diff --git a/.bumpversion.cfg b/.bumpversion.cfg index eb84b3c96b3f..bf67e4798de6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.44.12 +current_version = 0.50.21 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? @@ -12,14 +12,4 @@ serialize = [bumpversion:file:docs/operator-guides/upgrading-airbyte.md] -[bumpversion:file:octavia-cli/Dockerfile] - -[bumpversion:file:octavia-cli/README.md] - [bumpversion:file:run-ab-platform.sh] - -[bumpversion:file:octavia-cli/install.sh] - -[bumpversion:file:octavia-cli/setup.py] -search = version="{current_version}" -replace = version="{new_version}" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 55e90bfc08bf..bd3a2d1b2e00 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,5 +1,5 @@ # CDK and Connector Acceptance Tests -/airbyte-cdk/ @airbytehq/connector-extensibility +/airbyte-cdk/python @airbytehq/connector-extensibility /airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility /airbyte-integrations/bases/connector-acceptance-tests/ @airbytehq/connector-operations @@ -48,3 +48,4 @@ /airbyte-integrations/connectors/destination-rockset/ @airbytehq/destinations /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations +/airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations diff --git a/.github/actions/ci-tests-runner/action.yml b/.github/actions/ci-tests-runner/action.yml deleted file mode 100644 index cae685c81ed0..000000000000 --- a/.github/actions/ci-tests-runner/action.yml +++ /dev/null @@ -1,192 +0,0 @@ -name: "Setup CI Tests Env" -description: "Setup CI Tests Env for all module types" -inputs: - module-name: - description: "Unique module name. e.g.: connectors/source-s3, connectors/destination-s3" - required: true - module-folder: - description: "Path to module folder" - required: true - module-lang: - description: "Detected module language. Available values: py, java" - required: true - sonar-gcp-access-key: - required: true - sonar-token: - description: "Access token for using SonarQube API" - required: true - pull-request-id: - description: "Unique PR ID. For example: airbyte/1234" - default: "0" - token: - required: true - remove-sonar-project: - description: "This flag should be used if needed to remove sonar project after using" - default: false -runs: - using: "composite" - steps: - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - name: Tests of CI - shell: bash - run: | - # all CI python packages have the prefix "ci_" - pip install --quiet tox==3.24.4 - pip install --quiet -e ./tools/ci_* - tox -r -c ./tools/tox_ci.ini - echo "::echo::off" - - - name: Auth with gcloud CLI - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ inputs.sonar-gcp-access-key }} - project_id: dataline-integration-testing - export_default_credentials: true - - - name: Create IAP tunnel - id: gcloud-tunnel - shell: bash - run: | - while true; do - PORT=$(( ((RANDOM<<15)|RANDOM) % 49152 + 10000 )) - status="$(nc -z 127.0.0.1 $PORT < /dev/null &>/dev/null; echo $?)" - if [ "${status}" != "0" ]; then - echo "$PORT is free to use"; - break; - fi - done - IPS=($(hostname -I)) - LOCAL_IP_PORT="${IPS[0]}:${PORT}" - gcloud compute start-iap-tunnel sonarqube-1-vm 80 --local-host-port=${LOCAL_IP_PORT} --zone=europe-central2-a --project dataline-integration-testing & - echo pid=$! >> $GITHUB_OUTPUT - echo "sonar-host=http://${LOCAL_IP_PORT}/" >> $GITHUB_OUTPUT - echo "::echo::on" - - - name: Python Tests - id: ci-py-tests - if: ${{ inputs.module-lang == 'py' }} - uses: ./.github/actions/ci-py-tests - with: - module-name: ${{ inputs.module-name }} - module-folder: ${{ inputs.module-folder }} - - - name: Java Tests - id: ci-java-tests - if: ${{ inputs.module-lang == 'java' }} - uses: ./.github/actions/ci-java-tests - with: - module-name: ${{ inputs.module-name }} - module-folder: ${{ inputs.module-folder }} - - - name: Prepare SQ Options - shell: bash - id: sq-options - working-directory: ${{ inputs.module-folder }} - run: | - REPORT_FOLDER=reports - mkdir -p ${REPORT_FOLDER} - declare -a REPORT_FILES - declare -a OPTIONS - if [ ${{ inputs.module-lang }} == 'py' ]; then - [ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && ci_sonar_qube --mypy_log ${{ steps.ci-py-tests.outputs.mypy-logs }} --output_file ${REPORT_FOLDER}/issues_mypy.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} - [ -f ${{ steps.ci-py-tests.outputs.mypy-logs }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_mypy.json) - - [ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && ci_sonar_qube --black_diff ${{ steps.ci-py-tests.outputs.black-diff }} --output_file ${REPORT_FOLDER}/issues_black.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} - [ -f ${{ steps.ci-py-tests.outputs.black-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_black.json) - - [ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && ci_sonar_qube --isort_diff ${{ steps.ci-py-tests.outputs.isort-diff }} --output_file ${REPORT_FOLDER}/issues_isort.json --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} - [ -f ${{ steps.ci-py-tests.outputs.isort-diff }} ] && REPORT_FILES+=(${REPORT_FOLDER}/issues_isort.json) - - [ -f ${{ steps.ci-py-tests.outputs.coverage-paths }} ] && OPTIONS+=("-Dsonar.python.coverage.reportPaths=${{ steps.ci-py-tests.outputs.coverage-paths }}") - [ -f ${{ steps.ci-py-tests.outputs.flake8-logs }} ] && OPTIONS+=("-Dsonar.python.flake8.reportPaths=${{ steps.ci-py-tests.outputs.flake8-logs }}") - # TODO: figure out how to make this check work for Java-based connectors - # See more in https://github.com/airbytehq/airbyte/issues/10924 - cat ${REPORT_FOLDER}/* - fi - - if [ ${{ inputs.module-lang }} == 'java' ]; then - [ -d "./src/main/java" ] && OPTIONS+=("-Dsonar.sources=./src/main/java") - [ -d "./src/test/java" ] && OPTIONS+=("-Dsonar.tests=./src/test/java") - [ -d "./build/test-results" ] && OPTIONS+=("-Dsonar.junit.reportsPath=./build/test-results") - [ -f "./build/jacoco/test.exec" ] && OPTIONS+=("-Dsonar.jacoco.reportPaths=./build/jacoco/test.exec") - [ -d "./build/classes/java/main" ] && OPTIONS+=("-Dsonar.java.binaries=./build/classes/java/main") - [ -d "./build/classes/java/test" ] && OPTIONS+=("-Dsonar.test.binaries=./build/classes/java/test") - - fi - - # join the array to string format - echo external_reports=$(IFS=, ; echo "${REPORT_FILES[*]}") >> $GITHUB_OUTPUT - echo options=$(IFS=' ' ; echo "${OPTIONS[*]}") >> $GITHUB_OUTPUT - - - name: Create SonarQube Project - shell: bash - id: create-sq-project - run: | - ci_sonar_qube --pr ${{ inputs.pull-request-id }} --create --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} - echo "sq_project_name=$(ci_sonar_qube --pr ${{ inputs.pull-request-id }} --print_key --module ${{ inputs.module-name }})" >> $GITHUB_OUTPUT - ROOT_DIR=$(git rev-parse --show-toplevel) - MODULE_DIR=$(python -c "print('${{ inputs.module-folder }}'.replace('${ROOT_DIR}', '.'))") - echo "module_dir::${MODULE_DIR}" >> $GITHUB_OUTPUT - - - name: SonarQube Scan - - uses: sonarsource/sonarqube-scan-action@master - env: - SONAR_TOKEN: ${{ inputs.sonar-token }} - SONAR_HOST_URL: ${{ steps.gcloud-tunnel.outputs.sonar-host }} - with: - projectBaseDir: ${{ steps.create-sq-project.outputs.module_dir }} - args: > - -Dsonar.projectKey=${{ steps.create-sq-project.outputs.sq_project_name }} - -Dsonar.verbose=true - -Dsonar.working.directory=/tmp/scannerwork - -Dsonar.language=${{ inputs.module-lang }} - -Dsonar.sourceEncoding=UTF-8 - -Dsonar.projectBaseDir=${{ steps.create-sq-project.outputs.module_dir }} - -Dsonar.exclusions=reports/**,*.toml,*_tests/**,setup.py,main.py - -Dsonar.externalIssuesReportPaths=${{ steps.sq-options.outputs.external_reports }} - ${{ steps.sq-options.outputs.options }} - - - name: Generate SonarQube Report - shell: bash - id: generate-sq-report - run: | - # delay because SQ needs time for processing of all input data - sleep 10 - REPORT_FILE=/tmp/sq_report_$RANDOM.md - ci_sonar_qube --pr ${{ inputs.pull-request-id }} --report ${REPORT_FILE} --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} - body="$(cat ${REPORT_FILE})" - body="${body//'%'/'%25'}" - body="${body//$'\n'/'%0A'}" - body="${body//$'\r'/'%0D'}" - echo "sq-report=$body" >> $GITHUB_OUTPUT - - - name: Add Comment - if: ${{ github.event_name == 'pull_request' }} - uses: peter-evans/commit-comment@v1 - with: - body: ${{ steps.generate-sq-report.outputs.sq-report }} - token: ${{ inputs.token }} - - - name: Remove SonarQube Project - if: ${{ inputs.remove-sonar-project == true }} - shell: bash - id: remove-sq-project - run: | - ci_sonar_qube --pr ${{ inputs.pull-request-id }} --remove --module ${{ inputs.module-name }} --host ${{ steps.gcloud-tunnel.outputs.sonar-host }} --token ${{ inputs.sonar-token }} - - - name: Remove IAP tunnel - if: always() - shell: bash - run: | - kill ${{ steps.gcloud-tunnel.outputs.pid }} diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 6c5d27148dd8..8d413a9c658f 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -4,32 +4,85 @@ inputs: subcommand: description: "Subcommand for airbyte-ci" required: true - options: - description: "Options for the subcommand" - required: false context: description: "CI context (e.g., pull_request, manual)" required: true github_token: description: "GitHub token" required: true + docker_hub_username: + description: "Dockerhub username" + required: true + docker_hub_password: + description: "Dockerhub password" + required: true + options: + description: "Options for the subcommand" + required: false + production: + description: "Whether to run in production mode" + required: false + default: "True" + report_bucket_name: + description: "Bucket name for CI reports" + required: false + default: "airbyte-ci-reports-multi" + gcp_gsm_credentials: + description: "GCP credentials for GCP Secret Manager" + required: false + default: "" + git_branch: + description: "Git branch to checkout" + required: false + git_revision: + description: "Git revision to checkout" + required: false + slack_webhook_url: + description: "Slack webhook URL" + required: false + metadata_service_gcs_credentials: + description: "GCP credentials for metadata service" + required: false + metadata_service_bucket_name: + description: "Bucket name for metadata service" + required: false + default: "prod-airbyte-cloud-connector-metadata-service" + sentry_dsn: + description: "Sentry DSN" + required: false + spec_cache_bucket_name: + description: "Bucket name for GCS spec cache" + required: false + default: "io-airbyte-cloud-spec-cache" + spec_cache_gcs_credentials: + description: "GCP credentials for GCS spec cache" + required: false + gcs_credentials: + description: "GCP credentials for GCS" + required: false + ci_job_key: + description: "CI job key" + required: false runs: using: "composite" steps: - - name: Get start timestamp - id: get-start-timestamp - run: echo "::set-output name=start-timestamp::$(date +%s)" + - name: Check if PR is from a fork + if: github.event_name == 'pull_request' shell: bash - - name: Checkout Airbyte - uses: actions/checkout@v3 + run: | + if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then + echo "PR is from a fork. Exiting workflow..." + exit 78 + fi + - name: Docker login + uses: docker/login-action@v1 with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Extract branch name + username: ${{ inputs.docker_hub_username }} + password: ${{ inputs.docker_hub_password }} + - name: Get start timestamp + id: get-start-timestamp shell: bash - if: github.event_name == 'workflow_dispatch' - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch + run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT - name: Install Python 3.10 uses: actions/setup-python@v4 with: @@ -37,22 +90,34 @@ runs: token: ${{ inputs.github_token }} - name: Install ci-connector-ops package shell: bash - run: pip install --quiet -e ./tools/ci_connector_ops\[pipelines]\ + run: | + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/pipelines/ - name: Run airbyte-ci shell: bash run: | export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - DAGGER_CLI_COMMIT="2370d5eb65e511cf7f1611aebaacc6d48cf0907a" - DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" - export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" - if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then - mkdir -p "$DAGGER_TMP_BINDIR" - curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" - fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} - + airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} env: - CI_GIT_BRANCH: ${{ github.head_ref }} - CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }} + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" CI_CONTEXT: "${{ inputs.context }}" + CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }} + CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }} + CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }} + CI_JOB_KEY: ${{ inputs.ci_job_key }} CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }} + GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }} + GCS_CREDENTIALS: ${{ inputs.gcs_credentials }} + METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }} + METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }} + PRODUCTION: ${{ inputs.production }} + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + SENTRY_DSN: ${{ inputs.sentry_dsn }} + SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }} + SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }} + SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }} + DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }} + DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }} + CI: "True" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9d5fbde22cf7..476fd613df7e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,17 @@ + + ## What *Describe what the change is solving* *It helps to add screenshots if it affects the frontend.* diff --git a/.github/teams.yml b/.github/teams.yml deleted file mode 100644 index 34c12bfffa59..000000000000 --- a/.github/teams.yml +++ /dev/null @@ -1,3 +0,0 @@ -team/growth: - - "@letiescanciano" - - "@arnaudjnn" diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml new file mode 100644 index 000000000000..c5e5e6edb12b --- /dev/null +++ b/.github/workflows/airbyte-ci-tests.yml @@ -0,0 +1,34 @@ +name: Airbyte CI pipeline tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - airbyte-ci/** +jobs: + run-airbyte-ci-tests: + name: Run Airbyte CI tests + runs-on: "conn-prod-xlarge-runner" + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Run airbyte-ci/connectors/pipelines tests + id: run-airbyte-ci-connectors-pipelines-tests + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "test airbyte-ci/connectors/pipelines" diff --git a/.github/workflows/build-report.yml b/.github/workflows/build-report.yml deleted file mode 100644 index f9a0fe286fae..000000000000 --- a/.github/workflows/build-report.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Generate Build Report - -# Uses Python to Generate a build report -# and send it to Slack - -on: - workflow_dispatch: - schedule: - # 6AM UTC is 8AM EET, 7AM CET, 11PM PST. - - cron: "0 6 * * *" - -jobs: - build-report: - name: Build Report - timeout-minutes: 5 - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install requests slack_sdk pyyaml - - name: create and send report - run: python ./tools/bin/build_report.py - env: - SLACK_BUILD_REPORT: ${{ secrets.SLACK_BUILD_REPORT }} - - name: Slack Notification - Failure - if: failure() - uses: rtCamp/action-slack-notify@master - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_BUILD_REPORT }} - SLACK_USERNAME: Build Report - SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg - SLACK_COLOR: ${{ job.status }} - SLACK_TITLE: "Failed to create build report" - SLACK_MESSAGE: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - MSG_MINIMAL: True diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml new file mode 100644 index 000000000000..78b7ac86c3c4 --- /dev/null +++ b/.github/workflows/cat-tests.yml @@ -0,0 +1,34 @@ +name: CAT unit tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + pull_request: + types: + - opened + - reopened + - synchronize + paths: + - airbyte-integrations/bases/connector-acceptance-test/** +jobs: + run-cat-unit-tests: + name: Run CAT unit tests + runs-on: "conn-prod-xlarge-runner" + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Run CAT unit tests + id: run-cat-unit-tests + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "test airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests" diff --git a/.github/workflows/commands-for-testing-tool.yml b/.github/workflows/commands-for-testing-tool.yml deleted file mode 100644 index a6304c38ed47..000000000000 --- a/.github/workflows/commands-for-testing-tool.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: Run Testing Tool Commands -on: - issue_comment: - types: [created] -jobs: - set-params: - # Only allow slash commands on pull request (not on issues) - if: ${{ github.event.issue.pull_request }} - runs-on: ubuntu-latest - outputs: - repo: ${{ steps.getref.outputs.repo }} - ref: ${{ steps.getref.outputs.ref }} - comment-id: ${{ steps.comment-info.outputs.comment-id }} - command: ${{ steps.regex.outputs.first_match }} - steps: - - name: Get PR repo and ref - id: getref - run: | - pr_info="$(curl ${{ github.event.issue.pull_request.url }})" - echo ref="$(echo $pr_info | jq -r '.head.ref')" >> $GITHUB_OUTPUT - echo repo="$(echo $pr_info | jq -r '.head.repo.full_name')" >> $GITHUB_OUTPUT - - name: Get comment id - id: comment-info - run: | - echo comment-id="${{ github.event.comment.id }}" >> $GITHUB_OUTPUT - - name: Get command - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^/[a-zA-Z0-9_/-]+" - regex_flags: "i" - search_string: ${{ github.event.comment.body }} - helps-run: - runs-on: ubuntu-latest - if: | - needs.set-params.outputs.command == '/help-full' || - needs.set-params.outputs.command == '/help' || - needs.set-params.outputs.command == '/list-scenarios' - needs: set-params - steps: - - name: Update comment for processing - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ needs.set-params.outputs.comment-id }} - reactions: eyes, rocket - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Pull Testing Tool docker image - run: ./tools/bin/pull_image.sh -i airbyte/airbyte-e2e-testing-tool:latest - - name: Create input and output folders - run: | - mkdir secrets - mkdir result - - name: Run docker container with params - run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.comment.body }} - - name: Read file with results - id: read_file - uses: andstor/file-reader-action@v1 - with: - path: "result/log" - - name: Add Success Comment - if: needs.set-params.outputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ needs.set-params.outputs.comment-id }} - body: | - > :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - ${{ steps.read_file.outputs.contents }} - reactions: +1 - - name: Add Failure Comment - if: needs.set-params.outputs.comment-id && failure() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ needs.set-params.outputs.comment-id }} - body: | - > :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - reactions: -1 - scenarios-run: - runs-on: ubuntu-latest - if: | - needs.set-params.outputs.command == '/run-scenario' || - needs.set-params.outputs.command == '/run-scenario-local' - needs: set-params - steps: - - name: Update comment for processing - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ needs.set-params.outputs.comment-id }} - reactions: eyes, rocket - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ needs.set-params.outputs.repo }} - ref: ${{ needs.set-params.outputs.ref }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Pull Testing Tool docker image - run: ./tools/bin/pull_image.sh -i airbyte/airbyte-e2e-testing-tool:latest - - name: Change wrapper permissions - run: | - mkdir secrets - mkdir result - - name: Run Airbyte - run: docker compose up -d - - name: Connect to secret manager - uses: jsdaniell/create-json@1.1.2 - with: - name: "/secrets/service_account_credentials.json" - json: ${{ secrets.GCP_GSM_CREDENTIALS_FOR_TESTING_TOOL }} - - name: Run docker container with params - run: docker run -v $(pwd)/secrets:/secrets -v $(pwd)/result:/result airbyte/airbyte-e2e-testing-tool:latest ${{ github.event.comment.body }} - - name: Read file with results - id: read_file - uses: andstor/file-reader-action@v1 - with: - path: "result/log" - - name: Add Success Comment - if: needs.set-params.outputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ needs.set-params.outputs.comment-id }} - body: | - > :white_check_mark: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - ${{ steps.read_file.outputs.contents }} - reactions: +1 - - name: Add Failure Comment - if: needs.set-params.outputs.comment-id && failure() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ needs.set-params.outputs.comment-id }} - body: | - > :x: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - reactions: -1 diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index 8be33cae02a9..17216d62f12b 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -76,9 +76,9 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - #### Note: The following `dataset=` values are supported: `1m`(default), `10m`, `20m`, - `bottleneck_stream1`, `bottleneck_stream_randomseed. For destinations only: you can also use `stream-numbers=N` - to simulate N number of parallel streams. Additionally, `sync-mode=incremental` is supported for destinations. + #### Note: The following `dataset=` values are supported: `1m`(default), `10m`, `20m`, + `bottleneck_stream1`, `bottleneck_stream_randomseed. For destinations only: you can also use `stream-numbers=N` + to simulate N number of parallel streams. Additionally, `sync-mode=incremental` is supported for destinations. For example: `dataset=1m stream-numbers=2 sync-mode=incremental` > :runner: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}. - name: Search for valid connector name format @@ -92,7 +92,7 @@ jobs: if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && + if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && github.event.inputs.connector != 'connectors/source-mysql' && github.event.inputs.connector != 'connectors/destination-snowflake' }}" run: echo "Only connectors/source-postgres, source-mysql and destination-snowflake currently supported by harness" && exit 1 @@ -109,11 +109,13 @@ jobs: - name: Install Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Install CI scripts - # all CI python packages have the prefix "ci_" run: | - pip install --quiet -e ./tools/ci_* + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/ci_credentials + pipx install airbyte-ci/connectors/connector_ops - name: Source or Destination harness id: which-harness run: | @@ -139,7 +141,7 @@ jobs: echo "Building... ${{github.event.inputs.connector}}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2) - echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" + echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check - name: KIND Kubernetes Cluster Setup uses: helm/kind-action@v1.4.0 @@ -174,7 +176,7 @@ jobs: kubectl logs --follow $POD EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) echo "RUN_RESULT<<$EOF" >> $GITHUB_OUTPUT - kubectl logs --tail=1 $POD | while read line ; do line=${line#"$PREFIX"}; line=${line%"$SUFFIX"}; echo $line >> $GITHUB_OUTPUT ; done + kubectl logs --tail=1 $POD | while read line ; do line=${line#"$PREFIX"}; line=${line%"$SUFFIX"}; echo $line >> $GITHUB_OUTPUT ; done echo "$EOF" >> $GITHUB_OUTPUT - name: Link comment to workflow run uses: peter-evans/create-or-update-comment@v2 @@ -182,7 +184,7 @@ jobs: reactions: '+1' comment-id: ${{ github.event.inputs.comment-id }} body: | - ## Performance test Result: + ## Performance test Result: ``` ${{ steps.run-harness.outputs.RUN_RESULT }} ``` diff --git a/.github/workflows/connector_integration_test_single_dagger.yml b/.github/workflows/connector_integration_test_single_dagger.yml deleted file mode 100644 index 8131c866b645..000000000000 --- a/.github/workflows/connector_integration_test_single_dagger.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: POC Connectors CI - test pipeline - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - workflow_dispatch: - inputs: - test-connectors-options: - description: "Options to pass to the 'airbyte-ci connectors test' command" - default: "--modified" - pull_request: - paths: - - "airbyte-integrations/connectors/**" - #- ".github/workflows/connector_integration_test_single_dagger.yml" - types: - - opened - - synchronize - - ready_for_review -jobs: - connectors_ci: - name: Connectors CI - timeout-minutes: 240 # 4 hours - runs-on: dev-medium-runner - steps: - - name: Get start timestamp - id: get-start-timestamp - run: echo "::set-output name=start-timestamp::$(date +%s)" - - name: Login to DockerHub - run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Extract branch name - shell: bash - if: github.event_name == 'workflow_dispatch' - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - name: Install Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - - name: Install ci-connector-ops package - run: pip install -e ./tools/ci_connector_ops\[pipelines]\ - - name: Run airbyte-ci connectors test [WORKFLOW DISPATCH] - if: github.event_name == 'workflow_dispatch' - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" - DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" - export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" - if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then - mkdir -p "$DAGGER_TMP_BINDIR" - curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" - fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ github.event.inputs.test-connectors-options }} test - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - CI_GIT_REVISION: ${{ github.sha }} - CI_CONTEXT: "manual" - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - - name: Run airbyte-ci connectors test [PULL REQUESTS] - if: github.event_name == 'pull_request' - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" - DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" - export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" - if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then - mkdir -p "$DAGGER_TMP_BINDIR" - curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" - fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --modified test - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - CI_GIT_BRANCH: ${{ github.head_ref }} - CI_GIT_REVISION: ${{ github.event.pull_request.head.sha }} - CI_CONTEXT: "pull_request" - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/connector_integration_tests.yml b/.github/workflows/connector_integration_tests.yml deleted file mode 100644 index 7f18f5089143..000000000000 --- a/.github/workflows/connector_integration_tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Connector Integration Tests - -# Launches the connector integration tests - -on: - workflow_dispatch: - schedule: - # 3AM UTC is 5AM EET, 4AM CET, 8PM PST. - - cron: "0 3 * * *" - -jobs: - launch_integration_tests: - timeout-minutes: 30 - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyYAML requests - - name: Launch Integration Tests - run: python ./tools/bin/ci_integration_workflow_launcher.py base-normalization connector-acceptance-test source:beta source:GA destination:beta destination:GA - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} diff --git a/.github/workflows/connector_integration_tests_alpha.yml b/.github/workflows/connector_integration_tests_alpha.yml deleted file mode 100644 index f58df2671447..000000000000 --- a/.github/workflows/connector_integration_tests_alpha.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Connector Integration Tests (Alpha connectors only) - -# Launches the connector integration tests - -on: - workflow_dispatch: - schedule: - # 3AM UTC is 5AM EET, 4AM CET, 8PM PST. - - cron: "0 3 * * 0" - -jobs: - launch_integration_tests_alpha_only: - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install PyYAML requests - - name: Launch Integration Tests (Alpha connectors) - run: python ./tools/bin/ci_integration_workflow_launcher.py source:alpha destination:alpha - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} diff --git a/.github/workflows/connector_metadata_checks.yml b/.github/workflows/connector_metadata_checks.yml index f9096baa6766..2af441a2bbe9 100644 --- a/.github/workflows/connector_metadata_checks.yml +++ b/.github/workflows/connector_metadata_checks.yml @@ -16,9 +16,12 @@ jobs: - name: Install Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Install ci-connector-ops package - run: pip install --quiet -e ./tools/ci_connector_ops + run: | + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/connector_ops/ - name: Check test strictness level run: check-test-strictness-level - name: Check allowed hosts diff --git a/.github/workflows/connector_teams_review_requirements.yml b/.github/workflows/connector_teams_review_requirements.yml index e4755ae54a50..f4765eef488e 100644 --- a/.github/workflows/connector_teams_review_requirements.yml +++ b/.github/workflows/connector_teams_review_requirements.yml @@ -21,9 +21,12 @@ jobs: - name: Install Python uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Install ci-connector-ops package - run: pip install --quiet -e ./tools/ci_connector_ops + run: | + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/connector_ops - name: Write review requirements file id: write-review-requirements-file run: write-review-requirements-file >> $GITHUB_OUTPUT diff --git a/.github/workflows/connectors_manual_tests.yml b/.github/workflows/connectors_manual_tests.yml deleted file mode 100644 index 791398bd5650..000000000000 --- a/.github/workflows/connectors_manual_tests.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Connectors CI - Manual tests - -on: - workflow_dispatch: - inputs: - runs-on: - type: string - default: dev-large-runner - required: true - test-connectors-options: - default: --concurrency=10 --release-stage=generally_available --release-stage=beta - required: true - -run-name: "Test connectors ${{ inputs.test-connectors-options}} on ${{ inputs.runs-on }}" - -jobs: - test_connectors: - name: "Test connectors ${{ inputs.test-connectors-options}} on ${{ inputs.runs-on }}" - timeout-minutes: 600 # 10 hours - runs-on: ${{ inputs.runs-on || 'dev-large-runner' }} - steps: - - name: Get start timestamp - id: get-start-timestamp - run: echo "::set-output name=start-timestamp::$(date +%s)" - - name: Login to DockerHub - run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Extract branch name - shell: bash - if: github.event_name == 'workflow_dispatch' - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - name: Install Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - - name: Install ci-connector-ops package - run: pip install ./tools/ci_connector_ops\[pipelines]\ - - name: "Test connectors ${{ inputs.test-connectors-options}}" - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" - DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" - export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" - if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then - mkdir -p "$DAGGER_TMP_BINDIR" - curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" - fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options }} test - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - CI_CONTEXT: "manual" - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml new file mode 100644 index 000000000000..acd7d024e3bb --- /dev/null +++ b/.github/workflows/connectors_nightly_build.yml @@ -0,0 +1,44 @@ +name: Connectors nightly build + +on: + schedule: + # 0AM UTC is 2AM CEST, 3AM EEST, 5PM PDT. + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + runs-on: + type: string + default: conn-nightly-xlarge-runner + required: true + test-connectors-options: + default: --concurrency=5 --support-level=certified + required: true + +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + +jobs: + test_connectors: + name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + timeout-minutes: 720 # 12 hours + runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Test connectors + uses: ./.github/actions/run-dagger-pipeline + with: + context: "nightly_builds" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + git_branch: ${{ steps.extract_branch.outputs.branch }} + github_token: ${{ secrets.GITHUB_TOKEN }} + subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --support-level=certified' }} test" diff --git a/.github/workflows/connectors_nightly_builds.yml b/.github/workflows/connectors_nightly_builds.yml deleted file mode 100644 index 91e99b913d04..000000000000 --- a/.github/workflows/connectors_nightly_builds.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Connectors CI - nightly builds for GA and Beta connectors - -on: - schedule: - # 11AM UTC is 12AM CET, 4AM PST. - - cron: "0 11 * * *" - workflow_dispatch: - inputs: - runs-on: - type: string - default: dev-large-runner - required: true - -run-name: "Connectors CI - nightly builds for GA and Beta connectors - ${{ inputs.runs-on }}" - -jobs: - test_connectors: - name: Test GA and Beta connectors - timeout-minutes: 600 # 10 hours - runs-on: ${{ inputs.runs-on || 'dev-large-runner' }} - steps: - - name: Get start timestamp - id: get-start-timestamp - run: echo "::set-output name=start-timestamp::$(date +%s)" - - name: Login to DockerHub - run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Extract branch name - shell: bash - if: github.event_name == 'workflow_dispatch' - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - name: Install Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - - name: Install ci-connector-ops package - run: pip install ./tools/ci_connector_ops\[pipelines]\ - - name: Test Python connectors - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" - DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" - export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" - if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then - mkdir -p "$DAGGER_TMP_BINDIR" - curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" - fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=8 --release-stage=generally_available --release-stage=beta --language=python --language=low-code test - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - CI_CONTEXT: "nightly_builds" - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - - name: Test Java connectors - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" - DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" - export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" - if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then - mkdir -p "$DAGGER_TMP_BINDIR" - curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" - fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=1 --release-stage=generally_available --release-stage=beta --language=java test - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - CI_CONTEXT: "nightly_builds" - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml new file mode 100644 index 000000000000..56820c787cc8 --- /dev/null +++ b/.github/workflows/connectors_tests.yml @@ -0,0 +1,84 @@ +name: Connectors tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +on: + workflow_dispatch: + inputs: + test-connectors-options: + description: "Options to pass to the 'airbyte-ci connectors test' command" + default: "--modified" + runner: + description: "The runner to use for this job" + default: "conn-prod-xlarge-runner" + pull_request: + types: + - opened + - synchronize + - ready_for_review +jobs: + connectors_ci: + name: Connectors CI + timeout-minutes: 1440 # 24 hours + runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Extract branch name [WORKFLOW DISPATCH] + shell: bash + if: github.event_name == 'workflow_dispatch' + run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + id: extract_branch + # - name: Format connectors [PULL REQUESTS] + # if: github.event_name == 'pull_request' + # uses: ./.github/actions/run-dagger-pipeline + # with: + # context: "pull_request" + # docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + # docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + # gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + # git_branch: ${{ github.head_ref }} + # git_revision: ${{ github.sha }} + # github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + # subcommand: "connectors --modified format" + - name: Fetch last commit id from remote branch [PULL REQUESTS] + if: github.event_name == 'pull_request' + id: fetch_last_commit_id_pr + run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT + - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' + id: fetch_last_commit_id_wd + run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT + - name: Pull formatting changes [PULL REQUESTS] + if: github.event_name == 'pull_request' + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + - name: Test connectors [WORKFLOW DISPATCH] + if: github.event_name == 'workflow_dispatch' + uses: ./.github/actions/run-dagger-pipeline + with: + context: "manual" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + git_branch: ${{ steps.extract_branch.outputs.branch }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" + - name: Test connectors [PULL REQUESTS] + if: github.event_name == 'pull_request' + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + git_branch: ${{ github.head_ref }} + git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + subcommand: "connectors --modified test" diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml new file mode 100644 index 000000000000..63ee82dfe646 --- /dev/null +++ b/.github/workflows/connectors_weekly_build.yml @@ -0,0 +1,44 @@ +name: Connectors weekly build + +on: + schedule: + # 12PM UTC on Sunday is 2PM CEST, 3PM EEST, 5 PDT. + - cron: "0 12 * * 0" + workflow_dispatch: + inputs: + runs-on: + type: string + default: conn-nightly-xlarge-runner + required: true + test-connectors-options: + default: --concurrency=3 --support-level=community + required: true + +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + +jobs: + test_connectors: + name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + timeout-minutes: 8640 # 6 days + runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Test connectors + uses: ./.github/actions/run-dagger-pipeline + with: + context: "nightly_builds" + ci_job_key: "weekly_alpha_test" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + git_branch: ${{ steps.extract_branch.outputs.branch }} + github_token: ${{ secrets.GITHUB_TOKEN }} + subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=3 --support-level=community' }} test" diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index f4222cca7cc0..aead5f00f06e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -39,18 +39,22 @@ jobs: runs-on: ubuntu-latest outputs: build: ${{ steps.filter.outputs.build }} - cdk: ${{ steps.filter.outputs.cdk }} + java_cdk: ${{ steps.filter.outputs.java_cdk }} + python_cdk: ${{ steps.filter.outputs.python_cdk }} cli: ${{ steps.filter.outputs.cli }} - connectors: ${{ steps.filter.outputs.connectors }} + connectors_base: ${{ steps.filter.outputs.connectors_base }} db: ${{ steps.filter.outputs.db }} - frontend: ${{ steps.filter.outputs.frontend }} + any_change: ${{ steps.filter.outputs.any_change }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 - uses: dorny/paths-filter@v2 id: filter with: - # Note, the following glob expression within a filters are ORs. + # Note: The following glob expression within a filters are ORs. + # Note: If no filters match, the steps are all skipped WITHOUT reported their status check back to github. + # This can cause required checks to go unreported blocking PRs from being merged + # and this is why we have the any_change filter. filters: | build: - '.github/**' @@ -59,17 +63,21 @@ jobs: - '*.gradle' - 'deps.toml' - 'airbyte-config-oss/**' - cdk: - - 'airbyte-cdk/**' + python_cdk: + - 'airbyte-cdk/python/**' + java_cdk: + - 'airbyte-cdk/java/**' cli: - 'airbyte-api/**' - 'octavia-cli/**' - connectors: + connectors_base: - 'airbyte-integrations/bases/**' - 'airbyte-integrations/connectors-templates/**' - 'airbyte-connector-test-harnesses/acceptance-test-harness/**' db: - 'airbyte-db/**' + any_change: + - '**/*' # Uncomment to debug. # changes-output: @@ -81,14 +89,13 @@ jobs: # - run: | # echo '${{ toJSON(needs) }}' - ## BUILDS - octavia-cli-build: + format: needs: changes runs-on: ubuntu-latest # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. - if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') - name: "Octavia CLI: Build" - timeout-minutes: 90 + if: needs.changes.outputs.any_change == 'true' || (always() && github.ref == 'refs/heads/master') + name: "Apply All Formatting Rules" + timeout-minutes: 20 steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -98,8 +105,7 @@ jobs: - name: Cache Build Artifacts uses: ./.github/actions/cache-build-artifacts with: - cache-key: ${{ secrets.CACHE_VERSION }} - cache-python: "false" + cache-key: ${{ secrets.CACHE_VERSION }}-format - uses: actions/setup-java@v3 with: @@ -109,7 +115,6 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.9" - token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Set up CI Gradle Properties run: | @@ -126,44 +131,107 @@ jobs: EOF - name: Format - uses: Wandalen/wretry.action@master + uses: Wandalen/wretry.action@v1.0.42 with: - command: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace + command: ./gradlew format --scan --info --stacktrace attempt_limit: 3 attempt_delay: 5000 # in ms - - name: Ensure no file change - run: ./tools/bin/check_for_file_changes + # This is helpful in the case that we change a previously committed generated file to be ignored by git. + - name: Remove any files that have been gitignored + run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached + + - name: Commit Formatting Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Automated Commit - Formatting Changes + commit_user_name: Octavia Squidington III + commit_user_email: octavia-squidington-iii@users.noreply.github.com + + ## BUILDS + octavia-cli-build: + needs: + - changes + - format + runs-on: ubuntu-latest + # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. + if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master') + name: "Octavia CLI: Build" + timeout-minutes: 90 + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + # - name: Cache Build Artifacts + # uses: ./.github/actions/cache-build-artifacts + # with: + # cache-key: ${{ secrets.CACHE_VERSION }} + # cache-python: "false" + + - uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "17" + + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + + - name: Set up CI Gradle Properties + run: | + mkdir -p ~/.gradle/ + cat > ~/.gradle/gradle.properties < ~/.gradle/gradle.properties <> $GITHUB_ENV - name: Prepare Sentry if: startsWith(matrix.connector, 'connectors') - uses: Wandalen/wretry.action@master + uses: Wandalen/wretry.action@v1.0.42 with: attempt_limit: 3 attempt_delay: 5000 # in ms @@ -299,7 +299,7 @@ jobs: DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. TZ: UTC - uses: Wandalen/wretry.action@master + uses: Wandalen/wretry.action@v1.0.42 with: command: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} diff --git a/.github/workflows/legacy-test-command.yml b/.github/workflows/legacy-test-command.yml new file mode 100644 index 000000000000..6d681585f434 --- /dev/null +++ b/.github/workflows/legacy-test-command.yml @@ -0,0 +1,229 @@ +name: Run Integration Test +on: + workflow_dispatch: + inputs: + connector: + description: "Airbyte Connector" + required: true + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + uuid: + description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id." + required: false + connector-acceptance-test-version: + description: "Set a specific connector acceptance test version to use. Enter 'dev' to test, build and use a local version of Connector Acceptance Test." + required: false + default: "latest" + local_cdk: + description: "Run Connector Acceptance Tests against the CDK version on the current branch." + required: false +jobs: + uuid: + name: "Custom UUID of workflow run" + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: UUID ${{ github.event.inputs.uuid }} + run: true + start-test-runner: + name: Start Build EC2 Runner + needs: uuid + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + pipeline-start-timestamp: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + steps: + - name: Get start timestamp + id: get-start-timestamp + run: echo "::set-output name=start-timestamp::$(date +%s)" + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ env.PAT }} + integration-test: + timeout-minutes: 240 + needs: start-test-runner + runs-on: ${{ needs.start-test-runner.outputs.label }} + steps: + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ github.event.inputs.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != github.event.inputs.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Install Java + uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "17" + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install CI scripts + # all CI python packages have the prefix "ci_" + run: | + pip install --quiet -e ./airbyte-ci/connectors/ci_credentials + pip install --quiet -e ./airbyte-ci/connectors/connector_ops + - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + run: | + ci_credentials ${{ github.event.inputs.connector }} write-to-storage + # normalization also runs destination-specific tests, so fetch their creds also + if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then + ci_credentials destination-bigquery write-to-storage + ci_credentials destination-postgres write-to-storage + ci_credentials destination-snowflake write-to-storage + fi + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + - name: Build Java CDK Snapshot if Needed + # If a snapshot version is specified for the Java CDK, build publish locally. Otherwise, do nothing. + run: ./gradlew :airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded + - name: Test ${{ github.event.inputs.connector }} + id: test + env: + ACTION_RUN_ID: ${{github.run_id}} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + ORG_GRADLE_PROJECT_connectorAcceptanceTestVersion: ${{github.event.inputs.connector-acceptance-test-version}} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + uses: Wandalen/wretry.action@v1.0.42 + with: + command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} ${{ github.event.inputs.local_cdk }} + attempt_limit: 3 + attempt_delay: 10000 # in ms + - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} + if: always() + run: | + ci_credentials ${{ github.event.inputs.connector }} update-secrets + # normalization also runs destination-specific tests, so fetch their creds also + if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then + ci_credentials destination-bigquery update-secrets + ci_credentials destination-postgres update-secrets + ci_credentials destination-snowflake update-secrets + fi + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + - name: Archive test reports artifacts + if: github.event.inputs.comment-id && failure() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: | + **/${{ github.event.inputs.connector }}/build/reports/tests/**/** + **/${{ github.event.inputs.connector }}/acceptance_tests_logs/** + **/normalization_test_output/**/dbt_output.log + **/normalization_test_output/**/destination_output.log + **/normalization_test_output/**/build/compiled/airbyte_utils/** + **/normalization_test_output/**/build/run/airbyte_utils/** + **/normalization_test_output/**/models/generated/** + - name: Test coverage reports artifacts + if: github.event.inputs.comment-id && success() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: | + **/${{ github.event.inputs.connector }}/htmlcov/** + retention-days: 3 + - name: Run QA checks for ${{ github.event.inputs.connector }} + id: qa_checks + if: always() + run: | + run-qa-checks ${{ github.event.inputs.connector }} + - name: Report Observability + if: always() + run: ./tools/status/report_observability.sh ${{ github.event.inputs.connector }} ${{github.run_id}} ${{ needs.start-test-runner.outputs.pipeline-start-timestamp }} ${{ github.event.inputs.gitref }} ${{ github.sha }} ${{steps.test.outcome}} ${{steps.qa_checks.outcome}} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "us-east-2" + - name: Add Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + ${{env.PYTHON_UNITTEST_COVERAGE_REPORT}} + > ${{env.TEST_SUMMARY_INFO}} + - name: Add Failure Comment + if: github.event.inputs.comment-id && failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :bug: ${{env.GRADLE_SCAN_LINK}} + > ${{env.TEST_SUMMARY_INFO}} + # In case of self-hosted EC2 errors, remove this block. + stop-test-runner: + name: Stop Build EC2 Runner + timeout-minutes: 10 + needs: + - start-test-runner # required to get output from the start-runner job + - integration-test # required to wait when the main job is done + - uuid + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ env.PAT }} + label: ${{ needs.start-test-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml index b4a7dc13b846..2d36306392fc 100644 --- a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml +++ b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml @@ -19,6 +19,10 @@ jobs: uses: ./.github/actions/run-dagger-pipeline with: subcommand: "metadata deploy orchestrator" + context: "master" + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} env: DAGSTER_CLOUD_METADATA_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_METADATA_API_TOKEN }} - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} diff --git a/.github/workflows/metadata_service_tests_dagger.yml b/.github/workflows/metadata_service_tests_dagger.yml index dd296e5dd7db..83117161d1b4 100644 --- a/.github/workflows/metadata_service_tests_dagger.yml +++ b/.github/workflows/metadata_service_tests_dagger.yml @@ -10,7 +10,7 @@ jobs: name: Connector metadata service CI runs-on: medium-runner env: - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -20,9 +20,17 @@ jobs: with: subcommand: "metadata test lib" context: "pull_request" + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Run test for the metadata orchestrator id: metadata-orchestrator-test-pipeline uses: ./.github/actions/run-dagger-pipeline with: subcommand: "metadata test orchestrator" context: "pull_request" + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} diff --git a/.github/workflows/metadata_validate_manifest_dagger.yml b/.github/workflows/metadata_validate_manifest_dagger.yml deleted file mode 100644 index ecfa73f520ef..000000000000 --- a/.github/workflows/metadata_validate_manifest_dagger.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Connectors' metadata.yaml files validation - -on: - workflow_dispatch: - pull_request: - -jobs: - connectors_metadata_validation: - name: Connectors' metadata.yaml files validation - timeout-minutes: 10 # 10 minutes - runs-on: medium-runner - env: - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - PRODUCTION: "true" - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Connectors' metadata.yaml files validation - id: metadata-validate-pipeline - uses: ./.github/actions/run-dagger-pipeline - with: - subcommand: "metadata validate" - context: "pull_request" - github-token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} diff --git a/.github/workflows/publish-cdk-command-manually.yml b/.github/workflows/publish-cdk-command-manually.yml index 51370b77b529..d59f0fad16b8 100644 --- a/.github/workflows/publish-cdk-command-manually.yml +++ b/.github/workflows/publish-cdk-command-manually.yml @@ -240,7 +240,7 @@ jobs: DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. TZ: UTC - uses: Wandalen/wretry.action@master + uses: Wandalen/wretry.action@v1.0.42 with: command: | docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} @@ -293,10 +293,11 @@ jobs: - name: Update CDK version run: | PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) - sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-atelier-server/Dockerfile - sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-atelier-server/requirements.in + sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile + sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" airbyte-connector-builder-server-wrapped/Dockerfile + sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION - cd oss/airbyte-connector-atelier-server + cd oss/airbyte-connector-builder-server pip install pip-tools pip-compile --upgrade - name: Create Pull Request @@ -307,7 +308,7 @@ jobs: commit-message: Updating CDK version following release title: Updating CDK version following release body: This is an automatically generated PR triggered by a CDK release - branch: automatic-cdk-release-${{needs.bump-version.outputs.new_cdk_version}} + branch: automatic-cdk-release base: master delete-branch: true - name: Post success to Slack channel dev-connectors-extensibility diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index e5202ae11fdb..566fea7f1024 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -9,52 +9,52 @@ on: workflow_dispatch: inputs: connectors-options: - description: "Options to pass to the 'airbyte-ci connectors' command group" + description: "Options to pass to the 'airbyte-ci connectors' command group." default: "--name=source-pokeapi" publish-options: - description: "Options to pass to the 'airbyte-ci connectors publish' command" + description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " default: "--pre-release" + runs-on: + type: string + default: conn-prod-xlarge-runner + required: true jobs: publish_connectors: name: Publish connectors - runs-on: large-runner - env: - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_DEFAULT_REGION: "us-east-2" - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - METADATA_SERVICE_GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - METADATA_SERVICE_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service - SPEC_CACHE_BUCKET_NAME: io-airbyte-cloud-spec-cache - SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} - TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" - SLACK_WEBHOOK: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }} steps: - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Login to DockerHub - run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" + uses: actions/checkout@v3 - name: Publish modified connectors [On merge to master] id: publish-modified-connectors if: github.event_name == 'push' uses: ./.github/actions/run-dagger-pipeline with: - # Only pre-release images are published until the correct behavior is observed in prod. - # Setting concurrency to 1 for safety: - # High concurrency can lead to resource issues for java connectors. - # As speed is not a concern in this context I think not publishing connectors in parallel is fine. - subcommand: "connectors --concurrency=1 --execute-timeout=3600 --modified publish --main-release" context: "master" - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" + - name: Publish connectors [manual] id: publish-connectors if: github.event_name == 'workflow_dispatch' uses: ./.github/actions/run-dagger-pipeline with: - subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" context: "manual" - github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml new file mode 100644 index 000000000000..477db6af6d6a --- /dev/null +++ b/.github/workflows/publish_pypi.yml @@ -0,0 +1,16 @@ +name: Publish connectors to PyPI + +on: + workflow_dispatch: + inputs: + runs-on: + type: string + default: conn-prod-xlarge-runner + required: true + +jobs: + no-op: + name: No-op + runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }} + steps: + - run: echo 'hi!' diff --git a/.github/workflows/report-connectors-dependency.yml b/.github/workflows/report-connectors-dependency.yml deleted file mode 100644 index e8461cb15d91..000000000000 --- a/.github/workflows/report-connectors-dependency.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Report connectors dependency - -on: - pull_request: - -jobs: - report-affected-connectors: - name: "Report affected connectors" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. - - name: Extract git-diff changed files - run: | - git diff --name-only remotes/origin/master...HEAD -- airbyte-integrations/ > ./changed_files.txt - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - name: Create dependency report - id: dependency_report - run: | - python ./tools/bin/ci_check_dependency.py ./changed_files.txt - if [[ -f comment_body.md ]]; then - echo "comment=true" >> $GITHUB_OUTPUT - fi - - name: Find existing comment for connector dependencies - # Always run this step because the action may need to - # remove a comment created from a previous commit. - uses: peter-evans/find-comment@v2 - id: find_comment - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: "github-actions[bot]" - body-includes: "report-connectors-dependency.yml" - - name: Comment report in PR - if: steps.dependency_report.outputs.comment == 'true' - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.find_comment.outputs.comment-id }} - edit-mode: "replace" - body-file: "comment_body.md" - - name: Remove deprecated report in PR - if: steps.dependency_report.outputs.comment != 'true' && steps.find_comment.outputs.comment-id != '' - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.find_comment.outputs.comment-id }} - edit-mode: "replace" - body: | - - ## Affected Connector Report - The latest commit has removed all connector-related changes. There are no more dependent connectors for this PR. diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index d0f2cb02c9a5..6ac373540ca3 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -19,10 +19,11 @@ jobs: id: scd uses: peter-evans/slash-command-dispatch@v2 with: - token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + token: ${{ secrets.GITHUB_TOKEN }} permission: write commands: | test + legacy-test test-performance build-connector publish-connector diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index ef4fe0e975c5..72312f57938a 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -1,4 +1,4 @@ -name: Run Integration Test +name: Deprecation message for test slash command on: workflow_dispatch: inputs: @@ -27,206 +27,15 @@ on: description: "Run Connector Acceptance Tests against the CDK version on the current branch." required: false jobs: - uuid: - name: "Custom UUID of workflow run" - timeout-minutes: 10 + write-deprecation-message: runs-on: ubuntu-latest steps: - - name: UUID ${{ github.event.inputs.uuid }} - run: true - start-test-runner: - name: Start Build EC2 Runner - needs: uuid - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - pipeline-start-timestamp: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - steps: - - name: Get start timestamp - id: get-start-timestamp - run: echo "::set-output name=start-timestamp::$(date +%s)" - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - github-token: ${{ env.PAT }} - integration-test: - timeout-minutes: 240 - needs: start-test-runner - runs-on: ${{ needs.start-test-runner.outputs.label }} - steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: Search for valid connector name format - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$" - regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} - - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector - run: echo "The connector provided has an invalid format!" && exit 1 - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - name: Install CI scripts - # all CI python packages have the prefix "ci_" - run: | - pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} - run: | - ci_credentials ${{ github.event.inputs.connector }} write-to-storage - # normalization also runs destination-specific tests, so fetch their creds also - if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then - ci_credentials destination-bigquery write-to-storage - ci_credentials destination-postgres write-to-storage - ci_credentials destination-snowflake write-to-storage - fi - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Test ${{ github.event.inputs.connector }} - id: test - env: - ACTION_RUN_ID: ${{github.run_id}} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - ORG_GRADLE_PROJECT_connectorAcceptanceTestVersion: ${{github.event.inputs.connector-acceptance-test-version}} - S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - uses: Wandalen/wretry.action@master - with: - command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} ${{ github.event.inputs.local_cdk }} - attempt_limit: 3 - attempt_delay: 10000 # in ms - - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} - if: always() - run: | - ci_credentials ${{ github.event.inputs.connector }} update-secrets - # normalization also runs destination-specific tests, so fetch their creds also - if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then - ci_credentials destination-bigquery update-secrets - ci_credentials destination-postgres update-secrets - ci_credentials destination-snowflake update-secrets - fi - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Archive test reports artifacts - if: github.event.inputs.comment-id && failure() - uses: actions/upload-artifact@v3 - with: - name: test-reports - path: | - **/${{ github.event.inputs.connector }}/build/reports/tests/**/** - **/${{ github.event.inputs.connector }}/acceptance_tests_logs/** - **/normalization_test_output/**/dbt_output.log - **/normalization_test_output/**/destination_output.log - **/normalization_test_output/**/build/compiled/airbyte_utils/** - **/normalization_test_output/**/build/run/airbyte_utils/** - **/normalization_test_output/**/models/generated/** - - name: Test coverage reports artifacts - if: github.event.inputs.comment-id && success() - uses: actions/upload-artifact@v3 - with: - name: test-reports - path: | - **/${{ github.event.inputs.connector }}/htmlcov/** - retention-days: 3 - - name: Run QA checks for ${{ github.event.inputs.connector }} - id: qa_checks - if: always() - run: | - run-qa-checks ${{ github.event.inputs.connector }} - - name: Report Status - if: github.event.inputs.gitref == 'master' && always() - run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}} ${{steps.qa_checks.outcome}} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - - name: Report Observability - if: always() - run: ./tools/status/report_observability.sh ${{ github.event.inputs.connector }} ${{github.run_id}} ${{ needs.start-test-runner.outputs.pipeline-start-timestamp }} ${{ github.event.inputs.gitref }} ${{ github.sha }} ${{steps.test.outcome}} ${{steps.qa_checks.outcome}} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - - name: Add Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - ${{env.PYTHON_UNITTEST_COVERAGE_REPORT}} - > ${{env.TEST_SUMMARY_INFO}} - - name: Add Failure Comment - if: github.event.inputs.comment-id && failure() + - name: Print deprecation message uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - > :bug: ${{env.GRADLE_SCAN_LINK}} - > ${{env.TEST_SUMMARY_INFO}} - # In case of self-hosted EC2 errors, remove this block. - stop-test-runner: - name: Stop Build EC2 Runner - timeout-minutes: 10 - needs: - - start-test-runner # required to get output from the start-runner job - - integration-test # required to wait when the main job is done - - uuid - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Check PAT rate limits - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ - ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ env.PAT }} - label: ${{ needs.start-test-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }} + > :warning: The test slash command is now deprecated.
+ The connector tests are automatically triggered as CI checks.
+ Please use /legacy-test if you need to test CDK or CAT changes.
+ Please join post to #pipeline-complaint-hotline slack channel if something is not working as expected.
diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index 703be55140e1..0fa936b646b8 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -88,20 +88,14 @@ jobs: - name: Install Python uses: actions/setup-python@v4 with: - python-version: "3.9" - - name: Install Pyenv - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate + python-version: "3.10" - name: Install CI scripts - # all CI python packages have the prefix "ci_" run: | - source venv/bin/activate - pip install --quiet -e ./tools/ci_* + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/ci_credentials - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} run: | - source venv/bin/activate ci_credentials ${{ github.event.inputs.connector }} write-to-storage # normalization also runs destination-specific tests, so fetch their creds also if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then @@ -122,7 +116,6 @@ jobs: - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }} if: always() run: | - source venv/bin/activate ci_credentials ${{ github.event.inputs.connector }} update-secrets # normalization also runs destination-specific tests, so fetch their creds also if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then @@ -154,14 +147,6 @@ jobs: path: | **/${{ github.event.inputs.connector }}/htmlcov/** retention-days: 3 - - - name: Report Status - if: github.ref == 'refs/heads/master' && always() - run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-2" - name: Add Success Comment if: github.event.inputs.comment-id && success() uses: peter-evans/create-or-update-comment@v1 diff --git a/.gitignore b/.gitignore index cee3dbcc9089..56790f53bd9a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ static_checker_reports/ # Logs acceptance_tests_logs/ +airbyte_ci_logs/ # Secrets secrets @@ -91,6 +92,13 @@ dd-java-agent.jar /.env /.env.dev /flags.yml +/temporal/dynamicconfig/development.yaml # Ignore generated credentials from google-github-actions/auth gha-creds-*.json + +# Legacy pipeline reports path +tools/ci_connector_ops/pipeline_reports/ + +# ignore local build scan uri output +scan-journal.log diff --git a/README.md b/README.md index 41920b3f0fb6..bdf4db5d1544 100644 --- a/README.md +++ b/README.md @@ -37,14 +37,14 @@ _Screenshot taken from [Airbyte Cloud](https://cloud.airbyte.com/signup)_. * [Deploy Airbyte Open Source](https://docs.airbyte.com/quickstart/deploy-airbyte) or set up [Airbyte Cloud](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud) to start centralizing your data. * Create connectors in minutes with our [no-code Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview) or [low-code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview). * Explore popular use cases in our [tutorials](https://airbyte.com/tutorials). -* Orchestrate Airbyte syncs with [Airflow](https://docs.airbyte.com/operator-guides/using-the-airflow-airbyte-operator), [Prefect](https://docs.airbyte.com/operator-guides/using-prefect-task), [Dagster](https://docs.airbyte.com/operator-guides/using-dagster-integration) or the [Airbyte API](https://reference.airbyte.com/reference/start). +* Orchestrate Airbyte syncs with [Airflow](https://docs.airbyte.com/operator-guides/using-the-airflow-airbyte-operator), [Prefect](https://docs.airbyte.com/operator-guides/using-prefect-task), [Dagster](https://docs.airbyte.com/operator-guides/using-dagster-integration), [Kestra](https://docs.airbyte.com/operator-guides/using-kestra-plugin) or the [Airbyte API](https://reference.airbyte.com/reference/start). * Easily transform loaded data with [SQL](https://docs.airbyte.com/operator-guides/transformation-and-normalization/transformations-with-sql) or [dbt](https://docs.airbyte.com/operator-guides/transformation-and-normalization/transformations-with-dbt). Try it out yourself with our [demo app](https://demo.airbyte.io/), visit our [full documentation](https://docs.airbyte.com/) and learn more about [recent announcements](https://airbyte.com/blog-categories/company-updates). See our [registry](https://connectors.airbyte.com/files/generated_reports/connector_registry_report.html) for a full list of connectors already available in Airbyte or Airbyte Cloud. ### Join the Airbyte Community -The Airbyte community can be found in the [Airbyte Community Slack](https://airbyte.com/community), where you can ask questions and voice ideas. You can also ask for help in our [Discourse forum](https://discuss.airbyte.io/), or join our [office hours](https://airbyte.io/weekly-office-hours/). Airbyte's roadmap is publicly viewable on [GitHub](https://github.com/orgs/airbytehq/projects/37/views/1?pane=issue&itemId=26937554). +The Airbyte community can be found in the [Airbyte Community Slack](https://airbyte.com/community), where you can ask questions and voice ideas. You can also ask for help in our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions), or join our [Office Hours](https://airbyte.io/daily-office-hours/). Airbyte's roadmap is publicly viewable on [GitHub](https://github.com/orgs/airbytehq/projects/37/views/1?pane=issue&itemId=26937554). For videos and blogs on data engineering and building your data stack, check out Airbyte's [Content Hub](https://airbyte.com/content-hub), [Youtube](https://www.youtube.com/c/AirbyteHQ), and sign up for our [newsletter](https://airbyte.com/newsletter). diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index cc72a6ea53e4..8888cf44878c 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -4,12 +4,22 @@ info: Airbyte Configuration API [https://airbyte.io](https://airbyte.io). + The Configuration API is an internal Airbyte API that is designed for communications between different Airbyte components. + * Its main purpose is to enable the Airbyte Engineering team to configure the internal state of [Airbyte Cloud](https://airbyte.com/airbyte-cloud) + * It is also sometimes used by OSS users to configure their own Self-Hosted Airbyte deployment (internal state, etc) + + WARNING + * Airbyte does NOT have active commitments to support this API long-term. + * OSS users can utilize the Configuration API, but at their own risk. + * This API is utilized internally by the Airbyte Engineering team and may be modified in the future if the need arises. + * Modifications by the Airbyte Engineering team could create breaking changes and OSS users would need to update their code to catch up to any backwards incompatible changes in the API. + This API is a collection of HTTP RPC-style methods. While it is not a REST API, those familiar with REST should find the conventions of this API recognizable. Here are some conventions that this API follows: * All endpoints are http POST methods. * All endpoints accept data via `application/json` request bodies. The API does not accept any data via query params. - * The naming convention for endpoints is: localhost:8000/{VERSION}/{METHOD_FAMILY}/{METHOD_NAME} e.g. `localhost:8000/v1/connections/create`. + * The naming convention for endpoints is: localhost:8000/api/{VERSION}/{METHOD_FAMILY}/{METHOD_NAME} e.g. `localhost:8000/api/v1/connections/create`. * For all `update` methods, the whole object must be passed in, even the fields that did not change. Authentication (OSS): @@ -508,6 +518,196 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/source_definition_specifications/get_for_source: + post: + tags: + - source_definition_specification + summary: Get specification for a source. + operationId: getSpecificationForSourceId + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceIdRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceDefinitionSpecificationRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" + /v1/declarative_source_definitions/create_manifest: + post: + tags: + - declarative_source_definitions + summary: Create a declarative manifest to be used by the specified source definition + operationId: createDeclarativeSourceDefinitionManifest + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DeclarativeSourceDefinitionCreateManifestRequestBody" + required: true + responses: + "201": + description: Successful operation + "400": + description: Definition is not declarative source + "404": + $ref: "#/components/responses/NotFoundResponse" + "409": + description: Version already exists for definition id + /v1/declarative_source_definitions/update_active_manifest: + post: + tags: + - declarative_source_definitions + summary: Update the declarative manifest version for a source + operationId: updateDeclarativeManifestVersion + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/UpdateActiveManifestRequestBody" + required: true + responses: + "204": + description: Successful operation + "404": + $ref: "#/components/responses/NotFoundResponse" + /v1/declarative_source_definitions/list_manifests: + post: + tags: + - declarative_source_definitions + summary: List all available declarative manifest versions of a declarative source definition + operationId: listDeclarativeManifests + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListDeclarativeManifestsRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DeclarativeManifestsReadList" + "400": + description: Definition is not declarative source + "404": + $ref: "#/components/responses/NotFoundResponse" + /v1/connector_builder_projects/create: + post: + tags: + - connector_builder_project + summary: Create new connector builder project + operationId: createConnectorBuilderProject + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderProjectWithWorkspaceId" + required: true + responses: + "201": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderProjectIdWithWorkspaceId" + /v1/connector_builder_projects/publish: + post: + tags: + - connector_builder_project + summary: Publish a connector to the workspace + operationId: publishConnectorBuilderProject + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderPublishRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceDefinitionIdBody" + /v1/connector_builder_projects/update: + post: + tags: + - connector_builder_project + summary: Update connector builder project + operationId: updateConnectorBuilderProject + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ExistingConnectorBuilderProjectWithWorkspaceId" + required: true + responses: + "204": + description: Successful operation + /v1/connector_builder_projects/delete: + post: + tags: + - connector_builder_project + summary: Deletes connector builder project + operationId: deleteConnectorBuilderProject + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderProjectIdWithWorkspaceId" + required: true + responses: + "204": + description: Successful operation + /v1/connector_builder_projects/list: + post: + tags: + - connector_builder_project + summary: List connector builder projects for workspace + operationId: listConnectorBuilderProjects + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceIdRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderProjectReadList" + /v1/connector_builder_projects/get_with_manifest: + post: + tags: + - connector_builder_project + summary: Get a connector builder project with draft manifest + operationId: getConnectorBuilderProject + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderProjectIdWithWorkspaceId" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectorBuilderProjectRead" /v1/sources/create: post: tags: @@ -552,6 +752,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/sources/partial_update: + post: + tags: + - source + summary: Partially update a source + operationId: partialUpdateSource + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PartialSourceUpdate" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/SourceRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/sources/list: post: tags: @@ -754,7 +977,27 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" - + /v1/sources/apply_schema_changes: + post: + tags: + - source + summary: + Auto propagate the change on a catalog to a catalog saved in the DB. It will fetch all the connections linked to + a source id and apply the provided diff to their catalog. + operationId: applySchemaChangeForSource + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SourceAutoPropagateChange" + required: true + responses: + "204": + description: The schema was properly auto propagate + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/sources/write_discover_catalog_result: post: tags: @@ -1013,6 +1256,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/destination_definition_specifications/get_for_destination: + post: + tags: + - destination_definition_specification + summary: Get specification for a destination + operationId: getSpecificationForDestinationId + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationIdRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationDefinitionSpecificationRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" # DESTINATIONS /v1/destinations/create: post: @@ -1056,6 +1322,27 @@ paths: $ref: "#/components/schemas/DestinationRead" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/destinations/partial_update: + post: + tags: + - destination + summary: Update a destination partially + operationId: partialUpdateDestination + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PartialDestinationUpdate" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DestinationRead" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/destinations/list: post: tags: @@ -1329,6 +1616,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/connections/list_by_actor_definition: + post: + tags: + - connection + summary: List all connections that use the provided actor definition + operationId: listConnectionsByActorDefinition + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ActorDefinitionRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/state/get: post: tags: @@ -1462,6 +1772,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/connections/reset/stream: + post: + tags: + - connection + summary: Reset the data for a specific stream in the connection. Deletes data generated by the stream in the destination. Resets any cursors back to initial state. + operationId: resetConnectionStream + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ConnectionStreamRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/JobInfoRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/operations/check: post: tags: @@ -1723,6 +2056,25 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/source_oauths/revoke: + post: + tags: + - source_oauth + summary: Given a source definition ID and workspace ID revoke access/refresh token etc. + operationId: revokeSourceOAuthTokens + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/RevokeSourceOauthTokensRequest" + required: true + responses: + "200": + description: Successful operation + "400": + $ref: "#/components/responses/ExceptionResponse" + "404": + $ref: "#/components/responses/NotFoundResponse" /v1/destination_oauths/get_consent_url: post: tags: @@ -2032,6 +2384,29 @@ paths: $ref: "#/components/responses/NotFoundResponse" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/jobs/get_without_logs: + post: + tags: + - jobs + summary: Get information about a job excluding logs + operationId: getJobInfoWithoutLogs + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/JobIdRequestBody" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/JobInfoRead" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/jobs/get_light: post: tags: @@ -2234,7 +2609,69 @@ paths: application/json: schema: $ref: "#/components/schemas/InternalOperationResult" - + /v1/stream_statuses/list: + post: + summary: Gets a list of stream statuses filtered by parameters (with AND semantics). + tags: + - stream_statuses + - streams + operationId: getStreamStatuses + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusListRequestBody" + responses: + "200": + description: Successfully queried stream statuses. + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusReadList" + /v1/stream_statuses/create: + post: + summary: Creates a stream status. + tags: + - stream_statuses + - streams + operationId: createStreamStatus + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusCreateRequestBody" + responses: + "201": + description: Successfully created stream status. + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusRead" + /v1/stream_statuses/update: + post: + summary: Updates a stream status. + tags: + - stream_statuses + - streams + operationId: updateStreamStatus + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusUpdateRequestBody" + responses: + "201": + description: Successfully created stream status. + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusRead" + "200": + description: Successfully updated stream status. + content: + application/json: + schema: + $ref: "#/components/schemas/StreamStatusRead" components: securitySchemes: bearerAuth: @@ -2269,6 +2706,8 @@ components: type: array items: $ref: "#/components/schemas/Notification" + notificationSettings: + $ref: "#/components/schemas/NotificationSettings" displaySetupWizard: type: boolean defaultGeography: @@ -2289,6 +2728,34 @@ components: validationUrl: type: string description: if supplied, the webhook config will be validated by checking that this URL returns a 2xx response. + NotificationItem: + type: object + properties: + notificationType: + type: array + items: + $ref: "#/components/schemas/NotificationType" + slackConfiguration: + $ref: "#/components/schemas/SlackNotificationConfiguration" + customerioConfiguration: + $ref: "#/components/schemas/CustomerioNotificationConfiguration" + + NotificationSettings: + type: object + properties: + sendOnSuccess: + $ref: "#/components/schemas/NotificationItem" + sendOnFailure: + $ref: "#/components/schemas/NotificationItem" + sendOnSyncDisabled: + $ref: "#/components/schemas/NotificationItem" + sendOnSyncDisabledWarning: + $ref: "#/components/schemas/NotificationItem" + sendOnConnectionUpdate: + $ref: "#/components/schemas/NotificationItem" + sendOnConnectionUpdateActionRequired: + $ref: "#/components/schemas/NotificationItem" + Notification: type: object required: @@ -2344,6 +2811,38 @@ components: properties: workspaceId: $ref: "#/components/schemas/WorkspaceId" + ListConnectionsForWorkspacesRequestBody: + type: object + required: + - workspaceIds + - userId + properties: + workspaceIds: + type: array + items: + $ref: "#/components/schemas/WorkspaceId" + userId: + type: string + format: uuid + pagination: + $ref: "#/components/schemas/Pagination" + includeDeleted: + type: boolean + default: false + ListResourcesForWorkspacesRequestBody: + type: object + required: + - workspaceIds + properties: + workspaceIds: + type: array + items: + $ref: "#/components/schemas/WorkspaceId" + includeDeleted: + type: boolean + default: false + pagination: + $ref: "#/components/schemas/Pagination" WorkspaceReadList: type: object required: @@ -2387,6 +2886,8 @@ components: type: array items: $ref: "#/components/schemas/Notification" + notificationSettings: + $ref: "#/components/schemas/NotificationSettings" firstCompletedSync: type: boolean feedbackDone: @@ -2445,6 +2946,8 @@ components: type: array items: $ref: "#/components/schemas/Notification" + notificationSettings: + $ref: "#/components/schemas/NotificationSettings" defaultGeography: $ref: "#/components/schemas/Geography" webhookConfigs: @@ -2502,6 +3005,145 @@ components: - auto - us - eu + # BuilderProject + ConnectorBuilderProjectId: + type: string + format: uuid + DeclarativeManifest: + description: Low code CDK manifest JSON object + type: object + DeclarativeManifestRead: + type: object + properties: + manifest: + $ref: "#/components/schemas/DeclarativeManifest" + isDraft: + type: boolean + version: + $ref: "#/components/schemas/ManifestVersion" + description: + type: string + ConnectorBuilderProjectDetails: + type: object + required: + - name + properties: + name: + type: string + draftManifest: + $ref: "#/components/schemas/DeclarativeManifest" + ConnectorBuilderProjectDetailsRead: + type: object + required: + - name + - builderProjectId + - hasDraft + properties: + name: + type: string + builderProjectId: + $ref: "#/components/schemas/ConnectorBuilderProjectId" + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" + activeDeclarativeManifestVersion: + $ref: "#/components/schemas/ManifestVersion" + hasDraft: + type: boolean + ConnectorBuilderProjectIdWithWorkspaceId: + type: object + required: + - workspaceId + - builderProjectId + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + builderProjectId: + $ref: "#/components/schemas/ConnectorBuilderProjectId" + version: + $ref: "#/components/schemas/ManifestVersion" + ExistingConnectorBuilderProjectWithWorkspaceId: + type: object + required: + - workspaceId + - builderProjectId + - builderProject + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + builderProjectId: + $ref: "#/components/schemas/ConnectorBuilderProjectId" + builderProject: + $ref: "#/components/schemas/ConnectorBuilderProjectDetails" + ConnectorBuilderProjectWithWorkspaceId: + type: object + required: + - workspaceId + - builderProject + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + builderProject: + $ref: "#/components/schemas/ConnectorBuilderProjectDetails" + ConnectorBuilderProjectRead: + type: object + required: + - builderProject + properties: + builderProject: + $ref: "#/components/schemas/ConnectorBuilderProjectDetailsRead" + declarativeManifest: + $ref: "#/components/schemas/DeclarativeManifestRead" + ConnectorBuilderProjectReadList: + type: object + required: + - projects + properties: + projects: + type: array + items: + $ref: "#/components/schemas/ConnectorBuilderProjectDetailsRead" + ConnectorBuilderPublishRequestBody: + type: object + required: + - workspaceId + - builderProjectId + - name + - initialDeclarativeManifest + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + builderProjectId: + $ref: "#/components/schemas/ConnectorBuilderProjectId" + name: + type: string + initialDeclarativeManifest: + $ref: "#/components/schemas/DeclarativeSourceManifest" + ManifestVersion: + type: integer + format: int64 + DeclarativeSourceManifest: + type: object + required: + - description + - manifest + - spec + - version + properties: + description: + type: string + manifest: + $ref: "#/components/schemas/DeclarativeManifest" + spec: + $ref: "#/components/schemas/SourceDefinitionSpecification" + version: + $ref: "#/components/schemas/ManifestVersion" + SourceDefinitionIdBody: + type: object + required: + - sourceDefinitionId + properties: + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" # SourceDefinition SourceDefinitionId: type: string @@ -2586,35 +3228,101 @@ components: - custom resourceRequirements: $ref: "#/components/schemas/ActorDefinitionResourceRequirements" + maxSecondsBetweenMessages: + description: Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach + type: integer + format: int64 SourceDefinitionReadList: type: object required: - - sourceDefinitions + - sourceDefinitions + properties: + sourceDefinitions: + type: array + items: + $ref: "#/components/schemas/SourceDefinitionRead" + CustomSourceDefinitionCreate: + type: object + required: + - workspaceId + - sourceDefinition + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + sourceDefinition: + $ref: "#/components/schemas/SourceDefinitionCreate" + SourceDefinitionIdWithWorkspaceId: + type: object + required: + - sourceDefinitionId + - workspaceId + properties: + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + ListDeclarativeManifestsRequestBody: + type: object + required: + - workspaceId + - sourceDefinitionId properties: - sourceDefinitions: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" + DeclarativeManifestsReadList: + type: object + required: + - manifestVersions + properties: + manifestVersions: type: array items: - $ref: "#/components/schemas/SourceDefinitionRead" - CustomSourceDefinitionCreate: + $ref: "#/components/schemas/DeclarativeManifestVersionRead" + DeclarativeManifestVersionRead: + type: object + required: + - version + - isActive + - description + properties: + version: + type: integer + format: int64 + isActive: + type: boolean + description: + type: string + DeclarativeSourceDefinitionCreateManifestRequestBody: type: object required: - workspaceId - - sourceDefinition + - sourceDefinitionId + - setAsActiveManifest + - declarativeManifest properties: workspaceId: $ref: "#/components/schemas/WorkspaceId" - sourceDefinition: - $ref: "#/components/schemas/SourceDefinitionCreate" - SourceDefinitionIdWithWorkspaceId: + sourceDefinitionId: + $ref: "#/components/schemas/ConnectorBuilderProjectId" + setAsActiveManifest: + type: boolean + declarativeManifest: + $ref: "#/components/schemas/DeclarativeSourceManifest" + UpdateActiveManifestRequestBody: type: object required: - - sourceDefinitionId - workspaceId + - sourceDefinitionId + - version properties: - sourceDefinitionId: - $ref: "#/components/schemas/SourceDefinitionId" workspaceId: $ref: "#/components/schemas/WorkspaceId" + sourceDefinitionId: + $ref: "#/components/schemas/ConnectorBuilderProjectId" + version: + $ref: "#/components/schemas/ManifestVersion" PrivateSourceDefinitionRead: type: object required: @@ -2639,64 +3347,6 @@ components: description: The specification for what values are required to configure the sourceDefinition. type: object example: { user: { type: string } } - SourceAuthSpecification: - $ref: "#/components/schemas/AuthSpecification" - AuthSpecification: - type: object - properties: - auth_type: - type: string - enum: ["oauth2.0"] # Future auth types should be added here - oauth2Specification: - "$ref": "#/components/schemas/OAuth2Specification" - OAuth2Specification: - description: An object containing any metadata needed to describe this connector's Oauth flow - type: object - required: - - rootObject - - oauthFlowInitParameters - - oauthFlowOutputParameters - properties: - rootObject: - description: - "A list of strings representing a pointer to the root object which contains any oauth parameters in the ConnectorSpecification. - - Examples: - - if oauth parameters were contained inside the top level, rootObject=[] - If they were nested inside another object {'credentials': {'app_id' etc...}, rootObject=['credentials'] - If they were inside a oneOf {'switch': {oneOf: [{client_id...}, {non_oauth_param]}}, rootObject=['switch', 0] - " - type: array - items: {} # <--- using generic any type. Build fails with oneOf (https://github.com/OpenAPITools/openapi-generator/issues/6161) - example: - - path - - 1 - oauthFlowInitParameters: - description: - "Pointers to the fields in the rootObject needed to obtain the initial refresh/access tokens for the OAuth flow. - Each inner array represents the path in the rootObject of the referenced field. - For example. - Assume the rootObject contains params 'app_secret', 'app_id' which are needed to get the initial refresh token. - If they are not nested in the rootObject, then the array would look like this [['app_secret'], ['app_id']] - If they are nested inside an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']]" - type: array - items: - description: A list of strings denoting a pointer into the rootObject for where to find this property - type: array - items: - type: string - oauthFlowOutputParameters: - description: - "Pointers to the fields in the rootObject which can be populated from successfully completing the oauth flow using the init parameters. - This is typically a refresh/access token. - Each inner array represents the path in the rootObject of the referenced field." - type: array - items: - description: A list of strings denoting a pointer into the rootObject for where to find this property - type: array - items: - type: string SourceDefinitionSpecificationRead: type: object required: @@ -2709,8 +3359,6 @@ components: type: string connectionSpecification: $ref: "#/components/schemas/SourceDefinitionSpecification" - authSpecification: - $ref: "#/components/schemas/SourceAuthSpecification" advancedAuth: $ref: "#/components/schemas/AdvancedAuth" jobInfo: @@ -2777,6 +3425,9 @@ components: $ref: "#/components/schemas/WorkspaceId" name: type: string + secretId: + example: "airbyte_oauth_workspace_0509f049-d671-48cb-8105-0a23d47e6db6_secret_e0d38206-034e-4d75-9d21-da5a99b02826_v1" + type: string SourceDiscoverSchemaRequestBody: type: object required: @@ -2791,6 +3442,20 @@ components: type: boolean notifySchemaChange: type: boolean + PartialSourceUpdate: + type: object + required: + - sourceId + properties: + sourceId: + $ref: "#/components/schemas/SourceId" + connectionConfiguration: + $ref: "#/components/schemas/SourceConfiguration" + name: + type: string + secretId: + example: "airbyte_oauth_workspace_0509f049-d671-48cb-8105-0a23d47e6db6_secret_e0d38206-034e-4d75-9d21-da5a99b02826_v1" + type: string SourceUpdate: type: object required: @@ -2804,6 +3469,9 @@ components: $ref: "#/components/schemas/SourceConfiguration" name: type: string + secretId: + example: "airbyte_oauth_workspace_0509f049-d671-48cb-8105-0a23d47e6db6_secret_e0d38206-034e-4d75-9d21-da5a99b02826_v1" + type: string SourceRead: type: object required: @@ -2889,6 +3557,28 @@ components: type: boolean connectionStatus: $ref: "#/components/schemas/ConnectionStatus" + SourceAutoPropagateChange: + description: + Input of the source propagation, it contains the discovered catalog and a list of diff that need to be applied + to the existing catalog. + type: object + required: + - catalog + - catalogId + - sourceId + - workspaceId + properties: + catalog: + $ref: "#/components/schemas/AirbyteCatalog" + catalogId: + type: string + format: uuid + sourceId: + type: string + format: uuid + workspaceId: + type: string + format: uuid SourceSearch: type: object properties: @@ -2908,8 +3598,6 @@ components: DestinationDefinitionId: type: string format: uuid - DestinationAuthSpecification: - $ref: "#/components/schemas/AuthSpecification" DestinationDefinitionIdRequestBody: type: object required: @@ -3042,6 +3730,7 @@ components: # DESTINATION DEFINITION SPECIFICATION DestinationDefinitionSpecification: description: The specification for what values are required to configure the destinationDefinition. + type: object example: { user: { type: string } } DestinationDefinitionSpecificationRead: type: object @@ -3055,8 +3744,6 @@ components: type: string connectionSpecification: $ref: "#/components/schemas/DestinationDefinitionSpecification" - authSpecification: - $ref: "#/components/schemas/DestinationAuthSpecification" advancedAuth: $ref: "#/components/schemas/AdvancedAuth" jobInfo: @@ -3123,6 +3810,15 @@ components: $ref: "#/components/schemas/DestinationConfiguration" name: type: string + PartialDestinationUpdate: + type: object + properties: + destinationId: + $ref: "#/components/schemas/DestinationId" + connectionConfiguration: + $ref: "#/components/schemas/DestinationConfiguration" + name: + type: string DestinationCloneRequestBody: description: The values required to configure the destination. The schema for this should have an id of the existing destination along with the configuration you want to change in case. type: object @@ -3225,6 +3921,28 @@ components: properties: connectionId: $ref: "#/components/schemas/ConnectionId" + ConnectionStream: + type: object + required: + - streamName + - streamNamespace + properties: + streamName: + type: string + streamNamespace: + type: string + ConnectionStreamRequestBody: + type: object + required: + - connectionId + - streams + properties: + connectionId: + $ref: "#/components/schemas/ConnectionId" + streams: + type: array + items: + $ref: "#/components/schemas/ConnectionStream" DbMigrationRequestBody: type: object required: @@ -3288,6 +4006,8 @@ components: $ref: "#/components/schemas/Geography" notifySchemaChanges: type: boolean + notifySchemaChangesByEmail: + type: boolean nonBreakingChangesPreference: $ref: "#/components/schemas/NonBreakingChangesPreference" WebBackendConnectionCreate: @@ -3395,6 +4115,8 @@ components: $ref: "#/components/schemas/Geography" notifySchemaChanges: type: boolean + notifySchemaChangesByEmail: + type: boolean nonBreakingChangesPreference: $ref: "#/components/schemas/NonBreakingChangesPreference" breakingChange: @@ -3445,6 +4167,8 @@ components: $ref: "#/components/schemas/Geography" notifySchemaChanges: type: boolean + notifySchemaChangesByEmail: + type: boolean nonBreakingChangesPreference: $ref: "#/components/schemas/NonBreakingChangesPreference" ConnectionRead: @@ -3501,8 +4225,12 @@ components: type: boolean notifySchemaChanges: type: boolean + notifySchemaChangesByEmail: + type: boolean nonBreakingChangesPreference: $ref: "#/components/schemas/NonBreakingChangesPreference" + workspaceId: + $ref: "#/components/schemas/WorkspaceId" SchemaChange: enum: - no_change @@ -3984,6 +4712,10 @@ components: $ref: "#/components/schemas/JobConfigType" configId: type: string + enabledStreams: + type: array + items: + $ref: "#/components/schemas/StreamDescriptor" createdAt: type: integer format: int64 @@ -4116,6 +4848,9 @@ components: stateMessagesEmitted: type: integer format: int64 + bytesCommitted: + type: integer + format: int64 recordsCommitted: type: integer format: int64 @@ -4145,19 +4880,19 @@ components: failures: type: array items: - $ref: "#/components/schemas/AttemptFailureReason" + $ref: "#/components/schemas/FailureReason" partialSuccess: description: True if the number of committed records for this attempt was greater than 0. False if 0 records were committed. If not set, the number of committed records is unknown. type: boolean - AttemptFailureReason: + FailureReason: type: object required: - timestamp properties: failureOrigin: - $ref: "#/components/schemas/AttemptFailureOrigin" + $ref: "#/components/schemas/FailureOrigin" failureType: - $ref: "#/components/schemas/AttemptFailureType" + $ref: "#/components/schemas/FailureType" externalMessage: type: string internalMessage: @@ -4170,7 +4905,7 @@ components: timestamp: type: integer format: int64 - AttemptFailureOrigin: + FailureOrigin: description: Indicates where the error originated. If not set, the origin of error is not well known. type: string enum: @@ -4182,7 +4917,7 @@ components: - dbt - airbyte_platform - unknown - AttemptFailureType: + FailureType: description: Categorizes well known errors into types for programmatic handling. If not set, the type of error is not well known. type: string enum: @@ -4190,6 +4925,7 @@ components: - system_error - manual_cancellation - refresh_schema + - heartbeat_timeout AttemptStatus: type: string enum: @@ -4297,6 +5033,8 @@ components: default: false logs: $ref: "#/components/schemas/LogRead" + failureReason: + $ref: "#/components/schemas/FailureReason" Pagination: type: object properties: @@ -4316,7 +5054,6 @@ components: CheckConnectionRead: type: object required: - - status - jobInfo properties: status: @@ -4469,6 +5206,18 @@ components: type: array items: "$ref": "#/components/schemas/JobTypeResourceLimit" + ActorDefinitionRequestBody: + type: object + additionalProperties: false + required: + - actorDefinitionId + - actorType + properties: + actorDefinitionId: + type: string + format: uuid + actorType: + $ref: "#/components/schemas/ActorType" NormalizationDestinationDefinitionConfig: description: describes a normalization config for destination definition type: object @@ -4757,8 +5506,25 @@ components: additionalProperties: true # Oauth parameters like code, state, etc.. will be different per API so we don't specify them in advance oAuthInputConfiguration: $ref: "#/components/schemas/OAuthInputConfiguration" + returnSecretCoordinate: + type: boolean + description: If set to true, returns a secret coordinate which references the stored tokens. By default, returns raw tokens. + default: false + sourceId: + $ref: "#/components/schemas/SourceId" + RevokeSourceOauthTokensRequest: + type: object + required: + - sourceId + - sourceDefinitionId + - workspaceId + properties: + workspaceId: + $ref: "#/components/schemas/WorkspaceId" sourceId: $ref: "#/components/schemas/SourceId" + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" CompleteDestinationOAuthRequest: type: object required: @@ -4782,7 +5548,17 @@ components: $ref: "#/components/schemas/DestinationId" CompleteOAuthResponse: type: object - additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API so we don't specify them in advance + required: + - request_succeeded + - auth_payload + properties: + request_succeeded: + type: boolean + request_error: + type: string + auth_payload: + type: object + additionalProperties: true # Oauth parameters like refresh/access token etc.. will be different per API, so we don't specify them in advance SetInstancewideSourceOauthParamsRequestBody: type: object required: @@ -4805,6 +5581,29 @@ components: params: type: object additionalProperties: true + WorkspaceOverrideOauthParamsRequestBody: + type: object + required: + - definitionId + - params + - workspaceId + - actorType + properties: + definitionId: + type: string + format: uuid + params: + type: object + additionalProperties: true + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + actorType: + $ref: "#/components/schemas/ActorType" + ActorType: + enum: + - source + - destination + type: string # Web Backend WebBackendCheckUpdatesRead: type: object @@ -4880,6 +5679,7 @@ components: - isSyncing - schemaChange - notifySchemaChanges + - notifySchemaChangesByEmail - nonBreakingChangesPreference properties: connectionId: @@ -4941,12 +5741,16 @@ components: $ref: "#/components/schemas/SchemaChange" notifySchemaChanges: type: boolean + notifySchemaChangesByEmail: + type: boolean nonBreakingChangesPreference: $ref: "#/components/schemas/NonBreakingChangesPreference" NonBreakingChangesPreference: enum: - - ignore - - disable + - ignore # do nothing if we detect a schema change + - disable # disable the connection, pausing the sync + - propagate_columns # automatically propagate the changes on selected streams and continue syncing + - propagate_fully # automatically propagate the changes including new streams and continue syncing type: string WebBackendConnectionReadList: type: object @@ -5107,7 +5911,145 @@ components: format: int64 hasNormalizationFailed: type: boolean - + StreamStatusId: + type: string + format: uuid + StreamStatusRunState: + type: string + description: > + Values: + * `PENDING` - The stream operation has been selected to run + * `RUNNING` - The stream operation is running + * `COMPLETE` - The stream operation ran successfully to completion + * `INCOMPLETE` - The stream operation has terminated in an incomplete state. + See StreamStatusIncompleteRunCause for more details. + enum: + - PENDING + - RUNNING + - COMPLETE + - INCOMPLETE + StreamStatusIncompleteRunCause: + type: string + description: > + Values: + * `FAILED` - A failure has occurred + * `CANCELED` - The run has been canceled + enum: + - FAILED + - CANCELED + StreamStatusJobType: + type: string + enum: + - SYNC + - RESET + StreamStatusListRequestBody: + type: object + required: + - pagination + - workspaceId + properties: + attemptNumber: + $ref: "#/components/schemas/AttemptNumber" + connectionId: + $ref: "#/components/schemas/ConnectionId" + jobId: + $ref: "#/components/schemas/JobId" + jobType: + $ref: "#/components/schemas/StreamStatusJobType" + pagination: + $ref: "#/components/schemas/Pagination" + streamName: + type: string + streamNamespace: + type: string + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + StreamStatusCreateRequestBody: + type: object + required: + - attemptNumber + - connectionId + - jobId + - jobType + - runState + - streamName + - transitionedAt + - workspaceId + properties: + attemptNumber: + $ref: "#/components/schemas/AttemptNumber" + connectionId: + $ref: "#/components/schemas/ConnectionId" + jobId: + $ref: "#/components/schemas/JobId" + incompleteRunCause: + $ref: "#/components/schemas/StreamStatusIncompleteRunCause" + jobType: + $ref: "#/components/schemas/StreamStatusJobType" + runState: + $ref: "#/components/schemas/StreamStatusRunState" + streamName: + type: string + streamNamespace: + type: string + transitionedAt: + type: integer + format: int64 + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + StreamStatusUpdateRequestBody: + type: object + allOf: + - $ref: "#/components/schemas/StreamStatusCreateRequestBody" + required: + - id + properties: + id: + $ref: "#/components/schemas/StreamStatusId" + StreamStatusRead: + type: object + required: + - attemptNumber + - connectionId + - id + - jobId + - jobType + - runState + - streamName + - streamNamespace + - transitionedAt + - workspaceId + properties: + attemptNumber: + $ref: "#/components/schemas/AttemptNumber" + connectionId: + $ref: "#/components/schemas/ConnectionId" + id: + $ref: "#/components/schemas/StreamStatusId" + jobId: + $ref: "#/components/schemas/JobId" + incompleteRunCause: + $ref: "#/components/schemas/StreamStatusIncompleteRunCause" + jobType: + $ref: "#/components/schemas/StreamStatusJobType" + runState: + $ref: "#/components/schemas/StreamStatusRunState" + streamName: + type: string + streamNamespace: + type: string + transitionedAt: + type: integer + format: int64 + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + StreamStatusReadList: + type: object + properties: + streamStatuses: + type: array + items: + $ref: "#/components/schemas/StreamStatusRead" InvalidInputProperty: type: object required: diff --git a/airbyte-base-java-image/Dockerfile b/airbyte-base-java-image/Dockerfile index 13b1fae880ed..e672d3b71ce0 100644 --- a/airbyte-base-java-image/Dockerfile +++ b/airbyte-base-java-image/Dockerfile @@ -4,7 +4,7 @@ ARG DOCKER_BUILD_ARCH=amd64 WORKDIR /app -RUN yum install -y tar +RUN yum install -y tar && yum clean all # Add the Datadog Java APM agent ADD https://dtdg.co/latest-java-tracer dd-java-agent.jar diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md new file mode 100644 index 000000000000..e8e106c02372 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -0,0 +1,102 @@ +# Developing with the Java CDK + +This page will walk through the process of developing with the Java CDK. + +## Building the CDK + +To build and test the Java CDK, execute the following: + +```sh +./gradlew :airbyte-cdk:java:airbyte-cdk:build +``` + +## Bumping the declared CDK version + +You will need to bump this version manually whenever you are making changes to code inside the CDK. + +While under development, the next version number for the CDK is tracked in the file: `airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties`. + +If the CDK is not being modified, this file will contain the most recently published version number. + +## Publishing the CDK to Local Maven + +If your connector pins to a work-in-progress `-SNAPSHOT` version of the CDK (e.g. `0.0.1-SNAPSHOT` or `0.2.0-SNAPSHOT`), Gradle will notice this and automatically run the task to build and publish it to your MavenLocal repository before running the connector's own build and test tasks. + +## Referencing the CDK from Java connectors + +You can reference the CDK in your connector's `build.gradle` file: + +```groovy +dependencies { + implementation 'io.airbyte:airbyte-cdk:0.0.1-SNAPSHOT' +} +``` + +Replace `0.0.1-SNAPSHOT` with the version you are working with. If you're actively developing the CDK and want to use the latest version locally, use the `-SNAPSHOT` suffix to reference a bumped version number. (See below for version bump instructions.) + +## Developing a connector alongside the CDK + +You can iterate on changes in the CDK local and test them in the connector without needing to publish the CDK changes publicly. + +When modifying the CDK and a connector in the same PR or branch, please use the following steps: + +1. Set the version of the CDK in `version.properties` to the next appropriate version number, along with a `-SNAPSHOT` suffix, as explained above. +1. In your connector project, modify the `build.gradle` to use the _new_ local CDK version with the `-SNAPSHOT` suffix, as explained above. +1. Build and test your connector as usual. Gradle will automatically build the snapshot version of the CDK, and it will use this version when building and testing your connector. +1. As you make additional changes to the CDK, Gradle will automatically rebuild and republish the CDK locally in order to incorporate the latest changes. + +## Developing a connector against a pinned CDK version + +You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`. + +Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your connector, and you can use your local IDE to browse the prior version of the codebase that corresponds to that version. + + + +## Publish and release + +_⚠️ These steps should only be performed after all testing and approvals are in place on the PR. ⚠️_ + +1. Remove `-SNAPSHOT` suffix from CDK version. + - e.g. by running `nano airbyte-cdk/java/airbyte-cdk/src/main/resources/ +version.properties`. +2. Publish the CDK to Maven ([mycloudrepo](https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/)) + - `./gradlew :airbyte-cdk:java:airbyte-cdk:publish` + - Note: you will need to export the env vars `CLOUDREPO_USER` and `CLOUDREPO_PASSWORD` before publishing. +3. Remove the `-SNAPSHOT` suffix from any connector(s) using the latest version. + - E.g. If modifying `source-mysql`, then remove `-SNAPSHOT` from the CDK `implements` declaration in `airbyte-integrations/connectors/source-mysql/build.gradle`. +4. As per the normal process, modified connector(s) will be automatically published after they are merged to the main branch. + +Note: + +- This is documented as a manual process, but we will automate it into a CI workflow. +- You can view and administer published CDK versions here: https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/airbyte-cdk +- The corresponding public endpoint for published CDK versions is here: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/ + +## Debugging + +MavenLocal debugging steps: + +1. Confirm local publish status by running: + `ls -la ~/.m2/repository/io/airbyte/airbyte-cdk\*` +2. Confirm jar contents by running: + `jar tf ~/.m2/repository/io/airbyte/airbyte-cdk/0.0.2-SNAPSHOT/airbyte-cdk-0.0.2-SNAPSHOT.jar` +3. Remove CDK artifacts from MavenLocal by running: + `rm -rf ~/.m2/repository/io/airbyte/airbyte-cdk\*` +4. Rebuid CDK artifacts by running: + `./gradlew :airbyte-cdk:java:airbyte-cdk:build` + or + `./gradlew :airbyte-cdk:java:airbyte-cdk:publishToMavenLocal` + +## Changelog + +### Java CDK + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------ | +| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | +| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle new file mode 100644 index 000000000000..8b7800baa7a1 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -0,0 +1,72 @@ +plugins { + id 'java-library' + id 'maven-publish' +} + +group 'io.airbyte' + +// Version is dynamically loaded from version.properties file. +def props = new Properties() +file("src/main/resources/version.properties").withInputStream { props.load(it) } +version = props.getProperty('version') +description = "Airbyte Connector Development Kit (CDK) for Java." + +dependencies { + testImplementation 'org.junit.jupiter:junit-jupiter:5.7.2' +} + +publishing { + publications { + maven(MavenPublication) { + groupId = 'io.airbyte' + artifactId = 'airbyte-cdk' + from components.java + } + } + repositories { + maven { + name 'cloudrepo' + url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars' + credentials { + username System.getenv('CLOUDREPO_USER') + password System.getenv('CLOUDREPO_PASSWORD') + } + } + } +} + +// Adds publishToMavenLocal as final command in the list of 'build' tasks. +build.dependsOn(publishToMavenLocal) + +publishToMavenLocal { + // Always re-publish the artifact to MavenLocal + outputs.upToDateWhen { false } + + doFirst { + println "Running CDK publishToMavenLocal..." + } + doLast { + println "Finished CDK publishToMavenLocal." + } +} + +// This task will be a no-op if CDK version does not end with '-SNAPSHOT'. +task publishSnapshotIfNeeded {} + +if (version.endsWith("-SNAPSHOT")) { + logger.lifecycle("Version ${version} ends with '-SNAPSHOT'. Enqueing 'publishToMavenLocal'...") + publishSnapshotIfNeeded.dependsOn publishToMavenLocal +} else { + logger.lifecycle("Version ${version} does not end with '-SNAPSHOT'. Skipping task 'publishToMavenLocal'.") +} + +test { + useJUnitPlatform() + testLogging { + exceptionFormat = 'full' + showExceptions = true + showCauses = true + showStackTraces = false + events = ['passed', 'skipped', 'failed'] + } +} diff --git a/airbyte-cdk/java/airbyte-cdk/src/main/java/io/airbyte/cdk/CDKConstants.java b/airbyte-cdk/java/airbyte-cdk/src/main/java/io/airbyte/cdk/CDKConstants.java new file mode 100644 index 000000000000..a7e73115d9fe --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/src/main/java/io/airbyte/cdk/CDKConstants.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public final class CDKConstants { + + private CDKConstants() { + // restrict instantiation + } + + public static final String VERSION = getVersion(); + + private static String getVersion() { + Properties prop = new Properties(); + + try (InputStream inputStream = CDKConstants.class.getClassLoader().getResourceAsStream("version.properties")) { + prop.load(inputStream); + return prop.getProperty("version"); + } catch (IOException e) { + throw new RuntimeException("Could not read version properties file", e); + } + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties new file mode 100644 index 000000000000..6360017a8ee7 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties @@ -0,0 +1 @@ +version=0.0.2 diff --git a/airbyte-cdk/java/airbyte-cdk/src/test/java/io/airbyte/cdk/CDKConstantsTest.java b/airbyte-cdk/java/airbyte-cdk/src/test/java/io/airbyte/cdk/CDKConstantsTest.java new file mode 100644 index 000000000000..a4942ebf0080 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/src/test/java/io/airbyte/cdk/CDKConstantsTest.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class CDKConstantsTest { + + /* TODO: Remove these canary tests once real tests are in place. */ + @Test + void getVersion() { + assertEquals("0.0.2", CDKConstants.VERSION.replace("-SNAPSHOT", "")); + } + + @Test + // Comment out this line to force failure: + @Disabled("This is an intentionally failing test (skipped).") + void mustFail() { + fail("This is an intentionally failing test."); + } + +} diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 2aae694a1b6a..4d6fec045246 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.39.1 +current_version = 0.51.6 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/.mypy.ini b/airbyte-cdk/python/.mypy.ini deleted file mode 100644 index b255df24bae5..000000000000 --- a/airbyte-cdk/python/.mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -# The jsonschema package does not have signature definitions checked into typeshed. -ignore_missing_imports = True - diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 425f61dfcdd0..34ab52a8c526 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,131 @@ # Changelog +## 0.51.6 +File-based CDK: allow for extension mismatch + +## 0.51.5 +File-based CDK: Remove CSV noisy log + +## 0.51.4 +Source-S3 V4: feature parity rollout + +## 0.51.3 +File-based CDK: Do not stop processing files in slice on error + +## 0.51.2 +Check config against spec in embedded sources and remove list endpoint from connector builder module + +## 0.51.1 +low-code: allow formatting datetime as milliseconds since unix epoch + +## 0.51.0 +File-based CDK: handle legacy options + +## 0.50.2 +Fix title and description of datetime_format fields + +## 0.50.1 +File-based CDK cursor and entrypoint updates + +## 0.50.0 +Low code CDK: Decouple SimpleRetriever and HttpStream + +## 0.49.0 +Add utils for embedding sources in other Python applications + +## 0.48.0 +Relax pydantic version requirement and update to protocol models version 0.4.0 + +## 0.47.5 +Support many format for cursor datetime + +## 0.47.4 +File-based CDK updates + +## 0.47.3 +Connector Builder: Ensure we return when there are no slices + +## 0.47.2 +low-code: deduplicate query params if they are already encoded in the URL + +## 0.47.1 +Fix RemoveFields transformation issue + +## 0.47.0 +Breaking change: Rename existing SessionTokenAuthenticator to LegacySessionTokenAuthenticator and make SessionTokenAuthenticator more generic + +## 0.46.1 +Connector builder: warn if the max number of records was reached + +## 0.46.0 +Remove pyarrow from main dependency and add it to extras + +## 0.45.0 +Fix pyyaml and cython incompatibility + +## 0.44.4 +Connector builder: Show all request/responses as part of the testing panel + +## 0.44.3 +[ISSUE #27494] allow for state to rely on transformed field + +## 0.44.2 +Ensuring the state value format matches the cursor value from the record + +## 0.44.1 +Fix issue with incremental sync following data feed release + +## 0.44.0 +Support data feed like incremental syncs + +## 0.43.3 +Fix return type of RecordFilter: changed from generator to list + +## 0.43.2 +Connector builder module: serialize request body as string + +## 0.43.1 +Fix availability check to handle HttpErrors which happen during slice extraction + +## 0.43.0 +Refactoring declarative state management + +## 0.42.1 +Error message on state per partition state discrepancy + +## 0.42.0 +Supporting state per partition given incremental sync and partition router + +## 0.41.0 +Use x-www-urlencoded for access token refresh requests + +## 0.40.5 +Replace with when making oauth calls + +## 0.40.4 +Emit messages using message repository + +## 0.40.3 +Add utils for inferring datetime formats + +## 0.40.2 +Add a metadata field to the declarative component schema + +## 0.40.1 +make DatetimeBasedCursor.end_datetime optional + +## 0.40.0 +Remove SingleUseRefreshTokenOAuthAuthenticator from low code CDK and add generic injection capabilities to ApiKeyAuthenticator + +## 0.39.4 +Connector builder: add latest connector config control message to read calls + +## 0.39.3 +Add refresh token update capabilities to OAuthAuthenticator + +## 0.39.2 +Make step and cursor_granularity optional + ## 0.39.1 Improve connector builder error messages diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index cfcac9afe798..db173bb548c5 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.39.1 +RUN pip install --prefix=/install airbyte-cdk==0.51.6 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.39.1 +LABEL io.airbyte.version=0.51.6 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md index e1789af3f6bc..1efd6f48104a 100644 --- a/airbyte-cdk/python/README.md +++ b/airbyte-cdk/python/README.md @@ -63,7 +63,8 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies * Iterate on the code locally * Run tests via `python -m pytest -s unit_tests` -* Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `.mypy.ini`. +* Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`. + * Run `mypy ` to only check specific files. This is useful as the CDK still contains code that is not compliant. * The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! ##### Autogenerated files @@ -120,7 +121,7 @@ To run acceptance tests for a single connectors using the local CDK, from the co ```bash LOCAL_CDK=1 sh acceptance-test-docker.sh ``` -To additionally fetch secrets required by CATs, set the `FETCH_SECRETS` environment variable. This requires you to have a Google Service Account, and the GCP_GSM_CREDENTIALS environment variable to be set, per the instructions [here](https://github.com/airbytehq/airbyte/tree/b03653a24ef16be641333380f3a4d178271df0ee/tools/ci_credentials). +To additionally fetch secrets required by CATs, set the `FETCH_SECRETS` environment variable. This requires you to have a Google Service Account, and the GCP_GSM_CREDENTIALS environment variable to be set, per the instructions [here](https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/ci_credentials). ##### Running Connector Acceptance Tests for multiple connectors in Docker with your local CDK installed @@ -136,7 +137,7 @@ To run acceptance tests for multiple connectors using the local CDK, from the ro ## Coming Soon -* Full OAuth 2.0 support \(including refresh token issuing flow via UI or CLI\) +* Full OAuth 2.0 support \(including refresh token issuing flow via UI or CLI\) * Airbyte Java HTTP CDK * CDK for Async HTTP endpoints \(request-poll-wait style endpoints\) * CDK for other protocols diff --git a/airbyte-cdk/python/airbyte_cdk/config_observation.py b/airbyte-cdk/python/airbyte_cdk/config_observation.py index 3ead7b295945..c7a9b8960962 100644 --- a/airbyte-cdk/python/airbyte_cdk/config_observation.py +++ b/airbyte-cdk/python/airbyte_cdk/config_observation.py @@ -68,10 +68,18 @@ def observe_connector_config(non_observed_connector_config: MutableMapping[str, def emit_configuration_as_airbyte_control_message(config: MutableMapping): + """ + WARNING: deprecated - emit_configuration_as_airbyte_control_message is being deprecated in favor of the MessageRepository mechanism. + See the airbyte_cdk.sources.message package + """ + airbyte_message = create_connector_config_control_message(config) + print(airbyte_message.json(exclude_unset=True)) + + +def create_connector_config_control_message(config): control_message = AirbyteControlMessage( type=OrchestratorType.CONNECTOR_CONFIG, emitted_at=time.time() * 1000, connectorConfig=AirbyteControlConnectorConfigMessage(config=config), ) - airbyte_message = AirbyteMessage(type=Type.CONTROL, control=control_message) - print(airbyte_message.json(exclude_unset=True)) + return AirbyteMessage(type=Type.CONTROL, control=control_message) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md b/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md index 91045414a864..3f3402fab536 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/README.md @@ -11,17 +11,17 @@ python main.py read --config path/to/config --catalog path/to/catalog ``` Note: -- Requires the keys `__injected_declarative_manifest` and `__command` in its config, where `__injected_declarative_manifest` is a JSON manifest and `__command` is one of the commands handled by the ConnectorBuilderHandler (`stream_read`, `list_streams`, or `resolve_manifest`), i.e. +- Requires the keys `__injected_declarative_manifest` and `__command` in its config, where `__injected_declarative_manifest` is a JSON manifest and `__command` is one of the commands handled by the ConnectorBuilderHandler (`stream_read` or `resolve_manifest`), i.e. ``` { "config": , "__injected_declarative_manifest": {...}, - "__command": <"resolve_manifest" | "list_streams" | "test_read"> + "__command": <"resolve_manifest" | "test_read"> } ``` *See [ConnectionSpecification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#actor-specification) for details on the `"config"` key if needed. -- When the `__command` is `list_streams` or `resolve_manifest`, the argument to `catalog` should be an empty string. +- When the `__command` is `resolve_manifest`, the argument to `catalog` should be an empty string. ### Locally running the docker image diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py index 964995fd3eea..c53bda0dfc62 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py @@ -4,18 +4,15 @@ import dataclasses from datetime import datetime -from typing import Any, Dict, List, Mapping -from urllib.parse import urljoin +from typing import Any, Mapping from airbyte_cdk.connector_builder.message_grouper import MessageGrouper from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog from airbyte_cdk.models import Type from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource -from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.utils.traced_exception import AirbyteTracedException DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE = 5 @@ -51,12 +48,14 @@ def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> Manifest emit_connector_builder_messages=True, limit_pages_fetched_per_slice=limits.max_pages_per_slice, limit_slices_fetched=limits.max_slices, - disable_retries=True - ) + disable_retries=True, + ), ) -def read_stream(source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, limits: TestReadLimits) -> AirbyteMessage: +def read_stream( + source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, limits: TestReadLimits +) -> AirbyteMessage: try: handler = MessageGrouper(limits.max_pages_per_slice, limits.max_slices) stream_name = configured_catalog.streams[0].stream.name # The connector builder only supports a single stream @@ -87,38 +86,5 @@ def resolve_manifest(source: ManifestDeclarativeSource) -> AirbyteMessage: return error.as_airbyte_message() -def list_streams(source: ManifestDeclarativeSource, config: Dict[str, Any]) -> AirbyteMessage: - try: - streams = [ - {"name": http_stream.name, "url": urljoin(http_stream.url_base, http_stream.path())} - for http_stream in _get_http_streams(source, config) - ] - return AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - data={"streams": streams}, - emitted_at=_emitted_at(), - stream="list_streams", - ), - ) - except Exception as exc: - return AirbyteTracedException.from_exception(exc, message=f"Error listing streams: {str(exc)}").as_airbyte_message() - - -def _get_http_streams(source: ManifestDeclarativeSource, config: Dict[str, Any]) -> List[HttpStream]: - http_streams = [] - for stream in source.streams(config=config): - if isinstance(stream, DeclarativeStream): - if isinstance(stream.retriever, HttpStream): - http_streams.append(stream.retriever) - else: - raise TypeError( - f"A declarative stream should only have a retriever of type HttpStream, but received: {stream.retriever.__class__}" - ) - else: - raise TypeError(f"A declarative source should only contain streams of type DeclarativeStream, but received: {stream.__class__}") - return http_streams - - -def _emitted_at(): +def _emitted_at() -> int: return int(datetime.now().timestamp()) * 1000 diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py index ea1190677416..c26e5292d7f9 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py @@ -7,16 +7,9 @@ from typing import Any, List, Mapping, Optional, Tuple from airbyte_cdk.connector import BaseConnector -from airbyte_cdk.connector_builder.connector_builder_handler import ( - TestReadLimits, - create_source, - get_limits, - list_streams, - read_stream, - resolve_manifest, -) +from airbyte_cdk.connector_builder.connector_builder_handler import TestReadLimits, create_source, get_limits, read_stream, resolve_manifest from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -50,19 +43,17 @@ def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, def handle_connector_builder_request( source: ManifestDeclarativeSource, command: str, config: Mapping[str, Any], catalog: Optional[ConfiguredAirbyteCatalog], limits: TestReadLimits -): +) -> AirbyteMessage: if command == "resolve_manifest": return resolve_manifest(source) elif command == "test_read": assert catalog is not None, "`test_read` requires a valid `ConfiguredAirbyteCatalog`, got None." return read_stream(source, config, catalog, limits) - elif command == "list_streams": - return list_streams(source, config) else: raise ValueError(f"Unrecognized command {command}.") -def handle_request(args: List[str]): +def handle_request(args: List[str]) -> AirbyteMessage: command, config, catalog = get_config_and_catalog_from_args(args) limits = get_limits(config) source = create_source(config, limits) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index 669588927ba7..b787fe5d43c9 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -9,18 +9,29 @@ from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Union from urllib.parse import parse_qs, urlparse -from airbyte_cdk.connector_builder.models import HttpRequest, HttpResponse, LogMessage, StreamRead, StreamReadPages, StreamReadSlices +from airbyte_cdk.connector_builder.models import ( + AuxiliaryRequest, + HttpRequest, + HttpResponse, + LogMessage, + StreamRead, + StreamReadPages, + StreamReadSlices, +) from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource +from airbyte_cdk.sources.utils.types import JsonType from airbyte_cdk.utils import AirbyteTracedException +from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer from airbyte_cdk.utils.schema_inferrer import SchemaInferrer from airbyte_protocol.models.airbyte_protocol import ( + AirbyteControlMessage, AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage, ConfiguredAirbyteCatalog, - Level, + OrchestratorType, TraceType, ) from airbyte_protocol.models.airbyte_protocol import Type as MessageType @@ -44,6 +55,7 @@ def get_message_groups( if record_limit is not None and not (1 <= record_limit <= 1000): raise ValueError(f"Record limit must be between 1 and 1000. Got {record_limit}") schema_inferrer = SchemaInferrer() + datetime_format_inferrer = DatetimeFormatInferrer() if record_limit is None: record_limit = self._max_record_limit @@ -52,9 +64,12 @@ def get_message_groups( slices = [] log_messages = [] + latest_config_update: AirbyteControlMessage = None + auxiliary_requests = [] for message_group in self._get_message_groups( self._read_stream(source, config, configured_catalog), schema_inferrer, + datetime_format_inferrer, record_limit, ): if isinstance(message_group, AirbyteLogMessage): @@ -63,22 +78,31 @@ def get_message_groups( if message_group.type == TraceType.ERROR: error_message = f"{message_group.error.message} - {message_group.error.stack_trace}" log_messages.append(LogMessage(**{"message": error_message, "level": "ERROR"})) - - else: + elif isinstance(message_group, AirbyteControlMessage): + if not latest_config_update or latest_config_update.emitted_at <= message_group.emitted_at: + latest_config_update = message_group + elif isinstance(message_group, AuxiliaryRequest): + auxiliary_requests.append(message_group) + elif isinstance(message_group, StreamReadSlices): slices.append(message_group) + else: + raise ValueError(f"Unknown message group type: {type(message_group)}") return StreamRead( logs=log_messages, slices=slices, test_read_limit_reached=self._has_reached_limit(slices), + auxiliary_requests=auxiliary_requests, inferred_schema=schema_inferrer.get_stream_schema( configured_catalog.streams[0].stream.name ), # The connector builder currently only supports reading from a single stream at a time + latest_config_update=self._clean_config(latest_config_update.connectorConfig.config) if latest_config_update else None, + inferred_datetime_formats=datetime_format_inferrer.get_inferred_datetime_formats(), ) def _get_message_groups( - self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, limit: int - ) -> Iterable[Union[StreamReadPages, AirbyteLogMessage, AirbyteTraceMessage]]: + self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, datetime_format_inferrer: DatetimeFormatInferrer, limit: int + ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage, AuxiliaryRequest]]: """ Message groups are partitioned according to when request log messages are received. Subsequent response log messages and record messages belong to the prior request log message and when we encounter another request, append the latest @@ -96,16 +120,19 @@ def _get_message_groups( """ records_count = 0 at_least_one_page_in_group = False - current_page_records = [] - current_slice_descriptor: Dict[str, Any] = None - current_slice_pages = [] + current_page_records: List[Mapping[str, Any]] = [] + current_slice_descriptor: Optional[Dict[str, Any]] = None + current_slice_pages: List[StreamReadPages] = [] current_page_request: Optional[HttpRequest] = None current_page_response: Optional[HttpResponse] = None - had_error = False while records_count < limit and (message := next(messages, None)): - if self._need_to_close_page(at_least_one_page_in_group, message): - self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, True) + json_object = self._parse_json(message.log) if message.type == MessageType.LOG else None + if json_object is not None and not isinstance(json_object, dict): + raise ValueError(f"Expected log message to be a dict, got {json_object} of type {type(json_object)}") + json_message: Optional[Dict[str, JsonType]] = json_object + if self._need_to_close_page(at_least_one_page_in_group, message, json_message): + self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records) current_page_request = None current_page_response = None @@ -117,48 +144,88 @@ def _get_message_groups( elif message.type == MessageType.LOG and message.log.message.startswith(AbstractSource.SLICE_LOG_PREFIX): # parsing the first slice current_slice_descriptor = self._parse_slice_description(message.log.message) - elif message.type == MessageType.LOG and message.log.message.startswith("request:"): - if not at_least_one_page_in_group: - at_least_one_page_in_group = True - current_page_request = self._create_request_from_log_message(message.log) - elif message.type == MessageType.LOG and message.log.message.startswith("response:"): - current_page_response = self._create_response_from_log_message(message.log) elif message.type == MessageType.LOG: - if message.log.level == Level.ERROR: - had_error = True - yield message.log + if json_message is not None and self._is_http_log(json_message): + if self._is_auxiliary_http_request(json_message): + airbyte_cdk = json_message.get("airbyte_cdk", {}) + if not isinstance(airbyte_cdk, dict): + raise ValueError(f"Expected airbyte_cdk to be a dict, got {airbyte_cdk} of type {type(airbyte_cdk)}") + stream = airbyte_cdk.get("stream", {}) + if not isinstance(stream, dict): + raise ValueError(f"Expected stream to be a dict, got {stream} of type {type(stream)}") + title_prefix = ( + "Parent stream: " if stream.get("is_substream", False) else "" + ) + http = json_message.get("http", {}) + if not isinstance(http, dict): + raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}") + yield AuxiliaryRequest( + title=title_prefix + str(http.get("title", None)), + description=str(http.get("description", None)), + request=self._create_request_from_log_message(json_message), + response=self._create_response_from_log_message(json_message), + ) + else: + at_least_one_page_in_group = True + current_page_request = self._create_request_from_log_message(json_message) + current_page_response = self._create_response_from_log_message(json_message) + else: + yield message.log elif message.type == MessageType.TRACE: if message.trace.type == TraceType.ERROR: - had_error = True yield message.trace elif message.type == MessageType.RECORD: current_page_records.append(message.record.data) records_count += 1 schema_inferrer.accumulate(message.record) + datetime_format_inferrer.accumulate(message.record) + elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG: + yield message.control else: - self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, validate_page_complete=not had_error) - yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor) + if current_page_request or current_page_response or current_page_records: + self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records) + yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor) @staticmethod - def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessage) -> bool: + def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessage, json_message: Optional[Dict[str, Any]]) -> bool: return ( at_least_one_page_in_group and message.type == MessageType.LOG - and (message.log.message.startswith("request:") or message.log.message.startswith("slice:")) + and (MessageGrouper._is_page_http_request(json_message) or message.log.message.startswith("slice:")) ) @staticmethod - def _close_page(current_page_request, current_page_response, current_slice_pages, current_page_records, validate_page_complete: bool): + def _is_page_http_request(json_message: Optional[Dict[str, Any]]) -> bool: + if not json_message: + return False + else: + return MessageGrouper._is_http_log(json_message) and not MessageGrouper._is_auxiliary_http_request(json_message) + + @staticmethod + def _is_http_log(message: Dict[str, JsonType]) -> bool: + return bool(message.get("http", False)) + + @staticmethod + def _is_auxiliary_http_request(message: Optional[Dict[str, Any]]) -> bool: """ - Close a page when parsing message groups - @param validate_page_complete: in some cases, we expect the CDK to not return a response. As of today, this will only happen before - an uncaught exception and therefore, the assumption is that `validate_page_complete=True` only on the last page that is being closed + A auxiliary request is a request that is performed and will not directly lead to record for the specific stream it is being queried. + A couple of examples are: + * OAuth authentication + * Substream slice generation """ - if validate_page_complete and (not current_page_request or not current_page_response): - raise ValueError("Every message grouping should have at least one request and response") + if not message: + return False + + is_http = MessageGrouper._is_http_log(message) + return is_http and message.get("http", {}).get("is_auxiliary", False) + @staticmethod + def _close_page(current_page_request: Optional[HttpRequest], current_page_response: Optional[HttpResponse], current_slice_pages: List[StreamReadPages], current_page_records: List[Mapping[str, Any]]) -> None: + """ + Close a page when parsing message groups + """ current_slice_pages.append( - StreamReadPages(request=current_page_request, response=current_page_response, records=deepcopy(current_page_records)) + StreamReadPages(request=current_page_request, response=current_page_response, records=deepcopy(current_page_records)) # type: ignore ) current_page_records.clear() @@ -171,66 +238,59 @@ def _read_stream(self, source: DeclarativeSource, config: Mapping[str, Any], con error_message = f"{e.args[0] if len(e.args) > 0 else str(e)}" yield AirbyteTracedException.from_exception(e, message=error_message).as_airbyte_message() - def _create_request_from_log_message(self, log_message: AirbyteLogMessage) -> Optional[HttpRequest]: - # TODO: As a temporary stopgap, the CDK emits request data as a log message string. Ideally this should come in the + @staticmethod + def _parse_json(log_message: AirbyteLogMessage) -> JsonType: + # TODO: As a temporary stopgap, the CDK emits request/response data as a log message string. Ideally this should come in the # form of a custom message object defined in the Airbyte protocol, but this unblocks us in the immediate while the # protocol change is worked on. - raw_request = log_message.message.partition("request:")[2] try: - request = json.loads(raw_request) - url = urlparse(request.get("url", "")) - full_path = f"{url.scheme}://{url.hostname}{url.path}" if url else "" - parameters = parse_qs(url.query) or None - return HttpRequest( - url=full_path, - http_method=request.get("http_method", ""), - headers=request.get("headers"), - parameters=parameters, - body=request.get("body"), - ) - except JSONDecodeError as error: - self.logger.warning(f"Failed to parse log message into request object with error: {error}") + json_object: JsonType = json.loads(log_message.message) + return json_object + except JSONDecodeError: return None - def _create_response_from_log_message(self, log_message: AirbyteLogMessage) -> Optional[HttpResponse]: - # TODO: As a temporary stopgap, the CDK emits response data as a log message string. Ideally this should come in the - # form of a custom message object defined in the Airbyte protocol, but this unblocks us in the immediate while the - # protocol change is worked on. - raw_response = log_message.message.partition("response:")[2] - try: - response = json.loads(raw_response) - body = response.get("body", "{}") - return HttpResponse(status=response.get("status_code"), body=body, headers=response.get("headers")) - except JSONDecodeError as error: - self.logger.warning(f"Failed to parse log message into response object with error: {error}") - return None + @staticmethod + def _create_request_from_log_message(json_http_message: Dict[str, Any]) -> HttpRequest: + url = urlparse(json_http_message.get("url", {}).get("full", "")) + full_path = f"{url.scheme}://{url.hostname}{url.path}" if url else "" + request = json_http_message.get("http", {}).get("request", {}) + parameters = parse_qs(url.query) or None + return HttpRequest( + url=full_path, + http_method=request.get("method", ""), + headers=request.get("headers"), + parameters=parameters, + body=request.get("body", {}).get("content", ""), + ) + + @staticmethod + def _create_response_from_log_message(json_http_message: Dict[str, Any]) -> HttpResponse: + response = json_http_message.get("http", {}).get("response", {}) + body = response.get("body", {}).get("content", "") + return HttpResponse(status=response.get("status_code"), body=body, headers=response.get("headers")) - def _has_reached_limit(self, slices: List[StreamReadPages]): + def _has_reached_limit(self, slices: List[StreamReadSlices]) -> bool: if len(slices) >= self._max_slices: return True - for slice in slices: - if len(slice.pages) >= self._max_pages_per_slice: + record_count = 0 + + for _slice in slices: + if len(_slice.pages) >= self._max_pages_per_slice: return True + for page in _slice.pages: + record_count += len(page.records) + if record_count >= self._max_record_limit: + return True return False - def _parse_slice_description(self, log_message): - return json.loads(log_message.replace(AbstractSource.SLICE_LOG_PREFIX, "", 1)) - - @classmethod - def _create_configure_catalog(cls, stream_name: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog.parse_obj( - { - "streams": [ - { - "stream": { - "name": stream_name, - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - } - ] - } - ) + def _parse_slice_description(self, log_message: str) -> Dict[str, Any]: + return json.loads(log_message.replace(AbstractSource.SLICE_LOG_PREFIX, "", 1)) # type: ignore + + @staticmethod + def _clean_config(config: Dict[str, Any]) -> Dict[str, Any]: + cleaned_config = deepcopy(config) + for key in config.keys(): + if key.startswith("__"): + del cleaned_config[key] + return cleaned_config diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/models.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/models.py index 0f9e3f8f5bf0..06cbe215e447 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/models.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/models.py @@ -17,9 +17,9 @@ class HttpResponse: class HttpRequest: url: str parameters: Optional[Dict[str, Any]] - body: Optional[Dict[str, Any]] headers: Optional[Dict[str, Any]] http_method: str + body: Optional[str] = None @dataclass @@ -32,7 +32,7 @@ class StreamReadPages: @dataclass class StreamReadSlices: pages: List[StreamReadPages] - slice_descriptor: Dict[str, Any] + slice_descriptor: Optional[Dict[str, Any]] state: Optional[Dict[str, Any]] = None @@ -42,12 +42,23 @@ class LogMessage: level: str +@dataclass +class AuxiliaryRequest: + title: str + description: str + request: HttpRequest + response: HttpResponse + + @dataclass class StreamRead(object): logs: List[LogMessage] slices: List[StreamReadSlices] test_read_limit_reached: bool + auxiliary_requests: List[AuxiliaryRequest] inferred_schema: Optional[Dict[str, Any]] + inferred_datetime_formats: Optional[Dict[str, str]] + latest_config_update: Optional[Dict[str, Any]] @dataclass diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index 40604d4be95b..3590d48bded1 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -2,32 +2,45 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import argparse import importlib +import ipaddress import logging import os.path +import socket import sys import tempfile -from typing import Any, Iterable, List, Mapping +from functools import wraps +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from urllib.parse import urlparse from airbyte_cdk.connector import TConfig from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.logger import init_logger from airbyte_cdk.models import AirbyteMessage, Status, Type -from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification # type: ignore [attr-defined] from airbyte_cdk.sources import Source -from airbyte_cdk.sources.source import TCatalog, TState from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from requests import PreparedRequest, Response, Session logger = init_logger("airbyte") +VALID_URL_SCHEMES = ["https"] +CLOUD_DEPLOYMENT_MODE = "cloud" + class AirbyteEntrypoint(object): def __init__(self, source: Source): init_uncaught_exception_handler(logger) + + # DEPLOYMENT_MODE is read when instantiating the entrypoint because it is the common path shared by syncs and connector + # builder test requests + deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") + if deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE: + _init_internal_request_filter() + self.source = source self.logger = logging.getLogger(f"airbyte.{getattr(source, 'name', '')}") @@ -77,27 +90,32 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: else: self.logger.setLevel(logging.INFO) - # todo: add try catch for exceptions with different exit codes source_spec: ConnectorSpecification = self.source.spec(self.logger) - with tempfile.TemporaryDirectory() as temp_dir: - if cmd == "spec": - message = AirbyteMessage(type=Type.SPEC, spec=source_spec) - yield message.json(exclude_unset=True) - else: - raw_config = self.source.read_config(parsed_args.config) - config = self.source.configure(raw_config, temp_dir) - - if cmd == "check": - yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config)) - elif cmd == "discover": - yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config)) - elif cmd == "read": - config_catalog = self.source.read_catalog(parsed_args.catalog) - state = self.source.read_state(parsed_args.state) - - yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state)) + try: + with tempfile.TemporaryDirectory() as temp_dir: + if cmd == "spec": + message = AirbyteMessage(type=Type.SPEC, spec=source_spec) + yield from [ + self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source) + ] + yield self.airbyte_message_to_string(message) else: - raise Exception("Unexpected command " + cmd) + raw_config = self.source.read_config(parsed_args.config) + config = self.source.configure(raw_config, temp_dir) + + if cmd == "check": + yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config)) + elif cmd == "discover": + yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.discover(source_spec, config)) + elif cmd == "read": + config_catalog = self.source.read_catalog(parsed_args.catalog) + state = self.source.read_state(parsed_args.state) + + yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.read(source_spec, config, config_catalog, state)) + else: + raise Exception("Unexpected command " + cmd) + finally: + yield from [self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source)] def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]: self.set_up_secret_filter(config, source_spec.connectionSpecification) @@ -106,6 +124,7 @@ def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterabl except AirbyteTracedException as traced_exc: connection_status = traced_exc.as_connection_status_message() if connection_status: + yield from self._emit_queued_messages(self.source) yield connection_status return @@ -115,6 +134,7 @@ def check(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterabl else: self.logger.error("Check failed") + yield from self._emit_queued_messages(self.source) yield AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result) def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iterable[AirbyteMessage]: @@ -122,42 +142,117 @@ def discover(self, source_spec: ConnectorSpecification, config: TConfig) -> Iter if self.source.check_config_against_spec: self.validate_connection(source_spec, config) catalog = self.source.discover(self.logger, config) + + yield from self._emit_queued_messages(self.source) yield AirbyteMessage(type=Type.CATALOG, catalog=catalog) - def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: TCatalog, state: TState) -> Iterable[AirbyteMessage]: + def read( + self, source_spec: ConnectorSpecification, config: TConfig, catalog: Any, state: Union[list[Any], MutableMapping[str, Any]] + ) -> Iterable[AirbyteMessage]: self.set_up_secret_filter(config, source_spec.connectionSpecification) if self.source.check_config_against_spec: self.validate_connection(source_spec, config) yield from self.source.read(self.logger, config, catalog, state) + yield from self._emit_queued_messages(self.source) @staticmethod - def validate_connection(source_spec: ConnectorSpecification, config: Mapping[str, Any]) -> None: + def validate_connection(source_spec: ConnectorSpecification, config: TConfig) -> None: # Remove internal flags from config before validating so # jsonschema's additionalProperties flag won't fail the validation connector_config, _ = split_config(config) check_config_against_spec_or_exit(connector_config, source_spec) @staticmethod - def set_up_secret_filter(config, connection_specification: Mapping[str, Any]): + def set_up_secret_filter(config: TConfig, connection_specification: Mapping[str, Any]) -> None: # Now that we have the config, we can use it to get a list of ai airbyte_secrets # that we should filter in logging to avoid leaking secrets config_secrets = get_secrets(connection_specification, config) update_secrets(config_secrets) @staticmethod - def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str: + def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> Any: return airbyte_message.json(exclude_unset=True) + @classmethod + def extract_catalog(cls, args: List[str]) -> Optional[Any]: + parsed_args = cls.parse_args(args) + if hasattr(parsed_args, "catalog"): + return parsed_args.catalog + return None + + @classmethod + def extract_config(cls, args: List[str]) -> Optional[Any]: + parsed_args = cls.parse_args(args) + if hasattr(parsed_args, "config"): + return parsed_args.config + return None -def launch(source: Source, args: List[str]): + def _emit_queued_messages(self, source: Source) -> Iterable[AirbyteMessage]: + if hasattr(source, "message_repository") and source.message_repository: + yield from source.message_repository.consume_queue() + return + + +def launch(source: Source, args: List[str]) -> None: source_entrypoint = AirbyteEntrypoint(source) parsed_args = source_entrypoint.parse_args(args) for message in source_entrypoint.run(parsed_args): print(message) -def main(): +def _init_internal_request_filter() -> None: + """ + Wraps the Python requests library to prevent sending requests to internal URL endpoints. + """ + wrapped_fn = Session.send + + @wraps(wrapped_fn) + def filtered_send(self: Any, request: PreparedRequest, **kwargs: Any) -> Response: + parsed_url = urlparse(request.url) + + if parsed_url.scheme not in VALID_URL_SCHEMES: + raise ValueError( + "Invalid Protocol Scheme: The endpoint that data is being requested from is using an invalid or insecure " + + f"protocol {parsed_url.scheme!r}. Valid protocol schemes: {','.join(VALID_URL_SCHEMES)}" + ) + + if not parsed_url.hostname: + raise ValueError("Invalid URL specified: The endpoint that data is being requested from is not a valid URL") + + try: + is_private = _is_private_url(parsed_url.hostname, parsed_url.port) # type: ignore [arg-type] + if is_private: + raise ValueError( + "Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source " + + "connectors only support requesting data from public API endpoints." + ) + except socket.gaierror: + # This is a special case where the developer specifies an IP address string that is not formatted correctly like trailing + # whitespace which will fail the socket IP lookup. This only happens when using IP addresses and not text hostnames. + raise ValueError(f"Invalid hostname or IP address '{parsed_url.hostname!r}' specified.") + + return wrapped_fn(self, request, **kwargs) + + Session.send = filtered_send # type: ignore [method-assign] + + +def _is_private_url(hostname: str, port: int) -> bool: + """ + Helper method that checks if any of the IP addresses associated with a hostname belong to a private network. + """ + address_info_entries = socket.getaddrinfo(hostname, port) + for entry in address_info_entries: + # getaddrinfo() returns entries in the form of a 5-tuple where the IP is stored as the sockaddr. For IPv4 this + # is a 2-tuple and for IPv6 it is a 4-tuple, but the address is always the first value of the tuple at 0. + # See https://docs.python.org/3/library/socket.html#socket.getaddrinfo for more details. + ip_address = entry[4][0] + if ipaddress.ip_address(ip_address).is_private: + return True + return False + + +def main() -> None: impl_module = os.environ.get("AIRBYTE_IMPL_MODULE", Source.__module__) impl_class = os.environ.get("AIRBYTE_IMPL_PATH", Source.__name__) module = importlib.import_module(impl_module) diff --git a/airbyte-cdk/python/airbyte_cdk/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/models/__init__.py index e3127e1d519a..9545af7b044c 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/models/__init__.py @@ -5,5 +5,55 @@ # are just wrappers on top of that stand-alone package which do some namespacing magic # to make the airbyte_protocol python classes available to the airbyte-cdk consumer as part # of airbyte-cdk rather than a standalone package. -from .airbyte_protocol import * -from .well_known_types import * +from .airbyte_protocol import ( + AdvancedAuth, + AirbyteCatalog, + AirbyteConnectionStatus, + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteErrorTraceMessage, + AirbyteEstimateTraceMessage, + AirbyteGlobalState, + AirbyteLogMessage, + AirbyteMessage, + AirbyteProtocol, + AirbyteRecordMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStream, + AirbyteStreamState, + AirbyteStreamStatus, + AirbyteStreamStatusTraceMessage, + AirbyteTraceMessage, + AuthFlowType, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + EstimateType, + FailureType, + Level, + OAuthConfigSpecification, + OrchestratorType, + Status, + StreamDescriptor, + SyncMode, + TraceType, + Type, +) +from .well_known_types import ( + BinaryData, + Boolean, + Date, + Integer, + IntegerEnum, + Model, + Number, + NumberEnum, + String, + TimestampWithoutTimezone, + TimestampWithTimezone, + TimeWithoutTimezone, + TimeWithTimezone, +) diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index e634c18acd69..74639c8bf3c1 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -2,4 +2,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_protocol.models.airbyte_protocol import * +from airbyte_protocol.models import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index c6fc000b149d..68d804666f42 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -22,6 +22,7 @@ ) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.source import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.core import StreamData @@ -109,12 +110,13 @@ def read( f"The requested stream {configured_stream.stream.name} was not found in the source." f" Available streams: {stream_instances.keys()}" ) - stream_is_available, error = stream_instance.check_availability(logger, self) - if not stream_is_available: - logger.warning(f"Skipped syncing stream '{stream_instance.name}' because it was unavailable. Error: {error}") - continue + try: timer.start_event(f"Syncing stream {configured_stream.stream.name}") + stream_is_available, reason = stream_instance.check_availability(logger, self) + if not stream_is_available: + logger.warning(f"Skipped syncing stream '{stream_instance.name}' because it was unavailable. {reason}") + continue logger.info(f"Marking stream {configured_stream.stream.name} as STARTED") yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.STARTED) yield from self._read_stream( @@ -130,6 +132,7 @@ def read( yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) raise e except Exception as e: + yield from self._emit_queued_messages() logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) @@ -198,6 +201,7 @@ def _read_stream( logger.info(f"Marking stream {stream_name} as RUNNING") # If we just read the first record of the stream, emit the transition to the RUNNING state yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING) + yield from self._emit_queued_messages() yield record logger.info(f"Read {record_counter} records from {stream_name} stream") @@ -251,10 +255,7 @@ def _read_incremental( for _slice in slices: has_slices = True if self.should_log_slice_message(logger): - yield AirbyteMessage( - type=MessageType.LOG, - log=AirbyteLogMessage(level=Level.INFO, message=f"{self.SLICE_LOG_PREFIX}{json.dumps(_slice, default=str)}"), - ) + yield self._create_slice_log_message(_slice) records = stream_instance.read_records( sync_mode=SyncMode.incremental, stream_slice=_slice, @@ -264,6 +265,7 @@ def _read_incremental( record_counter = 0 for message_counter, record_data_or_message in enumerate(records, start=1): message = self._get_message(record_data_or_message, stream_instance) + yield from self._emit_queued_messages() yield message if message.type == MessageType.RECORD: record = message.record @@ -298,6 +300,11 @@ def should_log_slice_message(self, logger: logging.Logger): """ return logger.isEnabledFor(logging.DEBUG) + def _emit_queued_messages(self): + if self.message_repository: + yield from self.message_repository.consume_queue() + return + def _read_full_refresh( self, logger: logging.Logger, @@ -312,10 +319,7 @@ def _read_full_refresh( total_records_counter = 0 for _slice in slices: if self.should_log_slice_message(logger): - yield AirbyteMessage( - type=MessageType.LOG, - log=AirbyteLogMessage(level=Level.INFO, message=f"{self.SLICE_LOG_PREFIX}{json.dumps(_slice, default=str)}"), - ) + yield self._create_slice_log_message(_slice) record_data_or_messages = stream_instance.read_records( stream_slice=_slice, sync_mode=SyncMode.full_refresh, @@ -329,6 +333,17 @@ def _read_full_refresh( if self._limit_reached(internal_config, total_records_counter): return + def _create_slice_log_message(self, _slice: Optional[Mapping[str, Any]]) -> AirbyteMessage: + """ + Mapping is an interface that can be implemented in various ways. However, json.dumps will just do a `str()` if + the slice is a class implementing Mapping. Therefore, we want to cast this as a dict before passing this to json.dump + """ + printable_slice = dict(_slice) if _slice else _slice + return AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage(level=Level.INFO, message=f"{self.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}"), + ) + def _checkpoint_state(self, stream: Stream, stream_state, state_manager: ConnectorStateManager): # First attempt to retrieve the current state using the stream's state property. We receive an AttributeError if the state # property is not implemented by the stream instance and as a fallback, use the stream_state retrieved from the stream @@ -357,3 +372,7 @@ def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage] return record_data_or_message else: return stream_data_to_airbyte_message(stream.name, record_data_or_message, stream.transformer, stream.get_json_schema()) + + @property + def message_repository(self) -> Union[None, MessageRepository]: + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/declarative_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/declarative_authenticator.py index 83c293d6cf16..5517f546209a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/declarative_authenticator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/declarative_authenticator.py @@ -3,20 +3,32 @@ # from dataclasses import InitVar, dataclass -from typing import Any, Mapping +from typing import Any, Mapping, Union from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator @dataclass -class DeclarativeAuthenticator: +class DeclarativeAuthenticator(AbstractHeaderAuthenticator): """ Interface used to associate which authenticators can be used as part of the declarative framework """ + def get_request_params(self) -> Mapping[str, Any]: + """HTTP request parameter to add to the requests""" + return {} + + def get_request_body_data(self) -> Union[Mapping[str, Any], str]: + """Form-encoded body data to set on the requests""" + return {} + + def get_request_body_json(self) -> Mapping[str, Any]: + """JSON-encoded body data to set on the requests""" + return {} + @dataclass -class NoAuth(AbstractHeaderAuthenticator, DeclarativeAuthenticator): +class NoAuth(DeclarativeAuthenticator): parameters: InitVar[Mapping[str, Any]] @property diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index 939aee4b1539..a7621693f0a2 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -9,7 +9,9 @@ from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator +from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator @dataclass @@ -32,6 +34,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request grant_type: The grant_type to request for access_token. If set to refresh_token, the refresh_token parameter has to be provided + message_repository (MessageRepository): the message repository used to emit logs on HTTP requests """ token_refresh_endpoint: Union[InterpolatedString, str] @@ -48,6 +51,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut expires_in_name: Union[InterpolatedString, str] = "expires_in" refresh_request_body: Optional[Mapping[str, Any]] = None grant_type: Union[InterpolatedString, str] = "refresh_token" + message_repository: MessageRepository = NoopMessageRepository() def __post_init__(self, parameters: Mapping[str, Any]): self.token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, parameters=parameters) @@ -133,3 +137,20 @@ def access_token(self) -> str: @access_token.setter def access_token(self, value: str): self._access_token = value + + @property + def _message_repository(self) -> MessageRepository: + """ + Overriding AbstractOauth2Authenticator._message_repository to allow for HTTP request logs + """ + return self.message_repository + + +@dataclass +class DeclarativeSingleUseRefreshTokenOauth2Authenticator(SingleUseRefreshTokenOauth2Authenticator, DeclarativeAuthenticator): + """ + Declarative version of SingleUseRefreshTokenOauth2Authenticator which can be used in declarative connectors. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py index b2e8c48daaa6..d0046c3d8b64 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py @@ -9,14 +9,15 @@ import requests from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator +from airbyte_cdk.sources.declarative.auth.token_provider import TokenProvider from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.types import Config -from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator from cachetools import TTLCache, cached @dataclass -class ApiKeyAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator): +class ApiKeyAuthenticator(DeclarativeAuthenticator): """ ApiKeyAuth sets a request header on the HTTP requests sent. @@ -29,32 +30,47 @@ class ApiKeyAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator) `"Authorization": "Bearer hello"` Attributes: - header (Union[InterpolatedString, str]): Header key to set on the HTTP requests - api_token (Union[InterpolatedString, str]): Header value to set on the HTTP requests + request_option (RequestOption): request option how to inject the token into the request + token_provider (TokenProvider): Provider of the token config (Config): The user-provided configuration as specified by the source's spec parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - header: Union[InterpolatedString, str] - api_token: Union[InterpolatedString, str] + request_option: RequestOption + token_provider: TokenProvider config: Config parameters: InitVar[Mapping[str, Any]] - def __post_init__(self, parameters: Mapping[str, Any]): - self._header = InterpolatedString.create(self.header, parameters=parameters) - self._token = InterpolatedString.create(self.api_token, parameters=parameters) + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._field_name = InterpolatedString.create(self.request_option.field_name, parameters=parameters) @property def auth_header(self) -> str: - return self._header.eval(self.config) + options = self._get_request_options(RequestOptionType.header) + return next(iter(options.keys()), "") @property def token(self) -> str: - return self._token.eval(self.config) + return self.token_provider.get_token() + + def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]: + options = {} + if self.request_option.inject_into == option_type: + options[self._field_name.eval(self.config)] = self.token + return options + + def get_request_params(self) -> Mapping[str, Any]: + return self._get_request_options(RequestOptionType.request_parameter) + + def get_request_body_data(self) -> Union[Mapping[str, Any], str]: + return self._get_request_options(RequestOptionType.body_data) + + def get_request_body_json(self) -> Mapping[str, Any]: + return self._get_request_options(RequestOptionType.body_json) @dataclass -class BearerAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator): +class BearerAuthenticator(DeclarativeAuthenticator): """ Authenticator that sets the Authorization header on the HTTP requests sent. @@ -62,29 +78,26 @@ class BearerAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator) `"Authorization": "Bearer "` Attributes: - api_token (Union[InterpolatedString, str]): The bearer token + token_provider (TokenProvider): Provider of the token config (Config): The user-provided configuration as specified by the source's spec parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - api_token: Union[InterpolatedString, str] + token_provider: TokenProvider config: Config parameters: InitVar[Mapping[str, Any]] - def __post_init__(self, parameters: Mapping[str, Any]): - self._token = InterpolatedString.create(self.api_token, parameters=parameters) - @property def auth_header(self) -> str: return "Authorization" @property def token(self) -> str: - return f"Bearer {self._token.eval(self.config)}" + return f"Bearer {self.token_provider.get_token()}" @dataclass -class BasicHttpAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator): +class BasicHttpAuthenticator(DeclarativeAuthenticator): """ Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using base64 https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme @@ -104,7 +117,7 @@ class BasicHttpAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticat parameters: InitVar[Mapping[str, Any]] password: Union[InterpolatedString, str] = "" - def __post_init__(self, parameters): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._username = InterpolatedString.create(self.username, parameters=parameters) self._password = InterpolatedString.create(self.password, parameters=parameters) @@ -127,7 +140,7 @@ def token(self) -> str: i.e. by adding another item the cache would exceed its maximum size, the cache must choose which item(s) to discard ttl=86400 means that cached token will live for 86400 seconds (one day) """ -cacheSessionTokenAuthenticator = TTLCache(maxsize=1000, ttl=86400) +cacheSessionTokenAuthenticator: TTLCache[str, str] = TTLCache(maxsize=1000, ttl=86400) @cached(cacheSessionTokenAuthenticator) @@ -152,11 +165,11 @@ def get_new_session_token(api_url: str, username: str, password: str, response_k response.raise_for_status() if not response.ok: raise ConnectionError(f"Failed to retrieve new session token, response code {response.status_code} because {response.reason}") - return response.json()[response_key] + return str(response.json()[response_key]) @dataclass -class SessionTokenAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator): +class LegacySessionTokenAuthenticator(DeclarativeAuthenticator): """ Builds auth based on session tokens. A session token is a random value generated by a server to identify @@ -189,7 +202,7 @@ class SessionTokenAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenti validate_session_url: Union[InterpolatedString, str] password: Union[InterpolatedString, str] = "" - def __post_init__(self, parameters): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._username = InterpolatedString.create(self.username, parameters=parameters) self._password = InterpolatedString.create(self.password, parameters=parameters) self._api_url = InterpolatedString.create(self.api_url, parameters=parameters) @@ -203,13 +216,13 @@ def __post_init__(self, parameters): @property def auth_header(self) -> str: - return self._header.eval(self.config) + return str(self._header.eval(self.config)) @property def token(self) -> str: if self._session_token.eval(self.config): if self.is_valid_session_token(): - return self._session_token.eval(self.config) + return str(self._session_token.eval(self.config)) if self._password.eval(self.config) and self._username.eval(self.config): username = self._username.eval(self.config) password = self._password.eval(self.config) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token_provider.py new file mode 100644 index 000000000000..52383d2d1b59 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token_provider.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import datetime +from abc import abstractmethod +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Optional, Union + +import dpath.util +import pendulum +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.exceptions import ReadException +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.requester import Requester +from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.http_logger import format_http_message +from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository +from isodate import Duration +from pendulum import DateTime + + +class TokenProvider: + @abstractmethod + def get_token(self) -> str: + pass + + +@dataclass +class SessionTokenProvider(TokenProvider): + login_requester: Requester + session_token_path: List[str] + expiration_duration: Optional[Union[datetime.timedelta, Duration]] + parameters: InitVar[Mapping[str, Any]] + message_repository: MessageRepository = NoopMessageRepository() + + _decoder: Decoder = JsonDecoder(parameters={}) + _next_expiration_time: Optional[DateTime] = None + _token: Optional[str] = None + + def get_token(self) -> str: + self._refresh_if_necessary() + if self._token is None: + raise ReadException("Failed to get session token, token is None") + return self._token + + def _refresh_if_necessary(self) -> None: + if self._next_expiration_time is None or self._next_expiration_time < pendulum.now(): + self._refresh() + + def _refresh(self) -> None: + response = self.login_requester.send_request( + log_formatter=lambda response: format_http_message( + response, + "Login request", + "Obtains session token", + None, + is_auxiliary=True, + ), + ) + if response is None: + raise ReadException("Failed to get session token, response got ignored by requester") + session_token = dpath.util.get(self._decoder.decode(response), self.session_token_path) + if self.expiration_duration is not None: + self._next_expiration_time = pendulum.now() + self.expiration_duration + self._token = session_token + + +@dataclass +class InterpolatedStringTokenProvider(TokenProvider): + config: Config + api_token: Union[InterpolatedString, str] + parameters: Mapping[str, Any] + + def __post_init__(self) -> None: + self._token = InterpolatedString.create(self.api_token, parameters=self.parameters) + + def get_token(self) -> str: + return str(self._token.eval(self.config)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py index b1baacde083a..ac7fd6d90371 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/checks/check_stream.py @@ -40,10 +40,9 @@ def check_connection(self, source: Source, logger: logging.Logger, config: Mappi availability_strategy = stream.availability_strategy or HttpAvailabilityStrategy() try: stream_is_available, reason = availability_strategy.check_availability(stream, logger, source) - if stream_is_available: - return True, None - else: + if not stream_is_available: return False, reason except Exception as error: logger.error(f"Encountered an error trying to connect to stream {stream_name}. Error: \n {traceback.format_exc()}") return False, f"Unable to connect to stream {stream_name} - {error}" + return True, None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py index c28ef80266a6..932a1f58071a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py @@ -16,7 +16,9 @@ class DatetimeParser: Instead of using the directive directly, we can use datetime.fromtimestamp and dt.timestamp() """ - def parse(self, date: Union[str, int], format: str): + _UNIX_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + + def parse(self, date: Union[str, int], format: str) -> datetime.datetime: # "%s" is a valid (but unreliable) directive for formatting, but not for parsing # It is defined as # The number of seconds since the Epoch, 1970-01-01 00:00:00+0000 (UTC). https://man7.org/linux/man-pages/man3/strptime.3.html @@ -25,6 +27,8 @@ def parse(self, date: Union[str, int], format: str): # See https://stackoverflow.com/a/4974930 if format == "%s": return datetime.datetime.fromtimestamp(int(date), tz=datetime.timezone.utc) + elif format == "%ms": + return self._UNIX_EPOCH + datetime.timedelta(milliseconds=int(date)) parsed_datetime = datetime.datetime.strptime(str(date), format) if self._is_naive(parsed_datetime): @@ -37,6 +41,9 @@ def format(self, dt: datetime.datetime, format: str) -> str: # See https://stackoverflow.com/a/4974930 if format == "%s": return str(int(dt.timestamp())) + if format == "%ms": + # timstamp() returns a float representing the number of seconds since the unix epoch + return str(int(dt.timestamp() * 1000)) else: return dt.strftime(format) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index e02533107d5c..87c1ef911e18 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -27,6 +27,10 @@ properties: type: object spec: "$ref": "#/definitions/Spec" + metadata: + type: object + description: For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata. + additionalProperties: true additionalProperties: false definitions: AddedFieldDefinition: @@ -94,7 +98,6 @@ definitions: type: object required: - type - - api_token properties: type: type: string @@ -110,7 +113,7 @@ definitions: - "Token token={{ config['api_key'] }}" header: title: Header Name - description: The name of the HTTP header that will be set to the API key. + description: The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time. type: string interpolation_context: - config @@ -118,6 +121,15 @@ definitions: - Authorization - Api-Token - X-Auth-Token + inject_into: + title: Inject API Key Into Outgoing HTTP Request + description: Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined. + "$ref": "#/definitions/RequestOption" + examples: + - inject_into: header + field_name: Authorization + - inject_into: request_parameter + field_name: authKey $parameters: type: object additionalProperties: true @@ -339,7 +351,7 @@ definitions: enum: [CustomAuthenticator] class_name: title: Class Name - description: Fully-qualified name of the class that will be implementing the custom authentication strategy. The format is `source_..`. + description: Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`. type: string additionalProperties: true examples: @@ -550,11 +562,8 @@ definitions: required: - type - cursor_field - - end_datetime - datetime_format - - cursor_granularity - start_datetime - - step properties: type: type: string @@ -565,58 +574,102 @@ definitions: type: string interpolation_context: - config - - stream_state examples: - "created_at" - "{{ config['record_cursor'] }}" datetime_format: - title: Cursor Field Datetime Format - description: The datetime format of the Cursor Field. + title: Outgoing Datetime Format + description: | + The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available: + * **%s**: Epoch unix timestamp - `1686218963` + * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123` + * **%a**: Weekday (abbreviated) - `Sun` + * **%A**: Weekday (full) - `Sunday` + * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday) + * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31` + * **%b**: Month (abbreviated) - `Jan` + * **%B**: Month (full) - `January` + * **%m**: Month (zero-padded) - `01`, `02`, ..., `12` + * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99` + * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999` + * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23` + * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12` + * **%p**: AM/PM indicator + * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59` + * **%S**: Second (zero-padded) - `00`, `01`, ..., `59` + * **%f**: Microsecond (zero-padded to 6 digits) - `000000` + * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00` + * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT` + * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366` + * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53` + * **%W**: Week number of the year (starting Monday) - `00`, ..., `53` + * **%c**: Date and time - `Tue Aug 16 21:30:00 1988` + * **%x**: Date standard format - `08/16/1988` + * **%X**: Time standard format - `21:30:00` + * **%%**: Literal '%' character + + Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes). type: string examples: - "%Y-%m-%dT%H:%M:%S.%f%z" + - "%Y-%m-%d" + - "%s" + - "%ms" + start_datetime: + title: Start Datetime + description: The datetime that determines the earliest record that should be synced. + anyOf: + - type: string + - "$ref": "#/definitions/MinMaxDatetime" + interpolation_context: + - config + examples: + - "2020-01-1T00:00:00Z" + - "{{ config['start_time'] }}" + cursor_datetime_formats: + title: Cursor Datetime Formats + description: The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used. + type: array + items: + type: string + examples: + - "%Y-%m-%dT%H:%M:%S.%f%z" + - "%Y-%m-%d" + - "%s" cursor_granularity: title: Cursor Granularity description: Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should - be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. + be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well. type: string examples: - "PT1S" end_datetime: title: End Datetime - description: The datetime that determines the last record that should be synced. + description: The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used. anyOf: - type: string - "$ref": "#/definitions/MinMaxDatetime" + interpolation_context: + - config examples: - "2021-01-1T00:00:00Z" - "{{ now_utc() }}" - "{{ day_delta(-1) }}" - start_datetime: - title: Start Datetime - description: The datetime that determines the earliest record that should be synced. - anyOf: - - type: string - - "$ref": "#/definitions/MinMaxDatetime" - examples: - - "2020-01-1T00:00:00Z" - - "{{ config['start_time'] }}" - step: - title: Step - description: The size of the time window (ISO8601 duration). - type: string - examples: - - "P1W" - - "{{ config['step_increment'] }}" end_time_option: title: Inject End Time Into Outgoing HTTP Request description: Optionally configures how the end datetime will be sent in requests to the source API. "$ref": "#/definitions/RequestOption" + is_data_feed: + title: Whether the target API is formatted as a data feed + description: A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. + type: boolean lookback_window: title: Lookback Window description: Time interval before the start_datetime to read data for, e.g. P1M for looking back one month. type: string + interpolation_context: + - config examples: - "P1D" - "P{{ config['lookback_days'] }}D" @@ -636,6 +689,13 @@ definitions: title: Inject Start Time Into Outgoing HTTP Request description: Optionally configures how the start datetime will be sent in requests to the source API. "$ref": "#/definitions/RequestOption" + step: + title: Step + description: The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well. + type: string + examples: + - "P1W" + - "{{ config['step_increment'] }}" $parameters: type: object additionalProperties: true @@ -735,124 +795,46 @@ definitions: type: string examples: - "%Y-%m-%d %H:%M:%S.%f+00:00" - $parameters: - type: object - additionalProperties: true - SingleUseRefreshTokenOAuthAuthenticator: - tile: OAuth 2.0 With Single-Use Refresh Token - description: Authenticator for requests using OAuth 2.0 authentication with a single use refresh token. - type: object - required: - - type - - token_refresh_endpoint - properties: - type: - type: string - enum: [SingleUseRefreshTokenOAuthAuthenticator] - token_refresh_endpoint: - title: Token Refresh Endpoint - description: Endpoint to send a POST request in order to get a new access token. - type: string - examples: - - https://connect.squareup.com/oauth2/token - client_id_config_path: - title: Config Path To Client ID - description: Config path to the client ID. - type: array - items: - type: string - default: ["credentials", "client_id"] - examples: - - ["credentials", "client_id"] - - ["client_id"] - client_secret_config_path: - title: Config Path To Client Secret - description: Config path to the client secret. - type: array - items: - type: string - default: ["credentials", "client_secret"] - examples: - - ["credentials", "client_secret"] - - ["client_secret"] - access_token_config_path: - title: Config Path To Access Token - description: Config path to the access token. - type: array - items: - type: string - default: ["credentials", "access_token"] - examples: - - ["credentials", "access_token"] - - ["access_token"] - refresh_token_config_path: - title: Config Path To Refresh Token - description: Config path to the access token. - type: array - items: - type: string - default: ["credentials", "refresh_token"] - examples: - - ["credentials", "refresh_token"] - - ["refresh_token"] - token_expiry_date_config_path: - title: Config Path To Expiry Date - description: Config path to the expiry date. - type: array - items: - type: string - default: ["credentials", "token_expiry_date"] - examples: - - ["credentials", "token_expiry_date"] - access_token_name: - title: Access Token Response Field Name - description: The name of the field to extract the access token from in the token refresh response. - type: string - default: "access_token" - examples: - - "access_token" - refresh_token_name: - title: Refresh Token Response Field Name - type: string - default: "refresh_token" - examples: - - "refresh_token" - expires_in_name: - title: Time To Expiration Response Field Name - description: The name of the field the extract the number of seconds the access token will be valid from the token refresh response. - type: string - default: "expires_in" - examples: - - "expires_in" - grant_type: - title: Grant Type - description: Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported. - type: string - default: "refresh_token" - examples: - - refresh_token - - client_credentials - refresh_request_body: - title: Refresh Request Body - description: Body of the request sent to get a new access token. - type: object - additionalProperties: true - examples: - - applicationId: "{{ config['application_id'] }}" - applicationSecret: "{{ config['application_secret'] }}" - token: "{{ config['token'] }}" - scopes: - title: Scopes - description: List of scopes that should be granted to the access token. - type: array - items: - type: string - examples: - - [ - "crm.list.read", - "crm.objects.contacts.read", - "crm.schema.contacts.read", - ] + refresh_token_updater: + title: Token Updater + description: When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once. + properties: + refresh_token_name: + title: Refresh Token Property Name + description: The name of the property which contains the updated refresh token in the response from the token refresh endpoint. + type: string + default: "refresh_token" + examples: + - "refresh_token" + access_token_config_path: + title: Config Path To Access Token + description: Config path to the access token. Make sure the field actually exists in the config. + type: array + items: + type: string + default: ["credentials", "access_token"] + examples: + - ["credentials", "access_token"] + - ["access_token"] + refresh_token_config_path: + title: Config Path To Refresh Token + description: Config path to the access token. Make sure the field actually exists in the config. + type: array + items: + type: string + default: ["credentials", "refresh_token"] + examples: + - ["credentials", "refresh_token"] + - ["refresh_token"] + token_expiry_date_config_path: + title: Config Path To Expiry Date + description: Config path to the expiry date. Make sure actually exists in the config. + type: array + items: + type: string + default: ["credentials", "token_expiry_date"] + examples: + - ["credentials", "token_expiry_date"] $parameters: type: object additionalProperties: true @@ -898,6 +880,8 @@ definitions: anyOf: - "$ref": "#/definitions/InlineSchemaLoader" - "$ref": "#/definitions/JsonFileSchemaLoader" + # TODO we have move the transformation to the RecordSelector level in the code but kept this here for + # compatibility reason. We should eventually move this to align with the code. transformations: title: Transformations description: A list of transformations to be applied to each output record. @@ -994,7 +978,7 @@ definitions: enum: [DpathExtractor] field_path: title: Field Path - description: List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. + description: List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector). type: array items: - type: string @@ -1038,6 +1022,81 @@ definitions: $parameters: type: object additionalProperties: true + SessionTokenAuthenticator: + type: object + required: + - type + - login_requester + - session_token_path + - request_authentication + properties: + type: + type: string + enum: [SessionTokenAuthenticator] + login_requester: + title: Login Requester + description: Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property. + "$ref": "#/definitions/HttpRequester" + examples: + - type: HttpRequester + url_base: "https://my_api.com" + path: "/login" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config.username }}" + password: "{{ config.password }}" + session_token_path: + title: Session Token Path + description: The path in the response body returned from the login requester to the session token. + examples: + - ["access_token"] + - ["result", "token"] + type: array + items: + type: string + expiration_duration: + title: Expiration Duration + description: The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request. + type: string + examples: + - "PT1H" + - "P1D" + request_authentication: + title: Data Request Authentication + description: Authentication method to use for requests sent to the API, specifying how to inject the session token. + anyOf: + - "$ref": "#/definitions/SessionTokenRequestApiKeyAuthenticator" + - "$ref": "#/definitions/SessionTokenRequestBearerAuthenticator" + $parameters: + type: object + additionalProperties: true + SessionTokenRequestApiKeyAuthenticator: + type: object + title: API Key Authenticator + description: Authenticator for requests using the session token as an API key that's injected into the request. + required: + - type + - inject_into + properties: + type: + enum: [ApiKey] + inject_into: + title: Inject API Key Into Outgoing HTTP Request + description: Configure how the API Key will be sent in requests to the source API. + "$ref": "#/definitions/RequestOption" + examples: + - inject_into: header + field_name: Authorization + - inject_into: request_parameter + field_name: authKey + SessionTokenRequestBearerAuthenticator: + title: Bearer Authenticator + description: Authenticator for requests using the session token as a standard bearer token. + required: + - type + properties: + type: + enum: [Bearer] HttpRequester: title: HTTP Requester description: Requester submitting HTTP requests and extracting records from the response. @@ -1083,9 +1142,9 @@ definitions: - "$ref": "#/definitions/BearerAuthenticator" - "$ref": "#/definitions/CustomAuthenticator" - "$ref": "#/definitions/OAuthAuthenticator" - - "$ref": "#/definitions/SingleUseRefreshTokenOAuthAuthenticator" - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" + - "$ref": "#/definitions/LegacySessionTokenAuthenticator" error_handler: title: Error Handler description: Error handler component that defines how to handle errors. @@ -1108,7 +1167,7 @@ definitions: - POST request_body_data: title: Request Body Payload (Non-JSON) - description: Specifies how to populate the body of the request with a non-JSON payload. If returns a ready text that it will be sent as is. If returns a dict that it will be converted to a urlencoded form. + description: Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form. anyOf: - type: string - type: object @@ -1353,11 +1412,40 @@ definitions: - "{{ config['start_time'] }}" datetime_format: title: Datetime Format - description: Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp). + description: | + Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available: + * **%s**: Epoch unix timestamp - `1686218963` + * **%ms**: Epoch unix timestamp - `1686218963123` + * **%a**: Weekday (abbreviated) - `Sun` + * **%A**: Weekday (full) - `Sunday` + * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday) + * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31` + * **%b**: Month (abbreviated) - `Jan` + * **%B**: Month (full) - `January` + * **%m**: Month (zero-padded) - `01`, `02`, ..., `12` + * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99` + * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999` + * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23` + * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12` + * **%p**: AM/PM indicator + * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59` + * **%S**: Second (zero-padded) - `00`, `01`, ..., `59` + * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999` + * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00` + * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT` + * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366` + * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53` + * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53` + * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988` + * **%x**: Date representation - `08/16/1988` + * **%X**: Time representation - `21:30:00` + * **%%**: Literal '%' character + + Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes). type: string default: "" examples: - - "%Y-%m-%dT%H:%M:%S.%f%" + - "%Y-%m-%dT%H:%M:%S.%f%z" - "%Y-%m-%d" - "%s" max_datetime: @@ -1725,7 +1813,7 @@ definitions: - segment_id inject_into: title: Inject Into - description: Configures where the descriptor should be set on the HTTP requests. + description: Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated. enum: - request_parameter - header @@ -1741,9 +1829,9 @@ definitions: description: The stream schemas representing the shape of the data emitted by the stream. type: object additionalProperties: true - SessionTokenAuthenticator: + LegacySessionTokenAuthenticator: title: Session Token Authenticator - description: Authenticator for requests authenticated using session tokens. A session token is a random value generated by a server to identify a specific user for the duration of one interaction session. + description: Deprecated - use SessionTokenAuthenticator instead. Authenticator for requests authenticated using session tokens. A session token is a random value generated by a server to identify a specific user for the duration of one interaction session. type: object required: - type @@ -1754,7 +1842,7 @@ definitions: properties: type: type: string - enum: [SessionTokenAuthenticator] + enum: [LegacySessionTokenAuthenticator] header: title: Session Request Header description: The name of the session token header that will be injected in the request @@ -2046,33 +2134,34 @@ interpolation: arguments: {} return_type: Datetime examples: - - "{{ now_utc() }}" - - "{{ now_utc().strftime('%Y-%m-%d') }}" + - "'{{ now_utc() }}' -> '2021-09-01 00:00:00+00:00'" + - "'{{ now_utc().strftime('%Y-%m-%d') }}' -> '2021-09-01'" - title: Today (UTC) description: Returns the current date in UTC timezone. The output is a date object. arguments: {} return_type: Date examples: - - "{{ today_utc() }}" + - "'{{ today_utc() }}' -> '2021-09-01'" + - "'{{ today_utc().strftime('%Y/%m/%d')}}' -> '2021/09/01'" - title: Timestamp description: Converts a number or a string representing a datetime (formatted as ISO8601) to a timestamp. If the input is a number, it is converted to an int. If no timezone is specified, the string is interpreted as UTC. arguments: datetime: A string formatted as ISO8601 or an integer representing a unix timestamp return_type: int examples: - - "{{ timestamp(1646006400) }}" - - "{{ timestamp('2022-02-28') }}" - - "{{ timestamp('2022-02-28T00:00:00Z') }}" - - "{{ timestamp('2022-02-28 00:00:00Z') }}" - - "{{ timestamp('2022-02-28T00:00:00:-08:00') }}" + - "'{{ timestamp(1646006400) }}' -> 1646006400" + - "'{{ timestamp('2022-02-28') }}' -> 1646006400" + - "'{{ timestamp('2022-02-28T00:00:00Z') }}' -> 1646006400" + - "'{{ timestamp('2022-02-28 00:00:00Z') }}' -> 1646006400" + - "'{{ timestamp('2022-02-28T00:00:00-08:00') }}' -> 1646035200" - title: Max description: Returns the largest object of a iterable, or or two or more arguments. arguments: args: iterable or a sequence of two or more arguments return_type: Any examples: - - "{{ max(2, 3) }}" - - "{{ max([2, 3]) }}" + - "'{{ max(2, 3) }}' -> 3" + - "'{{ max([2, 3]) }}' -> 3" - title: Day Delta description: Returns the datetime of now() + num_days. arguments: @@ -2080,17 +2169,18 @@ interpolation: format: How to format the output string return_type: str examples: - - "{{ day_delta(25) }}" - - "{{ day_delta(25, format='%Y-%m-%d') }}" + - "'{{ day_delta(1) }}' -> '2021-09-02T00:00:00.000000+0000'" + - "'{{ day_delta(-1) }}' -> '2021-08-31:00:00.000000+0000'" + - "'{{ day_delta(25, format='%Y-%m-%d') }}' -> '2021-09-02'" - title: Duration description: Converts an ISO8601 duratioin to datetime.timedelta. arguments: duration_string: "A string representing an ISO8601 duration. See https://www.digi.com/resources/documentation/digidocs//90001488-13/reference/r_iso_8601_duration_format.htm for more details." return_type: datetime.timedelta examples: - - "{{ duration('P1D') }}" - - "{{ duration('P6DT23H') }}" - - "{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}" + - "'{{ duration('P1D') }}' -> '1 day, 0:00:00'" + - "'{{ duration('P6DT23H') }}' -> '6 days, 23:00:00'" + - "'{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}' -> '2021-08-31T00:00:00Z'" - title: Format Datetime description: Converts a datetime or a datetime-string to the specified format. arguments: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index 2a10e64e3fb7..56d92dfc5639 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -5,14 +5,13 @@ from dataclasses import InitVar, dataclass, field from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union -from airbyte_cdk.models import AirbyteMessage, SyncMode +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader -from airbyte_cdk.sources.declarative.transformations import RecordTransformation -from airbyte_cdk.sources.declarative.types import Config, StreamSlice -from airbyte_cdk.sources.streams.core import Stream, StreamData +from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.streams.core import Stream @dataclass @@ -27,7 +26,6 @@ class DeclarativeStream(Stream): retriever (Retriever): The retriever config (Config): The user-provided configuration as specified by the source's spec stream_cursor_field (Optional[Union[InterpolatedString, str]]): The cursor field - transformations (List[RecordTransformation]): A list of transformations to be applied to each output record in the stream. Transformations are applied in the order in which they are defined. """ @@ -39,16 +37,17 @@ class DeclarativeStream(Stream): schema_loader: Optional[SchemaLoader] = None _name: str = field(init=False, repr=False, default="") _primary_key: str = field(init=False, repr=False, default="") - _schema_loader: SchemaLoader = field(init=False, repr=False, default=None) stream_cursor_field: Optional[Union[InterpolatedString, str]] = None - transformations: List[RecordTransformation] = None - def __post_init__(self, parameters: Mapping[str, Any]): - self.stream_cursor_field = InterpolatedString.create(self.stream_cursor_field, parameters=parameters) - self.transformations = self.transformations or [] + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._stream_cursor_field = ( + InterpolatedString.create(self.stream_cursor_field, parameters=parameters) + if isinstance(self.stream_cursor_field, str) + else self.stream_cursor_field + ) self._schema_loader = self.schema_loader if self.schema_loader else DefaultSchemaLoader(config=self.config, parameters=parameters) - @property + @property # type: ignore def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: return self._primary_key @@ -57,7 +56,7 @@ def primary_key(self, value: str) -> None: if not isinstance(value, property): self._primary_key = value - @property + @property # type: ignore def name(self) -> str: """ :return: Stream name. By default this is the implementing class name, but it can be overridden as needed. @@ -71,14 +70,16 @@ def name(self, value: str) -> None: @property def state(self) -> MutableMapping[str, Any]: - return self.retriever.state + return self.retriever.state # type: ignore @state.setter - def state(self, value: MutableMapping[str, Any]): + def state(self, value: MutableMapping[str, Any]) -> None: """State setter, accept state serialized by state getter.""" self.retriever.state = value - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + def get_updated_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: return self.state @property @@ -87,46 +88,22 @@ def cursor_field(self) -> Union[str, List[str]]: Override to return the default cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. :return: The name of the field used as a cursor. If the cursor is nested, return an array consisting of the path to the cursor. """ - cursor = self.stream_cursor_field.eval(self.config) + cursor = self._stream_cursor_field.eval(self.config) return cursor if cursor else [] def read_records( self, sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: - for record in self.retriever.read_records(sync_mode, cursor_field, stream_slice, stream_state): - yield self._apply_transformations(record, self.config, stream_slice) + """ + :param: stream_state We knowingly avoid using stream_state as we want cursors to manage their own state. + """ + yield from self.retriever.read_records(stream_slice) - def _apply_transformations( - self, - message_or_record_data: StreamData, - config: Config, - stream_slice: StreamSlice, - ): - # If the input is an AirbyteMessage with a record, transform the record's data - # If the input is another type of AirbyteMessage, return it as is - # If the input is a dict, transform it - if isinstance(message_or_record_data, AirbyteMessage): - if message_or_record_data.record: - record = message_or_record_data.record.data - else: - return message_or_record_data - elif isinstance(message_or_record_data, dict): - record = message_or_record_data - else: - # Raise an error because this is unexpected and indicative of a typing problem in the CDK - raise ValueError( - f"Unexpected record type. Expected {StreamData}. Got {type(message_or_record_data)}. This is probably due to a bug in the CDK." - ) - for transformation in self.transformations: - transformation.transform(record, config=config, stream_state=self.state, stream_slice=stream_slice) - - return message_or_record_data - - def get_json_schema(self) -> Mapping[str, Any]: + def get_json_schema(self) -> Mapping[str, Any]: # type: ignore """ :return: A dict of the JSON schema representing this stream. @@ -136,15 +113,25 @@ def get_json_schema(self) -> Mapping[str, Any]: return self._schema_loader.get_json_schema() def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None ) -> Iterable[Optional[Mapping[str, Any]]]: """ Override to define the slices for this stream. See the stream slicing section of the docs for more information. :param sync_mode: :param cursor_field: - :param stream_state: + :param stream_state: we knowingly avoid using stream_state as we want cursors to manage their own state :return: """ - # this is not passing the cursor field because it is known at init time - return self.retriever.stream_slices(sync_mode=sync_mode, stream_state=stream_state) + return self.retriever.stream_slices() + + @property + def state_checkpoint_interval(self) -> Optional[int]: + """ + We explicitly disable checkpointing here. There are a couple reasons for that and not all are documented here but: + * In the case where records are not ordered, the granularity of what is ordered is the slice. Therefore, we will only update the + cursor value once at the end of every slice. + * Updating the state once every record would generate issues for data feed stop conditions or semi-incremental syncs where the + important state is the one at the beginning of the slice + """ + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py index 1ee9baa96018..99fe9378b6fe 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py @@ -11,7 +11,7 @@ from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Config, Record +from airbyte_cdk.sources.declarative.types import Config @dataclass @@ -63,7 +63,7 @@ def __post_init__(self, parameters: Mapping[str, Any]): if isinstance(self.field_path[path_index], str): self.field_path[path_index] = InterpolatedString.create(self.field_path[path_index], parameters=parameters) - def extract_records(self, response: requests.Response) -> List[Record]: + def extract_records(self, response: requests.Response) -> List[Mapping[str, Any]]: response_body = self.decoder.decode(response) if len(self.field_path) == 0: extracted = response_body diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py index f57fa5f31b36..792499c6dac6 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py @@ -4,10 +4,9 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import List +from typing import Any, List, Mapping import requests -from airbyte_cdk.sources.declarative.types import Record @dataclass @@ -20,7 +19,7 @@ class RecordExtractor: def extract_records( self, response: requests.Response, - ) -> List[Record]: + ) -> List[Mapping[str, Any]]: """ Selects records from the response :param response: The response to extract the records from diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py index fd41e206c4ac..6bd46113e3ba 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py @@ -6,7 +6,7 @@ from typing import Any, List, Mapping, Optional from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean -from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState @dataclass @@ -22,17 +22,15 @@ class RecordFilter: config: Config condition: str = "" - def __post_init__(self, parameters: Mapping[str, Any]): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._filter_interpolator = InterpolatedBoolean(condition=self.condition, parameters=parameters) def filter_records( self, - records: List[Record], + records: List[Mapping[str, Any]], stream_state: StreamState, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> List[Record]: + ) -> List[Mapping[str, Any]]: kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - for record in records: - if self._filter_interpolator.eval(self.config, record=record, **kwargs): - yield record + return [record for record in records if self._filter_interpolator.eval(self.config, record=record, **kwargs)] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index a3affe9f27e9..d08068a952e0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -2,14 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import InitVar, dataclass +from dataclasses import InitVar, dataclass, field from typing import Any, List, Mapping, Optional import requests from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.transformations import RecordTransformation +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState @dataclass @@ -21,13 +22,16 @@ class RecordSelector(HttpSelector): Attributes: extractor (RecordExtractor): The record extractor responsible for extracting records from a response record_filter (RecordFilter): The record filter responsible for filtering extracted records + transformations (List[RecordTransformation]): The transformations to be done on the records """ extractor: RecordExtractor + config: Config parameters: InitVar[Mapping[str, Any]] - record_filter: RecordFilter = None + record_filter: Optional[RecordFilter] = None + transformations: List[RecordTransformation] = field(default_factory=lambda: []) - def __post_init__(self, parameters: Mapping[str, Any]): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._parameters = parameters def select_records( @@ -37,9 +41,30 @@ def select_records( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: - all_records = self.extractor.extract_records(response) + all_data = self.extractor.extract_records(response) + filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token) + self._transform(filtered_data, stream_state, stream_slice) + return [Record(data, stream_slice) for data in filtered_data] + + def _filter( + self, + records: List[Mapping[str, Any]], + stream_state: StreamState, + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + ) -> List[Mapping[str, Any]]: if self.record_filter: return self.record_filter.filter_records( - all_records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - return all_records + return records + + def _transform( + self, + records: List[Mapping[str, Any]], + stream_state: StreamState, + stream_slice: Optional[StreamSlice] = None, + ) -> None: + for record in records: + for transformation in self.transformations: + transformation.transform(record, config=self.config, stream_state=stream_state, stream_slice=stream_slice) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py index 7d4572084492..aea13ab8a629 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/__init__.py @@ -2,6 +2,8 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import CursorFactory, PerPartitionCursor -__all__ = ["DatetimeBasedCursor"] +__all__ = ["Cursor", "CursorFactory", "DatetimeBasedCursor", "PerPartitionCursor"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/cursor.py new file mode 100644 index 000000000000..9e2c6d4eb065 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/cursor.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Optional + +from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState + + +class Cursor(ABC, StreamSlicer): + """ + Cursors are components that allow for incremental syncs. They keep track of what data has been consumed and slices the requests based on + that information. + """ + + @abstractmethod + def set_initial_state(self, stream_state: StreamState) -> None: + """ + Cursors are not initialized with their state. As state is needed in order to function properly, this method should be called + before calling anything else + + :param stream_state: The state of the stream as returned by get_stream_state + """ + + @abstractmethod + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + """ + Update state based on the stream slice and the latest record. Note that `stream_slice.cursor_slice` and + `last_record.associated_slice` are expected to be the same but we make it explicit here that `stream_slice` should be leveraged to + update the state. + + :param stream_slice: slice to close + :param last_record: the latest record we have received for the slice. This is important to consider because even if the cursor emits + a slice, some APIs are not able to enforce the upper boundary. The outcome is that the last_record might have a higher cursor + value than the slice upper boundary and if we want to reduce the duplication as much as possible, we need to consider the highest + value between the internal cursor, the stream slice upper boundary and the record cursor value. + """ + + @abstractmethod + def get_stream_state(self) -> StreamState: + """ + Returns the current stream state. We would like to restrict it's usage since it does expose internal of state. As of 2023-06-14, it + is used for two things: + * Interpolation of the requests + * Transformation of records + * Saving the state + + For the first case, we are probably stuck with exposing the stream state. For the second, we can probably expose a method that + allows for emitting the state to the platform. + """ + + @abstractmethod + def should_be_synced(self, record: Record) -> bool: + """ + Evaluating if a record should be synced allows for filtering and stop condition on pagination + """ + + @abstractmethod + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py index b64e433ea009..685f0b7e6876 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/datetime_based_cursor.py @@ -4,23 +4,24 @@ import datetime from dataclasses import InitVar, dataclass, field -from typing import Any, Iterable, Mapping, Optional, Union +from typing import Any, Iterable, List, Mapping, Optional, Union -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType -from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.message import MessageRepository from isodate import Duration, parse_duration @dataclass -class DatetimeBasedCursor(StreamSlicer): +class DatetimeBasedCursor(Cursor): """ - Slices the stream over a datetime range. + Slices the stream over a datetime range and create a state with format {: } Given a start time, end time, a step function, and an optional lookback window, the stream slicer will partition the date range from start time - lookback window to end time. @@ -33,11 +34,11 @@ class DatetimeBasedCursor(StreamSlicer): Attributes: start_datetime (Union[MinMaxDatetime, str]): the datetime that determines the earliest record that should be synced - end_datetime (Union[MinMaxDatetime, str]): the datetime that determines the last record that should be synced - step (str): size of the timewindow (ISO8601 duration) + end_datetime (Optional[Union[MinMaxDatetime, str]]): the datetime that determines the last record that should be synced cursor_field (Union[InterpolatedString, str]): record's cursor field datetime_format (str): format of the datetime - cursor_granularity (str): smallest increment the datetime_format has (ISO 8601 duration) that will be used to ensure that the start of a slice does not overlap with the end of the previous one + step (Optional[str]): size of the timewindow (ISO8601 duration) + cursor_granularity (Optional[str]): smallest increment the datetime_format has (ISO 8601 duration) that will be used to ensure that the start of a slice does not overlap with the end of the previous one config (Config): connection config start_time_option (Optional[RequestOption]): request option for start time end_time_option (Optional[RequestOption]): request option for end time @@ -47,31 +48,41 @@ class DatetimeBasedCursor(StreamSlicer): """ start_datetime: Union[MinMaxDatetime, str] - end_datetime: Union[MinMaxDatetime, str] - step: Union[InterpolatedString, str] cursor_field: Union[InterpolatedString, str] datetime_format: str - cursor_granularity: str config: Config parameters: InitVar[Mapping[str, Any]] - _cursor: dict = field(repr=False, default=None) # tracks current datetime - _cursor_end: dict = field(repr=False, default=None) # tracks end of current stream slice + _cursor: Optional[str] = field(repr=False, default=None) # tracks current datetime + end_datetime: Optional[Union[MinMaxDatetime, str]] = None + step: Optional[Union[InterpolatedString, str]] = None + cursor_granularity: Optional[str] = None start_time_option: Optional[RequestOption] = None end_time_option: Optional[RequestOption] = None partition_field_start: Optional[str] = None partition_field_end: Optional[str] = None lookback_window: Optional[Union[InterpolatedString, str]] = None - - def __post_init__(self, parameters: Mapping[str, Any]): + message_repository: Optional[MessageRepository] = None + cursor_datetime_formats: List[str] = field(default_factory=lambda: []) + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + if (self.step and not self.cursor_granularity) or (not self.step and self.cursor_granularity): + raise ValueError( + f"If step is defined, cursor_granularity should be as well and vice-versa. " + f"Right now, step is `{self.step}` and cursor_granularity is `{self.cursor_granularity}`" + ) if not isinstance(self.start_datetime, MinMaxDatetime): self.start_datetime = MinMaxDatetime(self.start_datetime, parameters) - if not isinstance(self.end_datetime, MinMaxDatetime): + if self.end_datetime and not isinstance(self.end_datetime, MinMaxDatetime): self.end_datetime = MinMaxDatetime(self.end_datetime, parameters) self._timezone = datetime.timezone.utc self._interpolation = JinjaInterpolation() - self._step = self._parse_timedelta(InterpolatedString.create(self.step, parameters=parameters).eval(self.config)) + self._step = ( + self._parse_timedelta(InterpolatedString.create(self.step, parameters=parameters).eval(self.config)) + if self.step + else datetime.timedelta.max + ) self._cursor_granularity = self._parse_timedelta(self.cursor_granularity) self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) self.lookback_window = InterpolatedString.create(self.lookback_window, parameters=parameters) @@ -82,66 +93,72 @@ def __post_init__(self, parameters: Mapping[str, Any]): # If datetime format is not specified then start/end datetime should inherit it from the stream slicer if not self.start_datetime.datetime_format: self.start_datetime.datetime_format = self.datetime_format - if not self.end_datetime.datetime_format: + if self.end_datetime and not self.end_datetime.datetime_format: self.end_datetime.datetime_format = self.datetime_format + if not self.cursor_datetime_formats: + self.cursor_datetime_formats = [self.datetime_format] + def get_stream_state(self) -> StreamState: return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + def set_initial_state(self, stream_state: StreamState) -> None: """ - Update the cursor value to the max datetime between the last record, the start of the stream_slice, and the current cursor value. - Update the cursor_end value with the stream_slice's end time. + Cursors are not initialized with their state. As state is needed in order to function properly, this method should be called + before calling anything else - :param stream_slice: current stream slice - :param last_record: last record read - :return: None + :param stream_state: The state of the stream as returned by get_stream_state """ - stream_slice_value = stream_slice.get(self.cursor_field.eval(self.config)) + self._cursor = stream_state.get(self.cursor_field.eval(self.config)) if stream_state else None + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + last_record_cursor_value = most_recent_record.get(self.cursor_field.eval(self.config)) if most_recent_record else None stream_slice_value_end = stream_slice.get(self.partition_field_end.eval(self.config)) - last_record_value = last_record.get(self.cursor_field.eval(self.config)) if last_record else None - cursor = None - if stream_slice_value and last_record_value: - cursor = max(stream_slice_value, last_record_value) - elif stream_slice_value: - cursor = stream_slice_value - else: - cursor = last_record_value - if self._cursor and cursor: - self._cursor = max(cursor, self._cursor) - elif cursor: - self._cursor = cursor - if self.partition_field_end: - self._cursor_end = stream_slice_value_end - - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: + cursor_value_str_by_cursor_value_datetime = dict( + map( + # we need to ensure the cursor value is preserved as is in the state else the CATs might complain of something like + # 2023-01-04T17:30:19.000Z' <= '2023-01-04T17:30:19.000000Z' + lambda datetime_str: (self.parse_date(datetime_str), datetime_str), + filter(lambda item: item, [self._cursor, last_record_cursor_value, stream_slice_value_end]), + ) + ) + self._cursor = ( + cursor_value_str_by_cursor_value_datetime[max(cursor_value_str_by_cursor_value_datetime.keys())] + if cursor_value_str_by_cursor_value_datetime + else None + ) + + def stream_slices(self) -> Iterable[StreamSlice]: """ Partition the daterange into slices of size = step. The start of the window is the minimum datetime between start_datetime - lookback_window and the stream_state's datetime The end of the window is the minimum datetime between the start of the window and end_datetime. - :param sync_mode: - :param stream_state: current stream state. If set, the start_date will be the day following the stream_state. :return: """ - stream_state = stream_state or {} - kwargs = {"stream_state": stream_state} - end_datetime = min(self.end_datetime.get_datetime(self.config, **kwargs), datetime.datetime.now(tz=self._timezone)) - lookback_delta = self._parse_timedelta(self.lookback_window.eval(self.config, **kwargs) if self.lookback_window else "P0D") + end_datetime = self._select_best_end_datetime() + start_datetime = self._calculate_earliest_possible_value(self._select_best_end_datetime()) + return self._partition_daterange(start_datetime, end_datetime, self._step) - earliest_possible_start_datetime = min(self.start_datetime.get_datetime(self.config, **kwargs), end_datetime) - cursor_datetime = self._calculate_cursor_datetime_from_state(stream_state) - start_datetime = max(earliest_possible_start_datetime, cursor_datetime) - lookback_delta + def _calculate_earliest_possible_value(self, end_datetime: datetime.datetime) -> datetime.datetime: + lookback_delta = self._parse_timedelta(self.lookback_window.eval(self.config) if self.lookback_window else "P0D") + earliest_possible_start_datetime = min(self.start_datetime.get_datetime(self.config), end_datetime) + cursor_datetime = self._calculate_cursor_datetime_from_state(self.get_stream_state()) + return max(earliest_possible_start_datetime, cursor_datetime) - lookback_delta - return self._partition_daterange(start_datetime, end_datetime, self._step) + def _select_best_end_datetime(self) -> datetime.datetime: + now = datetime.datetime.now(tz=self._timezone) + if not self.end_datetime: + return now + return min(self.end_datetime.get_datetime(self.config), now) def _calculate_cursor_datetime_from_state(self, stream_state: Mapping[str, Any]) -> datetime.datetime: if self.cursor_field.eval(self.config, stream_state=stream_state) in stream_state: return self.parse_date(stream_state[self.cursor_field.eval(self.config)]) return datetime.datetime.min.replace(tzinfo=datetime.timezone.utc) - def _format_datetime(self, dt: datetime.datetime): + def _format_datetime(self, dt: datetime.datetime) -> str: return self._parser.format(dt, self.datetime_format) def _partition_daterange(self, start: datetime.datetime, end: datetime.datetime, step: Union[datetime.timedelta, Duration]): @@ -149,17 +166,34 @@ def _partition_daterange(self, start: datetime.datetime, end: datetime.datetime, end_field = self.partition_field_end.eval(self.config) dates = [] while start <= end: - end_date = self._get_date(start + step - self._cursor_granularity, end, min) + next_start = self._evaluate_next_start_date_safely(start, step) + end_date = self._get_date(next_start - self._cursor_granularity, end, min) dates.append({start_field: self._format_datetime(start), end_field: self._format_datetime(end_date)}) - start += step + start = next_start return dates + def _evaluate_next_start_date_safely(self, start, step): + """ + Given that we set the default step at datetime.timedelta.max, we will generate an OverflowError when evaluating the next start_date + This method assumes that users would never enter a step that would generate an overflow. Given that would be the case, the code + would have broken anyway. + """ + try: + return start + step + except OverflowError: + return datetime.datetime.max.replace(tzinfo=datetime.timezone.utc) + def _get_date(self, cursor_value, default_date: datetime.datetime, comparator) -> datetime.datetime: cursor_date = cursor_value or default_date return comparator(cursor_date, default_date) def parse_date(self, date: str) -> datetime.datetime: - return self._parser.parse(date, self.datetime_format) + for datetime_format in self.cursor_datetime_formats + [self.datetime_format]: + try: + return self._parser.parse(date, datetime_format) + except ValueError: + pass + raise ValueError(f"No format in {self.cursor_datetime_formats} matching {date}") @classmethod def _parse_timedelta(cls, time_str) -> Union[datetime.timedelta, Duration]: @@ -217,3 +251,37 @@ def _get_request_options(self, option_type: RequestOptionType, stream_slice: Str if self.end_time_option and self.end_time_option.inject_into == option_type: options[self.end_time_option.field_name] = stream_slice.get(self.partition_field_end.eval(self.config)) return options + + def should_be_synced(self, record: Record) -> bool: + cursor_field = self.cursor_field.eval(self.config) + record_cursor_value = record.get(cursor_field) + if not record_cursor_value: + self._send_log( + Level.WARN, + f"Could not find cursor field `{cursor_field}` in record. The incremental sync will assume it needs to be synced", + ) + return True + + latest_possible_cursor_value = self._select_best_end_datetime() + earliest_possible_cursor_value = self._calculate_earliest_possible_value(latest_possible_cursor_value) + return earliest_possible_cursor_value <= self.parse_date(record_cursor_value) <= latest_possible_cursor_value + + def _send_log(self, level: Level, message: str) -> None: + if self.message_repository: + self.message_repository.emit_message( + AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage(level=level, message=message), + ) + ) + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + cursor_field = self.cursor_field.eval(self.config) + first_cursor_value = first.get(cursor_field) + second_cursor_value = second.get(cursor_field) + if first_cursor_value and second_cursor_value: + return self.parse_date(first_cursor_value) >= self.parse_date(second_cursor_value) + elif first_cursor_value: + return True + else: + return False diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py new file mode 100644 index 000000000000..75af991970d3 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py @@ -0,0 +1,279 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any, Callable, Iterable, Mapping, Optional + +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor +from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState + + +class PerPartitionKeySerializer: + """ + We are concerned of the performance of looping through the `states` list and evaluating equality on the partition. To reduce this + concern, we wanted to use dictionaries to map `partition -> cursor`. However, partitions are dict and dict can't be used as dict keys + since they are not hashable. By creating json string using the dict, we can have a use the dict as a key to the dict since strings are + hashable. + """ + + @staticmethod + def to_partition_key(to_serialize: Any) -> str: + # separators have changed in Python 3.4. To avoid being impacted by further change, we explicitly specify our own value + return json.dumps(to_serialize, indent=None, separators=(",", ":"), sort_keys=True) + + @staticmethod + def to_partition(to_deserialize: Any): + return json.loads(to_deserialize) + + +class PerPartitionStreamSlice(StreamSlice): + def __init__(self, partition: Mapping[str, Any], cursor_slice: Mapping[str, Any]): + self._partition = partition + self._cursor_slice = cursor_slice + if partition.keys() & cursor_slice.keys(): + raise ValueError("Keys for partition and incremental sync cursor should not overlap") + self._stream_slice = dict(partition) | dict(cursor_slice) + + @property + def partition(self): + return self._partition + + @property + def cursor_slice(self): + return self._cursor_slice + + def __repr__(self): + return repr(self._stream_slice) + + def __setitem__(self, key: str, value: Any): + raise ValueError("PerPartitionStreamSlice is immutable") + + def __getitem__(self, key: str): + return self._stream_slice[key] + + def __len__(self): + return len(self._stream_slice) + + def __iter__(self): + return iter(self._stream_slice) + + def __contains__(self, item: str): + return item in self._stream_slice + + def keys(self): + return self._stream_slice.keys() + + def items(self): + return self._stream_slice.items() + + def values(self): + return self._stream_slice.values() + + def get(self, key: str, default: Any) -> Any: + return self._stream_slice.get(key, default) + + def __eq__(self, other): + if isinstance(other, dict): + return self._stream_slice == other + if isinstance(other, PerPartitionStreamSlice): + # noinspection PyProtectedMember + return self._partition == other._partition and self._cursor_slice == other._cursor_slice + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class CursorFactory: + def __init__(self, create_function: Callable[[], StreamSlicer]): + self._create_function = create_function + + def create(self) -> StreamSlicer: + return self._create_function() + + +class PerPartitionCursor(Cursor): + """ + Given a stream has many partitions, it is important to provide a state per partition. + + Record | Stream Slice | Last Record | DatetimeCursorBased cursor + -- | -- | -- | -- + 1 | {"start_time": "2021-01-01","end_time": "2021-01-31","owner_resource": "1"''} | cursor_field: “2021-01-15” | 2021-01-15 + 2 | {"start_time": "2021-02-01","end_time": "2021-02-28","owner_resource": "1"''} | cursor_field: “2021-02-15” | 2021-02-15 + 3 | {"start_time": "2021-01-01","end_time": "2021-01-31","owner_resource": "2"''} | cursor_field: “2021-01-03” | 2021-01-03 + 4 | {"start_time": "2021-02-01","end_time": "2021-02-28","owner_resource": "2"''} | cursor_field: “2021-02-14” | 2021-02-14 + + Given the following errors, this can lead to some loss or duplication of records: + When | Problem | Affected Record + -- | -- | -- + Between record #1 and #2 | Loss | #3 + Between record #2 and #3 | Loss | #3, #4 + Between record #3 and #4 | Duplication | #1, #2 + + Therefore, we need to manage state per partition. + """ + + _NO_STATE = {} + _NO_CURSOR_STATE = {} + _KEY = 0 + _VALUE = 1 + + def __init__(self, cursor_factory: CursorFactory, partition_router: StreamSlicer): + self._cursor_factory = cursor_factory + self._partition_router = partition_router + self._cursor_per_partition = {} + self._partition_serializer = PerPartitionKeySerializer() + + def stream_slices(self) -> Iterable[PerPartitionStreamSlice]: + slices = self._partition_router.stream_slices() + for partition in slices: + cursor = self._cursor_per_partition.get(self._to_partition_key(partition)) + if not cursor: + cursor = self._create_cursor(self._NO_CURSOR_STATE) + self._cursor_per_partition[self._to_partition_key(partition)] = cursor + + for cursor_slice in cursor.stream_slices(): + yield PerPartitionStreamSlice(partition, cursor_slice) + + def set_initial_state(self, stream_state: StreamState) -> None: + if not stream_state: + return + + for state in stream_state["states"]: + self._cursor_per_partition[self._to_partition_key(state["partition"])] = self._create_cursor(state["cursor"]) + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + try: + cursor_most_recent_record = ( + Record(most_recent_record.data, stream_slice.cursor_slice) if most_recent_record else most_recent_record + ) + self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].close_slice( + stream_slice.cursor_slice, cursor_most_recent_record + ) + except KeyError as exception: + raise ValueError( + f"Partition {str(exception)} could not be found in current state based on the record. This is unexpected because " + f"we should only update state for partition that where emitted during `stream_slices`" + ) + + def get_stream_state(self) -> StreamState: + states = [] + for partition_tuple, cursor in self._cursor_per_partition.items(): + cursor_state = cursor.get_stream_state() + if cursor_state: + states.append( + { + "partition": self._to_dict(partition_tuple), + "cursor": cursor_state, + } + ) + return {"states": states} + + def _get_state_for_partition(self, partition: Mapping[str, Any]) -> Optional[StreamState]: + cursor = self._cursor_per_partition.get(self._to_partition_key(partition)) + if cursor: + return cursor.get_stream_state() + + return None + + @staticmethod + def _is_new_state(stream_state): + return not bool(stream_state) + + def _to_partition_key(self, partition) -> tuple: + return self._partition_serializer.to_partition_key(partition) + + def _to_dict(self, partition_key: tuple) -> StreamSlice: + return self._partition_serializer.to_partition(partition_key) + + def select_state(self, stream_slice: Optional[PerPartitionStreamSlice] = None) -> Optional[StreamState]: + if not stream_slice: + raise ValueError("A partition needs to be provided in order to extract a state") + + if not stream_slice: + return None + + return self._get_state_for_partition(stream_slice.partition) + + def _create_cursor(self, cursor_state: Any) -> StreamSlicer: + cursor = self._cursor_factory.create() + cursor.set_initial_state(cursor_state) + return cursor + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return self._partition_router.get_request_params( + stream_state=stream_state, stream_slice=stream_slice.partition, next_page_token=next_page_token + ) | self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].get_request_params( + stream_state=stream_state, stream_slice=stream_slice.cursor_slice, next_page_token=next_page_token + ) + + def get_request_headers( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return self._partition_router.get_request_headers( + stream_state=stream_state, stream_slice=stream_slice.partition, next_page_token=next_page_token + ) | self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].get_request_headers( + stream_state=stream_state, stream_slice=stream_slice.cursor_slice, next_page_token=next_page_token + ) + + def get_request_body_data( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return self._partition_router.get_request_body_data( + stream_state=stream_state, stream_slice=stream_slice.partition, next_page_token=next_page_token + ) | self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].get_request_body_data( + stream_state=stream_state, stream_slice=stream_slice.cursor_slice, next_page_token=next_page_token + ) + + def get_request_body_json( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + return self._partition_router.get_request_body_json( + stream_state=stream_state, stream_slice=stream_slice.partition, next_page_token=next_page_token + ) | self._cursor_per_partition[self._to_partition_key(stream_slice.partition)].get_request_body_json( + stream_state=stream_state, stream_slice=stream_slice.cursor_slice, next_page_token=next_page_token + ) + + def should_be_synced(self, record: Record) -> bool: + return self._get_cursor(record).should_be_synced(self._convert_record_to_cursor_record(record)) + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + if first.associated_slice.partition != second.associated_slice.partition: + raise ValueError( + f"To compare records, partition should be the same but got {first.associated_slice.partition} and {second.associated_slice.partition}" + ) + + return self._get_cursor(first).is_greater_than_or_equal( + self._convert_record_to_cursor_record(first), self._convert_record_to_cursor_record(second) + ) + + @staticmethod + def _convert_record_to_cursor_record(record: Record): + return Record(record.data, record.associated_slice.cursor_slice) + + def _get_cursor(self, record: Record) -> Cursor: + partition_key = self._to_partition_key(record.associated_slice.partition) + if partition_key not in self._cursor_per_partition: + raise ValueError("Invalid state as stream slices that are emitted should refer to an existing cursor") + cursor = self._cursor_per_partition[partition_key] + return cursor diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py index 1dcaccbfa4f4..ee014912c68a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -26,6 +26,7 @@ from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory from airbyte_cdk.sources.declarative.types import ConnectionDefinition +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.core import Stream from jsonschema.exceptions import ValidationError from jsonschema.validators import validate @@ -61,6 +62,7 @@ def __init__( self._debug = debug self._emit_connector_builder_messages = emit_connector_builder_messages self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages) + self._message_repository = self._constructor.get_message_repository() self._validate_source() @@ -68,6 +70,10 @@ def __init__( def resolved_manifest(self) -> Mapping[str, Any]: return self._source_config + @property + def message_repository(self) -> Union[None, MessageRepository]: + return self._message_repository + @property def connection_checker(self) -> ConnectionChecker: check = self._source_config["check"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index e694981fb97f..24e3a907b115 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -45,23 +41,6 @@ class AddFields(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class ApiKeyAuthenticator(BaseModel): - type: Literal["ApiKeyAuthenticator"] - api_token: str = Field( - ..., - description="The API key to inject in the request. Fill it in the user inputs.", - examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], - title="API Key", - ) - header: Optional[str] = Field( - None, - description="The name of the HTTP header that will be set to the API key.", - examples=["Authorization", "Api-Token", "X-Auth-Token"], - title="Header Name", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class AuthFlowType(Enum): oauth2_0 = "oauth2.0" oauth1_0 = "oauth1.0" @@ -123,7 +102,7 @@ class Config: type: Literal["CustomAuthenticator"] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom authentication strategy. The format is `source_..`.", + description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.", examples=["source_railz.components.ShortLivedTokenAuthenticator"], title="Class Name", ) @@ -260,6 +239,33 @@ class Config: parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class RefreshTokenUpdater(BaseModel): + refresh_token_name: Optional[str] = Field( + "refresh_token", + description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", + examples=["refresh_token"], + title="Refresh Token Property Name", + ) + access_token_config_path: Optional[List[str]] = Field( + ["credentials", "access_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "access_token"], ["access_token"]], + title="Config Path To Access Token", + ) + refresh_token_config_path: Optional[List[str]] = Field( + ["credentials", "refresh_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "refresh_token"], ["refresh_token"]], + title="Config Path To Refresh Token", + ) + token_expiry_date_config_path: Optional[List[str]] = Field( + ["credentials", "token_expiry_date"], + description="Config path to the expiry date. Make sure actually exists in the config.", + examples=[["credentials", "token_expiry_date"]], + title="Config Path To Expiry Date", + ) + + class OAuthAuthenticator(BaseModel): type: Literal["OAuthAuthenticator"] client_id: str = Field( @@ -340,87 +346,10 @@ class OAuthAuthenticator(BaseModel): examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], title="Token Expiry Date Format", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class SingleUseRefreshTokenOAuthAuthenticator(BaseModel): - type: Literal["SingleUseRefreshTokenOAuthAuthenticator"] - token_refresh_endpoint: str = Field( - ..., - description="Endpoint to send a POST request in order to get a new access token.", - examples=["https://connect.squareup.com/oauth2/token"], - title="Token Refresh Endpoint", - ) - client_id_config_path: Optional[List[str]] = Field( - ["credentials", "client_id"], - description="Config path to the client ID.", - examples=[["credentials", "client_id"], ["client_id"]], - title="Config Path To Client ID", - ) - client_secret_config_path: Optional[List[str]] = Field( - ["credentials", "client_secret"], - description="Config path to the client secret.", - examples=[["credentials", "client_secret"], ["client_secret"]], - title="Config Path To Client Secret", - ) - access_token_config_path: Optional[List[str]] = Field( - ["credentials", "access_token"], - description="Config path to the access token.", - examples=[["credentials", "access_token"], ["access_token"]], - title="Config Path To Access Token", - ) - refresh_token_config_path: Optional[List[str]] = Field( - ["credentials", "refresh_token"], - description="Config path to the access token.", - examples=[["credentials", "refresh_token"], ["refresh_token"]], - title="Config Path To Refresh Token", - ) - token_expiry_date_config_path: Optional[List[str]] = Field( - ["credentials", "token_expiry_date"], - description="Config path to the expiry date.", - examples=[["credentials", "token_expiry_date"]], - title="Config Path To Expiry Date", - ) - access_token_name: Optional[str] = Field( - "access_token", - description="The name of the field to extract the access token from in the token refresh response.", - examples=["access_token"], - title="Access Token Response Field Name", - ) - refresh_token_name: Optional[str] = Field( - "refresh_token", - examples=["refresh_token"], - title="Refresh Token Response Field Name", - ) - expires_in_name: Optional[str] = Field( - "expires_in", - description="The name of the field the extract the number of seconds the access token will be valid from the token refresh response.", - examples=["expires_in"], - title="Time To Expiration Response Field Name", - ) - grant_type: Optional[str] = Field( - "refresh_token", - description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", - examples=["refresh_token", "client_credentials"], - title="Grant Type", - ) - refresh_request_body: Optional[Dict[str, Any]] = Field( - None, - description="Body of the request sent to get a new access token.", - examples=[ - { - "applicationId": "{{ config['application_id'] }}", - "applicationSecret": "{{ config['application_secret'] }}", - "token": "{{ config['token'] }}", - } - ], - title="Refresh Request Body", - ) - scopes: Optional[List[str]] = Field( + refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], - title="Scopes", + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", + title="Token Updater", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -436,6 +365,10 @@ class ExponentialBackoffStrategy(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class SessionTokenRequestBearerAuthenticator(BaseModel): + type: Literal["Bearer"] + + class HttpMethodEnum(Enum): GET = "GET" POST = "POST" @@ -520,8 +453,8 @@ class MinMaxDatetime(BaseModel): ) datetime_format: Optional[str] = Field( "", - description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use %s if the datetime value is in epoch time (Unix timestamp).', - examples=["%Y-%m-%dT%H:%M:%S.%f%", "%Y-%m-%d", "%s"], + description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], title="Datetime Format", ) max_datetime: Optional[str] = Field( @@ -685,7 +618,7 @@ class RequestOption(BaseModel): ) inject_into: InjectInto = Field( ..., - description="Configures where the descriptor should be set on the HTTP requests.", + description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", examples=["request_parameter", "header", "body_data", "body_json"], title="Inject Into", ) @@ -698,8 +631,8 @@ class Config: extra = Extra.allow -class SessionTokenAuthenticator(BaseModel): - type: Literal["SessionTokenAuthenticator"] +class LegacySessionTokenAuthenticator(BaseModel): + type: Literal["LegacySessionTokenAuthenticator"] header: str = Field( ..., description="The name of the session token header that will be injected in the request", @@ -785,6 +718,32 @@ class WaitUntilTimeFromHeader(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class ApiKeyAuthenticator(BaseModel): + type: Literal["ApiKeyAuthenticator"] + api_token: Optional[str] = Field( + None, + description="The API key to inject in the request. Fill it in the user inputs.", + examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], + title="API Key", + ) + header: Optional[str] = Field( + None, + description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", + examples=["Authorization", "Api-Token", "X-Auth-Token"], + title="Header Name", + ) + inject_into: Optional[RequestOption] = Field( + None, + description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", + examples=[ + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, + ], + title="Inject API Key Into Outgoing HTTP Request", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class AuthFlow(BaseModel): auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type") predicate_key: Optional[List[str]] = Field( @@ -847,21 +806,9 @@ class DatetimeBasedCursor(BaseModel): ) datetime_format: str = Field( ..., - description="The datetime format of the Cursor Field.", - examples=["%Y-%m-%dT%H:%M:%S.%f%z"], - title="Cursor Field Datetime Format", - ) - cursor_granularity: str = Field( - ..., - description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S.", - examples=["PT1S"], - title="Cursor Granularity", - ) - end_datetime: Union[str, MinMaxDatetime] = Field( - ..., - description="The datetime that determines the last record that should be synced.", - examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], - title="End Datetime", + description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms"], + title="Outgoing Datetime Format", ) start_datetime: Union[str, MinMaxDatetime] = Field( ..., @@ -869,17 +816,33 @@ class DatetimeBasedCursor(BaseModel): examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], title="Start Datetime", ) - step: str = Field( - ..., - description="The size of the time window (ISO8601 duration).", - examples=["P1W", "{{ config['step_increment'] }}"], - title="Step", + cursor_datetime_formats: Optional[List[str]] = Field( + None, + description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", + title="Cursor Datetime Formats", + ) + cursor_granularity: Optional[str] = Field( + None, + description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", + examples=["PT1S"], + title="Cursor Granularity", + ) + end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( + None, + description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", + examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], + title="End Datetime", ) end_time_option: Optional[RequestOption] = Field( None, description="Optionally configures how the end datetime will be sent in requests to the source API.", title="Inject End Time Into Outgoing HTTP Request", ) + is_data_feed: Optional[bool] = Field( + None, + description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", + title="Whether the target API is formatted as a data feed", + ) lookback_window: Optional[str] = Field( None, description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", @@ -903,6 +866,12 @@ class DatetimeBasedCursor(BaseModel): description="Optionally configures how the start datetime will be sent in requests to the source API.", title="Inject Start Time Into Outgoing HTTP Request", ) + step: Optional[str] = Field( + None, + description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", + examples=["P1W", "{{ config['step_increment'] }}"], + title="Step", + ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -958,7 +927,7 @@ class DpathExtractor(BaseModel): type: Literal["DpathExtractor"] field_path: List[str] = Field( ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array.', + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ ["data"], ["data", "records"], @@ -975,6 +944,19 @@ class DpathExtractor(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class SessionTokenRequestApiKeyAuthenticator(BaseModel): + type: Literal["ApiKey"] + inject_into: RequestOption = Field( + ..., + description="Configure how the API Key will be sent in requests to the source API.", + examples=[ + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, + ], + title="Inject API Key Into Outgoing HTTP Request", + ) + + class ListPartitionRouter(BaseModel): type: Literal["ListPartitionRouter"] cursor_field: str = Field( @@ -1038,6 +1020,92 @@ class CompositeErrorHandler(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class DeclarativeSource(BaseModel): + class Config: + extra = Extra.forbid + + type: Literal["DeclarativeSource"] + check: CheckStream + streams: List[DeclarativeStream] + version: str + schemas: Optional[Schemas] = None + definitions: Optional[Dict[str, Any]] = None + spec: Optional[Spec] = None + metadata: Optional[Dict[str, Any]] = Field( + None, + description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", + ) + + +class DeclarativeStream(BaseModel): + class Config: + extra = Extra.allow + + type: Literal["DeclarativeStream"] + retriever: Union[CustomRetriever, SimpleRetriever] = Field( + ..., + description="Component used to coordinate how records are extracted across stream slices and request pages.", + title="Retriever", + ) + incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", + ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") + schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field( + None, + description="Component used to retrieve the schema for the current stream.", + title="Schema Loader", + ) + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( + None, + description="A list of transformations to be applied to each output record.", + title="Transformations", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class SessionTokenAuthenticator(BaseModel): + type: Literal["SessionTokenAuthenticator"] + login_requester: HttpRequester = Field( + ..., + description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", + examples=[ + { + "type": "HttpRequester", + "url_base": "https://my_api.com", + "path": "/login", + "authenticator": { + "type": "BasicHttpAuthenticator", + "username": "{{ config.username }}", + "password": "{{ config.password }}", + }, + } + ], + title="Login Requester", + ) + session_token_path: List[str] = Field( + ..., + description="The path in the response body returned from the login requester to the session token.", + examples=[["access_token"], ["result", "token"]], + title="Session Token Path", + ) + expiration_duration: Optional[str] = Field( + None, + description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", + examples=["PT1H", "P1D"], + title="Expiration Duration", + ) + request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field( + ..., + description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", + title="Data Request Authentication", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class HttpRequester(BaseModel): type: Literal["HttpRequester"] url_base: str = Field( @@ -1066,9 +1134,9 @@ class HttpRequester(BaseModel): BearerAuthenticator, CustomAuthenticator, OAuthAuthenticator, - SingleUseRefreshTokenOAuthAuthenticator, NoAuth, SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, ] ] = Field( None, @@ -1088,7 +1156,7 @@ class HttpRequester(BaseModel): ) request_body_data: Optional[Union[str, Dict[str, str]]] = Field( None, - description="Specifies how to populate the body of the request with a non-JSON payload. If returns a ready text that it will be sent as is. If returns a dict that it will be converted to a urlencoded form.", + description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", examples=[ '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' ], @@ -1126,49 +1194,6 @@ class HttpRequester(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class DeclarativeSource(BaseModel): - class Config: - extra = Extra.forbid - - type: Literal["DeclarativeSource"] - check: CheckStream - streams: List[DeclarativeStream] - version: str - schemas: Optional[Schemas] = None - definitions: Optional[Dict[str, Any]] = None - spec: Optional[Spec] = None - - -class DeclarativeStream(BaseModel): - class Config: - extra = Extra.allow - - type: Literal["DeclarativeStream"] - retriever: Union[CustomRetriever, SimpleRetriever] = Field( - ..., - description="Component used to coordinate how records are extracted across stream slices and request pages.", - title="Retriever", - ) - incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", - ) - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") - primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") - schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field( - None, - description="Component used to retrieve the schema for the current stream.", - title="Schema Loader", - ) - transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( - None, - description="A list of transformations to be applied to each output record.", - title="Transformations", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ParentStreamConfig(BaseModel): type: Literal["ParentStreamConfig"] parent_key: str = Field( @@ -1234,4 +1259,5 @@ class SubstreamPartitionRouter(BaseModel): CompositeErrorHandler.update_forward_refs() DeclarativeSource.update_forward_refs() DeclarativeStream.update_forward_refs() +SessionTokenAuthenticator.update_forward_refs() SimpleRetriever.update_forward_refs() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 02e84c555e72..4025b752ed88 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -7,23 +7,27 @@ import importlib import inspect import re -from typing import Any, Callable, List, Literal, Mapping, Optional, Type, Union, get_args, get_origin, get_type_hints +from typing import Any, Callable, List, Mapping, Optional, Type, Union, get_args, get_origin, get_type_hints +from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator from airbyte_cdk.sources.declarative.auth.token import ( ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, - SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, ) +from airbyte_cdk.sources.declarative.auth.token_provider import InterpolatedStringTokenProvider, SessionTokenProvider, TokenProvider from airbyte_cdk.sources.declarative.checks import CheckStream from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders import JsonDecoder from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.incremental import Cursor, CursorFactory, DatetimeBasedCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString +from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping from airbyte_cdk.sources.declarative.models.declarative_component_schema import AddedFieldDefinition as AddedFieldDefinitionModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import AddFields as AddFieldsModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import ApiKeyAuthenticator as ApiKeyAuthenticatorModel @@ -56,6 +60,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import InlineSchemaLoader as InlineSchemaLoaderModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import JsonDecoder as JsonDecoderModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import JsonFileSchemaLoader as JsonFileSchemaLoaderModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + LegacySessionTokenAuthenticator as LegacySessionTokenAuthenticatorModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ListPartitionRouter as ListPartitionRouterModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import MinMaxDatetime as MinMaxDatetimeModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import NoAuth as NoAuthModel @@ -71,9 +78,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestPath as RequestPathModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SessionTokenAuthenticator as SessionTokenAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SimpleRetriever as SimpleRetrieverModel -from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( - SingleUseRefreshTokenOAuthAuthenticator as SingleUseRefreshTokenOAuthAuthenticatorModel, -) from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SubstreamPartitionRouter as SubstreamPartitionRouterModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import WaitTimeFromHeader as WaitTimeFromHeaderModel @@ -90,7 +94,13 @@ ) from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator, NoPagination, PaginatorTestReadDecorator -from airbyte_cdk.sources.declarative.requesters.paginators.strategies import CursorPaginationStrategy, OffsetIncrement, PageIncrement +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( + CursorPaginationStrategy, + CursorStopCondition, + OffsetIncrement, + PageIncrement, + StopConditionPaginationStrategyDecorator, +) from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath @@ -98,13 +108,14 @@ from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.spec import Spec from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer, StreamSlicer -from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields +from airbyte_cdk.sources.declarative.transformations import AddFields, RecordTransformation, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from airbyte_cdk.sources.declarative.types import Config -from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator +from airbyte_cdk.sources.message import InMemoryMessageRepository, LogAppenderMessageRepositoryDecorator, MessageRepository +from isodate import parse_duration from pydantic import BaseModel -ComponentDefinition: Union[Literal, Mapping, List] +ComponentDefinition = Mapping[str, Any] DEFAULT_BACKOFF_STRATEGY = ExponentialBackoffStrategy @@ -113,19 +124,23 @@ class ModelToComponentFactory: def __init__( self, - limit_pages_fetched_per_slice: int = None, - limit_slices_fetched: int = None, + limit_pages_fetched_per_slice: Optional[int] = None, + limit_slices_fetched: Optional[int] = None, emit_connector_builder_messages: bool = False, - disable_retries=False, + disable_retries: bool = False, + message_repository: Optional[MessageRepository] = None, ): self._init_mappings() self._limit_pages_fetched_per_slice = limit_pages_fetched_per_slice self._limit_slices_fetched = limit_slices_fetched self._emit_connector_builder_messages = emit_connector_builder_messages self._disable_retries = disable_retries + self._message_repository = message_repository or InMemoryMessageRepository( # type: ignore + self._evaluate_log_level(emit_connector_builder_messages) + ) - def _init_mappings(self): - self.PYDANTIC_MODEL_TO_CONSTRUCTOR: [Type[BaseModel], Callable] = { + def _init_mappings(self) -> None: + self.PYDANTIC_MODEL_TO_CONSTRUCTOR: Mapping[Type[BaseModel], Callable[..., Any]] = { AddedFieldDefinitionModel: self.create_added_field_definition, AddFieldsModel: self.create_add_fields, ApiKeyAuthenticatorModel: self.create_api_key_authenticator, @@ -151,6 +166,7 @@ def _init_mappings(self): DefaultPaginatorModel: self.create_default_paginator, DpathExtractorModel: self.create_dpath_extractor, ExponentialBackoffStrategyModel: self.create_exponential_backoff_strategy, + SessionTokenAuthenticatorModel: self.create_session_token_authenticator, HttpRequesterModel: self.create_http_requester, HttpResponseFilterModel: self.create_http_response_filter, InlineSchemaLoaderModel: self.create_inline_schema_loader, @@ -161,7 +177,6 @@ def _init_mappings(self): NoAuthModel: self.create_no_auth, NoPaginationModel: self.create_no_pagination, OAuthAuthenticatorModel: self.create_oauth_authenticator, - SingleUseRefreshTokenOAuthAuthenticatorModel: self.create_single_use_refresh_token_oauth_authenticator, OffsetIncrementModel: self.create_offset_increment, PageIncrementModel: self.create_page_increment, ParentStreamConfigModel: self.create_parent_stream_config, @@ -170,7 +185,7 @@ def _init_mappings(self): RemoveFieldsModel: self.create_remove_fields, RequestPathModel: self.create_request_path, RequestOptionModel: self.create_request_option, - SessionTokenAuthenticatorModel: self.create_session_token_authenticator, + LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, SimpleRetrieverModel: self.create_simple_retriever, SpecModel: self.create_spec, SubstreamPartitionRouterModel: self.create_substream_partition_router, @@ -181,7 +196,9 @@ def _init_mappings(self): # Needed for the case where we need to perform a second parse on the fields of a custom component self.TYPE_NAME_TO_MODEL = {cls.__name__: cls for cls in self.PYDANTIC_MODEL_TO_CONSTRUCTOR} - def create_component(self, model_type: Type[BaseModel], component_definition: ComponentDefinition, config: Config, **kwargs) -> type: + def create_component( + self, model_type: Type[BaseModel], component_definition: ComponentDefinition, config: Config, **kwargs: Any + ) -> Any: """ Takes a given Pydantic model type and Mapping representing a component definition and creates a declarative component and subcomponents which will be used at runtime. This is done by first parsing the mapping into a Pydantic model and then creating @@ -204,63 +221,126 @@ def create_component(self, model_type: Type[BaseModel], component_definition: Co return self._create_component_from_model(model=declarative_component_model, config=config, **kwargs) - def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs) -> Any: + def _create_component_from_model(self, model: BaseModel, config: Config, **kwargs: Any) -> Any: if model.__class__ not in self.PYDANTIC_MODEL_TO_CONSTRUCTOR: raise ValueError(f"{model.__class__} with attributes {model} is not a valid component type") component_constructor = self.PYDANTIC_MODEL_TO_CONSTRUCTOR.get(model.__class__) + if not component_constructor: + raise ValueError(f"Could not find constructor for {model.__class__}") return component_constructor(model=model, config=config, **kwargs) @staticmethod - def create_added_field_definition(model: AddedFieldDefinitionModel, config: Config, **kwargs) -> AddedFieldDefinition: - interpolated_value = InterpolatedString.create(model.value, parameters=model.parameters) - return AddedFieldDefinition(path=model.path, value=interpolated_value, parameters=model.parameters) + def create_added_field_definition(model: AddedFieldDefinitionModel, config: Config, **kwargs: Any) -> AddedFieldDefinition: + interpolated_value = InterpolatedString.create(model.value, parameters=model.parameters or {}) + return AddedFieldDefinition(path=model.path, value=interpolated_value, parameters=model.parameters or {}) - def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs) -> AddFields: + def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: added_field_definitions = [ self._create_component_from_model(model=added_field_definition_model, config=config) for added_field_definition_model in model.fields ] - return AddFields(fields=added_field_definitions, parameters=model.parameters) + return AddFields(fields=added_field_definitions, parameters=model.parameters or {}) @staticmethod - def create_api_key_authenticator(model: ApiKeyAuthenticatorModel, config: Config, **kwargs) -> ApiKeyAuthenticator: - return ApiKeyAuthenticator(api_token=model.api_token, header=model.header, config=config, parameters=model.parameters) + def create_api_key_authenticator( + model: ApiKeyAuthenticatorModel, config: Config, token_provider: Optional[TokenProvider] = None, **kwargs: Any + ) -> ApiKeyAuthenticator: + if model.inject_into is None and model.header is None: + raise ValueError("Expected either inject_into or header to be set for ApiKeyAuthenticator") + + if model.inject_into is not None and model.header is not None: + raise ValueError("inject_into and header cannot be set both for ApiKeyAuthenticator - remove the deprecated header option") + + if token_provider is not None and model.api_token != "": + raise ValueError("If token_provider is set, api_token is ignored and has to be set to empty string.") + + request_option = ( + RequestOption( + inject_into=RequestOptionType(model.inject_into.inject_into.value), + field_name=model.inject_into.field_name, + parameters=model.parameters or {}, + ) + if model.inject_into + else RequestOption( + inject_into=RequestOptionType.header, + field_name=model.header or "", + parameters=model.parameters or {}, + ) + ) + return ApiKeyAuthenticator( + token_provider=token_provider + if token_provider is not None + else InterpolatedStringTokenProvider(api_token=model.api_token or "", config=config, parameters=model.parameters or {}), + request_option=request_option, + config=config, + parameters=model.parameters or {}, + ) + + def create_session_token_authenticator( + self, model: SessionTokenAuthenticatorModel, config: Config, name: str, **kwargs: Any + ) -> Union[ApiKeyAuthenticator, BearerAuthenticator]: + login_requester = self._create_component_from_model(model=model.login_requester, config=config, name=f"{name}_login_requester") + token_provider = SessionTokenProvider( + login_requester=login_requester, + session_token_path=model.session_token_path, + expiration_duration=parse_duration(model.expiration_duration) if model.expiration_duration else None, + parameters=model.parameters or {}, + message_repository=self._message_repository, + ) + if model.request_authentication.type == "Bearer": + return ModelToComponentFactory.create_bearer_authenticator( + BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), config, token_provider=token_provider + ) + else: + return ModelToComponentFactory.create_api_key_authenticator( + ApiKeyAuthenticatorModel(type="ApiKeyAuthenticator", api_token="", inject_into=model.request_authentication.inject_into), + config=config, + token_provider=token_provider, + ) @staticmethod - def create_basic_http_authenticator(model: BasicHttpAuthenticatorModel, config: Config, **kwargs) -> BasicHttpAuthenticator: - return BasicHttpAuthenticator(password=model.password, username=model.username, config=config, parameters=model.parameters) + def create_basic_http_authenticator(model: BasicHttpAuthenticatorModel, config: Config, **kwargs: Any) -> BasicHttpAuthenticator: + return BasicHttpAuthenticator( + password=model.password or "", username=model.username, config=config, parameters=model.parameters or {} + ) @staticmethod - def create_bearer_authenticator(model: BearerAuthenticatorModel, config: Config, **kwargs) -> BearerAuthenticator: + def create_bearer_authenticator( + model: BearerAuthenticatorModel, config: Config, token_provider: Optional[TokenProvider] = None, **kwargs: Any + ) -> BearerAuthenticator: + if token_provider is not None and model.api_token != "": + raise ValueError("If token_provider is set, api_token is ignored and has to be set to empty string.") return BearerAuthenticator( - api_token=model.api_token, + token_provider=token_provider + if token_provider is not None + else InterpolatedStringTokenProvider(api_token=model.api_token or "", config=config, parameters=model.parameters or {}), config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) @staticmethod - def create_check_stream(model: CheckStreamModel, config: Config, **kwargs): + def create_check_stream(model: CheckStreamModel, config: Config, **kwargs: Any) -> CheckStream: return CheckStream(stream_names=model.stream_names, parameters={}) - def create_composite_error_handler(self, model: CompositeErrorHandlerModel, config: Config, **kwargs) -> CompositeErrorHandler: + def create_composite_error_handler(self, model: CompositeErrorHandlerModel, config: Config, **kwargs: Any) -> CompositeErrorHandler: error_handlers = [ self._create_component_from_model(model=error_handler_model, config=config) for error_handler_model in model.error_handlers ] - return CompositeErrorHandler(error_handlers=error_handlers, parameters=model.parameters) + return CompositeErrorHandler(error_handlers=error_handlers, parameters=model.parameters or {}) @staticmethod - def create_constant_backoff_strategy(model: ConstantBackoffStrategyModel, config: Config, **kwargs) -> ConstantBackoffStrategy: + def create_constant_backoff_strategy(model: ConstantBackoffStrategyModel, config: Config, **kwargs: Any) -> ConstantBackoffStrategy: return ConstantBackoffStrategy( backoff_time_in_seconds=model.backoff_time_in_seconds, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) - def create_cursor_pagination(self, model: CursorPaginationModel, config: Config, **kwargs) -> CursorPaginationStrategy: + def create_cursor_pagination(self, model: CursorPaginationModel, config: Config, **kwargs: Any) -> CursorPaginationStrategy: if model.decoder: decoder = self._create_component_from_model(model=model.decoder, config=config) else: - decoder = JsonDecoder(parameters=model.parameters) + decoder = JsonDecoder(parameters=model.parameters or {}) return CursorPaginationStrategy( cursor_value=model.cursor_value, @@ -268,10 +348,10 @@ def create_cursor_pagination(self, model: CursorPaginationModel, config: Config, page_size=model.page_size, stop_condition=model.stop_condition, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) - def create_custom_component(self, model, config: Config, **kwargs) -> type: + def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> Any: """ Generically creates a custom component based on the model type and a class_name reference to the custom Python class being instantiated. Only the model's additional properties that match the custom class definition are passed to the constructor @@ -319,14 +399,14 @@ def create_custom_component(self, model, config: Config, **kwargs) -> type: return custom_component_class(**kwargs) @staticmethod - def _get_class_from_fully_qualified_class_name(class_name: str) -> type: + def _get_class_from_fully_qualified_class_name(class_name: str) -> Any: split = class_name.split(".") module = ".".join(split[:-1]) class_name = split[-1] return getattr(importlib.import_module(module), class_name) @staticmethod - def _derive_component_type_from_type_hints(field_type: str) -> Optional[str]: + def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: interface = field_type while True: origin = get_origin(interface) @@ -343,7 +423,7 @@ def _derive_component_type_from_type_hints(field_type: str) -> Optional[str]: return None @staticmethod - def is_builtin_type(cls) -> bool: + def is_builtin_type(cls: Optional[Type[Any]]) -> bool: if not cls: return False return cls.__module__ == "builtins" @@ -356,7 +436,7 @@ def _extract_missing_parameters(error: TypeError) -> List[str]: else: return [] - def _create_nested_component(self, model, model_field: str, model_value: Any, config: Config) -> Any: + def _create_nested_component(self, model: Any, model_field: str, model_value: Any, config: Config) -> Any: type_name = model_value.get("type", None) if not type_name: # If no type is specified, we can assume this is a dictionary object which can be returned instead of a subcomponent @@ -392,21 +472,25 @@ def _create_nested_component(self, model, model_field: str, model_value: Any, co @staticmethod def _is_component(model_value: Any) -> bool: - return isinstance(model_value, dict) and model_value.get("type") + return isinstance(model_value, dict) and model_value.get("type") is not None - def create_datetime_based_cursor(self, model: DatetimeBasedCursorModel, config: Config, **kwargs) -> DatetimeBasedCursor: - start_datetime = ( + def create_datetime_based_cursor(self, model: DatetimeBasedCursorModel, config: Config, **kwargs: Any) -> DatetimeBasedCursor: + start_datetime: Union[str, MinMaxDatetime] = ( model.start_datetime if isinstance(model.start_datetime, str) else self.create_min_max_datetime(model.start_datetime, config) ) - end_datetime = ( - model.end_datetime if isinstance(model.end_datetime, str) else self.create_min_max_datetime(model.end_datetime, config) - ) + end_datetime: Union[str, MinMaxDatetime, None] = None + if model.is_data_feed and model.end_datetime: + raise ValueError("Data feed does not support end_datetime") + if model.end_datetime: + end_datetime = ( + model.end_datetime if isinstance(model.end_datetime, str) else self.create_min_max_datetime(model.end_datetime, config) + ) end_time_option = ( RequestOption( inject_into=RequestOptionType(model.end_time_option.inject_into.value), field_name=model.end_time_option.field_name, - parameters=model.parameters, + parameters=model.parameters or {}, ) if model.end_time_option else None @@ -415,7 +499,7 @@ def create_datetime_based_cursor(self, model: DatetimeBasedCursorModel, config: RequestOption( inject_into=RequestOptionType(model.start_time_option.inject_into.value), field_name=model.start_time_option.field_name, - parameters=model.parameters, + parameters=model.parameters or {}, ) if model.start_time_option else None @@ -423,6 +507,7 @@ def create_datetime_based_cursor(self, model: DatetimeBasedCursorModel, config: return DatetimeBasedCursor( cursor_field=model.cursor_field, + cursor_datetime_formats=model.cursor_datetime_formats if model.cursor_datetime_formats else [], cursor_granularity=model.cursor_granularity, datetime_format=model.datetime_format, end_datetime=end_datetime, @@ -433,11 +518,12 @@ def create_datetime_based_cursor(self, model: DatetimeBasedCursorModel, config: start_time_option=start_time_option, partition_field_end=model.partition_field_end, partition_field_start=model.partition_field_start, + message_repository=self._message_repository, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) - def create_declarative_stream(self, model: DeclarativeStreamModel, config: Config, **kwargs) -> DeclarativeStream: + def create_declarative_stream(self, model: DeclarativeStreamModel, config: Config, **kwargs: Any) -> DeclarativeStream: # When constructing a declarative stream, we assemble the incremental_sync component and retriever's partition_router field # components if they exist into a single CartesianProductStreamSlicer. This is then passed back as an argument when constructing the # Retriever. This is done in the declarative stream not the retriever to support custom retrievers. The custom create methods in @@ -445,10 +531,22 @@ def create_declarative_stream(self, model: DeclarativeStreamModel, config: Confi combined_slicers = self._merge_stream_slicers(model=model, config=config) primary_key = model.primary_key.__root__ if model.primary_key else None + stop_condition_on_cursor = ( + model.incremental_sync and hasattr(model.incremental_sync, "is_data_feed") and model.incremental_sync.is_data_feed + ) + transformations = [] + if model.transformations: + for transformation_model in model.transformations: + transformations.append(self._create_component_from_model(model=transformation_model, config=config)) retriever = self._create_component_from_model( - model=model.retriever, config=config, name=model.name, primary_key=primary_key, stream_slicer=combined_slicers + model=model.retriever, + config=config, + name=model.name, + primary_key=primary_key, + stream_slicer=combined_slicers, + stop_condition_on_cursor=stop_condition_on_cursor, + transformations=transformations, ) - cursor_field = model.incremental_sync.cursor_field if model.incremental_sync else None if model.schema_loader: @@ -459,53 +557,49 @@ def create_declarative_stream(self, model: DeclarativeStreamModel, config: Confi options["name"] = model.name schema_loader = DefaultSchemaLoader(config=config, parameters=options) - transformations = [] - if model.transformations: - for transformation_model in model.transformations: - transformations.append(self._create_component_from_model(model=transformation_model, config=config)) return DeclarativeStream( - name=model.name, + name=model.name or "", primary_key=primary_key, retriever=retriever, schema_loader=schema_loader, stream_cursor_field=cursor_field or "", - transformations=transformations, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) def _merge_stream_slicers(self, model: DeclarativeStreamModel, config: Config) -> Optional[StreamSlicer]: - incremental_sync = ( - self._create_component_from_model(model=model.incremental_sync, config=config) if model.incremental_sync else None - ) - stream_slicer = None if hasattr(model.retriever, "partition_router") and model.retriever.partition_router: stream_slicer_model = model.retriever.partition_router - stream_slicer = ( - CartesianProductStreamSlicer( + if isinstance(stream_slicer_model, list): + stream_slicer = CartesianProductStreamSlicer( [self._create_component_from_model(model=slicer, config=config) for slicer in stream_slicer_model], parameters={} ) - if type(stream_slicer_model) == list - else self._create_component_from_model(model=stream_slicer_model, config=config) + else: + stream_slicer = self._create_component_from_model(model=stream_slicer_model, config=config) + + if model.incremental_sync and stream_slicer: + incremental_sync_model = model.incremental_sync + return PerPartitionCursor( + cursor_factory=CursorFactory( + lambda: self._create_component_from_model(model=incremental_sync_model, config=config), + ), + partition_router=stream_slicer, ) - - if incremental_sync and stream_slicer: - return CartesianProductStreamSlicer(stream_slicers=[incremental_sync, stream_slicer], parameters=model.parameters) - elif incremental_sync: - return incremental_sync + elif model.incremental_sync: + return self._create_component_from_model(model=model.incremental_sync, config=config) if model.incremental_sync else None elif stream_slicer: return stream_slicer else: return None - def create_default_error_handler(self, model: DefaultErrorHandlerModel, config: Config, **kwargs) -> DefaultErrorHandler: + def create_default_error_handler(self, model: DefaultErrorHandlerModel, config: Config, **kwargs: Any) -> DefaultErrorHandler: backoff_strategies = [] if model.backoff_strategies: for backoff_strategy_model in model.backoff_strategies: backoff_strategies.append(self._create_component_from_model(model=backoff_strategy_model, config=config)) else: - backoff_strategies.append(DEFAULT_BACKOFF_STRATEGY(config=config, parameters=model.parameters)) + backoff_strategies.append(DEFAULT_BACKOFF_STRATEGY(config=config, parameters=model.parameters or {})) response_filters = [] if model.response_filters: @@ -514,20 +608,25 @@ def create_default_error_handler(self, model: DefaultErrorHandlerModel, config: else: response_filters.append( HttpResponseFilter( - ResponseAction.RETRY, http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS, config=config, parameters=model.parameters + ResponseAction.RETRY, + http_codes=HttpResponseFilter.DEFAULT_RETRIABLE_ERRORS, + config=config, + parameters=model.parameters or {}, ) ) - response_filters.append(HttpResponseFilter(ResponseAction.IGNORE, config=config, parameters=model.parameters)) + response_filters.append(HttpResponseFilter(ResponseAction.IGNORE, config=config, parameters=model.parameters or {})) return DefaultErrorHandler( backoff_strategies=backoff_strategies, max_retries=model.max_retries, response_filters=response_filters, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) - def create_default_paginator(self, model: DefaultPaginatorModel, config: Config, *, url_base: str) -> DefaultPaginator: + def create_default_paginator( + self, model: DefaultPaginatorModel, config: Config, *, url_base: str, cursor_used_for_stop_condition: Optional[Cursor] = None + ) -> Union[DefaultPaginator, PaginatorTestReadDecorator]: decoder = self._create_component_from_model(model=model.decoder, config=config) if model.decoder else JsonDecoder(parameters={}) page_size_option = ( self._create_component_from_model(model=model.page_size_option, config=config) if model.page_size_option else None @@ -536,6 +635,10 @@ def create_default_paginator(self, model: DefaultPaginatorModel, config: Config, self._create_component_from_model(model=model.page_token_option, config=config) if model.page_token_option else None ) pagination_strategy = self._create_component_from_model(model=model.pagination_strategy, config=config) + if cursor_used_for_stop_condition: + pagination_strategy = StopConditionPaginationStrategyDecorator( + pagination_strategy, CursorStopCondition(cursor_used_for_stop_condition) + ) paginator = DefaultPaginator( decoder=decoder, @@ -544,30 +647,31 @@ def create_default_paginator(self, model: DefaultPaginatorModel, config: Config, pagination_strategy=pagination_strategy, url_base=url_base, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) if self._limit_pages_fetched_per_slice: return PaginatorTestReadDecorator(paginator, self._limit_pages_fetched_per_slice) return paginator - def create_dpath_extractor(self, model: DpathExtractorModel, config: Config, **kwargs) -> DpathExtractor: + def create_dpath_extractor(self, model: DpathExtractorModel, config: Config, **kwargs: Any) -> DpathExtractor: decoder = self._create_component_from_model(model.decoder, config=config) if model.decoder else JsonDecoder(parameters={}) - return DpathExtractor(decoder=decoder, field_path=model.field_path, config=config, parameters=model.parameters) + model_field_path: List[Union[InterpolatedString, str]] = [x for x in model.field_path] + return DpathExtractor(decoder=decoder, field_path=model_field_path, config=config, parameters=model.parameters or {}) @staticmethod def create_exponential_backoff_strategy(model: ExponentialBackoffStrategyModel, config: Config) -> ExponentialBackoffStrategy: - return ExponentialBackoffStrategy(factor=model.factor, parameters=model.parameters, config=config) + return ExponentialBackoffStrategy(factor=model.factor or 5, parameters=model.parameters or {}, config=config) def create_http_requester(self, model: HttpRequesterModel, config: Config, *, name: str) -> HttpRequester: authenticator = ( - self._create_component_from_model(model=model.authenticator, config=config, url_base=model.url_base) + self._create_component_from_model(model=model.authenticator, config=config, url_base=model.url_base, name=name) if model.authenticator else None ) error_handler = ( self._create_component_from_model(model=model.error_handler, config=config) if model.error_handler - else DefaultErrorHandler(backoff_strategies=[], response_filters=[], config=config, parameters=model.parameters) + else DefaultErrorHandler(backoff_strategies=[], response_filters=[], config=config, parameters=model.parameters or {}) ) request_options_provider = InterpolatedRequestOptionsProvider( @@ -576,7 +680,11 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na request_headers=model.request_headers, request_parameters=model.request_parameters, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, + ) + + model_http_method = ( + model.http_method if isinstance(model.http_method, str) else model.http_method.value if model.http_method is not None else "GET" ) return HttpRequester( @@ -585,14 +693,16 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na path=model.path, authenticator=authenticator, error_handler=error_handler, - http_method=model.http_method, + http_method=model_http_method, request_options_provider=request_options_provider, config=config, - parameters=model.parameters, + disable_retries=self._disable_retries, + parameters=model.parameters or {}, + message_repository=self._message_repository, ) @staticmethod - def create_http_response_filter(model: HttpResponseFilterModel, config: Config, **kwargs) -> HttpResponseFilter: + def create_http_response_filter(model: HttpResponseFilterModel, config: Config, **kwargs: Any) -> HttpResponseFilter: action = ResponseAction(model.action.value) http_codes = ( set(model.http_codes) if model.http_codes else set() @@ -601,32 +711,32 @@ def create_http_response_filter(model: HttpResponseFilterModel, config: Config, return HttpResponseFilter( action=action, error_message=model.error_message or "", - error_message_contains=model.error_message_contains, + error_message_contains=model.error_message_contains or "", http_codes=http_codes, predicate=model.predicate or "", config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) @staticmethod - def create_inline_schema_loader(model: InlineSchemaLoaderModel, config: Config, **kwargs) -> InlineSchemaLoader: - return InlineSchemaLoader(schema=model.schema_, parameters={}) + def create_inline_schema_loader(model: InlineSchemaLoaderModel, config: Config, **kwargs: Any) -> InlineSchemaLoader: + return InlineSchemaLoader(schema=model.schema_ or {}, parameters={}) @staticmethod - def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs) -> JsonDecoder: + def create_json_decoder(model: JsonDecoderModel, config: Config, **kwargs: Any) -> JsonDecoder: return JsonDecoder(parameters={}) @staticmethod - def create_json_file_schema_loader(model: JsonFileSchemaLoaderModel, config: Config, **kwargs) -> JsonFileSchemaLoader: - return JsonFileSchemaLoader(file_path=model.file_path, config=config, parameters=model.parameters) + def create_json_file_schema_loader(model: JsonFileSchemaLoaderModel, config: Config, **kwargs: Any) -> JsonFileSchemaLoader: + return JsonFileSchemaLoader(file_path=model.file_path or "", config=config, parameters=model.parameters or {}) @staticmethod - def create_list_partition_router(model: ListPartitionRouterModel, config: Config, **kwargs) -> ListPartitionRouter: + def create_list_partition_router(model: ListPartitionRouterModel, config: Config, **kwargs: Any) -> ListPartitionRouter: request_option = ( RequestOption( inject_into=RequestOptionType(model.request_option.inject_into.value), field_name=model.request_option.field_name, - parameters=model.parameters, + parameters=model.parameters or {}, ) if model.request_option else None @@ -636,74 +746,78 @@ def create_list_partition_router(model: ListPartitionRouterModel, config: Config request_option=request_option, values=model.values, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) @staticmethod - def create_min_max_datetime(model: MinMaxDatetimeModel, config: Config, **kwargs) -> MinMaxDatetime: + def create_min_max_datetime(model: MinMaxDatetimeModel, config: Config, **kwargs: Any) -> MinMaxDatetime: return MinMaxDatetime( datetime=model.datetime, - datetime_format=model.datetime_format, - max_datetime=model.max_datetime, - min_datetime=model.min_datetime, - parameters=model.parameters, + datetime_format=model.datetime_format or "", + max_datetime=model.max_datetime or "", + min_datetime=model.min_datetime or "", + parameters=model.parameters or {}, ) @staticmethod - def create_no_auth(model: NoAuthModel, config: Config, **kwargs) -> NoAuth: - return NoAuth(parameters=model.parameters) + def create_no_auth(model: NoAuthModel, config: Config, **kwargs: Any) -> NoAuth: + return NoAuth(parameters=model.parameters or {}) @staticmethod - def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs) -> NoPagination: + def create_no_pagination(model: NoPaginationModel, config: Config, **kwargs: Any) -> NoPagination: return NoPagination(parameters={}) - @staticmethod - def create_oauth_authenticator(model: OAuthAuthenticatorModel, config: Config, **kwargs) -> DeclarativeOauth2Authenticator: - return DeclarativeOauth2Authenticator( - access_token_name=model.access_token_name, + def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeOauth2Authenticator: + if model.refresh_token_updater: + # ignore type error beause fixing it would have a lot of dependencies, revisit later + return DeclarativeSingleUseRefreshTokenOauth2Authenticator( # type: ignore + config, + InterpolatedString.create(model.token_refresh_endpoint, parameters=model.parameters or {}).eval(config), + access_token_name=InterpolatedString.create( + model.access_token_name or "access_token", parameters=model.parameters or {} + ).eval(config), + refresh_token_name=model.refresh_token_updater.refresh_token_name, + expires_in_name=InterpolatedString.create(model.expires_in_name or "expires_in", parameters=model.parameters or {}).eval( + config + ), + client_id=InterpolatedString.create(model.client_id, parameters=model.parameters or {}).eval(config), + client_secret=InterpolatedString.create(model.client_secret, parameters=model.parameters or {}).eval(config), + access_token_config_path=model.refresh_token_updater.access_token_config_path, + refresh_token_config_path=model.refresh_token_updater.refresh_token_config_path, + token_expiry_date_config_path=model.refresh_token_updater.token_expiry_date_config_path, + grant_type=InterpolatedString.create(model.grant_type or "refresh_token", parameters=model.parameters or {}).eval(config), + refresh_request_body=InterpolatedMapping(model.refresh_request_body or {}, parameters=model.parameters or {}).eval(config), + scopes=model.scopes, + token_expiry_date_format=model.token_expiry_date_format, + message_repository=self._message_repository, + ) + # ignore type error beause fixing it would have a lot of dependencies, revisit later + return DeclarativeOauth2Authenticator( # type: ignore + access_token_name=model.access_token_name or "access_token", client_id=model.client_id, client_secret=model.client_secret, - expires_in_name=model.expires_in_name, - grant_type=model.grant_type, + expires_in_name=model.expires_in_name or "expires_in", + grant_type=model.grant_type or "refresh_token", refresh_request_body=model.refresh_request_body, refresh_token=model.refresh_token, scopes=model.scopes, token_expiry_date=model.token_expiry_date, - token_expiry_date_format=model.token_expiry_date_format, + token_expiry_date_format=model.token_expiry_date_format, # type: ignore token_refresh_endpoint=model.token_refresh_endpoint, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, + message_repository=self._message_repository, ) @staticmethod - def create_single_use_refresh_token_oauth_authenticator( - model: SingleUseRefreshTokenOAuthAuthenticatorModel, config: Config, **kwargs - ) -> SingleUseRefreshTokenOauth2Authenticator: - return SingleUseRefreshTokenOauth2Authenticator( - config, - model.token_refresh_endpoint, - access_token_name=model.access_token_name, - refresh_token_name=model.refresh_token_name, - expires_in_name=model.expires_in_name, - client_id_config_path=model.client_id_config_path, - client_secret_config_path=model.client_secret_config_path, - access_token_config_path=model.access_token_config_path, - refresh_token_config_path=model.refresh_token_config_path, - token_expiry_date_config_path=model.token_expiry_date_config_path, - grant_type=model.grant_type, - refresh_request_body=model.refresh_request_body, - scopes=model.scopes, - ) + def create_offset_increment(model: OffsetIncrementModel, config: Config, **kwargs: Any) -> OffsetIncrement: + return OffsetIncrement(page_size=model.page_size, config=config, parameters=model.parameters or {}) @staticmethod - def create_offset_increment(model: OffsetIncrementModel, config: Config, **kwargs) -> OffsetIncrement: - return OffsetIncrement(page_size=model.page_size, config=config, parameters=model.parameters) + def create_page_increment(model: PageIncrementModel, config: Config, **kwargs: Any) -> PageIncrement: + return PageIncrement(page_size=model.page_size, start_from_page=model.start_from_page or 0, parameters=model.parameters or {}) - @staticmethod - def create_page_increment(model: PageIncrementModel, config: Config, **kwargs) -> PageIncrement: - return PageIncrement(page_size=model.page_size, start_from_page=model.start_from_page, parameters=model.parameters) - - def create_parent_stream_config(self, model: ParentStreamConfigModel, config: Config, **kwargs) -> ParentStreamConfig: + def create_parent_stream_config(self, model: ParentStreamConfigModel, config: Config, **kwargs: Any) -> ParentStreamConfig: declarative_stream = self._create_component_from_model(model.stream, config=config) request_option = self._create_component_from_model(model.request_option, config=config) if model.request_option else None return ParentStreamConfig( @@ -712,47 +826,55 @@ def create_parent_stream_config(self, model: ParentStreamConfigModel, config: Co stream=declarative_stream, partition_field=model.partition_field, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) @staticmethod - def create_record_filter(model: RecordFilterModel, config: Config, **kwargs) -> RecordFilter: - return RecordFilter(condition=model.condition, config=config, parameters=model.parameters) + def create_record_filter(model: RecordFilterModel, config: Config, **kwargs: Any) -> RecordFilter: + return RecordFilter(condition=model.condition or "", config=config, parameters=model.parameters or {}) @staticmethod - def create_request_path(model: RequestPathModel, config: Config, **kwargs) -> RequestPath: + def create_request_path(model: RequestPathModel, config: Config, **kwargs: Any) -> RequestPath: return RequestPath(parameters={}) @staticmethod - def create_request_option(model: RequestOptionModel, config: Config, **kwargs) -> RequestOption: + def create_request_option(model: RequestOptionModel, config: Config, **kwargs: Any) -> RequestOption: inject_into = RequestOptionType(model.inject_into.value) return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={}) - def create_record_selector(self, model: RecordSelectorModel, config: Config, **kwargs) -> RecordSelector: + def create_record_selector( + self, model: RecordSelectorModel, config: Config, *, transformations: List[RecordTransformation], **kwargs: Any + ) -> RecordSelector: extractor = self._create_component_from_model(model=model.extractor, config=config) record_filter = self._create_component_from_model(model.record_filter, config=config) if model.record_filter else None - return RecordSelector(extractor=extractor, record_filter=record_filter, parameters=model.parameters) + return RecordSelector( + extractor=extractor, + config=config, + record_filter=record_filter, + transformations=transformations, + parameters=model.parameters or {}, + ) @staticmethod - def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs) -> RemoveFields: + def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs: Any) -> RemoveFields: return RemoveFields(field_pointers=model.field_pointers, parameters={}) @staticmethod - def create_session_token_authenticator( - model: SessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs - ) -> SessionTokenAuthenticator: - return SessionTokenAuthenticator( + def create_legacy_session_token_authenticator( + model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any + ) -> LegacySessionTokenAuthenticator: + return LegacySessionTokenAuthenticator( api_url=url_base, header=model.header, login_url=model.login_url, - password=model.password, - session_token=model.session_token, - session_token_response_key=model.session_token_response_key, - username=model.username, + password=model.password or "", + session_token=model.session_token or "", + session_token_response_key=model.session_token_response_key or "", + username=model.username or "", validate_session_url=model.validate_session_url, config=config, - parameters=model.parameters, + parameters=model.parameters or {}, ) def create_simple_retriever( @@ -763,12 +885,20 @@ def create_simple_retriever( name: str, primary_key: Optional[Union[str, List[str], List[List[str]]]], stream_slicer: Optional[StreamSlicer], + stop_condition_on_cursor: bool = False, + transformations: List[RecordTransformation], ) -> SimpleRetriever: requester = self._create_component_from_model(model=model.requester, config=config, name=name) - record_selector = self._create_component_from_model(model=model.record_selector, config=config) + record_selector = self._create_component_from_model(model=model.record_selector, config=config, transformations=transformations) url_base = model.requester.url_base if hasattr(model.requester, "url_base") else requester.get_url_base() + stream_slicer = stream_slicer or SinglePartitionRouter(parameters={}) + cursor = stream_slicer if isinstance(stream_slicer, Cursor) else None + + cursor_used_for_stop_condition = cursor if stop_condition_on_cursor else None paginator = ( - self._create_component_from_model(model=model.paginator, config=config, url_base=url_base) + self._create_component_from_model( + model=model.paginator, config=config, url_base=url_base, cursor_used_for_stop_condition=cursor_used_for_stop_condition + ) if model.paginator else NoPagination(parameters={}) ) @@ -780,11 +910,11 @@ def create_simple_retriever( primary_key=primary_key, requester=requester, record_selector=record_selector, - stream_slicer=stream_slicer or SinglePartitionRouter(parameters={}), + stream_slicer=stream_slicer, + cursor=cursor, config=config, - maximum_number_of_slices=self._limit_slices_fetched, - parameters=model.parameters, - disable_retries=self._disable_retries, + maximum_number_of_slices=self._limit_slices_fetched or 5, + parameters=model.parameters or {}, ) return SimpleRetriever( name=name, @@ -792,14 +922,14 @@ def create_simple_retriever( primary_key=primary_key, requester=requester, record_selector=record_selector, - stream_slicer=stream_slicer or SinglePartitionRouter(parameters={}), + stream_slicer=stream_slicer, + cursor=cursor, config=config, - parameters=model.parameters, - disable_retries=self._disable_retries, + parameters=model.parameters or {}, ) @staticmethod - def create_spec(model: SpecModel, config: Config, **kwargs) -> Spec: + def create_spec(model: SpecModel, config: Config, **kwargs: Any) -> Spec: return Spec( connection_specification=model.connection_specification, documentation_url=model.documentation_url, @@ -807,26 +937,48 @@ def create_spec(model: SpecModel, config: Config, **kwargs) -> Spec: parameters={}, ) - def create_substream_partition_router(self, model: SubstreamPartitionRouterModel, config: Config, **kwargs) -> SubstreamPartitionRouter: + def create_substream_partition_router( + self, model: SubstreamPartitionRouterModel, config: Config, **kwargs: Any + ) -> SubstreamPartitionRouter: parent_stream_configs = [] if model.parent_stream_configs: parent_stream_configs.extend( [ - self._create_component_from_model(model=parent_stream_config, config=config) + self._create_message_repository_substream_wrapper(model=parent_stream_config, config=config) for parent_stream_config in model.parent_stream_configs ] ) - return SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters=model.parameters, config=config) + return SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters=model.parameters or {}, config=config) + + def _create_message_repository_substream_wrapper(self, model: ParentStreamConfigModel, config: Config) -> Any: + substream_factory = ModelToComponentFactory( + limit_pages_fetched_per_slice=self._limit_pages_fetched_per_slice, + limit_slices_fetched=self._limit_slices_fetched, + emit_connector_builder_messages=self._emit_connector_builder_messages, + disable_retries=self._disable_retries, + message_repository=LogAppenderMessageRepositoryDecorator( + {"airbyte_cdk": {"stream": {"is_substream": True}}, "http": {"is_auxiliary": True}}, + self._message_repository, + self._evaluate_log_level(self._emit_connector_builder_messages), + ), + ) + return substream_factory._create_component_from_model(model=model, config=config) @staticmethod - def create_wait_time_from_header(model: WaitTimeFromHeaderModel, config: Config, **kwargs) -> WaitTimeFromHeaderBackoffStrategy: - return WaitTimeFromHeaderBackoffStrategy(header=model.header, parameters=model.parameters, config=config, regex=model.regex) + def create_wait_time_from_header(model: WaitTimeFromHeaderModel, config: Config, **kwargs: Any) -> WaitTimeFromHeaderBackoffStrategy: + return WaitTimeFromHeaderBackoffStrategy(header=model.header, parameters=model.parameters or {}, config=config, regex=model.regex) @staticmethod def create_wait_until_time_from_header( - model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs + model: WaitUntilTimeFromHeaderModel, config: Config, **kwargs: Any ) -> WaitUntilTimeFromHeaderBackoffStrategy: return WaitUntilTimeFromHeaderBackoffStrategy( - header=model.header, parameters=model.parameters, config=config, min_wait=model.min_wait, regex=model.regex + header=model.header, parameters=model.parameters or {}, config=config, min_wait=model.min_wait, regex=model.regex ) + + def get_message_repository(self) -> MessageRepository: + return self._message_repository + + def _evaluate_log_level(self, emit_connector_builder_messages: bool) -> Level: + return Level.DEBUG if emit_connector_builder_messages else Level.INFO diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py index 0357db5fed95..9841bbd51dba 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/list_partition_router.py @@ -5,11 +5,10 @@ from dataclasses import InitVar, dataclass from typing import Any, Iterable, List, Mapping, Optional, Union -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer -from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState @dataclass @@ -39,17 +38,6 @@ def __post_init__(self, parameters: Mapping[str, Any]): self._cursor = None - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - # This method is called after the records are processed. - slice_value = stream_slice.get(self.cursor_field.eval(self.config)) - if slice_value and slice_value in self.values: - self._cursor = slice_value - else: - raise ValueError(f"Unexpected stream slice: {slice_value}") - - def get_stream_state(self) -> StreamState: - return {self.cursor_field.eval(self.config): self._cursor} if self._cursor else {} - def get_request_params( self, stream_state: Optional[StreamState] = None, @@ -86,7 +74,7 @@ def get_request_body_json( # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response return self._get_request_option(RequestOptionType.body_json, stream_slice) - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: + def stream_slices(self) -> Iterable[StreamSlice]: return [{self.cursor_field.eval(self.config): slice_value} for slice_value in self.values] def _get_request_option(self, request_option_type: RequestOptionType, stream_slice: StreamSlice): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py index db90fa250a63..4697d114eb1a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/single_partition_router.py @@ -5,9 +5,8 @@ from dataclasses import InitVar, dataclass from typing import Any, Iterable, Mapping, Optional -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState @dataclass @@ -16,12 +15,6 @@ class SinglePartitionRouter(StreamSlicer): parameters: InitVar[Mapping[str, Any]] - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - pass - - def get_stream_state(self) -> StreamState: - return {} - def get_request_params( self, stream_state: Optional[StreamState] = None, @@ -54,5 +47,5 @@ def get_request_body_json( ) -> Mapping[str, Any]: return {} - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[StreamSlice]: + def stream_slices(self) -> Iterable[StreamSlice]: yield dict() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py index 9beda2e1270f..c080e56a49ce 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py @@ -54,19 +54,8 @@ class SubstreamPartitionRouter(StreamSlicer): def __post_init__(self, parameters: Mapping[str, Any]): if not self.parent_stream_configs: raise ValueError("SubstreamPartitionRouter needs at least 1 parent stream") - self._cursor = None self._parameters = parameters - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - # This method is called after the records are processed. - cursor = {} - for parent_stream_config in self.parent_stream_configs: - partition_field = parent_stream_config.partition_field.eval(self.config) - slice_value = stream_slice.get(partition_field) - if slice_value: - cursor.update({partition_field: slice_value}) - self._cursor = cursor - def get_request_params( self, stream_state: Optional[StreamState] = None, @@ -114,10 +103,7 @@ def _get_request_option(self, option_type: RequestOptionType, stream_slice: Stre params.update({parent_config.request_option.field_name: value}) return params - def get_stream_state(self) -> StreamState: - return self._cursor if self._cursor else {} - - def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]: + def stream_slices(self) -> Iterable[StreamSlice]: """ Iterate over each parent stream's record and create a StreamSlice for each record. @@ -139,7 +125,9 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Itera parent_stream = parent_stream_config.stream parent_field = parent_stream_config.parent_key.eval(self.config) stream_state_field = parent_stream_config.partition_field.eval(self.config) - for parent_stream_slice in parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=None, stream_state=stream_state): + for parent_stream_slice in parent_stream.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=None, stream_state=None + ): empty_parent_slice = True parent_slice = parent_stream_slice @@ -152,6 +140,8 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Itera parent_record = parent_record.record.data else: continue + elif isinstance(parent_record, Record): + parent_record = parent_record.data try: stream_state_value = dpath.util.get(parent_record, parent_field) except KeyError: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index 295c2af6a2b5..3887613ee652 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -2,22 +2,34 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import os +import urllib from dataclasses import InitVar, dataclass from functools import lru_cache -from typing import Any, Mapping, MutableMapping, Optional, Union +from typing import Any, Callable, Mapping, MutableMapping, Optional, Union +from urllib.parse import urljoin import requests +from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler +from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( InterpolatedRequestOptionsProvider, ) from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException +from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS +from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler, user_defined_backoff_handler +from airbyte_cdk.utils.mapping_helpers import combine_mappings +from requests.auth import AuthBase @dataclass @@ -41,54 +53,64 @@ class HttpRequester(Requester): path: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] + authenticator: Optional[DeclarativeAuthenticator] = None http_method: Union[str, HttpMethod] = HttpMethod.GET request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None - authenticator: DeclarativeAuthenticator = None error_handler: Optional[ErrorHandler] = None + disable_retries: bool = False + message_repository: MessageRepository = NoopMessageRepository() - def __post_init__(self, parameters: Mapping[str, Any]): - self.url_base = InterpolatedString.create(self.url_base, parameters=parameters) - self.path = InterpolatedString.create(self.path, parameters=parameters) + _DEFAULT_MAX_RETRY = 5 + _DEFAULT_RETRY_FACTOR = 5 + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._url_base = InterpolatedString.create(self.url_base, parameters=parameters) + self._path = InterpolatedString.create(self.path, parameters=parameters) if self.request_options_provider is None: self._request_options_provider = InterpolatedRequestOptionsProvider(config=self.config, parameters=parameters) elif isinstance(self.request_options_provider, dict): self._request_options_provider = InterpolatedRequestOptionsProvider(config=self.config, **self.request_options_provider) else: self._request_options_provider = self.request_options_provider - self.authenticator = self.authenticator or NoAuth(parameters=parameters) - if type(self.http_method) == str: - self.http_method = HttpMethod[self.http_method] - self._method = self.http_method - self.error_handler = self.error_handler or DefaultErrorHandler(parameters=parameters, config=self.config) + self._authenticator = self.authenticator or NoAuth(parameters=parameters) + self._http_method = HttpMethod[self.http_method] if isinstance(self.http_method, str) else self.http_method + self.error_handler = self.error_handler self._parameters = parameters + self.decoder = JsonDecoder(parameters={}) + self._session = requests.Session() + + if isinstance(self._authenticator, AuthBase): + self._session.auth = self._authenticator # We are using an LRU cache in should_retry() method which requires all incoming arguments (including self) to be hashable. # Dataclasses by default are not hashable, so we need to define __hash__(). Alternatively, we can set @dataclass(frozen=True), # but this has a cascading effect where all dataclass fields must also be set to frozen. - def __hash__(self): + def __hash__(self) -> int: return hash(tuple(self.__dict__)) - def get_authenticator(self): - return self.authenticator + def get_authenticator(self) -> DeclarativeAuthenticator: + return self._authenticator - def get_url_base(self): - return os.path.join(self.url_base.eval(self.config), "") + def get_url_base(self) -> str: + return os.path.join(self._url_base.eval(self.config), "") def get_path( self, *, stream_state: Optional[StreamState], stream_slice: Optional[StreamSlice], next_page_token: Optional[Mapping[str, Any]] ) -> str: kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - path = self.path.eval(self.config, **kwargs) + path = str(self._path.eval(self.config, **kwargs)) return path.lstrip("/") - def get_method(self): - return self._method + def get_method(self) -> HttpMethod: + return self._http_method # use a tiny cache to limit the memory footprint. It doesn't have to be large because we mostly # only care about the status of the last response received @lru_cache(maxsize=10) def interpret_response_status(self, response: requests.Response) -> ResponseStatus: # Cache the result because the HttpStream first checks if we should retry before looking at the backoff time + if self.error_handler is None: + raise ValueError("Cannot interpret response status without an error handler") return self.error_handler.interpret_response(response) def get_request_params( @@ -113,45 +135,403 @@ def get_request_headers( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - def get_request_body_data( + # fixing request options provider types has a lot of dependencies + def get_request_body_data( # type: ignore self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Union[Mapping, str]]: - return self._request_options_provider.get_request_body_data( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ) -> Union[Mapping[str, Any], str]: + return ( + self._request_options_provider.get_request_body_data( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ) + or {} ) - def get_request_body_json( + # fixing request options provider types has a lot of dependencies + def get_request_body_json( # type: ignore self, *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Mapping]: + ) -> Optional[Mapping[str, Any]]: return self._request_options_provider.get_request_body_json( stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - def request_kwargs( + @property + def max_retries(self) -> Union[int, None]: + if self.disable_retries: + return 0 + if self.error_handler is None: + return self._DEFAULT_MAX_RETRY + return self.error_handler.max_retries + + @property + def logger(self) -> logging.Logger: + return logging.getLogger(f"airbyte.HttpRequester.{self.name}") + + def _should_retry(self, response: requests.Response) -> bool: + """ + Specifies conditions for backoff based on the response from the server. + + By default, back off on the following HTTP response statuses: + - 429 (Too Many Requests) indicating rate limiting + - 500s to handle transient server errors + + Unexpected but transient exceptions (connection timeout, DNS resolution failed, etc..) are retried by default. + """ + if self.error_handler is None: + return response.status_code == 429 or 500 <= response.status_code < 600 + return bool(self.interpret_response_status(response).action == ResponseAction.RETRY) + + def _backoff_time(self, response: requests.Response) -> Optional[float]: + """ + Specifies backoff time. + + This method is called only if should_backoff() returns True for the input request. + + :param response: + :return how long to backoff in seconds. The return value may be a floating point number for subsecond precision. Returning None defers backoff + to the default backoff behavior (e.g using an exponential algorithm). + """ + if self.error_handler is None: + return None + should_retry = self.interpret_response_status(response) + if should_retry.action != ResponseAction.RETRY: + raise ValueError(f"backoff_time can only be applied on retriable response action. Got {should_retry.action}") + assert should_retry.action == ResponseAction.RETRY + return should_retry.retry_in + + def _error_message(self, response: requests.Response) -> str: + """ + Constructs an error message which can incorporate the HTTP response received from the partner API. + + :param response: The incoming HTTP response from the partner API + :return The error message string to be emitted + """ + return self.interpret_response_status(response).error_message + + def _get_request_options( + self, + stream_state: Optional[StreamState], + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + requester_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], + auth_options_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], + extra_options: Optional[Union[Mapping[str, Any], str]] = None, + ) -> Union[Mapping[str, Any], str]: + """ + Get the request_option from the requester, the authenticator and extra_options passed in. + Raise a ValueError if there's a key collision + Returned merged mapping otherwise + """ + return combine_mappings( + [ + requester_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + auth_options_method(), + extra_options, + ] + ) + + def _request_headers( self, - *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - # todo: there are a few integrations that override the request_kwargs() method, but the use case for why kwargs over existing - # constructs is a little unclear. We may revisit this, but for now lets leave it out of the DSL - return {} + """ + Specifies request headers. + Authentication headers will overwrite any overlapping headers returned from this method. + """ + headers = self._get_request_options( + stream_state, + stream_slice, + next_page_token, + self.get_request_headers, + self.get_authenticator().get_auth_header, + extra_headers, + ) + if isinstance(headers, str): + raise ValueError("Request headers cannot be a string") + return {str(k): str(v) for k, v in headers.items()} - @property - def cache_filename(self) -> str: - # FIXME: this should be declarative - return f"{self.name}.yml" + def _request_params( + self, + stream_state: Optional[StreamState], + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + extra_params: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + """ + Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. - @property - def use_cache(self) -> bool: - # FIXME: this should be declarative - return False + E.g: you might want to define query parameters for paging if next_page_token is not None. + """ + options = self._get_request_options( + stream_state, stream_slice, next_page_token, self.get_request_params, self.get_authenticator().get_request_params, extra_params + ) + if isinstance(options, str): + raise ValueError("Request params cannot be a string") + return options + + def _request_body_data( + self, + stream_state: Optional[StreamState], + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + extra_body_data: Optional[Union[Mapping[str, Any], str]] = None, + ) -> Optional[Union[Mapping[str, Any], str]]: + """ + Specifies how to populate the body of the request with a non-JSON payload. + + If returns a ready text that it will be sent as is. + If returns a dict that it will be converted to a urlencoded form. + E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2" + + At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden. + """ + # Warning: use self.state instead of the stream_state passed as argument! + return self._get_request_options( + stream_state, + stream_slice, + next_page_token, + self.get_request_body_data, + self.get_authenticator().get_request_body_data, + extra_body_data, + ) + + def _request_body_json( + self, + stream_state: Optional[StreamState], + stream_slice: Optional[StreamSlice], + next_page_token: Optional[Mapping[str, Any]], + extra_body_json: Optional[Mapping[str, Any]] = None, + ) -> Optional[Mapping[str, Any]]: + """ + Specifies how to populate the body of the request with a JSON payload. + + At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden. + """ + # Warning: use self.state instead of the stream_state passed as argument! + options = self._get_request_options( + stream_state, + stream_slice, + next_page_token, + self.get_request_body_json, + self.get_authenticator().get_request_body_json, + extra_body_json, + ) + if isinstance(options, str): + raise ValueError("Request body json cannot be a string") + return options + + def deduplicate_query_params(self, url: str, params: Optional[Mapping[str, Any]]) -> Mapping[str, Any]: + """ + Remove query parameters from params mapping if they are already encoded in the URL. + :param url: URL with + :param params: + :return: + """ + if params is None: + params = {} + query_string = urllib.parse.urlparse(url).query + query_dict = {k: v[0] for k, v in urllib.parse.parse_qs(query_string).items()} + + duplicate_keys_with_same_value = {k for k in query_dict.keys() if str(params.get(k)) == str(query_dict[k])} + return {k: v for k, v in params.items() if k not in duplicate_keys_with_same_value} + + @classmethod + def _join_url(cls, url_base: str, path: str) -> str: + return urljoin(url_base, path) + + def _create_prepared_request( + self, + path: str, + headers: Optional[Mapping[str, str]] = None, + params: Optional[Mapping[str, Any]] = None, + json: Any = None, + data: Any = None, + ) -> requests.PreparedRequest: + url = urljoin(self.get_url_base(), path) + http_method = str(self._http_method.value) + query_params = self.deduplicate_query_params(url, params) + args = {"method": http_method, "url": url, "headers": headers, "params": query_params} + if http_method.upper() in BODY_REQUEST_METHODS: + if json and data: + raise RequestBodyException( + "At the same time only one of the 'request_body_data' and 'request_body_json' functions can return data" + ) + elif json: + args["json"] = json + elif data: + args["data"] = data + + return self._session.prepare_request(requests.Request(**args)) + + def send_request( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + path: Optional[str] = None, + request_headers: Optional[Mapping[str, Any]] = None, + request_params: Optional[Mapping[str, Any]] = None, + request_body_data: Optional[Union[Mapping[str, Any], str]] = None, + request_body_json: Optional[Mapping[str, Any]] = None, + log_formatter: Optional[Callable[[requests.Response], Any]] = None, + ) -> Optional[requests.Response]: + request = self._create_prepared_request( + path=path + if path is not None + else self.get_path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + headers=self._request_headers(stream_state, stream_slice, next_page_token, request_headers), + params=self._request_params(stream_state, stream_slice, next_page_token, request_params), + json=self._request_body_json(stream_state, stream_slice, next_page_token, request_body_json), + data=self._request_body_data(stream_state, stream_slice, next_page_token, request_body_data), + ) + + response = self._send_with_retry(request, log_formatter=log_formatter) + return self._validate_response(response) + + def _send_with_retry( + self, + request: requests.PreparedRequest, + log_formatter: Optional[Callable[[requests.Response], Any]] = None, + ) -> requests.Response: + """ + Creates backoff wrappers which are responsible for retry logic + """ + + """ + Backoff package has max_tries parameter that means total number of + tries before giving up, so if this number is 0 no calls expected to be done. + But for this class we call it max_REtries assuming there would be at + least one attempt and some retry attempts, to comply this logic we add + 1 to expected retries attempts. + """ + max_tries = self.max_retries + """ + According to backoff max_tries docstring: + max_tries: The maximum number of attempts to make before giving + up ...The default value of None means there is no limit to + the number of tries. + This implies that if max_tries is explicitly set to None there is no + limit to retry attempts, otherwise it is limited number of tries. But + this is not true for current version of backoff packages (1.8.0). Setting + max_tries to 0 or negative number would result in endless retry attempts. + Add this condition to avoid an endless loop if it hasn't been set + explicitly (i.e. max_retries is not None). + """ + if max_tries is not None: + max_tries = max(0, max_tries) + 1 + + user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries)(self._send) # type: ignore # we don't pass in kwargs to the backoff handler + backoff_handler = default_backoff_handler(max_tries=max_tries, factor=self._DEFAULT_RETRY_FACTOR) + # backoff handlers wrap _send, so it will always return a response + return backoff_handler(user_backoff_handler)(request, log_formatter=log_formatter) # type: ignore + + def _send( + self, + request: requests.PreparedRequest, + log_formatter: Optional[Callable[[requests.Response], Any]] = None, + ) -> requests.Response: + """ + Wraps sending the request in rate limit and error handlers. + Please note that error handling for HTTP status codes will be ignored if raise_on_http_errors is set to False + + This method handles two types of exceptions: + 1. Expected transient exceptions e.g: 429 status code. + 2. Unexpected transient exceptions e.g: timeout. + + To trigger a backoff, we raise an exception that is handled by the backoff decorator. If an exception is not handled by the decorator will + fail the sync. + + For expected transient exceptions, backoff time is determined by the type of exception raised: + 1. CustomBackoffException uses the user-provided backoff value + 2. DefaultBackoffException falls back on the decorator's default behavior e.g: exponential backoff + + Unexpected transient exceptions use the default backoff parameters. + Unexpected persistent exceptions are not handled and will cause the sync to fail. + """ + self.logger.debug( + "Making outbound API request", extra={"headers": request.headers, "url": request.url, "request_body": request.body} + ) + response: requests.Response = self._session.send(request) + self.logger.debug("Receiving response", extra={"headers": response.headers, "status": response.status_code, "body": response.text}) + if log_formatter: + formatter = log_formatter + self.message_repository.log_message( + Level.DEBUG, + lambda: formatter(response), + ) + if self._should_retry(response): + custom_backoff_time = self._backoff_time(response) + if custom_backoff_time: + raise UserDefinedBackoffException(backoff=custom_backoff_time, request=request, response=response) + else: + raise DefaultBackoffException(request=request, response=response) + return response + + def _validate_response( + self, + response: requests.Response, + ) -> Optional[requests.Response]: + # if fail -> raise exception + # if ignore -> ignore response and return None + # else -> delegate to caller + if self.error_handler is None: + return response + + response_status = self.interpret_response_status(response) + if response_status.action == ResponseAction.FAIL: + error_message = ( + response_status.error_message + or f"Request to {response.request.url} failed with status code {response.status_code} and error message {HttpRequester.parse_response_error_message(response)}" + ) + raise ReadException(error_message) + elif response_status.action == ResponseAction.IGNORE: + self.logger.info( + f"Ignoring response for failed request with error message {HttpRequester.parse_response_error_message(response)}" + ) + + return response + + @classmethod + def parse_response_error_message(cls, response: requests.Response) -> Optional[str]: + """ + Parses the raw response object from a failed request into a user-friendly error message. + By default, this method tries to grab the error message from JSON responses by following common API patterns. Override to parse differently. + + :param response: + :return: A user-friendly message that indicates the cause of the error + """ + + # default logic to grab error from common fields + def _try_get_error(value: Any) -> Any: + if isinstance(value, str): + return value + elif isinstance(value, list): + return ", ".join(_try_get_error(v) for v in value) + elif isinstance(value, dict): + new_value = ( + value.get("message") + or value.get("messages") + or value.get("error") + or value.get("errors") + or value.get("failures") + or value.get("failure") + ) + return _try_get_error(new_value) + return None + + try: + body = response.json() + error = _try_get_error(body) + return str(error) if error else None + except requests.exceptions.JSONDecodeError: + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index 3ea4c578cda5..5f4bf69dd306 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -13,7 +13,7 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath -from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState @dataclass @@ -101,7 +101,7 @@ def __post_init__(self, parameters: Mapping[str, Any]): if isinstance(self.url_base, str): self.url_base = InterpolatedString(string=self.url_base, parameters=parameters) - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: + def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Optional[Mapping[str, Any]]: self._token = self.pagination_strategy.next_page_token(response, last_records) if self._token: return {"next_page_token": self._token} @@ -185,7 +185,7 @@ def __init__(self, decorated, maximum_number_of_pages: int = 5): self._decorated = decorated self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: + def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Optional[Mapping[str, Any]]: if self._page_count >= self._maximum_number_of_pages: return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py index d2cbde960d42..683508c761aa 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/no_pagination.py @@ -3,11 +3,11 @@ # from dataclasses import InitVar, dataclass -from typing import Any, List, Mapping, Optional, Union +from typing import Any, List, Mapping, MutableMapping, Optional, Union import requests from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator -from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState @dataclass @@ -27,7 +27,7 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + ) -> MutableMapping[str, Any]: return {} def get_request_headers( @@ -57,9 +57,9 @@ def get_request_body_json( ) -> Mapping[str, Any]: return {} - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Mapping[str, Any]: + def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Mapping[str, Any]: return {} - def reset(self): + def reset(self) -> None: # No state to reset pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py index 0891849acd92..2138712875dc 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/paginator.py @@ -8,6 +8,7 @@ import requests from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider +from airbyte_cdk.sources.declarative.types import Record @dataclass @@ -20,13 +21,13 @@ class Paginator(ABC, RequestOptionsProvider): """ @abstractmethod - def reset(self): + def reset(self) -> None: """ Reset the pagination's inner state """ @abstractmethod - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: + def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Optional[Mapping[str, Any]]: """ Returns the next_page_token to use to fetch the next page of records. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py index 2275c67add9c..03e5ecae532e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/__init__.py @@ -5,5 +5,15 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.cursor_pagination_strategy import CursorPaginationStrategy from airbyte_cdk.sources.declarative.requesters.paginators.strategies.offset_increment import OffsetIncrement from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.stop_condition import ( + CursorStopCondition, + StopConditionPaginationStrategyDecorator, +) -__all__ = ["CursorPaginationStrategy", "OffsetIncrement", "PageIncrement"] +__all__ = [ + "CursorPaginationStrategy", + "CursorStopCondition", + "OffsetIncrement", + "PageIncrement", + "StopConditionPaginationStrategyDecorator", +] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py new file mode 100644 index 000000000000..827171bcf705 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, List, Optional + +import requests +from airbyte_cdk.sources.declarative.incremental import Cursor +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy +from airbyte_cdk.sources.declarative.types import Record + + +class PaginationStopCondition(ABC): + @abstractmethod + def is_met(self, record: Record) -> bool: + """ + Given a condition is met, the pagination will stop + + :param record: a record used to evaluate the condition + """ + raise NotImplementedError() + + +class CursorStopCondition(PaginationStopCondition): + def __init__(self, cursor: Cursor): + self._cursor = cursor + + def is_met(self, record: Record) -> bool: + return not self._cursor.should_be_synced(record) + + +class StopConditionPaginationStrategyDecorator(PaginationStrategy): + def __init__(self, _delegate: PaginationStrategy, stop_condition: PaginationStopCondition): + self._delegate = _delegate + self._stop_condition = stop_condition + + def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Optional[Any]: + # We evaluate in reverse order because the assumption is that most of the APIs using data feed structure will return records in + # descending order. In terms of performance/memory, we return the records lazily + if last_records and any(self._stop_condition.is_met(record) for record in reversed(last_records)): + return None + return self._delegate.next_page_token(response, last_records) + + def reset(self): + self._delegate.reset() + + def get_page_size(self) -> Optional[int]: + return self._delegate.get_page_size() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py index d384df632458..3b8396756aa0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py @@ -4,13 +4,13 @@ from abc import abstractmethod from enum import Enum -from typing import Any, Mapping, MutableMapping, Optional +from typing import Any, Callable, Mapping, MutableMapping, Optional, Union import requests +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState -from requests.auth import AuthBase class HttpMethod(Enum): @@ -24,7 +24,7 @@ class HttpMethod(Enum): class Requester(RequestOptionsProvider): @abstractmethod - def get_authenticator(self) -> AuthBase: + def get_authenticator(self) -> DeclarativeAuthenticator: """ Specifies the authenticator to use when submitting requests """ @@ -125,29 +125,22 @@ def get_request_body_json( """ @abstractmethod - def request_kwargs( + def send_request( self, - *, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - """ - Returns a mapping of keyword arguments to be used when creating the HTTP request. - Any option listed in https://docs.python-requests.org/en/latest/api/#requests.adapters.BaseAdapter.send for can be returned from - this method. Note that these options do not conflict with request-level options such as headers, request params, etc.. - """ - - @property - @abstractmethod - def cache_filename(self) -> str: - """ - Return the name of cache file - """ - - @property - @abstractmethod - def use_cache(self) -> bool: - """ - If True, all records will be cached. + path: Optional[str] = None, + request_headers: Optional[Mapping[str, Any]] = None, + request_params: Optional[Mapping[str, Any]] = None, + request_body_data: Optional[Union[Mapping[str, Any], str]] = None, + request_body_json: Optional[Mapping[str, Any]] = None, + log_formatter: Optional[Callable[[requests.Response], Any]] = None, + ) -> Optional[requests.Response]: + """ + Sends a request and returns the response. Might return no response if the error handler chooses to ignore the response or throw an exception in case of an error. + If path is set, the path configured on the requester itself is ignored. + If header, params and body are set, they are merged with the ones configured on the requester itself. + + If a log formatter is provided, it's used to log the performed request and response. If it's not provided, no logging is performed. """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py index 45f9cce1940b..d46dc9463487 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py @@ -4,9 +4,8 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Iterable, List, Optional +from typing import Iterable, Optional -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.core import StreamData @@ -20,10 +19,7 @@ class Retriever: @abstractmethod def read_records( self, - sync_mode: SyncMode, - cursor_field: Optional[List[str]] = None, stream_slice: Optional[StreamSlice] = None, - stream_state: Optional[StreamState] = None, ) -> Iterable[StreamData]: """ Fetch a stream's records from an HTTP API source @@ -36,7 +32,7 @@ def read_records( """ @abstractmethod - def stream_slices(self, *, sync_mode: SyncMode, stream_state: Optional[StreamState] = None) -> Iterable[Optional[StreamSlice]]: + def stream_slices(self) -> Iterable[Optional[StreamSlice]]: """Returns the stream slices""" @property @@ -56,5 +52,5 @@ def state(self) -> StreamState: @state.setter @abstractmethod - def state(self, value: StreamState): + def state(self, value: StreamState) -> None: """State setter, accept state serialized by state getter.""" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index c58414127b5a..f269e35ebeab 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -2,33 +2,29 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json from dataclasses import InitVar, dataclass, field from itertools import islice -from json import JSONDecodeError -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from typing import Any, Callable, Iterable, List, Mapping, Optional, Set, Tuple, Union import requests -from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode -from airbyte_cdk.models import Type as MessageType -from airbyte_cdk.sources.declarative.exceptions import ReadException +from airbyte_cdk.models import AirbyteMessage from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter -from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.paginators.no_pagination import NoPagination from airbyte_cdk.sources.declarative.requesters.paginators.paginator import Paginator from airbyte_cdk.sources.declarative.requesters.requester import Requester from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.http_logger import format_http_message from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets +from airbyte_cdk.utils.mapping_helpers import combine_mappings @dataclass -class SimpleRetriever(Retriever, HttpStream): +class SimpleRetriever(Retriever): """ Retrieves records by synchronously sending requests to fetch records. @@ -47,11 +43,10 @@ class SimpleRetriever(Retriever, HttpStream): record_selector (HttpSelector): The record selector paginator (Optional[Paginator]): The paginator stream_slicer (Optional[StreamSlicer]): The stream slicer + cursor (Optional[cursor]): The cursor parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation """ - _DEFAULT_MAX_RETRY = 5 - requester: Requester record_selector: HttpSelector config: Config @@ -61,164 +56,110 @@ class SimpleRetriever(Retriever, HttpStream): primary_key: Optional[Union[str, List[str], List[List[str]]]] _primary_key: str = field(init=False, repr=False, default="") paginator: Optional[Paginator] = None - stream_slicer: Optional[StreamSlicer] = SinglePartitionRouter(parameters={}) - emit_connector_builder_messages: bool = False - disable_retries: bool = False - - def __post_init__(self, parameters: Mapping[str, Any]): - self.paginator = self.paginator or NoPagination(parameters=parameters) - HttpStream.__init__(self, self.requester.get_authenticator()) - self._last_response = None - self._last_records = None + stream_slicer: StreamSlicer = SinglePartitionRouter(parameters={}) + cursor: Optional[Cursor] = None + + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + self._paginator = self.paginator or NoPagination(parameters=parameters) + self._last_response: Optional[requests.Response] = None + self._records_from_last_response: List[Record] = [] self._parameters = parameters - self.name = InterpolatedString(self._name, parameters=parameters) + self._name = InterpolatedString(self._name, parameters=parameters) if isinstance(self._name, str) else self._name - @property + @property # type: ignore def name(self) -> str: """ :return: Stream name """ - return self._name.eval(self.config) + return str(self._name.eval(self.config)) if isinstance(self._name, InterpolatedString) else self._name @name.setter def name(self, value: str) -> None: if not isinstance(value, property): self._name = value - @property - def url_base(self) -> str: - return self.requester.get_url_base() - - @property - def http_method(self) -> str: - return str(self.requester.get_method().value) - - @property - def raise_on_http_errors(self) -> bool: - # never raise on http_errors because this overrides the error handler logic... - return False - - @property - def max_retries(self) -> Union[int, None]: - if self.disable_retries: - return 0 - if hasattr(self.requester.error_handler, "max_retries"): - return self.requester.error_handler.max_retries - return self._DEFAULT_MAX_RETRY - - def should_retry(self, response: requests.Response) -> bool: + def _get_mapping( + self, method: Callable[..., Optional[Union[Mapping[str, Any], str]]], **kwargs: Any + ) -> Tuple[Union[Mapping[str, Any], str], Set[str]]: """ - Specifies conditions for backoff based on the response from the server. - - By default, back off on the following HTTP response statuses: - - 429 (Too Many Requests) indicating rate limiting - - 500s to handle transient server errors - - Unexpected but transient exceptions (connection timeout, DNS resolution failed, etc..) are retried by default. + Get mapping from the provided method, and get the keys of the mapping. + If the method returns a string, it will return the string and an empty set. + If the method returns a dict, it will return the dict and its keys. """ - return self.requester.interpret_response_status(response).action == ResponseAction.RETRY - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """ - Specifies backoff time. - - This method is called only if should_backoff() returns True for the input request. - - :param response: - :return how long to backoff in seconds. The return value may be a floating point number for subsecond precision. Returning None defers backoff - to the default backoff behavior (e.g using an exponential algorithm). - """ - should_retry = self.requester.interpret_response_status(response) - if should_retry.action != ResponseAction.RETRY: - raise ValueError(f"backoff_time can only be applied on retriable response action. Got {should_retry.action}") - assert should_retry.action == ResponseAction.RETRY - return should_retry.retry_in - - def error_message(self, response: requests.Response) -> str: - """ - Constructs an error message which can incorporate the HTTP response received from the partner API. - - :param response: The incoming HTTP response from the partner API - :return The error message string to be emitted - """ - return self.requester.interpret_response_status(response).error_message + mapping = method(**kwargs) or {} + keys = set(mapping.keys()) if not isinstance(mapping, str) else set() + return mapping, keys def _get_request_options( self, + stream_state: Optional[StreamData], stream_slice: Optional[StreamSlice], next_page_token: Optional[Mapping[str, Any]], - requester_method, - paginator_method, - stream_slicer_method, - ): + paginator_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], + stream_slicer_method: Callable[..., Optional[Union[Mapping[str, Any], str]]], + ) -> Union[Mapping[str, Any], str]: """ - Get the request_option from the requester and from the paginator + Get the request_option from the paginator and the stream slicer. Raise a ValueError if there's a key collision Returned merged mapping otherwise - :param stream_slice: - :param next_page_token: - :param requester_method: - :param paginator_method: - :return: """ - - requester_mapping = requester_method(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) - requester_mapping_keys = set(requester_mapping.keys()) - paginator_mapping = paginator_method(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) - paginator_mapping_keys = set(paginator_mapping.keys()) - stream_slicer_mapping = stream_slicer_method(stream_slice=stream_slice) - stream_slicer_mapping_keys = set(stream_slicer_mapping.keys()) - - intersection = ( - (requester_mapping_keys & paginator_mapping_keys) - | (requester_mapping_keys & stream_slicer_mapping_keys) - | (paginator_mapping_keys & stream_slicer_mapping_keys) + # FIXME we should eventually remove the usage of stream_state as part of the interpolation + return combine_mappings( + [ + paginator_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + stream_slicer_method(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + ] ) - if intersection: - raise ValueError(f"Duplicate keys found: {intersection}") - return {**requester_mapping, **paginator_mapping, **stream_slicer_mapping} - def request_headers( - self, stream_state: StreamState, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None + def _request_headers( + self, + stream_state: Optional[StreamData] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: """ Specifies request headers. Authentication headers will overwrite any overlapping headers returned from this method. """ headers = self._get_request_options( + stream_state, stream_slice, next_page_token, - self.requester.get_request_headers, - self.paginator.get_request_headers, + self._paginator.get_request_headers, self.stream_slicer.get_request_headers, ) + if isinstance(headers, str): + raise ValueError("Request headers cannot be a string") return {str(k): str(v) for k, v in headers.items()} - def request_params( + def _request_params( self, - stream_state: StreamSlice, + stream_state: Optional[StreamData] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: + ) -> Mapping[str, Any]: """ Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. E.g: you might want to define query parameters for paging if next_page_token is not None. """ - return self._get_request_options( + params = self._get_request_options( + stream_state, stream_slice, next_page_token, - self.requester.get_request_params, - self.paginator.get_request_params, + self._paginator.get_request_params, self.stream_slicer.get_request_params, ) + if isinstance(params, str): + raise ValueError("Request params cannot be a string") + return params - def request_body_data( + def _request_body_data( self, - stream_state: StreamState, + stream_state: Optional[StreamData] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Union[Mapping, str]]: + ) -> Union[Mapping[str, Any], str]: """ Specifies how to populate the body of the request with a non-JSON payload. @@ -228,127 +169,68 @@ def request_body_data( At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden. """ - # Warning: use self.state instead of the stream_state passed as argument! - base_body_data = self.requester.get_request_body_data( - stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token - ) - if isinstance(base_body_data, str): - paginator_body_data = self.paginator.get_request_body_data() - if paginator_body_data: - raise ValueError( - f"Cannot combine requester's body data= {base_body_data} with paginator's body_data: {paginator_body_data}" - ) - else: - return base_body_data return self._get_request_options( + stream_state, stream_slice, next_page_token, - self.requester.get_request_body_data, - self.paginator.get_request_body_data, + self._paginator.get_request_body_data, self.stream_slicer.get_request_body_data, ) - def request_body_json( + def _request_body_json( self, - stream_state: StreamState, + stream_state: Optional[StreamData] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Mapping]: + ) -> Optional[Mapping[str, Any]]: """ Specifies how to populate the body of the request with a JSON payload. At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden. """ - # Warning: use self.state instead of the stream_state passed as argument! - return self._get_request_options( + body_json = self._get_request_options( + stream_state, stream_slice, next_page_token, - self.requester.get_request_body_json, - self.paginator.get_request_body_json, + self._paginator.get_request_body_json, self.stream_slicer.get_request_body_json, ) + if isinstance(body_json, str): + raise ValueError("Request body json cannot be a string") + return body_json - def request_kwargs( + def _paginator_path( self, - stream_state: StreamState, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + ) -> Optional[str]: """ - Specifies how to configure a mapping of keyword arguments to be used when creating the HTTP request. - Any option listed in https://docs.python-requests.org/en/latest/api/#requests.adapters.BaseAdapter.send for can be returned from - this method. Note that these options do not conflict with request-level options such as headers, request params, etc.. - """ - # Warning: use self.state instead of the stream_state passed as argument! - return self.requester.request_kwargs(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) - - def path( - self, - *, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - """ - Return the path the submit the next request to. - If the paginator points to a path, follow it, else return the requester's path + If the paginator points to a path, follow it, else return nothing so the requester is used. :param stream_state: :param stream_slice: :param next_page_token: :return: """ - # Warning: use self.state instead of the stream_state passed as argument! - paginator_path = self.paginator.path() - if paginator_path: - return paginator_path - else: - return self.requester.get_path(stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token) - - @property - def cache_filename(self) -> str: - """ - Return the name of cache file - """ - return self.requester.cache_filename - - @property - def use_cache(self) -> bool: - """ - If True, all records will be cached. - """ - return self.requester.use_cache + return self._paginator.path() - def parse_response( + def _parse_response( self, - response: requests.Response, - *, + response: Optional[requests.Response], stream_state: StreamState, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Record]: - # if fail -> raise exception - # if ignore -> ignore response and return no records - # else -> delegate to record selector - response_status = self.requester.interpret_response_status(response) - if response_status.action == ResponseAction.FAIL: - error_message = ( - response_status.error_message - or f"Request to {response.request.url} failed with status code {response.status_code} and error message {HttpStream.parse_response_error_message(response)}" - ) - raise ReadException(error_message) - elif response_status.action == ResponseAction.IGNORE: - self.logger.info(f"Ignoring response for failed request with error message {HttpStream.parse_response_error_message(response)}") + if not response: + self._last_response = None + self._records_from_last_response = [] return [] - # Warning: use self.state instead of the stream_state passed as argument! self._last_response = response records = self.record_selector.select_records( - response=response, stream_state=self.state, stream_slice=stream_slice, next_page_token=next_page_token + response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - self._last_records = records + self._records_from_last_response = records return records - @property + @property # type: ignore def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: """The stream's primary key""" return self._primary_key @@ -358,7 +240,7 @@ def primary_key(self, value: str) -> None: if not isinstance(value, property): self._primary_key = value - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + def _next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ Specifies a pagination strategy. @@ -366,39 +248,95 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, :return: The token for the next page from the input response object. Returning None means there are no more pages to read in this response. """ - return self.paginator.next_page_token(response, self._last_records) + return self._paginator.next_page_token(response, self._records_from_last_response) + + def _fetch_next_page( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any], next_page_token: Optional[Mapping[str, Any]] = None + ) -> Optional[requests.Response]: + return self.requester.send_request( + path=self._paginator_path(), + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + request_headers=self._request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + request_params=self._request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + request_body_data=self._request_body_data( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ), + request_body_json=self._request_body_json( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ), + ) + + # This logic is similar to _read_pages in the HttpStream class. When making changes here, consider making changes there as well. + def _read_pages( + self, + records_generator_fn: Callable[[Optional[requests.Response], Mapping[str, Any], Mapping[str, Any]], Iterable[StreamData]], + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any], + ) -> Iterable[StreamData]: + stream_state = stream_state or {} + pagination_complete = False + next_page_token = None + while not pagination_complete: + response = self._fetch_next_page(stream_state, stream_slice, next_page_token) + yield from records_generator_fn(response, stream_state, stream_slice) + + if not response: + pagination_complete = True + else: + next_page_token = self._next_page_token(response) + if not next_page_token: + pagination_complete = True + + # Always return an empty generator just in case no records were ever yielded + yield from [] def read_records( self, - sync_mode: SyncMode, - cursor_field: Optional[List[str]] = None, stream_slice: Optional[StreamSlice] = None, - stream_state: Optional[StreamState] = None, ) -> Iterable[StreamData]: - # Warning: use self.state instead of the stream_state passed as argument! stream_slice = stream_slice or {} # None-check - self.paginator.reset() - records_generator = self._read_pages( - self.parse_records, - stream_slice, - stream_state, - ) - cursor_updated = False - for record in records_generator: - # Only record messages should be parsed to update the cursor which is indicated by the Mapping type - if isinstance(record, Mapping): - self.stream_slicer.update_cursor(stream_slice, last_record=record) - cursor_updated = True - yield record - if not cursor_updated: - last_record = self._last_records[-1] if self._last_records else None - if last_record and isinstance(last_record, Mapping): - self.stream_slicer.update_cursor(stream_slice, last_record=last_record) - yield from [] - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Optional[StreamState] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: + # Fixing paginator types has a long tail of dependencies + self._paginator.reset() + + most_recent_record_from_slice = None + for stream_data in self._read_pages(self._parse_records, self.state, stream_slice): + most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, stream_data, stream_slice) + yield stream_data + + if self.cursor: + self.cursor.close_slice(stream_slice, most_recent_record_from_slice) + return + + def _get_most_recent_record( + self, current_most_recent: Optional[Record], stream_data: StreamData, stream_slice: StreamSlice + ) -> Optional[Record]: + if self.cursor and (record := self._extract_record(stream_data, stream_slice)): + if not current_most_recent: + return record + else: + return current_most_recent if self.cursor.is_greater_than_or_equal(current_most_recent, record) else record + else: + return None + + @staticmethod + def _extract_record(stream_data: StreamData, stream_slice: StreamSlice) -> Optional[Record]: + """ + As we allow the output of _read_pages to be StreamData, it can be multiple things. Therefore, we need to filter out and normalize + to data to streamline the rest of the process. + """ + if isinstance(stream_data, Record): + # Record is not part of `StreamData` but is the most common implementation of `Mapping[str, Any]` which is part of `StreamData` + return stream_data + elif isinstance(stream_data, (dict, Mapping)): + return Record(dict(stream_data), stream_slice) + elif isinstance(stream_data, AirbyteMessage) and stream_data.record: + return Record(stream_data.record.data, stream_slice) + return None + + # stream_slices is defined with arguments on http stream and fixing this has a long tail of dependencies. Will be resolved by the decoupling of http stream and simple retriever + def stream_slices(self) -> Iterable[Optional[Mapping[str, Any]]]: # type: ignore """ Specifies the slices for this stream. See the stream slicing section of the docs for more information. @@ -407,26 +345,28 @@ def stream_slices( :param stream_state: :return: """ - # Warning: use self.state instead of the stream_state passed as argument! - return self.stream_slicer.stream_slices(sync_mode, self.state) + return self.stream_slicer.stream_slices() @property - def state(self) -> MutableMapping[str, Any]: - return self.stream_slicer.get_stream_state() + def state(self) -> Mapping[str, Any]: + return self.cursor.get_stream_state() if self.cursor else {} @state.setter - def state(self, value: StreamState): + def state(self, value: StreamState) -> None: """State setter, accept state serialized by state getter.""" - self.stream_slicer.update_cursor(value) + if self.cursor: + self.cursor.set_initial_state(value) - def parse_records( + def _parse_records( self, - request: requests.PreparedRequest, - response: requests.Response, + response: Optional[requests.Response], stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]], ) -> Iterable[StreamData]: - yield from self.parse_response(response, stream_slice=stream_slice, stream_state=stream_state) + yield from self._parse_response(response, stream_slice=stream_slice, stream_state=stream_state) + + def must_deduplicate_query_params(self) -> bool: + return True @dataclass @@ -438,56 +378,37 @@ class SimpleRetrieverTestReadDecorator(SimpleRetriever): maximum_number_of_slices: int = 5 - def __post_init__(self, options: Mapping[str, Any]): + def __post_init__(self, options: Mapping[str, Any]) -> None: super().__post_init__(options) if self.maximum_number_of_slices and self.maximum_number_of_slices < 1: raise ValueError( f"The maximum number of slices on a test read needs to be strictly positive. Got {self.maximum_number_of_slices}" ) - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Optional[StreamState] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - return islice(super().stream_slices(sync_mode=sync_mode, stream_state=stream_state), self.maximum_number_of_slices) - - def parse_records( - self, - request: requests.PreparedRequest, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any], - ) -> Iterable[StreamData]: - yield _prepared_request_to_airbyte_message(request) - yield _response_to_airbyte_message(response) - yield from self.parse_response(response, stream_slice=stream_slice, stream_state=stream_state) - - -def _prepared_request_to_airbyte_message(request: requests.PreparedRequest) -> AirbyteMessage: - # FIXME: this should return some sort of trace message - request_dict = { - "url": request.url, - "http_method": request.method, - "headers": dict(request.headers), - "body": _body_binary_string_to_dict(request.body), - } - log_message = filter_secrets(f"request:{json.dumps(request_dict)}") - return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=log_message)) - - -def _body_binary_string_to_dict(body_str: str) -> Optional[Mapping[str, str]]: - if body_str: - if isinstance(body_str, (bytes, bytearray)): - body_str = body_str.decode() - try: - return json.loads(body_str) - except JSONDecodeError: - return {k: v for k, v in [s.split("=") for s in body_str.split("&")]} - else: - return None - - -def _response_to_airbyte_message(response: requests.Response) -> AirbyteMessage: - # FIXME: this should return some sort of trace message - response_dict = {"body": response.text, "headers": dict(response.headers), "status_code": response.status_code} - log_message = filter_secrets(f"response:{json.dumps(response_dict)}") - return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=log_message)) + # stream_slices is defined with arguments on http stream and fixing this has a long tail of dependencies. Will be resolved by the decoupling of http stream and simple retriever + def stream_slices(self) -> Iterable[Optional[Mapping[str, Any]]]: # type: ignore + return islice(super().stream_slices(), self.maximum_number_of_slices) + + def _fetch_next_page( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any], next_page_token: Optional[Mapping[str, Any]] = None + ) -> Optional[requests.Response]: + return self.requester.send_request( + path=self._paginator_path(), + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + request_headers=self._request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + request_params=self._request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + request_body_data=self._request_body_data( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ), + request_body_json=self._request_body_json( + stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ), + log_formatter=lambda response: format_http_message( + response, + f"Stream '{self.name}' request", + f"Request performed in order to extract records for stream '{self.name}'", + self.name, + ), + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py index 4aaf1af7c20d..ea57fe4fcf66 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/cartesian_product_stream_slicer.py @@ -7,7 +7,6 @@ from dataclasses import InitVar, dataclass from typing import Any, Iterable, List, Mapping, Optional -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState @@ -36,10 +35,6 @@ class CartesianProductStreamSlicer(StreamSlicer): stream_slicers: List[StreamSlicer] parameters: InitVar[Mapping[str, Any]] - def update_cursor(self, stream_slice: Mapping[str, Any], last_record: Optional[Mapping[str, Any]] = None): - for slicer in self.stream_slicers: - slicer.update_cursor(stream_slice, last_record) - def get_request_params( self, *, @@ -104,9 +99,6 @@ def get_request_body_json( ) ) - def get_stream_state(self) -> Mapping[str, Any]: - return dict(ChainMap(*[slicer.get_stream_state() for slicer in self.stream_slicers])) - - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: - sub_slices = (s.stream_slices(sync_mode, stream_state) for s in self.stream_slicers) + def stream_slices(self) -> Iterable[StreamSlice]: + sub_slices = (s.stream_slices() for s in self.stream_slicers) return (dict(ChainMap(*a)) for a in itertools.product(*sub_slices)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py index d68bcaac210a..35629fa63b60 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/stream_slicer.py @@ -4,11 +4,10 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Iterable, Optional +from typing import Iterable -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider -from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import StreamSlice @dataclass @@ -23,24 +22,9 @@ class StreamSlicer(RequestOptionsProvider): """ @abstractmethod - def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]: + def stream_slices(self) -> Iterable[StreamSlice]: """ Defines stream slices - :param sync_mode: The sync mode used the read data - :param stream_state: The current stream state :return: List of stream slices """ - - @abstractmethod - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - """ - State setter, accept state serialized by state getter. - - :param stream_slice: Current stream_slice - :param last_record: Last record read from the source - """ - - @abstractmethod - def get_stream_state(self) -> StreamState: - """Returns the current stream state""" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py index f7f00e227cb6..b0d222273ef3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/remove_fields.py @@ -3,12 +3,12 @@ # from dataclasses import InitVar, dataclass -from typing import Any, List, Mapping +from typing import Any, List, Mapping, Optional import dpath.exceptions import dpath.util from airbyte_cdk.sources.declarative.transformations import RecordTransformation -from airbyte_cdk.sources.declarative.types import FieldPointer, Record +from airbyte_cdk.sources.declarative.types import Config, FieldPointer, StreamSlice, StreamState @dataclass @@ -41,7 +41,13 @@ class RemoveFields(RecordTransformation): field_pointers: List[FieldPointer] parameters: InitVar[Mapping[str, Any]] - def transform(self, record: Record, **kwargs) -> Record: + def transform( + self, + record: Mapping[str, Any], + config: Optional[Config] = None, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Mapping[str, Any]: """ :param record: The record to be transformed :return: the input record with the requested fields removed diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py index 621460b60b55..560bf39e1b08 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py @@ -4,9 +4,9 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Optional +from typing import Any, Mapping, Optional -from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState @dataclass @@ -18,11 +18,11 @@ class RecordTransformation: @abstractmethod def transform( self, - record: Record, + record: Mapping[str, Any], config: Optional[Config] = None, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, - ) -> Record: + ) -> Mapping[str, Any]: """ Transform a record by adding, deleting, or mutating fields. @@ -33,5 +33,5 @@ def transform( :return: The transformed record """ - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return other.__dict__ == self.__dict__ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/types.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/types.py index 3eb88b578c3c..fd0eba51676c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/types.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/types.py @@ -4,9 +4,8 @@ from __future__ import annotations -from typing import Any, List, Mapping +from typing import Any, List, Mapping, Optional -Record = Mapping[str, Any] # A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2": # "hello"}] returns "hello" FieldPointer = List[str] @@ -14,3 +13,41 @@ ConnectionDefinition = Mapping[str, Any] StreamSlice = Mapping[str, Any] StreamState = Mapping[str, Any] + + +class Record(Mapping[str, Any]): + def __init__(self, data: Mapping[str, Any], associated_slice: Optional[StreamSlice]): + self._data = data + self._associated_slice = associated_slice + + @property + def data(self) -> Mapping[str, Any]: + return self._data + + @property + def associated_slice(self) -> Optional[StreamSlice]: + return self._associated_slice + + def __repr__(self) -> str: + return repr(self._data) + + def __getitem__(self, key: str) -> Any: + return self._data[key] + + def __len__(self) -> int: + return len(self._data) + + def __iter__(self) -> Any: + return iter(self._data) + + def __contains__(self, item: object) -> bool: + return item in self._data + + def __eq__(self, other: object) -> bool: + if isinstance(other, Record): + # noinspection PyProtectedMember + return self._data == other._data + return False + + def __ne__(self, other: object) -> bool: + return not self.__eq__(other) diff --git a/airbyte-integrations/connectors/source-babelforce/unit_tests/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-babelforce/unit_tests/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/embedded/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py new file mode 100644 index 000000000000..158dea4d135a --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Generic, Iterable, Optional, TypeVar + +from airbyte_cdk.connector import TConfig +from airbyte_cdk.sources.embedded.catalog import create_configured_catalog, get_stream, get_stream_names +from airbyte_cdk.sources.embedded.runner import SourceRunner +from airbyte_cdk.sources.embedded.tools import get_defined_id +from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit +from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type + +TOutput = TypeVar("TOutput") + + +class BaseEmbeddedIntegration(ABC, Generic[TConfig, TOutput]): + def __init__(self, runner: SourceRunner[TConfig], config: TConfig): + check_config_against_spec_or_exit(config, runner.spec()) + + self.source = runner + self.config = config + + self.last_state: Optional[AirbyteStateMessage] = None + + @abstractmethod + def _handle_record(self, record: AirbyteRecordMessage, id: Optional[str]) -> Optional[TOutput]: + """ + Turn an Airbyte record into the appropriate output type for the integration. + """ + pass + + def _load_data(self, stream_name: str, state: Optional[AirbyteStateMessage] = None) -> Iterable[TOutput]: + catalog = self.source.discover(self.config) + stream = get_stream(catalog, stream_name) + if not stream: + raise ValueError(f"Stream {stream_name} not found, the following streams are available: {', '.join(get_stream_names(catalog))}") + if SyncMode.incremental not in stream.supported_sync_modes: + configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.full_refresh) + else: + configured_catalog = create_configured_catalog(stream, sync_mode=SyncMode.incremental) + + for message in self.source.read(self.config, configured_catalog, state): + if message.type == Type.RECORD: + output = self._handle_record(message.record, get_defined_id(stream, message.record.data)) + if output: + yield output + elif message.type is Type.STATE and message.state: + self.last_state = message.state diff --git a/airbyte-cdk/python/airbyte_cdk/sources/embedded/catalog.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/catalog.py new file mode 100644 index 000000000000..765e9b260233 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/embedded/catalog.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from airbyte_cdk.models import ( + AirbyteCatalog, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) +from airbyte_cdk.sources.embedded.tools import get_first + + +def get_stream(catalog: AirbyteCatalog, stream_name: str) -> Optional[AirbyteStream]: + return get_first(catalog.streams, lambda s: s.name == stream_name) + + +def get_stream_names(catalog: AirbyteCatalog) -> List[str]: + return [stream.name for stream in catalog.streams] + + +def to_configured_stream( + stream: AirbyteStream, + sync_mode: SyncMode = SyncMode.full_refresh, + destination_sync_mode: DestinationSyncMode = DestinationSyncMode.append, + cursor_field: Optional[List[str]] = None, + primary_key: Optional[List[List[str]]] = None, +) -> ConfiguredAirbyteStream: + return ConfiguredAirbyteStream( + stream=stream, sync_mode=sync_mode, destination_sync_mode=destination_sync_mode, cursor_field=cursor_field, primary_key=primary_key + ) + + +def to_configured_catalog(configured_streams: List[ConfiguredAirbyteStream]) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog(streams=configured_streams) + + +def create_configured_catalog(stream: AirbyteStream, sync_mode: SyncMode = SyncMode.full_refresh) -> ConfiguredAirbyteCatalog: + configured_streams = [to_configured_stream(stream, sync_mode=sync_mode, primary_key=stream.source_defined_primary_key)] + + return to_configured_catalog(configured_streams) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/embedded/runner.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/runner.py new file mode 100644 index 000000000000..c64e66ed581e --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/embedded/runner.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from abc import ABC, abstractmethod +from typing import Generic, Iterable, Optional + +from airbyte_cdk.connector import TConfig +from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification +from airbyte_cdk.sources.source import Source + + +class SourceRunner(ABC, Generic[TConfig]): + @abstractmethod + def spec(self) -> ConnectorSpecification: + pass + + @abstractmethod + def discover(self, config: TConfig) -> AirbyteCatalog: + pass + + @abstractmethod + def read(self, config: TConfig, catalog: ConfiguredAirbyteCatalog, state: Optional[AirbyteStateMessage]) -> Iterable[AirbyteMessage]: + pass + + +class CDKRunner(SourceRunner[TConfig]): + def __init__(self, source: Source, name: str): + self._source = source + self._logger = logging.getLogger(name) + + def spec(self) -> ConnectorSpecification: + return self._source.spec(self._logger) + + def discover(self, config: TConfig) -> AirbyteCatalog: + return self._source.discover(self._logger, config) + + def read(self, config: TConfig, catalog: ConfiguredAirbyteCatalog, state: Optional[AirbyteStateMessage]) -> Iterable[AirbyteMessage]: + return self._source.read(self._logger, config, catalog, state=[state] if state else []) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/embedded/tools.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/tools.py new file mode 100644 index 000000000000..5777e567dd4c --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/embedded/tools.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Callable, Dict, Iterable, Optional + +import dpath +from airbyte_cdk.models import AirbyteStream + + +def get_first(iterable: Iterable[Any], predicate: Callable[[Any], bool] = lambda m: True) -> Optional[Any]: + return next(filter(predicate, iterable), None) + + +def get_defined_id(stream: AirbyteStream, data: Dict[str, Any]) -> Optional[str]: + if not stream.source_defined_primary_key: + return None + primary_key = [] + for key in stream.source_defined_primary_key: + try: + primary_key.append(str(dpath.util.get(data, key))) + except KeyError: + primary_key.append("__not_found__") + return "_".join(primary_key) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md b/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md new file mode 100644 index 000000000000..558f189fd85c --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md @@ -0,0 +1,120 @@ +## Behavior + +The Airbyte protocol defines the actions `spec`, `discover`, `check` and `read` for a source to be compliant. Here is the high-level description of the flow for a file-based source: +* spec: calls AbstractFileBasedSpec.documentation_url and AbstractFileBasedSpec.schema to return a ConnectorSpecification. +* discover: calls Source.streams, and subsequently Stream.get_json_schema; this uses Source.open_file to open files during schema discovery. +* check: Source.check_connection is called from the entrypoint code (in the main CDK). +* read: Stream.read_records calls Stream.list_files which calls Source.list_matching_files, and then also uses Source.open_file to parse records from the file handle. + +## How to Implement Your Own +To create a file-based source a user must extend three classes – AbstractFileBasedSource, AbstractFileBasedSpec, and AbstractStreamReader – to create an implementation for the connector’s specific storage system. They then initialize a FileBasedSource with the instance of AbstractStreamReader specific to their storage system. + +The abstract classes house the vast majority of the logic required by file-based sources. For example, when extending AbstractStreamReader, users only have to implement three methods: +* list_matching_files: lists files matching the glob pattern(s) provided in the config. +* open_file: returns a file handle for reading. +* config property setter: concrete implementations of AbstractFileBasedStreamReader's config setter should assert that `value` is of the correct config type for that type of StreamReader. + +The result is that an implementation of a source might look like this: +``` +class CustomStreamReader(AbstractStreamReader): + def open_file(self, remote_file: RemoteFile) -> FileHandler: + <...> + + def get_matching_files( + self, + globs: List[str], + logger: logging.Logger, + ) -> Iterable[RemoteFile]: + <...> + + @config.setter + def config(self, value: Config): + assert isinstance(value, CustomConfig) + self._config = value + + +class CustomConfig(AbstractFileBasedSpec): + @classmethod + def documentation_url(cls) -> AnyUrl: + return AnyUrl("https://docs.airbyte.com/integrations/sources/s3", scheme="https") + + a_spec_field: str = Field(title="A Spec Field", description="This is where you describe the fields of the spec", order=0) + <...> +``` + +For more information, feel free to check the docstrings of each classes or check specific implementations (like source-s3). + +## Supported File Types + +### Avro +Avro is a serialization format developed by [Apache](https://avro.apache.org/docs/). Avro configuration options for the file-based CDK: +* `double_as_string`: Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers. + +### CSV +CSV is a format loosely described by [RFC 4180](https://www.rfc-editor.org/rfc/rfc4180). The format is quite flexible which leads to a ton of options to consider: +* `delimiter`: The character delimiting individual cells in the CSV data. By name, CSV is comma separated so the default value is `,` +* `quote_char`: When quoted fields are used, it is possible for a field to span multiple lines, even when line breaks appear within such field. The default quote character is `"`. +* `escape_char`: The character used for escaping special characters. +* `encoding`: The character encoding of the file. By default, `UTF-8` +* `double_quote`: Whether two quotes in a quoted CSV value denote a single quote in the data. +* `quoting_behavior`: The quoting behavior determines when a value in a row should have quote marks added around it. +* `skip_rows_before_header`: The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field. +* `skip_rows_after_header`: The number of rows to skip after the header row. +* `autogenerate_column_names`: If your CSV does not have a header row, the file-based CDK will need this enable to generate column names. +* `null_values`: As CSV does not explicitly define a value for null values, the user can specify a set of case-sensitive strings that should be interpreted as null values. +* `true_values`: As CSV does not explicitly define a value for positive boolean, the user can specify a set of case-sensitive strings that should be interpreted as true values. +* `false_values`: As CSV does not explicitly define a value for negative boolean, the user can specify a set of case-sensitive strings that should be interpreted as false values. + +### JSONL +[JSONL](https://jsonlines.org/) (or JSON Lines) is a format where each row is a JSON object. There are no configuration option for this format. For backward compatibility reasons, the JSONL parser currently supports multiline objects even though this is not part of the JSONL standard. Following some data gathering, we reserve the right to remove the support for this. Given that files have multiline JSON objects, performances will be slow. + +### Parquet +Parquet is a file format defined by [Apache](https://parquet.apache.org/). Configuration options are: +* `decimal_as_float`: Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended. + +## Schema + +Having a schema allows for the file-based CDK to take action when there is a discrepancy between a record and what are the expected types of the record fields. + +Schema can be either inferred or user provided. +* If the user defines it a format using JSON types, inference will not apply. Input schemas are a key/value pair of strings describing column name and data type. Supported types are `["string", "number", "integer", "object", "array", "boolean", "null"]`. For example, `{"col1": "string", "col2": "boolean"}`. +* If the user enables schemaless sync, schema will `{"data": "object"}` and therefore emitted records will look like `{"data": {"col1": val1, …}}`. This is recommended if the contents between files in the stream vary significantly, and/or if data is very nested. +* Else, the file-based CDK will infer the schema depending on the file type. Some file formats defined the schema as part of their metadata (like Parquet), some do on the record-level (like Avro) and some don't have any explicit typing (like JSON or CSV). Note that all CSV values are inferred as strings except where we are supporting legacy configurations. Any file format that does not define their schema on a metadata level will require the file-based CDK to iterate to a number of records. There is a limit of bytes that will be consumed in order to infer the schema. + +### Validation Policies +Users will be required to select one of 3 different options, in the event that records are encountered that don’t conform to the schema. + +* Skip nonconforming records: check each record to see if it conforms to the user-input or inferred schema; skip the record if it doesn't conform. We keep a count of the number of records in each file that do and do not conform and emit a log message with these counts once we’re done reading the file. +* Emit all records: emit all records, even if they do not conform to the user-provided or inferred schema. Columns that don't exist in the configured catalog probably won't be available in the destination's table since that's the current behavior. +Only error if there are conflicting field types or malformed rows. +* Stop the sync and wait for schema re-discovery: if a record is encountered that does not conform to the configured catalog’s schema, we log a message and stop the whole sync. Note: this option is not recommended if the files have very different columns or datatypes, because the inferred schema may vary significantly at discover time. + +When the `schemaless` is enabled, validation will be skipped. + +## Breaking Changes (compared to previous S3 implementation) + +* [CSV] Mapping of type `array` and `object`: before, they were mapped as `large_string` and hence casted as strings. Given the new changes, if `array` or `object` is specified, the value will be casted as `array` and `object` respectively. +* [CSV] Before, a string value would not be considered as `null_values` if the column type was a string. We will now start to cast string columns with values matching `null_values` to null. +* [CSV] `decimal_point` option is deprecated: It is not possible anymore to use another character than `.` to separate the integer part from non-integer part. Given that the float is format with another character than this, it will be considered as a string. +* [Parquet] `columns` option is deprecated: You can use Airbyte column selection in order to have the same behavior. We don't expect it, but this could have impact on the performance as payload could be bigger. + +## Incremental syncs +The file-based connectors supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :--------------------------------------------- |:-----------| +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| Replicate Incremental Deletes | No | +| Replicate Multiple Files \(pattern matching\) | Yes | +| Replicate Multiple Streams \(distinct tables\) | Yes | +| Namespaces | No | + +We recommend you do not manually modify files that are already synced. The connector has file-level granularity, which means adding or modifying a row in a CSV file will trigger a re-sync of the content of that file. + +### Incremental sync +After the initial sync, the connector only pulls files that were modified since the last sync. + +The connector checkpoints the connection states when it is done syncing all files for a given timestamp. The connection's state only keeps track of the last 10 000 files synced. If more than 10 000 files are synced, the connector won't be able to rely on the connection state to deduplicate files. In this case, the connector will initialize its cursor to the minimum between the earliest file in the history, or 3 days ago. + +Both the maximum number of files, and the time buffer can be configured by connector developers. diff --git a/airbyte-integrations/connectors/source-exchange-rates/unit_tests/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-exchange-rates/unit_tests/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/file_based/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py new file mode 100644 index 000000000000..983f4eeb8bf7 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/__init__.py @@ -0,0 +1,4 @@ +from .abstract_file_based_availability_strategy import AbstractFileBasedAvailabilityStrategy +from .default_file_based_availability_strategy import DefaultFileBasedAvailabilityStrategy + +__all__ = ["AbstractFileBasedAvailabilityStrategy", "DefaultFileBasedAvailabilityStrategy"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py new file mode 100644 index 000000000000..1ba12f64febd --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/abstract_file_based_availability_strategy.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import abstractmethod +from typing import TYPE_CHECKING, Optional, Tuple + +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.core import Stream + +if TYPE_CHECKING: + from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream + + +class AbstractFileBasedAvailabilityStrategy(AvailabilityStrategy): + @abstractmethod + def check_availability(self, stream: Stream, logger: logging.Logger, _: Optional[Source]) -> Tuple[bool, Optional[str]]: + """ + Perform a connection check for the stream. + + Returns (True, None) if successful, otherwise (False, ). + """ + ... + + @abstractmethod + def check_availability_and_parsability( + self, stream: "AbstractFileBasedStream", logger: logging.Logger, _: Optional[Source] + ) -> Tuple[bool, Optional[str]]: + """ + Performs a connection check for the stream, as well as additional checks that + verify that the connection is working as expected. + + Returns (True, None) if successful, otherwise (False, ). + """ + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py new file mode 100644 index 000000000000..ff7390167068 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +import traceback +from typing import TYPE_CHECKING, List, Optional, Tuple + +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy +from airbyte_cdk.sources.file_based.exceptions import CheckAvailabilityError, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import conforms_to_schema + +if TYPE_CHECKING: + from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream + + +class DefaultFileBasedAvailabilityStrategy(AbstractFileBasedAvailabilityStrategy): + def __init__(self, stream_reader: AbstractFileBasedStreamReader): + self.stream_reader = stream_reader + + def check_availability(self, stream: "AbstractFileBasedStream", logger: logging.Logger, _: Optional[Source]) -> Tuple[bool, Optional[str]]: # type: ignore[override] + """ + Perform a connection check for the stream (verify that we can list files from the stream). + + Returns (True, None) if successful, otherwise (False, ). + """ + try: + self._check_list_files(stream) + except CheckAvailabilityError: + return False, "".join(traceback.format_exc()) + + return True, None + + def check_availability_and_parsability( + self, stream: "AbstractFileBasedStream", logger: logging.Logger, _: Optional[Source] + ) -> Tuple[bool, Optional[str]]: + """ + Perform a connection check for the stream. + + Returns (True, None) if successful, otherwise (False, ). + + For the stream: + - Verify that we can list files from the stream using the configured globs. + - Verify that we can read one file from the stream. + + This method will also check that the files and their contents are consistent + with the configured options, as follows: + - If the files have extensions, verify that they don't disagree with the + configured file type. + - If the user provided a schema in the config, check that a subset of records in + one file conform to the schema via a call to stream.conforms_to_schema(schema). + """ + try: + files = self._check_list_files(stream) + self._check_parse_record(stream, files[0], logger) + except CheckAvailabilityError: + return False, "".join(traceback.format_exc()) + + return True, None + + def _check_list_files(self, stream: "AbstractFileBasedStream") -> List[RemoteFile]: + try: + files = stream.list_files() + except Exception as exc: + raise CheckAvailabilityError(FileBasedSourceError.ERROR_LISTING_FILES, stream=stream.name) from exc + + if not files: + raise CheckAvailabilityError(FileBasedSourceError.EMPTY_STREAM, stream=stream.name) + + return files + + def _check_parse_record(self, stream: "AbstractFileBasedStream", file: RemoteFile, logger: logging.Logger) -> None: + parser = stream.get_parser(stream.config.file_type) + + try: + record = next(iter(parser.parse_records(stream.config, file, self.stream_reader, logger, discovered_schema=None))) + except StopIteration: + # The file is empty. We've verified that we can open it, so will + # consider the connection check successful even though it means + # we skip the schema validation check. + return + except Exception as exc: + raise CheckAvailabilityError(FileBasedSourceError.ERROR_READING_FILE, stream=stream.name, file=file.uri) from exc + + schema = stream.catalog_schema or stream.config.input_schema + if schema and stream.validation_policy.validate_schema_before_sync: + if not conforms_to_schema(record, schema): # type: ignore + raise CheckAvailabilityError( + FileBasedSourceError.ERROR_VALIDATING_RECORD, + stream=stream.name, + file=file.uri, + ) + + return None diff --git a/tools/ci_code_validator/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/__init__.py similarity index 100% rename from tools/ci_code_validator/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/file_based/config/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py new file mode 100644 index 000000000000..531502e66206 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import copy +from abc import abstractmethod +from typing import Any, Dict, List, Optional + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.utils import schema_helpers +from pydantic import AnyUrl, BaseModel, Field + + +class AbstractFileBasedSpec(BaseModel): + """ + Used during spec; allows the developer to configure the cloud provider specific options + that are needed when users configure a file-based source. + """ + + start_date: Optional[str] = Field( + title="Start Date", + description="UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + examples=["2021-01-01T00:00:00.000000Z"], + format="date-time", + pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + pattern_descriptor="YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + order=1, + ) + + streams: List[FileBasedStreamConfig] = Field( + title="The list of streams to sync", + description='Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.', + order=10, + ) + + @classmethod + @abstractmethod + def documentation_url(cls) -> AnyUrl: + """ + :return: link to docs page for this source e.g. "https://docs.airbyte.com/integrations/sources/s3" + """ + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the mapping comprised of the config fields + """ + schema = super().schema(*args, **kwargs) + transformed_schema = copy.deepcopy(schema) + schema_helpers.expand_refs(transformed_schema) + cls.replace_enum_allOf_and_anyOf(transformed_schema) + + return transformed_schema + + @staticmethod + def replace_enum_allOf_and_anyOf(schema: Dict[str, Any]) -> Dict[str, Any]: + """ + allOfs are not supported by the UI, but pydantic is automatically writing them for enums. + Unpacks the enums under allOf and moves them up a level under the enum key + anyOfs are also not supported by the UI, so we replace them with the similar oneOf, with the + additional validation that an incoming config only matches exactly one of a field's types. + """ + objects_to_check = schema["properties"]["streams"]["items"]["properties"]["format"] + objects_to_check["type"] = "object" + objects_to_check["oneOf"] = objects_to_check.pop("anyOf", []) + for format in objects_to_check["oneOf"]: + for key in format["properties"]: + object_property = format["properties"][key] + if "allOf" in object_property and "enum" in object_property["allOf"][0]: + object_property["enum"] = object_property["allOf"][0]["enum"] + object_property.pop("allOf") + + properties_to_change = ["validation_policy"] + for property_to_change in properties_to_change: + property_object = schema["properties"]["streams"]["items"]["properties"][property_to_change] + if "anyOf" in property_object: + schema["properties"]["streams"]["items"]["properties"][property_to_change]["type"] = "object" + schema["properties"]["streams"]["items"]["properties"][property_to_change]["oneOf"] = property_object.pop("anyOf") + if "allOf" in property_object and "enum" in property_object["allOf"][0]: + property_object["enum"] = property_object["allOf"][0]["enum"] + property_object.pop("allOf") + return schema diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py new file mode 100644 index 000000000000..ee7b955a325b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pydantic import BaseModel, Field +from typing_extensions import Literal + + +class AvroFormat(BaseModel): + class Config: + title = "Avro Format" + + filetype: Literal["avro"] = "avro" + + double_as_string: bool = Field( + title="Convert Double Fields to Strings", + description="Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + default=False, + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py new file mode 100644 index 000000000000..2d8106131c70 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import codecs +from enum import Enum +from typing import Optional, Set + +from pydantic import BaseModel, Field, validator +from typing_extensions import Literal + + +class InferenceType(Enum): + NONE = "None" + PRIMITIVE_TYPES_ONLY = "Primitive Types Only" + + +DEFAULT_TRUE_VALUES = ["y", "yes", "t", "true", "on", "1"] +DEFAULT_FALSE_VALUES = ["n", "no", "f", "false", "off", "0"] + + +class CsvFormat(BaseModel): + class Config: + title = "CSV Format" + + filetype: Literal["csv"] = "csv" + delimiter: str = Field( + title="Delimiter", + description="The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + default=",", + ) + quote_char: str = Field( + title="Quote Character", + default='"', + description="The character used for quoting CSV values. To disallow quoting, make this field blank.", + ) + escape_char: Optional[str] = Field( + title="Escape Character", + default=None, + description="The character used for escaping special characters. To disallow escaping, leave this field blank.", + ) + encoding: Optional[str] = Field( + default="utf8", + description='The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.', + ) + double_quote: bool = Field( + title="Double Quote", default=True, description="Whether two quotes in a quoted CSV value denote a single quote in the data." + ) + null_values: Set[str] = Field( + title="Null Values", + default=[], + description="A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + ) + strings_can_be_null: bool = Field( + title="Strings Can Be Null", + default=True, + description="Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + ) + skip_rows_before_header: int = Field( + title="Skip Rows Before Header", + default=0, + description="The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + ) + skip_rows_after_header: int = Field( + title="Skip Rows After Header", default=0, description="The number of rows to skip after the header row." + ) + autogenerate_column_names: bool = Field( + title="Autogenerate Column Names", + default=False, + description="Whether to autogenerate column names if column_names is empty. If true, column names will be of the form “f0”, “f1”… If false, column names will be read from the first CSV row after skip_rows_before_header.", + ) + true_values: Set[str] = Field( + title="True Values", + default=DEFAULT_TRUE_VALUES, + description="A set of case-sensitive strings that should be interpreted as true values.", + ) + false_values: Set[str] = Field( + title="False Values", + default=DEFAULT_FALSE_VALUES, + description="A set of case-sensitive strings that should be interpreted as false values.", + ) + inference_type: InferenceType = Field( + title="Inference Type", + default=InferenceType.NONE, + description="How to infer the types of the columns. If none, inference default to strings.", + airbyte_hidden=True, + ) + + @validator("delimiter") + def validate_delimiter(cls, v: str) -> str: + if len(v) != 1: + raise ValueError("delimiter should only be one character") + if v in {"\r", "\n"}: + raise ValueError(f"delimiter cannot be {v}") + return v + + @validator("quote_char") + def validate_quote_char(cls, v: str) -> str: + if len(v) != 1: + raise ValueError("quote_char should only be one character") + return v + + @validator("escape_char") + def validate_escape_char(cls, v: str) -> str: + if v is not None and len(v) != 1: + raise ValueError("escape_char should only be one character") + return v + + @validator("encoding") + def validate_encoding(cls, v: str) -> str: + try: + codecs.lookup(v) + except LookupError: + raise ValueError(f"invalid encoding format: {v}") + return v diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py new file mode 100644 index 000000000000..6f38ed4abf56 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from enum import Enum +from typing import Any, List, Mapping, Optional, Type, Union + +from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat +from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.schema_helpers import type_mapping_to_jsonschema +from pydantic import BaseModel, Field, validator + +PrimaryKeyType = Optional[Union[str, List[str]]] + + +VALID_FILE_TYPES: Mapping[str, Type[BaseModel]] = {"avro": AvroFormat, "csv": CsvFormat, "jsonl": JsonlFormat, "parquet": ParquetFormat} + + +class ValidationPolicy(Enum): + emit_record = "Emit Record" + skip_record = "Skip Record" + wait_for_discover = "Wait for Discover" + + +class FileBasedStreamConfig(BaseModel): + name: str = Field(title="Name", description="The name of the stream.") + file_type: str = Field(title="File Type", description="The data file type that is being extracted for a stream.") + globs: Optional[List[str]] = Field( + title="Globs", + description='The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.', + ) + legacy_prefix: Optional[str] = Field( + title="Legacy Prefix", + description="The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.", + airbyte_hidden=True, + ) + validation_policy: ValidationPolicy = Field( + title="Validation Policy", + description="The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + default=ValidationPolicy.emit_record, + ) + input_schema: Optional[str] = Field( + title="Input Schema", + description="The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + ) + primary_key: Optional[str] = Field( + title="Primary Key", description="The column or columns (for a composite key) that serves as the unique identifier of a record." + ) + days_to_sync_if_history_is_full: int = Field( + title="Days To Sync If History Is Full", + description="When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + default=3, + ) + format: Optional[Union[AvroFormat, CsvFormat, JsonlFormat, ParquetFormat]] = Field( + title="Format", + description="The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + ) + schemaless: bool = Field( + title="Schemaless", + description="When enabled, syncs will not validate or structure records against the stream's schema.", + default=False, + ) + + @validator("file_type", pre=True) + def validate_file_type(cls, v: str) -> str: + if v not in VALID_FILE_TYPES: + raise ValueError(f"Format filetype {v} is not a supported file type") + return v + + @classmethod + def _transform_legacy_config(cls, legacy_config: Mapping[str, Any], file_type: str) -> Mapping[str, Any]: + if file_type.casefold() not in VALID_FILE_TYPES: + raise ValueError(f"Format filetype {file_type} is not a supported file type") + if file_type.casefold() == "parquet" or file_type.casefold() == "avro": + legacy_config = cls._transform_legacy_parquet_or_avro_config(legacy_config) + return {file_type: VALID_FILE_TYPES[file_type.casefold()].parse_obj({key: val for key, val in legacy_config.items()})} + + @classmethod + def _transform_legacy_parquet_or_avro_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + The legacy parquet parser converts decimal fields to numbers. This isn't desirable because it can lead to precision loss. + To avoid introducing a breaking change with the new default, we will set decimal_as_float to True in the legacy configs. + """ + filetype = config.get("filetype") + if filetype != "parquet" and filetype != "avro": + raise ValueError( + f"Expected {filetype} format, got {config}. This is probably due to a CDK bug. Please reach out to the Airbyte team for support." + ) + if config.get("decimal_as_float"): + raise ValueError( + f"Received legacy {filetype} file form with 'decimal_as_float' set. This is unexpected. Please reach out to the Airbyte team for support." + ) + return {**config, **{"decimal_as_float": True}} + + @validator("input_schema", pre=True) + def validate_input_schema(cls, v: Optional[str]) -> Optional[str]: + if v: + if type_mapping_to_jsonschema(v): + return v + else: + raise ConfigValidationError(FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA) + return None + + def get_input_schema(self) -> Optional[Mapping[str, Any]]: + """ + User defined input_schema is defined as a string in the config. This method takes the string representation + and converts it into a Mapping[str, Any] which is used by file-based CDK components. + """ + if self.input_schema: + schema = type_mapping_to_jsonschema(self.input_schema) + if not schema: + raise ValueError(f"Unable to create JSON schema from input schema {self.input_schema}") + return schema + return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py new file mode 100644 index 000000000000..99010d3aeca5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pydantic import BaseModel +from typing_extensions import Literal + + +class JsonlFormat(BaseModel): + class Config: + title = "Jsonl Format" + + filetype: Literal["jsonl"] = "jsonl" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py new file mode 100644 index 000000000000..de7f1b62969d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pydantic import BaseModel, Field +from typing_extensions import Literal + + +class ParquetFormat(BaseModel): + class Config: + title = "Parquet Format" + + filetype: Literal["parquet"] = "parquet" + # This option is not recommended, but necessary for backwards compatibility + decimal_as_float: bool = Field( + title="Convert Decimal Fields to Floats", + description="Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + default=False, + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/__init__.py new file mode 100644 index 000000000000..c50aa1a4e70f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/__init__.py @@ -0,0 +1,4 @@ +from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import AbstractDiscoveryPolicy +from airbyte_cdk.sources.file_based.discovery_policy.default_discovery_policy import DefaultDiscoveryPolicy + +__all__ = ["AbstractDiscoveryPolicy", "DefaultDiscoveryPolicy"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py new file mode 100644 index 000000000000..d8cc5d2c4a74 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod + + +class AbstractDiscoveryPolicy(ABC): + """ + Used during discovery; allows the developer to configure the number of concurrent + requests to send to the source, and the number of files to use for schema discovery. + """ + + @property + @abstractmethod + def n_concurrent_requests(self) -> int: + ... + + @property + @abstractmethod + def max_n_files_for_schema_inference(self) -> int: + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py new file mode 100644 index 000000000000..56bd19d01f16 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import AbstractDiscoveryPolicy + +DEFAULT_N_CONCURRENT_REQUESTS = 10 +DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE = 10 + + +class DefaultDiscoveryPolicy(AbstractDiscoveryPolicy): + """ + Default number of concurrent requests to send to the source on discover, and number + of files to use for schema inference. + """ + + @property + def n_concurrent_requests(self) -> int: + return DEFAULT_N_CONCURRENT_REQUESTS + + @property + def max_n_files_for_schema_inference(self) -> int: + return DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py new file mode 100644 index 000000000000..3d2cf212fe58 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from enum import Enum + + +class FileBasedSourceError(Enum): + EMPTY_STREAM = "No files were identified in the stream. This may be because there are no files in the specified container, or because your glob patterns did not match any files. Please verify that your source contains files last modified after the start_date and that your glob patterns are not overly strict." + GLOB_PARSE_ERROR = ( + "Error parsing glob pattern. Please refer to the glob pattern rules at https://facelessuser.github.io/wcmatch/glob/#split." + ) + ERROR_CASTING_VALUE = "Could not cast the value to the expected type." + ERROR_CASTING_VALUE_UNRECOGNIZED_TYPE = "Could not cast the value to the expected type because the type is not recognized. Valid types are null, array, boolean, integer, number, object, and string." + ERROR_DECODING_VALUE = "Expected a JSON-decodeable value but could not decode record." + ERROR_LISTING_FILES = ( + "Error listing files. Please check the credentials provided in the config and verify that they provide permission to list files." + ) + ERROR_READING_FILE = ( + "Error opening file. Please check the credentials provided in the config and verify that they provide permission to read files." + ) + ERROR_PARSING_RECORD = "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable." + ERROR_PARSING_USER_PROVIDED_SCHEMA = "The provided schema could not be transformed into valid JSON Schema." + ERROR_VALIDATING_RECORD = "One or more records do not pass the schema validation policy. Please modify your input schema, or select a more lenient validation policy." + STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY = ( + "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema." + ) + NULL_VALUE_IN_SCHEMA = "Error during schema inference: no type was detected for key." + UNRECOGNIZED_TYPE = "Error during schema inference: unrecognized type." + SCHEMA_INFERENCE_ERROR = "Error inferring schema from files. Are the files valid?" + INVALID_SCHEMA_ERROR = "No fields were identified for this schema. This may happen if the stream is empty. Please check your configuration to verify that there are files that match the stream's glob patterns." + CONFIG_VALIDATION_ERROR = "Error creating stream config object." + MISSING_SCHEMA = "Expected `json_schema` in the configured catalog but it is missing." + UNDEFINED_PARSER = "No parser is defined for this file type." + UNDEFINED_VALIDATION_POLICY = "The validation policy defined in the config does not exist for the source." + + +class BaseFileBasedSourceError(Exception): + def __init__(self, error: FileBasedSourceError, **kwargs): # type: ignore # noqa + super().__init__( + f"{FileBasedSourceError(error).value} Contact Support if you need assistance.\n{' '.join([f'{k}={v}' for k, v in kwargs.items()])}" + ) + + +class ConfigValidationError(BaseFileBasedSourceError): + pass + + +class InvalidSchemaError(BaseFileBasedSourceError): + pass + + +class MissingSchemaError(BaseFileBasedSourceError): + pass + + +class RecordParseError(BaseFileBasedSourceError): + pass + + +class SchemaInferenceError(BaseFileBasedSourceError): + pass + + +class CheckAvailabilityError(BaseFileBasedSourceError): + pass + + +class UndefinedParserError(BaseFileBasedSourceError): + pass + + +class StopSyncPerValidationPolicy(BaseFileBasedSourceError): + pass + + +class ErrorListingFiles(BaseFileBasedSourceError): + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py new file mode 100644 index 000000000000..c5b643d6fa63 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py @@ -0,0 +1,134 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +import traceback +from abc import ABC +from typing import Any, List, Mapping, Optional, Tuple, Type + +from airbyte_cdk.models import ConnectorSpecification +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy +from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types import default_parsers +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES, AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream, DefaultFileBasedStream +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.streams import Stream +from pydantic.error_wrappers import ValidationError + + +class FileBasedSource(AbstractSource, ABC): + def __init__( + self, + stream_reader: AbstractFileBasedStreamReader, + spec_class: Type[AbstractFileBasedSpec], + catalog_path: Optional[str] = None, + availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None, + discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy(), + parsers: Mapping[str, FileTypeParser] = default_parsers, + validation_policies: Mapping[ValidationPolicy, AbstractSchemaValidationPolicy] = DEFAULT_SCHEMA_VALIDATION_POLICIES, + cursor_cls: Type[AbstractFileBasedCursor] = DefaultFileBasedCursor, + ): + self.stream_reader = stream_reader + self.spec_class = spec_class + self.availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(stream_reader) + self.discovery_policy = discovery_policy + self.parsers = parsers + self.validation_policies = validation_policies + catalog = self.read_catalog(catalog_path) if catalog_path else None + self.stream_schemas = {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {} + self.cursor_cls = cursor_cls + self.logger = logging.getLogger(f"airbyte.{self.name}") + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + """ + Check that the source can be accessed using the user-provided configuration. + + For each stream, verify that we can list and read files. + + Returns (True, None) if the connection check is successful. + + Otherwise, the "error" object should describe what went wrong. + """ + streams = self.streams(config) + if len(streams) == 0: + return ( + False, + f"No streams are available for source {self.name}. This is probably an issue with the connector. Please verify that your " + f"configuration provides permissions to list and read files from the source. Contact support if you are unable to " + f"resolve this issue.", + ) + + errors = [] + for stream in streams: + if not isinstance(stream, AbstractFileBasedStream): + raise ValueError(f"Stream {stream} is not a file-based stream.") + try: + ( + stream_is_available, + reason, + ) = stream.availability_strategy.check_availability_and_parsability(stream, logger, self) + except Exception: + errors.append(f"Unable to connect to stream {stream} - {''.join(traceback.format_exc())}") + else: + if not stream_is_available and reason: + errors.append(reason) + + return not bool(errors), (errors or None) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + Return a list of this source's streams. + """ + try: + parsed_config = self.spec_class(**config) + self.stream_reader.config = parsed_config + streams: List[Stream] = [] + for stream_config in parsed_config.streams: + self._validate_input_schema(stream_config) + streams.append( + DefaultFileBasedStream( + config=stream_config, + catalog_schema=self.stream_schemas.get(stream_config.name), + stream_reader=self.stream_reader, + availability_strategy=self.availability_strategy, + discovery_policy=self.discovery_policy, + parsers=self.parsers, + validation_policy=self._validate_and_get_validation_policy(stream_config), + cursor=self.cursor_cls(stream_config), + ) + ) + return streams + + except ValidationError as exc: + raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) from exc + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + """ + Returns the specification describing what fields can be configured by a user when setting up a file-based source. + """ + + return ConnectorSpecification( + documentationUrl=self.spec_class.documentation_url(), + connectionSpecification=self.spec_class.schema(), + ) + + def _validate_and_get_validation_policy(self, stream_config: FileBasedStreamConfig) -> AbstractSchemaValidationPolicy: + if stream_config.validation_policy not in self.validation_policies: + # This should never happen because we validate the config against the schema's validation_policy enum + raise ValidationError( + f"`validation_policy` must be one of {list(self.validation_policies.keys())}", model=FileBasedStreamConfig + ) + return self.validation_policies[stream_config.validation_policy] + + def _validate_input_schema(self, stream_config: FileBasedStreamConfig) -> None: + if stream_config.schemaless and stream_config.input_schema: + raise ValidationError("`input_schema` and `schemaless` options cannot both be set", model=FileBasedStreamConfig) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_stream_reader.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_stream_reader.py new file mode 100644 index 000000000000..4a7de3bb6992 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_stream_reader.py @@ -0,0 +1,107 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import ABC, abstractmethod +from datetime import datetime +from enum import Enum +from io import IOBase +from typing import Iterable, List, Optional, Set + +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from wcmatch.glob import GLOBSTAR, globmatch + + +class FileReadMode(Enum): + READ = "r" + READ_BINARY = "rb" + + +class AbstractFileBasedStreamReader(ABC): + DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + def __init__(self) -> None: + self._config = None + + @property + def config(self) -> Optional[AbstractFileBasedSpec]: + return self._config + + @config.setter + @abstractmethod + def config(self, value: AbstractFileBasedSpec) -> None: + """ + FileBasedSource reads the config from disk and parses it, and once parsed, the source sets the config on its StreamReader. + + Note: FileBasedSource only requires the keys defined in the abstract config, whereas concrete implementations of StreamReader + will require keys that (for example) allow it to authenticate with the 3rd party. + + Therefore, concrete implementations of AbstractFileBasedStreamReader's config setter should assert that `value` is of the correct + config type for that type of StreamReader. + """ + ... + + @abstractmethod + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + """ + Return a file handle for reading. + + Many sources will be able to use smart_open to implement this method, + for example: + + client = boto3.Session(...) + return smart_open.open(remote_file.uri, transport_params={"client": client}) + """ + ... + + @abstractmethod + def get_matching_files( + self, + globs: List[str], + prefix: Optional[str], + logger: logging.Logger, + ) -> Iterable[RemoteFile]: + """ + Return all files that match any of the globs. + + Example: + + The source has files "a.json", "foo/a.json", "foo/bar/a.json" + + If globs = ["*.json"] then this method returns ["a.json"]. + + If globs = ["foo/*.json"] then this method returns ["foo/a.json"]. + + Utility method `self.filter_files_by_globs` and `self.get_prefixes_from_globs` + are available, which may be helpful when implementing this method. + """ + ... + + def filter_files_by_globs_and_start_date(self, files: List[RemoteFile], globs: List[str]) -> Iterable[RemoteFile]: + """ + Utility method for filtering files based on globs. + """ + start_date = datetime.strptime(self.config.start_date, self.DATE_TIME_FORMAT) if self.config and self.config.start_date else None + seen = set() + + for file in files: + if self.file_matches_globs(file, globs): + if file.uri not in seen and (not start_date or file.last_modified >= start_date): + seen.add(file.uri) + yield file + + @staticmethod + def file_matches_globs(file: RemoteFile, globs: List[str]) -> bool: + # Use the GLOBSTAR flag to enable recursive ** matching + # (https://facelessuser.github.io/wcmatch/wcmatch/#globstar) + return any(globmatch(file.uri, g, flags=GLOBSTAR) for g in globs) + + @staticmethod + def get_prefixes_from_globs(globs: List[str]) -> Set[str]: + """ + Utility method for extracting prefixes from the globs. + """ + prefixes = {glob.split("*")[0] for glob in globs} + return set(filter(lambda x: bool(x), prefixes)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py new file mode 100644 index 000000000000..80922439a45f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py @@ -0,0 +1,16 @@ +from typing import Mapping + +from .avro_parser import AvroParser +from .csv_parser import CsvParser +from .file_type_parser import FileTypeParser +from .jsonl_parser import JsonlParser +from .parquet_parser import ParquetParser + +default_parsers: Mapping[str, FileTypeParser] = { + "avro": AvroParser(), + "csv": CsvParser(), + "jsonl": JsonlParser(), + "parquet": ParquetParser(), +} + +__all__ = ["AvroParser", "CsvParser", "JsonlParser", "ParquetParser", "default_parsers"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py new file mode 100644 index 000000000000..d578ce957c79 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py @@ -0,0 +1,173 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +import uuid +from typing import Any, Dict, Iterable, Mapping, Optional + +import fastavro +from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import SchemaType + +AVRO_TYPE_TO_JSON_TYPE = { + "null": "null", + "boolean": "boolean", + "int": "integer", + "long": "integer", + "float": "number", + "double": "string", # double -> number conversions can lose precision + "bytes": "string", + "string": "string", +} + +AVRO_LOGICAL_TYPE_TO_JSON = { + "decimal": {"type": "string"}, + "uuid": {"type": "string"}, + "date": {"type": "string", "format": "date"}, + "time-millis": {"type": "integer"}, + "time-micros": {"type": "integer"}, + "timestamp-millis": {"type": "string", "format": "date-time"}, + "timestamp-micros": {"type": "string"}, + "local-timestamp-millis": {"type": "string", "format": "date-time"}, + "local-timestamp-micros": {"type": "string"}, + # fastavro does not support duration https://fastavro.readthedocs.io/en/latest/logical_types.html +} + + +class AvroParser(FileTypeParser): + ENCODING = None + + async def infer_schema( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + ) -> SchemaType: + avro_format = config.format or AvroFormat() + if not isinstance(avro_format, AvroFormat): + raise ValueError(f"Expected ParquetFormat, got {avro_format}") + + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + avro_reader = fastavro.reader(fp) + avro_schema = avro_reader.writer_schema + if not avro_schema["type"] == "record": + unsupported_type = avro_schema["type"] + raise ValueError(f"Only record based avro files are supported. Found {unsupported_type}") + json_schema = { + field["name"]: AvroParser._convert_avro_type_to_json(avro_format, field["name"], field["type"]) + for field in avro_schema["fields"] + } + return json_schema + + @classmethod + def _convert_avro_type_to_json(cls, avro_format: AvroFormat, field_name: str, avro_field: str) -> Mapping[str, Any]: + if isinstance(avro_field, str) and avro_field in AVRO_TYPE_TO_JSON_TYPE: + # Legacy behavior to retain backwards compatibility. Long term we should always represent doubles as strings + if avro_field == "double" and not avro_format.double_as_string: + return {"type": "number"} + return {"type": AVRO_TYPE_TO_JSON_TYPE[avro_field]} + if isinstance(avro_field, Mapping): + if avro_field["type"] == "record": + return { + "type": "object", + "properties": { + object_field["name"]: AvroParser._convert_avro_type_to_json(avro_format, object_field["name"], object_field["type"]) + for object_field in avro_field["fields"] + }, + } + elif avro_field["type"] == "array": + if "items" not in avro_field: + raise ValueError(f"{field_name} array type does not have a required field items") + return {"type": "array", "items": AvroParser._convert_avro_type_to_json(avro_format, "", avro_field["items"])} + elif avro_field["type"] == "enum": + if "symbols" not in avro_field: + raise ValueError(f"{field_name} enum type does not have a required field symbols") + if "name" not in avro_field: + raise ValueError(f"{field_name} enum type does not have a required field name") + return {"type": "string", "enum": avro_field["symbols"]} + elif avro_field["type"] == "map": + if "values" not in avro_field: + raise ValueError(f"{field_name} map type does not have a required field values") + return { + "type": "object", + "additionalProperties": AvroParser._convert_avro_type_to_json(avro_format, "", avro_field["values"]), + } + elif avro_field["type"] == "fixed" and avro_field.get("logicalType") != "duration": + if "size" not in avro_field: + raise ValueError(f"{field_name} fixed type does not have a required field size") + if not isinstance(avro_field["size"], int): + raise ValueError(f"{field_name} fixed type size value is not an integer") + return { + "type": "string", + "pattern": f"^[0-9A-Fa-f]{{{avro_field['size'] * 2}}}$", + } + elif avro_field.get("logicalType") == "decimal": + if "precision" not in avro_field: + raise ValueError(f"{field_name} decimal type does not have a required field precision") + if "scale" not in avro_field: + raise ValueError(f"{field_name} decimal type does not have a required field scale") + max_whole_number_range = avro_field["precision"] - avro_field["scale"] + decimal_range = avro_field["scale"] + + # This regex looks like a mess, but it is validation for at least one whole number and optional fractional numbers + # For example: ^-?\d{1,5}(?:\.\d{1,3})?$ would accept 12345.123 and 123456.12345 would be rejected + return {"type": "string", "pattern": f"^-?\\d{{{1,max_whole_number_range}}}(?:\\.\\d{1,decimal_range})?$"} + elif "logicalType" in avro_field: + if avro_field["logicalType"] not in AVRO_LOGICAL_TYPE_TO_JSON: + raise ValueError(f"{avro_field['logical_type']} is not a valid Avro logical type") + return AVRO_LOGICAL_TYPE_TO_JSON[avro_field["logicalType"]] + else: + raise ValueError(f"Unsupported avro type: {avro_field}") + else: + raise ValueError(f"Unsupported avro type: {avro_field}") + + def parse_records( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + discovered_schema: Optional[Mapping[str, SchemaType]], + ) -> Iterable[Dict[str, Any]]: + avro_format = config.format or AvroFormat() + if not isinstance(avro_format, AvroFormat): + raise ValueError(f"Expected ParquetFormat, got {avro_format}") + + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + avro_reader = fastavro.reader(fp) + schema = avro_reader.writer_schema + schema_field_name_to_type = {field["name"]: field["type"] for field in schema["fields"]} + for record in avro_reader: + yield { + record_field: self._to_output_value(avro_format, schema_field_name_to_type[record_field], record[record_field]) + for record_field, record_value in schema_field_name_to_type.items() + } + + @property + def file_read_mode(self) -> FileReadMode: + return FileReadMode.READ_BINARY + + @staticmethod + def _to_output_value(avro_format: AvroFormat, record_type: Mapping[str, Any], record_value: Any) -> Any: + if not isinstance(record_type, Mapping): + if record_type == "double" and avro_format.double_as_string: + return str(record_value) + return record_value + if record_type.get("logicalType") == "uuid": + return uuid.UUID(bytes=record_value) + elif record_type.get("logicalType") == "decimal": + return str(record_value) + elif record_type.get("logicalType") == "date": + return record_value.isoformat() + elif record_type.get("logicalType") == "local-timestamp-millis": + return record_value.isoformat(sep="T", timespec="milliseconds") + elif record_type.get("logicalType") == "local-timestamp-micros": + return record_value.isoformat(sep="T", timespec="microseconds") + else: + return record_value diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py new file mode 100644 index 000000000000..5468a4cc6144 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py @@ -0,0 +1,408 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import csv +import json +import logging +from abc import ABC, abstractmethod +from collections import defaultdict +from functools import partial +from io import IOBase +from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set + +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, InferenceType +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import TYPE_PYTHON_MAPPING, SchemaType + +DIALECT_NAME = "_config_dialect" + + +class _CsvReader: + def read_data( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + file_read_mode: FileReadMode, + ) -> Generator[Dict[str, Any], None, None]: + config_format = _extract_format(config) + + # Formats are configured individually per-stream so a unique dialect should be registered for each stream. + # We don't unregister the dialect because we are lazily parsing each csv file to generate records + # This will potentially be a problem if we ever process multiple streams concurrently + dialect_name = config.name + DIALECT_NAME + csv.register_dialect( + dialect_name, + delimiter=config_format.delimiter, + quotechar=config_format.quote_char, + escapechar=config_format.escape_char, + doublequote=config_format.double_quote, + quoting=csv.QUOTE_MINIMAL, + ) + with stream_reader.open_file(file, file_read_mode, config_format.encoding, logger) as fp: + headers = self._get_headers(fp, config_format, dialect_name) + + # we assume that if we autogenerate columns, it is because we don't have headers + # if a user wants to autogenerate_column_names with a CSV having headers, he can skip rows + rows_to_skip = ( + config_format.skip_rows_before_header + + (0 if config_format.autogenerate_column_names else 1) + + config_format.skip_rows_after_header + ) + self._skip_rows(fp, rows_to_skip) + + reader = csv.DictReader(fp, dialect=dialect_name, fieldnames=headers) # type: ignore + try: + for row in reader: + # The row was not properly parsed if any of the values are None. This will most likely occur if there are more columns + # than headers or more headers dans columns + if None in row or None in row.values(): + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD) + yield row + finally: + # due to RecordParseError or GeneratorExit + csv.unregister_dialect(dialect_name) + + def _get_headers(self, fp: IOBase, config_format: CsvFormat, dialect_name: str) -> List[str]: + """ + Assumes the fp is pointing to the beginning of the files and will reset it as such + """ + # Note that this method assumes the dialect has already been registered if we're parsing the headers + self._skip_rows(fp, config_format.skip_rows_before_header) + if config_format.autogenerate_column_names: + headers = self._auto_generate_headers(fp, dialect_name) + else: + # Then read the header + reader = csv.reader(fp, dialect=dialect_name) # type: ignore + headers = list(next(reader)) + + fp.seek(0) + return headers + + def _auto_generate_headers(self, fp: IOBase, dialect_name: str) -> List[str]: + """ + Generates field names as [f0, f1, ...] in the same way as pyarrow's csv reader with autogenerate_column_names=True. + See https://arrow.apache.org/docs/python/generated/pyarrow.csv.ReadOptions.html + """ + reader = csv.reader(fp, dialect=dialect_name) # type: ignore + number_of_columns = len(next(reader)) # type: ignore + return [f"f{i}" for i in range(number_of_columns)] + + @staticmethod + def _skip_rows(fp: IOBase, rows_to_skip: int) -> None: + """ + Skip rows before the header. This has to be done on the file object itself, not the reader + """ + for _ in range(rows_to_skip): + fp.readline() + + +class CsvParser(FileTypeParser): + _MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE = 1_000_000 + + def __init__(self, csv_reader: Optional[_CsvReader] = None): + self._csv_reader = csv_reader if csv_reader else _CsvReader() + + async def infer_schema( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + ) -> SchemaType: + input_schema = config.get_input_schema() + if input_schema: + return input_schema + + # todo: the existing InMemoryFilesSource.open_file() test source doesn't currently require an encoding, but actual + # sources will likely require one. Rather than modify the interface now we can wait until the real use case + config_format = _extract_format(config) + type_inferrer_by_field: Dict[str, _TypeInferrer] = defaultdict( + lambda: _JsonTypeInferrer(config_format.true_values, config_format.false_values, config_format.null_values) + if config_format.inference_type != InferenceType.NONE + else _DisabledTypeInferrer() + ) + data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode) + read_bytes = 0 + for row in data_generator: + for header, value in row.items(): + type_inferrer_by_field[header].add_value(value) + # This is not accurate as a representation of how many bytes were read because csv does some processing on the actual value + # before returning. Given we would like to be more accurate, we could wrap the IO file using a decorator + read_bytes += len(value) + read_bytes += len(row) - 1 # for separators + if read_bytes >= self._MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE: + break + + schema = {header.strip(): {"type": type_inferred.infer()} for header, type_inferred in type_inferrer_by_field.items()} + data_generator.close() + return schema + + def parse_records( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + discovered_schema: Optional[Mapping[str, SchemaType]], + ) -> Iterable[Dict[str, Any]]: + config_format = _extract_format(config) + if discovered_schema: + property_types = {col: prop["type"] for col, prop in discovered_schema["properties"].items()} # type: ignore # discovered_schema["properties"] is known to be a mapping + deduped_property_types = CsvParser._pre_propcess_property_types(property_types) + else: + deduped_property_types = {} + cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless) + data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode) + for row in data_generator: + yield CsvParser._to_nullable(cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null) + data_generator.close() + + @property + def file_read_mode(self) -> FileReadMode: + return FileReadMode.READ + + @staticmethod + def _get_cast_function( + deduped_property_types: Mapping[str, str], config_format: CsvFormat, logger: logging.Logger, schemaless: bool + ) -> Callable[[Mapping[str, str]], Mapping[str, str]]: + # Only cast values if the schema is provided + if deduped_property_types and not schemaless: + return partial(CsvParser._cast_types, deduped_property_types=deduped_property_types, config_format=config_format, logger=logger) + else: + # If no schema is provided, yield the rows as they are + return _no_cast + + @staticmethod + def _to_nullable( + row: Mapping[str, str], deduped_property_types: Mapping[str, str], null_values: Set[str], strings_can_be_null: bool + ) -> Dict[str, Optional[str]]: + nullable = row | { + k: None if CsvParser._value_is_none(v, deduped_property_types.get(k), null_values, strings_can_be_null) else v + for k, v in row.items() + } + return nullable + + @staticmethod + def _value_is_none(value: Any, deduped_property_type: Optional[str], null_values: Set[str], strings_can_be_null: bool) -> bool: + return value in null_values and (strings_can_be_null or deduped_property_type != "string") + + @staticmethod + def _pre_propcess_property_types(property_types: Dict[str, Any]) -> Mapping[str, str]: + """ + Transform the property types to be non-nullable and remove duplicate types if any. + Sample input: + { + "col1": ["string", "null"], + "col2": ["string", "string", "null"], + "col3": "integer" + } + + Sample output: + { + "col1": "string", + "col2": "string", + "col3": "integer", + } + """ + output = {} + for prop, prop_type in property_types.items(): + if isinstance(prop_type, list): + prop_type_distinct = set(prop_type) + prop_type_distinct.remove("null") + if len(prop_type_distinct) != 1: + raise ValueError(f"Could not get non nullable type from {prop_type}") + output[prop] = next(iter(prop_type_distinct)) + else: + output[prop] = prop_type + return output + + @staticmethod + def _cast_types( + row: Dict[str, str], deduped_property_types: Dict[str, str], config_format: CsvFormat, logger: logging.Logger + ) -> Dict[str, Any]: + """ + Casts the values in the input 'row' dictionary according to the types defined in the JSON schema. + + Array and object types are only handled if they can be deserialized as JSON. + + If any errors are encountered, the value will be emitted as a string. + """ + warnings = [] + result = {} + + for key, value in row.items(): + prop_type = deduped_property_types.get(key) + cast_value: Any = value + + if prop_type in TYPE_PYTHON_MAPPING and prop_type is not None: + _, python_type = TYPE_PYTHON_MAPPING[prop_type] + + if python_type is None: + if value == "": + cast_value = None + else: + warnings.append(_format_warning(key, value, prop_type)) + + elif python_type == bool: + try: + cast_value = _value_to_bool(value, config_format.true_values, config_format.false_values) + except ValueError: + warnings.append(_format_warning(key, value, prop_type)) + + elif python_type == dict: + try: + # we don't re-use _value_to_object here because we type the column as object as long as there is only one object + cast_value = json.loads(value) + except json.JSONDecodeError: + warnings.append(_format_warning(key, value, prop_type)) + + elif python_type == list: + try: + cast_value = _value_to_list(value) + except (ValueError, json.JSONDecodeError): + warnings.append(_format_warning(key, value, prop_type)) + + elif python_type: + try: + cast_value = _value_to_python_type(value, python_type) + except ValueError: + warnings.append(_format_warning(key, value, prop_type)) + + result[key] = cast_value + + if warnings: + logger.warning( + f"{FileBasedSourceError.ERROR_CASTING_VALUE.value}: {','.join([w for w in warnings])}", + ) + return result + + +class _TypeInferrer(ABC): + @abstractmethod + def add_value(self, value: Any) -> None: + pass + + @abstractmethod + def infer(self) -> str: + pass + + +class _DisabledTypeInferrer(_TypeInferrer): + def add_value(self, value: Any) -> None: + pass + + def infer(self) -> str: + return "string" + + +class _JsonTypeInferrer(_TypeInferrer): + _NULL_TYPE = "null" + _BOOLEAN_TYPE = "boolean" + _INTEGER_TYPE = "integer" + _NUMBER_TYPE = "number" + _STRING_TYPE = "string" + + def __init__(self, boolean_trues: Set[str], boolean_falses: Set[str], null_values: Set[str]) -> None: + self._boolean_trues = boolean_trues + self._boolean_falses = boolean_falses + self._null_values = null_values + self._values: Set[str] = set() + + def add_value(self, value: Any) -> None: + self._values.add(value) + + def infer(self) -> str: + types_by_value = {value: self._infer_type(value) for value in self._values} + types_excluding_null_values = [types for types in types_by_value.values() if self._NULL_TYPE not in types] + if not types_excluding_null_values: + # this is highly unusual but we will consider the column as a string + return self._STRING_TYPE + + types = set.intersection(*types_excluding_null_values) + if self._BOOLEAN_TYPE in types: + return self._BOOLEAN_TYPE + elif self._INTEGER_TYPE in types: + return self._INTEGER_TYPE + elif self._NUMBER_TYPE in types: + return self._NUMBER_TYPE + return self._STRING_TYPE + + def _infer_type(self, value: str) -> Set[str]: + inferred_types = set() + + if value in self._null_values: + inferred_types.add(self._NULL_TYPE) + if self._is_boolean(value): + inferred_types.add(self._BOOLEAN_TYPE) + if self._is_integer(value): + inferred_types.add(self._INTEGER_TYPE) + inferred_types.add(self._NUMBER_TYPE) + elif self._is_number(value): + inferred_types.add(self._NUMBER_TYPE) + + inferred_types.add(self._STRING_TYPE) + return inferred_types + + def _is_boolean(self, value: str) -> bool: + try: + _value_to_bool(value, self._boolean_trues, self._boolean_falses) + return True + except ValueError: + return False + + @staticmethod + def _is_integer(value: str) -> bool: + try: + _value_to_python_type(value, int) + return True + except ValueError: + return False + + @staticmethod + def _is_number(value: str) -> bool: + try: + _value_to_python_type(value, float) + return True + except ValueError: + return False + + +def _value_to_bool(value: str, true_values: Set[str], false_values: Set[str]) -> bool: + if value in true_values: + return True + if value in false_values: + return False + raise ValueError(f"Value {value} is not a valid boolean value") + + +def _value_to_list(value: str) -> List[Any]: + parsed_value = json.loads(value) + if isinstance(parsed_value, list): + return parsed_value + raise ValueError(f"Value {parsed_value} is not a valid list value") + + +def _value_to_python_type(value: str, python_type: type) -> Any: + return python_type(value) + + +def _format_warning(key: str, value: str, expected_type: Optional[Any]) -> str: + return f"{key}: value={value},expected_type={expected_type}" + + +def _no_cast(row: Mapping[str, str]) -> Mapping[str, str]: + return row + + +def _extract_format(config: FileBasedStreamConfig) -> CsvFormat: + config_format = config.format or CsvFormat() + if not isinstance(config_format, CsvFormat): + raise ValueError(f"Invalid format config: {config_format}") + return config_format diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py new file mode 100644 index 000000000000..0e52fb5e04df --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, Mapping, Optional + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import SchemaType + +Record = Dict[str, Any] + + +class FileTypeParser(ABC): + """ + An abstract class containing methods that must be implemented for each + supported file type. + """ + + @abstractmethod + async def infer_schema( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + ) -> SchemaType: + """ + Infer the JSON Schema for this file. + """ + ... + + @abstractmethod + def parse_records( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + discovered_schema: Optional[Mapping[str, SchemaType]], + ) -> Iterable[Record]: + """ + Parse and emit each record. + """ + ... + + @property + @abstractmethod + def file_read_mode(self) -> FileReadMode: + """ + The mode in which the file should be opened for reading. + """ + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py new file mode 100644 index 000000000000..27efe8a005b4 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from typing import Any, Dict, Iterable, Mapping, Optional, Union + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import PYTHON_TYPE_MAPPING, SchemaType, merge_schemas + + +class JsonlParser(FileTypeParser): + + MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE = 1_000_000 + ENCODING = "utf8" + + async def infer_schema( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + ) -> SchemaType: + """ + Infers the schema for the file by inferring the schema for each line, and merging + it with the previously-inferred schema. + """ + inferred_schema: Mapping[str, Any] = {} + + for entry in self._parse_jsonl_entries(file, stream_reader, logger, read_limit=True): + line_schema = self._infer_schema_for_record(entry) + inferred_schema = merge_schemas(inferred_schema, line_schema) + + return inferred_schema + + def parse_records( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + discovered_schema: Optional[Mapping[str, SchemaType]], + ) -> Iterable[Dict[str, Any]]: + """ + This code supports parsing json objects over multiple lines even though this does not align with the JSONL format. This is for + backward compatibility reasons i.e. the previous source-s3 parser did support this. The drawback is: + * performance as the way we support json over multiple lines is very brute forced + * given that we don't have `newlines_in_values` config to scope the possible inputs, we might parse the whole file before knowing if + the input is improperly formatted or if the json is over multiple lines + + The goal is to run the V4 of source-s3 in production, track the warning log emitted when there are multiline json objects and + deprecate this feature if it's not a valid use case. + """ + yield from self._parse_jsonl_entries(file, stream_reader, logger) + + @classmethod + def _infer_schema_for_record(cls, record: Dict[str, Any]) -> Dict[str, Any]: + record_schema = {} + for key, value in record.items(): + if value is None: + record_schema[key] = {"type": "null"} + else: + record_schema[key] = {"type": PYTHON_TYPE_MAPPING[type(value)]} + + return record_schema + + @property + def file_read_mode(self) -> FileReadMode: + return FileReadMode.READ + + def _parse_jsonl_entries( + self, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + read_limit: bool = False, + ) -> Iterable[Dict[str, Any]]: + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + read_bytes = 0 + + had_json_parsing_error = False + has_warned_for_multiline_json_object = False + yielded_at_least_once = False + + accumulator = None + for line in fp: + if not accumulator: + accumulator = self._instantiate_accumulator(line) + read_bytes += len(line) + accumulator += line # type: ignore [operator] # In reality, it's either bytes or string and we add the same type + try: + record = json.loads(accumulator) + if had_json_parsing_error and not has_warned_for_multiline_json_object: + logger.warning(f"File at {file.uri} is using multiline JSON. Performance could be greatly reduced") + has_warned_for_multiline_json_object = True + + yield record + yielded_at_least_once = True + accumulator = self._instantiate_accumulator(line) + except json.JSONDecodeError: + had_json_parsing_error = True + + if read_limit and yielded_at_least_once and read_bytes >= self.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE: + logger.warning( + f"Exceeded the maximum number of bytes per file for schema inference ({self.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE}). " + f"Inferring schema from an incomplete set of records." + ) + break + + if had_json_parsing_error and not yielded_at_least_once: + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD) + + @staticmethod + def _instantiate_accumulator(line: Union[bytes, str]) -> Union[bytes, str]: + if isinstance(line, bytes): + return bytes("", json.detect_encoding(line)) + elif isinstance(line, str): + return "" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py new file mode 100644 index 000000000000..851608cfffd5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py @@ -0,0 +1,198 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +import os +from typing import Any, Dict, Iterable, List, Mapping, Optional +from urllib.parse import unquote + +import pyarrow as pa +import pyarrow.parquet as pq +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ParquetFormat +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import SchemaType +from pyarrow import Scalar + + +class ParquetParser(FileTypeParser): + + ENCODING = None + + async def infer_schema( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + ) -> SchemaType: + parquet_format = config.format or ParquetFormat() + if not isinstance(parquet_format, ParquetFormat): + raise ValueError(f"Expected ParquetFormat, got {parquet_format}") + + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + parquet_file = pq.ParquetFile(fp) + parquet_schema = parquet_file.schema_arrow + + # Inferred non-partition schema + schema = {field.name: ParquetParser.parquet_type_to_schema_type(field.type, parquet_format) for field in parquet_schema} + # Inferred partition schema + partition_columns = {partition.split("=")[0]: {"type": "string"} for partition in self._extract_partitions(file.uri)} + + schema.update(partition_columns) + return schema + + def parse_records( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + discovered_schema: Optional[Mapping[str, SchemaType]], + ) -> Iterable[Dict[str, Any]]: + parquet_format = config.format or ParquetFormat() + if not isinstance(parquet_format, ParquetFormat): + logger.info(f"Expected ParquetFormat, got {parquet_format}") + raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + reader = pq.ParquetFile(fp) + partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)} + for row_group in range(reader.num_row_groups): + batch = reader.read_row_group(row_group) + for row in range(batch.num_rows): + yield { + **{ + column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format) + for column in batch.column_names + }, + **partition_columns, + } + + @staticmethod + def _extract_partitions(filepath: str) -> List[str]: + return [unquote(partition) for partition in filepath.split(os.sep) if "=" in partition] + + @property + def file_read_mode(self) -> FileReadMode: + return FileReadMode.READ_BINARY + + @staticmethod + def _to_output_value(parquet_value: Scalar, parquet_format: ParquetFormat) -> Any: + """ + Convert a pyarrow scalar to a value that can be output by the source. + """ + # Convert date and datetime objects to isoformat strings + if pa.types.is_time(parquet_value.type) or pa.types.is_timestamp(parquet_value.type) or pa.types.is_date(parquet_value.type): + return parquet_value.as_py().isoformat() + + # Convert month_day_nano_interval to array + if parquet_value.type == pa.month_day_nano_interval(): + return json.loads(json.dumps(parquet_value.as_py())) + + # Decode binary strings to utf-8 + if ParquetParser._is_binary(parquet_value.type): + return parquet_value.as_py().decode("utf-8") + if pa.types.is_decimal(parquet_value.type): + if parquet_format.decimal_as_float: + return parquet_value.as_py() + else: + return str(parquet_value.as_py()) + + # Dictionaries are stored as two columns: indices and values + # The indices column is an array of integers that maps to the values column + if pa.types.is_dictionary(parquet_value.type): + return { + "indices": parquet_value.indices.tolist(), + "values": parquet_value.dictionary.tolist(), + } + if pa.types.is_map(parquet_value.type): + return {k: v for k, v in parquet_value.as_py()} + + if pa.types.is_null(parquet_value.type): + return None + + # Convert duration to seconds, then convert to the appropriate unit + if pa.types.is_duration(parquet_value.type): + duration = parquet_value.as_py() + duration_seconds = duration.total_seconds() + if parquet_value.type.unit == "s": + return duration_seconds + elif parquet_value.type.unit == "ms": + return duration_seconds * 1000 + elif parquet_value.type.unit == "us": + return duration_seconds * 1_000_000 + elif parquet_value.type.unit == "ns": + return duration_seconds * 1_000_000_000 + duration.nanoseconds + else: + raise ValueError(f"Unknown duration unit: {parquet_value.type.unit}") + else: + return parquet_value.as_py() + + @staticmethod + def parquet_type_to_schema_type(parquet_type: pa.DataType, parquet_format: ParquetFormat) -> Mapping[str, str]: + """ + Convert a pyarrow data type to an Airbyte schema type. + Parquet data types are defined at https://arrow.apache.org/docs/python/api/datatypes.html + """ + + if pa.types.is_timestamp(parquet_type): + return {"type": "string", "format": "date-time"} + elif pa.types.is_date(parquet_type): + return {"type": "string", "format": "date"} + elif ParquetParser._is_string(parquet_type, parquet_format): + return {"type": "string"} + elif pa.types.is_boolean(parquet_type): + return {"type": "boolean"} + elif ParquetParser._is_integer(parquet_type): + return {"type": "integer"} + elif ParquetParser._is_float(parquet_type, parquet_format): + return {"type": "number"} + elif ParquetParser._is_object(parquet_type): + return {"type": "object"} + elif ParquetParser._is_list(parquet_type): + return {"type": "array"} + elif pa.types.is_null(parquet_type): + return {"type": "null"} + else: + raise ValueError(f"Unsupported parquet type: {parquet_type}") + + @staticmethod + def _is_binary(parquet_type: pa.DataType) -> bool: + return bool( + pa.types.is_binary(parquet_type) or pa.types.is_large_binary(parquet_type) or pa.types.is_fixed_size_binary(parquet_type) + ) + + @staticmethod + def _is_integer(parquet_type: pa.DataType) -> bool: + return bool(pa.types.is_integer(parquet_type) or pa.types.is_duration(parquet_type)) + + @staticmethod + def _is_float(parquet_type: pa.DataType, parquet_format: ParquetFormat) -> bool: + if pa.types.is_decimal(parquet_type): + return parquet_format.decimal_as_float + else: + return bool(pa.types.is_floating(parquet_type)) + + @staticmethod + def _is_string(parquet_type: pa.DataType, parquet_format: ParquetFormat) -> bool: + if pa.types.is_decimal(parquet_type): + return not parquet_format.decimal_as_float + else: + return bool( + pa.types.is_time(parquet_type) + or pa.types.is_string(parquet_type) + or pa.types.is_large_string(parquet_type) + or ParquetParser._is_binary(parquet_type) # Best we can do is return as a string since we do not support binary + ) + + @staticmethod + def _is_object(parquet_type: pa.DataType) -> bool: + return bool(pa.types.is_dictionary(parquet_type) or pa.types.is_struct(parquet_type) or pa.types.is_map(parquet_type)) + + @staticmethod + def _is_list(parquet_type: pa.DataType) -> bool: + return bool(pa.types.is_list(parquet_type) or pa.types.is_large_list(parquet_type) or parquet_type == pa.month_day_nano_interval()) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py new file mode 100644 index 000000000000..c78065f8d2d3 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + + +class RemoteFile(BaseModel): + """ + A file in a file-based stream. + """ + + uri: str + last_modified: datetime + + def extension_agrees_with_file_type(self, file_type: Optional[str]) -> bool: + extensions = self.uri.split(".")[1:] + if not extensions: + return True + if not file_type: + return True + return any(file_type.casefold() in e.casefold() for e in extensions) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_helpers.py new file mode 100644 index 000000000000..3f7b2151653f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_helpers.py @@ -0,0 +1,245 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from copy import deepcopy +from enum import Enum +from functools import total_ordering +from typing import Any, Dict, List, Literal, Mapping, Optional, Tuple, Type, Union + +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, SchemaInferenceError + +JsonSchemaSupportedType = Union[List[str], Literal["string"], str] +SchemaType = Mapping[str, Mapping[str, JsonSchemaSupportedType]] + +schemaless_schema = {"type": "object", "properties": {"data": {"type": "object"}}} + + +@total_ordering +class ComparableType(Enum): + NULL = 0 + BOOLEAN = 1 + INTEGER = 2 + NUMBER = 3 + STRING = 4 + OBJECT = 5 + + def __lt__(self, other: Any) -> bool: + if self.__class__ is other.__class__: + return self.value < other.value # type: ignore + else: + return NotImplemented + + +TYPE_PYTHON_MAPPING: Mapping[str, Tuple[str, Optional[Type[Any]]]] = { + "null": ("null", None), + "array": ("array", list), + "boolean": ("boolean", bool), + "float": ("number", float), + "integer": ("integer", int), + "number": ("number", float), + "object": ("object", dict), + "string": ("string", str), +} +PYTHON_TYPE_MAPPING = {t: k for k, (_, t) in TYPE_PYTHON_MAPPING.items()} + + +def get_comparable_type(value: Any) -> Optional[ComparableType]: + if value == "null": + return ComparableType.NULL + if value == "boolean": + return ComparableType.BOOLEAN + if value == "integer": + return ComparableType.INTEGER + if value == "number": + return ComparableType.NUMBER + if value == "string": + return ComparableType.STRING + if value == "object": + return ComparableType.OBJECT + else: + return None + + +def get_inferred_type(value: Any) -> Optional[ComparableType]: + if value is None: + return ComparableType.NULL + if isinstance(value, bool): + return ComparableType.BOOLEAN + if isinstance(value, int): + return ComparableType.INTEGER + if isinstance(value, float): + return ComparableType.NUMBER + if isinstance(value, str): + return ComparableType.STRING + if isinstance(value, dict): + return ComparableType.OBJECT + else: + return None + + +def merge_schemas(schema1: SchemaType, schema2: SchemaType) -> SchemaType: + """ + Returns a new dictionary that contains schema1 and schema2. + + Schemas are merged as follows + - If a key is in one schema but not the other, add it to the base schema with its existing type. + - If a key is in both schemas but with different types, use the wider type. + - If the type is a list in one schema but a different type of element in the other schema, raise an exception. + - If the type is an object in both schemas but the objects are different raise an exception. + - If the type is an object in one schema but not in the other schema, raise an exception. + + In other words, we support merging + - any atomic type with any other atomic type (choose the wider of the two) + - list with list (union) + and nothing else. + """ + for k, t in list(schema1.items()) + list(schema2.items()): + if not isinstance(t, dict) or "type" not in t or not _is_valid_type(t["type"]): + raise SchemaInferenceError(FileBasedSourceError.UNRECOGNIZED_TYPE, key=k, type=t) + + merged_schema: Dict[str, Any] = deepcopy(schema1) # type: ignore # as of 2023-08-08, deepcopy can copy Mapping + for k2, t2 in schema2.items(): + t1 = merged_schema.get(k2) + if t1 is None: + merged_schema[k2] = t2 + elif t1 == t2: + continue + else: + merged_schema[k2] = _choose_wider_type(k2, t1, t2) + + return merged_schema + + +def _is_valid_type(t: JsonSchemaSupportedType) -> bool: + return t == "array" or get_comparable_type(t) is not None + + +def _choose_wider_type(key: str, t1: Mapping[str, Any], t2: Mapping[str, Any]) -> Mapping[str, Any]: + if (t1["type"] == "array" or t2["type"] == "array") and t1 != t2: + raise SchemaInferenceError( + FileBasedSourceError.SCHEMA_INFERENCE_ERROR, + details="Cannot merge schema for unequal array types.", + key=key, + detected_types=f"{t1},{t2}", + ) + elif (t1["type"] == "object" or t2["type"] == "object") and t1 != t2: + raise SchemaInferenceError( + FileBasedSourceError.SCHEMA_INFERENCE_ERROR, + details="Cannot merge schema for unequal object types.", + key=key, + detected_types=f"{t1},{t2}", + ) + else: + comparable_t1 = get_comparable_type(TYPE_PYTHON_MAPPING[t1["type"]][0]) # accessing the type_mapping value + comparable_t2 = get_comparable_type(TYPE_PYTHON_MAPPING[t2["type"]][0]) # accessing the type_mapping value + if not comparable_t1 and comparable_t2: + raise SchemaInferenceError(FileBasedSourceError.UNRECOGNIZED_TYPE, key=key, detected_types=f"{t1},{t2}") + return max( + [t1, t2], key=lambda x: ComparableType(get_comparable_type(TYPE_PYTHON_MAPPING[x["type"]][0])) + ) # accessing the type_mapping value + + +def is_equal_or_narrower_type(value: Any, expected_type: str) -> bool: + if isinstance(value, list): + # We do not compare lists directly; the individual items are compared. + # If we hit this condition, it means that the expected type is not + # compatible with the inferred type. + return False + + inferred_type = ComparableType(get_inferred_type(value)) + + if inferred_type is None: + return False + + return ComparableType(inferred_type) <= ComparableType(get_comparable_type(expected_type)) + + +def conforms_to_schema(record: Mapping[str, Any], schema: Mapping[str, Any]) -> bool: + """ + Return true iff the record conforms to the supplied schema. + + The record conforms to the supplied schema iff: + - All columns in the record are in the schema. + - For every column in the record, that column's type is equal to or narrower than the same column's + type in the schema. + """ + schema_columns = set(schema.get("properties", {}).keys()) + record_columns = set(record.keys()) + + if not record_columns.issubset(schema_columns): + return False + + for column, definition in schema.get("properties", {}).items(): + expected_type = definition.get("type") + value = record.get(column) + + if value is not None: + if isinstance(expected_type, list): + return any(is_equal_or_narrower_type(value, e) for e in expected_type) + elif expected_type == "object": + return isinstance(value, dict) + elif expected_type == "array": + if not isinstance(value, list): + return False + array_type = definition.get("items", {}).get("type") + if not all(is_equal_or_narrower_type(v, array_type) for v in value): + return False + elif not is_equal_or_narrower_type(value, expected_type): + return False + + return True + + +def _parse_json_input(input_schema: Union[str, Mapping[str, str]]) -> Optional[Mapping[str, str]]: + try: + if isinstance(input_schema, str): + schema: Mapping[str, str] = json.loads(input_schema) + else: + schema = input_schema + if not all(isinstance(s, str) for s in schema.values()): + raise ConfigValidationError( + FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, details="Invalid input schema; nested schemas are not supported." + ) + + except json.decoder.JSONDecodeError: + return None + + return schema + + +def type_mapping_to_jsonschema(input_schema: Optional[Union[str, Mapping[str, str]]]) -> Optional[Mapping[str, Any]]: + """ + Return the user input schema (type mapping), transformed to JSON Schema format. + + Verify that the input schema: + - is a key:value map + - all values in the map correspond to a JsonSchema datatype + """ + if not input_schema: + return None + + result_schema = {} + + json_mapping = _parse_json_input(input_schema) or {} + + for col_name, type_name in json_mapping.items(): + col_name, type_name = col_name.strip(), type_name.strip() + if not (col_name and type_name): + raise ConfigValidationError( + FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, + details=f"Invalid input schema; expected mapping in the format column_name: type, got {input_schema}.", + ) + + _json_schema_type = TYPE_PYTHON_MAPPING.get(type_name.casefold()) + + if not _json_schema_type: + raise ConfigValidationError( + FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA, details=f"Invalid type '{type_name}' for property '{col_name}'." + ) + + json_schema_type = _json_schema_type[0] + result_schema[col_name] = {"type": json_schema_type} + + return {"type": "object", "properties": result_schema} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py new file mode 100644 index 000000000000..d2cc0e63b214 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/__init__.py @@ -0,0 +1,15 @@ +from airbyte_cdk.sources.file_based.schema_validation_policies.abstract_schema_validation_policy import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.schema_validation_policies.default_schema_validation_policies import ( + DEFAULT_SCHEMA_VALIDATION_POLICIES, + EmitRecordPolicy, + SkipRecordPolicy, + WaitForDiscoverPolicy, +) + +__all__ = [ + "DEFAULT_SCHEMA_VALIDATION_POLICIES", + "AbstractSchemaValidationPolicy", + "EmitRecordPolicy", + "SkipRecordPolicy", + "WaitForDiscoverPolicy", +] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py new file mode 100644 index 000000000000..004139b78b10 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/abstract_schema_validation_policy.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, Mapping, Optional + + +class AbstractSchemaValidationPolicy(ABC): + name: str + validate_schema_before_sync = False # Whether to verify that records conform to the schema during the stream's availabilty check + + @abstractmethod + def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool: + """ + Return True if the record passes the user's validation policy. + """ + raise NotImplementedError() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py new file mode 100644 index 000000000000..02134d1b839f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_validation_policies/default_schema_validation_policies.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, Optional + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import ValidationPolicy +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, StopSyncPerValidationPolicy +from airbyte_cdk.sources.file_based.schema_helpers import conforms_to_schema +from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy + + +class EmitRecordPolicy(AbstractSchemaValidationPolicy): + name = "emit_record" + + def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool: + return True + + +class SkipRecordPolicy(AbstractSchemaValidationPolicy): + name = "skip_record" + + def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool: + return schema is not None and conforms_to_schema(record, schema) + + +class WaitForDiscoverPolicy(AbstractSchemaValidationPolicy): + name = "wait_for_discover" + validate_schema_before_sync = True + + def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool: + if schema is None or not conforms_to_schema(record, schema): + raise StopSyncPerValidationPolicy(FileBasedSourceError.STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY) + return True + + +DEFAULT_SCHEMA_VALIDATION_POLICIES = { + ValidationPolicy.emit_record: EmitRecordPolicy(), + ValidationPolicy.skip_record: SkipRecordPolicy(), + ValidationPolicy.wait_for_discover: WaitForDiscoverPolicy(), +} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/__init__.py new file mode 100644 index 000000000000..4b5c4bc2edd5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/__init__.py @@ -0,0 +1,4 @@ +from airbyte_cdk.sources.file_based.stream.abstract_file_based_stream import AbstractFileBasedStream +from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream + +__all__ = ["AbstractFileBasedStream", "DefaultFileBasedStream"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py new file mode 100644 index 000000000000..7499316ea2d7 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py @@ -0,0 +1,144 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import abstractmethod +from functools import cached_property, lru_cache +from typing import Any, Dict, Iterable, List, Mapping, Optional + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, PrimaryKeyType +from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError, UndefinedParserError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.types import StreamSlice +from airbyte_cdk.sources.streams import Stream + + +class AbstractFileBasedStream(Stream): + """ + A file-based stream in an Airbyte source. + + In addition to the base Stream attributes, a file-based stream has + - A config object (derived from the corresponding stream section in source config). + This contains the globs defining the stream's files. + - A StreamReader, which knows how to list and open files in the stream. + - A FileBasedAvailabilityStrategy, which knows how to verify that we can list and open + files in the stream. + - A DiscoveryPolicy that controls the number of concurrent requests sent to the source + during discover, and the number of files used for schema discovery. + - A dictionary of FileType:Parser that holds all of the file types that can be handled + by the stream. + """ + + def __init__( + self, + config: FileBasedStreamConfig, + catalog_schema: Optional[Mapping[str, Any]], + stream_reader: AbstractFileBasedStreamReader, + availability_strategy: AbstractFileBasedAvailabilityStrategy, + discovery_policy: AbstractDiscoveryPolicy, + parsers: Dict[str, FileTypeParser], + validation_policy: AbstractSchemaValidationPolicy, + ): + super().__init__() + self.config = config + self.catalog_schema = catalog_schema + self.validation_policy = validation_policy + self._stream_reader = stream_reader + self._discovery_policy = discovery_policy + self._availability_strategy = availability_strategy + self._parsers = parsers + + @property + @abstractmethod + def primary_key(self) -> PrimaryKeyType: + ... + + @abstractmethod + def list_files(self) -> List[RemoteFile]: + """ + List all files that belong to the stream. + """ + ... + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[StreamSlice] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[Mapping[str, Any]]: + """ + Yield all records from all remote files in `list_files_for_this_sync`. + This method acts as an adapter between the generic Stream interface and the file-based's + stream since file-based streams manage their own states. + """ + if stream_slice is None: + raise ValueError("stream_slice must be set") + return self.read_records_from_slice(stream_slice) + + @abstractmethod + def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Mapping[str, Any]]: + """ + Yield all records from all remote files in `list_files_for_this_sync`. + """ + ... + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + """ + This method acts as an adapter between the generic Stream interface and the file-based's + stream since file-based streams manage their own states. + """ + return self.compute_slices() + + @abstractmethod + def compute_slices(self) -> Iterable[Optional[StreamSlice]]: + """ + Return a list of slices that will be used to read files in the current sync. + :return: The slices to use for the current sync. + """ + ... + + @abstractmethod + @lru_cache(maxsize=None) + def get_json_schema(self) -> Mapping[str, Any]: + """ + Return the JSON Schema for a stream. + """ + ... + + @abstractmethod + def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: + """ + Infer the schema for files in the stream. + """ + ... + + def get_parser(self, file_type: str) -> FileTypeParser: + try: + return self._parsers[file_type] + except KeyError: + raise UndefinedParserError(FileBasedSourceError.UNDEFINED_PARSER, stream=self.name, file_type=file_type) + + def record_passes_validation_policy(self, record: Mapping[str, Any]) -> bool: + if self.validation_policy: + return self.validation_policy.record_passes_validation_policy(record=record, schema=self.catalog_schema) + else: + raise RecordParseError( + FileBasedSourceError.UNDEFINED_VALIDATION_POLICY, stream=self.name, validation_policy=self.config.validation_policy + ) + + @cached_property + def availability_strategy(self) -> AbstractFileBasedAvailabilityStrategy: + return self._availability_strategy + + @property + def name(self) -> str: + return self.config.name diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/__init__.py new file mode 100644 index 000000000000..c1bf15a5d01f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/__init__.py @@ -0,0 +1,4 @@ +from .abstract_file_based_cursor import AbstractFileBasedCursor +from .default_file_based_cursor import DefaultFileBasedCursor + +__all__ = ["AbstractFileBasedCursor", "DefaultFileBasedCursor"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py new file mode 100644 index 000000000000..f38a5364135c --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/abstract_file_based_cursor.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, Iterable, MutableMapping + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.types import StreamState + + +class AbstractFileBasedCursor(ABC): + """ + Abstract base class for cursors used by file-based streams. + """ + + @abstractmethod + def __init__(self, stream_config: FileBasedStreamConfig, **kwargs: Any): + """ + Common interface for all cursors. + """ + ... + + @abstractmethod + def add_file(self, file: RemoteFile) -> None: + """ + Add a file to the cursor. This method is called when a file is processed by the stream. + :param file: The file to add + """ + ... + + @abstractmethod + def set_initial_state(self, value: StreamState) -> None: + """ + Set the initial state of the cursor. The cursor cannot be initialized at construction time because the stream doesn't know its state yet. + :param value: The stream state + """ + + @abstractmethod + def get_state(self) -> MutableMapping[str, Any]: + """ + Get the state of the cursor. + """ + ... + + @abstractmethod + def get_start_time(self) -> datetime: + """ + Returns the start time of the current sync. + """ + ... + + @abstractmethod + def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Given the list of files in the source, return the files that should be synced. + :param all_files: All files in the source + :param logger: + :return: The files that should be synced + """ + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py new file mode 100644 index 000000000000..58d64acbf63d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/cursor/default_file_based_cursor.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime, timedelta +from typing import Any, Iterable, MutableMapping, Optional + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.cursor.abstract_file_based_cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamState + + +class DefaultFileBasedCursor(AbstractFileBasedCursor): + DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL = 3 + DEFAULT_MAX_HISTORY_SIZE = 10_000 + DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + CURSOR_FIELD = "_ab_source_file_last_modified" + + def __init__(self, stream_config: FileBasedStreamConfig, **_: Any): + super().__init__(stream_config) + self._file_to_datetime_history: MutableMapping[str, str] = {} + self._time_window_if_history_is_full = timedelta( + days=stream_config.days_to_sync_if_history_is_full or self.DEFAULT_DAYS_TO_SYNC_IF_HISTORY_IS_FULL + ) + + if self._time_window_if_history_is_full <= timedelta(): + raise ValueError(f"days_to_sync_if_history_is_full must be a positive timedelta, got {self._time_window_if_history_is_full}") + + self._start_time = self._compute_start_time() + self._initial_earliest_file_in_history: Optional[RemoteFile] = None + + def set_initial_state(self, value: StreamState) -> None: + self._file_to_datetime_history = value.get("history", {}) + self._start_time = self._compute_start_time() + self._initial_earliest_file_in_history = self._compute_earliest_file_in_history() + + def add_file(self, file: RemoteFile) -> None: + self._file_to_datetime_history[file.uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT) + if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE: + # Get the earliest file based on its last modified date and its uri + oldest_file = self._compute_earliest_file_in_history() + if oldest_file: + del self._file_to_datetime_history[oldest_file.uri] + else: + raise Exception( + "The history is full but there is no files in the history. This should never happen and might be indicative of a bug in the CDK." + ) + + def get_state(self) -> StreamState: + state = {"history": self._file_to_datetime_history, self.CURSOR_FIELD: self._get_cursor()} + return state + + def _get_cursor(self) -> Optional[str]: + """ + Returns the cursor value. + + Files are synced in order of last-modified with secondary sort on filename, so the cursor value is + a string joining the last-modified timestamp of the last synced file and the name of the file. + """ + if self._file_to_datetime_history.items(): + filename, timestamp = max(self._file_to_datetime_history.items(), key=lambda x: (x[1], x[0])) + return f"{timestamp}_{filename}" + return None + + def _is_history_full(self) -> bool: + """ + Returns true if the state's history is full, meaning new entries will start to replace old entries. + """ + return len(self._file_to_datetime_history) >= self.DEFAULT_MAX_HISTORY_SIZE + + def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool: + if file.uri in self._file_to_datetime_history: + # If the file's uri is in the history, we should sync the file if it has been modified since it was synced + updated_at_from_history = datetime.strptime(self._file_to_datetime_history[file.uri], self.DATE_TIME_FORMAT) + if file.last_modified < updated_at_from_history: + logger.warning( + f"The file {file.uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file." + ) + else: + return file.last_modified > updated_at_from_history + return file.last_modified > updated_at_from_history + if self._is_history_full(): + if self._initial_earliest_file_in_history is None: + return True + if file.last_modified > self._initial_earliest_file_in_history.last_modified: + # If the history is partial and the file's datetime is strictly greater than the earliest file in the history, + # we should sync it + return True + elif file.last_modified == self._initial_earliest_file_in_history.last_modified: + # If the history is partial and the file's datetime is equal to the earliest file in the history, + # we should sync it if its uri is strictly greater than the earliest file in the history + return file.uri > self._initial_earliest_file_in_history.uri + else: + # Otherwise, only sync the file if it has been modified since the start of the time window + return file.last_modified >= self.get_start_time() + else: + # The file is not in the history and the history is complete. We know we need to sync the file + return True + + def get_files_to_sync(self, all_files: Iterable[RemoteFile], logger: logging.Logger) -> Iterable[RemoteFile]: + if self._is_history_full(): + logger.warning( + f"The state history is full. " + f"This sync and future syncs won't be able to use the history to filter out duplicate files. " + f"It will instead use the time window of {self._time_window_if_history_is_full} to filter out files." + ) + for f in all_files: + if self._should_sync_file(f, logger): + yield f + + def get_start_time(self) -> datetime: + return self._start_time + + def _compute_earliest_file_in_history(self) -> Optional[RemoteFile]: + if self._file_to_datetime_history: + filename, last_modified = min(self._file_to_datetime_history.items(), key=lambda f: (f[1], f[0])) + return RemoteFile(uri=filename, last_modified=datetime.strptime(last_modified, self.DATE_TIME_FORMAT)) + else: + return None + + def _compute_start_time(self) -> datetime: + if not self._file_to_datetime_history: + return datetime.min + else: + earliest = min(self._file_to_datetime_history.values()) + earliest_dt = datetime.strptime(earliest, self.DATE_TIME_FORMAT) + if self._is_history_full(): + time_window = datetime.now() - self._time_window_if_history_is_full + earliest_dt = min(earliest_dt, time_window) + return earliest_dt diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py new file mode 100644 index 000000000000..087ea525b3af --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py @@ -0,0 +1,262 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncio +import itertools +import traceback +from functools import cache +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Union + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType +from airbyte_cdk.sources.file_based.exceptions import ( + FileBasedSourceError, + InvalidSchemaError, + MissingSchemaError, + RecordParseError, + SchemaInferenceError, + StopSyncPerValidationPolicy, +) +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import SchemaType, merge_schemas, schemaless_schema +from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamSlice +from airbyte_cdk.sources.streams import IncrementalMixin +from airbyte_cdk.sources.streams.core import JsonSchema +from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message + + +class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin): + + """ + The default file-based stream. + """ + + DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + ab_last_mod_col = "_ab_source_file_last_modified" + ab_file_name_col = "_ab_source_file_url" + airbyte_columns = [ab_last_mod_col, ab_file_name_col] + + def __init__(self, cursor: AbstractFileBasedCursor, **kwargs: Any): + super().__init__(**kwargs) + self._cursor = cursor + + @property + def state(self) -> MutableMapping[str, Any]: + return self._cursor.get_state() + + @state.setter + def state(self, value: MutableMapping[str, Any]) -> None: + """State setter, accept state serialized by state getter.""" + self._cursor.set_initial_state(value) + + @property + def primary_key(self) -> PrimaryKeyType: + return self.config.primary_key + + def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]: + # Sort files by last_modified, uri and return them grouped by last_modified + all_files = self.list_files() + files_to_read = self._cursor.get_files_to_sync(all_files, self.logger) + sorted_files_to_read = sorted(files_to_read, key=lambda f: (f.last_modified, f.uri)) + slices = [{"files": list(group[1])} for group in itertools.groupby(sorted_files_to_read, lambda f: f.last_modified)] + return slices + + def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[AirbyteMessage]: + """ + Yield all records from all remote files in `list_files_for_this_sync`. + + If an error is encountered reading records from a file, log a message and do not attempt + to sync the rest of the file. + """ + schema = self.catalog_schema + if schema is None: + # On read requests we should always have the catalog available + raise MissingSchemaError(FileBasedSourceError.MISSING_SCHEMA, stream=self.name) + # The stream only supports a single file type, so we can use the same parser for all files + parser = self.get_parser(self.config.file_type) + for file in stream_slice["files"]: + # only serialize the datetime once + file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT) + n_skipped = line_no = 0 + + try: + for record in parser.parse_records(self.config, file, self._stream_reader, self.logger, schema): + line_no += 1 + if self.config.schemaless: + record = {"data": record} + elif not self.record_passes_validation_policy(record): + n_skipped += 1 + continue + record[self.ab_last_mod_col] = file_datetime_string + record[self.ab_file_name_col] = file.uri + yield stream_data_to_airbyte_message(self.name, record) + self._cursor.add_file(file) + + except StopSyncPerValidationPolicy: + yield AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.WARN, + message=f"Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream={self.name} file={file.uri} validation_policy={self.config.validation_policy.value} n_skipped={n_skipped}", + ), + ) + break + + except RecordParseError: + # Increment line_no because the exception was raised before we could increment it + line_no += 1 + yield AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.ERROR, + message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name} file={file.uri} line_no={line_no} n_skipped={n_skipped}", + stack_trace=traceback.format_exc(), + ), + ) + + except Exception: + yield AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.ERROR, + message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name} file={file.uri} line_no={line_no} n_skipped={n_skipped}", + stack_trace=traceback.format_exc(), + ), + ) + + finally: + if n_skipped: + yield AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.WARN, + message=f"Records in file did not pass validation policy. stream={self.name} file={file.uri} n_skipped={n_skipped} validation_policy={self.validation_policy.name}", + ), + ) + + @property + def cursor_field(self) -> Union[str, List[str]]: + """ + Override to return the default cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. + :return: The name of the field used as a cursor. If the cursor is nested, return an array consisting of the path to the cursor. + """ + return self.ab_last_mod_col + + @cache + def get_json_schema(self) -> JsonSchema: + extra_fields = { + self.ab_last_mod_col: {"type": "string"}, + self.ab_file_name_col: {"type": "string"}, + } + try: + schema = self._get_raw_json_schema() + except Exception as exc: + raise SchemaInferenceError(FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name) from exc + else: + return {"type": "object", "properties": {**extra_fields, **schema["properties"]}} + + def _get_raw_json_schema(self) -> JsonSchema: + if self.config.input_schema: + return self.config.get_input_schema() # type: ignore + elif self.config.schemaless: + return schemaless_schema + else: + files = self.list_files() + total_n_files = len(files) + + if total_n_files == 0: + raise SchemaInferenceError(FileBasedSourceError.EMPTY_STREAM, stream=self.name) + + max_n_files_for_schema_inference = self._discovery_policy.max_n_files_for_schema_inference + if total_n_files > max_n_files_for_schema_inference: + # Use the most recent files for schema inference, so we pick up schema changes during discovery. + files = sorted(files, key=lambda x: x.last_modified, reverse=True)[:max_n_files_for_schema_inference] + self.logger.warn( + msg=f"Refusing to infer schema for all {total_n_files} files; using {max_n_files_for_schema_inference} files." + ) + + inferred_schema = self.infer_schema(files) + + if not inferred_schema: + raise InvalidSchemaError( + FileBasedSourceError.INVALID_SCHEMA_ERROR, + details=f"Empty schema. Please check that the files are valid {self.config.file_type}", + stream=self.name, + ) + + schema = {"type": "object", "properties": inferred_schema} + + return schema + + @cache + def list_files(self) -> List[RemoteFile]: + """ + List all files that belong to the stream as defined by the stream's globs. + The output of this method is cached so we don't need to list the files more than once. + This means we won't pick up changes to the files during a sync. + """ + return list(self._stream_reader.get_matching_files(self.config.globs or [], self.config.legacy_prefix, self.logger)) + + def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: + loop = asyncio.get_event_loop() + schema = loop.run_until_complete(self._infer_schema(files)) + return self._fill_nulls(schema) + + @staticmethod + def _fill_nulls(schema: Mapping[str, Any]) -> Mapping[str, Any]: + if isinstance(schema, dict): + for k, v in schema.items(): + if k == "type": + if isinstance(v, list): + if "null" not in v: + schema[k] = ["null"] + v + elif v != "null": + schema[k] = ["null", v] + else: + DefaultFileBasedStream._fill_nulls(v) + elif isinstance(schema, list): + for item in schema: + DefaultFileBasedStream._fill_nulls(item) + return schema + + async def _infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: + """ + Infer the schema for a stream. + + Each file type has a corresponding `infer_schema` handler. + Dispatch on file type. + """ + base_schema: SchemaType = {} + pending_tasks: Set[asyncio.tasks.Task[SchemaType]] = set() + + n_started, n_files = 0, len(files) + files_iterator = iter(files) + while pending_tasks or n_started < n_files: + while len(pending_tasks) <= self._discovery_policy.n_concurrent_requests and (file := next(files_iterator, None)): + pending_tasks.add(asyncio.create_task(self._infer_file_schema(file))) + n_started += 1 + # Return when the first task is completed so that we can enqueue a new task as soon as the + # number of concurrent tasks drops below the number allowed. + done, pending_tasks = await asyncio.wait(pending_tasks, return_when=asyncio.FIRST_COMPLETED) + for task in done: + try: + base_schema = merge_schemas(base_schema, task.result()) + except Exception as exc: + self.logger.error(f"An error occurred inferring the schema. \n {traceback.format_exc()}", exc_info=exc) + + return base_schema + + async def _infer_file_schema(self, file: RemoteFile) -> SchemaType: + try: + return await self.get_parser(self.config.file_type).infer_schema(self.config, file, self._stream_reader, self.logger) + except Exception as exc: + raise SchemaInferenceError( + FileBasedSourceError.SCHEMA_INFERENCE_ERROR, + file=file.uri, + stream_file_type=self.config.file_type, + stream=self.name, + ) from exc diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/types.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/types.py new file mode 100644 index 000000000000..b83bf37a37a7 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/types.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +from typing import Any, Mapping, MutableMapping + +StreamSlice = Mapping[str, Any] +StreamState = MutableMapping[str, Any] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/http_logger.py b/airbyte-cdk/python/airbyte_cdk/sources/http_logger.py new file mode 100644 index 000000000000..7158c8003e5b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/http_logger.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Optional, Union + +import requests +from airbyte_cdk.sources.message import LogMessage + + +def format_http_message( + response: requests.Response, title: str, description: str, stream_name: Optional[str], is_auxiliary: bool = None +) -> LogMessage: + request = response.request + log_message = { + "http": { + "title": title, + "description": description, + "request": { + "method": request.method, + "body": { + "content": _normalize_body_string(request.body), + }, + "headers": dict(request.headers), + }, + "response": { + "body": { + "content": response.text, + }, + "headers": dict(response.headers), + "status_code": response.status_code, + }, + }, + "log": { + "level": "debug", + }, + "url": {"full": request.url}, + } + if is_auxiliary is not None: + log_message["http"]["is_auxiliary"] = is_auxiliary + if stream_name: + log_message["airbyte_cdk"] = {"stream": {"name": stream_name}} + return log_message + + +def _normalize_body_string(body_str: Optional[Union[str, bytes]]) -> Optional[str]: + return body_str.decode() if isinstance(body_str, (bytes, bytearray)) else body_str diff --git a/airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py new file mode 100644 index 000000000000..c545c0d736ab --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/message/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from .repository import ( + InMemoryMessageRepository, + LogAppenderMessageRepositoryDecorator, + LogMessage, + MessageRepository, + NoopMessageRepository, +) + +__all__ = ["InMemoryMessageRepository", "LogAppenderMessageRepositoryDecorator", "LogMessage", "MessageRepository", "NoopMessageRepository"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py b/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py new file mode 100644 index 000000000000..124d8ec416c0 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from abc import ABC, abstractmethod +from collections import deque +from typing import Callable, Deque, Iterable, List, Optional + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, Type +from airbyte_cdk.sources.utils.types import JsonType +from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets + +_LOGGER = logging.getLogger("MessageRepository") +_SUPPORTED_MESSAGE_TYPES = {Type.CONTROL, Type.LOG} +LogMessage = dict[str, JsonType] + +_SEVERITY_BY_LOG_LEVEL = { + Level.FATAL: 1, + Level.ERROR: 2, + Level.WARN: 3, + Level.INFO: 4, + Level.DEBUG: 5, + Level.TRACE: 5, +} + + +def _is_severe_enough(threshold: Level, level: Level) -> bool: + if threshold not in _SEVERITY_BY_LOG_LEVEL: + _LOGGER.warning(f"Log level {threshold} for threshold is not supported. This is probably a CDK bug. Please contact Airbyte.") + return True + + if level not in _SEVERITY_BY_LOG_LEVEL: + _LOGGER.warning( + f"Log level {level} is not supported. This is probably a source bug. Please contact the owner of the source or Airbyte." + ) + return True + + return _SEVERITY_BY_LOG_LEVEL[threshold] >= _SEVERITY_BY_LOG_LEVEL[level] + + +class MessageRepository(ABC): + @abstractmethod + def emit_message(self, message: AirbyteMessage) -> None: + raise NotImplementedError() + + @abstractmethod + def log_message(self, level: Level, message_provider: Callable[[], LogMessage]) -> None: + """ + Computing messages can be resource consuming. This method is specialized for logging because we want to allow for lazy evaluation if + the log level is less severe than what is configured + """ + raise NotImplementedError() + + @abstractmethod + def consume_queue(self) -> Iterable[AirbyteMessage]: + raise NotImplementedError() + + +class NoopMessageRepository(MessageRepository): + def emit_message(self, message: AirbyteMessage) -> None: + pass + + def log_message(self, level: Level, message_provider: Callable[[], LogMessage]) -> None: + pass + + def consume_queue(self) -> Iterable[AirbyteMessage]: + return [] + + +class InMemoryMessageRepository(MessageRepository): + def __init__(self, log_level: Level = Level.INFO) -> None: + self._message_queue: Deque[AirbyteMessage] = deque() + self._log_level = log_level + + def emit_message(self, message: AirbyteMessage) -> None: + """ + :param message: As of today, only AirbyteControlMessages are supported given that supporting other types of message will need more + work and therefore this work has been postponed + """ + if message.type not in _SUPPORTED_MESSAGE_TYPES: + raise ValueError(f"As of today, only {_SUPPORTED_MESSAGE_TYPES} are supported as part of the InMemoryMessageRepository") + self._message_queue.append(message) + + def log_message(self, level: Level, message_provider: Callable[[], LogMessage]) -> None: + if _is_severe_enough(self._log_level, level): + self.emit_message( + AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=level, message=filter_secrets(json.dumps(message_provider())))) + ) + + def consume_queue(self) -> Iterable[AirbyteMessage]: + while self._message_queue: + yield self._message_queue.popleft() + + +class LogAppenderMessageRepositoryDecorator(MessageRepository): + def __init__(self, dict_to_append: LogMessage, decorated: MessageRepository, log_level: Level = Level.INFO): + self._dict_to_append = dict_to_append + self._decorated = decorated + self._log_level = log_level + + def emit_message(self, message: AirbyteMessage) -> None: + self._decorated.emit_message(message) + + def log_message(self, level: Level, message_provider: Callable[[], LogMessage]) -> None: + if _is_severe_enough(self._log_level, level): + message = message_provider() + self._append_second_to_first(message, self._dict_to_append) + self._decorated.log_message(level, lambda: message) + + def consume_queue(self) -> Iterable[AirbyteMessage]: + return self._decorated.consume_queue() + + def _append_second_to_first(self, first: LogMessage, second: LogMessage, path: Optional[List[str]] = None) -> LogMessage: + if path is None: + path = [] + + for key in second: + if key in first: + if isinstance(first[key], dict) and isinstance(second[key], dict): + self._append_second_to_first(first[key], second[key], path + [str(key)]) # type: ignore # type is verified above + else: + if first[key] != second[key]: + _LOGGER.warning("Conflict at %s" % ".".join(path + [str(key)])) + first[key] = second[key] + else: + first[key] = second[key] + return first diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py index 9dacdb6c216e..03698afa5747 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py @@ -27,11 +27,16 @@ # AirbyteMessage: An AirbyteMessage. Could be of any type StreamData = Union[Mapping[str, Any], AirbyteMessage] +JsonSchema = Mapping[str, Any] + def package_name_from_class(cls: object) -> str: """Find the package name given a class name""" - module: Any = inspect.getmodule(cls) - return module.__name__.split(".")[0] + module = inspect.getmodule(cls) + if module is not None: + return module.__name__.split(".")[0] + else: + raise ValueError(f"Could not find package name for class {cls}") class IncrementalMixin(ABC): @@ -64,7 +69,7 @@ def state(self) -> MutableMapping[str, Any]: @state.setter @abstractmethod - def state(self, value: MutableMapping[str, Any]): + def state(self, value: MutableMapping[str, Any]) -> None: """State setter, accept state serialized by state getter.""" @@ -75,7 +80,7 @@ class Stream(ABC): # Use self.logger in subclasses to log any messages @property - def logger(self): + def logger(self) -> logging.Logger: return logging.getLogger(f"airbyte.streams.{self.name}") # TypeTransformer object to perform output data transformation @@ -104,9 +109,9 @@ def get_error_display_message(self, exception: BaseException) -> Optional[str]: def read_records( self, sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: """ This method should be overridden by subclasses to read records based on the inputs @@ -204,7 +209,7 @@ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: """ def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None ) -> Iterable[Optional[Mapping[str, Any]]]: """ Override to define the slices for this stream. See the stream slicing section of the docs for more information. @@ -231,7 +236,9 @@ def state_checkpoint_interval(self) -> Optional[int]: return None @deprecated(version="0.1.49", reason="You should use explicit state property instead, see IncrementalMixin docs.") - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + def get_updated_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: """Override to extract state from the latest record. Needed to implement incremental sync. Inspects the latest record extracted from the data source and the current state object and return an updated state object. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py index 939050f60ce7..3f8755070c4b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/availability_strategy.py @@ -40,6 +40,11 @@ def check_availability(self, stream: Stream, logger: logging.Logger, source: Opt # without accounting for the case in which the parent stream is empty. reason = f"Cannot attempt to connect to stream {stream.name} - no stream slices were found, likely because the parent stream is empty." return False, reason + except HTTPError as error: + is_available, reason = self.handle_http_error(stream, logger, source, error) + if not is_available: + reason = f"Unable to get slices for {stream.name} stream, because of error in parent stream. {reason}" + return is_available, reason try: get_first_record_for_slice(stream, stream_slice) @@ -48,7 +53,10 @@ def check_availability(self, stream: Stream, logger: logging.Logger, source: Opt logger.info(f"Successfully connected to stream {stream.name}, but got 0 records.") return True, None except HTTPError as error: - return self.handle_http_error(stream, logger, source, error) + is_available, reason = self.handle_http_error(stream, logger, source, error) + if not is_available: + reason = f"Unable to read {stream.name} stream. {reason}" + return is_available, reason def handle_http_error( self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError @@ -69,17 +77,20 @@ def handle_http_error( for some reason and the str should describe what went wrong and how to resolve the unavailability, if possible. """ - try: - status_code = error.response.status_code - reason = self.reasons_for_unavailable_status_codes(stream, logger, source, error)[status_code] - response_error_message = stream.parse_response_error_message(error.response) - if response_error_message: - reason += response_error_message - return False, reason - except KeyError: - # If the HTTPError is not in the dictionary of errors we know how to handle, don't except it + status_code = error.response.status_code + known_status_codes = self.reasons_for_unavailable_status_codes(stream, logger, source, error) + known_reason = known_status_codes.get(status_code) + if not known_reason: + # If the HTTPError is not in the dictionary of errors we know how to handle, don't except raise error + doc_ref = self._visit_docs_message(logger, source) + reason = f"The endpoint {error.response.url} returned {status_code}: {error.response.reason}. {known_reason}. {doc_ref} " + response_error_message = stream.parse_response_error_message(error.response) + if response_error_message: + reason += response_error_message + return False, reason + def reasons_for_unavailable_status_codes( self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError ) -> Dict[int, str]: @@ -95,17 +106,16 @@ def reasons_for_unavailable_status_codes( why 'status code' may have occurred and how the user can resolve that error, if applicable. """ - forbidden_error_message = f"The endpoint to access stream '{stream.name}' returned 403: Forbidden. " - forbidden_error_message += "This is most likely due to insufficient permissions on the credentials in use. " - forbidden_error_message += self._visit_docs_message(logger, source) - - reasons_for_codes: Dict[int, str] = {requests.codes.FORBIDDEN: forbidden_error_message} + reasons_for_codes: Dict[int, str] = { + requests.codes.FORBIDDEN: "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate" + } return reasons_for_codes @staticmethod def _visit_docs_message(logger: logging.Logger, source: Optional["Source"]) -> str: """ - Creates a message indicicating where to look in the documentation for + Creates a message indicating where to look in the documentation for more information on a given source by checking the spec of that source (if provided) for a 'documentationUrl'. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 892ce3eeaf54..af0936e13cb8 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -5,6 +5,7 @@ import logging import os +import urllib from abc import ABC, abstractmethod from contextlib import suppress from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -17,8 +18,8 @@ from airbyte_cdk.sources.streams.core import Stream, StreamData from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from requests.auth import AuthBase -from requests_cache.session import CachedSession +from ...utils.types import JsonType from .auth.core import HttpAuthenticator, NoAuth from .exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException from .rate_limiting import default_backoff_handler, user_defined_backoff_handler @@ -36,7 +37,7 @@ class HttpStream(Stream, ABC): page_size: Optional[int] = None # Use this variable to define page size for API http requests with pagination support # TODO: remove legacy HttpAuthenticator authenticator references - def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None): + def __init__(self, authenticator: Optional[Union[AuthBase, HttpAuthenticator]] = None): if self.use_cache: self._session = self.request_cache() else: @@ -49,24 +50,24 @@ def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None): self._authenticator = authenticator @property - def cache_filename(self): + def cache_filename(self) -> str: """ Override if needed. Return the name of cache file """ return f"{self.name}.sqlite" @property - def use_cache(self): + def use_cache(self) -> bool: """ Override if needed. If True, all records will be cached. """ return False - def request_cache(self) -> CachedSession: + def request_cache(self) -> requests.Session: self.clear_cache() return requests_cache.CachedSession(self.cache_filename) - def clear_cache(self): + def clear_cache(self) -> None: """ remove cache file only once """ @@ -133,9 +134,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def path( self, *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> str: """ Returns the URL path for the API endpoint e.g: if you wanted to hit https://myapi.com/v1/some_entity then this should return "some_entity" @@ -143,9 +144,9 @@ def path( def request_params( self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + stream_state: Optional[Mapping[str, Any]], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: """ Override this method to define the query parameters that should be set on an outgoing HTTP request given the inputs. @@ -155,7 +156,10 @@ def request_params( return {} def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Optional[Mapping[str, Any]], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: """ Override to return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method. @@ -164,10 +168,10 @@ def request_headers( def request_body_data( self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Union[Mapping, str]]: + stream_state: Optional[Mapping[str, Any]], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Optional[Union[Mapping[str, Any], str]]: """ Override when creating POST/PUT/PATCH requests to populate the body of the request with a non-JSON payload. @@ -181,10 +185,10 @@ def request_body_data( def request_body_json( self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: + stream_state: Optional[Mapping[str, Any]], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Optional[Mapping[str, Any]]: """ Override when creating POST/PUT/PATCH requests to populate the body of the request with a JSON payload. @@ -194,9 +198,9 @@ def request_body_json( def request_kwargs( self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + stream_state: Optional[Mapping[str, Any]], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: """ Override to return a mapping of keyword arguments to be used when creating the HTTP request. @@ -211,9 +215,9 @@ def parse_response( response: requests.Response, *, stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Iterable[Mapping[str, Any]]: """ Parses the raw response object into a list of records. By default, this returns an iterable containing the input. Override to parse differently. @@ -258,15 +262,38 @@ def error_message(self, response: requests.Response) -> str: """ return "" + def must_deduplicate_query_params(self) -> bool: + return False + + def deduplicate_query_params(self, url: str, params: Optional[Mapping[str, Any]]) -> Mapping[str, Any]: + """ + Remove query parameters from params mapping if they are already encoded in the URL. + :param url: URL with + :param params: + :return: + """ + if params is None: + params = {} + query_string = urllib.parse.urlparse(url).query + query_dict = {k: v[0] for k, v in urllib.parse.parse_qs(query_string).items()} + + duplicate_keys_with_same_value = {k for k in query_dict.keys() if str(params.get(k)) == str(query_dict[k])} + return {k: v for k, v in params.items() if k not in duplicate_keys_with_same_value} + def _create_prepared_request( self, path: str, - headers: Mapping = None, - params: Mapping = None, - json: Any = None, - data: Any = None, + headers: Optional[Mapping[str, str]] = None, + params: Optional[Mapping[str, str]] = None, + json: Optional[Mapping[str, Any]] = None, + data: Optional[Union[str, Mapping[str, Any]]] = None, ) -> requests.PreparedRequest: - args = {"method": self.http_method, "url": self._join_url(self.url_base, path), "headers": headers, "params": params} + url = self._join_url(self.url_base, path) + if self.must_deduplicate_query_params(): + query_params = self.deduplicate_query_params(url, params) + else: + query_params = params or {} + args = {"method": self.http_method, "url": url, "headers": headers, "params": query_params} if self.http_method.upper() in BODY_REQUEST_METHODS: if json and data: raise RequestBodyException( @@ -276,11 +303,10 @@ def _create_prepared_request( args["json"] = json elif data: args["data"] = data - return self._session.prepare_request(requests.Request(**args)) @classmethod - def _join_url(cls, url_base: str, path: str): + def _join_url(cls, url_base: str, path: str) -> str: return urljoin(url_base, path) def _send(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: @@ -374,11 +400,12 @@ def parse_response_error_message(cls, response: requests.Response) -> Optional[s """ # default logic to grab error from common fields - def _try_get_error(value): + def _try_get_error(value: Optional[JsonType]) -> Optional[str]: if isinstance(value, str): return value elif isinstance(value, list): - return ", ".join(_try_get_error(v) for v in value) + errors_in_value = [_try_get_error(v) for v in value] + return ", ".join(v for v in errors_in_value if v is not None) elif isinstance(value, dict): new_value = ( value.get("message") @@ -416,9 +443,9 @@ def get_error_display_message(self, exception: BaseException) -> Optional[str]: def read_records( self, sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: yield from self._read_pages( lambda req, res, state, _slice: self.parse_response(res, stream_slice=_slice, stream_state=state), stream_slice, stream_state @@ -427,10 +454,10 @@ def read_records( def _read_pages( self, records_generator_fn: Callable[ - [requests.PreparedRequest, requests.Response, Mapping[str, Any], Mapping[str, Any]], Iterable[StreamData] + [requests.PreparedRequest, requests.Response, Mapping[str, Any], Optional[Mapping[str, Any]]], Iterable[StreamData] ], - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: stream_state = stream_state or {} pagination_complete = False @@ -447,7 +474,10 @@ def _read_pages( yield from [] def _fetch_next_page( - self, stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Tuple[requests.PreparedRequest, requests.Response]: request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) request = self._create_prepared_request( @@ -464,7 +494,7 @@ def _fetch_next_page( class HttpSubStream(HttpStream, ABC): - def __init__(self, parent: HttpStream, **kwargs): + def __init__(self, parent: HttpStream, **kwargs: Any): """ :param parent: should be the instance of HttpStream class """ @@ -472,7 +502,7 @@ def __init__(self, parent: HttpStream, **kwargs): self.parent = parent def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None ) -> Iterable[Optional[Mapping[str, Any]]]: parent_stream_slices = self.parent.stream_slices( sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py index f38d70568393..9bc580d500fe 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py @@ -5,10 +5,10 @@ import logging import sys import time -from typing import Optional +from typing import Any, Callable, Mapping, Optional import backoff -from requests import codes, exceptions +from requests import PreparedRequest, RequestException, Response, codes, exceptions from .exceptions import DefaultBackoffException, UserDefinedBackoffException @@ -23,21 +23,31 @@ logger = logging.getLogger("airbyte") -def default_backoff_handler(max_tries: Optional[int], factor: float, **kwargs): - def log_retry_attempt(details): +SendRequestCallableType = Callable[[PreparedRequest, Mapping[str, Any]], Response] + + +def default_backoff_handler( + max_tries: Optional[int], factor: float, **kwargs: Any +) -> Callable[[SendRequestCallableType], SendRequestCallableType]: + def log_retry_attempt(details: Mapping[str, Any]) -> None: _, exc, _ = sys.exc_info() - if exc.response: + if isinstance(exc, RequestException) and exc.response: logger.info(f"Status code: {exc.response.status_code}, Response Content: {exc.response.content}") logger.info( f"Caught retryable error '{str(exc)}' after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." ) - def should_give_up(exc): + def should_give_up(exc: Exception) -> bool: # If a non-rate-limiting related 4XX error makes it this far, it means it was unexpected and probably consistent, so we shouldn't back off - give_up = exc.response is not None and exc.response.status_code != codes.too_many_requests and 400 <= exc.response.status_code < 500 - if give_up: - logger.info(f"Giving up for returned HTTP status: {exc.response.status_code}") - return give_up + if isinstance(exc, RequestException): + give_up: bool = ( + exc.response is not None and exc.response.status_code != codes.too_many_requests and 400 <= exc.response.status_code < 500 + ) + if give_up: + logger.info(f"Giving up for returned HTTP status: {exc.response.status_code}") + return give_up + # Only RequestExceptions are retryable, so if we get here, it's not retryable + return False return backoff.on_exception( backoff.expo, @@ -51,8 +61,8 @@ def should_give_up(exc): ) -def user_defined_backoff_handler(max_tries: Optional[int], **kwargs): - def sleep_on_ratelimit(details): +def user_defined_backoff_handler(max_tries: Optional[int], **kwargs: Any) -> Callable[[SendRequestCallableType], SendRequestCallableType]: + def sleep_on_ratelimit(details: Mapping[str, Any]) -> None: _, exc, _ = sys.exc_info() if isinstance(exc, UserDefinedBackoffException): if exc.response: @@ -61,9 +71,12 @@ def sleep_on_ratelimit(details): logger.info(f"Retrying. Sleeping for {retry_after} seconds") time.sleep(retry_after + 1) # extra second to cover any fractions of second - def log_give_up(details): + def log_give_up(details: Mapping[str, Any]) -> None: _, exc, _ = sys.exc_info() - logger.error(f"Max retry limit reached. Request: {exc.request}, Response: {exc.response}") + if isinstance(exc, RequestException): + logger.error(f"Max retry limit reached. Request: {exc.request}, Response: {exc.response}") + else: + logger.error("Max retry limit reached for unknown request and response") return backoff.on_exception( backoff.constant, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 642cdbf35a82..371f06b34d51 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -9,11 +9,15 @@ import backoff import pendulum import requests +from airbyte_cdk.models import Level +from airbyte_cdk.sources.http_logger import format_http_message +from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository from requests.auth import AuthBase from ..exceptions import DefaultBackoffException logger = logging.getLogger("airbyte") +_NOOP_MESSAGE_REPOSITORY = NoopMessageRepository() class AbstractOauth2Authenticator(AuthBase): @@ -23,6 +27,8 @@ class AbstractOauth2Authenticator(AuthBase): delegating that behavior to the classes implementing the interface. """ + _NO_STREAM_NAME = None + def __call__(self, request: requests.Request) -> requests.Request: """Attach the HTTP headers required to authenticate on the HTTP request""" request.headers.update(self.get_auth_header()) @@ -80,6 +86,7 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: def _get_refresh_access_token_response(self): try: response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body()) + self._log_response(response) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: @@ -151,3 +158,22 @@ def access_token(self) -> str: @abstractmethod def access_token(self, value: str) -> str: """Setter for the access token""" + + @property + def _message_repository(self) -> Optional[MessageRepository]: + """ + The implementation can define a message_repository if it wants debugging logs for HTTP requests + """ + return _NOOP_MESSAGE_REPOSITORY + + def _log_response(self, response: requests.Response): + self._message_repository.log_message( + Level.DEBUG, + lambda: format_http_message( + response, + "Refresh token", + "Obtains access token", + self._NO_STREAM_NAME, + is_auxiliary=True, + ), + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index b0262a94dc97..d7a93157ed99 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -2,11 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Sequence, Tuple, Union +from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union import dpath import pendulum -from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message +from airbyte_cdk.config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message +from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator @@ -109,11 +110,13 @@ def __init__( refresh_token_name: str = "refresh_token", refresh_request_body: Mapping[str, Any] = None, grant_type: str = "refresh_token", - client_id_config_path: Sequence[str] = ("credentials", "client_id"), - client_secret_config_path: Sequence[str] = ("credentials", "client_secret"), + client_id: Optional[str] = None, + client_secret: Optional[str] = None, access_token_config_path: Sequence[str] = ("credentials", "access_token"), refresh_token_config_path: Sequence[str] = ("credentials", "refresh_token"), token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"), + token_expiry_date_format: Optional[str] = None, + message_repository: MessageRepository = NoopMessageRepository(), ): """ @@ -126,20 +129,25 @@ def __init__( refresh_token_name (str, optional): Name of the name of the refresh token field, used to parse the refresh token response. Defaults to "refresh_token". refresh_request_body (Mapping[str, Any], optional): Custom key value pair that will be added to the refresh token request body. Defaults to None. grant_type (str, optional): OAuth grant type. Defaults to "refresh_token". - client_id_config_path (Sequence[str]): Dpath to the client_id field in the connector configuration. Defaults to ("credentials", "client_id"). - client_secret_config_path (Sequence[str]): Dpath to the client_secret field in the connector configuration. Defaults to ("credentials", "client_secret"). + client_id (Optional[str]): The client id to authenticate. If not specified, defaults to credentials.client_id in the config object. + client_secret (Optional[str]): The client secret to authenticate. If not specified, defaults to credentials.client_secret in the config object. access_token_config_path (Sequence[str]): Dpath to the access_token field in the connector configuration. Defaults to ("credentials", "access_token"). refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). token_expiry_date_config_path (Sequence[str]): Dpath to the token_expiry_date field in the connector configuration. Defaults to ("credentials", "token_expiry_date"). + token_expiry_date_format (Optional[str]): Date format of the token expiry date field (set by expires_in_name). If not specified the token expiry date is interpreted as number of seconds until expiration. + message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update """ - self._client_id_config_path = client_id_config_path - self._client_secret_config_path = client_secret_config_path + self._client_id = client_id if client_id is not None else dpath.util.get(connector_config, ("credentials", "client_id")) + self._client_secret = ( + client_secret if client_secret is not None else dpath.util.get(connector_config, ("credentials", "client_secret")) + ) self._access_token_config_path = access_token_config_path self._refresh_token_config_path = refresh_token_config_path self._token_expiry_date_config_path = token_expiry_date_config_path + self._token_expiry_date_format = token_expiry_date_format self._refresh_token_name = refresh_token_name self._connector_config = connector_config - self._validate_connector_config() + self.__message_repository = message_repository super().__init__( token_refresh_endpoint, self.get_client_id(), @@ -151,69 +159,49 @@ def __init__( expires_in_name=expires_in_name, refresh_request_body=refresh_request_body, grant_type=grant_type, + token_expiry_date_format=token_expiry_date_format, ) - def _validate_connector_config(self): - """Validates the defined getters for configuration values are returning values. - - Raises: - ValueError: Raised if the defined getters are not returning a value. - """ - try: - assert self.access_token - except KeyError: - raise ValueError( - f"This authenticator expects a value under the {self._access_token_config_path} field path. Please check your configuration structure or change the access_token_config_path value at initialization of this authenticator." - ) - for field_path, getter, parameter_name in [ - (self._client_id_config_path, self.get_client_id, "client_id_config_path"), - (self._client_secret_config_path, self.get_client_secret, "client_secret_config_path"), - (self._refresh_token_config_path, self.get_refresh_token, "refresh_token_config_path"), - (self._token_expiry_date_config_path, self.get_token_expiry_date, "token_expiry_date_config_path"), - ]: - try: - assert getter() - except KeyError: - raise ValueError( - f"This authenticator expects a value under the {field_path} field path. Please check your configuration structure or change the {parameter_name} value at initialization of this authenticator." - ) - def get_refresh_token_name(self) -> str: return self._refresh_token_name def get_client_id(self) -> str: - return dpath.util.get(self._connector_config, self._client_id_config_path) + return self._client_id def get_client_secret(self) -> str: - return dpath.util.get(self._connector_config, self._client_secret_config_path) + return self._client_secret @property def access_token(self) -> str: - return dpath.util.get(self._connector_config, self._access_token_config_path) + return dpath.util.get(self._connector_config, self._access_token_config_path, default="") @access_token.setter def access_token(self, new_access_token: str): - dpath.util.set(self._connector_config, self._access_token_config_path, new_access_token) + dpath.util.new(self._connector_config, self._access_token_config_path, new_access_token) def get_refresh_token(self) -> str: - return dpath.util.get(self._connector_config, self._refresh_token_config_path) + return dpath.util.get(self._connector_config, self._refresh_token_config_path, default="") def set_refresh_token(self, new_refresh_token: str): - dpath.util.set(self._connector_config, self._refresh_token_config_path, new_refresh_token) + dpath.util.new(self._connector_config, self._refresh_token_config_path, new_refresh_token) def get_token_expiry_date(self) -> pendulum.DateTime: - return pendulum.parse(dpath.util.get(self._connector_config, self._token_expiry_date_config_path)) + expiry_date = dpath.util.get(self._connector_config, self._token_expiry_date_config_path, default="") + return pendulum.now().subtract(days=1) if expiry_date == "" else pendulum.parse(expiry_date) def set_token_expiry_date(self, new_token_expiry_date): - dpath.util.set(self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)) + dpath.util.new(self._connector_config, self._token_expiry_date_config_path, str(new_token_expiry_date)) def token_has_expired(self) -> bool: """Returns True if the token is expired""" return pendulum.now("UTC") > self.get_token_expiry_date() @staticmethod - def get_new_token_expiry_date(access_token_expires_in: int): - return pendulum.now("UTC").add(seconds=access_token_expires_in) + def get_new_token_expiry_date(access_token_expires_in: str, token_expiry_date_format: str = None) -> pendulum.DateTime: + if token_expiry_date_format: + return pendulum.from_format(access_token_expires_in, token_expiry_date_format) + else: + return pendulum.now("UTC").add(seconds=int(access_token_expires_in)) def get_access_token(self) -> str: """Retrieve new access and refresh token if the access token has expired. @@ -223,17 +211,30 @@ def get_access_token(self) -> str: """ if self.token_has_expired(): new_access_token, access_token_expires_in, new_refresh_token = self.refresh_access_token() - new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in) + new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, self._token_expiry_date_format) self.access_token = new_access_token self.set_refresh_token(new_refresh_token) self.set_token_expiry_date(new_token_expiry_date) - emit_configuration_as_airbyte_control_message(self._connector_config) + # FIXME emit_configuration_as_airbyte_control_message as been deprecated in favor of package airbyte_cdk.sources.message + # Usually, a class shouldn't care about the implementation details but to keep backward compatibility where we print the + # message directly in the console, this is needed + if not isinstance(self._message_repository, NoopMessageRepository): + self._message_repository.emit_message(create_connector_config_control_message(self._connector_config)) + else: + emit_configuration_as_airbyte_control_message(self._connector_config) return self.access_token - def refresh_access_token(self) -> Tuple[str, int, str]: + def refresh_access_token(self) -> Tuple[str, str, str]: response_json = self._get_refresh_access_token_response() return ( response_json[self.get_access_token_name()], - int(response_json[self.get_expires_in_name()]), + response_json[self.get_expires_in_name()], response_json[self.get_refresh_token_name()], ) + + @property + def _message_repository(self) -> MessageRepository: + """ + Overriding AbstractOauth2Authenticator._message_repository to allow for HTTP request logs + """ + return self.__message_repository diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/casing.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/casing.py index bf0ac86b16cf..806e077ae00c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/casing.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/casing.py @@ -7,6 +7,6 @@ # https://stackoverflow.com/a/1176023 -def camel_to_snake(s): +def camel_to_snake(s: str) -> str: s = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", s) return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s).lower() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py index 4a590c6bcbd8..b9a8d97e3849 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/record_helper.py @@ -20,8 +20,8 @@ def stream_data_to_airbyte_message( if schema is None: schema = {} - if isinstance(data_or_message, dict): - data = data_or_message + if isinstance(data_or_message, Mapping): + data = dict(data_or_message) now_millis = int(datetime.datetime.now().timestamp() * 1000) # Transform object fields according to config. Most likely you will # need it to normalize values against json schema. By default no action diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/types.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/types.py new file mode 100644 index 000000000000..9dc5e253bf29 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/types.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Union + +JsonType = Union[dict[str, "JsonType"], list["JsonType"], str, int, float, bool, None] diff --git a/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py b/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py new file mode 100644 index 000000000000..8e29a274d25d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, Optional + +from airbyte_cdk.models import AirbyteRecordMessage +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser + + +class DatetimeFormatInferrer: + """ + This class is used to detect toplevel fields in records that might be datetime values, along with the used format. + """ + + def __init__(self) -> None: + self._parser = DatetimeParser() + self._datetime_candidates: Optional[Dict[str, str]] = None + self._formats = [ + "%Y-%m-%d", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S.%fZ", + "%Y-%m-%d %H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%s", + "%ms", + "%d/%m/%Y %H:%M", + "%Y-%m", + "%d-%m-%Y", + ] + self._timestamp_heuristic_ranges = [range(1_000_000_000, 2_000_000_000), range(1_000_000_000_000, 2_000_000_000_000)] + + def _can_be_datetime(self, value: Any) -> bool: + """Checks if the value can be a datetime. + This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000 for seconds + or between 1_000_000_000_000 and 2_000_000_000_000 for milliseconds. + This is separate from the format check for performance reasons""" + for timestamp_range in self._timestamp_heuristic_ranges: + if isinstance(value, str) and (not value.isdecimal() or int(value) in timestamp_range): + return True + if isinstance(value, int) and value in timestamp_range: + return True + return False + + def _matches_format(self, value: Any, format: str) -> bool: + """Checks if the value matches the format""" + try: + self._parser.parse(value, format) + return True + except ValueError: + return False + + def _initialize(self, record: AirbyteRecordMessage) -> None: + """Initializes the internal state of the class""" + self._datetime_candidates = {} + for field_name, field_value in record.data.items(): + if not self._can_be_datetime(field_value): + continue + for format in self._formats: + if self._matches_format(field_value, format): + self._datetime_candidates[field_name] = format + break + + def _validate(self, record: AirbyteRecordMessage) -> None: + """Validates that the record is consistent with the inferred datetime formats""" + if self._datetime_candidates: + for candidate_field_name in list(self._datetime_candidates.keys()): + candidate_field_format = self._datetime_candidates[candidate_field_name] + current_value = record.data.get(candidate_field_name, None) + if ( + current_value is None + or not self._can_be_datetime(current_value) + or not self._matches_format(current_value, candidate_field_format) + ): + self._datetime_candidates.pop(candidate_field_name) + + def accumulate(self, record: AirbyteRecordMessage) -> None: + """Analyzes the record and updates the internal state of candidate datetime fields""" + self._initialize(record) if self._datetime_candidates is None else self._validate(record) + + def get_inferred_datetime_formats(self) -> Dict[str, str]: + """ + Returns the list of candidate datetime fields - the keys are the field names and the values are the inferred datetime formats. + For these fields the format was consistent across all visited records. + """ + return self._datetime_candidates or {} diff --git a/airbyte-cdk/python/airbyte_cdk/utils/mapping_helpers.py b/airbyte-cdk/python/airbyte_cdk/utils/mapping_helpers.py new file mode 100644 index 000000000000..ae5e898f667d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/mapping_helpers.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, List, Mapping, Optional, Set, Union + + +def combine_mappings(mappings: List[Optional[Union[Mapping[str, Any], str]]]) -> Union[Mapping[str, Any], str]: + """ + Combine multiple mappings into a single mapping. If any of the mappings are a string, return + that string. Raise errors in the following cases: + * If there are duplicate keys across mappings + * If there are multiple string mappings + * If there are multiple mappings containing keys and one of them is a string + """ + all_keys: List[Set[str]] = [] + for part in mappings: + if part is None: + continue + keys = set(part.keys()) if not isinstance(part, str) else set() + all_keys.append(keys) + + string_options = sum(isinstance(mapping, str) for mapping in mappings) + # If more than one mapping is a string, raise a ValueError + if string_options > 1: + raise ValueError("Cannot combine multiple string options") + + if string_options == 1 and sum(len(keys) for keys in all_keys) > 0: + raise ValueError("Cannot combine multiple options if one is a string") + + # If any mapping is a string, return it + for mapping in mappings: + if isinstance(mapping, str): + return mapping + + # If there are duplicate keys across mappings, raise a ValueError + intersection = set().union(*all_keys) + if len(intersection) < sum(len(keys) for keys in all_keys): + raise ValueError(f"Duplicate keys found: {intersection}") + + # Return the combined mappings + return {key: value for mapping in mappings if mapping for key, value in mapping.items()} # type: ignore # mapping can't be string here diff --git a/airbyte-cdk/python/airbyte_cdk/utils/schema_inferrer.py b/airbyte-cdk/python/airbyte_cdk/utils/schema_inferrer.py index 37c87ac87197..41f8e179469e 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/schema_inferrer.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/schema_inferrer.py @@ -3,10 +3,10 @@ # from collections import defaultdict -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Mapping, Optional from airbyte_cdk.models import AirbyteRecordMessage -from genson import SchemaBuilder +from genson import SchemaBuilder, SchemaNode from genson.schema.strategies.object import Object from genson.schema.strategies.scalar import Number @@ -17,8 +17,8 @@ class NoRequiredObj(Object): every time it parses object. So we dont add unnecessary extra field. """ - def to_schema(self): - schema = super(NoRequiredObj, self).to_schema() + def to_schema(self) -> Mapping[str, Any]: + schema: Dict[str, Any] = super(NoRequiredObj, self).to_schema() schema.pop("required", None) return schema @@ -28,7 +28,7 @@ class IntegerToNumber(Number): This class has the regular Number behaviour, but it will never emit an integer type. """ - def __init__(self, node_class): + def __init__(self, node_class: SchemaNode): super().__init__(node_class) self._type = "number" @@ -38,7 +38,7 @@ class NoRequiredSchemaBuilder(SchemaBuilder): # This type is inferred from the genson lib, but there is no alias provided for it - creating it here for type safety -InferredSchema = Dict[str, Union[str, Any, List, List[Dict[str, Union[Any, List]]]]] +InferredSchema = Dict[str, Any] class SchemaInferrer: @@ -53,10 +53,10 @@ class SchemaInferrer: stream_to_builder: Dict[str, SchemaBuilder] - def __init__(self): + def __init__(self) -> None: self.stream_to_builder = defaultdict(NoRequiredSchemaBuilder) - def accumulate(self, record: AirbyteRecordMessage): + def accumulate(self, record: AirbyteRecordMessage) -> None: """Uses the input record to add to the inferred schemas maintained by this object""" self.stream_to_builder[record.stream].add_object(record.data) @@ -70,7 +70,7 @@ def get_inferred_schemas(self) -> Dict[str, InferredSchema]: schemas[stream_name] = self._clean(builder.to_schema()) return schemas - def _clean(self, node: InferredSchema): + def _clean(self, node: InferredSchema) -> InferredSchema: """ Recursively cleans up a produced schema: - remove anyOf if one of them is just a null value @@ -83,7 +83,7 @@ def _clean(self, node: InferredSchema): node.update(real_type) node["type"] = [node["type"], "null"] node.pop("anyOf") - if "properties" in node: + if "properties" in node and isinstance(node["properties"], dict): for key, value in list(node["properties"].items()): if isinstance(value, dict) and value.get("type", None) == "null": node["properties"].pop(key) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/spec_schema_transformations.py b/airbyte-cdk/python/airbyte_cdk/utils/spec_schema_transformations.py new file mode 100644 index 000000000000..2a772d50b6c3 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/spec_schema_transformations.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import re + +from jsonschema import RefResolver + + +def resolve_refs(schema: dict) -> dict: + """ + For spec schemas generated using Pydantic models, the resulting JSON schema can contain refs between object + relationships. + """ + json_schema_ref_resolver = RefResolver.from_schema(schema) + str_schema = json.dumps(schema) + for ref_block in re.findall(r'{"\$ref": "#\/definitions\/.+?(?="})"}', str_schema): + ref = json.loads(ref_block)["$ref"] + str_schema = str_schema.replace(ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])) + pyschema: dict = json.loads(str_schema) + del pyschema["definitions"] + return pyschema diff --git a/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh b/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh new file mode 100755 index 000000000000..6b45a7548f9d --- /dev/null +++ b/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh @@ -0,0 +1,3 @@ +set -e +# TODO change this to include unit_tests as well once it's in a good state +{ git diff --name-only --relative ':(exclude)unit_tests'; git diff --name-only --staged --relative ':(exclude)unit_tests'; git diff --name-only master... --relative ':(exclude)unit_tests'; } | grep -E '\.py$' | sort | uniq | xargs .venv/bin/python -m mypy --config-file mypy.ini --install-types --non-interactive diff --git a/airbyte-cdk/python/build.gradle b/airbyte-cdk/python/build.gradle index 8e21b03a7f59..9624bcac5804 100644 --- a/airbyte-cdk/python/build.gradle +++ b/airbyte-cdk/python/build.gradle @@ -23,7 +23,13 @@ task runLowCodeConnectorUnitTests(type: Exec) { commandLine 'bin/low-code-unit-tests.sh' } -blackFormat.dependsOn generateComponentManifestClassFiles +task runMypyOnModifiedFiles(type: Exec) { + environment 'ROOT_DIR', rootDir.absolutePath + commandLine 'bin/run-mypy-on-modified-files.sh' +} + +blackFormat.dependsOn runMypyOnModifiedFiles isortFormat.dependsOn generateComponentManifestClassFiles flakeCheck.dependsOn generateComponentManifestClassFiles installReqs.dependsOn generateComponentManifestClassFiles +runMypyOnModifiedFiles.dependsOn generateComponentManifestClassFiles \ No newline at end of file diff --git a/airbyte-cdk/python/mypy.ini b/airbyte-cdk/python/mypy.ini new file mode 100644 index 000000000000..f51c0846fb8f --- /dev/null +++ b/airbyte-cdk/python/mypy.ini @@ -0,0 +1,26 @@ +# Global options: + +[mypy] +warn_unused_configs = True +warn_redundant_casts = True +ignore_missing_imports = True +strict_equality = True +check_untyped_defs = True +disallow_untyped_decorators = False +disallow_any_generics = True +disallow_untyped_calls = True +disallow_incomplete_defs = True +disallow_untyped_defs = True +warn_return_any = True + +# Only alert on the files we want to check +follow_imports = silent + +# Allow re-exporting types for airbyte-protocol +no_implicit_reexport = False + +[tool.mypy] +plugins = ["pydantic.mypy", "pendulum", "pytest-mypy-plugins"] + +[mypy-airbyte_cdk.models] +ignore_errors = True diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index bfb29bf1d4b7..a2860804f5e8 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -13,11 +13,15 @@ # The text of the README file README = (HERE / "README.md").read_text() +avro_dependency = "avro~=1.11.2" +fastavro_dependency = "fastavro~=1.8.0" +pyarrow_dependency = "pyarrow==12.0.1" + setup( name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.39.1", + version="0.51.6", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -46,7 +50,7 @@ packages=find_packages(exclude=("unit_tests",)), package_data={"airbyte_cdk": ["py.typed", "sources/declarative/declarative_component_schema.yaml"]}, install_requires=[ - "airbyte-protocol-models==0.3.6", + "airbyte-protocol-models==0.4.0", "backoff", "dpath~=2.0.1", "isodate~=0.6.1", @@ -54,29 +58,39 @@ "jsonref~=0.2", "pendulum", "genson==1.2.2", - "pydantic~=1.9.2", + "pydantic>=1.9.2,<2.0.0", "python-dateutil", - "PyYAML~=5.4", + "PyYAML>=6.0.1", "requests", "requests_cache", "Deprecated~=1.2", "Jinja2~=3.1.2", "cachetools", + "wcmatch==8.4", ], python_requires=">=3.8", extras_require={ "dev": [ + avro_dependency, + fastavro_dependency, "freezegun", - "MyPy~=0.812", + "mypy", "pytest", "pytest-cov", "pytest-mock", "requests-mock", "pytest-httpserver", + "pandas==2.0.3", + pyarrow_dependency, ], "sphinx-docs": [ "Sphinx~=4.2", "sphinx-rtd-theme~=1.0", ], + "file-based": [ + avro_dependency, + fastavro_dependency, + pyarrow_dependency, + ], }, ) diff --git a/airbyte-cdk/python/type_check_and_test.sh b/airbyte-cdk/python/type_check_and_test.sh index 93fccd7e1e19..37220d8f8e51 100755 --- a/airbyte-cdk/python/type_check_and_test.sh +++ b/airbyte-cdk/python/type_check_and_test.sh @@ -5,7 +5,7 @@ # Static Type Checking echo "Running MyPy to static check and test files." -mypy airbyte_cdk/ unit_tests/ +mypy airbyte_cdk/ unit_tests/ --config mypy.ini printf "\n" diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index b21a393bc520..f9cd37e2750a 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -6,8 +6,9 @@ import dataclasses import json import logging +import os from unittest import mock -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest import requests @@ -19,7 +20,6 @@ TestReadLimits, create_source, get_limits, - list_streams, resolve_manifest, ) from airbyte_cdk.connector_builder.main import handle_connector_builder_request, handle_request, read_stream @@ -41,8 +41,7 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.declarative.retrievers import SimpleRetrieverTestReadDecorator -from airbyte_cdk.sources.streams.core import Stream -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from unit_tests.connector_builder.utils import create_configured_catalog _stream_name = "stream_with_custom_requester" @@ -96,6 +95,54 @@ } } +OAUTH_MANIFEST = { + "version": "0.30.3", + "definitions": { + "retriever": { + "paginator": { + "type": "DefaultPaginator", + "page_size": _page_size, + "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, + "page_token_option": {"inject_into": "path", "type": "RequestPath"}, + "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}", "page_size": _page_size}, + }, + "partition_router": { + "type": "ListPartitionRouter", + "values": ["0", "1", "2", "3", "4", "5", "6", "7"], + "cursor_field": "item_id" + }, + "" + "requester": { + "path": "/v3/marketing/lists", + "authenticator": { + "type": "OAuthAuthenticator", + "api_token": "{{ config.apikey }}" + }, + "request_parameters": {"a_param": "10"}, + }, + "record_selector": {"extractor": {"field_path": ["result"]}}, + }, + }, + "streams": [ + { + "type": "DeclarativeStream", + "$parameters": _stream_options, + "retriever": "#/definitions/retriever", + }, + ], + "check": {"type": "CheckStream", "stream_names": ["lists"]}, + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": True + }, + "type": "Spec" + } +} + RESOLVE_MANIFEST_CONFIG = { "__injected_declarative_manifest": MANIFEST, "__command": "resolve_manifest", @@ -137,6 +184,14 @@ ] } +MOCK_RESPONSE = { + "result": [ + {"id": 1, "name": "Nora Moon", "position": "director"}, + {"id": 2, "name": "Hae Sung Jung", "position": "cinematographer"}, + {"id": 3, "name": "Arthur Zenneranski", "position": "composer"}, + ] +} + @pytest.fixture def valid_resolve_manifest_config_file(tmp_path): @@ -175,10 +230,23 @@ def invalid_config_file(tmp_path): return config_file +def _mocked_send(self, request, **kwargs) -> requests.Response: + """ + Mocks the outbound send operation to provide faster and more reliable responses compared to actual API requests + """ + response = requests.Response() + response.request = request + response.status_code = 200 + response.headers = {"header": "value"} + response_body = MOCK_RESPONSE + response._content = json.dumps(response_body).encode("utf-8") + return response + + def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catalog): - with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patch: + with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patched_handle: handle_request(["read", "--config", str(valid_resolve_manifest_config_file), "--catalog", str(dummy_catalog)]) - assert patch.call_count == 1 + assert patched_handle.call_count == 1 def test_handle_test_read(valid_read_config_file, configured_catalog): @@ -352,8 +420,11 @@ def test_read(): state=None, ) ], + auxiliary_requests=[], test_read_limit_reached=False, inferred_schema=None, + inferred_datetime_formats=None, + latest_config_update={} ) expected_airbyte_message = AirbyteMessage( @@ -366,7 +437,10 @@ def test_read(): {"pages": [{"records": [real_record], "request": None, "response": None}], "slice_descriptor": None, "state": None} ], "test_read_limit_reached": False, + "auxiliary_requests": [], "inferred_schema": None, + "inferred_datetime_formats": None, + "latest_config_update": {} }, emitted_at=1, ), @@ -380,6 +454,37 @@ def test_read(): assert output_record == expected_airbyte_message +def test_config_update(): + manifest = copy.deepcopy(MANIFEST) + manifest["definitions"]["retriever"]["requester"]["authenticator"] = { + "type": "OAuthAuthenticator", + "token_refresh_endpoint": "https://oauth.endpoint.com/tokens/bearer", + "client_id": "{{ config['credentials']['client_id'] }}", + "client_secret": "{{ config['credentials']['client_secret'] }}", + "refresh_token": "{{ config['credentials']['refresh_token'] }}", + "refresh_token_updater": {} + } + config = copy.deepcopy(TEST_READ_CONFIG) + config["__injected_declarative_manifest"] = manifest + config["credentials"] = { + "client_id": "a client id", + "client_secret": "a client secret", + "refresh_token": "a refresh token", + } + source = ManifestDeclarativeSource(manifest) + + refresh_request_response = { + "access_token": "an updated access token", + "refresh_token": "an updated refresh token", + "expires_in": 3600, + } + with patch("airbyte_cdk.sources.streams.http.requests_native_auth.SingleUseRefreshTokenOauth2Authenticator._get_refresh_access_token_response", return_value=refresh_request_response): + output = handle_connector_builder_request( + source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), TestReadLimits() + ) + assert output.record.data["latest_config_update"] + + @patch("traceback.TracebackException.from_exception") def test_read_returns_error_response(mock_from_exception): class MockManifestDeclarativeSource: @@ -403,11 +508,12 @@ def check_config_against_spec(self): response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), limits) expected_stream_read = StreamRead(logs=[LogMessage("error_message - a stack trace", "ERROR")], - slices=[StreamReadSlices( - pages=[StreamReadPages(records=[], request=None, response=None)], - slice_descriptor=None, state=None)], + slices=[], test_read_limit_reached=False, - inferred_schema=None) + auxiliary_requests=[], + inferred_schema=None, + inferred_datetime_formats={}, + latest_config_update=None) expected_message = AirbyteMessage( type=MessageType.RECORD, @@ -429,7 +535,7 @@ def check_config_against_spec(self): ) def test_invalid_protocol_command(command, valid_resolve_manifest_config_file): config = copy.deepcopy(RESOLVE_MANIFEST_CONFIG) - config["__command"] = "list_streams" + config["__command"] = "resolve_manifest" with pytest.raises(SystemExit): handle_request([command, "--config", str(valid_resolve_manifest_config_file), "--catalog", ""]) @@ -459,76 +565,13 @@ def manifest_declarative_source(): return mock.Mock(spec=ManifestDeclarativeSource, autospec=True) -def test_list_streams(manifest_declarative_source): - manifest_declarative_source.streams.return_value = [ - create_mock_declarative_stream(create_mock_http_stream("a name", "https://a-url-base.com", "a-path")), - create_mock_declarative_stream(create_mock_http_stream("another name", "https://another-url-base.com", "another-path")), - ] - - result = list_streams(manifest_declarative_source, {}) - - assert result.type == MessageType.RECORD - assert result.record.stream == "list_streams" - assert result.record.data == { - "streams": [ - {"name": "a name", "url": "https://a-url-base.com/a-path"}, - {"name": "another name", "url": "https://another-url-base.com/another-path"}, - ] - } - - -def test_given_stream_is_not_declarative_stream_when_list_streams_then_return_exception_message(manifest_declarative_source): - manifest_declarative_source.streams.return_value = [mock.Mock(spec=Stream)] - - error_message = list_streams(manifest_declarative_source, {}) - - assert error_message.type == MessageType.TRACE - assert error_message.trace.error.message.startswith("Error listing streams") - assert "A declarative source should only contain streams of type DeclarativeStream" in error_message.trace.error.internal_message - - -def test_given_declarative_stream_retriever_is_not_http_when_list_streams_then_return_exception_message(manifest_declarative_source): - declarative_stream = mock.Mock(spec=DeclarativeStream) - # `spec=DeclarativeStream` is needed for `isinstance` work but `spec` does not expose dataclasses fields, so we create one ourselves - declarative_stream.retriever = mock.Mock() - manifest_declarative_source.streams.return_value = [declarative_stream] - - error_message = list_streams(manifest_declarative_source, {}) - - assert error_message.type == MessageType.TRACE - assert error_message.trace.error.message.startswith("Error listing streams") - assert "A declarative stream should only have a retriever of type HttpStream" in error_message.trace.error.internal_message - - -def test_given_unexpected_error_when_list_streams_then_return_exception_message(manifest_declarative_source): - manifest_declarative_source.streams.side_effect = Exception("unexpected error") - - error_message = list_streams(manifest_declarative_source, {}) - - assert error_message.type == MessageType.TRACE - assert error_message.trace.error.message.startswith("Error listing streams") - assert "unexpected error" == error_message.trace.error.internal_message - - -def test_list_streams_integration_test(): - config = copy.deepcopy(RESOLVE_MANIFEST_CONFIG) - command = "list_streams" - config["__command"] = command - source = ManifestDeclarativeSource(MANIFEST) - limits = TestReadLimits() - - list_streams = handle_connector_builder_request(source, command, config, None, limits) - - assert list_streams.record.data == { - "streams": [{"name": "stream_with_custom_requester", "url": "https://api.sendgrid.com/v3/marketing/lists"}] - } - - -def create_mock_http_stream(name, url_base, path): - http_stream = mock.Mock(spec=HttpStream, autospec=True) +def create_mock_retriever(name, url_base, path): + http_stream = mock.Mock(spec=SimpleRetriever, autospec=True) http_stream.name = name - http_stream.url_base = url_base - http_stream.path.return_value = path + http_stream.requester = MagicMock() + http_stream.requester.get_url_base.return_value = url_base + http_stream.requester.get_path.return_value = path + http_stream._paginator_path.return_value = None return http_stream @@ -566,7 +609,7 @@ def test_create_source(): assert isinstance(source, ManifestDeclarativeSource) assert source._constructor._limit_pages_fetched_per_slice == limits.max_pages_per_slice assert source._constructor._limit_slices_fetched == limits.max_slices - assert source.streams(config={})[0].retriever.max_retries == 0 + assert source.streams(config={})[0].retriever.requester.max_retries == 0 def request_log_message(request: dict) -> AirbyteMessage: @@ -583,19 +626,21 @@ def _create_request(): return requests.Request('POST', url, headers=headers, json={"key": "value"}).prepare() -def _create_response(body): +def _create_response(body, request): response = requests.Response() response.status_code = 200 response._content = bytes(json.dumps(body), "utf-8") response.headers["Content-Type"] = "application/json" + response.request = request return response -def _create_page(response_body): - return _create_request(), _create_response(response_body) +def _create_page_response(response_body): + request = _create_request() + return _create_response(response_body, request) -@patch.object(HttpStream, "_fetch_next_page", side_effect=(_create_page({"result": [{"id": 0}, {"id": 1}],"_metadata": {"next": "next"}}), _create_page({"result": [{"id": 2}],"_metadata": {"next": "next"}})) * 10) +@patch.object(requests.Session, "send", side_effect=(_create_page_response({"result": [{"id": 0}, {"id": 1}],"_metadata": {"next": "next"}}), _create_page_response({"result": [{"id": 2}],"_metadata": {"next": "next"}})) * 10) def test_read_source(mock_http_stream): """ This test sort of acts as an integration test for the connector builder. @@ -636,7 +681,7 @@ def test_read_source(mock_http_stream): assert isinstance(s.retriever, SimpleRetrieverTestReadDecorator) -@patch.object(HttpStream, "_fetch_next_page", side_effect=(_create_page({"result": [{"id": 0}, {"id": 1}],"_metadata": {"next": "next"}}), _create_page({"result": [{"id": 2}],"_metadata": {"next": "next"}}))) +@patch.object(requests.Session, "send", side_effect=(_create_page_response({"result": [{"id": 0}, {"id": 1}],"_metadata": {"next": "next"}}), _create_page_response({"result": [{"id": 2}],"_metadata": {"next": "next"}}))) def test_read_source_single_page_single_slice(mock_http_stream): max_records = 100 max_pages_per_slice = 1 @@ -665,3 +710,109 @@ def test_read_source_single_page_single_slice(mock_http_stream): streams = source.streams(config) for s in streams: assert isinstance(s.retriever, SimpleRetrieverTestReadDecorator) + + +@pytest.mark.parametrize( + "deployment_mode, url_base, expected_error", + [ + pytest.param("CLOUD", "https://airbyte.com/api/v1/characters", None, id="test_cloud_read_with_public_endpoint"), + pytest.param("CLOUD", "https://10.0.27.27", "ValueError", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "ValueError", id="test_cloud_read_with_localhost"), + pytest.param("CLOUD", "http://unsecured.protocol/api/v1", "ValueError", id="test_cloud_read_with_unsecured_endpoint"), + pytest.param("OSS", "https://airbyte.com/api/v1/", None, id="test_oss_read_with_public_endpoint"), + pytest.param("OSS", "https://10.0.27.27/api/v1/", None, id="test_oss_read_with_private_endpoint"), + ] +) +@patch.object(requests.Session, "send", _mocked_send) +def test_handle_read_external_requests(deployment_mode, url_base, expected_error): + """ + This test acts like an integration test for the connector builder when it receives Test Read requests. + + The scenario being tested is whether requests should be denied if they are done on an unsecure channel or are made to internal + endpoints when running on Cloud or OSS deployments + """ + + limits = TestReadLimits(max_records=100, max_pages_per_slice=1, max_slices=1) + + catalog = ConfiguredAirbyteCatalog(streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name=_stream_name, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ]) + + test_manifest = MANIFEST + test_manifest["streams"][0]["$parameters"]["url_base"] = url_base + config = {"__injected_declarative_manifest": test_manifest} + + source = create_source(config, limits) + + with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): + output_data = read_stream(source, config, catalog, limits).record.data + if expected_error: + assert len(output_data["logs"]) > 0, "Expected at least one log message with the expected error" + error_message = output_data["logs"][0] + assert error_message["level"] == "ERROR" + assert expected_error in error_message["message"] + else: + page_records = output_data["slices"][0]["pages"][0] + assert len(page_records) == len(MOCK_RESPONSE["result"]) + + +@pytest.mark.parametrize( + "deployment_mode, token_url, expected_error", + [ + pytest.param("CLOUD", "https://airbyte.com/tokens/bearer", None, id="test_cloud_read_with_public_endpoint"), + pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "ValueError", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "http://unsecured.protocol/tokens/bearer", "ValueError", id="test_cloud_read_with_unsecured_endpoint"), + pytest.param("OSS", "https://airbyte.com/tokens/bearer", None, id="test_oss_read_with_public_endpoint"), + pytest.param("OSS", "https://10.0.27.27/tokens/bearer", None, id="test_oss_read_with_private_endpoint"), + ] +) +@patch.object(requests.Session, "send", _mocked_send) +def test_handle_read_external_oauth_request(deployment_mode, token_url, expected_error): + """ + This test acts like an integration test for the connector builder when it receives Test Read requests. + + The scenario being tested is whether requests should be denied if they are done on an unsecure channel or are made to internal + endpoints when running on Cloud or OSS deployments + """ + + limits = TestReadLimits(max_records=100, max_pages_per_slice=1, max_slices=1) + + catalog = ConfiguredAirbyteCatalog(streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name=_stream_name, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ]) + + oauth_authenticator_config: dict[str, str] = { + "type": "OAuthAuthenticator", + "token_refresh_endpoint": token_url, + "client_id": "greta", + "client_secret": "teo", + "refresh_token": "john", + } + + test_manifest = MANIFEST + test_manifest["definitions"]["retriever"]["requester"]["authenticator"] = oauth_authenticator_config + config = {"__injected_declarative_manifest": test_manifest} + + source = create_source(config, limits) + + with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): + output_data = read_stream(source, config, catalog, limits).record.data + if expected_error: + assert len(output_data["logs"]) > 0, "Expected at least one log message with the expected error" + error_message = output_data["logs"][0] + assert error_message["level"] == "ERROR" + assert expected_error in error_message["message"] diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index 0e1feb1398a3..67d437dfac91 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -3,13 +3,21 @@ # import json -from typing import Iterator -from unittest.mock import MagicMock, patch +from typing import Any, Iterator, List, Mapping +from unittest.mock import MagicMock, Mock, patch import pytest from airbyte_cdk.connector_builder.message_grouper import MessageGrouper from airbyte_cdk.connector_builder.models import HttpRequest, HttpResponse, LogMessage, StreamRead, StreamReadPages -from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, Level +from airbyte_cdk.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + Level, + OrchestratorType, +) from airbyte_cdk.models import Type as MessageType from unit_tests.connector_builder.utils import create_configured_catalog @@ -78,49 +86,48 @@ @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages(mock_entrypoint_read): +def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: + url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { - "url": "https://demonslayers.com/api/v1/hashiras?era=taisho", "headers": {"Content-Type": "application/json"}, - "http_method": "GET", - "body": {"custom": "field"}, + "method": "GET", + "body": {"content": '{"custom": "field"}'}, } - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}', "http_method": "GET"} - expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}}, "type": "object"} + response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} + expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}, "date": {"type": "string"}}, "type": "object"} + expected_datetime_fields = {"date": "%Y-%m-%d"} expected_pages = [ StreamReadPages( request=HttpRequest( url="https://demonslayers.com/api/v1/hashiras", parameters={"era": ["taisho"]}, headers={"Content-Type": "application/json"}, - body={"custom": "field"}, + body='{"custom": "field"}', http_method="GET", ), response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'), - records=[{"name": "Shinobu Kocho"}, {"name": "Muichiro Tokito"}], + records=[{"name": "Shinobu Kocho", "date": "2023-03-03"}, {"name": "Muichiro Tokito", "date": "2023-03-04"}], ), StreamReadPages( request=HttpRequest( url="https://demonslayers.com/api/v1/hashiras", parameters={"era": ["taisho"]}, headers={"Content-Type": "application/json"}, - body={"custom": "field"}, + body='{"custom": "field"}', http_method="GET", ), response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'), - records=[{"name": "Mitsuri Kanroji"}], + records=[{"name": "Mitsuri Kanroji", "date": "2023-03-05"}], ), ] mock_source = make_mock_source(mock_entrypoint_read, iter( [ - request_log_message(request), - response_log_message(response), - record_message("hashiras", {"name": "Shinobu Kocho"}), - record_message("hashiras", {"name": "Muichiro Tokito"}), - request_log_message(request), - response_log_message(response), - record_message("hashiras", {"name": "Mitsuri Kanroji"}), + request_response_log_message(request, response, url), + record_message("hashiras", {"name": "Shinobu Kocho", "date": "2023-03-03"}), + record_message("hashiras", {"name": "Muichiro Tokito", "date": "2023-03-04"}), + request_response_log_message(request, response, url), + record_message("hashiras", {"name": "Mitsuri Kanroji", "date": "2023-03-05"}), ] )) @@ -130,6 +137,7 @@ def test_get_grouped_messages(mock_entrypoint_read): ) assert actual_response.inferred_schema == expected_schema + assert actual_response.inferred_datetime_formats == expected_datetime_fields single_slice = actual_response.slices[0] for i, actual_page in enumerate(single_slice.pages): @@ -137,21 +145,21 @@ def test_get_grouped_messages(mock_entrypoint_read): @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_with_logs(mock_entrypoint_read): +def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: + url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { - "url": "https://demonslayers.com/api/v1/hashiras?era=taisho", "headers": {"Content-Type": "application/json"}, - "body": {"custom": "field"}, - "http_method": "GET", + "method": "GET", + "body": {"content": '{"custom": "field"}'}, } - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} + response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} expected_pages = [ StreamReadPages( request=HttpRequest( url="https://demonslayers.com/api/v1/hashiras", parameters={"era": ["taisho"]}, headers={"Content-Type": "application/json"}, - body={"custom": "field"}, + body='{"custom": "field"}', http_method="GET", ), response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'), @@ -162,7 +170,7 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read): url="https://demonslayers.com/api/v1/hashiras", parameters={"era": ["taisho"]}, headers={"Content-Type": "application/json"}, - body={"custom": "field"}, + body='{"custom": "field"}', http_method="GET", ), response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'), @@ -178,8 +186,7 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read): mock_source = make_mock_source(mock_entrypoint_read, iter( [ AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message="log message before the request")), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message="log message during the page")), record_message("hashiras", {"name": "Muichiro Tokito"}), @@ -209,23 +216,21 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read): ], ) @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_record_limit(mock_entrypoint_read, request_record_limit, max_record_limit): +def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int) -> None: + url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { - "url": "https://demonslayers.com/api/v1/hashiras?era=taisho", "headers": {"Content-Type": "application/json"}, - "body": {"custom": "field"}, + "method": "GET", + "body": {"content": '{"custom": "field"}'}, } - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} + response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} mock_source = make_mock_source(mock_entrypoint_read, iter( [ - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), record_message("hashiras", {"name": "Muichiro Tokito"}), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Mitsuri Kanroji"}), - response_log_message(response), ] ) ) @@ -242,6 +247,8 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read, request_record_ total_records += len(actual_page.records) assert total_records == min([record_limit, n_records]) + assert (total_records >= max_record_limit) == actual_response.test_read_limit_reached + @pytest.mark.parametrize( "max_record_limit", @@ -251,23 +258,21 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read, request_record_ ], ) @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_default_record_limit(mock_entrypoint_read, max_record_limit): +def test_get_grouped_messages_default_record_limit(mock_entrypoint_read: Mock, max_record_limit: int) -> None: + url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { - "url": "https://demonslayers.com/api/v1/hashiras?era=taisho", "headers": {"Content-Type": "application/json"}, - "body": {"custom": "field"}, + "method": "GET", + "body": {"content": '{"custom": "field"}'}, } - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} + response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} mock_source = make_mock_source(mock_entrypoint_read, iter( [ - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), record_message("hashiras", {"name": "Muichiro Tokito"}), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Mitsuri Kanroji"}), - response_log_message(response), ] ) ) @@ -285,23 +290,21 @@ def test_get_grouped_messages_default_record_limit(mock_entrypoint_read, max_rec @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_limit_0(mock_entrypoint_read): +def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: + url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { - "url": "https://demonslayers.com/api/v1/hashiras?era=taisho", "headers": {"Content-Type": "application/json"}, - "body": {"custom": "field"}, + "method": "GET", + "body": {"content": '{"custom": "field"}'}, } - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} + response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} mock_source = make_mock_source(mock_entrypoint_read, iter( [ - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), record_message("hashiras", {"name": "Muichiro Tokito"}), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Mitsuri Kanroji"}), - response_log_message(response), ] ) ) @@ -312,21 +315,21 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read): @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_no_records(mock_entrypoint_read): +def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: + url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { - "url": "https://demonslayers.com/api/v1/hashiras?era=taisho", "headers": {"Content-Type": "application/json"}, - "body": {"custom": "field"}, - "http_method": "GET", + "method": "GET", + "body": {"content": '{"custom": "field"}'}, } - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} + response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} expected_pages = [ StreamReadPages( request=HttpRequest( url="https://demonslayers.com/api/v1/hashiras", parameters={"era": ["taisho"]}, headers={"Content-Type": "application/json"}, - body={"custom": "field"}, + body='{"custom": "field"}', http_method="GET", ), response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'), @@ -337,7 +340,7 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read): url="https://demonslayers.com/api/v1/hashiras", parameters={"era": ["taisho"]}, headers={"Content-Type": "application/json"}, - body={"custom": "field"}, + body='{"custom": "field"}', http_method="GET", ), response=HttpResponse(status=200, headers={"field": "value"}, body='{"name": "field"}'), @@ -347,10 +350,8 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read): mock_source = make_mock_source(mock_entrypoint_read, iter( [ - request_log_message(request), - response_log_message(response), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), + request_response_log_message(request, response, url), ] ) ) @@ -366,48 +367,33 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read): assert actual_page == expected_pages[i] -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_invalid_group_format(mock_entrypoint_read): - response = {"status_code": 200, "headers": {"field": "value"}, "body": '{"name": "field"}'} - - mock_source = make_mock_source(mock_entrypoint_read, iter( - [ - response_log_message(response), - record_message("hashiras", {"name": "Shinobu Kocho"}), - record_message("hashiras", {"name": "Muichiro Tokito"}), - ] - ) - ) - - api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) - - with pytest.raises(ValueError): - api.get_message_groups(source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras")) - - @pytest.mark.parametrize( "log_message, expected_response", [ pytest.param( - {"status_code": 200, "headers": {"field": "name"}, "body": '{"id": "fire", "owner": "kyojuro_rengoku"}'}, + {"http": {"response": {"status_code": 200, "headers": {"field": "name"}, "body": {"content": '{"id": "fire", "owner": "kyojuro_rengoku"}'}}}}, HttpResponse(status=200, headers={"field": "name"}, body='{"id": "fire", "owner": "kyojuro_rengoku"}'), id="test_create_response_with_all_fields", ), pytest.param( - {"status_code": 200, "headers": {"field": "name"}}, - HttpResponse(status=200, headers={"field": "name"}, body="{}"), + {"http": {"response": {"status_code": 200, "headers": {"field": "name"}}}}, + HttpResponse(status=200, headers={"field": "name"}, body=""), id="test_create_response_with_no_body", ), pytest.param( - {"status_code": 200, "body": '{"id": "fire", "owner": "kyojuro_rengoku"}'}, + {"http": {"response": {"status_code": 200, "body": {"content": '{"id": "fire", "owner": "kyojuro_rengoku"}'}}}}, HttpResponse(status=200, body='{"id": "fire", "owner": "kyojuro_rengoku"}'), id="test_create_response_with_no_headers", ), pytest.param( { - "status_code": 200, - "headers": {"field": "name"}, - "body": '[{"id": "fire", "owner": "kyojuro_rengoku"}, {"id": "mist", "owner": "muichiro_tokito"}]', + "http": { + "response": { + "status_code": 200, + "headers": {"field": "name"}, + "body": {"content": '[{"id": "fire", "owner": "kyojuro_rengoku"}, {"id": "mist", "owner": "muichiro_tokito"}]'}, + } + } }, HttpResponse( status=200, @@ -417,55 +403,49 @@ def test_get_grouped_messages_invalid_group_format(mock_entrypoint_read): id="test_create_response_with_array", ), pytest.param( - {"status_code": 200, "body": "tomioka"}, + {"http": {"response": {"status_code": 200, "body": {"content": "tomioka"}}}}, HttpResponse(status=200, body="tomioka"), id="test_create_response_with_string", ), - pytest.param("request:{invalid_json: }", None, id="test_invalid_json_still_does_not_crash"), - pytest.param("just a regular log message", None, id="test_no_response:_prefix_does_not_crash"), ], ) -def test_create_response_from_log_message(log_message, expected_response): +def test_create_response_from_log_message(log_message: str, expected_response: HttpResponse) -> None: if isinstance(log_message, str): - response_message = log_message + response_message = json.loads(log_message) else: - response_message = f"response:{json.dumps(log_message)}" + response_message = log_message - airbyte_log_message = AirbyteLogMessage(level=Level.INFO, message=response_message) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) - actual_response = connector_builder_handler._create_response_from_log_message(airbyte_log_message) + actual_response = connector_builder_handler._create_response_from_log_message(response_message) assert actual_response == expected_response @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_with_many_slices(mock_entrypoint_read): - request = {} +def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> None: + url = "http://a-url.com" + request: Mapping[str, Any] = {} response = {"status_code": 200} mock_source = make_mock_source(mock_entrypoint_read, iter( [ slice_message('{"descriptor": "first_slice"}'), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Muichiro Tokito"}), slice_message('{"descriptor": "second_slice"}'), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), record_message("hashiras", {"name": "Mitsuri Kanroji"}), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), record_message("hashiras", {"name": "Obanai Iguro"}), - request_log_message(request), - response_log_message(response), + request_response_log_message(request, response, url), ] ) ) - connecto_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) - stream_read: StreamRead = connecto_builder_handler.get_message_groups( + stream_read: StreamRead = connector_builder_handler.get_message_groups( source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") ) @@ -484,11 +464,11 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read): @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limit_reached(mock_entrypoint_read): +def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limit_reached(mock_entrypoint_read: Mock) -> None: maximum_number_of_slices = 5 - request = {} + request: Mapping[str, Any] = {} response = {"status_code": 200} - mock_source = make_mock_source(mock_entrypoint_read, iter([slice_message(), request_log_message(request), response_log_message(response)] * maximum_number_of_slices)) + mock_source = make_mock_source(mock_entrypoint_read, iter([slice_message(), request_response_log_message(request, response, "a_url")] * maximum_number_of_slices)) api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -500,11 +480,11 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi @patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit_reached(mock_entrypoint_read): +def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit_reached(mock_entrypoint_read: Mock) -> None: maximum_number_of_pages_per_slice = 5 - request = {} + request: Mapping[str, Any] = {} response = {"status_code": 200} - mock_source = make_mock_source(mock_entrypoint_read, iter([slice_message()] + [request_log_message(request), response_log_message(response)] * maximum_number_of_pages_per_slice)) + mock_source = make_mock_source(mock_entrypoint_read, iter([slice_message()] + [request_response_log_message(request, response, "a_url")] * maximum_number_of_pages_per_slice)) api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -515,11 +495,11 @@ def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit assert stream_read.test_read_limit_reached -def test_read_stream_returns_error_if_stream_does_not_exist(): +def test_read_stream_returns_error_if_stream_does_not_exist() -> None: mock_source = MagicMock() mock_source.read.side_effect = ValueError("error") - full_config = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}} + full_config: Mapping[str, Any] = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}} message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response = message_grouper.get_message_groups(source=mock_source, config=full_config, @@ -530,23 +510,152 @@ def test_read_stream_returns_error_if_stream_does_not_exist(): assert "ERROR" in actual_response.logs[0].level -def make_mock_source(mock_entrypoint_read, return_value: Iterator) -> MagicMock: +@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +def test_given_control_message_then_stream_read_has_config_update(mock_entrypoint_read: Mock) -> None: + updated_config = {"x": 1} + mock_source = make_mock_source(mock_entrypoint_read, iter( + any_request_and_response_with_a_record() + [connector_configuration_control_message(1, updated_config)] + )) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + stream_read: StreamRead = connector_builder_handler.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") + ) + + assert stream_read.latest_config_update == updated_config + + +@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_emitted_at(mock_entrypoint_read: Mock) -> None: + earliest = 0 + earliest_config = {"earliest": 0} + latest = 1 + latest_config = {"latest": 1} + mock_source = make_mock_source(mock_entrypoint_read, iter( + any_request_and_response_with_a_record() + + [ + # here, we test that even if messages are emitted in a different order, we still rely on `emitted_at` + connector_configuration_control_message(latest, latest_config), + connector_configuration_control_message(earliest, earliest_config), + ] + ) + ) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + stream_read: StreamRead = connector_builder_handler.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") + ) + + assert stream_read.latest_config_update == latest_config + + +@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_has_latest_based_on_message_order(mock_entrypoint_read: Mock) -> None: + emitted_at = 0 + earliest_config = {"earliest": 0} + latest_config = {"latest": 1} + mock_source = make_mock_source(mock_entrypoint_read, iter( + any_request_and_response_with_a_record() + + [ + connector_configuration_control_message(emitted_at, earliest_config), + connector_configuration_control_message(emitted_at, latest_config), + ] + ) + ) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + stream_read: StreamRead = connector_builder_handler.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") + ) + + assert stream_read.latest_config_update == latest_config + + +@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_read: Mock) -> None: + mock_source = make_mock_source(mock_entrypoint_read, iter( + any_request_and_response_with_a_record() + + [ + auxiliary_request_log_message() + ] + )) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + stream_read: StreamRead = connector_builder_handler.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") + ) + + assert len(stream_read.auxiliary_requests) == 1 + + +@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> None: + mock_source = make_mock_source(mock_entrypoint_read, iter([auxiliary_request_log_message()])) + connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) + stream_read: StreamRead = connector_builder_handler.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") + ) + + assert len(stream_read.slices) == 0 + + +def make_mock_source(mock_entrypoint_read: Mock, return_value: Iterator[AirbyteMessage]) -> MagicMock: mock_source = MagicMock() mock_entrypoint_read.return_value = return_value return mock_source -def request_log_message(request: dict) -> AirbyteMessage: +def request_log_message(request: Mapping[str, Any]) -> AirbyteMessage: return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=f"request:{json.dumps(request)}")) -def response_log_message(response: dict) -> AirbyteMessage: +def response_log_message(response: Mapping[str, Any]) -> AirbyteMessage: return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=f"response:{json.dumps(response)}")) -def record_message(stream: str, data: dict) -> AirbyteMessage: +def record_message(stream: str, data: Mapping[str, Any]) -> AirbyteMessage: return AirbyteMessage(type=MessageType.RECORD, record=AirbyteRecordMessage(stream=stream, data=data, emitted_at=1234)) def slice_message(slice_descriptor: str = '{"key": "value"}') -> AirbyteMessage: return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message="slice:" + slice_descriptor)) + + +def connector_configuration_control_message(emitted_at: float, config: Mapping[str, Any]) -> AirbyteMessage: + return AirbyteMessage( + type=MessageType.CONTROL, + control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=emitted_at, + connectorConfig=AirbyteControlConnectorConfigMessage(config=config), + ) + ) + + +def auxiliary_request_log_message() -> AirbyteMessage: + return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=json.dumps({ + "http": { + "is_auxiliary": True, + "title": "a title", + "description": "a description", + "request": {}, + "response": {}, + }, + "url": {"full": "https://a-url.com"} + }))) + + +def request_response_log_message(request: Mapping[str, Any], response: Mapping[str, Any], url: str) -> AirbyteMessage: + return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=json.dumps({ + "airbyte_cdk": {"stream": {"name": "a stream name"}}, + "http": { + "title": "a title", + "description": "a description", + "request": request, + "response": response + }, + "url": {"full": url} + }))) + + +def any_request_and_response_with_a_record() -> List[AirbyteMessage]: + return [ + request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), + record_message("hashiras", {"name": "Shinobu Kocho"}), + ] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py index 93eaa02f178a..ff425380ed3c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py @@ -3,6 +3,7 @@ # import logging +from unittest.mock import Mock import freezegun import pendulum @@ -138,6 +139,7 @@ def test_refresh_access_token(self, mocker): def test_refresh_access_token_expire_format(self, mocker, expires_in_response, token_expiry_date_format): next_day = "2020-01-02T00:00:00Z" config.update({"token_expiry_date": pendulum.parse(next_day).subtract(days=2).to_rfc3339_string()}) + message_repository = Mock() oauth = DeclarativeOauth2Authenticator( token_refresh_endpoint="{{ config['refresh_endpoint'] }}", client_id="{{ config['client_id'] }}", @@ -152,6 +154,7 @@ def test_refresh_access_token_expire_format(self, mocker, expires_in_response, t "another_field": "{{ config['another_field'] }}", "scopes": ["no_override"], }, + message_repository=message_repository, parameters={}, ) @@ -161,6 +164,7 @@ def test_refresh_access_token_expire_format(self, mocker, expires_in_response, t token = oauth.get_access_token() assert "access_token" == token assert oauth.get_token_expiry_date() == pendulum.parse(next_day) + assert message_repository.log_message.call_count == 1 @pytest.mark.parametrize( "expires_in_response, next_day, raises", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py index bfb27c7e151b..5f99fabf00b3 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_session_token_auth.py @@ -3,7 +3,7 @@ # import pytest -from airbyte_cdk.sources.declarative.auth.token import SessionTokenAuthenticator, get_new_session_token +from airbyte_cdk.sources.declarative.auth.token import LegacySessionTokenAuthenticator, get_new_session_token from requests.exceptions import HTTPError parameters = {"hello": "world"} @@ -56,7 +56,7 @@ def test_auth_header(): - auth_header = SessionTokenAuthenticator( + auth_header = LegacySessionTokenAuthenticator( config=config, parameters=parameters, api_url=input_instance_api_url, @@ -76,7 +76,7 @@ def test_get_token_valid_session(requests_mock): f"{config_session_token['instance_api_url']}user/current", json={"common_name": "common_name", "last_login": "last_login"} ) - token = SessionTokenAuthenticator( + token = LegacySessionTokenAuthenticator( config=config_session_token, parameters=parameters, api_url=input_instance_api_url, @@ -93,7 +93,7 @@ def test_get_token_valid_session(requests_mock): def test_get_token_invalid_session_unauthorized(): with pytest.raises(ConnectionError): - _ = SessionTokenAuthenticator( + _ = LegacySessionTokenAuthenticator( config=config_session_token, parameters=parameters, api_url=input_instance_api_url, @@ -109,7 +109,7 @@ def test_get_token_invalid_session_unauthorized(): def test_get_token_invalid_username_password_unauthorized(): with pytest.raises(HTTPError): - _ = SessionTokenAuthenticator( + _ = LegacySessionTokenAuthenticator( config=config_username_password, parameters=parameters, api_url=input_instance_api_url, @@ -126,7 +126,7 @@ def test_get_token_invalid_username_password_unauthorized(): def test_get_token_username_password(requests_mock): requests_mock.post(f"{config['instance_api_url']}session", json={"id": "some session id"}) - token = SessionTokenAuthenticator( + token = LegacySessionTokenAuthenticator( config=config_username_password, parameters=parameters, api_url=input_instance_api_url, @@ -144,7 +144,7 @@ def test_get_token_username_password(requests_mock): def test_check_is_valid_session_token(requests_mock): requests_mock.get(f"{config['instance_api_url']}user/current", json={"common_name": "common_name", "last_login": "last_login"}) - assert SessionTokenAuthenticator( + assert LegacySessionTokenAuthenticator( config=config, parameters=parameters, api_url=input_instance_api_url, @@ -159,7 +159,7 @@ def test_check_is_valid_session_token(requests_mock): def test_check_is_valid_session_token_unauthorized(): - assert not SessionTokenAuthenticator( + assert not LegacySessionTokenAuthenticator( config=config, parameters=parameters, api_url=input_instance_api_url, diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py index 357ca047a92b..6217d72fe9c7 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py @@ -7,6 +7,8 @@ import pytest import requests from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator +from airbyte_cdk.sources.declarative.auth.token_provider import InterpolatedStringTokenProvider +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from requests import Response LOGGER = logging.getLogger(__name__) @@ -28,7 +30,8 @@ def test_bearer_token_authenticator(test_name, token, expected_header_value): """ Should match passed in token, no matter how many times token is retrieved. """ - token_auth = BearerAuthenticator(token, config, parameters=parameters) + token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) + token_auth = BearerAuthenticator(token_provider, config, parameters=parameters) header1 = token_auth.get_auth_header() header2 = token_auth.get_auth_header() @@ -78,7 +81,17 @@ def test_api_key_authenticator(test_name, header, token, expected_header, expect """ Should match passed in token, no matter how many times token is retrieved. """ - token_auth = ApiKeyAuthenticator(header=header, api_token=token, config=config, parameters=parameters) + token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) + token_auth = ApiKeyAuthenticator( + request_option=RequestOption( + inject_into=RequestOptionType.header, + field_name=header, + parameters={} + ), + token_provider=token_provider, + config=config, + parameters=parameters + ) header1 = token_auth.get_auth_header() header2 = token_auth.get_auth_header() @@ -89,3 +102,35 @@ def test_api_key_authenticator(test_name, header, token, expected_header, expect assert {expected_header: expected_header_value} == prepared_request.headers assert {expected_header: expected_header_value} == header1 assert {expected_header: expected_header_value} == header2 + + +@pytest.mark.parametrize( + "test_name, field_name, token, expected_field_name, expected_field_value, inject_type, validation_fn", + [ + ("test_static_token", "Authorization", "test-token", "Authorization", "test-token", RequestOptionType.request_parameter, "get_request_params"), + ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user", RequestOptionType.request_parameter, "get_request_params"), + ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user", RequestOptionType.request_parameter, "get_request_params"), + ("test_static_token", "Authorization", "test-token", "Authorization", "test-token", RequestOptionType.body_data, "get_request_body_data"), + ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user", RequestOptionType.body_data, "get_request_body_data"), + ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user", RequestOptionType.body_data, "get_request_body_data"), + ("test_static_token", "Authorization", "test-token", "Authorization", "test-token", RequestOptionType.body_json, "get_request_body_json"), + ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user", RequestOptionType.body_json, "get_request_body_json"), + ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user", RequestOptionType.body_json, "get_request_body_json"), + ], +) +def test_api_key_authenticator_inject(test_name, field_name, token, expected_field_name, expected_field_value, inject_type, validation_fn): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) + token_auth = ApiKeyAuthenticator( + request_option=RequestOption( + inject_into=inject_type, + field_name=field_name, + parameters={} + ), + token_provider=token_provider, + config=config, + parameters=parameters + ) + assert {expected_field_name: expected_field_value} == getattr(token_auth, validation_fn)() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py new file mode 100644 index 000000000000..7badce21801f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pendulum +import pytest +from airbyte_cdk.sources.declarative.auth.token_provider import InterpolatedStringTokenProvider, SessionTokenProvider +from airbyte_cdk.sources.declarative.exceptions import ReadException +from isodate import parse_duration + + +def create_session_token_provider(): + login_requester = MagicMock() + login_response = MagicMock() + login_response.json.return_value = {"nested": {"token": "my_token"}} + login_requester.send_request.return_value = login_response + + return SessionTokenProvider(login_requester=login_requester, session_token_path=["nested", "token"], expiration_duration=parse_duration("PT1H"), parameters={"test": "test"}) + + +def test_interpolated_string_token_provider(): + provider = InterpolatedStringTokenProvider(config={"config_key": "val"}, api_token="{{ config.config_key }}-{{ parameters.test }}", parameters={"test": "test"}) + assert provider.get_token() == "val-test" + + +def test_session_token_provider(): + provider = create_session_token_provider() + assert provider.get_token() == "my_token" + + +def test_session_token_provider_cache(): + provider = create_session_token_provider() + provider.get_token() + assert provider.get_token() == "my_token" + assert provider.login_requester.send_request.call_count == 1 + + +def test_session_token_provider_cache_expiration(): + with pendulum.test(pendulum.datetime(2001, 5, 21, 12)): + provider = create_session_token_provider() + provider.get_token() + + provider.login_requester.send_request.return_value.json.return_value = {"nested": {"token": "updated_token"}} + + with pendulum.test(pendulum.datetime(2001, 5, 21, 14)): + assert provider.get_token() == "updated_token" + + assert provider.login_requester.send_request.call_count == 2 + + +def test_session_token_provider_no_cache(): + provider = create_session_token_provider() + provider.expiration_duration = None + provider.get_token() + assert provider.login_requester.send_request.call_count == 1 + provider.get_token() + assert provider.login_requester.send_request.call_count == 2 + + +def test_session_token_provider_ignored_response(): + provider = create_session_token_provider() + provider.login_requester.send_request.return_value = None + with pytest.raises(ReadException): + provider.get_token() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 606ceb730efa..552f810f629f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -96,7 +96,7 @@ def test_check_stream_with_no_stream_slices_aborts(): 403, False, [ - "The endpoint to access stream 'mock_http_stream' returned 403: Forbidden.", + "Unable to read mock_http_stream stream", "This is most likely due to insufficient permissions on the credentials in use.", ], ), diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py index 06e0a5e1cac2..0e81f9987859 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py @@ -18,10 +18,10 @@ datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), ( - "test_parse_date_iso_with_timezone_not_utc", - "2021-01-01T00:00:00.000000+0400", - "%Y-%m-%dT%H:%M:%S.%f%z", - datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))), + "test_parse_date_iso_with_timezone_not_utc", + "2021-01-01T00:00:00.000000+0400", + "%Y-%m-%dT%H:%M:%S.%f%z", + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))), ), ( "test_parse_timestamp", @@ -29,7 +29,13 @@ "%s", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), - ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), + ( + "test_parse_timestamp", + "1609459200001", + "%ms", + datetime.datetime(2021, 1, 1, 0, 0, 0, 1000, tzinfo=datetime.timezone.utc), + ), + ("test_parse_date_ms", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ], ) def test_parse_date(test_name, input_date, date_format, expected_output_date): @@ -42,6 +48,7 @@ def test_parse_date(test_name, input_date, date_format, expected_output_date): "test_name, input_dt, datetimeformat, expected_output", [ ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), + ("test_format_timestamp_ms", datetime.datetime(2021, 1, 1, 0, 0, 0, 1000, tzinfo=datetime.timezone.utc), "%ms", "1609459200001"), ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py index c1c2d1b13158..89d003b77652 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_filter.py @@ -52,4 +52,4 @@ def test_record_filter(test_name, filter_template, records, expected_records): actual_records = record_filter.filter_records( records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - assert list(actual_records) == expected_records + assert actual_records == expected_records diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index 503cb5e054e3..619b228b009f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -3,6 +3,7 @@ # import json +from unittest.mock import Mock, call import pytest import requests @@ -10,10 +11,12 @@ from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector +from airbyte_cdk.sources.declarative.transformations import RecordTransformation +from airbyte_cdk.sources.declarative.types import Record @pytest.mark.parametrize( - "test_name, field_path, filter_template, body, expected_records", + "test_name, field_path, filter_template, body, expected_data", [ ( "test_with_extractor_and_filter", @@ -59,12 +62,15 @@ ), ], ) -def test_record_filter(test_name, field_path, filter_template, body, expected_records): +def test_record_filter(test_name, field_path, filter_template, body, expected_data): config = {"response_override": "stop_if_you_see_me"} parameters = {"parameters_field": "data", "created_at": "06-07-21"} stream_state = {"created_at": "06-06-21"} stream_slice = {"last_seen": "06-10-21"} next_page_token = {"last_seen_id": 14} + first_transformation = Mock(spec=RecordTransformation) + second_transformation = Mock(spec=RecordTransformation) + transformations = [first_transformation, second_transformation] response = create_response(body) decoder = JsonDecoder(parameters={}) @@ -73,12 +79,24 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_re record_filter = None else: record_filter = RecordFilter(config=config, condition=filter_template, parameters=parameters) - record_selector = RecordSelector(extractor=extractor, record_filter=record_filter, parameters=parameters) + record_selector = RecordSelector( + extractor=extractor, + record_filter=record_filter, + transformations=transformations, + config=config, + parameters=parameters + ) actual_records = record_selector.select_records( response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) - assert list(actual_records) == expected_records + assert actual_records == [Record(data, stream_slice) for data in expected_data] + calls = [] + for record in expected_data: + calls.append(call(record, config=config, stream_state=stream_state, stream_slice=stream_slice)) + for transformation in transformations: + assert transformation.transform.call_count == len(expected_data) + transformation.transform.assert_has_calls(calls) def create_response(body): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py index b26bf1b56e83..67617a9f0124 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py @@ -6,11 +6,11 @@ import unittest import pytest -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType +from airbyte_cdk.sources.declarative.types import Record datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z" cursor_granularity = "PT0.000001S" @@ -20,6 +20,8 @@ end_date_now = InterpolatedString(string="{{ today_utc() }}", parameters={}) cursor_field = "created" timezone = datetime.timezone.utc +NO_STATE = {} +ANY_SLICE = {} class MockedNowDatetime(datetime.datetime): @@ -38,7 +40,7 @@ def mock_datetime_now(monkeypatch): [ ( "test_1_day", - None, + NO_STATE, MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), "P1D", @@ -61,7 +63,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_2_day", - None, + NO_STATE, MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), "P2D", @@ -79,7 +81,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_1_week", - None, + NO_STATE, MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), MinMaxDatetime(datetime="2021-02-10T00:00:00.000000+0000", parameters={}), "P1W", @@ -98,7 +100,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_1_month", - None, + NO_STATE, MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), MinMaxDatetime(datetime="2021-06-10T00:00:00.000000+0000", parameters={}), "P1M", @@ -117,7 +119,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_1_year", - None, + NO_STATE, MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), MinMaxDatetime(datetime="2022-06-10T00:00:00.000000+0000", parameters={}), "P1Y", @@ -132,8 +134,8 @@ def mock_datetime_now(monkeypatch): ), ( "test_from_stream_state", - {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime(datetime="{{ stream_state['date'] }}", parameters={}), + {cursor_field: "2021-01-05T00:00:00.000000+0000"}, + MinMaxDatetime(datetime="2020-01-05T00:00:00.000000+0000", parameters={}), MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), "P1D", cursor_field, @@ -151,7 +153,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_12_day", - None, + NO_STATE, MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), "P12D", @@ -165,7 +167,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_end_time_greater_than_now", - None, + NO_STATE, MinMaxDatetime(datetime="2021-12-28T00:00:00.000000+0000", parameters={}), MinMaxDatetime(datetime=f"{(FAKE_NOW + datetime.timedelta(days=1)).strftime(datetime_format)}", parameters={}), "P1D", @@ -183,7 +185,7 @@ def mock_datetime_now(monkeypatch): ), ( "test_start_date_greater_than_end_time", - None, + NO_STATE, MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), MinMaxDatetime(datetime="2021-01-05T00:00:00.000000+0000", parameters={}), "P1D", @@ -197,11 +199,11 @@ def mock_datetime_now(monkeypatch): ), ( "test_cursor_date_greater_than_start_date", - {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime(datetime="{{ stream_state['date'] }}", parameters={}), + {cursor_field: "2021-01-05T00:00:00.000000+0000"}, + MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}), MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), "P1D", - InterpolatedString(string="{{ stream_state['date'] }}", parameters={}), + cursor_field, None, datetime_format, cursor_granularity, @@ -230,76 +232,17 @@ def mock_datetime_now(monkeypatch): {"start_time": "2021-01-09T00:00:00.000000+0000", "end_time": "2021-01-10T00:00:00.000000+0000"}, ], ), - ( - "test_start_date_less_than_min_date", - {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime(datetime="{{ config['start_date'] }}", min_datetime="{{ stream_state['date'] }}", parameters={}), - MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), - "P1D", - InterpolatedString(string="{{ stream_state['date'] }}", parameters={}), - None, - datetime_format, - cursor_granularity, - [ - {"start_time": "2021-01-05T00:00:00.000000+0000", "end_time": "2021-01-05T23:59:59.999999+0000"}, - {"start_time": "2021-01-06T00:00:00.000000+0000", "end_time": "2021-01-06T23:59:59.999999+0000"}, - {"start_time": "2021-01-07T00:00:00.000000+0000", "end_time": "2021-01-07T23:59:59.999999+0000"}, - {"start_time": "2021-01-08T00:00:00.000000+0000", "end_time": "2021-01-08T23:59:59.999999+0000"}, - {"start_time": "2021-01-09T00:00:00.000000+0000", "end_time": "2021-01-09T23:59:59.999999+0000"}, - {"start_time": "2021-01-10T00:00:00.000000+0000", "end_time": "2021-01-10T00:00:00.000000+0000"}, - ], - ), - ( - "test_end_date_greater_than_max_date", - {"date": "2021-01-05T00:00:00.000000+0000"}, - MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), - MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", max_datetime="{{ stream_state['date'] }}", parameters={}), - "P1D", - cursor_field, - None, - datetime_format, - cursor_granularity, - [ - {"start_time": "2021-01-01T00:00:00.000000+0000", "end_time": "2021-01-01T23:59:59.999999+0000"}, - {"start_time": "2021-01-02T00:00:00.000000+0000", "end_time": "2021-01-02T23:59:59.999999+0000"}, - {"start_time": "2021-01-03T00:00:00.000000+0000", "end_time": "2021-01-03T23:59:59.999999+0000"}, - {"start_time": "2021-01-04T00:00:00.000000+0000", "end_time": "2021-01-04T23:59:59.999999+0000"}, - {"start_time": "2021-01-05T00:00:00.000000+0000", "end_time": "2021-01-05T00:00:00.000000+0000"}, - ], - ), - ( - "test_start_end_min_max_inherits_datetime_format_from_stream_slicer", - {"date": "2021-01-05"}, - MinMaxDatetime(datetime="{{ config['start_date_ymd'] }}", parameters={}), - MinMaxDatetime(datetime="2021-01-10", max_datetime="{{ stream_state['date'] }}", parameters={}), - "P1D", - cursor_field, - None, - "%Y-%m-%d", - "P1D", - [ - {"start_time": "2021-01-01", "end_time": "2021-01-01"}, - {"start_time": "2021-01-02", "end_time": "2021-01-02"}, - {"start_time": "2021-01-03", "end_time": "2021-01-03"}, - {"start_time": "2021-01-04", "end_time": "2021-01-04"}, - {"start_time": "2021-01-05", "end_time": "2021-01-05"}, - ], - ), ( "test_with_lookback_window_from_start_date", - {"date": "2021-01-05"}, - MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), - MinMaxDatetime(datetime="2021-01-10", max_datetime="{{ stream_state['date'] }}", datetime_format="%Y-%m-%d", parameters={}), + NO_STATE, + MinMaxDatetime(datetime="2021-01-05", datetime_format="%Y-%m-%d", parameters={}), + MinMaxDatetime(datetime="2021-01-05", datetime_format="%Y-%m-%d", parameters={}), "P1D", cursor_field, "P3D", datetime_format, cursor_granularity, [ - {"start_time": "2020-12-29T00:00:00.000000+0000", "end_time": "2020-12-29T23:59:59.999999+0000"}, - {"start_time": "2020-12-30T00:00:00.000000+0000", "end_time": "2020-12-30T23:59:59.999999+0000"}, - {"start_time": "2020-12-31T00:00:00.000000+0000", "end_time": "2020-12-31T23:59:59.999999+0000"}, - {"start_time": "2021-01-01T00:00:00.000000+0000", "end_time": "2021-01-01T23:59:59.999999+0000"}, {"start_time": "2021-01-02T00:00:00.000000+0000", "end_time": "2021-01-02T23:59:59.999999+0000"}, {"start_time": "2021-01-03T00:00:00.000000+0000", "end_time": "2021-01-03T23:59:59.999999+0000"}, {"start_time": "2021-01-04T00:00:00.000000+0000", "end_time": "2021-01-04T23:59:59.999999+0000"}, @@ -326,9 +269,9 @@ def mock_datetime_now(monkeypatch): ), ( "test_with_lookback_window_defaults_to_0d", - {"date": "2021-01-05"}, - MinMaxDatetime(datetime="{{ config['start_date'] }}", parameters={}), - MinMaxDatetime(datetime="2021-01-10", max_datetime="{{ stream_state['date'] }}", datetime_format="%Y-%m-%d", parameters={}), + {}, + MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), + MinMaxDatetime(datetime="2021-01-05", datetime_format="%Y-%m-%d", parameters={}), "P1D", cursor_field, "{{ config['does_not_exist'] }}", @@ -377,7 +320,7 @@ def test_stream_slices( expected_slices, ): lookback_window = InterpolatedString(string=lookback_window, parameters={}) if lookback_window else None - slicer = DatetimeBasedCursor( + cursor = DatetimeBasedCursor( start_datetime=start, end_datetime=end, step=step, @@ -388,68 +331,97 @@ def test_stream_slices( config=config, parameters={}, ) - stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state) + cursor.set_initial_state(stream_state) + stream_slices = cursor.stream_slices() assert stream_slices == expected_slices @pytest.mark.parametrize( - "test_name, previous_cursor, stream_slice, last_record, expected_state", + "test_name, previous_cursor, stream_slice, latest_record_data, expected_state", [ - ("test_update_cursor_no_state_no_record", None, {}, None, {}), ( - "test_update_cursor_with_state_no_record", - None, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, - None, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, + "test_close_slice_previous_cursor_is_highest", + "2023-01-01", + {"end_time": "2022-01-01"}, + {cursor_field: "2021-01-01"}, + {cursor_field: "2023-01-01"}, ), ( - "test_update_cursor_with_state_equals_record", - None, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, + "test_close_slice_stream_slice_partition_end_is_highest", + "2021-01-01", + {"end_time": "2023-01-01"}, + {cursor_field: "2021-01-01"}, + {cursor_field: "2023-01-01"}, ), ( - "test_update_cursor_with_state_greater_than_record", - None, - {cursor_field: "2021-01-03T00:00:00.000000+0000"}, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, - {cursor_field: "2021-01-03T00:00:00.000000+0000"}, + "test_close_slice_latest_record_cursor_value_is_highest", + "2021-01-01", + {"end_time": "2022-01-01"}, + {cursor_field: "2023-01-01"}, + {cursor_field: "2023-01-01"}, ), ( - "test_update_cursor_with_state_less_than_record", + "test_close_slice_without_latest_record", + "2021-01-01", + {"end_time": "2022-01-01"}, None, - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, - {cursor_field: "2021-01-03T00:00:00.000000+0000"}, - {cursor_field: "2021-01-03T00:00:00.000000+0000"}, + {cursor_field: "2022-01-01"}, ), ( - "test_update_cursor_with_state_less_than_previous_cursor", - "2021-01-03T00:00:00.000000+0000", - {cursor_field: "2021-01-02T00:00:00.000000+0000"}, - {}, - {cursor_field: "2021-01-03T00:00:00.000000+0000"}, + "test_close_slice_without_cursor", + None, + {"end_time": "2022-01-01"}, + {cursor_field: "2023-01-01"}, + {cursor_field: "2023-01-01"}, ), ], ) -def test_update_cursor(test_name, previous_cursor, stream_slice, last_record, expected_state): - slicer = DatetimeBasedCursor( +def test_close_slice(test_name, previous_cursor, stream_slice, latest_record_data, expected_state): + cursor = DatetimeBasedCursor( start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}), - end_datetime=MinMaxDatetime(datetime="2021-01-10T00:00:00.000000+0000", parameters={}), - step="P1D", cursor_field=InterpolatedString(string=cursor_field, parameters={}), - datetime_format=datetime_format, - cursor_granularity=cursor_granularity, - lookback_window=InterpolatedString(string="0d", parameters={}), + datetime_format="%Y-%m-%d", config=config, parameters={}, ) - slicer._cursor = previous_cursor - slicer.update_cursor(stream_slice, last_record) - updated_state = slicer.get_stream_state() - assert expected_state == updated_state + cursor._cursor = previous_cursor + cursor.close_slice(stream_slice, Record(latest_record_data, stream_slice) if latest_record_data else None) + updated_state = cursor.get_stream_state() + assert updated_state == expected_state + + +def test_given_different_format_and_slice_is_highest_when_close_slice_then_slice_datetime_format(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}), + cursor_field=cursor_field, + datetime_format="%Y-%m-%dT%H:%M:%S.%fZ", + cursor_datetime_formats=["%Y-%m-%d"], + config=config, + parameters={}, + ) + + _slice = {"end_time": "2023-01-04T17:30:19.000Z"} + record_cursor_value = "2023-01-03" + cursor.close_slice(_slice, Record({cursor_field: record_cursor_value}, _slice)) + + assert cursor.get_stream_state()[cursor_field] == "2023-01-04T17:30:19.000Z" + + +def test_given_partition_end_is_specified_and_greater_than_record_when_close_slice_then_use_partition_end(): + partition_field_end = "partition_field_end" + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime(datetime="2021-01-01T00:00:00.000000+0000", parameters={}), + cursor_field=InterpolatedString(string=cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + partition_field_end=partition_field_end, + config=config, + parameters={}, + ) + stream_slice = {partition_field_end: "2025-01-01"} + cursor.close_slice(stream_slice, Record({cursor_field: "2020-01-01"}, stream_slice)) + updated_state = cursor.get_stream_state() + assert {cursor_field: "2025-01-01"} == updated_state @pytest.mark.parametrize( @@ -511,9 +483,6 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, parameters={}, ) stream_slice = {"start_time": "2021-01-01T00:00:00.000000+0000", "end_time": "2021-01-04T00:00:00.000000+0000"} - - slicer.update_cursor(stream_slice) - assert expected_req_params == slicer.get_request_params(stream_slice=stream_slice) assert expected_headers == slicer.get_request_headers(stream_slice=stream_slice) assert expected_body_json == slicer.get_request_body_json(stream_slice=stream_slice) @@ -540,7 +509,7 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, ("test_parse_date_number", "20210101", "%Y%m%d", "P1D", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ], ) -def test_parse_date(test_name, input_date, date_format, date_format_granularity, expected_output_date): +def test_parse_date_legacy_merge_datetime_format_in_cursor_datetime_format(test_name, input_date, date_format, date_format_granularity, expected_output_date): slicer = DatetimeBasedCursor( start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000", parameters={}), end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000", parameters={}), @@ -556,6 +525,48 @@ def test_parse_date(test_name, input_date, date_format, date_format_granularity, assert expected_output_date == output_date +@pytest.mark.parametrize( + "test_name, input_date, date_formats, expected_output_date", + [ + ( + "test_match_first_format", + "2021-01-01T00:00:00.000000+0000", + ["%Y-%m-%dT%H:%M:%S.%f%z", "%s"], + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + ), + ( + "test_match_second_format", + "1609459200", + ["%Y-%m-%dT%H:%M:%S.%f%z", "%s"], + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + ), + ], +) +def test_parse_date(test_name, input_date, date_formats, expected_output_date): + slicer = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + cursor_datetime_formats=date_formats, + config=config, + parameters={}, + ) + assert slicer.parse_date(input_date) == expected_output_date + + +def test_given_unknown_format_when_parse_date_then_raise_error(): + slicer = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + cursor_datetime_formats=["%Y-%m-%d", "%s"], + config=config, + parameters={}, + ) + with pytest.raises(ValueError): + slicer.parse_date("2021-01-01T00:00:00.000000+0000") + + @pytest.mark.parametrize( "test_name, input_dt, datetimeformat, datetimeformat_granularity, expected_output", [ @@ -581,5 +592,183 @@ def test_format_datetime(test_name, input_dt, datetimeformat, datetimeformat_gra assert expected_output == output_date +def test_step_but_no_cursor_granularity(): + with pytest.raises(ValueError): + DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000", parameters={}), + end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000", parameters={}), + step="P1D", + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + + +def test_cursor_granularity_but_no_step(): + with pytest.raises(ValueError): + DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000", parameters={}), + end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000", parameters={}), + cursor_granularity="P1D", + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + + +def test_given_multiple_cursor_datetime_format_then_slice_using_first_format(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01", parameters={}), + end_datetime=MinMaxDatetime("2023-01-10", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + cursor_datetime_formats=["%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"], + config=config, + parameters={}, + ) + stream_slices = cursor.stream_slices() + assert stream_slices == [{"start_time": "2021-01-01", "end_time": "2023-01-10"}] + + +def test_no_cursor_granularity_and_no_step_then_only_return_one_slice(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01", parameters={}), + end_datetime=MinMaxDatetime("2023-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + stream_slices = cursor.stream_slices() + assert stream_slices == [{"start_time": "2021-01-01", "end_time": "2023-01-01"}] + + +def test_no_end_datetime(mock_datetime_now): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + stream_slices = cursor.stream_slices() + assert stream_slices == [{"start_time": "2021-01-01", "end_time": FAKE_NOW.strftime("%Y-%m-%d")}] + + +def test_given_no_state_and_start_before_cursor_value_when_should_be_synced_then_return_true(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert cursor.should_be_synced(Record({cursor_field: "2022-01-01"}, ANY_SLICE)) + + +def test_given_no_state_and_start_after_cursor_value_when_should_be_synced_then_return_false(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2022-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert not cursor.should_be_synced(Record({cursor_field: "2021-01-01"}, ANY_SLICE)) + + +def test_given_state_earliest_to_start_datetime_when_should_be_synced_then_use_state_as_earliest_boundary(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2021-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + cursor.set_initial_state({cursor_field: "2023-01-01"}) + assert not cursor.should_be_synced(Record({cursor_field: "2022-01-01"}, ANY_SLICE)) + + +def test_given_start_datetime_earliest_to_state_when_should_be_synced_then_use_start_datetime_as_earliest_boundary(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2023-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + cursor.set_initial_state({cursor_field: "2021-01-01"}) + assert not cursor.should_be_synced(Record({cursor_field: "2022-01-01"}, ANY_SLICE)) + + +def test_given_end_datetime_before_cursor_value_when_should_be_synced_then_return_false(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("2023-01-01", parameters={}), + end_datetime=MinMaxDatetime("2025-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert not cursor.should_be_synced(Record({cursor_field: "2030-01-01"}, ANY_SLICE)) + + +def test_given_record_without_cursor_value_when_should_be_synced_then_return_true(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("3000-01-01", parameters={}), + cursor_field=InterpolatedString(cursor_field, parameters={}), + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert cursor.should_be_synced(Record({"record without cursor value": "any"}, ANY_SLICE)) + + +def test_given_first_greater_than_second_then_return_true(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("3000-01-01", parameters={}), + cursor_field="cursor_field", + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert cursor.is_greater_than_or_equal(Record({"cursor_field": "2023-01-01"}, {}), Record({"cursor_field": "2021-01-01"}, {})) + + +def test_given_first_lesser_than_second_then_return_false(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("3000-01-01", parameters={}), + cursor_field="cursor_field", + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert not cursor.is_greater_than_or_equal(Record({"cursor_field": "2021-01-01"}, {}), Record({"cursor_field": "2023-01-01"}, {})) + + +def test_given_no_cursor_value_for_second_than_second_then_return_true(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("3000-01-01", parameters={}), + cursor_field="cursor_field", + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert cursor.is_greater_than_or_equal(Record({"cursor_field": "2021-01-01"}, {}), Record({}, {})) + + +def test_given_no_cursor_value_for_first_than_second_then_return_false(): + cursor = DatetimeBasedCursor( + start_datetime=MinMaxDatetime("3000-01-01", parameters={}), + cursor_field="cursor_field", + datetime_format="%Y-%m-%d", + config=config, + parameters={}, + ) + assert not cursor.is_greater_than_or_equal(Record({}, {}), Record({"cursor_field": "2021-01-01"}, {})) + + if __name__ == "__main__": unittest.main() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py new file mode 100644 index 000000000000..2f1b8eccba18 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py @@ -0,0 +1,339 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from collections import OrderedDict +from unittest.mock import Mock + +import pytest +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor +from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import ( + PerPartitionCursor, + PerPartitionKeySerializer, + PerPartitionStreamSlice, +) +from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer +from airbyte_cdk.sources.declarative.types import Record + +PARTITION = { + "partition_key string": "partition value", + "partition_key int": 1, + "partition_key list str": ["list item 1", "list item 2"], + "partition_key list dict": [ + { + "dict within list key 1-1": "dict within list value 1-1", + "dict within list key 1-2": "dict within list value 1-2" + }, + {"dict within list key 2": "dict within list value 2"}, + ], + "partition_key nested dict": { + "nested_partition_key 1": "a nested value", + "nested_partition_key 2": "another nested value", + }, +} + +CURSOR_SLICE_FIELD = "cursor slice field" +CURSOR_STATE_KEY = "cursor state" +CURSOR_STATE = {CURSOR_STATE_KEY: "a state value"} +NOT_CONSIDERED_BECAUSE_MOCKED_CURSOR_HAS_NO_STATE = "any" +STATE = { + "states": [ + { + "partition": { + "partition_router_field_1": "X1", + "partition_router_field_2": "Y1", + }, + "cursor": { + "cursor state field": 1 + } + }, + { + "partition": { + "partition_router_field_1": "X2", + "partition_router_field_2": "Y2", + }, + "cursor": { + "cursor state field": 2 + } + }, + ] +} + + +def test_partition_serialization(): + serializer = PerPartitionKeySerializer() + assert serializer.to_partition(serializer.to_partition_key(PARTITION)) == PARTITION + + +def test_partition_with_different_key_orders(): + ordered_dict = OrderedDict({"1": 1, "2": 2}) + same_dict_with_different_order = OrderedDict({"2": 2, "1": 1}) + serializer = PerPartitionKeySerializer() + + assert serializer.to_partition_key(ordered_dict) == serializer.to_partition_key(same_dict_with_different_order) + + +def test_given_tuples_in_json_then_deserialization_convert_to_list(): + """ + This is a known issue with the current implementation. However, the assumption is that this wouldn't be a problem as we only use the + immutability and we expect stream slices to be immutable anyway + """ + serializer = PerPartitionKeySerializer() + partition_with_tuple = {"key": (1, 2, 3)} + + assert partition_with_tuple != serializer.to_partition(serializer.to_partition_key(partition_with_tuple)) + + +def test_stream_slice_merge_dictionaries(): + stream_slice = PerPartitionStreamSlice({"partition key": "partition value"}, {"cursor key": "cursor value"}) + assert stream_slice == {"partition key": "partition value", "cursor key": "cursor value"} + + +def test_overlapping_slice_keys_raise_error(): + with pytest.raises(ValueError): + PerPartitionStreamSlice({"overlapping key": "partition value"}, {"overlapping key": "cursor value"}) + + +class MockedCursorBuilder: + def __init__(self): + self._stream_slices = [] + self._stream_state = {} + + def with_stream_slices(self, stream_slices): + self._stream_slices = stream_slices + return self + + def with_stream_state(self, stream_state): + self._stream_state = stream_state + return self + + def build(self): + cursor = Mock(spec=Cursor) + cursor.get_stream_state.return_value = self._stream_state + cursor.stream_slices.return_value = self._stream_slices + return cursor + + +@pytest.fixture() +def mocked_partition_router(): + return Mock(spec=StreamSlicer) + + +@pytest.fixture() +def mocked_cursor_factory(): + cursor_factory = Mock() + cursor_factory.create.return_value = MockedCursorBuilder().build() + return cursor_factory + + +def test_given_no_partition_when_stream_slices_then_no_slices(mocked_cursor_factory, mocked_partition_router): + mocked_partition_router.stream_slices.return_value = [] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + + slices = cursor.stream_slices() + + assert not next(slices, None) + + +def test_given_partition_router_without_state_has_one_partition_then_return_one_slice_per_cursor_slice(mocked_cursor_factory, + mocked_partition_router): + partition = {"partition_field_1": "a value", "partition_field_2": "another value"} + mocked_partition_router.stream_slices.return_value = [partition] + cursor_slices = [{"start_datetime": 1}, {"start_datetime": 2}] + mocked_cursor_factory.create.return_value = MockedCursorBuilder().with_stream_slices(cursor_slices).build() + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + + slices = cursor.stream_slices() + + assert list(slices) == [PerPartitionStreamSlice(partition, cursor_slice) for cursor_slice in cursor_slices] + + +def test_given_partition_associated_with_state_when_stream_slices_then_do_not_recreate_cursor(mocked_cursor_factory, + mocked_partition_router): + partition = {"partition_field_1": "a value", "partition_field_2": "another value"} + mocked_partition_router.stream_slices.return_value = [partition] + cursor_slices = [{"start_datetime": 1}] + mocked_cursor_factory.create.return_value = MockedCursorBuilder().with_stream_slices(cursor_slices).build() + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + + cursor.set_initial_state({ + "states": [{ + "partition": partition, + "cursor": CURSOR_STATE + }] + }) + mocked_cursor_factory.create.assert_called_once() + slices = list(cursor.stream_slices()) + + mocked_cursor_factory.create.assert_called_once() + assert len(slices) == 1 + + +def test_given_multiple_partitions_then_each_have_their_state(mocked_cursor_factory, mocked_partition_router): + first_partition = {"first_partition_key": "first_partition_value"} + mocked_partition_router.stream_slices.return_value = [ + first_partition, + {"second_partition_key": "second_partition_value"} + ] + first_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "first slice cursor value"}]).build() + second_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "second slice cursor value"}]).build() + mocked_cursor_factory.create.side_effect = [first_cursor, second_cursor] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + + cursor.set_initial_state({ + "states": [{ + "partition": first_partition, + "cursor": CURSOR_STATE + }] + }) + slices = list(cursor.stream_slices()) + + first_cursor.stream_slices.assert_called_once() + second_cursor.stream_slices.assert_called_once() + assert slices == [ + PerPartitionStreamSlice( + partition={"first_partition_key": "first_partition_value"}, + cursor_slice={CURSOR_SLICE_FIELD: "first slice cursor value"} + ), + PerPartitionStreamSlice( + partition={"second_partition_key": "second_partition_value"}, + cursor_slice={CURSOR_SLICE_FIELD: "second slice cursor value"} + ), + ] + + +def test_given_stream_slices_when_get_stream_state_then_return_updated_state(mocked_cursor_factory, mocked_partition_router): + mocked_cursor_factory.create.side_effect = [ + MockedCursorBuilder().with_stream_state({CURSOR_STATE_KEY: "first slice cursor value"}).build(), + MockedCursorBuilder().with_stream_state({CURSOR_STATE_KEY: "second slice cursor value"}).build() + ] + mocked_partition_router.stream_slices.return_value = [{"partition key": "first partition"}, {"partition key": "second partition"}] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + list(cursor.stream_slices()) + assert cursor.get_stream_state() == { + "states": [ + { + "partition": {"partition key": "first partition"}, + "cursor": {CURSOR_STATE_KEY: "first slice cursor value"} + }, + { + "partition": {"partition key": "second partition"}, + "cursor": {CURSOR_STATE_KEY: "second slice cursor value"} + } + ] + } + + +def test_when_get_stream_state_then_delegate_to_underlying_cursor(mocked_cursor_factory, mocked_partition_router): + underlying_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "first slice cursor value"}]).build() + mocked_cursor_factory.create.side_effect = [underlying_cursor] + mocked_partition_router.stream_slices.return_value = [{"partition key": "first partition"}] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + first_slice = list(cursor.stream_slices())[0] + + cursor.should_be_synced( + Record( + {}, + first_slice + ) + ) + + underlying_cursor.should_be_synced.assert_called_once_with( + Record( + {}, + first_slice.cursor_slice + ) + ) + + +def test_close_slice(mocked_cursor_factory, mocked_partition_router): + underlying_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "first slice cursor value"}]).build() + mocked_cursor_factory.create.side_effect = [underlying_cursor] + stream_slice = PerPartitionStreamSlice(partition={"partition key": "first partition"}, cursor_slice={}) + mocked_partition_router.stream_slices.return_value = [stream_slice.partition] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + last_record = Mock() + list(cursor.stream_slices()) # generate internal state + + cursor.close_slice(stream_slice, last_record) + + underlying_cursor.close_slice.assert_called_once_with(stream_slice.cursor_slice, Record(last_record.data, stream_slice.cursor_slice)) + + +def test_given_no_last_record_when_close_slice_then_do_not_raise_error(mocked_cursor_factory, mocked_partition_router): + underlying_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "first slice cursor value"}]).build() + mocked_cursor_factory.create.side_effect = [underlying_cursor] + stream_slice = PerPartitionStreamSlice(partition={"partition key": "first partition"}, cursor_slice={}) + mocked_partition_router.stream_slices.return_value = [stream_slice.partition] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + list(cursor.stream_slices()) # generate internal state + + cursor.close_slice(stream_slice, None) + + underlying_cursor.close_slice.assert_called_once_with(stream_slice.cursor_slice, None) + + +def test_given_unknown_partition_when_close_slice_then_raise_error(): + any_cursor_factory = Mock() + any_partition_router = Mock() + cursor = PerPartitionCursor(any_cursor_factory, any_partition_router) + stream_slice = PerPartitionStreamSlice(partition={"unknown_partition": "unknown"}, cursor_slice={}) + with pytest.raises(ValueError): + cursor.close_slice( + stream_slice, + Record({}, stream_slice) + ) + + +def test_given_unknown_partition_when_should_be_synced_then_raise_error(): + any_cursor_factory = Mock() + any_partition_router = Mock() + cursor = PerPartitionCursor(any_cursor_factory, any_partition_router) + with pytest.raises(ValueError): + cursor.should_be_synced( + Record( + {}, + PerPartitionStreamSlice( + partition={"unknown_partition": "unknown"}, + cursor_slice={} + ) + ) + ) + + +def test_given_records_with_different_slice_when_is_greater_than_or_equal_then_raise_error(): + any_cursor_factory = Mock() + any_partition_router = Mock() + cursor = PerPartitionCursor(any_cursor_factory, any_partition_router) + with pytest.raises(ValueError): + cursor.is_greater_than_or_equal( + Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})), + Record({}, PerPartitionStreamSlice(partition={"another slice": "value"}, cursor_slice={})) + ) + + +def test_given_slice_is_unknown_when_is_greater_than_or_equal_then_raise_error(): + any_cursor_factory = Mock() + any_partition_router = Mock() + cursor = PerPartitionCursor(any_cursor_factory, any_partition_router) + with pytest.raises(ValueError): + cursor.is_greater_than_or_equal( + Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})), + Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})) + ) + + +def test_when_is_greater_than_or_equal_then_return_underlying_cursor_response(mocked_cursor_factory, mocked_partition_router): + underlying_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "first slice cursor value"}]).build() + mocked_cursor_factory.create.side_effect = [underlying_cursor] + stream_slice = PerPartitionStreamSlice(partition={"partition key": "first partition"}, cursor_slice={}) + mocked_partition_router.stream_slices.return_value = [stream_slice.partition] + cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) + first_record = Record({"first": "value"}, stream_slice) + second_record = Record({"second": "value"}, stream_slice) + list(cursor.stream_slices()) # generate internal state + + result = cursor.is_greater_than_or_equal(first_record, second_record) + + assert result == underlying_cursor.is_greater_than_or_equal.return_value + underlying_cursor.is_greater_than_or_equal.assert_called_once_with(first_record, second_record) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py new file mode 100644 index 000000000000..0dd19c66fc3c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py @@ -0,0 +1,165 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import PerPartitionStreamSlice +from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource +from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever +from airbyte_cdk.sources.declarative.types import Record + +CURSOR_FIELD = "cursor_field" +SYNC_MODE = SyncMode.incremental + + +class ManifestBuilder: + def __init__(self): + self._incremental_sync = None + self._partition_router = None + + def with_list_partition_router(self, cursor_field, partitions): + self._partition_router = { + "type": "ListPartitionRouter", + "cursor_field": cursor_field, + "values": partitions, + } + return self + + def with_incremental_sync(self, start_datetime, end_datetime, datetime_format, cursor_field, step, cursor_granularity): + self._incremental_sync = { + "type": "DatetimeBasedCursor", + "start_datetime": start_datetime, + "end_datetime": end_datetime, + "datetime_format": datetime_format, + "cursor_field": cursor_field, + "step": step, + "cursor_granularity": cursor_granularity + } + return self + + def build(self): + manifest = { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": { + "type": "CheckStream", + "stream_names": [ + "Rates" + ] + }, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Rates", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "properties": {}, + "type": "object" + } + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.apilayer.com", + "path": "/exchangerates_data/latest", + "http_method": "GET", + }, + "record_selector": { + "type": "RecordSelector", + "extractor": { + "type": "DpathExtractor", + "field_path": [] + } + }, + } + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": [], + "properties": {}, + "additionalProperties": True + }, + "documentation_url": "https://example.org", + "type": "Spec" + } + } + if self._incremental_sync: + manifest["streams"][0]["incremental_sync"] = self._incremental_sync + if self._partition_router: + manifest["streams"][0]["retriever"]["partition_router"] = self._partition_router + return manifest + + +def test_given_state_for_only_some_partition_when_stream_slices_then_create_slices_using_state_or_start_from_start_datetime(): + source = ManifestDeclarativeSource( + source_config=ManifestBuilder().with_list_partition_router("partition_field", ["1", "2"]).with_incremental_sync( + start_datetime="2022-01-01", + end_datetime="2022-02-28", + datetime_format="%Y-%m-%d", + cursor_field=CURSOR_FIELD, + step="P1M", + cursor_granularity="P1D", + ).build() + ) + stream_instance = source.streams({})[0] + stream_instance.state = { + "states": [ + { + "partition": {"partition_field": "1"}, + "cursor": {CURSOR_FIELD: "2022-02-01"}, + } + ] + } + + slices = stream_instance.stream_slices( + sync_mode=SYNC_MODE, + stream_state={}, + ) + + assert list(slices) == [ + {"partition_field": "1", "start_time": "2022-02-01", "end_time": "2022-02-28"}, + {"partition_field": "2", "start_time": "2022-01-01", "end_time": "2022-01-31"}, + {"partition_field": "2", "start_time": "2022-02-01", "end_time": "2022-02-28"}, + ] + + +def test_given_record_for_partition_when_read_then_update_state(): + source = ManifestDeclarativeSource( + source_config=ManifestBuilder().with_list_partition_router("partition_field", ["1", "2"]).with_incremental_sync( + start_datetime="2022-01-01", + end_datetime="2022-02-28", + datetime_format="%Y-%m-%d", + cursor_field=CURSOR_FIELD, + step="P1M", + cursor_granularity="P1D", + ).build() + ) + stream_instance = source.streams({})[0] + list(stream_instance.stream_slices(sync_mode=SYNC_MODE)) + + stream_slice = PerPartitionStreamSlice({"partition_field": "1"}, {"start_time": "2022-01-01", "end_time": "2022-01-31"}) + with patch.object(SimpleRetriever, "_read_pages", side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]]): + list( + stream_instance.read_records( + sync_mode=SYNC_MODE, + stream_slice=stream_slice, + stream_state={"states": []}, + cursor_field=CURSOR_FIELD, + ) + ) + + assert stream_instance.state == {"states": [ + { + "partition": {"partition_field": "1"}, + "cursor": {CURSOR_FIELD: "2022-01-31"}, + } + ]} diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py index 7541d5fd3344..34105f99f497 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py @@ -190,3 +190,30 @@ def test_undeclared_variables(template_string, expected_error, expected_value): else: actual_value = interpolation.eval(template_string, config=config, **{"to_be": "that_is_the_question"}) assert actual_value == expected_value + + +@freeze_time("2021-09-01") +@pytest.mark.parametrize("template_string, expected_value",[ + pytest.param("{{ now_utc() }}", "2021-09-01 00:00:00+00:00", id="test_now_utc"), + pytest.param("{{ now_utc().strftime('%Y-%m-%d') }}", "2021-09-01", id="test_now_utc_strftime"), + pytest.param("{{ today_utc() }}", "2021-09-01", id="test_today_utc"), + pytest.param("{{ today_utc().strftime('%Y/%m/%d') }}", "2021/09/01", id="test_todat_utc_stftime"), + pytest.param("{{ timestamp(1646006400) }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28') }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28T00:00:00Z') }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28 00:00:00Z') }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28T00:00:00-08:00') }}", 1646035200, id="test_timestamp_from_date_with_tz"), + pytest.param("{{ max(2, 3) }}", 3, id="test_max_with_arguments"), + pytest.param("{{ max([2, 3]) }}", 3, id="test_max_with_list"), + pytest.param("{{ day_delta(1) }}", "2021-09-02T00:00:00.000000+0000", id="test_day_delta"), + pytest.param("{{ day_delta(-1) }}", "2021-08-31T00:00:00.000000+0000", id="test_day_delta_negative"), + pytest.param("{{ day_delta(1, format='%Y-%m-%d') }}", "2021-09-02", id="test_day_delta_with_format"), + pytest.param("{{ duration('P1D') }}", "1 day, 0:00:00", id="test_duration_one_day"), + pytest.param("{{ duration('P6DT23H') }}", "6 days, 23:00:00", id="test_duration_six_days_and_23_hours"), + pytest.param("{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}", "2021-08-31T00:00:00Z", id="test_now_utc_with_duration_and_format"), +]) +def test_macros_examples(template_string, expected_value): + # The outputs of this test are referenced in declarative_component_schema.yaml + # If you change the expected output, you must also change the expected output in declarative_component_schema.yaml + now_utc = interpolation.eval(template_string, {}) + assert now_utc == expected_value diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py index 4fcf4b4da00f..0a19c9ab4f86 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py @@ -28,14 +28,14 @@ def test_macros_export(test_name, fn_name, found_in_macros): @pytest.mark.parametrize("test_name, input_value, format, expected_output", [ - ("test_datetime_string_to_date", "2022-01-01T01:01:01Z", "%Y-%m-%d","2022-01-01"), + ("test_datetime_string_to_date", "2022-01-01T01:01:01Z", "%Y-%m-%d", "2022-01-01"), ("test_date_string_to_date", "2022-01-01", "%Y-%m-%d", "2022-01-01"), ("test_datetime_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), ("test_datetime_with_tz_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), ("test_datetime_string_to_datetime", "2022-01-01T01:01:01Z", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), ("test_datetime_string_with_tz_to_datetime", "2022-01-01T01:01:01-0800", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T09:01:01Z"), - ("test_datetime_object_tz_to_date", datetime.datetime(2022,1,1,1,1,1), "%Y-%m-%d", "2022-01-01"), - ("test_datetime_object_tz_to_datetime", datetime.datetime(2022,1,1,1,1,1), "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), + ("test_datetime_object_tz_to_date", datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%d", "2022-01-01"), + ("test_datetime_object_tz_to_datetime", datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), ]) def test_format_datetime(test_name, input_value, format, expected_output): format_datetime = macros["format_datetime"] @@ -54,7 +54,7 @@ def test_duration(test_name, input_value, expected_output): @pytest.mark.parametrize( - "test_name, input_value, expected_output",[ + "test_name, input_value, expected_output", [ ("test_int_input", 1646006400, 1646006400), ("test_float_input", 100.0, 100), ("test_float_input_is_floored", 100.9, 100), diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index c268b75557c3..7d9fb6bbd0ac 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -2,17 +2,26 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +# mypy: ignore-errors + import datetime import pytest +from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator, SessionTokenAuthenticator +from airbyte_cdk.sources.declarative.auth.token import ( + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + LegacySessionTokenAuthenticator, +) +from airbyte_cdk.sources.declarative.auth.token_provider import SessionTokenProvider from airbyte_cdk.sources.declarative.checks import CheckStream from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders import JsonDecoder from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.models import CheckStream as CheckStreamModel from airbyte_cdk.sources.declarative.models import CompositeErrorHandler as CompositeErrorHandlerModel @@ -42,7 +51,12 @@ ) from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator -from airbyte_cdk.sources.declarative.requesters.paginators.strategies import CursorPaginationStrategy, OffsetIncrement, PageIncrement +from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( + CursorPaginationStrategy, + OffsetIncrement, + PageIncrement, + StopConditionPaginationStrategyDecorator, +) from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath @@ -54,6 +68,7 @@ from airbyte_cdk.sources.declarative.transformations import AddFields, RemoveFields from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent factory = ModelToComponentFactory() @@ -188,13 +203,13 @@ def test_full_config_stream(): assert isinstance(stream, DeclarativeStream) assert stream.primary_key == "id" assert stream.name == "lists" - assert stream.stream_cursor_field.string == "created" + assert stream._stream_cursor_field.string == "created" assert isinstance(stream.schema_loader, JsonFileSchemaLoader) assert stream.schema_loader._get_json_filepath() == "./source_sendgrid/schemas/lists.json" - assert len(stream.transformations) == 1 - add_fields = stream.transformations[0] + assert len(stream.retriever.record_selector.transformations) == 1 + add_fields = stream.retriever.record_selector.transformations[0] assert isinstance(add_fields, AddFields) assert add_fields.fields[0].path == ["extra"] assert add_fields.fields[0].value.string == "{{ response.to_add }}" @@ -227,13 +242,13 @@ def test_full_config_stream(): assert stream.retriever.paginator.pagination_strategy.page_size == 10 assert isinstance(stream.retriever.requester, HttpRequester) - assert stream.retriever.requester.http_method == HttpMethod.GET + assert stream.retriever.requester._http_method == HttpMethod.GET assert stream.retriever.requester.name == stream.name - assert stream.retriever.requester.path.string == "{{ next_page_token['next_page_url'] }}" - assert stream.retriever.requester.path.default == "{{ next_page_token['next_page_url'] }}" + assert stream.retriever.requester._path.string == "{{ next_page_token['next_page_url'] }}" + assert stream.retriever.requester._path.default == "{{ next_page_token['next_page_url'] }}" assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator) - assert stream.retriever.requester.authenticator._token.eval(input_config) == "verysecrettoken" + assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken" assert isinstance(stream.retriever.requester.request_options_provider, InterpolatedRequestOptionsProvider) assert stream.retriever.requester.request_options_provider.request_parameters.get("unit") == "day" @@ -293,6 +308,45 @@ def test_interpolate_config(): assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} +def test_single_use_oauth_branch(): + single_use_input_config = {"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"], "credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"}} + + content = """ + authenticator: + type: OAuthAuthenticator + client_id: "some_client_id" + client_secret: "some_client_secret" + token_refresh_endpoint: "https://api.sendgrid.com/v3/auth" + refresh_token: "{{ config['apikey'] }}" + refresh_request_body: + body_field: "yoyoyo" + interpolated_body_field: "{{ config['apikey'] }}" + refresh_token_updater: + refresh_token_name: "the_refresh_token" + refresh_token_config_path: + - apikey + """ + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {}) + + authenticator: SingleUseRefreshTokenOauth2Authenticator = factory.create_component( + model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=single_use_input_config + ) + + assert isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator) + assert authenticator._client_id == "some_client_id" + assert authenticator._client_secret == "some_client_secret" + assert authenticator._token_refresh_endpoint == "https://api.sendgrid.com/v3/auth" + assert authenticator._refresh_token == "verysecrettoken" + assert authenticator._refresh_request_body == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} + assert authenticator._refresh_token_name == "the_refresh_token" + assert authenticator._refresh_token_config_path == ["apikey"] + # default values + assert authenticator._access_token_config_path == ["credentials", "access_token"] + assert authenticator._token_expiry_date_config_path == ["credentials", "token_expiry_date"] + + def test_list_based_stream_slicer_with_values_refd(): content = """ repositories: ["airbyte", "airbyte-cloud"] @@ -545,10 +599,9 @@ def test_stream_with_incremental_and_retriever_with_partition_router(): assert isinstance(stream, DeclarativeStream) assert isinstance(stream.retriever, SimpleRetriever) - assert isinstance(stream.retriever.stream_slicer, CartesianProductStreamSlicer) - assert len(stream.retriever.stream_slicer.stream_slicers) == 2 + assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor) - datetime_stream_slicer = stream.retriever.stream_slicer.stream_slicers[0] + datetime_stream_slicer = stream.retriever.stream_slicer._cursor_factory.create() assert isinstance(datetime_stream_slicer, DatetimeBasedCursor) assert isinstance(datetime_stream_slicer.start_datetime, MinMaxDatetime) assert datetime_stream_slicer.start_datetime.datetime.string == "{{ config['start_time'] }}" @@ -557,12 +610,86 @@ def test_stream_with_incremental_and_retriever_with_partition_router(): assert datetime_stream_slicer.step == "P10D" assert datetime_stream_slicer.cursor_field.string == "created" - list_stream_slicer = stream.retriever.stream_slicer.stream_slicers[1] + list_stream_slicer = stream.retriever.stream_slicer._partition_router assert isinstance(list_stream_slicer, ListPartitionRouter) assert list_stream_slicer.values == ["airbyte", "airbyte-cloud"] assert list_stream_slicer.cursor_field.string == "a_key" +def test_incremental_data_feed(): + content = """ +selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["extractor_path"] + record_filter: + type: RecordFilter + condition: "{{ record['id'] > stream_state['id'] }}" +requester: + type: HttpRequester + name: "{{ parameters['name'] }}" + url_base: "https://api.sendgrid.com/v3/" + http_method: "GET" +list_stream: + type: DeclarativeStream + incremental_sync: + type: DatetimeBasedCursor + $parameters: + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: "{{ config['start_time'] }}" + cursor_field: "created" + is_data_feed: true + retriever: + type: SimpleRetriever + name: "{{ parameters['name'] }}" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response._metadata.next }}" + page_size: 10 + requester: + $ref: "#/requester" + path: "/" + record_selector: + $ref: "#/selector" + $parameters: + name: "lists" + """ + + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["list_stream"], {}) + + stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config) + + assert isinstance(stream.retriever.paginator.pagination_strategy, StopConditionPaginationStrategyDecorator) + + +def test_given_data_feed_and_incremental_then_raise_error(): + content = """ +incremental_sync: + type: DatetimeBasedCursor + $parameters: + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: "{{ config['start_time'] }}" + end_datetime: "2023-01-01" + cursor_field: "created" + is_data_feed: true""" + + parsed_incremental_sync = YamlDeclarativeSource._parse(content) + resolved_incremental_sync = resolver.preprocess_manifest(parsed_incremental_sync) + datetime_based_cursor_definition = transformer.propagate_types_and_parameters("", resolved_incremental_sync["incremental_sync"], {}) + + with pytest.raises(ValueError): + factory.create_component( + model_type=DatetimeBasedCursorModel, + component_definition=datetime_based_cursor_definition, + config=input_config + ) + + @pytest.mark.parametrize( "test_name, record_selector, expected_runtime_selector", [("test_static_record_selector", "result", "result"), ("test_options_record_selector", "{{ parameters['name'] }}", "lists")], @@ -586,7 +713,7 @@ def test_create_record_selector(test_name, record_selector, expected_runtime_sel resolved_manifest = resolver.preprocess_manifest(parsed_manifest) selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {}) - selector = factory.create_component(model_type=RecordSelectorModel, component_definition=selector_manifest, config=input_config) + selector = factory.create_component(model_type=RecordSelectorModel, component_definition=selector_manifest, transformations=[], config=input_config) assert isinstance(selector, RecordSelector) assert isinstance(selector.extractor, DpathExtractor) @@ -669,10 +796,10 @@ def test_create_requester(test_name, error_handler, expected_backoff_strategy_ty ) assert isinstance(selector, HttpRequester) - assert selector._method == HttpMethod.GET + assert selector._http_method == HttpMethod.GET assert selector.name == "name" - assert selector.path.string == "/v3/marketing/lists" - assert selector.url_base.string == "https://api.sendgrid.com" + assert selector._path.string == "/v3/marketing/lists" + assert selector._url_base.string == "https://api.sendgrid.com" assert isinstance(selector.error_handler, DefaultErrorHandler) assert len(selector.error_handler.backoff_strategies) == 1 @@ -687,7 +814,7 @@ def test_create_requester(test_name, error_handler, expected_backoff_strategy_ty assert selector._request_options_provider._headers_interpolator._interpolator.mapping["header"] == "header_value" -def test_create_request_with_session_authenticator(): +def test_create_request_with_leacy_session_authenticator(): content = """ requester: type: HttpRequester @@ -696,7 +823,7 @@ def test_create_request_with_session_authenticator(): name: 'lists' url_base: "https://api.sendgrid.com" authenticator: - type: "SessionTokenAuthenticator" + type: "LegacySessionTokenAuthenticator" username: "{{ parameters.name}}" password: "{{ config.apikey }}" login_url: "login" @@ -718,12 +845,62 @@ def test_create_request_with_session_authenticator(): ) assert isinstance(selector, HttpRequester) - assert isinstance(selector.authenticator, SessionTokenAuthenticator) + assert isinstance(selector.authenticator, LegacySessionTokenAuthenticator) assert selector.authenticator._username.eval(input_config) == "lists" assert selector.authenticator._password.eval(input_config) == "verysecrettoken" assert selector.authenticator._api_url.eval(input_config) == "https://api.sendgrid.com" +def test_create_request_with_session_authenticator(): + content = """ +requester: + type: HttpRequester + path: "/v3/marketing/lists" + $parameters: + name: 'lists' + url_base: "https://api.sendgrid.com" + authenticator: + type: SessionTokenAuthenticator + expiration_duration: P10D + login_requester: + path: /session + type: HttpRequester + url_base: 'https://api.sendgrid.com' + http_method: POST + request_body_json: + password: '{{ config.apikey }}' + username: '{{ parameters.name }}' + session_token_path: + - id + request_authentication: + type: ApiKey + inject_into: + type: RequestOption + field_name: X-Metabase-Session + inject_into: header + request_parameters: + a_parameter: "something_here" + request_headers: + header: header_value + """ + name = "name" + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {}) + + selector = factory.create_component( + model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name + ) + + assert isinstance(selector.authenticator, ApiKeyAuthenticator) + assert isinstance(selector.authenticator.token_provider, SessionTokenProvider) + assert selector.authenticator.token_provider.session_token_path == ["id"] + assert isinstance(selector.authenticator.token_provider.login_requester, HttpRequester) + assert selector.authenticator.token_provider.session_token_path == ["id"] + assert selector.authenticator.token_provider.login_requester._url_base.eval(input_config) == "https://api.sendgrid.com" + assert selector.authenticator.token_provider.login_requester.get_request_body_json() == {"username": "lists", "password": "verysecrettoken"} + + def test_create_composite_error_handler(): content = """ error_handler: @@ -818,10 +995,10 @@ def test_config_with_defaults(): assert stream.schema_loader.file_path.default == "./source_sendgrid/schemas/{{ parameters.name }}.yaml" assert isinstance(stream.retriever.requester, HttpRequester) - assert stream.retriever.requester.http_method == HttpMethod.GET + assert stream.retriever.requester._http_method == HttpMethod.GET assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator) - assert stream.retriever.requester.authenticator._token.eval(input_config) == "verysecrettoken" + assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken" assert isinstance(stream.retriever.record_selector, RecordSelector) assert isinstance(stream.retriever.record_selector.extractor, DpathExtractor) @@ -1127,7 +1304,7 @@ def test_no_transformations(self): stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config) assert isinstance(stream, DeclarativeStream) - assert [] == stream.transformations + assert [] == stream.retriever.record_selector.transformations def test_remove_fields(self): content = f""" @@ -1150,7 +1327,7 @@ def test_remove_fields(self): assert isinstance(stream, DeclarativeStream) expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]], parameters={})] - assert stream.transformations == expected + assert stream.retriever.record_selector.transformations == expected def test_add_fields(self): content = f""" @@ -1184,7 +1361,7 @@ def test_add_fields(self): parameters={}, ) ] - assert stream.transformations == expected + assert stream.retriever.record_selector.transformations == expected def test_default_schema_loader(self): component_definition = { @@ -1220,7 +1397,7 @@ def test_default_schema_loader(self): @pytest.mark.parametrize( - "incremental, partition_router, expected_type, expected_slicer_count", + "incremental, partition_router, expected_type", [ pytest.param( { @@ -1234,7 +1411,6 @@ def test_default_schema_loader(self): }, None, DatetimeBasedCursor, - 1, id="test_create_simple_retriever_with_incremental", ), pytest.param( @@ -1245,7 +1421,6 @@ def test_default_schema_loader(self): "cursor_field": "a_key", }, ListPartitionRouter, - 1, id="test_create_simple_retriever_with_partition_router", ), pytest.param( @@ -1263,8 +1438,7 @@ def test_default_schema_loader(self): "values": "{{config['repos']}}", "cursor_field": "a_key", }, - CartesianProductStreamSlicer, - 2, + PerPartitionCursor, id="test_create_simple_retriever_with_incremental_and_partition_router", ), pytest.param( @@ -1289,14 +1463,13 @@ def test_default_schema_loader(self): "cursor_field": "b_key", }, ], - CartesianProductStreamSlicer, - 2, + PerPartitionCursor, id="test_create_simple_retriever_with_partition_routers_multiple_components", ), - pytest.param(None, None, SinglePartitionRouter, 1, id="test_create_simple_retriever_with_no_incremental_or_partition_router"), + pytest.param(None, None, SinglePartitionRouter, id="test_create_simple_retriever_with_no_incremental_or_partition_router"), ], ) -def test_merge_incremental_and_partition_router(incremental, partition_router, expected_type, expected_slicer_count): +def test_merge_incremental_and_partition_router(incremental, partition_router, expected_type): stream_model = { "type": "DeclarativeStream", "retriever": { @@ -1329,9 +1502,14 @@ def test_merge_incremental_and_partition_router(incremental, partition_router, e assert isinstance(stream.retriever, SimpleRetriever) assert isinstance(stream.retriever.stream_slicer, expected_type) - if expected_slicer_count > 1: - assert isinstance(stream.retriever.stream_slicer, CartesianProductStreamSlicer) - assert len(stream.retriever.stream_slicer.stream_slicers) == expected_slicer_count + if incremental and partition_router: + assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor) + if type(partition_router) == list and len(partition_router) > 1: + assert type(stream.retriever.stream_slicer._partition_router) == CartesianProductStreamSlicer + assert len(stream.retriever.stream_slicer._partition_router.stream_slicers) == len(partition_router) + elif partition_router and type(partition_router) == list and len(partition_router) > 1: + assert isinstance(stream.retriever.stream_slicer, PerPartitionCursor) + assert len(stream.retriever.stream_slicer.stream_slicerS) == len(partition_router) def test_simple_retriever_emit_log_messages(): @@ -1355,32 +1533,24 @@ def test_simple_retriever_emit_log_messages(): name="Test", primary_key="id", stream_slicer=None, + transformations=[], ) assert isinstance(retriever, SimpleRetrieverTestReadDecorator) + assert connector_builder_factory._message_repository._log_level == Level.DEBUG def test_ignore_retry(): requester_model = { - "type": "SimpleRetriever", - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [], - }, - }, - "requester": {"type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api"}, + "type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api", } connector_builder_factory = ModelToComponentFactory(disable_retries=True) - retriever = connector_builder_factory.create_component( - model_type=SimpleRetrieverModel, + requester = connector_builder_factory.create_component( + model_type=HttpRequesterModel, component_definition=requester_model, config={}, name="Test", - primary_key="id", - stream_slicer=None, ) - assert retriever.max_retries == 0 + assert requester.max_retries == 0 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py index 52350e640396..c8c8ef1dd6eb 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py @@ -3,7 +3,6 @@ # import pytest as pytest -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType @@ -37,29 +36,10 @@ ) def test_list_partition_router(test_name, partition_values, cursor_field, expected_slices): slicer = ListPartitionRouter(values=partition_values, cursor_field=cursor_field, config={}, parameters=parameters) - slices = [s for s in slicer.stream_slices(SyncMode.incremental, stream_state=None)] + slices = [s for s in slicer.stream_slices()] assert slices == expected_slices -@pytest.mark.parametrize( - "test_name, stream_slice, last_record, expected_state", - [ - ("test_update_cursor_no_state_no_record", {}, None, {}), - ("test_update_cursor_with_state_no_record", {"owner_resource": "customer"}, None, {"owner_resource": "customer"}), - ("test_update_cursor_value_not_in_list", {"owner_resource": "invalid"}, None, {}), - ], -) -def test_update_cursor(test_name, stream_slice, last_record, expected_state): - slicer = ListPartitionRouter(values=partition_values, cursor_field=cursor_field, config={}, parameters={}) - if expected_state: - slicer.update_cursor(stream_slice, last_record) - updated_state = slicer.get_stream_state() - assert expected_state == updated_state - else: - with pytest.raises(ValueError): - slicer.update_cursor(stream_slice, last_record) - - @pytest.mark.parametrize( "test_name, request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data", [ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_single_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_single_partition_router.py index 8db94e030486..1f9570955038 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_single_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_single_partition_router.py @@ -2,13 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.partition_routers.single_partition_router import SinglePartitionRouter def test(): iterator = SinglePartitionRouter(parameters={}) - stream_slices = iterator.stream_slices(SyncMode.incremental, None) + stream_slices = iterator.stream_slices() next_slice = next(stream_slices) assert next_slice == dict() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py index 7a54cb625a6c..f8c7f0cb332b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py @@ -5,9 +5,10 @@ from typing import Any, Iterable, List, Mapping, Optional, Union import pytest as pytest -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, SyncMode, Type from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig, SubstreamPartitionRouter from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType +from airbyte_cdk.sources.declarative.types import Record from airbyte_cdk.sources.streams.core import Stream parent_records = [{"id": 1, "data": "data1"}, {"id": 2, "data": "data2"}] @@ -160,47 +161,10 @@ def test_substream_slicer(test_name, parent_stream_configs, expected_slices): except ValueError: return partition_router = SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters={}, config={}) - slices = [s for s in partition_router.stream_slices(SyncMode.incremental, stream_state=None)] + slices = [s for s in partition_router.stream_slices()] assert slices == expected_slices -@pytest.mark.parametrize( - "test_name, stream_slice, expected_state", - [ - ("test_update_cursor_no_state_no_record", {}, {}), - ("test_update_cursor_with_state_single_parent", {"first_stream_id": "1234"}, {"first_stream_id": "1234"}), - ("test_update_cursor_with_unknown_state_field", {"unknown_stream_id": "1234"}, {}), - ( - "test_update_cursor_with_state_from_both_parents", - {"first_stream_id": "1234", "second_stream_id": "4567"}, - {"first_stream_id": "1234", "second_stream_id": "4567"}, - ), - ], -) -def test_update_cursor(test_name, stream_slice, expected_state): - parent_stream_name_to_config = [ - ParentStreamConfig( - stream=MockStream(parent_slices, data_first_parent_slice + data_second_parent_slice, "first_stream"), - parent_key="id", - partition_field="first_stream_id", - parameters={}, - config={}, - ), - ParentStreamConfig( - stream=MockStream(second_parent_stream_slice, more_records, "second_stream"), - parent_key="id", - partition_field="second_stream_id", - parameters={}, - config={}, - ), - ] - - partition_router = SubstreamPartitionRouter(parent_stream_configs=parent_stream_name_to_config, parameters={}, config={}) - partition_router.update_cursor(stream_slice, None) - updated_state = partition_router.get_stream_state() - assert expected_state == updated_state - - @pytest.mark.parametrize( "test_name, parent_stream_request_parameters, expected_req_params, expected_headers, expected_body_json, expected_body_data", [ @@ -297,3 +261,43 @@ def test_request_option( assert expected_headers == partition_router.get_request_headers(stream_slice=stream_slice) assert expected_body_json == partition_router.get_request_body_json(stream_slice=stream_slice) assert expected_body_data == partition_router.get_request_body_data(stream_slice=stream_slice) + + +def test_given_record_is_airbyte_message_when_stream_slices_then_use_record_data(): + parent_slice = {} + partition_router = SubstreamPartitionRouter( + parent_stream_configs=[ + ParentStreamConfig( + stream=MockStream([parent_slice], [AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(data={"id": "record value"}, emitted_at=0, stream="stream"))], "first_stream"), + parent_key="id", + partition_field="partition_field", + parameters={}, + config={} + ) + ], + parameters={}, + config={} + ) + + slices = list(partition_router.stream_slices()) + assert slices == [{"partition_field": "record value", "parent_slice": parent_slice}] + + +def test_given_record_is_record_object_when_stream_slices_then_use_record_data(): + parent_slice = {} + partition_router = SubstreamPartitionRouter( + parent_stream_configs=[ + ParentStreamConfig( + stream=MockStream([parent_slice], [Record({"id": "record value"}, {})], "first_stream"), + parent_key="id", + partition_field="partition_field", + parameters={}, + config={} + ) + ], + parameters={}, + config={} + ) + + slices = list(partition_router.stream_slices()) + assert slices == [{"partition_field": "record value", "parent_slice": parent_slice}] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py new file mode 100644 index 000000000000..41d71e75623f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock, call + +from airbyte_cdk.sources.declarative.incremental import Cursor +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.pagination_strategy import PaginationStrategy +from airbyte_cdk.sources.declarative.requesters.paginators.strategies.stop_condition import ( + CursorStopCondition, + PaginationStopCondition, + StopConditionPaginationStrategyDecorator, +) +from airbyte_cdk.sources.declarative.types import Record +from pytest import fixture + +ANY_RECORD = Mock() +NO_RECORDS = [] +ANY_RESPONSE = Mock() + + +@fixture +def mocked_cursor(): + return Mock(spec=Cursor) + + +@fixture +def mocked_pagination_strategy(): + return Mock(spec=PaginationStrategy) + + +@fixture +def mocked_stop_condition(): + return Mock(spec=PaginationStopCondition) + + +def test_given_record_should_be_synced_when_is_met_return_false(mocked_cursor): + mocked_cursor.should_be_synced.return_value = True + assert not CursorStopCondition(mocked_cursor).is_met(ANY_RECORD) + + +def test_given_record_should_not_be_synced_when_is_met_return_true(mocked_cursor): + mocked_cursor.should_be_synced.return_value = False + assert CursorStopCondition(mocked_cursor).is_met(ANY_RECORD) + + +def test_given_stop_condition_is_met_when_next_page_token_then_return_none(mocked_pagination_strategy, mocked_stop_condition): + mocked_stop_condition.is_met.side_effect = [False, True] + first_record = Mock(spec=Record) + last_record = Mock(spec=Record) + + decorator = StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition) + + assert not decorator.next_page_token(ANY_RESPONSE, [first_record, last_record]) + mocked_stop_condition.is_met.assert_has_calls([call(last_record), call(first_record)]) + + +def test_given_last_record_meets_condition_when_next_page_token_then_do_not_check_for_other_records(mocked_pagination_strategy, mocked_stop_condition): + mocked_stop_condition.is_met.return_value = True + last_record = Mock(spec=Record) + + StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition).next_page_token( + ANY_RESPONSE, + [Mock(spec=Record), last_record] + ) + + mocked_stop_condition.is_met.assert_called_once_with(last_record) + + +def test_given_stop_condition_is_not_met_when_next_page_token_then_delegate(mocked_pagination_strategy, mocked_stop_condition): + mocked_stop_condition.is_met.return_value = False + first_record = Mock(spec=Record) + last_record = Mock(spec=Record) + decorator = StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition) + + next_page_token = decorator.next_page_token(ANY_RESPONSE, [first_record, last_record]) + + assert next_page_token == mocked_pagination_strategy.next_page_token.return_value + mocked_pagination_strategy.next_page_token.assert_called_once_with(ANY_RESPONSE, [first_record, last_record]) + mocked_stop_condition.is_met.assert_has_calls([call(last_record), call(first_record)]) + + +def test_given_no_records_when_next_page_token_then_delegate(mocked_pagination_strategy, mocked_stop_condition): + decorator = StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition) + + next_page_token = decorator.next_page_token(ANY_RESPONSE, NO_RECORDS) + + assert next_page_token == mocked_pagination_strategy.next_page_token.return_value + mocked_pagination_strategy.next_page_token.assert_called_once_with(ANY_RESPONSE, NO_RECORDS) + + +def test_when_reset_then_delegate(mocked_pagination_strategy, mocked_stop_condition): + decorator = StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition) + decorator.reset() + mocked_pagination_strategy.reset.assert_called_once_with() + + +def test_when_get_page_size_then_delegate(mocked_pagination_strategy, mocked_stop_condition): + decorator = StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition) + + page_size = decorator.get_page_size() + + assert page_size == mocked_pagination_strategy.get_page_size.return_value + mocked_pagination_strategy.get_page_size.assert_called_once_with() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index f00f7006a3fe..b3a5bc772261 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -2,12 +2,24 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from http import HTTPStatus +from typing import Any, Mapping, Optional +from unittest import mock from unittest.mock import MagicMock +from urllib.parse import parse_qs, urlparse import pytest as pytest import requests +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth +from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator +from airbyte_cdk.sources.declarative.exceptions import ReadException from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler +from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler from airbyte_cdk.sources.declarative.requesters.http_requester import HttpMethod, HttpRequester +from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException +from requests import PreparedRequest def test_http_requester(): @@ -61,7 +73,6 @@ def test_http_requester(): assert requester.get_request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data assert requester.get_request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json assert requester.interpret_response_status(requests.Response()) == response_status - assert {} == requester.request_kwargs(stream_state={}, stream_slice=None, next_page_token=None) @pytest.mark.parametrize( @@ -111,3 +122,481 @@ def test_path(test_name, path, expected_path): parameters={}, ) assert requester.get_path(stream_state={}, stream_slice={}, next_page_token={}) == expected_path + + +def create_requester(url_base: Optional[str] = None, parameters: Optional[Mapping[str, Any]] = {}, config: Optional[Config] = None, path: Optional[str] = None, authenticator: Optional[DeclarativeAuthenticator] = None, error_handler: Optional[ErrorHandler] = None) -> HttpRequester: + requester = HttpRequester( + name="name", + url_base=url_base or "https://example.com", + path=path or "deals", + http_method=HttpMethod.GET, + request_options_provider=None, + authenticator=authenticator, + error_handler=error_handler, + config=config or {}, + parameters=parameters or {}, + ) + requester._session.send = MagicMock() + req = requests.Response() + req.status_code = 200 + requester._session.send.return_value = req + return requester + + +def test_basic_send_request(): + options_provider = MagicMock() + options_provider.get_request_headers.return_value = {"my_header": "my_value"} + requester = create_requester() + requester._request_options_provider = options_provider + requester.send_request() + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.method == "GET" + assert sent_request.url == "https://example.com/deals" + assert sent_request.headers["my_header"] == "my_value" + assert sent_request.body is None + + +@pytest.mark.parametrize( + "provider_data, provider_json, param_data, param_json, authenticator_data, authenticator_json, expected_exception, expected_body", + [ + # merging data params from the three sources + ({"field": "value"}, None, None, None, None, None, None, "field=value"), + ({"field": "value"}, None, {"field2": "value"}, None, None, None, None, "field=value&field2=value"), + ({"field": "value"}, None, {"field2": "value"}, None, {"authfield": "val"}, None, None, "field=value&field2=value&authfield=val"), + ({"field": "value"}, None, {"field": "value"}, None, None, None, ValueError, None), + ({"field": "value"}, None, None, None, {"field": "value"}, None, ValueError, None), + ({"field": "value"}, None, {"field2": "value"}, None, {"field": "value"}, None, ValueError, None), + # merging json params from the three sources + (None, {"field": "value"}, None, None, None, None, None, '{"field": "value"}'), + (None, {"field": "value"}, None, {"field2": "value"}, None, None, None, '{"field": "value", "field2": "value"}'), + (None, {"field": "value"}, None, {"field2": "value"}, None, {"authfield": "val"}, None, '{"field": "value", "field2": "value", "authfield": "val"}'), + (None, {"field": "value"}, None, {"field": "value"}, None, None, ValueError, None), + (None, {"field": "value"}, None, None, None, {"field": "value"}, ValueError, None), + # raise on mixed data and json params + ({"field": "value"}, {"field": "value"}, None, None, None, None, RequestBodyException, None), + ({"field": "value"}, None, None, {"field": "value"}, None, None, RequestBodyException, None), + (None, None, {"field": "value"}, {"field": "value"}, None, None, RequestBodyException, None), + (None, None, None, None, {"field": "value"}, {"field": "value"}, RequestBodyException, None), + ({"field": "value"}, None, None, None, None, {"field": "value"}, RequestBodyException, None), + + ]) +def test_send_request_data_json(provider_data, provider_json, param_data, param_json, authenticator_data, authenticator_json, expected_exception, expected_body): + options_provider = MagicMock() + options_provider.get_request_body_data.return_value = provider_data + options_provider.get_request_body_json.return_value = provider_json + authenticator = MagicMock() + authenticator.get_request_body_data.return_value = authenticator_data + authenticator.get_request_body_json.return_value = authenticator_json + requester = create_requester(authenticator=authenticator) + requester._request_options_provider = options_provider + if expected_exception is not None: + with pytest.raises(expected_exception): + requester.send_request(request_body_data=param_data, request_body_json=param_json) + else: + requester.send_request(request_body_data=param_data, request_body_json=param_json) + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + if expected_body is not None: + assert sent_request.body == expected_body.decode('UTF-8') if not isinstance(expected_body, str) else expected_body + + +@pytest.mark.parametrize( + "provider_data, param_data, authenticator_data, expected_exception, expected_body", + [ + # assert body string from one source works + ("field=value", None, None, None, "field=value"), + (None, "field=value", None, None, "field=value"), + (None, None, "field=value", None, "field=value"), + # assert body string from multiple sources fails + ("field=value", "field=value", None, ValueError, None), + ("field=value", None, "field=value", ValueError, None), + (None, "field=value", "field=value", ValueError, None), + ("field=value", "field=value", "field=value", ValueError, None), + # assert body string and mapping from different source fails + ("field=value", {"abc": "def"}, None, ValueError, None), + ({"abc": "def"}, "field=value", None, ValueError, None), + ("field=value", None, {"abc": "def"}, ValueError, None), + ] +) +def test_send_request_string_data(provider_data, param_data, authenticator_data, expected_exception, expected_body): + options_provider = MagicMock() + options_provider.get_request_body_data.return_value = provider_data + authenticator = MagicMock() + authenticator.get_request_body_data.return_value = authenticator_data + requester = create_requester(authenticator=authenticator) + requester._request_options_provider = options_provider + if expected_exception is not None: + with pytest.raises(expected_exception): + requester.send_request(request_body_data=param_data) + else: + requester.send_request(request_body_data=param_data) + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + if expected_body is not None: + assert sent_request.body == expected_body + + +@pytest.mark.parametrize( + "provider_headers, param_headers, authenticator_headers, expected_exception, expected_headers", + [ + # merging headers from the three sources + ({"header": "value"}, None, None, None, {"header": "value"}), + ({"header": "value"}, {"header2": "value"}, None, None, {"header": "value", "header2": "value"}), + ({"header": "value"}, {"header2": "value"}, {"authheader": "val"}, None, {"header": "value", "header2": "value", "authheader": "val"}), + # raise on conflicting headers + ({"header": "value"}, {"header": "value"}, None, ValueError, None), + ({"header": "value"}, None, {"header": "value"}, ValueError, None), + ({"header": "value"}, {"header2": "value"}, {"header": "value"}, ValueError, None), + ]) +def test_send_request_headers(provider_headers, param_headers, authenticator_headers, expected_exception, expected_headers): + # headers set by the requests framework, do not validate + default_headers = {'User-Agent': mock.ANY, 'Accept-Encoding': mock.ANY, 'Accept': mock.ANY, 'Connection': mock.ANY} + options_provider = MagicMock() + options_provider.get_request_headers.return_value = provider_headers + authenticator = MagicMock() + authenticator.get_auth_header.return_value = authenticator_headers or {} + requester = create_requester(authenticator=authenticator) + requester._request_options_provider = options_provider + if expected_exception is not None: + with pytest.raises(expected_exception): + requester.send_request(request_headers=param_headers) + else: + requester.send_request(request_headers=param_headers) + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.headers == {**default_headers, **expected_headers} + + +@pytest.mark.parametrize( + "provider_params, param_params, authenticator_params, expected_exception, expected_params", + [ + # merging params from the three sources + ({"param": "value"}, None, None, None, {"param": "value"}), + ({"param": "value"}, {"param2": "value"}, None, None, {"param": "value", "param2": "value"}), + ({"param": "value"}, {"param2": "value"}, {"authparam": "val"}, None, {"param": "value", "param2": "value", "authparam": "val"}), + # raise on conflicting params + ({"param": "value"}, {"param": "value"}, None, ValueError, None), + ({"param": "value"}, None, {"param": "value"}, ValueError, None), + ({"param": "value"}, {"param2": "value"}, {"param": "value"}, ValueError, None), + ]) +def test_send_request_params(provider_params, param_params, authenticator_params, expected_exception, expected_params): + options_provider = MagicMock() + options_provider.get_request_params.return_value = provider_params + authenticator = MagicMock() + authenticator.get_request_params.return_value = authenticator_params + requester = create_requester(authenticator=authenticator) + requester._request_options_provider = options_provider + if expected_exception is not None: + with pytest.raises(expected_exception): + requester.send_request(request_params=param_params) + else: + requester.send_request(request_params=param_params) + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + parsed_url = urlparse(sent_request.url) + query_params = {key: value[0] for key, value in parse_qs(parsed_url.query).items()} + assert query_params == expected_params + + +@pytest.mark.parametrize( + "requester_path, param_path, expected_path", + [ + ("deals", None, "/deals"), + ("deals", "deals2", "/deals2"), + ("deals", "/deals2", "/deals2"), + ("deals/{{ stream_slice.start }}/{{ next_page_token.next_page_token }}/{{ config.config_key }}/{{ parameters.param_key }}", None, "/deals/2012/pagetoken/config_value/param_value"), + ]) +def test_send_request_path(requester_path, param_path, expected_path): + requester = create_requester(config={"config_key": "config_value"}, path=requester_path, parameters={"param_key": "param_value"}) + requester.send_request(stream_slice={"start": "2012"}, next_page_token={"next_page_token": "pagetoken"},path=param_path) + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + parsed_url = urlparse(sent_request.url) + assert parsed_url.path == expected_path + + +def test_send_request_url_base(): + requester = create_requester(url_base="https://example.org/{{ config.config_key }}/{{ parameters.param_key }}", config={"config_key": "config_value"}, parameters={"param_key": "param_value"}) + requester.send_request() + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.url == "https://example.org/config_value/param_value/deals" + + +def test_send_request_stream_slice_next_page_token(): + options_provider = MagicMock() + requester = create_requester() + requester._request_options_provider = options_provider + stream_slice = {"id": "1234"} + next_page_token = {"next_page_token": "next_page_token"} + requester.send_request(stream_slice=stream_slice, next_page_token=next_page_token) + options_provider.get_request_params.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) + options_provider.get_request_body_data.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) + options_provider.get_request_body_json.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) + options_provider.get_request_headers.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) + + +def test_default_authenticator(): + requester = create_requester() + assert isinstance(requester._authenticator, NoAuth) + assert isinstance(requester._session.auth, NoAuth) + + +def test_token_authenticator(): + requester = create_requester(authenticator=BearerAuthenticator(token_provider=MagicMock(), config={}, parameters={})) + assert isinstance(requester.authenticator, BearerAuthenticator) + assert isinstance(requester._session.auth, BearerAuthenticator) + + +def test_stub_custom_backoff_http_stream(mocker): + mocker.patch("time.sleep", lambda x: None) + req = requests.Response() + req.status_code = 429 + + requester = create_requester() + requester._backoff_time = lambda _: 0.5 + + requester._session.send.return_value = req + + with pytest.raises(UserDefinedBackoffException): + requester.send_request() + assert requester._session.send.call_count == requester.max_retries + 1 + + +@pytest.mark.parametrize("retries", [-20, -1, 0, 1, 2, 10]) +def test_stub_custom_backoff_http_stream_retries(mocker, retries): + mocker.patch("time.sleep", lambda x: None) + error_handler = DefaultErrorHandler(parameters={}, config={}, max_retries=retries) + requester = create_requester(error_handler=error_handler) + req = requests.Response() + req.status_code = HTTPStatus.TOO_MANY_REQUESTS + requester._session.send.return_value = req + + with pytest.raises(UserDefinedBackoffException, match="Request URL: https://example.com/deals, Response Code: 429") as excinfo: + requester.send_request() + assert isinstance(excinfo.value.request, requests.PreparedRequest) + assert isinstance(excinfo.value.response, requests.Response) + if retries <= 0: + assert requester._session.send.call_count == 1 + else: + assert requester._session.send.call_count == requester.max_retries + 1 + + +def test_stub_custom_backoff_http_stream_endless_retries(mocker): + mocker.patch("time.sleep", lambda x: None) + error_handler = DefaultErrorHandler(parameters={}, config={}, max_retries=None) + requester = create_requester(error_handler=error_handler) + req = requests.Response() + req.status_code = HTTPStatus.TOO_MANY_REQUESTS + infinite_number = 20 + + req = requests.Response() + req.status_code = HTTPStatus.TOO_MANY_REQUESTS + send_mock = mocker.patch.object(requester._session, "send", side_effect=[req] * infinite_number) + + # Expecting mock object to raise a RuntimeError when the end of side_effect list parameter reached. + with pytest.raises(StopIteration): + requester.send_request() + assert send_mock.call_count == infinite_number + 1 + + +@pytest.mark.parametrize("http_code", [400, 401, 403]) +def test_4xx_error_codes_http_stream(mocker, http_code): + requester = create_requester(error_handler=DefaultErrorHandler(parameters={}, config={}, max_retries=0)) + requester._DEFAULT_RETRY_FACTOR = 0.01 + req = requests.Response() + req.request = requests.Request() + req.status_code = http_code + requester._session.send.return_value = req + + with pytest.raises(ReadException): + requester.send_request() + + +def test_raise_on_http_errors_off_429(mocker): + requester = create_requester() + requester._DEFAULT_RETRY_FACTOR = 0.01 + req = requests.Response() + req.status_code = 429 + requester._session.send.return_value = req + + with pytest.raises(DefaultBackoffException, match="Request URL: https://example.com/deals, Response Code: 429"): + requester.send_request() + + +@pytest.mark.parametrize("status_code", [500, 501, 503, 504]) +def test_raise_on_http_errors_off_5xx(mocker, status_code): + requester = create_requester() + req = requests.Response() + req.status_code = status_code + requester._session.send.return_value = req + requester._DEFAULT_RETRY_FACTOR = 0.01 + + with pytest.raises(DefaultBackoffException): + requester.send_request() + assert requester._session.send.call_count == requester.max_retries + 1 + + +@pytest.mark.parametrize("status_code", [400, 401, 402, 403, 416]) +def test_raise_on_http_errors_off_non_retryable_4xx(mocker, status_code): + requester = create_requester() + req = requests.Response() + req.status_code = status_code + requester._session.send.return_value = req + requester._DEFAULT_RETRY_FACTOR = 0.01 + + response = requester.send_request() + assert response.status_code == status_code + + +@pytest.mark.parametrize( + "error", + ( + requests.exceptions.ConnectTimeout, + requests.exceptions.ConnectionError, + requests.exceptions.ChunkedEncodingError, + requests.exceptions.ReadTimeout, + ), +) +def test_raise_on_http_errors(mocker, error): + requester = create_requester() + req = requests.Response() + req.status_code = 200 + requester._session.send.return_value = req + requester._DEFAULT_RETRY_FACTOR = 0.01 + mocker.patch.object(requester._session, "send", side_effect=error()) + + with pytest.raises(error): + requester.send_request() + assert requester._session.send.call_count == requester.max_retries + 1 + + +@pytest.mark.parametrize( + "api_response, expected_message", + [ + ({"error": "something broke"}, "something broke"), + ({"error": {"message": "something broke"}}, "something broke"), + ({"error": "err-001", "message": "something broke"}, "something broke"), + ({"failure": {"message": "something broke"}}, "something broke"), + ({"error": {"errors": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}}, "one, two, three"), + ({"errors": ["one", "two", "three"]}, "one, two, three"), + ({"messages": ["one", "two", "three"]}, "one, two, three"), + ({"errors": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + ({"error": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + ({"errors": [{"error": "one"}, {"error": "two"}, {"error": "three"}]}, "one, two, three"), + ({"failures": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + (["one", "two", "three"], "one, two, three"), + ([{"error": "one"}, {"error": "two"}, {"error": "three"}], "one, two, three"), + ({"error": True}, None), + ({"something_else": "hi"}, None), + ({}, None), + ], +) +def test_default_parse_response_error_message(api_response: dict, expected_message: Optional[str]): + response = MagicMock() + response.json.return_value = api_response + + message = HttpRequester.parse_response_error_message(response) + assert message == expected_message + + +def test_default_parse_response_error_message_not_json(requests_mock): + requests_mock.register_uri("GET", "mock://test.com/not_json", text="this is not json") + response = requests.get("mock://test.com/not_json") + + message = HttpRequester.parse_response_error_message(response) + assert message is None + + +@pytest.mark.parametrize( + "test_name, base_url, path, expected_full_url",[ + ("test_no_slashes", "https://airbyte.io", "my_endpoint", "https://airbyte.io/my_endpoint"), + ("test_trailing_slash_on_base_url", "https://airbyte.io/", "my_endpoint", "https://airbyte.io/my_endpoint"), + ("test_trailing_slash_on_base_url_and_leading_slash_on_path", "https://airbyte.io/", "/my_endpoint", "https://airbyte.io/my_endpoint"), + ("test_leading_slash_on_path", "https://airbyte.io", "/my_endpoint", "https://airbyte.io/my_endpoint"), + ("test_trailing_slash_on_path", "https://airbyte.io", "/my_endpoint/", "https://airbyte.io/my_endpoint/"), + ("test_nested_path_no_leading_slash", "https://airbyte.io", "v1/my_endpoint", "https://airbyte.io/v1/my_endpoint"), + ("test_nested_path_with_leading_slash", "https://airbyte.io", "/v1/my_endpoint", "https://airbyte.io/v1/my_endpoint"), + ] +) +def test_join_url(test_name, base_url, path, expected_full_url): + requester = HttpRequester( + name="name", + url_base=base_url, + path=path, + http_method=HttpMethod.GET, + request_options_provider=None, + config={}, + parameters={}, + ) + requester._session.send = MagicMock() + response = requests.Response() + response.status_code = 200 + requester._session.send.return_value = response + requester.send_request() + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.url == expected_full_url + + +@pytest.mark.parametrize( + "path, params, expected_url", [ + pytest.param("v1/endpoint?param1=value1", {}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), + pytest.param("v1/endpoint", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), + pytest.param("v1/endpoint", None, "https://test_base_url.com/v1/endpoint", id="test_params_is_none_and_no_params_in_path"), + pytest.param("v1/endpoint?param1=value1", None, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_is_none_and_no_params_in_path"), + pytest.param("v1/endpoint?param1=value1", {"param2": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m2=value2", id="test_no_duplicate_params"), + pytest.param("v1/endpoint?param1=value1", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_duplicate_params_same_value"), + pytest.param("v1/endpoint?param1=1", {"param1": 1}, "https://test_base_url.com/v1/endpoint?param1=1", id="test_duplicate_params_same_value_not_string"), + pytest.param("v1/endpoint?param1=value1", {"param1": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", id="test_duplicate_params_different_value"), + ] +) +def test_duplicate_request_params_are_deduped(path, params, expected_url): + requester = HttpRequester( + name="name", + url_base="https://test_base_url.com", + path=path, + http_method=HttpMethod.GET, + request_options_provider=None, + config={}, + parameters={}, + ) + + if expected_url is None: + with pytest.raises(ValueError): + requester._create_prepared_request(path=path, params=params) + else: + prepared_request = requester._create_prepared_request(path=path, params=params) + assert prepared_request.url == expected_url + + +@pytest.mark.parametrize( + "should_log, status_code, should_throw", [ + (True, 200, False), + (True, 400, False), + (True, 500, True), + (False, 200, False), + (False, 400, False), + (False, 500, True), + ] +) +def test_log_requests(should_log, status_code, should_throw): + repository = MagicMock() + requester = HttpRequester( + name="name", + url_base="https://test_base_url.com", + path="/", + http_method=HttpMethod.GET, + request_options_provider=None, + config={}, + parameters={}, + message_repository=repository, + disable_retries=True + ) + requester._session.send = MagicMock() + response = requests.Response() + response.status_code = status_code + requester._session.send.return_value = response + formatter = MagicMock() + formatter.return_value = "formatted_response" + if should_throw: + with pytest.raises(DefaultBackoffException): + requester.send_request(log_formatter=formatter if should_log else None) + else: + requester.send_request(log_formatter=formatter if should_log else None) + if should_log: + assert repository.log_message.call_args_list[0].args[1]() == "formatted_response" + formatter.assert_called_once_with(response) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index c767a6989705..ebdc7a6201fd 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -2,28 +2,23 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Mapping -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch -import airbyte_cdk.sources.declarative.requesters.error_handlers.response_status as response_status import pytest import requests from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode, Type -from airbyte_cdk.sources.declarative.exceptions import ReadException -from airbyte_cdk.sources.declarative.incremental import DatetimeBasedCursor +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.incremental import Cursor, DatetimeBasedCursor from airbyte_cdk.sources.declarative.partition_routers import SinglePartitionRouter -from airbyte_cdk.sources.declarative.requesters.error_handlers.response_action import ResponseAction from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod -from airbyte_cdk.sources.declarative.retrievers.simple_retriever import ( - SimpleRetriever, - SimpleRetrieverTestReadDecorator, - _prepared_request_to_airbyte_message, - _response_to_airbyte_message, -) -from airbyte_cdk.sources.streams.http.auth import NoAuth -from airbyte_cdk.sources.streams.http.http import HttpStream +from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever, SimpleRetrieverTestReadDecorator +from airbyte_cdk.sources.declarative.types import Record + +A_SLICE_STATE = {"slice_state": "slice state value"} +A_STREAM_SLICE = {"stream slice": "slice value"} +A_STREAM_STATE = {"stream state": "state value"} primary_key = "pk" records = [{"id": 1}, {"id": 2}] @@ -34,7 +29,7 @@ config = {} -@patch.object(HttpStream, "_read_pages", return_value=iter([])) +@patch.object(SimpleRetriever, "_read_pages", return_value=iter([])) def test_simple_retriever_full(mock_http_stream): requester = MagicMock() request_params = {"param": "value"} @@ -44,20 +39,22 @@ def test_simple_retriever_full(mock_http_stream): next_page_token = {"cursor": "cursor_value"} paginator.path.return_value = None paginator.next_page_token.return_value = next_page_token + paginator.get_requesyyt_headers.return_value = {} record_selector = MagicMock() record_selector.select_records.return_value = records - stream_slicer = MagicMock() + cursor = MagicMock(spec=Cursor) stream_slices = [{"date": "2022-01-01"}, {"date": "2022-01-02"}] - stream_slicer.stream_slices.return_value = stream_slices + cursor.stream_slices.return_value = stream_slices response = requests.Response() + response.status_code = 200 underlying_state = {"date": "2021-01-01"} - stream_slicer.get_stream_state.return_value = underlying_state + cursor.get_stream_state.return_value = underlying_state - requester.get_authenticator.return_value = NoAuth() + requester.get_authenticator.return_value = NoAuth({}) url_base = "https://airbyte.io" requester.get_url_base.return_value = url_base path = "/v1" @@ -76,10 +73,6 @@ def test_simple_retriever_full(mock_http_stream): requester.get_request_body_json.return_value = request_body_json request_kwargs = {"kwarg": "value"} requester.request_kwargs.return_value = request_kwargs - cache_filename = "cache" - requester.cache_filename = cache_filename - use_cache = True - requester.use_cache = use_cache retriever = SimpleRetriever( name="stream_name", @@ -87,39 +80,29 @@ def test_simple_retriever_full(mock_http_stream): requester=requester, paginator=paginator, record_selector=record_selector, - stream_slicer=stream_slicer, + stream_slicer=cursor, + cursor=cursor, parameters={}, config={}, ) assert retriever.primary_key == primary_key - assert retriever.url_base == url_base - assert retriever.path() == path assert retriever.state == underlying_state - assert retriever.next_page_token(response) == next_page_token - assert retriever.request_params(None, None, None) == request_params - assert retriever.stream_slices(sync_mode=SyncMode.incremental) == stream_slices + assert retriever._next_page_token(response) == next_page_token + assert retriever._request_params(None, None, None) == {} + assert retriever.stream_slices() == stream_slices assert retriever._last_response is None - assert retriever._last_records is None - assert retriever.parse_response(response, stream_state={}) == records + assert retriever._records_from_last_response == [] + assert retriever._parse_response(response, stream_state={}) == records assert retriever._last_response == response - assert retriever._last_records == records - - assert retriever.http_method == "GET" - assert not retriever.raise_on_http_errors - assert retriever.should_retry(requests.Response()) - assert retriever.backoff_time(requests.Response()) == backoff_time - assert retriever.request_body_json(None, None, None) == request_body_json - assert retriever.request_kwargs(None, None, None) == request_kwargs - assert retriever.cache_filename == cache_filename - assert retriever.use_cache == use_cache + assert retriever._records_from_last_response == records [r for r in retriever.read_records(SyncMode.full_refresh)] paginator.reset.assert_called() -@patch.object(HttpStream, "_read_pages", return_value=iter([*request_response_logs, *records])) +@patch.object(SimpleRetriever, "_read_pages", return_value=iter([*request_response_logs, *records])) def test_simple_retriever_with_request_response_logs(mock_http_stream): requester = MagicMock() paginator = MagicMock() @@ -155,13 +138,14 @@ def test_simple_retriever_with_request_response_logs(mock_http_stream): assert actual_messages[3] == records[1] -@patch.object(HttpStream, "_read_pages", return_value=iter([])) +@patch.object(SimpleRetriever, "_read_pages", return_value=iter([])) def test_simple_retriever_with_request_response_log_last_records(mock_http_stream): requester = MagicMock() paginator = MagicMock() record_selector = MagicMock() record_selector.select_records.return_value = request_response_logs response = requests.Response() + response.status_code = 200 stream_slicer = DatetimeBasedCursor( start_datetime="", end_datetime="", @@ -185,157 +169,22 @@ def test_simple_retriever_with_request_response_log_last_records(mock_http_strea ) assert retriever._last_response is None - assert retriever._last_records is None - assert retriever.parse_response(response, stream_state={}) == request_response_logs + assert retriever._records_from_last_response == [] + assert retriever._parse_response(response, stream_state={}) == request_response_logs assert retriever._last_response == response - assert retriever._last_records == request_response_logs + assert retriever._records_from_last_response == request_response_logs [r for r in retriever.read_records(SyncMode.full_refresh)] paginator.reset.assert_called() -@pytest.mark.parametrize( - "test_name, requester_response, expected_should_retry, expected_backoff_time", - [ - ("test_should_retry_fail", response_status.FAIL, False, None), - ("test_should_retry_none_backoff", ResponseStatus.retry(None), True, None), - ("test_should_retry_custom_backoff", ResponseStatus.retry(60), True, 60), - ], -) -def test_should_retry(test_name, requester_response, expected_should_retry, expected_backoff_time): - requester = MagicMock(use_cache=False) - retriever = SimpleRetriever( - name="stream_name", primary_key=primary_key, requester=requester, record_selector=MagicMock(), parameters={}, config={} - ) - requester.interpret_response_status.return_value = requester_response - assert retriever.should_retry(requests.Response()) == expected_should_retry - if requester_response.action == ResponseAction.RETRY: - assert retriever.backoff_time(requests.Response()) == expected_backoff_time - - -@pytest.mark.parametrize( - "test_name, status_code, response_status, len_expected_records, expected_error", - [ - ( - "test_parse_response_fails_if_should_retry_is_fail", - 404, - response_status.FAIL, - None, - ReadException("Request None failed with response "), - ), - ("test_parse_response_succeeds_if_should_retry_is_ok", 200, response_status.SUCCESS, 1, None), - ("test_parse_response_succeeds_if_should_retry_is_ignore", 404, response_status.IGNORE, 0, None), - ( - "test_parse_response_fails_with_custom_error_message", - 404, - ResponseStatus(response_action=ResponseAction.FAIL, error_message="Custom error message override"), - None, - ReadException("Custom error message override"), - ), - ], -) -def test_parse_response(test_name, status_code, response_status, len_expected_records, expected_error): - requester = MagicMock(use_cache=False) - record_selector = MagicMock() - record_selector.select_records.return_value = [{"id": 100}] - retriever = SimpleRetriever( - name="stream_name", primary_key=primary_key, requester=requester, record_selector=record_selector, parameters={}, config={} - ) - response = requests.Response() - response.request = requests.Request() - response.status_code = status_code - requester.interpret_response_status.return_value = response_status - if len_expected_records is None: - try: - retriever.parse_response(response, stream_state={}) - assert False - except ReadException as actual_exception: - assert type(expected_error) is type(actual_exception) - else: - records = retriever.parse_response(response, stream_state={}) - assert len(records) == len_expected_records - - -def test_max_retries_given_error_handler_has_max_retries(): - requester = MagicMock() - requester.error_handler = MagicMock() - requester.error_handler.max_retries = 10 - retriever = SimpleRetriever( - name="stream_name", - primary_key=primary_key, - requester=requester, - record_selector=MagicMock(), - parameters={}, - config={} - ) - assert retriever.max_retries == 10 - - -def test_max_retries_given_error_handler_without_max_retries(): - requester = MagicMock() - requester.error_handler = MagicMock(spec=[u'without_max_retries_attribute']) - retriever = SimpleRetriever( - name="stream_name", - primary_key=primary_key, - requester=requester, - record_selector=MagicMock(), - parameters={}, - config={} - ) - assert retriever.max_retries == 5 - - -def test_max_retries_given_disable_retries(): - retriever = SimpleRetriever( - name="stream_name", - primary_key=primary_key, - requester=MagicMock(), - record_selector=MagicMock(), - disable_retries=True, - parameters={}, - config={} - ) - assert retriever.max_retries == 0 - - -@pytest.mark.parametrize( - "test_name, response_action, retry_in, expected_backoff_time", - [ - ("test_backoff_retriable_request", ResponseAction.RETRY, 10, 10), - ("test_backoff_fail_request", ResponseAction.FAIL, 10, None), - ("test_backoff_ignore_request", ResponseAction.IGNORE, 10, None), - ("test_backoff_success_request", ResponseAction.IGNORE, 10, None), - ], -) -def test_backoff_time(test_name, response_action, retry_in, expected_backoff_time): - requester = MagicMock(use_cache=False) - record_selector = MagicMock() - record_selector.select_records.return_value = [{"id": 100}] - response = requests.Response() - retriever = SimpleRetriever( - name="stream_name", primary_key=primary_key, requester=requester, record_selector=record_selector, parameters={}, config={} - ) - if expected_backoff_time: - requester.interpret_response_status.return_value = ResponseStatus(response_action, retry_in) - actual_backoff_time = retriever.backoff_time(response) - assert expected_backoff_time == actual_backoff_time - else: - try: - retriever.backoff_time(response) - assert False - except ValueError: - pass - - @pytest.mark.parametrize( "test_name, paginator_mapping, stream_slicer_mapping, expected_mapping", [ - ("test_only_base_headers", {}, {}, {"key": "value"}), - ("test_header_from_pagination", {"offset": 1000}, {}, {"key": "value", "offset": 1000}), - ("test_header_from_stream_slicer", {}, {"slice": "slice_value"}, {"key": "value", "slice": "slice_value"}), - ("test_duplicate_header_slicer", {}, {"key": "slice_value"}, None), + ("test_empty_headers", {}, {}, {}), + ("test_header_from_pagination_and_slicer", {"offset": 1000}, {"key": "value"}, {"key": "value", "offset": 1000}), + ("test_header_from_stream_slicer", {}, {"slice": "slice_value"}, {"slice": "slice_value"}), ("test_duplicate_header_slicer_paginator", {"k": "v"}, {"k": "slice_value"}, None), - ("test_duplicate_header_paginator", {"key": 1000}, {}, None), ], ) def test_get_request_options_from_pagination(test_name, paginator_mapping, stream_slicer_mapping, expected_mapping): @@ -350,17 +199,11 @@ def test_get_request_options_from_pagination(test_name, paginator_mapping, strea stream_slicer.get_request_body_data.return_value = stream_slicer_mapping stream_slicer.get_request_body_json.return_value = stream_slicer_mapping - base_mapping = {"key": "value"} - requester = MagicMock(use_cache=False) - requester.get_request_params.return_value = base_mapping - requester.get_request_body_data.return_value = base_mapping - requester.get_request_body_json.return_value = base_mapping - record_selector = MagicMock() retriever = SimpleRetriever( name="stream_name", primary_key=primary_key, - requester=requester, + requester=MagicMock(), record_selector=record_selector, paginator=paginator, stream_slicer=stream_slicer, @@ -369,13 +212,13 @@ def test_get_request_options_from_pagination(test_name, paginator_mapping, strea ) request_option_type_to_method = { - RequestOptionType.request_parameter: retriever.request_params, - RequestOptionType.body_data: retriever.request_body_data, - RequestOptionType.body_json: retriever.request_body_json, + RequestOptionType.request_parameter: retriever._request_params, + RequestOptionType.body_data: retriever._request_body_data, + RequestOptionType.body_json: retriever._request_body_json, } for _, method in request_option_type_to_method.items(): - if expected_mapping: + if expected_mapping is not None: actual_mapping = method(None, None, None) assert expected_mapping == actual_mapping else: @@ -400,8 +243,8 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping): paginator.get_request_headers.return_value = paginator_mapping requester = MagicMock(use_cache=False) - base_mapping = {"key": "value"} - requester.get_request_headers.return_value = base_mapping + stream_slicer = MagicMock() + stream_slicer.get_request_headers.return_value = {"key": "value"} record_selector = MagicMock() retriever = SimpleRetriever( @@ -409,13 +252,14 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping): primary_key=primary_key, requester=requester, record_selector=record_selector, + stream_slicer=stream_slicer, paginator=paginator, parameters={}, config={}, ) request_option_type_to_method = { - RequestOptionType.header: retriever.request_headers, + RequestOptionType.header: retriever._request_headers, } for _, method in request_option_type_to_method.items(): @@ -431,21 +275,22 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping): @pytest.mark.parametrize( - "test_name, requester_body_data, paginator_body_data, expected_body_data", + "test_name, slicer_body_data, paginator_body_data, expected_body_data", [ - ("test_only_requester_mapping", {"key": "value"}, {}, {"key": "value"}), - ("test_only_requester_string", "key=value", {}, "key=value"), - ("test_requester_mapping_and_paginator_no_duplicate", {"key": "value"}, {"offset": 1000}, {"key": "value", "offset": 1000}), - ("test_requester_mapping_and_paginator_with_duplicate", {"key": "value"}, {"key": 1000}, None), - ("test_requester_string_and_paginator", "key=value", {"offset": 1000}, None), + ("test_only_slicer_mapping", {"key": "value"}, {}, {"key": "value"}), + ("test_only_slicer_string", "key=value", {}, "key=value"), + ("test_slicer_mapping_and_paginator_no_duplicate", {"key": "value"}, {"offset": 1000}, {"key": "value", "offset": 1000}), + ("test_slicer_mapping_and_paginator_with_duplicate", {"key": "value"}, {"key": 1000}, None), + ("test_slicer_string_and_paginator", "key=value", {"offset": 1000}, None), ], ) -def test_request_body_data(test_name, requester_body_data, paginator_body_data, expected_body_data): +def test_request_body_data(test_name, slicer_body_data, paginator_body_data, expected_body_data): paginator = MagicMock() paginator.get_request_body_data.return_value = paginator_body_data requester = MagicMock(use_cache=False) - requester.get_request_body_data.return_value = requester_body_data + stream_slicer = MagicMock() + stream_slicer.get_request_body_data.return_value = slicer_body_data record_selector = MagicMock() retriever = SimpleRetriever( @@ -454,16 +299,17 @@ def test_request_body_data(test_name, requester_body_data, paginator_body_data, requester=requester, record_selector=record_selector, paginator=paginator, + stream_slicer=stream_slicer, parameters={}, config={}, ) if expected_body_data: - actual_body_data = retriever.request_body_data(None, None, None) + actual_body_data = retriever._request_body_data(None, None, None) assert expected_body_data == actual_body_data else: try: - retriever.request_body_data(None, None, None) + retriever._request_body_data(None, None, None) assert False except ValueError: pass @@ -472,7 +318,7 @@ def test_request_body_data(test_name, requester_body_data, paginator_body_data, @pytest.mark.parametrize( "test_name, requester_path, paginator_path, expected_path", [ - ("test_path_from_requester", "/v1/path", None, "/v1/path"), + ("test_path_from_requester", "/v1/path", None, None), ("test_path_from_paginator", "/v1/path/", "/v2/paginator", "/v2/paginator"), ], ) @@ -494,202 +340,10 @@ def test_path(test_name, requester_path, paginator_path, expected_path): config={}, ) - actual_path = retriever.path(stream_state=None, stream_slice=None, next_page_token=None) + actual_path = retriever._paginator_path() assert expected_path == actual_path -@pytest.mark.parametrize( - "test_name, http_method, url, headers, params, body_json, body_data, expected_airbyte_message", - [ - ( - "test_basic_get_request", - HttpMethod.GET, - "https://airbyte.io", - {}, - {}, - {}, - {}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {}, "body": null}' - ), - ), - ), - ( - "test_get_request_with_headers", - HttpMethod.GET, - "https://airbyte.io", - {"h1": "v1", "h2": "v2"}, - {}, - {}, - {}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"h1": "v1", "h2": "v2"}, "body": null}', - ), - ), - ), - ( - "test_get_request_with_request_params", - HttpMethod.GET, - "https://airbyte.io", - {}, - {"p1": "v1", "p2": "v2"}, - {}, - {}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='request:{"url": "https://airbyte.io/?p1=v1&p2=v2", "http_method": "GET", "headers": {}, "body": null}', - ), - ), - ), - ( - "test_get_request_with_request_body_json", - HttpMethod.GET, - "https://airbyte.io", - {"Content-Type": "application/json"}, - {}, - {"b1": "v1", "b2": "v2"}, - {}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"Content-Type": "application/json", "Content-Length": "24"}, "body": {"b1": "v1", "b2": "v2"}}', - ), - ), - ), - ( - "test_get_request_with_headers_params_and_body", - HttpMethod.GET, - "https://airbyte.io", - {"Content-Type": "application/json", "h1": "v1"}, - {"p1": "v1", "p2": "v2"}, - {"b1": "v1", "b2": "v2"}, - {}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='request:{"url": "https://airbyte.io/?p1=v1&p2=v2", "http_method": "GET", "headers": {"Content-Type": "application/json", "h1": "v1", "Content-Length": "24"}, "body": {"b1": "v1", "b2": "v2"}}', - ), - ), - ), - ( - "test_get_request_with_request_body_data", - HttpMethod.GET, - "https://airbyte.io", - {"Content-Type": "application/json"}, - {}, - {}, - {"b1": "v1", "b2": "v2"}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='request:{"url": "https://airbyte.io/", "http_method": "GET", "headers": {"Content-Type": "application/json", "Content-Length": "11"}, "body": {"b1": "v1", "b2": "v2"}}', - ), - ), - ), - ( - "test_basic_post_request", - HttpMethod.POST, - "https://airbyte.io", - {}, - {}, - {}, - {}, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='request:{"url": "https://airbyte.io/", "http_method": "POST", "headers": {"Content-Length": "0"}, "body": null}', - ), - ), - ), - ], -) -def test_prepared_request_to_airbyte_message(test_name, http_method, url, headers, params, body_json, body_data, expected_airbyte_message): - request = requests.Request(method=http_method.name, url=url, headers=headers, params=params) - if body_json: - request.json = body_json - if body_data: - request.data = body_data - prepared_request = request.prepare() - - actual_airbyte_message = _prepared_request_to_airbyte_message(prepared_request) - - assert expected_airbyte_message == actual_airbyte_message - - -@pytest.mark.parametrize( - "test_name, response_body, response_headers, status_code, expected_airbyte_message", - [ - ( - "test_response_no_body_no_headers", - b"", - {}, - 200, - AirbyteMessage( - type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message='response:{"body": "", "headers": {}, "status_code": 200}') - ), - ), - ( - "test_response_no_body_with_headers", - b"", - {"h1": "v1", "h2": "v2"}, - 200, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, message='response:{"body": "", "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200}' - ), - ), - ), - ( - "test_response_with_body_no_headers", - b'{"b1": "v1", "b2": "v2"}', - {}, - 200, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='response:{"body": "{\\"b1\\": \\"v1\\", \\"b2\\": \\"v2\\"}", "headers": {}, "status_code": 200}', - ), - ), - ), - ( - "test_response_with_body_and_headers", - b'{"b1": "v1", "b2": "v2"}', - {"h1": "v1", "h2": "v2"}, - 200, - AirbyteMessage( - type=Type.LOG, - log=AirbyteLogMessage( - level=Level.INFO, - message='response:{"body": "{\\"b1\\": \\"v1\\", \\"b2\\": \\"v2\\"}", "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200}', - ), - ), - ), - ], -) -def test_response_to_airbyte_message(test_name, response_body, response_headers, status_code, expected_airbyte_message): - response = requests.Response() - response.status_code = status_code - response.headers = response_headers - response._content = response_body - - actual_airbyte_message = _response_to_airbyte_message(response) - - assert expected_airbyte_message == actual_airbyte_message - - def test_limit_stream_slices(): maximum_number_of_slices = 4 stream_slicer = MagicMock() @@ -706,48 +360,101 @@ def test_limit_stream_slices(): config={}, ) - truncated_slices = list(retriever.stream_slices(sync_mode=SyncMode.incremental, stream_state=None)) + truncated_slices = list(retriever.stream_slices()) assert truncated_slices == _generate_slices(maximum_number_of_slices) @pytest.mark.parametrize( - "test_name, last_records, records, expected_stream_slicer_update_count", + "test_name, first_greater_than_second", [ - ("test_two_records", [{"id": -1}], records, 2), - ("test_no_records", [{"id": -1}], [], 1), - ("test_no_records_no_previous_records", [], [], 0) - ] + ("test_first_greater_than_second", True), + ("test_second_greater_than_first", False), + ], ) -def test_read_records_updates_stream_slicer_once_if_no_records(test_name, last_records, records, expected_stream_slicer_update_count): - with patch.object(HttpStream, "_read_pages", return_value=iter(records)): - requester = MagicMock() - paginator = MagicMock() - record_selector = MagicMock() - stream_slicer = MagicMock() - - retriever = SimpleRetriever( - name="stream_name", - primary_key=primary_key, - requester=requester, - paginator=paginator, - record_selector=record_selector, - stream_slicer=stream_slicer, - parameters={}, - config={}, - ) - retriever._last_records = last_records - - list(retriever.read_records(sync_mode=SyncMode.incremental, stream_slice={"repository": "airbyte"})) - - assert stream_slicer.update_cursor.call_count == expected_stream_slicer_update_count +def test_when_read_records_then_cursor_close_slice_with_greater_record(test_name, first_greater_than_second): + first_record = Record({"first": 1}, {}) + second_record = Record({"second": 2}, {}) + records = [first_record, second_record] + record_selector = MagicMock() + record_selector.select_records.return_value = records + cursor = MagicMock(spec=Cursor) + cursor.is_greater_than_or_equal.return_value = first_greater_than_second + paginator = MagicMock() + paginator.get_request_headers.return_value = {} + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=MagicMock(), + paginator=paginator, + record_selector=record_selector, + stream_slicer=cursor, + cursor=cursor, + parameters={}, + config={}, + ) + stream_slice = {"repository": "airbyte"} + + with patch.object(SimpleRetriever, "_read_pages", return_value=iter([first_record, second_record]), side_effect=lambda _, __, ___: retriever._parse_records(response=MagicMock(), stream_state=None, stream_slice=stream_slice)): + list(retriever.read_records(stream_slice=stream_slice)) + cursor.close_slice.assert_called_once_with(stream_slice, first_record if first_greater_than_second else second_record) + + +def test_given_stream_data_is_not_record_when_read_records_then_update_slice_with_optional_record(): + stream_data = [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="a log message"))] + record_selector = MagicMock() + record_selector.select_records.return_value = [] + cursor = MagicMock(spec=Cursor) + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=MagicMock(), + paginator=Mock(), + record_selector=record_selector, + stream_slicer=cursor, + cursor=cursor, + parameters={}, + config={}, + ) + stream_slice = {"repository": "airbyte"} + + with patch.object(SimpleRetriever, "_read_pages", return_value=iter(stream_data), side_effect=lambda _, __, ___: retriever._parse_records(response=MagicMock(), stream_state=None, stream_slice=stream_slice)): + list(retriever.read_records(stream_slice=stream_slice)) + cursor.close_slice.assert_called_once_with(stream_slice, None) def _generate_slices(number_of_slices): return [{"date": f"2022-01-0{day + 1}"} for day in range(number_of_slices)] -def test_emit_log_request_response_messages(): +@patch.object(SimpleRetriever, "_read_pages", return_value=iter([])) +def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages): + requester = MagicMock() + paginator = MagicMock() + record_selector = MagicMock() + cursor = MagicMock(spec=Cursor) + cursor.select_state = MagicMock(return_value=A_SLICE_STATE) + cursor.get_stream_state = MagicMock(return_value=A_STREAM_STATE) + + retriever = SimpleRetriever( + name="stream_name", + primary_key=primary_key, + requester=requester, + paginator=paginator, + record_selector=record_selector, + stream_slicer=cursor, + cursor=cursor, + parameters={}, + config={}, + ) + list(retriever.read_records(stream_slice=A_STREAM_SLICE)) + + http_stream_read_pages.assert_called_once_with(retriever._parse_records, A_STREAM_STATE, A_STREAM_SLICE) + + +def test_emit_log_request_response_messages(mocker): record_selector = MagicMock() record_selector.select_records.return_value = records @@ -759,10 +466,12 @@ def test_emit_log_request_response_messages(): response.request = request response.status_code = 200 + format_http_message_mock = mocker.patch("airbyte_cdk.sources.declarative.retrievers.simple_retriever.format_http_message") + requester = MagicMock() retriever = SimpleRetrieverTestReadDecorator( name="stream_name", primary_key=primary_key, - requester=MagicMock(), + requester=requester, paginator=MagicMock(), record_selector=record_selector, stream_slicer=SinglePartitionRouter(parameters={}), @@ -770,17 +479,7 @@ def test_emit_log_request_response_messages(): config={}, ) - request_log_message, response_log_message, record_1, record_2 = [ - record for record in retriever.parse_records(request=request, response=response, stream_slice={}, stream_state={}) - ] - - assert isinstance(request_log_message, AirbyteMessage) - assert request_log_message.type == Type.LOG - assert "request:" in request_log_message.log.message - assert isinstance(response_log_message, AirbyteMessage) - assert response_log_message.type == Type.LOG - assert "response:" in response_log_message.log.message - assert isinstance(record_1, Mapping) - assert record_1 == records[0] - assert isinstance(record_1, Mapping) - assert record_2 == records[1] + retriever._fetch_next_page(stream_state={}, stream_slice={}) + + assert requester.send_request.call_args_list[0][1]["log_formatter"] is not None + assert requester.send_request.call_args_list[0][1]["log_formatter"](response) == format_http_message_mock.return_value diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py index 5e24c75dee84..74e4d5fece7c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py @@ -3,7 +3,6 @@ # import pytest as pytest -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -70,47 +69,10 @@ ) def test_substream_slicer(test_name, stream_slicers, expected_slices): slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, parameters={}) - slices = [s for s in slicer.stream_slices(SyncMode.incremental, stream_state=None)] + slices = [s for s in slicer.stream_slices()] assert slices == expected_slices -@pytest.mark.parametrize( - "test_name, stream_slice, expected_state", - [ - ("test_update_cursor_no_state_no_record", {}, {}), - ("test_update_cursor_partial_state", {"owner_resource": "customer"}, {"owner_resource": "customer"}), - ( - "test_update_cursor_full_state", - {"owner_resource": "customer", "date": "2021-01-01"}, - {"owner_resource": "customer", "date": "2021-01-01"}, - ), - ], -) -def test_update_cursor(test_name, stream_slice, expected_state): - stream_slicers = [ - ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={}), - DatetimeBasedCursor( - start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), - end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}), - step="P1D", - cursor_field=InterpolatedString(string="date", parameters={}), - datetime_format="%Y-%m-%d", - cursor_granularity="P1D", - config={}, - parameters={}, - ), - ] - slicer = CartesianProductStreamSlicer(stream_slicers=stream_slicers, parameters={}) - - if expected_state: - slicer.update_cursor(stream_slice, None) - updated_state = slicer.get_stream_state() - assert expected_state == updated_state - else: - with pytest.raises(ValueError): - slicer.update_cursor(stream_slice, None) - - @pytest.mark.parametrize( "test_name, stream_1_request_option, stream_2_request_option, expected_req_params, expected_headers,expected_body_json, expected_body_data", [ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py index 62e52eaf0e44..a5da7e092139 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py @@ -2,22 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest import mock -from unittest.mock import MagicMock, call +from unittest.mock import MagicMock -from airbyte_cdk.models import ( - AirbyteLogMessage, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteTraceMessage, - Level, - SyncMode, - TraceType, - Type, -) +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage, Level, SyncMode, TraceType, Type from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream -from airbyte_cdk.sources.declarative.transformations import AddFields, RecordTransformation -from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition + +SLICE_NOT_CONSIDERED_FOR_EQUALITY = {} def test_declarative_stream(): @@ -47,10 +37,6 @@ def test_declarative_stream(): retriever.read_records.return_value = records retriever.stream_slices.return_value = stream_slices - no_op_transform = mock.create_autospec(spec=RecordTransformation) - no_op_transform.transform = MagicMock(side_effect=lambda record, config, stream_slice, stream_state: record) - transformations = [no_op_transform] - config = {"api_key": "open_sesame"} stream = DeclarativeStream( @@ -60,7 +46,6 @@ def test_declarative_stream(): schema_loader=schema_loader, retriever=retriever, config=config, - transformations=transformations, parameters={"cursor_field": "created_at"}, ) @@ -72,69 +57,17 @@ def test_declarative_stream(): assert stream.primary_key == primary_key assert stream.cursor_field == cursor_field assert stream.stream_slices(sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_state=None) == stream_slices - for transformation in transformations: - expected_calls = [ - call(record, config=config, stream_slice=input_slice, stream_state=state) for record in records if isinstance(record, dict) - ] - assert len(transformation.transform.call_args_list) == len(expected_calls) - transformation.transform.assert_has_calls(expected_calls, any_order=False) - - -def test_declarative_stream_with_add_fields_transform(): - name = "stream" - primary_key = "pk" - cursor_field = "created_at" - - schema_loader = MagicMock() - json_schema = {"name": {"type": "string"}} - schema_loader.get_json_schema.return_value = json_schema - state = MagicMock() - retriever_records = [ - {"pk": 1234, "field": "value"}, - {"pk": 4567, "field": "different_value"}, - AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(data={"pk": 1357, "field": "a_value"}, emitted_at=12344, stream="stream")), - AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is a log message")), - AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345)), - ] - - expected_records = [ - {"pk": 1234, "field": "value", "added_key": "added_value"}, - {"pk": 4567, "field": "different_value", "added_key": "added_value"}, - AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(data={"pk": 1357, "field": "a_value", "added_key": "added_value"}, emitted_at=12344, stream="stream")), - AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="This is a log message")), - AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.ERROR, emitted_at=12345)), - ] - stream_slices = [ - {"date": "2021-01-01"}, - {"date": "2021-01-02"}, - {"date": "2021-01-03"}, - ] - - retriever = MagicMock() - retriever.state = state - retriever.read_records.return_value = retriever_records - retriever.stream_slices.return_value = stream_slices - - inputs = [AddedFieldDefinition(path=["added_key"], value="added_value", parameters={})] - add_fields_transform = AddFields(fields=inputs, parameters={}) - transformations = [add_fields_transform] - - config = {"api_key": "open_sesame"} +def test_state_checkpoint_interval(): stream = DeclarativeStream( - name=name, - primary_key=primary_key, + name="any name", + primary_key="any primary key", stream_cursor_field="{{ parameters['cursor_field'] }}", - schema_loader=schema_loader, - retriever=retriever, - config=config, - transformations=transformations, - parameters={"cursor_field": "created_at"}, + schema_loader=MagicMock(), + retriever=MagicMock(), + config={}, + parameters={}, ) - assert stream.name == name - assert stream.get_json_schema() == json_schema - assert stream.state == state - input_slice = stream_slices[0] - assert list(stream.read_records(SyncMode.full_refresh, cursor_field, input_slice, state)) == expected_records + assert stream.state_checkpoint_interval is None diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index 9c28997e917e..e139e4ac2062 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -25,7 +25,7 @@ ) from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from jsonschema.exceptions import ValidationError logger = logging.getLogger("airbyte") @@ -767,7 +767,9 @@ def _create_response(body): def _create_page(response_body): - return _create_request(), _create_response(response_body) + response = _create_response(response_body) + response.request = _create_request() + return response @pytest.mark.parametrize("test_name, manifest, pages, expected_records, expected_calls",[ @@ -1135,7 +1137,7 @@ def _create_page(response_body): (_create_page({"rates": [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}], "_metadata": {"next": "next"}}), _create_page({"rates": [{"ABC": 2, "partition": 1}], "_metadata": {"next": "next"}})), [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"ABC": 2, "partition": 1}], - [call({"partition": "0"}, {}, None), call({"partition": "1"}, {}, None)] + [call({}, {"partition": "0"}, None), call({}, {"partition": "1"}, None)] ), ("test_with_pagination_and_partition_router", { @@ -1236,15 +1238,15 @@ def _create_page(response_body): _create_page({"rates": [{"ABC": 2, "partition": 1}], "_metadata": {}}), ), [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"USD": 3, "partition": 0}, {"ABC": 2, "partition": 1}], - [call({"partition": "0"}, {}, None), call({"partition": "0"}, {}, {"next_page_token": "next"}), call({"partition": "1"}, {}, None),] + [call({}, {"partition": "0"}, None), call({}, {"partition": "0"},{"next_page_token": "next"}), call({}, {"partition": "1"},None),] ) ]) def test_read_manifest_declarative_source(test_name, manifest, pages, expected_records, expected_calls): _stream_name = "Rates" - with patch.object(HttpStream, "_fetch_next_page", side_effect=pages) as mock_http_stream: + with patch.object(SimpleRetriever, "_fetch_next_page", side_effect=pages) as mock_retriever: output_data = [message.record.data for message in _run_read(manifest, _stream_name) if message.record] assert expected_records == output_data - mock_http_stream.assert_has_calls(expected_calls) + mock_retriever.assert_has_calls(expected_calls) def _run_read(manifest: Mapping[str, Any], stream_name: str) -> List[AirbyteMessage]: diff --git a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py new file mode 100644 index 000000000000..28ff10bc8660 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from typing import Any, Mapping, Optional +from unittest.mock import MagicMock + +from airbyte_cdk.sources.embedded.base_integration import BaseEmbeddedIntegration +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import ( + AirbyteCatalog, + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + Level, + SyncMode, + Type, +) + + +class TestIntegration(BaseEmbeddedIntegration): + def _handle_record(self, record: AirbyteRecordMessage, id: Optional[str]) -> Mapping[str, Any]: + return {"data": record.data, "id": id} + + +class EmbeddedIntegrationTestCase(unittest.TestCase): + def setUp(self): + self.source_class = MagicMock() + self.source = MagicMock() + self.source_class.return_value = self.source + self.source.spec.return_value = ConnectorSpecification(connectionSpecification={ + "properties": { + "test": { + "type": "string", + } + } + }) + self.config = {"test": "abc"} + self.integration = TestIntegration(self.source, self.config) + self.stream1 = AirbyteStream( + name="test", + source_defined_primary_key=[["test"]], + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + ) + self.stream2 = AirbyteStream(name="test2", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]) + self.source.discover.return_value = AirbyteCatalog(streams=[self.stream2, self.stream1]) + + def test_integration(self): + self.source.read.return_value = [ + AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="test")), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test", data={"test": 1}, emitted_at=1)), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test", data={"test": 2}, emitted_at=2)), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test", data={"test": 3}, emitted_at=3)), + ] + result = list(self.integration._load_data("test", None)) + self.assertEqual( + result, + [ + {"data": {"test": 1}, "id": "1"}, + {"data": {"test": 2}, "id": "2"}, + {"data": {"test": 3}, "id": "3"}, + ], + ) + self.source.discover.assert_called_once_with(self.config) + self.source.read.assert_called_once_with( + self.config, + ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=self.stream1, + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + primary_key=[["test"]], + ) + ] + ), + None, + ) + + def test_failed_check(self): + self.config = {"test": 123} + with self.assertRaises(AirbyteTracedException) as error: + TestIntegration(self.source, self.config) + assert str(error.exception) == "123 is not of type 'string'" + + def test_state(self): + state = AirbyteStateMessage(data={}) + self.source.read.return_value = [ + AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="test")), + AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test", data={"test": 1}, emitted_at=1)), + AirbyteMessage(type=Type.STATE, state=state), + ] + result = list(self.integration._load_data("test", None)) + self.assertEqual( + result, + [ + {"data": {"test": 1}, "id": "1"}, + ], + ) + self.integration.last_state = state + + def test_incremental(self): + state = AirbyteStateMessage(data={}) + list(self.integration._load_data("test", state)) + self.source.read.assert_called_once_with( + self.config, + ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=self.stream1, + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + primary_key=[["test"]], + ) + ] + ), + state, + ) + + def test_incremental_without_state(self): + list(self.integration._load_data("test")) + self.source.read.assert_called_once_with( + self.config, + ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=self.stream1, + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + primary_key=[["test"]], + ) + ] + ), + None, + ) + + def test_incremental_unsupported(self): + state = AirbyteStateMessage(data={}) + list(self.integration._load_data("test2", state)) + self.source.read.assert_called_once_with( + self.config, + ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=self.stream2, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ), + state, + ) diff --git a/tools/ci_code_validator/ci_changes_detection/__init__.py b/airbyte-cdk/python/unit_tests/sources/file_based/__init__.py similarity index 100% rename from tools/ci_code_validator/ci_changes_detection/__init__.py rename to airbyte-cdk/python/unit_tests/sources/file_based/__init__.py diff --git a/tools/ci_code_validator/tests/__init__.py b/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/__init__.py similarity index 100% rename from tools/ci_code_validator/tests/__init__.py rename to airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py new file mode 100644 index 000000000000..fc6656d5983e --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from datetime import datetime +from unittest.mock import Mock, PropertyMock + +from airbyte_cdk.sources.file_based.availability_strategy.default_file_based_availability_strategy import ( + DefaultFileBasedAvailabilityStrategy, +) +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream import AbstractFileBasedStream + +_FILE_WITH_UNKNOWN_EXTENSION = RemoteFile(uri="a.unknown_extension", last_modified=datetime.now(), file_type="csv") +_ANY_CONFIG = FileBasedStreamConfig( + name="config.name", + file_type="parquet", + format=JsonlFormat(), +) +_ANY_SCHEMA = {"key": "value"} + + +class DefaultFileBasedAvailabilityStrategyTest(unittest.TestCase): + + def setUp(self) -> None: + self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) + self._strategy = DefaultFileBasedAvailabilityStrategy(self._stream_reader) + + self._parser = Mock(spec=FileTypeParser) + self._stream = Mock(spec=AbstractFileBasedStream) + self._stream.get_parser.return_value = self._parser + self._stream.catalog_schema = _ANY_SCHEMA + self._stream.config = _ANY_CONFIG + self._stream.validation_policy = PropertyMock(validate_schema_before_sync=False) + + def test_given_file_extension_does_not_match_when_check_availability_and_parsability_then_stream_is_still_available(self) -> None: + """ + Before, we had a validation on the file extension but it turns out that in production, users sometimes have mismatch there. The + example we've seen was for JSONL parser but the file extension was just `.json`. Note that there we more than one record extracted + from this stream so it's not just that the file is one JSON object + """ + self._stream.list_files.return_value = [_FILE_WITH_UNKNOWN_EXTENSION] + self._parser.parse_records.return_value = [{"a record": 1}] + + is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock()) + + assert is_available diff --git a/tools/ci_code_validator/tests/simple_package/__init__.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/__init__.py similarity index 100% rename from tools/ci_code_validator/tests/simple_package/__init__.py rename to airbyte-cdk/python/unit_tests/sources/file_based/config/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py new file mode 100644 index 000000000000..961a52d49782 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Type + +import pytest +from airbyte_cdk.sources.file_based.config.file_based_stream_config import AvroFormat, CsvFormat, ParquetFormat +from jsonschema import ValidationError, validate +from pydantic import BaseModel + + +@pytest.mark.parametrize( + "file_format, file_type, expected_error", + [ + pytest.param(ParquetFormat, "parquet", None, id="test_parquet_format_is_a_valid_parquet_file_type"), + pytest.param(AvroFormat, "avro", None, id="test_avro_format_is_a_valid_avro_file_type"), + pytest.param(CsvFormat, "parquet", ValidationError, id="test_csv_format_is_not_a_valid_parquet_file_type"), + ] +) +def test_parquet_file_type_is_not_a_valid_csv_file_type(file_format: BaseModel, file_type: str, expected_error: Type[Exception]) -> None: + format_config = { + file_type: { + "filetype": file_type, + "decimal_as_float": True + } + } + + if expected_error: + with pytest.raises(expected_error): + validate(instance=format_config[file_type], schema=file_format.schema()) + else: + validate(instance=format_config[file_type], schema=file_format.schema()) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py new file mode 100644 index 000000000000..ebd6a2571d0d --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py @@ -0,0 +1,62 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, Type + +import pytest as pytest +from airbyte_cdk.sources.file_based.config.file_based_stream_config import CsvFormat, FileBasedStreamConfig +from pydantic import ValidationError + + +@pytest.mark.parametrize( + "file_type, input_format, expected_format, expected_error", + [ + pytest.param("csv", {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True}, {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True}, None, id="test_valid_format"), + pytest.param("csv", {"filetype": "csv", "double_quote": False}, {"delimiter": ",", "quote_char": "\"", "encoding": "utf8", "double_quote": False}, None, id="test_default_format_values"), + pytest.param("csv", {"filetype": "csv", "delimiter": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_delimiter"), + pytest.param("csv", {"filetype": "csv", "quote_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_quote_char"), + pytest.param("csv", {"filetype": "csv", "escape_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_escape_char"), + pytest.param("csv", {"filetype": "csv", "delimiter": ",", "quote_char": "\"", "encoding": "not_a_format", "double_quote": True}, {}, ValidationError, id="test_invalid_encoding_type"), + pytest.param("invalid", {"filetype": "invalid", "double_quote": False}, {}, ValidationError, id="test_config_format_file_type_mismatch"), + ] +) +def test_csv_config(file_type: str, input_format: Mapping[str, Any], expected_format: Mapping[str, Any], expected_error: Type[Exception]) -> None: + stream_config = { + "name": "stream1", + "file_type": file_type, + "globs": ["*"], + "validation_policy": "Emit Record", + "format": input_format + } + + if expected_error: + with pytest.raises(expected_error): + FileBasedStreamConfig(**stream_config) + else: + actual_config = FileBasedStreamConfig(**stream_config) + if actual_config.format is not None: + for expected_format_field, expected_format_value in expected_format.items(): + assert isinstance(actual_config.format, CsvFormat) + assert getattr(actual_config.format, expected_format_field) == expected_format_value + else: + assert False, "Expected format to be set" + + +def test_invalid_validation_policy() -> None: + stream_config = { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Not Valid Policy", + "format": { + "filetype": "csv", + "delimiter": "d", + "quote_char": "q", + "escape_char": "e", + "encoding": "ascii", + "double_quote": True, + }, + } + with pytest.raises(ValidationError): + FileBasedStreamConfig(**stream_config) diff --git a/tools/ci_code_validator/tests/simple_smell_package/__init__.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/__init__.py similarity index 100% rename from tools/ci_code_validator/tests/simple_smell_package/__init__.py rename to airbyte-cdk/python/unit_tests/sources/file_based/file_types/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py new file mode 100644 index 000000000000..106e9dc1e68c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +import uuid + +import pytest +from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat +from airbyte_cdk.sources.file_based.file_types import AvroParser + +_default_avro_format = AvroFormat() +_double_as_string_avro_format = AvroFormat(double_as_string=True) + + +@pytest.mark.parametrize( + "avro_format, avro_type, expected_json_type, expected_error", + [ + # Primitive types + pytest.param(_default_avro_format, "null", {"type": "null"}, None, id="test_null"), + pytest.param(_default_avro_format, "boolean", {"type": "boolean"}, None, id="test_boolean"), + pytest.param(_default_avro_format, "int", {"type": "integer"}, None, id="test_int"), + pytest.param(_default_avro_format, "long", {"type": "integer"}, None, id="test_long"), + pytest.param(_default_avro_format, "float", {"type": "number"}, None, id="test_float"), + pytest.param(_default_avro_format, "double", {"type": "number"}, None, id="test_double"), + pytest.param(_double_as_string_avro_format, "double", {"type": "string"}, None, id="test_double_as_string"), + pytest.param(_default_avro_format, "bytes", {"type": "string"}, None, id="test_bytes"), + pytest.param(_default_avro_format, "string", {"type": "string"}, None, id="test_string"), + pytest.param(_default_avro_format, "void", None, ValueError, id="test_invalid_type"), + # Complex types + pytest.param( + _default_avro_format, + { + "type": "record", + "name": "SubRecord", + "fields": [{"name": "precise", "type": "double"}, {"name": "robo", "type": "bytes"}, {"name": "simple", "type": "long"}], + }, + { + "type": "object", + "properties": { + "precise": {"type": "number"}, + "robo": {"type": "string"}, + "simple": {"type": "integer"}, + }, + }, + None, + id="test_record", + ), + pytest.param( + _default_avro_format, + { + "type": "record", + "name": "SubRecord", + "fields": [{"name": "precise", "type": "double"}, {"name": "obj_array", "type": {"type": "array", "items": "float"}}], + }, + {"type": "object", "properties": {"precise": {"type": "number"}, "obj_array": {"type": "array", "items": {"type": "number"}}}}, + None, + id="test_record_with_nested_array", + ), + pytest.param( + _default_avro_format, + { + "type": "record", + "name": "SubRecord", + "fields": [ + { + "name": "nested_record", + "type": {"type": "record", "name": "SubRecord", "fields": [{"name": "question", "type": "boolean"}]}, + } + ], + }, + { + "type": "object", + "properties": { + "nested_record": { + "type": "object", + "properties": {"question": {"type": "boolean"}}, + } + }, + }, + None, + id="test_record_with_nested_record", + ), + pytest.param(_default_avro_format, {"type": "array", "items": "float"}, {"type": "array", "items": {"type": "number"}}, None, + id="test_array"), + pytest.param( + _default_avro_format, + {"type": "array", "items": {"type": "record", "name": "SubRecord", "fields": [{"name": "precise", "type": "double"}]}}, + { + "type": "array", + "items": { + "type": "object", + "properties": { + "precise": {"type": "number"}, + }, + }, + }, + None, + id="test_array_of_records", + ), + pytest.param(_default_avro_format, {"type": "array", "not_items": "string"}, None, ValueError, id="test_array_missing_items"), + pytest.param(_default_avro_format, {"type": "array", "items": "invalid_avro_type"}, None, ValueError, + id="test_array_invalid_item_type"), + pytest.param( + _default_avro_format, + {"type": "enum", "name": "IMF", "symbols": ["Ethan", "Benji", "Luther"]}, + {"type": "string", "enum": ["Ethan", "Benji", "Luther"]}, + None, + id="test_enum", + ), + pytest.param(_default_avro_format, {"type": "enum", "name": "IMF"}, None, ValueError, id="test_enum_missing_symbols"), + pytest.param(_default_avro_format, {"type": "enum", "symbols": ["mission", "not", "accepted"]}, None, ValueError, + id="test_enum_missing_name"), + pytest.param( + _default_avro_format, + {"type": "map", "values": "int"}, {"type": "object", "additionalProperties": {"type": "integer"}}, None, id="test_map" + ), + pytest.param( + _default_avro_format, + {"type": "map", "values": {"type": "record", "name": "SubRecord", "fields": [{"name": "agent", "type": "string"}]}}, + {"type": "object", "additionalProperties": {"type": "object", "properties": {"agent": {"type": "string"}}}}, + None, + id="test_map_object", + ), + pytest.param(_default_avro_format, {"type": "map"}, None, ValueError, id="test_map_missing_values"), + pytest.param( + _default_avro_format, + {"type": "fixed", "name": "limit", "size": 12}, {"type": "string", "pattern": "^[0-9A-Fa-f]{24}$"}, None, id="test_fixed" + ), + pytest.param(_default_avro_format, {"type": "fixed", "name": "limit"}, None, ValueError, id="test_fixed_missing_size"), + pytest.param(_default_avro_format, {"type": "fixed", "name": "limit", "size": "50"}, None, ValueError, + id="test_fixed_size_not_integer"), + # Logical types + pytest.param( + _default_avro_format, + {"type": "bytes", "logicalType": "decimal", "precision": 9, "scale": 4}, + {"type": "string", "pattern": f"^-?\\d{{{1, 5}}}(?:\\.\\d{1, 4})?$"}, + None, + id="test_decimal", + ), + pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "decimal", "scale": 4}, None, ValueError, + id="test_decimal_missing_precision"), + pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "decimal", "precision": 9}, None, ValueError, + id="test_decimal_missing_scale"), + pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "uuid"}, {"type": ["null", "string"]}, None, id="test_uuid"), + pytest.param(_default_avro_format, {"type": "int", "logicalType": "date"}, {"type": ["null", "string"], "format": "date"}, None, + id="test_date"), + pytest.param(_default_avro_format, {"type": "int", "logicalType": "time-millis"}, {"type": ["null", "integer"]}, None, id="test_time_millis"), + pytest.param(_default_avro_format, {"type": "long", "logicalType": "time-micros"}, {"type": ["null", "integer"]}, None, + id="test_time_micros"), + pytest.param( + _default_avro_format, + {"type": "long", "logicalType": "timestamp-millis"}, {"type": ["null", "string"], "format": "date-time"}, None, id="test_timestamp_millis" + ), + pytest.param(_default_avro_format, {"type": "long", "logicalType": "timestamp-micros"}, {"type": ["null", "string"]}, None, + id="test_timestamp_micros"), + pytest.param( + _default_avro_format, + {"type": "long", "logicalType": "local-timestamp-millis"}, {"type": "string", "format": "date-time"}, None, + id="test_local_timestamp_millis" + ), + pytest.param(_default_avro_format, {"type": "long", "logicalType": "local-timestamp-micros"}, {"type": "string"}, None, + id="test_local_timestamp_micros"), + + ], +) +def test_convert_primitive_avro_type_to_json(avro_format, avro_type, expected_json_type, expected_error): + if expected_error: + with pytest.raises(expected_error): + AvroParser._convert_avro_type_to_json(avro_format, "field_name", avro_type) + else: + actual_json_type = AvroParser._convert_avro_type_to_json(avro_format, "field_name", avro_type) + assert actual_json_type == expected_json_type + + +@pytest.mark.parametrize( + "avro_format, record_type, record_value, expected_value", [ + pytest.param(_default_avro_format, "boolean", True, True, id="test_boolean"), + pytest.param(_default_avro_format, "int", 123, 123, id="test_int"), + pytest.param(_default_avro_format, "long", 123, 123, id="test_long"), + pytest.param(_default_avro_format, "float", 123.456, 123.456, id="test_float"), + pytest.param(_default_avro_format, "double", 123.456, 123.456, id="test_double_default_config"), + pytest.param(_double_as_string_avro_format, "double", 123.456, "123.456", id="test_double_as_string"), + pytest.param(_default_avro_format, "bytes", b"hello world", b"hello world", id="test_bytes"), + pytest.param(_default_avro_format, "string", "hello world", "hello world", id="test_string"), + pytest.param(_default_avro_format, {"logicalType": "decimal"}, 3.1415, "3.1415", id="test_decimal"), + pytest.param(_default_avro_format, {"logicalType": "uuid"}, b"abcdefghijklmnop", uuid.UUID(bytes=b"abcdefghijklmnop"), id="test_uuid"), + pytest.param(_default_avro_format, + {"logicalType": "date"}, + datetime.date(2023, 8, 7), + "2023-08-07", + id="test_date"), + pytest.param(_default_avro_format, {"logicalType": "time-millis"}, 70267068, 70267068, id="test_time_millis"), + pytest.param(_default_avro_format, {"logicalType": "time-micros"}, 70267068, 70267068, id="test_time_micros"), + pytest.param(_default_avro_format, + {"logicalType": "local-timestamp-millis"}, + datetime.datetime(2023, 8, 7, 19, 31, 7, 68000, tzinfo=datetime.timezone.utc), + "2023-08-07T19:31:07.068+00:00", + id="test_timestamp_millis"), + pytest.param(_default_avro_format, + {"logicalType": "local-timestamp-micros"}, + datetime.datetime(2023, 8, 7, 19, 31, 7, 68000, tzinfo=datetime.timezone.utc), + "2023-08-07T19:31:07.068000+00:00", + id="test_timestamo_micros"), + ] +) +def test_to_output_value(avro_format, record_type, record_value, expected_value): + parser = AvroParser() + assert parser._to_output_value(avro_format, record_type, record_value) == expected_value diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py new file mode 100644 index 000000000000..de86ad200bc2 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py @@ -0,0 +1,498 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncio +import csv +import io +import logging +import unittest +from datetime import datetime +from typing import Any, Dict, Generator, List, Set +from unittest import TestCase, mock +from unittest.mock import Mock + +import pytest +from airbyte_cdk.sources.file_based.config.csv_format import DEFAULT_FALSE_VALUES, DEFAULT_TRUE_VALUES, CsvFormat, InferenceType +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.exceptions import RecordParseError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.csv_parser import CsvParser, _CsvReader +from airbyte_cdk.sources.file_based.remote_file import RemoteFile + +PROPERTY_TYPES = { + "col1": "null", + "col2": "boolean", + "col3": "integer", + "col4": "number", + "col5": "string", + "col6": "object", + "col7": "array", + "col8": "array", + "col9": "array", + "col10": "string", +} + +logger = logging.getLogger() + + +@pytest.mark.parametrize( + "row, true_values, false_values, expected_output", + [ + pytest.param( + { + "col1": "", + "col2": "true", + "col3": "1", + "col4": "1.1", + "col5": "asdf", + "col6": '{"a": "b"}', + "col7": "[1, 2]", + "col8": '["1", "2"]', + "col9": '[{"a": "b"}, {"a": "c"}]', + "col10": "asdf", + }, + DEFAULT_TRUE_VALUES, + DEFAULT_FALSE_VALUES, + { + "col1": None, + "col2": True, + "col3": 1, + "col4": 1.1, + "col5": "asdf", + "col6": {"a": "b"}, + "col7": [1, 2], + "col8": ["1", "2"], + "col9": [{"a": "b"}, {"a": "c"}], + "col10": "asdf", + }, + id="cast-all-cols", + ), + pytest.param({"col1": "1"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col1": "1"}, id="cannot-cast-to-null"), + pytest.param({"col2": "1"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col2": True}, id="cast-1-to-bool"), + pytest.param({"col2": "0"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col2": False}, id="cast-0-to-bool"), + pytest.param({"col2": "yes"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col2": True}, id="cast-yes-to-bool"), + pytest.param( + {"col2": "this_is_a_true_value"}, + ["this_is_a_true_value"], + DEFAULT_FALSE_VALUES, + {"col2": True}, + id="cast-custom-true-value-to-bool", + ), + pytest.param( + {"col2": "this_is_a_false_value"}, + DEFAULT_TRUE_VALUES, + ["this_is_a_false_value"], + {"col2": False}, + id="cast-custom-false-value-to-bool", + ), + pytest.param({"col2": "no"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col2": False}, id="cast-no-to-bool"), + pytest.param({"col2": "10"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col2": "10"}, id="cannot-cast-to-bool"), + pytest.param({"col3": "1.1"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col3": "1.1"}, id="cannot-cast-to-int"), + pytest.param({"col4": "asdf"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col4": "asdf"}, id="cannot-cast-to-float"), + pytest.param({"col6": "{'a': 'b'}"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col6": "{'a': 'b'}"}, id="cannot-cast-to-dict"), + pytest.param( + {"col7": "['a', 'b']"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col7": "['a', 'b']"}, id="cannot-cast-to-list-of-ints" + ), + pytest.param( + {"col8": "['a', 'b']"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col8": "['a', 'b']"}, id="cannot-cast-to-list-of-strings" + ), + pytest.param( + {"col9": "['a', 'b']"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {"col9": "['a', 'b']"}, id="cannot-cast-to-list-of-objects" + ), + pytest.param({"col11": "x"}, DEFAULT_TRUE_VALUES, DEFAULT_FALSE_VALUES, {}, id="item-not-in-props-doesn't-error"), + ], +) +def test_cast_to_python_type(row: Dict[str, str], true_values: Set[str], false_values: Set[str], expected_output: Dict[str, Any]) -> None: + csv_format = CsvFormat(true_values=true_values, false_values=false_values) + assert CsvParser._cast_types(row, PROPERTY_TYPES, csv_format, logger) == expected_output + + +@pytest.mark.parametrize( + "row, strings_can_be_null, expected_output", + [ + pytest.param( + {"id": "1", "name": "bob", "age": 10, "is_cool": False}, + False, + {"id": "1", "name": "bob", "age": 10, "is_cool": False}, + id="test-no-values-are-null", + ), + pytest.param( + {"id": "1", "name": "bob", "age": "null", "is_cool": "null"}, + False, + {"id": "1", "name": "bob", "age": None, "is_cool": None}, + id="test-non-string-values-are-none-if-in-null-values", + ), + pytest.param( + {"id": "1", "name": "null", "age": 10, "is_cool": False}, + False, + {"id": "1", "name": "null", "age": 10, "is_cool": False}, + id="test-string-values-are-not-none-if-strings-cannot-be-null", + ), + pytest.param( + {"id": "1", "name": "null", "age": 10, "is_cool": False}, + True, + {"id": "1", "name": None, "age": 10, "is_cool": False}, + id="test-string-values-none-if-strings-can-be-null", + ), + ], +) +def test_to_nullable(row, strings_can_be_null, expected_output): + property_types = {"id": "string", "name": "string", "age": "integer", "is_cool": "boolean"} + null_values = {"null"} + nulled_row = CsvParser._to_nullable(row, property_types, null_values, strings_can_be_null) + assert nulled_row == expected_output + + +_DEFAULT_TRUE_VALUES = {"1", "yes", "yeah", "right"} +_DEFAULT_FALSE_VALUES = {"0", "no", "nop", "wrong"} + + +class SchemaInferenceTestCase(TestCase): + _A_NULL_VALUE = "null" + _HEADER_NAME = "header" + + def setUp(self) -> None: + self._config_format = CsvFormat() + self._config_format.true_values = _DEFAULT_TRUE_VALUES + self._config_format.false_values = _DEFAULT_FALSE_VALUES + self._config_format.null_values = {self._A_NULL_VALUE} + self._config_format.inference_type = InferenceType.NONE + self._config = Mock() + self._config.get_input_schema.return_value = None + self._config.format = self._config_format + + self._file = Mock(spec=RemoteFile) + self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) + self._logger = Mock(spec=logging.Logger) + self._csv_reader = Mock(spec=_CsvReader) + self._parser = CsvParser(self._csv_reader) + + def test_given_user_schema_defined_when_infer_schema_then_return_user_schema(self) -> None: + self._config.get_input_schema.return_value = {self._HEADER_NAME: {"type": "potato"}} + self._test_infer_schema(list(_DEFAULT_TRUE_VALUES.union(_DEFAULT_FALSE_VALUES)), "potato") + + def test_given_booleans_only_when_infer_schema_then_type_is_boolean(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(list(_DEFAULT_TRUE_VALUES.union(_DEFAULT_FALSE_VALUES)), "boolean") + + def test_given_integers_only_when_infer_schema_then_type_is_integer(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(["2", "90329", "5645"], "integer") + + def test_given_integer_overlap_with_bool_value_only_when_infer_schema_then_type_is_integer(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(["1", "90329", "5645"], "integer") # here, "1" is also considered a boolean + + def test_given_numbers_and_integers_when_infer_schema_then_type_is_number(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(["2", "90329", "2.312"], "number") + + def test_given_arrays_when_infer_schema_then_type_is_string(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(['["first_item", "second_item"]', '["first_item_again", "second_item_again"]'], "string") + + def test_given_objects_when_infer_schema_then_type_is_object(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(['{"object1_key": 1}', '{"object2_key": 2}'], "string") + + def test_given_strings_only_when_infer_schema_then_type_is_string(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(["a string", "another string"], "string") + + def test_given_a_null_value_when_infer_then_ignore_null(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema(["2", "90329", "5645", self._A_NULL_VALUE], "integer") + + def test_given_only_null_values_when_infer_then_type_is_string(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._test_infer_schema([self._A_NULL_VALUE, self._A_NULL_VALUE, self._A_NULL_VALUE], "string") + + def test_given_big_file_when_infer_schema_then_stop_early(self) -> None: + self._config_format.inference_type = InferenceType.PRIMITIVE_TYPES_ONLY + self._csv_reader.read_data.return_value = ({self._HEADER_NAME: row} for row in ["2." + "2" * 1_000_000] + ["this is a string"]) + inferred_schema = self._infer_schema() + # since the type is number, we know the string at the end was not considered + assert inferred_schema == {self._HEADER_NAME: {"type": "number"}} + + def _test_infer_schema(self, rows: List[str], expected_type: str) -> None: + self._csv_reader.read_data.return_value = ({self._HEADER_NAME: row} for row in rows) + inferred_schema = self._infer_schema() + assert inferred_schema == {self._HEADER_NAME: {"type": expected_type}} + + def _infer_schema(self): + loop = asyncio.new_event_loop() + task = loop.create_task(self._parser.infer_schema(self._config, self._file, self._stream_reader, self._logger)) + loop.run_until_complete(task) + return task.result() + + +class CsvFileBuilder: + def __init__(self) -> None: + self._prefixed_rows: List[str] = [] + self._data: List[str] = [] + + def with_prefixed_rows(self, rows: List[str]) -> "CsvFileBuilder": + self._prefixed_rows = rows + return self + + def with_data(self, data: List[str]) -> "CsvFileBuilder": + self._data = data + return self + + def build(self) -> io.StringIO: + return io.StringIO("\n".join(self._prefixed_rows + self._data)) + + +class CsvReaderTest(unittest.TestCase): + _CONFIG_NAME = "config_name" + + def setUp(self) -> None: + self._config_format = CsvFormat() + self._config = Mock() + self._config.name = self._CONFIG_NAME + self._config.format = self._config_format + + self._file = Mock(spec=RemoteFile) + self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) + self._logger = Mock(spec=logging.Logger) + self._csv_reader = _CsvReader() + + def test_given_skip_rows_when_read_data_then_do_not_considered_prefixed_rows(self) -> None: + self._config_format.skip_rows_before_header = 2 + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_prefixed_rows(["first line", "second line"]) + .with_data( + [ + "header", + "a value", + "another value", + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"header": "a value"}, {"header": "another value"}] + + def test_given_autogenerated_headers_when_read_data_then_generate_headers_with_format_fX(self) -> None: + self._config_format.autogenerate_column_names = True + self._stream_reader.open_file.return_value = CsvFileBuilder().with_data(["0,1,2,3,4,5,6"]).build() + + data_generator = self._read_data() + + assert list(data_generator) == [{"f0": "0", "f1": "1", "f2": "2", "f3": "3", "f4": "4", "f5": "5", "f6": "6"}] + + def test_given_skip_rows_after_header_when_read_data_then_do_not_parse_skipped_rows(self) -> None: + self._config_format.skip_rows_after_header = 1 + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1,header2", + "skipped row: important that the is no comma in this string to test if columns do not match in skipped rows", + "a value 1,a value 2", + "another value 1,another value 2", + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [ + {"header1": "a value 1", "header2": "a value 2"}, + {"header1": "another value 1", "header2": "another value 2"}, + ] + + def test_given_quote_delimiter_when_read_data_then_parse_properly(self) -> None: + self._config_format.delimiter = "|" + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1|header2", + "a value 1|a value 2", + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"header1": "a value 1", "header2": "a value 2"}] + + def test_given_quote_char_when_read_data_then_parse_properly(self) -> None: + self._config_format.quote_char = "|" + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1,header2", + "|a,value,1|,|a,value,2|", + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"header1": "a,value,1", "header2": "a,value,2"}] + + def test_given_escape_char_when_read_data_then_parse_properly(self) -> None: + self._config_format.escape_char = "|" + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1,header2", + '"a |"value|", 1",a value 2', + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"header1": 'a "value", 1', "header2": "a value 2"}] + + def test_given_double_quote_on_when_read_data_then_parse_properly(self) -> None: + self._config_format.double_quote = True + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1,header2", + '1,"Text with doublequote: ""This is a text."""', + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"header1": "1", "header2": 'Text with doublequote: "This is a text."'}] + + def test_given_double_quote_off_when_read_data_then_parse_properly(self) -> None: + self._config_format.double_quote = False + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1,header2", + '1,"Text with doublequote: ""This is a text."""', + ] + ) + .build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"header1": "1", "header2": 'Text with doublequote: "This is a text."""'}] + + def test_given_generator_closed_when_read_data_then_unregister_dialect(self) -> None: + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header", + "a value", + "another value", + ] + ) + .build() + ) + + data_generator = self._read_data() + next(data_generator) + assert f"{self._CONFIG_NAME}_config_dialect" in csv.list_dialects() + data_generator.close() + assert f"{self._CONFIG_NAME}_config_dialect" not in csv.list_dialects() + + def test_given_too_many_values_for_columns_when_read_data_then_raise_exception_and_unregister_dialect(self) -> None: + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header", + "a value", + "too many values,value,value,value", + ] + ) + .build() + ) + + data_generator = self._read_data() + next(data_generator) + assert f"{self._CONFIG_NAME}_config_dialect" in csv.list_dialects() + + with pytest.raises(RecordParseError): + next(data_generator) + assert f"{self._CONFIG_NAME}_config_dialect" not in csv.list_dialects() + + def test_given_too_few_values_for_columns_when_read_data_then_raise_exception_and_unregister_dialect(self) -> None: + self._stream_reader.open_file.return_value = ( + CsvFileBuilder() + .with_data( + [ + "header1,header2,header3", + "value1,value2,value3", + "a value", + ] + ) + .build() + ) + + data_generator = self._read_data() + next(data_generator) + assert f"{self._CONFIG_NAME}_config_dialect" in csv.list_dialects() + + with pytest.raises(RecordParseError): + next(data_generator) + assert f"{self._CONFIG_NAME}_config_dialect" not in csv.list_dialects() + + def _read_data(self) -> Generator[Dict[str, str], None, None]: + data_generator = self._csv_reader.read_data( + self._config, + self._file, + self._stream_reader, + self._logger, + FileReadMode.READ, + ) + return data_generator + + +def test_encoding_is_passed_to_stream_reader() -> None: + parser = CsvParser() + encoding = "ascii" + stream_reader = Mock() + mock_obj = stream_reader.open_file.return_value + mock_obj.__enter__ = Mock(return_value=io.StringIO("c1,c2\nv1,v2")) + mock_obj.__exit__ = Mock(return_value=None) + file = RemoteFile(uri="s3://bucket/key.csv", last_modified=datetime.now()) + config = FileBasedStreamConfig(name="test", validation_policy="Emit Record", file_type="csv", format=CsvFormat(encoding=encoding)) + list(parser.parse_records(config, file, stream_reader, logger, {"properties": {"c1": {"type": "string"}, "c2": {"type": "string"}}})) + stream_reader.open_file.assert_has_calls( + [ + mock.call(file, FileReadMode.READ, encoding, logger), + mock.call().__enter__(), + mock.call().__exit__(None, None, None), + ] + ) + + mock_obj.__enter__ = Mock(return_value=io.StringIO("c1,c2\nv1,v2")) + loop = asyncio.get_event_loop() + loop.run_until_complete(parser.infer_schema(config, file, stream_reader, logger)) + stream_reader.open_file.assert_called_with(file, FileReadMode.READ, encoding, logger) + stream_reader.open_file.assert_has_calls( + [ + mock.call(file, FileReadMode.READ, encoding, logger), + mock.call().__enter__(), + mock.call().__exit__(None, None, None), + mock.call(file, FileReadMode.READ, encoding, logger), + mock.call().__enter__(), + mock.call().__exit__(None, None, None), + ] + ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_jsonl_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_jsonl_parser.py new file mode 100644 index 000000000000..af5d83d77d04 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_jsonl_parser.py @@ -0,0 +1,158 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncio +import io +import json +from typing import Any, Dict +from unittest.mock import MagicMock, Mock + +import pytest +from airbyte_cdk.sources.file_based.exceptions import RecordParseError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types import JsonlParser + +JSONL_CONTENT_WITHOUT_MULTILINE_JSON_OBJECTS = [ + b'{"a": 1, "b": "1"}', + b'{"a": 2, "b": "2"}', +] +JSONL_CONTENT_WITH_MULTILINE_JSON_OBJECTS = [ + b"{", + b' "a": 1,', + b' "b": "1"', + b"}", + b"{", + b' "a": 2,', + b' "b": "2"', + b"}", +] +INVALID_JSON_CONTENT = [ + b"{", + b' "a": 1,', + b' "b": "1"', + b"{", + b' "a": 2,', + b' "b": "2"', + b"}", +] + + +@pytest.fixture +def stream_reader() -> MagicMock: + return MagicMock(spec=AbstractFileBasedStreamReader) + + +def _infer_schema(stream_reader: MagicMock) -> Dict[str, Any]: + loop = asyncio.new_event_loop() + task = loop.create_task(JsonlParser().infer_schema(Mock(), Mock(), stream_reader, Mock())) + loop.run_until_complete(task) + return task.result() # type: ignore # asyncio has no typing + + +def test_when_infer_then_return_proper_types(stream_reader: MagicMock) -> None: + record = {"col1": 1, "col2": 2.2, "col3": "3", "col4": ["a", "list"], "col5": {"inner": "obj"}, "col6": None, "col7": True} + stream_reader.open_file.return_value.__enter__.return_value = io.BytesIO(json.dumps(record).encode("utf-8")) + + schema = _infer_schema(stream_reader) + + assert schema == { + "col1": {"type": "integer"}, + "col2": {"type": "number"}, + "col3": {"type": "string"}, + "col4": {"type": "array"}, + "col5": {"type": "object"}, + "col6": {"type": "null"}, + "col7": {"type": "boolean"}, + } + + +def test_given_str_io_when_infer_then_return_proper_types(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = io.StringIO('{"col": 1}') + + schema = _infer_schema(stream_reader) + + assert schema == {"col": {"type": "integer"}} + + +def test_given_empty_record_when_infer_then_return_empty_schema(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = io.BytesIO("{}".encode("utf-8")) + schema = _infer_schema(stream_reader) + assert schema == {} + + +def test_given_no_records_when_infer_then_return_empty_schema(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = io.BytesIO("".encode("utf-8")) + schema = _infer_schema(stream_reader) + assert schema == {} + + +def test_given_limit_hit_when_infer_then_stop_considering_records(stream_reader: MagicMock) -> None: + jsonl_file_content = '{"key": 2.' + "2" * JsonlParser.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE + '}\n{"key": "a string"}' + stream_reader.open_file.return_value.__enter__.return_value = io.BytesIO(jsonl_file_content.encode("utf-8")) + + schema = _infer_schema(stream_reader) + + assert schema == {"key": {"type": "number"}} + + +def test_given_multiline_json_objects_and_read_limit_hit_when_infer_then_return_parse_until_at_least_one_record( + stream_reader: MagicMock, +) -> None: + jsonl_file_content = '{\n"key": 2.' + "2" * JsonlParser.MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE + "\n}" + stream_reader.open_file.return_value.__enter__.return_value = io.BytesIO(jsonl_file_content.encode("utf-8")) + + schema = _infer_schema(stream_reader) + + assert schema == {"key": {"type": "number"}} + + +def test_given_multiline_json_objects_and_hits_read_limit_when_infer_then_return_proper_types(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = JSONL_CONTENT_WITH_MULTILINE_JSON_OBJECTS + schema = _infer_schema(stream_reader) + assert schema == {"a": {"type": "integer"}, "b": {"type": "string"}} + + +def test_given_multiple_records_then_merge_types(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = io.BytesIO('{"col1": 1}\n{"col1": 2.3}'.encode("utf-8")) + schema = _infer_schema(stream_reader) + assert schema == {"col1": {"type": "number"}} + + +def test_given_one_json_per_line_when_parse_records_then_return_records(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = JSONL_CONTENT_WITHOUT_MULTILINE_JSON_OBJECTS + records = list(JsonlParser().parse_records(Mock(), Mock(), stream_reader, Mock(), None)) + assert records == [{"a": 1, "b": "1"}, {"a": 2, "b": "2"}] + + +def test_given_one_json_per_line_when_parse_records_then_do_not_send_warning(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = JSONL_CONTENT_WITHOUT_MULTILINE_JSON_OBJECTS + logger = Mock() + + list(JsonlParser().parse_records(Mock(), Mock(), stream_reader, logger, None)) + + assert logger.warning.call_count == 0 + + +def test_given_multiline_json_object_when_parse_records_then_return_records(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = JSONL_CONTENT_WITH_MULTILINE_JSON_OBJECTS + records = list(JsonlParser().parse_records(Mock(), Mock(), stream_reader, Mock(), None)) + assert records == [{"a": 1, "b": "1"}, {"a": 2, "b": "2"}] + + +def test_given_multiline_json_object_when_parse_records_then_log_once_one_record_yielded(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = JSONL_CONTENT_WITH_MULTILINE_JSON_OBJECTS + logger = Mock() + + next(iter(JsonlParser().parse_records(Mock(), Mock(), stream_reader, logger, None))) + + assert logger.warning.call_count == 1 + + +def test_given_unparsable_json_when_parse_records_then_raise_error(stream_reader: MagicMock) -> None: + stream_reader.open_file.return_value.__enter__.return_value = INVALID_JSON_CONTENT + logger = Mock() + + with pytest.raises(RecordParseError): + list(JsonlParser().parse_records(Mock(), Mock(), stream_reader, logger, None)) + assert logger.warning.call_count == 0 diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py new file mode 100644 index 000000000000..028be78b5b5e --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py @@ -0,0 +1,180 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncio +import datetime +import math +from typing import Any, Mapping, Union +from unittest.mock import Mock + +import pyarrow as pa +import pytest +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy +from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat +from airbyte_cdk.sources.file_based.file_types import ParquetParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from pyarrow import Scalar + +_default_parquet_format = ParquetFormat() +_decimal_as_float_parquet_format = ParquetFormat(decimal_as_float=True) + + +@pytest.mark.parametrize( + "parquet_type, expected_type, parquet_format", + [ + pytest.param(pa.bool_(), {"type": "boolean"}, _default_parquet_format, id="test_parquet_bool"), + pytest.param(pa.int8(), {"type": "integer"}, _default_parquet_format, id="test_parquet_int8"), + pytest.param(pa.int16(), {"type": "integer"}, _default_parquet_format, id="test_parquet_int16"), + pytest.param(pa.int32(), {"type": "integer"}, _default_parquet_format, id="test_parquet_int32"), + pytest.param(pa.int64(), {"type": "integer"}, _default_parquet_format, id="test_parquet_int64"), + pytest.param(pa.uint8(), {"type": "integer"}, _default_parquet_format, id="test_parquet_uint8"), + pytest.param(pa.uint16(), {"type": "integer"}, _default_parquet_format, id="test_parquet_uint16"), + pytest.param(pa.uint32(), {"type": "integer"}, _default_parquet_format, id="test_parquet_uint32"), + pytest.param(pa.uint64(), {"type": "integer"}, _default_parquet_format, id="test_parquet_uint64"), + pytest.param(pa.float16(), {"type": "number"}, _default_parquet_format, id="test_parquet_float16"), + pytest.param(pa.float32(), {"type": "number"}, _default_parquet_format, id="test_parquet_float32"), + pytest.param(pa.float64(), {"type": "number"}, _default_parquet_format, id="test_parquet_float64"), + pytest.param(pa.time32("s"), {"type": "string"}, _default_parquet_format, id="test_parquet_time32s"), + pytest.param(pa.time32("ms"), {"type": "string"}, _default_parquet_format, id="test_parquet_time32ms"), + pytest.param(pa.time64("us"), {"type": "string"}, _default_parquet_format, id="test_parquet_time64us"), + pytest.param(pa.time64("ns"), {"type": "string"}, _default_parquet_format, id="test_parquet_time64us"), + pytest.param(pa.timestamp("s"), {"type": "string", "format": "date-time"}, _default_parquet_format, id="test_parquet_timestamps_s"), + pytest.param(pa.timestamp("ms"), {"type": "string", "format": "date-time"}, _default_parquet_format, + id="test_parquet_timestamp_ms"), + pytest.param(pa.timestamp("s", "utc"), {"type": "string", "format": "date-time"}, _default_parquet_format, + id="test_parquet_timestamps_s_with_tz"), + pytest.param(pa.timestamp("ms", "est"), {"type": "string", "format": "date-time"}, _default_parquet_format, + id="test_parquet_timestamps_ms_with_tz"), + pytest.param(pa.date32(), {"type": "string", "format": "date"}, _default_parquet_format, id="test_parquet_date32"), + pytest.param(pa.date64(), {"type": "string", "format": "date"}, _default_parquet_format, id="test_parquet_date64"), + pytest.param(pa.duration("s"), {"type": "integer"}, _default_parquet_format, id="test_duration_s"), + pytest.param(pa.duration("ms"), {"type": "integer"}, _default_parquet_format, id="test_duration_ms"), + pytest.param(pa.duration("us"), {"type": "integer"}, _default_parquet_format, id="test_duration_us"), + pytest.param(pa.duration("ns"), {"type": "integer"}, _default_parquet_format, id="test_duration_ns"), + pytest.param(pa.month_day_nano_interval(), {"type": "array"}, _default_parquet_format, id="test_parquet_month_day_nano_interval"), + pytest.param(pa.binary(), {"type": "string"}, _default_parquet_format, id="test_binary"), + pytest.param(pa.binary(2), {"type": "string"}, _default_parquet_format, id="test_fixed_size_binary"), + pytest.param(pa.string(), {"type": "string"}, _default_parquet_format, id="test_parquet_string"), + pytest.param(pa.utf8(), {"type": "string"}, _default_parquet_format, id="test_utf8"), + pytest.param(pa.large_binary(), {"type": "string"}, _default_parquet_format, id="test_large_binary"), + pytest.param(pa.large_string(), {"type": "string"}, _default_parquet_format, id="test_large_string"), + pytest.param(pa.large_utf8(), {"type": "string"}, _default_parquet_format, id="test_large_utf8"), + pytest.param(pa.dictionary(pa.int32(), pa.string()), {"type": "object"}, _default_parquet_format, id="test_dictionary"), + pytest.param(pa.struct([pa.field("field", pa.int32())]), {"type": "object"}, _default_parquet_format, id="test_struct"), + pytest.param(pa.list_(pa.int32()), {"type": "array"}, _default_parquet_format, id="test_list"), + pytest.param(pa.large_list(pa.int32()), {"type": "array"}, _default_parquet_format, id="test_large_list"), + pytest.param(pa.decimal128(2), {"type": "string"}, _default_parquet_format, id="test_decimal128"), + pytest.param(pa.decimal256(2), {"type": "string"}, _default_parquet_format, id="test_decimal256"), + pytest.param(pa.decimal128(2), {"type": "number"}, _decimal_as_float_parquet_format, id="test_decimal128_as_float"), + pytest.param(pa.decimal256(2), {"type": "number"}, _decimal_as_float_parquet_format, id="test_decimal256_as_float"), + pytest.param(pa.map_(pa.int32(), pa.int32()), {"type": "object"}, _default_parquet_format, id="test_map"), + pytest.param(pa.null(), {"type": "null"}, _default_parquet_format, id="test_null"), + ] +) +def test_type_mapping(parquet_type: pa.DataType, expected_type: Mapping[str, str], parquet_format: ParquetFormat) -> None: + if expected_type is None: + with pytest.raises(ValueError): + ParquetParser.parquet_type_to_schema_type(parquet_type, parquet_format) + else: + assert ParquetParser.parquet_type_to_schema_type(parquet_type, parquet_format) == expected_type + + +@pytest.mark.parametrize( + "pyarrow_type, parquet_format, parquet_object, expected_value", + [ + pytest.param(pa.bool_(), _default_parquet_format, True, True, id="test_bool"), + pytest.param(pa.int8(), _default_parquet_format, -1, -1, id="test_int8"), + pytest.param(pa.int16(), _default_parquet_format, 2, 2, id="test_int16"), + pytest.param(pa.int32(), _default_parquet_format, 3, 3, id="test_int32"), + pytest.param(pa.int64(), _default_parquet_format, 4, 4, id="test_int64"), + pytest.param(pa.uint8(), _default_parquet_format, 4, 4, id="test_parquet_uint8"), + pytest.param(pa.uint16(), _default_parquet_format, 5, 5, id="test_parquet_uint16"), + pytest.param(pa.uint32(), _default_parquet_format, 6, 6, id="test_parquet_uint32"), + pytest.param(pa.uint64(), _default_parquet_format, 6, 6, id="test_parquet_uint64"), + pytest.param(pa.float32(), _default_parquet_format, 2.7, 2.7, id="test_parquet_float32"), + pytest.param(pa.float64(), _default_parquet_format, 3.14, 3.14, id="test_parquet_float64"), + pytest.param(pa.time32("s"), _default_parquet_format, datetime.time(1, 2, 3), "01:02:03", id="test_parquet_time32s"), + pytest.param(pa.time32("ms"), _default_parquet_format, datetime.time(3, 4, 5), "03:04:05", id="test_parquet_time32ms"), + pytest.param(pa.time64("us"), _default_parquet_format, datetime.time(6, 7, 8), "06:07:08", id="test_parquet_time64us"), + pytest.param(pa.time64("ns"), _default_parquet_format, datetime.time(9, 10, 11), "09:10:11", id="test_parquet_time64us"), + pytest.param(pa.timestamp("s"), _default_parquet_format, datetime.datetime(2023, 7, 7, 10, 11, 12), "2023-07-07T10:11:12", + id="test_parquet_timestamps_s"), + pytest.param(pa.timestamp("ms"), _default_parquet_format, datetime.datetime(2024, 8, 8, 11, 12, 13), "2024-08-08T11:12:13", + id="test_parquet_timestamp_ms"), + pytest.param(pa.timestamp("s", "utc"), _default_parquet_format, + datetime.datetime(2020, 1, 1, 1, 1, 1, tzinfo=datetime.timezone.utc), + "2020-01-01T01:01:01+00:00", id="test_parquet_timestamps_s_with_tz"), + pytest.param(pa.timestamp("ms", "utc"), _default_parquet_format, datetime.datetime(2021, 2, 3, 4, 5, tzinfo=datetime.timezone.utc), + "2021-02-03T04:05:00+00:00", id="test_parquet_timestamps_ms_with_tz"), + pytest.param(pa.date32(), _default_parquet_format, datetime.date(2023, 7, 7), "2023-07-07", id="test_parquet_date32"), + pytest.param(pa.date64(), _default_parquet_format, datetime.date(2023, 7, 8), "2023-07-08", id="test_parquet_date64"), + pytest.param(pa.duration("s"), _default_parquet_format, 12345, 12345, id="test_duration_s"), + pytest.param(pa.duration("ms"), _default_parquet_format, 12345, 12345, id="test_duration_ms"), + pytest.param(pa.duration("us"), _default_parquet_format, 12345, 12345, id="test_duration_us"), + pytest.param(pa.duration("ns"), _default_parquet_format, 12345, 12345, id="test_duration_ns"), + pytest.param(pa.month_day_nano_interval(), _default_parquet_format, datetime.timedelta(days=3, microseconds=4), [0, 3, 4000], + id="test_parquet_month_day_nano_interval"), + pytest.param(pa.binary(), _default_parquet_format, b"this is a binary string", "this is a binary string", id="test_binary"), + pytest.param(pa.binary(2), _default_parquet_format, b"t1", "t1", id="test_fixed_size_binary"), + pytest.param(pa.string(), _default_parquet_format, "this is a string", "this is a string", id="test_parquet_string"), + pytest.param(pa.utf8(), _default_parquet_format, "utf8".encode("utf8"), "utf8", id="test_utf8"), + pytest.param(pa.large_binary(), _default_parquet_format, b"large binary string", "large binary string", id="test_large_binary"), + pytest.param(pa.large_string(), _default_parquet_format, "large string", "large string", id="test_large_string"), + pytest.param(pa.large_utf8(), _default_parquet_format, "large utf8", "large utf8", id="test_large_utf8"), + pytest.param(pa.struct([pa.field("field", pa.int32())]), _default_parquet_format, {"field": 1}, {"field": 1}, id="test_struct"), + pytest.param(pa.list_(pa.int32()), _default_parquet_format, [1, 2, 3], [1, 2, 3], id="test_list"), + pytest.param(pa.large_list(pa.int32()), _default_parquet_format, [4, 5, 6], [4, 5, 6], id="test_large_list"), + pytest.param(pa.decimal128(5, 3), _default_parquet_format, 12, "12.000", id="test_decimal128"), + pytest.param(pa.decimal256(8, 2), _default_parquet_format, 13, "13.00", id="test_decimal256"), + pytest.param(pa.decimal128(5, 3), _decimal_as_float_parquet_format, 12, 12.000, id="test_decimal128"), + pytest.param(pa.decimal256(8, 2), _decimal_as_float_parquet_format, 13, 13.00, id="test_decimal256"), + pytest.param(pa.map_(pa.string(), pa.int32()), _default_parquet_format, {"hello": 1, "world": 2}, {"hello": 1, "world": 2}, + id="test_map"), + pytest.param(pa.null(), _default_parquet_format, None, None, id="test_null"), + ] +) +def test_value_transformation(pyarrow_type: pa.DataType, parquet_format: ParquetFormat, parquet_object: Scalar, + expected_value: Any) -> None: + pyarrow_value = pa.array([parquet_object], type=pyarrow_type)[0] + py_value = ParquetParser._to_output_value(pyarrow_value, parquet_format) + if isinstance(py_value, float): + assert math.isclose(py_value, expected_value, abs_tol=0.01) + else: + assert py_value == expected_value + + +def test_value_dictionary() -> None: + # Setting the dictionary is more involved than other data types so we test it in a separate test + dictionary_values = ["apple", "banana", "cherry"] + indices = [0, 1, 2, 0, 1] + indices_array = pa.array(indices, type=pa.int8()) + dictionary = pa.DictionaryArray.from_arrays(indices_array, dictionary_values) + py_value = ParquetParser._to_output_value(dictionary, _default_parquet_format) + assert py_value == {"indices": [0, 1, 2, 0, 1], "values": ["apple", "banana", "cherry"]} + + +@pytest.mark.parametrize( + "file_format", [ + pytest.param(CsvFormat( + filetype="csv", + delimiter=",", + escape_char="\\", + quote_char='"', + ), id="test_csv_format"), + pytest.param(JsonlFormat(), id="test_jsonl_format"), + ] +) +def test_wrong_file_format(file_format: Union[CsvFormat, JsonlFormat]) -> None: + parser = ParquetParser() + config = FileBasedStreamConfig(name="test.parquet", file_type=file_format.filetype, format={file_format.filetype: file_format}, + validation_policy=ValidationPolicy.emit_record) + file = RemoteFile(uri="s3://mybucket/test.parquet", last_modified=datetime.datetime.now()) + stream_reader = Mock() + logger = Mock() + with pytest.raises(ValueError): + asyncio.get_event_loop().run_until_complete( + parser.infer_schema(config, file, stream_reader, logger) + ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py b/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py new file mode 100644 index 000000000000..491b5658ad0c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime +from io import IOBase +from typing import Any, Dict, List, Mapping, Optional + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.discovery_policy import DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.csv_parser import CsvParser +from airbyte_cdk.sources.file_based.file_types.jsonl_parser import JsonlParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor +from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesStreamReader + + +class EmptySchemaParser(CsvParser): + async def infer_schema(self, config: FileBasedStreamConfig, file: RemoteFile, stream_reader: AbstractFileBasedStreamReader, logger: logging.Logger) -> Dict[str, Any]: + return {} + + +class LowInferenceLimitDiscoveryPolicy(DefaultDiscoveryPolicy): + @property + def max_n_files_for_schema_inference(self) -> int: + return 1 + + +class LowInferenceBytesJsonlParser(JsonlParser): + MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE = 1 + + +class TestErrorListMatchingFilesInMemoryFilesStreamReader(InMemoryFilesStreamReader): + def get_matching_files( + self, + globs: List[str], + from_date: Optional[datetime] = None, + ) -> List[RemoteFile]: + raise Exception("Error listing files") + + +class TestErrorOpenFileInMemoryFilesStreamReader(InMemoryFilesStreamReader): + def open_file(self, file: RemoteFile, file_read_mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + raise Exception("Error opening file") + + +class FailingSchemaValidationPolicy(AbstractSchemaValidationPolicy): + ALWAYS_FAIL = "always_fail" + validate_schema_before_sync = True + + def record_passes_validation_policy(self, record: Mapping[str, Any], schema: Optional[Mapping[str, Any]]) -> bool: + return False + + +class LowHistoryLimitCursor(DefaultFileBasedCursor): + DEFAULT_MAX_HISTORY_SIZE = 3 + + +def make_remote_files(files: List[str]) -> List[RemoteFile]: + return [ + RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) + for f in files + ] diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py new file mode 100644 index 000000000000..ca25289bbf2c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py @@ -0,0 +1,185 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import csv +import io +import json +import logging +import tempfile +from datetime import datetime +from io import IOBase +from typing import Any, Iterable, List, Mapping, Optional + +import avro.io as ai +import avro.schema as avro_schema +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES, AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor, DefaultFileBasedCursor +from avro import datafile +from pydantic import AnyUrl + + +class InMemoryFilesSource(FileBasedSource): + def __init__( + self, + files: Mapping[str, Any], + file_type: str, + availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy], + discovery_policy: Optional[AbstractDiscoveryPolicy], + validation_policies: Mapping[str, AbstractSchemaValidationPolicy], + parsers: Mapping[str, FileTypeParser], + stream_reader: Optional[AbstractFileBasedStreamReader], + catalog: Optional[Mapping[str, Any]], + file_write_options: Mapping[str, Any], + cursor_cls: Optional[AbstractFileBasedCursor], + ): + # Attributes required for test purposes + self.files = files + self.file_type = file_type + self.catalog = catalog + + # Source setup + stream_reader = stream_reader or InMemoryFilesStreamReader(files=files, file_type=file_type, file_write_options=file_write_options) + availability_strategy = availability_strategy or DefaultFileBasedAvailabilityStrategy(stream_reader) # type: ignore[assignment] + super().__init__( + stream_reader, + spec_class=InMemorySpec, + catalog_path="fake_path" if catalog else None, + availability_strategy=availability_strategy, + discovery_policy=discovery_policy or DefaultDiscoveryPolicy(), + parsers=parsers, + validation_policies=validation_policies or DEFAULT_SCHEMA_VALIDATION_POLICIES, + cursor_cls=cursor_cls or DefaultFileBasedCursor, + ) + + def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog(streams=self.catalog["streams"]) if self.catalog else None + + +class InMemoryFilesStreamReader(AbstractFileBasedStreamReader): + def __init__(self, files: Mapping[str, Mapping[str, Any]], file_type: str, file_write_options: Optional[Mapping[str, Any]] = None): + self.files = files + self.file_type = file_type + self.file_write_options = file_write_options + super().__init__() + + @property + def config(self) -> Optional[AbstractFileBasedSpec]: + return self._config + + @config.setter + def config(self, value: AbstractFileBasedSpec) -> None: + self._config = value + + def get_matching_files( + self, + globs: List[str], + prefix: Optional[str], + logger: logging.Logger, + ) -> Iterable[RemoteFile]: + yield from self.filter_files_by_globs_and_start_date([ + RemoteFile(uri=f, last_modified=datetime.strptime(data["last_modified"], "%Y-%m-%dT%H:%M:%S.%fZ")) + for f, data in self.files.items() + ], globs) + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + if self.file_type == "csv": + return self._make_csv_file_contents(file.uri) + elif self.file_type == "jsonl": + return self._make_jsonl_file_contents(file.uri) + else: + raise NotImplementedError(f"No implementation for file type: {self.file_type}") + + def _make_csv_file_contents(self, file_name: str) -> IOBase: + + # Some tests define the csv as an array of strings to make it easier to validate the handling + # of quotes, delimiter, and escpare chars. + if isinstance(self.files[file_name]["contents"][0], str): + return io.StringIO("\n".join([s.strip() for s in self.files[file_name]["contents"]])) + + fh = io.StringIO() + + if self.file_write_options: + csv.register_dialect("in_memory_dialect", **self.file_write_options) + writer = csv.writer(fh, dialect="in_memory_dialect") + writer.writerows(self.files[file_name]["contents"]) + csv.unregister_dialect("in_memory_dialect") + else: + writer = csv.writer(fh) + writer.writerows(self.files[file_name]["contents"]) + fh.seek(0) + return fh + + def _make_jsonl_file_contents(self, file_name: str) -> IOBase: + fh = io.BytesIO() + + for line in self.files[file_name]["contents"]: + try: + fh.write((json.dumps(line) + "\n").encode("utf-8")) + except TypeError: + # Intentionally trigger json validation error + fh.write((str(line) + "\n").encode("utf-8")) + fh.seek(0) + return fh + + +class InMemorySpec(AbstractFileBasedSpec): + @classmethod + def documentation_url(cls) -> AnyUrl: + return AnyUrl(scheme="https", url="https://docs.airbyte.com/integrations/sources/in_memory_files") # type: ignore + + +class TemporaryParquetFilesStreamReader(InMemoryFilesStreamReader): + """ + A file reader that writes RemoteFiles to a temporary file and then reads them back. + """ + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + return io.BytesIO(self._create_file(file.uri)) + + def _create_file(self, file_name: str) -> bytes: + contents = self.files[file_name]["contents"] + schema = self.files[file_name].get("schema") + + df = pd.DataFrame(contents[1:], columns=contents[0]) + with tempfile.TemporaryFile() as fp: + table = pa.Table.from_pandas(df, schema) + pq.write_table(table, fp) + + fp.seek(0) + return fp.read() + + +class TemporaryAvroFilesStreamReader(InMemoryFilesStreamReader): + """ + A file reader that writes RemoteFiles to a temporary file and then reads them back. + """ + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + return io.BytesIO(self._make_file_contents(file.uri)) + + def _make_file_contents(self, file_name: str) -> bytes: + contents = self.files[file_name]["contents"] + schema = self.files[file_name]["schema"] + stream_schema = avro_schema.make_avsc_object(schema) + + rec_writer = ai.DatumWriter(stream_schema) + with tempfile.TemporaryFile() as fp: + file_writer = datafile.DataFileWriter(fp, rec_writer, stream_schema) + for content in contents: + data = {col["name"]: content[i] for i, col in enumerate(schema["fields"])} + file_writer.append(data) + file_writer.flush() + fp.seek(0) + return fp.read() diff --git a/tools/ci_common_utils/tests/__init__.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/__init__.py similarity index 100% rename from tools/ci_common_utils/tests/__init__.py rename to airbyte-cdk/python/unit_tests/sources/file_based/scenarios/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py new file mode 100644 index 000000000000..27ef71fd9005 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py @@ -0,0 +1,722 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +import decimal +import uuid + +from unit_tests.sources.file_based.in_memory_files_source import TemporaryAvroFilesStreamReader +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +_single_avro_file = { + "a.avro": { + "schema": { + "type": "record", + "name": "sampleAvro", + "fields": [ + {"name": "col1", "type": "string"}, + {"name": "col2", "type": "int"}, + ], + }, + "contents": [ + ("val11", 12), + ("val21", 22), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } +} + +_multiple_avro_combine_schema_file = { + "a.avro": { + "schema": { + "type": "record", + "name": "sampleAvro", + "fields": [ + {"name": "col_double", "type": "double"}, + {"name": "col_string", "type": "string"}, + {"name": "col_album", "type": {"type": "record", "name": "Album", "fields": [{"name": "album", "type": "string"}]}}, + ], + }, + "contents": [ + (20.02, "Robbers", {"album": "The 1975"}), + (20.23, "Somebody Else", {"album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It"}), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.avro": { + "schema": { + "type": "record", + "name": "sampleAvro", + "fields": [ + {"name": "col_double", "type": "double"}, + {"name": "col_string", "type": "string"}, + {"name": "col_song", "type": {"type": "record", "name": "Song", "fields": [{"name": "title", "type": "string"}]}}, + ], + }, + "contents": [ + (1975.1975, "It's Not Living (If It's Not with You)", {"title": "Love It If We Made It"}), + (5791.5791, "The 1975", {"title": "About You"}), + ], + "last_modified": "2023-06-06T03:54:07.000Z", + }, +} + +_avro_all_types_file = { + "a.avro": { + "schema": { + "type": "record", + "name": "sampleAvro", + "fields": [ + # Primitive Types + {"name": "col_bool", "type": "boolean"}, + {"name": "col_int", "type": "int"}, + {"name": "col_long", "type": "long"}, + {"name": "col_float", "type": "float"}, + {"name": "col_double", "type": "double"}, + {"name": "col_bytes", "type": "bytes"}, + {"name": "col_string", "type": "string"}, + # Complex Types + { + "name": "col_record", + "type": { + "type": "record", + "name": "SongRecord", + "fields": [ + {"name": "artist", "type": "string"}, + {"name": "song", "type": "string"}, + {"name": "year", "type": "int"}, + ], + }, + }, + {"name": "col_enum", "type": {"type": "enum", "name": "Genre", "symbols": ["POP_ROCK", "INDIE_ROCK", "ALTERNATIVE_ROCK"]}}, + {"name": "col_array", "type": {"type": "array", "items": "string"}}, + {"name": "col_map", "type": {"type": "map", "values": "string"}}, + {"name": "col_fixed", "type": {"type": "fixed", "name": "MyFixed", "size": 4}}, + # Logical Types + {"name": "col_decimal", "type": {"type": "bytes", "logicalType": "decimal", "precision": 10, "scale": 5}}, + {"name": "col_uuid", "type": {"type": "bytes", "logicalType": "uuid"}}, + {"name": "col_date", "type": {"type": "int", "logicalType": "date"}}, + {"name": "col_time_millis", "type": {"type": "int", "logicalType": "time-millis"}}, + {"name": "col_time_micros", "type": {"type": "long", "logicalType": "time-micros"}}, + {"name": "col_timestamp_millis", "type": {"type": "long", "logicalType": "timestamp-millis"}}, + {"name": "col_timestamp_micros", "type": {"type": "long", "logicalType": "timestamp-micros"}}, + ], + }, + "contents": [ + ( + True, + 27, + 1992, + 999.09723456, + 9123456.12394, + b"\x00\x01\x02\x03", + "Love It If We Made It", + {"artist": "The 1975", "song": "About You", "year": 2022}, + "POP_ROCK", + [ + "The 1975", + "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It", + "The 1975 A Brief Inquiry into Online Relationships", + "Notes on a Conditional Form", + "Being Funny in a Foreign Language", + ], + {"lead_singer": "Matty Healy", "lead_guitar": "Adam Hann", "bass_guitar": "Ross MacDonald", "drummer": "George Daniel"}, + b"\x12\x34\x56\x78", + decimal.Decimal("1234.56789"), + uuid.UUID('123e4567-e89b-12d3-a456-426655440000').bytes, + datetime.date(2022, 5, 29), + datetime.time(6, 0, 0, 456000), + datetime.time(12, 0, 0, 456789), + datetime.datetime(2022, 5, 29, 0, 0, 0, 456000, tzinfo=datetime.timezone.utc), + datetime.datetime(2022, 5, 30, 0, 0, 0, 456789, tzinfo=datetime.timezone.utc), + ), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } +} + +_multiple_avro_stream_file = { + "odesza_songs.avro": { + "schema": { + "type": "record", + "name": "sampleAvro", + "fields": [ + {"name": "col_title", "type": "string"}, + { + "name": "col_album", + "type": { + "type": "enum", + "name": "Album", + "symbols": ["SUMMERS_GONE", "IN_RETURN", "A_MOMENT_APART", "THE_LAST_GOODBYE"], + }, + }, + {"name": "col_year", "type": "int"}, + {"name": "col_vocals", "type": "boolean"}, + ], + }, + "contents": [ + ("Late Night", "A_MOMENT_APART", 2017, False), + ("White Lies", "IN_RETURN", 2014, True), + ("Wide Awake", "THE_LAST_GOODBYE", 2022, True), + ("Sun Models", "SUMMERS_GONE", 2012, True), + ("All We Need", "IN_RETURN", 2014, True), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "california_festivals.avro": { + "schema": { + "type": "record", + "name": "sampleAvro", + "fields": [ + {"name": "col_name", "type": "string"}, + { + "name": "col_location", + "type": { + "type": "record", + "name": "LocationRecord", + "fields": [ + {"name": "country", "type": "string"}, + {"name": "state", "type": "string"}, + {"name": "city", "type": "string"}, + ], + }, + }, + {"name": "col_attendance", "type": "long"}, + ], + }, + "contents": [ + ("Coachella", {"country": "USA", "state": "California", "city": "Indio"}, 250000), + ("CRSSD", {"country": "USA", "state": "California", "city": "San Diego"}, 30000), + ("Lightning in a Bottle", {"country": "USA", "state": "California", "city": "Buena Vista Lake"}, 18000), + ("Outside Lands", {"country": "USA", "state": "California", "city": "San Francisco"}, 220000), + ], + "last_modified": "2023-06-06T03:54:07.000Z", + }, +} + +single_avro_scenario = ( + TestScenarioBuilder() + .set_name("single_avro_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "avro", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_single_avro_file, file_type="avro")) + .set_file_type("avro") + .set_expected_check_status("SUCCEEDED") + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": 12, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": 22, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "integer"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +multiple_avro_combine_schema_scenario = ( + TestScenarioBuilder() + .set_name("multiple_avro_combine_schema_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "avro", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_combine_schema_file, file_type="avro")) + .set_file_type("avro") + .set_expected_records( + [ + { + "data": { + "col_double": 20.02, + "col_string": "Robbers", + "col_album": {"album": "The 1975"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col_double": 20.23, + "col_string": "Somebody Else", + "col_album": {"album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col_double": 1975.1975, + "col_string": "It's Not Living (If It's Not with You)", + "col_song": {"title": "Love It If We Made It"}, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col_double": 5791.5791, + "col_string": "The 1975", + "col_song": {"title": "About You"}, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.avro", + }, + "stream": "stream1", + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col_double": {"type": ["null", "number"]}, + "col_string": {"type": ["null", "string"]}, + "col_album": { + "properties": { + "album": {"type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "col_song": { + "properties": { + "title": {"type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +avro_all_types_scenario = ( + TestScenarioBuilder() + .set_name("avro_all_types_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "avro", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_avro_all_types_file, file_type="avro")) + .set_file_type("avro") + .set_expected_records( + [ + { + "data": { + "col_bool": True, + "col_int": 27, + "col_long": 1992, + "col_float": 999.09723456, + "col_double": 9123456.12394, + "col_bytes": "\x00\x01\x02\x03", + "col_string": "Love It If We Made It", + "col_record": {"artist": "The 1975", "song": "About You", "year": 2022}, + "col_enum": "POP_ROCK", + "col_array": [ + "The 1975", + "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It", + "The 1975 A Brief Inquiry into Online Relationships", + "Notes on a Conditional Form", + "Being Funny in a Foreign Language", + ], + "col_map": { + "lead_singer": "Matty Healy", + "lead_guitar": "Adam Hann", + "bass_guitar": "Ross MacDonald", + "drummer": "George Daniel", + }, + "col_fixed": "\x12\x34\x56\x78", + "col_decimal": "1234.56789", + "col_uuid": "123e4567-e89b-12d3-a456-426655440000", + "col_date": "2022-05-29", + "col_time_millis": "06:00:00.456000", + "col_time_micros": "12:00:00.456789", + "col_timestamp_millis": "2022-05-29T00:00:00.456000+00:00", + "col_timestamp_micros": "2022-05-30T00:00:00.456789+00:00", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col_array": {"items": {"type": ["null", "string"]}, "type": ["null", "array"]}, + "col_bool": {"type": ["null", "boolean"]}, + "col_bytes": {"type": ["null", "string"]}, + "col_double": {"type": ["null", "number"]}, + "col_enum": {"enum": ["POP_ROCK", "INDIE_ROCK", "ALTERNATIVE_ROCK"], "type": ["null", "string"]}, + "col_fixed": {"pattern": "^[0-9A-Fa-f]{8}$", "type": ["null", "string"]}, + "col_float": {"type": ["null", "number"]}, + "col_int": {"type": ["null", "integer"]}, + "col_long": {"type": ["null", "integer"]}, + "col_map": {"additionalProperties": {"type": ["null", "string"]}, "type": ["null", "object"]}, + "col_record": { + "properties": {"artist": {"type": ["null", "string"]}, "song": {"type": ["null", "string"]}, "year": {"type": ["null", "integer"]}}, + "type": ["null", "object"], + }, + "col_string": {"type": ["null", "string"]}, + "col_decimal": {"pattern": "^-?\\d{(1, 5)}(?:\\.\\d(1, 5))?$", "type": ["null", "string"]}, + "col_uuid": {"type": ["null", "string"]}, + "col_date": {"format": "date", "type": ["null", "string"]}, + "col_time_millis": {"type": ["null", "integer"]}, + "col_time_micros": {"type": ["null", "integer"]}, + "col_timestamp_millis": {"format": "date-time", "type": ["null", "string"]}, + "col_timestamp_micros": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +multiple_streams_avro_scenario = ( + TestScenarioBuilder() + .set_name("multiple_streams_avro_stream") + .set_config( + { + "streams": [ + { + "name": "songs_stream", + "file_type": "avro", + "globs": ["*_songs.avro"], + "validation_policy": "Emit Record", + }, + { + "name": "festivals_stream", + "file_type": "avro", + "globs": ["*_festivals.avro"], + "validation_policy": "Emit Record", + }, + ] + } + ) + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_stream_file, file_type="avro")) + .set_file_type("avro") + .set_expected_records( + [ + { + "data": { + "col_title": "Late Night", + "col_album": "A_MOMENT_APART", + "col_year": 2017, + "col_vocals": False, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "odesza_songs.avro", + }, + "stream": "songs_stream", + }, + { + "data": { + "col_title": "White Lies", + "col_album": "IN_RETURN", + "col_year": 2014, + "col_vocals": True, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "odesza_songs.avro", + }, + "stream": "songs_stream", + }, + { + "data": { + "col_title": "Wide Awake", + "col_album": "THE_LAST_GOODBYE", + "col_year": 2022, + "col_vocals": True, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "odesza_songs.avro", + }, + "stream": "songs_stream", + }, + { + "data": { + "col_title": "Sun Models", + "col_album": "SUMMERS_GONE", + "col_year": 2012, + "col_vocals": True, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "odesza_songs.avro", + }, + "stream": "songs_stream", + }, + { + "data": { + "col_title": "All We Need", + "col_album": "IN_RETURN", + "col_year": 2014, + "col_vocals": True, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "odesza_songs.avro", + }, + "stream": "songs_stream", + }, + { + "data": { + "col_name": "Coachella", + "col_location": {"country": "USA", "state": "California", "city": "Indio"}, + "col_attendance": 250000, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "california_festivals.avro", + }, + "stream": "festivals_stream", + }, + { + "data": { + "col_name": "CRSSD", + "col_location": {"country": "USA", "state": "California", "city": "San Diego"}, + "col_attendance": 30000, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "california_festivals.avro", + }, + "stream": "festivals_stream", + }, + { + "data": { + "col_name": "Lightning in a Bottle", + "col_location": {"country": "USA", "state": "California", "city": "Buena Vista Lake"}, + "col_attendance": 18000, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "california_festivals.avro", + }, + "stream": "festivals_stream", + }, + { + "data": { + "col_name": "Outside Lands", + "col_location": {"country": "USA", "state": "California", "city": "San Francisco"}, + "col_attendance": 220000, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "california_festivals.avro", + }, + "stream": "festivals_stream", + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col_title": {"type": ["null", "string"]}, + "col_album": {"type": ["null", "string"], "enum": ["SUMMERS_GONE", "IN_RETURN", "A_MOMENT_APART", "THE_LAST_GOODBYE"]}, + "col_year": {"type": ["null", "integer"]}, + "col_vocals": {"type": ["null", "boolean"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "songs_stream", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col_name": {"type": ["null", "string"]}, + "col_location": { + "properties": {"country": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}}, + "type": ["null", "object"], + }, + "col_attendance": {"type": ["null", "integer"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "festivals_stream", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) +).build() + +avro_file_with_double_as_number_scenario = ( + TestScenarioBuilder() + .set_name("avro_file_with_double_as_number_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "avro", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "avro", + "double_as_string": False + } + } + ] + } + ) + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_combine_schema_file, file_type="avro")) + .set_file_type("avro") + .set_expected_records( + [ + { + "data": { + "col_double": 20.02, + "col_string": "Robbers", + "col_album": {"album": "The 1975"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col_double": 20.23, + "col_string": "Somebody Else", + "col_album": {"album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col_double": 1975.1975, + "col_string": "It's Not Living (If It's Not with You)", + "col_song": {"title": "Love It If We Made It"}, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.avro", + }, + "stream": "stream1", + }, + { + "data": { + "col_double": 5791.5791, + "col_string": "The 1975", + "col_song": {"title": "About You"}, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.avro", + }, + "stream": "stream1", + }, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col_double": {"type": ["null", "number"]}, + "col_string": {"type": ["null", "string"]}, + "col_album": { + "properties": { + "album": {"type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "col_song": { + "properties": { + "title": {"type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py new file mode 100644 index 000000000000..880046bb1ef2 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py @@ -0,0 +1,196 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError +from unit_tests.sources.file_based.helpers import ( + FailingSchemaValidationPolicy, + TestErrorListMatchingFilesInMemoryFilesStreamReader, + TestErrorOpenFileInMemoryFilesStreamReader, +) +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +_base_success_scenario = ( + TestScenarioBuilder() + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_check_status("SUCCEEDED") +) + + +success_csv_scenario = ( + _base_success_scenario.copy() + .set_name("success_csv_scenario") +).build() + + +success_multi_stream_scenario = ( + _base_success_scenario.copy() + .set_name("success_multi_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv", "*.gz"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["*.csv", "*.gz"], + "validation_policy": "Emit Record", + } + ] + } + ) +).build() + + +success_extensionless_scenario = ( + _base_success_scenario.copy() + .set_name("success_extensionless_file_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) +).build() + + +success_user_provided_schema_scenario = ( + _base_success_scenario.copy() + .set_name("success_user_provided_schema_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "string"}', + } + ], + } + ) +).build() + + +_base_failure_scenario = ( + _base_success_scenario.copy() + .set_expected_check_status("FAILED") +) + + +error_empty_stream_scenario = ( + _base_failure_scenario.copy() + .set_name("error_empty_stream_scenario") + .set_files({}) + .set_expected_check_error(None, FileBasedSourceError.EMPTY_STREAM.value) +).build() + + +error_listing_files_scenario = ( + _base_failure_scenario.copy() + .set_name("error_listing_files_scenario") + .set_stream_reader(TestErrorListMatchingFilesInMemoryFilesStreamReader(files=_base_failure_scenario._files, file_type="csv")) + .set_expected_check_error(None, FileBasedSourceError.ERROR_LISTING_FILES.value) +).build() + + +error_reading_file_scenario = ( + _base_failure_scenario.copy() + .set_name("error_reading_file_scenario") + .set_stream_reader(TestErrorOpenFileInMemoryFilesStreamReader(files=_base_failure_scenario._files, file_type="csv")) + .set_expected_check_error(None, FileBasedSourceError.ERROR_READING_FILE.value) +).build() + + +error_record_validation_user_provided_schema_scenario = ( + _base_failure_scenario.copy() + .set_name("error_record_validation_user_provided_schema_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "always_fail", + "input_schema": '{"col1": "number", "col2": "string"}', + } + ], + } + ) + .set_validation_policies({FailingSchemaValidationPolicy.ALWAYS_FAIL: FailingSchemaValidationPolicy()}) + .set_expected_check_error(None, FileBasedSourceError.ERROR_VALIDATING_RECORD.value) +).build() + + +error_multi_stream_scenario = ( + _base_failure_scenario.copy() + .set_name("error_multi_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "file_type": "jsonl", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ], + } + ) + .set_expected_check_error(None, FileBasedSourceError.ERROR_READING_FILE.value) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py new file mode 100644 index 000000000000..585cb1b403d7 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -0,0 +1,2613 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, SchemaInferenceError +from unit_tests.sources.file_based.helpers import EmptySchemaParser, LowInferenceLimitDiscoveryPolicy +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +single_csv_scenario = ( + TestScenarioBuilder() + .set_name("single_csv_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_spec( + { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/in_memory_files", + "connectionSpecification": { + "title": "InMemorySpec", + "description": "Used during spec; allows the developer to configure the cloud provider specific options\nthat are needed when users configure a file-based source.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string", + }, + "streams": { + "title": "The list of streams to sync", + "description": 'Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.', + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": {"title": "Name", "description": "The name of the stream.", "type": "string"}, + "file_type": { + "title": "File Type", + "description": "The data file type that is being extracted for a stream.", + "type": "string", + }, + "globs": { + "title": "Globs", + "description": 'The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.', + "type": "array", + "items": {"type": "string"}, + }, + "legacy_prefix": { + "title": "Legacy Prefix", + "airbyte_hidden": True, + "type": "string", + "description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.", + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"], + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string", + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "type": "string", + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer", + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": {"title": "Filetype", "default": "avro", "enum": ["avro"], "type": "string"}, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": False, + "type": "boolean", + }, + }, + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": {"title": "Filetype", "default": "csv", "enum": ["csv"], "type": "string"}, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string", + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": '"', + "type": "string", + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string", + }, + "encoding": { + "title": "Encoding", + "description": 'The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.', + "default": "utf8", + "type": "string", + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": True, + "type": "boolean", + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": True, + "type": "boolean", + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer", + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer", + }, + "autogenerate_column_names": { + "title": "Autogenerate Column Names", + "description": "Whether to autogenerate column names if column_names is empty. If true, column names will be of the form \u201cf0\u201d, \u201cf1\u201d\u2026 If false, column names will be read from the first CSV row after skip_rows_before_header.", + "default": False, + "type": "boolean", + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": {"type": "string"}, + "uniqueItems": True, + }, + "inference_type": { + "title": "Inference Type", + "description": "How to infer the types of the columns. If none, inference default to strings.", + "default": "None", + "airbyte_hidden": True, + "enum": ["None", "Primitive Types Only"], + }, + }, + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": {"title": "Filetype", "default": "jsonl", "enum": ["jsonl"], "type": "string"} + }, + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "enum": ["parquet"], + "type": "string", + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": False, + "type": "boolean", + }, + }, + }, + ], + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": False, + "type": "boolean", + }, + }, + "required": ["name", "file_type"], + }, + }, + }, + "required": ["streams"], + }, + } + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +multi_csv_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +multi_csv_stream_n_file_exceeds_limit_for_inference = ( + TestScenarioBuilder() + .set_name("multi_csv_stream_n_file_exceeds_limit_for_inference") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + ] + ) + .set_discovery_policy(LowInferenceLimitDiscoveryPolicy()) +).build() + +invalid_csv_scenario = ( + TestScenarioBuilder() + .set_name("invalid_csv_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1",), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records([]) + .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_logs( + { + "read": [ + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a.csv line_no=1 n_skipped=0", + }, + ] + } + ) +).build() + +csv_single_stream_scenario = ( + TestScenarioBuilder() + .set_name("csv_single_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + {"col1": "val12b", "col2": "val22b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_multi_stream_scenario = ( + TestScenarioBuilder() + .set_name("csv_multi_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Emit Record", + }, + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": {"col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream1", + }, + { + "data": {"col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream1", + }, + { + "data": {"col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream2", + }, + { + "data": {"col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream2", + }, + ] + ) +).build() + + +csv_custom_format_scenario = ( + TestScenarioBuilder() + .set_name("csv_custom_format") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "delimiter": "#", + "quote_char": "|", + "escape_char": "!", + "double_quote": True, + }, + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11", "val12", "val |13|"), + ("val21", "val22", "val23"), + ("val,31", "val |,32|", "val, !!!! 33"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "col3": "val |13|", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "col3": "val23", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val,31", + "col2": "val |,32|", + "col3": "val, !! 33", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) + .set_file_write_options( + { + "delimiter": "#", + "quotechar": "|", + } + ) +).build() + + +multi_stream_custom_format = ( + TestScenarioBuilder() + .set_name("multi_stream_custom_format_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "format": {"filetype": "csv", "delimiter": "#", "escape_char": "!", "double_quote": True, "newlines_in_values": False}, + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "delimiter": "#", + "escape_char": "@", + "double_quote": True, + "newlines_in_values": False, + }, + }, + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val !! 12a"), + ("val !! 21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val @@@@ 13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val ! 12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val ! 21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col3": "val @@@@ 13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": {"col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream1", + }, + { + "data": { + "col3": "val @@ 13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", + }, + { + "data": {"col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream2", + }, + ] + ) + .set_file_write_options( + { + "delimiter": "#", + } + ) +).build() + + +empty_schema_inference_scenario = ( + TestScenarioBuilder() + .set_name("empty_schema_inference_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_parsers({"csv": EmptySchemaParser()}) + .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + + +schemaless_csv_scenario = ( + TestScenarioBuilder() + .set_name("schemaless_csv_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Skip Record", + "schemaless": True, + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "data": {"type": "object"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "data": {"col1": "val11a", "col2": "val12a"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": "val21a", "col2": "val22a"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": "val21b", "col2": "val22b", "col3": "val23b"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + + +schemaless_csv_multi_stream_scenario = ( + TestScenarioBuilder() + .set_name("schemaless_csv_multi_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a.csv"], + "validation_policy": "Skip Record", + "schemaless": True, + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Skip Record", + }, + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "data": {"type": "object"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records( + [ + { + "data": { + "data": {"col1": "val11a", "col2": "val12a"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": "val21a", "col2": "val22a"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": {"col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream2", + }, + { + "data": {"col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, + "stream": "stream2", + }, + ] + ) +).build() + + +schemaless_with_user_input_schema_fails_connection_check_scenario = ( + TestScenarioBuilder() + .set_name("schemaless_with_user_input_schema_fails_connection_check_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Skip Record", + "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}', + "schemaless": True, + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "data": {"type": "object"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) + .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) + .set_expected_read_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) +).build() + + +schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario = ( + TestScenarioBuilder() + .set_name("schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a.csv"], + "validation_policy": "Skip Record", + "schemaless": True, + "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}', + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Skip Record", + }, + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "data": {"type": "object"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) + .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) + .set_expected_read_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) +).build() + + +csv_string_can_be_null_with_input_schemas_scenario = ( + TestScenarioBuilder() + .set_name("csv_string_can_be_null_with_input_schema") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "string"}', + "format": { + "filetype": "csv", + "null_values": ["null"], + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": "string"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "2", + "col2": None, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_string_are_not_null_if_strings_can_be_null_is_false_scenario = ( + TestScenarioBuilder() + .set_name("csv_string_are_not_null_if_strings_can_be_null_is_false") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "string"}', + "format": { + "filetype": "csv", + "null_values": ["null"], + "strings_can_be_null": False, + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": "string"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "2", + "col2": "null", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_string_not_null_if_no_null_values_scenario = ( + TestScenarioBuilder() + .set_name("csv_string_not_null_if_no_null_values") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "2", + "col2": "null", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_strings_can_be_null_not_quoted_scenario = ( + TestScenarioBuilder() + .set_name("csv_strings_can_be_null_no_input_schema") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": {"filetype": "csv", "null_values": ["null"]}, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "2", + "col2": None, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_newline_in_values_quoted_value_scenario = ( + TestScenarioBuilder() + .set_name("csv_newline_in_values_quoted_value") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + } + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + '''"col1","col2"''', + '''"2","val\n2"''', + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "2", + "col2": "val\n2", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_newline_in_values_not_quoted_scenario = ( + TestScenarioBuilder() + .set_name("csv_newline_in_values_not_quoted") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + """col1,col2""", + """2,val\n2""", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # Note that the value for col2 is truncated to "val" because the newline is not escaped + { + "data": { + "col1": "2", + "col2": "val", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) + .set_expected_logs( + { + "read": [ + { + "level": "ERROR", + "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=a.csv line_no=2 n_skipped=0", + } + ] + } + ) + .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) +).build() + +csv_escape_char_is_set_scenario = ( + TestScenarioBuilder() + .set_name("csv_escape_char_is_set") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "double_quotes": False, + "quote_char": '"', + "delimiter": ",", + "escape_char": "\\", + } + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + """col1,col2""", + '''val11,"val\\"2"''', + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": 'val"2', + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_double_quote_is_set_scenario = ( + TestScenarioBuilder() + .set_name("csv_doublequote_is_set") + # This scenario tests that quotes are properly escaped when double_quotes is True + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "double_quotes": True, + "quote_char": '"', + "delimiter": ",", + } + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + """col1,col2""", + '''val11,"val""2"''', + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": 'val"2', + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_custom_delimiter_with_escape_char_scenario = ( + TestScenarioBuilder() + .set_name("csv_custom_delimiter_with_escape_char") + # This scenario tests that a value can contain the delimiter if it is wrapped in the quote_char + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": {"filetype": "csv", "double_quotes": True, "quote_char": "@", "delimiter": "|", "escape_char": "+"}, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + """col1|col2""", + """val"1,1|val+|2""", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": 'val"1,1', + "col2": "val|2", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_custom_delimiter_in_double_quotes_scenario = ( + TestScenarioBuilder() + .set_name("csv_custom_delimiter_in_double_quotes") + # This scenario tests that a value can contain the delimiter if it is wrapped in the quote_char + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "double_quotes": True, + "quote_char": "@", + "delimiter": "|", + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + """col1|col2""", + """val"1,1|@val|2@""", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": 'val"1,1', + "col2": "val|2", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + + +csv_skip_before_header_scenario = ( + TestScenarioBuilder() + .set_name("csv_skip_before_header") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": {"filetype": "csv", "skip_rows_before_header": 2}, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("skip_this", "skip_this"), + ("skip_this_too", "skip_this_too"), + ("col1", "col2"), + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_skip_after_header_scenario = ( + TestScenarioBuilder() + .set_name("csv_skip_after_header") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": {"filetype": "csv", "skip_rows_after_header": 2}, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("skip_this", "skip_this"), + ("skip_this_too", "skip_this_too"), + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + + +csv_skip_before_and_after_header_scenario = ( + TestScenarioBuilder() + .set_name("csv_skip_before_after_header") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "skip_rows_before_header": 1, + "skip_rows_after_header": 1, + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("skip_this", "skip_this"), + ("col1", "col2"), + ("skip_this_too", "skip_this_too"), + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_autogenerate_column_names_scenario = ( + TestScenarioBuilder() + .set_name("csv_autogenerate_column_names") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv", + "autogenerate_column_names": True, + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "f0": {"type": ["null", "string"]}, + "f1": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "f0": "val11", + "f1": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_custom_bool_values_scenario = ( + TestScenarioBuilder() + .set_name("csv_custom_bool_values") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "boolean", "col2": "boolean"}', + "format": { + "filetype": "csv", + "true_values": ["this_is_true"], + "false_values": ["this_is_false"], + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("this_is_true", "this_is_false"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": "boolean"}, + "col2": {"type": "boolean"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": True, + "col2": False, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + +csv_custom_null_values_scenario = ( + TestScenarioBuilder() + .set_name("csv_custom_null_values") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "boolean", "col2": "string"}', + "format": { + "filetype": "csv", + "null_values": ["null"], + }, + } + ], + "start_date": "2023-06-04T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("null", "na"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": "boolean"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "col1": None, + "col2": "na", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + ] + ) +).build() + + +earlier_csv_scenario = ( + TestScenarioBuilder() + .set_name("earlier_csv_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ], + "start_date": "2023-06-10T03:54:07.000000Z", + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_expected_check_status("FAILED") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": "string"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records([]) + .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py new file mode 100644 index 000000000000..3f4a95933b02 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py @@ -0,0 +1,1593 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unit_tests.sources.file_based.helpers import LowHistoryLimitCursor +from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder + +single_csv_input_state_is_earlier_scenario = ( + TestScenarioBuilder() + .set_name("single_csv_input_state_is_earlier") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "some_old_file.csv": "2023-06-01T03:54:07.000000Z" + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + )) + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "some_old_file.csv": "2023-06-01T03:54:07.000000Z", + "a.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + } + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + } + ] + } + ) +).build() + +single_csv_file_is_skipped_if_same_modified_at_as_in_history = ( + TestScenarioBuilder() + .set_name("single_csv_file_is_skipped_if_same_modified_at_as_in_history") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z" + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + )) + .set_expected_records( + [ + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + } + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + } + ] + } + ) +).build() + +single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history = ( + TestScenarioBuilder() + .set_name("single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "a.csv": "2023-06-01T03:54:07.000000Z" + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + )) + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + } + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + } + ] + } + ) +).build() + +single_csv_no_input_state_scenario = ( + TestScenarioBuilder() + .set_name("single_csv_input_state_is_earlier_again") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", + } + } + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[], + ))).build() + +multi_csv_same_timestamp_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_same_timestamp") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + } + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[], + ))).build() + +single_csv_input_state_is_later_scenario = ( + TestScenarioBuilder() + .set_name("single_csv_input_state_is_later") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "recent_file.csv": "2023-07-15T23:59:59.000000Z", + "a.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv", + } + } + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "recent_file.csv": "2023-07-15T23:59:59.000000Z" + } + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() + +multi_csv_different_timestamps_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_stream_different_timestamps") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-04T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-04T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv", + } + }, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-04T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + } + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[], + ))).build() + +multi_csv_per_timestamp_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_per_timestamp") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + }, + {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[], + ))).build() + +multi_csv_skip_file_if_already_in_history = ( + TestScenarioBuilder() + .set_name("skip_files_already_in_history") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", + } + }, + {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"} + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() + +multi_csv_include_missing_files_within_history_range = ( + TestScenarioBuilder() + .set_name("include_missing_files_within_history_range") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped + # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-06T03:54:07.000000Z" + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() + +multi_csv_remove_old_files_if_history_is_full_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_remove_old_files_if_history_is_full") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-10T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "a.csv": "2023-06-06T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv", + } + }, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "a.csv": "2023-06-06T03:54:07.000000Z", + "b.csv": "2023-06-07T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv", + } + }, + {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + "b.csv": "2023-06-07T03:54:07.000000Z", + "c.csv": "2023-06-10T03:54:07.000000Z" + }, + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv", + } + }, + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() + +multi_csv_same_timestamp_more_files_than_history_size_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_same_timestamp_more_files_than_history_size") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_files( + { + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + {"data": {"col1": "val11d", "col2": "val12d", "col3": "val13d", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21d", "col2": "val22d", "col3": "val23d", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + } + } + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[], + ))).build() + +multi_csv_sync_recent_files_if_history_is_incomplete_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_recent_files_if_history_is_incomplete") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_cursor_cls(LowHistoryLimitCursor) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "stream1": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + } + } + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() + +multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_recent_files_if_history_is_incomplete__different_timestamps") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # This file is skipped because it is older than the time_window + # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_e.csv", + } + }, + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() + +multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario = ( + TestScenarioBuilder() + .set_name("multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3, + } + ] + } + ) + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "a.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + } + }, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "stream1": { + "history": { + "b.csv": "2023-06-06T03:54:07.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", + } + }, + ] + ) + .set_incremental_scenario_config(IncrementalScenarioConfig( + input_state=[{ + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "old_file.csv": "2023-06-05T00:00:00.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"} + } + } + ], + ))).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py new file mode 100644 index 000000000000..6e305f75ba0a --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py @@ -0,0 +1,737 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, SchemaInferenceError +from unit_tests.sources.file_based.helpers import LowInferenceBytesJsonlParser, LowInferenceLimitDiscoveryPolicy +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +single_jsonl_scenario = ( + TestScenarioBuilder() + .set_name("single_jsonl_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11", "col2": "val12"}, + {"col1": "val21", "col2": "val22"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + ] + ) +).build() + + +multi_jsonl_with_different_keys_scenario = ( + TestScenarioBuilder() + .set_name("multi_jsonl_with_different_keys_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11a", "col2": "val12a"}, + {"col1": "val21a", "col2": "val22a"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + {"col1": "val21b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + ] + ) +).build() + + +multi_jsonl_stream_n_file_exceeds_limit_for_inference = ( + TestScenarioBuilder() + .set_name("multi_jsonl_stream_n_file_exceeds_limit_for_inference") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11a", "col2": "val12a"}, + {"col1": "val21a", "col2": "val22a"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + {"col1": "val21b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + ] + ) + .set_discovery_policy(LowInferenceLimitDiscoveryPolicy()) +).build() + + +multi_jsonl_stream_n_bytes_exceeds_limit_for_inference = ( + TestScenarioBuilder() + .set_name("multi_jsonl_stream_n_bytes_exceeds_limit_for_inference") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11a", "col2": "val12a"}, + {"col1": "val21a", "col2": "val22a"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b"}, + {"col1": "val21b", "col2": "val22b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, "col2": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + ] + ) + .set_parsers({"jsonl": LowInferenceBytesJsonlParser()}) +).build() + + +invalid_jsonl_scenario = ( + TestScenarioBuilder() + .set_name("invalid_jsonl_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val1"}, + "invalid", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records([ + {"data": {"col1": "val1", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + ]) + .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_logs( + { + "read": [ + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a.jsonl line_no=2 n_skipped=0", + }, + ] + } + ) +).build() + + +jsonl_multi_stream_scenario = ( + TestScenarioBuilder() + .set_name("jsonl_multi_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*.jsonl"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "file_type": "jsonl", + "globs": ["b.jsonl"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "record1"}, + {"col1": 2, "col2": "record2"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col3": 1.1}, + {"col3": 2.2}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "integer"] + }, + "col2": { + "type": ["null", "string"], + }, + "col3": { + "type": ["null", "number"] + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": { + "type": ["null", "number"] + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": 1, "col2": "record1", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": 2, "col2": "record2", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream1"}, + {"data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream1"}, + {"data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2"}, + {"data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2"}, + ] + ) +).build() + + +schemaless_jsonl_scenario = ( + TestScenarioBuilder() + .set_name("schemaless_jsonl_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Skip Record", + "schemaless": True, + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "record1"}, + {"col1": 2, "col2": "record2"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": 3, "col2": "record3", "col3": 1.1}, + {"col1": 4, "col2": "record4", "col3": 1.1}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "data": { + "type": "object" + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"data": {"col1": 1, "col2": "record1"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"data": {"col1": 2, "col2": "record2"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"data": {"col1": 3, "col2": "record3", "col3": 1.1}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + {"data": {"data": {"col1": 4, "col2": "record4", "col3": 1.1}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + ] + ) +).build() + + +schemaless_jsonl_multi_stream_scenario = ( + TestScenarioBuilder() + .set_name("schemaless_jsonl_multi_stream_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["a.jsonl"], + "validation_policy": "Skip Record", + "schemaless": True, + }, + { + "name": "stream2", + "file_type": "jsonl", + "globs": ["b.jsonl"], + "validation_policy": "Skip Record", + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "record1"}, + {"col1": 2, "col2": "record2"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col3": 1.1}, + {"col3": 2.2}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "data": { + "type": "object" + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": { + "type": ["null", "number"] + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records( + [ + {"data": {"data": {"col1": 1, "col2": "record1"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"data": {"col1": 2, "col2": "record2"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2"}, + {"data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2"}, + ] + ) +).build() + +jsonl_user_input_schema_scenario = ( + TestScenarioBuilder() + .set_name("jsonl_user_input_schema_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "jsonl", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "integer", "col2": "string"}' + } + ] + } + ) + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "val12"}, + {"col1": 2, "col2": "val22"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("jsonl") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "integer" + }, + "col2": { + "type": "string", + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": 1, "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + {"data": {"col1": 2, "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + ] + ) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py new file mode 100644 index 000000000000..efba1c4bc043 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py @@ -0,0 +1,763 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +import decimal + +import pyarrow as pa +from airbyte_cdk.sources.file_based.exceptions import SchemaInferenceError +from unit_tests.sources.file_based.in_memory_files_source import TemporaryParquetFilesStreamReader +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +_single_parquet_file = { + "a.parquet": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } +} + +_single_partitioned_parquet_file = { + "path_prefix/partition1=1/partition2=2/a.parquet": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } +} + +_parquet_file_with_decimal = { + "a.parquet": { + "contents": [ + ("col1", ), + (decimal.Decimal("13.00"),), + ], + "schema": pa.schema([ + pa.field("col1", pa.decimal128(5, 2)), + ]), + "last_modified": "2023-06-05T03:54:07.000Z", + } +} + +_multiple_parquet_file = { + "a.parquet": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.parquet": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, +} + +_parquet_file_with_various_types = { + "a.parquet": { + "contents": [ + ("col_bool", + + "col_int8", + "col_int16", + "col_int32", + + "col_uint8", + "col_uint16", + "col_uint32", + "col_uint64", + + "col_float32", + "col_float64", + + "col_string", + + "col_date32", + "col_date64", + + "col_timestamp_without_tz", + "col_timestamp_with_tz", + + "col_time32s", + "col_time32ms", + "col_time64us", + + "col_struct", + "col_list", + "col_duration", + "col_binary", + ), + (True, + + -1, + 1, + 2, + + 2, + 3, + 4, + 5, + + 3.14, + 5.0, + + "2020-01-01", + + datetime.date(2021, 1, 1), + datetime.date(2022, 1, 1), + datetime.datetime(2023, 1, 1, 1, 2, 3), + datetime.datetime(2024, 3, 4, 5, 6, 7, tzinfo=datetime.timezone.utc), + + datetime.time(1, 2, 3), + datetime.time(2, 3, 4), + datetime.time(1, 2, 3, 4), + {"struct_key": "struct_value"}, + [1, 2, 3, 4], + 12345, + b"binary string. Hello world!", + ), + ], + "schema": pa.schema([ + pa.field("col_bool", pa.bool_()), + + pa.field("col_int8", pa.int8()), + pa.field("col_int16", pa.int16()), + pa.field("col_int32", pa.int32()), + + pa.field("col_uint8", pa.uint8()), + pa.field("col_uint16", pa.uint16()), + pa.field("col_uint32", pa.uint32()), + pa.field("col_uint64", pa.uint64()), + + pa.field("col_float32", pa.float32()), + pa.field("col_float64", pa.float64()), + + pa.field("col_string", pa.string()), + + pa.field("col_date32", pa.date32()), + pa.field("col_date64", pa.date64()), + pa.field("col_timestamp_without_tz", pa.timestamp("s")), + pa.field("col_timestamp_with_tz", pa.timestamp("s", tz="UTC")), + + pa.field("col_time32s", pa.time32("s")), + pa.field("col_time32ms", pa.time32("ms")), + pa.field("col_time64us", pa.time64("us")), + + pa.field("col_struct", pa.struct([pa.field("struct_key", pa.string())])), + pa.field("col_list", pa.list_(pa.int32())), + pa.field("col_duration", pa.duration("s")), + pa.field("col_binary", pa.binary()) + ]), + "last_modified": "2023-06-05T03:54:07.000Z", + } +} + +single_parquet_scenario = ( + TestScenarioBuilder() + .set_name("single_parquet_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_parquet_file, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"] + }, + "col2": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +single_partitioned_parquet_scenario = ( + TestScenarioBuilder() + .set_name("single_partitioned_parquet_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["path_prefix/**/*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_partitioned_parquet_file, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "partition1": "1", "partition2": "2","_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "path_prefix/partition1=1/partition2=2/a.parquet"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "partition1": "1", "partition2": "2", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "path_prefix/partition1=1/partition2=2/a.parquet"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"] + }, + "col2": { + "type": ["null", "string"] + }, + "partition1": { + "type": ["null", "string"] + }, + "partition2": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +multi_parquet_scenario = ( + TestScenarioBuilder() + .set_name("multi_parquet_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_file_type("parquet") + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_multiple_parquet_file, file_type="parquet")) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"] + }, + "col2": { + "type": ["null", "string"] + }, + "col3": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.parquet"}, "stream": "stream1"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.parquet"}, "stream": "stream1"}, + ] + ) +).build() + +parquet_various_types_scenario = ( + TestScenarioBuilder() + .set_name("parquet_various_types") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_various_types, file_type="parquet")) + .set_file_type("parquet") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col_bool": { + "type": ["null", "boolean"], + }, + "col_int8": { + "type": ["null", "integer"], + }, + "col_int16": { + "type": ["null", "integer"], + }, + "col_int32": { + "type": ["null", "integer"], + }, + "col_uint8": { + "type": ["null", "integer"], + }, + "col_uint16": { + "type": ["null", "integer"], + }, + "col_uint32": { + "type": ["null", "integer"], + }, + "col_uint64": { + "type": ["null", "integer"], + }, + "col_float32": { + "type": ["null", "number"], + }, + "col_float64": { + "type": ["null", "number"], + }, + "col_string": { + "type": ["null", "string"], + }, + "col_date32": { + "type": ["null", "string"], + "format": "date" + }, + "col_date64": { + "type": ["null", "string"], + "format": "date" + }, + "col_timestamp_without_tz": { + "type": ["null", "string"], + "format": "date-time" + }, + "col_timestamp_with_tz": { + "type": ["null", "string"], + "format": "date-time" + }, + "col_time32s": { + "type": ["null", "string"], + }, + "col_time32ms": { + "type": ["null", "string"], + }, + "col_time64us": { + "type": ["null", "string"], + }, + "col_struct": { + "type": ["null", "object"], + }, + "col_list": { + "type": ["null", "array"], + }, + "col_duration": { + "type": ["null", "integer"], + }, + "col_binary": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string", + }, + "_ab_source_file_url": { + "type": "string", + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col_bool": True, + "col_int8": -1, + "col_int16": 1, + "col_int32": 2, + "col_uint8": 2, + "col_uint16": 3, + "col_uint32": 4, + "col_uint64": 5, + "col_float32": 3.14, + "col_float64": 5.0, + "col_string": "2020-01-01", + "col_date32": "2021-01-01", + "col_date64": "2022-01-01", + "col_timestamp_without_tz": "2023-01-01T01:02:03", + "col_timestamp_with_tz": "2024-03-04T05:06:07+00:00", + "col_time32s": "01:02:03", + "col_time32ms": "02:03:04", + "col_time64us": "01:02:03.000004", + "col_struct": {"struct_key": "struct_value"}, + "col_list": [1, 2, 3, 4], + "col_duration": 12345, + "col_binary": "binary string. Hello world!", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1" + }, + ] + ) +).build() + +parquet_file_with_decimal_no_config_scenario = ( + TestScenarioBuilder() + .set_name("parquet_file_with_decimal_no_config") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + {"data": {"col1": "13.00", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +parquet_file_with_decimal_as_string_scenario = ( + TestScenarioBuilder() + .set_name("parquet_file_with_decimal_as_string") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "parquet", + "decimal_as_float": False + } + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + {"data": {"col1": "13.00", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +parquet_file_with_decimal_as_float_scenario = ( + TestScenarioBuilder() + .set_name("parquet_file_with_decimal_as_float") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "parquet", + "decimal_as_float": True + } + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + {"data": {"col1": 13.00, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "number"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +parquet_file_with_decimal_legacy_config_scenario = ( + TestScenarioBuilder() + .set_name("parquet_file_with_decimal_legacy_config") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "format": { + "filetype": "parquet", + }, + "globs": ["*"], + "validation_policy": "emit_record", + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + {"data": {"col1": 13.00, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "number"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + +parquet_with_invalid_config_scenario = ( + TestScenarioBuilder() + .set_name("parquet_with_invalid_config") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "parquet", + "globs": ["*"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv" + } + } + ] + } + ) + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_parquet_file, file_type="parquet")) + .set_file_type("parquet") + .set_expected_records( + [ + ] + ) + .set_expected_logs({"read": [ + { + "level": "ERROR", + "message": "Error parsing record" + } + ]}) + .set_expected_discover_error(SchemaInferenceError, "Error inferring schema from files") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"] + }, + "col2": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py new file mode 100644 index 000000000000..b3cf8c44037e --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -0,0 +1,237 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any, List, Mapping, Optional, Tuple, Type + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.file_based.availability_strategy.abstract_file_based_availability_strategy import ( + AbstractFileBasedAvailabilityStrategy, +) +from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_based_source import default_parsers +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource + + +@dataclass +class IncrementalScenarioConfig: + input_state: List[Mapping[str, Any]] = field(default_factory=list) + expected_output_state: Optional[Mapping[str, Any]] = None + + +class TestScenario: + def __init__( + self, + name: str, + config: Mapping[str, Any], + files: Mapping[str, Any], + file_type: str, + expected_spec: Optional[Mapping[str, Any]], + expected_check_status: Optional[str], + expected_catalog: Optional[Mapping[str, Any]], + expected_logs: Optional[Mapping[str, List[Mapping[str, Any]]]], + expected_records: List[Mapping[str, Any]], + availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy], + discovery_policy: Optional[AbstractDiscoveryPolicy], + validation_policies: Mapping[str, AbstractSchemaValidationPolicy], + parsers: Mapping[str, FileTypeParser], + stream_reader: Optional[AbstractFileBasedStreamReader], + expected_check_error: Tuple[Optional[Type[Exception]], Optional[str]], + expected_discover_error: Tuple[Optional[Type[Exception]], Optional[str]], + expected_read_error: Tuple[Optional[Type[Exception]], Optional[str]], + incremental_scenario_config: Optional[IncrementalScenarioConfig], + file_write_options: Mapping[str, Any], + cursor_cls: Optional[Type[AbstractFileBasedCursor]], + ): + self.name = name + self.config = config + self.expected_spec = expected_spec + self.expected_check_status = expected_check_status + self.expected_catalog = expected_catalog + self.expected_logs = expected_logs + self.expected_records = expected_records + self.expected_check_error = expected_check_error + self.expected_discover_error = expected_discover_error + self.expected_read_error = expected_read_error + self.source = InMemoryFilesSource( + files, + file_type, + availability_strategy, + discovery_policy, + validation_policies, + parsers, + stream_reader, + self.configured_catalog(SyncMode.incremental if incremental_scenario_config else SyncMode.full_refresh), + file_write_options, + cursor_cls, + ) + self.incremental_scenario_config = incremental_scenario_config + self.validate() + + def validate(self) -> None: + assert self.name + if not self.expected_catalog: + return + streams = {s["name"] for s in self.config["streams"]} + expected_streams = {s["name"] for s in self.expected_catalog["streams"]} + assert expected_streams <= streams + + def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]]: + if not self.expected_catalog: + return None + catalog: Mapping[str, Any] = {"streams": []} + for stream in self.expected_catalog["streams"]: + catalog["streams"].append( + { + "stream": stream, + "sync_mode": sync_mode.value, + "destination_sync_mode": "append", + } + ) + + return catalog + + def input_state(self) -> List[Mapping[str, Any]]: + if self.incremental_scenario_config: + return self.incremental_scenario_config.input_state + else: + return [] + + +class TestScenarioBuilder: + def __init__(self) -> None: + self._name = "" + self._config: Mapping[str, Any] = {} + self._files: Mapping[str, Any] = {} + self._file_type: Optional[str] = None + self._expected_spec: Optional[Mapping[str, Any]] = None + self._expected_check_status: Optional[str] = None + self._expected_catalog: Mapping[str, Any] = {} + self._expected_logs: Optional[Mapping[str, Any]] = None + self._expected_records: List[Mapping[str, Any]] = [] + self._availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None + self._discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy() + self._validation_policies: Optional[Mapping[str, AbstractSchemaValidationPolicy]] = None + self._parsers = default_parsers + self._stream_reader: Optional[AbstractFileBasedStreamReader] = None + self._expected_check_error: Tuple[Optional[Type[Exception]], Optional[str]] = None, None + self._expected_discover_error: Tuple[Optional[Type[Exception]], Optional[str]] = None, None + self._expected_read_error: Tuple[Optional[Type[Exception]], Optional[str]] = None, None + self._incremental_scenario_config: Optional[IncrementalScenarioConfig] = None + self._file_write_options: Mapping[str, Any] = {} + self._cursor_cls: Optional[Type[AbstractFileBasedCursor]] = None + + def set_name(self, name: str) -> "TestScenarioBuilder": + self._name = name + return self + + def set_config(self, config: Mapping[str, Any]) -> "TestScenarioBuilder": + self._config = config + return self + + def set_files(self, files: Mapping[str, Any]) -> "TestScenarioBuilder": + self._files = files + return self + + def set_file_type(self, file_type: str) -> "TestScenarioBuilder": + self._file_type = file_type + return self + + def set_expected_spec(self, expected_spec: Mapping[str, Any]) -> "TestScenarioBuilder": + self._expected_spec = expected_spec + return self + + def set_expected_check_status(self, expected_check_status: str) -> "TestScenarioBuilder": + self._expected_check_status = expected_check_status + return self + + def set_expected_catalog(self, expected_catalog: Mapping[str, Any]) -> "TestScenarioBuilder": + self._expected_catalog = expected_catalog + return self + + def set_expected_logs(self, expected_logs: Mapping[str, List[Mapping[str, Any]]]) -> "TestScenarioBuilder": + self._expected_logs = expected_logs + return self + + def set_expected_records(self, expected_records: List[Mapping[str, Any]]) -> "TestScenarioBuilder": + self._expected_records = expected_records + return self + + def set_parsers(self, parsers: Mapping[str, FileTypeParser]) -> "TestScenarioBuilder": + self._parsers = parsers + return self + + def set_availability_strategy(self, availability_strategy: AbstractFileBasedAvailabilityStrategy) -> "TestScenarioBuilder": + self._availability_strategy = availability_strategy + return self + + def set_discovery_policy(self, discovery_policy: AbstractDiscoveryPolicy) -> "TestScenarioBuilder": + self._discovery_policy = discovery_policy + return self + + def set_validation_policies(self, validation_policies: Mapping[str, AbstractSchemaValidationPolicy]) -> "TestScenarioBuilder": + self._validation_policies = validation_policies + return self + + def set_stream_reader(self, stream_reader: AbstractFileBasedStreamReader) -> "TestScenarioBuilder": + self._stream_reader = stream_reader + return self + + def set_cursor_cls(self, cursor_cls: AbstractFileBasedCursor) -> "TestScenarioBuilder": + self._cursor_cls = cursor_cls + return self + + def set_incremental_scenario_config(self, incremental_scenario_config: IncrementalScenarioConfig) -> "TestScenarioBuilder": + self._incremental_scenario_config = incremental_scenario_config + return self + + def set_expected_check_error(self, error: Optional[Type[Exception]], message: str) -> "TestScenarioBuilder": + self._expected_check_error = error, message + return self + + def set_expected_discover_error(self, error: Type[Exception], message: str) -> "TestScenarioBuilder": + self._expected_discover_error = error, message + return self + + def set_expected_read_error(self, error: Type[Exception], message: str) -> "TestScenarioBuilder": + self._expected_read_error = error, message + return self + + def set_file_write_options(self, file_write_options: Mapping[str, Any]) -> "TestScenarioBuilder": + self._file_write_options = file_write_options + return self + + def copy(self) -> "TestScenarioBuilder": + return deepcopy(self) + + def build(self) -> TestScenario: + if self._file_type is None: + raise ValueError("file_type is not set") + return TestScenario( + self._name, + self._config, + self._files, + self._file_type, + self._expected_spec, + self._expected_check_status, + self._expected_catalog, + self._expected_logs, + self._expected_records, + self._availability_strategy, + self._discovery_policy, + self._validation_policies or {}, + self._parsers, + self._stream_reader, + self._expected_check_error, + self._expected_discover_error, + self._expected_read_error, + self._incremental_scenario_config, + self._file_write_options, + self._cursor_cls, + ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py new file mode 100644 index 000000000000..33c8587a04d2 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py @@ -0,0 +1,708 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +""" +User input schema rules: + - `check`: Successful if the schema conforms to a record, otherwise ConfigValidationError. + - `discover`: User-input schema is output if the schema is valid, otherwise ConfigValidationError. + - `read`: If the schema is valid, record values are cast to types in the schema; if this is successful + the records are emitted. otherwise an error is logged. If the schema is not valid, ConfigValidationError. +""" + + +_base_user_input_schema_scenario = ( + TestScenarioBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "string" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + ] + ) +) + + +valid_single_stream_user_input_schema_scenario = ( + _base_user_input_schema_scenario.copy() + .set_name("valid_single_stream_user_input_schema_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "string"}', + } + ] + } + ) + .set_expected_check_status("SUCCEEDED") +).build() + + +single_stream_user_input_schema_scenario_schema_is_invalid = ( + _base_user_input_schema_scenario.copy() + .set_name("single_stream_user_input_schema_scenario_schema_is_invalid") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "x", "col2": "string"}', + } + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) + .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) + .set_expected_read_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) +).build() + + +single_stream_user_input_schema_scenario_emit_nonconforming_records = ( + _base_user_input_schema_scenario.copy() + .set_name("single_stream_user_input_schema_scenario_emit_nonconforming_records") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "integer", "col2": "string"}', + } + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(None, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "integer" + }, + "col2": { + "type": "string" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +).build() + + +single_stream_user_input_schema_scenario_skip_nonconforming_records = ( + _base_user_input_schema_scenario.copy() + .set_name("single_stream_user_input_schema_scenario_skip_nonconforming_records") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*"], + "validation_policy": "Skip Record", + "input_schema": '{"col1": "integer", "col2": "string"}', + } + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(None, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "integer" + }, + "col2": { + "type": "string" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records([]) + .set_expected_logs({ + "read": [ + { + 'level': 'WARN', + 'message': 'Records in file did not pass validation policy. stream=stream1 file=a.csv n_skipped=2 validation_policy=skip_record', + }, + { + 'level': "WARN", + 'message': 'Could not cast the value to the expected type.: col1: value=val11,expected_type=integer', + }, + { + 'level': "WARN", + 'message': 'Could not cast the value to the expected type.: col1: value=val21,expected_type=integer', + }, + ] + }) +).build() + + +_base_multi_stream_user_input_schema_scenario = ( + TestScenarioBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", 21), + ("val12a", 22), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "c.csv": { + "contents": [ + ("col1",), + ("val11c",), + ("val21c",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string", + }, + "col2": { + "type": "integer", + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string", + }, + "col2": { + "type": "string", + }, + "col3": { + "type": "string", + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream2", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + } + }, + "name": "stream3", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": 21, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val12a", "col2": 22, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + # The files in b.csv are emitted despite having an invalid schema + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, + {"data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + {"data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + ] + ) +) + + +valid_multi_stream_user_input_schema_scenario = ( + _base_multi_stream_user_input_schema_scenario.copy() + .set_name("valid_multi_stream_user_input_schema_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "integer"}', + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}', + }, + { + "name": "stream3", + "file_type": "csv", + "globs": ["c.csv"], + "validation_policy": "Emit Record", + }, + + ] + } + ) + .set_expected_check_status("SUCCEEDED") +).build() + + +multi_stream_user_input_schema_scenario_schema_is_invalid = ( + _base_multi_stream_user_input_schema_scenario.copy() + .set_name("multi_stream_user_input_schema_scenario_schema_is_invalid") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "integer"}', + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "x", "col2": "string", "col3": "string"}', # this stream's schema is invalid + }, + { + "name": "stream3", + "file_type": "csv", + "globs": ["c.csv"], + "validation_policy": "Emit Record", + }, + + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) + .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) + .set_expected_read_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) +).build() + + +multi_stream_user_input_schema_scenario_emit_nonconforming_records = ( + _base_multi_stream_user_input_schema_scenario.copy() + .set_name("multi_stream_user_input_schema_scenario_emit_nonconforming_records") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "integer"}', + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}', # this stream's records do not conform to the schema + }, + { + "name": "stream3", + "file_type": "csv", + "globs": ["c.csv"], + "validation_policy": "Emit Record", + }, + + ] + } + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "integer" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "integer" + }, + "col3": { + "type": "string" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream3", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": 21, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val12a", "col2": 22, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, + {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, + {"data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + {"data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + ] + ) + .set_expected_logs({ + "read": [ + { + 'level': "WARN", + 'message': 'Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer', + }, + { + 'level': "WARN", + 'message': 'Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer', + }, + ] + + }) +).build() + + +multi_stream_user_input_schema_scenario_skip_nonconforming_records = ( + _base_multi_stream_user_input_schema_scenario.copy() + .set_name("multi_stream_user_input_schema_scenario_skip_nonconforming_records") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a.csv"], + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "integer"}', + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b.csv"], + "validation_policy": "Skip Record", + "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}', # this stream's records do not conform to the schema + }, + { + "name": "stream3", + "file_type": "csv", + "globs": ["c.csv"], + "validation_policy": "Emit Record", + }, + + ] + } + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "integer" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "integer" + }, + "col3": { + "type": "string" + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": ["null", "string"], + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream3", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_check_status("FAILED") + .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) + .set_expected_records( + [ + {"data": {"col1": "val11a", "col2": 21, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val12a", "col2": 22, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + # "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, + # {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + # "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, + {"data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + {"data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + ] + ) + .set_expected_logs({ + "read": [ + { + 'level': 'WARN', + 'message': 'Records in file did not pass validation policy. stream=stream2 file=b.csv n_skipped=2 validation_policy=skip_record', + }, + { + 'level': "WARN", + 'message': 'Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer', + }, + { + 'level': "WARN", + 'message': 'Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer', + }, + ] + }) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py new file mode 100644 index 000000000000..fb2abf648e1e --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py @@ -0,0 +1,508 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +_base_single_stream_scenario = ( + TestScenarioBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val_a_11", "1"), + ("val_a_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { # The records in this file do not conform to the schema + "contents": [ + ("col1", "col2"), + ("val_b_11", "this is text that will trigger validation policy"), + ("val_b_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "c.csv": { + "contents": [ + ("col1",), + ("val_c_11",), + ("val_c_12", "val_c_22"), # This record is not parsable + ("val_c_13",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "d.csv": { + "contents": [ + ("col1",), + ("val_d_11",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + 'default_cursor_field': ['_ab_source_file_last_modified'], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string", + }, + "col2": { + "type": "integer", + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) +) + + +_base_multi_stream_scenario = ( + TestScenarioBuilder() + .set_files( + { + "a/a1.csv": { + "contents": [ + ("col1", "col2"), + ("val_aa1_11", "1"), + ("val_aa1_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "a/a2.csv": { + "contents": [ + ("col1", "col2"), + ("val_aa2_11", "this is text that will trigger validation policy"), + ("val_aa2_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "a/a3.csv": { + "contents": [ + ("col1",), + ("val_aa3_11",), + ("val_aa3_12", "val_aa3_22"), # This record is not parsable + ("val_aa3_13",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "a/a4.csv": { + "contents": [ + ("col1",), + ("val_aa4_11",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + + "b/b1.csv": { # The records in this file do not conform to the schema + "contents": [ + ("col1", "col2"), + ("val_bb1_11", "1"), + ("val_bb1_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b/b2.csv": { + "contents": [ + ("col1", "col2"), + ("val_bb2_11", "this is text that will trigger validation policy"), + ("val_bb2_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b/b3.csv": { + "contents": [ + ("col1",), + ("val_bb3_11",), + ("val_bb3_12",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + + } + ) + .set_file_type("csv") + .set_expected_catalog( + { + "streams": [ + { + 'default_cursor_field': ['_ab_source_file_last_modified'], + "json_schema": { + "type": "object", + "properties": { + "col1": { + "type": "string", + }, + "col2": { + "type": "integer", + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "json_schema": { + 'default_cursor_field': ['_ab_source_file_last_modified'], + "type": "object", + "properties": { + "col1": { + "type": "string", + }, + "col2": { + "type": "integer", + }, + "_ab_source_file_last_modified": { + "type": "string" + }, + "_ab_source_file_url": { + "type": "string" + }, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) +) + + +skip_record_scenario_single_stream = ( + _base_single_stream_scenario.copy() + .set_name("skip_record_scenario_single_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Skip Record", + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val_a_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_a_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_b_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, # This record is skipped because it does not conform + {"data": {"col1": "val_b_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_c_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_c_12", None: "val_c_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted + # {"data": {"col1": "val_c_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed + {"data": {"col1": "val_d_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, + ] + ) + .set_expected_logs({ + "read": [ + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream1 file=b.csv n_skipped=1 validation_policy=skip_record", + }, + { + "level": "ERROR", + "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=c.csv line_no=2 n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + }) +).build() + + +skip_record_scenario_multi_stream = ( + _base_multi_stream_scenario.copy() + .set_name("skip_record_scenario_multi_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a/*.csv"], + "validation_policy": "Skip Record", + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b/*.csv"], + "validation_policy": "Skip Record", + } + + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val_aa1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, # This record is skipped because it does not conform + {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted + # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed + {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_bb1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + # {"data": {"col1": "val_bb2_11", "col2": "val_bb2_21", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, # This record is skipped because it does not conform + {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, + ] + ) + .set_expected_logs({ + "read": [ + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream1 file=a/a2.csv n_skipped=1 validation_policy=skip_record", + }, + { + "level": "ERROR", + "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=a/a3.csv line_no=2 n_skipped=0", + }, + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream2 file=b/b2.csv n_skipped=1 validation_policy=skip_record", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + }) +).build() + + +emit_record_scenario_single_stream = ( + _base_single_stream_scenario.copy() + .set_name("emit_record_scenario_single_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val_a_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_a_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_b_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, # This record is skipped because it does not conform + {"data": {"col1": "val_b_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_c_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_c_12", None: "val_c_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted + # {"data": {"col1": "val_c_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # No more records from this stream are emitted after we hit a parse error + {"data": {"col1": "val_d_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, + ] + ) + .set_expected_logs({ + "read": [ + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=c.csv line_no=2 n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + }) +).build() + + +emit_record_scenario_multi_stream = ( + _base_multi_stream_scenario.copy() + .set_name("emit_record_scenario_multi_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a/*.csv"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b/*.csv"], + "validation_policy": "Emit Record", + } + + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val_aa1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted + # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed + {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_bb1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, + ] + ) + .set_expected_logs({ + "read": [ + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a/a3.csv line_no=2 n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + }) +).build() + + +wait_for_rediscovery_scenario_single_stream = ( + _base_single_stream_scenario.copy() + .set_name("wait_for_rediscovery_scenario_single_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["*.csv"], + "validation_policy": "Wait for Discover", + } + ] + } + ) + .set_expected_records([ + {"data": {"col1": "val_a_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_a_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + # No records past that because the first record for the second file did not conform to the schema + ]) + .set_expected_logs({ + "read": [ + { + "level": "WARN", + "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream1 file=b.csv validation_policy=Wait for Discover n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + }) +).build() + + +wait_for_rediscovery_scenario_multi_stream = ( + _base_multi_stream_scenario.copy() + .set_name("wait_for_rediscovery_scenario_multi_stream") + .set_config( + { + "streams": [ + { + "name": "stream1", + "file_type": "csv", + "globs": ["a/*.csv"], + "validation_policy": "Wait for Discover", + }, + { + "name": "stream2", + "file_type": "csv", + "globs": ["b/*.csv"], + "validation_policy": "Wait for Discover", + } + + ] + } + ) + .set_expected_records( + [ + {"data": {"col1": "val_aa1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_aa1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + # {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, + {"data": {"col1": "val_bb1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + {"data": {"col1": "val_bb1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + # {"data": {"col1": "val_bb2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, + # {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, + # {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, + # {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, + ] + ) + .set_expected_logs({ + "read": [ + { + "level": "WARN", + "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream1 file=a/a2.csv validation_policy=Wait for Discover n_skipped=0", + }, + { + "level": "WARN", + "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream2 file=b/b2.csv validation_policy=Wait for Discover n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + }) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py b/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py new file mode 100644 index 000000000000..957dcd356bad --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping + +import pytest +from airbyte_cdk.sources.file_based.config.file_based_stream_config import ValidationPolicy +from airbyte_cdk.sources.file_based.exceptions import StopSyncPerValidationPolicy +from airbyte_cdk.sources.file_based.schema_validation_policies import DEFAULT_SCHEMA_VALIDATION_POLICIES + +CONFORMING_RECORD = { + "col1": "val1", + "col2": 1, +} + +NONCONFORMING_RECORD = { + "col1": "val1", + "extra_col": "x", +} + + +SCHEMA = { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "integer" + }, + } +} + + +@pytest.mark.parametrize( + "record,schema,validation_policy,expected_result", + [ + pytest.param(CONFORMING_RECORD, SCHEMA, ValidationPolicy.emit_record, True, id="record-conforms_emit_record"), + pytest.param(NONCONFORMING_RECORD, SCHEMA, ValidationPolicy.emit_record, True, id="nonconforming_emit_record"), + pytest.param(CONFORMING_RECORD, SCHEMA, ValidationPolicy.skip_record, True, id="record-conforms_skip_record"), + pytest.param(NONCONFORMING_RECORD, SCHEMA, ValidationPolicy.skip_record, False, id="nonconforming_skip_record"), + pytest.param(CONFORMING_RECORD, SCHEMA, ValidationPolicy.wait_for_discover, True, id="record-conforms_wait_for_discover"), + pytest.param(NONCONFORMING_RECORD, SCHEMA, ValidationPolicy.wait_for_discover, False, id="nonconforming_wait_for_discover"), + ] +) +def test_record_passes_validation_policy( + record: Mapping[str, Any], + schema: Mapping[str, Any], + validation_policy: ValidationPolicy, + expected_result: bool +) -> None: + if validation_policy == ValidationPolicy.wait_for_discover and expected_result is False: + with pytest.raises(StopSyncPerValidationPolicy): + DEFAULT_SCHEMA_VALIDATION_POLICIES[validation_policy].record_passes_validation_policy(record, schema) + else: + assert DEFAULT_SCHEMA_VALIDATION_POLICIES[validation_policy].record_passes_validation_policy(record, schema) == expected_result diff --git a/tools/ci_credentials/tests/__init__.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/__init__.py similarity index 100% rename from tools/ci_credentials/tests/__init__.py rename to airbyte-cdk/python/unit_tests/sources/file_based/stream/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py new file mode 100644 index 000000000000..fdf755b42cdd --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py @@ -0,0 +1,268 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timedelta +from typing import Any, List, Mapping +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from freezegun import freeze_time + + +@pytest.mark.parametrize( + "files_to_add, expected_start_time, expected_state_dict", + [ + pytest.param([ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv") + + ], + [datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2020, 12, 31)], + { + "history": { + "a.csv": "2021-01-01T00:00:00.000000Z", + "b.csv": "2021-01-02T00:00:00.000000Z", + "c.csv": "2020-12-31T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2021-01-02T00:00:00.000000Z_b.csv", + }, + id="test_file_start_time_is_earliest_time_in_history"), + pytest.param([ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2021-01-03T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="d.csv", + last_modified=datetime.strptime("2021-01-04T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + + ], + [datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2021, 1, 2)], + { + "history": { + "b.csv": "2021-01-02T00:00:00.000000Z", + "c.csv": "2021-01-03T00:00:00.000000Z", + "d.csv": "2021-01-04T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2021-01-04T00:00:00.000000Z_d.csv", + }, + id="test_earliest_file_is_removed_from_history_if_history_is_full"), + pytest.param([ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="file_with_same_timestamp_as_b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2021-01-03T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="d.csv", + last_modified=datetime.strptime("2021-01-04T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + + ], + [datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2021, 1, 2), + datetime(2021, 1, 2), + ], + { + "history": { + "file_with_same_timestamp_as_b.csv": "2021-01-02T00:00:00.000000Z", + "c.csv": "2021-01-03T00:00:00.000000Z", + "d.csv": "2021-01-04T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2021-01-04T00:00:00.000000Z_d.csv", + }, + id="test_files_are_sorted_by_timestamp_and_by_name"), + ], +) +def test_add_file(files_to_add: List[RemoteFile], expected_start_time: List[datetime], expected_state_dict: Mapping[str, Any]) -> None: + cursor = get_cursor(max_history_size=3, days_to_sync_if_history_is_full=3) + assert cursor._compute_start_time() == datetime.min + + for index, f in enumerate(files_to_add): + cursor.add_file(f) + assert expected_start_time[index] == cursor._compute_start_time() + assert expected_state_dict == cursor.get_state() + + +@pytest.mark.parametrize("files, expected_files_to_sync, max_history_size, history_is_partial", [ + pytest.param([ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv") + ], + [ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv") + ], 3, True, id="test_all_files_should_be_synced"), + pytest.param([ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv") + ], [ + RemoteFile(uri="a.csv", + last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv"), + RemoteFile(uri="c.csv", + last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv") + + ], 2, True, id="test_sync_more_files_than_history_size"), +]) +def test_get_files_to_sync(files: List[RemoteFile], expected_files_to_sync: List[RemoteFile], max_history_size: int, history_is_partial: bool) -> None: + logger = MagicMock() + cursor = get_cursor(max_history_size, 3) + + files_to_sync = list(cursor.get_files_to_sync(files, logger)) + for f in files_to_sync: + cursor.add_file(f) + + assert files_to_sync == expected_files_to_sync + assert cursor._is_history_full() == history_is_partial + + +@freeze_time("2023-06-16T00:00:00Z") +def test_only_recent_files_are_synced_if_history_is_full() -> None: + logger = MagicMock() + cursor = get_cursor(2, 3) + + files_in_history = [ + RemoteFile(uri="b1.csv", last_modified=datetime(2021, 1, 2), file_type="csv"), + RemoteFile(uri="b2.csv", last_modified=datetime(2021, 1, 3), file_type="csv"), + ] + + state = { + "history": { + f.uri: f.last_modified.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) for f in files_in_history + }, + } + cursor.set_initial_state(state) + + files = [ + RemoteFile(uri="a.csv", last_modified=datetime(2021, 1, 1), file_type="csv"), + RemoteFile(uri="c.csv", last_modified=datetime(2021, 1, 2), file_type="csv"), + RemoteFile(uri="d.csv", last_modified=datetime(2021, 1, 4), file_type="csv"), + ] + + expected_files_to_sync = [ + RemoteFile(uri="c.csv", last_modified=datetime(2021, 1, 2), file_type="csv"), + RemoteFile(uri="d.csv", last_modified=datetime(2021, 1, 4), file_type="csv"), + ] + + files_to_sync = list(cursor.get_files_to_sync(files, logger)) + assert files_to_sync == expected_files_to_sync + logger.warning.assert_called_once() + + +@pytest.mark.parametrize("modified_at_delta, should_sync_file", [ + pytest.param(timedelta(days=-1), False, id="test_modified_at_is_earlier"), + pytest.param(timedelta(days=0), False, id="test_modified_at_is_equal"), + pytest.param(timedelta(days=1), True, id="test_modified_at_is_more_recent"), +]) +def test_sync_file_already_present_in_history(modified_at_delta: timedelta, should_sync_file: bool) -> None: + logger = MagicMock() + cursor = get_cursor(2, 3) + original_modified_at = datetime(2021, 1, 2) + filename = "a.csv" + files_in_history = [ + RemoteFile(uri=filename, last_modified=original_modified_at, file_type="csv"), + ] + + state = { + "history": { + f.uri: f.last_modified.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) for f in files_in_history + }, + } + cursor.set_initial_state(state) + + files = [ + RemoteFile(uri=filename, last_modified=original_modified_at + modified_at_delta, file_type="csv"), + ] + + files_to_sync = list(cursor.get_files_to_sync(files, logger)) + assert bool(files_to_sync) == should_sync_file + + +@freeze_time("2023-06-06T00:00:00Z") +@pytest.mark.parametrize( + "file_name, last_modified, earliest_dt_in_history, should_sync_file", [ + pytest.param("a.csv", datetime(2023, 6, 3), datetime(2023, 6, 6), True, id="test_last_modified_is_equal_to_time_buffer"), + pytest.param("b.csv", datetime(2023, 6, 6), datetime(2023, 6, 6), False, id="test_file_was_already_synced"), + pytest.param("b.csv", datetime(2023, 6, 7), datetime(2023, 6, 6), True, id="test_file_was_synced_in_the_past"), + pytest.param("b.csv", datetime(2023, 6, 3), datetime(2023, 6, 6), False, id="test_file_was_synced_in_the_past_but_last_modified_is_earlier_in_history"), + pytest.param("a.csv", datetime(2023, 6, 3), datetime(2023, 6, 3), False, id="test_last_modified_is_equal_to_earliest_dt_in_history_and_lexicographically_smaller"), + pytest.param("c.csv", datetime(2023, 6, 3), datetime(2023, 6, 3), True, id="test_last_modified_is_equal_to_earliest_dt_in_history_and_lexicographically_greater"), + ] +) +def test_should_sync_file(file_name: str, last_modified: datetime, earliest_dt_in_history: datetime, should_sync_file: bool) -> None: + logger = MagicMock() + cursor = get_cursor(1, 3) + + cursor.add_file(RemoteFile(uri="b.csv", last_modified=earliest_dt_in_history, file_type="csv")) + cursor._start_time = cursor._compute_start_time() + cursor._initial_earliest_file_in_history = cursor._compute_earliest_file_in_history() + + assert bool(list(cursor.get_files_to_sync([RemoteFile(uri=file_name, last_modified=last_modified, file_type="csv")], logger))) == should_sync_file + + +def test_set_initial_state_no_history() -> None: + cursor = get_cursor(1, 3) + cursor.set_initial_state({}) + + +def get_cursor(max_history_size: int, days_to_sync_if_history_is_full: int) -> DefaultFileBasedCursor: + cursor_cls = DefaultFileBasedCursor + cursor_cls.DEFAULT_MAX_HISTORY_SIZE = max_history_size + config = FileBasedStreamConfig( + file_type="csv", name="test", validation_policy=ValidationPolicy.emit_record, days_to_sync_if_history_is_full=days_to_sync_if_history_is_full) + return cursor_cls(config) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py new file mode 100644 index 000000000000..99b2ae789a4e --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from datetime import datetime, timezone +from typing import Any, Iterable, Iterator, Mapping +from unittest.mock import Mock + +import pytest +from airbyte_cdk.models import Level +from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy +from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream + + +@pytest.mark.parametrize( + "input_schema, expected_output", + [ + pytest.param({}, {}, id="empty-schema"), + pytest.param( + {"type": "string"}, + {"type": ["null", "string"]}, + id="simple-schema", + ), + pytest.param( + {"type": ["string"]}, + {"type": ["null", "string"]}, + id="simple-schema-list-type", + ), + pytest.param( + {"type": ["null", "string"]}, + {"type": ["null", "string"]}, + id="simple-schema-already-has-null", + ), + pytest.param( + {"properties": {"type": "string"}}, + {"properties": {"type": ["null", "string"]}}, + id="nested-schema", + ), + pytest.param( + {"items": {"type": "string"}}, + {"items": {"type": ["null", "string"]}}, + id="array-schema", + ), + pytest.param( + {"type": "object", "properties": {"prop": {"type": "string"}}}, + {"type": ["null", "object"], "properties": {"prop": {"type": ["null", "string"]}}}, + id="deeply-nested-schema", + ), + ], +) +def test_fill_nulls(input_schema: Mapping[str, Any], expected_output: Mapping[str, Any]) -> None: + assert DefaultFileBasedStream._fill_nulls(input_schema) == expected_output + + +class DefaultFileBasedStreamTest(unittest.TestCase): + _FILE_TYPE = "file_type" + _NOW = datetime(2022, 10, 22, tzinfo=timezone.utc) + _A_RECORD = {"a_record": 1} + + def setUp(self) -> None: + self._stream_config = Mock() + self._stream_config.file_type = self._FILE_TYPE + self._stream_config.name = "a stream name" + self._catalog_schema = Mock() + self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) + self._availability_strategy = Mock(spec=AbstractFileBasedAvailabilityStrategy) + self._discovery_policy = Mock(spec=AbstractDiscoveryPolicy) + self._parser = Mock(spec=FileTypeParser) + self._validation_policy = Mock(spec=AbstractSchemaValidationPolicy) + self._validation_policy.name = "validation policy name" + self._cursor = Mock(spec=AbstractFileBasedCursor) + + self._stream = DefaultFileBasedStream( + config=self._stream_config, + catalog_schema=self._catalog_schema, + stream_reader=self._stream_reader, + availability_strategy=self._availability_strategy, + discovery_policy=self._discovery_policy, + parsers={self._FILE_TYPE: self._parser}, + validation_policy=self._validation_policy, + cursor=self._cursor, + ) + + def test_when_read_records_from_slice_then_return_records(self) -> None: + self._parser.parse_records.return_value = [self._A_RECORD] + messages = list(self._stream.read_records_from_slice({"files": [RemoteFile(uri="uri", last_modified=self._NOW)]})) + assert list(map(lambda message: message.record.data["data"], messages)) == [self._A_RECORD] + + def test_given_exception_when_read_records_from_slice_then_do_process_other_files(self) -> None: + """ + The current behavior for source-s3 v3 does not fail sync on some errors and hence, we will keep this behaviour for now. One example + we can easily reproduce this is by having a file with gzip extension that is not actually a gzip file. The reader will fail to open + the file but the sync won't fail. + Ticket: https://github.com/airbytehq/airbyte/issues/29680 + """ + self._parser.parse_records.side_effect = [ValueError("An error"), [self._A_RECORD]] + + messages = list(self._stream.read_records_from_slice({"files": [ + RemoteFile(uri="invalid_file", last_modified=self._NOW), + RemoteFile(uri="valid_file", last_modified=self._NOW), + ]})) + + assert messages[0].log.level == Level.ERROR + assert messages[1].record.data["data"] == self._A_RECORD + + def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning(self) -> None: + self._stream_config.schemaless = False + self._validation_policy.record_passes_validation_policy.return_value = False + self._parser.parse_records.side_effect = [self._iter([self._A_RECORD, ValueError("An error")])] + + messages = list(self._stream.read_records_from_slice({"files": [ + RemoteFile(uri="invalid_file", last_modified=self._NOW), + RemoteFile(uri="valid_file", last_modified=self._NOW), + ]})) + + assert messages[0].log.level == Level.ERROR + assert messages[1].log.level == Level.WARN + + def _iter(self, x: Iterable[Any]) -> Iterator[Any]: + for item in x: + if isinstance(item, Exception): + raise item + yield item diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py new file mode 100644 index 000000000000..0f32399fd6df --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from io import IOBase +from typing import Any, Iterable, List, Mapping, Optional, Set + +import pytest +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from pydantic import AnyUrl +from unit_tests.sources.file_based.helpers import make_remote_files + +reader = AbstractFileBasedStreamReader + +""" +The rules are: + +- All files at top-level: /* +- All files at top-level of mydir: mydir/* +- All files anywhere under mydir: mydir/**/* +- All files in any directory: **/* +- All files in any directory that end in .csv: **/*.csv +- All files in any directory that have a .csv extension: **/*.csv* +""" + +FILEPATHS = [ + "a", "a.csv", "a.csv.gz", "a.jsonl", + "a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", + "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl", + "a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl", + "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl", + "a/b/c/d", "a/b/c/d.csv", "a/b/c/d.csv.gz", "a/b/c/d.jsonl" +] +FILES = make_remote_files(FILEPATHS) + +DEFAULT_CONFIG = { + "streams": [], +} + + +class TestStreamReader(AbstractFileBasedStreamReader): + @property + def config(self) -> Optional[AbstractFileBasedSpec]: + return self._config + + @config.setter + def config(self, value: AbstractFileBasedSpec) -> None: + self._config = value + + def get_matching_files(self, globs: List[str]) -> Iterable[RemoteFile]: + pass + + def open_file(self, file: RemoteFile) -> IOBase: + pass + + +class TestSpec(AbstractFileBasedSpec): + @classmethod + def documentation_url(cls) -> AnyUrl: + return AnyUrl(scheme="https", url="https://docs.airbyte.com/integrations/sources/test") # type: ignore + + +@pytest.mark.parametrize( + "globs,config,expected_matches,expected_path_prefixes", + [ + pytest.param([], DEFAULT_CONFIG, set(), set(), id="no-globs"), + pytest.param([""], DEFAULT_CONFIG, set(), set(), id="empty-string"), + pytest.param(["**"], DEFAULT_CONFIG, set(FILEPATHS), set(), id="**"), + pytest.param(["**/*.csv"], DEFAULT_CONFIG, {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), id="**/*.csv"), + pytest.param(["**/*.csv*"], DEFAULT_CONFIG, + {"a.csv", "a.csv.gz", "a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz", "a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", + "a/c/c.csv.gz", "a/b/c/d.csv", "a/b/c/d.csv.gz"}, set(), id="**/*.csv*"), + pytest.param(["*"], DEFAULT_CONFIG, {"a", "a.csv", "a.csv.gz", "a.jsonl"}, set(), id="*"), + pytest.param(["*.csv"], DEFAULT_CONFIG, {"a.csv"}, set(), id="*.csv"), + pytest.param(["*.csv*"], DEFAULT_CONFIG, {"a.csv", "a.csv.gz"}, set(), id="*.csv*"), + pytest.param(["*/*"], DEFAULT_CONFIG, {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl"}, set(), id="*/*"), + pytest.param(["*/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv"}, set(), id="*/*.csv"), + pytest.param(["*/*.csv*"], DEFAULT_CONFIG, {"a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz"}, set(), id="*/*.csv*"), + pytest.param(["*/**"], DEFAULT_CONFIG, + {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl", "a/b/c", "a/b/c.csv", + "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl", "a/b/c/d", "a/b/c/d.csv", + "a/b/c/d.csv.gz", "a/b/c/d.jsonl"}, set(), id="*/**"), + pytest.param(["a/*"], DEFAULT_CONFIG, {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl"}, + {"a/"}, id="a/*"), + pytest.param(["a/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv"}, {"a/"}, id="a/*.csv"), + pytest.param(["a/*.csv*"], DEFAULT_CONFIG, {"a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz"}, {"a/"}, id="a/*.csv*"), + pytest.param(["a/b/*"], DEFAULT_CONFIG, {"a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl"}, {"a/b/"}, id="a/b/*"), + pytest.param(["a/b/*.csv"], DEFAULT_CONFIG, {"a/b/c.csv"}, {"a/b/"}, id="a/b/*.csv"), + pytest.param(["a/b/*.csv*"], DEFAULT_CONFIG, {"a/b/c.csv", "a/b/c.csv.gz"}, {"a/b/"}, id="a/b/*.csv*"), + pytest.param(["a/*/*"], DEFAULT_CONFIG, + {"a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl"}, + {"a/"}, id="a/*/*"), + pytest.param(["a/*/*.csv"], DEFAULT_CONFIG, {"a/b/c.csv", "a/c/c.csv"}, {"a/"}, id="a/*/*.csv"), + pytest.param(["a/*/*.csv*"], DEFAULT_CONFIG, {"a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", "a/c/c.csv.gz"}, {"a/"}, id="a/*/*.csv*"), + pytest.param(["a/**/*"], DEFAULT_CONFIG, + {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl", "a/b/c", "a/b/c.csv", + "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl", "a/b/c/d", "a/b/c/d.csv", + "a/b/c/d.csv.gz", "a/b/c/d.jsonl"}, {"a/"}, id="a/**/*"), + pytest.param(["a/**/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, {"a/"}, + id="a/**/*.csv"), + pytest.param(["a/**/*.csv*"], DEFAULT_CONFIG, + {"a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz", "a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", "a/c/c.csv.gz", + "a/b/c/d.csv", "a/b/c/d.csv.gz"}, {"a/"}, id="a/**/*.csv*"), + pytest.param(["**/*.csv", "**/*.gz"], DEFAULT_CONFIG, + {"a.csv", "a.csv.gz", "a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz", "a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", + "a/c/c.csv.gz", "a/b/c/d.csv", "a/b/c/d.csv.gz"}, set(), id="**/*.csv,**/*.gz"), + pytest.param(["*.csv", "*.gz"], DEFAULT_CONFIG, {"a.csv", "a.csv.gz"}, set(), id="*.csv,*.gz"), + pytest.param(["a/*.csv", "a/*/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv"}, {"a/"}, + id="a/*.csv,a/*/*.csv"), + pytest.param(["a/*.csv", "a/b/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv"}, {"a/", "a/b/"}, id="a/*.csv,a/b/*.csv"), + pytest.param(["**/*.csv"], {"start_date": "2023-06-01T03:54:07.000Z", "streams": []}, + {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), + id="all_csvs_modified_after_start_date"), + pytest.param(["**/*.csv"], {"start_date": "2023-06-10T03:54:07.000Z", "streams": []}, set(), set(), + id="all_csvs_modified_before_start_date"), + pytest.param(["**/*.csv"], {"start_date": "2023-06-05T03:54:07.000Z", "streams": []}, + {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), + id="all_csvs_modified_exactly_on_start_date"), + ], +) +def test_globs_and_prefixes_from_globs(globs: List[str], config: Mapping[str, Any], expected_matches: Set[str], expected_path_prefixes: Set[str]) -> None: + reader = TestStreamReader() + reader.config = TestSpec(**config) + assert set([f.uri for f in reader.filter_files_by_globs_and_start_date(FILES, globs)]) == expected_matches + assert set(reader.get_prefixes_from_globs(globs)) == expected_path_prefixes diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py new file mode 100644 index 000000000000..e89e320430a6 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py @@ -0,0 +1,425 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import math +from pathlib import Path, PosixPath +from typing import Any, Dict, List, Mapping, Optional, Union + +import pytest +from _pytest.capture import CaptureFixture +from _pytest.reports import ExceptionInfo +from airbyte_cdk.entrypoint import launch +from airbyte_cdk.logger import AirbyteLogFormatter +from airbyte_cdk.models import SyncMode +from freezegun import freeze_time +from pytest import LogCaptureFixture +from unit_tests.sources.file_based.scenarios.avro_scenarios import ( + avro_all_types_scenario, + avro_file_with_double_as_number_scenario, + multiple_avro_combine_schema_scenario, + multiple_streams_avro_scenario, + single_avro_scenario, +) +from unit_tests.sources.file_based.scenarios.check_scenarios import ( + error_empty_stream_scenario, + error_listing_files_scenario, + error_multi_stream_scenario, + error_reading_file_scenario, + error_record_validation_user_provided_schema_scenario, + success_csv_scenario, + success_extensionless_scenario, + success_multi_stream_scenario, + success_user_provided_schema_scenario, +) +from unit_tests.sources.file_based.scenarios.csv_scenarios import ( + csv_autogenerate_column_names_scenario, + csv_custom_bool_values_scenario, + csv_custom_delimiter_in_double_quotes_scenario, + csv_custom_delimiter_with_escape_char_scenario, + csv_custom_format_scenario, + csv_custom_null_values_scenario, + csv_double_quote_is_set_scenario, + csv_escape_char_is_set_scenario, + csv_multi_stream_scenario, + csv_newline_in_values_not_quoted_scenario, + csv_newline_in_values_quoted_value_scenario, + csv_single_stream_scenario, + csv_skip_after_header_scenario, + csv_skip_before_and_after_header_scenario, + csv_skip_before_header_scenario, + csv_string_are_not_null_if_strings_can_be_null_is_false_scenario, + csv_string_can_be_null_with_input_schemas_scenario, + csv_string_not_null_if_no_null_values_scenario, + csv_strings_can_be_null_not_quoted_scenario, + earlier_csv_scenario, + empty_schema_inference_scenario, + invalid_csv_scenario, + multi_csv_scenario, + multi_csv_stream_n_file_exceeds_limit_for_inference, + multi_stream_custom_format, + schemaless_csv_multi_stream_scenario, + schemaless_csv_scenario, + schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_scenario, + single_csv_scenario, +) +from unit_tests.sources.file_based.scenarios.incremental_scenarios import ( + multi_csv_different_timestamps_scenario, + multi_csv_include_missing_files_within_history_range, + multi_csv_per_timestamp_scenario, + multi_csv_remove_old_files_if_history_is_full_scenario, + multi_csv_same_timestamp_more_files_than_history_size_scenario, + multi_csv_same_timestamp_scenario, + multi_csv_skip_file_if_already_in_history, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario, + single_csv_file_is_skipped_if_same_modified_at_as_in_history, + single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history, + single_csv_input_state_is_earlier_scenario, + single_csv_input_state_is_later_scenario, + single_csv_no_input_state_scenario, +) +from unit_tests.sources.file_based.scenarios.jsonl_scenarios import ( + invalid_jsonl_scenario, + jsonl_multi_stream_scenario, + jsonl_user_input_schema_scenario, + multi_jsonl_stream_n_bytes_exceeds_limit_for_inference, + multi_jsonl_stream_n_file_exceeds_limit_for_inference, + multi_jsonl_with_different_keys_scenario, + schemaless_jsonl_multi_stream_scenario, + schemaless_jsonl_scenario, + single_jsonl_scenario, +) +from unit_tests.sources.file_based.scenarios.parquet_scenarios import ( + multi_parquet_scenario, + parquet_file_with_decimal_as_float_scenario, + parquet_file_with_decimal_as_string_scenario, + parquet_file_with_decimal_no_config_scenario, + parquet_various_types_scenario, + parquet_with_invalid_config_scenario, + single_parquet_scenario, + single_partitioned_parquet_scenario, +) +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario +from unit_tests.sources.file_based.scenarios.user_input_schema_scenarios import ( + multi_stream_user_input_schema_scenario_emit_nonconforming_records, + multi_stream_user_input_schema_scenario_schema_is_invalid, + multi_stream_user_input_schema_scenario_skip_nonconforming_records, + single_stream_user_input_schema_scenario_emit_nonconforming_records, + single_stream_user_input_schema_scenario_schema_is_invalid, + single_stream_user_input_schema_scenario_skip_nonconforming_records, + valid_multi_stream_user_input_schema_scenario, + valid_single_stream_user_input_schema_scenario, +) +from unit_tests.sources.file_based.scenarios.validation_policy_scenarios import ( + emit_record_scenario_multi_stream, + emit_record_scenario_single_stream, + skip_record_scenario_multi_stream, + skip_record_scenario_single_stream, + wait_for_rediscovery_scenario_multi_stream, + wait_for_rediscovery_scenario_single_stream, +) + +discover_scenarios = [ + csv_multi_stream_scenario, + csv_single_stream_scenario, + invalid_csv_scenario, + single_csv_scenario, + multi_csv_scenario, + multi_csv_stream_n_file_exceeds_limit_for_inference, + single_csv_input_state_is_earlier_scenario, + single_csv_no_input_state_scenario, + single_csv_input_state_is_later_scenario, + multi_csv_same_timestamp_scenario, + multi_csv_different_timestamps_scenario, + multi_csv_per_timestamp_scenario, + multi_csv_skip_file_if_already_in_history, + multi_csv_include_missing_files_within_history_range, + multi_csv_remove_old_files_if_history_is_full_scenario, + multi_csv_same_timestamp_more_files_than_history_size_scenario, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario, + single_csv_file_is_skipped_if_same_modified_at_as_in_history, + single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history, + csv_custom_format_scenario, + earlier_csv_scenario, + multi_stream_custom_format, + empty_schema_inference_scenario, + single_parquet_scenario, + multi_parquet_scenario, + parquet_various_types_scenario, + parquet_file_with_decimal_no_config_scenario, + parquet_file_with_decimal_as_string_scenario, + parquet_file_with_decimal_as_float_scenario, + schemaless_csv_scenario, + schemaless_csv_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_scenario, + single_stream_user_input_schema_scenario_schema_is_invalid, + single_stream_user_input_schema_scenario_emit_nonconforming_records, + single_stream_user_input_schema_scenario_skip_nonconforming_records, + multi_stream_user_input_schema_scenario_emit_nonconforming_records, + multi_stream_user_input_schema_scenario_skip_nonconforming_records, + multi_stream_user_input_schema_scenario_schema_is_invalid, + valid_multi_stream_user_input_schema_scenario, + valid_single_stream_user_input_schema_scenario, + single_jsonl_scenario, + multi_jsonl_with_different_keys_scenario, + multi_jsonl_stream_n_file_exceeds_limit_for_inference, + multi_jsonl_stream_n_bytes_exceeds_limit_for_inference, + invalid_jsonl_scenario, + jsonl_multi_stream_scenario, + jsonl_user_input_schema_scenario, + schemaless_jsonl_scenario, + schemaless_jsonl_multi_stream_scenario, + csv_string_can_be_null_with_input_schemas_scenario, + csv_string_are_not_null_if_strings_can_be_null_is_false_scenario, + csv_string_not_null_if_no_null_values_scenario, + csv_strings_can_be_null_not_quoted_scenario, + csv_newline_in_values_quoted_value_scenario, + csv_escape_char_is_set_scenario, + csv_double_quote_is_set_scenario, + csv_custom_delimiter_with_escape_char_scenario, + csv_custom_delimiter_in_double_quotes_scenario, + csv_skip_before_header_scenario, + csv_skip_after_header_scenario, + csv_skip_before_and_after_header_scenario, + csv_custom_bool_values_scenario, + csv_custom_null_values_scenario, + single_avro_scenario, + avro_all_types_scenario, + multiple_avro_combine_schema_scenario, + multiple_streams_avro_scenario, + avro_file_with_double_as_number_scenario, + csv_newline_in_values_not_quoted_scenario, + csv_autogenerate_column_names_scenario, + parquet_with_invalid_config_scenario, + single_partitioned_parquet_scenario, +] + + +@pytest.mark.parametrize("scenario", discover_scenarios, ids=[s.name for s in discover_scenarios]) +def test_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> None: + expected_exc, expected_msg = scenario.expected_discover_error + expected_logs = scenario.expected_logs + if expected_exc: + with pytest.raises(expected_exc) as exc: + discover(capsys, tmp_path, scenario) + if expected_msg: + assert expected_msg in get_error_message_from_exc(exc) + else: + output = discover(capsys, tmp_path, scenario) + catalog, logs = output["catalog"], output["logs"] + assert catalog == scenario.expected_catalog + if expected_logs: + discover_logs = expected_logs.get("discover") + logs = [log for log in logs if log.get("log", {}).get("level") in ("ERROR", "WARN")] + _verify_expected_logs(logs, discover_logs) + + +read_scenarios = discover_scenarios + [ + emit_record_scenario_multi_stream, + emit_record_scenario_single_stream, + skip_record_scenario_multi_stream, + skip_record_scenario_single_stream, + wait_for_rediscovery_scenario_multi_stream, + wait_for_rediscovery_scenario_single_stream, +] + + +@pytest.mark.parametrize("scenario", read_scenarios, ids=[s.name for s in read_scenarios]) +@freeze_time("2023-06-09T00:00:00Z") +def test_read(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None: + caplog.handler.setFormatter(AirbyteLogFormatter()) + if scenario.incremental_scenario_config: + run_test_read_incremental(capsys, caplog, tmp_path, scenario) + else: + run_test_read_full_refresh(capsys, caplog, tmp_path, scenario) + + +def run_test_read_full_refresh(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None: + expected_exc, expected_msg = scenario.expected_read_error + if expected_exc: + with pytest.raises(expected_exc) as exc: # noqa + read(capsys, caplog, tmp_path, scenario) + if expected_msg: + assert expected_msg in get_error_message_from_exc(exc) + else: + output = read(capsys, caplog, tmp_path, scenario) + _verify_read_output(output, scenario) + + +def run_test_read_incremental(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None: + expected_exc, expected_msg = scenario.expected_read_error + if expected_exc: + with pytest.raises(expected_exc): + read_with_state(capsys, caplog, tmp_path, scenario) + else: + output = read_with_state(capsys, caplog, tmp_path, scenario) + _verify_read_output(output, scenario) + + +def _verify_read_output(output: Dict[str, Any], scenario: TestScenario) -> None: + records, logs = output["records"], output["logs"] + logs = [log for log in logs if log.get("level") in ("ERROR", "WARN", "WARNING")] + expected_records = scenario.expected_records + assert len(records) == len(expected_records) + for actual, expected in zip(records, expected_records): + if "record" in actual: + assert len(actual["record"]["data"]) == len(expected["data"]) + for key, value in actual["record"]["data"].items(): + if isinstance(value, float): + assert math.isclose(value, expected["data"][key], abs_tol=1e-04) + else: + assert value == expected["data"][key] + assert actual["record"]["stream"] == expected["stream"] + elif "state" in actual: + assert actual["state"]["data"] == expected + + if scenario.expected_logs: + read_logs = scenario.expected_logs.get("read") + assert len(logs) == (len(read_logs) if read_logs else 0) + _verify_expected_logs(logs, read_logs) + + +def _verify_expected_logs(logs: List[Dict[str, Any]], expected_logs: Optional[List[Mapping[str, Any]]]) -> None: + if expected_logs: + for actual, expected in zip(logs, expected_logs): + actual_level, actual_message = actual["level"], actual["message"] + expected_level = expected["level"] + expected_message = expected["message"] + assert actual_level == expected_level + assert expected_message in actual_message + + +spec_scenarios = [ + single_csv_scenario, +] + + +@pytest.mark.parametrize("scenario", spec_scenarios, ids=[c.name for c in spec_scenarios]) +def test_spec(capsys: CaptureFixture[str], scenario: TestScenario) -> None: + assert spec(capsys, scenario) == scenario.expected_spec + + +check_scenarios = [ + error_empty_stream_scenario, + error_listing_files_scenario, + error_reading_file_scenario, + error_record_validation_user_provided_schema_scenario, + error_multi_stream_scenario, + success_csv_scenario, + success_extensionless_scenario, + success_multi_stream_scenario, + success_user_provided_schema_scenario, + schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_scenario, + valid_single_stream_user_input_schema_scenario, + single_avro_scenario, + earlier_csv_scenario, +] + + +@pytest.mark.parametrize("scenario", check_scenarios, ids=[c.name for c in check_scenarios]) +def test_check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> None: + expected_exc, expected_msg = scenario.expected_check_error + + if expected_exc: + with pytest.raises(expected_exc): + output = check(capsys, tmp_path, scenario) + if expected_msg: + # expected_msg is a string. what's the expected value field? + assert expected_msg.value in output["message"] # type: ignore + assert output["status"] == scenario.expected_check_status + + else: + output = check(capsys, tmp_path, scenario) + assert output["status"] == scenario.expected_check_status + + +def spec(capsys: CaptureFixture[str], scenario: TestScenario) -> Mapping[str, Any]: + launch( + scenario.source, + ["spec"], + ) + captured = capsys.readouterr() + return json.loads(captured.out.splitlines()[0])["spec"] # type: ignore + + +def check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> Dict[str, Any]: + launch( + scenario.source, + ["check", "--config", make_file(tmp_path / "config.json", scenario.config)], + ) + captured = capsys.readouterr() + return json.loads(captured.out.splitlines()[0])["connectionStatus"] # type: ignore + + +def discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> Dict[str, Any]: + launch( + scenario.source, + ["discover", "--config", make_file(tmp_path / "config.json", scenario.config)], + ) + output = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + [catalog] = [o["catalog"] for o in output if o.get("catalog")] # type: ignore + return { + "catalog": catalog, + "logs": [o["log"] for o in output if o.get("log")], + } + + +def read(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> Dict[str, Any]: + with caplog.handler.stream as logger_stream: + launch( + scenario.source, + [ + "read", + "--config", + make_file(tmp_path / "config.json", scenario.config), + "--catalog", + make_file(tmp_path / "catalog.json", scenario.configured_catalog(SyncMode.full_refresh)), + ], + ) + captured = capsys.readouterr().out.splitlines() + logger_stream.getvalue().split("\n")[:-1] + + return { + "records": [msg for msg in (json.loads(line) for line in captured) if msg["type"] == "RECORD"], + "logs": [msg["log"] for msg in (json.loads(line) for line in captured) if msg["type"] == "LOG"], + } + + +def read_with_state( + capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario +) -> Dict[str, List[Any]]: + launch( + scenario.source, + [ + "read", + "--config", + make_file(tmp_path / "config.json", scenario.config), + "--catalog", + make_file(tmp_path / "catalog.json", scenario.configured_catalog(SyncMode.incremental)), + "--state", + make_file(tmp_path / "state.json", scenario.input_state()), + ], + ) + captured = capsys.readouterr() + logs = caplog.records + return { + "records": [msg for msg in (json.loads(line) for line in captured.out.splitlines()) if msg["type"] in ("RECORD", "STATE")], + "logs": [msg["log"] for msg in (json.loads(line) for line in captured.out.splitlines()) if msg["type"] == "LOG"] + + [{"level": log.levelname, "message": log.message} for log in logs], + } + + +def make_file(path: Path, file_contents: Optional[Union[Mapping[str, Any], List[Mapping[str, Any]]]]) -> str: + path.write_text(json.dumps(file_contents)) + return str(path) + + +def get_error_message_from_exc(exc: ExceptionInfo[Any]) -> str: + return str(exc.value.args[0]) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py new file mode 100644 index 000000000000..625fdb23cf87 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py @@ -0,0 +1,351 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, Optional + +import pytest +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, SchemaInferenceError +from airbyte_cdk.sources.file_based.schema_helpers import ( + ComparableType, + SchemaType, + conforms_to_schema, + merge_schemas, + type_mapping_to_jsonschema, +) + +COMPLETE_CONFORMING_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + + +NONCONFORMING_EXTRA_COLUMN_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, + "column_x": "extra" +} + +CONFORMING_WITH_MISSING_COLUMN_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], +} + +CONFORMING_WITH_NARROWER_TYPE_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": True, + "number_field": True, + "string_field": True, + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + +NONCONFORMING_WIDER_TYPE_RECORD = { + "null_field": "not None", + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + +NONCONFORMING_NON_OBJECT_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": "not an object", +} + +NONCONFORMING_NON_ARRAY_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": "not an array", + "object_field": {"col": "val"}, +} + +CONFORMING_MIXED_TYPE_NARROWER_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + +NONCONFORMING_MIXED_TYPE_WIDER_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + +CONFORMING_MIXED_TYPE_WITHIN_TYPE_RANGE_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "val1", + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + +NONCONFORMING_INVALID_ARRAY_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": ["this should not be an array"], + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + +NONCONFORMING_TOO_WIDE_ARRAY_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "okay", + "array_field": ["val1", "val2"], + "object_field": {"col": "val"}, +} + + +CONFORMING_NARROWER_ARRAY_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": "okay", + "array_field": [1, 2], + "object_field": {"col": "val"}, +} + + +NONCONFORMING_INVALID_OBJECT_RECORD = { + "null_field": None, + "boolean_field": True, + "integer_field": 1, + "number_field": 1.5, + "string_field": {"this": "should not be an object"}, + "array_field": [1.1, 2.2], + "object_field": {"col": "val"}, +} + + +SCHEMA = { + "type": "object", + "properties": { + "null_field": { + "type": "null" + }, + "boolean_field": { + "type": "boolean" + }, + "integer_field": { + "type": "integer" + }, + "number_field": { + "type": "number" + }, + "string_field": { + "type": "string" + }, + "array_field": { + "type": "array", + "items": { + "type": "number", + }, + }, + "object_field": { + "type": "object" + }, + } +} + + +@pytest.mark.parametrize( + "record,schema,expected_result", + [ + pytest.param(COMPLETE_CONFORMING_RECORD, SCHEMA, True, id="record-conforms"), + pytest.param(NONCONFORMING_EXTRA_COLUMN_RECORD, SCHEMA, False, id="nonconforming-extra-column"), + pytest.param(CONFORMING_WITH_MISSING_COLUMN_RECORD, SCHEMA, True, id="record-conforms-with-missing-column"), + pytest.param(CONFORMING_WITH_NARROWER_TYPE_RECORD, SCHEMA, True, id="record-conforms-with-narrower-type"), + pytest.param(NONCONFORMING_WIDER_TYPE_RECORD, SCHEMA, False, id="nonconforming-wider-type"), + pytest.param(NONCONFORMING_NON_OBJECT_RECORD, SCHEMA, False, id="nonconforming-string-is-not-an-object"), + pytest.param(NONCONFORMING_NON_ARRAY_RECORD, SCHEMA, False, id="nonconforming-string-is-not-an-array"), + pytest.param(NONCONFORMING_TOO_WIDE_ARRAY_RECORD, SCHEMA, False, id="nonconforming-array-values-too-wide"), + pytest.param(CONFORMING_NARROWER_ARRAY_RECORD, SCHEMA, True, id="conforming-array-values-narrower-than-schema"), + pytest.param(NONCONFORMING_INVALID_ARRAY_RECORD, SCHEMA, False, id="nonconforming-array-is-not-a-string"), + pytest.param(NONCONFORMING_INVALID_OBJECT_RECORD, SCHEMA, False, id="nonconforming-object-is-not-a-string"), + ] +) +def test_conforms_to_schema( + record: Mapping[str, Any], + schema: Mapping[str, Any], + expected_result: bool +) -> None: + assert conforms_to_schema(record, schema) == expected_result + + +def test_comparable_types() -> None: + assert ComparableType.OBJECT > ComparableType.STRING + assert ComparableType.STRING > ComparableType.NUMBER + assert ComparableType.NUMBER > ComparableType.INTEGER + assert ComparableType.INTEGER > ComparableType.BOOLEAN + assert ComparableType["OBJECT"] == ComparableType.OBJECT + + +@pytest.mark.parametrize( + "schema1,schema2,expected_result", + [ + pytest.param({}, {}, {}, id="empty-schemas"), + pytest.param({"a": None}, {}, None, id="null-value-in-schema"), + pytest.param({"a": {"type": "integer"}}, {}, {"a": {"type": "integer"}}, id="single-key-schema1"), + pytest.param({}, {"a": {"type": "integer"}}, {"a": {"type": "integer"}}, id="single-key-schema2"), + pytest.param({"a": {"type": "integer"}}, {"a": {"type": "integer"}}, {"a": {"type": "integer"}}, id="single-key-both-schemas"), + pytest.param({"a": {"type": "integer"}}, {"a": {"type": "number"}}, {"a": {"type": "number"}}, id="single-key-schema2-is-wider"), + pytest.param({"a": {"type": "number"}}, {"a": {"type": "integer"}}, {"a": {"type": "number"}}, id="single-key-schema1-is-wider"), + pytest.param({"a": {"type": "array"}}, {"a": {"type": "integer"}}, None, id="single-key-with-array-schema1"), + pytest.param({"a": {"type": "integer"}}, {"a": {"type": "array"}}, None, id="single-key-with-array-schema2"), + pytest.param({"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, id="single-key-same-object"), + pytest.param({"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "object", "properties": {"b": {"type": "string"}}}}, None, id="single-key-different-objects"), + pytest.param({"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "number"}}, None, id="single-key-with-object-schema1"), + pytest.param({"a": {"type": "number"}}, {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, None, id="single-key-with-object-schema2"), + pytest.param({"a": {"type": "array", "items": {"type": "number"}}}, {"a": {"type": "array", "items": {"type": "number"}}}, {"a": {"type": "array", "items": {"type": "number"}}}, id="equal-arrays-in-both-schemas"), + pytest.param({"a": {"type": "array", "items": {"type": "integer"}}}, {"a": {"type": "array", "items": {"type": "number"}}}, None, id="different-arrays-in-both-schemas"), + pytest.param({"a": {"type": "integer"}, "b": {"type": "string"}}, {"c": {"type": "number"}}, {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "number"}}, id=""), + pytest.param({"a": {"type": "invalid_type"}}, {"b": {"type": "integer"}}, None, id="invalid-type"), + ] +) +def test_merge_schemas(schema1: SchemaType, schema2: SchemaType, expected_result: Optional[SchemaType]) -> None: + if expected_result is not None: + assert merge_schemas(schema1, schema2) == expected_result + else: + with pytest.raises(SchemaInferenceError): + merge_schemas(schema1, schema2) + + +@pytest.mark.parametrize( + "type_mapping,expected_schema,expected_exc_msg", + [ + pytest.param( + '{"col1": "null", "col2": "array", "col3": "boolean", "col4": "float", "col5": "integer", "col6": "number", "col7": "object", "col8": "string"}', + { + "type": "object", + "properties": { + "col1": { + "type": "null" + }, + "col2": { + "type": "array" + }, + "col3": { + "type": "boolean" + }, + "col4": { + "type": "number" + }, + "col5": { + "type": "integer" + }, + "col6": { + "type": "number" + }, + "col7": { + "type": "object" + }, + "col8": { + "type": "string" + } + } + }, + None, + id="valid_all_types" + ), + pytest.param( + '{"col1 ": " string", "col2": " integer"}', + { + "type": "object", + "properties": { + "col1": { + "type": "string" + }, + "col2": { + "type": "integer" + } + } + }, + None, + id="valid_extra_spaces", + ), + pytest.param( + "", + None, + None, + id="valid_empty_string", + ), + pytest.param( + '{"col1": "x", "col2": "integer"}', + None, + "Invalid type 'x' for property 'col1'", + id="invalid_type", + ), + pytest.param( + '{"col1": "", "col2": "integer"}', + None, + "Invalid input schema", + id="invalid_missing_type", + ), + pytest.param( + '{"": "string", "col2": "integer"}', + None, + "Invalid input schema", + id="invalid_missing_name", + ), + pytest.param( + '{"type": "object", "properties": {"col1": {"type": "string"}, "col2": {"type": "integer"}}}', + None, + "Invalid input schema; nested schemas are not supported.", + id="invalid_nested_input_string", + ), + pytest.param( + '{"type": "object", "properties": {"col1": {"type": "string"}, "col2": {"type": "integer"}}}', + None, + "Invalid input schema; nested schemas are not supported.", + id="invalid_nested_input_json", + ), + ], +) +def test_type_mapping_to_jsonschema(type_mapping: Mapping[str, Any], expected_schema: Optional[Mapping[str, Any]], expected_exc_msg: Optional[str]) -> None: + if expected_exc_msg: + with pytest.raises(ConfigValidationError) as exc: + type_mapping_to_jsonschema(type_mapping) + assert expected_exc_msg in exc.value.args[0] + else: + assert type_mapping_to_jsonschema(type_mapping) == expected_schema diff --git a/airbyte-integrations/connectors/source-chargify/unit_tests/__init__.py b/airbyte-cdk/python/unit_tests/sources/fixtures/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-chargify/unit_tests/__init__.py rename to airbyte-cdk/python/unit_tests/sources/fixtures/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py b/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py new file mode 100644 index 000000000000..101b5dced98c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from abc import ABC +from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union + +import requests +from airbyte_cdk.models import ( + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + SyncMode, +) +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator +from requests.auth import AuthBase + + +class SourceTestFixture(AbstractSource): + """ + This is a concrete implementation of a Source connector that provides implementations of all the methods needed to run sync + operations. For simplicity, it also overrides functions that read from files in favor of returning the data directly avoiding + the need to load static files (ex. spec.yaml, config.json, configured_catalog.json) into the unit-test package. + """ + def __init__(self, streams: Optional[List[Stream]] = None, authenticator: Optional[AuthBase] = None): + self._streams = streams + self._authenticator = authenticator + + def spec(self, logger: logging.Logger) -> ConnectorSpecification: + return ConnectorSpecification(connectionSpecification={ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Test Fixture Spec", + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": { + "type": "string", + "title": "API token", + "description": "The token used to authenticate requests to the API.", + "airbyte_secret": True + } + } + }) + + def read_config(self, config_path: str) -> Mapping[str, Any]: + return { + "api_token": "just_some_token" + } + + @classmethod + def read_catalog(cls, catalog_path: str) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog(streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="http_test_stream", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + default_cursor_field=["updated_at"], + source_defined_cursor=True, + source_defined_primary_key=[["id"]] + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + ]) + + def check_connection(self, *args, **kwargs) -> Tuple[bool, Optional[Any]]: + return True, "" + + def streams(self, *args, **kwargs) -> List[Stream]: + return [HttpTestStream(authenticator=self._authenticator)] + + +class HttpTestStream(HttpStream, ABC): + url_base = "https://airbyte.com/api/v1/" + + def supports_incremental(self): + return True + + @property + def availability_strategy(self): + return None + + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + return "id" + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + return "cast" + + def parse_response( + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + body = response.json() or {} + return body["records"] + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def get_json_schema(self) -> Mapping[str, Any]: + return {} + + +def fixture_mock_send(self, request, **kwargs) -> requests.Response: + """ + Helper method that can be used by a test to patch the Session.send() function and mock the outbound send operation to provide + faster and more reliable responses compared to actual API requests + """ + response = requests.Response() + response.request = request + response.status_code = 200 + response.headers = {"header": "value"} + response_body = {"records": [ + {"id": 1, "name": "Celine Song", "position": "director"}, + {"id": 2, "name": "Shabier Kirchner", "position": "cinematographer"}, + {"id": 3, "name": "Christopher Bear", "position": "composer"}, + {"id": 4, "name": "Daniel Rossen", "position": "composer"} + ]} + response._content = json.dumps(response_body).encode("utf-8") + return response + + +class SourceFixtureOauthAuthenticator(Oauth2Authenticator): + """ + Test OAuth authenticator that only overrides the request and response aspect of the authenticator flow + """ + def refresh_access_token(self) -> Tuple[str, int]: + response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), params={}) + response.raise_for_status() + return "some_access_token", 1800 # Mock oauth response values to be used during the data retrieval step diff --git a/airbyte-cdk/python/unit_tests/sources/message/__init__.py b/airbyte-cdk/python/unit_tests/sources/message/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py new file mode 100644 index 000000000000..b8db5e08e53f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock + +import pytest +from airbyte_cdk.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteMessage, + AirbyteStateMessage, + Level, + OrchestratorType, + Type, +) +from airbyte_cdk.sources.message import ( + InMemoryMessageRepository, + LogAppenderMessageRepositoryDecorator, + MessageRepository, + NoopMessageRepository, +) +from pydantic.error_wrappers import ValidationError + +A_CONTROL = AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=0, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"a config": "value"}), +) +ANY_MESSAGE = AirbyteMessage(type=Type.CONTROL, control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=0, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"any message": "value"}), +)) +ANOTHER_CONTROL = AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=0, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"another config": "another value"}), +) +UNKNOWN_LEVEL = "potato" + + +class TestInMemoryMessageRepository: + def test_given_no_messages_when_consume_queue_then_return_empty(self): + repo = InMemoryMessageRepository() + messages = list(repo.consume_queue()) + assert messages == [] + + def test_given_messages_when_consume_queue_then_return_messages(self): + repo = InMemoryMessageRepository() + first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL) + repo.emit_message(first_message) + second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL) + repo.emit_message(second_message) + + messages = repo.consume_queue() + + assert list(messages) == [first_message, second_message] + + def test_given_message_is_consumed_when_consume_queue_then_remove_message_from_queue(self): + repo = InMemoryMessageRepository() + first_message = AirbyteMessage(type=Type.CONTROL, control=A_CONTROL) + repo.emit_message(first_message) + second_message = AirbyteMessage(type=Type.CONTROL, control=ANOTHER_CONTROL) + repo.emit_message(second_message) + + message_generator = repo.consume_queue() + consumed_message = next(message_generator) + assert consumed_message == first_message + + second_message_generator = repo.consume_queue() + assert list(second_message_generator) == [second_message] + + def test_given_message_is_not_control_nor_log_message_when_emit_message_then_raise_error(self): + repo = InMemoryMessageRepository() + with pytest.raises(ValueError): + repo.emit_message(AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "state value"}))) + + def test_given_log_level_is_severe_enough_when_log_message_then_allow_message_to_be_consumed(self): + repo = InMemoryMessageRepository(Level.DEBUG) + repo.log_message(Level.INFO, lambda: "this is a log message") + assert list(repo.consume_queue()) + + def test_given_log_level_is_severe_enough_when_log_message_then_filter_secrets(self, mocker): + filtered_message = "a filtered message" + mocker.patch("airbyte_cdk.sources.message.repository.filter_secrets", return_value=filtered_message) + repo = InMemoryMessageRepository(Level.DEBUG) + + repo.log_message(Level.INFO, lambda: "this is a log message") + + assert list(repo.consume_queue())[0].log.message == filtered_message + + def test_given_log_level_not_severe_enough_when_log_message_then_do_not_allow_message_to_be_consumed(self): + repo = InMemoryMessageRepository(Level.ERROR) + repo.log_message(Level.INFO, lambda: "this is a log message") + assert not list(repo.consume_queue()) + + def test_given_unknown_log_level_as_threshold_when_log_message_then_allow_message_to_be_consumed(self): + repo = InMemoryMessageRepository(UNKNOWN_LEVEL) + repo.log_message(Level.DEBUG, lambda: "this is a log message") + assert list(repo.consume_queue()) + + def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self): + """ + Pydantic will fail if the log level is unknown but on our side, we should try to log at least + """ + repo = InMemoryMessageRepository(Level.ERROR) + with pytest.raises(ValidationError): + repo.log_message(UNKNOWN_LEVEL, lambda: "this is a log message") + + +class TestNoopMessageRepository: + def test_given_message_emitted_when_consume_queue_then_return_empty(self): + repo = NoopMessageRepository() + repo.emit_message(AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)) + repo.log_message(Level.INFO, lambda: "this is a log message") + + assert not list(repo.consume_queue()) + + +class TestLogAppenderMessageRepositoryDecorator: + + _DICT_TO_APPEND = {"airbyte_cdk": {"stream": {"is_substream": False}}} + + @pytest.fixture() + def decorated(self): + return Mock(spec=MessageRepository) + + def test_when_emit_message_then_delegate_call(self, decorated): + repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.DEBUG) + repo.emit_message(ANY_MESSAGE) + decorated.emit_message.assert_called_once_with(ANY_MESSAGE) + + def test_when_log_message_then_append(self, decorated): + repo = LogAppenderMessageRepositoryDecorator({"a": {"dict_to_append": "appended value"}}, decorated, Level.DEBUG) + repo.log_message(Level.INFO, lambda: {"a": {"original": "original value"}}) + assert decorated.log_message.call_args_list[0].args[1]() == { + "a": { + "dict_to_append": "appended value", + "original": "original value" + } + } + + def test_given_value_clash_when_log_message_then_overwrite_value(self, decorated): + repo = LogAppenderMessageRepositoryDecorator({"clash": "appended value"}, decorated, Level.DEBUG) + repo.log_message(Level.INFO, lambda: {"clash": "original value"}) + assert decorated.log_message.call_args_list[0].args[1]() == {"clash": "appended value"} + + def test_given_log_level_is_severe_enough_when_log_message_then_allow_message_to_be_consumed(self, decorated): + repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.DEBUG) + repo.log_message(Level.INFO, lambda: {}) + assert decorated.log_message.call_count == 1 + + def test_given_log_level_not_severe_enough_when_log_message_then_do_not_allow_message_to_be_consumed(self, decorated): + repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.ERROR) + repo.log_message(Level.INFO, lambda: {}) + assert decorated.log_message.call_count == 0 + + def test_when_consume_queue_then_return_delegate_queue(self, decorated): + repo = LogAppenderMessageRepositoryDecorator(self._DICT_TO_APPEND, decorated, Level.DEBUG) + queue = [ANY_MESSAGE, ANY_MESSAGE, ANY_MESSAGE] + decorated.consume_queue.return_value = iter(queue) + + result = list(repo.consume_queue()) + + assert result == queue diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index 6724dd0343d6..df2eb08e8464 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -4,11 +4,13 @@ import json import logging +from unittest.mock import Mock import freezegun import pendulum import pytest import requests +from airbyte_cdk.models import OrchestratorType, Type from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, MultipleTokenAuthenticator, @@ -227,25 +229,31 @@ def test_init(self, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], ) assert authenticator.access_token == connector_config["credentials"]["access_token"] assert authenticator.get_refresh_token() == connector_config["credentials"]["refresh_token"] assert authenticator.get_token_expiry_date() == pendulum.parse(connector_config["credentials"]["token_expiry_date"]) - def test_init_with_invalid_config(self, invalid_connector_config): - with pytest.raises(ValueError): - SingleUseRefreshTokenOauth2Authenticator( - invalid_connector_config, - token_refresh_endpoint="foobar", - ) - @freezegun.freeze_time("2022-12-31") - def test_get_access_token(self, capsys, mocker, connector_config): + @pytest.mark.parametrize( + "test_name, expires_in_value, expiry_date_format, expected_expiry_date", + [ + ("number_of_seconds", 42, None, "2022-12-31T00:00:42+00:00"), + ("string_of_seconds", "42", None, "2022-12-31T00:00:42+00:00"), + ("date_format", "2023-04-04", "YYYY-MM-DD", "2023-04-04T00:00:00+00:00"), + ] + ) + def test_given_no_message_repository_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], + token_expiry_date_format=expiry_date_format, ) - authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", 42, "new_refresh_token")) + authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", expires_in_value, "new_refresh_token")) authenticator.token_has_expired = mocker.Mock(return_value=True) access_token = authenticator.get_access_token() captured = capsys.readouterr() @@ -253,7 +261,7 @@ def test_get_access_token(self, capsys, mocker, connector_config): expected_new_config = connector_config.copy() expected_new_config["credentials"]["access_token"] = "new_access_token" expected_new_config["credentials"]["refresh_token"] = "new_refresh_token" - expected_new_config["credentials"]["token_expiry_date"] = "2022-12-31T00:00:42+00:00" + expected_new_config["credentials"]["token_expiry_date"] = expected_expiry_date assert airbyte_message["control"]["connectorConfig"]["config"] == expected_new_config assert authenticator.access_token == access_token == "new_access_token" assert authenticator.get_refresh_token() == "new_refresh_token" @@ -264,30 +272,63 @@ def test_get_access_token(self, capsys, mocker, connector_config): assert not captured.out assert authenticator.access_token == access_token == "new_access_token" - def test_refresh_access_token(self, mocker, connector_config): + def test_given_message_repository_when_get_access_token_then_emit_message(self, mocker, connector_config): + message_repository = Mock() authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], + token_expiry_date_format="YYYY-MM-DD", + message_repository=message_repository, ) + authenticator.refresh_access_token = mocker.Mock(return_value=("new_access_token", "2023-04-04", "new_refresh_token")) + authenticator.token_has_expired = mocker.Mock(return_value=True) - authenticator._get_refresh_access_token_response = mocker.Mock( - return_value={ - authenticator.get_access_token_name(): "new_access_token", - authenticator.get_expires_in_name(): 42, - authenticator.get_refresh_token_name(): "new_refresh_token", - } + authenticator.get_access_token() + + emitted_message = message_repository.emit_message.call_args_list[0].args[0] + assert emitted_message.type == Type.CONTROL + assert emitted_message.control.type == OrchestratorType.CONNECTOR_CONFIG + assert emitted_message.control.connectorConfig.config["credentials"]["access_token"] == "new_access_token" + assert emitted_message.control.connectorConfig.config["credentials"]["refresh_token"] == "new_refresh_token" + assert emitted_message.control.connectorConfig.config["credentials"]["token_expiry_date"] == "2023-04-04T00:00:00+00:00" + assert emitted_message.control.connectorConfig.config["credentials"]["client_id"] == "my_client_id" + assert emitted_message.control.connectorConfig.config["credentials"]["client_secret"] == "my_client_secret" + + def test_given_message_repository_when_get_access_token_then_log_request(self, mocker, connector_config): + message_repository = Mock() + authenticator = SingleUseRefreshTokenOauth2Authenticator( + connector_config, + token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], + message_repository=message_repository, + ) + mocker.patch("airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.requests.request") + mocker.patch("airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.format_http_message", return_value="formatted json") + authenticator.token_has_expired = mocker.Mock(return_value=True) + + authenticator.get_access_token() + + assert message_repository.log_message.call_count == 1 + + def test_refresh_access_token(self, mocker, connector_config): + authenticator = SingleUseRefreshTokenOauth2Authenticator( + connector_config, + token_refresh_endpoint="foobar", + client_id=connector_config["credentials"]["client_id"], + client_secret=connector_config["credentials"]["client_secret"], ) - assert authenticator.refresh_access_token() == ("new_access_token", 42, "new_refresh_token") - # Test with expires_in as str authenticator._get_refresh_access_token_response = mocker.Mock( return_value={ authenticator.get_access_token_name(): "new_access_token", - authenticator.get_expires_in_name(): "1000", + authenticator.get_expires_in_name(): "42", authenticator.get_refresh_token_name(): "new_refresh_token", } ) - assert authenticator.refresh_access_token() == ("new_access_token", 1000, "new_refresh_token") + assert authenticator.refresh_access_token() == ("new_access_token", "42", "new_refresh_token") def mock_request(method, url, data): diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py index 6b37bd51671d..9cd89fb59967 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py @@ -22,9 +22,10 @@ class StubBasicReadHttpStream(HttpStream): url_base = "https://test_base_url.com" primary_key = "" - def __init__(self, **kwargs): + def __init__(self, deduplicate_query_params: bool = False, **kwargs): super().__init__(**kwargs) self.resp_counter = 1 + self._deduplicate_query_params = deduplicate_query_params def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None @@ -37,6 +38,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp self.resp_counter += 1 yield stubResp + def must_deduplicate_query_params(self) -> bool: + return self._deduplicate_query_params + def test_default_authenticator(): stream = StubBasicReadHttpStream() @@ -525,3 +529,28 @@ def test_default_get_error_display_message_handles_http_error(mocker): def test_join_url(test_name, base_url, path, expected_full_url): actual_url = HttpStream._join_url(base_url, path) assert actual_url == expected_full_url + + +@pytest.mark.parametrize( + "deduplicate_query_params, path, params, expected_url", [ + pytest.param(True, "v1/endpoint?param1=value1", {}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), + pytest.param(True, "v1/endpoint", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), + pytest.param(True, "v1/endpoint", None, "https://test_base_url.com/v1/endpoint", id="test_params_is_none_and_no_params_in_path"), + pytest.param(True, "v1/endpoint?param1=value1", None, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_is_none_and_no_params_in_path"), + pytest.param(True, "v1/endpoint?param1=value1", {"param2": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m2=value2", id="test_no_duplicate_params"), + pytest.param(True, "v1/endpoint?param1=value1", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_duplicate_params_same_value"), + pytest.param(True, "v1/endpoint?param1=1", {"param1": 1}, "https://test_base_url.com/v1/endpoint?param1=1", id="test_duplicate_params_same_value_not_string"), + pytest.param(True, "v1/endpoint?param1=value1", {"param1": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", id="test_duplicate_params_different_value"), + pytest.param(False, "v1/endpoint?param1=value1", {"param1": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", id="test_same_params_different_value_no_deduplication"), + pytest.param(False, "v1/endpoint?param1=value1", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value1", id="test_same_params_same_value_no_deduplication"), + ] +) +def test_duplicate_request_params_are_deduped(deduplicate_query_params, path, params, expected_url): + stream = StubBasicReadHttpStream(deduplicate_query_params) + + if expected_url is None: + with pytest.raises(ValueError): + stream._create_prepared_request(path=path, params=params) + else: + prepared_request = stream._create_prepared_request(path=path, params=params) + assert prepared_request.url == expected_url diff --git a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py index aaa347ed351e..7bbcc0e1813e 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py @@ -7,7 +7,7 @@ import logging from collections import defaultdict from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from unittest.mock import call +from unittest.mock import Mock, call import pytest from airbyte_cdk.models import ( @@ -37,9 +37,11 @@ from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from pytest import fixture logger = logging.getLogger("airbyte") @@ -50,10 +52,12 @@ def __init__( check_lambda: Callable[[], Tuple[bool, Optional[Any]]] = None, streams: List[Stream] = None, per_stream: bool = True, + message_repository: MessageRepository = None ): self._streams = streams self.check_lambda = check_lambda self.per_stream = per_stream + self._message_repository = message_repository def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: if self.check_lambda: @@ -69,6 +73,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: def per_stream_state_enabled(self) -> bool: return self.per_stream + @property + def message_repository(self): + return self._message_repository + class StreamNoStateMethod(Stream): name = "managers" @@ -97,6 +105,16 @@ def state(self, value: MutableMapping[str, Any]): self._cursor_value = value.get(self.cursor_field, self.start_date) +MESSAGE_FROM_REPOSITORY = Mock() + + +@fixture +def message_repository(): + message_repository = Mock(spec=MessageRepository) + message_repository.consume_queue.return_value = [message for message in [MESSAGE_FROM_REPOSITORY]] + return message_repository + + def test_successful_check(): """Tests that if a source returns TRUE for the connection check the appropriate connectionStatus success message is returned""" expected = AirbyteConnectionStatus(status=Status.SUCCEEDED) @@ -221,6 +239,34 @@ def test_read_nonexistent_stream_raises_exception(mocker): list(src.read(logger, {}, catalog)) +def test_read_stream_emits_repository_message_before_record(mocker, message_repository): + stream = MockStream(name="my_stream") + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(MockStream, "read_records", side_effect=[[{"a record": "a value"}, {"another record": "another value"}]]) + message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], []] + + source = MockSource(streams=[stream], message_repository=message_repository) + + messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)]))) + + assert messages.count(MESSAGE_FROM_REPOSITORY) == 1 + record_messages = (message for message in messages if message.type == Type.RECORD) + assert all(messages.index(MESSAGE_FROM_REPOSITORY) < messages.index(record) for record in record_messages) + + +def test_read_stream_emits_repository_message_on_error(mocker, message_repository): + stream = MockStream(name="my_stream") + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(MockStream, "read_records", side_effect=RuntimeError("error")) + message_repository.consume_queue.return_value = [message for message in [MESSAGE_FROM_REPOSITORY]] + + source = MockSource(streams=[stream], message_repository=message_repository) + + with pytest.raises(RuntimeError): + messages = list(source.read(logger, {}, ConfiguredAirbyteCatalog(streams=[_configured_stream(stream, SyncMode.full_refresh)]))) + assert MESSAGE_FROM_REPOSITORY in messages + + def test_read_stream_with_error_gets_display_message(mocker): stream = MockStream(name="my_stream") diff --git a/airbyte-cdk/python/unit_tests/sources/test_config.py b/airbyte-cdk/python/unit_tests/sources/test_config.py index c988e60df406..e9617da3684a 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_config.py +++ b/airbyte-cdk/python/unit_tests/sources/test_config.py @@ -43,7 +43,7 @@ class TestBaseConfig: "properties": { "count": {"title": "Count", "type": "integer"}, "name": {"title": "Name", "type": "string"}, - "selected_strategy": {"const": "option1", "title": "Selected " "Strategy", "type": "string"}, + "selected_strategy": {"const": "option1", "title": "Selected " "Strategy", "type": "string", "default": "option1"}, }, "required": ["name", "count"], "title": "Choice1", @@ -51,7 +51,7 @@ class TestBaseConfig: }, { "properties": { - "selected_strategy": {"const": "option2", "title": "Selected " "Strategy", "type": "string"}, + "selected_strategy": {"const": "option2", "title": "Selected " "Strategy", "type": "string", "default": "option2"}, "sequence": {"items": {"type": "string"}, "title": "Sequence", "type": "array"}, }, "required": ["sequence"], diff --git a/airbyte-cdk/python/unit_tests/sources/test_http_logger.py b/airbyte-cdk/python/unit_tests/sources/test_http_logger.py new file mode 100644 index 000000000000..a79a5216e2eb --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/test_http_logger.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +import requests +from airbyte_cdk.sources.http_logger import format_http_message + +A_TITLE = "a title" +A_DESCRIPTION = "a description" +A_STREAM_NAME = "a stream name" +ANY_REQUEST = requests.Request(method="POST", url="http://a-url.com", headers={}, params={}).prepare() + + +class ResponseBuilder: + def __init__(self): + self._body_content = "" + self._headers = {} + self._request = ANY_REQUEST + self._status_code = 100 + + def body_content(self, body_content: bytes) -> "ResponseBuilder": + self._body_content = body_content + return self + + def headers(self, headers: dict) -> "ResponseBuilder": + self._headers = headers + return self + + def request(self, request: requests.PreparedRequest) -> "ResponseBuilder": + self._request = request + return self + + def status_code(self, status_code: int) -> "ResponseBuilder": + self._status_code = status_code + return self + + def build(self): + response = requests.Response() + response._content = self._body_content + response.headers = self._headers + response.request = self._request + response.status_code = self._status_code + return response + + +EMPTY_RESPONSE = {"body": {"content": ""}, "headers": {}, "status_code": 100} + + +@pytest.mark.parametrize( + "test_name, http_method, url, headers, params, body_json, body_data, expected_airbyte_message", + [ + ( + "test_basic_get_request", + "GET", + "https://airbyte.io", + {}, + {}, + {}, + {}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": None}, "headers": {}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}}, + ), + ( + "test_get_request_with_headers", + "GET", + "https://airbyte.io", + {"h1": "v1", "h2": "v2"}, + {}, + {}, + {}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": None}, "headers": {"h1": "v1", "h2": "v2"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}}, + ), + ( + "test_get_request_with_request_params", + "GET", + "https://airbyte.io", + {}, + {"p1": "v1", "p2": "v2"}, + {}, + {}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": None}, "headers": {}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/?p1=v1&p2=v2"}}, + ), + ( + "test_get_request_with_request_body_json", + "GET", + "https://airbyte.io", + {"Content-Type": "application/json"}, + {}, + {"b1": "v1", "b2": "v2"}, + {}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"Content-Type": "application/json", "Content-Length": "24"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}} + ), + ( + "test_get_request_with_headers_params_and_body", + "GET", + "https://airbyte.io", + {"Content-Type": "application/json", "h1": "v1"}, + {"p1": "v1", "p2": "v2"}, + {"b1": "v1", "b2": "v2"}, + {}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"Content-Type": "application/json", "Content-Length": "24", "h1": "v1"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/?p1=v1&p2=v2"}}, + ), + ( + "test_get_request_with_request_body_data", + "GET", + "https://airbyte.io", + {"Content-Type": "application/x-www-form-urlencoded"}, + {}, + {}, + {"b1": "v1", "b2": "v2"}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": "b1=v1&b2=v2"}, "headers": {"Content-Type": "application/x-www-form-urlencoded", "Content-Length": "11"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}}, + ), + ( + "test_basic_post_request", + "POST", + "https://airbyte.io", + {}, + {}, + {}, + {}, + {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "POST", "body": {"content": None}, "headers": {"Content-Length": "0"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}} + ), + ], +) +def test_prepared_request_to_airbyte_message(test_name, http_method, url, headers, params, body_json, body_data, expected_airbyte_message): + request = requests.Request(method=http_method, url=url, headers=headers, params=params) + if body_json: + request.json = body_json + if body_data: + request.data = body_data + prepared_request = request.prepare() + + actual_airbyte_message = format_http_message(ResponseBuilder().request(prepared_request).build(), A_TITLE, A_DESCRIPTION, A_STREAM_NAME) + + assert actual_airbyte_message == expected_airbyte_message + + +@pytest.mark.parametrize( + "test_name, response_body, response_headers, status_code, expected_airbyte_message", + [ + ( + "test_response_no_body_no_headers", + b"", + {}, + 200, + {"body": {"content": ""}, "headers": {}, "status_code": 200} + ), + ( + "test_response_no_body_with_headers", + b"", + {"h1": "v1", "h2": "v2"}, + 200, + {"body": {"content": ""}, "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200} + ), + ( + "test_response_with_body_no_headers", + b'{"b1": "v1", "b2": "v2"}', + {}, + 200, + {"body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {}, "status_code": 200} + ), + ( + "test_response_with_body_and_headers", + b'{"b1": "v1", "b2": "v2"}', + {"h1": "v1", "h2": "v2"}, + 200, + {"body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200} + ), + ], +) +def test_response_to_airbyte_message(test_name, response_body, response_headers, status_code, expected_airbyte_message): + response = ResponseBuilder().body_content(response_body).headers(response_headers).status_code(status_code).build() + + actual_airbyte_message = format_http_message(response, A_TITLE, A_DESCRIPTION, A_STREAM_NAME) + + assert actual_airbyte_message["http"]["response"] == expected_airbyte_message diff --git a/airbyte-cdk/python/unit_tests/sources/test_integration_source.py b/airbyte-cdk/python/unit_tests/sources/test_integration_source.py new file mode 100644 index 000000000000..01f58b5adec7 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/test_integration_source.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +from unittest import mock +from unittest.mock import patch + +import pytest +import requests +from airbyte_cdk.entrypoint import launch +from unit_tests.sources.fixtures.source_test_fixture import ( + HttpTestStream, + SourceFixtureOauthAuthenticator, + SourceTestFixture, + fixture_mock_send, +) + + +@pytest.mark.parametrize( + "deployment_mode, url_base, expected_records, expected_error", + [ + pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"), + pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"), + pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://localhost:80/api/v1/", [], ValueError, id="test_cloud_read_with_localhost"), + pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"), + pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"), + ] +) +@patch.object(requests.Session, "send", fixture_mock_send) +def test_external_request_source(capsys, deployment_mode, url_base, expected_records, expected_error): + source = SourceTestFixture() + + with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict + with mock.patch.object(HttpTestStream, 'url_base', url_base): + args = ['read', '--config', 'config.json', '--catalog', 'configured_catalog.json'] + if expected_error: + with pytest.raises(expected_error): + launch(source, args) + else: + launch(source, args) + + +@pytest.mark.parametrize( + "deployment_mode, token_refresh_url, expected_records, expected_error", + [ + pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"), + pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"), + pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"), + pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"), + pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"), + ] +) +@patch.object(requests.Session, "send", fixture_mock_send) +def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error): + oauth_authenticator = SourceFixtureOauthAuthenticator( + client_id="nora", + client_secret="hae_sung", + refresh_token="arthur", + token_refresh_endpoint=token_refresh_url + ) + source = SourceTestFixture(authenticator=oauth_authenticator) + + with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict + args = ['read', '--config', 'config.json', '--catalog', 'configured_catalog.json'] + if expected_error: + with pytest.raises(expected_error): + launch(source, args) + else: + launch(source, args) diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index 5237c81f87ef..1996c56914c7 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -25,7 +25,7 @@ from airbyte_cdk.sources import AbstractSource, Source from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from airbyte_cdk.sources.streams.http.http import HttpStream +from airbyte_cdk.sources.streams.http.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from pydantic import ValidationError @@ -584,12 +584,95 @@ def __init__(self, *args, **kvargs): with caplog.at_level(logging.WARNING): records = [r for r in source.read(logger=logger, config={}, catalog=catalog, state={})] - # 0 for http stream, 3 for non http stream and 3 status trace meessages + # 0 for http stream, 3 for non http stream and 3 status trace messages assert len(records) == 0 + 3 + 3 assert non_http_stream.read_records.called expected_logs = [ f"Skipped syncing stream '{http_stream.name}' because it was unavailable.", - f"The endpoint to access stream '{http_stream.name}' returned 403: Forbidden.", + f"Unable to read {http_stream.name} stream.", + "This is most likely due to insufficient permissions on the credentials in use.", + f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more." + ] + for message in expected_logs: + assert message in caplog.text + + +def test_read_default_http_availability_strategy_parent_stream_unavailable(catalog, mocker, caplog): + """Test default availability strategy if error happens during slice extraction (reading of parent stream)""" + mocker.patch.multiple(Stream, __abstractmethods__=set()) + + class MockHttpParentStream(HttpStream): + url_base = "https://test_base_url.com" + primary_key = "" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.resp_counter = 1 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + stub_response = {"data": self.resp_counter} + self.resp_counter += 1 + yield stub_response + + class MockHttpStream(HttpSubStream): + url_base = "https://test_base_url.com" + primary_key = "" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.resp_counter = 1 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + stub_response = {"data": self.resp_counter} + self.resp_counter += 1 + yield stub_response + + http_stream = MockHttpStream(parent=MockHttpParentStream()) + streams = [http_stream] + assert isinstance(http_stream, HttpSubStream) + assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) + + # Patch HTTP request to stream endpoint to make it unavailable + req = requests.Response() + req.status_code = 403 + mocker.patch.object(requests.Session, "send", return_value=req) + + source = MockAbstractSource(streams=streams) + logger = logging.getLogger("test_read_default_http_availability_strategy_parent_stream_unavailable") + configured_catalog = { + "streams": [ + { + "stream": { + "name": "mock_http_stream", + "json_schema": {"type": "object", "properties": {"k": "v"}}, + "supported_sync_modes": ["full_refresh"], + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh", + } + ] + } + catalog = ConfiguredAirbyteCatalog.parse_obj(configured_catalog) + with caplog.at_level(logging.WARNING): + records = [r for r in source.read(logger=logger, config={}, catalog=catalog, state={})] + + # 0 for http stream, 3 for non http stream and 3 status trace messages + assert len(records) == 0 + expected_logs = [ + f"Skipped syncing stream '{http_stream.name}' because it was unavailable.", + f"Unable to get slices for {http_stream.name} stream, because of error in parent stream", "This is most likely due to insufficient permissions on the credentials in use.", f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more." ] diff --git a/airbyte-cdk/python/unit_tests/test_config_observation.py b/airbyte-cdk/python/unit_tests/test_config_observation.py index 50791080cf18..a182854493dc 100644 --- a/airbyte-cdk/python/unit_tests/test_config_observation.py +++ b/airbyte-cdk/python/unit_tests/test_config_observation.py @@ -6,7 +6,8 @@ import time import pytest -from airbyte_cdk.config_observation import ConfigObserver, ObservedDict, observe_connector_config +from airbyte_cdk.config_observation import ConfigObserver, ObservedDict, create_connector_config_control_message, observe_connector_config +from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, OrchestratorType, Type class TestObservedDict: @@ -74,3 +75,14 @@ def test_observe_already_observed_config(): observed_config = observe_connector_config({"foo": "bar"}) with pytest.raises(ValueError): observe_connector_config(observed_config) + + +def test_create_connector_config_control_message(): + A_CONFIG = {"config key": "config value"} + + message = create_connector_config_control_message(A_CONFIG) + + assert message.type == Type.CONTROL + assert message.control.type == OrchestratorType.CONNECTOR_CONFIG + assert message.control.connectorConfig == AirbyteControlConnectorConfigMessage(config=A_CONFIG) + assert message.control.emitted_at is not None diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index ab31a784d946..5cd37e5d503b 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -2,22 +2,27 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import os from argparse import Namespace from copy import deepcopy from typing import Any, List, Mapping, MutableMapping, Union -from unittest.mock import MagicMock +from unittest import mock +from unittest.mock import MagicMock, patch import pytest +import requests from airbyte_cdk import AirbyteEntrypoint from airbyte_cdk import entrypoint as entrypoint_module from airbyte_cdk.models import ( AirbyteCatalog, AirbyteConnectionStatus, + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, AirbyteMessage, AirbyteRecordMessage, AirbyteStream, ConnectorSpecification, + OrchestratorType, Status, SyncMode, Type, @@ -35,6 +40,10 @@ def discover(self, **kwargs): def check(self, **kwargs): pass + @property + def message_repository(self): + pass + def _as_arglist(cmd: str, named_args: Mapping[str, Any]) -> List[str]: out = [cmd] @@ -53,8 +62,21 @@ def spec_mock(mocker): return mock +MESSAGE_FROM_REPOSITORY = AirbyteMessage( + type=Type.CONTROL, + control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=10, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"any config": "a config value"}), + ) +) + + @pytest.fixture -def entrypoint() -> AirbyteEntrypoint: +def entrypoint(mocker) -> AirbyteEntrypoint: + message_repository = MagicMock() + message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], []] + mocker.patch.object(MockSource, "message_repository", new_callable=mocker.PropertyMock, return_value=message_repository) return AirbyteEntrypoint(MockSource()) @@ -124,7 +146,10 @@ def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): parsed_args = Namespace(command="spec") expected = ConnectorSpecification(connectionSpecification={"hi": "hi"}) mocker.patch.object(MockSource, "spec", return_value=expected) - assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages @pytest.fixture @@ -158,41 +183,116 @@ def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock, sch messages = list(entrypoint.run(parsed_args)) if config_valid: - assert [_wrap_message(check_value)] == messages + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages else: - assert len(messages) == 1 - airbyte_message = AirbyteMessage.parse_raw(messages[0]) - assert airbyte_message.type == Type.CONNECTION_STATUS - assert airbyte_message.connectionStatus.status == Status.FAILED - assert airbyte_message.connectionStatus.message.startswith("Config validation error:") + assert len(messages) == 2 + assert messages[0] == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) + connection_status_message = AirbyteMessage.parse_raw(messages[1]) + assert connection_status_message.type == Type.CONNECTION_STATUS + assert connection_status_message.connectionStatus.status == Status.FAILED + assert connection_status_message.connectionStatus.message.startswith("Config validation error:") def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="check", config="config_path") check_value = AirbyteConnectionStatus(status=Status.SUCCEEDED) mocker.patch.object(MockSource, "check", return_value=check_value) - assert [_wrap_message(check_value)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages assert spec_mock.called +def test_run_check_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="check", config="config_path") + mocker.patch.object(MockSource, "check", side_effect=ValueError("Any error")) + + with pytest.raises(ValueError): + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + + def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="discover", config="config_path") expected = AirbyteCatalog(streams=[AirbyteStream(name="stream", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh])]) mocker.patch.object(MockSource, "discover", return_value=expected) - assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages assert spec_mock.called +def test_run_discover_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="discover", config="config_path") + mocker.patch.object(MockSource, "discover", side_effect=ValueError("Any error")) + + with pytest.raises(ValueError): + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + + def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") expected = AirbyteRecordMessage(stream="stream", data={"data": "stuff"}, emitted_at=1) mocker.patch.object(MockSource, "read_state", return_value={}) mocker.patch.object(MockSource, "read_catalog", return_value={}) mocker.patch.object(MockSource, "read", return_value=[AirbyteMessage(record=expected, type=Type.RECORD)]) - assert [_wrap_message(expected)] == list(entrypoint.run(parsed_args)) + + messages = list(entrypoint.run(parsed_args)) + + assert [_wrap_message(expected), MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages assert spec_mock.called -def test_invalid_command(entrypoint: AirbyteEntrypoint, mocker, config_mock): +def test_run_read_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") + mocker.patch.object(MockSource, "read_state", return_value={}) + mocker.patch.object(MockSource, "read_catalog", return_value={}) + mocker.patch.object(MockSource, "read", side_effect=ValueError("Any error")) + + with pytest.raises(ValueError): + messages = list(entrypoint.run(parsed_args)) + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + + +def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock): with pytest.raises(Exception): list(entrypoint.run(Namespace(command="invalid", config="conf"))) + + +@pytest.mark.parametrize( + "deployment_mode, url, expected_error", + [ + pytest.param("CLOUD", "https://airbyte.com", None, id="test_cloud_public_endpoint_is_successful"), + pytest.param("CLOUD", "https://192.168.27.30", ValueError, id="test_cloud_private_ip_address_is_rejected"), + pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", ValueError, id="test_cloud_private_endpoint_is_rejected"), + pytest.param("CLOUD", "http://past.lives.net/api/v1/inyun", ValueError, id="test_cloud_unsecured_endpoint_is_rejected"), + pytest.param("CLOUD", "https://not:very/cash:443.money", ValueError, id="test_cloud_invalid_url_format"), + pytest.param("CLOUD", "https://192.168.27.30 ", ValueError, id="test_cloud_incorrect_ip_format_is_rejected"), + pytest.param("cloud", "https://192.168.27.30", ValueError, id="test_case_insensitive_cloud_environment_variable"), + pytest.param("OSS", "https://airbyte.com", None, id="test_oss_public_endpoint_is_successful"), + pytest.param("OSS", "https://192.168.27.30", None, id="test_oss_private_endpoint_is_successful"), + pytest.param("OSS", "https://localhost:8080/api/v1/cast", None, id="test_oss_private_endpoint_is_successful"), + pytest.param("OSS", "http://past.lives.net/api/v1/inyun", None, id="test_oss_unsecured_endpoint_is_successful"), + ] +) +@patch.object(requests.Session, "send", lambda self, request, **kwargs: requests.Response()) +def test_filter_internal_requests(deployment_mode, url, expected_error): + with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): + AirbyteEntrypoint(source=MockSource()) + + session = requests.Session() + + prepared_request = requests.PreparedRequest() + prepared_request.method = "GET" + prepared_request.headers = {"header": "value"} + prepared_request.url = url + + if expected_error: + with pytest.raises(expected_error): + session.send(request=prepared_request) + else: + actual_response = session.send(request=prepared_request) + assert isinstance(actual_response, requests.Response) diff --git a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py new file mode 100644 index 000000000000..68152184b66f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Dict, List + +import pytest +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer + +NOW = 1234567 + + +@pytest.mark.parametrize( + "test_name,input_records,expected_candidate_fields", + [ + ("empty", [], {}), + ("simple_match", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}), + ("timestamp_match_integer", [{"d": 1686058051}], {"d": "%s"}), + ("timestamp_match_string", [{"d": "1686058051"}], {"d": "%s"}), + ("timestamp_ms_match_integer", [{"d": 1686058051000}], {"d": "%ms"}), + ("timestamp_ms_match_string", [{"d": "1686058051000"}], {"d": "%ms"}), + ("timestamp_no_match_integer", [{"d": 99}], {}), + ("timestamp_no_match_string", [{"d": "99999999999999999999"}], {}), + ("simple_no_match", [{"d": "20220203"}], {}), + ("multiple_match", [{"d": "2022-02-03", "e": "2022-02-03"}], {"d": "%Y-%m-%d", "e": "%Y-%m-%d"}), + ( + "multiple_no_match", + [{"d": "20220203", "r": "ccc", "e": {"something-else": "2023-03-03"}, "s": ["2023-03-03"], "x": False, "y": 123}], + {}, + ), + ("format_1", [{"d": "2022-02-03"}], {"d": "%Y-%m-%d"}), + ("format_2", [{"d": "2022-02-03 12:34:56"}], {"d": "%Y-%m-%d %H:%M:%S"}), + ("format_3", [{"d": "2022-02-03T12:34:56Z"}], {"d": "%Y-%m-%dT%H:%M:%SZ"}), + ("format_4 1", [{"d": "2022-02-03T12:34:56.000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%fZ"}), + ("format_4 2", [{"d": "2022-02-03T12:34:56.000000Z"}], {"d": "%Y-%m-%dT%H:%M:%S.%fZ"}), + ("format_5", [{"d": "2022-02-03 12:34:56.123456+00:00"}], {"d": "%Y-%m-%d %H:%M:%S.%f%z"}), + ("format_5 2", [{"d": "2022-02-03 12:34:56.123456+02:00"}], {"d": "%Y-%m-%d %H:%M:%S.%f%z"}), + ("format_6", [{"d": "2022-02-03T12:34:56.123456+0000"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}), + ("format_6 2", [{"d": "2022-02-03T12:34:56.123456+00:00"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}), + ("format_6 3", [{"d": "2022-02-03T12:34:56.123456-03:00"}], {"d": "%Y-%m-%dT%H:%M:%S.%f%z"}), + ("format_7", [{"d": "03/02/2022 12:34"}], {"d": "%d/%m/%Y %H:%M"}), + ("format_8", [{"d": "2022-02"}], {"d": "%Y-%m"}), + ("format_9", [{"d": "03-02-2022"}], {"d": "%d-%m-%Y"}), + ("limit_down", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "2022-02-03", "x": "another thing"}], {"d": "%Y-%m-%d"}), + ("limit_down all", [{"d": "2022-02-03", "x": "2022-02-03"}, {"d": "also another thing", "x": "another thing"}], {}), + ("limit_down empty", [{"d": "2022-02-03", "x": "2022-02-03"}, {}], {}), + ("limit_down unsupported type", [{"d": "2022-02-03"}, {"d": False}], {}), + ("limit_down complex type", [{"d": "2022-02-03"}, {"d": {"date": "2022-03-03"}}], {}), + ("limit_down different format", [{"d": "2022-02-03"}, {"d": 1686058051}], {}), + ("limit_down different format", [{"d": "2022-02-03"}, {"d": "2022-02-03T12:34:56.000000Z"}], {}), + ("no scope expand", [{}, {"d": "2022-02-03"}], {}), + ], +) +def test_schema_inferrer(test_name, input_records: List, expected_candidate_fields: Dict[str, str]): + inferrer = DatetimeFormatInferrer() + for record in input_records: + inferrer.accumulate(AirbyteRecordMessage(stream="abc", data=record, emitted_at=NOW)) + assert inferrer.get_inferred_datetime_formats() == expected_candidate_fields diff --git a/airbyte-cdk/python/unit_tests/utils/test_mapping_helpers.py b/airbyte-cdk/python/unit_tests/utils/test_mapping_helpers.py new file mode 100644 index 000000000000..f5dc979e3477 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/utils/test_mapping_helpers.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.utils.mapping_helpers import combine_mappings + + +def test_basic_merge(): + mappings = [{"a": 1}, {"b": 2}, {"c": 3}, {}] + result = combine_mappings(mappings) + assert result == {"a": 1, "b": 2, "c": 3} + + +def test_combine_with_string(): + mappings = [{"a": 1}, "option"] + with pytest.raises(ValueError, match="Cannot combine multiple options if one is a string"): + combine_mappings(mappings) + + +def test_overlapping_keys(): + mappings = [{"a": 1, "b": 2}, {"b": 3}] + with pytest.raises(ValueError, match="Duplicate keys found"): + combine_mappings(mappings) + + +def test_multiple_strings(): + mappings = ["option1", "option2"] + with pytest.raises(ValueError, match="Cannot combine multiple string options"): + combine_mappings(mappings) + + +def test_handle_none_values(): + mappings = [{"a": 1}, None, {"b": 2}] + result = combine_mappings(mappings) + assert result == {"a": 1, "b": 2} + + +def test_empty_mappings(): + mappings = [] + result = combine_mappings(mappings) + assert result == {} + + +def test_single_mapping(): + mappings = [{"a": 1}] + result = combine_mappings(mappings) + assert result == {"a": 1} + + +def test_combine_with_string_and_empty_mappings(): + mappings = ["option", {}] + result = combine_mappings(mappings) + assert result == "option" diff --git a/airbyte-ci/README.md b/airbyte-ci/README.md index 81b4733ca0c1..b7985d1c9585 100644 --- a/airbyte-ci/README.md +++ b/airbyte-ci/README.md @@ -1,3 +1,5 @@ # Airbyte CI -This folder is a collection of systems, tools and scripts that are used to run Airbyte's CI/CD \ No newline at end of file +This folder is a collection of systems, tools and scripts that are used to run Airbyte's CI/CD + +The installation instructions for the `airbyte-ci` CLI tool cal be found here [airbyte-ci/connectors/pipelines](connectors/pipelines/README.md) \ No newline at end of file diff --git a/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml b/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml index 61db82fc08e8..7957973df4e7 100644 --- a/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml +++ b/airbyte-ci/connectors/CONNECTOR_CHECKLIST.yaml @@ -1,10 +1,11 @@ paths: "airbyte-integrations/connectors/**": - - PR name follows [PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/issues-and-pull-requests#pull-request-title-convention) - - "[Breaking changes are considered](https://docs.airbyte.com/contributing-to-airbyte/#breaking-changes-to-connectors). If a **Breaking Change** is being introduced, ensure an Airbyte engineer has created a Breaking Change Plan and you've followed all steps in the [Breaking Changes Checklist](https://docs.airbyte.com/contributing-to-airbyte/#checklist-for-contributors)" - - Connector version has been incremented in the Dockerfile and metadata.yaml according to our [Semantic Versioning for Connectors](https://docs.airbyte.com/contributing-to-airbyte/#semantic-versioning-for-connectors) guidelines + - PR name follows [PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention) + - "[Breaking changes are considered](https://docs.airbyte.com/contributing-to-airbyte/change-cdk-connector/#breaking-changes-to-connectors). If a **Breaking Change** is being introduced, ensure an Airbyte engineer has created a Breaking Change Plan." + - Connector version has been incremented in the Dockerfile and metadata.yaml according to our [Semantic Versioning for Connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors) guidelines + - You've updated the connector's `metadata.yaml` file any other relevant changes, including a `breakingChanges` entry for major version bumps. See [metadata.yaml docs](https://docs.airbyte.com/connector-development/connector-metadata-file/) - Secrets in the connector's spec are annotated with `airbyte_secret` - All documentation files are up to date. (README.md, bootstrap.md, docs.md, etc...) - Changelog updated in `docs/integrations//.md` with an entry for the new version. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog) - - You, or an Airbyter, have run `/test` successfully on this PR - or on a non-forked branch - - You've updated the connector's `metadata.yaml` file (new!) + - Migration guide updated in `docs/integrations//-migrations.md` with an entry for the new version, if the version is a breaking change. See migration guide [example](https://docs.airbyte.io/integrations/sources/faker-migrations) + - If set, you've ensured the icon is present in the `platform-internal` repo. ([Docs](https://docs.airbyte.com/connector-development/connector-metadata-file/#the-icon-field)) diff --git a/airbyte-ci/connectors/ci_credentials/README.md b/airbyte-ci/connectors/ci_credentials/README.md new file mode 100644 index 000000000000..238ae35351f6 --- /dev/null +++ b/airbyte-ci/connectors/ci_credentials/README.md @@ -0,0 +1,88 @@ +# CI Credentials +CLI tooling to read and manage GSM secrets: +- `write-to-storage` download a connector's secrets locally in the connector's `secret` folder +- `update-secrets` uploads new connector secret version that were locally updated. + +## Requirements + +This project requires Python 3.10 and pipx. + +## Installation + +The recommended way to install `ci_credentials` is using pipx. This ensures the tool and its dependencies are isolated from your other Python projects. + +If you haven't installed pipx, you can do it with pip: + +```bash +python -m pip install --user pipx +python -m pipx ensurepath +``` + +Once pipx is installed, navigate to the root directory of the project, then run: + +```bash +pipx install airbyte-ci/connectors/ci_credentials/ +``` + +This command installs ci_credentials and makes it globally available in your terminal. + +## Get GSM access +Download a Service account json key that has access to Google Secrets Manager. + +### Create Service Account +* Go to https://console.cloud.google.com/iam-admin/serviceaccounts/create?project=dataline-integration-testing +* In step #1 `Service account details`, set a name and a relevant description +* In step #2 `Grant this service account access to project`, select role `Owner` (there is a role that is more scope but I based this decision on others `-testing` service account) + +### Create Service Account Token +* Go to https://console.cloud.google.com/iam-admin/serviceaccounts?project=dataline-integration-testing +* Find your service account and click on it +* Go in the tab "KEYS" +* Click on "ADD KEY -> Create new key" and select JSON. This will download a file on your computer + +### Setup ci_credentials +* In your .zshrc, add: `export GCP_GSM_CREDENTIALS=$(cat )` + +## Development +During development, you can use the `--editable` option to make changes to the `ci_credentials` package and have them immediately take effect without needing to reinstall the package: + +```bash +pipx install --editable airbyte-ci/connectors/ci_credentials/ +``` + +This is useful when you are making changes to the package and want to test them in real-time. + +## Usage +After installation, you can use the ci_credentials command in your terminal. + +## Run it + +The `VERSION=dev` will make it so it knows to use your local current working directory and not the Github Action one. + +### Help +```bash +VERSION=dev ci_credentials --help +``` + +### Write credentials for a specific connector to local storage +To download GSM secrets to `airbyte-integrations/connectors/source-bings-ads/secrets`: +```bash +VERSION=dev ci_credentials source-bing-ads write-to-storage +``` + +### Write credentials for all connectors to local storage +To download GSM secrets to for all available connectors into their respective `secrets` directories: +```bash +VERSION=dev ci_credentials all write-to-storage +``` + +### Update secrets +To upload to GSM newly updated configurations from `airbyte-integrations/connectors/source-bings-ads/secrets/updated_configurations`: + +```bash +VERSION=dev ci_credentials source-bing-ads update-secrets +``` + +## FAQ +### What is `VERSION=dev`? +This is a way to tell the tool to write secrets using your local current working directory and not the Github Action runner one. diff --git a/tools/ci_credentials/ci_credentials/__init__.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/__init__.py similarity index 100% rename from tools/ci_credentials/ci_credentials/__init__.py rename to airbyte-ci/connectors/ci_credentials/ci_credentials/__init__.py diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py new file mode 100644 index 000000000000..9f2f47456848 --- /dev/null +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/main.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import sys +from json.decoder import JSONDecodeError + +import click +from common_utils import Logger + +from . import SecretsManager + +logger = Logger() + +ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS" + + +# credentials of GSM and GitHub secrets should be shared via shell environment +@click.group() +@click.argument("connector_name") +@click.option("--gcp-gsm-credentials", envvar="GCP_GSM_CREDENTIALS") +@click.pass_context +def ci_credentials(ctx, connector_name: str, gcp_gsm_credentials): + ctx.ensure_object(dict) + ctx.obj["connector_name"] = connector_name + # parse unique connector name, because it can have the common prefix "connectors/" + connector_name = connector_name.split("/")[-1] + if connector_name == "all": + # if needed to load all secrets + connector_name = None + + # parse GCP_GSM_CREDENTIALS + try: + gsm_credentials = json.loads(gcp_gsm_credentials) if gcp_gsm_credentials else {} + except JSONDecodeError as e: + return logger.error(f"incorrect GCP_GSM_CREDENTIALS value, error: {e}") + + if not gsm_credentials: + return logger.error("GCP_GSM_CREDENTIALS shouldn't be empty!") + + secret_manager = SecretsManager( + connector_name=connector_name, + gsm_credentials=gsm_credentials, + ) + ctx.obj["secret_manager"] = secret_manager + ctx.obj["connector_secrets"] = secret_manager.read_from_gsm() + + +@ci_credentials.command(help="Download GSM secrets locally to the connector's secrets directory.") +@click.pass_context +def write_to_storage(ctx): + written_files = ctx.obj["secret_manager"].write_to_storage(ctx.obj["connector_secrets"]) + written_files_count = len(written_files) + click.echo(f"{written_files_count} secret files were written: {','.join([str(path) for path in written_files])}") + + +@ci_credentials.command(help="Update GSM secrets according to the content of the secrets/updated_configurations directory.") +@click.pass_context +def update_secrets(ctx): + new_remote_secrets = ctx.obj["secret_manager"].update_secrets(ctx.obj["connector_secrets"]) + updated_secret_names = [secret.name for secret in new_remote_secrets] + updated_secrets_count = len(new_remote_secrets) + click.echo(f"Updated {updated_secrets_count} secrets: {','.join(updated_secret_names)}") + + +if __name__ == "__main__": + sys.exit(ci_credentials(obj={})) diff --git a/tools/ci_credentials/ci_credentials/models.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/models.py similarity index 100% rename from tools/ci_credentials/ci_credentials/models.py rename to airbyte-ci/connectors/ci_credentials/ci_credentials/models.py diff --git a/tools/ci_credentials/ci_credentials/secrets_manager.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py similarity index 89% rename from tools/ci_credentials/ci_credentials/secrets_manager.py rename to airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py index b6d345199657..2c02785957b4 100644 --- a/tools/ci_credentials/ci_credentials/secrets_manager.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import base64 import json import os @@ -10,7 +11,9 @@ from pathlib import Path from typing import Any, ClassVar, List, Mapping -from ci_common_utils import GoogleApi, Logger +import requests +import yaml +from common_utils import GoogleApi, Logger from .models import DEFAULT_SECRET_FILE, RemoteSecret, Secret @@ -18,7 +21,7 @@ GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) -MASK_KEY_PATTERNS = [ +DEFAULT_MASK_KEY_PATTERNS = [ "password", "host", "user", @@ -42,14 +45,17 @@ "survey_", "appid", "apikey", + "api_key", ] class SecretsManager: """Loading, saving and updating all requested secrets into connector folders""" + SPEC_MASK_URL = "https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml" + logger: ClassVar[Logger] = Logger() - if os.getenv("VERSION") == "dev": + if os.getenv("VERSION") in ["dev", "dagger_ci"]: base_folder = Path(os.getcwd()) else: base_folder = Path("/actions-runner/_work/airbyte/airbyte") @@ -65,6 +71,10 @@ def api(self) -> GoogleApi: self._api = GoogleApi(self.gsm_credentials, GSM_SCOPES) return self._api + @property + def mask_key_patterns(self) -> List[str]: + return self._get_spec_mask() + DEFAULT_MASK_KEY_PATTERNS + def __load_gsm_secrets(self) -> List[RemoteSecret]: """Loads needed GSM secrets""" secrets = [] @@ -145,15 +155,19 @@ def mask_secrets_from_action_log(self, key, value): else: if key: # regular value, check for what to mask - for pattern in MASK_KEY_PATTERNS: + for pattern in self.mask_key_patterns: if re.search(pattern, key): self.logger.info(f"Add mask for key: {key}") for line in str(value).splitlines(): line = str(line).strip() # don't output } and such - if len(line) > 1 and not os.getenv("VERSION") == "dev": - # has to be at the beginning of line for Github to notice it - print(f"::add-mask::{line}") + if len(line) > 1: + if not os.getenv("VERSION") in ["dev", "dagger_ci"]: + # has to be at the beginning of line for Github to notice it + print(f"::add-mask::{line}") + if os.getenv("VERSION") == "dagger_ci": + with open("/tmp/secrets_to_mask.txt", "a") as f: + f.write(f"{line}\n") break # see if it's really embedded json and get those values too try: @@ -271,3 +285,13 @@ def update_secrets(self, existing_secrets: List[RemoteSecret]) -> List[RemoteSec new_remote_secrets.append(new_remote_secret) self.logger.info(f"Updated {new_remote_secret.name} with new value") return new_remote_secrets + + def _get_spec_mask(self) -> List[str]: + response = requests.get(self.SPEC_MASK_URL, allow_redirects=True) + if not response.ok: + self.logger.error(f"Failed to fetch spec mask: {response.content}") + try: + return yaml.safe_load(response.content)["properties"] + except Exception as e: + self.logger.error(f"Failed to parse spec mask: {e}") + return [] diff --git a/airbyte-ci/connectors/ci_credentials/poetry.lock b/airbyte-ci/connectors/ci_credentials/poetry.lock new file mode 100644 index 000000000000..ebc60b07b062 --- /dev/null +++ b/airbyte-ci/connectors/ci_credentials/poetry.lock @@ -0,0 +1,503 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2023.5.7" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, +] + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] + +[[package]] +name = "common_utils" +version = "0.0.0" +description = "" +optional = false +python-versions = ">=3.9" +files = [] +develop = true + +[package.dependencies] +cryptography = "*" +pyjwt = ">=2.6.0,<2.7.0" +requests = "*" + +[package.extras] +tests = ["requests-mock"] + +[package.source] +type = "directory" +url = "../common_utils" + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "41.0.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:f73bff05db2a3e5974a6fd248af2566134d8981fd7ab012e5dd4ddb1d9a70699"}, + {file = "cryptography-41.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:1a5472d40c8f8e91ff7a3d8ac6dfa363d8e3138b961529c996f3e2df0c7a411a"}, + {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fa01527046ca5facdf973eef2535a27fec4cb651e4daec4d043ef63f6ecd4ca"}, + {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b46e37db3cc267b4dea1f56da7346c9727e1209aa98487179ee8ebed09d21e43"}, + {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d198820aba55660b4d74f7b5fd1f17db3aa5eb3e6893b0a41b75e84e4f9e0e4b"}, + {file = "cryptography-41.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:948224d76c4b6457349d47c0c98657557f429b4e93057cf5a2f71d603e2fc3a3"}, + {file = "cryptography-41.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:059e348f9a3c1950937e1b5d7ba1f8e968508ab181e75fc32b879452f08356db"}, + {file = "cryptography-41.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:b4ceb5324b998ce2003bc17d519080b4ec8d5b7b70794cbd2836101406a9be31"}, + {file = "cryptography-41.0.1-cp37-abi3-win32.whl", hash = "sha256:8f4ab7021127a9b4323537300a2acfb450124b2def3756f64dc3a3d2160ee4b5"}, + {file = "cryptography-41.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:1fee5aacc7367487b4e22484d3c7e547992ed726d14864ee33c0176ae43b0d7c"}, + {file = "cryptography-41.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9a6c7a3c87d595608a39980ebaa04d5a37f94024c9f24eb7d10262b92f739ddb"}, + {file = "cryptography-41.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5d092fdfedaec4cbbffbf98cddc915ba145313a6fdaab83c6e67f4e6c218e6f3"}, + {file = "cryptography-41.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1a8e6c2de6fbbcc5e14fd27fb24414507cb3333198ea9ab1258d916f00bc3039"}, + {file = "cryptography-41.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cb33ccf15e89f7ed89b235cff9d49e2e62c6c981a6061c9c8bb47ed7951190bc"}, + {file = "cryptography-41.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5f0ff6e18d13a3de56f609dd1fd11470918f770c6bd5d00d632076c727d35485"}, + {file = "cryptography-41.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7bfc55a5eae8b86a287747053140ba221afc65eb06207bedf6e019b8934b477c"}, + {file = "cryptography-41.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eb8163f5e549a22888c18b0d53d6bb62a20510060a22fd5a995ec8a05268df8a"}, + {file = "cryptography-41.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8dde71c4169ec5ccc1087bb7521d54251c016f126f922ab2dfe6649170a3b8c5"}, + {file = "cryptography-41.0.1.tar.gz", hash = "sha256:d34579085401d3f49762d2f7d6634d6b6c2ae1242202e860f4d26b046e3a1006"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "exceptiongroup" +version = "1.1.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pluggy" +version = "1.2.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pyjwt" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pytest" +version = "7.3.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, + {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.10.0" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, + {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "urllib3" +version = "1.26.16" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "5c858b1988aed2273d7268987a724d848bc9eef618b5592cf0ef78d6b2c91ff8" diff --git a/airbyte-ci/connectors/ci_credentials/pyproject.toml b/airbyte-ci/connectors/ci_credentials/pyproject.toml new file mode 100644 index 000000000000..5ca538a9828d --- /dev/null +++ b/airbyte-ci/connectors/ci_credentials/pyproject.toml @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +[tool.poetry] +name = "ci_credentials" +version = "1.1.0" +description = "CLI tooling to read and manage GSM secrets" +authors = ["Airbyte "] +readme = "README.md" +packages = [{ include = "ci_credentials" }] + +[tool.poetry.dependencies] +python = "^3.10" +requests = "^2.28.2" +click = "^8.1.3" +pyyaml = "^6.0" +common_utils = { path = "../common_utils", develop = true } + +[tool.poetry.group.test.dependencies] +requests-mock = "^1.10.0" +pytest = "^7.3.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +ci_credentials = "ci_credentials.main:ci_credentials" diff --git a/airbyte-ci/connectors/ci_credentials/tests/__init__.py b/airbyte-ci/connectors/ci_credentials/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/ci_credentials/tests/test_models.py b/airbyte-ci/connectors/ci_credentials/tests/test_models.py similarity index 99% rename from tools/ci_credentials/tests/test_models.py rename to airbyte-ci/connectors/ci_credentials/tests/test_models.py index 1f36b1dee239..dd3b749b53d2 100644 --- a/tools/ci_credentials/tests/test_models.py +++ b/airbyte-ci/connectors/ci_credentials/tests/test_models.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest from ci_credentials.models import Secret diff --git a/tools/ci_credentials/tests/test_secrets_manager.py b/airbyte-ci/connectors/ci_credentials/tests/test_secrets_manager.py similarity index 94% rename from tools/ci_credentials/tests/test_secrets_manager.py rename to airbyte-ci/connectors/ci_credentials/tests/test_secrets_manager.py index 06855cf10806..b88fa17d7957 100644 --- a/tools/ci_credentials/tests/test_secrets_manager.py +++ b/airbyte-ci/connectors/ci_credentials/tests/test_secrets_manager.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import base64 import json import re @@ -20,6 +21,7 @@ def matchers(): "addVersion": re.compile("https://secretmanager.googleapis.com/v1/.+:addVersion"), "access": re.compile("https://secretmanager.googleapis.com/v1/.+/1:access"), "disable": re.compile("https://secretmanager.googleapis.com/v1/.+:disable"), + "spec_secret_mask": re.compile("https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml"), } @@ -52,8 +54,8 @@ def matchers(): "gsm_only", ], ) -@patch("ci_common_utils.GoogleApi.get_access_token", lambda *args: ("fake_token", None)) -@patch("ci_common_utils.GoogleApi.project_id", "fake_id") +@patch("common_utils.GoogleApi.get_access_token", lambda *args: ("fake_token", None)) +@patch("common_utils.GoogleApi.project_id", "fake_id") def test_read(matchers, connector_name, gsm_secrets, expected_secrets): secrets_list = { "secrets": [ @@ -92,6 +94,7 @@ def test_read(matchers, connector_name, gsm_secrets, expected_secrets): m.post(matchers["secrets"], json={"name": ""}) m.get(matchers["versions"], versions_response_list) m.get(matchers["access"], secrets_response_list) + m.get(matchers["spec_secret_mask"], json={"spec_secret_mask": "test"}) secrets = manager.read_from_gsm() assert secrets == expected_secrets @@ -160,8 +163,8 @@ def test_validate_mask_values(connector_name, dict_json_value, expected_secret, assert expected_secret in capsys.readouterr().out -@patch("ci_common_utils.GoogleApi.get_access_token", lambda *args: ("fake_token", None)) -@patch("ci_common_utils.GoogleApi.project_id", "fake_id") +@patch("common_utils.GoogleApi.get_access_token", lambda *args: ("fake_token", None)) +@patch("common_utils.GoogleApi.project_id", "fake_id") @pytest.mark.parametrize( "old_secret_value, updated_configurations", [ diff --git a/tools/ci_common_utils/ci_common_utils/__init__.py b/airbyte-ci/connectors/common_utils/common_utils/__init__.py similarity index 100% rename from tools/ci_common_utils/ci_common_utils/__init__.py rename to airbyte-ci/connectors/common_utils/common_utils/__init__.py diff --git a/tools/ci_common_utils/ci_common_utils/google_api.py b/airbyte-ci/connectors/common_utils/common_utils/google_api.py similarity index 79% rename from tools/ci_common_utils/ci_common_utils/google_api.py rename to airbyte-ci/connectors/common_utils/common_utils/google_api.py index 2ce4ef2a0f7c..68ff38ae5a9f 100644 --- a/tools/ci_common_utils/ci_common_utils/google_api.py +++ b/airbyte-ci/connectors/common_utils/common_utils/google_api.py @@ -1,6 +1,10 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import time from dataclasses import dataclass -from typing import Mapping, Any, List, ClassVar +from typing import Any, ClassVar, List, Mapping import jwt import requests @@ -15,6 +19,7 @@ class GoogleApi: """ Simple Google API client """ + logger: ClassVar[Logger] = Logger() config: Mapping[str, Any] @@ -24,11 +29,7 @@ class GoogleApi: def get(self, url: str, params: Mapping = None) -> Mapping[str, Any]: """Sends a GET request""" token = self.get_access_token() - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "X-Goog-User-Project": self.project_id - } + headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json", "X-Goog-User-Project": self.project_id} # Making a get request response = requests.get(url, headers=headers, params=params) response.raise_for_status() @@ -38,10 +39,7 @@ def post(self, url: str, json: Mapping = None, params: Mapping = None) -> Mappin """Sends a POST request""" token = self.get_access_token() - headers = { - "Authorization": f"Bearer {token}", - "X-Goog-User-Project": self.project_id - } + headers = {"Authorization": f"Bearer {token}", "X-Goog-User-Project": self.project_id} # Making a get request response = requests.post(url, headers=headers, json=json, params=params) try: @@ -81,8 +79,11 @@ def get_access_token(self) -> str: def __get_access_token(self) -> str: jwt = self.__generate_jwt() - resp = requests.post(self.token_uri, data={ - "assertion": jwt, - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", - }) + resp = requests.post( + self.token_uri, + data={ + "assertion": jwt, + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + }, + ) return resp.json()["access_token"] diff --git a/tools/ci_common_utils/ci_common_utils/logger.py b/airbyte-ci/connectors/common_utils/common_utils/logger.py similarity index 76% rename from tools/ci_common_utils/ci_common_utils/logger.py rename to airbyte-ci/connectors/common_utils/common_utils/logger.py index 00ec0a7c7162..0e163ef1fff0 100644 --- a/tools/ci_common_utils/ci_common_utils/logger.py +++ b/airbyte-ci/connectors/common_utils/common_utils/logger.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import datetime as dt import inspect import logging @@ -7,13 +11,12 @@ class MyFormatter(logging.Formatter): - """Custom formatter for logging - """ + """Custom formatter for logging""" + converter = dt.datetime.fromtimestamp def formatTime(self, record, datefmt=None): - """! @brief redefinition of format of log - """ + """! @brief redefinition of format of log""" ct = self.converter(record.created) if datefmt: s = ct.strftime(datefmt) @@ -25,19 +28,17 @@ def formatTime(self, record, datefmt=None): class Logger: """Simple logger with a pretty log header - the method error returns the value 1 - the method critical terminates a script work + the method error returns the value 1 + the method critical terminates a script work """ def __init__(self): - formatter = MyFormatter( - fmt='[%(asctime)s] - %(levelname)-6s - %(message)s', - datefmt='%d/%m/%Y %H:%M:%S.%f') + formatter = MyFormatter(fmt="[%(asctime)s] - %(levelname)-6s - %(message)s", datefmt="%d/%m/%Y %H:%M:%S.%f") logger_name = __name__ stack_items = inspect.stack() for i in range(len(stack_items)): - if stack_items[i].filename.endswith("ci_common_utils/logger.py"): + if stack_items[i].filename.endswith("common_utils/logger.py"): logger_name = ".".join(stack_items[i + 1].filename.split("/")[-3:])[:-3] self._logger = logging.getLogger(logger_name) @@ -55,7 +56,7 @@ def wrapper(*args): prefix = "" stack_items = inspect.stack() for i in range(len(stack_items)): - if stack_items[i].filename.endswith("ci_common_utils/logger.py"): + if stack_items[i].filename.endswith("common_utils/logger.py"): filepath = stack_items[i + 1].filename line_number = stack_items[i + 1].lineno @@ -78,4 +79,7 @@ def wrapper(*args): def __getattr__(self, function_name: str): if not hasattr(self._logger, function_name): return super().__getattr__(function_name) - return self.__prepare_log_line(function_name, getattr(self._logger, function_name), ) + return self.__prepare_log_line( + function_name, + getattr(self._logger, function_name), + ) diff --git a/airbyte-ci/connectors/common_utils/poetry.lock b/airbyte-ci/connectors/common_utils/poetry.lock new file mode 100644 index 000000000000..750a922b5cca --- /dev/null +++ b/airbyte-ci/connectors/common_utils/poetry.lock @@ -0,0 +1,339 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "certifi" +version = "2023.5.7" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, +] + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "chardet" +version = "4.0.0" +description = "Universal encoding detector for Python 2 and 3" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, + {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "3.4.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-3.4.7-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1"}, + {file = "cryptography-3.4.7-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250"}, + {file = "cryptography-3.4.7-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2"}, + {file = "cryptography-3.4.7-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6"}, + {file = "cryptography-3.4.7-cp36-abi3-manylinux2014_x86_64.whl", hash = "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959"}, + {file = "cryptography-3.4.7-cp36-abi3-win32.whl", hash = "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d"}, + {file = "cryptography-3.4.7-cp36-abi3-win_amd64.whl", hash = "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca"}, + {file = "cryptography-3.4.7-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873"}, + {file = "cryptography-3.4.7-pp36-pypy36_pp73-manylinux2014_x86_64.whl", hash = "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b01fd6f2737816cb1e08ed4807ae194404790eac7ad030b34f2ce72b332f5586"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-manylinux2014_x86_64.whl", hash = "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:bf40af59ca2465b24e54f671b2de2c59257ddc4f7e5706dbd6930e26823668d3"}, + {file = "cryptography-3.4.7.tar.gz", hash = "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "exceptiongroup" +version = "1.1.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "idna" +version = "2.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, + {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pluggy" +version = "1.2.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pyjwt" +version = "2.1.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyJWT-2.1.0-py3-none-any.whl", hash = "sha256:934d73fbba91b0483d3857d1aff50e96b2a892384ee2c17417ed3203f173fca1"}, + {file = "PyJWT-2.1.0.tar.gz", hash = "sha256:fba44e7898bbca160a2b2b501f492824fc8382485d3a6f11ba5d0c1937ce6130"}, +] + +[package.extras] +crypto = ["cryptography (>=3.3.1,<4.0.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.3.1,<4.0.0)", "mypy", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pytest" +version = "7.4.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "requests" +version = "2.25.1" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, + {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +chardet = ">=3.0.2,<5" +idna = ">=2.5,<3" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +security = ["cryptography (>=1.3.4)", "pyOpenSSL (>=0.14)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] + +[[package]] +name = "requests-mock" +version = "1.9.3" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "urllib3" +version = "1.26.16" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "13161cded140a3c9476808685981630c07a214edc26a537a9b30031f669903d4" diff --git a/airbyte-ci/connectors/common_utils/pyproject.toml b/airbyte-ci/connectors/common_utils/pyproject.toml new file mode 100644 index 000000000000..97e23e9ffac5 --- /dev/null +++ b/airbyte-ci/connectors/common_utils/pyproject.toml @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +[tool.poetry] +name = "common_utils" +version = "0.0.0" +description = "Suite of all often used classes and common functions" +authors = ["Airbyte "] + +[tool.poetry.dependencies] +python = "^3.10" +cryptography = "^3.4.7" +requests = "^2.28.2" +pyjwt = "^2.1.0" + +[tool.poetry.group.test.dependencies] +pytest = "^7.2.2" + +[tool.poetry.group.dev.dependencies] +requests-mock = "^1.9.3" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/airbyte-ci/connectors/common_utils/tests/__init__.py b/airbyte-ci/connectors/common_utils/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/common_utils/tests/test_logger.py b/airbyte-ci/connectors/common_utils/tests/test_logger.py new file mode 100644 index 000000000000..1d12ad29325c --- /dev/null +++ b/airbyte-ci/connectors/common_utils/tests/test_logger.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +from datetime import datetime, timedelta + +import pytest +from common_utils import Logger + +LOG_RE = re.compile(r"^\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\.\d{6})\] -" r"\s+(\w+)\s+- \[.*tests/test_logger.py:(\d+)\] # (.+)") +LOGGER = Logger() +TEST_MESSAGE = "sbhY=)9'v-}LT=)jjF66(XrZh=]>7Xp\"?/zCz,=eu8K47u8" + + +def check_output(msg: str, expected_line_number: int, expected_log_level: str): + m = LOG_RE.match(msg) + assert m is not None, f"incorrect message format, pattern: {LOG_RE.pattern}" + date_time, log_level, line_number, msg = m.groups() + + assert int(line_number) == expected_line_number + assert expected_log_level == log_level + assert expected_log_level == log_level + dt = datetime.strptime(date_time, "%d/%m/%Y %H:%M:%S.%f") + now = datetime.now() + delta = timedelta(seconds=1) + assert now - delta < dt < now + + +@pytest.mark.parametrize( + "log_func,expected_log_level,expected_code", + ((LOGGER.debug, "DEBUG", 0), (LOGGER.warning, "WARNING", 0), (LOGGER.info, "INFO", 0), (LOGGER.error, "ERROR", 1)), +) +def test_log_message(capfd, log_func, expected_log_level, expected_code): + assert log_func(TEST_MESSAGE) == expected_code + _, err = capfd.readouterr() + check_output(err, 36, expected_log_level) + + +def test_critical_message(capfd): + with pytest.raises(SystemExit) as (err): + LOGGER.critical(TEST_MESSAGE) + _, err = capfd.readouterr() + check_output(err, 43, "CRITICAL") diff --git a/tools/ci_connector_ops/.python-version b/airbyte-ci/connectors/connector_ops/.python-version similarity index 100% rename from tools/ci_connector_ops/.python-version rename to airbyte-ci/connectors/connector_ops/.python-version diff --git a/airbyte-ci/connectors/connector_ops/README.md b/airbyte-ci/connectors/connector_ops/README.md new file mode 100644 index 000000000000..ef306478b7cb --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/README.md @@ -0,0 +1,35 @@ +# connector_ops + +A collection of tools and checks run by Github Actions + +## Running Locally + +From this directory, create a virtual environment: + +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: + +```bash +source .venv/bin/activate +pip install -e . # assuming you are in the ./airbyte-ci/connectors/connector_ops directory +``` + +pip will make binaries for all the commands in setup.py, so you can run `allowed-hosts-checks` directly from the virtual-env + +## Testing Locally + +To install requirements to run unit tests, use: + +``` +pip install -e ".[tests]" +``` + +Unit tests are currently configured to be run from the base `airbyte` directory. You can run the tests from that directory with the following command: + +``` +pytest -s airbyte-ci/connector_ops/connectors/tests +``` \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-appfollow/unit_tests/__init__.py b/airbyte-ci/connectors/connector_ops/connector_ops/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-appfollow/unit_tests/__init__.py rename to airbyte-ci/connectors/connector_ops/connector_ops/__init__.py diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py new file mode 100644 index 000000000000..7d2b064468df --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +import sys +from typing import Dict, List, Set, Union + +import yaml +from connector_ops import utils + +BACKWARD_COMPATIBILITY_REVIEWERS = {"connector-operations", "connector-extensibility"} +TEST_STRICTNESS_LEVEL_REVIEWERS = {"connector-operations"} +GA_BYPASS_REASON_REVIEWERS = {"connector-operations"} +GA_CONNECTOR_REVIEWERS = {"gl-python"} +REVIEW_REQUIREMENTS_FILE_PATH = ".github/connector_org_review_requirements.yaml" + + +def find_connectors_with_bad_strictness_level() -> List[utils.Connector]: + """Check if changed connectors have the expected connector acceptance test strictness level according to their release stage. + 1. Identify changed connectors + 2. Retrieve their release stage from the catalog + 3. Parse their acceptance test config file + 4. Check if the test strictness level matches the strictness level expected for their release stage. + + Returns: + List[utils.Connector]: List of changed connector that are not matching test strictness level expectations. + """ + connectors_with_bad_strictness_level = [] + changed_connector = utils.get_changed_connectors(destination=False, third_party=False) + for connector in changed_connector: + check_for_high_strictness = connector.acceptance_test_config is not None and connector.requires_high_test_strictness_level + if check_for_high_strictness: + try: + assert connector.acceptance_test_config.get("test_strictness_level") == "high" + except AssertionError: + connectors_with_bad_strictness_level.append(connector) + return connectors_with_bad_strictness_level + + +def find_changed_important_connectors() -> Set[utils.Connector]: + """Find important connectors modified on the current branch. + + Returns: + Set[utils.Connector]: The set of GA connectors that were modified on the current branch. + """ + changed_connectors = utils.get_changed_connectors(destination=False, third_party=False) + return {connector for connector in changed_connectors if connector.is_important_connector} + + +def get_bypass_reason_changes() -> Set[utils.Connector]: + """Find connectors that have modified bypass_reasons. + + Returns: + Set[str]: Set of connector names e.g {"source-github"}: The set of GA connectors that have changed bypass_reasons. + """ + bypass_reason_changes = utils.get_changed_acceptance_test_config(diff_regex="bypass_reason") + return bypass_reason_changes.intersection(find_changed_important_connectors()) + + +def find_mandatory_reviewers() -> List[Union[str, Dict[str, List]]]: + important_connector_changes = find_changed_important_connectors() + backward_compatibility_changes = utils.get_changed_acceptance_test_config(diff_regex="disable_for_version") + test_strictness_level_changes = utils.get_changed_acceptance_test_config(diff_regex="test_strictness_level") + ga_bypass_reason_changes = get_bypass_reason_changes() + + if backward_compatibility_changes: + return [{"any-of": list(BACKWARD_COMPATIBILITY_REVIEWERS)}] + if test_strictness_level_changes: + return [{"any-of": list(TEST_STRICTNESS_LEVEL_REVIEWERS)}] + if ga_bypass_reason_changes: + return [{"any-of": list(GA_BYPASS_REASON_REVIEWERS)}] + if important_connector_changes: + return list(GA_CONNECTOR_REVIEWERS) + return [] + + +def check_test_strictness_level(): + connectors_with_bad_strictness_level = find_connectors_with_bad_strictness_level() + if connectors_with_bad_strictness_level: + logging.error( + f"The following connectors must enable high test strictness level: {connectors_with_bad_strictness_level}. Please check this documentation for details: https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/#strictness-level" + ) + sys.exit(1) + else: + sys.exit(0) + + +def write_review_requirements_file(): + mandatory_reviewers = find_mandatory_reviewers() + + if mandatory_reviewers: + requirements_file_content = [ + {"name": "Required reviewers from the connector org teams", "paths": "unmatched", "teams": mandatory_reviewers} + ] + with open(REVIEW_REQUIREMENTS_FILE_PATH, "w") as requirements_file: + yaml.safe_dump(requirements_file_content, requirements_file) + print("CREATED_REQUIREMENTS_FILE=true") + else: + print("CREATED_REQUIREMENTS_FILE=false") + + +def print_mandatory_reviewers(): + teams = [] + mandatory_reviewers = find_mandatory_reviewers() + for mandatory_reviewer in mandatory_reviewers: + if isinstance(mandatory_reviewer, dict): + teams += mandatory_reviewer["any-of"] + else: + teams.append(mandatory_reviewer) + print(f"MANDATORY_REVIEWERS=A review is required from these teams: {','.join(teams)}") diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/allowed_hosts_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/allowed_hosts_checks.py new file mode 100644 index 000000000000..9a74371716ea --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/connector_ops/allowed_hosts_checks.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +import sys +from typing import List + +from connector_ops import utils + + +def get_connectors_missing_allowed_hosts() -> List[utils.Connector]: + connectors_missing_allowed_hosts: List[utils.Connector] = [] + changed_connectors = utils.get_changed_connectors(destination=False, third_party=False) + + for connector in changed_connectors: + if connector.requires_allowed_hosts_check: + missing = not connector_has_allowed_hosts(connector) + if missing: + connectors_missing_allowed_hosts.append(connector) + + return connectors_missing_allowed_hosts + + +def connector_has_allowed_hosts(connector: utils.Connector) -> bool: + return connector.allowed_hosts is not None + + +def check_allowed_hosts(): + connectors_missing_allowed_hosts = get_connectors_missing_allowed_hosts() + if connectors_missing_allowed_hosts: + logging.error(f"The following connectors must include allowedHosts: {connectors_missing_allowed_hosts}") + sys.exit(1) + else: + sys.exit(0) diff --git a/tools/ci_connector_ops/ci_connector_ops/qa_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py similarity index 75% rename from tools/ci_connector_ops/ci_connector_ops/qa_checks.py rename to airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py index c2b789f06fea..462a05c1c4d0 100644 --- a/tools/ci_connector_ops/ci_connector_ops/qa_checks.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py @@ -7,7 +7,59 @@ from pathlib import Path from typing import Iterable, Optional, Set, Tuple -from ci_connector_ops.utils import Connector +from connector_ops.utils import Connector +from pydash.objects import get + + +def check_migration_guide(connector: Connector) -> bool: + """Check if a migration guide is available for the connector if a breaking change was introduced.""" + + breaking_changes = get(connector.metadata, f"releases.breakingChanges") + if not breaking_changes: + return True + + migration_guide_file_path = connector.migration_guide_file_path + if not migration_guide_file_path.exists(): + print( + f"Migration guide file is missing for {connector.name}. Please create a {connector.migration_guide_file_name} file in the docs folder." + ) + return False + + # Check that the migration guide begins with # {connector name} Migration Guide + expected_title = f"# {connector.name_from_metadata} Migration Guide" + expected_version_header_start = f"## Upgrading to " + with open(migration_guide_file_path) as f: + first_line = f.readline().strip() + if not first_line == expected_title: + print( + f"Migration guide file for {connector.technical_name} does not start with the correct header. Expected '{expected_title}', got '{first_line}'" + ) + return False + + # Check that the migration guide contains a section for each breaking change key ## Upgrading to {version} + # Note that breaking change is a dict where the version is the key + # Note that the migration guide must have the sections in order of the version descending + # 3.0.0, 2.0.0, 1.0.0, etc + # This means we have to record the headings in the migration guide and then check that they are in order + # We also have to check that the headings are in the breaking changes dict + + ordered_breaking_changes = sorted(breaking_changes.keys(), reverse=True) + ordered_expected_headings = [f"{expected_version_header_start}{version}" for version in ordered_breaking_changes] + + ordered_heading_versions = [] + for line in f: + stripped_line = line.strip() + if stripped_line.startswith(expected_version_header_start): + version = stripped_line.replace(expected_version_header_start, "") + ordered_heading_versions.append(version) + + if ordered_breaking_changes != ordered_heading_versions: + print(f"Migration guide file for {connector.name} has incorrect version headings.") + print(f"Check for missing, extra, or misordered headings, or headers with typos.") + print(f"Expected headings: {ordered_expected_headings}") + return False + + return True def check_documentation_file_exists(connector: Connector) -> bool: @@ -116,7 +168,7 @@ def read_all_files_in_directory( ".hypothesis", } -IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = {"*Test.java", "*.pyc", "*.gz", "*.svg"} +IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = {"*Test.java", "*.jar", "*.pyc", "*.gz", "*.svg"} IGNORED_URLS_PREFIX = { "http://json-schema.org", "http://localhost", @@ -188,6 +240,7 @@ def check_metadata_version_matches_dockerfile_label(connector: Connector) -> boo QA_CHECKS = [ check_documentation_file_exists, + check_migration_guide, # Disabling the following check because it's likely to not pass on a lot of connectors. # check_documentation_follows_guidelines, check_changelog_entry_is_updated, diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py new file mode 100644 index 000000000000..ef3b83e9dd80 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -0,0 +1,463 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import functools +import logging +import os +import re +from dataclasses import dataclass +from enum import Enum +from glob import glob +from pathlib import Path +from typing import List, Optional, Set, Tuple, Union + +import git +import requests +import yaml +from ci_credentials import SecretsManager +from pydash.objects import get +from rich.console import Console + +console = Console() + +DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") +OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" +CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" +SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" +DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" +THIRD_PARTY_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/third_party/" +SCAFFOLD_CONNECTOR_GLOB = "-scaffold-" + + +ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml" +AIRBYTE_DOCKER_REPO = "airbyte" +AIRBYTE_REPO_DIRECTORY_NAME = "airbyte" +GRADLE_PROJECT_RE_PATTERN = r"project\((['\"])(.+?)\1\)" +TEST_GRADLE_DEPENDENCIES = ["testImplementation", "integrationTestJavaImplementation", "performanceTestJavaImplementation"] + + +def download_catalog(catalog_url): + response = requests.get(catalog_url) + return response.json() + + +OSS_CATALOG = download_catalog(OSS_CATALOG_URL) +METADATA_FILE_NAME = "metadata.yaml" +ICON_FILE_NAME = "icon.svg" + +IMPORTANT_CONNECTOR_THRESHOLDS = { + "sl": 300, + "ql": 400, +} + +ALLOWED_HOST_THRESHOLD = { + "ql": 300, +} + + +class ConnectorInvalidNameError(Exception): + pass + + +class ConnectorVersionNotFound(Exception): + pass + + +def get_connector_name_from_path(path): + return path.split("/")[2] + + +def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[str]: + """Retrieve the set of connectors for which the acceptance_test_config file was changed in the current branch (compared to master). + + Args: + diff_regex (str): Find the edited files that contain the following regex in their change. + + Returns: + Set[Connector]: Set of connectors that were changed + """ + airbyte_repo = git.Repo(search_parent_directories=True) + + if diff_regex is None: + diff_command_args = ("--name-only", DIFFED_BRANCH) + else: + diff_command_args = ("--name-only", f"-G{diff_regex}", DIFFED_BRANCH) + + changed_acceptance_test_config_paths = { + file_path + for file_path in airbyte_repo.git.diff(*diff_command_args).split("\n") + if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX) and file_path.endswith(ACCEPTANCE_TEST_CONFIG_FILE_NAME) + } + return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths} + + +def get_gradle_dependencies_block(build_file: Path) -> str: + """Get the dependencies block of a Gradle file. + + Args: + build_file (Path): Path to the build.gradle file of the project. + + Returns: + str: The dependencies block of the Gradle file. + """ + contents = build_file.read_text().split("\n") + dependency_block = [] + in_dependencies_block = False + for line in contents: + if line.strip().startswith("dependencies"): + in_dependencies_block = True + continue + if in_dependencies_block: + if line.startswith("}"): + in_dependencies_block = False + break + else: + dependency_block.append(line) + dependencies_block = "\n".join(dependency_block) + return dependencies_block + + +def parse_gradle_dependencies(build_file: Path) -> Tuple[List[Path], List[Path]]: + """Parse the dependencies block of a Gradle file and return the list of project dependencies and test dependencies. + + Args: + build_file (Path): _description_ + + Returns: + Tuple[List[Tuple[str, Path]], List[Tuple[str, Path]]]: _description_ + """ + + dependencies_block = get_gradle_dependencies_block(build_file) + + project_dependencies: List[Path] = [] + test_dependencies: List[Path] = [] + + # Find all matches for test dependencies and regular dependencies + matches = re.findall( + r"(testImplementation|integrationTestJavaImplementation|performanceTestJavaImplementation|implementation|api).*?project\(['\"](.*?)['\"]\)", + dependencies_block, + ) + if matches: + # Iterate through each match + for match in matches: + dependency_type, project_path = match + path_parts = project_path.split(":") + path = Path(*path_parts) + + if dependency_type in TEST_GRADLE_DEPENDENCIES: + test_dependencies.append(path) + else: + project_dependencies.append(path) + + project_dependencies.append(Path("airbyte-cdk", "java", "airbyte-cdk")) + return project_dependencies, test_dependencies + + +def get_all_gradle_dependencies( + build_file: Path, with_test_dependencies: bool = True, found_dependencies: Optional[List[Path]] = None +) -> List[Path]: + """Recursively retrieve all transitive dependencies of a Gradle project. + + Args: + build_file (Path): Path to the build.gradle file of the project. + found_dependencies (List[Path]): List of dependencies that have already been found. Defaults to None. + + Returns: + List[Path]: All dependencies of the project. + """ + if found_dependencies is None: + found_dependencies = [] + project_dependencies, test_dependencies = parse_gradle_dependencies(build_file) + all_dependencies = project_dependencies + test_dependencies if with_test_dependencies else project_dependencies + for dependency_path in all_dependencies: + if dependency_path not in found_dependencies and Path(dependency_path / "build.gradle").exists(): + found_dependencies.append(dependency_path) + get_all_gradle_dependencies(dependency_path / "build.gradle", with_test_dependencies, found_dependencies) + + return found_dependencies + + +class ConnectorLanguage(str, Enum): + PYTHON = "python" + JAVA = "java" + LOW_CODE = "low-code" + + +class ConnectorLanguageError(Exception): + pass + + +@dataclass(frozen=True) +class Connector: + """Utility class to gather metadata about a connector.""" + + technical_name: str + + def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: + if "-" not in self.technical_name: + raise ConnectorInvalidNameError(f"Connector type and name could not be inferred from {self.technical_name}") + _type = self.technical_name.split("-")[0] + name = self.technical_name[len(_type) + 1 :] + return _type, name + + @property + def name(self): + return self._get_type_and_name_from_technical_name()[1] + + @property + def connector_type(self) -> str: + return self._get_type_and_name_from_technical_name()[0] + + @property + def documentation_directory(self) -> Path: + return Path(f"./docs/integrations/{self.connector_type}s") + + @property + def documentation_file_path(self) -> Path: + readme_file_name = f"{self.name}.md" + return self.documentation_directory / readme_file_name + + @property + def migration_guide_file_name(self) -> str: + return f"{self.name}-migrations.md" + + @property + def migration_guide_file_path(self) -> Path: + return self.documentation_directory / self.migration_guide_file_name + + @property + def icon_path(self) -> Path: + file_path = self.code_directory / ICON_FILE_NAME + return file_path + + @property + def code_directory(self) -> Path: + return Path(f"./airbyte-integrations/connectors/{self.technical_name}") + + @property + def metadata_file_path(self) -> Path: + return self.code_directory / METADATA_FILE_NAME + + @property + def metadata(self) -> Optional[dict]: + file_path = self.metadata_file_path + if not file_path.is_file(): + return None + return yaml.safe_load((self.code_directory / METADATA_FILE_NAME).read_text())["data"] + + @property + def language(self) -> ConnectorLanguage: + if Path(self.code_directory / self.technical_name.replace("-", "_") / "manifest.yaml").is_file(): + return ConnectorLanguage.LOW_CODE + if Path(self.code_directory / "setup.py").is_file(): + return ConnectorLanguage.PYTHON + try: + with open(self.code_directory / "Dockerfile") as dockerfile: + if "FROM airbyte/integration-base-java" in dockerfile.read(): + return ConnectorLanguage.JAVA + except FileNotFoundError: + pass + return None + # raise ConnectorLanguageError(f"We could not infer {self.technical_name} connector language") + + @property + def version(self) -> str: + if self.metadata is None: + return self.version_in_dockerfile_label + return self.metadata["dockerImageTag"] + + @property + def version_in_dockerfile_label(self) -> str: + with open(self.code_directory / "Dockerfile") as f: + for line in f: + if "io.airbyte.version" in line: + return line.split("=")[1].strip() + raise ConnectorVersionNotFound( + """ + Could not find the connector version from its Dockerfile. + The io.airbyte.version tag is missing. + """ + ) + + @property + def name_from_metadata(self) -> Optional[str]: + return self.metadata.get("name") if self.metadata else None + + @property + def support_level(self) -> Optional[str]: + return self.metadata.get("supportLevel") if self.metadata else None + + @property + def ab_internal_sl(self) -> int: + """Airbyte Internal Field. + + More info can be found here: https://www.notion.so/Internal-Metadata-Fields-32b02037e7b244b7934214019d0b7cc9 + + Returns: + int: The value + """ + default_value = 100 + sl_value = get(self.metadata, "ab_internal.sl") + + if sl_value is None: + logging.warning( + f"Connector {self.technical_name} does not have a `ab_internal.sl` defined in metadata.yaml. Defaulting to {default_value}" + ) + return default_value + + return sl_value + + @property + def ab_internal_ql(self) -> int: + """Airbyte Internal Field. + + More info can be found here: https://www.notion.so/Internal-Metadata-Fields-32b02037e7b244b7934214019d0b7cc9 + + Returns: + int: The value + """ + default_value = 100 + ql_value = get(self.metadata, "ab_internal.ql") + + if ql_value is None: + logging.warning( + f"Connector {self.technical_name} does not have a `ab_internal.ql` defined in metadata.yaml. Defaulting to {default_value}" + ) + return default_value + + return ql_value + + @property + def is_important_connector(self) -> bool: + """Check if a connector qualifies as an important connector. + + Returns: + bool: True if the connector is a high value connector, False otherwise. + """ + if self.ab_internal_sl >= IMPORTANT_CONNECTOR_THRESHOLDS["sl"]: + return True + + if self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"]: + return True + + return False + + @property + def requires_high_test_strictness_level(self) -> bool: + """Check if a connector requires high strictness CAT tests. + + Returns: + bool: True if the connector requires high test strictness level, False otherwise. + """ + return self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"] + + @property + def requires_allowed_hosts_check(self) -> bool: + """Check if a connector requires allowed hosts. + + Returns: + bool: True if the connector requires allowed hosts, False otherwise. + """ + return self.ab_internal_ql >= ALLOWED_HOST_THRESHOLD["ql"] + + @property + def allowed_hosts(self) -> Optional[List[str]]: + return self.metadata.get("allowedHosts") if self.metadata else None + + @property + def suggested_streams(self) -> Optional[List[str]]: + return self.metadata.get("suggestedStreams") if self.metadata else None + + @property + def acceptance_test_config_path(self) -> Path: + return self.code_directory / ACCEPTANCE_TEST_CONFIG_FILE_NAME + + @property + def acceptance_test_config(self) -> Optional[dict]: + try: + with open(self.acceptance_test_config_path) as acceptance_test_config_file: + return yaml.safe_load(acceptance_test_config_file) + except FileNotFoundError: + logging.warning(f"No {ACCEPTANCE_TEST_CONFIG_FILE_NAME} file found for {self.technical_name}") + return None + + @property + def supports_normalization(self) -> bool: + return self.metadata and self.metadata.get("normalizationConfig") is not None + + @property + def normalization_repository(self) -> Optional[str]: + if self.supports_normalization: + return f"{self.metadata['normalizationConfig']['normalizationRepository']}" + + @property + def normalization_tag(self) -> Optional[str]: + if self.supports_normalization: + return f"{self.metadata['normalizationConfig']['normalizationTag']}" + + def get_secret_manager(self, gsm_credentials: str): + return SecretsManager(connector_name=self.technical_name, gsm_credentials=gsm_credentials) + + def __repr__(self) -> str: + return self.technical_name + + @functools.lru_cache(maxsize=2) + def get_local_dependency_paths(self, with_test_dependencies: bool = True) -> Set[Path]: + dependencies_paths = [] + if self.language == ConnectorLanguage.JAVA: + dependencies_paths += get_all_gradle_dependencies( + self.code_directory / "build.gradle", with_test_dependencies=with_test_dependencies + ) + return sorted(list(set(dependencies_paths))) + + +def get_changed_connectors( + modified_files: Optional[Set[Union[str, Path]]] = None, source: bool = True, destination: bool = True, third_party: bool = True +) -> Set[Connector]: + """Retrieve a set of Connectors that were changed in the current branch (compared to master).""" + if modified_files is None: + airbyte_repo = git.Repo(search_parent_directories=True) + modified_files = airbyte_repo.git.diff("--name-only", DIFFED_BRANCH).split("\n") + + prefix_to_check = [] + if source: + prefix_to_check.append(SOURCE_CONNECTOR_PATH_PREFIX) + if destination: + prefix_to_check.append(DESTINATION_CONNECTOR_PATH_PREFIX) + if third_party: + prefix_to_check.append(THIRD_PARTY_CONNECTOR_PATH_PREFIX) + + changed_source_connector_files = { + file_path + for file_path in modified_files + if any(file_path.startswith(prefix) for prefix in prefix_to_check) and SCAFFOLD_CONNECTOR_GLOB not in file_path + } + return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} + + +def get_all_connectors_in_repo() -> Set[Connector]: + """Retrieve a set of all Connectors in the repo. + We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. + + Returns: + A set of Connectors. + """ + repo = git.Repo(search_parent_directories=True) + repo_path = repo.working_tree_dir + + return { + Connector(Path(metadata_file).parent.name) + for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) + if SCAFFOLD_CONNECTOR_GLOB not in metadata_file + } + + +class ConnectorTypeEnum(str, Enum): + source = "source" + destination = "destination" + + +class SupportLevelEnum(str, Enum): + certified = "certified" + community = "community" diff --git a/airbyte-ci/connectors/connector_ops/poetry.lock b/airbyte-ci/connectors/connector_ops/poetry.lock new file mode 100644 index 000000000000..1a1e547ab392 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/poetry.lock @@ -0,0 +1,1251 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "cachetools" +version = "5.3.1" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, +] + +[[package]] +name = "certifi" +version = "2023.7.22" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.2.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] + +[[package]] +name = "ci-credentials" +version = "1.1.0" +description = "CLI tooling to read and manage GSM secrets" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +click = "^8.1.3" +common_utils = {path = "../common_utils", develop = true} +pyyaml = "^6.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../ci_credentials" + +[[package]] +name = "click" +version = "8.1.6" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "common-utils" +version = "0.0.0" +description = "Suite of all often used classes and common functions" +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +cryptography = "^3.4.7" +pyjwt = "^2.1.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../common_utils" + +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +optional = false +python-versions = "*" +files = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "cryptography" +version = "3.4.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89"}, + {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, + {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, + {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "exceptiongroup" +version = "1.1.3" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.6" +files = [ + {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, + {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "gitdb" +version = "4.0.10" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, + {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.32" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.32-py3-none-any.whl", hash = "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"}, + {file = "GitPython-3.1.32.tar.gz", hash = "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[[package]] +name = "google-api-core" +version = "2.11.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, + {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.22.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, + {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" +six = ">=1.9.0" +urllib3 = "<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-core" +version = "2.3.3" +description = "Google Cloud API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, + {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, +] + +[package.dependencies] +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0dev)"] + +[[package]] +name = "google-cloud-storage" +version = "2.10.0" +description = "Google Cloud Storage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, + {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-resumable-media = ">=2.3.2" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +protobuf = ["protobuf (<5.0.0dev)"] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"}, +] + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.5.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "google-resumable-media-2.5.0.tar.gz", hash = "sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93"}, + {file = "google_resumable_media-2.5.0-py2.py3-none-any.whl", hash = "sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec"}, +] + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.60.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, + {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "numpy" +version = "1.25.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, + {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, + {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, + {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, + {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, + {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, + {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, + {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, + {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, + {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, + {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, + {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pandas" +version = "2.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, + {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, + {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, + {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, + {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, + {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, + {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, + {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, + {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, + {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, + {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] + +[[package]] +name = "pluggy" +version = "1.2.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.24.0" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, + {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, + {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, + {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, + {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, + {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, + {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, + {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, + {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, + {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, + {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.12" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, + {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, + {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, + {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, + {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, + {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, + {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, + {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, + {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, + {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, + {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, + {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, + {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, + {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, + {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, + {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, + {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, + {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, + {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, + {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, + {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, + {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, + {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pydash" +version = "7.0.6" +description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydash-7.0.6-py3-none-any.whl", hash = "sha256:10e506935953fde4b0d6fe21a88e17783cd1479256ae96f285b5f89063b4efd6"}, + {file = "pydash-7.0.6.tar.gz", hash = "sha256:7d9df7e9f36f2bbb08316b609480e7c6468185473a21bdd8e65dda7915565a26"}, +] + +[package.dependencies] +typing-extensions = ">=3.10,<4.6.0 || >4.6.0" + +[package.extras] +dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] + +[[package]] +name = "pygithub" +version = "1.59.1" +description = "Use the full Github API v3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyGithub-1.59.1-py3-none-any.whl", hash = "sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9"}, + {file = "PyGithub-1.59.1.tar.gz", hash = "sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217"}, +] + +[package.dependencies] +deprecated = "*" +pyjwt = {version = ">=2.4.0", extras = ["crypto"]} +pynacl = ">=1.4.0" +requests = ">=2.14.0" + +[[package]] +name = "pygments" +version = "2.16.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pynacl" +version = "1.5.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, + {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, +] + +[package.dependencies] +cffi = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] + +[[package]] +name = "pytest" +version = "7.4.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.11.1" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, + {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2023.3" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, + {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rich" +version = "11.2.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.6.2,<4.0.0" +files = [ + {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, + {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, +] + +[package.dependencies] +colorama = ">=0.4.0,<0.5.0" +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.6" +files = [ + {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, + {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] + +[[package]] +name = "tzdata" +version = "2023.3" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, + {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, +] + +[[package]] +name = "urllib3" +version = "1.26.16" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wrapt" +version = "1.15.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ + {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, + {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, + {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, + {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, + {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, + {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, + {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, + {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, + {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, + {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, + {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, + {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, + {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, + {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, + {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, + {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, + {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, + {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, + {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "e601115553c94d23c6d25303190d48216b7a6ccc9d93fd3dec2369ffc5094a5b" diff --git a/airbyte-ci/connectors/connector_ops/pyproject.toml b/airbyte-ci/connectors/connector_ops/pyproject.toml new file mode 100644 index 000000000000..3faca6122351 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "connector_ops" +version = "0.2.2" +description = "Packaged maintained by the connector operations team to perform CI for connectors" +authors = ["Airbyte "] + +[tool.poetry.dependencies] +python = "^3.10" +click = "^8.1.3" +requests = "^2.28.2" +PyYAML = "^6.0" +GitPython = "^3.1.29" +pydantic = "^1.9" +PyGithub = "^1.58.0" +rich = "^11.0.1" +pydash = "^7.0.4" +google-cloud-storage = "^2.8.0" +ci-credentials = {path = "../ci_credentials"} +pandas = "^2.0.3" + +[tool.poetry.group.test.dependencies] +pytest = "^7.4.0" +pytest-mock = "^3.10.0" +freezegun = "^1.1.0" + +[tool.poetry.scripts] +check-test-strictness-level = "connector_ops.acceptance_test_config_checks:check_test_strictness_level" +write-review-requirements-file = "connector_ops.acceptance_test_config_checks:write_review_requirements_file" +print-mandatory-reviewers = "connector_ops.acceptance_test_config_checks:print_mandatory_reviewers" +allowed-hosts-checks = "connector_ops.allowed_hosts_checks:check_allowed_hosts" +run-qa-checks = "connector_ops.qa_checks:run_qa_checks" diff --git a/tools/ci_connector_ops/pytest.ini b/airbyte-ci/connectors/connector_ops/pytest.ini similarity index 100% rename from tools/ci_connector_ops/pytest.ini rename to airbyte-ci/connectors/connector_ops/pytest.ini diff --git a/airbyte-ci/connectors/connector_ops/tests/conftest.py b/airbyte-ci/connectors/connector_ops/tests/conftest.py new file mode 100644 index 000000000000..fbc90e8a1f59 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/conftest.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +import pandas as pd +import pytest + + +@pytest.fixture(scope="module") +def adoption_metrics_per_connector_version(): + return pd.DataFrame( + [ + { + "connector_definition_id": "dfd88b22-b603-4c3d-aad7-3701784586b1", + "connector_version": "2.0.0", + "number_of_connections": 0, + "number_of_users": 0, + "succeeded_syncs_count": 0, + "failed_syncs_count": 0, + "total_syncs_count": 0, + "sync_success_rate": 0.0, + } + ] + ) + + +@pytest.fixture +def dummy_qa_report() -> pd.DataFrame: + return pd.DataFrame( + [ + { + "connector_type": "source", + "connector_name": "test", + "connector_technical_name": "source-test", + "connector_definition_id": "foobar", + "connector_version": "0.0.0", + "support_level": "community", + "is_on_cloud": False, + "is_appropriate_for_cloud_use": True, + "latest_build_is_successful": True, + "documentation_is_available": False, + "number_of_connections": 0, + "number_of_users": 0, + "sync_success_rate": 0.99, + "total_syncs_count": 0, + "failed_syncs_count": 0, + "succeeded_syncs_count": 0, + "is_eligible_for_promotion_to_cloud": True, + "report_generation_datetime": datetime.utcnow(), + } + ] + ) diff --git a/tools/ci_connector_ops/tests/test_acceptance_test_config_checks.py b/airbyte-ci/connectors/connector_ops/tests/test_acceptance_test_config_checks.py similarity index 99% rename from tools/ci_connector_ops/tests/test_acceptance_test_config_checks.py rename to airbyte-ci/connectors/connector_ops/tests/test_acceptance_test_config_checks.py index 30af67a2b651..c100cd3fb483 100644 --- a/tools/ci_connector_ops/tests/test_acceptance_test_config_checks.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_acceptance_test_config_checks.py @@ -8,7 +8,7 @@ import git import pytest import yaml -from ci_connector_ops import acceptance_test_config_checks +from connector_ops import acceptance_test_config_checks @pytest.fixture diff --git a/airbyte-ci/connectors/connector_ops/tests/test_migration_files/bad-header.md b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/bad-header.md new file mode 100644 index 000000000000..2d430959e1c6 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/bad-header.md @@ -0,0 +1,9 @@ +# Foobar Migration Guide + +## 2.0.0 + +This is something else + +## 1.0.0 + +This is something diff --git a/airbyte-ci/connectors/connector_ops/tests/test_migration_files/bad-title.md b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/bad-title.md new file mode 100644 index 000000000000..10fd8674485a --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/bad-title.md @@ -0,0 +1,9 @@ +# source-foobar Migration Guide + +## Upgrading to 2.0.0 + +This is something else + +## Upgrading to 1.0.0 + +This is something diff --git a/airbyte-ci/connectors/connector_ops/tests/test_migration_files/extra-header.md b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/extra-header.md new file mode 100644 index 000000000000..20d0e6e56332 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/extra-header.md @@ -0,0 +1,13 @@ +# Foobar Migration Guide + +## Upgrading to 2.0.0 + +This is something else + +## Upgrading to 1.0.0 + +This is something + +## Upgrading to 1.0.0 + +This is extra \ No newline at end of file diff --git a/airbyte-ci/connectors/connector_ops/tests/test_migration_files/missing-entry.md b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/missing-entry.md new file mode 100644 index 000000000000..6bc3ef77252b --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/missing-entry.md @@ -0,0 +1,5 @@ +# Foobar Migration Guide + +## Upgrading to 1.0.0 + +This is something \ No newline at end of file diff --git a/airbyte-ci/connectors/connector_ops/tests/test_migration_files/out-of-order.md b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/out-of-order.md new file mode 100644 index 000000000000..12e6bdd37058 --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_migration_files/out-of-order.md @@ -0,0 +1,9 @@ +# Foobar Migration Guide + +## Upgrading to 1.0.0 + +This is something + +## Upgrading to 2.0.0 + +This is something else \ No newline at end of file diff --git a/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py b/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py new file mode 100644 index 000000000000..a7860b65505a --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py @@ -0,0 +1,246 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from pathlib import Path + +import pytest +from connector_ops import qa_checks, utils + + +@pytest.mark.parametrize( + "connector, expect_exists", + [ + (utils.Connector("source-faker"), True), + (utils.Connector("source-foobar"), False), + ], +) +def test_check_documentation_file_exists(connector, expect_exists): + assert qa_checks.check_documentation_file_exists(connector) == expect_exists + + +def test_check_changelog_entry_is_updated_missing_doc(mocker): + mocker.patch.object(qa_checks, "check_documentation_file_exists", mocker.Mock(return_value=False)) + assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) is False + + +def test_check_changelog_entry_is_updated_no_changelog_section(mocker, tmp_path): + mock_documentation_file_path = Path(tmp_path / "doc.md") + mock_documentation_file_path.touch() + + mocker.patch.object(qa_checks.Connector, "documentation_file_path", mock_documentation_file_path) + assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) is False + + +def test_check_changelog_entry_is_updated_version_not_in_changelog(mocker, tmp_path): + mock_documentation_file_path = Path(tmp_path / "doc.md") + with open(mock_documentation_file_path, "w") as f: + f.write("# Changelog") + + mocker.patch.object(qa_checks.Connector, "documentation_file_path", mock_documentation_file_path) + + mocker.patch.object(qa_checks.Connector, "version", "0.0.0") + + assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) is False + + +def test_check_changelog_entry_is_updated_version_in_changelog(mocker, tmp_path): + mock_documentation_file_path = Path(tmp_path / "doc.md") + with open(mock_documentation_file_path, "w") as f: + f.write("# Changelog\n0.0.0") + + mocker.patch.object(qa_checks.Connector, "documentation_file_path", mock_documentation_file_path) + + mocker.patch.object(qa_checks.Connector, "version", "0.0.0") + assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) + + +@pytest.mark.parametrize( + "connector, expect_exists", + [ + (utils.Connector("source-faker"), True), + (utils.Connector("source-foobar"), False), + ], +) +def test_check_connector_icon_is_available(connector, expect_exists): + assert qa_checks.check_connector_icon_is_available(connector) == expect_exists + + +@pytest.mark.parametrize( + "user_input, expect_qa_checks_to_run", + [ + ("not-a-connector", False), + ("connectors/source-faker", True), + ("source-faker", True), + ], +) +def test_run_qa_checks_success(capsys, mocker, user_input, expect_qa_checks_to_run): + mocker.patch.object(qa_checks.sys, "argv", ["", user_input]) + mocker.patch.object(qa_checks, "Connector") + mock_qa_check = mocker.Mock(return_value=True, __name__="mock_qa_check") + if expect_qa_checks_to_run: + mocker.patch.object(qa_checks, "QA_CHECKS", [mock_qa_check]) + with pytest.raises(SystemExit) as wrapped_error: + qa_checks.run_qa_checks() + assert wrapped_error.value.code == 0 + if not expect_qa_checks_to_run: + qa_checks.Connector.assert_not_called() + stdout, _ = capsys.readouterr() + assert "No QA check to run" in stdout + else: + expected_connector_technical_name = user_input.split("/")[-1] + qa_checks.Connector.assert_called_with(expected_connector_technical_name) + mock_qa_check.assert_called_with(qa_checks.Connector.return_value) + stdout, _ = capsys.readouterr() + assert f"Running QA checks for {expected_connector_technical_name}" in stdout + assert f"All QA checks succeeded for {expected_connector_technical_name}" in stdout + + +def test_run_qa_checks_error(capsys, mocker): + mocker.patch.object(qa_checks.sys, "argv", ["", "source-faker"]) + mocker.patch.object(qa_checks, "Connector") + mock_qa_check = mocker.Mock(return_value=False, __name__="mock_qa_check") + mocker.patch.object(qa_checks, "QA_CHECKS", [mock_qa_check]) + with pytest.raises(SystemExit) as wrapped_error: + qa_checks.run_qa_checks() + assert wrapped_error.value.code == 1 + stdout, _ = capsys.readouterr() + assert "QA checks failed for source-faker" in stdout + assert "❌ - mock_qa_check" in stdout + + +@pytest.mark.parametrize( + "file_name, file_line, expected_in_stdout", + [ + ("file_with_http_url.foo", "http://foo.bar", True), + ("file_without_https_url.foo", "", False), + ("file_with_https_url.foo", "https://airbyte.com", False), + ("file_with_http_url_and_ignored.foo", "http://localhost http://airbyte.com", True), + ("file_with_ignored_url.foo", "http://localhost", False), + ("file_with_http_url_in_comment.py", "# http://dev.foo", False), + ("file_with_http_url_in_comment.yml", "# http://dev.foo", False), + ("file_with_http_url_in_comment.yaml", "# http://dev.foo", False), + ("file_with_http_url_in_comment.java", "// http://dev.foo", False), + ("file_with_http_url_in_comment.md", " incr + end + subgraph tests ["Tests"] + build[Build connector docker image] + unit[Run unit tests] + integration[Run integration tests] + cat[Run connector acceptance tests] + secret[Load connector configuration] + + unit-->secret + unit-->build + secret-->integration + secret-->cat + build-->integration + build-->cat + end + entrypoint-->static + entrypoint-->tests + report["Build test report"] + tests-->report + static-->report +``` + +#### Options + +| Option | Multiple | Default value | Description | +| ------------------- | -------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | +| `--fast-tests-only` | True | False | Run unit tests only, skipping integration tests or any tests explicitly tagged as slow. Use this for more frequent checks, when it is not feasible to run the entire test suite. | +| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development.| + +Note: + +* The above options are implemented for Java connectors but may not be available for Python connectors. If an option is not supported, the pipeline will not fail but instead the 'default' behavior will be executed. + +### `connectors build` command +Run a build pipeline for one or multiple connectors and export the built docker image to the local docker host. +It's mainly purposed for local use. + +Build a single connector: +`airbyte-ci connectors --name=source-pokeapi build` + +Build multiple connectors: +`airbyte-ci connectors --name=source-pokeapi --name=source-bigquery build` + +Build certified connectors: +`airbyte-ci connectors --support-level=certified build` + +Build connectors changed on the current branch: +`airbyte-ci connectors --modified build` + +#### What it runs + +For Python and Low Code connectors: + +```mermaid +flowchart TD + arch(For each platform amd64/arm64) + connector[Build connector image] + load[Load to docker host with :dev tag, current platform] + spec[Get spec] + arch-->connector-->spec--"if success"-->load +``` + +For Java connectors: +```mermaid +flowchart TD + arch(For each platform amd64/arm64) + distTar[Gradle distTar task run] + base[Build integration base] + java_base[Build integration base Java] + normalization[Build Normalization] + connector[Build connector image] + + arch-->base-->java_base-->connector + distTar-->connector + normalization--"if supports normalization"-->connector + + load[Load to docker host with :dev tag, current platform] + spec[Get spec] + connector-->spec--"if success"-->load +``` + +### `connectors publish` command +Run a publish pipeline for one or multiple connectors. +It's mainly purposed for CI use to release a connector update. + +### Examples +Publish all connectors modified in the head commit: `airbyte-ci connectors --modified publish` + +### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------------------ | -------- | --------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--pre-release/--main-release` | False | `--pre-release` | | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. | +| `--docker-hub-username` | True | | `DOCKER_HUB_USERNAME` | Your username to connect to DockerHub. | +| `--docker-hub-password` | True | | `DOCKER_HUB_PASSWORD` | Your password to connect to DockerHub. | +| `--spec-cache-gcs-credentials` | False | | `SPEC_CACHE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting spec cache. | +| `--spec-cache-bucket-name` | False | | `SPEC_CACHE_BUCKET_NAME` | The name of the GCS bucket where specs will be cached. | +| `--metadata-service-gcs-credentials` | False | | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files. | +| `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | +| `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | +| `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | + +I've added an empty "Default" column, and you can fill in the default values as needed. +#### What it runs +```mermaid +flowchart TD + validate[Validate the metadata file] + check[Check if the connector image already exists] + build[Build the connector image for all platform variants] + upload_spec[Upload connector spec to the spec cache bucket] + push[Push the connector image from DockerHub, with platform variants] + pull[Pull the connector image from DockerHub to check SPEC can be run and the image layers are healthy] + upload_metadata[Upload its metadata file to the metadata service bucket] + + validate-->check-->build-->upload_spec-->push-->pull-->upload_metadata +``` + +### `metadata` command subgroup + +Available commands: +* `airbyte-ci metadata validate` +* `airbyte-ci metadata upload` +* `airbyte-ci metadata test lib` +* `airbyte-ci metadata test orchestrator` +* `airbyte-ci metadata deploy orchestrator` + +### `metadata validate` command +This commands validates the modified `metadata.yaml` files in the head commit, or all the `metadata.yaml` files. + +#### Example +Validate all `metadata.yaml` files in the repo: +`airbyte-ci metadata validate --all` + +#### Options +| Option | Default | Description | +| ------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------------- | +| `--modified/--all` | `--modified` | Flag to run validation of `metadata.yaml` files on the modified files in the head commit or all the `metadata.yaml` files. | + +### `metadata upload` command +This command upload the modified `metadata.yaml` files in the head commit, or all the `metadata.yaml` files, to a GCS bucket. + +#### Example +Upload all the `metadata.yaml` files to a GCS bucket: +`airbyte-ci metadata upload --all ` + +#### Options +| Option | Required | Default | Mapped environment variable | Description | +| ------------------- | -------- | ------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `--gcs-credentials` | True | | `GCS_CREDENTIALS` | Service account credentials in JSON format with permission to get and upload on the GCS bucket | +| `--modified/--all` | True | `--modified` | | Flag to upload the modified `metadata.yaml` files in the head commit or all the `metadata.yaml` files to a GCS bucket. | + +### `metadata deploy orchestrator` command +This command deploys the metadata service orchestrator to production. +The `DAGSTER_CLOUD_METADATA_API_TOKEN` environment variable must be set. + +#### Example +`airbyte-ci metadata deploy orchestrator` + +#### What it runs +```mermaid +flowchart TD + test[Run orchestrator tests] --> deploy[Deploy orchestrator to Dagster Cloud] +``` + +### `metadata test lib` command +This command runs tests for the metadata service library. + +#### Example +`airbyte-ci metadata test lib` + +### `metadata test orchestrator` command +This command runs tests for the metadata service orchestrator. + +#### Example +`airbyte-ci metadata test orchestrator` + +### `tests` command +This command runs the Python tests for a airbyte-ci poetry package. + +#### Arguments +| Option | Required | Default | Mapped environment variable | Description | +| ------------------ | -------- | ------- | --------------------------- | ---------------------------------------------------------------- | +| `poetry_package_path` | True | | | The path to poetry package to test. | + +#### Options +| Option | Required | Default | Mapped environment variable | Description | +| ------------------ | -------- | ------- | --------------------------- | ---------------------------------------------------------------- | +| `--test-directory` | False | tests | | The path to the directory on which pytest should discover tests, relative to the poetry package. | + + +#### Example +`airbyte-ci test airbyte-ci/connectors/pipelines --test-directory=tests` +`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests` + +## Changelog +| Version | PR | Description | +| ------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 1.1.0 | [#29509](https://github.com/airbytehq/airbyte/pull/29509) | Refactor the airbyte-ci test command to run tests on any poetry package. | +| 1.0.0 | [#28000](https://github.com/airbytehq/airbyte/pull/29232) | Remove release stages in favor of support level from airbyte-ci. | +| 0.5.0 | [#28000](https://github.com/airbytehq/airbyte/pull/28000) | Run connector acceptance tests with dagger-in-dagger. | +| 0.4.7 | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. | +| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | +| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | +| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | +| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | +| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | +| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | +| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | +| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | +| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | +| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | +| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | +| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | +| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | +| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | +| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | +| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | + +## More info +This project is owned by the Connectors Operations team. +We share project updates and remaining stories before its release to production in this [EPIC](https://github.com/airbytehq/airbyte/issues/24403). \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/pipelines/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/__init__.py new file mode 100644 index 000000000000..371bafaa1370 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/__init__.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""The pipelines package.""" +import logging +import os + +from rich.logging import RichHandler + +from . import sentry_utils + +sentry_utils.initialize() + +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("httpx").setLevel(logging.WARNING) +logging_handlers = [RichHandler(rich_tracebacks=True)] +if "CI" in os.environ: + # RichHandler does not work great in the CI + logging_handlers = [logging.StreamHandler()] + +logging.basicConfig( + level=logging.INFO, + format="%(name)s: %(message)s", + datefmt="[%X]", + handlers=logging_handlers, +) + +main_logger = logging.getLogger(__name__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py new file mode 100644 index 000000000000..09bf0600a802 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py @@ -0,0 +1,4 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""The actions package is made to declare reusable pipeline components.""" diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py new file mode 100644 index 000000000000..3f75ffbdc3d6 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py @@ -0,0 +1,1009 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This modules groups functions made to create reusable environments packaged in dagger containers.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import uuid +from pathlib import Path +from typing import TYPE_CHECKING, Callable, List, Optional + +import toml +from dagger import CacheVolume, Client, Container, DaggerError, Directory, File, Platform, Secret +from dagger.engine._version import CLI_VERSION as dagger_engine_version +from pipelines import consts +from pipelines.consts import ( + CI_CREDENTIALS_SOURCE_PATH, + CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH, + CONNECTOR_TESTING_REQUIREMENTS, + LICENSE_SHORT_FILE_PATH, + PYPROJECT_TOML_FILE_PATH, +) +from pipelines.utils import check_path_in_workdir, get_file_contents + +if TYPE_CHECKING: + from pipelines.contexts import ConnectorContext, PipelineContext + + +def with_python_base(context: PipelineContext, python_version: str = "3.10") -> Container: + """Build a Python container with a cache volume for pip cache. + + Args: + context (PipelineContext): The current test context, providing a dagger client and a repository directory. + python_image_name (str, optional): The python image to use to build the python base environment. Defaults to "python:3.9-slim". + + Raises: + ValueError: Raised if the python_image_name is not a python image. + + Returns: + Container: The python base environment container. + """ + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + + base_container = ( + context.dagger_client.container() + .from_(f"python:{python_version}-slim") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "install", "-y", "build-essential", "cmake", "g++", "libffi-dev", "libstdc++6", "git"]) + .with_mounted_cache("/root/.cache/pip", pip_cache) + .with_exec(["pip", "install", "pip==23.1.2"]) + ) + + return base_container + + +def with_testing_dependencies(context: PipelineContext) -> Container: + """Build a testing environment by installing testing dependencies on top of a python base environment. + + Args: + context (PipelineContext): The current test context, providing a dagger client and a repository directory. + + Returns: + Container: The testing environment container. + """ + python_environment: Container = with_python_base(context) + pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH) + license_short_file = context.get_repo_dir(".", include=[LICENSE_SHORT_FILE_PATH]).file(LICENSE_SHORT_FILE_PATH) + + return ( + python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS) + .with_file(f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file) + .with_file(f"/{LICENSE_SHORT_FILE_PATH}", license_short_file) + ) + + +def with_git(dagger_client, ci_github_access_token_secret, ci_git_user) -> Container: + return ( + dagger_client.container() + .from_("alpine:latest") + .with_secret_variable("GITHUB_TOKEN", ci_github_access_token_secret) + .with_exec(["apk", "update"]) + .with_exec(["apk", "add", "git", "tar", "wget"]) + .with_workdir("/ghcli") + .with_exec(["wget", "https://github.com/cli/cli/releases/download/v2.30.0/gh_2.30.0_linux_amd64.tar.gz", "-O", "ghcli.tar.gz"]) + .with_exec(["tar", "--strip-components=1", "-xf", "ghcli.tar.gz"]) + .with_exec(["rm", "ghcli.tar.gz"]) + .with_exec(["cp", "bin/gh", "/usr/local/bin/gh"]) + .with_exec(["git", "config", "--global", "user.email", f"{ci_git_user}@users.noreply.github.com"]) + .with_exec(["git", "config", "--global", "user.name", ci_git_user]) + .with_exec(["git", "config", "--global", "--add", "--bool", "push.autoSetupRemote", "true"]) + ) + + +async def with_installed_pipx_package( + context: PipelineContext, + python_environment: Container, + package_source_code_path: str, + exclude: Optional[List] = None, +) -> Container: + """Install a python package in a python environment container using pipx. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. + python_environment (Container): An existing python environment in which the package will be installed. + package_source_code_path (str): The local path to the package source code. + exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + + Returns: + Container: A python environment container with the python package installed. + """ + pipx_python_environment = with_pipx(python_environment) + container = with_python_package(context, pipx_python_environment, package_source_code_path, exclude=exclude) + + local_dependencies = await find_local_dependencies_in_pyproject_toml(context, container, package_source_code_path, exclude=exclude) + for dependency_directory in local_dependencies: + container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) + + container = container.with_exec(["pipx", "install", f"/{package_source_code_path}"]) + + return container + + +def with_python_package( + context: PipelineContext, + python_environment: Container, + package_source_code_path: str, + exclude: Optional[List] = None, +) -> Container: + """Load a python package source code to a python environment container. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. + python_environment (Container): An existing python environment in which the package will be installed. + package_source_code_path (str): The local path to the package source code. + additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. + exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + + Returns: + Container: A python environment container with the python package source code. + """ + package_source_code_directory: Directory = context.get_repo_dir(package_source_code_path, exclude=exclude) + work_dir_path = f"/{package_source_code_path}" + container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory).with_workdir(work_dir_path) + return container + + +async def find_local_python_dependencies( + context: PipelineContext, + package_source_code_path: str, + search_dependencies_in_setup_py: bool = True, + search_dependencies_in_requirements_txt: bool = True, +) -> List[str]: + """Find local python dependencies of a python package. The dependencies are found in the setup.py and requirements.txt files. + + Args: + context (PipelineContext): The current pipeline context, providing a dagger client and a repository directory. + package_source_code_path (str): The local path to the python package source code. + search_dependencies_in_setup_py (bool, optional): Whether to search for local dependencies in the setup.py file. Defaults to True. + search_dependencies_in_requirements_txt (bool, optional): Whether to search for local dependencies in the requirements.txt file. Defaults to True. + + Returns: + List[str]: Paths to the local dependencies relative to the airbyte repo. + """ + python_environment = with_python_base(context) + container = with_python_package(context, python_environment, package_source_code_path) + + local_dependency_paths = [] + if search_dependencies_in_setup_py: + local_dependency_paths += await find_local_dependencies_in_setup_py(container) + if search_dependencies_in_requirements_txt: + local_dependency_paths += await find_local_dependencies_in_requirements_txt(container, package_source_code_path) + + transitive_dependency_paths = [] + for local_dependency_path in local_dependency_paths: + # Transitive local dependencies installation is achieved by calling their setup.py file, not their requirements.txt file. + transitive_dependency_paths += await find_local_python_dependencies(context, local_dependency_path, True, False) + + all_dependency_paths = local_dependency_paths + transitive_dependency_paths + if all_dependency_paths: + context.logger.debug(f"Found local dependencies for {package_source_code_path}: {all_dependency_paths}") + return all_dependency_paths + + +async def find_local_dependencies_in_setup_py(python_package: Container) -> List[str]: + """Find local dependencies of a python package in its setup.py file. + + Args: + python_package (Container): A python package container. + + Returns: + List[str]: Paths to the local dependencies relative to the airbyte repo. + """ + setup_file_content = await get_file_contents(python_package, "setup.py") + if not setup_file_content: + return [] + + local_setup_dependency_paths = [] + with_egg_info = python_package.with_exec(["python", "setup.py", "egg_info"]) + egg_info_output = await with_egg_info.stdout() + dependency_in_requires_txt = [] + for line in egg_info_output.split("\n"): + if line.startswith("writing requirements to"): + # Find the path to the requirements.txt file that was generated by calling egg_info + requires_txt_path = line.replace("writing requirements to", "").strip() + requirements_txt_content = await with_egg_info.file(requires_txt_path).contents() + dependency_in_requires_txt = requirements_txt_content.split("\n") + + for dependency_line in dependency_in_requires_txt: + if "file://" in dependency_line: + match = re.search(r"file:///(.+)", dependency_line) + if match: + local_setup_dependency_paths.append([match.group(1)][0]) + return local_setup_dependency_paths + + +async def find_local_dependencies_in_requirements_txt(python_package: Container, package_source_code_path: str) -> List[str]: + """Find local dependencies of a python package in a requirements.txt file. + + Args: + python_package (Container): A python environment container with the python package source code. + package_source_code_path (str): The local path to the python package source code. + + Returns: + List[str]: Paths to the local dependencies relative to the airbyte repo. + """ + requirements_txt_content = await get_file_contents(python_package, "requirements.txt") + if not requirements_txt_content: + return [] + + local_requirements_dependency_paths = [] + for line in requirements_txt_content.split("\n"): + # Some package declare themselves as a requirement in requirements.txt, + # #Without line != "-e ." the package will be considered a dependency of itself which can cause an infinite loop + if line.startswith("-e .") and line != "-e .": + local_dependency_path = Path(line[3:]) + package_source_code_path = Path(package_source_code_path) + local_dependency_path = str((package_source_code_path / local_dependency_path).resolve().relative_to(Path.cwd())) + local_requirements_dependency_paths.append(local_dependency_path) + return local_requirements_dependency_paths + + +async def find_local_dependencies_in_pyproject_toml( + context: PipelineContext, + base_container: Container, + pyproject_file_path: str, + exclude: Optional[List] = None, +) -> list: + """Find local dependencies of a python package in a pyproject.toml file. + + Args: + python_package (Container): A python environment container with the python package source code. + pyproject_file_path (str): The path to the pyproject.toml file. + + Returns: + list: Paths to the local dependencies relative to the current directory. + """ + python_package = with_python_package(context, base_container, pyproject_file_path) + pyproject_content_raw = await get_file_contents(python_package, "pyproject.toml") + if not pyproject_content_raw: + return [] + + pyproject_content = toml.loads(pyproject_content_raw) + local_dependency_paths = [] + for dep, value in pyproject_content["tool"]["poetry"]["dependencies"].items(): + if isinstance(value, dict) and "path" in value: + local_dependency_path = Path(value["path"]) + pyproject_file_path = Path(pyproject_file_path) + local_dependency_path = str((pyproject_file_path / local_dependency_path).resolve().relative_to(Path.cwd())) + local_dependency_paths.append(local_dependency_path) + + # Ensure we parse the child dependencies + # TODO handle more than pyproject.toml + child_local_dependencies = await find_local_dependencies_in_pyproject_toml( + context, base_container, local_dependency_path, exclude=exclude + ) + local_dependency_paths += child_local_dependencies + + return local_dependency_paths + + +async def with_installed_python_package( + context: PipelineContext, + python_environment: Container, + package_source_code_path: str, + additional_dependency_groups: Optional[List] = None, + exclude: Optional[List] = None, +) -> Container: + """Install a python package in a python environment container. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. + python_environment (Container): An existing python environment in which the package will be installed. + package_source_code_path (str): The local path to the package source code. + additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. + exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + + Returns: + Container: A python environment container with the python package installed. + """ + install_requirements_cmd = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + install_connector_package_cmd = ["python", "-m", "pip", "install", "."] + + container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude) + + local_dependencies = await find_local_python_dependencies(context, package_source_code_path) + + for dependency_directory in local_dependencies: + container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) + + has_setup_py, has_requirements_txt = await check_path_in_workdir(container, "setup.py"), await check_path_in_workdir( + container, "requirements.txt" + ) + + if has_setup_py: + container = container.with_exec(install_connector_package_cmd) + if has_requirements_txt: + container = container.with_exec(install_requirements_cmd) + + if additional_dependency_groups: + container = container.with_exec( + install_connector_package_cmd[:-1] + [install_connector_package_cmd[-1] + f"[{','.join(additional_dependency_groups)}]"] + ) + + return container + + +def with_python_connector_source(context: ConnectorContext) -> Container: + """Load an airbyte connector source code in a testing environment. + + Args: + context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. + Returns: + Container: A python environment container (with the connector source code). + """ + connector_source_path = str(context.connector.code_directory) + testing_environment: Container = with_testing_dependencies(context) + + return with_python_package(context, testing_environment, connector_source_path) + + +async def with_python_connector_installed(context: ConnectorContext) -> Container: + """Install an airbyte connector python package in a testing environment. + + Args: + context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. + Returns: + Container: A python environment container (with the connector installed). + """ + connector_source_path = str(context.connector.code_directory) + testing_environment: Container = with_testing_dependencies(context) + exclude = [ + f"{context.connector.code_directory}/{item}" + for item in [ + "secrets", + "metadata.yaml", + "bootstrap.md", + "icon.svg", + "README.md", + "Dockerfile", + "acceptance-test-docker.sh", + "build.gradle", + ".hypothesis", + ".dockerignore", + ] + ] + return await with_installed_python_package( + context, testing_environment, connector_source_path, additional_dependency_groups=["dev", "tests", "main"], exclude=exclude + ) + + +async def with_ci_credentials(context: PipelineContext, gsm_secret: Secret) -> Container: + """Install the ci_credentials package in a python environment. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. + gsm_secret (Secret): The secret holding GCP_GSM_CREDENTIALS env variable value. + + Returns: + Container: A python environment with the ci_credentials package installed. + """ + python_base_environment: Container = with_python_base(context) + ci_credentials = await with_installed_pipx_package(context, python_base_environment, CI_CREDENTIALS_SOURCE_PATH) + ci_credentials = ci_credentials.with_env_variable("VERSION", "dagger_ci") + return ci_credentials.with_secret_variable("GCP_GSM_CREDENTIALS", gsm_secret).with_workdir("/") + + +def with_alpine_packages(base_container: Container, packages_to_install: List[str]) -> Container: + """Installs packages using apk-get. + Args: + context (Container): A alpine based container. + + Returns: + Container: A container with the packages installed. + + """ + package_install_command = ["apk", "add"] + return base_container.with_exec(package_install_command + packages_to_install) + + +def with_debian_packages(base_container: Container, packages_to_install: List[str]) -> Container: + """Installs packages using apt-get. + Args: + context (Container): A alpine based container. + + Returns: + Container: A container with the packages installed. + + """ + update_packages_command = ["apt-get", "update"] + package_install_command = ["apt-get", "install", "-y"] + return base_container.with_exec(update_packages_command).with_exec(package_install_command + packages_to_install) + + +def with_pip_packages(base_container: Container, packages_to_install: List[str]) -> Container: + """Installs packages using pip + Args: + context (Container): A container with python installed + + Returns: + Container: A container with the pip packages installed. + + """ + package_install_command = ["pip", "install"] + return base_container.with_exec(package_install_command + packages_to_install) + + +async def with_connector_ops(context: PipelineContext) -> Container: + """Installs the connector_ops package in a Container running Python > 3.10 with git.. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_connector_sources sources will be pulled. + + Returns: + Container: A python environment container with connector_ops installed. + """ + python_base_environment: Container = with_python_base(context) + + return await with_installed_pipx_package(context, python_base_environment, CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH) + + +def with_global_dockerd_service(dagger_client: Client) -> Container: + """Create a container with a docker daemon running. + We expose its 2375 port to use it as a docker host for docker-in-docker use cases. + Args: + dagger_client (Client): The dagger client used to create the container. + Returns: + Container: The container running dockerd as a service + """ + return ( + dagger_client.container() + .from_(consts.DOCKER_DIND_IMAGE) + .with_mounted_cache( + "/tmp", + dagger_client.cache_volume("shared-tmp"), + ) + .with_exposed_port(2375) + .with_exec(["dockerd", "--log-level=error", "--host=tcp://0.0.0.0:2375", "--tls=false"], insecure_root_capabilities=True) + ) + + +def with_bound_docker_host( + context: ConnectorContext, + container: Container, +) -> Container: + """Bind a container to a docker host. It will use the dockerd service as a docker host. + + Args: + context (ConnectorContext): The current connector context. + container (Container): The container to bind to the docker host. + Returns: + Container: The container bound to the docker host. + """ + dockerd = context.dockerd_service + docker_hostname = "global-docker-host" + return ( + container.with_env_variable("DOCKER_HOST", f"tcp://{docker_hostname}:2375") + .with_service_binding(docker_hostname, dockerd) + .with_mounted_cache("/tmp", context.dagger_client.cache_volume("shared-tmp")) + ) + + +def bound_docker_host(context: ConnectorContext) -> Container: + def bound_docker_host_inner(container: Container) -> Container: + return with_bound_docker_host(context, container) + + return bound_docker_host_inner + + +def with_docker_cli(context: ConnectorContext) -> Container: + """Create a container with the docker CLI installed and bound to a persistent docker host. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + Container: A docker cli container bound to a docker host. + """ + docker_cli = context.dagger_client.container().from_(consts.DOCKER_CLI_IMAGE) + return with_bound_docker_host(context, docker_cli) + + +def with_gradle( + context: ConnectorContext, + sources_to_include: List[str] = None, + bind_to_docker_host: bool = True, +) -> Container: + """Create a container with Gradle installed and bound to a persistent docker host. + + Args: + context (ConnectorContext): The current connector context. + sources_to_include (List[str], optional): List of additional source path to mount to the container. Defaults to None. + bind_to_docker_host (bool): Whether to bind the gradle container to a docker host. + + Returns: + Container: A container with Gradle installed and Java sources from the repository. + """ + + include = [ + ".root", + ".env", + "build.gradle", + "deps.toml", + "gradle.properties", + "gradle", + "gradlew", + "LICENSE_SHORT", + "publish-repositories.gradle", + "settings.gradle", + "build.gradle", + "tools/gradle", + "spotbugs-exclude-filter-file.xml", + "buildSrc", + "tools/bin/build_image.sh", + "tools/lib/lib.sh", + "tools/gradle/codestyle", + "pyproject.toml", + ] + + if sources_to_include: + include += sources_to_include + # TODO re-enable once we have fixed the over caching issue + # gradle_dependency_cache: CacheVolume = context.dagger_client.cache_volume("gradle-dependencies-caching") + # gradle_build_cache: CacheVolume = context.dagger_client.cache_volume(f"{context.connector.technical_name}-gradle-build-cache") + + openjdk_with_docker = ( + context.dagger_client.container() + .from_("openjdk:17.0.1-jdk-slim") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "install", "-y", "curl", "jq", "rsync", "npm", "pip"]) + .with_env_variable("VERSION", consts.DOCKER_VERSION) + .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"]) + .with_env_variable("GRADLE_HOME", "/root/.gradle") + .with_exec(["mkdir", "/airbyte"]) + .with_workdir("/airbyte") + .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include)) + .with_exec(["mkdir", "-p", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH]) + # TODO (ben) reenable once we have fixed the over caching issue + # .with_mounted_cache(consts.GRADLE_BUILD_CACHE_PATH, gradle_build_cache, sharing=CacheSharingMode.LOCKED) + # .with_mounted_cache(consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH, gradle_dependency_cache) + .with_env_variable("GRADLE_RO_DEP_CACHE", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH) + ) + + if bind_to_docker_host: + return with_bound_docker_host(context, openjdk_with_docker) + else: + return openjdk_with_docker + + +async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str): + """Load a docker image tar archive to the docker host. + + Args: + context (ConnectorContext): The current connector context. + tar_file (File): The file object holding the docker image tar archive. + image_tag (str): The tag to create on the image if it has no tag. + """ + # Hacky way to make sure the image is always loaded + tar_name = f"{str(uuid.uuid4())}.tar" + docker_cli = with_docker_cli(context).with_mounted_file(tar_name, tar_file) + + image_load_output = await docker_cli.with_exec(["docker", "load", "--input", tar_name]).stdout() + # Not tagged images only have a sha256 id the load output shares. + if "sha256:" in image_load_output: + image_id = image_load_output.replace("\n", "").replace("Loaded image ID: sha256:", "") + await docker_cli.with_exec(["docker", "tag", image_id, image_tag]) + image_sha = json.loads(await docker_cli.with_exec(["docker", "inspect", image_tag]).stdout())[0].get("Id") + return image_sha + + +def with_pipx(base_python_container: Container) -> Container: + """Installs pipx in a python container. + + Args: + base_python_container (Container): The container to install pipx on. + + Returns: + Container: A python environment with pipx installed. + """ + python_with_pipx = with_pip_packages(base_python_container, ["pipx"]).with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + + return python_with_pipx + + +def with_poetry(context: PipelineContext) -> Container: + """Install poetry in a python environment. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. + Returns: + Container: A python environment with poetry installed. + """ + python_base_environment: Container = with_python_base(context) + python_with_git = with_debian_packages(python_base_environment, ["git"]) + python_with_poetry = with_pip_packages(python_with_git, ["poetry"]) + + # poetry_cache: CacheVolume = context.dagger_client.cache_volume("poetry_cache") + # poetry_with_cache = python_with_poetry.with_mounted_cache("/root/.cache/pypoetry", poetry_cache, sharing=CacheSharingMode.SHARED) + + return python_with_poetry + + +def with_poetry_module(context: PipelineContext, parent_dir: Directory, module_path: str) -> Container: + """Sets up a Poetry module. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. + Returns: + Container: A python environment with dependencies installed using poetry. + """ + poetry_install_dependencies_cmd = ["poetry", "install"] + + python_with_poetry = with_poetry(context) + return ( + python_with_poetry.with_mounted_directory("/src", parent_dir) + .with_workdir(f"/src/{module_path}") + .with_exec(poetry_install_dependencies_cmd) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + ) + + +def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container: + return ( + context.dagger_client.container(platform=build_platform) + .from_("amazonlinux:2022.0.20220831.1") + .with_workdir("/airbyte") + .with_file("base.sh", context.get_repo_dir("airbyte-integrations/bases/base", include=["base.sh"]).file("base.sh")) + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") + .with_label("io.airbyte.version", "0.1.0") + .with_label("io.airbyte.name", "airbyte/integration-base") + ) + + +def with_integration_base_java(context: PipelineContext, build_platform: Platform, jdk_version: str = "17.0.4") -> Container: + integration_base = with_integration_base(context, build_platform) + return ( + context.dagger_client.container(platform=build_platform) + .from_(f"amazoncorretto:{jdk_version}") + .with_directory("/airbyte", integration_base.directory("/airbyte")) + .with_exec(["yum", "install", "-y", "tar", "openssl"]) + .with_exec(["yum", "clean", "all"]) + .with_workdir("/airbyte") + .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer")) + .with_file("javabase.sh", context.get_repo_dir("airbyte-integrations/bases/base-java", include=["javabase.sh"]).file("javabase.sh")) + .with_env_variable("AIRBYTE_SPEC_CMD", "/airbyte/javabase.sh --spec") + .with_env_variable("AIRBYTE_CHECK_CMD", "/airbyte/javabase.sh --check") + .with_env_variable("AIRBYTE_DISCOVER_CMD", "/airbyte/javabase.sh --discover") + .with_env_variable("AIRBYTE_READ_CMD", "/airbyte/javabase.sh --read") + .with_env_variable("AIRBYTE_WRITE_CMD", "/airbyte/javabase.sh --write") + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") + .with_label("io.airbyte.version", "0.1.2") + .with_label("io.airbyte.name", "airbyte/integration-base-java") + ) + + +BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { + "destination-bigquery": { + "dockerfile": "Dockerfile", + "dbt_adapter": "dbt-bigquery==1.0.0", + "integration_name": "bigquery", + "normalization_image": "airbyte/normalization:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, + "destination-clickhouse": { + "dockerfile": "clickhouse.Dockerfile", + "dbt_adapter": "dbt-clickhouse>=1.4.0", + "integration_name": "clickhouse", + "normalization_image": "airbyte/normalization-clickhouse:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-duckdb": { + "dockerfile": "duckdb.Dockerfile", + "dbt_adapter": "dbt-duckdb==1.0.1", + "integration_name": "duckdb", + "normalization_image": "airbyte/normalization-duckdb:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-mssql": { + "dockerfile": "mssql.Dockerfile", + "dbt_adapter": "dbt-sqlserver==1.0.0", + "integration_name": "mssql", + "normalization_image": "airbyte/normalization-mssql:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, + "destination-mysql": { + "dockerfile": "mysql.Dockerfile", + "dbt_adapter": "dbt-mysql==1.0.0", + "integration_name": "mysql", + "normalization_image": "airbyte/normalization-mysql:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-oracle": { + "dockerfile": "oracle.Dockerfile", + "dbt_adapter": "dbt-oracle==0.4.3", + "integration_name": "oracle", + "normalization_image": "airbyte/normalization-oracle:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-postgres": { + "dockerfile": "Dockerfile", + "dbt_adapter": "dbt-postgres==1.0.0", + "integration_name": "postgres", + "normalization_image": "airbyte/normalization:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-redshift": { + "dockerfile": "redshift.Dockerfile", + "dbt_adapter": "dbt-redshift==1.0.0", + "integration_name": "redshift", + "normalization_image": "airbyte/normalization-redshift:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, + "destination-snowflake": { + "dockerfile": "snowflake.Dockerfile", + "dbt_adapter": "dbt-snowflake==1.0.0", + "integration_name": "snowflake", + "normalization_image": "airbyte/normalization-snowflake:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": ["gcc-c++"], + }, + "destination-tidb": { + "dockerfile": "tidb.Dockerfile", + "dbt_adapter": "dbt-tidb==1.0.1", + "integration_name": "tidb", + "normalization_image": "airbyte/normalization-tidb:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, +} + +DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { + **BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, + **{f"{k}-strict-encrypt": v for k, v in BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION.items()}, +} + + +def with_normalization(context: ConnectorContext, build_platform: Platform) -> Container: + return context.dagger_client.container(platform=build_platform).from_( + DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["normalization_image"] + ) + + +def with_integration_base_java_and_normalization(context: PipelineContext, build_platform: Platform) -> Container: + yum_packages_to_install = [ + "python3", + "python3-devel", + "jq", + "sshpass", + "git", + ] + + additional_yum_packages = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["yum_packages"] + yum_packages_to_install += additional_yum_packages + + dbt_adapter_package = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["dbt_adapter"] + normalization_integration_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["integration_name"] + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + + return ( + with_integration_base_java(context, build_platform) + .with_exec(["yum", "install", "-y"] + yum_packages_to_install) + .with_exec(["yum", "clean", "all"]) + .with_exec(["alternatives", "--install", "/usr/bin/python", "python", "/usr/bin/python3", "60"]) + .with_mounted_cache("/root/.cache/pip", pip_cache) + .with_exec(["python", "-m", "ensurepip", "--upgrade"]) + # Workaround for https://github.com/yaml/pyyaml/issues/601 + .with_exec(["pip3", "install", "Cython<3.0", "pyyaml~=5.4", "--no-build-isolation"]) + .with_exec(["pip3", "install", dbt_adapter_package]) + .with_directory("airbyte_normalization", with_normalization(context, build_platform).directory("/airbyte")) + .with_workdir("airbyte_normalization") + .with_exec(["sh", "-c", "mv * .."]) + .with_workdir("/airbyte") + .with_exec(["rm", "-rf", "airbyte_normalization"]) + # We don't install the airbyte-protocol legacy package as its not used anymore and not compatible with Cython 3.x + # .with_workdir("/airbyte/base_python_structs") + # .with_exec(["pip3", "install", "--force-reinstall", "Cython<3.0", ".",]) + .with_workdir("/airbyte/normalization_code") + .with_exec(["pip3", "install", "."]) + .with_workdir("/airbyte/normalization_code/dbt-template/") + # amazon linux 2 isn't compatible with urllib3 2.x, so force 1.x + .with_exec(["pip3", "install", "urllib3<2"]) + .with_exec(["dbt", "deps"]) + .with_workdir("/airbyte") + .with_file( + "run_with_normalization.sh", + context.get_repo_dir("airbyte-integrations/bases/base-java", include=["run_with_normalization.sh"]).file( + "run_with_normalization.sh" + ), + ) + .with_env_variable("AIRBYTE_NORMALIZATION_INTEGRATION", normalization_integration_name) + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/run_with_normalization.sh") + ) + + +async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container: + application = context.connector.technical_name + + build_stage = ( + with_integration_base_java(context, build_platform) + .with_workdir("/airbyte") + .with_env_variable("APPLICATION", context.connector.technical_name) + .with_file(f"{application}.tar", connector_java_tar_file) + .with_exec(["tar", "xf", f"{application}.tar", "--strip-components=1"]) + .with_exec(["rm", "-rf", f"{application}.tar"]) + ) + + if ( + context.connector.supports_normalization + and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"] + ): + base = with_integration_base_java_and_normalization(context, build_platform) + entrypoint = ["/airbyte/run_with_normalization.sh"] + else: + base = with_integration_base_java(context, build_platform) + entrypoint = ["/airbyte/base.sh"] + + connector_container = ( + base.with_workdir("/airbyte") + .with_env_variable("APPLICATION", application) + .with_mounted_directory("builts_artifacts", build_stage.directory("/airbyte")) + .with_exec(["sh", "-c", "mv builts_artifacts/* ."]) + .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) + .with_label("io.airbyte.name", context.metadata["dockerRepository"]) + .with_entrypoint(entrypoint) + ) + return await finalize_build(context, connector_container) + + +async def get_cdk_version_from_python_connector(python_connector: Container) -> Optional[str]: + pip_freeze_stdout = await python_connector.with_entrypoint("pip").with_exec(["freeze"]).stdout() + pip_dependencies = [dep.split("==") for dep in pip_freeze_stdout.split("\n")] + for package_name, package_version in pip_dependencies: + if package_name == "airbyte-cdk": + return package_version + return None + + +async def with_airbyte_python_connector(context: ConnectorContext, build_platform: Platform) -> Container: + if context.connector.technical_name == "source-file-secure": + return await with_airbyte_python_connector_full_dagger(context, build_platform) + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + connector_container = ( + context.dagger_client.container(platform=build_platform) + .with_mounted_cache("/root/.cache/pip", pip_cache) + .build(await context.get_connector_dir()) + .with_label("io.airbyte.name", context.metadata["dockerRepository"]) + ) + cdk_version = await get_cdk_version_from_python_connector(connector_container) + if cdk_version: + connector_container = connector_container.with_label("io.airbyte.cdk_version", cdk_version) + context.cdk_version = cdk_version + if not await connector_container.label("io.airbyte.version") == context.metadata["dockerImageTag"]: + raise DaggerError( + "Abusive caching might be happening. The connector container should have been built with the correct version as defined in metadata.yaml" + ) + return await finalize_build(context, connector_container) + + +async def finalize_build(context: ConnectorContext, connector_container: Container) -> Container: + """Finalize build by adding dagger engine version label and running finalize_build.sh or finalize_build.py if present in the connector directory.""" + connector_container = connector_container.with_label("io.dagger.engine_version", dagger_engine_version) + connector_dir_with_finalize_script = await context.get_connector_dir(include=["finalize_build.sh", "finalize_build.py"]) + finalize_scripts = await connector_dir_with_finalize_script.entries() + if not finalize_scripts: + return connector_container + + # We don't want finalize scripts to override the entrypoint so we keep it in memory to reset it after finalization + original_entrypoint = await connector_container.entrypoint() + + has_finalize_bash_script = "finalize_build.sh" in finalize_scripts + has_finalize_python_script = "finalize_build.py" in finalize_scripts + if has_finalize_python_script and has_finalize_bash_script: + raise Exception("Connector has both finalize_build.sh and finalize_build.py, please remove one of them") + + if has_finalize_python_script: + context.logger.info(f"{context.connector.technical_name} has a finalize_build.py script, running it to finalize build...") + module_path = context.connector.code_directory / "finalize_build.py" + connector_finalize_module_spec = importlib.util.spec_from_file_location( + f"{context.connector.code_directory.name}_finalize", module_path + ) + connector_finalize_module = importlib.util.module_from_spec(connector_finalize_module_spec) + connector_finalize_module_spec.loader.exec_module(connector_finalize_module) + try: + connector_container = await connector_finalize_module.finalize_build(context, connector_container) + except AttributeError: + raise Exception("Connector has a finalize_build.py script but it doesn't have a finalize_build function.") + + if has_finalize_bash_script: + context.logger.info(f"{context.connector.technical_name} has finalize_build.sh script, running it to finalize build...") + connector_container = ( + connector_container.with_file("/tmp/finalize_build.sh", connector_dir_with_finalize_script.file("finalize_build.sh")) + .with_entrypoint("sh") + .with_exec(["/tmp/finalize_build.sh"]) + ) + + return connector_container.with_entrypoint(original_entrypoint) + + +async def with_airbyte_python_connector_full_dagger(context: ConnectorContext, build_platform: Platform) -> Container: + setup_dependencies_to_mount = await find_local_python_dependencies( + context, str(context.connector.code_directory), search_dependencies_in_setup_py=True, search_dependencies_in_requirements_txt=False + ) + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + base = context.dagger_client.container(platform=build_platform).from_("python:3.9-slim") + snake_case_name = context.connector.technical_name.replace("-", "_") + entrypoint = ["python", "/airbyte/integration_code/main.py"] + builder = ( + base.with_workdir("/airbyte/integration_code") + .with_env_variable("DAGGER_BUILD", "True") + .with_exec(["apt-get", "update"]) + .with_mounted_cache("/root/.cache/pip", pip_cache) + .with_exec(["pip", "install", "--upgrade", "pip"]) + .with_exec(["apt-get", "install", "-y", "tzdata"]) + .with_file("setup.py", (await context.get_connector_dir(include="setup.py")).file("setup.py")) + ) + + for dependency_path in setup_dependencies_to_mount: + in_container_dependency_path = f"/local_dependencies/{Path(dependency_path).name}" + builder = builder.with_mounted_directory(in_container_dependency_path, context.get_repo_dir(dependency_path)) + + builder = builder.with_exec(["pip", "install", "--prefix=/install", "."]) + + connector_container = ( + base.with_workdir("/airbyte/integration_code") + .with_directory("/usr/local", builder.directory("/install")) + .with_file("/usr/localtime", builder.file("/usr/share/zoneinfo/Etc/UTC")) + .with_new_file("/etc/timezone", contents="Etc/UTC") + .with_exec(["apt-get", "install", "-y", "bash"]) + .with_file("main.py", (await context.get_connector_dir(include="main.py")).file("main.py")) + .with_directory(snake_case_name, (await context.get_connector_dir(include=snake_case_name)).directory(snake_case_name)) + .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) + .with_entrypoint(entrypoint) + .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) + .with_label("io.airbyte.name", context.metadata["dockerRepository"]) + ) + return await finalize_build(context, connector_container) + + +def with_crane( + context: PipelineContext, +) -> Container: + """Crane is a tool to analyze and manipulate container images. + We can use it to extract the image manifest and the list of layers or list the existing tags on an image repository. + https://github.com/google/go-containerregistry/tree/main/cmd/crane + """ + + # We use the debug image as it contains a shell which we need to properly use environment variables + # https://github.com/google/go-containerregistry/tree/main/cmd/crane#images + base_container = context.dagger_client.container().from_("gcr.io/go-containerregistry/crane/debug:v0.15.1") + + if context.docker_hub_username_secret and context.docker_hub_password_secret: + base_container = ( + base_container.with_secret_variable("DOCKER_HUB_USERNAME", context.docker_hub_username_secret).with_secret_variable( + "DOCKER_HUB_PASSWORD", context.docker_hub_password_secret + ) + # We need to use skip_entrypoint=True to avoid the entrypoint to be overridden by the crane command + # We use sh -c to be able to use environment variables in the command + # This is a workaround as the default crane entrypoint doesn't support environment variables + .with_exec( + ["sh", "-c", "crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"], skip_entrypoint=True + ) + ) + + return base_container + + +def mounted_connector_secrets(context: PipelineContext, secret_directory_path="secrets") -> Callable: + def mounted_connector_secrets_inner(container: Container): + container = container.with_exec(["mkdir", secret_directory_path], skip_entrypoint=True) + for secret_file_name, secret in context.connector_secrets.items(): + container = container.with_mounted_secret(f"{secret_directory_path}/{secret_file_name}", secret) + return container + + return mounted_connector_secrets_inner diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py b/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py new file mode 100644 index 000000000000..3024cf8378c0 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups functions to interact with remote storage services like S3 or GCS.""" + +import uuid +from pathlib import Path +from typing import List, Optional, Tuple + +from dagger import Client, File, Secret +from pipelines.utils import get_exec_result, secret_host_variable, with_exit_code + +GOOGLE_CLOUD_SDK_TAG = "425.0.0-slim" + + +async def upload_to_s3(dagger_client: Client, file_to_upload_path: Path, key: str, bucket: str) -> int: + """Upload a local file to S3 using the AWS CLI docker image and running aws s3 cp command. + + Args: + dagger_client (Client): The dagger client. + file_to_upload_path (Path): The local path to the file to upload. + key (str): The key that will be written on the S3 bucket. + bucket (str): The S3 bucket name. + + Returns: + int: Exit code of the upload process. + """ + s3_uri = f"s3://{bucket}/{key}" + file_to_upload: File = dagger_client.host().directory(".", include=[str(file_to_upload_path)]).file(str(file_to_upload_path)) + return await with_exit_code( + dagger_client.container() + .from_("amazon/aws-cli:latest") + .with_file(str(file_to_upload_path), file_to_upload) + .with_(secret_host_variable(dagger_client, "AWS_ACCESS_KEY_ID")) + .with_(secret_host_variable(dagger_client, "AWS_SECRET_ACCESS_KEY")) + .with_(secret_host_variable(dagger_client, "AWS_DEFAULT_REGION")) + .with_exec(["s3", "cp", str(file_to_upload_path), s3_uri]) + ) + + +async def upload_to_gcs( + dagger_client: Client, + file_to_upload: File, + key: str, + bucket: str, + gcs_credentials: Secret, + flags: Optional[List] = None, + cache_upload: bool = False, +) -> Tuple[int, str, str]: + """Upload a local file to GCS using the AWS CLI docker image and running aws s3 cp command. + Args: + dagger_client (Client): The dagger client. + file_to_upload (File): The dagger File to upload. + key (str): The key that will be written on the S3 bucket. + bucket (str): The S3 bucket name. + gcs_credentials (Secret): The dagger secret holding the credentials to get and upload the targeted GCS bucket. + flags (List[str]): Flags to be passed to the 'gcloud storage cp' command. + cache_upload (bool): If false, the gcloud commands will be executed on each call. + Returns: + Tuple[int, str, str]: Exit code, stdout, stderr + """ + flags = [] if flags is None else flags + gcs_uri = f"gs://{bucket}/{key}" + dagger_client = dagger_client.pipeline(f"Upload file to {gcs_uri}") + cp_command = ["gcloud", "storage", "cp"] + flags + ["to_upload", gcs_uri] + + gcloud_container = ( + dagger_client.container() + .from_(f"google/cloud-sdk:{GOOGLE_CLOUD_SDK_TAG}") + .with_workdir("/upload") + .with_new_file("credentials.json", contents=await gcs_credentials.plaintext()) + .with_env_variable("GOOGLE_APPLICATION_CREDENTIALS", "/upload/credentials.json") + .with_file("to_upload", file_to_upload) + ) + if not cache_upload: + gcloud_container = gcloud_container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + else: + gcloud_container = gcloud_container.without_env_variable("CACHEBUSTER") + + gcloud_auth_container = gcloud_container.with_exec(["gcloud", "auth", "login", "--cred-file=credentials.json"]) + if (await with_exit_code(gcloud_auth_container)) == 1: + gcloud_auth_container = gcloud_container.with_exec(["gcloud", "auth", "activate-service-account", "--key-file", "credentials.json"]) + + gcloud_cp_container = gcloud_auth_container.with_exec(cp_command) + return await get_exec_result(gcloud_cp_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py new file mode 100644 index 000000000000..985ca064b5b9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This modules groups functions made to download/upload secrets from/to a remote secret service and provide these secret in a dagger Directory.""" +from __future__ import annotations + +import datetime +from typing import TYPE_CHECKING + +from dagger import Secret +from pipelines.actions import environments +from pipelines.utils import get_file_contents, get_secret_host_variable + +if TYPE_CHECKING: + from dagger import Container + from pipelines.contexts import ConnectorContext + + +async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: + """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix. + We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, + And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. + """ + secrets_to_mask = [] + if secrets_to_mask_file := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): + for secret_to_mask in secrets_to_mask_file.splitlines(): + # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" + # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked + print(f"::add-mask::{secret_to_mask}") + secrets_to_mask.append(secret_to_mask) + return secrets_to_mask + + +async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> dict[str, Secret]: + """Use the ci-credentials tool to download the secrets stored for a specific connector to a Directory. + + Args: + context (ConnectorContext): The context providing a connector object. + gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS". + + Returns: + Directory: A directory with the downloaded secrets. + """ + gsm_secret = get_secret_host_variable(context.dagger_client, gcp_gsm_env_variable_name) + secrets_path = f"/{context.connector.code_directory}/secrets" + ci_credentials = await environments.with_ci_credentials(context, gsm_secret) + with_downloaded_secrets = ( + ci_credentials.with_exec(["mkdir", "-p", secrets_path]) + .with_env_variable( + "CACHEBUSTER", datetime.datetime.now().isoformat() + ) # Secrets can be updated on GSM anytime, we can't cache this step... + .with_exec(["ci_credentials", context.connector.technical_name, "write-to-storage"]) + ) + # We don't want to print secrets in the logs when running locally. + if context.is_ci: + context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets) + connector_secrets = {} + for secret_file in await with_downloaded_secrets.directory(secrets_path).entries(): + secret_plaintext = await with_downloaded_secrets.directory(secrets_path).file(secret_file).contents() + # We have to namespace secrets as Dagger derives session wide secret ID from their name + unique_secret_name = f"{context.connector.technical_name}_{secret_file}" + connector_secrets[secret_file] = context.dagger_client.set_secret(unique_secret_name, secret_plaintext) + + return connector_secrets + + +async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS"): + """Use the ci-credentials tool to upload the secrets stored in the context's updated_secrets-dir. + + Args: + context (ConnectorContext): The context providing a connector object and the update secrets dir. + gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS". + + Returns: + container (Container): The executed ci-credentials update-secrets command. + + Raises: + ExecError: If the command returns a non-zero exit code. + """ + gsm_secret = get_secret_host_variable(context.dagger_client, gcp_gsm_env_variable_name) + secrets_path = f"/{context.connector.code_directory}/secrets" + + ci_credentials = await environments.with_ci_credentials(context, gsm_secret) + + return await ci_credentials.with_directory(secrets_path, context.updated_secrets_dir).with_exec( + ["ci_credentials", context.connector.technical_name, "update-secrets"] + ) + + +async def get_connector_secrets(context: ConnectorContext) -> dict[str, Secret]: + """Download the secrets from GSM or use the local secrets directory for a connector. + + Args: + context (ConnectorContext): The context providing the connector directory and the use_remote_secrets flag. + + Returns: + Directory: A directory with the downloaded connector secrets. + """ + if context.use_remote_secrets: + connector_secrets = await download(context) + else: + raise NotImplementedError("Local secrets are not implemented yet. See https://github.com/airbytehq/airbyte/issues/25621") + return connector_secrets diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py new file mode 100644 index 000000000000..b5f397d69ff2 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/bases.py @@ -0,0 +1,659 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module declare base / abstract models to be reused in a pipeline lifecycle.""" + +from __future__ import annotations + +import json +import logging +import webbrowser +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +from typing import TYPE_CHECKING, Any, ClassVar, List, Optional, Set + +import anyio +import asyncer +from anyio import Path +from connector_ops.utils import Connector, console +from dagger import Container, DaggerError +from jinja2 import Environment, PackageLoader, select_autoescape +from pipelines import sentry_utils +from pipelines.actions import remote_storage +from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT, PYPROJECT_TOML_FILE_PATH +from pipelines.utils import METADATA_FILE_NAME, check_path_in_workdir, format_duration, get_exec_result +from rich.console import Group +from rich.panel import Panel +from rich.style import Style +from rich.table import Table +from rich.text import Text +from tabulate import tabulate + +if TYPE_CHECKING: + from pipelines.contexts import PipelineContext + + +@dataclass(frozen=True) +class ConnectorWithModifiedFiles(Connector): + modified_files: Set[Path] = field(default_factory=frozenset) + + @property + def has_metadata_change(self) -> bool: + return any(path.name == METADATA_FILE_NAME for path in self.modified_files) + + +class CIContext(str, Enum): + """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"].""" + + MANUAL = "manual" + PULL_REQUEST = "pull_request" + NIGHTLY_BUILDS = "nightly_builds" + MASTER = "master" + + def __str__(self) -> str: + return self.value + + +class StepStatus(Enum): + """An Enum to characterize the success, failure or skipping of a Step.""" + + SUCCESS = "Successful" + FAILURE = "Failed" + SKIPPED = "Skipped" + + def get_rich_style(self) -> Style: + """Match color used in the console output to the step status.""" + if self is StepStatus.SUCCESS: + return Style(color="green") + if self is StepStatus.FAILURE: + return Style(color="red", bold=True) + if self is StepStatus.SKIPPED: + return Style(color="yellow") + + def get_emoji(self) -> str: + """Match emoji used in the console output to the step status.""" + if self is StepStatus.SUCCESS: + return "✅" + if self is StepStatus.FAILURE: + return "❌" + if self is StepStatus.SKIPPED: + return "🟡" + + def __str__(self) -> str: # noqa D105 + return self.value + + +class Step(ABC): + """An abstract class to declare and run pipeline step.""" + + title: ClassVar[str] + max_retries: ClassVar[int] = 0 + max_dagger_error_retries: ClassVar[int] = 3 + should_log: ClassVar[bool] = True + success_exit_code: ClassVar[int] = 0 + skipped_exit_code: ClassVar[int] = None + # The max duration of a step run. If the step run for more than this duration it will be considered as timed out. + # The default of 5 hours is arbitrary and can be changed if needed. + max_duration: ClassVar[timedelta] = timedelta(hours=5) + + retry_delay = timedelta(seconds=10) + + def __init__(self, context: PipelineContext) -> None: # noqa D107 + self.context = context + self.retry_count = 0 + self.started_at = None + self.stopped_at = None + + @property + def run_duration(self) -> timedelta: + if self.started_at and self.stopped_at: + return self.stopped_at - self.started_at + else: + return timedelta(seconds=0) + + @property + def logger(self) -> logging.Logger: + if self.should_log: + return logging.getLogger(f"{self.context.pipeline_name} - {self.title}") + else: + disabled_logger = logging.getLogger() + disabled_logger.disabled = True + return disabled_logger + + @property + def dagger_client(self) -> Container: + return self.context.dagger_client.pipeline(self.title) + + async def log_progress(self, completion_event: anyio.Event) -> None: + """Log the step progress every 30 seconds until the step is done.""" + while not completion_event.is_set(): + duration = datetime.utcnow() - self.started_at + elapsed_seconds = duration.total_seconds() + if elapsed_seconds > 30 and round(elapsed_seconds) % 30 == 0: + self.logger.info(f"⏳ Still running... (duration: {format_duration(duration)})") + await anyio.sleep(1) + + async def run_with_completion(self, completion_event: anyio.Event, *args, **kwargs) -> StepResult: + """Run the step with a timeout and set the completion event when the step is done.""" + try: + with anyio.fail_after(self.max_duration.total_seconds()): + result = await self._run(*args, **kwargs) + completion_event.set() + return result + except TimeoutError: + self.retry_count = self.max_retries + 1 + self.logger.error(f"🚨 {self.title} timed out after {self.max_duration}. No additional retry will happen.") + completion_event.set() + return self._get_timed_out_step_result() + + @sentry_utils.with_step_context + async def run(self, *args, **kwargs) -> StepResult: + """Public method to run the step. It output a step result. + + If an unexpected dagger error happens it outputs a failed step result with the exception payload. + + Returns: + StepResult: The step result following the step run. + """ + self.logger.info(f"🚀 Start {self.title}") + self.started_at = datetime.utcnow() + completion_event = anyio.Event() + try: + async with asyncer.create_task_group() as task_group: + soon_result = task_group.soonify(self.run_with_completion)(completion_event, *args, **kwargs) + task_group.soonify(self.log_progress)(completion_event) + step_result = soon_result.value + except DaggerError as e: + self.logger.error("Step failed with an unexpected dagger error", exc_info=e) + step_result = StepResult(self, StepStatus.FAILURE, stderr=str(e), exc_info=e) + + self.stopped_at = datetime.utcnow() + self.log_step_result(step_result) + + lets_retry = self.should_retry(step_result) + step_result = await self.retry(step_result, *args, **kwargs) if lets_retry else step_result + return step_result + + def should_retry(self, step_result: StepResult) -> bool: + """Return True if the step should be retried.""" + if step_result.status is not StepStatus.FAILURE: + return False + max_retries = self.max_dagger_error_retries if step_result.exc_info else self.max_retries + return self.retry_count < max_retries and max_retries > 0 + + async def retry(self, step_result, *args, **kwargs) -> StepResult: + self.retry_count += 1 + self.logger.warn( + f"Failed with error: {step_result.stderr}.\nRetry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..." + ) + await anyio.sleep(self.retry_delay.total_seconds()) + return await self.run(*args, **kwargs) + + def log_step_result(self, result: StepResult) -> None: + """Log the step result. + + Args: + result (StepResult): The step result to log. + """ + duration = format_duration(self.run_duration) + if result.status is StepStatus.FAILURE: + self.logger.info(f"{result.status.get_emoji()} failed (duration: {duration})") + if result.status is StepStatus.SKIPPED: + self.logger.info(f"{result.status.get_emoji()} was skipped (duration: {duration})") + if result.status is StepStatus.SUCCESS: + self.logger.info(f"{result.status.get_emoji()} was successful (duration: {duration})") + + @abstractmethod + async def _run(self, *args, **kwargs) -> StepResult: + """Implement the execution of the step and return a step result. + + Returns: + StepResult: The result of the step run. + """ + ... + + def skip(self, reason: str = None) -> StepResult: + """Declare a step as skipped. + + Args: + reason (str, optional): Reason why the step was skipped. + + Returns: + StepResult: A skipped step result. + """ + return StepResult(self, StepStatus.SKIPPED, stdout=reason) + + def get_step_status_from_exit_code( + self, + exit_code: int, + ) -> StepStatus: + """Map an exit code to a step status. + + Args: + exit_code (int): A process exit code. + + Raises: + ValueError: Raised if the exit code is not mapped to a step status. + + Returns: + StepStatus: The step status inferred from the exit code. + """ + if exit_code == self.success_exit_code: + return StepStatus.SUCCESS + elif self.skipped_exit_code is not None and exit_code == self.skipped_exit_code: + return StepStatus.SKIPPED + else: + return StepStatus.FAILURE + + async def get_step_result(self, container: Container) -> StepResult: + """Concurrent retrieval of exit code, stdout and stdout of a container. + + Create a StepResult object from these objects. + + Args: + container (Container): The container from which we want to infer a step result/ + + Returns: + StepResult: Failure or success with stdout and stderr. + """ + exit_code, stdout, stderr = await get_exec_result(container) + return StepResult( + self, + self.get_step_status_from_exit_code(exit_code), + stderr=stderr, + stdout=stdout, + output_artifact=container, + ) + + def _get_timed_out_step_result(self) -> StepResult: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Timed out after the max duration of {format_duration(self.max_duration)}. Please checkout the Dagger logs to see what happened.", + ) + + +class PytestStep(Step, ABC): + """An abstract class to run pytest tests and evaluate success or failure according to pytest logs.""" + + skipped_exit_code = 5 + + async def _run_tests_in_directory(self, connector_under_test: Container, test_directory: str) -> StepResult: + """Run the pytest tests in the test_directory that was passed. + + A StepStatus.SKIPPED is returned if no tests were discovered. + + Args: + connector_under_test (Container): The connector under test container. + test_directory (str): The directory in which the python test modules are declared + + Returns: + Tuple[StepStatus, Optional[str], Optional[str]]: Tuple of StepStatus, stderr and stdout. + """ + test_config = "pytest.ini" if await check_path_in_workdir(connector_under_test, "pytest.ini") else "/" + PYPROJECT_TOML_FILE_PATH + if await check_path_in_workdir(connector_under_test, test_directory): + tester = connector_under_test.with_exec( + [ + "python", + "-m", + "pytest", + "-s", + test_directory, + "-c", + test_config, + ] + ) + return await self.get_step_result(tester) + + else: + return StepResult(self, StepStatus.SKIPPED) + + +class NoOpStep(Step): + """A step that does nothing.""" + + title = "No Op" + should_log = False + + def __init__(self, context: PipelineContext, step_status: StepStatus) -> None: + super().__init__(context) + self.step_status = step_status + + async def _run(self, *args, **kwargs) -> StepResult: + return StepResult(self, self.step_status) + + +@dataclass(frozen=True) +class StepResult: + """A dataclass to capture the result of a step.""" + + step: Step + status: StepStatus + created_at: datetime = field(default_factory=datetime.utcnow) + stderr: Optional[str] = None + stdout: Optional[str] = None + output_artifact: Any = None + exc_info: Optional[Exception] = None + + def __repr__(self) -> str: # noqa D105 + return f"{self.step.title}: {self.status.value}" + + def __str__(self) -> str: # noqa D105 + return f"{self.step.title}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" + + def __post_init__(self): + if self.stderr: + super().__setattr__("stderr", self.redact_secrets_from_string(self.stderr)) + if self.stdout: + super().__setattr__("stdout", self.redact_secrets_from_string(self.stdout)) + + def redact_secrets_from_string(self, value: str) -> str: + for secret in self.step.context.secrets_to_mask: + value = value.replace(secret, "********") + return value + + +@dataclass(frozen=True) +class Report: + """A dataclass to build reports to share pipelines executions results with the user.""" + + pipeline_context: PipelineContext + steps_results: List[StepResult] + created_at: datetime = field(default_factory=datetime.utcnow) + name: str = "REPORT" + filename: str = "output" + + @property + def report_output_prefix(self) -> str: # noqa D102 + return self.pipeline_context.report_output_prefix + + @property + def json_report_file_name(self) -> str: # noqa D102 + return self.filename + ".json" + + @property + def json_report_remote_storage_key(self) -> str: # noqa D102 + return f"{self.report_output_prefix}/{self.json_report_file_name}" + + @property + def failed_steps(self) -> List[StepResult]: # noqa D102 + return [step_result for step_result in self.steps_results if step_result.status is StepStatus.FAILURE] + + @property + def successful_steps(self) -> List[StepResult]: # noqa D102 + return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SUCCESS] + + @property + def skipped_steps(self) -> List[StepResult]: # noqa D102 + return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SKIPPED] + + @property + def success(self) -> bool: # noqa D102 + return len(self.failed_steps) == 0 and (len(self.skipped_steps) > 0 or len(self.successful_steps) > 0) + + @property + def run_duration(self) -> timedelta: # noqa D102 + return self.pipeline_context.stopped_at - self.pipeline_context.started_at + + @property + def lead_duration(self) -> timedelta: # noqa D102 + return self.pipeline_context.stopped_at - self.pipeline_context.created_at + + @property + def remote_storage_enabled(self) -> bool: # noqa D102 + return self.pipeline_context.is_ci + + async def save_local(self, filename: str, content: str) -> Path: + """Save the report files locally.""" + local_path = anyio.Path(f"{LOCAL_REPORTS_PATH_ROOT}/{self.report_output_prefix}/{filename}") + await local_path.parents[0].mkdir(parents=True, exist_ok=True) + await local_path.write_text(content) + return local_path + + async def save_remote(self, local_path: Path, remote_key: str, content_type: str = None) -> int: + gcs_cp_flags = None if content_type is None else [f"--content-type={content_type}"] + local_file = self.pipeline_context.dagger_client.host().directory(".", include=[str(local_path)]).file(str(local_path)) + report_upload_exit_code, _, _ = await remote_storage.upload_to_gcs( + dagger_client=self.pipeline_context.dagger_client, + file_to_upload=local_file, + key=remote_key, + bucket=self.pipeline_context.ci_report_bucket, + gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, + flags=gcs_cp_flags, + ) + gcs_uri = "gs://" + self.pipeline_context.ci_report_bucket + "/" + remote_key + public_url = f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{remote_key}" + if report_upload_exit_code != 0: + self.pipeline_context.logger.error(f"Uploading {local_path} to {gcs_uri} failed.") + else: + self.pipeline_context.logger.info(f"Uploading {local_path} to {gcs_uri} succeeded. Public URL: {public_url}") + return report_upload_exit_code + + async def save(self) -> None: + """Save the report files.""" + local_json_path = await self.save_local(self.json_report_file_name, self.to_json()) + absolute_path = await local_json_path.absolute() + self.pipeline_context.logger.info(f"Report saved locally at {absolute_path}") + if self.remote_storage_enabled: + await self.save_remote(local_json_path, self.json_report_remote_storage_key, "application/json") + + def to_json(self) -> str: + """Create a JSON representation of the report. + + Returns: + str: The JSON representation of the report. + """ + return json.dumps( + { + "pipeline_name": self.pipeline_context.pipeline_name, + "run_timestamp": self.pipeline_context.started_at.isoformat(), + "run_duration": self.run_duration.total_seconds(), + "success": self.success, + "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], + "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], + "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], + "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, + "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, + "pipeline_end_timestamp": round(self.pipeline_context.stopped_at.timestamp()), + "pipeline_duration": round(self.pipeline_context.stopped_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp, + "git_branch": self.pipeline_context.git_branch, + "git_revision": self.pipeline_context.git_revision, + "ci_context": self.pipeline_context.ci_context, + "pull_request_url": self.pipeline_context.pull_request.html_url if self.pipeline_context.pull_request else None, + "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, + } + ) + + def print(self): + """Print the test report to the console in a nice way.""" + pipeline_name = self.pipeline_context.pipeline_name + main_panel_title = Text(f"{pipeline_name.upper()} - {self.name}") + main_panel_title.stylize(Style(color="blue", bold=True)) + duration_subtitle = Text(f"⏲️ Total pipeline duration for {pipeline_name}: {format_duration(self.run_duration)}") + step_results_table = Table(title="Steps results") + step_results_table.add_column("Step") + step_results_table.add_column("Result") + step_results_table.add_column("Finished after") + + for step_result in self.steps_results: + step = Text(step_result.step.title) + step.stylize(step_result.status.get_rich_style()) + result = Text(step_result.status.value) + result.stylize(step_result.status.get_rich_style()) + + if step_result.status is StepStatus.SKIPPED: + step_results_table.add_row(step, result, "N/A") + else: + run_time = format_duration((step_result.created_at - step_result.step.started_at)) + step_results_table.add_row(step, result, run_time) + + to_render = [step_results_table] + if self.failed_steps: + sub_panels = [] + for failed_step in self.failed_steps: + errors = Text(failed_step.stderr) + panel_title = Text(f"{pipeline_name} {failed_step.step.title.lower()} failures") + panel_title.stylize(Style(color="red", bold=True)) + sub_panel = Panel(errors, title=panel_title) + sub_panels.append(sub_panel) + failures_group = Group(*sub_panels) + to_render.append(failures_group) + + if self.pipeline_context.dagger_cloud_url: + self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") + + main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) + console.print(main_panel) + + +@dataclass(frozen=True) +class ConnectorReport(Report): + """A dataclass to build connector test reports to share pipelines executions results with the user.""" + + @property + def report_output_prefix(self) -> str: # noqa D102 + return f"{self.pipeline_context.report_output_prefix}/{self.pipeline_context.connector.technical_name}/{self.pipeline_context.connector.version}" + + @property + def html_report_file_name(self) -> str: # noqa D102 + return self.filename + ".html" + + @property + def html_report_remote_storage_key(self) -> str: # noqa D102 + return f"{self.report_output_prefix}/{self.html_report_file_name}" + + @property + def html_report_url(self) -> str: # noqa D102 + return f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{self.html_report_remote_storage_key}" + + @property + def should_be_commented_on_pr(self) -> bool: # noqa D102 + return ( + self.pipeline_context.should_save_report + and self.pipeline_context.is_ci + and self.pipeline_context.pull_request + and self.pipeline_context.PRODUCTION + ) + + def to_json(self) -> str: + """Create a JSON representation of the connector test report. + + Returns: + str: The JSON representation of the report. + """ + return json.dumps( + { + "connector_technical_name": self.pipeline_context.connector.technical_name, + "connector_version": self.pipeline_context.connector.version, + "run_timestamp": self.created_at.isoformat(), + "run_duration": self.run_duration.total_seconds(), + "success": self.success, + "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], + "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], + "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], + "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, + "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, + "pipeline_end_timestamp": round(self.created_at.timestamp()), + "pipeline_duration": round(self.created_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp, + "git_branch": self.pipeline_context.git_branch, + "git_revision": self.pipeline_context.git_revision, + "ci_context": self.pipeline_context.ci_context, + "cdk_version": self.pipeline_context.cdk_version, + "html_report_url": self.html_report_url, + "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, + } + ) + + def post_comment_on_pr(self) -> None: + icon_url = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + global_status_emoji = "✅" if self.success else "❌" + commit_url = f"{self.pipeline_context.pull_request.html_url}/commits/{self.pipeline_context.git_revision}" + markdown_comment = f'## {self.pipeline_context.connector.technical_name} test report (commit [`{self.pipeline_context.git_revision[:10]}`]({commit_url})) - {global_status_emoji}\n\n' + markdown_comment += f"⏲️ Total pipeline duration: {format_duration(self.run_duration)} \n\n" + report_data = [ + [step_result.step.title, step_result.status.get_emoji()] + for step_result in self.steps_results + if step_result.status is not StepStatus.SKIPPED + ] + markdown_comment += tabulate(report_data, headers=["Step", "Result"], tablefmt="pipe") + "\n\n" + markdown_comment += f"🔗 [View the logs here]({self.html_report_url})\n\n" + + if self.pipeline_context.dagger_cloud_url: + markdown_comment += f"☁️ [View runs for commit in Dagger Cloud]({self.pipeline_context.dagger_cloud_url})\n\n" + + markdown_comment += "*Please note that tests are only run on PR ready for review. Please set your PR to draft mode to not flood the CI engine and upstream service on following commits.*\n" + markdown_comment += "**You can run the same pipeline locally on this branch with the [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool with the following command**\n" + markdown_comment += f"```bash\nairbyte-ci connectors --name={self.pipeline_context.connector.technical_name} test\n```\n\n" + self.pipeline_context.pull_request.create_issue_comment(markdown_comment) + + async def to_html(self) -> str: + env = Environment(loader=PackageLoader("pipelines.tests"), autoescape=select_autoescape(), trim_blocks=False, lstrip_blocks=True) + template = env.get_template("test_report.html.j2") + template.globals["StepStatus"] = StepStatus + template.globals["format_duration"] = format_duration + local_icon_path = await Path(f"{self.pipeline_context.connector.code_directory}/icon.svg").resolve() + template_context = { + "connector_name": self.pipeline_context.connector.technical_name, + "step_results": self.steps_results, + "run_duration": self.run_duration, + "created_at": self.created_at.isoformat(), + "connector_version": self.pipeline_context.connector.version, + "gha_workflow_run_url": None, + "dagger_logs_url": None, + "git_branch": self.pipeline_context.git_branch, + "git_revision": self.pipeline_context.git_revision, + "commit_url": None, + "icon_url": local_icon_path.as_uri(), + } + + if self.pipeline_context.is_ci: + template_context["commit_url"] = f"https://github.com/airbytehq/airbyte/commit/{self.pipeline_context.git_revision}" + template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url + template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url + template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url + template_context[ + "icon_url" + ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + return template.render(template_context) + + async def save(self) -> None: + local_html_path = await self.save_local(self.html_report_file_name, await self.to_html()) + absolute_path = await local_html_path.resolve() + if self.pipeline_context.is_local: + self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}") + self.pipeline_context.logger.info("Opening HTML report in browser.") + webbrowser.open(absolute_path.as_uri()) + if self.remote_storage_enabled: + await self.save_remote(local_html_path, self.html_report_remote_storage_key, "text/html") + self.pipeline_context.logger.info(f"HTML report uploaded to {self.html_report_url}") + await super().save() + + def print(self): + """Print the test report to the console in a nice way.""" + connector_name = self.pipeline_context.connector.technical_name + main_panel_title = Text(f"{connector_name.upper()} - {self.name}") + main_panel_title.stylize(Style(color="blue", bold=True)) + duration_subtitle = Text(f"⏲️ Total pipeline duration for {connector_name}: {format_duration(self.run_duration)}") + step_results_table = Table(title="Steps results") + step_results_table.add_column("Step") + step_results_table.add_column("Result") + step_results_table.add_column("Duration") + + for step_result in self.steps_results: + step = Text(step_result.step.title) + step.stylize(step_result.status.get_rich_style()) + result = Text(step_result.status.value) + result.stylize(step_result.status.get_rich_style()) + step_results_table.add_row(step, result, format_duration(step_result.step.run_duration)) + + details_instructions = Text("ℹ️ You can find more details with step executions logs in the saved HTML report.") + to_render = [step_results_table, details_instructions] + + if self.pipeline_context.dagger_cloud_url: + self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") + + main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) + console.print(main_panel) diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py new file mode 100644 index 000000000000..3d92eabf6234 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""This module groups factory like functions to dispatch builds steps according to the connector language.""" + +from __future__ import annotations + +import platform + +import anyio +from connector_ops.utils import ConnectorLanguage +from dagger import Platform +from pipelines.bases import ConnectorReport, StepResult +from pipelines.builds import common, java_connectors, python_connectors +from pipelines.contexts import ConnectorContext + + +class NoBuildStepForLanguageError(Exception): + pass + + +LANGUAGE_BUILD_CONNECTOR_MAPPING = { + ConnectorLanguage.PYTHON: python_connectors.run_connector_build, + ConnectorLanguage.LOW_CODE: python_connectors.run_connector_build, + ConnectorLanguage.JAVA: java_connectors.run_connector_build, +} + +BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] +LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}") + + +async def run_connector_build(context: ConnectorContext) -> StepResult: + """Run a build pipeline for a single connector.""" + if context.connector.language not in LANGUAGE_BUILD_CONNECTOR_MAPPING: + raise NoBuildStepForLanguageError(f"No build step for connector language {context.connector.language}.") + return await LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language](context) + + +async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: + """Run a build pipeline for a single connector. + + Args: + context (ConnectorContext): The initialized connector context. + + Returns: + ConnectorReport: The reports holding builds results. + """ + step_results = [] + async with semaphore: + async with context: + build_result = await run_connector_build(context) + step_results.append(build_result) + if context.is_local and build_result.status is common.StepStatus.SUCCESS: + connector_to_load_to_local_docker_host = build_result.output_artifact[LOCAL_BUILD_PLATFORM] + load_image_result = await common.LoadContainerToLocalDockerHost(context, connector_to_load_to_local_docker_host).run() + step_results.append(load_image_result) + context.report = ConnectorReport(context, step_results, name="BUILD RESULTS") + return context.report diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/common.py b/airbyte-ci/connectors/pipelines/pipelines/builds/common.py new file mode 100644 index 000000000000..d8de2bffc4a1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/common.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Tuple + +import docker +from dagger import Container, Platform +from pipelines.bases import Step, StepResult, StepStatus +from pipelines.consts import BUILD_PLATFORMS +from pipelines.contexts import ConnectorContext +from pipelines.utils import export_container_to_tarball + + +class BuildConnectorImageBase(Step, ABC): + @property + def title(self): + return f"Build {self.context.connector.technical_name} docker image for platform {self.build_platform}" + + def __init__(self, context: ConnectorContext, build_platform: Platform) -> None: + self.build_platform = build_platform + super().__init__(context) + + +class BuildConnectorImageForAllPlatformsBase(Step, ABC): + + ALL_PLATFORMS = BUILD_PLATFORMS + + title = f"Build connector image for {BUILD_PLATFORMS}" + + def get_success_result(self, build_results_per_platform: dict[Platform, Container]) -> StepResult: + return StepResult( + self, + StepStatus.SUCCESS, + stdout="The connector image was successfully built for all platforms.", + output_artifact=build_results_per_platform, + ) + + +class LoadContainerToLocalDockerHost(Step): + IMAGE_TAG = "dev" + + def __init__(self, context: ConnectorContext, container: Container) -> None: + super().__init__(context) + self.container = container + + @property + def title(self): + return f"Load {self.image_name}:{self.IMAGE_TAG} to the local docker host." + + @property + def image_name(self) -> Tuple: + return f"airbyte/{self.context.connector.technical_name}" + + async def _run(self) -> StepResult: + _, exported_tarball_path = await export_container_to_tarball(self.context, self.container) + client = docker.from_env() + try: + with open(exported_tarball_path, "rb") as tarball_content: + new_image = client.images.load(tarball_content.read())[0] + new_image.tag(self.image_name, tag=self.IMAGE_TAG) + return StepResult(self, StepStatus.SUCCESS) + except ConnectionError: + return StepResult(self, StepStatus.FAILURE, stderr="The connection to the local docker host failed.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py new file mode 100644 index 000000000000..ffd6ff1e8bd4 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py @@ -0,0 +1,88 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import ExecError, File, QueryError +from pipelines.actions import environments +from pipelines.bases import StepResult, StepStatus +from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase +from pipelines.contexts import ConnectorContext +from pipelines.gradle import GradleTask + + +class BuildConnectorDistributionTar(GradleTask): + title = "Build connector tar" + gradle_task_name = "distTar" + + async def _run(self) -> StepResult: + cdk_includes = ["./airbyte-cdk/java/airbyte-cdk/**"] + with_built_tar = ( + environments.with_gradle( + self.context, + self.build_include + cdk_includes, + ) + .with_exec(["./gradlew", ":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"]) + .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) + .with_exec(self._get_gradle_command()) + .with_workdir(f"{self.context.connector.code_directory}/build/distributions") + ) + distributions = await with_built_tar.directory(".").entries() + tar_files = [f for f in distributions if f.endswith(".tar")] + await self._export_gradle_dependency_cache(with_built_tar) + if len(tar_files) == 1: + return StepResult( + self, + StepStatus.SUCCESS, + stdout="The tar file for the current connector was successfully built.", + output_artifact=with_built_tar.file(tar_files[0]), + ) + else: + return StepResult( + self, + StepStatus.FAILURE, + stderr="The distributions directory contains multiple connector tar files. We can't infer which one should be used. Please review and delete any unnecessary tar files.", + ) + + +class BuildConnectorImage(BuildConnectorImageBase): + """ + A step to build a Java connector image using the distTar Gradle task. + """ + + async def _run(self, distribution_tar: File) -> StepResult: + try: + java_connector = await environments.with_airbyte_java_connector(self.context, distribution_tar, self.build_platform) + try: + await java_connector.with_exec(["spec"]) + except ExecError: + return StepResult( + self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {self.build_platform}." + ) + return StepResult( + self, StepStatus.SUCCESS, stdout="The connector image was successfully built.", output_artifact=java_connector + ) + except QueryError as e: + return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + + +class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase): + """Build a Java connector image for all platforms.""" + + async def _run(self, distribution_tar: File) -> StepResult: + build_results_per_platform = {} + for platform in self.ALL_PLATFORMS: + build_connector_step_result = await BuildConnectorImage(self.context, platform).run(distribution_tar) + if build_connector_step_result.status is not StepStatus.SUCCESS: + return build_connector_step_result + build_results_per_platform[platform] = build_connector_step_result.output_artifact + return self.get_success_result(build_results_per_platform) + + +async def run_connector_build(context: ConnectorContext) -> StepResult: + """Create the java connector distribution tar file and build the connector image.""" + + build_connector_tar_result = await BuildConnectorDistributionTar(context).run() + if build_connector_tar_result.status is not StepStatus.SUCCESS: + return build_connector_tar_result + + return await BuildConnectorImageForAllPlatforms(context).run(build_connector_tar_result.output_artifact) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py similarity index 80% rename from tools/ci_connector_ops/ci_connector_ops/pipelines/builds/normalization.py rename to airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py index 85cc61cf79e9..3494086eee8c 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from ci_connector_ops.pipelines.actions import environments -from ci_connector_ops.pipelines.bases import Step, StepResult, StepStatus -from ci_connector_ops.pipelines.contexts import ConnectorContext +from dagger import Platform +from pipelines.actions import environments +from pipelines.bases import Step, StepResult, StepStatus +from pipelines.contexts import ConnectorContext # TODO this class could be deleted @@ -12,7 +13,7 @@ class BuildOrPullNormalization(Step): """A step to build or pull the normalization image for a connector according to the image name.""" - def __init__(self, context: ConnectorContext, normalization_image: str) -> None: + def __init__(self, context: ConnectorContext, normalization_image: str, build_platform: Platform) -> None: """Initialize the step to build or pull the normalization image. Args: @@ -20,13 +21,14 @@ def __init__(self, context: ConnectorContext, normalization_image: str) -> None: normalization_image (str): The normalization image to build (if :dev) or pull. """ super().__init__(context) + self.build_platform = build_platform self.use_dev_normalization = normalization_image.endswith(":dev") self.normalization_image = normalization_image self.title = f"Build {self.normalization_image}" if self.use_dev_normalization else f"Pull {self.normalization_image}" async def _run(self) -> StepResult: if self.use_dev_normalization: - build_normalization_container = environments.with_normalization(self.context) + build_normalization_container = environments.with_normalization(self.context, self.build_platform) else: build_normalization_container = self.context.dagger_client.container().from_(self.normalization_image) return StepResult(self, StepStatus.SUCCESS, output_artifact=build_normalization_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py new file mode 100644 index 000000000000..d18dc9537d8d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import QueryError +from pipelines.actions.environments import with_airbyte_python_connector +from pipelines.bases import StepResult, StepStatus +from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase +from pipelines.contexts import ConnectorContext + + +class BuildConnectorImage(BuildConnectorImageBase): + """ + A step to build a Python connector image. + A spec command is run on the container to validate it was built successfully. + """ + + async def _run(self) -> StepResult: + connector = await with_airbyte_python_connector(self.context, self.build_platform) + try: + return await self.get_step_result(connector.with_exec(["spec"])) + except QueryError as e: + return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + + +class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase): + """Build a Python connector image for all platforms.""" + + async def _run(self) -> StepResult: + build_results_per_platform = {} + for platform in self.ALL_PLATFORMS: + build_connector_step_result = await BuildConnectorImage(self.context, platform).run() + if build_connector_step_result.status is not StepStatus.SUCCESS: + return build_connector_step_result + build_results_per_platform[platform] = build_connector_step_result.output_artifact + return self.get_success_result(build_results_per_platform) + + +async def run_connector_build(context: ConnectorContext) -> StepResult: + return await BuildConnectorImageForAllPlatforms(context).run() diff --git a/airbyte-integrations/connectors/source-coda/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/commands/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-coda/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/commands/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py new file mode 100644 index 000000000000..3ff8a0531c98 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py @@ -0,0 +1,146 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module is the CLI entrypoint to the airbyte-ci commands.""" + +from typing import List + +import click +from github import PullRequest +from pipelines import github, main_logger +from pipelines.bases import CIContext +from pipelines.utils import ( + get_current_epoch_time, + get_current_git_branch, + get_current_git_revision, + get_modified_files_in_branch, + get_modified_files_in_commit, + get_modified_files_in_pull_request, + transform_strs_to_paths, +) + +from .groups.connectors import connectors +from .groups.metadata import metadata +from .groups.tests import test + +# HELPERS + + +def get_modified_files( + git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, pull_request: PullRequest +) -> List[str]: + """Get the list of modified files in the current git branch. + If the current branch is master, it will return the list of modified files in the head commit. + The head commit on master should be the merge commit of the latest merged pull request as we squash commits on merge. + Pipelines like "publish on merge" are triggered on each new commit on master. + + If the CI context is a pull request, it will return the list of modified files in the pull request, without using git diff. + If the current branch is not master, it will return the list of modified files in the current branch. + This latest case is the one we encounter when running the pipeline locally, on a local branch, or manually on GHA with a workflow dispatch event. + """ + if ci_context is CIContext.MASTER or ci_context is CIContext.NIGHTLY_BUILDS: + return get_modified_files_in_commit(git_branch, git_revision, is_local) + if ci_context is CIContext.PULL_REQUEST and pull_request is not None: + return get_modified_files_in_pull_request(pull_request) + if ci_context is CIContext.MANUAL: + if git_branch == "master": + return get_modified_files_in_commit(git_branch, git_revision, is_local) + else: + return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) + return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) + + +# COMMANDS + + +@click.group(help="Airbyte CI top-level command group.") +@click.option("--is-local/--is-ci", default=True) +@click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH") +@click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION") +@click.option( + "--diffed-branch", + help="Branch to which the git diff will happen to detect new or modified connectors", + default="origin/master", + type=str, +) +@click.option("--gha-workflow-run-id", help="[CI Only] The run id of the GitHub action workflow", default=None, type=str) +@click.option("--ci-context", default=CIContext.MANUAL, envvar="CI_CONTEXT", type=click.Choice(CIContext)) +@click.option("--pipeline-start-timestamp", default=get_current_epoch_time, envvar="CI_PIPELINE_START_TIMESTAMP", type=int) +@click.option("--pull-request-number", envvar="PULL_REQUEST_NUMBER", type=int) +@click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str) +@click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str) +@click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str) +@click.option( + "--ci-gcs-credentials", + help="The service account to use during CI.", + type=click.STRING, + required=False, # Not required for pre-release or local pipelines + envvar="GCP_GSM_CREDENTIALS", +) +@click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) +@click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) +@click.pass_context +def airbyte_ci( + ctx: click.Context, + is_local: bool, + git_branch: str, + git_revision: str, + diffed_branch: str, + gha_workflow_run_id: str, + ci_context: str, + pipeline_start_timestamp: int, + pull_request_number: int, + ci_git_user: str, + ci_github_access_token: str, + ci_report_bucket_name: str, + ci_gcs_credentials: str, + ci_job_key: str, + show_dagger_logs: bool, +): # noqa D103 + ctx.ensure_object(dict) + ctx.obj["is_local"] = is_local + ctx.obj["is_ci"] = not is_local + ctx.obj["git_branch"] = git_branch + ctx.obj["git_revision"] = git_revision + ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id + ctx.obj["gha_workflow_run_url"] = ( + f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None + ) + ctx.obj["ci_context"] = ci_context + ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name + ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials + ctx.obj["ci_git_user"] = ci_git_user + ctx.obj["ci_github_access_token"] = ci_github_access_token + ctx.obj["ci_job_key"] = ci_job_key + ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp + ctx.obj["show_dagger_logs"] = show_dagger_logs + + if pull_request_number and ci_github_access_token: + ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) + else: + ctx.obj["pull_request"] = None + + ctx.obj["modified_files"] = transform_strs_to_paths( + get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"]) + ) + + if not is_local: + main_logger.info("Running airbyte-ci in CI mode.") + main_logger.info(f"CI Context: {ci_context}") + main_logger.info(f"CI Report Bucket Name: {ci_report_bucket_name}") + main_logger.info(f"Git Branch: {git_branch}") + main_logger.info(f"Git Revision: {git_revision}") + main_logger.info(f"GitHub Workflow Run ID: {gha_workflow_run_id}") + main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") + main_logger.info(f"Pull Request Number: {pull_request_number}") + main_logger.info(f"Pipeline Start Timestamp: {pipeline_start_timestamp}") + main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") + + +airbyte_ci.add_command(connectors) +airbyte_ci.add_command(metadata) +airbyte_ci.add_command(test) + +if __name__ == "__main__": + airbyte_ci() diff --git a/airbyte-integrations/connectors/source-fastbill/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-fastbill/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/commands/groups/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py new file mode 100644 index 000000000000..1fe82c244cff --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py @@ -0,0 +1,505 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module declares the CLI commands to run the connectors CI pipelines.""" + +import os +import sys +from pathlib import Path +from typing import List, Set, Tuple + +import anyio +import click +from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, console, get_all_connectors_in_repo +from pipelines import main_logger +from pipelines.bases import ConnectorWithModifiedFiles +from pipelines.builds import run_connector_build_pipeline +from pipelines.contexts import ConnectorContext, ContextState, PublishConnectorContext +from pipelines.format import run_connectors_format_pipelines +from pipelines.github import update_global_commit_status_check_for_tests +from pipelines.pipelines.connectors import run_connectors_pipelines +from pipelines.publish import reorder_contexts, run_connector_publish_pipeline +from pipelines.tests import run_connector_test_pipeline +from pipelines.utils import DaggerPipelineCommand, get_connector_modified_files, get_modified_connectors + +# HELPERS + +ALL_CONNECTORS = get_all_connectors_in_repo() + + +def validate_environment(is_local: bool, use_remote_secrets: bool): + """Check if the required environment variables exist.""" + if is_local: + if not (os.getcwd().endswith("/airbyte") and Path(".git").is_dir()): + raise click.UsageError("You need to run this command from the airbyte repository root.") + else: + required_env_vars_for_ci = [ + "GCP_GSM_CREDENTIALS", + "CI_REPORT_BUCKET_NAME", + "CI_GITHUB_ACCESS_TOKEN", + ] + for required_env_var in required_env_vars_for_ci: + if os.getenv(required_env_var) is None: + raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.") + if use_remote_secrets and os.getenv("GCP_GSM_CREDENTIALS") is None: + raise click.UsageError( + "You have to set the GCP_GSM_CREDENTIALS if you want to download secrets from GSM. Set the --use-remote-secrets option to false otherwise." + ) + + +def get_selected_connectors_with_modified_files( + selected_names: Tuple[str], + selected_support_levels: Tuple[str], + selected_languages: Tuple[str], + modified: bool, + metadata_changes_only: bool, + modified_files: Set[Path], + enable_dependency_scanning: bool = False, +) -> List[ConnectorWithModifiedFiles]: + """Get the connectors that match the selected criteria. + + Args: + selected_names (Tuple[str]): Selected connector names. + selected_support_levels (Tuple[str]): Selected connector support levels. + selected_languages (Tuple[str]): Selected connector languages. + modified (bool): Whether to select the modified connectors. + metadata_changes_only (bool): Whether to select only the connectors with metadata changes. + modified_files (Set[Path]): The modified files. + enable_dependency_scanning (bool): Whether to enable the dependency scanning. + Returns: + List[ConnectorWithModifiedFiles]: The connectors that match the selected criteria. + """ + + if metadata_changes_only and not modified: + main_logger.info("--metadata-changes-only overrides --modified") + modified = True + + selected_modified_connectors = ( + get_modified_connectors(modified_files, ALL_CONNECTORS, enable_dependency_scanning) if modified else set() + ) + selected_connectors_by_name = {c for c in ALL_CONNECTORS if c.technical_name in selected_names} + selected_connectors_by_support_level = {connector for connector in ALL_CONNECTORS if connector.support_level in selected_support_levels} + selected_connectors_by_language = {connector for connector in ALL_CONNECTORS if connector.language in selected_languages} + non_empty_connector_sets = [ + connector_set + for connector_set in [ + selected_connectors_by_name, + selected_connectors_by_support_level, + selected_connectors_by_language, + selected_modified_connectors, + ] + if connector_set + ] + # The selected connectors are the intersection of the selected connectors by name, support_level, language and modified. + selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set() + + selected_connectors_with_modified_files = [] + for connector in selected_connectors: + connector_with_modified_files = ConnectorWithModifiedFiles( + technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files) + ) + if not metadata_changes_only: + selected_connectors_with_modified_files.append(connector_with_modified_files) + else: + if connector_with_modified_files.has_metadata_change: + selected_connectors_with_modified_files.append(connector_with_modified_files) + return selected_connectors_with_modified_files + + +# COMMANDS + + +@click.group(help="Commands related to connectors and connector acceptance tests.") +@click.option("--use-remote-secrets", default=True) # specific to connectors +@click.option( + "--name", + "names", + multiple=True, + help="Only test a specific connector. Use its technical name. e.g source-pokeapi.", + type=click.Choice([c.technical_name for c in ALL_CONNECTORS]), +) +@click.option("--language", "languages", multiple=True, help="Filter connectors to test by language.", type=click.Choice(ConnectorLanguage)) +@click.option( + "--support-level", + "support_levels", + multiple=True, + help="Filter connectors to test by support_level.", + type=click.Choice(SupportLevelEnum), +) +@click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool) +@click.option( + "--metadata-changes-only/--not-metadata-changes-only", + help="Only test connectors with modified metadata files in the current branch.", + default=False, + type=bool, +) +@click.option("--concurrency", help="Number of connector tests pipeline to run in parallel.", default=5, type=int) +@click.option( + "--execute-timeout", + help="The maximum time in seconds for the execution of a Dagger request before an ExecuteTimeoutError is raised. Passing None results in waiting forever.", + default=None, + type=int, +) +@click.option( + "--enable-dependency-scanning/--disable-dependency-scanning", + help="When enabled, the dependency scanning will be performed to detect the connectors to test according to a dependency change.", + default=False, + type=bool, +) +@click.pass_context +def connectors( + ctx: click.Context, + use_remote_secrets: bool, + names: Tuple[str], + languages: Tuple[ConnectorLanguage], + support_levels: Tuple[str], + modified: bool, + metadata_changes_only: bool, + concurrency: int, + execute_timeout: int, + enable_dependency_scanning: bool, +): + """Group all the connectors-ci command.""" + validate_environment(ctx.obj["is_local"], use_remote_secrets) + + ctx.ensure_object(dict) + ctx.obj["use_remote_secrets"] = use_remote_secrets + ctx.obj["concurrency"] = concurrency + ctx.obj["execute_timeout"] = execute_timeout + ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( + names, support_levels, languages, modified, metadata_changes_only, ctx.obj["modified_files"], enable_dependency_scanning + ) + log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) + + +@connectors.command(cls=DaggerPipelineCommand, help="Test all the selected connectors.") +@click.option( + "--code-tests-only", + is_flag=True, + help=("Only execute code tests. " "Metadata checks, QA, and acceptance tests will be skipped."), + default=False, + type=bool, +) +@click.option( + "--fail-fast", + help="When enabled, tests will fail fast.", + default=False, + type=bool, + is_flag=True, +) +@click.option( + "--fast-tests-only", + help="When enabled, slow tests are skipped.", + default=False, + type=bool, + is_flag=True, +) +@click.pass_context +def test( + ctx: click.Context, + code_tests_only: bool, + fail_fast: bool, + fast_tests_only: bool, +) -> bool: + """Runs a test pipeline for the selected connectors. + + Args: + ctx (click.Context): The click context. + """ + if ctx.obj["is_ci"] and ctx.obj["pull_request"] and ctx.obj["pull_request"].draft: + main_logger.info("Skipping connectors tests for draft pull request.") + sys.exit(0) + + if ctx.obj["selected_connectors_with_modified_files"]: + update_global_commit_status_check_for_tests(ctx.obj, "pending") + else: + main_logger.warn("No connector were selected for testing.") + update_global_commit_status_check_for_tests(ctx.obj, "success") + return True + + connectors_tests_contexts = [ + ConnectorContext( + pipeline_name=f"Testing connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + pull_request=ctx.obj.get("pull_request"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + fail_fast=fail_fast, + fast_tests_only=fast_tests_only, + code_tests_only=code_tests_only, + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + try: + anyio.run( + run_connectors_pipelines, + [connector_context for connector_context in connectors_tests_contexts], + run_connector_test_pipeline, + "Test Pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + ) + except Exception as e: + main_logger.error("An error occurred while running the test pipeline", exc_info=e) + update_global_commit_status_check_for_tests(ctx.obj, "failure") + return False + + @ctx.call_on_close + def send_commit_status_check() -> None: + if ctx.obj["is_ci"]: + global_success = all(connector_context.state is ContextState.SUCCESSFUL for connector_context in connectors_tests_contexts) + update_global_commit_status_check_for_tests(ctx.obj, "success" if global_success else "failure") + + # If we reach this point, it means that all the connectors have been tested so the pipeline did its job and can exit with success. + return True + + +@connectors.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.") +@click.pass_context +def build(ctx: click.Context) -> bool: + """Runs a build pipeline for the selected connectors.""" + + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Build connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + anyio.run( + run_connectors_pipelines, + connectors_contexts, + run_connector_build_pipeline, + "Build Pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + ) + + return True + + +@connectors.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.") +@click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool) +@click.option( + "--spec-cache-gcs-credentials", + help="The service account key to upload files to the GCS bucket hosting spec cache.", + type=click.STRING, + required=True, + envvar="SPEC_CACHE_GCS_CREDENTIALS", +) +@click.option( + "--spec-cache-bucket-name", + help="The name of the GCS bucket where specs will be cached.", + type=click.STRING, + required=True, + envvar="SPEC_CACHE_BUCKET_NAME", +) +@click.option( + "--metadata-service-gcs-credentials", + help="The service account key to upload files to the GCS bucket hosting the metadata files.", + type=click.STRING, + required=True, + envvar="METADATA_SERVICE_GCS_CREDENTIALS", +) +@click.option( + "--metadata-service-bucket-name", + help="The name of the GCS bucket where metadata files will be uploaded.", + type=click.STRING, + required=True, + envvar="METADATA_SERVICE_BUCKET_NAME", +) +@click.option( + "--docker-hub-username", + help="Your username to connect to DockerHub.", + type=click.STRING, + required=True, + envvar="DOCKER_HUB_USERNAME", +) +@click.option( + "--docker-hub-password", + help="Your password to connect to DockerHub.", + type=click.STRING, + required=True, + envvar="DOCKER_HUB_PASSWORD", +) +@click.option( + "--slack-webhook", + help="The Slack webhook URL to send notifications to.", + type=click.STRING, + envvar="SLACK_WEBHOOK", +) +@click.option( + "--slack-channel", + help="The Slack webhook URL to send notifications to.", + type=click.STRING, + envvar="SLACK_CHANNEL", + default="#publish-on-merge-updates", +) +@click.pass_context +def publish( + ctx: click.Context, + pre_release: bool, + spec_cache_gcs_credentials: str, + spec_cache_bucket_name: str, + metadata_service_bucket_name: str, + metadata_service_gcs_credentials: str, + docker_hub_username: str, + docker_hub_password: str, + slack_webhook: str, + slack_channel: str, +): + ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials + ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name + ctx.obj["metadata_service_bucket_name"] = metadata_service_bucket_name + ctx.obj["metadata_service_gcs_credentials"] = metadata_service_gcs_credentials + if ctx.obj["is_local"]: + click.confirm( + "Publishing from a local environment is not recommended and requires to be logged in Airbyte's DockerHub registry, do you want to continue?", + abort=True, + ) + + publish_connector_contexts = reorder_contexts( + [ + PublishConnectorContext( + connector=connector, + pre_release=pre_release, + spec_cache_gcs_credentials=spec_cache_gcs_credentials, + spec_cache_bucket_name=spec_cache_bucket_name, + metadata_service_gcs_credentials=metadata_service_gcs_credentials, + metadata_bucket_name=metadata_service_bucket_name, + docker_hub_username=docker_hub_username, + docker_hub_password=docker_hub_password, + slack_webhook=slack_webhook, + reporting_slack_channel=slack_channel, + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + pull_request=ctx.obj.get("pull_request"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + ) + + main_logger.warn("Concurrency is forced to 1. For stability reasons we disable parallel publish pipelines.") + ctx.obj["concurrency"] = 1 + + publish_connector_contexts = anyio.run( + run_connectors_pipelines, + publish_connector_contexts, + run_connector_publish_pipeline, + "Publishing connectors", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + ) + return all(context.state is ContextState.SUCCESSFUL for context in publish_connector_contexts) + + +@connectors.command(cls=DaggerPipelineCommand, help="List all selected connectors.") +@click.pass_context +def list( + ctx: click.Context, +): + selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) + table = Table(title=f"{len(selected_connectors)} selected connectors") + table.add_column("Modified") + table.add_column("Connector") + table.add_column("Language") + table.add_column("Release stage") + table.add_column("Version") + table.add_column("Folder") + + for connector in selected_connectors: + modified = "X" if connector.modified_files else "" + connector_name = Text(connector.technical_name) + language = Text(connector.language.value) if connector.language else "N/A" + try: + support_level = Text(connector.support_level) + except Exception: + support_level = "N/A" + try: + version = Text(connector.version) + except Exception: + version = "N/A" + folder = Text(str(connector.code_directory)) + table.add_row(modified, connector_name, language, support_level, version, folder) + + console.print(table) + return True + + +@connectors.command(name="format", cls=DaggerPipelineCommand, help="Autoformat connector code.") +@click.pass_context +def format_code(ctx: click.Context) -> bool: + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Format connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ci_git_user=ctx.obj["ci_git_user"], + ci_github_access_token=ctx.obj["ci_github_access_token"], + pull_request=ctx.obj.get("pull_request"), + should_save_report=False, + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + anyio.run( + run_connectors_format_pipelines, + connectors_contexts, + ctx.obj["ci_git_user"], + ctx.obj["ci_github_access_token"], + ctx.obj["git_branch"], + ctx.obj["is_local"], + ctx.obj["execute_timeout"], + ) + + return True + + +def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None: + if selected_connectors_with_modified_files: + selected_connectors_names = [c.technical_name for c in selected_connectors_with_modified_files] + main_logger.info(f"Will run on the following connectors: {', '.join(selected_connectors_names)}.") + else: + main_logger.info("No connectors to run.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py new file mode 100644 index 000000000000..a9988db89fe2 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import anyio +import click +from pipelines.bases import CIContext +from pipelines.pipelines.metadata import ( + run_metadata_lib_test_pipeline, + run_metadata_orchestrator_deploy_pipeline, + run_metadata_orchestrator_test_pipeline, + run_metadata_upload_pipeline, + run_metadata_validation_pipeline, +) +from pipelines.utils import DaggerPipelineCommand, get_all_metadata_files, get_expected_metadata_files, get_modified_metadata_files + +# MAIN GROUP + + +@click.group(help="Commands related to the metadata service.") +@click.pass_context +def metadata(ctx: click.Context): + pass + + +# VALIDATE COMMAND + + +@metadata.command(cls=DaggerPipelineCommand, help="Commands related to validating the metadata files.") +@click.option("--modified-only/--all", default=True) +@click.pass_context +def validate(ctx: click.Context, modified_only: bool) -> bool: + if modified_only: + metadata_to_validate = get_expected_metadata_files(ctx.obj["modified_files"]) + else: + click.secho("Will run metadata validation on all the metadata files found in the repo.") + metadata_to_validate = get_all_metadata_files() + + click.secho(f"Will validate {len(metadata_to_validate)} metadata files.") + + return anyio.run( + run_metadata_validation_pipeline, + ctx.obj["is_local"], + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj.get("gha_workflow_run_url"), + ctx.obj.get("pipeline_start_timestamp"), + ctx.obj.get("ci_context"), + metadata_to_validate, + ) + + +# UPLOAD COMMAND + + +@metadata.command(cls=DaggerPipelineCommand, help="Commands related to uploading the metadata files to remote storage.") +@click.argument("gcs-bucket-name", type=click.STRING) +@click.option("--modified-only/--all", default=True) +@click.pass_context +def upload(ctx: click.Context, gcs_bucket_name: str, modified_only: bool) -> bool: + if modified_only: + if ctx.obj["ci_context"] is not CIContext.MASTER and ctx.obj["git_branch"] != "master": + click.secho("Not on the master branch. Skipping metadata upload.") + return True + metadata_to_upload = get_modified_metadata_files(ctx.obj["modified_files"]) + if not metadata_to_upload: + click.secho("No modified metadata found. Skipping metadata upload.") + return True + else: + metadata_to_upload = get_all_metadata_files() + + click.secho(f"Will upload {len(metadata_to_upload)} metadata files.") + + return anyio.run( + run_metadata_upload_pipeline, + ctx.obj["is_local"], + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj.get("gha_workflow_run_url"), + ctx.obj.get("dagger_logs_url"), + ctx.obj.get("pipeline_start_timestamp"), + ctx.obj.get("ci_context"), + metadata_to_upload, + gcs_bucket_name, + ) + + +# DEPLOY GROUP + + +@metadata.group(help="Commands related to deploying components of the metadata service.") +@click.pass_context +def deploy(ctx: click.Context): + pass + + +@deploy.command(cls=DaggerPipelineCommand, name="orchestrator", help="Deploy the metadata service orchestrator to production") +@click.pass_context +def deploy_orchestrator(ctx: click.Context) -> bool: + return anyio.run( + run_metadata_orchestrator_deploy_pipeline, + ctx.obj["is_local"], + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj.get("gha_workflow_run_url"), + ctx.obj.get("dagger_logs_url"), + ctx.obj.get("pipeline_start_timestamp"), + ctx.obj.get("ci_context"), + ) + + +# TEST GROUP + + +@metadata.group(help="Commands related to testing the metadata service.") +@click.pass_context +def test(ctx: click.Context): + pass + + +@test.command(cls=DaggerPipelineCommand, name="lib", help="Run tests for the metadata service library.") +@click.pass_context +def test_lib(ctx: click.Context) -> bool: + return anyio.run( + run_metadata_lib_test_pipeline, + ctx.obj["is_local"], + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj.get("gha_workflow_run_url"), + ctx.obj.get("dagger_logs_url"), + ctx.obj.get("pipeline_start_timestamp"), + ctx.obj.get("ci_context"), + ) + + +@test.command(cls=DaggerPipelineCommand, name="orchestrator", help="Run tests for the metadata service orchestrator.") +@click.pass_context +def test_orchestrator(ctx: click.Context) -> bool: + return anyio.run( + run_metadata_orchestrator_test_pipeline, + ctx.obj["is_local"], + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj.get("gha_workflow_run_url"), + ctx.obj.get("dagger_logs_url"), + ctx.obj.get("pipeline_start_timestamp"), + ctx.obj.get("ci_context"), + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py new file mode 100644 index 000000000000..c9cd4248a170 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the tests command to test airbyte-ci projects. +""" + +import logging +import os +import sys + +import anyio +import click +import dagger + + +@click.command() +@click.argument("poetry_package_path") +@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") +def test( + poetry_package_path: str, + test_directory: str, +): + """Runs the tests for the given airbyte-ci package. + + Args: + poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. + test_directory (str): The directory containing the tests to run. + """ + success = anyio.run(run_test, poetry_package_path, test_directory) + if not success: + click.Abort() + + +async def run_test(poetry_package_path: str, test_directory: str) -> bool: + """Runs the tests for the given airbyte-ci package in a Dagger container. + + Args: + airbyte_ci_package_path (str): Path to the airbyte-ci package to test, relative to airbyte-ci directory. + Returns: + bool: True if the tests passed, False otherwise. + """ + logger = logging.getLogger(f"{poetry_package_path}.tests") + logger.info(f"Running tests for {poetry_package_path}") + # The following directories are always mounted because a lot of tests rely on them + directories_to_always_mount = [".git", "airbyte-integrations", "airbyte-ci"] + directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + try: + docker_host_socket = dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") + pytest_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "install", "-y", "bash", "git", "curl"]) + .with_env_variable("VERSION", "24.0.2") + .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"]) + .with_exec(["pip", "install", "pipx"]) + .with_exec(["pipx", "ensurepath"]) + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec(["pipx", "install", "poetry"]) + .with_mounted_directory( + "/airbyte", + dagger_client.host().directory( + ".", + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/build", "**/.gradle"], + include=directories_to_mount, + ), + ) + .with_workdir(f"/airbyte/{poetry_package_path}") + .with_exec(["poetry", "install"]) + .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_exec(["poetry", "run", "pytest", test_directory]) + ) + if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: + logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") + pytest_container = pytest_container.with_env_variable( + "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" + ).with_unix_socket("/var/run/buildkit/buildkitd.sock", docker_host_socket) + + await pytest_container + return True + except dagger.ExecError as e: + logger.error("Tests failed") + logger.error(e.stderr) + sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py new file mode 100644 index 000000000000..445479449ddc --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import platform +from pathlib import Path + +from dagger import Platform + +PYPROJECT_TOML_FILE_PATH = "pyproject.toml" +LICENSE_SHORT_FILE_PATH = "LICENSE_SHORT" +CONNECTOR_TESTING_REQUIREMENTS = [ + "pip==21.3.1", + "mccabe==0.6.1", + "flake8==4.0.1", + "pyproject-flake8==0.0.1a2", + "black==22.3.0", + "isort==5.6.4", + "pytest==6.2.5", + "coverage[toml]==6.3.1", + "pytest-custom_exit_code", + "licenseheaders==0.8.8", +] + +CI_CREDENTIALS_SOURCE_PATH = "airbyte-ci/connectors/ci_credentials" +CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH = "airbyte-ci/connectors/connector_ops" +BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] +LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}") +DOCKER_VERSION = "24.0.2" +DOCKER_DIND_IMAGE = "docker:24-dind" +DOCKER_CLI_IMAGE = "docker:24-cli" +GRADLE_CACHE_PATH = "/root/.gradle/caches" +GRADLE_BUILD_CACHE_PATH = f"{GRADLE_CACHE_PATH}/build-cache-1" +GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH = "/root/gradle_dependency_cache" +LOCAL_REPORTS_PATH_ROOT = "airbyte-ci/connectors/pipelines/pipeline_reports/" +GCS_PUBLIC_DOMAIN = "https://storage.cloud.google.com" diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/contexts.py similarity index 79% rename from tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py rename to airbyte-ci/connectors/pipelines/pipelines/contexts.py index e289f340e89d..897006696f90 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/contexts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/contexts.py @@ -15,14 +15,14 @@ import yaml from anyio import Path from asyncer import asyncify -from ci_connector_ops.pipelines.actions import remote_storage, secrets -from ci_connector_ops.pipelines.bases import CIContext, ConnectorReport, Report -from ci_connector_ops.pipelines.github import update_commit_status_check -from ci_connector_ops.pipelines.slack import send_message_to_webhook -from ci_connector_ops.pipelines.utils import AIRBYTE_REPO_URL, METADATA_FILE_NAME, sanitize_gcs_credentials -from ci_connector_ops.utils import Connector from dagger import Client, Directory, Secret from github import PullRequest +from pipelines import hacks +from pipelines.actions import secrets +from pipelines.bases import CIContext, ConnectorReport, ConnectorWithModifiedFiles, Report +from pipelines.github import update_commit_status_check +from pipelines.slack import send_message_to_webhook +from pipelines.utils import AIRBYTE_REPO_URL, METADATA_FILE_NAME, format_duration, sanitize_gcs_credentials class ContextState(Enum): @@ -41,7 +41,7 @@ class PipelineContext: PRODUCTION = bool(os.environ.get("PRODUCTION", False)) # Set this to True to enable production mode (e.g. to send PR comments) DEFAULT_EXCLUDED_FILES = ( - [".git"] + [".git", "airbyte-ci/connectors/pipelines/*"] + glob("**/build", recursive=True) + glob("**/.venv", recursive=True) + glob("**/secrets", recursive=True) @@ -52,6 +52,7 @@ class PipelineContext: + glob("**/.eggs", recursive=True) + glob("**/.mypy_cache", recursive=True) + glob("**/.DS_Store", recursive=True) + + glob("**/airbyte_ci_logs", recursive=True) ) def __init__( @@ -61,12 +62,17 @@ def __init__( git_branch: str, git_revision: str, gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, ci_context: Optional[str] = None, is_ci_optional: bool = False, slack_webhook: Optional[str] = None, reporting_slack_channel: Optional[str] = None, pull_request: PullRequest = None, + ci_report_bucket: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + ci_git_user: Optional[str] = None, + ci_github_access_token: Optional[str] = None, ): """Initialize a pipeline context. @@ -76,6 +82,7 @@ def __init__( git_branch (str): The current git branch name. git_revision (str): The current git revision, commit hash. gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. + dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None. is_ci_optional (bool, optional): Whether the CI is optional. Defaults to False. @@ -88,6 +95,7 @@ def __init__( self.git_branch = git_branch self.git_revision = git_revision self.gha_workflow_run_url = gha_workflow_run_url + self.dagger_logs_url = dagger_logs_url self.pipeline_start_timestamp = pipeline_start_timestamp self.created_at = datetime.utcnow() self.ci_context = ci_context @@ -99,6 +107,14 @@ def __init__( self.logger = logging.getLogger(self.pipeline_name) self.dagger_client = None self._report = None + self.dockerd_service = None + self.ci_gcs_credentials = sanitize_gcs_credentials(ci_gcs_credentials) if ci_gcs_credentials else None + self.ci_report_bucket = ci_report_bucket + self.ci_git_user = ci_git_user + self.ci_github_access_token = ci_github_access_token + self.started_at = None + self.stopped_at = None + self.secrets_to_mask = [] update_commit_status_check(**self.github_commit_status) @property @@ -129,6 +145,14 @@ def report(self) -> Report: # noqa D102 def report(self, report: Report): # noqa D102 self._report = report + @property + def ci_gcs_credentials_secret(self) -> Secret: + return self.dagger_client.set_secret("ci_gcs_credentials", self.ci_gcs_credentials) + + @property + def ci_github_access_token_secret(self) -> Secret: + return self.dagger_client.set_secret("ci_github_access_token", self.ci_github_access_token) + @property def github_commit_status(self) -> dict: """Build a dictionary used as kwargs to the update_commit_status_check function.""" @@ -147,14 +171,22 @@ def github_commit_status(self) -> dict: def should_send_slack_message(self) -> bool: return self.slack_webhook is not None and self.reporting_slack_channel is not None + @property + def has_dagger_cloud_token(self) -> bool: + return "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN" in os.environ + + @property + def dagger_cloud_url(self) -> str: + """Gets the link to the Dagger Cloud runs page for the current commit.""" + if self.is_local or not self.has_dagger_cloud_token: + return None + + return f"https://alpha.dagger.cloud/changeByPipelines?filter=dagger.io/git.ref:{self.git_revision}" + def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> Directory: """Get a directory from the current repository. - If running in the CI: - The directory is extracted from the git branch. - - If running locally: - The directory is extracted from your host file system. + The directory is extracted from the host file system. A couple of files or directories that could corrupt builds are exclude by default (check DEFAULT_EXCLUDED_FILES). Args: @@ -170,6 +202,7 @@ def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, i else: exclude += self.DEFAULT_EXCLUDED_FILES exclude = list(set(exclude)) + exclude.sort() # sort to make sure the order is always the same to not burst the cache. Casting exclude to set can change the order if subdir != ".": subdir = f"{subdir}/" if not subdir.endswith("/") else subdir exclude = [f.replace(subdir, "") for f in exclude if subdir in f] @@ -191,6 +224,9 @@ async def __aenter__(self): if self.dagger_client is None: raise Exception("A Pipeline can't be entered with an undefined dagger_client") self.state = ContextState.RUNNING + self.started_at = datetime.utcnow() + self.logger.info("Caching the latest CDK version...") + await hacks.cache_latest_cdk(self.dagger_client) await asyncify(update_commit_status_check)(**self.github_commit_status) if self.should_send_slack_message: await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) @@ -236,6 +272,7 @@ async def __aexit__( bool: Whether the teardown operation ran successfully. """ self.state = self.determine_final_state(self.report, exception_value) + self.stopped_at = datetime.utcnow() if exception_value: self.logger.error("An error was handled by the Pipeline", exc_info=True) @@ -244,7 +281,6 @@ async def __aexit__( self.report = Report(self, steps_results=[]) self.report.print() - self.logger.info(self.report.to_json()) await asyncify(update_commit_status_check)(**self.github_commit_status) if self.should_send_slack_message: @@ -256,25 +292,33 @@ async def __aexit__( class ConnectorContext(PipelineContext): """The connector context is used to store configuration for a specific connector pipeline run.""" - DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE = "airbyte/connector-acceptance-test:latest" + DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE = "airbyte/connector-acceptance-test:dev" def __init__( self, pipeline_name: str, - connector: Connector, + connector: ConnectorWithModifiedFiles, is_local: bool, git_branch: bool, git_revision: bool, - modified_files: List[str], - s3_report_key: str, + report_output_prefix: str, use_remote_secrets: bool = True, + ci_report_bucket: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + ci_git_user: Optional[str] = None, + ci_github_access_token: Optional[str] = None, connector_acceptance_test_image: Optional[str] = DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE, gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, ci_context: Optional[str] = None, slack_webhook: Optional[str] = None, reporting_slack_channel: Optional[str] = None, pull_request: PullRequest = None, + should_save_report: bool = True, + fail_fast: bool = False, + fast_tests_only: bool = False, + code_tests_only: bool = False, ): """Initialize a connector context. @@ -283,40 +327,56 @@ def __init__( is_local (bool): Whether the context is for a local run or a CI run. git_branch (str): The current git branch name. git_revision (str): The current git revision, commit hash. - modified_files (List[str]): The list of modified files in the current git branch. - s3_report_key (str): The S3 key to upload the test report to. + report_output_prefix (str): The S3 key to upload the test report to. use_remote_secrets (bool, optional): Whether to download secrets for GSM or use the local secrets. Defaults to True. connector_acceptance_test_image (Optional[str], optional): The image to use to run connector acceptance tests. Defaults to DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE. gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. + dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None. slack_webhook (Optional[str], optional): The slack webhook to send messages to. Defaults to None. reporting_slack_channel (Optional[str], optional): The slack channel to send messages to. Defaults to None. pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. + fail_fast (bool, optional): Whether to fail fast. Defaults to False. + fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False. + code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. """ self.pipeline_name = pipeline_name self.connector = connector self.use_remote_secrets = use_remote_secrets self.connector_acceptance_test_image = connector_acceptance_test_image - self.modified_files = modified_files - self.s3_report_key = s3_report_key + self.report_output_prefix = report_output_prefix self._secrets_dir = None self._updated_secrets_dir = None self.cdk_version = None + self.should_save_report = should_save_report + self.fail_fast = fail_fast + self.fast_tests_only = fast_tests_only + self.code_tests_only = code_tests_only + super().__init__( pipeline_name=pipeline_name, is_local=is_local, git_branch=git_branch, git_revision=git_revision, gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, pipeline_start_timestamp=pipeline_start_timestamp, ci_context=ci_context, slack_webhook=slack_webhook, reporting_slack_channel=reporting_slack_channel, pull_request=pull_request, + ci_report_bucket=ci_report_bucket, + ci_gcs_credentials=ci_gcs_credentials, + ci_git_user=ci_git_user, + ci_github_access_token=ci_github_access_token, ) + @property + def modified_files(self): + return self.connector.modified_files + @property def secrets_dir(self) -> Directory: # noqa D102 return self._secrets_dir @@ -354,10 +414,18 @@ def metadata(self) -> dict: return yaml.safe_load(self.metadata_path.read_text())["data"] @property - def docker_image_from_metadata(self) -> str: - return f"{self.metadata['dockerRepository']}:{self.metadata['dockerImageTag']}" + def docker_repository(self) -> str: + return self.metadata["dockerRepository"] + + @property + def docker_image_tag(self) -> str: + return self.metadata["dockerImageTag"] + + @property + def docker_image(self) -> str: + return f"{self.docker_repository}:{self.docker_image_tag}" - def get_connector_dir(self, exclude=None, include=None) -> Directory: + async def get_connector_dir(self, exclude=None, include=None) -> Directory: """Get the connector under test source code directory. Args: @@ -367,7 +435,8 @@ def get_connector_dir(self, exclude=None, include=None) -> Directory: Returns: Directory: The connector under test source code directory. """ - return self.get_repo_dir(str(self.connector.code_directory), exclude=exclude, include=include) + vanilla_connector_dir = self.get_repo_dir(str(self.connector.code_directory), exclude=exclude, include=include) + return await hacks.patch_connector_dir(self, vanilla_connector_dir) async def __aexit__( self, exception_type: Optional[type[BaseException]], exception_value: Optional[BaseException], traceback: Optional[TracebackType] @@ -386,6 +455,7 @@ async def __aexit__( Returns: bool: Whether the teardown operation ran successfully. """ + self.stopped_at = datetime.utcnow() self.state = self.determine_final_state(self.report, exception_value) if exception_value: self.logger.error("An error got handled by the ConnectorContext", exc_info=True) @@ -397,29 +467,18 @@ async def __aexit__( await secrets.upload(self) self.report.print() - self.logger.info(self.report.to_json()) - - local_reports_path_root = "tools/ci_connector_ops/pipeline_reports/" - connector_name = self.report.pipeline_context.connector.technical_name - connector_version = self.report.pipeline_context.connector.version - git_revision = self.report.pipeline_context.git_revision - git_branch = self.report.pipeline_context.git_branch.replace("/", "_") - suffix = f"{connector_name}/{git_branch}/{connector_version}/{git_revision}.json" - local_report_path = Path(local_reports_path_root + suffix) - await local_report_path.parents[0].mkdir(parents=True, exist_ok=True) - await local_report_path.write_text(self.report.to_json()) - if self.report.should_be_saved: - s3_key = self.s3_report_key + suffix - report_upload_exit_code = await remote_storage.upload_to_s3( - self.dagger_client, str(local_report_path), s3_key, os.environ["TEST_REPORTS_BUCKET_NAME"] - ) - if report_upload_exit_code != 0: - self.logger.error("Uploading the report to S3 failed.") + + if self.should_save_report: + await self.report.save() + if self.report.should_be_commented_on_pr: self.report.post_comment_on_pr() + await asyncify(update_commit_status_check)(**self.github_commit_status) + if self.should_send_slack_message: await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) + # Supress the exception if any return True @@ -430,9 +489,8 @@ def create_slack_message(self) -> str: class PublishConnectorContext(ConnectorContext): def __init__( self, - connector: Connector, + connector: ConnectorWithModifiedFiles, pre_release: bool, - modified_files: List[str], spec_cache_gcs_credentials: str, spec_cache_bucket_name: str, metadata_service_gcs_credentials: str, @@ -441,12 +499,16 @@ def __init__( docker_hub_password: str, slack_webhook: str, reporting_slack_channel: str, + ci_report_bucket: str, + report_output_prefix: str, is_local: bool, git_branch: bool, git_revision: bool, gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, ci_context: Optional[str] = None, + ci_gcs_credentials: str = None, pull_request: PullRequest = None, ): self.pre_release = pre_release @@ -463,16 +525,19 @@ def __init__( super().__init__( pipeline_name=pipeline_name, connector=connector, - modified_files=modified_files, - s3_report_key="python-poc/publish/history/", + report_output_prefix=report_output_prefix, + ci_report_bucket=ci_report_bucket, is_local=is_local, git_branch=git_branch, git_revision=git_revision, gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, pipeline_start_timestamp=pipeline_start_timestamp, ci_context=ci_context, slack_webhook=slack_webhook, reporting_slack_channel=reporting_slack_channel, + ci_gcs_credentials=ci_gcs_credentials, + should_save_report=True, ) @property @@ -492,15 +557,17 @@ def spec_cache_gcs_credentials_secret(self) -> Secret: return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials) @property - def docker_image_name(self): + def docker_image_tag(self): + # get the docker image tag from the parent class + metadata_tag = super().docker_image_tag if self.pre_release: - return f"{self.docker_image_from_metadata}-dev.{self.git_revision[:10]}" + return f"{metadata_tag}-dev.{self.git_revision[:10]}" else: - return self.docker_image_from_metadata + return metadata_tag def create_slack_message(self) -> str: docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags" - message = f"*Publish <{docker_hub_url}|{self.docker_image_name}>*\n" + message = f"*Publish <{docker_hub_url}|{self.docker_image}>*\n" if self.is_ci: message += f"🤖 <{self.gha_workflow_run_url}|GitHub Action workflow>\n" else: @@ -519,7 +586,7 @@ def create_slack_message(self) -> str: message += "🔴" message += f" {self.state.value['description']}\n" if self.state is ContextState.SUCCESSFUL: - message += f"⏲️ Run duration: {round(self.report.run_duration)}s\n" + message += f"⏲️ Run duration: {format_duration(self.report.run_duration)}\n" if self.state is ContextState.FAILURE: - message += "\ncc. " + message += "\ncc. " # @dev-connector-ops return message diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py new file mode 100644 index 000000000000..d9ef70879617 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module execute the airbyte-ci-internal CLI wrapped in a dagger run command to use the Dagger Terminal UI.""" + +import logging +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Optional + +import pkg_resources +import requests + +LOGGER = logging.getLogger(__name__) +BIN_DIR = Path.home() / "bin" +BIN_DIR.mkdir(exist_ok=True) +DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE = ( + "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN", + "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k", +) +ARGS_DISABLING_TUI = ["--no-tui", "publish"] + + +def get_dagger_path() -> Optional[str]: + try: + return ( + subprocess.run(["which", "dagger"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8").strip() + ) + except subprocess.CalledProcessError: + if Path(BIN_DIR / "dagger").exists(): + return str(Path(BIN_DIR / "dagger")) + + +def get_current_dagger_sdk_version() -> str: + version = pkg_resources.get_distribution("dagger-io").version + return version + + +def install_dagger_cli(dagger_version: str) -> None: + install_script_path = "/tmp/install_dagger.sh" + with open(install_script_path, "w") as f: + response = requests.get("https://dl.dagger.io/dagger/install.sh") + response.raise_for_status() + f.write(response.text) + subprocess.run(["chmod", "+x", install_script_path], check=True) + os.environ["BIN_DIR"] = str(BIN_DIR) + os.environ["DAGGER_VERSION"] = dagger_version + subprocess.run([install_script_path], check=True) + + +def get_dagger_cli_version(dagger_path: Optional[str]) -> Optional[str]: + if not dagger_path: + return None + version_output = ( + subprocess.run([dagger_path, "version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8").strip() + ) + version_pattern = r"v(\d+\.\d+\.\d+)" + + match = re.search(version_pattern, version_output) + + if match: + version = match.group(1) + return version + else: + raise Exception("Could not find dagger version in output: " + version_output) + + +def check_dagger_cli_install() -> str: + expected_dagger_cli_version = get_current_dagger_sdk_version() + dagger_path = get_dagger_path() + if dagger_path is None: + LOGGER.info(f"The Dagger CLI is not installed. Installing {expected_dagger_cli_version}...") + install_dagger_cli(expected_dagger_cli_version) + dagger_path = get_dagger_path() + + cli_version = get_dagger_cli_version(dagger_path) + if cli_version != expected_dagger_cli_version: + LOGGER.warning( + f"The Dagger CLI version '{cli_version}' does not match the expected version '{expected_dagger_cli_version}'. Installing Dagger CLI '{expected_dagger_cli_version}'..." + ) + install_dagger_cli(expected_dagger_cli_version) + return check_dagger_cli_install() + return dagger_path + + +def main(): + os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1] + exit_code = 0 + if len(sys.argv) > 1 and any([arg in ARGS_DISABLING_TUI for arg in sys.argv]): + command = ["airbyte-ci-internal"] + [arg for arg in sys.argv[1:] if arg != "--no-tui"] + else: + dagger_path = check_dagger_cli_install() + command = [dagger_path, "run", "airbyte-ci-internal"] + sys.argv[1:] + try: + try: + subprocess.run(command, check=True) + except KeyboardInterrupt: + LOGGER.info("Keyboard interrupt detected. Exiting...") + exit_code = 1 + except subprocess.CalledProcessError as e: + exit_code = e.returncode + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py new file mode 100644 index 000000000000..730874bd6b9b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""This module groups factory like functions to dispatch formatting steps according to the connector language.""" + +from __future__ import annotations + +import sys +from typing import List, Optional + +import anyio +import dagger +from connector_ops.utils import ConnectorLanguage +from pipelines.actions import environments +from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus +from pipelines.contexts import ConnectorContext +from pipelines.format import java_connectors, python_connectors +from pipelines.git import GitPushChanges +from pipelines.pipelines.connectors import run_report_complete_pipeline + + +class NoFormatStepForLanguageError(Exception): + pass + + +FORMATTING_STEP_TO_CONNECTOR_LANGUAGE_MAPPING = { + ConnectorLanguage.PYTHON: python_connectors.FormatConnectorCode, + ConnectorLanguage.LOW_CODE: python_connectors.FormatConnectorCode, + ConnectorLanguage.JAVA: java_connectors.FormatConnectorCode, +} + + +class ExportChanges(Step): + title = "Export changes to local repository" + + async def _run(self, changed_directory: dagger.Directory, changed_directory_path_in_repo: str) -> StepResult: + await changed_directory.export(changed_directory_path_in_repo) + return StepResult(self, StepStatus.SUCCESS, stdout=f"Changes exported to {changed_directory_path_in_repo}") + + +async def run_connector_format_pipeline(context: ConnectorContext) -> ConnectorReport: + """Run a format pipeline for a single connector. + + Args: + context (ConnectorContext): The initialized connector context. + + Returns: + ConnectorReport: The reports holding formats results. + """ + steps_results = [] + async with context: + FormatConnectorCode = FORMATTING_STEP_TO_CONNECTOR_LANGUAGE_MAPPING.get(context.connector.language) + if not FormatConnectorCode: + raise NoFormatStepForLanguageError( + f"No formatting step found for connector {context.connector.technical_name} with language {context.connector.language}" + ) + format_connector_code_result = await FormatConnectorCode(context).run() + steps_results.append(format_connector_code_result) + + if context.is_local: + export_changes_results = await ExportChanges(context).run( + format_connector_code_result.output_artifact, str(context.connector.code_directory) + ) + steps_results.append(export_changes_results) + else: + git_push_changes_results = await GitPushChanges(context).run( + format_connector_code_result.output_artifact, + str(context.connector.code_directory), + f"Auto format {context.connector.technical_name} code", + skip_ci=True, + ) + steps_results.append(git_push_changes_results) + context.report = ConnectorReport(context, steps_results, name="FORMAT RESULTS") + return context.report + + +async def run_connectors_format_pipelines( + contexts: List[ConnectorContext], + ci_git_user: str, + ci_github_access_token: str, + git_branch: str, + is_local: bool, + execute_timeout: Optional[int], +) -> List[ConnectorContext]: + async with dagger.Connection(dagger.Config(log_output=sys.stderr, execute_timeout=execute_timeout)) as dagger_client: + requires_dind = any(context.connector.language == ConnectorLanguage.JAVA for context in contexts) + dockerd_service = environments.with_global_dockerd_service(dagger_client) + async with anyio.create_task_group() as tg_main: + if requires_dind: + tg_main.start_soon(dockerd_service.sync) + await anyio.sleep(10) # Wait for the docker service to be ready + for context in contexts: + context.dagger_client = dagger_client.pipeline(f"Format - {context.connector.technical_name}") + context.dockerd_service = dockerd_service + await run_connector_format_pipeline(context) + # When the connectors pipelines are done, we can stop the dockerd service + tg_main.cancel_scope.cancel() + + await run_report_complete_pipeline(dagger_client, contexts) + + return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py new file mode 100644 index 000000000000..7d73f3ab40fb --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from pipelines.actions import environments +from pipelines.bases import StepResult +from pipelines.gradle import GradleTask +from pipelines.utils import get_exec_result + + +class FormatConnectorCode(GradleTask): + """ + A step to format a Java connector code. + """ + + title = "Format connector code" + + async def _run(self) -> StepResult: + formatted = ( + environments.with_gradle(self.context, self.build_include, bind_to_docker_host=self.BIND_TO_DOCKER_HOST) + .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) + .with_exec(["./gradlew", "format"]) + ) + exit_code, stdout, stderr = await get_exec_result(formatted) + return StepResult( + self, + self.get_step_status_from_exit_code(exit_code), + stderr=stderr, + stdout=stdout, + output_artifact=formatted.directory(str(self.context.connector.code_directory)), + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py new file mode 100644 index 000000000000..e2ebbcf68d8a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncer +from pipelines.actions import environments +from pipelines.bases import Step, StepResult +from pipelines.utils import with_exit_code, with_stderr, with_stdout + + +class FormatConnectorCode(Step): + """ + A step to format a Python connector code. + """ + + title = "Format connector code" + + @property + def black_cmd(self): + return ["python", "-m", "black", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] + + @property + def isort_cmd(self): + return ["python", "-m", "isort", f"--settings-file=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] + + @property + def licenseheaders_cmd(self): + return [ + "python", + "-m", + "licenseheaders", + f"--tmpl=/{environments.LICENSE_SHORT_FILE_PATH}", + "--ext=py", + "--exclude=**/models/__init__.py", + ] + + async def _run(self) -> StepResult: + formatted = ( + environments.with_testing_dependencies(self.context) + .with_mounted_directory("/connector_code", await self.context.get_connector_dir()) + .with_workdir("/connector_code") + .with_exec(self.licenseheaders_cmd) + .with_exec(self.isort_cmd) + .with_exec(self.black_cmd) + ) + async with asyncer.create_task_group() as task_group: + soon_exit_code = task_group.soonify(with_exit_code)(formatted) + soon_stderr = task_group.soonify(with_stderr)(formatted) + soon_stdout = task_group.soonify(with_stdout)(formatted) + + return StepResult( + self, + self.get_step_status_from_exit_code(await soon_exit_code), + stderr=soon_stderr.value, + stdout=soon_stdout.value, + output_artifact=formatted.directory("/connector_code"), + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/git.py b/airbyte-ci/connectors/pipelines/pipelines/git.py new file mode 100644 index 000000000000..acf23c2e8eef --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/git.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import Client, Directory, Secret +from pipelines.actions import environments +from pipelines.bases import Step, StepResult +from pipelines.github import AIRBYTE_GITHUB_REPO + + +class GitPushChanges(Step): + """ + A step to push changes to the remote repository. + """ + + title = "Push changes to the remote repository" + + GITHUB_REPO_URL = f"https://github.com/{AIRBYTE_GITHUB_REPO}.git" + + @property + def ci_git_user(self) -> str: + return self.context.ci_git_user + + @property + def ci_github_access_token(self) -> str: + return self.context.ci_github_access_token + + @property + def dagger_client(self) -> Client: + return self.context.dagger_client + + @property + def git_branch(self) -> str: + return self.context.git_branch + + @property + def authenticated_repo_url(self) -> Secret: + url = self.GITHUB_REPO_URL.replace("https://", f"https://{self.ci_git_user}:{self.ci_github_access_token}@") + return self.dagger_client.set_secret("authenticated_repo_url", url) + + @property + def airbyte_repo(self) -> Directory: + return self.dagger_client.git(self.GITHUB_REPO_URL, keep_git_dir=True).branch(self.git_branch).tree() + + def get_commit_message(self, commit_message: str, skip_ci: bool) -> str: + commit_message = f"🤖 {commit_message}" + return f"{commit_message} [skip ci]" if skip_ci else commit_message + + async def _run( + self, changed_directory: Directory, changed_directory_path: str, commit_message: str, skip_ci: bool = True + ) -> StepResult: + diff = ( + environments.with_git(self.dagger_client, self.context.ci_github_access_token_secret, self.ci_git_user) + .with_secret_variable("AUTHENTICATED_REPO_URL", self.authenticated_repo_url) + .with_mounted_directory("/airbyte", self.airbyte_repo) + .with_workdir("/airbyte") + .with_exec(["git", "checkout", self.git_branch]) + .with_mounted_directory(f"/airbyte/{changed_directory_path}", changed_directory) + .with_exec(["git", "diff", "--name-only"]) + ) + + if not await diff.stdout(): + return self.skip("No changes to push") + + commit_and_push = ( + diff.with_exec(["sh", "-c", "git remote set-url origin $AUTHENTICATED_REPO_URL"]) + .with_exec(["git", "add", "."]) + .with_exec(["git", "commit", "-m", self.get_commit_message(commit_message, skip_ci)]) + .with_exec(["git", "pull", "--rebase", "origin", self.git_branch]) + .with_exec(["git", "push"]) + ) + return await self.get_step_result(commit_and_push) + + +class GitPushEmptyCommit(GitPushChanges): + """ + A step to push an empty commit to the remote repository. + """ + + title = "Push empty commit to the remote repository" + + def __init__(self, dagger_client, ci_git_user, ci_github_access_token, git_branch): + self._dagger_client = dagger_client + self._ci_github_access_token = ci_github_access_token + self._ci_git_user = ci_git_user + self._git_branch = git_branch + self.ci_github_access_token_secret = dagger_client.set_secret("ci_github_access_token", ci_github_access_token) + + @property + def dagger_client(self) -> Client: + return self._dagger_client + + @property + def ci_git_user(self) -> str: + return self._ci_git_user + + @property + def ci_github_access_token(self) -> Secret: + return self._ci_github_access_token + + @property + def git_branch(self) -> str: + return self._git_branch + + async def _run(self, commit_message: str, skip_ci: bool = True) -> StepResult: + push_empty_commit = ( + environments.with_git(self.dagger_client, self.ci_github_access_token_secret, self.ci_git_user) + .with_secret_variable("AUTHENTICATED_REPO_URL", self.authenticated_repo_url) + .with_mounted_directory("/airbyte", self.airbyte_repo) + .with_workdir("/airbyte") + .with_exec(["git", "checkout", self.git_branch]) + .with_exec(["sh", "-c", "git remote set-url origin $AUTHENTICATED_REPO_URL"]) + .with_exec(["git", "commit", "--allow-empty", "-m", self.get_commit_message(commit_message, skip_ci)]) + .with_exec(["git", "pull", "--rebase", "origin", self.git_branch]) + .with_exec(["git", "push"]) + ) + return await self.get_step_result(push_empty_commit) diff --git a/airbyte-ci/connectors/pipelines/pipelines/github.py b/airbyte-ci/connectors/pipelines/pipelines/github.py new file mode 100644 index 000000000000..fd6bb7e47530 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/github.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""Module grouping functions interacting with the GitHub API.""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING, Optional + +from connector_ops.utils import console +from pipelines import main_logger +from pipelines.bases import CIContext + +if TYPE_CHECKING: + from logging import Logger + +from github import Github, PullRequest + +AIRBYTE_GITHUB_REPO = "airbytehq/airbyte" +GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests" +GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests" + + +def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> None: + """Log a message to a logger if one is available, otherwise print to the console.""" + if logger: + log_method = getattr(logger, level.lower()) + log_method(message) + else: + main_logger.info(message) + + +def update_commit_status_check( + sha: str, state: str, target_url: str, description: str, context: str, is_optional=False, should_send=True, logger: Logger = None +): + """Call the GitHub API to create commit status check. + + Args: + sha (str): Hash of the commit for which you want to create a status check. + state (str): The check state (success, failure, pending) + target_url (str): The URL to attach to the commit check for details. + description (str): Description of the check that is run. + context (str): Name of the Check context e.g: source-pokeapi tests + should_send (bool, optional): Whether the commit check should actually be sent to GitHub API. Defaults to True. + logger (Logger, optional): A logger to log info about updates. Defaults to None. + """ + if not should_send: + return + + safe_log(logger, f"Attempting to create {state} status for commit {sha} on Github in {context} context.") + try: + github_client = Github(os.environ["CI_GITHUB_ACCESS_TOKEN"]) + airbyte_repo = github_client.get_repo(AIRBYTE_GITHUB_REPO) + except Exception as e: + if logger: + logger.error("No commit status check sent, the connection to Github API failed", exc_info=True) + else: + console.print(e) + return + + # If the check is optional, we don't want to fail the build if it fails. + # Instead, we want to mark it as a warning. + # Unfortunately, Github doesn't have a warning state, so we use success instead. + if is_optional and state == "failure": + state = "success" + description = f"[WARNING] optional check failed {context}: {description}" + + context = context if bool(os.environ.get("PRODUCTION", False)) is True else f"[please ignore] {context}" + airbyte_repo.get_commit(sha=sha).create_status( + state=state, + target_url=target_url, + description=description, + context=context, + ) + safe_log(logger, f"Created {state} status for commit {sha} on Github in {context} context with desc: {description}.") + + +def get_pull_request(pull_request_number: int, github_access_token: str) -> PullRequest: + """Get a pull request object from its number. + + Args: + pull_request_number (str): The number of the pull request to get. + github_access_token (str): The GitHub access token to use to authenticate. + Returns: + PullRequest: The pull request object. + """ + github_client = Github(github_access_token) + airbyte_repo = github_client.get_repo(AIRBYTE_GITHUB_REPO) + return airbyte_repo.get_pull(pull_request_number) + + +def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Logger = None): + update_commit_status_check( + click_context["git_revision"], + github_state, + click_context["gha_workflow_run_url"], + GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS, + GITHUB_GLOBAL_CONTEXT_FOR_TESTS, + should_send=click_context.get("ci_context") == CIContext.PULL_REQUEST, + logger=logger, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/gradle.py new file mode 100644 index 000000000000..c1dbf283d7a5 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/gradle.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +from abc import ABC +from typing import ClassVar, Tuple + +from dagger import CacheVolume, Container, Directory, QueryError +from pipelines import consts +from pipelines.actions import environments +from pipelines.bases import Step, StepResult +from pipelines.contexts import PipelineContext + + +class GradleTask(Step, ABC): + """ + A step to run a Gradle task. + + Attributes: + task_name (str): The Gradle task name to run. + title (str): The step title. + """ + + DEFAULT_TASKS_TO_EXCLUDE = ["airbyteDocker"] + BIND_TO_DOCKER_HOST = True + gradle_task_name: ClassVar + gradle_task_options: Tuple[str, ...] = () + + def __init__(self, context: PipelineContext, with_java_cdk_snapshot: bool = True) -> None: + super().__init__(context) + self.with_java_cdk_snapshot = with_java_cdk_snapshot + + @property + def connector_java_build_cache(self) -> CacheVolume: + return self.context.dagger_client.cache_volume("connector_java_build_cache") + + @property + def build_include(self) -> List[str]: + """Retrieve the list of source code directory required to run a Java connector Gradle task. + + The list is different according to the connector type. + + Returns: + List[str]: List of directories or files to be mounted to the container to run a Java connector Gradle task. + """ + return [ + str(dependency_directory) + for dependency_directory in self.context.connector.get_local_dependency_paths(with_test_dependencies=True) + ] + + async def _get_patched_build_src_dir(self) -> Directory: + """Patch some gradle plugins. + + Returns: + Directory: The patched buildSrc directory + """ + + build_src_dir = self.context.get_repo_dir("buildSrc") + cat_gradle_plugin_content = await build_src_dir.file("src/main/groovy/airbyte-connector-acceptance-test.gradle").contents() + # When running integrationTest in Dagger we don't want to run connectorAcceptanceTest + # connectorAcceptanceTest is run in the AcceptanceTest step + cat_gradle_plugin_content = cat_gradle_plugin_content.replace( + "project.integrationTest.dependsOn(project.connectorAcceptanceTest)", "" + ) + return build_src_dir.with_new_file("src/main/groovy/airbyte-connector-acceptance-test.gradle", contents=cat_gradle_plugin_content) + + def _get_gradle_command(self, extra_options: Tuple[str, ...] = ("--no-daemon", "--scan", "--build-cache")) -> List: + command = ( + ["./gradlew"] + + list(extra_options) + + [f":airbyte-integrations:connectors:{self.context.connector.technical_name}:{self.gradle_task_name}"] + + list(self.gradle_task_options) + ) + for task in self.DEFAULT_TASKS_TO_EXCLUDE: + command += ["-x", task] + return command + + async def _run(self) -> StepResult: + includes = self.build_include + if self.with_java_cdk_snapshot: + includes + ["./airbyte-cdk/java/airbyte-cdk/**"] + + connector_under_test = ( + environments.with_gradle(self.context, includes, bind_to_docker_host=self.BIND_TO_DOCKER_HOST) + .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) + .with_mounted_directory("buildSrc", await self._get_patched_build_src_dir()) + # Disable the Ryuk container because it needs privileged docker access that does not work: + .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true") + .with_(environments.mounted_connector_secrets(self.context, f"{self.context.connector.code_directory}/secrets")) + ) + if self.with_java_cdk_snapshot: + connector_under_test = connector_under_test.with_exec(["./gradlew", ":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"]) + connector_under_test = connector_under_test.with_exec(self._get_gradle_command()) + + results = await self.get_step_result(connector_under_test) + + await self._export_gradle_dependency_cache(connector_under_test) + return results + + async def _export_gradle_dependency_cache(self, gradle_container: Container) -> Container: + """Export the Gradle writable dependency cache to the read-only dependency cache path. + The read-only dependency cache is persisted thanks to mounted cache volumes in environments.with_gradle(). + You can read more about Shared readonly cache here: https://docs.gradle.org/current/userguide/dependency_resolution.html#sub:shared-readonly-cache + Args: + gradle_container (Container): The Gradle container. + + Returns: + Container: The Gradle container, with the updated cache. + """ + try: + cache_dirs = await gradle_container.directory(consts.GRADLE_CACHE_PATH).entries() + except QueryError: + cache_dirs = [] + if "modules-2" in cache_dirs: + with_cache = gradle_container.with_exec( + [ + "rsync", + "--archive", + "--quiet", + "--times", + "--exclude", + "*.lock", + "--exclude", + "gc.properties", + f"{consts.GRADLE_CACHE_PATH}/modules-2/", + f"{consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH}/modules-2/", + ] + ) + return await with_cache + return gradle_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py new file mode 100644 index 000000000000..b6a4d79cce9d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -0,0 +1,135 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module contains hacks used in connectors pipelines. They're gathered here for tech debt visibility.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable, List + +import requests +from connector_ops.utils import ConnectorLanguage +from dagger import DaggerError + +if TYPE_CHECKING: + from dagger import Client, Container, Directory + from pipelines.contexts import ConnectorContext + + +LINES_TO_REMOVE_FROM_GRADLE_FILE = [ + # Do not build normalization with Gradle - we build normalization with Dagger in the BuildOrPullNormalization step. + "project(':airbyte-integrations:bases:base-normalization').airbyteDocker.output", +] + + +async def _patch_gradle_file(context: ConnectorContext, connector_dir: Directory) -> Directory: + """Patch the build.gradle file of the connector under test by removing the lines declared in LINES_TO_REMOVE_FROM_GRADLE_FILE. + + Underlying issue: + Java connectors build.gradle declare a dependency to the normalization module. + It means every time we test a java connector the normalization is built. + This is time consuming and not required as normalization is now baked in containers. + Normalization is going away soon so hopefully this hack will be removed soon. + + Args: + context (ConnectorContext): The initialized connector context. + connector_dir (Directory): The directory containing the build.gradle file to patch. + Returns: + Directory: The directory containing the patched gradle file. + """ + if context.connector.language is not ConnectorLanguage.JAVA: + context.logger.info(f"Connector language {context.connector.language} does not require a patched build.gradle file.") + return connector_dir + + try: + gradle_file_content = await connector_dir.file("build.gradle").contents() + except DaggerError: + context.logger.info("Could not find build.gradle file in the connector directory. Skipping patching.") + return connector_dir + + context.logger.warn("Patching build.gradle file to remove normalization build.") + + patched_gradle_file = [] + + for line in gradle_file_content.splitlines(): + if not any(line_to_remove in line for line_to_remove in LINES_TO_REMOVE_FROM_GRADLE_FILE): + patched_gradle_file.append(line) + return connector_dir.with_new_file("build.gradle", contents="\n".join(patched_gradle_file)) + + +async def patch_connector_dir(context: ConnectorContext, connector_dir: Directory) -> Directory: + """Patch a connector directory: patch cat config, gradle file and dockerfile. + + Args: + context (ConnectorContext): The initialized connector context. + connector_dir (Directory): The directory containing the connector to patch. + Returns: + Directory: The directory containing the patched connector. + """ + patched_connector_dir = await _patch_gradle_file(context, connector_dir) + return patched_connector_dir.with_timestamps(1) + + +async def cache_latest_cdk(dagger_client: Client, pip_cache_volume_name: str = "pip_cache") -> None: + """ + Download the latest CDK version to update the pip cache. + + Underlying issue: + Most Python connectors, or normalization, are not pinning the CDK version they use. + It means that the will get whatever version is in the pip cache. + But the original goal of not pinning the CDK version is to always get the latest version. + + Hack: + Call this function before building connector test environment to update the cache with the latest CDK version. + + Github Issue: + Revisiting and aligning how we build Python connectors and using the same container for test, build and publish will provide better control over the CDK version. + https://github.com/airbytehq/airbyte/issues/25523 + Args: + dagger_client (Client): Dagger client. + """ + + # We get the latest version of the CDK from PyPI using their API. + # It allows us to explicitly install the latest version of the CDK in the container + # while keeping buildkit layer caching when the version value does not change. + # In other words: we only update the pip cache when the latest CDK version changes. + # When the CDK version does not change, the pip cache is not updated as the with_exec command remains the same. + cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" + response = requests.get(cdk_pypi_url) + response.raise_for_status() + package_info = response.json() + cdk_latest_version = package_info["info"]["version"] + + await ( + dagger_client.container() + .from_("python:3.9-slim") + .with_mounted_cache("/root/.cache/pip", dagger_client.cache_volume(pip_cache_volume_name)) + .with_exec(["pip", "install", "--force-reinstall", f"airbyte-cdk=={cdk_latest_version}"]) + .sync() + ) + + +def never_fail_exec(command: List[str]) -> Callable: + """ + Wrap a command execution with some bash sugar to always exit with a 0 exit code but write the actual exit code to a file. + + Underlying issue: + When a classic dagger with_exec is returning a >0 exit code an ExecError is raised. + It's OK for the majority of our container interaction. + But some execution, like running CAT, are expected to often fail. + In CAT we don't want ExecError to be raised on container interaction because CAT might write updated secrets that we need to pull from the container after the test run. + The bash trick below is a hack to always return a 0 exit code but write the actual exit code to a file. + The file is then read by the pipeline to determine the exit code of the container. + + Args: + command (List[str]): The command to run in the container. + + Returns: + Callable: _description_ + """ + + def never_fail_exec_inner(container: Container): + return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True) + + return never_fail_exec_inner diff --git a/tools/ci_connector_ops/ci_connector_ops/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/__init__.py similarity index 100% rename from tools/ci_connector_ops/ci_connector_ops/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/helpers/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py new file mode 100644 index 000000000000..c2456122778b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py @@ -0,0 +1,62 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""The actions package is made to declare reusable pipeline components.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Tuple, Union + +import asyncer +from pipelines.bases import Step, StepStatus + +if TYPE_CHECKING: + from pipelines.bases import StepResult + + +async def run_steps( + steps_and_run_args: List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]], results: List[StepResult] = [] +) -> List[StepResult]: + """Run multiple steps sequentially, or in parallel if steps are wrapped into a sublist. + + Args: + steps_and_run_args (List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]]): List of steps to run, if steps are wrapped in a sublist they will be executed in parallel. run function arguments can be passed as a tuple along the Step instance. + results (List[StepResult], optional): List of step results, used for recursion. + + Returns: + List[StepResult]: List of step results. + """ + # If there are no steps to run, return the results + if not steps_and_run_args: + return results + + # If any of the previous steps failed, skip the remaining steps + if any(result.status is StepStatus.FAILURE for result in results): + skipped_results = [] + for step_and_run_args in steps_and_run_args: + if isinstance(step_and_run_args, Tuple): + skipped_results.append(step_and_run_args[0].skip()) + else: + skipped_results.append(step_and_run_args.skip()) + return results + skipped_results + + # Pop the next step to run + steps_to_run, remaining_steps = steps_and_run_args[0], steps_and_run_args[1:] + + # wrap the step in a list if it is not already (allows for parallel steps) + if not isinstance(steps_to_run, list): + steps_to_run = [steps_to_run] + + async with asyncer.create_task_group() as task_group: + tasks = [] + for step in steps_to_run: + if isinstance(step, Step): + tasks.append(task_group.soonify(step.run)()) + elif isinstance(step, Tuple) and isinstance(step[0], Step) and isinstance(step[1], Tuple): + step, run_args = step + tasks.append(task_group.soonify(step.run)(*run_args)) + + new_results = [task.value for task in tasks] + + return await run_steps(remaining_steps, results + new_results) diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/__init__.py similarity index 100% rename from tools/ci_connector_ops/ci_connector_ops/pipelines/commands/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/pipelines/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py new file mode 100644 index 000000000000..5acb57b9558f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups the functions to run full pipelines for connector testing.""" + +import sys +from pathlib import Path +from typing import Callable, List, Optional + +import anyio +import dagger +from connector_ops.utils import ConnectorLanguage +from dagger import Config +from pipelines.actions import environments +from pipelines.bases import NoOpStep, Report, StepResult, StepStatus +from pipelines.contexts import ConnectorContext, ContextState +from pipelines.utils import create_and_open_file + +GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI" +GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests" + +CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING = { + # We run the Java connectors tests sequentially because we currently have memory issues when Java integration tests are run in parallel. + # See https://github.com/airbytehq/airbyte/issues/27168 + ConnectorLanguage.JAVA: anyio.Semaphore(1), +} + + +async def context_to_step_result(context: ConnectorContext) -> StepResult: + if context.state == ContextState.SUCCESSFUL: + return await NoOpStep(context, StepStatus.SUCCESS).run() + + if context.state == ContextState.FAILURE: + return await NoOpStep(context, StepStatus.FAILURE).run() + + if context.state == ContextState.ERROR: + return await NoOpStep(context, StepStatus.FAILURE).run() + + raise ValueError(f"Could not convert context state: {context.state} to step status") + + +# HACK: This is to avoid wrapping the whole pipeline in a dagger pipeline to avoid instability just prior to launch +# TODO (ben): Refactor run_connectors_pipelines to wrap the whole pipeline in a dagger pipeline once Steps are refactored +async def run_report_complete_pipeline(dagger_client: dagger.Client, contexts: List[ConnectorContext]) -> List[ConnectorContext]: + """Create and Save a report representing the run of the encompassing pipeline. + + This is to denote when the pipeline is complete, useful for long running pipelines like nightlies. + """ + + if not contexts: + return [] + + # Repurpose the first context to be the pipeline upload context to preserve timestamps + first_connector_context = contexts[0] + + pipeline_name = f"Report upload {first_connector_context.report_output_prefix}" + first_connector_context.pipeline_name = pipeline_name + + # Transform contexts into a list of steps + steps_results = [await context_to_step_result(context) for context in contexts] + + report = Report( + name=pipeline_name, + pipeline_context=first_connector_context, + steps_results=steps_results, + filename="complete", + ) + + return await report.save() + + +async def run_connectors_pipelines( + contexts: List[ConnectorContext], + connector_pipeline: Callable, + pipeline_name: str, + concurrency: int, + dagger_logs_path: Optional[Path], + execute_timeout: Optional[int], + *args, +) -> List[ConnectorContext]: + """Run a connector pipeline for all the connector contexts.""" + + default_connectors_semaphore = anyio.Semaphore(concurrency) + dagger_logs_output = sys.stderr if not dagger_logs_path else create_and_open_file(dagger_logs_path) + async with dagger.Connection(Config(log_output=dagger_logs_output, execute_timeout=execute_timeout)) as dagger_client: + # HACK: This is to get a long running dockerd service to be shared across all the connectors pipelines + # Using the "normal" service binding leads to restart of dockerd during pipeline run that can cause corrupted docker state + # See https://github.com/airbytehq/airbyte/issues/27233 + dockerd_service = environments.with_global_dockerd_service(dagger_client) + async with anyio.create_task_group() as tg_main: + tg_main.start_soon(dockerd_service.sync) + await anyio.sleep(10) # Wait for the docker service to be ready + async with anyio.create_task_group() as tg_connectors: + for context in contexts: + context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") + context.dockerd_service = dockerd_service + tg_connectors.start_soon( + connector_pipeline, + context, + CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), + *args, + ) + # When the connectors pipelines are done, we can stop the dockerd service + tg_main.cancel_scope.cancel() + await run_report_complete_pipeline(dagger_client, contexts) + + return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py new file mode 100644 index 000000000000..d5288ef05ba9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py @@ -0,0 +1,340 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import functools +import uuid +from pathlib import Path +from typing import Optional, Set + +import dagger +from pipelines.actions.environments import with_pip_packages, with_poetry_module, with_python_base +from pipelines.bases import Report, Step, StepResult +from pipelines.contexts import PipelineContext +from pipelines.helpers.steps import run_steps +from pipelines.utils import DAGGER_CONFIG, METADATA_FILE_NAME, METADATA_ICON_FILE_NAME, execute_concurrently, get_secret_host_variable + +METADATA_DIR = "airbyte-ci/connectors/metadata_service" +METADATA_LIB_MODULE_PATH = "lib" +METADATA_ORCHESTRATOR_MODULE_PATH = "orchestrator" + +# HELPERS + + +def get_metadata_file_from_path(context: PipelineContext, metadata_path: Path) -> dagger.File: + if metadata_path.is_file() and metadata_path.name != METADATA_FILE_NAME: + raise ValueError(f"The metadata file name is not {METADATA_FILE_NAME}, it is {metadata_path.name} .") + if metadata_path.is_dir(): + metadata_path = metadata_path / METADATA_FILE_NAME + if not metadata_path.exists(): + raise FileNotFoundError(f"{str(metadata_path)} does not exist.") + return context.get_repo_dir(str(metadata_path.parent), include=[METADATA_FILE_NAME]).file(METADATA_FILE_NAME) + + +def get_metadata_icon_file_from_path(context: PipelineContext, metadata_icon_path: Path) -> dagger.File: + return context.get_repo_dir(str(metadata_icon_path.parent), include=[METADATA_ICON_FILE_NAME]).file(METADATA_ICON_FILE_NAME) + + +# STEPS + + +class PoetryRun(Step): + def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, module_path: str): + self.title = title + super().__init__(context) + self.parent_dir = self.context.get_repo_dir(parent_dir_path) + self.module_path = module_path + self.poetry_run_container = with_poetry_module(self.context, self.parent_dir, self.module_path).with_entrypoint(["poetry", "run"]) + + async def _run(self, poetry_run_args: list) -> StepResult: + poetry_run_exec = self.poetry_run_container.with_exec(poetry_run_args) + return await self.get_step_result(poetry_run_exec) + + +class MetadataValidation(PoetryRun): + def __init__(self, context: PipelineContext, metadata_path: Path): + title = f"Validate {metadata_path}" + super().__init__(context, title, METADATA_DIR, METADATA_LIB_MODULE_PATH) + self.poetry_run_container = self.poetry_run_container.with_mounted_file( + METADATA_FILE_NAME, get_metadata_file_from_path(context, metadata_path) + ) + + async def _run(self) -> StepResult: + return await super()._run(["metadata_service", "validate", METADATA_FILE_NAME]) + + +class MetadataUpload(PoetryRun): + # When the metadata service exits with this code, it means the metadata is valid but the upload was skipped because the metadata is already uploaded + skipped_exit_code = 5 + + def __init__( + self, + context: PipelineContext, + metadata_path: Path, + metadata_bucket_name: str, + metadata_service_gcs_credentials_secret: dagger.Secret, + docker_hub_username_secret: dagger.Secret, + docker_hub_password_secret: dagger.Secret, + pre_release: bool = False, + pre_release_tag: Optional[str] = None, + ): + title = f"Upload {metadata_path}" + self.gcs_bucket_name = metadata_bucket_name + self.pre_release = pre_release + self.pre_release_tag = pre_release_tag + super().__init__(context, title, METADATA_DIR, METADATA_LIB_MODULE_PATH) + + # Ensure the icon file is included in the upload + base_container = self.poetry_run_container.with_file(METADATA_FILE_NAME, get_metadata_file_from_path(context, metadata_path)) + metadata_icon_path = metadata_path.parent / METADATA_ICON_FILE_NAME + if metadata_icon_path.exists(): + base_container = base_container.with_file( + METADATA_ICON_FILE_NAME, get_metadata_icon_file_from_path(context, metadata_icon_path) + ) + + self.poetry_run_container = ( + base_container.with_secret_variable("DOCKER_HUB_USERNAME", docker_hub_username_secret) + .with_secret_variable("DOCKER_HUB_PASSWORD", docker_hub_password_secret) + .with_secret_variable("GCS_CREDENTIALS", metadata_service_gcs_credentials_secret) + # The cache buster ensures we always run the upload command (in case of remote bucket change) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + ) + + async def _run(self) -> StepResult: + upload_command = ["metadata_service", "upload", METADATA_FILE_NAME, self.gcs_bucket_name] + + if self.pre_release: + upload_command += ["--prerelease", self.pre_release_tag] + + return await super()._run(upload_command) + + +class DeployOrchestrator(Step): + title = "Deploy Metadata Orchestrator to Dagster Cloud" + deploy_dagster_command = [ + "dagster-cloud", + "serverless", + "deploy-python-executable", + "--location-name", + "metadata_service_orchestrator", + "--location-file", + "dagster_cloud.yaml", + "--organization", + "airbyte-connectors", + "--deployment", + "prod", + "--python-version", + "3.9", + ] + + async def _run(self) -> StepResult: + parent_dir = self.context.get_repo_dir(METADATA_DIR) + python_base = with_python_base(self.context, "3.9") + python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud==1.2.6", "pydantic==1.10.6", "poetry2setup==1.1.0"]) + dagster_cloud_api_token_secret: dagger.Secret = get_secret_host_variable( + self.context.dagger_client, "DAGSTER_CLOUD_METADATA_API_TOKEN" + ) + + container_to_run = ( + python_with_dependencies.with_mounted_directory("/src", parent_dir) + .with_secret_variable("DAGSTER_CLOUD_API_TOKEN", dagster_cloud_api_token_secret) + .with_workdir(f"/src/{METADATA_ORCHESTRATOR_MODULE_PATH}") + .with_exec(["/bin/sh", "-c", "poetry2setup >> setup.py"]) + .with_exec(self.deploy_dagster_command) + ) + return await self.get_step_result(container_to_run) + + +class TestOrchestrator(PoetryRun): + def __init__(self, context: PipelineContext): + super().__init__( + context=context, + title="Test Metadata Orchestrator", + parent_dir_path=METADATA_DIR, + module_path=METADATA_ORCHESTRATOR_MODULE_PATH, + ) + + async def _run(self) -> StepResult: + return await super()._run(["pytest"]) + + +# PIPELINES + + +async def run_metadata_validation_pipeline( + is_local: bool, + git_branch: str, + git_revision: str, + gha_workflow_run_url: Optional[str], + dagger_logs_url: Optional[str], + pipeline_start_timestamp: Optional[int], + ci_context: Optional[str], + metadata_to_validate: Set[Path], +) -> bool: + metadata_pipeline_context = PipelineContext( + pipeline_name="Validate metadata.yaml files", + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ) + + async with dagger.Connection(DAGGER_CONFIG) as dagger_client: + metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) + async with metadata_pipeline_context: + validation_steps = [MetadataValidation(metadata_pipeline_context, metadata_path).run for metadata_path in metadata_to_validate] + + results = await execute_concurrently(validation_steps, concurrency=10) + metadata_pipeline_context.report = Report( + pipeline_context=metadata_pipeline_context, steps_results=results, name="METADATA VALIDATION RESULTS" + ) + + return metadata_pipeline_context.report.success + + +async def run_metadata_lib_test_pipeline( + is_local: bool, + git_branch: str, + git_revision: str, + gha_workflow_run_url: Optional[str], + dagger_logs_url: Optional[str], + pipeline_start_timestamp: Optional[int], + ci_context: Optional[str], +) -> bool: + metadata_pipeline_context = PipelineContext( + pipeline_name="Metadata Service Lib Unit Test Pipeline", + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ) + + async with dagger.Connection(DAGGER_CONFIG) as dagger_client: + metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) + async with metadata_pipeline_context: + test_lib_step = PoetryRun( + context=metadata_pipeline_context, + title="Test Metadata Service Lib", + parent_dir_path=METADATA_DIR, + module_path=METADATA_LIB_MODULE_PATH, + ) + result = await test_lib_step.run(["pytest"]) + metadata_pipeline_context.report = Report( + pipeline_context=metadata_pipeline_context, steps_results=[result], name="METADATA LIB TEST RESULTS" + ) + + return metadata_pipeline_context.report.success + + +async def run_metadata_orchestrator_test_pipeline( + is_local: bool, + git_branch: str, + git_revision: str, + gha_workflow_run_url: Optional[str], + dagger_logs_url: Optional[str], + pipeline_start_timestamp: Optional[int], + ci_context: Optional[str], +) -> bool: + metadata_pipeline_context = PipelineContext( + pipeline_name="Metadata Service Orchestrator Unit Test Pipeline", + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ) + + async with dagger.Connection(DAGGER_CONFIG) as dagger_client: + metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) + async with metadata_pipeline_context: + test_orch_step = TestOrchestrator(context=metadata_pipeline_context) + result = await test_orch_step.run() + metadata_pipeline_context.report = Report( + pipeline_context=metadata_pipeline_context, steps_results=[result], name="METADATA ORCHESTRATOR TEST RESULTS" + ) + + return metadata_pipeline_context.report.success + + +async def run_metadata_upload_pipeline( + is_local: bool, + git_branch: str, + git_revision: str, + gha_workflow_run_url: Optional[str], + dagger_logs_url: Optional[str], + pipeline_start_timestamp: Optional[int], + ci_context: Optional[str], + metadata_to_upload: Set[Path], + gcs_bucket_name: str, +) -> bool: + pipeline_context = PipelineContext( + pipeline_name="Metadata Upload Pipeline", + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ) + + async with dagger.Connection(DAGGER_CONFIG) as dagger_client: + pipeline_context.dagger_client = dagger_client.pipeline(pipeline_context.pipeline_name) + async with pipeline_context: + get_secret = functools.partial(get_secret_host_variable, pipeline_context.dagger_client) + results = await execute_concurrently( + [ + MetadataUpload( + context=pipeline_context, + metadata_service_gcs_credentials_secret=get_secret("GCS_CREDENTIALS"), + docker_hub_username_secret=get_secret("DOCKER_HUB_USERNAME"), + docker_hub_password_secret=get_secret("DOCKER_HUB_PASSWORD"), + metadata_bucket_name=gcs_bucket_name, + metadata_path=metadata_path, + ).run + for metadata_path in metadata_to_upload + ] + ) + pipeline_context.report = Report(pipeline_context, results, name="METADATA UPLOAD RESULTS") + + return pipeline_context.report.success + + +async def run_metadata_orchestrator_deploy_pipeline( + is_local: bool, + git_branch: str, + git_revision: str, + gha_workflow_run_url: Optional[str], + dagger_logs_url: Optional[str], + pipeline_start_timestamp: Optional[int], + ci_context: Optional[str], +) -> bool: + metadata_pipeline_context = PipelineContext( + pipeline_name="Metadata Service Orchestrator Unit Test Pipeline", + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ) + + async with dagger.Connection(DAGGER_CONFIG) as dagger_client: + metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) + + async with metadata_pipeline_context: + steps = [TestOrchestrator(context=metadata_pipeline_context), DeployOrchestrator(context=metadata_pipeline_context)] + steps_results = await run_steps(steps) + metadata_pipeline_context.report = Report( + pipeline_context=metadata_pipeline_context, steps_results=steps_results, name="METADATA ORCHESTRATOR DEPLOY RESULTS" + ) + return metadata_pipeline_context.report.success diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/publish.py b/airbyte-ci/connectors/pipelines/pipelines/publish.py similarity index 76% rename from tools/ci_connector_ops/ci_connector_ops/pipelines/publish.py rename to airbyte-ci/connectors/pipelines/pipelines/publish.py index 27d3a98cdd44..2948cc9ef86a 100644 --- a/tools/ci_connector_ops/ci_connector_ops/pipelines/publish.py +++ b/airbyte-ci/connectors/pipelines/pipelines/publish.py @@ -1,20 +1,20 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json import uuid from typing import List, Tuple import anyio from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification -from ci_connector_ops.pipelines import builds, consts -from ci_connector_ops.pipelines.actions import environments -from ci_connector_ops.pipelines.actions.remote_storage import upload_to_gcs -from ci_connector_ops.pipelines.bases import ConnectorReport, Step, StepResult, StepStatus -from ci_connector_ops.pipelines.contexts import PublishConnectorContext -from ci_connector_ops.pipelines.pipelines import metadata -from ci_connector_ops.pipelines.utils import with_exit_code, with_stderr, with_stdout -from dagger import Container, File, ImageLayerCompression, QueryError +from dagger import Container, ExecError, File, ImageLayerCompression, QueryError +from pipelines import builds, consts +from pipelines.actions import environments +from pipelines.actions.remote_storage import upload_to_gcs +from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus +from pipelines.contexts import PublishConnectorContext +from pipelines.pipelines import metadata from pydantic import ValidationError @@ -22,7 +22,7 @@ class CheckConnectorImageDoesNotExist(Step): title = "Check if the connector docker image does not exist on the registry." async def _run(self) -> StepResult: - docker_repository, docker_tag = self.context.docker_image_name.split(":") + docker_repository, docker_tag = self.context.docker_image.split(":") crane_ls = ( environments.with_crane( self.context, @@ -30,20 +30,19 @@ async def _run(self) -> StepResult: .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_exec(["ls", docker_repository]) ) - crane_ls_exit_code = await with_exit_code(crane_ls) - crane_ls_stderr = await with_stderr(crane_ls) - crane_ls_stdout = await with_stdout(crane_ls) - if crane_ls_exit_code != 0: - if "NAME_UNKNOWN" in crane_ls_stderr: + try: + crane_ls_stdout = await crane_ls.stdout() + except ExecError as e: + if "NAME_UNKNOWN" in e.stderr: return StepResult(self, status=StepStatus.SUCCESS, stdout=f"The docker repository {docker_repository} does not exist.") else: - return StepResult(self, status=StepStatus.FAILURE, stderr=crane_ls_stderr, stdout=crane_ls_stdout) + return StepResult(self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout) else: # The docker repo exists and ls was successful existing_tags = crane_ls_stdout.split("\n") docker_tag_already_exists = docker_tag in existing_tags if docker_tag_already_exists: - return StepResult(self, status=StepStatus.SKIPPED, stderr=f"{self.context.docker_image_name} already exists.") - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image_name}.") + return StepResult(self, status=StepStatus.SKIPPED, stderr=f"{self.context.docker_image} already exists.") + return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") class PushConnectorImageToRegistry(Step): @@ -51,12 +50,12 @@ class PushConnectorImageToRegistry(Step): @property def latest_docker_image_name(self): - return f"{self.context.metadata['dockerRepository']}:latest" + return f"{self.context.docker_repository}:latest" async def _run(self, built_containers_per_platform: List[Container], attempts: int = 3) -> StepResult: try: image_ref = await built_containers_per_platform[0].publish( - f"docker.io/{self.context.docker_image_name}", + f"docker.io/{self.context.docker_image}", platform_variants=built_containers_per_platform[1:], forced_compression=ImageLayerCompression.Gzip, ) @@ -70,7 +69,7 @@ async def _run(self, built_containers_per_platform: List[Container], attempts: i except QueryError as e: if attempts > 0: self.context.logger.error(str(e)) - self.context.logger.warn(f"Failed to publish {self.context.docker_image_name}. Retrying. {attempts} attempts left.") + self.context.logger.warn(f"Failed to publish {self.context.docker_image}. Retrying. {attempts} attempts left.") await anyio.sleep(5) return await self._run(built_containers_per_platform, attempts - 1) return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) @@ -88,43 +87,41 @@ async def check_if_image_only_has_gzip_layers(self) -> bool: """ for platform in consts.BUILD_PLATFORMS: inspect = environments.with_crane(self.context).with_exec( - ["manifest", "--platform", f"{str(platform)}", f"docker.io/{self.context.docker_image_name}"] + ["manifest", "--platform", f"{str(platform)}", f"docker.io/{self.context.docker_image}"] ) - inspect_exit_code = await with_exit_code(inspect) - inspect_stderr = await with_stderr(inspect) - inspect_stdout = await with_stdout(inspect) - if inspect_exit_code != 0: - raise Exception(f"Failed to inspect {self.context.docker_image_name}: {inspect_stderr}") + try: + inspect_stdout = await inspect.stdout() + except ExecError as e: + raise Exception(f"Failed to inspect {self.context.docker_image}: {e.stderr}") from e try: for layer in json.loads(inspect_stdout)["layers"]: - if not layer["mediaType"].endswith(".gzip"): + if not layer["mediaType"].endswith("gzip"): return False return True except (KeyError, json.JSONDecodeError) as e: - raise Exception(f"Failed to parse manifest for {self.context.docker_image_name}: {inspect_stdout}") from e + raise Exception(f"Failed to parse manifest for {self.context.docker_image}: {inspect_stdout}") from e async def _run(self, attempt: int = 3) -> StepResult: try: - exit_code = await with_exit_code( - self.context.dagger_client.container().from_(f"docker.io/{self.context.docker_image_name}").with_exec(["spec"]) - ) - if exit_code != 0: + try: + await self.context.dagger_client.container().from_(f"docker.io/{self.context.docker_image}").with_exec(["spec"]) + except ExecError: if attempt > 0: await anyio.sleep(10) return await self._run(attempt - 1) else: - return StepResult(self, status=StepStatus.FAILURE, stderr=f"Failed to pull {self.context.docker_image_name}") + return StepResult(self, status=StepStatus.FAILURE, stderr=f"Failed to pull {self.context.docker_image}") if not await self.check_if_image_only_has_gzip_layers(): return StepResult( self, status=StepStatus.FAILURE, - stderr=f"Image {self.context.docker_image_name} does not only have gzip compressed layers. Please rebuild the connector with Docker < 21.", + stderr=f"Image {self.context.docker_image} does not only have gzip compressed layers. Please rebuild the connector with Docker < 21.", ) else: return StepResult( self, status=StepStatus.SUCCESS, - stdout=f"Pulled {self.context.docker_image_name} and validated it has gzip only compressed layers and we can run spec on it.", + stdout=f"Pulled {self.context.docker_image} and validated it has gzip only compressed layers and we can run spec on it.", ) except QueryError as e: if attempt > 0: @@ -144,7 +141,7 @@ class UploadSpecToCache(Step): @property def spec_key_prefix(self): - return "specs/" + self.context.docker_image_name.replace(":", "/") + return "specs/" + self.context.docker_image.replace(":", "/") @property def cloud_spec_key(self): @@ -177,8 +174,8 @@ async def _get_connector_spec(self, connector: Container, deployment_mode: str) spec_output = await connector.with_env_variable("DEPLOYMENT_MODE", deployment_mode).with_exec(["spec"]).stdout() return self._parse_spec_output(spec_output) - def _get_spec_as_file(self, spec: str, name="spec_to_cache.json") -> File: - return self.context.get_connector_dir().with_new_file(name, spec).file(name) + async def _get_spec_as_file(self, spec: str, name="spec_to_cache.json") -> File: + return (await self.context.get_connector_dir()).with_new_file(name, contents=spec).file(name) async def _run(self, built_connector: Container) -> StepResult: try: @@ -187,10 +184,10 @@ async def _run(self, built_connector: Container) -> StepResult: except InvalidSpecOutputError as e: return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) - specs_to_uploads: List[Tuple[str, File]] = [(self.oss_spec_key, self._get_spec_as_file(oss_spec))] + specs_to_uploads: List[Tuple[str, File]] = [(self.oss_spec_key, await self._get_spec_as_file(oss_spec))] if oss_spec != cloud_spec: - specs_to_uploads.append(self.cloud_spec_key, self._get_spec_as_file(cloud_spec, "cloud_spec_to_cache.json")) + specs_to_uploads.append((self.cloud_spec_key, await self._get_spec_as_file(cloud_spec, "cloud_spec_to_cache.json"))) for key, file in specs_to_uploads: exit_code, stdout, stderr = await upload_to_gcs( @@ -220,17 +217,16 @@ async def run_connector_publish_pipeline(context: PublishConnectorContext, semap ConnectorReport: The reports holding publish results. """ - if not context.pre_release: - metadata_upload_step = metadata.MetadataUpload( - context=context, - metadata_service_gcs_credentials_secret=context.metadata_service_gcs_credentials_secret, - docker_hub_username_secret=context.docker_hub_username_secret, - docker_hub_password_secret=context.docker_hub_password_secret, - metadata_bucket_name=context.metadata_bucket_name, - metadata_path=context.metadata_path, - ) - else: - metadata_upload_step = None + metadata_upload_step = metadata.MetadataUpload( + context=context, + metadata_service_gcs_credentials_secret=context.metadata_service_gcs_credentials_secret, + docker_hub_username_secret=context.docker_hub_username_secret, + docker_hub_password_secret=context.docker_hub_password_secret, + metadata_bucket_name=context.metadata_bucket_name, + metadata_path=context.metadata_path, + pre_release=context.pre_release, + pre_release_tag=context.docker_image_tag, + ) def create_connector_report(results: List[StepResult]) -> ConnectorReport: report = ConnectorReport(context, results, name="PUBLISH RESULTS") @@ -239,7 +235,7 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: async with semaphore: async with context: - # TODO add a strucutre to hold the results of each step. and perform skips and failures. + # TODO add a strucutre to hold the results of each step. and perform skips and failures results = [] @@ -255,11 +251,11 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. # We also need to upload the spec to the spec cache bucket. - if check_connector_image_results.status is StepStatus.SKIPPED and not context.pre_release: + if check_connector_image_results.status is StepStatus.SKIPPED: context.logger.info( "The connector version is already published. Let's upload metadata.yaml and spec to GCS even if no version bump happened." ) - already_published_connector = context.dagger_client.container().from_(context.docker_image_from_metadata) + already_published_connector = context.dagger_client.container().from_(context.docker_image) upload_to_spec_cache_results = await UploadSpecToCache(context).run(already_published_connector) results.append(upload_to_spec_cache_results) if upload_to_spec_cache_results.status is not StepStatus.SUCCESS: @@ -300,10 +296,8 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: if upload_to_spec_cache_results.status is not StepStatus.SUCCESS: return create_connector_report(results) - if not context.pre_release: - # Only upload to metadata service bucket if the connector is not a pre-release. - metadata_upload_results = await metadata_upload_step.run() - results.append(metadata_upload_results) + metadata_upload_results = await metadata_upload_step.run() + results.append(metadata_upload_results) return create_connector_report(results) @@ -314,8 +308,10 @@ def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishCon Non strict-encrypt variant reference the strict-encrypt variant in their metadata file for cloud. So if we publish the non strict-encrypt variant first, the metadata upload will fail if the strict-encrypt variant is not published yet. As strict-encrypt variant are often modified in the same PR as the non strict-encrypt variant, we want to publish them first. - This is an hacky approach: as connector names with -strict-encrypt/secure prefix are longer, - they will be sorted first with our reverse sort below. """ - return sorted(contexts, key=lambda context: context.connector.technical_name, reverse=True) + def is_secure_variant(context: PublishConnectorContext) -> bool: + SECURE_VARIANT_KEYS = ["secure", "strict-encrypt"] + return any(key in context.connector.technical_name for key in SECURE_VARIANT_KEYS) + + return sorted(contexts, key=lambda context: (is_secure_variant(context), context.connector.technical_name), reverse=True) diff --git a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py new file mode 100644 index 000000000000..da36bb015ebd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import importlib.metadata +import os + +import sentry_sdk +from connector_ops.utils import Connector + + +def initialize(): + if "SENTRY_DSN" in os.environ: + sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + before_send=before_send, + release=f"pipelines@{importlib.metadata.version('pipelines')}", + ) + + +def before_send(event, hint): + # Ignore logged errors that do not contain an exception + if "log_record" in hint and "exc_info" not in hint: + return None + + return event + + +def with_step_context(func): + def wrapper(self, *args, **kwargs): + with sentry_sdk.configure_scope() as scope: + step_name = self.__class__.__name__ + scope.set_tag("pipeline_step", step_name) + scope.set_context( + "Pipeline Step", + { + "name": step_name, + "step_title": self.title, + "max_retries": self.max_retries, + "max_duration": self.max_duration, + "retry_count": self.retry_count, + }, + ) + + if hasattr(self.context, "connector"): + connector: Connector = self.context.connector + scope.set_tag("connector", connector.technical_name) + scope.set_context( + "Connector", + { + "name": connector.name, + "technical_name": connector.technical_name, + "language": connector.language, + "version": connector.version, + "support_level": connector.support_level, + }, + ) + + return func(self, *args, **kwargs) + + return wrapper + + +def with_command_context(func): + def wrapper(self, ctx, *args, **kwargs): + with sentry_sdk.configure_scope() as scope: + scope.set_tag("pipeline_command", self.name) + scope.set_context( + "Pipeline Command", + { + "name": self.name, + "params": self.params, + }, + ) + + scope.set_context("Click Context", ctx.obj) + scope.set_tag("git_branch", ctx.obj.get("git_branch", "unknown")) + scope.set_tag("git_revision", ctx.obj.get("git_revision", "unknown")) + + return func(self, ctx, *args, **kwargs) + + return wrapper diff --git a/airbyte-ci/connectors/pipelines/pipelines/slack.py b/airbyte-ci/connectors/pipelines/pipelines/slack.py new file mode 100644 index 000000000000..ecc4e23e0996 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/slack.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json + +import requests +from pipelines import main_logger + + +def send_message_to_webhook(message: str, channel: str, webhook: str) -> dict: + payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message} + response = requests.post(webhook, data={"payload": json.dumps(payload)}) + + # log if the request failed, but don't fail the pipeline + if not response.ok: + main_logger.error(f"Failed to send message to slack webhook: {response.text}") + + return response diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py new file mode 100644 index 000000000000..0d3d26c27e1a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""This module groups factory like functions to dispatch tests steps according to the connector under test language.""" + +import itertools +from typing import List + +import anyio +import asyncer +from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage +from pipelines.bases import ConnectorReport, StepResult +from pipelines.contexts import ConnectorContext +from pipelines.pipelines.metadata import MetadataValidation +from pipelines.tests import java_connectors, python_connectors +from pipelines.tests.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck + +LANGUAGE_MAPPING = { + "run_all_tests": { + ConnectorLanguage.PYTHON: python_connectors.run_all_tests, + ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, + ConnectorLanguage.JAVA: java_connectors.run_all_tests, + }, + "run_code_format_checks": { + ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, + ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, + # ConnectorLanguage.JAVA: java_connectors.run_code_format_checks + }, +} + + +async def run_metadata_validation(context: ConnectorContext) -> List[StepResult]: + """Run the metadata validation on a connector. + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of the metadata validation steps. + """ + return [await MetadataValidation(context, context.connector.code_directory / METADATA_FILE_NAME).run()] + + +async def run_version_checks(context: ConnectorContext) -> List[StepResult]: + """Run the version checks on a connector. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of the version checks steps. + """ + return [await VersionFollowsSemverCheck(context).run(), await VersionIncrementCheck(context).run()] + + +async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: + """Run the QA checks on a connector. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of the QA checks steps. + """ + return [await QaChecks(context).run()] + + +async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: + """Run the code format checks according to the connector language. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of the code format checks steps. + """ + if _run_code_format_checks := LANGUAGE_MAPPING["run_code_format_checks"].get(context.connector.language): + return await _run_code_format_checks(context) + else: + context.logger.warning(f"No code format checks defined for connector language {context.connector.language}!") + return [] + + +async def run_all_tests(context: ConnectorContext) -> List[StepResult]: + """Run all the tests steps according to the connector language. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of the tests steps. + """ + if _run_all_tests := LANGUAGE_MAPPING["run_all_tests"].get(context.connector.language): + return await _run_all_tests(context) + else: + context.logger.warning(f"No tests defined for connector language {context.connector.language}!") + return [] + + +async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: + """Run a test pipeline for a single connector. + + A visual DAG can be found on the README.md file of the pipelines modules. + + Args: + context (ConnectorContext): The initialized connector context. + + Returns: + ConnectorReport: The test reports holding tests results. + """ + async with semaphore: + async with context: + async with asyncer.create_task_group() as task_group: + tasks = [ + task_group.soonify(run_all_tests)(context), + task_group.soonify(run_code_format_checks)(context), + ] + if not context.code_tests_only: + tasks += [ + task_group.soonify(run_metadata_validation)(context), + task_group.soonify(run_version_checks)(context), + task_group.soonify(run_qa_checks)(context), + ] + results = list(itertools.chain(*(task.value for task in tasks))) + context.report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") + + return context.report diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/common.py b/airbyte-ci/connectors/pipelines/pipelines/tests/common.py new file mode 100644 index 000000000000..374152d87e8a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/common.py @@ -0,0 +1,283 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups steps made to run tests agnostic to a connector language.""" + +import datetime +import os +from abc import ABC, abstractmethod +from functools import cached_property +from typing import ClassVar, List, Optional + +import requests +import semver +import yaml +from connector_ops.utils import Connector +from dagger import Container, Directory, File +from pipelines import hacks +from pipelines.actions import environments +from pipelines.bases import CIContext, PytestStep, Step, StepResult, StepStatus +from pipelines.utils import METADATA_FILE_NAME + + +class VersionCheck(Step, ABC): + """A step to validate the connector version was bumped if files were modified""" + + GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors" + failure_message: ClassVar + should_run = True + + @property + def github_master_metadata_url(self): + return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}" + + @cached_property + def master_metadata(self) -> Optional[dict]: + response = requests.get(self.github_master_metadata_url) + + # New connectors will not have a metadata file in master + if not response.ok: + return None + return yaml.safe_load(response.text) + + @property + def master_connector_version(self) -> semver.Version: + metadata = self.master_metadata + if not metadata: + return semver.Version.parse("0.0.0") + + return semver.Version.parse(str(metadata["data"]["dockerImageTag"])) + + @property + def current_connector_version(self) -> semver.Version: + return semver.Version.parse(str(self.context.metadata["dockerImageTag"])) + + @property + def success_result(self) -> StepResult: + return StepResult(self, status=StepStatus.SUCCESS) + + @property + def failure_result(self) -> StepResult: + return StepResult(self, status=StepStatus.FAILURE, stderr=self.failure_message) + + @abstractmethod + def validate(self) -> StepResult: + raise NotImplementedError() + + async def _run(self) -> StepResult: + if not self.should_run: + return StepResult(self, status=StepStatus.SKIPPED, stdout="No modified files required a version bump.") + if self.context.ci_context in [CIContext.MASTER, CIContext.NIGHTLY_BUILDS]: + return StepResult(self, status=StepStatus.SKIPPED, stdout="Version check are not running in master context.") + try: + return self.validate() + except (requests.HTTPError, ValueError, TypeError) as e: + return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + + +class VersionIncrementCheck(VersionCheck): + title = "Connector version increment check" + + BYPASS_CHECK_FOR = [ + METADATA_FILE_NAME, + "acceptance-test-config.yml", + "README.md", + "bootstrap.md", + ".dockerignore", + "unit_tests", + "integration_tests", + "src/test", + "src/test-integration", + "src/test-performance", + "build.gradle", + ] + + @property + def failure_message(self) -> str: + return f"The dockerImageTag in {METADATA_FILE_NAME} was not incremented. The files you modified should lead to a version bump. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" + + @property + def should_run(self) -> bool: + for filename in self.context.modified_files: + relative_path = str(filename).replace(str(self.context.connector.code_directory) + "/", "") + if not any([relative_path.startswith(to_bypass) for to_bypass in self.BYPASS_CHECK_FOR]): + return True + return False + + def validate(self) -> StepResult: + if not self.current_connector_version > self.master_connector_version: + return self.failure_result + return self.success_result + + +class VersionFollowsSemverCheck(VersionCheck): + title = "Connector version semver check" + + @property + def failure_message(self) -> str: + return f"The dockerImageTag in {METADATA_FILE_NAME} is not following semantic versioning or was decremented. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" + + def validate(self) -> StepResult: + try: + if not self.current_connector_version >= self.master_connector_version: + return self.failure_result + except ValueError: + return self.failure_result + return self.success_result + + +class QaChecks(Step): + """A step to run QA checks for a connector.""" + + title = "QA checks" + + async def _run(self) -> StepResult: + """Run QA checks on a connector. + + The QA checks are defined in this module: + https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connector_ops/connector_ops/qa_checks.py + + Args: + context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. + Returns: + StepResult: Failure or success of the QA checks with stdout and stderr. + """ + connector_ops = await environments.with_connector_ops(self.context) + include = [ + str(self.context.connector.code_directory), + str(self.context.connector.documentation_file_path), + str(self.context.connector.migration_guide_file_path), + str(self.context.connector.icon_path), + ] + if ( + self.context.connector.technical_name.endswith("strict-encrypt") + or self.context.connector.technical_name == "source-file-secure" + ): + original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) + include += [ + str(original_connector.code_directory), + str(original_connector.documentation_file_path), + str(original_connector.icon_path), + ] + + filtered_repo = self.context.get_repo_dir( + include=include, + ) + + qa_checks = ( + connector_ops.with_mounted_directory("/airbyte", filtered_repo) + .with_workdir("/airbyte") + .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"]) + ) + + return await self.get_step_result(qa_checks) + + +class AcceptanceTests(PytestStep): + """A step to run acceptance tests for a connector if it has an acceptance test config file.""" + + title = "Acceptance tests" + CONTAINER_TEST_INPUT_DIRECTORY = "/test_input" + CONTAINER_SECRETS_DIRECTORY = "/test_input/secrets" + + @property + def base_cat_command(self) -> List[str]: + return [ + "python", + "-m", + "pytest", + "-p", + "connector_acceptance_test.plugin", + "--acceptance-test-config", + self.CONTAINER_TEST_INPUT_DIRECTORY, + ] + + async def get_cat_command(self, connector_dir: Directory) -> List[str]: + """ + Connectors can optionally setup or teardown resources before and after the acceptance tests are run. + This is done via the acceptance.py file in their integration_tests directory. + We append this module as a plugin the acceptance will use. + """ + cat_command = self.base_cat_command + if "integration_tests" in await connector_dir.entries(): + if "acceptance.py" in await connector_dir.directory("integration_tests").entries(): + cat_command += ["-p", "integration_tests.acceptance"] + return cat_command + + async def _run(self, connector_under_test_image_tar: File) -> StepResult: + """Run the acceptance test suite on a connector dev image. Build the connector acceptance test image if the tag is :dev. + + Args: + connector_under_test_image_tar (File): The file holding the tar archive of the connector image. + + Returns: + StepResult: Failure or success of the acceptances tests with stdout and stderr. + """ + if not self.context.connector.acceptance_test_config: + return StepResult(self, StepStatus.SKIPPED) + connector_dir = await self.context.get_connector_dir() + cat_container = await self._build_connector_acceptance_test(connector_under_test_image_tar, connector_dir) + cat_command = await self.get_cat_command(connector_dir) + cat_container = cat_container.with_(hacks.never_fail_exec(cat_command)) + step_result = await self.get_step_result(cat_container) + secret_dir = cat_container.directory(self.CONTAINER_SECRETS_DIRECTORY) + + if secret_files := await secret_dir.entries(): + for file_path in secret_files: + if file_path.startswith("updated_configurations"): + self.context.updated_secrets_dir = secret_dir + break + return step_result + + async def get_cache_buster(self, connector_under_test_image_tar: File) -> str: + """ + This bursts the CAT cached results everyday and on new version or image size change. + It's cool because in case of a partially failing nightly build the connectors that already ran CAT won't re-run CAT. + We keep the guarantee that a CAT runs everyday. + + Args: + connector_under_test_image_tar (File): The file holding the tar archive of the connector image. + Returns: + str: A string representing the cachebuster value. + """ + return ( + datetime.datetime.utcnow().strftime("%Y%m%d") + + self.context.connector.version + + str(await connector_under_test_image_tar.size()) + ) + + async def _build_connector_acceptance_test(self, connector_under_test_image_tar: File, test_input: Directory) -> Container: + """Create a container to run connector acceptance tests. + + Args: + connector_under_test_image_tar (File): The file containing the tar archive of the image of the connector under test. + test_input (Directory): The connector under test directory. + Returns: + Container: A container with connector acceptance tests installed. + """ + + if self.context.connector_acceptance_test_image.endswith(":dev"): + cat_container = self.context.connector_acceptance_test_source_dir.docker_build() + else: + cat_container = self.dagger_client.container().from_(self.context.connector_acceptance_test_image) + + cat_container = ( + cat_container.with_env_variable("RUN_IN_AIRBYTE_CI", "1") + .with_exec(["mkdir", "/dagger_share"], skip_entrypoint=True) + .with_env_variable("CACHEBUSTER", await self.get_cache_buster(connector_under_test_image_tar)) + .with_mounted_file("/dagger_share/connector_under_test_image.tar", connector_under_test_image_tar) + .with_env_variable("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH", "/dagger_share/connector_under_test_image.tar") + .with_workdir("/test_input") + .with_mounted_directory("/test_input", test_input) + .with_(environments.mounted_connector_secrets(self.context, secret_directory_path="/test_input/secrets")) + ) + if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: + self.context.logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") + cat_container = cat_container.with_env_variable( + "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" + ).with_unix_socket( + "/var/run/buildkit/buildkitd.sock", self.context.dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") + ) + + return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py new file mode 100644 index 000000000000..052547ac46e4 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups steps made to run tests for a specific Java connector given a test context.""" + +from typing import List, Optional + +import anyio +from dagger import File, QueryError +from pipelines.actions import environments, secrets +from pipelines.bases import StepResult, StepStatus +from pipelines.builds import LOCAL_BUILD_PLATFORM +from pipelines.builds.java_connectors import BuildConnectorDistributionTar, BuildConnectorImage +from pipelines.builds.normalization import BuildOrPullNormalization +from pipelines.contexts import ConnectorContext +from pipelines.gradle import GradleTask +from pipelines.tests.common import AcceptanceTests +from pipelines.utils import export_container_to_tarball + + +class IntegrationTests(GradleTask): + """A step to run integrations tests for Java connectors using the integrationTestJava Gradle task.""" + + gradle_task_name = "integrationTest" + DEFAULT_TASKS_TO_EXCLUDE = ["airbyteDocker"] + title = "Java Connector Integration Tests" + + async def _load_normalization_image(self, normalization_tar_file: File): + normalization_image_tag = f"{self.context.connector.normalization_repository}:dev" + self.context.logger.info("Load the normalization image to the docker host.") + await environments.load_image_to_docker_host(self.context, normalization_tar_file, normalization_image_tag) + self.context.logger.info("Successfully loaded the normalization image to the docker host.") + + async def _load_connector_image(self, connector_tar_file: File): + connector_image_tag = f"airbyte/{self.context.connector.technical_name}:dev" + self.context.logger.info("Load the connector image to the docker host") + await environments.load_image_to_docker_host(self.context, connector_tar_file, connector_image_tag) + self.context.logger.info("Successfully loaded the connector image to the docker host.") + + async def _run(self, connector_tar_file: File, normalization_tar_file: Optional[File]) -> StepResult: + try: + async with anyio.create_task_group() as tg: + if normalization_tar_file: + tg.start_soon(self._load_normalization_image, normalization_tar_file) + tg.start_soon(self._load_connector_image, connector_tar_file) + return await super()._run() + except QueryError as e: + return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + + +class UnitTests(GradleTask): + """A step to run unit tests for Java connectors.""" + + title = "Java Connector Unit Tests" + gradle_task_name = "test" + context: ConnectorContext + + @property + def gradle_task_options(self) -> tuple[str, ...]: + """Return the Gradle task options to use when running unit tests.""" + if self.context.fail_fast: + return ("--fail-fast",) + + return () + + +async def run_all_tests(context: ConnectorContext) -> List[StepResult]: + """Run all tests for a Java connectors. + + - Build the normalization image if the connector supports it. + - Run unit tests with Gradle. + - Build connector image with Gradle. + - Run integration and acceptance test in parallel using the built connector and normalization images. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of all the tests steps. + """ + context.connector_secrets = await secrets.get_connector_secrets(context) + step_results = [] + + unit_tests_results = await UnitTests(context).run() + step_results.append(unit_tests_results) + + if context.fail_fast and unit_tests_results.status is StepStatus.FAILURE: + return step_results + + build_distribution_tar_results = await BuildConnectorDistributionTar(context).run() + step_results.append(build_distribution_tar_results) + if build_distribution_tar_results.status is StepStatus.FAILURE: + return step_results + + build_connector_image_results = await BuildConnectorImage(context, LOCAL_BUILD_PLATFORM).run( + build_distribution_tar_results.output_artifact + ) + step_results.append(build_connector_image_results) + if build_connector_image_results.status is StepStatus.FAILURE: + return step_results + + if context.connector.supports_normalization: + normalization_image = f"{context.connector.normalization_repository}:dev" + context.logger.info(f"This connector supports normalization: will build {normalization_image}.") + build_normalization_results = await BuildOrPullNormalization(context, normalization_image, LOCAL_BUILD_PLATFORM).run() + normalization_container = build_normalization_results.output_artifact + normalization_tar_file, _ = await export_container_to_tarball( + context, normalization_container, tar_file_name=f"{context.connector.normalization_repository}_{context.git_revision}.tar" + ) + step_results.append(build_normalization_results) + else: + normalization_tar_file = None + + connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact) + + integration_tests_results = await IntegrationTests(context).run(connector_image_tar_file, normalization_tar_file) + step_results.append(integration_tests_results) + + acceptance_tests_results = await AcceptanceTests(context).run(connector_image_tar_file) + step_results.append(acceptance_tests_results) + return step_results diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py new file mode 100644 index 000000000000..8470fc6ff84b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py @@ -0,0 +1,161 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups steps made to run tests for a specific Python connector given a test context.""" + +from datetime import timedelta +from typing import List + +import asyncer +from dagger import Container +from pipelines.actions import environments, secrets +from pipelines.bases import Step, StepResult, StepStatus +from pipelines.builds import LOCAL_BUILD_PLATFORM +from pipelines.builds.python_connectors import BuildConnectorImage +from pipelines.contexts import ConnectorContext +from pipelines.helpers.steps import run_steps +from pipelines.tests.common import AcceptanceTests, PytestStep +from pipelines.utils import export_container_to_tarball + + +class CodeFormatChecks(Step): + """A step to run the code format checks on a Python connector using Black, Isort and Flake.""" + + title = "Code format checks" + + RUN_BLACK_CMD = ["python", "-m", "black", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "--check", "."] + RUN_ISORT_CMD = ["python", "-m", "isort", f"--settings-file=/{environments.PYPROJECT_TOML_FILE_PATH}", "--check-only", "--diff", "."] + RUN_FLAKE_CMD = ["python", "-m", "pflake8", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] + + async def _run(self) -> StepResult: + """Run a code format check on the container source code. + + We call black, isort and flake commands: + - Black formats the code: fails if the code is not formatted. + - Isort checks the import orders: fails if the import are not properly ordered. + - Flake enforces style-guides: fails if the style-guide is not followed. + + Args: + context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. + step (Step): The step in which the code format checks are run. Defaults to Step.CODE_FORMAT_CHECKS + Returns: + StepResult: Failure or success of the code format checks with stdout and stderr. + """ + connector_under_test = environments.with_python_connector_source(self.context) + + formatter = ( + connector_under_test.with_exec(["echo", "Running black"]) + .with_exec(self.RUN_BLACK_CMD) + .with_exec(["echo", "Running Isort"]) + .with_exec(self.RUN_ISORT_CMD) + .with_exec(["echo", "Running Flake"]) + .with_exec(self.RUN_FLAKE_CMD) + ) + return await self.get_step_result(formatter) + + +class ConnectorPackageInstall(Step): + """A step to install the Python connector package in a container.""" + + title = "Connector package install" + max_duration = timedelta(minutes=20) + max_retries = 3 + + async def _run(self) -> StepResult: + """Install the connector under test package in a Python container. + + Returns: + StepResult: Failure or success of the package installation and the connector under test container (with the connector package installed). + """ + connector_under_test = await environments.with_python_connector_installed(self.context) + return await self.get_step_result(connector_under_test) + + +class UnitTests(PytestStep): + """A step to run the connector unit tests with Pytest.""" + + title = "Unit tests" + + async def _run(self, connector_under_test: Container) -> StepResult: + """Run all pytest tests declared in the unit_tests directory of the connector code. + + Args: + connector_under_test (Container): The connector under test container. + + Returns: + StepResult: Failure or success of the unit tests with stdout and stdout. + """ + connector_under_test_with_secrets = connector_under_test.with_(environments.mounted_connector_secrets(self.context)) + return await self._run_tests_in_directory(connector_under_test_with_secrets, "unit_tests") + + +class IntegrationTests(PytestStep): + """A step to run the connector integration tests with Pytest.""" + + title = "Integration tests" + + async def _run(self, connector_under_test: Container) -> StepResult: + """Run all pytest tests declared in the integration_tests directory of the connector code. + + Args: + connector_under_test (Container): The connector under test container. + + Returns: + StepResult: Failure or success of the integration tests with stdout and stdout. + """ + + connector_under_test = connector_under_test.with_(environments.bound_docker_host(self.context)).with_( + environments.mounted_connector_secrets(self.context) + ) + return await self._run_tests_in_directory(connector_under_test, "integration_tests") + + +async def run_all_tests(context: ConnectorContext) -> List[StepResult]: + """Run all tests for a Python connector. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: The results of all the steps that ran or were skipped. + """ + + step_results = await run_steps( + [ + ConnectorPackageInstall(context), + BuildConnectorImage(context, LOCAL_BUILD_PLATFORM), + ] + ) + if any([step_result.status is StepStatus.FAILURE for step_result in step_results]): + return step_results + connector_package_install_results, build_connector_image_results = step_results[0], step_results[1] + connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact) + connector_container = connector_package_install_results.output_artifact + + context.connector_secrets = await secrets.get_connector_secrets(context) + + unit_test_results = await UnitTests(context).run(connector_container) + + if unit_test_results.status is StepStatus.FAILURE: + return step_results + [unit_test_results] + step_results.append(unit_test_results) + async with asyncer.create_task_group() as task_group: + tasks = [ + task_group.soonify(IntegrationTests(context).run)(connector_container), + task_group.soonify(AcceptanceTests(context).run)(connector_image_tar_file), + ] + + return step_results + [task.value for task in tasks] + + +async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: + """Run the code format check steps for Python connectors. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + List[StepResult]: Results of the code format checks. + """ + return [await CodeFormatChecks(context).run()] diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 b/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 new file mode 100644 index 000000000000..5ac9282ac5bd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 @@ -0,0 +1,177 @@ + + + + {{ connector_name }} test report + + + + + +

{{ connector_name }} test report

+
    +
  • Created at: {{ created_at }} UTC
    • +
  • Run duration: {{ format_duration(run_duration) }}
    • + {% if commit_url %} +
  • Commit: {{ git_revision[:10] }}
    • + {% else %} +
  • Commit: {{ git_revision[:10] }}
    • + {% endif %} +
  • Branch: {{ git_branch }}
    • + {% if gha_workflow_run_url %} +
    +
  • Github Actions logs
    • + {% endif %} + {% if dagger_logs_url %} +
  • Dagger logs
    • + {% endif %} + {% if dagger_cloud_url %} +
  • Dagger Cloud UI
    • + {% endif %} +
+

Summary

+ + + + + + + {% for step_result in step_results %} + + + + + + {% endfor %} +
StepStatusDuration
{{ step_result.step.title }}{{ step_result.status }}{{ format_duration(step_result.step.run_duration) }}
+

Step details

+ {% for step_result in step_results %} +
+ + {% if step_result.status == StepStatus.SUCCESS %} + + {% elif step_result.status == StepStatus.FAILURE %} + + {% else %} + + {% endif %} +
+
+ {% if step_result.stdout %} + Standard output: + 
{{ step_result.stdout }}
+ {% endif %} + {% if step_result.stderr %} + Standard error: + 
{{ step_result.stderr }}
+ {% endif %} +
+
+
+ {% endfor %} +

These reports are generated from this code, please reach out to the Connector Operations team for support.

+ + diff --git a/airbyte-ci/connectors/pipelines/pipelines/utils.py b/airbyte-ci/connectors/pipelines/pipelines/utils.py new file mode 100644 index 000000000000..d80fe8d744ed --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/utils.py @@ -0,0 +1,635 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups util function used in pipelines.""" +from __future__ import annotations + +import contextlib +import datetime +import json +import os +import re +import sys +import unicodedata +from glob import glob +from io import TextIOWrapper +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, FrozenSet, List, Optional, Set, Tuple, Union + +import anyio +import asyncer +import click +import git +from connector_ops.utils import get_changed_connectors +from dagger import Client, Config, Connection, Container, DaggerError, ExecError, File, ImageLayerCompression, QueryError, Secret +from google.cloud import storage +from google.oauth2 import service_account +from more_itertools import chunked +from pipelines import consts, main_logger, sentry_utils +from pipelines.consts import GCS_PUBLIC_DOMAIN + +if TYPE_CHECKING: + from connector_ops.utils import Connector + from github import PullRequest + from pipelines.contexts import ConnectorContext + +DAGGER_CONFIG = Config(log_output=sys.stderr) +AIRBYTE_REPO_URL = "https://github.com/airbytehq/airbyte.git" +METADATA_FILE_NAME = "metadata.yaml" +METADATA_ICON_FILE_NAME = "icon.svg" +DIFF_FILTER = "MADRT" # Modified, Added, Deleted, Renamed, Type changed +IGNORED_FILE_EXTENSIONS = [".md"] +STATIC_REPORT_PREFIX = "airbyte-ci" + + +# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented +async def check_path_in_workdir(container: Container, path: str) -> bool: + """Check if a local path is mounted to the working directory of a container. + + Args: + container (Container): The container on which we want the check the path existence. + path (str): Directory or file path we want to check the existence in the container working directory. + + Returns: + bool: Whether the path exists in the container working directory. + """ + workdir = (await container.with_exec(["pwd"]).stdout()).strip() + mounts = await container.mounts() + if workdir in mounts: + expected_file_path = Path(workdir[1:]) / path + return expected_file_path.is_file() or expected_file_path.is_dir() + else: + return False + + +def secret_host_variable(client: Client, name: str, default: str = ""): + """Add a host environment variable as a secret in a container. + + Example: + container.with_(secret_host_variable(client, "MY_SECRET")) + + Args: + client (Client): The dagger client. + name (str): The name of the environment variable. The same name will be + used in the container, for the secret name and for the host variable. + default (str): The default value to use if the host variable is not set. Defaults to "". + + Returns: + Callable[[Container], Container]: A function that can be used in a `Container.with_()` method. + """ + + def _secret_host_variable(container: Container): + return container.with_secret_variable(name, get_secret_host_variable(client, name, default)) + + return _secret_host_variable + + +def get_secret_host_variable(client: Client, name: str, default: str = "") -> Secret: + """Creates a dagger.Secret from a host environment variable. + + Args: + client (Client): The dagger client. + name (str): The name of the environment variable. The same name will be used for the secret. + default (str): The default value to use if the host variable is not set. Defaults to "". + + Returns: + Secret: A dagger secret. + """ + return client.set_secret(name, os.environ.get(name, default)) + + +# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented +async def get_file_contents(container: Container, path: str) -> Optional[str]: + """Retrieve a container file contents. + + Args: + container (Container): The container hosting the file you want to read. + path (str): Path, in the container, to the file you want to read. + + Returns: + Optional[str]: The file content if the file exists in the container, None otherwise. + """ + try: + return await container.file(path).contents() + except QueryError as e: + if "no such file or directory" not in str(e): + # this error could come from a network issue + raise + return None + + +@contextlib.contextmanager +def catch_exec_error_group(): + try: + yield + except anyio.ExceptionGroup as eg: + for e in eg.exceptions: + if isinstance(e, ExecError): + raise e + raise + + +async def get_container_output(container: Container) -> Tuple[str, str]: + """Retrieve both stdout and stderr of a container, concurrently. + + Args: + container (Container): The container to execute. + + Returns: + Tuple[str, str]: The stdout and stderr of the container, respectively. + """ + with catch_exec_error_group(): + async with asyncer.create_task_group() as task_group: + soon_stdout = task_group.soonify(container.stdout)() + soon_stderr = task_group.soonify(container.stderr)() + return soon_stdout.value, soon_stderr.value + + +async def get_exec_result(container: Container) -> Tuple[int, str, str]: + """Retrieve the exit_code along with stdout and stderr of a container by handling the ExecError. + + Note: It is preferrable to not worry about the exit code value and just capture + ExecError to handle errors. This is offered as a convenience when the exit code + value is actually needed. + + If the container has a file at /exit_code, the exit code will be read from it. + See hacks.never_fail_exec for more details. + + Args: + container (Container): The container to execute. + + Returns: + Tuple[int, str, str]: The exit_code, stdout and stderr of the container, respectively. + """ + try: + exit_code = 0 + in_file_exit_code = await get_file_contents(container, "/exit_code") + if in_file_exit_code: + exit_code = int(in_file_exit_code) + return exit_code, *(await get_container_output(container)) + except ExecError as e: + return e.exit_code, e.stdout, e.stderr + + +async def with_exit_code(container: Container) -> int: + """Read the container exit code. + + Args: + container (Container): The container from which you want to read the exit code. + + Returns: + int: The exit code. + """ + try: + await container + except ExecError as e: + return e.exit_code + return 0 + + +async def with_stderr(container: Container) -> str: + """Retrieve the stderr of a container even on execution error.""" + try: + return await container.stderr() + except ExecError as e: + return e.stderr + + +async def with_stdout(container: Container) -> str: + """Retrieve the stdout of a container even on execution error.""" + try: + return await container.stdout() + except ExecError as e: + return e.stdout + + +def get_current_git_branch() -> str: # noqa D103 + return git.Repo().active_branch.name + + +def get_current_git_revision() -> str: # noqa D103 + return git.Repo().head.object.hexsha + + +def get_current_epoch_time() -> int: # noqa D103 + return round(datetime.datetime.utcnow().timestamp()) + + +async def get_modified_files_in_branch_remote( + current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master" +) -> Set[str]: + """Use git diff to spot the modified files on the remote branch.""" + async with Connection(DAGGER_CONFIG) as dagger_client: + modified_files = await ( + dagger_client.container() + .from_("alpine/git:latest") + .with_workdir("/repo") + .with_exec(["init"]) + .with_env_variable("CACHEBUSTER", current_git_revision) + .with_exec( + [ + "remote", + "add", + "--fetch", + "--track", + diffed_branch.split("/")[-1], + "--track", + current_git_branch, + "origin", + AIRBYTE_REPO_URL, + ] + ) + .with_exec(["checkout", "-t", f"origin/{current_git_branch}"]) + .with_exec(["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_revision}"]) + .stdout() + ) + return set(modified_files.split("\n")) + + +def get_modified_files_in_branch_local(current_git_revision: str, diffed_branch: str = "master") -> Set[str]: + """Use git diff and git status to spot the modified files on the local branch.""" + airbyte_repo = git.Repo() + modified_files = airbyte_repo.git.diff( + f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_revision}" + ).split("\n") + status_output = airbyte_repo.git.status("--porcelain") + for not_committed_change in status_output.split("\n"): + file_path = not_committed_change.strip().split(" ")[-1] + if file_path: + modified_files.append(file_path) + return set(modified_files) + + +def get_modified_files_in_branch(current_git_branch: str, current_git_revision: str, diffed_branch: str, is_local: bool = True) -> Set[str]: + """Retrieve the list of modified files on the branch.""" + if is_local: + return get_modified_files_in_branch_local(current_git_revision, diffed_branch) + else: + return anyio.run(get_modified_files_in_branch_remote, current_git_branch, current_git_revision, diffed_branch) + + +async def get_modified_files_in_commit_remote(current_git_branch: str, current_git_revision: str) -> Set[str]: + async with Connection(DAGGER_CONFIG) as dagger_client: + modified_files = await ( + dagger_client.container() + .from_("alpine/git:latest") + .with_workdir("/repo") + .with_exec(["init"]) + .with_env_variable("CACHEBUSTER", current_git_revision) + .with_exec( + [ + "remote", + "add", + "--fetch", + "--track", + current_git_branch, + "origin", + AIRBYTE_REPO_URL, + ] + ) + .with_exec(["checkout", "-t", f"origin/{current_git_branch}"]) + .with_exec(["diff-tree", "--no-commit-id", "--name-only", current_git_revision, "-r"]) + .stdout() + ) + return set(modified_files.split("\n")) + + +def get_modified_files_in_commit_local(current_git_revision: str) -> Set[str]: + airbyte_repo = git.Repo() + modified_files = airbyte_repo.git.diff_tree("--no-commit-id", "--name-only", current_git_revision, "-r").split("\n") + return set(modified_files) + + +def get_modified_files_in_commit(current_git_branch: str, current_git_revision: str, is_local: bool = True) -> Set[str]: + if is_local: + return get_modified_files_in_commit_local(current_git_revision) + else: + return anyio.run(get_modified_files_in_commit_remote, current_git_branch, current_git_revision) + + +def get_modified_files_in_pull_request(pull_request: PullRequest) -> List[str]: + """Retrieve the list of modified files in a pull request.""" + return [f.filename for f in pull_request.get_files()] + + +def get_last_commit_message() -> str: + """Retrieve the last commit message.""" + return git.Repo().head.commit.message + + +def _is_ignored_file(file_path: Union[str, Path]) -> bool: + """Check if the provided file has an ignored extension.""" + return Path(file_path).suffix in IGNORED_FILE_EXTENSIONS + + +def _find_modified_connectors( + file_path: Union[str, Path], all_connectors: Set[Connector], dependency_scanning: bool = True +) -> Set[Connector]: + """Find all connectors impacted by the file change.""" + modified_connectors = set() + + for connector in all_connectors: + if Path(file_path).is_relative_to(Path(connector.code_directory)): + main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.") + modified_connectors.add(connector) + + if dependency_scanning: + for connector_dependency in connector.get_local_dependency_paths(): + if Path(file_path).is_relative_to(Path(connector_dependency)): + # Add the connector to the modified connectors + modified_connectors.add(connector) + main_logger.info(f"Adding connector '{connector}' due to dependency modification: '{file_path}'.") + return modified_connectors + + +def get_modified_connectors(modified_files: Set[Path], all_connectors: Set[Connector], dependency_scanning: bool) -> Set[Connector]: + """Create a mapping of modified connectors (key) and modified files (value). + If dependency scanning is enabled any modification to a dependency will trigger connector pipeline for all connectors that depend on it. + It currently works only for Java connectors . + It's especially useful to trigger tests of strict-encrypt variant when a change is made to the base connector. + Or to tests all jdbc connectors when a change is made to source-jdbc or base-java. + We'll consider extending the dependency resolution to Python connectors once we confirm that it's needed and feasible in term of scale. + """ + # Ignore files with certain extensions + modified_connectors = set() + for modified_file in modified_files: + if not _is_ignored_file(modified_file): + modified_connectors.update(_find_modified_connectors(modified_file, all_connectors, dependency_scanning)) + return modified_connectors + + +def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> FrozenSet[Path]: + connector_modified_files = set() + for modified_file in all_modified_files: + modified_file_path = Path(modified_file) + if modified_file_path.is_relative_to(connector.code_directory): + connector_modified_files.add(modified_file) + return frozenset(connector_modified_files) + + +def get_modified_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]: + return { + Path(str(f)) + for f in modified_files + if str(f).endswith(METADATA_FILE_NAME) and str(f).startswith("airbyte-integrations/connectors") and "-scaffold-" not in str(f) + } + + +def get_expected_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]: + changed_connectors = get_changed_connectors(modified_files=modified_files) + return {changed_connector.metadata_file_path for changed_connector in changed_connectors} + + +def get_all_metadata_files() -> Set[Path]: + return { + Path(metadata_file) + for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True) + if "-scaffold-" not in metadata_file + } + + +def slugify(value: Any, allow_unicode: bool = False): + """ + Taken from https://github.com/django/django/blob/master/django/utils/text.py. + + Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated + dashes to single dashes. Remove characters that aren't alphanumerics, + underscores, or hyphens. Convert to lowercase. Also strip leading and + trailing whitespace, dashes, and underscores. + """ + value = str(value) + if allow_unicode: + value = unicodedata.normalize("NFKC", value) + else: + value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + value = re.sub(r"[^\w\s-]", "", value.lower()) + return re.sub(r"[-\s]+", "-", value).strip("-_") + + +def key_value_text_to_dict(text: str) -> dict: + kv = {} + for line in text.split("\n"): + if "=" in line: + try: + k, v = line.split("=") + except ValueError: + continue + kv[k] = v + return kv + + +async def key_value_file_to_dict(file: File) -> dict: + return key_value_text_to_dict(await file.contents()) + + +async def get_dockerfile_labels(dockerfile: File) -> dict: + return {k.replace("LABEL ", ""): v for k, v in (await key_value_file_to_dict(dockerfile)).items() if k.startswith("LABEL")} + + +async def get_version_from_dockerfile(dockerfile: File) -> str: + dockerfile_labels = await get_dockerfile_labels(dockerfile) + try: + return dockerfile_labels["io.airbyte.version"] + except KeyError: + raise Exception("Could not get the version from the Dockerfile labels.") + + +def create_and_open_file(file_path: Path) -> TextIOWrapper: + """Create a file and open it for writing. + + Args: + file_path (Path): The path to the file to create. + + Returns: + File: The file object. + """ + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch() + return file_path.open("w") + + +class DaggerPipelineCommand(click.Command): + @sentry_utils.with_command_context + def invoke(self, ctx: click.Context) -> Any: + """Wrap parent invoke in a try catch suited to handle pipeline failures. + Args: + ctx (click.Context): The invocation context. + Raises: + e: Raise whatever exception that was caught. + Returns: + Any: The invocation return value. + """ + command_name = self.name + main_logger.info(f"Running Dagger Command {command_name}...") + main_logger.info( + "If you're running this command for the first time the Dagger engine image will be pulled, it can take a short minute..." + ) + ctx.obj["report_output_prefix"] = self.render_report_output_prefix(ctx) + dagger_logs_gcs_key = f"{ctx.obj['report_output_prefix']}/dagger-logs.txt" + try: + if not ctx.obj["show_dagger_logs"]: + dagger_log_dir = Path(f"{consts.LOCAL_REPORTS_PATH_ROOT}/{ctx.obj['report_output_prefix']}") + dagger_log_path = Path(f"{dagger_log_dir}/dagger.log").resolve() + ctx.obj["dagger_logs_path"] = dagger_log_path + main_logger.info(f"Saving dagger logs to: {dagger_log_path}") + if ctx.obj["is_ci"]: + ctx.obj["dagger_logs_url"] = f"{GCS_PUBLIC_DOMAIN}/{ctx.obj['ci_report_bucket_name']}/{dagger_logs_gcs_key}" + else: + ctx.obj["dagger_logs_url"] = None + else: + ctx.obj["dagger_logs_path"] = None + pipeline_success = super().invoke(ctx) + if not pipeline_success: + raise DaggerError(f"Dagger Command {command_name} failed.") + except DaggerError as e: + main_logger.error(f"Dagger Command {command_name} failed", exc_info=e) + sys.exit(1) + finally: + if ctx.obj.get("dagger_logs_path"): + if ctx.obj["is_local"]: + main_logger.info(f"Dagger logs saved to {ctx.obj['dagger_logs_path']}") + if ctx.obj["is_ci"]: + gcs_uri, public_url = upload_to_gcs( + ctx.obj["dagger_logs_path"], ctx.obj["ci_report_bucket_name"], dagger_logs_gcs_key, ctx.obj["ci_gcs_credentials"] + ) + main_logger.info(f"Dagger logs saved to {gcs_uri}. Public URL: {public_url}") + + @staticmethod + def render_report_output_prefix(ctx: click.Context) -> str: + """Render the report output prefix for any command in the Connector CLI. + + The goal is to standardize the output of all logs and reports generated by the CLI + related to a specific command, and to a specific CI context. + + Note: We cannot hoist this higher in the command hierarchy because only one level of + subcommands are available at the time the context is created. + """ + + git_branch = ctx.obj["git_branch"] + git_revision = ctx.obj["git_revision"] + pipeline_start_timestamp = ctx.obj["pipeline_start_timestamp"] + ci_context = ctx.obj["ci_context"] + ci_job_key = ctx.obj["ci_job_key"] if ctx.obj.get("ci_job_key") else ci_context + + sanitized_branch = slugify(git_branch.replace("/", "_")) + + # get the command name for the current context, if a group then prepend the parent command name + if ctx.command_path: + cmd_components = ctx.command_path.split(" ") + cmd_components[0] = STATIC_REPORT_PREFIX + cmd = "/".join(cmd_components) + else: + cmd = None + + path_values = [ + cmd, + ci_job_key, + sanitized_branch, + pipeline_start_timestamp, + git_revision, + ] + + # check all values are defined + if None in path_values: + raise ValueError(f"Missing value required to render the report output prefix: {path_values}") + + # join all values with a slash, and convert all values to string + return "/".join(map(str, path_values)) + + +async def execute_concurrently(steps: List[Callable], concurrency=5): + tasks = [] + # Asyncer does not have builtin semaphore, so control concurrency via chunks of steps + # Anyio has semaphores but does not have the soonify method which allow access to results via the value task attribute. + for chunk in chunked(steps, concurrency): + async with asyncer.create_task_group() as task_group: + tasks += [task_group.soonify(step)() for step in chunk] + return [task.value for task in tasks] + + +async def export_container_to_tarball( + context: ConnectorContext, container: Container, tar_file_name: Optional[str] = None +) -> Tuple[Optional[File], Optional[Path]]: + """Save the container image to the host filesystem as a tar archive. + + Exporting a container image as a tar archive allows user to have a dagger built container image available on their host filesystem. + They can load this tar file to their main docker host with 'docker load'. + This mechanism is also used to share dagger built containers with other steps like AcceptanceTest that have their own dockerd service. + We 'docker load' this tar file to AcceptanceTest's docker host to make sure the container under test image is available for testing. + + Returns: + Tuple[Optional[File], Optional[Path]]: A tuple with the file object holding the tar archive on the host and its path. + """ + if tar_file_name is None: + tar_file_name = f"{context.connector.technical_name}_{context.git_revision}.tar" + tar_file_name = slugify(tar_file_name) + local_path = Path(f"{context.host_image_export_dir_path}/{tar_file_name}") + export_success = await container.export(str(local_path), forced_compression=ImageLayerCompression.Gzip) + if export_success: + exported_file = ( + context.dagger_client.host().directory(context.host_image_export_dir_path, include=[tar_file_name]).file(tar_file_name) + ) + return exported_file, local_path + else: + return None, None + + +def sanitize_gcs_credentials(raw_value: Optional[str]) -> Optional[str]: + """Try to parse the raw string input that should contain a json object with the GCS credentials. + It will raise an exception if the parsing fails and help us to fail fast on invalid credentials input. + + Args: + raw_value (str): A string representing a json object with the GCS credentials. + + Returns: + str: The raw value string if it was successfully parsed. + """ + if raw_value is None: + return None + return json.dumps(json.loads(raw_value)) + + +def format_duration(time_delta: datetime.timedelta) -> str: + total_seconds = time_delta.total_seconds() + if total_seconds < 60: + return "{:.2f}s".format(total_seconds) + minutes = int(total_seconds // 60) + seconds = int(total_seconds % 60) + return "{:02d}mn{:02d}s".format(minutes, seconds) + + +def upload_to_gcs(file_path: Path, bucket_name: str, object_name: str, credentials: str) -> Tuple[str, str]: + """Upload a file to a GCS bucket. + + Args: + file_path (Path): The path to the file to upload. + bucket_name (str): The name of the GCS bucket. + object_name (str): The name of the object in the GCS bucket. + credentials (str): The GCS credentials as a JSON string. + """ + # Exit early if file does not exist + if not file_path.exists(): + main_logger.warning(f"File {file_path} does not exist. Skipping upload to GCS.") + return "", "" + + credentials = service_account.Credentials.from_service_account_info(json.loads(credentials)) + client = storage.Client(credentials=credentials) + bucket = client.get_bucket(bucket_name) + blob = bucket.blob(object_name) + blob.upload_from_filename(str(file_path)) + gcs_uri = f"gs://{bucket_name}/{object_name}" + public_url = f"{GCS_PUBLIC_DOMAIN}/{bucket_name}/{object_name}" + return gcs_uri, public_url + + +def transform_strs_to_paths(str_paths: List[str]) -> List[Path]: + """Transform a list of string paths to a list of Path objects. + + Args: + str_paths (List[str]): A list of string paths. + + Returns: + List[Path]: A list of Path objects. + """ + return [Path(str_path) for str_path in str_paths] diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock new file mode 100644 index 000000000000..f48430f798f6 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -0,0 +1,1925 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "airbyte-protocol-models" +version = "1.0.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.9" +files = [ + {file = "airbyte_protocol_models-1.0.1-py3-none-any.whl", hash = "sha256:2c214fb8cb42b74aa6408beeea2cd52f094bc8a3ba0e78af20bb358e5404f4a8"}, + {file = "airbyte_protocol_models-1.0.1.tar.gz", hash = "sha256:caa860d15c9c9073df4b221f58280b9855d36de07519e010d1e610546458d0a7"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<1.10.0" + +[[package]] +name = "anyio" +version = "3.7.1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.7" +files = [ + {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, + {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, +] + +[package.dependencies] +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] +test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (<0.22)"] + +[[package]] +name = "asyncer" +version = "0.0.2" +description = "Asyncer, async and await, focused on developer experience." +optional = false +python-versions = ">=3.6.2,<4.0.0" +files = [ + {file = "asyncer-0.0.2-py3-none-any.whl", hash = "sha256:46e0e1423ce21588350ad425875e81795280b9e1f517e8a389de940b86c348bd"}, + {file = "asyncer-0.0.2.tar.gz", hash = "sha256:d546c85f3626ebbaf06bb4395db49761c902a61a6ac802b1a74133cab4f7f433"}, +] + +[package.dependencies] +anyio = ">=3.4.0,<4.0.0" + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "beartype" +version = "0.15.0" +description = "Unbearably fast runtime type checking in pure Python." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "beartype-0.15.0-py3-none-any.whl", hash = "sha256:52cd2edea72fdd84e4e7f8011a9e3007bf0125c3d6d7219e937b9d8868169177"}, + {file = "beartype-0.15.0.tar.gz", hash = "sha256:2af6a8d8a7267ccf7d271e1a3bd908afbc025d2a09aa51123567d7d7b37438df"}, +] + +[package.extras] +all = ["typing-extensions (>=3.10.0.0)"] +dev = ["autoapi (>=0.9.0)", "coverage (>=5.5)", "mypy (>=0.800)", "numpy", "pandera", "pydata-sphinx-theme (<=0.7.2)", "pytest (>=4.0.0)", "sphinx", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)", "tox (>=3.20.1)", "typing-extensions (>=3.10.0.0)"] +doc-rtd = ["autoapi (>=0.9.0)", "pydata-sphinx-theme (<=0.7.2)", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)"] +test-tox = ["mypy (>=0.800)", "numpy", "pandera", "pytest (>=4.0.0)", "sphinx", "typing-extensions (>=3.10.0.0)"] +test-tox-coverage = ["coverage (>=5.5)"] + +[[package]] +name = "cachetools" +version = "5.3.1" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, +] + +[[package]] +name = "cattrs" +version = "23.1.2" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, + {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, +] + +[package.dependencies] +attrs = ">=20" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.2.0,<5.0.0)"] +cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] +msgpack = ["msgpack (>=1.0.2,<2.0.0)"] +orjson = ["orjson (>=3.5.2,<4.0.0)"] +pyyaml = ["PyYAML (>=6.0,<7.0)"] +tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] +ujson = ["ujson (>=5.4.0,<6.0.0)"] + +[[package]] +name = "certifi" +version = "2023.7.22" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.2.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] + +[[package]] +name = "ci-credentials" +version = "1.1.0" +description = "CLI tooling to read and manage GSM secrets" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +click = "^8.1.3" +common_utils = {path = "../common_utils", develop = true} +pyyaml = "^6.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../ci_credentials" + +[[package]] +name = "click" +version = "8.1.6" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "common-utils" +version = "0.0.0" +description = "Suite of all often used classes and common functions" +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +cryptography = "^3.4.7" +pyjwt = "^2.1.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../common_utils" + +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +optional = false +python-versions = "*" +files = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "connector-ops" +version = "0.2.2" +description = "Packaged maintained by the connector operations team to perform CI for connectors" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +ci-credentials = {path = "../ci_credentials"} +click = "^8.1.3" +GitPython = "^3.1.29" +google-cloud-storage = "^2.8.0" +pydantic = "^1.9" +pydash = "^7.0.4" +PyGithub = "^1.58.0" +PyYAML = "^6.0" +requests = "^2.28.2" +rich = "^11.0.1" + +[package.source] +type = "directory" +url = "../connector_ops" + +[[package]] +name = "coverage" +version = "7.2.7" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "3.4.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89"}, + {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, + {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, + {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "dagger-io" +version = "0.6.4" +description = "A client package for running Dagger pipelines in Python." +optional = false +python-versions = ">=3.10" +files = [ + {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, + {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, +] + +[package.dependencies] +anyio = ">=3.6.2" +beartype = ">=0.11.0" +cattrs = ">=22.2.0" +gql = ">=3.4.0" +graphql-core = ">=3.2.3" +httpx = ">=0.23.1" +platformdirs = ">=2.6.2" +typing-extensions = ">=4.4.0" + +[package.extras] +cli = ["typer[all] (>=0.6.1)"] +server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "docker" +version = "5.0.3" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.6" +files = [ + {file = "docker-5.0.3-py2.py3-none-any.whl", hash = "sha256:7a79bb439e3df59d0a72621775d600bc8bc8b422d285824cb37103eab91d1ce0"}, + {file = "docker-5.0.3.tar.gz", hash = "sha256:d916a26b62970e7c2f554110ed6af04c7ccff8e9f81ad17d0d40c75637e227fb"}, +] + +[package.dependencies] +pywin32 = {version = "227", markers = "sys_platform == \"win32\""} +requests = ">=2.14.2,<2.18.0 || >2.18.0" +websocket-client = ">=0.32.0" + +[package.extras] +ssh = ["paramiko (>=2.4.2)"] +tls = ["cryptography (>=3.4.7)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] + +[[package]] +name = "exceptiongroup" +version = "1.1.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.6" +files = [ + {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, + {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + +[[package]] +name = "gitdb" +version = "4.0.10" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, + {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.32" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.32-py3-none-any.whl", hash = "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"}, + {file = "GitPython-3.1.32.tar.gz", hash = "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[[package]] +name = "google-api-core" +version = "2.11.1" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, + {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.22.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, + {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" +six = ">=1.9.0" +urllib3 = "<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-core" +version = "2.3.3" +description = "Google Cloud API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, + {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, +] + +[package.dependencies] +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0dev)"] + +[[package]] +name = "google-cloud-storage" +version = "2.10.0" +description = "Google Cloud Storage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, + {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-resumable-media = ">=2.3.2" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +protobuf = ["protobuf (<5.0.0dev)"] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"}, +] + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.5.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "google-resumable-media-2.5.0.tar.gz", hash = "sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93"}, + {file = "google_resumable_media-2.5.0-py2.py3-none-any.whl", hash = "sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec"}, +] + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.60.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, + {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "gql" +version = "3.4.1" +description = "GraphQL client for Python" +optional = false +python-versions = "*" +files = [ + {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, + {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, +] + +[package.dependencies] +backoff = ">=1.11.1,<3.0" +graphql-core = ">=3.2,<3.3" +yarl = ">=1.6,<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] +all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +botocore = ["botocore (>=1.21,<2)"] +dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] +test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] +websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "0.17.3" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"}, + {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"}, +] + +[package.dependencies] +anyio = ">=3.0,<5.0" +certifi = "*" +h11 = ">=0.13,<0.15" +sniffio = "==1.*" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "httpx" +version = "0.24.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"}, + {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"}, +] + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.18.0" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "more-itertools" +version = "8.14.0" +description = "More routines for operating on iterables, beyond itertools" +optional = false +python-versions = ">=3.5" +files = [ + {file = "more-itertools-8.14.0.tar.gz", hash = "sha256:c09443cd3d5438b8dafccd867a6bc1cb0894389e90cb53d227456b0b0bccb750"}, + {file = "more_itertools-8.14.0-py3-none-any.whl", hash = "sha256:1bc4f91ee5b1b31ac7ceacc17c09befe6a40a503907baf9c839c229b5095cfd2"}, +] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "platformdirs" +version = "3.10.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.2.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.24.0" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, + {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, + {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, + {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, + {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, + {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, + {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, + {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, + {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, + {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, + {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.0" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, + {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.9.2" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, + {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, + {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, + {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, + {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, + {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, + {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, + {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, + {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, + {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pydash" +version = "7.0.6" +description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydash-7.0.6-py3-none-any.whl", hash = "sha256:10e506935953fde4b0d6fe21a88e17783cd1479256ae96f285b5f89063b4efd6"}, + {file = "pydash-7.0.6.tar.gz", hash = "sha256:7d9df7e9f36f2bbb08316b609480e7c6468185473a21bdd8e65dda7915565a26"}, +] + +[package.dependencies] +typing-extensions = ">=3.10,<4.6.0 || >4.6.0" + +[package.extras] +dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] + +[[package]] +name = "pygithub" +version = "1.59.1" +description = "Use the full Github API v3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyGithub-1.59.1-py3-none-any.whl", hash = "sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9"}, + {file = "PyGithub-1.59.1.tar.gz", hash = "sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217"}, +] + +[package.dependencies] +deprecated = "*" +pyjwt = {version = ">=2.4.0", extras = ["crypto"]} +pynacl = ">=1.4.0" +requests = ">=2.14.0" + +[[package]] +name = "pygments" +version = "2.16.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pynacl" +version = "1.5.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, + {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, +] + +[package.dependencies] +cffi = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "4.1.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + +[[package]] +name = "pytest-mock" +version = "3.11.1" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, + {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pywin32" +version = "227" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-227-cp27-cp27m-win32.whl", hash = "sha256:371fcc39416d736401f0274dd64c2302728c9e034808e37381b5e1b22be4a6b0"}, + {file = "pywin32-227-cp27-cp27m-win_amd64.whl", hash = "sha256:4cdad3e84191194ea6d0dd1b1b9bdda574ff563177d2adf2b4efec2a244fa116"}, + {file = "pywin32-227-cp35-cp35m-win32.whl", hash = "sha256:f4c5be1a293bae0076d93c88f37ee8da68136744588bc5e2be2f299a34ceb7aa"}, + {file = "pywin32-227-cp35-cp35m-win_amd64.whl", hash = "sha256:a929a4af626e530383a579431b70e512e736e9588106715215bf685a3ea508d4"}, + {file = "pywin32-227-cp36-cp36m-win32.whl", hash = "sha256:300a2db938e98c3e7e2093e4491439e62287d0d493fe07cce110db070b54c0be"}, + {file = "pywin32-227-cp36-cp36m-win_amd64.whl", hash = "sha256:9b31e009564fb95db160f154e2aa195ed66bcc4c058ed72850d047141b36f3a2"}, + {file = "pywin32-227-cp37-cp37m-win32.whl", hash = "sha256:47a3c7551376a865dd8d095a98deba954a98f326c6fe3c72d8726ca6e6b15507"}, + {file = "pywin32-227-cp37-cp37m-win_amd64.whl", hash = "sha256:31f88a89139cb2adc40f8f0e65ee56a8c585f629974f9e07622ba80199057511"}, + {file = "pywin32-227-cp38-cp38-win32.whl", hash = "sha256:7f18199fbf29ca99dff10e1f09451582ae9e372a892ff03a28528a24d55875bc"}, + {file = "pywin32-227-cp38-cp38-win_amd64.whl", hash = "sha256:7c1ae32c489dc012930787f06244426f8356e129184a02c25aef163917ce158e"}, + {file = "pywin32-227-cp39-cp39-win32.whl", hash = "sha256:c054c52ba46e7eb6b7d7dfae4dbd987a1bb48ee86debe3f245a2884ece46e295"}, + {file = "pywin32-227-cp39-cp39-win_amd64.whl", hash = "sha256:f27cec5e7f588c3d1051651830ecc00294f90728d19c3bf6916e6dba93ea357c"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rich" +version = "11.2.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.6.2,<4.0.0" +files = [ + {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, + {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, +] + +[package.dependencies] +colorama = ">=0.4.0,<0.5.0" +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "semver" +version = "3.0.1" +description = "Python helper for Semantic Versioning (https://semver.org)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, + {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, +] + +[[package]] +name = "sentry-sdk" +version = "1.29.2" +description = "Python client for Sentry (https://sentry.io)" +optional = false +python-versions = "*" +files = [ + {file = "sentry-sdk-1.29.2.tar.gz", hash = "sha256:a99ee105384788c3f228726a88baf515fe7b5f1d2d0f215a03d194369f158df7"}, + {file = "sentry_sdk-1.29.2-py2.py3-none-any.whl", hash = "sha256:3e17215d8006612e2df02b0e73115eb8376c37e3f586d8436fa41644e605074d"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} + +[package.extras] +aiohttp = ["aiohttp (>=3.5)"] +arq = ["arq (>=0.23)"] +beam = ["apache-beam (>=2.12)"] +bottle = ["bottle (>=0.12.13)"] +celery = ["celery (>=3)"] +chalice = ["chalice (>=1.16.0)"] +django = ["django (>=1.8)"] +falcon = ["falcon (>=1.4)"] +fastapi = ["fastapi (>=0.79.0)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] +grpcio = ["grpcio (>=1.21.1)"] +httpx = ["httpx (>=0.16.0)"] +huey = ["huey (>=2)"] +loguru = ["loguru (>=0.5)"] +opentelemetry = ["opentelemetry-distro (>=0.35b0)"] +pure-eval = ["asttokens", "executing", "pure-eval"] +pymongo = ["pymongo (>=3.1)"] +pyspark = ["pyspark (>=2.4.4)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] +rq = ["rq (>=0.6)"] +sanic = ["sanic (>=0.8)"] +sqlalchemy = ["sqlalchemy (>=1.2)"] +starlette = ["starlette (>=0.19.1)"] +starlite = ["starlite (>=1.48)"] +tornado = ["tornado (>=5)"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.6" +files = [ + {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, + {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, +] + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "tabulate" +version = "0.8.10" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "tabulate-0.8.10-py3-none-any.whl", hash = "sha256:0ba055423dbaa164b9e456abe7920c5e8ed33fcc16f6d1b2f2d152c8e1e8b4fc"}, + {file = "tabulate-0.8.10.tar.gz", hash = "sha256:6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519"}, +] + +[package.extras] +widechars = ["wcwidth"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] + +[[package]] +name = "urllib3" +version = "1.26.16" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "websocket-client" +version = "1.6.1" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.7" +files = [ + {file = "websocket-client-1.6.1.tar.gz", hash = "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd"}, + {file = "websocket_client-1.6.1-py3-none-any.whl", hash = "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"}, +] + +[package.extras] +docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "wrapt" +version = "1.15.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +files = [ + {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, + {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, + {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, + {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, + {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, + {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, + {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, + {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, + {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, + {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, + {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, + {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, + {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, + {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, + {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, + {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, + {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, + {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, + {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, + {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, + {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, + {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, + {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, + {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, + {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, + {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, + {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, + {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, + {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, + {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, + {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, + {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, + {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, + {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, + {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, + {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, + {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, + {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +] + +[[package]] +name = "yarl" +version = "1.9.2" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, + {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, + {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, + {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, + {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, + {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, + {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, + {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, + {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, + {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, + {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, + {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, + {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "2903a2c7f1a7ebc91ada22a4e6b73aa96e7195e0d3da72310254862e49c6f22b" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml new file mode 100644 index 000000000000..20001f87d260 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "pipelines" +version = "1.1.0" +description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" +authors = ["Airbyte "] + +[tool.poetry.dependencies] +python = "^3.10" +dagger-io = "^0.6.4" +asyncer = "^0.0.2" +anyio = "^3.4.1" +more-itertools = "^8.11.0" +docker = "^5.0.3" +semver = "^3.0.1" +airbyte-protocol-models = "*" +tabulate = "^0.8.9" +jinja2 = "^3.0.2" +requests = "^2.28.2" +connector-ops = {path = "../connector_ops"} +toml = "^0.10.2" +sentry-sdk = "^1.28.1" + +[tool.poetry.group.test.dependencies] +pytest = "^6.2.5" +pytest-mock = "^3.10.0" + + +[tool.poetry.group.dev.dependencies] +freezegun = "^1.2.2" +pytest-cov = "^4.1.0" + +[tool.poetry.scripts] +airbyte-ci-internal = "pipelines.commands.airbyte_ci:airbyte_ci" +airbyte-ci = "pipelines.dagger_run:main" diff --git a/airbyte-ci/connectors/pipelines/pytest.ini b/airbyte-ci/connectors/pipelines/pytest.ini new file mode 100644 index 000000000000..0bd08b038c23 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=pipelines diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/__init__.py b/airbyte-ci/connectors/pipelines/tests/__init__.py similarity index 100% rename from tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/__init__.py rename to airbyte-ci/connectors/pipelines/tests/__init__.py diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py new file mode 100644 index 000000000000..47cfb0fc195f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +import sys +from pathlib import Path +from typing import Set + +import dagger +import git +import pytest +import requests +from connector_ops.utils import Connector +from pipelines import utils +from tests.utils import ALL_CONNECTORS + + +@pytest.fixture(scope="module") +def anyio_backend(): + return "asyncio" + + +@pytest.fixture(scope="module") +async def dagger_client(): + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: + yield client + + +@pytest.fixture(scope="session") +def oss_registry(): + response = requests.get("https://connectors.airbyte.com/files/registries/v0/oss_registry.json") + response.raise_for_status() + return response.json() + + +@pytest.fixture(scope="session") +def airbyte_repo_path() -> Path: + return Path(git.Repo(search_parent_directories=True).working_tree_dir) + + +@pytest.fixture +def new_connector(airbyte_repo_path: Path, mocker) -> Connector: + new_connector_code_directory = airbyte_repo_path / "airbyte-integrations/connectors/source-new-connector" + Path(new_connector_code_directory).mkdir() + + new_connector_code_directory.joinpath("metadata.yaml").touch() + mocker.patch.object( + utils, + "ALL_CONNECTOR_DEPENDENCIES", + [(connector, connector.get_local_dependency_paths()) for connector in utils.get_all_connectors_in_repo()], + ) + yield Connector("source-new-connector") + new_connector_code_directory.joinpath("metadata.yaml").unlink() + new_connector_code_directory.rmdir() + + +@pytest.fixture(autouse=True, scope="session") +def from_airbyte_root(airbyte_repo_path): + """ + Change the working directory to the root of the Airbyte repo. + This will make all the tests current working directory to be the root of the Airbyte repo as we've set autouse=True. + """ + original_dir = Path.cwd() + os.chdir(airbyte_repo_path) + yield airbyte_repo_path + os.chdir(original_dir) + + +@pytest.fixture(scope="session") +def all_connectors() -> Set[Connector]: + return ALL_CONNECTORS diff --git a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py new file mode 100644 index 000000000000..f48c061dbec2 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from connector_ops.utils import Connector +from pipelines.actions import environments +from pipelines.contexts import PipelineContext + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def python_connector() -> Connector: + return Connector("source-openweather") + + +@pytest.fixture +def context(dagger_client): + context = PipelineContext( + pipeline_name="test", + is_local=True, + git_branch="test", + git_revision="test", + ) + context.dagger_client = dagger_client + return context + + +async def test_with_installed_python_package(context, python_connector): + python_environment = context.dagger_client.container().from_("python:3.9") + installed_connector_package = await environments.with_installed_python_package( + context, + python_environment, + str(python_connector.code_directory), + ) + await installed_connector_package.with_exec(["python", "main.py", "spec"]) diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py new file mode 100644 index 000000000000..e1f1ebee2f73 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_bases.py @@ -0,0 +1,107 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import timedelta + +import anyio +import pytest +from dagger import DaggerError +from pipelines import bases + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestStep: + class DummyStep(bases.Step): + title = "Dummy step" + max_retries = 3 + max_duration = timedelta(seconds=2) + + async def _run(self, run_duration: timedelta) -> bases.StepResult: + await anyio.sleep(run_duration.total_seconds()) + return bases.StepResult(self, bases.StepStatus.SUCCESS) + + @pytest.fixture + def test_context(self, mocker): + return mocker.Mock(secrets_to_mask=[]) + + async def test_run_with_timeout(self, test_context): + step = self.DummyStep(test_context) + step_result = await step.run(run_duration=step.max_duration - timedelta(seconds=1)) + assert step_result.status == bases.StepStatus.SUCCESS + assert step.retry_count == 0 + + step_result = await step.run(run_duration=step.max_duration + timedelta(seconds=1)) + timed_out_step_result = step._get_timed_out_step_result() + assert step_result.status == timed_out_step_result.status + assert step_result.stdout == timed_out_step_result.stdout + assert step_result.stderr == timed_out_step_result.stderr + assert step_result.output_artifact == timed_out_step_result.output_artifact + assert step.retry_count == step.max_retries + 1 + + @pytest.mark.parametrize( + "step_status, exc_info, max_retries, max_dagger_error_retries, expect_retry", + [ + (bases.StepStatus.SUCCESS, None, 0, 0, False), + (bases.StepStatus.SUCCESS, None, 3, 0, False), + (bases.StepStatus.SUCCESS, None, 0, 3, False), + (bases.StepStatus.SUCCESS, None, 3, 3, False), + (bases.StepStatus.SKIPPED, None, 0, 0, False), + (bases.StepStatus.SKIPPED, None, 3, 0, False), + (bases.StepStatus.SKIPPED, None, 0, 3, False), + (bases.StepStatus.SKIPPED, None, 3, 3, False), + (bases.StepStatus.FAILURE, DaggerError(), 0, 0, False), + (bases.StepStatus.FAILURE, DaggerError(), 0, 3, True), + (bases.StepStatus.FAILURE, None, 0, 0, False), + (bases.StepStatus.FAILURE, None, 0, 3, False), + (bases.StepStatus.FAILURE, None, 3, 0, True), + ], + ) + async def test_run_with_retries(self, mocker, test_context, step_status, exc_info, max_retries, max_dagger_error_retries, expect_retry): + step = self.DummyStep(test_context) + step.max_dagger_error_retries = max_dagger_error_retries + step.max_retries = max_retries + step.max_duration = timedelta(seconds=60) + step.retry_delay = timedelta(seconds=0) + step._run = mocker.AsyncMock( + side_effect=[bases.StepResult(step, step_status, exc_info=exc_info)] * (max(max_dagger_error_retries, max_retries) + 1) + ) + + step_result = await step.run() + + if expect_retry: + assert step.retry_count > 0 + else: + assert step.retry_count == 0 + assert step_result.status == step_status + + +class TestReport: + @pytest.fixture + def test_context(self, mocker): + return mocker.Mock() + + def test_report_failed_if_it_has_no_step_result(self, test_context): + report = bases.Report(test_context, []) + assert not report.success + report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.FAILURE)]) + assert not report.success + + report = bases.Report( + test_context, [bases.StepResult(None, bases.StepStatus.FAILURE), bases.StepResult(None, bases.StepStatus.SUCCESS)] + ) + assert not report.success + + report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.SUCCESS)]) + assert report.success + + report = bases.Report( + test_context, [bases.StepResult(None, bases.StepStatus.SUCCESS), bases.StepResult(None, bases.StepStatus.SKIPPED)] + ) + assert report.success + + report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.SKIPPED)]) + assert report.success diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_commands/__init__.py similarity index 100% rename from tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/__init__.py rename to airbyte-ci/connectors/pipelines/tests/test_commands/__init__.py diff --git a/tools/ci_connector_ops/tests/test_pipelines/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/__init__.py similarity index 100% rename from tools/ci_connector_ops/tests/test_pipelines/__init__.py rename to airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/__init__.py diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py new file mode 100644 index 000000000000..0f9cda290e27 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -0,0 +1,262 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Callable + +import pytest +from click.testing import CliRunner +from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage +from pipelines.bases import ConnectorWithModifiedFiles +from pipelines.commands.groups import connectors +from tests.utils import pick_a_random_connector + + +@pytest.fixture(scope="session") +def runner(): + return CliRunner() + + +def test_get_selected_connectors_by_name_no_file_modification(): + connector = pick_a_random_connector() + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(connector.technical_name,), + selected_support_levels=(), + selected_languages=(), + modified=False, + metadata_changes_only=False, + modified_files=set(), + ) + + assert len(selected_connectors) == 1 + assert isinstance(selected_connectors[0], ConnectorWithModifiedFiles) + assert selected_connectors[0].technical_name == connector.technical_name + assert not selected_connectors[0].modified_files + + +def test_get_selected_connectors_by_support_level_no_file_modification(): + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=["certified"], + selected_languages=(), + modified=False, + metadata_changes_only=False, + modified_files=set(), + ) + + set([c.support_level for c in selected_connectors]) == {"certified"} + + +def test_get_selected_connectors_by_language_no_file_modification(): + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=(), + selected_languages=(ConnectorLanguage.LOW_CODE,), + modified=False, + metadata_changes_only=False, + modified_files=set(), + ) + + set([c.language for c in selected_connectors]) == {ConnectorLanguage.LOW_CODE} + + +def test_get_selected_connectors_by_name_with_file_modification(): + connector = pick_a_random_connector() + modified_files = {connector.code_directory / "setup.py"} + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(connector.technical_name,), + selected_support_levels=(), + selected_languages=(), + modified=False, + metadata_changes_only=False, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 1 + assert isinstance(selected_connectors[0], ConnectorWithModifiedFiles) + assert selected_connectors[0].technical_name == connector.technical_name + assert selected_connectors[0].modified_files == modified_files + + +def test_get_selected_connectors_by_name_and_support_level_or_languages_leads_to_intersection(): + connector = pick_a_random_connector() + modified_files = {connector.code_directory / "setup.py"} + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(connector.technical_name,), + selected_support_levels=(connector.support_level,), + selected_languages=(connector.language,), + modified=False, + metadata_changes_only=False, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 1 + + +def test_get_selected_connectors_with_modified(): + first_modified_connector = pick_a_random_connector() + second_modified_connector = pick_a_random_connector(other_picked_connectors=[first_modified_connector]) + modified_files = {first_modified_connector.code_directory / "setup.py", second_modified_connector.code_directory / "setup.py"} + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=(), + selected_languages=(), + modified=True, + metadata_changes_only=False, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 2 + + +def test_get_selected_connectors_with_modified_and_language(): + first_modified_connector = pick_a_random_connector(language=ConnectorLanguage.PYTHON) + second_modified_connector = pick_a_random_connector(language=ConnectorLanguage.JAVA, other_picked_connectors=[first_modified_connector]) + modified_files = {first_modified_connector.code_directory / "setup.py", second_modified_connector.code_directory / "setup.py"} + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=(), + selected_languages=(ConnectorLanguage.JAVA,), + modified=True, + metadata_changes_only=False, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 1 + assert selected_connectors[0].technical_name == second_modified_connector.technical_name + + +def test_get_selected_connectors_with_modified_and_support_level(): + first_modified_connector = pick_a_random_connector(support_level="community") + second_modified_connector = pick_a_random_connector(support_level="certified", other_picked_connectors=[first_modified_connector]) + modified_files = {first_modified_connector.code_directory / "setup.py", second_modified_connector.code_directory / "setup.py"} + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=["certified"], + selected_languages=(), + modified=True, + metadata_changes_only=False, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 1 + assert selected_connectors[0].technical_name == second_modified_connector.technical_name + + +def test_get_selected_connectors_with_modified_and_metadata_only(): + first_modified_connector = pick_a_random_connector() + second_modified_connector = pick_a_random_connector(other_picked_connectors=[first_modified_connector]) + modified_files = { + first_modified_connector.code_directory / "setup.py", + second_modified_connector.code_directory / METADATA_FILE_NAME, + second_modified_connector.code_directory / "setup.py", + } + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=(), + selected_languages=(), + modified=True, + metadata_changes_only=True, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 1 + assert selected_connectors[0].technical_name == second_modified_connector.technical_name + assert selected_connectors[0].modified_files == { + second_modified_connector.code_directory / METADATA_FILE_NAME, + second_modified_connector.code_directory / "setup.py", + } + + +def test_get_selected_connectors_with_metadata_only(): + first_modified_connector = pick_a_random_connector() + second_modified_connector = pick_a_random_connector(other_picked_connectors=[first_modified_connector]) + modified_files = { + first_modified_connector.code_directory / "setup.py", + second_modified_connector.code_directory / METADATA_FILE_NAME, + second_modified_connector.code_directory / "setup.py", + } + selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=(), + selected_languages=(), + modified=False, + metadata_changes_only=True, + modified_files=modified_files, + ) + + assert len(selected_connectors) == 1 + assert selected_connectors[0].technical_name == second_modified_connector.technical_name + assert selected_connectors[0].modified_files == { + second_modified_connector.code_directory / METADATA_FILE_NAME, + second_modified_connector.code_directory / "setup.py", + } + + +@pytest.fixture() +def click_context_obj(): + return { + "git_branch": "test_branch", + "git_revision": "test_revision", + "pipeline_start_timestamp": 0, + "ci_context": "manual", + "show_dagger_logs": False, + "is_local": True, + "is_ci": False, + "select_modified_connectors": False, + "selected_connectors_with_modified_files": {}, + "gha_workflow_run_url": None, + "ci_report_bucket_name": None, + "use_remote_secrets": False, + "ci_gcs_credentials": None, + "execute_timeout": 0, + "concurrency": 1, + "ci_git_user": None, + "ci_github_access_token": None, + } + + +@pytest.mark.parametrize( + "command, command_args", + [ + (connectors.test, []), + ( + connectors.publish, + [ + "--spec-cache-gcs-credentials", + "test", + "--spec-cache-bucket-name", + "test", + "--metadata-service-gcs-credentials", + "test", + "--metadata-service-bucket-name", + "test", + "--docker-hub-username", + "test", + "--docker-hub-password", + "test", + ], + ), + (connectors.format_code, []), + (connectors.build, []), + ], +) +def test_commands_do_not_override_connector_selection( + mocker, runner: CliRunner, click_context_obj: dict, command: Callable, command_args: list +): + """ + This test is here to make sure that the commands do not override the connector selection + This is important because we want to control the connector selection in a single place. + """ + + selected_connector = mocker.MagicMock() + click_context_obj["selected_connectors_with_modified_files"] = [selected_connector] + + mocker.patch.object(connectors.click, "confirm") + mock_connector_context = mocker.MagicMock() + mocker.patch.object(connectors, "ConnectorContext", mock_connector_context) + mocker.patch.object(connectors, "PublishConnectorContext", mock_connector_context) + runner.invoke(command, command_args, catch_exceptions=False, obj=click_context_obj) + assert mock_connector_context.call_count == 1 + # If the connector selection is overriden the context won't be instantiated with the selected connector mock instance + assert mock_connector_context.call_args_list[0].kwargs["connector"] == selected_connector diff --git a/airbyte-ci/connectors/pipelines/tests/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_environments.py new file mode 100644 index 000000000000..db48914fdeb7 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_environments.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py new file mode 100644 index 000000000000..e45027c860dc --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path + +import pytest +from pipelines import bases, gradle + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestGradleTask: + class DummyStep(gradle.GradleTask): + gradle_task_name = "dummyTask" + + async def _run(self) -> bases.StepResult: + return bases.StepResult(self, bases.StepStatus.SUCCESS) + + @pytest.fixture + def test_context(self, mocker, dagger_client): + return mocker.Mock( + secrets_to_mask=[], + dagger_client=dagger_client, + connector=bases.ConnectorWithModifiedFiles( + "source-postgres", frozenset({Path("airbyte-integrations/connectors/source-postgres/metadata.yaml")}) + ), + ) + + async def test_build_include(self, test_context): + step = self.DummyStep(test_context) + assert step.build_include diff --git a/tools/ci_connector_ops/tests/test_pipelines/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py similarity index 76% rename from tools/ci_connector_ops/tests/test_pipelines/test_publish.py rename to airbyte-ci/connectors/pipelines/tests/test_publish.py index d464ff73263d..9bcf38a9bfcd 100644 --- a/tools/ci_connector_ops/tests/test_pipelines/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -1,41 +1,32 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json import random from typing import List import anyio -import dagger import pytest -import requests -from ci_connector_ops.pipelines import publish -from ci_connector_ops.pipelines.bases import StepStatus - - -@pytest.fixture(scope="module") -def anyio_backend(): - return "asyncio" - - -@pytest.fixture(scope="module") -async def dagger_client(): - async with dagger.Connection() as client: - yield client - - -@pytest.fixture(scope="module") -def oss_registry(): - response = requests.get("https://connectors.airbyte.com/files/registries/v0/oss_registry.json") - response.raise_for_status() - return response.json() - +from pipelines import publish +from pipelines.bases import StepStatus pytestmark = [ pytest.mark.anyio, ] +@pytest.fixture +def publish_context(mocker, dagger_client, tmpdir): + return mocker.MagicMock( + dagger_client=dagger_client, + get_connector_dir=mocker.MagicMock(return_value=dagger_client.host().directory(str(tmpdir))), + docker_hub_username_secret=None, + docker_hub_password_secret=None, + docker_image="hello-world:latest", + ) + + class TestCheckConnectorImageDoesNotExists: @pytest.fixture(scope="class") def three_random_connectors_image_names(self, oss_registry: dict) -> List[str]: @@ -43,35 +34,28 @@ def three_random_connectors_image_names(self, oss_registry: dict) -> List[str]: random.shuffle(connectors) return [f"{connector['dockerRepository']}:{connector['dockerImageTag']}" for connector in connectors[:3]] - @pytest.mark.slow - async def test_run(self, mocker, dagger_client, three_random_connectors_image_names): - """We pick the first three connectors from the OSS registry and check that they are already published.""" + async def test_run_skipped_when_already_published(self, three_random_connectors_image_names, publish_context): + """We pick three random connectors from the OSS registry. They should be published. We check that the step is skipped.""" for image_name in three_random_connectors_image_names: - context = mocker.MagicMock(dagger_client=dagger_client, docker_image_name=image_name) - step = publish.CheckConnectorImageDoesNotExist(context) + publish_context.docker_image = image_name + step = publish.CheckConnectorImageDoesNotExist(publish_context) step_result = await step.run() assert step_result.status == StepStatus.SKIPPED - image_name = "airbyte/source-pokeapi:0.0.0" - context = mocker.MagicMock(dagger_client=dagger_client, docker_image_name=image_name) - step = publish.CheckConnectorImageDoesNotExist(context) + + async def test_run_success_when_already_published(self, publish_context): + publish_context.docker_image = "airbyte/source-pokeapi:0.0.0" + step = publish.CheckConnectorImageDoesNotExist(publish_context) step_result = await step.run() assert step_result.status == StepStatus.SUCCESS class TestUploadSpecToCache: @pytest.fixture(scope="class") - def random_connector(self, oss_registry): - return random.choice(oss_registry["sources"] + oss_registry["destinations"]) - - @pytest.fixture - def context(self, mocker, dagger_client, random_connector, tmpdir): - image_name = f"{random_connector['dockerRepository']}:{random_connector['dockerImageTag']}" - tmp_dir = dagger_client.host().directory(str(tmpdir)) - return mocker.MagicMock( - dagger_client=dagger_client, get_connector_dir=mocker.MagicMock(return_value=tmp_dir), docker_image_name=image_name - ) + def random_connector(self, oss_registry: dict) -> dict: + connectors = oss_registry["sources"] + oss_registry["destinations"] + random.shuffle(connectors) + return connectors[0] - @pytest.mark.slow @pytest.mark.parametrize( "valid_spec, successful_upload", [ @@ -81,14 +65,15 @@ def context(self, mocker, dagger_client, random_connector, tmpdir): [False, False], ], ) - async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, random_connector, context): + async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, random_connector, publish_context): """Test that the spec is correctly uploaded to the spec cache bucket. We pick a random connector from the oss registry, by nature this connector should have a valid spec and be published. - We use load this connector as a Dagger container and run spec against it. + We load this connector as a Dagger container and run spec against it. We validate that the outputted spec is the same as the one in the OSS registry. We also artificially set the spec to be invalid and check that the step fails. """ image_name = f"{random_connector['dockerRepository']}:{random_connector['dockerImageTag']}" + publish_context.docker_image = image_name expected_spec = random_connector["spec"] connector_container = dagger_client.container().from_(image_name) @@ -101,15 +86,15 @@ async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, r publish.UploadSpecToCache, "_get_connector_spec", mocker.Mock(side_effect=publish.InvalidSpecOutputError("Invalid spec.")) ) - step = publish.UploadSpecToCache(context) + step = publish.UploadSpecToCache(publish_context) step_result = await step.run(connector_container) if valid_spec: publish.upload_to_gcs.assert_called_once_with( - context.dagger_client, + publish_context.dagger_client, mocker.ANY, f"specs/{image_name.replace(':', '/')}/spec.json", - context.spec_cache_bucket_name, - context.spec_cache_gcs_credentials_secret, + publish_context.spec_cache_bucket_name, + publish_context.spec_cache_gcs_credentials_secret, flags=['--cache-control="no-cache"'], ) @@ -131,27 +116,27 @@ async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, r assert step_result.stdout is None publish.upload_to_gcs.assert_not_called() - def test_parse_spec_output_valid(self, context, random_connector): - step = publish.UploadSpecToCache(context) + def test_parse_spec_output_valid(self, publish_context, random_connector): + step = publish.UploadSpecToCache(publish_context) correct_spec_message = json.dumps({"type": "SPEC", "spec": random_connector["spec"]}) spec_output = f'random_stuff\n{{"type": "RANDOM_MESSAGE"}}\n{correct_spec_message}' result = step._parse_spec_output(spec_output) assert json.loads(result) == random_connector["spec"] - def test_parse_spec_output_invalid_json(self, context): - step = publish.UploadSpecToCache(context) + def test_parse_spec_output_invalid_json(self, publish_context): + step = publish.UploadSpecToCache(publish_context) spec_output = "Invalid JSON" with pytest.raises(publish.InvalidSpecOutputError): step._parse_spec_output(spec_output) - def test_parse_spec_output_invalid_key(self, context): - step = publish.UploadSpecToCache(context) + def test_parse_spec_output_invalid_key(self, publish_context): + step = publish.UploadSpecToCache(publish_context) spec_output = '{"type": "SPEC", "spec": {"invalid_key": "value"}}' with pytest.raises(publish.InvalidSpecOutputError): step._parse_spec_output(spec_output) - def test_parse_spec_output_no_spec(self, context): - step = publish.UploadSpecToCache(context) + def test_parse_spec_output_no_spec(self, publish_context): + step = publish.UploadSpecToCache(publish_context) spec_output = '{"type": "OTHER"}' with pytest.raises(publish.InvalidSpecOutputError): step._parse_spec_output(spec_output) @@ -170,7 +155,7 @@ def test_parse_spec_output_no_spec(self, context): @pytest.mark.parametrize("pre_release", [True, False]) async def test_run_connector_publish_pipeline_when_failed_validation(mocker, pre_release): - """We validate the no other steps are called if the metadata validation step fails.""" + """We validate that no other steps are called if the metadata validation step fails.""" for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) @@ -197,10 +182,10 @@ async def test_run_connector_publish_pipeline_when_failed_validation(mocker, pre @pytest.mark.parametrize( - "check_image_exists_status, pre_release", - [(StepStatus.SKIPPED, False), (StepStatus.SKIPPED, True), (StepStatus.FAILURE, True), (StepStatus.FAILURE, False)], + "check_image_exists_status", + [StepStatus.SKIPPED, StepStatus.FAILURE], ) -async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker, check_image_exists_status, pre_release): +async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker, check_image_exists_status, publish_context): """We validate that when the connector image exists or the check fails, we don't run the rest of the pipeline. We also validate that the metadata upload step is called when the image exists (Skipped status). We do this to ensure that the metadata is still updated in the case where the connector image already exists. @@ -222,9 +207,8 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker run_metadata_upload = publish.metadata.MetadataUpload.return_value.run - context = mocker.MagicMock(pre_release=pre_release) semaphore = anyio.Semaphore(1) - report = await publish.run_connector_publish_pipeline(context, semaphore) + report = await publish.run_connector_publish_pipeline(publish_context, semaphore) run_metadata_validation.assert_called_once() run_check_connector_image_does_not_exist.assert_called_once() @@ -233,22 +217,11 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker if to_mock not in ["MetadataValidation", "MetadataUpload", "CheckConnectorImageDoesNotExist", "UploadSpecToCache"]: getattr(module, to_mock).return_value.run.assert_not_called() - if check_image_exists_status is StepStatus.SKIPPED and pre_release: - run_metadata_upload.assert_not_called() - assert ( - report.steps_results - == context.report.steps_results - == [ - run_metadata_validation.return_value, - run_check_connector_image_does_not_exist.return_value, - ] - ) - - if check_image_exists_status is StepStatus.SKIPPED and not pre_release: + if check_image_exists_status is StepStatus.SKIPPED: run_metadata_upload.assert_called_once() assert ( report.steps_results - == context.report.steps_results + == publish_context.report.steps_results == [ run_metadata_validation.return_value, run_check_connector_image_does_not_exist.return_value, @@ -261,7 +234,7 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker run_metadata_upload.assert_not_called() assert ( report.steps_results - == context.report.steps_results + == publish_context.report.steps_results == [ run_metadata_validation.return_value, run_check_connector_image_does_not_exist.return_value, @@ -323,7 +296,9 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( name="metadata_upload_result", status=metadata_upload_step_status ) - context = mocker.MagicMock(pre_release=pre_release, ) + context = mocker.MagicMock( + pre_release=pre_release, + ) semaphore = anyio.Semaphore(1) report = await publish.run_connector_publish_pipeline(context, semaphore) @@ -335,15 +310,12 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( publish.PullConnectorImageFromRegistry.return_value.run, ] - if not pre_release: - steps_to_run += [publish.metadata.MetadataUpload.return_value.run] - for i, step_to_run in enumerate(steps_to_run): if step_to_run.return_value.status is StepStatus.FAILURE or i == len(steps_to_run) - 1: assert len(report.steps_results) == len(context.report.steps_results) previous_steps = steps_to_run[:i] - for k, step_ran in enumerate(previous_steps): + for _, step_ran in enumerate(previous_steps): step_ran.assert_called_once() step_ran.return_value @@ -358,6 +330,3 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( publish.PullConnectorImageFromRegistry.return_value.run.assert_not_called() publish.UploadSpecToCache.return_value.run.assert_not_called() publish.metadata.MetadataUpload.return_value.run.assert_not_called() - - if pre_release: - publish.metadata.MetadataUpload.return_value.run.assert_not_called() diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py new file mode 100644 index 000000000000..0bdcd5158c93 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -0,0 +1,227 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +import pathlib +import time +from typing import List + +import dagger +import pytest +import yaml +from freezegun import freeze_time +from pipelines.bases import ConnectorWithModifiedFiles, StepStatus +from pipelines.tests import common + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestAcceptanceTests: + @staticmethod + def get_dummy_cat_container(dagger_client: dagger.Client, exit_code: int, secret_file_paths: List, stdout: str, stderr: str): + secret_file_paths = secret_file_paths or [] + container = ( + dagger_client.container() + .from_("bash:latest") + .with_exec(["mkdir", "-p", common.AcceptanceTests.CONTAINER_TEST_INPUT_DIRECTORY]) + .with_exec(["mkdir", "-p", common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY]) + ) + + for secret_file_path in secret_file_paths: + secret_dir_name = str(pathlib.Path(secret_file_path).parent) + container = container.with_exec(["mkdir", "-p", secret_dir_name]) + container = container.with_exec(["sh", "-c", f"echo foo > {secret_file_path}"]) + return container.with_new_file("/stupid_bash_script.sh", contents=f"echo {stdout}; echo {stderr} >&2; exit {exit_code}") + + @pytest.fixture + def test_context(self, mocker, dagger_client): + return mocker.MagicMock(connector=ConnectorWithModifiedFiles("source-faker", frozenset()), dagger_client=dagger_client) + + @pytest.fixture + def dummy_connector_under_test_image_tar(self, dagger_client, tmpdir) -> dagger.File: + dummy_tar_file = tmpdir / "dummy.tar" + dummy_tar_file.write_text("dummy", encoding="utf8") + return dagger_client.host().directory(str(tmpdir), include=["dummy.tar"]).file("dummy.tar") + + @pytest.fixture + def another_dummy_connector_under_test_image_tar(self, dagger_client, tmpdir) -> dagger.File: + dummy_tar_file = tmpdir / "another_dummy.tar" + dummy_tar_file.write_text("another_dummy", encoding="utf8") + return dagger_client.host().directory(str(tmpdir), include=["another_dummy.tar"]).file("another_dummy.tar") + + async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context): + test_context.connector = mocker.MagicMock(acceptance_test_config=None) + acceptance_test_step = common.AcceptanceTests(test_context) + step_result = await acceptance_test_step._run(None) + assert step_result.status == StepStatus.SKIPPED + + @pytest.mark.parametrize( + "exit_code,expected_status,secrets_file_names,expect_updated_secrets", + [ + (0, StepStatus.SUCCESS, [], False), + (1, StepStatus.FAILURE, [], False), + (2, StepStatus.FAILURE, [], False), + (common.AcceptanceTests.skipped_exit_code, StepStatus.SKIPPED, [], False), + (0, StepStatus.SUCCESS, [f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json"], False), + (1, StepStatus.FAILURE, [f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json"], False), + (2, StepStatus.FAILURE, [f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json"], False), + ( + common.AcceptanceTests.skipped_exit_code, + StepStatus.SKIPPED, + [f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json"], + False, + ), + ( + 0, + StepStatus.SUCCESS, + [ + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json", + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/updated_configurations/updated_config.json", + ], + True, + ), + ( + 1, + StepStatus.FAILURE, + [ + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json", + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/updated_configurations/updated_config.json", + ], + True, + ), + ( + 2, + StepStatus.FAILURE, + [ + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json", + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/updated_configurations/updated_config.json", + ], + True, + ), + ( + common.AcceptanceTests.skipped_exit_code, + StepStatus.SKIPPED, + [ + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/config.json", + f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}/updated_configurations/updated_config.json", + ], + True, + ), + ], + ) + async def test__run( + self, + test_context, + mocker, + exit_code: int, + expected_status: StepStatus, + secrets_file_names: List, + expect_updated_secrets: bool, + test_input_dir: dagger.Directory, + ): + """Test the behavior of the run function using a dummy container.""" + cat_container = self.get_dummy_cat_container( + test_context.dagger_client, exit_code, secrets_file_names, stdout="hello", stderr="world" + ) + async_mock = mocker.AsyncMock(return_value=cat_container) + mocker.patch.object(common.AcceptanceTests, "_build_connector_acceptance_test", side_effect=async_mock) + mocker.patch.object(common.AcceptanceTests, "get_cat_command", return_value=["bash", "/stupid_bash_script.sh"]) + test_context.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) + acceptance_test_step = common.AcceptanceTests(test_context) + step_result = await acceptance_test_step._run(None) + assert step_result.status == expected_status + assert step_result.stdout.strip() == "hello" + assert step_result.stderr.strip() == "world" + if expect_updated_secrets: + assert ( + await test_context.updated_secrets_dir.entries() + == await cat_container.directory(f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}").entries() + ) + assert any("updated_configurations" in str(file_name) for file_name in await test_context.updated_secrets_dir.entries()) + + @pytest.fixture + def test_input_dir(self, dagger_client, tmpdir): + with open(tmpdir / "acceptance-test-config.yml", "w") as f: + yaml.safe_dump({"connector_image": "airbyte/connector_under_test_image:dev"}, f) + return dagger_client.host().directory(str(tmpdir)) + + def get_patched_acceptance_test_step(self, dagger_client, mocker, test_context, test_input_dir): + test_context.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) + test_context.connector_acceptance_test_image = "bash:latest" + test_context.connector_secrets = {"config.json": dagger_client.set_secret("config.json", "connector_secret")} + + mocker.patch.object(common.environments, "load_image_to_docker_host", return_value="image_sha") + mocker.patch.object(common.environments, "with_bound_docker_host", lambda _, cat_container: cat_container) + return common.AcceptanceTests(test_context) + + async def test_cat_container_provisioning( + self, dagger_client, mocker, test_context, test_input_dir, dummy_connector_under_test_image_tar + ): + """Check that the acceptance test container is correctly provisioned. + We check that: + - the test input and secrets are correctly mounted. + - the cache buster and image sha are correctly set as environment variables. + - that the entrypoint is correctly set. + - the current working directory is correctly set. + """ + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context, test_input_dir) + cat_container = await acceptance_test_step._build_connector_acceptance_test(dummy_connector_under_test_image_tar, test_input_dir) + assert (await cat_container.with_exec(["pwd"]).stdout()).strip() == acceptance_test_step.CONTAINER_TEST_INPUT_DIRECTORY + test_input_ls_result = await cat_container.with_exec(["ls"]).stdout() + assert all( + file_or_directory in test_input_ls_result.splitlines() for file_or_directory in ["secrets", "acceptance-test-config.yml"] + ) + assert await cat_container.with_exec(["cat", f"{acceptance_test_step.CONTAINER_SECRETS_DIRECTORY}/config.json"]).stdout() == "***" + env_vars = {await env_var.name(): await env_var.value() for env_var in await cat_container.env_variables()} + assert "CACHEBUSTER" in env_vars + + async def test_cat_container_caching( + self, + dagger_client, + mocker, + test_context, + test_input_dir, + dummy_connector_under_test_image_tar, + another_dummy_connector_under_test_image_tar, + ): + """Check that the acceptance test container caching behavior is correct.""" + + initial_datetime = datetime.datetime(year=1992, month=6, day=19, hour=13, minute=1, second=0) + + with freeze_time(initial_datetime) as frozen_datetime: + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context, test_input_dir) + cat_container = await acceptance_test_step._build_connector_acceptance_test( + dummy_connector_under_test_image_tar, test_input_dir + ) + cat_container = cat_container.with_exec(["date"]) + fist_date_result = await cat_container.stdout() + + frozen_datetime.tick(delta=datetime.timedelta(hours=5)) + # Check that cache is used in the same day + cat_container = await acceptance_test_step._build_connector_acceptance_test( + dummy_connector_under_test_image_tar, test_input_dir + ) + cat_container = cat_container.with_exec(["date"]) + second_date_result = await cat_container.stdout() + assert fist_date_result == second_date_result + + # Check that cache bursted after a day + frozen_datetime.tick(delta=datetime.timedelta(days=1, seconds=1)) + cat_container = await acceptance_test_step._build_connector_acceptance_test( + dummy_connector_under_test_image_tar, test_input_dir + ) + cat_container = cat_container.with_exec(["date"]) + third_date_result = await cat_container.stdout() + assert third_date_result != second_date_result + + time.sleep(1) + # Check that changing the tarball invalidates the cache + cat_container = await acceptance_test_step._build_connector_acceptance_test( + another_dummy_connector_under_test_image_tar, test_input_dir + ) + cat_container = cat_container.with_exec(["date"]) + fourth_date_result = await cat_container.stdout() + assert fourth_date_result != third_date_result diff --git a/airbyte-ci/connectors/pipelines/tests/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py new file mode 100644 index 000000000000..58b9c9fd78cd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_utils.py @@ -0,0 +1,183 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path +from unittest import mock + +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from pipelines import utils +from tests.utils import pick_a_random_connector + + +@pytest.mark.parametrize( + "ctx, expected", + [ + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": None, + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_context/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch/with/slashes", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashes/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch/with/slashes#and!special@characters", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="airbyte-ci command path", + obj={ + "git_branch": "my_branch/with/slashes#and!special@characters", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="airbyte-ci-internal command path", + obj={ + "git_branch": "my_branch/with/slashes#and!special@characters", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", + ), + ], +) +def test_render_report_output_prefix(ctx, expected): + assert utils.DaggerPipelineCommand.render_report_output_prefix(ctx) == expected + + +@pytest.mark.parametrize("enable_dependency_scanning", [True, False]) +def test_get_modified_connectors_with_dependency_scanning(all_connectors, enable_dependency_scanning): + base_java_changed_file = Path("airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/BaseConnector.java") + modified_files = [base_java_changed_file] + + not_modified_java_connector = pick_a_random_connector(language=ConnectorLanguage.JAVA) + modified_java_connector = pick_a_random_connector( + language=ConnectorLanguage.JAVA, other_picked_connectors=[not_modified_java_connector] + ) + modified_files.append(modified_java_connector.code_directory / "foo.bar") + + modified_connectors = utils.get_modified_connectors(modified_files, all_connectors, enable_dependency_scanning) + if enable_dependency_scanning: + assert not_modified_java_connector in modified_connectors + else: + assert not_modified_java_connector not in modified_connectors + assert modified_java_connector in modified_connectors + + +def test_get_connector_modified_files(): + connector = pick_a_random_connector() + other_connector = pick_a_random_connector(other_picked_connectors=[connector]) + + all_modified_files = { + connector.code_directory / "setup.py", + other_connector.code_directory / "README.md", + } + + result = utils.get_connector_modified_files(connector, all_modified_files) + assert result == frozenset({connector.code_directory / "setup.py"}) + + +def test_no_modified_files_in_connector_directory(): + connector = pick_a_random_connector() + other_connector = pick_a_random_connector(other_picked_connectors=[connector]) + + all_modified_files = { + other_connector.code_directory / "README.md", + } + + result = utils.get_connector_modified_files(connector, all_modified_files) + assert result == frozenset() + + +@pytest.mark.anyio +async def test_check_path_in_workdir(dagger_client): + connector = Connector("source-openweather") + container = ( + dagger_client.container() + .from_("bash") + .with_mounted_directory(str(connector.code_directory), dagger_client.host().directory(str(connector.code_directory))) + .with_workdir(str(connector.code_directory)) + ) + assert await utils.check_path_in_workdir(container, "metadata.yaml") + assert await utils.check_path_in_workdir(container, "setup.py") + assert await utils.check_path_in_workdir(container, "requirements.txt") + assert await utils.check_path_in_workdir(container, "not_existing_file") is False diff --git a/airbyte-ci/connectors/pipelines/tests/utils.py b/airbyte-ci/connectors/pipelines/tests/utils.py new file mode 100644 index 000000000000..fc1c4ae5d6e6 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/utils.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import random + +from connector_ops.utils import Connector, ConnectorLanguage, get_all_connectors_in_repo + +ALL_CONNECTORS = get_all_connectors_in_repo() + + +def pick_a_random_connector( + language: ConnectorLanguage = None, support_level: str = None, other_picked_connectors: list = None +) -> Connector: + """Pick a random connector from the list of all connectors.""" + all_connectors = list(ALL_CONNECTORS) + if language: + all_connectors = [c for c in all_connectors if c.language is language] + if support_level: + all_connectors = [c for c in all_connectors if c.support_level == support_level] + picked_connector = random.choice(all_connectors) + if other_picked_connectors: + while picked_connector in other_picked_connectors: + picked_connector = random.choice(all_connectors) + return picked_connector diff --git a/airbyte-ci/connectors/qa-engine/poetry.lock b/airbyte-ci/connectors/qa-engine/poetry.lock index a82d2601178d..9feaa409fd5f 100644 --- a/airbyte-ci/connectors/qa-engine/poetry.lock +++ b/airbyte-ci/connectors/qa-engine/poetry.lock @@ -1,100 +1,99 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "aiohttp" -version = "3.8.4" +version = "3.8.5" description = "Async http client/server framework (asyncio)" -category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a"}, - {file = "aiohttp-3.8.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5"}, - {file = "aiohttp-3.8.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949"}, - {file = "aiohttp-3.8.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea"}, - {file = "aiohttp-3.8.4-cp310-cp310-win32.whl", hash = "sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1"}, - {file = "aiohttp-3.8.4-cp310-cp310-win_amd64.whl", hash = "sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4"}, - {file = "aiohttp-3.8.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05"}, - {file = "aiohttp-3.8.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b"}, - {file = "aiohttp-3.8.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24"}, - {file = "aiohttp-3.8.4-cp311-cp311-win32.whl", hash = "sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d"}, - {file = "aiohttp-3.8.4-cp311-cp311-win_amd64.whl", hash = "sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc"}, - {file = "aiohttp-3.8.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01"}, - {file = "aiohttp-3.8.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71"}, - {file = "aiohttp-3.8.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win32.whl", hash = "sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777"}, - {file = "aiohttp-3.8.4-cp36-cp36m-win_amd64.whl", hash = "sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e"}, - {file = "aiohttp-3.8.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab"}, - {file = "aiohttp-3.8.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6"}, - {file = "aiohttp-3.8.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win32.whl", hash = "sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a"}, - {file = "aiohttp-3.8.4-cp37-cp37m-win_amd64.whl", hash = "sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15"}, - {file = "aiohttp-3.8.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8"}, - {file = "aiohttp-3.8.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275"}, - {file = "aiohttp-3.8.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d"}, - {file = "aiohttp-3.8.4-cp38-cp38-win32.whl", hash = "sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54"}, - {file = "aiohttp-3.8.4-cp38-cp38-win_amd64.whl", hash = "sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567"}, - {file = "aiohttp-3.8.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2"}, - {file = "aiohttp-3.8.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14"}, - {file = "aiohttp-3.8.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4"}, - {file = "aiohttp-3.8.4-cp39-cp39-win32.whl", hash = "sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a"}, - {file = "aiohttp-3.8.4-cp39-cp39-win_amd64.whl", hash = "sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04"}, - {file = "aiohttp-3.8.4.tar.gz", hash = "sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"}, + {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"}, + {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"}, + {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"}, + {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"}, + {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"}, + {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"}, + {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"}, + {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"}, + {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"}, + {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"}, + {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"}, + {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"}, + {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"}, + {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"}, + {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"}, + {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"}, + {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"}, + {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"}, + {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"}, + {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"}, + {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"}, + {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"}, + {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"}, + {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"}, + {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"}, + {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"}, + {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"}, ] [package.dependencies] @@ -113,7 +112,6 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -126,21 +124,19 @@ frozenlist = ">=1.1.0" [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, ] [[package]] name = "atomicwrites" version = "1.4.1" description = "Atomic file writes." -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -151,7 +147,6 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -168,33 +163,30 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte [[package]] name = "cachetools" -version = "5.3.0" +version = "5.3.1" description = "Extensible memoizing collections and decorators" -category = "main" optional = false -python-versions = "~=3.7" +python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, - {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, + {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, + {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, ] [[package]] name = "certifi" -version = "2023.5.7" +version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, ] [[package]] name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = "*" files = [ @@ -269,111 +261,127 @@ pycparser = "*" [[package]] name = "chardet" -version = "5.1.0" +version = "5.2.0" description = "Universal encoding detector for Python 3" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, - {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, ] [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.2.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] + +[[package]] +name = "ci-credentials" +version = "1.1.0" +description = "CLI tooling to read and manage GSM secrets" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +click = "^8.1.3" +common_utils = {path = "../common_utils", develop = true} +pyyaml = "^6.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../ci_credentials" [[package]] name = "click" -version = "8.1.3" +version = "8.1.6" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, + {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, + {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, ] [package.dependencies] @@ -383,7 +391,6 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -391,58 +398,111 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "common-utils" +version = "0.0.0" +description = "Suite of all often used classes and common functions" +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +cryptography = "^3.4.7" +pyjwt = "^2.1.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../common_utils" + +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +optional = false +python-versions = "*" +files = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "connector-ops" +version = "0.2.2" +description = "Packaged maintained by the connector operations team to perform CI for connectors" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +ci-credentials = {path = "../ci_credentials"} +click = "^8.1.3" +GitPython = "^3.1.29" +google-cloud-storage = "^2.8.0" +pydantic = "^1.9" +pydash = "^7.0.4" +PyGithub = "^1.58.0" +PyYAML = "^6.0" +requests = "^2.28.2" +rich = "^11.0.1" + +[package.source] +type = "directory" +url = "../connector_ops" + [[package]] name = "cryptography" -version = "40.0.2" +version = "3.4.8" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "cryptography-40.0.2-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b"}, - {file = "cryptography-40.0.2-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440"}, - {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d"}, - {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288"}, - {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2"}, - {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b"}, - {file = "cryptography-40.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9"}, - {file = "cryptography-40.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c"}, - {file = "cryptography-40.0.2-cp36-abi3-win32.whl", hash = "sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9"}, - {file = "cryptography-40.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b"}, - {file = "cryptography-40.0.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b"}, - {file = "cryptography-40.0.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e"}, - {file = "cryptography-40.0.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a"}, - {file = "cryptography-40.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958"}, - {file = "cryptography-40.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b"}, - {file = "cryptography-40.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636"}, - {file = "cryptography-40.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e"}, - {file = "cryptography-40.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404"}, - {file = "cryptography-40.0.2.tar.gz", hash = "sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89"}, + {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, + {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, + {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, ] [package.dependencies] cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "check-manifest", "mypy", "ruff"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist"] -test-randomorder = ["pytest-randomly"] -tox = ["tox"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] [[package]] name = "dataproperty" -version = "0.55.1" +version = "1.0.1" description = "Python library for extract property from data." -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "DataProperty-0.55.1-py3-none-any.whl", hash = "sha256:54a5c1440b0b5ab2e902c29b451dd408dadefd93232ae21672e6f1305698f802"}, - {file = "DataProperty-0.55.1.tar.gz", hash = "sha256:08113748e164c17d60b9e66c256cbabec0c654b7aaf5e8441c4ec4ee51d5c921"}, + {file = "DataProperty-1.0.1-py3-none-any.whl", hash = "sha256:0b8b07d4fb6453fcf975b53d35dea41f3cfd69c9d79b5010c3cf224ff0407a7a"}, + {file = "DataProperty-1.0.1.tar.gz", hash = "sha256:723e5729fa6e885e127a771a983ee1e0e34bb141aca4ffe1f0bfa7cde34650a4"}, ] [package.dependencies] @@ -457,7 +517,6 @@ test = ["pytest (>=6.0.1)", "pytest-md-report (>=0.3)", "tcolorpy (>=0.1.2)"] name = "db-dtypes" version = "1.1.1" description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -475,7 +534,6 @@ pyarrow = ">=3.0.0" name = "decorator" version = "5.1.1" description = "Decorators for Humans" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -485,27 +543,25 @@ files = [ [[package]] name = "deprecated" -version = "1.2.13" +version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, - {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, ] [package.dependencies] wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "freezegun" version = "1.2.2" description = "Let your Python tests travel through time" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -518,93 +574,78 @@ python-dateutil = ">=2.7" [[package]] name = "frozenlist" -version = "1.3.3" +version = "1.4.0" description = "A list-like structure which implements collections.abc.MutableSequence" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"}, - {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"}, - {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"}, - {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"}, - {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"}, - {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"}, - {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"}, - {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"}, - {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"}, - {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"}, - {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"}, - {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"}, - {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"}, - {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"}, - {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"}, - {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"}, - {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"}, - {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"}, - {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"}, - {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"}, - {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"}, - {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"}, - {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"}, - {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"}, - {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"}, - {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559"}, + {file = "frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963"}, + {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9"}, + {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62"}, + {file = "frozenlist-1.4.0-cp310-cp310-win32.whl", hash = "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0"}, + {file = "frozenlist-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3"}, + {file = "frozenlist-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b"}, + {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467"}, + {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb"}, + {file = "frozenlist-1.4.0-cp311-cp311-win32.whl", hash = "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431"}, + {file = "frozenlist-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503"}, + {file = "frozenlist-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672"}, + {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781"}, + {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8"}, + {file = "frozenlist-1.4.0-cp38-cp38-win32.whl", hash = "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc"}, + {file = "frozenlist-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963"}, + {file = "frozenlist-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6"}, + {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a"}, + {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3"}, + {file = "frozenlist-1.4.0-cp39-cp39-win32.whl", hash = "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f"}, + {file = "frozenlist-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167"}, + {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"}, ] [[package]] name = "fsspec" version = "2023.1.0" description = "File-system specification" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -639,7 +680,6 @@ tqdm = ["tqdm"] name = "gcsfs" version = "2023.1.0" description = "Convenient Filesystem interface over GCS" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -664,7 +704,6 @@ gcsfuse = ["fusepy"] name = "gitdb" version = "4.0.10" description = "Git Object Database" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -677,14 +716,13 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.31" +version = "3.1.32" description = "GitPython is a Python library used to interact with Git repositories" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.31-py3-none-any.whl", hash = "sha256:f04893614f6aa713a60cbbe1e6a97403ef633103cdd0ef5eb6efe0deb98dbe8d"}, - {file = "GitPython-3.1.31.tar.gz", hash = "sha256:8ce3bcf69adfdf7c7d503e78fd3b1c492af782d58893b650adb2ac8912ddd573"}, + {file = "GitPython-3.1.32-py3-none-any.whl", hash = "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"}, + {file = "GitPython-3.1.32.tar.gz", hash = "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6"}, ] [package.dependencies] @@ -692,66 +730,63 @@ gitdb = ">=4.0.1,<5" [[package]] name = "google-api-core" -version = "2.11.0" +version = "2.11.1" description = "Google API client core library" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.11.0.tar.gz", hash = "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22"}, - {file = "google_api_core-2.11.0-py3-none-any.whl", hash = "sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e"}, + {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, + {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, ] [package.dependencies] -google-auth = ">=2.14.1,<3.0dev" -googleapis-common-protos = ">=1.56.2,<2.0dev" +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" -requests = ">=2.18.0,<3.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)", "grpcio-status (>=1.49.1,<2.0dev)"] -grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.18.1" +version = "2.22.0" description = "Google Authentication Library" -category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +python-versions = ">=3.6" files = [ - {file = "google-auth-2.18.1.tar.gz", hash = "sha256:d7a3249027e7f464fbbfd7ee8319a08ad09d2eea51578575c4bd360ffa049ccb"}, - {file = "google_auth-2.18.1-py2.py3-none-any.whl", hash = "sha256:55a395cdfd3f3dd3f649131d41f97c17b4ed8a2aac1be3502090c716314e8a37"}, + {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, + {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, ] [package.dependencies] cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" -rsa = {version = ">=3.1.4,<5", markers = "python_version >= \"3.6\""} +rsa = ">=3.1.4,<5" six = ">=1.9.0" urllib3 = "<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "requests (>=2.20.0,<3.0.0dev)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] -requests = ["requests (>=2.20.0,<3.0.0dev)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-auth-oauthlib" version = "1.0.0" description = "Google Authentication Library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -768,18 +803,17 @@ tool = ["click (>=6.0.0)"] [[package]] name = "google-cloud-bigquery" -version = "3.10.0" +version = "3.11.4" description = "Google BigQuery API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.10.0.tar.gz", hash = "sha256:4b02def076e2db8cec66f65fb627d13904a9fc3cf4fee315ede43dcb7038a8df"}, - {file = "google_cloud_bigquery-3.10.0-py2.py3-none-any.whl", hash = "sha256:848a3cbce0ba7d4f1e9551400a7c99aa0eab72290d5a1bbbe69f18a24a10bd3a"}, + {file = "google-cloud-bigquery-3.11.4.tar.gz", hash = "sha256:697df117241a2283bcbb93b21e10badc14e51c9a90800d2a7e1a3e1c7d842974"}, + {file = "google_cloud_bigquery-3.11.4-py2.py3-none-any.whl", hash = "sha256:5fa7897743a0ed949ade25a0942fc9e7557d8fce307c6f8a76d1b604cf27f1b1"}, ] [package.dependencies] -google-api-core = {version = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev", extras = ["grpc"]} +google-api-core = {version = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev", extras = ["grpc"]} google-cloud-core = ">=1.6.0,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" grpcio = [ @@ -804,18 +838,17 @@ tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] [[package]] name = "google-cloud-bigquery-storage" -version = "2.19.1" +version = "2.22.0" description = "Google Cloud Bigquery Storage API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-storage-2.19.1.tar.gz", hash = "sha256:0d9b5f42a703f3210b4b3ad45a18139191f934740fdf3629b1b22fd95adf0bba"}, - {file = "google_cloud_bigquery_storage-2.19.1-py2.py3-none-any.whl", hash = "sha256:e2cb6cb7f0fe027f440903d70c33bf843a2af5714eb3f15b1473335cbd736365"}, + {file = "google-cloud-bigquery-storage-2.22.0.tar.gz", hash = "sha256:f6d8c7b3ab9b574c66977fcee9d336e334ad1a3843a722be19123640e7808ea3"}, + {file = "google_cloud_bigquery_storage-2.22.0-py2.py3-none-any.whl", hash = "sha256:7f11b2ae590a5b3874fb6ddf705a66a070340db238f971cf7b53349eee9ca317"}, ] [package.dependencies] -google-api-core = {version = ">=1.34.0,<2.0.0 || >=2.11.0,<3.0.0dev", extras = ["grpc"]} +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} proto-plus = [ {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, @@ -829,18 +862,17 @@ pyarrow = ["pyarrow (>=0.15.0)"] [[package]] name = "google-cloud-core" -version = "2.3.2" +version = "2.3.3" description = "Google Cloud API client core library" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-core-2.3.2.tar.gz", hash = "sha256:b9529ee7047fd8d4bf4a2182de619154240df17fbe60ead399078c1ae152af9a"}, - {file = "google_cloud_core-2.3.2-py2.py3-none-any.whl", hash = "sha256:8417acf6466be2fa85123441696c4badda48db314c607cf1e5d543fa8bdc22fe"}, + {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, + {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, ] [package.dependencies] -google-api-core = ">=1.31.6,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] @@ -848,18 +880,17 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] [[package]] name = "google-cloud-storage" -version = "2.9.0" +version = "2.10.0" description = "Google Cloud Storage API client library" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.9.0.tar.gz", hash = "sha256:9b6ae7b509fc294bdacb84d0f3ea8e20e2c54a8b4bbe39c5707635fec214eff3"}, - {file = "google_cloud_storage-2.9.0-py2.py3-none-any.whl", hash = "sha256:83a90447f23d5edd045e0037982c270302e3aeb45fc1288d2c2ca713d27bad94"}, + {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, + {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" google-resumable-media = ">=2.3.2" @@ -872,7 +903,6 @@ protobuf = ["protobuf (<5.0.0dev)"] name = "google-crc32c" version = "1.5.0" description = "A python wrapper of the C library 'Google CRC32C'" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -953,7 +983,6 @@ testing = ["pytest"] name = "google-resumable-media" version = "2.5.0" description = "Utilities for Google Media Downloads and Resumable Uploads" -category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -970,102 +999,98 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.59.0" +version = "1.60.0" description = "Common protobufs used in Google APIs" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.59.0.tar.gz", hash = "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44"}, - {file = "googleapis_common_protos-1.59.0-py2.py3-none-any.whl", hash = "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"}, + {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, + {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, ] [package.dependencies] -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "grpcio" -version = "1.54.2" +version = "1.57.0" description = "HTTP/2-based RPC framework" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.54.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:40e1cbf69d6741b40f750f3cccc64326f927ac6145a9914d33879e586002350c"}, - {file = "grpcio-1.54.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2288d76e4d4aa7ef3fe7a73c1c470b66ea68e7969930e746a8cd8eca6ef2a2ea"}, - {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:c0e3155fc5335ec7b3b70f15230234e529ca3607b20a562b6c75fb1b1218874c"}, - {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bf88004fe086c786dc56ef8dd6cb49c026833fdd6f42cb853008bce3f907148"}, - {file = "grpcio-1.54.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2be88c081e33f20630ac3343d8ad9f1125f32987968e9c8c75c051c9800896e8"}, - {file = "grpcio-1.54.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:33d40954199bddbb6a78f8f6f2b2082660f381cd2583ec860a6c2fa7c8400c08"}, - {file = "grpcio-1.54.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b52d00d1793d290c81ad6a27058f5224a7d5f527867e5b580742e1bd211afeee"}, - {file = "grpcio-1.54.2-cp310-cp310-win32.whl", hash = "sha256:881d058c5ccbea7cc2c92085a11947b572498a27ef37d3eef4887f499054dca8"}, - {file = "grpcio-1.54.2-cp310-cp310-win_amd64.whl", hash = "sha256:0212e2f7fdf7592e4b9d365087da30cb4d71e16a6f213120c89b4f8fb35a3ab3"}, - {file = "grpcio-1.54.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:1e623e0cf99a0ac114f091b3083a1848dbc64b0b99e181473b5a4a68d4f6f821"}, - {file = "grpcio-1.54.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:66233ccd2a9371158d96e05d082043d47dadb18cbb294dc5accfdafc2e6b02a7"}, - {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:4cb283f630624ebb16c834e5ac3d7880831b07cbe76cb08ab7a271eeaeb8943e"}, - {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a1e601ee31ef30a9e2c601d0867e236ac54c922d32ed9f727b70dd5d82600d5"}, - {file = "grpcio-1.54.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8da84bbc61a4e92af54dc96344f328e5822d574f767e9b08e1602bb5ddc254a"}, - {file = "grpcio-1.54.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:5008964885e8d23313c8e5ea0d44433be9bfd7e24482574e8cc43c02c02fc796"}, - {file = "grpcio-1.54.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a2f5a1f1080ccdc7cbaf1171b2cf384d852496fe81ddedeb882d42b85727f610"}, - {file = "grpcio-1.54.2-cp311-cp311-win32.whl", hash = "sha256:b74ae837368cfffeb3f6b498688a123e6b960951be4dec0e869de77e7fa0439e"}, - {file = "grpcio-1.54.2-cp311-cp311-win_amd64.whl", hash = "sha256:8cdbcbd687e576d48f7886157c95052825ca9948c0ed2afdc0134305067be88b"}, - {file = "grpcio-1.54.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:782f4f8662a2157c4190d0f99eaaebc602899e84fb1e562a944e5025929e351c"}, - {file = "grpcio-1.54.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:714242ad0afa63a2e6dabd522ae22e1d76e07060b5af2ddda5474ba4f14c2c94"}, - {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:f900ed4ad7a0f1f05d35f955e0943944d5a75f607a836958c6b8ab2a81730ef2"}, - {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96a41817d2c763b1d0b32675abeb9179aa2371c72aefdf74b2d2b99a1b92417b"}, - {file = "grpcio-1.54.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70fcac7b94f4c904152809a050164650ac81c08e62c27aa9f156ac518029ebbe"}, - {file = "grpcio-1.54.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fd6c6c29717724acf9fc1847c4515d57e4dc12762452457b9cb37461f30a81bb"}, - {file = "grpcio-1.54.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c2392f5b5d84b71d853918687d806c1aa4308109e5ca158a16e16a6be71041eb"}, - {file = "grpcio-1.54.2-cp37-cp37m-win_amd64.whl", hash = "sha256:51630c92591d6d3fe488a7c706bd30a61594d144bac7dee20c8e1ce78294f474"}, - {file = "grpcio-1.54.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:b04202453941a63b36876a7172b45366dc0cde10d5fd7855c0f4a4e673c0357a"}, - {file = "grpcio-1.54.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:89dde0ac72a858a44a2feb8e43dc68c0c66f7857a23f806e81e1b7cc7044c9cf"}, - {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:09d4bfd84686cd36fd11fd45a0732c7628308d094b14d28ea74a81db0bce2ed3"}, - {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7fc2b4edb938c8faa4b3c3ea90ca0dd89b7565a049e8e4e11b77e60e4ed2cc05"}, - {file = "grpcio-1.54.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61f7203e2767800edee7a1e1040aaaf124a35ce0c7fe0883965c6b762defe598"}, - {file = "grpcio-1.54.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e416c8baf925b5a1aff31f7f5aecc0060b25d50cce3a5a7255dc5cf2f1d4e5eb"}, - {file = "grpcio-1.54.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dc80c9c6b608bf98066a038e0172013a49cfa9a08d53335aefefda2c64fc68f4"}, - {file = "grpcio-1.54.2-cp38-cp38-win32.whl", hash = "sha256:8d6192c37a30a115f4663592861f50e130caed33efc4eec24d92ec881c92d771"}, - {file = "grpcio-1.54.2-cp38-cp38-win_amd64.whl", hash = "sha256:46a057329938b08e5f0e12ea3d7aed3ecb20a0c34c4a324ef34e00cecdb88a12"}, - {file = "grpcio-1.54.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:2296356b5c9605b73ed6a52660b538787094dae13786ba53080595d52df13a98"}, - {file = "grpcio-1.54.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:c72956972e4b508dd39fdc7646637a791a9665b478e768ffa5f4fe42123d5de1"}, - {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:9bdbb7624d65dc0ed2ed8e954e79ab1724526f09b1efa88dcd9a1815bf28be5f"}, - {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c44e1a765b31e175c391f22e8fc73b2a2ece0e5e6ff042743d8109b5d2eff9f"}, - {file = "grpcio-1.54.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cc928cfe6c360c1df636cf7991ab96f059666ac7b40b75a769410cc6217df9c"}, - {file = "grpcio-1.54.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a08920fa1a97d4b8ee5db2f31195de4a9def1a91bc003544eb3c9e6b8977960a"}, - {file = "grpcio-1.54.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4864f99aac207e3e45c5e26c6cbb0ad82917869abc2f156283be86c05286485c"}, - {file = "grpcio-1.54.2-cp39-cp39-win32.whl", hash = "sha256:b38b3de8cff5bc70f8f9c615f51b48eff7313fc9aca354f09f81b73036e7ddfa"}, - {file = "grpcio-1.54.2-cp39-cp39-win_amd64.whl", hash = "sha256:be48496b0e00460717225e7680de57c38be1d8629dc09dadcd1b3389d70d942b"}, - {file = "grpcio-1.54.2.tar.gz", hash = "sha256:50a9f075eeda5097aa9a182bb3877fe1272875e45370368ac0ee16ab9e22d019"}, + {file = "grpcio-1.57.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:092fa155b945015754bdf988be47793c377b52b88d546e45c6a9f9579ac7f7b6"}, + {file = "grpcio-1.57.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f7349786da979a94690cc5c2b804cab4e8774a3cf59be40d037c4342c906649"}, + {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:82640e57fb86ea1d71ea9ab54f7e942502cf98a429a200b2e743d8672171734f"}, + {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40b72effd4c789de94ce1be2b5f88d7b9b5f7379fe9645f198854112a6567d9a"}, + {file = "grpcio-1.57.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f708a6a17868ad8bf586598bee69abded4996b18adf26fd2d91191383b79019"}, + {file = "grpcio-1.57.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:60fe15288a0a65d5c1cb5b4a62b1850d07336e3ba728257a810317be14f0c527"}, + {file = "grpcio-1.57.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6907b1cf8bb29b058081d2aad677b15757a44ef2d4d8d9130271d2ad5e33efca"}, + {file = "grpcio-1.57.0-cp310-cp310-win32.whl", hash = "sha256:57b183e8b252825c4dd29114d6c13559be95387aafc10a7be645462a0fc98bbb"}, + {file = "grpcio-1.57.0-cp310-cp310-win_amd64.whl", hash = "sha256:7b400807fa749a9eb286e2cd893e501b110b4d356a218426cb9c825a0474ca56"}, + {file = "grpcio-1.57.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c6ebecfb7a31385393203eb04ed8b6a08f5002f53df3d59e5e795edb80999652"}, + {file = "grpcio-1.57.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:00258cbe3f5188629828363ae8ff78477ce976a6f63fb2bb5e90088396faa82e"}, + {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:23e7d8849a0e58b806253fd206ac105b328171e01b8f18c7d5922274958cc87e"}, + {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5371bcd861e679d63b8274f73ac281751d34bd54eccdbfcd6aa00e692a82cd7b"}, + {file = "grpcio-1.57.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aed90d93b731929e742967e236f842a4a2174dc5db077c8f9ad2c5996f89f63e"}, + {file = "grpcio-1.57.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fe752639919aad9ffb0dee0d87f29a6467d1ef764f13c4644d212a9a853a078d"}, + {file = "grpcio-1.57.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fada6b07ec4f0befe05218181f4b85176f11d531911b64c715d1875c4736d73a"}, + {file = "grpcio-1.57.0-cp311-cp311-win32.whl", hash = "sha256:bb396952cfa7ad2f01061fbc7dc1ad91dd9d69243bcb8110cf4e36924785a0fe"}, + {file = "grpcio-1.57.0-cp311-cp311-win_amd64.whl", hash = "sha256:e503cb45ed12b924b5b988ba9576dc9949b2f5283b8e33b21dcb6be74a7c58d0"}, + {file = "grpcio-1.57.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:fd173b4cf02b20f60860dc2ffe30115c18972d7d6d2d69df97ac38dee03be5bf"}, + {file = "grpcio-1.57.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:d7f8df114d6b4cf5a916b98389aeaf1e3132035420a88beea4e3d977e5f267a5"}, + {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:76c44efa4ede1f42a9d5b2fed1fe9377e73a109bef8675fb0728eb80b0b8e8f2"}, + {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4faea2cfdf762a664ab90589b66f416274887641ae17817de510b8178356bf73"}, + {file = "grpcio-1.57.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c60b83c43faeb6d0a9831f0351d7787a0753f5087cc6fa218d78fdf38e5acef0"}, + {file = "grpcio-1.57.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b363bbb5253e5f9c23d8a0a034dfdf1b7c9e7f12e602fc788c435171e96daccc"}, + {file = "grpcio-1.57.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:f1fb0fd4a1e9b11ac21c30c169d169ef434c6e9344ee0ab27cfa6f605f6387b2"}, + {file = "grpcio-1.57.0-cp37-cp37m-win_amd64.whl", hash = "sha256:34950353539e7d93f61c6796a007c705d663f3be41166358e3d88c45760c7d98"}, + {file = "grpcio-1.57.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:871f9999e0211f9551f368612460442a5436d9444606184652117d6a688c9f51"}, + {file = "grpcio-1.57.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:a8a8e560e8dbbdf29288872e91efd22af71e88b0e5736b0daf7773c1fecd99f0"}, + {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2313b124e475aa9017a9844bdc5eafb2d5abdda9d456af16fc4535408c7d6da6"}, + {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b4098b6b638d9e0ca839a81656a2fd4bc26c9486ea707e8b1437d6f9d61c3941"}, + {file = "grpcio-1.57.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e5b58e32ae14658085c16986d11e99abd002ddbf51c8daae8a0671fffb3467f"}, + {file = "grpcio-1.57.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0f80bf37f09e1caba6a8063e56e2b87fa335add314cf2b78ebf7cb45aa7e3d06"}, + {file = "grpcio-1.57.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5b7a4ce8f862fe32b2a10b57752cf3169f5fe2915acfe7e6a1e155db3da99e79"}, + {file = "grpcio-1.57.0-cp38-cp38-win32.whl", hash = "sha256:9338bacf172e942e62e5889b6364e56657fbf8ac68062e8b25c48843e7b202bb"}, + {file = "grpcio-1.57.0-cp38-cp38-win_amd64.whl", hash = "sha256:e1cb52fa2d67d7f7fab310b600f22ce1ff04d562d46e9e0ac3e3403c2bb4cc16"}, + {file = "grpcio-1.57.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fee387d2fab144e8a34e0e9c5ca0f45c9376b99de45628265cfa9886b1dbe62b"}, + {file = "grpcio-1.57.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:b53333627283e7241fcc217323f225c37783b5f0472316edcaa4479a213abfa6"}, + {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:f19ac6ac0a256cf77d3cc926ef0b4e64a9725cc612f97228cd5dc4bd9dbab03b"}, + {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fdf04e402f12e1de8074458549337febb3b45f21076cc02ef4ff786aff687e"}, + {file = "grpcio-1.57.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5613a2fecc82f95d6c51d15b9a72705553aa0d7c932fad7aed7afb51dc982ee5"}, + {file = "grpcio-1.57.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b670c2faa92124b7397b42303e4d8eb64a4cd0b7a77e35a9e865a55d61c57ef9"}, + {file = "grpcio-1.57.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7a635589201b18510ff988161b7b573f50c6a48fae9cb567657920ca82022b37"}, + {file = "grpcio-1.57.0-cp39-cp39-win32.whl", hash = "sha256:d78d8b86fcdfa1e4c21f8896614b6cc7ee01a2a758ec0c4382d662f2a62cf766"}, + {file = "grpcio-1.57.0-cp39-cp39-win_amd64.whl", hash = "sha256:20ec6fc4ad47d1b6e12deec5045ec3cd5402d9a1597f738263e98f490fe07056"}, + {file = "grpcio-1.57.0.tar.gz", hash = "sha256:4b089f7ad1eb00a104078bab8015b0ed0ebcb3b589e527ab009c53893fd4e613"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.54.2)"] +protobuf = ["grpcio-tools (>=1.57.0)"] [[package]] name = "grpcio-status" -version = "1.54.2" +version = "1.57.0" description = "Status proto mapping for gRPC" -category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.54.2.tar.gz", hash = "sha256:3255cbec5b7c706caa3d4dd584606c080e6415e15631bb2f6215e2b70055836d"}, - {file = "grpcio_status-1.54.2-py3-none-any.whl", hash = "sha256:2a7cb4838225f1b53bd0448a3008c5b5837941e1f3a0b13fa38768f08a7b68c2"}, + {file = "grpcio-status-1.57.0.tar.gz", hash = "sha256:b098da99df1eebe58337f8f78e50df990273ccacc1226fddeb47c590e3df9e02"}, + {file = "grpcio_status-1.57.0-py3-none-any.whl", hash = "sha256:15d6af055914ebbc4ed17e55ebfb8e6bb17a45a57fea32e6af19978fb7844690"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.54.2" +grpcio = ">=1.57.0" protobuf = ">=4.21.6" [[package]] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1077,7 +1102,6 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1085,41 +1109,15 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "markdown-it-py" -version = "2.2.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, - {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["attrs", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - [[package]] name = "mbstrdecoder" -version = "1.1.2" -description = "multi-byte character string decoder" -category = "main" +version = "1.1.3" +description = "mbstrdecoder is a Python library for multi-byte character string decoder" optional = false python-versions = ">=3.7" files = [ - {file = "mbstrdecoder-1.1.2-py3-none-any.whl", hash = "sha256:27dd749eb46484a3d032aa1cea80743cd094922245764579e0153f70813845de"}, - {file = "mbstrdecoder-1.1.2.tar.gz", hash = "sha256:c3a258e5e00192281eb774c00637f68d2d460854cea0c5c820aa241732aa0b51"}, + {file = "mbstrdecoder-1.1.3-py3-none-any.whl", hash = "sha256:d66c1ed3f2dc4e7c5d87cd44a75be10bc5af4250f95b38bbaedd7851308ce938"}, + {file = "mbstrdecoder-1.1.3.tar.gz", hash = "sha256:dcfd2c759322eb44fe193a9e0b1b86c5b87f3ec5ea8e1bb43b3e9ae423f1e8fe"}, ] [package.dependencies] @@ -1128,23 +1126,10 @@ chardet = ">=3.0.4,<6" [package.extras] test = ["Faker (>=1.0.2)", "pytest (>=6.0.1)", "pytest-md-report (>=0.1)"] -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -category = "main" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - [[package]] name = "multidict" version = "6.0.4" description = "multidict implementation" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1226,47 +1211,42 @@ files = [ [[package]] name = "numpy" -version = "1.24.3" +version = "1.25.2" description = "Fundamental package for array computing in Python" -category = "main" optional = false -python-versions = ">=3.8" -files = [ - {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, - {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, - {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, - {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, - {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, - {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, - {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, - {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, - {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, - {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, +python-versions = ">=3.9" +files = [ + {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, + {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, + {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, + {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, + {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, + {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, + {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, + {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, + {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, + {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, + {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, + {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, ] [[package]] name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1283,7 +1263,6 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1295,7 +1274,6 @@ files = [ name = "pandas" version = "1.5.3" description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1330,7 +1308,6 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] @@ -1344,7 +1321,6 @@ test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] name = "pandas-gbq" version = "0.19.2" description = "Google BigQuery connector for pandas" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1372,7 +1348,6 @@ tqdm = ["tqdm (>=4.23.0)"] name = "pathvalidate" version = "2.5.2" description = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1385,14 +1360,13 @@ test = ["allpairspy", "click", "faker", "pytest (>=6.0.1)", "pytest-discord (>=0 [[package]] name = "pluggy" -version = "1.0.0" +version = "1.2.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, ] [package.extras] @@ -1401,14 +1375,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "proto-plus" -version = "1.22.2" +version = "1.22.3" description = "Beautiful, Pythonic protocol buffers." -category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "proto-plus-1.22.2.tar.gz", hash = "sha256:0e8cda3d5a634d9895b75c573c9352c16486cb75deb0e078b5fda34db4243165"}, - {file = "proto_plus-1.22.2-py3-none-any.whl", hash = "sha256:de34e52d6c9c6fcd704192f09767cb561bb4ee64e70eede20b0834d841f0be4d"}, + {file = "proto-plus-1.22.3.tar.gz", hash = "sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b"}, + {file = "proto_plus-1.22.3-py3-none-any.whl", hash = "sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df"}, ] [package.dependencies] @@ -1419,32 +1392,30 @@ testing = ["google-api-core[grpc] (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.23.1" +version = "4.24.0" description = "" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.23.1-cp310-abi3-win32.whl", hash = "sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be"}, - {file = "protobuf-4.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee"}, - {file = "protobuf-4.23.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a"}, - {file = "protobuf-4.23.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436"}, - {file = "protobuf-4.23.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0"}, - {file = "protobuf-4.23.1-cp37-cp37m-win32.whl", hash = "sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39"}, - {file = "protobuf-4.23.1-cp37-cp37m-win_amd64.whl", hash = "sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa"}, - {file = "protobuf-4.23.1-cp38-cp38-win32.whl", hash = "sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1"}, - {file = "protobuf-4.23.1-cp38-cp38-win_amd64.whl", hash = "sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38"}, - {file = "protobuf-4.23.1-cp39-cp39-win32.whl", hash = "sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61"}, - {file = "protobuf-4.23.1-cp39-cp39-win_amd64.whl", hash = "sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0"}, - {file = "protobuf-4.23.1-py3-none-any.whl", hash = "sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477"}, - {file = "protobuf-4.23.1.tar.gz", hash = "sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e"}, + {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, + {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, + {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, + {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, + {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, + {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, + {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, + {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, + {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, + {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, + {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, + {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, ] [[package]] name = "py" version = "1.11.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1454,37 +1425,36 @@ files = [ [[package]] name = "pyarrow" -version = "12.0.0" +version = "12.0.1" description = "Python library for Apache Arrow" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pyarrow-12.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:3b97649c8a9a09e1d8dc76513054f1331bd9ece78ee39365e6bf6bc7503c1e94"}, - {file = "pyarrow-12.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc4ea634dacb03936f50fcf59574a8e727f90c17c24527e488d8ceb52ae284de"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d568acfca3faa565d663e53ee34173be8e23a95f78f2abfdad198010ec8f745"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b50bb9a82dca38a002d7cbd802a16b1af0f8c50ed2ec94a319f5f2afc047ee9"}, - {file = "pyarrow-12.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3d1733b1ea086b3c101427d0e57e2be3eb964686e83c2363862a887bb5c41fa8"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:a7cd32fe77f967fe08228bc100433273020e58dd6caced12627bcc0a7675a513"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:92fb031e6777847f5c9b01eaa5aa0c9033e853ee80117dce895f116d8b0c3ca3"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:280289ebfd4ac3570f6b776515baa01e4dcbf17122c401e4b7170a27c4be63fd"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:272f147d4f8387bec95f17bb58dcfc7bc7278bb93e01cb7b08a0e93a8921e18e"}, - {file = "pyarrow-12.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:0846ace49998825eda4722f8d7f83fa05601c832549c9087ea49d6d5397d8cec"}, - {file = "pyarrow-12.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:993287136369aca60005ee7d64130f9466489c4f7425f5c284315b0a5401ccd9"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7b6a765ee4f88efd7d8348d9a1f804487d60799d0428b6ddf3344eaef37282"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1c4fce253d5bdc8d62f11cfa3da5b0b34b562c04ce84abb8bd7447e63c2b327"}, - {file = "pyarrow-12.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e6be4d85707fc8e7a221c8ab86a40449ce62559ce25c94321df7c8500245888f"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ea830d9f66bfb82d30b5794642f83dd0e4a718846462d22328981e9eb149cba8"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7b5b9f60d9ef756db59bec8d90e4576b7df57861e6a3d6a8bf99538f68ca15b3"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99e559d27db36ad3a33868a475f03e3129430fc065accc839ef4daa12c6dab6"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b0810864a593b89877120972d1f7af1d1c9389876dbed92b962ed81492d3ffc"}, - {file = "pyarrow-12.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:23a77d97f4d101ddfe81b9c2ee03a177f0e590a7e68af15eafa06e8f3cf05976"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2cc63e746221cddb9001f7281dee95fd658085dd5b717b076950e1ccc607059c"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8c26912607e26c2991826bbaf3cf2b9c8c3e17566598c193b492f058b40d3a4"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d8b90efc290e99a81d06015f3a46601c259ecc81ffb6d8ce288c91bd1b868c9"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2466be046b81863be24db370dffd30a2e7894b4f9823fb60ef0a733c31ac6256"}, - {file = "pyarrow-12.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:0e36425b1c1cbf5447718b3f1751bf86c58f2b3ad299f996cd9b1aa040967656"}, - {file = "pyarrow-12.0.0.tar.gz", hash = "sha256:19c812d303610ab5d664b7b1de4051ae23565f9f94d04cbea9e50569746ae1ee"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, + {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, + {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, + {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, + {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, + {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, + {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, + {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, + {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, + {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, + {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, + {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, + {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, + {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, + {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, + {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, + {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, ] [package.dependencies] @@ -1494,7 +1464,6 @@ numpy = ">=1.16.6" name = "pyasn1" version = "0.5.0" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1506,7 +1475,6 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1521,7 +1489,6 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycparser" version = "2.21" description = "C parser in Python" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1533,7 +1500,6 @@ files = [ name = "pydantic" version = "1.9.2" description = "Data validation and settings management using python type hints" -category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -1583,29 +1549,30 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pydash" -version = "6.0.2" +version = "7.0.6" description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library." -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pydash-6.0.2-py3-none-any.whl", hash = "sha256:6d3ce5cbbc8ca3533c12782ac201c2ec756d1e1703ec3efc88f2b95d1ed2bb31"}, - {file = "pydash-6.0.2.tar.gz", hash = "sha256:35caa588e01d293713655e0870544d25128cd414c5e19477a0d63adc2b2ca03e"}, + {file = "pydash-7.0.6-py3-none-any.whl", hash = "sha256:10e506935953fde4b0d6fe21a88e17783cd1479256ae96f285b5f89063b4efd6"}, + {file = "pydash-7.0.6.tar.gz", hash = "sha256:7d9df7e9f36f2bbb08316b609480e7c6468185473a21bdd8e65dda7915565a26"}, ] +[package.dependencies] +typing-extensions = ">=3.10,<4.6.0 || >4.6.0" + [package.extras] -dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "importlib-metadata (<5)", "invoke", "isort", "pylint", "pytest", "pytest-cov", "sphinx-rtd-theme", "tox", "twine", "wheel"] +dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] [[package]] name = "pydata-google-auth" -version = "1.8.0" +version = "1.8.2" description = "PyData helpers for authenticating to Google APIs" -category = "main" optional = false python-versions = "*" files = [ - {file = "pydata-google-auth-1.8.0.tar.gz", hash = "sha256:8790aca74cfa6c18da01a53b4b8cb2467ef519d1e891f855a9af6885dfa8c626"}, - {file = "pydata_google_auth-1.8.0-py2.py3-none-any.whl", hash = "sha256:8a093294bbae6de980a83794345b634b163f7374fa9d84c7b0059ea4b2001700"}, + {file = "pydata-google-auth-1.8.2.tar.gz", hash = "sha256:547b6c0fbea657dcecd50887c5db8640ebec062a59a2b88e8ff8e53a04818303"}, + {file = "pydata_google_auth-1.8.2-py2.py3-none-any.whl", hash = "sha256:a9dce59af4a170ea60c4b2ebbc83ee1f74d34255a4f97b2469ae9a4a0dc98e99"}, ] [package.dependencies] @@ -1617,7 +1584,6 @@ setuptools = "*" name = "pygithub" version = "1.58.2" description = "Use the full Github API v3" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1633,14 +1599,13 @@ requests = ">=2.14.0" [[package]] name = "pygments" -version = "2.15.1" +version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, ] [package.extras] @@ -1648,62 +1613,61 @@ plugins = ["importlib-metadata"] [[package]] name = "pyinstrument" -version = "4.4.0" +version = "4.5.1" description = "Call stack profiler for Python. Shows you why your code is slow!" -category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstrument-4.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8874f8f58cfcb1ff134dc8e4a2b31ab9175adb271a4423596ed7ac8183592cf8"}, - {file = "pyinstrument-4.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e5f4d6e1c395f259f67a923a9c54dc3eaccd5f02540598da4f865c4bb3545762"}, - {file = "pyinstrument-4.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d70fed48ddd0078e287fb580daaeede4d8703a9edc8bf4f703308a77920bac37"}, - {file = "pyinstrument-4.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9fda1bd596e81ecd2b6a976eb9b930a757a5dd04071583d0141d059e34eed83f"}, - {file = "pyinstrument-4.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f57b61d39d3b1a4d773da16baa8456aa66102d6016ce1f39817051550cbe47e"}, - {file = "pyinstrument-4.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5a9aead0ca5579473f66fed4c449c693feee464802b5ba9b98772e64e02c575c"}, - {file = "pyinstrument-4.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:375a340c3fbebd922a35b0834de1c82d1b4fea681df49f99729439a6cb5e6ad4"}, - {file = "pyinstrument-4.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cbaf3bcda5ad9af4c9a7bf4f1b8f15bb32c4cadf554d0a2c723892c898021b"}, - {file = "pyinstrument-4.4.0-cp310-cp310-win32.whl", hash = "sha256:97cbeb5f5a048dc6eb047495f73db90c9e2ec97606e65298c7ea2c61fa52de38"}, - {file = "pyinstrument-4.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:73edbce7fda1b3d8cab0b6c39c43b012167d783c072f40928600c3357d1a5dc5"}, - {file = "pyinstrument-4.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7db8cb55182883be48520eb915bd1769f176a4813ce0cc38243aa2d1182e7ce7"}, - {file = "pyinstrument-4.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c614e2c241fb558a25973019ff43ce027ba4958bcb87383f0b0789af9c4d03b"}, - {file = "pyinstrument-4.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c012422c851f0457c3cb82d8b1259d96fa0dcddc0f1e8bf4d97f0b2efe54485"}, - {file = "pyinstrument-4.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4f5ad100710dda68f9f345961780bf4f0cbb9fd3e46295d099bb9ad65b179ea"}, - {file = "pyinstrument-4.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a4a053cd67102c6fcc313366ea6be97cfce7eae2b9e57e62c9be8adbbdebc17"}, - {file = "pyinstrument-4.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1d2a1e53615c8ef210286e4d2d93be0d3e8296995b090df29a0b7ddeae5d874b"}, - {file = "pyinstrument-4.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2a6609ef74ad8ba292a11fbd975660bc86466c7eaab1ff11360d24e0300800b"}, - {file = "pyinstrument-4.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3643084ee8ad22d9ea2adb13d65d4b6e18810113e4176b19d026a011957f8c7c"}, - {file = "pyinstrument-4.4.0-cp311-cp311-win32.whl", hash = "sha256:fcd717910a8ab6deca353aded890403bbaea14a6dd99a87c3367f24721d2d6aa"}, - {file = "pyinstrument-4.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:be9ac54a4dd07d969d5941e4dcba67d5aef5f6826f43b9ddda65553816f6abca"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:39584c0fec147e3bbfa7b28454332f9801af5f93331f4143f24a4b0f9e3cb470"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5491a5cb3ae5e88436d48b4b3de8328286e843e7307116dc2cca397c9c2ffe21"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d66fcc94f0ebaab6bcbbdfa2482f833dd634352a20295616ea45286e990f7446"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b72bde0b1a03d1b2dc9b9d79546f551df6f67673cca816614e98ea0aebd3bc50"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0bbd169b92147ec5d67ed160c300dda504059cfd81e953ed5b059e8ef92bb482"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e5233022ba511ef7ecfef2e07d162c0817048c995f0940f9aa2f6a1936afcb9c"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5e750fc3afb9acc288ad84b183a5ccd863e9185c435b445fcc62e0c133af9b7f"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-win32.whl", hash = "sha256:2d131b98f116fb895d759dfb8c1078e0e9fa8987a9f44f566d29221545f75bd4"}, - {file = "pyinstrument-4.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:de83152bafc9eed4e5469e340b6002be825151f0654c32bbb9a3a7e31708d227"}, - {file = "pyinstrument-4.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a7c774c4b8df21664b082d3e72fa8cbc0631fe9bb222bb9d285ccfe9cd9b4909"}, - {file = "pyinstrument-4.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7526f0b1dab721ddc19920fa1f4eeaa5bcb658a4d18ac9c50868e84f911f794b"}, - {file = "pyinstrument-4.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59f5f479db277b3dbeb0c6843a7e9a38ee8b7c23d75b9ef764d96cb522d96212"}, - {file = "pyinstrument-4.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffeeaa0d036a8bef31da6fc13c4ea097160f913d86319897314113bb9271af4c"}, - {file = "pyinstrument-4.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfc4e2fd670a570ea847f6897283d10d4b9606170e491f01b75488ed1aa37a81"}, - {file = "pyinstrument-4.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ffd9a9fa73fd83a40252430c6ebf8dfff7c668cc68eab4a92562b8b27c302598"}, - {file = "pyinstrument-4.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:09167ece8802bc03a63e97536dcefd9c1a340dae686f40914cf995099bc0d0af"}, - {file = "pyinstrument-4.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9e7d1cc3affef4a7e4695bb87c6cfcd577e2dac508624a91481f24217ef78c57"}, - {file = "pyinstrument-4.4.0-cp38-cp38-win32.whl", hash = "sha256:b50cf50513a5318738c3c7147f02596cda4891089acf2f627bb65954fc5bcbfd"}, - {file = "pyinstrument-4.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:dd9625cf136eb6684d9ca555a5088f21a7ac6c6cb2ece3ae45d09772906ceba8"}, - {file = "pyinstrument-4.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a8afee175335005d2964848b77579bfc18f011ea74b59b79ab6d5b35433bf3e3"}, - {file = "pyinstrument-4.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebc63b70845e3a44b673f7dcdc78ac2c475684db41b0402eea370f194da2a287"}, - {file = "pyinstrument-4.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb89033e41e74dc2ac4fd882269e91ddf677588efa665d2be8b718e96ea4cec6"}, - {file = "pyinstrument-4.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b2bcd803d273c8addf01eaf75a42ae0a2a9196a58fb0ebb8d29be75abb88701"}, - {file = "pyinstrument-4.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50727b686a0961a11eba2fe6205d0899f3479c983bcf34abb114d6da70bc1b93"}, - {file = "pyinstrument-4.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4f07941bb5dd5cd730fc84eef6497ef9f0807c686e68d0c6b1f464589646a3b7"}, - {file = "pyinstrument-4.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30f5ce299c3219559870117c5b0825f33243808be375be9c3525572ba050c2db"}, - {file = "pyinstrument-4.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a13c75b24bf8eed5a4356ffa8a419cc534284a529f2b314f3e10275a820420f"}, - {file = "pyinstrument-4.4.0-cp39-cp39-win32.whl", hash = "sha256:e5583b0d23f87631af06bb9f3c184190c889c194b02553eed132de966324bdf9"}, - {file = "pyinstrument-4.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:a19784a898133b7e0ffe4489155bacd2d07ec48ea059f9bf50033dc2b814c273"}, - {file = "pyinstrument-4.4.0.tar.gz", hash = "sha256:be34a2e8118c14a616a64538e02430d9099d5d67d8a370f2888e4ac71e52bbb7"}, + {file = "pyinstrument-4.5.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f334250b158010d1e2c70d9d10b880f848e03a917079b366b1e2d8890348d41"}, + {file = "pyinstrument-4.5.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:55537cd763aee8bce65a201d5ec1aef74677d9ff3ab3391316604ca68740d92a"}, + {file = "pyinstrument-4.5.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d7933bd83e913e21c4031d5c1aeeb2483147e4037363f43475df9ad962c748"}, + {file = "pyinstrument-4.5.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0d8f6b6df7ce338af35b213cd89b685b2a7c15569f482476c4e0942700b3e71"}, + {file = "pyinstrument-4.5.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98101d064b7af008189dd6f0bdd01f9be39bc6a4630505dfb13ff6ef51a0c67c"}, + {file = "pyinstrument-4.5.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:46f1607e29f93da16d38be41ad2062a56731ff4efa24e561ac848719e8b8ca41"}, + {file = "pyinstrument-4.5.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e287ebc1a8b00d3a767829c03f210df0824ab2e0f6340e8f63bab6fcef1b3546"}, + {file = "pyinstrument-4.5.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d15613b8d5d509c29001f2edfadd73d418c2814262433fd1225c4f7893e4010a"}, + {file = "pyinstrument-4.5.1-cp310-cp310-win32.whl", hash = "sha256:04c67f08bac41173bc6b44396c60bf1a1879864d0684a7717b1bb8be27793bd9"}, + {file = "pyinstrument-4.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:dc07267447935d28ee914f955613b04d621e5bb44995f793508d6f0eb3ec2818"}, + {file = "pyinstrument-4.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8285cfb25b9ee72766bdac8db8c276755115a6e729cda4571005d1ba58c99dda"}, + {file = "pyinstrument-4.5.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b58239f4a0fe64f688260be0e5b4a1d19a23b890b284cf6c1c8bd0ead4616f41"}, + {file = "pyinstrument-4.5.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4039210a80015ae0ad2016a3b3311b068f5b334d5f5ce3c54d473f8624db0d35"}, + {file = "pyinstrument-4.5.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b28a4c5926036155062c83e15ca93437dbe2d41dd5feeac96f72d4d16b3431c"}, + {file = "pyinstrument-4.5.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89d2c2a9de60712abd2228033e4ac63cdee86783af5288f2d7f8efc365e33425"}, + {file = "pyinstrument-4.5.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bf0fdb17cb245c53826c77e2b95095a8fb5053e49ae8ef18aecbbd184028f9e7"}, + {file = "pyinstrument-4.5.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:65ac43f8a1b74a331b5a4f60985531654a8d71a7698e6be5ac7e8493e7a37f37"}, + {file = "pyinstrument-4.5.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:61632d287f70d850a517533b9e1bf8da41527ffc4d781d4b65106f64ee33cb98"}, + {file = "pyinstrument-4.5.1-cp311-cp311-win32.whl", hash = "sha256:22ae739152ed2366c654f80aa073579f9d5a93caffa74dcb839a62640ffe429f"}, + {file = "pyinstrument-4.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:c72a33168485172a7c2dbd6c4aa3262c8d2a6154bc0792403d8e0689c6ff5304"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8c3dabcb70b705d1342f52f0c3a00647c8a244d1e6ffe46459c05d4533ffabfc"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17d469572d48ee0b78d4ff7ed3972ff40abc70c7dab4777897c843cb03a6ab7b"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66416fa4b3413bc60e6b499e60e8d009384c85cd03535f82337dce55801c43f"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c888fca16c3ae04a6d7b5a29ee0c12f9fa23792fab695117160c48c3113428f"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:861fe8c41ac7e54a57ed6ef63268c2843fbc695012427a3d19b2eb1307d9bc61"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:0bf91cd5d6c80ff25fd1a136545a5cf752522190b6e6f3806559c352f18d0e73"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b16afb5e67d4d901ef702160e85e04001183b7cdea7e38c8dfb37e491986ccff"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-win32.whl", hash = "sha256:f12312341c505e7441e5503b7c77974cff4156d072f0e7f9f822a6b5fdafbc20"}, + {file = "pyinstrument-4.5.1-cp37-cp37m-win_amd64.whl", hash = "sha256:06d96b442a1ae7c267aa34450b028d80559c4f968b10e4d3ce631b0a6ccea6ef"}, + {file = "pyinstrument-4.5.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c6234094ff0ea7d51e7d4699f192019359bf12d5bbe9e1c9c5d1983562162d58"}, + {file = "pyinstrument-4.5.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f025522edc35831af34bcdbe300b272b432d2afd9811eb780e326116096cbff5"}, + {file = "pyinstrument-4.5.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0a091c575367af427e80829ec414f69a8398acdd68ddfaeb335598071329b44"}, + {file = "pyinstrument-4.5.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ec169cd288f230cbc6a1773384f20481b0a14d2d7cceecf1fb65e56835eaa9a"}, + {file = "pyinstrument-4.5.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:004745e83c79d0db7ea8787aba476f13d8bb6d00d75b00d8dbd933a9c7ee1685"}, + {file = "pyinstrument-4.5.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:54be442df5039bc7c73e3e86de0093ca82f3e446392bebab29e51a1512c796cb"}, + {file = "pyinstrument-4.5.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:35e5be8621b3381cf10b1f16bbae527cb7902e87b64e0c9706bc244f6fee51b1"}, + {file = "pyinstrument-4.5.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:50e93fac7e42dba8b3c630ed00808e7664d0d6c6b0c477462e7b061a31be23dc"}, + {file = "pyinstrument-4.5.1-cp38-cp38-win32.whl", hash = "sha256:b0a88bfe24d4efb129ef2ae7e2d50fa29908634e893bf154e29f91655c558692"}, + {file = "pyinstrument-4.5.1-cp38-cp38-win_amd64.whl", hash = "sha256:b8a71ef9c2ad81e5f3d5f92e1d21a0c9b5f9992e94d0bfcfa9020ea88df4e69f"}, + {file = "pyinstrument-4.5.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9882827e681466d1aff931479387ed77e29674c179bc10fc67f1fa96f724dd20"}, + {file = "pyinstrument-4.5.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:427228a011d5be21ff009dc05fcd512cee86ea2a51687a3300b8b822bad6815b"}, + {file = "pyinstrument-4.5.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50501756570352e78aaf2aee509b5eb6c68706a2f2701dc3a84b066e570c61ca"}, + {file = "pyinstrument-4.5.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6471f47860f1a5807c182be7184839d747e2702625d44ec19a8f652380541020"}, + {file = "pyinstrument-4.5.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59727936e862677e9716b9317e209e5e31aa1da7eb03c65083d9dee8b5fbe0f8"}, + {file = "pyinstrument-4.5.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9341a07885cba57c2a134847aacb629f27b4ce06a4950a4619629d35a6d8619c"}, + {file = "pyinstrument-4.5.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:63c27f2ae8f0501dca4d52b42285be36095f4461dd9e340d32104c2b2df3a731"}, + {file = "pyinstrument-4.5.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1bda9b73dde7df63d7606e37340ba0a63ad59053e59eff318f3b67d5a7ea5579"}, + {file = "pyinstrument-4.5.1-cp39-cp39-win32.whl", hash = "sha256:300ed27714c43ae2feb7572e9b3ca39660fb89b3b298e94ad24b64609f823d3c"}, + {file = "pyinstrument-4.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:f2d8e4a9a8167c2a47874d72d6ab0a4266ed484e9ae30f35a515f8594b224b51"}, + {file = "pyinstrument-4.5.1.tar.gz", hash = "sha256:b55a93be883c65650515319455636d32ab32692b097faa1e07f8cd9d4e0eeaa9"}, ] [package.extras] @@ -1711,14 +1675,13 @@ jupyter = ["ipython"] [[package]] name = "pyjwt" -version = "2.7.0" +version = "2.8.0" description = "JSON Web Token implementation in Python" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "PyJWT-2.7.0-py3-none-any.whl", hash = "sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1"}, - {file = "PyJWT-2.7.0.tar.gz", hash = "sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074"}, + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, ] [package.dependencies] @@ -1734,7 +1697,6 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pynacl" version = "1.5.0" description = "Python binding to the Networking and Cryptography (NaCl) library" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1761,7 +1723,6 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] name = "pytablewriter" version = "0.64.2" description = "pytablewriter is a Python library to write a table in various formats: AsciiDoc / CSV / Elasticsearch / HTML / JavaScript / JSON / LaTeX / LDJSON / LTSV / Markdown / MediaWiki / NumPy / Excel / Pandas / Python / reStructuredText / SQLite / TOML / TSV / YAML." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1797,7 +1758,6 @@ yaml = ["PyYAML (>=3.11,<7)"] name = "pytest" version = "6.2.5" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1822,7 +1782,6 @@ testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xm name = "pytest-mock" version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" -category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1840,7 +1799,6 @@ dev = ["pre-commit", "pytest-asyncio", "tox"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1855,7 +1813,6 @@ six = ">=1.5" name = "pytz" version = "2023.3" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" files = [ @@ -1865,59 +1822,57 @@ files = [ [[package]] name = "pyyaml" -version = "6.0" +version = "6.0.1" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] [[package]] name = "requests" version = "2.31.0" description = "Python HTTP for Humans." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1939,7 +1894,6 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-oauthlib" version = "1.3.1" description = "OAuthlib authentication support for Requests." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1956,28 +1910,27 @@ rsa = ["oauthlib[signedtoken] (>=3.0.0)"] [[package]] name = "rich" -version = "13.3.5" +version = "11.2.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "main" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.6.2,<4.0.0" files = [ - {file = "rich-13.3.5-py3-none-any.whl", hash = "sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704"}, - {file = "rich-13.3.5.tar.gz", hash = "sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c"}, + {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, + {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, ] [package.dependencies] -markdown-it-py = ">=2.2.0,<3.0.0" -pygments = ">=2.13.0,<3.0.0" +colorama = ">=0.4.0,<0.5.0" +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" [package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" -category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -1990,14 +1943,13 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruamel-yaml" -version = "0.17.30" +version = "0.17.32" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "main" optional = false python-versions = ">=3" files = [ - {file = "ruamel.yaml-0.17.30-py3-none-any.whl", hash = "sha256:b73821c1e0a9daa459e215db7c4d6d23d7159fb61c4c536653a2b4fb5b8d08a7"}, - {file = "ruamel.yaml-0.17.30.tar.gz", hash = "sha256:7e47a3d9ccfa2e41d353e6615661308bcbbc2ebad393bdc13b725bac15baf8a9"}, + {file = "ruamel.yaml-0.17.32-py3-none-any.whl", hash = "sha256:23cd2ed620231677564646b0c6a89d138b6822a0d78656df7abda5879ec4f447"}, + {file = "ruamel.yaml-0.17.32.tar.gz", hash = "sha256:ec939063761914e14542972a5cba6d33c23b0859ab6342f61cf070cfc600efc2"}, ] [package.dependencies] @@ -2011,7 +1963,6 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] name = "ruamel-yaml-clib" version = "0.2.7" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "main" optional = false python-versions = ">=3.5" files = [ @@ -2055,14 +2006,13 @@ files = [ [[package]] name = "setuptools" -version = "67.8.0" +version = "68.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.8.0-py3-none-any.whl", hash = "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f"}, - {file = "setuptools-67.8.0.tar.gz", hash = "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102"}, + {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, + {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, ] [package.extras] @@ -2074,7 +2024,6 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2086,7 +2035,6 @@ files = [ name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2098,7 +2046,6 @@ files = [ name = "tabledata" version = "1.3.1" description = "tabledata is a Python library to represent tabular data. Used for pytablewriter/pytablereader/SimpleSQLite/etc." -category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2118,7 +2065,6 @@ test = ["pytablewriter (>=0.46)", "pytest"] name = "tcolorpy" version = "0.1.3" description = "tcolopy is a Python library to apply true color for terminal text." -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2133,7 +2079,6 @@ test = ["pytest", "pytest-md-report (>=0.1)"] name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" -category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2143,14 +2088,13 @@ files = [ [[package]] name = "typepy" -version = "1.3.0" +version = "1.3.1" description = "typepy is a Python library for variable type checker/validator/converter at a run time." -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "typepy-1.3.0-py3-none-any.whl", hash = "sha256:cf1913982969cf6348152c4a5feec08e324addd99670999e57cdb3ad87a61e9a"}, - {file = "typepy-1.3.0.tar.gz", hash = "sha256:96788530614083164993d1443959f6c58e6bb8e2da839812ddf462c203e4b84c"}, + {file = "typepy-1.3.1-py3-none-any.whl", hash = "sha256:892566bff279368d63f02901aba0a3ce78cd7a319ec1f2bf6c8baab3520207a3"}, + {file = "typepy-1.3.1.tar.gz", hash = "sha256:dfc37b888d6eed8542208389efa60ec8454e06fd84b276b45b2e33897f9d7825"}, ] [package.dependencies] @@ -2165,26 +2109,24 @@ test = ["packaging", "pytest (>=6.0.1)", "python-dateutil (>=2.8.0,<3.0.0)", "py [[package]] name = "typing-extensions" -version = "4.5.0" +version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] name = "urllib3" -version = "1.26.15" +version = "1.26.16" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, - {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, ] [package.extras] @@ -2196,7 +2138,6 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2281,7 +2222,6 @@ files = [ name = "yarl" version = "1.9.2" description = "Yet another URL library" -category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2367,5 +2307,5 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" -python-versions = "^3.9" -content-hash = "671b302a276c6ac2f0c592d1ef2e82273021a5d0801a0e548474e9b4165a7299" +python-versions = "^3.10" +content-hash = "bc89c823b268bf41879da6dbc8559a4662ee564b407dd9da858f59e79cf7d058" diff --git a/airbyte-ci/connectors/qa-engine/pyproject.toml b/airbyte-ci/connectors/qa-engine/pyproject.toml index ca8a85ab7caa..2ca0e4554103 100644 --- a/airbyte-ci/connectors/qa-engine/pyproject.toml +++ b/airbyte-ci/connectors/qa-engine/pyproject.toml @@ -11,7 +11,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = "^3.9" +python = "^3.10" click = "~8.1.3" requests = "*" PyYAML = "~6.0" @@ -24,8 +24,9 @@ pandas-gbq = "~0.19.0" fsspec = "~2023.1.0" gcsfs = "~2023.1.0" pytablewriter = "~0.64.2" -pydash = "^6.0.2" +pydash = "^7.0.4" ruamel-yaml = "^0.17.30" +connector-ops = {path = "../connector_ops"} [tool.poetry.group.dev.dependencies] pyinstrument = "*" diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/constants.py b/airbyte-ci/connectors/qa-engine/qa_engine/constants.py index 0f9eccc82750..92d98ae6172c 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/constants.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/constants.py @@ -1,9 +1,10 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import os -CONNECTOR_BUILD_OUTPUT_URL = "https://dnsgjos7lj2fu.cloudfront.net/tests/history/connectors" +CONNECTOR_TEST_SUMMARY_URL = "https://connectors.airbyte.com/files/generated_reports/test_summary" CLOUD_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/cloud_registry.json" OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" @@ -29,6 +30,7 @@ "4528e960-6f7b-4412-8555-7e0097e1da17", # destination-starburst-galaxy, no strict-encrypt variant "aa8ba6fd-4875-d94e-fc8d-4e1e09aa2503", # source-teradata, no strict-encrypt variant "447e0381-3780-4b46-bb62-00a4e3c8b8e2", # source-db2, no strict-encrypt variant + "0eeee7fb-518f-4045-bacc-9619e31c43ea", # destination-amazon-sqs, hide Amazon SQS Destination https://github.com/airbytehq/airbyte/issues/16316 ] GCS_QA_REPORT_PATH = "gs://airbyte-data-connectors-qa-engine/" diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/enrichments.py b/airbyte-ci/connectors/qa-engine/qa_engine/enrichments.py index e5e9d4d7cfad..5bdfd48fe60e 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/enrichments.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/enrichments.py @@ -14,7 +14,7 @@ def get_enriched_catalog( - Rename columns to snake case. - Rename name column to connector_name. - Rename docker_image_tag to connector_version. - - Replace null value for release_stage with alpha. + - Replace null value for support_level with unknown. Enrichments: - is_on_cloud: determined by the merge operation results. - connector_technical_name: built from the docker repository field. airbyte/source-pokeapi -> source-pokeapi. @@ -45,7 +45,7 @@ def get_enriched_catalog( enriched_catalog["connector_name"] = enriched_catalog["name"] enriched_catalog["connector_technical_name"] = enriched_catalog["docker_repository"].str.replace("airbyte/", "") enriched_catalog["connector_version"] = enriched_catalog["docker_image_tag"] - enriched_catalog["release_stage"] = enriched_catalog["release_stage"].fillna("unknown") + enriched_catalog["support_level"] = enriched_catalog["support_level"].fillna("unknown") enriched_catalog = enriched_catalog.merge( adoption_metrics_per_connector_version, how="left", on=["connector_definition_id", "connector_version"] ) diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/inputs.py b/airbyte-ci/connectors/qa-engine/qa_engine/inputs.py index cca5e8150560..8f481ae8ac82 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/inputs.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/inputs.py @@ -10,7 +10,7 @@ import pandas as pd import requests -from .constants import CONNECTOR_BUILD_OUTPUT_URL +from .constants import CONNECTOR_TEST_SUMMARY_URL LOGGER = logging.getLogger(__name__) @@ -28,28 +28,43 @@ def from_string(cls, string_value: Optional[str]) -> "BUILD_STATUSES": return BUILD_STATUSES[string_value.upper()] -def get_connector_build_output_url(connector_technical_name: str, connector_version: str) -> str: +def get_connector_build_output_url(connector_technical_name: str) -> str: """ Get the connector build output url. - Documentation of the larger build output system can be found here: https://internal-docs.airbyte.io/Generated-Reports/Build-Status-Reports """ - return f"{CONNECTOR_BUILD_OUTPUT_URL}/{connector_technical_name}/version-{connector_version}.json" + # remove connectors/ prefix from connector_technical_name + connector_technical_name = connector_technical_name.replace("connectors/", "") + return f"{CONNECTOR_TEST_SUMMARY_URL}/{connector_technical_name}/index.json" -def fetch_latest_build_status_for_connector_version(connector_technical_name: str, connector_version: str) -> BUILD_STATUSES: +def fetch_latest_build_status_for_connector(connector_technical_name: str) -> BUILD_STATUSES: """Fetch the latest build status for a given connector version.""" - connector_build_output_url = get_connector_build_output_url(connector_technical_name, connector_version) + connector_build_output_url = get_connector_build_output_url(connector_technical_name) connector_build_output_response = requests.get(connector_build_output_url) # if the connector returned successfully, return the outcome if connector_build_output_response.status_code == 200: connector_build_output = connector_build_output_response.json() - outcome = connector_build_output.get("outcome") + + # we want to get the latest build status + # sort by date and get the first element + latest_connector_run = sorted(connector_build_output, key=lambda x: x["date"], reverse=True)[0] + + outcome = latest_connector_run.get("success") + if outcome is None: + LOGGER.error(f"Error: No outcome value for connector {connector_technical_name}") + return BUILD_STATUSES.NOT_FOUND + + if outcome == True: + return BUILD_STATUSES.SUCCESS + + if outcome == False: + return BUILD_STATUSES.FAILURE try: return BUILD_STATUSES.from_string(outcome) except KeyError: - LOGGER.error(f"Error: Unexpected build status value: {outcome} for connector {connector_technical_name}:{connector_version}") + LOGGER.error(f"Error: Unexpected build status value: {outcome} for connector {connector_technical_name}") return BUILD_STATUSES.NOT_FOUND else: diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/models.py b/airbyte-ci/connectors/qa-engine/qa_engine/models.py index 0c26a25d8dfc..c5c009a7b083 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/models.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/models.py @@ -4,24 +4,11 @@ from datetime import datetime -from enum import Enum from typing import List +from connector_ops.utils import ConnectorTypeEnum, SupportLevelEnum from pydantic import BaseModel, Field - -class ConnectorTypeEnum(str, Enum): - source = "source" - destination = "destination" - - -class ReleaseStageEnum(str, Enum): - unknown = "unknown" - alpha = "alpha" - beta = "beta" - generally_available = "generally_available" - - PUBLIC_FIELD = Field(..., is_public=True) PRIVATE_FIELD = Field(..., is_public=False) @@ -32,7 +19,7 @@ class ConnectorQAReport(BaseModel): connector_technical_name: str = PUBLIC_FIELD connector_definition_id: str = PUBLIC_FIELD connector_version: str = PUBLIC_FIELD - release_stage: ReleaseStageEnum = PUBLIC_FIELD + support_level: SupportLevelEnum = PUBLIC_FIELD is_on_cloud: bool = PUBLIC_FIELD is_appropriate_for_cloud_use: bool = PUBLIC_FIELD latest_build_is_successful: bool = PUBLIC_FIELD diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/validations.py b/airbyte-ci/connectors/qa-engine/qa_engine/validations.py index f94923386539..058e1af5ee22 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/validations.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/validations.py @@ -10,7 +10,7 @@ import requests from .constants import INAPPROPRIATE_FOR_CLOUD_USE_CONNECTORS -from .inputs import BUILD_STATUSES, fetch_latest_build_status_for_connector_version +from .inputs import BUILD_STATUSES, fetch_latest_build_status_for_connector from .models import ConnectorQAReport, QAReport logger = logging.getLogger(__name__) @@ -40,8 +40,7 @@ def is_eligible_for_promotion_to_cloud(connector_qa_data: pd.Series) -> bool: def latest_build_is_successful(connector_qa_data: pd.Series) -> bool: connector_technical_name = connector_qa_data["connector_technical_name"] - connector_version = connector_qa_data["connector_version"] - latest_build_status = fetch_latest_build_status_for_connector_version(connector_technical_name, connector_version) + latest_build_status = fetch_latest_build_status_for_connector(connector_technical_name) return latest_build_status == BUILD_STATUSES.SUCCESS diff --git a/airbyte-ci/connectors/qa-engine/tests/conftest.py b/airbyte-ci/connectors/qa-engine/tests/conftest.py index d79c12a42043..e68c067c61fe 100644 --- a/airbyte-ci/connectors/qa-engine/tests/conftest.py +++ b/airbyte-ci/connectors/qa-engine/tests/conftest.py @@ -4,10 +4,10 @@ from datetime import datetime + import pandas as pd import pytest - -from qa_engine.constants import OSS_CATALOG_URL, CLOUD_CATALOG_URL +from qa_engine.constants import CLOUD_CATALOG_URL, OSS_CATALOG_URL from qa_engine.inputs import fetch_remote_catalog @@ -49,7 +49,7 @@ def dummy_qa_report() -> pd.DataFrame: "connector_technical_name": "source-test", "connector_definition_id": "foobar", "connector_version": "0.0.0", - "release_stage": "alpha", + "support_level": "community", "is_on_cloud": False, "is_appropriate_for_cloud_use": True, "latest_build_is_successful": True, diff --git a/airbyte-ci/connectors/qa-engine/tests/test_cloud_availability_updater.py b/airbyte-ci/connectors/qa-engine/tests/test_cloud_availability_updater.py index 70981d39918b..c143b49916b9 100644 --- a/airbyte-ci/connectors/qa-engine/tests/test_cloud_availability_updater.py +++ b/airbyte-ci/connectors/qa-engine/tests/test_cloud_availability_updater.py @@ -26,7 +26,7 @@ def eligible_connectors(): models.ConnectorQAReport( connector_type="source", connector_name="PokeAPI", - release_stage="alpha", + support_level="community", is_on_cloud=False, is_appropriate_for_cloud_use=True, latest_build_is_successful=True, @@ -52,7 +52,7 @@ def excluded_connectors(): models.ConnectorQAReport( connector_type="source", connector_name="excluded", - release_stage="alpha", + support_level="community", is_on_cloud=False, is_appropriate_for_cloud_use=True, latest_build_is_successful=True, diff --git a/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py b/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py index d72f87e94d43..73ae7fde8c41 100644 --- a/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py +++ b/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py @@ -7,7 +7,6 @@ import pandas as pd import pytest - from qa_engine import enrichments @@ -46,5 +45,5 @@ def test_no_column_are_removed_and_lowercased(enriched_catalog_columns, oss_cata assert re.sub(r"(? arg); } @Override public String strictComparisonNormalizationTag() { - return ""; + return getEnvOrDefault(STRICT_COMPARISON_NORMALIZATION_TAG, "strict_comparison2", (arg) -> arg); } // TODO: refactor in order to use the same method than the ones in EnvConfigs.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java index f03dc46d8dd2..b3da9ac764bb 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java @@ -10,17 +10,13 @@ */ public interface FeatureFlags { - boolean autoDisablesFailingConnections(); - - boolean forceSecretMigration(); - boolean useStreamCapableState(); boolean autoDetectSchema(); boolean logConnectorMessages(); - boolean needStateValidation(); + boolean concurrentSourceStreamRead(); /** * Return true if field selection should be applied. See also fieldSelectionWorkspaces. @@ -39,7 +35,7 @@ public interface FeatureFlags { /** * Get the workspaces allow-listed for strict incremental comparison in normalization. This takes - * precedence over the normalization version in oss_registry.json . + * precedence over the normalization version in destination_definitions.yaml. * * @return a comma-separated list of workspace ids where strict incremental comparison should be * enabled in normalization. diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index d3f05739c00b..18b40c7c5489 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.core.util.Separators; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -41,6 +42,13 @@ public class Jsons { // Object Mapper is thread-safe private static final ObjectMapper OBJECT_MAPPER = MoreMappers.initMapper(); + // sort of a hotfix; I don't know how bad the performance hit is so not turning this on by default + // at time of writing (2023-08-18) this is only used in tests, so we don't care. + private static final ObjectMapper OBJECT_MAPPER_EXACT; + static { + OBJECT_MAPPER_EXACT = MoreMappers.initMapper(); + OBJECT_MAPPER_EXACT.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + } private static final ObjectMapper YAML_OBJECT_MAPPER = MoreMappers.initYamlMapper(new YAMLFactory()); private static final ObjectWriter OBJECT_WRITER = OBJECT_MAPPER.writer(new JsonPrettyPrinter()); @@ -97,6 +105,14 @@ public static JsonNode deserialize(final String jsonString) { } } + public static JsonNode deserializeExact(final String jsonString) { + try { + return OBJECT_MAPPER_EXACT.readTree(jsonString); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + public static Optional tryDeserialize(final String jsonString, final Class klass) { try { return Optional.of(OBJECT_MAPPER.readValue(jsonString, klass)); @@ -326,7 +342,7 @@ public static void mergeMaps(final Map originalMap, final String Entry::getValue))); } - public static Map deserializeToStringMap(JsonNode json) { + public static Map deserializeToStringMap(final JsonNode json) { return OBJECT_MAPPER.convertValue(json, new TypeReference<>() {}); } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java b/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java new file mode 100644 index 000000000000..f4f748bef993 --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.stream; + +import io.airbyte.commons.util.AirbyteStreamAware; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus; +import java.util.Optional; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of utility methods that support the generation of stream status updates. + */ +public class StreamStatusUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamStatusUtils.class); + + /** + * Creates a new {@link Consumer} that wraps the provided {@link Consumer} with stream status + * reporting capabilities. Specifically, this consumer will emit an + * {@link AirbyteStreamStatus#RUNNING} status after the first message is consumed by the delegated + * {@link Consumer}. + * + * @param stream The stream from which the delegating {@link Consumer} will consume messages for + * processing. + * @param delegateRecordCollector The delegated {@link Consumer} that will be called when this + * consumer accepts a message for processing. + * @param streamStatusEmitter The optional {@link Consumer} that will be used to emit stream status + * updates. + * @return A wrapping {@link Consumer} that provides stream status updates when the provided + * delegate {@link Consumer} is invoked. + */ + public static Consumer statusTrackingRecordCollector(final AutoCloseableIterator stream, + final Consumer delegateRecordCollector, + final Optional> streamStatusEmitter) { + return new Consumer<>() { + + private boolean firstRead = true; + + @Override + public void accept(final AirbyteMessage airbyteMessage) { + try { + delegateRecordCollector.accept(airbyteMessage); + } finally { + if (firstRead) { + emitRunningStreamStatus(stream, streamStatusEmitter); + firstRead = false; + } + } + } + + }; + } + + /** + * Emits a {@link AirbyteStreamStatus#RUNNING} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitRunningStreamStatus(final AutoCloseableIterator airbyteStream, + final Optional> statusEmitter) { + if (airbyteStream instanceof AirbyteStreamAware) { + emitRunningStreamStatus((AirbyteStreamAware) airbyteStream, statusEmitter); + } + } + + /** + * Emits a {@link AirbyteStreamStatus#RUNNING} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitRunningStreamStatus(final AirbyteStreamAware airbyteStream, + final Optional> statusEmitter) { + emitRunningStreamStatus(airbyteStream.getAirbyteStream(), statusEmitter); + } + + /** + * Emits a {@link AirbyteStreamStatus#RUNNING} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitRunningStreamStatus(final Optional airbyteStream, + final Optional> statusEmitter) { + airbyteStream.ifPresent(s -> { + LOGGER.debug("RUNNING -> {}", s); + emitStreamStatus(s, AirbyteStreamStatus.RUNNING, statusEmitter); + }); + } + + /** + * Emits a {@link AirbyteStreamStatus#STARTED} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitStartStreamStatus(final AutoCloseableIterator airbyteStream, + final Optional> statusEmitter) { + if (airbyteStream instanceof AirbyteStreamAware) { + emitStartStreamStatus((AirbyteStreamAware) airbyteStream, statusEmitter); + } + } + + /** + * Emits a {@link AirbyteStreamStatus#STARTED} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitStartStreamStatus(final AirbyteStreamAware airbyteStream, + final Optional> statusEmitter) { + emitStartStreamStatus(airbyteStream.getAirbyteStream(), statusEmitter); + } + + /** + * Emits a {@link AirbyteStreamStatus#STARTED} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitStartStreamStatus(final Optional airbyteStream, + final Optional> statusEmitter) { + airbyteStream.ifPresent(s -> { + LOGGER.debug("STARTING -> {}", s); + emitStreamStatus(s, AirbyteStreamStatus.STARTED, statusEmitter); + }); + } + + /** + * Emits a {@link AirbyteStreamStatus#COMPLETE} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitCompleteStreamStatus(final AutoCloseableIterator airbyteStream, + final Optional> statusEmitter) { + if (airbyteStream instanceof AirbyteStreamAware) { + emitCompleteStreamStatus((AirbyteStreamAware) airbyteStream, statusEmitter); + } + } + + /** + * Emits a {@link AirbyteStreamStatus#COMPLETE} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitCompleteStreamStatus(final AirbyteStreamAware airbyteStream, + final Optional> statusEmitter) { + emitCompleteStreamStatus(airbyteStream.getAirbyteStream(), statusEmitter); + } + + /** + * Emits a {@link AirbyteStreamStatus#COMPLETE} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitCompleteStreamStatus(final Optional airbyteStream, + final Optional> statusEmitter) { + airbyteStream.ifPresent(s -> { + LOGGER.debug("COMPLETE -> {}", s); + emitStreamStatus(s, AirbyteStreamStatus.COMPLETE, statusEmitter); + }); + } + + /** + * Emits a {@link AirbyteStreamStatus#INCOMPLETE} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitIncompleteStreamStatus(final AutoCloseableIterator airbyteStream, + final Optional> statusEmitter) { + if (airbyteStream instanceof AirbyteStreamAware) { + emitIncompleteStreamStatus((AirbyteStreamAware) airbyteStream, statusEmitter); + } + } + + /** + * Emits a {@link AirbyteStreamStatus#INCOMPLETE} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitIncompleteStreamStatus(final AirbyteStreamAware airbyteStream, + final Optional> statusEmitter) { + emitIncompleteStreamStatus(airbyteStream.getAirbyteStream(), statusEmitter); + } + + /** + * Emits a {@link AirbyteStreamStatus#INCOMPLETE} stream status for the provided stream. + * + * @param airbyteStream The stream that should be associated with the stream status. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + public static void emitIncompleteStreamStatus(final Optional airbyteStream, + final Optional> statusEmitter) { + airbyteStream.ifPresent(s -> { + LOGGER.debug("INCOMPLETE -> {}", s); + emitStreamStatus(s, AirbyteStreamStatus.INCOMPLETE, statusEmitter); + }); + } + + /** + * Emits a stream status for the provided stream. + * + * @param airbyteStreamNameNamespacePair The stream identifier. + * @param airbyteStreamStatus The status update. + * @param statusEmitter The {@link Optional} stream status emitter. + */ + private static void emitStreamStatus(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final AirbyteStreamStatus airbyteStreamStatus, + final Optional> statusEmitter) { + statusEmitter.ifPresent(consumer -> consumer.accept(new AirbyteStreamStatusHolder(airbyteStreamNameNamespacePair, airbyteStreamStatus))); + } + +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java b/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java index 06949c479fb2..ccbc11e10a11 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java @@ -12,4 +12,4 @@ * * @param type */ -public interface AutoCloseableIterator extends Iterator, AutoCloseable {} +public interface AutoCloseableIterator extends Iterator, AutoCloseable, AirbyteStreamAware {} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java b/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java index 26152d962479..9423f54c5eb9 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java @@ -174,6 +174,12 @@ public static AutoCloseableIterator transform(final Function(iteratorCreator.apply(autoCloseableIterator), autoCloseableIterator::close, airbyteStream); } + public static AutoCloseableIterator transformIterator(final Function, Iterator> iteratorCreator, + final AutoCloseableIterator autoCloseableIterator, + final AirbyteStreamNameNamespacePair airbyteStream) { + return new DefaultAutoCloseableIterator(iteratorCreator.apply(autoCloseableIterator), autoCloseableIterator::close, airbyteStream); + } + @SafeVarargs public static CompositeIterator concatWithEagerClose(final Consumer airbyteStreamStatusConsumer, final AutoCloseableIterator... iterators) { @@ -185,6 +191,15 @@ public static CompositeIterator concatWithEagerClose(final AutoCloseableI return concatWithEagerClose(List.of(iterators), null); } + /** + * Creates a {@link CompositeIterator} that reads from the provided iterators in a serial fashion. + * + * @param iterators The list of iterators to be used in a serial fashion. + * @param airbyteStreamStatusConsumer The stream status consumer used to report stream status during + * iteration. + * @return A {@link CompositeIterator}. + * @param The type of data contained in each iterator. + */ public static CompositeIterator concatWithEagerClose(final List> iterators, final Consumer airbyteStreamStatusConsumer) { return new CompositeIterator<>(iterators, airbyteStreamStatusConsumer); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java b/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java index 2f92c1b68e92..7c5997d344c6 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java @@ -7,8 +7,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.AbstractIterator; import io.airbyte.commons.stream.AirbyteStreamStatusHolder; +import io.airbyte.commons.stream.StreamStatusUtils; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -37,7 +37,7 @@ * * @param type */ -public final class CompositeIterator extends AbstractIterator implements AutoCloseableIterator, AirbyteStreamAware { +public final class CompositeIterator extends AbstractIterator implements AutoCloseableIterator { private static final Logger LOGGER = LoggerFactory.getLogger(CompositeIterator.class); @@ -72,15 +72,15 @@ protected T computeNext() { while (!currentIterator().hasNext()) { try { currentIterator().close(); - emitCompleteStreamStatus(getAirbyteStream()); + StreamStatusUtils.emitCompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); } catch (final Exception e) { - emitIncompleteStreamStatus(getAirbyteStream()); + StreamStatusUtils.emitIncompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); throw new RuntimeException(e); } if (i + 1 < iterators.size()) { i++; - emitStartStreamStatus(getAirbyteStream()); + StreamStatusUtils.emitStartStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); firstRead = true; } else { return endOfData(); @@ -89,15 +89,15 @@ protected T computeNext() { try { if (isFirstStream()) { - emitStartStreamStatus(getAirbyteStream()); + StreamStatusUtils.emitStartStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); } return currentIterator().next(); } catch (final RuntimeException e) { - emitIncompleteStreamStatus(getAirbyteStream()); + StreamStatusUtils.emitIncompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); throw e; } finally { if (firstRead) { - emitRunningStreamStatus(getAirbyteStream()); + StreamStatusUtils.emitRunningStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); firstRead = false; } } @@ -143,37 +143,4 @@ private void assertHasNotClosed() { Preconditions.checkState(!hasClosed); } - private void emitRunningStreamStatus(final Optional airbyteStream) { - airbyteStream.ifPresent(s -> { - LOGGER.info("RUNNING -> {}", s); - emitStreamStatus(s, AirbyteStreamStatus.RUNNING); - }); - } - - private void emitStartStreamStatus(final Optional airbyteStream) { - airbyteStream.ifPresent(s -> { - LOGGER.info("STARTING -> {}", s); - emitStreamStatus(s, AirbyteStreamStatus.STARTED); - }); - } - - private void emitCompleteStreamStatus(final Optional airbyteStream) { - airbyteStream.ifPresent(s -> { - LOGGER.info("COMPLETE -> {}", s); - emitStreamStatus(s, AirbyteStreamStatus.COMPLETE); - }); - } - - private void emitIncompleteStreamStatus(final Optional airbyteStream) { - airbyteStream.ifPresent(s -> { - LOGGER.info("COMPLETE -> {}", s); - emitStreamStatus(s, AirbyteStreamStatus.INCOMPLETE); - }); - } - - private void emitStreamStatus(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, - final AirbyteStreamStatus airbyteStreamStatus) { - airbyteStreamStatusConsumer.ifPresent(c -> c.accept(new AirbyteStreamStatusHolder(airbyteStreamNameNamespacePair, airbyteStreamStatus))); - } - } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java b/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java index e6051910c023..effd09566e37 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java @@ -17,7 +17,7 @@ * * @param type */ -class DefaultAutoCloseableIterator extends AbstractIterator implements AutoCloseableIterator, AirbyteStreamAware { +class DefaultAutoCloseableIterator extends AbstractIterator implements AutoCloseableIterator { private final AirbyteStreamNameNamespacePair airbyteStream; private final Iterator iterator; diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java b/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java index 5479e63ac333..77fcbeb51308 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java @@ -20,7 +20,7 @@ * * @param type */ -class LazyAutoCloseableIterator extends AbstractIterator implements AutoCloseableIterator, AirbyteStreamAware { +class LazyAutoCloseableIterator extends AbstractIterator implements AutoCloseableIterator { private final Supplier> iteratorSupplier; diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java new file mode 100644 index 000000000000..5ddbbd2ed288 --- /dev/null +++ b/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java @@ -0,0 +1,498 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.stream; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.commons.util.AirbyteStreamAware; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus; +import java.util.Optional; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.junit.jupiter.MockitoExtension; + +/** + * Test suite for the {@link StreamStatusUtils} class. + */ +@ExtendWith(MockitoExtension.class) +class StreamStatusUtilsTest { + + private static final String NAME = "name"; + private static final String NAMESPACE = "namespace"; + + @Captor + private ArgumentCaptor airbyteStreamStatusHolderArgumentCaptor; + + @Test + void testCreateStreamStatusConsumerWrapper() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Optional> streamStatusEmitter = Optional.empty(); + final Consumer messageConsumer = mock(Consumer.class); + + final Consumer wrappedMessageConsumer = + StreamStatusUtils.statusTrackingRecordCollector(stream, messageConsumer, streamStatusEmitter); + + assertNotEquals(messageConsumer, wrappedMessageConsumer); + } + + @Test + void testStreamStatusConsumerWrapperProduceStreamStatus() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + final Consumer messageConsumer = mock(Consumer.class); + final AirbyteMessage airbyteMessage = mock(AirbyteMessage.class); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + final Consumer wrappedMessageConsumer = + StreamStatusUtils.statusTrackingRecordCollector(stream, messageConsumer, streamStatusEmitter); + + assertNotEquals(messageConsumer, wrappedMessageConsumer); + + wrappedMessageConsumer.accept(airbyteMessage); + wrappedMessageConsumer.accept(airbyteMessage); + wrappedMessageConsumer.accept(airbyteMessage); + + verify(messageConsumer, times(3)).accept(any()); + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.RUNNING, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitRunningStreamStatusIterator() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitRunningStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.RUNNING, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitRunningStreamStatusIteratorEmptyAirbyteStream() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitRunningStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitRunningStreamStatusIteratorEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitRunningStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitRunningStreamStatusAirbyteStreamAware() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitRunningStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.RUNNING, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitRunningStreamStatusAirbyteStreamAwareEmptyStream() { + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitRunningStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitRunningStreamStatusAirbyteStreamAwareEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitRunningStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitRunningStreamStatusAirbyteStream() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + StreamStatusUtils.emitRunningStreamStatus(Optional.of(airbyteStream), streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.RUNNING, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitRunningStreamStatusEmptyAirbyteStream() { + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + assertDoesNotThrow(() -> StreamStatusUtils.emitRunningStreamStatus(Optional.empty(), streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitRunningStreamStatusAirbyteStreamEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Optional> streamStatusEmitter = Optional.empty(); + + assertDoesNotThrow(() -> StreamStatusUtils.emitRunningStreamStatus(Optional.of(airbyteStream), streamStatusEmitter)); + } + + @Test + void testEmitStartedStreamStatusIterator() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.STARTED, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitStartedStreamStatusIteratorEmptyAirbyteStream() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitStartedStreamStatusIteratorEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitStartedStreamStatusAirbyteStreamAware() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.STARTED, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitStartedStreamStatusAirbyteStreamAwareEmptyStream() { + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitStartedStreamStatusAirbyteStreamAwareEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitStartedStreamStatusAirbyteStream() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + StreamStatusUtils.emitStartStreamStatus(Optional.of(airbyteStream), streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.STARTED, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitStartedStreamStatusEmptyAirbyteStream() { + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + assertDoesNotThrow(() -> StreamStatusUtils.emitStartStreamStatus(Optional.empty(), streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitStartedStreamStatusAirbyteStreamEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Optional> streamStatusEmitter = Optional.empty(); + + assertDoesNotThrow(() -> StreamStatusUtils.emitStartStreamStatus(Optional.of(airbyteStream), streamStatusEmitter)); + } + + @Test + void testEmitCompleteStreamStatusIterator() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.COMPLETE, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitCompleteStreamStatusIteratorEmptyAirbyteStream() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitCompleteStreamStatusIteratorEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitCompleteStreamStatusAirbyteStreamAware() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.COMPLETE, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitCompleteStreamStatusAirbyteStreamAwareEmptyStream() { + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitCompleteStreamStatusAirbyteStreamAwareEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitCompleteStreamStatusAirbyteStream() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + StreamStatusUtils.emitCompleteStreamStatus(Optional.of(airbyteStream), streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.COMPLETE, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitCompleteStreamStatusEmptyAirbyteStream() { + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + assertDoesNotThrow(() -> StreamStatusUtils.emitCompleteStreamStatus(Optional.empty(), streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitCompleteStreamStatusAirbyteStreamEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Optional> streamStatusEmitter = Optional.empty(); + + assertDoesNotThrow(() -> StreamStatusUtils.emitCompleteStreamStatus(Optional.of(airbyteStream), streamStatusEmitter)); + } + + @Test + void testEmitIncompleteStreamStatusIterator() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.INCOMPLETE, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitIncompleteStreamStatusIteratorEmptyAirbyteStream() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitIncompleteStreamStatusIteratorEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitIncompleteStreamStatusAirbyteStreamAware() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.INCOMPLETE, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitIncompleteStreamStatusAirbyteStreamAwareEmptyStream() { + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + when(stream.getAirbyteStream()).thenReturn(Optional.empty()); + + assertDoesNotThrow(() -> StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitIncompleteStreamStatusAirbyteStreamAwareEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final AirbyteStreamAware stream = mock(AirbyteStreamAware.class); + final Optional> streamStatusEmitter = Optional.empty(); + + when(stream.getAirbyteStream()).thenReturn(Optional.of(airbyteStream)); + + assertDoesNotThrow(() -> StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter)); + } + + @Test + void testEmitIncompleteStreamStatusAirbyteStream() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + StreamStatusUtils.emitIncompleteStreamStatus(Optional.of(airbyteStream), streamStatusEmitter); + + verify(statusEmitter, times(1)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + assertEquals(AirbyteStreamStatus.INCOMPLETE, airbyteStreamStatusHolderArgumentCaptor.getValue().toTraceMessage().getStreamStatus().getStatus()); + } + + @Test + void testEmitIncompleteStreamStatusEmptyAirbyteStream() { + final Consumer statusEmitter = mock(Consumer.class); + final Optional> streamStatusEmitter = Optional.of(statusEmitter); + + assertDoesNotThrow(() -> StreamStatusUtils.emitIncompleteStreamStatus(Optional.empty(), streamStatusEmitter)); + verify(statusEmitter, times(0)).accept(airbyteStreamStatusHolderArgumentCaptor.capture()); + } + + @Test + void testEmitIncompleteStreamStatusAirbyteStreamEmptyStatusEmitter() { + final AirbyteStreamNameNamespacePair airbyteStream = new AirbyteStreamNameNamespacePair(NAME, NAMESPACE); + final Optional> streamStatusEmitter = Optional.empty(); + + assertDoesNotThrow(() -> StreamStatusUtils.emitIncompleteStreamStatus(Optional.of(airbyteStream), streamStatusEmitter)); + } + +} diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java index bc4661282d41..3fbb86b2d950 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java @@ -12,7 +12,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import io.airbyte.commons.concurrency.VoidCallable; import java.util.Iterator; @@ -73,7 +72,7 @@ void testAppendOnClose() throws Exception { @Test void testTransform() { final Iterator transform = Iterators.transform(MoreIterators.of(1, 2, 3), i -> i + 1); - assertEquals(ImmutableList.of(2, 3, 4), MoreIterators.toList(transform)); + assertEquals(List.of(2, 3, 4), MoreIterators.toList(transform)); } @Test @@ -81,17 +80,17 @@ void testConcatWithEagerClose() throws Exception { final VoidCallable onClose1 = mock(VoidCallable.class); final VoidCallable onClose2 = mock(VoidCallable.class); - final AutoCloseableIterator iterator = new CompositeIterator<>(ImmutableList.of( + final AutoCloseableIterator iterator = new CompositeIterator<>(List.of( AutoCloseableIterators.fromIterator(MoreIterators.of("a", "b"), onClose1, null), AutoCloseableIterators.fromIterator(MoreIterators.of("d"), onClose2, null)), null); - assertOnCloseInvocations(ImmutableList.of(), ImmutableList.of(onClose1, onClose2)); + assertOnCloseInvocations(List.of(), List.of(onClose1, onClose2)); assertNext(iterator, "a"); assertNext(iterator, "b"); assertNext(iterator, "d"); - assertOnCloseInvocations(ImmutableList.of(onClose1), ImmutableList.of(onClose2)); + assertOnCloseInvocations(List.of(onClose1), List.of(onClose2)); assertFalse(iterator.hasNext()); - assertOnCloseInvocations(ImmutableList.of(onClose1, onClose2), ImmutableList.of()); + assertOnCloseInvocations(List.of(onClose1, onClose2), List.of()); iterator.close(); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java index 4f9caeb3f476..084dfdee7940 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java @@ -17,9 +17,9 @@ public class PostgresUtils { public static PgLsn getLsn(final JdbcDatabase database) throws SQLException { - // pg version 10+. + // pg version >= 10. For versions < 10 use query select * from pg_current_xlog_location() final List jsonNodes = database - .bufferedResultSetQuery(conn -> conn.createStatement().executeQuery("SELECT pg_current_wal_lsn()"), + .bufferedResultSetQuery(conn -> conn.createStatement().executeQuery("select * from pg_current_wal_lsn()"), resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); Preconditions.checkState(jsonNodes.size() == 1); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/ConnectionFactory.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/ConnectionFactory.java new file mode 100644 index 000000000000..cd428f9e8130 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/ConnectionFactory.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.factory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + +/** + * This class as been added in order to be able to save the connection in a test. It was found that + * the {@link javax.sql.DataSource} close method wasn't propagating the connection properly. It + * shouldn't be needed in our application code. + */ +public class ConnectionFactory { + + /** + * Construct a new {@link Connection} instance using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param connectionProperties The extra properties to add to the connection. + * @param jdbcConnectionString The JDBC connection string. + * @return The configured {@link Connection} + */ + public static Connection create(final String username, + final String password, + final Map connectionProperties, + final String jdbcConnectionString) { + try { + Properties properties = new Properties(); + properties.put("user", username); + properties.put("password", password); + connectionProperties.forEach((k, v) -> properties.put(k, v)); + + return DriverManager.getConnection(jdbcConnectionString, + properties); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java index 6ae184d674c1..592d1b59b62d 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java @@ -27,5 +27,6 @@ public final class JdbcConstants { public static final String INTERNAL_COLUMN_SIZE = "columnSize"; public static final String INTERNAL_IS_NULLABLE = "isNullable"; public static final String INTERNAL_DECIMAL_DIGITS = "decimalDigits"; + public static final String KEY_SEQ = "KEY_SEQ"; } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java index 502cce75d737..d525a3e3df1b 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java @@ -107,9 +107,7 @@ public boolean tryAdvance(final Consumer action) { LOGGER.error("SQLState: {}, Message: {}", e.getSQLState(), e.getMessage()); streamException = e; isStreamFailed = true; - // throwing an exception in tryAdvance() method lead to the endless loop in Spliterator and stream - // will never close - return false; + throw new RuntimeException(e); } } diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java index d35fa52809dc..b5c4fa3cde34 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java @@ -156,7 +156,7 @@ public String uploadRecordsToBucket(final SerializableBuffer recordsData, private String loadDataIntoBucket(final String objectPath, final SerializableBuffer recordsData) throws IOException { final long partSize = DEFAULT_PART_SIZE; final String bucket = s3Config.getBucketName(); - final String partId = getPartId(objectPath); + final String partId = UUID.randomUUID().toString(); final String fileExtension = getExtension(recordsData.getFilename()); final String fullObjectKey; if (StringUtils.isNotBlank(s3Config.getFileNamePattern())) { @@ -230,17 +230,6 @@ protected static String getExtension(final String filename) { return "." + result; } - private String getPartId(final String objectPath) { - final String bucket = s3Config.getBucketName(); - final ObjectListing objects = s3Client.listObjects(bucket, objectPath); - if (objects.isTruncated()) { - // bucket contains too many objects, use an uuid instead - return UUID.randomUUID().toString(); - } else { - return Integer.toString(objects.getObjectSummaries().size()); - } - } - @Override public void dropBucketObject(final String objectPath) { cleanUpBucketObject(objectPath, List.of()); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java index 6bc2fcec059d..e5ad1755fa57 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.UUID; import org.apache.avro.Schema; @@ -32,8 +33,14 @@ public AvroRecordFactory(final Schema schema, final JsonAvroConverter converter) public GenericData.Record getAvroRecord(final UUID id, final AirbyteRecordMessage recordMessage) throws JsonProcessingException { final ObjectNode jsonRecord = MAPPER.createObjectNode(); - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_ID, id.toString()); - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, recordMessage.getEmittedAt()); + if (TypingAndDedupingFlag.isDestinationV2()) { + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, id.toString()); + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, recordMessage.getEmittedAt()); + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, (Long) null); + } else { + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_ID, id.toString()); + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, recordMessage.getEmittedAt()); + } jsonRecord.setAll((ObjectNode) recordMessage.getData()); return converter.convertToGenericDataRecord(WRITER.writeValueAsBytes(jsonRecord), schema); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java index 0ed30bf2fb15..e813f1ad52ae 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java @@ -25,10 +25,14 @@ public List getDataRow(final UUID id, final AirbyteRecordMessage recordM } @Override - public List getDataRow(JsonNode formattedData) { + public List getDataRow(final JsonNode formattedData) { return new LinkedList<>(getRecordColumns(formattedData)); } + public List getDataRow(final UUID id, final String formattedString, final long emittedAt) { + throw new UnsupportedOperationException("Not implemented in BaseSheetGenerator"); + } + abstract List getRecordColumns(JsonNode json); } diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java index 9cb2b2a4f719..b535e20660f1 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java @@ -59,11 +59,23 @@ protected void initWriter(final OutputStream outputStream) throws IOException { csvPrinter = new CSVPrinter(new PrintWriter(outputStream, true, StandardCharsets.UTF_8), csvFormat); } + /** + * TODO: (ryankfu) remove this call within {@link SerializedBufferingStrategy} and move to use + * recordString + * + * @param record AirbyteRecordMessage to be written + * @throws IOException + */ @Override protected void writeRecord(final AirbyteRecordMessage record) throws IOException { csvPrinter.printRecord(csvSheetGenerator.getDataRow(UUID.randomUUID(), record)); } + @Override + protected void writeRecord(final String recordString, final long emittedAt) throws IOException { + csvPrinter.printRecord(csvSheetGenerator.getDataRow(UUID.randomUUID(), recordString, emittedAt)); + } + @Override protected void flushWriter() throws IOException { // in an async world, it is possible that flush writer gets called even if no records were accepted. diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java index 4c174eb80313..72512c26a717 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java @@ -18,10 +18,14 @@ public interface CsvSheetGenerator { List getHeaderRow(); + // TODO: (ryankfu) remove this and switch over all destinations to pass in serialized recordStrings, + // both for performance and lowers memory footprint List getDataRow(UUID id, AirbyteRecordMessage recordMessage); List getDataRow(JsonNode formattedData); + List getDataRow(UUID id, String formattedString, long emittedAt); + final class Factory { public static CsvSheetGenerator create(final JsonNode jsonSchema, final S3CsvFormatConfig formatConfig) { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java index 8fddaf2a2179..2d86f26d55f3 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java @@ -22,7 +22,7 @@ public class RootLevelFlatteningSheetGenerator extends BaseSheetGenerator implem public RootLevelFlatteningSheetGenerator(final JsonNode jsonSchema) { this.recordHeaders = MoreIterators.toList(jsonSchema.get("properties").fieldNames()) - .stream().sorted().collect(Collectors.toList());; + .stream().sorted().collect(Collectors.toList()); } @Override diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java index 8dbfc4bedc7e..60f34128358d 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.Timestamp; import java.time.Instant; @@ -21,27 +22,31 @@ *

* This intentionally does not extend {@link BaseSheetGenerator}, because it needs the columns in a * different order (ABID, JSON, timestamp) vs (ABID, timestamp, JSON) + *

+ * In 1s1t mode, the column ordering is also different (raw_id, extracted_at, loaded_at, data). Note + * that the loaded_at column is rendered as an empty string; callers are expected to configure their + * destination to parse this as NULL. For example, Snowflake's COPY into command accepts a NULL_IF + * parameter, and Redshift accepts an EMPTYASNULL option. */ public class StagingDatabaseCsvSheetGenerator implements CsvSheetGenerator { - /** - * This method is implemented for clarity, but not actually used. S3StreamCopier disables headers on - * S3CsvWriter. - */ + private final boolean use1s1t; + private final List header; + + public StagingDatabaseCsvSheetGenerator() { + use1s1t = TypingAndDedupingFlag.isDestinationV2(); + this.header = use1s1t ? JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES : JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; + } + + // TODO is this even used anywhere? @Override public List getHeaderRow() { - return List.of( - JavaBaseConstants.COLUMN_NAME_AB_ID, - JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + return header; } @Override public List getDataRow(final UUID id, final AirbyteRecordMessage recordMessage) { - return List.of( - id, - Jsons.serialize(recordMessage.getData()), - Timestamp.from(Instant.ofEpochMilli(recordMessage.getEmittedAt()))); + return getDataRow(id, Jsons.serialize(recordMessage.getData()), recordMessage.getEmittedAt()); } @Override @@ -49,4 +54,20 @@ public List getDataRow(final JsonNode formattedData) { return new LinkedList<>(Collections.singletonList(Jsons.serialize(formattedData))); } + @Override + public List getDataRow(final UUID id, final String formattedString, final long emittedAt) { + if (use1s1t) { + return List.of( + id, + Timestamp.from(Instant.ofEpochMilli(emittedAt)), + "", + formattedString); + } else { + return List.of( + id, + formattedString, + Timestamp.from(Instant.ofEpochMilli(emittedAt))); + } + } + } diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java index 8aa8a9d78b8d..b2e05d704abd 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java @@ -53,7 +53,7 @@ protected void writeRecord(final AirbyteRecordMessage record) { json.put(JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString()); json.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, record.getEmittedAt()); if (flattenData) { - Map data = MAPPER.convertValue(record.getData(), new TypeReference<>() {}); + final Map data = MAPPER.convertValue(record.getData(), new TypeReference<>() {}); json.setAll(data); } else { json.set(JavaBaseConstants.COLUMN_NAME_DATA, record.getData()); diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java index 746c279fc9ea..67896c2de35c 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java @@ -38,7 +38,7 @@ * The {@link io.airbyte.integrations.destination.record_buffer.BaseSerializedBuffer} class * abstracts the {@link io.airbyte.integrations.destination.record_buffer.BufferStorage} from the * details of the format the data is going to be stored in. - * + *

* Unfortunately, the Parquet library doesn't allow us to manipulate the output stream and forces us * to go through {@link HadoopOutputFile} instead. So we can't benefit from the abstraction * described above. Therefore, we re-implement the necessary methods to be used as @@ -72,7 +72,7 @@ public ParquetSerializedBuffer(final S3DestinationConfig config, Files.deleteIfExists(bufferFile); avroRecordFactory = new AvroRecordFactory(schema, AvroConstants.JSON_CONVERTER); final S3ParquetFormatConfig formatConfig = (S3ParquetFormatConfig) config.getFormatConfig(); - Configuration avroConfig = new Configuration(); + final Configuration avroConfig = new Configuration(); avroConfig.setBoolean(WRITE_OLD_LIST_STRUCTURE, false); parquetWriter = AvroParquetWriter.builder(HadoopOutputFile .fromPath(new org.apache.hadoop.fs.Path(bufferFile.toUri()), avroConfig)) @@ -101,6 +101,11 @@ public long accept(final AirbyteRecordMessage record) throws Exception { } } + @Override + public long accept(final String recordString, final long emittedAt) throws Exception { + throw new UnsupportedOperationException("This method is not supported for ParquetSerializedBuffer"); + } + @Override public void flush() throws Exception { if (inputStream == null && !isClosed) { diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java index 2bc57eedfcda..334fae868442 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.record_buffer.BufferStorage; import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.destination.record_buffer.InMemoryBuffer; @@ -27,6 +28,7 @@ import org.apache.avro.file.SeekableByteArrayInput; import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumReader; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; public class AvroSerializedBufferTest { @@ -49,6 +51,11 @@ public class AvroSerializedBufferTest { Field.of("nested_column", JsonSchemaType.OBJECT)); private static final ConfiguredAirbyteCatalog catalog = CatalogHelpers.createConfiguredAirbyteCatalog(STREAM, null, FIELDS); + @BeforeAll + public static void setup() { + DestinationConfig.initialize(Jsons.deserialize("{}")); + } + @Test public void testSnappyAvroWriter() throws Exception { final S3AvroFormatConfig config = new S3AvroFormatConfig(Jsons.jsonNode(Map.of("compression_codec", Map.of( diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java index 38fefce36d5c..3fa5f1c78497 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.record_buffer.BufferStorage; import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.destination.record_buffer.InMemoryBuffer; @@ -31,6 +32,7 @@ import java.util.UUID; import java.util.zip.GZIPInputStream; import org.apache.commons.csv.CSVFormat; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class CsvSerializedBufferTest { @@ -55,6 +57,11 @@ public class CsvSerializedBufferTest { private static final String CSV_FILE_EXTENSION = ".csv"; private static final CSVFormat csvFormat = CSVFormat.newFormat(','); + @BeforeEach + public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + } + @Test public void testUncompressedDefaultCsvFormatWriter() throws Exception { runTest(new InMemoryBuffer(CSV_FILE_EXTENSION), CSVFormat.DEFAULT, false, 350L, 365L, null, diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java index 38a680f4fe6a..48a08a6e4feb 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java @@ -22,6 +22,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.integrations.destination.s3.csv.S3CsvWriter.Builder; import io.airbyte.integrations.destination.s3.util.CompressionType; @@ -246,6 +248,7 @@ public void writesContentsCorrectly_when_headerDisabled() throws IOException { */ @Test public void writesContentsCorrectly_when_stagingDatabaseConfig() throws IOException { + DestinationConfig.initialize(Jsons.emptyObject()); final S3DestinationConfig s3Config = S3DestinationConfig.create( "fake-bucket", "fake-bucketPath", diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java index 2f2911cb51ac..a9ca83f66389 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java +++ b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java @@ -11,6 +11,7 @@ import com.amazonaws.util.IOUtils; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.Field; @@ -33,6 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.hadoop.ParquetReader; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; public class ParquetSerializedBufferTest { @@ -60,6 +62,11 @@ public class ParquetSerializedBufferTest { Field.of("datetime_with_timezone", JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE)); private static final ConfiguredAirbyteCatalog catalog = CatalogHelpers.createConfiguredAirbyteCatalog(STREAM, null, FIELDS); + @BeforeAll + public static void setup() { + DestinationConfig.initialize(Jsons.deserialize("{}")); + } + @Test public void testUncompressedParquetWriter() throws Exception { final S3DestinationConfig config = S3DestinationConfig.getS3DestinationConfig(Jsons.jsonNode(Map.of( diff --git a/airbyte-integrations/bases/base-java/run_with_normalization.sh b/airbyte-integrations/bases/base-java/run_with_normalization.sh index 4f59898e9268..f61cfea63b9a 100755 --- a/airbyte-integrations/bases/base-java/run_with_normalization.sh +++ b/airbyte-integrations/bases/base-java/run_with_normalization.sh @@ -6,10 +6,34 @@ set -o pipefail destination_exit_code=$? echo '{"type": "LOG","log":{"level":"INFO","message":"Destination process done (exit code '"$destination_exit_code"')"}}' +# store original args +args=$@ + +while [ $# -ne 0 ]; do + case "$1" in + --config) + CONFIG_FILE="$2" + shift 2 + ;; + *) + # move on + shift + ;; + esac +done + +# restore original args after shifts +set -- $args + +USE_1S1T_FORMAT="false" +if [[ -s "$CONFIG_FILE" ]]; then + USE_1S1T_FORMAT=$(jq -r '.use_1s1t_format' "$CONFIG_FILE") +fi + if test "$1" != 'write' then normalization_exit_code=0 -elif test "$NORMALIZATION_TECHNIQUE" = 'LEGACY' +elif test "$NORMALIZATION_TECHNIQUE" = 'LEGACY' && test "$USE_1S1T_FORMAT" != "true" then echo '{"type": "LOG","log":{"level":"INFO","message":"Starting in-connector normalization"}}' # the args in a write command are `write --catalog foo.json --config bar.json` diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java index 5a2403a6d981..d2e7cfc0b344 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java @@ -37,7 +37,8 @@ AirbyteMessageConsumer getConsumer(JsonNode config, /** * Default implementation allows us to not have to touch existing destinations while avoiding a lot - * of conditional statements in {@link IntegrationRunner}. + * of conditional statements in {@link IntegrationRunner}. This is preferred over #getConsumer and + * is the default Async Framework method. * * @param config config * @param catalog catalog diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/DestinationConfig.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/DestinationConfig.java new file mode 100644 index 000000000000..bd27cb2255f3 --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/DestinationConfig.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import jakarta.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Singleton of destination config for easy lookup of values. + */ +@Singleton +public class DestinationConfig { + + private static final Logger LOGGER = LoggerFactory.getLogger(DestinationConfig.class); + + private static DestinationConfig config; + + @VisibleForTesting + protected JsonNode root; + + private DestinationConfig() {} + + public static void initialize(final JsonNode root) { + if (config == null) { + if (root == null) { + throw new IllegalArgumentException("Cannot create DestinationConfig from null."); + } + config = new DestinationConfig(); + config.root = root; + } else { + LOGGER.warn("Singleton was already initialized."); + } + } + + public static DestinationConfig getInstance() { + if (config == null) { + throw new IllegalStateException("Singleton not initialized."); + } + return config; + } + + public JsonNode getNodeValue(final String key) { + final JsonNode node = config.root.get(key); + if (node == null) { + LOGGER.debug("Cannot find node with key {} ", key); + } + return node; + } + + // string value, otherwise empty string + public String getTextValue(final String key) { + final JsonNode node = getNodeValue(key); + if (node == null || !node.isTextual()) { + LOGGER.debug("Cannot retrieve text value for node with key {}", key); + return ""; + } + return node.asText(); + } + + // boolean value, otherwise false + public Boolean getBooleanValue(final String key) { + final JsonNode node = getNodeValue(key); + if (node == null || !node.isBoolean()) { + LOGGER.debug("Cannot retrieve boolean value for node with key {}", key); + return false; + } + return node.asBoolean(); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java index 3fba41ef0f54..f35a78407842 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java @@ -7,14 +7,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import datadog.trace.api.Trace; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.lang.Exceptions.Procedure; +import io.airbyte.commons.stream.StreamStatusUtils; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.integrations.util.ApmTraceUtils; import io.airbyte.integrations.util.ConnectorExceptionUtil; +import io.airbyte.integrations.util.concurrent.ConcurrentStreamConsumer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; @@ -24,6 +28,7 @@ import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; @@ -31,6 +36,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; import org.apache.commons.lang3.ThreadUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.slf4j.Logger; @@ -45,16 +52,32 @@ public class IntegrationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationRunner.class); + /** + * Filters threads that should not be considered when looking for orphaned threads at shutdown of + * the integration runner. + *

+ *

+ * N.B. Daemon threads don't block the JVM if the main `currentThread` exits, so they are not + * problematic. Additionally, ignore database connection pool threads, which stay active so long as + * the database connection pool is open. + */ + @VisibleForTesting + static final Predicate ORPHANED_THREAD_FILTER = runningThread -> !runningThread.getName().equals(Thread.currentThread().getName()) + && !runningThread.isDaemon(); + public static final int INTERRUPT_THREAD_DELAY_MINUTES = 60; public static final int EXIT_THREAD_DELAY_MINUTES = 70; public static final int FORCED_EXIT_CODE = 2; + private static final Runnable EXIT_HOOK = () -> System.exit(FORCED_EXIT_CODE); + private final IntegrationCliParser cliParser; private final Consumer outputRecordCollector; private final Integration integration; private final Destination destination; private final Source source; + private final FeatureFlags featureFlags; private static JsonSchemaValidator validator; public IntegrationRunner(final Destination destination) { @@ -77,6 +100,7 @@ public IntegrationRunner(final Source source) { integration = source != null ? source : destination; this.source = source; this.destination = destination; + this.featureFlags = new EnvVariableFeatureFlags(); validator = new JsonSchemaValidator(); Thread.setDefaultUncaughtExceptionHandler(new AirbyteExceptionHandler()); @@ -136,29 +160,36 @@ private void runInternal(final IntegrationConfig parsed) throws Exception { validateConfig(integration.spec().getConnectionSpecification(), config, "READ"); final ConfiguredAirbyteCatalog catalog = parseConfig(parsed.getCatalogPath(), ConfiguredAirbyteCatalog.class); final Optional stateOptional = parsed.getStatePath().map(IntegrationRunner::parseConfig); - try (final AutoCloseableIterator messageIterator = source.read(config, catalog, stateOptional.orElse(null))) { - produceMessages(messageIterator); + try { + if (featureFlags.concurrentSourceStreamRead()) { + LOGGER.info("Concurrent source stream read enabled."); + readConcurrent(config, catalog, stateOptional); + } else { + readSerial(config, catalog, stateOptional); + } + } finally { + if (source instanceof AutoCloseable) { + ((AutoCloseable) source).close(); + } } } // destination only case WRITE -> { final JsonNode config = parseConfig(parsed.getConfigPath()); validateConfig(integration.spec().getConnectionSpecification(), config, "WRITE"); + // save config to singleton + DestinationConfig.initialize(config); final ConfiguredAirbyteCatalog catalog = parseConfig(parsed.getCatalogPath(), ConfiguredAirbyteCatalog.class); - final Procedure consumeWriteStreamCallable = () -> { - try (final SerializedAirbyteMessageConsumer consumer = destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector)) { - consumeWriteStream(consumer); - } - }; - - watchForOrphanThreads( - consumeWriteStreamCallable, - () -> System.exit(FORCED_EXIT_CODE), - INTERRUPT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES, - EXIT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES); + try (final SerializedAirbyteMessageConsumer consumer = destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector)) { + consumeWriteStream(consumer); + } finally { + stopOrphanedThreads(EXIT_HOOK, + INTERRUPT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES, + EXIT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES); + } } default -> throw new IllegalStateException("Unexpected value: " + parsed.getCommand()); } @@ -195,14 +226,66 @@ private void runInternal(final IntegrationConfig parsed) throws Exception { LOGGER.info("Completed integration: {}", integration.getClass().getName()); } - private void produceMessages(final AutoCloseableIterator messageIterator) throws Exception { - watchForOrphanThreads( - () -> messageIterator.forEachRemaining(outputRecordCollector), - () -> System.exit(FORCED_EXIT_CODE), - INTERRUPT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES, - EXIT_THREAD_DELAY_MINUTES, - TimeUnit.MINUTES); + private void produceMessages(final AutoCloseableIterator messageIterator, final Consumer recordCollector) { + messageIterator.getAirbyteStream().ifPresent(s -> LOGGER.debug("Producing messages for stream {}...", s)); + messageIterator.forEachRemaining(recordCollector); + messageIterator.getAirbyteStream().ifPresent(s -> LOGGER.debug("Finished producing messages for stream {}...")); + } + + private void readConcurrent(final JsonNode config, ConfiguredAirbyteCatalog catalog, final Optional stateOptional) throws Exception { + final Collection> streams = source.readStreams(config, catalog, stateOptional.orElse(null)); + + try (final ConcurrentStreamConsumer streamConsumer = new ConcurrentStreamConsumer(this::consumeFromStream, streams.size())) { + /* + * Break the streams into partitions equal to the number of concurrent streams supported by the + * stream consumer. + */ + final Integer partitionSize = streamConsumer.getParallelism(); + final List>> partitions = Lists.partition(streams.stream().toList(), + partitionSize); + + // Submit each stream partition for concurrent execution + partitions.forEach(partition -> { + streamConsumer.accept(partition); + }); + + // Check for any exceptions that were raised during the concurrent execution + if (streamConsumer.getException().isPresent()) { + throw streamConsumer.getException().get(); + } + } catch (final Exception e) { + LOGGER.error("Unable to perform concurrent read.", e); + throw e; + } finally { + stopOrphanedThreads(EXIT_HOOK, + INTERRUPT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES, + EXIT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES); + } + } + + private void readSerial(final JsonNode config, ConfiguredAirbyteCatalog catalog, final Optional stateOptional) throws Exception { + try (final AutoCloseableIterator messageIterator = source.read(config, catalog, stateOptional.orElse(null))) { + produceMessages(messageIterator, outputRecordCollector); + } finally { + stopOrphanedThreads(EXIT_HOOK, + INTERRUPT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES, + EXIT_THREAD_DELAY_MINUTES, + TimeUnit.MINUTES); + } + } + + private void consumeFromStream(final AutoCloseableIterator stream) { + try { + final Consumer streamStatusTrackingRecordConsumer = StreamStatusUtils.statusTrackingRecordCollector(stream, + outputRecordCollector, Optional.of(AirbyteTraceMessageUtility::emitStreamStatusTrace)); + produceMessages(stream, streamStatusTrackingRecordConsumer); + } catch (final Exception e) { + stream.getAirbyteStream().ifPresent(s -> LOGGER.error("Failed to consume from stream {}.", s, e)); + throw new RuntimeException(e); + } } @VisibleForTesting @@ -247,60 +330,58 @@ static void consumeWriteStream(final SerializedAirbyteMessageConsumer consumer, } /** - * This method calls a runMethod and make sure that it won't produce orphan non-daemon active - * threads once it is done. Active non-daemon threads blocks JVM from exiting when the main thread - * is done, whereas daemon ones don't. + * Stops any non-daemon threads that could block the JVM from exiting when the main thread is done. *

* If any active non-daemon threads would be left as orphans, this method will schedule some * interrupt/exit hooks after giving it some time delay to close up properly. It is generally * preferred to have a proper closing sequence from children threads instead of interrupting or * force exiting the process, so this mechanism serve as a fallback while surfacing warnings in logs * for maintainers to fix the code behavior instead. + * + * @param exitHook The {@link Runnable} exit hook to execute for any orphaned threads. + * @param interruptTimeDelay The time to delay execution of the orphaned thread interrupt attempt. + * @param interruptTimeUnit The time unit of the interrupt delay. + * @param exitTimeDelay The time to delay execution of the orphaned thread exit hook. + * @param exitTimeUnit The time unit of the exit delay. */ @VisibleForTesting - static void watchForOrphanThreads(final Procedure runMethod, - final Runnable exitHook, - final int interruptTimeDelay, - final TimeUnit interruptTimeUnit, - final int exitTimeDelay, - final TimeUnit exitTimeUnit) - throws Exception { + static void stopOrphanedThreads(final Runnable exitHook, + final int interruptTimeDelay, + final TimeUnit interruptTimeUnit, + final int exitTimeDelay, + final TimeUnit exitTimeUnit) { final Thread currentThread = Thread.currentThread(); - try { - runMethod.call(); - } finally { - final List runningThreads = ThreadUtils.getAllThreads() - .stream() - // daemon threads don't block the JVM if the main `currentThread` exits, so they are not problematic - .filter(runningThread -> !runningThread.getName().equals(currentThread.getName()) && !runningThread.isDaemon()) - .toList(); - if (!runningThreads.isEmpty()) { - LOGGER.warn(""" - The main thread is exiting while children non-daemon threads from a connector are still active. - Ideally, this situation should not happen... - Please check with maintainers if the connector or library code should safely clean up its threads before quitting instead. - The main thread is: {}""", dumpThread(currentThread)); - final ScheduledExecutorService scheduledExecutorService = Executors - .newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder() - // this thread executor will create daemon threads, so it does not block exiting if all other active - // threads are already stopped. - .daemon(true).build()); - for (final Thread runningThread : runningThreads) { - final String str = "Active non-daemon thread: " + dumpThread(runningThread); - LOGGER.warn(str); - // even though the main thread is already shutting down, we still leave some chances to the children - // threads to close properly on their own. - // So, we schedule an interrupt hook after a fixed time delay instead... - scheduledExecutorService.schedule(runningThread::interrupt, interruptTimeDelay, interruptTimeUnit); - } - scheduledExecutorService.schedule(() -> { - if (ThreadUtils.getAllThreads().stream() - .anyMatch(runningThread -> !runningThread.isDaemon() && !runningThread.getName().equals(currentThread.getName()))) { - LOGGER.error("Failed to interrupt children non-daemon threads, forcefully exiting NOW...\n"); - exitHook.run(); - } - }, exitTimeDelay, exitTimeUnit); + + final List runningThreads = ThreadUtils.getAllThreads() + .stream() + .filter(ORPHANED_THREAD_FILTER) + .collect(Collectors.toList()); + if (!runningThreads.isEmpty()) { + LOGGER.warn(""" + The main thread is exiting while children non-daemon threads from a connector are still active. + Ideally, this situation should not happen... + Please check with maintainers if the connector or library code should safely clean up its threads before quitting instead. + The main thread is: {}""", dumpThread(currentThread)); + final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder() + // this thread executor will create daemon threads, so it does not block exiting if all other active + // threads are already stopped. + .daemon(true).build()); + for (final Thread runningThread : runningThreads) { + final String str = "Active non-daemon thread: " + dumpThread(runningThread); + LOGGER.warn(str); + // even though the main thread is already shutting down, we still leave some chances to the children + // threads to close properly on their own. + // So, we schedule an interrupt hook after a fixed time delay instead... + scheduledExecutorService.schedule(runningThread::interrupt, interruptTimeDelay, interruptTimeUnit); } + scheduledExecutorService.schedule(() -> { + if (ThreadUtils.getAllThreads().stream() + .anyMatch(runningThread -> !runningThread.isDaemon() && !runningThread.getName().equals(currentThread.getName()))) { + LOGGER.error("Failed to interrupt children non-daemon threads, forcefully exiting NOW...\n"); + exitHook.run(); + } + }, exitTimeDelay, exitTimeUnit); } } @@ -317,7 +398,7 @@ private static void validateConfig(final JsonNode schemaJson, final JsonNode obj } } - private static JsonNode parseConfig(final Path path) { + public static JsonNode parseConfig(final Path path) { return Jsons.deserialize(IOs.readFile(path)); } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java index 4b3a4896dc4a..0c5672becbda 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.base; +import java.util.List; + public final class JavaBaseConstants { private JavaBaseConstants() {} @@ -19,5 +21,26 @@ private JavaBaseConstants() {} public static final String COLUMN_NAME_AB_ID = "_airbyte_ab_id"; public static final String COLUMN_NAME_EMITTED_AT = "_airbyte_emitted_at"; public static final String COLUMN_NAME_DATA = "_airbyte_data"; + public static final List LEGACY_RAW_TABLE_COLUMNS = List.of( + COLUMN_NAME_AB_ID, + COLUMN_NAME_DATA, + COLUMN_NAME_EMITTED_AT); + + // destination v2 + public static final String COLUMN_NAME_AB_RAW_ID = "_airbyte_raw_id"; + public static final String COLUMN_NAME_AB_LOADED_AT = "_airbyte_loaded_at"; + public static final String COLUMN_NAME_AB_EXTRACTED_AT = "_airbyte_extracted_at"; + public static final String COLUMN_NAME_AB_META = "_airbyte_meta"; + public static final List V2_RAW_TABLE_COLUMN_NAMES = List.of( + COLUMN_NAME_AB_RAW_ID, + COLUMN_NAME_AB_EXTRACTED_AT, + COLUMN_NAME_AB_LOADED_AT, + COLUMN_NAME_DATA); + public static final List V2_FINAL_TABLE_METADATA_COLUMNS = List.of( + COLUMN_NAME_AB_RAW_ID, + COLUMN_NAME_AB_EXTRACTED_AT, + COLUMN_NAME_AB_META); + + public static final String DEFAULT_AIRBYTE_INTERNAL_NAMESPACE = "airbyte_internal"; } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java index dba2770335f1..69b866252328 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.base; +import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.functional.CheckedBiConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -53,4 +54,29 @@ public interface SerializedAirbyteMessageConsumer extends CheckedBiConsumer read(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) throws Exception; + /** + * Returns a collection of iterators of messages pulled from the source, each representing a + * "stream". + * + * @param config - integration-specific configuration object as json. e.g. { "username": "airbyte", + * "password": "super secure" } + * @param catalog - schema of the incoming messages. + * @param state - state of the incoming messages. + * @return The collection of {@link AutoCloseableIterator} instances that produce messages for each + * configured "stream" + * @throws Exception - any exception + */ + default Collection> readStreams(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) + throws Exception { + return List.of(read(config, catalog, state)); + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/TypingAndDedupingFlag.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/TypingAndDedupingFlag.java new file mode 100644 index 000000000000..aea71ee4006d --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/TypingAndDedupingFlag.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base; + +import java.util.Optional; +import org.elasticsearch.common.Strings; + +public class TypingAndDedupingFlag { + + public static boolean isDestinationV2() { + return DestinationConfig.getInstance().getBooleanValue("use_1s1t_format"); + } + + public static Optional getRawNamespaceOverride(String option) { + String rawOverride = DestinationConfig.getInstance().getTextValue(option); + if (Strings.isEmpty(rawOverride)) { + return Optional.empty(); + } else { + return Optional.of(rawOverride); + } + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java index a6b512537e28..72528ce06ce4 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java @@ -4,7 +4,12 @@ package io.airbyte.integrations.base.adaptive; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.Command; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.DestinationConfig; +import io.airbyte.integrations.base.IntegrationCliParser; +import io.airbyte.integrations.base.IntegrationConfig; import io.airbyte.integrations.base.IntegrationRunner; import java.util.function.Supplier; import org.slf4j.Logger; @@ -82,6 +87,15 @@ private Destination getDestination() { } public void run(final String[] args) throws Exception { + // getDestination() sometimes depends on the singleton being initialized. + // Parse the CLI args just so we can accomplish that. + IntegrationConfig parsedArgs = new IntegrationCliParser().parse(args); + if (parsedArgs.getCommand() != Command.SPEC) { + DestinationConfig.initialize(IntegrationRunner.parseConfig(parsedArgs.getConfigPath())); + } else { + DestinationConfig.initialize(Jsons.emptyObject()); + } + final Destination destination = getDestination(); LOGGER.info("Starting destination: {}", destination.getClass().getName()); new IntegrationRunner(destination).run(args); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java index f7cfef4df5af..a0e26a5bcc9c 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java @@ -12,6 +12,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; +import java.util.Collection; /** * In some cases we want to prune or mutate the spec for an existing source. The common case is that @@ -49,4 +50,10 @@ public AutoCloseableIterator read(final JsonNode config, final C return source.read(config, catalog, state); } + @Override + public Collection> readStreams(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) + throws Exception { + return source.readStreams(config, catalog, state); + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java index 954bd58d4c8f..0f8d6a650373 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java @@ -11,6 +11,7 @@ import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -79,7 +80,7 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) throws Exception { - final SshTunnel tunnel = (endPointKey != null) ? SshTunnel.getInstance(config, endPointKey) : SshTunnel.getInstance(config, hostKey, portKey); + final SshTunnel tunnel = getTunnelInstance(config); final AirbyteMessageConsumer delegateConsumer; try { @@ -92,4 +93,27 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, return AirbyteMessageConsumer.appendOnClose(delegateConsumer, tunnel::close); } + @Override + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) + throws Exception { + final SshTunnel tunnel = getTunnelInstance(config); + final SerializedAirbyteMessageConsumer delegateConsumer; + try { + delegateConsumer = delegate.getSerializedMessageConsumer(tunnel.getConfigInTunnel(), catalog, outputRecordCollector); + } catch (final Exception e) { + LOGGER.error("Exception occurred while getting the delegate consumer, closing SSH tunnel", e); + tunnel.close(); + throw e; + } + return SerializedAirbyteMessageConsumer.appendOnClose(delegateConsumer, tunnel::close); + } + + protected SshTunnel getTunnelInstance(final JsonNode config) throws Exception { + return (endPointKey != null) + ? SshTunnel.getInstance(config, endPointKey) + : SshTunnel.getInstance(config, hostKey, portKey); + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java index bb3b7de21fe2..08971e9ec768 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java @@ -15,6 +15,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; +import java.util.Collection; import java.util.List; import java.util.Optional; import org.slf4j.Logger; @@ -80,4 +81,17 @@ public AutoCloseableIterator read(final JsonNode config, final C return AutoCloseableIterators.appendOnClose(delegateRead, tunnel::close); } + @Override + public Collection> readStreams(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) + throws Exception { + final SshTunnel tunnel = SshTunnel.getInstance(config, hostKey, portKey); + try { + return delegate.readStreams(tunnel.getConfigInTunnel(), catalog, state); + } catch (final Exception e) { + LOGGER.error("Exception occurred while getting the delegate read stream iterators, closing SSH tunnel", e); + tunnel.close(); + throw e; + } + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md index 24588f205704..749b2df491a0 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md @@ -31,7 +31,7 @@ Our SSH connector support is designed to be easy to plug into any existing conne Replace port_key and host_key as necessary. Look at `transform_postgres()` to see an example. 2. To make sure your changes are present in Normalization when running tests on the connector locally, you'll need to change [this version tag](https://github.com/airbytehq/airbyte/blob/6d9ba022646441c7f298ca4dcaa3df59b9a19fbb/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java#L50) to `dev` so that the new locally built docker image for Normalization is used. Don't push this change with the PR though. 3. If your `host_key="host"` and `port_key="port"` then this step is not necessary. However if the key names differ for your connector, you will also need to add some logic into `sshtunneling.sh` (within airbyte-workers) to handle this, as currently it assumes that the keys are exactly `host` and `port`. -4. When making your PR, make sure that you've version bumped Normalization (in `airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java` and `airbyte-integrations/bases/base-normalization/Dockerfile`). You'll need to /test & /legacy-publish Normalization _first_ so that when you /test the connector, it can use the new version. +4. When making your PR, make sure that you've version bumped Normalization (in `airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java` and `airbyte-integrations/bases/base-normalization/Dockerfile`). You'll need to /legacy-test & /legacy-publish Normalization _first_ so that when you the connector is tested, it can use the new version. ## Misc diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java index efdbd2a019cd..8c1da914938f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.AirbyteMessageConsumer; @@ -92,7 +93,13 @@ public class BufferedStreamConsumer extends FailureTrackingAirbyteMessageConsume private Instant nextFlushDeadline; private final Duration bufferFlushFrequency; + private final String defaultNamespace; + /** + * Feel free to continue using this in non-1s1t destinations - it may be easier to use. However, + * 1s1t destinations should prefer the version which accepts a {@code defaultNamespace}. + */ + @Deprecated public BufferedStreamConsumer(final Consumer outputRecordCollector, final OnStartFunction onStart, final BufferingStrategy bufferingStrategy, @@ -105,7 +112,27 @@ public BufferedStreamConsumer(final Consumer outputRecordCollect onClose, catalog, isValidRecord, - Duration.ofMinutes(15)); + Duration.ofMinutes(15), + // This is purely for backwards compatibility. Many older destinations handle this internally. + // Starting with Destinations V2, we recommend passing in an explicit namespace. + null); + } + + public BufferedStreamConsumer(final Consumer outputRecordCollector, + final OnStartFunction onStart, + final BufferingStrategy bufferingStrategy, + final OnCloseFunction onClose, + final ConfiguredAirbyteCatalog catalog, + final CheckedFunction isValidRecord, + final String defaultNamespace) { + this(outputRecordCollector, + onStart, + bufferingStrategy, + onClose, + catalog, + isValidRecord, + Duration.ofMinutes(15), + defaultNamespace); } /* @@ -119,7 +146,8 @@ public BufferedStreamConsumer(final Consumer outputRecordCollect final OnCloseFunction onClose, final ConfiguredAirbyteCatalog catalog, final CheckedFunction isValidRecord, - final Duration flushFrequency) { + final Duration flushFrequency, + final String defaultNamespace) { this.outputRecordCollector = outputRecordCollector; this.hasStarted = false; this.hasClosed = false; @@ -130,8 +158,9 @@ public BufferedStreamConsumer(final Consumer outputRecordCollect this.isValidRecord = isValidRecord; this.streamToIgnoredRecordCount = new HashMap<>(); this.bufferingStrategy = bufferingStrategy; - this.stateManager = new DefaultDestStateLifecycleManager(); + this.stateManager = new DefaultDestStateLifecycleManager(defaultNamespace); this.bufferFlushFrequency = flushFrequency; + this.defaultNamespace = defaultNamespace; } @Override @@ -157,7 +186,11 @@ protected void acceptTracked(final AirbyteMessage message) throws Exception { Preconditions.checkState(hasStarted, "Cannot accept records until consumer has started"); if (message.getType() == Type.RECORD) { final AirbyteRecordMessage record = message.getRecord(); - final AirbyteStreamNameNamespacePair stream = AirbyteStreamNameNamespacePair.fromRecordMessage(record); + if (Strings.isNullOrEmpty(record.getNamespace())) { + record.setNamespace(defaultNamespace); + } + final AirbyteStreamNameNamespacePair stream; + stream = AirbyteStreamNameNamespacePair.fromRecordMessage(record); // if stream is not part of list of streams to sync to then throw invalid stream exception if (!streamNames.contains(stream)) { @@ -177,7 +210,7 @@ protected void acceptTracked(final AirbyteMessage message) throws Exception { } else if (BufferFlushType.FLUSH_SINGLE_STREAM.equals(flushType.get())) { if (stateManager.supportsPerStreamFlush()) { // per-stream instance can handle flush of just a single stream - markStatesAsFlushedToDestination(); + markStatesAsFlushedToDestination(stream); } /* * We don't mark {@link AirbyteStateMessage} as committed in the case with GLOBAL/LEGACY because @@ -207,6 +240,13 @@ private void markStatesAsFlushedToDestination() { nextFlushDeadline = Instant.now().plus(bufferFlushFrequency); } + private void markStatesAsFlushedToDestination(final AirbyteStreamNameNamespacePair stream) { + stateManager.markPendingAsCommitted(stream); + stateManager.listCommitted().forEach(outputRecordCollector); + stateManager.clearCommitted(); + nextFlushDeadline = Instant.now().plus(bufferFlushFrequency); + } + /** * Periodically flushes buffered data to destination storage when exceeding flush deadline. Also * resets the last time a flush occurred diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java index f5a85f322781..cbde8d7a497c 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java @@ -9,6 +9,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Queue; import java.util.function.Supplier; @@ -33,8 +34,8 @@ public class DefaultDestStateLifecycleManager implements DestStateLifecycleManag private AirbyteStateType stateType; private final Supplier internalStateManagerSupplier; - public DefaultDestStateLifecycleManager() { - this(new DestSingleStateLifecycleManager(), new DestStreamStateLifecycleManager()); + public DefaultDestStateLifecycleManager(final String defaultNamespace) { + this(new DestSingleStateLifecycleManager(), new DestStreamStateLifecycleManager(defaultNamespace)); } @VisibleForTesting @@ -115,6 +116,11 @@ public void markPendingAsCommitted() { internalStateManagerSupplier.get().markPendingAsCommitted(); } + @Override + public void markPendingAsCommitted(final AirbyteStreamNameNamespacePair stream) { + internalStateManagerSupplier.get().markPendingAsCommitted(stream); + } + @Override public void clearCommitted() { internalStateManagerSupplier.get().clearCommitted(); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java index b5fbadef03cc..acc64d35a88a 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java @@ -6,6 +6,7 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -71,6 +72,12 @@ public void markPendingAsCommitted() { } } + @Override + public void markPendingAsCommitted(final AirbyteStreamNameNamespacePair stream) { + // We declare supportsPerStreamFlush as false, so this method should never be called. + throw new IllegalStateException("Committing a single stream state is not supported for this state type."); + } + @Override public Queue listCommitted() { return stateMessageToQueue(lastCommittedState); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java index ebe9b4516408..503c790f6a43 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.dest_state_lifecycle_manager; import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Queue; /** @@ -70,6 +71,11 @@ public interface DestStateLifecycleManager { */ void markPendingAsCommitted(); + /** + * Mark all pending states for the given stream as committed. + */ + void markPendingAsCommitted(AirbyteStreamNameNamespacePair stream); + /** * List all tracked state messages that are committed. * diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java index 3d69907af14e..797d5baa6833 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java @@ -4,10 +4,12 @@ package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +import com.amazonaws.util.StringUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.Comparator; import java.util.HashMap; @@ -34,8 +36,10 @@ public class DestStreamStateLifecycleManager implements DestStateLifecycleManage private final Map streamToLastPendingState; private final Map streamToLastFlushedState; private final Map streamToLastCommittedState; + private final String defaultNamespace; - public DestStreamStateLifecycleManager() { + public DestStreamStateLifecycleManager(final String defaultNamespace) { + this.defaultNamespace = defaultNamespace; streamToLastPendingState = new HashMap<>(); streamToLastFlushedState = new HashMap<>(); streamToLastCommittedState = new HashMap<>(); @@ -44,7 +48,20 @@ public DestStreamStateLifecycleManager() { @Override public void addState(final AirbyteMessage message) { Preconditions.checkArgument(message.getState().getType() == AirbyteStateType.STREAM); - streamToLastPendingState.put(message.getState().getStream().getStreamDescriptor(), message); + final StreamDescriptor originalStreamId = message.getState().getStream().getStreamDescriptor(); + final StreamDescriptor actualStreamId; + if (StringUtils.isNullOrEmpty(originalStreamId.getNamespace())) { + // If the state's namespace is null/empty, we need to be able to find it using the default namespace + // (because many destinations actually set records' namespace to the default namespace before + // they make it into this class). + // Clone the streamdescriptor so that we don't modify the original state message. + actualStreamId = new StreamDescriptor() + .withName(originalStreamId.getName()) + .withNamespace(defaultNamespace); + } else { + actualStreamId = originalStreamId; + } + streamToLastPendingState.put(actualStreamId, message); } @VisibleForTesting @@ -88,6 +105,21 @@ public void markPendingAsCommitted() { moveToNextPhase(streamToLastPendingState, streamToLastCommittedState); } + @Override + public void markPendingAsCommitted(final AirbyteStreamNameNamespacePair stream) { + // streamToLastCommittedState is keyed using defaultNamespace instead of namespace=null. (see + // #addState) + // Many destinations actually modify the records' namespace immediately after reading them from + // stdin, + // but we should have a null-check here just in case. + final String actualNamespace = stream.getNamespace() == null ? defaultNamespace : stream.getNamespace(); + final StreamDescriptor sd = new StreamDescriptor().withName(stream.getName()).withNamespace(actualNamespace); + final AirbyteMessage lastPendingState = streamToLastPendingState.remove(sd); + if (lastPendingState != null) { + streamToLastCommittedState.put(sd, lastPendingState); + } + } + @Override public Queue listCommitted() { return listStatesInOrder(streamToLastCommittedState); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java index 73d16162aa3f..39b2925f9ee1 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.record_buffer; import com.google.common.io.CountingOutputStream; +import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.IOException; @@ -18,7 +19,7 @@ /** * Base implementation of a {@link SerializableBuffer}. It is composed of a {@link BufferStorage} * where the actual data is being stored in a serialized format. - * + *

* Such data format is defined by concrete implementation inheriting from this base abstract class. * To do so, necessary methods on handling "writer" methods should be defined. This writer would * take care of converting {@link AirbyteRecordMessage} into the serialized form of the data such as @@ -57,8 +58,21 @@ protected BaseSerializedBuffer(final BufferStorage bufferStorage) throws Excepti * Transform the @param record into a serialized form of the data and writes it to the registered * OutputStream provided when {@link BaseSerializedBuffer#initWriter} was called. */ + @Deprecated protected abstract void writeRecord(AirbyteRecordMessage record) throws IOException; + /** + * TODO: (ryankfu) move destination to use serialized record string instead of passing entire + * AirbyteRecord + * + * @param recordString serialized record + * @param emittedAt timestamp of the record in milliseconds + * @throws IOException + */ + protected void writeRecord(final String recordString, final long emittedAt) throws IOException { + writeRecord(Jsons.deserialize(recordString, AirbyteRecordMessage.class).withEmittedAt(emittedAt)); + } + /** * Stops the writer from receiving new data and prepares it for being finalized and converted into * an InputStream to read from instead. This is used when flushing the buffer into some other @@ -96,6 +110,26 @@ public long accept(final AirbyteRecordMessage record) throws Exception { } } + @Override + public long accept(final String recordString, final long emittedAt) throws Exception { + if (!isStarted) { + if (useCompression) { + compressedBuffer = new GzipCompressorOutputStream(byteCounter); + initWriter(compressedBuffer); + } else { + initWriter(byteCounter); + } + isStarted = true; + } + if (inputStream == null && !isClosed) { + final long startCount = byteCounter.getCount(); + writeRecord(recordString, emittedAt); + return byteCounter.getCount() - startCount; + } else { + throw new IllegalCallerException("Buffer is already closed, it cannot accept more messages"); + } + } + @Override public String getFilename() throws IOException { if (useCompression && !bufferStorage.getFilename().endsWith(GZ_SUFFIX)) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java index 2762ab055ecb..1870d779b70f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java @@ -35,8 +35,20 @@ public interface SerializableBuffer extends AutoCloseable { * @param record {@link AirbyteRecordMessage} to be added to buffer * @return number of bytes written to the buffer */ + @Deprecated long accept(AirbyteRecordMessage record) throws Exception; + /** + * TODO: (ryankfu) Move all destination connectors to pass the serialized record string instead of + * the entire AirbyteRecordMessage + * + * @param recordString serialized record + * @param emittedAt timestamp of the record in milliseconds + * @return number of bytes written to the buffer + * @throws Exception + */ + long accept(String recordString, long emittedAt) throws Exception; + /** * Flush a buffer implementation. */ diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java index 600352e2759b..435a0b11a94c 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java @@ -4,10 +4,9 @@ package io.airbyte.integrations.destination_async; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; @@ -47,17 +46,25 @@ public class AsyncStreamConsumer implements SerializedAirbyteMessageConsumer { private final FlushWorkers flushWorkers; private final Set streamNames; private final FlushFailure flushFailure; + private final String defaultNamespace; private boolean hasStarted; private boolean hasClosed; + // This is to account for the references when deserialization to a PartialAirbyteMessage. The + // calculation is as follows: + // PartialAirbyteMessage (4) + Max( PartialRecordMessage(4), PartialStateMessage(6)) with + // PartialStateMessage being larger with more nested objects within it. Using 8 bytes as we assumed + // a 64 bit JVM. + final int PARTIAL_DESERIALIZE_REF_BYTES = 10 * 8; public AsyncStreamConsumer(final Consumer outputRecordCollector, final OnStartFunction onStart, final OnCloseFunction onClose, final DestinationFlushFunction flusher, final ConfiguredAirbyteCatalog catalog, - final BufferManager bufferManager) { - this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, new FlushFailure()); + final BufferManager bufferManager, + final String defaultNamespace) { + this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, new FlushFailure(), defaultNamespace); } @VisibleForTesting @@ -67,7 +74,9 @@ public AsyncStreamConsumer(final Consumer outputRecordCollector, final DestinationFlushFunction flusher, final ConfiguredAirbyteCatalog catalog, final BufferManager bufferManager, - final FlushFailure flushFailure) { + final FlushFailure flushFailure, + final String defaultNamespace) { + this.defaultNamespace = defaultNamespace; hasStarted = false; hasClosed = false; @@ -103,11 +112,13 @@ public void accept(final String messageString, final Integer sizeInBytes) throws */ deserializeAirbyteMessage(messageString) .ifPresent(message -> { - if (message.getType() == Type.RECORD) { + if (Type.RECORD.equals(message.getType())) { + if (Strings.isNullOrEmpty(message.getRecord().getNamespace())) { + message.getRecord().setNamespace(defaultNamespace); + } validateRecord(message); } - - bufferEnqueue.addRecord(message, sizeInBytes); + bufferEnqueue.addRecord(message, sizeInBytes + PARTIAL_DESERIALIZE_REF_BYTES); }); } @@ -115,55 +126,32 @@ public void accept(final String messageString, final Integer sizeInBytes) throws * Deserializes to a {@link PartialAirbyteMessage} which can represent both a Record or a State * Message * + * PartialAirbyteMessage holds either: + *

  • entire serialized message string when message is a valid State Message + *
  • serialized AirbyteRecordMessage when message is a valid Record Message
    • + * * @param messageString the string to deserialize * @return PartialAirbyteMessage if the message is valid, empty otherwise */ - private Optional deserializeAirbyteMessage(final String messageString) { + @VisibleForTesting + public static Optional deserializeAirbyteMessage(final String messageString) { + // TODO: (ryankfu) plumb in the serialized AirbyteStateMessage to match AirbyteRecordMessage code + // parity. https://github.com/airbytehq/airbyte/issues/27530 for additional context final Optional messageOptional = Jsons.tryDeserialize(messageString, PartialAirbyteMessage.class) - .map(partial -> partial.withSerialized(messageString)); + .map(partial -> { + if (Type.RECORD.equals(partial.getType()) && partial.getRecord().getData() != null) { + return partial.withSerialized(partial.getRecord().getData().toString()); + } else if (Type.STATE.equals(partial.getType())) { + return partial.withSerialized(messageString); + } else { + return null; + } + }); + if (messageOptional.isPresent()) { return messageOptional; - } else { - if (isStateMessage(messageString)) { - throw new IllegalStateException("Invalid state message: " + messageString); - } else { - LOGGER.error("Received invalid message: " + messageString); - return Optional.empty(); - } - } - } - - /** - * Tests whether the provided JSON string represents a state message. - * - * @param input a JSON string that represents an {@link AirbyteMessage}. - * @return {@code true} if the message is a state message, {@code false} otherwise. - */ - private static boolean isStateMessage(final String input) { - final Optional deserialized = Jsons.tryDeserialize(input, AirbyteTypeMessage.class); - return deserialized.filter(airbyteTypeMessage -> airbyteTypeMessage.getType() == Type.STATE).isPresent(); - } - - /** - * Custom class that can be used to parse a JSON message to determine the type of the represented - * {@link AirbyteMessage}. - */ - private static class AirbyteTypeMessage { - - @JsonProperty("type") - @JsonPropertyDescription("Message type") - private AirbyteMessage.Type type; - - @JsonProperty("type") - public AirbyteMessage.Type getType() { - return type; } - - @JsonProperty("type") - public void setType(final AirbyteMessage.Type type) { - this.type = type; - } - + throw new RuntimeException("Invalid serialized message"); } @Override diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java index 9b0cd4acceab..6d230dec56d7 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java @@ -11,6 +11,7 @@ import java.time.temporal.ChronoUnit; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -210,11 +211,18 @@ long estimateSizeOfRunningWorkers(final StreamDescriptor stream, final long curr */ @VisibleForTesting List orderStreamsByPriority(final Set streams) { + // eagerly pull attributes so that values are consistent throughout comparison + final Map> sdToQueueSize = streams.stream() + .collect(Collectors.toMap(s -> s, bufferDequeue::getQueueSizeBytes)); + + final Map> sdToTimeOfLastRecord = streams.stream() + .collect(Collectors.toMap(s -> s, bufferDequeue::getTimeOfLastRecord)); + return streams.stream() - .sorted(Comparator.comparing((StreamDescriptor s) -> bufferDequeue.getQueueSizeBytes(s).orElseThrow(), Comparator.reverseOrder()) + .sorted(Comparator.comparing((StreamDescriptor s) -> sdToQueueSize.get(s).orElseThrow(), Comparator.reverseOrder()) // if no time is present, it suggests the queue has no records. set MAX time as a sentinel value to // represent no records. - .thenComparing(s -> bufferDequeue.getTimeOfLastRecord(s).orElse(Instant.MAX)) + .thenComparing(s -> sdToTimeOfLastRecord.get(s).orElse(Instant.MAX)) .thenComparing(s -> s.getNamespace() + s.getName())) .collect(Collectors.toList()); } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java index 1376ca629e8a..94be07f6f485 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java @@ -86,6 +86,7 @@ public FlushWorkers(final BufferDequeue bufferDequeue, } public void start() { + log.info("Start async buffer supervisor"); supervisorThread.scheduleAtFixedRate(this::retrieveWork, SUPERVISOR_INITIAL_DELAY_SECS, SUPERVISOR_PERIOD_SECS, @@ -98,7 +99,9 @@ public void start() { private void retrieveWork() { try { - log.info("Retrieve Work -- Finding queues to flush"); + // This will put a new log line every second which is too much, sampling it doesn't bring much value + // so it is set to debug + log.debug("Retrieve Work -- Finding queues to flush"); final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) workerPool; int allocatableThreads = threadPoolExecutor.getMaximumPoolSize() - threadPoolExecutor.getActiveCount(); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java index 99f5de4f963e..87064d638c39 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java @@ -32,6 +32,10 @@ public void addMaxMemory(final long maxMemoryUsage) { } public Optional getTimeOfLastMessage() { + // if the queue is empty, the time of last message is irrelevant + if (size() == 0) { + return Optional.empty(); + } return Optional.ofNullable(timeOfLastMessage.get()); } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java index 9fb7e5ef9a2d..b6218de45473 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java @@ -5,8 +5,13 @@ package io.airbyte.integrations.destination_async.partial_messages; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.fasterxml.jackson.databind.JsonNode; import java.util.Objects; +// TODO: (ryankfu) remove this and test with low memory resources to ensure OOM is still not a +// factor, shouldn't be +// but weird things have happened public class PartialAirbyteRecordMessage { @JsonProperty("namespace") @@ -14,6 +19,13 @@ public class PartialAirbyteRecordMessage { @JsonProperty("stream") private String stream; + @JsonProperty("data") + private JsonNode data; + + @JsonProperty("emitted_at") + @JsonPropertyDescription("when the data was emitted from the source. epoch in millisecond.") + private long emittedAt; + public PartialAirbyteRecordMessage() {} @JsonProperty("namespace") @@ -46,6 +58,36 @@ public PartialAirbyteRecordMessage withStream(final String stream) { return this; } + @JsonProperty("data") + public JsonNode getData() { + return data; + } + + @JsonProperty("data") + public void setData(final JsonNode data) { + this.data = data; + } + + public PartialAirbyteRecordMessage withData(final JsonNode data) { + this.data = data; + return this; + } + + @JsonProperty("emitted_at") + public Long getEmittedAt() { + return this.emittedAt; + } + + @JsonProperty("emitted_at") + public void setEmittedAt(final long emittedAt) { + this.emittedAt = emittedAt; + } + + public PartialAirbyteRecordMessage withEmittedAt(final Long emittedAt) { + this.emittedAt = emittedAt; + return this; + } + @Override public boolean equals(final Object o) { if (this == o) { @@ -55,12 +97,14 @@ public boolean equals(final Object o) { return false; } final PartialAirbyteRecordMessage that = (PartialAirbyteRecordMessage) o; - return Objects.equals(namespace, that.namespace) && Objects.equals(stream, that.stream); + return Objects.equals(namespace, that.namespace) + && Objects.equals(stream, that.stream) + && Objects.equals(emittedAt, that.emittedAt); } @Override public int hashCode() { - return Objects.hash(namespace, stream); + return Objects.hash(namespace, stream, emittedAt); } @Override @@ -68,6 +112,7 @@ public String toString() { return "PartialAirbyteRecordMessage{" + "namespace='" + namespace + '\'' + ", stream='" + stream + '\'' + + ", emittedAt='" + emittedAt + '\'' + '}'; } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java index cd9a8f8fd9bb..420f892a66ef 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java @@ -145,24 +145,32 @@ public List flushStates() { final List output = new ArrayList<>(); Long bytesFlushed = 0L; for (final Map.Entry> entry : streamToStateIdQ.entrySet()) { - // remove all states with 0 counters. - final LinkedList stateIdQueue = entry.getValue(); - while (true) { - final Long oldestState = stateIdQueue.peek(); - final boolean emptyQ = oldestState == null; - final boolean noCorrespondingStateMsg = stateIdToState.get(oldestState) == null; - if (emptyQ || noCorrespondingStateMsg) { - break; - } - - final boolean noPrevRecs = !stateIdToCounter.containsKey(oldestState); - final boolean allRecsEmitted = stateIdToCounter.get(oldestState).get() == 0; - if (noPrevRecs || allRecsEmitted) { - entry.getValue().poll(); // poll to remove. no need to read as the earlier peek is still valid. - output.add(stateIdToState.get(oldestState).getLeft()); - bytesFlushed += stateIdToState.get(oldestState).getRight(); - } else { - break; + // Remove all states with 0 counters. + // Per-stream synchronized is required to make sure the state (at the head of the queue) + // logic is applied to is the state actually removed. + synchronized (this) { + final LinkedList stateIdQueue = entry.getValue(); + while (true) { + final Long oldestState = stateIdQueue.peek(); + if (oldestState == null) { + break; + } + + // technically possible this map hasn't been updated yet. + final boolean noCorrespondingStateMsg = stateIdToState.get(oldestState) == null; + if (noCorrespondingStateMsg) { + break; + } + + final boolean noPrevRecs = !stateIdToCounter.containsKey(oldestState); + final boolean allRecsEmitted = stateIdToCounter.get(oldestState).get() == 0; + if (noPrevRecs || allRecsEmitted) { + var polled = entry.getValue().poll(); // poll to remove. no need to read as the earlier peek is still valid. + output.add(stateIdToState.get(oldestState).getLeft()); + bytesFlushed += stateIdToState.get(oldestState).getRight(); + } else { + break; + } } } } @@ -173,7 +181,9 @@ public List flushStates() { private Long getStateIdAndIncrement(final StreamDescriptor streamDescriptor, final long increment) { final StreamDescriptor resolvedDescriptor = stateType == AirbyteStateMessage.AirbyteStateType.STREAM ? streamDescriptor : SENTINEL_GLOBAL_DESC; - if (!streamToStateIdQ.containsKey(resolvedDescriptor)) { + // As concurrent collections do not guarantee data consistency when iterating, use `get` instead of + // `containsKey`. + if (streamToStateIdQ.get(resolvedDescriptor) == null) { registerNewStreamDescriptor(resolvedDescriptor); } final Long stateId = streamToStateIdQ.get(resolvedDescriptor).peekLast(); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumer.java new file mode 100644 index 000000000000..7d9bdb15ead7 --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumer.java @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.util.concurrent; + +import io.airbyte.commons.stream.AirbyteStreamStatusHolder; +import io.airbyte.commons.stream.StreamStatusUtils; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor.AbortPolicy; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * {@link Consumer} implementation that consumes {@link AirbyteMessage} records from each provided + * stream concurrently. + *

    + *

    + * The consumer calculates the parallelism based on the provided requested parallelism. If the + * requested parallelism is greater than zero, the minimum value between the requested parallelism + * and the maximum number of allowed threads is chosen as the parallelism value. Otherwise, the + * minimum parallelism value is selected. This is to avoid issues with attempting to execute with a + * parallelism value of zero, which is not allowed by the underlying {@link ExecutorService}. + *

    + *

    + * This consumer will capture any raised exceptions during execution of each stream. Anu exceptions + * are stored and made available by calling the {@link #getException()} method. + */ +public class ConcurrentStreamConsumer implements Consumer>>, AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrentStreamConsumer.class); + + /** + * Name of threads spawned by the {@link ConcurrentStreamConsumer}. + */ + public static final String CONCURRENT_STREAM_THREAD_NAME = "concurrent-stream-thread"; + + private final ExecutorService executorService; + private final List exceptions; + private final Integer parallelism; + private final Consumer> streamConsumer; + private final Optional> streamStatusEmitter = + Optional.of(AirbyteTraceMessageUtility::emitStreamStatusTrace); + + /** + * Constructs a new {@link ConcurrentStreamConsumer} that will use the provided stream consumer to + * execute each stream submitted to the {@link #accept(Collection)} method of + * this consumer. Streams submitted to the {@link #accept(Collection)} method + * will be converted to a {@link Runnable} and executed on an {@link ExecutorService} configured by + * this consumer to ensure concurrent execution of each stream. + * + * @param streamConsumer The {@link Consumer} that accepts streams as an + * {@link AutoCloseableIterator}. + * @param requestedParallelism The requested amount of parallelism that will be used as a hint to + * determine the appropriate number of threads to execute concurrently. + */ + public ConcurrentStreamConsumer(final Consumer> streamConsumer, final Integer requestedParallelism) { + this.parallelism = computeParallelism(requestedParallelism); + this.executorService = createExecutorService(parallelism); + this.exceptions = new ArrayList<>(); + this.streamConsumer = streamConsumer; + } + + @Override + public void accept(final Collection> streams) { + /* + * Submit the provided streams to the underlying executor service for concurrent execution. This + * thread will track the status of each stream as well as consuming all messages produced from each + * stream, passing them to the provided message consumer for further processing. Any exceptions + * raised within the thread will be captured and exposed to the caller. + */ + final Collection> futures = streams.stream() + .map(stream -> new ConcurrentStreamRunnable(stream, this)) + .map(runnable -> CompletableFuture.runAsync(runnable, executorService)) + .collect(Collectors.toList()); + + /* + * Wait for the submitted streams to complete before returning. This uses the join() method to allow + * all streams to complete even if one or more encounters an exception. + */ + LOGGER.debug("Waiting for all streams to complete...."); + CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).join(); + LOGGER.debug("Completed consuming from all streams."); + } + + /** + * Returns the first captured {@link Exception}. + * + * @return The first captured {@link Exception} or an empty {@link Optional} if no exceptions were + * captured during execution. + */ + public Optional getException() { + if (!exceptions.isEmpty()) { + return Optional.of(exceptions.get(0)); + } else { + return Optional.empty(); + } + } + + /** + * Returns the list of exceptions captured during execution of the streams, if any. + * + * @return The collection of captured exceptions or an empty list. + */ + public List getExceptions() { + return Collections.unmodifiableList(exceptions); + } + + /** + * Returns the parallelism value that will be used by this consumer to execute the consumption of + * data from the provided streams in parallel. + * + * @return The parallelism value of this consumer. + */ + public Integer getParallelism() { + return computeParallelism(parallelism); + } + + /** + * Calculates the parallelism based on the requested parallelism. If the requested parallelism is + * greater than zero, the minimum value between the parallelism and the maximum parallelism is + * chosen as the parallelism count. Otherwise, the minimum parallelism is selected. This is to avoid + * issues with attempting to create an executor service with a thread pool size of 0, which is not + * allowed. + * + * @param requestedParallelism The requested parallelism. + * @return The selected parallelism based on the factors outlined above. + */ + private Integer computeParallelism(final Integer requestedParallelism) { + /* + * Selects the default thread pool size based on the provided value via an environment variable or + * the number of available processors if the environment variable is not set/present. This is to + * ensure that we do not over-parallelize unless requested explicitly. + */ + final Integer defaultPoolSize = Optional.ofNullable(System.getenv("DEFAULT_CONCURRENT_STREAM_CONSUMER_THREADS")) + .map(Integer::parseInt) + .orElseGet(() -> Runtime.getRuntime().availableProcessors()); + LOGGER.debug("Default parallelism: {}, Requested parallelism: {}", defaultPoolSize, requestedParallelism); + final Integer parallelism = Math.min(defaultPoolSize, requestedParallelism > 0 ? requestedParallelism : 1); + LOGGER.debug("Computed concurrent stream consumer parallelism: {}", parallelism); + return parallelism; + } + + /** + * Creates the {@link ExecutorService} that will be used by the consumer to consume from the + * provided streams in parallel. + * + * @param nThreads The number of threads to execute concurrently. + * @return The configured {@link ExecutorService}. + */ + private ExecutorService createExecutorService(final Integer nThreads) { + return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(), + new ConcurrentStreamThreadFactory(), new AbortPolicy()); + } + + /** + * Executes the stream by providing it to the configured {@link #streamConsumer}. + * + * @param stream The stream to be executed. + */ + private void executeStream(final AutoCloseableIterator stream) { + try (stream) { + stream.getAirbyteStream().ifPresent(s -> LOGGER.debug("Consuming from stream {}...", s)); + StreamStatusUtils.emitStartStreamStatus(stream, streamStatusEmitter); + streamConsumer.accept(stream); + StreamStatusUtils.emitCompleteStreamStatus(stream, streamStatusEmitter); + stream.getAirbyteStream().ifPresent(s -> LOGGER.debug("Consumption from stream {} complete.", s)); + } catch (final Exception e) { + stream.getAirbyteStream().ifPresent(s -> LOGGER.error("Unable to consume from stream {}.", s, e)); + StreamStatusUtils.emitIncompleteStreamStatus(stream, streamStatusEmitter); + exceptions.add(e); + } + } + + @Override + public void close() throws Exception { + // Block waiting for the executor service to close + executorService.shutdownNow(); + executorService.awaitTermination(30, TimeUnit.SECONDS); + } + + /** + * Custom {@link ThreadFactory} that names the threads used to concurrently execute streams. + */ + private static class ConcurrentStreamThreadFactory implements ThreadFactory { + + @Override + public Thread newThread(final Runnable r) { + final Thread thread = new Thread(r); + if (r instanceof ConcurrentStreamRunnable) { + final AutoCloseableIterator stream = ((ConcurrentStreamRunnable) r).stream(); + if (stream.getAirbyteStream().isPresent()) { + final AirbyteStreamNameNamespacePair airbyteStream = stream.getAirbyteStream().get(); + thread.setName(String.format("%s-%s-%s", CONCURRENT_STREAM_THREAD_NAME, airbyteStream.getNamespace(), airbyteStream.getName())); + } else { + thread.setName(CONCURRENT_STREAM_THREAD_NAME); + } + } else { + thread.setName(CONCURRENT_STREAM_THREAD_NAME); + } + return thread; + } + + } + + /** + * Custom {@link Runnable} that exposes the stream for thread naming purposes. + * + * @param stream The stream that is part of the {@link Runnable} execution. + * @param consumer The {@link ConcurrentStreamConsumer} that will execute the stream. + */ + private record ConcurrentStreamRunnable(AutoCloseableIterator stream, ConcurrentStreamConsumer consumer) implements Runnable { + + @Override + public void run() { + consumer.executeStream(stream); + } + + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/DestinationConfigTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/DestinationConfigTest.java new file mode 100644 index 000000000000..00182f989c26 --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/DestinationConfigTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import org.junit.jupiter.api.Test; + +public class DestinationConfigTest { + + private static final String JSON = """ + { + "foo": "bar", + "baz": true + } + """; + + private static final JsonNode NODE = Jsons.deserialize(JSON); + + @Test + public void testInitialization() { + // bad initialization + assertThrows(IllegalArgumentException.class, () -> DestinationConfig.initialize(null)); + assertThrows(IllegalStateException.class, DestinationConfig::getInstance); + + // good initialization + DestinationConfig.initialize(NODE); + assertNotNull(DestinationConfig.getInstance()); + assertEquals(NODE, DestinationConfig.getInstance().root); + + // initializing again doesn't change the config + final JsonNode nodeUnused = Jsons.deserialize("{}"); + DestinationConfig.initialize(nodeUnused); + assertEquals(NODE, DestinationConfig.getInstance().root); + } + + @Test + public void testValues() { + DestinationConfig.initialize(NODE); + + assertEquals("bar", DestinationConfig.getInstance().getTextValue("foo")); + assertEquals("", DestinationConfig.getInstance().getTextValue("baz")); + + assertFalse(DestinationConfig.getInstance().getBooleanValue("foo")); + assertTrue(DestinationConfig.getInstance().getBooleanValue("baz")); + + // non-existent key + assertEquals("", DestinationConfig.getInstance().getTextValue("blah")); + assertFalse(DestinationConfig.getInstance().getBooleanValue("blah")); + + assertEquals(Jsons.deserialize("\"bar\""), DestinationConfig.getInstance().getNodeValue("foo")); + assertEquals(Jsons.deserialize("true"), DestinationConfig.getInstance().getNodeValue("baz")); + assertNull(DestinationConfig.getInstance().getNodeValue("blah")); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java index fbfbae6d13f6..8f2aaf57615c 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.base; +import static io.airbyte.integrations.base.IntegrationRunner.ORPHANED_THREAD_FILTER; import static io.airbyte.integrations.util.ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; @@ -371,24 +372,20 @@ void testDestinationConsumerLifecycleFailure() throws Exception { } @Test - void testInterruptOrphanThreadFailure() { - final String testName = Thread.currentThread().getName(); + void testInterruptOrphanThread() { final List caughtExceptions = new ArrayList<>(); startSleepingThread(caughtExceptions, false); - assertThrows(IOException.class, () -> IntegrationRunner.watchForOrphanThreads( - () -> { - throw new IOException("random error"); - }, + IntegrationRunner.stopOrphanedThreads( Assertions::fail, 3, TimeUnit.SECONDS, - 10, TimeUnit.SECONDS)); + 10, TimeUnit.SECONDS); try { TimeUnit.SECONDS.sleep(15); } catch (final Exception e) { throw new RuntimeException(e); } final List runningThreads = ThreadUtils.getAllThreads().stream() - .filter(runningThread -> !runningThread.isDaemon() && !runningThread.getName().equals(testName)) + .filter(ORPHANED_THREAD_FILTER) .collect(Collectors.toList()); // all threads should be interrupted assertEquals(List.of(), runningThreads); @@ -396,26 +393,23 @@ void testInterruptOrphanThreadFailure() { } @Test - void testNoInterruptOrphanThreadFailure() { - final String testName = Thread.currentThread().getName(); + void testNoInterruptOrphanThread() { final List caughtExceptions = new ArrayList<>(); final AtomicBoolean exitCalled = new AtomicBoolean(false); startSleepingThread(caughtExceptions, true); - assertThrows(IOException.class, () -> IntegrationRunner.watchForOrphanThreads( - () -> { - throw new IOException("random error"); - }, + IntegrationRunner.stopOrphanedThreads( () -> exitCalled.set(true), 3, TimeUnit.SECONDS, - 10, TimeUnit.SECONDS)); + 10, TimeUnit.SECONDS); try { TimeUnit.SECONDS.sleep(15); } catch (final Exception e) { throw new RuntimeException(e); } + final List runningThreads = ThreadUtils.getAllThreads().stream() - .filter(runningThread -> !runningThread.isDaemon() && !runningThread.getName().equals(testName)) - .toList(); + .filter(ORPHANED_THREAD_FILTER) + .collect(Collectors.toList()); // a thread that refuses to be interrupted should remain assertEquals(1, runningThreads.size()); assertEquals(1, caughtExceptions.size()); @@ -423,7 +417,13 @@ void testNoInterruptOrphanThreadFailure() { } private void startSleepingThread(final List caughtExceptions, final boolean ignoreInterrupt) { - final ExecutorService executorService = Executors.newFixedThreadPool(1); + final ExecutorService executorService = Executors.newFixedThreadPool(1, r -> { + // Create a thread that should be identified as orphaned if still running during shutdown + final Thread thread = new Thread(r); + thread.setName("sleeping-thread"); + thread.setDaemon(false); + return thread; + }); executorService.submit(() -> { for (int tries = 0; tries < 3; tries++) { try { diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java index 11ef6402e14e..eae9d74b83a8 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java @@ -8,6 +8,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -18,20 +19,26 @@ import com.google.common.collect.Lists; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.record_buffer.BufferFlushType; +import io.airbyte.integrations.destination.record_buffer.BufferingStrategy; import io.airbyte.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -350,6 +357,141 @@ void testSlowStreamReturnsMultipleStates() throws Exception { verify(outputRecordCollector).accept(STATE_MESSAGE2); } + /** + * Verify that if we ack a state message for stream2 while stream1 has unflushed records+state, that + * we do _not_ ack stream1's state message. + */ + @Test + void testStreamTail() throws Exception { + // InMemoryRecordBufferingStrategy always returns FLUSH_ALL, so just mock a new strategy here + final BufferingStrategy strategy = mock(BufferingStrategy.class); + // The first two records that we push will not trigger any flushes, but the third record _will_ + // trigger a flush + when(strategy.addRecord(any(), any())).thenReturn( + Optional.empty(), + Optional.empty(), + Optional.of(BufferFlushType.FLUSH_SINGLE_STREAM)); + consumer = new BufferedStreamConsumer( + outputRecordCollector, + onStart, + strategy, + onClose, + CATALOG, + isValidRecord, + // Never periodic flush + Duration.ofHours(24), + null); + final List expectedRecordsStream1 = List.of(new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME) + .withNamespace(SCHEMA_NAME))); + final List expectedRecordsStream2 = List.of(new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME2) + .withNamespace(SCHEMA_NAME))); + + final AirbyteMessage state1 = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME).withNamespace(SCHEMA_NAME)) + .withStreamState(Jsons.jsonNode(ImmutableMap.of("state_message_id", 1))))); + final AirbyteMessage state2 = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME2).withNamespace(SCHEMA_NAME)) + .withStreamState(Jsons.jsonNode(ImmutableMap.of("state_message_id", 2))))); + + consumer.start(); + consumeRecords(consumer, expectedRecordsStream1); + consumer.accept(state1); + // At this point, we have not yet flushed anything + consumeRecords(consumer, expectedRecordsStream2); + consumer.accept(state2); + consumeRecords(consumer, expectedRecordsStream2); + // Now we have flushed stream 2, but not stream 1 + // Verify that we have only acked stream 2's state. + verify(outputRecordCollector).accept(state2); + verify(outputRecordCollector, never()).accept(state1); + + consumer.close(); + // Now we've closed the consumer, which flushes everything. + // Verify that we ack stream 1's pending state. + verify(outputRecordCollector).accept(state1); + } + + /** + * Same idea as {@link #testStreamTail()} but with global state. We shouldn't emit any state + * messages until we close the consumer. + */ + @Test + void testStreamTailGlobalState() throws Exception { + // InMemoryRecordBufferingStrategy always returns FLUSH_ALL, so just mock a new strategy here + final BufferingStrategy strategy = mock(BufferingStrategy.class); + // The first two records that we push will not trigger any flushes, but the third record _will_ + // trigger a flush + when(strategy.addRecord(any(), any())).thenReturn( + Optional.empty(), + Optional.empty(), + Optional.of(BufferFlushType.FLUSH_SINGLE_STREAM)); + consumer = new BufferedStreamConsumer( + outputRecordCollector, + onStart, + strategy, + onClose, + CATALOG, + isValidRecord, + // Never periodic flush + Duration.ofHours(24), + null); + final List expectedRecordsStream1 = List.of(new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME) + .withNamespace(SCHEMA_NAME))); + final List expectedRecordsStream2 = List.of(new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME2) + .withNamespace(SCHEMA_NAME))); + + final AirbyteMessage state1 = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.GLOBAL) + .withGlobal(new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(ImmutableMap.of("state_message_id", 1))))); + final AirbyteMessage state2 = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.GLOBAL) + .withGlobal(new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(ImmutableMap.of("state_message_id", 2))))); + + consumer.start(); + consumeRecords(consumer, expectedRecordsStream1); + consumer.accept(state1); + // At this point, we have not yet flushed anything + consumeRecords(consumer, expectedRecordsStream2); + consumer.accept(state2); + consumeRecords(consumer, expectedRecordsStream2); + // Now we have flushed stream 2, but not stream 1 + // We should not have acked any state yet, because we haven't written stream1's records yet. + verify(outputRecordCollector, never()).accept(any()); + + consumer.close(); + // Now we've closed the consumer, which flushes everything. + // Verify that we ack the final state. + // Note that we discard state1 entirely - this is OK. As long as we ack the last state message, + // the source can correctly resume from that point. + verify(outputRecordCollector).accept(state2); + } + private BufferedStreamConsumer getConsumerWithFlushFrequency() { final BufferedStreamConsumer flushFrequencyConsumer = new BufferedStreamConsumer( outputRecordCollector, @@ -358,7 +500,8 @@ private BufferedStreamConsumer getConsumerWithFlushFrequency() { onClose, CATALOG, isValidRecord, - Duration.ofSeconds(PERIODIC_BUFFER_FREQUENCY)); + Duration.ofSeconds(PERIODIC_BUFFER_FREQUENCY), + null); return flushFrequencyConsumer; } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java index b24350b969c5..a8e69fa8fc2e 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java @@ -41,7 +41,7 @@ class DestStreamStateLifecycleManagerTest { @BeforeEach void setup() { - mgr = new DestStreamStateLifecycleManager(); + mgr = new DestStreamStateLifecycleManager("default_namespace"); } /** diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java index 68afe5c6b8d9..06e399d28f36 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java @@ -24,6 +24,7 @@ import io.airbyte.integrations.destination_async.state.FlushFailure; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteLogMessage; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; @@ -37,6 +38,8 @@ import java.time.Instant; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -44,6 +47,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang.RandomStringUtils; import org.junit.jupiter.api.BeforeEach; @@ -74,6 +78,12 @@ class AsyncStreamConsumerTest { Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)))); + private static final JsonNode PAYLOAD = Jsons.jsonNode(Map.of( + "created_at", "2022-02-01T17:02:19+00:00", + "id", 1, + "make", "Mazda", + "nested_column", Map.of("array_column", List.of(1, 2, 3)))); + private static final AirbyteMessage STATE_MESSAGE1 = new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() @@ -107,14 +117,15 @@ void setup() { flushFunction, CATALOG, new BufferManager(), - flushFailure); + flushFailure, + "default_ns"); when(flushFunction.getOptimalBatchSizeBytes()).thenReturn(10_000L); } @Test void test1StreamWith1State() throws Exception { - final List expectedRecords = generateRecords(1_000); + final List expectedRecords = generateRecords(1_000); consumer.start(); consumeRecords(consumer, expectedRecords); @@ -130,7 +141,7 @@ void test1StreamWith1State() throws Exception { @Test void test1StreamWith2State() throws Exception { - final List expectedRecords = generateRecords(1_000); + final List expectedRecords = generateRecords(1_000); consumer.start(); consumeRecords(consumer, expectedRecords); @@ -147,21 +158,20 @@ void test1StreamWith2State() throws Exception { @Test void test1StreamWith0State() throws Exception { - final List expectedRecords = generateRecords(1_000); + final List allRecords = generateRecords(1_000); consumer.start(); - consumeRecords(consumer, expectedRecords); + consumeRecords(consumer, allRecords); consumer.close(); verifyStartAndClose(); - verifyRecords(STREAM_NAME, SCHEMA_NAME, expectedRecords); + verifyRecords(STREAM_NAME, SCHEMA_NAME, allRecords); } @Test void testShouldBlockWhenQueuesAreFull() throws Exception { consumer.start(); - } /* @@ -179,7 +189,8 @@ void testBackPressure() throws Exception { flushFunction, CATALOG, new BufferManager(1024 * 10), - flushFailure); + flushFailure, + "default_ns"); when(flushFunction.getOptimalBatchSizeBytes()).thenReturn(0L); final AtomicLong recordCount = new AtomicLong(); @@ -215,6 +226,60 @@ void testBackPressure() throws Exception { assertTrue(recordCount.get() < 1000, String.format("Record count was %s", recordCount.get())); } + @Test + void deserializeAirbyteMessageWithAirbyteRecord() { + final AirbyteMessage airbyteMessage = new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME) + .withNamespace(SCHEMA_NAME) + .withData(PAYLOAD)); + final String serializedAirbyteMessage = Jsons.serialize(airbyteMessage); + final String airbyteRecordString = Jsons.serialize(PAYLOAD); + final Optional partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(airbyteRecordString, partial.get().getSerialized()); + } + + @Test + void deserializeAirbyteMessageWithEmptyAirbyteRecord() { + final Map emptyMap = Map.of(); + final AirbyteMessage airbyteMessage = new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME) + .withNamespace(SCHEMA_NAME) + .withData(Jsons.jsonNode(emptyMap))); + final String serializedAirbyteMessage = Jsons.serialize(airbyteMessage); + final Optional partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(emptyMap.toString(), partial.get().getSerialized()); + } + + @Test + void deserializeAirbyteMessageWithNoStateOrRecord() { + final AirbyteMessage airbyteMessage = new AirbyteMessage() + .withType(Type.LOG) + .withLog(new AirbyteLogMessage()); + final String serializedAirbyteMessage = Jsons.serialize(airbyteMessage); + assertThrows(RuntimeException.class, () -> AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage)); + } + + @Test + void deserializeAirbyteMessageWithAirbyteState() { + final String serializedAirbyteMessage = Jsons.serialize(STATE_MESSAGE1); + final Optional partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(serializedAirbyteMessage, partial.get().getSerialized()); + } + + @Test + void deserializeAirbyteMessageWithBadAirbyteState() { + final AirbyteMessage badState = new AirbyteMessage() + .withState(new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(STREAM1_DESC).withStreamState(Jsons.jsonNode(1)))); + final String serializedAirbyteMessage = Jsons.serialize(badState); + assertThrows(RuntimeException.class, () -> AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage)); + } + @Nested class ErrorHandling { @@ -246,10 +311,10 @@ void testErrorOnClose() throws Exception { } - private static void consumeRecords(final AsyncStreamConsumer consumer, final Collection records) { + private static void consumeRecords(final AsyncStreamConsumer consumer, final Collection records) { records.forEach(m -> { try { - consumer.accept(m.getSerialized(), RECORD_SIZE_20_BYTES); + consumer.accept(Jsons.serialize(m), RECORD_SIZE_20_BYTES); } catch (final Exception e) { throw new RuntimeException(e); } @@ -258,25 +323,20 @@ private static void consumeRecords(final AsyncStreamConsumer consumer, final Col // NOTE: Generates records at chunks of 160 bytes @SuppressWarnings("SameParameterValue") - private static List generateRecords(final long targetSizeInBytes) { - final List output = Lists.newArrayList(); + private static List generateRecords(final long targetSizeInBytes) { + final List output = Lists.newArrayList(); long bytesCounter = 0; for (int i = 0;; i++) { final JsonNode payload = Jsons.jsonNode(ImmutableMap.of("id", RandomStringUtils.randomAlphabetic(7), "name", "human " + String.format("%8d", i))); final long sizeInBytes = RecordSizeEstimator.getStringByteSize(payload); bytesCounter += sizeInBytes; - final PartialAirbyteMessage airbyteMessage = new PartialAirbyteMessage() + final AirbyteMessage airbyteMessage = new AirbyteMessage() .withType(Type.RECORD) - .withRecord(new PartialAirbyteRecordMessage() + .withRecord(new AirbyteRecordMessage() .withStream(STREAM_NAME) - .withNamespace(SCHEMA_NAME)) - .withSerialized(Jsons.serialize(new AirbyteMessage() - .withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream(STREAM_NAME) - .withNamespace(SCHEMA_NAME) - .withData(payload)))); + .withNamespace(SCHEMA_NAME) + .withData(payload)); if (bytesCounter > targetSizeInBytes) { break; } else { @@ -292,7 +352,7 @@ private void verifyStartAndClose() throws Exception { } @SuppressWarnings({"unchecked", "SameParameterValue"}) - private void verifyRecords(final String streamName, final String namespace, final Collection expectedRecords) + private void verifyRecords(final String streamName, final String namespace, final List allRecords) throws Exception { final ArgumentCaptor> argumentCaptor = ArgumentCaptor.forClass(Stream.class); verify(flushFunction, atLeast(1)).flush( @@ -306,7 +366,15 @@ private void verifyRecords(final String streamName, final String namespace, fina // flatten those results into a single list for the simplicity of comparison .flatMap(s -> s) .toList(); - assertEquals(expectedRecords.stream().toList(), actualRecords); + + final var expRecords = allRecords.stream().map(m -> new PartialAirbyteMessage() + .withType(Type.RECORD) + .withRecord(new PartialAirbyteRecordMessage() + .withStream(m.getRecord().getStream()) + .withNamespace(m.getRecord().getNamespace()) + .withData(m.getRecord().getData())) + .withSerialized(Jsons.serialize(m.getRecord().getData()))).collect(Collectors.toList()); + assertEquals(expRecords, actualRecords); } } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java index 6e4e6a491519..723aec616e8c 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java @@ -76,9 +76,15 @@ void testGetNextPicksUpOnTimeTrigger() { when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); when(bufferDequeue.getTimeOfLastRecord(DESC1)) + // because we eagerly load values and later access them again + // double the mocks for correctness; two calls here equals one test case. .thenReturn(Optional.empty()) + .thenReturn(Optional.empty()) + .thenReturn(Optional.of(NOW)) .thenReturn(Optional.of(NOW)) + .thenReturn(Optional.of(FIVE_MIN_AGO)) .thenReturn(Optional.of(FIVE_MIN_AGO)); + final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); when(runningFlushWorkers.getSizesOfRunningWorkerBatches(any())).thenReturn(List.of(Optional.of(SIZE_10MB))); final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/PartialAirbyteMessageTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/PartialAirbyteMessageTest.java new file mode 100644 index 000000000000..d9d2e3e5edff --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/PartialAirbyteMessageTest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination_async; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.StreamDescriptor; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import java.time.Instant; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PartialAirbyteMessageTest { + + @Test + void testDeserializeRecord() { + final long emittedAt = Instant.now().toEpochMilli(); + final var serializedRec = Jsons.serialize(new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream("users") + .withNamespace("public") + .withEmittedAt(emittedAt) + .withData(Jsons.jsonNode("data")))); + + final var rec = Jsons.tryDeserialize(serializedRec, PartialAirbyteMessage.class).get(); + Assertions.assertEquals(AirbyteMessage.Type.RECORD, rec.getType()); + Assertions.assertEquals("users", rec.getRecord().getStream()); + Assertions.assertEquals("public", rec.getRecord().getNamespace()); + Assertions.assertEquals("\"data\"", rec.getRecord().getData().toString()); + Assertions.assertEquals(emittedAt, rec.getRecord().getEmittedAt()); + } + + @Test + void testDeserializeState() { + final var serializedState = Jsons.serialize(new io.airbyte.protocol.models.AirbyteMessage() + .withType(io.airbyte.protocol.models.AirbyteMessage.Type.STATE) + .withState(new AirbyteStateMessage().withStream( + new AirbyteStreamState().withStreamDescriptor( + new StreamDescriptor().withName("user").withNamespace("public")) + .withStreamState(Jsons.jsonNode("data"))) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM))); + + final var rec = Jsons.tryDeserialize(serializedState, PartialAirbyteMessage.class).get(); + Assertions.assertEquals(AirbyteMessage.Type.STATE, rec.getType()); + + final var streamDesc = rec.getState().getStream().getStreamDescriptor(); + Assertions.assertEquals("user", streamDesc.getName()); + Assertions.assertEquals("public", streamDesc.getNamespace()); + Assertions.assertEquals(io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType.STREAM, rec.getState().getType()); + } + + @Test + void testGarbage() { + final var badSerialization = "messed up data"; + + final var rec = Jsons.tryDeserialize(badSerialization, PartialAirbyteMessage.class); + Assertions.assertTrue(rec.isEmpty()); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java index d9561fac9d63..3e0632d372e1 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java @@ -30,6 +30,7 @@ void testTimeTrigger() { .thenReturn(Optional.empty()) .thenReturn(Optional.of(NOW)) .thenReturn(Optional.of(FIVE_MIN_AGO)); + final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, null, null); assertEquals(false, detect.isTimeTriggered(DESC1).getLeft()); assertEquals(false, detect.isTimeTriggered(DESC1).getLeft()); diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java index 73ea94163037..446b988d30f2 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; import org.junit.jupiter.api.Test; @@ -32,7 +33,8 @@ void test() throws InterruptedException { queue.take(); assertEquals(0, queue.getCurrentMemoryUsage()); - assertNotNull(queue.getTimeOfLastMessage().orElse(null)); + // This should be null because the queue is empty + assertTrue(queue.getTimeOfLastMessage().isEmpty(), "Expected empty optional; got " + queue.getTimeOfLastMessage()); } } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumerTest.java new file mode 100644 index 000000000000..db9f92492d88 --- /dev/null +++ b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumerTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.util.concurrent; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.node.IntNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link ConcurrentStreamConsumer} class. + */ +class ConcurrentStreamConsumerTest { + + private static final String NAME = "name"; + private static final String NAMESPACE = "namespace"; + + @Test + void testAcceptMessage() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer> streamConsumer = mock(Consumer.class); + + final ConcurrentStreamConsumer concurrentStreamConsumer = new ConcurrentStreamConsumer(streamConsumer, 1); + + assertDoesNotThrow(() -> concurrentStreamConsumer.accept(List.of(stream))); + + verify(streamConsumer, times(1)).accept(stream); + } + + @Test + void testAcceptMessageWithException() { + final AutoCloseableIterator stream = mock(AutoCloseableIterator.class); + final Consumer> streamConsumer = mock(Consumer.class); + final Exception e = new NullPointerException("test"); + + doThrow(e).when(streamConsumer).accept(any()); + + final ConcurrentStreamConsumer concurrentStreamConsumer = new ConcurrentStreamConsumer(streamConsumer, 1); + + assertDoesNotThrow(() -> concurrentStreamConsumer.accept(List.of(stream))); + + verify(streamConsumer, times(1)).accept(stream); + assertTrue(concurrentStreamConsumer.getException().isPresent()); + assertEquals(e, concurrentStreamConsumer.getException().get()); + assertEquals(1, concurrentStreamConsumer.getExceptions().size()); + assertTrue(concurrentStreamConsumer.getExceptions().contains(e)); + } + + @Test + void testAcceptMessageWithMultipleExceptions() { + final AutoCloseableIterator stream1 = mock(AutoCloseableIterator.class); + final AutoCloseableIterator stream2 = mock(AutoCloseableIterator.class); + final AutoCloseableIterator stream3 = mock(AutoCloseableIterator.class); + final Consumer> streamConsumer = mock(Consumer.class); + final Exception e1 = new NullPointerException("test1"); + final Exception e2 = new NullPointerException("test2"); + final Exception e3 = new NullPointerException("test3"); + + doThrow(e1).when(streamConsumer).accept(stream1); + doThrow(e2).when(streamConsumer).accept(stream2); + doThrow(e3).when(streamConsumer).accept(stream3); + + final ConcurrentStreamConsumer concurrentStreamConsumer = new ConcurrentStreamConsumer(streamConsumer, 1); + + assertDoesNotThrow(() -> concurrentStreamConsumer.accept(List.of(stream1, stream2, stream3))); + + verify(streamConsumer, times(3)).accept(any(AutoCloseableIterator.class)); + assertTrue(concurrentStreamConsumer.getException().isPresent()); + assertEquals(e1, concurrentStreamConsumer.getException().get()); + assertEquals(3, concurrentStreamConsumer.getExceptions().size()); + assertTrue(concurrentStreamConsumer.getExceptions().contains(e1)); + assertTrue(concurrentStreamConsumer.getExceptions().contains(e2)); + assertTrue(concurrentStreamConsumer.getExceptions().contains(e3)); + } + + @Test + void testMoreStreamsThanAvailableThreads() { + final List baseData = List.of(2, 4, 6, 8, 10, 12, 14, 16, 18, 20); + final List> streams = new ArrayList<>(); + for (int i = 0; i < 20; i++) { + final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = + new AirbyteStreamNameNamespacePair(String.format("%s_%d", NAME, i), NAMESPACE); + final List messages = new ArrayList<>(); + for (int d : baseData) { + final AirbyteMessage airbyteMessage = mock(AirbyteMessage.class); + final AirbyteRecordMessage recordMessage = mock(AirbyteRecordMessage.class); + when(recordMessage.getData()).thenReturn(new IntNode(d * i)); + when(airbyteMessage.getRecord()).thenReturn(recordMessage); + messages.add(airbyteMessage); + } + streams.add(AutoCloseableIterators.fromIterator(messages.iterator(), airbyteStreamNameNamespacePair)); + } + final Consumer> streamConsumer = mock(Consumer.class); + + final ConcurrentStreamConsumer concurrentStreamConsumer = new ConcurrentStreamConsumer(streamConsumer, streams.size()); + final Integer partitionSize = concurrentStreamConsumer.getParallelism(); + final List>> partitions = Lists.partition(streams.stream().toList(), + partitionSize); + + for (final List> partition : partitions) { + assertDoesNotThrow(() -> concurrentStreamConsumer.accept(partition)); + } + + verify(streamConsumer, times(streams.size())).accept(any(AutoCloseableIterator.class)); + } + +} diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index e8ee2ddd0354..c0ee635f3045 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -15,6 +15,11 @@ COPY dbt-project-template/ ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 85112083e41b..03740e0f9019 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -58,11 +58,11 @@ def buildAirbyteDocker(String customConnector) { arch = 'linux/arm64' } - def baseCommand = ['docker', 'buildx', 'build', '--load', '--platform', arch, '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector), '.'] - println("Building normalization container: " + baseCommand.join(" ")) + def cmdArray = ['docker', 'buildx', 'build', '--load', '--platform', arch, '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector), '.'] + // println("Building normalization container: " + cmdArray.join(" ")) return { - commandLine baseCommand + commandLine cmdArray } } diff --git a/airbyte-integrations/bases/base-normalization/clickhouse.Dockerfile b/airbyte-integrations/bases/base-normalization/clickhouse.Dockerfile index d12e36ba3226..18005ea89872 100644 --- a/airbyte-integrations/bases/base-normalization/clickhouse.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/clickhouse.Dockerfile @@ -14,6 +14,11 @@ COPY dbt-project-template/ ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/dbt.Dockerfile b/airbyte-integrations/bases/base-normalization/dbt.Dockerfile new file mode 100644 index 000000000000..09b0e3c94064 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/dbt.Dockerfile @@ -0,0 +1,3 @@ +# This dockerfile only exists to pull and re-export this image converted to the local arch of this machine +# It is then consumed by the Dockerfile in this direcotry as "fishtownanalytics/dbt:1.0.0-dev" +FROM fishtownanalytics/dbt:1.0.0 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/duckdb.Dockerfile b/airbyte-integrations/bases/base-normalization/duckdb.Dockerfile index 638d1e6afbeb..af039e7114ec 100644 --- a/airbyte-integrations/bases/base-normalization/duckdb.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/duckdb.Dockerfile @@ -15,6 +15,11 @@ COPY dbt-project-template/ ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json index 7840b4c835d7..584f7f98d359 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json @@ -124,10 +124,10 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [] + "default_cursor_field": ["_ab_cdc_updated_at"] }, "sync_mode": "incremental", - "cursor_field": [], + "cursor_field": ["_ab_cdc_updated_at"], "destination_sync_mode": "append_dedup", "primary_key": [["id"]] }, @@ -156,10 +156,10 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [] + "default_cursor_field": ["_ab_cdc_lsn"] }, "sync_mode": "incremental", - "cursor_field": [], + "cursor_field": ["_ab_cdc_lsn"], "destination_sync_mode": "append_dedup", "primary_key": [["id"]] }, @@ -191,10 +191,10 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [] + "default_cursor_field": ["_ab_cdc_lsn"] }, "sync_mode": "full_refresh", - "cursor_field": [], + "cursor_field": ["_ab_cdc_lsn"], "destination_sync_mode": "append_dedup", "primary_key": [["id"]] }, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json index 220d17b17c87..1f334071c928 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json @@ -113,10 +113,10 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [] + "default_cursor_field": ["_ab_cdc_lsn"] }, "sync_mode": "incremental", - "cursor_field": [], + "cursor_field": ["_ab_cdc_lsn"], "destination_sync_mode": "append_dedup", "primary_key": [["id"]] }, @@ -145,10 +145,10 @@ }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [] + "default_cursor_field": ["_ab_cdc_lsn"] }, "sync_mode": "incremental", - "cursor_field": [], + "cursor_field": ["_ab_cdc_lsn"], "destination_sync_mode": "append_dedup", "primary_key": [["id"]] } diff --git a/airbyte-integrations/bases/base-normalization/mssql.Dockerfile b/airbyte-integrations/bases/base-normalization/mssql.Dockerfile index a5f30879f862..449c0c60982b 100644 --- a/airbyte-integrations/bases/base-normalization/mssql.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/mssql.Dockerfile @@ -49,6 +49,11 @@ COPY dbt-project-template-mssql/* ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/mysql.Dockerfile b/airbyte-integrations/bases/base-normalization/mysql.Dockerfile index 8e0f4ebc5b11..efc25fcb38d9 100644 --- a/airbyte-integrations/bases/base-normalization/mysql.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/mysql.Dockerfile @@ -16,6 +16,11 @@ COPY dbt-project-template-mysql/* ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/oracle.Dockerfile b/airbyte-integrations/bases/base-normalization/oracle.Dockerfile index 971338980e09..6041ea3cf1c5 100644 --- a/airbyte-integrations/bases/base-normalization/oracle.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/oracle.Dockerfile @@ -34,6 +34,11 @@ COPY dbt-project-template/ ./dbt-template/ COPY dbt-project-template-oracle/* ./dbt-template/ WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/redshift.Dockerfile b/airbyte-integrations/bases/base-normalization/redshift.Dockerfile index b1a35debe8b6..9b8124ebe9ed 100644 --- a/airbyte-integrations/bases/base-normalization/redshift.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/redshift.Dockerfile @@ -16,6 +16,11 @@ COPY dbt-project-template-redshift/* ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile index bdc5a914889e..41d74e50621a 100644 --- a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile @@ -16,6 +16,11 @@ COPY dbt-project-template-snowflake/* ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-normalization/tidb.Dockerfile b/airbyte-integrations/bases/base-normalization/tidb.Dockerfile index 84ec144b6f25..a749f88a66d8 100644 --- a/airbyte-integrations/bases/base-normalization/tidb.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/tidb.Dockerfile @@ -15,6 +15,11 @@ COPY dbt-project-template/ ./dbt-template/ # Install python dependencies WORKDIR /airbyte/base_python_structs + +# workaround for https://github.com/yaml/pyyaml/issues/601 +# this should be fixed in the airbyte/base-airbyte-protocol-python image +RUN pip install "Cython<3.0" "pyyaml==5.4" --no-build-isolation + RUN pip install . WORKDIR /airbyte/normalization_code diff --git a/airbyte-integrations/bases/base-typing-deduping-test/build.gradle b/airbyte-integrations/bases/base-typing-deduping-test/build.gradle new file mode 100644 index 000000000000..5c786c2f79c0 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'java-library' +} + +dependencies { + implementation project(':airbyte-config-oss:config-models-oss') + implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') + implementation project(':airbyte-integrations:bases:base-typing-deduping') + implementation libs.airbyte.protocol + + implementation(enforcedPlatform('org.junit:junit-bom:5.8.2')) + implementation 'org.junit.jupiter:junit-jupiter-api' + implementation 'org.junit.jupiter:junit-jupiter-params' + implementation 'org.mockito:mockito-core:4.6.1' +} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..8f26b1699dc0 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java @@ -0,0 +1,975 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Streams; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class exercises {@link SqlGenerator} implementations. All destinations should extend this + * class for their respective implementation. Subclasses are encouraged to add additional tests with + * destination-specific behavior (for example, verifying that datasets are created in the correct + * BigQuery region). + *

    + * Subclasses should implement a {@link org.junit.jupiter.api.BeforeAll} method to load any secrets + * and connect to the destination. This test expects to be able to run + * {@link #getDestinationHandler()} in a {@link org.junit.jupiter.api.BeforeEach} method. + */ +@Execution(ExecutionMode.CONCURRENT) +public abstract class BaseSqlGeneratorIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseSqlGeneratorIntegrationTest.class); + /** + * This, along with {@link #FINAL_TABLE_COLUMN_NAMES_CDC}, is the list of columns that should be in + * the final table. They're useful for generating SQL queries to insert records into the final + * table. + */ + protected static final List FINAL_TABLE_COLUMN_NAMES = List.of( + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta", + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown"); + protected static final List FINAL_TABLE_COLUMN_NAMES_CDC; + + static { + FINAL_TABLE_COLUMN_NAMES_CDC = Streams.concat( + FINAL_TABLE_COLUMN_NAMES.stream(), + Stream.of("_ab_cdc_deleted_at")).toList(); + } + + protected static final RecordDiffer DIFFER = new RecordDiffer( + Pair.of("id1", AirbyteProtocolType.INTEGER), + Pair.of("id2", AirbyteProtocolType.INTEGER), + Pair.of("updated_at", AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE)); + + /** + * Subclasses may use these four StreamConfigs in their tests. + */ + protected StreamConfig incrementalDedupStream; + /** + * We intentionally don't have full refresh overwrite/append streams. Those actually behave + * identically in the sqlgenerator. Overwrite mode is actually handled in + * {@link DefaultTyperDeduper}. + */ + protected StreamConfig incrementalAppendStream; + protected StreamConfig cdcIncrementalDedupStream; + /** + * This isn't particularly realistic, but it's technically possible. + */ + protected StreamConfig cdcIncrementalAppendStream; + + protected SqlGenerator generator; + protected DestinationHandler destinationHandler; + protected String namespace; + + protected StreamId streamId; + private List primaryKey; + private ColumnId cursor; + private LinkedHashMap COLUMNS; + + protected abstract SqlGenerator getSqlGenerator(); + + protected abstract DestinationHandler getDestinationHandler(); + + /** + * Do any setup work to create a namespace for this test run. For example, this might create a + * BigQuery dataset, or a Snowflake schema. + */ + protected abstract void createNamespace(String namespace) throws Exception; + + /** + * Create a raw table using the StreamId's rawTableId. + */ + protected abstract void createRawTable(StreamId streamId) throws Exception; + + /** + * Creates a raw table in the v1 format + */ + protected abstract void createV1RawTable(StreamId v1RawTable) throws Exception; + + /** + * Create a final table usingi the StreamId's finalTableId. Subclasses are recommended to hardcode + * the columns from {@link #FINAL_TABLE_COLUMN_NAMES} or {@link #FINAL_TABLE_COLUMN_NAMES_CDC}. The + * only difference between those two column lists is the inclusion of the _ab_cdc_deleted_at column, + * which is controlled by the includeCdcDeletedAt parameter. + */ + protected abstract void createFinalTable(boolean includeCdcDeletedAt, StreamId streamId, String suffix) throws Exception; + + protected abstract void insertRawTableRecords(StreamId streamId, List records) throws Exception; + + protected abstract void insertV1RawTableRecords(StreamId streamId, List records) throws Exception; + + protected abstract void insertFinalTableRecords(boolean includeCdcDeletedAt, StreamId streamId, String suffix, List records) + throws Exception; + + /** + * The two dump methods are defined identically as in {@link BaseTypingDedupingTest}, but with + * slightly different method signature. This test expects subclasses to respect the raw/finalTableId + * on the StreamId object, rather than hardcoding e.g. the airbyte_internal dataset. + *

    + * The {@code _airbyte_data} field must be deserialized into an ObjectNode, even if it's stored in + * the destination as a string. + */ + protected abstract List dumpRawTableRecords(StreamId streamId) throws Exception; + + protected abstract List dumpFinalTableRecords(StreamId streamId, String suffix) throws Exception; + + /** + * Clean up all resources in the namespace. For example, this might delete the BigQuery dataset + * created in {@link #createNamespace(String)}. + */ + protected abstract void teardownNamespace(String namespace) throws Exception; + + /** + * This test implementation is extremely destination-specific, but all destinations must implement + * it. This test should verify that creating a table using {@link #incrementalDedupStream} works as + * expected, including column types, indexing, partitioning, etc. + *

    + * Note that subclasses must also annotate their implementation with @Test. + */ + @Test + public abstract void testCreateTableIncremental() throws Exception; + + @BeforeEach + public void setup() throws Exception { + generator = getSqlGenerator(); + destinationHandler = getDestinationHandler(); + final ColumnId id1 = generator.buildColumnId("id1"); + final ColumnId id2 = generator.buildColumnId("id2"); + primaryKey = List.of(id1, id2); + cursor = generator.buildColumnId("updated_at"); + + COLUMNS = new LinkedHashMap<>(); + COLUMNS.put(id1, AirbyteProtocolType.INTEGER); + COLUMNS.put(id2, AirbyteProtocolType.INTEGER); + COLUMNS.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + COLUMNS.put(generator.buildColumnId("struct"), new Struct(new LinkedHashMap<>())); + COLUMNS.put(generator.buildColumnId("array"), new Array(AirbyteProtocolType.UNKNOWN)); + COLUMNS.put(generator.buildColumnId("string"), AirbyteProtocolType.STRING); + COLUMNS.put(generator.buildColumnId("number"), AirbyteProtocolType.NUMBER); + COLUMNS.put(generator.buildColumnId("integer"), AirbyteProtocolType.INTEGER); + COLUMNS.put(generator.buildColumnId("boolean"), AirbyteProtocolType.BOOLEAN); + COLUMNS.put(generator.buildColumnId("timestamp_with_timezone"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + COLUMNS.put(generator.buildColumnId("timestamp_without_timezone"), AirbyteProtocolType.TIMESTAMP_WITHOUT_TIMEZONE); + COLUMNS.put(generator.buildColumnId("time_with_timezone"), AirbyteProtocolType.TIME_WITH_TIMEZONE); + COLUMNS.put(generator.buildColumnId("time_without_timezone"), AirbyteProtocolType.TIME_WITHOUT_TIMEZONE); + COLUMNS.put(generator.buildColumnId("date"), AirbyteProtocolType.DATE); + COLUMNS.put(generator.buildColumnId("unknown"), AirbyteProtocolType.UNKNOWN); + + final LinkedHashMap cdcColumns = new LinkedHashMap<>(COLUMNS); + cdcColumns.put(generator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + + namespace = Strings.addRandomSuffix("sql_generator_test", "_", 5); + // This is not a typical stream ID would look like, but SqlGenerator isn't allowed to make any + // assumptions about StreamId structure. + // In practice, the final table would be testDataset.users, and the raw table would be + // airbyte_internal.testDataset_raw__stream_users. + streamId = new StreamId(namespace, "users_final", namespace, "users_raw", namespace, "users_final"); + + incrementalDedupStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + COLUMNS); + incrementalAppendStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + primaryKey, + Optional.of(cursor), + COLUMNS); + + cdcIncrementalDedupStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + cdcColumns); + cdcIncrementalAppendStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + primaryKey, + Optional.of(cursor), + cdcColumns); + + LOGGER.info("Running with namespace {}", namespace); + createNamespace(namespace); + } + + @AfterEach + public void teardown() throws Exception { + teardownNamespace(namespace); + } + + /** + * Create a table and verify that we correctly recognize it as identical to itself. + */ + @Test + public void detectNoSchemaChange() throws Exception { + final String createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + assertTrue( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Unchanged schema was incorrectly detected as a schema change."); + } + + /** + * Verify that adding a new column is detected as a schema change. + */ + @Test + public void detectColumnAdded() throws Exception { + final String createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + incrementalDedupStream.columns().put( + generator.buildColumnId("new_column"), + AirbyteProtocolType.STRING); + + assertFalse( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Adding a new column was not detected as a schema change."); + } + + /** + * Verify that removing a column is detected as a schema change. + */ + @Test + public void detectColumnRemoved() throws Exception { + final String createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + incrementalDedupStream.columns().remove(generator.buildColumnId("string")); + + assertFalse( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Removing a column was not detected as a schema change."); + } + + /** + * Verify that changing a column's type is detected as a schema change. + */ + @Test + public void detectColumnChanged() throws Exception { + final String createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + incrementalDedupStream.columns().put( + generator.buildColumnId("string"), + AirbyteProtocolType.INTEGER); + + assertFalse( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Altering a column was not detected as a schema change."); + } + + /** + * Test that T+D throws an error for an incremental-dedup sync where at least one record has a null + * primary key, and that we don't write any final records. + */ + @Test + public void incrementalDedupInvalidPrimaryKey() throws Exception { + createRawTable(streamId); + createFinalTable(false, streamId, ""); + insertRawTableRecords( + streamId, + List.of( + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "10d6e27d-ae7a-41b5-baf8-c4c277ef9c11", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": {} + } + """), + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "5ce60e70-98aa-4fe3-8159-67207352c4f0", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": {"id1": 1, "id2": 100} + } + """))); + + final String sql = generator.updateTable(incrementalDedupStream, ""); + assertThrows( + Exception.class, + () -> destinationHandler.execute(sql)); + DIFFER.diffFinalTableRecords( + emptyList(), + dumpFinalTableRecords(streamId, "")); + } + + /** + * Run a full T+D update for an incremental-dedup stream, writing to a final table with "_foo" + * suffix, with values for all data types. Verifies all behaviors for all types: + *

      + *
    • A valid, nonnull value
      • + *
    • No value (i.e. the column is missing from the record)
      • + *
    • A JSON null value
      • + *
    • An invalid value
      • + *
    + *

    + * In practice, incremental streams never write to a suffixed table, but SqlGenerator isn't allowed + * to make that assumption (and we might as well exercise that code path). + */ + @Test + public void allTypes() throws Exception { + createRawTable(streamId); + createFinalTable(false, streamId, "_foo"); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); + + final String sql = generator.updateTable(incrementalDedupStream, "_foo"); + destinationHandler.execute(sql); + + verifyRecords( + "sqlgenerator/alltypes_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/alltypes_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "_foo")); + } + + @Test + public void timestampFormats() throws Exception { + createRawTable(streamId); + createFinalTable(false, streamId, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/timestampformats_inputrecords.jsonl")); + + final String sql = generator.updateTable(incrementalAppendStream, ""); + destinationHandler.execute(sql); + + DIFFER.diffFinalTableRecords( + BaseTypingDedupingTest.readRecords("sqlgenerator/timestampformats_expectedrecords_final.jsonl"), + dumpFinalTableRecords(streamId, "")); + } + + @Test + public void incrementalDedup() throws Exception { + createRawTable(streamId); + createFinalTable(false, streamId, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/incrementaldedup_inputrecords.jsonl")); + + final String sql = generator.updateTable(incrementalDedupStream, ""); + destinationHandler.execute(sql); + + verifyRecords( + "sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/incrementaldedup_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * We shouldn't crash on a sync with null cursor. Insert two records and verify that we keep the + * record with higher extracted_at. + */ + @Test + public void incrementalDedupNoCursor() throws Exception { + createRawTable(streamId); + createFinalTable(false, streamId, ""); + insertRawTableRecords( + streamId, + List.of( + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "c5bcae50-962e-4b92-b2eb-1659eae31693", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "string": "foo" + } + } + """), + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "93f1bdd8-1916-4e6c-94dc-29a5d9701179", + "_airbyte_extracted_at": "2023-01-01T01:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "string": "bar" + } + } + """))); + final StreamConfig streamConfig = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.empty(), + COLUMNS); + + final String sql = generator.updateTable(streamConfig, ""); + destinationHandler.execute(sql); + + final List actualRawRecords = dumpRawTableRecords(streamId); + final List actualFinalRecords = dumpFinalTableRecords(streamId, ""); + verifyRecordCounts( + 1, + actualRawRecords, + 1, + actualFinalRecords); + assertAll( + () -> assertEquals("bar", actualRawRecords.get(0).get("_airbyte_data").get("string").asText()), + () -> assertEquals("bar", actualFinalRecords.get(0).get("string").asText())); + } + + @Test + public void incrementalAppend() throws Exception { + createRawTable(streamId); + createFinalTable(false, streamId, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/incrementaldedup_inputrecords.jsonl")); + + final String sql = generator.updateTable(incrementalAppendStream, ""); + destinationHandler.execute(sql); + + verifyRecordCounts( + 3, + dumpRawTableRecords(streamId), + 3, + dumpFinalTableRecords(streamId, "")); + } + + /** + * Create a nonempty users_final_tmp table. Overwrite users_final from users_final_tmp. Verify that + * users_final now exists and contains nonzero records. + */ + @Test + public void overwriteFinalTable() throws Exception { + createFinalTable(false, streamId, "_tmp"); + final List records = singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_meta": {} + } + """)); + insertFinalTableRecords( + false, + streamId, + "_tmp", + records); + + final String sql = generator.overwriteFinalTable(streamId, "_tmp"); + destinationHandler.execute(sql); + + assertEquals(1, dumpFinalTableRecords(streamId, "").size()); + } + + @Test + public void cdcImmediateDeletion() throws Exception { + createRawTable(streamId); + createFinalTable(true, streamId, ""); + insertRawTableRecords( + streamId, + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "updated_at": "2023-01-01T00:00:00Z", + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + } + """))); + + final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); + destinationHandler.execute(sql); + + verifyRecordCounts( + 1, + dumpRawTableRecords(streamId), + 0, + dumpFinalTableRecords(streamId, "")); + } + + /** + * Verify that running T+D twice is idempotent. Previously there was a bug where non-dedup syncs + * with an _ab_cdc_deleted_at column would duplicate "deleted" records on each run. + */ + @Test + public void cdcIdempotent() throws Exception { + createRawTable(streamId); + createFinalTable(true, streamId, ""); + insertRawTableRecords( + streamId, + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "updated_at": "2023-01-01T00:00:00Z", + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + } + """))); + + final String sql = generator.updateTable(cdcIncrementalAppendStream, ""); + // Execute T+D twice + destinationHandler.execute(sql); + destinationHandler.execute(sql); + + verifyRecordCounts( + 1, + dumpRawTableRecords(streamId), + 1, + dumpFinalTableRecords(streamId, "")); + } + + @Test + public void cdcComplexUpdate() throws Exception { + createRawTable(streamId); + createFinalTable(true, streamId, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcupdate_inputrecords_raw.jsonl")); + insertFinalTableRecords( + true, + streamId, + "", + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcupdate_inputrecords_final.jsonl")); + + final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); + destinationHandler.execute(sql); + + verifyRecordCounts( + // We keep the newest raw record per PK + 7, + dumpRawTableRecords(streamId), + 5, + dumpFinalTableRecords(streamId, "")); + } + + /** + * source operations: + *

      + *
    1. insert id=1 (lsn 10000)
      2. + *
    2. delete id=1 (lsn 10001)
      3. + *
    + *

    + * But the destination writes lsn 10001 before 10000. We should still end up with no records in the + * final table. + *

    + * All records have the same emitted_at timestamp. This means that we live or die purely based on + * our ability to use _ab_cdc_lsn. + */ + @Test + public void testCdcOrdering_updateAfterDelete() throws Exception { + createRawTable(streamId); + createFinalTable(true, streamId, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl")); + + final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); + destinationHandler.execute(sql); + + verifyRecordCounts( + 1, + dumpRawTableRecords(streamId), + 0, + dumpFinalTableRecords(streamId, "")); + } + + /** + * source operations: + *

      + *
    1. arbitrary history...
      2. + *
    2. delete id=1 (lsn 10001)
      3. + *
    3. reinsert id=1 (lsn 10002)
      4. + *
    + *

    + * But the destination receives LSNs 10002 before 10001. In this case, we should keep the reinserted + * record in the final table. + *

    + * All records have the same emitted_at timestamp. This means that we live or die purely based on + * our ability to use _ab_cdc_lsn. + */ + @Test + public void testCdcOrdering_insertAfterDelete() throws Exception { + createRawTable(streamId); + createFinalTable(true, streamId, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl")); + insertFinalTableRecords( + true, + streamId, + "", + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl")); + + final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); + destinationHandler.execute(sql); + + verifyRecordCounts( + 1, + dumpRawTableRecords(streamId), + 1, + dumpFinalTableRecords(streamId, "")); + } + + /** + * Create a table which includes the _ab_cdc_deleted_at column, then soft reset it using the non-cdc + * stream config. Verify that the deleted_at column gets dropped. + */ + @Test + public void softReset() throws Exception { + createRawTable(streamId); + createFinalTable(true, streamId, ""); + insertRawTableRecords( + streamId, + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "arst", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_loaded_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + } + """))); + insertFinalTableRecords( + true, + streamId, + "", + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "arst", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_meta": {}, + "id1": 1, + "id2": 100, + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + """))); + + final String sql = generator.softReset(incrementalAppendStream); + destinationHandler.execute(sql); + + final List actualRawRecords = dumpRawTableRecords(streamId); + final List actualFinalRecords = dumpFinalTableRecords(streamId, ""); + assertAll( + () -> assertEquals(1, actualRawRecords.size()), + () -> assertEquals(1, actualFinalRecords.size()), + () -> assertTrue( + actualFinalRecords.stream().noneMatch(record -> record.has("_ab_cdc_deleted_at")), + "_ab_cdc_deleted_at column was expected to be dropped. Actual final table had: " + actualFinalRecords)); + } + + @Test + public void weirdColumnNames() throws Exception { + createRawTable(streamId); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl")); + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + new LinkedHashMap<>() { + + { + put(generator.buildColumnId("id1"), AirbyteProtocolType.INTEGER); + put(generator.buildColumnId("id2"), AirbyteProtocolType.INTEGER); + put(generator.buildColumnId("updated_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + put(generator.buildColumnId("$starts_with_dollar_sign"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes\"doublequote"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes'singlequote"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes`backtick"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes.period"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes$$doubledollar"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("endswithbackslash\\"), AirbyteProtocolType.STRING); + } + + }); + + final String createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + final String updateTable = generator.updateTable(stream, ""); + destinationHandler.execute(updateTable); + + verifyRecords( + "sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * Verify that we don't crash when there are special characters in the stream namespace, name, + * primary key, or cursor. + */ + @ParameterizedTest + @ValueSource(strings = {"$", "\"", "'", "`", ".", "$$", "\\"}) + public void noCrashOnSpecialCharacters(final String specialChars) throws Exception { + final String str = namespace + "_" + specialChars; + final StreamId originalStreamId = generator.buildStreamId(str, str, "unused"); + final StreamId modifiedStreamId = new StreamId( + originalStreamId.finalNamespace(), + originalStreamId.finalName(), + // hack for testing simplicity: put the raw tables in the final namespace. This makes cleanup + // easier. + originalStreamId.finalNamespace(), + "raw_table", + null, + null); + final ColumnId columnId = generator.buildColumnId(str); + try { + createNamespace(modifiedStreamId.finalNamespace()); + createRawTable(modifiedStreamId); + insertRawTableRecords( + modifiedStreamId, + List.of(Jsons.jsonNode(Map.of( + "_airbyte_raw_id", "758989f2-b148-4dd3-8754-30d9c17d05fb", + "_airbyte_extracted_at", "2023-01-01T00:00:00Z", + "_airbyte_data", Map.of(str, "bar"))))); + final StreamConfig stream = new StreamConfig( + modifiedStreamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + List.of(columnId), + Optional.of(columnId), + new LinkedHashMap<>() { + + { + put(columnId, AirbyteProtocolType.STRING); + } + + }); + + final String createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + final String updateTable = generator.updateTable(stream, ""); + // Not verifying anything about the data; let's just make sure we don't crash. + destinationHandler.execute(updateTable); + } finally { + teardownNamespace(modifiedStreamId.finalNamespace()); + } + } + + /** + * A stream with no columns is weird, but we shouldn't treat it specially in any way. It should + * create a final table as usual, and populate it with the relevant metadata columns. + */ + @Test + public void noColumns() throws Exception { + createRawTable(streamId); + insertRawTableRecords( + streamId, + List.of(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": {} + } + """))); + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + emptyList(), + Optional.empty(), + new LinkedHashMap<>()); + + final String createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + final String updateTable = generator.updateTable(stream, ""); + destinationHandler.execute(updateTable); + + verifyRecords( + "sqlgenerator/nocolumns_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/nocolumns_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + @Test + public void testV1V2migration() throws Exception { + // This is maybe a little hacky, but it avoids having to refactor this entire class and subclasses + // for something that is going away + final StreamId v1RawTableStreamId = new StreamId(null, null, streamId.finalNamespace(), "v1_" + streamId.rawName(), null, null); + createV1RawTable(v1RawTableStreamId); + insertV1RawTableRecords(v1RawTableStreamId, BaseTypingDedupingTest.readRecords("sqlgenerator/all_types_v1_inputrecords.jsonl")); + final String migration = generator.migrateFromV1toV2(streamId, v1RawTableStreamId.rawNamespace(), v1RawTableStreamId.rawName()); + destinationHandler.execute(migration); + final List v1RawRecords = dumpV1RawTableRecords(v1RawTableStreamId); + final List v2RawRecords = dumpRawTableRecords(streamId); + migrationAssertions(v1RawRecords, v2RawRecords); + } + + protected void migrationAssertions(final List v1RawRecords, final List v2RawRecords) { + final var v2RecordMap = v2RawRecords.stream().collect(Collectors.toMap( + record -> record.get("_airbyte_raw_id").asText(), + Function.identity())); + assertAll( + () -> assertEquals(5, v1RawRecords.size()), + () -> assertEquals(5, v2RawRecords.size())); + v1RawRecords.forEach(v1Record -> { + final var v1id = v1Record.get("_airbyte_ab_id").asText(); + assertAll( + () -> assertEquals(v1id, v2RecordMap.get(v1id).get("_airbyte_raw_id").asText()), + () -> assertEquals(v1Record.get("_airbyte_emitted_at").asText(), v2RecordMap.get(v1id).get("_airbyte_extracted_at").asText()), + () -> assertNull(v2RecordMap.get(v1id).get("_airbyte_loaded_at"))); + final JsonNode originalData = v1Record.get("_airbyte_data"); + JsonNode migratedData = v2RecordMap.get(v1id).get("_airbyte_data"); + if (migratedData.isTextual()) { + migratedData = Jsons.deserializeExact(migratedData.asText()); + } + // hacky thing because we only care about the data contents. + // diffRawTableRecords makes some assumptions about the structure of the blob. + DIFFER.diffFinalTableRecords(List.of(originalData), List.of(migratedData)); + }); + } + + protected List dumpV1RawTableRecords(final StreamId streamId) throws Exception { + return dumpRawTableRecords(streamId); + } + + @Test + public void testCreateTableForce() throws Exception { + final String createTableNoForce = generator.createTable(incrementalDedupStream, "", false); + final String createTableForce = generator.createTable(incrementalDedupStream, "", true); + + destinationHandler.execute(createTableNoForce); + assertThrows(Exception.class, () -> destinationHandler.execute(createTableNoForce)); + // This should not throw an exception + destinationHandler.execute(createTableForce); + + assertTrue(destinationHandler.findExistingTable(streamId).isPresent()); + } + + private void verifyRecords(final String expectedRawRecordsFile, + final List actualRawRecords, + final String expectedFinalRecordsFile, + final List actualFinalRecords) { + assertAll( + () -> DIFFER.diffRawTableRecords( + BaseTypingDedupingTest.readRecords(expectedRawRecordsFile), + actualRawRecords), + () -> assertEquals( + 0, + actualRawRecords.stream() + .filter(record -> !record.hasNonNull("_airbyte_loaded_at")) + .count()), + () -> DIFFER.diffFinalTableRecords( + BaseTypingDedupingTest.readRecords(expectedFinalRecordsFile), + actualFinalRecords)); + } + + private void verifyRecordCounts(final int expectedRawRecords, + final List actualRawRecords, + final int expectedFinalRecords, + final List actualFinalRecords) { + assertAll( + () -> assertEquals( + expectedRawRecords, + actualRawRecords.size(), + "Raw record count was incorrect"), + () -> assertEquals( + 0, + actualRawRecords.stream() + .filter(record -> !record.hasNonNull("_airbyte_loaded_at")) + .count()), + () -> assertEquals( + expectedFinalRecords, + actualFinalRecords.size(), + "Final record count was incorrect")); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java new file mode 100644 index 000000000000..48ca40f32f88 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java @@ -0,0 +1,670 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.lang.Exceptions; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.configoss.WorkerDestinationConfig; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import io.airbyte.workers.internal.AirbyteDestination; +import io.airbyte.workers.internal.DefaultAirbyteDestination; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessFactory; +import io.airbyte.workers.process.ProcessFactory; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is loosely based on standard-destination-tests's DestinationAcceptanceTest class. The + * sync-running code is copy-pasted from there. + *

    + * All tests use a single stream, whose schema is defined in {@code resources/schema.json}. Each + * test case constructs a ConfiguredAirbyteCatalog dynamically. + *

    + * For sync modes which use a primary key, the stream provides a composite key of (id1, id2). For + * sync modes which use a cursor, the stream provides an updated_at field. The stream also has an + * _ab_cdc_deleted_at field. + */ +// If you're running from inside intellij, you must run your specific subclass to get concurrent +// execution. +@Execution(ExecutionMode.CONCURRENT) +public abstract class BaseTypingDedupingTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseTypingDedupingTest.class); + protected static final JsonNode SCHEMA; + static { + try { + SCHEMA = Jsons.deserialize(MoreResources.readResource("dat/schema.json")); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + private static final RecordDiffer DIFFER = new RecordDiffer( + Pair.of("id1", AirbyteProtocolType.INTEGER), + Pair.of("id2", AirbyteProtocolType.INTEGER), + Pair.of("updated_at", AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE), + Pair.of("old_cursor", AirbyteProtocolType.INTEGER)); + + private String randomSuffix; + private JsonNode config; + protected String streamNamespace; + protected String streamName; + private List streamsToTearDown; + + /** + * @return the docker image to run, e.g. {@code "airbyte/destination-bigquery:dev"}. + */ + protected abstract String getImageName(); + + /** + * Get the destination connector config. Subclasses may use this method for other setup work, e.g. + * opening a connection to the destination. + *

    + * Subclasses should _not_ start testcontainers in this method; that belongs in a BeforeAll method. + * The tests in this class are intended to be run concurrently on a shared database and will not + * interfere with each other. + *

    + * Sublcasses which need access to the config may use {@link #getConfig()}. + */ + protected abstract JsonNode generateConfig() throws Exception; + + /** + * For a given stream, return the records that exist in the destination's raw table. Each record + * must be in the format {"_airbyte_raw_id": "...", "_airbyte_extracted_at": "...", + * "_airbyte_loaded_at": "...", "_airbyte_data": {fields...}}. + *

    + * The {@code _airbyte_data} column must be an + * {@link com.fasterxml.jackson.databind.node.ObjectNode} (i.e. it cannot be a string value). + *

    + * streamNamespace may be null, in which case you should query from the default namespace. + */ + protected abstract List dumpRawTableRecords(String streamNamespace, String streamName) throws Exception; + + /** + * For a given stream, return the records that exist in the destination's final table. Each record + * must be in the format {"_airbyte_raw_id": "...", "_airbyte_extracted_at": "...", "_airbyte_meta": + * {...}, "field1": ..., "field2": ..., ...}. + *

    + * For JSON-valued columns, there is some nuance: a SQL null should be represented as a missing + * entry, whereas a JSON null should be represented as a + * {@link com.fasterxml.jackson.databind.node.NullNode}. For example, in the JSON blob {"name": + * null}, the `name` field is a JSON null, and the `address` field is a SQL null. + *

    + * The corresponding SQL looks like + * {@code INSERT INTO ... (name, address) VALUES ('null' :: jsonb, NULL)}. + *

    + * streamNamespace may be null, in which case you should query from the default namespace. + */ + protected abstract List dumpFinalTableRecords(String streamNamespace, String streamName) throws Exception; + + /** + * Delete any resources in the destination associated with this stream AND its namespace. We need + * this because we write raw tables to a shared {@code airbyte} namespace, which we can't drop + * wholesale. Must handle the case where the table/namespace doesn't exist (e.g. if the connector + * crashed without writing any data). + *

    + * In general, this should resemble + * {@code DROP TABLE IF EXISTS airbyte._; DROP SCHEMA IF EXISTS }. + */ + protected abstract void teardownStreamAndNamespace(String streamNamespace, String streamName) throws Exception; + + /** + * Destinations which need to clean up resources after an entire test finishes should override this + * method. For example, if you want to gracefully close a database connection, you should do that + * here. + */ + protected void globalTeardown() throws Exception {} + + /** + * @return A suffix which is different for each concurrent test, but stable within a single test. + */ + protected synchronized String getUniqueSuffix() { + if (randomSuffix == null) { + randomSuffix = "_" + RandomStringUtils.randomAlphabetic(5).toLowerCase(); + } + return randomSuffix; + } + + protected JsonNode getConfig() { + return config; + } + + @BeforeEach + public void setup() throws Exception { + config = generateConfig(); + streamNamespace = "typing_deduping_test" + getUniqueSuffix(); + streamName = "test_stream" + getUniqueSuffix(); + streamsToTearDown = new ArrayList<>(); + LOGGER.info("Using stream namespace {} and name {}", streamNamespace, streamName); + } + + @AfterEach + public void teardown() throws Exception { + for (final AirbyteStreamNameNamespacePair streamId : streamsToTearDown) { + teardownStreamAndNamespace(streamId.getNamespace(), streamId.getName()); + } + globalTeardown(); + } + + /** + * Starting with an empty destination, execute a full refresh overwrite sync. Verify that the + * records are written to the destination table. Then run a second sync, and verify that the records + * are overwritten. + */ + @Test + public void fullRefreshOverwrite() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + /** + * Starting with an empty destination, execute a full refresh append sync. Verify that the records + * are written to the destination table. Then run a second sync, and verify that the old and new + * records are all present. + */ + @Test + public void fullRefreshAppend() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + /** + * Starting with an empty destination, execute an incremental append sync. + *

    + * This is (not so secretly) identical to {@link #fullRefreshAppend()}, and uses the same set of + * expected records. Incremental as a concept only exists in the source. From the destination's + * perspective, we only care about the destination sync mode. + */ + @Test + public void incrementalAppend() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + // These two lines are literally the only difference between this test and fullRefreshAppend + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + /** + * Starting with an empty destination, execute an incremental dedup sync. Verify that the records + * are written to the destination table. Then run a second sync, and verify that the raw/final + * tables contain the correct records. + */ + @Test + public void incrementalDedup() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_dedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + /** + * Identical to {@link #incrementalDedup()}, except that the stream has no namespace. + */ + @Test + public void incrementalDedupDefaultNamespace() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + // NB: we don't call `withNamespace` here + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl", null, streamName); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_dedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, null, streamName); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl", null, streamName); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, null, streamName); + } + + @Test + @Disabled("Not yet implemented") + public void testLineBreakCharacters() throws Exception { + // TODO verify that we can handle strings with interesting characters + // build an airbyterecordmessage using something like this, and add it to the input messages: + Jsons.jsonNode(ImmutableMap.builder() + .put("id", 1) + .put("currency", "USD\u2028") + .put("date", "2020-03-\n31T00:00:00Z\r") + // TODO(sherifnada) hack: write decimals with sigfigs because Snowflake stores 10.1 as "10" which + // fails destination tests + .put("HKD", 10.1) + .put("NZD", 700.1) + .build()); + } + + /** + * Run a sync, then remove the {@code name} column from the schema and run a second sync. Verify + * that the final table doesn't contain the `name` column after the second sync. + */ + @Test + public void testIncrementalSyncDropOneColumn() throws Exception { + final AirbyteStream stream = new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(stream))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + final JsonNode trimmedSchema = SCHEMA.deepCopy(); + ((ObjectNode) trimmedSchema.get("properties")).remove("name"); + stream.setJsonSchema(trimmedSchema); + + runSync(catalog, messages2); + + // The raw data is unaffected by the schema, but the final table should not have a `name` column. + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl").stream() + .peek(record -> ((ObjectNode) record).remove("name")) + .toList(); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + @Test + @Disabled("Not yet implemented") + public void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { + // TODO duplicate this test for each sync mode. Run 1st+2nd syncs using a stream with null + // namespace: + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(null) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + } + + // TODO duplicate this test for each sync mode. Run 1st+2nd syncs using two streams with the same + // name but different namespace + // TODO maybe we don't even need the single-stream versions... + /** + * Identical to {@link #incrementalDedup()}, except there are two streams with the same name and + * different namespace. + */ + @Test + public void incrementalDedupIdenticalName() throws Exception { + final String namespace1 = streamNamespace + "_1"; + final String namespace2 = streamNamespace + "_2"; + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(namespace1) + .withName(streamName) + .withJsonSchema(SCHEMA)), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(namespace2) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + // Read the same set of messages for both streams + final List messages1 = Stream.concat( + readMessages("dat/sync1_messages.jsonl", namespace1, streamName).stream(), + readMessages("dat/sync1_messages.jsonl", namespace2, streamName).stream()).toList(); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_dedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, namespace1, streamName); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, namespace2, streamName); + + // Second sync + final List messages2 = Stream.concat( + readMessages("dat/sync2_messages.jsonl", namespace1, streamName).stream(), + readMessages("dat/sync2_messages.jsonl", namespace2, streamName).stream()).toList(); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, namespace1, streamName); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, namespace2, streamName); + } + + @Test + @Disabled("Not yet implemented") + public void testSyncNotFailsWithNewFields() throws Exception { + // TODO duplicate this test for each sync mode. Run a sync, then add a new field to the schema, then + // run another sync + // We might want to write a test that verifies more general schema evolution (e.g. all valid + // evolutions) + } + + /** + * Change the cursor column in the second sync to a column that doesn't exist in the first sync. + * Verify that we overwrite everything correctly. + *

    + * This essentially verifies that the destination connector correctly recognizes NULL cursors as + * older than non-NULL cursors. + */ + @Test + public void incrementalDedupChangeCursor() throws Exception { + final JsonNode mangledSchema = SCHEMA.deepCopy(); + ((ObjectNode) mangledSchema.get("properties")).remove("updated_at"); + ((ObjectNode) mangledSchema.get("properties")).set( + "old_cursor", + Jsons.deserialize( + """ + {"type": "integer"} + """)); + final ConfiguredAirbyteStream configuredStream = new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("old_cursor")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(mangledSchema)); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(configuredStream)); + + // First sync + final List messages1 = readMessages("dat/sync1_cursorchange_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + configuredStream.getStream().setJsonSchema(SCHEMA); + configuredStream.setCursorField(List.of("updated_at")); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + @Test + @Disabled("Not yet implemented") + public void testSyncWithLargeRecordBatch() throws Exception { + // TODO duplicate this test for each sync mode. Run a single sync with many records + /* + * copied from DATs: This serves to test MSSQL 2100 limit parameters in a single query. this means + * that for Airbyte insert data need to limit to ~ 700 records (3 columns for the raw tables) = 2100 + * params + * + * this maybe needs configuration per destination to specify that limit? + */ + } + + @Test + @Disabled("Not yet implemented") + public void testDataTypes() throws Exception { + // TODO duplicate this test for each sync mode. See DataTypeTestArgumentProvider for what this test + // does in DAT-land + // we probably don't want to do the exact same thing, but the general spirit of testing a wide range + // of values for every data type is approximately correct + // this test probably needs some configuration per destination to specify what values are supported? + } + + protected void verifySyncResult(final List expectedRawRecords, final List expectedFinalRecords) throws Exception { + verifySyncResult(expectedRawRecords, expectedFinalRecords, streamNamespace, streamName); + } + + private void verifySyncResult(final List expectedRawRecords, + final List expectedFinalRecords, + final String streamNamespace, + final String streamName) + throws Exception { + final List actualRawRecords = dumpRawTableRecords(streamNamespace, streamName); + final List actualFinalRecords = dumpFinalTableRecords(streamNamespace, streamName); + DIFFER.verifySyncResult(expectedRawRecords, actualRawRecords, expectedFinalRecords, actualFinalRecords); + } + + public static List readRecords(final String filename) throws IOException { + return MoreResources.readResource(filename).lines() + .map(String::trim) + .filter(line -> !line.isEmpty()) + .filter(line -> !line.startsWith("//")) + .map(Jsons::deserializeExact) + .toList(); + } + + protected List readMessages(final String filename) throws IOException { + return readMessages(filename, streamNamespace, streamName); + } + + private static List readMessages(final String filename, final String streamNamespace, final String streamName) throws IOException { + return readRecords(filename).stream() + .map(record -> Jsons.convertValue(record, AirbyteMessage.class)) + .peek(message -> { + message.getRecord().setNamespace(streamNamespace); + message.getRecord().setStream(streamName); + }).toList(); + } + + /* + * !!!!!! WARNING !!!!!! The code below was mostly copypasted from DestinationAcceptanceTest. If you + * make edits here, you probably want to also edit there. + */ + + // These contain some state, so they are instanced per test (i.e. cannot be static) + private Path jobRoot; + private ProcessFactory processFactory; + + @BeforeEach + public void setupProcessFactory() throws IOException { + final Path testDir = Path.of("/tmp/airbyte_tests/"); + Files.createDirectories(testDir); + final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); + jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); + final Path localRoot = Files.createTempDirectory(testDir, "output"); + processFactory = new DockerProcessFactory( + workspaceRoot, + workspaceRoot.toString(), + localRoot.toString(), + "host", + Collections.emptyMap()); + } + + protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages) throws Exception { + runSync(catalog, messages, getImageName()); + } + + protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages, final String imageName) throws Exception { + catalog.getStreams().forEach(s -> streamsToTearDown.add(AirbyteStreamNameNamespacePair.fromAirbyteStream(s.getStream()))); + + final WorkerDestinationConfig destinationConfig = new WorkerDestinationConfig() + .withConnectionId(UUID.randomUUID()) + .withCatalog(convertProtocolObject(catalog, io.airbyte.protocol.models.ConfiguredAirbyteCatalog.class)) + .withDestinationConnectionConfiguration(config); + + final AirbyteDestination destination = new DefaultAirbyteDestination(new AirbyteIntegrationLauncher( + "0", + 0, + imageName, + processFactory, + null, + null, + false, + new EnvVariableFeatureFlags())); + + destination.start(destinationConfig, jobRoot, Collections.emptyMap()); + messages.forEach( + message -> Exceptions.toRuntime(() -> destination.accept(convertProtocolObject(message, io.airbyte.protocol.models.AirbyteMessage.class)))); + destination.notifyEndOfInput(); + + while (!destination.isFinished()) { + destination.attemptRead(); + } + + destination.close(); + } + + private static V0 convertProtocolObject(final V1 v1, final Class klass) { + return Jsons.object(Jsons.jsonNode(v1), klass); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java new file mode 100644 index 000000000000..058986346d29 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static java.util.stream.Collectors.toList; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Streams; +import io.airbyte.commons.json.Jsons; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetTime; +import java.time.ZoneOffset; +import java.util.Arrays; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Utility class to generate human-readable diffs between expected and actual records. Assumes 1s1t + * output format. + */ +public class RecordDiffer { + + private final Comparator recordIdentityComparator; + private final Comparator recordSortComparator; + private final Function recordIdentityExtractor; + + /** + * @param identifyingColumns Which fields constitute a unique record (typically PK+cursor). Do _not_ + * include extracted_at; it is handled automatically. + */ + @SafeVarargs + public RecordDiffer(final Pair... identifyingColumns) { + this.recordIdentityComparator = buildIdentityComparator(identifyingColumns); + this.recordSortComparator = recordIdentityComparator.thenComparing(record -> asString(record.get("_airbyte_raw_id"))); + this.recordIdentityExtractor = buildIdentityExtractor(identifyingColumns); + } + + /** + * In the expected records, a SQL null is represented as a JsonNode without that field at all, and a + * JSON null is represented as a NullNode. For example, in the JSON blob {"name": null}, the `name` + * field is a JSON null, and the `address` field is a SQL null. + */ + public void verifySyncResult(final List expectedRawRecords, + final List actualRawRecords, + final List expectedFinalRecords, + final List actualFinalRecords) { + assertAll( + () -> diffRawTableRecords(expectedRawRecords, actualRawRecords), + () -> diffFinalTableRecords(expectedFinalRecords, actualFinalRecords)); + } + + public void diffRawTableRecords(final List expectedRecords, final List actualRecords) { + final String diff = diffRecords( + expectedRecords.stream().map(RecordDiffer::copyWithLiftedData).collect(toList()), + actualRecords.stream().map(RecordDiffer::copyWithLiftedData).collect(toList()), + recordIdentityComparator, + recordSortComparator, + recordIdentityExtractor); + + if (!diff.isEmpty()) { + fail("Raw table was incorrect.\n" + diff); + } + } + + public void diffFinalTableRecords(final List expectedRecords, final List actualRecords) { + final String diff = diffRecords( + expectedRecords, + actualRecords, + recordIdentityComparator, + recordSortComparator, + recordIdentityExtractor); + + if (!diff.isEmpty()) { + fail("Final table was incorrect.\n" + diff); + } + } + + /** + * Lift _airbyte_data fields to the root level. If _airbyte_data is a string, deserialize it first. + * + * @return A copy of the record, but with all fields in _airbyte_data lifted to the top level. + */ + private static JsonNode copyWithLiftedData(final JsonNode record) { + final ObjectNode copy = record.deepCopy(); + copy.remove("_airbyte_data"); + JsonNode airbyteData = record.get("_airbyte_data"); + if (airbyteData.isTextual()) { + airbyteData = Jsons.deserializeExact(airbyteData.asText()); + } + Streams.stream(airbyteData.fields()).forEach(field -> { + if (!copy.has(field.getKey())) { + copy.set(field.getKey(), field.getValue()); + } else { + // This would only happen if the record has one of the metadata columns (e.g. _airbyte_raw_id) + // We don't support that in production, so we don't support it here either. + throw new RuntimeException("Cannot lift field " + field.getKey() + " because it already exists in the record."); + } + }); + return copy; + } + + /** + * Build a Comparator to detect equality between two records. It first compares all the identifying + * columns in order, and breaks ties using extracted_at. + */ + private Comparator buildIdentityComparator(final Pair[] identifyingColumns) { + // Start with a noop comparator for convenience + Comparator comp = Comparator.comparing(record -> 0); + for (final Pair column : identifyingColumns) { + comp = comp.thenComparing(record -> extract(record, column.getKey(), column.getValue())); + } + comp = comp.thenComparing(record -> asTimestampWithTimezone(record.get("_airbyte_extracted_at"))); + return comp; + } + + /** + * See {@link #buildIdentityComparator(Pair[])} for an explanation of dataExtractor. + */ + private Function buildIdentityExtractor(final Pair[] identifyingColumns) { + return record -> Arrays.stream(identifyingColumns) + .map(column -> getPrintableFieldIfPresent(record, column.getKey())) + .collect(Collectors.joining(", ")) + + getPrintableFieldIfPresent(record, "_airbyte_extracted_at"); + } + + private static String getPrintableFieldIfPresent(final JsonNode record, final String field) { + if (record.has(field)) { + return field + "=" + record.get(field); + } else { + return ""; + } + } + + /** + * Generate a human-readable diff between the two lists. Assumes (in general) that two records with + * the same PK, cursor, and extracted_at are the same record. + *

    + * Verifies that all values specified in the expected records are correct (_including_ raw_id), and + * that no other fields are present (except for loaded_at and raw_id). We assume that it's + * impossible to verify loaded_at, since it's generated dynamically; however, we do provide the + * ability to assert on the exact raw_id if desired; we simply assume that raw_id is always expected + * to be present. + * + * @param identityComparator Returns 0 iff two records are the "same" record (i.e. have the same + * PK+cursor+extracted_at) + * @param sortComparator Behaves identically to identityComparator, but if two records are the same, + * breaks that tie using _airbyte_raw_id + * @param recordIdExtractor Dump the record's PK+cursor+extracted_at into a human-readable string + * @return The diff, or empty string if there were no differences + */ + private static String diffRecords(final List originalExpectedRecords, + final List originalActualRecords, + final Comparator identityComparator, + final Comparator sortComparator, + final Function recordIdExtractor) { + final List expectedRecords = originalExpectedRecords.stream().sorted(sortComparator).toList(); + final List actualRecords = originalActualRecords.stream().sorted(sortComparator).toList(); + + // Iterate through both lists in parallel and compare each record. + // Build up an error message listing any incorrect, missing, or unexpected records. + String message = ""; + int expectedRecordIndex = 0; + int actualRecordIndex = 0; + while (expectedRecordIndex < expectedRecords.size() && actualRecordIndex < actualRecords.size()) { + final JsonNode expectedRecord = expectedRecords.get(expectedRecordIndex); + final JsonNode actualRecord = actualRecords.get(actualRecordIndex); + final int compare = identityComparator.compare(expectedRecord, actualRecord); + if (compare == 0) { + // These records should be the same. Find the specific fields that are different and move on + // to the next records in both lists. + message += diffSingleRecord(recordIdExtractor, expectedRecord, actualRecord); + expectedRecordIndex++; + actualRecordIndex++; + } else if (compare < 0) { + // The expected record is missing from the actual records. Print it and move on to the next expected + // record. + message += "Row was expected but missing: " + expectedRecord + "\n"; + expectedRecordIndex++; + } else { + // There's an actual record which isn't present in the expected records. Print it and move on to the + // next actual record. + message += "Row was not expected but present: " + actualRecord + "\n"; + actualRecordIndex++; + } + } + // Tail loops in case we reached the end of one list before the other. + while (expectedRecordIndex < expectedRecords.size()) { + message += "Row was expected but missing: " + expectedRecords.get(expectedRecordIndex) + "\n"; + expectedRecordIndex++; + } + while (actualRecordIndex < actualRecords.size()) { + message += "Row was not expected but present: " + actualRecords.get(actualRecordIndex) + "\n"; + actualRecordIndex++; + } + + return message; + } + + private static String diffSingleRecord(final Function recordIdExtractor, + final JsonNode expectedRecord, + final JsonNode actualRecord) { + boolean foundMismatch = false; + String mismatchedRecordMessage = "Row had incorrect data: " + recordIdExtractor.apply(expectedRecord) + "\n"; + // Iterate through each column in the expected record and compare it to the actual record's value. + for (final String column : Streams.stream(expectedRecord.fieldNames()).sorted().toList()) { + // For all other columns, we can just compare their values directly. + final JsonNode expectedValue = expectedRecord.get(column); + final JsonNode actualValue = actualRecord.get(column); + if (!areJsonNodesEquivalent(expectedValue, actualValue)) { + mismatchedRecordMessage += generateFieldError("column " + column, expectedValue, actualValue); + foundMismatch = true; + } + } + // Then check the entire actual record for any columns that we weren't expecting. + final LinkedHashMap extraColumns = checkForExtraOrNonNullFields(expectedRecord, actualRecord); + if (extraColumns.size() > 0) { + for (final Map.Entry extraColumn : extraColumns.entrySet()) { + mismatchedRecordMessage += generateFieldError("column " + extraColumn.getKey(), null, extraColumn.getValue()); + foundMismatch = true; + } + } + if (foundMismatch) { + return mismatchedRecordMessage; + } else { + return ""; + } + } + + private static boolean areJsonNodesEquivalent(final JsonNode expectedValue, final JsonNode actualValue) { + if (expectedValue == null || actualValue == null) { + // If one of the values is null, then we expect both of them to be null. + return expectedValue == null && actualValue == null; + } else if (expectedValue instanceof final ArrayNode expectedArrayNode && actualValue instanceof final ArrayNode actualArrayNode) { + // If both values are arrays, compare each of their elements. Order should be preserved + return IntStream.range(0, expectedArrayNode.size()) + .allMatch(i -> areJsonNodesEquivalent(expectedArrayNode.get(i), actualArrayNode.get(i))); + } else if (expectedValue instanceof final ObjectNode expectedObjectNode && actualValue instanceof final ObjectNode actualObjectNode) { + // If both values are objects compare their fields and values + return expectedObjectNode.size() == actualObjectNode.size() && Stream.generate(expectedObjectNode.fieldNames()::next) + .limit(expectedObjectNode.size()) + .allMatch(field -> areJsonNodesEquivalent(expectedObjectNode.get(field), actualObjectNode.get(field))); + } else { + // Otherwise, we need to compare the actual values. + // This is kind of sketchy, but seems to work fine for the data we have in our test cases. + return expectedValue.equals(actualValue) + // equals() expects the two values to be the same class. + // We need to handle comparisons between e.g. LongNode and IntNode. + || (expectedValue.isIntegralNumber() && actualValue.isIntegralNumber() + && expectedValue.bigIntegerValue().equals(actualValue.bigIntegerValue())) + || (expectedValue.isNumber() && actualValue.isNumber() && expectedValue.decimalValue().equals(actualValue.decimalValue())); + } + } + + /** + * Verify that all fields in the actual record are present in the expected record. This is primarily + * relevant for detecting fields that we expected to be null, but actually were not. See + * {@link BaseTypingDedupingTest#dumpFinalTableRecords(String, String)} for an explanation of how + * SQL/JSON nulls are represented in the expected record. + *

    + * This has the side benefit of detecting completely unexpected columns, which would be a very weird + * bug but is probably still useful to catch. + */ + private static LinkedHashMap checkForExtraOrNonNullFields(final JsonNode expectedRecord, final JsonNode actualRecord) { + final LinkedHashMap extraFields = new LinkedHashMap<>(); + for (final String column : Streams.stream(actualRecord.fieldNames()).sorted().toList()) { + // loaded_at and raw_id are generated dynamically, so we just ignore them. + if (!"_airbyte_loaded_at".equals(column) && !"_airbyte_raw_id".equals(column) && !expectedRecord.has(column)) { + extraFields.put(column, actualRecord.get(column)); + } + } + return extraFields; + } + + /** + * Produce a pretty-printed error message, e.g. " For column foo, expected 1 but got 2". The leading + * spaces are intentional, to make the message easier to read when it's embedded in a larger + * stacktrace. + */ + private static String generateFieldError(final String fieldname, final JsonNode expectedValue, final JsonNode actualValue) { + final String expectedString = expectedValue == null ? "SQL NULL (i.e. no value)" : expectedValue.toString(); + final String actualString = actualValue == null ? "SQL NULL (i.e. no value)" : actualValue.toString(); + return " For " + fieldname + ", expected " + expectedString + " but got " + actualString + "\n"; + } + + // These asFoo methods are used for sorting records, so their defaults are intended to make broken + // records stand out. + private static String asString(final JsonNode node) { + if (node == null || node.isNull()) { + return ""; + } else if (node.isTextual()) { + return node.asText(); + } else { + return Jsons.serialize(node); + } + } + + private static BigDecimal asNumber(final JsonNode node) { + if (node == null || !node.isNumber()) { + return new BigDecimal(Double.MIN_VALUE); + } else { + return node.decimalValue(); + } + } + + private static long asInt(final JsonNode node) { + if (node == null || !node.isIntegralNumber()) { + return Long.MIN_VALUE; + } else { + return node.longValue(); + } + } + + private static boolean asBoolean(final JsonNode node) { + if (node == null || !node.isBoolean()) { + return false; + } else { + return node.asBoolean(); + } + } + + private static Instant asTimestampWithTimezone(final JsonNode node) { + if (node == null || !node.isTextual()) { + return Instant.ofEpochMilli(Long.MIN_VALUE); + } else { + try { + return Instant.parse(node.asText()); + } catch (final Exception e) { + return Instant.ofEpochMilli(Long.MIN_VALUE); + } + } + } + + private static LocalDateTime asTimestampWithoutTimezone(final JsonNode node) { + if (node == null || !node.isTextual()) { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.MIN_VALUE), ZoneOffset.UTC); + } else { + try { + return LocalDateTime.parse(node.asText()); + } catch (final Exception e) { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(Long.MIN_VALUE), ZoneOffset.UTC); + } + } + } + + private static OffsetTime asTimeWithTimezone(final JsonNode node) { + if (node == null || !node.isTextual()) { + return OffsetTime.of(0, 0, 0, 0, ZoneOffset.UTC); + } else { + return OffsetTime.parse(node.asText()); + } + } + + private static LocalTime asTimeWithoutTimezone(final JsonNode node) { + if (node == null || !node.isTextual()) { + return LocalTime.of(0, 0, 0); + } else { + try { + return LocalTime.parse(node.asText()); + } catch (final Exception e) { + return LocalTime.of(0, 0, 0); + } + } + } + + private static LocalDate asDate(final JsonNode node) { + if (node == null || !node.isTextual()) { + return LocalDate.ofInstant(Instant.ofEpochMilli(Long.MIN_VALUE), ZoneOffset.UTC); + } else { + try { + return LocalDate.parse(node.asText()); + } catch (final Exception e) { + return LocalDate.ofInstant(Instant.ofEpochMilli(Long.MIN_VALUE), ZoneOffset.UTC); + } + } + } + + // Generics? Never heard of 'em. (I'm sorry) + private static Comparable extract(final JsonNode node, final String field, final AirbyteType type) { + if (type instanceof final AirbyteProtocolType t) { + return switch (t) { + case STRING -> asString(node.get(field)); + case NUMBER -> asNumber(node.get(field)); + case INTEGER -> asInt(node.get(field)); + case BOOLEAN -> asBoolean(node.get(field)); + case TIMESTAMP_WITH_TIMEZONE -> asTimestampWithTimezone(node.get(field)); + case TIMESTAMP_WITHOUT_TIMEZONE -> asTimestampWithoutTimezone(node.get(field)); + case TIME_WITH_TIMEZONE -> asTimeWithTimezone(node.get(field)); + case TIME_WITHOUT_TIMEZONE -> asTimeWithoutTimezone(node.get(field)); + case DATE -> asDate(node.get(field)); + case UNKNOWN -> node.toString(); + }; + } else { + return node.toString(); + } + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/schema.json b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/schema.json new file mode 100644 index 000000000000..c6a19f184a56 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/schema.json @@ -0,0 +1,31 @@ +{ + "type": "object", + "properties": { + "id1": { "type": "integer" }, + "id2": { "type": "integer" }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "_ab_cdc_deleted_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "name": { "type": "string" }, + "address": { + "type": "object", + "properties": { + "city": { "type": "string" }, + "state": { "type": "string" } + } + }, + "age": { "type": "integer" }, + "registration_date": { + "type": "string", + "format": "date", + "airbyte_type": "date" + } + } +} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_cursorchange_messages.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_cursorchange_messages.jsonl new file mode 100644 index 000000000000..e8262c202587 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_cursorchange_messages.jsonl @@ -0,0 +1,4 @@ +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}} +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}} +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}} +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_messages.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_messages.jsonl new file mode 100644 index 000000000000..4c5dec1a24ea --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_messages.jsonl @@ -0,0 +1,12 @@ +// emitted_at:1000 is equal to 1970-01-01 00:00:01Z, which is what you'll see in the expected records. +// This obviously makes no sense in relation to updated_at being in the year 2000, but that's OK +// because (from destinations POV) updated_at has no relation to emitted_at. +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}} +// Emit a second record for id=(1,200) with a different updated_at. This generally doesn't happen +// in full refresh syncs - but if T+D is implemented correctly, it shouldn't matter +// (i.e. both records should be written to the final table). +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}}} +// Emit a record with no _ab_cdc_deleted_at field. CDC sources typically emit an explicit null, but we should handle both cases. +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}}} +// Emit a record with an invalid age. +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync2_messages.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync2_messages.jsonl new file mode 100644 index 000000000000..801ca7267d72 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync2_messages.jsonl @@ -0,0 +1,5 @@ +{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}}} +{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}}} +// Set deleted_at to something non-null. Again, T+D doesn't check the actual _value_ of deleted_at (i.e. the fact that it's in the past is irrelevant). +// It only cares whether deleted_at is non-null. So this should delete Bob from the final table (in dedup mode). +{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/all_types_v1_inputrecords.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/all_types_v1_inputrecords.jsonl new file mode 100644 index 000000000000..71e96f28af46 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/all_types_v1_inputrecords.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_ab_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_ab_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_ab_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +// Note that array and struct have invalid values ({} and [] respectively). +{"_airbyte_ab_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_ab_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/alltypes_inputrecords.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/alltypes_inputrecords.jsonl new file mode 100644 index 000000000000..ca82be9ffdc4 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/alltypes_inputrecords.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}}' +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +// Note that array and struct have invalid values ({} and [] respectively). +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl new file mode 100644 index 000000000000..047f9e9a85f7 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "alice"} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl new file mode 100644 index 000000000000..30a996600d40 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl @@ -0,0 +1,4 @@ +// First batch +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_loaded_at": "2023-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "alice"}} +// Second batch - this is an outdated deletion record, which should be ignored +{"_airbyte_raw_id": "87ff57d7-41a7-4962-a9dc-d684276283da", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T00:00:00Z", "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl new file mode 100644 index 000000000000..0a0c67270d03 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl @@ -0,0 +1,5 @@ +// Write raw deletion record from the first batch, which resulted in an empty final table. +// Note the non-null loaded_at - this is to simulate that we previously ran T+D on this record. +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_loaded_at": "2023-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z"}} +// insert raw record from the second record batch - this is an outdated record that should be ignored. +{"_airbyte_raw_id": "87ff57d7-41a7-4962-a9dc-d684276283da", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T00:00:00Z", "string": "alice"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl new file mode 100644 index 000000000000..304819047e36 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_raw_id": "d5790c04-52df-42f3-8f77-a543268822a7", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_meta": {}, "id1": 1, "id2": 100, "updated_at": "2022-12-31T00:00:00Z", "string": "spooky ghost"} +{"_airbyte_raw_id": "e3b03d92-0f7c-49e5-b203-573dbb7bd1cb", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_meta": {}, "id1": 5, "id2": 100, "updated_at": "2022-12-31T01:00:00Z", "string": "will be deleted"} +{"_airbyte_raw_id": "687718e4-a2a9-4233-80a9-9671f83d61ae", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_meta": {}, "id1": 6, "id2": 100, "updated_at": "2022-12-31T03:00:00Z", "string": "should be untouched"} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl new file mode 100644 index 000000000000..2327710d6e84 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl @@ -0,0 +1,16 @@ +// Records from the first sync (note the non-null loaded_at value) +{"_airbyte_raw_id": "d5790c04-52df-42f3-8f77-a543268822a7", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_loaded_at": "2022-12-31T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2022-12-01T00:00:00Z", "string": "spooky ghost", "_ab_cdc_deleted_at": null}} +{"_airbyte_raw_id": "3593a002-3ab2-4e67-8b4a-e62f0f9a26f9", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_loaded_at": "2022-12-31T00:00:01Z", "_airbyte_data": {"id1": 0, "id2": 100, "updated_at": "2022-12-01T01:00:00Z", "string": "zombie", "_ab_cdc_deleted_at": "2022-12-31T00:00:00Z"}} +{"_airbyte_raw_id": "e3b03d92-0f7c-49e5-b203-573dbb7bd1cb", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_loaded_at": "2022-12-31T00:00:01Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2022-12-01T02:00:00Z", "string": "will be deleted", "_ab_cdc_deleted_at": null}} +{"_airbyte_raw_id": "687718e4-a2a9-4233-80a9-9671f83d61ae", "_airbyte_extracted_at": "2022-12-31T00:00:00Z", "_airbyte_loaded_at": "2022-12-31T00:00:01Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2022-12-01T03:00:00Z", "string": "should be untouched", "_ab_cdc_deleted_at": null}} + +// Records from the second sync +{"_airbyte_raw_id": "5f959152-0db0-44b9-b7e4-0d5c44dc2664", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "_ab_cdc_deleted_at": null, "string": "alice"}} +{"_airbyte_raw_id": "a182ff97-8868-42b9-b3cf-c0753fba55e1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "_ab_cdc_deleted_at": null, "string": "alice2"}} +{"_airbyte_raw_id": "65a6c31f-9ded-4e3d-9339-38ee85b0ae81", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "_ab_cdc_deleted_at": null, "string": "bob"}} +{"_airbyte_raw_id": "f7fffb67-cd05-4cf7-bcd9-00f2fe796168", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T04:00:00Z", "_ab_cdc_deleted_at": "2022-12-31T23:59:59Z"}} +{"_airbyte_raw_id": "4d8674a5-eb6e-41ca-a310-69c64c88d101", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 0, "id2": 100, "updated_at": "2023-01-01T05:00:00Z", "_ab_cdc_deleted_at": null, "string": "zombie_returned"}} +// CDC generally outputs an explicit null for deleted_at, but verify that we can also handle the case where deleted_at is unset. +{"_airbyte_raw_id": "f0b59e49-8c74-4101-9f14-cb4d1193fd5a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T06:00:00Z", "string": "charlie"}} +// Verify that we can handle weird values in deleted_at +{"_airbyte_raw_id": "d4e1d989-c115-403c-9e68-5d320e6376bb", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T07:00:00Z", "_ab_cdc_deleted_at": {}, "string": "david1"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl new file mode 100644 index 000000000000..1d850d9dc74b --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_raw_id": "d7b81af0-01da-4846-a650-cc398986bc99", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "string": "Alice", "struct": {"city": "San Francisco", "state": "CA"}, "integer": 42}} +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/timestampformats_inputrecords.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/timestampformats_inputrecords.jsonl new file mode 100644 index 000000000000..4501ab29a102 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/timestampformats_inputrecords.jsonl @@ -0,0 +1,23 @@ +// types with timezones: +// Z offset +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56Z", "time_with_timezone": "12:34:56Z"}} +// -08:00 offset +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56-08:00", "time_with_timezone": "12:34:56-08:00"}} +// -0800 offset +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56-0800", "time_with_timezone": "12:34:56-0800"}} +// -08 offset +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56-08", "time_with_timezone": "12:34:56-08"}} +// +08:00 offset +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56+08:00", "time_with_timezone": "12:34:56+08:00"}} +// +0800 offset +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56+0800", "time_with_timezone": "12:34:56+0800"}} +// +08 offset +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56+08", "time_with_timezone": "12:34:56+08"}} +// decimal precision +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_with_timezone": "2023-01-23T12:34:56.123Z", "time_with_timezone": "12:34:56.123Z"}} + +// types without timezones: +// basic +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"}} +// decimal precision +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"timestamp_without_timezone": "2023-01-23T12:34:56.123", "time_without_timezone": "12:34:56.123"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl new file mode 100644 index 000000000000..757f7c357a12 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"}} diff --git a/airbyte-integrations/bases/base-typing-deduping/build.gradle b/airbyte-integrations/bases/base-typing-deduping/build.gradle new file mode 100644 index 000000000000..296403745343 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'java-library' +} + +dependencies { + implementation libs.airbyte.protocol + implementation project(path: ':airbyte-integrations:bases:base-java') +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java new file mode 100644 index 000000000000..ab800697b0e1 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Protocol types are ordered by precedence in the case of a Union that contains multiple types. + * Priority is given to wider scope types over narrower ones. (Note that because of dedup logic in + * {@link AirbyteType#fromJsonSchema(JsonNode)}, at most one string or date/time type can exist in a + * Union.) + */ +public enum AirbyteProtocolType implements AirbyteType { + + STRING, + DATE, + TIME_WITHOUT_TIMEZONE, + TIME_WITH_TIMEZONE, + TIMESTAMP_WITHOUT_TIMEZONE, + TIMESTAMP_WITH_TIMEZONE, + NUMBER, + INTEGER, + BOOLEAN, + UNKNOWN; + + private static AirbyteProtocolType matches(final String type) { + try { + return AirbyteProtocolType.valueOf(type.toUpperCase()); + } catch (final IllegalArgumentException e) { + LOGGER.error(String.format("Could not find matching AirbyteProtocolType for \"%s\": %s", type, e)); + return UNKNOWN; + } + } + + // Extracts the appropriate protocol type from the representative JSON + protected static AirbyteProtocolType fromJson(final JsonNode node) { + // JSON could be a string (ex: "number") + if (node.isTextual()) { + return matches(node.asText()); + } + + // or, JSON could be a node with fields + final JsonNode propertyType = node.get("type"); + final JsonNode airbyteType = node.get("airbyte_type"); + final JsonNode format = node.get("format"); + + if (AirbyteType.nodeMatches(propertyType, "boolean")) { + return BOOLEAN; + } else if (AirbyteType.nodeMatches(propertyType, "integer")) { + return INTEGER; + } else if (AirbyteType.nodeMatches(propertyType, "number")) { + return AirbyteType.nodeMatches(airbyteType, "integer") ? INTEGER : NUMBER; + } else if (AirbyteType.nodeMatches(propertyType, "string")) { + if (AirbyteType.nodeMatches(format, "date")) { + return DATE; + } else if (AirbyteType.nodeMatches(format, "time")) { + if (AirbyteType.nodeMatches(airbyteType, "time_without_timezone")) { + return TIME_WITHOUT_TIMEZONE; + } else if (AirbyteType.nodeMatches(airbyteType, "time_with_timezone")) { + return TIME_WITH_TIMEZONE; + } + } else if (AirbyteType.nodeMatches(format, "date-time")) { + if (AirbyteType.nodeMatches(airbyteType, "timestamp_without_timezone")) { + return TIMESTAMP_WITHOUT_TIMEZONE; + } else if (airbyteType == null || AirbyteType.nodeMatches(airbyteType, "timestamp_with_timezone")) { + return TIMESTAMP_WITH_TIMEZONE; + } + } else { + return STRING; + } + } + + return UNKNOWN; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java new file mode 100644 index 000000000000..de59c763ed9c --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public sealed interface AirbyteType permits AirbyteProtocolType,Struct,Array,UnsupportedOneOf,Union { + + Logger LOGGER = LoggerFactory.getLogger(AirbyteType.class); + + /** + * The most common call pattern is probably to use this method on the stream schema, verify that + * it's an {@link Struct} schema, and then call {@link Struct#properties()} to get the columns. + *

    + * If the top-level schema is not an object, then we can't really do anything with it, and should + * probably fail the sync. (but see also {@link Union#asColumns()}). + */ + static AirbyteType fromJsonSchema(final JsonNode schema) { + try { + final JsonNode topLevelType = schema.get("type"); + if (topLevelType != null) { + if (topLevelType.isTextual()) { + if (nodeMatches(topLevelType, "object")) { + return getStruct(schema); + } else if (nodeMatches(topLevelType, "array")) { + return getArray(schema); + } + } else if (topLevelType.isArray()) { + return fromArrayJsonSchema(schema, topLevelType); + } + } else if (schema.hasNonNull("oneOf")) { + final List options = new ArrayList<>(); + schema.get("oneOf").elements().forEachRemaining(element -> options.add(fromJsonSchema(element))); + return new UnsupportedOneOf(options); + } else if (schema.hasNonNull("properties")) { + // The schema has neither type nor oneof, but it does have properties. Assume we're looking at a + // struct. + // This is for backwards-compatibility with legacy normalization. + return getStruct(schema); + } + return AirbyteProtocolType.fromJson(schema); + } catch (final Exception e) { + LOGGER.error("Exception parsing JSON schema {}: {}; returning UNKNOWN.", schema, e); + return AirbyteProtocolType.UNKNOWN; + } + } + + static boolean nodeMatches(final JsonNode node, final String value) { + if (node == null || !node.isTextual()) { + return false; + } + return node.equals(TextNode.valueOf(value)); + } + + private static Struct getStruct(final JsonNode schema) { + final LinkedHashMap propertiesMap = new LinkedHashMap<>(); + final JsonNode properties = schema.get("properties"); + if (properties != null) { + properties.fields().forEachRemaining(property -> { + final String key = property.getKey(); + final JsonNode value = property.getValue(); + propertiesMap.put(key, fromJsonSchema(value)); + }); + } + return new Struct(propertiesMap); + } + + private static Array getArray(final JsonNode schema) { + final JsonNode items = schema.get("items"); + if (items == null) { + return new Array(AirbyteProtocolType.UNKNOWN); + } else { + return new Array(fromJsonSchema(items)); + } + } + + private static AirbyteType fromArrayJsonSchema(final JsonNode schema, final JsonNode array) { + final List typeOptions = new ArrayList<>(); + array.elements().forEachRemaining(element -> { + // ignore "null" type and remove duplicates + final String type = element.asText(""); + if (!"null".equals(type) && !typeOptions.contains(type)) { + typeOptions.add(element.asText()); + } + }); + + // we encounter an array of types that actually represents a single type rather than a Union + if (typeOptions.size() == 1) { + if (typeOptions.get(0).equals("object")) { + return getStruct(schema); + } else if (typeOptions.get(0).equals("array")) { + return getArray(schema); + } else { + return AirbyteProtocolType.fromJson(getTrimmedJsonSchema(schema, typeOptions.get(0))); + } + } + + // Recurse into a schema that forces a specific one of each option + final List options = typeOptions.stream().map(typeOption -> fromJsonSchema(getTrimmedJsonSchema(schema, typeOption))).toList(); + return new Union(options); + } + + // Duplicates the JSON schema but keeps only one type + private static JsonNode getTrimmedJsonSchema(final JsonNode schema, final String type) { + final JsonNode schemaClone = schema.deepCopy(); + // schema is guaranteed to be an object here, because we know it has a `type` key + ((ObjectNode) schemaClone).put("type", type); + return schemaClone; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java new file mode 100644 index 000000000000..64ce0d9fdd78 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Set; +import java.util.stream.Stream; + +public record AlterTableReport(Set columnsToAdd, + Set columnsToRemove, + Set columnsToChangeType, + boolean isDestinationV2Format) { + + /** + * A no-op for an AlterTableReport is when the existing table matches the expected schema + * + * @return whether the schema matches + */ + public boolean isNoOp() { + return isDestinationV2Format && Stream.of(this.columnsToAdd, this.columnsToRemove, this.columnsToChangeType) + .allMatch(Set::isEmpty); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java new file mode 100644 index 000000000000..dccd687e033e --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public record Array(AirbyteType items) implements AirbyteType { + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java new file mode 100644 index 000000000000..d7f2e6ab67ed --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; +import static io.airbyte.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.Collection; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseDestinationV1V2Migrator implements DestinationV1V2Migrator { + + protected static final Logger LOGGER = LoggerFactory.getLogger(BaseDestinationV1V2Migrator.class); + + @Override + public void migrateIfNecessary( + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final StreamConfig streamConfig) + throws TableNotMigratedException, UnexpectedSchemaException { + LOGGER.info("Assessing whether migration is necessary for stream {}", streamConfig.id().finalName()); + if (shouldMigrate(streamConfig)) { + LOGGER.info("Starting v2 Migration for stream {}", streamConfig.id().finalName()); + migrate(sqlGenerator, destinationHandler, streamConfig); + LOGGER.info("V2 Migration completed successfully for stream {}", streamConfig.id().finalName()); + } else { + LOGGER.info("No Migration Required for stream: {}", streamConfig.id().finalName()); + } + + } + + /** + * Determine whether a given stream needs to be migrated from v1 to v2 + * + * @param streamConfig the stream in question + * @return whether to migrate the stream + */ + protected boolean shouldMigrate(final StreamConfig streamConfig) { + final var v1RawTable = convertToV1RawName(streamConfig); + LOGGER.info("Checking whether v1 raw table {} in dataset {} exists", v1RawTable.tableName(), v1RawTable.namespace()); + final var syncModeNeedsMigration = isMigrationRequiredForSyncMode(streamConfig.destinationSyncMode()); + final var noValidV2RawTableExists = !doesValidV2RawTableAlreadyExist(streamConfig); + final var aValidV1RawTableExists = doesValidV1RawTableExist(v1RawTable.namespace(), v1RawTable.tableName()); + LOGGER.info("Migration Info: Required for Sync mode: {}, No existing v2 raw tables: {}, A v1 raw table exists: {}", + syncModeNeedsMigration, noValidV2RawTableExists, aValidV1RawTableExists); + return syncModeNeedsMigration && noValidV2RawTableExists && aValidV1RawTableExists; + } + + /** + * Execute sql statements that converts a v1 raw table to a v2 raw table. Leaves the v1 raw table + * intact + * + * @param sqlGenerator the class which generates dialect specific sql statements + * @param destinationHandler the class which executes the sql statements + * @param streamConfig the stream to migrate the raw table of + */ + public void migrate(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final StreamConfig streamConfig) + throws TableNotMigratedException { + final var namespacedTableName = convertToV1RawName(streamConfig); + try { + destinationHandler.execute(sqlGenerator.migrateFromV1toV2(streamConfig.id(), namespacedTableName.namespace(), namespacedTableName.tableName())); + } catch (Exception e) { + final var message = "Attempted and failed to migrate stream %s".formatted(streamConfig.id().finalName()); + throw new TableNotMigratedException(message, e); + } + } + + /** + * Checks the schema of the v1 raw table to ensure it matches the expected format + * + * @param existingV2AirbyteRawTable the v1 raw table + * @return whether the schema is as expected + */ + private boolean doesV1RawTableMatchExpectedSchema(DialectTableDefinition existingV2AirbyteRawTable) { + + return schemaMatchesExpectation(existingV2AirbyteRawTable, LEGACY_RAW_TABLE_COLUMNS); + } + + /** + * Checks the schema of the v2 raw table to ensure it matches the expected format + * + * @param existingV2AirbyteRawTable the v2 raw table + */ + private void validateAirbyteInternalNamespaceRawTableMatchExpectedV2Schema(DialectTableDefinition existingV2AirbyteRawTable) { + if (!schemaMatchesExpectation(existingV2AirbyteRawTable, V2_RAW_TABLE_COLUMN_NAMES)) { + throw new UnexpectedSchemaException("Destination V2 Raw Table does not match expected Schema"); + } + } + + /** + * If the sync mode is a full refresh and we overwrite the table then there is no need to migrate + * + * @param destinationSyncMode destination sync mode + * @return whether this is full refresh overwrite + */ + private boolean isMigrationRequiredForSyncMode(final DestinationSyncMode destinationSyncMode) { + return !DestinationSyncMode.OVERWRITE.equals(destinationSyncMode); + } + + /** + * Checks if a valid destinations v2 raw table already exists + * + * @param streamConfig the raw table to check + * @return whether it exists and is in the correct format + */ + private boolean doesValidV2RawTableAlreadyExist(final StreamConfig streamConfig) { + if (doesAirbyteInternalNamespaceExist(streamConfig)) { + final var existingV2Table = getTableIfExists(streamConfig.id().rawNamespace(), streamConfig.id().rawName()); + existingV2Table.ifPresent(this::validateAirbyteInternalNamespaceRawTableMatchExpectedV2Schema); + return existingV2Table.isPresent(); + } + return false; + } + + /** + * Checks if a valid v1 raw table already exists + * + * @param namespace + * @param tableName + * @return whether it exists and is in the correct format + */ + protected boolean doesValidV1RawTableExist(final String namespace, final String tableName) { + final var existingV1RawTable = getTableIfExists(namespace, tableName); + return existingV1RawTable.isPresent() && doesV1RawTableMatchExpectedSchema(existingV1RawTable.get()); + } + + /** + * Checks to see if Airbyte's internal schema for destinations v2 exists + * + * @param streamConfig the stream to check + * @return whether the schema exists + */ + abstract protected boolean doesAirbyteInternalNamespaceExist(StreamConfig streamConfig); + + /** + * Checks a Table's schema and compares it to an expected schema to make sure it matches + * + * @param existingTable the table to check + * @param columns the expected schema + * @return whether the existing table schema matches the expectation + */ + abstract protected boolean schemaMatchesExpectation(DialectTableDefinition existingTable, Collection columns); + + /** + * Get a reference ta a table if it exists + * + * @param namespace + * @param tableName + * @return an optional potentially containing a reference to the table + */ + abstract protected Optional getTableIfExists(String namespace, String tableName); + + /** + * We use different naming conventions for raw table names in destinations v2, we need a way to map + * that back to v1 names + * + * @param streamConfig the stream in question + * @return the valid v1 name and namespace for the same stream + */ + abstract protected NamespacedTableName convertToV1RawName(StreamConfig streamConfig); + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java new file mode 100644 index 000000000000..03ec4e06a8f0 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.Optional; +import org.apache.commons.codec.digest.DigestUtils; + +public class CatalogParser { + + private final SqlGenerator sqlGenerator; + private final String rawNamespace; + + public CatalogParser(final SqlGenerator sqlGenerator) { + this(sqlGenerator, DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); + } + + public CatalogParser(final SqlGenerator sqlGenerator, final String rawNamespace) { + this.sqlGenerator = sqlGenerator; + this.rawNamespace = rawNamespace; + } + + public ParsedCatalog parseCatalog(final ConfiguredAirbyteCatalog catalog) { + // this code is bad and I feel bad + // it's mostly a port of the old normalization logic to prevent tablename collisions. + // tbh I have no idea if it works correctly. + final List streamConfigs = new ArrayList<>(); + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + final StreamConfig originalStreamConfig = toStreamConfig(stream); + // Use empty string quote because we don't really care + if (streamConfigs.stream().anyMatch(s -> s.id().finalTableId("").equals(originalStreamConfig.id().finalTableId(""))) + || streamConfigs.stream().anyMatch(s -> s.id().rawTableId("").equals(originalStreamConfig.id().rawTableId("")))) { + final String originalNamespace = stream.getStream().getNamespace(); + final String originalName = stream.getStream().getName(); + // ... this logic is ported from legacy normalization, and maybe should change? + // We're taking a hash of the quoted namespace and the unquoted stream name + final String hash = DigestUtils.sha1Hex(originalStreamConfig.id().finalNamespace() + "&airbyte&" + originalName).substring(0, 3); + final String newName = originalName + "_" + hash; + streamConfigs.add(new StreamConfig( + sqlGenerator.buildStreamId(originalNamespace, newName, rawNamespace), + originalStreamConfig.syncMode(), + originalStreamConfig.destinationSyncMode(), + originalStreamConfig.primaryKey(), + originalStreamConfig.cursor(), + originalStreamConfig.columns())); + } else { + streamConfigs.add(originalStreamConfig); + } + } + return new ParsedCatalog(streamConfigs); + } + + private StreamConfig toStreamConfig(final ConfiguredAirbyteStream stream) { + final AirbyteType schema = AirbyteType.fromJsonSchema(stream.getStream().getJsonSchema()); + final LinkedHashMap airbyteColumns; + if (schema instanceof final Struct o) { + airbyteColumns = o.properties(); + } else if (schema instanceof final Union u) { + airbyteColumns = u.asColumns(); + } else { + throw new IllegalArgumentException("Top-level schema must be an object"); + } + + if (stream.getPrimaryKey().stream().anyMatch(key -> key.size() > 1)) { + throw new IllegalArgumentException("Only top-level primary keys are supported"); + } + final List primaryKey = stream.getPrimaryKey().stream().map(key -> sqlGenerator.buildColumnId(key.get(0))).toList(); + + if (stream.getCursorField().size() > 1) { + throw new IllegalArgumentException("Only top-level cursors are supported"); + } + final Optional cursor; + if (stream.getCursorField().size() > 0) { + cursor = Optional.of(sqlGenerator.buildColumnId(stream.getCursorField().get(0))); + } else { + cursor = Optional.empty(); + } + + // this code is really bad and I'm not convinced we need to preserve this behavior. + // as with the tablename collisions thing above - we're trying to preserve legacy normalization's + // naming conventions here. + final LinkedHashMap columns = new LinkedHashMap<>(); + for (final Entry entry : airbyteColumns.entrySet()) { + final ColumnId originalColumnId = sqlGenerator.buildColumnId(entry.getKey()); + ColumnId columnId; + if (columns.keySet().stream().noneMatch(c -> c.canonicalName().equals(originalColumnId.canonicalName()))) { + // None of the existing columns have the same name. We can add this new column as-is. + columnId = originalColumnId; + } else { + // One of the existing columns has the same name. We need to handle this collision. + // Append _1, _2, _3, ... to the column name until we find one that doesn't collide. + int i = 1; + while (true) { + columnId = sqlGenerator.buildColumnId(entry.getKey() + "_" + i); + final String canonicalName = columnId.canonicalName(); + if (columns.keySet().stream().noneMatch(c -> c.canonicalName().equals(canonicalName))) { + break; + } else { + i++; + } + } + // But we need to keep the original name so that we can still fetch it out of the JSON records. + columnId = new ColumnId( + columnId.name(), + originalColumnId.originalName(), + columnId.canonicalName()); + } + + columns.put(columnId, entry.getValue()); + } + + return new StreamConfig( + sqlGenerator.buildStreamId(stream.getStream().getNamespace(), stream.getStream().getName(), rawNamespace), + stream.getSyncMode(), + stream.getDestinationSyncMode(), + primaryKey, + cursor, + columns); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java new file mode 100644 index 000000000000..22e0e3b58dd8 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Collection; +import java.util.Optional; + +/** + * TODO these are not in the right place, they probably belongs in a base library but to avoid + * having to publish a bunch of connectors I'm putting it here temporarily + * + */ +public class CollectionUtils { + + /** + * Pass in a collection and search term to determine whether any of the values match ignoring case + * + * @param collection the collection of values + * @param search the value to look for + * @return whether the value matches anything in the collection + */ + public static boolean containsIgnoreCase(final Collection collection, final String search) { + return matchingKey(collection, search).isPresent(); + } + + /** + * Convenience method for when you need to check an entire collection for membership in another + * collection. + * + * @param searchCollection the collection you want to check membership in + * @param searchTerms the keys you're looking for + * @return whether all searchTerms are in the searchCollection + */ + public static boolean containsAllIgnoreCase(final Collection searchCollection, final Collection searchTerms) { + if (searchTerms.isEmpty()) { + // There isn't a good behavior for an empty collection. Without this check, an empty collection + // would always return + // true, but it feels misleading to say that the searchCollection does "contain all" when + // searchTerms is empty + throw new IllegalArgumentException("Search Terms collection may not be empty"); + } + return searchTerms.stream().allMatch(term -> containsIgnoreCase(searchCollection, term)); + } + + /** + * From a collection of strings, return an entry which matches the search term ignoring case + * + * @param collection the collection to search + * @param search the key you're looking for + * @return an Optional value which might contain the key that matches the search + */ + public static Optional matchingKey(final Collection collection, final String search) { + if (collection.contains(search)) { + return Optional.of(search); + } + return collection.stream().filter(s -> s.equalsIgnoreCase(search)).findFirst(); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java new file mode 100644 index 000000000000..5ad836b48fda --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +/** + * In general, callers should not directly instantiate this class. Use + * {@link SqlGenerator#buildColumnId(String)} instead. + * + * @param name the name of the column in the final table. Callers should prefer + * {@link #name(String)} when using the column in a query. + * @param originalName the name of the field in the raw JSON blob + * @param canonicalName the name of the field according to the destination. Used for deduping. + * Useful if a destination warehouse handles columns ignoring case, but preserves case in the + * table schema. + */ +public record ColumnId(String name, String originalName, String canonicalName) { + + public String name(final String quote) { + return quote + name + quote; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java new file mode 100644 index 000000000000..5c078e83c3c5 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An abstraction over SqlGenerator and DestinationHandler. Destinations will still need to call + * {@code new CatalogParser(new FooSqlGenerator()).parseCatalog()}, but should otherwise avoid + * interacting directly with these classes. + *

    + * In a typical sync, destinations should call the methods: + *

      + *
    1. {@link #prepareTables()} once at the start of the sync
      2. + *
    2. {@link #typeAndDedupe(String, String)} as needed throughout the sync
      3. + *
    3. {@link #commitFinalTables()} once at the end of the sync
      4. + *
    + * Note that createFinalTables initializes some internal state. The other methods will throw an + * exception if that method was not called. + */ +public class DefaultTyperDeduper implements TyperDeduper { + + private static final Logger LOGGER = LoggerFactory.getLogger(TyperDeduper.class); + + private static final String NO_SUFFIX = ""; + private static final String TMP_OVERWRITE_TABLE_SUFFIX = "_airbyte_tmp"; + + private final SqlGenerator sqlGenerator; + private final DestinationHandler destinationHandler; + + private final DestinationV1V2Migrator v1V2Migrator; + private final V2RawTableMigrator v2RawTableMigrator; + private final ParsedCatalog parsedCatalog; + private Set overwriteStreamsWithTmpTable; + private final Set streamsWithSuccesfulSetup; + + public DefaultTyperDeduper(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final ParsedCatalog parsedCatalog, + final DestinationV1V2Migrator v1V2Migrator, + final V2RawTableMigrator v2RawTableMigrator) { + this.sqlGenerator = sqlGenerator; + this.destinationHandler = destinationHandler; + this.parsedCatalog = parsedCatalog; + this.v1V2Migrator = v1V2Migrator; + this.v2RawTableMigrator = v2RawTableMigrator; + this.streamsWithSuccesfulSetup = new HashSet<>(); + } + + public DefaultTyperDeduper( + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final ParsedCatalog parsedCatalog, + final DestinationV1V2Migrator v1V2Migrator) { + this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2RawTableMigrator<>()); + } + + /** + * Create the tables that T+D will write to during the sync. In OVERWRITE mode, these might not be + * the true final tables. Specifically, other than an initial sync (i.e. table does not exist, or is + * empty) we write to a temporary final table, and swap it into the true final table at the end of + * the sync. This is to prevent user downtime during a sync. + */ + public void prepareTables() throws Exception { + if (overwriteStreamsWithTmpTable != null) { + throw new IllegalStateException("Tables were already prepared."); + } + overwriteStreamsWithTmpTable = new HashSet<>(); + LOGGER.info("Preparing final tables"); + + // For each stream, make sure that its corresponding final table exists. + // Also, for OVERWRITE streams, decide if we're writing directly to the final table, or into an + // _airbyte_tmp table. + for (final StreamConfig stream : parsedCatalog.streams()) { + // Migrate the Raw Tables if this is the first v2 sync after a v1 sync + v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); + v2RawTableMigrator.migrateIfNecessary(stream); + + final Optional existingTable = destinationHandler.findExistingTable(stream.id()); + if (existingTable.isPresent()) { + LOGGER.info("Final Table exists for stream {}", stream.id().finalName()); + // The table already exists. Decide whether we're writing to it directly, or using a tmp table. + if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { + if (!destinationHandler.isFinalTableEmpty(stream.id()) || !sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { + // We want to overwrite an existing table. Write into a tmp table. We'll overwrite the table at the + // end of the sync. + overwriteStreamsWithTmpTable.add(stream.id()); + // overwrite an existing tmp table if needed. + destinationHandler.execute(sqlGenerator.createTable(stream, TMP_OVERWRITE_TABLE_SUFFIX, true)); + LOGGER.info("Using temp final table for stream {}, will overwrite existing table at end of sync", stream.id().finalName()); + } else { + LOGGER.info("Final Table for stream {} is empty and matches the expected v2 format, writing to table directly", stream.id().finalName()); + } + + } else if (!sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { + // We're loading data directly into the existing table. Make sure it has the right schema. + LOGGER.info("Existing schema for stream {} is different from expected schema. Executing soft reset.", stream.id().finalTableId("")); + destinationHandler.execute(sqlGenerator.softReset(stream)); + } + } else { + LOGGER.info("Final Table does not exist for stream {}, creating.", stream.id().finalName()); + // The table doesn't exist. Create it. Don't force. + destinationHandler.execute(sqlGenerator.createTable(stream, NO_SUFFIX, false)); + } + + streamsWithSuccesfulSetup.add(stream.id()); + } + } + + /** + * Execute typing and deduping for a single stream (i.e. fetch new raw records into the final table, + * etc.). + *

    + * This method is thread-safe; multiple threads can call it concurrently. + * + * @param originalNamespace The stream's namespace, as declared in the configured catalog + * @param originalName The stream's name, as declared in the configured catalog + */ + public void typeAndDedupe(final String originalNamespace, final String originalName) throws Exception { + LOGGER.info("Attempting typing and deduping for {}.{}", originalNamespace, originalName); + final var streamConfig = parsedCatalog.getStream(originalNamespace, originalName); + if (streamsWithSuccesfulSetup.stream() + .noneMatch(streamId -> streamId.originalNamespace().equals(originalNamespace) && streamId.originalName().equals(originalName))) { + // For example, if T+D setup fails, but the consumer tries to run T+D on all streams during close, + // we should skip it. + LOGGER.warn("Skipping typing and deduping for {}.{} because we could not set up the tables for this stream.", originalNamespace, originalName); + return; + } + final String suffix = getFinalTableSuffix(streamConfig.id()); + final String sql = sqlGenerator.updateTable(streamConfig, suffix); + destinationHandler.execute(sql); + } + + /** + * Does any "end of sync" work. For most streams, this is a noop. + *

    + * For OVERWRITE streams where we're writing to a temp table, this is where we swap the temp table + * into the final table. + */ + public void commitFinalTables() throws Exception { + LOGGER.info("Committing final tables"); + for (final StreamConfig streamConfig : parsedCatalog.streams()) { + if (!streamsWithSuccesfulSetup.contains(streamConfig.id())) { + LOGGER.warn("Skipping committing final table for for {}.{} because we could not set up the tables for this stream.", + streamConfig.id().originalNamespace(), streamConfig.id().originalName()); + continue; + } + if (DestinationSyncMode.OVERWRITE.equals(streamConfig.destinationSyncMode())) { + final StreamId streamId = streamConfig.id(); + final String finalSuffix = getFinalTableSuffix(streamId); + if (!StringUtils.isEmpty(finalSuffix)) { + final String overwriteFinalTable = sqlGenerator.overwriteFinalTable(streamId, finalSuffix); + LOGGER.info("Overwriting final table with tmp table for stream {}.{}", streamId.originalNamespace(), streamId.originalName()); + destinationHandler.execute(overwriteFinalTable); + } + } + } + } + + private String getFinalTableSuffix(final StreamId streamId) { + return overwriteStreamsWithTmpTable.contains(streamId) ? TMP_OVERWRITE_TABLE_SUFFIX : NO_SUFFIX; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java new file mode 100644 index 000000000000..9ace9bd64c65 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Optional; + +public interface DestinationHandler { + + Optional findExistingTable(StreamId id) throws Exception; + + boolean isFinalTableEmpty(StreamId id) throws Exception; + + void execute(final String sql) throws Exception; + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java new file mode 100644 index 000000000000..bfe3973e7d31 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public interface DestinationV1V2Migrator { + + /** + * This is the primary entrypoint to this interface + *

    + * Determine whether a migration is necessary for a given stream and if so, migrate the raw table + * and rebuild the final table with a soft reset + * + * @param sqlGenerator the class to use to generate sql + * @param destinationHandler the handler to execute the sql statements + * @param streamConfig the stream to assess migration needs + */ + void migrateIfNecessary( + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final StreamConfig streamConfig) + throws TableNotMigratedException, UnexpectedSchemaException; + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java new file mode 100644 index 000000000000..89f5a4ba4695 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +// yet another namespace, name combo class +public record NamespacedTableName(String namespace, String tableName) { + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java new file mode 100644 index 000000000000..d9e49257d0a7 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public class NoOpDestinationV1V2Migrator implements DestinationV1V2Migrator { + + @Override + public void migrateIfNecessary(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final StreamConfig streamConfig) + throws TableNotMigratedException, UnexpectedSchemaException { + // Do nothing + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java new file mode 100644 index 000000000000..a503914efa6a --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public class NoopTyperDeduper implements TyperDeduper { + + @Override + public void prepareTables() throws Exception { + + } + + @Override + public void typeAndDedupe(String originalNamespace, String originalName) throws Exception { + + } + + @Override + public void commitFinalTables() throws Exception { + + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2RawTableMigrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2RawTableMigrator.java new file mode 100644 index 000000000000..8535481d7847 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2RawTableMigrator.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public class NoopV2RawTableMigrator implements V2RawTableMigrator { + + @Override + public void migrateIfNecessary(final StreamConfig streamConfig) { + // do nothing + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java new file mode 100644 index 000000000000..fb8d2245232c --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import java.util.List; + +public record ParsedCatalog(List streams) { + + public StreamConfig getStream(AirbyteStreamNameNamespacePair streamId) { + return getStream(streamId.getNamespace(), streamId.getName()); + } + + public StreamConfig getStream(StreamId streamId) { + return getStream(streamId.originalNamespace(), streamId.originalName()); + } + + public StreamConfig getStream(String originalNamespace, String originalName) { + return streams.stream() + .filter(s -> s.id().originalNamespace().equals(originalNamespace) && s.id().originalName().equals(originalName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(String.format( + "Could not find stream %s.%s out of streams %s", + originalNamespace, + originalName, + streams.stream().map(stream -> stream.id().originalNamespace() + "." + stream.id().originalName()).toList()))); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java new file mode 100644 index 000000000000..4fe85355a931 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public interface SqlGenerator { + + String SOFT_RESET_SUFFIX = "_ab_soft_reset"; + + StreamId buildStreamId(String namespace, String name, String rawNamespaceOverride); + + ColumnId buildColumnId(String name); + + /** + * Generate a SQL statement to create a fresh table to match the given stream. + *

    + * The generated SQL should throw an exception if the table already exists and {@code force} is + * false. Callers should use + * {@link #existingSchemaMatchesStreamConfig(StreamConfig, java.lang.Object)} if the table is known + * to exist, and potentially {@link #softReset(StreamConfig)}. + * + * @param suffix A suffix to add to the stream name. Useful for full refresh overwrite syncs, where + * we write the entire sync to a temp table. + * @param force If true, will overwrite an existing table. If false, will throw an exception if the + * table already exists. If you're passing a non-empty prefix, you likely want to set this to + * true. + */ + String createTable(final StreamConfig stream, final String suffix, boolean force); + + /** + * Check the final table's schema and compare it to what the stream config would generate. + * + * @param stream the stream/stable in question + * @param existingTable the existing table mapped to the stream + * @return whether the existing table matches the expected schema + */ + boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final DialectTableDefinition existingTable); + + /** + * SQL Statement which will rebuild the final table using the raw table data. Should not cause data + * downtime. Typically this will resemble "create tmp_table; update raw_table set loaded_at=null; + * (t+d into tmp table); (overwrite final table from tmp table);" + * + * @param stream the stream to rebuild + */ + String softReset(final StreamConfig stream); + + /** + * Generate a SQL statement to copy new data from the raw table into the final table. + *

    + * Responsible for: + *

      + *
    • Pulling new raw records from a table (i.e. records with null _airbyte_loaded_at)
      • + *
    • Extracting the JSON fields and casting to the appropriate types
      • + *
    • Handling errors in those casts
      • + *
    • Merging those typed records into an existing table
      • + *
    • Updating the raw records with SET _airbyte_loaded_at = now()
      • + *
    + *

    + * Implementing classes are recommended to break this into smaller methods, which can be tested in + * isolation. However, this interface only requires a single mega-method. + * + * @param finalSuffix the suffix of the final table to write to. If empty string, writes to the + * final table directly. Useful for full refresh overwrite syncs, where we write the entire + * sync to a temp table and then swap it into the final table at the end. + */ + String updateTable(final StreamConfig stream, String finalSuffix); + + /** + * Drop the previous final table, and rename the new final table to match the old final table. + *

    + * This method may assume that the stream is an OVERWRITE stream, and that the final suffix is + * non-empty. Callers are responsible for verifying those are true. + */ + String overwriteFinalTable(StreamId stream, String finalSuffix); + + /** + * Creates a sql query which will create a v2 raw table from the v1 raw table, then performs a soft + * reset. + * + * @param streamId the stream to migrate + * @param namespace + * @param tableName + * @return a string containing the necessary sql to migrate + */ + String migrateFromV1toV2(StreamId streamId, String namespace, String tableName); + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java new file mode 100644 index 000000000000..67952f916cb3 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; + +public record StreamConfig(StreamId id, + SyncMode syncMode, + DestinationSyncMode destinationSyncMode, + List primaryKey, + Optional cursor, + LinkedHashMap columns) { + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java new file mode 100644 index 000000000000..9851ee7b7e59 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; + +/** + * In general, callers should not directly instantiate this class. Use + * {@link SqlGenerator#buildStreamId(String, String, String)} instead. + *

    + * All names/namespaces are intended to be quoted, but do not explicitly contain quotes. For + * example, finalName might be "foo bar"; the caller is required to wrap that in quotes before using + * it in a query. + * + * @param finalNamespace the namespace where the final table will be created + * @param finalName the name of the final table + * @param rawNamespace the namespace where the raw table will be created (typically "airbyte") + * @param rawName the name of the raw table (typically namespace_name, but may be different if there + * are collisions). There is no rawNamespace because we assume that we're writing raw tables + * to the airbyte namespace. + */ +public record StreamId(String finalNamespace, + String finalName, + String rawNamespace, + String rawName, + String originalNamespace, + String originalName) { + + /** + * Most databases/warehouses use a `schema.name` syntax to identify tables. This is a convenience + * method to generate that syntax. + */ + public String finalTableId(String quote) { + return quote + finalNamespace + quote + "." + quote + finalName + quote; + } + + public String finalTableId(String quote, String suffix) { + return quote + finalNamespace + quote + "." + quote + finalName + suffix + quote; + } + + public String rawTableId(String quote) { + return quote + rawNamespace + quote + "." + quote + rawName + quote; + } + + public String finalName(final String quote) { + return quote + finalName + quote; + } + + public String finalNamespace(final String quote) { + return quote + finalNamespace + quote; + } + + public AirbyteStreamNameNamespacePair asPair() { + return new AirbyteStreamNameNamespacePair(originalName, originalNamespace); + } + + /** + * Build the raw table name as namespace + (delimiter) + name. For example, given a stream with + * namespace "public__ab" and name "abab_users", we will end up with raw table name + * "public__ab_ab___ab_abab_users". + *

    + * This logic is intended to solve two problems: + *

      + *
    • The raw table name should be unambiguously parsable into the namespace/name.
      • + *
    • It must be impossible for two different streams to generate the same raw table name.
      • + *
    + * The generated delimiter is guaranteed to not be present in the namespace or name, so it + * accomplishes both of these goals. + */ + public static String concatenateRawTableName(String namespace, String name) { + String plainConcat = namespace + name; + // Pretend we always have at least one underscore, so that we never generate `_raw_stream_` + int longestUnderscoreRun = 1; + for (int i = 0; i < plainConcat.length(); i++) { + // If we've found an underscore, count the number of consecutive underscores + int underscoreRun = 0; + while (i < plainConcat.length() && plainConcat.charAt(i) == '_') { + underscoreRun++; + i++; + } + longestUnderscoreRun = Math.max(longestUnderscoreRun, underscoreRun); + } + + return namespace + "_raw" + "_".repeat(longestUnderscoreRun + 1) + "stream_" + name; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java new file mode 100644 index 000000000000..80eb61be79c5 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.LinkedHashMap; + +/** + * @param properties Use LinkedHashMap to preserve insertion order. + */ +public record Struct(LinkedHashMap properties) implements AirbyteType { + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java new file mode 100644 index 000000000000..ee0fa6c10a22 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +/** + * Exception thrown when a destination's v2 sync is attempting to write to a table which does not + * have the expected columns used by airbyte. + */ +public class TableNotMigratedException extends RuntimeException { + + public TableNotMigratedException(String message) { + super(message); + } + + public TableNotMigratedException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java new file mode 100644 index 000000000000..524c052db0a1 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * A slightly more complicated way to keep track of when to perform type and dedupe operations per + * stream + */ +public class TypeAndDedupeOperationValve extends ConcurrentHashMap { + + private static final long NEGATIVE_MILLIS = -1; + private static final long FIFTEEN_MINUTES_MILLIS = 1000 * 60 * 15; + private static final long ONE_HOUR_MILLIS = 1000 * 60 * 60 * 1; + private static final long TWO_HOURS_MILLIS = 1000 * 60 * 60 * 2; + private static final long FOUR_HOURS_MILLIS = 1000 * 60 * 60 * 4; + + // New users of airbyte likely want to see data flowing into their tables as soon as possible, and + // we want to catch new errors which might appear early within an incremental sync. + // However, as their destination tables grow in size, typing and de-duping data becomes an expensive + // operation. + // To strike a balance between showing data quickly and not slowing down the entire sync, we use an + // increasing interval based approach, from 0 up to 4 hours. + // This is not fancy, just hard coded intervals. + private static final List typeAndDedupeIncreasingIntervals = + List.of(NEGATIVE_MILLIS, FIFTEEN_MINUTES_MILLIS, ONE_HOUR_MILLIS, TWO_HOURS_MILLIS, FOUR_HOURS_MILLIS); + + private static final Supplier SYSTEM_NOW = () -> System.currentTimeMillis(); + + private ConcurrentHashMap incrementalIndex; + + private final Supplier nowness; + + public TypeAndDedupeOperationValve() { + this(SYSTEM_NOW); + } + + /** + * This constructor is here because mocking System.currentTimeMillis() is a pain :( + * + * @param nownessSupplier Supplier which will return a long value representing now + */ + public TypeAndDedupeOperationValve(Supplier nownessSupplier) { + super(); + incrementalIndex = new ConcurrentHashMap<>(); + this.nowness = nownessSupplier; + } + + @Override + public Long put(final AirbyteStreamNameNamespacePair key, final Long value) { + if (!incrementalIndex.containsKey(key)) { + incrementalIndex.put(key, 0); + } + return super.put(key, value); + + } + + /** + * Adds a stream specific timestamp to track type and dedupe operations + * + * @param key the AirbyteStreamNameNamespacePair to track + */ + public void addStream(final AirbyteStreamNameNamespacePair key) { + put(key, nowness.get()); + } + + /** + * Whether we should type and dedupe at this point in time for this particular stream. + * + * @param key the stream in question + * @return a boolean indicating whether we have crossed the interval threshold for typing and + * deduping. + */ + public boolean readyToTypeAndDedupe(final AirbyteStreamNameNamespacePair key) { + if (!containsKey(key)) { + return false; + } + + return nowness.get() - get(key) > typeAndDedupeIncreasingIntervals.get(incrementalIndex.get(key)); + } + + /** + * Increment the interval at which typing and deduping should occur for the stream, max out at last + * index of {@link TypeAndDedupeOperationValve#typeAndDedupeIncreasingIntervals} + * + * @param key the stream to increment the interval of + * @return the index of the typing and deduping interval associated with this stream + */ + public int incrementInterval(final AirbyteStreamNameNamespacePair key) { + if (incrementalIndex.get(key) < typeAndDedupeIncreasingIntervals.size() - 1) { + incrementalIndex.put(key, incrementalIndex.get(key) + 1); + } + return incrementalIndex.get(key); + } + + /** + * Meant to be called after + * {@link TypeAndDedupeOperationValve#readyToTypeAndDedupe(AirbyteStreamNameNamespacePair)} will set + * a streams last operation to the current time and increase its index reference in + * {@link TypeAndDedupeOperationValve#typeAndDedupeIncreasingIntervals} + * + * @param key the stream to update + */ + public void updateTimeAndIncreaseInterval(final AirbyteStreamNameNamespacePair key) { + put(key, nowness.get()); + incrementInterval(key); + } + + /** + * Get the current interval for the stream + * + * @param key the stream in question + * @return a long value representing the length of the interval milliseconds + */ + public Long getIncrementInterval(final AirbyteStreamNameNamespacePair key) { + return typeAndDedupeIncreasingIntervals.get(incrementalIndex.get(key)); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java new file mode 100644 index 000000000000..8a90791359f8 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public interface TyperDeduper { + + void prepareTables() throws Exception; + + void typeAndDedupe(String originalNamespace, String originalName) throws Exception; + + void commitFinalTables() throws Exception; + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java new file mode 100644 index 000000000000..05f0fe6041cd --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public class UnexpectedSchemaException extends RuntimeException { + + public UnexpectedSchemaException(String message) { + super(message); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java new file mode 100644 index 000000000000..e8b62dc36eed --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; + +/** + * Represents a {type: [a, b, ...]} schema. This is theoretically equivalent to {oneOf: [{type: a}, + * {type: b}, ...]} but legacy normalization only handles the {type: [...]} schemas. + *

    + * Eventually we should: + *

      + *
    1. Announce a breaking change to handle both oneOf styles the same
      2. + *
    2. Test against some number of API sources to verify that they won't break badly
      3. + *
    3. Update {@link AirbyteType#fromJsonSchema(JsonNode)} to parse both styles into + * SupportedOneOf
      4. + *
    4. Delete UnsupportedOneOf
      5. + *
    + */ +public record Union(List options) implements AirbyteType { + + /** + * This is a hack to handle weird schemas like {type: [object, string]}. If a stream's top-level + * schema looks like this, we still want to be able to extract the object properties (i.e. treat it + * as though the string option didn't exist). + * + * @throws IllegalArgumentException if we cannot extract columns from this schema + */ + public LinkedHashMap asColumns() { + final long numObjectOptions = options.stream().filter(o -> o instanceof Struct).count(); + if (numObjectOptions > 1) { + LOGGER.error("Can't extract columns from a schema with multiple object options"); + return new LinkedHashMap<>(); + } + + return (options.stream().filter(o -> o instanceof Struct).findFirst()) + .map(o -> ((Struct) o).properties()) + .orElseGet(() -> { + LOGGER.error("Can't extract columns from a schema with no object options"); + return new LinkedHashMap<>(); + }); + } + + // Picks which type in a Union takes precedence + public AirbyteType chooseType() { + final Comparator comparator = Comparator.comparing(t -> { + if (t instanceof Array) { + return -2; + } else if (t instanceof Struct) { + return -1; + } else if (t instanceof final AirbyteProtocolType p) { + return List.of(AirbyteProtocolType.values()).indexOf(p); + } + return Integer.MAX_VALUE; + }); + + return options.stream().min(comparator).orElse(AirbyteProtocolType.UNKNOWN); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java new file mode 100644 index 000000000000..3d3c84636a3c --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.List; + +/** + * Represents a {oneOf: [...]} schema. + *

    + * This is purely a legacy type that we should eventually delete. See also {@link Union}. + */ +public record UnsupportedOneOf(List options) implements AirbyteType { + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2RawTableMigrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2RawTableMigrator.java new file mode 100644 index 000000000000..d17722ac1279 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2RawTableMigrator.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public interface V2RawTableMigrator { + + void migrateIfNecessary(final StreamConfig streamConfig) throws InterruptedException; + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java new file mode 100644 index 000000000000..da80eeee31c5 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java @@ -0,0 +1,502 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType.*; +import static io.airbyte.integrations.base.destination.typing_deduping.AirbyteType.fromJsonSchema; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.airbyte.commons.json.Jsons; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class AirbyteTypeTest { + + @Test + public void testStruct() { + final List structSchema = new ArrayList<>(); + structSchema.add(""" + { + "type": "object", + "properties": { + "key1": { + "type": "boolean" + }, + "key2": { + "type": "integer" + }, + "key3": { + "type": "number", + "airbyte_type": "integer" + }, + "key4": { + "type": "number" + }, + "key5": { + "type": "string", + "format": "date" + }, + "key6": { + "type": "string", + "format": "time", + "airbyte_type": "time_without_timezone" + }, + "key7": { + "type": "string", + "format": "time", + "airbyte_type": "time_with_timezone" + }, + "key8": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "key9": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "key10": { + "type": "string", + "format": "date-time" + }, + "key11": { + "type": "string" + } + } + } + """); + structSchema.add(""" + { + "type": ["object"], + "properties": { + "key1": { + "type": ["boolean"] + }, + "key2": { + "type": ["integer"] + }, + "key3": { + "type": ["number"], + "airbyte_type": "integer" + }, + "key4": { + "type": ["number"] + }, + "key5": { + "type": ["string"], + "format": "date" + }, + "key6": { + "type": ["string"], + "format": "time", + "airbyte_type": "time_without_timezone" + }, + "key7": { + "type": ["string"], + "format": "time", + "airbyte_type": "time_with_timezone" + }, + "key8": { + "type": ["string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "key9": { + "type": ["string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "key10": { + "type": ["string"], + "format": "date-time" + }, + "key11": { + "type": ["string"] + } + } + } + """); + structSchema.add(""" + { + "type": ["null", "object"], + "properties": { + "key1": { + "type": ["null", "boolean"] + }, + "key2": { + "type": ["null", "integer"] + }, + "key3": { + "type": ["null", "number"], + "airbyte_type": "integer" + }, + "key4": { + "type": ["null", "number"] + }, + "key5": { + "type": ["null", "string"], + "format": "date" + }, + "key6": { + "type": ["null", "string"], + "format": "time", + "airbyte_type": "time_without_timezone" + }, + "key7": { + "type": ["null", "string"], + "format": "time", + "airbyte_type": "time_with_timezone" + }, + "key8": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "key9": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "key10": { + "type": ["null", "string"], + "format": "date-time" + }, + "key11": { + "type": ["null", "string"] + } + } + } + """); + + final LinkedHashMap propertiesMap = new LinkedHashMap<>(); + propertiesMap.put("key1", BOOLEAN); + propertiesMap.put("key2", INTEGER); + propertiesMap.put("key3", INTEGER); + propertiesMap.put("key4", NUMBER); + propertiesMap.put("key5", DATE); + propertiesMap.put("key6", TIME_WITHOUT_TIMEZONE); + propertiesMap.put("key7", TIME_WITH_TIMEZONE); + propertiesMap.put("key8", TIMESTAMP_WITHOUT_TIMEZONE); + propertiesMap.put("key9", TIMESTAMP_WITH_TIMEZONE); + propertiesMap.put("key10", TIMESTAMP_WITH_TIMEZONE); + propertiesMap.put("key11", STRING); + + final AirbyteType struct = new Struct(propertiesMap); + for (final String schema : structSchema) { + assertEquals(struct, fromJsonSchema(Jsons.deserialize(schema))); + } + } + + @Test + public void testEmptyStruct() { + final List structSchema = new ArrayList<>(); + structSchema.add(""" + { + "type": "object" + } + """); + structSchema.add(""" + { + "type": ["object"] + } + """); + structSchema.add(""" + { + "type": ["null", "object"] + } + """); + + final AirbyteType struct = new Struct(new LinkedHashMap<>()); + for (final String schema : structSchema) { + assertEquals(struct, fromJsonSchema(Jsons.deserialize(schema))); + } + } + + @Test + public void testImplicitStruct() { + final String structSchema = """ + { + "properties": { + "key1": { + "type": "boolean" + } + } + } + """; + + final LinkedHashMap propertiesMap = new LinkedHashMap<>(); + propertiesMap.put("key1", BOOLEAN); + + final AirbyteType struct = new Struct(propertiesMap); + assertEquals(struct, fromJsonSchema(Jsons.deserialize(structSchema))); + } + + @Test + public void testArray() { + final List arraySchema = new ArrayList<>(); + arraySchema.add(""" + { + "type": "array", + "items": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + """); + arraySchema.add(""" + { + "type": ["array"], + "items": { + "type": ["string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + """); + arraySchema.add(""" + { + "type": ["null", "array"], + "items": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + } + } + """); + + final AirbyteType array = new Array(TIMESTAMP_WITH_TIMEZONE); + for (final String schema : arraySchema) { + assertEquals(array, fromJsonSchema(Jsons.deserialize(schema))); + } + } + + @Test + public void testEmptyArray() { + final List arraySchema = new ArrayList<>(); + arraySchema.add(""" + { + "type": "array" + } + """); + arraySchema.add(""" + { + "type": ["array"] + } + """); + + arraySchema.add(""" + { + "type": ["null", "array"] + } + """); + + final AirbyteType array = new Array(UNKNOWN); + for (final String schema : arraySchema) { + assertEquals(array, fromJsonSchema(Jsons.deserialize(schema))); + } + } + + @Test + public void testUnsupportedOneOf() { + final String unsupportedOneOfSchema = """ + { + "oneOf": ["number", "string"] + } + """; + + final List options = new ArrayList<>(); + options.add(NUMBER); + options.add(STRING); + + final UnsupportedOneOf unsupportedOneOf = new UnsupportedOneOf(options); + assertEquals(unsupportedOneOf, fromJsonSchema(Jsons.deserialize(unsupportedOneOfSchema))); + } + + @Test + public void testUnion() { + + final String unionSchema = """ + { + "type": ["string", "number"] + } + """; + + final List options = new ArrayList<>(); + options.add(STRING); + options.add(NUMBER); + + final Union union = new Union(options); + assertEquals(union, fromJsonSchema(Jsons.deserialize(unionSchema))); + } + + @Test + public void testUnionComplex() { + final JsonNode schema = Jsons.deserialize(""" + { + "type": ["string", "object", "array", "null", "string", "object", "array", "null"], + "properties": { + "foo": {"type": "string"} + }, + "items": {"type": "string"} + } + """); + + final AirbyteType parsed = fromJsonSchema(schema); + + final AirbyteType expected = new Union(List.of( + STRING, + new Struct(new LinkedHashMap<>() { + + { + put("foo", STRING); + } + + }), + new Array(STRING))); + assertEquals(expected, parsed); + } + + @Test + public void testUnionUnderspecifiedNonPrimitives() { + final JsonNode schema = Jsons.deserialize(""" + { + "type": ["string", "object", "array", "null", "string", "object", "array", "null"] + } + """); + + final AirbyteType parsed = fromJsonSchema(schema); + + final AirbyteType expected = new Union(List.of( + STRING, + new Struct(new LinkedHashMap<>()), + new Array(UNKNOWN))); + assertEquals(expected, parsed); + } + + @Test + public void testInvalidTextualType() { + final String invalidTypeSchema = """ + { + "type": "foo" + } + """; + assertEquals(UNKNOWN, fromJsonSchema(Jsons.deserialize(invalidTypeSchema))); + } + + @Test + public void testInvalidBooleanType() { + final String invalidTypeSchema = """ + { + "type": true + } + """; + assertEquals(UNKNOWN, fromJsonSchema(Jsons.deserialize(invalidTypeSchema))); + } + + @Test + public void testInvalid() { + final List invalidSchema = new ArrayList<>(); + invalidSchema.add(""); + invalidSchema.add("null"); + invalidSchema.add("true"); + invalidSchema.add("false"); + invalidSchema.add("1"); + invalidSchema.add("\"\""); + invalidSchema.add("[]"); + invalidSchema.add("{}"); + + for (final String schema : invalidSchema) { + assertEquals(UNKNOWN, fromJsonSchema(Jsons.deserialize(schema))); + } + } + + @Test + public void testChooseUnion() { + final Map unionToType = new HashMap<>(); + + final Array a = new Array(BOOLEAN); + + final LinkedHashMap properties = new LinkedHashMap<>(); + properties.put("key1", UNKNOWN); + properties.put("key2", INTEGER); + final Struct s = new Struct(properties); + + unionToType.put(new Union(ImmutableList.of(s, a)), a); + unionToType.put(new Union(ImmutableList.of(NUMBER, a)), a); + unionToType.put(new Union(ImmutableList.of(INTEGER, s)), s); + unionToType.put(new Union(ImmutableList.of(NUMBER, DATE, BOOLEAN)), DATE); + unionToType.put(new Union(ImmutableList.of(INTEGER, BOOLEAN, NUMBER)), NUMBER); + unionToType.put(new Union(ImmutableList.of(BOOLEAN, INTEGER)), INTEGER); + + assertAll( + unionToType.entrySet().stream().map(e -> () -> assertEquals(e.getValue(), e.getKey().chooseType()))); + } + + @Test + public void testAsColumns() { + final Union u = new Union(List.of( + STRING, + new Struct(new LinkedHashMap<>() { + + { + put("foo", STRING); + } + + }), + new Array(STRING), + // This is bad behavior, but it matches current behavior so we'll test it. + // Ideally, we would recognize that the sub-unions are also objects. + new Union(List.of(new Struct(new LinkedHashMap<>()))), + new UnsupportedOneOf(List.of(new Struct(new LinkedHashMap<>()))))); + + final LinkedHashMap columns = u.asColumns(); + + assertEquals( + new LinkedHashMap<>() { + + { + put("foo", STRING); + } + + }, + columns); + } + + @Test + public void testAsColumnsMultipleObjects() { + final Union u = new Union(List.of( + new Struct(new LinkedHashMap<>()), + new Struct(new LinkedHashMap<>()))); + + // This prooobably should throw an exception, but for the sake of smooth rollout it just logs a + // warning for now. + assertEquals(new LinkedHashMap<>(), u.asColumns()); + } + + @Test + public void testAsColumnsNoObjects() { + final Union u = new Union(List.of( + STRING, + new Array(STRING), + new UnsupportedOneOf(new ArrayList<>()), + // Similar to testAsColumns(), this is bad behavior. + new Union(List.of(new Struct(new LinkedHashMap<>()))), + new UnsupportedOneOf(List.of(new Struct(new LinkedHashMap<>()))))); + + // This prooobably should throw an exception, but for the sake of smooth rollout it just logs a + // warning for now. + assertEquals(new LinkedHashMap<>(), u.asColumns()); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java new file mode 100644 index 000000000000..f79da6a374f6 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.mockito.Mockito.*; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class CatalogParserTest { + + private SqlGenerator sqlGenerator; + private CatalogParser parser; + + @BeforeEach + public void setup() { + sqlGenerator = mock(SqlGenerator.class); + // noop quoting logic + when(sqlGenerator.buildColumnId(any())).thenAnswer(invocation -> { + String fieldName = invocation.getArgument(0); + return new ColumnId(fieldName, fieldName, fieldName); + }); + when(sqlGenerator.buildStreamId(any(), any(), any())).thenAnswer(invocation -> { + String namespace = invocation.getArgument(0); + String name = invocation.getArgument(1); + String rawNamespace = invocation.getArgument(1); + return new StreamId(namespace, name, rawNamespace, namespace + "_abab_" + name, namespace, name); + }); + + parser = new CatalogParser(sqlGenerator); + } + + /** + * Both these streams will write to the same final table name ("foofoo"). Verify that they don't + * actually use the same tablename. + */ + @Test + public void finalNameCollision() { + when(sqlGenerator.buildStreamId(any(), any(), any())).thenAnswer(invocation -> { + String originalNamespace = invocation.getArgument(0); + String originalName = (invocation.getArgument(1)); + String originalRawNamespace = (invocation.getArgument(1)); + + // emulate quoting logic that causes a name collision + String quotedName = originalName.replaceAll("bar", ""); + return new StreamId(originalNamespace, quotedName, originalRawNamespace, originalNamespace + "_abab_" + quotedName, originalNamespace, + originalName); + }); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + stream("a", "foobarfoo"), + stream("a", "foofoo"))); + + final ParsedCatalog parsedCatalog = parser.parseCatalog(catalog); + + assertNotEquals( + parsedCatalog.streams().get(0).id().finalName(), + parsedCatalog.streams().get(1).id().finalName()); + } + + /** + * The schema contains two fields, which will both end up named "foofoo" after quoting. Verify that + * they don't actually use the same column name. + */ + @Test + public void columnNameCollision() { + when(sqlGenerator.buildColumnId(any())).thenAnswer(invocation -> { + String originalName = invocation.getArgument(0); + + // emulate quoting logic that causes a name collision + String quotedName = originalName.replaceAll("bar", ""); + return new ColumnId(quotedName, originalName, quotedName); + }); + JsonNode schema = Jsons.deserialize(""" + { + "type": "object", + "properties": { + "foobarfoo": {"type": "string"}, + "foofoo": {"type": "string"} + } + } + """); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(stream("a", "a", schema))); + + final ParsedCatalog parsedCatalog = parser.parseCatalog(catalog); + + assertEquals(2, parsedCatalog.streams().get(0).columns().size()); + } + + private static ConfiguredAirbyteStream stream(String namespace, String name) { + return stream( + namespace, + name, + Jsons.deserialize(""" + { + "type": "object", + "properties": { + "name": {"type": "string"} + } + } + """)); + } + + private static ConfiguredAirbyteStream stream(String namespace, String name, JsonNode schema) { + return new ConfiguredAirbyteStream().withStream( + new AirbyteStream() + .withNamespace(namespace) + .withName(name) + .withJsonSchema(schema)); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java new file mode 100644 index 000000000000..84718062c16c --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Optional; +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +public class CollectionUtilsTest { + + static Set TEST_COLLECTION = Set.of("foo", "BAR", "fizz", "zip_ZOP"); + + @ParameterizedTest + @CsvSource({"foo,foo", "bar,BAR", "fIzZ,fizz", "ZIP_zop,zip_ZOP", "nope,"}) + public void testMatchingKey(final String input, final String output) { + final var expected = Optional.ofNullable(output); + Assertions.assertEquals(CollectionUtils.matchingKey(TEST_COLLECTION, input), expected); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java new file mode 100644 index 000000000000..62fb1374eed9 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.ignoreStubs; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class DefaultTyperDeduperTest { + + private MockSqlGenerator sqlGenerator; + private DestinationHandler destinationHandler; + + private DestinationV1V2Migrator migrator; + private TyperDeduper typerDeduper; + + @BeforeEach + void setup() { + sqlGenerator = spy(new MockSqlGenerator()); + destinationHandler = mock(DestinationHandler.class); + migrator = new NoOpDestinationV1V2Migrator<>(); + + final ParsedCatalog parsedCatalog = new ParsedCatalog(List.of( + new StreamConfig( + new StreamId("overwrite_ns", "overwrite_stream", null, null, "overwrite_ns", "overwrite_stream"), + null, + DestinationSyncMode.OVERWRITE, + null, + null, + null), + new StreamConfig( + new StreamId("append_ns", "append_stream", null, null, "append_ns", "append_stream"), + null, + DestinationSyncMode.APPEND, + null, + null, + null), + new StreamConfig( + new StreamId("dedup_ns", "dedup_stream", null, null, "dedup_ns", "dedup_stream"), + null, + DestinationSyncMode.APPEND_DEDUP, + null, + null, + null))); + + typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator); + } + + /** + * When there are no existing tables, we should create them and write to them directly. + */ + @Test + void emptyDestination() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.empty()); + + typerDeduper.prepareTables(); + verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream"); + verify(destinationHandler).execute("CREATE TABLE append_ns.append_stream"); + verify(destinationHandler).execute("CREATE TABLE dedup_ns.dedup_stream"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream"); + verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream"); + typerDeduper.typeAndDedupe("append_ns", "append_stream"); + verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream"); + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); + verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.commitFinalTables(); + verify(destinationHandler, never()).execute(any()); + } + + /** + * When there's an existing table but it's empty, we should ensure it has the right schema and write + * to it directly. + */ + @Test + void existingEmptyTable() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); + when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(false); + + typerDeduper.prepareTables(); + verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); + verify(destinationHandler).execute("SOFT RESET append_ns.append_stream"); + verify(destinationHandler).execute("SOFT RESET dedup_ns.dedup_stream"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream"); + verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); + typerDeduper.typeAndDedupe("append_ns", "append_stream"); + verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream"); + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); + verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.commitFinalTables(); + verify(destinationHandler).execute("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + } + + /** + * When there's an existing empty table with the right schema, we don't need to do anything during + * setup. + */ + @Test + void existingEmptyTableMatchingSchema() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); + when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); + + typerDeduper.prepareTables(); + verify(destinationHandler, never()).execute(any()); + } + + /** + * When there's an existing nonempty table, we should alter it. For the OVERWRITE stream, we also + * need to write to a tmp table, and overwrite the real table at the end of the sync. + */ + @Test + void existingNonemptyTable() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); + + typerDeduper.prepareTables(); + // NB: We only create a tmp table for the overwrite stream, and do _not_ soft reset the existing + // overwrite stream's table. + verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); + verify(destinationHandler).execute("SOFT RESET append_ns.append_stream"); + verify(destinationHandler).execute("SOFT RESET dedup_ns.dedup_stream"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream"); + // NB: no airbyte_tmp suffix on the non-overwrite streams + verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); + typerDeduper.typeAndDedupe("append_ns", "append_stream"); + verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream"); + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); + verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.commitFinalTables(); + verify(destinationHandler).execute("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + } + + /** + * When there's an existing nonempty table with the right schema, we don't need to modify it, but + * OVERWRITE streams still need to create a tmp table. + */ + @Test + void existingNonemptyTableMatchingSchema() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); + when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); + + typerDeduper.prepareTables(); + // NB: We only create one tmp table here. + // Also, we need to alter the existing _real_ table, not the tmp table! + verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + } + + @Test + void nonexistentStream() { + assertThrows(IllegalArgumentException.class, + () -> typerDeduper.typeAndDedupe("nonexistent_ns", "nonexistent_stream")); + verifyNoInteractions(ignoreStubs(destinationHandler)); + } + + @Test + void failedSetup() throws Exception { + doThrow(new RuntimeException("foo")).when(destinationHandler).execute(any()); + + assertThrows(RuntimeException.class, () -> typerDeduper.prepareTables()); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); + typerDeduper.commitFinalTables(); + + verifyNoInteractions(ignoreStubs(destinationHandler)); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java new file mode 100644 index 000000000000..8fe695b81ed0 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; +import static io.airbyte.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.Optional; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.mockito.Mockito; + +public class DestinationV1V2MigratorTest { + + private static final StreamId STREAM_ID = new StreamId("final", "final_table", "raw", "raw_table", null, null); + + public static class ShouldMigrateTestArgumentProvider implements ArgumentsProvider { + + @Override + public Stream provideArguments(ExtensionContext context) throws Exception { + + // Don't throw an exception + final boolean v2SchemaMatches = true; + + return Stream.of( + // Doesn't Migrate because of sync mode + Arguments.of(DestinationSyncMode.OVERWRITE, makeMockMigrator(true, false, v2SchemaMatches, true, true), false), + // Doesn't migrate because v2 table already exists + Arguments.of(DestinationSyncMode.APPEND, makeMockMigrator(true, true, v2SchemaMatches, true, true), false), + Arguments.of(DestinationSyncMode.APPEND_DEDUP, makeMockMigrator(true, true, v2SchemaMatches, true, true), false), + // Doesn't migrate because no valid v1 raw table exists + Arguments.of(DestinationSyncMode.APPEND, makeMockMigrator(true, false, v2SchemaMatches, false, true), false), + Arguments.of(DestinationSyncMode.APPEND_DEDUP, makeMockMigrator(true, false, v2SchemaMatches, false, true), false), + Arguments.of(DestinationSyncMode.APPEND, makeMockMigrator(true, false, v2SchemaMatches, true, false), false), + Arguments.of(DestinationSyncMode.APPEND_DEDUP, makeMockMigrator(true, false, v2SchemaMatches, true, false), false), + // Migrates + Arguments.of(DestinationSyncMode.APPEND, noIssuesMigrator(), true), + Arguments.of(DestinationSyncMode.APPEND_DEDUP, noIssuesMigrator(), true)); + } + + } + + @ParameterizedTest + @ArgumentsSource(ShouldMigrateTestArgumentProvider.class) + public void testShouldMigrate(final DestinationSyncMode destinationSyncMode, final BaseDestinationV1V2Migrator migrator, boolean expected) { + final StreamConfig config = new StreamConfig(STREAM_ID, null, destinationSyncMode, null, null, null); + final var actual = migrator.shouldMigrate(config); + Assertions.assertEquals(expected, actual); + } + + @Test + public void testMismatchedSchemaThrowsException() { + final StreamConfig config = new StreamConfig(STREAM_ID, null, DestinationSyncMode.APPEND_DEDUP, null, null, null); + final var migrator = makeMockMigrator(true, true, false, false, false); + UnexpectedSchemaException exception = Assertions.assertThrows(UnexpectedSchemaException.class, + () -> migrator.shouldMigrate(config)); + Assertions.assertEquals("Destination V2 Raw Table does not match expected Schema", exception.getMessage()); + } + + @SneakyThrows + @Test + public void testMigrate() { + final var sqlGenerator = new MockSqlGenerator(); + final StreamConfig stream = new StreamConfig(STREAM_ID, null, DestinationSyncMode.APPEND_DEDUP, null, null, null); + final DestinationHandler handler = Mockito.mock(DestinationHandler.class); + final var sql = sqlGenerator.migrateFromV1toV2(STREAM_ID, "v1_raw_namespace", "v1_raw_table"); + // All is well + final var migrator = noIssuesMigrator(); + migrator.migrate(sqlGenerator, handler, stream); + Mockito.verify(handler).execute(sql); + // Exception thrown when executing sql, TableNotMigratedException thrown + Mockito.doThrow(Exception.class).when(handler).execute(Mockito.anyString()); + TableNotMigratedException exception = Assertions.assertThrows(TableNotMigratedException.class, + () -> migrator.migrate(sqlGenerator, handler, stream)); + Assertions.assertEquals("Attempted and failed to migrate stream final_table", exception.getMessage()); + } + + public static BaseDestinationV1V2Migrator makeMockMigrator(final boolean v2NamespaceExists, + final boolean v2TableExists, + final boolean v2RawSchemaMatches, + boolean v1RawTableExists, + boolean v1RawTableSchemaMatches) { + final BaseDestinationV1V2Migrator migrator = Mockito.spy(BaseDestinationV1V2Migrator.class); + Mockito.when(migrator.doesAirbyteInternalNamespaceExist(Mockito.any())).thenReturn(v2NamespaceExists); + final var existingTable = v2TableExists ? Optional.of("v2_raw") : Optional.empty(); + Mockito.when(migrator.getTableIfExists("raw", "raw_table")).thenReturn(existingTable); + Mockito.when(migrator.schemaMatchesExpectation("v2_raw", V2_RAW_TABLE_COLUMN_NAMES)).thenReturn(v2RawSchemaMatches); + + Mockito.when(migrator.convertToV1RawName(Mockito.any())).thenReturn(new NamespacedTableName("v1_raw_namespace", "v1_raw_table")); + final var existingV1RawTable = v1RawTableExists ? Optional.of("v1_raw") : Optional.empty(); + Mockito.when(migrator.getTableIfExists("v1_raw_namespace", "v1_raw_table")).thenReturn(existingV1RawTable); + Mockito.when(migrator.schemaMatchesExpectation("v1_raw", LEGACY_RAW_TABLE_COLUMNS)).thenReturn(v1RawTableSchemaMatches); + return migrator; + } + + public static BaseDestinationV1V2Migrator noIssuesMigrator() { + return makeMockMigrator(true, false, true, true, true); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java new file mode 100644 index 000000000000..3f56b61114e2 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +/** + * Basic SqlGenerator mock. See {@link DefaultTyperDeduperTest} for example usage. + */ +class MockSqlGenerator implements SqlGenerator { + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + return null; + } + + @Override + public ColumnId buildColumnId(final String name) { + return null; + } + + @Override + public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + return "CREATE TABLE " + stream.id().finalTableId("", suffix); + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final String existingTable) throws TableNotMigratedException { + return false; + } + + @Override + public String softReset(final StreamConfig stream) { + return "SOFT RESET " + stream.id().finalTableId(""); + } + + @Override + public String updateTable(final StreamConfig stream, final String finalSuffix) { + return "UPDATE TABLE " + stream.id().finalTableId("", finalSuffix); + } + + @Override + public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return "OVERWRITE TABLE " + stream.finalTableId("") + " FROM " + stream.finalTableId("", finalSuffix); + } + + @Override + public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return "MIGRATE TABLE " + String.join(".", namespace, tableName) + " TO " + streamId.rawTableId(""); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java new file mode 100644 index 000000000000..d9ef0d6f4c85 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class StreamIdTest { + + /** + * Both these streams naively want the same raw table name ("aaa_abab_bbb_abab_ccc"). Verify that + * they don't actually use the same raw table. + */ + @Test + public void rawNameCollision() { + String stream1 = StreamId.concatenateRawTableName("aaa_abab_bbb", "ccc"); + String stream2 = StreamId.concatenateRawTableName("aaa", "bbb_abab_ccc"); + + assertAll( + () -> assertEquals("aaa_abab_bbb_raw__stream_ccc", stream1), + () -> assertEquals("aaa_raw__stream_bbb_abab_ccc", stream2)); + } + + @Test + public void noUnderscores() { + String stream = StreamId.concatenateRawTableName("a", "b"); + + assertEquals("a_raw__stream_b", stream); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java new file mode 100644 index 000000000000..3f6b35e6eaa3 --- /dev/null +++ b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TypeAndDedupeOperationValveTest { + + private static final AirbyteStreamNameNamespacePair STREAM_A = new AirbyteStreamNameNamespacePair("a", "a"); + private static final AirbyteStreamNameNamespacePair STREAM_B = new AirbyteStreamNameNamespacePair("b", "b"); + private static final Supplier ALWAYS_ZERO = () -> 0l; + + private Supplier minuteUpdates; + + @BeforeEach + public void setup() { + AtomicLong start = new AtomicLong(0); + minuteUpdates = () -> start.getAndUpdate(l -> l + (60 * 1000)); + } + + private void elapseTime(Supplier timing, int iterations) { + IntStream.range(0, iterations).forEach(__ -> { + timing.get(); + }); + } + + @Test + public void testAddStream() { + final var valve = new TypeAndDedupeOperationValve(ALWAYS_ZERO); + valve.addStream(STREAM_A); + Assertions.assertEquals(-1, valve.getIncrementInterval(STREAM_A)); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + Assertions.assertEquals(valve.get(STREAM_A), 0l); + } + + @Test + public void testReadyToTypeAndDedupe() { + final var valve = new TypeAndDedupeOperationValve(minuteUpdates); + // method call increments time + valve.addStream(STREAM_A); + elapseTime(minuteUpdates, 1); + // method call increments time + valve.addStream(STREAM_B); + // method call increments time + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + elapseTime(minuteUpdates, 1); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_B)); + valve.updateTimeAndIncreaseInterval(STREAM_A); + Assertions.assertEquals(1000 * 60 * 15, + valve.getIncrementInterval(STREAM_A)); + // method call increments time + Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A)); + // More than enough time has passed now + elapseTime(minuteUpdates, 15); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + } + + @Test + public void testIncrementInterval() { + final var valve = new TypeAndDedupeOperationValve(ALWAYS_ZERO); + valve.addStream(STREAM_A); + IntStream.rangeClosed(1, 4).forEach(i -> { + final var index = valve.incrementInterval(STREAM_A); + Assertions.assertEquals(i, index); + }); + Assertions.assertEquals(4, valve.incrementInterval(STREAM_A)); + // Twice to be sure + Assertions.assertEquals(4, valve.incrementInterval(STREAM_A)); + } + + @Test + public void testUpdateTimeAndIncreaseInterval() { + final var valve = new TypeAndDedupeOperationValve(minuteUpdates); + valve.addStream(STREAM_A); + IntStream.range(0, 1).forEach(__ -> Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A))); // start ready to T&D + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + valve.updateTimeAndIncreaseInterval(STREAM_A); + IntStream.range(0, 15).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + valve.updateTimeAndIncreaseInterval(STREAM_A); + IntStream.range(0, 60).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + valve.updateTimeAndIncreaseInterval(STREAM_A); + IntStream.range(0, 120).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + valve.updateTimeAndIncreaseInterval(STREAM_A); + IntStream.range(0, 240).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + valve.updateTimeAndIncreaseInterval(STREAM_A); + IntStream.range(0, 240).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); + Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/build.gradle b/airbyte-integrations/bases/bases-destination-jdbc/build.gradle index 57c98181a3e9..4b731d310b42 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/build.gradle +++ b/airbyte-integrations/bases/bases-destination-jdbc/build.gradle @@ -11,6 +11,7 @@ dependencies { implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:base-java-s3') + implementation project(':airbyte-integrations:bases:base-typing-deduping') implementation libs.airbyte.protocol implementation 'org.apache.commons:commons-lang3:3.11' diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java index 522c5fda8c47..d9bd6e6212d2 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java @@ -32,7 +32,7 @@ public abstract class JdbcSqlOperations implements SqlOperations { // this adapter modifies record message before inserting them to the destination protected final Optional dataAdapter; - private final Set schemaSet = new HashSet<>(); + protected final Set schemaSet = new HashSet<>(); protected JdbcSqlOperations() { this.dataAdapter = Optional.empty(); @@ -49,7 +49,7 @@ public void createSchemaIfNotExists(final JdbcDatabase database, final String sc database.execute(String.format("CREATE SCHEMA IF NOT EXISTS %s;", schemaName)); schemaSet.add(schemaName); } - } catch (Exception e) { + } catch (final Exception e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } @@ -62,7 +62,7 @@ public void createSchemaIfNotExists(final JdbcDatabase database, final String sc * @param e the exception to check. * @return A ConfigErrorException with a message with actionable feedback to the user. */ - protected Optional checkForKnownConfigExceptions(Exception e) { + protected Optional checkForKnownConfigExceptions(final Exception e) { return Optional.empty(); } @@ -70,7 +70,7 @@ protected Optional checkForKnownConfigExceptions(Exception public void createTableIfNotExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { try { database.execute(createTableQuery(database, schemaName, tableName)); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } @@ -127,12 +127,12 @@ public void executeTransaction(final JdbcDatabase database, final List q public void dropTableIfExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { try { database.execute(dropTableIfExistsQuery(schemaName, tableName)); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } - private String dropTableIfExistsQuery(final String schemaName, final String tableName) { + public String dropTableIfExistsQuery(final String schemaName, final String tableName) { return String.format("DROP TABLE IF EXISTS %s.%s;\n", schemaName, tableName); } diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java index 8398c5adeafa..ddfa0f535c1f 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java @@ -9,6 +9,7 @@ import io.airbyte.integrations.BaseConnector; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -66,4 +67,14 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, return typeToDestination.get(destinationType).getConsumer(config, catalog, outputRecordCollector); } + @Override + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) + throws Exception { + final T destinationType = configToType.apply(config); + LOGGER.info("Using destination type: " + destinationType.name()); + return typeToDestination.get(destinationType).getSerializedMessageConsumer(config, catalog, outputRecordCollector); + } + } diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java index c6a25fb43aba..645280485b28 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java @@ -6,13 +6,14 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.jdbc.WriteConfig; import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; import io.airbyte.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; import io.airbyte.integrations.destination_async.DestinationFlushFunction; import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.List; @@ -31,15 +32,38 @@ class AsyncFlush implements DestinationFlushFunction { private final StagingOperations stagingOperations; private final JdbcDatabase database; private final ConfiguredAirbyteCatalog catalog; + private final TypeAndDedupeOperationValve typerDeduperValve; + private final TyperDeduper typerDeduper; + private final long optimalBatchSizeBytes; public AsyncFlush(final Map streamDescToWriteConfig, final StagingOperations stagingOperations, final JdbcDatabase database, - final ConfiguredAirbyteCatalog catalog) { + final ConfiguredAirbyteCatalog catalog, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper) { + this(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper, 50 * 1024 * 1024); + } + + public AsyncFlush(final Map streamDescToWriteConfig, + final StagingOperations stagingOperations, + final JdbcDatabase database, + final ConfiguredAirbyteCatalog catalog, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + // In general, this size is chosen to improve the performance of lower memory connectors. With 1 Gi + // of + // resource the connector will usually at most fill up around 150 MB in a single queue. By lowering + // the batch size, the AsyncFlusher will flush in smaller batches which allows for memory to be + // freed earlier similar to a sliding window effect + long optimalBatchSizeBytes) { this.streamDescToWriteConfig = streamDescToWriteConfig; this.stagingOperations = stagingOperations; this.database = database; this.catalog = catalog; + this.typerDeduperValve = typerDeduperValve; + this.typerDeduper = typerDeduper; + this.optimalBatchSizeBytes = optimalBatchSizeBytes; } @Override @@ -57,7 +81,7 @@ public void flush(final StreamDescriptor decs, final Stream writeConfigs) { + final List writeConfigs, + final TyperDeduper typerDeduper) { return () -> { log.info("Preparing raw tables in destination started for {} streams", writeConfigs.size()); + typerDeduper.prepareTables(); final List queryList = new ArrayList<>(); for (final WriteConfig writeConfig : writeConfigs) { final String schema = writeConfig.getOutputSchemaName(); @@ -66,7 +70,11 @@ public static void copyIntoTableFromStage(final JdbcDatabase database, final List stagedFiles, final String tableName, final String schemaName, - final StagingOperations stagingOperations) + final StagingOperations stagingOperations, + final String streamNamespace, + final String streamName, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper) throws Exception { try { stagingOperations.copyIntoTableFromStage(database, stageName, stagingPath, stagedFiles, @@ -90,7 +98,8 @@ public static void copyIntoTableFromStage(final JdbcDatabase database, public static OnCloseFunction onCloseFunction(final JdbcDatabase database, final StagingOperations stagingOperations, final List writeConfigs, - final boolean purgeStagingData) { + final boolean purgeStagingData, + final TyperDeduper typerDeduper) { return (hasFailed) -> { // After moving data from staging area to the target table (airybte_raw) clean up the staging // area (if user configured) @@ -103,7 +112,11 @@ public static OnCloseFunction onCloseFunction(final JdbcDatabase database, stageName); stagingOperations.dropStageIfExists(database, stageName); } + + typerDeduper.typeAndDedupe(writeConfig.getNamespace(), writeConfig.getStreamName()); } + + typerDeduper.commitFinalTables(); log.info("Cleaning up destination completed."); }; } diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java index 57a1054dbaff..1757bfbd3c23 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java @@ -10,6 +10,8 @@ import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.jdbc.WriteConfig; import io.airbyte.integrations.destination.record_buffer.FlushBufferFunction; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -47,7 +49,9 @@ public static FlushBufferFunction function( final JdbcDatabase database, final StagingOperations stagingOperations, final List writeConfigs, - final ConfiguredAirbyteCatalog catalog) { + final ConfiguredAirbyteCatalog catalog, + TypeAndDedupeOperationValve typerDeduperValve, + TyperDeduper typerDeduper) { // TODO: (ryankfu) move this block of code that executes before the lambda to #onStartFunction final Set conflictingStreams = new HashSet<>(); final Map pairToWriteConfig = new HashMap<>(); @@ -86,7 +90,11 @@ public static FlushBufferFunction function( final String stagedFile = stagingOperations.uploadRecordsToStage(database, writer, schemaName, stageName, stagingPath); GeneralStagingFunctions.copyIntoTableFromStage(database, stageName, stagingPath, List.of(stagedFile), writeConfig.getOutputTableName(), schemaName, - stagingOperations); + stagingOperations, + writeConfig.getNamespace(), + writeConfig.getStreamName(), + typerDeduperValve, + typerDeduper); } catch (final Exception e) { log.error("Failed to flush and commit buffer data into destination's raw table", e); throw new RuntimeException("Failed to upload buffer to stage and commit to destination", e); diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java index c6cd63aebfe8..4334e90a1d2d 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java @@ -13,6 +13,11 @@ import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; import io.airbyte.integrations.destination.jdbc.WriteConfig; @@ -65,39 +70,48 @@ public AirbyteMessageConsumer create(final Consumer outputRecord final BufferCreateFunction onCreateBuffer, final JsonNode config, final ConfiguredAirbyteCatalog catalog, - final boolean purgeStagingData) { - final List writeConfigs = createWriteConfigs(namingResolver, config, catalog); + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace) { + final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog); return new BufferedStreamConsumer( outputRecordCollector, - GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs), + GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), new SerializedBufferingStrategy( onCreateBuffer, catalog, - SerialFlush.function(database, stagingOperations, writeConfigs, catalog)), - GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData), + SerialFlush.function(database, stagingOperations, writeConfigs, catalog, typerDeduperValve, typerDeduper)), + GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData, typerDeduper), catalog, - stagingOperations::isValidData); + stagingOperations::isValidData, + defaultNamespace); } public SerializedAirbyteMessageConsumer createAsync(final Consumer outputRecordCollector, final JdbcDatabase database, final StagingOperations stagingOperations, final NamingConventionTransformer namingResolver, - final BufferCreateFunction onCreateBuffer, final JsonNode config, final ConfiguredAirbyteCatalog catalog, - final boolean purgeStagingData) { - final List writeConfigs = createWriteConfigs(namingResolver, config, catalog); + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace) { + final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog); final var streamDescToWriteConfig = streamDescToWriteConfig(writeConfigs); - final var flusher = new AsyncFlush(streamDescToWriteConfig, stagingOperations, database, catalog); + final var flusher = new AsyncFlush(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper); return new AsyncStreamConsumer( outputRecordCollector, - GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs), + GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), // todo (cgardens) - wrapping the old close function to avoid more code churn. - () -> GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData).accept(false), + () -> GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData, typerDeduper).accept(false), flusher, catalog, - new BufferManager()); + new BufferManager(), + defaultNamespace); } private static Map streamDescToWriteConfig(final List writeConfigs) { @@ -141,21 +155,30 @@ private static StreamDescriptor toStreamDescriptor(final WriteConfig config) { */ private static List createWriteConfigs(final NamingConventionTransformer namingResolver, final JsonNode config, - final ConfiguredAirbyteCatalog catalog) { + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog) { - return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config)).collect(toList()); + return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config, parsedCatalog)).collect(toList()); } private static Function toWriteConfig(final NamingConventionTransformer namingResolver, - final JsonNode config) { + final JsonNode config, + final ParsedCatalog parsedCatalog) { return stream -> { Preconditions.checkNotNull(stream.getDestinationSyncMode(), "Undefined destination sync mode"); final AirbyteStream abStream = stream.getStream(); - - final String outputSchema = getOutputSchema(abStream, config.get("schema").asText(), namingResolver); - final String streamName = abStream.getName(); - final String tableName = namingResolver.getRawTableName(streamName); + + final String outputSchema; + final String tableName; + if (TypingAndDedupingFlag.isDestinationV2()) { + final StreamId streamId = parsedCatalog.getStream(abStream.getNamespace(), streamName).id(); + outputSchema = streamId.rawNamespace(); + tableName = streamId.rawName(); + } else { + outputSchema = getOutputSchema(abStream, config.get("schema").asText(), namingResolver); + tableName = namingResolver.getRawTableName(streamName); + } final String tmpTableName = namingResolver.getTmpTableName(streamName); final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java b/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java index 8933da9b0f48..5c375a0bc6d9 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java @@ -15,7 +15,9 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.jdbc.SqlOperations; import io.airbyte.integrations.destination.s3.S3DestinationConfig; @@ -95,6 +97,8 @@ private record CopyArguments(JdbcDatabase database, @BeforeEach public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + s3Client = mock(AmazonS3Client.class); db = mock(JdbcDatabase.class); sqlOperations = mock(SqlOperations.class); diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java b/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java index fa7ebb26d7f1..7d3f76f43119 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java +++ b/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java @@ -23,6 +23,8 @@ void detectConflictingStreams() { List.of( new WriteConfig("example_stream", "source_schema", "destination_default_schema", null, null, null), new WriteConfig("example_stream", "source_schema", "destination_default_schema", null, null, null)), + null, + null, null)); assertEquals( diff --git a/airbyte-integrations/bases/connector-acceptance-test/.dockerignore b/airbyte-integrations/bases/connector-acceptance-test/.dockerignore index e8e8f8e5814b..b93d78698493 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/.dockerignore +++ b/airbyte-integrations/bases/connector-acceptance-test/.dockerignore @@ -1,5 +1,7 @@ * !Dockerfile !connector_acceptance_test -!setup.py +!pyproject.toml +!poetry.lock !pytest.ini +!requirements.txt \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index 8687ca454db9..993653cff454 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## 1.0.1 +Pin airbyte-protocol-model to <1.0.0. + +## 1.0.0 +Bump to Python 3.10, use dagger instead of docker-py in the ConnectorRunner. + +## 0.11.5 +Changing test output and adding diff to test_read + +## 0.11.4 +Relax checking of `oneOf` common property and allow optional `default` keyword additional to `const` keyword. + +## 0.11.3 +Refactor test_oauth_flow_parameters to validate advanced_auth instead of the deprecated authSpecification + +## 0.11.2 +Do not enforce spec.json/spec.yaml + +## 0.11.1 +Test connector image labels and make sure they are set correctly and match metadata.yaml. + +## 0.11.0 +Add backward_compatibility.check_if_field_removed test to check if a field has been removed from the catalog. + +## 0.10.8 +Increase the connection timeout to Docker client to 2 minutes ([context](https://github.com/airbytehq/airbyte/issues/27401)) + +## 0.10.7 +Fix on supporting arrays in the state (ensure string are parsed as string and not int) + +## 0.10.6 +Supporting arrays in the state by allowing ints in cursor_paths + ## 0.10.5 Skipping test_catalog_has_supported_data_types as it is failing on too many connectors. Will first address globally the type/format problems at scale and then re-enable it. diff --git a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile index 66065cb8f566..d70d45c824d7 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile @@ -1,39 +1,30 @@ -FROM python:3.9.11-alpine3.15 as base +FROM python:3.10.12 as base -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ +ENV ACCEPTANCE_TEST_DOCKER_CONTAINER 1 +ENV DOCKER_VERSION = "24.0.2" + +RUN apt-get update \ && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base + && apt-get install tzdata bash curl -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN python setup.py egg_info -RUN pip install --prefix=/install -r *.egg-info/requires.txt +# Docker is required for the dagger in docker use case. +RUN curl -fsSL https://get.docker.com | sh -# build a clean environment -FROM base -WORKDIR /airbyte/connector_acceptance_test -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone -# Bash is installed for more convenient debugging. -RUN apk --no-cache add bash -ENV ACCEPTANCE_TEST_DOCKER_CONTAINER 1 +RUN pip install poetry==1.5.1 +RUN poetry config virtualenvs.create false +RUN echo "Etc/UTC" > /etc/timezone -# copy payload code only -COPY pytest.ini setup.py ./ -COPY connector_acceptance_test ./connector_acceptance_test -RUN pip install . +WORKDIR /app +COPY pyproject.toml /app +COPY poetry.lock /app +RUN poetry install --no-root --only main --no-interaction --no-ansi +COPY . /app +RUN poetry install --only main --no-cache --no-interaction --no-ansi -LABEL io.airbyte.version=0.10.5 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/connector-acceptance-test - -ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx"] +WORKDIR /test_input +ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx", "--show-capture=log"] diff --git a/airbyte-integrations/bases/connector-acceptance-test/README.md b/airbyte-integrations/bases/connector-acceptance-test/README.md index c7fc6c2be00a..ec0671e4dbf6 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/README.md +++ b/airbyte-integrations/bases/connector-acceptance-test/README.md @@ -1,78 +1,116 @@ -# Connector Acceptance Tests +# Connector Acceptance Tests (CAT) This package gathers multiple test suites to assess the sanity of any Airbyte connector. It is shipped as a [pytest](https://docs.pytest.org/en/7.1.x/) plugin and relies on pytest to discover, configure and execute tests. Test-specific documentation can be found [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/)). -## Running the acceptance tests on a source connector: -1. `cd` into your connector project (e.g. `airbyte-integrations/connectors/source-pokeapi`) -2. Edit `acceptance-test-config.yml` according to your need. Please refer to our [Connector Acceptance Test Reference](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) if you need details about the available options. -3. Build the connector docker image ( e.g.: `docker build . -t airbyte/source-pokeapi:dev`) -4. Use one of the following ways to run tests (**from your connector project directory**) +## Configuration +The acceptance tests are configured via the `acceptance-test-config.yml` YAML file, which is passed to the plugin via the `--acceptance-test-config` option. + +## Running the acceptance tests locally +Note there are MANY ways to do this at this time, but we are working on consolidating them. + +Which method you choose to use depends on the context you are in. + +Pre-requisites: +- Setting up a Service Account for Google Secrets Manager (GSM) access. See [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md) +- Ensuring that you have the `GCP_GSM_CREDENTIALS` environment variable set to the contents of your GSM service account key file. +- [Poetry](https://python-poetry.org/docs/#installation) installed +- [Pipx](https://pypa.github.io/pipx/installation/) installed + +### Running CAT in the same environment as our CI/CD pipeline (`airbyte-ci`) -### Using python -_Note: these will assume that docker image for connector is already built_ +_Note: Install instructions for airbyte-ci are [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) _ -**Running the whole suite** +**This runs connector acceptance and other tests that run in our CI** ```bash -python -m pytest integration_tests -p integration_tests.acceptance +airbyte-ci connectors --name= test +``` + +### Running CAT locally for Debugging/Development Purposes + +**Pre-requisites:** + +To learn how to set up `ci_credentials` and your GSM Service account see [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md) + +```bash +# Hook up your GSM service account +export GCP_GSM_CREDENTIALS=`cat ` + +# Install the credentials tool +pipx install airbyte-ci/connectors/ci_credentials/ ``` -**Running a specific test** +**Retrieve a connectors sandbox secrets** + ```bash -python -m pytest integration_tests -p integration_tests.acceptance -k "" +# From the root of the airbyte repo + +# Writes the secrets to airbyte-integrations/connectors/source-faker/secrets +VERSION=dev ci_credentials connectors/source-faker write-to-storage ``` +**Run install dependencies** -### Using Gradle ```bash -./gradlew :airbyte-integrations:connectors:source-:connectorAcceptanceTest +# Navigate to our CAT test directory +cd airbyte-integrations/bases/connector-acceptance-test/ + +# Install dependencies +poetry install ``` -_Note: this way will also build docker image for the connector_ -### Using Bash +**Run the tests** + ```bash -./acceptance-test-docker.sh +# Run tests against your connector +poetry run pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/source-faker --pdb ``` -_Note: this will use the latest docker image for connector-acceptance-test and will also build docker image for the connector_ -You can also use the following environment variables with the Gradle and Bash commands: -- `LOCAL_CDK=1`: Run tests against the local python CDK, if relevant. If not set, tests against the latest package published to pypi, or the version specified in the connector's setup.py. -- `FETCH_SECRETS=1`: Fetch secrets required by CATs. This requires you to have a Google Service Account, and the GCP_GSM_CREDENTIALS environment variable to be set, per the instructions [here](https://github.com/airbytehq/airbyte/tree/b03653a24ef16be641333380f3a4d178271df0ee/tools/ci_credentials). +### Running CAT via the production docker image (deprecated) +This is the old method and is not useful outside of helping third party connector developers run their tests. -## Running the acceptance tests on multiple connectors: -If you are contributing to the python CDK, you may want to validate your changes by running acceptance tests against multiple connectors. +Ideally you should use `airbyte-ci` as described above. + +_Note: To use `FETCH_SECRETS=1` you must have `ci_credentials` and your GSM Service account setup see [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md)_ + + +```bash +# Navigate to the connectors folder +cd airbyte-integrations/connectors/source-faker -To do so, from the root of the `airbyte` repo, run `./airbyte-cdk/python/bin/run-cats-with-local-cdk.sh -c ,,...` +# Run the tests +FETCH_SECRETS=1 ./acceptance-test-docker.sh +``` + +Note you can also use `LOCAL_CDK=1` to run tests against the local python CDK, if relevant. If not set, tests against the latest package published to pypi, or the version specified in the connector's setup.py. + +### Manually +1. `cd` into your connector project (e.g. `airbyte-integrations/connectors/source-pokeapi`) +2. Edit `acceptance-test-config.yml` according to your need. Please refer to our [Connector Acceptance Test Reference](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) if you need details about the available options. +3. Build the connector docker image ( e.g.: `docker build . -t airbyte/source-pokeapi:dev`) +4. Use one of the following ways to run tests (**from your connector project directory**) -## When does acceptance test run? -* When running local acceptance tests on connector: - * When running `connectorAcceptanceTest` `gradle` task - * When running or `./acceptance-test-docker.sh` in a connector project -* When running `/test` command on a GitHub pull request. -* When running ` integration-test` GitHub action. This is the same action that creates and uploads the test report JSON files that power the badges in the [connector registry summary report](https://connectors.airbyte.com/files/generated_reports/connector_registry_report.html). ## Developing on the acceptance tests You may want to iterate on the acceptance test project itself: adding new tests, fixing a bug etc. These iterations are more conveniently achieved by remaining in the current directory. -1. Create a `virtualenv`: `python -m venv .venv` -2. Activate the `virtualenv`: `source ./.venv/bin/activate` -3. Install requirements: `pip install -e .` -4. Run the unit tests on the acceptance tests themselves: `python -m pytest unit_tests` (add the `--pdb` option if you want to enable the debugger on test failure) -5. Make the changes you want: +1. Install dependencies via `poetry install` +3. Run the unit tests on the acceptance tests themselves: `poetry run python -m pytest unit_tests` (add the `--pdb` option if you want to enable the debugger on test failure) +4. Make the changes you want: * Global pytest fixtures are defined in `./connector_acceptance_test/conftest.py` * Existing test modules are defined in `./connector_acceptance_test/tests` * `acceptance-test-config.yaml` structure is defined in `./connector_acceptance_test/config.py` -6. Unit test your changes by adding tests to `./unit_tests` -7. Run the unit tests on the acceptance tests again: `python -m pytest unit_tests`, make sure the coverage did not decrease. You can bypass slow tests by using the `slow` marker: `python -m pytest unit_tests -m "not slow"`. -8. Manually test the changes you made by running acceptance tests on a specific connector. e.g. `python -m pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/source-pokeapi` -9. Make sure you updated `docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md` according to your changes -10. Bump the acceptance test docker image version in `airbyte-integrations/bases/connector-acceptance-test/Dockerfile` -11. Update the project changelog `airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md` -12. Open a PR on our GitHub repository -13. Run the unit test on the CI by running `/test connector=bases/connector-acceptance-test` in a GitHub comment -14. Publish the new acceptance test version if your PR is approved by running `/publish connector=bases/connector-acceptance-test auto-bump-version=false` in a GitHub comment -15. Merge your PR +5. Unit test your changes by adding tests to `./unit_tests` +6. Run the unit tests on the acceptance tests again: `poetry run pytest unit_tests`, make sure the coverage did not decrease. You can bypass slow tests by using the `slow` marker: `poetry run pytest unit_tests -m "not slow"`. +7. Manually test the changes you made by running acceptance tests on a specific connector. e.g. `poetry run pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/source-pokeapi` +8. Make sure you updated `docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md` according to your changes +9. Bump the acceptance test docker image version in `airbyte-integrations/bases/connector-acceptance-test/Dockerfile` +10. Update the project changelog `airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md` +11. Open a PR on our GitHub repository +12. This [Github action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered an run the unit tests on your branch. +13. Publish the new acceptance test version if your PR is approved by running `/legacy-publish connector=bases/connector-acceptance-test run-tests=false` in a GitHub comment +14. Merge your PR ## Migrating `acceptance-test-config.yml` to latest configuration format We introduced changes in the structure of `acceptance-test-config.yml` files in version 0.2.12. diff --git a/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh b/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh index 9c8fbf9d6324..e599e8549b5a 100755 --- a/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh +++ b/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh @@ -16,7 +16,10 @@ CONNECTOR_DIR="$ROOT_DIR/airbyte-integrations/connectors/$CONNECTOR_NAME" if [ -n "$FETCH_SECRETS" ]; then cd $ROOT_DIR - VERSION=dev $ROOT_DIR/tools/.venv/bin/ci_credentials $CONNECTOR_NAME write-to-storage || true + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/ci_credentials + VERSION=dev ci_credentials $CONNECTOR_NAME write-to-storage || true cd - fi diff --git a/airbyte-integrations/bases/connector-acceptance-test/build.gradle b/airbyte-integrations/bases/connector-acceptance-test/build.gradle index 2f20e9f479c6..55c378f71fb7 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/build.gradle +++ b/airbyte-integrations/bases/connector-acceptance-test/build.gradle @@ -1,9 +1,6 @@ plugins { + // airbyte-docker is kept to support /legacy-publish until airbyte-ci cat publish command exists. id 'airbyte-docker' - id 'airbyte-python' } -airbytePython { - moduleDirectory 'connector_acceptance_test' -} diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index b4ce92f9e607..b79b939d958b 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -7,7 +7,7 @@ from copy import deepcopy from enum import Enum from pathlib import Path -from typing import Generic, List, Mapping, Optional, Set, TypeVar +from typing import Generic, List, Mapping, Optional, Set, TypeVar, Union from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel @@ -160,7 +160,7 @@ class FutureStateConfig(BaseConfig): class IncrementalConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path - cursor_paths: Optional[Mapping[str, List[str]]] = Field( + cursor_paths: Optional[Mapping[str, List[Union[int, str]]]] = Field( description="For each stream, the path of its cursor field in the output state messages." ) future_state: Optional[FutureStateConfig] = Field(description="Configuration for the future state.") @@ -174,6 +174,9 @@ class IncrementalConfig(BaseConfig): description="Determines whether to skip more granular testing for incremental syncs", default=False ) + class Config: + smart_union = True + class GenericTestConfig(GenericModel, Generic[TestConfigT]): bypass_reason: Optional[str] @@ -200,9 +203,6 @@ class TestStrictnessLevel(str, Enum): high = "high" low = "low" - cache_discovered_catalog: bool = Field( - default=True, description="Enable or disable caching of discovered catalog for reuse in multiple tests." - ) connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'") acceptance_tests: AcceptanceTestConfigurations = Field(description="List of the acceptance test to run with their configs") base_path: Optional[str] = Field(description="Base path for all relative paths") diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 0dddf960dad4..2d25cba6fd29 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -8,14 +8,16 @@ import json import logging import os +import sys from glob import glob from logging import Logger from pathlib import Path from subprocess import STDOUT, check_output, run from typing import Any, List, Mapping, MutableMapping, Optional, Set +import dagger import pytest -from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConnectorSpecification, Type +from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, ConnectorSpecification, Type from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import Config, EmptyStreamConfiguration, ExpectedRecordsConfig, IgnoredFieldsConfiguration from connector_acceptance_test.tests import TestBasicRead @@ -27,9 +29,7 @@ filter_output, load_config, load_yaml_or_json_path, - make_hashable, ) -from docker import errors @pytest.fixture(name="acceptance_test_config", scope="session") @@ -51,11 +51,6 @@ def test_strictness_level_fixture(acceptance_test_config: Config) -> Config.Test return acceptance_test_config.test_strictness_level -@pytest.fixture(name="cache_discovered_catalog", scope="session") -def cache_discovered_catalog_fixture(acceptance_test_config: Config) -> bool: - return acceptance_test_config.cache_discovered_catalog - - @pytest.fixture(name="custom_environment_variables", scope="session") def custom_environment_variables_fixture(acceptance_test_config: Config) -> Mapping: return acceptance_test_config.custom_environment_variables @@ -140,19 +135,42 @@ def malformed_connector_config_fixture(connector_config) -> MutableMapping[str, @pytest.fixture(name="connector_spec") -def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification: - spec_obj = load_yaml_or_json_path(connector_spec_path) - return ConnectorSpecification.parse_obj(spec_obj) +def connector_spec_fixture(connector_spec_path) -> Optional[ConnectorSpecification]: + try: + spec_obj = load_yaml_or_json_path(connector_spec_path) + return ConnectorSpecification.parse_obj(spec_obj) + except FileNotFoundError: + return None + + +@pytest.fixture(scope="session") +def anyio_backend(): + """Determine the anyio backend to use for the tests. + anyio allows us to run async tests. + """ + return "asyncio" -@pytest.fixture(name="docker_runner") -def docker_runner_fixture(image_tag, tmp_path, connector_config_path, custom_environment_variables) -> ConnectorRunner: - return ConnectorRunner( +@pytest.fixture(scope="session") +async def dagger_client(anyio_backend): + """Exposes a Dagger client available for the whole test session. + Dagger is a tool to programmatically create and interact with containers with out of the box caching. + More info here: https://dagger.io/ + """ + async with dagger.Connection(config=dagger.Config(log_output=sys.stderr)) as client: + yield client + + +@pytest.fixture(name="docker_runner", autouse=True) +async def docker_runner_fixture(image_tag, connector_config_path, custom_environment_variables, dagger_client) -> ConnectorRunner: + runner = ConnectorRunner( image_tag, - volume=tmp_path, + dagger_client, connector_configuration_path=connector_config_path, custom_environment_variables=custom_environment_variables, ) + await runner.load_container() + return runner @pytest.fixture(name="previous_connector_image_name") @@ -162,32 +180,14 @@ def previous_connector_image_name_fixture(image_tag, inputs) -> str: @pytest.fixture(name="previous_connector_docker_runner") -def previous_connector_docker_runner_fixture(previous_connector_image_name, tmp_path) -> ConnectorRunner: +async def previous_connector_docker_runner_fixture(previous_connector_image_name, dagger_client) -> ConnectorRunner: """Fixture to create a connector runner with the previous connector docker image. Returns None if the latest image was not found, to skip downstream tests if the current connector is not yet published to the docker registry. Raise not found error if the previous connector image is not latest and expected to be published. """ - try: - return ConnectorRunner(previous_connector_image_name, volume=tmp_path / "previous_connector") - except (errors.NotFound, errors.ImageNotFound) as e: - if previous_connector_image_name.endswith("latest"): - logging.warning( - f"\n We did not find the {previous_connector_image_name} image for this connector. This probably means this version has not yet been published to an accessible docker registry like DockerHub." - ) - return None - else: - raise e - - -@pytest.fixture(scope="session", autouse=True) -def pull_docker_image(acceptance_test_config) -> None: - """Startup fixture to pull docker image""" - image_name = acceptance_test_config.connector_image - config_filename = "acceptance-test-config.yml" - try: - ConnectorRunner(image_name=image_name, volume=Path(".")) - except errors.ImageNotFound: - pytest.exit(f"Docker image `{image_name}` not found, please check your {config_filename} file", returncode=1) + runner = ConnectorRunner(previous_connector_image_name, dagger_client) + await runner.load_container() + return runner @pytest.fixture(name="empty_streams") @@ -266,41 +266,23 @@ def find_not_seeded_streams( return expected_seeded_stream_names - expected_record_stream_names -@pytest.fixture(name="cached_schemas", scope="session") -def cached_schemas_fixture() -> MutableMapping[str, AirbyteStream]: - """Simple cache for discovered catalog: stream_name -> json_schema""" - return {} - - -@pytest.fixture(name="previous_cached_schemas", scope="session") -def previous_cached_schemas_fixture() -> MutableMapping[str, AirbyteStream]: - """Simple cache for discovered catalog of previous connector: stream_name -> json_schema""" - return {} - - @pytest.fixture(name="discovered_catalog") -def discovered_catalog_fixture( - connector_config, docker_runner: ConnectorRunner, cached_schemas, cache_discovered_catalog: bool +async def discovered_catalog_fixture( + connector_config, + docker_runner: ConnectorRunner, ) -> MutableMapping[str, AirbyteStream]: """JSON schemas for each stream""" - cached_schemas = cached_schemas.setdefault(make_hashable(connector_config), {}) - if not cache_discovered_catalog: - cached_schemas.clear() - if not cached_schemas: - output = docker_runner.call_discover(config=connector_config) - catalogs = [message.catalog for message in output if message.type == Type.CATALOG] - for stream in catalogs[-1].streams: - cached_schemas[stream.name] = stream - return cached_schemas + + output = await docker_runner.call_discover(config=connector_config) + catalogs = [message.catalog for message in output if message.type == Type.CATALOG] + return {stream.name: stream for stream in catalogs[-1].streams} @pytest.fixture(name="previous_discovered_catalog") -def previous_discovered_catalog_fixture( +async def previous_discovered_catalog_fixture( connector_config, previous_connector_image_name, previous_connector_docker_runner: ConnectorRunner, - previous_cached_schemas, - cache_discovered_catalog: bool, ) -> MutableMapping[str, AirbyteStream]: """JSON schemas for each stream""" if previous_connector_docker_runner is None: @@ -308,21 +290,15 @@ def previous_discovered_catalog_fixture( f"\n We could not retrieve the previous discovered catalog as a connector runner for the previous connector version ({previous_connector_image_name}) could not be instantiated." ) return None - previous_cached_schemas = previous_cached_schemas.setdefault(make_hashable(connector_config), {}) - if not cache_discovered_catalog: - previous_cached_schemas.clear() - if not previous_cached_schemas: - try: - output = previous_connector_docker_runner.call_discover(config=connector_config) - except errors.ContainerError: - logging.warning( - "\n DISCOVER on the previous connector version failed. This could be because the current connector config is not compatible with the previous connector version." - ) - return None - catalogs = [message.catalog for message in output if message.type == Type.CATALOG] - for stream in catalogs[-1].streams: - previous_cached_schemas[stream.name] = stream - return previous_cached_schemas + try: + output = await previous_connector_docker_runner.call_discover(config=connector_config) + except dagger.DaggerError: + logging.warning( + "\n DISCOVER on the previous connector version failed. This could be because the current connector config is not compatible with the previous connector version." + ) + return None + catalogs = [message.catalog for message in output if message.type == Type.CATALOG] + return {stream.name: stream for stream in catalogs[-1].streams} @pytest.fixture @@ -347,18 +323,15 @@ def detailed_logger() -> Logger: @pytest.fixture(name="actual_connector_spec") -def actual_connector_spec_fixture(request: BaseTest, docker_runner: ConnectorRunner) -> ConnectorSpecification: - if not request.instance.spec_cache: - output = docker_runner.call_spec() - spec_messages = filter_output(output, Type.SPEC) - assert len(spec_messages) == 1, "Spec message should be emitted exactly once" - spec = spec_messages[0].spec - request.instance.spec_cache = spec - return request.instance.spec_cache +async def actual_connector_spec_fixture(docker_runner: ConnectorRunner) -> ConnectorSpecification: + output = await docker_runner.call_spec() + spec_messages = filter_output(output, Type.SPEC) + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + return spec_messages[0].spec @pytest.fixture(name="previous_connector_spec") -def previous_connector_spec_fixture( +async def previous_connector_spec_fixture( request: BaseTest, previous_connector_docker_runner: ConnectorRunner ) -> Optional[ConnectorSpecification]: if previous_connector_docker_runner is None: @@ -366,13 +339,10 @@ def previous_connector_spec_fixture( "\n We could not retrieve the previous connector spec as a connector runner for the previous connector version could not be instantiated." ) return None - if not request.instance.previous_spec_cache: - output = previous_connector_docker_runner.call_spec() - spec_messages = filter_output(output, Type.SPEC) - assert len(spec_messages) == 1, "Spec message should be emitted exactly once" - spec = spec_messages[0].spec - request.instance.previous_spec_cache = spec - return request.instance.previous_spec_cache + output = await previous_connector_docker_runner.call_spec() + spec_messages = filter_output(output, Type.SPEC) + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + return spec_messages[0].spec def pytest_sessionfinish(session, exitstatus): @@ -396,3 +366,8 @@ def pytest_sessionfinish(session, exitstatus): except Exception as e: logger.info(e) # debug pass + + +@pytest.fixture(name="connector_metadata") +def connector_metadata_fixture(base_path) -> dict: + return load_yaml_or_json_path(base_path / "metadata.yaml") diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index 17a1ff606cc1..b0fb44c41fdb 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -15,7 +15,7 @@ import dpath.util import jsonschema import pytest -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteRecordMessage, AirbyteStream, AirbyteTraceMessage, @@ -46,8 +46,14 @@ find_all_values_for_key_in_schema, find_keyword_schema, ) -from connector_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure -from jsonschema._utils import flatten +from connector_acceptance_test.utils.compare import diff_dicts +from connector_acceptance_test.utils.json_schema_helper import ( + JsonSchemaHelper, + flatten_tuples, + get_expected_schema_structure, + get_object_structure, + get_paths_in_connector_config, +) @pytest.fixture(name="connector_spec_dict") @@ -82,12 +88,9 @@ def secret_property_names_fixture(): DATETIME_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$" -@pytest.mark.default_timeout(10) +# The connector fixture can be long to load, we have to increase the default timeout... +@pytest.mark.default_timeout(5 * 60) class TestSpec(BaseTest): - - spec_cache: ConnectorSpecification = None - previous_spec_cache: ConnectorSpecification = None - @pytest.fixture(name="skip_backward_compatibility_tests") def skip_backward_compatibility_tests_fixture( self, @@ -113,7 +116,6 @@ def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, """Check that config matches the actual schema from the spec call""" # Getting rid of technical variables that start with an underscore config = {key: value for key, value in connector_config.data.items() if not key.startswith("_")} - try: jsonschema.validate(instance=config, schema=actual_connector_spec.connectionSpecification) except jsonschema.exceptions.ValidationError as err: @@ -125,13 +127,8 @@ def test_match_expected(self, connector_spec: ConnectorSpecification, actual_con """Check that spec call returns a spec equals to expected one""" if connector_spec: assert actual_connector_spec == connector_spec, "Spec should be equal to the one in spec.yaml or spec.json file" - - def test_docker_env(self, actual_connector_spec: ConnectorSpecification, docker_runner: ConnectorRunner): - """Check that connector's docker image has required envs""" - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT"), "AIRBYTE_ENTRYPOINT must be set in dockerfile" - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT") == " ".join( - docker_runner.entry_point - ), "env should be equal to space-joined entrypoint" + else: + pytest.skip("The spec.yaml or spec.json does not exist. Hence, comparison with the actual one can't be performed") def test_enum_usage(self, actual_connector_spec: ConnectorSpecification): """Check that enum lists in specs contain distinct values.""" @@ -184,11 +181,11 @@ def test_oneof_usage(self, actual_connector_spec: ConnectorSpecification): for n, variant in enumerate(variants): prop_obj = variant["properties"][const_common_prop] assert ( - "default" not in prop_obj - ), f"There should not be 'default' keyword in common property {oneof_path}[{n}].{const_common_prop}. Use `const` instead. {docs_msg}" - assert ( - "enum" not in prop_obj - ), f"There should not be 'enum' keyword in common property {oneof_path}[{n}].{const_common_prop}. Use `const` instead. {docs_msg}" + "default" not in prop_obj or prop_obj["default"] == prop_obj["const"] + ), f"'default' needs to be identical to const in common property {oneof_path}[{n}].{const_common_prop}. It's recommended to just use `const`. {docs_msg}" + assert "enum" not in prop_obj or ( + len(prop_obj["enum"]) == 1 and prop_obj["enum"][0] == prop_obj["const"] + ), f"'enum' needs to be an array with a single item identical to const in common property {oneof_path}[{n}].{const_common_prop}. It's recommended to just use `const`. {docs_msg}" def test_required(self): """Check that connector will fail if any required field is missing""" @@ -283,12 +280,12 @@ def _fail_on_errors(self, errors: List[str]): if len(errors) > 0: pytest.fail("\n".join(errors)) - def test_property_type_is_not_array(self, connector_spec: ConnectorSpecification): + def test_property_type_is_not_array(self, actual_connector_spec: ConnectorSpecification): """ Each field has one or multiple types, but the UI only supports a single type and optionally "null" as a second type. """ errors = [] - for type_path, type_value in dpath.util.search(connector_spec.connectionSpecification, "**/properties/*/type", yielded=True): + for type_path, type_value in dpath.util.search(actual_connector_spec.connectionSpecification, "**/properties/*/type", yielded=True): if isinstance(type_value, List): number_of_types = len(type_value) if number_of_types != 2 and number_of_types != 1: @@ -301,14 +298,14 @@ def test_property_type_is_not_array(self, connector_spec: ConnectorSpecification ) self._fail_on_errors(errors) - def test_object_not_empty(self, connector_spec: ConnectorSpecification): + def test_object_not_empty(self, actual_connector_spec: ConnectorSpecification): """ Each object field needs to have at least one property as the UI won't be able to show them otherwise. If the whole spec is empty, it's allowed to have a single empty object at the top level """ - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) errors = [] - for type_path, type_value in dpath.util.search(connector_spec.connectionSpecification, "**/type", yielded=True): + for type_path, type_value in dpath.util.search(actual_connector_spec.connectionSpecification, "**/type", yielded=True): if type_path == "type": # allow empty root object continue @@ -320,13 +317,13 @@ def test_object_not_empty(self, connector_spec: ConnectorSpecification): ) self._fail_on_errors(errors) - def test_array_type(self, connector_spec: ConnectorSpecification): + def test_array_type(self, actual_connector_spec: ConnectorSpecification): """ Each array has one or multiple types for its items, but the UI only supports a single type which can either be object, string or an enum """ - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) errors = [] - for type_path, type_type in dpath.util.search(connector_spec.connectionSpecification, "**/type", yielded=True): + for type_path, type_type in dpath.util.search(actual_connector_spec.connectionSpecification, "**/type", yielded=True): property_definition = schema_helper.get_parent(type_path) if type_type != "array": # unrelated "items", not an array definition @@ -340,7 +337,7 @@ def test_array_type(self, connector_spec: ConnectorSpecification): errors.append(f"Items of {type_path} has to be either object or string or define an enum") self._fail_on_errors(errors) - def test_forbidden_complex_types(self, connector_spec: ConnectorSpecification): + def test_forbidden_complex_types(self, actual_connector_spec: ConnectorSpecification): """ not, anyOf, patternProperties, prefixItems, allOf, if, then, else, dependentSchemas and dependentRequired are not allowed """ @@ -358,25 +355,27 @@ def test_forbidden_complex_types(self, connector_spec: ConnectorSpecification): ] found_keys = set() for forbidden_key in forbidden_keys: - for path, value in dpath.util.search(connector_spec.connectionSpecification, f"**/{forbidden_key}", yielded=True): + for path, value in dpath.util.search(actual_connector_spec.connectionSpecification, f"**/{forbidden_key}", yielded=True): found_keys.add(path) for forbidden_key in forbidden_keys: # remove forbidden keys if they are used as properties directly - for path, _value in dpath.util.search(connector_spec.connectionSpecification, f"**/properties/{forbidden_key}", yielded=True): + for path, _value in dpath.util.search( + actual_connector_spec.connectionSpecification, f"**/properties/{forbidden_key}", yielded=True + ): found_keys.remove(path) if len(found_keys) > 0: key_list = ", ".join(found_keys) pytest.fail(f"Found the following disallowed JSON schema features: {key_list}") - def test_date_pattern(self, connector_spec: ConnectorSpecification, detailed_logger): + def test_date_pattern(self, actual_connector_spec: ConnectorSpecification, detailed_logger): """ Properties with format date or date-time should always have a pattern defined how the date/date-time should be formatted that corresponds with the format the datepicker component is creating. """ - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) - for format_path, format in dpath.util.search(connector_spec.connectionSpecification, "**/format", yielded=True): + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) + for format_path, format in dpath.util.search(actual_connector_spec.connectionSpecification, "**/format", yielded=True): if not isinstance(format, str): # format is not a format definition here but a property named format continue @@ -391,12 +390,12 @@ def test_date_pattern(self, connector_spec: ConnectorSpecification, detailed_log f"{format_path} is defining a date-time format without the corresponding pattern Consider setting the pattern to {DATETIME_PATTERN} to make it easier for users to edit this field in the UI." ) - def test_date_format(self, connector_spec: ConnectorSpecification, detailed_logger): + def test_date_format(self, actual_connector_spec: ConnectorSpecification, detailed_logger): """ Properties with a pattern that looks like a date should have their format set to date or date-time. """ - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) - for pattern_path, pattern in dpath.util.search(connector_spec.connectionSpecification, "**/pattern", yielded=True): + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) + for pattern_path, pattern in dpath.util.search(actual_connector_spec.connectionSpecification, "**/pattern", yielded=True): if not isinstance(pattern, str): # pattern is not a pattern definition here but a property named pattern continue @@ -412,15 +411,15 @@ def test_date_format(self, connector_spec: ConnectorSpecification, detailed_logg f"{pattern_path} is defining a pattern that looks like a date-time without setting the format to `date-time`. Consider specifying the format to make it easier for users to edit this field in the UI." ) - def test_duplicate_order(self, connector_spec: ConnectorSpecification): + def test_duplicate_order(self, actual_connector_spec: ConnectorSpecification): """ Custom ordering of field (via the "order" property defined in the field) is not allowed to have duplicates within the same group. `{ "a": { "order": 1 }, "b": { "order": 1 } }` is invalid because there are two fields with order 1 `{ "a": { "order": 1 }, "b": { "order": 1, "group": "x" } }` is valid because the fields with the same order are in different groups """ - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) errors = [] - for properties_path, properties in dpath.util.search(connector_spec.connectionSpecification, "**/properties", yielded=True): + for properties_path, properties in dpath.util.search(actual_connector_spec.connectionSpecification, "**/properties", yielded=True): definition = schema_helper.get_parent(properties_path) if definition.get("type") != "object": # unrelated "properties", not an actual object definition @@ -439,15 +438,15 @@ def test_duplicate_order(self, connector_spec: ConnectorSpecification): orders_for_group.add(order) self._fail_on_errors(errors) - def test_nested_group(self, connector_spec: ConnectorSpecification): + def test_nested_group(self, actual_connector_spec: ConnectorSpecification): """ Groups can only be defined on the top level properties `{ "a": { "group": "x" }}` is valid because field "a" is a top level field `{ "a": { "oneOf": [{ "type": "object", "properties": { "b": { "group": "x" } } }] }}` is invalid because field "b" is nested in a oneOf """ errors = [] - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) - for result in dpath.util.search(connector_spec.connectionSpecification, "/properties/**/group", yielded=True): + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) + for result in dpath.util.search(actual_connector_spec.connectionSpecification, "/properties/**/group", yielded=True): group_path = result[0] parent_path = schema_helper.get_parent_path(group_path) is_property_named_group = parent_path.endswith("properties") @@ -456,13 +455,13 @@ def test_nested_group(self, connector_spec: ConnectorSpecification): errors.append(f"Groups can only be defined on top level, is defined at {group_path}") self._fail_on_errors(errors) - def test_required_always_show(self, connector_spec: ConnectorSpecification): + def test_required_always_show(self, actual_connector_spec: ConnectorSpecification): """ Fields with always_show are not allowed to be required fields because only optional fields can be hidden in the form in the first place. """ errors = [] - schema_helper = JsonSchemaHelper(connector_spec.connectionSpecification) - for result in dpath.util.search(connector_spec.connectionSpecification, "/properties/**/always_show", yielded=True): + schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) + for result in dpath.util.search(actual_connector_spec.connectionSpecification, "/properties/**/always_show", yielded=True): always_show_path = result[0] parent_path = schema_helper.get_parent_path(always_show_path) is_property_named_always_show = parent_path.endswith("properties") @@ -488,25 +487,31 @@ def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecificati """Check if connector has correct oauth flow parameters according to https://docs.airbyte.io/connector-development/connector-specification-reference """ - if not actual_connector_spec.authSpecification: + advanced_auth = actual_connector_spec.advanced_auth + if not advanced_auth: return spec_schema = actual_connector_spec.connectionSpecification - oauth_spec = actual_connector_spec.authSpecification.oauth2Specification - parameters: List[List[str]] = oauth_spec.oauthFlowInitParameters + oauth_spec.oauthFlowOutputParameters - root_object = oauth_spec.rootObject - if len(root_object) == 0: - params = {"/" + "/".join(p) for p in parameters} - schema_path = set(get_expected_schema_structure(spec_schema)) - elif len(root_object) == 1: - params = {"/" + "/".join([root_object[0], *p]) for p in parameters} - schema_path = set(get_expected_schema_structure(spec_schema)) - elif len(root_object) == 2: - params = {"/" + "/".join([f"{root_object[0]}({root_object[1]})", *p]) for p in parameters} - schema_path = set(get_expected_schema_structure(spec_schema, annotate_one_of=True)) - else: - pytest.fail("rootObject cannot have more than 2 elements") + paths_to_validate = set() + if advanced_auth.predicate_key: + paths_to_validate.add("/" + "/".join(advanced_auth.predicate_key)) + oauth_config_specification = advanced_auth.oauth_config_specification + if oauth_config_specification: + if oauth_config_specification.oauth_user_input_from_connector_config_specification: + paths_to_validate.update( + get_paths_in_connector_config( + oauth_config_specification.oauth_user_input_from_connector_config_specification["properties"] + ) + ) + if oauth_config_specification.complete_oauth_output_specification: + paths_to_validate.update( + get_paths_in_connector_config(oauth_config_specification.complete_oauth_output_specification["properties"]) + ) + if oauth_config_specification.complete_oauth_server_output_specification: + paths_to_validate.update( + get_paths_in_connector_config(oauth_config_specification.complete_oauth_server_output_specification["properties"]) + ) - diff = params - schema_path + diff = paths_to_validate - set(get_expected_schema_structure(spec_schema)) assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}" @pytest.mark.default_timeout(60) @@ -538,24 +543,44 @@ def test_additional_properties_is_true(self, actual_connector_spec: ConnectorSpe [additional_properties_value is True for additional_properties_value in additional_properties_values] ), "When set, additionalProperties field value must be true for backward compatibility." + # This test should not be part of TestSpec because it's testing the connector's docker image content, not the spec itself + # But it's cumbersome to declare a separate, non configurable, test class + # See https://github.com/airbytehq/airbyte/issues/15551 + async def test_image_labels(self, docker_runner: ConnectorRunner, connector_metadata: dict): + """Check that connector's docker image has required labels""" + assert ( + await docker_runner.get_container_label("io.airbyte.name") == connector_metadata["data"]["dockerRepository"] + ), "io.airbyte.name must be equal to dockerRepository in metadata.yaml" + assert ( + await docker_runner.get_container_label("io.airbyte.version") == connector_metadata["data"]["dockerImageTag"] + ), "io.airbyte.version must be equal to dockerImageTag in metadata.yaml" + + # This test should not be part of TestSpec because it's testing the connector's docker image content, not the spec itself + # But it's cumbersome to declare a separate, non configurable, test class + # See https://github.com/airbytehq/airbyte/issues/15551 + async def test_image_environment_variables(self, docker_runner: ConnectorRunner): + """Check that connector's docker image has required envs""" + assert await docker_runner.get_container_env_variable_value("AIRBYTE_ENTRYPOINT"), "AIRBYTE_ENTRYPOINT must be set in dockerfile" + assert await docker_runner.get_container_env_variable_value("AIRBYTE_ENTRYPOINT") == await docker_runner.get_container_entrypoint() + @pytest.mark.default_timeout(30) class TestConnection(BaseTest): - def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runner: ConnectorRunner): + async def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runner: ConnectorRunner): if inputs.status == ConnectionTestConfig.Status.Succeed: - output = docker_runner.call_check(config=connector_config) + output = await docker_runner.call_check(config=connector_config) con_messages = filter_output(output, Type.CONNECTION_STATUS) assert len(con_messages) == 1, "Connection status message should be emitted exactly once" assert con_messages[0].connectionStatus.status == Status.SUCCEEDED elif inputs.status == ConnectionTestConfig.Status.Failed: - output = docker_runner.call_check(config=connector_config) + output = await docker_runner.call_check(config=connector_config) con_messages = filter_output(output, Type.CONNECTION_STATUS) assert len(con_messages) == 1, "Connection status message should be emitted exactly once" assert con_messages[0].connectionStatus.status == Status.FAILED elif inputs.status == ConnectionTestConfig.Status.Exception: - output = docker_runner.call_check(config=connector_config, raise_container_error=False) + output = await docker_runner.call_check(config=connector_config, raise_container_error=False) trace_messages = filter_output(output, Type.TRACE) assert len(trace_messages) == 1, "A trace message should be emitted in case of unexpected errors" trace = trace_messages[0].trace @@ -607,9 +632,9 @@ def skip_backward_compatibility_tests_fixture( pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") return False - def test_discover(self, connector_config, docker_runner: ConnectorRunner): + async def test_discover(self, connector_config, docker_runner: ConnectorRunner): """Verify that discover produce correct schema.""" - output = docker_runner.call_discover(config=connector_config) + output = await docker_runner.call_discover(config=connector_config) catalog_messages = filter_output(output, Type.CATALOG) assert len(catalog_messages) == 1, "Catalog message should be emitted exactly once" @@ -752,7 +777,7 @@ def primary_keys_for_records(streams, records): yield pk_values, stream_record -@pytest.mark.default_timeout(5 * 60) +@pytest.mark.default_timeout(10 * 60) class TestBasicRead(BaseTest): @staticmethod def _validate_records_structure(records: List[AirbyteRecordMessage], configured_catalog: ConfiguredAirbyteCatalog): @@ -819,7 +844,7 @@ def _validate_field_appears_at_least_once_in_stream(self, records: List, schema: In case of `oneOf` or `anyOf` schema props, compare only choice which is present in records. """ expected_paths = get_expected_schema_structure(schema, annotate_one_of=True) - expected_paths = set(flatten(tuple(expected_paths))) + expected_paths = set(flatten_tuples(tuple(expected_paths))) for record in records: record_paths = set(get_object_structure(record)) @@ -867,14 +892,8 @@ def _validate_expected_records( for stream_name, expected in expected_records_by_stream.items(): actual = actual_by_stream.get(stream_name, []) detailed_logger.info(f"Actual records for stream {stream_name}:") - detailed_logger.log_json_list(actual) - detailed_logger.info(f"Expected records for stream {stream_name}:") - detailed_logger.log_json_list(expected) - + detailed_logger.info(actual) ignored_field_names = [field.name for field in ignored_fields.get(stream_name, [])] - detailed_logger.info(f"Ignored fields for stream {stream_name}:") - detailed_logger.log_json_list(ignored_field_names) - self.compare_records( stream_name=stream_name, actual=actual, @@ -935,7 +954,7 @@ def configured_catalog_fixture( else: return build_configured_catalog_from_custom_catalog(configured_catalog_path, discovered_catalog) - def test_read( + async def test_read( self, connector_config, configured_catalog, @@ -949,7 +968,7 @@ def test_read( docker_runner: ConnectorRunner, detailed_logger, ): - output = docker_runner.call_read(connector_config, configured_catalog) + output = await docker_runner.call_read(connector_config, configured_catalog) records = [message.record for message in filter_output(output, Type.RECORD)] assert records, "At least one record should be read using provided catalog" @@ -979,7 +998,7 @@ def test_read( detailed_logger=detailed_logger, ) - def test_airbyte_trace_message_on_failure(self, connector_config, inputs: BasicReadTestConfig, docker_runner: ConnectorRunner): + async def test_airbyte_trace_message_on_failure(self, connector_config, inputs: BasicReadTestConfig, docker_runner: ConnectorRunner): if not inputs.expect_trace_message_on_failure: pytest.skip("Skipping `test_airbyte_trace_message_on_failure` because `inputs.expect_trace_message_on_failure=False`") return @@ -999,7 +1018,7 @@ def test_airbyte_trace_message_on_failure(self, connector_config, inputs: BasicR ] ) - output = docker_runner.call_read(connector_config, invalid_configured_catalog, raise_container_error=False) + output = await docker_runner.call_read(connector_config, invalid_configured_catalog, raise_container_error=False) trace_messages = filter_output(output, Type.TRACE) error_trace_messages = list(filter(lambda m: m.trace.type == TraceType.ERROR, trace_messages)) @@ -1033,16 +1052,32 @@ def compare_records( ): """Compare records using combination of restrictions""" if exact_order: - for r1, r2 in zip(expected, actual): + if ignored_fields: + for item in actual: + delete_fields(item, ignored_fields) + for item in expected: + delete_fields(item, ignored_fields) + + cleaned_actual = [] + if extra_fields: + for r1, r2 in zip(expected, actual): + if r1 and r2: + cleaned_actual.append(TestBasicRead.remove_extra_fields(r2, r1)) + else: + break + + cleaned_actual = cleaned_actual or actual + complete_diff = "\n".join( + diff_dicts(cleaned_actual if not extra_records else cleaned_actual[: len(expected)], expected, use_markup=False) + ) + for r1, r2 in zip(expected, cleaned_actual): if r1 is None: assert extra_records, f"Stream {stream_name}: There are more records than expected, but extra_records is off" break - if extra_fields: - r2 = TestBasicRead.remove_extra_fields(r2, r1) - if ignored_fields: - delete_fields(r1, ignored_fields) - delete_fields(r2, ignored_fields) - assert r1 == r2, f"Stream {stream_name}: Mismatch of record order or values" + + # to avoid printing the diff twice, we avoid the == operator here (see plugin.pytest_assertrepr_compare) + equals = r1 == r2 + assert equals, f"Stream {stream_name}: Mismatch of record order or values\nDiff actual vs expected:{complete_diff}" else: _make_hashable = functools.partial(make_hashable, exclude_fields=ignored_fields) if ignored_fields else make_hashable expected = set(map(_make_hashable, expected)) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py index 6694c77fbab4..d98e8d8e778c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py @@ -9,12 +9,14 @@ from typing import List, Mapping, Optional import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, Type +from airbyte_protocol.models import ConfiguredAirbyteCatalog, Type from connector_acceptance_test.base import BaseTest from connector_acceptance_test.config import IgnoredFieldsConfiguration from connector_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, full_refresh_only_catalog, make_hashable from connector_acceptance_test.utils.json_schema_helper import CatalogField +# from airbyte_pr import ConfiguredAirbyteCatalog, Type + def primary_keys_by_stream(configured_catalog: ConfiguredAirbyteCatalog) -> Mapping[str, List[CatalogField]]: """Get PK fields for each stream @@ -82,7 +84,7 @@ def assert_two_sequential_reads_produce_same_or_subset_records( detailed_logger.log_json_list(missing_records) pytest.fail(msg) - def test_sequential_reads( + async def test_sequential_reads( self, connector_config: SecretDict, configured_catalog: ConfiguredAirbyteCatalog, @@ -91,13 +93,17 @@ def test_sequential_reads( detailed_logger: Logger, ): configured_catalog = full_refresh_only_catalog(configured_catalog) - output_1 = docker_runner.call_read(connector_config, configured_catalog) + output_1 = await docker_runner.call_read( + connector_config, + configured_catalog, + enable_caching=False, + ) records_1 = [message.record for message in output_1 if message.type == Type.RECORD] # sleep for 1 second to ensure that the emitted_at timestamp is different time.sleep(1) - output_2 = docker_runner.call_read(connector_config, configured_catalog) + output_2 = await docker_runner.call_read(connector_config, configured_catalog, enable_caching=False) records_2 = [message.record for message in output_2 if message.type == Type.RECORD] self.assert_emitted_at_increase_on_subsequent_runs(records_1, records_2) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py index 9379e5db8e8b..4fc301ecb8cd 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py @@ -9,7 +9,7 @@ import pendulum import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog, SyncMode, Type +from airbyte_protocol.models import AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog, SyncMode, Type from connector_acceptance_test import BaseTest from connector_acceptance_test.config import Config, EmptyStreamConfiguration, IncrementalConfig from connector_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, filter_output, incremental_only_catalog @@ -150,18 +150,18 @@ def construct_latest_state_from_messages(messages: List[AirbyteMessage]) -> Dict @pytest.mark.default_timeout(20 * 60) class TestIncremental(BaseTest): - def test_two_sequential_reads( + async def test_two_sequential_reads( self, inputs: IncrementalConfig, connector_config: SecretDict, configured_catalog_for_incremental: ConfiguredAirbyteCatalog, - cursor_paths: dict[str, list[str]], + cursor_paths: dict[str, list[Union[int, str]]], docker_runner: ConnectorRunner, ): threshold_days = getattr(inputs, "threshold_days") or 0 stream_mapping = {stream.stream.name: stream for stream in configured_catalog_for_incremental.streams} - output = docker_runner.call_read(connector_config, configured_catalog_for_incremental) + output = await docker_runner.call_read(connector_config, configured_catalog_for_incremental) records_1 = filter_output(output, type_=Type.RECORD) states_1 = filter_output(output, type_=Type.STATE) @@ -191,7 +191,7 @@ def test_two_sequential_reads( record_value <= state_value ), f"First incremental sync should produce records younger or equal to cursor value from the state. Stream: {stream_name}" - output = docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) + output = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) records_2 = filter_output(output, type_=Type.RECORD) for record_value, state_value, stream_name in records_with_state(records_2, latest_state, stream_mapping, cursor_paths): @@ -199,7 +199,7 @@ def test_two_sequential_reads( record_value, state_value, threshold_days ), f"Second incremental sync should produce records older or equal to cursor value from the state. Stream: {stream_name}" - def test_read_sequential_slices( + async def test_read_sequential_slices( self, inputs: IncrementalConfig, connector_config, configured_catalog_for_incremental, cursor_paths, docker_runner: ConnectorRunner ): """ @@ -215,7 +215,7 @@ def test_read_sequential_slices( threshold_days = getattr(inputs, "threshold_days") or 0 stream_mapping = {stream.stream.name: stream for stream in configured_catalog_for_incremental.streams} - output = docker_runner.call_read(connector_config, configured_catalog_for_incremental) + output = await docker_runner.call_read(connector_config, configured_catalog_for_incremental) records_1 = filter_output(output, type_=Type.RECORD) states_1 = filter_output(output, type_=Type.STATE) @@ -251,7 +251,7 @@ def test_read_sequential_slices( if len(checkpoint_messages) >= min_batches_to_test and idx % sample_rate != 0: continue - output = docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) + output = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) records = filter_output(output, type_=Type.RECORD) for record_value, state_value, stream_name in records_with_state(records, complete_state, stream_mapping, cursor_paths): @@ -259,9 +259,11 @@ def test_read_sequential_slices( record_value, state_value, threshold_days ), f"Second incremental sync should produce records older or equal to cursor value from the state. Stream: {stream_name}" - def test_state_with_abnormally_large_values(self, connector_config, configured_catalog, future_state, docker_runner: ConnectorRunner): + async def test_state_with_abnormally_large_values( + self, connector_config, configured_catalog, future_state, docker_runner: ConnectorRunner + ): configured_catalog = incremental_only_catalog(configured_catalog) - output = docker_runner.call_read_with_state(config=connector_config, catalog=configured_catalog, state=future_state) + output = await docker_runner.call_read_with_state(config=connector_config, catalog=configured_catalog, state=future_state) records = filter_output(output, type_=Type.RECORD) states = filter_output(output, type_=Type.STATE) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py index 8d87e69d8709..89680d1745da 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/asserts.py @@ -10,7 +10,7 @@ import dpath.util import pendulum -from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog +from airbyte_protocol.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog from jsonschema import Draft7Validator, FormatChecker, FormatError, ValidationError, validators # fmt: off diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py index ff9b50a9e781..13bcf9af0089 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py @@ -4,9 +4,10 @@ from abc import ABC, abstractmethod from enum import Enum +from typing import Any, Dict import jsonschema -from airbyte_cdk.models import ConnectorSpecification +from airbyte_protocol.models import ConnectorSpecification from connector_acceptance_test.utils import SecretDict from deepdiff import DeepDiff from hypothesis import HealthCheck, Verbosity, given, settings @@ -180,17 +181,48 @@ def check_if_declared_new_enum_field(self, diff: DeepDiff): self._raise_error("An 'enum' field was declared on an existing property", diff) +def remove_date_time_pattern_format(schema: Dict[str, Any]) -> Dict[str, Any]: + """ + This function traverses a JSON schema and removes the 'format' field for properties + that are of 'date-time' format and have a 'pattern' field. + + The 'pattern' is often more restrictive than the 'date-time' format, and Hypothesis can't natively generate + date-times that match a specific pattern. Therefore, in this case, we've opted to + remove the 'date-time' format since the 'pattern' is more restrictive and more likely + to cause a breaking change if not adhered to. + + On the otherside we also validate the output of hypothesis against the new schema to ensure + that the generated data matches the new schema. In this case we will catch whether or not the + date-time format is still being adhered to. + + Args: + schema (Dict[str, Any]): The JSON schema to be processed. + + Returns: + Dict[str, Any]: The processed JSON schema where 'date-time' format has been removed + for properties that have a 'pattern'. + """ + if isinstance(schema, dict): + for key, value in schema.items(): + if isinstance(value, dict): + if value.get("format") == "date-time" and "pattern" in value: + del value["format"] + remove_date_time_pattern_format(value) + return schema + + def validate_previous_configs( previous_connector_spec: ConnectorSpecification, actual_connector_spec: ConnectorSpecification, number_of_configs_to_generate=100 ): """Use hypothesis and hypothesis-jsonschema to run property based testing: 1. Generate fake previous config with the previous connector specification json schema. 2. Validate a fake previous config against the actual connector specification json schema.""" + prev_con_spec = previous_connector_spec.dict()["connectionSpecification"] - @given(from_schema(previous_connector_spec.dict()["connectionSpecification"])) + @given(from_schema(remove_date_time_pattern_format(prev_con_spec))) @settings( max_examples=number_of_configs_to_generate, - verbosity=Verbosity.quiet, + verbosity=Verbosity.verbose, suppress_health_check=(HealthCheck.too_slow, HealthCheck.filter_too_much), ) def check_fake_previous_config_against_actual_spec(fake_previous_config): @@ -227,8 +259,20 @@ def assert_is_backward_compatible(self): self.check_if_stream_was_removed(self.streams_json_schemas_diff) self.check_if_value_of_type_field_changed(self.streams_json_schemas_diff) self.check_if_type_of_type_field_changed(self.streams_json_schemas_diff, allow_type_widening=False) + self.check_if_field_removed(self.streams_json_schemas_diff) self.check_if_cursor_field_was_changed(self.streams_cursor_fields_diff) + def check_if_field_removed(self, diff: DeepDiff): + """Check if a property was removed from the catalog.""" + removed_properties = [] + for removal in diff.get("dictionary_item_removed", []): + removal_path_parts = removal.path(output_format="list") + if "properties" in removal_path_parts: + removal_path_human_readable = ".".join(removal_path_parts) + removed_properties.append(removal_path_human_readable) + if removed_properties: + self._raise_error(f"The following properties were removed: {', '.join(removed_properties)}", diff) + def check_if_stream_was_removed(self, diff: DeepDiff): """Check if a stream was removed from the catalog.""" removed_streams = [] diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py index 946871ddb806..929309ca0e67 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/common.py @@ -16,7 +16,7 @@ except ImportError: from yaml import Loader -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteMessage, AirbyteStream, ConfiguredAirbyteCatalog, diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py index cf88e83522f2..b54904ceeab4 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py @@ -5,188 +5,226 @@ import json import logging +import os +import uuid from pathlib import Path -from typing import Iterable, List, Mapping, Optional +from typing import List, Mapping, Optional, Union +import dagger import docker -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, OrchestratorType -from airbyte_cdk.models import Type as AirbyteMessageType -from docker.errors import ContainerError, NotFound -from docker.models.containers import Container +import pytest +import yaml +from airbyte_protocol.models import AirbyteMessage, ConfiguredAirbyteCatalog, OrchestratorType +from airbyte_protocol.models import Type as AirbyteMessageType +from anyio import Path as AnyioPath +from connector_acceptance_test.utils import SecretDict from pydantic import ValidationError class ConnectorRunner: + IN_CONTAINER_CONFIG_PATH = "/data/config.json" + IN_CONTAINER_CATALOG_PATH = "/data/catalog.json" + IN_CONTAINER_STATE_PATH = "/data/state.json" + IN_CONTAINER_OUTPUT_PATH = "/output.txt" + def __init__( self, - image_name: str, - volume: Path, + image_tag: str, + dagger_client: dagger.Client, connector_configuration_path: Optional[Path] = None, custom_environment_variables: Optional[Mapping] = {}, ): - self._client = docker.from_env() - try: - self._image = self._client.images.get(image_name) - except docker.errors.ImageNotFound: - print("Pulling docker image", image_name) - self._image = self._client.images.pull(image_name) - print("Pulling completed") - self._runs = 0 - self._volume_base = volume + self._check_connector_under_test() + self.image_tag = image_tag + self.dagger_client = dagger_client self._connector_configuration_path = connector_configuration_path self._custom_environment_variables = custom_environment_variables + connector_image_tarball_path = self._get_connector_image_tarball_path() + self._connector_under_test_container = self._get_connector_container(connector_image_tarball_path) - @property - def output_folder(self) -> Path: - return self._volume_base / f"run_{self._runs}" / "output" + async def load_container(self): + """This is to pre-load the container following instantiation of the class. + This is useful to make sure that when using the connector runner fixture the costly _import is already done. + """ + await self._connector_under_test_container.with_exec(["spec"]) - @property - def input_folder(self) -> Path: - return self._volume_base / f"run_{self._runs}" / "input" + async def call_spec(self, raise_container_error=False) -> List[AirbyteMessage]: + return await self._run(["spec"], raise_container_error) - def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping], catalog: Optional[ConfiguredAirbyteCatalog]): - self.input_folder.mkdir(parents=True) - self.output_folder.mkdir(parents=True) + async def call_check(self, config: SecretDict, raise_container_error: bool = False) -> List[AirbyteMessage]: + return await self._run(["check", "--config", self.IN_CONTAINER_CONFIG_PATH], raise_container_error, config=config) - # using "is not None" to allow falsey config objects like {} to still write - if config is not None: - with open(str(self.input_folder / "tap_config.json"), "w") as outfile: - json.dump(dict(config), outfile) + async def call_discover(self, config: SecretDict, raise_container_error: bool = False) -> List[AirbyteMessage]: + return await self._run(["discover", "--config", self.IN_CONTAINER_CONFIG_PATH], raise_container_error, config=config) - if state: - with open(str(self.input_folder / "state.json"), "w") as outfile: - if isinstance(state, List): - json.dump(state, outfile) - else: - json.dump(dict(state), outfile) + async def call_read( + self, config: SecretDict, catalog: ConfiguredAirbyteCatalog, raise_container_error: bool = False, enable_caching: bool = True + ) -> List[AirbyteMessage]: + return await self._run( + ["read", "--config", self.IN_CONTAINER_CONFIG_PATH, "--catalog", self.IN_CONTAINER_CATALOG_PATH], + raise_container_error, + config=config, + catalog=catalog, + enable_caching=enable_caching, + ) - if catalog: - with open(str(self.input_folder / "catalog.json"), "w") as outfile: - outfile.write(catalog.json()) - - volumes = { - str(self.input_folder): { - "bind": "/data", - # "mode": "ro", - }, - str(self.output_folder): { - "bind": "/local", - "mode": "rw", - }, - } - return volumes - - def call_spec(self, **kwargs) -> List[AirbyteMessage]: - cmd = "spec" - output = list(self.run(cmd=cmd, **kwargs)) - return output + async def call_read_with_state( + self, + config: SecretDict, + catalog: ConfiguredAirbyteCatalog, + state: dict, + raise_container_error: bool = False, + enable_caching: bool = True, + ) -> List[AirbyteMessage]: + return await self._run( + [ + "read", + "--config", + self.IN_CONTAINER_CONFIG_PATH, + "--catalog", + self.IN_CONTAINER_CATALOG_PATH, + "--state", + self.IN_CONTAINER_STATE_PATH, + ], + raise_container_error, + config=config, + catalog=catalog, + state=state, + enable_caching=enable_caching, + ) - def call_check(self, config, **kwargs) -> List[AirbyteMessage]: - cmd = "check --config /data/tap_config.json" - output = list(self.run(cmd=cmd, config=config, **kwargs)) - return output + async def get_container_env_variable_value(self, name: str) -> str: + return await self._connector_under_test_container.env_variable(name) - def call_discover(self, config, **kwargs) -> List[AirbyteMessage]: - cmd = "discover --config /data/tap_config.json" - output = list(self.run(cmd=cmd, config=config, **kwargs)) - return output + async def get_container_label(self, label: str): + return await self._connector_under_test_container.label(label) - def call_read(self, config, catalog, **kwargs) -> List[AirbyteMessage]: - cmd = "read --config /data/tap_config.json --catalog /data/catalog.json" - output = list(self.run(cmd=cmd, config=config, catalog=catalog, **kwargs)) - return output + async def get_container_entrypoint(self): + entrypoint = await self._connector_under_test_container.entrypoint() + return " ".join(entrypoint) - def call_read_with_state(self, config, catalog, state, **kwargs) -> List[AirbyteMessage]: - cmd = "read --config /data/tap_config.json --catalog /data/catalog.json --state /data/state.json" - output = list(self.run(cmd=cmd, config=config, catalog=catalog, state=state, **kwargs)) - return output + def _get_connector_image_tarball_path(self) -> Optional[Path]: + if "CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH" not in os.environ and not self.image_tag.endswith(":dev"): + return None + if "CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH" in os.environ: + connector_under_test_image_tar_path = Path(os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"]) + elif self.image_tag.endswith(":dev"): + connector_under_test_image_tar_path = self._export_local_connector_image_to_tarball(self.image_tag) + assert connector_under_test_image_tar_path.exists(), "Connector image tarball does not exist" + return connector_under_test_image_tar_path - def run(self, cmd, config=None, state=None, catalog=None, raise_container_error: bool = True, **kwargs) -> Iterable[AirbyteMessage]: + def _export_local_connector_image_to_tarball(self, local_image_name: str) -> Optional[Path]: + tarball_path = Path("/tmp/connector_under_test_image.tar") - self._runs += 1 - volumes = self._prepare_volumes(config, state, catalog) - logging.debug(f"Docker run {self._image}: \n{cmd}\n" f"input: {self.input_folder}\noutput: {self.output_folder}") + docker_client = docker.from_env() + try: + image = docker_client.images.get(local_image_name) + with open(tarball_path, "wb") as f: + for chunk in image.save(named=True): + f.write(chunk) + + except docker.errors.ImageNotFound: + pytest.fail(f"Image {local_image_name} not found, please make sure to build or pull it before running the tests") + return tarball_path - container = self._client.containers.run( - image=self._image, - command=cmd, - volumes=volumes, - network_mode="host", - detach=True, - environment=self._custom_environment_variables, - **kwargs, + def _get_connector_container_from_tarball(self, tarball_path: Path) -> dagger.Container: + container_under_test_tar_file = ( + self.dagger_client.host().directory(str(tarball_path.parent), include=tarball_path.name).file(tarball_path.name) ) - with open(self.output_folder / "raw", "wb+") as f: - for line in self.read(container, command=cmd, with_ext=raise_container_error): - f.write(line.encode()) - try: - airbyte_message = AirbyteMessage.parse_raw(line) - if ( - airbyte_message.type is AirbyteMessageType.CONTROL - and airbyte_message.control.type is OrchestratorType.CONNECTOR_CONFIG - ): - self._persist_new_configuration( - airbyte_message.control.connectorConfig.config, int(airbyte_message.control.emitted_at) - ) - yield airbyte_message - except ValidationError as exc: - logging.warning("Unable to parse connector's output %s, error: %s", line, exc) - - @classmethod - def read(cls, container: Container, command: str = None, with_ext: bool = True) -> Iterable[str]: - """Reads connector's logs per line""" - buffer = b"" - exception = "" - line = "" - for chunk in container.logs(stdout=True, stderr=True, stream=True, follow=True): - - buffer += chunk - while True: - # every chunk can include several lines - found = buffer.find(b"\n") - if found <= -1: - break - - line = buffer[: found + 1].decode("utf-8") - if len(exception) > 0 or line.startswith("Traceback (most recent call last)"): - exception += line - else: - yield line - buffer = buffer[found + 1 :] - - if buffer: - # send the latest chunk if exists - line = buffer.decode("utf-8") - if exception: - exception += line - else: - yield line + return self.dagger_client.container().import_(container_under_test_tar_file) + + def _get_connector_container(self, connector_image_tarball_path: Optional[Path]) -> dagger.Container: + if connector_image_tarball_path is not None: + container = self._get_connector_container_from_tarball(connector_image_tarball_path) + else: + # Try to pull the image from DockerHub + container = self.dagger_client.container().from_(self.image_tag) + # Client might pass a cachebuster env var to force recreation of the container + # We pass this env var to the container to ensure the cache is busted + if cachebuster_value := os.environ.get("CACHEBUSTER"): + container = container.with_env_variable("CACHEBUSTER", cachebuster_value) + for key, value in self._custom_environment_variables.items(): + container = container.with_env_variable(key, str(value)) + return container + + async def _run( + self, + airbyte_command: List[str], + raise_container_error: bool, + config: SecretDict = None, + catalog: dict = None, + state: Union[dict, list] = None, + enable_caching=True, + ) -> List[AirbyteMessage]: + """_summary_ + + Args: + airbyte_command (List[str]): The command to run in the connector container. + raise_container_error (bool): Whether to raise an error if the container fails to run the command. + config (SecretDict, optional): The config to mount to the container. Defaults to None. + catalog (dict, optional): The catalog to mount to the container. Defaults to None. + state (Union[dict, list], optional): The state to mount to the container. Defaults to None. + enable_caching (bool, optional): Whether to enable command output caching. Defaults to True. + + Raises: + e: _description_ + + Returns: + List[AirbyteMessage]: _description_ + """ + container = self._connector_under_test_container + if not enable_caching: + container = container.with_env_variable("CAT_CACHEBUSTER", str(uuid.uuid4())) + if config: + container = container.with_new_file(self.IN_CONTAINER_CONFIG_PATH, json.dumps(dict(config))) + if state: + container = container.with_new_file(self.IN_CONTAINER_STATE_PATH, json.dumps(state)) + if catalog: + container = container.with_new_file(self.IN_CONTAINER_CATALOG_PATH, catalog.json()) + for key, value in self._custom_environment_variables.items(): + container = container.with_env_variable(key, str(value)) try: - exit_status = container.wait() - container.remove() - except NotFound as err: - logging.error(f"Waiting error: {err}, logs: {exception or line}") - raise - if exit_status["StatusCode"]: - error = exit_status.get("Error") or exception or line - logging.error(f"Docker container failed, " f'code {exit_status["StatusCode"]}, error:\n{error}') - if with_ext: - raise ContainerError( - container=container, - exit_status=exit_status["StatusCode"], - command=command, - image=container.image, - stderr=error, - ) + output = await self._read_output_from_stdout(airbyte_command, container) + except dagger.QueryError as e: + output_too_big = bool([error for error in e.errors if error.message.startswith("file size")]) + if output_too_big: + output = await self._read_output_from_file(airbyte_command, container) + elif raise_container_error: + raise e + else: + if isinstance(e, dagger.ExecError): + output = e.stdout + e.stderr + else: + pytest.fail(f"Failed to run command {airbyte_command} in container {self.image_tag} with error: {e}") + return self.parse_airbyte_messages_from_command_output(output) + + async def _read_output_from_stdout(self, airbyte_command: list, container: dagger.Container) -> str: + return await container.with_exec(airbyte_command).stdout() - @property - def env_variables(self): - env_vars = self._image.attrs["Config"]["Env"] - return {env.split("=", 1)[0]: env.split("=", 1)[1] for env in env_vars} + async def _read_output_from_file(self, airbyte_command: list, container: dagger.Container) -> str: + local_output_file_path = f"/tmp/{str(uuid.uuid4())}" + entrypoint = await container.entrypoint() + airbyte_command = entrypoint + airbyte_command + container = container.with_exec( + ["sh", "-c", " ".join(airbyte_command) + f" > {self.IN_CONTAINER_OUTPUT_PATH} 2>&1 | tee -a {self.IN_CONTAINER_OUTPUT_PATH}"], + skip_entrypoint=True, + ) + await container.file(self.IN_CONTAINER_OUTPUT_PATH).export(local_output_file_path) + output = await AnyioPath(local_output_file_path).read_text() + await AnyioPath(local_output_file_path).unlink() + return output - @property - def entry_point(self): - return self._image.attrs["Config"]["Entrypoint"] + def parse_airbyte_messages_from_command_output(self, command_output: str) -> List[AirbyteMessage]: + airbyte_messages = [] + for line in command_output.splitlines(): + try: + airbyte_message = AirbyteMessage.parse_raw(line) + if airbyte_message.type is AirbyteMessageType.CONTROL and airbyte_message.control.type is OrchestratorType.CONNECTOR_CONFIG: + self._persist_new_configuration(airbyte_message.control.connectorConfig.config, int(airbyte_message.control.emitted_at)) + airbyte_messages.append(airbyte_message) + except ValidationError as exc: + logging.warning("Unable to parse connector's output %s, error: %s", line, exc) + return airbyte_messages def _persist_new_configuration(self, new_configuration: dict, configuration_emitted_at: int) -> Optional[Path]: """Store new configuration values to an updated_configurations subdir under the original configuration path. @@ -221,3 +259,16 @@ def _persist_new_configuration(self, new_configuration: dict, configuration_emit json.dump(new_configuration, new_configuration_file) logging.info(f"Stored most recent configuration value to {new_configuration_file_path}") return new_configuration_file_path + + def _check_connector_under_test(self): + """ + As a safety measure, we check that the connector under test matches the connector being tested by comparing the content of the metadata.yaml file to the CONNECTOR_UNDER_TEST_TECHNICAL_NAME environment varialbe. + When running CAT from airbyte-ci we set this CONNECTOR_UNDER_TEST_TECHNICAL_NAME env var name, + This is a safety check to ensure the correct test inputs are mounted to the CAT container. + """ + if connector_under_test_technical_name := os.environ.get("CONNECTOR_UNDER_TEST_TECHNICAL_NAME"): + metadata = yaml.safe_load(Path("/test_input/metadata.yaml").read_text()) + assert metadata["data"]["dockerRepository"] == f"airbyte/{connector_under_test_technical_name}", ( + f"Connector under test env var {connector_under_test_technical_name} does not match the connector " + f"being tested {metadata['data']['dockerRepository']}" + ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py index fe959d79bdb4..52455bccfc3f 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/json_schema_helper.py @@ -43,7 +43,7 @@ def _parse_value(self, value: Any) -> Any: return pendulum.parse(value) return value - def parse(self, record: Mapping[str, Any], path: Optional[List[str]] = None) -> Any: + def parse(self, record: Mapping[str, Any], path: Optional[List[Union[int, str]]] = None) -> Any: """Extract field value from the record and cast it to native type""" path = path or self.path value = reduce(lambda data, key: data[key], path, record) @@ -240,3 +240,26 @@ def _scan_schema(subschema, path=""): _scan_schema(schema) return paths + + +def flatten_tuples(to_flatten): + """Flatten a tuple of tuples into a single tuple.""" + types = set() + + if not isinstance(to_flatten, tuple): + to_flatten = (to_flatten,) + for thing in to_flatten: + if isinstance(thing, tuple): + types.update(flatten_tuples(thing)) + else: + types.add(thing) + return tuple(types) + + +def get_paths_in_connector_config(schema: dict) -> List[str]: + """ + Traverse through the provided schema's values and extract the path_in_connector_config paths + :param properties: jsonschema containing values which may have path_in_connector_config attributes + :returns list of path_in_connector_config paths + """ + return ["/" + "/".join(value["path_in_connector_config"]) for value in schema.values()] diff --git a/airbyte-integrations/bases/connector-acceptance-test/poetry.lock b/airbyte-integrations/bases/connector-acceptance-test/poetry.lock new file mode 100644 index 000000000000..50a12f5ee023 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/poetry.lock @@ -0,0 +1,1492 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.1-py3-none-any.whl", hash = "sha256:95f1197c800d7867ba067f75770b83aeff4c2cec9b3d1def2dbf70261fee89ee"}, + {file = "airbyte_protocol_models-0.4.1.tar.gz", hash = "sha256:92602134eab4c921d1328fa4f24e9a810a679c117ccb352cf6b1521f95f0ed53"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "anyio" +version = "3.7.1" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.7" +files = [ + {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, + {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, +] + +[package.dependencies] +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] +test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (<0.22)"] + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "beartype" +version = "0.15.0" +description = "Unbearably fast runtime type checking in pure Python." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "beartype-0.15.0-py3-none-any.whl", hash = "sha256:52cd2edea72fdd84e4e7f8011a9e3007bf0125c3d6d7219e937b9d8868169177"}, + {file = "beartype-0.15.0.tar.gz", hash = "sha256:2af6a8d8a7267ccf7d271e1a3bd908afbc025d2a09aa51123567d7d7b37438df"}, +] + +[package.extras] +all = ["typing-extensions (>=3.10.0.0)"] +dev = ["autoapi (>=0.9.0)", "coverage (>=5.5)", "mypy (>=0.800)", "numpy", "pandera", "pydata-sphinx-theme (<=0.7.2)", "pytest (>=4.0.0)", "sphinx", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)", "tox (>=3.20.1)", "typing-extensions (>=3.10.0.0)"] +doc-rtd = ["autoapi (>=0.9.0)", "pydata-sphinx-theme (<=0.7.2)", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)"] +test-tox = ["mypy (>=0.800)", "numpy", "pandera", "pytest (>=4.0.0)", "sphinx", "typing-extensions (>=3.10.0.0)"] +test-tox-coverage = ["coverage (>=5.5)"] + +[[package]] +name = "cattrs" +version = "23.1.2" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, + {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, +] + +[package.dependencies] +attrs = ">=20" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.2.0,<5.0.0)"] +cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] +msgpack = ["msgpack (>=1.0.2,<2.0.0)"] +orjson = ["orjson (>=3.5.2,<4.0.0)"] +pyyaml = ["PyYAML (>=6.0,<7.0)"] +tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] +ujson = ["ujson (>=5.4.0,<6.0.0)"] + +[[package]] +name = "certifi" +version = "2023.7.22" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.2.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, + {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, + {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, + {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, + {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, + {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, + {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.2.7" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "dagger-io" +version = "0.6.4" +description = "A client package for running Dagger pipelines in Python." +optional = false +python-versions = ">=3.10" +files = [ + {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, + {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, +] + +[package.dependencies] +anyio = ">=3.6.2" +beartype = ">=0.11.0" +cattrs = ">=22.2.0" +gql = ">=3.4.0" +graphql-core = ">=3.2.3" +httpx = ">=0.23.1" +platformdirs = ">=2.6.2" +typing-extensions = ">=4.4.0" + +[package.extras] +cli = ["typer[all] (>=0.6.1)"] +server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] + +[[package]] +name = "deepdiff" +version = "5.8.1" +description = "Deep Difference and Search of any Python object/data." +optional = false +python-versions = ">=3.6" +files = [ + {file = "deepdiff-5.8.1-py3-none-any.whl", hash = "sha256:e9aea49733f34fab9a0897038d8f26f9d94a97db1790f1b814cced89e9e0d2b7"}, + {file = "deepdiff-5.8.1.tar.gz", hash = "sha256:8d4eb2c4e6cbc80b811266419cb71dd95a157094a3947ccf937a94d44943c7b8"}, +] + +[package.dependencies] +ordered-set = ">=4.1.0,<4.2.0" + +[package.extras] +cli = ["clevercsv (==0.7.1)", "click (==8.0.3)", "pyyaml (==5.4.1)", "toml (==0.10.2)"] + +[[package]] +name = "docker" +version = "6.1.3" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.7" +files = [ + {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"}, + {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"}, +] + +[package.dependencies] +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" +websocket-client = ">=0.32.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, + {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fancycompleter" +version = "0.9.1" +description = "colorful TAB completion for Python prompt" +optional = false +python-versions = "*" +files = [ + {file = "fancycompleter-0.9.1-py3-none-any.whl", hash = "sha256:dd076bca7d9d524cc7f25ec8f35ef95388ffef9ef46def4d3d25e9b044ad7080"}, + {file = "fancycompleter-0.9.1.tar.gz", hash = "sha256:09e0feb8ae242abdfd7ef2ba55069a46f011814a80fe5476be48f51b00247272"}, +] + +[package.dependencies] +pyreadline = {version = "*", markers = "platform_system == \"Windows\""} +pyrepl = ">=0.8.2" + +[[package]] +name = "gql" +version = "3.4.1" +description = "GraphQL client for Python" +optional = false +python-versions = "*" +files = [ + {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, + {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, +] + +[package.dependencies] +backoff = ">=1.11.1,<3.0" +graphql-core = ">=3.2,<3.3" +yarl = ">=1.6,<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] +all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +botocore = ["botocore (>=1.21,<2)"] +dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] +test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] +websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "0.17.3" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"}, + {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"}, +] + +[package.dependencies] +anyio = ">=3.0,<5.0" +certifi = "*" +h11 = ">=0.13,<0.15" +sniffio = "==1.*" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "httpx" +version = "0.24.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.7" +files = [ + {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"}, + {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"}, +] + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.18.0" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "hypothesis" +version = "6.82.3" +description = "A library for property-based testing" +optional = false +python-versions = ">=3.8" +files = [ + {file = "hypothesis-6.82.3-py3-none-any.whl", hash = "sha256:7ff0f6a12d3cd9372e30f84d300e2468c3923e813198a93b9e479dda91858460"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +exceptiongroup = {version = ">=1.0.0", markers = "python_version < \"3.11\""} +sortedcontainers = ">=2.1.0,<3.0.0" + +[package.extras] +all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.3)"] +cli = ["black (>=19.10b0)", "click (>=7.0)", "rich (>=9.0.0)"] +codemods = ["libcst (>=0.3.16)"] +dateutil = ["python-dateutil (>=1.4)"] +django = ["django (>=3.2)"] +dpcontracts = ["dpcontracts (>=0.4)"] +ghostwriter = ["black (>=19.10b0)"] +lark = ["lark (>=0.10.1)"] +numpy = ["numpy (>=1.17.3)"] +pandas = ["pandas (>=1.1)"] +pytest = ["pytest (>=4.6)"] +pytz = ["pytz (>=2014.1)"] +redis = ["redis (>=3.0.0)"] +zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.3)"] + +[[package]] +name = "hypothesis-jsonschema" +version = "0.22.1" +description = "Generate test data from JSON schemata with Hypothesis" +optional = false +python-versions = ">=3.7" +files = [ + {file = "hypothesis-jsonschema-0.22.1.tar.gz", hash = "sha256:5dd7449009f323e408a9aa64afb4d18bd1f60ea2eabf5bf152a510da728b34f2"}, + {file = "hypothesis_jsonschema-0.22.1-py3-none-any.whl", hash = "sha256:082968cb86a6aac2369627b08753cbf714c08054b1ebfce3588e3756e652cde6"}, +] + +[package.dependencies] +hypothesis = ">=6.31.6" +jsonschema = ">=4.0.0" + +[[package]] +name = "icdiff" +version = "1.9.1" +description = "improved colored diff" +optional = false +python-versions = "*" +files = [ + {file = "icdiff-1.9.1.tar.gz", hash = "sha256:66972dd03318da55280991db375d3ef6b66d948c67af96c1ebdb21587e86655e"}, +] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "inflection" +version = "0.5.1" +description = "A port of Ruby on Rails inflector to Python" +optional = false +python-versions = ">=3.5" +files = [ + {file = "inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2"}, + {file = "inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jsonref" +version = "0.2" +description = "An implementation of JSON Reference for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, + {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, +] + +[[package]] +name = "jsonschema" +version = "4.19.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema-4.19.0-py3-none-any.whl", hash = "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb"}, + {file = "jsonschema-4.19.0.tar.gz", hash = "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +jsonschema-specifications = ">=2023.03.6" +referencing = ">=0.28.4" +rpds-py = ">=0.7.1" + +[package.extras] +format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] +format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] + +[[package]] +name = "jsonschema-specifications" +version = "2023.7.1" +description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jsonschema_specifications-2023.7.1-py3-none-any.whl", hash = "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1"}, + {file = "jsonschema_specifications-2023.7.1.tar.gz", hash = "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"}, +] + +[package.dependencies] +referencing = ">=0.28.0" + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] + +[[package]] +name = "ordered-set" +version = "4.1.0" +description = "An OrderedSet is a custom MutableSet that remembers its order, so that every" +optional = false +python-versions = ">=3.7" +files = [ + {file = "ordered-set-4.1.0.tar.gz", hash = "sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8"}, + {file = "ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562"}, +] + +[package.extras] +dev = ["black", "mypy", "pytest"] + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + +[[package]] +name = "pdbpp" +version = "0.10.3" +description = "pdb++, a drop-in replacement for pdb" +optional = false +python-versions = "*" +files = [ + {file = "pdbpp-0.10.3-py2.py3-none-any.whl", hash = "sha256:79580568e33eb3d6f6b462b1187f53e10cd8e4538f7d31495c9181e2cf9665d1"}, + {file = "pdbpp-0.10.3.tar.gz", hash = "sha256:d9e43f4fda388eeb365f2887f4e7b66ac09dce9b6236b76f63616530e2f669f5"}, +] + +[package.dependencies] +fancycompleter = ">=0.8" +pygments = "*" +wmctrl = "*" + +[package.extras] +funcsigs = ["funcsigs"] +testing = ["funcsigs", "pytest"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "3.10.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.2.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, + {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pprintpp" +version = "0.4.0" +description = "A drop-in replacement for pprint that's actually pretty" +optional = false +python-versions = "*" +files = [ + {file = "pprintpp-0.4.0-py2.py3-none-any.whl", hash = "sha256:b6b4dcdd0c0c0d75e4d7b2f21a9e933e5b2ce62b26e1a54537f9651ae5a5c01d"}, + {file = "pprintpp-0.4.0.tar.gz", hash = "sha256:ea826108e2c7f49dc6d66c752973c3fc9749142a798d6b254e1e301cfdbc6403"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pydantic" +version = "1.9.2" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.6.1" +files = [ + {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, + {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, + {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, + {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, + {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, + {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, + {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, + {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, + {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, + {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, + {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, + {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, + {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, + {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, + {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, + {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, + {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, + {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, + {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, + {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, + {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, + {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, + {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4.3" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.16.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, +] + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "pyreadline" +version = "2.1" +description = "A python implmementation of GNU readline." +optional = false +python-versions = "*" +files = [ + {file = "pyreadline-2.1.zip", hash = "sha256:4530592fc2e85b25b1a9f79664433da09237c1a270e4d78ea5aa3a2c7229e2d1"}, +] + +[[package]] +name = "pyrepl" +version = "0.9.0" +description = "A library for building flexible command line interfaces" +optional = false +python-versions = "*" +files = [ + {file = "pyrepl-0.9.0.tar.gz", hash = "sha256:292570f34b5502e871bbb966d639474f2b57fbfcd3373c2d6a2f3d56e681a775"}, +] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "3.0.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"}, + {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + +[[package]] +name = "pytest-mock" +version = "3.6.1" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-mock-3.6.1.tar.gz", hash = "sha256:40217a058c52a63f1042f0784f62009e976ba824c418cced42e88d5f40ab0e62"}, + {file = "pytest_mock-3.6.1-py3-none-any.whl", hash = "sha256:30c2f2cc9759e76eee674b81ea28c9f0b94f8f0445a1b87762cadf774f0df7e3"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "pytest-sugar" +version = "0.9.7" +description = "pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly)." +optional = false +python-versions = "*" +files = [ + {file = "pytest-sugar-0.9.7.tar.gz", hash = "sha256:f1e74c1abfa55f7241cf7088032b6e378566f16b938f3f08905e2cf4494edd46"}, + {file = "pytest_sugar-0.9.7-py2.py3-none-any.whl", hash = "sha256:8cb5a4e5f8bbcd834622b0235db9e50432f4cbd71fef55b467fe44e43701e062"}, +] + +[package.dependencies] +packaging = ">=21.3" +pytest = ">=6.2.0" +termcolor = ">=2.1.0" + +[package.extras] +dev = ["black", "flake8", "pre-commit"] + +[[package]] +name = "pytest-timeout" +version = "1.4.2" +description = "py.test plugin to abort hanging tests" +optional = false +python-versions = "*" +files = [ + {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, + {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, +] + +[package.dependencies] +pytest = ">=3.6.0" + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "referencing" +version = "0.30.2" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.30.2-py3-none-any.whl", hash = "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf"}, + {file = "referencing-0.30.2.tar.gz", hash = "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-mock" +version = "1.9.3" +description = "Mock out responses from the requests package" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, + {file = "requests_mock-1.9.3-py2.py3-none-any.whl", hash = "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] + +[[package]] +name = "rpds-py" +version = "0.9.2" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.9.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7"}, + {file = "rpds_py-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f"}, + {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f"}, + {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f"}, + {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe"}, + {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6"}, + {file = "rpds_py-0.9.2-cp310-none-win32.whl", hash = "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764"}, + {file = "rpds_py-0.9.2-cp310-none-win_amd64.whl", hash = "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e"}, + {file = "rpds_py-0.9.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3"}, + {file = "rpds_py-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f"}, + {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d"}, + {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c"}, + {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae"}, + {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de"}, + {file = "rpds_py-0.9.2-cp311-none-win32.whl", hash = "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab"}, + {file = "rpds_py-0.9.2-cp311-none-win_amd64.whl", hash = "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298"}, + {file = "rpds_py-0.9.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386"}, + {file = "rpds_py-0.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238"}, + {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1"}, + {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b"}, + {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90"}, + {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d"}, + {file = "rpds_py-0.9.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d"}, + {file = "rpds_py-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55"}, + {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55"}, + {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d"}, + {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32"}, + {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e"}, + {file = "rpds_py-0.9.2-cp38-none-win32.whl", hash = "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920"}, + {file = "rpds_py-0.9.2-cp38-none-win_amd64.whl", hash = "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044"}, + {file = "rpds_py-0.9.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260"}, + {file = "rpds_py-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f"}, + {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324"}, + {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3"}, + {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535"}, + {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26"}, + {file = "rpds_py-0.9.2-cp39-none-win32.whl", hash = "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0"}, + {file = "rpds_py-0.9.2-cp39-none-win_amd64.whl", hash = "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67"}, + {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8"}, + {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03"}, + {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe"}, + {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + +[[package]] +name = "termcolor" +version = "2.3.0" +description = "ANSI color formatting for output in terminal" +optional = false +python-versions = ">=3.7" +files = [ + {file = "termcolor-2.3.0-py3-none-any.whl", hash = "sha256:3afb05607b89aed0ffe25202399ee0867ad4d3cb4180d98aaf8eefa6a5f7d475"}, + {file = "termcolor-2.3.0.tar.gz", hash = "sha256:b5b08f68937f138fe92f6c089b99f1e2da0ae56c52b78bf7075fd95420fd9a5a"}, +] + +[package.extras] +tests = ["pytest", "pytest-cov"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.7.1" +description = "Backported and Experimental Type Hints for Python 3.7+" +optional = false +python-versions = ">=3.7" +files = [ + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, +] + +[[package]] +name = "urllib3" +version = "1.26.16" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, + {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "websocket-client" +version = "1.6.1" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.7" +files = [ + {file = "websocket-client-1.6.1.tar.gz", hash = "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd"}, + {file = "websocket_client-1.6.1-py3-none-any.whl", hash = "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"}, +] + +[package.extras] +docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "wmctrl" +version = "0.4" +description = "A tool to programmatically control windows inside X" +optional = false +python-versions = "*" +files = [ + {file = "wmctrl-0.4.tar.gz", hash = "sha256:66cbff72b0ca06a22ec3883ac3a4d7c41078bdae4fb7310f52951769b10e14e0"}, +] + +[[package]] +name = "yarl" +version = "1.9.2" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, + {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, + {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, + {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, + {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, + {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, + {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, + {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, + {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, + {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, + {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, + {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, + {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, + {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, + {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, + {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, + {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, + {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, + {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, + {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, + {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, + {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, + {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, + {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, + {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, + {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, + {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "d0bfe69918b1133a0ea31368570203e0413bccf22aabfca6b5339cca59e2df2d" diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml new file mode 100644 index 000000000000..141db6810674 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "connector-acceptance-test" +version = "1.0.0" +description = "Contains acceptance tests for connectors." +authors = ["Airbyte "] +license = "MIT" +homepage = "https://github.com/airbytehq/airbyte" + +[tool.poetry.dependencies] +python = "^3.10" +airbyte-protocol-models = "<1.0.0" +dagger-io = "==0.6.4" +PyYAML = "~=6.0" +icdiff = "~=1.9" +inflection = "~=0.5" +pdbpp = "~=0.10" +pydantic = "*" +pytest = "~=6.2" +pytest-sugar = "~=0.9" +pytest-timeout = "~=1.4" +pprintpp = "~=0.4" +dpath = "~=2.0.1" +jsonschema = "*" +jsonref = "==0.2" +deepdiff = "~=5.8.0" +requests-mock = "~=1.9.3" +pytest-mock = "~=3.6.1" +pendulum = "*" +pytest-cov = "~=3.0.0" +hypothesis = "*" +hypothesis-jsonschema = "*" +anyio = "^3.4.1" +docker = ">=6,<7" +# Pinning requests and urllib3 to avoid an issue with dockerpy and requests 2. +# Related issue: https://github.com/docker/docker-py/issues/3113 +urllib3 = "<2.0" +requests = "<2.29.0" + +[tool.poetry.dev-dependencies] + diff --git a/airbyte-integrations/bases/connector-acceptance-test/setup.py b/airbyte-integrations/bases/connector-acceptance-test/setup.py deleted file mode 100644 index e596f8a57600..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import setuptools - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", - "docker~=5.0.3", - "PyYAML~=5.4", - "icdiff~=1.9", - "inflection~=0.5", - "pdbpp~=0.10", - "pydantic~=1.6", - "pytest~=6.2", - "pytest-sugar~=0.9", - "pytest-timeout~=1.4", - "pprintpp~=0.4", - "dpath~=2.0.1", - "jsonschema~=3.2.0", - "jsonref==0.2", - "deepdiff~=5.8.0", - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest-cov~=3.0.0", - "hypothesis~=6.54.1", - "hypothesis-jsonschema~=0.20.1", # TODO alafanechere upgrade to latest when jsonschema lib is upgraded to >= 4.0.0 in airbyte-cdk and connector acceptance tests - # Pinning requests and urllib3 to avoid an issue with dockerpy and requests 2. - # Related issue: https://github.com/docker/docker-py/issues/3113 - "urllib3<2.0", - "requests<2.29.0", -] - -setuptools.setup( - name="connector-acceptance-test", - description="Contains acceptance tests for connectors.", - author="Airbyte", - author_email="contact@airbyte.io", - url="https://github.com/airbytehq/airbyte", - packages=setuptools.find_packages(), - install_requires=MAIN_REQUIREMENTS, -) diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md index 399adf9ab52e..9883481f76f9 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md @@ -11,7 +11,7 @@ pip install -r requirements.txt brew install gh ``` -Then create a module to contain the information for your migration/issues etc.: +Then create a module to contain the information for your migration/issues etc.: ``` mkdir migrations/ touch migrations//__init__.py @@ -19,7 +19,7 @@ touch migrations//config.py ``` Copy a config.py file from another migration and fill in the `MODULE_NAME` variable. The other variables -can be filled in when you use certain scripts. +can be filled in when you use certain scripts. ```python # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. @@ -36,7 +36,7 @@ MODULE_NAME: str = "" ## Scripts -The scripts perform operations on a set of connectors. +The scripts perform operations on a set of connectors. ### `run_tests.py`: Run CAT tests and save results by exit code @@ -49,24 +49,24 @@ TODO: Replace this process with Dagger #### Before running -1. The tests will run on the `latest` version of CAT by default. To run the `dev` version of CAT, and select a specific +1. The tests will run on the `latest` version of CAT by default. To run the `dev` version of CAT, and select a specific test, commit the hacky changes in [this commit](https://github.com/airbytehq/airbyte/pull/24377/commits/7d9fb1414911a512cd5d5ffafe2a384e8004fb1e). 2. Give Docker a _lot_ of space to build all the connector images! -3. Make sure you have the secrets downloaded from GSM for all of the connectors you want to run tests on. Please keep +3. Make sure you have the secrets downloaded from GSM for all of the connectors you want to run tests on. Please keep in mind that secrets need to be re-uploaded for connectors with single-use Oauth tokens. -#### How to run +#### How to run -Typical usage: +Typical usage: ``` python run_tests.py ``` -Full options: +Full options: ``` -usage: run_tests.py [-h] --connectors [CONNECTORS ...] [--allow_alpha | --no-allow_alpha] [--allow_beta | --no-allow_beta] [--max_concurrency MAX_CONCURRENCY] +usage: run_tests.py [-h] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] [--max_concurrency MAX_CONCURRENCY] Run connector acceptance tests for a list of connectors. @@ -76,7 +76,7 @@ options: A list of connectors (separated by spaces) to run a script on. (default: all connectors) --allow_alpha, --no-allow_alpha Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) - --allow_beta, --no-allow_beta + --allow_community, --no-allow_community Whether to apply the change to bets connectors, if they are included in the list of connectors. (default: False) --max_concurrency MAX_CONCURRENCY The maximum number of acceptance tests that should happen at once. @@ -100,9 +100,9 @@ Issues get created with the title according to `ISSUE_TITLE`. Labels are added a # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - + from typing import Optional, List - + # SET THESE BEFORE USING THE SCRIPT MODULE_NAME: str = "" GITHUB_PROJECT_NAME: Optional[str] = "" @@ -116,12 +116,12 @@ Issues get created with the title according to `ISSUE_TITLE`. Labels are added a ```bash touch migrations//issue.md.j2 ``` - + If you need to fill more variables than are currently defined in the call to `template.render()` in `create_issues.py`, edit the script to allow filling of that variable and define how it should be filled. Please keep in mind the other migrations when you do this. -3. Update the following line in the script so that it points to the config file from your migration: +3. Update the following line in the script so that it points to the config file from your migration: ```python ## Update this line before running the script @@ -140,9 +140,9 @@ Typical usage (real execution): python create_issues.py --connectors --no-dry ``` -Full options: +Full options: ``` -usage: create_issues.py [-h] [-d | --dry | --no-dry] --connectors [CONNECTORS ...] [--allow_beta | --no-allow_beta] [--allow_alpha | --no-allow_alpha] +usage: create_issues.py [-h] [-d | --dry | --no-dry] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] [--allow_alpha | --no-allow_alpha] Create issues for a list of connectors from a template. @@ -151,14 +151,14 @@ options: -d, --dry, --no-dry Whether the action performed is a dry run. In the case of a dry run, no git actions will be pushed to the remote. (default: True) --connectors [CONNECTORS ...] A list of connectors (separated by spaces) to run a script on. (default: all connectors) - --allow_beta, --no-allow_beta + --allow_community, --no-allow_community Whether to apply the change to bets connectors, if they are included in the list of connectors. (default: False) --allow_alpha, --no-allow_alpha Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) ``` -### `config_migration.py`: Perform migrations on `acceptance-test-config.yml` files +### `config_migration.py`: Perform migrations on `acceptance-test-config.yml` files #### What it does: For each connector: @@ -189,7 +189,7 @@ python config_migration.py --connectors Full options: ``` -usage: config_migration.py [-h] --connectors [CONNECTORS ...] [--allow_alpha | --no-allow_alpha] [--allow_beta | --no-allow_beta] [--migrate_from_legacy | --no-migrate_from_legacy] +usage: config_migration.py [-h] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] [--migrate_from_legacy | --no-migrate_from_legacy] Migrate acceptance-test-config.yml files for a list of connectors. @@ -197,10 +197,8 @@ options: -h, --help show this help message and exit --connectors [CONNECTORS ...] A list of connectors (separated by spaces) to run a script on. (default: all connectors) - --allow_alpha, --no-allow_alpha - Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) - --allow_beta, --no-allow_beta - Whether to apply the change to bets connectors, if they are included in the list of connectors. (default: False) + --allow_community, --no-allow_community + Whether to apply the change to community connectors, if they are included in the list of connectors. (default: False) --migrate_from_legacy, --no-migrate_from_legacy Whether to migrate config files from the legacy format before applying the migration. (default: False) ``` @@ -208,7 +206,7 @@ options: ### `create_prs.py`: Create a PR per connector that performs a config migration and pushes it -## Create migration PRs for GA connectors (`create_prs.py`) +## Create migration PRs for Certified connectors (`create_prs.py`) #### What it does: For each connector: @@ -230,9 +228,9 @@ PRs get created with the title according to `ISSUE_TITLE`. Labels are added acco # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - + from typing import Optional, List - + # SET THESE BEFORE USING THE SCRIPT MODULE_NAME: str = "" GITHUB_PROJECT_NAME: Optional[str] = "" @@ -275,7 +273,7 @@ python create_prs.py --connectors --no-dry Full options: ``` -usage: create_prs.py [-h] [-d | --dry | --no-dry] --connectors [CONNECTORS ...] [--allow_alpha | --no-allow_alpha] [--allow_beta | --no-allow_beta] +usage: create_prs.py [-h] [-d | --dry | --no-dry] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] Create PRs for a list of connectors from a template. @@ -286,8 +284,8 @@ options: A list of connectors (separated by spaces) to run a script on. (default: all connectors) --allow_alpha, --no-allow_alpha Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) - --allow_beta, --no-allow_beta - Whether to apply the change to bets connectors, if they are included in the list of connectors. (default: False) + --allow_community, --no-allow_community + Whether to apply the change to community connectors, if they are included in the list of connectors. (default: False) ``` ## Existing migrations diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py index 9b77374a53d5..afd93f83b5da 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py @@ -52,7 +52,7 @@ def get_issue_content(source_definition) -> Optional[Dict[Text, Any]]: # TODO: Make list of variables to render, and how to render them, configurable issue_body = template.render( connector_name=source_definition["name"], - release_stage=source_definition["releaseStage"], + support_level=source_definition["supportLevel"], test_failure_logs=get_test_failure_logs(source_definition), ) file_definition, issue_body_path = tempfile.mkstemp() diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py index 94118ccad780..b9d441871c30 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py @@ -55,7 +55,7 @@ def commit_push_migrated_config(config_path, connector_name, new_branch, dry_run def get_pr_content(definition): pr_title = f"Source {definition['name']}: {config.ISSUE_TITLE}" - pr_body = PR_TEMPLATE.render(connector_name=definition["name"], release_stage=definition["releaseStage"]) + pr_body = PR_TEMPLATE.render(connector_name=definition["name"], support_level=definition["supportLevel"]) file_definition, pr_body_path = tempfile.mkstemp() with os.fdopen(file_definition, "w") as tmp: diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py index 0ed4d03f03b0..519bf5abb66f 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py @@ -22,15 +22,11 @@ def read_source_definitions(): return download_and_parse_registry_json()["sources"] -def find_by_release_stage(source_definitions, release_stage): - if release_stage == "other": - return [ - definition - for definition in source_definitions - if definition.get("releaseStage", "") not in ["alpha", "beta", "generally_available"] - ] +def find_by_support_level(source_definitions, support_level): + if support_level == "other": + return [definition for definition in source_definitions if definition.get("supportLevel", "") not in ["community", "certified"]] else: - return [definition for definition in source_definitions if definition.get("releaseStage") == release_stage] + return [definition for definition in source_definitions if definition.get("supportLevel") == support_level] def find_by_name(connector_names: List[str]): @@ -51,7 +47,6 @@ def is_airbyte_connector(connector_definition): ALL_DEFINITIONS = read_source_definitions() -GA_DEFINITIONS = find_by_release_stage(ALL_DEFINITIONS, "generally_available") -BETA_DEFINITIONS = find_by_release_stage(ALL_DEFINITIONS, "beta") -ALPHA_DEFINTIONS = find_by_release_stage(ALL_DEFINITIONS, "alpha") -OTHER_DEFINITIONS = find_by_release_stage(ALL_DEFINITIONS, "other") +CERTIFIED_DEFINITIONS = find_by_support_level(ALL_DEFINITIONS, "certified") +COMMUNITY_DEFINITIONS = find_by_support_level(ALL_DEFINITIONS, "community") +OTHER_DEFINITIONS = find_by_support_level(ALL_DEFINITIONS, "other") diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md index 22edcd917b27..228226040f9c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md @@ -1,6 +1,6 @@ # Bypassing column selection validation for sources which fail it This migration adds `fail_on_extra_columns: false` to the `basic_read` test in the `acceptance-test-config.yml` -file for all Beta and GA connectors which fail the stricter validation added to the `basic_read` test. It creates +file for all community and certified connectors which fail the stricter validation added to the `basic_read` test. It creates issues for each of the connectors whose configs were modified as a result. Before following this README, please reference the `acceptance_test_config_migraton` README for general @@ -9,10 +9,10 @@ usage information for the given scripts. ## Add bypass for connectors that fail the new CAT test ### Run tests on all connectors -Run CAT on all Beta and GA connectors. +Run CAT on all community and certified connectors. ``` -python run_tests.py --allow_beta +python run_tests.py --allow_community ``` ### Collect output from connectors that fail due to `additionalProperties` @@ -22,18 +22,18 @@ sh get_failures.sh ``` ### Migrate configs for failed connectors -For the connectors that failed due to `Additional properties are not allowed:`, we want to add the new -`fail_on_extra_columns` input parameter to the basic read test. To do this, +For the connectors that failed due to `Additional properties are not allowed:`, we want to add the new +`fail_on_extra_columns` input parameter to the basic read test. To do this, ``` -python config_migration.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_beta +python config_migration.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_community ``` Add these bypasses to the PR that adds the new CAT test! ## Create issues for failing connectors (`create_issues.py`) -Create one issue per GA connectors to add the missing columns to the spec and remove the `fail_on_extra_columns` bypass. +Create one issue per certified connectors to add the missing columns to the spec and remove the `fail_on_extra_columns` bypass. Issues get created with the following labels: * `area/connectors` @@ -44,10 +44,10 @@ Issues get created with the following labels: ### How to run: **Dry run**: ``` -python create_issues.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_beta +python create_issues.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_community ``` **Real execution**: ``` -python create_issues.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_beta --no-dry +python create_issues.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_community --no-dry ``` diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 index 5bdbcce1f6b9..395fc61376bd 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 @@ -1,6 +1,6 @@ ## What A `test_strictness_level` field was introduced to Connector Acceptance Tests. -{{ connector_name }} is a {{ release_stage }} connector, we want it to have a `high` test strictness level. +{{ connector_name }} is a {{ support_level }} connector, we want it to have a `high` test strictness level. **This will help**: - maximize the acceptance test coverage on this connector. diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 index f04a5c189565..c6f023e2b013 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 @@ -1,6 +1,6 @@ ## What A `test_strictness_level` field was introduced to Connector Acceptance Tests. -{{ connector_name }} is a {{ release_stage }} connector, we want it to have a `high` test strictness level. +{{ connector_name }} is a {{ support_level }} connector, we want it to have a `high` test strictness level. **This will help**: - maximize the acceptance test coverage on this connector. @@ -10,10 +10,10 @@ A `test_strictness_level` field was introduced to Connector Acceptance Tests. 1. Migrate the existing `acceptance-test-config.yml` file to the latest configuration format. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/connector-acceptance-test/README.md#L61)) 2. Enable `high` test strictness level in `acceptance-test-config.yml`. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md#L240)) -⚠️ ⚠️ ⚠️ +⚠️ ⚠️ ⚠️ **If tests are failing please fix the failing test by changing the `acceptance-test-config.yml` file or use `bypass_reason` fields to explain why a specific test can't be run.** -Please open a new PR if the new enabled tests help discover a new bug. +Please open a new PR if the new enabled tests help discover a new bug. Once this bug fix is merged please rebase this branch and run `/test` again. You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/). diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py index 9858915e9032..044a3621d3ba 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py +++ b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py @@ -31,18 +31,9 @@ def add_dry_param(parser: argparse.ArgumentParser): ) -def add_allow_alpha_param(parser: argparse.ArgumentParser): +def add_allow_community_param(parser: argparse.ArgumentParser): parser.add_argument( - "--allow_alpha", - action=argparse.BooleanOptionalAction, - default=False, - help="Whether to apply the change to alpha connectors, if they are included in the list of connectors.", - ) - - -def add_allow_beta_param(parser: argparse.ArgumentParser): - parser.add_argument( - "--allow_beta", + "--allow_community", action=argparse.BooleanOptionalAction, default=False, help="Whether to apply the change to bets connectors, if they are included in the list of connectors.", @@ -66,12 +57,12 @@ def get_valid_definitions_from_args(args): connector_technical_name = definitions.get_airbyte_connector_name_from_definition(definition) if not definitions.is_airbyte_connector(definition): logging.warning(f"Skipping {connector_technical_name} since it's not an airbyte connector.") - elif not args.allow_beta and definition in definitions.BETA_DEFINITIONS: - logging.warning(f"Skipping {connector_technical_name} since it's a beta connector. This is configurable via `--allow_beta`") - elif not args.allow_alpha and definition in definitions.ALPHA_DEFINTIONS: - logging.warning(f"Skipping {connector_technical_name} since it's an alpha connector. This is configurable via `--allow_alpha`") + elif not args.allow_community and definition in definitions.COMMUNITY_DEFINITIONS: + logging.warning( + f"Skipping {connector_technical_name} since it's a community connector. This is configurable via `--allow_community`" + ) elif definition in definitions.OTHER_DEFINITIONS: - logging.warning(f"Skipping {connector_technical_name} since it doesn't have a release stage.") + logging.warning(f"Skipping {connector_technical_name} since it doesn't have a support level.") else: valid_definitions.append(definition) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py index 38ae2f96fd71..21184060b502 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py @@ -3,8 +3,10 @@ # import json +import sys from contextlib import contextmanager +import dagger import pytest @@ -23,3 +25,14 @@ def postgres_source_spec_schema(): @contextmanager def does_not_raise(): yield + + +@pytest.fixture(scope="module") +def anyio_backend(): + return "asyncio" + + +@pytest.fixture(scope="module") +async def dagger_client(): + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: + yield client diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py index 76b9868c5cc9..0b78b93148c6 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_asserts.py @@ -3,7 +3,7 @@ # import pytest -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, @@ -109,30 +109,6 @@ def test_verify_records_schema(configured_catalog: ConfiguredAirbyteCatalog): ({"a": "2021-08-10T12:43:15Z"}, {"type": "object", "properties": {"a": {"type": "string", "format": "date-time"}}}, True), ({"a": "2018-11-13T20:20:39+00:00"}, {"type": "object", "properties": {"a": {"type": "string", "format": "date-time"}}}, True), ({"a": "2018-21-13T20:20:39+00:00"}, {"type": "object", "properties": {"a": {"type": "string", "format": "date-time"}}}, False), - # This is valid for postgres sql but not valid for bigquery - ({"a": "2014-09-27 9:35z"}, {"type": "object", "properties": {"a": {"type": "string", "format": "date-time"}}}, False), - # Seconds are obligatory for bigquery timestamp - ({"a": "2014-09-27 9:35"}, {"type": "object", "properties": {"a": {"type": "string", "format": "date-time"}}}, False), - ({"a": "2014-09-27 9:35:0z"}, {"type": "object", "properties": {"a": {"type": "string", "format": "date-time"}}}, True), - # email - ({"a": "2018-11-13 20:20:39"}, {"type": "object", "properties": {"a": {"type": "string", "format": "email"}}}, False), - ({"a": "hi@example.com"}, {"type": "object", "properties": {"a": {"type": "string", "format": "email"}}}, True), - ({"a": "Пример@example.com"}, {"type": "object", "properties": {"a": {"type": "string", "format": "email"}}}, True), - ({"a": "写电子邮件@子邮件"}, {"type": "object", "properties": {"a": {"type": "string", "format": "email"}}}, True), - # hostname - ({"a": "2018-11-13 20:20:39"}, {"type": "object", "properties": {"a": {"type": "string", "format": "hostname"}}}, False), - ({"a": "hi@example.com"}, {"type": "object", "properties": {"a": {"type": "string", "format": "hostname"}}}, False), - ({"a": "localhost"}, {"type": "object", "properties": {"a": {"type": "string", "format": "hostname"}}}, True), - ({"a": "example.com"}, {"type": "object", "properties": {"a": {"type": "string", "format": "hostname"}}}, True), - # ipv4 - ({"a": "example.com"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv4"}}}, False), - ({"a": "0.0.0.1000"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv4"}}}, False), - ({"a": "0.0.0.0"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv4"}}}, True), - # ipv6 - ({"a": "example.com"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv6"}}}, False), - ({"a": "1080:0:0:0:8:800:200C:417A"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv6"}}}, True), - ({"a": "::1"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv6"}}}, True), - ({"a": "::"}, {"type": "object", "properties": {"a": {"type": "string", "format": "ipv6"}}}, True), ], indirect=["configured_catalog"], ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py index bf1a0497f166..d2fe626c25f8 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py @@ -6,7 +6,7 @@ from typing import MutableMapping, Union import pytest -from airbyte_cdk.models import AirbyteStream, ConnectorSpecification +from airbyte_protocol.models import AirbyteStream, ConnectorSpecification from connector_acceptance_test.tests.test_core import TestDiscovery as _TestDiscovery from connector_acceptance_test.tests.test_core import TestSpec as _TestSpec from connector_acceptance_test.utils.backward_compatibility import NonBackwardCompatibleError, validate_previous_configs @@ -1193,17 +1193,16 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe ), }, ), -] - -VALID_CATALOG_TRANSITIONS = [ Transition( - name="Adding a stream to a catalog should not fail.", - should_fail=False, + name="Removing a top level field should fail.", + should_fail=True, previous={ "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", - "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "json_schema": { + "properties": {"username": {"type": "string"}, "email": {"type": "string"}}, + }, "supported_sync_modes": ["full_refresh"], } ) @@ -1212,21 +1211,45 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", - "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "json_schema": { + "properties": {"username": {"type": "string"}}, + }, "supported_sync_modes": ["full_refresh"], } - ), - "other_test_stream": AirbyteStream.parse_obj( + ) + }, + ), + Transition( + name="Removing a nested field should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( { - "name": "other_test_stream", + "name": "test_stream", + "json_schema": { + "properties": { + "user": {"type": "object", "properties": {"username": {"type": "string"}, "email": {"type": "string"}}} + } + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, "supported_sync_modes": ["full_refresh"], } - ), + ) }, ), +] + +VALID_CATALOG_TRANSITIONS = [ Transition( - name="Making a field nullable should not fail.", + name="Adding a stream to a catalog should not fail.", should_fail=False, previous={ "test_stream": AirbyteStream.parse_obj( @@ -1241,21 +1264,28 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, "supported_sync_modes": ["full_refresh"], - "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string", "null"]}}}}}, } - ) + ), + "other_test_stream": AirbyteStream.parse_obj( + { + "name": "other_test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], + } + ), }, ), Transition( - name="Changing 'type' field to list should not fail.", + name="Making a field nullable should not fail.", should_fail=False, previous={ "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", - "supported_sync_modes": ["full_refresh"], "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], } ) }, @@ -1263,25 +1293,21 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", - "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string"]}}}}}, "supported_sync_modes": ["full_refresh"], + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string", "null"]}}}}}, } ) }, ), Transition( - name="Removing a field should not fail.", + name="Changing 'type' field to list should not fail.", should_fail=False, previous={ "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", - "json_schema": { - "properties": { - "user": {"type": "object", "properties": {"username": {"type": "string"}, "email": {"type": "string"}}} - } - }, "supported_sync_modes": ["full_refresh"], + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, } ) }, @@ -1289,7 +1315,7 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe "test_stream": AirbyteStream.parse_obj( { "name": "test_stream", - "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": "string"}}}}}, + "json_schema": {"properties": {"user": {"type": "object", "properties": {"username": {"type": ["string"]}}}}}, "supported_sync_modes": ["full_refresh"], } ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py index 2687bfaf5101..671129ceb922 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py @@ -100,6 +100,33 @@ def test_config_parsing(self, raw_config, expected_output_config, expected_error parsed_config = config.Config.parse_obj(raw_config) assert parsed_config == expected_output_config + def test_cursor_path_union_str(self): + parsed_config = config.Config.parse_obj(self._config_with_incremental_cursor_paths(["2331"])) + assert type(parsed_config.acceptance_tests.incremental.tests[0].cursor_paths["stream_name"][0]) == str + + def test_cursor_path_union_int(self): + parsed_config = config.Config.parse_obj(self._config_with_incremental_cursor_paths([2331])) + assert type(parsed_config.acceptance_tests.incremental.tests[0].cursor_paths["stream_name"][0]) == int + + @staticmethod + def _config_with_incremental_cursor_paths(cursor_paths): + return { + "connector_image": "foo", + "acceptance_tests": { + "incremental": { + "tests": [ + { + "config_path": "config_path.json", + "cursor_paths": { + "stream_name": cursor_paths + } + } + ] + } + }, + "test_strictness_level": "low" + } + @pytest.mark.parametrize( "legacy_config, expected_parsed_config", [ diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py new file mode 100644 index 000000000000..0ef84e81b0dd --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py @@ -0,0 +1,134 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import os + +import pytest +from airbyte_protocol.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteMessage, + AirbyteRecordMessage, + OrchestratorType, +) +from airbyte_protocol.models import Type as AirbyteMessageType +from connector_acceptance_test.utils import connector_runner + +pytestmark = pytest.mark.anyio + + +class TestContainerRunner: + @pytest.fixture + def dev_image_name(self): + return "airbyte/source-faker:dev" + + @pytest.fixture + def released_image_name(self): + return "airbyte/source-faker:latest" + + @pytest.fixture() + async def local_tar_image(self, dagger_client, tmpdir, released_image_name): + local_image_tar_path = str(tmpdir / "local_image.tar") + await dagger_client.container().from_(released_image_name).export(local_image_tar_path) + os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"] = local_image_tar_path + yield local_image_tar_path + os.environ.pop("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH") + + async def test_load_container_from_tar(self, dagger_client, dev_image_name, local_tar_image): + runner = connector_runner.ConnectorRunner(dev_image_name, dagger_client) + await runner.load_container() + assert await runner._connector_under_test_container.with_exec(["spec"]) + + async def test_load_container_from_released_connector(self, dagger_client, released_image_name): + runner = connector_runner.ConnectorRunner(released_image_name, dagger_client) + await runner.load_container() + assert await runner._connector_under_test_container.with_exec(["spec"]) + + async def test_get_container_env_variable_value(self, dagger_client, dev_image_name, local_tar_image): + runner = connector_runner.ConnectorRunner(dev_image_name, dagger_client, custom_environment_variables={"FOO": "BAR"}) + assert await runner.get_container_env_variable_value("FOO") == "BAR" + + def test_parse_airbyte_messages_from_command_output(self, mocker, tmp_path): + old_configuration_path = tmp_path / "config.json" + new_configuration = {"field_a": "new_value_a"} + mock_logging = mocker.MagicMock() + mocker.patch.object(connector_runner, "logging", mock_logging) + mocker.patch.object(connector_runner, "docker") + raw_command_output = "\n".join( + [ + AirbyteMessage( + type=AirbyteMessageType.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"foo": "bar"}, emitted_at=1.0) + ).json(exclude_unset=False), + AirbyteMessage( + type=AirbyteMessageType.CONTROL, + control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=1.0, + connectorConfig=AirbyteControlConnectorConfigMessage(config=new_configuration), + ), + ).json(exclude_unset=False), + "invalid message", + ] + ) + + mocker.patch.object(connector_runner.ConnectorRunner, "_persist_new_configuration") + runner = connector_runner.ConnectorRunner( + "source-test:dev", + mocker.Mock(), + connector_configuration_path=old_configuration_path, + custom_environment_variables={"foo": "bar"}, + ) + runner.parse_airbyte_messages_from_command_output(raw_command_output) + runner._persist_new_configuration.assert_called_once_with(new_configuration, 1) + mock_logging.warning.assert_called_once() + + @pytest.mark.parametrize( + "pass_configuration_path, old_configuration, new_configuration, new_configuration_emitted_at, expect_new_configuration", + [ + pytest.param( + True, + {"field_a": "value_a"}, + {"field_a": "value_a"}, + 1, + False, + id="Config unchanged: No new configuration persisted", + ), + pytest.param( + True, {"field_a": "value_a"}, {"field_a": "new_value_a"}, 1, True, id="Config changed: New configuration persisted" + ), + pytest.param( + False, + {"field_a": "value_a"}, + {"field_a": "new_value_a"}, + 1, + False, + id="Config changed but persistence is disable: New configuration not persisted", + ), + ], + ) + def test_persist_new_configuration( + self, + mocker, + tmp_path, + pass_configuration_path, + old_configuration, + new_configuration, + new_configuration_emitted_at, + expect_new_configuration, + ): + if pass_configuration_path: + old_configuration_path = tmp_path / "config.json" + with open(old_configuration_path, "w") as old_configuration_file: + json.dump(old_configuration, old_configuration_file) + else: + old_configuration_path = None + mocker.patch.object(connector_runner, "docker") + runner = connector_runner.ConnectorRunner("source-test:dev", mocker.MagicMock(), old_configuration_path) + new_configuration_path = runner._persist_new_configuration(new_configuration, new_configuration_emitted_at) + if not expect_new_configuration: + assert new_configuration_path is None + else: + assert new_configuration_path == tmp_path / "updated_configurations" / f"config|{new_configuration_emitted_at}.json" diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py deleted file mode 100644 index d1716de8f76e..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_container_runner.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import json - -import pytest -from airbyte_cdk.models import ( - AirbyteControlConnectorConfigMessage, - AirbyteControlMessage, - AirbyteMessage, - AirbyteRecordMessage, - OrchestratorType, -) -from airbyte_cdk.models import Type as AirbyteMessageType -from connector_acceptance_test.utils import connector_runner - - -class TestContainerRunner: - def test_run_call_persist_configuration(self, mocker, tmp_path): - old_configuration_path = tmp_path / "config.json" - new_configuration = {"field_a": "new_value_a"} - mocker.patch.object(connector_runner, "docker") - records_reads = [ - AirbyteMessage( - type=AirbyteMessageType.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"foo": "bar"}, emitted_at=1.0) - ).json(exclude_unset=False), - AirbyteMessage( - type=AirbyteMessageType.CONTROL, - control=AirbyteControlMessage( - type=OrchestratorType.CONNECTOR_CONFIG, - emitted_at=1.0, - connectorConfig=AirbyteControlConnectorConfigMessage(config=new_configuration), - ), - ).json(exclude_unset=False), - ] - mocker.patch.object(connector_runner.ConnectorRunner, "read", mocker.Mock(return_value=records_reads)) - mocker.patch.object(connector_runner.ConnectorRunner, "_persist_new_configuration") - runner = connector_runner.ConnectorRunner( - "source-test:dev", tmp_path, connector_configuration_path=old_configuration_path, custom_environment_variables={"foo": "bar"} - ) - list(runner.run("dummy_cmd")) - runner._persist_new_configuration.assert_called_once_with(new_configuration, 1) - runner._client.containers.run.assert_called_once_with( - image=runner._image, - command="dummy_cmd", - volumes={str(tmp_path) + "/run_1/input": {"bind": "/data"}, str(tmp_path) + "/run_1/output": {"bind": "/local", "mode": "rw"}}, - network_mode="host", - detach=True, - environment={"foo": "bar"}, - ) - - @pytest.mark.parametrize( - "pass_configuration_path, old_configuration, new_configuration, new_configuration_emitted_at, expect_new_configuration", - [ - pytest.param( - True, - {"field_a": "value_a"}, - {"field_a": "value_a"}, - 1, - False, - id="Config unchanged: No new configuration persisted", - ), - pytest.param( - True, {"field_a": "value_a"}, {"field_a": "new_value_a"}, 1, True, id="Config changed: New configuration persisted" - ), - pytest.param( - False, - {"field_a": "value_a"}, - {"field_a": "new_value_a"}, - 1, - False, - id="Config changed but persistence is disable: New configuration not persisted", - ), - ], - ) - def test_persist_new_configuration( - self, - mocker, - tmp_path, - pass_configuration_path, - old_configuration, - new_configuration, - new_configuration_emitted_at, - expect_new_configuration, - ): - if pass_configuration_path: - old_configuration_path = tmp_path / "config.json" - with open(old_configuration_path, "w") as old_configuration_file: - json.dump(old_configuration, old_configuration_file) - else: - old_configuration_path = None - mocker.patch.object(connector_runner, "docker") - runner = connector_runner.ConnectorRunner("source-test:dev", tmp_path, old_configuration_path) - new_configuration_path = runner._persist_new_configuration(new_configuration, new_configuration_emitted_at) - if not expect_new_configuration: - assert new_configuration_path is None - else: - assert new_configuration_path == tmp_path / "updated_configurations" / f"config|{new_configuration_emitted_at}.json" diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index 66872a4ba500..7d894b0450f6 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -7,7 +7,7 @@ import pytest from _pytest.outcomes import Failed -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, @@ -26,6 +26,8 @@ from .conftest import does_not_raise +pytestmark = pytest.mark.anyio + @pytest.mark.parametrize( "schema, cursors, should_fail", @@ -562,7 +564,7 @@ def test_configured_catalog_fixture(mocker, test_strictness_level, configured_ca ), ], ) -def test_read(schema, ignored_fields, expect_records_config, record, expected_records_by_stream, expectation): +async def test_read(mocker, schema, ignored_fields, expect_records_config, record, expected_records_by_stream, expectation): configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -572,13 +574,14 @@ def test_read(schema, ignored_fields, expect_records_config, record, expected_re ) ] ) - docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = [ - AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) - ] + docker_runner_mock = mocker.MagicMock( + call_read=mocker.AsyncMock( + return_value=[AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111))] + ) + ) t = test_core.TestBasicRead() with expectation: - t.test_read( + await t.test_read( connector_config=None, configured_catalog=configured_catalog, expect_records_config=expect_records_config, @@ -603,7 +606,9 @@ def test_read(schema, ignored_fields, expect_records_config, record, expected_re ], ) @pytest.mark.parametrize("additional_properties", [True, False, None]) -def test_fail_on_extra_columns(config_fail_on_extra_columns, record_has_unexpected_column, expectation_should_fail, additional_properties): +async def test_fail_on_extra_columns( + mocker, config_fail_on_extra_columns, record_has_unexpected_column, expectation_should_fail, additional_properties +): schema = {"type": "object", "properties": {"field_1": {"type": ["string"]}, "field_2": {"type": ["string"]}}} if additional_properties: schema["additionalProperties"] = additional_properties @@ -621,14 +626,16 @@ def test_fail_on_extra_columns(config_fail_on_extra_columns, record_has_unexpect ) ] ) - docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = [ - AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111)) - ] + docker_runner_mock = mocker.MagicMock( + call_read=mocker.AsyncMock( + return_value=[AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data=record, emitted_at=111))] + ) + ) + t = test_core.TestBasicRead() if expectation_should_fail: with pytest.raises(Failed, match="test_stream"): - t.test_read( + await t.test_read( connector_config=None, configured_catalog=configured_catalog, expect_records_config=ExpectedRecordsConfig(path="foobar"), @@ -732,18 +739,17 @@ def test_fail_on_extra_columns(config_fail_on_extra_columns, record_has_unexpect ([], False, False), ], ) -def test_airbyte_trace_message_on_failure(output, expect_trace_message_on_failure, should_fail): +async def test_airbyte_trace_message_on_failure(mocker, output, expect_trace_message_on_failure, should_fail): t = test_core.TestBasicRead() input_config = BasicReadTestConfig(expect_trace_message_on_failure=expect_trace_message_on_failure) - docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = output + docker_runner_mock = mocker.MagicMock(call_read=mocker.AsyncMock(return_value=output)) with patch.object(pytest, "skip", return_value=None): if should_fail: with pytest.raises(AssertionError, match="Connector should emit at least one error trace message"): - t.test_airbyte_trace_message_on_failure(None, input_config, docker_runner_mock) + await t.test_airbyte_trace_message_on_failure(None, input_config, docker_runner_mock) else: - t.test_airbyte_trace_message_on_failure(None, input_config, docker_runner_mock) + await t.test_airbyte_trace_message_on_failure(None, input_config, docker_runner_mock) @pytest.mark.parametrize( diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py index 4c545cb55429..ad7c45046304 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_global_fixtures.py @@ -6,7 +6,7 @@ import time import pytest -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode +from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from connector_acceptance_test import conftest from connector_acceptance_test.config import ( BasicReadTestConfig, @@ -65,10 +65,7 @@ def test_empty_streams_fixture(mocker, test_strictness_level, basic_read_test_co [ pytest.param( Config.TestStrictnessLevel.low, - BasicReadTestConfig( - config_path="config_path", - ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me")]} - ), + BasicReadTestConfig(config_path="config_path", ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me")]}), False, id="[LOW test strictness level] Ignored fields can be declared without bypass_reason.", ), @@ -76,17 +73,14 @@ def test_empty_streams_fixture(mocker, test_strictness_level, basic_read_test_co Config.TestStrictnessLevel.low, BasicReadTestConfig( config_path="config_path", - ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me", bypass_reason="test")]} + ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me", bypass_reason="test")]}, ), False, id="[LOW test strictness level] Ignored fields can be declared with a bypass_reason.", ), pytest.param( Config.TestStrictnessLevel.high, - BasicReadTestConfig( - config_path="config_path", - ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me")]} - ), + BasicReadTestConfig(config_path="config_path", ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me")]}), True, id="[HIGH test strictness level] Ignored fields can't be declared without bypass_reason.", ), @@ -94,7 +88,7 @@ def test_empty_streams_fixture(mocker, test_strictness_level, basic_read_test_co Config.TestStrictnessLevel.high, BasicReadTestConfig( config_path="config_path", - ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me", bypass_reason="test")]} + ignored_fields={"test_stream": [IgnoredFieldsConfiguration(name="ignore_me", bypass_reason="test")]}, ), False, id="[HIGH test strictness level] Ignored fields can be declared with a bypass_reason.", @@ -104,7 +98,9 @@ def test_empty_streams_fixture(mocker, test_strictness_level, basic_read_test_co def test_ignored_fields_fixture(mocker, test_strictness_level, basic_read_test_config, expect_test_failure): mocker.patch.object(conftest.pytest, "fail") # Pytest prevents fixture to be directly called. Using __wrapped__ allows us to call the actual function before it's been wrapped by the decorator. - assert conftest.ignored_fields_fixture.__wrapped__(basic_read_test_config, test_strictness_level) == basic_read_test_config.ignored_fields + assert ( + conftest.ignored_fields_fixture.__wrapped__(basic_read_test_config, test_strictness_level) == basic_read_test_config.ignored_fields + ) if expect_test_failure: conftest.pytest.fail.assert_called_once() else: diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index d8c44190b6f6..58256392d49f 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -11,7 +11,7 @@ import pendulum import pytest -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, AirbyteStateBlob, @@ -35,6 +35,10 @@ future_state_fixture, ) +pytestmark = [ + pytest.mark.anyio, +] + def build_messages_from_record_data(stream: str, records: list[dict]) -> list[AirbyteMessage]: return [build_record_message(stream, data) for data in records] @@ -140,12 +144,12 @@ def test_compare_cursor_with_threshold(record_value, state_value, threshold_days @pytest.mark.parametrize( "run_per_stream_test", [ - pytest.param(False, id="test_two_sequential_reads_using_a_mock_connector_emitting_legacy_state"), + # pytest.param(False, id="test_two_sequential_reads_using_a_mock_connector_emitting_legacy_state"), pytest.param(True, id="test_two_sequential_reads_using_a_mock_connector_emitting_per_stream_state"), ], ) -def test_incremental_two_sequential_reads( - records1, records2, latest_state, threshold_days, cursor_type, expected_error, run_per_stream_test +async def test_incremental_two_sequential_reads( + mocker, records1, records2, latest_state, threshold_days, cursor_type, expected_error, run_per_stream_test ): input_config = IncrementalConfig(threshold_days=threshold_days) cursor_paths = {"test_stream": ["date"]} @@ -178,12 +182,12 @@ def test_incremental_two_sequential_reads( call_read_with_state_output_messages = build_messages_from_record_data("test_stream", records2) docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = call_read_output_messages - docker_runner_mock.call_read_with_state.return_value = call_read_with_state_output_messages + docker_runner_mock.call_read = mocker.AsyncMock(return_value=call_read_output_messages) + docker_runner_mock.call_read_with_state = mocker.AsyncMock(return_value=call_read_with_state_output_messages) t = _TestIncremental() with expected_error: - t.test_two_sequential_reads( + await t.test_two_sequential_reads( inputs=input_config, connector_config=MagicMock(), configured_catalog_for_incremental=catalog, @@ -197,13 +201,8 @@ def test_incremental_two_sequential_reads( [ ( "test_stream", - { - "dateCreated": { - "type": "string", - "format": "date-time" - } - }, - {'test_stream': ['dateCreated']}, + {"dateCreated": {"type": "string", "format": "date-time"}}, + {"test_stream": ["dateCreated"]}, [{"dateCreated": "2020-01-01T01:01:01.000000Z"}, {"dateCreated": "2020-01-02T01:01:01.000000Z"}], [], {"dateCreated": "2020-01-02T01:01:01.000000Z"}, @@ -211,17 +210,12 @@ def test_incremental_two_sequential_reads( ), ( "test_stream", - { - "dateCreated": { - "type": "string", - "format": "date-time" - } - }, - {'test_stream': ['dateCreated']}, + {"dateCreated": {"type": "string", "format": "date-time"}}, + {"test_stream": ["dateCreated"]}, [{"dateCreated": "2020-01-01T01:01:01.000000Z"}, {"dateCreated": "2020-01-02T01:01:01.000000Z"}], [], {}, - pytest.raises(AssertionError, match="At least one valid state should be produced, given a cursor path") + pytest.raises(AssertionError, match="At least one valid state should be produced, given a cursor path"), ), ], ) @@ -232,8 +226,8 @@ def test_incremental_two_sequential_reads( pytest.param(True, id="test_two_sequential_reads_using_a_mock_connector_emitting_per_stream_state"), ], ) -def test_incremental_two_sequential_reads_state_invalid( - stream_name, records1, records2, latest_state, cursor_type, cursor_paths, expected_error, run_per_stream_test +async def test_incremental_two_sequential_reads_state_invalid( + mocker, stream_name, records1, records2, latest_state, cursor_type, cursor_paths, expected_error, run_per_stream_test ): input_config = IncrementalConfig() catalog = ConfiguredAirbyteCatalog( @@ -268,12 +262,12 @@ def test_incremental_two_sequential_reads_state_invalid( call_read_with_state_output_messages = build_messages_from_record_data(stream_name, records2) docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = call_read_output_messages - docker_runner_mock.call_read_with_state.return_value = call_read_with_state_output_messages + docker_runner_mock.call_read = mocker.AsyncMock(return_value=call_read_output_messages) + docker_runner_mock.call_read_with_state = mocker.AsyncMock(return_value=call_read_with_state_output_messages) t = _TestIncremental() with expected_error: - t.test_two_sequential_reads( + await t.test_two_sequential_reads( inputs=input_config, connector_config=MagicMock(), configured_catalog_for_incremental=catalog, @@ -602,7 +596,7 @@ def test_incremental_two_sequential_reads_state_invalid( pytest.param(True, id="test_read_with_multiple_states_using_a_mock_connector_emitting_per_stream_state"), ], ) -def test_per_stream_read_with_multiple_states(records, state_records, threshold_days, expected_error, run_per_stream_test): +async def test_per_stream_read_with_multiple_states(mocker, records, state_records, threshold_days, expected_error, run_per_stream_test): input_config = IncrementalConfig(threshold_days=threshold_days) cursor_paths = {"test_stream": ["date"], "test_stream_2": ["date"]} catalog = ConfiguredAirbyteCatalog( @@ -668,12 +662,12 @@ def test_per_stream_read_with_multiple_states(records, state_records, threshold_ ] docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = call_read_output_messages - docker_runner_mock.call_read_with_state.side_effect = call_read_with_state_output_messages + docker_runner_mock.call_read = mocker.AsyncMock(return_value=call_read_output_messages) + docker_runner_mock.call_read_with_state = mocker.AsyncMock(side_effect=call_read_with_state_output_messages) t = _TestIncremental() with expected_error: - t.test_read_sequential_slices( + await t.test_read_sequential_slices( inputs=input_config, connector_config=MagicMock(), configured_catalog_for_incremental=catalog, @@ -754,12 +748,12 @@ def test_config_skip_test(): ), ], ) -def test_state_with_abnormally_large_values(mocker, read_output, expectation): +async def test_state_with_abnormally_large_values(mocker, read_output, expectation): docker_runner_mock = mocker.MagicMock() - docker_runner_mock.call_read_with_state.return_value = read_output + docker_runner_mock.call_read_with_state = mocker.AsyncMock(return_value=read_output) t = _TestIncremental() with expectation: - t.test_state_with_abnormally_large_values( + await t.test_state_with_abnormally_large_values( connector_config=mocker.MagicMock(), configured_catalog=ConfiguredAirbyteCatalog( streams=[ @@ -834,8 +828,12 @@ def test_future_state_configuration_fixture(mocker, test_strictness_level, input test_incremental.pytest.fail.assert_not_called() -TEST_AIRBYTE_STREAM_A = AirbyteStream(name="test_stream_a", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]) -TEST_AIRBYTE_STREAM_B = AirbyteStream(name="test_stream_b", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental]) +TEST_AIRBYTE_STREAM_A = AirbyteStream( + name="test_stream_a", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] +) +TEST_AIRBYTE_STREAM_B = AirbyteStream( + name="test_stream_b", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] +) TEST_CONFIGURED_AIRBYTE_STREAM_A = ConfiguredAirbyteStream( stream=TEST_AIRBYTE_STREAM_A, diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py index b526c6733938..afb0ae67889c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py @@ -7,7 +7,7 @@ import pendulum import pytest -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, AirbyteStream, @@ -217,7 +217,7 @@ def test_get_expected_schema_structure(schema, pathes): (["option1"], 2, {"a_key": "a_value"}), (["option2"], 1, ["value1", "value2"]), (["nonexistent_key"], 0, None), - (["option1", "option2"], 3, ["value1", "value2"]) + (["option1", "option2"], 3, ["value1", "value2"]), ], ) def test_find_and_get_nodes(keys: List[Text], num_paths: int, last_value: Any): diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index 88d6226cfc74..fdca8ac03994 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict import pytest -from airbyte_cdk.models import ConnectorSpecification +from airbyte_protocol.models import ConnectorSpecification from connector_acceptance_test import conftest from connector_acceptance_test.tests.test_core import TestSpec as _TestSpec @@ -414,6 +414,76 @@ def parametrize_test_case(*test_cases: Dict[str, Any]) -> Callable: } }, }, + "should_fail": False, + }, + { + "test_id": "different_default_in_common_property", + "connector_spec": { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "common": {"type": "string", "const": "option1", "default": "optionX"}, + "option1": {"type": "string"}, + }, + } + ], + } + }, + }, + "should_fail": True, + }, + { + "test_id": "enum_keyword_in_common_property", + "connector_spec": { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "common": {"type": "string", "const": "option1", "enum": ["option1"]}, + "option1": {"type": "string"}, + }, + }, + { + "type": "object", + "properties": { + "common": {"type": "string", "const": "option2", "enum": ["option2"]}, + "option2": {"type": "string"}, + }, + }, + ], + } + }, + }, + "should_fail": False, + }, + { + "test_id": "different_enum_in_common_property", + "connector_spec": { + "type": "object", + "properties": { + "credentials": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "common": {"type": "string", "const": "option1", "enum": ["option1", "option2"]}, + "option1": {"type": "string"}, + }, + } + ], + } + }, + }, "should_fail": True, }, ) @@ -472,24 +542,28 @@ def test_enum_usage(connector_spec, should_fail): @pytest.mark.parametrize( "connector_spec, expected_error", [ - # SUCCESS: no authSpecification specified + # SUCCESS: no advancedAuth specified (ConnectorSpecification(connectionSpecification={}), ""), - # FAIL: Field specified in root object does not exist + # SUCCESS: empty predicate_key and oauth_config_specification + ( + ConnectorSpecification( + connectionSpecification={"type": "object"}, + advanced_auth={"auth_type": "oauth2.0", "oauth_config_specification": {}}, + ), + "", + ), + # FAIL: Field specified in predicate_key does not exist ( ConnectorSpecification( connectionSpecification={"type": "object"}, - authSpecification={ + advanced_auth={ "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, + "predicate_key": ["credentials", "auth_type"], }, ), "Specified oauth fields are missed from spec schema:", ), - # SUCCESS: Empty root object + # FAIL: Field specified in oauth_user_input_from_connector_config_specification does not exist ( ConnectorSpecification( connectionSpecification={ @@ -501,178 +575,108 @@ def test_enum_usage(connector_spec, should_fail): "refresh_token": {"type": "string"}, }, }, - authSpecification={ + advanced_auth={ "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "", - ), - # FAIL: Some oauth fields missed - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - }, + "properties": {"api_url": {"type": "string", "path_in_connector_config": ["api_url"]}}, } }, }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, ), "Specified oauth fields are missed from spec schema:", ), - # SUCCESS: case w/o oneOf property + # FAIL: Field specified in complete_oauth_output_specification does not exist ( ConnectorSpecification( connectionSpecification={ "type": "object", - "properties": { - "credentials": { - "type": "object", - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - }, - } - }, + "properties": {"authentication": {"type": "object", "properties": {"client_id": {"type": "string"}}}}, }, - authSpecification={ + advanced_auth={ "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, - ), - "", - ), - # SUCCESS: case w/ oneOf property - ( - ConnectorSpecification( - connectionSpecification={ - "type": "object", - "properties": { - "credentials": { + "oauth_config_specification": { + "complete_oauth_output_specification": { "type": "object", - "oneOf": [ - { - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - } - }, - { - "properties": { - "api_key": {"type": "string"}, - } - }, - ], + "properties": {"client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}}, } }, }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], - }, - }, ), - "", + "Specified oauth fields are missed from spec schema:", ), - # FAIL: Wrong root object index + # FAIL: Field specified in complete_oauth_server_output_specification does not exist ( ConnectorSpecification( connectionSpecification={ "type": "object", - "properties": { - "credentials": { - "type": "object", - "oneOf": [ - { - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - } - }, - { - "properties": { - "api_key": {"type": "string"}, - } - }, - ], - } - }, + "properties": {"authentication": {"type": "object", "properties": {"client_id": {"type": "string"}}}}, }, - authSpecification={ + advanced_auth={ "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 1], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + "oauth_config_specification": { + "complete_oauth_server_output_specification": { + "type": "object", + "properties": {"client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}}, + } }, }, ), "Specified oauth fields are missed from spec schema:", ), - # SUCCESS: root object index equal to 1 + # SUCCESS: Fields specified in advanced_auth exist in spec ( ConnectorSpecification( connectionSpecification={ "type": "object", "properties": { + "api_url": {"type": "object"}, "credentials": { "type": "object", - "oneOf": [ - { - "properties": { - "api_key": {"type": "string"}, - } - }, - { - "properties": { - "client_id": {"type": "string"}, - "client_secret": {"type": "string"}, - "access_token": {"type": "string"}, - "refresh_token": {"type": "string"}, - } - }, - ], - } + "properties": { + "auth_type": {"type": "string", "const": "oauth2.0"}, + "client_id": {"type": "string"}, + "client_secret": {"type": "string"}, + "access_token": {"type": "string"}, + "refresh_token": {"type": "string"}, + "token_expiry_date": {"type": "string", "format": "date-time"}, + }, + }, }, }, - authSpecification={ - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 1], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]], + advanced_auth={ + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": {"domain": {"type": "string", "path_in_connector_config": ["api_url"]}}, + }, + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": {"type": "string", "path_in_connector_config": ["credentials", "access_token"]}, + "refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"], + }, + }, + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, }, }, ), diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec_unit.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec_unit.py index ddf0928bfce3..56975ca38126 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec_unit.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec_unit.py @@ -6,7 +6,6 @@ import docker import pytest -from connector_acceptance_test.utils import ConnectorRunner def build_docker_image(text: str, tag: str) -> docker.models.images.Image: @@ -59,23 +58,3 @@ def connector_image_with_ne_properties(): yield tag client = docker.from_env() client.images.remove(image=tag, force=True) - - -class TestEnvAttributes: - def test_correct_connector_image(self, correct_connector_image, tmp_path): - docker_runner = ConnectorRunner(image_name=correct_connector_image, volume=tmp_path) - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT"), "AIRBYTE_ENTRYPOINT must be set in dockerfile" - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT") == " ".join( - docker_runner.entry_point - ), "env should be equal to space-joined entrypoint" - - def test_connector_image_without_env(self, connector_image_without_env, tmp_path): - docker_runner = ConnectorRunner(image_name=connector_image_without_env, volume=tmp_path) - assert not docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT"), "this test should fail if AIRBYTE_ENTRYPOINT defined" - - def test_docker_image_env_ne_entrypoint(self, connector_image_with_ne_properties, tmp_path): - docker_runner = ConnectorRunner(image_name=connector_image_with_ne_properties, volume=tmp_path) - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT"), "AIRBYTE_ENTRYPOINT must be set in dockerfile" - assert docker_runner.env_variables.get("AIRBYTE_ENTRYPOINT") != " ".join(docker_runner.entry_point), ( - "This test should fail if we have " ".join(ENTRYPOINT)==ENV" - ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py index 4cb190dfc1be..0d913e3a1daa 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_test_full_refresh.py @@ -7,7 +7,7 @@ import pytest from _pytest.outcomes import Failed -from airbyte_cdk.models import ( +from airbyte_protocol.models import ( AirbyteMessage, AirbyteRecordMessage, AirbyteStream, @@ -19,12 +19,14 @@ from connector_acceptance_test.config import ConnectionTestConfig, IgnoredFieldsConfiguration from connector_acceptance_test.tests.test_full_refresh import TestFullRefresh as _TestFullRefresh +pytestmark = pytest.mark.anyio + class ReadTestConfigWithIgnoreFields(ConnectionTestConfig): ignored_fields: Dict[str, List[IgnoredFieldsConfiguration]] = { "test_stream": [ IgnoredFieldsConfiguration(name="ignore_me", bypass_reason="test"), - IgnoredFieldsConfiguration(name="ignore_me_too", bypass_reason="test") + IgnoredFieldsConfiguration(name="ignore_me_too", bypass_reason="test"), ] } @@ -108,7 +110,7 @@ def get_default_catalog(schema, **kwargs): "schema, record, expected_record, fail_context", ignored_fields_test_cases, ) -def test_read_with_ignore_fields(mocker, schema, record, expected_record, fail_context): +async def test_read_with_ignore_fields(mocker, schema, record, expected_record, fail_context): catalog = get_default_catalog(schema) input_config = ReadTestConfigWithIgnoreFields() docker_runner_mock = mocker.MagicMock() @@ -120,11 +122,19 @@ def test_read_with_ignore_fields(mocker, schema, record, expected_record, fail_c sequence_of_docker_callread_results, list(reversed(sequence_of_docker_callread_results)), ): - docker_runner_mock.call_read.side_effect = [record_message_from_record([first], emitted_at=111), record_message_from_record([second], emitted_at=112)] + + docker_runner_mock = mocker.MagicMock( + call_read=mocker.AsyncMock( + side_effect=[ + record_message_from_record([first], emitted_at=111), + record_message_from_record([second], emitted_at=112), + ] + ) + ) t = _TestFullRefresh() with fail_context: - t.test_sequential_reads( + await t.test_sequential_reads( ignored_fields=input_config.ignored_fields, connector_config=mocker.MagicMock(), configured_catalog=catalog, @@ -203,23 +213,26 @@ def test_read_with_ignore_fields(mocker, schema, record, expected_record, fail_c "primary_key, first_read_records, second_read_records, fail_context", recordset_comparison_test_cases, ) -def test_recordset_comparison(mocker, primary_key, first_read_records, second_read_records, fail_context): +async def test_recordset_comparison(mocker, primary_key, first_read_records, second_read_records, fail_context): schema = { "type": "object", "properties": {"id": {"type": "integer"}, "first_name": {"type": "string"}, "last_name": {"type": "string"}}, } catalog = get_default_catalog(schema, primary_key=primary_key) input_config = ReadTestConfigWithIgnoreFields() - docker_runner_mock = mocker.MagicMock() - docker_runner_mock.call_read.side_effect = [ - record_message_from_record(first_read_records, emitted_at=111), - record_message_from_record(second_read_records, emitted_at=112), - ] + docker_runner_mock = mocker.MagicMock( + call_read=mocker.AsyncMock( + side_effect=[ + record_message_from_record(first_read_records, emitted_at=111), + record_message_from_record(second_read_records, emitted_at=112), + ] + ) + ) t = _TestFullRefresh() with fail_context: - t.test_sequential_reads( + await t.test_sequential_reads( ignored_fields=input_config.ignored_fields, connector_config=mocker.MagicMock(), configured_catalog=catalog, @@ -241,7 +254,7 @@ def test_recordset_comparison(mocker, primary_key, first_read_records, second_re AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"aa": 23}, emitted_at=112)), AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"aa": 24}, emitted_at=112)), ], - does_not_raise() + does_not_raise(), ), ( {"type": "object"}, @@ -253,7 +266,7 @@ def test_recordset_comparison(mocker, primary_key, first_read_records, second_re AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"aa": 24}, emitted_at=112)), AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"aa": 23}, emitted_at=112)), ], - does_not_raise() + does_not_raise(), ), ( {"type": "object"}, @@ -265,11 +278,11 @@ def test_recordset_comparison(mocker, primary_key, first_read_records, second_re AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"aa": 23}, emitted_at=111)), AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"aa": 24}, emitted_at=112)), ], - pytest.raises(AssertionError, match="emitted_at should increase on subsequent runs") + pytest.raises(AssertionError, match="emitted_at should increase on subsequent runs"), ), ], ) -def test_emitted_at_increase_on_subsequent_runs(mocker, schema, records_1, records_2, expectation): +async def test_emitted_at_increase_on_subsequent_runs(mocker, schema, records_1, records_2, expectation): configured_catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -279,13 +292,12 @@ def test_emitted_at_increase_on_subsequent_runs(mocker, schema, records_1, recor ) ] ) - docker_runner_mock = mocker.MagicMock() - docker_runner_mock.call_read.side_effect = [records_1, records_2] + docker_runner_mock = mocker.MagicMock(call_read=mocker.AsyncMock(side_effect=[records_1, records_2])) input_config = ReadTestConfigWithIgnoreFields() t = _TestFullRefresh() with expectation: - t.test_sequential_reads( + await t.test_sequential_reads( ignored_fields=input_config.ignored_fields, connector_config=mocker.MagicMock(), configured_catalog=configured_catalog, diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py index 0079c8860551..cc994347268a 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_utils.py @@ -12,15 +12,12 @@ from typing import Iterable from unittest.mock import Mock -import docker import pytest import yaml -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode +from airbyte_protocol.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from connector_acceptance_test.config import EmptyStreamConfiguration from connector_acceptance_test.utils import common from connector_acceptance_test.utils.compare import make_hashable -from connector_acceptance_test.utils.connector_runner import ConnectorRunner -from docker.errors import ContainerError, NotFound def not_sorted_data(): @@ -208,93 +205,6 @@ def binary_generator(lengths, last_line=None): yield ("bla-1234567890-bla\n" + last_line).encode() -def test_successful_logs_reading(): - line_count = 100 - line_lengths = [random.randint(0, 256) for _ in range(line_count)] - lines = [ - line for line in ConnectorRunner.read(container=MockContainer(status={"StatusCode": 0}, iter_logs=binary_generator(line_lengths))) - ] - assert line_count == len(lines) - for line, length in zip(lines, line_lengths): - assert len(line) - 1 == length - - -@pytest.mark.parametrize( - "traceback,container_error,last_line,expected_error", - ( - # container returns a some internal error - ( - "Traceback (most recent call last):\n File \"\", line 1, in \nKeyError: 'bbbb'", - "Some Container Error", - "Last Container Logs Line", - "Some Container Error", - ), - # container returns a raw traceback - ( - "Traceback (most recent call last):\n File \"\", line 1, in \nKeyError: 'bbbb'", - None, - "Last Container Logs Line", - "Traceback (most recent call last):\n File \"\", line 1, in \nKeyError: 'bbbb'", - ), - # container doesn't return any tracebacks or errors - ( - None, - None, - "Last Container Logs Line", - "Last Container Logs Line", - ), - ), - ids=["interal_error", "traceback", "last_line"], -) -def test_failed_reading(traceback, container_error, last_line, expected_error): - line_count = 10 - line_lengths = [random.randint(0, 32) for _ in range(line_count)] - - with pytest.raises(ContainerError) as exc: - status = {"StatusCode": 1} - if container_error: - status["Error"] = container_error - list( - ConnectorRunner.read( - container=MockContainer( - status=status, iter_logs=binary_generator(line_lengths, traceback or last_line) - ) - ) - ) - - assert expected_error == exc.value.stderr - - -@pytest.mark.parametrize( - "command,wait_timeout,expected_count", - ( - ( - "cnt=0; while [ $cnt -lt 10 ]; do cnt=$((cnt+1)); echo something; done", - 0, - 10, - ), - # Sometimes a container can finish own work before python tries to read it - ("echo something;", 0.1, 1), - ), - ids=["standard", "waiting"], -) -def test_docker_runner(command, wait_timeout, expected_count): - client = docker.from_env() - new_container = client.containers.run( - image="busybox", - command=f"""sh -c '{command}'""", - detach=True, - ) - if wait_timeout: - time.sleep(wait_timeout) - lines = list(ConnectorRunner.read(new_container, command=command)) - assert set(lines) == set(["something\n"]) - assert len(lines) == expected_count - - for container in client.containers.list(all=True, ignore_removed=True): - assert container.id != new_container.id, "Container should be removed after reading" - - def wait_status(container, expected_statuses): """Waits expected_statuses for 5 sec""" for _ in range(500): @@ -305,23 +215,6 @@ def wait_status(container, expected_statuses): f"expected statuses: {expected_statuses}" -def test_not_found_container(): - """Case when a container was removed before its reading""" - client = docker.from_env() - cmd = """sh -c 'sleep 100; exit 0'""" - new_container = client.containers.run( - image="busybox", - command=cmd, - detach=True, - auto_remove=True, - ) - wait_status(new_container, ["running", "created"]) - new_container.remove(force=True) - - with pytest.raises(NotFound): - list(ConnectorRunner.read(new_container, command=cmd)) - - class TestLoadYamlOrJsonPath: VALID_SPEC = { "documentationUrl": "https://google.com", diff --git a/airbyte-integrations/bases/debezium/build.gradle b/airbyte-integrations/bases/debezium/build.gradle index a82922b63951..5e515f0c6876 100644 --- a/airbyte-integrations/bases/debezium/build.gradle +++ b/airbyte-integrations/bases/debezium/build.gradle @@ -17,6 +17,7 @@ dependencies { testImplementation project(':airbyte-test-utils') testImplementation libs.connectors.testcontainers.jdbc + testImplementation libs.connectors.testcontainers.mysql testImplementation libs.connectors.testcontainers.postgresql testFixturesImplementation 'org.junit.jupiter:junit-jupiter-engine:5.4.2' diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java index 57e79d6be32a..9b27dc3b5280 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java @@ -12,7 +12,7 @@ * Postgres we add the lsn to the records. In MySql we add the file name and position to the * records. */ -public interface CdcMetadataInjector { +public interface CdcMetadataInjector { /** * A debezium record contains multiple pieces. Ref : @@ -24,6 +24,10 @@ public interface CdcMetadataInjector { */ void addMetaData(ObjectNode event, JsonNode source); + default void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, final String transactionTimestamp, final T metadataToAdd) { + throw new RuntimeException("Not Supported"); + } + /** * As part of Airbyte record we need to add the namespace (schema name) * diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java index ca2880649e79..63cf1866f8ae 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java @@ -59,9 +59,9 @@ default boolean isHeartbeatSupported() { * * @param offset DB CDC offset * @param event Event from the CDC load - * @return Returns `true` when the record is behind the offset. Otherwise, it returns `false` + * @return Returns `true` when the event is ahead of the offset. Otherwise, it returns `false` */ - default boolean isRecordBehindOffset(final Map offset, final ChangeEventWithMetadata event) { + default boolean isEventAheadOffset(final Map offset, final ChangeEventWithMetadata event) { return false; } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java index d46fbeef925f..aa689edd4be0 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java @@ -162,7 +162,7 @@ protected AirbyteMessage computeNext() { if (checkpointOffsetToSend.size() == 1 && changeEventIterator.hasNext() && !event.isSnapshotEvent() - && targetPosition.isRecordBehindOffset(checkpointOffsetToSend, event)) { + && targetPosition.isEventAheadOffset(checkpointOffsetToSend, event)) { sendCheckpointMessage = true; } } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java new file mode 100644 index 000000000000..92f6623d49e6 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import com.google.common.annotations.VisibleForTesting; +import com.mongodb.client.MongoClient; +import io.airbyte.integrations.debezium.CdcTargetPosition; +import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.integrations.debezium.internals.SnapshotMetadata; +import io.debezium.connector.mongodb.ResumeTokens; +import java.util.Map; +import java.util.Objects; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of the {@link CdcTargetPosition} interface that provides methods for determining + * when a sync has reached the target position of the CDC log for MongoDB. In this case, the target + * position is a resume token value from the MongoDB oplog. This implementation compares the + * timestamp present in the Debezium change event against the timestamp of the resume token recorded + * at the start of a sync. When the event timestamp exceeds the resume token timestamp, the sync + * should stop to prevent it from running forever. + */ +public class MongoDbCdcTargetPosition implements CdcTargetPosition { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcTargetPosition.class); + + private final BsonTimestamp resumeTokenTimestamp; + + public MongoDbCdcTargetPosition(final BsonDocument resumeToken) { + this.resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeToken); + } + + /** + * Constructs a new {@link MongoDbCdcTargetPosition} by fetching the most recent resume token from + * the MongoDB database. + * + * @param mongoClient A {@link MongoClient} used to retrieve the resume token. + * @return The {@link MongoDbCdcTargetPosition} set to the most recent resume token present in the + * database. + */ + public static MongoDbCdcTargetPosition targetPosition(final MongoClient mongoClient) { + final BsonDocument resumeToken = MongoDbResumeTokenHelper.getResumeToken(mongoClient); + return new MongoDbCdcTargetPosition(resumeToken); + } + + @VisibleForTesting + BsonTimestamp getResumeTokenTimestamp() { + return resumeTokenTimestamp; + } + + @Override + public boolean isHeartbeatSupported() { + return true; + } + + @Override + public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { + if (changeEventWithMetadata.isSnapshotEvent()) { + return false; + } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { + LOGGER.info("Signalling close because Snapshot is complete"); + return true; + } else { + final BsonTimestamp eventResumeTokenTimestamp = + MongoDbResumeTokenHelper.extractTimestamp(changeEventWithMetadata.eventValueAsJson()); + boolean isEventResumeTokenAfter = resumeTokenTimestamp.compareTo(eventResumeTokenTimestamp) <= 0; + if (isEventResumeTokenAfter) { + LOGGER.info("Signalling close because record's event timestamp {} is after target event timestamp {}.", + eventResumeTokenTimestamp, resumeTokenTimestamp); + } + return isEventResumeTokenAfter; + } + } + + @Override + public boolean reachedTargetPosition(final BsonTimestamp positionFromHeartbeat) { + return positionFromHeartbeat != null && positionFromHeartbeat.compareTo(resumeTokenTimestamp) >= 0; + } + + @Override + public BsonTimestamp extractPositionFromHeartbeatOffset(final Map sourceOffset) { + return ResumeTokens.getTimestamp( + ResumeTokens.fromData( + sourceOffset.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN).toString())); + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + MongoDbCdcTargetPosition that = (MongoDbCdcTargetPosition) o; + return Objects.equals(resumeTokenTimestamp, that.resumeTokenTimestamp); + } + + @Override + public int hashCode() { + return Objects.hash(resumeTokenTimestamp); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java new file mode 100644 index 000000000000..198a5f9eb781 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import io.debezium.connector.mongodb.SourceInfo; + +/** + * A collection of constants for use with the Debezium MongoDB Connector. + */ +public class MongoDbDebeziumConstants { + + /** + * Constants for Debezium Source Event data. + */ + public static class ChangeEvent { + + public static final String SOURCE = "source"; + + public static final String SOURCE_ORDER = SourceInfo.ORDER; + + public static final String SOURCE_RESUME_TOKEN = "resume_token"; + + public static final String SOURCE_SECONDS = SourceInfo.TIMESTAMP; + + public static final String SOURCE_TIMESTAMP_MS = "ts_ms"; + + } + + /** + * Constants for Debezium Offset State storage. + */ + public static class OffsetState { + + public static final String KEY_REPLICA_SET = SourceInfo.REPLICA_SET_NAME; + + public static final String KEY_SERVER_ID = SourceInfo.SERVER_ID_KEY; + + public static final String VALUE_INCREMENT = SourceInfo.ORDER; + + public static final String VALUE_RESUME_TOKEN = "resume_token"; + + public static final String VALUE_SECONDS = SourceInfo.TIMESTAMP; + + public static final String VALUE_TRANSACTION_ID = "transaction_id"; + + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java new file mode 100644 index 000000000000..5b8c6759dbca --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.client.MongoClient; +import io.airbyte.commons.json.Jsons; +import io.debezium.connector.mongodb.ResumeTokens; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.bson.BsonDocument; +import org.bson.BsonString; +import org.bson.BsonTimestamp; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of utility methods related to the Debezium offset state. + */ +public class MongoDbDebeziumStateUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbDebeziumStateUtil.class); + + /** + * Constructs the initial Debezium offset state that will be used by the incremental CDC snapshot + * after an initial snapshot sync. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server. + * @param database The database associated with the sync. + * @param replicaSet The replication set associated with the sync. + * @return The initial Debezium offset state storage document as a {@link JsonNode}. + */ + public JsonNode constructInitialDebeziumState(final MongoClient mongoClient, final String database, final String replicaSet) { + final BsonDocument resumeToken = MongoDbResumeTokenHelper.getResumeToken(mongoClient); + final String resumeTokenData = ((BsonString) ResumeTokens.getData(resumeToken)).getValue(); + final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeToken); + + final List> key = List.of( + Map.of(MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET, replicaSet, + MongoDbDebeziumConstants.OffsetState.KEY_SERVER_ID, database)); + + final Map value = new HashMap<>(); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_SECONDS, timestamp.getTime()); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_INCREMENT, timestamp.getInc()); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID, null); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN, resumeTokenData); + + final JsonNode state = Jsons.jsonNode(Map.of(key, value)); + LOGGER.info("Initial Debezium state constructed: {}", state); + return state; + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java new file mode 100644 index 000000000000..74c726beddb1 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of utility helper methods for dealing with MongoDB resume tokens. + */ +public class MongoDbResumeTokenHelper { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbResumeTokenHelper.class); + + /** + * Retrieves the most recent resume token from MongoDB server. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server. + * @return The most recent resume token value. + */ + public static BsonDocument getResumeToken(final MongoClient mongoClient) { + final ChangeStreamIterable eventStream = mongoClient.watch(BsonDocument.class); + try (final MongoChangeStreamCursor> eventStreamCursor = eventStream.cursor()) { + /* + * Must call tryNext before attempting to get the resume token from the cursor directly. Otherwise, + * the call to getResumeToken() will return null! + */ + eventStreamCursor.tryNext(); + return eventStreamCursor.getResumeToken(); + } + } + + /** + * Extracts the timestamp from a Debezium MongoDB change event. + * + * @param event The Debezium MongoDB change event as JSON. + * @return The extracted timestamp + * @throws IllegalStateException if the timestamp could not be extracted from the change event. + */ + public static BsonTimestamp extractTimestamp(final JsonNode event) { + return Optional.ofNullable(event.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE)) + .flatMap(MongoDbResumeTokenHelper::createTimestampFromSource) + .orElseThrow(() -> new IllegalStateException("Could not find timestamp")); + } + + private static Optional createTimestampFromSource(final JsonNode source) { + try { + return Optional.ofNullable( + new BsonTimestamp( + Long.valueOf(TimeUnit.MILLISECONDS.toSeconds( + source.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_TIMESTAMP_MS) + .asLong())) + .intValue(), + source.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER).asInt())); + } catch (final Exception e) { + LOGGER.warn("Unable to extract timestamp data from event source.", e); + return Optional.empty(); + } + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java index 5d70dfebeb89..3fed0293920b 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.debezium.internals.mysql; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.debezium.CdcTargetPosition; import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; @@ -102,6 +104,43 @@ public boolean isHeartbeatSupported() { return true; } + @Override + public boolean isEventAheadOffset(final Map offset, final ChangeEventWithMetadata event) { + if (offset.size() != 1) { + return false; + } + + final String eventFileName = event.eventValueAsJson().get("source").get("file").asText(); + final long eventPosition = event.eventValueAsJson().get("source").get("pos").asLong(); + + final JsonNode offsetJson = Jsons.deserialize((String) offset.values().toArray()[0]); + + final String offsetFileName = offsetJson.get("file").asText(); + final long offsetPosition = offsetJson.get("pos").asLong(); + if (eventFileName.compareTo(offsetFileName) != 0) { + return eventFileName.compareTo(offsetFileName) > 0; + } + + return eventPosition > offsetPosition; + } + + @Override + public boolean isSameOffset(final Map offsetA, final Map offsetB) { + if ((offsetA == null || offsetA.size() != 1) || (offsetB == null || offsetB.size() != 1)) { + return false; + } + + final JsonNode offsetJsonA = Jsons.deserialize((String) offsetA.values().toArray()[0]); + final String offsetAFileName = offsetJsonA.get("file").asText(); + final long offsetAPosition = offsetJsonA.get("pos").asLong(); + + final JsonNode offsetJsonB = Jsons.deserialize((String) offsetB.values().toArray()[0]); + final String offsetBFileName = offsetJsonB.get("file").asText(); + final long offsetBPosition = offsetJsonB.get("pos").asLong(); + + return offsetAFileName.equals(offsetBFileName) && offsetAPosition == offsetBPosition; + } + @Override public MySqlCdcPosition extractPositionFromHeartbeatOffset(final Map sourceOffset) { return new MySqlCdcPosition(sourceOffset.get("file").toString(), (Long) sourceOffset.get("pos")); diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java new file mode 100644 index 000000000000..08b01c187fa7 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mysql; + +import static io.debezium.relational.RelationalDatabaseConnectorConfig.DATABASE_NAME; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.integrations.debezium.internals.DebeziumRecordPublisher; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.debezium.config.Configuration; +import io.debezium.connector.common.OffsetReader; +import io.debezium.connector.mysql.GtidSet; +import io.debezium.connector.mysql.MySqlConnectorConfig; +import io.debezium.connector.mysql.MySqlOffsetContext; +import io.debezium.connector.mysql.MySqlOffsetContext.Loader; +import io.debezium.connector.mysql.MySqlPartition; +import io.debezium.engine.ChangeEvent; +import io.debezium.pipeline.spi.Offsets; +import io.debezium.pipeline.spi.Partition; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.storage.FileOffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlDebeziumStateUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlDebeziumStateUtil.class); + public static final String MYSQL_CDC_OFFSET = "mysql_cdc_offset"; + public static final String MYSQL_DB_HISTORY = "mysql_db_history"; + + public boolean savedOffsetStillPresentOnServer(final JdbcDatabase database, final MysqlDebeziumStateAttributes savedState) { + if (savedState.gtidSet().isPresent()) { + final Optional availableGtidStr = getStateAttributesFromDB(database).gtidSet(); + if (availableGtidStr.isEmpty()) { + // Last offsets had GTIDs but the server does not use them + LOGGER.info("Connector used GTIDs previously, but MySQL server does not know of any GTIDs or they are not enabled"); + return false; + } + final GtidSet gtidSetFromSavedState = new GtidSet(savedState.gtidSet().get()); + // Get the GTID set that is available in the server + final GtidSet availableGtidSet = new GtidSet(availableGtidStr.get()); + if (gtidSetFromSavedState.isContainedWithin(availableGtidSet)) { + LOGGER.info("MySQL server current GTID set {} does contain the GTID set required by the connector {}", availableGtidSet, + gtidSetFromSavedState); + final Optional gtidSetToReplicate = subtractGtidSet(availableGtidSet, gtidSetFromSavedState, database); + if (gtidSetToReplicate.isPresent()) { + final Optional purgedGtidSet = purgedGtidSet(database); + if (purgedGtidSet.isPresent()) { + LOGGER.info("MySQL server has already purged {} GTIDs", purgedGtidSet.get()); + final Optional nonPurgedGtidSetToReplicate = subtractGtidSet(gtidSetToReplicate.get(), purgedGtidSet.get(), database); + if (nonPurgedGtidSetToReplicate.isPresent()) { + LOGGER.info("GTIDs known by the MySQL server but not processed yet {}, for replication are available only {}", gtidSetToReplicate, + nonPurgedGtidSetToReplicate); + if (!gtidSetToReplicate.equals(nonPurgedGtidSetToReplicate)) { + LOGGER.info("Some of the GTIDs needed to replicate have been already purged by MySQL server"); + return false; + } + } + } + } + return true; + } + LOGGER.info("Connector last known GTIDs are {}, but MySQL server only has {}", gtidSetFromSavedState, availableGtidSet); + return false; + } + + final List existingLogFiles = getExistingLogFiles(database); + final boolean found = existingLogFiles.stream().anyMatch(savedState.binlogFilename()::equals); + if (!found) { + LOGGER.info("Connector requires binlog file '{}', but MySQL server only has {}", savedState.binlogFilename(), + String.join(", ", existingLogFiles)); + } else { + LOGGER.info("MySQL server has the binlog file '{}' required by the connector", savedState.binlogFilename()); + } + + return found; + + } + + private List getExistingLogFiles(final JdbcDatabase database) { + try (final Stream stream = database.unsafeResultSetQuery( + connection -> connection.createStatement().executeQuery("SHOW BINARY LOGS"), + resultSet -> resultSet.getString(1))) { + return stream.toList(); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + private Optional subtractGtidSet(final GtidSet set1, final GtidSet set2, final JdbcDatabase database) { + try (final Stream stream = database.unsafeResultSetQuery( + connection -> { + final PreparedStatement ps = connection.prepareStatement("SELECT GTID_SUBTRACT(?, ?)"); + ps.setString(1, set1.toString()); + ps.setString(2, set2.toString()); + return ps.executeQuery(); + }, + resultSet -> new GtidSet(resultSet.getString(1)))) { + final List gtidSets = stream.toList(); + if (gtidSets.isEmpty()) { + return Optional.empty(); + } else if (gtidSets.size() == 1) { + return Optional.of(gtidSets.get(0)); + } else { + throw new RuntimeException("Not expecting gtid set size to be greater than 1"); + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + private Optional purgedGtidSet(final JdbcDatabase database) { + try (final Stream> stream = database.unsafeResultSetQuery( + connection -> connection.createStatement().executeQuery("SELECT @@global.gtid_purged"), + resultSet -> { + if (resultSet.getMetaData().getColumnCount() > 0) { + String string = resultSet.getString(1); + if (string != null && !string.isEmpty()) { + return Optional.of(new GtidSet(string)); + } + } + return Optional.empty(); + })) { + List> gtidSet = stream.toList(); + if (gtidSet.isEmpty()) { + return Optional.empty(); + } else if (gtidSet.size() == 1) { + return gtidSet.get(0); + } else { + throw new RuntimeException("Not expecting the size to be greater than 1"); + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + public Optional savedOffset(final Properties baseProperties, + final ConfiguredAirbyteCatalog catalog, + final JsonNode cdcOffset, + final JsonNode config) { + if (Objects.isNull(cdcOffset)) { + return Optional.empty(); + } + + final DebeziumPropertiesManager debeziumPropertiesManager = new DebeziumPropertiesManager(baseProperties, config, catalog, + AirbyteFileOffsetBackingStore.initializeState(cdcOffset, Optional.empty()), + Optional.empty()); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + return parseSavedOffset(debeziumProperties); + } + + private Optional parseSavedOffset(final Properties properties) { + + FileOffsetBackingStore fileOffsetBackingStore = null; + OffsetStorageReaderImpl offsetStorageReader = null; + try { + fileOffsetBackingStore = new FileOffsetBackingStore(); + final Map propertiesMap = Configuration.from(properties).asMap(); + propertiesMap.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); + propertiesMap.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); + fileOffsetBackingStore.configure(new StandaloneConfig(propertiesMap)); + fileOffsetBackingStore.start(); + + final Map internalConverterConfig = Collections.singletonMap(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false"); + final JsonConverter keyConverter = new JsonConverter(); + keyConverter.configure(internalConverterConfig, true); + final JsonConverter valueConverter = new JsonConverter(); + valueConverter.configure(internalConverterConfig, false); + + final MySqlConnectorConfig connectorConfig = new MySqlConnectorConfig(Configuration.from(properties)); + final MySqlOffsetContext.Loader loader = new MySqlOffsetContext.Loader(connectorConfig); + final Set partitions = + Collections.singleton(new MySqlPartition(connectorConfig.getLogicalName(), properties.getProperty(DATABASE_NAME.name()))); + + offsetStorageReader = new OffsetStorageReaderImpl(fileOffsetBackingStore, properties.getProperty("name"), keyConverter, + valueConverter); + final OffsetReader offsetReader = new OffsetReader<>(offsetStorageReader, + loader); + final Map offsets = offsetReader.offsets(partitions); + + return extractStateAttributes(partitions, offsets); + + } finally { + LOGGER.info("Closing offsetStorageReader and fileOffsetBackingStore"); + if (offsetStorageReader != null) { + offsetStorageReader.close(); + } + + if (fileOffsetBackingStore != null) { + fileOffsetBackingStore.stop(); + } + } + } + + private Optional extractStateAttributes(final Set partitions, + final Map offsets) { + boolean found = false; + for (final Partition partition : partitions) { + final MySqlOffsetContext mySqlOffsetContext = offsets.get(partition); + + if (mySqlOffsetContext != null) { + found = true; + LOGGER.info("Found previous partition offset {}: {}", partition, mySqlOffsetContext.getOffset()); + } + } + + if (!found) { + LOGGER.info("No previous offsets found"); + return Optional.empty(); + } + + final Offsets of = Offsets.of(offsets); + final MySqlOffsetContext previousOffset = of.getTheOnlyOffset(); + + return Optional.of(new MysqlDebeziumStateAttributes(previousOffset.getSource().binlogFilename(), previousOffset.getSource().binlogPosition(), + Optional.ofNullable(previousOffset.gtidSet()))); + + } + + public JsonNode constructInitialDebeziumState(final Properties properties, + final ConfiguredAirbyteCatalog catalog, + final JdbcDatabase database) { + // https://debezium.io/documentation/reference/2.2/connectors/mysql.html#mysql-property-snapshot-mode + // We use the schema_only_recovery property cause using this mode will instruct Debezium to + // construct the db schema history. + properties.setProperty("snapshot.mode", "schema_only_recovery"); + final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState( + constructBinlogOffset(database, database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText()), + Optional.empty()); + final AirbyteSchemaHistoryStorage schemaHistoryStorage = AirbyteSchemaHistoryStorage.initializeDBHistory(Optional.empty()); + final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(); + try (final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(properties, database.getSourceConfig(), catalog, offsetManager, + Optional.of(schemaHistoryStorage))) { + publisher.start(queue); + while (!publisher.hasClosed()) { + final ChangeEvent event = queue.poll(10, TimeUnit.SECONDS); + if (event == null) { + continue; + } + LOGGER.info("A record is returned, closing the engine since the state is constructed"); + publisher.close(); + break; + } + } catch (final Exception e) { + throw new RuntimeException(e); + } + + final Map offset = offsetManager.read(); + final String dbHistory = schemaHistoryStorage.read(); + + assert !offset.isEmpty(); + assert Objects.nonNull(dbHistory); + + final Map state = new HashMap<>(); + state.put(MYSQL_CDC_OFFSET, offset); + state.put(MYSQL_DB_HISTORY, dbHistory); + + final JsonNode asJson = Jsons.jsonNode(state); + LOGGER.info("Initial Debezium state constructed: {}", asJson); + + return asJson; + } + + /** + * Method to construct initial Debezium state which can be passed onto Debezium engine to make it + * process binlogs from a specific file and position and skip snapshot phase + */ + private JsonNode constructBinlogOffset(final JdbcDatabase database, final String dbName) { + return format(getStateAttributesFromDB(database), dbName, Instant.now()); + } + + @VisibleForTesting + public JsonNode format(final MysqlDebeziumStateAttributes attributes, final String dbName, final Instant time) { + final String key = "[\"" + dbName + "\",{\"server\":\"" + dbName + "\"}]"; + final String gtidSet = attributes.gtidSet().isPresent() ? ",\"gtids\":\"" + attributes.gtidSet().get() + "\"" : ""; + final String value = + "{\"transaction_id\":null,\"ts_sec\":" + time.getEpochSecond() + ",\"file\":\"" + attributes.binlogFilename() + "\",\"pos\":" + + attributes.binlogPosition() + + gtidSet + "}"; + + final Map result = new HashMap<>(); + result.put(key, value); + + final JsonNode jsonNode = Jsons.jsonNode(result); + LOGGER.info("Initial Debezium state offset constructed: {}", jsonNode); + + return jsonNode; + } + + public static MysqlDebeziumStateAttributes getStateAttributesFromDB(final JdbcDatabase database) { + try (final Stream stream = database.unsafeResultSetQuery( + connection -> connection.createStatement().executeQuery("SHOW MASTER STATUS"), + resultSet -> { + final String file = resultSet.getString("File"); + final long position = resultSet.getLong("Position"); + assert file != null; + assert position >= 0; + if (resultSet.getMetaData().getColumnCount() > 4) { + // This column exists only in MySQL 5.6.5 or later ... + final String gtidSet = resultSet.getString(5); // GTID set, may be null, blank, or contain a GTID set + return new MysqlDebeziumStateAttributes(file, position, removeNewLineChars(gtidSet)); + } + return new MysqlDebeziumStateAttributes(file, position, Optional.empty()); + })) { + final List stateAttributes = stream.toList(); + assert stateAttributes.size() == 1; + return stateAttributes.get(0); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + private static Optional removeNewLineChars(final String gtidSet) { + if (gtidSet != null && !gtidSet.trim().isEmpty()) { + // Remove all the newline chars that exist in the GTID set string ... + return Optional.of(gtidSet.replace("\n", "").replace("\r", "")); + } + + return Optional.empty(); + } + + public record MysqlDebeziumStateAttributes(String binlogFilename, long binlogPosition, Optional gtidSet) { + + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java index 1d47fa18198b..d8a672fac1d1 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java @@ -94,7 +94,7 @@ public Long extractPositionFromHeartbeatOffset(final Map sourceOffset } @Override - public boolean isRecordBehindOffset(final Map offset, final ChangeEventWithMetadata event) { + public boolean isEventAheadOffset(final Map offset, final ChangeEventWithMetadata event) { if (offset.size() != 1) { return false; } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java index 43c2ee1e2730..2121fac0ea48 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java +++ b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java @@ -9,6 +9,11 @@ import static io.debezium.relational.RelationalDatabaseConnectorConfig.DATABASE_NAME; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.PostgresUtils; +import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.debezium.internals.AirbyteFileOffsetBackingStore; import io.airbyte.integrations.debezium.internals.DebeziumPropertiesManager; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -21,8 +26,12 @@ import io.debezium.connector.postgresql.connection.Lsn; import io.debezium.pipeline.spi.Offsets; import io.debezium.pipeline.spi.Partition; +import io.debezium.time.Conversions; import java.sql.SQLException; +import java.time.Instant; import java.util.Collections; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -224,4 +233,43 @@ private OptionalLong extractLsn(final Set partitions, } + /** + * Method to construct initial Debezium state which can be passed onto Debezium engine to make it + * process WAL from a specific LSN and skip snapshot phase + */ + public JsonNode constructInitialDebeziumState(final JdbcDatabase database, final String dbName) { + try { + return format(currentXLogLocation(database), currentTransactionId(database), dbName, Instant.now()); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @VisibleForTesting + public JsonNode format(final Long currentXLogLocation, final Long currentTransactionId, final String dbName, final Instant time) { + final String key = "[\"" + dbName + "\",{\"server\":\"" + dbName + "\"}]"; + final String value = + "{\"transaction_id\":null,\"lsn\":" + currentXLogLocation + ",\"txId\":" + currentTransactionId + ",\"ts_usec\":" + Conversions.toEpochMicros( + time) + "}"; + + final Map result = new HashMap<>(); + result.put(key, value); + + final JsonNode jsonNode = Jsons.jsonNode(result); + LOGGER.info("Initial Debezium state constructed: {}", jsonNode); + + return jsonNode; + } + + private long currentXLogLocation(JdbcDatabase database) throws SQLException { + return PostgresUtils.getLsn(database).asLong(); + } + + private Long currentTransactionId(final JdbcDatabase database) throws SQLException { + final List transactionId = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery("select * from txid_current()"), + resultSet -> resultSet.getLong(1)); + Preconditions.checkState(transactionId.size() == 1); + return transactionId.get(0); + } + } diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java new file mode 100644 index 000000000000..3cddad443ba6 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; +import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.SyncMode; +import java.sql.SQLException; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MySQLContainer; + +public class MysqlDebeziumStateUtilTest { + + private static final String DB_NAME = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); + private static final String TABLE_NAME = Strings.addRandomSuffix("table", "_", 10).toLowerCase(); + private static final Properties MYSQL_PROPERTIES = new Properties(); + private static final String DB_CREATE_QUERY = "CREATE DATABASE " + DB_NAME; + private static final String TABLE_CREATE_QUERY = "CREATE TABLE " + DB_NAME + "." + TABLE_NAME + " (id INTEGER, name VARCHAR(200), PRIMARY KEY(id))"; + private static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + TABLE_NAME, + DB_NAME, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("string", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); + protected static final ConfiguredAirbyteCatalog CONFIGURED_CATALOG = CatalogHelpers.toDefaultConfiguredCatalog(CATALOG); + + static { + CONFIGURED_CATALOG.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); + MYSQL_PROPERTIES.setProperty("connector.class", "io.debezium.connector.mysql.MySqlConnector"); + MYSQL_PROPERTIES.setProperty("database.server.id", "5000"); + } + + @Test + public void debeziumInitialStateConstructTest() throws SQLException { + try (final MySQLContainer container = new MySQLContainer<>("mysql:8.0")) { + container.start(); + initDB(container); + final JdbcDatabase database = getJdbcDatabase(container); + final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); + final JsonNode debeziumState = mySqlDebeziumStateUtil.constructInitialDebeziumState(MYSQL_PROPERTIES, CONFIGURED_CATALOG, database); + Assertions.assertEquals(2, Jsons.object(debeziumState, Map.class).size()); + Assertions.assertTrue(debeziumState.has("mysql_db_history")); + Assertions.assertNotNull(debeziumState.get("mysql_db_history")); + Assertions.assertTrue(debeziumState.has("mysql_cdc_offset")); + final Map mysqlCdcOffset = Jsons.object(debeziumState.get("mysql_cdc_offset"), Map.class); + Assertions.assertEquals(1, mysqlCdcOffset.size()); + Assertions.assertTrue(mysqlCdcOffset.containsKey("[\"" + DB_NAME + "\",{\"server\":\"" + DB_NAME + "\"}]")); + Assertions.assertNotNull(mysqlCdcOffset.get("[\"" + DB_NAME + "\",{\"server\":\"" + DB_NAME + "\"}]")); + + final Optional parsedOffset = mySqlDebeziumStateUtil.savedOffset(MYSQL_PROPERTIES, CONFIGURED_CATALOG, + debeziumState.get("mysql_cdc_offset"), database.getSourceConfig()); + Assertions.assertTrue(parsedOffset.isPresent()); + Assertions.assertNotNull(parsedOffset.get().binlogFilename()); + Assertions.assertTrue(parsedOffset.get().binlogPosition() > 0); + Assertions.assertTrue(parsedOffset.get().gtidSet().isEmpty()); + container.stop(); + } + } + + @Test + public void formatTestWithGtid() { + final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); + final JsonNode debeziumState = mySqlDebeziumStateUtil.format(new MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes("binlog.000002", 633, + Optional.of("3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5")), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + final Map stateAsMap = Jsons.object(debeziumState, Map.class); + Assertions.assertEquals(1, stateAsMap.size()); + Assertions.assertTrue(stateAsMap.containsKey("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); + Assertions.assertEquals( + "{\"transaction_id\":null,\"ts_sec\":1686040570,\"file\":\"binlog.000002\",\"pos\":633,\"gtids\":\"3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5\"}", + stateAsMap.get("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); + + final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, "host") + .put(JdbcUtils.PORT_KEY, "5432") + .put(JdbcUtils.DATABASE_KEY, "db_fgnfxvllud") + .put(JdbcUtils.USERNAME_KEY, "username") + .put(JdbcUtils.PASSWORD_KEY, "password") + .put(JdbcUtils.SSL_KEY, false) + .build()); + + final Optional parsedOffset = mySqlDebeziumStateUtil.savedOffset(MYSQL_PROPERTIES, CONFIGURED_CATALOG, + debeziumState, config); + Assertions.assertTrue(parsedOffset.isPresent()); + final JsonNode stateGeneratedUsingParsedOffset = + mySqlDebeziumStateUtil.format(parsedOffset.get(), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + Assertions.assertEquals(debeziumState, stateGeneratedUsingParsedOffset); + } + + @Test + public void formatTestWithoutGtid() { + final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); + final JsonNode debeziumState = mySqlDebeziumStateUtil.format(new MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes("binlog.000002", 633, + Optional.empty()), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + final Map stateAsMap = Jsons.object(debeziumState, Map.class); + Assertions.assertEquals(1, stateAsMap.size()); + Assertions.assertTrue(stateAsMap.containsKey("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); + Assertions.assertEquals("{\"transaction_id\":null,\"ts_sec\":1686040570,\"file\":\"binlog.000002\",\"pos\":633}", + stateAsMap.get("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); + + final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, "host") + .put(JdbcUtils.PORT_KEY, "5432") + .put(JdbcUtils.DATABASE_KEY, "db_fgnfxvllud") + .put(JdbcUtils.USERNAME_KEY, "username") + .put(JdbcUtils.PASSWORD_KEY, "password") + .put(JdbcUtils.SSL_KEY, false) + .build()); + + final Optional parsedOffset = mySqlDebeziumStateUtil.savedOffset(MYSQL_PROPERTIES, CONFIGURED_CATALOG, + debeziumState, config); + Assertions.assertTrue(parsedOffset.isPresent()); + final JsonNode stateGeneratedUsingParsedOffset = + mySqlDebeziumStateUtil.format(parsedOffset.get(), "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + Assertions.assertEquals(debeziumState, stateGeneratedUsingParsedOffset); + } + + private JdbcDatabase getJdbcDatabase(final MySQLContainer container) { + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + "root", + "test", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format(DatabaseDriver.MYSQL.getUrlFormatString(), + container.getHost(), + container.getFirstMappedPort(), + DB_NAME))); + database.setSourceConfig(getSourceConfig(container)); + return database; + } + + private void initDB(final MySQLContainer container) throws SQLException { + final Database db = new Database(DSLContextFactory.create( + "root", + "test", + DatabaseDriver.MYSQL.getDriverClassName(), + String.format("jdbc:mysql://%s:%s", + container.getHost(), + container.getFirstMappedPort()), + SQLDialect.MYSQL)); + db.query(ctx -> ctx.execute(DB_CREATE_QUERY)); + db.query(ctx -> ctx.execute(TABLE_CREATE_QUERY)); + } + + private JsonNode getSourceConfig(final MySQLContainer container) { + final Map config = new HashMap<>(); + config.put(JdbcUtils.USERNAME_KEY, "root"); + config.put(JdbcUtils.PASSWORD_KEY, "test"); + config.put(JdbcUtils.HOST_KEY, container.getHost()); + config.put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()); + config.put(JdbcUtils.DATABASE_KEY, DB_NAME); + return Jsons.jsonNode(config); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java index 713f1f9fb95c..ec3b90bde596 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java +++ b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java @@ -20,6 +20,7 @@ import io.airbyte.test.utils.PostgreSQLContainerHelper; import io.debezium.connector.postgresql.connection.Lsn; import java.sql.SQLException; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.OptionalLong; @@ -207,4 +208,51 @@ private JsonNode getReplicationSlot(final JdbcDatabase database, String slotName } } + @Test + public void formatTest() { + final PostgresDebeziumStateUtil postgresDebeziumStateUtil = new PostgresDebeziumStateUtil(); + final JsonNode debeziumState = postgresDebeziumStateUtil.format(23904232L, 506L, "db_fgnfxvllud", Instant.parse("2023-06-06T08:36:10.341842Z")); + final Map stateAsMap = Jsons.object(debeziumState, Map.class); + Assertions.assertEquals(1, stateAsMap.size()); + Assertions.assertTrue(stateAsMap.containsKey("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); + Assertions.assertEquals("{\"transaction_id\":null,\"lsn\":23904232,\"txId\":506,\"ts_usec\":1686040570341842}", + stateAsMap.get("[\"db_fgnfxvllud\",{\"server\":\"db_fgnfxvllud\"}]")); + + } + + @Test + public void debeziumInitialStateConstructTest() { + final DockerImageName myImage = DockerImageName.parse("debezium/postgres:13-alpine").asCompatibleSubstituteFor("postgres"); + final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); + try (final PostgreSQLContainer container = new PostgreSQLContainer<>(myImage)) { + container.start(); + + final String initScriptName = "init_" + dbName.concat(".sql"); + final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); + PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), container); + + final Map databaseConfig = Map.of(JdbcUtils.USERNAME_KEY, container.getUsername(), + JdbcUtils.PASSWORD_KEY, container.getPassword(), + JdbcUtils.JDBC_URL_KEY, String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + container.getHost(), + container.getFirstMappedPort(), + dbName)); + + final JdbcDatabase database = new DefaultJdbcDatabase( + DataSourceFactory.create( + databaseConfig.get(JdbcUtils.USERNAME_KEY), + databaseConfig.get(JdbcUtils.PASSWORD_KEY), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + databaseConfig.get(JdbcUtils.JDBC_URL_KEY))); + + final PostgresDebeziumStateUtil postgresDebeziumStateUtil = new PostgresDebeziumStateUtil(); + final JsonNode debeziumState = postgresDebeziumStateUtil.constructInitialDebeziumState(database, dbName); + final Map stateAsMap = Jsons.object(debeziumState, Map.class); + Assertions.assertEquals(1, stateAsMap.size()); + Assertions.assertTrue(stateAsMap.containsKey("[\"" + dbName + "\",{\"server\":\"" + dbName + "\"}]")); + Assertions.assertNotNull(stateAsMap.get("[\"" + dbName + "\",{\"server\":\"" + dbName + "\"}]")); + container.stop(); + } + } + } diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java new file mode 100644 index 000000000000..b6a3346bde7a --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import static com.mongodb.assertions.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; +import io.debezium.connector.mongodb.ResumeTokens; +import io.debezium.engine.ChangeEvent; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.Test; + +class MongoDbCdcTargetPositionTest { + + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + + @Test + void testCreateTargetPosition() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + assertNotNull(targetPosition); + assertEquals(ResumeTokens.getTimestamp(resumeTokenDocument), targetPosition.getResumeTokenTimestamp()); + } + + @Test + void testReachedTargetPosition() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + + when(changeEvent.value()).thenReturn(changeEventJson.replaceAll("\"ts_ms\"\\: \\d+,", "\"ts_ms\": 1590221043000,")); + final ChangeEventWithMetadata changeEventWithMetadata2 = new ChangeEventWithMetadata(changeEvent); + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata2)); + } + + @Test + void testReachedTargetPositionSnapshotEvent() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + } + + @Test + void testReachedTargetPositionSnapshotLastEvent() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot_last.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + } + + @Test + void testReachedTargetPositionFromHeartbeat() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + final BsonTimestamp heartbeatTimestamp = new BsonTimestamp( + Long.valueOf(ResumeTokens.getTimestamp(resumeTokenDocument).getTime() + TimeUnit.HOURS.toSeconds(1)).intValue(), + 0); + + assertTrue(targetPosition.reachedTargetPosition(heartbeatTimestamp)); + assertFalse(targetPosition.reachedTargetPosition((BsonTimestamp) null)); + } + + @Test + void testIsHeartbeatSupported() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + + assertTrue(targetPosition.isHeartbeatSupported()); + } + + @Test + void testExtractPositionFromHeartbeatOffset() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final BsonTimestamp resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeTokenDocument); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); + + final Map sourceOffset = Map.of(MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS, resumeTokenTimestamp.getTime(), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, resumeTokenTimestamp.getInc(), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN, RESUME_TOKEN); + + final BsonTimestamp timestamp = targetPosition.extractPositionFromHeartbeatOffset(sourceOffset); + assertEquals(resumeTokenTimestamp, timestamp); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java new file mode 100644 index 000000000000..642a6e0ad601 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.debezium.connector.mongodb.ResumeTokens; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbDebeziumStateUtilTest { + + private MongoDbDebeziumStateUtil mongoDbDebeziumStateUtil; + + @BeforeEach + void setup() { + mongoDbDebeziumStateUtil = new MongoDbDebeziumStateUtil(); + } + + @Test + void testConstructInitialDebeziumState() { + final String database = "test"; + final String replicaSet = "test_rs"; + final String resumeToken = "8264BEB9F3000000012B0229296E04"; + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final JsonNode initialState = mongoDbDebeziumStateUtil.constructInitialDebeziumState(mongoClient, + database, replicaSet); + + assertNotNull(initialState); + assertEquals(1, initialState.size()); + final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeTokenDocument); + final JsonNode offsetState = initialState.fields().next().getValue(); + assertEquals(resumeToken, offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN).asText()); + assertEquals(timestamp.getTime(), offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_SECONDS).asInt()); + assertEquals(timestamp.getInc(), offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_INCREMENT).asInt()); + assertEquals("null", offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID).asText()); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java new file mode 100644 index 000000000000..d2a3e10fd455 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.debezium.internals.mongodb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.debezium.connector.mongodb.ResumeTokens; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.Test; + +class MongoDbResumeTokenHelperTest { + + @Test + void testRetrievingResumeToken() { + final String resumeToken = "8264BEB9F3000000012B0229296E04"; + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getResumeToken(mongoClient); + assertEquals(resumeTokenDocument, actualResumeToken); + } + + @Test + void testTimestampExtraction() throws IOException { + final int timestampSec = Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(1692651270000L)).intValue(); + final BsonTimestamp expectedTimestamp = new BsonTimestamp(timestampSec, 2); + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final JsonNode changeEvent = Jsons.deserialize(changeEventJson); + + final BsonTimestamp timestamp = MongoDbResumeTokenHelper.extractTimestamp(changeEvent); + assertNotNull(timestamp); + assertEquals(expectedTimestamp, timestamp); + } + + @Test + void testTimestampExtractionSourceNotPresent() { + final JsonNode changeEvent = Jsons.deserialize("{}"); + assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestamp(changeEvent)); + } + + @Test + void testTimestampExtractionTimestampNotPresent() { + final JsonNode changeEvent = Jsons.deserialize("{\"source\":{}}"); + assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestamp(changeEvent)); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event.json b/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event.json new file mode 100644 index 000000000000..9c4470daed49 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event.json @@ -0,0 +1,23 @@ +{ + "before": null, + "after": null, + "updateDescription": null, + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "test_db", + "ts_ms": 1692651270000, + "snapshot": "false", + "db": "test_db", + "sequence": null, + "rs": "atlas-abcdef-shard-0", + "collection": "test_collection", + "ord": 2, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "r", + "ts_ms": 1692651277722, + "transaction": null +} diff --git a/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot.json b/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot.json new file mode 100644 index 000000000000..f97acf330542 --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot.json @@ -0,0 +1,23 @@ +{ + "before": null, + "after": null, + "updateDescription": null, + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "test_db", + "ts_ms": 1692651270000, + "snapshot": "true", + "db": "test_db", + "sequence": null, + "rs": "atlas-abcdef-shard-0", + "collection": "test_collection", + "ord": 2, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "r", + "ts_ms": 1692651277722, + "transaction": null +} diff --git a/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot_last.json b/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot_last.json new file mode 100644 index 000000000000..40419efb485b --- /dev/null +++ b/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot_last.json @@ -0,0 +1,23 @@ +{ + "before": null, + "after": null, + "updateDescription": null, + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "test_db", + "ts_ms": 1692651270000, + "snapshot": "last", + "db": "test_db", + "sequence": null, + "rs": "atlas-abcdef-shard-0", + "collection": "test_collection", + "ord": 2, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "r", + "ts_ms": 1692651277722, + "transaction": null +} diff --git a/airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java b/airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java index 02d2045e48c3..281aeee5924b 100644 --- a/airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java +++ b/airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java @@ -48,6 +48,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -59,7 +60,7 @@ public abstract class CdcSourceTest { protected static final String MODELS_SCHEMA = "models_schema"; protected static final String MODELS_STREAM_NAME = "models"; - private static final Set STREAM_NAMES = Sets + protected static final Set STREAM_NAMES = Sets .newHashSet(MODELS_STREAM_NAME); protected static final String COL_ID = "id"; protected static final String COL_MAKE_ID = "make_id"; @@ -305,6 +306,8 @@ protected void assertExpectedRecords(final Set expectedRecords, assertEquals(expectedRecords, actualData); } + // Failing on `source-postgres`, possibly others as well. + @Disabled("The 'testExistingData()' test is flaky. https://github.com/airbytehq/airbyte/issues/29411") @Test @DisplayName("On the first sync, produce returns records that exist in the database.") void testExistingData() throws Exception { @@ -321,8 +324,6 @@ void testExistingData() throws Exception { }); assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages); - assertEquals(1, stateMessages.size()); - assertNotNull(stateMessages.get(0).getData()); assertExpectedStateMessages(stateMessages); } @@ -333,29 +334,29 @@ void testDelete() throws Exception { .read(getConfig(), CONFIGURED_CATALOG, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final List stateMessages1 = extractStateMessages(actualRecords1); - assertEquals(1, stateMessages1.size()); - assertNotNull(stateMessages1.get(0).getData()); assertExpectedStateMessages(stateMessages1); executeQuery(String .format("DELETE FROM %s.%s WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_ID, 11)); - final JsonNode state = Jsons.jsonNode(stateMessages1); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final List recordMessages2 = new ArrayList<>( extractRecordMessages(actualRecords2)); final List stateMessages2 = extractStateMessages(actualRecords2); - assertEquals(1, stateMessages2.size()); - assertNotNull(stateMessages2.get(0).getData()); - assertExpectedStateMessages(stateMessages2); + assertExpectedStateMessagesFromIncrementalSync(stateMessages2); assertEquals(1, recordMessages2.size()); assertEquals(11, recordMessages2.get(0).getData().get(COL_ID).asInt()); assertCdcMetaData(recordMessages2.get(0).getData(), false); } + protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { + assertExpectedStateMessages(stateMessages); + } + @Test @DisplayName("When a record is updated, produces an update record.") void testUpdate() throws Exception { @@ -364,24 +365,20 @@ void testUpdate() throws Exception { .read(getConfig(), CONFIGURED_CATALOG, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final List stateMessages1 = extractStateMessages(actualRecords1); - assertEquals(1, stateMessages1.size()); - assertNotNull(stateMessages1.get(0).getData()); assertExpectedStateMessages(stateMessages1); executeQuery(String .format("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_MODEL, updatedModel, COL_ID, 11)); - final JsonNode state = Jsons.jsonNode(stateMessages1); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final List recordMessages2 = new ArrayList<>( extractRecordMessages(actualRecords2)); final List stateMessages2 = extractStateMessages(actualRecords2); - assertEquals(1, stateMessages2.size()); - assertNotNull(stateMessages2.get(0).getData()); - assertExpectedStateMessages(stateMessages2); + assertExpectedStateMessagesFromIncrementalSync(stateMessages2); assertEquals(1, recordMessages2.size()); assertEquals(11, recordMessages2.get(0).getData().get(COL_ID).asInt()); assertEquals(updatedModel, recordMessages2.get(0).getData().get(COL_MODEL).asText()); @@ -408,9 +405,7 @@ protected void testRecordsProducedDuringAndAfterSync() throws Exception { final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); - assertEquals(1, stateAfterFirstBatch.size()); - assertNotNull(stateAfterFirstBatch.get(0).getData()); - assertExpectedStateMessages(stateAfterFirstBatch); + assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(stateAfterFirstBatch); final Set recordsFromFirstBatch = extractRecordMessages( dataFromFirstBatch); assertEquals((MODEL_RECORDS.size() + recordsToCreate), recordsFromFirstBatch.size()); @@ -424,16 +419,14 @@ protected void testRecordsProducedDuringAndAfterSync() throws Exception { writeModelRecord(record); } - final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); final AutoCloseableIterator secondBatchIterator = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); - assertEquals(1, stateAfterSecondBatch.size()); - assertNotNull(stateAfterSecondBatch.get(0).getData()); - assertExpectedStateMessages(stateAfterSecondBatch); + assertExpectedStateMessagesFromIncrementalSync(stateAfterSecondBatch); final Set recordsFromSecondBatch = extractRecordMessages( dataFromSecondBatch); @@ -455,6 +448,10 @@ protected void testRecordsProducedDuringAndAfterSync() throws Exception { .size()); } + protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(final List stateAfterFirstBatch) { + assertExpectedStateMessages(stateAfterFirstBatch); + } + @Test @DisplayName("When both incremental CDC and full refresh are configured for different streams in a sync, the data is replicated as expected.") void testCdcAndFullRefreshInSameSync() throws Exception { @@ -500,8 +497,6 @@ void testCdcAndFullRefreshInSameSync() throws Exception { final List stateMessages1 = extractStateMessages(actualRecords1); final HashSet names = new HashSet<>(STREAM_NAMES); names.add(MODELS_STREAM_NAME + "_2"); - assertEquals(1, stateMessages1.size()); - assertNotNull(stateMessages1.get(0).getData()); assertExpectedStateMessages(stateMessages1); assertExpectedRecords(Streams.concat(MODEL_RECORDS_2.stream(), MODEL_RECORDS.stream()) .collect(Collectors.toSet()), @@ -514,16 +509,14 @@ void testCdcAndFullRefreshInSameSync() throws Exception { .jsonNode(ImmutableMap.of(COL_ID, 100, COL_MAKE_ID, 3, COL_MODEL, "Punto")); writeModelRecord(puntoRecord); - final JsonNode state = Jsons.jsonNode(extractStateMessages(actualRecords1)); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); final AutoCloseableIterator read2 = getSource() .read(getConfig(), configuredCatalog, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final Set recordMessages2 = extractRecordMessages(actualRecords2); final List stateMessages2 = extractStateMessages(actualRecords2); - assertEquals(1, stateMessages2.size()); - assertNotNull(stateMessages2.get(0).getData()); - assertExpectedStateMessages(stateMessages2); + assertExpectedStateMessagesFromIncrementalSync(stateMessages2); assertExpectedRecords( Streams.concat(MODEL_RECORDS_2.stream(), Stream.of(puntoRecord)) .collect(Collectors.toSet()), @@ -545,10 +538,11 @@ void testNoData() throws Exception { final Set recordMessages = extractRecordMessages(actualRecords); final List stateMessages = extractStateMessages(actualRecords); - assertExpectedRecords(Collections.emptySet(), recordMessages); - assertEquals(1, stateMessages.size()); - assertNotNull(stateMessages.get(0).getData()); + assertExpectedStateMessagesForNoData(stateMessages); + } + + protected void assertExpectedStateMessagesForNoData(final List stateMessages) { assertExpectedStateMessages(stateMessages); } @@ -558,7 +552,8 @@ void testNoDataOnSecondSync() throws Exception { final AutoCloseableIterator read1 = getSource() .read(getConfig(), CONFIGURED_CATALOG, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); - final JsonNode state = Jsons.jsonNode(extractStateMessages(actualRecords1)); + final List stateMessagesFromFirstSync = extractStateMessages(actualRecords1); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessagesFromFirstSync.get(stateMessagesFromFirstSync.size() - 1))); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); @@ -568,9 +563,7 @@ void testNoDataOnSecondSync() throws Exception { final List stateMessages2 = extractStateMessages(actualRecords2); assertExpectedRecords(Collections.emptySet(), recordMessages2); - assertEquals(1, stateMessages2.size()); - assertNotNull(stateMessages2.get(0).getData()); - assertExpectedStateMessages(stateMessages2); + assertExpectedStateMessagesFromIncrementalSync(stateMessages2); } @Test @@ -600,9 +593,9 @@ public void newTableSnapshotTest() throws Exception { final Set recordsFromFirstBatch = extractRecordMessages( dataFromFirstBatch); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); - assertEquals(1, stateAfterFirstBatch.size()); + assertExpectedStateMessages(stateAfterFirstBatch); - final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion = stateAfterFirstBatch.get(0); + final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion = stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1); assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterFirstSyncCompletion.getType()); assertNotNull(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState()); final Set streamsInStateAfterFirstSyncCompletion = stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getStreamStates() @@ -616,7 +609,7 @@ public void newTableSnapshotTest() throws Exception { assertEquals((MODEL_RECORDS.size()), recordsFromFirstBatch.size()); assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordsFromFirstBatch); - final JsonNode state = stateAfterFirstBatch.get(0).getData(); + final JsonNode state = stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1).getData(); final ConfiguredAirbyteCatalog newTables = CatalogHelpers .toDefaultConfiguredCatalog(new AirbyteCatalog().withStreams(List.of( @@ -655,36 +648,7 @@ public void newTableSnapshotTest() throws Exception { .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); - assertEquals(2, stateAfterSecondBatch.size()); - - final AirbyteStateMessage stateMessageEmittedAfterSnapshotCompletionInSecondSync = stateAfterSecondBatch.get(0); - assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSnapshotCompletionInSecondSync.getType()); - assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), - stateMessageEmittedAfterSnapshotCompletionInSecondSync.getGlobal().getSharedState()); - final Set streamsInSnapshotState = stateMessageEmittedAfterSnapshotCompletionInSecondSync.getGlobal().getStreamStates() - .stream() - .map(AirbyteStreamState::getStreamDescriptor) - .collect(Collectors.toSet()); - assertEquals(2, streamsInSnapshotState.size()); - assertTrue( - streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); - assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); - assertNotNull(stateMessageEmittedAfterSnapshotCompletionInSecondSync.getData()); - - final AirbyteStateMessage stateMessageEmittedAfterSecondSyncCompletion = stateAfterSecondBatch.get(1); - assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSecondSyncCompletion.getType()); - assertNotEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), - stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); - final Set streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() - .stream() - .map(AirbyteStreamState::getStreamDescriptor) - .collect(Collectors.toSet()); - assertEquals(2, streamsInSnapshotState.size()); - assertTrue( - streamsInSyncCompletionState.contains( - new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); - assertTrue(streamsInSyncCompletionState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); - assertNotNull(stateMessageEmittedAfterSecondSyncCompletion.getData()); + assertStateMessagesForNewTableSnapshotTest(stateAfterSecondBatch, stateMessageEmittedAfterFirstSyncCompletion); final Map> recordsStreamWise = extractRecordMessagesStreamWise(dataFromSecondBatch); assertTrue(recordsStreamWise.containsKey(MODELS_STREAM_NAME)); @@ -725,19 +689,19 @@ public void newTableSnapshotTest() throws Exception { recordsWrittenInRandomTable.add(record2); } - final JsonNode state2 = stateAfterSecondBatch.get(1).getData(); + final JsonNode state2 = stateAfterSecondBatch.get(stateAfterSecondBatch.size() - 1).getData(); final AutoCloseableIterator thirdBatchIterator = getSource() .read(getConfig(), updatedCatalog, state2); final List dataFromThirdBatch = AutoCloseableIterators .toListAndClose(thirdBatchIterator); final List stateAfterThirdBatch = extractStateMessages(dataFromThirdBatch); - assertEquals(1, stateAfterThirdBatch.size()); + assertTrue(stateAfterThirdBatch.size() >= 1); - final AirbyteStateMessage stateMessageEmittedAfterThirdSyncCompletion = stateAfterThirdBatch.get(0); + final AirbyteStateMessage stateMessageEmittedAfterThirdSyncCompletion = stateAfterThirdBatch.get(stateAfterThirdBatch.size() - 1); assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterThirdSyncCompletion.getType()); assertNotEquals(stateMessageEmittedAfterThirdSyncCompletion.getGlobal().getSharedState(), - stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); + stateAfterSecondBatch.get(stateAfterSecondBatch.size() - 1).getGlobal().getSharedState()); final Set streamsInSyncCompletionStateAfterThirdSync = stateMessageEmittedAfterThirdSyncCompletion.getGlobal().getStreamStates() .stream() .map(AirbyteStreamState::getStreamDescriptor) @@ -766,6 +730,39 @@ public void newTableSnapshotTest() throws Exception { randomTableSchema()); } + protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, + final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion) { + assertEquals(2, stateMessages.size()); + final AirbyteStateMessage stateMessageEmittedAfterSnapshotCompletionInSecondSync = stateMessages.get(0); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSnapshotCompletionInSecondSync.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessageEmittedAfterSnapshotCompletionInSecondSync.getGlobal().getSharedState()); + final Set streamsInSnapshotState = stateMessageEmittedAfterSnapshotCompletionInSecondSync.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + assertNotNull(stateMessageEmittedAfterSnapshotCompletionInSecondSync.getData()); + + final AirbyteStateMessage stateMessageEmittedAfterSecondSyncCompletion = stateMessages.get(1); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSecondSyncCompletion.getType()); + assertNotEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); + final Set streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSyncCompletionState.contains( + new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSyncCompletionState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + assertNotNull(stateMessageEmittedAfterSecondSyncCompletion.getData()); + } + protected AirbyteCatalog expectedCatalogForDiscover() { final AirbyteCatalog expectedCatalog = Jsons.clone(CATALOG); @@ -776,6 +773,7 @@ protected AirbyteCatalog expectedCatalogForDiscover() { // stream with PK streams.get(0).setSourceDefinedCursor(true); addCdcMetadataColumns(streams.get(0)); + addCdcDefaultCursorField(streams.get(0)); final AirbyteStream streamWithoutPK = CatalogHelpers.createAirbyteStream( MODELS_STREAM_NAME + "_2", @@ -785,6 +783,7 @@ protected AirbyteCatalog expectedCatalogForDiscover() { Field.of(COL_MODEL, JsonSchemaType.STRING)); streamWithoutPK.setSourceDefinedPrimaryKey(Collections.emptyList()); streamWithoutPK.setSupportedSyncModes(List.of(SyncMode.FULL_REFRESH)); + addCdcDefaultCursorField(streamWithoutPK); addCdcMetadataColumns(streamWithoutPK); final AirbyteStream randomStream = CatalogHelpers.createAirbyteStream( @@ -796,6 +795,8 @@ protected AirbyteCatalog expectedCatalogForDiscover() { .withSourceDefinedCursor(true) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID + "_random"))); + + addCdcDefaultCursorField(randomStream); addCdcMetadataColumns(randomStream); streams.add(streamWithoutPK); @@ -821,6 +822,8 @@ protected AirbyteCatalog expectedCatalogForDiscover() { protected abstract void addCdcMetadataColumns(final AirbyteStream stream); + protected abstract void addCdcDefaultCursorField(final AirbyteStream stream); + protected abstract Source getSource(); protected abstract JsonNode getConfig(); diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java b/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java index 869a539e9624..fe19b22414c4 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java @@ -22,6 +22,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Path; import java.util.Comparator; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -125,7 +126,7 @@ protected List getAllSyncedObjects(final String streamName, fin *

  • Construct the S3 client.
    • */ @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { final JsonNode baseConfigJson = getBaseConfigJson(); // Set a random s3 bucket path for each integration test final JsonNode configJson = Jsons.clone(baseConfigJson); diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index 6e2857b30ed1..5a7425710a03 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -24,7 +24,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.configoss.JobGetSpecConfig; import io.airbyte.configoss.OperatorDbt; @@ -100,6 +99,8 @@ public abstract class DestinationAcceptanceTest { + protected static final HashSet TEST_SCHEMAS = new HashSet<>(); + private static final Random RANDOM = new Random(); private static final String NORMALIZATION_VERSION = "dev"; @@ -210,7 +211,9 @@ protected String getDefaultSchema(final JsonNode config) throws Exception { if (config.get("schema") == null) { return null; } - return config.get("schema").asText(); + final String schema = config.get("schema").asText(); + TEST_SCHEMAS.add(schema); + return schema; } /** @@ -294,14 +297,6 @@ protected boolean implementsOverwrite() throws TestHarnessException { } } - /** - * Override to return true if a destination implements size limits on record size (then destination - * should redefine getMaxRecordValueLimit() too) - */ - protected boolean implementsRecordSizeLimitChecks() { - return false; - } - /** * Same idea as {@link #retrieveRecords(TestDestinationEnv, String, String, JsonNode)}. Except this * method should pull records from the table that contains the normalized records and convert them @@ -327,9 +322,10 @@ protected List retrieveNormalizedRecords(final TestDestinationEnv test * postgres database. This function will be called before EACH test. * * @param testEnv - information about the test environment. + * @param TEST_SCHEMAS * @throws Exception - can throw any exception, test framework will handle. */ - protected abstract void setup(TestDestinationEnv testEnv) throws Exception; + protected abstract void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception; /** * Function that performs any clean up of external resources required for the test. e.g. delete a @@ -362,7 +358,7 @@ void setUpInternal() throws Exception { testEnv = new TestDestinationEnv(localRoot); mConnectorConfigUpdater = Mockito.mock(ConnectorConfigUpdater.class); - setup(testEnv); + setup(testEnv, TEST_SCHEMAS); processFactory = new DockerProcessFactory( workspaceRoot, @@ -838,66 +834,6 @@ public void testIncrementalDedupeSync() throws Exception { assertSameMessages(expectedMessages, actualMessages, true); } - /** - * This test is running a sync using the exchange rate catalog and messages. However it also - * generates and adds two extra messages with big records (near the destination limit as defined by - * getMaxValueLengthLimit() - *

    - * The first big message should be small enough to fit into the destination while the second message - * would be too big and fails to replicate. - */ - @Test - void testSyncVeryBigRecords() throws Exception { - if (!implementsRecordSizeLimitChecks()) { - return; - } - - final AirbyteCatalog catalog = - Jsons.deserialize( - MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), - AirbyteCatalog.class); - final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( - catalog); - final List messages = MoreResources.readResource( - DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() - .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) - .collect(Collectors.toList()); - // Add a big message that barely fits into the limits of the destination - messages.add(new AirbyteMessage() - .withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream(catalog.getStreams().get(0).getName()) - .withEmittedAt(Instant.now().toEpochMilli()) - .withData(Jsons.jsonNode(ImmutableMap.builder() - .put("id", 3) - // remove enough characters from max limit to fit the other columns and json characters - .put("currency", generateBigString(-150)) - .put("date", "2020-10-10T00:00:00Z") - .put("HKD", 10.5) - .put("NZD", 1.14) - .build())))); - // Add a big message that does not fit into the limits of the destination - final AirbyteMessage bigMessage = new AirbyteMessage() - .withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream(catalog.getStreams().get(0).getName()) - .withEmittedAt(Instant.now().toEpochMilli()) - .withData(Jsons.jsonNode(ImmutableMap.builder() - .put("id", 3) - .put("currency", generateBigString(getGenerateBigStringAddExtraCharacters())) - .put("date", "2020-10-10T00:00:00Z") - .put("HKD", 10.5) - .put("NZD", 1.14) - .build()))); - final JsonNode config = getConfig(); - final String defaultSchema = getDefaultSchema(config); - final List allMessages = new ArrayList<>(); - allMessages.add(bigMessage); - allMessages.addAll(messages); - runSyncAndVerifyStateOutput(config, allMessages, configuredCatalog, false); - retrieveRawRecordsAndAssertSameMessages(catalog, messages, defaultSchema); - } - private String generateBigString(final int addExtraCharacters) { final int length = getMaxRecordValueLimit() + addExtraCharacters; return RANDOM @@ -1046,9 +982,9 @@ void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { Jsons.deserialize( MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); - // A randomized namespace is required otherwise you can generate a "false success" with data from a - // previous run. - final String namespace = Strings.addRandomSuffix("airbyte_source_namespace", "_", 8); + // A unique namespace is required to avoid test isolation problems. + final String namespace = TestingNamespaces.generate("source_namespace"); + TEST_SCHEMAS.add(namespace); catalog.getStreams().forEach(stream -> stream.setNamespace(namespace)); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( @@ -1081,11 +1017,13 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { Jsons.deserialize( MoreResources.readResource(DataArgumentsProvider.EXCHANGE_RATE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); - final var namespace1 = "sourcenamespace"; + final var namespace1 = TestingNamespaces.generate("source_namespace"); + TEST_SCHEMAS.add(namespace1); catalog.getStreams().forEach(stream -> stream.setNamespace(namespace1)); final var diffNamespaceStreams = new ArrayList(); - final var namespace2 = "diff_source_namespace"; + final var namespace2 = TestingNamespaces.generate("diff_source_namespace"); + TEST_SCHEMAS.add(namespace2); final var mapper = MoreMappers.initMapper(); for (final AirbyteStream stream : catalog.getStreams()) { final var clonedStream = mapper.readValue(mapper.writeValueAsString(stream), @@ -1115,16 +1053,30 @@ void testSyncWriteSameTableNameDifferentNamespace() throws Exception { retrieveRawRecordsAndAssertSameMessages(catalog, allMessages, defaultSchema); } + /** + * The goal of this test is to verify the expected conversions of a namespace as it appears in the + * catalog to how it appears in the destination. Each database has its own rules, so this test runs + * through several "edge" case sorts of names and checks the behavior. + * + * @param testCaseId - the id of each test case in namespace_test_cases.json so that we can handle + * an individual case specially for a specific database. + * @param namespaceInCatalog - namespace as it would appear in the catalog + * @param namespaceInDst - namespace as we would expect it to appear in the destination (this may be + * overridden for different databases). + * @throws Exception - broad catch of exception to hydrate log information with additional test case + * context. + */ @ParameterizedTest @ArgumentsSource(NamespaceTestCaseProvider.class) public void testNamespaces(final String testCaseId, - final String namespace, - final String normalizedNamespace) + final String namespaceInCatalog, + final String namespaceInDst) throws Exception { final Optional nameTransformer = getNameTransformer(); nameTransformer.ifPresent( - namingConventionTransformer -> assertNamespaceNormalization(testCaseId, normalizedNamespace, - namingConventionTransformer.getNamespace(namespace))); + namingConventionTransformer -> assertNamespaceNormalization(testCaseId, + namespaceInDst, + namingConventionTransformer.getNamespace(namespaceInCatalog))); if (!implementsNamespaces() || !supportNamespaceTest()) { return; @@ -1133,7 +1085,7 @@ public void testNamespaces(final String testCaseId, final AirbyteCatalog catalog = Jsons.deserialize( MoreResources.readResource(DataArgumentsProvider.NAMESPACE_CONFIG.getCatalogFileVersion(getProtocolVersion())), AirbyteCatalog.class); - catalog.getStreams().forEach(stream -> stream.setNamespace(namespace)); + catalog.getStreams().forEach(stream -> stream.setNamespace(namespaceInCatalog)); final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( catalog); @@ -1141,16 +1093,17 @@ public void testNamespaces(final String testCaseId, DataArgumentsProvider.NAMESPACE_CONFIG.getMessageFileVersion(getProtocolVersion())).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)) .collect(Collectors.toList()); - final List messagesWithNewNamespace = getRecordMessagesWithNewNamespace( - messages, namespace); + final List messagesWithNewNamespace = getRecordMessagesWithNewNamespace(messages, namespaceInCatalog); final JsonNode config = getConfig(); try { runSyncAndVerifyStateOutput(config, messagesWithNewNamespace, configuredCatalog, false); + // Add to the list of schemas to clean up. + TEST_SCHEMAS.add(namespaceInCatalog); } catch (final Exception e) { throw new IOException(String.format( "[Test Case %s] Destination failed to sync data to namespace %s, see \"namespace_test_cases.json for details\"", - testCaseId, namespace), e); + testCaseId, namespaceInCatalog), e); } } @@ -1862,10 +1815,16 @@ public Stream provideArguments(final ExtensionContext conte Jsons.deserialize(MoreResources.readResource(NAMESPACE_TEST_CASES_JSON)); return MoreIterators.toList(testCases.elements()).stream() .filter(testCase -> testCase.get("enabled").asBoolean()) - .map(testCase -> Arguments.of( - testCase.get("id").asText(), - testCase.get("namespace").asText(), - testCase.get("normalized").asText())); + .map(testCase -> { + final String namespaceInCatalog = TestingNamespaces.generate(testCase.get("namespace").asText()); + final String namespaceInDst = TestingNamespaces + .generateFromOriginal(namespaceInCatalog, testCase.get("namespace").asText(), testCase.get("normalized").asText()); + return Arguments.of( + testCase.get("id").asText(), + // Add uniqueness to namespace to avoid collisions between tests. + namespaceInCatalog, + namespaceInDst); + }); } } diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/TestingNamespaces.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/TestingNamespaces.java new file mode 100644 index 000000000000..a4391db59870 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/TestingNamespaces.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.standardtest.destination; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import org.apache.commons.lang3.RandomStringUtils; + +/** + * This class is used to generate unique namespaces for tests that follow a convention so that we + * can identify and delete old namespaces. Ideally tests would always clean up their own namespaces, + * but there are exception cases that can prevent that from happening. We want to be able to + * identify namespaces for which this has happened from their name, so we can take action. + *

    + * The convention we follow is `_test_YYYYMMDD_<8-character random suffix>`. + */ +public class TestingNamespaces { + + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + private static final int SUFFIX_LENGTH = 5; + public static final String STANDARD_PREFIX = "test_"; + + /** + * Generates a namespace that matches our testing namespace convention. + * + * @return convention-compliant namespace + */ + public static String generate() { + return generate(null); + } + + /** + * Generates a namespace that matches our testing namespace convention. + * + * @param prefix prefix to use for the namespace + * @return convention-compliant namespace + */ + public static String generate(final String prefix) { + final String userDefinedPrefix = prefix != null ? prefix + "_" : ""; + return userDefinedPrefix + STANDARD_PREFIX + FORMATTER.format(Instant.now().atZone(ZoneId.of("UTC"))) + "_" + generateSuffix(); + } + + public static String generateFromOriginal(final String toOverwrite, final String oldPrefix, final String newPrefix) { + return toOverwrite.replace(oldPrefix, newPrefix); + } + + /** + * Checks if a namespace is older than 2 days. + * + * @param namespace to check + * @return true if the namespace is older than 2 days, otherwise false + */ + public static boolean isOlderThan2Days(final String namespace) { + return isOlderThan(namespace, 2, ChronoUnit.DAYS); + } + + @SuppressWarnings("SameParameterValue") + private static boolean isOlderThan(final String namespace, final int timeMagnitude, final ChronoUnit timeUnit) { + return ifTestNamespaceGetDate(namespace) + .map(namespaceInstant -> namespaceInstant.isBefore(Instant.now().minus(timeMagnitude, timeUnit))) + .orElse(false); + } + + private static Optional ifTestNamespaceGetDate(final String namespace) { + final String[] parts = namespace.split("_"); + + if (parts.length < 3) { + return Optional.empty(); + } + + // need to re-add the _ since it gets pruned out by the split. + if (!STANDARD_PREFIX.equals(parts[parts.length - 3] + "_")) { + return Optional.empty(); + } + + return parseDateOrEmpty(parts[parts.length - 2]); + } + + private static Optional parseDateOrEmpty(final String dateCandidate) { + try { + return Optional.ofNullable(LocalDate.parse(dateCandidate, FORMATTER).atStartOfDay().toInstant(ZoneOffset.UTC)); + } catch (final DateTimeParseException e) { + return Optional.empty(); + } + } + + private static String generateSuffix() { + return RandomStringUtils.randomAlphabetic(SUFFIX_LENGTH).toLowerCase(); + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_test_cases.json b/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_test_cases.json index f9ad2f047859..0f52f07696db 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_test_cases.json +++ b/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_test_cases.json @@ -1,5 +1,6 @@ [ { + "_comment": "todo: normalized in this context means how the namespace is transformed in the destination. it does not refer to the Airbyte Normalization step. We should change it, but out of scope for this PR.", "id": "S1-1", "description": "namespace are converted to lowercase", "namespace": "NAMESPACE", diff --git a/airbyte-integrations/bases/standard-destination-test/src/test/java/io/airbyte/integrations/standardtest/destination/TestingNamespacesTest.java b/airbyte-integrations/bases/standard-destination-test/src/test/java/io/airbyte/integrations/standardtest/destination/TestingNamespacesTest.java new file mode 100644 index 000000000000..1963174c0052 --- /dev/null +++ b/airbyte-integrations/bases/standard-destination-test/src/test/java/io/airbyte/integrations/standardtest/destination/TestingNamespacesTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.standardtest.destination; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Test; + +class TestingNamespacesTest { + + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + + @Test + void testGenerate() { + final String[] namespace = TestingNamespaces.generate().split("_"); + assertEquals("test", namespace[0]); + assertEquals(FORMATTER.format(Instant.now().atZone(ZoneId.of("UTC")).toLocalDate()), namespace[1]); + assertFalse(namespace[2].isBlank()); + } + + @Test + void testGenerateWithPrefix() { + final String[] namespace = TestingNamespaces.generate("myprefix").split("_"); + assertEquals("myprefix", namespace[0]); + assertEquals("test", namespace[1]); + assertEquals(FORMATTER.format(Instant.now().atZone(ZoneId.of("UTC")).toLocalDate()), namespace[2]); + assertFalse(namespace[3].isBlank()); + } + + @Test + void testIsOlderThan2Days() { + assertFalse(TestingNamespaces.isOlderThan2Days("myprefix_test_" + getDate(0) + "_12345")); + assertTrue(TestingNamespaces.isOlderThan2Days("myprefix_test_" + getDate(2) + "_12345")); + } + + @Test + void doesNotFailOnNonConventionalNames() { + assertFalse(TestingNamespaces.isOlderThan2Days("12345")); + assertFalse(TestingNamespaces.isOlderThan2Days("test_12345")); + assertFalse(TestingNamespaces.isOlderThan2Days("hello_test_12345")); + assertFalse(TestingNamespaces.isOlderThan2Days("myprefix_test1_" + getDate(2) + "_12345")); + + } + + private static String getDate(final int daysAgo) { + return FORMATTER.format(Instant.now().minus(daysAgo, ChronoUnit.DAYS).atZone(ZoneId.of("UTC")).toLocalDate()); + } + +} diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java index 5d07cf1d6232..1ff01947da71 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java @@ -26,6 +26,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -43,6 +44,7 @@ public abstract class SourceAcceptanceTest extends AbstractSourceConnectorTest { public static final String CDC_DELETED_AT = "_ab_cdc_deleted_at"; public static final String CDC_LOG_FILE = "_ab_cdc_log_file"; public static final String CDC_LOG_POS = "_ab_cdc_log_pos"; + public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; public static final String CDC_EVENT_SERIAL_NO = "_ab_cdc_event_serial_no"; private static final Logger LOGGER = LoggerFactory.getLogger(SourceAcceptanceTest.class); @@ -174,6 +176,11 @@ protected void verifyCatalog(final AirbyteCatalog catalog) throws Exception { */ @Test public void testFullRefreshRead() throws Exception { + if (!sourceSupportsFullRefresh()) { + LOGGER.info("Test skipped. Source does not support full refresh."); + return; + } + final ConfiguredAirbyteCatalog catalog = withFullRefreshSyncModes(getConfiguredCatalog()); final List allMessages = runRead(catalog); @@ -195,6 +202,11 @@ protected void assertFullRefreshMessages(final List allMessages) */ @Test public void testIdenticalFullRefreshes() throws Exception { + if (!sourceSupportsFullRefresh()) { + LOGGER.info("Test skipped. Source does not support full refresh."); + return; + } + if (IMAGES_TO_SKIP_IDENTICAL_FULL_REFRESHES.contains(getImageName().split(":")[0])) { return; } @@ -274,15 +286,20 @@ public void testEmptyStateIncrementalIdenticalToFullRefresh() throws Exception { return; } + if (!sourceSupportsFullRefresh()) { + LOGGER.info("Test skipped. Source does not support full refresh."); + return; + } + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); final ConfiguredAirbyteCatalog fullRefreshCatalog = withFullRefreshSyncModes(configuredCatalog); final List fullRefreshRecords = filterRecords(runRead(fullRefreshCatalog)); final List emptyStateRecords = filterRecords(runRead(configuredCatalog, Jsons.jsonNode(new HashMap<>()))); - final String assertionMessage = "Expected a full refresh sync and incremental sync with no input state to produce identical records"; - assertFalse(fullRefreshRecords.isEmpty(), assertionMessage); - assertFalse(emptyStateRecords.isEmpty(), assertionMessage); - assertSameRecords(fullRefreshRecords, emptyStateRecords, assertionMessage); + assertFalse(fullRefreshRecords.isEmpty(), "Expected a full refresh sync to produce records"); + assertFalse(emptyStateRecords.isEmpty(), "Expected state records to not be empty"); + assertSameRecords(fullRefreshRecords, emptyStateRecords, + "Expected a full refresh sync and incremental sync with no input state to produce identical records"); } /** @@ -326,9 +343,17 @@ protected ConfiguredAirbyteCatalog withFullRefreshSyncModes(final ConfiguredAirb } private boolean sourceSupportsIncremental() throws Exception { + return sourceSupports(INCREMENTAL); + } + + private boolean sourceSupportsFullRefresh() throws Exception { + return sourceSupports(FULL_REFRESH); + } + + private boolean sourceSupports(final SyncMode syncMode) throws Exception { final ConfiguredAirbyteCatalog catalog = getConfiguredCatalog(); for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { - if (stream.getStream().getSupportedSyncModes().contains(INCREMENTAL)) { + if (stream.getStream().getSupportedSyncModes().contains(syncMode)) { return true; } } @@ -359,6 +384,7 @@ private AirbyteRecordMessage pruneCdcMetadata(final AirbyteRecordMessage m) { ((ObjectNode) clone.getData()).remove(CDC_UPDATED_AT); ((ObjectNode) clone.getData()).remove(CDC_DELETED_AT); ((ObjectNode) clone.getData()).remove(CDC_EVENT_SERIAL_NO); + ((ObjectNode) clone.getData()).remove(CDC_DEFAULT_CURSOR); return clone; } diff --git a/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs b/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs index 9d5782685c3e..fdd5c3deb969 100644 --- a/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/destination-java/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: database connectorType: destination definitionId: {{generateDefinitionId}} @@ -16,6 +18,7 @@ data: name: {{capitalCase name}} releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/destinations/{{dashCase name}} tags: - language:python diff --git a/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs index 9d5782685c3e..fdd5c3deb969 100644 --- a/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: database connectorType: destination definitionId: {{generateDefinitionId}} @@ -16,6 +18,7 @@ data: name: {{capitalCase name}} releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/destinations/{{dashCase name}} tags: - language:python diff --git a/airbyte-integrations/connector-templates/generator/build.gradle b/airbyte-integrations/connector-templates/generator/build.gradle index ee220700f2eb..53df8f43d219 100644 --- a/airbyte-integrations/connector-templates/generator/build.gradle +++ b/airbyte-integrations/connector-templates/generator/build.gradle @@ -1,6 +1,6 @@ plugins { id "base" - id "com.github.node-gradle.node" version "2.2.4" + id "com.github.node-gradle.node" version "3.5.1" } def nodeVersion = System.getenv('NODE_VERSION') ?: '16.13.0' diff --git a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs index a614ed2e6ad0..6499f05d2d71 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} @@ -16,6 +18,7 @@ data: name: {{capitalCase name}} releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: - language:lowcode diff --git a/airbyte-integrations/connector-templates/source-configuration-based/requirements.txt.hbs b/airbyte-integrations/connector-templates/source-configuration-based/requirements.txt.hbs index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/requirements.txt.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/requirements.txt.hbs @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs index 1fc29f343d68..38d921d99b11 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs @@ -10,6 +10,7 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test", diff --git a/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs index 651c8393bc35..4f8331bb191c 100644 --- a/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-generic/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} @@ -16,5 +18,6 @@ data: name: {{capitalCase name}} releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs index 0adc873f7fc0..fa29cee2516d 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: database connectorType: source definitionId: {{generateDefinitionId}} @@ -15,6 +17,7 @@ data: license: MIT name: {{capitalCase name}} releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: diff --git a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs index 1c73077c3d03..629b17607a6b 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} @@ -15,6 +17,7 @@ data: license: MIT name: {{capitalCase name}} releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: diff --git a/airbyte-integrations/connector-templates/source-python-http-api/requirements.txt.hbs b/airbyte-integrations/connector-templates/source-python-http-api/requirements.txt.hbs index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/requirements.txt.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/requirements.txt.hbs @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs index b4975a7cb028..667a27713662 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs @@ -10,6 +10,7 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test", diff --git a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs index 1c73077c3d03..629b17607a6b 100644 --- a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} @@ -15,6 +17,7 @@ data: license: MIT name: {{capitalCase name}} releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: diff --git a/airbyte-integrations/connector-templates/source-python/requirements.txt.hbs b/airbyte-integrations/connector-templates/source-python/requirements.txt.hbs index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connector-templates/source-python/requirements.txt.hbs +++ b/airbyte-integrations/connector-templates/source-python/requirements.txt.hbs @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connector-templates/source-python/setup.py.hbs b/airbyte-integrations/connector-templates/source-python/setup.py.hbs index d7ef522bb1ee..563d13c3708c 100644 --- a/airbyte-integrations/connector-templates/source-python/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/setup.py.hbs @@ -10,6 +10,8 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.2", "connector-acceptance-test", ] diff --git a/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs index 1c73077c3d03..629b17607a6b 100644 --- a/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-singer/metadata.yaml.hbs @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} @@ -15,6 +17,7 @@ data: license: MIT name: {{capitalCase name}} releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: diff --git a/airbyte-integrations/connector-templates/source-singer/requirements.txt.hbs b/airbyte-integrations/connector-templates/source-singer/requirements.txt.hbs index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connector-templates/source-singer/requirements.txt.hbs +++ b/airbyte-integrations/connector-templates/source-singer/requirements.txt.hbs @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connector-templates/source-singer/setup.py.hbs b/airbyte-integrations/connector-templates/source-singer/setup.py.hbs index c73f9e16c162..adfdf1ecdbf6 100644 --- a/airbyte-integrations/connector-templates/source-singer/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-singer/setup.py.hbs @@ -12,7 +12,6 @@ MAIN_REQUIREMENTS = [ TEST_REQUIREMENTS = [ "pytest~=6.2", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/__init__.py b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/__init__.py index 4a5cfa2864e6..ff5ba7b7242c 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/__init__.py +++ b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml index 65429c8d3764..832eb8ca9454 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml @@ -10,11 +10,15 @@ data: name: Amazon SQS registries: cloud: - enabled: true + enabled: false # hide Amazon SQS Destination https://github.com/airbytehq/airbyte/issues/16316 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/amazon-sqs tags: - language:python + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml b/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml index 315a71a99d74..baf2cbf1ad98 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/aws-datalake tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml index 7792b9802c79..86283b46945f 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml @@ -20,7 +20,11 @@ data: resourceRequirements: memory_limit: 1Gi memory_request: 1Gi - documentationUrl: https://docs.airbyte.com/integrations/destinations/azureblobstorage + documentationUrl: https://docs.airbyte.com/integrations/destinations/azure-blob-storage tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java index 51fa2db7ca28..e5799a6b9556 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -79,19 +80,19 @@ protected List getAppendBlobClient(final String streamName) th .buildAppendBlobClient(); final BlobContainerClient containerClient = streamAppendBlobClient.getContainerClient(); - var blobItemList = StreamSupport.stream(containerClient.listBlobs().spliterator(), false) + final var blobItemList = StreamSupport.stream(containerClient.listBlobs().spliterator(), false) .collect(Collectors.toList()); - var filteredBlobList = blobItemList.stream() + final var filteredBlobList = blobItemList.stream() .filter(blob -> blob.getName().startsWith(streamName + "/")) .toList(); if (!filteredBlobList.isEmpty()) { - List clobClientList = new ArrayList<>(); + final List clobClientList = new ArrayList<>(); filteredBlobList.forEach(blobItem -> { clobClientList.add(specializedBlobClientBuilder.blobName(blobItem.getName()).buildAppendBlobClient()); }); return clobClientList; } else { - var errorText = String.format("Can not find blob started with: %s/", streamName); + final var errorText = String.format("Can not find blob started with: %s/", streamName); LOGGER.error(errorText); throw new Exception(errorText); } @@ -125,7 +126,7 @@ protected boolean supportObjectDataTypeTest() { *

  • Construct the Azure Blob client.
    • */ @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { final JsonNode baseConfigJson = getBaseConfigJson(); configJson = Jsons.jsonNode(ImmutableMap.builder() diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index bb9528d167cf..e0706ac44eae 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-bigquery-denormalized COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.4.1 +LABEL io.airbyte.version=1.5.3 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle index 2516657a500d..bab42c9c7c4a 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle @@ -26,7 +26,6 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-bigquery-denormalized') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) integrationTestJavaImplementation project(':airbyte-db:db-lib') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml index 0bc8f1873f84..6ce69ccbc34e 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 079d5540-f236-4294-ba7c-ade8fd918496 - dockerImageTag: 1.4.1 + dockerImageTag: 1.5.3 dockerRepository: airbyte/destination-bigquery-denormalized githubIssueLabel: destination-bigquery-denormalized icon: bigquery.svg - license: MIT + license: ELv2 name: BigQuery (denormalized typed struct) registries: cloud: @@ -23,4 +23,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery tags: - language:java + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java index e10dcadce4b1..7a1abff0188a 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java @@ -45,6 +45,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.io.IOException; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.TimeZone; @@ -203,7 +204,7 @@ protected JsonNode createConfig() throws IOException { } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { config = createConfig(); bigquery = configureBigQuery(config); dataset = getBigQueryDataSet(config, bigquery); diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 76e06e1346fc..3117f0af1636 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -18,14 +18,17 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar FROM airbyte/integration-base-java:dev -RUN yum install -y python3 python3-devel jq sshpass git && \ +RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ python -m ensurepip --upgrade && \ + # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 + pip3 install wheel && \ + pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ pip3 install dbt-bigquery==1.0.0 # Luckily, none of normalization's files conflict with destination-bigquery's files :) # We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization:dev /airbyte /airbyte +COPY --from=airbyte/normalization:0.4.3 /airbyte /airbyte # Install python dependencies WORKDIR /airbyte/base_python_structs RUN pip3 install . @@ -44,7 +47,7 @@ ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.4.4 +LABEL io.airbyte.version=1.10.2 LABEL io.airbyte.name=airbyte/destination-bigquery ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 49cef6bab508..5b0004d9e808 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -10,9 +10,12 @@ application { } dependencies { - implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' + implementation 'io.airbyte:airbyte-cdk:0.0.1' + + implementation 'com.google.cloud:google-cloud-bigquery:2.27.0' implementation 'org.apache.commons:commons-lang3:3.11' implementation 'org.apache.commons:commons-csv:1.4' + implementation 'org.apache.commons:commons-text:1.10.0' implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.12.0' implementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.4.5' @@ -22,10 +25,12 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:base-java-s3') + implementation project(':airbyte-integrations:bases:base-typing-deduping') implementation project(':airbyte-integrations:connectors:destination-gcs') implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} testImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:bases:base-typing-deduping-test') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-bigquery') @@ -34,12 +39,6 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } -tasks.named("airbyteDocker") { - // this is really inefficent (because base-normalization:airbyteDocker builds 9 docker images) - // but it's also just simple to implement - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDocker -} - configurations.all { resolutionStrategy { // at time of writing: deps.toml declares google-cloud-storage 2.17.2 diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index bc7853654944..0e191513ea61 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 1.4.4 + dockerImageTag: 1.10.2 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg - license: MIT + license: ELv2 name: BigQuery normalizationConfig: normalizationIntegrationType: bigquery @@ -28,4 +28,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java index 7c966c3c0e05..9f68f48e70fd 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java @@ -19,7 +19,6 @@ import java.util.function.Function; import org.apache.avro.Schema; import org.apache.avro.file.CodecFactory; -import org.apache.commons.lang3.StringUtils; /** * This class differs from {@link AvroSerializedBuffer} in that 1) the Avro schema can be customized @@ -52,7 +51,7 @@ public static BufferCreateFunction createBufferFunction(final S3AvroFormatConfig return (pair, catalog) -> { final AirbyteStream stream = catalog.getStreams() .stream() - .filter(s -> s.getStream().getName().equals(pair.getName()) && StringUtils.equals(s.getStream().getNamespace(), pair.getNamespace())) + .filter(s -> s.getStream().getName().equals(pair.getName())) .findFirst() .orElseThrow(() -> new RuntimeException(String.format("No such stream %s.%s", pair.getNamespace(), pair.getName()))) .getStream(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 58519e1ad4ce..5ab5589e4abb 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -22,11 +22,22 @@ import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; +import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.GcsAvroBigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.GcsCsvBigQueryRecordFormatter; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryDestinationHandler; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV1V2Migrator; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV2RawTableMigrator; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; @@ -66,6 +77,8 @@ public class BigQueryDestination extends BaseConnector implements Destination { + private static final String RAW_DATA_DATASET = "raw_data_dataset"; + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestination.class); private static final List REQUIRED_PERMISSIONS = List.of( "storage.multipartUploads.abort", @@ -160,7 +173,7 @@ private AirbyteConnectionStatus checkGcsPermission(final JsonNode config) { } } - protected BigQuery getBigQuery(final JsonNode config) { + public static BigQuery getBigQuery(final JsonNode config) { final String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); try { @@ -199,45 +212,92 @@ public static GoogleCredentials getServiceAccountCredentials(final JsonNode conf * @param config - integration-specific configuration object as json. e.g. { "username": "airbyte", * "password": "super secure" } * @param catalog - schema of the incoming messages. - * @param outputRecordCollector - * @return - * @throws IOException */ @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) - throws IOException { + throws Exception { + // Set the default namespace on streams with null namespace. This means we don't need to repeat this + // logic in the rest of the connector. + // (record messages still need to handle null namespaces though, which currently happens in e.g. + // BigQueryRecordConsumer#acceptTracked) + // This probably should be shared logic amongst destinations eventually. + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + if (StringUtils.isEmpty(stream.getStream().getNamespace())) { + stream.getStream().withNamespace(BigQueryUtils.getDatasetId(config)); + } + } + + final String datasetLocation = BigQueryUtils.getDatasetLocation(config); + final BigQuerySqlGenerator sqlGenerator = new BigQuerySqlGenerator(datasetLocation); + final CatalogParser catalogParser; + if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_DATA_DATASET).isPresent()) { + catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_DATA_DATASET).get()); + } else { + catalogParser = new CatalogParser(sqlGenerator); + } + final ParsedCatalog parsedCatalog; + + final BigQuery bigquery = getBigQuery(config); + final TyperDeduper typerDeduper; + if (TypingAndDedupingFlag.isDestinationV2()) { + parsedCatalog = catalogParser.parseCatalog(catalog); + final BigQueryV1V2Migrator migrator = new BigQueryV1V2Migrator(bigquery, namingResolver); + final BigQueryV2RawTableMigrator v2RawTableMigrator = new BigQueryV2RawTableMigrator(bigquery); + typerDeduper = new DefaultTyperDeduper<>( + sqlGenerator, + new BigQueryDestinationHandler(bigquery, datasetLocation), + parsedCatalog, + migrator, + v2RawTableMigrator); + } else { + parsedCatalog = null; + typerDeduper = new NoopTyperDeduper(); + } + final UploadingMethod uploadingMethod = BigQueryUtils.getLoadingMethod(config); if (uploadingMethod == UploadingMethod.STANDARD) { LOGGER.warn("The \"standard\" upload mode is not performant, and is not recommended for production. " + "Please use the GCS upload mode if you are syncing a large amount of data."); - return getStandardRecordConsumer(config, catalog, outputRecordCollector); + return getStandardRecordConsumer(bigquery, config, catalog, parsedCatalog, outputRecordCollector, typerDeduper); } else { - return getGcsRecordConsumer(config, catalog, outputRecordCollector); + return getGcsRecordConsumer(bigquery, config, catalog, parsedCatalog, outputRecordCollector, typerDeduper); } } - protected Map> getUploaderMap(final JsonNode config, - final ConfiguredAirbyteCatalog catalog) + protected Map> getUploaderMap( + final BigQuery bigquery, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final boolean use1s1t) throws IOException { - final BigQuery bigquery = getBigQuery(config); - final Map> uploaderMap = new HashMap<>(); for (final ConfiguredAirbyteStream configStream : catalog.getStreams()) { final AirbyteStream stream = configStream.getStream(); - if (StringUtils.isEmpty(stream.getNamespace())) { - stream.setNamespace(BigQueryUtils.getDatasetId(config)); - } + final StreamConfig parsedStream; + final String streamName = stream.getName(); + final String targetTableName; + if (use1s1t) { + parsedStream = parsedCatalog.getStream(stream.getNamespace(), stream.getName()); + targetTableName = parsedStream.id().rawName(); + } else { + parsedStream = null; + targetTableName = getTargetTableName(streamName); + } + final UploaderConfig uploaderConfig = UploaderConfig .builder() .bigQuery(bigquery) .configStream(configStream) + .parsedStream(parsedStream) .config(config) .formatterMap(getFormatterMap(stream.getJsonSchema())) .tmpTableName(namingResolver.getTmpTableName(streamName)) - .targetTableName(getTargetTableName(streamName)) + .targetTableName(targetTableName) + // This refers to whether this is BQ denormalized or not .isDefaultAirbyteTmpSchema(isDefaultAirbyteTmpTableSchema()) .build(); @@ -276,20 +336,38 @@ protected String getTargetTableName(final String streamName) { return namingResolver.getRawTableName(streamName); } - private AirbyteMessageConsumer getStandardRecordConsumer(final JsonNode config, + private AirbyteMessageConsumer getStandardRecordConsumer(final BigQuery bigquery, + final JsonNode config, final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) - throws IOException { - final Map> writeConfigs = getUploaderMap(config, catalog); - return new BigQueryRecordConsumer(writeConfigs, outputRecordCollector, BigQueryUtils.getDatasetId(config)); + final ParsedCatalog parsedCatalog, + final Consumer outputRecordCollector, + final TyperDeduper typerDeduper) + throws Exception { + typerDeduper.prepareTables(); + final Map> writeConfigs = getUploaderMap( + bigquery, + config, + catalog, + parsedCatalog, + TypingAndDedupingFlag.isDestinationV2()); + + return new BigQueryRecordConsumer( + bigquery, + writeConfigs, + outputRecordCollector, + BigQueryUtils.getDatasetId(config), + typerDeduper, + parsedCatalog); } - public AirbyteMessageConsumer getGcsRecordConsumer(final JsonNode config, + public AirbyteMessageConsumer getGcsRecordConsumer(final BigQuery bigQuery, + final JsonNode config, final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { - + final ParsedCatalog parsedCatalog, + final Consumer outputRecordCollector, + final TyperDeduper typerDeduper) + throws Exception { final StandardNameTransformer gcsNameTransformer = new GcsNameTransformer(); - final BigQuery bigQuery = getBigQuery(config); final GcsDestinationConfig gcsConfig = BigQueryUtils.getGcsAvroDestinationConfig(config); final UUID stagingId = UUID.randomUUID(); final DateTime syncDatetime = DateTime.now(DateTimeZone.UTC); @@ -323,6 +401,7 @@ public AirbyteMessageConsumer getGcsRecordConsumer(final JsonNode config, () -> new FileBuffer(S3AvroFormatConfig.DEFAULT_SUFFIX, numberOfFileBuffers)); LOGGER.info("Creating BigQuery staging message consumer with staging ID {} at {}", stagingId, syncDatetime); + return new BigQueryStagingConsumerFactory().create( config, catalog, @@ -331,7 +410,10 @@ public AirbyteMessageConsumer getGcsRecordConsumer(final JsonNode config, onCreateBuffer, recordFormatterCreator, namingResolver::getTmpTableName, - getTargetTableNameTransformer(namingResolver)); + getTargetTableNameTransformer(namingResolver), + typerDeduper, + parsedCatalog, + BigQueryUtils.getDatasetId(config)); } protected BiFunction getAvroSchemaCreator() { @@ -349,7 +431,7 @@ protected Function getTargetTableNameTransformer(final BigQueryS /** * Retrieves user configured file buffer amount so as long it doesn't exceed the maximum number of * file buffers and sets the minimum number to the default - * + *

    * NOTE: If Out Of Memory Exceptions (OOME) occur, this can be a likely cause as this hard limit has * not been thoroughly load tested across all instance sizes * diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java index aa3069a06b04..23d34d929057 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java @@ -4,13 +4,22 @@ package io.airbyte.integrations.destination.bigquery; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.TableId; import io.airbyte.commons.string.Strings; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; +import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -26,22 +35,54 @@ public class BigQueryRecordConsumer extends FailureTrackingAirbyteMessageConsume private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryRecordConsumer.class); + private final BigQuery bigquery; private final Map> uploaderMap; private final Consumer outputRecordCollector; - private final String datasetId; + private final String defaultDatasetId; private AirbyteMessage lastStateMessage = null; - public BigQueryRecordConsumer(final Map> uploaderMap, + private final TypeAndDedupeOperationValve streamTDValve = new TypeAndDedupeOperationValve(); + private final ParsedCatalog catalog; + private final boolean use1s1t; + private final TyperDeduper typerDeduper; + + public BigQueryRecordConsumer(final BigQuery bigquery, + final Map> uploaderMap, final Consumer outputRecordCollector, - final String datasetId) { + final String defaultDatasetId, + final TyperDeduper typerDeduper, + final ParsedCatalog catalog) { + this.bigquery = bigquery; this.uploaderMap = uploaderMap; this.outputRecordCollector = outputRecordCollector; - this.datasetId = datasetId; + this.defaultDatasetId = defaultDatasetId; + this.typerDeduper = typerDeduper; + this.catalog = catalog; + this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); + + LOGGER.info("Got parsed catalog {}", catalog); + LOGGER.info("Got canonical stream IDs {}", uploaderMap.keySet()); } @Override protected void startTracked() { // todo (cgardens) - move contents of #write into this method. + if (use1s1t) { + // Set up our raw tables + uploaderMap.forEach((streamId, uploader) -> { + final StreamConfig stream = catalog.getStream(streamId); + if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { + // For streams in overwrite mode, truncate the raw table. + // non-1s1t syncs actually overwrite the raw table at the end of the sync, so we only do this in + // 1s1t mode. + final TableId rawTableId = TableId.of(stream.id().rawNamespace(), stream.id().rawName()); + bigquery.delete(rawTableId); + BigQueryUtils.createPartitionedTableIfNotExists(bigquery, rawTableId, DefaultBigQueryRecordFormatter.SCHEMA_V2); + } else { + uploader.createRawTable(); + } + }); + } } /** @@ -54,13 +95,13 @@ protected void startTracked() { * @param message {@link AirbyteMessage} to be processed */ @Override - public void acceptTracked(final AirbyteMessage message) { + public void acceptTracked(final AirbyteMessage message) throws Exception { if (message.getType() == Type.STATE) { lastStateMessage = message; outputRecordCollector.accept(message); } else if (message.getType() == Type.RECORD) { if (StringUtils.isEmpty(message.getRecord().getNamespace())) { - message.getRecord().setNamespace(datasetId); + message.getRecord().setNamespace(defaultDatasetId); } processRecord(message); } else { @@ -75,22 +116,26 @@ public void acceptTracked(final AirbyteMessage message) { * @param message record to be written */ private void processRecord(final AirbyteMessage message) { - final var pair = AirbyteStreamNameNamespacePair.fromRecordMessage(message.getRecord()); - uploaderMap.get(pair).upload(message); + final var streamId = AirbyteStreamNameNamespacePair.fromRecordMessage(message.getRecord()); + uploaderMap.get(streamId).upload(message); + // We are not doing any incremental typing and de-duping for Standard Inserts, see + // https://github.com/airbytehq/airbyte/issues/27586 } @Override - public void close(final boolean hasFailed) { + public void close(final boolean hasFailed) throws Exception { LOGGER.info("Started closing all connections"); final List exceptionsThrown = new ArrayList<>(); - uploaderMap.values().forEach(uploader -> { + uploaderMap.forEach((streamId, uploader) -> { try { uploader.close(hasFailed, outputRecordCollector, lastStateMessage); + typerDeduper.typeAndDedupe(streamId.getNamespace(), streamId.getName()); } catch (final Exception e) { exceptionsThrown.add(e); LOGGER.error("Exception while closing uploader {}", uploader, e); } }); + typerDeduper.commitFinalTables(); if (!exceptionsThrown.isEmpty()) { throw new RuntimeException(String.format("Exceptions thrown while closing consumer: %s", Strings.join(exceptionsThrown, "\n"))); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java index 3e6b96042378..22e910857e7a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java @@ -4,11 +4,19 @@ package io.airbyte.integrations.destination.bigquery; +import static io.airbyte.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Functions; import com.google.common.base.Preconditions; +import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.AirbyteMessageConsumer; +import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; @@ -46,28 +54,51 @@ public AirbyteMessageConsumer create(final JsonNode config, final BufferCreateFunction onCreateBuffer, final Function recordFormatterCreator, final Function tmpTableNameTransformer, - final Function targetTableNameTransformer) { + final Function targetTableNameTransformer, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace) + throws Exception { final Map writeConfigs = createWriteConfigs( config, catalog, + parsedCatalog, recordFormatterCreator, tmpTableNameTransformer, targetTableNameTransformer); + CheckedConsumer typeAndDedupeStreamFunction = + incrementalTypingAndDedupingStreamConsumer(typerDeduper); + return new BufferedStreamConsumer( outputRecordCollector, - onStartFunction(bigQueryGcsOperations, writeConfigs), + onStartFunction(bigQueryGcsOperations, writeConfigs, typerDeduper), new SerializedBufferingStrategy( onCreateBuffer, catalog, - flushBufferFunction(bigQueryGcsOperations, writeConfigs, catalog)), - onCloseFunction(bigQueryGcsOperations, writeConfigs), + flushBufferFunction(bigQueryGcsOperations, writeConfigs, catalog, typeAndDedupeStreamFunction)), + onCloseFunction(bigQueryGcsOperations, writeConfigs, typerDeduper), catalog, - json -> true); + json -> true, + defaultNamespace); + } + + private CheckedConsumer incrementalTypingAndDedupingStreamConsumer(final TyperDeduper typerDeduper) { + final TypeAndDedupeOperationValve valve = new TypeAndDedupeOperationValve(); + return (streamId) -> { + if (!valve.containsKey(streamId)) { + valve.addStream(streamId); + } + if (valve.readyToTypeAndDedupe(streamId)) { + typerDeduper.typeAndDedupe(streamId.getNamespace(), streamId.getName()); + valve.updateTimeAndIncreaseInterval(streamId); + } + }; } private Map createWriteConfigs(final JsonNode config, final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, final Function recordFormatterCreator, final Function tmpTableNameTransformer, final Function targetTableNameTransformer) { @@ -76,16 +107,27 @@ private Map createWriteConf Preconditions.checkNotNull(configuredStream.getDestinationSyncMode(), "Undefined destination sync mode"); final AirbyteStream stream = configuredStream.getStream(); + final StreamConfig streamConfig; + if (TypingAndDedupingFlag.isDestinationV2()) { + streamConfig = parsedCatalog.getStream(stream.getNamespace(), stream.getName()); + } else { + streamConfig = null; + } final String streamName = stream.getName(); final BigQueryRecordFormatter recordFormatter = recordFormatterCreator.apply(stream.getJsonSchema()); + final var internalTableNamespace = + TypingAndDedupingFlag.isDestinationV2() ? streamConfig.id().rawNamespace() : BigQueryUtils.sanitizeDatasetId(stream.getNamespace()); + final var targetTableName = + TypingAndDedupingFlag.isDestinationV2() ? streamConfig.id().rawName() : targetTableNameTransformer.apply(streamName); + final BigQueryWriteConfig writeConfig = new BigQueryWriteConfig( streamName, stream.getNamespace(), - BigQueryUtils.getSchema(config, configuredStream), + internalTableNamespace, BigQueryUtils.getDatasetLocation(config), tmpTableNameTransformer.apply(streamName), - targetTableNameTransformer.apply(streamName), + targetTableName, recordFormatter.getBigQuerySchema(), configuredStream.getDestinationSyncMode()); @@ -107,27 +149,36 @@ private Map createWriteConf * * @param bigQueryGcsOperations collection of Google Cloud Storage Operations * @param writeConfigs configuration settings used to describe how to write data and where it exists - * @return */ private OnStartFunction onStartFunction(final BigQueryStagingOperations bigQueryGcsOperations, - final Map writeConfigs) { + final Map writeConfigs, + final TyperDeduper typerDeduper) { return () -> { LOGGER.info("Preparing airbyte_raw tables in destination started for {} streams", writeConfigs.size()); + if (TypingAndDedupingFlag.isDestinationV2()) { + typerDeduper.prepareTables(); + } for (final BigQueryWriteConfig writeConfig : writeConfigs.values()) { LOGGER.info("Preparing staging are in destination for schema: {}, stream: {}, target table: {}, stage: {}", writeConfig.tableSchema(), writeConfig.streamName(), writeConfig.targetTableId(), writeConfig.streamName()); - final String datasetId = writeConfig.datasetId(); - bigQueryGcsOperations.createSchemaIfNotExists(datasetId, writeConfig.datasetLocation()); + // In Destinations V2, we will always use the 'airbyte' schema/namespace for raw tables + final String rawDatasetId = TypingAndDedupingFlag.isDestinationV2() ? DEFAULT_AIRBYTE_INTERNAL_NAMESPACE : writeConfig.datasetId(); + // Regardless, ensure the schema the customer wants to write to exists + bigQueryGcsOperations.createSchemaIfNotExists(writeConfig.datasetId(), writeConfig.datasetLocation()); + // Schema used for raw and airbyte internal tables + bigQueryGcsOperations.createSchemaIfNotExists(rawDatasetId, writeConfig.datasetLocation()); + // Customer's destination schema // With checkpointing, we will be creating the target table earlier in the setup such that // the data can be immediately loaded from the staging area bigQueryGcsOperations.createTableIfNotExists(writeConfig.targetTableId(), writeConfig.tableSchema()); - bigQueryGcsOperations.createStageIfNotExists(datasetId, writeConfig.streamName()); + bigQueryGcsOperations.createStageIfNotExists(rawDatasetId, writeConfig.streamName()); // When OVERWRITE mode, truncate the destination's raw table prior to syncing data if (writeConfig.syncMode() == DestinationSyncMode.OVERWRITE) { - bigQueryGcsOperations.truncateTableIfExists(datasetId, writeConfig.targetTableId(), writeConfig.tableSchema()); + // TODO: this might need special handling during the migration + bigQueryGcsOperations.truncateTableIfExists(rawDatasetId, writeConfig.targetTableId(), writeConfig.tableSchema()); } } - LOGGER.info("Preparing airbyte_raw tables in destination completed."); + LOGGER.info("Preparing tables in destination completed."); }; } @@ -142,12 +193,14 @@ private OnStartFunction onStartFunction(final BigQueryStagingOperations bigQuery private FlushBufferFunction flushBufferFunction( final BigQueryStagingOperations bigQueryGcsOperations, final Map writeConfigs, - final ConfiguredAirbyteCatalog catalog) { + final ConfiguredAirbyteCatalog catalog, + final CheckedConsumer incrementalTypeAndDedupeConsumer) { return (pair, writer) -> { LOGGER.info("Flushing buffer for stream {} ({}) to staging", pair.getName(), FileUtils.byteCountToDisplaySize(writer.getByteCount())); if (!writeConfigs.containsKey(pair)) { throw new IllegalArgumentException( - String.format("Message contained record from a stream that was not in the catalog. \ncatalog: %s", Jsons.serialize(catalog))); + String.format("Message contained record from a stream that was not in the catalog: %s.\nKeys: %s\ncatalog: %s", pair, + writeConfigs.keySet(), Jsons.serialize(catalog))); } final BigQueryWriteConfig writeConfig = writeConfigs.get(pair); @@ -164,6 +217,7 @@ private FlushBufferFunction flushBufferFunction( writeConfig.addStagedFile(stagedFile); bigQueryGcsOperations.copyIntoTableFromStage(datasetId, stream, writeConfig.targetTableId(), writeConfig.tableSchema(), List.of(stagedFile)); + incrementalTypeAndDedupeConsumer.accept(new AirbyteStreamNameNamespacePair(writeConfig.streamName(), writeConfig.namespace())); } catch (final Exception e) { LOGGER.error("Failed to flush and commit buffer data into destination's raw table:", e); throw new RuntimeException("Failed to upload buffer to stage and commit to destination", e); @@ -176,10 +230,10 @@ private FlushBufferFunction flushBufferFunction( * * @param bigQueryGcsOperations collection of staging operations * @param writeConfigs configuration settings used to describe how to write data and where it exists - * @return */ private OnCloseFunction onCloseFunction(final BigQueryStagingOperations bigQueryGcsOperations, - final Map writeConfigs) { + final Map writeConfigs, + final TyperDeduper typerDeduper) { return (hasFailed) -> { /* * Previously the hasFailed value was used to commit any remaining staged files into destination, @@ -188,9 +242,11 @@ private OnCloseFunction onCloseFunction(final BigQueryStagingOperations bigQuery */ LOGGER.info("Cleaning up destination started for {} streams", writeConfigs.size()); - for (final BigQueryWriteConfig writeConfig : writeConfigs.values()) { - bigQueryGcsOperations.dropStageIfExists(writeConfig.datasetId(), writeConfig.streamName()); + for (final Map.Entry entry : writeConfigs.entrySet()) { + typerDeduper.typeAndDedupe(entry.getKey().getNamespace(), entry.getKey().getName()); + bigQueryGcsOperations.dropStageIfExists(entry.getValue().datasetId(), entry.getValue().streamName()); } + typerDeduper.commitFinalTables(); LOGGER.info("Cleaning up destination completed."); }; } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 8eebe36f2763..57fe09be3282 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -38,8 +38,8 @@ import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.time.Instant; import java.time.LocalDateTime; @@ -203,12 +203,14 @@ public static Table createTable(final BigQuery bigquery, final String datasetNam */ static void createPartitionedTableIfNotExists(final BigQuery bigquery, final TableId tableId, final Schema schema) { try { + final var chunkingColumn = + TypingAndDedupingFlag.isDestinationV2() ? JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT : JavaBaseConstants.COLUMN_NAME_EMITTED_AT; final TimePartitioning partitioning = TimePartitioning.newBuilder(TimePartitioning.Type.DAY) - .setField(JavaBaseConstants.COLUMN_NAME_EMITTED_AT) + .setField(chunkingColumn) .build(); final Clustering clustering = Clustering.newBuilder() - .setFields(ImmutableList.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) + .setFields(ImmutableList.of(chunkingColumn)) .build(); final StandardTableDefinition tableDefinition = @@ -221,6 +223,7 @@ static void createPartitionedTableIfNotExists(final BigQuery bigquery, final Tab final Table table = bigquery.getTable(tableInfo.getTableId()); if (table != null && table.exists()) { + // TODO: Handle migration from v1 -> v2 LOGGER.info("Partitioned table ALREADY EXISTS: {}", tableId); } else { bigquery.create(tableInfo); @@ -353,15 +356,6 @@ private static String getFormattedBigQueryDateTime(final String dateTimeValue) { : null); } - /** - * @return BigQuery dataset ID - */ - public static String getSchema(final JsonNode config, final ConfiguredAirbyteStream stream) { - final String srcNamespace = stream.getStream().getNamespace(); - final String schemaName = srcNamespace == null ? getDatasetId(config) : srcNamespace; - return sanitizeDatasetId(schemaName); - } - public static String sanitizeDatasetId(final String datasetId) { return NAME_TRANSFORMER.getNamespace(datasetId); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java index bd6756e94b08..569a2772fdeb 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java @@ -11,8 +11,10 @@ import com.google.cloud.bigquery.StandardSQLTypeName; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import java.util.HashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -23,38 +25,58 @@ */ public class DefaultBigQueryRecordFormatter extends BigQueryRecordFormatter { - private static final com.google.cloud.bigquery.Schema SCHEMA = com.google.cloud.bigquery.Schema.of( + public static final com.google.cloud.bigquery.Schema SCHEMA = com.google.cloud.bigquery.Schema.of( Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING), Field.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, StandardSQLTypeName.TIMESTAMP), Field.of(JavaBaseConstants.COLUMN_NAME_DATA, StandardSQLTypeName.STRING)); - public DefaultBigQueryRecordFormatter(JsonNode jsonSchema, StandardNameTransformer namingResolver) { + public static final com.google.cloud.bigquery.Schema SCHEMA_V2 = com.google.cloud.bigquery.Schema.of( + Field.of(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, StandardSQLTypeName.STRING), + Field.of(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, StandardSQLTypeName.TIMESTAMP), + Field.of(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, StandardSQLTypeName.TIMESTAMP), + Field.of(JavaBaseConstants.COLUMN_NAME_DATA, StandardSQLTypeName.STRING)); + + public DefaultBigQueryRecordFormatter(final JsonNode jsonSchema, final StandardNameTransformer namingResolver) { super(jsonSchema, namingResolver); } @Override - public JsonNode formatRecord(AirbyteRecordMessage recordMessage) { - return Jsons.jsonNode(Map.of( - JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString(), - JavaBaseConstants.COLUMN_NAME_EMITTED_AT, getEmittedAtField(recordMessage), - JavaBaseConstants.COLUMN_NAME_DATA, getData(recordMessage))); + public JsonNode formatRecord(final AirbyteRecordMessage recordMessage) { + if (TypingAndDedupingFlag.isDestinationV2()) { + // Map.of has a @NonNull requirement, so creating a new Hash map + final HashMap destinationV2record = new HashMap<>(); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, UUID.randomUUID().toString()); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, getEmittedAtField(recordMessage)); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, null); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_DATA, getData(recordMessage)); + return Jsons.jsonNode(destinationV2record); + } else { + return Jsons.jsonNode(Map.of( + JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString(), + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, getEmittedAtField(recordMessage), + JavaBaseConstants.COLUMN_NAME_DATA, getData(recordMessage))); + } } - protected Object getEmittedAtField(AirbyteRecordMessage recordMessage) { + protected Object getEmittedAtField(final AirbyteRecordMessage recordMessage) { // Bigquery represents TIMESTAMP to the microsecond precision, so we convert to microseconds then // use BQ helpers to string-format correctly. final long emittedAtMicroseconds = TimeUnit.MICROSECONDS.convert(recordMessage.getEmittedAt(), TimeUnit.MILLISECONDS); return QueryParameterValue.timestamp(emittedAtMicroseconds).getValue(); } - protected Object getData(AirbyteRecordMessage recordMessage) { + protected Object getData(final AirbyteRecordMessage recordMessage) { final JsonNode formattedData = StandardNameTransformer.formatJsonPath(recordMessage.getData()); return Jsons.serialize(formattedData); } @Override - public Schema getBigQuerySchema(JsonNode jsonSchema) { - return SCHEMA; + public Schema getBigQuerySchema(final JsonNode jsonSchema) { + if (TypingAndDedupingFlag.isDestinationV2()) { + return SCHEMA_V2; + } else { + return SCHEMA; + } } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java index 44425f33bb28..8ee69576a1ce 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java @@ -13,17 +13,17 @@ */ public class GcsAvroBigQueryRecordFormatter extends DefaultBigQueryRecordFormatter { - public GcsAvroBigQueryRecordFormatter(JsonNode jsonSchema, StandardNameTransformer namingResolver) { + public GcsAvroBigQueryRecordFormatter(final JsonNode jsonSchema, final StandardNameTransformer namingResolver) { super(jsonSchema, namingResolver); } @Override - protected Object getEmittedAtField(AirbyteRecordMessage recordMessage) { + protected Object getEmittedAtField(final AirbyteRecordMessage recordMessage) { return recordMessage.getEmittedAt(); } @Override - protected Object getData(AirbyteRecordMessage recordMessage) { + protected Object getData(final AirbyteRecordMessage recordMessage) { return StandardNameTransformer.formatJsonPath(recordMessage.getData()).toString(); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java new file mode 100644 index 000000000000..a9c3a2949913 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobConfiguration; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobStatistics; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.math.BigInteger; +import java.util.Comparator; +import java.util.Optional; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO this stuff almost definitely exists somewhere else in our codebase. +public class BigQueryDestinationHandler implements DestinationHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestinationHandler.class); + + private final BigQuery bq; + private final String datasetLocation; + + public BigQueryDestinationHandler(final BigQuery bq, String datasetLocation) { + this.bq = bq; + this.datasetLocation = datasetLocation; + } + + @Override + public Optional findExistingTable(StreamId id) { + final Table table = bq.getTable(id.finalNamespace(), id.finalName()); + return Optional.ofNullable(table).map(Table::getDefinition); + } + + @Override + public boolean isFinalTableEmpty(StreamId id) { + return BigInteger.ZERO.equals(bq.getTable(TableId.of(id.finalNamespace(), id.finalName())).getNumRows()); + } + + @Override + public void execute(final String sql) throws InterruptedException { + if ("".equals(sql)) { + return; + } + final UUID queryId = UUID.randomUUID(); + LOGGER.info("Executing sql {}: {}", queryId, sql); + + /* + * If you run a query like CREATE SCHEMA ... OPTIONS(location=foo); CREATE TABLE ...;, bigquery + * doesn't do a good job of inferring the query location. Pass it in explicitly. + */ + Job job = bq.create(JobInfo.of(JobId.newBuilder().setLocation(datasetLocation).build(), QueryJobConfiguration.newBuilder(sql).build())); + job = job.waitFor(); + // waitFor() seems to throw an exception if the query failed, but javadoc says we're supposed to + // handle this case + if (job.getStatus().getError() != null) { + throw new RuntimeException(job.getStatus().getError().toString()); + } + + JobStatistics.QueryStatistics statistics = job.getStatistics(); + LOGGER.info("Root-level job {} completed in {} ms; processed {} bytes; billed for {} bytes", + queryId, + statistics.getEndTime() - statistics.getStartTime(), + statistics.getTotalBytesProcessed(), + statistics.getTotalBytesBilled()); + + // SQL transactions can spawn child jobs, which are billed individually. Log their stats too. + if (statistics.getNumChildJobs() != null) { + // There isn't (afaict) anything resembling job.getChildJobs(), so we have to ask bq for them + bq.listJobs(BigQuery.JobListOption.parentJobId(job.getJobId().getJob())).streamAll() + .sorted(Comparator.comparing(childJob -> childJob.getStatistics().getEndTime())) + .forEach(childJob -> { + JobConfiguration configuration = childJob.getConfiguration(); + if (configuration instanceof QueryJobConfiguration qc) { + JobStatistics.QueryStatistics childQueryStats = childJob.getStatistics(); + String truncatedQuery = qc.getQuery() + .replaceAll("\n", " ") + .replaceAll(" +", " ") + .substring(0, Math.min(100, qc.getQuery().length())); + if (!truncatedQuery.equals(qc.getQuery())) { + truncatedQuery += "..."; + } + LOGGER.info("Child sql {} completed in {} ms; processed {} bytes; billed for {} bytes", + truncatedQuery, + childQueryStats.getEndTime() - childQueryStats.getStartTime(), + childQueryStats.getTotalBytesProcessed(), + childQueryStats.getTotalBytesBilled()); + } else { + // other job types are extract/copy/load + // we're probably not using them, but handle just in case? + JobStatistics childJobStats = childJob.getStatistics(); + LOGGER.info("Non-query child job ({}) completed in {} ms", + configuration.getType(), + childJobStats.getEndTime() - childJobStats.getStartTime()); + } + }); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java new file mode 100644 index 000000000000..288bcd99d36b --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java @@ -0,0 +1,654 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.containsAllIgnoreCase; +import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.containsIgnoreCase; +import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.matchingKey; +import static java.util.stream.Collectors.joining; + +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TimePartitioning; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.AlterTableReport; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.TableNotMigratedException; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import io.airbyte.integrations.destination.bigquery.BigQuerySQLNameTransformer; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BigQuerySqlGenerator implements SqlGenerator { + + public static final String QUOTE = "`"; + private static final BigQuerySQLNameTransformer nameTransformer = new BigQuerySQLNameTransformer(); + + private final ColumnId CDC_DELETED_AT_COLUMN = buildColumnId("_ab_cdc_deleted_at"); + + private final Logger LOGGER = LoggerFactory.getLogger(BigQuerySqlGenerator.class); + private final String datasetLocation; + + /** + * @param datasetLocation This is technically redundant with {@link BigQueryDestinationHandler} + * setting the query execution location, but let's be explicit since this is typically a + * compliance requirement. + */ + public BigQuerySqlGenerator(final String datasetLocation) { + this.datasetLocation = datasetLocation; + } + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + return new StreamId( + nameTransformer.getNamespace(namespace), + nameTransformer.convertStreamName(name), + nameTransformer.getNamespace(rawNamespaceOverride), + nameTransformer.convertStreamName(StreamId.concatenateRawTableName(namespace, name)), + namespace, + name); + } + + @Override + public ColumnId buildColumnId(final String name) { + // Bigquery columns are case-insensitive, so do all our validation on the lowercased name + final String canonicalized = name.toLowerCase(); + return new ColumnId(nameTransformer.getIdentifier(name), name, canonicalized); + } + + public StandardSQLTypeName toDialectType(final AirbyteType type) { + // switch pattern-matching is still in preview at language level 17 :( + if (type instanceof final AirbyteProtocolType p) { + return toDialectType(p); + } else if (type instanceof Struct) { + return StandardSQLTypeName.JSON; + } else if (type instanceof Array) { + return StandardSQLTypeName.JSON; + } else if (type instanceof UnsupportedOneOf) { + return StandardSQLTypeName.JSON; + } else if (type instanceof final Union u) { + final AirbyteType typeWithPrecedence = u.chooseType(); + final StandardSQLTypeName dialectType; + if ((typeWithPrecedence instanceof Struct) || (typeWithPrecedence instanceof Array)) { + dialectType = StandardSQLTypeName.JSON; + } else { + dialectType = toDialectType((AirbyteProtocolType) typeWithPrecedence); + } + return dialectType; + } + + // Literally impossible; AirbyteType is a sealed interface. + throw new IllegalArgumentException("Unsupported AirbyteType: " + type); + } + + private String extractAndCast(final ColumnId column, final AirbyteType airbyteType) { + if (airbyteType instanceof final Union u) { + // This is guaranteed to not be a Union, so we won't recurse infinitely + final AirbyteType chosenType = u.chooseType(); + return extractAndCast(column, chosenType); + } else if (airbyteType instanceof Struct) { + // We need to validate that the struct is actually a struct. + // Note that struct columns are actually nullable in two ways. For a column `foo`: + // {foo: null} and {} are both valid, and are both written to the final table as a SQL NULL (_not_ a + // JSON null). + // JSON_QUERY(JSON'{}', '$."foo"') returns a SQL null. + // JSON_QUERY(JSON'{"foo": null}', '$."foo"') returns a JSON null. + return new StringSubstitutor(Map.of("column_name", escapeColumnNameForJsonPath(column.originalName()))).replace( + """ + PARSE_JSON(CASE + WHEN JSON_QUERY(`_airbyte_data`, '$."${column_name}"') IS NULL + OR JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$."${column_name}"'), wide_number_mode=>'round')) != 'object' + THEN NULL + ELSE JSON_QUERY(`_airbyte_data`, '$."${column_name}"') + END, wide_number_mode=>'round') + """); + } else if (airbyteType instanceof Array) { + // Much like the Struct case above, arrays need special handling. + return new StringSubstitutor(Map.of("column_name", escapeColumnNameForJsonPath(column.originalName()))).replace( + """ + PARSE_JSON(CASE + WHEN JSON_QUERY(`_airbyte_data`, '$."${column_name}"') IS NULL + OR JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$."${column_name}"'), wide_number_mode=>'round')) != 'array' + THEN NULL + ELSE JSON_QUERY(`_airbyte_data`, '$."${column_name}"') + END, wide_number_mode=>'round') + """); + } else if (airbyteType instanceof UnsupportedOneOf || airbyteType == AirbyteProtocolType.UNKNOWN) { + // JSON_QUERY returns a SQL null if the field contains a JSON null, so we actually parse the + // airbyte_data to json + // and json_query it directly (which preserves nulls correctly). + return new StringSubstitutor(Map.of("column_name", escapeColumnNameForJsonPath(column.originalName()))).replace( + """ + JSON_QUERY(PARSE_JSON(`_airbyte_data`, wide_number_mode=>'round'), '$."${column_name}"') + """); + } else { + final StandardSQLTypeName dialectType = toDialectType(airbyteType); + return "SAFE_CAST(JSON_VALUE(`_airbyte_data`, '$.\"" + escapeColumnNameForJsonPath(column.originalName()) + "\"') as " + dialectType.name() + + ")"; + } + } + + // TODO maybe make this a BiMap and elevate this method and its inverse (toDestinationSQLType?) to + // the SQLGenerator? + public StandardSQLTypeName toDialectType(final AirbyteProtocolType airbyteProtocolType) { + return switch (airbyteProtocolType) { + case STRING -> StandardSQLTypeName.STRING; + case NUMBER -> StandardSQLTypeName.NUMERIC; + case INTEGER -> StandardSQLTypeName.INT64; + case BOOLEAN -> StandardSQLTypeName.BOOL; + case TIMESTAMP_WITH_TIMEZONE -> StandardSQLTypeName.TIMESTAMP; + case TIMESTAMP_WITHOUT_TIMEZONE -> StandardSQLTypeName.DATETIME; + case TIME_WITH_TIMEZONE -> StandardSQLTypeName.STRING; + case TIME_WITHOUT_TIMEZONE -> StandardSQLTypeName.TIME; + case DATE -> StandardSQLTypeName.DATE; + case UNKNOWN -> StandardSQLTypeName.JSON; + }; + } + + @Override + public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + final String columnDeclarations = columnsAndTypes(stream); + final String clusterConfig = clusteringColumns(stream).stream() + .map(c -> StringUtils.wrap(c, QUOTE)) + .collect(joining(", ")); + final String forceCreateTable = force ? "OR REPLACE" : ""; + + return new StringSubstitutor(Map.of( + "final_namespace", stream.id().finalNamespace(QUOTE), + "dataset_location", datasetLocation, + "force_create_table", forceCreateTable, + "final_table_id", stream.id().finalTableId(QUOTE, suffix), + "column_declarations", columnDeclarations, + "cluster_config", clusterConfig)).replace( + """ + CREATE SCHEMA IF NOT EXISTS ${final_namespace} + OPTIONS(location="${dataset_location}"); + + CREATE ${force_create_table} TABLE ${final_table_id} ( + _airbyte_raw_id STRING NOT NULL, + _airbyte_extracted_at TIMESTAMP NOT NULL, + _airbyte_meta JSON NOT NULL, + ${column_declarations} + ) + PARTITION BY (DATE_TRUNC(_airbyte_extracted_at, DAY)) + CLUSTER BY ${cluster_config}; + """); + } + + private List clusteringColumns(final StreamConfig stream) { + final List clusterColumns = new ArrayList<>(); + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + // We're doing de-duping, therefore we have a primary key. + // Cluster on the first 3 PK columns since BigQuery only allows up to 4 clustering columns, + // and we're always clustering on _airbyte_extracted_at + stream.primaryKey().stream().limit(3).forEach(columnId -> { + clusterColumns.add(columnId.name()); + }); + } + clusterColumns.add("_airbyte_extracted_at"); + return clusterColumns; + } + + private String columnsAndTypes(final StreamConfig stream) { + return stream.columns().entrySet().stream() + .map(column -> String.join(" ", column.getKey().name(QUOTE), toDialectType(column.getValue()).name())) + .collect(joining(",\n")); + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, + final TableDefinition existingTable) + throws TableNotMigratedException { + final var alterTableReport = buildAlterTableReport(stream, existingTable); + boolean tableClusteringMatches = false; + boolean tablePartitioningMatches = false; + if (existingTable instanceof final StandardTableDefinition standardExistingTable) { + tableClusteringMatches = clusteringMatches(stream, standardExistingTable); + tablePartitioningMatches = partitioningMatches(standardExistingTable); + } + LOGGER.info("Alter Table Report {} {} {}; Clustering {}; Partitioning {}", + alterTableReport.columnsToAdd(), + alterTableReport.columnsToRemove(), + alterTableReport.columnsToChangeType(), + tableClusteringMatches, + tablePartitioningMatches); + + return alterTableReport.isNoOp() && tableClusteringMatches && tablePartitioningMatches; + } + + @VisibleForTesting + public boolean clusteringMatches(final StreamConfig stream, final StandardTableDefinition existingTable) { + return existingTable.getClustering() != null + && containsAllIgnoreCase( + new HashSet<>(existingTable.getClustering().getFields()), + clusteringColumns(stream)); + } + + @VisibleForTesting + public boolean partitioningMatches(final StandardTableDefinition existingTable) { + return existingTable.getTimePartitioning() != null + && existingTable.getTimePartitioning() + .getField() + .equalsIgnoreCase("_airbyte_extracted_at") + && TimePartitioning.Type.DAY.equals(existingTable.getTimePartitioning().getType()); + } + + public AlterTableReport buildAlterTableReport(final StreamConfig stream, final TableDefinition existingTable) { + final Map streamSchema = stream.columns().entrySet().stream() + .collect(Collectors.toMap( + entry -> entry.getKey().name(), + entry -> toDialectType(entry.getValue()))); + + final Map existingSchema = existingTable.getSchema().getFields().stream() + .collect(Collectors.toMap( + field -> field.getName(), + field -> field.getType().getStandardType())); + + // Columns in the StreamConfig that don't exist in the TableDefinition + final Set columnsToAdd = streamSchema.keySet().stream() + .filter(name -> !containsIgnoreCase(existingSchema.keySet(), name)) + .collect(Collectors.toSet()); + + // Columns in the current schema that are no longer in the StreamConfig + final Set columnsToRemove = existingSchema.keySet().stream() + .filter(name -> !containsIgnoreCase(streamSchema.keySet(), name) && !containsIgnoreCase( + JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS, name)) + .collect(Collectors.toSet()); + + // Columns that are typed differently than the StreamConfig + final Set columnsToChangeType = streamSchema.keySet().stream() + // If it's not in the existing schema, it should already be in the columnsToAdd Set + .filter(name -> { + // Big Query Columns are case-insensitive, first find the correctly cased key if it exists + return matchingKey(existingSchema.keySet(), name) + // if it does exist, only include it in this set if the type (the value in each respective map) + // is different between the stream and existing schemas + .map(key -> !existingSchema.get(key).equals(streamSchema.get(name))) + // if there is no matching key, then don't include it because it is probably already in columnsToAdd + .orElse(false); + }) + .collect(Collectors.toSet()); + + final boolean isDestinationV2Format = schemaContainAllFinalTableV2AirbyteColumns(existingSchema.keySet()); + + return new AlterTableReport(columnsToAdd, columnsToRemove, columnsToChangeType, isDestinationV2Format); + } + + /** + * Checks the schema to determine whether the table contains all expected final table airbyte + * columns + * + * @param columnNames the column names of the schema to check + * @return whether all the {@link JavaBaseConstants#V2_FINAL_TABLE_METADATA_COLUMNS} are present + */ + @VisibleForTesting + public static boolean schemaContainAllFinalTableV2AirbyteColumns(final Collection columnNames) { + return JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.stream() + .allMatch(column -> containsIgnoreCase(columnNames, column)); + } + + @Override + public String softReset(final StreamConfig stream) { + final String createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); + final String clearLoadedAt = clearLoadedAt(stream.id()); + final String rebuildInTempTable = updateTable(stream, SOFT_RESET_SUFFIX, false); + final String overwriteFinalTable = overwriteFinalTable(stream.id(), SOFT_RESET_SUFFIX); + return String.join("\n", createTempTable, clearLoadedAt, rebuildInTempTable, overwriteFinalTable); + } + + private String clearLoadedAt(final StreamId streamId) { + return new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) + .replace(""" + UPDATE ${raw_table_id} SET _airbyte_loaded_at = NULL WHERE 1=1; + """); + } + + @Override + public String updateTable(final StreamConfig stream, final String finalSuffix) { + return updateTable(stream, finalSuffix, true); + } + + private String updateTable(final StreamConfig stream, final String finalSuffix, final boolean verifyPrimaryKeys) { + String pkVarDeclaration = ""; + String validatePrimaryKeys = ""; + if (verifyPrimaryKeys && stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + pkVarDeclaration = "DECLARE missing_pk_count INT64;"; + validatePrimaryKeys = validatePrimaryKeys(stream.id(), stream.primaryKey(), stream.columns()); + } + final String insertNewRecords = insertNewRecords(stream, finalSuffix, stream.columns()); + String dedupFinalTable = ""; + String cdcDeletes = ""; + String dedupRawTable = ""; + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + dedupRawTable = dedupRawTable(stream.id(), finalSuffix); + // If we're in dedup mode, then we must have a cursor + dedupFinalTable = dedupFinalTable(stream.id(), finalSuffix, stream.primaryKey(), stream.cursor()); + cdcDeletes = cdcDeletes(stream, finalSuffix, stream.columns()); + } + final String commitRawTable = commitRawTable(stream.id()); + + return new StringSubstitutor(Map.of( + "pk_var_declaration", pkVarDeclaration, + "validate_primary_keys", validatePrimaryKeys, + "insert_new_records", insertNewRecords, + "dedup_final_table", dedupFinalTable, + "cdc_deletes", cdcDeletes, + "dedupe_raw_table", dedupRawTable, + "commit_raw_table", commitRawTable)).replace( + """ + ${pk_var_declaration} + + BEGIN TRANSACTION; + + ${validate_primary_keys} + + ${insert_new_records} + + ${dedup_final_table} + + ${dedupe_raw_table} + + ${cdc_deletes} + + ${commit_raw_table} + + COMMIT TRANSACTION; + """); + } + + @VisibleForTesting + String validatePrimaryKeys(final StreamId id, + final List primaryKeys, + final LinkedHashMap streamColumns) { + final String pkNullChecks = primaryKeys.stream().map( + pk -> { + final String jsonExtract = extractAndCast(pk, streamColumns.get(pk)); + return "AND " + jsonExtract + " IS NULL"; + }).collect(joining("\n")); + + return new StringSubstitutor(Map.of( + "raw_table_id", id.rawTableId(QUOTE), + "pk_null_checks", pkNullChecks)).replace( + """ + SET missing_pk_count = ( + SELECT COUNT(1) + FROM ${raw_table_id} + WHERE + `_airbyte_loaded_at` IS NULL + ${pk_null_checks} + ); + + IF missing_pk_count > 0 THEN + RAISE USING message = FORMAT('Raw table has %s rows missing a primary key', CAST(missing_pk_count AS STRING)); + END IF + ;"""); + } + + @VisibleForTesting + String insertNewRecords(final StreamConfig stream, final String finalSuffix, final LinkedHashMap streamColumns) { + final String columnCasts = streamColumns.entrySet().stream().map( + col -> extractAndCast(col.getKey(), col.getValue()) + " as " + col.getKey().name(QUOTE) + ",") + .collect(joining("\n")); + final String columnErrors; + if (streamColumns.isEmpty()) { + // ARRAY_CONCAT doesn't like having an empty argument list, so handle that case separately + columnErrors = "[]"; + } else { + columnErrors = "ARRAY_CONCAT(" + streamColumns.entrySet().stream().map( + col -> new StringSubstitutor(Map.of( + "raw_col_name", escapeColumnNameForJsonPath(col.getKey().originalName()), + "col_type", toDialectType(col.getValue()).name(), + "json_extract", extractAndCast(col.getKey(), col.getValue()))).replace( + // Explicitly parse json here. This is safe because we're not using the actual value anywhere, + // and necessary because json_query + """ + CASE + WHEN (JSON_QUERY(PARSE_JSON(`_airbyte_data`, wide_number_mode=>'round'), '$."${raw_col_name}"') IS NOT NULL) + AND (JSON_TYPE(JSON_QUERY(PARSE_JSON(`_airbyte_data`, wide_number_mode=>'round'), '$."${raw_col_name}"')) != 'null') + AND (${json_extract} IS NULL) + THEN ['Problem with `${raw_col_name}`'] + ELSE [] + END""")) + .collect(joining(",\n")) + ")"; + } + final String columnList = streamColumns.keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + + String cdcConditionalOrIncludeStatement = ""; + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { + cdcConditionalOrIncludeStatement = """ + OR ( + _airbyte_loaded_at IS NOT NULL + AND JSON_VALUE(`_airbyte_data`, '$._ab_cdc_deleted_at') IS NOT NULL + ) + """; + } + + return new StringSubstitutor(Map.of( + "raw_table_id", stream.id().rawTableId(QUOTE), + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), + "column_casts", columnCasts, + "column_errors", columnErrors, + "cdcConditionalOrIncludeStatement", cdcConditionalOrIncludeStatement, + "column_list", columnList)).replace( + """ + INSERT INTO ${final_table_id} + ( + ${column_list} + _airbyte_meta, + _airbyte_raw_id, + _airbyte_extracted_at + ) + WITH intermediate_data AS ( + SELECT + ${column_casts} + ${column_errors} as _airbyte_cast_errors, + _airbyte_raw_id, + _airbyte_extracted_at + FROM ${raw_table_id} + WHERE + _airbyte_loaded_at IS NULL + ${cdcConditionalOrIncludeStatement} + ) + SELECT + ${column_list} + to_json(struct(_airbyte_cast_errors AS errors)) AS _airbyte_meta, + _airbyte_raw_id, + _airbyte_extracted_at + FROM intermediate_data;"""); + } + + @VisibleForTesting + String dedupFinalTable(final StreamId id, + final String finalSuffix, + final List primaryKey, + final Optional cursor) { + final String pkList = primaryKey.stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); + final String cursorOrderClause = cursor + .map(cursorId -> cursorId.name(QUOTE) + " DESC NULLS LAST,") + .orElse(""); + + return new StringSubstitutor(Map.of( + "final_table_id", id.finalTableId(QUOTE, finalSuffix), + "pk_list", pkList, + "cursor_order_clause", cursorOrderClause)).replace( + """ + DELETE FROM ${final_table_id} + WHERE + `_airbyte_raw_id` IN ( + SELECT `_airbyte_raw_id` FROM ( + SELECT `_airbyte_raw_id`, row_number() OVER ( + PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} `_airbyte_extracted_at` DESC + ) as row_number FROM ${final_table_id} + ) + WHERE row_number != 1 + ) + ;"""); + } + + @VisibleForTesting + String cdcDeletes(final StreamConfig stream, + final String finalSuffix, + final LinkedHashMap streamColumns) { + + if (stream.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { + return ""; + } + + if (!streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { + return ""; + } + + final String pkList = stream.primaryKey().stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); + final String pkCasts = stream.primaryKey().stream().map(pk -> extractAndCast(pk, streamColumns.get(pk))).collect(joining(",\n")); + + // we want to grab IDs for deletion from the raw table (not the final table itself) to hand + // out-of-order record insertions after the delete has been registered + return new StringSubstitutor(Map.of( + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), + "raw_table_id", stream.id().rawTableId(QUOTE), + "pk_list", pkList, + "pk_extracts", pkCasts, + "quoted_cdc_delete_column", QUOTE + "_ab_cdc_deleted_at" + QUOTE)).replace( + """ + DELETE FROM ${final_table_id} + WHERE + (${pk_list}) IN ( + SELECT ( + ${pk_extracts} + ) + FROM ${raw_table_id} + WHERE JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$._ab_cdc_deleted_at'), wide_number_mode=>'round')) != 'null' + ) + ;"""); + } + + @VisibleForTesting + String dedupRawTable(final StreamId id, final String finalSuffix) { + return new StringSubstitutor(Map.of( + "raw_table_id", id.rawTableId(QUOTE), + "final_table_id", id.finalTableId(QUOTE, finalSuffix))).replace( + // Note that this leaves _all_ deletion records in the raw table. We _could_ clear them out, but it + // would be painful, + // and it only matters in a few edge cases. + """ + DELETE FROM + ${raw_table_id} + WHERE + `_airbyte_raw_id` NOT IN ( + SELECT `_airbyte_raw_id` FROM ${final_table_id} + ) + ;"""); + } + + @VisibleForTesting + String commitRawTable(final StreamId id) { + return new StringSubstitutor(Map.of( + "raw_table_id", id.rawTableId(QUOTE))).replace( + """ + UPDATE ${raw_table_id} + SET `_airbyte_loaded_at` = CURRENT_TIMESTAMP() + WHERE `_airbyte_loaded_at` IS NULL + ;"""); + } + + @Override + public String overwriteFinalTable(final StreamId streamId, final String finalSuffix) { + return new StringSubstitutor(Map.of( + "final_table_id", streamId.finalTableId(QUOTE), + "tmp_final_table", streamId.finalTableId(QUOTE, finalSuffix), + "real_final_table", streamId.finalName(QUOTE))).replace( + """ + DROP TABLE IF EXISTS ${final_table_id}; + ALTER TABLE ${tmp_final_table} RENAME TO ${real_final_table}; + """); + } + + private String wrapAndQuote(final String namespace, final String tableName) { + return Stream.of(namespace, tableName) + .map(part -> StringUtils.wrap(part, QUOTE)) + .collect(joining(".")); + } + + @Override + public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return new StringSubstitutor(Map.of( + "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), + "dataset_location", datasetLocation, + "v2_raw_table", streamId.rawTableId(QUOTE), + "v1_raw_table", wrapAndQuote(namespace, tableName))).replace( + """ + CREATE SCHEMA IF NOT EXISTS ${raw_namespace} + OPTIONS(location="${dataset_location}"); + + CREATE OR REPLACE TABLE ${v2_raw_table} ( + _airbyte_raw_id STRING, + _airbyte_data STRING, + _airbyte_extracted_at TIMESTAMP, + _airbyte_loaded_at TIMESTAMP + ) + PARTITION BY DATE(_airbyte_extracted_at) + CLUSTER BY _airbyte_extracted_at + AS ( + SELECT + _airbyte_ab_id AS _airbyte_raw_id, + _airbyte_data AS _airbyte_data, + _airbyte_emitted_at AS _airbyte_extracted_at, + CAST(NULL AS TIMESTAMP) AS _airbyte_loaded_at + FROM ${v1_raw_table} + ); + """); + } + + /** + * Does two things: escape single quotes (for use inside sql string literals),and escape double + * quotes (for use inside JSON paths). For example, if a column name is foo'bar"baz, then we want to + * end up with something like {@code SELECT JSON_QUERY(..., '$."foo\'bar\\"baz"')}. Note the + * single-backslash for single-quotes (needed for SQL) and the double-backslash for double-quotes + * (needed for JSON path). + */ + private String escapeColumnNameForJsonPath(final String stringContents) { + // This is not a place of honor. + return stringContents + // Consider the JSON blob {"foo\\bar": 42}. + // This is an object with key foo\bar. + // The JSONPath for this is (something like...?) $."foo\\bar" (i.e. 2 backslashes). + // TODO is that jsonpath correct? + // When we represent that path as a SQL string, the backslashes are doubled (to 4): '$."foo\\\\bar"' + // And we're writing that in a Java string, so we have to type out 8 backslashes: + // "'$.\"foo\\\\\\\\bar\"'" + .replace("\\", "\\\\\\\\") + // Similar situation here: + // a literal " needs to be \" in a JSONPath: $."foo\"bar" + // which is \\" in a SQL string: '$."foo\\"bar"' + // The backslashes become \\\\ in java, and the quote becomes \": "'$.\"foo\\\\\"bar\"'" + .replace("\"", "\\\\\"") + // Here we're escaping a SQL string, so we only need a single backslash (which is 2, beacuse Java). + .replace("'", "\\'"); + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV1V2Migrator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV1V2Migrator.java new file mode 100644 index 000000000000..6f1a06a2a073 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV1V2Migrator.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import io.airbyte.integrations.base.destination.typing_deduping.BaseDestinationV1V2Migrator; +import io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils; +import io.airbyte.integrations.base.destination.typing_deduping.NamespacedTableName; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.destination.bigquery.BigQuerySQLNameTransformer; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class BigQueryV1V2Migrator extends BaseDestinationV1V2Migrator { + + private final BigQuery bq; + + private final BigQuerySQLNameTransformer nameTransformer; + + public BigQueryV1V2Migrator(final BigQuery bq, BigQuerySQLNameTransformer nameTransformer) { + this.bq = bq; + this.nameTransformer = nameTransformer; + } + + @Override + protected boolean doesAirbyteInternalNamespaceExist(StreamConfig streamConfig) { + final var dataset = bq.getDataset(streamConfig.id().rawNamespace()); + return dataset != null && dataset.exists(); + } + + @Override + protected Optional getTableIfExists(String namespace, String tableName) { + Table table = bq.getTable(TableId.of(namespace, tableName)); + return table != null && table.exists() ? Optional.of(table.getDefinition()) : Optional.empty(); + } + + @Override + protected boolean schemaMatchesExpectation(TableDefinition existingTable, Collection expectedColumnNames) { + Set existingSchemaColumns = Optional.ofNullable(existingTable.getSchema()) + .map(schema -> schema.getFields().stream() + .map(Field::getName) + .collect(Collectors.toSet())) + .orElse(Collections.emptySet()); + + return !existingSchemaColumns.isEmpty() && + CollectionUtils.containsAllIgnoreCase(expectedColumnNames, existingSchemaColumns); + } + + @Override + protected NamespacedTableName convertToV1RawName(StreamConfig streamConfig) { + return new NamespacedTableName( + this.nameTransformer.getNamespace(streamConfig.id().originalNamespace()), + this.nameTransformer.getRawTableName(streamConfig.id().originalName())); + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2RawTableMigrator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2RawTableMigrator.java new file mode 100644 index 000000000000..cbbf5e122f37 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2RawTableMigrator.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.V2RawTableMigrator; +import java.util.Map; +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BigQueryV2RawTableMigrator implements V2RawTableMigrator { + + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryV2RawTableMigrator.class); + + private final BigQuery bq; + + public BigQueryV2RawTableMigrator(final BigQuery bq) { + this.bq = bq; + } + + @Override + public void migrateIfNecessary(final StreamConfig streamConfig) throws InterruptedException { + final Table rawTable = bq.getTable(TableId.of(streamConfig.id().rawNamespace(), streamConfig.id().rawName())); + if (rawTable != null && rawTable.exists()) { + final Schema existingRawSchema = rawTable.getDefinition().getSchema(); + final FieldList fields = existingRawSchema.getFields(); + if (fields.stream().noneMatch(f -> JavaBaseConstants.COLUMN_NAME_DATA.equals(f.getName()))) { + throw new IllegalStateException( + "Table does not have a column named _airbyte_data. We are likely colliding with a completely different table."); + } + final Field dataColumn = fields.get(JavaBaseConstants.COLUMN_NAME_DATA); + if (dataColumn.getType() == LegacySQLTypeName.JSON) { + LOGGER.info("Raw table has _airbyte_data of type JSON. Migrating to STRING."); + final String tmpRawTableId = BigQuerySqlGenerator.QUOTE + streamConfig.id().rawNamespace() + BigQuerySqlGenerator.QUOTE + "." + + BigQuerySqlGenerator.QUOTE + streamConfig.id().rawName() + "_airbyte_tmp" + BigQuerySqlGenerator.QUOTE; + bq.query(QueryJobConfiguration.of( + new StringSubstitutor(Map.of( + "raw_table", streamConfig.id().rawTableId(BigQuerySqlGenerator.QUOTE), + "tmp_raw_table", tmpRawTableId, + "real_raw_table", BigQuerySqlGenerator.QUOTE + streamConfig.id().rawName() + BigQuerySqlGenerator.QUOTE)).replace( + // In full refresh / append mode, standard inserts is creating a non-partitioned raw table. + // (possibly also in overwrite mode?). + // We can't just CREATE OR REPLACE the table because bigquery will complain that we're trying to + // change the partitioning scheme. + // Do an explicit CREATE tmp + DROP + RENAME, similar to how we overwrite the final tables in + // OVERWRITE mode. + """ + CREATE TABLE ${tmp_raw_table} + PARTITION BY DATE(_airbyte_extracted_at) + CLUSTER BY _airbyte_extracted_at + AS ( + SELECT + _airbyte_raw_id, + _airbyte_extracted_at, + _airbyte_loaded_at, + to_json_string(_airbyte_data) as _airbyte_data + FROM ${raw_table} + ); + DROP TABLE IF EXISTS ${raw_table}; + ALTER TABLE ${tmp_raw_table} RENAME TO ${real_raw_table}; + """))); + LOGGER.info("Completed Data column Migration for stream {}", streamConfig.id().rawName()); + } else { + LOGGER.info("No Data column Migration Required for stream {}", streamConfig.id().rawName()); + } + } + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java index f61a2a9bd023..390e2f7fdfd8 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java @@ -14,9 +14,13 @@ import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; import io.airbyte.commons.string.Strings; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.s3.writer.DestinationWriter; @@ -38,6 +42,7 @@ public abstract class AbstractBigQueryUploader { protected final T writer; protected final BigQuery bigQuery; protected final BigQueryRecordFormatter recordFormatter; + protected final boolean use1s1t; AbstractBigQueryUploader(final TableId table, final TableId tmpTable, @@ -45,6 +50,7 @@ public abstract class AbstractBigQueryUploader { final WriteDisposition syncMode, final BigQuery bigQuery, final BigQueryRecordFormatter recordFormatter) { + this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); this.table = table; this.tmpTable = tmpTable; this.writer = writer; @@ -96,9 +102,13 @@ public void close(final boolean hasFailed, final Consumer output protected void uploadData(final Consumer outputRecordCollector, final AirbyteMessage lastStateMessage) throws Exception { try { - LOGGER.info("Uploading data from the tmp table {} to the source table {}.", tmpTable.getTable(), table.getTable()); - uploadDataToTableFromTmpTable(); - LOGGER.info("Data is successfully loaded to the source table {}!", table.getTable()); + if (!use1s1t) { + // This only needs to happen if we actually wrote to a tmp table. + LOGGER.info("Uploading data from the tmp table {} to the source table {}.", tmpTable.getTable(), table.getTable()); + uploadDataToTableFromTmpTable(); + LOGGER.info("Data is successfully loaded to the source table {}!", table.getTable()); + } + outputRecordCollector.accept(lastStateMessage); LOGGER.info("Final state message is accepted."); } catch (final Exception e) { @@ -109,6 +119,18 @@ protected void uploadData(final Consumer outputRecordCollector, } } + public void createRawTable() { + // Ensure that this table exists. + // TODO alter an existing raw table? + final Table rawTable = bigQuery.getTable(table); + if (rawTable == null) { + LOGGER.info("Creating raw table {}.", table); + bigQuery.create(TableInfo.newBuilder(table, StandardTableDefinition.of(recordFormatter.getBigQuerySchema())).build()); + } else { + LOGGER.info("Found raw table {}.", rawTable.getTableId()); + } + } + protected void dropTmpTable() { try { // clean up tmp tables; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java index 3bd481c03b58..81a4641395ff 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java @@ -4,7 +4,9 @@ package io.airbyte.integrations.destination.bigquery.uploader; -import com.google.cloud.bigquery.*; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.TableId; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.writer.BigQueryTableWriter; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java index 67d52a68334c..3ee2fdafa23b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java @@ -20,6 +20,7 @@ import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.WriteChannelConfiguration; import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; @@ -31,9 +32,13 @@ import java.sql.Timestamp; import java.util.HashSet; import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BigQueryUploaderFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryUploaderFactory.class); + private static final String CONFIG_ERROR_MSG = """ Failed to write to destination schema. @@ -50,20 +55,27 @@ public class BigQueryUploaderFactory { public static AbstractBigQueryUploader getUploader(final UploaderConfig uploaderConfig) throws IOException { - final String schemaName = BigQueryUtils.getSchema(uploaderConfig.getConfig(), uploaderConfig.getConfigStream()); + final String dataset; + if (TypingAndDedupingFlag.isDestinationV2()) { + dataset = uploaderConfig.getParsedStream().id().rawNamespace(); + } else { + // This previously needed to handle null namespaces. That's now happening at the top of the + // connector, so we can assume namespace is non-null here. + dataset = BigQueryUtils.sanitizeDatasetId(uploaderConfig.getConfigStream().getStream().getNamespace()); + } final String datasetLocation = BigQueryUtils.getDatasetLocation(uploaderConfig.getConfig()); - final Set existingSchemas = new HashSet<>(); + final Set existingDatasets = new HashSet<>(); final BigQueryRecordFormatter recordFormatter = uploaderConfig.getFormatter(); final Schema bigQuerySchema = recordFormatter.getBigQuerySchema(); - final TableId targetTable = TableId.of(schemaName, uploaderConfig.getTargetTableName()); - final TableId tmpTable = TableId.of(schemaName, uploaderConfig.getTmpTableName()); + final TableId targetTable = TableId.of(dataset, uploaderConfig.getTargetTableName()); + final TableId tmpTable = TableId.of(dataset, uploaderConfig.getTmpTableName()); BigQueryUtils.createSchemaAndTableIfNeeded( uploaderConfig.getBigQuery(), - existingSchemas, - schemaName, + existingDatasets, + dataset, tmpTable, datasetLocation, bigQuerySchema); @@ -146,8 +158,10 @@ private static BigQueryDirectUploader getBigQueryDirectUploader( final String datasetLocation, final BigQueryRecordFormatter formatter) { // https://cloud.google.com/bigquery/docs/loading-data-local#loading_data_from_a_local_data_source + final TableId tableToWriteRawData = TypingAndDedupingFlag.isDestinationV2() ? targetTable : tmpTable; + LOGGER.info("Will write raw data to {} with schema {}", tableToWriteRawData, formatter.getBigQuerySchema()); final WriteChannelConfiguration writeChannelConfiguration = - WriteChannelConfiguration.newBuilder(tmpTable) + WriteChannelConfiguration.newBuilder(tableToWriteRawData) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .setSchema(formatter.getBigQuerySchema()) .setFormatOptions(FormatOptions.json()) diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java index ab13465c9cc6..6dad08ee4e11 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.cloud.bigquery.BigQuery; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.UploadingMethod; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; @@ -20,7 +21,15 @@ public class UploaderConfig { private JsonNode config; + /** + * Taken directly from the {@link ConfiguredAirbyteStream}, except if the namespace was null, we set + * it to the destination default namespace. + */ private ConfiguredAirbyteStream configStream; + /** + * Parsed directly from {@link #configStream}. + */ + private StreamConfig parsedStream; private String targetTableName; private String tmpTableName; private BigQuery bigQuery; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index e458ebfaa863..da8b9d83093b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -206,6 +206,18 @@ "default": 15, "examples": ["15"], "order": 6 + }, + "use_1s1t_format": { + "type": "boolean", + "description": "(Early Access) Use Destinations V2.", + "title": "Use Destinations V2 (Early Access)", + "order": 7 + }, + "raw_data_dataset": { + "type": "string", + "description": "(Early Access) The dataset to write raw tables into", + "title": "Destinations V2 Raw Table Dataset (Early Access)", + "order": 8 } } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java index 6df9b73c91d8..f15a5f30072e 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java @@ -9,6 +9,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQuery.DatasetListOption; +import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.ConnectionProperty; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.FieldList; @@ -23,6 +26,7 @@ import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.TestingNamespaces; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.nio.file.Path; @@ -115,9 +119,18 @@ protected void assertNamespaceNormalization(final String testCaseId, final String actualNormalizedNamespace) { final String message = String.format("Test case %s failed; if this is expected, please override assertNamespaceNormalization", testCaseId); if (testCaseId.equals("S3A-1")) { - // bigquery allows namespace starting with a number, and prepending underscore - // will hide the dataset, so we don't do it as we do for other destinations - assertEquals("99namespace", actualNormalizedNamespace, message); + /* + * See NamespaceTestCaseProvider for how this suffix is generated.

    expectedNormalizedNamespace + * will look something like this: `_99namespace_test_20230824_bicrt`. We want to grab the part after + * `_99namespace`. + */ + final int underscoreIndex = expectedNormalizedNamespace.indexOf("_", 1); + final String randomSuffix = expectedNormalizedNamespace.substring(underscoreIndex); + /* + * bigquery allows namespace starting with a number, and prepending underscore will hide the + * dataset, so we don't do it as we do for other destinations + */ + assertEquals("99namespace" + randomSuffix, actualNormalizedNamespace, message); } else { assertEquals(expectedNormalizedNamespace, actualNormalizedNamespace, message); } @@ -173,13 +186,31 @@ protected void setUpBigQuery() throws IOException { // secrets file should be set by the inhereting class Assertions.assertNotNull(secretsFile); final String datasetId = Strings.addRandomSuffix("airbyte_tests", "_", 8); - config = BigQueryDestinationTestUtils.createConfig(secretsFile, datasetId); + final String stagingPathSuffix = Strings.addRandomSuffix("test_path", "_", 8); + config = BigQueryDestinationTestUtils.createConfig(secretsFile, datasetId, stagingPathSuffix); final String projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); bigquery = BigQueryDestinationTestUtils.initBigQuery(config, projectId); dataset = BigQueryDestinationTestUtils.initDataSet(config, bigquery, datasetId); } + protected void removeOldNamespaces() { + int datasetsDeletedCount = 0; + // todo (cgardens) - hardcoding to testing project to de-risk this running somewhere unexpected. + for (final Dataset dataset1 : bigquery.listDatasets("dataline-integration-testing", DatasetListOption.all()) + .iterateAll()) { + if (TestingNamespaces.isOlderThan2Days(dataset1.getDatasetId().getDataset())) { + try { + bigquery.delete(dataset1.getDatasetId(), DatasetDeleteOption.deleteContents()); + datasetsDeletedCount++; + } catch (final BigQueryException e) { + LOGGER.error("Failed to delete old dataset: {}", dataset1.getDatasetId().getDataset(), e); + } + } + } + LOGGER.info("Deleted {} old datasets.", datasetsDeletedCount); + } + protected void tearDownBigQuery() { BigQueryDestinationTestUtils.tearDownBigQuery(bigquery, dataset, LOGGER); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index de865f5ad80e..de75a017a3d7 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -30,6 +30,7 @@ import io.airbyte.commons.string.Strings; import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; @@ -171,30 +172,33 @@ public static void beforeAll() throws IOException { } datasetId = Strings.addRandomSuffix(DATASET_NAME_PREFIX, "_", 8); + String stagingPath = Strings.addRandomSuffix("test_path", "_", 8); // Set up config objects for test scenarios // config - basic config for standard inserts that should succeed check and write tests // this config is also used for housekeeping (checking records, and cleaning up) - config = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_STANDARD_INSERT_PATH, datasetId); + config = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_STANDARD_INSERT_PATH, datasetId, stagingPath); + + DestinationConfig.initialize(config); // all successful configs use the same project ID projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); // configWithProjectId - config that uses project:dataset notation for datasetId final String dataSetWithProjectId = String.format("%s:%s", projectId, datasetId); - configWithProjectId = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_STANDARD_INSERT_PATH, dataSetWithProjectId); + configWithProjectId = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_STANDARD_INSERT_PATH, dataSetWithProjectId, stagingPath); // configWithBadProjectId - config that uses "fake" project ID and should fail final String dataSetWithBadProjectId = String.format("%s:%s", "fake", datasetId); - configWithBadProjectId = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_BAD_PROJECT_PATH, dataSetWithBadProjectId); + configWithBadProjectId = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_BAD_PROJECT_PATH, dataSetWithBadProjectId, stagingPath); // config that has insufficient privileges - insufficientRoleConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_NO_DATASET_CREATION_PATH, datasetId); + insufficientRoleConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_NO_DATASET_CREATION_PATH, datasetId, stagingPath); // config that tries to write to a project with disabled billing (free tier) - nonBillableConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_NON_BILLABLE_PROJECT_PATH, "testnobilling"); + nonBillableConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_NON_BILLABLE_PROJECT_PATH, "testnobilling", stagingPath); // config that has no privileges to edit anything in Public schema - noEditPublicSchemaRoleConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_NO_EDIT_PUBLIC_SCHEMA_ROLE_PATH, "public"); + noEditPublicSchemaRoleConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_NO_EDIT_PUBLIC_SCHEMA_ROLE_PATH, "public", stagingPath); // config with GCS staging - gcsStagingConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_WITH_GCS_STAGING_PATH, datasetId); + gcsStagingConfig = BigQueryDestinationTestUtils.createConfig(CREDENTIALS_WITH_GCS_STAGING_PATH, datasetId, stagingPath); MESSAGE_USERS1.getRecord().setNamespace(datasetId); MESSAGE_USERS2.getRecord().setNamespace(datasetId); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTestUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTestUtils.java index cdc5e042078f..912f18316d9f 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTestUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTestUtils.java @@ -21,21 +21,38 @@ import java.util.LinkedList; import java.util.List; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BigQueryDestinationTestUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDestinationTestUtils.class); + /** - * Parse the config file and replace dataset with datasetId randomly generated by the test + * Parse the config file and replace dataset with datasetId and stagingPath randomly generated by + * the test. * - * @param configFile - * @param datasetId - * @return - * @throws IOException + * @param configFile Path to the config file + * @param datasetId Dataset id to use in the test. Should be randomized per test case. + * @param stagingPath Staging GCS path to use in the test, or null if the test is running in + * standard inserts mode. Should be randomized per test case. */ - public static JsonNode createConfig(Path configFile, String datasetId) throws IOException { + public static ObjectNode createConfig(Path configFile, String datasetId, String stagingPath) throws IOException { + LOGGER.info("Setting default dataset to {}", datasetId); final String tmpConfigAsString = Files.readString(configFile); - final JsonNode tmpConfigJson = Jsons.deserialize(tmpConfigAsString); - return Jsons.jsonNode(((ObjectNode) tmpConfigJson).put(BigQueryConsts.CONFIG_DATASET_ID, datasetId)); + final ObjectNode config = (ObjectNode) Jsons.deserialize(tmpConfigAsString); + config.put(BigQueryConsts.CONFIG_DATASET_ID, datasetId); + + // This is sort of a hack. Ideally tests shouldn't interfere with each other even when using the + // same staging path. + // Most likely there's a real bug in the connector - but we should investigate that and write a real + // test case, + // rather than relying on tests randomly failing to indicate that bug. + // See https://github.com/airbytehq/airbyte/issues/28372. + if (stagingPath != null && BigQueryUtils.getLoadingMethod(config) == UploadingMethod.GCS) { + ObjectNode loadingMethodNode = (ObjectNode) config.get(BigQueryConsts.LOADING_METHOD); + loadingMethodNode.put(BigQueryConsts.GCS_BUCKET_PATH, stagingPath); + } + return config; } /** diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java index 5f69f0ccee36..eb60ef15aaa7 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java @@ -11,10 +11,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; +import java.util.HashSet; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; @@ -30,15 +32,19 @@ public class BigQueryGcsDestinationAcceptanceTest extends AbstractBigQueryDestin * Sets up secretsFile path as well as BigQuery and GCS instances for verification and cleanup This * function will be called before EACH test. * - * @see DestinationAcceptanceTest#setUpInternal() * @param testEnv - information about the test environment. + * @param TEST_SCHEMAS * @throws Exception - can throw any exception, test framework will handle. + * @see DestinationAcceptanceTest#setUpInternal() */ @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { // use secrets file with GCS staging config secretsFile = Path.of("secrets/credentials-gcs-staging.json"); setUpBigQuery(); + removeOldNamespaces(); + + DestinationConfig.initialize(config); // the setup steps below are specific to GCS staging use case final GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig @@ -49,12 +55,12 @@ protected void setup(TestDestinationEnv testEnv) throws Exception { /** * Removes data from bigquery and GCS This function will be called after EACH test * - * @see DestinationAcceptanceTest#tearDownInternal() * @param testEnv - information about the test environment. * @throws Exception - can throw any exception, test framework will handle. + * @see DestinationAcceptanceTest#tearDownInternal() */ @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { tearDownBigQuery(); tearDownGcs(); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java index b7d1197df089..5e106e5b8caa 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java @@ -8,6 +8,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; +import java.util.HashSet; import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,25 +22,27 @@ public class BigQueryStandardDestinationAcceptanceTest extends AbstractBigQueryD * Sets up secretsFile path and BigQuery instance for verification and cleanup This function will be * called before EACH test. * - * @see DestinationAcceptanceTest#setUpInternal() * @param testEnv - information about the test environment. + * @param TEST_SCHEMAS * @throws Exception - can throw any exception, test framework will handle. + * @see DestinationAcceptanceTest#setUpInternal() */ @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { secretsFile = Path.of("secrets/credentials-standard.json"); setUpBigQuery(); + removeOldNamespaces(); } /** * Removes data from bigquery This function will be called after EACH test * - * @see DestinationAcceptanceTest#tearDownInternal() * @param testEnv - information about the test environment. * @throws Exception - can throw any exception, test framework will handle. + * @see DestinationAcceptanceTest#tearDownInternal() */ @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { tearDownBigQuery(); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java new file mode 100644 index 000000000000..ffc5104f5cd7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.bigquery.BigQueryDestination; +import io.airbyte.integrations.destination.bigquery.BigQueryDestinationTestUtils; +import io.airbyte.integrations.destination.bigquery.BigQueryUtils; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import org.junit.jupiter.api.Test; + +public abstract class AbstractBigQueryTypingDedupingTest extends BaseTypingDedupingTest { + + private BigQuery bq; + + protected abstract String getConfigPath(); + + @Override + public JsonNode generateConfig() throws IOException { + final String datasetId = "typing_deduping_default_dataset" + getUniqueSuffix(); + final String stagingPath = "test_path" + getUniqueSuffix(); + final ObjectNode config = BigQueryDestinationTestUtils.createConfig(Path.of(getConfigPath()), datasetId, stagingPath); + bq = BigQueryDestination.getBigQuery(config); + return config; + } + + @Override + protected String getImageName() { + return "airbyte/destination-bigquery:dev"; + } + + @Override + protected List dumpRawTableRecords(String streamNamespace, final String streamName) throws InterruptedException { + if (streamNamespace == null) { + streamNamespace = BigQueryUtils.getDatasetId(getConfig()); + } + final TableResult result = + bq.query(QueryJobConfiguration.of("SELECT * FROM " + getRawDataset() + "." + StreamId.concatenateRawTableName(streamNamespace, streamName))); + return BigQuerySqlGeneratorIntegrationTest.toJsonRecords(result); + } + + @Override + protected List dumpFinalTableRecords(String streamNamespace, final String streamName) throws InterruptedException { + if (streamNamespace == null) { + streamNamespace = BigQueryUtils.getDatasetId(getConfig()); + } + final TableResult result = bq.query(QueryJobConfiguration.of("SELECT * FROM " + streamNamespace + "." + streamName)); + return BigQuerySqlGeneratorIntegrationTest.toJsonRecords(result); + } + + @Override + protected void teardownStreamAndNamespace(String streamNamespace, final String streamName) { + if (streamNamespace == null) { + streamNamespace = BigQueryUtils.getDatasetId(getConfig()); + } + // bq.delete simply returns false if the table/schema doesn't exist (e.g. if the connector failed to + // create it) + // so we don't need to do any existence checks here. + bq.delete(TableId.of(getRawDataset(), StreamId.concatenateRawTableName(streamNamespace, streamName))); + bq.delete(DatasetId.of(streamNamespace), BigQuery.DatasetDeleteOption.deleteContents()); + } + + /** + * Run a sync using 1.9.0 (which is the highest version that still creates v2 raw tables with JSON + * _airbyte_data). Then run a sync using our current version. + */ + @Test + public void testRawTableJsonToStringMigration() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1, "airbyte/destination-bigquery:1.9.0"); + + // 1.9.0 is known-good, but we might as well check that we're in good shape before continuing. + // If this starts erroring out because we added more test records and 1.9.0 had a latent bug, + // just delete these three lines :P + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + } + + /** + * Subclasses using a config with a nonstandard raw table dataset should override this method. + */ + protected String getRawDataset() { + return JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsTypingDedupingTest.java new file mode 100644 index 000000000000..40ddb07b7f36 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsTypingDedupingTest.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +public class BigQueryGcsTypingDedupingTest extends AbstractBigQueryTypingDedupingTest { + + @Override + public String getConfigPath() { + return "secrets/credentials-1s1t-gcs.json"; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..b9a30769fb9b --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java @@ -0,0 +1,506 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import static com.google.cloud.bigquery.LegacySQLTypeName.legacySQLTypeName; +import static java.util.stream.Collectors.joining; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableResult; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.bigquery.BigQueryDestination; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.commons.text.StringSubstitutor; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Execution(ExecutionMode.CONCURRENT) +public class BigQuerySqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BigQuerySqlGeneratorIntegrationTest.class); + + private static BigQuery bq; + + @BeforeAll + public static void setupBigquery() throws Exception { + final String rawConfig = Files.readString(Path.of("secrets/credentials-gcs-staging.json")); + final JsonNode config = Jsons.deserialize(rawConfig); + bq = BigQueryDestination.getBigQuery(config); + } + + @Override + protected BigQuerySqlGenerator getSqlGenerator() { + return new BigQuerySqlGenerator("US"); + } + + @Override + protected BigQueryDestinationHandler getDestinationHandler() { + return new BigQueryDestinationHandler(bq, "US"); + } + + @Override + protected void createNamespace(final String namespace) { + bq.create(DatasetInfo.newBuilder(namespace) + // This unfortunately doesn't delete the actual dataset after 3 days, but at least we'll clear out + // old tables automatically + .setDefaultTableLifetime(Duration.ofDays(3).toMillis()) + .build()); + } + + @Override + protected void createRawTable(final StreamId streamId) throws InterruptedException { + bq.query(QueryJobConfiguration.newBuilder( + new StringSubstitutor(Map.of( + "raw_table_id", streamId.rawTableId(BigQuerySqlGenerator.QUOTE))).replace( + """ + CREATE TABLE ${raw_table_id} ( + _airbyte_raw_id STRING NOT NULL, + _airbyte_data STRING NOT NULL, + _airbyte_extracted_at TIMESTAMP NOT NULL, + _airbyte_loaded_at TIMESTAMP + ) PARTITION BY ( + DATE_TRUNC(_airbyte_extracted_at, DAY) + ) CLUSTER BY _airbyte_loaded_at; + """)) + .build()); + } + + @Override + protected void createV1RawTable(final StreamId v1RawTable) throws Exception { + bq.query( + QueryJobConfiguration + .newBuilder( + new StringSubstitutor(Map.of( + "raw_table_id", v1RawTable.rawTableId(BigQuerySqlGenerator.QUOTE))).replace( + """ + CREATE TABLE ${raw_table_id} ( + _airbyte_ab_id STRING NOT NULL, + _airbyte_data STRING NOT NULL, + _airbyte_emitted_at TIMESTAMP NOT NULL, + ) PARTITION BY ( + DATE_TRUNC(_airbyte_emitted_at, DAY) + ) CLUSTER BY _airbyte_emitted_at; + """)) + .build()); + } + + @Override + protected void createFinalTable(final boolean includeCdcDeletedAt, final StreamId streamId, final String suffix) throws InterruptedException { + final String cdcDeletedAt = includeCdcDeletedAt ? "`_ab_cdc_deleted_at` TIMESTAMP," : ""; + bq.query(QueryJobConfiguration.newBuilder( + new StringSubstitutor(Map.of( + "final_table_id", streamId.finalTableId(BigQuerySqlGenerator.QUOTE, suffix), + "cdc_deleted_at", cdcDeletedAt)).replace( + """ + CREATE TABLE ${final_table_id} ( + _airbyte_raw_id STRING NOT NULL, + _airbyte_extracted_at TIMESTAMP NOT NULL, + _airbyte_meta JSON NOT NULL, + `id1` INT64, + `id2` INT64, + `updated_at` TIMESTAMP, + ${cdc_deleted_at} + `struct` JSON, + `array` JSON, + `string` STRING, + `number` NUMERIC, + `integer` INT64, + `boolean` BOOL, + `timestamp_with_timezone` TIMESTAMP, + `timestamp_without_timezone` DATETIME, + `time_with_timezone` STRING, + `time_without_timezone` TIME, + `date` DATE, + `unknown` JSON + ) + PARTITION BY (DATE_TRUNC(_airbyte_extracted_at, DAY)) + CLUSTER BY id1, id2, _airbyte_extracted_at; + """)) + .build()); + } + + @Override + protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, + final StreamId streamId, + final String suffix, + final List records) + throws InterruptedException { + final List columnNames = includeCdcDeletedAt ? FINAL_TABLE_COLUMN_NAMES_CDC : FINAL_TABLE_COLUMN_NAMES; + final String cdcDeletedAtDecl = includeCdcDeletedAt ? ",`_ab_cdc_deleted_at` TIMESTAMP" : ""; + final String cdcDeletedAtName = includeCdcDeletedAt ? ",`_ab_cdc_deleted_at`" : ""; + final String recordsText = records.stream() + // For each record, convert it to a string like "(rawId, extractedAt, loadedAt, data)" + .map(record -> columnNames.stream() + .map(record::get) + .map(r -> { + if (r == null) { + return "NULL"; + } + final String stringContents; + if (r.isTextual()) { + stringContents = r.asText(); + } else { + stringContents = r.toString(); + } + return '"' + stringContents + // Serialized json might contain backslashes and double quotes. Escape them. + .replace("\\", "\\\\") + .replace("\"", "\\\"") + '"'; + }) + .collect(joining(","))) + .map(row -> "(" + row + ")") + .collect(joining(",")); + + bq.query(QueryJobConfiguration.newBuilder( + new StringSubstitutor(Map.of( + "final_table_id", streamId.finalTableId(BigQuerySqlGenerator.QUOTE, suffix), + "cdc_deleted_at_name", cdcDeletedAtName, + "cdc_deleted_at_decl", cdcDeletedAtDecl, + "records", recordsText)).replace( + // Similar to insertRawTableRecords, some of these columns are declared as string and wrapped in + // parse_json(). + // There's also a bunch of casting, because bigquery doesn't coerce strings to e.g. int + """ + insert into ${final_table_id} ( + _airbyte_raw_id, + _airbyte_extracted_at, + _airbyte_meta, + `id1`, + `id2`, + `updated_at`, + `struct`, + `array`, + `string`, + `number`, + `integer`, + `boolean`, + `timestamp_with_timezone`, + `timestamp_without_timezone`, + `time_with_timezone`, + `time_without_timezone`, + `date`, + `unknown` + ${cdc_deleted_at_name} + ) + select + _airbyte_raw_id, + _airbyte_extracted_at, + parse_json(_airbyte_meta), + cast(`id1` as int64), + cast(`id2` as int64), + `updated_at`, + parse_json(`struct`), + parse_json(`array`), + `string`, + cast(`number` as numeric), + cast(`integer` as int64), + cast(`boolean` as boolean), + `timestamp_with_timezone`, + `timestamp_without_timezone`, + `time_with_timezone`, + `time_without_timezone`, + `date`, + parse_json(`unknown`) + ${cdc_deleted_at_name} + from unnest([ + STRUCT< + _airbyte_raw_id STRING, + _airbyte_extracted_at TIMESTAMP, + _airbyte_meta STRING, + `id1` STRING, + `id2` STRING, + `updated_at` TIMESTAMP, + `struct` STRING, + `array` STRING, + `string` STRING, + `number` STRING, + `integer` STRING, + `boolean` STRING, + `timestamp_with_timezone` TIMESTAMP, + `timestamp_without_timezone` DATETIME, + `time_with_timezone` STRING, + `time_without_timezone` TIME, + `date` DATE, + `unknown` STRING + ${cdc_deleted_at_decl} + > + ${records} + ]) + """)) + .build()); + } + + private String stringifyRecords(final List records, final List columnNames) { + return records.stream() + // For each record, convert it to a string like "(rawId, extractedAt, loadedAt, data)" + .map(record -> columnNames.stream() + .map(record::get) + .map(r -> { + if (r == null) { + return "NULL"; + } + final String stringContents; + if (r.isTextual()) { + stringContents = r.asText(); + } else { + stringContents = r.toString(); + } + return '"' + stringContents + // Serialized json might contain backslashes and double quotes. Escape them. + .replace("\\", "\\\\") + .replace("\"", "\\\"") + '"'; + }) + .collect(joining(","))) + .map(row -> "(" + row + ")") + .collect(joining(",")); + } + + @Override + protected void insertRawTableRecords(final StreamId streamId, final List records) throws InterruptedException { + final String recordsText = stringifyRecords(records, JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES); + + bq.query(QueryJobConfiguration.newBuilder( + new StringSubstitutor(Map.of( + "raw_table_id", streamId.rawTableId(BigQuerySqlGenerator.QUOTE), + "records", recordsText)).replace( + // Note the parse_json call, and that _airbyte_data is declared as a string. + // This is needed because you can't insert a string literal into a JSON column + // so we build a struct literal with a string field, and then parse the field when inserting to the + // table. + """ + INSERT INTO ${raw_table_id} (_airbyte_raw_id, _airbyte_extracted_at, _airbyte_loaded_at, _airbyte_data) + SELECT _airbyte_raw_id, _airbyte_extracted_at, _airbyte_loaded_at, _airbyte_data FROM UNNEST([ + STRUCT<`_airbyte_raw_id` STRING, `_airbyte_extracted_at` TIMESTAMP, `_airbyte_loaded_at` TIMESTAMP, _airbyte_data STRING> + ${records} + ]) + """)) + .build()); + } + + @Override + protected void insertV1RawTableRecords(final StreamId streamId, final List records) throws Exception { + final String recordsText = stringifyRecords(records, JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS); + bq.query( + QueryJobConfiguration + .newBuilder( + new StringSubstitutor(Map.of( + "v1_raw_table_id", streamId.rawTableId(BigQuerySqlGenerator.QUOTE), + "records", recordsText)).replace( + """ + INSERT INTO ${v1_raw_table_id} (_airbyte_ab_id, _airbyte_data, _airbyte_emitted_at) + SELECT _airbyte_ab_id, _airbyte_data, _airbyte_emitted_at FROM UNNEST([ + STRUCT<`_airbyte_ab_id` STRING, _airbyte_data STRING, `_airbyte_emitted_at` TIMESTAMP> + ${records} + ]) + """)) + .build()); + } + + @Override + protected List dumpRawTableRecords(final StreamId streamId) throws Exception { + final TableResult result = bq.query(QueryJobConfiguration.of("SELECT * FROM " + streamId.rawTableId(BigQuerySqlGenerator.QUOTE))); + return BigQuerySqlGeneratorIntegrationTest.toJsonRecords(result).stream().peek(record -> { + final JsonNode deserializedData = Jsons.deserializeExact(record.get("_airbyte_data").asText()); + ((ObjectNode) record).set("_airbyte_data", deserializedData); + }).toList(); + } + + @Override + protected List dumpFinalTableRecords(final StreamId streamId, final String suffix) throws Exception { + final TableResult result = bq.query(QueryJobConfiguration.of("SELECT * FROM " + streamId.finalTableId(BigQuerySqlGenerator.QUOTE, suffix))); + return BigQuerySqlGeneratorIntegrationTest.toJsonRecords(result); + } + + @Override + protected void teardownNamespace(final String namespace) { + bq.delete(namespace, BigQuery.DatasetDeleteOption.deleteContents()); + } + + @Override + @Test + public void testCreateTableIncremental() throws Exception { + destinationHandler.execute(generator.createTable(incrementalDedupStream, "", false)); + + final Table table = bq.getTable(namespace, "users_final"); + // The table should exist + assertNotNull(table); + final Schema schema = table.getDefinition().getSchema(); + // And we should know exactly what columns it contains + assertEquals( + // Would be nice to assert directly against StandardSQLTypeName, but bigquery returns schemas of + // LegacySQLTypeName. So we have to translate. + Schema.of( + Field.newBuilder("_airbyte_raw_id", legacySQLTypeName(StandardSQLTypeName.STRING)).setMode(Field.Mode.REQUIRED).build(), + Field.newBuilder("_airbyte_extracted_at", legacySQLTypeName(StandardSQLTypeName.TIMESTAMP)).setMode(Field.Mode.REQUIRED).build(), + Field.newBuilder("_airbyte_meta", legacySQLTypeName(StandardSQLTypeName.JSON)).setMode(Field.Mode.REQUIRED).build(), + Field.of("id1", legacySQLTypeName(StandardSQLTypeName.INT64)), + Field.of("id2", legacySQLTypeName(StandardSQLTypeName.INT64)), + Field.of("updated_at", legacySQLTypeName(StandardSQLTypeName.TIMESTAMP)), + Field.of("struct", legacySQLTypeName(StandardSQLTypeName.JSON)), + Field.of("array", legacySQLTypeName(StandardSQLTypeName.JSON)), + Field.of("string", legacySQLTypeName(StandardSQLTypeName.STRING)), + Field.of("number", legacySQLTypeName(StandardSQLTypeName.NUMERIC)), + Field.of("integer", legacySQLTypeName(StandardSQLTypeName.INT64)), + Field.of("boolean", legacySQLTypeName(StandardSQLTypeName.BOOL)), + Field.of("timestamp_with_timezone", legacySQLTypeName(StandardSQLTypeName.TIMESTAMP)), + Field.of("timestamp_without_timezone", legacySQLTypeName(StandardSQLTypeName.DATETIME)), + Field.of("time_with_timezone", legacySQLTypeName(StandardSQLTypeName.STRING)), + Field.of("time_without_timezone", legacySQLTypeName(StandardSQLTypeName.TIME)), + Field.of("date", legacySQLTypeName(StandardSQLTypeName.DATE)), + Field.of("unknown", legacySQLTypeName(StandardSQLTypeName.JSON))), + schema); + // TODO this should assert partitioning/clustering configs + } + + @Test + public void testCreateTableInOtherRegion() throws InterruptedException { + final BigQueryDestinationHandler destinationHandler = new BigQueryDestinationHandler(bq, "asia-east1"); + // We're creating the dataset in the wrong location in the @BeforeEach block. Explicitly delete it. + bq.getDataset(namespace).delete(); + + destinationHandler.execute(new BigQuerySqlGenerator("asia-east1").createTable(incrementalDedupStream, "", false)); + + // Empirically, it sometimes takes Bigquery nearly 30 seconds to propagate the dataset's existence. + // Give ourselves 2 minutes just in case. + for (int i = 0; i < 120; i++) { + final Dataset dataset = bq.getDataset(DatasetId.of(bq.getOptions().getProjectId(), namespace)); + if (dataset == null) { + LOGGER.info("Sleeping and trying again... ({})", i); + Thread.sleep(1000); + } else { + assertEquals("asia-east1", dataset.getLocation()); + return; + } + } + fail("Dataset does not exist"); + } + + /** + * Bigquery column names aren't allowed to start with certain prefixes. Verify that we throw an + * error in these cases. + */ + @ParameterizedTest + @ValueSource(strings = { + "_table_", + "_file_", + "_partition_", + "_row_timestamp_", + "__root__", + "_colidentifier_" + }) + public void testFailureOnReservedColumnNamePrefix(final String prefix) { + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + null, + Optional.empty(), + new LinkedHashMap<>() { + + { + put(generator.buildColumnId(prefix + "the_column_name"), AirbyteProtocolType.STRING); + } + + }); + + final String createTable = generator.createTable(stream, "", false); + assertThrows( + BigQueryException.class, + () -> destinationHandler.execute(createTable)); + } + + /** + * Something about this test is borked on bigquery. It fails because the raw table doesn't exist, + * but you can go into the UI and see that it does exist. + */ + @Override + @Disabled + public void noCrashOnSpecialCharacters(final String specialChars) throws Exception { + super.noCrashOnSpecialCharacters(specialChars); + } + + /** + * TableResult contains records in a somewhat nonintuitive format (and it avoids loading them all + * into memory). That's annoying for us since we're working with small test data, so just pull + * everything into a list. + */ + public static List toJsonRecords(final TableResult result) { + return result.streamAll().map(row -> toJson(result.getSchema(), row)).toList(); + } + + /** + * FieldValueList stores everything internally as string (I think?) but provides conversions to more + * useful types. This method does that conversion, using the schema to determine which type is most + * appropriate. Then we just dump everything into a jsonnode for interop with RecordDiffer. + */ + private static JsonNode toJson(final Schema schema, final FieldValueList row) { + final ObjectNode json = (ObjectNode) Jsons.emptyObject(); + for (int i = 0; i < schema.getFields().size(); i++) { + final Field field = schema.getFields().get(i); + final FieldValue value = row.get(i); + final JsonNode typedValue; + if (!value.isNull()) { + typedValue = switch (field.getType().getStandardType()) { + case BOOL -> Jsons.jsonNode(value.getBooleanValue()); + case INT64 -> Jsons.jsonNode(value.getLongValue()); + case FLOAT64 -> Jsons.jsonNode(value.getDoubleValue()); + case NUMERIC, BIGNUMERIC -> Jsons.jsonNode(value.getNumericValue()); + case STRING -> Jsons.jsonNode(value.getStringValue()); + // naively converting an Instant returns a DecimalNode with the unix epoch, so instead we manually + // stringify it + case TIMESTAMP -> Jsons.jsonNode(value.getTimestampInstant().toString()); + // value.getTimestampInstant() fails to parse these types + case DATE, DATETIME, TIME -> Jsons.jsonNode(value.getStringValue()); + // bigquery returns JSON columns as string; manually parse it into a JsonNode + case JSON -> Jsons.jsonNode(Jsons.deserializeExact(value.getStringValue())); + + // Default case for weird types (struct, array, geography, interval, bytes) + default -> Jsons.jsonNode(value.getStringValue()); + }; + json.set(field.getName(), typedValue); + } + } + return json; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsRawOverrideTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsRawOverrideTypingDedupingTest.java new file mode 100644 index 000000000000..9c629902a022 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsRawOverrideTypingDedupingTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +public class BigQueryStandardInsertsRawOverrideTypingDedupingTest extends AbstractBigQueryTypingDedupingTest { + + @Override + public String getConfigPath() { + return "secrets/credentials-1s1t-standard-raw-override.json"; + } + + @Override + protected String getRawDataset() { + return "overridden_raw_dataset"; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsTypingDedupingTest.java new file mode 100644 index 000000000000..29b5f0f44f39 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsTypingDedupingTest.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +public class BigQueryStandardInsertsTypingDedupingTest extends AbstractBigQueryTypingDedupingTest { + + @Override + public String getConfigPath() { + return "secrets/credentials-1s1t-standard.json"; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..6ea7612c5abc --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..a9bf479e4e3e --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..e456f48d443a --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -0,0 +1,4 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..88411c9e4de3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,4 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl new file mode 100644 index 000000000000..623527f41e75 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +// Invalid columns are nulled out (i.e. SQL null, not JSON null) +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl new file mode 100644 index 000000000000..4b4db08115e5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +// Note the duplicate record. In this sync mode, we don't dedup anything. +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..7fa0d8339a64 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with updated_at, so it still has a null cursor +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..4f3f04233ec1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} +// Charlie wasn't reemitted in sync2. This record still has an old_cursor value. +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl new file mode 100644 index 000000000000..1f4d620add7b --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -0,0 +1,8 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} + +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl new file mode 100644 index 000000000000..0f44480d1b5b --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl @@ -0,0 +1,9 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl new file mode 100644 index 000000000000..c62531e41ed7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl new file mode 100644 index 000000000000..df099ff75ea0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..10cd001e22f6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Delete Bob, keep Charlie +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..5a3209db5e22 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +// Keep the record that deleted Bob, but delete the other records associated with id=(1, 201) +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} +// And keep Charlie's record, even though it wasn't reemitted in sync2. +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl new file mode 100644 index 000000000000..faf1bda26c1a --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -0,0 +1,7 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `string`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`", "Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`", "Problem with `time_without_timezone`", "Problem with `date`"]}} +// Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. +// But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.17411800000001}, "array": [67.17411800000001], "unknown": 67.17411800000001, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..bc145f60abd3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}}' +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl new file mode 100644 index 000000000000..ecd140e04aad --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": ["Problem with `integer`"]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..e2c19ff210a9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl new file mode 100644 index 000000000000..fc5f35381948 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..fc7aaebdd9c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl new file mode 100644 index 000000000000..20041e0db3da --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -0,0 +1,14 @@ +// Bigquery converts timestamp_with_timezone to UTC +// But we're using STRING for time_with_timezone, so that column isn't modified at all +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "time_with_timezone": "12:34:56Z"} +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56Z", "time_with_timezone": "12:34:56-0800"} +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56Z", "time_with_timezone": "12:34:56-08"} +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56Z", "time_with_timezone": "12:34:56+0800"} +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56Z", "time_with_timezone": "12:34:56+08"} +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123Z", "time_with_timezone": "12:34:56.123Z"} + +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"} +// Bigquery returns 6 decimal places if there are any decimal places... but not for timestamp_with_timezone +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123000", "time_without_timezone": "12:34:56.123000"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl new file mode 100644 index 000000000000..fa1b306416e0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -0,0 +1,10 @@ +// column renamings: +// * $starts_with_dollar_sign -> _starts_with_dollar_sign +// * includes"doublequote -> includes_doublequote +// * includes'singlequote -> includes_singlequote +// * includes`backtick -> includes_backtick +// * includes$$doubledollar -> includes__doubledollar +// * includes.period -> includes_period +// columns with issues: +// * endswithbackslash\ -> nulled out, and no error in airbyte_meta. This actually extracts the value correctly if _airbyte_data is a JSON column, but JSON_VALUE seems to return null if the data is a string and the key contains a backslash. +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "_starts_with_dollar_sign": "foo", "includes_doublequote": "foo", "includes_singlequote": "foo", "includes_backtick": "foo", "includes_period": "foo", "includes__doubledollar": "foo"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..757f7c357a12 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java index 27511e993453..881ee3973a7a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java @@ -4,16 +4,26 @@ package io.airbyte.integrations.destination.bigquery; +import static org.mockito.Mockito.mock; + +import com.google.cloud.bigquery.BigQuery; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV1V2Migrator; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import java.util.Collections; import java.util.Map; import java.util.function.Consumer; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -24,9 +34,23 @@ public class BigQueryRecordConsumerTest extends PerStreamStateMessageTest { @Mock private Consumer outputRecordCollector; - @InjectMocks private BigQueryRecordConsumer bigQueryRecordConsumer; + @BeforeEach + public void setup() { + DestinationConfig.initialize(Jsons.deserialize("{}")); + + ParsedCatalog parsedCatalog = new ParsedCatalog(Collections.emptyList()); + BigQueryV1V2Migrator migrator = Mockito.mock(BigQueryV1V2Migrator.class); + bigQueryRecordConsumer = new BigQueryRecordConsumer( + mock(BigQuery.class), + uploaderMap, + outputRecordCollector, + "test-dataset-id", + new NoopTyperDeduper(), + parsedCatalog); + } + @Override protected Consumer getMockedConsumer() { return outputRecordCollector; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/CdkImportTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/CdkImportTest.java new file mode 100644 index 000000000000..2b8400ee05ca --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/CdkImportTest.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.cdk.CDKConstants; +import org.junit.jupiter.api.Test; + +class CdkImportTest { + + /** + * This test ensures that the CDK is able to be imported and that its version number matches the + * expected pinned version. + */ + @Test + void cdkVersionShouldMatch() { + assertEquals("0.0.1", CDKConstants.VERSION.replace("-SNAPSHOT", "")); + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java new file mode 100644 index 000000000000..f04008e47a1f --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.cloud.bigquery.Clustering; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TimePartitioning; +import com.google.common.collect.ImmutableList; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class BigQuerySqlGeneratorTest { + + private final BigQuerySqlGenerator generator = new BigQuerySqlGenerator("US"); + + @Test + public void testToDialectType() { + final Struct s = new Struct(new LinkedHashMap<>()); + final Array a = new Array(AirbyteProtocolType.BOOLEAN); + + assertEquals(StandardSQLTypeName.INT64, generator.toDialectType((AirbyteType) AirbyteProtocolType.INTEGER)); + assertEquals(StandardSQLTypeName.JSON, generator.toDialectType(s)); + assertEquals(StandardSQLTypeName.JSON, generator.toDialectType(a)); + assertEquals(StandardSQLTypeName.JSON, generator.toDialectType(new UnsupportedOneOf(new ArrayList<>()))); + + Union u = new Union(ImmutableList.of(s)); + assertEquals(StandardSQLTypeName.JSON, generator.toDialectType(u)); + u = new Union(ImmutableList.of(a)); + assertEquals(StandardSQLTypeName.JSON, generator.toDialectType(u)); + u = new Union(ImmutableList.of(AirbyteProtocolType.BOOLEAN, AirbyteProtocolType.NUMBER)); + assertEquals(StandardSQLTypeName.NUMERIC, generator.toDialectType(u)); + } + + @Test + public void testBuildColumnId() { + // Uninteresting names are unchanged + assertEquals( + new ColumnId("foo", "foo", "foo"), + generator.buildColumnId("foo")); + } + + @Test + public void testClusteringMatches() { + StreamConfig stream = new StreamConfig(null, + null, + DestinationSyncMode.APPEND_DEDUP, + List.of(new ColumnId("foo", "bar", "fizz")), + null, + null); + + // Clustering is null + final StandardTableDefinition existingTable = Mockito.mock(StandardTableDefinition.class); + Mockito.when(existingTable.getClustering()).thenReturn(null); + Assertions.assertFalse(generator.clusteringMatches(stream, existingTable)); + + // Clustering does not contain all fields + Mockito.when(existingTable.getClustering()) + .thenReturn(Clustering.newBuilder().setFields(List.of("_airbyte_extracted_at")).build()); + Assertions.assertFalse(generator.clusteringMatches(stream, existingTable)); + + // Clustering matches + stream = new StreamConfig(null, + null, + DestinationSyncMode.OVERWRITE, + null, + null, + null); + Assertions.assertTrue(generator.clusteringMatches(stream, existingTable)); + + // Clustering only the first 3 PK columns (See https://github.com/airbytehq/oncall/issues/2565) + final var expectedStreamColumnNames = List.of("a", "b", "c"); + Mockito.when(existingTable.getClustering()) + .thenReturn(Clustering.newBuilder().setFields( + Stream.concat(expectedStreamColumnNames.stream(), Stream.of("_airbyte_extracted_at")) + .collect(Collectors.toList())) + .build()); + stream = new StreamConfig(null, + null, + DestinationSyncMode.APPEND_DEDUP, + Stream.concat(expectedStreamColumnNames.stream(), Stream.of("d", "e")) + .map(name -> new ColumnId(name, "foo", "bar")) + .collect(Collectors.toList()), + null, + null); + Assertions.assertTrue(generator.clusteringMatches(stream, existingTable)); + } + + @Test + public void testPartitioningMatches() { + final StandardTableDefinition existingTable = Mockito.mock(StandardTableDefinition.class); + // Partitioning is null + Mockito.when(existingTable.getTimePartitioning()).thenReturn(null); + Assertions.assertFalse(generator.partitioningMatches(existingTable)); + // incorrect field + Mockito.when(existingTable.getTimePartitioning()) + .thenReturn(TimePartitioning.newBuilder(TimePartitioning.Type.DAY).setField("_foo").build()); + Assertions.assertFalse(generator.partitioningMatches(existingTable)); + // incorrect partitioning scheme + Mockito.when(existingTable.getTimePartitioning()) + .thenReturn(TimePartitioning.newBuilder(TimePartitioning.Type.YEAR).setField("_airbyte_extracted_at").build()); + Assertions.assertFalse(generator.partitioningMatches(existingTable)); + + // partitioning matches + Mockito.when(existingTable.getTimePartitioning()) + .thenReturn(TimePartitioning.newBuilder(TimePartitioning.Type.DAY).setField("_airbyte_extracted_at").build()); + Assertions.assertTrue(generator.partitioningMatches(existingTable)); + } + + @Test + public void testSchemaContainAllFinalTableV2AirbyteColumns() { + Assertions.assertTrue( + BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of("_airbyte_meta", "_airbyte_extracted_at", "_airbyte_raw_id"))); + Assertions.assertFalse(BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of("_airbyte_extracted_at", "_airbyte_raw_id"))); + Assertions.assertFalse(BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of("_airbyte_meta", "_airbyte_raw_id"))); + Assertions.assertFalse(BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of("_airbyte_meta", "_airbyte_extracted_at"))); + Assertions.assertFalse(BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of())); + Assertions.assertTrue( + BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of("_AIRBYTE_META", "_AIRBYTE_EXTRACTED_AT", "_AIRBYTE_RAW_ID"))); + } + +} diff --git a/airbyte-integrations/connectors/destination-cassandra/metadata.yaml b/airbyte-integrations/connectors/destination-cassandra/metadata.yaml index 63546b1ab50c..825ba9edb491 100644 --- a/airbyte-integrations/connectors/destination-cassandra/metadata.yaml +++ b/airbyte-integrations/connectors/destination-cassandra/metadata.yaml @@ -10,11 +10,15 @@ data: name: Cassandra registries: cloud: - enabled: true + enabled: false # hide Cassandra Destination https://github.com/airbytehq/airbyte-cloud/issues/2606 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/cassandra tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java index 6a022e2337ca..c34561d2a9c6 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java @@ -9,6 +9,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.util.HostPortResolver; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; @@ -29,21 +30,21 @@ static void initContainer() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { configJson = TestDataFactory.createJsonConfig( cassandraContainer.getUsername(), cassandraContainer.getPassword(), HostPortResolver.resolveHost(cassandraContainer), HostPortResolver.resolvePort(cassandraContainer)); - var cassandraConfig = new CassandraConfig(configJson); + final var cassandraConfig = new CassandraConfig(configJson); cassandraCqlProvider = new CassandraCqlProvider(cassandraConfig); cassandraNameTransformer = new CassandraNameTransformer(cassandraConfig); } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { cassandraCqlProvider.retrieveMetadata().forEach(meta -> { - var keyspace = meta.value1(); + final var keyspace = meta.value1(); meta.value2().forEach(table -> cassandraCqlProvider.truncate(keyspace, table)); }); } @@ -73,12 +74,12 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) { - var keyspace = cassandraNameTransformer.outputKeyspace(namespace); - var table = cassandraNameTransformer.outputTable(streamName); + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + final var keyspace = cassandraNameTransformer.outputKeyspace(namespace); + final var table = cassandraNameTransformer.outputTable(streamName); return cassandraCqlProvider.select(keyspace, table).stream() .sorted(Comparator.comparing(CassandraRecord::getTimestamp)) .map(CassandraRecord::getData) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 3b3774f499d6..a86c49a6ce01 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -8,6 +8,25 @@ # Please reach out to the Connectors Operations team if you have any question. FROM airbyte/integration-base-java:dev AS build +RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ + alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ + python -m ensurepip --upgrade && \ + pip3 install dbt-clickhouse>=1.4.0 + +# Luckily, none of normalization's files conflict with destination-clickhouse's files :) +# We don't enforce that in any way, but hopefully we're only living in this state for a short time. +COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte +# Install python dependencies +WORKDIR /airbyte/base_python_structs +RUN pip3 install . +WORKDIR /airbyte/normalization_code +RUN pip3 install . +WORKDIR /airbyte/normalization_code/dbt-template/ +# Download external dbt dependencies +# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 +RUN pip3 install "urllib3<2" +RUN dbt deps + WORKDIR /airbyte ENV APPLICATION destination-clickhouse-strict-encrypt @@ -21,8 +40,12 @@ FROM airbyte/integration-base-java:dev WORKDIR /airbyte ENV APPLICATION destination-clickhouse-strict-encrypt +ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt + +ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" +ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index 4115757b1510..bfdc74c9a53f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -28,3 +28,7 @@ dependencies { integrationTestJavaImplementation libs.connectors.destination.testcontainers.clickhouse integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } + +tasks.named("airbyteDocker") { + dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerClickhouse +} diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml index 2b1f588a283a..b58fc5f5d3e5 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: database connectorType: destination definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.5 dockerRepository: airbyte/destination-clickhouse-strict-encrypt githubIssueLabel: destination-clickhouse icon: clickhouse.svg diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index 6bd26cacdba1..d311d2a53f2f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -24,6 +24,7 @@ import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.params.ParameterizedTest; @@ -164,7 +165,7 @@ protected List resolveIdentifier(final String identifier) { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") .withFileFromClasspath("Dockerfile", "docker/Dockerfile") .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index d09a2b3d34ac..98560ddb4f04 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -8,9 +8,32 @@ # Please reach out to the Connectors Operations team if you have any question. FROM airbyte/integration-base-java:dev AS build +RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ + alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ + python -m ensurepip --upgrade && \ + # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 + pip3 install wheel && \ + pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ + pip3 install dbt-clickhouse>=1.4.0 + +# Luckily, none of normalization's files conflict with destination-clickhouse's files :) +# We don't enforce that in any way, but hopefully we're only living in this state for a short time. +COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte +# Install python dependencies +WORKDIR /airbyte/base_python_structs +RUN pip3 install . +WORKDIR /airbyte/normalization_code +RUN pip3 install . +WORKDIR /airbyte/normalization_code/dbt-template/ +# Download external dbt dependencies +# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 +RUN pip3 install "urllib3<2" +RUN dbt deps + WORKDIR /airbyte ENV APPLICATION destination-clickhouse +ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar @@ -24,5 +47,8 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/destination-clickhouse + +ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" +ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index 046db83deb8a..b636c4258aed 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -30,3 +30,7 @@ dependencies { integrationTestJavaImplementation libs.connectors.destination.testcontainers.clickhouse integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } + +tasks.named("airbyteDocker") { + dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerClickhouse +} diff --git a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml index 89ee099a0910..cf10ca7aa667 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/destination-clickhouse/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.5 dockerRepository: airbyte/destination-clickhouse githubIssueLabel: destination-clickhouse icon: clickhouse.svg @@ -23,4 +23,8 @@ data: supportsDbt: false tags: - language:java + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 3fa1f7e365c6..421a97bcd9b6 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -23,6 +23,7 @@ import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; import java.time.Duration; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.params.ParameterizedTest; @@ -137,7 +138,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") .waitingFor(Wait.forHttp("/ping").forPort(8123) .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 57651ea3a412..0dd47c99402d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -20,6 +20,7 @@ import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.params.ParameterizedTest; @@ -154,7 +155,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { bastion.initAndStartBastion(network); db = (ClickHouseContainer) new ClickHouseContainer("clickhouse/clickhouse-server:22.5").withNetwork(network); db.start(); diff --git a/airbyte-integrations/connectors/destination-convex/metadata.yaml b/airbyte-integrations/connectors/destination-convex/metadata.yaml index 958433481a10..88cbb5c9616f 100644 --- a/airbyte-integrations/connectors/destination-convex/metadata.yaml +++ b/airbyte-integrations/connectors/destination-convex/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/destinations/convex + documentationUrl: https://docs.airbyte.com/integrations/destinations/convex tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-csv/metadata.yaml b/airbyte-integrations/connectors/destination-csv/metadata.yaml index 6904e04d8531..63bb9b6625f8 100644 --- a/airbyte-integrations/connectors/destination-csv/metadata.yaml +++ b/airbyte-integrations/connectors/destination-csv/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/local-csv + documentationUrl: https://docs.airbyte.com/integrations/destinations/csv tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java index 614d8dcc8d90..30856da6aa84 100644 --- a/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java @@ -24,6 +24,7 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -51,7 +52,7 @@ protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.of("destination_path", Path.of("/local").resolve(RELATIVE_PATH).toString())); } - protected JsonNode getConfigWithDelimiter(String delimiter) { + protected JsonNode getConfigWithDelimiter(final String delimiter) { config = Jsons.jsonNode(ImmutableMap.of("destination_path", Path.of("/local").resolve(RELATIVE_PATH).toString(), "delimiter", delimiter)); return config; } @@ -75,7 +76,7 @@ public void testCheckConnectionInvalidCredentials() {} @ParameterizedTest @ArgumentsSource(CSVDataArgumentsProvider.class) - public void testSyncWithDelimiter(final String messagesFilename, final String catalogFilename, String delimiter) + public void testSyncWithDelimiter(final String messagesFilename, final String catalogFilename, final String delimiter) throws Exception { final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); @@ -118,7 +119,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { // no op } @@ -133,7 +134,7 @@ public static class CSVDataArgumentsProvider extends DataArgumentsProvider { @Override public Stream provideArguments(final ExtensionContext context) throws Exception { - ProtocolVersion protocolVersion = ArgumentProviderUtil.getProtocolVersion(context); + final ProtocolVersion protocolVersion = ArgumentProviderUtil.getProtocolVersion(context); return Stream.of( Arguments.of(EXCHANGE_RATE_CONFIG.getMessageFileVersion(protocolVersion), EXCHANGE_RATE_CONFIG.getCatalogFileVersion(protocolVersion), "\\u002c"), diff --git a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py index fc8035b249b7..61c6c5ac4afb 100644 --- a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py +++ b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/destination.py @@ -15,7 +15,6 @@ class DestinationCumulio(Destination): - def write( self, config: Mapping[str, Any], diff --git a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/spec.json b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/spec.json index 63abc48320af..dff9ec31cb64 100644 --- a/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/spec.json +++ b/airbyte-integrations/connectors/destination-cumulio/destination_cumulio/spec.json @@ -1,9 +1,6 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/destinations/cumulio", - "supported_destination_sync_modes": [ - "overwrite", - "append" - ], + "supported_destination_sync_modes": ["overwrite", "append"], "supportsIncremental": true, "supportsDBT": false, "supportsNormalization": false, diff --git a/airbyte-integrations/connectors/destination-cumulio/integration_tests/sample_config.json b/airbyte-integrations/connectors/destination-cumulio/integration_tests/sample_config.json index dca95d675aa3..2a1ca74c862b 100644 --- a/airbyte-integrations/connectors/destination-cumulio/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/destination-cumulio/integration_tests/sample_config.json @@ -2,4 +2,4 @@ "api_host": "https://api.cumul.io", "api_key": "CUMULIO_API_KEY", "api_token": "CUMULIO_API_TOKEN" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-cumulio/metadata.yaml b/airbyte-integrations/connectors/destination-cumulio/metadata.yaml index a2182597e713..d45dd2ff46db 100644 --- a/airbyte-integrations/connectors/destination-cumulio/metadata.yaml +++ b/airbyte-integrations/connectors/destination-cumulio/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/cumulio tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-databend/destination_databend/spec.json b/airbyte-integrations/connectors/destination-databend/destination_databend/spec.json index 01d086fe6553..e77d3301152c 100644 --- a/airbyte-integrations/connectors/destination-databend/destination_databend/spec.json +++ b/airbyte-integrations/connectors/destination-databend/destination_databend/spec.json @@ -1,67 +1,56 @@ { - "documentationUrl" : "https://docs.airbyte.com/integrations/destinations/databend", - "supported_destination_sync_modes" : [ - "overwrite", - "append" - ], - "supportsIncremental" : true, - "connectionSpecification" : { - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "Destination Databend", - "type" : "object", - "required" : [ - "host", - "username", - "database" - ], - "additionalProperties" : true, - "properties" : { - "host" : { - "title" : "Host", - "description" : "Hostname of the database.", - "type" : "string", - "order" : 0 + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/databend", + "supported_destination_sync_modes": ["overwrite", "append"], + "supportsIncremental": true, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Databend", + "type": "object", + "required": ["host", "username", "database"], + "additionalProperties": true, + "properties": { + "host": { + "title": "Host", + "description": "Hostname of the database.", + "type": "string", + "order": 0 }, - "port" : { - "title" : "Port", - "description" : "Port of the database.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 443, - "examples" : [ - "443" - ], - "order" : 2 + "port": { + "title": "Port", + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 443, + "examples": ["443"], + "order": 2 }, - "database" : { - "title" : "DB Name", - "description" : "Name of the database.", - "type" : "string", - "order" : 3 + "database": { + "title": "DB Name", + "description": "Name of the database.", + "type": "string", + "order": 3 }, - "table" : { - "title" : "Default Table", - "description" : "The default table was written to.", - "type" : "string", - "examples" : [ - "default" - ], - "default" : "default", - "order" : 4 + "table": { + "title": "Default Table", + "description": "The default table was written to.", + "type": "string", + "examples": ["default"], + "default": "default", + "order": 4 }, - "username" : { - "title" : "User", - "description" : "Username to use to access the database.", - "type" : "string", - "order" : 5 + "username": { + "title": "User", + "description": "Username to use to access the database.", + "type": "string", + "order": 5 }, - "password" : { - "title" : "Password", - "description" : "Password associated with the username.", - "type" : "string", - "airbyte_secret" : true, - "order" : 6 + "password": { + "title": "Password", + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 6 } } } diff --git a/airbyte-integrations/connectors/destination-databend/integration_tests/sample_config.json b/airbyte-integrations/connectors/destination-databend/integration_tests/sample_config.json index cc8ac8584d94..62c0cdb78b7f 100644 --- a/airbyte-integrations/connectors/destination-databend/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/destination-databend/integration_tests/sample_config.json @@ -1,9 +1,9 @@ { - "protocol" : "https", - "host" : "tnc7yee14--xxxx.ch.datafusecloud.com", - "port" : 443, - "username" : "username", - "password" : "password", - "database" : "default", - "table" : "default" + "protocol": "https", + "host": "tnc7yee14--xxxx.ch.datafusecloud.com", + "port": 443, + "username": "username", + "password": "password", + "database": "default", + "table": "default" } diff --git a/airbyte-integrations/connectors/destination-databend/metadata.yaml b/airbyte-integrations/connectors/destination-databend/metadata.yaml index 2ee179caac68..4b2de407755e 100644 --- a/airbyte-integrations/connectors/destination-databend/metadata.yaml +++ b/airbyte-integrations/connectors/destination-databend/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/databend tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-databricks/Dockerfile b/airbyte-integrations/connectors/destination-databricks/Dockerfile index ce0effa21c53..450c77f83318 100644 --- a/airbyte-integrations/connectors/destination-databricks/Dockerfile +++ b/airbyte-integrations/connectors/destination-databricks/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-databricks COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.2 +LABEL io.airbyte.version=1.1.0 LABEL io.airbyte.name=airbyte/destination-databricks diff --git a/airbyte-integrations/connectors/destination-databricks/metadata.yaml b/airbyte-integrations/connectors/destination-databricks/metadata.yaml index 6726b37d9455..0b707d0c110f 100644 --- a/airbyte-integrations/connectors/destination-databricks/metadata.yaml +++ b/airbyte-integrations/connectors/destination-databricks/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 072d5540-f236-4294-ba7c-ade8fd918496 - dockerImageTag: 1.0.2 + dockerImageTag: 1.1.0 dockerRepository: airbyte/destination-databricks githubIssueLabel: destination-databricks icon: databricks.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/databricks tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfig.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfig.java index d363e9bb440d..c744234faab9 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfig.java @@ -6,6 +6,7 @@ import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_CATALOG_KEY; import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_DATA_SOURCE_KEY; +import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_ENABLE_SCHEMA_EVOLUTION_KEY; import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_HTTP_PATH_KEY; import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_PERSONAL_ACCESS_TOKEN_KEY; import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_PORT_KEY; @@ -23,12 +24,14 @@ public record DatabricksDestinationConfig(String serverHostname, String catalog, String schema, boolean isPurgeStagingData, + boolean enableSchemaEvolution, DatabricksStorageConfigProvider storageConfig) { static final String DEFAULT_DATABRICKS_PORT = "443"; static final String DEFAULT_DATABASE_SCHEMA = "default"; static final String DEFAULT_CATALOG = "hive_metastore"; static final boolean DEFAULT_PURGE_STAGING_DATA = true; + static final boolean DEFAULT_ENABLE_SCHEMA_EVOLUTION = false; public static DatabricksDestinationConfig get(final JsonNode config) { Preconditions.checkArgument( @@ -42,6 +45,8 @@ public static DatabricksDestinationConfig get(final JsonNode config) { config.get(DATABRICKS_PERSONAL_ACCESS_TOKEN_KEY).asText(), config.has(DATABRICKS_CATALOG_KEY) ? config.get(DATABRICKS_CATALOG_KEY).asText() : DEFAULT_CATALOG, config.has(DATABRICKS_SCHEMA_KEY) ? config.get(DATABRICKS_SCHEMA_KEY).asText() : DEFAULT_DATABASE_SCHEMA, + config.has(DATABRICKS_ENABLE_SCHEMA_EVOLUTION_KEY) ? config.get(DATABRICKS_ENABLE_SCHEMA_EVOLUTION_KEY).asBoolean() + : DEFAULT_ENABLE_SCHEMA_EVOLUTION, config.has(DATABRICKS_PURGE_STAGING_DATA_KEY) ? config.get(DATABRICKS_PURGE_STAGING_DATA_KEY).asBoolean() : DEFAULT_PURGE_STAGING_DATA, DatabricksStorageConfigProvider.getDatabricksStorageConfig(config.get(DATABRICKS_DATA_SOURCE_KEY))); } diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java index fade966d0940..3d4d4773d30c 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java @@ -115,12 +115,14 @@ public String generateMergeStatement(final String destTableName) { "COPY INTO %s.%s.%s " + "FROM '%s' " + "FILEFORMAT = PARQUET " + - "PATTERN = '%s'", + "PATTERN = '%s' " + + "COPY_OPTIONS ('mergeSchema' = '%s')", catalogName, schemaName, destTableName, getTmpTableLocation(), - parquetWriter.getOutputFilename()); + parquetWriter.getOutputFilename(), + databricksConfig.enableSchemaEvolution()); LOGGER.info(copyData); return copyData; } diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java index f92eea022e12..04226ddf801f 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java @@ -16,6 +16,7 @@ public class DatabricksConstants { public static final String DATABRICKS_PORT_KEY = "databricks_port"; public static final String DATABRICKS_CATALOG_KEY = "database"; public static final String DATABRICKS_SCHEMA_KEY = "schema"; + public static final String DATABRICKS_ENABLE_SCHEMA_EVOLUTION_KEY = "enable_schema_evolution"; public static final String DATABRICKS_CATALOG_JDBC_KEY = "ConnCatalog"; public static final String DATABRICKS_SCHEMA_JDBC_KEY = "ConnSchema"; public static final String DATABRICKS_PURGE_STAGING_DATA_KEY = "purge_staging_data"; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json index c5dca06931d3..19b74c77a80f 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json @@ -67,12 +67,19 @@ "default": "default", "order": 7 }, + "enable_schema_evolution": { + "title": "Support schema evolution for all streams.", + "type": "boolean", + "description": "Support schema evolution for all streams. If \"false\", the connector might fail when a stream's schema changes.", + "default": false, + "order": 8 + }, "data_source": { "title": "Data Source", "type": "object", "description": "Storage on which the delta lake is built.", "default": "MANAGED_TABLES_STORAGE", - "order": 8, + "order": 9, "oneOf": [ { "title": "[Recommended] Managed tables", @@ -236,7 +243,7 @@ "type": "boolean", "description": "Default to 'true'. Switch it to 'false' for debugging purpose.", "default": true, - "order": 9 + "order": 10 } } } diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java index eaf0376e11fc..c7ab5cf11220 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import java.nio.file.Path; import java.sql.SQLException; +import java.util.HashSet; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; @@ -45,7 +46,7 @@ protected JsonNode getFailCheckConfig() { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { final JsonNode baseConfigJson = Jsons.deserialize(IOs.readFile(Path.of(SECRETS_CONFIG_JSON))); // Set a random Azure path and database schema for each integration test diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java index 8f65c74af7f9..f51008bc7659 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java @@ -22,6 +22,7 @@ import java.nio.file.Path; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -80,7 +81,7 @@ protected JsonNode getFailCheckConfig() throws Exception { } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { this.configJson = Jsons.deserialize(IOs.readFile(Path.of(SECRETS_CONFIG_JSON))); this.databricksConfig = DatabricksDestinationConfig.get(configJson); } diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java index 64030483a82d..1b714915c222 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.integrations.destination.s3.S3DestinationConfig; import java.nio.file.Path; import java.sql.SQLException; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import org.apache.commons.lang3.RandomStringUtils; @@ -46,7 +47,7 @@ protected JsonNode getFailCheckConfig() { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { final JsonNode baseConfigJson = Jsons.deserialize(IOs.readFile(Path.of(SECRETS_CONFIG_JSON))); // Set a random s3 bucket path and database schema for each integration test diff --git a/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfigTest.java b/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfigTest.java index 77cc43081f90..54f7dfb0fdbf 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfigTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationConfigTest.java @@ -44,10 +44,11 @@ public void testConfigCreationFromJsonS3() { assertEquals(DatabricksDestinationConfig.DEFAULT_DATABRICKS_PORT, config1.port()); assertEquals(DatabricksDestinationConfig.DEFAULT_DATABASE_SCHEMA, config1.schema()); - databricksConfig.put("databricks_port", "1000").put("schema", "testing_schema"); + databricksConfig.put("databricks_port", "1000").put("schema", "testing_schema").put("enable_schema_evolution", true); final DatabricksDestinationConfig config2 = DatabricksDestinationConfig.get(databricksConfig); assertEquals("1000", config2.port()); assertEquals("testing_schema", config2.schema()); + assertEquals(true, config2.enableSchemaEvolution()); assertEquals(DatabricksS3StorageConfigProvider.class, config2.storageConfig().getClass()); } @@ -76,10 +77,11 @@ public void testConfigCreationFromJsonAzure() { assertEquals(DatabricksDestinationConfig.DEFAULT_DATABRICKS_PORT, config1.port()); assertEquals(DatabricksDestinationConfig.DEFAULT_DATABASE_SCHEMA, config1.schema()); - databricksConfig.put("databricks_port", "1000").put("schema", "testing_schema"); + databricksConfig.put("databricks_port", "1000").put("schema", "testing_schema").put("enable_schema_evolution", true); final DatabricksDestinationConfig config2 = DatabricksDestinationConfig.get(databricksConfig); assertEquals("1000", config2.port()); assertEquals("testing_schema", config2.schema()); + assertEquals(true, config2.enableSchemaEvolution()); assertEquals(DatabricksAzureBlobStorageConfigProvider.class, config2.storageConfig().getClass()); } diff --git a/airbyte-integrations/connectors/destination-databricks/src/test/resources/azure_config.json b/airbyte-integrations/connectors/destination-databricks/src/test/resources/azure_config.json index 17ebdf366a3f..9e1df47cbd6a 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test/resources/azure_config.json +++ b/airbyte-integrations/connectors/destination-databricks/src/test/resources/azure_config.json @@ -4,7 +4,7 @@ "databricks_http_path": "test_http_path", "databricks_port": "443", "databricks_personal_access_token": "test_token", - "database" : "integration_test_catalog", + "database": "integration_test_catalog", "schema": "integration_test_external", "data_source": { "data_source_type": "AZURE_BLOB_STORAGE", diff --git a/airbyte-integrations/connectors/destination-databricks/src/test/resources/config.json b/airbyte-integrations/connectors/destination-databricks/src/test/resources/config.json index 61b5234260ea..6e9554396169 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test/resources/config.json +++ b/airbyte-integrations/connectors/destination-databricks/src/test/resources/config.json @@ -4,8 +4,9 @@ "databricks_http_path": "test_http_path", "databricks_port": "443", "databricks_personal_access_token": "test_token", - "database" : "test", + "database": "test", "schema": "test", + "enable_schema_evolution": "true", "data_source": { "data_source_type": "S3_STORAGE", "s3_bucket_name": "required", diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml index 440b260361d2..ad5594d4b173 100644 --- a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: false releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/destinations/e2e-test + documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java index af0e1c419d75..07df3387a066 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java @@ -12,6 +12,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.Collections; +import java.util.HashSet; import java.util.List; public class DevNullDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -41,7 +42,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { // do nothing } diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json index b6f2a4730264..bec2c3f05c92 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/destination-dev-null/src/test/resources/expected_spec.json @@ -12,9 +12,9 @@ "required": ["test_destination"], "properties": { "test_destination": { - "title" : "Test Destination", - "type" : "object", - "description" : "The type of destination to be used", + "title": "Test Destination", + "type": "object", + "description": "The type of destination to be used", "oneOf": [ { "title": "Silent", diff --git a/airbyte-integrations/connectors/destination-doris/metadata.yaml b/airbyte-integrations/connectors/destination-doris/metadata.yaml index d87f01164f91..86f0098edb73 100644 --- a/airbyte-integrations/connectors/destination-doris/metadata.yaml +++ b/airbyte-integrations/connectors/destination-doris/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/doris tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java index b8189986d3e6..1221c9a7a4fe 100644 --- a/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import java.nio.file.Paths; import java.sql.*; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import org.apache.commons.lang3.StringEscapeUtils; import org.junit.jupiter.api.AfterAll; @@ -44,13 +45,13 @@ protected String getImageName() { @BeforeAll public static void getConnect() { - JsonNode config = Jsons.deserialize(IOs.readFile(Paths.get("../../../secrets/config.json"))); - String dbUrl = String.format(DB_URL_PATTERN, config.get("host").asText(), PORT); + final JsonNode config = Jsons.deserialize(IOs.readFile(Paths.get("../../../secrets/config.json"))); + final String dbUrl = String.format(DB_URL_PATTERN, config.get("host").asText(), PORT); try { Class.forName(JDBC_DRIVER); conn = DriverManager.getConnection(dbUrl, config.get("username").asText(), config.get("password") == null ? "" : config.get("password").asText()); - } catch (Exception e) { + } catch (final Exception e) { e.printStackTrace(); } @@ -80,10 +81,10 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws IOException, SQLException { // TODO Implement this method to retrieve records which written to the destination by the connector. // Records returned from this method will be compared against records provided to the connector @@ -91,15 +92,15 @@ protected List retrieveRecords(TestDestinationEnv testEnv, final String tableName = namingResolver.getIdentifier(streamName); - String query = String.format( + final String query = String.format( "SELECT * FROM %s.%s ORDER BY %s ASC;", configJson.get("database").asText(), tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - PreparedStatement stmt = conn.prepareStatement(query); - ResultSet resultSet = stmt.executeQuery(); + final PreparedStatement stmt = conn.prepareStatement(query); + final ResultSet resultSet = stmt.executeQuery(); - List res = new ArrayList<>(); + final List res = new ArrayList<>(); while (resultSet.next()) { - String sss = resultSet.getString(JavaBaseConstants.COLUMN_NAME_DATA); + final String sss = resultSet.getString(JavaBaseConstants.COLUMN_NAME_DATA); res.add(Jsons.deserialize(StringEscapeUtils.unescapeJava(sss))); } stmt.close(); @@ -107,12 +108,12 @@ protected List retrieveRecords(TestDestinationEnv testEnv, } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { // TODO Implement this method to run any setup actions needed before every test case } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { // TODO Implement this method to run any cleanup actions needed after every test case } diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json index 9686cea6d8ad..2517ac9ed772 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json @@ -16,12 +16,11 @@ "description": "Path to the .duckdb file. The file will be placed inside that local mount. For more information check out our docs", "example": "/local/destination.duckdb" }, - "schema": { + "schema": { "type": "string", "description": "database schema, default for duckdb is main", "example": "main" } - } } } diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/config.json b/airbyte-integrations/connectors/destination-duckdb/integration_tests/config.json index 1e6f086a7be0..6ff20b5e462f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/integration_tests/config.json +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/config.json @@ -1 +1 @@ -{"destination_path": "/local/destination.duckdb"} +{ "destination_path": "/local/destination.duckdb" } diff --git a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml index 8f0d96bc4562..3d751bfe1f80 100644 --- a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/destinations/duckdb + documentationUrl: https://docs.airbyte.com/integrations/destinations/duckdb tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml b/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml index 6199e1a92cff..961fa4be5cd3 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/dynamodb tags: - language:java + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java index 434b6083a05a..56936761a4d6 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java @@ -105,7 +105,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { final JsonNode baseConfigJson = getBaseConfigJson(); // Set a random s3 bucket path for each integration test final JsonNode configJson = Jsons.clone(baseConfigJson); diff --git a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml index f7865f617bc3..31b19e2f5b12 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml +++ b/airbyte-integrations/connectors/destination-e2e-test/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json index dd8f2703ad3a..eaf220172e66 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json @@ -12,142 +12,118 @@ "required": ["test_destination"], "properties": { "test_destination": { - "title" : "Test Destination", - "type" : "object", - "description" : "The type of destination to be used", - "oneOf" : [ + "title": "Test Destination", + "type": "object", + "description": "The type of destination to be used", + "oneOf": [ { - "title" : "Logging", - "required" : [ - "test_destination_type", - "logging_config" - ], - "properties" : { - "test_destination_type" : { - "type" : "string", - "const" : "LOGGING", - "default" : "LOGGING" + "title": "Logging", + "required": ["test_destination_type", "logging_config"], + "properties": { + "test_destination_type": { + "type": "string", + "const": "LOGGING", + "default": "LOGGING" }, - "logging_config" : { - "title" : "Logging Configuration", - "type" : "object", - "description" : "Configurate how the messages are logged.", - "oneOf" : [ + "logging_config": { + "title": "Logging Configuration", + "type": "object", + "description": "Configurate how the messages are logged.", + "oneOf": [ { - "title" : "First N Entries", - "description" : "Log first N entries per stream.", - "type" : "object", - "required" : [ - "logging_type", - "max_entry_count" - ], - "properties" : { - "logging_type" : { - "type" : "string", - "enum" : [ - "FirstN" - ], - "default" : "FirstN" + "title": "First N Entries", + "description": "Log first N entries per stream.", + "type": "object", + "required": ["logging_type", "max_entry_count"], + "properties": { + "logging_type": { + "type": "string", + "enum": ["FirstN"], + "default": "FirstN" }, - "max_entry_count" : { - "title" : "N", - "description" : "Number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", - "type" : "number", - "default" : 100, - "examples" : [ - 100 - ], - "minimum" : 1, - "maximum" : 1000 + "max_entry_count": { + "title": "N", + "description": "Number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 } } }, { - "title" : "Every N-th Entry", - "description" : "For each stream, log every N-th entry with a maximum cap.", - "type" : "object", - "required" : [ + "title": "Every N-th Entry", + "description": "For each stream, log every N-th entry with a maximum cap.", + "type": "object", + "required": [ "logging_type", "nth_entry_to_log", "max_entry_count" ], - "properties" : { - "logging_type" : { - "type" : "string", - "enum" : [ - "EveryNth" - ], - "default" : "EveryNth" + "properties": { + "logging_type": { + "type": "string", + "enum": ["EveryNth"], + "default": "EveryNth" }, - "nth_entry_to_log" : { - "title" : "N", - "description" : "The N-th entry to log for each stream. N starts from 1. For example, when N = 1, every entry is logged; when N = 2, every other entry is logged; when N = 3, one out of three entries is logged.", - "type" : "number", - "example" : [ - 3 - ], - "minimum" : 1, - "maximum" : 1000 + "nth_entry_to_log": { + "title": "N", + "description": "The N-th entry to log for each stream. N starts from 1. For example, when N = 1, every entry is logged; when N = 2, every other entry is logged; when N = 3, one out of three entries is logged.", + "type": "number", + "example": [3], + "minimum": 1, + "maximum": 1000 }, - "max_entry_count" : { - "title" : "Max Log Entries", - "description" : "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", - "type" : "number", - "default" : 100, - "examples" : [ - 100 - ], - "minimum" : 1, - "maximum" : 1000 + "max_entry_count": { + "title": "Max Log Entries", + "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 } } }, { - "title" : "Random Sampling", - "description" : "For each stream, randomly log a percentage of the entries with a maximum cap.", - "type" : "object", - "required" : [ + "title": "Random Sampling", + "description": "For each stream, randomly log a percentage of the entries with a maximum cap.", + "type": "object", + "required": [ "logging_type", "sampling_ratio", "max_entry_count" ], - "properties" : { - "logging_type" : { - "type" : "string", - "enum" : [ - "RandomSampling" - ], - "default" : "RandomSampling" + "properties": { + "logging_type": { + "type": "string", + "enum": ["RandomSampling"], + "default": "RandomSampling" }, - "sampling_ratio" : { - "title" : "Sampling Ratio", - "description" : "A positive floating number smaller than 1.", - "type" : "number", - "default" : 0.001, - "examples" : [ - 0.001 - ], - "minimum" : 0, - "maximum" : 1 + "sampling_ratio": { + "title": "Sampling Ratio", + "description": "A positive floating number smaller than 1.", + "type": "number", + "default": 0.001, + "examples": [0.001], + "minimum": 0, + "maximum": 1 }, - "seed" : { - "title" : "Random Number Generator Seed", - "description" : "When the seed is unspecified, the current time millis will be used as the seed.", - "type" : "number", - "examples" : [ - 1900 - ] + "seed": { + "title": "Random Number Generator Seed", + "description": "When the seed is unspecified, the current time millis will be used as the seed.", + "type": "number", + "examples": [1900] }, - "max_entry_count" : { - "title" : "Max Log Entries", - "description" : "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", - "type" : "number", - "default" : 100, - "examples" : [ - 100 - ], - "minimum" : 1, - "maximum" : 1000 + "max_entry_count": { + "title": "Max Log Entries", + "description": "Max number of entries to log. This destination is for testing only. So it won't make sense to log infinitely. The maximum is 1,000 entries.", + "type": "number", + "default": 100, + "examples": [100], + "minimum": 1, + "maximum": 1000 } } } @@ -156,51 +132,43 @@ } }, { - "title" : "Silent", - "required" : [ - "test_destination_type" - ], - "properties" : { - "test_destination_type" : { - "type" : "string", - "const" : "SILENT", - "default" : "SILENT" + "title": "Silent", + "required": ["test_destination_type"], + "properties": { + "test_destination_type": { + "type": "string", + "const": "SILENT", + "default": "SILENT" } } }, { - "title" : "Throttled", - "required" : [ - "test_destination_type", - "millis_per_record" - ], - "properties" : { - "test_destination_type" : { - "type" : "string", - "const" : "THROTTLED", - "default" : "THROTTLED" + "title": "Throttled", + "required": ["test_destination_type", "millis_per_record"], + "properties": { + "test_destination_type": { + "type": "string", + "const": "THROTTLED", + "default": "THROTTLED" }, - "millis_per_record" : { - "description" : "Number of milli-second to pause in between records.", - "type" : "integer" + "millis_per_record": { + "description": "Number of milli-second to pause in between records.", + "type": "integer" } } }, { - "title" : "Failing", - "required" : [ - "test_destination_type", - "num_messages" - ], - "properties" : { - "test_destination_type" : { - "type" : "string", - "const" : "FAILING", - "default" : "FAILING" + "title": "Failing", + "required": ["test_destination_type", "num_messages"], + "properties": { + "test_destination_type": { + "type": "string", + "const": "FAILING", + "default": "FAILING" }, - "num_messages" : { - "description" : "Number of messages after which to fail.", - "type" : "integer" + "num_messages": { + "description": "Number of messages after which to fail.", + "type": "integer" } } } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java index 3acb7b50cbc8..25ad63f098bd 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java @@ -13,6 +13,7 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.Collections; +import java.util.HashSet; import java.util.List; public class TestingSilentDestinationAcceptanceTest extends DestinationAcceptanceTest { @@ -42,7 +43,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { // do nothing } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index a88a908b9b38..66cae14c5ed4 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.HashSet; import java.util.List; import java.util.Map; import org.junit.jupiter.api.AfterAll; @@ -134,10 +135,10 @@ protected List retrieveRecords(final DestinationAcceptanceTest.TestDes } @Override - protected void setup(final DestinationAcceptanceTest.TestDestinationEnv testEnv) {} + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) {} @Override - protected void tearDown(final DestinationAcceptanceTest.TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { final ElasticsearchConnection connection = new ElasticsearchConnection(mapper.convertValue(getConfig(), ConnectorConfiguration.class)); connection.allIndices().forEach(connection::deleteIndexIfPresent); } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/metadata.yaml b/airbyte-integrations/connectors/destination-elasticsearch/metadata.yaml index 542f67ec9f07..a9c8f1d09a1a 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/metadata.yaml +++ b/airbyte-integrations/connectors/destination-elasticsearch/metadata.yaml @@ -18,4 +18,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/elasticsearch tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java index 253db3d2761a..e35d59ba56b7 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java @@ -10,6 +10,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.time.Duration; +import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -22,7 +23,7 @@ public class ElasticsearchDestinationAcceptanceTest extends DestinationAcceptanc private static final String IMAGE_NAME = "docker.elastic.co/elasticsearch/elasticsearch:8.3.3"; private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchDestinationAcceptanceTest.class); - private ObjectMapper mapper = new ObjectMapper(); + private final ObjectMapper mapper = new ObjectMapper(); private static ElasticsearchContainer container; @BeforeAll @@ -83,7 +84,7 @@ protected TestDataComparator getTestDataComparator() { @Override protected JsonNode getConfig() throws Exception { - var configJson = mapper.createObjectNode(); + final var configJson = mapper.createObjectNode(); configJson.put("endpoint", String.format("http://%s:%s", container.getHost(), container.getMappedPort(9200))); return configJson; } @@ -91,16 +92,16 @@ protected JsonNode getConfig() throws Exception { @Override protected JsonNode getFailCheckConfig() throws Exception { // should result in a failed connection check - var configJson = mapper.createObjectNode(); + final var configJson = mapper.createObjectNode(); configJson.put("endpoint", String.format("htp::/%s:-%s", container.getHost(), container.getMappedPort(9200))); return configJson; } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { // Records returned from this method will be compared against records provided to the connector // to verify they were written correctly @@ -109,16 +110,16 @@ protected List retrieveRecords(TestDestinationEnv testEnv, .setStreamName(streamName) .getIndexName(); - ElasticsearchConnection connection = new ElasticsearchConnection(mapper.convertValue(getConfig(), ConnectorConfiguration.class)); + final ElasticsearchConnection connection = new ElasticsearchConnection(mapper.convertValue(getConfig(), ConnectorConfiguration.class)); return connection.getRecords(indexName); } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception {} + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception {} @Override - protected void tearDown(TestDestinationEnv testEnv) throws Exception { - ElasticsearchConnection connection = new ElasticsearchConnection(mapper.convertValue(getConfig(), ConnectorConfiguration.class)); + protected void tearDown(final TestDestinationEnv testEnv) throws Exception { + final ElasticsearchConnection connection = new ElasticsearchConnection(mapper.convertValue(getConfig(), ConnectorConfiguration.class)); connection.allIndices().forEach(connection::deleteIndexIfPresent); connection.close(); } diff --git a/airbyte-integrations/connectors/destination-exasol/metadata.yaml b/airbyte-integrations/connectors/destination-exasol/metadata.yaml index 236eb2f2ad11..45c7215bafee 100644 --- a/airbyte-integrations/connectors/destination-exasol/metadata.yaml +++ b/airbyte-integrations/connectors/destination-exasol/metadata.yaml @@ -16,4 +16,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/exasol tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java index f56aed797572..23326d046f35 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -43,7 +44,7 @@ static void startExasolContainer() { config = createExasolConfig(EXASOL); } - private static JsonNode createExasolConfig(ExasolContainer> exasol) { + private static JsonNode createExasolConfig(final ExasolContainer> exasol) { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, exasol.getHost()) .put(JdbcUtils.PORT_KEY, exasol.getFirstMappedDatabasePort()) @@ -97,10 +98,10 @@ protected boolean implementsNamespaces() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws SQLException { return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), "\"" + namespace + "\"") .stream() @@ -110,7 +111,7 @@ protected List retrieveRecords(TestDestinationEnv testEnv, } private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, ExasolSqlOperations.COLUMN_NAME_EMITTED_AT); + final String query = String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, ExasolSqlOperations.COLUMN_NAME_EMITTED_AT); LOGGER.info("Retrieving records using query {}", query); try (final DSLContext dslContext = getDSLContext(config)) { final List result = new Database(dslContext) @@ -124,9 +125,9 @@ private List retrieveRecordsFromTable(final String tableName, final St } private static DSLContext getDSLContext(final JsonNode config) { - String jdbcUrl = + final String jdbcUrl = String.format(DatabaseDriver.EXASOL.getUrlFormatString(), config.get(JdbcUtils.HOST_KEY).asText(), config.get(JdbcUtils.PORT_KEY).asInt()); - Map jdbcConnectionProperties = Map.of("fingerprint", config.get("certificateFingerprint").asText()); + final Map jdbcConnectionProperties = Map.of("fingerprint", config.get("certificateFingerprint").asText()); return DSLContextFactory.create( config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), @@ -137,12 +138,12 @@ private static DSLContext getDSLContext(final JsonNode config) { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { // Nothing to do } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { EXASOL.purgeDatabase(); } diff --git a/airbyte-integrations/connectors/destination-firebolt/metadata.yaml b/airbyte-integrations/connectors/destination-firebolt/metadata.yaml index d5ad1319d7b0..942439e480c1 100644 --- a/airbyte-integrations/connectors/destination-firebolt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-firebolt/metadata.yaml @@ -18,4 +18,8 @@ data: supportsDbt: true tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-firestore/metadata.yaml b/airbyte-integrations/connectors/destination-firestore/metadata.yaml index 16b5d697db51..e2f730979597 100644 --- a/airbyte-integrations/connectors/destination-firestore/metadata.yaml +++ b/airbyte-integrations/connectors/destination-firestore/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/firestore tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index b4d5b18e6dbc..df7f1d9ffd94 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-gcs COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.4.4 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index 3aa35bf0f4e1..fe7fa51f3eb9 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -45,3 +45,12 @@ dependencies { integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-gcs') integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') } + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-gcs/metadata.yaml b/airbyte-integrations/connectors/destination-gcs/metadata.yaml index 506d95b5ad20..3d904cb25b47 100644 --- a/airbyte-integrations/connectors/destination-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/destination-gcs/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: file connectorType: destination definitionId: ca8f6566-e555-4b40-943a-545bf123117a - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.4 dockerRepository: airbyte/destination-gcs githubIssueLabel: destination-gcs icon: googlecloudstorage.svg - license: MIT + license: ELv2 name: Google Cloud Storage (GCS) registries: cloud: @@ -23,4 +23,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/gcs tags: - language:java + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java index c8d22cb0ed4b..9ea223b318f6 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.gcs.util; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.s3.avro.AvroConstants; import javax.annotation.Nullable; import org.apache.avro.LogicalTypes; @@ -18,6 +19,7 @@ public class GcsUtils { private static final Logger LOGGER = LoggerFactory.getLogger(GcsUtils.class); private static final Schema UUID_SCHEMA = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)); private static final Schema TIMESTAMP_MILLIS_SCHEMA = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)); + private static final Schema NULLABLE_TIMESTAMP_MILLIS = SchemaBuilder.builder().unionOf().nullType().and().type(TIMESTAMP_MILLIS_SCHEMA).endUnion(); public static Schema getDefaultAvroSchema(final String name, @Nullable final String namespace, @@ -30,14 +32,25 @@ public static Schema getDefaultAvroSchema(final String name, if (stdNamespace != null) { builder = builder.namespace(stdNamespace); } + if (TypingAndDedupingFlag.isDestinationV2()) { + builder.namespace("airbyte"); + } SchemaBuilder.FieldAssembler assembler = builder.fields(); - - if (appendAirbyteFields) { - assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_ID).type(UUID_SCHEMA).noDefault(); - assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_EMITTED_AT).type(TIMESTAMP_MILLIS_SCHEMA).noDefault(); + if (TypingAndDedupingFlag.isDestinationV2()) { + if (appendAirbyteFields) { + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).type(UUID_SCHEMA).noDefault(); + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT).type(TIMESTAMP_MILLIS_SCHEMA).noDefault(); + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT).type(NULLABLE_TIMESTAMP_MILLIS).withDefault(null); + } + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_DATA).type().stringType().noDefault(); + } else { + if (appendAirbyteFields) { + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_ID).type(UUID_SCHEMA).noDefault(); + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_EMITTED_AT).type(TIMESTAMP_MILLIS_SCHEMA).noDefault(); + } + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_DATA).type().stringType().noDefault(); } - assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_DATA).type().stringType().noDefault(); return assembler.endRecord(); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java index 3897d7e73de9..243e3c71005f 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java @@ -29,6 +29,7 @@ import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.nio.file.Path; import java.util.Comparator; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -160,7 +161,7 @@ protected List getAllSyncedObjects(final String streamName, fin *

  • Construct the GCS client.
    • */ @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { final JsonNode baseConfigJson = getBaseConfigJson(); // Set a random GCS bucket path for each integration test final JsonNode configJson = Jsons.clone(baseConfigJson); diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java index 52f5d68907e7..fe1b3b30eb13 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java @@ -11,6 +11,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; @@ -26,6 +28,8 @@ class GcsAvroWriterTest { @Test public void generatesCorrectObjectPath() throws IOException { + DestinationConfig.initialize(Jsons.deserialize("{}")); + final GcsAvroWriter writer = new GcsAvroWriter( new GcsDestinationConfig( "fake-bucket", diff --git a/airbyte-integrations/connectors/destination-google-sheets/Dockerfile b/airbyte-integrations/connectors/destination-google-sheets/Dockerfile index dcbd96c665b5..0d239b92faa6 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/destination-google-sheets/Dockerfile @@ -6,12 +6,12 @@ RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" WORKDIR /airbyte/integration_code -COPY destination_google_sheets ./destination_google_sheets COPY setup.py ./ -COPY main.py ./ RUN pip install . +COPY destination_google_sheets ./destination_google_sheets +COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/destination-google-sheets diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py index ec072f9e5909..29d27ae744fc 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/helpers.py @@ -18,7 +18,7 @@ def get_spreadsheet_id(id_or_url: str) -> str: - if re.match(r"(http://)|(https://)", id_or_url): + if re.match(r"(https://)", id_or_url): m = re.search(r"(/)([-\w]{40,})([/]?)", id_or_url) if m.group(2): return m.group(2) diff --git a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/spec.json b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/spec.json index ee3013a461c8..d32d6ff96271 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/spec.json +++ b/airbyte-integrations/connectors/destination-google-sheets/destination_google_sheets/spec.json @@ -45,12 +45,42 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml b/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml index cd356804f6dd..223fda282f35 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: api connectorType: destination definitionId: a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a - dockerImageTag: 0.1.2 + dockerImageTag: 0.2.2 dockerRepository: airbyte/destination-google-sheets githubIssueLabel: destination-google-sheets icon: google-sheets.svg - license: MIT + license: ELv2 name: Google Sheets registries: cloud: @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/google-sheets tags: - language:python + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-iceberg/Dockerfile b/airbyte-integrations/connectors/destination-iceberg/Dockerfile index 241e82f2adde..283e51dd1f32 100644 --- a/airbyte-integrations/connectors/destination-iceberg/Dockerfile +++ b/airbyte-integrations/connectors/destination-iceberg/Dockerfile @@ -29,5 +29,5 @@ ENV JAVA_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED \ COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-iceberg diff --git a/airbyte-integrations/connectors/destination-iceberg/build.gradle b/airbyte-integrations/connectors/destination-iceberg/build.gradle index 8dcc5bb3c3f2..a16bb53aa73f 100644 --- a/airbyte-integrations/connectors/destination-iceberg/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg/build.gradle @@ -14,21 +14,30 @@ dependencies { implementation project(':airbyte-integrations:bases:base-java') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation('org.apache.spark:spark-sql_2.13:3.3.0') { + implementation('org.apache.spark:spark-sql_2.13:3.3.2') { exclude(group: 'org.apache.hadoop', module: 'hadoop-common') } - implementation('org.apache.spark:spark-hive_2.13:3.3.0') { + implementation('org.apache.spark:spark-hive_2.13:3.3.2') { exclude(group: 'org.apache.hadoop', module: 'hadoop-common') } - implementation 'org.apache.iceberg:iceberg-spark-runtime-3.3_2.13:1.0.0' - implementation "software.amazon.awssdk:bundle:2.17.131" - implementation "software.amazon.awssdk:url-connection-client:2.17.131" + implementation 'org.apache.iceberg:iceberg-spark-runtime-3.3_2.13:1.3.0' + + // force awssdk version required by Iceberg + implementation "software.amazon.awssdk:utils:2.20.18" + implementation "software.amazon.awssdk:url-connection-client:2.20.18" + implementation "software.amazon.awssdk:s3:2.20.18" + implementation "software.amazon.awssdk:glue:2.20.18" + implementation "software.amazon.awssdk:dynamodb:2.20.18" + implementation "software.amazon.awssdk:kms:2.20.18" + implementation "software.amazon.awssdk:sts:2.20.18" + implementation "software.amazon.awssdk:sdk-core:2.20.18" + implementation "software.amazon.awssdk:aws-core:2.20.18" + implementation "org.apache.hadoop:hadoop-aws:3.3.2" implementation "org.apache.hadoop:hadoop-client-api:3.3.2" implementation "org.apache.hadoop:hadoop-client-runtime:3.3.2" implementation "org.postgresql:postgresql:42.5.0" implementation "commons-collections:commons-collections:3.2.2" -// implementation "software.amazon.awssdk:utils:2.17.131" testImplementation libs.connectors.testcontainers.postgresql integrationTestJavaImplementation libs.connectors.testcontainers.postgresql diff --git a/airbyte-integrations/connectors/destination-iceberg/iceberg.svg b/airbyte-integrations/connectors/destination-iceberg/iceberg.svg new file mode 100644 index 000000000000..bed01742ad40 --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/iceberg.svg @@ -0,0 +1 @@ +iceberg \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml index 939304fa8afb..659f3fcafe61 100644 --- a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml +++ b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: df65a8f3-9908-451b-aa9b-445462803560 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.4 dockerRepository: airbyte/destination-iceberg githubIssueLabel: destination-iceberg license: MIT @@ -16,4 +16,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/iceberg tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConstants.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConstants.java index f4b0a620c854..2d83fb09c5f8 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConstants.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConstants.java @@ -27,6 +27,9 @@ public class IcebergConstants { public static final String JDBC_PASSWORD_CONFIG_KEY = "password"; public static final String JDBC_SSL_CONFIG_KEY = "ssl"; public static final String JDBC_CATALOG_SCHEMA_CONFIG_KEY = "catalog_schema"; + public static final String REST_CATALOG_URI_CONFIG_KEY = "rest_uri"; + public static final String REST_CATALOG_CREDENTIAL_CONFIG_KEY = "rest_credential"; + public static final String REST_CATALOG_TOKEN_CONFIG_KEY = "rest_token"; /** * Storage Config keys @@ -38,6 +41,7 @@ public class IcebergConstants { public static final String S3_BUCKET_REGION_CONFIG_KEY = "s3_bucket_region"; public static final String S3_ENDPOINT_CONFIG_KEY = "s3_endpoint"; public static final String S3_PATH_STYLE_ACCESS_CONFIG_KEY = "s3_path_style_access"; + public static final String MANAGED_WAREHOUSE_NAME = "managed_warehouse_name"; /** * Format Config keys diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java index ffa2090d8cb2..b67c277fef97 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java @@ -10,13 +10,13 @@ import static org.apache.logging.log4j.util.Strings.isNotBlank; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; import io.airbyte.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.integrations.destination.iceberg.config.WriteConfig; import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfig; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; @@ -93,7 +93,7 @@ protected void startTracked() throws Exception { throw new IllegalStateException("Undefined destination sync mode"); } final boolean isAppendMode = syncMode != DestinationSyncMode.OVERWRITE; - AirbyteStreamNameNamespacePair nameNamespacePair = AirbyteStreamNameNamespacePair.fromAirbyteSteam(stream.getStream()); + AirbyteStreamNameNamespacePair nameNamespacePair = AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()); Integer flushBatchSize = catalogConfig.getFormatConfig().getFlushBatchSize(); WriteConfig writeConfig = new WriteConfig(namespace, streamName, isAppendMode, flushBatchSize); configs.put(nameNamespacePair, writeConfig); diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/CatalogType.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/CatalogType.java index 2adfe0462592..3cbd63eedbb2 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/CatalogType.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/CatalogType.java @@ -10,5 +10,6 @@ public enum CatalogType { HIVE, HADOOP, - JDBC + JDBC, + REST } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfigFactory.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfigFactory.java index bd96a24d7dd1..f77dacfe4283 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfigFactory.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfigFactory.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.destination.iceberg.config.format.FormatConfig; import io.airbyte.integrations.destination.iceberg.config.storage.S3Config; +import io.airbyte.integrations.destination.iceberg.config.storage.ServerManagedStorageConfig; import io.airbyte.integrations.destination.iceberg.config.storage.StorageConfig; import io.airbyte.integrations.destination.iceberg.config.storage.StorageType; import javax.annotation.Nonnull; @@ -38,7 +39,10 @@ public IcebergCatalogConfig fromJsonNodeConfig(@Nonnull final JsonNode config) { IcebergCatalogConfig icebergCatalogConfig = genIcebergCatalogConfig(catalogConfigJson); icebergCatalogConfig.formatConfig = formatConfig; icebergCatalogConfig.storageConfig = storageConfig; - icebergCatalogConfig.setDefaultOutputDatabase(catalogConfigJson.get(DEFAULT_DATABASE_CONFIG_KEY).asText()); + JsonNode defaultDb = catalogConfigJson.get(DEFAULT_DATABASE_CONFIG_KEY); + if (null != defaultDb) { + icebergCatalogConfig.setDefaultOutputDatabase(defaultDb.asText()); + } return icebergCatalogConfig; } @@ -52,6 +56,8 @@ private StorageConfig genStorageConfig(JsonNode storageConfigJson) { switch (storageType) { case S3: return S3Config.fromDestinationConfig(storageConfigJson); + case MANAGED: + return ServerManagedStorageConfig.fromDestinationConfig(storageConfigJson); case HDFS: default: throw new RuntimeException("Unexpected storage config: " + storageTypeStr); @@ -70,6 +76,7 @@ private static IcebergCatalogConfig genIcebergCatalogConfig(@NotNull JsonNode ca case HIVE -> new HiveCatalogConfig(catalogConfigJson); case HADOOP -> new HadoopCatalogConfig(catalogConfigJson); case JDBC -> new JdbcCatalogConfig(catalogConfigJson); + case REST -> new RESTCatalogConfig(catalogConfigJson); default -> throw new RuntimeException("Unexpected catalog config: " + catalogTypeStr); }; } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/RESTCatalogConfig.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/RESTCatalogConfig.java new file mode 100644 index 000000000000..9498d0e5035d --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/RESTCatalogConfig.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg.config.catalog; + +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.CATALOG_NAME; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.REST_CATALOG_CREDENTIAL_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.REST_CATALOG_TOKEN_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.REST_CATALOG_URI_CONFIG_KEY; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import java.util.HashMap; +import java.util.Map; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.jetbrains.annotations.NotNull; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = false) +public class RESTCatalogConfig + extends IcebergCatalogConfig { + + private final String uri; + private final String credential; + private final String token; + + public RESTCatalogConfig(@NotNull JsonNode catalogConfig) { + Preconditions.checkArgument(null != catalogConfig.get(REST_CATALOG_URI_CONFIG_KEY), "%s is required", REST_CATALOG_URI_CONFIG_KEY); + this.uri = catalogConfig.get(REST_CATALOG_URI_CONFIG_KEY).asText(); + JsonNode credentialNode = catalogConfig.get(REST_CATALOG_CREDENTIAL_CONFIG_KEY); + JsonNode tokenNode = catalogConfig.get(REST_CATALOG_TOKEN_CONFIG_KEY); + this.credential = null != credentialNode ? credentialNode.asText() : null; + this.token = null != tokenNode ? tokenNode.asText() : null; + } + + @Override + public Map sparkConfigMap() { + Map configMap = new HashMap<>(); + configMap.put("spark.network.timeout", "300000"); + configMap.put("spark.sql.defaultCatalog", CATALOG_NAME); + configMap.put("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions"); + configMap.put("spark.sql.catalog." + CATALOG_NAME, "org.apache.iceberg.spark.SparkCatalog"); + configMap.put("spark.sql.catalog." + CATALOG_NAME + ".catalog-impl", "org.apache.iceberg.rest.RESTCatalog"); + configMap.put("spark.sql.catalog." + CATALOG_NAME + ".uri", this.uri); + configMap.put("spark.driver.extraJavaOptions", "-Dpackaging.type=jar -Djava.io.tmpdir=/tmp"); + + if (isNotBlank(this.credential)) { + configMap.put("spark.sql.catalog." + CATALOG_NAME + ".credential", this.credential); + } + if (isNotBlank(this.token)) { + configMap.put("spark.sql.catalog." + CATALOG_NAME + ".token", this.token); + } + + configMap.putAll(this.storageConfig.sparkConfigMap(CATALOG_NAME)); + return configMap; + } + + @Override + public Catalog genCatalog() { + RESTCatalog catalog = new RESTCatalog(); + Map properties = new HashMap<>(this.storageConfig.catalogInitializeProperties()); + properties.put(CatalogProperties.URI, this.uri); + if (isNotBlank(this.credential)) { + properties.put(OAuth2Properties.CREDENTIAL, this.credential); + } + if (isNotBlank(this.token)) { + properties.put(OAuth2Properties.TOKEN, this.token); + } + properties.put(CatalogProperties.WAREHOUSE_LOCATION, this.storageConfig.getWarehouseUri()); + catalog.initialize(CATALOG_NAME, properties); + return catalog; + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/ServerManagedStorageConfig.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/ServerManagedStorageConfig.java new file mode 100644 index 000000000000..3ae250ac01a5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/ServerManagedStorageConfig.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg.config.storage; + +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.MANAGED_WAREHOUSE_NAME; +import static io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfigFactory.getProperty; +import static org.apache.commons.lang3.StringUtils.isBlank; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nonnull; + +public class ServerManagedStorageConfig implements StorageConfig { + + private final String warehouseName; + + public ServerManagedStorageConfig(String warehouseName) { + this.warehouseName = warehouseName; + } + + @Override + public void check() throws Exception {} + + @Override + public String getWarehouseUri() { + return warehouseName; + } + + public static ServerManagedStorageConfig fromDestinationConfig(@Nonnull final JsonNode config) { + String warehouseName = getProperty(config, MANAGED_WAREHOUSE_NAME); + if (isBlank(warehouseName)) { + throw new IllegalArgumentException(MANAGED_WAREHOUSE_NAME + " cannot be null"); + } + + return new ServerManagedStorageConfig(warehouseName); + } + + @Override + public Map sparkConfigMap(String catalogName) { + Map sparkConfig = new HashMap<>(); + sparkConfig.put("spark.sql.catalog." + catalogName + ".warehouse", warehouseName); + return sparkConfig; + } + + @Override + public Map catalogInitializeProperties() { + return ImmutableMap.of(); + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/StorageType.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/StorageType.java index 05f133853310..f5e5d7b6f302 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/StorageType.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/storage/StorageType.java @@ -9,5 +9,6 @@ */ public enum StorageType { S3, - HDFS; + HDFS, + MANAGED; } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json index 8b95191c41b0..503b24a6cf71 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json @@ -118,6 +118,42 @@ "order": 6 } } + }, + { + "title": "RESTCatalog", + "description": "The RESTCatalog connects to a REST server at the specified URI", + "required": ["catalog_type", "rest_uri"], + "properties": { + "catalog_type": { + "title": "Catalog Type", + "type": "string", + "default": "Rest", + "enum": ["Rest"], + "order": 0 + }, + "rest_uri": { + "title": "REST Server URI", + "type": "string", + "examples": ["http://localhost:12345"], + "order": 1 + }, + "rest_credential": { + "title": "A credential to exchange for a token in the OAuth2 client credentials flow.", + "type": "string", + "airbyte_secret": true, + "examples": ["username:password"], + "order": 2 + }, + "rest_token": { + "title": "A Bearer token which will be used for interaction with the server.", + "type": "string", + "airbyte_secret": true, + "examples": [ + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c" + ], + "order": 3 + } + } } ], "order": 0 @@ -222,6 +258,27 @@ "order": 5 } } + }, + { + "title": "Server-managed", + "type": "object", + "description": "Server-managed object storage", + "required": ["storage_type", "managed_warehouse_name"], + "properties": { + "storage_type": { + "title": "Storage Type", + "type": "string", + "default": "MANAGED", + "enum": ["MANAGED"], + "order": 0 + }, + "managed_warehouse_name": { + "type": "string", + "description": "The name of the managed warehouse", + "title": "Warehouse name", + "order": 0 + } + } } ], "order": 1 diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/container/RESTServerWithMinioCompose.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/container/RESTServerWithMinioCompose.java new file mode 100644 index 000000000000..0a2742c0aacf --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/container/RESTServerWithMinioCompose.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg.container; + +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.ICEBERG_CATALOG_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.ICEBERG_CATALOG_TYPE_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.ICEBERG_FORMAT_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.ICEBERG_STORAGE_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.ICEBERG_STORAGE_TYPE_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.REST_CATALOG_URI_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.S3_ACCESS_KEY_ID_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.S3_BUCKET_REGION_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.S3_ENDPOINT_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.S3_SECRET_KEY_CONFIG_KEY; +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.S3_WAREHOUSE_URI_CONFIG_KEY; +import static java.util.Map.entry; +import static java.util.Map.ofEntries; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; +import io.airbyte.integrations.destination.iceberg.hive.IcebergHiveCatalogS3ParquetIntegrationTest; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +public class RESTServerWithMinioCompose extends DockerComposeContainer { + + private static final Logger LOGGER = LoggerFactory.getLogger(IcebergHiveCatalogS3ParquetIntegrationTest.class); + private static final String LOCAL_RELATIVE_PATH = "src/test-integration/resources/"; + private static final String COMPOSE_PATH = LOCAL_RELATIVE_PATH + "rest-catalog-compose.yml"; + private static final int REST_SERVER_PORT = 8181; + private static final int MINIO_PORT = 9000; + private static final String REST_SERVICE_NAME = "rest_1"; + private static final String MINIO_SERVICE_NAME = "minio_1"; + + public RESTServerWithMinioCompose() { + super(Path.of(COMPOSE_PATH).toFile()); + super.withExposedService(REST_SERVICE_NAME, + REST_SERVER_PORT, + Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(30))) + .withExposedService(MINIO_SERVICE_NAME, + MinioContainer.MINIO_PORT, + Wait.forHttp(MinioContainer.HEALTH_ENDPOINT).withStartupTimeout(Duration.ofSeconds(60))) + .withLocalCompose(true); + } + + @Override + public void start() { + long startTime = System.currentTimeMillis(); + super.start(); + LOGGER.info("REST Server port: {}", getServicePort(REST_SERVICE_NAME, REST_SERVER_PORT)); + LOGGER.info("Minio port: {}", getServicePort(MINIO_SERVICE_NAME, MINIO_PORT)); + LOGGER.info("REST Server docker-compose startup cost: {} ms", System.currentTimeMillis() - startTime); + } + + public String s3Endpoint() { + return "http://localhost:" + getServicePort(MINIO_SERVICE_NAME, MINIO_PORT); + } + + public String restServerUri() { + return "http://localhost:" + getServicePort(REST_SERVICE_NAME, REST_SERVER_PORT); + } + + public JsonNode getComposeConfig(DataFileFormat fileFormat) { + String s3Endpoint = this.s3Endpoint(); + LOGGER.info("Configure S3 endpoint to {}", s3Endpoint); + return Jsons.jsonNode(ofEntries( + entry(ICEBERG_CATALOG_CONFIG_KEY, + Jsons.jsonNode(ofEntries( + entry(ICEBERG_CATALOG_TYPE_CONFIG_KEY, "Rest"), + entry(REST_CATALOG_URI_CONFIG_KEY, this.restServerUri())))), + entry(ICEBERG_STORAGE_CONFIG_KEY, + Jsons.jsonNode(ofEntries( + entry(ICEBERG_STORAGE_TYPE_CONFIG_KEY, "S3"), + entry(S3_ACCESS_KEY_ID_CONFIG_KEY, "admin"), + entry(S3_SECRET_KEY_CONFIG_KEY, "password"), + entry(S3_WAREHOUSE_URI_CONFIG_KEY, "s3://warehouse/rest"), + entry(S3_BUCKET_REGION_CONFIG_KEY, "us-east-1"), + entry(S3_ENDPOINT_CONFIG_KEY, s3Endpoint)))), + entry(ICEBERG_FORMAT_CONFIG_KEY, + Jsons.jsonNode(Map.of("format", fileFormat.getConfigValue()))))); + } + + public JsonNode getWrongConfig() { + return Jsons.jsonNode(ofEntries( + entry(ICEBERG_CATALOG_CONFIG_KEY, + Jsons.jsonNode(ofEntries( + entry(ICEBERG_CATALOG_TYPE_CONFIG_KEY, "Rest"), + entry(REST_CATALOG_URI_CONFIG_KEY, "wrong-host:1234")))), + entry(ICEBERG_STORAGE_CONFIG_KEY, + Jsons.jsonNode(ofEntries(entry(ICEBERG_STORAGE_TYPE_CONFIG_KEY, "S3"), + entry(S3_ACCESS_KEY_ID_CONFIG_KEY, "wrong_access_key"), + entry(S3_SECRET_KEY_CONFIG_KEY, "wrong_secret_key"), + entry(S3_WAREHOUSE_URI_CONFIG_KEY, "s3://warehouse/"), + entry(S3_BUCKET_REGION_CONFIG_KEY, "us-east-1"), + entry(S3_ENDPOINT_CONFIG_KEY, this.s3Endpoint())))), + entry(ICEBERG_FORMAT_CONFIG_KEY, Jsons.jsonNode(Map.of("format", "wrong-format"))))); + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java index c3bba8d9147e..db297bb4cab8 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java @@ -29,6 +29,7 @@ import io.airbyte.integrations.destination.iceberg.container.MinioContainer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.util.HostPortResolver; +import java.util.HashSet; import java.util.List; import java.util.Map; import org.slf4j.Logger; @@ -43,7 +44,7 @@ public abstract class BaseIcebergHadoopCatalogS3IntegrationTest extends Destinat private MinioContainer s3Storage; @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { s3Storage = IcebergIntegrationTestUtil.createAndStartMinioContainer(null); IcebergIntegrationTestUtil.createS3WarehouseBucket(getConfig()); } @@ -60,7 +61,7 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - String s3Endpoint = "http://" + s3Storage.getHostAddress(); + final String s3Endpoint = "http://" + s3Storage.getHostAddress(); LOGGER.info("Configurate S3 endpoint to {}", s3Endpoint); return Jsons.jsonNode(ofEntries( entry(ICEBERG_CATALOG_CONFIG_KEY, @@ -79,7 +80,7 @@ protected JsonNode getConfig() { @Override protected JsonNode getFailCheckConfig() { - String s3Endpoint = "http://%s:%s".formatted(HostPortResolver.resolveHost(s3Storage), + final String s3Endpoint = "http://%s:%s".formatted(HostPortResolver.resolveHost(s3Storage), HostPortResolver.resolvePort(s3Storage)); return Jsons.jsonNode(ofEntries( entry(ICEBERG_CATALOG_CONFIG_KEY, @@ -97,10 +98,10 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { return IcebergIntegrationTestUtil.retrieveRecords(getConfig(), namespace, streamName); } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java index 1f40b7a4e25c..734184a887a5 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java @@ -11,6 +11,7 @@ import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.HiveMetastoreS3PostgresCompose; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -48,7 +49,7 @@ public static void stopCompose() { } @Override - protected void setup(final TestDestinationEnv testEnv) {} + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) {} @Override protected void tearDown(final TestDestinationEnv testEnv) {} @@ -69,10 +70,10 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { return IcebergIntegrationTestUtil.retrieveRecords(getConfig(), namespace, streamName); } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java index 128a48a81145..9fca8b9682aa 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java @@ -11,6 +11,7 @@ import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.HiveMetastoreS3PostgresCompose; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -48,7 +49,7 @@ public static void stopCompose() { } @Override - protected void setup(final TestDestinationEnv testEnv) {} + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) {} @Override protected void tearDown(final TestDestinationEnv testEnv) {} @@ -69,10 +70,10 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { return IcebergIntegrationTestUtil.retrieveRecords(getConfig(), namespace, streamName); } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java index 6a5c385838e9..3839f0d3fb8f 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java @@ -34,6 +34,7 @@ import io.airbyte.integrations.destination.iceberg.container.MinioContainer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.util.HostPortResolver; +import java.util.HashSet; import java.util.List; import java.util.Map; import org.slf4j.Logger; @@ -52,7 +53,7 @@ public abstract class BaseIcebergJdbcCatalogS3IntegrationTest extends Destinatio private MinioContainer s3Storage; @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { catalogDb = new PostgreSQLContainer<>("postgres:13-alpine"); catalogDb.start(); LOGGER.info("==> Started PostgreSQL docker container..."); @@ -74,9 +75,9 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - String jdbcUrl = catalogDb.getJdbcUrl(); + final String jdbcUrl = catalogDb.getJdbcUrl(); LOGGER.info("Postgresql jdbc url: {}", jdbcUrl); - String s3Endpoint = "http://" + s3Storage.getHostAddress(); + final String s3Endpoint = "http://" + s3Storage.getHostAddress(); return Jsons.jsonNode(ofEntries( entry(ICEBERG_CATALOG_CONFIG_KEY, Jsons.jsonNode(ofEntries( @@ -99,10 +100,10 @@ protected JsonNode getConfig() { @Override protected JsonNode getFailCheckConfig() { - String jdbcUrl = "jdbc:postgresql://%s:%d/%s".formatted(HostPortResolver.resolveHost(catalogDb), + final String jdbcUrl = "jdbc:postgresql://%s:%d/%s".formatted(HostPortResolver.resolveHost(catalogDb), HostPortResolver.resolvePort(catalogDb), catalogDb.getDatabaseName()); - String s3Endpoint = "http://%s:%s".formatted(HostPortResolver.resolveHost(s3Storage), + final String s3Endpoint = "http://%s:%s".formatted(HostPortResolver.resolveHost(s3Storage), HostPortResolver.resolvePort(s3Storage)); return Jsons.jsonNode(ofEntries( entry(ICEBERG_CATALOG_CONFIG_KEY, @@ -125,10 +126,10 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws Exception { return IcebergIntegrationTestUtil.retrieveRecords(getConfig(), namespace, streamName); } diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java new file mode 100644 index 000000000000..49c2f66986fe --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg.rest; + +import static io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil.ICEBERG_IMAGE_NAME; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil; +import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; +import io.airbyte.integrations.destination.iceberg.container.RESTServerWithMinioCompose; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.util.HashSet; +import java.util.List; +import org.junit.jupiter.api.AfterAll; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BaseIcebergRESTCatalogS3IntegrationTest extends DestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseIcebergRESTCatalogS3IntegrationTest.class); + + private static RESTServerWithMinioCompose composeContainer; + private static JsonNode config; + + static void startCompose(final DataFileFormat fileFormat) { + composeContainer = new RESTServerWithMinioCompose(); + composeContainer.start(); + config = composeContainer.getComposeConfig(fileFormat); + IcebergIntegrationTestUtil.createS3WarehouseBucket(config); + LOGGER.info("==> Started REST Server with Minio - Docker Compose..."); + } + + @AfterAll + public static void stopCompose() { + IcebergIntegrationTestUtil.stopAndCloseContainer(composeContainer, "REST Server with Minio - Docker Compose"); + } + + @Override + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) {} + + @Override + protected void tearDown(final TestDestinationEnv testEnv) {} + + @Override + protected String getImageName() { + return ICEBERG_IMAGE_NAME; + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected JsonNode getFailCheckConfig() { + return composeContainer.getWrongConfig(); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws Exception { + return IcebergIntegrationTestUtil.retrieveRecords(getConfig(), namespace, streamName); + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/IcebergRESTCatalogS3AvroIntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/IcebergRESTCatalogS3AvroIntegrationTest.java new file mode 100644 index 000000000000..c2071e22737c --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/IcebergRESTCatalogS3AvroIntegrationTest.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg.rest; + +import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; +import org.junit.jupiter.api.BeforeAll; + +public class IcebergRESTCatalogS3AvroIntegrationTest extends BaseIcebergRESTCatalogS3IntegrationTest { + + @BeforeAll + public static void startCompose() { + startCompose(DataFileFormat.AVRO); + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/IcebergRESTCatalogS3ParquetIntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/IcebergRESTCatalogS3ParquetIntegrationTest.java new file mode 100644 index 000000000000..78573afef283 --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/IcebergRESTCatalogS3ParquetIntegrationTest.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg.rest; + +import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; +import org.junit.jupiter.api.BeforeAll; + +public class IcebergRESTCatalogS3ParquetIntegrationTest extends BaseIcebergRESTCatalogS3IntegrationTest { + + @BeforeAll + public static void startCompose() { + startCompose(DataFileFormat.PARQUET); + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/metastore-log4j.properties b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/metastore-log4j.properties index c8e23faffb3f..52aca04c4e98 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/metastore-log4j.properties +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/metastore-log4j.properties @@ -6,7 +6,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml new file mode 100644 index 000000000000..9eb4ad04d010 --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml @@ -0,0 +1,42 @@ +version: "3" + +services: + rest: + image: tabulario/iceberg-rest + networks: + iceberg_net: + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + - CATALOG_WAREHOUSE=s3://warehouse/rest + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio:9000 + minio: + image: minio/minio + environment: + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password + - MINIO_DOMAIN=minio + networks: + iceberg_net: + aliases: + - warehouse.minio + command: ["server", "/data"] + mc: + depends_on: + - minio + image: minio/mc + networks: + iceberg_net: + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + entrypoint: > + /bin/sh -c " + until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; + tail -f /dev/null + " +networks: + iceberg_net: \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test/java/io/airbyte/integrations/destination/iceberg/IcebergRESTCatalogConfigTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test/java/io/airbyte/integrations/destination/iceberg/IcebergRESTCatalogConfigTest.java new file mode 100644 index 000000000000..dd87c414e111 --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test/java/io/airbyte/integrations/destination/iceberg/IcebergRESTCatalogConfigTest.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg; + +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.FORMAT_TYPE_CONFIG_KEY; +import static java.util.Map.entry; +import static java.util.Map.ofEntries; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.UploadPartRequest; +import com.amazonaws.services.s3.model.UploadPartResult; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfig; +import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfigFactory; +import io.airbyte.integrations.destination.iceberg.config.catalog.RESTCatalogConfig; +import io.airbyte.integrations.destination.iceberg.config.format.FormatConfig; +import io.airbyte.integrations.destination.iceberg.config.storage.S3Config; +import io.airbyte.integrations.destination.iceberg.config.storage.credential.S3AccessKeyCredentialConfig; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableScan; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.IcebergGenerics.ScanBuilder; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.spark.SparkCatalog; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +@Slf4j +class IcebergRESTCatalogConfigTest { + + private static final String FAKE_WAREHOUSE_URI = "s3://fake-bucket"; + private static final String FAKE_ENDPOINT = "fake-endpoint"; + private static final String FAKE_ENDPOINT_WITH_SCHEMA = "https://fake-endpoint"; + private static final String FAKE_ACCESS_KEY_ID = "fake-accessKeyId"; + private static final String FAKE_SECRET_ACCESS_KEY = "fake-secretAccessKey"; + private static final String FAKE_REST_URI = "http://fake-rest-uri"; + private static MockedStatic mockedIcebergGenerics; + + private AmazonS3 s3; + private RESTCatalogConfig config; + private Catalog catalog; + private IcebergCatalogConfigFactory factory; + + @BeforeAll + static void staticSetup() { + IcebergRESTCatalogConfigTest.mockedIcebergGenerics = mockStatic(IcebergGenerics.class); + } + + @AfterAll + static void staticStop() { + IcebergRESTCatalogConfigTest.mockedIcebergGenerics.close(); + } + + @BeforeEach + void setup() { + s3 = mock(AmazonS3.class); + final InitiateMultipartUploadResult uploadResult = mock(InitiateMultipartUploadResult.class); + final UploadPartResult uploadPartResult = mock(UploadPartResult.class); + when(s3.uploadPart(any(UploadPartRequest.class))).thenReturn(uploadPartResult); + when(s3.initiateMultipartUpload(any(InitiateMultipartUploadRequest.class))).thenReturn(uploadResult); + + TableScan tableScan = mock(TableScan.class); + when(tableScan.schema()).thenReturn(null); + Table tempTable = mock(Table.class); + when(tempTable.newScan()).thenReturn(tableScan); + ScanBuilder scanBuilder = mock(ScanBuilder.class); + when(scanBuilder.build()).thenReturn(new EmptyIterator()); + when(IcebergGenerics.read(tempTable)).thenReturn(scanBuilder); + + catalog = mock(Catalog.class); + when(catalog.createTable(any(), any())).thenReturn(tempTable); + when(catalog.dropTable(any())).thenReturn(true); + + JsonNode jsonNode = Jsons.jsonNode(ofEntries(entry(IcebergConstants.REST_CATALOG_URI_CONFIG_KEY, FAKE_REST_URI))); + + config = new RESTCatalogConfig(jsonNode); + config.setStorageConfig(S3Config.builder() + .warehouseUri(FAKE_WAREHOUSE_URI) + .bucketRegion("fake-region") + .endpoint(FAKE_ENDPOINT) + .endpointWithSchema(FAKE_ENDPOINT_WITH_SCHEMA) + .accessKeyId(FAKE_ACCESS_KEY_ID) + .secretKey(FAKE_SECRET_ACCESS_KEY) + .credentialConfig(new S3AccessKeyCredentialConfig(FAKE_ACCESS_KEY_ID, FAKE_SECRET_ACCESS_KEY)) + .s3Client(s3) + .build()); + config.setFormatConfig(new FormatConfig(Jsons.jsonNode(ImmutableMap.of(FORMAT_TYPE_CONFIG_KEY, "Parquet")))); + config.setDefaultOutputDatabase("default"); + + factory = new IcebergCatalogConfigFactory() { + + @Override + public IcebergCatalogConfig fromJsonNodeConfig(final @NotNull JsonNode jsonConfig) { + return config; + } + + }; + } + + @Test + public void checksRESTServerUri() { + final IcebergDestination destinationFail = new IcebergDestination(); + final AirbyteConnectionStatus status = destinationFail.check(Jsons.deserialize(""" + { + "catalog_config": { + "catalog_type": "REST", + "database": "test" + }, + "storage_config": { + "storage_type": "S3", + "access_key_id": "xxxxxxxxxxx", + "secret_access_key": "yyyyyyyyyyyy", + "s3_warehouse_uri": "s3://warehouse/hive", + "s3_bucket_region": "us-east-1", + "s3_endpoint": "your-own-minio-host:9000" + }, + "format_config": { + "format": "Parquet" + } + }""")); + log.info("status={}", status); + assertThat(status.getStatus()).isEqualTo(Status.FAILED); + assertThat(status.getMessage()).contains("rest_uri is required"); + } + + @Test + public void restCatalogSparkConfigTest() { + Map sparkConfig = config.sparkConfigMap(); + log.info("Spark Config for REST catalog: {}", sparkConfig); + + // Catalog config + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.catalog-impl")).isEqualTo(RESTCatalog.class.getName()); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.uri")).isEqualTo(FAKE_REST_URI); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg")).isEqualTo(SparkCatalog.class.getName()); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.io-impl")).isEqualTo(S3FileIO.class.getName()); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.warehouse")).isEqualTo(FAKE_WAREHOUSE_URI); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.s3.access-key-id")).isEqualTo(FAKE_ACCESS_KEY_ID); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.s3.secret-access-key")).isEqualTo(FAKE_SECRET_ACCESS_KEY); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.s3.endpoint")).isEqualTo(FAKE_ENDPOINT_WITH_SCHEMA); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.s3.path-style-access")).isEqualTo("false"); + + // Hadoop config + assertThat(sparkConfig.get("spark.hadoop.fs.s3a.endpoint")).isEqualTo(FAKE_ENDPOINT); + assertThat(sparkConfig.get("spark.hadoop.fs.s3a.access.key")).isEqualTo(FAKE_ACCESS_KEY_ID); + assertThat(sparkConfig.get("spark.hadoop.fs.s3a.secret.key")).isEqualTo(FAKE_SECRET_ACCESS_KEY); + assertThat(sparkConfig.get("spark.hadoop.fs.s3a.impl")).isEqualTo(S3AFileSystem.class.getName()); + assertThat(sparkConfig.get("spark.hadoop.fs.s3a.connection.ssl.enabled")).isEqualTo("false"); + } + + @Test + public void s3ConfigForCatalogInitializeTest() { + Map properties = config.getStorageConfig().catalogInitializeProperties(); + log.info("S3 Config for RESTCatalog Initialize: {}", properties); + + assertThat(properties.get("io-impl")).isEqualTo(S3FileIO.class.getName()); + assertThat(properties.get("s3.endpoint")).isEqualTo(FAKE_ENDPOINT_WITH_SCHEMA); + assertThat(properties.get("s3.access-key-id")).isEqualTo(FAKE_ACCESS_KEY_ID); + assertThat(properties.get("s3.secret-access-key")).isEqualTo(FAKE_SECRET_ACCESS_KEY); + assertThat(properties.get("s3.path-style-access")).isEqualTo("false"); + } + +} diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test/java/io/airbyte/integrations/destination/iceberg/IcebergRESTCatalogServerManagedConfigTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test/java/io/airbyte/integrations/destination/iceberg/IcebergRESTCatalogServerManagedConfigTest.java new file mode 100644 index 000000000000..4082d709707d --- /dev/null +++ b/airbyte-integrations/connectors/destination-iceberg/src/test/java/io/airbyte/integrations/destination/iceberg/IcebergRESTCatalogServerManagedConfigTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.iceberg; + +import static io.airbyte.integrations.destination.iceberg.IcebergConstants.FORMAT_TYPE_CONFIG_KEY; +import static java.util.Map.entry; +import static java.util.Map.ofEntries; +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.iceberg.config.catalog.RESTCatalogConfig; +import io.airbyte.integrations.destination.iceberg.config.format.FormatConfig; +import io.airbyte.integrations.destination.iceberg.config.storage.ServerManagedStorageConfig; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.spark.SparkCatalog; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@Slf4j +class IcebergRESTCatalogServerManagedConfigTest { + + private static final String FAKE_WAREHOUSE_NAME = "fake-warehouse"; + private static final String FAKE_REST_URI = "http://fake-rest-uri"; + private static final String FAKE_CREDENTIAL = "fake-credential"; + private static final String FAKE_TOKEN = "fake-token"; + + private RESTCatalogConfig config; + + @BeforeEach + void setup() { + JsonNode jsonNode = Jsons.jsonNode(ofEntries(entry(IcebergConstants.REST_CATALOG_URI_CONFIG_KEY, FAKE_REST_URI), + entry(IcebergConstants.REST_CATALOG_CREDENTIAL_CONFIG_KEY, FAKE_CREDENTIAL), + entry(IcebergConstants.REST_CATALOG_TOKEN_CONFIG_KEY, FAKE_TOKEN))); + + config = new RESTCatalogConfig(jsonNode); + config.setStorageConfig(new ServerManagedStorageConfig(FAKE_WAREHOUSE_NAME)); + config.setFormatConfig(new FormatConfig(Jsons.jsonNode(ImmutableMap.of(FORMAT_TYPE_CONFIG_KEY, "Parquet")))); + config.setDefaultOutputDatabase("default"); + } + + @Test + public void checksRESTServerUri() { + final IcebergDestination destinationFail = new IcebergDestination(); + final AirbyteConnectionStatus status = destinationFail.check(Jsons.deserialize(""" + { + "catalog_config": { + "catalog_type": "REST", + "rest_credential": "fake-credential", + "rest_token": "fake-token", + "database": "test" + }, + "storage_config": { + "storage_type": "MANAGED", + "managed_warehouse_name": "fake-warehouse" + }, + "format_config": { + "format": "Parquet" + } + }""")); + log.info("status={}", status); + assertThat(status.getStatus()).isEqualTo(Status.FAILED); + assertThat(status.getMessage()).contains("rest_uri is required"); + } + + @Test + public void restCatalogSparkConfigTest() { + Map sparkConfig = config.sparkConfigMap(); + log.info("Spark Config for REST catalog: {}", sparkConfig); + + // Catalog config + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.catalog-impl")).isEqualTo(RESTCatalog.class.getName()); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.uri")).isEqualTo(FAKE_REST_URI); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.credential")).isEqualTo(FAKE_CREDENTIAL); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.token")).isEqualTo(FAKE_TOKEN); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg")).isEqualTo(SparkCatalog.class.getName()); + assertThat(sparkConfig.get("spark.sql.catalog.iceberg.warehouse")).isEqualTo(FAKE_WAREHOUSE_NAME); + } + + @Test + public void s3ConfigForCatalogInitializeTest() { + Map properties = config.getStorageConfig().catalogInitializeProperties(); + assertThat(properties).isEmpty(); + } + +} diff --git a/airbyte-integrations/connectors/destination-kafka/metadata.yaml b/airbyte-integrations/connectors/destination-kafka/metadata.yaml index 3d45a04fe177..36c58d103c20 100644 --- a/airbyte-integrations/connectors/destination-kafka/metadata.yaml +++ b/airbyte-integrations/connectors/destination-kafka/metadata.yaml @@ -10,11 +10,15 @@ data: name: Kafka registries: cloud: - enabled: false + enabled: false # hide Kafka Destination https://github.com/airbytehq/airbyte-cloud/issues/2610 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/kafka tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java index 11d4653f6245..9de3ea4ab3d0 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java @@ -19,6 +19,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.kafka.clients.consumer.ConsumerConfig; @@ -166,7 +167,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { KAFKA = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.0")); KAFKA.start(); } diff --git a/airbyte-integrations/connectors/destination-keen/metadata.yaml b/airbyte-integrations/connectors/destination-keen/metadata.yaml index 006307752275..64963c7874be 100644 --- a/airbyte-integrations/connectors/destination-keen/metadata.yaml +++ b/airbyte-integrations/connectors/destination-keen/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/keen tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java b/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java index aa79277d811b..07c5d7eababa 100644 --- a/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java +++ b/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java @@ -96,7 +96,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { if (!Files.exists(Path.of(SECRET_FILE_PATH))) { throw new IllegalStateException( "Must provide path to a file containing Keen account credentials: Project ID and Master API Key. " + diff --git a/airbyte-integrations/connectors/destination-kinesis/metadata.yaml b/airbyte-integrations/connectors/destination-kinesis/metadata.yaml index 80024ee48fc2..144113267bf6 100644 --- a/airbyte-integrations/connectors/destination-kinesis/metadata.yaml +++ b/airbyte-integrations/connectors/destination-kinesis/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/kinesis tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java index c6946b195a72..965f4d7c1d59 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java @@ -10,6 +10,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; @@ -34,7 +35,7 @@ static void initContainer() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { configJson = KinesisDataFactory.jsonConfig( kinesisContainer.getEndpointOverride().toString(), kinesisContainer.getRegion(), @@ -65,7 +66,7 @@ protected boolean supportObjectDataTypeTest() { } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { kinesisStream.deleteAllStreams(); } @@ -94,11 +95,11 @@ protected JsonNode getFailCheckConfig() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) { - var stream = kinesisNameTransformer.streamName(namespace, streamName); + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + final var stream = kinesisNameTransformer.streamName(namespace, streamName); return kinesisStream.getRecords(stream).stream() .sorted(Comparator.comparing(KinesisRecord::getTimestamp)) .map(KinesisRecord::getData) diff --git a/airbyte-integrations/connectors/destination-langchain/.dockerignore b/airbyte-integrations/connectors/destination-langchain/.dockerignore new file mode 100644 index 000000000000..188ab8cdf977 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_langchain +!setup.py diff --git a/airbyte-integrations/connectors/destination-langchain/Dockerfile b/airbyte-integrations/connectors/destination-langchain/Dockerfile new file mode 100644 index 000000000000..1472c50f57d4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/Dockerfile @@ -0,0 +1,46 @@ +FROM python:3.10-slim as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version + +RUN apt-get update \ + && pip install --upgrade pip \ + && apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 + +COPY setup.py ./ + +RUN pip install --upgrade pip + +# This is required because the current connector dependency is not compatible with the CDK version +# An older CDK version will be used, which depends on pyYAML 5.4, for which we need to pin Cython to <3.0 +# As of today the CDK version that satisfies the main dependency requirements, is 0.1.80 ... +RUN pip install --prefix=/install "Cython<3.0" "pyyaml~=5.4" --no-build-isolation + +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apt-get install bash + +# copy payload code only +COPY main.py ./ +COPY destination_langchain ./destination_langchain + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.0.8 +LABEL io.airbyte.name=airbyte/destination-langchain diff --git a/airbyte-integrations/connectors/destination-langchain/README.md b/airbyte-integrations/connectors/destination-langchain/README.md new file mode 100644 index 000000000000..5de181d487d8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/README.md @@ -0,0 +1,123 @@ +# Langchain Destination + +This is the repository for the Langchain destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/langchain). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.10.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-langchain:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/langchain) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_langchain/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination langchain test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-langchain:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-langchain:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-langchain:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-langchain:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-langchain:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-langchain:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-langchain:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-langchain/bootstrap.md b/airbyte-integrations/connectors/destination-langchain/bootstrap.md new file mode 100644 index 000000000000..2554eaf1cafe --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/bootstrap.md @@ -0,0 +1,30 @@ +# Langchain Destination Connector Bootstrap + +This destination does three things: +* Split records into chunks and separates metadata from text data +* Embeds text data into an embedding vector +* Stores the metadata and embedding vector in a vector database + +The record processing is using the text split components from https://python.langchain.com/docs/modules/data_connection/document_transformers/. + +There are two possible providers for generating embeddings, [OpenAI](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/openai) and [Fake embeddings](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) for testing purposes. + +Embedded documents are stored in a vector database. Currently, [Pinecone](https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/pinecone) and the locally stored [DocArrayHnswSearch](https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/docarray_hnsw) are supported. + +For all three components, it's easily possible to add new integrations based on the existing abstractions of the langchain library. In some cases (like the pinecone integration), it's necessary to use the underlying APIs directly to implement more features or improve performance. + +## Pinecone integration + +The pinecone integration is adding stream and primary key to the vector metadata which allows for deduped incremental and full refreshes. It's using the [official pinecone python client](https://github.com/pinecone-io/pinecone-python-client). + +You can use the `test_pinecone.py` file to check whether the pipeline works as expected. + +## DocArrayHnswSearch integration + +The DocArrayHnswSearch integration is storing the vector metadata in a local file in the local root (`/local` in the container, `/tmp/airbyte_local` on the host). It's not possible to dedupe records, so only full refresh syncs are supported. DocArrayHnswSearch uses hnswlib under the hood, but the integration is fully relying on the langchain abstraction. + +## Chroma integration + +The Chroma integration is storing the vector metadata in a local file in the local root (`/local` in the container, `/tmp/airbyte_local` on the host), similar to the DocArrayHnswSearch. This is called the "persistent client" mode in Chroma. The integration is mostly using langchains abstraction, but it can also dedupe records and reset streams independently. + +You can use the `test_local.py` file to check whether the pipeline works as expected. \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-langchain/build.gradle b/airbyte-integrations/connectors/destination-langchain/build.gradle new file mode 100644 index 000000000000..9ac8fa2191ac --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'destination_langchain' +} diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/__init__.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/__init__.py new file mode 100644 index 000000000000..3e63cd2eb391 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationLangchain + +__all__ = ["DestinationLangchain"] diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/batcher.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/batcher.py new file mode 100644 index 000000000000..ab3ad18c83d8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/batcher.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Callable, List + + +class Batcher: + def __init__(self, batch_size: int, flush_handler: Callable[[List[Any]], None]): + self.batch_size = batch_size + self.buffer = [] + self.flush_handler = flush_handler + + def add(self, item: Any): + self.buffer.append(item) + self._flush_if_necessary() + + def flush(self): + if len(self.buffer) == 0: + return + self.flush_handler(list(self.buffer)) + self.buffer.clear() + + def _flush_if_necessary(self): + if len(self.buffer) >= self.batch_size: + self.flush() diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py new file mode 100644 index 000000000000..a5b00b0cfeaf --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py @@ -0,0 +1,151 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import re +from typing import List, Literal, Optional, Union + +import dpath.util +from jsonschema import RefResolver +from pydantic import BaseModel, Field + + +class ProcessingConfigModel(BaseModel): + chunk_size: int = Field( + ..., + title="Chunk size", + maximum=8191, + description="Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + ) + chunk_overlap: int = Field( + title="Chunk overlap", + description="Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + default=0, + ) + text_fields: Optional[List[str]] = Field( + ..., + title="Text fields to embed", + description="List of fields in the record that should be used to calculate the embedding. All other fields are passed along as meta fields. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + always_show=True, + examples=["text", "user.name", "users.*.name"], + ) + + class Config: + schema_extra = {"group": "processing"} + + +class OpenAIEmbeddingConfigModel(BaseModel): + mode: Literal["openai"] = Field("openai", const=True) + openai_key: str = Field(..., title="OpenAI API key", airbyte_secret=True) + + class Config: + title = "OpenAI" + schema_extra = { + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + } + + +class FakeEmbeddingConfigModel(BaseModel): + mode: Literal["fake"] = Field("fake", const=True) + + class Config: + title = "Fake" + schema_extra = { + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." + } + + +class PineconeIndexingModel(BaseModel): + mode: Literal["pinecone"] = Field("pinecone", const=True) + pinecone_key: str = Field(..., title="Pinecone API key", airbyte_secret=True) + pinecone_environment: str = Field(..., title="Pinecone environment", description="Pinecone environment to use") + index: str = Field(..., title="Index", description="Pinecone index to use") + + class Config: + title = "Pinecone" + schema_extra = { + "description": "Pinecone is a popular vector store that can be used to store and retrieve embeddings. It is a managed service and can also be queried from outside of langchain." + } + + +class ChromaLocalIndexingModel(BaseModel): + mode: Literal["chroma_local"] = Field("chroma_local", const=True) + destination_path: str = Field( + ..., + title="Destination Path", + description="Path to the directory where chroma files will be written. The files will be placed inside that local mount.", + examples=["/local/my_chroma_db"], + ) + collection_name: str = Field( + title="Collection Name", + description="Name of the collection to use.", + default="langchain", + ) + + class Config: + title = "Chroma (local persistance)" + schema_extra = { + "description": "Chroma is a popular vector store that can be used to store and retrieve embeddings. It will build its index in memory and persist it to disk by the end of the sync." + } + + +class DocArrayHnswSearchIndexingModel(BaseModel): + mode: Literal["DocArrayHnswSearch"] = Field("DocArrayHnswSearch", const=True) + destination_path: str = Field( + ..., + title="Destination Path", + description="Path to the directory where hnswlib and meta data files will be written. The files will be placed inside that local mount. All files in the specified destination directory will be deleted on each run.", + examples=["/local/my_hnswlib_index"], + ) + + class Config: + title = "DocArrayHnswSearch" + schema_extra = { + "description": "DocArrayHnswSearch is a lightweight Document Index implementation provided by Docarray that runs fully locally and is best suited for small- to medium-sized datasets. It stores vectors on disk in hnswlib, and stores all other data in SQLite." + } + + +class ConfigModel(BaseModel): + processing: ProcessingConfigModel + embedding: Union[OpenAIEmbeddingConfigModel, FakeEmbeddingConfigModel] = Field( + ..., title="Embedding", description="Embedding configuration", discriminator="mode", group="embedding", type="object" + ) + indexing: Union[PineconeIndexingModel, DocArrayHnswSearchIndexingModel, ChromaLocalIndexingModel] = Field( + ..., title="Indexing", description="Indexing configuration", discriminator="mode", group="indexing", type="object" + ) + + class Config: + title = "Langchain Destination Config" + schema_extra = { + "groups": [ + {"id": "processing", "title": "Processing"}, + {"id": "embedding", "title": "Embedding"}, + {"id": "indexing", "title": "Indexing"}, + ] + } + + @staticmethod + def resolve_refs(schema: dict) -> dict: + # config schemas can't contain references, so inline them + json_schema_ref_resolver = RefResolver.from_schema(schema) + str_schema = json.dumps(schema) + for ref_block in re.findall(r'{"\$ref": "#\/definitions\/.+?(?="})"}', str_schema): + ref = json.loads(ref_block)["$ref"] + str_schema = str_schema.replace(ref_block, json.dumps(json_schema_ref_resolver.resolve(ref)[1])) + pyschema: dict = json.loads(str_schema) + del pyschema["definitions"] + return pyschema + + @staticmethod + def remove_discriminator(schema: dict) -> None: + """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references""" + dpath.util.delete(schema, "properties/*/discriminator") + + @classmethod + def schema(cls): + """we're overriding the schema classmethod to enable some post-processing""" + schema = super().schema() + schema = cls.resolve_refs(schema) + cls.remove_discriminator(schema) + return schema diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py new file mode 100644 index 000000000000..94e3ade7b8d4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py @@ -0,0 +1,88 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, List, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteMessage, + AirbyteRecordMessage, + ConfiguredAirbyteCatalog, + ConnectorSpecification, + Status, + Type, +) +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_langchain.batcher import Batcher +from destination_langchain.config import ConfigModel +from destination_langchain.document_processor import DocumentProcessor +from destination_langchain.embedder import Embedder, FakeEmbedder, OpenAIEmbedder +from destination_langchain.indexer import ChromaLocalIndexer, DocArrayHnswSearchIndexer, Indexer, PineconeIndexer +from langchain.document_loaders.base import Document + +BATCH_SIZE = 128 + +indexer_map = {"pinecone": PineconeIndexer, "DocArrayHnswSearch": DocArrayHnswSearchIndexer, "chroma_local": ChromaLocalIndexer} + +embedder_map = {"openai": OpenAIEmbedder, "fake": FakeEmbedder} + + +class DestinationLangchain(Destination): + indexer: Indexer + processor: DocumentProcessor + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = embedder_map[config.embedding.mode](config.embedding) + self.indexer = indexer_map[config.indexing.mode](config.indexing, self.embedder) + + def _process_batch(self, batch: List[AirbyteRecordMessage]): + documents: List[Document] = [] + ids_to_delete = [] + for record in batch: + record_documents, record_id_to_delete = self.processor.process(record) + documents.extend(record_documents) + if record_id_to_delete is not None: + ids_to_delete.append(record_id_to_delete) + self.indexer.index(documents, ids_to_delete) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + self.processor = DocumentProcessor(config_model.processing, configured_catalog, max_metadata_size=self.indexer.max_metadata_size) + batcher = Batcher(BATCH_SIZE, lambda batch: self._process_batch(batch)) + self.indexer.pre_sync(configured_catalog) + for message in input_messages: + if message.type == Type.STATE: + # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush + # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state + batcher.flush() + yield message + elif message.type == Type.RECORD: + batcher.add(message.record) + batcher.flush() + yield from self.indexer.post_sync() + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + self._init_indexer(ConfigModel.parse_obj(config)) + embedder_error = self.embedder.check() + indexer_error = self.indexer.check() + errors = [error for error in [embedder_error, indexer_error] if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/langchain", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py new file mode 100644 index 000000000000..460a7614bf37 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import List, Mapping, Optional, Tuple, Union + +import dpath.util +from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode +from destination_langchain.config import ProcessingConfigModel +from dpath.exceptions import PathNotFound +from langchain.document_loaders.base import Document +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.utils import stringify_dict + +METADATA_STREAM_FIELD = "_airbyte_stream" +METADATA_RECORD_ID_FIELD = "_record_id" + + +class DocumentProcessor: + streams: Mapping[str, ConfiguredAirbyteStream] + + def __init__(self, config: ProcessingConfigModel, catalog: ConfiguredAirbyteCatalog, max_metadata_size: Optional[int] = None): + self.streams = {self._stream_identifier(stream.stream): stream for stream in catalog.streams} + self.max_metadata_size = max_metadata_size + + self.splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=config.chunk_size, chunk_overlap=config.chunk_overlap + ) + self.text_fields = config.text_fields + self.logger = logging.getLogger("airbyte.document_processor") + + def _stream_identifier(self, stream: Union[AirbyteStream, AirbyteRecordMessage]) -> str: + if isinstance(stream, AirbyteStream): + return stream.name if stream.namespace is None else f"{stream.namespace}_{stream.name}" + else: + return stream.stream if stream.namespace is None else f"{stream.namespace}_{stream.stream}" + + def process(self, record: AirbyteRecordMessage) -> Tuple[List[Document], Optional[str]]: + """ + Generate documents from records. + :param records: List of AirbyteRecordMessages + :return: Tuple of (List of document chunks, record id to delete if a stream is in dedup mode to avoid stale documents in the vector store) + """ + doc = self._generate_document(record) + if doc is None: + self.logger.warning(f"Record {str(record.data)[:250]}... does not contain any text fields. Skipping.") + return [], None + chunks = self._split_document(doc) + id_to_delete = doc.metadata[METADATA_RECORD_ID_FIELD] if METADATA_RECORD_ID_FIELD in doc.metadata else None + return chunks, id_to_delete + + def _generate_document(self, record: AirbyteRecordMessage) -> Optional[Document]: + relevant_fields = self._extract_relevant_fields(record) + if len(relevant_fields) == 0: + return None + metadata = self._extract_metadata(record) + text = stringify_dict(relevant_fields) + return Document(page_content=text, metadata=metadata) + + def _extract_relevant_fields(self, record: AirbyteRecordMessage) -> dict: + relevant_fields = {} + if self.text_fields: + for field in self.text_fields: + values = dpath.util.values(record.data, field, separator=".") + if values and len(values) > 0: + relevant_fields[field] = values + else: + relevant_fields = record.data + return relevant_fields + + def _extract_metadata(self, record: AirbyteRecordMessage) -> dict: + metadata = record.data + if self.text_fields: + for field in self.text_fields: + try: + dpath.util.delete(metadata, field, separator=".") + except PathNotFound: + pass # if the field doesn't exist, do nothing + metadata = self._truncate_metadata(metadata) + stream_identifier = self._stream_identifier(record) + current_stream: ConfiguredAirbyteStream = self.streams[stream_identifier] + metadata[METADATA_STREAM_FIELD] = stream_identifier + # if the sync mode is deduping, use the primary key to upsert existing records instead of appending new ones + if current_stream.primary_key and current_stream.destination_sync_mode == DestinationSyncMode.append_dedup: + metadata[METADATA_RECORD_ID_FIELD] = self._extract_primary_key(record, current_stream) + return metadata + + def _extract_primary_key(self, record: AirbyteRecordMessage, stream: ConfiguredAirbyteStream) -> dict: + primary_key = [] + for key in stream.primary_key: + try: + primary_key.append(str(dpath.util.get(record.data, key))) + except KeyError: + primary_key.append("__not_found__") + return "_".join(primary_key) + + def _truncate_metadata(self, metadata: dict) -> dict: + """ + Normalize metadata to ensure it is within the size limit and doesn't contain complex objects. + """ + result = {} + current_size = 0 + + for key, value in metadata.items(): + if isinstance(value, (str, int, float, bool)): + # Calculate the size of the key and value + item_size = len(str(key)) + len(str(value)) + + # Check if adding the item exceeds the size limit + if self.max_metadata_size is None or current_size + item_size <= self.max_metadata_size: + result[key] = value + current_size += item_size + + return result + + def _split_document(self, doc: Document) -> List[Document]: + chunks = self.splitter.split_documents([doc]) + return chunks diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/embedder.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/embedder.py new file mode 100644 index 000000000000..55e0674e88a3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/embedder.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Optional + +from destination_langchain.config import FakeEmbeddingConfigModel, OpenAIEmbeddingConfigModel +from destination_langchain.utils import format_exception +from langchain.embeddings.base import Embeddings +from langchain.embeddings.fake import FakeEmbeddings +from langchain.embeddings.openai import OpenAIEmbeddings + + +class Embedder(ABC): + def __init__(self): + pass + + @abstractmethod + def check(self) -> Optional[str]: + pass + + @property + @abstractmethod + def langchain_embeddings(self) -> Embeddings: + pass + + @property + @abstractmethod + def embedding_dimensions(self) -> int: + pass + + +OPEN_AI_VECTOR_SIZE = 1536 + + +class OpenAIEmbedder(Embedder): + def __init__(self, config: OpenAIEmbeddingConfigModel): + super().__init__() + self.embeddings = OpenAIEmbeddings(openai_api_key=config.openai_key, chunk_size=8191) + + def check(self) -> Optional[str]: + try: + self.embeddings.embed_query("test") + except Exception as e: + return format_exception(e) + return None + + @property + def langchain_embeddings(self) -> Embeddings: + return self.embeddings + + @property + def embedding_dimensions(self) -> int: + # vector size produced by text-embedding-ada-002 model + return OPEN_AI_VECTOR_SIZE + + +class FakeEmbedder(Embedder): + def __init__(self, config: FakeEmbeddingConfigModel): + super().__init__() + self.embeddings = FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE) + + def check(self) -> Optional[str]: + try: + self.embeddings.embed_query("test") + except Exception as e: + return format_exception(e) + return None + + @property + def langchain_embeddings(self) -> Embeddings: + return self.embeddings + + @property + def embedding_dimensions(self) -> int: + # use same vector size as for OpenAI embeddings to keep it realistic + return OPEN_AI_VECTOR_SIZE diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py new file mode 100644 index 000000000000..6f8fbaeafb0c --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py @@ -0,0 +1,205 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import itertools +import os +import uuid +from abc import ABC, abstractmethod +from typing import Any, List, Optional + +import pinecone +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, DestinationSyncMode, Level, Type +from destination_langchain.config import ChromaLocalIndexingModel, DocArrayHnswSearchIndexingModel, PineconeIndexingModel +from destination_langchain.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from destination_langchain.embedder import Embedder +from destination_langchain.measure_time import measure_time +from destination_langchain.utils import format_exception +from langchain.document_loaders.base import Document +from langchain.vectorstores import Chroma +from langchain.vectorstores.docarray import DocArrayHnswSearch + + +class Indexer(ABC): + def __init__(self, config: Any, embedder: Embedder): + self.config = config + self.embedder = embedder + pass + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + pass + + def post_sync(self) -> List[AirbyteMessage]: + return [] + + @abstractmethod + def index(self, document_chunks: List[Document], delete_ids: List[str]): + pass + + @abstractmethod + def check(self) -> Optional[str]: + pass + + @property + def max_metadata_size(self) -> Optional[int]: + return None + + +def chunks(iterable, batch_size): + """A helper function to break an iterable into chunks of size batch_size.""" + it = iter(iterable) + chunk = tuple(itertools.islice(it, batch_size)) + while chunk: + yield chunk + chunk = tuple(itertools.islice(it, batch_size)) + + +# large enough to speed up processing, small enough to not hit pinecone request limits +PINECONE_BATCH_SIZE = 40 + + +class PineconeIndexer(Indexer): + config: PineconeIndexingModel + + def __init__(self, config: PineconeIndexingModel, embedder: Embedder): + super().__init__(config, embedder) + pinecone.init(api_key=config.pinecone_key, environment=config.pinecone_environment, threaded=True) + self.pinecone_index = pinecone.Index(config.index, pool_threads=10) + self.embed_fn = measure_time(self.embedder.langchain_embeddings.embed_documents) + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + index_description = pinecone.describe_index(self.config.index) + self._pod_type = index_description.pod_type + for stream in catalog.streams: + if stream.destination_sync_mode == DestinationSyncMode.overwrite: + self._delete_vectors({METADATA_STREAM_FIELD: stream.stream.name}) + + def post_sync(self): + return [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.WARN, message=self.embed_fn._get_stats()))] + + def _delete_vectors(self, filter): + if self._pod_type == "starter": + # Starter pod types have a maximum of 1000000 rows + top_k = 10000 + self._delete_by_metadata(filter, top_k) + else: + self.pinecone_index.delete(filter=filter) + + def _delete_by_metadata(self, filter, top_k): + zero_vector = [0.0] * self.embedder.embedding_dimensions + query_result = self.pinecone_index.query(vector=zero_vector, filter=filter, top_k=top_k) + vector_ids = [doc.id for doc in query_result.matches] + if len(vector_ids) > 0: + self.pinecone_index.delete(ids=vector_ids) + + def index(self, document_chunks, delete_ids): + if len(delete_ids) > 0: + self._delete_vectors({METADATA_RECORD_ID_FIELD: {"$in": delete_ids}}) + embedding_vectors = self.embed_fn([chunk.page_content for chunk in document_chunks]) + pinecone_docs = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + metadata = chunk.metadata + metadata["text"] = chunk.page_content + pinecone_docs.append((str(uuid.uuid4()), embedding_vectors[i], metadata)) + async_results = [ + self.pinecone_index.upsert(vectors=ids_vectors_chunk, async_req=True, show_progress=False) + for ids_vectors_chunk in chunks(pinecone_docs, batch_size=PINECONE_BATCH_SIZE) + ] + # Wait for and retrieve responses (this raises in case of error) + [async_result.get() for async_result in async_results] + + def check(self) -> Optional[str]: + try: + description = pinecone.describe_index(self.config.index) + actual_dimension = int(description.dimension) + if actual_dimension != self.embedder.embedding_dimensions: + return f"Your embedding configuration will produce vectors with dimension {self.embedder.embedding_dimensions:d}, but your index is configured with dimension {actual_dimension:d}. Make sure embedding and indexing configurations match." + except Exception as e: + return format_exception(e) + return None + + @property + def max_metadata_size(self) -> int: + # leave some space for the text field + return 40_960 - 10_000 + + +class DocArrayHnswSearchIndexer(Indexer): + config: DocArrayHnswSearchIndexingModel + + def __init__(self, config: DocArrayHnswSearchIndexingModel, embedder: Embedder): + super().__init__(config, embedder) + + def _init_vectorstore(self): + self.vectorstore = DocArrayHnswSearch.from_params( + embedding=self.embedder.langchain_embeddings, work_dir=self.config.destination_path, n_dim=self.embedder.embedding_dimensions + ) + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + for stream in catalog.streams: + if stream.destination_sync_mode != DestinationSyncMode.overwrite: + raise Exception( + f"DocArrayHnswSearchIndexer only supports overwrite mode, got {stream.destination_sync_mode} for stream {stream.stream.name}" + ) + for file in os.listdir(self.config.destination_path): + os.remove(os.path.join(self.config.destination_path, file)) + self._init_vectorstore() + + def post_sync(self): + return [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.WARN, message=self.index._get_stats()))] + + @measure_time + def index(self, document_chunks, delete_ids: List[str]): + # does not support deleting documents, always full refresh sync + self.vectorstore.add_documents(document_chunks) + + def check(self) -> Optional[str]: + try: + self._init_vectorstore() + except Exception as e: + return format_exception(e) + return None + + +class ChromaLocalIndexer(Indexer): + config: ChromaLocalIndexingModel + + def __init__(self, config: ChromaLocalIndexingModel, embedder: Embedder): + super().__init__(config, embedder) + + def _init_vectorstore(self): + self.vectorstore = Chroma( + collection_name=self.config.collection_name, + embedding_function=self.embedder.langchain_embeddings, + persist_directory=self.config.destination_path, + ) + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + self._init_vectorstore() + for stream in catalog.streams: + if stream.destination_sync_mode == DestinationSyncMode.overwrite: + self.vectorstore._collection.delete(where={METADATA_STREAM_FIELD: {"$eq": stream.stream.name}}) + + def index(self, document_chunks, delete_ids): + for delete_in in delete_ids: + self.vectorstore._collection.delete(where={METADATA_RECORD_ID_FIELD: {"$eq": delete_in}}) + for chunk in document_chunks: + self._normalize_metadata(chunk) + self.vectorstore.add_documents(document_chunks) + + def _normalize_metadata(self, document: Document): + for key, value in document.metadata.items(): + # check bool separately because isinstance(True, int) == True + if not isinstance(value, (str, float, int)) or isinstance(value, bool): + document.metadata[key] = str(value) + + def check(self) -> Optional[str]: + try: + self._init_vectorstore() + # try reading collections to make sure it works + self.vectorstore._client.list_collections() + except Exception as e: + return format_exception(e) + return None diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/measure_time.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/measure_time.py new file mode 100644 index 000000000000..da074de5801e --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/measure_time.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import time + + +def measure_time(func): + def wrapper(*args, **kwargs): + wrapper.count += 1 + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + execution_time = end_time - start_time + + wrapper.total_time += execution_time + wrapper.average_time = wrapper.total_time / wrapper.count + + return result + + wrapper.count = 0 + wrapper.total_time = 0 + wrapper.average_time = 0 + wrapper._get_stats = lambda: get_stats(wrapper) + + def get_stats(wrapper): + return f"Function '{func.__name__}' called {wrapper.count} time(s).\nAverage execution time: {wrapper.average_time:.6f} seconds.\nTotal execution time: {wrapper.total_time:.6f} seconds." + + return wrapper diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/utils.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/utils.py new file mode 100644 index 000000000000..05644e2e7709 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/utils.py @@ -0,0 +1,9 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import traceback + + +def format_exception(exception: Exception) -> None: + return str(exception) + "\n" + "".join(traceback.TracebackException.from_exception(exception).format()) diff --git a/airbyte-integrations/connectors/destination-langchain/examples/configured_catalog.json b/airbyte-integrations/connectors/destination-langchain/examples/configured_catalog.json new file mode 100644 index 000000000000..fab8a309fdf6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/examples/configured_catalog.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/destination-langchain/examples/messages.jsonl b/airbyte-integrations/connectors/destination-langchain/examples/messages.jsonl new file mode 100644 index 000000000000..80e15fb36948 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/examples/messages.jsonl @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "value1", "field2": "value2" }, "emitted_at": 1625383200000}} +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "value2", "field2": "value2" }, "emitted_at": 1625383200000}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-langchain/icon.svg b/airbyte-integrations/connectors/destination-langchain/icon.svg new file mode 100644 index 000000000000..6796e747d783 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/base_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/base_integration_test.py new file mode 100644 index 000000000000..37121df0925a --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/base_integration_test.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import shutil +import tempfile +import unittest +from typing import Any, Dict + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, + Type, +) + + +class BaseIntegrationTest(unittest.TestCase): + def _get_configured_catalog(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream( + name="mystream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + ), + primary_key=[["int_col"]], + sync_mode=SyncMode.incremental, + destination_sync_mode=destination_mode, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + def _state(self, data: Dict[str, Any]) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) + + def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) + ) + + +class LocalIntegrationTest(BaseIntegrationTest): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir) diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py new file mode 100644 index 000000000000..c5fa445add06 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging + +import chromadb +from airbyte_cdk.models import DestinationSyncMode, Status +from chromadb.api.types import QueryResult +from destination_langchain.destination import DestinationLangchain +from destination_langchain.embedder import OPEN_AI_VECTOR_SIZE +from integration_tests.base_integration_test import LocalIntegrationTest +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Chroma + + +class ChromaLocalIntegrationTest(LocalIntegrationTest): + def setUp(self): + super().setUp() + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + self.config["indexing"] = { + "destination_path": self.temp_dir, + "mode": "chroma_local", + } + self.chroma_client = chromadb.PersistentClient(path=self.temp_dir) + + def test_check_valid_config(self): + outcome = DestinationLangchain().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_write(self): + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationLangchain() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + assert self.chroma_client.get_collection("langchain").count() == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + chroma_result: QueryResult = self.chroma_client.get_collection("langchain").query( + query_embeddings=[0] * OPEN_AI_VECTOR_SIZE, n_results=10, where={"_record_id": "2"}, include=["documents"] + ) + assert len(chroma_result["documents"][0]) == 1 + assert chroma_result["documents"][0] == ["str_col: Cats are nice"] + + # test langchain integration + embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"]) + vector_store = Chroma(embedding_function=embeddings, persist_directory=self.temp_dir) + result = vector_store.similarity_search("feline animals", 1) + assert result[0].metadata["_record_id"] == "2" diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py new file mode 100644 index 000000000000..ba93129e02bc --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging + +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_langchain.destination import DestinationLangchain +from destination_langchain.embedder import OPEN_AI_VECTOR_SIZE +from integration_tests.base_integration_test import LocalIntegrationTest +from langchain.embeddings import FakeEmbeddings +from langchain.vectorstores import DocArrayHnswSearch + + +class DocArrayIntegrationTest(LocalIntegrationTest): + def setUp(self): + super().setUp() + self.config = { + "processing": {"text_fields": ["str_col"], "chunk_size": 1000}, + "embedding": {"mode": "fake"}, + "indexing": {"mode": "DocArrayHnswSearch", "destination_path": self.temp_dir}, + } + + def test_check_valid_config(self): + outcome = DestinationLangchain().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_write(self): + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are nice, number {i}", i) for i in range(5)] + + destination = DestinationLangchain() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + + vector_store = DocArrayHnswSearch.from_params(embedding=FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE), work_dir=self.temp_dir, n_dim=OPEN_AI_VECTOR_SIZE) + + result = vector_store.similarity_search("does not match anyway", 10) + assert len(result) == 5 diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py new file mode 100644 index 000000000000..55d6b01a98ee --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from time import sleep + +import pinecone +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_langchain.destination import DestinationLangchain +from destination_langchain.embedder import OPEN_AI_VECTOR_SIZE +from integration_tests.base_integration_test import BaseIntegrationTest +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Pinecone + + +class PineconeIntegrationTest(BaseIntegrationTest): + def _init_pinecone(self): + pinecone.init(api_key=self.config["indexing"]["pinecone_key"], environment=self.config["indexing"]["pinecone_environment"]) + self._index = pinecone.Index(self.config["indexing"]["index"]) + + def _clean_index(self): + self._init_pinecone() + zero_vector = [0.0] * OPEN_AI_VECTOR_SIZE + query_result = self._index.query(vector=zero_vector, top_k=10_000) + vector_ids = [doc.id for doc in query_result.matches] + if len(vector_ids) > 0: + self._index.delete(ids=vector_ids) + + def setUp(self): + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + + def tearDown(self): + self._clean_index() + + def test_check_valid_config(self): + outcome = DestinationLangchain().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + outcome = DestinationLangchain().check( + logging.getLogger("airbyte"), + { + "processing": {"text_fields": ["str_col"], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "mode": "pinecone", + "pinecone_key": "mykey", + "index": "testdata", + "pinecone_environment": "asia-southeast1-gcp-free", + }, + }, + ) + assert outcome.status == Status.FAILED + + def test_write(self): + self._init_pinecone() + is_starter_pod = pinecone.describe_index(self.config["indexing"]["index"]).pod_type == "starter" + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationLangchain() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + if is_starter_pod: + # Documents might not be available right away because Pinecone is handling them async + sleep(20) + assert self._index.describe_index_stats().total_vector_count == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + if is_starter_pod: + # Documents might not be available right away because Pinecone is handling them async + sleep(20) + result = self._index.query( + vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_record_id": "2"}, include_metadata=True + ) + assert len(result.matches) == 1 + assert result.matches[0].metadata["text"] == "str_col: Cats are nice" + + # test langchain integration + embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"]) + pinecone.init(api_key=self.config["indexing"]["pinecone_key"], environment=self.config["indexing"]["pinecone_environment"]) + vector_store = Pinecone(self._index, embeddings.embed_query, "text") + result = vector_store.similarity_search("feline animals", 1) + assert result[0].metadata["_record_id"] == "2" diff --git a/airbyte-integrations/connectors/destination-langchain/main.py b/airbyte-integrations/connectors/destination-langchain/main.py new file mode 100644 index 000000000000..717d9a760388 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_langchain import DestinationLangchain + +if __name__ == "__main__": + DestinationLangchain().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-langchain/metadata.yaml b/airbyte-integrations/connectors/destination-langchain/metadata.yaml new file mode 100644 index 000000000000..191f9d8c8f39 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/metadata.yaml @@ -0,0 +1,25 @@ +data: + registries: + cloud: + enabled: true + oss: + enabled: true + connectorSubtype: database + connectorType: destination + definitionId: cf98d52c-ba5a-4dfd-8ada-c1baebfa6e73 + dockerImageTag: 0.0.8 + dockerRepository: airbyte/destination-langchain + githubIssueLabel: destination-langchain + icon: langchain.svg + license: MIT + name: Vector Database (powered by LangChain) + releaseDate: 2023-07-15 + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/langchain + tags: + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-langchain/requirements.txt b/airbyte-integrations/connectors/destination-langchain/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-langchain/setup.py b/airbyte-integrations/connectors/destination-langchain/setup.py new file mode 100644 index 000000000000..80f25bd65f1e --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/setup.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", + "langchain", + "openai", + "requests", + "tiktoken", + "docarray[hnswlib]>=0.32.0", + "pinecone-client", + "typing-inspect==0.8.0", + "typing_extensions==4.5.0", + "pydantic==1.10.8", + "pandas==1.4.2", + "chromadb==0.4.3", +] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_langchain", + description="Destination implementation for Langchain.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-langchain/test_local.py b/airbyte-integrations/connectors/destination-langchain/test_local.py new file mode 100644 index 000000000000..87740999f518 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/test_local.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from langchain.chains import RetrievalQA +from langchain.embeddings import OpenAIEmbeddings +from langchain.llms import OpenAI +from langchain.vectorstores import Chroma + +# Run with OPENAI_API_KEY set in the environment + +embeddings = OpenAIEmbeddings() +vector_store = Chroma(embedding_function=embeddings, persist_directory="/tmp/airbyte_local/my_chroma_db") + +# print(vector_store.similarity_search("python", 1)) + +qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever()) + +print("Chat Langchain Demo") +print("Ask a question to begin:") +while True: + query = input("") + answer = qa.run(query) + print(answer) + print("\nWhat else can I help you with:") diff --git a/airbyte-integrations/connectors/destination-langchain/test_pinecone.py b/airbyte-integrations/connectors/destination-langchain/test_pinecone.py new file mode 100644 index 000000000000..41e87a8cf243 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/test_pinecone.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os + +import pinecone +from langchain.chains import RetrievalQA +from langchain.embeddings import OpenAIEmbeddings +from langchain.llms import OpenAI +from langchain.vectorstores import Pinecone + +# Run with OPENAI_API_KEY, PINECONE_KEY and PINECONE_ENV set in the environment + +embeddings = OpenAIEmbeddings() +pinecone.init(api_key=os.environ["PINECONE_KEY"], environment=os.environ["PINECONE_ENV"]) +index = pinecone.Index("testdata") +vector_store = Pinecone(index, embeddings.embed_query, "text") + + +# Playing with a Github issue search use case + +# prompt_template = """You are a question-answering bot operating on Github issues. Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. In the end, state the issue number you based your answer on. + +# {context} + +# Question: {question} +# Helpful Answer:""" +# prompt = PromptTemplate( +# template=prompt_template, input_variables=["context", "question"] +# ) +# document_prompt = PromptTemplate(input_variables=["page_content", "number"], template="{page_content}, issue number: {number}") +# qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever(), chain_type_kwargs={"prompt": prompt, "document_prompt": document_prompt}) + +qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever()) + +print("Chat Langchain Demo") +print("Ask a question to begin:") +while True: + query = input("") + answer = qa.run(query) + print(answer) + print("\nWhat else can I help you with:") diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/batcher_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/batcher_test.py new file mode 100644 index 000000000000..50538ea324b7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/batcher_test.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock + +from destination_langchain.batcher import Batcher + + +class BatcherTestCase(unittest.TestCase): + def test_add_single_item(self): + # Arrange + batch_size = 3 + flush_handler_mock = MagicMock() + batcher = Batcher(batch_size, flush_handler_mock) + + # Act + batcher.add(1) + + # Assert + self.assertFalse(flush_handler_mock.called) + + def test_add_flushes_batch(self): + # Arrange + batch_size = 3 + flush_handler_mock = MagicMock() + batcher = Batcher(batch_size, flush_handler_mock) + + # Act + batcher.add(1) + batcher.add(2) + batcher.add(3) + + # Assert + flush_handler_mock.assert_called_once_with([1, 2, 3]) + + def test_flush_empty_buffer(self): + # Arrange + batch_size = 3 + flush_handler_mock = MagicMock() + batcher = Batcher(batch_size, flush_handler_mock) + + # Act + batcher.flush() + + # Assert + self.assertFalse(flush_handler_mock.called) + + def test_flush_non_empty_buffer(self): + # Arrange + batch_size = 3 + flush_handler_mock = MagicMock() + batcher = Batcher(batch_size, flush_handler_mock) + batcher.add(1) + batcher.add(2) + + # Act + batcher.flush() + + # Assert + flush_handler_mock.assert_called_once_with([1, 2]) + self.assertEqual(len(batcher.buffer), 0) + + def test_flush_if_necessary_flushes_batch(self): + # Arrange + batch_size = 3 + flush_handler_mock = MagicMock() + batcher = Batcher(batch_size, flush_handler_mock) + batcher.add(1) + batcher.add(2) + batcher.add(3) + + # Act + batcher.add(4) + batcher.add(5) + + # Assert + flush_handler_mock.assert_called_once_with([1, 2, 3]) + self.assertEqual(len(batcher.buffer), 2) + + def test_flush_if_necessary_does_not_flush_incomplete_batch(self): + # Arrange + batch_size = 3 + flush_handler_mock = MagicMock() + batcher = Batcher(batch_size, flush_handler_mock) + batcher.add(1) + + # Act + batcher.add(2) + + # Assert + self.assertFalse(flush_handler_mock.called) + self.assertEqual(len(batcher.buffer), 2) diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py new file mode 100644 index 000000000000..01452e3c74ef --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, call + +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from destination_langchain.config import ChromaLocalIndexingModel +from destination_langchain.indexer import ChromaLocalIndexer +from langchain.document_loaders.base import Document + + +def create_chroma_local_indexer(): + config = ChromaLocalIndexingModel(mode="chroma_local", destination_path="/test", collection_name="myindex") + embedder = MagicMock() + indexer = ChromaLocalIndexer(config, embedder) + + indexer.vectorstore = MagicMock() + indexer.vectorstore._collection = MagicMock() + indexer.vectorstore._collection.delete = MagicMock() + indexer.embed_fn = MagicMock(return_value=[[1, 2, 3], [4, 5, 6]]) + indexer.vectorstore.add_documents = MagicMock() + return indexer + + +def test_chroma_local_index_upsert_and_delete(): + indexer = create_chroma_local_indexer() + docs = [ + Document(page_content="test", metadata={"_airbyte_stream": "abc"}), + Document(page_content="test2", metadata={"_airbyte_stream": "abc"}), + ] + indexer.index( + docs, + ["delete_id1", "delete_id2"], + ) + indexer.vectorstore._collection.delete.assert_has_calls( + [ + call(where={"_record_id": {"$eq": "delete_id1"}}), + call(where={"_record_id": {"$eq": "delete_id2"}}), + ] + ) + indexer.vectorstore.add_documents.assert_called_with(docs) + + +def test_chroma_local_normalize_metadata(): + indexer = create_chroma_local_indexer() + docs = [ + Document(page_content="test", metadata={"_airbyte_stream": "abc", "a_boolean_value": True}), + ] + indexer.index( + docs, + [], + ) + indexer.vectorstore.add_documents.assert_called_with([ + Document(page_content="test", metadata={"_airbyte_stream": "abc", "a_boolean_value": "True"}), + ]) + + +def test_chroma_local_index_empty_batch(): + indexer = create_chroma_local_indexer() + indexer.index( + [], + [], + ) + indexer.vectorstore._collection.delete.assert_not_called() + indexer.vectorstore.add_documents.assert_called_with([]) + + +def test_chroma_local_pre_sync(): + indexer = create_chroma_local_indexer() + indexer._init_vectorstore = MagicMock() + indexer.pre_sync( + ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + ] + } + ) + ) + indexer._init_vectorstore.assert_called() + indexer.vectorstore._collection.delete.assert_called_with(where={"_airbyte_stream": {"$eq": "example_stream2"}}) diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py new file mode 100644 index 000000000000..2b27d4d25ed6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, patch + +from airbyte_cdk.models.airbyte_protocol import ( + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + Level, + Type, +) +from destination_langchain.config import ConfigModel +from destination_langchain.destination import BATCH_SIZE, DestinationLangchain, embedder_map, indexer_map + + +def _generate_record_message(index: int): + return AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="example_stream", emitted_at=1234, data={"column_name": f"value {index}", "id": index})) + + +@patch.dict(embedder_map, {"openai": MagicMock()}) +@patch.dict(indexer_map, {"pinecone": MagicMock()}) +def test_write(): + """ + Basic test for the write method, batcher and document processor. + """ + config = { + "processing": {"text_fields": ["column_name"], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "mode": "pinecone", + "pinecone_key": "mykey", + "index": "myindex", + "pinecone_environment": "myenv", + }, + } + config_model = ConfigModel.parse_obj(config) + + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + } + ] + } + ) + # messages are flushed after 32 records or after a state message, so this will trigger two batches to be processed + input_messages = [_generate_record_message(i) for i in range(BATCH_SIZE + 5)] + state_message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage()) + input_messages.append(state_message) + # messages are also flushed once the input messages are exhausted, so this will trigger another batch + input_messages.extend([_generate_record_message(i) for i in range(5)]) + + mock_embedder = MagicMock() + embedder_map["openai"].return_value = mock_embedder + + mock_indexer = MagicMock() + indexer_map["pinecone"].return_value = mock_indexer + mock_indexer.max_metadata_size = 1000 + post_sync_log_message = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="post sync")) + mock_indexer.post_sync.return_value = [post_sync_log_message] + + # Create the DestinationLangchain instance + destination = DestinationLangchain() + + output_messages = destination.write(config, configured_catalog, input_messages) + output_message = next(output_messages) + # assert state message is + assert output_message == state_message + + embedder_map["openai"].assert_called_with(config_model.embedding) + indexer_map["pinecone"].assert_called_with(config_model.indexing, mock_embedder) + mock_indexer.pre_sync.assert_called_with(configured_catalog) + + # 1 batches due to max batch size reached and 1 batch due to state message + assert mock_indexer.index.call_count == 2 + + output_message = next(output_messages) + assert output_message == post_sync_log_message + + try: + next(output_messages) + assert False, "Expected end of message stream" + except StopIteration: + pass + + # 1 batch due to end of message stream + assert mock_indexer.index.call_count == 3 + + mock_indexer.post_sync.assert_called() diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py new file mode 100644 index 000000000000..d4486811d2dc --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, patch + +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from destination_langchain.config import DocArrayHnswSearchIndexingModel +from destination_langchain.indexer import DocArrayHnswSearchIndexer +from langchain.document_loaders.base import Document + + +class DocArrayIndexerTest(unittest.TestCase): + def setUp(self): + self.config = DocArrayHnswSearchIndexingModel(mode="DocArrayHnswSearch", destination_path="/tmp/made_up") + self.embedder = MagicMock() + self.embedder.embedding_dimensions = 3 + self.indexer = DocArrayHnswSearchIndexer(self.config, self.embedder) + self.indexer.vectorstore = MagicMock() + + def test_docarray_index(self): + docs = [ + Document(page_content="test", metadata={"_airbyte_stream": "abc"}), + Document(page_content="test2", metadata={"_airbyte_stream": "abc"}), + ] + + self.indexer.index( + docs, + ["delete_id1", "delete_id2"], + ) + # deleted documents are ignored + self.indexer.vectorstore.add_documents.assert_called_with(docs) + + def test_docarray_pre_sync_fail(self): + try: + self.indexer.pre_sync(ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + ] + } + )) + assert False, "Expected exception" + except Exception as e: + assert str(e) == "DocArrayHnswSearchIndexer only supports overwrite mode, got DestinationSyncMode.append_dedup for stream example_stream" + + @patch('os.listdir') + @patch('os.remove') + def test_docarray_pre_sync_succeed(self, remove_mock, listdir_mock): + listdir_mock.return_value = ["file1", "file2"] + self.indexer._init_vectorstore = MagicMock() + self.indexer.pre_sync(ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + ] + } + )) + assert remove_mock.call_count == 2 + assert self.indexer._init_vectorstore.call_count == 1 diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py new file mode 100644 index 000000000000..b8076debe157 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py @@ -0,0 +1,244 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode, SyncMode +from destination_langchain.config import ProcessingConfigModel +from destination_langchain.document_processor import DocumentProcessor + + +def initialize_processor(): + config = ProcessingConfigModel(chunk_size=48, chunk_overlap=0, text_fields=None) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name="stream1", json_schema={}, namespace="namespace1", supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + primary_key=[["id"]], + ), + ConfiguredAirbyteStream( + stream=AirbyteStream(name="stream2", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + return DocumentProcessor(config=config, catalog=catalog) + + +def test_process_single_chunk_without_metadata(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "text": "This is the text", + }, + emitted_at=1234, + ) + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 1 + # natural id is only set for dedup mode + assert "_record_id" not in chunks[0].metadata + assert chunks[0].metadata["_airbyte_stream"] == "namespace1_stream1" + assert chunks[0].page_content == "id: 1\ntext: This is the text" + assert id_to_delete is None + + +def test_process_single_chunk_without_namespace(): + config = ProcessingConfigModel(chunk_size=48, chunk_overlap=0, text_fields=None) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + processor = DocumentProcessor(config=config, catalog=catalog) + + record = AirbyteRecordMessage( + stream="stream1", + data={ + "id": 1, + "text": "This is the text", + }, + emitted_at=1234, + ) + + chunks, _ = processor.process(record) + assert chunks[0].metadata["_airbyte_stream"] == "stream1" + + +def test_complex_text_fields(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "nested": { + "texts": [ + {"text": "This is the text"}, + {"text": "And another"}, + ] + }, + "non_text": "a", + "non_text_2": 1, + "text": "This is the regular text", + "other_nested": { + "non_text": { + "a": "xyz", + "b": "abc" + } + } + }, + emitted_at=1234, + ) + + processor.text_fields = ["nested.texts.*.text", "text", "other_nested.non_text", "non.*.existing"] + + chunks, _ = processor.process(record) + + assert len(chunks) == 1 + assert chunks[0].page_content == """nested.texts.*.text: This is the text +And another +text: This is the regular text +other_nested.non_text: \na: xyz +b: abc""" + assert chunks[0].metadata == { + "id": 1, + "non_text": "a", + "non_text_2": 1, + "_airbyte_stream": "namespace1_stream1" + } + + +def test_non_text_fields(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "text": "This is the regular text", + }, + emitted_at=1234, + ) + + processor.text_fields = ["another_field"] + processor.logger = MagicMock() + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 0 + assert id_to_delete is None + assert processor.logger.warning.called + + +def test_metadata_normalization(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "a_complex_field": {"a_nested_field": "a_nested_value"}, + "too_big": "a" * 1000, + "small": "a", + "text": "This is the text", + }, + emitted_at=1234, + ) + + processor.text_fields = ["text"] + processor.max_metadata_size = 100 + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 1 + assert chunks[0].page_content == "text: This is the text" + assert id_to_delete is None + + for chunk in chunks: + assert len(chunk.metadata) == 3 + assert "a_complex_field" not in chunk.metadata + assert "too_big" not in chunk.metadata + assert "small" in chunk.metadata + + +def test_process_multiple_chunks_with_relevant_fields(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "name": "John Doe", + "text": "This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks", + "age": 25, + }, + emitted_at=1234, + ) + + processor.text_fields = ["text"] + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 2 + + for chunk in chunks: + assert chunk.metadata["age"] == 25 + assert id_to_delete is None + + +@pytest.mark.parametrize( + "primary_key_value, stringified_primary_key, primary_key", + [ + ({"id": 99}, "99", [["id"]]), + ({"id": 99, "name": "John Doe"}, "99_John Doe", [["id"], ["name"]]), + ({"id": 99, "name": "John Doe", "age": 25}, "99_John Doe_25", [["id"], ["name"], ["age"]]), + ({"nested": {"id": "abc"}, "name": "John Doe"}, "abc_John Doe", [["nested", "id"], ["name"]]), + ({"nested": {"id": "abc"}}, "abc___not_found__", [["nested", "id"], ["name"]]), + ] +) +def test_process_multiple_chunks_with_dedupe_mode(primary_key_value: Mapping[str, Any], stringified_primary_key: str, primary_key: List[List[str]]): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "text": "This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks", + "age": 25, + **primary_key_value + }, + emitted_at=1234, + ) + + processor.text_fields = ["text"] + + processor.streams["namespace1_stream1"].destination_sync_mode = DestinationSyncMode.append_dedup + processor.streams["namespace1_stream1"].primary_key = primary_key + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) > 1 + for chunk in chunks: + assert chunk.metadata["_record_id"] == stringified_primary_key + assert id_to_delete == stringified_primary_key diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py new file mode 100644 index 000000000000..b7e430277dc2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import ANY, MagicMock, patch + +import pytest +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from destination_langchain.config import PineconeIndexingModel +from destination_langchain.indexer import PineconeIndexer +from langchain.document_loaders.base import Document +from pinecone import IndexDescription + + +def create_pinecone_indexer(): + config = PineconeIndexingModel(mode="pinecone", pinecone_environment="myenv", pinecone_key="mykey", index="myindex") + embedder = MagicMock() + embedder.embedding_dimensions = 3 + indexer = PineconeIndexer(config, embedder) + + indexer.pinecone_index.delete = MagicMock() + indexer.embed_fn = MagicMock(return_value=[[1, 2, 3], [4, 5, 6]]) + indexer.pinecone_index.upsert = MagicMock() + indexer.pinecone_index.query = MagicMock() + return indexer + + +def create_index_description(dimensions=3, pod_type="p1"): + return IndexDescription( + name="", + metric="", + replicas=1, + dimension=dimensions, + shards=1, + pods=1, + pod_type=pod_type, + status=None, + metadata_config=None, + source_collection=None, + ) + + +@pytest.fixture(scope="module", autouse=True) +def mock_describe_index(): + with patch('pinecone.describe_index') as mock: + mock.return_value = create_index_description() + yield mock + + +def test_pinecone_index_upsert_and_delete(mock_describe_index): + indexer = create_pinecone_indexer() + indexer._pod_type = "p1" + indexer.index( + [ + Document(page_content="test", metadata={"_airbyte_stream": "abc"}), + Document(page_content="test2", metadata={"_airbyte_stream": "abc"}), + ], + ["delete_id1", "delete_id2"], + ) + indexer.pinecone_index.delete.assert_called_with(filter={"_record_id": {"$in": ["delete_id1", "delete_id2"]}}) + indexer.pinecone_index.upsert.assert_called_with( + vectors=( + (ANY, [1, 2, 3], {"_airbyte_stream": "abc", "text": "test"}), + (ANY, [4, 5, 6], {"_airbyte_stream": "abc", "text": "test2"}), + ), + async_req=True, + show_progress=False, + ) + + +def test_pinecone_index_upsert_and_delete_starter(mock_describe_index): + indexer = create_pinecone_indexer() + indexer._pod_type = "starter" + indexer.pinecone_index.query.return_value = MagicMock(matches=[MagicMock(id="doc_id1"), MagicMock(id="doc_id2")]) + indexer.index( + [ + Document(page_content="test", metadata={"_airbyte_stream": "abc"}), + Document(page_content="test2", metadata={"_airbyte_stream": "abc"}), + ], + ["delete_id1", "delete_id2"], + ) + indexer.pinecone_index.query.assert_called_with(vector=[0,0,0],filter={"_record_id": {"$in": ["delete_id1", "delete_id2"]}}, top_k=10_000) + indexer.pinecone_index.delete.assert_called_with(ids=["doc_id1", "doc_id2"]) + indexer.pinecone_index.upsert.assert_called_with( + vectors=( + (ANY, [1, 2, 3], {"_airbyte_stream": "abc", "text": "test"}), + (ANY, [4, 5, 6], {"_airbyte_stream": "abc", "text": "test2"}), + ), + async_req=True, + show_progress=False, + ) + + +def test_pinecone_index_empty_batch(): + indexer = create_pinecone_indexer() + indexer.index( + [], + [], + ) + indexer.pinecone_index.delete.assert_not_called() + indexer.pinecone_index.upsert.assert_not_called() + + +def test_pinecone_index_upsert_batching(): + indexer = create_pinecone_indexer() + indexer.embed_fn = MagicMock(return_value=[[i, i, i] for i in range(50)]) + indexer.index( + [Document(page_content=f"test {i}", metadata={"_airbyte_stream": "abc"}) for i in range(50)], + [], + ) + assert indexer.pinecone_index.upsert.call_count == 2 + for i in range(40): + assert indexer.pinecone_index.upsert.call_args_list[0].kwargs["vectors"][i] == ( + ANY, + [i, i, i], + {"_airbyte_stream": "abc", "text": f"test {i}"}, + ) + for i in range(40, 50): + assert indexer.pinecone_index.upsert.call_args_list[1].kwargs["vectors"][i - 40] == ( + ANY, + [i, i, i], + {"_airbyte_stream": "abc", "text": f"test {i}"}, + ) + + +def generate_catalog(): + return ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + ] + } + ) + + +def test_pinecone_pre_sync(mock_describe_index): + indexer = create_pinecone_indexer() + indexer.pre_sync(generate_catalog()) + indexer.pinecone_index.delete.assert_called_with(filter={"_airbyte_stream": "example_stream2"}) + + +def test_pinecone_pre_sync_starter(mock_describe_index): + mock_describe_index.return_value = create_index_description(pod_type="starter") + indexer = create_pinecone_indexer() + indexer.pinecone_index.query.return_value = MagicMock(matches=[MagicMock(id="doc_id1"), MagicMock(id="doc_id2")]) + indexer.pre_sync(generate_catalog()) + indexer.pinecone_index.query.assert_called_with(vector=[0,0,0],filter={"_airbyte_stream": "example_stream2"}, top_k=10_000) + indexer.pinecone_index.delete.assert_called_with(ids=["doc_id1", "doc_id2"]) + + +@pytest.mark.parametrize( + "describe_throws,reported_dimensions,check_succeeds", + [ + (False, 3, True), + (False, 4, False), + (True, 3, False), + (True, 4, False), + ], +) +@patch("pinecone.describe_index") +def test_pinecone_check(describe_mock, describe_throws, reported_dimensions, check_succeeds): + indexer = create_pinecone_indexer() + indexer.embedder.embedding_dimensions = 3 + if describe_throws: + describe_mock.side_effect = Exception("describe failed") + describe_mock.return_value = create_index_description(dimensions=reported_dimensions) + result = indexer.check() + if check_succeeds: + assert result is None + else: + assert result is not None diff --git a/airbyte-integrations/connectors/destination-local-json/metadata.yaml b/airbyte-integrations/connectors/destination-local-json/metadata.yaml index 6bb05961faa8..2f2b89c7d22e 100644 --- a/airbyte-integrations/connectors/destination-local-json/metadata.yaml +++ b/airbyte-integrations/connectors/destination-local-json/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/local-json tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java index 65a814c140f1..77907527225b 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java @@ -16,6 +16,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -91,7 +92,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { // no op } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/metadata.yaml b/airbyte-integrations/connectors/destination-mariadb-columnstore/metadata.yaml index 4b3b1d9c4f13..dbb6081f1f5f 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/metadata.yaml @@ -10,11 +10,15 @@ data: name: MariaDB ColumnStore registries: cloud: - enabled: true + enabled: false # hide MariaDB Destination https://github.com/airbytehq/airbyte-cloud/issues/2611 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/mariadb-columnstore tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java index 8727a6c27c83..a75e4969e89b 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.testcontainers.containers.MariaDBContainer; @@ -116,7 +117,7 @@ private static JdbcDatabase getDatabase(final JsonNode config) { } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { final DockerImageName mcsImage = DockerImageName.parse("fengdi/columnstore:1.5.2").asCompatibleSubstituteFor("mariadb"); db = new MariaDBContainer(mcsImage); db.start(); diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java index 0ee45d0c3e07..57b3b64db323 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -129,7 +130,7 @@ protected List resolveIdentifier(final String identifier) { } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { bastion.initAndStartBastion(network); startAndInitJdbcContainer(); } diff --git a/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml b/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml index 62087cf3d688..7826092b697b 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml +++ b/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml @@ -10,11 +10,15 @@ data: name: MeiliSearch registries: cloud: - enabled: true + enabled: false # hide MeiliSearch Destination https://github.com/airbytehq/airbyte/issues/16313 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/meilisearch tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile index 18f5f0c16b37..a20abf6c4715 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/metadata.yaml index a051a72df044..b81fdc490ef8 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 8b746512-8c2e-6ac1-4adc-b59faafd473c - dockerImageTag: 0.1.9 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mongodb-strict-encrypt githubIssueLabel: destination-mongodb icon: mongodb.svg - license: MIT + license: ELv2 name: MongoDB releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/mongodb diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java index 882442b2ad46..72cc7a8f830b 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java @@ -23,6 +23,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import org.bson.Document; import org.junit.jupiter.api.BeforeAll; @@ -127,7 +128,7 @@ void testCheck() throws Exception { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { final var credentials = String.format("%s:%s@", config.get(AUTH_TYPE).get(JdbcUtils.USERNAME_KEY).asText(), config.get(AUTH_TYPE).get(JdbcUtils.PASSWORD_KEY).asText()); final String connectionString = String.format("mongodb+srv://%s%s/%s?retryWrites=true&w=majority&tls=true", diff --git a/airbyte-integrations/connectors/destination-mongodb/Dockerfile b/airbyte-integrations/connectors/destination-mongodb/Dockerfile index 389bb3d934b3..94afbd1579ad 100644 --- a/airbyte-integrations/connectors/destination-mongodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-mongodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-mongodb diff --git a/airbyte-integrations/connectors/destination-mongodb/metadata.yaml b/airbyte-integrations/connectors/destination-mongodb/metadata.yaml index b26b928380ac..68c8852b7538 100644 --- a/airbyte-integrations/connectors/destination-mongodb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mongodb/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 8b746512-8c2e-6ac1-4adc-b59faafd473c - dockerImageTag: 0.1.9 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mongodb githubIssueLabel: destination-mongodb icon: mongodb.svg - license: MIT + license: ELv2 name: MongoDB registries: cloud: @@ -18,4 +18,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/mongodb tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index 80f0c4bc5bb0..a1e0798ea6f5 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -21,6 +21,7 @@ import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import org.bson.Document; import org.junit.jupiter.api.Test; @@ -180,7 +181,7 @@ public void testCheckIncorrectPort() { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { container = new MongoDBContainer(DOCKER_IMAGE_NAME); container.start(); } diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java index dbbcb2592646..84b2b992d71e 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java @@ -16,6 +16,7 @@ import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.util.HostPortResolver; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import org.bson.Document; import org.testcontainers.containers.MongoDBContainer; @@ -31,7 +32,7 @@ public abstract class SshMongoDbDestinationAcceptanceTest extends MongodbDestina public abstract SshTunnel.TunnelMethod getTunnelMethod(); @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { container = new MongoDBContainer(DOCKER_IMAGE_NAME) .withNetwork(network) .withExposedPorts(DEFAULT_PORT); diff --git a/airbyte-integrations/connectors/destination-mqtt/metadata.yaml b/airbyte-integrations/connectors/destination-mqtt/metadata.yaml index 8dd1d512f663..85d5e49baa42 100644 --- a/airbyte-integrations/connectors/destination-mqtt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mqtt/metadata.yaml @@ -10,11 +10,15 @@ data: name: MQTT registries: cloud: - enabled: false + enabled: false # hide MQTT Destination https://github.com/airbytehq/airbyte-cloud/issues/2613 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/mqtt tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java index 5d6b8ae63257..67cf210e381e 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; @@ -133,13 +134,13 @@ private String getIpAddress() throws UnknownHostException { .map(InetAddress::getHostAddress) .filter(InetAddresses::isUriInetAddress) .findFirst().orElse(InetAddress.getLocalHost().getHostAddress()); - } catch (SocketException e) { + } catch (final SocketException e) { return InetAddress.getLocalHost().getHostAddress(); } } @Override - protected void setup(final TestDestinationEnv testEnv) throws MqttException { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws MqttException { recordsPerTopic.clear(); client = new MqttClient("tcp://" + extension.getHost() + ":" + extension.getMqttPort(), UUID.randomUUID().toString(), new MemoryPersistence()); @@ -149,7 +150,7 @@ protected void setup(final TestDestinationEnv testEnv) throws MqttException { client.connect(options); client.subscribe(TOPIC_PREFIX + "#", (topic, msg) -> { - List records = recordsPerTopic.getOrDefault(topic, new ArrayList<>()); + final List records = recordsPerTopic.getOrDefault(topic, new ArrayList<>()); records.add(READER.readTree(msg.getPayload()).get(MqttDestination.COLUMN_NAME_DATA)); recordsPerTopic.put(topic, records); }); diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile index 1140b9766d69..d2d36689b8d1 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile @@ -8,9 +8,29 @@ # Please reach out to the Connectors Operations team if you have any question. FROM airbyte/integration-base-java:dev AS build +RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ + alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ + python -m ensurepip --upgrade && \ + pip3 install dbt-sqlserver==1.0.0 + +# Luckily, none of normalization's files conflict with destination-mssql's files :) +# We don't enforce that in any way, but hopefully we're only living in this state for a short time. +COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte +# Install python dependencies +WORKDIR /airbyte/base_python_structs +RUN pip3 install . +WORKDIR /airbyte/normalization_code +RUN pip3 install . +WORKDIR /airbyte/normalization_code/dbt-template/ +# Download external dbt dependencies +# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 +RUN pip3 install "urllib3<2" +RUN dbt deps + WORKDIR /airbyte ENV APPLICATION destination-mssql-strict-encrypt +ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar @@ -24,5 +44,8 @@ ENV APPLICATION destination-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.23 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt + +ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" +ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle index 6c18e69206a1..45378da4cf27 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle @@ -29,3 +29,7 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } + +tasks.named("airbyteDocker") { + dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql +} diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml index 2f77aeb9de4b..073064b4272c 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: database connectorType: destination definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf - dockerImageTag: 0.1.23 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mssql-strict-encrypt githubIssueLabel: destination-mssql icon: mssql.svg - license: MIT + license: ELv2 name: MS SQL Server normalizationConfig: normalizationIntegrationType: mssql diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java index 45a579872b83..2dff33999955 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java @@ -24,6 +24,7 @@ import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -146,7 +147,7 @@ private static Database getDatabase(final DSLContext dslContext) { } @Override - protected void setup(final TestDestinationEnv testEnv) throws SQLException { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws SQLException { final JsonNode configWithoutDbName = getConfig(db); final String dbName = Strings.addRandomSuffix("db", "_", 10); dslContext = getDslContext(configWithoutDbName); diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile index 8400c99e6690..d51950f7c0d5 100644 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql/Dockerfile @@ -8,9 +8,32 @@ # Please reach out to the Connectors Operations team if you have any question. FROM airbyte/integration-base-java:dev AS build +RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ + alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ + python -m ensurepip --upgrade && \ + # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 + pip3 install wheel && \ + pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ + pip3 install dbt-sqlserver==1.0.0 + +# Luckily, none of normalization's files conflict with destination-mssql's files :) +# We don't enforce that in any way, but hopefully we're only living in this state for a short time. +COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte +# Install python dependencies +WORKDIR /airbyte/base_python_structs +RUN pip3 install . +WORKDIR /airbyte/normalization_code +RUN pip3 install . +WORKDIR /airbyte/normalization_code/dbt-template/ +# Download external dbt dependencies +# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 +RUN pip3 install "urllib3<2" +RUN dbt deps + WORKDIR /airbyte ENV APPLICATION destination-mssql +ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar @@ -24,5 +47,8 @@ ENV APPLICATION destination-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.23 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-mssql + +ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" +ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-mssql/build.gradle b/airbyte-integrations/connectors/destination-mssql/build.gradle index fa8d153e1b62..300c78a44c21 100644 --- a/airbyte-integrations/connectors/destination-mssql/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql/build.gradle @@ -27,3 +27,7 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } + +tasks.named("airbyteDocker") { + dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql +} diff --git a/airbyte-integrations/connectors/destination-mssql/metadata.yaml b/airbyte-integrations/connectors/destination-mssql/metadata.yaml index 5508c142332e..53ce25ce47bd 100644 --- a/airbyte-integrations/connectors/destination-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mssql/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: d4353156-9217-4cad-8dd7-c108fd4f74cf - dockerImageTag: 0.1.23 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mssql githubIssueLabel: destination-mssql icon: mssql.svg - license: MIT + license: ELv2 name: MS SQL Server normalizationConfig: normalizationIntegrationType: mssql @@ -23,4 +23,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java index 26edb690eb00..584c6d8bacad 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java @@ -20,6 +20,7 @@ import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -130,7 +131,7 @@ private static Database getDatabase(final DSLContext dslContext) { // 1. exec into mssql container (not the test container container) // 2. /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "A_Str0ng_Required_Password" @Override - protected void setup(final TestDestinationEnv testEnv) throws SQLException { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws SQLException { final JsonNode configWithoutDbName = getConfig(db); final String dbName = Strings.addRandomSuffix("db", "_", 10); dslContext = getDslContext(configWithoutDbName); diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java index 589f83ede858..eaeb3ffc16f2 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java @@ -20,6 +20,7 @@ import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -141,7 +142,7 @@ private static Database getDatabase(final DSLContext dslContext) { // 1. exec into mssql container (not the test container container) // 2. /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "A_Str0ng_Required_Password" @Override - protected void setup(final TestDestinationEnv testEnv) throws SQLException { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws SQLException { configWithoutDbName = getConfig(db); final String dbName = Strings.addRandomSuffix("db", "_", 10); dslContext = getDslContext(configWithoutDbName); diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java index d6bc3625aab8..1cc0ff6a8075 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; @@ -114,7 +115,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { startTestContainers(); SshTunnel.sshWrap( @@ -126,7 +127,7 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception { ctx.fetch(String.format("CREATE DATABASE %s;", database)); ctx.fetch(String.format("USE %s;", database)); ctx.fetch(String.format("CREATE SCHEMA %s;", schemaName)); - + TEST_SCHEMAS.add(schemaName); return null; }); }); diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile index 615078ce8558..3488862613b2 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.21 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml index c90171695938..d838a707043b 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: database connectorType: destination definitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 - dockerImageTag: 0.1.21 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mysql-strict-encrypt githubIssueLabel: destination-mysql icon: mysql.svg - license: MIT + license: ELv2 name: MySQL normalizationConfig: normalizationIntegrationType: mysql diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java index c409004b77ef..77f88e227538 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java @@ -27,6 +27,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.sql.SQLException; import java.time.Instant; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -140,7 +141,7 @@ protected List retrieveNormalizedRecords(final TestDestinationEnv test } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { db = new MySQLContainer<>("mysql:8.0"); db.start(); setLocalInFileToTrue(); diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index b8adb4467da0..8152df3ad141 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.20 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-mysql/metadata.yaml b/airbyte-integrations/connectors/destination-mysql/metadata.yaml index ac62824acaca..ac3c702c6c58 100644 --- a/airbyte-integrations/connectors/destination-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mysql/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 - dockerImageTag: 0.1.20 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mysql githubIssueLabel: destination-mysql icon: mysql.svg - license: MIT + license: ELv2 name: MySQL normalizationConfig: normalizationIntegrationType: mysql @@ -23,4 +23,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index 583d31b56fae..9706f188dff8 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -30,6 +30,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.sql.SQLException; import java.time.Instant; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -148,7 +149,7 @@ protected List retrieveNormalizedRecords(final TestDestinationEnv test } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new MySQLContainer<>("mysql:8.0"); db.start(); setLocalInFileToTrue(); diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java index 8df6ef3555a2..bceb783f701e 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java @@ -19,6 +19,7 @@ import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; @@ -135,7 +136,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase(); final var config = getConfig(); SshTunnel.sshWrap( diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java index ce8e060c9676..0d7636503e16 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java @@ -17,6 +17,7 @@ import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -84,7 +85,7 @@ public void testCustomDbtTransformations() { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new MySQLContainer<>("mysql:8.0"); db.start(); diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile index 178d0dc3af2f..9ce70339438b 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-oracle-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml index 96a99d10260e..bd323c6fdbc5 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 3986776d-2319-4de9-8af8-db14c0996e72 - dockerImageTag: 0.1.19 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-oracle-strict-encrypt githubIssueLabel: destination-oracle icon: oracle.svg - license: MIT + license: ELv2 name: Oracle normalizationConfig: normalizationIntegrationType: oracle diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index f367195fca07..68b1f7411622 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -25,6 +25,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import javax.sql.DataSource; @@ -141,7 +142,7 @@ private static DSLContext getDslContext(final JsonNode config) { } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { final String dbName = Strings.addRandomSuffix("db", "_", 10); db = new OracleContainer() .withUsername("test") diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 5499f231424a..04cd94af0dc0 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-oracle COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-oracle/metadata.yaml b/airbyte-integrations/connectors/destination-oracle/metadata.yaml index 47d33e9843f2..f6f1acf44c37 100644 --- a/airbyte-integrations/connectors/destination-oracle/metadata.yaml +++ b/airbyte-integrations/connectors/destination-oracle/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 3986776d-2319-4de9-8af8-db14c0996e72 - dockerImageTag: 0.1.19 + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-oracle githubIssueLabel: destination-oracle icon: oracle.svg - license: MIT + license: ELv2 name: Oracle normalizationConfig: normalizationIntegrationType: oracle @@ -23,4 +23,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index c486c8a6d581..2e1e10b1bab7 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -20,6 +20,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -131,7 +132,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { startTestContainers(); SshTunnel.sshWrap( getConfig(), diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index e1f1adc9fe15..f492d9850437 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -27,6 +27,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import javax.sql.DataSource; @@ -149,7 +150,7 @@ private static Database getDatabase(final DSLContext dslContext) { } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { final String dbName = Strings.addRandomSuffix("db", "_", 10); db = new OracleContainer() .withUsername("test") diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile index 37969de69e48..990198346407 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.27 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml index 6a95ad2edddb..b24719b753e4 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.3.27 + dockerImageTag: 0.4.0 dockerRepository: airbyte/destination-postgres-strict-encrypt githubIssueLabel: destination-postgres icon: postgresql.svg - license: MIT + license: ELv2 name: Postgres normalizationConfig: normalizationIntegrationType: postgres diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java index ffd4b02d226f..37964dab269c 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java @@ -22,6 +22,7 @@ import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -140,7 +141,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { db = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") .asCompatibleSubstituteFor("postgres")); db.start(); diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 23f1f283d200..7d9f429bff5b 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.27 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-postgres/metadata.yaml b/airbyte-integrations/connectors/destination-postgres/metadata.yaml index 5d936f1a7c0a..2b7a003fb13f 100644 --- a/airbyte-integrations/connectors/destination-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.3.27 + dockerImageTag: 0.4.0 dockerRepository: airbyte/destination-postgres githubIssueLabel: destination-postgres icon: postgresql.svg - license: MIT + license: ELv2 name: Postgres normalizationConfig: normalizationIntegrationType: postgres @@ -23,4 +23,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java index 9ff0f7413df1..f70d06171432 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java @@ -17,6 +17,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -127,7 +128,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new PostgreSQLContainer<>("postgres:13-alpine"); db.start(); } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java index 9931d1876f75..942c53b2439e 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -133,7 +134,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { db = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") .asCompatibleSubstituteFor("postgres")); db.start(); diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java index fa76a9723d9e..9755a3d60fb5 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java @@ -18,6 +18,7 @@ import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; @@ -133,7 +134,7 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { startTestContainers(); // do everything in a randomly generated schema so that we can wipe it out at the end. @@ -143,6 +144,7 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception { JdbcUtils.PORT_LIST_KEY, mangledConfig -> { getDatabaseFromConfig(mangledConfig).query(ctx -> ctx.fetch(String.format("CREATE SCHEMA %s;", schemaName))); + TEST_SCHEMAS.add(schemaName); }); } diff --git a/airbyte-integrations/connectors/destination-pubsub/metadata.yaml b/airbyte-integrations/connectors/destination-pubsub/metadata.yaml index 105e67c8c7d4..491d06998acf 100644 --- a/airbyte-integrations/connectors/destination-pubsub/metadata.yaml +++ b/airbyte-integrations/connectors/destination-pubsub/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/pubsub tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java index ebbecc89eed5..9d3a7b8d774c 100644 --- a/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java @@ -47,6 +47,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -185,7 +186,7 @@ NAMESPACE, nullToEmpty(n), } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { if (!Files.exists(CREDENTIALS_PATH)) { throw new IllegalStateException( "Must provide path to a gcp service account credentials file. By default {module-root}/" diff --git a/airbyte-integrations/connectors/destination-pulsar/metadata.yaml b/airbyte-integrations/connectors/destination-pulsar/metadata.yaml index 5afb9c0c0496..d910ddb6f8b9 100644 --- a/airbyte-integrations/connectors/destination-pulsar/metadata.yaml +++ b/airbyte-integrations/connectors/destination-pulsar/metadata.yaml @@ -10,11 +10,15 @@ data: name: Pulsar registries: cloud: - enabled: true + enabled: false # hide Pulsar Destination https://github.com/airbytehq/airbyte-cloud/issues/2614 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/pulsar tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java index 8e88d44ece8b..76a9fa933dfd 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -179,7 +180,7 @@ private List getIpAddresses() throws UnknownHostException { } @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { PULSAR = new PulsarContainer(DockerImageName.parse("apachepulsar/pulsar:2.8.1")); PULSAR.start(); } diff --git a/airbyte-integrations/connectors/destination-r2/metadata.yaml b/airbyte-integrations/connectors/destination-r2/metadata.yaml index 1bdb3346ff02..13e4ae9a353b 100644 --- a/airbyte-integrations/connectors/destination-r2/metadata.yaml +++ b/airbyte-integrations/connectors/destination-r2/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/r2 tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-rabbitmq/metadata.yaml b/airbyte-integrations/connectors/destination-rabbitmq/metadata.yaml index 44ae324f5ec1..4a5dcdbb4f45 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/metadata.yaml +++ b/airbyte-integrations/connectors/destination-rabbitmq/metadata.yaml @@ -10,11 +10,15 @@ data: name: RabbitMQ registries: cloud: - enabled: true + enabled: false # hide RabbitMQ Destination https://github.com/airbytehq/airbyte/issues/16315 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/rabbitmq tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-redis/metadata.yaml b/airbyte-integrations/connectors/destination-redis/metadata.yaml index 0e4a06381456..7d90e8dfbc4e 100644 --- a/airbyte-integrations/connectors/destination-redis/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redis/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/redis tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java index 149243619602..6321e2b2ab3e 100644 --- a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java @@ -10,6 +10,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; @@ -30,7 +31,7 @@ static void initContainer() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { jsonConfig = RedisDataFactory.jsonConfig( redisContainer.getHost(), redisContainer.getFirstMappedPort()); diff --git a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java index fad4da3d871e..d8f0a539882c 100644 --- a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.AfterAll; @@ -48,7 +49,7 @@ static void stop() { public abstract SshTunnel.TunnelMethod getTunnelMethod(); @Override - protected void setup(final TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { jsonConfig = RedisDataFactory.jsonConfig( redisContainer.getHost(), redisContainer.getFirstMappedPort()); diff --git a/airbyte-integrations/connectors/destination-redpanda/metadata.yaml b/airbyte-integrations/connectors/destination-redpanda/metadata.yaml index eaef3910a4d5..6ad22c5a4a55 100644 --- a/airbyte-integrations/connectors/destination-redpanda/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redpanda/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/redpanda tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java index 04df235eea36..6ac4e241ab8e 100644 --- a/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java @@ -14,6 +14,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -51,7 +52,7 @@ static void stopContainer() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { this.redpandaNameTransformer = new RedpandaNameTransformer(); this.adminClient = AdminClient.create(Map.of( AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, redpandaContainer.getBootstrapServers(), @@ -61,8 +62,8 @@ protected void setup(TestDestinationEnv testEnv) { } @Override - protected void tearDown(TestDestinationEnv testEnv) throws ExecutionException, InterruptedException { - var topics = adminClient.listTopics().listings().get().stream() + protected void tearDown(final TestDestinationEnv testEnv) throws ExecutionException, InterruptedException { + final var topics = adminClient.listTopics().listings().get().stream() .filter(tl -> !tl.isInternal()) .map(TopicListing::name) .collect(Collectors.toSet()); @@ -131,15 +132,15 @@ protected boolean implementsNamespaces() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) { - List records = new ArrayList<>(); - String bootstrapServers = redpandaContainer.getBootstrapServers(); - String groupId = redpandaNameTransformer.getIdentifier(namespace + "-" + streamName); - try (RedpandaConsumer redpandaConsumer = RedpandaConsumerFactory.getInstance(bootstrapServers, groupId)) { - String topicName = redpandaNameTransformer.topicName(namespace, streamName); + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + final List records = new ArrayList<>(); + final String bootstrapServers = redpandaContainer.getBootstrapServers(); + final String groupId = redpandaNameTransformer.getIdentifier(namespace + "-" + streamName); + try (final RedpandaConsumer redpandaConsumer = RedpandaConsumerFactory.getInstance(bootstrapServers, groupId)) { + final String topicName = redpandaNameTransformer.topicName(namespace, streamName); redpandaConsumer.subscribe(Collections.singletonList(topicName)); redpandaConsumer.poll(Duration.ofSeconds(5)).iterator() .forEachRemaining(r -> records.add(r.value().get(JavaBaseConstants.COLUMN_NAME_DATA))); diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index c68329876333..85e41effc438 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -9,9 +9,12 @@ FROM airbyte/integration-base-java:dev AS build # amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN yum install -y python3 python3-devel jq sshpass git && \ +RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ python -m ensurepip --upgrade && \ + # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 + pip3 install wheel && \ + pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ pip3 install dbt-redshift==1.0.0 "urllib3<2" # Luckily, none of normalization's files conflict with destination-bigquery's files :) @@ -43,7 +46,7 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.8 +LABEL io.airbyte.version=0.6.5 LABEL io.airbyte.name=airbyte/destination-redshift ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index f9087714fb26..98bed326e017 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -21,6 +21,7 @@ dependencies { implementation libs.airbyte.protocol implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:bases:base-java-s3') + implementation project(':airbyte-integrations:bases:base-typing-deduping') implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index a5498b216108..0b7d33a37ced 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 0.4.8 + dockerImageTag: 0.6.5 dockerRepository: airbyte/destination-redshift githubIssueLabel: destination-redshift icon: redshift.svg @@ -28,4 +28,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index b6216584442d..7d4250dd2f73 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -22,6 +22,9 @@ import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; import io.airbyte.integrations.base.ssh.SshWrappedDestination; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; @@ -35,7 +38,6 @@ import io.airbyte.integrations.destination.s3.S3BaseChecks; import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; import io.airbyte.integrations.destination.staging.StagingConsumerFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; @@ -44,6 +46,7 @@ import java.util.Map; import java.util.function.Consumer; import javax.sql.DataSource; +import org.apache.commons.lang3.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,15 +137,23 @@ public JsonNode toJdbcConfig(final JsonNode config) { } @Override + @Deprecated public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) { + throw new NotImplementedException("Should use the getSerializedMessageConsumer instead"); + } + + @Override + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(JsonNode config, + ConfiguredAirbyteCatalog catalog, + Consumer outputRecordCollector) + throws Exception { final EncryptionConfig encryptionConfig = config.has(UPLOADING_METHOD) ? EncryptionConfig.fromJson(config.get(UPLOADING_METHOD).get(JdbcUtils.ENCRYPTION_KEY)) : new NoEncryption(); final JsonNode s3Options = findS3Options(config); final S3DestinationConfig s3Config = getS3DestinationConfig(s3Options); final int numberOfFileBuffers = getNumberOfFileBuffers(s3Options); - if (numberOfFileBuffers > FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER) { LOGGER.warn(""" Increasing the number of file buffers past {} can lead to increased performance but @@ -151,15 +162,20 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, """, FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER, catalog.getStreams().size()); } - return new StagingConsumerFactory().create( + return new StagingConsumerFactory().createAsync( outputRecordCollector, getDatabase(getDataSource(config)), new RedshiftS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), getNamingResolver(), - CsvSerializedBuffer.createFunction(null, () -> new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX, numberOfFileBuffers)), config, catalog, - isPurgeStagingData(s3Options)); + isPurgeStagingData(s3Options), + new TypeAndDedupeOperationValve(), + new NoopTyperDeduper(), + // The parsedcatalog is only used in v2 mode, so just pass null for now + null, + // Overwriting null namespace with null is perfectly safe + null); } /** diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java index 688f27ae1aed..d114e3e5e3c9 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java @@ -4,17 +4,13 @@ package io.airbyte.integrations.destination.redshift.operations; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.integrations.destination.jdbc.SqlOperationsUtils; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.List; -import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,27 +58,4 @@ public void insertRecordsInternal(final JdbcDatabase database, SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, database, records); } - @Override - public boolean isValidData(final JsonNode data) { - // check overall size of the SUPER data - final String stringData = Jsons.serialize(data); - final int dataSize = stringData.getBytes(StandardCharsets.UTF_8).length; - boolean isValid = dataSize <= REDSHIFT_SUPER_MAX_BYTE_SIZE; - - // check VARCHAR limits for VARCHAR fields within the SUPER object, if overall object is valid - if (isValid) { - final Map dataMap = Jsons.flatten(data, true); - for (final Object value : dataMap.values()) { - if (value instanceof String stringValue) { - final int stringDataSize = stringValue.getBytes(StandardCharsets.UTF_8).length; - isValid = stringDataSize <= REDSHIFT_VARCHAR_MAX_BYTE_SIZE; - if (!isValid) { - break; - } - } - } - } - return isValid; - } - } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionHandler.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionHandler.java new file mode 100644 index 000000000000..eff12d4d33d0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionHandler.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import java.sql.Connection; +import java.sql.SQLException; + +public class RedshiftConnectionHandler { + + /** + * For to close a connection. Aimed to be use in test only. + * + * @param connection The connection to close + */ + public static void close(Connection connection) { + try { + connection.setAutoCommit(false); + connection.commit(); + connection.close(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java index 2f820492ce90..ad4f7dafd755 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java @@ -14,20 +14,24 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.ConnectionFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.TestingNamespaces; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.io.IOException; import java.nio.file.Path; +import java.sql.Connection; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; +import org.jooq.impl.DSL; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +51,8 @@ public abstract class RedshiftStagingS3DestinationAcceptanceTest extends JdbcDes private final RedshiftSQLNameTransformer namingResolver = new RedshiftSQLNameTransformer(); private final String USER_WITHOUT_CREDS = Strings.addRandomSuffix("test_user", "_", 5); + private Database database; + private Connection connection; protected TestDestinationEnv testDestinationEnv; private final ObjectMapper mapper = new ObjectMapper(); @@ -118,8 +124,7 @@ public void testCheckIncorrectDataBaseFailure() throws Exception { @Test public void testGetFileBufferDefault() { final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - assertEquals(destination.getNumberOfFileBuffers(config), - FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); } @Test @@ -204,49 +209,80 @@ private List retrieveRecordsFromTable(final String tableName, final St // for each test we create a new schema in the database. run the test in there and then remove it. @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { - final String schemaName = Strings.addRandomSuffix("integration_test", "_", 5); + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { + final String schemaName = TestingNamespaces.generate(); final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); baseConfig = getStaticConfig(); + database = createDatabase(); + removeOldNamespaces(); getDatabase().query(ctx -> ctx.execute(createSchemaQuery)); final String createUser = String.format("create user %s with password '%s' SESSION TIMEOUT 60;", USER_WITHOUT_CREDS, baseConfig.get("password").asText()); getDatabase().query(ctx -> ctx.execute(createUser)); final JsonNode configForSchema = Jsons.clone(baseConfig); ((ObjectNode) configForSchema).put("schema", schemaName); + TEST_SCHEMAS.add(schemaName); config = configForSchema; - this.testDestinationEnv = testEnv; + testDestinationEnv = testEnv; + } + + private void removeOldNamespaces() { + final List schemas; + try { + schemas = getDatabase().query(ctx -> ctx.fetch("SELECT schema_name FROM information_schema.schemata;")) + .stream() + .map(record -> record.get("schema_name").toString()) + .toList(); + } catch (final SQLException e) { + // if we can't fetch the schemas, just return. + return; + } + + int schemasDeletedCount = 0; + for (final String schema : schemas) { + if (TestingNamespaces.isOlderThan2Days(schema)) { + try { + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); + schemasDeletedCount++; + } catch (final SQLException e) { + LOGGER.error("Failed to delete old dataset: {}", schema, e); + } + } + } + LOGGER.info("Deleted {} old schemas.", schemasDeletedCount); } @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { + System.out.println("TEARING_DOWN_SCHEMAS: " + TEST_SCHEMAS); getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", config.get("schema").asText()))); + for (final String schema : TEST_SCHEMAS) { + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); + } getDatabase().query(ctx -> ctx.execute(String.format("drop user if exists %s;", USER_WITHOUT_CREDS))); + RedshiftConnectionHandler.close(connection); + } + + protected Database createDatabase() { + connection = ConnectionFactory.create(baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), + baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), + RedshiftInsertDestination.SSL_JDBC_PARAMETERS, + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + baseConfig.get(JdbcUtils.HOST_KEY).asText(), + baseConfig.get(JdbcUtils.PORT_KEY).asInt(), + baseConfig.get(JdbcUtils.DATABASE_KEY).asText())); + + return new Database(DSL.using(connection)); } protected Database getDatabase() { - return new Database( - DSLContextFactory.create( - baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), - baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.REDSHIFT.getDriverClassName(), - String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), - baseConfig.get(JdbcUtils.HOST_KEY).asText(), - baseConfig.get(JdbcUtils.PORT_KEY).asInt(), - baseConfig.get(JdbcUtils.DATABASE_KEY).asText()), - null, - RedshiftInsertDestination.SSL_JDBC_PARAMETERS)); + return database; } public RedshiftSQLNameTransformer getNamingResolver() { return namingResolver; } - @Override - protected boolean implementsRecordSizeLimitChecks() { - return true; - } - @Override protected int getMaxRecordValueLimit() { return RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE; diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java index 3be0b91396ed..bd0d9f6639c0 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java @@ -17,18 +17,22 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.ConnectionFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.integrations.standardtest.destination.TestingNamespaces; import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; +import java.sql.Connection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.jooq.impl.DSL; public abstract class SshRedshiftDestinationBaseAcceptanceTest extends JdbcDestinationAcceptanceTest { @@ -38,6 +42,10 @@ public abstract class SshRedshiftDestinationBaseAcceptanceTest extends JdbcDesti // config which refers to the schema that the test is being run in. protected JsonNode config; + private Database database; + + private Connection connection; + private final RedshiftSQLNameTransformer namingResolver = new RedshiftSQLNameTransformer(); private final String USER_WITHOUT_CREDS = Strings.addRandomSuffix("test_user", "_", 5); @@ -121,7 +129,7 @@ private List retrieveRecordsFromTable(final String tableName, final St JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, config -> { - return getDatabaseFromConfig(config).query(ctx -> ctx + return getDatabase().query(ctx -> ctx .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) .stream() .map(this::getJsonFromRecord) @@ -135,18 +143,20 @@ protected TestDataComparator getTestDataComparator() { return new RedshiftTestDataComparator(); } - private static Database getDatabaseFromConfig(final JsonNode config) { - return new Database( - DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.REDSHIFT.getDriverClassName(), - String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - null, - RedshiftInsertDestination.SSL_JDBC_PARAMETERS)); + private Database createDatabaseFromConfig(final JsonNode config) { + connection = ConnectionFactory.create(config.get(JdbcUtils.USERNAME_KEY).asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + RedshiftInsertDestination.SSL_JDBC_PARAMETERS, + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asInt(), + config.get(JdbcUtils.DATABASE_KEY).asText())); + + return new Database(DSL.using(connection)); + } + + private Database getDatabase() { + return database; } @Override @@ -155,20 +165,22 @@ protected int getMaxRecordValueLimit() { } @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { baseConfig = getStaticConfig(); final JsonNode configForSchema = Jsons.clone(baseConfig); - schemaName = Strings.addRandomSuffix("integration_test", "_", 5); + schemaName = TestingNamespaces.generate(); + TEST_SCHEMAS.add(schemaName); ((ObjectNode) configForSchema).put("schema", schemaName); config = configForSchema; - + database = createDatabaseFromConfig(config); // create the schema + SshTunnel.sshWrap( getConfig(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, config -> { - getDatabaseFromConfig(config).query(ctx -> ctx.fetch(String.format("CREATE SCHEMA %s;", schemaName))); + getDatabase().query(ctx -> ctx.fetch(String.format("CREATE SCHEMA %s;", schemaName))); }); // create the user @@ -177,7 +189,7 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception { JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, config -> { - getDatabaseFromConfig(config).query(ctx -> ctx.fetch(String.format("CREATE USER %s WITH PASSWORD '%s' SESSION TIMEOUT 60;", + getDatabase().query(ctx -> ctx.fetch(String.format("CREATE USER %s WITH PASSWORD '%s' SESSION TIMEOUT 60;", USER_WITHOUT_CREDS, baseConfig.get("password").asText()))); }); } @@ -190,7 +202,7 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception { JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, config -> { - getDatabaseFromConfig(config).query(ctx -> ctx.fetch(String.format("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName))); + getDatabase().query(ctx -> ctx.fetch(String.format("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName))); }); // blow away the user at the end. @@ -199,8 +211,9 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception { JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, config -> { - getDatabaseFromConfig(config).query(ctx -> ctx.fetch(String.format("DROP USER IF EXISTS %s;", USER_WITHOUT_CREDS))); + getDatabase().query(ctx -> ctx.fetch(String.format("DROP USER IF EXISTS %s;", USER_WITHOUT_CREDS))); }); + RedshiftConnectionHandler.close(connection); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java index 3e23ce98ff67..f1011abf42eb 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java @@ -16,7 +16,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.jdbc.SqlOperations; import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; @@ -61,6 +63,7 @@ class RedshiftStreamCopierTest { @BeforeEach public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); s3Client = mock(AmazonS3Client.class, RETURNS_DEEP_STUBS); db = mock(JdbcDatabase.class); sqlOperations = mock(SqlOperations.class); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java index 182f57e03285..1ff61d389c76 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperationsTest.java @@ -43,51 +43,4 @@ public void isValidDataForValid() { assertEquals(true, isValid); } - @Test - @DisplayName("isValidData should return false for invalid data - string too long") - public void isValidDataForInvalidNode() { - JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() - .put("id", 3) - .put("currency", generateBigString(1)) - .put("date", "2020-10-10T00:00:00Z") - .put("HKD", 10.5) - .put("NZD", 1.14) - .build()); - - RedshiftSqlOperations uut = new RedshiftSqlOperations(); - boolean isValid = uut.isValidData(testNode); - assertEquals(false, isValid); - } - - @Test - @DisplayName("isValidData should return false for invalid data - total object too big") - public void isValidDataForInvalidObject() { - JsonNode testNode = Jsons.jsonNode(ImmutableMap.builder() - .put("key1", generateBigString(-1)) - .put("key2", generateBigString(-1)) - .put("key3", generateBigString(-1)) - .put("key4", generateBigString(-1)) - .put("key5", generateBigString(-1)) - .put("key6", generateBigString(-1)) - .put("key7", generateBigString(-1)) - .put("key8", generateBigString(-1)) - .put("key9", generateBigString(-1)) - .put("key10", generateBigString(-1)) - .put("key11", generateBigString(-1)) - .put("key12", generateBigString(-1)) - .put("key13", generateBigString(-1)) - .put("key14", generateBigString(-1)) - .put("key15", generateBigString(-1)) - .put("key16", generateBigString(-1)) - .put("key17", generateBigString(-1)) - .put("key18", generateBigString(-1)) - .put("key19", generateBigString(-1)) - .put("key20", generateBigString(-1)) - .build()); - - RedshiftSqlOperations uut = new RedshiftSqlOperations(); - boolean isValid = uut.isValidData(testNode); - assertEquals(false, isValid); - } - } diff --git a/airbyte-integrations/connectors/destination-rockset/metadata.yaml b/airbyte-integrations/connectors/destination-rockset/metadata.yaml index c65839e71f9d..9c1839c55999 100644 --- a/airbyte-integrations/connectors/destination-rockset/metadata.yaml +++ b/airbyte-integrations/connectors/destination-rockset/metadata.yaml @@ -9,11 +9,15 @@ data: name: Rockset registries: cloud: - enabled: true + enabled: false # hide Rockset Destination https://github.com/airbytehq/airbyte-cloud/issues/2615 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/rockset tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java index 1c71d1883b44..043a12b529bf 100644 --- a/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java @@ -24,6 +24,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -147,7 +148,7 @@ private static void dropFields(JsonNode node, String... fields) { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { // Nothing to do } diff --git a/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml b/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml index 51c30a2d0c12..dc0a004c20bd 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/s3-glue tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java index 5654c34466d6..dc25fa123ccf 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java @@ -12,11 +12,11 @@ public class S3GlueJsonlDestinationAcceptanceTest extends S3BaseJsonlDestinationAcceptanceTest { @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { super.tearDown(testEnv); - GlueDestinationConfig glueDestinationConfig = GlueDestinationConfig.getInstance(configJson); - try (var glueTestClient = new GlueTestClient(glueDestinationConfig.getAWSGlueInstance())) { + final GlueDestinationConfig glueDestinationConfig = GlueDestinationConfig.getInstance(configJson); + try (final var glueTestClient = new GlueTestClient(glueDestinationConfig.getAWSGlueInstance())) { glueTestClient.purgeDatabase(glueDestinationConfig.getDatabase()); diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java index f3c4960e7e6e..e5d46921114f 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java @@ -9,11 +9,11 @@ public class S3GlueJsonlGzipDestinationAcceptanceTest extends S3BaseJsonlGzipDestinationAcceptanceTest { @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { super.tearDown(testEnv); - GlueDestinationConfig glueDestinationConfig = GlueDestinationConfig.getInstance(configJson); - try (var glueTestClient = new GlueTestClient(glueDestinationConfig.getAWSGlueInstance())) { + final GlueDestinationConfig glueDestinationConfig = GlueDestinationConfig.getInstance(configJson); + try (final var glueTestClient = new GlueTestClient(glueDestinationConfig.getAWSGlueInstance())) { glueTestClient.purgeDatabase(glueDestinationConfig.getDatabase()); diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 87e4d41ee6cf..18668204b383 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -28,25 +28,8 @@ RUN /bin/bash -c 'set -e && \ if [ "$ARCH" == "x86_64" ] || [ "$ARCH" = "amd64" ]; then \ echo "$ARCH" && \ yum install lzop lzo lzo-dev -y; \ - elif [ "$ARCH" == "aarch64" ] || [ "$ARCH" = "arm64" ]; then \ - echo "$ARCH" && \ - yum group install -y "Development Tools"; \ - yum install lzop lzo lzo-dev wget curl unzip zip maven git which -y; \ - wget https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz -P /tmp; \ - cd /tmp && tar xvfz lzo-2.10.tar.gz; \ - cd /tmp/lzo-2.10/ && ./configure --enable-shared --prefix /usr/local/lzo-2.10; \ - cd /tmp/lzo-2.10/ && make; \ - cd /tmp/lzo-2.10/ && make install; \ - git clone https://github.com/twitter/hadoop-lzo.git /usr/lib/hadoop/lib/hadoop-lzo/; \ - curl -s "https://get.sdkman.io" | bash; \ - source /root/.sdkman/bin/sdkman-init.sh; \ - sdk install java 8.0.342-librca; \ - sdk use java 8.0.342-librca; \ - cd /usr/lib/hadoop/lib/hadoop-lzo/ && C_INCLUDE_PATH=/usr/local/lzo-2.10/include LIBRARY_PATH=/usr/local/lzo-2.10/lib mvn clean package; \ - find /usr/lib/hadoop/lib/hadoop-lzo/ -name '*libgplcompression*' -exec cp {} /usr/lib/ \; ;\ - else \ - echo "unknown arch" ;\ fi' -LABEL io.airbyte.version=0.4.1 +RUN yum clean all +LABEL io.airbyte.version=0.5.1 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-s3/finalize_build.sh b/airbyte-integrations/connectors/destination-s3/finalize_build.sh index 9d1f3a566426..256688b7f687 100644 --- a/airbyte-integrations/connectors/destination-s3/finalize_build.sh +++ b/airbyte-integrations/connectors/destination-s3/finalize_build.sh @@ -1,36 +1,13 @@ #!/bin/bash set -e +echo "Running destination-s3 docker custom steps..." + ARCH=$(uname -m) if [ "$ARCH" == "x86_64" ] || [ "$ARCH" = "amd64" ]; then echo "$ARCH" yum install lzop lzo lzo-dev -y - -# alanechere: I'm not sure we need this custom install of lzo anymore. Using the yum install above works in the build context. -elif [ "$ARCH" == "aarch64" ] || [ "$ARCH" = "arm64" ]; then - echo "$ARCH" - yum group install -y "Development Tools" - yum install lzop lzo lzo-dev wget curl unzip zip maven git which -y - wget https://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz -P /tmp - cd /tmp - tar xvfz lzo-2.10.tar.gz - cd /tmp/lzo-2.10/ - ./configure --enable-shared --prefix=/usr/local/lzo-2.10 - make - make install - git clone https://github.com/twitter/hadoop-lzo.git /usr/lib/hadoop/lib/hadoop-lzo/ - curl -s "https://get.sdkman.io" | bash - source /root/.sdkman/bin/sdkman-init.sh - # alafanechere: The following command exits with 1, it makes the build fail. - sdk install java 8.0.342-librca - sdk use java 8.0.342-librca - cd /usr/lib/hadoop/lib/hadoop-lzo/ - - C_INCLUDE_PATH=/usr/local/lzo-2.10/include LIBRARY_PATH=/usr/local/lzo-2.10/lib mvn clean package - find /usr/lib/hadoop/lib/hadoop-lzo/ -name '*libgplcompression*' -exec cp {} /usr/lib/ \; - echo "AFTER FIND" - -else - echo "Unknown architecture" fi + +yum clean all diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index 5a7d26eb7589..444ffc504893 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 0.4.1 + dockerImageTag: 0.5.1 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg - license: MIT + license: ELv2 name: S3 registries: cloud: @@ -23,4 +23,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/s3 tags: - language:java + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml b/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml index 0d7746f5f17b..3b8c369a5312 100644 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml +++ b/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: database connectorType: destination definitionId: FAKE-UUID-0000-0000-000000000000 @@ -16,6 +18,7 @@ data: name: Scaffold Destination Python releaseDate: TODO releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/destinations/scaffold-destination-python tags: - language:python diff --git a/airbyte-integrations/connectors/destination-scylla/metadata.yaml b/airbyte-integrations/connectors/destination-scylla/metadata.yaml index 72e6b3467be2..c1c8e06d6a5e 100644 --- a/airbyte-integrations/connectors/destination-scylla/metadata.yaml +++ b/airbyte-integrations/connectors/destination-scylla/metadata.yaml @@ -10,11 +10,15 @@ data: name: Scylla registries: cloud: - enabled: true + enabled: false # hide Scylla Destination https://github.com/airbytehq/airbyte-cloud/issues/2617 oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/scylla tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java index 1416b236909e..6ea7c8dca661 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java @@ -12,6 +12,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.junit.jupiter.api.BeforeAll; @@ -37,21 +38,21 @@ static void initContainer() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { configJson = TestDataFactory.jsonConfig( HostPortResolver.resolveHost(scyllaContainer), HostPortResolver.resolvePort(scyllaContainer)); - var scyllaConfig = new ScyllaConfig(configJson); + final var scyllaConfig = new ScyllaConfig(configJson); this.scyllaCqlProvider = new ScyllaCqlProvider(scyllaConfig); this.nameTransformer = new ScyllaNameTransformer(scyllaConfig); } @Override - protected void tearDown(TestDestinationEnv testEnv) { + protected void tearDown(final TestDestinationEnv testEnv) { scyllaCqlProvider.metadata().stream() .filter(m -> !m.value1().startsWith("system")) .forEach(meta -> { - var keyspace = meta.value1(); + final var keyspace = meta.value1(); meta.value2().forEach(table -> scyllaCqlProvider.truncate(keyspace, table)); }); } @@ -92,12 +93,12 @@ protected boolean supportObjectDataTypeTest() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) { - var keyspace = nameTransformer.outputKeyspace(namespace); - var table = nameTransformer.outputTable(streamName); + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) { + final var keyspace = nameTransformer.outputKeyspace(namespace); + final var table = nameTransformer.outputTable(streamName); return scyllaCqlProvider.select(keyspace, table).stream() .sorted(Comparator.comparing(Triplet::value3)) .map(Triplet::value2) diff --git a/airbyte-integrations/connectors/destination-selectdb/metadata.yaml b/airbyte-integrations/connectors/destination-selectdb/metadata.yaml index c12b615a32e4..fe717849c4a5 100644 --- a/airbyte-integrations/connectors/destination-selectdb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-selectdb/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/selectdb tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-selectdb/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-selectdb/src/main/resources/spec.json index c0f6aa5755cf..e06e54a5c9ac 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-selectdb/src/main/resources/spec.json @@ -9,7 +9,12 @@ "title": "SelectDB Destination Spec", "type": "object", "required": [ - "load_url", "jdbc_url", "cluster_name", "user_name", "password", "database" + "load_url", + "jdbc_url", + "cluster_name", + "user_name", + "password", + "database" ], "properties": { "load_url": { diff --git a/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java index a8817d7a90e7..8ed3769edb9e 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java @@ -15,6 +15,7 @@ import java.nio.file.Paths; import java.sql.*; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import org.apache.commons.lang3.StringEscapeUtils; import org.junit.jupiter.api.AfterAll; @@ -106,7 +107,7 @@ protected List retrieveRecords(TestDestinationEnv testEnv, } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { // TODO Implement this method to run any setup actions needed before every test case } diff --git a/airbyte-integrations/connectors/destination-sftp-json/metadata.yaml b/airbyte-integrations/connectors/destination-sftp-json/metadata.yaml index 3dd6c3a3f0df..eec56d8268be 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/metadata.yaml +++ b/airbyte-integrations/connectors/destination-sftp-json/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/sftp-json tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index edf31d438d2a..f5b15634eddd 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -16,9 +16,12 @@ FROM airbyte/integration-base-java:dev # rm /tmp/YourKit-JavaProfiler-2021.3-docker.zip # amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && \ +RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ python -m ensurepip --upgrade && \ + # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 + pip3 install wheel && \ + pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ pip3 install dbt-snowflake==1.0.0 "urllib3<2" COPY --from=airbyte/normalization-snowflake:dev /airbyte /airbyte @@ -45,7 +48,8 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=1.0.5 + +LABEL io.airbyte.version=2.1.3 LABEL io.airbyte.name=airbyte/destination-snowflake ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" diff --git a/airbyte-integrations/connectors/destination-snowflake/README.md b/airbyte-integrations/connectors/destination-snowflake/README.md index f19cf11f18bf..79de239617d3 100644 --- a/airbyte-integrations/connectors/destination-snowflake/README.md +++ b/airbyte-integrations/connectors/destination-snowflake/README.md @@ -1,11 +1,14 @@ # Snowflake Destination ## Documentation -* [User Documentation](https://docs.airbyte.io/integrations/destinations/snowflake) + +- [User Documentation](https://docs.airbyte.io/integrations/destinations/snowflake) ## Community Contributor + 1. Look at the integration documentation to see how to create a warehouse/database/schema/user/role for Airbyte to sync into. 1. Create a file at `secrets/config.json` with the following format: + ``` { "host": "testhost.snowflakecomputing.com", @@ -25,7 +28,7 @@ Put the contents of the following LastPass secrets into corresponding files under the `secrets` directory: | LastPass Secret | File | -|--------------------------------------------------------------------------------------------------------|------------------------------------------| +| ------------------------------------------------------------------------------------------------------ | ---------------------------------------- | | `destination snowflake - test creds (secrets/config.json)` | `secrets/config.json` | | `destination snowflake - insert test creds (secrets/insert_config.json)` | `secrets/insert_config.json` | | `destination snowflake - internal staging test creds (secrets/internal_staging_config.json)` | `secrets/internal_staging_config.json` | @@ -83,9 +86,21 @@ DESC STORAGE INTEGRATION GCS_AIRBYTE_INTEGRATION; That last query (`DESC STORAGE`) will show a `STORAGE_GCP_SERVICE_ACCOUNT` property with an email as the property value. Add read/write permissions to your bucket with that email if it's not already there. If you ever need to start over, use this: + ```sql DROP DATABASE IF EXISTS INTEGRATION_TEST_DESTINATION; DROP USER IF EXISTS INTEGRATION_TEST_USER_DESTINATION; DROP ROLE IF EXISTS INTEGRATION_TESTER_DESTINATION; DROP WAREHOUSE IF EXISTS INTEGRATION_TEST_WAREHOUSE_DESTINATION; ``` + +### Setup for various error-case users: +Log in as the `INTEGRATION_TEST_USER_DESTINATION` user, and run this: +```sql +drop schema if exists INTEGRATION_TEST_DESTINATION.TEXT_SCHEMA; +create schema INTEGRATION_TEST_DESTINATION.TEXT_SCHEMA; +grant ownership on schema INTEGRATION_TEST_DESTINATION.TEXT_SCHEMA to role INTEGRATION_TESTER_DESTINATION revoke current grants; +grant all privileges on schema INTEGRATION_TEST_DESTINATION.TEXT_SCHEMA to role NO_ACTIVE_WAREHOUSE_ROLE; +``` + +These tests are currently disabled (`testCheckWithNoProperStagingPermissionConnection`, `testCheckWithNoActiveWarehouseConnection`). Their test users keep breaking (i.e. becoming the schema owner) because our tests are tearing down `TEXT_SCHEMA` after every test. diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index d7416653caa0..9059aeb8be09 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -12,10 +12,10 @@ application { '-XX:MaxRAMPercentage=75.0', // '-Xmx2000m', // '-XX:NativeMemoryTracking=detail', -// "-Djava.rmi.server.hostname=localhost", +// '-Djava.rmi.server.hostname=localhost', // '-Dcom.sun.management.jmxremote=true', // '-Dcom.sun.management.jmxremote.port=6000', -// "-Dcom.sun.management.jmxremote.rmi.port=6000", +// '-Dcom.sun.management.jmxremote.rmi.port=6000', // '-Dcom.sun.management.jmxremote.local.only=false', // '-Dcom.sun.management.jmxremote.authenticate=false', // '-Dcom.sun.management.jmxremote.ssl=false', @@ -35,6 +35,7 @@ dependencies { // TODO (edgao) explain how you built this jar implementation files('lib/snowflake-jdbc.jar') implementation 'org.apache.commons:commons-csv:1.4' + implementation 'org.apache.commons:commons-text:1.10.0' implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' implementation "io.aesy:datasize:1.0.0" implementation 'com.zaxxer:HikariCP:5.0.1' @@ -45,6 +46,7 @@ dependencies { implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-gcs') implementation project(':airbyte-integrations:bases:base-java-s3') + implementation project(':airbyte-integrations:bases:base-typing-deduping') implementation libs.airbyte.protocol // this is a configuration to make mockito work with final classes @@ -52,6 +54,7 @@ dependencies { integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:bases:base-typing-deduping-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-snowflake') integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index ee21eabf74c0..54b819007784 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 1.0.5 + dockerImageTag: 2.1.3 dockerRepository: airbyte/destination-snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg - license: MIT + license: ELv2 name: Snowflake normalizationConfig: normalizationIntegrationType: snowflake @@ -22,10 +22,19 @@ data: jobSpecific: - jobType: sync resourceRequirements: - memory_limit: 1Gi - memory_request: 1Gi + memory_limit: 2Gi + memory_request: 2Gi documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake supportsDbt: true tags: - language:java + releases: + breakingChanges: + 2.0.0: + message: "Remove GCS/S3 loading method support." + upgradeDeadline: "2023-08-31" + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index de4afc4f0651..f0c8244f15b0 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -14,6 +14,7 @@ import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; +// TODO: Remove the Switching Destination from this class as part of code cleanup. @Slf4j public class SnowflakeDestination extends SwitchingDestination { @@ -21,8 +22,6 @@ public class SnowflakeDestination extends SwitchingDestination outputRecordCollector) - throws Exception { - log.info("destination class: {}", getClass()); - // this is how we toggle async snowflake on. - final boolean useAsyncSnowflake = false; - // final boolean useAsyncSnowflake = config.has("loading_method") - // && config.get("loading_method").has("method") - // && config.get("loading_method").get("method").asText().equals("Internal Staging"); - - log.info("using async snowflake: {}", useAsyncSnowflake); - if (useAsyncSnowflake) { - return new SnowflakeInternalStagingDestination(airbyteEnvironment).getSerializedMessageConsumer(config, catalog, outputRecordCollector); - } else { - return new ShimToSerializedAirbyteMessageConsumer(getConsumer(config, catalog, outputRecordCollector)); - } - + final Consumer outputRecordCollector) { + return new SnowflakeInternalStagingDestination(airbyteEnvironment).getSerializedMessageConsumer(config, catalog, outputRecordCollector); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java index 1d64d09a4ac5..6f6044f3dd65 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java @@ -7,49 +7,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; import io.airbyte.integrations.destination.snowflake.SnowflakeDestination.DestinationType; import java.util.Map; public class SnowflakeDestinationResolver { public static DestinationType getTypeFromConfig(final JsonNode config) { - if (isS3Copy(config)) { - return DestinationType.COPY_S3; - } else if (isGcsCopy(config)) { - return DestinationType.COPY_GCS; - } else { - return DestinationType.INTERNAL_STAGING; - } + return DestinationType.INTERNAL_STAGING; } - public static boolean isS3Copy(final JsonNode config) { - return config.has("loading_method") && config.get("loading_method").isObject() && config.get("loading_method").has("s3_bucket_name"); - } - - public static boolean isGcsCopy(final JsonNode config) { - return config.has("loading_method") && config.get("loading_method").isObject() && config.get("loading_method").has("project_id"); - } - - public static int getNumberOfFileBuffers(final JsonNode config) { - int numOfFileBuffers = FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER; - if (config.has(FileBuffer.FILE_BUFFER_COUNT_KEY)) { - numOfFileBuffers = Math.min(config.get(FileBuffer.FILE_BUFFER_COUNT_KEY).asInt(), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); - } - // Only allows for values 10 <= numOfFileBuffers <= 50 - return Math.max(numOfFileBuffers, FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - public static Map getTypeToDestination( - final String airbyteEnvironment) { - final SnowflakeS3StagingDestination s3StagingDestination = new SnowflakeS3StagingDestination(airbyteEnvironment); - final SnowflakeGcsStagingDestination gcsStagingDestination = new SnowflakeGcsStagingDestination(airbyteEnvironment); + public static Map getTypeToDestination(final String airbyteEnvironment) { final SnowflakeInternalStagingDestination internalStagingDestination = new SnowflakeInternalStagingDestination(airbyteEnvironment); - return ImmutableMap.of( - DestinationType.COPY_S3, s3StagingDestination, - DestinationType.COPY_GCS, gcsStagingDestination, - DestinationType.INTERNAL_STAGING, internalStagingDestination); + return ImmutableMap.of(DestinationType.INTERNAL_STAGING, internalStagingDestination); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStagingDestination.java deleted file mode 100644 index cd744429839e..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStagingDestination.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static io.airbyte.integrations.destination.snowflake.SnowflakeDestinationResolver.getNumberOfFileBuffers; -import static io.airbyte.integrations.destination.snowflake.SnowflakeS3StagingDestination.isPurgeStagingData; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.WriteChannel; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsConfig; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; -import io.airbyte.integrations.destination.staging.StagingConsumerFactory; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.Map; -import java.util.function.Consumer; -import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowflakeGcsStagingDestination extends AbstractJdbcDestination implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeGcsStagingDestination.class); - private final String airbyteEnvironment; - - public SnowflakeGcsStagingDestination(final String airbyteEnvironment) { - this(new SnowflakeSQLNameTransformer(), airbyteEnvironment); - } - - public SnowflakeGcsStagingDestination(final SnowflakeSQLNameTransformer nameTransformer, final String airbyteEnvironment) { - super("", nameTransformer, new SnowflakeSqlOperations()); - this.airbyteEnvironment = airbyteEnvironment; - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) { - final GcsConfig gcsConfig = GcsConfig.getGcsConfig(config); - final NamingConventionTransformer nameTransformer = getNamingResolver(); - final SnowflakeGcsStagingSqlOperations snowflakeGcsStagingSqlOperations = - new SnowflakeGcsStagingSqlOperations(nameTransformer, gcsConfig); - final DataSource dataSource = getDataSource(config); - - try { - final JdbcDatabase database = getDatabase(dataSource); - final String outputSchema = super.getNamingResolver().getIdentifier(config.get("schema").asText()); - - attemptTableOperations(outputSchema, database, nameTransformer, snowflakeGcsStagingSqlOperations, - true); - attemptWriteAndDeleteGcsObject(gcsConfig, outputSchema); - - return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); - } catch (final Exception e) { - LOGGER.error("Exception while checking connection: ", e); - return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED) - .withMessage("Could not connect with provided configuration. \n" + e.getMessage()); - } finally { - try { - DataSourceFactory.close(dataSource); - } catch (final Exception e) { - LOGGER.warn("Unable to close data source.", e); - } - } - } - - private static void attemptWriteAndDeleteGcsObject(final GcsConfig gcsConfig, final String outputTableName) throws IOException { - final Storage storageClient = getStorageClient(gcsConfig); - final BlobId blobId = BlobId.of(gcsConfig.getBucketName(), "check-content/" + outputTableName); - final BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build(); - - storageClient.create(blobInfo); - - try (WriteChannel writer = storageClient.writer(blobInfo)) { - // Try to write a dummy message to make sure user has all required permissions - final byte[] content = "Hello, World!".getBytes(UTF_8); - writer.write(ByteBuffer.wrap(content, 0, content.length)); - } finally { - storageClient.delete(blobId); - } - } - - public static Storage getStorageClient(final GcsConfig gcsConfig) throws IOException { - final InputStream credentialsInputStream = new ByteArrayInputStream(gcsConfig.getCredentialsJson().getBytes(StandardCharsets.UTF_8)); - final GoogleCredentials credentials = GoogleCredentials.fromStream(credentialsInputStream); - return StorageOptions.newBuilder() - .setCredentials(credentials) - .setProjectId(gcsConfig.getProjectId()) - .build() - .getService(); - } - - @Override - protected DataSource getDataSource(final JsonNode config) { - return SnowflakeDatabase.createDataSource(config, airbyteEnvironment); - } - - @Override - protected JdbcDatabase getDatabase(final DataSource dataSource) { - return SnowflakeDatabase.getDatabase(dataSource); - } - - @Override - protected Map getDefaultConnectionProperties(final JsonNode config) { - return Collections.emptyMap(); - } - - // this is a no op since we override getDatabase. - @Override - public JsonNode toJdbcConfig(final JsonNode config) { - return Jsons.emptyObject(); - } - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { - final GcsConfig gcsConfig = GcsConfig.getGcsConfig(config); - return new StagingConsumerFactory().create( - outputRecordCollector, - getDatabase(getDataSource(config)), - new SnowflakeGcsStagingSqlOperations(getNamingResolver(), gcsConfig), - getNamingResolver(), - CsvSerializedBuffer.createFunction(null, () -> new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX, getNumberOfFileBuffers(config))), - config, - catalog, - isPurgeStagingData(config)); - - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStagingSqlOperations.java deleted file mode 100644 index 88438b85a0c3..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStagingSqlOperations.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static io.airbyte.integrations.destination.snowflake.SnowflakeInternalStagingSqlOperations.UPLOAD_RETRY_LIMIT; - -import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import io.airbyte.commons.lang.Exceptions; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsConfig; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.staging.StagingOperations; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.channels.Channels; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import org.joda.time.DateTime; - -public class SnowflakeGcsStagingSqlOperations extends SnowflakeSqlOperations implements StagingOperations { - - private final NamingConventionTransformer nameTransformer; - private final Storage storageClient; - private final GcsConfig gcsConfig; - private final Set fullObjectKeys = new HashSet<>(); - - public SnowflakeGcsStagingSqlOperations(final NamingConventionTransformer nameTransformer, final GcsConfig gcsConfig) { - this.nameTransformer = nameTransformer; - this.gcsConfig = gcsConfig; - this.storageClient = getStorageClient(gcsConfig); - } - - private Storage getStorageClient(final GcsConfig gcsConfig) { - try { - final InputStream credentialsInputStream = new ByteArrayInputStream(gcsConfig.getCredentialsJson().getBytes(StandardCharsets.UTF_8)); - final GoogleCredentials credentials = GoogleCredentials.fromStream(credentialsInputStream); - return StorageOptions.newBuilder() - .setCredentials(credentials) - .setProjectId(gcsConfig.getProjectId()) - .build() - .getService(); - } catch (final Exception e) { - throw new RuntimeException(e); - } - - } - - @Override - public String getStageName(final String namespace, final String streamName) { - return nameTransformer.applyDefaultCase(String.join("_", - nameTransformer.convertStreamName(namespace), - nameTransformer.convertStreamName(streamName))); - } - - @Override - public String getStagingPath(final UUID connectionId, final String namespace, final String streamName, final DateTime writeDatetime) { - // see https://docs.snowflake.com/en/user-guide/data-load-considerations-stage.html - return nameTransformer.applyDefaultCase(String.format("%s/%s_%02d_%02d_%02d/%s/", - getStageName(namespace, streamName), - writeDatetime.year().get(), - writeDatetime.monthOfYear().get(), - writeDatetime.dayOfMonth().get(), - writeDatetime.hourOfDay().get(), - connectionId)); - } - - @Override - public void createStageIfNotExists(final JdbcDatabase database, final String stageName) throws Exception { - final String bucket = gcsConfig.getBucketName(); - if (!doesBucketExist(bucket)) { - LOGGER.info("Bucket {} does not exist; creating...", bucket); - storageClient.create(BucketInfo.newBuilder(bucket).build()); - LOGGER.info("Bucket {} has been created.", bucket); - } - } - - private boolean doesBucketExist(final String bucket) { - return storageClient.get(bucket, Storage.BucketGetOption.fields()) != null; - } - - @Override - public String uploadRecordsToStage(final JdbcDatabase database, - final SerializableBuffer recordsData, - final String schemaName, - final String stageName, - final String stagingPath) - throws Exception { - final List exceptionsThrown = new ArrayList<>(); - while (exceptionsThrown.size() < UPLOAD_RETRY_LIMIT) { - try { - final String fileName = loadDataIntoBucket(stagingPath, recordsData); - LOGGER.info("Successfully loaded records to stage {} with {} re-attempt(s)", stagingPath, exceptionsThrown.size()); - return fileName; - } catch (final Exception e) { - LOGGER.error("Failed to upload records into storage {}", stagingPath, e); - exceptionsThrown.add(e); - } - } - throw new RuntimeException(String.format("Exceptions thrown while uploading records into storage: %s", Strings.join(exceptionsThrown, "\n"))); - } - - /** - * Upload the file from {@code recordsData} to S3 and simplify the filename as .. - * - *

    - * Method mirrors similarly named method within - * {@link io.airbyte.integrations.destination.s3.S3StorageOperations} - *

    - * - * @param objectPath filepath to the object - * @param recordsData serialized {@link io.airbyte.protocol.models.AirbyteRecordMessage}s - * @return the uploaded filename, which is different from the serialized buffer filename - */ - private String loadDataIntoBucket(final String objectPath, final SerializableBuffer recordsData) throws IOException { - - final String fullObjectKey = objectPath + recordsData.getFilename(); - fullObjectKeys.add(fullObjectKey); - - final var blobId = BlobId.of(gcsConfig.getBucketName(), fullObjectKey); - final var blobInfo = BlobInfo.newBuilder(blobId).build(); - final var blob = storageClient.create(blobInfo); - final var channel = blob.writer(); - try (channel) { - final OutputStream outputStream = Channels.newOutputStream(channel); - final InputStream dataInputStream = recordsData.getInputStream(); - dataInputStream.transferTo(outputStream); - } catch (final Exception e) { - LOGGER.error("Failed to load data into storage {}", objectPath, e); - throw new RuntimeException(e); - } - return recordsData.getFilename(); - } - - @Override - public void copyIntoTableFromStage(final JdbcDatabase database, - final String stageName, - final String stagingPath, - final List stagedFiles, - final String tableName, - final String schemaName) - throws Exception { - LOGGER.info("Starting copy to target table from stage: {} in destination from stage: {}, schema: {}, .", - tableName, stagingPath, schemaName); - // Print actual SQL query if user needs to manually force reload from staging - Exceptions.toRuntime(() -> database.execute(getCopyQuery(stagingPath, stagedFiles, - tableName, schemaName))); - LOGGER.info("Copy to target table {}.{} in destination complete.", schemaName, tableName); - } - - private String getCopyQuery(final String stagingPath, final List stagedFiles, final String dstTableName, final String schemaName) { - - return String.format( - "COPY INTO %s.%s FROM '%s' storage_integration = gcs_airbyte_integration " - + " file_format = (type = csv compression = auto field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') " - + generateFilesList(stagedFiles) + ";", - schemaName, - dstTableName, - generateBucketPath(stagingPath)); - } - - private String generateBucketPath(final String stagingPath) { - return "gcs://" + gcsConfig.getBucketName() + "/" + stagingPath; - } - - @Override - public void cleanUpStage(final JdbcDatabase database, final String stageName, final List stagedFiles) throws Exception { - cleanUpBucketObject(stagedFiles); - } - - private void cleanUpBucketObject(final List currentStagedFiles) { - currentStagedFiles.forEach(candidate -> fullObjectKeys.forEach(fullBlobPath -> { - if (fullBlobPath.contains(candidate)) { - removeBlob(fullBlobPath); - } - })); - } - - private void removeBlob(final String file) { - final var blobId = BlobId.of(gcsConfig.getBucketName(), file); - storageClient.delete(blobId); - } - - @Override - public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception { - dropBucketObject(); - } - - private void dropBucketObject() { - if (!fullObjectKeys.isEmpty()) { - final Iterator iterator = fullObjectKeys.iterator(); - while (iterator.hasNext()) { - final String element = iterator.next(); - if (element != null) { - removeBlob(element); - iterator.remove(); - } - } - } - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopier.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopier.java deleted file mode 100644 index a28fa798b0b2..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopier.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static io.airbyte.integrations.destination.snowflake.SnowflakeS3StreamCopier.MAX_FILES_PER_COPY; - -import com.google.cloud.storage.Storage; -import com.google.common.collect.Lists; -import io.airbyte.commons.lang.Exceptions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.StagingFilenameGenerator; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsConfig; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsStreamCopier; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowflakeGcsStreamCopier extends GcsStreamCopier implements SnowflakeParallelCopyStreamCopier { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeGcsStreamCopier.class); - - public SnowflakeGcsStreamCopier(final String stagingFolder, - final DestinationSyncMode destSyncMode, - final String schema, - final String streamName, - final Storage storageClient, - final JdbcDatabase db, - final GcsConfig gcsConfig, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final StagingFilenameGenerator stagingFilenameGenerator) { - super(stagingFolder, destSyncMode, schema, streamName, storageClient, db, gcsConfig, nameTransformer, sqlOperations); - this.filenameGenerator = stagingFilenameGenerator; - } - - @Override - public void copyStagingFileToTemporaryTable() throws Exception { - List> partitions = Lists.partition(new ArrayList<>(gcsStagingFiles), MAX_FILES_PER_COPY); - LOGGER.info("Starting parallel copy to tmp table: {} in destination for stream: {}, schema: {}. Chunks count {}", tmpTableName, streamName, - schemaName, partitions.size()); - - copyFilesInParallel(partitions); - LOGGER.info("Copy to tmp table {} in destination for stream {} complete.", tmpTableName, streamName); - } - - @Override - public void copyIntoStage(List files) { - - final var copyQuery = String.format( - "COPY INTO %s.%s FROM '%s' storage_integration = gcs_airbyte_integration " - + " file_format = (type = csv field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') " - + "files = (" + generateFilesList(files) + " );", - schemaName, - tmpTableName, - generateBucketPath()); - - Exceptions.toRuntime(() -> db.execute(copyQuery)); - } - - @Override - public String generateBucketPath() { - return "gcs://" + gcsConfig.getBucketName() + "/" + stagingFolder + "/" + schemaName + "/"; - } - - @Override - public void copyGcsCsvFileIntoTable(final JdbcDatabase database, - final String gcsFileLocation, - final String schema, - final String tableName, - final GcsConfig gcsConfig) - throws SQLException { - throw new RuntimeException("Snowflake GCS Stream Copier should not copy individual files without use of a parallel copy"); - - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java deleted file mode 100644 index a56bef096997..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsStreamCopierFactory.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import com.google.cloud.storage.Storage; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.StagingFilenameGenerator; -import io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsConfig; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsStreamCopierFactory; -import io.airbyte.protocol.models.v0.DestinationSyncMode; - -public class SnowflakeGcsStreamCopierFactory extends GcsStreamCopierFactory { - - @Override - public StreamCopier create(final String stagingFolder, - final DestinationSyncMode syncMode, - final String schema, - final String streamName, - final Storage storageClient, - final JdbcDatabase db, - final GcsConfig gcsConfig, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations) - throws Exception { - return new SnowflakeGcsStreamCopier( - stagingFolder, - syncMode, - schema, - streamName, - storageClient, - db, - gcsConfig, - nameTransformer, - sqlOperations, - new StagingFilenameGenerator(streamName, GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES)); - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 9dbd25e220e4..99f8f1200e4e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -4,34 +4,43 @@ package io.airbyte.integrations.destination.snowflake; -import static io.airbyte.integrations.destination.snowflake.SnowflakeDestinationResolver.getNumberOfFileBuffers; - import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.Destination; import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; +import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeDestinationHandler; +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator; +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeV1V2Migrator; import io.airbyte.integrations.destination.staging.StagingConsumerFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.util.Collections; import java.util.Map; import java.util.UUID; import java.util.function.Consumer; import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SnowflakeInternalStagingDestination extends AbstractJdbcDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeInternalStagingDestination.class); + private static final String RAW_SCHEMA_OVERRIDE = "raw_data_schema"; private final String airbyteEnvironment; public SnowflakeInternalStagingDestination(final String airbyteEnvironment) { @@ -110,34 +119,50 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.emptyObject(); } - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { - return new StagingConsumerFactory().create( - outputRecordCollector, - getDatabase(getDataSource(config)), - new SnowflakeInternalStagingSqlOperations(getNamingResolver()), - getNamingResolver(), - CsvSerializedBuffer.createFunction(null, () -> new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX, getNumberOfFileBuffers(config))), - config, - catalog, - true); - } - @Override public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) { + final String defaultNamespace = config.get("schema").asText(); + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + if (StringUtils.isEmpty(stream.getStream().getNamespace())) { + stream.getStream().setNamespace(defaultNamespace); + } + } + + final SnowflakeSqlGenerator sqlGenerator = new SnowflakeSqlGenerator(); + final ParsedCatalog parsedCatalog; + final TyperDeduper typerDeduper; + final JdbcDatabase database = getDatabase(getDataSource(config)); + if (TypingAndDedupingFlag.isDestinationV2()) { + final String databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + final SnowflakeDestinationHandler snowflakeDestinationHandler = new SnowflakeDestinationHandler(databaseName, database); + final CatalogParser catalogParser; + if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).isPresent()) { + catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get()); + } else { + catalogParser = new CatalogParser(sqlGenerator); + } + parsedCatalog = catalogParser.parseCatalog(catalog); + final SnowflakeV1V2Migrator migrator = new SnowflakeV1V2Migrator(getNamingResolver(), database, databaseName); + typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator); + } else { + parsedCatalog = null; + typerDeduper = new NoopTyperDeduper(); + } + return new StagingConsumerFactory().createAsync( outputRecordCollector, - getDatabase(getDataSource(config)), + database, new SnowflakeInternalStagingSqlOperations(getNamingResolver()), getNamingResolver(), - CsvSerializedBuffer.createFunction(null, () -> new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX, getNumberOfFileBuffers(config))), config, catalog, - true); + true, + new TypeAndDedupeOperationValve(), + typerDeduper, + parsedCatalog, + defaultNamespace); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java index 76617f0f8af7..2391ef11ac5e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.string.Strings; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import java.io.IOException; @@ -27,24 +28,52 @@ public class SnowflakeInternalStagingSqlOperations extends SnowflakeSqlStagingOp "CREATE STAGE IF NOT EXISTS %s encryption = (type = 'SNOWFLAKE_SSE') copy_options = (on_error='skip_file');"; private static final String PUT_FILE_QUERY = "PUT file://%s @%s/%s PARALLEL = %d;"; private static final String LIST_STAGE_QUERY = "LIST @%s/%s/%s;"; - private static final String COPY_QUERY = "COPY INTO %s.%s FROM '@%s/%s' " - + "file_format = (type = csv compression = auto field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"')"; + // the 1s1t copy query explicitly quotes the raw table+schema name. + private static final String COPY_QUERY_1S1T = + """ + COPY INTO "%s"."%s" FROM '@%s/%s' + file_format = ( + type = csv + compression = auto + field_delimiter = ',' + skip_header = 0 + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + NULL_IF=('') + )"""; + private static final String COPY_QUERY = """ + COPY INTO %s.%s FROM '@%s/%s' + file_format = ( + type = csv + compression = auto + field_delimiter = ',' + skip_header = 0 + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + NULL_IF=('') + )"""; private static final String DROP_STAGE_QUERY = "DROP STAGE IF EXISTS %s;"; private static final String REMOVE_QUERY = "REMOVE @%s;"; private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeSqlOperations.class); private final NamingConventionTransformer nameTransformer; + private final boolean use1s1t; public SnowflakeInternalStagingSqlOperations(final NamingConventionTransformer nameTransformer) { this.nameTransformer = nameTransformer; + this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); } @Override public String getStageName(final String namespace, final String streamName) { - return nameTransformer.applyDefaultCase(String.join("_", - nameTransformer.convertStreamName(namespace), - nameTransformer.convertStreamName(streamName))); + if (use1s1t) { + return String.join(".", + '"' + nameTransformer.convertStreamName(namespace) + '"', + '"' + nameTransformer.convertStreamName(streamName) + '"'); + } else { + return nameTransformer.applyDefaultCase(String.join(".", + nameTransformer.convertStreamName(namespace), + nameTransformer.convertStreamName(streamName))); + } } @Override @@ -163,7 +192,7 @@ public void copyIntoTableFromStage(final JdbcDatabase database, final String query = getCopyQuery(stageName, stagingPath, stagedFiles, tableName, schemaName); LOGGER.debug("Executing query: {}", query); database.execute(query); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } @@ -184,7 +213,11 @@ protected String getCopyQuery(final String stageName, final List stagedFiles, final String dstTableName, final String schemaName) { - return String.format(COPY_QUERY + generateFilesList(stagedFiles) + ";", schemaName, dstTableName, stageName, stagingPath); + if (use1s1t) { + return String.format(COPY_QUERY_1S1T + generateFilesList(stagedFiles) + ";", schemaName, dstTableName, stageName, stagingPath); + } else { + return String.format(COPY_QUERY + generateFilesList(stagedFiles) + ";", schemaName, dstTableName, stageName, stagingPath); + } } @Override @@ -193,7 +226,7 @@ public void dropStageIfExists(final JdbcDatabase database, final String stageNam final String query = getDropQuery(stageName); LOGGER.debug("Executing query: {}", query); database.execute(query); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } @@ -214,7 +247,7 @@ public void cleanUpStage(final JdbcDatabase database, final String stageName, fi final String query = getRemoveQuery(stageName); LOGGER.debug("Executing query: {}", query); database.execute(query); - } catch (SQLException e) { + } catch (final SQLException e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeParallelCopyStreamCopier.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeParallelCopyStreamCopier.java deleted file mode 100644 index efdb132c4b33..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeParallelCopyStreamCopier.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import java.util.List; -import java.util.StringJoiner; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -interface SnowflakeParallelCopyStreamCopier { - - /** - * Generates list of staging files. See more - * https://docs.snowflake.com/en/user-guide/data-load-considerations-load.html#lists-of-files - */ - default String generateFilesList(List files) { - StringJoiner joiner = new StringJoiner(","); - files.forEach(filename -> joiner.add("'" + filename.substring(filename.lastIndexOf("/") + 1) + "'")); - return joiner.toString(); - } - - /** - * Executes async copying of staging files.This method should block until the copy/upload has - * completed. - */ - default void copyFilesInParallel(List> partitions) { - ExecutorService executorService = Executors.newFixedThreadPool(5); - List> futures = partitions.stream() - .map(partition -> CompletableFuture.runAsync(() -> copyIntoStage(partition), executorService)) - .collect(Collectors.toList()); - - try { - // wait until all futures ready - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); - } catch (Exception e) { - throw new RuntimeException("Failed to copy files from stage to tmp table {}" + e); - } finally { - executorService.shutdown(); - } - } - - /** - * Copies staging files to the temporary table using statement - */ - void copyIntoStage(List files); - - /** - * Generates full bucket/container path to staging files - */ - String generateBucketPath(); - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingDestination.java deleted file mode 100644 index 105bb47fe50a..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingDestination.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static io.airbyte.integrations.destination.snowflake.SnowflakeDestinationResolver.getNumberOfFileBuffers; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryption; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryption.KeyType; -import io.airbyte.integrations.destination.s3.EncryptionConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; -import io.airbyte.integrations.destination.staging.StagingConsumerFactory; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.util.Collections; -import java.util.Map; -import java.util.UUID; -import java.util.function.Consumer; -import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowflakeS3StagingDestination extends AbstractJdbcDestination implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeS3StagingDestination.class); - private final String airbyteEnvironment; - - public SnowflakeS3StagingDestination(final String airbyteEnvironment) { - this(new SnowflakeSQLNameTransformer(), airbyteEnvironment); - } - - public SnowflakeS3StagingDestination(final SnowflakeSQLNameTransformer nameTransformer, final String airbyteEnvironment) { - super("", nameTransformer, new SnowflakeSqlOperations()); - this.airbyteEnvironment = airbyteEnvironment; - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) { - final S3DestinationConfig s3Config = getS3DestinationConfig(config); - final EncryptionConfig encryptionConfig = EncryptionConfig.fromJson(config.get("loading_method").get("encryption")); - if (!isPurgeStagingData(config) && encryptionConfig instanceof AesCbcEnvelopeEncryption c && c.keyType() == KeyType.EPHEMERAL) { - return new AirbyteConnectionStatus() - .withStatus(Status.FAILED) - .withMessage( - "You cannot use ephemeral keys and disable purging your staging data. This would produce S3 objects that you cannot decrypt."); - } - final NamingConventionTransformer nameTransformer = getNamingResolver(); - final SnowflakeS3StagingSqlOperations snowflakeS3StagingSqlOperations = - new SnowflakeS3StagingSqlOperations(nameTransformer, s3Config.getS3Client(), s3Config, encryptionConfig); - final DataSource dataSource = getDataSource(config); - try { - final JdbcDatabase database = getDatabase(dataSource); - final String outputSchema = super.getNamingResolver().getIdentifier(config.get("schema").asText()); - attemptTableOperations(outputSchema, database, nameTransformer, snowflakeS3StagingSqlOperations, - true); - attemptStageOperations(outputSchema, database, nameTransformer, snowflakeS3StagingSqlOperations); - return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); - } catch (final Exception e) { - LOGGER.error("Exception while checking connection: ", e); - return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED) - .withMessage("Could not connect with provided configuration. \n" + e.getMessage()); - } finally { - try { - DataSourceFactory.close(dataSource); - } catch (final Exception e) { - LOGGER.warn("Unable to close data source.", e); - } - } - } - - private static void attemptStageOperations(final String outputSchema, - final JdbcDatabase database, - final NamingConventionTransformer namingResolver, - final SnowflakeS3StagingSqlOperations sqlOperations) - throws Exception { - - // verify we have permissions to create/drop stage - final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID()); - final String stageName = sqlOperations.getStageName(outputSchema, outputTableName); - sqlOperations.createStageIfNotExists(database, stageName); - - // try to make test write to make sure we have required role - try { - sqlOperations.attemptWriteToStage(outputSchema, stageName, database); - } finally { - // drop created tmp stage - sqlOperations.dropStageIfExists(database, stageName); - } - } - - @Override - protected DataSource getDataSource(final JsonNode config) { - return SnowflakeDatabase.createDataSource(config, airbyteEnvironment); - } - - @Override - protected JdbcDatabase getDatabase(final DataSource dataSource) { - return SnowflakeDatabase.getDatabase(dataSource); - } - - @Override - protected Map getDefaultConnectionProperties(final JsonNode config) { - return Collections.emptyMap(); - } - - // this is a no op since we override getDatabase. - @Override - public JsonNode toJdbcConfig(final JsonNode config) { - return Jsons.emptyObject(); - } - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { - final S3DestinationConfig s3Config = getS3DestinationConfig(config); - final EncryptionConfig encryptionConfig = EncryptionConfig.fromJson(config.get("loading_method").get("encryption")); - return new StagingConsumerFactory().create( - outputRecordCollector, - getDatabase(getDataSource(config)), - new SnowflakeS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), - getNamingResolver(), - CsvSerializedBuffer.createFunction(null, () -> new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX, getNumberOfFileBuffers(config))), - config, - catalog, - isPurgeStagingData(config)); - } - - private S3DestinationConfig getS3DestinationConfig(final JsonNode config) { - final JsonNode loadingMethod = config.get("loading_method"); - return S3DestinationConfig.getS3DestinationConfig(loadingMethod); - } - - public static boolean isPurgeStagingData(final JsonNode config) { - final JsonNode loadingMethod = config.get("loading_method"); - if (!loadingMethod.has("purge_staging_data")) { - return true; - } else { - return loadingMethod.get("purge_staging_data").asBoolean(); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java deleted file mode 100644 index d36ad77b8cec..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperations.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.commons.lang.Exceptions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryption; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryptionBlobDecorator; -import io.airbyte.integrations.destination.s3.EncryptionConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import java.util.Base64; -import java.util.Base64.Encoder; -import java.util.List; -import java.util.UUID; -import org.joda.time.DateTime; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowflakeS3StagingSqlOperations extends SnowflakeSqlStagingOperations { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeSqlOperations.class); - private static final Encoder BASE64_ENCODER = Base64.getEncoder(); - private static final String COPY_QUERY = "COPY INTO %s.%s FROM '%s' " - + "CREDENTIALS=(aws_key_id='%s' aws_secret_key='%s') " - + "file_format = (type = csv compression = auto field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"')"; - - private final NamingConventionTransformer nameTransformer; - private final S3StorageOperations s3StorageOperations; - private final S3DestinationConfig s3Config; - private final byte[] keyEncryptingKey; - - public SnowflakeS3StagingSqlOperations(final NamingConventionTransformer nameTransformer, - final AmazonS3 s3Client, - final S3DestinationConfig s3Config, - final EncryptionConfig encryptionConfig) { - this.nameTransformer = nameTransformer; - this.s3StorageOperations = new S3StorageOperations(nameTransformer, s3Client, s3Config); - this.s3Config = s3Config; - if (encryptionConfig instanceof AesCbcEnvelopeEncryption e) { - this.s3StorageOperations.addBlobDecorator(new AesCbcEnvelopeEncryptionBlobDecorator(e.key())); - this.keyEncryptingKey = e.key(); - } else { - this.keyEncryptingKey = null; - } - } - - @Override - public String getStageName(final String namespace, final String streamName) { - return nameTransformer.applyDefaultCase(String.join("_", - nameTransformer.convertStreamName(namespace), - nameTransformer.convertStreamName(streamName))); - } - - @Override - public String getStagingPath(final UUID connectionId, final String namespace, final String streamName, final DateTime writeDatetime) { - // see https://docs.snowflake.com/en/user-guide/data-load-considerations-stage.html - return nameTransformer.applyDefaultCase(String.format("%s/%s/%02d/%02d/%02d/%s/", - getStageName(namespace, streamName), - writeDatetime.year().get(), - writeDatetime.monthOfYear().get(), - writeDatetime.dayOfMonth().get(), - writeDatetime.hourOfDay().get(), - connectionId)); - } - - @Override - public String uploadRecordsToStage(final JdbcDatabase database, - final SerializableBuffer recordsData, - final String schemaName, - final String stageName, - final String stagingPath) { - return s3StorageOperations.uploadRecordsToBucket(recordsData, schemaName, stageName, stagingPath); - } - - @Override - public void createStageIfNotExists(final JdbcDatabase database, final String stageName) { - s3StorageOperations.createBucketIfNotExists(); - } - - @Override - public void copyIntoTableFromStage(final JdbcDatabase database, - final String stageName, - final String stagingPath, - final List stagedFiles, - final String tableName, - final String schemaName) { - LOGGER.info("Starting copy to target table from stage: {} in destination from stage: {}, schema: {}, .", - tableName, stagingPath, schemaName); - // Print actual SQL query if user needs to manually force reload from staging - Exceptions.toRuntime(() -> database.execute(getCopyQuery(stagingPath, stagedFiles, - tableName, schemaName))); - LOGGER.info("Copy to target table {}.{} in destination complete.", schemaName, tableName); - } - - protected String getCopyQuery(final String stagingPath, - final List stagedFiles, - final String dstTableName, - final String schemaName) { - final S3AccessKeyCredentialConfig credentialConfig = (S3AccessKeyCredentialConfig) s3Config.getS3CredentialConfig(); - final String encryptionClause; - if (keyEncryptingKey == null) { - encryptionClause = ""; - } else { - encryptionClause = String.format(" encryption = (type = 'aws_cse' master_key = '%s')", BASE64_ENCODER.encodeToString(keyEncryptingKey)); - } - return String.format(COPY_QUERY + generateFilesList(stagedFiles) + encryptionClause + ";", - schemaName, - dstTableName, - generateBucketPath(stagingPath), - credentialConfig.getAccessKeyId(), - credentialConfig.getSecretAccessKey()); - } - - private String generateBucketPath(final String stagingPath) { - return "s3://" + s3Config.getBucketName() + "/" + stagingPath; - } - - @Override - public void dropStageIfExists(final JdbcDatabase database, final String stageName) { - s3StorageOperations.dropBucketObject(stageName); - } - - @Override - public void cleanUpStage(final JdbcDatabase database, final String stageName, final List stagedFiles) { - s3StorageOperations.cleanUpBucketObject(stageName, stagedFiles); - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java deleted file mode 100644 index c689b62bd1db..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopier.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import com.amazonaws.services.s3.AmazonS3; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Lists; -import io.airbyte.commons.lang.Exceptions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3StreamCopier; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.util.S3OutputPathHelper; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowflakeS3StreamCopier extends S3StreamCopier implements SnowflakeParallelCopyStreamCopier { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeS3StreamCopier.class); - - // From https://docs.aws.amazon.com/redshift/latest/dg/t_loading-tables-from-s3.html - // "Split your load data files so that the files are about equal size, between 1 MB and 1 GB after - // compression" - public static final int MAX_PARTS_PER_FILE = 4; - public static final int MAX_FILES_PER_COPY = 1000; - - public SnowflakeS3StreamCopier(final String stagingFolder, - final String schema, - final AmazonS3 client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final ConfiguredAirbyteStream configuredAirbyteStream) { - this( - stagingFolder, - schema, - client, - db, - config, - nameTransformer, - sqlOperations, - Timestamp.from(Instant.now()), - configuredAirbyteStream); - } - - @VisibleForTesting - SnowflakeS3StreamCopier(final String stagingFolder, - final String schema, - final AmazonS3 client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final Timestamp uploadTime, - final ConfiguredAirbyteStream configuredAirbyteStream) { - - super(stagingFolder, - schema, - client, - db, - config, - nameTransformer, - sqlOperations, - configuredAirbyteStream, - uploadTime, - MAX_PARTS_PER_FILE); - } - - @Override - public void copyStagingFileToTemporaryTable() throws Exception { - final List> partitions = Lists.partition(new ArrayList<>(getStagingFiles()), MAX_FILES_PER_COPY); - LOGGER.info("Starting parallel copy to tmp table: {} in destination for stream: {}, schema: {}. Chunks count {}", tmpTableName, streamName, - schemaName, partitions.size()); - - copyFilesInParallel(partitions); - LOGGER.info("Copy to tmp table {} in destination for stream {} complete.", tmpTableName, streamName); - } - - @Override - public void copyIntoStage(final List files) { - final S3AccessKeyCredentialConfig credentialConfig = (S3AccessKeyCredentialConfig) s3Config.getS3CredentialConfig(); - final var copyQuery = String.format( - "COPY INTO %s.%s FROM '%s' " - + "CREDENTIALS=(aws_key_id='%s' aws_secret_key='%s') " - + "file_format = (type = csv field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') " - + "files = (" + generateFilesList(files) + " );", - schemaName, - tmpTableName, - generateBucketPath(), - credentialConfig.getAccessKeyId(), - credentialConfig.getSecretAccessKey()); - - Exceptions.toRuntime(() -> db.execute(copyQuery)); - } - - @Override - public String generateBucketPath() { - return "s3://" + s3Config.getBucketName() + "/" - + S3OutputPathHelper.getOutputPrefix(s3Config.getBucketPath(), configuredAirbyteStream.getStream()) + "/"; - } - - @Override - public void copyS3CsvFileIntoTable(final JdbcDatabase database, - final String s3FileLocation, - final String schema, - final String tableName, - final S3DestinationConfig s3Config) - throws SQLException { - throw new RuntimeException("Snowflake Stream Copier should not copy individual files without use of a parallel copy"); - - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierFactory.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierFactory.java deleted file mode 100644 index 4ccbb29053c9..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3StreamCopierFactory; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; - -public class SnowflakeS3StreamCopierFactory extends S3StreamCopierFactory { - - @Override - protected StreamCopier create(final String stagingFolder, - final String schema, - final AmazonS3 s3Client, - final JdbcDatabase db, - final S3CopyConfig config, - final StandardNameTransformer nameTransformer, - final SqlOperations sqlOperations, - final ConfiguredAirbyteStream configuredStream) - throws Exception { - return new SnowflakeS3StreamCopier(stagingFolder, schema, s3Client, db, config, nameTransformer, - sqlOperations, configuredStream); - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java index 96cf77dcc7b1..5a517d17c715 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java @@ -8,6 +8,7 @@ import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.integrations.destination.jdbc.SqlOperations; import io.airbyte.integrations.destination.jdbc.SqlOperationsUtils; @@ -31,26 +32,89 @@ class SnowflakeSqlOperations extends JdbcSqlOperations implements SqlOperations private static final String NO_PRIVILEGES_ERROR_MESSAGE = "but current role has no privileges on it"; private static final String IP_NOT_IN_WHITE_LIST_ERR_MSG = "not allowed to access Snowflake"; + private final boolean use1s1t; + + public SnowflakeSqlOperations() { + this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); + } + + @Override + public void createSchemaIfNotExists(final JdbcDatabase database, final String schemaName) throws Exception { + try { + if (!schemaSet.contains(schemaName) && !isSchemaExists(database, schemaName)) { + if (use1s1t) { + // 1s1t is assuming a lowercase airbyte_internal schema name, so we need to quote it + database.execute(String.format("CREATE SCHEMA IF NOT EXISTS \"%s\";", schemaName)); + } else { + database.execute(String.format("CREATE SCHEMA IF NOT EXISTS %s;", schemaName)); + } + schemaSet.add(schemaName); + } + } catch (final Exception e) { + throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); + } + } + @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { - return String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s VARCHAR PRIMARY KEY,\n" - + "%s VARIANT,\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp()\n" - + ") data_retention_time_in_days = 0;", - schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + if (use1s1t) { + return String.format( + """ + CREATE TABLE IF NOT EXISTS "%s"."%s" ( + "%s" VARCHAR PRIMARY KEY, + "%s" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), + "%s" TIMESTAMP WITH TIME ZONE DEFAULT NULL, + "%s" VARIANT + ) data_retention_time_in_days = 0;""", + schemaName, + tableName, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + JavaBaseConstants.COLUMN_NAME_DATA); + } else { + return String.format( + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s VARIANT, + %s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp() + ) data_retention_time_in_days = 0;""", + schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + } } @Override public boolean isSchemaExists(final JdbcDatabase database, final String outputSchema) throws Exception { try (final Stream results = database.unsafeQuery(SHOW_SCHEMAS)) { - return results.map(schemas -> schemas.get(NAME).asText()).anyMatch(outputSchema::equalsIgnoreCase); - } catch (Exception e) { + if (use1s1t) { + return results.map(schemas -> schemas.get(NAME).asText()).anyMatch(outputSchema::equals); + } else { + return results.map(schemas -> schemas.get(NAME).asText()).anyMatch(outputSchema::equalsIgnoreCase); + } + } catch (final Exception e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } } + @Override + public String truncateTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { + if (use1s1t) { + return String.format("TRUNCATE TABLE \"%s\".\"%s\";\n", schemaName, tableName); + } else { + return String.format("TRUNCATE TABLE %s.%s;\n", schemaName, tableName); + } + } + + @Override + public String dropTableIfExistsQuery(final String schemaName, final String tableName) { + if (use1s1t) { + return String.format("DROP TABLE IF EXISTS \"%s\".\"%s\";\n", schemaName, tableName); + } else { + return String.format("DROP TABLE IF EXISTS %s.%s;\n", schemaName, tableName); + } + } + @Override public void insertRecordsInternal(final JdbcDatabase database, final List records, @@ -65,9 +129,19 @@ public void insertRecordsInternal(final JdbcDatabase database, // FROM VALUES // (?, ?, ?), // ... - final String insertQuery = String.format( - "INSERT INTO %s.%s (%s, %s, %s) SELECT column1, parse_json(column2), column3 FROM VALUES\n", - schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + final String insertQuery; + if (use1s1t) { + // Note that the column order is weird here - that's intentional, to avoid needing to change + // SqlOperationsUtils.insertRawRecordsInSingleQuery to support a different column order. + insertQuery = String.format( + "INSERT INTO \"%s\".\"%s\" (\"%s\", \"%s\", \"%s\") SELECT column1, parse_json(column2), column3 FROM VALUES\n", + schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); + } else { + insertQuery = String.format( + "INSERT INTO %s.%s (%s, %s, %s) SELECT column1, parse_json(column2), column3 FROM VALUES\n", + schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + } final String recordQuery = "(?, ?, ?),\n"; SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQuery, recordQuery, database, records); } @@ -84,7 +158,7 @@ protected String generateFilesList(final List files) { } @Override - protected Optional checkForKnownConfigExceptions(Exception e) { + protected Optional checkForKnownConfigExceptions(final Exception e) { if (e instanceof SnowflakeSQLException && e.getMessage().contains(NO_PRIVILEGES_ERROR_MESSAGE)) { return Optional.of(new ConfigErrorException( "Encountered Error with Snowflake Configuration: Current role does not have permissions on the target schema please verify your privileges", diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumn.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumn.java new file mode 100644 index 000000000000..8415fedf587c --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumn.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +/** + * type is notably _not_ a {@link net.snowflake.client.jdbc.SnowflakeType}. That enum doesn't + * contain all the types that snowflake supports (specifically NUMBER). + */ +public record SnowflakeColumn(String name, String type) {} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java new file mode 100644 index 000000000000..ba8794927743 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Optional; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SnowflakeDestinationHandler implements DestinationHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDestinationHandler.class); + + private final String databaseName; + private final JdbcDatabase database; + + public SnowflakeDestinationHandler(String databaseName, JdbcDatabase database) { + this.databaseName = databaseName; + this.database = database; + } + + @Override + public Optional findExistingTable(StreamId id) throws SQLException { + // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates + // VARIANT as VARCHAR + LinkedHashMap columns = database.queryJsons( + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_catalog = ? + AND table_schema = ? + AND table_name = ? + ORDER BY ordinal_position; + """, + databaseName, + id.finalNamespace(), + id.finalName()).stream() + .collect(LinkedHashMap::new, + (map, row) -> map.put(row.get("COLUMN_NAME").asText(), row.get("DATA_TYPE").asText()), + LinkedHashMap::putAll); + // TODO query for indexes/partitioning/etc + + if (columns.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(new SnowflakeTableDefinition(columns)); + } + } + + @Override + public boolean isFinalTableEmpty(StreamId id) throws SQLException { + int rowCount = database.queryInt( + """ + SELECT row_count + FROM information_schema.tables + WHERE table_catalog = ? + AND table_schema = ? + AND table_name = ? + """, + databaseName, + id.finalNamespace(), + id.finalName()); + return rowCount == 0; + } + + @Override + public void execute(String sql) throws Exception { + if ("".equals(sql)) { + return; + } + final UUID queryId = UUID.randomUUID(); + LOGGER.info("Executing sql {}: {}", queryId, sql); + long startTime = System.currentTimeMillis(); + + database.execute(sql); + + LOGGER.info("Sql {} completed in {} ms", queryId, System.currentTimeMillis() - startTime); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java new file mode 100644 index 000000000000..5f77e4445447 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java @@ -0,0 +1,532 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import static java.util.stream.Collectors.joining; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.TableNotMigratedException; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringSubstitutor; + +public class SnowflakeSqlGenerator implements SqlGenerator { + + public static final String QUOTE = "\""; + + private final ColumnId CDC_DELETED_AT_COLUMN = buildColumnId("_ab_cdc_deleted_at"); + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + // No escaping needed, as far as I can tell. We quote all our identifier names. + return new StreamId( + escapeIdentifier(namespace), + escapeIdentifier(name), + escapeIdentifier(rawNamespaceOverride), + escapeIdentifier(StreamId.concatenateRawTableName(namespace, name)), + namespace, + name); + } + + @Override + public ColumnId buildColumnId(final String name) { + // No escaping needed, as far as I can tell. We quote all our identifier names. + return new ColumnId(escapeIdentifier(name), name, name); + } + + public String toDialectType(final AirbyteType type) { + if (type instanceof final AirbyteProtocolType p) { + return toDialectType(p); + } else if (type instanceof Struct) { + // TODO should this+array just be VARIANT? + return "OBJECT"; + } else if (type instanceof Array) { + return "ARRAY"; + } else if (type instanceof UnsupportedOneOf) { + return "VARIANT"; + } else if (type instanceof final Union u) { + final AirbyteType typeWithPrecedence = u.chooseType(); + // typeWithPrecedence is never a Union, so this recursion is safe. + return toDialectType(typeWithPrecedence); + } + + // Literally impossible; AirbyteType is a sealed interface. + throw new IllegalArgumentException("Unsupported AirbyteType: " + type); + } + + public String toDialectType(final AirbyteProtocolType airbyteProtocolType) { + // TODO verify these types against normalization + return switch (airbyteProtocolType) { + case STRING -> "TEXT"; + case NUMBER -> "FLOAT"; + case INTEGER -> "NUMBER"; + case BOOLEAN -> "BOOLEAN"; + case TIMESTAMP_WITH_TIMEZONE -> "TIMESTAMP_TZ"; + case TIMESTAMP_WITHOUT_TIMEZONE -> "TIMESTAMP_NTZ"; + // If you change this - also change the logic in extractAndCast + case TIME_WITH_TIMEZONE -> "TEXT"; + case TIME_WITHOUT_TIMEZONE -> "TIME"; + case DATE -> "DATE"; + case UNKNOWN -> "VARIANT"; + }; + } + + @Override + public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + final String columnDeclarations = stream.columns().entrySet().stream() + .map(column -> "," + column.getKey().name(QUOTE) + " " + toDialectType(column.getValue())) + .collect(joining("\n")); + final String forceCreateTable = force ? "OR REPLACE" : ""; + + return new StringSubstitutor(Map.of( + "final_namespace", stream.id().finalNamespace(QUOTE), + "final_table_id", stream.id().finalTableId(QUOTE, suffix), + "force_create_table", forceCreateTable, + "column_declarations", columnDeclarations)).replace( + """ + CREATE SCHEMA IF NOT EXISTS ${final_namespace}; + + CREATE ${force_create_table} TABLE ${final_table_id} ( + "_airbyte_raw_id" TEXT NOT NULL, + "_airbyte_extracted_at" TIMESTAMP_TZ NOT NULL, + "_airbyte_meta" VARIANT NOT NULL + ${column_declarations} + ); + """); + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final SnowflakeTableDefinition existingTable) + throws TableNotMigratedException { + + // Check that the columns match, with special handling for the metadata columns. + final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey().name(), toDialectType(column.getValue())), + LinkedHashMap::putAll); + final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() + .filter(column -> !JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.contains(column.getKey())) + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey(), column.getValue()), + LinkedHashMap::putAll); + final boolean sameColumns = actualColumns.equals(intendedColumns) + && "TEXT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID)) + && "TIMESTAMP_TZ".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)) + && "VARIANT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META)); + + return sameColumns; + } + + @Override + public String updateTable(final StreamConfig stream, final String finalSuffix) { + return updateTable(stream, finalSuffix, true); + } + + private String updateTable(final StreamConfig stream, final String finalSuffix, final boolean verifyPrimaryKeys) { + String validatePrimaryKeys = ""; + if (verifyPrimaryKeys && stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + validatePrimaryKeys = validatePrimaryKeys(stream.id(), stream.primaryKey(), stream.columns()); + } + final String insertNewRecords = insertNewRecords(stream, finalSuffix, stream.columns()); + String dedupFinalTable = ""; + String cdcDeletes = ""; + String dedupRawTable = ""; + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + dedupRawTable = dedupRawTable(stream.id(), finalSuffix); + // If we're in dedup mode, then we must have a cursor + dedupFinalTable = dedupFinalTable(stream.id(), finalSuffix, stream.primaryKey(), stream.cursor()); + cdcDeletes = cdcDeletes(stream, finalSuffix, stream.columns()); + } + final String commitRawTable = commitRawTable(stream.id()); + + return new StringSubstitutor(Map.of( + "validate_primary_keys", validatePrimaryKeys, + "insert_new_records", insertNewRecords, + "dedup_final_table", dedupFinalTable, + "cdc_deletes", cdcDeletes, + "dedupe_raw_table", dedupRawTable, + "commit_raw_table", commitRawTable)).replace( + """ + BEGIN TRANSACTION; + ${validate_primary_keys} + ${insert_new_records} + ${dedup_final_table} + ${dedupe_raw_table} + ${cdc_deletes} + ${commit_raw_table} + COMMIT; + """); + } + + private String extractAndCast(final ColumnId column, final AirbyteType airbyteType) { + if (airbyteType instanceof final Union u) { + // This is guaranteed to not be a Union, so we won't recurse infinitely + final AirbyteType chosenType = u.chooseType(); + return extractAndCast(column, chosenType); + } else if (airbyteType == AirbyteProtocolType.TIME_WITH_TIMEZONE) { + // We're using TEXT for this type, so need to explicitly check the string format. + // There's a bunch of ways we could do this; this regex is approximately correct and easy to + // implement. + // It'll match anything like HH:MM:SS[.SSS](Z|[+-]HH[:]MM), e.g.: + // 12:34:56Z + // 12:34:56.7+08:00 + // 12:34:56.7890123-0800 + // 12:34:56-08 + return new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + """ + CASE + WHEN NOT ("_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{1,2}:\\\\d{2}:\\\\d{2}(\\\\.\\\\d+)?(Z|[+\\\\-]\\\\d{1,2}(:?\\\\d{2})?)') + THEN NULL + ELSE "_airbyte_data":"${column_name}" + END + """); + } else { + final String dialectType = toDialectType(airbyteType); + return switch (dialectType) { + case "TIMESTAMP_TZ" -> new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + // Handle offsets in +/-HHMM and +/-HH formats + // The four cases, in order, match: + // 2023-01-01T12:34:56-0800 + // 2023-01-01T12:34:56-08 + // 2023-01-01T12:34:56.7890123-0800 + // 2023-01-01T12:34:56.7890123-08 + // And the ELSE will try to handle everything else. + """ + CASE + WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}(\\\\+|-)\\\\d{4}' + THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}(\\\\+|-)\\\\d{2}' + THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SSTZH') + WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}\\\\.\\\\d{1,7}(\\\\+|-)\\\\d{4}' + THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}\\\\.\\\\d{1,7}(\\\\+|-)\\\\d{2}' + THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + ELSE TRY_CAST("_airbyte_data":"${column_name}"::TEXT AS TIMESTAMP_TZ) + END + """); + // try_cast doesn't support variant/array/object, so handle them specially + case "VARIANT" -> "\"_airbyte_data\":\"" + escapeIdentifier(column.originalName()) + "\""; + // We need to validate that the struct is actually a struct. + // Note that struct columns are actually nullable in two ways. For a column `foo`: + // {foo: null} and {} are both valid, and are both written to the final table as a SQL NULL (_not_ a + // JSON null). + case "OBJECT" -> new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + """ + CASE + WHEN TYPEOF("_airbyte_data":"${column_name}") != 'OBJECT' + THEN NULL + ELSE "_airbyte_data":"${column_name}" + END + """); + // Much like the object case, arrays need special handling. + case "ARRAY" -> new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + """ + CASE + WHEN TYPEOF("_airbyte_data":"${column_name}") != 'ARRAY' + THEN NULL + ELSE "_airbyte_data":"${column_name}" + END + """); + default -> "TRY_CAST(\"_airbyte_data\":\"" + escapeIdentifier(column.originalName()) + "\"::text as " + dialectType + ")"; + }; + } + } + + @VisibleForTesting + String validatePrimaryKeys(final StreamId id, + final List primaryKeys, + final LinkedHashMap streamColumns) { + if (primaryKeys.stream().anyMatch(c -> c.originalName().contains("`"))) { + // TODO why is snowflake throwing a bizarre error when we try to use a column with a backtick in it? + // E.g. even this trivial procedure fails: (it should return the string `'foo`bar') + // execute immediate 'BEGIN RETURN \'foo`bar\'; END;' + return ""; + } + + final String pkNullChecks = primaryKeys.stream().map( + pk -> { + final String jsonExtract = extractAndCast(pk, streamColumns.get(pk)); + return "AND " + jsonExtract + " IS NULL"; + }).collect(joining("\n")); + + final String script = new StringSubstitutor(Map.of( + "raw_table_id", id.rawTableId(QUOTE), + "pk_null_checks", pkNullChecks)).replace( + // Wrap this inside a script block so that we can use the scripting language + """ + BEGIN + LET missing_pk_count INTEGER := ( + SELECT COUNT(1) + FROM ${raw_table_id} + WHERE + "_airbyte_loaded_at" IS NULL + ${pk_null_checks} + ); + + IF (missing_pk_count > 0) THEN + RAISE STATEMENT_ERROR; + END IF; + RETURN 'SUCCESS'; + END; + """); + return "EXECUTE IMMEDIATE '" + escapeSingleQuotedString(script) + "';"; + } + + @VisibleForTesting + String insertNewRecords(final StreamConfig stream, final String finalSuffix, final LinkedHashMap streamColumns) { + final String columnCasts = streamColumns.entrySet().stream().map( + col -> extractAndCast(col.getKey(), col.getValue()) + " as " + col.getKey().name(QUOTE) + ",") + .collect(joining("\n")); + final String columnErrors = streamColumns.entrySet().stream().map( + col -> new StringSubstitutor(Map.of( + "raw_col_name", escapeIdentifier(col.getKey().originalName()), + "printable_col_name", escapeSingleQuotedString(col.getKey().originalName()), + "col_type", toDialectType(col.getValue()), + "json_extract", extractAndCast(col.getKey(), col.getValue()))).replace( + // TYPEOF returns "NULL_VALUE" for a JSON null and "NULL" for a SQL null + """ + CASE + WHEN (TYPEOF("_airbyte_data":"${raw_col_name}") NOT IN ('NULL', 'NULL_VALUE')) + AND (${json_extract} IS NULL) + THEN ['Problem with `${printable_col_name}`'] + ELSE [] + END""")) + .reduce( + "ARRAY_CONSTRUCT()", + (acc, col) -> "ARRAY_CAT(" + acc + ", " + col + ")"); + final String columnList = streamColumns.keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + + String cdcConditionalOrIncludeStatement = ""; + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { + cdcConditionalOrIncludeStatement = """ + OR ( + "_airbyte_loaded_at" IS NOT NULL + AND TYPEOF("_airbyte_data":"_ab_cdc_deleted_at") NOT IN ('NULL', 'NULL_VALUE') + ) + """; + } + + return new StringSubstitutor(Map.of( + "raw_table_id", stream.id().rawTableId(QUOTE), + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), + "column_casts", columnCasts, + "column_errors", columnErrors, + "cdcConditionalOrIncludeStatement", cdcConditionalOrIncludeStatement, + "column_list", columnList)).replace( + """ + INSERT INTO ${final_table_id} + ( + ${column_list} + "_airbyte_meta", + "_airbyte_raw_id", + "_airbyte_extracted_at" + ) + WITH intermediate_data AS ( + SELECT + ${column_casts} + ${column_errors} as "_airbyte_cast_errors", + "_airbyte_raw_id", + "_airbyte_extracted_at" + FROM ${raw_table_id} + WHERE + "_airbyte_loaded_at" IS NULL + ${cdcConditionalOrIncludeStatement} + ) + SELECT + ${column_list} + OBJECT_CONSTRUCT('errors', "_airbyte_cast_errors") AS "_airbyte_meta", + "_airbyte_raw_id", + "_airbyte_extracted_at" + FROM intermediate_data;"""); + } + + @VisibleForTesting + String dedupFinalTable(final StreamId id, + final String finalSuffix, + final List primaryKey, + final Optional cursor) { + final String pkList = primaryKey.stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); + final String cursorOrderClause = cursor + .map(cursorId -> cursorId.name(QUOTE) + " DESC NULLS LAST,") + .orElse(""); + + return new StringSubstitutor(Map.of( + "final_table_id", id.finalTableId(QUOTE, finalSuffix), + "pk_list", pkList, + "cursor_order_clause", cursorOrderClause)).replace( + """ + DELETE FROM ${final_table_id} + WHERE "_airbyte_raw_id" IN ( + SELECT "_airbyte_raw_id" FROM ( + SELECT "_airbyte_raw_id", row_number() OVER ( + PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} "_airbyte_extracted_at" DESC + ) as row_number FROM ${final_table_id} + ) + WHERE row_number != 1 + ); + """); + } + + @VisibleForTesting + String cdcDeletes(final StreamConfig stream, + final String finalSuffix, + final LinkedHashMap streamColumns) { + + if (stream.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { + return ""; + } + + if (!streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { + return ""; + } + + final String pkList = stream.primaryKey().stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); + final String pkCasts = stream.primaryKey().stream().map(pk -> extractAndCast(pk, streamColumns.get(pk))).collect(joining(",\n")); + + // we want to grab IDs for deletion from the raw table (not the final table itself) to hand + // out-of-order record insertions after the delete has been registered + return new StringSubstitutor(Map.of( + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), + "raw_table_id", stream.id().rawTableId(QUOTE), + "pk_list", pkList, + "pk_extracts", pkCasts, + "quoted_cdc_delete_column", QUOTE + "_ab_cdc_deleted_at" + QUOTE)).replace( + """ + DELETE FROM ${final_table_id} + WHERE ARRAY_CONSTRUCT(${pk_list}) IN ( + SELECT ARRAY_CONSTRUCT( + ${pk_extracts} + ) + FROM ${raw_table_id} + WHERE "_airbyte_data":"_ab_cdc_deleted_at" != 'null' + ); + """); + } + + @VisibleForTesting + String dedupRawTable(final StreamId id, final String finalSuffix) { + return new StringSubstitutor(Map.of( + "raw_table_id", id.rawTableId(QUOTE), + "final_table_id", id.finalTableId(QUOTE, finalSuffix))).replace( + // Note that this leaves _all_ deletion records in the raw table. We _could_ clear them out, but it + // would be painful, + // and it only matters in a few edge cases. + """ + DELETE FROM ${raw_table_id} + WHERE "_airbyte_raw_id" NOT IN ( + SELECT "_airbyte_raw_id" FROM ${final_table_id} + ); + """); + } + + @VisibleForTesting + String commitRawTable(final StreamId id) { + return new StringSubstitutor(Map.of( + "raw_table_id", id.rawTableId(QUOTE))).replace( + """ + UPDATE ${raw_table_id} + SET "_airbyte_loaded_at" = CURRENT_TIMESTAMP() + WHERE "_airbyte_loaded_at" IS NULL + ;"""); + } + + @Override + public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return new StringSubstitutor(Map.of( + "final_table", stream.finalTableId(QUOTE), + "tmp_final_table", stream.finalTableId(QUOTE, finalSuffix))).replace( + """ + BEGIN TRANSACTION; + DROP TABLE IF EXISTS ${final_table}; + ALTER TABLE ${tmp_final_table} RENAME TO ${final_table}; + COMMIT; + """); + } + + @Override + public String softReset(final StreamConfig stream) { + final String createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); + final String clearLoadedAt = clearLoadedAt(stream.id()); + final String rebuildInTempTable = updateTable(stream, SOFT_RESET_SUFFIX, false); + final String overwriteFinalTable = overwriteFinalTable(stream.id(), SOFT_RESET_SUFFIX); + return String.join("\n", createTempTable, clearLoadedAt, rebuildInTempTable, overwriteFinalTable); + } + + private String clearLoadedAt(final StreamId streamId) { + return new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) + .replace(""" + UPDATE ${raw_table_id} SET "_airbyte_loaded_at" = NULL; + """); + } + + @Override + public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + // In the SQL below, the v2 values are quoted to preserve their case while the v1 values are + // intentionally _not_ quoted. This is to preserve the implicit upper-casing behavior in v1. + return new StringSubstitutor(Map.of( + "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), + "raw_table_name", streamId.rawTableId(QUOTE), + "raw_id", JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, + "extracted_at", JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + "loaded_at", JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + "data", JavaBaseConstants.COLUMN_NAME_DATA, + "v1_raw_id", JavaBaseConstants.COLUMN_NAME_AB_ID, + "emitted_at", JavaBaseConstants.COLUMN_NAME_EMITTED_AT, + "v1_raw_table", String.join(".", namespace, tableName))) + .replace( + """ + CREATE SCHEMA IF NOT EXISTS ${raw_namespace}; + + CREATE OR REPLACE TABLE ${raw_table_name} ( + "${raw_id}" VARCHAR PRIMARY KEY, + "${extracted_at}" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), + "${loaded_at}" TIMESTAMP WITH TIME ZONE DEFAULT NULL, + "${data}" VARIANT + ) + data_retention_time_in_days = 0 + AS ( + SELECT + ${v1_raw_id} AS "${raw_id}", + ${emitted_at} AS "${extracted_at}", + CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS "${loaded_at}", + PARSE_JSON(${data}) AS "${data}" + FROM ${v1_raw_table} + ) + ; + """); + } + + public static String escapeIdentifier(final String identifier) { + // Note that we don't need to escape backslashes here! + // The only special character in an identifier is the double-quote, which needs to be doubled. + return identifier.replace("\"", "\"\""); + } + + public static String escapeSingleQuotedString(final String str) { + return str + .replace("\\", "\\\\") + .replace("'", "\\'"); + } + + public static String escapeDollarString(final String str) { + return str.replace("$$", "\\$\\$"); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java new file mode 100644 index 000000000000..8524222fcd35 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import java.util.LinkedHashMap; + +/** + * @param columns Map from column name to type. Type is a plain string because + * {@link net.snowflake.client.jdbc.SnowflakeType} doesn't actually have all the types that + * Snowflake supports. + */ +// TODO fields for columns + indexes... or other stuff we want to set? +public record SnowflakeTableDefinition(LinkedHashMap columns) {} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java new file mode 100644 index 000000000000..cafb4ddb8e63 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.destination.typing_deduping.BaseDestinationV1V2Migrator; +import io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils; +import io.airbyte.integrations.base.destination.typing_deduping.NamespacedTableName; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.destination.NamingConventionTransformer; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Optional; +import lombok.SneakyThrows; + +public class SnowflakeV1V2Migrator extends BaseDestinationV1V2Migrator { + + private final NamingConventionTransformer namingConventionTransformer; + + private final JdbcDatabase database; + + private final String databaseName; + + public SnowflakeV1V2Migrator(final NamingConventionTransformer namingConventionTransformer, + final JdbcDatabase database, + final String databaseName) { + this.namingConventionTransformer = namingConventionTransformer; + this.database = database; + this.databaseName = databaseName; + } + + @SneakyThrows + @Override + protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamConfig) { + return !database + .queryJsons( + """ + SELECT SCHEMA_NAME + FROM information_schema.schemata + WHERE schema_name = ? + AND catalog_name = ?; + """, + streamConfig.id().rawNamespace(), + databaseName) + .isEmpty(); + } + + @Override + protected boolean schemaMatchesExpectation(final SnowflakeTableDefinition existingTable, final Collection columns) { + return CollectionUtils.containsAllIgnoreCase(existingTable.columns().keySet(), columns); + } + + @SneakyThrows + @Override + protected Optional getTableIfExists(final String namespace, final String tableName) { + // TODO this is mostly copied from SnowflakeDestinationHandler#findExistingTable, we should probably + // reuse this logic + // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates + // VARIANT as VARCHAR + LinkedHashMap columns = + database.queryJsons( + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_catalog = ? + AND table_schema = ? + AND table_name = ? + ORDER BY ordinal_position; + """, + databaseName, + namespace, + tableName) + .stream() + .collect(LinkedHashMap::new, + (map, row) -> map.put(row.get("COLUMN_NAME").asText(), row.get("DATA_TYPE").asText()), + LinkedHashMap::putAll); + if (columns.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(new SnowflakeTableDefinition(columns)); + } + } + + @Override + protected NamespacedTableName convertToV1RawName(final StreamConfig streamConfig) { + // The implicit upper-casing happens for this in the SqlGenerator + return new NamespacedTableName( + this.namingConventionTransformer.getIdentifier(streamConfig.id().originalNamespace()), + this.namingConventionTransformer.getRawTableName(streamConfig.id().originalName())); + } + + @Override + protected boolean doesValidV1RawTableExist(final String namespace, final String tableName) { + // Previously we were not quoting table names and they were being implicitly upper-cased. + // In v2 we preserve cases + return super.doesValidV1RawTableExist(namespace.toUpperCase(), tableName.toUpperCase()); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index 5298d766596f..f59b63a692cc 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -161,230 +161,17 @@ "type": "string", "order": 7 }, - "loading_method": { - "type": "object", - "title": "Data Staging Method", - "description": "Select a data staging method", - "order": 8, - "oneOf": [ - { - "title": "Select another option", - "description": "Select another option", - "required": ["method"], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Standard"], - "default": "Standard" - } - } - }, - { - "title": "[Recommended] Internal Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": ["method"], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["Internal Staging"], - "default": "Internal Staging" - } - } - }, - { - "title": "AWS S3 Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "s3_bucket_name", - "access_key_id", - "secret_access_key" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["S3 Staging"], - "default": "S3 Staging", - "order": 0 - }, - "s3_bucket_name": { - "title": "S3 Bucket Name", - "type": "string", - "description": "Enter your S3 bucket name", - "examples": ["airbyte.staging"], - "order": 1 - }, - "s3_bucket_region": { - "title": "S3 Bucket Region", - "type": "string", - "default": "", - "description": "Enter the region where your S3 bucket resides", - "enum": [ - "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", - "af-south-1", - "ap-east-1", - "ap-south-1", - "ap-northeast-1", - "ap-northeast-2", - "ap-northeast-3", - "ap-southeast-1", - "ap-southeast-2", - "ca-central-1", - "cn-north-1", - "cn-northwest-1", - "eu-central-1", - "eu-west-1", - "eu-west-2", - "eu-west-3", - "eu-south-1", - "eu-north-1", - "sa-east-1", - "me-south-1" - ], - "order": 2 - }, - "access_key_id": { - "type": "string", - "description": "Enter your AWS access key ID. Airbyte requires Read and Write permissions on your S3 bucket ", - "title": "AWS access key ID", - "airbyte_secret": true, - "order": 3 - }, - "secret_access_key": { - "type": "string", - "description": "Enter your AWS secret access key", - "title": "AWS secret access key", - "airbyte_secret": true, - "order": 4 - }, - "purge_staging_data": { - "title": "Purge Staging Files and Tables", - "type": "boolean", - "description": "Toggle to delete staging files from the S3 bucket after a successful sync", - "default": true, - "order": 5 - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "Choose a data encryption method for the staging data", - "default": { "encryption_type": "none" }, - "order": 6, - "oneOf": [ - { - "title": "No encryption", - "description": "Staging data will be stored in plaintext.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "none", - "enum": ["none"], - "default": "none" - } - } - }, - { - "title": "AES-CBC envelope encryption", - "description": "Staging data will be encrypted using AES-CBC envelope encryption.", - "type": "object", - "required": ["encryption_type"], - "properties": { - "encryption_type": { - "type": "string", - "const": "aes_cbc_envelope", - "enum": ["aes_cbc_envelope"], - "default": "aes_cbc_envelope" - }, - "key_encrypting_key": { - "type": "string", - "title": "Key", - "description": "The key, base64-encoded. Must be either 128, 192, or 256 bits. Leave blank to have Airbyte generate an ephemeral key for each sync.", - "airbyte_secret": true - } - } - } - ] - }, - "file_name_pattern": { - "type": "string", - "description": "The pattern allows you to set the file-name format for the S3 staging file(s)", - "title": "S3 Filename pattern", - "examples": [ - "{date}", - "{date:yyyy_MM}", - "{timestamp}", - "{part_number}", - "{sync_id}" - ], - "order": 7 - } - } - }, - { - "title": "Google Cloud Storage Staging", - "description": "Recommended for large production workloads for better speed and scalability.", - "required": [ - "method", - "project_id", - "bucket_name", - "credentials_json" - ], - "properties": { - "method": { - "title": "", - "description": "", - "type": "string", - "enum": ["GCS Staging"], - "default": "GCS Staging", - "order": 0 - }, - "project_id": { - "title": "Google Cloud project ID", - "type": "string", - "description": "Enter the Google Cloud project ID", - "examples": ["my-project"], - "order": 1 - }, - "bucket_name": { - "title": "Cloud Storage bucket name", - "type": "string", - "description": "Enter the Cloud Storage bucket name", - "examples": ["airbyte-staging"], - "order": 2 - }, - "credentials_json": { - "title": "Google Application Credentials", - "type": "string", - "description": "Enter your Google Cloud service account key in the JSON format with read/write access to your Cloud Storage staging bucket", - "airbyte_secret": true, - "multiline": true, - "order": 3 - } - } - } - ] + "use_1s1t_format": { + "type": "boolean", + "description": "(Beta) Use Destinations V2. Contact Airbyte Support to participate in the beta program.", + "title": "Use Destinations V2 (Early Access)", + "order": 10 }, - "file_buffer_count": { - "title": "File Buffer Count", - "type": "integer", - "minimum": 10, - "maximum": 50, - "default": 10, - "description": "Number of file buffers allocated for writing data. Increasing this number is beneficial for connections using Change Data Capture (CDC) and up to the number of streams within a connection. Increasing the number of file buffers past the maximum number of streams has deteriorating effects", - "examples": ["10"], - "order": 9 + "raw_data_schema": { + "type": "string", + "description": "(Beta) The schema to write raw tables into", + "title": "Destinations V2 Raw Table Schema (Early Access)", + "order": 11 } } }, diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java index 719440458081..8d7db2702655 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java @@ -14,18 +14,25 @@ import io.airbyte.commons.string.Strings; import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.sql.SQLException; import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class SnowflakeDestinationIntegrationTest { private final SnowflakeSQLNameTransformer namingResolver = new SnowflakeSQLNameTransformer(); + @BeforeEach + public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + } + @Test void testCheckFailsWithInvalidPermissions() throws Exception { // TODO(sherifnada) this test case is assumes config.json does not have permission to access the diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsCopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsCopyDestinationAcceptanceTest.java deleted file mode 100644 index 23d19f967dd1..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeGcsCopyDestinationAcceptanceTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.configoss.StandardCheckConnectionOutput; -import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import java.nio.file.Path; -import org.junit.jupiter.api.Test; - -public class SnowflakeGcsCopyDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest { - - private static final String NO_GCS_PRIVILEGES_ERR_MSG = - "Permission 'storage.objects.create' denied on resource (or it may not exist)."; - - @Override - public JsonNode getStaticConfig() { - final JsonNode copyConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/copy_gcs_config.json"))); - Preconditions.checkArgument(SnowflakeDestinationResolver.isGcsCopy(copyConfig)); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isS3Copy(copyConfig)); - return copyConfig; - } - - @Test - public void testCheckWithNoProperGcsPermissionConnection() throws Exception { - // Config to user (creds) that has no permission to schema - final JsonNode config = Jsons.deserialize(IOs.readFile( - Path.of("secrets/copy_insufficient_gcs_roles_config.json"))); - - final StandardCheckConnectionOutput standardCheckConnectionOutput = runCheck(config); - - assertEquals(Status.FAILED, standardCheckConnectionOutput.getStatus()); - assertThat(standardCheckConnectionOutput.getMessage()).contains(NO_GCS_PRIVILEGES_ERR_MSG); - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java index d82ac8eed581..5cfd62d01da6 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java @@ -10,7 +10,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Preconditions; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -19,6 +18,7 @@ import io.airbyte.configoss.StandardCheckConnectionOutput.Status; import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; @@ -32,9 +32,14 @@ import java.nio.file.Path; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.*; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.TimeZone; import java.util.stream.Collectors; import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -57,6 +62,11 @@ public class SnowflakeInsertDestinationAcceptanceTest extends DestinationAccepta private JdbcDatabase database; private DataSource dataSource; + @BeforeEach + public void setup() { + DestinationConfig.initialize(getConfig()); + } + @Override protected String getImageName() { return "airbyte/destination-snowflake:dev"; @@ -87,10 +97,12 @@ protected boolean supportObjectDataTypeTest() { return true; } + protected boolean supportsInDestinationNormalization() { + return true; + } + public JsonNode getStaticConfig() { final JsonNode insertConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/insert_config.json"))); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isS3Copy(insertConfig)); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isGcsCopy(insertConfig)); return insertConfig; } @@ -158,9 +170,10 @@ private List retrieveRecordsFromTable(final String tableName, final St // for each test we create a new schema in the database. run the test in there and then remove it. @Override - protected void setup(final TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { final String schemaName = Strings.addRandomSuffix("integration_test", "_", 5); final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); + TEST_SCHEMAS.add(schemaName); this.config = Jsons.clone(getStaticConfig()); ((ObjectNode) config).put("schema", schemaName); @@ -172,23 +185,21 @@ protected void setup(final TestDestinationEnv testEnv) throws Exception { @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - final String createSchemaQuery = String.format("DROP SCHEMA IF EXISTS %s", config.get("schema").asText()); - database.execute(createSchemaQuery); - DataSourceFactory.close(dataSource); - } + TEST_SCHEMAS.add(config.get("schema").asText()); + for (final String schema : TEST_SCHEMAS) { + // we need to wrap namespaces in quotes, but that means we have to manually upcase them. + // thanks, v1 destinations! + // this probably doesn't actually work, because v1 destinations are mangling namespaces and names + // but it's approximately correct and maybe works for some things. + final String mangledSchema = schema.toUpperCase(); + final String dropSchemaQuery = String.format("DROP SCHEMA IF EXISTS \"%s\"", mangledSchema); + database.execute(dropSchemaQuery); + } - @Test - public void testCheckWithNoActiveWarehouseConnection() throws Exception { - // Config to user(creds) that has no warehouse assigned - final JsonNode config = Jsons.deserialize(IOs.readFile( - Path.of("secrets/internal_staging_config_no_active_warehouse.json"))); - - final StandardCheckConnectionOutput standardCheckConnectionOutput = runCheck(config); - - assertEquals(Status.FAILED, standardCheckConnectionOutput.getStatus()); - assertThat(standardCheckConnectionOutput.getMessage()).contains(NO_ACTIVE_WAREHOUSE_ERR_MSG); + DataSourceFactory.close(dataSource); } + @Disabled("See README for why this test is disabled") @Test public void testCheckWithNoTextSchemaPermissionConnection() throws Exception { // Config to user (creds) that has no permission to schema diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java index e819ed6b7e00..b03c8b8111f1 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java @@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -23,6 +22,7 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -30,12 +30,10 @@ public class SnowflakeInternalStagingDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest { public JsonNode getStaticConfig() { - final JsonNode internalStagingConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/internal_staging_config.json"))); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isS3Copy(internalStagingConfig)); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isGcsCopy(internalStagingConfig)); - return internalStagingConfig; + return Jsons.deserialize(IOs.readFile(Path.of("secrets/internal_staging_config.json"))); } + @Disabled("See README for why this test is disabled") @Test public void testCheckWithNoProperStagingPermissionConnection() throws Exception { // Config to user (creds) that has no permission to write @@ -48,6 +46,19 @@ public void testCheckWithNoProperStagingPermissionConnection() throws Exception assertThat(standardCheckConnectionOutput.getMessage()).contains(NO_USER_PRIVILEGES_ERR_MSG); } + @Disabled("See README for why this test is disabled") + @Test + public void testCheckWithNoActiveWarehouseConnection() throws Exception { + // Config to user(creds) that has no warehouse assigned + final JsonNode config = Jsons.deserialize(IOs.readFile( + Path.of("secrets/internal_staging_config_no_active_warehouse.json"))); + + final StandardCheckConnectionOutput standardCheckConnectionOutput = runCheck(config); + + assertEquals(Status.FAILED, standardCheckConnectionOutput.getStatus()); + assertThat(standardCheckConnectionOutput.getMessage()).contains(NO_ACTIVE_WAREHOUSE_ERR_MSG); + } + @ParameterizedTest @ArgumentsSource(DataArgumentsProvider.class) public void testSyncWithNormalizationWithKeyPairAuth(final String messagesFilename, final String catalogFilename) throws Exception { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3CopyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3CopyDestinationAcceptanceTest.java deleted file mode 100644 index f067ddb40b02..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3CopyDestinationAcceptanceTest.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import java.nio.file.Path; - -public class SnowflakeS3CopyDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest { - - @Override - public JsonNode getStaticConfig() { - final JsonNode copyConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/copy_s3_config.json"))); - Preconditions.checkArgument(SnowflakeDestinationResolver.isS3Copy(copyConfig)); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isGcsCopy(copyConfig)); - return copyConfig; - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3CopyEncryptedDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3CopyEncryptedDestinationAcceptanceTest.java deleted file mode 100644 index c434b24ea27e..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3CopyEncryptedDestinationAcceptanceTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.configoss.StandardCheckConnectionOutput; -import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import java.nio.file.Path; -import org.junit.jupiter.api.Test; - -public class SnowflakeS3CopyEncryptedDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest { - - private static final String NO_S3_PRIVILEGES_ERR_MSG = "Could not connect with provided configuration."; - - @Override - public JsonNode getStaticConfig() { - final JsonNode copyConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/copy_s3_encrypted_config.json"))); - Preconditions.checkArgument(SnowflakeDestinationResolver.isS3Copy(copyConfig)); - Preconditions.checkArgument(!SnowflakeDestinationResolver.isGcsCopy(copyConfig)); - return copyConfig; - } - - @Test - public void testCheckWithNoProperS3PermissionConnection() throws Exception { - // Config to user (creds) that has no permission to schema - final JsonNode config = Jsons.deserialize(IOs.readFile( - Path.of("secrets/copy_s3_wrong_location_config.json"))); - - final StandardCheckConnectionOutput standardCheckConnectionOutput = runCheck(config); - - assertEquals(Status.FAILED, standardCheckConnectionOutput.getStatus()); - assertThat(standardCheckConnectionOutput.getMessage()).contains(NO_S3_PRIVILEGES_ERR_MSG); - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java index 55d825448285..37cea8bc1400 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java @@ -8,16 +8,23 @@ import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcSourceOperations; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTestUtils; import java.sql.ResultSet; import java.sql.SQLException; public class SnowflakeTestSourceOperations extends JdbcSourceOperations { @Override - protected void putString(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { - DestinationAcceptanceTestUtils.putStringIntoJson(resultSet.getString(index), columnName, node); + public void copyToJsonField(final ResultSet resultSet, final int colIndex, final ObjectNode json) throws SQLException { + final String columnName = resultSet.getMetaData().getColumnName(colIndex); + final String columnTypeName = resultSet.getMetaData().getColumnTypeName(colIndex).toLowerCase(); + + switch (columnTypeName) { + // jdbc converts VARIANT columns to serialized JSON, so we need to deserialize these. + case "variant", "array", "object" -> json.set(columnName, Jsons.deserializeExact(resultSet.getString(colIndex))); + default -> super.copyToJsonField(resultSet, colIndex, json); + } } @Override diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java new file mode 100644 index 000000000000..a4f0aadd1cd5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake; + +import static java.util.stream.Collectors.joining; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import org.apache.commons.text.StringSubstitutor; + +public class SnowflakeTestUtils { + + public static List dumpRawTable(final JdbcDatabase database, final String tableIdentifier) throws SQLException { + return dumpTable( + List.of( + quote(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID), + timestampToString(quote(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)), + timestampToString(quote(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT)), + quote(JavaBaseConstants.COLUMN_NAME_DATA)), + database, + tableIdentifier); + } + + public static List dumpFinalTable(final JdbcDatabase database, final String databaseName, final String schema, final String table) + throws SQLException { + // We have to discover the column names, because if we just SELECT * then snowflake will upcase all + // column names. + final List columns = database.queryJsons( + """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_catalog = ? + AND table_schema = ? + AND table_name = ? + ORDER BY ordinal_position; + """, + unescapeIdentifier(databaseName), + unescapeIdentifier(schema), + unescapeIdentifier(table)).stream() + .map(column -> { + final String quotedName = quote(column.get("COLUMN_NAME").asText()); + final String type = column.get("DATA_TYPE").asText(); + return switch (type) { + // something about JDBC is mangling date/time values + // E.g. 2023-01-01T00:00:00Z becomes 2022-12-31T16:00:00Z + // Explicitly convert to varchar to prevent jdbc from doing this. + case "TIMESTAMP_TZ" -> timestampToString(quotedName); + case "TIMESTAMP_NTZ", "TIMESTAMP_LTZ" -> "TO_VARCHAR(" + quotedName + ", 'YYYY-MM-DD\"T\"HH24:MI:SS.FF') as " + quotedName; + case "TIME" -> "TO_VARCHAR(" + quotedName + ", 'HH24:MI:SS.FF') as " + quotedName; + case "DATE" -> "TO_VARCHAR(" + quotedName + ", 'YYYY-MM-DD') as " + quotedName; + default -> quotedName; + }; + }) + .toList(); + return dumpTable(columns, database, quote(schema) + "." + quote(table)); + } + + /** + * This is mostly identical to SnowflakeInsertDestinationAcceptanceTest, except it doesn't verify + * table type. + *

    + * The columns param is a list of column names/aliases. For example, + * {@code "_airbyte_extracted_at :: varchar AS "_airbyte_extracted_at"}. + * + * @param tableIdentifier Table identifier (e.g. "schema.table"), with quotes if necessary. + */ + public static List dumpTable(final List columns, final JdbcDatabase database, final String tableIdentifier) throws SQLException { + return database.bufferedResultSetQuery(connection -> connection.createStatement().executeQuery(new StringSubstitutor(Map.of( + "columns", columns.stream().collect(joining(",")), + "table", tableIdentifier)).replace( + """ + SELECT ${columns} FROM ${table} ORDER BY "_airbyte_extracted_at" ASC + """)), + new SnowflakeTestSourceOperations()::rowToJson); + } + + private static String quote(final String name) { + return '"' + SnowflakeSqlGenerator.escapeIdentifier(name) + '"'; + } + + public static String timestampToString(final String quotedName) { + return "TO_VARCHAR(" + quotedName + ", 'YYYY-MM-DD\"T\"HH24:MI:SS.FFTZH:TZM') as " + quotedName; + } + + private static String unescapeIdentifier(final String escapedIdentifier) { + return escapedIdentifier.replace("\"\"", "\""); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java new file mode 100644 index 000000000000..8fb2205f62da --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.snowflake.OssCloudEnvVarConsts; +import io.airbyte.integrations.destination.snowflake.SnowflakeDatabase; +import io.airbyte.integrations.destination.snowflake.SnowflakeTestUtils; +import java.nio.file.Path; +import java.util.List; +import javax.sql.DataSource; + +public abstract class AbstractSnowflakeTypingDedupingTest extends BaseTypingDedupingTest { + + private String databaseName; + private JdbcDatabase database; + private DataSource dataSource; + + protected abstract String getConfigPath(); + + @Override + protected String getImageName() { + return "airbyte/destination-snowflake:dev"; + } + + @Override + protected JsonNode generateConfig() { + final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of(getConfigPath()))); + ((ObjectNode) config).put("schema", "typing_deduping_default_schema" + getUniqueSuffix()); + databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + dataSource = SnowflakeDatabase.createDataSource(config, OssCloudEnvVarConsts.AIRBYTE_OSS); + database = SnowflakeDatabase.getDatabase(dataSource); + return config; + } + + @Override + protected List dumpRawTableRecords(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(); + } + final String tableName = StreamId.concatenateRawTableName(streamNamespace, streamName); + final String schema = getRawSchema(); + return SnowflakeTestUtils.dumpRawTable( + database, + // Explicitly wrap in quotes to prevent snowflake from upcasing + '"' + schema + "\".\"" + tableName + '"'); + } + + @Override + protected List dumpFinalTableRecords(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(); + } + return SnowflakeTestUtils.dumpFinalTable(database, databaseName, streamNamespace, streamName); + } + + @Override + protected void teardownStreamAndNamespace(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(); + } + database.execute( + String.format( + """ + DROP TABLE IF EXISTS "%s"."%s"; + DROP SCHEMA IF EXISTS "%s" CASCADE + """, + getRawSchema(), + StreamId.concatenateRawTableName(streamNamespace, streamName), + streamNamespace)); + } + + @Override + protected void globalTeardown() throws Exception { + DataSourceFactory.close(dataSource); + } + + /** + * Subclasses using a config with a nonstandard raw table schema should override this method. + */ + protected String getRawSchema() { + return JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + } + + private String getDefaultSchema() { + return getConfig().get("schema").asText(); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingRawSchemaOverrideTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingRawSchemaOverrideTypingDedupingTest.java new file mode 100644 index 000000000000..6d766b6fee7e --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingRawSchemaOverrideTypingDedupingTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +public class SnowflakeInternalStagingRawSchemaOverrideTypingDedupingTest extends AbstractSnowflakeTypingDedupingTest { + + @Override + protected String getConfigPath() { + return "secrets/1s1t_internal_staging_config_raw_schema_override.json"; + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingTypingDedupingTest.java new file mode 100644 index 000000000000..a80d38d2bcf6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingTypingDedupingTest.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +public class SnowflakeInternalStagingTypingDedupingTest extends AbstractSnowflakeTypingDedupingTest { + + @Override + protected String getConfigPath() { + return "secrets/1s1t_internal_staging_config.json"; + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..eabe272d3e5e --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import static io.airbyte.integrations.destination.snowflake.SnowflakeTestUtils.timestampToString; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import autovalue.shaded.com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.snowflake.OssCloudEnvVarConsts; +import io.airbyte.integrations.destination.snowflake.SnowflakeDatabase; +import io.airbyte.integrations.destination.snowflake.SnowflakeTestSourceOperations; +import io.airbyte.integrations.destination.snowflake.SnowflakeTestUtils; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.StringSubstitutor; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class SnowflakeSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { + + private static String databaseName; + private static JdbcDatabase database; + private static DataSource dataSource; + + @BeforeAll + public static void setupSnowflake() { + final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_internal_staging_config.json"))); + databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + dataSource = SnowflakeDatabase.createDataSource(config, OssCloudEnvVarConsts.AIRBYTE_OSS); + database = SnowflakeDatabase.getDatabase(dataSource); + } + + @AfterAll + public static void teardownSnowflake() throws Exception { + DataSourceFactory.close(dataSource); + } + + @Override + protected SnowflakeSqlGenerator getSqlGenerator() { + return new SnowflakeSqlGenerator(); + } + + @Override + protected SnowflakeDestinationHandler getDestinationHandler() { + return new SnowflakeDestinationHandler(databaseName, database); + } + + @Override + protected void createNamespace(final String namespace) throws SQLException { + database.execute("CREATE SCHEMA IF NOT EXISTS \"" + namespace + '"'); + } + + @Override + protected void createRawTable(final StreamId streamId) throws Exception { + database.execute(new StringSubstitutor(Map.of( + "raw_table_id", streamId.rawTableId(SnowflakeSqlGenerator.QUOTE))).replace( + """ + CREATE TABLE ${raw_table_id} ( + "_airbyte_raw_id" TEXT NOT NULL, + "_airbyte_data" VARIANT NOT NULL, + "_airbyte_extracted_at" TIMESTAMP_TZ NOT NULL, + "_airbyte_loaded_at" TIMESTAMP_TZ + ) + """)); + } + + @Override + protected void createFinalTable(final boolean includeCdcDeletedAt, final StreamId streamId, final String suffix) throws Exception { + final String cdcDeletedAt = includeCdcDeletedAt ? "\"_ab_cdc_deleted_at\" TIMESTAMP_TZ," : ""; + database.execute(new StringSubstitutor(Map.of( + "final_table_id", streamId.finalTableId(SnowflakeSqlGenerator.QUOTE, suffix), + "cdc_deleted_at", cdcDeletedAt)).replace( + """ + CREATE TABLE ${final_table_id} ( + "_airbyte_raw_id" TEXT NOT NULL, + "_airbyte_extracted_at" TIMESTAMP_TZ NOT NULL, + "_airbyte_meta" VARIANT NOT NULL, + "id1" NUMBER, + "id2" NUMBER, + "updated_at" TIMESTAMP_TZ, + ${cdc_deleted_at} + "struct" OBJECT, + "array" ARRAY, + "string" TEXT, + "number" FLOAT, + "integer" NUMBER, + "boolean" BOOLEAN, + "timestamp_with_timezone" TIMESTAMP_TZ, + "timestamp_without_timezone" TIMESTAMP_NTZ, + "time_with_timezone" TEXT, + "time_without_timezone" TIME, + "date" DATE, + "unknown" VARIANT + ) + """)); + } + + @Override + protected List dumpRawTableRecords(final StreamId streamId) throws Exception { + return SnowflakeTestUtils.dumpRawTable(database, streamId.rawTableId(SnowflakeSqlGenerator.QUOTE)); + } + + @Override + protected List dumpFinalTableRecords(final StreamId streamId, final String suffix) throws Exception { + return SnowflakeTestUtils.dumpFinalTable( + database, + databaseName, + streamId.finalNamespace(), + streamId.finalName() + suffix); + } + + @Override + protected void teardownNamespace(final String namespace) throws SQLException { + database.execute("DROP SCHEMA IF EXISTS \"" + namespace + '"'); + } + + @Override + protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, + final StreamId streamId, + final String suffix, + final List records) + throws Exception { + final List columnNames = includeCdcDeletedAt ? FINAL_TABLE_COLUMN_NAMES_CDC : FINAL_TABLE_COLUMN_NAMES; + final String cdcDeletedAtName = includeCdcDeletedAt ? ",\"_ab_cdc_deleted_at\"" : ""; + final String cdcDeletedAtExtract = includeCdcDeletedAt ? ",column19" : ""; + final String recordsText = records.stream() + // For each record, convert it to a string like "(rawId, extractedAt, loadedAt, data)" + .map(record -> columnNames.stream() + .map(record::get) + .map(this::dollarQuoteWrap) + .collect(joining(","))) + .map(row -> "(" + row + ")") + .collect(joining(",")); + + database.execute(new StringSubstitutor( + Map.of( + "final_table_id", streamId.finalTableId(SnowflakeSqlGenerator.QUOTE, suffix), + "cdc_deleted_at_name", cdcDeletedAtName, + "cdc_deleted_at_extract", cdcDeletedAtExtract, + "records", recordsText), + "#{", + "}").replace( + // Similar to insertRawTableRecords, some of these columns are declared as string and wrapped in + // parse_json(). + """ + INSERT INTO #{final_table_id} ( + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta", + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown" + #{cdc_deleted_at_name} + ) + SELECT + column1, + column2, + PARSE_JSON(column3), + column4, + column5, + column6, + PARSE_JSON(column7), + PARSE_JSON(column8), + column9, + column10, + column11, + column12, + column13, + column14, + column15, + column16, + column17, + PARSE_JSON(column18) + #{cdc_deleted_at_extract} + FROM VALUES + #{records} + """)); + } + + private String dollarQuoteWrap(final JsonNode node) { + if (node == null) { + return "NULL"; + } + final String stringContents = node.isTextual() ? node.asText() : node.toString(); + // Use dollar quotes to avoid needing to escape quotes + return StringUtils.wrap(stringContents.replace("$$", "\\$\\$"), "$$"); + } + + @Override + protected void insertRawTableRecords(final StreamId streamId, final List records) throws Exception { + final String recordsText = records.stream() + // For each record, convert it to a string like "(rawId, extractedAt, loadedAt, data)" + .map(record -> JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES + .stream() + .map(record::get) + .map(this::dollarQuoteWrap) + .collect(joining(","))) + .map(row -> "(" + row + ")") + .collect(joining(",")); + database.execute(new StringSubstitutor( + Map.of( + "raw_table_id", streamId.rawTableId(SnowflakeSqlGenerator.QUOTE), + "records_text", recordsText), + // Use different delimiters because we're using dollar quotes in the query. + "#{", + "}").replace( + // Snowflake doesn't let you directly insert a parse_json expression, so we have to use a subquery. + """ + INSERT INTO #{raw_table_id} ( + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_loaded_at", + "_airbyte_data" + ) + SELECT + column1, + column2, + column3, + PARSE_JSON(column4) + FROM VALUES + #{records_text}; + """)); + } + + @Override + @Test + public void testCreateTableIncremental() throws Exception { + final String sql = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(sql); + + final Optional tableKind = database.queryJsons(String.format("SHOW TABLES LIKE '%s' IN SCHEMA \"%s\";", "users_final", namespace)) + .stream().map(record -> record.get("kind").asText()) + .findFirst(); + final Map columns = database.queryJsons( + """ + SELECT column_name, data_type, numeric_precision, numeric_scale + FROM information_schema.columns + WHERE table_catalog = ? + AND table_schema = ? + AND table_name = ? + ORDER BY ordinal_position; + """, + databaseName, + namespace, + "users_final").stream() + .collect(toMap( + record -> record.get("COLUMN_NAME").asText(), + record -> { + final String type = record.get("DATA_TYPE").asText(); + if (type.equals("NUMBER")) { + return String.format("NUMBER(%s, %s)", record.get("NUMERIC_PRECISION").asText(), + record.get("NUMERIC_SCALE").asText()); + } + return type; + })); + assertAll( + () -> assertEquals(Optional.of("TABLE"), tableKind, "Table should be permanent, not transient"), + () -> assertEquals( + ImmutableMap.builder() + .put("_airbyte_raw_id", "TEXT") + .put("_airbyte_extracted_at", "TIMESTAMP_TZ") + .put("_airbyte_meta", "VARIANT") + .put("id1", "NUMBER(38, 0)") + .put("id2", "NUMBER(38, 0)") + .put("updated_at", "TIMESTAMP_TZ") + .put("struct", "OBJECT") + .put("array", "ARRAY") + .put("string", "TEXT") + .put("number", "FLOAT") + .put("integer", "NUMBER(38, 0)") + .put("boolean", "BOOLEAN") + .put("timestamp_with_timezone", "TIMESTAMP_TZ") + .put("timestamp_without_timezone", "TIMESTAMP_NTZ") + .put("time_with_timezone", "TEXT") + .put("time_without_timezone", "TIME") + .put("date", "DATE") + .put("unknown", "VARIANT") + .build(), + columns)); + } + + @Override + protected void createV1RawTable(final StreamId v1RawTable) throws Exception { + database.execute(String.format( + """ + CREATE SCHEMA IF NOT EXISTS %s; + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s VARIANT, + %s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp() + ) data_retention_time_in_days = 0; + """, + v1RawTable.rawNamespace(), + v1RawTable.rawNamespace(), + v1RawTable.rawName(), + JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_DATA, + JavaBaseConstants.COLUMN_NAME_EMITTED_AT)); + } + + @Override + protected void insertV1RawTableRecords(final StreamId streamId, final List records) throws Exception { + final var recordsText = records + .stream() + .map(record -> JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS + .stream() + .map(record::get) + .map(value -> value == null ? "NULL" : value.isTextual() ? value.asText() : value.toString()) + .map(v -> "NULL".equals(v) ? v : StringUtils.wrap(v, "$$")) + .collect(joining(","))) + .map(row -> "(%s)".formatted(row)) + .collect(joining(",")); + final var insert = new StringSubstitutor(Map.of( + "v1_raw_table_id", String.join(".", streamId.rawNamespace(), streamId.rawName()), + "records", recordsText), + // Use different delimiters because we're using dollar quotes in the query. + "#{", + "}").replace( + """ + INSERT INTO #{v1_raw_table_id} (_airbyte_ab_id, _airbyte_data, _airbyte_emitted_at) + SELECT column1, PARSE_JSON(column2), column3 FROM VALUES + #{records}; + """); + database.execute(insert); + } + + @Override + protected List dumpV1RawTableRecords(final StreamId streamId) throws Exception { + final var columns = Stream.of( + JavaBaseConstants.COLUMN_NAME_AB_ID, + timestampToString(JavaBaseConstants.COLUMN_NAME_EMITTED_AT), + JavaBaseConstants.COLUMN_NAME_DATA).collect(joining(",")); + return database.bufferedResultSetQuery(connection -> connection.createStatement().executeQuery(new StringSubstitutor(Map.of( + "columns", columns, + "table", String.join(".", streamId.rawNamespace(), streamId.rawName()))).replace( + """ + SELECT ${columns} FROM ${table} ORDER BY _airbyte_emitted_at ASC + """)), + new SnowflakeTestSourceOperations()::rowToJson); + } + + @Override + protected void migrationAssertions(final List v1RawRecords, final List v2RawRecords) { + final var v2RecordMap = v2RawRecords.stream().collect(Collectors.toMap( + record -> record.get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).asText(), + Function.identity())); + assertAll( + () -> assertEquals(5, v1RawRecords.size()), + () -> assertEquals(5, v2RawRecords.size())); + v1RawRecords.forEach(v1Record -> { + final var v1id = v1Record.get(JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase()).asText(); + assertAll( + () -> assertEquals(v1id, v2RecordMap.get(v1id).get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).asText()), + () -> assertEquals(v1Record.get(JavaBaseConstants.COLUMN_NAME_EMITTED_AT.toUpperCase()).asText(), + v2RecordMap.get(v1id).get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT).asText()), + () -> assertNull(v2RecordMap.get(v1id).get(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT))); + JsonNode originalData = v1Record.get(JavaBaseConstants.COLUMN_NAME_DATA.toUpperCase()); + JsonNode migratedData = v2RecordMap.get(v1id).get(JavaBaseConstants.COLUMN_NAME_DATA); + migratedData = migratedData.isTextual() ? Jsons.deserializeExact(migratedData.asText()) : migratedData; + originalData = originalData.isTextual() ? Jsons.deserializeExact(migratedData.asText()) : originalData; + // hacky thing because we only care about the data contents. + // diffRawTableRecords makes some assumptions about the structure of the blob. + DIFFER.diffFinalTableRecords(List.of(originalData), List.of(migratedData)); + }); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..bf928d688997 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..4a370ae69377 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..f30261d6154c --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -0,0 +1,4 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..7f6e7aa9438b --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,6 @@ +// Keep the Alice record with more recent updated_at +// Note that extracted_at uses microseconds precision (because it's parsed directly by snowflake) +// but updated_at is still using seconds precision (because Snowflake treats it as a normal string inside a JSON blob) +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl new file mode 100644 index 000000000000..a26cf1d5289d --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +// Invalid columns are nulled out (i.e. SQL null, not JSON null) +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl new file mode 100644 index 000000000000..75901736b545 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +// Note the duplicate record. In this sync mode, we don't dedup anything. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..7ac3264abcc4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with updated_at, so it still has a null cursor +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..fc788ceeb4a5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} +// Charlie wasn't reemitted in sync2. This record still has an old_cursor value. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl new file mode 100644 index 000000000000..fd96bc516169 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -0,0 +1,8 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} + +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000000Z"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl new file mode 100644 index 000000000000..b9a53cffb59a --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl @@ -0,0 +1,9 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl new file mode 100644 index 000000000000..da89677485f0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000000Z"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl new file mode 100644 index 000000000000..2607c9f73a49 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..96442a4e7fcd --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Delete Bob, keep Charlie +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..fe2377ede753 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +// Keep the record that deleted Bob, but delete the other records associated with id=(1, 201) +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} +// And keep Charlie's record, even though it wasn't reemitted in sync2. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl new file mode 100644 index 000000000000..c17a8134a49e --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -0,0 +1,6 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56.000000000", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56.000000000", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`", "Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`", "Problem with `time_without_timezone`", "Problem with `date`"]}, "string": "{}"} +// Note: no loss of precision on these numbers. A naive float64 conversion would yield 67.17411800000001. +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..45e13560e19f --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}}' +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl new file mode 100644 index 000000000000..293be295e24d --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": ["Problem with `integer`"]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00.000000000Z", "string": "Bob"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..0cdce4a5e75e --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl new file mode 100644 index 000000000000..b36b5ea4b450 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..749862aafea9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl new file mode 100644 index 000000000000..0064d5adf1b3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -0,0 +1,12 @@ +// snowflake/jdbc is returning 9 decimals for all timestamp/time types +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000Z", "time_with_timezone": "12:34:56Z"} +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000-08:00", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000-08:00", "time_with_timezone": "12:34:56-0800"} +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000-08:00", "time_with_timezone": "12:34:56-08"} +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000+08:00", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000+08:00", "time_with_timezone": "12:34:56+0800"} +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000+08:00", "time_with_timezone": "12:34:56+08"} +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000000Z", "time_with_timezone": "12:34:56.123Z"} + +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.000000000", "time_without_timezone": "12:34:56.000000000", "date": "2023-01-23"} +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123000000", "time_without_timezone": "12:34:56.123000000"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl new file mode 100644 index 000000000000..b61de2de5f58 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -0,0 +1,3 @@ +// columns with issues: +// * endswithbackslash\ -> written as null to the final table +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000000Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..cc14be85e880 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java index 556a064efb6f..178933c9a653 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java @@ -5,24 +5,21 @@ package io.airbyte.integrations.destination.snowflake; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.integrations.destination.snowflake.SnowflakeDestination.DestinationType; +import io.airbyte.integrations.destination_async.AsyncStreamConsumer; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; -import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -30,7 +27,10 @@ public class SnowflakeDestinationTest { - private static final ObjectMapper mapper = MoreMappers.initMapper(); + @BeforeEach + public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + } private static Stream urlsDataProvider() { return Stream.of( @@ -74,43 +74,6 @@ void testUrlPattern(final String url, final boolean isMatch) throws Exception { assertEquals(isMatch, matcher.find()); } - @Test - @DisplayName("When given S3 credentials should use COPY") - public void useS3CopyStrategyTest() { - final var stubLoadingMethod = mapper.createObjectNode(); - stubLoadingMethod.put("s3_bucket_name", "fake-bucket"); - stubLoadingMethod.put("access_key_id", "test"); - stubLoadingMethod.put("secret_access_key", "test key"); - - final var stubConfig = mapper.createObjectNode(); - stubConfig.set("loading_method", stubLoadingMethod); - - assertTrue(SnowflakeDestinationResolver.isS3Copy(stubConfig)); - } - - @Test - @DisplayName("When given GCS credentials should use COPY") - public void useGcsCopyStrategyTest() { - final var stubLoadingMethod = mapper.createObjectNode(); - stubLoadingMethod.put("project_id", "my-project"); - stubLoadingMethod.put("bucket_name", "my-bucket"); - stubLoadingMethod.put("credentials_json", "hunter2"); - - final var stubConfig = mapper.createObjectNode(); - stubConfig.set("loading_method", stubLoadingMethod); - - assertTrue(SnowflakeDestinationResolver.isGcsCopy(stubConfig)); - } - - @Test - @DisplayName("When not given S3 credentials should use INSERT") - public void useInsertStrategyTest() { - final var stubLoadingMethod = mapper.createObjectNode(); - final var stubConfig = mapper.createObjectNode(); - stubConfig.set("loading_method", stubLoadingMethod); - assertFalse(SnowflakeDestinationResolver.isS3Copy(stubConfig)); - } - @ParameterizedTest @MethodSource("destinationTypeToConfig") public void testS3ConfigType(final String configFileName, final DestinationType expectedDestinationType) throws Exception { @@ -120,10 +83,7 @@ public void testS3ConfigType(final String configFileName, final DestinationType } private static Stream destinationTypeToConfig() { - return Stream.of( - arguments("copy_gcs_config.json", DestinationType.COPY_GCS), - arguments("copy_s3_config.json", DestinationType.COPY_S3), - arguments("insert_config.json", DestinationType.INTERNAL_STAGING)); + return Stream.of(arguments("insert_config.json", DestinationType.INTERNAL_STAGING)); } @Test @@ -131,7 +91,7 @@ void testWriteSnowflakeInternal() throws Exception { final JsonNode config = Jsons.deserialize(MoreResources.readResource("internal_staging_config.json"), JsonNode.class); final SerializedAirbyteMessageConsumer consumer = new SnowflakeDestination(OssCloudEnvVarConsts.AIRBYTE_OSS) .getSerializedMessageConsumer(config, new ConfiguredAirbyteCatalog(), null); - assertEquals(Destination.ShimToSerializedAirbyteMessageConsumer.class, consumer.getClass()); + assertEquals(AsyncStreamConsumer.class, consumer.getClass()); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeGCSStreamCopierTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeGCSStreamCopierTest.java deleted file mode 100644 index f198101ae161..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeGCSStreamCopierTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static io.airbyte.integrations.destination.snowflake.SnowflakeS3StreamCopier.MAX_PARTS_PER_FILE; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -import com.google.cloud.storage.Storage; -import com.google.common.collect.Lists; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.gcs.GcsConfig; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.util.ArrayList; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class SnowflakeGCSStreamCopierTest { - - private JdbcDatabase db; - private SnowflakeGcsStreamCopier copier; - - @BeforeEach - public void setup() throws Exception { - Storage storageClient = mock(Storage.class, RETURNS_DEEP_STUBS); - db = mock(JdbcDatabase.class); - SqlOperations sqlOperations = mock(SqlOperations.class); - - copier = (SnowflakeGcsStreamCopier) new SnowflakeGcsStreamCopierFactory().create( - "fake-staging-folder", - DestinationSyncMode.OVERWRITE, - "fake-schema", - "fake-stream", - storageClient, - db, - new GcsConfig("fake-project-id", "fake-bucket-name", "fake-credentials"), - new StandardNameTransformer(), - sqlOperations); - } - - @Test - public void copiesCorrectFilesToTable() throws Exception { - for (int i = 0; i < MAX_PARTS_PER_FILE + 1; i++) { - copier.prepareStagingFile(); - } - - copier.copyStagingFileToTemporaryTable(); - final List> partition = Lists.partition(new ArrayList<>(copier.getGcsStagingFiles()), 1000); - for (final List files : partition) { - verify(db).execute(String.format( - "COPY INTO fake-schema.%s FROM '%s' storage_integration = gcs_airbyte_integration " - + " file_format = (type = csv field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') " - + "files = (" + copier.generateFilesList(files) + " );", - copier.getTmpTableName(), - copier.generateBucketPath())); - } - - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java index ccfa7bb8d66e..66f0ff677898 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java @@ -7,7 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.DestinationConfig; import java.util.List; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class SnowflakeInternalStagingSqlOperationsTest { @@ -17,8 +20,14 @@ class SnowflakeInternalStagingSqlOperationsTest { private static final String STAGE_PATH = "stagePath/2022/"; private static final String FILE_PATH = "filepath/filename"; - private final SnowflakeInternalStagingSqlOperations snowflakeStagingSqlOperations = - new SnowflakeInternalStagingSqlOperations(new SnowflakeSQLNameTransformer()); + private SnowflakeInternalStagingSqlOperations snowflakeStagingSqlOperations; + + @BeforeEach + public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + snowflakeStagingSqlOperations = + new SnowflakeInternalStagingSqlOperations(new SnowflakeSQLNameTransformer()); + } @Test void createStageIfNotExists() { @@ -44,9 +53,17 @@ void listStage() { @Test void copyIntoTmpTableFromStage() { - final String expectedQuery = "COPY INTO schemaName.tableName FROM '@" + STAGE_NAME + "/" + STAGE_PATH + "' " - + "file_format = (type = csv compression = auto field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') " - + "files = ('filename1','filename2');"; + final String expectedQuery = + """ + COPY INTO schemaName.tableName FROM '@stageName/stagePath/2022/' + file_format = ( + type = csv + compression = auto + field_delimiter = ',' + skip_header = 0 + FIELD_OPTIONALLY_ENCLOSED_BY = '"' + NULL_IF=('') + ) files = ('filename1','filename2');"""; final String actualCopyQuery = snowflakeStagingSqlOperations.getCopyQuery(STAGE_NAME, STAGE_PATH, List.of("filename1", "filename2"), "tableName", SCHEMA_NAME); assertEquals(expectedQuery, actualCopyQuery); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperationsTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperationsTest.java deleted file mode 100644 index 2d3213c48465..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StagingSqlOperationsTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.integrations.destination.s3.NoEncryption; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import java.util.List; -import org.junit.jupiter.api.Test; - -class SnowflakeS3StagingSqlOperationsTest { - - private static final String SCHEMA_NAME = "schemaName"; - private static final String STAGE_PATH = "stagePath/2022/"; - private static final String TABLE_NAME = "tableName"; - private static final String BUCKET_NAME = "bucket_name"; - - private final AmazonS3 s3Client = mock(AmazonS3.class); - private final S3DestinationConfig s3Config = mock(S3DestinationConfig.class); - private final S3AccessKeyCredentialConfig credentialConfig = mock(S3AccessKeyCredentialConfig.class); - - private final SnowflakeS3StagingSqlOperations snowflakeStagingSqlOperations = - new SnowflakeS3StagingSqlOperations(new SnowflakeSQLNameTransformer(), s3Client, s3Config, new NoEncryption()); - - @Test - void copyIntoTmpTableFromStage() { - final String expectedQuery = "COPY INTO " + SCHEMA_NAME + "." + TABLE_NAME + " FROM 's3://" + BUCKET_NAME + "/" + STAGE_PATH + "' " + - "CREDENTIALS=(aws_key_id='aws_access_key_id' aws_secret_key='aws_secret_access_key') file_format = (type = csv compression = auto " + - "field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') files = ('filename1','filename2');"; - when(s3Config.getBucketName()).thenReturn(BUCKET_NAME); - when(s3Config.getS3CredentialConfig()).thenReturn(credentialConfig); - when(credentialConfig.getAccessKeyId()).thenReturn("aws_access_key_id"); - when(credentialConfig.getSecretAccessKey()).thenReturn("aws_secret_access_key"); - final String actualCopyQuery = - snowflakeStagingSqlOperations.getCopyQuery(STAGE_PATH, List.of("filename1", "filename2"), TABLE_NAME, SCHEMA_NAME); - assertEquals(expectedQuery, actualCopyQuery); - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java deleted file mode 100644 index b6539f65ddc8..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeS3StreamCopierTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.snowflake; - -import static io.airbyte.integrations.destination.snowflake.SnowflakeS3StreamCopier.MAX_PARTS_PER_FILE; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -import com.amazonaws.services.s3.AmazonS3Client; -import com.google.common.collect.Lists; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class SnowflakeS3StreamCopierTest { - - // equivalent to Thu, 09 Dec 2021 19:17:54 GMT - private static final Timestamp UPLOAD_TIME = Timestamp.from(Instant.ofEpochMilli(1639077474000L)); - - private AmazonS3Client s3Client; - private JdbcDatabase db; - private SqlOperations sqlOperations; - private SnowflakeS3StreamCopier copier; - - @BeforeEach - public void setup() throws Exception { - s3Client = mock(AmazonS3Client.class, RETURNS_DEEP_STUBS); - db = mock(JdbcDatabase.class); - sqlOperations = mock(SqlOperations.class); - - final S3DestinationConfig s3Config = S3DestinationConfig.create( - "fake-bucket", - "fake-bucketPath", - "fake-region") - .withEndpoint("fake-endpoint") - .withAccessKeyCredential("fake-access-key-id", "fake-secret-access-key") - .get(); - - copier = (SnowflakeS3StreamCopier) new SnowflakeS3StreamCopierFactory().create( - // In reality, this is normally a UUID - see CopyConsumerFactory#createWriteConfigs - "fake-staging-folder", - "fake-schema", - s3Client, - db, - new S3CopyConfig(true, s3Config), - new StandardNameTransformer(), - sqlOperations, - new ConfiguredAirbyteStream() - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(new AirbyteStream() - .withName("fake-stream") - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH)) - .withNamespace("fake-namespace"))); - } - - @Test - public void copiesCorrectFilesToTable() throws Exception { - // Generate two files - for (int i = 0; i < MAX_PARTS_PER_FILE + 1; i++) { - copier.prepareStagingFile(); - } - - copier.copyStagingFileToTemporaryTable(); - final Set stagingFiles = copier.getStagingFiles(); - // check the use of all files for staging - Assertions.assertTrue(stagingFiles.size() > 1); - - final List> partition = Lists.partition(new ArrayList<>(stagingFiles), 1000); - for (final List files : partition) { - verify(db).execute(String.format( - "COPY INTO fake-schema.%s FROM '%s' " - + "CREDENTIALS=(aws_key_id='fake-access-key-id' aws_secret_key='fake-secret-access-key') " - + "file_format = (type = csv field_delimiter = ',' skip_header = 0 FIELD_OPTIONALLY_ENCLOSED_BY = '\"') " - + "files = (" + copier.generateFilesList(files) + " );", - copier.getTmpTableName(), - copier.generateBucketPath())); - } - - } - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java index eadef6ad7b76..d02f71ec95c5 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java @@ -12,28 +12,38 @@ import static org.mockito.Mockito.verify; import io.airbyte.commons.functional.CheckedConsumer; +import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.SQLException; import java.util.List; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class SnowflakeSqlOperationsTest { - SnowflakeSqlOperations snowflakeSqlOperations = new SnowflakeSqlOperations(); + private SnowflakeSqlOperations snowflakeSqlOperations; public static String SCHEMA_NAME = "schemaName"; public static final String TABLE_NAME = "tableName"; JdbcDatabase db = mock(JdbcDatabase.class); + @BeforeEach + public void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + snowflakeSqlOperations = new SnowflakeSqlOperations(); + } + @Test void createTableQuery() { String expectedQuery = String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s VARCHAR PRIMARY KEY,\n" - + "%s VARIANT,\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp()\n" - + ") data_retention_time_in_days = 0;", + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s VARIANT, + %s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp() + ) data_retention_time_in_days = 0;""", SCHEMA_NAME, TABLE_NAME, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); String actualQuery = snowflakeSqlOperations.createTableQuery(db, SCHEMA_NAME, TABLE_NAME); assertEquals(expectedQuery, actualQuery); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java index 26ef815b8575..cb131b139189 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java @@ -8,12 +8,15 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.DestinationConfig; import java.sql.SQLException; import java.util.List; import java.util.stream.Stream; import net.snowflake.client.jdbc.SnowflakeSQLException; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.function.Executable; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -37,24 +40,41 @@ class SnowflakeSqlOperationsThrowConfigExceptionTest { private static final String TEST_PERMISSION_EXCEPTION_CATCHED = "but current role has no privileges on it"; private static final String TEST_IP_NOT_IN_WHITE_LIST_EXCEPTION_CATCHED = "not allowed to access Snowflake"; - private static final SnowflakeInternalStagingSqlOperations snowflakeStagingSqlOperations = - new SnowflakeInternalStagingSqlOperations(new SnowflakeSQLNameTransformer()); + private static SnowflakeInternalStagingSqlOperations snowflakeStagingSqlOperations; - private static final SnowflakeSqlOperations snowflakeSqlOperations = new SnowflakeSqlOperations(); + private static SnowflakeSqlOperations snowflakeSqlOperations; - final static JdbcDatabase dbForExecuteQuery = Mockito.mock(JdbcDatabase.class); - final static JdbcDatabase dbForRunUnsafeQuery = Mockito.mock(JdbcDatabase.class); + private static final JdbcDatabase dbForExecuteQuery = Mockito.mock(JdbcDatabase.class); + private static final JdbcDatabase dbForRunUnsafeQuery = Mockito.mock(JdbcDatabase.class); - final static Executable createStageIfNotExists = () -> snowflakeStagingSqlOperations.createStageIfNotExists(dbForExecuteQuery, STAGE_NAME); - final static Executable dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME); - final static Executable cleanUpStage = () -> snowflakeStagingSqlOperations.cleanUpStage(dbForExecuteQuery, STAGE_NAME, FILE_PATH); - final static Executable copyIntoTableFromStage = - () -> snowflakeStagingSqlOperations.copyIntoTableFromStage(dbForExecuteQuery, STAGE_NAME, STAGE_PATH, FILE_PATH, TABLE_NAME, SCHEMA_NAME); + private static Executable createStageIfNotExists; + private static Executable dropStageIfExists; + private static Executable cleanUpStage; + private static Executable copyIntoTableFromStage; - final static Executable createSchemaIfNotExists = () -> snowflakeSqlOperations.createSchemaIfNotExists(dbForExecuteQuery, SCHEMA_NAME); - final static Executable isSchemaExists = () -> snowflakeSqlOperations.isSchemaExists(dbForRunUnsafeQuery, SCHEMA_NAME); - final static Executable createTableIfNotExists = () -> snowflakeSqlOperations.createTableIfNotExists(dbForExecuteQuery, SCHEMA_NAME, TABLE_NAME); - final static Executable dropTableIfExists = () -> snowflakeSqlOperations.dropTableIfExists(dbForExecuteQuery, SCHEMA_NAME, TABLE_NAME); + private static Executable createSchemaIfNotExists; + private static Executable isSchemaExists; + private static Executable createTableIfNotExists; + private static Executable dropTableIfExists; + + @BeforeAll + public static void setup() { + DestinationConfig.initialize(Jsons.emptyObject()); + + snowflakeStagingSqlOperations = new SnowflakeInternalStagingSqlOperations(new SnowflakeSQLNameTransformer()); + snowflakeSqlOperations = new SnowflakeSqlOperations(); + + createStageIfNotExists = () -> snowflakeStagingSqlOperations.createStageIfNotExists(dbForExecuteQuery, STAGE_NAME); + dropStageIfExists = () -> snowflakeStagingSqlOperations.dropStageIfExists(dbForExecuteQuery, STAGE_NAME); + cleanUpStage = () -> snowflakeStagingSqlOperations.cleanUpStage(dbForExecuteQuery, STAGE_NAME, FILE_PATH); + copyIntoTableFromStage = + () -> snowflakeStagingSqlOperations.copyIntoTableFromStage(dbForExecuteQuery, STAGE_NAME, STAGE_PATH, FILE_PATH, TABLE_NAME, SCHEMA_NAME); + + createSchemaIfNotExists = () -> snowflakeSqlOperations.createSchemaIfNotExists(dbForExecuteQuery, SCHEMA_NAME); + isSchemaExists = () -> snowflakeSqlOperations.isSchemaExists(dbForRunUnsafeQuery, SCHEMA_NAME); + createTableIfNotExists = () -> snowflakeSqlOperations.createTableIfNotExists(dbForExecuteQuery, SCHEMA_NAME, TABLE_NAME); + dropTableIfExists = () -> snowflakeSqlOperations.dropTableIfExists(dbForExecuteQuery, SCHEMA_NAME, TABLE_NAME); + } private static Stream testArgumentsForDbExecute() { return Stream.of( diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_gcs_config.json b/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_gcs_config.json deleted file mode 100644 index 3bcf98a5736d..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_gcs_config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "host": "testhost.snowflakecomputing.com", - "role": "AIRBYTE_ROLE", - "warehouse": "AIRBYTE_WAREHOUSE", - "database": "AIRBYTE_DATABASE", - "schema": "AIRBYTE_SCHEMA", - "username": "AIRBYTE_USER", - "credentials": { - "password": "test" - }, - "loading_method": { - "method": "GCS Staging", - "project_id": "test", - "bucket_name": "test", - "credentials_json": "{\n\"type\": \"test\"}\n" - } -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_s3_config.json b/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_s3_config.json deleted file mode 100644 index bf55f9a2fd92..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_s3_config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "host": "testhost.snowflakecomputing.com", - "role": "AIRBYTE_ROLE", - "warehouse": "AIRBYTE_WAREHOUSE", - "database": "AIRBYTE_DATABASE", - "schema": "AIRBYTE_SCHEMA", - "username": "AIRBYTE_USER", - "credentials": { - "password": "test" - }, - "loading_method": { - "method": "S3 Staging", - "s3_bucket_name": "airbyte-snowflake-integration-tests", - "s3_bucket_region": "us-east-2", - "access_key_id": "test", - "secret_access_key": "test" - } -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_s3_encrypted_config.json b/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_s3_encrypted_config.json deleted file mode 100644 index e0c5e3b62344..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/resources/copy_s3_encrypted_config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "host": "testhost.snowflakecomputing.com", - "role": "AIRBYTE_ROLE", - "warehouse": "AIRBYTE_WAREHOUSE", - "database": "AIRBYTE_DATABASE", - "schema": "AIRBYTE_SCHEMA", - "username": "AIRBYTE_USER", - "credentials": { - "password": "test" - }, - "loading_method": { - "method": "S3 Staging", - "s3_bucket_name": "airbyte-snowflake-integration-tests", - "s3_bucket_region": "us-east-2", - "access_key_id": "test", - "secret_access_key": "test", - "encryption": { - "encryption_type": "aes_cbc_envelope" - } - } -} diff --git a/airbyte-integrations/connectors/destination-sqlite/metadata.yaml b/airbyte-integrations/connectors/destination-sqlite/metadata.yaml index f92f1510b6a9..679b7722c374 100644 --- a/airbyte-integrations/connectors/destination-sqlite/metadata.yaml +++ b/airbyte-integrations/connectors/destination-sqlite/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/local-sqlite + documentationUrl: https://docs.airbyte.com/integrations/destinations/sqlite tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/destination-starburst-galaxy/integration_tests/configured_catalog.json index da8be1ef3bd2..f9f0e034ad2c 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/integration_tests/configured_catalog.json @@ -1,7 +1,7 @@ { "streams": [ { - "stream" : { + "stream": { "name": "users", "json_schema": { "type": "object", @@ -21,4 +21,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/metadata.yaml b/airbyte-integrations/connectors/destination-starburst-galaxy/metadata.yaml index 6cfb75c93754..d9831e3bb121 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/metadata.yaml +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/starburst-galaxy tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/resources/spec.json index 96e14b888bad..34f0e3fca5b1 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/resources/spec.json @@ -4,7 +4,14 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Starburst Galaxy Destination Spec", "type": "object", - "required": [ "accept_terms", "server_hostname", "username", "password", "catalog", "staging_object_store" ], + "required": [ + "accept_terms", + "server_hostname", + "username", + "password", + "catalog", + "staging_object_store" + ], "properties": { "accept_terms": { "title": "Agree to the Starburst Galaxy terms & conditions", @@ -17,7 +24,7 @@ "title": "Hostname", "type": "string", "description": "Starburst Galaxy cluster hostname.", - "examples": [ "abc-12345678-wxyz.trino.galaxy-demo.io" ], + "examples": ["abc-12345678-wxyz.trino.galaxy-demo.io"], "order": 2 }, "port": { @@ -25,21 +32,21 @@ "type": "string", "description": "Starburst Galaxy cluster port.", "default": "443", - "examples": [ "443" ], + "examples": ["443"], "order": 3 }, "username": { "title": "User", "type": "string", "description": "Starburst Galaxy user.", - "examples": [ "user@example.com" ], + "examples": ["user@example.com"], "order": 4 }, "password": { "title": "Password", "type": "string", "description": "Starburst Galaxy password for the specified user.", - "examples": [ "password" ], + "examples": ["password"], "airbyte_secret": true, "order": 5 }, @@ -47,7 +54,7 @@ "title": "Amazon S3 catalog", "type": "string", "description": "Name of the Starburst Galaxy Amazon S3 catalog.", - "examples": [ "sample_s3_catalog" ], + "examples": ["sample_s3_catalog"], "order": 6 }, "catalog_schema": { @@ -55,63 +62,85 @@ "type": "string", "description": "The default Starburst Galaxy Amazon S3 catalog schema where tables are written to if the source does not specify a namespace. Defaults to \"public\".", "default": "public", - "examples": [ "public" ], + "examples": ["public"], "order": 7 }, "staging_object_store": { "title": "Staging object store", "type": "object", "description": "Temporary storage on which temporary Iceberg table is created.", - "oneOf": [ { - "title": "Amazon S3", - "required": [ "object_store_type", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "s3_access_key_id", "s3_secret_access_key" ], - "properties": { - "object_store_type": { - "type": "string", - "enum": [ "S3" ], - "default": "S3", - "order": 1 - }, - "s3_bucket_name": { - "title": "S3 bucket name", - "type": "string", - "description": "Name of the S3 bucket", - "examples": [ "airbyte_staging" ], - "order": 1 - }, - "s3_bucket_path": { - "title": "S3 bucket path", - "type": "string", - "description": "Directory in the S3 bucket where staging data is stored.", - "examples": [ "temp_airbyte__sync/test" ], - "order": 2 - }, - "s3_bucket_region": { - "title": "S3 bucket region", - "type": "string", - "default": "us-east-1", - "description": "The region of the S3 bucket.", - "enum": [ "ap-northeast-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "us-east-1", "us-east-2", "us-west-1", "us-west-2" ], - "order": 3 - }, - "s3_access_key_id": { - "title": "Access key", - "type": "string", - "description": "Access key with access to the bucket. Airbyte requires read and write permissions to a given bucket.", - "examples": [ "A012345678910EXAMPLE" ], - "airbyte_secret": true, - "order": 4 - }, - "s3_secret_access_key": { - "title": "Secret key", - "type": "string", - "description": "Secret key used with the specified access key.", - "examples": [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ], - "airbyte_secret": true, - "order": 5 + "oneOf": [ + { + "title": "Amazon S3", + "required": [ + "object_store_type", + "s3_bucket_name", + "s3_bucket_path", + "s3_bucket_region", + "s3_access_key_id", + "s3_secret_access_key" + ], + "properties": { + "object_store_type": { + "type": "string", + "enum": ["S3"], + "default": "S3", + "order": 1 + }, + "s3_bucket_name": { + "title": "S3 bucket name", + "type": "string", + "description": "Name of the S3 bucket", + "examples": ["airbyte_staging"], + "order": 1 + }, + "s3_bucket_path": { + "title": "S3 bucket path", + "type": "string", + "description": "Directory in the S3 bucket where staging data is stored.", + "examples": ["temp_airbyte__sync/test"], + "order": 2 + }, + "s3_bucket_region": { + "title": "S3 bucket region", + "type": "string", + "default": "us-east-1", + "description": "The region of the S3 bucket.", + "enum": [ + "ap-northeast-1", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "eu-central-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2" + ], + "order": 3 + }, + "s3_access_key_id": { + "title": "Access key", + "type": "string", + "description": "Access key with access to the bucket. Airbyte requires read and write permissions to a given bucket.", + "examples": ["A012345678910EXAMPLE"], + "airbyte_secret": true, + "order": 4 + }, + "s3_secret_access_key": { + "title": "Secret key", + "type": "string", + "description": "Secret key used with the specified access key.", + "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"], + "airbyte_secret": true, + "order": 5 + } } } - } ], + ], "order": 8 }, "purge_staging_table": { @@ -126,5 +155,5 @@ "supportsIncremental": true, "supportsNormalization": false, "supportsDBT": false, - "supported_destination_sync_modes": [ "overwrite", "append" ] -} \ No newline at end of file + "supported_destination_sync_modes": ["overwrite", "append"] +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java index 92a9bce20da6..8e3cf6fea561 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java @@ -45,6 +45,7 @@ import java.nio.file.Path; import java.sql.SQLException; import java.time.Instant; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -67,7 +68,7 @@ public abstract class StarburstGalaxyDestinationAcceptanceTest extends Destinati private Database database; @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { dslContext = create(galaxyDestinationConfig.galaxyUsername(), galaxyDestinationConfig.galaxyPassword(), STARBURST_GALAXY_DRIVER_CLASS, getGalaxyConnectionString(galaxyDestinationConfig), SQLDialect.DEFAULT); database = new Database(dslContext); @@ -108,21 +109,21 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, @Override protected void tearDown(final TestDestinationEnv testEnv) throws SQLException { // clean up database - List schemas = executeQuery(format("SHOW SCHEMAS LIKE '%s'", galaxyDestinationConfig.galaxyCatalogSchema().toLowerCase(ENGLISH))); + final List schemas = executeQuery(format("SHOW SCHEMAS LIKE '%s'", galaxyDestinationConfig.galaxyCatalogSchema().toLowerCase(ENGLISH))); schemas.stream().map(node -> node.get("Schema").asText()) .forEach(schema -> { try { - List tables = executeQuery(format("SHOW TABLES FROM %s", galaxyDestinationConfig.galaxyCatalogSchema())); + final List tables = executeQuery(format("SHOW TABLES FROM %s", galaxyDestinationConfig.galaxyCatalogSchema())); tables.forEach(table -> { try { - String tableName = table.get("Table").asText(); + final String tableName = table.get("Table").asText(); LOGGER.info("Dropping table : {}.{}", schema, tableName); executeQuery(format("DROP TABLE IF EXISTS %s.%s", schema, tableName)); - } catch (SQLException e) { + } catch (final SQLException e) { throw new RuntimeException(e); } }); - } catch (SQLException e) { + } catch (final SQLException e) { throw new RuntimeException(e); } }); @@ -131,12 +132,12 @@ protected void tearDown(final TestDestinationEnv testEnv) throws SQLException { dslContext.close(); } - private List executeQuery(ContextQueryFunction> transform) + private List executeQuery(final ContextQueryFunction> transform) throws SQLException { return database.query(transform); } - private List executeQuery(String query) + private List executeQuery(final String query) throws SQLException { return executeQuery(ctx -> ctx.resultQuery(query) .stream() @@ -146,31 +147,31 @@ private List executeQuery(String query) @Test public void testPromoteSourceSchemaChanges() throws Exception { - String sampleStream = "sample_stream_1"; + final String sampleStream = "sample_stream_1"; testStreamSync(OVERWRITE, sampleStream, "schema-overwrite.json", "data-overwrite.json", "expected-schema-overwrite.json"); testStreamSync(APPEND, sampleStream, "schema-append.json", "data-append.json", "expected-schema-append.json"); assertEquals(2, executeQuery(format("SELECT COUNT(*) FROM %s.%s", galaxyDestinationConfig.galaxyCatalogSchema(), sampleStream)).get(0).get("_col0").asInt()); } - private void testStreamSync(DestinationSyncMode syncMode, - String streamName, - String schemaFileName, - String dataFileName, - String expectedSchemaFileName) + private void testStreamSync(final DestinationSyncMode syncMode, + final String streamName, + final String schemaFileName, + final String dataFileName, + final String expectedSchemaFileName) throws Exception { - JsonNode overwriteSchema = getTestDataFromResourceJson(schemaFileName); - AirbyteMessage overwriteMessage = createRecordMessage(streamName, getTestDataFromResourceJson(dataFileName)); + final JsonNode overwriteSchema = getTestDataFromResourceJson(schemaFileName); + final AirbyteMessage overwriteMessage = createRecordMessage(streamName, getTestDataFromResourceJson(dataFileName)); runDestinationWrite(getCommonCatalog(streamName, overwriteSchema, syncMode), configJson, overwriteMessage); validateTableSchema(streamName, expectedSchemaFileName); } - private void validateTableSchema(String streamName, String expectedSchemaFileName) + private void validateTableSchema(final String streamName, final String expectedSchemaFileName) throws SQLException { - List describeRecords = executeQuery(format("DESCRIBE %s.%s", galaxyDestinationConfig.galaxyCatalogSchema(), streamName)); - Map actualDataTypes = + final List describeRecords = executeQuery(format("DESCRIBE %s.%s", galaxyDestinationConfig.galaxyCatalogSchema(), streamName)); + final Map actualDataTypes = describeRecords.stream().collect(Collectors.toMap(column -> column.get("Column").asText(), column -> column.get("Type").asText())); - JsonNode expectedDataTypes = getTestDataFromResourceJson(expectedSchemaFileName); + final JsonNode expectedDataTypes = getTestDataFromResourceJson(expectedSchemaFileName); assertEquals(expectedDataTypes.size(), actualDataTypes.size()); expectedDataTypes.fields().forEachRemaining(field -> assertEquals(field.getValue().asText(), actualDataTypes.get(field.getKey()))); } @@ -185,17 +186,21 @@ public void testJsonV1Types() throws Exception { testDifferentTypes("sample_stream_3", "datatypeV1.json", "dataV1.json", "expected-datatypeV1.json", "expected-dataV1.json"); } - private void testDifferentTypes(String streamName, String dataTypeFile, String dataFile, String expectedDataTypeFile, String expectedDataFile) + private void testDifferentTypes(final String streamName, + final String dataTypeFile, + final String dataFile, + final String expectedDataTypeFile, + final String expectedDataFile) throws Exception { - JsonNode datatypeSchema = getTestDataFromResourceJson(dataTypeFile); - AirbyteMessage datatypeMessage = createRecordMessage(streamName, getTestDataFromResourceJson(dataFile)); + final JsonNode datatypeSchema = getTestDataFromResourceJson(dataTypeFile); + final AirbyteMessage datatypeMessage = createRecordMessage(streamName, getTestDataFromResourceJson(dataFile)); runDestinationWrite(getCommonCatalog(streamName, datatypeSchema, OVERWRITE), configJson, datatypeMessage); final JsonFieldNameUpdater nameUpdater = AvroRecordHelper.getFieldNameUpdater(streamName, galaxyDestinationConfig.galaxyCatalogSchema(), datatypeSchema); validateTableSchema(streamName, expectedDataTypeFile); - List records = executeQuery(ctx -> ctx.select(asterisk()) + final List records = executeQuery(ctx -> ctx.select(asterisk()) .from(format("%s.%s", galaxyDestinationConfig.galaxyCatalogSchema(), streamName)) .orderBy(field(COLUMN_NAME_EMITTED_AT).asc()) .fetch().stream() @@ -205,30 +210,31 @@ private void testDifferentTypes(String streamName, String dataTypeFile, String d return pruneAirbyteJson(jsonWithOriginalFields); }) .collect(toList())); - JsonNode actualData = records.get(0); - JsonNode expectedData = getTestDataFromResourceJson(expectedDataFile); + final JsonNode actualData = records.get(0); + final JsonNode expectedData = getTestDataFromResourceJson(expectedDataFile); assertEquals(expectedData.size(), actualData.size()); expectedData.fields().forEachRemaining(field -> assertEquals(field.getValue(), actualData.get(field.getKey()))); } - private static AirbyteMessage createRecordMessage(String streamName, final JsonNode data) { + private static AirbyteMessage createRecordMessage(final String streamName, final JsonNode data) { return new AirbyteMessage() .withType(RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withData(data).withEmittedAt(Instant.now().toEpochMilli())); } - public static ConfiguredAirbyteCatalog getCommonCatalog(String stream, final JsonNode schema, DestinationSyncMode destinationSyncMode) { + public static ConfiguredAirbyteCatalog getCommonCatalog(final String stream, final JsonNode schema, final DestinationSyncMode destinationSyncMode) { return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(new ConfiguredAirbyteStream() .withStream(new AirbyteStream().withName(stream).withJsonSchema(schema) .withSupportedSyncModes(Lists.newArrayList(FULL_REFRESH))) .withSyncMode(FULL_REFRESH).withDestinationSyncMode(destinationSyncMode))); } - private static void runDestinationWrite(ConfiguredAirbyteCatalog catalog, JsonNode config, AirbyteMessage... messages) throws Exception { + private static void runDestinationWrite(final ConfiguredAirbyteCatalog catalog, final JsonNode config, final AirbyteMessage... messages) + throws Exception { final StarburstGalaxyDestination destination = new StarburstGalaxyDestination(); final AirbyteMessageConsumer consumer = destination.getConsumer(config, catalog, Destination::defaultOutputRecordCollector); consumer.start(); - for (AirbyteMessage message : messages) { + for (final AirbyteMessage message : messages) { consumer.accept(message); } consumer.close(); @@ -236,7 +242,7 @@ private static void runDestinationWrite(ConfiguredAirbyteCatalog catalog, JsonNo private static JsonNode getTestDataFromResourceJson(final String fileName) { try { - String fileContent = readString(Path.of(Objects.requireNonNull(StarburstGalaxyDestinationAcceptanceTest.class.getClassLoader() + final String fileContent = readString(Path.of(Objects.requireNonNull(StarburstGalaxyDestinationAcceptanceTest.class.getClassLoader() .getResource(INPUT_FILES_BASE_LOCATION + fileName)).getPath())); return Jsons.deserialize(fileContent); } catch (final IOException e) { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java index 460e564a0457..459b2580eaab 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java @@ -20,6 +20,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.s3.S3DestinationConfig; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import org.slf4j.Logger; @@ -40,7 +41,7 @@ protected JsonNode getFailCheckConfig() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { JsonNode baseConfigJson = Jsons.deserialize(IOs.readFile(Path.of(SECRETS_CONFIG_JSON))); // Set a random s3 bucket path and database schema for each integration test @@ -55,7 +56,7 @@ protected void setup(TestDestinationEnv testEnv) { S3DestinationConfig s3Config = galaxyDestinationConfig.storageConfig().getS3DestinationConfigOrThrow(); LOGGER.info("Test full path: s3://{}/{}", s3Config.getBucketName(), s3Config.getBucketPath()); - super.setup(testEnv); // Create a database + super.setup(testEnv, TEST_SCHEMAS); // Create a database } @Override diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/data-append.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/data-append.json index c29e7ddb1437..7e49453ccad3 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/data-append.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/data-append.json @@ -3,5 +3,5 @@ "c_row": { "first_name": "charles" }, - "c_new_column" : 345 + "c_new_column": 345 } diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV0.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV0.json index 20b5a73a7124..e0c1ae5b9f4b 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV0.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV0.json @@ -1,18 +1,18 @@ { - "string_type" : "sample_string", - "number_integer_type" : 948, - "string_big_integer_type" : "54667543256786653424", - "number_float_type" : 34.657322, - "number_type" : 78.32, - "integer_type" : 5389, - "boolean_type" : false, - "date_time_with_tz_type" : "2021-07-12T03:12:22+05:00", - "date_time_without_tz_type" : "2023-11-07T02:10:32", - "date_type" : "2021-01-01", + "string_type": "sample_string", + "number_integer_type": 948, + "string_big_integer_type": "54667543256786653424", + "number_float_type": 34.657322, + "number_type": 78.32, + "integer_type": 5389, + "boolean_type": false, + "date_time_with_tz_type": "2021-07-12T03:12:22+05:00", + "date_time_without_tz_type": "2023-11-07T02:10:32", + "date_type": "2021-01-01", "time_type": "12:23:01.541", "array_type": ["151", "152"], "row_type": { "first_name": "charles", "last_name": "darwin" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV1.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV1.json index f25dd7f7b706..632eaf6f39a2 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV1.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/dataV1.json @@ -1,12 +1,12 @@ { - "string_type" : "sample_string", - "integer_type" : 5389, - "number_type" : 78.32, - "boolean_type" : false, - "binary_type" : "FDX", - "date_type" : "2021-01-01", - "timestamp_with_tz_type" : "2021-01-01T07:06:13+05:00", - "timestamp_without_tz_type" : "2023-03-01T04:05:01", - "time_with_tz_type" : "12:23:01.541+05:00", - "time_without_tz_type" : "12:16:04.541Z" -} \ No newline at end of file + "string_type": "sample_string", + "integer_type": 5389, + "number_type": 78.32, + "boolean_type": false, + "binary_type": "FDX", + "date_type": "2021-01-01", + "timestamp_with_tz_type": "2021-01-01T07:06:13+05:00", + "timestamp_without_tz_type": "2023-03-01T04:05:01", + "time_with_tz_type": "12:23:01.541+05:00", + "time_without_tz_type": "12:16:04.541Z" +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV0.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV0.json index b6babed11902..4bf92d0def69 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV0.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV0.json @@ -6,15 +6,15 @@ }, "number_integer_type": { "type": "number", - "airbyte_type" : "integer" + "airbyte_type": "integer" }, "string_big_integer_type": { "type": "string", - "airbyte_type" : "big_integer" + "airbyte_type": "big_integer" }, "number_float_type": { "type": "number", - "airbyte_type" : "float" + "airbyte_type": "float" }, "number_type": { "type": "number" @@ -59,4 +59,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV1.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV1.json index dc1f5f144e45..9a3d5d9fa718 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV1.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/datatypeV1.json @@ -32,4 +32,4 @@ "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV0.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV0.json index 31f5c47e0c5a..4dbce39c0cb1 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV0.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV0.json @@ -1,15 +1,15 @@ { - "string_type" : "sample_string", - "number_integer_type" : 948, - "string_big_integer_type" : "54667543256786653424", - "number_float_type" : 34.657322, - "number_type" : 78.32, - "integer_type" : 5389, - "boolean_type" : false, - "date_time_with_tz_type" : "2021-07-11T22:12:22Z", - "date_time_without_tz_type" : "2023-11-07T02:10:32Z", - "date_type" : "2021-01-01", - "time_type" : "12:23:01", + "string_type": "sample_string", + "number_integer_type": 948, + "string_big_integer_type": "54667543256786653424", + "number_float_type": 34.657322, + "number_type": 78.32, + "integer_type": 5389, + "boolean_type": false, + "date_time_with_tz_type": "2021-07-11T22:12:22Z", + "date_time_without_tz_type": "2023-11-07T02:10:32Z", + "date_type": "2021-01-01", + "time_type": "12:23:01", "array_type": "[151, 152]", "row_type": "{first_name=charles, last_name=darwin, _airbyte_additional_properties=null}" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV1.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV1.json index c2034ac0c965..d0598d10819a 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV1.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-dataV1.json @@ -1,12 +1,12 @@ { - "string_type" : "sample_string", - "integer_type" : 5389, - "number_type" : 78.32, - "boolean_type" : false, - "binary_type" : "RkRY", - "date_type" : "2021-01-01", - "timestamp_with_tz_type" : "2021-01-01T02:06:13Z", - "timestamp_without_tz_type" : "2023-03-01T04:05:01Z", - "time_with_tz_type" : "12:23:01.541+05:00", - "time_without_tz_type" : "12:16:04" -} \ No newline at end of file + "string_type": "sample_string", + "integer_type": 5389, + "number_type": 78.32, + "boolean_type": false, + "binary_type": "RkRY", + "date_type": "2021-01-01", + "timestamp_with_tz_type": "2021-01-01T02:06:13Z", + "timestamp_without_tz_type": "2023-03-01T04:05:01Z", + "time_with_tz_type": "12:23:01.541+05:00", + "time_without_tz_type": "12:16:04" +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV0.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV0.json index 042a3bf74f1f..4f8668231140 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV0.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV0.json @@ -1,18 +1,18 @@ { - "_airbyte_ab_id" : "varchar", - "_airbyte_emitted_at" : "timestamp(6) with time zone", - "string_type" : "varchar", - "number_integer_type" : "integer", - "string_big_integer_type" : "varchar", - "number_float_type" : "real", - "number_type" : "double", - "integer_type" : "bigint", - "boolean_type" : "boolean", - "date_time_without_tz_type" : "timestamp(6) with time zone", - "date_time_with_tz_type" : "timestamp(6) with time zone", - "date_type" : "date", - "time_type" : "time(6)", - "array_type" : "array(varchar)", - "row_type" : "row(first_name varchar, last_name varchar, _airbyte_additional_properties map(varchar, varchar))", - "_airbyte_additional_properties" : "map(varchar, varchar)" -} \ No newline at end of file + "_airbyte_ab_id": "varchar", + "_airbyte_emitted_at": "timestamp(6) with time zone", + "string_type": "varchar", + "number_integer_type": "integer", + "string_big_integer_type": "varchar", + "number_float_type": "real", + "number_type": "double", + "integer_type": "bigint", + "boolean_type": "boolean", + "date_time_without_tz_type": "timestamp(6) with time zone", + "date_time_with_tz_type": "timestamp(6) with time zone", + "date_type": "date", + "time_type": "time(6)", + "array_type": "array(varchar)", + "row_type": "row(first_name varchar, last_name varchar, _airbyte_additional_properties map(varchar, varchar))", + "_airbyte_additional_properties": "map(varchar, varchar)" +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV1.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV1.json index ebac7cbcf9df..a7b57bbbb74a 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV1.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-datatypeV1.json @@ -1,15 +1,15 @@ { - "_airbyte_ab_id" : "varchar", - "_airbyte_emitted_at" : "timestamp(6) with time zone", - "string_type" : "varchar", - "integer_type" : "bigint", - "number_type" : "double", - "boolean_type" : "boolean", - "binary_type" : "varbinary", - "date_type" : "date", - "timestamp_with_tz_type" : "timestamp(6) with time zone", - "timestamp_without_tz_type" : "timestamp(6) with time zone", - "time_with_tz_type" : "varchar", - "time_without_tz_type" : "time(6)", - "_airbyte_additional_properties" : "map(varchar, varchar)" -} \ No newline at end of file + "_airbyte_ab_id": "varchar", + "_airbyte_emitted_at": "timestamp(6) with time zone", + "string_type": "varchar", + "integer_type": "bigint", + "number_type": "double", + "boolean_type": "boolean", + "binary_type": "varbinary", + "date_type": "date", + "timestamp_with_tz_type": "timestamp(6) with time zone", + "timestamp_without_tz_type": "timestamp(6) with time zone", + "time_with_tz_type": "varchar", + "time_without_tz_type": "time(6)", + "_airbyte_additional_properties": "map(varchar, varchar)" +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-append.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-append.json index 7f09a6b6b363..3c791c1f9d5d 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-append.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-append.json @@ -1,8 +1,8 @@ { - "_airbyte_ab_id" : "varchar", - "c_row" : "row(first_name varchar, _airbyte_additional_properties map(varchar, varchar))", - "_airbyte_additional_properties" : "map(varchar, varchar)", - "c_array" : "array(varchar)", - "_airbyte_emitted_at" : "timestamp(6) with time zone", - "c_new_column" : "bigint" -} \ No newline at end of file + "_airbyte_ab_id": "varchar", + "c_row": "row(first_name varchar, _airbyte_additional_properties map(varchar, varchar))", + "_airbyte_additional_properties": "map(varchar, varchar)", + "c_array": "array(varchar)", + "_airbyte_emitted_at": "timestamp(6) with time zone", + "c_new_column": "bigint" +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-overwrite.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-overwrite.json index 2a31cd4ec5cc..0d2e687ed325 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-overwrite.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/expected-schema-overwrite.json @@ -1,7 +1,7 @@ { - "_airbyte_ab_id" : "varchar", - "c_row" : "row(first_name varchar, _airbyte_additional_properties map(varchar, varchar))", - "_airbyte_additional_properties" : "map(varchar, varchar)", - "c_array" : "array(varchar)", - "_airbyte_emitted_at" : "timestamp(6) with time zone" -} \ No newline at end of file + "_airbyte_ab_id": "varchar", + "c_row": "row(first_name varchar, _airbyte_additional_properties map(varchar, varchar))", + "_airbyte_additional_properties": "map(varchar, varchar)", + "c_array": "array(varchar)", + "_airbyte_emitted_at": "timestamp(6) with time zone" +} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-append.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-append.json index 5da1007582f8..f6322d6814b6 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-append.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-append.json @@ -11,8 +11,7 @@ }, "c_array": { "type": "array", - "items": - { + "items": { "type": "string" } }, diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-overwrite.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-overwrite.json index ab7cf67917f6..bd4f7f084f7c 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-overwrite.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/resources/testdata/schema-overwrite.json @@ -11,8 +11,7 @@ }, "c_array": { "type": "array", - "items": - { + "items": { "type": "string" } } diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v0.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v0.json index 628d4718a7e0..d6ee7b39479c 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v0.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v0.json @@ -9,11 +9,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -37,12 +37,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"user\"" : "row(first_name varchar, last_name varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"user\"": "row(first_name varchar, last_name varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -55,11 +55,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifier\"" : "row(member0 double, member1 varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifier\"": "row(member0 double, member1 varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -75,11 +75,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifier\"" : "array(varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifier\"": "array(varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -106,11 +106,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifiers\"" : "array(row(member0 varchar, member1 bigint, member2 boolean))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifiers\"": "array(row(member0 varchar, member1 bigint, member2 boolean))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -133,11 +133,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"created_at\"" : "row(member0 varchar, member1 bigint)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"created_at\"": "row(member0 varchar, member1 bigint)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -165,11 +165,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"user\"" : "row(created_at row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"user\"": "row(created_at row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -196,11 +196,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifiers\"" : "array(row(member0 bigint, member1 varchar, member2 boolean))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifiers\"": "array(row(member0 bigint, member1 varchar, member2 boolean))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -216,11 +216,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -236,11 +236,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -248,10 +248,10 @@ "jsonSchema": { "type": "object" }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -296,12 +296,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"author\"" : "row(id bigint, _airbyte_additional_properties map(varchar, varchar))", - "\"commit\"" : "row(message varchar, author row(name varchar, pr row(id varchar, message varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"author\"": "row(id bigint, _airbyte_additional_properties map(varchar, varchar))", + "\"commit\"": "row(message varchar, author row(name varchar, pr row(id varchar, message varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -314,11 +314,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifier\"" : "array(varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifier\"": "array(varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -374,11 +374,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"parent_object\"" : "row(object_array array(row(member0 bigint, member1 boolean, member2 row(id row(id_part_1 row(member0 bigint, member1 varchar), id_part_2 row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar)), _message varchar, _airbyte_additional_properties map(varchar, varchar)))), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"parent_object\"": "row(object_array array(row(member0 bigint, member1 boolean, member2 row(id row(id_part_1 row(member0 bigint, member1 varchar), id_part_2 row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar)), _message varchar, _airbyte_additional_properties map(varchar, varchar)))), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -402,11 +402,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"filters\"" : "array(array(row(filterFamily varchar, _airbyte_additional_properties map(varchar, varchar))))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"filters\"": "array(array(row(filterFamily varchar, _airbyte_additional_properties map(varchar, varchar))))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -420,11 +420,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"array_field\"" : "array(varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"array_field\"": "array(varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -437,11 +437,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "double", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "double", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -476,12 +476,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"same_record_name_field\"" : "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"any_of_field\"" : "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"same_record_name_field\"": "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"any_of_field\"": "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -516,12 +516,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"same_record_name_field\"" : "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"all_of_field\"" : "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"same_record_name_field\"": "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"all_of_field\"": "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -556,12 +556,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"same_record_name_field\"" : "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"any_of_field\"" : "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"same_record_name_field\"": "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"any_of_field\"": "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -569,7 +569,7 @@ "jsonSchema": { "type": "object", "properties": { - "null_type" : { + "null_type": { "type": "null" }, "string_type": { @@ -577,15 +577,15 @@ }, "number_integer_type": { "type": "number", - "airbyte_type" : "integer" + "airbyte_type": "integer" }, "string_big_integer_type": { "type": "string", - "airbyte_type" : "big_integer" + "airbyte_type": "big_integer" }, "number_float_type": { "type": "number", - "airbyte_type" : "float" + "airbyte_type": "float" }, "number_type": { "type": "number" @@ -610,20 +610,20 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"string_type\"" : "varchar", - "\"number_integer_type\"" : "integer", - "\"string_big_integer_type\"" : "varchar", - "\"number_float_type\"" : "real", - "\"number_type\"" : "double", - "\"integer_type\"" : "bigint", - "\"boolean_type\"" : "boolean", - "\"date_time_type\"" : "timestamp(6) with time zone", - "\"date_type\"" : "date", - "\"time_type\"" : "time(6)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"string_type\"": "varchar", + "\"number_integer_type\"": "integer", + "\"string_big_integer_type\"": "varchar", + "\"number_float_type\"": "real", + "\"number_type\"": "double", + "\"integer_type\"": "bigint", + "\"boolean_type\"": "boolean", + "\"date_time_type\"": "timestamp(6) with time zone", + "\"date_type\"": "date", + "\"time_type\"": "time(6)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } } ] diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v1.json b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v1.json index 98c8eede9b33..28a55c3efe5f 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v1.json +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/resources/schemas/type_conversion_test_cases_v1.json @@ -9,11 +9,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -37,12 +37,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"user\"" : "row(first_name varchar, last_name varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"user\"": "row(first_name varchar, last_name varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -58,11 +58,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifier\"" : "row(member0 double, member1 varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifier\"": "row(member0 double, member1 varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -78,11 +78,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifier\"" : "array(varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifier\"": "array(varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -101,11 +101,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifiers\"" : "array(row(member0 varchar, member1 bigint, member2 boolean))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifiers\"": "array(row(member0 varchar, member1 bigint, member2 boolean))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -127,11 +127,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"created_at\"" : "row(member0 varchar, member1 bigint)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"created_at\"": "row(member0 varchar, member1 bigint)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -158,11 +158,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"user\"" : "row(created_at row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"user\"": "row(created_at row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -189,11 +189,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifiers\"" : "array(row(member0 bigint, member1 varchar, member2 boolean))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifiers\"": "array(row(member0 bigint, member1 varchar, member2 boolean))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -209,11 +209,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -229,11 +229,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "varchar", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "varchar", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -241,10 +241,10 @@ "jsonSchema": { "type": "object" }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -289,12 +289,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"author\"" : "row(id bigint, _airbyte_additional_properties map(varchar, varchar))", - "\"commit\"" : "row(message varchar, author row(name varchar, pr row(id varchar, message varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"author\"": "row(id bigint, _airbyte_additional_properties map(varchar, varchar))", + "\"commit\"": "row(message varchar, author row(name varchar, pr row(id varchar, message varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -307,11 +307,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"identifier\"" : "array(varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"identifier\"": "array(varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -367,11 +367,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"parent_object\"" : "row(object_array array(row(member0 bigint, member1 boolean, member2 row(id row(id_part_1 row(member0 bigint, member1 varchar), id_part_2 row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar)), _message varchar, _airbyte_additional_properties map(varchar, varchar)))), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"parent_object\"": "row(object_array array(row(member0 bigint, member1 boolean, member2 row(id row(id_part_1 row(member0 bigint, member1 varchar), id_part_2 row(member0 varchar, member1 bigint), _airbyte_additional_properties map(varchar, varchar)), _message varchar, _airbyte_additional_properties map(varchar, varchar)))), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -395,11 +395,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"filters\"" : "array(array(row(filterFamily varchar, _airbyte_additional_properties map(varchar, varchar))))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"filters\"": "array(array(row(filterFamily varchar, _airbyte_additional_properties map(varchar, varchar))))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -413,11 +413,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"array_field\"" : "array(varchar)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"array_field\"": "array(varchar)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -430,11 +430,11 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"node_id\"" : "double", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"node_id\"": "double", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -469,12 +469,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"same_record_name_field\"" : "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"any_of_field\"" : "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"same_record_name_field\"": "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"any_of_field\"": "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -509,12 +509,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"same_record_name_field\"" : "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"all_of_field\"" : "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"same_record_name_field\"": "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"all_of_field\"": "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -549,12 +549,12 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"same_record_name_field\"" : "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", - "\"any_of_field\"" : "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"same_record_name_field\"": "row(sub_field_1 varchar, _airbyte_additional_properties map(varchar, varchar))", + "\"any_of_field\"": "row(same_record_name_field row(sub_field_2 varchar, _airbyte_additional_properties map(varchar, varchar)), _airbyte_additional_properties map(varchar, varchar))", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } }, { @@ -594,20 +594,20 @@ } } }, - "galaxyIcebergSchema" : { - "\"_airbyte_ab_id\"" : "varchar", - "\"_airbyte_emitted_at\"" : "timestamp(6) with time zone", - "\"string_type\"" : "varchar", - "\"integer_type\"" : "bigint", - "\"number_type\"" : "double", - "\"boolean_type\"" : "boolean", - "\"binary_type\"" : "varbinary", - "\"date_type\"" : "date", - "\"timestamp_with_tz_type\"" : "timestamp(6) with time zone", - "\"timestamp_without_tz_type\"" : "timestamp(6) with time zone", - "\"time_with_tz_type\"" : "varchar", - "\"time_without_tz_type\"" : "time(6)", - "\"_airbyte_additional_properties\"" : "map(varchar, varchar)" + "galaxyIcebergSchema": { + "\"_airbyte_ab_id\"": "varchar", + "\"_airbyte_emitted_at\"": "timestamp(6) with time zone", + "\"string_type\"": "varchar", + "\"integer_type\"": "bigint", + "\"number_type\"": "double", + "\"boolean_type\"": "boolean", + "\"binary_type\"": "varbinary", + "\"date_type\"": "date", + "\"timestamp_with_tz_type\"": "timestamp(6) with time zone", + "\"timestamp_without_tz_type\"": "timestamp(6) with time zone", + "\"time_with_tz_type\"": "varchar", + "\"time_without_tz_type\"": "time(6)", + "\"_airbyte_additional_properties\"": "map(varchar, varchar)" } } ] diff --git a/airbyte-integrations/connectors/destination-teradata/Dockerfile b/airbyte-integrations/connectors/destination-teradata/Dockerfile index 655dea677eb0..85b0198c5758 100644 --- a/airbyte-integrations/connectors/destination-teradata/Dockerfile +++ b/airbyte-integrations/connectors/destination-teradata/Dockerfile @@ -22,5 +22,5 @@ ENV APPLICATION destination-teradata COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-teradata diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 892ad3b7b842..8df89e805032 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/destinations/teradata + documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index 4b082661dab6..3c9971d3a6a8 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -31,15 +31,23 @@ public class TeradataDestination extends AbstractJdbcDestination implements Dest /** * Default schema name */ - public static final String DEFAULT_SCHEMA_NAME = "def_airbyte_db"; - public static final String PARAM_MODE = "mode"; - public static final String PARAM_SSL = "ssl"; - public static final String PARAM_SSL_MODE = "ssl_mode"; - public static final String PARAM_SSLMODE = "sslmode"; - public static final String PARAM_SSLCA = "sslca"; - public static final String REQUIRE = "require"; - - private static final String CA_CERTIFICATE = "ca.pem"; + protected static final String DEFAULT_SCHEMA_NAME = "def_airbyte_db"; + protected static final String PARAM_MODE = "mode"; + protected static final String PARAM_SSL = "ssl"; + protected static final String PARAM_SSL_MODE = "ssl_mode"; + protected static final String PARAM_SSLMODE = "sslmode"; + protected static final String PARAM_SSLCA = "sslca"; + protected static final String REQUIRE = "require"; + + protected static final String VERIFY_CA = "verify-ca"; + + protected static final String VERIFY_FULL = "verify-full"; + + protected static final String ALLOW = "allow"; + + protected static final String CA_CERTIFICATE = "ca.pem"; + + protected static final String CA_CERT_KEY = "ssl_ca_certificate"; public static void main(String[] args) throws Exception { new IntegrationRunner(new TeradataDestination()).run(args); diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 573d403b942d..76df7bf8fbeb 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -24,10 +24,10 @@ public class TeradataSqlOperations extends JdbcSqlOperations { private static final Logger LOGGER = LoggerFactory.getLogger(TeradataSqlOperations.class); @Override - public void insertRecordsInternal(JdbcDatabase database, - List records, - String schemaName, - String tableName) + public void insertRecordsInternal(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) throws SQLException { if (records.isEmpty()) { return; @@ -39,7 +39,7 @@ public void insertRecordsInternal(JdbcDatabase database, database.execute(con -> { try { - PreparedStatement pstmt = con.prepareStatement(insertQueryComponent); + final PreparedStatement pstmt = con.prepareStatement(insertQueryComponent); for (final AirbyteRecordMessage record : records) { @@ -65,7 +65,7 @@ public void insertRecordsInternal(JdbcDatabase database, AirbyteTraceMessageUtility.emitSystemErrorTrace(se, "Connector failed while inserting records to staging table"); throw new RuntimeException(se); - } catch (Exception e) { + } catch (final Exception e) { AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Connector failed while inserting records to staging table"); throw new RuntimeException(e); @@ -78,7 +78,7 @@ public void insertRecordsInternal(JdbcDatabase database, public void createSchemaIfNotExists(final JdbcDatabase database, final String schemaName) throws Exception { try { database.execute(String.format("CREATE DATABASE \"%s\" AS PERMANENT = 120e6, SPOOL = 120e6;", schemaName)); - } catch (SQLException e) { + } catch (final SQLException e) { if (e.getMessage().contains("already exists")) { LOGGER.warn("Database " + schemaName + " already exists."); } else { @@ -94,7 +94,7 @@ public void createTableIfNotExists(final JdbcDatabase database, final String sch throws SQLException { try { database.execute(createTableQuery(database, schemaName, tableName)); - } catch (SQLException e) { + } catch (final SQLException e) { if (e.getMessage().contains("already exists")) { LOGGER.warn("Table " + schemaName + "." + tableName + " already exists."); } else { @@ -117,8 +117,8 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN public void dropTableIfExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { try { - database.execute(dropTableIfExistsQuery(schemaName, tableName)); - } catch (SQLException e) { + database.execute(dropTableIfExistsQueryInternal(schemaName, tableName)); + } catch (final SQLException e) { AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Connector failed while dropping table " + schemaName + "." + tableName); } @@ -128,17 +128,17 @@ public void dropTableIfExists(final JdbcDatabase database, final String schemaNa public String truncateTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { try { return String.format("DELETE %s.%s ALL;\n", schemaName, tableName); - } catch (Exception e) { + } catch (final Exception e) { AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Connector failed while truncating table " + schemaName + "." + tableName); } return ""; } - private String dropTableIfExistsQuery(final String schemaName, final String tableName) { + private String dropTableIfExistsQueryInternal(final String schemaName, final String tableName) { try { return String.format("DROP TABLE %s.%s;\n", schemaName, tableName); - } catch (Exception e) { + } catch (final Exception e) { AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Connector failed while dropping table " + schemaName + "." + tableName); } @@ -153,7 +153,7 @@ public void executeTransaction(final JdbcDatabase database, final List q appendedQueries.append(query); } database.execute(appendedQueries.toString()); - } catch (SQLException e) { + } catch (final SQLException e) { AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Connector failed while executing queries : " + appendedQueries.toString()); } diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json index 33cbd7d0856a..25e54e8f2b0c 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json @@ -138,10 +138,7 @@ "title": "verify-full", "additionalProperties": false, "description": "Verify-full SSL mode.", - "required": [ - "mode", - "ssl_ca_certificate" - ], + "required": ["mode", "ssl_ca_certificate"], "properties": { "mode": { "type": "string", @@ -160,7 +157,6 @@ } } } - ] }, "jdbc_url_params": { @@ -171,4 +167,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java index 469263e971d5..1386c172d7bd 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java @@ -4,8 +4,6 @@ package io.airbyte.integrations.destination.teradata; -import static org.junit.jupiter.api.Assertions.assertEquals; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; @@ -19,25 +17,40 @@ import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.integrations.destination.teradata.envclient.TeradataHttpClient; +import io.airbyte.integrations.destination.teradata.envclient.dto.*; +import io.airbyte.integrations.destination.teradata.envclient.exception.BaseException; import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.nio.file.Files; import java.nio.file.Paths; import java.sql.SQLException; -import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; import javax.sql.DataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TeradataDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(TeradataDestinationAcceptanceTest.class); private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + private static final String SCHEMA_NAME = Strings.addRandomSuffix("acc_test", "_", 5); + + private static final String CREATE_DATABASE = "CREATE DATABASE \"%s\" AS PERMANENT = 60e6, SPOOL = 60e6 SKEW = 10 PERCENT"; + + private static final String DELETE_DATABASE = "DELETE DATABASE \"%s\""; + + private static final String DROP_DATABASE = "DROP DATABASE \"%s\""; + private JsonNode configJson; private JdbcDatabase database; private DataSource dataSource; @@ -54,18 +67,55 @@ protected JsonNode getConfig() { return configJson; } + @BeforeAll + void initEnvironment() throws Exception { + this.configJson = Jsons.clone(getStaticConfig()); + TeradataHttpClient teradataHttpClient = new TeradataHttpClient(configJson.get("env_url").asText()); + String name = configJson.get("env_name").asText(); + String token = configJson.get("env_token").asText(); + var getRequest = new GetEnvironmentRequest(name); + EnvironmentResponse response = null; + try { + response = teradataHttpClient.getEnvironment(getRequest, token); + } catch (BaseException be) { + LOGGER.error("Environemnt " + name + " is not available. " + be.getMessage()); + } + if (response == null || response.ip() == null) { + var request = new CreateEnvironmentRequest( + name, + configJson.get("env_region").asText(), + configJson.get("env_password").asText()); + response = teradataHttpClient.createEnvironment(request, token).get(); + } else if (response.state() == EnvironmentResponse.State.STOPPED) { + var request = new EnvironmentRequest(name, new OperationRequest("start")); + teradataHttpClient.startEnvironment(request, token); + } + ((ObjectNode) configJson).put("host", response.ip()); + if (configJson.get("password") == null) { + ((ObjectNode) configJson).put("password", configJson.get("env_password").asText()); + } + } + + @AfterAll + void cleanupEnvironment() throws ExecutionException, InterruptedException { + try { + TeradataHttpClient teradataHttpClient = new TeradataHttpClient(configJson.get("env_url").asText()); + var request = new EnvironmentRequest(configJson.get("env_name").asText(), new OperationRequest("stop")); + teradataHttpClient.stopEnvironment(request, configJson.get("env_token").asText()); + } catch (BaseException be) { + LOGGER.error("Environemnt " + configJson.get("env_name").asText() + " is not available. " + be.getMessage()); + } + } + public JsonNode getStaticConfig() throws Exception { - final JsonNode config = Jsons.deserialize(Files.readString(Paths.get("secrets/config.json"))); - return config; + return Jsons.deserialize(Files.readString(Paths.get("secrets/config.json"))); } @Override protected JsonNode getFailCheckConfig() throws Exception { - final JsonNode credentialsJsonString = Jsons - .deserialize(Files.readString(Paths.get("secrets/failureconfig.json"))); - final AirbyteConnectionStatus check = new TeradataDestination().check(credentialsJsonString); - assertEquals(AirbyteConnectionStatus.Status.FAILED, check.getStatus()); - return credentialsJsonString; + JsonNode failureConfig = Jsons.clone(this.configJson); + ((ObjectNode) failureConfig).put("password", "wrongpassword"); + return failureConfig; } @Override @@ -74,8 +124,7 @@ protected List retrieveRecords(final TestDestinationEnv testEnv, final String namespace, final JsonNode streamSchema) throws Exception { - final List records = retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); - return records; + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace); } @@ -92,19 +141,29 @@ private List retrieveRecordsFromTable(final String tableName, final St } @Override - protected void setup(TestDestinationEnv testEnv) { - final String schemaName = Strings.addRandomSuffix("integration_test_teradata", "_", 5); - final String createSchemaQuery = String - .format(String.format("CREATE DATABASE \"%s\" AS PERMANENT = 60e6, SPOOL = 60e6 SKEW = 10 PERCENT", schemaName)); + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { + final String createSchemaQuery = String.format(CREATE_DATABASE, SCHEMA_NAME); try { - this.configJson = Jsons.clone(getStaticConfig()); - ((ObjectNode) configJson).put("schema", schemaName); - + ((ObjectNode) configJson).put("schema", SCHEMA_NAME); dataSource = getDataSource(configJson); database = getDatabase(dataSource); database.execute(createSchemaQuery); } catch (Exception e) { - AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Database " + schemaName + " creation got failed."); + AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Database " + SCHEMA_NAME + " creation got failed."); + } + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) throws Exception { + final String deleteQuery = String.format(String.format(DELETE_DATABASE, SCHEMA_NAME)); + final String dropQuery = String.format(String.format(DROP_DATABASE, SCHEMA_NAME)); + try { + database.execute(deleteQuery); + database.execute(dropQuery); + } catch (Exception e) { + AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "Database " + SCHEMA_NAME + " delete got failed."); + } finally { + DataSourceFactory.close(dataSource); } } @@ -120,9 +179,6 @@ public void testSecondSync() { // overrides test in coming releases } - @Override - protected void tearDown(TestDestinationEnv testEnv) {} - protected DataSource getDataSource(final JsonNode config) { final JsonNode jdbcConfig = destination.toJdbcConfig(config); return DataSourceFactory.create(jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), @@ -144,7 +200,7 @@ protected Map getConnectionProperties(final JsonNode config) { } protected Map getDefaultConnectionProperties(final JsonNode config) { - return Collections.emptyMap(); + return destination.getDefaultConnectionProperties(config); } private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationSSLAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationSSLAcceptanceTest.java index 1b5121873ef7..c740ec8e0afa 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationSSLAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationSSLAcceptanceTest.java @@ -4,15 +4,21 @@ package io.airbyte.integrations.destination.teradata; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TeradataDestinationSSLAcceptanceTest extends TeradataDestinationAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(TeradataDestinationSSLAcceptanceTest.class); - protected String getConfigFileName() { - return "secrets/sslconfig.json"; + public JsonNode getStaticConfig() throws Exception { + return Jsons.deserialize(Files.readString(Paths.get("secrets/sslconfig.json"))); } } diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/Headers.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/Headers.java new file mode 100644 index 000000000000..6fc2d055f0cc --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/Headers.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient; + +public class Headers { + + private Headers() { + + } + + public static final String CONTENT_TYPE = "Content-Type"; + + public static final String AUTHORIZATION = "Authorization"; + + public static final String APPLICATION_JSON = "application/json"; + + public static final String BEARER = "Bearer "; + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/TeradataHttpClient.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/TeradataHttpClient.java new file mode 100644 index 000000000000..db2204f57d8f --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/TeradataHttpClient.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient; + +import static io.airbyte.integrations.destination.teradata.envclient.Headers.*; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import io.airbyte.integrations.destination.teradata.envclient.dto.*; +import io.airbyte.integrations.destination.teradata.envclient.exception.BaseException; +import io.airbyte.integrations.destination.teradata.envclient.exception.Error4xxException; +import io.airbyte.integrations.destination.teradata.envclient.exception.Error5xxException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.concurrent.CompletableFuture; + +public class TeradataHttpClient { + + private final String baseUrl; + + private final HttpClient httpClient; + + private final ObjectMapper objectMapper; + + public TeradataHttpClient(String baseUrl) { + this(HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(), baseUrl); + } + + public TeradataHttpClient(HttpClient httpClient, String baseUrl) { + this.httpClient = httpClient; + this.baseUrl = baseUrl; + this.objectMapper = JsonMapper.builder() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false) + .build(); + } + + // Creating an environment is a blocking operation by default, and it takes ~1.5min to finish + public CompletableFuture createEnvironment(CreateEnvironmentRequest createEnvironmentRequest, + String token) { + var requestBody = handleCheckedException(() -> objectMapper.writeValueAsString(createEnvironmentRequest)); + + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl.concat("/environments"))) + .headers( + AUTHORIZATION, BEARER + token, + CONTENT_TYPE, APPLICATION_JSON) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + return httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString()) + .thenApply(httpResponse -> handleHttpResponse(httpResponse, new TypeReference<>() {})); + } + + // Avoids long connections and the risk of connection termination by intermediary + public CompletableFuture pollingCreateEnvironment( + CreateEnvironmentRequest createEnvironmentRequest, + String token) { + throw new UnsupportedOperationException(); + } + + public EnvironmentResponse getEnvironment(GetEnvironmentRequest getEnvironmentRequest, String token) { + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl + .concat("/environments/") + .concat(getEnvironmentRequest.name()))) + .headers(AUTHORIZATION, BEARER + token) + .GET() + .build(); + + var httpResponse = + handleCheckedException(() -> httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString())); + return handleHttpResponse(httpResponse, new TypeReference<>() {}); + } + + // Deleting an environment is a blocking operation by default, and it takes ~1.5min to finish + public CompletableFuture deleteEnvironment(DeleteEnvironmentRequest deleteEnvironmentRequest, String token) { + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl + .concat("/environments/") + .concat(deleteEnvironmentRequest.name()))) + .headers(AUTHORIZATION, BEARER + token) + .DELETE() + .build(); + + return httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString()) + .thenApply(httpResponse -> handleHttpResponse(httpResponse, new TypeReference<>() {})); + } + + public CompletableFuture startEnvironment(EnvironmentRequest environmentRequest, String token) { + var requestBody = handleCheckedException(() -> objectMapper.writeValueAsString(environmentRequest.request())); + return getVoidCompletableFuture(environmentRequest.name(), token, requestBody); + } + + public CompletableFuture stopEnvironment(EnvironmentRequest environmentRequest, String token) { + var requestBody = handleCheckedException(() -> objectMapper.writeValueAsString(environmentRequest.request())); + return getVoidCompletableFuture(environmentRequest.name(), token, requestBody); + } + + private CompletableFuture getVoidCompletableFuture(String name, String token, String jsonPayLoadString) { + HttpRequest.BodyPublisher publisher = HttpRequest.BodyPublishers.ofString(jsonPayLoadString); + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl + .concat("/environments/") + .concat(name))) + .headers(AUTHORIZATION, BEARER + token, + CONTENT_TYPE, APPLICATION_JSON) + .method("PATCH", publisher) + .build(); + var httpResponse = + handleCheckedException(() -> httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString())); + return handleHttpResponse(httpResponse, new TypeReference<>() {}); + + } + + private T handleHttpResponse(HttpResponse httpResponse, TypeReference typeReference) { + var body = httpResponse.body(); + if (httpResponse.statusCode() >= 200 && httpResponse.statusCode() <= 299) { + return handleCheckedException(() -> { + if (typeReference.getType().getTypeName().equals(Void.class.getTypeName())) { + return null; + } else { + return objectMapper.readValue(body, typeReference); + } + }); + } else if (httpResponse.statusCode() >= 400 && httpResponse.statusCode() <= 499) { + throw new Error4xxException(httpResponse.statusCode(), body); + } else if (httpResponse.statusCode() >= 500 && httpResponse.statusCode() <= 599) { + throw new Error5xxException(httpResponse.statusCode(), body); + } else { + throw new BaseException(httpResponse.statusCode(), body); + } + } + + private T handleCheckedException(CheckedSupplier checkedSupplier) { + try { + return checkedSupplier.get(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + @FunctionalInterface + private interface CheckedSupplier { + + T get() throws IOException, InterruptedException; + + } + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/CreateEnvironmentRequest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/CreateEnvironmentRequest.java new file mode 100644 index 000000000000..c2a56cb359b8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/CreateEnvironmentRequest.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +public record CreateEnvironmentRequest( + + String name, + + String region, + + String password + +) { + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/DeleteEnvironmentRequest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/DeleteEnvironmentRequest.java new file mode 100644 index 000000000000..a7672782b625 --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/DeleteEnvironmentRequest.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +public record DeleteEnvironmentRequest( + + String name + +) {} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/EnvironmentRequest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/EnvironmentRequest.java new file mode 100644 index 000000000000..8706c955c06c --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/EnvironmentRequest.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +public record EnvironmentRequest( + + String name, + + OperationRequest request + +) {} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/EnvironmentResponse.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/EnvironmentResponse.java new file mode 100644 index 000000000000..67a579bcc945 --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/EnvironmentResponse.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +import java.util.List; + +public record EnvironmentResponse( + + State state, + + String region, + + // Use for subsequent environment operations i.e GET, DELETE, etc + String name, + + // Use for connecting with JDBC driver + String ip, + + String dnsName, + + String owner, + + String type, + + List services + +) { + + record Service( + + List credentials, + + String name, + + String url + + ) { + + } + + record Credential( + + String name, + + String value + + ) { + + } + + public enum State { + + PROVISIONING, + INITIALIZING, + RUNNING, + STARTING, + STOPPING, + STOPPED, + TERMINATING, + TERMINATED, + REPAIRING + + } + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/GetEnvironmentRequest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/GetEnvironmentRequest.java new file mode 100644 index 000000000000..5781d3c8058c --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/GetEnvironmentRequest.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +public record GetEnvironmentRequest( + + String name + +) {} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/OperationRequest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/OperationRequest.java new file mode 100644 index 000000000000..1c1da32ddf9e --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/OperationRequest.java @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +public record OperationRequest( + String operation) {} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/Region.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/Region.java new file mode 100644 index 000000000000..a6ba3c489d15 --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/dto/Region.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.dto; + +public enum Region { + + US_CENTRAL("us-central"), + + US_EAST("us-east"), + + US_WEST("us-west"), + + SOUTHAMERICA_EAST("southamerica-east"), + + EUROPE_WEST("europe-west"), + + ASIA_SOUTH("asia-south"), + + ASIA_NORTHEAST("asia-northeast"), + + ASIA_SOUTHEAST("asia-southeast"), + + AUSTRALIA_SOUTHEAST("australia-southeast"); + + private final String regionName; + + Region(String regionName) { + this.regionName = regionName; + } + + public String getRegionName() { + return regionName; + } + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/BaseException.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/BaseException.java new file mode 100644 index 000000000000..4132821a120d --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/BaseException.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.exception; + +public class BaseException extends RuntimeException { + + private final int statusCode; + + private final String body; + + private final String reason; + + public BaseException(int statusCode, String body) { + super(body); + this.statusCode = statusCode; + this.body = body; + this.reason = null; + } + + public BaseException(int statusCode, String body, String reason) { + super(body); + this.statusCode = statusCode; + this.body = body; + this.reason = reason; + } + + public int getStatusCode() { + return statusCode; + } + + public String getBody() { + return body; + } + + public String getReason() { + return reason; + } + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/Error4xxException.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/Error4xxException.java new file mode 100644 index 000000000000..c56a7c60b33e --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/Error4xxException.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.exception; + +public class Error4xxException extends BaseException { + + public Error4xxException(int statusCode, String body, String reason) { + super(statusCode, body, reason); + } + + public Error4xxException(int statusCode, String body) { + super(statusCode, body); + } + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/Error5xxException.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/Error5xxException.java new file mode 100644 index 000000000000..8ab59887ad67 --- /dev/null +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/envclient/exception/Error5xxException.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.teradata.envclient.exception; + +public class Error5xxException extends BaseException { + + public Error5xxException(int statusCode, String body, String reason) { + super(statusCode, body, reason); + } + + public Error5xxException(int statusCode, String body) { + super(statusCode, body); + } + +} diff --git a/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java b/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java index 6c7acc7e0253..2fb453e3162f 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java @@ -4,33 +4,25 @@ package io.airbyte.integrations.destination.teradata; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import java.nio.file.Files; -import java.nio.file.Paths; -import org.junit.jupiter.api.AfterAll; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class TeradataDestinationTest { - private static final Logger LOGGER = LoggerFactory.getLogger(TeradataDestinationTest.class); - private JsonNode config; final TeradataDestination destination = new TeradataDestination(); - private static String EXPECTED_JDBC_URL = "jdbc:teradata://localhost/"; - private static String EXPECTED_JDBC_ESCAPED_URL = "jdbc:teradata://localhost/"; + private final String EXPECTED_JDBC_URL = "jdbc:teradata://localhost/"; + + private final String EXTRA_JDBC_PARAMS = "key1=value1&key2=value2&key3=value3"; private String getUserName() { return config.get(JdbcUtils.USERNAME_KEY).asText(); @@ -44,112 +36,161 @@ private String getHostName() { return config.get(JdbcUtils.HOST_KEY).asText(); } - private JsonNode buildConfigNoJdbcParameters() { - return Jsons.jsonNode(ImmutableMap.of(JdbcUtils.HOST_KEY, config.get(JdbcUtils.HOST_KEY).asText(), - JdbcUtils.USERNAME_KEY, config.get(JdbcUtils.USERNAME_KEY).asText(), JdbcUtils.DATABASE_KEY, - config.get(JdbcUtils.PASSWORD_KEY).asText())); + private String getSchemaName() { + return config.get(JdbcUtils.SCHEMA_KEY).asText(); } @BeforeEach void setup() { + this.config = createConfig(); + } - try { - this.config = Jsons.clone(Jsons.deserialize(Files.readString(Paths.get("secrets/config.json")))); - this.EXPECTED_JDBC_URL = String.format("jdbc:teradata://%s/", config.get(JdbcUtils.HOST_KEY).asText()); - this.EXPECTED_JDBC_ESCAPED_URL = String.format("jdbc:teradata://%s/", - config.get(JdbcUtils.HOST_KEY).asText()); - } catch (Exception e) { - AirbyteTraceMessageUtility.emitSystemErrorTrace(e, "setup failed"); + private JsonNode createConfig() { + return Jsons.jsonNode(baseParameters()); + } + + private JsonNode createConfig(boolean sslEnable) { + JsonNode jsonNode; + if (sslEnable) { + jsonNode = Jsons.jsonNode(sslBaseParameters()); + } else { + jsonNode = createConfig(); } + return jsonNode; + } + + private JsonNode createConfig(final String sslMethod) { + Map additionalParameters = getAdditionalParams(sslMethod); + return Jsons.jsonNode(MoreMaps.merge(sslBaseParameters(), additionalParameters)); + } + + private Map getAdditionalParams(final String sslMethod) { + Map additionalParameters; + switch (sslMethod) { + case "verify-ca", "verify-full" -> { + additionalParameters = ImmutableMap.of( + TeradataDestination.PARAM_SSL_MODE, Jsons.jsonNode(ImmutableMap.of( + TeradataDestination.PARAM_MODE, sslMethod, + TeradataDestination.CA_CERT_KEY, "dummycertificatecontent"))); + } + default -> { + additionalParameters = ImmutableMap.of( + TeradataDestination.PARAM_SSL_MODE, Jsons.jsonNode(ImmutableMap.of( + TeradataDestination.PARAM_MODE, sslMethod))); + } + } + return additionalParameters; + } + + private Map baseParameters() { + return ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, "localhost") + .put(JdbcUtils.SCHEMA_KEY, "db") + .put(JdbcUtils.USERNAME_KEY, "username") + .put(JdbcUtils.PASSWORD_KEY, "verysecure") + .build(); } - @AfterAll - static void cleanUp() {} + private Map sslBaseParameters() { + return ImmutableMap.builder() + .put(TeradataDestination.PARAM_SSL, "true") + .put(JdbcUtils.HOST_KEY, getHostName()) + .put(JdbcUtils.SCHEMA_KEY, getSchemaName()) + .put(JdbcUtils.USERNAME_KEY, getUserName()) + .put(JdbcUtils.PASSWORD_KEY, getPassword()) + .build(); + } + + private JsonNode buildConfigNoJdbcParameters() { + return Jsons.jsonNode(baseParameters()); + } - private JsonNode buildConfigEscapingNeeded() { - return Jsons.jsonNode(ImmutableMap.of(JdbcUtils.HOST_KEY, getHostName(), JdbcUtils.USERNAME_KEY, getUserName(), - JdbcUtils.DATABASE_KEY, "db/foo")); + private JsonNode buildConfigDefaultSchema() { + return Jsons.jsonNode(ImmutableMap.of( + JdbcUtils.HOST_KEY, getHostName(), + JdbcUtils.USERNAME_KEY, getUserName(), + JdbcUtils.PASSWORD_KEY, + getPassword())); } private JsonNode buildConfigWithExtraJdbcParameters(final String extraParam) { - return Jsons.jsonNode(ImmutableMap.of(JdbcUtils.HOST_KEY, getHostName(), JdbcUtils.USERNAME_KEY, getUserName(), - JdbcUtils.DATABASE_KEY, "db", JdbcUtils.JDBC_URL_PARAMS_KEY, extraParam)); + return Jsons.jsonNode(ImmutableMap.of( + JdbcUtils.HOST_KEY, getHostName(), + JdbcUtils.USERNAME_KEY, getUserName(), + JdbcUtils.SCHEMA_KEY, getSchemaName(), + JdbcUtils.JDBC_URL_PARAMS_KEY, extraParam)); } @Test void testJdbcUrlAndConfigNoExtraParams() { - final JsonNode jdbcConfig = new TeradataDestination().toJdbcConfig(buildConfigNoJdbcParameters()); + final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigNoJdbcParameters()); assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); + assertEquals("username", jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText()); + assertEquals("db", jdbcConfig.get(JdbcUtils.SCHEMA_KEY).asText()); + assertEquals("verysecure", jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText()); } @Test - void testJdbcUrlWithEscapedDatabaseName() { - final JsonNode jdbcConfig = new TeradataDestination().toJdbcConfig(buildConfigEscapingNeeded()); - assertEquals(EXPECTED_JDBC_ESCAPED_URL, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); + void testJdbcUrlEmptyExtraParams() { + final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigWithExtraJdbcParameters("")); + assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); + assertEquals("username", jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText()); + assertEquals("db", jdbcConfig.get(JdbcUtils.SCHEMA_KEY).asText()); + assertEquals("", jdbcConfig.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); } @Test - void testJdbcUrlEmptyExtraParams() { - final JsonNode jdbcConfig = new TeradataDestination().toJdbcConfig(buildConfigWithExtraJdbcParameters("")); + void testJdbcUrlExtraParams() { + + final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigWithExtraJdbcParameters(EXTRA_JDBC_PARAMS)); assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); + assertEquals("username", jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText()); + assertEquals("db", jdbcConfig.get(JdbcUtils.SCHEMA_KEY).asText()); + assertEquals(EXTRA_JDBC_PARAMS, jdbcConfig.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); } @Test - void testJdbcUrlExtraParams() { - final String extraParam = "key1=value1&key2=value2&key3=value3"; - final JsonNode jdbcConfig = new TeradataDestination() - .toJdbcConfig(buildConfigWithExtraJdbcParameters(extraParam)); + void testDefaultSchemaName() { + final JsonNode jdbcConfig = destination.toJdbcConfig(buildConfigDefaultSchema()); assertEquals(EXPECTED_JDBC_URL, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); + assertEquals(TeradataDestination.DEFAULT_SCHEMA_NAME, jdbcConfig.get(JdbcUtils.SCHEMA_KEY).asText()); + } + + @Test + void testSSLDisable() { + final JsonNode jdbcConfig = createConfig(false); + final Map properties = destination.getDefaultConnectionProperties(jdbcConfig); + assertNull(properties.get(TeradataDestination.PARAM_SSLMODE)); } @Test - void testCheckIncorrectPasswordFailure() { - final var config = buildConfigNoJdbcParameters(); - ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, "public"); - final TeradataDestination destination = new TeradataDestination(); - final AirbyteConnectionStatus status = destination.check(config); - // State code: 28000; Error code: 8017; Message: [Teradata Database] [TeraJDBC - // 17.20.00.12] [Error 8017] [SQLState 28000] The UserId, Password or Account is - // invalid."}} - assertTrue(status.getMessage().contains("SQLState 28000")); + void testSSLDefaultMode() { + final JsonNode jdbcConfig = createConfig(true); + final Map properties = destination.getDefaultConnectionProperties(jdbcConfig); + assertEquals(TeradataDestination.REQUIRE, properties.get(TeradataDestination.PARAM_SSLMODE).toString()); } @Test - public void testCheckIncorrectUsernameFailure() { - final var config = buildConfigNoJdbcParameters(); - LOGGER.info(" config in testCheckIncorrectUsernameFailure - " + config); - ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "dummy"); - ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, "public"); - final TeradataDestination destination = new TeradataDestination(); - final AirbyteConnectionStatus status = destination.check(config); - // State code: 28000; Error code: 8017; Message: [Teradata Database] [TeraJDBC - // 17.20.00.12] [Error 8017] [SQLState 28000] The UserId, Password or Account is - // invalid."}} - assertTrue(status.getMessage().contains("SQLState 28000")); + void testSSLAllowMode() { + final JsonNode jdbcConfig = createConfig(TeradataDestination.ALLOW); + final Map properties = destination.getDefaultConnectionProperties(jdbcConfig); + assertEquals(TeradataDestination.ALLOW, properties.get(TeradataDestination.PARAM_SSLMODE).toString()); } @Test - public void testCheckIncorrectHostFailure() { - final var config = buildConfigNoJdbcParameters(); - ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, "public"); - final TeradataDestination destination = new TeradataDestination(); - final AirbyteConnectionStatus status = destination.check(config); - assertTrue(status.getMessage().contains("SQLState 08S01")); + void testSSLVerfifyCAMode() { + final JsonNode jdbcConfig = createConfig(TeradataDestination.VERIFY_CA); + final Map properties = destination.getDefaultConnectionProperties(jdbcConfig); + assertEquals(TeradataDestination.VERIFY_CA, properties.get(TeradataDestination.PARAM_SSLMODE).toString()); + assertNotNull(properties.get(TeradataDestination.PARAM_SSLCA).toString()); } @Test - public void testCheckIncorrectDataBaseFailure() { - final var config = buildConfigNoJdbcParameters(); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); - ((ObjectNode) config).put(JdbcUtils.SCHEMA_KEY, "public"); - final TeradataDestination destination = new TeradataDestination(); - final AirbyteConnectionStatus status = destination.check(config); - // State code: 28000; Error code: 8017; Message: [Teradata Database] [TeraJDBC - // 17.20.00.12] [Error 8017] [SQLState 28000] The UserId, Password or Account is - // invalid."}} - assertTrue(status.getMessage().contains("SQLState 28000")); + void testSSLVerfifyFullMode() { + final JsonNode jdbcConfig = createConfig(TeradataDestination.VERIFY_FULL); + final Map properties = destination.getDefaultConnectionProperties(jdbcConfig); + assertEquals(TeradataDestination.VERIFY_FULL, properties.get(TeradataDestination.PARAM_SSLMODE).toString()); + assertNotNull(properties.get(TeradataDestination.PARAM_SSLCA).toString()); } } diff --git a/airbyte-integrations/connectors/destination-tidb/Dockerfile b/airbyte-integrations/connectors/destination-tidb/Dockerfile index d8a958adffb4..35f6f8bd5d2f 100644 --- a/airbyte-integrations/connectors/destination-tidb/Dockerfile +++ b/airbyte-integrations/connectors/destination-tidb/Dockerfile @@ -8,8 +8,31 @@ # Please reach out to the Connectors Operations team if you have any question. FROM airbyte/integration-base-java:dev AS build +RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ + alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ + python -m ensurepip --upgrade && \ + # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 + pip3 install wheel && \ + pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ + pip3 install dbt-tidb==1.0.1 + +# Luckily, none of normalization's files conflict with destination-tidb's files :) +# We don't enforce that in any way, but hopefully we're only living in this state for a short time. +COPY --from=airbyte/normalization-tidb:dev /airbyte /airbyte +# Install python dependencies +WORKDIR /airbyte/base_python_structs +RUN pip3 install . +WORKDIR /airbyte/normalization_code +RUN pip3 install . +WORKDIR /airbyte/normalization_code/dbt-template/ +# Download external dbt dependencies +# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 +RUN pip3 install "urllib3<2" +RUN dbt deps + WORKDIR /airbyte ENV APPLICATION destination-tidb +ENV AIRBYTE_NORMALIZATION_INTEGRATION tidb COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar @@ -22,5 +45,8 @@ ENV APPLICATION destination-tidb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-tidb + +ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" +ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-tidb/build.gradle b/airbyte-integrations/connectors/destination-tidb/build.gradle index c2e1b7a55719..674e4473783e 100644 --- a/airbyte-integrations/connectors/destination-tidb/build.gradle +++ b/airbyte-integrations/connectors/destination-tidb/build.gradle @@ -24,3 +24,7 @@ dependencies { integrationTestJavaImplementation libs.connectors.testcontainers.tidb integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } + +tasks.named("airbyteDocker") { + dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerTiDB +} diff --git a/airbyte-integrations/connectors/destination-tidb/metadata.yaml b/airbyte-integrations/connectors/destination-tidb/metadata.yaml index 20c0fbee1b8a..1d7ff079f914 100644 --- a/airbyte-integrations/connectors/destination-tidb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-tidb/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 06ec60c7-7468-45c0-91ac-174f6e1a788b - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.4 dockerRepository: airbyte/destination-tidb githubIssueLabel: destination-tidb icon: tidb.svg @@ -22,4 +22,8 @@ data: supportsDbt: true tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java index 181507bac400..608c2ddbd2a5 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java @@ -17,6 +17,7 @@ import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -135,7 +136,7 @@ protected List retrieveNormalizedRecords(final TestDestinationEnv test } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) .withExposedPorts(4000); container.start(); diff --git a/airbyte-integrations/connectors/destination-timeplus/.dockerignore b/airbyte-integrations/connectors/destination-timeplus/.dockerignore new file mode 100755 index 000000000000..40dea8ad1f6f --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_timeplus +!setup.py diff --git a/airbyte-integrations/connectors/destination-timeplus/Dockerfile b/airbyte-integrations/connectors/destination-timeplus/Dockerfile new file mode 100755 index 000000000000..34f3c7492dd3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY destination_timeplus ./destination_timeplus + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-timeplus diff --git a/airbyte-integrations/connectors/destination-timeplus/README.md b/airbyte-integrations/connectors/destination-timeplus/README.md new file mode 100755 index 000000000000..bfd9cf3a6680 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/README.md @@ -0,0 +1,122 @@ +# Timeplus Destination + +This is the repository for the Timeplus destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/timeplus). +This connector is built by following the [Building a Python Destination](https://docs.airbyte.com/connector-development/tutorials/building-a-python-destination) + +## Local development + +### Prerequisites + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-timeplus:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/timeplus) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_timeplus/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination timeplus test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +cat integration_tests/messages.jsonl | python main.py write --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-timeplus:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-timeplus:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-timeplus:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-timeplus:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-timeplus:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install pytest +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-timeplus:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-timeplus:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-timeplus/build.gradle b/airbyte-integrations/connectors/destination-timeplus/build.gradle new file mode 100755 index 000000000000..022593982902 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'destination_timeplus' +} diff --git a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/__init__.py b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/__init__.py new file mode 100755 index 000000000000..fa8a30eb633c --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationTimeplus + +__all__ = ["DestinationTimeplus"] diff --git a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py new file mode 100755 index 000000000000..3cf5c8920e78 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/destination.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from logging import getLogger +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + DestinationSyncMode, + Status, + Type, +) +from timeplus import Environment, Stream + +logger = getLogger("airbyte") + + +class DestinationTimeplus(Destination): + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + """ + Reads the input stream of messages, config, and catalog to write data to the destination. + + This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received + in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been + successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, + then the source is given the last state message output from this method as the starting point of the next sync. + + :param config: dict of JSON configuration matching the configuration declared in spec.json + :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the + destination + :param input_messages: The stream of input messages received from the source + :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs + """ + endpoint = config["endpoint"] + apikey = config["apikey"] + if endpoint[-1] == "/": + endpoint = endpoint[0 : len(endpoint) - 1] + env = Environment().address(endpoint).apikey(apikey) + stream_list = Stream(env=env).list() + all_streams = {s.name for s in stream_list} + + # only support "overwrite", "append" + for configured_stream in configured_catalog.streams: + is_overwrite = configured_stream.destination_sync_mode == DestinationSyncMode.overwrite + stream_exists = configured_stream.stream.name in all_streams + logger.info(f"Stream {configured_stream.stream.name} {configured_stream.destination_sync_mode}") + need_delete_stream = False + need_create_stream = False + if is_overwrite: + if stream_exists: + # delete all data in the existing stream and recreate the stream. + need_delete_stream = True + need_create_stream = True + else: + # only need to create the stream + need_create_stream = True + else: + if stream_exists: + # for append mode, just add more data to the existing stream. No need to do anything. + pass + else: + # for append mode, create the stream and append data to it. + need_create_stream = True + + if need_delete_stream: + # delete the existing stream + Stream(env=env).name(configured_stream.stream.name).get().delete() + logger.info(f"Stream {configured_stream.stream.name} deleted successfully") + if need_create_stream: + # create a new stream + DestinationTimeplus.create_stream(env, configured_stream.stream) + logger.info(f"Stream {configured_stream.stream.name} created successfully") + + for message in input_messages: + if message.type == Type.STATE: + # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush + # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state + yield message + elif message.type == Type.RECORD: + record = message.record + + # this code is to send data to a single-column stream + # Stream(env=env).name(record.stream).column("raw", "string").ingest(payload=record.data) + + Stream(env=env).name(record.stream).ingest(payload=record.data, format="streaming") + else: + # ignore other message types for now + continue + + @staticmethod + def create_stream(env, stream: AirbyteStream): + # singlel-column stream + # Stream(env=env).name(stream.name).column('raw','string').create() + + tp_stream = Stream(env=env).name(stream.name.strip()) + for name, v in stream.json_schema["properties"].items(): + tp_stream.column(name.strip(), DestinationTimeplus.type_mapping(v)) + tp_stream.create() + + @staticmethod + def type_mapping(v) -> str: + airbyte_type = v["type"] + if type(airbyte_type) is list: + for t in list(airbyte_type): + if t != "null": + type_def = {"type": t} + if t == "array": + type_def["items"] = v["items"] + return DestinationTimeplus.type_mapping(type_def) + if airbyte_type == "number": + return "float" + elif airbyte_type == "integer": + return "integer" + elif airbyte_type == "boolean": + return "bool" + elif airbyte_type == "object": + return "string" + elif airbyte_type == "array": + return f"array({DestinationTimeplus.type_mapping(v['items'])})" + else: + return "string" + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the destination with the needed permissions + e.g: if a provided API token or password can be used to connect and write to the destination. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this destination, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + try: + endpoint = config["endpoint"] + apikey = config["apikey"] + if not endpoint.startswith("http"): + return AirbyteConnectionStatus(status=Status.FAILED, message="Endpoint must start with http or https") + if len(apikey) != 60: + return AirbyteConnectionStatus(status=Status.FAILED, message="API Key must be 60 characters") + if endpoint[-1] == "/": + endpoint = endpoint[0 : len(endpoint) - 1] + env = Environment().address(endpoint).apikey(apikey) + Stream(env=env).list() + logger.info("Successfully connected to " + endpoint) + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + return AirbyteConnectionStatus( + status=Status.FAILED, message=f"Fail to connect to Timeplus endpoint with the given API key: {repr(e)}" + ) diff --git a/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/spec.json b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/spec.json new file mode 100755 index 000000000000..6a56f1b0252e --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/destination_timeplus/spec.json @@ -0,0 +1,31 @@ +{ + "documentationUrl": "https://docs.timeplus.com", + "supported_destination_sync_modes": ["overwrite", "append"], + "supportsIncremental": true, + "supportsDBT": false, + "supportsNormalization": false, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Timeplus", + "type": "object", + "required": ["endpoint", "apikey"], + "additionalProperties": false, + "properties": { + "endpoint": { + "title": "Endpoint", + "description": "Timeplus workspace endpoint", + "type": "string", + "default": "https://us.timeplus.cloud/", + "examples": ["https://us.timeplus.cloud/workspace_id"], + "order": 0 + }, + "apikey": { + "title": "API key", + "description": "Personal API key", + "type": "string", + "airbyte_secret": true, + "order": 1 + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-timeplus/icon.svg b/airbyte-integrations/connectors/destination-timeplus/icon.svg new file mode 100644 index 000000000000..392443d947ae --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/icon.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-timeplus/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/destination-timeplus/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..96540519acb1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/integration_tests/configured_catalog.json @@ -0,0 +1,263 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte_single_str_col", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": { + "type": "object", + "properties": { + "raw": { + "type": "string" + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "airbyte_acceptance_table", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": { + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + }, + "column3": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone" + }, + "column4": { + "type": "number" + }, + "column5": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "airbyte_test_boolean", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": { + "type": "object", + "properties": { + "column1": { + "type": "boolean" + }, + "column2": { + "type": "number" + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "destination_sync_mode": "overwrite", + "stream": { + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] }, + "base_experience": { "type": ["null", "integer"] }, + "height": { "type": ["null", "integer"] }, + "is_default": { "type": ["null", "boolean"] }, + "order": { "type": ["null", "integer"] }, + "weight": { "type": ["null", "integer"] }, + "abilities": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "is_hidden": { "type": ["null", "boolean"] }, + "slot": { "type": ["null", "integer"] }, + "ability": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + } + } + }, + "forms": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + }, + "game_indices": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "game_index": { "type": ["null", "integer"] }, + "version": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + } + } + }, + "held_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "item": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "version_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "version": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "rarity": { "type": ["null", "integer"] } + } + } + } + } + } + }, + "location_area_encounters": { "type": ["null", "string"] }, + "moves": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "move": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "version_group_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "move_learn_method": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "version_group": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "level_learned_at": { "type": ["null", "integer"] } + } + } + } + } + } + }, + "sprites": { + "type": ["null", "object"], + "properties": { + "front_default": { "type": ["null", "string"] }, + "front_shiny": { "type": ["null", "string"] }, + "front_female": { "type": ["null", "string"] }, + "front_shiny_female": { "type": ["null", "string"] }, + "back_default": { "type": ["null", "string"] }, + "back_shiny": { "type": ["null", "string"] }, + "back_female": { "type": ["null", "string"] }, + "back_shiny_female": { "type": ["null", "string"] } + } + }, + "species": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "stats": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "stat": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + }, + "effort": { "type": ["null", "integer"] }, + "base_stat": { "type": ["null", "integer"] } + } + } + }, + "types": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "slot": { "type": ["null", "integer"] }, + "type": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } + } + } + } + } + } + } + }, + "name": "pokemon", + "source_defined_cursor": false, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py new file mode 100755 index 000000000000..e3de7dac9e71 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/integration_tests/integration_test.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from datetime import datetime +from typing import Any, Mapping + +import pytest +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from destination_timeplus import DestinationTimeplus + + +@pytest.fixture(name="config") +def config_fixture() -> Mapping[str, Any]: + with open("secrets/config.json", "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(name="configured_catalog") +def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} + append_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + ) + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + + return ConfiguredAirbyteCatalog(streams=[append_stream, overwrite_stream]) + + +def test_check_valid_config(config: Mapping): + outcome = DestinationTimeplus().check(logging.getLogger("airbyte"), config) + assert outcome.status == Status.SUCCEEDED + + +def test_check_invalid_config(): + outcome = DestinationTimeplus().check(logging.getLogger("airbyte"), {"secret_key": "not_a_real_secret"}) + assert outcome.status == Status.FAILED + + +def test_write(config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog): + records = [ + AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream="append_stream", + data={ + "string_col": "example", + "int_col": 1, + }, + emitted_at=int(datetime.now().timestamp()) * 1000, + ), + ) + ] + dest = DestinationTimeplus() + dest.write(config, configured_catalog, records) diff --git a/airbyte-integrations/connectors/destination-timeplus/integration_tests/messages.jsonl b/airbyte-integrations/connectors/destination-timeplus/integration_tests/messages.jsonl new file mode 100644 index 000000000000..6db122f96411 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/integration_tests/messages.jsonl @@ -0,0 +1,5 @@ +{"type": "RECORD", "record": {"stream": "airbyte_single_str_col", "data": {"raw": "my_value"}, "emitted_at": 1626172757000}} +{"type": "RECORD", "record": {"stream": "airbyte_acceptance_table", "data": {"column1": "my_value", "column2": 221, "column3": "2021-01-01T20:10:22", "column4": 1.214, "column5": [1,2,3]}, "emitted_at": 1626172757000}} +{"type": "RECORD", "record": {"stream": "airbyte_acceptance_table", "data": {"column1": "my_value2", "column2": 222, "column3": "2021-01-02T22:10:22", "column5": [1,2,null]}, "emitted_at": 1626172757000}} +{"type": "RECORD", "record": {"stream": "airbyte_test_boolean", "data": {"column1": true, "column2": 222}, "emitted_at": 1626172757000}} +{"type": "RECORD", "record": {"stream": "pokemon","data": { "abilities": [ { "ability": { "name": "limber", "url": "https://pokeapi.co/api/v2/ability/7/" }, "is_hidden": false, "slot": 1 }, { "ability": { "name": "imposter", "url": "https://pokeapi.co/api/v2/ability/150/" }, "is_hidden": true, "slot": 3 } ], "base_experience": 101, "forms": [ { "name": "ditto", "url": "https://pokeapi.co/api/v2/pokemon-form/132/" } ], "game_indices": [ { "game_index": 76, "version": { "name": "red", "url": "https://pokeapi.co/api/v2/version/1/" } }, { "game_index": 76, "version": { "name": "blue", "url": "https://pokeapi.co/api/v2/version/2/" } }, { "game_index": 76, "version": { "name": "yellow", "url": "https://pokeapi.co/api/v2/version/3/" } }, { "game_index": 132, "version": { "name": "gold", "url": "https://pokeapi.co/api/v2/version/4/" } }, { "game_index": 132, "version": { "name": "silver", "url": "https://pokeapi.co/api/v2/version/5/" } }, { "game_index": 132, "version": { "name": "crystal", "url": "https://pokeapi.co/api/v2/version/6/" } }, { "game_index": 132, "version": { "name": "ruby", "url": "https://pokeapi.co/api/v2/version/7/" } }, { "game_index": 132, "version": { "name": "sapphire", "url": "https://pokeapi.co/api/v2/version/8/" } }, { "game_index": 132, "version": { "name": "emerald", "url": "https://pokeapi.co/api/v2/version/9/" } }, { "game_index": 132, "version": { "name": "firered", "url": "https://pokeapi.co/api/v2/version/10/" } }, { "game_index": 132, "version": { "name": "leafgreen", "url": "https://pokeapi.co/api/v2/version/11/" } }, { "game_index": 132, "version": { "name": "diamond", "url": "https://pokeapi.co/api/v2/version/12/" } }, { "game_index": 132, "version": { "name": "pearl", "url": "https://pokeapi.co/api/v2/version/13/" } }, { "game_index": 132, "version": { "name": "platinum", "url": "https://pokeapi.co/api/v2/version/14/" } }, { "game_index": 132, "version": { "name": "heartgold", "url": "https://pokeapi.co/api/v2/version/15/" } }, { "game_index": 132, "version": { "name": "soulsilver", "url": "https://pokeapi.co/api/v2/version/16/" } }, { "game_index": 132, "version": { "name": "black", "url": "https://pokeapi.co/api/v2/version/17/" } }, { "game_index": 132, "version": { "name": "white", "url": "https://pokeapi.co/api/v2/version/18/" } }, { "game_index": 132, "version": { "name": "black-2", "url": "https://pokeapi.co/api/v2/version/21/" } }, { "game_index": 132, "version": { "name": "white-2", "url": "https://pokeapi.co/api/v2/version/22/" } } ], "height": 3, "held_items": [ { "item": { "name": "metal-powder", "url": "https://pokeapi.co/api/v2/item/234/" }, "version_details": [ { "rarity": 5, "version": { "name": "ruby", "url": "https://pokeapi.co/api/v2/version/7/" } }, { "rarity": 5, "version": { "name": "sapphire", "url": "https://pokeapi.co/api/v2/version/8/" } }, { "rarity": 5, "version": { "name": "emerald", "url": "https://pokeapi.co/api/v2/version/9/" } }, { "rarity": 5, "version": { "name": "firered", "url": "https://pokeapi.co/api/v2/version/10/" } }, { "rarity": 5, "version": { "name": "leafgreen", "url": "https://pokeapi.co/api/v2/version/11/" } }, { "rarity": 5, "version": { "name": "diamond", "url": "https://pokeapi.co/api/v2/version/12/" } }, { "rarity": 5, "version": { "name": "pearl", "url": "https://pokeapi.co/api/v2/version/13/" } }, { "rarity": 5, "version": { "name": "platinum", "url": "https://pokeapi.co/api/v2/version/14/" } }, { "rarity": 5, "version": { "name": "heartgold", "url": "https://pokeapi.co/api/v2/version/15/" } }, { "rarity": 5, "version": { "name": "soulsilver", "url": "https://pokeapi.co/api/v2/version/16/" } }, { "rarity": 5, "version": { "name": "black", "url": "https://pokeapi.co/api/v2/version/17/" } }, { "rarity": 5, "version": { "name": "white", "url": "https://pokeapi.co/api/v2/version/18/" } }, { "rarity": 5, "version": { "name": "black-2", "url": "https://pokeapi.co/api/v2/version/21/" } }, { "rarity": 5, "version": { "name": "white-2", "url": "https://pokeapi.co/api/v2/version/22/" } }, { "rarity": 5, "version": { "name": "x", "url": "https://pokeapi.co/api/v2/version/23/" } }, { "rarity": 5, "version": { "name": "y", "url": "https://pokeapi.co/api/v2/version/24/" } }, { "rarity": 5, "version": { "name": "omega-ruby", "url": "https://pokeapi.co/api/v2/version/25/" } }, { "rarity": 5, "version": { "name": "alpha-sapphire", "url": "https://pokeapi.co/api/v2/version/26/" } }, { "rarity": 5, "version": { "name": "sun", "url": "https://pokeapi.co/api/v2/version/27/" } }, { "rarity": 5, "version": { "name": "moon", "url": "https://pokeapi.co/api/v2/version/28/" } }, { "rarity": 5, "version": { "name": "ultra-sun", "url": "https://pokeapi.co/api/v2/version/29/" } }, { "rarity": 5, "version": { "name": "ultra-moon", "url": "https://pokeapi.co/api/v2/version/30/" } } ] }, { "item": { "name": "quick-powder", "url": "https://pokeapi.co/api/v2/item/251/" }, "version_details": [ { "rarity": 50, "version": { "name": "diamond", "url": "https://pokeapi.co/api/v2/version/12/" } }, { "rarity": 50, "version": { "name": "pearl", "url": "https://pokeapi.co/api/v2/version/13/" } }, { "rarity": 50, "version": { "name": "platinum", "url": "https://pokeapi.co/api/v2/version/14/" } }, { "rarity": 50, "version": { "name": "heartgold", "url": "https://pokeapi.co/api/v2/version/15/" } }, { "rarity": 50, "version": { "name": "soulsilver", "url": "https://pokeapi.co/api/v2/version/16/" } }, { "rarity": 50, "version": { "name": "black", "url": "https://pokeapi.co/api/v2/version/17/" } }, { "rarity": 50, "version": { "name": "white", "url": "https://pokeapi.co/api/v2/version/18/" } }, { "rarity": 50, "version": { "name": "black-2", "url": "https://pokeapi.co/api/v2/version/21/" } }, { "rarity": 50, "version": { "name": "white-2", "url": "https://pokeapi.co/api/v2/version/22/" } }, { "rarity": 50, "version": { "name": "x", "url": "https://pokeapi.co/api/v2/version/23/" } }, { "rarity": 50, "version": { "name": "y", "url": "https://pokeapi.co/api/v2/version/24/" } }, { "rarity": 50, "version": { "name": "omega-ruby", "url": "https://pokeapi.co/api/v2/version/25/" } }, { "rarity": 50, "version": { "name": "alpha-sapphire", "url": "https://pokeapi.co/api/v2/version/26/" } }, { "rarity": 50, "version": { "name": "sun", "url": "https://pokeapi.co/api/v2/version/27/" } }, { "rarity": 50, "version": { "name": "moon", "url": "https://pokeapi.co/api/v2/version/28/" } }, { "rarity": 50, "version": { "name": "ultra-sun", "url": "https://pokeapi.co/api/v2/version/29/" } }, { "rarity": 50, "version": { "name": "ultra-moon", "url": "https://pokeapi.co/api/v2/version/30/" } } ] } ], "id": 132, "is_default": true, "location_area_encounters": "https://pokeapi.co/api/v2/pokemon/132/encounters", "moves": [ { "move": { "name": "transform", "url": "https://pokeapi.co/api/v2/move/144/" }, "version_group_details": [ { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "red-blue", "url": "https://pokeapi.co/api/v2/version-group/1/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "yellow", "url": "https://pokeapi.co/api/v2/version-group/2/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "gold-silver", "url": "https://pokeapi.co/api/v2/version-group/3/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "crystal", "url": "https://pokeapi.co/api/v2/version-group/4/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "ruby-sapphire", "url": "https://pokeapi.co/api/v2/version-group/5/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "emerald", "url": "https://pokeapi.co/api/v2/version-group/6/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "firered-leafgreen", "url": "https://pokeapi.co/api/v2/version-group/7/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "diamond-pearl", "url": "https://pokeapi.co/api/v2/version-group/8/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "platinum", "url": "https://pokeapi.co/api/v2/version-group/9/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "heartgold-soulsilver", "url": "https://pokeapi.co/api/v2/version-group/10/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "black-white", "url": "https://pokeapi.co/api/v2/version-group/11/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "colosseum", "url": "https://pokeapi.co/api/v2/version-group/12/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "xd", "url": "https://pokeapi.co/api/v2/version-group/13/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "black-2-white-2", "url": "https://pokeapi.co/api/v2/version-group/14/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "x-y", "url": "https://pokeapi.co/api/v2/version-group/15/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "omega-ruby-alpha-sapphire", "url": "https://pokeapi.co/api/v2/version-group/16/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "sun-moon", "url": "https://pokeapi.co/api/v2/version-group/17/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "ultra-sun-ultra-moon", "url": "https://pokeapi.co/api/v2/version-group/18/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "lets-go-pikachu-lets-go-eevee", "url": "https://pokeapi.co/api/v2/version-group/19/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "sword-shield", "url": "https://pokeapi.co/api/v2/version-group/20/" } }, { "level_learned_at": 1, "move_learn_method": { "name": "level-up", "url": "https://pokeapi.co/api/v2/move-learn-method/1/" }, "version_group": { "name": "scarlet-violet", "url": "https://pokeapi.co/api/v2/version-group/25/" } } ] } ], "name": "ditto", "order": 214, "species": { "name": "ditto", "url": "https://pokeapi.co/api/v2/pokemon-species/132/" }, "sprites": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/132.png", "back_female": null, "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/132.png", "back_shiny_female": null, "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/132.png", "front_shiny_female": null, "other": { "dream_world": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/dream-world/132.svg", "front_female": null }, "home": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/132.png", "front_shiny_female": null }, "official-artwork": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/official-artwork/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/official-artwork/shiny/132.png" } }, "versions": { "generation-i": { "red-blue": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/132.png", "back_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/gray/132.png", "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/back/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/132.png", "front_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/gray/132.png", "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/132.png" }, "yellow": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/132.png", "back_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/gray/132.png", "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/back/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/132.png", "front_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/gray/132.png", "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/132.png" } }, "generation-ii": { "crystal": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/132.png", "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/shiny/132.png", "back_shiny_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/shiny/132.png", "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/shiny/132.png", "front_shiny_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/shiny/132.png", "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/132.png" }, "gold": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/132.png", "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/shiny/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/shiny/132.png", "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/transparent/132.png" }, "silver": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/132.png", "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/shiny/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/shiny/132.png", "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/transparent/132.png" } }, "generation-iii": { "emerald": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/shiny/132.png" }, "firered-leafgreen": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/132.png", "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/shiny/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/shiny/132.png" }, "ruby-sapphire": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/132.png", "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/shiny/132.png", "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/132.png", "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/shiny/132.png" } }, "generation-iv": { "diamond-pearl": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/132.png", "back_female": null, "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/132.png", "back_shiny_female": null, "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/132.png", "front_shiny_female": null }, "heartgold-soulsilver": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/132.png", "back_female": null, "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/132.png", "back_shiny_female": null, "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/132.png", "front_shiny_female": null }, "platinum": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/132.png", "back_female": null, "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/132.png", "back_shiny_female": null, "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/132.png", "front_shiny_female": null } }, "generation-v": { "black-white": { "animated": { "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/132.gif", "back_female": null, "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/132.gif", "back_shiny_female": null, "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/132.gif", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/132.gif", "front_shiny_female": null }, "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/132.png", "back_female": null, "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/132.png", "back_shiny_female": null, "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/132.png", "front_shiny_female": null } }, "generation-vi": { "omegaruby-alphasapphire": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/132.png", "front_shiny_female": null }, "x-y": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/132.png", "front_shiny_female": null } }, "generation-vii": { "icons": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/icons/132.png", "front_female": null }, "ultra-sun-ultra-moon": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/132.png", "front_female": null, "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/132.png", "front_shiny_female": null } }, "generation-viii": { "icons": { "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/132.png", "front_female": null } } } }, "stats": [ { "base_stat": 48, "effort": 1, "stat": { "name": "hp", "url": "https://pokeapi.co/api/v2/stat/1/" } }, { "base_stat": 48, "effort": 0, "stat": { "name": "attack", "url": "https://pokeapi.co/api/v2/stat/2/" } }, { "base_stat": 48, "effort": 0, "stat": { "name": "defense", "url": "https://pokeapi.co/api/v2/stat/3/" } }, { "base_stat": 48, "effort": 0, "stat": { "name": "special-attack", "url": "https://pokeapi.co/api/v2/stat/4/" } }, { "base_stat": 48, "effort": 0, "stat": { "name": "special-defense", "url": "https://pokeapi.co/api/v2/stat/5/" } }, { "base_stat": 48, "effort": 0, "stat": { "name": "speed", "url": "https://pokeapi.co/api/v2/stat/6/" } } ], "types": [ { "slot": 1, "type": { "name": "normal", "url": "https://pokeapi.co/api/v2/type/1/" } } ], "weight": 40 }, "emitted_at": 1673989852906 }} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-timeplus/main.py b/airbyte-integrations/connectors/destination-timeplus/main.py new file mode 100755 index 000000000000..a6f1b6b49d3c --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_timeplus import DestinationTimeplus + +if __name__ == "__main__": + DestinationTimeplus().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-timeplus/metadata.yaml b/airbyte-integrations/connectors/destination-timeplus/metadata.yaml new file mode 100644 index 000000000000..f1b8331630ab --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/metadata.yaml @@ -0,0 +1,24 @@ +data: + connectorSubtype: database + connectorType: destination + definitionId: f70a8ece-351e-4790-b37b-cb790bcd6d54 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/destination-timeplus + githubIssueLabel: destination-timeplus + icon: timeplus.svg + license: MIT + name: Timeplus + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/timeplus + tags: + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-timeplus/requirements.txt b/airbyte-integrations/connectors/destination-timeplus/requirements.txt new file mode 100755 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-timeplus/setup.py b/airbyte-integrations/connectors/destination-timeplus/setup.py new file mode 100755 index 000000000000..c082df533d8c --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/setup.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", + "timeplus~=1.2.1", +] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_timeplus", + description="Destination implementation for Timeplus.", + author="Airbyte", + author_email="jove@timeplus.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-timeplus/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-timeplus/unit_tests/unit_test.py new file mode 100755 index 000000000000..0b6359090af8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-timeplus/unit_tests/unit_test.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from destination_timeplus import DestinationTimeplus + + +def test_type_mapping(): + expected = { + "float": {"type": "number"}, + "bool": {"type": "boolean"}, + "string": {"type": "string"}, + "integer": {"type": "integer"}, + "array(integer)": {"type": "array", "items": {"type": "integer"}}, + } + for k, v in expected.items(): + assert k == DestinationTimeplus.type_mapping(v) diff --git a/airbyte-integrations/connectors/destination-typesense/Dockerfile b/airbyte-integrations/connectors/destination-typesense/Dockerfile index 0f5659af70c9..f5036e89ab0c 100644 --- a/airbyte-integrations/connectors/destination-typesense/Dockerfile +++ b/airbyte-integrations/connectors/destination-typesense/Dockerfile @@ -34,5 +34,5 @@ COPY destination_typesense ./destination_typesense ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-typesense diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py index ee3674debbe3..5e4de404d2af 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py @@ -18,7 +18,7 @@ def get_client(config: Mapping[str, Any]) -> Client: port = config.get("port") or "8108" protocol = config.get("protocol") or "https" - client = Client({"api_key": api_key, "nodes": [{"host": host, "port": port, "protocol": protocol}], "connection_timeout_seconds": 2}) + client = Client({"api_key": api_key, "nodes": [{"host": host, "port": port, "protocol": protocol}], "connection_timeout_seconds": 3600}) return client diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json b/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json index deda56407a91..46462eb37b93 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/spec.json @@ -35,7 +35,7 @@ }, "batch_size": { "title": "Batch size", - "type": "string", + "type": "integer", "description": "How many documents should be imported together. Default 1000", "order": 4 } diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py index a11b285d2d03..fd9c0e3b5868 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -14,10 +14,10 @@ class TypesenseWriter: write_buffer = [] - def __init__(self, client: Client, steam_name: str, batch_size: int = 1000): + def __init__(self, client: Client, steam_name: str, batch_size: int = None): self.client = client self.steam_name = steam_name - self.batch_size = batch_size + self.batch_size = batch_size or 10000 def queue_write_operation(self, data: Mapping): random_key = str(uuid4()) diff --git a/airbyte-integrations/connectors/destination-typesense/metadata.yaml b/airbyte-integrations/connectors/destination-typesense/metadata.yaml index cd8cac0aa5d0..b3ee53fe4fee 100644 --- a/airbyte-integrations/connectors/destination-typesense/metadata.yaml +++ b/airbyte-integrations/connectors/destination-typesense/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 36be8dc6-9851-49af-b776-9d4c30e4ab6a - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/destination-typesense githubIssueLabel: destination-typesense icon: typesense.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/typesense tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py index b282d9f510c4..ba065cb9fc02 100644 --- a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py @@ -7,6 +7,24 @@ from destination_typesense.writer import TypesenseWriter +@patch("typesense.Client") +def test_default_batch_size(client): + writer = TypesenseWriter(client, "steam_name") + assert writer.batch_size == 10000 + + +@patch("typesense.Client") +def test_empty_batch_size(client): + writer = TypesenseWriter(client, "steam_name", "") + assert writer.batch_size == 10000 + + +@patch("typesense.Client") +def test_custom_batch_size(client): + writer = TypesenseWriter(client, "steam_name", 9000) + assert writer.batch_size == 9000 + + @patch("typesense.Client") def test_queue_write_operation(client): writer = TypesenseWriter(client, "steam_name") diff --git a/airbyte-integrations/connectors/destination-vertica/metadata.yaml b/airbyte-integrations/connectors/destination-vertica/metadata.yaml index b4689848030f..ba8ce1298b98 100644 --- a/airbyte-integrations/connectors/destination-vertica/metadata.yaml +++ b/airbyte-integrations/connectors/destination-vertica/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/vertica tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-vertica/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-vertica/src/main/resources/spec.json index d809216efeee..682ccef2629c 100644 --- a/airbyte-integrations/connectors/destination-vertica/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-vertica/src/main/resources/spec.json @@ -8,7 +8,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Vertica Destination Spec", "type": "object", - "required": ["host", "port", "username", "database","schema"], + "required": ["host", "port", "username", "database", "schema"], "additionalProperties": true, "properties": { "host": { @@ -60,4 +60,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java index 4c634d505ccf..5ba85840d228 100644 --- a/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java @@ -16,6 +16,7 @@ import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import java.io.IOException; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import org.jooq.DSLContext; @@ -118,7 +119,7 @@ static void cleanUp() { } @Override - protected void setup(TestDestinationEnv testEnv) { + protected void setup(TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { // TODO Implement this method to run any setup actions needed before every test case } diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json index eb21a5795318..8980482fa3c5 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json @@ -1,61 +1,51 @@ { - "documentationUrl" : "https://docs.airbyte.com/integrations/destinations/weaviate", - "supported_destination_sync_modes" : [ - "append", - "overwrite" - ], - "supportsIncremental" : true, - "supportsDBT" : false, - "supportsNormalization" : false, - "connectionSpecification" : { - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "Destination Weaviate", - "type" : "object", - "required" : [ - "url" - ], - "additionalProperties" : false, - "properties" : { - "url" : { - "type" : "string", - "description" : "The URL to the weaviate instance", - "examples" : [ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/weaviate", + "supported_destination_sync_modes": ["append", "overwrite"], + "supportsIncremental": true, + "supportsDBT": false, + "supportsNormalization": false, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Weaviate", + "type": "object", + "required": ["url"], + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "The URL to the weaviate instance", + "examples": [ "http://localhost:8080", "https://your-instance.semi.network" ] }, - "username" : { - "type" : "string", - "description" : "Username used with OIDC authentication", - "examples" : [ - "xyz@weaviate.io" - ] + "username": { + "type": "string", + "description": "Username used with OIDC authentication", + "examples": ["xyz@weaviate.io"] }, - "password" : { - "type" : "string", - "description" : "Password used with OIDC authentication", - "airbyte_secret" : true + "password": { + "type": "string", + "description": "Password used with OIDC authentication", + "airbyte_secret": true }, - "batch_size" : { - "type" : "integer", - "description" : "Batch size for writing to Weaviate", - "default" : 100 + "batch_size": { + "type": "integer", + "description": "Batch size for writing to Weaviate", + "default": 100 }, - "vectors" : { - "type" : "string", - "description" : "Comma separated list of strings of `stream_name.vector_column_name` to specify which field holds the vectors.", - "examples" : [ + "vectors": { + "type": "string", + "description": "Comma separated list of strings of `stream_name.vector_column_name` to specify which field holds the vectors.", + "examples": [ "my_table.my_vector_column, another_table.vector", "mytable.vector" ] }, - "id_schema" : { - "type" : "string", - "description" : "Comma separated list of strings of `stream_name.id_column_name` to specify which field holds the ID of the record.", - "examples" : [ - "my_table.my_id_column, another_table.id", - "users.user_id" - ] + "id_schema": { + "type": "string", + "description": "Comma separated list of strings of `stream_name.id_column_name` to specify which field holds the ID of the record.", + "examples": ["my_table.my_id_column, another_table.id", "users.user_id"] } } } diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json index 4521b2af2dda..5d47610a805d 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json @@ -20,11 +20,8 @@ "creationTimeUnix": 1614852753746, "id": "b7b1cfbe-20da-496c-b932-008d35805f26", "properties": {}, - "vector": [ - -0.05244319, - 0.076136276 - ], + "vector": [-0.05244319, 0.076136276], "deprecations": null, "result": {} } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json index acf07c93140c..ca97fc7fc0ef 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json @@ -1 +1 @@ -{ "url": "http://localhost:8081"} +{ "url": "http://localhost:8081" } diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json index f8cf0bd05ffd..48d9ee0d8cb4 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json @@ -1,23 +1,23 @@ { - "type" : "object", - "properties" : { - "id" : { - "type" : "integer" + "type": "object", + "properties": { + "id": { + "type": "integer" }, - "currency" : { - "type" : "string" + "currency": { + "type": "string" }, - "date" : { - "type" : "string" + "date": { + "type": "string" }, - "HKD" : { - "type" : "number" + "HKD": { + "type": "number" }, - "NZD" : { - "type" : "number" + "NZD": { + "type": "number" }, - "USD" : { - "type" : "number" + "USD": { + "type": "number" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json index 602d8eb47a49..264481d37723 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json @@ -1 +1,11125 @@ -{"abilities":[{"ability":{"name":"static","url":"https://pokeapi.co/api/v2/ability/9/"},"is_hidden":false,"slot":1},{"ability":{"name":"lightning-rod","url":"https://pokeapi.co/api/v2/ability/31/"},"is_hidden":true,"slot":3}],"base_experience":112,"forms":[{"name":"pikachu","url":"https://pokeapi.co/api/v2/pokemon-form/25/"}],"game_indices":[{"game_index":84,"version":{"name":"red","url":"https://pokeapi.co/api/v2/version/1/"}},{"game_index":84,"version":{"name":"blue","url":"https://pokeapi.co/api/v2/version/2/"}},{"game_index":84,"version":{"name":"yellow","url":"https://pokeapi.co/api/v2/version/3/"}},{"game_index":25,"version":{"name":"gold","url":"https://pokeapi.co/api/v2/version/4/"}},{"game_index":25,"version":{"name":"silver","url":"https://pokeapi.co/api/v2/version/5/"}},{"game_index":25,"version":{"name":"crystal","url":"https://pokeapi.co/api/v2/version/6/"}},{"game_index":25,"version":{"name":"ruby","url":"https://pokeapi.co/api/v2/version/7/"}},{"game_index":25,"version":{"name":"sapphire","url":"https://pokeapi.co/api/v2/version/8/"}},{"game_index":25,"version":{"name":"emerald","url":"https://pokeapi.co/api/v2/version/9/"}},{"game_index":25,"version":{"name":"firered","url":"https://pokeapi.co/api/v2/version/10/"}},{"game_index":25,"version":{"name":"leafgreen","url":"https://pokeapi.co/api/v2/version/11/"}},{"game_index":25,"version":{"name":"diamond","url":"https://pokeapi.co/api/v2/version/12/"}},{"game_index":25,"version":{"name":"pearl","url":"https://pokeapi.co/api/v2/version/13/"}},{"game_index":25,"version":{"name":"platinum","url":"https://pokeapi.co/api/v2/version/14/"}},{"game_index":25,"version":{"name":"heartgold","url":"https://pokeapi.co/api/v2/version/15/"}},{"game_index":25,"version":{"name":"soulsilver","url":"https://pokeapi.co/api/v2/version/16/"}},{"game_index":25,"version":{"name":"black","url":"https://pokeapi.co/api/v2/version/17/"}},{"game_index":25,"version":{"name":"white","url":"https://pokeapi.co/api/v2/version/18/"}},{"game_index":25,"version":{"name":"black-2","url":"https://pokeapi.co/api/v2/version/21/"}},{"game_index":25,"version":{"name":"white-2","url":"https://pokeapi.co/api/v2/version/22/"}}],"height":4,"held_items":[{"item":{"name":"oran-berry","url":"https://pokeapi.co/api/v2/item/132/"},"version_details":[{"rarity":50,"version":{"name":"ruby","url":"https://pokeapi.co/api/v2/version/7/"}},{"rarity":50,"version":{"name":"sapphire","url":"https://pokeapi.co/api/v2/version/8/"}},{"rarity":50,"version":{"name":"emerald","url":"https://pokeapi.co/api/v2/version/9/"}},{"rarity":50,"version":{"name":"diamond","url":"https://pokeapi.co/api/v2/version/12/"}},{"rarity":50,"version":{"name":"pearl","url":"https://pokeapi.co/api/v2/version/13/"}},{"rarity":50,"version":{"name":"platinum","url":"https://pokeapi.co/api/v2/version/14/"}},{"rarity":50,"version":{"name":"heartgold","url":"https://pokeapi.co/api/v2/version/15/"}},{"rarity":50,"version":{"name":"soulsilver","url":"https://pokeapi.co/api/v2/version/16/"}},{"rarity":50,"version":{"name":"black","url":"https://pokeapi.co/api/v2/version/17/"}},{"rarity":50,"version":{"name":"white","url":"https://pokeapi.co/api/v2/version/18/"}}]},{"item":{"name":"light-ball","url":"https://pokeapi.co/api/v2/item/213/"},"version_details":[{"rarity":5,"version":{"name":"ruby","url":"https://pokeapi.co/api/v2/version/7/"}},{"rarity":5,"version":{"name":"sapphire","url":"https://pokeapi.co/api/v2/version/8/"}},{"rarity":5,"version":{"name":"emerald","url":"https://pokeapi.co/api/v2/version/9/"}},{"rarity":5,"version":{"name":"diamond","url":"https://pokeapi.co/api/v2/version/12/"}},{"rarity":5,"version":{"name":"pearl","url":"https://pokeapi.co/api/v2/version/13/"}},{"rarity":5,"version":{"name":"platinum","url":"https://pokeapi.co/api/v2/version/14/"}},{"rarity":5,"version":{"name":"heartgold","url":"https://pokeapi.co/api/v2/version/15/"}},{"rarity":5,"version":{"name":"soulsilver","url":"https://pokeapi.co/api/v2/version/16/"}},{"rarity":1,"version":{"name":"black","url":"https://pokeapi.co/api/v2/version/17/"}},{"rarity":1,"version":{"name":"white","url":"https://pokeapi.co/api/v2/version/18/"}},{"rarity":5,"version":{"name":"black-2","url":"https://pokeapi.co/api/v2/version/21/"}},{"rarity":5,"version":{"name":"white-2","url":"https://pokeapi.co/api/v2/version/22/"}},{"rarity":5,"version":{"name":"x","url":"https://pokeapi.co/api/v2/version/23/"}},{"rarity":5,"version":{"name":"y","url":"https://pokeapi.co/api/v2/version/24/"}},{"rarity":5,"version":{"name":"omega-ruby","url":"https://pokeapi.co/api/v2/version/25/"}},{"rarity":5,"version":{"name":"alpha-sapphire","url":"https://pokeapi.co/api/v2/version/26/"}},{"rarity":5,"version":{"name":"sun","url":"https://pokeapi.co/api/v2/version/27/"}},{"rarity":5,"version":{"name":"moon","url":"https://pokeapi.co/api/v2/version/28/"}},{"rarity":5,"version":{"name":"ultra-sun","url":"https://pokeapi.co/api/v2/version/29/"}},{"rarity":5,"version":{"name":"ultra-moon","url":"https://pokeapi.co/api/v2/version/30/"}}]}],"id":25,"is_default":true,"location_area_encounters":"https://pokeapi.co/api/v2/pokemon/25/encounters","moves":[{"move":{"name":"mega-punch","url":"https://pokeapi.co/api/v2/move/5/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"pay-day","url":"https://pokeapi.co/api/v2/move/6/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"thunder-punch","url":"https://pokeapi.co/api/v2/move/9/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"slam","url":"https://pokeapi.co/api/v2/move/21/"},"version_group_details":[{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":24,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":28,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"double-kick","url":"https://pokeapi.co/api/v2/move/24/"},"version_group_details":[{"level_learned_at":9,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}}]},{"move":{"name":"mega-kick","url":"https://pokeapi.co/api/v2/move/25/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"headbutt","url":"https://pokeapi.co/api/v2/move/29/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}}]},{"move":{"name":"body-slam","url":"https://pokeapi.co/api/v2/move/34/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"take-down","url":"https://pokeapi.co/api/v2/move/36/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}}]},{"move":{"name":"double-edge","url":"https://pokeapi.co/api/v2/move/38/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}}]},{"move":{"name":"tail-whip","url":"https://pokeapi.co/api/v2/move/39/"},"version_group_details":[{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":3,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"growl","url":"https://pokeapi.co/api/v2/move/45/"},"version_group_details":[{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":5,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"surf","url":"https://pokeapi.co/api/v2/move/57/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"stadium-surfing-pikachu","url":"https://pokeapi.co/api/v2/move-learn-method/5/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"stadium-surfing-pikachu","url":"https://pokeapi.co/api/v2/move-learn-method/5/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"submission","url":"https://pokeapi.co/api/v2/move/66/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}}]},{"move":{"name":"counter","url":"https://pokeapi.co/api/v2/move/68/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}}]},{"move":{"name":"seismic-toss","url":"https://pokeapi.co/api/v2/move/69/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}}]},{"move":{"name":"strength","url":"https://pokeapi.co/api/v2/move/70/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}}]},{"move":{"name":"thunder-shock","url":"https://pokeapi.co/api/v2/move/84/"},"version_group_details":[{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"thunderbolt","url":"https://pokeapi.co/api/v2/move/85/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":36,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"thunder-wave","url":"https://pokeapi.co/api/v2/move/86/"},"version_group_details":[{"level_learned_at":9,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":4,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"thunder","url":"https://pokeapi.co/api/v2/move/87/"},"version_group_details":[{"level_learned_at":43,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":41,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":58,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":58,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":58,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":30,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":44,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"dig","url":"https://pokeapi.co/api/v2/move/91/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"toxic","url":"https://pokeapi.co/api/v2/move/92/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}}]},{"move":{"name":"agility","url":"https://pokeapi.co/api/v2/move/97/"},"version_group_details":[{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":33,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":27,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":24,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"quick-attack","url":"https://pokeapi.co/api/v2/move/98/"},"version_group_details":[{"level_learned_at":16,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":11,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":10,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":6,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"rage","url":"https://pokeapi.co/api/v2/move/99/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}}]},{"move":{"name":"mimic","url":"https://pokeapi.co/api/v2/move/102/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}}]},{"move":{"name":"double-team","url":"https://pokeapi.co/api/v2/move/104/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":15,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":23,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":23,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":23,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":12,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":8,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"defense-curl","url":"https://pokeapi.co/api/v2/move/111/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}}]},{"move":{"name":"light-screen","url":"https://pokeapi.co/api/v2/move/113/"},"version_group_details":[{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":45,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":53,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":53,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":53,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":40,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"reflect","url":"https://pokeapi.co/api/v2/move/115/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"bide","url":"https://pokeapi.co/api/v2/move/117/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}}]},{"move":{"name":"swift","url":"https://pokeapi.co/api/v2/move/129/"},"version_group_details":[{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"skull-bash","url":"https://pokeapi.co/api/v2/move/130/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}}]},{"move":{"name":"flash","url":"https://pokeapi.co/api/v2/move/148/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}}]},{"move":{"name":"rest","url":"https://pokeapi.co/api/v2/move/156/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"substitute","url":"https://pokeapi.co/api/v2/move/164/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"red-blue","url":"https://pokeapi.co/api/v2/version-group/1/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"yellow","url":"https://pokeapi.co/api/v2/version-group/2/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"thief","url":"https://pokeapi.co/api/v2/move/168/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"snore","url":"https://pokeapi.co/api/v2/move/173/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"curse","url":"https://pokeapi.co/api/v2/move/174/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}}]},{"move":{"name":"reversal","url":"https://pokeapi.co/api/v2/move/179/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"protect","url":"https://pokeapi.co/api/v2/move/182/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"sweet-kiss","url":"https://pokeapi.co/api/v2/move/186/"},"version_group_details":[{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"mud-slap","url":"https://pokeapi.co/api/v2/move/189/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}}]},{"move":{"name":"zap-cannon","url":"https://pokeapi.co/api/v2/move/192/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}}]},{"move":{"name":"detect","url":"https://pokeapi.co/api/v2/move/197/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}}]},{"move":{"name":"endure","url":"https://pokeapi.co/api/v2/move/203/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"charm","url":"https://pokeapi.co/api/v2/move/204/"},"version_group_details":[{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"rollout","url":"https://pokeapi.co/api/v2/move/205/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}}]},{"move":{"name":"swagger","url":"https://pokeapi.co/api/v2/move/207/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"spark","url":"https://pokeapi.co/api/v2/move/209/"},"version_group_details":[{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":26,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":20,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"attract","url":"https://pokeapi.co/api/v2/move/213/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"sleep-talk","url":"https://pokeapi.co/api/v2/move/214/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"return","url":"https://pokeapi.co/api/v2/move/216/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"frustration","url":"https://pokeapi.co/api/v2/move/218/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"dynamic-punch","url":"https://pokeapi.co/api/v2/move/223/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}}]},{"move":{"name":"encore","url":"https://pokeapi.co/api/v2/move/227/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"iron-tail","url":"https://pokeapi.co/api/v2/move/231/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"hidden-power","url":"https://pokeapi.co/api/v2/move/237/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"rain-dance","url":"https://pokeapi.co/api/v2/move/240/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"gold-silver","url":"https://pokeapi.co/api/v2/version-group/3/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"crystal","url":"https://pokeapi.co/api/v2/version-group/4/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"rock-smash","url":"https://pokeapi.co/api/v2/move/249/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}}]},{"move":{"name":"uproar","url":"https://pokeapi.co/api/v2/move/253/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"facade","url":"https://pokeapi.co/api/v2/move/263/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"focus-punch","url":"https://pokeapi.co/api/v2/move/264/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"helping-hand","url":"https://pokeapi.co/api/v2/move/270/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"brick-break","url":"https://pokeapi.co/api/v2/move/280/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"knock-off","url":"https://pokeapi.co/api/v2/move/282/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"secret-power","url":"https://pokeapi.co/api/v2/move/290/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}}]},{"move":{"name":"signal-beam","url":"https://pokeapi.co/api/v2/move/324/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"covet","url":"https://pokeapi.co/api/v2/move/343/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"volt-tackle","url":"https://pokeapi.co/api/v2/move/344/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"calm-mind","url":"https://pokeapi.co/api/v2/move/347/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"lets-go-pikachu-lets-go-eevee","url":"https://pokeapi.co/api/v2/version-group/19/"}}]},{"move":{"name":"shock-wave","url":"https://pokeapi.co/api/v2/move/351/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ruby-sapphire","url":"https://pokeapi.co/api/v2/version-group/5/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"emerald","url":"https://pokeapi.co/api/v2/version-group/6/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"firered-leafgreen","url":"https://pokeapi.co/api/v2/version-group/7/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"colosseum","url":"https://pokeapi.co/api/v2/version-group/12/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"xd","url":"https://pokeapi.co/api/v2/version-group/13/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"natural-gift","url":"https://pokeapi.co/api/v2/move/363/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}}]},{"move":{"name":"feint","url":"https://pokeapi.co/api/v2/move/364/"},"version_group_details":[{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":21,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":16,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"fling","url":"https://pokeapi.co/api/v2/move/374/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"magnet-rise","url":"https://pokeapi.co/api/v2/move/393/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"nasty-plot","url":"https://pokeapi.co/api/v2/move/417/"},"version_group_details":[{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"discharge","url":"https://pokeapi.co/api/v2/move/435/"},"version_group_details":[{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":37,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":42,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":34,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":32,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"captivate","url":"https://pokeapi.co/api/v2/move/445/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}}]},{"move":{"name":"grass-knot","url":"https://pokeapi.co/api/v2/move/447/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"charge-beam","url":"https://pokeapi.co/api/v2/move/451/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"diamond-pearl","url":"https://pokeapi.co/api/v2/version-group/8/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"platinum","url":"https://pokeapi.co/api/v2/version-group/9/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"heartgold-soulsilver","url":"https://pokeapi.co/api/v2/version-group/10/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"electro-ball","url":"https://pokeapi.co/api/v2/move/486/"},"version_group_details":[{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":18,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":13,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":12,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"round","url":"https://pokeapi.co/api/v2/move/496/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"echoed-voice","url":"https://pokeapi.co/api/v2/move/497/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"volt-switch","url":"https://pokeapi.co/api/v2/move/521/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"electroweb","url":"https://pokeapi.co/api/v2/move/527/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"wild-charge","url":"https://pokeapi.co/api/v2/move/528/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-white","url":"https://pokeapi.co/api/v2/version-group/11/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"black-2-white-2","url":"https://pokeapi.co/api/v2/version-group/14/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":50,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"draining-kiss","url":"https://pokeapi.co/api/v2/move/577/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"play-rough","url":"https://pokeapi.co/api/v2/move/583/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"play-nice","url":"https://pokeapi.co/api/v2/move/589/"},"version_group_details":[{"level_learned_at":7,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":7,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":7,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":7,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"confide","url":"https://pokeapi.co/api/v2/move/590/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"electric-terrain","url":"https://pokeapi.co/api/v2/move/604/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"machine","url":"https://pokeapi.co/api/v2/move-learn-method/4/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"nuzzle","url":"https://pokeapi.co/api/v2/move/609/"},"version_group_details":[{"level_learned_at":23,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"x-y","url":"https://pokeapi.co/api/v2/version-group/15/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"omega-ruby-alpha-sapphire","url":"https://pokeapi.co/api/v2/version-group/16/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sun-moon","url":"https://pokeapi.co/api/v2/version-group/17/"}},{"level_learned_at":29,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}},{"level_learned_at":1,"move_learn_method":{"name":"level-up","url":"https://pokeapi.co/api/v2/move-learn-method/1/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]},{"move":{"name":"laser-focus","url":"https://pokeapi.co/api/v2/move/673/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"ultra-sun-ultra-moon","url":"https://pokeapi.co/api/v2/version-group/18/"}}]},{"move":{"name":"rising-voltage","url":"https://pokeapi.co/api/v2/move/804/"},"version_group_details":[{"level_learned_at":0,"move_learn_method":{"name":"tutor","url":"https://pokeapi.co/api/v2/move-learn-method/3/"},"version_group":{"name":"sword-shield","url":"https://pokeapi.co/api/v2/version-group/20/"}}]}],"name":"pikachu","order":35,"past_types":[],"species":{"name":"pikachu","url":"https://pokeapi.co/api/v2/pokemon-species/25/"},"sprites":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/25.png","back_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/female/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/25.png","back_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/female/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/female/25.png","other":{"dream_world":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/dream-world/25.svg","front_female":null},"home":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/female/25.png"},"official-artwork":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/official-artwork/25.png"}},"versions":{"generation-i":{"red-blue":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/25.png","back_gray":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/gray/25.png","back_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/back/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/25.png","front_gray":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/gray/25.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/25.png"},"yellow":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/25.png","back_gray":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/gray/25.png","back_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/back/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/25.png","front_gray":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/gray/25.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/25.png"}},"generation-ii":{"crystal":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/shiny/25.png","back_shiny_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/shiny/25.png","back_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/shiny/25.png","front_shiny_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/shiny/25.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/25.png"},"gold":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/shiny/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/shiny/25.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/transparent/25.png"},"silver":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/shiny/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/shiny/25.png","front_transparent":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/transparent/25.png"}},"generation-iii":{"emerald":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/shiny/25.png"},"firered-leafgreen":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/shiny/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/shiny/25.png"},"ruby-sapphire":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/shiny/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/shiny/25.png"}},"generation-iv":{"diamond-pearl":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/25.png","back_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/female/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/25.png","back_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/female/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/female/25.png"},"heartgold-soulsilver":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/25.png","back_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/female/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/25.png","back_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/female/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/female/25.png"},"platinum":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/25.png","back_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/female/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/25.png","back_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/female/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/female/25.png"}},"generation-v":{"black-white":{"animated":{"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/25.gif","back_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/female/25.gif","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/25.gif","back_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/female/25.gif","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/25.gif","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/female/25.gif","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/25.gif","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/female/25.gif"},"back_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/25.png","back_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/female/25.png","back_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/25.png","back_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/female/25.png","front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/female/25.png"}},"generation-vi":{"omegaruby-alphasapphire":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/female/25.png"},"x-y":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/female/25.png"}},"generation-vii":{"icons":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/icons/25.png","front_female":null},"ultra-sun-ultra-moon":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/female/25.png","front_shiny":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/25.png","front_shiny_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/female/25.png"}},"generation-viii":{"icons":{"front_default":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/25.png","front_female":"https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/female/25.png"}}}},"stats":[{"base_stat":35,"effort":0,"stat":{"name":"hp","url":"https://pokeapi.co/api/v2/stat/1/"}},{"base_stat":55,"effort":0,"stat":{"name":"attack","url":"https://pokeapi.co/api/v2/stat/2/"}},{"base_stat":40,"effort":0,"stat":{"name":"defense","url":"https://pokeapi.co/api/v2/stat/3/"}},{"base_stat":50,"effort":0,"stat":{"name":"special-attack","url":"https://pokeapi.co/api/v2/stat/4/"}},{"base_stat":50,"effort":0,"stat":{"name":"special-defense","url":"https://pokeapi.co/api/v2/stat/5/"}},{"base_stat":90,"effort":2,"stat":{"name":"speed","url":"https://pokeapi.co/api/v2/stat/6/"}}],"types":[{"slot":1,"type":{"name":"electric","url":"https://pokeapi.co/api/v2/type/13/"}}],"weight":60} \ No newline at end of file +{ + "abilities": [ + { + "ability": { + "name": "static", + "url": "https://pokeapi.co/api/v2/ability/9/" + }, + "is_hidden": false, + "slot": 1 + }, + { + "ability": { + "name": "lightning-rod", + "url": "https://pokeapi.co/api/v2/ability/31/" + }, + "is_hidden": true, + "slot": 3 + } + ], + "base_experience": 112, + "forms": [ + { "name": "pikachu", "url": "https://pokeapi.co/api/v2/pokemon-form/25/" } + ], + "game_indices": [ + { + "game_index": 84, + "version": { + "name": "red", + "url": "https://pokeapi.co/api/v2/version/1/" + } + }, + { + "game_index": 84, + "version": { + "name": "blue", + "url": "https://pokeapi.co/api/v2/version/2/" + } + }, + { + "game_index": 84, + "version": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version/3/" + } + }, + { + "game_index": 25, + "version": { + "name": "gold", + "url": "https://pokeapi.co/api/v2/version/4/" + } + }, + { + "game_index": 25, + "version": { + "name": "silver", + "url": "https://pokeapi.co/api/v2/version/5/" + } + }, + { + "game_index": 25, + "version": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version/6/" + } + }, + { + "game_index": 25, + "version": { + "name": "ruby", + "url": "https://pokeapi.co/api/v2/version/7/" + } + }, + { + "game_index": 25, + "version": { + "name": "sapphire", + "url": "https://pokeapi.co/api/v2/version/8/" + } + }, + { + "game_index": 25, + "version": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version/9/" + } + }, + { + "game_index": 25, + "version": { + "name": "firered", + "url": "https://pokeapi.co/api/v2/version/10/" + } + }, + { + "game_index": 25, + "version": { + "name": "leafgreen", + "url": "https://pokeapi.co/api/v2/version/11/" + } + }, + { + "game_index": 25, + "version": { + "name": "diamond", + "url": "https://pokeapi.co/api/v2/version/12/" + } + }, + { + "game_index": 25, + "version": { + "name": "pearl", + "url": "https://pokeapi.co/api/v2/version/13/" + } + }, + { + "game_index": 25, + "version": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version/14/" + } + }, + { + "game_index": 25, + "version": { + "name": "heartgold", + "url": "https://pokeapi.co/api/v2/version/15/" + } + }, + { + "game_index": 25, + "version": { + "name": "soulsilver", + "url": "https://pokeapi.co/api/v2/version/16/" + } + }, + { + "game_index": 25, + "version": { + "name": "black", + "url": "https://pokeapi.co/api/v2/version/17/" + } + }, + { + "game_index": 25, + "version": { + "name": "white", + "url": "https://pokeapi.co/api/v2/version/18/" + } + }, + { + "game_index": 25, + "version": { + "name": "black-2", + "url": "https://pokeapi.co/api/v2/version/21/" + } + }, + { + "game_index": 25, + "version": { + "name": "white-2", + "url": "https://pokeapi.co/api/v2/version/22/" + } + } + ], + "height": 4, + "held_items": [ + { + "item": { + "name": "oran-berry", + "url": "https://pokeapi.co/api/v2/item/132/" + }, + "version_details": [ + { + "rarity": 50, + "version": { + "name": "ruby", + "url": "https://pokeapi.co/api/v2/version/7/" + } + }, + { + "rarity": 50, + "version": { + "name": "sapphire", + "url": "https://pokeapi.co/api/v2/version/8/" + } + }, + { + "rarity": 50, + "version": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version/9/" + } + }, + { + "rarity": 50, + "version": { + "name": "diamond", + "url": "https://pokeapi.co/api/v2/version/12/" + } + }, + { + "rarity": 50, + "version": { + "name": "pearl", + "url": "https://pokeapi.co/api/v2/version/13/" + } + }, + { + "rarity": 50, + "version": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version/14/" + } + }, + { + "rarity": 50, + "version": { + "name": "heartgold", + "url": "https://pokeapi.co/api/v2/version/15/" + } + }, + { + "rarity": 50, + "version": { + "name": "soulsilver", + "url": "https://pokeapi.co/api/v2/version/16/" + } + }, + { + "rarity": 50, + "version": { + "name": "black", + "url": "https://pokeapi.co/api/v2/version/17/" + } + }, + { + "rarity": 50, + "version": { + "name": "white", + "url": "https://pokeapi.co/api/v2/version/18/" + } + } + ] + }, + { + "item": { + "name": "light-ball", + "url": "https://pokeapi.co/api/v2/item/213/" + }, + "version_details": [ + { + "rarity": 5, + "version": { + "name": "ruby", + "url": "https://pokeapi.co/api/v2/version/7/" + } + }, + { + "rarity": 5, + "version": { + "name": "sapphire", + "url": "https://pokeapi.co/api/v2/version/8/" + } + }, + { + "rarity": 5, + "version": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version/9/" + } + }, + { + "rarity": 5, + "version": { + "name": "diamond", + "url": "https://pokeapi.co/api/v2/version/12/" + } + }, + { + "rarity": 5, + "version": { + "name": "pearl", + "url": "https://pokeapi.co/api/v2/version/13/" + } + }, + { + "rarity": 5, + "version": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version/14/" + } + }, + { + "rarity": 5, + "version": { + "name": "heartgold", + "url": "https://pokeapi.co/api/v2/version/15/" + } + }, + { + "rarity": 5, + "version": { + "name": "soulsilver", + "url": "https://pokeapi.co/api/v2/version/16/" + } + }, + { + "rarity": 1, + "version": { + "name": "black", + "url": "https://pokeapi.co/api/v2/version/17/" + } + }, + { + "rarity": 1, + "version": { + "name": "white", + "url": "https://pokeapi.co/api/v2/version/18/" + } + }, + { + "rarity": 5, + "version": { + "name": "black-2", + "url": "https://pokeapi.co/api/v2/version/21/" + } + }, + { + "rarity": 5, + "version": { + "name": "white-2", + "url": "https://pokeapi.co/api/v2/version/22/" + } + }, + { + "rarity": 5, + "version": { + "name": "x", + "url": "https://pokeapi.co/api/v2/version/23/" + } + }, + { + "rarity": 5, + "version": { + "name": "y", + "url": "https://pokeapi.co/api/v2/version/24/" + } + }, + { + "rarity": 5, + "version": { + "name": "omega-ruby", + "url": "https://pokeapi.co/api/v2/version/25/" + } + }, + { + "rarity": 5, + "version": { + "name": "alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version/26/" + } + }, + { + "rarity": 5, + "version": { + "name": "sun", + "url": "https://pokeapi.co/api/v2/version/27/" + } + }, + { + "rarity": 5, + "version": { + "name": "moon", + "url": "https://pokeapi.co/api/v2/version/28/" + } + }, + { + "rarity": 5, + "version": { + "name": "ultra-sun", + "url": "https://pokeapi.co/api/v2/version/29/" + } + }, + { + "rarity": 5, + "version": { + "name": "ultra-moon", + "url": "https://pokeapi.co/api/v2/version/30/" + } + } + ] + } + ], + "id": 25, + "is_default": true, + "location_area_encounters": "https://pokeapi.co/api/v2/pokemon/25/encounters", + "moves": [ + { + "move": { + "name": "mega-punch", + "url": "https://pokeapi.co/api/v2/move/5/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "pay-day", "url": "https://pokeapi.co/api/v2/move/6/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "thunder-punch", + "url": "https://pokeapi.co/api/v2/move/9/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "slam", "url": "https://pokeapi.co/api/v2/move/21/" }, + "version_group_details": [ + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 24, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 28, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "double-kick", + "url": "https://pokeapi.co/api/v2/move/24/" + }, + "version_group_details": [ + { + "level_learned_at": 9, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + } + ] + }, + { + "move": { + "name": "mega-kick", + "url": "https://pokeapi.co/api/v2/move/25/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "headbutt", + "url": "https://pokeapi.co/api/v2/move/29/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + } + ] + }, + { + "move": { + "name": "body-slam", + "url": "https://pokeapi.co/api/v2/move/34/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "take-down", + "url": "https://pokeapi.co/api/v2/move/36/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + } + ] + }, + { + "move": { + "name": "double-edge", + "url": "https://pokeapi.co/api/v2/move/38/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + } + ] + }, + { + "move": { + "name": "tail-whip", + "url": "https://pokeapi.co/api/v2/move/39/" + }, + "version_group_details": [ + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 3, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "growl", "url": "https://pokeapi.co/api/v2/move/45/" }, + "version_group_details": [ + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 5, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "surf", "url": "https://pokeapi.co/api/v2/move/57/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "stadium-surfing-pikachu", + "url": "https://pokeapi.co/api/v2/move-learn-method/5/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "stadium-surfing-pikachu", + "url": "https://pokeapi.co/api/v2/move-learn-method/5/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "submission", + "url": "https://pokeapi.co/api/v2/move/66/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + } + ] + }, + { + "move": { + "name": "counter", + "url": "https://pokeapi.co/api/v2/move/68/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + } + ] + }, + { + "move": { + "name": "seismic-toss", + "url": "https://pokeapi.co/api/v2/move/69/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + } + ] + }, + { + "move": { + "name": "strength", + "url": "https://pokeapi.co/api/v2/move/70/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + } + ] + }, + { + "move": { + "name": "thunder-shock", + "url": "https://pokeapi.co/api/v2/move/84/" + }, + "version_group_details": [ + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "thunderbolt", + "url": "https://pokeapi.co/api/v2/move/85/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 36, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "thunder-wave", + "url": "https://pokeapi.co/api/v2/move/86/" + }, + "version_group_details": [ + { + "level_learned_at": 9, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 4, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "thunder", + "url": "https://pokeapi.co/api/v2/move/87/" + }, + "version_group_details": [ + { + "level_learned_at": 43, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 41, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 58, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 58, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 58, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 30, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 44, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "dig", "url": "https://pokeapi.co/api/v2/move/91/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "toxic", "url": "https://pokeapi.co/api/v2/move/92/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + } + ] + }, + { + "move": { + "name": "agility", + "url": "https://pokeapi.co/api/v2/move/97/" + }, + "version_group_details": [ + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 33, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 27, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 24, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "quick-attack", + "url": "https://pokeapi.co/api/v2/move/98/" + }, + "version_group_details": [ + { + "level_learned_at": 16, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 11, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 10, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 6, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "rage", "url": "https://pokeapi.co/api/v2/move/99/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + } + ] + }, + { + "move": { "name": "mimic", "url": "https://pokeapi.co/api/v2/move/102/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + } + ] + }, + { + "move": { + "name": "double-team", + "url": "https://pokeapi.co/api/v2/move/104/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 15, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 23, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 23, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 23, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 12, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 8, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "defense-curl", + "url": "https://pokeapi.co/api/v2/move/111/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + } + ] + }, + { + "move": { + "name": "light-screen", + "url": "https://pokeapi.co/api/v2/move/113/" + }, + "version_group_details": [ + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 45, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 53, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 53, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 53, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 40, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "reflect", + "url": "https://pokeapi.co/api/v2/move/115/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "bide", "url": "https://pokeapi.co/api/v2/move/117/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + } + ] + }, + { + "move": { "name": "swift", "url": "https://pokeapi.co/api/v2/move/129/" }, + "version_group_details": [ + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "skull-bash", + "url": "https://pokeapi.co/api/v2/move/130/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + } + ] + }, + { + "move": { "name": "flash", "url": "https://pokeapi.co/api/v2/move/148/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + } + ] + }, + { + "move": { "name": "rest", "url": "https://pokeapi.co/api/v2/move/156/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "substitute", + "url": "https://pokeapi.co/api/v2/move/164/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "red-blue", + "url": "https://pokeapi.co/api/v2/version-group/1/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "yellow", + "url": "https://pokeapi.co/api/v2/version-group/2/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "thief", "url": "https://pokeapi.co/api/v2/move/168/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "snore", "url": "https://pokeapi.co/api/v2/move/173/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "curse", "url": "https://pokeapi.co/api/v2/move/174/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + } + ] + }, + { + "move": { + "name": "reversal", + "url": "https://pokeapi.co/api/v2/move/179/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "protect", + "url": "https://pokeapi.co/api/v2/move/182/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "sweet-kiss", + "url": "https://pokeapi.co/api/v2/move/186/" + }, + "version_group_details": [ + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "mud-slap", + "url": "https://pokeapi.co/api/v2/move/189/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + } + ] + }, + { + "move": { + "name": "zap-cannon", + "url": "https://pokeapi.co/api/v2/move/192/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + } + ] + }, + { + "move": { + "name": "detect", + "url": "https://pokeapi.co/api/v2/move/197/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + } + ] + }, + { + "move": { + "name": "endure", + "url": "https://pokeapi.co/api/v2/move/203/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "charm", "url": "https://pokeapi.co/api/v2/move/204/" }, + "version_group_details": [ + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "rollout", + "url": "https://pokeapi.co/api/v2/move/205/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + } + ] + }, + { + "move": { + "name": "swagger", + "url": "https://pokeapi.co/api/v2/move/207/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { "name": "spark", "url": "https://pokeapi.co/api/v2/move/209/" }, + "version_group_details": [ + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 26, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 20, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "attract", + "url": "https://pokeapi.co/api/v2/move/213/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "sleep-talk", + "url": "https://pokeapi.co/api/v2/move/214/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "return", + "url": "https://pokeapi.co/api/v2/move/216/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "frustration", + "url": "https://pokeapi.co/api/v2/move/218/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "dynamic-punch", + "url": "https://pokeapi.co/api/v2/move/223/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + } + ] + }, + { + "move": { + "name": "encore", + "url": "https://pokeapi.co/api/v2/move/227/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "iron-tail", + "url": "https://pokeapi.co/api/v2/move/231/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "hidden-power", + "url": "https://pokeapi.co/api/v2/move/237/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "rain-dance", + "url": "https://pokeapi.co/api/v2/move/240/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "gold-silver", + "url": "https://pokeapi.co/api/v2/version-group/3/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "crystal", + "url": "https://pokeapi.co/api/v2/version-group/4/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "rock-smash", + "url": "https://pokeapi.co/api/v2/move/249/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + } + ] + }, + { + "move": { + "name": "uproar", + "url": "https://pokeapi.co/api/v2/move/253/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "facade", + "url": "https://pokeapi.co/api/v2/move/263/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "focus-punch", + "url": "https://pokeapi.co/api/v2/move/264/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "helping-hand", + "url": "https://pokeapi.co/api/v2/move/270/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "brick-break", + "url": "https://pokeapi.co/api/v2/move/280/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "knock-off", + "url": "https://pokeapi.co/api/v2/move/282/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "secret-power", + "url": "https://pokeapi.co/api/v2/move/290/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + } + ] + }, + { + "move": { + "name": "signal-beam", + "url": "https://pokeapi.co/api/v2/move/324/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { "name": "covet", "url": "https://pokeapi.co/api/v2/move/343/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "volt-tackle", + "url": "https://pokeapi.co/api/v2/move/344/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "calm-mind", + "url": "https://pokeapi.co/api/v2/move/347/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "lets-go-pikachu-lets-go-eevee", + "url": "https://pokeapi.co/api/v2/version-group/19/" + } + } + ] + }, + { + "move": { + "name": "shock-wave", + "url": "https://pokeapi.co/api/v2/move/351/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ruby-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/5/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "emerald", + "url": "https://pokeapi.co/api/v2/version-group/6/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "firered-leafgreen", + "url": "https://pokeapi.co/api/v2/version-group/7/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "colosseum", + "url": "https://pokeapi.co/api/v2/version-group/12/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "xd", + "url": "https://pokeapi.co/api/v2/version-group/13/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "natural-gift", + "url": "https://pokeapi.co/api/v2/move/363/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + } + ] + }, + { + "move": { "name": "feint", "url": "https://pokeapi.co/api/v2/move/364/" }, + "version_group_details": [ + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 21, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 16, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "fling", "url": "https://pokeapi.co/api/v2/move/374/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "magnet-rise", + "url": "https://pokeapi.co/api/v2/move/393/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "nasty-plot", + "url": "https://pokeapi.co/api/v2/move/417/" + }, + "version_group_details": [ + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "discharge", + "url": "https://pokeapi.co/api/v2/move/435/" + }, + "version_group_details": [ + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 37, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 42, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 34, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 32, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "captivate", + "url": "https://pokeapi.co/api/v2/move/445/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + } + ] + }, + { + "move": { + "name": "grass-knot", + "url": "https://pokeapi.co/api/v2/move/447/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "charge-beam", + "url": "https://pokeapi.co/api/v2/move/451/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "diamond-pearl", + "url": "https://pokeapi.co/api/v2/version-group/8/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "platinum", + "url": "https://pokeapi.co/api/v2/version-group/9/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "heartgold-soulsilver", + "url": "https://pokeapi.co/api/v2/version-group/10/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "electro-ball", + "url": "https://pokeapi.co/api/v2/move/486/" + }, + "version_group_details": [ + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 18, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 13, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 12, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { "name": "round", "url": "https://pokeapi.co/api/v2/move/496/" }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "echoed-voice", + "url": "https://pokeapi.co/api/v2/move/497/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "volt-switch", + "url": "https://pokeapi.co/api/v2/move/521/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "electroweb", + "url": "https://pokeapi.co/api/v2/move/527/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "wild-charge", + "url": "https://pokeapi.co/api/v2/move/528/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-white", + "url": "https://pokeapi.co/api/v2/version-group/11/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "black-2-white-2", + "url": "https://pokeapi.co/api/v2/version-group/14/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 50, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "draining-kiss", + "url": "https://pokeapi.co/api/v2/move/577/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "play-rough", + "url": "https://pokeapi.co/api/v2/move/583/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "play-nice", + "url": "https://pokeapi.co/api/v2/move/589/" + }, + "version_group_details": [ + { + "level_learned_at": 7, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 7, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 7, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 7, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "confide", + "url": "https://pokeapi.co/api/v2/move/590/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "electric-terrain", + "url": "https://pokeapi.co/api/v2/move/604/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "machine", + "url": "https://pokeapi.co/api/v2/move-learn-method/4/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "nuzzle", + "url": "https://pokeapi.co/api/v2/move/609/" + }, + "version_group_details": [ + { + "level_learned_at": 23, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "x-y", + "url": "https://pokeapi.co/api/v2/version-group/15/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "omega-ruby-alpha-sapphire", + "url": "https://pokeapi.co/api/v2/version-group/16/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sun-moon", + "url": "https://pokeapi.co/api/v2/version-group/17/" + } + }, + { + "level_learned_at": 29, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + }, + { + "level_learned_at": 1, + "move_learn_method": { + "name": "level-up", + "url": "https://pokeapi.co/api/v2/move-learn-method/1/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + }, + { + "move": { + "name": "laser-focus", + "url": "https://pokeapi.co/api/v2/move/673/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "ultra-sun-ultra-moon", + "url": "https://pokeapi.co/api/v2/version-group/18/" + } + } + ] + }, + { + "move": { + "name": "rising-voltage", + "url": "https://pokeapi.co/api/v2/move/804/" + }, + "version_group_details": [ + { + "level_learned_at": 0, + "move_learn_method": { + "name": "tutor", + "url": "https://pokeapi.co/api/v2/move-learn-method/3/" + }, + "version_group": { + "name": "sword-shield", + "url": "https://pokeapi.co/api/v2/version-group/20/" + } + } + ] + } + ], + "name": "pikachu", + "order": 35, + "past_types": [], + "species": { + "name": "pikachu", + "url": "https://pokeapi.co/api/v2/pokemon-species/25/" + }, + "sprites": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/25.png", + "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/female/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/25.png", + "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/female/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/female/25.png", + "other": { + "dream_world": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/dream-world/25.svg", + "front_female": null + }, + "home": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/female/25.png" + }, + "official-artwork": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/official-artwork/25.png" + } + }, + "versions": { + "generation-i": { + "red-blue": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/25.png", + "back_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/gray/25.png", + "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/back/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/25.png", + "front_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/gray/25.png", + "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/25.png" + }, + "yellow": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/25.png", + "back_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/gray/25.png", + "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/back/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/25.png", + "front_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/gray/25.png", + "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/25.png" + } + }, + "generation-ii": { + "crystal": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/shiny/25.png", + "back_shiny_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/shiny/25.png", + "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/shiny/25.png", + "front_shiny_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/shiny/25.png", + "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/25.png" + }, + "gold": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/shiny/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/shiny/25.png", + "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/transparent/25.png" + }, + "silver": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/shiny/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/shiny/25.png", + "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/transparent/25.png" + } + }, + "generation-iii": { + "emerald": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/shiny/25.png" + }, + "firered-leafgreen": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/shiny/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/shiny/25.png" + }, + "ruby-sapphire": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/shiny/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/shiny/25.png" + } + }, + "generation-iv": { + "diamond-pearl": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/25.png", + "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/female/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/25.png", + "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/female/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/female/25.png" + }, + "heartgold-soulsilver": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/25.png", + "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/female/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/25.png", + "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/female/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/female/25.png" + }, + "platinum": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/25.png", + "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/female/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/25.png", + "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/female/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/female/25.png" + } + }, + "generation-v": { + "black-white": { + "animated": { + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/25.gif", + "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/female/25.gif", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/25.gif", + "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/female/25.gif", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/25.gif", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/female/25.gif", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/25.gif", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/female/25.gif" + }, + "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/25.png", + "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/female/25.png", + "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/25.png", + "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/female/25.png", + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/female/25.png" + } + }, + "generation-vi": { + "omegaruby-alphasapphire": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/female/25.png" + }, + "x-y": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/female/25.png" + } + }, + "generation-vii": { + "icons": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/icons/25.png", + "front_female": null + }, + "ultra-sun-ultra-moon": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/female/25.png", + "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/25.png", + "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/female/25.png" + } + }, + "generation-viii": { + "icons": { + "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/25.png", + "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/female/25.png" + } + } + } + }, + "stats": [ + { + "base_stat": 35, + "effort": 0, + "stat": { "name": "hp", "url": "https://pokeapi.co/api/v2/stat/1/" } + }, + { + "base_stat": 55, + "effort": 0, + "stat": { "name": "attack", "url": "https://pokeapi.co/api/v2/stat/2/" } + }, + { + "base_stat": 40, + "effort": 0, + "stat": { "name": "defense", "url": "https://pokeapi.co/api/v2/stat/3/" } + }, + { + "base_stat": 50, + "effort": 0, + "stat": { + "name": "special-attack", + "url": "https://pokeapi.co/api/v2/stat/4/" + } + }, + { + "base_stat": 50, + "effort": 0, + "stat": { + "name": "special-defense", + "url": "https://pokeapi.co/api/v2/stat/5/" + } + }, + { + "base_stat": 90, + "effort": 2, + "stat": { "name": "speed", "url": "https://pokeapi.co/api/v2/stat/6/" } + } + ], + "types": [ + { + "slot": 1, + "type": { + "name": "electric", + "url": "https://pokeapi.co/api/v2/type/13/" + } + } + ], + "weight": 60 +} diff --git a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml index d3e73db4804c..c23934d934d0 100644 --- a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml +++ b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-xata/.dockerignore b/airbyte-integrations/connectors/destination-xata/.dockerignore new file mode 100644 index 000000000000..40370594ddc6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_xata +!setup.py diff --git a/airbyte-integrations/connectors/destination-xata/Dockerfile b/airbyte-integrations/connectors/destination-xata/Dockerfile new file mode 100644 index 000000000000..a2ac681f7b79 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY destination_xata ./destination_xata + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.name=airbyte/destination-xata diff --git a/airbyte-integrations/connectors/destination-xata/README.md b/airbyte-integrations/connectors/destination-xata/README.md new file mode 100644 index 000000000000..59d4bbd09b13 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/README.md @@ -0,0 +1,123 @@ +# Xata Destination + +This is the repository for the Xata destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/xata). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-xata:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/xata) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_xata/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination xata test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-xata:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-xata:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-xata:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-xata:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-xata:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-xata:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-xata:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-xata/bootstrap.md b/airbyte-integrations/connectors/destination-xata/bootstrap.md new file mode 100644 index 000000000000..bac35e3ae53c --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/bootstrap.md @@ -0,0 +1 @@ +# Xata Destination Connector diff --git a/airbyte-integrations/connectors/destination-xata/build.gradle b/airbyte-integrations/connectors/destination-xata/build.gradle new file mode 100644 index 000000000000..f52818d7ca2f --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'destination_xata' +} diff --git a/airbyte-integrations/connectors/destination-xata/destination_xata/__init__.py b/airbyte-integrations/connectors/destination-xata/destination_xata/__init__.py new file mode 100644 index 000000000000..d03079997c13 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/destination_xata/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationXata + +__all__ = ["DestinationXata"] diff --git a/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py b/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py new file mode 100644 index 000000000000..a9698c49c446 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/destination_xata/destination.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type +from xata.client import XataClient +from xata.helpers import BulkProcessor + +__version__ = "0.0.1" + +logger = logging.getLogger("airbyte") + + +class DestinationXata(Destination): + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + """ + Reads the input stream of messages, config, and catalog to write data to the destination. + + This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received + in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been + successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, + then the source is given the last state message output from this method as the starting point of the next sync. + + :param config: dict of JSON configuration matching the configuration declared in spec.json + :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the + destination + :param input_messages: The stream of input messages received from the source + :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs + """ + + xata = XataClient(api_key=config["api_key"], db_url=config["db_url"]) + xata.set_header("user-agent", f"airbyte/destination-xata:{__version__}") + + bp = BulkProcessor(xata) + count = 0 + for message in input_messages: + if message.type == Type.RECORD: + # Put record to processing queue + bp.put_record(message.record.stream, message.record.data) + count += 1 + if message.type == Type.STATE: + yield message + bp.flush_queue() + logger.info(bp.get_stats()) + if count != bp.get_stats()["total"] or bp.get_stats()["failed_batches"] != 0: + raise Exception( + "inconsistency found, expected %d records pushed, actual: %d with %d failures." + % (count, bp.get_stats()["total"], bp.get_stats()["failed_batches"]) + ) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the destination with the needed permissions + e.g: if a provided API token or password can be used to connect and write to the destination. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this destination, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + try: + xata = XataClient(api_key=config["api_key"], db_url=config["db_url"]) + xata.set_header("user-agent", f"airbyte/destination-xata:{__version__}") + + r = xata.users().getUser() + if r.status_code != 200: + raise Exception("Invalid connection parameters.") + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") diff --git a/airbyte-integrations/connectors/destination-xata/destination_xata/spec.json b/airbyte-integrations/connectors/destination-xata/destination_xata/spec.json new file mode 100644 index 000000000000..6e73b6cec519 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/destination_xata/spec.json @@ -0,0 +1,28 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/xata", + "supported_destination_sync_modes": ["append"], + "supportsIncremental": false, + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Xata", + "type": "object", + "required": ["api_key", "db_url"], + "additionalProperties": true, + "properties": { + "api_key": { + "title": "API Key", + "description": "API Key to connect.", + "type": "string", + "order": 0, + "airbyte_secret": true + }, + "db_url": { + "title": "Database URL", + "description": "URL pointing to your workspace.", + "type": "string", + "order": 1, + "example": "https://my-workspace-abc123.us-east-1.xata.sh/db/nyc-taxi-fares:main" + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-xata/icon.svg b/airbyte-integrations/connectors/destination-xata/icon.svg new file mode 100644 index 000000000000..8950b358dc8f --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py new file mode 100644 index 000000000000..8aa903b715ba --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py @@ -0,0 +1,109 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any, Mapping +from unittest.mock import Mock + +import pytest +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from destination_xata import DestinationXata +from xata.client import XataClient + + +@pytest.fixture(name="config") +def config_fixture() -> Mapping[str, Any]: + with open("secrets/config.json", "r") as f: + return json.loads(f.read()) + + +@pytest.fixture(name="configured_catalog") +def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"string_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + append_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="append_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + ) + # TODO implement overwrite + """ + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="overwrite_stream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + """ + return ConfiguredAirbyteCatalog(streams=[append_stream]) + + +def test_check_valid_config(config: Mapping): + outcome = DestinationXata().check(logger=Mock(), config=config) + assert outcome.status == Status.SUCCEEDED + + +def test_check_invalid_config(): + f = open("integration_tests/invalid_config.json") + config = json.load(f) + outcome = DestinationXata().check(logger=Mock(), config=config) + assert outcome.status == Status.FAILED + + +def test_write(config: Mapping): + test_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + test_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name="test_stream", json_schema=test_schema, supported_sync_modes=[SyncMode.incremental]), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + ) + + records = [AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={ + "str_col": "example", + "int_col": 1, + }, emitted_at=0) + )] + + # setup Xata workspace + xata = XataClient(api_key=config["api_key"], db_url=config["db_url"]) + db_name = xata.get_config()["dbName"] + # database exists ? + assert xata.databases().getDatabaseMetadata(db_name).status_code == 200, f"database '{db_name}' does not exist." + assert xata.table().createTable("test_stream").status_code == 201, "could not create table, if it already exists, please delete it." + assert xata.table().setTableSchema("test_stream", { + "columns": [ + {"name": "str_col", "type": "string"}, + {"name": "int_col", "type": "int"}, + ] + }).status_code == 200, "failed to set table schema" + + dest = DestinationXata() + list(dest.write( + config=config, + configured_catalog=test_stream, + input_messages=records + )) + + # fetch record + records = xata.data().queryTable("test_stream", {}) + assert records.status_code == 200 + assert len(records.json()["records"]) == 1 + + proof = records.json()["records"][0] + assert proof["str_col"] == "example" + assert proof["int_col"] == 1 + + # cleanup + assert xata.table().deleteTable("test_stream").status_code == 200 diff --git a/airbyte-integrations/connectors/destination-xata/integration_tests/invalid_config.json b/airbyte-integrations/connectors/destination-xata/integration_tests/invalid_config.json new file mode 100644 index 000000000000..36bd35acc0b5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "husenvasen", + "database_url": "https://invalid" +} diff --git a/airbyte-integrations/connectors/destination-xata/main.py b/airbyte-integrations/connectors/destination-xata/main.py new file mode 100644 index 000000000000..76e7d8f087c0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_xata import DestinationXata + +if __name__ == "__main__": + DestinationXata().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-xata/metadata.yaml b/airbyte-integrations/connectors/destination-xata/metadata.yaml new file mode 100644 index 000000000000..967f1ec2a534 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/metadata.yaml @@ -0,0 +1,24 @@ +data: + registries: + cloud: + enabled: true + oss: + enabled: true + connectorSubtype: database + connectorType: destination + definitionId: 2a51c92d-0fb4-4e54-94d2-cce631f24d1f + dockerImageTag: 0.1.1 + dockerRepository: airbyte/destination-xata + githubIssueLabel: destination-xata + icon: xata.svg + license: MIT + name: Xata + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/xata + tags: + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-xata/requirements.txt b/airbyte-integrations/connectors/destination-xata/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-xata/sample_files/configured_catalog.json b/airbyte-integrations/connectors/destination-xata/sample_files/configured_catalog.json new file mode 100644 index 000000000000..f526611d3df1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/sample_files/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/destination-xata/setup.py b/airbyte-integrations/connectors/destination-xata/setup.py new file mode 100644 index 000000000000..5fcb33e94fbb --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk", "xata==0.10.1"] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_xata", + description="Destination implementation for Xata.io", + author="Philip Krauss ", + author_email="support@xata.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py new file mode 100644 index 000000000000..340fbf62c948 --- /dev/null +++ b/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest + +from xata.client import XataClient +from xata.helpers import BulkProcessor + + +class DestinationConnectorXataTestCase(unittest.TestCase): + + def test_request(self): + xata = XataClient(db_url="https://unit_tests-mock.results-store.xata.sh/db/mock-db", api_key="mock-key") + bp = BulkProcessor(xata, thread_pool_size=1, batch_size=2, flush_interval=1) + stats = bp.get_stats() + + assert "total" in stats + assert "queue" in stats + assert "failed_batches" in stats + assert "tables" in stats + + assert stats["total"] == 0 + assert stats["queue"] == 0 + assert stats["failed_batches"] == 0 + + +if __name__ == '__main__': + unittest.main() diff --git a/airbyte-integrations/connectors/destination-yugabytedb/metadata.yaml b/airbyte-integrations/connectors/destination-yugabytedb/metadata.yaml index 316588e36331..29f00360f6d1 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-yugabytedb/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/yugabytedb tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java index 2c8b0f701236..26b6a92d37f8 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java @@ -49,7 +49,7 @@ protected String getImageName() { } @Override - protected void setup(TestDestinationEnv testEnv) throws Exception { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { jsonConfig = Jsons.jsonNode(ImmutableMap.builder() .put("host", yugabytedbContainer.getHost()) .put("port", yugabytedbContainer.getMappedPort(5433)) @@ -69,13 +69,13 @@ protected void setup(TestDestinationEnv testEnv) throws Exception { } @Override - protected void tearDown(TestDestinationEnv testEnv) throws Exception { + protected void tearDown(final TestDestinationEnv testEnv) throws Exception { database.execute(connection -> { - var statement = connection.createStatement(); + final var statement = connection.createStatement(); cleanupTables.forEach(tb -> { try { statement.execute("DROP TABLE " + tb + ";"); - } catch (SQLException e) { + } catch (final SQLException e) { throw new RuntimeException(e); } }); @@ -126,14 +126,14 @@ protected boolean supportObjectDataTypeTest() { } @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) + protected List retrieveRecords(final TestDestinationEnv testEnv, + final String streamName, + final String namespace, + final JsonNode streamSchema) throws SQLException { - String tableName = namingResolver.getRawTableName(streamName); - String schemaName = namingResolver.getNamespace(namespace); + final String tableName = namingResolver.getRawTableName(streamName); + final String schemaName = namingResolver.getNamespace(namespace); cleanupTables.add(schemaName + "." + tableName); return retrieveRecordsFromTable(tableName, schemaName); } @@ -143,7 +143,7 @@ private List retrieveRecordsFromTable(final String tableName, final St return database.bufferedResultSetQuery( connection -> { - var statement = connection.createStatement(); + final var statement = connection.createStatement(); return statement.executeQuery( String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)); diff --git a/airbyte-integrations/connectors/source-activecampaign/metadata.yaml b/airbyte-integrations/connectors/source-activecampaign/metadata.yaml index d4b2aecfcc8e..7b302f920aa0 100644 --- a/airbyte-integrations/connectors/source-activecampaign/metadata.yaml +++ b/airbyte-integrations/connectors/source-activecampaign/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 9f32dab3-77cb-45a1-9d33-347aa5fbe363 dockerImageTag: 0.1.0 dockerRepository: airbyte/source-activecampaign + documentationUrl: https://docs.airbyte.com/integrations/sources/activecampaign githubIssueLabel: source-activecampaign icon: activecampaign.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/activecampaign + supportLevel: community tags: - language:low-code - language:python diff --git a/airbyte-integrations/connectors/source-activecampaign/requirements.txt b/airbyte-integrations/connectors/source-activecampaign/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-activecampaign/requirements.txt +++ b/airbyte-integrations/connectors/source-activecampaign/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-activecampaign/setup.py b/airbyte-integrations/connectors/source-activecampaign/setup.py index d858dd2d9fc8..d539a3c2757c 100644 --- a/airbyte-integrations/connectors/source-activecampaign/setup.py +++ b/airbyte-integrations/connectors/source-activecampaign/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-adjust/metadata.yaml b/airbyte-integrations/connectors/source-adjust/metadata.yaml index 3770867fc1cc..f3a7d7bb290a 100644 --- a/airbyte-integrations/connectors/source-adjust/metadata.yaml +++ b/airbyte-integrations/connectors/source-adjust/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: d3b7fa46-111b-419a-998a-d7f046f6d66d dockerImageTag: 0.1.0 dockerRepository: airbyte/source-adjust + documentationUrl: https://docs.airbyte.com/integrations/sources/adjust githubIssueLabel: source-adjust icon: adjust.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/adjust + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-adjust/requirements.txt b/airbyte-integrations/connectors/source-adjust/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-adjust/requirements.txt +++ b/airbyte-integrations/connectors/source-adjust/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-adjust/setup.py b/airbyte-integrations/connectors/source-adjust/setup.py index 0ea2be7cdf3e..99691631d22c 100644 --- a/airbyte-integrations/connectors/source-adjust/setup.py +++ b/airbyte-integrations/connectors/source-adjust/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-aha/Dockerfile b/airbyte-integrations/connectors/source-aha/Dockerfile index 8620e0d11000..674019d2f2d5 100644 --- a/airbyte-integrations/connectors/source-aha/Dockerfile +++ b/airbyte-integrations/connectors/source-aha/Dockerfile @@ -34,5 +34,5 @@ COPY source_aha ./source_aha ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-aha diff --git a/airbyte-integrations/connectors/source-aha/metadata.yaml b/airbyte-integrations/connectors/source-aha/metadata.yaml index fcc246842a9a..cbbecab8c080 100644 --- a/airbyte-integrations/connectors/source-aha/metadata.yaml +++ b/airbyte-integrations/connectors/source-aha/metadata.yaml @@ -1,20 +1,24 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 81ca39dc-4534-4dd2-b848-b0cfd2c11fce - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-aha + documentationUrl: https://docs.airbyte.com/integrations/sources/aha githubIssueLabel: source-aha icon: aha.svg license: MIT name: Aha registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/aha + supportLevel: community tags: - language:low-code - language:python diff --git a/airbyte-integrations/connectors/source-aha/requirements.txt b/airbyte-integrations/connectors/source-aha/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-aha/requirements.txt +++ b/airbyte-integrations/connectors/source-aha/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-aha/setup.py b/airbyte-integrations/connectors/source-aha/setup.py index d0b3d5a49a8c..30789cbe1a0a 100644 --- a/airbyte-integrations/connectors/source-aha/setup.py +++ b/airbyte-integrations/connectors/source-aha/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_comments.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_comments.json index e28a9c8935d0..1b1687af0c00 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_comments.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_comments.json @@ -21,8 +21,8 @@ "type": ["null", "string"] }, "idea_commenter_user": { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "id": { "type": ["null", "string"] }, @@ -41,8 +41,8 @@ } }, "idea": { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "id": { "type": ["null", "string"] }, @@ -59,8 +59,8 @@ "type": ["null", "string"] }, "workflow_status": { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "id": { "type": ["null", "string"] }, @@ -79,8 +79,8 @@ } }, "description": { - "type" : "object", - "properties" : { + "type": "object", + "properties": { "id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_endorsements.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_endorsements.json index bb1b210f1866..5e8b9cc61252 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_endorsements.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/idea_endorsements.json @@ -1,96 +1,96 @@ { - "$schema" : "http://json-schema.org/draft-07/schema#", - "type" : "object", - "properties" : { - "id" : { - "type" : ["null", "string"] + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] }, - "idea_id" : { - "type" : ["null", "string"] + "idea_id": { + "type": ["null", "string"] }, - "created_at" : { - "type" : ["null", "string"] + "created_at": { + "type": ["null", "string"] }, - "updated_at" : { - "type" : ["null", "string"] + "updated_at": { + "type": ["null", "string"] }, - "value" : { - "type" : ["null", "string"] + "value": { + "type": ["null", "string"] }, - "link" : { - "type" : ["null", "string"] + "link": { + "type": ["null", "string"] }, - "weight" : { - "type" : ["null", "integer"] + "weight": { + "type": ["null", "integer"] }, - "endorsed_by_portal_user" : { - "type" : "object", - "properties" : { - "id" : { - "type" : ["null", "string"] + "endorsed_by_portal_user": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] }, - "name" : { - "type" : ["null", "string"] + "name": { + "type": ["null", "string"] }, - "email" : { - "type" : ["null", "string"] + "email": { + "type": ["null", "string"] }, - "created_at" : { - "type" : ["null", "string"] + "created_at": { + "type": ["null", "string"] } } }, - "endorsed_by_idea_user" : { - "type" : "object", - "properties" : { - "id" : { - "type" : ["null", "string"] + "endorsed_by_idea_user": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] }, - "name" : { - "type" : ["null", "string"] + "name": { + "type": ["null", "string"] }, - "email" : { - "type" : ["null", "string"] + "email": { + "type": ["null", "string"] }, - "created_at" : { - "type" : ["null", "string"] + "created_at": { + "type": ["null", "string"] } } }, - "endorsed_by_idea_organization" : { - "type" : "object", - "properties" : { - "id" : { - "type" : ["null", "string"] + "endorsed_by_idea_organization": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] }, - "name" : { - "type" : ["null", "string"] + "name": { + "type": ["null", "string"] }, - "created_at" : { - "type" : ["null", "string"] + "created_at": { + "type": ["null", "string"] }, - "url" : { - "type" : ["null", "string"] + "url": { + "type": ["null", "string"] }, - "resource" : { - "type" : ["null", "string"] + "resource": { + "type": ["null", "string"] } } }, "endorsed_by_user": { "type": "object", "properties": { - "id" : { - "type" : ["null", "string"] + "id": { + "type": ["null", "string"] }, - "name" : { - "type" : ["null", "string"] + "name": { + "type": ["null", "string"] }, - "email" : { - "type" : ["null", "string"] + "email": { + "type": ["null", "string"] }, - "created_at" : { - "type" : ["null", "string"] + "created_at": { + "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml index fc55fc9176f7..ad5568a53a59 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml @@ -12,6 +12,7 @@ connectionSpecification: type: string description: API Key title: API Bearer Token + airbyte_secret: true url: type: string description: URL diff --git a/airbyte-integrations/connectors/source-aircall/Dockerfile b/airbyte-integrations/connectors/source-aircall/Dockerfile index 3a10c2a5eaa0..95704317f229 100644 --- a/airbyte-integrations/connectors/source-aircall/Dockerfile +++ b/airbyte-integrations/connectors/source-aircall/Dockerfile @@ -34,5 +34,5 @@ COPY source_aircall ./source_aircall ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-aircall diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json index a568c6be9023..a464dd4be03c 100644 --- a/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/abnormal_state.json @@ -13,4 +13,4 @@ "stream_descriptor": { "name": "numbers" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json index 6af1ac52f359..dc8a81fc8adb 100644 --- a/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/configured_catalog.json @@ -56,7 +56,7 @@ }, { "stream": { - "name": "user_availablity", + "name": "user_availability", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -82,4 +82,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json index 48e215dc8a08..7fe815337900 100644 --- a/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-aircall/integration_tests/sample_state.json @@ -6,4 +6,4 @@ "stream_descriptor": { "name": "teams" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-aircall/metadata.yaml b/airbyte-integrations/connectors/source-aircall/metadata.yaml index ce86667858bc..1883e76b2e69 100644 --- a/airbyte-integrations/connectors/source-aircall/metadata.yaml +++ b/airbyte-integrations/connectors/source-aircall/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 912eb6b7-a893-4a5b-b1c0-36ebbe2de8cd - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-aircall githubIssueLabel: source-aircall icon: aircall.svg @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-aircall/requirements.txt b/airbyte-integrations/connectors/source-aircall/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-aircall/requirements.txt +++ b/airbyte-integrations/connectors/source-aircall/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-aircall/setup.py b/airbyte-integrations/connectors/source-aircall/setup.py index 658ef0d53c58..25b830a1e3cc 100644 --- a/airbyte-integrations/connectors/source-aircall/setup.py +++ b/airbyte-integrations/connectors/source-aircall/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-aircall/source_aircall/manifest.yaml b/airbyte-integrations/connectors/source-aircall/source_aircall/manifest.yaml index 8f3ded1c24ec..bf620875b582 100644 --- a/airbyte-integrations/connectors/source-aircall/source_aircall/manifest.yaml +++ b/airbyte-integrations/connectors/source-aircall/source_aircall/manifest.yaml @@ -115,11 +115,11 @@ definitions: $parameters: path: "/users" - user_availablity_stream: + user_availability_stream: type: DeclarativeStream retriever: $ref: "#/definitions/user_retriever" - name: "user_availablity" + name: "user_availability" primary_key: "id" $parameters: path: "/users/availabilities" @@ -165,7 +165,7 @@ streams: - "#/definitions/contacts_stream" - "#/definitions/numbers_stream" - "#/definitions/tags_stream" - - "#/definitions/user_availablity_stream" + - "#/definitions/user_availability_stream" - "#/definitions/users_stream" - "#/definitions/teams_stream" - "#/definitions/webhooks_stream" @@ -178,7 +178,7 @@ check: - "contacts" - "numbers" - "tags" - - "user_availablity" + - "user_availability" - "users" - "teams" - "webhooks" diff --git a/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml index 03e2909a6b72..d19419e1ff17 100644 --- a/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml @@ -17,7 +17,7 @@ acceptance_tests: - config_path: "integration_tests/invalid_config_oauth.json" status: "failed" - config_path: "integration_tests/invalid_config_oauth_missing_fields.json" - status: "exception" + status: "failed" discovery: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json index fb8a47c718c3..e441f0d0de77 100644 --- a/airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-airtable/integration_tests/configured_catalog.json @@ -13,7 +13,7 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, + }, { "stream": { "name": "users/table_2/tblCjIgm5yveWC4X5", diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config_oauth.json index 2e0cc51ef89c..ce53b3b2f25b 100644 --- a/airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config_oauth.json +++ b/airbyte-integrations/connectors/source-airtable/integration_tests/invalid_config_oauth.json @@ -1,10 +1,10 @@ { - "credentials": { - "auth_method": "oauth2.0", - "client_id": "client_id", - "client_secret": "client_secret", - "access_token": "access_token", - "refresh_token": "refresh_token", - "token_expiry_date": "2023-01-01T12:12:12.000000+00:00" - } + "credentials": { + "auth_method": "oauth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "access_token", + "refresh_token": "refresh_token", + "token_expiry_date": "2023-01-01T12:12:12.000000+00:00" + } } diff --git a/airbyte-integrations/connectors/source-airtable/metadata.yaml b/airbyte-integrations/connectors/source-airtable/metadata.yaml index 0decbd38bc80..ea6c51182a8c 100644 --- a/airbyte-integrations/connectors/source-airtable/metadata.yaml +++ b/airbyte-integrations/connectors/source-airtable/metadata.yaml @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/airtable tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-airtable/requirements.txt b/airbyte-integrations/connectors/source-airtable/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-airtable/requirements.txt +++ b/airbyte-integrations/connectors/source-airtable/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-airtable/setup.py b/airbyte-integrations/connectors/source-airtable/setup.py index 9a995a70447f..2c294e0b0dfb 100644 --- a/airbyte-integrations/connectors/source-airtable/setup.py +++ b/airbyte-integrations/connectors/source-airtable/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile index c671cf59cc10..7af590ca55e7 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-alloydb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.28 +LABEL io.airbyte.version=3.1.5 LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml index 80d7290fbb48..d3898532abe2 100644 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: database connectorType: source definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - dockerImageTag: 2.0.28 + dockerImageTag: 3.1.5 dockerRepository: airbyte/source-alloydb-strict-encrypt githubIssueLabel: source-alloydb icon: alloydb.svg diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile index c201e1965bfd..cf8e202ab39d 100644 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-alloydb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.28 +LABEL io.airbyte.version=3.1.5 LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-alloydb/metadata.yaml b/airbyte-integrations/connectors/source-alloydb/metadata.yaml index e0165214bdc1..aed3d7766c62 100644 --- a/airbyte-integrations/connectors/source-alloydb/metadata.yaml +++ b/airbyte-integrations/connectors/source-alloydb/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: database connectorType: source definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - dockerImageTag: 2.0.28 + dockerImageTag: 3.1.5 dockerRepository: airbyte/source-alloydb githubIssueLabel: source-alloydb icon: alloydb.svg @@ -22,4 +22,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb tags: - language:java + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json index 400e6dac3d9c..6c81f1b6ac87 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json @@ -26,7 +26,7 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, + }, { "stream": { "name": "quote", diff --git a/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml b/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml index 274c07509ea0..d4f4a1a63e0f 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml +++ b/airbyte-integrations/connectors/source-alpha-vantage/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt b/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt +++ b/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-alpha-vantage/setup.py b/airbyte-integrations/connectors/source-alpha-vantage/setup.py index 3dbca3f3111c..bf4ebe78c025 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/setup.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/quote.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/quote.json index 036b48903dc2..8f38ae34060b 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/quote.json +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/quote.json @@ -1,38 +1,38 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "Global Quote": { - "01. symbol": { - "type": ["string", "null"] - }, - "02. open": { - "type": ["string", "null"] - }, - "03. high": { - "type": ["string", "null"] - }, - "04. low": { - "type": ["string", "null"] - }, - "05. price": { - "type": ["string", "null"] - }, - "06. volume": { - "type": ["string", "null"] - }, - "07. latest trading day": { - "type": ["string", "null"] - }, - "08. previous close": { - "type": ["string", "null"] - }, - "09. change": { - "type": ["string", "null"] - }, - "10. change percent": { - "type": ["string", "null"] - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Global Quote": { + "01. symbol": { + "type": ["string", "null"] + }, + "02. open": { + "type": ["string", "null"] + }, + "03. high": { + "type": ["string", "null"] + }, + "04. low": { + "type": ["string", "null"] + }, + "05. price": { + "type": ["string", "null"] + }, + "06. volume": { + "type": ["string", "null"] + }, + "07. latest trading day": { + "type": ["string", "null"] + }, + "08. previous close": { + "type": ["string", "null"] + }, + "09. change": { + "type": ["string", "null"] + }, + "10. change percent": { + "type": ["string", "null"] } } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 58deb6dda8c7..3ffd3401acb8 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -13,6 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=3.1.0 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index 8635a6866cec..d0be492d3369 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -21,10 +21,22 @@ acceptance_tests: bypass_reason: "can't populate stream because it requires real ad campaign" - name: sponsored_brands_report_stream bypass_reason: "can't populate stream because it requires real ad campaign" + - name: sponsored_brands_v3_report_stream + bypass_reason: "can't populate stream because it requires real ad campaign" - name: sponsored_brands_video_report_stream bypass_reason: "can't populate stream because it requires real ad campaign" - name: sponsored_products_report_stream bypass_reason: "can't populate stream because it requires real ad campaign" + ignored_fields: + sponsored_product_campaigns: + - name: dailyBudget + bypass_reason: "can be updated, also it is sometimes integer, sometimes float" + sponsored_product_ad_group_suggested_keywords: + - name: suggestedKeywords + bypass_reason: "value can be changed because it is real-life recommendation from Amazon" + sponsored_product_ad_group_bid_recommendations: + - name: suggestedBid + bypass_reason: "value can be changed because it is real-life recommendation from Amazon" timeout_seconds: 2400 expect_records: path: integration_tests/expected_records.jsonl @@ -37,6 +49,8 @@ acceptance_tests: discovery: tests: - config_path: secrets/config.json + backward_compatibility_tests_config: + disable_for_version: 2.3.1 full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json index 8394bbf8feff..3be1aab60f67 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog.json @@ -10,6 +10,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "portfolios", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["portfolioId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "sponsored_display_campaigns", @@ -130,6 +140,26 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "sponsored_product_ad_group_suggested_keywords", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["adGroupId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_product_ad_group_bid_recommendations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["adGroupId"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "sponsored_brands_keywords", diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json index ea2923cc85f7..938356b1a163 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/configured_catalog_report.json @@ -12,6 +12,23 @@ ["recordId"] ] }, + "cursor_field": ["reportDate"], + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "sponsored_brands_v3_report_stream", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["profileId"], + ["recordType"], + ["reportDate"], + ["recordId"] + ] + }, + "cursor_field": ["reportDate"], "sync_mode": "incremental", "destination_sync_mode": "overwrite" } diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl index 1159acc2fd5a..2c0ce04ab232 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl @@ -48,10 +48,10 @@ {"stream":"sponsored_display_product_ads","data":{"adId":195948665185008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBOA","state":"enabled"},"emitted_at":1659020219614} {"stream":"sponsored_display_product_ads","data":{"adId":130802512011075,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G1HT4P","state":"enabled"},"emitted_at":1659020219614} {"stream":"sponsored_display_targetings","data":{"adGroupId":239470166910761,"bid":0.4,"expression":[{"type":"similarProduct"}],"expressionType":"auto","resolvedExpression":[{"type":"similarProduct"}],"state":"enabled","targetId":124150067548052},"emitted_at":1659020220625} -{"stream": "sponsored_product_campaigns", "data": {"campaignId": 39413387973397, "name": "Test campaging for profileId 1861552880916640", "campaignType": "sponsoredProducts", "targetingType": "manual", "premiumBidAdjustment": true, "dailyBudget": 10.0, "ruleBasedBudget": {"isProcessing": false}, "startDate": "20220705", "endDate": "20220712", "state": "paused", "bidding": {"strategy": "legacyForSales", "adjustments": [{"predicate": "placementTop", "percentage": 50}]}, "tags": {"PONumber": "examplePONumber", "accountManager": "exampleAccountManager"}}, "emitted_at": 1677630109443} -{"stream": "sponsored_product_campaigns", "data": {"campaignId": 135264288913079, "name": "Campaign - 7/5/2022 18:14:02", "campaignType": "sponsoredProducts", "targetingType": "auto", "premiumBidAdjustment": false, "dailyBudget": 10.0, "startDate": "20220705", "state": "enabled", "bidding": {"strategy": "legacyForSales", "adjustments": []}}, "emitted_at": 1677630109611} -{"stream": "sponsored_product_campaigns", "data": {"campaignId": 191249325250025, "name": "Campaign - 7/8/2022 13:57:48", "campaignType": "sponsoredProducts", "targetingType": "auto", "premiumBidAdjustment": true, "dailyBudget": 50.0, "startDate": "20220708", "state": "enabled", "bidding": {"strategy": "legacyForSales", "adjustments": [{"predicate": "placementProductPage", "percentage": 100}, {"predicate": "placementTop", "percentage": 100}]}}, "emitted_at": 1677630109612} -{"stream": "sponsored_product_campaigns", "data": {"campaignId": 146003174711486, "name": "Test campaging for profileId 3039403378822505", "campaignType": "sponsoredProducts", "targetingType": "manual", "premiumBidAdjustment": true, "dailyBudget": 2.0, "startDate": "20220705", "endDate": "20231111", "state": "enabled", "bidding": {"strategy": "legacyForSales", "adjustments": [{"predicate": "placementTop", "percentage": 50}]}, "tags": {"PONumber": "examplePONumber", "accountManager": "exampleAccountManager"}}, "emitted_at": 1677630109776} +{"stream":"sponsored_product_campaigns","data":{"campaignId":39413387973397,"name":"Test campaging for profileId 1861552880916640","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":10,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220705","endDate":"20220712","state":"paused","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1687524797996} +{"stream":"sponsored_product_campaigns","data":{"campaignId":135264288913079,"name":"Campaign - 7/5/2022 18:14:02","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":false,"dailyBudget":10,"startDate":"20220705","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[]},"portfolioId":270076898441727},"emitted_at":1687524798170} +{"stream":"sponsored_product_campaigns","data":{"campaignId":191249325250025,"name":"Campaign - 7/8/2022 13:57:48","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":true,"dailyBudget":50,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220708","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementProductPage","percentage":100},{"predicate":"placementTop","percentage":100}]},"portfolioId":253945852845204},"emitted_at":1687524798171} +{"stream":"sponsored_product_campaigns","data":{"campaignId":146003174711486,"name":"Test campaging for profileId 3039403378822505","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":2,"startDate":"20220705","endDate":"20231111","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1687524798327} {"stream":"sponsored_product_ad_groups","data":{"adGroupId":226404883721634,"name":"My AdGroup for Campaign 39413387973397","campaignId":39413387973397,"defaultBid":10,"state":"enabled"},"emitted_at":1659020222108} {"stream":"sponsored_product_ad_groups","data":{"adGroupId":183961953969922,"name":"Ad group - 7/5/2022 18:14:02","campaignId":135264288913079,"defaultBid":0.75,"state":"enabled"},"emitted_at":1659020222276} {"stream":"sponsored_product_ad_groups","data":{"adGroupId":108551155050351,"name":"Ad group - 7/8/2022 13:57:48","campaignId":191249325250025,"defaultBid":1,"state":"enabled"},"emitted_at":1659020222276} @@ -120,3 +120,14 @@ {"stream":"sponsored_product_targetings","data":{"targetId":232221427954900,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"queryBroadRelMatches"}],"resolvedExpression":[{"type":"queryBroadRelMatches"}]},"emitted_at":1659020226436} {"stream":"sponsored_product_targetings","data":{"targetId":12739477778779,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"asinAccessoryRelated"}],"resolvedExpression":[{"type":"asinAccessoryRelated"}]},"emitted_at":1659020226436} {"stream":"sponsored_product_targetings","data":{"targetId":1189452552122,"adGroupId":108551155050351,"campaignId":191249325250025,"expressionType":"auto","state":"enabled","expression":[{"type":"asinSubstituteRelated"}],"resolvedExpression":[{"type":"asinSubstituteRelated"}]},"emitted_at":1659020226437} +{"stream":"sponsored_product_campaign_negative_keywords","data":{"keywordId":367623300145491,"campaignId":39413387973397,"keywordText":"campaign negative keyword","matchType":"negativeExact","state":"enabled"},"emitted_at":1687197089168} +{"stream":"sponsored_product_campaign_negative_keywords","data":{"keywordId":313227767817048,"campaignId":39413387973397,"keywordText":"campaign negative keyword2","matchType":"negativeExact","state":"enabled"},"emitted_at":1687197089170} +{"stream":"sponsored_product_campaign_negative_keywords","data":{"keywordId":362264772936192,"campaignId":191249325250025,"keywordText":"negative","matchType":"negativeExact","state":"enabled"},"emitted_at":1687197089322} +{"stream":"sponsored_product_campaign_negative_keywords","data":{"keywordId":318852596875190,"campaignId":191249325250025,"keywordText":"negative phrase","matchType":"negativeExact","state":"enabled"},"emitted_at":1687197089323} +{"stream":"sponsored_product_campaign_negative_keywords","data":{"keywordId":435410875929367,"campaignId":191249325250025,"keywordText":"another negative phrase","matchType":"negativeExact","state":"enabled"},"emitted_at":1687197089324} +{"stream":"sponsored_display_budget_rules","data":{"createdDate":1657024512836,"lastUpdatedDate":1657024512836,"ruleDetails":{"budgetIncreaseBy":{"type":"PERCENT","value":32},"duration":{"dateRangeTypeRuleDuration":null,"eventTypeRuleDuration":{"endDate":"20220713","eventId":"ae0226d3-9f97-5122-a749-2e9ba741a2dc","eventName":"Prime Day","startDate":"20220712"}},"name":"ex","performanceMeasureCondition":null,"recurrence":{"daysOfWeek":null,"intraDaySchedule":null,"type":"DAILY"},"ruleType":"SCHEDULE"},"ruleId":"b5abeec6-7624-49e7-8571-97b8ba61551e","ruleState":"ACTIVE","ruleStatus":"EXPIRED","ruleStatusDetails":null},"emitted_at":1687254964816} +{"stream":"sponsored_display_budget_rules","data":{"createdDate":1686765545918,"lastUpdatedDate":1686765545918,"ruleDetails":{"budgetIncreaseBy":{"type":"PERCENT","value":1},"duration":{"dateRangeTypeRuleDuration":null,"eventTypeRuleDuration":{"endDate":"20230619","eventId":"553ddee0-8178-544b-a54a-f8918d21ad5f","eventName":"Father's Day","startDate":"20230611"}},"name":"Rule for vadim","performanceMeasureCondition":null,"recurrence":{"daysOfWeek":null,"intraDaySchedule":null,"type":"DAILY"},"ruleType":"SCHEDULE"},"ruleId":"039ff522-f785-4409-8f3a-f6f884ec1750","ruleState":"ACTIVE","ruleStatus":"EXPIRED","ruleStatusDetails":null},"emitted_at":1687254965077} +{"stream": "portfolios", "data": {"portfolioId": 253945852845204, "name": "Test Portfolio 2", "inBudget": true, "state": "enabled", "creationDate": 1687510907465, "lastUpdatedDate": 1687510907465, "servingStatus": "PORTFOLIO_STATUS_ENABLED"}, "emitted_at": 1688475309870} +{"stream": "portfolios", "data": {"portfolioId": 270076898441727, "name": "Test Portfolio", "budget": {"amount": 1.0, "currencyCode": "USD", "policy": "dateRange", "startDate": "20230623", "endDate": "20230624"}, "inBudget": true, "state": "enabled", "creationDate": 1687510616329, "lastUpdatedDate": 1687514774484, "servingStatus": "PORTFOLIO_ENDED"}, "emitted_at": 1688475309871} +{"stream":"sponsored_product_ad_group_suggested_keywords","data":{"adGroupId":103188883625219,"suggestedKeywords":[{"keywordText":"disposable hotel slippers","matchType":"broad"},{"keywordText":"hotel slippers women","matchType":"broad"},{"keywordText":"slippers bulk","matchType":"broad"},{"keywordText":"spa slipper","matchType":"broad"},{"keywordText":"disposable guest slippers","matchType":"broad"},{"keywordText":"hotel slipper","matchType":"broad"},{"keywordText":"black bulk slippers","matchType":"broad"},{"keywordText":"disposable black slippers","matchType":"broad"},{"keywordText":"toothbrush oral b medium","matchType":"broad"},{"keywordText":"toothbrush soft oral b","matchType":"broad"},{"keywordText":"diamond cat food wet","matchType":"broad"},{"keywordText":"toothbrush 1 count","matchType":"broad"},{"keywordText":"black slipper pack","matchType":"broad"},{"keywordText":"toothbrush medium","matchType":"broad"},{"keywordText":"peach mango propel water","matchType":"broad"},{"keywordText":"black guest slippers","matchType":"broad"},{"keywordText":"black hotel slippers","matchType":"broad"},{"keywordText":"black house slippers bulk","matchType":"broad"},{"keywordText":"black spa slippers","matchType":"broad"},{"keywordText":"diamond natural wet cat food","matchType":"broad"},{"keywordText":"house slippers 6 pack","matchType":"broad"},{"keywordText":"house slippers guests bulk","matchType":"broad"},{"keywordText":"single toothbrush","matchType":"broad"},{"keywordText":"spa slippers women black","matchType":"broad"},{"keywordText":"toothbrush oral b manual","matchType":"broad"},{"keywordText":"medium toothbrush single","matchType":"broad"},{"keywordText":"toothbrush charcoal","matchType":"broad"},{"keywordText":"toothbrush hard","matchType":"broad"},{"keywordText":"tooth brush medium","matchType":"broad"},{"keywordText":"tooth brush soft","matchType":"broad"},{"keywordText":"house slippers disposable","matchType":"broad"},{"keywordText":"toothbrush oral b","matchType":"broad"},{"keywordText":"toothbrush soft extra","matchType":"broad"},{"keywordText":"black disposable slippers guests","matchType":"broad"},{"keywordText":"bulk slippers guests washable","matchType":"broad"},{"keywordText":"crest toothbrush","matchType":"broad"},{"keywordText":"toothbrush travel","matchType":"broad"},{"keywordText":"black spa slippers bulk","matchType":"broad"},{"keywordText":"house slippers bulk","matchType":"broad"},{"keywordText":"house slippers visitor","matchType":"broad"},{"keywordText":"disposable slipper","matchType":"broad"},{"keywordText":"spa slippers disposable black","matchType":"broad"},{"keywordText":"toothbrush small","matchType":"broad"},{"keywordText":"cepillo de dientes","matchType":"broad"},{"keywordText":"guest slippers bulk","matchType":"broad"},{"keywordText":"soft toothbrush","matchType":"broad"},{"keywordText":"spa house slippers","matchType":"broad"},{"keywordText":"toothbrush firm","matchType":"broad"},{"keywordText":"toothbrush sensitive","matchType":"broad"},{"keywordText":"bulk pack slippers","matchType":"broad"},{"keywordText":"house guest slippers","matchType":"broad"},{"keywordText":"propel peach water","matchType":"broad"},{"keywordText":"teeth brush","matchType":"broad"},{"keywordText":"tooth brush oral b","matchType":"broad"},{"keywordText":"toothbrush amazon fresh","matchType":"broad"},{"keywordText":"toothbrush oralb","matchType":"broad"},{"keywordText":"toothbrush whitening","matchType":"broad"},{"keywordText":"toothbrusj","matchType":"broad"},{"keywordText":"water propel","matchType":"broad"},{"keywordText":"extra soft tooth brush","matchType":"broad"},{"keywordText":"house slippers guests washable","matchType":"broad"},{"keywordText":"propel peach mango","matchType":"broad"},{"keywordText":"tootbrush","matchType":"broad"},{"keywordText":"toothbrush","matchType":"broad"},{"keywordText":"toothbrush bamboo","matchType":"broad"},{"keywordText":"diamond naturals canned cat food","matchType":"broad"},{"keywordText":"organic potato","matchType":"broad"},{"keywordText":"spa slippers bulk","matchType":"broad"},{"keywordText":"toothbrush oral b white","matchType":"broad"},{"keywordText":"toothbrush soft bristle","matchType":"broad"},{"keywordText":"slippers 12 pair","matchType":"broad"},{"keywordText":"black house guest slippers","matchType":"broad"},{"keywordText":"black house slippers pack","matchType":"broad"},{"keywordText":"black slippers set","matchType":"broad"},{"keywordText":"black washable slippers","matchType":"broad"},{"keywordText":"black washable spa slippers","matchType":"broad"},{"keywordText":"bulk house shoes guests","matchType":"broad"},{"keywordText":"diamond naturals cat food can","matchType":"broad"},{"keywordText":"diamond naturals kitten food wet","matchType":"broad"},{"keywordText":"dispisable slippers","matchType":"broad"},{"keywordText":"disposable house slippers black","matchType":"broad"},{"keywordText":"disposable spa slippers bulk","matchType":"broad"},{"keywordText":"disposable washable slippers","matchType":"broad"},{"keywordText":"disposal house slippers","matchType":"broad"},{"keywordText":"fisposable slippers","matchType":"broad"},{"keywordText":"guest slippers washable set","matchType":"broad"},{"keywordText":"hoise slippers","matchType":"broad"},{"keywordText":"home slipper set","matchType":"broad"},{"keywordText":"house alippers guests","matchType":"broad"},{"keywordText":"house shoes guests washable","matchType":"broad"},{"keywordText":"house slipeprs","matchType":"broad"},{"keywordText":"disposable house slippers guest","matchType":"broad"},{"keywordText":"disposable slippers women","matchType":"broad"},{"keywordText":"disposable spa slippers","matchType":"broad"},{"keywordText":"hotel slippers bulk","matchType":"broad"},{"keywordText":"disposable slippers travel","matchType":"broad"},{"keywordText":"one time use slippers","matchType":"broad"},{"keywordText":"pack slippers guest","matchType":"broad"},{"keywordText":"guest slipper","matchType":"broad"},{"keywordText":"guest slippers washable","matchType":"broad"}]},"emitted_at":1688632533382} +{"stream":"sponsored_product_ad_group_bid_recommendations","data":{"adGroupId":183961953969922,"suggestedBid":{"rangeEnd":1.71,"rangeStart":0.14,"suggested":0.62}},"emitted_at":1688632722904} diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index c18f6a115bcb..9881c233f8fc 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 1.1.0 + dockerImageTag: 3.1.0 dockerRepository: airbyte/source-amazon-ads githubIssueLabel: source-amazon-ads icon: amazonads.svg @@ -23,4 +23,13 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads tags: - language:python + releases: + breakingChanges: + 3.0.0: + message: "Attribution report stream schemas fix." + upgradeDeadline: "2023-07-24" + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amazon-ads/requirements.txt b/airbyte-integrations/connectors/source-amazon-ads/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/requirements.txt +++ b/airbyte-integrations/connectors/source-amazon-ads/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py index 6bec81cb59cf..13b317863d0e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ b/airbyte-integrations/connectors/source-amazon-ads/setup.py @@ -8,6 +8,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.16", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.7.0", "jsonschema~=3.2.0", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/__init__.py index 65afa0f7b0e9..e8c40cbed936 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py index f5a7f3532468..8c3f0963a5ae 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py @@ -1,32 +1,19 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from .attribution_report import AttributionReportModel -from .common import CatalogModel, Keywords, MetricsReport, NegativeKeywords +from .common import CatalogModel, Keywords, MetricsReport, NegativeKeywords, Portfolio from .profile import Profile from .sponsored_brands import BrandsAdGroup, BrandsCampaign -from .sponsored_display import DisplayAdGroup, DisplayCampaign, DisplayProductAds, DisplayTargeting -from .sponsored_products import ProductAd, ProductAdGroups, ProductCampaign, ProductTargeting +from .sponsored_display import DisplayAdGroup, DisplayBudgetRules, DisplayCampaign, DisplayProductAds, DisplayTargeting +from .sponsored_products import ( + ProductAd, + ProductAdGroupBidRecommendations, + ProductAdGroups, + ProductAdGroupSuggestedKeywords, + ProductCampaign, + ProductTargeting, +) __all__ = [ "BrandsAdGroup", @@ -36,11 +23,15 @@ "DisplayCampaign", "DisplayProductAds", "DisplayTargeting", + "DisplayBudgetRules", "Keywords", "MetricsReport", "NegativeKeywords", + "Portfolio", "ProductAd", "ProductAdGroups", + "ProductAdGroupBidRecommendations", + "ProductAdGroupSuggestedKeywords", "ProductCampaign", "ProductTargeting", "Profile", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py index eb6c2c4bac27..6aea054ea703 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py @@ -2,12 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import List - from .common import CatalogModel -class Report(CatalogModel): +class AttributionReportModel(CatalogModel): date: str brandName: str marketplace: str @@ -22,9 +20,3 @@ class Report(CatalogModel): productSubcategory: str productGroup: str publisher: str - - -class AttributionReportModel(CatalogModel): - reports: List[Report] - size: int - cursorId: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py index 3469b38be39b..89485a7a8d72 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/common.py @@ -39,6 +39,8 @@ def schema_extra(cls, schema: Dict[str, Any], model: Type["BaseModel"]) -> None: if prop["type"] == "array" and prop["items"]: if prop["items"].pop("additionalProperties", None): prop["items"]["additionalProperties"] = True + if schema["type"] == "object": + schema["type"] = ["object", "null"] class MetricsReport(CatalogModel): @@ -79,3 +81,22 @@ class Keywords(KeywordsBase): class NegativeKeywords(KeywordsBase): matchType: str + + +class Budget(CatalogModel): + amount: Decimal = None + currencyCode: str = None + policy: str = None + startDate: str = None + endDate: str = None + + +class Portfolio(CatalogModel): + portfolioId: int + name: str = None + budget: Budget = None + inBudget: bool = None + state: str = None + creationDate: int = None + lastUpdatedDate: int = None + servingStatus: str = None diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py index f16ee791e47c..510d2492d7b2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py @@ -19,7 +19,7 @@ class BrandsCampaign(CatalogModel): state: str servingStatus: str brandEntityId: str - portfolioId: Decimal + portfolioId: int bidOptimization: bool = None bidMultiplier: Decimal = None adFormat: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py index f0a4952d3856..ffaf34b72e06 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py @@ -3,7 +3,7 @@ # from decimal import Decimal -from typing import Dict, List +from typing import Dict, List, Optional from .common import CatalogModel, Targeting @@ -17,7 +17,7 @@ class DisplayCampaign(CatalogModel): endDate: str = None costType: str state: str - portfolioId: str = None + portfolioId: int = None tactic: str deliveryProfile: str @@ -45,3 +45,56 @@ class DisplayProductAds(CatalogModel): class DisplayTargeting(Targeting): expression: List[Dict[str, str]] resolvedExpression: List[Dict[str, str]] + + +class DisplayBudgetRuleDetailsPerformanceMeasureCondition(CatalogModel): + metricName: str + comparisonOperator: str + threshold: Decimal + + +class DisplayBudgetRuleDetailsRecurrence(CatalogModel): + type: str + daysOfWeek: List[str] = None + threshold: Decimal + + +class DisplayBudgetRuleDetailsBudgetIncreaseBy(CatalogModel): + type: str + value: Decimal + + +class DisplayBudgetRuleDetailsDurationEventTypeRuleDuration(CatalogModel): + eventId: str + endDate: str + eventName: str + startDate: str + + +class DisplayBudgetRuleDetailsDurationDateRangeTypeRuleDuration(CatalogModel): + endDate: str + startDate: str + + +class DisplayBudgetRuleDetailsDuration(CatalogModel): + eventTypeRuleDuration: Optional[DisplayBudgetRuleDetailsDurationEventTypeRuleDuration] = None + dateRangeTypeRuleDuration: Optional[DisplayBudgetRuleDetailsDurationDateRangeTypeRuleDuration] = None + + +class DisplayBudgetRuleDetails(CatalogModel): + name: str + ruleType: str = None + duration: Optional[DisplayBudgetRuleDetailsDuration] = None + budgetIncreaseBy: Optional[DisplayBudgetRuleDetailsBudgetIncreaseBy] = None + recurrence: Optional[DisplayBudgetRuleDetailsRecurrence] = None + performanceMeasureCondition: Optional[DisplayBudgetRuleDetailsPerformanceMeasureCondition] = None + + +class DisplayBudgetRules(CatalogModel): + ruleId: str + ruleStatus: str + ruleState: str + lastUpdatedDate: Decimal + createdDate: Decimal + ruleDetails: DisplayBudgetRuleDetails = None + ruleStatusDetails: Dict[str, str] = None diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py index 6a21b534176e..d60f1042592c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py @@ -3,7 +3,7 @@ # from decimal import Decimal -from typing import Dict, List +from typing import Dict, List, Optional from .common import CatalogModel, Targeting @@ -19,7 +19,7 @@ class Bidding(CatalogModel): class ProductCampaign(CatalogModel): - portfolioId: Decimal + portfolioId: int campaignId: Decimal name: str tags: Dict[str, str] @@ -42,6 +42,27 @@ class ProductAdGroups(CatalogModel): state: str +class SuggestedBid(CatalogModel): + suggested: Decimal + rangeStart: Decimal + rangeEnd: Decimal + + +class ProductAdGroupBidRecommendations(CatalogModel): + adGroupId: Decimal + suggestedBid: Optional[SuggestedBid] = None + + +class SuggestedKeyword(CatalogModel): + keywordText: str + matchType: str + + +class ProductAdGroupSuggestedKeywords(CatalogModel): + adGroupId: Decimal + suggestedKeywords: List[SuggestedKeyword] = None + + class ProductAd(CatalogModel): adId: Decimal campaignId: Decimal diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 2fe44f002b17..c521e04082a5 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -17,19 +17,25 @@ AttributionReportPerformanceCampaign, AttributionReportPerformanceCreative, AttributionReportProducts, + Portfolios, Profiles, SponsoredBrandsAdGroups, SponsoredBrandsCampaigns, SponsoredBrandsKeywords, SponsoredBrandsReportStream, + SponsoredBrandsV3ReportStream, SponsoredBrandsVideoReportStream, SponsoredDisplayAdGroups, + SponsoredDisplayBudgetRules, SponsoredDisplayCampaigns, SponsoredDisplayProductAds, SponsoredDisplayReportStream, SponsoredDisplayTargetings, + SponsoredProductAdGroupBidRecommendations, SponsoredProductAdGroups, + SponsoredProductAdGroupSuggestedKeywords, SponsoredProductAds, + SponsoredProductCampaignNegativeKeywords, SponsoredProductCampaigns, SponsoredProductKeywords, SponsoredProductNegativeKeywords, @@ -98,10 +104,14 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SponsoredDisplayProductAds, SponsoredDisplayTargetings, SponsoredDisplayReportStream, + SponsoredDisplayBudgetRules, SponsoredProductCampaigns, SponsoredProductAdGroups, + SponsoredProductAdGroupBidRecommendations, + SponsoredProductAdGroupSuggestedKeywords, SponsoredProductKeywords, SponsoredProductNegativeKeywords, + SponsoredProductCampaignNegativeKeywords, SponsoredProductAds, SponsoredProductTargetings, SponsoredProductsReportStream, @@ -109,13 +119,15 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SponsoredBrandsAdGroups, SponsoredBrandsKeywords, SponsoredBrandsReportStream, + SponsoredBrandsV3ReportStream, SponsoredBrandsVideoReportStream, AttributionReportPerformanceAdgroup, AttributionReportPerformanceCampaign, AttributionReportPerformanceCreative, AttributionReportProducts, ] - return [profiles_stream, *[stream_class(**stream_args) for stream_class in non_profile_stream_classes]] + portfolios_stream = Portfolios(**stream_args) + return [profiles_stream, portfolios_stream, *[stream_class(**stream_args) for stream_class in non_profile_stream_classes]] @staticmethod def _make_authenticator(config: Mapping[str, Any]): diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index c747d634fdee..795ca21589e7 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -81,8 +81,8 @@ connectionSpecification: title: "Look Back Window" description: "The amount of days to go back in time to get the updated data from Amazon Ads" examples: - - 3 - - 10 + - 3 + - 10 type: "integer" default: 3 order: 8 diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py index 1d7ab84d65df..fb27c5d5c792 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from .attribution_report import ( AttributionReportPerformanceAdgroup, @@ -7,18 +7,29 @@ AttributionReportPerformanceCreative, AttributionReportProducts, ) +from .portfolios import Portfolios from .profiles import Profiles from .report_streams import ( SponsoredBrandsReportStream, + SponsoredBrandsV3ReportStream, SponsoredBrandsVideoReportStream, SponsoredDisplayReportStream, SponsoredProductsReportStream, ) from .sponsored_brands import SponsoredBrandsAdGroups, SponsoredBrandsCampaigns, SponsoredBrandsKeywords -from .sponsored_display import SponsoredDisplayAdGroups, SponsoredDisplayCampaigns, SponsoredDisplayProductAds, SponsoredDisplayTargetings +from .sponsored_display import ( + SponsoredDisplayAdGroups, + SponsoredDisplayBudgetRules, + SponsoredDisplayCampaigns, + SponsoredDisplayProductAds, + SponsoredDisplayTargetings, +) from .sponsored_products import ( + SponsoredProductAdGroupBidRecommendations, SponsoredProductAdGroups, + SponsoredProductAdGroupSuggestedKeywords, SponsoredProductAds, + SponsoredProductCampaignNegativeKeywords, SponsoredProductCampaigns, SponsoredProductKeywords, SponsoredProductNegativeKeywords, @@ -26,16 +37,21 @@ ) __all__ = [ + "Portfolios", "Profiles", "SponsoredDisplayAdGroups", "SponsoredDisplayCampaigns", "SponsoredDisplayProductAds", "SponsoredDisplayTargetings", + "SponsoredDisplayBudgetRules", "SponsoredProductAdGroups", + "SponsoredProductAdGroupBidRecommendations", + "SponsoredProductAdGroupSuggestedKeywords", "SponsoredProductAds", "SponsoredProductCampaigns", "SponsoredProductKeywords", "SponsoredProductNegativeKeywords", + "SponsoredProductCampaignNegativeKeywords", "SponsoredProductTargetings", "SponsoredBrandsCampaigns", "SponsoredBrandsAdGroups", @@ -43,6 +59,7 @@ "SponsoredDisplayReportStream", "SponsoredProductsReportStream", "SponsoredBrandsReportStream", + "SponsoredBrandsV3ReportStream", "SponsoredBrandsVideoReportStream", "AttributionReportPerformanceAdgroup", "AttributionReportPerformanceCampaign", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py index 3fec199ef6c8..38d2073e5e2a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/attribution_report.py @@ -60,7 +60,7 @@ class AttributionReport(AmazonAdsStream): page_size = 300 report_type = "" - metrics = "" + custom_metrics = [] group_by = "" _next_page_token_field = "cursorId" @@ -74,6 +74,10 @@ def __init__(self, config: Mapping[str, Any], *args, **kwargs): self._start_date = config.get("start_date") super().__init__(config, *args, **kwargs) + @property + def metrics(self): + return METRICS_MAP[self.report_type] + self.custom_metrics + @property def http_method(self) -> str: return "POST" @@ -81,6 +85,12 @@ def http_method(self) -> str: def path(self, **kvargs) -> str: return "/attribution/report" + def get_json_schema(self): + schema = super().get_json_schema() + metrics_type_map = {metric: {"type": ["null", "string"]} for metric in self.metrics} + schema["properties"].update(metrics_type_map) + return schema + def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: @@ -135,7 +145,7 @@ def request_body_json( body = { "reportType": self.report_type, "count": self.page_size, - "metrics": self.metrics, + "metrics": ",".join(self.metrics), "startDate": stream_slice["startDate"], "endDate": stream_slice["endDate"], self._next_page_token_field: "", @@ -152,35 +162,21 @@ def request_body_json( class AttributionReportProducts(AttributionReport): report_type = "PRODUCTS" - - metrics = ",".join(METRICS_MAP[report_type]) - group_by = "" class AttributionReportPerformanceCreative(AttributionReport): report_type = "PERFORMANCE" - - metrics = ",".join(METRICS_MAP[report_type]) - group_by = "CREATIVE" class AttributionReportPerformanceAdgroup(AttributionReport): report_type = "PERFORMANCE" - - metrics_list = METRICS_MAP[report_type] - metrics_list.append(BRAND_REFERRAL_BONUS) - metrics = ",".join(metrics_list) - + custom_metrics = [BRAND_REFERRAL_BONUS] group_by = "ADGROUP" class AttributionReportPerformanceCampaign(AttributionReport): report_type = "PERFORMANCE" - - metrics_list = METRICS_MAP[report_type] - metrics_list.append(BRAND_REFERRAL_BONUS) - metrics = ",".join(metrics_list) - + custom_metrics = [BRAND_REFERRAL_BONUS] group_by = "CAMPAIGN" diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 46c428b75ee5..12f6fa0c7606 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -24,6 +24,7 @@ class to provide explanation why it had been done in this way. ├── airbyte_cdk.sources.streams.http.HttpStream │ └── AmazonAdsStream │ ├── Profiles + │ ├── Portfolios │ └── SubProfilesStream │ ├── SponsoredDisplayAdGroups │ ├── SponsoredDisplayCampaigns @@ -40,6 +41,7 @@ class to provide explanation why it had been done in this way. │ └── SponsoredBrandsKeywords └── ReportStream ├── SponsoredBrandsReportStream + ├── SponsoredBrandsV3ReportStream ├── SponsoredDisplayReportStream └── SponsoredProductsReportStream @@ -48,11 +50,11 @@ class to provide explanation why it had been done in this way. profile id. Also it stores pydantic model and API url for requests. AmazonAdsStream is Http based class, it used for making request that could be -accomlished by single http call (any but report streams). +accomplished by single http call (any but report streams). SubProfilesStream is subclass for http streams to perform read_records from basic class for EACH profile from self._profiles list. Also provides support -for Amazon Ads API pagintaion. This is base class for all the sync http streams +for Amazon Ads API pagination. This is base class for all the sync http streams that used by source. ReportStream (It implemented on report_stream.py file) is subclass for async diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/portfolios.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/portfolios.py new file mode 100644 index 000000000000..6892d8ffa896 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/portfolios.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Iterable, Mapping, MutableMapping + +from source_amazon_ads.schemas import Portfolio +from source_amazon_ads.streams.common import AmazonAdsStream + + +class Portfolios(AmazonAdsStream): + """ + This stream corresponds to Amazon Advertising API - Portfolios + https://advertising.amazon.com/API/docs/en-us/reference/2/portfolios + """ + + primary_key = "portfolioId" + model = Portfolio + + def path(self, **kvargs) -> str: + return "v2/portfolios/extended" + + def read_records(self, *args, **kvargs) -> Iterable[Mapping[str, Any]]: + """ + Iterate through self._profiles list and send read all records for each profile. + """ + for profile in self._profiles: + self._current_profile_id = profile.profileId + yield from super().read_records(*args, **kvargs) + + def request_headers(self, *args, **kvargs) -> MutableMapping[str, Any]: + headers = super().request_headers(*args, **kvargs) + headers["Amazon-Advertising-API-Scope"] = str(self._current_profile_id) + return headers diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py index 28e70950a759..314d3770336b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/__init__.py @@ -1,7 +1,7 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from .brands_report import SponsoredBrandsReportStream +from .brands_report import SponsoredBrandsReportStream, SponsoredBrandsV3ReportStream from .brands_video_report import SponsoredBrandsVideoReportStream from .display_report import SponsoredDisplayReportStream from .products_report import SponsoredProductsReportStream @@ -10,5 +10,6 @@ "SponsoredDisplayReportStream", "SponsoredProductsReportStream", "SponsoredBrandsReportStream", + "SponsoredBrandsV3ReportStream", "SponsoredBrandsVideoReportStream", ] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py index b5bd605fcaf1..fbe14a4bc118 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_report.py @@ -2,6 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from http import HTTPStatus + +from .products_report import SponsoredProductsReportStream from .report_streams import ReportStream METRICS_MAP = { @@ -99,7 +102,6 @@ ], } - METRICS_TYPE_TO_ID_MAP = { "keywords": "keywordBid", "adGroups": "adGroupId", @@ -123,4 +125,66 @@ def _get_init_report_body(self, report_date: str, record_type: str, profile): body = { "reportDate": report_date, } - return {**body, "metrics": ",".join(metrics_list)} + yield {**body, "metrics": ",".join(metrics_list)} + + +METRICS_MAP_V3 = { + "purchasedAsin": [ + "campaignBudgetCurrencyCode", + "campaignName", + "adGroupName", + "attributionType", + "purchasedAsin", + "productName", + "productCategory", + "sales14d", + "orders14d", + "unitsSold14d", + "newToBrandSales14d", + "newToBrandPurchases14d", + "newToBrandUnitsSold14d", + "newToBrandSalesPercentage14d", + "newToBrandPurchasesPercentage14d", + "newToBrandUnitsSoldPercentage14d", + ] +} + +METRICS_TYPE_TO_ID_MAP_V3 = { + "purchasedAsin": "purchasedAsin", +} + + +class SponsoredBrandsV3ReportStream(SponsoredProductsReportStream): + """ + https://advertising.amazon.com/API/docs/en-us/guides/reporting/v3/report-types#purchased-product-reports + """ + + API_VERSION = "reporting" # v3 + REPORT_DATE_FORMAT = "YYYY-MM-DD" + ad_product = "SPONSORED_BRANDS" + report_is_created = HTTPStatus.OK + metrics_map = METRICS_MAP_V3 + metrics_type_to_id_map = METRICS_TYPE_TO_ID_MAP_V3 + + def _get_init_report_body(self, report_date: str, record_type: str, profile): + metrics_list = self.metrics_map[record_type] + + reportTypeId = "sbPurchasedProduct" + group_by = ["purchasedAsin"] + + body = { + "name": f"{record_type} report {report_date}", + "startDate": report_date, + "endDate": report_date, + "configuration": { + "adProduct": self.ad_product, + "groupBy": group_by, + "columns": metrics_list, + "reportTypeId": reportTypeId, + "filters": [], + "timeUnit": "SUMMARY", + "format": "GZIP_JSON", + }, + } + + yield body diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py index cfa9ab7a21c8..b5da376396cd 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/brands_video_report.py @@ -143,4 +143,4 @@ def _get_init_report_body(self, report_date: str, record_type: str, profile): "reportDate": report_date, "creativeType": "video", } - return {**body, "metrics": ",".join(metrics_list)} + yield {**body, "metrics": ",".join(metrics_list)} diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py index 825e65f0fd11..de158949addc 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/display_report.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from enum import Enum from .report_streams import RecordType, ReportStream @@ -214,12 +213,7 @@ METRICS_TYPE_TO_ID_MAP = {"campaigns": "campaignId", "adGroups": "adGroupId", "productAds": "adId", "targets": "targetId", "asins": "asin"} - -class Tactics(str, Enum): - T00001 = "T00001" - T00020 = "T00020" - T00030 = "T00030" - REMARKETING = "remarketing" +TACTICS = ["T00020", "T00030"] class SponsoredDisplayReportStream(ReportStream): @@ -234,15 +228,15 @@ def report_init_endpoint(self, record_type: str) -> str: metrics_type_to_id_map = METRICS_TYPE_TO_ID_MAP def _get_init_report_body(self, report_date: str, record_type: str, profile): - metrics_list = self.metrics_map[record_type] - if record_type == RecordType.ASINS and profile.accountInfo.type == "vendor": - return None - elif record_type == RecordType.PRODUCTADS and profile.accountInfo.type != "seller": - # Remove SKU from metrics since it is only available for seller accounts in Product Ad report - metrics_list = [m for m in metrics_list if m != "sku"] - return { - "reportDate": report_date, - # Only for most common T00020 tactic for now - "tactic": Tactics.T00020, - "metrics": ",".join(metrics_list), - } + for tactic in TACTICS: + metrics_list = self.metrics_map[record_type] + if record_type == RecordType.ASINS and profile.accountInfo.type == "vendor": + return None + elif record_type == RecordType.PRODUCTADS and profile.accountInfo.type != "seller": + # Remove SKU from metrics since it is only available for seller accounts in Product Ad report + metrics_list = [m for m in metrics_list if m != "sku"] + yield { + "reportDate": report_date, + "tactic": tactic, + "metrics": ",".join(metrics_list), + } diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py index 036dc67d8258..359f9b424c30 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py @@ -316,4 +316,4 @@ def _get_init_report_body(self, report_date: str, record_type: str, profile): }, } - return body + yield body diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 047a8a75e497..836588eb0639 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -109,6 +109,7 @@ class ReportStream(BasicAmazonAdsStream, ABC): # Check if the connector received an error like: 'Tactic T00020 is not supported for report API in marketplace A1C3SOZRARQ6R3.' # https://docs.developer.amazonservices.com/en_UK/dev_guide/DG_Endpoints.html (400, re.compile(r"^Tactic T00020 is not supported for report API in marketplace [A-Z\d]+\.$")), + (400, re.compile(r"^Tactic T00030 is not supported for report API in marketplace [A-Z\d]+\.$")), # Check if the connector received an error: 'Report date is too far in the past. Reports are only available for 60 days.' # In theory, it does not have to get such an error because the connector correctly calculates the start date, # but from practice, we can still catch such errors from time to time. @@ -162,19 +163,22 @@ def read_records( return profile = stream_slice["profile"] report_date = stream_slice[self.cursor_field] - report_infos = self._init_and_try_read_records(profile, report_date) + report_info_list = self._init_and_try_read_records(profile, report_date) self._update_state(profile, report_date) - for report_info in report_infos: + for report_info in report_info_list: for metric_object in report_info.metric_objects: yield self._model( profileId=report_info.profile_id, recordType=report_info.record_type, reportDate=report_date, - recordId=metric_object.get(self.metrics_type_to_id_map[report_info.record_type], str(uuid.uuid4())), + recordId=self.get_record_id(metric_object, report_info.record_type), metric=metric_object, ).dict() + def get_record_id(self, metric_object: dict, record_type: str) -> str: + return metric_object.get(self.metrics_type_to_id_map[record_type]) or str(uuid.uuid4()) + def backoff_max_time(func): def wrapped(self, *args, **kwargs): return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout * 60, interval=10)(func)( @@ -193,16 +197,16 @@ def wrapped(self, *args, **kwargs): @backoff_max_tries def _init_and_try_read_records(self, profile: Profile, report_date): - report_infos = self._init_reports(profile, report_date) - self.logger.info(f"Waiting for {len(report_infos)} report(s) to be generated") - self._try_read_records(report_infos) - return report_infos + report_info_list = self._init_reports(profile, report_date) + self.logger.info(f"Waiting for {len(report_info_list)} report(s) to be generated") + self._try_read_records(report_info_list) + return report_info_list @backoff_max_time - def _try_read_records(self, report_infos): - incomplete_report_infos = self._incomplete_report_infos(report_infos) - self.logger.info(f"Checking report status, {len(incomplete_report_infos)} report(s) remaining") - for report_info in incomplete_report_infos: + def _try_read_records(self, report_info_list): + incomplete_report_info = self._incomplete_report_info(report_info_list) + self.logger.info(f"Checking report status, {len(incomplete_report_info)} report(s) remaining") + for report_info in incomplete_report_info: report_status, download_url = self._check_status(report_info) report_info.status = report_status @@ -215,13 +219,13 @@ def _try_read_records(self, report_infos): except requests.HTTPError as error: raise ReportGenerationFailure(error) - pending_report_status = [(r.profile_id, r.report_id, r.status) for r in self._incomplete_report_infos(report_infos)] + pending_report_status = [(r.profile_id, r.report_id, r.status) for r in self._incomplete_report_info(report_info_list)] if len(pending_report_status) > 0: message = f"Report generation in progress: {repr(pending_report_status)}" raise ReportGenerationInProgress(message) - def _incomplete_report_infos(self, report_infos): - return [r for r in report_infos if r.status != Status.SUCCESS and r.status != Status.COMPLETED] + def _incomplete_report_info(self, report_info_list): + return [r for r in report_info_list if r.status != Status.SUCCESS and r.status != Status.COMPLETED] def _generate_model(self): """ @@ -377,46 +381,48 @@ def _init_reports(self, profile: Profile, report_date: str) -> List[ReportInfo]: :report_date - date for generating metric report. :return List of ReportInfo objects each of them has reportId field to check report status. """ - report_infos = [] + report_info_list = [] for record_type, metrics in self.metrics_map.items(): if len(self._report_record_types) > 0 and record_type not in self._report_record_types: continue - report_init_body = self._get_init_report_body(report_date, record_type, profile) - if not report_init_body: - continue - # Some of the record types has subtypes. For example asins type - # for product report have keyword and targets subtypes and it - # represented as asins_keywords and asins_targets types. Those - # subtypes have mutually excluded parameters so we requesting - # different metric list for each record. - request_record_type = record_type.split("_")[0] - self.logger.info(f"Initiating report generation for {profile.profileId} profile with {record_type} type for {report_date} date") - response = self._send_http_request( - urljoin(self._url, self.report_init_endpoint(request_record_type)), - profile.profileId, - report_init_body, - ) - if response.status_code != self.report_is_created: - error_msg = f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}" - if self._skip_known_errors(response): - self.logger.warning(error_msg) - break - raise ReportInitFailure(error_msg) - - response = ReportInitResponse.parse_raw(response.text) - report_infos.append( - ReportInfo( - report_id=response.reportId, - record_type=record_type, - profile_id=profile.profileId, - status=Status.IN_PROGRESS, - metric_objects=[], + for report_init_body in self._get_init_report_body(report_date, record_type, profile): + if not report_init_body: + continue + # Some of the record types has subtypes. For example asins type + # for product report have keyword and targets subtypes and it + # represented as asins_keywords and asins_targets types. Those + # subtypes have mutually excluded parameters so we requesting + # different metric list for each record. + request_record_type = record_type.split("_")[0] + self.logger.info( + f"Initiating report generation for {profile.profileId} profile with {record_type} type for {report_date} date" ) - ) - self.logger.info("Initiated successfully") + response = self._send_http_request( + urljoin(self._url, self.report_init_endpoint(request_record_type)), + profile.profileId, + report_init_body, + ) + if response.status_code != self.report_is_created: + error_msg = f"Unexpected HTTP status code {response.status_code} when registering {record_type}, {type(self).__name__} for {profile.profileId} profile: {response.text}" + if self._skip_known_errors(response): + self.logger.warning(error_msg) + break + raise ReportInitFailure(error_msg) + + response = ReportInitResponse.parse_raw(response.text) + report_info_list.append( + ReportInfo( + report_id=response.reportId, + record_type=record_type, + profile_id=profile.profileId, + status=Status.IN_PROGRESS, + metric_objects=[], + ) + ) + self.logger.info("Initiated successfully") - return report_infos + return report_info_list @backoff.on_exception( backoff.expo, diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py index ccd335954ae2..35520f22f1c2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py @@ -2,7 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_amazon_ads.schemas import DisplayAdGroup, DisplayCampaign, DisplayProductAds, DisplayTargeting +from typing import Any, Mapping + +from source_amazon_ads.schemas import DisplayAdGroup, DisplayBudgetRules, DisplayCampaign, DisplayProductAds, DisplayTargeting from source_amazon_ads.streams.common import SubProfilesStream @@ -67,3 +69,27 @@ class SponsoredDisplayTargetings(SubProfilesStream): def path(self, **kwargs) -> str: return "sd/targets" + + +class SponsoredDisplayBudgetRules(SubProfilesStream): + """ + This stream corresponds to Amazon Advertising API - Sponsored Displays BudgetRules + https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi/prod#/BudgetRules/GetSDBudgetRulesForAdvertiser + + Important: API docs contains incorrect endpoint path: + sd/budgetRules - endpoint from API docs which always returns empty results + sp/budgetRules - working endpoint + """ + + primary_key = "ruleId" + model = DisplayBudgetRules + data_field = "budgetRulesForAdvertiserResponse" + page_size = 30 + + def path(self, **kwargs) -> str: + return "sp/budgetRules" + + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): + params = super().request_params(stream_slice=stream_slice, **kwargs) + params["pageSize"] = params.pop("count") + return params diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py index bc52a77f4dd3..14b86f888737 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py @@ -2,8 +2,23 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_amazon_ads.schemas import Keywords, NegativeKeywords, ProductAd, ProductAdGroups, ProductCampaign, ProductTargeting -from source_amazon_ads.streams.common import SubProfilesStream +from abc import ABC +from http import HTTPStatus +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional + +import requests as requests +from airbyte_protocol.models import SyncMode +from source_amazon_ads.schemas import ( + Keywords, + NegativeKeywords, + ProductAd, + ProductAdGroupBidRecommendations, + ProductAdGroups, + ProductAdGroupSuggestedKeywords, + ProductCampaign, + ProductTargeting, +) +from source_amazon_ads.streams.common import AmazonAdsStream, SubProfilesStream class SponsoredProductCampaigns(SubProfilesStream): @@ -43,6 +58,93 @@ def path(self, **kvargs) -> str: return "v2/sp/adGroups" +class SponsoredProductAdGroupsWithProfileId(SponsoredProductAdGroups): + """Add profileId attr for each records in SponsoredProductAdGroups stream""" + + def parse_response(self, *args, **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(*args, **kwargs): + record["profileId"] = self._current_profile_id + yield record + + +class SponsoredProductAdGroupWithSlicesABC(AmazonAdsStream, ABC): + """ABC Class for extraction of additional information for each known sp ad group""" + + primary_key = "adGroupId" + + def __init__(self, *args, **kwargs): + self.__args = args + self.__kwargs = kwargs + super().__init__(*args, **kwargs) + + def request_headers(self, *args, **kvargs) -> MutableMapping[str, Any]: + headers = super().request_headers(*args, **kvargs) + headers["Amazon-Advertising-API-Scope"] = str(kvargs["stream_slice"]["profileId"]) + return headers + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + yield from SponsoredProductAdGroupsWithProfileId(*self.__args, **self.__kwargs).read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=None, stream_state=stream_state + ) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + + resp = response.json() + if response.status_code == HTTPStatus.OK: + yield resp + + if response.status_code == HTTPStatus.BAD_REQUEST: + # 400 error message for bids recommendation: + # Bid recommendation for AD group in Manual Targeted Campaign is not supported. + # 400 error message for keywords recommendation: + # Getting keyword recommendations for AD Group in Auto Targeted Campaign is not supported + self.logger.warning( + f"Skip current AdGroup because it does not support request {response.request.url} for " + f"{response.request.headers['Amazon-Advertising-API-Scope']} profile: {response.text}" + ) + + else: + response.raise_for_status() + + +class SponsoredProductAdGroupBidRecommendations(SponsoredProductAdGroupWithSlicesABC): + """Docs: + Latest API: + https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Bid%20Recommendations/getTargetBidRecommendations + POST /sd/targets/bid/recommendations + Note: does not work, always get "403 Forbidden" + + V2 API: + https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Bid%20recommendations/getAdGroupBidRecommendations + GET /v2/sp/adGroups/{adGroupId}/bidRecommendations + """ + + model = ProductAdGroupBidRecommendations + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"v2/sp/adGroups/{stream_slice['adGroupId']}/bidRecommendations" + + +class SponsoredProductAdGroupSuggestedKeywords(SponsoredProductAdGroupWithSlicesABC): + """Docs: + Latest API: + https://advertising.amazon.com/API/docs/en-us/sponsored-products/3-0/openapi/prod#/Keyword%20Targets/getRankedKeywordRecommendation + POST /sp/targets/keywords/recommendations + Note: does not work, always get "403 Forbidden" + + V2 API: + https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Suggested%20keywords + GET /v2/sp/adGroups/{{adGroupId}}>/suggested/keywords + """ + + model = ProductAdGroupSuggestedKeywords + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"v2/sp/adGroups/{stream_slice['adGroupId']}/suggested/keywords" + + class SponsoredProductKeywords(SubProfilesStream): """ This stream corresponds to Amazon Advertising API - Sponsored Products Keywords @@ -69,6 +171,16 @@ def path(self, **kvargs) -> str: return "v2/sp/negativeKeywords" +class SponsoredProductCampaignNegativeKeywords(SponsoredProductNegativeKeywords): + """ + This stream corresponds to Amazon Advertising API - Sponsored Products Negative Keywords + https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Negative%20keywords + """ + + def path(self, **kvargs) -> str: + return "v2/sp/campaignNegativeKeywords" + + class SponsoredProductAds(SubProfilesStream): """ This stream corresponds to Amazon Advertising API - Sponsored Products Ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index 46f148bce5c8..a1335bfb5fb2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -38,6 +38,13 @@ def profiles_response(): """ +@fixture +def portfolios_response(): + return """ +[{"portfolioId":253945852845204,"name":"Test Portfolio 2","inBudget":true,"state":"enabled","creationDate":1687510907465,"lastUpdatedDate":1687510907465,"servingStatus":"PORTFOLIO_STATUS_ENABLED"},{"portfolioId":270076898441727,"name":"Test Portfolio","budget":{"amount":1.0,"currencyCode":"USD","policy":"dateRange","startDate":"20230623","endDate":"20230624"},"inBudget":true,"state":"enabled","creationDate":1687510616329,"lastUpdatedDate":1687514774484,"servingStatus":"PORTFOLIO_STATUS_ENABLED"}] +""" + + @fixture def campaigns_response(): return """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py index 83ba14a70231..cf24dd16ad7b 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py @@ -70,6 +70,7 @@ def test_attribution_report_schema(config, profiles_response, attribution_report profile_stream = get_stream_by_name(streams, "profiles") attribution_report_stream = get_stream_by_name(streams, stream_name) schema = attribution_report_stream.get_json_schema() + schema["additionalProperties"] = False profile_records = list(read_full_refresh(profile_stream)) attribution_records = list(read_full_refresh(attribution_report_stream)) diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 560de684e37e..9b4329f37031 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -22,12 +22,14 @@ from source_amazon_ads.streams import ( SponsoredBrandsCampaigns, SponsoredBrandsReportStream, + SponsoredBrandsV3ReportStream, SponsoredBrandsVideoReportStream, SponsoredDisplayCampaigns, SponsoredDisplayReportStream, SponsoredProductCampaigns, SponsoredProductsReportStream, ) +from source_amazon_ads.streams.report_streams.display_report import TACTICS from source_amazon_ads.streams.report_streams.report_streams import ( RecordType, ReportGenerationFailure, @@ -195,14 +197,14 @@ def test_display_report_stream(config): stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) stream_slice = {"profile": profiles[0], "reportDate": "20210725"} metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] - assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) + assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) * len(TACTICS) profiles = make_profiles(profile_type="vendor") stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) stream_slice["profile"] = profiles[0] metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] # Skip asins record for vendor profiles - assert len(metrics) == METRICS_COUNT * (len(stream.metrics_map) - 1) + assert len(metrics) == METRICS_COUNT * (len(stream.metrics_map) - 1) * len(TACTICS) @pytest.mark.parametrize( @@ -295,6 +297,22 @@ def test_brands_report_stream(config): assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) +@responses.activate +def test_brands_v3_report_stream(config): + setup_responses( + init_response_products=REPORT_INIT_RESPONSE, + status_response=REPORT_STATUS_RESPONSE, + metric_response=METRIC_RESPONSE, + ) + + profiles = make_profiles(profile_type="vendor") + + stream = SponsoredBrandsV3ReportStream(config, profiles, authenticator=mock.MagicMock()) + stream_slice = {"profile": profiles[0], "reportDate": "2021-07-25", "retry_count": 3} + metrics = [m for m in stream.read_records(SyncMode.incremental, stream_slice=stream_slice)] + assert len(metrics) == METRICS_COUNT * len(stream.metrics_map) + + @responses.activate def test_brands_video_report_stream(config): setup_responses( @@ -359,15 +377,15 @@ def test_display_report_stream_init_too_many_requests(mocker, config): [ ( [ - (lambda x: x <= 5, "SUCCESS", None), + (lambda x: x <= 10, "SUCCESS", None), ], - 5, + 10, ), ( [ - (lambda x: x > 5, "SUCCESS", None), + (lambda x: x > 10, "SUCCESS", None), ], - 10, + 20, ), ( [ @@ -381,7 +399,7 @@ def test_display_report_stream_init_too_many_requests(mocker, config): (lambda x: x >= 6 and x <= 10, None, "2021-01-02 03:23:05"), (lambda x: x >= 11, "SUCCESS", "2021-01-02 03:24:06"), ], - 15, + 20, ), ( [ @@ -881,3 +899,25 @@ def test_brands_video_report_with_custom_record_types(config_gen, custom_record_ if record['recordType'] not in expected_record_types: if flag_match_error: assert False + + +@pytest.mark.parametrize( + "metric_object, record_type", + [ + ({"campaignId": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}, "campaigns"), + ({"campaignId": ""}, "campaigns"), + ({"campaignId": None}, "campaigns") + ] +) +def test_get_record_id_by_report_type(config, metric_object, record_type): + """ + Test if a `recordId` is allways non-empty for any given `metric_object`. + `recordId` is not a contant key for every report. + We define suitable key for every report by its type and normally it should not be empty. + It may be `campaignId` or `adGroupId` or any other key depending on report type (See METRICS_TYPE_TO_ID_MAP). + In case when it is not defined or empty (sometimes we get one record with missing data while others are populated) + we must return `recordId` anyway so we generate it manually. + """ + profiles = make_profiles(profile_type="vendor") + stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) + assert stream.get_record_id(metric_object, record_type), "recordId must be non-empty value" diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index c51cb566f787..9126547b19ad 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -90,16 +90,20 @@ def test_source_streams(config): setup_responses() source = SourceAmazonAds() streams = source.streams(config) - assert len(streams) == 22 + assert len(streams) == 28 actual_stream_names = {stream.name for stream in streams} expected_stream_names = set( [ "profiles", + "portfolios", "sponsored_display_campaigns", "sponsored_product_campaigns", "sponsored_product_ad_groups", + "sponsored_product_ad_group_suggested_keywords", + "sponsored_product_ad_group_bid_recommendations", "sponsored_product_keywords", "sponsored_product_negative_keywords", + "sponsored_product_campaign_negative_keywords", "sponsored_product_ads", "sponsored_product_targetings", "sponsored_products_report_stream", @@ -111,6 +115,7 @@ def test_source_streams(config): "attribution_report_performance_campaign", "attribution_report_performance_creative", "attribution_report_products", + "sponsored_display_budget_rules" ] ) assert not expected_stream_names - actual_stream_names diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py index 1defa40de5e9..2a9abb1e4d16 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py @@ -16,6 +16,7 @@ def setup_responses( profiles_response=None, + portfolios_response=None, campaigns_response=None, adgroups_response=None, targeting_response=None, @@ -33,6 +34,12 @@ def setup_responses( "https://advertising-api.amazon.com/v2/profiles", body=profiles_response, ) + if portfolios_response: + responses.add( + responses.GET, + "https://advertising-api.amazon.com/v2/portfolios/extended", + body=portfolios_response, + ) if campaigns_response: responses.add( responses.GET, @@ -96,6 +103,24 @@ def test_streams_profile(config, profiles_response): assert record == expected_record +@responses.activate +def test_streams_portfolios(config, profiles_response, portfolios_response): + setup_responses(profiles_response=profiles_response, portfolios_response=portfolios_response) + + source = SourceAmazonAds() + streams = source.streams(config) + + portfolio_stream = get_stream_by_name(streams, "portfolios") + schema = portfolio_stream.get_json_schema() + records = get_all_stream_records(portfolio_stream) + assert len(responses.calls) == 6 + assert len(records) == 8 + expected_records = json.loads(portfolios_response) + for record, expected_record in zip(records, expected_records): + validate(schema=schema, instance=record) + assert record == expected_record + + @responses.activate def test_streams_campaigns_4_vendors(config, profiles_response, campaigns_response): profiles_response = json.loads(profiles_response) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile index 7e21858a3e57..3dd7d468d499 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.5.1 LABEL io.airbyte.name=airbyte/source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index f1940a4937bf..5b753c0a5ac1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -3,8 +3,9 @@ acceptance_tests: spec: tests: - spec_path: "integration_tests/spec.json" + # massively refactored `spec` backward_compatibility_tests_config: - disable_for_version: "1.1.0" + disable_for_version: "1.2.0" connection: tests: - config_path: "secrets/config.json" @@ -15,9 +16,11 @@ acceptance_tests: timeout_seconds: 60 discovery: tests: - - config_path: "secrets/config.json" + - config_path: + "secrets/config.json" + # refactored `spec`, but `app_id` is required for `1.2.0` backward_compatibility_tests_config: - disable_for_version: "0.2.33" + disable_for_version: "1.2.0" basic_read: tests: - config_path: "secrets/config.json" @@ -29,8 +32,10 @@ acceptance_tests: extra_records: yes # TODO: Add records for empty streams - https://github.com/airbytehq/airbyte/issues/21555 empty_streams: - - name: Orders - bypass_reason: "no records" + - name: GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING + bypass_reason: "no access and no data" + - name: GET_ORDER_REPORT_DATA_SHIPPING + bypass_reason: "no access and no data" - name: GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL bypass_reason: "no records" - name: GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL @@ -115,11 +120,15 @@ acceptance_tests: bypass_reason: "no records" - name: ListFinancialEvents bypass_reason: "no records" + - name: ListFinancialEventGroups + bypass_reason: "no records" - name: GET_FBA_REIMBURSEMENTS_DATA bypass_reason: "no records" + - name: GET_XML_BROWSE_TREE_DATA + bypass_reason: "no records" incremental: tests: - - config_path: "secrets/config_old_data.json" + - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_incremental.json" timeout_seconds: 3600 future_state: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_reimbursements_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_reimbursements_data.json index 4eb9a760bca0..c054fe96b7a6 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_reimbursements_data.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_reimbursements_data.json @@ -10,118 +10,62 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "approval-date": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "reimbursement-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "case-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "amazon-order-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "reason": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sku": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "fnsku": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "asin": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "product-name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "condition": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "currency-unit": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "amount-per-unit": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "quantity-reimbursed-cash": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "amount-total": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "quantity-reimbursed-inventory": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "quantity-reimbursed-total": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "original-reimbursement-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "original-reimbursement-type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json index 9f2a3465ecde..55c7437a3fc0 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json @@ -1,5 +1,17 @@ { "streams": [ + { + "stream": { + "name": "OrderItems", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["LastUpdateDate"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["LastUpdateDate"] + }, { "stream": { "name": "Orders", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl index f1a59643fb00..d3778d0803cc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl @@ -1,40 +1,17 @@ -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1673447236350} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1673447236353} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1673447236353} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "0", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1673447236354} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1673447236354} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1673447236354} -{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "I0-RALD-N1UR", "asin": "B0B68NBQ1Y", "price": "5.00", "quantity": "0", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": ""}, "emitted_at": 1673447298434} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20457973011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Produce - en_US", "browseNodeStoreContextName": "Produce - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US", "hasChildren": "true", "childNodes": {"count": "2", "id": ["20457993011", "20457992011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20457993011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Fruits - en_US", "browseNodeStoreContextName": "Fruits - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457993011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Fruits - en_US", "hasChildren": "true", "childNodes": {"count": "4", "id": ["20458032011", "20458033011", "20458034011", "20458035011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458032011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Apples - en_US", "browseNodeStoreContextName": "Apples - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457993011,20458032011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Fruits - en_US,Apples - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458033011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Bananas - en_US", "browseNodeStoreContextName": "Bananas - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457993011,20458033011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Fruits - en_US,Bananas - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458034011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Grapes - en_US", "browseNodeStoreContextName": "Grapes - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457993011,20458034011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Fruits - en_US,Grapes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458035011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Tomatoes - en_US", "browseNodeStoreContextName": "Tomatoes - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457993011,20458035011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Fruits - en_US,Tomatoes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20457992011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Vegetables - en_US", "browseNodeStoreContextName": "Veetables - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457992011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Vegetables - en_US", "hasChildren": "true", "childNodes": {"count": "4", "id": ["20458031011", "20458029011", "20458030011", "20458035011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458031011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Artichokes - en_US", "browseNodeStoreContextName": "Artichokes - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457992011,20458031011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Vegetables - en_US,Artichokes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458029011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Celery - en_US", "browseNodeStoreContextName": "Celery - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457992011,20458029011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Vegetables - en_US,Celery - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458030011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Eggplants - en_US", "browseNodeStoreContextName": "Eggplants - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457992011,20458030011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Vegetables - en_US,Eggplants - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20458035011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Tomatoes - en_US", "browseNodeStoreContextName": "Tomatoes - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904925011,20457973011,20457992011,20458035011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Vanessa - en_US,Produce - en_US,Vegetables - en_US,Tomatoes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "19904924011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Yvonne - en_US", "browseNodeStoreContextName": "Yvonne - en_US", "browsePathById": "19162063011,19162064011,19904871011,19904924011", "browsePathByName": "Yggdrasil,MUC Intro to GT Tooling,Yvonne - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370205} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355625011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Produce - en_US", "browseNodeStoreContextName": "Produce - en_US", "browsePathById": "19162063011,19162064011,20355625011", "browsePathByName": "Yggdrasil,Produce - en_US", "hasChildren": "true", "childNodes": {"count": "2", "id": ["20355629011", "20355628011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355629011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Fruits - en_US", "browseNodeStoreContextName": "Fruits - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355629011", "browsePathByName": "Yggdrasil,Produce - en_US,Fruits - en_US", "hasChildren": "true", "childNodes": {"count": "3", "id": ["20355648011", "20355646011", "20355647011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355648011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Apples - en_US", "browseNodeStoreContextName": "Apples - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355629011,20355648011", "browsePathByName": "Yggdrasil,Produce - en_US,Fruits - en_US,Apples - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355646011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Bananas - en_US", "browseNodeStoreContextName": "Bananas - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355629011,20355646011", "browsePathByName": "Yggdrasil,Produce - en_US,Fruits - en_US,Bananas - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355647011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Grapes - en_US", "browseNodeStoreContextName": "Grapes - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355629011,20355647011", "browsePathByName": "Yggdrasil,Produce - en_US,Fruits - en_US,Grapes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355628011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Vegetables - en_US", "browseNodeStoreContextName": "Vegetables - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US", "hasChildren": "true", "childNodes": {"count": "3", "id": ["20355644011", "20355643011", "20355645011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355644011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Artichokes - en_US", "browseNodeStoreContextName": "Artichokes - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355644011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Artichokes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355643011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Celery - en_US", "browseNodeStoreContextName": "Celery - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355643011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Celery - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355645011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Eggplant - en_US", "browseNodeStoreContextName": "Eggplant - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355645011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Eggplant - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "21354445011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Test2", "browseNodeStoreContextName": "Test2", "browsePathById": "19162063011,19162064011,21354445011", "browsePathByName": "Yggdrasil,Test2", "hasChildren": "true", "childNodes": {"count": "1", "id": ["21354444011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "21354444011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Test1", "browseNodeStoreContextName": "Test1", "browsePathById": "19162063011,19162064011,21354445011,21354444011", "browsePathByName": "Yggdrasil,Test2,Test1", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}}, "emitted_at": 1682444370206} -{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "G3-8N7Y-L93I", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "Issue Description": "'[brand]' is required but not supplied."}, "emitted_at": 1673450133231} -{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "I0-RALD-N1UR", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "Issue Description": "'[brand]' is required but not supplied."}, "emitted_at": 1673450133232} -{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "biM60XKT9qekhLpYdH9-ktjaaCDakRl5bhkXarpufys", "ProcessingStatus": "Open", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}, "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -58.86}, "FinancialEventGroupStart": "2022-08-08T22:51:31Z"}, "emitted_at": 1673450203988} -{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "0", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": ""}, "emitted_at": 1673450377952} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1673450467120} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1673450467121} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1673450467122} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "0", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1673450467122} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1673450467123} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1673450467123} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Active"}, "emitted_at": 1690214254096} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254097} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254098} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254098} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254098} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254099} +{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "I0-RALD-N1UR", "asin": "B0B68NBQ1Y", "price": "5.00", "quantity": "1000", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": ""}, "emitted_at": 1690217648401} +{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "G3-8N7Y-L93I", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "Issue Description": "'[brand]' is required but not supplied."}, "emitted_at": 1690219384531} +{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "I0-RALD-N1UR", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 11, 2022", "Issue Description": "'[brand]' is required but not supplied."}, "emitted_at": 1690219384532} +{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": ""}, "emitted_at": 1690220838938} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127427} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-4052057-4266618", "EarliestShipDate": "2022-07-25T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-26T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-22T20:25:05Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "7.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-09-01T13:16:42Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1691499338977} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "00860509139506", "LastUpdateDate": "2022-09-01T13:16:42Z", "AmazonOrderId": "112-4052057-4266618"}, "emitted_at": 1691499343416} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json index f7a73ecd7d11..7b295ecf757e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json @@ -1,13 +1,24 @@ [ - { - "type": "STREAM", - "stream": { - "stream_state": { - "LastUpdateDate": "2121-07-01T00:00:00Z" - }, - "stream_descriptor": { - "name": "Orders" - } - } - } - ] \ No newline at end of file + { + "type": "STREAM", + "stream": { + "stream_state": { + "LastUpdateDate": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "OrderItems" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "LastUpdateDate": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "Orders" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json index 2cf96f871abd..9c8e32370a3e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json @@ -4,67 +4,109 @@ "connectionSpecification": { "title": "Amazon Seller Partner Spec", "type": "object", + "required": [ + "aws_environment", + "region", + "lwa_app_id", + "lwa_client_secret", + "refresh_token", + "replication_start_date" + ], + "additionalProperties": true, "properties": { - "app_id": { - "title": "App Id", - "description": "Your Amazon App ID", - "airbyte_secret": true, - "order": 0, - "type": "string" - }, "auth_type": { "title": "Auth Type", "const": "oauth2.0", - "order": 1, - "type": "string" - }, - "lwa_app_id": { - "title": "LWA Client Id", - "description": "Your Login with Amazon Client ID.", - "order": 2, + "order": 0, "type": "string" }, - "lwa_client_secret": { - "title": "LWA Client Secret", - "description": "Your Login with Amazon Client Secret.", - "airbyte_secret": true, - "order": 3, - "type": "string" + "aws_environment": { + "title": "AWS Environment", + "description": "Select the AWS Environment.", + "enum": ["PRODUCTION", "SANDBOX"], + "default": "PRODUCTION", + "type": "string", + "order": 1 }, - "refresh_token": { - "title": "Refresh Token", - "description": "The Refresh Token obtained via OAuth flow authorization.", - "airbyte_secret": true, - "order": 4, - "type": "string" + "region": { + "title": "AWS Region", + "description": "Select the AWS Region.", + "enum": [ + "AE", + "AU", + "BE", + "BR", + "CA", + "DE", + "EG", + "ES", + "FR", + "GB", + "IN", + "IT", + "JP", + "MX", + "NL", + "PL", + "SA", + "SE", + "SG", + "TR", + "UK", + "US" + ], + "default": "US", + "type": "string", + "order": 2 }, "aws_access_key": { "title": "AWS Access Key", "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", "airbyte_secret": true, - "order": 5, + "order": 3, "type": "string" }, "aws_secret_key": { "title": "AWS Secret Access Key", "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", "airbyte_secret": true, - "order": 6, + "order": 4, "type": "string" }, "role_arn": { "title": "Role ARN", "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", "airbyte_secret": true, + "order": 5, + "type": "string" + }, + "lwa_app_id": { + "title": "LWA Client Id", + "description": "Your Login with Amazon Client ID.", + "order": 6, + "airbyte_secret": true, + "type": "string" + }, + "lwa_client_secret": { + "title": "LWA Client Secret", + "description": "Your Login with Amazon Client Secret.", + "airbyte_secret": true, "order": 7, "type": "string" }, + "refresh_token": { + "title": "Refresh Token", + "description": "The Refresh Token obtained via OAuth flow authorization.", + "airbyte_secret": true, + "order": 8, + "type": "string" + }, "replication_start_date": { "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "order": 8, + "order": 9, "type": "string" }, "replication_end_date": { @@ -72,17 +114,15 @@ "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", "examples": ["2017-01-25T00:00:00Z"], - "order": 9, + "order": 10, "type": "string" }, "period_in_days": { "title": "Period In Days", + "type": "integer", "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", "default": 90, - "maximum": 90, - "examples": ["1", "10", "30", "60", "90"], - "order": 10, - "type": "integer" + "order": 11 }, "report_options": { "title": "Report Options", @@ -91,7 +131,7 @@ "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" ], - "order": 11, + "order": 12, "type": "string" }, "max_wait_seconds": { @@ -99,7 +139,7 @@ "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", "default": 500, "examples": ["500", "1980"], - "order": 12, + "order": 13, "type": "integer" }, "advanced_stream_options": { @@ -109,89 +149,7 @@ "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" ], - "order": 13, - "type": "string" - }, - "aws_environment": { - "title": "AWSEnvironment", - "description": "An enumeration.", - "enum": ["PRODUCTION", "SANDBOX"], - "type": "string" - }, - "region": { - "title": "AWSRegion", - "description": "An enumeration.", - "enum": [ - "AE", - "AU", - "BE", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "type": "string" - } - }, - "required": [ - "app_id", - "lwa_app_id", - "lwa_client_secret", - "refresh_token", - "replication_start_date", - "aws_environment", - "region" - ], - "additionalProperties": true, - "definitions": { - "AWSEnvironment": { - "title": "AWSEnvironment", - "description": "An enumeration.", - "enum": ["PRODUCTION", "SANDBOX"], - "type": "string" - }, - "AWSRegion": { - "title": "AWSRegion", - "description": "An enumeration.", - "enum": [ - "AE", - "AU", - "BE", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], + "order": 14, "type": "string" } } @@ -201,16 +159,6 @@ "predicate_key": ["auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "app_id": { - "type": "string", - "path_in_connector_config": ["app_id"] - } - } - }, "complete_oauth_output_specification": { "type": "object", "additionalProperties": false, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 5bb3400333db..1e981bf36a47 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -1,20 +1,24 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 1.2.0 + dockerImageTag: 1.5.1 dockerRepository: airbyte/source-amazon-seller-partner + documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner icon: amazonsellerpartner.svg license: MIT name: Amazon Seller Partner registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/requirements.txt b/airbyte-integrations/connectors/source-amazon-seller-partner/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/requirements.txt +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index ba48f6b73f00..2e5ca2d1e6d5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -8,6 +8,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1", "pycryptodome~=3.10", "xmltodict~=0.12"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", ] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json index 477bd4ce9fa0..0c48c8651a11 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json @@ -1,117 +1,63 @@ { - "title" : "FBA Reimbursements Data", - "description" : "FBA Reimbursements Data Reports", - "type" : "object", - "$schema" : "http://json-schema.org/draft-07/schema#", + "title": "FBA Reimbursements Data", + "description": "FBA Reimbursements Data Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": true, - "properties" : { + "properties": { "approval-date": { - "type": [ - "null", - "string" - ] - }, + "type": ["null", "string"] + }, "reimbursement-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "case-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "amazon-order-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "reason": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sku": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "fnsku": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "asin": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "product-name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "condition": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "currency-unit": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "amount-per-unit": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "quantity-reimbursed-cash": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "amount-total": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "quantity-reimbursed-inventory": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "quantity-reimbursed-total": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "original-reimbursement-id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "original-reimbursement-type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json new file mode 100644 index 000000000000..acaa95d38547 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json @@ -0,0 +1,87 @@ +{ + "title": "Flat File All Orders Data Reports (by last update)", + "description": "Flat File All Orders Data by Last Update Date General Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "order-id": { + "type": "string" + }, + "order-item-id": { + "type": ["null", "string"] + }, + "purchase-date": { + "type": "string", + "format": "date-time" + }, + "payments-date": { + "type": "string", + "format": "date-time" + }, + "reporting-date": { + "type": "string", + "format": "date-time" + }, + "promise-date": { + "type": "string", + "format": "date-time" + }, + "days-past-promise": { + "type": ["null", "string"] + }, + "buyer-email": { + "type": ["null", "string"] + }, + "buyer-phone-number": { + "type": ["null", "string"] + }, + "is-business-order": { + "type": ["null", "string"] + }, + "quantity-purchased": { + "type": ["null", "string"] + }, + "quantity-shipped": { + "type": ["null", "string"] + }, + "quantity-to-ship": { + "type": ["null", "string"] + }, + "product-name": { + "type": ["null", "string"] + }, + "purchase-order-number": { + "type": ["null", "string"] + }, + "recipient-name": { + "type": ["null", "string"] + }, + "ship-city": { + "type": ["null", "string"] + }, + "ship-country": { + "type": ["null", "string"] + }, + "ship-postal-code": { + "type": ["null", "string"] + }, + "ship-promotion-discount": { + "type": ["null", "string"] + }, + "ship-service-level": { + "type": ["null", "string"] + }, + "ship-state": { + "type": ["null", "string"] + }, + "shipping-price": { + "type": ["null", "string"] + }, + "shipping-tax": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json new file mode 100644 index 000000000000..baa13fe62730 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json @@ -0,0 +1,192 @@ +{ + "title": "GET_ORDER_REPORT_DATA_SHIPPING", + "description": "GET_ORDER_REPORT_DATA_SHIPPING", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "AmazonOrderID": { + "type": ["null", "string"] + }, + "AmazonSessionID": { + "type": ["null", "string"] + }, + "OrderDate": { + "type": ["null", "string"], + "format": "date" + }, + "OrderPostedDate": { + "type": ["null", "string"], + "format": "date" + }, + "BillingData": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "BuyerEmailAddress": { + "type": ["null", "string"] + }, + "BuyerName": { + "type": ["null", "string"] + }, + "BuyerPhoneNumber": { + "type": ["null", "string"] + } + } + }, + "FulfillmentData": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "FulfillmentMethod": { + "type": ["null", "string"] + }, + "FulfillmentServiceLevel": { + "type": ["null", "string"] + }, + "Address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Name": { + "type": ["null", "string"] + }, + "AddressFieldOne": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "StateOrRegion": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "CountryCode": { + "type": ["null", "string"] + }, + "PhoneNumber": { + "type": ["null", "string"] + } + } + } + } + }, + "IsBusinessOrder": { + "type": ["null", "string"] + }, + "Item": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "AmazonOrderItemCode": { + "type": ["null", "string"] + }, + "SKU": { + "type": ["null", "string"] + }, + "Title": { + "type": ["null", "string"] + }, + "Quantity": { + "type": ["null", "string"] + }, + "ProductTaxCode": { + "type": ["null", "string"] + }, + "ItemPrice": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Component": { + "type": ["array"], + "items": { + "type": ["null", "object"] + }, + "properties": { + "Type": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "currency": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "ItemFees": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Fee": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Type": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "currency": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "Promotion": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "PromotionClaimCode": { + "type": ["null", "string"] + }, + "MerchantPromotionID": { + "type": ["null", "string"] + }, + "Component": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Type": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "currency": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "SignatureConfirmationRecommended": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json index d505714f157e..aaaf073bf2e1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json @@ -776,7 +776,7 @@ } }, "AdhocDisbursementEventList": { - "type": ["null","array"], + "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { @@ -845,7 +845,7 @@ } }, "ChargeRefundEventList": { - "type": ["null","array"], + "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { @@ -859,7 +859,7 @@ } }, "FailedAdhocDisbursementEventList": { - "type": ["null","array"], + "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { @@ -889,7 +889,7 @@ } }, "ValueAddedServiceChargeEventList": { - "type": ["null","array"], + "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json new file mode 100644 index 000000000000..e8fa6b726250 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json @@ -0,0 +1,290 @@ +{ + "title": "Order Items", + "description": "All order items that were updated after a specified date", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "AmazonOrderId": { + "type": ["null", "string"] + }, + "ASIN": { + "type": ["null", "string"] + }, + "OrderItemId": { + "type": ["null", "string"] + }, + "SellerSKU": { + "type": ["null", "string"] + }, + "Title": { + "type": ["null", "string"] + }, + "QuantityOrdered": { + "type": ["null", "integer"] + }, + "ProductInfo": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "NumberOfItems": { + "type": ["null", "string"] + } + } + }, + "QuantityShipped": { + "type": ["null", "integer"] + }, + "PointsGranted": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "PointsNumber": { + "type": ["null", "integer"] + }, + "PointsMonetaryValue": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + } + } + }, + "ItemPrice": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "PromotionIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ItemTax": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "ShippingPrice": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "ShippingTax": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "ShippingDiscount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "ShippingDiscountTax": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "PromotionDiscount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "PromotionDiscountTax": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "ScheduledDeliveryEndDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "ScheduledDeliveryStartDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "CODFee": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "IsGift": { + "type": ["null", "string"] + }, + "ConditionNote": { + "type": ["null", "string"] + }, + "ConditionId": { + "type": ["null", "string"] + }, + "ConditionSubtypeId": { + "type": ["null", "string"] + }, + "CODFeeDiscount": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "TaxCollection": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "Model": { + "type": ["null", "string"] + }, + "ResponsibleParty": { + "type": ["null", "string"] + } + } + }, + "IsTransparency": { + "type": ["null", "boolean"] + }, + "IossNumber": { + "type": ["null", "string"] + }, + "SerialNumberRequired": { + "type": ["null", "boolean"] + }, + "StoreChainStoreId": { + "type": ["null", "string"] + }, + "DeemedResellerCategory": { + "type": ["null", "string"] + }, + "PriceDesignation": { + "type": ["null", "string"] + }, + "BuyerInfo": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "BuyerCustomizedInfo": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CustomizedURL": { + "type": ["null", "string"] + } + } + }, + "GiftMessageText": { + "type": ["null", "string"] + }, + "GiftWrapPrice": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "CurrencyCode": { + "type": ["null", "string"] + }, + "Amount": { + "type": ["null", "string"] + } + } + }, + "GiftWrapLevel": { + "type": ["null", "string"] + } + } + }, + "BuyerRequestedCancel": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "IsBuyerRequestedCancel": { + "type": ["null", "string"] + }, + "BuyerCancelReason": { + "type": ["null", "string"] + } + } + }, + "SerialNumbers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "LastUpdateDate": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json index 6eef79420d93..942eeb73a650 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json @@ -3,6 +3,7 @@ "description": "All orders that were updated after a specified date", "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "seller_id": { "type": "string", @@ -11,6 +12,10 @@ "AmazonOrderId": { "type": ["null", "string"] }, + "BuyerInfo": { + "type": ["null", "object"], + "additionalProperties": true + }, "PurchaseDate": { "type": ["null", "string"] }, @@ -29,6 +34,18 @@ "SalesChannel": { "type": ["null", "string"] }, + "AutomatedShippingSettings": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "HasAutomatedShippingSettings": { + "type": ["null", "boolean"] + } + } + }, + "HasRegulatedItems": { + "type": ["null", "boolean"] + }, "ShipServiceLevel": { "type": ["null", "string"] }, @@ -58,6 +75,9 @@ "type": ["null", "string"] } }, + "IsAccessPointOrder": { + "type": ["null", "boolean"] + }, "IsReplacementOrder": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index ff00b9576e34..5f9bb1c5b7b0 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -6,12 +6,11 @@ import boto3 from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification, SyncMode +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner.auth import AWSAuthenticator, AWSSignature from source_amazon_seller_partner.constants import get_marketplaces -from source_amazon_seller_partner.spec import AmazonSellerPartnerConfig, advanced_auth from source_amazon_seller_partner.streams import ( BrandAnalyticsAlternatePurchaseReports, BrandAnalyticsItemComparisonReports, @@ -37,6 +36,7 @@ FbaSnsForecastReport, FbaSnsPerformanceReport, FbaStorageFeesReports, + FlatFileActionableOrderDataShipping, FlatFileArchivedOrdersDataByOrderDate, FlatFileOpenListingsReports, FlatFileOrdersReports, @@ -55,6 +55,8 @@ MerchantListingsReport, MerchantListingsReportBackCompat, MerchantListingsReports, + OrderItems, + OrderReportDataShipping, Orders, RestockInventoryReports, SellerAnalyticsSalesAndTrafficReports, @@ -68,8 +70,8 @@ class SourceAmazonSellerPartner(AbstractSource): - def _get_stream_kwargs(self, config: AmazonSellerPartnerConfig) -> Mapping[str, Any]: - endpoint, marketplace_id, region = get_marketplaces(config.aws_environment)[config.region] + def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: + endpoint, marketplace_id, region = get_marketplaces(config.get("aws_environment"))[config.get("region")] sts_credentials = self.get_sts_credentials(config) role_creds = sts_credentials["Credentials"] @@ -82,9 +84,9 @@ def _get_stream_kwargs(self, config: AmazonSellerPartnerConfig) -> Mapping[str, ) auth = AWSAuthenticator( token_refresh_endpoint="https://api.amazon.com/auth/o2/token", - client_id=config.lwa_app_id, - client_secret=config.lwa_client_secret, - refresh_token=config.refresh_token, + client_id=config.get("lwa_app_id"), + client_secret=config.get("lwa_client_secret"), + refresh_token=config.get("refresh_token"), host=endpoint.replace("https://", ""), refresh_access_token_headers={"Content-Type": "application/x-www-form-urlencoded"}, ) @@ -92,18 +94,18 @@ def _get_stream_kwargs(self, config: AmazonSellerPartnerConfig) -> Mapping[str, "url_base": endpoint, "authenticator": auth, "aws_signature": aws_signature, - "replication_start_date": config.replication_start_date, + "replication_start_date": config.get("replication_start_date"), "marketplace_id": marketplace_id, - "period_in_days": config.period_in_days, - "report_options": config.report_options, - "max_wait_seconds": config.max_wait_seconds, - "replication_end_date": config.replication_end_date, - "advanced_stream_options": config.advanced_stream_options, + "period_in_days": config.get("period_in_days", 90), + "report_options": config.get("report_options"), + "max_wait_seconds": config.get("max_wait_seconds", 500), + "replication_end_date": config.get("replication_end_date"), + "advanced_stream_options": config.get("advanced_stream_options"), } return stream_kwargs @staticmethod - def get_sts_credentials(config: AmazonSellerPartnerConfig) -> dict: + def get_sts_credentials(config: Mapping[str, Any]) -> dict: """ We can only use a IAM User arn entity or a IAM Role entity. If we use an IAM user arn entity in the connector configuration we need to get the credentials directly from the boto3 sts client @@ -111,12 +113,18 @@ def get_sts_credentials(config: AmazonSellerPartnerConfig) -> dict: :param config: """ - boto3_client = boto3.client("sts", aws_access_key_id=config.aws_access_key, aws_secret_access_key=config.aws_secret_key) - *_, arn_resource = config.role_arn.split(":") + boto3_client = boto3.client( + "sts", aws_access_key_id=config.get("aws_access_key"), aws_secret_access_key=config.get("aws_secret_key") + ) + + if config.get("role_arn") is None: + return boto3_client.get_session_token() + + *_, arn_resource = config.get("role_arn").split(":") if arn_resource.startswith("user"): sts_credentials = boto3_client.get_session_token() elif arn_resource.startswith("role"): - sts_credentials = boto3_client.assume_role(RoleArn=config.role_arn, RoleSessionName="guid") + sts_credentials = boto3_client.assume_role(RoleArn=config.get("role_arn"), RoleSessionName="guid") else: raise ValueError("Invalid ARN, your ARN is not for a user or a role") return sts_credentials @@ -132,9 +140,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Show error message in case of request exception or unexpected response. """ try: - config = AmazonSellerPartnerConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK stream_kwargs = self._get_stream_kwargs(config) - orders_stream = VendorSalesReports(**stream_kwargs) + orders_stream = Orders(**stream_kwargs) next(orders_stream.read_records(sync_mode=SyncMode.full_refresh)) return True, None @@ -143,8 +150,11 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> if isinstance(e, StopIteration): return True, None - # Additional check, since Vendor-ony accounts within Amazon Seller API will not pass the test without this exception + # Additional check, since Vendor-only accounts within Amazon Seller API + # will not pass the test without this exception if "403 Client Error" in str(e): + stream_to_check = VendorSalesReports(**stream_kwargs) + next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) return True, None return False, e @@ -153,9 +163,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - config = AmazonSellerPartnerConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK stream_kwargs = self._get_stream_kwargs(config) - return [ FbaCustomerReturnsReports(**stream_kwargs), FbaAfnInventoryReports(**stream_kwargs), @@ -165,6 +173,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: FbaReplacementsReports(**stream_kwargs), FbaStorageFeesReports(**stream_kwargs), RestockInventoryReports(**stream_kwargs), + FlatFileActionableOrderDataShipping(**stream_kwargs), FlatFileOpenListingsReports(**stream_kwargs), FlatFileOrdersReports(**stream_kwargs), FlatFileOrdersReportsByLastUpdate(**stream_kwargs), @@ -175,6 +184,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: VendorInventoryReports(**stream_kwargs), VendorSalesReports(**stream_kwargs), Orders(**stream_kwargs), + OrderItems(**stream_kwargs), + OrderReportDataShipping(**stream_kwargs), SellerAnalyticsSalesAndTrafficReports(**stream_kwargs), SellerFeedbackReports(**stream_kwargs), BrandAnalyticsMarketBasketReports(**stream_kwargs), @@ -209,20 +220,3 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: LedgerSummaryViewReport(**stream_kwargs), FbaReimbursementsReports(**stream_kwargs), ] - - def spec(self, *args, **kwargs) -> ConnectorSpecification: - """ - Returns the spec for this integration. The spec is a JSON-Schema object describing the required - configurations (e.g: username and password) required to run this integration. - """ - # FIXME: airbyte-cdk does not parse pydantic $ref correctly. This override won't be needed after the fix - schema = AmazonSellerPartnerConfig.schema() - schema["properties"]["aws_environment"] = schema["definitions"]["AWSEnvironment"] - schema["properties"]["region"] = schema["definitions"]["AWSRegion"] - - return ConnectorSpecification( - documentationUrl="https://docs.airbyte.com/integrations/sources/amazon-seller-partner", - changelogUrl="https://docs.airbyte.com/integrations/sources/amazon-seller-partner", - connectionSpecification=schema, - advanced_auth=advanced_auth, - ) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json new file mode 100644 index 000000000000..9c8e32370a3e --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -0,0 +1,200 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/amazon-seller-partner", + "changelogUrl": "https://docs.airbyte.com/integrations/sources/amazon-seller-partner", + "connectionSpecification": { + "title": "Amazon Seller Partner Spec", + "type": "object", + "required": [ + "aws_environment", + "region", + "lwa_app_id", + "lwa_client_secret", + "refresh_token", + "replication_start_date" + ], + "additionalProperties": true, + "properties": { + "auth_type": { + "title": "Auth Type", + "const": "oauth2.0", + "order": 0, + "type": "string" + }, + "aws_environment": { + "title": "AWS Environment", + "description": "Select the AWS Environment.", + "enum": ["PRODUCTION", "SANDBOX"], + "default": "PRODUCTION", + "type": "string", + "order": 1 + }, + "region": { + "title": "AWS Region", + "description": "Select the AWS Region.", + "enum": [ + "AE", + "AU", + "BE", + "BR", + "CA", + "DE", + "EG", + "ES", + "FR", + "GB", + "IN", + "IT", + "JP", + "MX", + "NL", + "PL", + "SA", + "SE", + "SG", + "TR", + "UK", + "US" + ], + "default": "US", + "type": "string", + "order": 2 + }, + "aws_access_key": { + "title": "AWS Access Key", + "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", + "airbyte_secret": true, + "order": 3, + "type": "string" + }, + "aws_secret_key": { + "title": "AWS Secret Access Key", + "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", + "airbyte_secret": true, + "order": 4, + "type": "string" + }, + "role_arn": { + "title": "Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", + "airbyte_secret": true, + "order": 5, + "type": "string" + }, + "lwa_app_id": { + "title": "LWA Client Id", + "description": "Your Login with Amazon Client ID.", + "order": 6, + "airbyte_secret": true, + "type": "string" + }, + "lwa_client_secret": { + "title": "LWA Client Secret", + "description": "Your Login with Amazon Client Secret.", + "airbyte_secret": true, + "order": 7, + "type": "string" + }, + "refresh_token": { + "title": "Refresh Token", + "description": "The Refresh Token obtained via OAuth flow authorization.", + "airbyte_secret": true, + "order": 8, + "type": "string" + }, + "replication_start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2017-01-25T00:00:00Z"], + "order": 9, + "type": "string" + }, + "replication_end_date": { + "title": "End Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", + "examples": ["2017-01-25T00:00:00Z"], + "order": 10, + "type": "string" + }, + "period_in_days": { + "title": "Period In Days", + "type": "integer", + "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", + "default": 90, + "order": 11 + }, + "report_options": { + "title": "Report Options", + "description": "Additional information passed to reports. This varies by report type. Must be a valid json string.", + "examples": [ + "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", + "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" + ], + "order": 12, + "type": "string" + }, + "max_wait_seconds": { + "title": "Max wait time for reports (in seconds)", + "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", + "default": 500, + "examples": ["500", "1980"], + "order": 13, + "type": "integer" + }, + "advanced_stream_options": { + "title": "Advanced Stream Options", + "description": "Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.", + "examples": [ + "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", + "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" + ], + "order": 14, + "type": "string" + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "lwa_app_id": { + "type": "string" + }, + "lwa_client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "lwa_app_id": { + "type": "string", + "path_in_connector_config": ["lwa_app_id"] + }, + "lwa_client_secret": { + "type": "string", + "path_in_connector_config": ["lwa_client_secret"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.py deleted file mode 100644 index 0294e8291329..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.models import AdvancedAuth, AuthFlowType, OAuthConfigSpecification -from pydantic import BaseModel, Field -from source_amazon_seller_partner.constants import AWSEnvironment, AWSRegion - - -class AmazonSellerPartnerConfig(BaseModel): - class Config: - title = "Amazon Seller Partner Spec" - schema_extra = {"additionalProperties": True} - - app_id: str = Field( - description="Your Amazon App ID", - title="App Id", - airbyte_secret=True, - order=0, - ) - - auth_type: str = Field( - default="oauth2.0", - const=True, - order=1, - ) - - lwa_app_id: str = Field( - description="Your Login with Amazon Client ID.", - title="LWA Client Id", - order=2, - ) - - lwa_client_secret: str = Field( - description="Your Login with Amazon Client Secret.", - title="LWA Client Secret", - airbyte_secret=True, - order=3, - ) - - refresh_token: str = Field( - description="The Refresh Token obtained via OAuth flow authorization.", - title="Refresh Token", - airbyte_secret=True, - order=4, - ) - - aws_access_key: str = Field( - None, - description="Specifies the AWS access key used as part of the credentials to authenticate the user.", - title="AWS Access Key", - airbyte_secret=True, - order=5, - ) - - aws_secret_key: str = Field( - None, - description="Specifies the AWS secret key used as part of the credentials to authenticate the user.", - title="AWS Secret Access Key", - airbyte_secret=True, - order=6, - ) - - role_arn: str = Field( - None, - description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - title="Role ARN", - airbyte_secret=True, - order=7, - ) - - replication_start_date: str = Field( - description="UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - title="Start Date", - pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - examples=["2017-01-25T00:00:00Z"], - order=8, - ) - - replication_end_date: str = Field( - None, - description="UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", - title="End Date", - pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - examples=["2017-01-25T00:00:00Z"], - order=9, - ) - - period_in_days: int = Field( - 90, - le=90, - examples=["1", "10", "30", "60", "90"], - description="Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", - order=10, - ) - report_options: str = Field( - None, - description="Additional information passed to reports. This varies by report type. Must be a valid json string.", - examples=['{"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT": {"reportPeriod": "WEEK"}}', '{"GET_SOME_REPORT": {"custom": "true"}}'], - order=11, - ) - max_wait_seconds: int = Field( - 500, - title="Max wait time for reports (in seconds)", - description="Sometimes report can take up to 30 minutes to generate. This will set the limit for how long " - "to wait for a successful report.", - examples=["500", "1980"], - order=12, - ) - - # This field has been introduced because some users experienced a delay on report data update, - # for certain reports the data availability SLA is not explicited this field allows to customize this value. Using this sort of - # options make possible to extend the usage adding more customization per stream. - advanced_stream_options: str = Field( - None, - description="Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.", - examples=['{"GET_SALES_AND_TRAFFIC_REPORT": {"availability_sla_days": 3}}', '{"GET_SOME_REPORT": {"custom": "true"}}'], - order=13, - ) - - aws_environment: AWSEnvironment = Field( - description="Select the AWS Environment.", - title="AWS Environment", - ) - - region: AWSRegion = Field( - description="Select the AWS Region.", - title="AWS Region", - ) - - -advanced_auth = AdvancedAuth( - auth_flow_type=AuthFlowType.oauth2_0, - predicate_key=["auth_type"], - predicate_value="oauth2.0", - oauth_config_specification=OAuthConfigSpecification( - complete_oauth_output_specification={ - "type": "object", - "additionalProperties": False, - "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["refresh_token"]}}, - }, - complete_oauth_server_input_specification={ - "type": "object", - "additionalProperties": False, - "properties": {"lwa_app_id": {"type": "string"}, "lwa_client_secret": {"type": "string"}}, - }, - complete_oauth_server_output_specification={ - "type": "object", - "additionalProperties": False, - "properties": { - "lwa_app_id": {"type": "string", "path_in_connector_config": ["lwa_app_id"]}, - "lwa_client_secret": {"type": "string", "path_in_connector_config": ["lwa_client_secret"]}, - }, - }, - oauth_user_input_from_connector_config_specification={ - "type": "object", - "additionalProperties": False, - "properties": {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - }, - ), -) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 95213116a0c9..d347fa185d87 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -441,6 +441,31 @@ class FbaOrdersReports(ReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA" +class FlatFileActionableOrderDataShipping(ReportsAmazonSPStream): + """ + Field definitions: https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_flat_file_actionable_order_data_shipping + """ + + name = "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING" + + +class OrderReportDataShipping(ReportsAmazonSPStream): + """ + Field definitions: https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping + """ + + name = "GET_ORDER_REPORT_DATA_SHIPPING" + + def parse_document(self, document): + parsed = xmltodict.parse(document, attr_prefix="", cdata_key="value", force_list={"Message"}) + reports = parsed.get("AmazonEnvelope", {}).get("Message", {}) + result = [] + for report in reports: + result.append(report.get("OrderReport", {})) + + return result + + class FbaShipmentsReports(ReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100 @@ -772,6 +797,7 @@ class Orders(IncrementalAmazonSPStream): next_page_token_field = "NextToken" page_size_field = "MaxResultsPerPage" default_backoff_time = 60 + use_cache = True def path(self, **kwargs) -> str: return f"orders/{ORDERS_API_VERSION}/orders" @@ -796,6 +822,80 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return self.default_backoff_time +class OrderItems(AmazonSPStream, ABC): + """ + API docs: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems + API model: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#orderitemslist + """ + + name = "OrderItems" + primary_key = "OrderItemId" + cursor_field = "LastUpdateDate" + parent_cursor_field = "LastUpdateDate" + next_page_token_field = "NextToken" + stream_slice_cursor_field = "AmazonOrderId" + page_size_field = None + default_backoff_time = 10 + default_stream_slice_delay_time = 1 + cached_state: Dict = {} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.stream_kwargs = kwargs + + def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + return f"orders/{ORDERS_API_VERSION}/orders/{stream_slice[self.stream_slice_cursor_field]}/orderItems" + + def request_params( + self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + if next_page_token: + return dict(next_page_token) + return {} + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + orders = Orders(**self.stream_kwargs) + for order_record in orders.read_records(sync_mode=SyncMode.incremental, stream_state=stream_state): + self.cached_state[self.parent_cursor_field] = order_record[self.parent_cursor_field] + self.logger.info(f"OrderItems stream slice for order {order_record[self.stream_slice_cursor_field]}") + time.sleep(self.default_stream_slice_delay_time) + yield { + self.stream_slice_cursor_field: order_record[self.stream_slice_cursor_field], + self.parent_cursor_field: order_record[self.parent_cursor_field], + } + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_benchmark = self.cached_state[self.parent_cursor_field] + if current_stream_state.get(self.parent_cursor_field): + return {self.parent_cursor_field: max(latest_benchmark, current_stream_state[self.parent_cursor_field])} + return {self.parent_cursor_field: latest_benchmark} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + stream_data = response.json() + next_page_token = stream_data.get("payload").get(self.next_page_token_field) + if next_page_token: + return {self.next_page_token_field: next_page_token} + + def backoff_time(self, response: requests.Response) -> Optional[float]: + rate_limit = response.headers.get("x-amzn-RateLimit-Limit", 0) + if rate_limit: + return 1 / float(rate_limit) + else: + return self.default_backoff_time + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + order_items_list = response.json().get(self.data_field, {}) + self.logger.info(f"order_items_list efim {order_items_list}") + if order_items_list.get(self.next_page_token_field) is None: + self.cached_state[self.parent_cursor_field] = stream_slice[self.parent_cursor_field] + for order_item in order_items_list.get(self.name, []): + order_item[self.cursor_field] = stream_slice.get(self.parent_cursor_field) + order_item[self.stream_slice_cursor_field] = order_items_list.get(self.stream_slice_cursor_field) + yield order_item + + class LedgerDetailedViewReports(IncrementalReportsAmazonSPStream): """ API docs: https://developer-docs.amazon.com/sp-api/docs/report-type-values diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py new file mode 100644 index 000000000000..4e5d1364ed99 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +import requests +from source_amazon_seller_partner.auth import AWSSignature +from source_amazon_seller_partner.streams import OrderItems + +list_order_items_payload_data = { + "payload": { + "OrderItems": [ + { + "ProductInfo": { + "NumberOfItems": "1" + }, + "IsGift": "false", + "BuyerInfo": {}, + "QuantityShipped": 0, + "IsTransparency": False, + "QuantityOrdered": 1, + "ASIN": "AKDDKDKD", + "SellerSKU": "AAA-VPx3-AMZ", + "Title": "Example product", + "OrderItemId": "88888888888" + } + ], + "AmazonOrderId": "111-0000000-2222222" + } +} + + +@pytest.fixture +def order_items_stream(): + def _internal(): + aws_signature = AWSSignature( + service="execute-api", + aws_access_key_id="AccessKeyId", + aws_secret_access_key="SecretAccessKey", + aws_session_token="SessionToken", + region="US", + ) + stream = OrderItems( + url_base="https://test.url", + aws_signature=aws_signature, + replication_start_date="2023-08-08T00:00:00Z", + replication_end_date=None, + marketplace_id="id", + authenticator=None, + period_in_days=0, + report_options=None, + advanced_stream_options=None, + max_wait_seconds=500, + ) + return stream + + return _internal + + +def test_order_items_stream_initialization(order_items_stream): + stream = order_items_stream() + assert stream._replication_start_date == "2023-08-08T00:00:00Z" + assert stream._replication_end_date is None + assert stream.marketplace_id == "id" + + +def test_order_items_stream_next_token(mocker, order_items_stream): + response = requests.Response() + token = "111111111" + expected = {"NextToken": token} + mocker.patch.object(response, "json", return_value={"payload": expected}) + assert expected == order_items_stream().next_page_token(response) + + mocker.patch.object(response, "json", return_value={"payload": {}}) + if order_items_stream().next_page_token(response) is not None: + assert False + + +def test_order_items_stream_parse_response(mocker, order_items_stream): + response = requests.Response() + mocker.patch.object(response, "json", return_value=list_order_items_payload_data) + + stream = order_items_stream() + stream.cached_state["LastUpdateDate"] = "2023-08-07T00:00:00Z" + parsed = stream.parse_response(response, stream_slice={"AmazonOrderId": "111-0000000-2222222", "LastUpdateDate": "2023-08-08T00:00:00Z"}) + + for record in parsed: + assert record["AmazonOrderId"] == "111-0000000-2222222" + assert record["OrderItemId"] == "88888888888" + assert record["SellerSKU"] == "AAA-VPx3-AMZ" + assert record["ASIN"] == "AKDDKDKD" + assert record["Title"] == "Example product" + assert record["QuantityOrdered"] == 1 + assert record["QuantityShipped"] == 0 + assert record["BuyerInfo"] == {} + assert record["IsGift"] == "false" + assert record["ProductInfo"] == {"NumberOfItems": "1"} + + assert stream.cached_state["LastUpdateDate"] == "2023-08-08T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index 241b1ccf33c5..4c24d0fccd65 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -5,11 +5,9 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner from source_amazon_seller_partner.source import boto3 -from source_amazon_seller_partner.spec import AmazonSellerPartnerConfig @pytest.fixture @@ -19,19 +17,19 @@ def connector_source(): @pytest.fixture def connector_config(): - return AmazonSellerPartnerConfig( - replication_start_date="2017-01-25T00:00:00Z", - replication_end_date="2017-02-25T00:00:00Z", - refresh_token="Atzr|IwEBIP-abc123", - app_id="amzn1.sp.solution.2cfa6ca8-2c35-4f01-a984-9fb65c7d84f2", - lwa_app_id="amzn1.application-oa2-client.abc123", - lwa_client_secret="abc123", - aws_access_key="aws_access_key", - aws_secret_key="aws_secret_key", - role_arn="arn:aws:iam::123456789098:role/some-role", - aws_environment="SANDBOX", - region="US", - ) + return { + "replication_start_date": "2017-01-25T00:00:00Z", + "replication_end_date": "2017-02-25T00:00:00Z", + "refresh_token": "Atzr|IwEBIP-abc123", + "app_id": "amzn1.sp.solution.2cfa6ca8-2c35-123-456-78910", + "lwa_app_id": "amzn1.application-oa2-client.abc123", + "lwa_client_secret": "abc123", + "aws_access_key": "aws_access_key", + "aws_secret_key": "aws_secret_key", + "role_arn": "arn:aws:iam::123456789098:role/some-role", + "aws_environment": "SANDBOX", + "region": "US", + } @pytest.fixture @@ -54,10 +52,6 @@ def mock_boto_client(mocker, sts_credentials): return boto_client -def test_spec(connector_source): - assert isinstance(connector_source.spec(), ConnectorSpecification) - - def test_streams(connector_source, connector_config, mock_boto_client): for stream in connector_source.streams(connector_config): assert isinstance(stream, Stream) @@ -65,7 +59,7 @@ def test_streams(connector_source, connector_config, mock_boto_client): @pytest.mark.parametrize("arn", ("arn:aws:iam::123456789098:user/some-user", "arn:aws:iam::123456789098:role/some-role")) def test_stream_with_good_iam_arn_value(mock_boto_client, connector_source, connector_config, arn): - connector_config.role_arn = arn + connector_config["role_arn"] = arn result = connector_source.get_sts_credentials(connector_config) assert "Credentials" in result if "user" in arn: @@ -75,7 +69,7 @@ def test_stream_with_good_iam_arn_value(mock_boto_client, connector_source, conn def test_stream_with_bad_iam_arn_value(connector_source, connector_config, mock_boto_client): - connector_config.role_arn = "bad-arn" + connector_config["role_arn"] = "bad-arn" with pytest.raises(ValueError) as e: connector_source.get_sts_credentials(connector_config) assert "Invalid" in e.message diff --git a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml index 460e412196c1..a62580bedfa1 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 983fd355-6bf3-4709-91b5-37afa391eeb6 dockerImageTag: 0.1.0 dockerRepository: airbyte/source-amazon-sqs + documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-sqs githubIssueLabel: source-amazon-sqs icon: awssqs.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-sqs + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amazon-sqs/requirements.txt b/airbyte-integrations/connectors/source-amazon-sqs/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/requirements.txt +++ b/airbyte-integrations/connectors/source-amazon-sqs/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-amazon-sqs/setup.py b/airbyte-integrations/connectors/source-amazon-sqs/setup.py index 2aaad0d7de7f..3ca84d0f1041 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/setup.py +++ b/airbyte-integrations/connectors/source-amazon-sqs/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "boto3"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "moto[sqs, iam]"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "moto[sqs, iam]"] setup( name="source_amazon_sqs", diff --git a/airbyte-integrations/connectors/source-amplitude/README.md b/airbyte-integrations/connectors/source-amplitude/README.md index 2318166113d0..17b153e7bbe2 100644 --- a/airbyte-integrations/connectors/source-amplitude/README.md +++ b/airbyte-integrations/connectors/source-amplitude/README.md @@ -50,7 +50,9 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat #### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` To run your integration tests with Docker, run: ``` ./acceptance-test-docker.sh diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml index 3aac2082e2b9..f2d615d750d4 100644 --- a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml @@ -36,6 +36,7 @@ acceptance_tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 3600 full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-amplitude/metadata.yaml b/airbyte-integrations/connectors/source-amplitude/metadata.yaml index bb14dc1e0772..be9ecbaf14e7 100644 --- a/airbyte-integrations/connectors/source-amplitude/metadata.yaml +++ b/airbyte-integrations/connectors/source-amplitude/metadata.yaml @@ -22,4 +22,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amplitude/requirements.txt b/airbyte-integrations/connectors/source-amplitude/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-amplitude/requirements.txt +++ b/airbyte-integrations/connectors/source-amplitude/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-amplitude/setup.py b/airbyte-integrations/connectors/source-amplitude/setup.py index 727c043aa667..ad70b2ff395f 100644 --- a/airbyte-integrations/connectors/source-amplitude/setup.py +++ b/airbyte-integrations/connectors/source-amplitude/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml b/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml index 94c41951fadf..adabb7f021aa 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml @@ -5,7 +5,7 @@ definitions: type: RecordSelector extractor: type: DpathExtractor - field_path: [ "{{ parameters.get('data_field') }}" ] + field_path: ["{{ parameters.get('data_field') }}"] requester: type: HttpRequester url_base: "https://{{'analytics.eu.' if config['data_region'] == 'EU Residency Server' else '' }}amplitude.com/api" @@ -17,13 +17,13 @@ definitions: error_handler: type: DefaultErrorHandler response_filters: - - http_codes: [ 400 ] + - http_codes: [400] action: FAIL error_message: The file size of the exported data is too large. Shorten the time ranges and try again. The limit size is 4GB. - - http_codes: [ 404 ] + - http_codes: [404] action: IGNORE error_message: No data collected - - http_codes: [ 504 ] + - http_codes: [504] action: FAIL error_message: The amount of data is large causing a timeout. For large amounts of data, the Amazon S3 destination is recommended. @@ -119,7 +119,6 @@ definitions: primary_key: "date" path: "/2/users" - streams: - "#/definitions/annotations_stream" - "#/definitions/cohorts_stream" diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml index 8b14b1ca9f5e..ee950f5e6658 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml @@ -4,9 +4,9 @@ connectionSpecification: title: Amplitude Spec type: object required: - - api_key - - secret_key - - start_date + - api_key + - secret_key + - start_date additionalProperties: true properties: data_region: @@ -14,33 +14,37 @@ connectionSpecification: title: Data region description: Amplitude data region server enum: - - Standard Server - - EU Residency Server + - Standard Server + - EU Residency Server default: Standard Server api_key: type: string title: API Key - description: Amplitude API Key. See the setup + description: + Amplitude API Key. See the setup guide for more information on how to obtain this key. airbyte_secret: true secret_key: type: string title: Secret Key - description: Amplitude Secret Key. See the setup + description: + Amplitude Secret Key. See the setup guide for more information on how to obtain this key. airbyte_secret: true start_date: type: string title: Replication Start Date pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - description: UTC date and time in the format 2021-01-25T00:00:00Z. Any data + description: + UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated. examples: - - "2021-01-25T00:00:00Z" + - "2021-01-25T00:00:00Z" request_time_range: type: integer title: Request time range - description: According to Considerations too + description: + According to Considerations too big time range in request can cause a timeout error. In this case, set shorter time interval in hours. default: 24 minimum: 1 diff --git a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile index 3c25c0ce7cbd..31496bd68d70 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile +++ b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-apify-dataset diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json index bffccc95487a..d736a2ebd156 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json @@ -6,7 +6,14 @@ "destination_sync_mode": "overwrite", "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object" + "type": "object", + "properties": { + "data": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": true } } ] diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json index 572bbe92137d..02e5fe573ea0 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json @@ -7,7 +7,17 @@ "name": "DatasetItems", "supported_sync_modes": ["full_refresh"], "destination_sync_mode": "overwrite", - "json_schema": {} + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "data": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": true + } } } ] diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index 057ccd5b0faa..e3ab131aa173 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 - dockerImageTag: 0.1.11 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-apify-dataset githubIssueLabel: source-apify-dataset icon: apify.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-apify-dataset/requirements.txt b/airbyte-integrations/connectors/source-apify-dataset/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/requirements.txt +++ b/airbyte-integrations/connectors/source-apify-dataset/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-apify-dataset/setup.py b/airbyte-integrations/connectors/source-apify-dataset/setup.py index e88d0344d42f..ee122749a6ac 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/setup.py +++ b/airbyte-integrations/connectors/source-apify-dataset/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "apify-client~=0.0.1"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py index c916d4f5ba74..68922a9f80a1 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py @@ -87,6 +87,12 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: json_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "properties": { + "data": { + "type": "object", + "additionalProperties": True, + } + }, } return AirbyteCatalog( @@ -133,6 +139,6 @@ def read( yield AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( - stream=DATASET_ITEMS_STREAM_NAME, data=data, emitted_at=int(datetime.now().timestamp()) * 1000 + stream=DATASET_ITEMS_STREAM_NAME, data={"data": data}, emitted_at=int(datetime.now().timestamp()) * 1000 ), ) diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json index 21ad77968932..62e8578d611d 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json @@ -5,7 +5,7 @@ "title": "Apify Dataset Spec", "type": "object", "required": ["datasetId"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "datasetId": { "type": "string", diff --git a/airbyte-integrations/connectors/source-appfollow/Dockerfile b/airbyte-integrations/connectors/source-appfollow/Dockerfile index 8b333c731db8..52753990acb8 100644 --- a/airbyte-integrations/connectors/source-appfollow/Dockerfile +++ b/airbyte-integrations/connectors/source-appfollow/Dockerfile @@ -1,38 +1,16 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base +FROM python:3.9-slim +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ COPY source_appfollow ./source_appfollow +COPY main.py ./ +COPY setup.py ./ +RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-appfollow diff --git a/airbyte-integrations/connectors/source-appfollow/README.md b/airbyte-integrations/connectors/source-appfollow/README.md index 30fbb84dbf2f..7d41040d954f 100644 --- a/airbyte-integrations/connectors/source-appfollow/README.md +++ b/airbyte-integrations/connectors/source-appfollow/README.md @@ -1,35 +1,10 @@ # Appfollow Source -This is the repository for the Appfollow source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/appfollow). +This is the repository for the Appfollow configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/appfollow). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,7 +14,7 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/appfollow) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/appfollow) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_appfollow/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,14 +22,6 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source appfollow test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python3 main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appfollow:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-appfollow:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-appfollow/__init__.py b/airbyte-integrations/connectors/source-appfollow/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml b/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml index ae277c417719..1f7f35bda6ab 100644 --- a/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml @@ -1,22 +1,30 @@ -# # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# # for more information about how to configure these tests - connector_image: airbyte/source-appfollow:dev -tests: +test_strictness_level: low +acceptance_tests: spec: - - spec_path: "source_appfollow/spec.yaml" -# Enable these once we get appfollow credentials -# connection: -# - config_path: "secrets/config.json" -# status: "succeed" -# - config_path: "integration_tests/invalid_config.json" -# status: "failed" -# discovery: -# - config_path: "secrets/config.json" -# basic_read: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# empty_streams: [] -# full_refresh: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - spec_path: "source_appfollow/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.1" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "ratings" + bypass_reason: "Endpoint not allowed in integration account" + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/abnormal_state.json similarity index 100% rename from airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/abnormal_state.json rename to airbyte-integrations/connectors/source-appfollow/integration_tests/abnormal_state.json diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py index 82823254d266..d49b55882333 100644 --- a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py @@ -10,5 +10,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json index e4d63c32e72d..91671831791f 100644 --- a/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json @@ -1,23 +1,45 @@ { "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "app_collections", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "app_lists", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stat_reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "ratings", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "ext_id": { - "type": "string" - }, - "cid": { - "type": "string" - }, - "country": { - "type": "string" - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json index f2311f73ce28..92a71d900e59 100644 --- a/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json @@ -1,6 +1,3 @@ { - "ext_id": "FIXME", - "cid": "FIXME", - "api_secret": "FIXME", - "country": "FIXME" + "api_key": "invalid-api-key" } diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json index f2311f73ce28..4bf6eab50a98 100644 --- a/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json @@ -1,6 +1,3 @@ { - "ext_id": "FIXME", - "cid": "FIXME", - "api_secret": "FIXME", - "country": "FIXME" + "api_key": "wrong-api=key" } diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_state.json similarity index 100% rename from airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/sample_state.json rename to airbyte-integrations/connectors/source-appfollow/integration_tests/sample_state.json diff --git a/airbyte-integrations/connectors/source-appfollow/metadata.yaml b/airbyte-integrations/connectors/source-appfollow/metadata.yaml index 5019b31ff29c..e65309bd6e97 100644 --- a/airbyte-integrations/connectors/source-appfollow/metadata.yaml +++ b/airbyte-integrations/connectors/source-appfollow/metadata.yaml @@ -1,20 +1,33 @@ data: + allowedHosts: + hosts: + - https://api.appfollow.io + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: b4375641-e270-41d3-9c20-4f9cecad87a8 - dockerImageTag: 0.1.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-appfollow githubIssueLabel: source-appfollow icon: appfollow.svg license: MIT name: Appfollow - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2023-08-10 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/appfollow tags: - - language:python + - language:lowcode + releases: + breakingChanges: + 1.0.0: + message: "Remove spec parameters and ingest all apps" + upgradeDeadline: "2023-08-21" + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-appfollow/requirements.txt b/airbyte-integrations/connectors/source-appfollow/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-appfollow/requirements.txt +++ b/airbyte-integrations/connectors/source-appfollow/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-appfollow/sample_files/config.json b/airbyte-integrations/connectors/source-appfollow/sample_files/config.json deleted file mode 100644 index f2311f73ce28..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/sample_files/config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "ext_id": "FIXME", - "cid": "FIXME", - "api_secret": "FIXME", - "country": "FIXME" -} diff --git a/airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json deleted file mode 100644 index e4d63c32e72d..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "ratings", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "ext_id": { - "type": "string" - }, - "cid": { - "type": "string" - }, - "country": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json deleted file mode 100644 index 4eb2229ced98..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "ext_id": "bar" -} diff --git a/airbyte-integrations/connectors/source-appfollow/setup.py b/airbyte-integrations/connectors/source-appfollow/setup.py index 82d45ecc2d29..c808dd5682e9 100644 --- a/airbyte-integrations/connectors/source-appfollow/setup.py +++ b/airbyte-integrations/connectors/source-appfollow/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "requests_mock~=1.9"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "requests_mock~=1.9"] setup( name="source_appfollow", diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/manifest.yaml b/airbyte-integrations/connectors/source-appfollow/source_appfollow/manifest.yaml new file mode 100644 index 000000000000..a48875b2b87f --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/manifest.yaml @@ -0,0 +1,136 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + defined_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.path_extractor }}"] + + requester: + type: HttpRequester + url_base: "https://api.appfollow.io/api/v2" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: X-AppFollow-API-Token + api_token: "{{ config['api_key'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + $parameters: + path: "/account/users" + + app_collections_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/defined_selector" + name: "app_collections" + $parameters: + path: "/account/apps" + path_extractor: apps + + apps_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/app_collections_stream" + parent_key: id + partition_field: app_collection_id + + app_lists_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/defined_selector" + requester: + $ref: "#/definitions/requester" + request_parameters: + apps_id: "{{ stream_slice.app_collection_id }}" + partition_router: + $ref: "#/definitions/apps_partition_router" + transformations: + - type: AddFields + fields: + - path: ["app_collection_id"] + value: "{{ stream_slice.app_collection_id }}" + - type: AddFields + fields: + - path: ["ext_id"] + value: "{{ record.app.ext_id }}" + name: "app_lists" + $parameters: + path: "/account/apps/app" + path_extractor: apps_app + + ext_id_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/app_lists_stream" + parent_key: ext_id + partition_field: ext_id + + stat_reviews_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + ext_id: "{{ stream_slice.ext_id }}" + partition_router: + $ref: "#/definitions/ext_id_partition_router" + name: "stat_reviews" + $parameters: + path: "/reviews/stats" + + ratings_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + ext_id: "{{ stream_slice.ext_id }}" + error_handler: + response_filters: + - http_codes: [400] + action: IGNORE + partition_router: + $ref: "#/definitions/ext_id_partition_router" + name: "ratings" + $parameters: + path: "/meta/ratings" + +streams: + - "#/definitions/users_stream" + - "#/definitions/app_collections_stream" + - "#/definitions/app_lists_stream" + - "#/definitions/stat_reviews_stream" + - "#/definitions/ratings_stream" + +check: + type: CheckStream + stream_names: + - "users" diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/app_collections.json b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/app_collections.json new file mode 100644 index 000000000000..3e3721a7f9bd --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/app_collections.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "title_normalized": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"] + }, + "count_apps": { + "type": ["null", "integer"] + }, + "languages": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "countries": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/app_lists.json b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/app_lists.json new file mode 100644 index 000000000000..d6f74ee9540d --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/app_lists.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "app": { + "type": ["null", "object"], + "additionalProperties": true + }, + "has_reply_integration": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "string"] + }, + "is_favorite": { + "type": ["null", "integer"] + }, + "count_whatsnew": { + "type": ["null", "integer"] + }, + "store": { + "type": ["null", "string"] + }, + "watch_url": { + "type": ["null", "string"] + }, + "ext_id": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "count_reviews": { + "type": ["null", "integer"] + }, + "app_collection_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json index af210d2a4000..3860d7a4fe2e 100644 --- a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "ratings": { "list": { diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/stat_reviews.json b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/stat_reviews.json new file mode 100644 index 000000000000..d233ef79fd66 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/stat_reviews.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "store": { + "type": ["null", "string"] + }, + "reviews_stat": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "store": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "reviews": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "replies": { + "type": ["null", "integer"] + } + } + }, + "ext_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/users.json b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/users.json new file mode 100644 index 000000000000..1eda654f315c --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/users.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py b/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py index 23c5f09e043f..bfbb9335fdea 100644 --- a/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py @@ -2,107 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from requests.auth import HTTPBasicAuth +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -logger = logging.getLogger("airbyte") +WARNING: Do not modify this file. +""" -# Basic full refresh stream - -class AppfollowStream(HttpStream, ABC): - - url_base = "https://api.appfollow.io/" - - def __init__(self, ext_id: str, cid: str, **kwargs): - super().__init__(**kwargs) - self.ext_id = ext_id - self.cid = cid - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - Include common app and client parameters - """ - return {"ext_id": self.ext_id, "cid": self.cid} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - response_json = response.json() - yield response_json - - -class Ratings(AppfollowStream): - """ - Ratings is a stream that pulls app ratings data from the Appfollow API. - """ - - primary_key = None - - def __init__(self, country: str, **kwargs): - super().__init__(**kwargs) - self.country = country - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params["country"] = self.country - return params - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "ratings" - - -# Source - - -class SourceAppfollow(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - A connection check to validate that the user-provided config can be used to connect to the underlying API - - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - logger.info("Checking Appfollow API connection...") - try: - ext_id = config["ext_id"] - cid = config["cid"] - api_secret = config["api_secret"] - response = requests.get( - f"https://api.appfollow.io/ratings?ext_id={ext_id}&cid={cid}", auth=HTTPBasicAuth(api_secret, api_secret) - ) - if response.status_code == 200: - return True, None - else: - return False, "Invalid Appfollow API credentials" - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - Appfollow streams - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = BasicHttpAuthenticator(username=config["api_secret"], password=config["api_secret"]) - args = {"ext_id": config["ext_id"], "cid": config["cid"]} - return [Ratings(authenticator=auth, country=config["country"], **args)] +# Declarative Source +class SourceAppfollow(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml b/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml index f2fa2d7b82d1..e2733582c957 100644 --- a/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml @@ -4,26 +4,11 @@ connectionSpecification: title: Appfollow Spec type: object required: - - ext_id - - country - - cid - - api_secret + - api_key additionalProperties: true properties: - ext_id: - type: string - title: app external id - description: for App Store — this is 9-10 digits identification number; for Google Play — this is bundle name; - cid: - type: string - title: client id - description: client id provided by Appfollow api_secret: type: string - title: api secret - description: api secret provided by Appfollow + title: API Key + description: API Key provided by Appfollow airbyte_secret: true - country: - type: string - title: country - description: getting data by Country diff --git a/airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py b/airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py deleted file mode 100644 index 0c7dbeed79e9..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_appfollow.source import SourceAppfollow - - -def test_check_connection(mocker, requests_mock): - source = SourceAppfollow() - logger_mock, config_mock = MagicMock(), MagicMock() - - # success - requests_mock.get("https://api.appfollow.io/ratings", json={"data": "pong!"}) - assert source.check_connection(logger_mock, config_mock) == (True, None) - - # failure - requests_mock.get("https://api.appfollow.io/ratings", status_code=500) - ok, err = source.check_connection(logger_mock, config_mock) - assert (ok, type(err)) == (False, str) - - -def test_streams(mocker): - source = SourceAppfollow() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py deleted file mode 100644 index 39e82f7342be..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_appfollow.source import AppfollowStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - def __init__(self): - self.ext_id = "00000" - self.cid = "000000" - - mocker.patch.object(AppfollowStream, "__init__", __init__) - mocker.patch.object(AppfollowStream, "path", "v0/example_endpoint") - mocker.patch.object(AppfollowStream, "primary_key", "test_primary_key") - mocker.patch.object(AppfollowStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = AppfollowStream() - inputs = {"stream_state": "test_stream_state"} - expected_params = {"ext_id": "00000", "cid": "000000"} - assert stream.request_params(**inputs) == expected_params - - -def test_parse_response(patch_base_class): - stream = AppfollowStream() - mock_response = MagicMock() - inputs = {"stream_state": "test_stream_state", "response": mock_response} - expected_parsed_object = mock_response.json() - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = AppfollowStream() - inputs = {"stream_state": "test_stream_state"} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = AppfollowStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = AppfollowStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = AppfollowStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-apple-search-ads/Dockerfile b/airbyte-integrations/connectors/source-apple-search-ads/Dockerfile index 2f1b015660a1..f82a8effc8e4 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-apple-search-ads/Dockerfile @@ -34,5 +34,5 @@ COPY source_apple_search_ads ./source_apple_search_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-apple-search-ads diff --git a/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/configured_catalog.json index 74c4977fc8ba..e4b552b66c61 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-apple-search-ads/integration_tests/configured_catalog.json @@ -37,10 +37,7 @@ "name": "campaigns_report_daily", "source_defined_cursor": true, "source_defined_primary_key": [["date"], ["campaignId"]], - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "destination_sync_mode": "append", "sync_mode": "incremental" @@ -52,10 +49,7 @@ "name": "adgroups_report_daily", "source_defined_cursor": true, "source_defined_primary_key": [["date"], ["adGroupId"]], - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "destination_sync_mode": "append", "sync_mode": "incremental" @@ -67,10 +61,7 @@ "name": "keywords_report_daily", "source_defined_cursor": true, "source_defined_primary_key": [["date"], ["keywordId"]], - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "destination_sync_mode": "append", "sync_mode": "incremental" diff --git a/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml b/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml index 98fbc20ceb44..1406f248390e 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-apple-search-ads/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: e59c8416-c2fa-4bd3-9e95-52677ea281c1 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-apple-search-ads githubIssueLabel: source-apple-search-ads icon: apple.svg @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-apple-search-ads/requirements.txt b/airbyte-integrations/connectors/source-apple-search-ads/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/requirements.txt +++ b/airbyte-integrations/connectors/source-apple-search-ads/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-apple-search-ads/setup.py b/airbyte-integrations/connectors/source-apple-search-ads/setup.py index f4070c78949d..a66de217e0fd 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/setup.py +++ b/airbyte-integrations/connectors/source-apple-search-ads/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/manifest.yaml b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/manifest.yaml index 8df0dc8934a1..6a2f40ff878e 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/manifest.yaml +++ b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/manifest.yaml @@ -15,7 +15,7 @@ definitions: X-AP-Context: orgId={{ config.org_id }} error_handler: response_filters: - - http_codes: [ 500, 429 ] + - http_codes: [500, 429] action: RETRY backoff_strategies: - type: "ExponentialBackoffStrategy" @@ -80,7 +80,7 @@ definitions: parent_stream_configs: - stream: "#/definitions/adgroups_stream" parent_key: "id" - partition_field: "adgroup_id" + partition_field: "adgroup_id" requester: $ref: "#/definitions/retriever/requester" path: "/campaigns/{{ stream_slice.parent_slice.campaign_id }}/adgroups/{{ stream_slice.adgroup_id }}/targetingkeywords" @@ -94,7 +94,6 @@ definitions: cursor_granularity: "P1D" cursor_field: "date" lookback_window: "P30D" - datetime_format: "%Y-%m-%d" report_stream: selector: @@ -118,12 +117,12 @@ definitions: granularity: "{{ parameters.granularity }}" selector: '{ "orderBy": [ - { - "field": "countryOrRegion", - "sortOrder": "ASCENDING" - } + { + "field": "countryOrRegion", + "sortOrder": "ASCENDING" + } ] - }' + }' groupBy: "[ 'countryOrRegion' ]" campaigns_report_daily_stream: @@ -140,9 +139,9 @@ definitions: transformations: - type: AddFields fields: - - path: [ "campaignId" ] + - path: ["campaignId"] value: "{{ record.metadata.campaignId }}" - - path: [ "date" ] + - path: ["date"] value: "{{ stream_slice.start_time }}" adgroups_report_daily_stream: @@ -165,9 +164,9 @@ definitions: transformations: - type: AddFields fields: - - path: [ "adGroupId" ] + - path: ["adGroupId"] value: "{{ record.metadata.adGroupId }}" - - path: [ "date" ] + - path: ["date"] value: "{{ stream_slice.start_time }}" keywords_report_daily_stream: @@ -192,19 +191,18 @@ definitions: response_filters: - predicate: "{{ 'CAMPAIGN DOES NOT CONTAIN KEYWORD' in response.error.errors[0].message }}" action: IGNORE - - http_codes: [ 500, 429 ] + - http_codes: [500, 429] action: RETRY backoff_strategies: - type: "ExponentialBackoffStrategy" transformations: - type: AddFields fields: - - path: [ "keywordId" ] + - path: ["keywordId"] value: "{{ record.metadata.keywordId }}" - - path: [ "date" ] + - path: ["date"] value: "{{ stream_slice.start_time }}" - streams: - "#/definitions/campaigns_stream" - "#/definitions/adgroups_stream" diff --git a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/adgroups.json b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/adgroups.json index fc2ad9992c39..a5ad5002513b 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/adgroups.json +++ b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/adgroups.json @@ -104,10 +104,7 @@ "type": "null" }, "minAge": { - "type": [ - "integer", - "null" - ] + "type": ["integer", "null"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/campaigns.json index 1b439e3cb102..87de15854d47 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-apple-search-ads/source_apple_search_ads/schemas/campaigns.json @@ -78,16 +78,10 @@ "type": "string" }, "clientName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "orderNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-appsflyer/metadata.yaml b/airbyte-integrations/connectors/source-appsflyer/metadata.yaml index 94f7dc4817f1..4f3c263c6cbb 100644 --- a/airbyte-integrations/connectors/source-appsflyer/metadata.yaml +++ b/airbyte-integrations/connectors/source-appsflyer/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/appsflyer tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-appsflyer/requirements.txt b/airbyte-integrations/connectors/source-appsflyer/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-appsflyer/requirements.txt +++ b/airbyte-integrations/connectors/source-appsflyer/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-appsflyer/setup.py b/airbyte-integrations/connectors/source-appsflyer/setup.py index ee6f585f997f..ac74ebcabfc9 100644 --- a/airbyte-integrations/connectors/source-appsflyer/setup.py +++ b/airbyte-integrations/connectors/source-appsflyer/setup.py @@ -8,9 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml b/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml index 4285415406ed..cb0d55be57b3 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-appstore-singer/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/appstore tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-appstore-singer/setup.py b/airbyte-integrations/connectors/source-appstore-singer/setup.py index e263b5f42dae..b6ecccc99a03 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/setup.py +++ b/airbyte-integrations/connectors/source-appstore-singer/setup.py @@ -13,6 +13,7 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index 0cbb9c20b0e9..5a14041b24e4 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/asana tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-asana/requirements.txt b/airbyte-integrations/connectors/source-asana/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-asana/requirements.txt +++ b/airbyte-integrations/connectors/source-asana/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-asana/setup.py b/airbyte-integrations/connectors/source-asana/setup.py index d5c276d8ca77..dda6ee977db1 100644 --- a/airbyte-integrations/connectors/source-asana/setup.py +++ b/airbyte-integrations/connectors/source-asana/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk~=0.2", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "requests-mock~=1.9.3", "connector-acceptance-test"] +TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock~=1.9.3"] setup( name="source_asana", diff --git a/airbyte-integrations/connectors/source-asana/source_asana/spec.json b/airbyte-integrations/connectors/source-asana/source_asana/spec.json index b6903849cb00..28b4e3107f20 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/spec.json +++ b/airbyte-integrations/connectors/source-asana/source_asana/spec.json @@ -67,10 +67,7 @@ }, "advanced_auth": { "auth_flow_type": "oauth2.0", - "predicate_key": [ - "credentials", - "option_title" - ], + "predicate_key": ["credentials", "option_title"], "predicate_value": "OAuth Credentials", "oauth_config_specification": { "complete_oauth_output_specification": { @@ -78,9 +75,7 @@ "properties": { "refresh_token": { "type": "string", - "path_in_connector_config": [ - "credentials", "refresh_token" - ] + "path_in_connector_config": ["credentials", "refresh_token"] } } }, @@ -100,15 +95,11 @@ "properties": { "client_id": { "type": "string", - "path_in_connector_config": [ - "credentials", "client_id" - ] + "path_in_connector_config": ["credentials", "client_id"] }, "client_secret": { "type": "string", - "path_in_connector_config": [ - "credentials","client_secret" - ] + "path_in_connector_config": ["credentials", "client_secret"] } } } diff --git a/airbyte-integrations/connectors/source-ashby/metadata.yaml b/airbyte-integrations/connectors/source-ashby/metadata.yaml index 270135f0e5b4..510c05376240 100644 --- a/airbyte-integrations/connectors/source-ashby/metadata.yaml +++ b/airbyte-integrations/connectors/source-ashby/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-ashby/requirements.txt b/airbyte-integrations/connectors/source-ashby/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-ashby/requirements.txt +++ b/airbyte-integrations/connectors/source-ashby/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-ashby/setup.py b/airbyte-integrations/connectors/source-ashby/setup.py index 93194d1cfe62..d4fd781c5f1e 100644 --- a/airbyte-integrations/connectors/source-ashby/setup.py +++ b/airbyte-integrations/connectors/source-ashby/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-auth0/Dockerfile b/airbyte-integrations/connectors/source-auth0/Dockerfile index 888936f8d3ac..796a01c6fd39 100644 --- a/airbyte-integrations/connectors/source-auth0/Dockerfile +++ b/airbyte-integrations/connectors/source-auth0/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_auth0 ./source_auth0 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-auth0 diff --git a/airbyte-integrations/connectors/source-auth0/README.md b/airbyte-integrations/connectors/source-auth0/README.md index 6be279a1d1ef..d6131a4feda3 100644 --- a/airbyte-integrations/connectors/source-auth0/README.md +++ b/airbyte-integrations/connectors/source-auth0/README.md @@ -1,35 +1,10 @@ # Auth0 Source -This is the repository for the Auth0 source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/auth0). +This is the repository for the Auth0 configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/auth0). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/auth0) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/auth0) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_auth0/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config_access-token.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source auth0 test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-auth0:dev discover --c docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-auth0:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-auth0/__init__.py b/airbyte-integrations/connectors/source-auth0/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml b/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml index e9b68ceeea84..938d14f308bd 100644 --- a/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-auth0/acceptance-test-config.yml @@ -1,22 +1,35 @@ connector_image: airbyte/source-auth0:dev -tests: +acceptance_tests: spec: - - spec_path: "source_auth0/spec.yaml" + tests: + - spec_path: "source_auth0/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + # bypass_reason: "Connection check getting lost" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json index 3237ca3c641f..2430fb768383 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/abnormal_state.json @@ -1,3 +1,9 @@ -{ - "users": { "updated_at": "3021-09-08T07:04:28.000Z" } -} +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "3021-09-08T07:04:28.000Z" }, + "stream_descriptor": { "name": "users" } + } + } +] diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py index 82823254d266..d49b55882333 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/acceptance.py @@ -10,5 +10,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/catalog.json b/airbyte-integrations/connectors/source-auth0/integration_tests/catalog.json deleted file mode 100644 index c9582a1dce62..000000000000 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/catalog.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "users", - "json_schema": {} - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updated_at"], - "primary_key": [["user_id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json index 0ab89c83f98e..e305bcb59312 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/configured_catalog.json @@ -7,9 +7,9 @@ "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", - "destination_sync_mode": "overwrite", "cursor_field": ["updated_at"], - "primary_key": [["user_id"]] + "primary_key": [["user_id"]], + "destination_sync_mode": "overwrite" }, { "stream": { @@ -20,6 +20,36 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", "primary_key": [["client_id"]] + }, + { + "stream": { + "name": "organizations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "organization_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "primary_key": [["user_id"]], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "organization_member_roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..fd248c8c1be1 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl @@ -0,0 +1 @@ +{"stream": "users", "data": {"created_at":"2023-08-03T14:47:51.713Z","email":"admin@medusa-test.com","email_verified":false,"identities":[{"connection":"Username-Password-Authentication","user_id":"64cbbe17741f518beae16346","provider":"auth0","isSocial":false}],"name":"admin@medusa-test.com","nickname":"admin","picture":"https://s.gravatar.com/avatar/36ded5b8b1df85ba3f21bd1382c92bbb?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fad.png","updated_at":"2023-08-23T01:40:56.928Z","user_id":"auth0|64cbbe17741f518beae16346","user_metadata":{"color":"blue"},"app_metadata":{}}, "emitted_at": 1691072031178} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json index 828d82b44254..defc51ad29b1 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/invalid_config.json @@ -3,5 +3,6 @@ "credentials": { "auth_type": "oauth2_access_token", "access_token": "Invalid-token" - } + }, + "start_date": "2099-08-05T00:43:59.244Z" } diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config.json new file mode 100644 index 000000000000..62797d9aaa03 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config.json @@ -0,0 +1,8 @@ +{ + "base_url": "https://dev-yourOrg.us.auth0.com", + "credentials": { + "auth_type": "oauth2_access_token", + "access_token": "api-key-just-for-testing" + }, + "start_date": "2023-08-05T00:43:59.244Z" +} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_access-token.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_access-token.json deleted file mode 100644 index 73af3bf1822c..000000000000 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_access-token.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "base_url": "https://dev-yourOrg.us.auth0.com", - "credentials": { - "auth_type": "oauth2_access_token", - "access_token": "api-key-just-for-testing" - } -} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json index 397458e958d4..82af98b8c27b 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_config_oauth2-confidential.json @@ -5,5 +5,6 @@ "client_secret": "top-secret", "client_id": "click-copy-icon-from-applications-page", "audience": "https://dev-your-org.us.auth0.com/api/v2/" - } + }, + "start_date": "2023-08-05T00:43:59.244Z" } diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json index b9b68f1828f3..f7527151cd3f 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/sample_state.json @@ -1,3 +1,9 @@ -{ - "users": { "updated_at": "2021-09-08T07:04:28.000Z" } -} +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2023-09-08T07:04:28.000Z" }, + "stream_descriptor": { "name": "users" } + } + } +] diff --git a/airbyte-integrations/connectors/source-auth0/metadata.yaml b/airbyte-integrations/connectors/source-auth0/metadata.yaml index 79354e4697d7..b0087a7a72f9 100644 --- a/airbyte-integrations/connectors/source-auth0/metadata.yaml +++ b/airbyte-integrations/connectors/source-auth0/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - "*.auth0.com" + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 6c504e48-14aa-4221-9a72-19cf5ff1ae78 - dockerImageTag: 0.2.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-auth0 githubIssueLabel: source-auth0 icon: auth0.svg license: MIT name: Auth0 - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-10 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/auth0 tags: - - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-auth0/setup.py b/airbyte-integrations/connectors/source-auth0/setup.py index d2eca1352c42..c634b4b3fc2d 100644 --- a/airbyte-integrations/connectors/source-auth0/setup.py +++ b/airbyte-integrations/connectors/source-auth0/setup.py @@ -6,11 +6,11 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/authenticator.py b/airbyte-integrations/connectors/source-auth0/source_auth0/authenticator.py deleted file mode 100644 index f4745f120ebe..000000000000 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/authenticator.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -from typing import Any, Mapping -from urllib import parse - -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator - -logger = logging.getLogger("airbyte") - - -class Auth0Oauth2Authenticator(Oauth2Authenticator): - def __init__(self, base_url: str, audience: str, client_id: str, client_secret: str): - super().__init__(parse.urljoin(base_url, "/oauth/token"), client_id, client_secret, "") - self.audience = audience.rstrip("/") + "/" - - def build_refresh_request_body(self) -> Mapping[str, Any]: - if not self.get_refresh_token(): - return { - "grant_type": "client_credentials", - "client_id": self.get_client_id(), - "client_secret": self.get_client_secret(), - "audience": self.audience, - } - else: - return super().build_refresh_request_body() diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml b/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml new file mode 100644 index 000000000000..f5b944162e5f --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml @@ -0,0 +1,126 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "{{ config['base_url'] }}" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "per_page" + pagination_strategy: + type: "PageIncrement" + page_size: 5 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + clients_stream: + $ref: "#/definitions/base_stream" + name: "clients" + primary_key: "client_id" + $parameters: + path: "clients" + + users_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "users" + primary_key: "user_id" + path: "users" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: "updated_at" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_granularity: "PT0.000001S" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + + organizations_stream: + $ref: "#/definitions/base_stream" + name: "organizations" + primary_key: "id" + $parameters: + path: "organizations" + + organization_members_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/organizations_stream" + parent_key: "id" + partition_field: "parent_id" + + organization_members_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "organization_members" + primary_key: "user_id" + path: "organizations/{{ stream_partition.parent_id }}/members" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/organization_members_partition_router" + + organization_member_roles_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/organization_members_stream" + parent_key: "user_id" + partition_field: "parent_user_id" + + organization_member_roles_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "organization_member_roles" + primary_key: "id" + path: "organizations/{{ stream_partition.parent_slice.parent_id }}/members/{{ stream_partition.parent_user_id }}/roles" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/organization_member_roles_partition_router" + +streams: + - "#/definitions/clients_stream" + - "#/definitions/users_stream" + - "#/definitions/organizations_stream" + - "#/definitions/organization_member_roles_stream" + - "#/definitions/organization_members_stream" + +check: + type: CheckStream + stream_names: + - "clients" + - "users" + - "organizations" + - "organization_member_roles" + - "organization_members" diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json index e17890d2a740..1d271e303881 100644 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/clients.json @@ -121,7 +121,7 @@ } }, "signing_keys": { - "type": ["array", "null"], + "type": ["null", "array"], "items": { "type": ["object", "null"], "additionalProperties": true diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_member_roles.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_member_roles.json new file mode 100644 index 000000000000..74ce51f9b84c --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_member_roles.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "org_id": { + "type": ["string", "null"] + }, + "user_id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_members.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_members.json new file mode 100644 index 000000000000..4b291d698b83 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organization_members.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "org_id": { + "type": ["string", "null"] + }, + "user_id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "picture": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organizations.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organizations.json new file mode 100644 index 000000000000..7e998508b666 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/organizations.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "display_name": { + "type": ["string", "null"] + }, + "branding": { + "type": ["object", "null"], + "additionalProperties": true + }, + "metadata": { + "type": ["object", "null"], + "additionalProperties": true + } + } +} diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json index ddf342e4c6ac..5079e526f7ad 100644 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/schemas/users.json @@ -1,4 +1,5 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": ["object", "null"], "additionalProperties": true, "properties": { @@ -32,10 +33,15 @@ "type": ["string", "null"] }, "identities": { - "type": "array", + "type": ["null", "array"], "items": { "type": ["object", "null"], - "additionalProperties": true + "additionalProperties": true, + "properties": { + "connection": { + "type": ["string", "null"] + } + } } }, "app_metadata": { @@ -56,8 +62,11 @@ "type": ["string", "null"] }, "multifactor": { - "type": ["object", "null"], - "additionalProperties": true + "type": ["null", "array"], + "additionalProperties": true, + "items": { + "type": ["string", "null"] + } }, "last_ip": { "type": ["string", "null"] diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/source.py b/airbyte-integrations/connectors/source-auth0/source_auth0/source.py index 4964c3e7ef81..7d05a3317c0c 100644 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/source.py +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/source.py @@ -2,154 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import logging -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib import parse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from airbyte_cdk.sources.streams.http import HttpStream -from source_auth0.utils import get_api_endpoint, initialize_authenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class Auth0Stream(HttpStream, ABC): - api_version = "v2" - page_size = 50 - resource_name = "entities" - - def __init__(self, url_base: str, *args, **kwargs): - super().__init__(*args, **kwargs) - self.api_endpoint = get_api_endpoint(url_base, self.api_version) - - def path(self, **kwargs) -> str: - return self.resource_name - - @property - def url_base(self) -> str: - return self.api_endpoint - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - body = response.json() - if "total" in body and "start" in body and "limit" in body and "length" in body: - try: - start = int(body["start"]) - limit = int(body["limit"]) - length = int(body["length"]) - total = int(body["total"]) - current = start // limit - if length < limit or (start + length) == total: - return None - else: - token = { - "page": current + 1, - "per_page": limit, - } - return token - except Exception: - return None - else: - if not body or len(body) < self.page_size: - return None - else: - return { - "page": 0, - "per_page": self.page_size, - } - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - return { - "page": 0, - "per_page": self.page_size, - "include_totals": "true", - **(next_page_token or {}), - } - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json().get(self.resource_name) - - def backoff_time(self, response: requests.Response) -> Optional[float]: - # The rate limit resets on the timestamp indicated - # https://auth0.com/docs/troubleshoot/customer-support/operational-policies/rate-limit-policy/management-api-endpoint-rate-limits - if response.status_code == requests.codes.TOO_MANY_REQUESTS: - next_reset_epoch = int(response.headers["x-ratelimit-reset"]) - next_reset = pendulum.from_timestamp(next_reset_epoch) - next_reset_duration = pendulum.now("UTC").diff(next_reset) - return next_reset_duration.seconds - - -class IncrementalAuth0Stream(Auth0Stream, IncrementalMixin): - min_id = "" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._cursor_value = self.min_id - - @property - @abstractmethod - def cursor_field(self) -> str: - pass - - @property - def state(self) -> MutableMapping[str, Any]: - return {self.cursor_field: self._cursor_value} - - @state.setter - def state(self, value: MutableMapping[str, Any]): - self._cursor_value = value.get(self.cursor_field) - - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=self.state, next_page_token=next_page_token, **kwargs) - latest_entry = self.state.get(self.cursor_field) - filter_param = {"include_totals": "false", "sort": f"{self.cursor_field}:1", "q": f"{self.cursor_field}:{{{latest_entry} TO *]"} - params.update(filter_param) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - entities = response.json() - if entities: - last_item = entities[-1] - self.state = last_item - yield from entities - - -class Clients(Auth0Stream): - primary_key = "client_id" - resource_name = "clients" - -class Users(IncrementalAuth0Stream): - min_id = "1900-01-01T00:00:00.000Z" - primary_key = "user_id" - resource_name = "users" - cursor_field = "updated_at" - - -# Source -class SourceAuth0(AbstractSource): - def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: - try: - auth = initialize_authenticator(config) - api_endpoint = get_api_endpoint(config.get("base_url"), "v2") - url = parse.urljoin(api_endpoint, "users") - response = requests.get( - url, - params={"per_page": 1}, - headers=auth.get_auth_header(), - ) - - if response.status_code == requests.codes.ok: - return True, None - - return False, response.json() - except Exception: - return False, "Failed to authenticate with the provided credentials" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - initialization_params = {"authenticator": initialize_authenticator(config), "url_base": config.get("base_url")} - return [Clients(**initialization_params), Users(**initialization_params)] +# Declarative Source +class SourceAuth0(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml b/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml index 4476700ddbce..d445b68dbb52 100644 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/spec.yaml @@ -68,6 +68,8 @@ connectionSpecification: type: string title: Authentication Method const: oauth2_access_token + examples: + - "oauth2_access_token" order: 0 access_token: title: OAuth2 Access Token @@ -77,3 +79,13 @@ connectionSpecification: The access token used to call the Auth0 Management API Token. It's a JWT that contains specific grant permissions knowns as scopes. type: string airbyte_secret: true + start_date: + type: string + title: Start Date + description: + UTC date and time in the format 2017-01-25T00:00:00Z. Any data + before this date will not be replicated. + examples: + - "2023-08-05T00:43:59.244Z" + default: "2023-08-05T00:43:59.244Z" + airbyte_secret: false diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/utils.py b/airbyte-integrations/connectors/source-auth0/source_auth0/utils.py deleted file mode 100644 index abafb4389fb3..000000000000 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/utils.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import datetime -import logging -from typing import Dict -from urllib import parse - -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator -from requests.auth import AuthBase - -from .authenticator import Auth0Oauth2Authenticator - -logger = logging.getLogger("airbyte") - - -def get_api_endpoint(url_base: str, version: str) -> str: - return parse.urljoin(url_base, f"/api/{version}/") - - -def initialize_authenticator(config: Dict) -> AuthBase: - credentials = config.get("credentials") - if not credentials: - raise Exception("Config validation error. `credentials` not specified.") - - auth_type = credentials.get("auth_type") - if not auth_type: - raise Exception("Config validation error. `auth_type` not specified.") - - if auth_type == "oauth2_access_token": - return TokenAuthenticator(credentials.get("access_token")) - - if auth_type == "oauth2_confidential_application": - return Auth0Oauth2Authenticator( - base_url=config.get("base_url"), - audience=credentials.get("audience"), - client_secret=credentials.get("client_secret"), - client_id=credentials.get("client_id"), - ) - - -def datetime_to_string(date: datetime.datetime) -> str: - return date.strftime("%Y-%m-%dT%H:%M:%S.000Z") diff --git a/airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py b/airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py deleted file mode 100644 index 151a75afc9e3..000000000000 --- a/airbyte-integrations/connectors/source-auth0/unit_tests/conftest.py +++ /dev/null @@ -1,269 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pendulum -import pytest - - -@pytest.fixture() -def url_base(): - """ - URL base for test - """ - return "https://dev-yourOrg.us.auth0.com" - - -@pytest.fixture() -def api_url(url_base): - """ - Just return API url based on url_base - """ - return f"{url_base}/api/v2" - - -@pytest.fixture() -def oauth_config(url_base): - """ - Credentials for oauth2.0 authorization - """ - return { - "credentials": { - "auth_type": "oauth2_confidential_application", - "client_secret": "test_client_secret", - "client_id": "test_client_id", - "audience": f"{url_base}/api/v2", - }, - "base_url": url_base, - } - - -@pytest.fixture() -def wrong_oauth_config_bad_credentials_record(url_base): - """ - Malformed Credentials for oauth2.0 authorization - credentials -> credential - """ - return { - "credential": { - "auth_type": "oauth2.0", - "client_secret": "test_client_secret", - "client_id": "test_client_id", - }, - "base_url": url_base, - } - - -@pytest.fixture() -def wrong_oauth_config_bad_auth_type(url_base): - """ - Wrong Credentials format for oauth2.0 authorization - absent "auth_type" field - """ - return { - "credentials": { - "client_secret": "test_client_secret", - "client_id": "test_client_id", - "refresh_token": "test_refresh_token", - }, - "base_url": url_base, - } - - -@pytest.fixture() -def token_config(url_base): - """ - Just test 'token' - """ - return { - "credentials": {"auth_type": "oauth2_access_token", "access_token": "test-token"}, - "base_url": url_base, - } - - -@pytest.fixture() -def user_status_filter(): - statuses = ["ACTIVE", "DEPROVISIONED", "LOCKED_OUT", "PASSWORD_EXPIRED", "PROVISIONED", "RECOVERY", "STAGED", "SUSPENDED"] - return " or ".join([f'status eq "{status}"' for status in statuses]) - - -@pytest.fixture() -def users_instance(): - """ - Users instance object response - """ - return { - "blocked": False, - "created_at": "2022-10-21T04:10:34.240Z", - "email": "rodrick_waelchi73@yahoo.com", - "email_verified": False, - "family_name": "Kerluke", - "given_name": "Nick", - "identities": [ - { - "user_id": "15164a44-8064-4ef9-ac31-fb08814da3f9", - "connection": "Username-Password-Authentication", - "provider": "auth0", - "isSocial": False, - } - ], - "name": "Linda Sporer IV", - "nickname": "Marty", - "picture": "https://secure.gravatar.com/avatar/15626c5e0c749cb912f9d1ad48dba440?s=480&r=pg&d=https%3A%2F%2Fssl.gstatic.com%2Fs2%2Fprofiles%2Fimages%2Fsilhouette80.png", - "updated_at": "2022-10-21T04:10:34.240Z", - "user_id": "auth0|15164a44-8064-4ef9-ac31-fb08814da3f9", - "user_metadata": {}, - "app_metadata": {}, - } - - -@pytest.fixture() -def clients_instance(): - """ - Clients instance object response - """ - return { - "client_id": "AaiyAPdpYdesoKnqjj8HJqRn4T5titww", - "tenant": "", - "name": "My application", - "description": "", - "global": False, - "client_secret": "MG_TNT2ver-SylNat-_VeMmd-4m0Waba0jr1troztBniSChEw0glxEmgEi2Kw40H", - "app_type": "", - "logo_uri": "", - "is_first_party": False, - "oidc_conformant": False, - "callbacks": ["http://localhost/callback"], - "allowed_origins": [""], - "web_origins": [""], - "client_aliases": [""], - "allowed_clients": [""], - "allowed_logout_urls": ["http://localhost/logoutCallback"], - "oidc_backchannel_logout": {"backchannel_logout_urls": [""]}, - "grant_types": [""], - "jwt_configuration": {"lifetime_in_seconds": 36000, "secret_encoded": True, "scopes": {}, "alg": "HS256"}, - "signing_keys": ["object"], - "encryption_key": {"pub": "", "cert": "", "subject": ""}, - "sso": False, - "sso_disabled": False, - "cross_origin_authentication": False, - "cross_origin_loc": "", - "custom_login_page_on": True, - "custom_login_page": "", - "custom_login_page_preview": "", - "form_template": "", - "addons": { - "aws": {"principal": "", "role": "", "lifetime_in_seconds": 0}, - "azure_blob": { - "accountName": "", - "storageAccessKey": "", - "containerName": "", - "blobName": "", - "expiration": 0, - "signedIdentifier": "", - "blob_read": False, - "blob_write": False, - "blob_delete": False, - "container_read": False, - "container_write": False, - "container_delete": False, - "container_list": False, - }, - "azure_sb": {"namespace": "", "sasKeyName": "", "sasKey": "", "entityPath": "", "expiration": 0}, - "rms": {"url": ""}, - "mscrm": {"url": ""}, - "slack": {"team": ""}, - "sentry": {"org_slug": "", "base_url": ""}, - "box": {}, - "cloudbees": {}, - "concur": {}, - "dropbox": {}, - "echosign": {"domain": ""}, - "egnyte": {"domain": ""}, - "firebase": {"secret": "", "private_key_id": "", "private_key": "", "client_email": "", "lifetime_in_seconds": 0}, - "newrelic": {"account": ""}, - "office365": {"domain": "", "connection": ""}, - "salesforce": {"entity_id": ""}, - "salesforce_api": {"clientid": "", "principal": "", "communityName": "", "community_url_section": ""}, - "salesforce_sandbox_api": {"clientid": "", "principal": "", "communityName": "", "community_url_section": ""}, - "samlp": { - "mappings": {}, - "audience": "", - "recipient": "", - "createUpnClaim": False, - "mapUnknownClaimsAsIs": False, - "passthroughClaimsWithNoMapping": False, - "mapIdentities": False, - "signatureAlgorithm": "", - "digestAlgorithm": "", - "issuer": "", - "destination": "", - "lifetimeInSeconds": 0, - "signResponse": False, - "nameIdentifierFormat": "", - "nameIdentifierProbes": [""], - "authnContextClassRef": "", - }, - "layer": {"providerId": "", "keyId": "", "privateKey": "", "principal": "", "expiration": 0}, - "sap_api": { - "clientid": "", - "usernameAttribute": "", - "tokenEndpointUrl": "", - "scope": "", - "servicePassword": "", - "nameIdentifierFormat": "", - }, - "sharepoint": {"url": "", "external_url": [""]}, - "springcm": {"acsurl": ""}, - "wams": {"masterkey": ""}, - "wsfed": {}, - "zendesk": {"accountName": ""}, - "zoom": {"account": ""}, - "sso_integration": {"name": "", "version": ""}, - }, - "token_endpoint_auth_method": "none", - "client_metadata": {}, - "mobile": { - "android": {"app_package_name": "", "sha256_cert_fingerprints": []}, - "ios": {"team_id": "", "app_bundle_identifier": ""}, - }, - "initiate_login_uri": "", - "native_social_login": {"apple": {"enabled": False}, "facebook": {"enabled": False}}, - "refresh_token": { - "rotation_type": "non-rotating", - "expiration_type": "non-expiring", - "leeway": 0, - "token_lifetime": 0, - "infinite_token_lifetime": False, - "idle_token_lifetime": 0, - "infinite_idle_token_lifetime": False, - }, - "organization_usage": "deny", - "organization_require_behavior": "no_prompt", - "client_authentication_methods": {"private_key_jwt": {"credentials": ["object"]}} - } - - -@pytest.fixture() -def latest_record_instance(): - """ - Last Record instance object response - """ - return { - "id": "test_user_group_id", - "created": "2022-07-18T07:58:11.000Z", - "lastUpdated": "2022-07-18T07:58:11.000Z", - } - - -@pytest.fixture() -def error_failed_to_authorize_with_provided_credentials(): - """ - Error raised when using incorrect oauth2.0 credentials - """ - return "Failed to authenticate with the provided credentials" - - -@pytest.fixture() -def start_date(): - return pendulum.parse("2021-03-21T20:49:13Z") diff --git a/airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py b/airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py deleted file mode 100644 index d6dde5648958..000000000000 --- a/airbyte-integrations/connectors/source-auth0/unit_tests/test_source.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator -from source_auth0.authenticator import Auth0Oauth2Authenticator -from source_auth0.source import SourceAuth0, Clients, Users, initialize_authenticator - - -class TestAuthentication: - def test_init_token_authentication_init(self, token_config): - token_auth = initialize_authenticator(config=token_config) - assert isinstance(token_auth, TokenAuthenticator) - - def test_init_oauth2_authentication_init(self, oauth_config): - oauth_auth = initialize_authenticator(config=oauth_config) - assert isinstance(oauth_auth, Auth0Oauth2Authenticator) - - def test_init_oauth2_authentication_wrong_credentials_record(self, wrong_oauth_config_bad_credentials_record): - try: - initialize_authenticator(config=wrong_oauth_config_bad_credentials_record) - except Exception as e: - assert e.args[0] == "Config validation error. `credentials` not specified." - - def test_init_oauth2_authentication_wrong_oauth_config_bad_auth_type(self, wrong_oauth_config_bad_auth_type): - try: - initialize_authenticator(config=wrong_oauth_config_bad_auth_type) - except Exception as e: - assert e.args[0] == "Config validation error. `auth_type` not specified." - - def test_check_connection_ok(self, requests_mock, oauth_config, url_base): - oauth_auth = initialize_authenticator(config=oauth_config) - assert isinstance(oauth_auth, Auth0Oauth2Authenticator) - - source_auth0 = SourceAuth0() - requests_mock.get(f"{url_base}/api/v2/users?per_page=1", json={"connect": "ok"}) - requests_mock.post(f"{url_base}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) - assert source_auth0.check_connection(logger=MagicMock(), config=oauth_config) == (True, None) - - def test_check_connection_error_status_code(self, requests_mock, oauth_config, url_base): - oauth_auth = initialize_authenticator(config=oauth_config) - assert isinstance(oauth_auth, Auth0Oauth2Authenticator) - - source_auth0 = SourceAuth0() - requests_mock.get(f"{url_base}/api/v2/users?per_page=1", status_code=400, json={}) - requests_mock.post(f"{url_base}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) - - assert source_auth0.check_connection(logger=MagicMock(), config=oauth_config) == (False, {}) - - def test_check_connection_error_with_exception( - self, requests_mock, oauth_config, url_base, error_failed_to_authorize_with_provided_credentials - ): - oauth_auth = initialize_authenticator(config=oauth_config) - assert isinstance(oauth_auth, Auth0Oauth2Authenticator) - - source_auth0 = SourceAuth0() - requests_mock.get(f"{url_base}/api/v2/users?per_page=1", status_code=400, json="ss") - requests_mock.post(f"{url_base}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) - - assert source_auth0.check_connection(logger=MagicMock(), config="wrong_config") == ( - False, - error_failed_to_authorize_with_provided_credentials, - ) - - def test_check_streams(self, requests_mock, oauth_config, api_url): - oauth_auth = initialize_authenticator(config=oauth_config) - assert isinstance(oauth_auth, Auth0Oauth2Authenticator) - - source_auth0 = SourceAuth0() - requests_mock.get(f"{api_url}/api/v2/users?per_page=1", json={"connect": "ok"}) - requests_mock.get(f"{api_url}/api/v2/clients?per_page=1", json={"connect": "ok"}) - requests_mock.post(f"{api_url}/oauth/token", json={"access_token": "test_token", "expires_in": 948}) - streams = source_auth0.streams(config=oauth_config) - - streams_supported = [Clients, Users] - - # check the number of streams supported - assert len(streams) == len(streams_supported) - - # and each stream to be specific stream - assert isinstance(streams[0], streams_supported[0]) - assert isinstance(streams[1], streams_supported[1]) diff --git a/airbyte-integrations/connectors/source-auth0/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-auth0/unit_tests/test_streams.py deleted file mode 100644 index 5c12d87238fa..000000000000 --- a/airbyte-integrations/connectors/source-auth0/unit_tests/test_streams.py +++ /dev/null @@ -1,260 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import time -from abc import ABC -from unittest.mock import MagicMock - -import pytest -import requests -from airbyte_cdk.models import SyncMode -from source_auth0.source import Auth0Stream, IncrementalAuth0Stream, Users, Clients - - -@pytest.fixture -def patch_base_class(mocker): - """ - Base patcher for used streams - """ - mocker.patch.object(Auth0Stream, "primary_key", "test_primary_key") - mocker.patch.object(Auth0Stream, "__abstractmethods__", set()) - mocker.patch.object(IncrementalAuth0Stream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalAuth0Stream, "__abstractmethods__", set()) - - -class TestAuth0Stream: - def test_auth0_stream_request_params(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = { - "page": 0, - "per_page": 50, - "include_totals": "true", - } - assert stream.request_params(**inputs) == expected_params - - def test_auth0_stream_parse_response(self, patch_base_class, requests_mock, url_base, api_url): - stream = Auth0Stream(url_base=url_base) - requests_mock.get(f"{api_url}", json={"entities": [{"a": 123}, {"b": "xx"}]}) - resp = requests.get(f"{api_url}") - inputs = {"response": resp, "stream_state": MagicMock()} - expected_parsed_object = [{"a": 123}, {"b": "xx"}] - assert list(stream.parse_response(**inputs)) == expected_parsed_object - - def test_auth0_stream_backoff_time(self, patch_base_class, url_base): - response_mock = requests.Response() - stream = Auth0Stream(url_base=url_base) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - def test_auth0_stream_incremental_request_params(self, patch_base_class, url_base): - stream = IncrementalAuth0Stream(url_base=url_base) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = { - "page": 0, - "per_page": 50, - "include_totals": "false", - "sort": "None:1", - "q": "None:{ TO *]", - } - assert stream.request_params(**inputs) == expected_params - - def test_incremental_auth0_stream_parse_response(self, patch_base_class, requests_mock, url_base, api_url): - stream = IncrementalAuth0Stream(url_base=url_base) - requests_mock.get(f"{api_url}", json=[{"a": 123}, {"b": "xx"}]) - resp = requests.get(f"{api_url}") - inputs = {"response": resp, "stream_state": MagicMock()} - expected_parsed_object = [{"a": 123}, {"b": "xx"}] - assert list(stream.parse_response(**inputs)) == expected_parsed_object - - def test_incremental_auth0_stream_backoff_time(self, patch_base_class, url_base): - response_mock = MagicMock() - stream = IncrementalAuth0Stream(url_base=url_base) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - def test_auth0_stream_incremental_backoff_time_empty(self, patch_base_class, url_base): - stream = IncrementalAuth0Stream(url_base=url_base) - response = MagicMock(requests.Response) - response.status_code = 200 - expected_params = None - inputs = {"response": response} - assert stream.backoff_time(**inputs) == expected_params - - def test_auth0_stream_incremental_back_off_now(self, patch_base_class, url_base): - stream = IncrementalAuth0Stream(url_base=url_base) - response = MagicMock(requests.Response) - response.status_code = requests.codes.TOO_MANY_REQUESTS - response.headers = {"x-ratelimit-reset": int(time.time())} - expected_params = (0, 2) - inputs = {"response": response} - get_backoff_time = stream.backoff_time(**inputs) - assert expected_params[0] <= get_backoff_time <= expected_params[1] - - def test_auth0_stream_incremental_get_updated_state(self, patch_base_class, latest_record_instance, url_base): - class TestIncrementalAuth0Stream(IncrementalAuth0Stream, ABC): - cursor_field = "lastUpdated" - - stream = TestIncrementalAuth0Stream(url_base=url_base) - stream._cursor_field = "lastUpdated" - assert stream._cursor_value == "" - stream.state = {"lastUpdated": "123"} - assert stream._cursor_value == "123" - - def test_auth0_stream_http_method(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - expected_method = "GET" - assert stream.http_method == expected_method - - -class TestNextPageToken: - def test_next_page_token(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - json = { - "start": "0", - "limit": 50, - "length": "50", - "total": 51, - } - response = MagicMock(requests.Response) - response.json = MagicMock(return_value=json) - inputs = {"response": response} - expected_token = {"page": 1, "per_page": 50} - result = stream.next_page_token(**inputs) - assert result == expected_token - - def test_next_page_token_invalid_cursor(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - json = { - "start": "0", - "limit": 50, - "length": "abc", - "total": 51, - } - response = MagicMock(requests.Response) - response.json = MagicMock(return_value=json) - inputs = {"response": response} - expected_token = None - result = stream.next_page_token(**inputs) - assert result == expected_token - - def test_next_page_token_missing_cursor(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - json = { - "limit": 50, - "total": 51, - } - response = MagicMock(requests.Response) - response.json = MagicMock(return_value=json) - inputs = {"response": response} - expected_token = None - result = stream.next_page_token(**inputs) - assert result == expected_token - - def test_next_page_token_one_page_only(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - json = { - "start": 0, - "limit": 50, - "length": 1, - "total": 1, - } - response = MagicMock(requests.Response) - response.json = MagicMock(return_value=json) - inputs = {"response": response} - expected_token = None - result = stream.next_page_token(**inputs) - assert result == expected_token - - def test_next_page_token_last_page_incomplete(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - json = { - "start": "50", - "limit": 50, - "length": "1", - "total": 51, - } - response = MagicMock(requests.Response) - response.json = MagicMock(return_value=json) - inputs = {"response": response} - expected_token = None - result = stream.next_page_token(**inputs) - assert result == expected_token - - def test_next_page_token_last_page_complete(self, patch_base_class, url_base): - stream = Auth0Stream(url_base=url_base) - json = { - "start": "50", - "limit": 50, - "length": "50", - "total": 100, - } - response = MagicMock(requests.Response) - response.json = MagicMock(return_value=json) - inputs = {"response": response} - expected_token = None - result = stream.next_page_token(**inputs) - assert result == expected_token - - -class TestStreamUsers: - def test_stream_users(self, patch_base_class, users_instance, url_base, api_url, requests_mock): - stream = Users(url_base=url_base) - requests_mock.get( - f"{api_url}/users", - json=[users_instance], - ) - inputs = {"sync_mode": SyncMode.incremental} - assert list(stream.read_records(**inputs)) == [users_instance] - - def test_users_request_params_out_of_next_page_token(self, patch_base_class, url_base): - stream = Users(url_base=url_base) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = { - "include_totals": "false", - "page": 0, - "per_page": 50, - "q": "updated_at:{1900-01-01T00:00:00.000Z TO *]", - "sort": "updated_at:1", - } - assert stream.request_params(**inputs) == expected_params - - def test_users_source_request_params_have_next_cursor(self, patch_base_class, url_base): - stream = Users(url_base=url_base) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 1, "per_page": 50}} - expected_params = { - "include_totals": "false", - "page": 1, - "per_page": 50, - "q": "updated_at:{1900-01-01T00:00:00.000Z TO *]", - "sort": "updated_at:1", - } - assert stream.request_params(**inputs) == expected_params - - def test_users_source_parse_response(self, requests_mock, patch_base_class, users_instance, url_base, api_url): - stream = Users(url_base=url_base) - requests_mock.get( - f"{api_url}/users", - json=[users_instance], - ) - assert list(stream.parse_response(response=requests.get(f"{api_url}/users"))) == [users_instance] - - -class TestStreamClients: - def test_stream_clients(self, patch_base_class, clients_instance, url_base, api_url, requests_mock): - stream = Clients(url_base=url_base) - requests_mock.get( - f"{api_url}/clients", - json={"total": 1, "start": 0, "limit": 50, "clients": [clients_instance]}, - ) - inputs = {"sync_mode": SyncMode.full_refresh} - assert list(stream.read_records(**inputs)) == [clients_instance] - - def test_clients_source_parse_response(self, requests_mock, patch_base_class, clients_instance, url_base, api_url): - stream = Clients(url_base=url_base) - requests_mock.get( - f"{api_url}/clients", - json={"total": 1, "start": 0, "limit": 50, "clients": [clients_instance]}, - ) - assert list(stream.parse_response(response=requests.get(f"{api_url}/clients"))) == [clients_instance] diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml b/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml index c8b36893a7b1..3b3240c49425 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 dockerImageTag: 0.1.5 dockerRepository: airbyte/source-aws-cloudtrail + documentationUrl: https://docs.airbyte.com/integrations/sources/aws-cloudtrail githubIssueLabel: source-aws-cloudtrail icon: awscloudtrail.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/aws-cloudtrail + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/requirements.txt b/airbyte-integrations/connectors/source-aws-cloudtrail/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/requirements.txt +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py index 6f6410df7368..bda35b60aa1e 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "boto3==1.17.*"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index 3170c6cf0618..395b78ee078c 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json index e00784cf8759..9e4d74450a36 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json @@ -44,17 +44,13 @@ "title": "Azure Blob Storage blobs prefix", "description": "The Azure blob storage prefix to be applied", "type": "string", - "examples": [ - "FolderA/FolderB/" - ] + "examples": ["FolderA/FolderB/"] }, "azure_blob_storage_schema_inference_limit": { "title": "Azure Blob Storage schema inference limit", "description": "The Azure blob storage blobs to scan for inferring the schema, useful on large amounts of data with consistent structure", "type": "integer", - "examples": [ - "500" - ] + "examples": ["500"] }, "format": { "title": "Input Format", diff --git a/airbyte-integrations/connectors/source-azure-table/metadata.yaml b/airbyte-integrations/connectors/source-azure-table/metadata.yaml index 349e9751fb59..06efb503b20c 100644 --- a/airbyte-integrations/connectors/source-azure-table/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-table/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/azure-table tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-azure-table/requirements.txt b/airbyte-integrations/connectors/source-azure-table/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-azure-table/requirements.txt +++ b/airbyte-integrations/connectors/source-azure-table/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-azure-table/setup.py b/airbyte-integrations/connectors/source-azure-table/setup.py index 9cd14c7b04aa..a04c790cbb8f 100644 --- a/airbyte-integrations/connectors/source-azure-table/setup.py +++ b/airbyte-integrations/connectors/source-azure-table/setup.py @@ -8,9 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "azure.data.tables"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-babelforce/Dockerfile b/airbyte-integrations/connectors/source-babelforce/Dockerfile index ab6f18079b99..0768c0d039ef 100644 --- a/airbyte-integrations/connectors/source-babelforce/Dockerfile +++ b/airbyte-integrations/connectors/source-babelforce/Dockerfile @@ -34,5 +34,5 @@ COPY source_babelforce ./source_babelforce ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-babelforce diff --git a/airbyte-integrations/connectors/source-babelforce/README.md b/airbyte-integrations/connectors/source-babelforce/README.md index 3a054caaa255..f85b00f928f7 100644 --- a/airbyte-integrations/connectors/source-babelforce/README.md +++ b/airbyte-integrations/connectors/source-babelforce/README.md @@ -1,35 +1,10 @@ # Babelforce Source -This is the repository for the Babelforce source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/babelforce). +This is the repository for the Babelforce configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/babelforce). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/babelforce) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_babelforce/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/babelforce) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_babelforce/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source babelforce test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-babelforce:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-babelforce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-babelforce/__init__.py b/airbyte-integrations/connectors/source-babelforce/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-babelforce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-babelforce/acceptance-test-config.yml index 343966dacb78..b3ddb6ff63a5 100644 --- a/airbyte-integrations/connectors/source-babelforce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-babelforce/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-babelforce:dev tests: spec: - - spec_path: "source_babelforce/spec.json" + - spec_path: "source_babelforce/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/__init__.py b/airbyte-integrations/connectors/source-babelforce/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-babelforce/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-babelforce/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py index d49b55882333..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/acceptance.py @@ -10,4 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-babelforce/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-babelforce/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-babelforce/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-babelforce/metadata.yaml b/airbyte-integrations/connectors/source-babelforce/metadata.yaml index 8100069105f5..9425eae3c84b 100644 --- a/airbyte-integrations/connectors/source-babelforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-babelforce/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - ${region}.babelforce.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 971c3e1e-78a5-411e-ad56-c4052b50876b - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-babelforce githubIssueLabel: source-babelforce icon: babelforce.svg license: MIT name: Babelforce - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2022-05-09 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/babelforce tags: - - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-babelforce/requirements.txt b/airbyte-integrations/connectors/source-babelforce/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-babelforce/requirements.txt +++ b/airbyte-integrations/connectors/source-babelforce/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-babelforce/setup.py b/airbyte-integrations/connectors/source-babelforce/setup.py index e0cb2d60a7e4..36286f264643 100644 --- a/airbyte-integrations/connectors/source-babelforce/setup.py +++ b/airbyte-integrations/connectors/source-babelforce/setup.py @@ -5,12 +5,14 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "python-dateutil==2.8.2"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( @@ -20,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/__init__.py b/airbyte-integrations/connectors/source-babelforce/source_babelforce/__init__.py index c03e529edd5f..0e51b2c619fc 100644 --- a/airbyte-integrations/connectors/source-babelforce/source_babelforce/__init__.py +++ b/airbyte-integrations/connectors/source-babelforce/source_babelforce/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml b/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml new file mode 100644 index 000000000000..40b0785582bd --- /dev/null +++ b/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml @@ -0,0 +1,121 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - calls + +streams: + - type: DeclarativeStream + name: calls + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{ config['region'] }}.babelforce.com/api/v2/ + path: calls/reporting/simple + http_method: GET + request_parameters: {} + request_headers: + X-Auth-Access-ID: "{{ config['access_key_id'] }}" + X-Auth-Access-Token: "{{ config['access_token'] }}" + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - items + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: max + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: "{{ response['pagination']['current'] + 1 }}" + stop_condition: "{{ 'current' not in response['pagination'] }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: dateCreated + cursor_datetime_formats: + - "%s" + datetime_format: "%s" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['date_created_from'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + inject_into: request_parameter + field_name: filters.dateCreated.from + type: RequestOption + end_time_option: + inject_into: request_parameter + field_name: filters.dateCreated.to + type: RequestOption + end_datetime: + type: MinMaxDatetime + datetime: "{{ config['date_created_to'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + +spec: + documentation_url: https://example.org + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - access_key_id + - access_token + - region + properties: + access_key_id: + title: Access Key ID + description: The Babelforce access key ID + airbyte_secret: true + type: string + order: 0 + access_token: + title: Access Token + description: The Babelforce access token + airbyte_secret: true + type: string + order: 1 + date_created_from: + title: Date Created from + description: >- + Timestamp in Unix the replication from Babelforce API will start from. + For example 1651363200 which corresponds to 2022-05-01 00:00:00. + type: integer + examples: [1651363200] + order: 2 + date_created_to: + title: Date Created to + description: >- + Timestamp in Unix the replication from Babelforce will be up to. For + example 1651363200 which corresponds to 2022-05-01 00:00:00. + type: integer + examples: [1651363200] + order: 3 + region: + title: Region + description: Babelforce region + default: services + enum: + - services + - us-east + - ap-southeast + type: string + order: 4 diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/source.py b/airbyte-integrations/connectors/source-babelforce/source_babelforce/source.py index a372701d3cc4..9d384f5a7b45 100644 --- a/airbyte-integrations/connectors/source-babelforce/source_babelforce/source.py +++ b/airbyte-integrations/connectors/source-babelforce/source_babelforce/source.py @@ -2,149 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import operator -from abc import ABC -from datetime import datetime -from time import mktime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from dateutil.tz import tzutc +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -DEFAULT_CURSOR = "dateCreated" +WARNING: Do not modify this file. +""" -class InvalidStartAndEndDateException(Exception): - pass - - -# Basic full refresh stream -class BabelforceStream(HttpStream, ABC): - page_size = 100 - - def __init__(self, region: str, **args): - super().__init__(**args) - - self.region = region - - @property - def url_base(self) -> str: - return f"https://{self.region}.babelforce.com/api/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - pagination = response.json().get("pagination") - - if pagination.get("current"): - return {"page": pagination.get("current", 0) + 1} - else: - return None - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - Babelforce calls are sorted in reverse order. To process the calls in ascending order an in-memory sort is performed - - :return an iterable containing each record in the response - """ - items = response.json().get("items") - items.sort(key=operator.itemgetter("dateCreated")) - keys = self.get_json_schema().get("properties").keys() - - for item in items: - yield {key: val for key, val in item.items() if key in keys} - - -# Basic incremental stream -class IncrementalBabelforceStream(BabelforceStream, ABC): - cursor_field = DEFAULT_CURSOR - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - if current_stream_state and current_stream_state.get(self.cursor_field): - current_updated_at = pendulum.parse(current_stream_state.get(self.cursor_field)) - else: - current_updated_at = datetime(1970, 1, 1) - - current_updated_at = current_updated_at.replace(tzinfo=tzutc()) - latest_record_updated_at = pendulum.parse(latest_record.get(self.cursor_field)).replace(tzinfo=tzutc()) - - return {self.cursor_field: max(latest_record_updated_at, current_updated_at).isoformat(timespec="seconds")} - - -class Calls(IncrementalBabelforceStream): - primary_key = "id" - - def __init__(self, date_created_from: int = None, date_created_to: int = None, **args): - super(Calls, self).__init__(**args) - - self.date_created_from = date_created_from - self.date_created_to = date_created_to - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "calls/reporting" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - page = next_page_token.get("page", 1) if next_page_token else 1 - - params = {"page": page, "max": self.page_size} - - if stream_state: - cursor_value = pendulum.parse(stream_state[self.cursor_field]) - self.date_created_from = int(mktime(cursor_value.timetuple())) - - if self.date_created_from and self.date_created_to and self.date_created_from > self.date_created_to: - raise InvalidStartAndEndDateException("`date_created_from` should be less than or equal to `date_created_to`") - - if self.date_created_from: - params.update({"filters.dateCreated.from": self.date_created_from}) - - if self.date_created_to: - params.update({"filters.dateCreated.to": self.date_created_to}) - - return params - - -class SourceBabelforce(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - authenticator = BabelforceAuthenticator(access_key_id=config.get("access_key_id"), access_token=config.get("access_token")) - calls = Calls(region=config.get("region"), authenticator=authenticator) - - test_url = f"{calls.url_base}{calls.path()}?max=1" - response = requests.request("GET", url=test_url, headers=authenticator.get_auth_header()) - - if response.ok: - return True, None - else: - response.raise_for_status() - except Exception as exception: - return False, exception - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - date_created_from = config.get("date_created_from") - date_created_to = config.get("date_created_to") - region = config.get("region") - - auth = BabelforceAuthenticator(access_key_id=config.get("access_key_id"), access_token=config.get("access_token")) - return [Calls(authenticator=auth, region=region, date_created_from=date_created_from, date_created_to=date_created_to)] - - -class BabelforceAuthenticator(HttpAuthenticator): - def __init__(self, access_key_id: str, access_token: str): - self.access_key_id = access_key_id - self.access_token = access_token - - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-Auth-Access-ID": self.access_key_id, "X-Auth-Access-Token": self.access_token} +# Declarative Source +class SourceBabelforce(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/spec.json b/airbyte-integrations/connectors/source-babelforce/source_babelforce/spec.json deleted file mode 100644 index fd41af17ce3f..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/source_babelforce/spec.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Babelforce Spec", - "type": "object", - "required": ["region", "access_key_id", "access_token"], - "additionalProperties": false, - "properties": { - "region": { - "type": "string", - "title": "Region", - "default": "services", - "description": "Babelforce region", - "enum": ["services", "us-east", "ap-southeast"], - "order": 1 - }, - "access_key_id": { - "type": "string", - "title": "Access Key ID", - "description": "The Babelforce access key ID", - "airbyte_secret": true, - "order": 2 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The Babelforce access token", - "airbyte_secret": true, - "order": 3 - }, - "date_created_from": { - "type": "integer", - "title": "Date Created from", - "description": "Timestamp in Unix the replication from Babelforce API will start from. For example 1651363200 which corresponds to 2022-05-01 00:00:00.", - "examples": [1651363200], - "order": 4 - }, - "date_created_to": { - "type": "integer", - "title": "Date Created to", - "description": "Timestamp in Unix the replication from Babelforce will be up to. For example 1651363200 which corresponds to 2022-05-01 00:00:00.", - "examples": [1651363200], - "order": 5 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_call_stream.py b/airbyte-integrations/connectors/source-babelforce/unit_tests/test_call_stream.py deleted file mode 100644 index fb9d58e214ae..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_call_stream.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime -from unittest.mock import MagicMock - -import pytest -from source_babelforce.source import Calls, InvalidStartAndEndDateException - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(Calls, "path", "v0/example_endpoint") - mocker.patch.object(Calls, "primary_key", "test_primary_key") - mocker.patch.object(Calls, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = Calls(region="services") - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 1}} - expected_params = {"max": 100, "page": 1} - assert stream.request_params(**inputs) == expected_params - - -def test_date_created_from_less_than_date_created_to_not_raise_exception(patch_base_class): - try: - stream = Calls(region="services", date_created_from=1, date_created_to=2) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 1}} - stream.request_params(**inputs) - except InvalidStartAndEndDateException as exception: - assert False, exception - - -def test_date_created_from_less_than_date_created_to_raise_exception(patch_base_class): - with pytest.raises(InvalidStartAndEndDateException): - stream = Calls(region="services", date_created_from=2, date_created_to=1) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"page": 1}} - stream.request_params(**inputs) - - -def test_parse_response(patch_base_class): - stream = Calls(region="services") - - fake_date_str = "2022-04-27T00:00:00" - fake_date = datetime.strptime(fake_date_str, "%Y-%m-%dT%H:%M:%S") - - fake_item = { - "id": "abc", - "parentId": "abc", - "sessionId": "abc", - "conversationId": "abc", - "dateCreated": fake_date, - "dateEstablished": None, - "dateFinished": fake_date, - "lastUpdated": fake_date, - "state": "completed", - "finishReason": "unreachable", - "from": "123", - "to": "123", - "type": "outbound", - "source": "queue", - "domain": "internal", - "duration": 0, - "anonymous": False, - "recordings": [], - "bridged": {} - } - - fake_call_json = { - "items": [fake_item] - } - inputs = {"response": MagicMock(json=MagicMock(return_value=fake_call_json))} - assert next(stream.parse_response(**inputs)) == fake_item diff --git a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-babelforce/unit_tests/test_incremental_streams.py deleted file mode 100644 index ca278dceee69..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dateutil.parser import parse -from dateutil.tz import tzutc -from pytest import fixture -from source_babelforce.source import DEFAULT_CURSOR, IncrementalBabelforceStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalBabelforceStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalBabelforceStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalBabelforceStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalBabelforceStream(region="services") - expected_cursor_field = DEFAULT_CURSOR - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalBabelforceStream(region="services") - fake_date = "2022-02-01T00:00:00" - inputs = {"current_stream_state": None, "latest_record": {DEFAULT_CURSOR: fake_date}} - expected_state = {DEFAULT_CURSOR: parse(fake_date).replace(tzinfo=tzutc()).isoformat(timespec="seconds")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalBabelforceStream, "cursor_field", "dummy_field") - stream = IncrementalBabelforceStream(region="services") - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalBabelforceStream(region="services") - assert stream.source_defined_cursor diff --git a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_source.py b/airbyte-integrations/connectors/source-babelforce/unit_tests/test_source.py deleted file mode 100644 index 9e3df894b1c8..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_source.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import requests -from source_babelforce.source import SourceBabelforce - - -def test_check_connection(mocker): - source = SourceBabelforce() - logger_mock, config_mock = MagicMock(), MagicMock() - mocker.patch.object(requests, "request", return_value=MagicMock(ok=True)) - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceBabelforce() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-babelforce/unit_tests/test_streams.py deleted file mode 100644 index 9660ffe2c804..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/unit_tests/test_streams.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_babelforce.source import BabelforceStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(BabelforceStream, "path", "v0/example_endpoint") - mocker.patch.object(BabelforceStream, "primary_key", "test_primary_key") - mocker.patch.object(BabelforceStream, "__abstractmethods__", set()) - - -def test_next_page_token(patch_base_class): - stream = BabelforceStream(region="services") - - json_response_mock = MagicMock() - json_response_mock.json.return_value = {"pagination": {"current": 1}} - - inputs = {"response": json_response_mock} - expected_token = {"page": 2} - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = BabelforceStream(region="services") - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = BabelforceStream(region="services") - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = BabelforceStream(region="services") - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = BabelforceStream(region="services") - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml b/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml index 7a844e7aeb4d..ea2b009e281f 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml +++ b/airbyte-integrations/connectors/source-bamboo-hr/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 90916976-a132-4ce9-8bce-82a03dd58788 dockerImageTag: 0.2.2 dockerRepository: airbyte/source-bamboo-hr + documentationUrl: https://docs.airbyte.com/integrations/sources/bamboo-hr githubIssueLabel: source-bamboo-hr icon: bamboohr.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/bamboo-hr + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bamboo-hr/requirements.txt b/airbyte-integrations/connectors/source-bamboo-hr/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/requirements.txt +++ b/airbyte-integrations/connectors/source-bamboo-hr/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-bamboo-hr/setup.py b/airbyte-integrations/connectors/source-bamboo-hr/setup.py index 1ba4b32a7e4c..a81279a9f2dd 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/setup.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "PyBambooHR"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml index 6f68f50baf61..0ac3c370391c 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 59c5501b-9f95-411e-9269-7143c939adbd dockerImageTag: 0.1.10 dockerRepository: airbyte/source-bigcommerce + documentationUrl: https://docs.airbyte.com/integrations/sources/bigcommerce githubIssueLabel: source-bigcommerce icon: bigcommerce.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/bigcommerce + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bigcommerce/requirements.txt b/airbyte-integrations/connectors/source-bigcommerce/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/requirements.txt +++ b/airbyte-integrations/connectors/source-bigcommerce/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-bigcommerce/setup.py b/airbyte-integrations/connectors/source-bigcommerce/setup.py index 80baceeb4e38..7b6eed1b049c 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/setup.py +++ b/airbyte-integrations/connectors/source-bigcommerce/setup.py @@ -10,8 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-bigquery/Dockerfile b/airbyte-integrations/connectors/source-bigquery/Dockerfile index 4f25769f3997..f3db983b9d4e 100644 --- a/airbyte-integrations/connectors/source-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/source-bigquery/Dockerfile @@ -25,5 +25,5 @@ ENV APPLICATION source-bigquery COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-bigquery diff --git a/airbyte-integrations/connectors/source-bigquery/metadata.yaml b/airbyte-integrations/connectors/source-bigquery/metadata.yaml index 08cbd1388d06..f6e8623837bb 100644 --- a/airbyte-integrations/connectors/source-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigquery/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: database connectorType: source definitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c - dockerImageTag: 0.2.3 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-bigquery + documentationUrl: https://docs.airbyte.com/integrations/sources/bigquery githubIssueLabel: source-bigquery icon: bigquery.svg - license: MIT + license: ELv2 name: BigQuery registries: cloud: @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/bigquery + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index a0bdea3b36cc..f2dd0cba5aa9 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.23 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index 322c1fca974f..e4ae31eea384 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -17,21 +17,23 @@ acceptance_tests: status: failed basic_read: tests: - - config_path: secrets/config.json - expect_records: - path: "integration_tests/expected_records.txt" - extra_records: yes - empty_streams: - - name: account_performance_report_hourly - bypass_reason: "Hourly reports are disabled, because sync is too long" - - name: ad_group_performance_report_hourly - bypass_reason: "Hourly reports are disabled, because sync is too long" - - name: ad_performance_report_hourly - bypass_reason: "Hourly reports are disabled, because sync is too long" - - name: campaign_performance_report_hourly - bypass_reason: "Hourly reports are disabled, because sync is too long" - - name: keyword_performance_report_hourly - bypass_reason: "Hourly reports are disabled, because sync is too long" + - config_path: secrets/config.json + expect_records: + path: "integration_tests/expected_records.txt" + extra_records: yes + empty_streams: + - name: account_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: ad_group_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: ad_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: campaign_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: keyword_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: geographic_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json index 968b99773072..501c98fb97e3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json @@ -156,6 +156,36 @@ "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "geographic_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "geographic_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "geographic_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, { "stream": { "name": "budget_summary_report", diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt index 7cbcc74a0f7a..e7c35531066f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt @@ -1,7 +1,7 @@ {"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":0.05},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1357897480389129,"Language":null,"Name":"Data Integration Tool","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":4,"Month":12,"Year":2020},"Status":"Paused","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null}},"emitted_at":1675189566179} {"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"ActiveLimited","FinalAppUrls":null,"FinalMobileUrls":{"string":["https://airbyte.io"]},"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.io"]},"ForwardCompatibilityMap":null,"Id":84525295496190,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239221964468,"Name":null,"Type":"TextAsset","Text":"Open data integration for modern data teams"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964466,"Name":null,"Type":"TextAsset","Text":"Get your data pipelines running in minutes. With pre-built or custom connectors"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.io","Headlines":{"AssetLink":[{"Asset":{"Id":10239221964471,"Name":null,"Type":"TextAsset","Text":"Data Integration Tool"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964469,"Name":null,"Type":"TextAsset","Text":"1,000+ Members"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964467,"Name":null,"Type":"TextAsset","Text":"Data Management Software"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null},"emitted_at":1675189577521} {"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":0.1,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":407519039,"MultimediaAdsBidAdjustment":40,"Name":"integration-test-campaign","Status":"Paused","SubType":null,"TimeZone":"Arizona","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false},"emitted_at":1675189585409} -{"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180278106,"Language":"English","LastModifiedByUserId":3,"LastModifiedTime":"2021-08-23T07:06:19.147000","Name":"Daxtarity Inc.","Number":"F149GKV5","ParentCustomerId":251186883,"PaymentMethodId":138188746,"PaymentMethodType":"CreditCard","PrimaryUserId":138225488,"AccountLifeCycleStatus":"Active","TimeStamp":"AAAAAE0a41E=","TimeZone":"Arizona","PauseReason":null,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149004358,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1675189590928} +{"stream": "accounts", "data": {"BillToCustomerId": 251186883, "CurrencyCode": "USD", "AccountFinancialStatus": "ClearFinancialStatus", "Id": 180278106, "Language": "English", "LastModifiedByUserId": 0, "LastModifiedTime": "2023-08-11T03:26:10.277000", "Name": "Daxtarity Inc.", "Number": "F149GKV5", "ParentCustomerId": 251186883, "PaymentMethodId": 138188746, "PaymentMethodType": "CreditCard", "PrimaryUserId": 138225488, "AccountLifeCycleStatus": "Active", "TimeStamp": "AAAAAH1yyMo=", "TimeZone": "Arizona", "PauseReason": null, "ForwardCompatibilityMap": null, "LinkedAgencies": null, "SalesHouseCustomerId": null, "TaxInformation": null, "BackUpPaymentInstrumentId": null, "BillingThresholdAmount": null, "BusinessAddress": {"City": "San Francisco", "CountryCode": "US", "Id": 149004358, "Line1": "350 29th avenue", "Line2": null, "Line3": null, "Line4": null, "PostalCode": "94121", "StateOrProvince": "CA", "TimeStamp": null, "BusinessName": "Daxtarity Inc."}, "AutoTagType": "Inactive", "SoldToPaymentInstrumentId": null, "AccountMode": "Expert"}, "emitted_at": 1692381691611} {"stream":"account_performance_report_daily", "data":{"AccountId": 180278106, "TimePeriod": "2021-08-03", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 1, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679944835117} {"stream":"account_performance_report_weekly", "data": {"AccountId": 180278106, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 1, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 1, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679944953111} {"stream":"account_performance_report_monthly", "data": {"AccountId": 180278106, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 1, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 1, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679945438344} @@ -18,3 +18,6 @@ {"stream": "keyword_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "KeywordId": 84525593559629, "AdId": 84525295496190, "TimePeriod": "2021-08-03", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Smartphone", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "AdGroupName": "Airbyte", "Keyword": "data integration tools", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 0.11, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": null, "MainlineBid": null, "FirstPageBid": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1679951505600} {"stream": "keyword_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "KeywordId": 84525593559629, "AdId": 84525295496190, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "AdGroupName": "Airbyte", "Keyword": "data integration tools", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 0.11, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": null, "MainlineBid": null, "FirstPageBid": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679951543951} {"stream": "keyword_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "KeywordId": 84525593559629, "AdId": 84525295496190, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "AdGroupName": "Airbyte", "Keyword": "data integration tools", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 0.11, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": null, "MainlineBid": null, "FirstPageBid": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679951588461} +{"stream": "geographic_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-06-09", "Country": "Australia", "State": "New South Wales", "MetroArea": null, "City": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Windows", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AdGroupName": "Airbyte", "Ctr": 0.0, "ProximityTargetLocation": null, "Radius": "0", "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "LocationType": "Physical location", "MostSpecificLocation": "2000", "AccountStatus": "Active", "CampaignStatus": "Paused", "AdGroupStatus": "Active", "County": null, "PostalCode": "2000", "LocationId": "122395", "BaseCampaignId": "413444833", "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "100.00%", "AllConversionsQualified": "0.00", "ViewThroughConversionsQualified": null, "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1692275353788} +{"stream": "geographic_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-06-06", "Country": "Australia", "State": null, "MetroArea": null, "City": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Unknown", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AdGroupName": "Airbyte", "Ctr": 0.0, "ProximityTargetLocation": null, "Radius": "0", "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "LocationType": "Physical location", "MostSpecificLocation": "Australia", "AccountStatus": "Active", "CampaignStatus": "Paused", "AdGroupStatus": "Active", "County": null, "PostalCode": null, "LocationId": "9", "BaseCampaignId": "413444833", "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "100.00%", "AllConversionsQualified": "0.00", "ViewThroughConversionsQualified": null, "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1692275162937} +{"stream": "geographic_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-06-01", "Country": "United Arab Emirates", "State": null, "MetroArea": null, "City": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Bing and Yahoo! search", "DeviceOS": "Windows", "TopVsOther": "Bing and Yahoo! search - Top", "BidMatchType": "Broad", "AdGroupName": "Airbyte", "Ctr": 0.0, "ProximityTargetLocation": null, "Radius": "0", "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "LocationType": "Physical location", "MostSpecificLocation": "United Arab Emirates", "AccountStatus": "Active", "CampaignStatus": "Paused", "AdGroupStatus": "Active", "County": null, "PostalCode": null, "LocationId": "218", "BaseCampaignId": "413444833", "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "100.00%", "AllConversionsQualified": "0.00", "ViewThroughConversionsQualified": null, "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1692275633242} diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 960960dd75e2..630df575f0a7 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 0.1.23 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-bing-ads githubIssueLabel: source-bing-ads icon: bingads.svg @@ -26,4 +26,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bing-ads/requirements.txt b/airbyte-integrations/connectors/source-bing-ads/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/requirements.txt +++ b/airbyte-integrations/connectors/source-bing-ads/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index 89bc72e6c01d..c342863f078e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2", "urllib3<2.0"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index eff342fd77a0..0d9c7506493e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -7,7 +7,7 @@ import sys from datetime import datetime, timedelta, timezone from functools import lru_cache -from typing import Any, Iterator, Mapping, Optional +from typing import Any, Iterator, Mapping, Optional, Union from urllib.error import URLError import backoff @@ -96,9 +96,14 @@ def is_token_expiring(self) -> bool: token_updated_expires_in: int = self.oauth.access_token_expires_in_seconds - token_total_lifetime.seconds return False if token_updated_expires_in > self.refresh_token_safe_delta else True - def should_give_up(self, error: WebFault) -> bool: - if isinstance(error, URLError) and (isinstance(error.reason, socket.timeout) or isinstance(error.reason, ssl.SSLError)): - return False + def should_give_up(self, error: Union[WebFault, URLError]) -> bool: + if isinstance(error, URLError): + if ( + isinstance(error.reason, socket.timeout) + or isinstance(error.reason, ssl.SSLError) + or isinstance(error.reason, socket.gaierror) # temporary failure in name resolution + ): + return False error_code = str(errorcode_of_exception(error)) give_up = error_code not in self.retry_on_codes diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json index 091bd133a951..d72fd09dcb25 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json @@ -82,7 +82,7 @@ "Conversions": { "type": ["null", "number"] }, - "ConversionsQualified":{ + "ConversionsQualified": { "type": ["null", "number"] }, "ConversionRate": { diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json new file mode 100644 index 000000000000..f2c65f08ff39 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json @@ -0,0 +1,211 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "Country": { + "type": ["null", "string"] + }, + "State": { + "type": ["null", "string"] + }, + "MetroArea": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "ProximityTargetLocation": { + "type": ["null", "string"] + }, + "Radius": { + "type": ["null", "string"] + }, + "LocationType": { + "type": ["null", "string"] + }, + "MostSpecificLocation": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "County": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "LocationId": { + "type": ["null", "string"] + }, + "BaseCampaignId": { + "type": ["null", "string"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "string"] + }, + "AllConversionsQualified": { + "type": ["null", "string"] + }, + "Neighborhood": { + "type": ["null", "string"] + }, + "ViewThroughRevenue": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AssetGroupId": { + "type": ["null", "string"] + }, + "AssetGroupName": { + "type": ["null", "string"] + }, + "AssetGroupStatus": { + "type": ["null", "string"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index fd78e126644a..dd9ec06b0387 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -690,6 +690,91 @@ class KeywordPerformanceReportMonthly(KeywordPerformanceReport): report_aggregation = "Monthly" +class GeographicPerformanceReport(PerformanceReportsMixin, BingAdsStream): + data_field: str = "" + service_name: str = "ReportingService" + report_name: str = "GeographicPerformanceReport" + operation_name: str = "download_report" + additional_fields: str = "" + cursor_field = "TimePeriod" + report_schema_name = "geographic_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "TimePeriod", + "Country", + "CurrencyCode", + "DeliveredMatchType", + "AdDistribution", + "DeviceType", + "Language", + "Network", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + report_columns = [ + *primary_key, + "MetroArea", + "State", + "City", + "AdGroupName", + "Ctr", + "ProximityTargetLocation", + "Radius", + "Assists", + "ReturnOnAdSpend", + "CostPerAssist", + "LocationType", + "MostSpecificLocation", + "AccountStatus", + "CampaignStatus", + "AdGroupStatus", + "County", + "PostalCode", + "LocationId", + "BaseCampaignId", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "ViewThroughConversions", + "Goal", + "GoalType", + "AbsoluteTopImpressionRatePercent", + "TopImpressionRatePercent", + "AllConversionsQualified", + "ViewThroughConversionsQualified", + "Neighborhood", + "ViewThroughRevenue", + "CampaignType", + "AssetGroupId", + "AssetGroupName", + "AssetGroupStatus", + *CONVERSION_FIELDS, + *AVERAGE_FIELDS, + *ALL_CONVERSION_FIELDS, + *ALL_REVENUE_FIELDS, + *REVENUE_FIELDS, + ] + + +class GeographicPerformanceReportHourly(GeographicPerformanceReport): + report_aggregation = "Hourly" + + +class GeographicPerformanceReportDaily(GeographicPerformanceReport): + report_aggregation = "Daily" + + +class GeographicPerformanceReportWeekly(GeographicPerformanceReport): + report_aggregation = "Weekly" + + +class GeographicPerformanceReportMonthly(GeographicPerformanceReport): + report_aggregation = "Monthly" + + class AccountPerformanceReport(PerformanceReportsMixin, BingAdsStream): data_field: str = "" service_name: str = "ReportingService" @@ -772,6 +857,7 @@ def get_report_streams(self, aggregation_type: str) -> List[Stream]: globals()[f"AdGroupPerformanceReport{aggregation_type}"], globals()[f"AdPerformanceReport{aggregation_type}"], globals()[f"CampaignPerformanceReport{aggregation_type}"], + globals()[f"GeographicPerformanceReport{aggregation_type}"], ] def streams(self, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index b19fea3127cc..23a53d6ac78e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -27,7 +27,7 @@ def logger_mock_fixture(): @patch.object(source_bing_ads.source, "Client") def test_streams_config_based(mocked_client, config): streams = SourceBingAds().streams(config) - assert len(streams) == 25 + assert len(streams) == 29 @patch.object(source_bing_ads.source, "Client") diff --git a/airbyte-integrations/connectors/source-braintree/Dockerfile b/airbyte-integrations/connectors/source-braintree/Dockerfile index 9c6eedaecba9..eaa0fafbec21 100644 --- a/airbyte-integrations/connectors/source-braintree/Dockerfile +++ b/airbyte-integrations/connectors/source-braintree/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-braintree diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json index 2f474d84f17d..15bf66616f91 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json @@ -1,11 +1,14 @@ { "customer_stream": { - "created_at": "2222" + "created_at": "2222-01-01 00:00:00" }, "dispute_stream": { - "received_date": "2222" + "received_date": "2222-01-01 00:00:00" }, "transaction_stream": { - "created_at": "2222" + "created_at": "2222-01-01 00:00:00" + }, + "subscription_stream": { + "created_at": "2222-01-01 00:00:00" } } diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog.json index e7eada6b3d7a..af0695a74427 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog.json @@ -70,7 +70,7 @@ "stream": { "name": "subscription_stream", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": false, "source_defined_primary_key": [["id"]] }, diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog_incremental.json index 6bebe1984999..99c304977dec 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/configured_catalog_incremental.json @@ -35,6 +35,18 @@ }, "sync_mode": "incremental", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "subscription_stream", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-braintree/integration_tests/sample_state.json index 912aa49c762b..b8d0d1ec5d5b 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/sample_state.json @@ -1,11 +1,14 @@ { "customer_stream": { - "created_at": "2021-08-01" + "created_at": "2023-08-08 00:00:00" }, "dispute_stream": { - "received_date": "2021-08-01" + "received_date": "2023-08-08 00:00:00" }, "transaction_stream": { - "created_at": "2021-08-01" + "created_at": "2023-08-08 00:00:00" + }, + "subscription_stream": { + "created_at": "2023-08-08 00:00:00" } } diff --git a/airbyte-integrations/connectors/source-braintree/metadata.yaml b/airbyte-integrations/connectors/source-braintree/metadata.yaml index 641afbdba0ec..330156721e05 100644 --- a/airbyte-integrations/connectors/source-braintree/metadata.yaml +++ b/airbyte-integrations/connectors/source-braintree/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 63cea06f-1c75-458d-88fe-ad48c7cb27fd - dockerImageTag: 0.1.5 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-braintree + documentationUrl: https://docs.airbyte.com/integrations/sources/braintree githubIssueLabel: source-braintree icon: braintree.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/braintree + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-braintree/requirements.txt b/airbyte-integrations/connectors/source-braintree/requirements.txt index 9ce85523c234..5b8864c417d3 100644 --- a/airbyte-integrations/connectors/source-braintree/requirements.txt +++ b/airbyte-integrations/connectors/source-braintree/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/connector-acceptance-test --e . diff --git a/airbyte-integrations/connectors/source-braintree/setup.py b/airbyte-integrations/connectors/source-braintree/setup.py index cba7d648a2d7..1930cb0c4fdb 100644 --- a/airbyte-integrations/connectors/source-braintree/setup.py +++ b/airbyte-integrations/connectors/source-braintree/setup.py @@ -5,18 +5,22 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "braintree~=4.18.1", "pendulum~=1.5.1", "inflection~=0.5.1", "backoff~=1.11.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "braintree~=4.21.0"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "freezegun~=1.1.0", "responses~=0.13.3"] +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] setup( - name="source_braintree", - description="Source implementation for Braintree.", + name="source_braintree_no_code", + description="Source implementation for Braintree No Code.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/__init__.py b/airbyte-integrations/connectors/source-braintree/source_braintree/__init__.py index 6abb596c712a..b123306f51ae 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/__init__.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml b/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml new file mode 100644 index 000000000000..ead085ac84e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml @@ -0,0 +1,15977 @@ +version: 0.50.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - merchant_account_stream +streams: + - type: DeclarativeStream + name: merchant_account_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + "$schema": http://json-schema.org/schema# + additionalProperties: true + required: + - id + properties: + business_details: + oneOf: + - type: "null" + - type: object + properties: + address_details: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + dba_name: + type: + - "null" + - string + legal_name: + type: + - "null" + - string + tax_id: + type: + - "null" + - string + currency_iso_code: + type: + - "null" + - string + funding_details: + oneOf: + - type: "null" + - type: object + properties: + account_number_last_4: + type: + - "null" + - string + descriptor: + type: + - "null" + - string + destination: + type: + - "null" + - string + email: + type: + - "null" + - string + mobile_phone: + type: + - "null" + - string + routing_number: + type: + - "null" + - string + id: + type: + - string + individual_details: + oneOf: + - type: "null" + - type: object + properties: + address_details: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + date_of_birth: + type: + - "null" + - string + email: + type: + - "null" + - string + first_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + phone: + type: + - "null" + - string + ssn_last_4: + type: + - "null" + - string + status: + type: + - "null" + - string + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /merchant_accounts + http_method: GET + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.MerchantAccountExtractor" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + pagination_strategy: + type: PageIncrement + start_from_page: 1 + - type: DeclarativeStream + name: customer_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + "$schema": http://json-schema.org/schema# + type: object + additionalProperties: true + required: + - id + - created_at + properties: + addresses: + type: + - "null" + - array + items: + type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + android_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + apple_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + company: + type: + - "null" + - string + created_at: + type: + - string + format: date-time + credit_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + custom_fields: + type: + - "null" + - string + email: + type: + - "null" + - string + fax: + type: + - "null" + - string + first_name: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - string + last_name: + type: + - "null" + - string + masterpass_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + payment_methods: + type: + - "null" + - array + items: + anyOf: + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + account_holder_name: + type: + - "null" + - string + account_type: + type: + - "null" + - string + ach_mandate: + type: + - "null" + - string + bank_name: + type: + - "null" + - string + business_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + owner_id: + type: + - "null" + - string + ownership_type: + type: + - "null" + - string + plaid_verified_at: + type: + - "null" + - string + format: date-time + routing_number: + type: + - "null" + - string + verifiable: + type: + - "null" + - boolean + verified: + type: + - "null" + - boolean + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + paypal_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + phone: + type: + - "null" + - string + samsung_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + updated_at: + type: + - "null" + - string + format: date-time + us_bank_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + account_holder_name: + type: + - "null" + - string + account_type: + type: + - "null" + - string + ach_mandate: + type: + - "null" + - string + bank_name: + type: + - "null" + - string + business_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + owner_id: + type: + - "null" + - string + ownership_type: + type: + - "null" + - string + plaid_verified_at: + type: + - "null" + - string + format: date-time + routing_number: + type: + - "null" + - string + verifiable: + type: + - "null" + - boolean + verified: + type: + - "null" + - boolean + venmo_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + visa_checkout_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + website: + type: + - "null" + - string + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /customers/advanced_search + http_method: POST + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: + search: + created_at: + min: "{{ stream_interval.start_time }}" + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.CustomerExtractor" + paginator: + type: NoPagination + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_at + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + - type: DeclarativeStream + name: discount_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + "$schema": http://json-schema.org/schema# + type: object + additionalProperties: true + required: + - id + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /discounts/ + http_method: GET + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.DiscountExtractor" + paginator: + type: NoPagination + - type: DeclarativeStream + name: plan_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + "$schema": http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - id + properties: + add_ons: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + billing_day_of_month: + type: + - "null" + - number + billing_frequency: + type: + - "null" + - number + created_at: + type: + - "null" + - string + format: date-time + currency_iso_code: + type: + - "null" + - string + description: + type: + - "null" + - string + discounts: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + id: + type: + - string + name: + type: + - "null" + - string + number_of_billing_cycles: + type: + - "null" + - number + price: + type: + - "null" + - number + trial_duration: + type: + - "null" + - number + trial_duration_unit: + type: + - "null" + - string + trial_period: + type: + - "null" + - boolean + updated_at: + type: + - "null" + - string + format: date-time + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /plans/ + http_method: GET + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.PlanExtractor" + paginator: + type: NoPagination + - type: DeclarativeStream + name: transaction_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + "$schema": http://json-schema.org/schema# + type: object + additionalProperties: true + required: + - id + - created_at + properties: + acquirer_reference_number: + type: + - "null" + - string + additional_processor_response: + type: + - "null" + - string + amount: + type: + - "null" + - string + android_pay_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + apple_pay_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + authorization_expires_at: + type: + - "null" + - string + format: date-time + avs_error_response_code: + type: + - "null" + - string + avs_postal_code_response_code: + type: + - "null" + - string + avs_street_address_response_code: + type: + - "null" + - string + billing_details: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + channel: + type: + - "null" + - string + created_at: + type: + - string + format: date-time + credit_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + currency_iso_code: + type: + - "null" + - string + custom_fields: + type: + - "null" + - string + customer_details: + oneOf: + - type: "null" + - type: object + properties: + addresses: + type: + - "null" + - array + items: + type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + android_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + apple_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + company: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + credit_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + custom_fields: + type: + - "null" + - string + email: + type: + - "null" + - string + fax: + type: + - "null" + - string + first_name: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + masterpass_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + payment_methods: + type: + - "null" + - array + items: + anyOf: + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + account_holder_name: + type: + - "null" + - string + account_type: + type: + - "null" + - string + ach_mandate: + type: + - "null" + - string + bank_name: + type: + - "null" + - string + business_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + owner_id: + type: + - "null" + - string + ownership_type: + type: + - "null" + - string + plaid_verified_at: + type: + - "null" + - string + format: date-time + routing_number: + type: + - "null" + - string + verifiable: + type: + - "null" + - boolean + verified: + type: + - "null" + - boolean + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + paypal_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + phone: + type: + - "null" + - string + samsung_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + updated_at: + type: + - "null" + - string + format: date-time + us_bank_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + account_holder_name: + type: + - "null" + - string + account_type: + type: + - "null" + - string + ach_mandate: + type: + - "null" + - string + bank_name: + type: + - "null" + - string + business_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + owner_id: + type: + - "null" + - string + ownership_type: + type: + - "null" + - string + plaid_verified_at: + type: + - "null" + - string + format: date-time + routing_number: + type: + - "null" + - string + verifiable: + type: + - "null" + - boolean + verified: + type: + - "null" + - boolean + venmo_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + visa_checkout_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + website: + type: + - "null" + - string + cvv_response_code: + type: + - "null" + - string + disbursement_details: + oneOf: + - type: "null" + - type: object + properties: + disbursement_date: + type: + - "null" + - string + format: date + funds_held: + type: + - "null" + - boolean + settlement_amount: + type: + - "null" + - number + settlement_base_currency_exchange_rate: + type: + - "null" + - number + settlement_currency_exchange_rate: + type: + - "null" + - number + settlement_currency_iso_code: + type: + - "null" + - string + success: + type: + - "null" + - boolean + discount_amount: + type: + - "null" + - number + discounts: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + disputes: + type: + - "null" + - array + items: + type: object + properties: + amount_disputed: + type: + - "null" + - number + amount_won: + type: + - "null" + - number + case_number: + type: + - "null" + - string + chargeback_protection_level: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + currency_iso_code: + type: + - "null" + - string + evidence: + anyOf: + - type: object + properties: + created_at: + type: + - "null" + - string + format: date-time + id: + type: + - "null" + - string + sent_to_processor_at: + type: + - "null" + - string + format: date-time + url: + type: + - "null" + - string + comment: + type: + - "null" + - string + - type: array + items: + type: object + properties: + created_at: + type: + - "null" + - string + format: date-time + id: + type: + - "null" + - string + sent_to_processor_at: + type: + - "null" + - string + format: date-time + url: + type: + - "null" + - string + comment: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + merchant_account_id: + type: + - "null" + - string + original_dispute_id: + type: + - "null" + - string + paypal_messages: + type: + - "null" + - array + items: + type: object + properties: + message: + type: + - "null" + - string + send_at: + type: + - "null" + - string + format: date-time + sender: + type: + - "null" + - string + processor_comments: + type: + - "null" + - string + reason: + type: + - "null" + - string + reason_code: + type: + - "null" + - string + reason_description: + type: + - "null" + - string + received_date: + type: + - "null" + - string + format: date + reference_number: + type: + - "null" + - string + reply_by_date: + type: + - "null" + - string + format: date + status: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + escrow_status: + type: + - "null" + - string + gateway_rejection_reason: + type: + - "null" + - string + global_id: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - string + installment_count: + type: + - "null" + - number + masterpass_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + merchant_account_id: + type: + - "null" + - string + merchant_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + merchant_identification_number: + type: + - "null" + - string + merchant_name: + type: + - "null" + - string + network_response_code: + type: + - "null" + - string + network_response_text: + type: + - "null" + - string + network_transaction_id: + type: + - "null" + - string + order_id: + type: + - "null" + - string + payment_instrument_type: + type: + - "null" + - string + pin_verified: + type: + - "null" + - boolean + plan_id: + type: + - "null" + - string + processed_with_network_token: + type: + - "null" + - boolean + processor_authorization_code: + type: + - "null" + - string + processor_response_code: + type: + - "null" + - string + processor_response_text: + type: + - "null" + - string + processor_response_type: + type: + - "null" + - string + processor_settlement_response_code: + type: + - "null" + - string + processor_settlement_response_text: + type: + - "null" + - string + purchase_order_number: + type: + - "null" + - string + paypal_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + recurring: + type: + - "null" + - boolean + refund_ids: + type: + - "null" + - array + items: + type: string + refund_global_ids: + type: + - "null" + - array + items: + type: string + refunded_transaction_id: + type: + - "null" + - string + response_emv_data: + type: + - "null" + - string + retrieval_reference_number: + type: + - "null" + - string + samsung_pay_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + sca_exemption_requested: + type: + - "null" + - string + service_fee_amount: + type: + - "null" + - number + settlement_batch_id: + type: + - "null" + - string + shipping_amount: + type: + - "null" + - number + shipping_details: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + ships_from_postal_code: + type: + - "null" + - string + status: + type: + - "null" + - string + status_history: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + status: + type: + - "null" + - string + timestamp: + type: + - "null" + - string + format: date-time + transaction_source: + type: + - "null" + - string + user: + type: + - "null" + - string + subscription_details: + oneOf: + - type: "null" + - type: object + properties: + billing_period_end_date: + type: + - "null" + - string + format: date-time + billing_period_start_date: + type: + - "null" + - string + format: date-time + subscription_id: + type: + - "null" + - string + tax_amount: + type: + - "null" + - number + tax_exempt: + type: + - "null" + - boolean + terminal_identification_number: + type: + - "null" + - string + type: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + venmo_account_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + visa_checkout_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + voice_referral_number: + type: + - "null" + - string + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /transactions/advanced_search + http_method: POST + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: + search: + created_at: + min: "{{ stream_interval.start_time }}" + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.TransactionExtractor" + paginator: + type: NoPagination + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_at + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + - type: DeclarativeStream + name: subscription_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + "$schema": http://json-schema.org/schema# + type: object + additionalProperties: true + required: + - id + - created_at + properties: + add_ons: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + balance: + type: + - "null" + - number + billing_day_of_month: + type: + - "null" + - number + billing_period_start_date: + type: + - "null" + - string + format: date-time + created_at: + type: + - string + format: date-time + current_billing_cycle: + type: + - "null" + - number + days_past_due: + type: + - "null" + - number + description: + type: + - "null" + - string + discounts: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + failure_count: + type: + - "null" + - number + first_billing_date: + type: + - "null" + - string + format: date + id: + type: + - string + merchant_account_id: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + next_bill_amount: + type: + - "null" + - number + next_billing_date: + type: + - "null" + - string + format: date + next_billing_period_amount: + type: + - "null" + - number + number_of_billing_cycles: + type: + - "null" + - number + paid_through_date: + type: + - "null" + - string + format: date-time + payment_method_token: + type: + - "null" + - string + plan_id: + type: + - "null" + - string + price: + type: + - "null" + - number + status: + type: + - "null" + - string + transactions: + type: + - "null" + - array + items: + type: object + properties: + acquirer_reference_number: + type: + - "null" + - string + additional_processor_response: + type: + - "null" + - string + amount: + type: + - "null" + - string + android_pay_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + apple_pay_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + authorization_expires_at: + type: + - "null" + - string + format: date-time + avs_error_response_code: + type: + - "null" + - string + avs_postal_code_response_code: + type: + - "null" + - string + avs_street_address_response_code: + type: + - "null" + - string + billing_details: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + channel: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + credit_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + currency_iso_code: + type: + - "null" + - string + custom_fields: + type: + - "null" + - string + customer_details: + oneOf: + - type: "null" + - type: object + properties: + addresses: + type: + - "null" + - array + items: + type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + android_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + apple_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + company: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + credit_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + custom_fields: + type: + - "null" + - string + email: + type: + - "null" + - string + fax: + type: + - "null" + - string + first_name: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + masterpass_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + payment_methods: + type: + - "null" + - array + items: + anyOf: + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + google_transaction_id: + type: + - "null" + - string + source_card_type: + type: + - "null" + - string + source_description: + type: + - "null" + - string + is_network_tokenized: + type: + - "null" + - boolean + source_card_last_4: + type: + - "null" + - string + virtual_card_last_4: + type: + - "null" + - string + virtual_card_type: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + payment_instrument_name: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + account_holder_name: + type: + - "null" + - string + account_type: + type: + - "null" + - string + ach_mandate: + type: + - "null" + - string + bank_name: + type: + - "null" + - string + business_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + owner_id: + type: + - "null" + - string + ownership_type: + type: + - "null" + - string + plaid_verified_at: + type: + - "null" + - string + format: date-time + routing_number: + type: + - "null" + - string + verifiable: + type: + - "null" + - boolean + verified: + type: + - "null" + - boolean + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + paypal_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + phone: + type: + - "null" + - string + samsung_pay_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + updated_at: + type: + - "null" + - string + format: date-time + us_bank_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + account_holder_name: + type: + - "null" + - string + account_type: + type: + - "null" + - string + ach_mandate: + type: + - "null" + - string + bank_name: + type: + - "null" + - string + business_name: + type: + - "null" + - string + last_name: + type: + - "null" + - string + owner_id: + type: + - "null" + - string + ownership_type: + type: + - "null" + - string + plaid_verified_at: + type: + - "null" + - string + format: date-time + routing_number: + type: + - "null" + - string + verifiable: + type: + - "null" + - boolean + verified: + type: + - "null" + - boolean + venmo_accounts: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + visa_checkout_cards: + type: + - "null" + - array + items: + type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + website: + type: + - "null" + - string + cvv_response_code: + type: + - "null" + - string + disbursement_details: + oneOf: + - type: "null" + - type: object + properties: + disbursement_date: + type: + - "null" + - string + format: date + funds_held: + type: + - "null" + - boolean + settlement_amount: + type: + - "null" + - number + settlement_base_currency_exchange_rate: + type: + - "null" + - number + settlement_currency_exchange_rate: + type: + - "null" + - number + settlement_currency_iso_code: + type: + - "null" + - string + success: + type: + - "null" + - boolean + discount_amount: + type: + - "null" + - number + discounts: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + current_billing_cycle: + type: + - "null" + - number + description: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + name: + type: + - "null" + - string + never_expires: + type: + - "null" + - boolean + number_of_billing_cycles: + type: + - "null" + - number + quantity: + type: + - "null" + - number + disputes: + type: + - "null" + - array + items: + type: object + properties: + amount_disputed: + type: + - "null" + - number + amount_won: + type: + - "null" + - number + case_number: + type: + - "null" + - string + chargeback_protection_level: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + currency_iso_code: + type: + - "null" + - string + evidence: + anyOf: + - type: object + properties: + created_at: + type: + - "null" + - string + format: date-time + id: + type: + - "null" + - string + sent_to_processor_at: + type: + - "null" + - string + format: date-time + url: + type: + - "null" + - string + comment: + type: + - "null" + - string + - type: array + items: + type: object + properties: + created_at: + type: + - "null" + - string + format: date-time + id: + type: + - "null" + - string + sent_to_processor_at: + type: + - "null" + - string + format: date-time + url: + type: + - "null" + - string + comment: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - "null" + - string + kind: + type: + - "null" + - string + merchant_account_id: + type: + - "null" + - string + original_dispute_id: + type: + - "null" + - string + paypal_messages: + type: + - "null" + - array + items: + type: object + properties: + message: + type: + - "null" + - string + send_at: + type: + - "null" + - string + format: date-time + sender: + type: + - "null" + - string + processor_comments: + type: + - "null" + - string + reason: + type: + - "null" + - string + reason_code: + type: + - "null" + - string + reason_description: + type: + - "null" + - string + received_date: + type: + - "null" + - string + format: date + reference_number: + type: + - "null" + - string + reply_by_date: + type: + - "null" + - string + format: date + status: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + escrow_status: + type: + - "null" + - string + gateway_rejection_reason: + type: + - "null" + - string + global_id: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - "null" + - string + installment_count: + type: + - "null" + - number + masterpass_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + merchant_account_id: + type: + - "null" + - string + merchant_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + merchant_identification_number: + type: + - "null" + - string + merchant_name: + type: + - "null" + - string + network_response_code: + type: + - "null" + - string + network_response_text: + type: + - "null" + - string + network_transaction_id: + type: + - "null" + - string + order_id: + type: + - "null" + - string + payment_instrument_type: + type: + - "null" + - string + pin_verified: + type: + - "null" + - boolean + plan_id: + type: + - "null" + - string + processed_with_network_token: + type: + - "null" + - boolean + processor_authorization_code: + type: + - "null" + - string + processor_response_code: + type: + - "null" + - string + processor_response_text: + type: + - "null" + - string + processor_response_type: + type: + - "null" + - string + processor_settlement_response_code: + type: + - "null" + - string + processor_settlement_response_text: + type: + - "null" + - string + purchase_order_number: + type: + - "null" + - string + paypal_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + billing_agreement_id: + type: + - "null" + - string + email: + type: + - "null" + - string + payer_id: + type: + - "null" + - string + revoked_at: + type: + - "null" + - string + format: date-time + recurring: + type: + - "null" + - boolean + refund_ids: + type: + - "null" + - array + items: + type: string + refund_global_ids: + type: + - "null" + - array + items: + type: string + refunded_transaction_id: + type: + - "null" + - string + response_emv_data: + type: + - "null" + - string + retrieval_reference_number: + type: + - "null" + - string + samsung_pay_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + sca_exemption_requested: + type: + - "null" + - string + service_fee_amount: + type: + - "null" + - number + settlement_batch_id: + type: + - "null" + - string + shipping_amount: + type: + - "null" + - number + shipping_details: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + ships_from_postal_code: + type: + - "null" + - string + status: + type: + - "null" + - string + status_history: + type: + - "null" + - array + items: + type: object + properties: + amount: + type: + - "null" + - number + status: + type: + - "null" + - string + timestamp: + type: + - "null" + - string + format: date-time + transaction_source: + type: + - "null" + - string + user: + type: + - "null" + - string + subscription_details: + oneOf: + - type: "null" + - type: object + properties: + billing_period_end_date: + type: + - "null" + - string + format: date-time + billing_period_start_date: + type: + - "null" + - string + format: date-time + subscription_id: + type: + - "null" + - string + tax_amount: + type: + - "null" + - number + tax_exempt: + type: + - "null" + - boolean + terminal_identification_number: + type: + - "null" + - string + type: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + venmo_account_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + source_description: + type: + - "null" + - string + username: + type: + - "null" + - string + venmo_user_id: + type: + - "null" + - string + visa_checkout_card_details: + oneOf: + - type: "null" + - type: object + properties: + billing_address: + oneOf: + - type: "null" + - type: object + properties: + company: + type: + - "null" + - string + country_code_alpha2: + type: + - "null" + - string + country_code_alpha3: + type: + - "null" + - string + country_code_numeric: + type: + - "null" + - string + country_name: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + extended_address: + type: + - "null" + - string + first_name: + type: + - "null" + - string + id: + type: + - "null" + - string + last_name: + type: + - "null" + - string + locality: + type: + - "null" + - string + postal_code: + type: + - "null" + - string + region: + type: + - "null" + - string + street_address: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + bin: + type: + - "null" + - string + card_type: + type: + - "null" + - string + cardholder_name: + type: + - "null" + - string + commercial: + type: + - "null" + - string + country_of_issuance: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + customer_id: + type: + - "null" + - string + customer_location: + type: + - "null" + - string + debit: + type: + - "null" + - string + default: + type: + - "null" + - boolean + durbin_regulated: + type: + - "null" + - string + expiration_date: + type: + - "null" + - string + expiration_month: + type: + - "null" + - string + expiration_year: + type: + - "null" + - string + expired: + type: + - "null" + - boolean + healthcare: + type: + - "null" + - string + image_url: + type: + - "null" + - string + issuing_bank: + type: + - "null" + - string + last_4: + type: + - "null" + - string + masked_number: + type: + - "null" + - string + payroll: + type: + - "null" + - string + prepaid: + type: + - "null" + - string + product_id: + type: + - "null" + - string + token: + type: + - "null" + - string + unique_number_identifier: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + voice_referral_number: + type: + - "null" + - string + trial_duration: + type: + - "null" + - number + trial_duration_unit: + type: + - "null" + - string + trial_period: + type: + - "null" + - boolean + updated_at: + type: + - "null" + - string + format: date-time + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /subscriptions/advanced_search + http_method: GET + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: + search: + created_at: + min: "{{ stream_interval.start_time }}" + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.SubscriptionExtractor" + paginator: + type: NoPagination + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_at + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + - type: DeclarativeStream + name: dispute_stream + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + "$schema": http://json-schema.org/schema# + type: object + additionalProperties: true + required: + - id + - received_date + properties: + amount_disputed: + type: + - "null" + - number + amount_won: + type: + - "null" + - number + case_number: + type: + - "null" + - string + chargeback_protection_level: + type: + - "null" + - string + created_at: + type: + - "null" + - string + format: date-time + currency_iso_code: + type: + - "null" + - string + evidence: + anyOf: + - type: object + properties: + created_at: + type: + - "null" + - string + format: date-time + id: + type: + - "null" + - string + sent_to_processor_at: + type: + - "null" + - string + format: date-time + url: + type: + - "null" + - string + comment: + type: + - "null" + - string + - type: array + items: + type: object + properties: + created_at: + type: + - "null" + - string + format: date-time + id: + type: + - "null" + - string + sent_to_processor_at: + type: + - "null" + - string + format: date-time + url: + type: + - "null" + - string + comment: + type: + - "null" + - string + graphql_id: + type: + - "null" + - string + id: + type: + - string + kind: + type: + - "null" + - string + merchant_account_id: + type: + - "null" + - string + original_dispute_id: + type: + - "null" + - string + paypal_messages: + type: + - "null" + - array + items: + type: object + properties: + message: + type: + - "null" + - string + send_at: + type: + - "null" + - string + format: date-time + sender: + type: + - "null" + - string + processor_comments: + type: + - "null" + - string + reason: + type: + - "null" + - string + reason_code: + type: + - "null" + - string + reason_description: + type: + - "null" + - string + received_date: + type: + - string + format: date + reference_number: + type: + - "null" + - string + reply_by_date: + type: + - "null" + - string + format: date + status: + type: + - "null" + - string + updated_at: + type: + - "null" + - string + format: date-time + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://{% if config['environment'] == 'Development' %}localhost:3000{% elif config['environment'] == 'Sandbox' %}api.sandbox.braintreegateway.com:443{% elif config['environment'] == 'Qa' %}gateway.qa.braintreepayments.com:443{% elif config['environment'] == 'Production' %}api.braintreegateway.com:443{% endif %}/merchants/{{config['merchant_id']}} + path: /disputes/advanced_search + http_method: POST + request_parameters: {} + request_headers: + X-ApiVersion: "6" + authenticator: + type: BasicHttpAuthenticator + password: "{{ config['private_key'] }}" + username: "{{ config['public_key'] }}" + request_body_json: + search: + received_date: + min: "{{ stream_interval.start_time }}" + record_selector: + type: RecordSelector + extractor: + class_name: "source_braintree.source.DisputeExtractor" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + pagination_strategy: + type: PageIncrement + start_from_page: 1 + incremental_sync: + type: DatetimeBasedCursor + cursor_field: received_date + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" +spec: + connection_specification: + title: Braintree Spec + type: object + properties: + merchant_id: + title: Merchant ID + description: >- + The unique identifier for your entire gateway account. See the docs + for more information on how to obtain this ID. + name: Merchant ID + type: string + public_key: + title: Public Key + description: >- + Braintree Public Key. See the docs + for more information on how to obtain this key. + name: Public Key + type: string + private_key: + title: Private Key + description: >- + Braintree Private Key. See the docs + for more information on how to obtain this key. + name: Private Key + airbyte_secret: true + type: string + start_date: + title: Start Date + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + name: Start Date + examples: + - "2020" + - "2020-12-30" + - "2020-11-22 20:20:05" + type: string + format: date-time + environment: + title: Environment + description: Environment specifies where the data will come from. + name: Environment + examples: + - sandbox + - production + - qa + - development + enum: + - Development + - Sandbox + - Qa + - Production + type: string + required: + - merchant_id + - public_key + - private_key + - environment + documentation_url: https://docs.airbyte.com/integrations/sources/braintree + type: Spec +metadata: + autoImportSchema: + merchant_account_stream: true + customer_stream: true + discount_stream: true + plan_stream: true + transaction_stream: true + subscription_stream: true + dispute_stream: true diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/subscription.py b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/subscription.py index cbaf83a3062f..fd18ff7ff44f 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/subscription.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/subscription.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from datetime import datetime +from datetime import date, datetime from decimal import Decimal from typing import List @@ -22,12 +22,12 @@ class Subscription(CatalogModel): description: str discounts: List[Discount] failure_count: Decimal - first_billing_date: datetime + first_billing_date: date id: str merchant_account_id: str never_expires: bool next_bill_amount: Decimal - next_billing_date: datetime + next_billing_date: date next_billing_period_amount: Decimal number_of_billing_cycles: Decimal paid_through_date: datetime diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/source.py b/airbyte-integrations/connectors/source-braintree/source_braintree/source.py index 6cad919bbe9d..29d30182144c 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/source.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/source.py @@ -2,53 +2,185 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from dataclasses import dataclass +from typing import List, Union -from typing import Any, List, Mapping, Optional, Tuple - -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification -from airbyte_cdk.sources.abstract_source import AbstractSource -from airbyte_cdk.sources.streams.core import Stream -from source_braintree.spec import BraintreeConfig -from source_braintree.streams import ( - BraintreeStream, - CustomerStream, - DiscountStream, - DisputeStream, - MerchantAccountStream, - PlanStream, - SubscriptionStream, - TransactionStream, -) - - -class SourceBraintree(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: - """ - :return: AirbyteConnectionStatus indicating a Success or Failure - """ - try: - config = BraintreeConfig(**config) - gateway = BraintreeStream.create_gateway(config) - gateway.customer.all() - except Exception as exc: - return False, repr(exc) - - return True, "" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - config = BraintreeConfig(**config) +import requests +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.types import Record +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from braintree.attribute_getter import AttributeGetter +from braintree.customer import Customer as BCustomer +from braintree.discount import Discount as BDiscount +from braintree.dispute import Dispute as BDispute +from braintree.merchant_account.merchant_account import MerchantAccount as BMerchantAccount +from braintree.plan import Plan as BPlan +from braintree.subscription import Subscription as BSubscription +from braintree.transaction import Transaction as BTransaction +from braintree.util.xml_util import XmlUtil +from source_braintree.schemas import Customer, Discount, Dispute, MerchantAccount, Plan, Subscription, Transaction + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +@dataclass +class BraintreeExtractor(RecordExtractor): + """ + Extractor Template for all BrainTree streams. + """ + + @staticmethod + def _extract_as_array(results, attribute): + if attribute not in results: + return [] + + value = results[attribute] + if not isinstance(value, list): + value = [value] + return value + + def _get_json_from_resource(self, resource_obj: Union[AttributeGetter, List[AttributeGetter]]): + if isinstance(resource_obj, list): + return [obj if not isinstance(obj, AttributeGetter) else self._get_json_from_resource(obj) for obj in resource_obj] + obj_dict = resource_obj.__dict__ + result = dict() + for attr in obj_dict: + if not attr.startswith("_"): + if callable(obj_dict[attr]): + continue + result[attr] = ( + self._get_json_from_resource(obj_dict[attr]) if isinstance(obj_dict[attr], (AttributeGetter, list)) else obj_dict[attr] + ) + return result + + +@dataclass +class MerchantAccountExtractor(BraintreeExtractor): + """ + Extractor for Merchant Accounts stream. + It parses output XML and finds all `Merchant Account` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text)["merchant_accounts"] + merchant_accounts = self._extract_as_array(data, "merchant_account") return [ - CustomerStream(config), - DiscountStream(config), - DisputeStream(config), - TransactionStream(config), - MerchantAccountStream(config), - PlanStream(config), - SubscriptionStream(config), + MerchantAccount(**self._get_json_from_resource(BMerchantAccount(None, merchant_account))).dict(exclude_unset=True) + for merchant_account in merchant_accounts ] - def spec(self, logger: AirbyteLogger) -> ConnectorSpecification: - return ConnectorSpecification( - connectionSpecification=BraintreeConfig.schema(), documentationUrl="https://docs.airbyte.com/integrations/sources/braintree" - ) + +@dataclass +class CustomerExtractor(BraintreeExtractor): + """ + Extractor for Customers stream. + It parses output XML and finds all `Customer` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text)["customers"] + customers = self._extract_as_array(data, "customer") + return [Customer(**self._get_json_from_resource(BCustomer(None, customer))).dict(exclude_unset=True) for customer in customers] + + +@dataclass +class DiscountExtractor(BraintreeExtractor): + """ + Extractor for Discounts stream. + It parses output XML and finds all `Discount` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text) + discounts = self._extract_as_array(data, "discounts") + return [Discount(**self._get_json_from_resource(BDiscount(None, discount))).dict(exclude_unset=True) for discount in discounts] + + +@dataclass +class TransactionExtractor(BraintreeExtractor): + """ + Extractor for Transactions stream. + It parses output XML and finds all `Transaction` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text)["credit_card_transactions"] + transactions = self._extract_as_array(data, "transaction") + return [ + Transaction(**self._get_json_from_resource(BTransaction(None, transaction))).dict(exclude_unset=True) + for transaction in transactions + ] + + +@dataclass +class SubscriptionExtractor(BraintreeExtractor): + """ + Extractor for Subscriptions stream. + It parses output XML and finds all `Subscription` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text)["subscriptions"] + subscriptions = self._extract_as_array(data, "subscription") + return [ + Subscription(**self._get_json_from_resource(BSubscription(None, subscription))).dict(exclude_unset=True) + for subscription in subscriptions + ] + + +@dataclass +class PlanExtractor(BraintreeExtractor): + """ + Extractor for Plans stream. + It parses output XML and finds all `Plan` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text) + plans = self._extract_as_array(data, "plans") + return [Plan(**self._get_json_from_resource(BPlan(None, plan))).dict(exclude_unset=True) for plan in plans] + + +@dataclass +class DisputeExtractor(BraintreeExtractor): + """ + Extractor for Disputes stream. + It parses output XML and finds all `Dispute` occurrences in it. + """ + + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + data = XmlUtil.dict_from_xml(response.text)["disputes"] + disputes = self._extract_as_array(data, "dispute") + return [Dispute(**self._get_json_from_resource(BDispute(dispute))).dict(exclude_unset=True) for dispute in disputes] + + +# Declarative Source +class SourceBraintree(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/spec.py b/airbyte-integrations/connectors/source-braintree/source_braintree/spec.py deleted file mode 100644 index 81a9cdb8c71a..000000000000 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/spec.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime -from enum import Enum - -from inflection import camelize -from pydantic import BaseModel, Field, validator - - -class Environment(str, Enum): - DEV = "Development" - SANDBOX = "Sandbox" - QA = "Qa" - PROD = "Production" - - -class BraintreeConfig(BaseModel): - class Config: - title = "Braintree Spec" - - merchant_id: str = Field( - name="Merchant ID", - title="Merchant ID", - description='The unique identifier for your entire gateway account. See the docs for more information on how to obtain this ID.', - ) - public_key: str = Field( - name="Public Key", - title="Public Key", - description='Braintree Public Key. See the docs for more information on how to obtain this key.', - ) - private_key: str = Field( - name="Private Key", - title="Private Key", - description='Braintree Private Key. See the docs for more information on how to obtain this key.', - airbyte_secret=True, - ) - start_date: datetime = Field( - None, - name="Start Date", - title="Start Date", - description="UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - examples=["2020", "2020-12-30", "2020-11-22 20:20:05"], - ) - environment: Environment = Field( - name="Environment", - title="Environment", - description="Environment specifies where the data will come from.", - examples=["sandbox", "production", "qa", "development"], - ) - - @validator("environment") - def to_lower_case(cls, v): - return v.lower() - - @validator("environment", pre=True) - def to_camel_case(cls, v): - return camelize(v) - - @classmethod - def schema(cls, **kwargs): - schema = super().schema(**kwargs) - if "definitions" in schema: - schema["definitions"]["Environment"].pop("description") - schema["properties"]["environment"].update(schema["definitions"]["Environment"]) - schema["properties"]["environment"].pop("allOf", None) - del schema["definitions"] - return schema diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/streams.py b/airbyte-integrations/connectors/source-braintree/source_braintree/streams.py deleted file mode 100644 index 418715ecaf0f..000000000000 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/streams.py +++ /dev/null @@ -1,211 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from datetime import datetime -from typing import Any, Generator, Iterable, List, Mapping, Optional, Union - -import backoff -import braintree -import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import Stream -from braintree.attribute_getter import AttributeGetter -from source_braintree.schemas import Customer, Discount, Dispute, MerchantAccount, Plan, Subscription, Transaction -from source_braintree.spec import BraintreeConfig - - -class BraintreeStream(Stream, ABC): - def __init__(self, config: BraintreeConfig): - self._start_date = config.start_date - self._gateway = BraintreeStream.create_gateway(config) - - @staticmethod - def create_gateway(config: BraintreeConfig): - return braintree.BraintreeGateway(braintree.Configuration(**config.dict())) - - @property - @abstractmethod - def model(self): - """ - Pydantic model to represent catalog schema - """ - - @abstractmethod - def get_items(self, start_date: datetime) -> Generator: - """ - braintree SDK gateway object for items list - """ - - def get_json_schema(self): - return self.model.schema() - - def stream_slices( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Optional[Mapping[str, Any]]]: - - current_datetime = pendulum.utcnow() - if sync_mode == SyncMode.full_refresh: - return [{self.cursor_field or "start_date": self._start_date or current_datetime}] - stream_state_start_date = stream_state.get(self.cursor_field) - if stream_state_start_date: - stream_state_start_date = pendulum.parse(stream_state_start_date) - start_date = stream_state_start_date or self._start_date or current_datetime - return [{self.cursor_field: start_date}] - - def get_updated_state( - self, - current_stream_state: Mapping[str, Any], - latest_record: Mapping[str, Any], - ): - next_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) - current_state = pendulum.parse(current_state) if current_state else next_state - return {self.cursor_field: max(current_state, next_state).strftime("%Y-%m-%d %H:%M:%S")} - - @staticmethod - def get_json_from_resource(resource_obj: Union[AttributeGetter, List[AttributeGetter]]): - if isinstance(resource_obj, list): - return [obj if not isinstance(obj, AttributeGetter) else BraintreeStream.get_json_from_resource(obj) for obj in resource_obj] - obj_dict = resource_obj.__dict__ - result = dict() - for attr in obj_dict: - if not attr.startswith("_"): - result[attr] = ( - BraintreeStream.get_json_from_resource(obj_dict[attr]) - if isinstance(obj_dict[attr], (AttributeGetter, list)) - else obj_dict[attr] - ) - return result - - @backoff.on_exception( - backoff.expo, - ( - braintree.exceptions.GatewayTimeoutError, - braintree.exceptions.RequestTimeoutError, - braintree.exceptions.ServerError, - braintree.exceptions.ServiceUnavailableError, - braintree.exceptions.TooManyRequestsError, - ), - max_tries=5, - ) - def _collect_items(self, stream_slice: Mapping[str, Any]) -> List[Mapping[str, Any]]: - """ - Fetch list of response object normalized acccording to catalog model. - Braintree pagination API is designed to use lazy evaluation and SDK is - built upon this approach: First its fetch list of ids, wraps it inside - generator object and then iterates each items and send for getting - additional details. Cause of this implementation we cant handle retry - in case of individual item fails. - :stream_slice Stream slice with cursor field in case of incremental stream. - :return List of objects - """ - start_date = stream_slice.get(self.cursor_field or "start_date") - items = self.get_items(start_date) - result = [] - for item in items: - item = self.get_json_from_resource(item) - item = self.model(**item) - result.append(item.dict(exclude_unset=True)) - return result - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - yield from self._collect_items(stream_slice) - - -class CustomerStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/request/customer/search - """ - - primary_key = "id" - model = Customer - cursor_field = "created_at" - - def get_items(self, start_date: datetime): - return self._gateway.customer.search(braintree.CustomerSearch.created_at >= start_date) - - -class DiscountStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/response/discount - """ - - primary_key = "id" - model = Discount - - def get_items(self, start_date: datetime): - return self._gateway.discount.all() - - -class DisputeStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/request/dispute/search - """ - - primary_key = "id" - model = Dispute - cursor_field = "received_date" - - def get_items(self, start_date: datetime): - return self._gateway.dispute.search(braintree.DisputeSearch.received_date >= start_date.date()).disputes.items - - -class TransactionStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/response/transaction - """ - - primary_key = "id" - model = Transaction - cursor_field = "created_at" - - def get_items(self, start_date: datetime): - return self._gateway.transaction.search(braintree.TransactionSearch.created_at >= start_date) - - -class MerchantAccountStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/response/merchant-account - """ - - primary_key = "id" - model = MerchantAccount - - def get_items(self, start_date: datetime): - return self._gateway.merchant_account.all().merchant_accounts - - -class PlanStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/response/plan - """ - - primary_key = "id" - model = Plan - - def get_items(self, start_date: datetime): - return self._gateway.plan.all() - - -class SubscriptionStream(BraintreeStream): - """ - https://developer.paypal.com/braintree/docs/reference/response/subscription - """ - - primary_key = "id" - model = Subscription - cursor_field = "created_at" - - def get_items(self, start_date: datetime): - return self._gateway.subscription.search(braintree.SubscriptionSearch.created_at >= start_date).items diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/__init__.py b/airbyte-integrations/connectors/source-braintree/unit_tests/__init__.py deleted file mode 100644 index 9db886e0930f..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/conftest.py b/airbyte-integrations/connectors/source-braintree/unit_tests/conftest.py deleted file mode 100644 index 7ceeb9688203..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture -def test_config(): - return { - "merchant_id": "mech_id", - "public_key": "pub_key", - "start_date": "2020-11-22T20:32:05Z", - "private_key": "p k", - "environment": "Sandbox", - } diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/customers_ids.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/customers_ids.txt deleted file mode 100644 index e0bbb1f51c79..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/customers_ids.txt +++ /dev/null @@ -1,6 +0,0 @@ - - 50 - - 896865626 - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/customers_obj_response.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/customers_obj_response.txt deleted file mode 100644 index 39657c24f7b3..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/customers_obj_response.txt +++ /dev/null @@ -1,56 +0,0 @@ - - - 1 - 50 - 1 - - 896865626 - v4h2qqxh3krhq7p3 - test - test - airbyte - customer@test.com - 1231231231 - - airbyte.io - 2020-12-02T06:21:40Z - 2020-12-02T06:29:56Z - - Y3VzdG9tZXJfODk2ODY1NjI2 - - - 411111 - Visa - test guy - Unknown - Unknown - 2020-12-02T06:29:56Z - 896865626 - Y3VzdG9tZXJfODk2ODY1NjI2 - US - Unknown - true - Unknown - 11 - 2022 - false - cGF5bWVudG1ldGhvZF9jY183anQ1NG5y - Unknown - https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox - Unknown - 1111 - Unknown - Unknown - Unknown - - 7jt54nr - 7ae98b4956f624db53908bece7cb0a47 - 2020-12-02T06:30:06Z - false - - false - - - - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/discounts.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/discounts.txt deleted file mode 100644 index 766a4de546a4..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/discounts.txt +++ /dev/null @@ -1,15 +0,0 @@ - - - - 2.00 - 2021-08-10T12:43:41Z - This is test discount - dissscount - discount - v4h2qqxh3krhq7p3 - test discount - true - - 2021-08-10T12:43:41Z - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/disputes.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/disputes.txt deleted file mode 100644 index ab2cd12c99da..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/disputes.txt +++ /dev/null @@ -1,53 +0,0 @@ - - - 1 - 50 - 1 - - yxnc73mfsvc9bzt3 - ZGlzcHV0ZV95eG5jNzNtZnN2YzlienQz - 666.00 - 666.00 - 0.00 - CB173337554006 - 2021-08-10T14:32:40Z - USD - 2021-08-10 - - - chargeback - airbyte - fraud - 62 - - 2021-08-10 - - 2021-08-16 - 2021-08-20 - 2021-08-15T04:59:59Z - open - 2021-08-10T14:32:40Z - - - - - - 2021-08-10 - 2021-08-10 - open - 2021-08-10T14:32:40Z - - - - rb476y0x - dHJhbnNhY3Rpb25fcmI0NzZ5MHg - 666.00 - 2021-08-10T14:32:39Z - - - - Visa - - - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/merch_account.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/merch_account.txt deleted file mode 100644 index c537bb3639b0..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/merch_account.txt +++ /dev/null @@ -1,13 +0,0 @@ - - - 1 - 20 - 1 - - active - airbyte - USD - true - false - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/plans.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/plans.txt deleted file mode 100644 index 54e3e37c25aa..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/plans.txt +++ /dev/null @@ -1,59 +0,0 @@ - - - - eqweqw - v4h2qqxh3krhq7p3 - - 1 - USD - This is plan created for testing - test plan - 3 - 20.00 - 2 - month - true - 2021-08-10T12:43:15Z - 2021-08-10T12:43:15Z - - - 2.00 - 2021-08-10T12:46:19Z - this is test addon - test_addon - add_on - v4h2qqxh3krhq7p3 - test addon - false - 34 - 2021-08-10T12:46:19Z - - - 1.00 - 2021-08-10T13:48:51Z - - test_addon_empty - add_on - v4h2qqxh3krhq7p3 - test_addon - true - - 2021-08-10T13:48:51Z - - - - - 2.00 - 2021-08-10T12:43:41Z - This is test discount - dissscount - discount - v4h2qqxh3krhq7p3 - test discount - true - - 2021-08-10T12:43:41Z - - - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/subscriptions.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/subscriptions.txt deleted file mode 100644 index 553f187c2446..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/subscriptions.txt +++ /dev/null @@ -1,5 +0,0 @@ - - 50 - - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/transaction__objs.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/transaction__objs.txt deleted file mode 100644 index 3872d032c997..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/transaction__objs.txt +++ /dev/null @@ -1,381 +0,0 @@ - - - 1 - 50 - 2 - - 2cdqmzsg - submitted_for_settlement - sale - USD - 2.00 - 2.00 - airbyte - - - - 2021-08-10T14:19:49Z - 2021-08-10T14:19:50Z - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - I - I - I - - T8LSNZ - 1000 - Approved - - - - - false - - false - - - 411111 - 1111 - Visa - 08 - 2100 - US - sss - https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - - - - false - - - - 2021-08-10T14:19:50Z - authorized - 2.00 - sherifairbyte - control_panel - - - 2021-08-10T14:19:50Z - submitted_for_settlement - 2.00 - sherifairbyte - control_panel - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - credit_card - - - XX - sample network response text - - - - - 020210810141950 - approved - 2021-08-18T14:19:49Z - - - - - - - dHJhbnNhY3Rpb25fMmNkcW16c2c - - - 1234567 - - - - - - 123456789012 - 00000001 - DESCRIPTORNAME - - - Braintree - MA - 02184 - 5555555555 - - false - - - - - hncnkbsy - settled - sale - USD - 500.00 - - airbyte - - - - 2020-12-02T06:30:06Z - 2020-12-02T08:20:40Z - - 896865626 - test - test - airbyte - customer@test.com - airbyte.io - 1231231231 - - Y3VzdG9tZXJfODk2ODY1NjI2 - - - - - - - - - - - - - - - - - - - - - - 2020-12-02_airbyte_gcdq2ac6 - - - - - - - - - - - - - - - - - - I - I - I - - MQ6H9W - 1000 - Approved - - - - - false - - false - - 7jt54nr - 411111 - 1111 - Visa - 11 - 2022 - US - test guy - https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - Unknown - cGF5bWVudG1ldGhvZF9jY183anQ1NG5y - - 7ae98b4956f624db53908bece7cb0a47 - false - - - - 2020-12-02T06:30:06Z - authorized - 500.00 - sherifairbyte - control_panel - - - 2020-12-02T06:30:06Z - submitted_for_settlement - 500.00 - sherifairbyte - control_panel - - - 2020-12-02T08:20:40Z - settled - 500.00 - - - - - - - - - - - - - - - - - false - - - - - 2020-12-03 - 500.00 - USD - 1 - - false - true - - - - credit_card - - - XX - sample network response text - - - - - 020201202063006 - approved - 2020-12-09T06:30:06Z - - - - - - - dHJhbnNhY3Rpb25faG5jbmtic3k - - - 1234567 - - - - - - 123456789012 - 00000001 - DESCRIPTORNAME - - - Braintree - MA - 02184 - 5555555555 - - false - - - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/data/transaction_ids.txt b/airbyte-integrations/connectors/source-braintree/unit_tests/data/transaction_ids.txt deleted file mode 100644 index 094f0487a03f..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/data/transaction_ids.txt +++ /dev/null @@ -1,7 +0,0 @@ - - 50 - - 2cdqmzsg - hncnkbsy - - diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/test_source.py b/airbyte-integrations/connectors/source-braintree/unit_tests/test_source.py deleted file mode 100644 index 09e6d652c497..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/test_source.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import jsonschema -import responses -from airbyte_cdk.models import AirbyteMessage, Type -from source_braintree.source import SourceBraintree - - -def get_stream_by_name(streams: list, stream_name: str): - for stream in streams: - if stream.name == stream_name: - return stream - - -def test_source_streams(test_config): - s = SourceBraintree() - streams = s.streams(test_config) - assert len(streams) == 7 - assert {stream.name for stream in streams} == { - "customer_stream", - "discount_stream", - "dispute_stream", - "transaction_stream", - "merchant_account_stream", - "plan_stream", - "subscription_stream", - } - customers = get_stream_by_name(streams, "customer_stream") - assert customers.supports_incremental - discount_stream = get_stream_by_name(streams, "discount_stream") - assert not discount_stream.supports_incremental - - -def test_discover(test_config): - source = SourceBraintree() - catalog = source.discover(None, test_config) - catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) - schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] - for schema in schemas: - jsonschema.Draft7Validator.check_schema(schema) - - -def test_spec(test_config): - s = SourceBraintree() - schema = s.spec(None).connectionSpecification - jsonschema.Draft4Validator.check_schema(schema) - jsonschema.validate(instance=test_config, schema=schema) - - -@responses.activate -def test_check(test_config): - s = SourceBraintree() - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/customers/advanced_search_ids", - body=open("unit_tests/data/customers_ids.txt").read(), - ) - assert s.check_connection(None, test_config) == (True, "") diff --git a/airbyte-integrations/connectors/source-braintree/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-braintree/unit_tests/test_streams.py deleted file mode 100644 index feba99d21317..000000000000 --- a/airbyte-integrations/connectors/source-braintree/unit_tests/test_streams.py +++ /dev/null @@ -1,165 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import os -from copy import deepcopy - -import pendulum -import responses -from airbyte_cdk.models import SyncMode -from freezegun import freeze_time -from source_braintree.spec import BraintreeConfig -from source_braintree.streams import ( - CustomerStream, - DiscountStream, - DisputeStream, - MerchantAccountStream, - PlanStream, - SubscriptionStream, - TransactionStream, -) - - -def load_file(fn): - return open(os.path.join("unit_tests", "data", fn)).read() - - -def read_all_records(stream): - stream_slice = {stream.cursor_field or "start_date": pendulum.utcnow()} - return [r for r in stream.read_records(None, None, stream_slice)] - - -@responses.activate -def test_customers_stream(test_config): - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/customers/advanced_search_ids", - body=load_file("customers_ids.txt"), - ) - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/customers/advanced_search", - body=load_file("customers_obj_response.txt"), - ) - config = BraintreeConfig(**test_config) - stream = CustomerStream(config) - records = read_all_records(stream) - assert len(records) == 1 - - -@responses.activate -def test_transaction_stream(test_config): - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/transactions/advanced_search_ids", - body=load_file("transaction_ids.txt"), - ) - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/transactions/advanced_search", - body=load_file("transaction__objs.txt"), - ) - config = BraintreeConfig(**test_config) - stream = TransactionStream(config) - records = read_all_records(stream) - assert len(records) == 2 - - -@responses.activate -def test_discount(test_config): - responses.add( - responses.GET, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/discounts/", - body=load_file("discounts.txt"), - ) - config = BraintreeConfig(**test_config) - stream = DiscountStream(config) - records = read_all_records(stream) - assert len(records) == 1 - - -@responses.activate -def test_merch_account(test_config): - responses.add( - responses.GET, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/merchant_accounts/", - body=load_file("merch_account.txt"), - ) - config = BraintreeConfig(**test_config) - stream = MerchantAccountStream(config) - records = read_all_records(stream) - assert len(records) == 1 - - -@responses.activate -def test_plan(test_config): - responses.add( - responses.GET, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/plans/", - body=load_file("plans.txt"), - ) - config = BraintreeConfig(**test_config) - stream = PlanStream(config) - records = read_all_records(stream) - assert len(records) == 1 - - -@responses.activate -def test_dispute(test_config): - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/disputes/advanced_search", - body=load_file("disputes.txt"), - ) - config = BraintreeConfig(**test_config) - stream = DisputeStream(config) - records = read_all_records(stream) - assert len(records) == 1 - - -@responses.activate -def test_subscription(test_config): - responses.add( - responses.POST, - "https://api.sandbox.braintreegateway.com:443/merchants/mech_id/subscriptions/advanced_search_ids", - body=load_file("subscriptions.txt"), - ) - config = BraintreeConfig(**test_config) - stream = SubscriptionStream(config) - records = read_all_records(stream) - assert len(records) == 0 - - -@freeze_time("2020-10-10") -def test_stream_slices(test_config): - config = BraintreeConfig(**test_config) - stream = TransactionStream(config) - - assert stream.stream_slices(SyncMode.incremental, None, {}) == [{stream.cursor_field: config.start_date}] - assert stream.stream_slices(SyncMode.incremental, None, {stream.cursor_field: "2010"}) == [ - {stream.cursor_field: pendulum.datetime(2010, 1, 1)} - ] - assert stream.stream_slices(SyncMode.full_refresh, None, {stream.cursor_field: "2010"}) == [{stream.cursor_field: config.start_date}] - - test_config = deepcopy(test_config) - test_config.pop("start_date") - config = BraintreeConfig(**test_config) - stream = TransactionStream(config) - assert stream.stream_slices(SyncMode.incremental, None, {}) == [{stream.cursor_field: pendulum.datetime(2020, 10, 10)}] - assert stream.stream_slices(SyncMode.full_refresh, None, {stream.cursor_field: "2010"}) == [ - {stream.cursor_field: pendulum.datetime(2020, 10, 10)} - ] - - -def test_updated_state(test_config): - config = BraintreeConfig(**test_config) - stream = TransactionStream(config) - - assert stream.get_updated_state({stream.cursor_field: "2021-08-10 14:32:39"}, {stream.cursor_field: pendulum.parse("2000")}) == { - stream.cursor_field: "2021-08-10 14:32:39" - } - assert stream.get_updated_state({stream.cursor_field: "2021-08-10 14:32:39"}, {stream.cursor_field: pendulum.parse("2100")}) == { - stream.cursor_field: "2100-01-01 00:00:00" - } - assert stream.get_updated_state({}, {stream.cursor_field: pendulum.parse("2100")}) == {stream.cursor_field: "2100-01-01 00:00:00"} diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json index ac1e946b5e4a..73ca9a0e3a24 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json @@ -7,14 +7,10 @@ "time": "2050-12-13", "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2" }, - "events_analytics": { - }, - "kpi_daily_new_users": { - }, - "kpi_daily_active_users": { - }, - "kpi_daily_app_uninstalls": { - }, + "events_analytics": {}, + "kpi_daily_new_users": {}, + "kpi_daily_active_users": {}, + "kpi_daily_app_uninstalls": {}, "cards_analytics": { "time": "2050-12-13", "card_id": "609e4d4c-367b-4c87-a8ad-b1f903d058fd" @@ -23,4 +19,4 @@ "time": "2050-12-13", "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json index c0877549b4ed..40da794e3f38 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json @@ -7,18 +7,13 @@ "time": "2022-09-01", "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2" }, - "events_analytics": { - }, - "kpi_daily_new_users": { - }, - "kpi_daily_active_users": { - }, - "kpi_daily_app_uninstalls": { - }, - "cards_analytics": { - }, + "events_analytics": {}, + "kpi_daily_new_users": {}, + "kpi_daily_active_users": {}, + "kpi_daily_app_uninstalls": {}, + "cards_analytics": {}, "segments_analytics": { "time": "2022-09-01", "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-braze/metadata.yaml b/airbyte-integrations/connectors/source-braze/metadata.yaml index 6fde2e295659..16aa19bfc0c8 100644 --- a/airbyte-integrations/connectors/source-braze/metadata.yaml +++ b/airbyte-integrations/connectors/source-braze/metadata.yaml @@ -1,11 +1,15 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 68b9c98e-0747-4c84-b05b-d30b47686725 dockerImageTag: 0.1.3 - icon: braze.svg dockerRepository: airbyte/source-braze + documentationUrl: https://docs.airbyte.com/integrations/sources/braze githubIssueLabel: source-braze + icon: braze.svg license: MIT name: Braze registries: @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/sources/braze + supportLevel: community tags: - language:low-code - language:python diff --git a/airbyte-integrations/connectors/source-braze/requirements.txt b/airbyte-integrations/connectors/source-braze/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-braze/requirements.txt +++ b/airbyte-integrations/connectors/source-braze/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-braze/setup.py b/airbyte-integrations/connectors/source-braze/setup.py index d9c9d2d6073b..3bdf009a6ece 100644 --- a/airbyte-integrations/connectors/source-braze/setup.py +++ b/airbyte-integrations/connectors/source-braze/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_analytics.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_analytics.json index a5fe24aa0f9e..ac6f66ec8e68 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_analytics.json +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_analytics.json @@ -59,4 +59,4 @@ "type": ["null", "number"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json index f347e2fe19be..c834118b67f0 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json @@ -34,4 +34,4 @@ "additionalProperties": true } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/cards_analytics.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/cards_analytics.json index 6a26469276e9..0b480216cb3a 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/schemas/cards_analytics.json +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/cards_analytics.json @@ -8,16 +8,16 @@ "time": { "type": ["null", "string"] }, - "clicks" : { + "clicks": { "type": ["null", "integer"] - } , - "impressions" : { + }, + "impressions": { "type": ["null", "integer"] }, - "unique_clicks" : { + "unique_clicks": { "type": ["null", "integer"] }, - "unique_impressions" : { + "unique_impressions": { "type": ["null", "integer"] } } diff --git a/airbyte-integrations/connectors/source-breezometer/metadata.yaml b/airbyte-integrations/connectors/source-breezometer/metadata.yaml index 42c14aebd434..b325179a45ca 100644 --- a/airbyte-integrations/connectors/source-breezometer/metadata.yaml +++ b/airbyte-integrations/connectors/source-breezometer/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-breezometer/requirements.txt b/airbyte-integrations/connectors/source-breezometer/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-breezometer/requirements.txt +++ b/airbyte-integrations/connectors/source-breezometer/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-breezometer/setup.py b/airbyte-integrations/connectors/source-breezometer/setup.py index 187d854ed23c..61bd2c525610 100644 --- a/airbyte-integrations/connectors/source-breezometer/setup.py +++ b/airbyte-integrations/connectors/source-breezometer/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-callrail/metadata.yaml b/airbyte-integrations/connectors/source-callrail/metadata.yaml index 3fc097041ec6..d876058aa9c0 100644 --- a/airbyte-integrations/connectors/source-callrail/metadata.yaml +++ b/airbyte-integrations/connectors/source-callrail/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-callrail/requirements.txt b/airbyte-integrations/connectors/source-callrail/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-callrail/requirements.txt +++ b/airbyte-integrations/connectors/source-callrail/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-callrail/setup.py b/airbyte-integrations/connectors/source-callrail/setup.py index d0c793161371..c744bbc957ff 100644 --- a/airbyte-integrations/connectors/source-callrail/setup.py +++ b/airbyte-integrations/connectors/source-callrail/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-captain-data/.dockerignore b/airbyte-integrations/connectors/source-captain-data/.dockerignore new file mode 100644 index 000000000000..f7cb8b7c520a --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_captain_data +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-captain-data/Dockerfile b/airbyte-integrations/connectors/source-captain-data/Dockerfile new file mode 100644 index 000000000000..3b3a4c8883ab --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_captain_data ./source_captain_data + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-captain-data diff --git a/airbyte-integrations/connectors/source-captain-data/README.md b/airbyte-integrations/connectors/source-captain-data/README.md new file mode 100644 index 000000000000..0d93d706ca0c --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/README.md @@ -0,0 +1,82 @@ +# Captain Data Source + +This is the repository for the Captain Data configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/captain-data). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/captain-data) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_captain_data/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source captain-data test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-captain-data:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-captain-data:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-captain-data:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-captain-data:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-captain-data:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-captain-data:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-captain-data/__init__.py b/airbyte-integrations/connectors/source-captain-data/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml b/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml new file mode 100644 index 000000000000..9f4ac3d46b6e --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-captain-data:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_captain_data/manifest.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-captain-data/build.gradle b/airbyte-integrations/connectors/source-captain-data/build.gradle new file mode 100644 index 000000000000..d92bf8ed87a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_captain_data' +} diff --git a/airbyte-integrations/connectors/source-captain-data/icon.svg b/airbyte-integrations/connectors/source-captain-data/icon.svg new file mode 100644 index 000000000000..7a54fe2fc550 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/icon.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py b/airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py new file mode 100644 index 000000000000..82823254d266 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..f89970b00f0c --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/configured_catalog.json @@ -0,0 +1,64 @@ +{ + "streams": [ + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "workspace", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["name"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "workflows", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["uid"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "jobs", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["uid"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "job_results", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["job_result_id"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json new file mode 100644 index 000000000000..a8a304c99f51 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "bad-api-key", + "project_uid": "bad-project-uid" +} diff --git a/airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json new file mode 100644 index 000000000000..daca574b8635 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "api-key", + "project_uid": "project-uid" +} diff --git a/airbyte-integrations/connectors/source-captain-data/main.py b/airbyte-integrations/connectors/source-captain-data/main.py new file mode 100644 index 000000000000..765d967fad15 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_captain_data import SourceCaptainData + +if __name__ == "__main__": + source = SourceCaptainData() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-captain-data/metadata.yaml b/airbyte-integrations/connectors/source-captain-data/metadata.yaml new file mode 100644 index 000000000000..715cb3f3c9c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/metadata.yaml @@ -0,0 +1,25 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: fa290790-1dca-43e7-8ced-6a40b2a66099 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-captain-data + githubIssueLabel: source-captain-data + icon: captain-data.svg + license: MIT + name: Captain Data + registries: + cloud: + enabled: false + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/captain-data + tags: + - language:low-code + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-captain-data/requirements.txt b/airbyte-integrations/connectors/source-captain-data/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/source-captain-data/setup.py b/airbyte-integrations/connectors/source-captain-data/setup.py new file mode 100644 index 000000000000..f6121791ddd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + +setup( + name="source_captain_data", + description="Source implementation for Captain Data.", + author="Elliot Trabac", + author_email="elliot.trabac1@gmail.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py b/airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py new file mode 100644 index 000000000000..bdb8c926c0a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCaptainData + +__all__ = ["SourceCaptainData"] diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml new file mode 100644 index 000000000000..5cb698d2aa9f --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/manifest.yaml @@ -0,0 +1,170 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.captaindata.co/v3/" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "x-api-key" + api_token: "{{ config['api_key'] }}" + request_headers: + x-project-id: "{{ config['project_uid'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + captain_data_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response['paging']['next']}}" + stop_condition: "{{ response['paging']['have_next_page'] is false }}" + page_token_option: + type: "RequestPath" + + # Base stream + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + # Streams + workspace_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "workspace" + primary_key: "name" + path: "/workspace" + transformations: + - type: AddFields + fields: + - path: ["project_uid"] + value: "{{ config['project_uid'] }}" + + workflows_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "workflows" + primary_key: "uid" + path: "/workflows" + + # Sliced streams + jobs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "jobs" + primary_key: "uid" + path: "/workflows/{{ stream_slice.parent_id }}/jobs" + retriever: + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/workflows_stream" + parent_key: "uid" + partition_field: "parent_id" + record_selector: + $ref: "#/definitions/selector" + + successful_jobs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "jobs" + primary_key: "uid" + path: "/workflows/{{ stream_slice.parent_id }}/jobs" + retriever: + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/workflows_stream" + parent_key: "uid" + partition_field: "parent_id" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + record_filter: + condition: "{{ record['total_row_count'] > 0 and record['status'] == 'finished' }}" + + job_results_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "job_results" + primary_key: "job_result_id" + path: "/jobs/{{ stream_slice.parent_id }}/results" + stream_cursor_field: "extracted_at" + retriever: + requester: + $ref: "#/definitions/requester" + paginator: + $ref: "#/definitions/captain_data_paginator" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/successful_jobs_stream" + parent_key: "uid" + partition_field: "parent_id" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["results"] + transformations: + - type: AddFields + fields: + - path: ["results"] + value: "{{ record }}" + - path: ["job_result_id"] + value: "{{ record|hash('md5') }}" + - path: ["job_uid"] + value: "{{ stream_slice.parent_id }}" + - path: ["extracted_at"] + value: "{{ now_utc() }}" + +streams: + - "#/definitions/workspace_stream" + - "#/definitions/workflows_stream" + - "#/definitions/jobs_stream" + - "#/definitions/job_results_stream" + +check: + type: CheckStream + stream_names: + - "workspace" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/captain-data + connection_specification: + title: Captain Data Spec + type: object + required: + - api_key + - project_uid + additionalProperties: true + properties: + api_key: + title: API Key + type: string + description: Your Captain Data project API key. + airbyte_secret: true + project_uid: + title: Project UID + type: string + description: Your Captain Data project uuid. diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json new file mode 100644 index 000000000000..f487b33fbfdc --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/job_results.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "job_uid": { + "type": ["null", "string"] + }, + "job_result_id": { + "type": ["null", "string"] + }, + "results": { + "type": ["null", "object"] + }, + "extracted_at": { + "type": ["null", "string"], + "format": "datetime" + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json new file mode 100644 index 000000000000..dd7cf2a8670c --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/jobs.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "accounts": { + "items": { + "type": ["null", "string"] + }, + "type": "array" + }, + "configuration": { + "type": ["null", "object"] + }, + "error_message": { + "type": ["null", "string"] + }, + "finish_time": { + "type": ["null", "string"] + }, + "main_job_uid": { + "type": ["null", "string"] + }, + "main_job_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "number_inputs": { + "type": ["null", "integer"] + }, + "row_count": { + "type": ["null", "integer"] + }, + "scheduled_time": { + "type": ["null", "string"] + }, + "start_time": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "step_uid": { + "type": ["null", "string"] + }, + "total_row_count": { + "type": ["null", "integer"] + }, + "uid": { + "type": ["null", "string"] + }, + "workflow_name": { + "type": ["null", "string"] + }, + "workflow_uid": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json new file mode 100644 index 000000000000..4709a5f21f51 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workflows.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "created_at": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "steps": { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "step_uid": { + "type": ["null", "string"] + }, + "uid": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "type": "array" + }, + "template_uid": { + "type": ["null", "string"] + }, + "uid": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json new file mode 100644 index 000000000000..7644d353733f --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/schemas/workspace.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "project_uid": { + "type": ["null", "string"] + }, + "current_month_end": { + "type": ["null", "string"] + }, + "current_month_start": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "plan_name": { + "type": ["null", "string"] + }, + "tasks_left": { + "type": ["null", "integer"] + }, + "tasks_max": { + "type": ["null", "integer"] + }, + "tasks_used": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py b/airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py new file mode 100644 index 000000000000..648e1fb181b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-captain-data/source_captain_data/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceCaptainData(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-cart/metadata.yaml b/airbyte-integrations/connectors/source-cart/metadata.yaml index 57964ee54a15..8f75a5db4ceb 100644 --- a/airbyte-integrations/connectors/source-cart/metadata.yaml +++ b/airbyte-integrations/connectors/source-cart/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/cart tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-cart/requirements.txt b/airbyte-integrations/connectors/source-cart/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-cart/requirements.txt +++ b/airbyte-integrations/connectors/source-cart/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-cart/setup.py b/airbyte-integrations/connectors/source-cart/setup.py index a0f6e142dd07..d23fa4f8daf4 100644 --- a/airbyte-integrations/connectors/source-cart/setup.py +++ b/airbyte-integrations/connectors/source-cart/setup.py @@ -10,8 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-chargebee/Dockerfile b/airbyte-integrations/connectors/source-chargebee/Dockerfile index fa5855f04f7f..3184e75a167b 100644 --- a/airbyte-integrations/connectors/source-chargebee/Dockerfile +++ b/airbyte-integrations/connectors/source-chargebee/Dockerfile @@ -34,5 +34,5 @@ COPY source_chargebee ./source_chargebee ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/source-chargebee diff --git a/airbyte-integrations/connectors/source-chargebee/README.md b/airbyte-integrations/connectors/source-chargebee/README.md index 58625880868c..9a2752b810bc 100644 --- a/airbyte-integrations/connectors/source-chargebee/README.md +++ b/airbyte-integrations/connectors/source-chargebee/README.md @@ -80,4 +80,4 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). 1. Create a Pull Request. 1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml index f9448af43623..27941b91fb62 100644 --- a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml @@ -49,19 +49,22 @@ acceptance_tests: extra_records: yes fail_on_extra_columns: false incremental: - tests: - - config_path: "secrets/config.json" - timeout_seconds: 2400 - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/future_state.json" - missing_streams: - - name: attached_item - bypass_reason: "This stream is Full-Refresh only" - - name: contact - bypass_reason: "This stream is Full-Refresh only" - - name: quote_line_group - bypass_reason: "This stream is Full-Refresh only" + # tests: + # - config_path: "secrets/config.json" + # timeout_seconds: 2400 + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/future_state.json" + # missing_streams: + # - name: attached_item + # bypass_reason: "This stream is Full-Refresh only" + # - name: contact + # bypass_reason: "This stream is Full-Refresh only" + # - name: quote_line_group + # bypass_reason: "This stream is Full-Refresh only" + bypass_reason: > + "Incrremental tests are disabled until CAT works with cursor data-types directly, + relatated slack thread: https://airbyte-globallogic.slack.com/archives/C02U9R3AF37/p1690810513681859" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json index 8d7904ccf5da..b7226a840311 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json @@ -97,14 +97,14 @@ "stream_descriptor": { "name": "quote" } } }, - { + { "type": "STREAM", "stream": { "stream_state": { "occurred_at": 2147483647 }, "stream_descriptor": { "name": "event" } } }, - { + { "type": "STREAM", "stream": { "stream_state": { "updated_at": 2147483647 }, diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json index a6c6fcb79734..468a1e167331 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json @@ -97,14 +97,14 @@ "stream_descriptor": { "name": "quote" } } }, - { + { "type": "STREAM", "stream": { "stream_state": { "occurred_at": 1625596058 }, "stream_descriptor": { "name": "event" } } }, - { + { "type": "STREAM", "stream": { "stream_state": { "updated_at": 1625596058 }, diff --git a/airbyte-integrations/connectors/source-chargebee/metadata.yaml b/airbyte-integrations/connectors/source-chargebee/metadata.yaml index 2d0440c148bb..789788ab5d56 100644 --- a/airbyte-integrations/connectors/source-chargebee/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargebee/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 686473f1-76d9-4994-9cc7-9b13da46147c - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 dockerRepository: airbyte/source-chargebee githubIssueLabel: source-chargebee icon: chargebee.svg @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-chargebee/requirements.txt b/airbyte-integrations/connectors/source-chargebee/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-chargebee/requirements.txt +++ b/airbyte-integrations/connectors/source-chargebee/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-chargebee/setup.py b/airbyte-integrations/connectors/source-chargebee/setup.py index 5419ced8034f..5c3b625e0b4c 100644 --- a/airbyte-integrations/connectors/source-chargebee/setup.py +++ b/airbyte-integrations/connectors/source-chargebee/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.29", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json index b951367b1756..32fde3838b79 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/gift.json @@ -72,7 +72,13 @@ "properties": { "status": { "type": ["string", "null"], - "enum": ["scheduled", "unclaimed", "claimed", "cancelled", "expired"] + "enum": [ + "scheduled", + "unclaimed", + "claimed", + "cancelled", + "expired" + ] }, "occurred_at": { "type": ["integer", "null"] diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json index fa151b99dc8d..8ad0b53168de 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/hosted_page.json @@ -9,8 +9,16 @@ "type": { "type": ["string", "null"], "enum": [ - "checkout_new", "checkout_existing", "update_payment_method", "manage_payment_sources", "collect_now", - "extend_subscription", "checkout_gift", "claim_gift", "checkout_one_time", "pre_cancel" + "checkout_new", + "checkout_existing", + "update_payment_method", + "manage_payment_sources", + "collect_now", + "extend_subscription", + "checkout_gift", + "claim_gift", + "checkout_one_time", + "pre_cancel" ] }, "url": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json index 961964bb61ad..26095b14a47e 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json @@ -21,8 +21,22 @@ "type": { "type": ["string", "null"], "enum": [ - "card", "paypal_express_checkout", "amazon_payments", "direct_debit", "generic", "alipay", "unionpay", - "apple_pay", "wechat_pay", "ideal", "google_pay", "sofort", "bancontact", "giropay", "dotpay", "upi", + "card", + "paypal_express_checkout", + "amazon_payments", + "direct_debit", + "generic", + "alipay", + "unionpay", + "apple_pay", + "wechat_pay", + "ideal", + "google_pay", + "sofort", + "bancontact", + "giropay", + "dotpay", + "upi", "netbanking_emandates" ] }, @@ -42,12 +56,51 @@ "gateway": { "type": ["string", "null"], "enum": [ - "chargebee", "chargebee_payments", "stripe", "wepay", "braintree", "authorize_net", "paypal_pro", "pin", - "eway", "eway_rapid", "worldpay", "balanced_payments", "beanstream", "bluepay", "elavon", "first_data_global", - "hdfc", "migs", "nmi", "ogone", "paymill", "paypal_payflow_pro", "sage_pay", "tco", "wirecard", - "amazon_payments", "paypal_express_checkout", "gocardless", "adyen", "orbital", "moneris_us", - "moneris", "bluesnap", "cybersource", "vantiv", "checkout_com", "paypal", "ingenico_direct", "exact", "mollie", - "quickbooks", "razorpay", "global_payments", "bank_of_america", "not_applicable" + "chargebee", + "chargebee_payments", + "stripe", + "wepay", + "braintree", + "authorize_net", + "paypal_pro", + "pin", + "eway", + "eway_rapid", + "worldpay", + "balanced_payments", + "beanstream", + "bluepay", + "elavon", + "first_data_global", + "hdfc", + "migs", + "nmi", + "ogone", + "paymill", + "paypal_payflow_pro", + "sage_pay", + "tco", + "wirecard", + "amazon_payments", + "paypal_express_checkout", + "gocardless", + "adyen", + "orbital", + "moneris_us", + "moneris", + "bluesnap", + "cybersource", + "vantiv", + "checkout_com", + "paypal", + "ingenico_direct", + "exact", + "mollie", + "quickbooks", + "razorpay", + "global_payments", + "bank_of_america", + "not_applicable" ] }, "gateway_account_id": { @@ -83,8 +136,15 @@ "brand": { "type": ["string", "null"], "enum": [ - "visa", "mastercard", "american_express", "discover", "jcb", - "diners_club", "other", "bancontact", "not_applicable" + "visa", + "mastercard", + "american_express", + "discover", + "jcb", + "diners_club", + "other", + "bancontact", + "not_applicable" ] }, "funding_type": { @@ -201,8 +261,8 @@ "type": ["string", "null"] }, "created_at": { - "type": ["integer", "null"] - } + "type": ["integer", "null"] + } } }, "custom_fields": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json index ef9eb4497355..0e4198814f2c 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote.json @@ -27,7 +27,11 @@ }, "operation_type": { "type": ["string", "null"], - "enum": ["create_subscription_for_customer", "change_subscription", "onetime_invoice"] + "enum": [ + "create_subscription_for_customer", + "change_subscription", + "onetime_invoice" + ] }, "vat_number": { "type": ["string", "null"] @@ -158,13 +162,25 @@ }, "entity_type": { "type": ["string", "null"], - "enum": ["adhoc", "plan_item_price", "addon_item_price", "charge_item_price"] + "enum": [ + "adhoc", + "plan_item_price", + "addon_item_price", + "charge_item_price" + ] }, "tax_exempt_reason": { "type": ["string", "null"], "enum": [ - "tax_not_configured", "region_non_taxable", "export", "customer_exempt", "product_exempt", - "zero_rated", "reverse_charge", "high_value_physical_goods", "zero_value_item" + "tax_not_configured", + "region_non_taxable", + "export", + "customer_exempt", + "product_exempt", + "zero_rated", + "reverse_charge", + "high_value_physical_goods", + "zero_value_item" ] }, "entity_id": { @@ -193,8 +209,12 @@ "entity_type": { "type": ["string", "null"], "enum": [ - "item_level_coupon", "document_level_coupon", "promotional_credits", - "prorated_credits", "item_level_discount", "document_level_discount" + "item_level_coupon", + "document_level_coupon", + "promotional_credits", + "prorated_credits", + "item_level_discount", + "document_level_discount" ] }, "discount_type": { @@ -221,8 +241,12 @@ "discount_type": { "type": ["string", "null"], "enum": [ - "item_level_coupon", "document_level_coupon", "promotional_credits", - "prorated_credits", "item_level_discount", "document_level_discount" + "item_level_coupon", + "document_level_coupon", + "promotional_credits", + "prorated_credits", + "item_level_discount", + "document_level_discount" ] }, "entity_id": { @@ -279,7 +303,16 @@ }, "tax_juris_type": { "type": ["string", "null"], - "enum": ["country", "federal", "state", "county", "city", "special", "unincorporated", "other"] + "enum": [ + "country", + "federal", + "state", + "county", + "city", + "special", + "unincorporated", + "other" + ] }, "tax_juris_name": { "type": ["string", "null"] @@ -471,4 +504,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json index 1052967d81d4..588c38bd7d4d 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/quote_line_group.json @@ -27,8 +27,12 @@ "charge_event": { "type": ["string", "null"], "enum": [ - "immediate", "subscription_creation", "trial_start", - "subscription_change", "subscription_renewal", "subscription_cancel" + "immediate", + "subscription_creation", + "trial_start", + "subscription_change", + "subscription_renewal", + "subscription_cancel" ] }, "billing_cycle_number": { @@ -99,13 +103,25 @@ }, "entity_type": { "type": ["string", "null"], - "enum": ["adhoc", "plan_item_price", "addon_item_price", "charge_item_price"] + "enum": [ + "adhoc", + "plan_item_price", + "addon_item_price", + "charge_item_price" + ] }, "tax_exempt_reason": { "type": ["string", "null"], "enum": [ - "tax_not_configured", "region_non_taxable", "export", "customer_exempt", "product_exempt", - "zero_rated", "reverse_charge", "high_value_physical_goods", "zero_value_item" + "tax_not_configured", + "region_non_taxable", + "export", + "customer_exempt", + "product_exempt", + "zero_rated", + "reverse_charge", + "high_value_physical_goods", + "zero_value_item" ] }, "entity_id": { @@ -134,8 +150,12 @@ "entity_type": { "type": ["string", "null"], "enum": [ - "item_level_coupon", "document_level_coupon", "promotional_credits", - "prorated_credits", "item_level_discount", "document_level_discount" + "item_level_coupon", + "document_level_coupon", + "promotional_credits", + "prorated_credits", + "item_level_discount", + "document_level_discount" ] }, "discount_type": { @@ -162,8 +182,12 @@ "discount_type": { "type": ["string", "null"], "enum": [ - "item_level_coupon", "document_level_coupon", "promotional_credits", - "prorated_credits", "item_level_discount", "document_level_discount" + "item_level_coupon", + "document_level_coupon", + "promotional_credits", + "prorated_credits", + "item_level_discount", + "document_level_discount" ] }, "entity_id": { @@ -220,7 +244,16 @@ }, "tax_juris_type": { "type": ["string", "null"], - "enum": ["country", "federal", "state", "county", "city", "special", "unincorporated", "other"] + "enum": [ + "country", + "federal", + "state", + "county", + "city", + "special", + "unincorporated", + "other" + ] }, "tax_juris_name": { "type": ["string", "null"] diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json index fe6e145ba65a..37c94c393aab 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/unbilled_charge.json @@ -42,7 +42,12 @@ }, "entity_type": { "type": ["string", "null"], - "enum": ["adhoc", "plan_item_price", "addon_item_price", "charge_item_price"] + "enum": [ + "adhoc", + "plan_item_price", + "addon_item_price", + "charge_item_price" + ] }, "entity_id": { "type": ["string", "null"] diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json index e9c2c51e0c43..daaa14b122fa 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/virtual_bank_account.json @@ -31,12 +31,51 @@ "gateway": { "type": ["string", "null"], "enum": [ - "chargebee", "chargebee_payments", "stripe", "wepay", "braintree", "authorize_net", "paypal_pro", "pin", - "eway", "eway_rapid", "worldpay", "balanced_payments", "beanstream", "bluepay", "elavon", "first_data_global", - "hdfc", "migs", "nmi", "ogone", "paymill", "paypal_payflow_pro", "sage_pay", "tco", "wirecard", - "amazon_payments", "paypal_express_checkout", "gocardless", "adyen", "orbital", "moneris_us", - "moneris", "bluesnap", "cybersource", "vantiv", "checkout_com", "paypal", "ingenico_direct", "exact", "mollie", - "quickbooks", "razorpay", "global_payments", "bank_of_america", "not_applicable" + "chargebee", + "chargebee_payments", + "stripe", + "wepay", + "braintree", + "authorize_net", + "paypal_pro", + "pin", + "eway", + "eway_rapid", + "worldpay", + "balanced_payments", + "beanstream", + "bluepay", + "elavon", + "first_data_global", + "hdfc", + "migs", + "nmi", + "ogone", + "paymill", + "paypal_payflow_pro", + "sage_pay", + "tco", + "wirecard", + "amazon_payments", + "paypal_express_checkout", + "gocardless", + "adyen", + "orbital", + "moneris_us", + "moneris", + "bluesnap", + "cybersource", + "vantiv", + "checkout_com", + "paypal", + "ingenico_direct", + "exact", + "mollie", + "quickbooks", + "razorpay", + "global_payments", + "bank_of_america", + "not_applicable" ] }, "resource_version": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml b/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml index 6bc819acf76c..0f8e20d15d36 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/spec.yaml @@ -37,4 +37,4 @@ connectionSpecification: title: Product Catalog description: Product Catalog version of your Chargebee site. Instructions on how to find your version you may find here under `API Version` section. enum: ["1.0", "2.0"] - order: 3 \ No newline at end of file + order: 3 diff --git a/airbyte-integrations/connectors/source-chargify/Dockerfile b/airbyte-integrations/connectors/source-chargify/Dockerfile index 8d9fd809b173..869d038aa4ce 100644 --- a/airbyte-integrations/connectors/source-chargify/Dockerfile +++ b/airbyte-integrations/connectors/source-chargify/Dockerfile @@ -34,5 +34,5 @@ COPY source_chargify ./source_chargify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-chargify diff --git a/airbyte-integrations/connectors/source-chargify/README.md b/airbyte-integrations/connectors/source-chargify/README.md index f44eca7c41f6..3f9b75013ad3 100644 --- a/airbyte-integrations/connectors/source-chargify/README.md +++ b/airbyte-integrations/connectors/source-chargify/README.md @@ -1,35 +1,10 @@ # Chargify Source -This is the repository for the Chargify source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/chargify). +This is the repository for the Chargify configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/chargify). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/chargify) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chargify/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/chargify) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chargify/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source chargify test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chargify:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-chargify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-chargify/__init__.py b/airbyte-integrations/connectors/source-chargify/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml index 4b7dd606b666..33cbcb687916 100644 --- a/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml @@ -1,20 +1,41 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-chargify:dev -tests: +acceptance_tests: spec: - - spec_path: "source_chargify/spec.json" + tests: + - spec_path: "source_chargify/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/__init__.py b/airbyte-integrations/connectors/source-chargify/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-chargify/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-chargify/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/catalog.json b/airbyte-integrations/connectors/source-chargify/integration_tests/catalog.json deleted file mode 100644 index 9a6c0e1fc5d5..000000000000 --- a/airbyte-integrations/connectors/source-chargify/integration_tests/catalog.json +++ /dev/null @@ -1,272 +0,0 @@ -{ - "streams": [ - { - "name": "customers", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "default_subscription_group_uid": { - "type": ["null", "string"] - }, - "portal_invite_last_sent_at": { - "type": "string", - "format": "date-time" - }, - "vat_number": { - "type": ["null", "string"] - }, - "email": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "zip": { - "type": "string" - }, - "first_name": { - "type": "string" - }, - "country_name": { - "type": ["null", "string"] - }, - "state": { - "type": "string" - }, - "city": { - "type": "string" - }, - "parent_id": { - "type": ["null", "integer"] - }, - "locale": { - "type": ["null", "string"] - }, - "portal_customer_created_at": { - "type": "string" - }, - "updated_at": { - "type": "string", - "format": "date-time" - }, - "country": { - "type": "string" - }, - "portal_invite_last_accepted_at": { - "type": ["null", "string"] - }, - "tax_exempt": { - "type": "boolean" - }, - "id": { - "type": "integer" - }, - "reference": { - "type": ["null", "string"] - }, - "last_name": { - "type": "string" - }, - "address_2": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "organization": { - "type": "string" - }, - "address": { - "type": "string" - }, - "verified": { - "type": ["null", "string"] - }, - "cc_emails": { - "type": "string" - }, - "state_name": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "subscriptions", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["integer"] - }, - "state": { - "type": ["string"] - }, - "balance_in_cents": { - "type": ["integer"] - }, - "total_revenue_in_cents": { - "type": ["integer"] - }, - "product_price_in_cents": { - "type": ["integer"] - }, - "product_version_number": { - "type": ["integer"] - }, - "current_period_ends_at": { - "type": ["string"] - }, - "next_assessment_at": { - "type": ["string"], - "format": "date-time" - }, - "trial_started_at": { - "type": ["string"], - "format": "date-time" - }, - "trial_ended_at": { - "type": ["string"], - "format": "date-time" - }, - "activated_at": { - "type": ["string"], - "format": "date-time" - }, - "expires_at": { - "type": ["string"], - "format": "date-time" - }, - "created_at": { - "type": ["string"], - "format": "date-time" - }, - "updated_at": { - "type": ["string"], - "format": "date-time" - }, - "cancellation_message": { - "type": ["string"] - }, - "cancellation_method": { - "type": ["null", "string"] - }, - "cancel_at_end_of_period": { - "type": ["boolean"] - }, - "canceled_at": { - "type": ["string"], - "format": "date-time" - }, - "current_period_started_at": { - "type": ["string"], - "format": "date-time" - }, - "previous_state": { - "type": ["string"] - }, - "signup_payment_id": { - "type": ["integer"] - }, - "signup_revenue": { - "type": ["string"] - }, - "delayed_cancel_at": { - "type": ["string"], - "format": "date-time" - }, - "coupon_code": { - "type": ["string"] - }, - "snap_day": { - "type": ["string"] - }, - "payment_collection_method": { - "type": ["string"] - }, - "customer": { - "type": ["object"] - }, - "product": { - "type": ["object"] - }, - "credit_card": { - "type": ["object"] - }, - "group": { - "type": ["null", "object"] - }, - "bank_account": { - "type": ["object"] - }, - "payment_type": { - "type": ["string"] - }, - "referral_code": { - "type": ["string"] - }, - "next_product_id": { - "type": ["string"] - }, - "next_product_handle": { - "type": ["string"] - }, - "coupon_use_count": { - "type": ["integer"] - }, - "coupon_uses_allowed": { - "type": ["integer"] - }, - "reason_code": { - "type": ["string"] - }, - "automatically_resume_at": { - "type": ["string"], - "format": "date-time" - }, - "coupon_codes": { - "type": ["array"] - }, - "offer_id": { - "type": ["string"] - }, - "payer_id": { - "type": ["integer"] - }, - "current_billing_amount_in_cents": { - "type": ["integer"] - }, - "product_price_point_id": { - "type": ["integer"] - }, - "next_product_price_point_id": { - "type": ["integer"] - }, - "net_terms": { - "type": ["integer"] - }, - "stored_credential_transaction_id": { - "type": ["integer"] - }, - "reference": { - "type": ["string"] - }, - "on_hold_at": { - "type": ["string"], - "format": "date-time" - }, - "prepaid_dunning": { - "type": ["boolean"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-chargify/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-chargify/metadata.yaml b/airbyte-integrations/connectors/source-chargify/metadata.yaml index a596bd6e181f..873ecf5a0113 100644 --- a/airbyte-integrations/connectors/source-chargify/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargify/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - ${domain} + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 9b2d3607-7222-4709-9fa2-c2abdebbdd88 - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-chargify githubIssueLabel: source-chargify icon: chargify.svg license: MIT name: Chargify - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2023-08-11 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/chargify tags: - - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-chargify/requirements.txt b/airbyte-integrations/connectors/source-chargify/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-chargify/requirements.txt +++ b/airbyte-integrations/connectors/source-chargify/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-chargify/setup.py b/airbyte-integrations/connectors/source-chargify/setup.py index 7cc100b04b21..521c61646023 100644 --- a/airbyte-integrations/connectors/source-chargify/setup.py +++ b/airbyte-integrations/connectors/source-chargify/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( @@ -22,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/__init__.py b/airbyte-integrations/connectors/source-chargify/source_chargify/__init__.py index fb413107366d..233a409d39e7 100644 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/__init__.py +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml b/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml new file mode 100644 index 000000000000..2b2f677e92b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml @@ -0,0 +1,99 @@ +version: 0.29.0 +type: DeclarativeSource + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - "*" + - "{{ parameters.field_path }}" + + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 200 + start_from_page: 1 + + requester: + type: HttpRequester + url_base: "https://{{ config['domain'] }}/" + path: "{{ parameters.path }}" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + password: "x" + username: "{{ config['api_key'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + + customers_stream: + $ref: "#/definitions/base_stream" + name: "customers" + $parameters: + path: "customers.json" + field_path: "customer" + + subscriptions_stream: + $ref: "#/definitions/base_stream" + name: "subscriptions" + $parameters: + path: "subscriptions.json" + field_path: "subscription" + +streams: + - "#/definitions/customers_stream" + - "#/definitions/subscriptions_stream" + +check: + type: CheckStream + stream_names: + - customers + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/chargify + connection_specification: + title: Chargify Spec + type: object + required: + - api_key + - domain + additionalProperties: true + properties: + api_key: + type: string + title: API Key + order: 0 + description: Maxio Advanced Billing/Chargify API Key. + airbyte_secret: true + domain: + type: string + order: 1 + title: Domain + description: >- + Chargify domain. Normally this domain follows the following format + companyname.chargify.com diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/customers.json b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/customers.json index 6b80d4074023..21003d566321 100644 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/customers.json +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/customers.json @@ -1,58 +1,39 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "required": [ - "portal_invite_last_sent_at", - "email", - "created_at", - "zip", - "first_name", - "state", - "city", - "portal_customer_created_at", - "updated_at", - "country", - "tax_exempt", - "id", - "last_name", - "address_2", - "phone", - "organization", - "address", - "cc_emails" - ], + "additionalProperties": true, "properties": { "default_subscription_group_uid": { "type": ["null", "string"] }, "portal_invite_last_sent_at": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "vat_number": { "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "zip": { - "type": "string" + "type": ["null", "string"] }, "first_name": { - "type": "string" + "type": ["null", "string"] }, "country_name": { "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "parent_id": { "type": ["null", "integer"] @@ -61,47 +42,47 @@ "type": ["null", "string"] }, "portal_customer_created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "country": { - "type": "string" + "type": ["null", "string"] }, "portal_invite_last_accepted_at": { "type": ["null", "string"] }, "tax_exempt": { - "type": "boolean" + "type": ["null", "boolean"] }, "id": { - "type": "integer" + "type": ["null", "integer"] }, "reference": { "type": ["null", "string"] }, "last_name": { - "type": "string" + "type": ["null", "string"] }, "address_2": { - "type": "string" + "type": ["null", "string"] }, "phone": { - "type": "string" + "type": ["null", "string"] }, "organization": { - "type": "string" + "type": ["null", "string"] }, "address": { - "type": "string" + "type": ["null", "string"] }, "verified": { "type": ["null", "string"] }, "cc_emails": { - "type": "string" + "type": ["null", "string"] }, "state_name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json index b8a9eed74af3..0a3c44a610d3 100644 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json @@ -1,293 +1,298 @@ { "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "balance_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "total_revenue_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "product_price_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "product_version_number": { - "type": "integer" + "type": ["null", "integer"] }, "current_period_ends_at": { - "type": "string" + "type": ["null", "string"] }, "next_assessment_at": { - "type": "string" + "type": ["null", "string"] }, "trial_started_at": { - "type": "string" + "type": ["null", "string"] }, "trial_ended_at": { - "type": "string" + "type": ["null", "string"] }, "activated_at": { - "type": "string" + "type": ["null", "string"] }, "expires_at": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] }, "cancellation_message": { - "type": "string" + "type": ["null", "string"] }, "cancellation_method": { - "type": "string" + "type": ["null", "string"] }, "cancel_at_end_of_period": { - "type": "boolean" + "type": ["null", "boolean"] }, "canceled_at": { - "type": "string" + "type": ["null", "string"] }, "current_period_started_at": { - "type": "string" + "type": ["null", "string"] }, "previous_state": { - "type": "string" + "type": ["null", "string"] }, "signup_payment_id": { - "type": "integer" + "type": ["null", "integer"] }, "signup_revenue": { - "type": "string" + "type": ["null", "string"] }, "delayed_cancel_at": { - "type": "string" + "type": ["null", "string"] }, "coupon_code": { - "type": "string" + "type": ["null", "string"] }, "snap_day": { - "type": "string" + "type": ["null", "string"] }, "payment_collection_method": { - "type": "string" + "type": ["null", "string"] }, "customer": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "first_name": { - "type": "string" + "type": ["null", "string"] }, "last_name": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "cc_emails": { - "type": "string" + "type": ["null", "string"] }, "organization": { - "type": "string" + "type": ["null", "string"] }, "reference": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "integer" + "type": ["null", "integer"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] }, "address": { - "type": "string" + "type": ["null", "string"] }, "address_2": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "state_name": { - "type": "string" + "type": ["null", "string"] }, "zip": { - "type": "string" + "type": ["null", "string"] }, "country": { - "type": "string" + "type": ["null", "string"] }, "country_name": { - "type": "string" + "type": ["null", "string"] }, "phone": { - "type": "string" + "type": ["null", "string"] }, "verified": { - "type": "boolean" + "type": ["null", "boolean"] }, "portal_customer_created_at": { - "type": "string" + "type": ["null", "string"] }, "portal_invite_last_sent_at": { - "type": "string" + "type": ["null", "string"] }, "portal_invite_last_accepted_at": { "type": ["null", "string"] }, "tax_exempt": { - "type": "boolean" + "type": ["null", "boolean"] }, "vat_number": { - "type": "string" + "type": ["null", "string"] }, "parent_id": { - "type": "integer" + "type": ["null", "integer"] }, "locale": { - "type": "string" + "type": ["null", "string"] }, "default_subscription_group_uid": { - "type": "string" + "type": ["null", "string"] } } }, "product": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "handle": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "accounting_code": { - "type": "string" + "type": ["null", "string"] }, "request_credit_card": { - "type": "boolean" + "type": ["null", "boolean"] }, "expiration_interval": { - "type": "integer" + "type": ["null", "integer"] }, "expiration_interval_unit": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] }, "price_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "interval": { - "type": "integer" + "type": ["null", "integer"] }, "interval_unit": { - "type": "string" + "type": ["null", "string"] }, "initial_charge_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "trial_price_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "trial_interval": { - "type": "integer" + "type": ["null", "integer"] }, "trial_interval_unit": { - "type": "string" + "type": ["null", "string"] }, "archived_at": { - "type": "string" + "type": ["null", "string"] }, "require_credit_card": { - "type": "boolean" + "type": ["null", "boolean"] }, "return_params": { - "type": "string" + "type": ["null", "string"] }, "taxable": { - "type": "boolean" + "type": ["null", "boolean"] }, "update_return_url": { - "type": "string" + "type": ["null", "string"] }, "initial_charge_after_trial": { - "type": "boolean" + "type": ["null", "boolean"] }, "version_number": { - "type": "integer" + "type": ["null", "integer"] }, "update_return_params": { - "type": "string" + "type": ["null", "string"] }, "product_family": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "handle": { - "type": "string" + "type": ["null", "string"] }, "accounting_code": { - "type": "null" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] } } }, "public_signup_pages": { - "type": "array", + "type": ["null", "array"], "items": { "anyOf": [ { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "return_url": { - "type": "string" + "type": ["null", "string"] }, "return_params": { - "type": "string" + "type": ["null", "string"] }, "url": { - "type": "string" + "type": ["null", "string"] } } } @@ -295,244 +300,259 @@ } }, "product_price_point_name": { - "type": "string" + "type": ["null", "string"] }, "request_billing_address": { - "type": "boolean" + "type": ["null", "boolean"] }, "require_billing_address": { - "type": "boolean" + "type": ["null", "boolean"] }, "require_shipping_address": { - "type": "boolean" + "type": ["null", "boolean"] }, "tax_code": { - "type": "string" + "type": ["null", "string"] }, "default_product_price_point_id": { - "type": "integer" + "type": ["null", "integer"] } } }, "credit_card": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "first_name": { - "type": "string" + "type": ["null", "string"] }, "last_name": { - "type": "string" + "type": ["null", "string"] }, "masked_card_number": { - "type": "string" + "type": ["null", "string"] }, "card_type": { - "type": "string" + "type": ["null", "string"] }, "expiration_month": { - "type": "integer" + "type": ["null", "integer"] }, "expiration_year": { - "type": "integer" + "type": ["null", "integer"] }, "customer_id": { - "type": "integer" + "type": ["null", "integer"] }, "current_vault": { - "type": "string" + "type": ["null", "string"] }, "vault_token": { - "type": "string" + "type": ["null", "string"] }, "billing_address": { - "type": "string" + "type": ["null", "string"] }, "billing_city": { - "type": "string" + "type": ["null", "string"] }, "billing_state": { - "type": "string" + "type": ["null", "string"] }, "billing_zip": { - "type": "string" + "type": ["null", "string"] }, "billing_country": { - "type": "string" + "type": ["null", "string"] }, "customer_vault_token": { - "type": "string" + "type": ["null", "string"] }, "billing_address_2": { - "type": "string" + "type": ["null", "string"] }, "payment_type": { - "type": "string" + "type": ["null", "string"] }, "disabled": { - "type": "boolean" + "type": ["null", "boolean"] }, "chargify_token": { - "type": "string" + "type": ["null", "string"] }, "site_gateway_setting_id": { - "type": "integer" + "type": ["null", "integer"] }, "gateway_handle": { - "type": "string" + "type": ["null", "string"] } } }, "group": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "uid": { - "type": "string" + "type": ["null", "string"] }, "scheme": { - "type": "string" + "type": ["null", "string"] }, "primary_subscription_id": { - "type": "string" + "type": ["null", "string"] }, "primary": { - "type": "string" + "type": ["null", "string"] } } }, "bank_account": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "bank_account_holder_type": { - "type": "string" + "type": ["null", "string"] }, "bank_account_type": { - "type": "string" + "type": ["null", "string"] }, "bank_name": { - "type": "string" + "type": ["null", "string"] }, "billing_address": { - "type": "string" + "type": ["null", "string"] }, "billing_address_2": { - "type": "string" + "type": ["null", "string"] }, "billing_city": { - "type": "string" + "type": ["null", "string"] }, "billing_state": { - "type": "string" + "type": ["null", "string"] }, "billing_zip": { - "type": "string" + "type": ["null", "string"] }, "billing_country": { - "type": "string" + "type": ["null", "string"] }, "current_vault": { - "type": "string" + "type": ["null", "string"] }, "customer_id": { - "type": "integer" + "type": ["null", "integer"] }, "customer_vault_token": { - "type": "string" + "type": ["null", "string"] }, "first_name": { - "type": "string" + "type": ["null", "string"] }, "last_name": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "integer" + "type": ["null", "integer"] }, "masked_bank_account_number": { - "type": "string" + "type": ["null", "string"] }, "masked_bank_routing_number": { - "type": "string" + "type": ["null", "string"] }, "vault_token": { - "type": "string" + "type": ["null", "string"] }, "chargify_token": { - "type": "string" + "type": ["null", "string"] }, "site_gateway_setting_id": { - "type": "integer" + "type": ["null", "integer"] }, "gateway_handle": { - "type": "string" + "type": ["null", "string"] } } }, "payment_type": { - "type": "string" + "type": ["null", "string"] }, "referral_code": { - "type": "string" + "type": ["null", "string"] }, "next_product_id": { - "type": "integer" + "type": ["null", "integer"] }, "next_product_handle": { - "type": "string" + "type": ["null", "string"] }, "coupon_use_count": { - "type": "integer" + "type": ["null", "integer"] }, "coupon_uses_allowed": { - "type": "integer" + "type": ["null", "integer"] + }, + "product_price_point_type": { + "type": ["null", "string"] + }, + "coupons": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } }, "reason_code": { - "type": "string" + "type": ["null", "string"] }, "automatically_resume_at": { - "type": "string" + "type": ["null", "string"] }, "coupon_codes": { - "type": "array", + "type": ["null", "array"], "items": { - "anyOf": [ - { - "type": "string" - } - ] + "type": ["null", "string"] } }, "offer_id": { - "type": "string" + "type": ["null", "integer"] }, "payer_id": { - "type": "integer" + "type": ["null", "integer"] }, "current_billing_amount_in_cents": { - "type": "integer" + "type": ["null", "integer"] }, "product_price_point_id": { - "type": "integer" + "type": ["null", "integer"] }, "next_product_price_point_id": { - "type": "integer" + "type": ["null", "integer"] }, "net_terms": { - "type": "integer" + "type": ["null", "integer"] }, "stored_credential_transaction_id": { - "type": "integer" + "type": ["null", "integer"] }, "reference": { - "type": "string" + "type": ["null", "string"] }, "on_hold_at": { - "type": "string" + "type": ["null", "string"] }, "prepaid_dunning": { - "type": "boolean" + "type": ["null", "boolean"] + }, + "dunning_communication_delay_enabled": { + "type": ["null", "boolean"] + }, + "dunning_communication_delay_time_zone": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/source.py b/airbyte-integrations/connectors/source-chargify/source_chargify/source.py index 4d40c437cae5..7c9d8f287428 100644 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/source.py +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/source.py @@ -2,115 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class ChargifyStream(HttpStream, ABC): - - PER_PAGE = 200 - FIRST_PAGE = 1 - - def __init__(self, *args, domain: str, **kwargs): - super().__init__(*args, **kwargs) - self._domain = domain - - @property - def url_base(self): - return f"https://{self._domain}" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - - results = response.json() - - if results: - if len(results) == self.PER_PAGE: - url_query = urlparse(response.url).query - query_params = parse_qs(url_query) - - new_params = {param_name: param_value[0] for param_name, param_value in query_params.items()} - if "page" in new_params: - new_params["page"] = int(new_params["page"]) + 1 - return new_params - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - if next_page_token is None: - return {"page": self.FIRST_PAGE, "per_page": self.PER_PAGE} - - return next_page_token - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - - yield response.json() - - -class Customers(ChargifyStream): - - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "customers.json" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # Chargify API: https://developers.chargify.com/docs/api-docs/b3A6MTQxMDgyNzY-list-or-find-customers - # it returns a generator of Customers objects. - customers = response.json() - for customer in customers: - yield customer["customer"] - - -class Subscriptions(ChargifyStream): - - primary_key = "id" - - def path(self, **kwargs) -> str: - - return "subscriptions.json" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # Chargify API: https://developers.chargify.com/docs/api-docs/b3A6MTQxMDgzODk-list-subscriptions - # it returns a generator of Subscriptions objects. - subscriptions = response.json() - for subscription in subscriptions: - yield subscription["subscription"] - - -# Source -class SourceChargify(AbstractSource): - BASIC_AUTH_PASSWORD = "x" - - def get_basic_auth(self, config: Mapping[str, Any]) -> requests.auth.HTTPBasicAuth: - return requests.auth.HTTPBasicAuth( - config["api_key"], SourceChargify.BASIC_AUTH_PASSWORD - ) # https://developers.chargify.com/docs/api-docs/YXBpOjE0MTA4MjYx-chargify-api-documentation - - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - try: - authenticator = self.get_basic_auth(config) - customers_gen = Customers(authenticator, domain=config["domain"]).read_records(SyncMode.full_refresh) - next(customers_gen) - subcriptions_gen = Subscriptions(authenticator, domain=config["domain"]).read_records(SyncMode.full_refresh) - next(subcriptions_gen) - return True, None - except Exception as error: - return False, f"Unable to connect to Chargify API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = self.get_basic_auth(config) - return [Customers(authenticator, domain=config["domain"]), Subscriptions(authenticator, domain=config["domain"])] +# Declarative Source +class SourceChargify(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/spec.json b/airbyte-integrations/connectors/source-chargify/source_chargify/spec.json deleted file mode 100644 index 5e0f26fde644..000000000000 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/spec.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/chargify", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Chargify Spec", - "type": "object", - "required": ["api_key", "domain"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "Chargify API Key.", - "airbyte_secret": true - }, - "domain": { - "type": "string", - "description": "Chargify domain. Normally this domain follows the following format companyname.chargify.com" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-chargify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-chargify/unit_tests/test_source.py deleted file mode 100644 index b330567842ac..000000000000 --- a/airbyte-integrations/connectors/source-chargify/unit_tests/test_source.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_chargify.source import SourceChargify - - -def test_streams(mocker): - source = SourceChargify() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-chargify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-chargify/unit_tests/test_streams.py deleted file mode 100644 index 02117eb57173..000000000000 --- a/airbyte-integrations/connectors/source-chargify/unit_tests/test_streams.py +++ /dev/null @@ -1,135 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pytest -import requests -from source_chargify.source import ChargifyStream, Customers, Subscriptions - - -@pytest.fixture() -def ChargifyStreamInstance(mocker) -> ChargifyStream: - - mocker.patch.object(ChargifyStream, "path", "v0/example_endpoint") - mocker.patch.object(ChargifyStream, "primary_key", "test_primary_key") - mocker.patch.object(ChargifyStream, "__abstractmethods__", set()) - - return ChargifyStream( - authenticator=MagicMock(), - domain="test", - ) - - -@pytest.fixture() -def CustomerStreamInstance(mocker) -> Customers: - - mocker.patch.object(Customers, "path", "v0/example_endpoint") - mocker.patch.object(Customers, "primary_key", "test_primary_key") - mocker.patch.object(Customers, "__abstractmethods__", set()) - - return Customers(authenticator=MagicMock(), domain="test") - - -@pytest.fixture() -def SubscriptionsStreamInstance(mocker) -> Subscriptions: - - mocker.patch.object(Subscriptions, "path", "v0/example_endpoint") - mocker.patch.object(Subscriptions, "primary_key", "test_primary_key") - mocker.patch.object(Subscriptions, "__abstractmethods__", set()) - - return Subscriptions( - authenticator=MagicMock(), - domain="test", - ) - - -@pytest.mark.parametrize("domain", [("test"), ("test1"), ("test2")]) -def test_stream_config(domain, mocker): - - mocker.patch.object(ChargifyStream, "path", "v0/example_endpoint") - mocker.patch.object(ChargifyStream, "primary_key", "test_primary_key") - mocker.patch.object(ChargifyStream, "__abstractmethods__", set()) - - stream: ChargifyStream = ChargifyStream( - domain=domain, - ) - assert stream._domain == domain - - customers_stream: Customers = Customers(domain=domain) - assert customers_stream.path() == "customers.json" - assert customers_stream.primary_key == "id" - - subscriptions_stream: Subscriptions = Subscriptions(domain=domain) - assert subscriptions_stream.path() == "subscriptions.json" - assert subscriptions_stream.primary_key == "id" - - -def test_next_page_token(ChargifyStreamInstance: ChargifyStream): - response = requests.Response() - response.url = "https://test.chargify.com/subscriptions.json?page=1&per_page=2" - response.json = MagicMock() - response.json.return_value = [{"id": 1}, {"id": 2}] - - ChargifyStream.PER_PAGE = 2 - - token_params = ChargifyStreamInstance.next_page_token(response=response) - - assert token_params == {"page": 2, "per_page": "2"} - - response = requests.Response() - response.url = "https://test.chargify.com/subscriptions.json?page=1&per_page=2" - response.json = MagicMock() - response.json.return_value = {} - - token_params = ChargifyStreamInstance.next_page_token(response=response) - - assert token_params is None - - -def test_requests_params(ChargifyStreamInstance: ChargifyStream): - - ChargifyStream.PER_PAGE = 200 - - params = ChargifyStreamInstance.request_params(stream_state={}, next_page_token=None) - - assert params == {"page": 1, "per_page": 200} - - params = ChargifyStreamInstance.request_params(stream_state={}, next_page_token={"page": 2, "per_page": 200}) - - assert params == {"page": 2, "per_page": 200} - - -def test_parse_subscriptions_response(SubscriptionsStreamInstance: Subscriptions): - - response = MagicMock() - response.json.return_value = [ - {"subscription": {"id": 0, "state": "string", "balance_in_cents": 0}}, - {"subscription": {"id": 2, "state": "string", "balance_in_cents": 1000}}, - {"subscription": {"id": 3, "state": "string", "balance_in_cents": 100}}, - ] - - response = list(SubscriptionsStreamInstance.parse_response(response=response)) - - assert len(response) == 3 - assert response[0] == {"id": 0, "state": "string", "balance_in_cents": 0} - assert response[1] == {"id": 2, "state": "string", "balance_in_cents": 1000} - assert response[2] == {"id": 3, "state": "string", "balance_in_cents": 100} - - -def test_parse_customers_response(CustomerStreamInstance: Customers): - - response = MagicMock() - response.json.return_value = [ - {"customer": {"id": 0, "state": "string", "balance_in_cents": 0}}, - {"customer": {"id": 2, "state": "string", "balance_in_cents": 1000}}, - {"customer": {"id": 3, "state": "string", "balance_in_cents": 100}}, - ] - - response = list(CustomerStreamInstance.parse_response(response=response)) - - assert len(response) == 3 - assert response[0] == {"id": 0, "state": "string", "balance_in_cents": 0} - assert response[1] == {"id": 2, "state": "string", "balance_in_cents": 1000} - assert response[2] == {"id": 3, "state": "string", "balance_in_cents": 100} diff --git a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml index e61589556fb0..6f4af3668c87 100644 --- a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-chartmogul/requirements.txt b/airbyte-integrations/connectors/source-chartmogul/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-chartmogul/requirements.txt +++ b/airbyte-integrations/connectors/source-chartmogul/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-chartmogul/setup.py b/airbyte-integrations/connectors/source-chartmogul/setup.py index dc371c20aaec..76bc706ad50c 100644 --- a/airbyte-integrations/connectors/source-chartmogul/setup.py +++ b/airbyte-integrations/connectors/source-chartmogul/setup.py @@ -12,7 +12,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-clickhouse/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-clickhouse/integration_tests/seed/basic.sql index bf8cc757300c..78b7552f0eb9 100644 --- a/airbyte-integrations/connectors/source-clickhouse/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-clickhouse/integration_tests/seed/basic.sql @@ -1,5 +1,51 @@ -CREATE TABLE IF NOT EXISTS default.id_and_name (id INTEGER, name VARCHAR(200)) ENGINE = TinyLog; -INSERT INTO default.id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash'); +CREATE + TABLE + IF NOT EXISTS default.id_and_name( + id INTEGER, + name VARCHAR(200) + ) ENGINE = TinyLog; -CREATE TABLE IF NOT EXISTS default.starships (id INTEGER, name VARCHAR(200)) ENGINE = TinyLog; -INSERT INTO default.starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato'); +INSERT + INTO + default.id_and_name( + id, + name + ) + VALUES( + 1, + 'picard' + ), + ( + 2, + 'crusher' + ), + ( + 3, + 'vash' + ); + +CREATE + TABLE + IF NOT EXISTS default.starships( + id INTEGER, + name VARCHAR(200) + ) ENGINE = TinyLog; + +INSERT + INTO + default.starships( + id, + name + ) + VALUES( + 1, + 'enterprise-d' + ), + ( + 2, + 'defiant' + ), + ( + 3, + 'yamato' + ); diff --git a/airbyte-integrations/connectors/source-clickhouse/metadata.yaml b/airbyte-integrations/connectors/source-clickhouse/metadata.yaml index f36cbb184561..07806c1237f3 100644 --- a/airbyte-integrations/connectors/source-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickhouse/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 200 + sl: 100 allowedHosts: hosts: - ${host} @@ -8,6 +11,7 @@ data: definitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerImageTag: 0.1.17 dockerRepository: airbyte/source-clickhouse + documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse githubIssueLabel: source-clickhouse icon: clickhouse.svg license: MIT @@ -20,7 +24,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-clickup-api/metadata.yaml b/airbyte-integrations/connectors/source-clickup-api/metadata.yaml index afda2daedf00..a59f15841d5d 100644 --- a/airbyte-integrations/connectors/source-clickup-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickup-api/metadata.yaml @@ -14,8 +14,12 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/click-up + documentationUrl: https://docs.airbyte.com/integrations/sources/clickup-api tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-clickup-api/requirements.txt b/airbyte-integrations/connectors/source-clickup-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-clickup-api/requirements.txt +++ b/airbyte-integrations/connectors/source-clickup-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-clickup-api/setup.py b/airbyte-integrations/connectors/source-clickup-api/setup.py index 2a61de913d8d..4fb2367a7516 100644 --- a/airbyte-integrations/connectors/source-clickup-api/setup.py +++ b/airbyte-integrations/connectors/source-clickup-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-clockify/Dockerfile b/airbyte-integrations/connectors/source-clockify/Dockerfile index a19cd8507ded..4a69b4505941 100644 --- a/airbyte-integrations/connectors/source-clockify/Dockerfile +++ b/airbyte-integrations/connectors/source-clockify/Dockerfile @@ -34,5 +34,5 @@ COPY source_clockify ./source_clockify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-clockify diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml index 34c057d5c0bd..b1555d69f8e3 100644 --- a/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml @@ -1,19 +1,27 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-clockify:dev -tests: +acceptance_tests: spec: - - spec_path: "source_clockify/spec.json" + tests: + - spec_path: "source_clockify/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + previous_connector_version: "0.2.0" + disable_for_version: "0.2.0" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-clockify/metadata.yaml b/airbyte-integrations/connectors/source-clockify/metadata.yaml index 16e677c9398a..f0425dc400cb 100644 --- a/airbyte-integrations/connectors/source-clockify/metadata.yaml +++ b/airbyte-integrations/connectors/source-clockify/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: e71aae8a-5143-11ed-bdc3-0242ac120002 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-clockify githubIssueLabel: source-clockify icon: clockify.svg @@ -10,11 +10,15 @@ data: name: Clockify registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/clockify tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-clockify/requirements.txt b/airbyte-integrations/connectors/source-clockify/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-clockify/requirements.txt +++ b/airbyte-integrations/connectors/source-clockify/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-clockify/setup.py b/airbyte-integrations/connectors/source-clockify/setup.py index 35831254f888..5836e4cbb175 100644 --- a/airbyte-integrations/connectors/source-clockify/setup.py +++ b/airbyte-integrations/connectors/source-clockify/setup.py @@ -6,10 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2.0", + "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "responses"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses"] setup( name="source_clockify", diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json index 00a10979e40c..069971965775 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/clients.json @@ -1,23 +1,27 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "address": { "type": ["null", "string"] }, "archived": { - "type": "boolean" + "type": ["null", "boolean"] }, "id": { - "type": "string" + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "note": { "type": ["null", "string"] }, "workspaceId": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json index faa01b58a0fd..71a34a58df0b 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/projects.json @@ -1,11 +1,12 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "archived": { - "type": "boolean" + "type": ["null", "boolean"] }, "billable": { - "type": "boolean" + "type": ["null", "boolean"] }, "budgetEstimate": { "anyOf": [ @@ -21,27 +22,27 @@ "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "resetOption": { "type": ["null", "string"] }, "active": { - "type": "boolean" + "type": ["null", "boolean"] } }, - "type": "object" + "type": ["null", "object"] } ] }, "clientId": { - "type": "string" + "type": ["null", "string"] }, "clientName": { - "type": "string" + "type": ["null", "string"] }, "color": { - "type": "string" + "type": ["null", "string"] }, "costRate": { "anyOf": [ @@ -60,12 +61,12 @@ "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] } ] }, "duration": { - "type": "string" + "type": ["null", "string"] }, "estimate": { "properties": { @@ -76,27 +77,27 @@ "type": "string" } }, - "type": "object" + "type": ["null", "object"] }, "hourlyRate": { "properties": { "amount": { - "type": "integer" + "type": ["null", "integer"] }, "currency": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "memberships": { "items": { "properties": { "costRate": { - "type": "null" + "type": ["null"] }, "hourlyRate": { "anyOf": [ @@ -106,67 +107,67 @@ { "properties": { "amount": { - "type": "integer" + "type": ["null", "integer"] }, "currency": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] } ] }, "membershipStatus": { - "type": "string" + "type": ["null", "string"] }, "membershipType": { - "type": "string" + "type": ["null", "string"] }, "targetId": { - "type": "string" + "type": ["null", "string"] }, "userId": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] }, - "type": "array" + "type": ["null", "array"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "note": { - "type": "string" + "type": ["null", "string"] }, "public": { - "type": "boolean" + "type": ["null", "boolean"] }, "template": { - "type": "boolean" + "type": ["null", "boolean"] }, "timeEstimate": { "properties": { "active": { - "type": "boolean" + "type": ["null", "boolean"] }, "estimate": { - "type": "string" + "type": ["null", "string"] }, "includeNonBillable": { - "type": "boolean" + "type": ["null", "boolean"] }, "resetOption": { "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] }, "workspaceId": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json index 75b53dd8cfea..cbef1bec6b9a 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tags.json @@ -1,17 +1,18 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "archived": { - "type": "boolean" + "type": ["null", "boolean"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "workspaceId": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json index 441785586cce..df6eef85a5e3 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json @@ -1,17 +1,18 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "assigneeId": { "type": ["null", "string"] }, "assigneeIds": { "items": { - "type": "string" + "type": ["null", "string"] }, - "type": "array" + "type": ["null", "array"] }, "billable": { - "type": "boolean" + "type": ["null", "boolean"] }, "costRate": { "anyOf": [ @@ -30,7 +31,7 @@ "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] } ] }, @@ -38,7 +39,7 @@ "type": ["null", "string"] }, "estimate": { - "type": "string" + "type": ["null", "string"] }, "hourlyRate": { "anyOf": [ @@ -48,30 +49,30 @@ { "properties": { "amount": { - "type": "integer" + "type": ["null", "integer"] }, "currency": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] } ] }, "id": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "projectId": { - "type": "string" + "type": ["null", "string"] }, "status": { - "type": "string" + "type": ["null", "string"] }, "userGroupIds": { - "type": "array" + "type": ["null", "array"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json index 8fb163736583..4dbcabeabad3 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/time_entries.json @@ -1,20 +1,21 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "billable": { - "type": "boolean" + "type": ["null", "boolean"] }, "customFieldValues": { - "type": "array" + "type": ["null", "array"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "isLocked": { - "type": "boolean" + "type": ["null", "boolean"] }, "kioskId": { "type": ["null", "string"] @@ -29,9 +30,9 @@ }, { "items": { - "type": "string" + "type": ["null", "string"] }, - "type": "array" + "type": ["null", "array"] } ] }, @@ -41,25 +42,25 @@ "timeInterval": { "properties": { "duration": { - "type": "string" + "type": ["null", "string"] }, "end": { - "type": "string" + "type": ["null", "string"] }, "start": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "userId": { - "type": "string" + "type": ["null", "string"] }, "workspaceId": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json index f7ffaae51e6b..7e51183f8b1e 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/user_groups.json @@ -1,20 +1,21 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "userIds": { "items": { - "type": "string" + "type": ["null", "string"] }, - "type": "array" + "type": ["null", "array"] }, "workspaceId": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json index 7d3d6d27cbd7..02e2a5d31bcb 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/users.json @@ -1,132 +1,133 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "properties": { "activeWorkspace": { - "type": "string" + "type": ["null", "string"] }, "customFields": { - "type": "array" + "type": ["null", "array"] }, "defaultWorkspace": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "memberships": { - "type": "array" + "type": ["null", "array"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "profilePicture": { - "type": "string" + "type": ["null", "string"] }, "settings": { "properties": { "alerts": { - "type": "boolean" + "type": ["null", "boolean"] }, "approval": { - "type": "boolean" + "type": ["null", "boolean"] }, "collapseAllProjectLists": { - "type": "boolean" + "type": ["null", "boolean"] }, "dashboardPinToTop": { - "type": "boolean" + "type": ["null", "boolean"] }, "dashboardSelection": { - "type": "string" + "type": ["null", "string"] }, "dashboardViewType": { - "type": "string" + "type": ["null", "string"] }, "dateFormat": { - "type": "string" + "type": ["null", "string"] }, "groupSimilarEntriesDisabled": { - "type": "boolean" + "type": ["null", "boolean"] }, "isCompactViewOn": { - "type": "boolean" + "type": ["null", "boolean"] }, "lang": { - "type": "string" + "type": ["null", "string"] }, "longRunning": { - "type": "boolean" + "type": ["null", "boolean"] }, "multiFactorEnabled": { - "type": "boolean" + "type": ["null", "boolean"] }, "myStartOfDay": { - "type": "string" + "type": ["null", "string"] }, "onboarding": { - "type": "boolean" + "type": ["null", "boolean"] }, "projectListCollapse": { - "type": "integer" + "type": ["null", "integer"] }, "projectPickerTaskFilter": { - "type": "boolean" + "type": ["null", "boolean"] }, "pto": { - "type": "boolean" + "type": ["null", "boolean"] }, "reminders": { - "type": "boolean" + "type": ["null", "boolean"] }, "scheduledReports": { - "type": "boolean" + "type": ["null", "boolean"] }, "scheduling": { - "type": "boolean" + "type": ["null", "boolean"] }, "sendNewsletter": { - "type": "boolean" + "type": ["null", "boolean"] }, "showOnlyWorkingDays": { - "type": "boolean" + "type": ["null", "boolean"] }, "summaryReportSettings": { "properties": { "group": { - "type": "string" + "type": ["null", "string"] }, "subgroup": { - "type": "string" + "type": ["null", "string"] } }, - "type": "object" + "type": ["null", "object"] }, "theme": { - "type": "string" + "type": ["null", "string"] }, "timeFormat": { - "type": "string" + "type": ["null", "string"] }, "timeTrackingManual": { - "type": "boolean" + "type": ["null", "boolean"] }, "timeZone": { - "type": "string" + "type": ["null", "string"] }, "weekStart": { - "type": "string" + "type": ["null", "string"] }, "weeklyUpdates": { - "type": "boolean" + "type": ["null", "boolean"] } }, - "type": "object" + "type": ["null", "object"] }, "status": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/source.py b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py index aeab35939950..37547b049a2f 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/source.py +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py @@ -19,6 +19,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: workspace_stream = Users( authenticator=TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method=""), workspace_id=config["workspace_id"], + api_url=config["api_url"], ) next(workspace_stream.read_records(sync_mode=SyncMode.full_refresh)) return True, None @@ -28,6 +29,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method="") - args = {"authenticator": authenticator, "workspace_id": config["workspace_id"]} + args = {"authenticator": authenticator, "workspace_id": config["workspace_id"], "api_url": config["api_url"]} return [Users(**args), Projects(**args), Clients(**args), Tags(**args), UserGroups(**args), TimeEntries(**args), Tasks(**args)] diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json index ecd182c8e160..42756964f11a 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json @@ -17,6 +17,12 @@ "description": "You can get your api access_key here This API is Case Sensitive.", "type": "string", "airbyte_secret": true + }, + "api_url": { + "title": "API Url", + "description": "The URL for the Clockify API. This should only need to be modified if connecting to an enterprise version of Clockify.", + "type": "string", + "default": "https://api.clockify.me" } } } diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py index 387e31adc5a9..f20462415943 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py @@ -13,13 +13,17 @@ class ClockifyStream(HttpStream, ABC): - url_base = "https://api.clockify.me/api/v1/" + url_base = "" + api_url = "" + api_path = "/api/v1/" page_size = 50 page = 1 primary_key = None - def __init__(self, workspace_id: str, **kwargs): + def __init__(self, workspace_id: str, api_url: str, **kwargs): super().__init__(**kwargs) + self.api_url = api_url + self.url_base = self.api_url + self.api_path self.workspace_id = workspace_id def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -27,6 +31,8 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, self.page = self.page + 1 if next_page: return {"page": self.page} + else: + self.page = 1 def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = { @@ -76,11 +82,12 @@ def path(self, **kwargs) -> str: class TimeEntries(HttpSubStream, ClockifyStream): - def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], **kwargs): + def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], api_url: str, **kwargs): super().__init__( authenticator=authenticator, workspace_id=workspace_id, - parent=Users(authenticator=authenticator, workspace_id=workspace_id, **kwargs), + api_url=api_url, + parent=Users(authenticator=authenticator, workspace_id=workspace_id, api_url=api_url, **kwargs), ) def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: @@ -90,7 +97,7 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: so self._session.auth is used instead """ - users_stream = Users(authenticator=self._session.auth, workspace_id=self.workspace_id) + users_stream = Users(authenticator=self._session.auth, workspace_id=self.workspace_id, api_url=self.api_url) for user in users_stream.read_records(sync_mode=SyncMode.full_refresh): yield {"user_id": user["id"]} @@ -100,11 +107,12 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Tasks(HttpSubStream, ClockifyStream): - def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], **kwargs): + def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], api_url: str, **kwargs): super().__init__( authenticator=authenticator, workspace_id=workspace_id, - parent=Projects(authenticator=authenticator, workspace_id=workspace_id, **kwargs), + api_url=api_url, + parent=Projects(authenticator=authenticator, workspace_id=workspace_id, api_url=api_url, **kwargs), ) def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: @@ -114,7 +122,7 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: so self._session.auth is used instead """ - projects_stream = Projects(authenticator=self._session.auth, workspace_id=self.workspace_id) + projects_stream = Projects(authenticator=self._session.auth, workspace_id=self.workspace_id, api_url=self.api_url) for project in projects_stream.read_records(sync_mode=SyncMode.full_refresh): yield {"project_id": project["id"]} diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py index fd2c2d776448..f712b6c15dd9 100644 --- a/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py @@ -7,4 +7,4 @@ @pytest.fixture(scope="session", name="config") def config_fixture(): - return {"api_key": "test_api_key", "workspace_id": "workspace_id"} + return {"api_key": "test_api_key", "workspace_id": "workspace_id", "api_url": "http://some.test.url"} diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py index b7f54d5f699c..3cca00a0c4a4 100644 --- a/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py @@ -11,7 +11,7 @@ def setup_responses(): responses.add( responses.GET, - "https://api.clockify.me/api/v1/workspaces/workspace_id/users", + "http://some.test.url/api/v1/workspaces/workspace_id/users", json={"access_token": "test_api_key", "expires_in": 3600}, ) diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py index 63dbf772109f..debe32e0d4ac 100644 --- a/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py @@ -18,32 +18,32 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock()) + stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} expected_params = {"page-size": 50} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock()) + stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) inputs = {"response": MagicMock()} expected_token = {"page": 2} assert stream.next_page_token(**inputs) == expected_token def test_read_records(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock()) + stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) assert stream.read_records(sync_mode=SyncMode.full_refresh) def test_request_headers(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock()) + stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} expected_headers = {} assert stream.request_headers(**inputs) == expected_headers def test_http_method(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock()) + stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) expected_method = "GET" assert stream.http_method == expected_method diff --git a/airbyte-integrations/connectors/source-close-com/Dockerfile b/airbyte-integrations/connectors/source-close-com/Dockerfile index 1d547ebf5fd3..4deb69dd321d 100644 --- a/airbyte-integrations/connectors/source-close-com/Dockerfile +++ b/airbyte-integrations/connectors/source-close-com/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.4.2 LABEL io.airbyte.name=airbyte/source-close-com diff --git a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml index 2fd39e745bec..c1382737b9f5 100644 --- a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml @@ -50,7 +50,9 @@ acceptance_tests: - name: incoming_email_tasks bypass_reason: "unable to test due to fast-changing data" - name: created_activities - bypass_reason: "data deleted after some time" + bypass_reason: "return records randomly" + - name: opportunity_status_change_activities + bypass_reason: "return records randomly" fail_on_extra_columns: false incremental: tests: diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl index d284dd43b706..bfcc3040ef80 100644 --- a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl @@ -1,90 +1,90 @@ -{"stream": "created_activities", "data": {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": [], "date_created": "2021-07-13T11:39:05.012000+00:00", "id": "acti_CO9Th9maeKbnjp8XiG6gPada9OL2icBKBjuWK0aSzi5", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-07-13T11:39:05.012000+00:00", "date_updated": "2021-07-13T11:39:05.012000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "_type": "Created", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "import_id": null, "source": "ui", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "contact_id": null}, "emitted_at": 1683623831405} -{"stream": "created_activities", "data": {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": [], "date_created": "2021-07-13T11:39:03.413000+00:00", "id": "acti_Vu4EowJ5vQVaTp2UviBUWsah2gfHfzEhToD0qt20awK", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-07-13T11:39:03.413000+00:00", "date_updated": "2021-07-13T11:39:03.413000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "_type": "Created", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "import_id": null, "source": "ui", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "contact_id": null}, "emitted_at": 1683623831410} -{"stream": "created_activities", "data": {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": [], "date_created": "2021-07-04T11:39:00.850000+00:00", "id": "acti_1vDHprrVKawVMTbb4XmRtOJeP407lC7VeHwJu47cNzQ", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-07-04T11:39:00.850000+00:00", "date_updated": "2021-07-13T11:39:04.485000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "_type": "Created", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "import_id": null, "source": "ui", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "contact_id": null}, "emitted_at": 1683623831413} -{"stream": "opportunity_status_change_activities", "data": {"created_by_name": "Airbyte Team", "new_pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "old_status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "opportunity_value": 50000, "opportunity_date_won": "2021-07-15T00:00:00+00:00", "user_name": "Airbyte Team", "opportunity_value_formatted": "$500", "id": "acti_wYiwpVt6MU3LVce1p8JHW8TowOW8GlDikXHkIHvl5xM", "users": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-18T10:26:44.228000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-08-18T10:26:44.228000+00:00", "date_updated": "2021-08-18T10:26:44.228000+00:00", "_type": "OpportunityStatusChange", "opportunity_id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "old_status_type": "active", "new_status_label": "Proposal Sent", "new_status_type": "active", "opportunity_confidence": 75, "opportunity_value_period": "one_time", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "new_status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "old_pipeline_name": "Sales", "new_pipeline_name": "Sales", "old_pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy", "contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "old_status_label": "Demo Completed", "updated_by_name": "Airbyte Team", "opportunity_value_currency": "USD"}, "emitted_at": 1683623833131} -{"stream": "note_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-09-08T09:52:33.209000+00:00", "_type": "Note", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": [], "created_by_name": "Airbyte Team", "date_updated": "2021-09-08T09:52:33.209000+00:00", "note_html": "

    test

    ", "contact_id": null, "id": "acti_S8FCREkmsz7QYukb5ce25aMwDD09ojKF5R9eXiPJczw", "activity_at": "2021-09-08T09:52:33.209000+00:00", "note": "test", "user_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623834944} -{"stream": "note_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-12T14:16:54.027000+00:00", "_type": "Note", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": [], "created_by_name": "Airbyte Team", "date_updated": "2021-08-12T14:16:55.644000+00:00", "note_html": "

    demo note

    ", "contact_id": null, "id": "acti_MEZjvRruzMu1YJP9VFz8AIlJPLNCINLJjKO3bw7TAW7", "activity_at": "2021-08-12T14:16:54.027000+00:00", "note": "demo note", "user_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623834949} -{"stream": "note_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-12T12:10:32.670000+00:00", "_type": "Note", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": [], "created_by_name": "Airbyte Team", "date_updated": "2021-08-18T10:00:40.586000+00:00", "note_html": "

    test 2

    ", "contact_id": null, "id": "acti_MWsXtWl1ouu73mqy8wFOsz6rskZ5fYky1XlB3UqlGwr", "activity_at": "2021-08-12T12:10:32.670000+00:00", "note": "test 2", "user_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623834952} -{"stream": "meeting_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "calendar_event_uids": ["1dep55p6mjf40lgbrv9r9j9hpo"], "id": "acti_AlRbqNk15jdt7Eq2phHXuV49qSpkLBB7af3mwvJkk1z", "updated_by_name": "Airbyte Team", "activity_at": "2022-11-12T18:00:00+00:00", "source": "calendar", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "ends_at": "2022-11-12T19:00:00+00:00", "connected_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "status": "completed", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attendees": [{"email": "irina.grankova@gmail.com", "name": null, "status": "yes", "is_organizer": false, "user_id": null, "contact_id": null}, {"email": "iryna.grankova@airbyte.io", "name": null, "status": "yes", "is_organizer": true, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null}], "integrations": [], "created_by_name": "Airbyte Team", "starts_at": "2022-11-12T18:00:00+00:00", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "contact_id": null, "duration": 3600, "calendar_event_link": "https://www.google.com/calendar/event?eid=MWRlcDU1cDZtamY0MGxnYnJ2OXI5ajlocG8gaXJ5bmEuZ3JhbmtvdmFAYWlyYnl0ZS5pbw", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-12T18:00:00+00:00", "note": "", "title": "Test meeting 2", "_type": "Meeting", "user_name": "Airbyte Team", "date_updated": "2022-11-12T19:00:03.639000+00:00", "location": null, "is_recurring": false}, "emitted_at": 1683623837546} -{"stream": "call_activities", "data": {"remote_phone": "+16505176539", "date_updated": "2021-07-16T00:00:12.646000+00:00", "status": "completed", "local_country_iso": "", "transferred_from": null, "updated_by": null, "_type": "Call", "disposition": "vm-left", "transferred_from_user_id": null, "cost": null, "phone": "+16505176539", "dialer_id": null, "user_name": null, "local_phone_formatted": null, "users": [], "direction": "inbound", "date_answered": null, "note": null, "sequence_id": null, "transferred_to": null, "recording_expires_at": null, "sequence_subscription_id": null, "call_method": "regular", "duration": 0, "remote_phone_formatted": "+1 650-517-6539", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_created": "2021-07-16T00:00:12.646000+00:00", "forwarded_to": null, "user_id": null, "recording_url": null, "created_by": null, "note_html": null, "dialer_saved_search_id": null, "remote_country_iso": "US", "coach_legs": [], "voicemail_duration": 28, "created_by_name": null, "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "has_recording": false, "updated_by_name": null, "source": "Close.io", "local_phone": null, "id": "acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt", "is_joinable": false, "is_to_group_number": false, "activity_at": "2021-07-16T00:00:12.646000+00:00", "is_forwarded": false, "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "voicemail_url": "https://api.close.com/call/acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt/voicemail/", "transferred_to_user_id": null}, "emitted_at": 1683623839016} -{"stream": "call_activities", "data": {"remote_phone": "+12025550186", "date_updated": "2021-07-13T11:39:04.536000+00:00", "status": "no-answer", "local_country_iso": "", "transferred_from": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "Call", "disposition": null, "transferred_from_user_id": null, "cost": null, "phone": "+12025550186", "dialer_id": null, "user_name": "Airbyte Team", "local_phone_formatted": null, "users": [], "direction": "outbound", "date_answered": null, "note": "Gob never answered.", "sequence_id": null, "transferred_to": null, "recording_expires_at": null, "sequence_subscription_id": null, "call_method": "regular", "duration": 0, "remote_phone_formatted": "+1 202-555-0186", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_created": "2021-07-05T11:39:00.850000+00:00", "forwarded_to": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "recording_url": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

    Gob never answered.

    ", "dialer_saved_search_id": null, "remote_country_iso": "US", "coach_legs": [], "voicemail_duration": 0, "created_by_name": "Airbyte Team", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "has_recording": false, "updated_by_name": "Airbyte Team", "source": "Close.io", "local_phone": null, "id": "acti_wszWUd92D7wNYSbn5gKWXCf55NeyU0jc1Vguv7DfUiH", "is_joinable": false, "is_to_group_number": false, "activity_at": "2021-07-05T11:39:00.850000+00:00", "is_forwarded": false, "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "voicemail_url": null, "transferred_to_user_id": null}, "emitted_at": 1683623839021} -{"stream": "call_activities", "data": {"local_phone_formatted": "+1 415-625-1293", "transferred_from": null, "sequence_subscription_id": null, "recording_expires_at": null, "date_updated": "2022-11-09T13:57:40.167000+00:00", "_type": "Call", "recording_url": null, "is_forwarded": false, "disposition": "answered", "sequence_id": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2022-11-09T13:57:14.751000+00:00", "transferred_to": null, "updated_by_name": "Airbyte Team", "transferred_from_user_id": null, "local_country_iso": "US", "status": "completed", "is_joinable": false, "remote_country_iso": "US", "users": [], "voicemail_url": null, "created_by_name": "Airbyte Team", "date_created": "2022-11-09T13:57:14.751000+00:00", "dialer_id": null, "note": "", "forwarded_to": null, "direction": "outbound", "note_html": "

    ", "sequence_name": null, "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "voicemail_duration": 0, "call_method": "regular", "remote_phone_formatted": "+1 415-623-6785", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "user_name": "Airbyte Team", "dialer_saved_search_id": null, "has_recording": false, "cost": "2", "source": "Close.io", "remote_phone": "+14156236785", "transferred_to_user_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "phone": "+14156236785", "local_phone": "+14156251293", "duration": 17, "coach_legs": [], "date_answered": "2022-11-09T13:57:22.063000+00:00", "is_to_group_number": false, "id": "acti_ZgqHg31m0XXDZPwaxUVUNOhnEoLSG3fY8rMn9iIS3cN", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623839599} -{"stream": "email_activities", "data": {"template_name": null, "send_as_id": null, "followup_sequence_delay": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "users": [], "in_reply_to_id": null, "sequence_subscription_id": null, "sequence_name": null, "contact_id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "date_scheduled": null, "bcc": [], "subject": "Re: Sandbox Account", "created_by": null, "id": "acti_aHMyKom3arlMpEVXQYPNxASS0Luskq7sNLAl3Z23vyE", "send_attempts": [], "date_updated": "2022-11-08T12:35:28.249000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "need_smtp_credentials": false, "cc": ["integration-test@airbyte.io", "nick@close.com", "sherif@airbyte.io", "yuri.cherniaiev@airbyte.io"], "updated_by": null, "_type": "Email", "opens_summary": null, "user_name": "Airbyte Team", "to": ["james.urie@close.com"], "references": ["", "", "", "", "", "", "<163000492090.2550.4736014357857312201@smtpgw.close.com>", "<163181368546.9449.10929845414794115430@smtpgw.close.com>", "", "", "<163190650280.3618.8817856985357292069@smtpgw.close.com>"], "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "has_reply": false, "date_created": "2021-09-17T21:29:12+00:00", "envelope": {"message_id": "", "in_reply_to": "<163190650280.3618.8817856985357292069@smtpgw.close.com>", "date": "Sat, 18 Sep 2021 08:29:12 +1100", "reply_to": [], "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "is_autoreply": false, "bcc": [], "subject": "Re: Sandbox Account", "to": [{"name": "James Urie", "email": "james.urie@close.com"}], "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}]}, "date_sent": "2021-09-17T21:29:12+00:00", "body_text_quoted": [{"expand": true, "text": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:"}, {"expand": false, "text": "\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Sep 16, 2021, at 01:21 PM, Jean Lafleur wrote:\n>\n> Hey James,\n>\n> Do you want to do a quick catch-up on co-marketing this week?\n> Here's a link to my calendar: https://calendly.com/john-lafleur/30min\n>\n> Thanks,\n>\n> John\n>\n> On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:\n>\n>> Hi James,\n>>\n>> Happy to report we finished working on the Close.com source and have\n>> released it in beta just this week! I'll let John handle the co-marketing\n>> efforts ;)\n>>\n>> Best,\n>> Shrif\n>>\n>> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>>\n>>> Hey John! Any updates on this integration thus far?\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>>\n>>> Excellent! Keep us in the loop.\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>>\n>>> Hey Nick and James,\n>>>\n>>> Thanks for the call today.\n>>> Following up on our conversation, we should be able to release the\n>>> integration within 1 or 2 weeks.\n>>>\n>>> Thanks,\n>>>\n>>> John\n>>>\n>>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>>\n>>>> Scheduled for next Tuesday!\n>>>>\n>>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>>\n>>>>> Jean!\n>>>>>\n>>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur \n>>>>> wrote:\n>>>>>\n>>>>>> Nice to meet you, Nick!\n>>>>>> Let's schedule some time together as soon as we got the integration\n>>>>>> working :).\n>>>>>>\n>>>>>> Looking forward to it!\n>>>>>>\n>>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>>\n>>>>>>> Great! :-)\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, Close \n>>>>>>>\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>>\n>>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>>\n>>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>>\n>>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>>\n>>>>>>>>> Cheers,\n>>>>>>>>>\n>>>>>>>>> Nick Persico\n>>>>>>>>> Director of Sales, *Close* \n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Thank you James!\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Extended the trial out until 2023. Just ping us then when you\n>>>>>>>>>> need more time.\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>>> working properly\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>>> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Hi James,\n>>>>>>>>>>>\n>>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Shrif\n>>>>>>>>>>>\n>>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>>> tests wrote:\n>>>>>>>>>>>\n>>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> James Urie\n>>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>>\n>>>>>>>>>>>> \n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>>\n>>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>>\n>>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We're happy to build and maintain this integration\n>>>>>>>>>>>> independently without support from Kustomer; the only thing we'd need is\n>>>>>>>>>>>> access to an API-enabled Close sandbox account where we can verify that the\n>>>>>>>>>>>> integration we've built is functional.\n>>>>>>>>>>>>\n>>>>>>>>>>>> Best,\n>>>>>>>>>>>> Yurii\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>\n\n"}], "thread_id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "body_html_quoted": [{"expand": true, "html": "
    Looking forward to it!

    On Sat, Sep 18, 2021 at 6:22 AM James Urie <james.urie@close.com> wrote:
    "}, {"expand": false, "html": "
    Scheduled some time next week!

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Sep 16, 2021, at 01:21 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    "}], "sender": "Jean Lafleur ", "activity_at": "2021-09-17T21:29:12+00:00", "sequence_id": null, "attachments": [], "message_ids": [""], "created_by_name": null, "status": "sent", "followup_sequence_id": null, "body_text": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:\n\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Sep 16, 2021, at 01:21 PM, Jean Lafleur wrote:\n>\n> Hey James,\n>\n> Do you want to do a quick catch-up on co-marketing this week?\n> Here's a link to my calendar: https://calendly.com/john-lafleur/30min\n>\n> Thanks,\n>\n> John\n>\n> On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:\n>\n>> Hi James,\n>>\n>> Happy to report we finished working on the Close.com source and have\n>> released it in beta just this week! I'll let John handle the co-marketing\n>> efforts ;)\n>>\n>> Best,\n>> Shrif\n>>\n>> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>>\n>>> Hey John! Any updates on this integration thus far?\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>>\n>>> Excellent! Keep us in the loop.\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>>\n>>> Hey Nick and James,\n>>>\n>>> Thanks for the call today.\n>>> Following up on our conversation, we should be able to release the\n>>> integration within 1 or 2 weeks.\n>>>\n>>> Thanks,\n>>>\n>>> John\n>>>\n>>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>>\n>>>> Scheduled for next Tuesday!\n>>>>\n>>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>>\n>>>>> Jean!\n>>>>>\n>>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur \n>>>>> wrote:\n>>>>>\n>>>>>> Nice to meet you, Nick!\n>>>>>> Let's schedule some time together as soon as we got the integration\n>>>>>> working :).\n>>>>>>\n>>>>>> Looking forward to it!\n>>>>>>\n>>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>>\n>>>>>>> Great! :-)\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, Close \n>>>>>>>\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>>\n>>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>>\n>>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>>\n>>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>>\n>>>>>>>>> Cheers,\n>>>>>>>>>\n>>>>>>>>> Nick Persico\n>>>>>>>>> Director of Sales, *Close* \n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Thank you James!\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Extended the trial out until 2023. Just ping us then when you\n>>>>>>>>>> need more time.\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>>> working properly\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>>> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Hi James,\n>>>>>>>>>>>\n>>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Shrif\n>>>>>>>>>>>\n>>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>>> tests wrote:\n>>>>>>>>>>>\n>>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> James Urie\n>>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>>\n>>>>>>>>>>>> \n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>>\n>>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>>\n>>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We're happy to build and maintain this integration\n>>>>>>>>>>>> independently without support from Kustomer; the only thing we'd need is\n>>>>>>>>>>>> access to an API-enabled Close sandbox account where we can verify that the\n>>>>>>>>>>>> integration we've built is functional.\n>>>>>>>>>>>>\n>>>>>>>>>>>> Best,\n>>>>>>>>>>>> Yurii\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>\n\n", "body_html": "
    Looking forward to it!

    On Sat, Sep 18, 2021 at 6:22 AM James Urie <james.urie@close.com> wrote:
    Scheduled some time next week!

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Sep 16, 2021, at 01:21 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n\n", "template_id": null, "opens": [], "body_preview": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:\n\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>", "bulk_email_action_id": null, "updated_by_name": null}, "emitted_at": 1683623841122} -{"stream": "email_activities", "data": {"template_name": null, "send_as_id": null, "followup_sequence_delay": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "users": [], "in_reply_to_id": null, "sequence_subscription_id": null, "sequence_name": null, "contact_id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "date_scheduled": null, "bcc": [], "subject": "Re: Sandbox Account", "created_by": null, "id": "acti_3Mapa6JtEWAYWsQU2xF6sPDFz97slwbmFqhye8vDxAV", "send_attempts": [], "date_updated": "2022-11-08T12:35:28.436000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "need_smtp_credentials": false, "cc": ["integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io", "sherif@airbyte.io"], "updated_by": null, "_type": "Email", "opens_summary": null, "user_name": "Airbyte Team", "to": ["james.urie@close.com"], "references": ["", "", "<162629568767.3923.5347313150012573840@smtpgw.close.com>", "", "", "", "", "", "", "<163000492090.2550.4736014357857312201@smtpgw.close.com>", "<163181368546.9449.10929845414794115430@smtpgw.close.com>", ""], "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "has_reply": false, "date_created": "2021-09-16T19:21:05+00:00", "envelope": {"message_id": "", "in_reply_to": "", "date": "Fri, 17 Sep 2021 06:21:05 +1100", "reply_to": [], "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "is_autoreply": false, "bcc": [], "subject": "Re: Sandbox Account", "to": [{"name": "James Urie", "email": "james.urie@close.com"}], "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}]}, "date_sent": "2021-09-16T19:21:05+00:00", "body_text_quoted": [{"expand": true, "text": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:"}, {"expand": false, "text": "\n> Hi James,\n>\n> Happy to report we finished working on the Close.com source and have\n> released it in beta just this week! I'll let John handle the co-marketing\n> efforts ;)\n>\n> Best,\n> Shrif\n>\n> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>\n>> Hey John! Any updates on this integration thus far?\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>\n>> Excellent! Keep us in the loop.\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>\n>> Hey Nick and James,\n>>\n>> Thanks for the call today.\n>> Following up on our conversation, we should be able to release the\n>> integration within 1 or 2 weeks.\n>>\n>> Thanks,\n>>\n>> John\n>>\n>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>\n>>> Scheduled for next Tuesday!\n>>>\n>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>\n>>>> Jean!\n>>>>\n>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>\n>>>> Cheers,\n>>>>\n>>>> Nick Persico\n>>>> Director of Sales, Close \n>>>>\n>>>>\n>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>>\n>>>>> Nice to meet you, Nick!\n>>>>> Let's schedule some time together as soon as we got the integration\n>>>>> working :).\n>>>>>\n>>>>> Looking forward to it!\n>>>>>\n>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>\n>>>>>> Great! :-)\n>>>>>>\n>>>>>> Cheers,\n>>>>>>\n>>>>>> Nick Persico\n>>>>>> Director of Sales, Close \n>>>>>>\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>> wrote:\n>>>>>>\n>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>\n>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>>\n>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>\n>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>\n>>>>>>>> Cheers,\n>>>>>>>>\n>>>>>>>> Nick Persico\n>>>>>>>> Director of Sales, *Close* \n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Thank you James!\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>>> more time.\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>> working properly\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Hi James,\n>>>>>>>>>>\n>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Shrif\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>> tests wrote:\n>>>>>>>>>>\n>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>\n>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>\n>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>\n>>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>>> we've built is functional.\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Yurii\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>\n\n"}], "thread_id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "body_html_quoted": [{"expand": true, "html": "
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    "}, {"expand": false, "html": "
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n
    "}], "sender": "Jean Lafleur ", "activity_at": "2021-09-16T19:21:05+00:00", "sequence_id": null, "attachments": [], "message_ids": [""], "created_by_name": null, "status": "sent", "followup_sequence_id": null, "body_text": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:\n\n> Hi James,\n>\n> Happy to report we finished working on the Close.com source and have\n> released it in beta just this week! I'll let John handle the co-marketing\n> efforts ;)\n>\n> Best,\n> Shrif\n>\n> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>\n>> Hey John! Any updates on this integration thus far?\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>\n>> Excellent! Keep us in the loop.\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>\n>> Hey Nick and James,\n>>\n>> Thanks for the call today.\n>> Following up on our conversation, we should be able to release the\n>> integration within 1 or 2 weeks.\n>>\n>> Thanks,\n>>\n>> John\n>>\n>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>\n>>> Scheduled for next Tuesday!\n>>>\n>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>\n>>>> Jean!\n>>>>\n>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>\n>>>> Cheers,\n>>>>\n>>>> Nick Persico\n>>>> Director of Sales, Close \n>>>>\n>>>>\n>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>>\n>>>>> Nice to meet you, Nick!\n>>>>> Let's schedule some time together as soon as we got the integration\n>>>>> working :).\n>>>>>\n>>>>> Looking forward to it!\n>>>>>\n>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>\n>>>>>> Great! :-)\n>>>>>>\n>>>>>> Cheers,\n>>>>>>\n>>>>>> Nick Persico\n>>>>>> Director of Sales, Close \n>>>>>>\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>> wrote:\n>>>>>>\n>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>\n>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>>\n>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>\n>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>\n>>>>>>>> Cheers,\n>>>>>>>>\n>>>>>>>> Nick Persico\n>>>>>>>> Director of Sales, *Close* \n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Thank you James!\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>>> more time.\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>> working properly\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Hi James,\n>>>>>>>>>>\n>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Shrif\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>> tests wrote:\n>>>>>>>>>>\n>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>\n>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>\n>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>\n>>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>>> we've built is functional.\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Yurii\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>\n\n", "body_html": "
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n
    \n\n", "template_id": null, "opens": [], "body_preview": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif N", "bulk_email_action_id": null, "updated_by_name": null}, "emitted_at": 1683623841128} -{"stream": "email_activities", "data": {"template_name": null, "send_as_id": null, "followup_sequence_delay": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "users": [], "in_reply_to_id": null, "sequence_subscription_id": null, "sequence_name": null, "contact_id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "date_scheduled": null, "bcc": [], "subject": "Re: Sandbox Account", "created_by": null, "id": "acti_DvkhzZr7BulOSmgomLFc3htdTwCDnu85eEoDuL8qQ9R", "send_attempts": [], "date_updated": "2022-11-08T12:35:28.662000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "need_smtp_credentials": false, "cc": ["john@airbyte.io", "integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io"], "updated_by": null, "_type": "Email", "opens_summary": null, "user_name": "Airbyte Team", "to": ["james.urie@close.com"], "references": ["", "", "<162629568767.3923.5347313150012573840@smtpgw.close.com>", "", "", "", "", "", "", "<163000492090.2550.4736014357857312201@smtpgw.close.com>", "<163181368546.9449.10929845414794115430@smtpgw.close.com>"], "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "has_reply": false, "date_created": "2021-09-16T17:48:53+00:00", "envelope": {"message_id": "", "in_reply_to": "<163181368546.9449.10929845414794115430@smtpgw.close.com>", "date": "Thu, 16 Sep 2021 10:48:53 -0700", "reply_to": [], "cc": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "from": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "is_autoreply": false, "bcc": [], "subject": "Re: Sandbox Account", "to": [{"name": "James Urie", "email": "james.urie@close.com"}], "sender": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}]}, "date_sent": "2021-09-16T17:48:53+00:00", "body_text_quoted": [{"expand": true, "text": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:"}, {"expand": false, "text": "\n> Hey John! Any updates on this integration thus far?\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>\n> Excellent! Keep us in the loop.\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>\n> Hey Nick and James,\n>\n> Thanks for the call today.\n> Following up on our conversation, we should be able to release the\n> integration within 1 or 2 weeks.\n>\n> Thanks,\n>\n> John\n>\n> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>\n>> Scheduled for next Tuesday!\n>>\n>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>\n>>> Jean!\n>>>\n>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>\n>>> Cheers,\n>>>\n>>> Nick Persico\n>>> Director of Sales, Close \n>>>\n>>>\n>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>\n>>>> Nice to meet you, Nick!\n>>>> Let's schedule some time together as soon as we got the integration\n>>>> working :).\n>>>>\n>>>> Looking forward to it!\n>>>>\n>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>\n>>>>> Great! :-)\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>> wrote:\n>>>>>\n>>>>>> +John, COO, since he would handle marketing.\n>>>>>>\n>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>\n>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>\n>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, *Close* \n>>>>>>>\n>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada wrote:\n>>>>>>>\n>>>>>>> Thank you James!\n>>>>>>>\n>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>> more time.\n>>>>>>>>\n>>>>>>>> James Urie\n>>>>>>>> Sr. Account Executive\n>>>>>>>>\n>>>>>>>> \n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>> working properly\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Hi James,\n>>>>>>>>>\n>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>\n>>>>>>>>> Best,\n>>>>>>>>> Shrif\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests\n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>\n>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>\n>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>\n>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>> we've built is functional.\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Yurii\n>>>>>>>>>>\n>>>>>>>>>>\n>>>\n\n"}], "thread_id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "body_html_quoted": [{"expand": true, "html": "
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    "}, {"expand": false, "html": "
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    "}], "sender": "Sherif Nada ", "activity_at": "2021-09-16T17:48:53+00:00", "sequence_id": null, "attachments": [], "message_ids": [""], "created_by_name": null, "status": "sent", "followup_sequence_id": null, "body_text": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n\n> Hey John! Any updates on this integration thus far?\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>\n> Excellent! Keep us in the loop.\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>\n> Hey Nick and James,\n>\n> Thanks for the call today.\n> Following up on our conversation, we should be able to release the\n> integration within 1 or 2 weeks.\n>\n> Thanks,\n>\n> John\n>\n> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>\n>> Scheduled for next Tuesday!\n>>\n>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>\n>>> Jean!\n>>>\n>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>\n>>> Cheers,\n>>>\n>>> Nick Persico\n>>> Director of Sales, Close \n>>>\n>>>\n>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>\n>>>> Nice to meet you, Nick!\n>>>> Let's schedule some time together as soon as we got the integration\n>>>> working :).\n>>>>\n>>>> Looking forward to it!\n>>>>\n>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>\n>>>>> Great! :-)\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>> wrote:\n>>>>>\n>>>>>> +John, COO, since he would handle marketing.\n>>>>>>\n>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>\n>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>\n>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, *Close* \n>>>>>>>\n>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada wrote:\n>>>>>>>\n>>>>>>> Thank you James!\n>>>>>>>\n>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>> more time.\n>>>>>>>>\n>>>>>>>> James Urie\n>>>>>>>> Sr. Account Executive\n>>>>>>>>\n>>>>>>>> \n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>> working properly\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Hi James,\n>>>>>>>>>\n>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>\n>>>>>>>>> Best,\n>>>>>>>>> Shrif\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests\n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>\n>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>\n>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>\n>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>> we've built is functional.\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Yurii\n>>>>>>>>>>\n>>>>>>>>>>\n>>>\n\n", "body_html": "
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n\n", "template_id": null, "opens": [], "body_preview": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 ", "bulk_email_action_id": null, "updated_by_name": null}, "emitted_at": 1683623841133} -{"stream": "email_thread_activities", "data": {"created_by_name": null, "date_created": "2021-09-17T21:29:12.001000+00:00", "n_emails": 6, "_type": "EmailThread", "latest_normalized_subject": "Sandbox Account", "contact_id": null, "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "updated_by_name": null, "created_by": null, "updated_by": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_updated": "2022-11-08T12:35:29.323000+00:00", "activity_at": "2021-09-17T21:29:12.001000+00:00", "participants": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "'Nick Persico' via Integration tests", "email": "integration-test@airbyte.io"}], "latest_emails": [{"to": ["james.urie@close.com"], "has_attachments": false, "body_preview": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:\n\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>", "date_created": "2021-09-17T21:29:12+00:00", "date_updated": "2022-11-08T12:35:28.249000+00:00", "envelope": {"to": [{"name": "James Urie", "email": "james.urie@close.com"}], "subject": "Re: Sandbox Account", "date": "Sat, 18 Sep 2021 08:29:12 +1100", "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "message_id": "", "is_autoreply": false, "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "in_reply_to": "<163190650280.3618.8817856985357292069@smtpgw.close.com>", "reply_to": [], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_aHMyKom3arlMpEVXQYPNxASS0Luskq7sNLAl3Z23vyE", "subject": "Re: Sandbox Account", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": ["integration-test@airbyte.io", "nick@close.com", "sherif@airbyte.io", "yuri.cherniaiev@airbyte.io"], "sender": "Jean Lafleur ", "date_sent": "2021-09-17T21:29:12+00:00"}, {"to": ["james.urie@close.com"], "has_attachments": false, "body_preview": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif N", "date_created": "2021-09-16T19:21:05+00:00", "date_updated": "2022-11-08T12:35:28.436000+00:00", "envelope": {"to": [{"name": "James Urie", "email": "james.urie@close.com"}], "subject": "Re: Sandbox Account", "date": "Fri, 17 Sep 2021 06:21:05 +1100", "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "message_id": "", "is_autoreply": false, "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "in_reply_to": "", "reply_to": [], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_3Mapa6JtEWAYWsQU2xF6sPDFz97slwbmFqhye8vDxAV", "subject": "Re: Sandbox Account", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": ["integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io", "sherif@airbyte.io"], "sender": "Jean Lafleur ", "date_sent": "2021-09-16T19:21:05+00:00"}, {"to": ["james.urie@close.com"], "has_attachments": false, "body_preview": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 ", "date_created": "2021-09-16T17:48:53+00:00", "date_updated": "2022-11-08T12:35:28.662000+00:00", "envelope": {"to": [{"name": "James Urie", "email": "james.urie@close.com"}], "subject": "Re: Sandbox Account", "date": "Thu, 16 Sep 2021 10:48:53 -0700", "from": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "message_id": "", "is_autoreply": false, "cc": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "sender": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "in_reply_to": "<163181368546.9449.10929845414794115430@smtpgw.close.com>", "reply_to": [], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_DvkhzZr7BulOSmgomLFc3htdTwCDnu85eEoDuL8qQ9R", "subject": "Re: Sandbox Account", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": ["john@airbyte.io", "integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io"], "sender": "Sherif Nada ", "date_sent": "2021-09-16T17:48:53+00:00"}, {"to": ["nick@close.com"], "has_attachments": false, "body_preview": "Hey Nick and James,\n\nThanks for the call today.\nFollowing up on our conversation, we should be able to release the\nintegration within 1 or 2 weeks.\n\nThanks,\n\nJohn\n\nOn Tue, Aug 17, 2021 at 7:29 AM Jean", "date_created": "2021-08-25T05:00:00+00:00", "date_updated": "2022-11-08T12:35:28.963000+00:00", "envelope": {"to": [{"name": "Nick Persico", "email": "nick@close.com"}], "subject": "Re: Sandbox Account", "date": "Wed, 25 Aug 2021 16:00:00 +1100", "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "message_id": "", "is_autoreply": false, "cc": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "James Urie", "email": "james.urie@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "in_reply_to": "", "reply_to": [], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_cz7ITkJ0XVxLg6IGHjISOt0ETJiMtvzCvuKkC5HNZlj", "subject": "Re: Sandbox Account", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": ["sherif@airbyte.io", "integration-test@airbyte.io", "james.urie@close.com", "yuri.cherniaiev@airbyte.io"], "sender": "Jean Lafleur ", "date_sent": "2021-08-25T05:00:00+00:00"}, {"to": ["nick@close.com"], "has_attachments": false, "body_preview": "Scheduled for next Tuesday!\n\nOn Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n\n> Jean!\n>\n> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>\n> Cheers,\n>\n> Nick Persico\n> Direct", "date_created": "2021-08-16T20:29:59+00:00", "date_updated": "2022-11-08T12:35:29.133000+00:00", "envelope": {"to": [{"name": "Nick Persico", "email": "nick@close.com"}], "subject": "Re: Sandbox Account", "date": "Tue, 17 Aug 2021 07:29:59 +1100", "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "message_id": "", "is_autoreply": false, "cc": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "James Urie", "email": "james.urie@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "in_reply_to": "", "reply_to": [], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_gF8wEbqW7Cx3fDIQTCnejQ3cOQKXLsE5md5JN88z4Lx", "subject": "Re: Sandbox Account", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": ["sherif@airbyte.io", "integration-test@airbyte.io", "james.urie@close.com", "yuri.cherniaiev@airbyte.io"], "sender": "Jean Lafleur ", "date_sent": "2021-08-16T20:29:59+00:00"}], "id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team"}, "emitted_at": 1683623843222} -{"stream": "email_thread_activities", "data": {"created_by_name": null, "date_created": "2021-08-25T21:15:36.001000+00:00", "n_emails": 2, "_type": "EmailThread", "latest_normalized_subject": "Your bulk edit for Airbyte is done", "contact_id": null, "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "updated_by_name": null, "created_by": null, "updated_by": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_updated": "2022-11-08T12:36:06.162000+00:00", "activity_at": "2021-08-25T21:15:36.001000+00:00", "participants": [{"name": "'Close CRM' via Integration tests", "email": "integration-test@airbyte.io"}], "latest_emails": [{"to": ["integration-test@airbyte.io"], "has_attachments": false, "body_preview": "Hello Jean,\n\nYour bulk edit in Close was completed successfully.\n\nSearch query: *\nAction: Change lead status to \"Interested\".\n\nAll 2 leads were modified successfully.\n\nThanks,\nClose Support\nsupport@cl", "date_created": "2021-08-25T21:15:36+00:00", "date_updated": "2022-11-08T12:36:05.979000+00:00", "envelope": {"to": [{"name": "", "email": "integration-test@airbyte.io"}], "subject": "Your bulk edit for Airbyte is done", "date": "Wed, 25 Aug 2021 21:15:36 +0000", "from": [{"name": "'Close CRM' via Integration tests", "email": "integration-test@airbyte.io"}], "message_id": "<162992613594.10130.17609834081606867635@closeio-tasktiger-other-7bd975b8f7-f5phd>", "is_autoreply": false, "cc": [], "sender": [{"name": "", "email": "support@close.com"}], "in_reply_to": null, "reply_to": [{"name": "Close CRM", "email": "support@close.com"}], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_iNuDRoYQ5w17aiH9BRIfN2gHbWBth11m4D16q9mxkdn", "subject": "Your bulk edit for Airbyte is done", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": [], "sender": "'Close CRM' via Integration tests ", "date_sent": "2021-08-25T21:15:36+00:00"}, {"to": ["integration-test@airbyte.io"], "has_attachments": false, "body_preview": "Hello Jean,\n\nYour bulk edit in Close was completed successfully.\n\nSearch query: *\nAction: Set custom field \"Lead Owner\" to \"user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg\".\n\nAll 2 leads were modifie", "date_created": "2021-08-25T21:03:54+00:00", "date_updated": "2022-11-08T12:36:06.154000+00:00", "envelope": {"to": [{"name": "", "email": "integration-test@airbyte.io"}], "subject": "Your bulk edit for Airbyte is done", "date": "Wed, 25 Aug 2021 21:03:54 +0000", "from": [{"name": "'Close CRM' via Integration tests", "email": "integration-test@airbyte.io"}], "message_id": "<162992543485.16564.1574709800437653559@closeio-tasktiger-other-7bd975b8f7-x9jpv>", "is_autoreply": false, "cc": [], "sender": [{"name": "", "email": "support@close.com"}], "in_reply_to": null, "reply_to": [{"name": "Close CRM", "email": "support@close.com"}], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_6pzzBcXnRUrC7ClGxkbfyifOL0tKELXmDQsGffzFx8N", "subject": "Your bulk edit for Airbyte is done", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "sent", "cc": [], "sender": "'Close CRM' via Integration tests ", "date_sent": "2021-08-25T21:03:54+00:00"}], "id": "acti_3rQgtNnE0L7Wo90WzgOXcbfSR2WCjOBipMrAq41yJFE", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team"}, "emitted_at": 1683623843227} -{"stream": "email_thread_activities", "data": {"created_by_name": "Airbyte Team", "date_created": "2021-08-18T10:35:50.166000+00:00", "n_emails": 1, "_type": "EmailThread", "latest_normalized_subject": "Airbyte Follow-up", "contact_id": null, "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "updated_by_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_updated": "2021-08-18T10:36:10.989000+00:00", "activity_at": "2021-08-18T10:35:50.166000+00:00", "participants": [], "latest_emails": [{"to": ["bluth@close.com"], "has_attachments": false, "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "date_created": "2021-08-18T10:35:50.165000+00:00", "date_updated": "2021-08-18T10:36:10.974000+00:00", "envelope": {"to": [{"name": "", "email": "bluth@close.com"}], "subject": "Airbyte Follow-up", "date": "Wed, 18 Aug 2021 10:35:50 -0000", "from": [], "message_id": null, "is_autoreply": false, "cc": [], "sender": [], "in_reply_to": null, "reply_to": [], "bcc": []}, "send_as_id": null, "bcc": [], "id": "acti_8MOghgAFLMW70Zj8pMTUA0NozWrEKeyckt4kFJcx65L", "subject": "Airbyte Follow-up", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outgoing", "opens_summary": null, "status": "draft", "cc": [], "sender": null, "date_sent": null}], "id": "acti_XFp5upc2w8MawOLJ89I5LNlWOYcJLBrHp6UOFOTfN0T", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team"}, "emitted_at": 1683623843231} -{"stream": "lead_status_change_activities", "data": {"user_name": "Airbyte Team", "contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "acti_wSO4ltT4XHGI6wkN2oCZpRdCtw1ALhsRttt8osqVjll", "new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-25T21:15:35.163000+00:00", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "updated_by_name": "Airbyte Team", "old_status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "_type": "LeadStatusChange", "created_by_name": "Airbyte Team", "new_status_label": "Interested", "date_updated": "2021-08-25T21:15:35.163000+00:00", "users": [], "old_status_label": "Potential", "activity_at": "2021-08-25T21:15:35.163000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623845335} -{"stream": "lead_status_change_activities", "data": {"user_name": "Airbyte Team", "contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "acti_T4XA2LBQYvrqAimflFjU2KXY0I2g1BPbOUn2jHZ59Dj", "new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-25T21:15:34.607000+00:00", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "updated_by_name": "Airbyte Team", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "_type": "LeadStatusChange", "created_by_name": "Airbyte Team", "new_status_label": "Interested", "date_updated": "2021-08-25T21:15:34.607000+00:00", "users": [], "old_status_label": "Qualified", "activity_at": "2021-08-25T21:15:34.607000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623845340} -{"stream": "lead_status_change_activities", "data": {"user_name": "Airbyte Team", "contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "acti_udWvk5RJ6SFuO9Ra6d1pAJg200Q0Zyq43kEq0FpceIx", "new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-25T21:15:34.044000+00:00", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by_name": "Airbyte Team", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "_type": "LeadStatusChange", "created_by_name": "Airbyte Team", "new_status_label": "Interested", "date_updated": "2021-08-25T21:15:34.044000+00:00", "users": [], "old_status_label": "Qualified", "activity_at": "2021-08-25T21:15:34.044000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623845343} -{"stream": "sms_activities", "data": {"date_sent": null, "remote_phone": "+12025550186", "date_updated": "2021-08-11T18:14:32.750000+00:00", "status": "draft", "text": "Hi! This is a reminder that we have a call scheduled for 12pm PT today.", "local_country_iso": "US", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "SMS", "cost": null, "user_name": "Airbyte Team", "local_phone_formatted": "+1 415-444-5555", "direction": "outbound", "sequence_id": null, "sequence_subscription_id": null, "date_scheduled": null, "remote_phone_formatted": "+1 202-555-0186", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_created": "2021-08-11T18:14:32.750000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "error_message": null, "remote_country_iso": "US", "created_by_name": "Airbyte Team", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "source": "Close.io", "updated_by_name": "Airbyte Team", "local_phone": "+14154445555", "id": "acti_GIVSys3F0wFeA519lDa5QKRfYOPgskyqKj2aXiCMSEO", "activity_at": "2021-08-11T18:14:32.750000+00:00", "attachments": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "template_id": null}, "emitted_at": 1683623847086} -{"stream": "sms_activities", "data": {"remote_phone": "+14156236785", "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "created_by_name": "Airbyte Team", "id": "acti_CXrlrlHc8QpP5MBXqCfcJKpctDqBREuG3Whj0JQODu4", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "activity_at": "2022-11-09T14:02:18.756000+00:00", "cost": null, "direction": "outbound", "status": "draft", "remote_phone_formatted": "+1 415-623-6785", "remote_country_iso": "US", "attachments": [], "_type": "SMS", "local_phone_formatted": "+1 415-625-1293", "sequence_name": null, "sequence_subscription_id": null, "date_scheduled": null, "error_message": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_sent": null, "date_updated": "2022-11-09T14:02:24.943000+00:00", "date_created": "2022-11-09T14:02:18.756000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "user_name": "Airbyte Team", "text": "Hi!", "updated_by_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "local_country_iso": "US"}, "emitted_at": 1683623847532} -{"stream": "sms_activities", "data": {"remote_phone": "+14156236785", "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "created_by_name": "Airbyte Team", "id": "acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "activity_at": "2022-11-09T12:54:44.404000+00:00", "cost": "3", "direction": "outbound", "status": "sent", "remote_phone_formatted": "+1 415-623-6785", "remote_country_iso": "US", "attachments": [{"media_id": "media_580CfGelcIvP5BzvIYTlhX", "url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/", "thumbnail_url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/thumbnail/", "content_type": "image/png", "size": 6132, "filename": "Airbyte_logo_75x75.png"}], "_type": "SMS", "local_phone_formatted": "+1 415-625-1293", "sequence_name": null, "sequence_subscription_id": null, "date_scheduled": null, "error_message": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_sent": "2022-11-09T12:54:44.404000+00:00", "date_updated": "2022-11-09T12:54:44.405000+00:00", "date_created": "2022-11-09T12:54:15.456000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "user_name": "Airbyte Team", "text": "Hi!", "updated_by_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "local_country_iso": "US"}, "emitted_at": 1683623847536} -{"stream": "task_completed_activities", "data": {"activity_at": "2021-08-18T10:31:52.002000+00:00", "created_by_name": "Airbyte Team", "task_text": "Send Steli an email", "id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "updated_by_name": "Airbyte Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "users": [], "task_id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "task_assigned_to_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "TaskCompleted", "task_assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-18T10:31:52.002000+00:00", "user_name": "Airbyte Team", "date_updated": "2021-08-18T10:31:52.002000+00:00"}, "emitted_at": 1683623848975} -{"stream": "task_completed_activities", "data": {"users": [], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "updated_by_name": "Airbyte Team", "created_by_name": "Airbyte Team", "_type": "TaskCompleted", "task_id": "task_KAzmd4tqcQ1dYypqOjeZqmpKVlFZf3LWJfxCSoX29GY", "task_text": "Follow up", "task_assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-09T12:44:26.185000+00:00", "user_name": "Airbyte Team", "date_updated": "2022-11-09T12:44:26.185000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2022-11-09T12:44:26.185000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "acti_tEFyhPH4AKZ3YHaqqFEdm9Lot1oEI4yEyKzEf0ncbGE", "task_assigned_to_name": "Airbyte Team"}, "emitted_at": 1683623849512} -{"stream": "task_completed_activities", "data": {"users": [], "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "updated_by_name": "Airbyte Team", "created_by_name": "Airbyte Team", "_type": "TaskCompleted", "task_id": "task_iHHSk7GuvunATuloc0hM2k2btiuJpasL12GkwHnIVjP", "task_text": "Follow up", "task_assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-09T11:06:11.134000+00:00", "user_name": "Airbyte Team", "date_updated": "2022-11-09T11:06:11.134000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2022-11-09T11:06:11.134000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "acti_DJOFYNqKoXSmPB4yJHVRafxAqhnvk5olHmmihw4cqND", "task_assigned_to_name": "Airbyte Team"}, "emitted_at": 1683623849515} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": null, "display_name": "Alex", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "name": "Alex", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY/", "tasks": [], "description": "", "updated_by_name": "Airbyte Team", "custom": {}, "date_created": "2022-07-05T21:01:25.608000+00:00", "addresses": [], "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Alex"}], "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Cooper", "phones": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2022-07-05T21:01:25.612000+00:00", "title": "", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "name": "Cooper", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "date_updated": "2022-07-05T21:01:25.612000+00:00", "emails": []}], "status_label": "Potential", "id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "opportunities": [], "date_updated": "2022-07-05T21:01:25.658000+00:00", "created_by_name": "Airbyte Team"}, "emitted_at": 1683623851437} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": null, "display_name": "Bluth Company (Example\u00a0Lead)", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "name": "Bluth Company (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2/", "tasks": [], "description": null, "updated_by_name": "Airbyte Team", "custom": {"Current Vendor/Software": "BiffCo", "Industry": "Real estate", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Google Search"}, "date_created": "2021-07-05T11:39:00.850000+00:00", "addresses": [{"zipcode": "90210", "country": "US", "city": "Los Angeles", "address_2": null, "label": "business", "state": "CA", "address_1": "100 Bluth Drive"}], "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Bluth%20Company%20%28Example%C2%A0Lead%29"}], "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Gob Bluth", "phones": [{"country": "US", "phone_formatted": "+1 202-555-0186", "type": "office", "phone": "+12025550186"}], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2021-07-13T11:39:04.430000+00:00", "title": "Magician", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Gob%20Bluth"}], "name": "Gob Bluth", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-07-13T11:39:04.430000+00:00", "emails": [{"email": "bluth@close.com", "type": "office"}], "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Tobias F\u00fcnke", "phones": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2021-07-13T11:39:04.441000+00:00", "title": "Blue Man Group (Understudy)", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Tobias%20F%C3%BCnke"}], "name": "Tobias F\u00fcnke", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-07-13T11:39:04.441000+00:00", "emails": [{"email": "tobiasfunke@close.com", "type": "office"}], "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Point of Contact"]}], "status_label": "Interested", "id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "opportunities": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "expected_value": 225000, "user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "lead_name": "Bluth Company (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "note": "Gob's ready to buy a $3,000 suit.", "value_currency": "USD", "value_formatted": "$3,000", "value_period": "one_time", "status_display_name": "Demo Completed", "value": 300000, "contact_name": null, "date_lost": null, "contact_id": null, "updated_by_name": "Airbyte Team", "date_created": "2021-07-13T11:39:04.817000+00:00", "status_label": "Demo Completed", "status_type": "active", "integration_links": [], "annualized_value": 300000, "annualized_expected_value": 225000, "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "confidence": 75, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_won": "2021-07-16", "date_updated": "2021-07-13T11:39:04.817000+00:00", "created_by_name": "Airbyte Team"}], "date_updated": "2022-11-08T16:28:54.174000+00:00", "created_by_name": "Airbyte Team", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "BiffCo", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Real estate", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Google Search"}, "emitted_at": 1683623851443} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": "https://close.com", "display_name": "Close (Example\u00a0Lead)", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz/", "tasks": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "text": "Call Steli", "date": "2021-07-18", "is_dateless": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_type": "taskcompleted", "is_complete": true, "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "due_date": "2021-07-18", "lead_name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "assigned_to_name": "Airbyte Team", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "contact_name": null, "contact_id": null, "updated_by_name": "Airbyte Team", "view": "archive", "date_created": "2021-07-13T11:39:03.520000+00:00", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "_type": "lead", "date_updated": "2022-11-08T12:21:15.730000+00:00", "created_by_name": "Airbyte Team"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "text": "Send Steli an email", "date": "2021-07-16", "is_dateless": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_type": "taskcompleted", "is_complete": true, "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "due_date": "2021-07-16", "lead_name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "assigned_to_name": "Airbyte Team", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "contact_name": null, "contact_id": null, "updated_by_name": "Airbyte Team", "view": "archive", "date_created": "2021-07-13T11:39:03.463000+00:00", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "_type": "lead", "date_updated": "2021-08-18T10:31:52.081000+00:00", "created_by_name": "Airbyte Team"}], "description": "Visit our blog for high quality sales content, blog.close.com!", "updated_by_name": "Airbyte Team", "custom": {"Current Vendor/Software": "Stark Industries", "Industry": "Software", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Website"}, "date_created": "2021-07-13T11:39:03.315000+00:00", "addresses": [{"zipcode": "94120", "country": "US", "city": "San Francisco", "address_2": null, "label": "mailing", "state": "CA", "address_1": "PO Box 7775 #69574"}], "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Close%20%28Example%C2%A0Lead%29"}], "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Steli Efti", "phones": [{"country": "US", "phone_formatted": "+1 650-517-6539", "type": "office", "phone": "+16505176539"}], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2021-07-13T11:39:03.354000+00:00", "title": "CEO & Co-Founder", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Steli%20Efti"}], "name": "Steli Efti", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-07-13T11:39:03.354000+00:00", "emails": [{"email": "sales@close.com", "type": "office"}], "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Nick Persico", "phones": [{"country": "US", "phone_formatted": "+1 833-462-5673", "type": "office", "phone": "+18334625673"}], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2021-07-13T11:39:03.366000+00:00", "title": "Director of Revenue", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Nick%20Persico"}], "name": "Nick Persico", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-07-13T11:39:03.366000+00:00", "emails": [{"email": "nick@close.com", "type": "office"}], "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper", "Point of Contact"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Customer Success Team", "phones": [{"country": "US", "phone_formatted": "+1 833-462-5673", "type": "office", "phone": "+18334625673"}], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2021-07-13T11:39:03.374000+00:00", "title": null, "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Customer%20Success%20Team"}], "name": "Customer Success Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_4cmimyQMTMi61kc72mLAV9XdHdddw6LR1LqzvoNdSuV", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-07-13T11:39:03.374000+00:00", "emails": [{"email": "success@close.com", "type": "office"}]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Support", "phones": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "date_created": "2021-07-13T11:39:03.380000+00:00", "title": null, "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Support"}], "name": "Support", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cont_CI5c6Ekew0cyhSgoFIXeaz2PJVmmtigF2XNIGUDXKnu", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-07-13T11:39:03.380000+00:00", "emails": [{"email": "support@close.com", "type": "office"}]}], "status_label": "Interested", "id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "opportunities": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "expected_value": 37500, "user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "lead_name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "value_currency": "USD", "value_formatted": "$500", "value_period": "one_time", "status_display_name": "Proposal Sent", "value": 50000, "contact_name": "Steli Efti", "date_lost": null, "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "updated_by_name": "Airbyte Team", "date_created": "2021-07-13T11:39:04.284000+00:00", "status_label": "Proposal Sent", "status_type": "active", "integration_links": [], "annualized_value": 50000, "annualized_expected_value": 37500, "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "confidence": 75, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_won": "2021-07-15", "date_updated": "2021-08-18T10:26:44.306000+00:00", "created_by_name": "Airbyte Team"}], "date_updated": "2022-11-09T02:40:26.489000+00:00", "created_by_name": "Airbyte Team", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Stark Industries", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Software", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Website"}, "emitted_at": 1683623851449} -{"stream": "lead_tasks", "data": {"contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "object_type": "taskcompleted", "_type": "lead", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "is_complete": true, "is_dateless": null, "due_date": "2021-07-16", "view": "archive", "created_by_name": "Airbyte Team", "assigned_to_name": "Airbyte Team", "date": "2021-07-16", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "lead_name": "Close (Example\u00a0Lead)", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-08-18T10:31:52.081000+00:00", "date_created": "2021-07-13T11:39:03.463000+00:00", "text": "Send Steli an email", "updated_by_name": "Airbyte Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_name": null}, "emitted_at": 1683623853073} -{"stream": "lead_tasks", "data": {"contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "object_type": "taskcompleted", "_type": "lead", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "is_complete": true, "is_dateless": null, "due_date": "2021-07-18", "view": "archive", "created_by_name": "Airbyte Team", "assigned_to_name": "Airbyte Team", "date": "2021-07-18", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "lead_name": "Close (Example\u00a0Lead)", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2022-11-08T12:21:15.730000+00:00", "date_created": "2021-07-13T11:39:03.520000+00:00", "text": "Call Steli", "updated_by_name": "Airbyte Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_name": null}, "emitted_at": 1683623853077} -{"stream": "lead_tasks", "data": {"view": "inbox", "is_complete": false, "date": "2022-11-12", "assigned_to_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_type": null, "is_dateless": false, "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "_type": "lead", "date_updated": "2022-11-09T11:06:06.145000+00:00", "text": "Follow up", "due_date": "2022-11-12", "updated_by_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "id": "task_Wgzt6t0ZGRlvZTnfhSfxHoFRWm6zXBMuuecKxRkfYmZ", "object_id": null, "date_created": "2022-11-08T15:10:57.220000+00:00"}, "emitted_at": 1683623853713} -{"stream": "email_followup_tasks", "data": {"lead_name": "Wayne Enterprises (Example\u00a0Lead)", "is_complete": false, "date_created": "2022-11-08T15:44:05.221000+00:00", "object_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "contact_name": "Bruce Wayne", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-15", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "view": "inbox", "body_preview": "Test", "date_updated": "2022-11-08T16:28:54.794000+00:00", "email_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "subject": "(no subject)", "id": "noti_3I4QaKqYel0nzGyKT8mpDKy4q2KKu4IuIZZ3Gb7gs4x", "object_type": "emailthread", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "assigned_to_name": "Airbyte Team", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "_type": "email_followup", "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6"}, "emitted_at": 1683623858573} -{"stream": "email_followup_tasks", "data": {"lead_name": "Test Lead", "is_complete": false, "date_created": "2022-11-09T12:54:10.330000+00:00", "object_id": "acti_fFBPZHGPKt60kZG2SzI5ws5OMbysXqCAGJuGlKucDy5", "contact_name": "User2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-16", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "view": "inbox", "body_preview": "Hi Iryna, \n \nI'm Jean with Airbyte. We help companies in the space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Test Lead\nand show you what we're working on. \n ", "date_updated": "2022-11-09T12:54:10.330000+00:00", "email_id": "acti_a3AsG3P38a8yDx0XgI8ZNZR4Ko8XRUBybauASuT8C2j", "subject": "Test Lead + Airbyte", "id": "noti_JU7vkNkhpywznaGu5P0WuBWxsMXtzZmkOSXaCxbxTsN", "object_type": "emailthread", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "assigned_to_name": "Airbyte Team", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "_type": "email_followup", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP"}, "emitted_at": 1683623858578} -{"stream": "opportunity_due_tasks", "data": {"assigned_to_name": "Airbyte Team", "created_by_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "opportunity_value": 50000, "id": "noti_ELGehUlDZ1pmlIeibPlLbTyruBTKdACAQ1q1XLiWo6B", "date_created": "2021-07-13T11:39:04.316000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "contact_name": "Steli Efti", "opportunity_value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_name": "Close (Example\u00a0Lead)", "object_type": "opportunity", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "view": "inbox", "updated_by_name": "Airbyte Team", "opportunity_note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "opportunity_value_formatted": "$500", "date": "2021-07-15", "_type": "opportunity_due", "is_complete": false, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-08-18T10:26:44.323000+00:00"}, "emitted_at": 1683623866914} -{"stream": "opportunity_due_tasks", "data": {"assigned_to_name": "Airbyte Team", "created_by_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "opportunity_value": 300000, "id": "noti_wcbYMl0dpbdcypRygf9jYd9axfIzvSsj06gXHmPmE4k", "date_created": "2021-07-13T11:39:04.839000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "contact_name": null, "opportunity_value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_name": "Bluth Company (Example\u00a0Lead)", "object_type": "opportunity", "contact_id": null, "view": "inbox", "updated_by_name": "Airbyte Team", "opportunity_note": "Gob's ready to buy a $3,000 suit.", "opportunity_value_formatted": "$3,000", "date": "2021-07-16", "_type": "opportunity_due", "is_complete": false, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-07-13T11:39:04.839000+00:00"}, "emitted_at": 1683623866919} -{"stream": "opportunity_due_tasks", "data": {"assigned_to_name": "Airbyte Team", "created_by_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "opportunity_value": 50000, "id": "noti_IPi8EZj1AtqSwbnSMg5OT2DpUqNHCl2RM7luvbx9zsp", "date_created": "2021-07-13T11:39:05.083000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "contact_name": null, "opportunity_value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_type": "opportunity", "contact_id": null, "view": "inbox", "updated_by_name": "Airbyte Team", "opportunity_note": "Bruce needs new software for the Bat Cave.", "opportunity_value_formatted": "$500", "date": "2021-07-15", "_type": "opportunity_due", "is_complete": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "date_updated": "2021-07-13T11:39:05.083000+00:00"}, "emitted_at": 1683623866923} -{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "next_billing_on"], "data": {"address_id": null, "bundle_id": null, "carrier": "twilio", "country": "US", "date_created": "2022-11-08T12:35:29.464000+00:00", "date_updated": "2023-05-07T10:01:09.836000+00:00", "forward_to": null, "forward_to_enabled": false, "forward_to_formatted": null, "id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "is_group_number": false, "is_verified": false, "label": null, "last_billed_price": "1.15", "mms_enabled": true, "next_billing_on": "2023-06-07", "number": "+14156251293", "number_formatted": "+1 415-625-1293", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "press_1_to_accept": true, "sms_enabled": true, "supports_mms_to_countries": ["CA", "US"], "supports_sms_to_countries": ["CA", "PR", "US"], "type": "internal", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "voicemail_greeting_url": "https://closeio-voicemail-greetings.s3.amazonaws.com/14bzVblrIaekmXSGQEKM11/undefined.mp3"}, "date_created": "2023-05-07T10:01:09.840000", "date_updated": "2023-05-07T10:01:09.840000", "id": "ev_5QmFTiaXszWD4dLYCX2kiS", "lead_id": null, "meta": {}, "object_id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "object_type": "phone_number", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-04-07T10:01:14.668000+00:00", "next_billing_on": "2023-05-07"}, "request_id": null, "user_id": null}, "emitted_at": 1683623872033} -{"stream": "lead_custom_fields", "data": {"date_created": "2021-07-13T11:39:01.352000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "description": null, "name": "Lead Owner", "is_shared": false, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "back_reference_is_visible": null, "date_updated": "2021-07-13T11:39:01.352000+00:00", "referenced_custom_type_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "choices": null, "editable_with_roles": [], "id": "cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh", "type": "user"}, "emitted_at": 1683623873222} -{"stream": "contact_custom_fields", "data": {"is_shared": false, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "referenced_custom_type_id": null, "date_updated": "2021-08-11T18:05:14.598000+00:00", "accepts_multiple_values": false, "choices": null, "editable_with_roles": [], "name": "Birthday", "description": null, "type": "date", "id": "cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-08-11T18:05:14.598000+00:00"}, "emitted_at": 1683623874178} -{"stream": "opportunity_custom_fields", "data": {"description": null, "is_shared": false, "accepts_multiple_values": false, "editable_with_roles": [], "date_updated": "2021-07-13T11:39:01.714000+00:00", "choices": null, "back_reference_is_visible": null, "type": "text", "id": "cf_pzwnr4IYZvEqq68ENLlWknEZdVsPk53rd3571eRcP56", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Contract Link (Example)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:01.714000+00:00", "referenced_custom_type_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"}, "emitted_at": 1683623874993} -{"stream": "opportunity_custom_fields", "data": {"description": null, "is_shared": false, "accepts_multiple_values": true, "editable_with_roles": [], "date_updated": "2021-07-13T11:39:01.447000+00:00", "choices": ["Widget A", "Widget B", "Widget C"], "back_reference_is_visible": null, "type": "choices", "id": "cf_aDbTIccjcJ9GQqM0xqA8uqgjYuEclhLMxnqfG3TAhsx", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Product (Example)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:01.447000+00:00", "referenced_custom_type_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"}, "emitted_at": 1683623874996} -{"stream": "opportunity_custom_fields", "data": {"description": null, "is_shared": false, "accepts_multiple_values": false, "editable_with_roles": [], "date_updated": "2021-07-13T11:39:01.676000+00:00", "choices": null, "back_reference_is_visible": null, "type": "date", "id": "cf_Z6Vyxe2J0lrRqFpWQQGLkPo8iwhmZ9hzWc69ll2oFox", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Renewal Date (Example)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:01.676000+00:00", "referenced_custom_type_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"}, "emitted_at": 1683623874998} -{"stream": "activity_custom_fields", "data": {"id": "cf_UftGNTS2rq9XMG9hOHkALp5cgn4Xl8nZqN7gReax3lc", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.212000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "type": "contact", "date_updated": "2021-07-13T11:39:02.212000+00:00", "editable_with_roles": [], "name": "Contact", "is_shared": false, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "back_reference_is_visible": null, "description": null, "choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1683623875781} -{"stream": "activity_custom_fields", "data": {"id": "cf_3W4n175LyZ3QMfr665jUS19nt2QATHyM7QaxXQC4QqA", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.744000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "type": "text", "date_updated": "2021-07-13T11:39:02.744000+00:00", "editable_with_roles": [], "name": "Current Vendor: Other (if applicable)", "is_shared": false, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "back_reference_is_visible": null, "description": null, "choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1683623875784} -{"stream": "activity_custom_fields", "data": {"id": "cf_bp93vNo2vxbmM2QhQ7JvdytgcozCCGmOklTG9E8rDYa", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.478000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "type": "text", "date_updated": "2021-07-13T11:39:02.478000+00:00", "editable_with_roles": [], "name": "Industry: Other (if applicable)", "is_shared": false, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "back_reference_is_visible": null, "description": null, "choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1683623875785} -{"stream": "users", "data": {"date_created": "2021-07-13T11:36:04.905000+00:00", "email": "integration-test@airbyte.io", "email_verified_at": "2021-07-13T11:37:23.175000+00:00", "date_updated": "2023-01-30T16:10:36.593000+00:00", "image": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef", "last_name": "Team", "organizations": ["orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"], "id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "google_profile_image_url": null, "last_used_timezone": "Europe/Kiev", "first_name": "Airbyte"}, "emitted_at": 1683623876657} -{"stream": "contacts", "data": {"date_updated": "2023-01-30T16:02:41.174000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "name": "User1", "id": "cont_b4h4BcmWn7rKbnsHQ0JfADwXGgndbpc5JlQEdHHyv78", "emails": [], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User1"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-11T09:00:52.289000+00:00", "urls": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "User1", "phones": [{"phone": "+16000000001", "phone_formatted": "+1 600-000-0001", "country": null, "type": "office"}], "title": "Product Manager"}, "emitted_at": 1683623877504} -{"stream": "contacts", "data": {"date_updated": "2023-01-30T16:03:28.787000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "name": "User2", "id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "emails": [{"type": "office", "email": "user2.sample@gmail.com"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User2"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T15:54:42.381000+00:00", "urls": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "User2", "phones": [{"phone": "+14156236785", "phone_formatted": "+1 415-623-6785", "country": "US", "type": "office"}], "title": "Test Lead", "custom.cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu": "2022-12-01"}, "emitted_at": 1683623877507} -{"stream": "contacts", "data": {"date_updated": "2022-07-05T21:01:25.612000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "name": "Cooper", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "emails": [], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-07-05T21:01:25.612000+00:00", "urls": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "display_name": "Cooper", "phones": [], "title": ""}, "emitted_at": 1683623877509} -{"stream": "opportunities", "data": {"date_created": "2021-07-13T11:39:04.817000+00:00", "updated_by_name": "Airbyte Team", "annualized_expected_value": 225000, "status_type": "active", "value_period": "one_time", "annualized_value": 300000, "expected_value": 225000, "value_formatted": "$3,000", "status_display_name": "Demo Completed", "user_name": "Airbyte Team", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:04.817000+00:00", "value": 300000, "date_won": "2021-07-16", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_label": "Demo Completed", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "confidence": 75, "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "date_lost": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note": "Gob's ready to buy a $3,000 suit.", "lead_name": "Bluth Company (Example\u00a0Lead)", "value_currency": "USD", "contact_name": null, "created_by_name": "Airbyte Team", "contact_id": null, "integration_links": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2"}, "emitted_at": 1683623878961} -{"stream": "opportunities", "data": {"date_created": "2021-07-13T11:39:05.061000+00:00", "updated_by_name": "Airbyte Team", "annualized_expected_value": 37500, "status_type": "active", "value_period": "one_time", "annualized_value": 50000, "expected_value": 37500, "value_formatted": "$500", "status_display_name": "Demo Completed", "user_name": "Airbyte Team", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:05.061000+00:00", "value": 50000, "date_won": "2021-07-15", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_label": "Demo Completed", "id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "confidence": 75, "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "date_lost": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note": "Bruce needs new software for the Bat Cave.", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "value_currency": "USD", "contact_name": null, "created_by_name": "Airbyte Team", "contact_id": null, "integration_links": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN"}, "emitted_at": 1683623878966} -{"stream": "opportunities", "data": {"date_created": "2021-07-13T11:39:04.284000+00:00", "updated_by_name": "Airbyte Team", "annualized_expected_value": 37500, "status_type": "active", "value_period": "one_time", "annualized_value": 50000, "expected_value": 37500, "value_formatted": "$500", "status_display_name": "Proposal Sent", "user_name": "Airbyte Team", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-08-18T10:26:44.306000+00:00", "value": 50000, "date_won": "2021-07-15", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_label": "Proposal Sent", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "confidence": 75, "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "date_lost": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "lead_name": "Close (Example\u00a0Lead)", "value_currency": "USD", "contact_name": "Steli Efti", "created_by_name": "Airbyte Team", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "integration_links": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, "emitted_at": 1683623878971} -{"stream": "roles", "data": {"date_created": "0001-01-01T00:00:00", "date_updated": "0001-01-01T00:00:00", "editable": false, "id": "admin", "name": "Admin", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "permissions": ["bulk_delete", "bulk_edit", "bulk_email", "bulk_import", "bulk_sequence_subscriptions", "call_coach_barge", "call_coach_listen", "calling", "delete_leads", "delete_own_activities", "delete_own_opportunities", "delete_own_tasks", "export", "manage_customizations", "manage_email_sequences", "manage_group_numbers", "manage_organization", "manage_others_activities", "manage_others_opportunities", "manage_others_tasks", "manage_team_email_templates", "manage_team_smart_views", "merge_leads", "view_all_leads"], "visibility_user_lcf_behavior": null, "visibility_user_lcf_ids": []}, "emitted_at": 1683623881039} -{"stream": "roles", "data": {"date_created": "0001-01-01T00:00:00", "date_updated": "0001-01-01T00:00:00", "editable": false, "id": "restricteduser", "name": "Restricted User", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "permissions": ["delete_own_activities", "delete_own_opportunities", "delete_own_tasks", "calling", "view_all_leads"], "visibility_user_lcf_behavior": null, "visibility_user_lcf_ids": []}, "emitted_at": 1683623881041} -{"stream": "roles", "data": {"date_created": "0001-01-01T00:00:00", "date_updated": "0001-01-01T00:00:00", "editable": false, "id": "superuser", "name": "Super User", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "permissions": ["bulk_delete", "bulk_edit", "bulk_email", "bulk_import", "bulk_sequence_subscriptions", "call_coach_barge", "call_coach_listen", "calling", "delete_leads", "delete_own_activities", "delete_own_opportunities", "delete_own_tasks", "export", "manage_customizations", "manage_email_sequences", "manage_group_numbers", "manage_others_activities", "manage_others_opportunities", "manage_others_tasks", "manage_team_email_templates", "manage_team_smart_views", "merge_leads", "view_all_leads"], "visibility_user_lcf_behavior": null, "visibility_user_lcf_ids": []}, "emitted_at": 1683623881043} -{"stream": "lead_statuses", "data": {"id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Potential"}, "emitted_at": 1683623882481} -{"stream": "lead_statuses", "data": {"id": "stat_I73shDEVu3FGzxWmP7Jti3MC6aJEdYH4e036HF1bk7k", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Customer"}, "emitted_at": 1683623882483} -{"stream": "lead_statuses", "data": {"id": "stat_K2bWWhgZqujhdsO1FPg189wAapaKMVH7bi7FwPqjFfy", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Bad Fit"}, "emitted_at": 1683623882484} -{"stream": "opportunity_statuses", "data": {"id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Proposal Sent", "type": "active", "pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy"}, "emitted_at": 1683623883350} -{"stream": "opportunity_statuses", "data": {"id": "stat_CZr5826cyG8wqIg4tD6bbjaqePP4HAYOMSLgOhi1Xbf", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Lost", "type": "lost", "pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy"}, "emitted_at": 1683623883352} -{"stream": "opportunity_statuses", "data": {"id": "stat_ObYTUqjVZW0nTZXjvHhzMGyK999e42WZdIhkaNq12En", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Contract Sent", "type": "active", "pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy"}, "emitted_at": 1683623883353} -{"stream": "pipelines", "data": {"created_by": null, "date_created": "2021-07-13T11:36:04.983404", "date_updated": "2021-07-13T11:36:04.983404", "id": "pipe_0IAl41rGk9OPls9CdxFpHy", "name": "Sales", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "statuses": [{"id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "label": "Demo Completed", "type": "active"}, {"id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "label": "Proposal Sent", "type": "active"}, {"id": "stat_ObYTUqjVZW0nTZXjvHhzMGyK999e42WZdIhkaNq12En", "label": "Contract Sent", "type": "active"}, {"id": "stat_gaqEGSVHIFzrofTfzzg5UfjyBZ1B6KERccIy2MOp8FG", "label": "Won", "type": "won"}, {"id": "stat_CZr5826cyG8wqIg4tD6bbjaqePP4HAYOMSLgOhi1Xbf", "label": "Lost", "type": "lost"}], "updated_by": null}, "emitted_at": 1683623884327} -{"stream": "email_templates", "data": {"date_updated": "2022-08-11T18:11:08.966000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "subject": "{{ lead.display_name }} + {{ organization.name }}", "name": "Email 1 - Intro", "id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "date_created": "2021-07-13T11:39:00.497000+00:00", "attachments": [], "is_archived": false, "is_shared": true, "body": "Hi {{ contact.first_name }},

    I'm {{ user.first_name }} with {{ organization.name }}. We help companies in the {{ lead.custom.[\"Industry\"] }} space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at {{ lead.display_name }} and show you what we're working on.

    Are you available for a quick call tomorrow afternoon?
    ", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623885200} -{"stream": "email_templates", "data": {"date_updated": "2022-08-11T18:11:08.972000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "subject": "{{ organization.name }} Follow-up", "name": "Email 2 - Follow-up #1", "id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "date_created": "2021-07-13T11:39:00.503000+00:00", "attachments": [], "is_archived": false, "is_shared": true, "body": "Hi {{ contact.first_name }},

    Friendly follow-up.

    I wanted to show you how {{ organization.name }} can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

    - Wed @ 11AM
    - Thur @ 2PM
    - Fri @ 3PM
    ", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623885203} -{"stream": "google_connected_accounts", "data": {"date_created": "2022-11-08T12:13:40.319000+00:00", "send_status": "ok", "receive_attempts_count": 0, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "imap": {"date_updated": "2022-11-08T12:13:40.319000+00:00", "port": 993, "host": "imap.gmail.com", "username": "iryna.grankova@airbyte.io", "use_ssl": true}, "latest_send_error": null, "smtp": {"date_updated": "2022-11-08T12:13:40.319000+00:00", "port": 465, "host": "smtp.gmail.com", "username": "iryna.grankova@airbyte.io", "use_ssl": true}, "calendar_receive_status": "ok", "email": "iryna.grankova@airbyte.io", "available_features": ["calendar_syncing", "email_sending", "email_syncing"], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "google", "available_calendars": [{"id": "iryna.grankova@airbyte.io", "name": "iryna.grankova@airbyte.io", "status": "ok", "default": true, "synced": true}, {"id": "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com", "name": "Close Tasks: Airbyte Team", "status": "ok", "default": false, "synced": true}], "id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "identities": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "default_identity": {"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}, "sync_all_calendars": false, "synced_calendars": ["iryna.grankova@airbyte.io", "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com"], "latest_receive_error": null, "email_receive_status": "ok", "receive_status": "ok", "lead_suggestions_updated_at": "2022-11-08T12:13:45.711000+00:00", "enabled_features": ["calendar_syncing", "email_sending", "email_syncing"], "is_imap_archive_sync_enabled": true}, "emitted_at": 1683623886595} -{"stream": "google_connected_accounts", "data": {"available_features": ["email_sending", "email_syncing"], "date_created": "2022-11-08T14:43:13.927000+00:00", "email": "integration-test@airbyte.io", "enabled_features": ["email_sending", "email_syncing"], "_type": "custom_email", "calendar_receive_status": null, "default_identity": {"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}, "email_receive_status": "error", "latest_receive_error": null, "latest_send_error": null, "receive_status": "error", "send_status": "ok", "id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "identities": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "imap": {"date_updated": "2022-11-08T14:43:13.927000+00:00", "port": 993, "host": "imap.gmail.com", "username": "iryna.grankova@globallogic.com", "use_ssl": true}, "is_imap_archive_sync_enabled": true, "lead_suggestions_updated_at": "2022-11-08T14:43:17.844000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "receive_attempts_count": 0, "smtp": {"date_updated": "2022-11-08T14:43:13.927000+00:00", "port": 465, "host": "smtp.gmail.com", "username": "iryna.grankova@globallogic.com", "use_ssl": true}, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623886598} -{"stream": "google_connected_accounts", "data": {"date_created": "2022-11-08T13:13:08.795000+00:00", "send_status": null, "receive_attempts_count": 0, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "imap": null, "latest_send_error": null, "smtp": null, "calendar_receive_status": null, "email": "integration-test@airbyte.io", "available_features": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "zoom", "id": "emailacct_pSuPSwRBjmzoTYudYJRn9ENpaR3RAaRp7K2jJ8syPsp", "zoom_account_plan": null, "identities": [], "default_identity": {"email": "integration-test@airbyte.io", "name": "Airbyte Team"}, "latest_receive_error": null, "email_receive_status": null, "receive_status": null, "lead_suggestions_updated_at": null, "enabled_features": [], "is_imap_archive_sync_enabled": true}, "emitted_at": 1683623886599} -{"stream": "custom_email_connected_accounts", "data": {"latest_receive_error": null, "smtp": {"username": "iryna.grankova@airbyte.io", "date_updated": "2022-11-08T12:13:40.319000+00:00", "use_ssl": true, "port": 465, "host": "smtp.gmail.com"}, "latest_send_error": null, "_type": "google", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:13:40.319000+00:00", "id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "available_features": ["calendar_syncing", "email_sending", "email_syncing"], "lead_suggestions_updated_at": "2022-11-08T12:13:45.711000+00:00", "default_identity": {"name": "Jean Lafleur", "email": "iryna.grankova@airbyte.io"}, "receive_attempts_count": 0, "email_receive_status": "ok", "email": "iryna.grankova@airbyte.io", "send_status": "ok", "is_imap_archive_sync_enabled": true, "sync_all_calendars": false, "identities": [{"name": "Jean Lafleur", "email": "iryna.grankova@airbyte.io"}], "calendar_receive_status": "ok", "receive_status": "ok", "synced_calendars": ["iryna.grankova@airbyte.io", "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com"], "enabled_features": ["calendar_syncing", "email_sending", "email_syncing"], "imap": {"username": "iryna.grankova@airbyte.io", "date_updated": "2022-11-08T12:13:40.319000+00:00", "use_ssl": true, "port": 993, "host": "imap.gmail.com"}, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "available_calendars": [{"id": "iryna.grankova@airbyte.io", "name": "iryna.grankova@airbyte.io", "status": "ok", "default": true, "synced": true}, {"id": "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com", "name": "Close Tasks: Airbyte Team", "status": "ok", "default": false, "synced": true}]}, "emitted_at": 1683623887494} -{"stream": "custom_email_connected_accounts", "data": {"available_features": ["email_sending", "email_syncing"], "date_created": "2022-11-08T14:43:13.927000+00:00", "email": "integration-test@airbyte.io", "enabled_features": ["email_sending", "email_syncing"], "_type": "custom_email", "calendar_receive_status": null, "default_identity": {"name": "Jean Lafleur", "email": "integration-test@airbyte.io"}, "email_receive_status": "error", "latest_receive_error": null, "latest_send_error": null, "receive_status": "error", "send_status": "ok", "id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "identities": [{"name": "Jean Lafleur", "email": "integration-test@airbyte.io"}], "imap": {"username": "iryna.grankova@globallogic.com", "date_updated": "2022-11-08T14:43:13.927000+00:00", "use_ssl": true, "port": 993, "host": "imap.gmail.com"}, "is_imap_archive_sync_enabled": true, "lead_suggestions_updated_at": "2022-11-08T14:43:17.844000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "receive_attempts_count": 0, "smtp": {"username": "iryna.grankova@globallogic.com", "date_updated": "2022-11-08T14:43:13.927000+00:00", "use_ssl": true, "port": 465, "host": "smtp.gmail.com"}, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623887496} -{"stream": "custom_email_connected_accounts", "data": {"latest_receive_error": null, "smtp": null, "latest_send_error": null, "_type": "zoom", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T13:13:08.795000+00:00", "id": "emailacct_pSuPSwRBjmzoTYudYJRn9ENpaR3RAaRp7K2jJ8syPsp", "available_features": [], "lead_suggestions_updated_at": null, "default_identity": {"name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "receive_attempts_count": 0, "email_receive_status": null, "email": "integration-test@airbyte.io", "send_status": null, "is_imap_archive_sync_enabled": true, "identities": [], "calendar_receive_status": null, "receive_status": null, "enabled_features": [], "imap": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "zoom_account_plan": null}, "emitted_at": 1683623887498} -{"stream": "zoom_connected_accounts", "data": {"smtp": {"date_updated": "2022-11-08T12:13:40.319000+00:00", "username": "iryna.grankova@airbyte.io", "use_ssl": true, "host": "smtp.gmail.com", "port": 465}, "is_imap_archive_sync_enabled": true, "synced_calendars": ["iryna.grankova@airbyte.io", "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com"], "default_identity": {"name": "Jean Lafleur", "email": "iryna.grankova@airbyte.io"}, "id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "available_features": ["calendar_syncing", "email_sending", "email_syncing"], "latest_send_error": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_suggestions_updated_at": "2022-11-08T12:13:45.711000+00:00", "imap": {"date_updated": "2022-11-08T12:13:40.319000+00:00", "username": "iryna.grankova@airbyte.io", "use_ssl": true, "host": "imap.gmail.com", "port": 993}, "send_status": "ok", "identities": [{"name": "Jean Lafleur", "email": "iryna.grankova@airbyte.io"}], "email_receive_status": "ok", "enabled_features": ["calendar_syncing", "email_sending", "email_syncing"], "_type": "google", "available_calendars": [{"id": "iryna.grankova@airbyte.io", "name": "iryna.grankova@airbyte.io", "status": "ok", "default": true, "synced": true}, {"id": "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com", "name": "Close Tasks: Airbyte Team", "status": "ok", "default": false, "synced": true}], "receive_attempts_count": 0, "receive_status": "ok", "calendar_receive_status": "ok", "date_created": "2022-11-08T12:13:40.319000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "latest_receive_error": null, "sync_all_calendars": false, "email": "iryna.grankova@airbyte.io"}, "emitted_at": 1683623888402} -{"stream": "zoom_connected_accounts", "data": {"available_features": ["email_sending", "email_syncing"], "date_created": "2022-11-08T14:43:13.927000+00:00", "email": "integration-test@airbyte.io", "enabled_features": ["email_sending", "email_syncing"], "_type": "custom_email", "calendar_receive_status": null, "default_identity": {"name": "Jean Lafleur", "email": "integration-test@airbyte.io"}, "email_receive_status": "error", "latest_receive_error": null, "latest_send_error": null, "receive_status": "error", "send_status": "ok", "id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "identities": [{"name": "Jean Lafleur", "email": "integration-test@airbyte.io"}], "imap": {"date_updated": "2022-11-08T14:43:13.927000+00:00", "username": "iryna.grankova@globallogic.com", "use_ssl": true, "host": "imap.gmail.com", "port": 993}, "is_imap_archive_sync_enabled": true, "lead_suggestions_updated_at": "2022-11-08T14:43:17.844000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "receive_attempts_count": 0, "smtp": {"date_updated": "2022-11-08T14:43:13.927000+00:00", "username": "iryna.grankova@globallogic.com", "use_ssl": true, "host": "smtp.gmail.com", "port": 465}, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623888404} -{"stream": "zoom_connected_accounts", "data": {"smtp": null, "is_imap_archive_sync_enabled": true, "default_identity": {"name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "id": "emailacct_pSuPSwRBjmzoTYudYJRn9ENpaR3RAaRp7K2jJ8syPsp", "available_features": [], "latest_send_error": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_suggestions_updated_at": null, "imap": null, "send_status": null, "identities": [], "email_receive_status": null, "enabled_features": [], "_type": "zoom", "receive_attempts_count": 0, "receive_status": null, "calendar_receive_status": null, "date_created": "2022-11-08T13:13:08.795000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "latest_receive_error": null, "email": "integration-test@airbyte.io", "zoom_account_plan": null}, "emitted_at": 1683623888406} -{"stream": "email_sequences", "data": {"allow_manual_enrollment": true, "created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:06.184401", "date_updated": "2022-11-08T12:07:06.184405", "id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "name": "Sequence 1", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "schedule_id": "sched_1eL83fHJodv0OtoibuJFAw", "status": "active", "steps": [{"created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:06.191662", "date_updated": "2022-11-08T12:07:06.191665", "delay": 0, "email_template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "id": "seqstep_5lFdCIwj4qVpdVMH3K26iQ", "required": true, "sms_template_id": null, "step_allowed_delay": null, "step_type": "email", "threading": "old_thread", "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, {"created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:06.191692", "date_updated": "2022-11-08T12:07:06.191693", "delay": 259200, "email_template_id": null, "id": "seqstep_4YVJ2uoVccNHYG95qvwDQw", "required": false, "sms_template_id": null, "step_allowed_delay": 3600, "step_type": "call", "threading": null, "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "timezone": "Europe/Kiev", "trigger_query": null, "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683885150263} -{"stream": "email_sequences", "data": {"allow_manual_enrollment": true, "created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:48.571475", "date_updated": "2022-11-08T12:07:48.571481", "id": "seq_7gEZ4ByvvLv1rI6szKolhR", "name": "Sequence 2", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "schedule_id": "sched_3MvlPuSqL0NGJyY95MBMpF", "status": "active", "steps": [{"created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:48.623996", "date_updated": "2022-11-08T12:07:48.624000", "delay": 3600, "email_template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "id": "seqstep_71ELVa12hUsLg2ba3K74Yq", "required": true, "sms_template_id": null, "step_allowed_delay": null, "step_type": "email", "threading": "old_thread", "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "timezone": "Europe/Kiev", "trigger_query": null, "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683885150272} -{"stream": "dialer", "data": {"caller_id": "", "date_created": "2022-11-08T14:27:23.185019", "date_updated": "2022-11-08T14:27:23.309015", "id": "dial_5qE6KYskn4PfT6oQLTe8ez", "is_user_dependent": false, "music_preference": "tone1", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "ring_mode": "more-voicemails", "source_type": "saved-search", "source_value": "save_vL2ENUTpFhqrYHf97tm61Y9Sh6QughnuMURuIH4VDCC", "status": "inactive", "target_type": "lead", "type": "power", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": [{"is_active": false, "state": "inactive", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}],"visibility_mode":"all"}, "emitted_at": 1683623891380} -{"stream": "smart_views", "data": {"date_created": "2022-11-08T15:50:57.032000+00:00", "date_updated": "2022-11-08T15:50:57.032000+00:00", "id": "save_UeQpNDWG0AuXKKunEIABD0qSjHvV62EFaTHzN70B6gN", "is_shared": false, "is_user_dependent": false, "name": "Deleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": null, "s_query": {"query": {"negate": false, "queries": [{"negate": false, "object_type": "lead", "type": "object_type"}, {"negate": false, "queries": [{"negate": false, "queries": [{"condition": {"before": {"range": "today", "type": "start_end_of_predefined_relative_period", "which": "end"}, "on_or_after": {"range": "today", "type": "start_end_of_predefined_relative_period", "which": "start"}, "type": "moment_range"}, "field": {"field_name": "date_created", "object_type": "lead", "type": "regular_field"}, "negate": false, "type": "field_condition"}], "type": "and"}], "type": "and"}], "type": "and"}, "results_limit": null, "sort": []}, "type": "lead", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623892300} -{"stream": "smart_views", "data": {"date_created": "2021-07-13T11:39:00.802000+00:00", "date_updated": "2021-07-13T11:39:00.802000+00:00", "id": "save_vL2ENUTpFhqrYHf97tm61Y9Sh6QughnuMURuIH4VDCC", "is_shared": true, "is_user_dependent": false, "name": "\ud83d\udce3 Untouched Leads", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "times_communicated:0 notes:0 tasks:0 opportunities:0", "s_query": null, "type": "lead", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623892302} -{"stream": "smart_views", "data": {"date_created": "2021-07-13T11:39:00.745000+00:00", "date_updated": "2021-07-13T11:39:00.745000+00:00", "id": "save_dnBmv1ixCRWNZ6P13oydsxGPLLxs94SWGzvBZIEqo3x", "is_shared": true, "is_user_dependent": false, "name": "\u260e\ufe0f Leads to Call", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "has:phone_numbers opportunities:0 calls:0 tasks:0", "s_query": null, "type": "lead", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623892304} -{"stream": "email_bulk_actions", "data": {"id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "date_created": "2022-11-08T12:58:33.566000+00:00", "date_updated": "2022-11-08T12:58:47.020000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "has:active_opportunities last_communication_date > \"7 days ago\"", "s_query": {"negate": false, "type": "match_all"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 1, "n_objects_processed": 1, "n_leads": 1, "n_leads_processed": 1, "send_done_email": true, "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "contact_preference": "lead", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "sender": "Jean Lafleur "}, "emitted_at": 1683623893223} -{"stream": "sequence_subscription_bulk_actions", "data": {"query": "", "from_phone_number_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status": "done", "action_type": "subscribe", "sender_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "bulkseqsub_nB54SIHOmtafCyAri59aJc24OfcCWudvQFtIW6k7oJv", "n_leads_processed": 11, "n_objects": 11, "sender_name": "Jean Lafleur", "n_leads": 11, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_preference": "contact", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sort": [], "date_created": "2022-11-08T15:09:57.014000+00:00", "s_query": {"negate": false, "queries": [{"negate": false, "object_type": "contact", "type": "object_type"}, {"negate": false, "queries": [], "type": "and"}], "type": "and"}, "n_objects_processed": 11, "calls_assigned_to": [], "sender_email": "integration-test@airbyte.io", "results_limit": null, "send_done_email": true, "date_updated": "2022-11-08T15:10:17.831000+00:00"}, "emitted_at": 1683623894094} -{"stream": "sequence_subscription_bulk_actions", "data": {"query": "", "from_phone_number_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status": "done", "action_type": "subscribe", "sender_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "bulkseqsub_YrswODZaYxUAhKYKcOcIQCWRxzbUjlsIgno8viR55dw", "n_leads_processed": 11, "n_objects": 11, "sender_name": "Jean Lafleur", "n_leads": 11, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_preference": "contact", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sort": [], "date_created": "2022-11-08T15:09:38.582000+00:00", "s_query": {"negate": false, "queries": [{"negate": false, "object_type": "contact", "type": "object_type"}, {"negate": false, "queries": [], "type": "and"}], "type": "and"}, "n_objects_processed": 11, "calls_assigned_to": ["group_2WQ2sEM5fBJ2Ip0lWMyRLg"], "sender_email": "iryna.grankova@airbyte.io", "results_limit": null, "send_done_email": true, "date_updated": "2022-11-08T15:09:55.730000+00:00"}, "emitted_at": 1683623894097} -{"stream": "delete_bulk_actions", "data": {"id": "bulkdelete_jfj3OxS8q9ri6uDwvr09Gwxl9Q6Oh3tgJ4cUPAaZq52", "date_created": "2022-11-08T15:49:38.627000+00:00", "date_updated": "2022-11-08T15:49:44.832000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "", "s_query": {"negate": false, "queries": [{"negate": false, "object_type": "lead", "type": "object_type"}, {"negate": false, "queries": [{"negate": false, "queries": [{"condition": {"before": {"range": "today", "which": "end", "type": "start_end_of_predefined_relative_period"}, "on_or_after": {"range": "today", "which": "start", "type": "start_end_of_predefined_relative_period"}, "type": "moment_range"}, "field": {"field_name": "date_created", "object_type": "lead", "type": "regular_field"}, "negate": false, "type": "field_condition"}], "type": "and"}], "type": "and"}], "type": "and"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 3, "n_objects_processed": 3, "n_leads": 3, "n_leads_processed": 3, "send_done_email": true, "bulk_object_type": "lead"}, "emitted_at": 1683623894896} -{"stream": "edit_bulk_actions", "data": {"id": "bulkedit_GMBUd9rUAIfYnKXokMKXV1AHMQUm9yCMGXhh96xYlxr", "date_created": "2021-08-25T21:15:27.477000+00:00", "date_updated": "2021-08-25T21:15:35.868000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "*", "s_query": {"negate": false, "type": "match_all"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 3, "n_objects_processed": 3, "n_leads": 3, "n_leads_processed": 3, "send_done_email": true, "type": "set_lead_status", "custom_field_name": null, "custom_field_value": null, "lead_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q"}, "emitted_at": 1683623895731} -{"stream": "edit_bulk_actions", "data": {"id": "bulkedit_mA19Kcb30egVp18LcMXP2OCTQaLsGJyWikJMXYu32kv", "date_created": "2021-08-25T21:03:43.590000+00:00", "date_updated": "2021-08-25T21:03:54.810000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "*", "s_query": {"negate": false, "type": "match_all"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 3, "n_objects_processed": 3, "n_leads": 3, "n_leads_processed": 3, "send_done_email": true, "type": "set_custom_field", "custom_field_name": "Lead Owner", "custom_field_value": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_status_id": null}, "emitted_at": 1683623895734} -{"stream": "integration_links", "data": {"type": "lead", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": "https://google.com/search?q={{ lead.display_name }}", "id": "ilink_CUrmAjZi7dy3T5nodx6INqBAYMIme3HQ9In2Zm4PHI4", "date_created": "2021-07-13T11:39:00.832000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Google Search", "date_updated": "2022-08-11T18:11:08.979000+00:00"}, "emitted_at": 1683623896553} -{"stream": "integration_links", "data": {"type": "contact", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": "https://www.linkedin.com/search/results/people/?keywords={{ contact.name }}", "id": "ilink_UFZJXoAlAiDsIbBMKPaK87FDp0hgjduPjRMcdjomGzS", "date_created": "2021-07-13T11:39:00.840000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "LinkedIn Search", "date_updated": "2022-08-11T18:11:08.993000+00:00"}, "emitted_at": 1683623896555} -{"stream": "custom_activities", "data": {"api_create_only": false, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:03:27.631229", "date_updated": "2022-11-08T12:03:27.631229", "description": null, "editable_with_roles": [], "fields": [{"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": "Test description", "editable_with_roles": [], "id": "cf_KLkKl1BRUm4o7zknJtTkvp6TMFgff1984UelnlGN9mZ", "is_shared": false, "name": "Test 1", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": "Test activity", "editable_with_roles": [], "id": "cf_aea8B3HHCHNaoNIqzpH6HLvYqvlxCbZNZU8OkhkAuuW", "is_shared": false, "name": "Test Actvity", "referenced_custom_type_id": null, "required": false, "type": "datetime"}], "id": "actitype_4cMlNQsx560XS5dhcUpxp2", "name": "Meeting Activity 1", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623897336} -{"stream": "custom_activities", "data": {"api_create_only": false, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:01.879565", "date_updated": "2021-07-13T11:39:01.879565", "description": null, "editable_with_roles": [], "fields": [{"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_UftGNTS2rq9XMG9hOHkALp5cgn4Xl8nZqN7gReax3lc", "is_shared": false, "name": "Contact", "referenced_custom_type_id": null, "required": false, "type": "contact"}, {"accepts_multiple_values": true, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi", "is_shared": true, "name": "Contact Role", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim", "is_shared": true, "name": "Industry", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_bp93vNo2vxbmM2QhQ7JvdytgcozCCGmOklTG9E8rDYa", "is_shared": false, "name": "Industry: Other (if applicable)", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D", "is_shared": true, "name": "Current Vendor/Software", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_3W4n175LyZ3QMfr665jUS19nt2QATHyM7QaxXQC4QqA", "is_shared": false, "name": "Current Vendor: Other (if applicable)", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_i1zuHB2hra0uMSHQuDvkx3niXapGCORwWL5t9mO6w6Q", "is_shared": false, "name": "Number of Potential Users", "referenced_custom_type_id": null, "required": false, "type": "number"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4", "is_shared": true, "name": "Referral Source", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_WKYpud81IJKhD0uFmxm12baU0YcAuTN3nE6TzktrJoT", "is_shared": false, "name": "Referral Source: Other (if applicable)", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_ldS8XBbJo4PxIpBYkA3XWQkwY3aGqHxJxDRV7sMTsaA", "is_shared": false, "name": "Next Steps", "referenced_custom_type_id": null, "required": false, "type": "textarea"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_jxTmWbZeP5LEVL4vokkPvTW0VPUU2UHobsphKgUB6xK", "is_shared": false, "name": "Notes", "referenced_custom_type_id": null, "required": false, "type": "textarea"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_1cL7txuLyrTH6Cpw15KtPLPT36l9eOdlJLKUdosxQLm", "is_shared": false, "name": "Qualified?", "referenced_custom_type_id": null, "required": false, "type": "choices"}], "id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "name": "Qualification Call", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1683623897340} +{"stream": "created_activities", "data": {"users": [], "activity_at": "2021-07-13T11:39:05.012000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "source": "ui", "contact_id": null, "_type": "Created", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "import_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "date_updated": "2021-07-13T11:39:05.012000+00:00", "id": "acti_CO9Th9maeKbnjp8XiG6gPada9OL2icBKBjuWK0aSzi5", "created_by_name": "Airbyte Team", "date_created": "2021-07-13T11:39:05.012000+00:00", "user_name": "Airbyte Team"}, "emitted_at": 1691417081566} +{"stream": "created_activities", "data": {"users": [], "activity_at": "2021-07-13T11:39:03.413000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "source": "ui", "contact_id": null, "_type": "Created", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "import_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-07-13T11:39:03.413000+00:00", "id": "acti_Vu4EowJ5vQVaTp2UviBUWsah2gfHfzEhToD0qt20awK", "created_by_name": "Airbyte Team", "date_created": "2021-07-13T11:39:03.413000+00:00", "user_name": "Airbyte Team"}, "emitted_at": 1691417081570} +{"stream": "created_activities", "data": {"users": [], "activity_at": "2021-07-04T11:39:00.850000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "source": "ui", "contact_id": null, "_type": "Created", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "import_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-07-13T11:39:04.485000+00:00", "id": "acti_1vDHprrVKawVMTbb4XmRtOJeP407lC7VeHwJu47cNzQ", "created_by_name": "Airbyte Team", "date_created": "2021-07-04T11:39:00.850000+00:00", "user_name": "Airbyte Team"}, "emitted_at": 1691417081573} +{"stream": "note_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note": "test", "users": [], "contact_id": null, "date_created": "2021-09-08T09:52:33.209000+00:00", "id": "acti_S8FCREkmsz7QYukb5ce25aMwDD09ojKF5R9eXiPJczw", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-09-08T09:52:33.209000+00:00", "activity_at": "2021-09-08T09:52:33.209000+00:00", "created_by_name": "Airbyte Team", "user_name": "Airbyte Team", "note_html": "

    test

    ", "updated_by_name": "Airbyte Team", "_type": "Note", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417084010} +{"stream": "note_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note": "demo note", "users": [], "contact_id": null, "date_created": "2021-08-12T14:16:54.027000+00:00", "id": "acti_MEZjvRruzMu1YJP9VFz8AIlJPLNCINLJjKO3bw7TAW7", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-08-12T14:16:55.644000+00:00", "activity_at": "2021-08-12T14:16:54.027000+00:00", "created_by_name": "Airbyte Team", "user_name": "Airbyte Team", "note_html": "

    demo note

    ", "updated_by_name": "Airbyte Team", "_type": "Note", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417084015} +{"stream": "note_activities", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note": "test 2", "users": [], "contact_id": null, "date_created": "2021-08-12T12:10:32.670000+00:00", "id": "acti_MWsXtWl1ouu73mqy8wFOsz6rskZ5fYky1XlB3UqlGwr", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-08-18T10:00:40.586000+00:00", "activity_at": "2021-08-12T12:10:32.670000+00:00", "created_by_name": "Airbyte Team", "user_name": "Airbyte Team", "note_html": "

    test 2

    ", "updated_by_name": "Airbyte Team", "_type": "Note", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417084018} +{"stream": "email_thread_activities", "data": {"n_emails": 6, "id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": null, "latest_emails": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_aHMyKom3arlMpEVXQYPNxASS0Luskq7sNLAl3Z23vyE", "date_sent": "2021-09-17T21:29:12+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:\n\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>", "envelope": {"message_id": "", "subject": "Re: Sandbox Account", "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "date": "Sat, 18 Sep 2021 08:29:12 +1100", "reply_to": [], "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "in_reply_to": "<163190650280.3618.8817856985357292069@smtpgw.close.com>", "is_autoreply": false, "bcc": [], "to": [{"name": "James Urie", "email": "james.urie@close.com"}]}, "date_updated": "2022-11-08T12:35:28.249000+00:00", "sender": "Jean Lafleur ", "subject": "Re: Sandbox Account", "direction": "outgoing", "date_created": "2021-09-17T21:29:12+00:00", "cc": ["integration-test@airbyte.io", "nick@close.com", "sherif@airbyte.io", "yuri.cherniaiev@airbyte.io"], "opens_summary": null, "status": "sent", "to": ["james.urie@close.com"]}, {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_3Mapa6JtEWAYWsQU2xF6sPDFz97slwbmFqhye8vDxAV", "date_sent": "2021-09-16T19:21:05+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif N", "envelope": {"message_id": "", "subject": "Re: Sandbox Account", "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "date": "Fri, 17 Sep 2021 06:21:05 +1100", "reply_to": [], "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "in_reply_to": "", "is_autoreply": false, "bcc": [], "to": [{"name": "James Urie", "email": "james.urie@close.com"}]}, "date_updated": "2022-11-08T12:35:28.436000+00:00", "sender": "Jean Lafleur ", "subject": "Re: Sandbox Account", "direction": "outgoing", "date_created": "2021-09-16T19:21:05+00:00", "cc": ["integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io", "sherif@airbyte.io"], "opens_summary": null, "status": "sent", "to": ["james.urie@close.com"]}, {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_DvkhzZr7BulOSmgomLFc3htdTwCDnu85eEoDuL8qQ9R", "date_sent": "2021-09-16T17:48:53+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 ", "envelope": {"message_id": "", "subject": "Re: Sandbox Account", "sender": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "from": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "date": "Thu, 16 Sep 2021 10:48:53 -0700", "reply_to": [], "cc": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "in_reply_to": "<163181368546.9449.10929845414794115430@smtpgw.close.com>", "is_autoreply": false, "bcc": [], "to": [{"name": "James Urie", "email": "james.urie@close.com"}]}, "date_updated": "2022-11-08T12:35:28.662000+00:00", "sender": "Sherif Nada ", "subject": "Re: Sandbox Account", "direction": "outgoing", "date_created": "2021-09-16T17:48:53+00:00", "cc": ["john@airbyte.io", "integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io"], "opens_summary": null, "status": "sent", "to": ["james.urie@close.com"]}, {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_cz7ITkJ0XVxLg6IGHjISOt0ETJiMtvzCvuKkC5HNZlj", "date_sent": "2021-08-25T05:00:00+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Hey Nick and James,\n\nThanks for the call today.\nFollowing up on our conversation, we should be able to release the\nintegration within 1 or 2 weeks.\n\nThanks,\n\nJohn\n\nOn Tue, Aug 17, 2021 at 7:29 AM Jean", "envelope": {"message_id": "", "subject": "Re: Sandbox Account", "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "date": "Wed, 25 Aug 2021 16:00:00 +1100", "reply_to": [], "cc": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "James Urie", "email": "james.urie@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "in_reply_to": "", "is_autoreply": false, "bcc": [], "to": [{"name": "Nick Persico", "email": "nick@close.com"}]}, "date_updated": "2022-11-08T12:35:28.963000+00:00", "sender": "Jean Lafleur ", "subject": "Re: Sandbox Account", "direction": "outgoing", "date_created": "2021-08-25T05:00:00+00:00", "cc": ["sherif@airbyte.io", "integration-test@airbyte.io", "james.urie@close.com", "yuri.cherniaiev@airbyte.io"], "opens_summary": null, "status": "sent", "to": ["nick@close.com"]}, {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_gF8wEbqW7Cx3fDIQTCnejQ3cOQKXLsE5md5JN88z4Lx", "date_sent": "2021-08-16T20:29:59+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Scheduled for next Tuesday!\n\nOn Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n\n> Jean!\n>\n> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>\n> Cheers,\n>\n> Nick Persico\n> Direct", "envelope": {"message_id": "", "subject": "Re: Sandbox Account", "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "date": "Tue, 17 Aug 2021 07:29:59 +1100", "reply_to": [], "cc": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "James Urie", "email": "james.urie@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "in_reply_to": "", "is_autoreply": false, "bcc": [], "to": [{"name": "Nick Persico", "email": "nick@close.com"}]}, "date_updated": "2022-11-08T12:35:29.133000+00:00", "sender": "Jean Lafleur ", "subject": "Re: Sandbox Account", "direction": "outgoing", "date_created": "2021-08-16T20:29:59+00:00", "cc": ["sherif@airbyte.io", "integration-test@airbyte.io", "james.urie@close.com", "yuri.cherniaiev@airbyte.io"], "opens_summary": null, "status": "sent", "to": ["nick@close.com"]}], "date_created": "2021-09-17T21:29:12.001000+00:00", "updated_by_name": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "activity_at": "2021-09-17T21:29:12.001000+00:00", "latest_normalized_subject": "Sandbox Account", "participants": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "'Nick Persico' via Integration tests", "email": "integration-test@airbyte.io"}], "_type": "EmailThread", "contact_id": null, "created_by": null, "date_updated": "2022-11-08T12:35:29.323000+00:00", "created_by_name": null, "user_name": "Airbyte Team"}, "emitted_at": 1691417087768} +{"stream": "email_thread_activities", "data": {"n_emails": 2, "id": "acti_3rQgtNnE0L7Wo90WzgOXcbfSR2WCjOBipMrAq41yJFE", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": null, "latest_emails": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_iNuDRoYQ5w17aiH9BRIfN2gHbWBth11m4D16q9mxkdn", "date_sent": "2021-08-25T21:15:36+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Hello Jean,\n\nYour bulk edit in Close was completed successfully.\n\nSearch query: *\nAction: Change lead status to \"Interested\".\n\nAll 2 leads were modified successfully.\n\nThanks,\nClose Support\nsupport@cl", "envelope": {"message_id": "<162992613594.10130.17609834081606867635@closeio-tasktiger-other-7bd975b8f7-f5phd>", "subject": "Your bulk edit for Airbyte is done", "sender": [{"name": "", "email": "support@close.com"}], "from": [{"name": "'Close CRM' via Integration tests", "email": "integration-test@airbyte.io"}], "date": "Wed, 25 Aug 2021 21:15:36 +0000", "reply_to": [{"name": "Close CRM", "email": "support@close.com"}], "cc": [], "in_reply_to": null, "is_autoreply": false, "bcc": [], "to": [{"name": "", "email": "integration-test@airbyte.io"}]}, "date_updated": "2022-11-08T12:36:05.979000+00:00", "sender": "'Close CRM' via Integration tests ", "subject": "Your bulk edit for Airbyte is done", "direction": "outgoing", "date_created": "2021-08-25T21:15:36+00:00", "cc": [], "opens_summary": null, "status": "sent", "to": ["integration-test@airbyte.io"]}, {"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_6pzzBcXnRUrC7ClGxkbfyifOL0tKELXmDQsGffzFx8N", "date_sent": "2021-08-25T21:03:54+00:00", "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Hello Jean,\n\nYour bulk edit in Close was completed successfully.\n\nSearch query: *\nAction: Set custom field \"Lead Owner\" to \"user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg\".\n\nAll 2 leads were modifie", "envelope": {"message_id": "<162992543485.16564.1574709800437653559@closeio-tasktiger-other-7bd975b8f7-x9jpv>", "subject": "Your bulk edit for Airbyte is done", "sender": [{"name": "", "email": "support@close.com"}], "from": [{"name": "'Close CRM' via Integration tests", "email": "integration-test@airbyte.io"}], "date": "Wed, 25 Aug 2021 21:03:54 +0000", "reply_to": [{"name": "Close CRM", "email": "support@close.com"}], "cc": [], "in_reply_to": null, "is_autoreply": false, "bcc": [], "to": [{"name": "", "email": "integration-test@airbyte.io"}]}, "date_updated": "2022-11-08T12:36:06.154000+00:00", "sender": "'Close CRM' via Integration tests ", "subject": "Your bulk edit for Airbyte is done", "direction": "outgoing", "date_created": "2021-08-25T21:03:54+00:00", "cc": [], "opens_summary": null, "status": "sent", "to": ["integration-test@airbyte.io"]}], "date_created": "2021-08-25T21:15:36.001000+00:00", "updated_by_name": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "activity_at": "2021-08-25T21:15:36.001000+00:00", "latest_normalized_subject": "Your bulk edit for Airbyte is done", "participants": [{"name": "'Close CRM' via Integration tests", "email": "integration-test@airbyte.io"}], "_type": "EmailThread", "contact_id": null, "created_by": null, "date_updated": "2022-11-08T12:36:06.162000+00:00", "created_by_name": null, "user_name": "Airbyte Team"}, "emitted_at": 1691417087772} +{"stream": "email_thread_activities", "data": {"n_emails": 1, "id": "acti_XFp5upc2w8MawOLJ89I5LNlWOYcJLBrHp6UOFOTfN0T", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "latest_emails": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_8MOghgAFLMW70Zj8pMTUA0NozWrEKeyckt4kFJcx65L", "date_sent": null, "send_as_id": null, "has_attachments": false, "bcc": [], "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "envelope": {"message_id": null, "subject": "Airbyte Follow-up", "sender": [], "from": [], "date": "Wed, 18 Aug 2021 10:35:50 -0000", "reply_to": [], "cc": [], "in_reply_to": null, "is_autoreply": false, "bcc": [], "to": [{"name": "", "email": "bluth@close.com"}]}, "date_updated": "2021-08-18T10:36:10.974000+00:00", "sender": null, "subject": "Airbyte Follow-up", "direction": "outgoing", "date_created": "2021-08-18T10:35:50.165000+00:00", "cc": [], "opens_summary": null, "status": "draft", "to": ["bluth@close.com"]}], "date_created": "2021-08-18T10:35:50.166000+00:00", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "activity_at": "2021-08-18T10:35:50.166000+00:00", "latest_normalized_subject": "Airbyte Follow-up", "participants": [], "_type": "EmailThread", "contact_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-08-18T10:36:10.989000+00:00", "created_by_name": "Airbyte Team", "user_name": "Airbyte Team"}, "emitted_at": 1691417087775} +{"stream": "email_activities", "data": {"body_html_quoted": [{"expand": true, "html": "
    Looking forward to it!

    On Sat, Sep 18, 2021 at 6:22 AM James Urie <james.urie@close.com> wrote:
    "}, {"expand": false, "html": "
    Scheduled some time next week!

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Sep 16, 2021, at 01:21 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    "}], "updated_by_name": null, "sequence_name": null, "direction": "outgoing", "date_updated": "2022-11-08T12:35:28.249000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "bulk_email_action_id": null, "_type": "Email", "envelope": {"subject": "Re: Sandbox Account", "in_reply_to": "<163190650280.3618.8817856985357292069@smtpgw.close.com>", "bcc": [], "to": [{"name": "James Urie", "email": "james.urie@close.com"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "reply_to": [], "message_id": "", "is_autoreply": false, "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "date": "Sat, 18 Sep 2021 08:29:12 +1100"}, "template_name": null, "status": "sent", "send_attempts": [], "thread_id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "has_reply": false, "body_preview": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:\n\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>", "attachments": [], "date_sent": "2021-09-17T21:29:12+00:00", "created_by_name": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "opens": [], "to": ["james.urie@close.com"], "send_as_id": null, "followup_sequence_id": null, "activity_at": "2021-09-17T21:29:12+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "in_reply_to_id": null, "users": [], "sequence_id": null, "subject": "Re: Sandbox Account", "template_id": null, "need_smtp_credentials": false, "date_created": "2021-09-17T21:29:12+00:00", "user_name": "Airbyte Team", "opens_summary": null, "sequence_subscription_id": null, "date_scheduled": null, "id": "acti_aHMyKom3arlMpEVXQYPNxASS0Luskq7sNLAl3Z23vyE", "message_ids": [""], "updated_by": null, "created_by": null, "bcc": [], "body_text": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:\n\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Sep 16, 2021, at 01:21 PM, Jean Lafleur wrote:\n>\n> Hey James,\n>\n> Do you want to do a quick catch-up on co-marketing this week?\n> Here's a link to my calendar: https://calendly.com/john-lafleur/30min\n>\n> Thanks,\n>\n> John\n>\n> On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:\n>\n>> Hi James,\n>>\n>> Happy to report we finished working on the Close.com source and have\n>> released it in beta just this week! I'll let John handle the co-marketing\n>> efforts ;)\n>>\n>> Best,\n>> Shrif\n>>\n>> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>>\n>>> Hey John! Any updates on this integration thus far?\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>>\n>>> Excellent! Keep us in the loop.\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>>\n>>> Hey Nick and James,\n>>>\n>>> Thanks for the call today.\n>>> Following up on our conversation, we should be able to release the\n>>> integration within 1 or 2 weeks.\n>>>\n>>> Thanks,\n>>>\n>>> John\n>>>\n>>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>>\n>>>> Scheduled for next Tuesday!\n>>>>\n>>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>>\n>>>>> Jean!\n>>>>>\n>>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur \n>>>>> wrote:\n>>>>>\n>>>>>> Nice to meet you, Nick!\n>>>>>> Let's schedule some time together as soon as we got the integration\n>>>>>> working :).\n>>>>>>\n>>>>>> Looking forward to it!\n>>>>>>\n>>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>>\n>>>>>>> Great! :-)\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, Close \n>>>>>>>\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>>\n>>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>>\n>>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>>\n>>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>>\n>>>>>>>>> Cheers,\n>>>>>>>>>\n>>>>>>>>> Nick Persico\n>>>>>>>>> Director of Sales, *Close* \n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Thank you James!\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Extended the trial out until 2023. Just ping us then when you\n>>>>>>>>>> need more time.\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>>> working properly\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>>> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Hi James,\n>>>>>>>>>>>\n>>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Shrif\n>>>>>>>>>>>\n>>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>>> tests wrote:\n>>>>>>>>>>>\n>>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> James Urie\n>>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>>\n>>>>>>>>>>>> \n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>>\n>>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>>\n>>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We're happy to build and maintain this integration\n>>>>>>>>>>>> independently without support from Kustomer; the only thing we'd need is\n>>>>>>>>>>>> access to an API-enabled Close sandbox account where we can verify that the\n>>>>>>>>>>>> integration we've built is functional.\n>>>>>>>>>>>>\n>>>>>>>>>>>> Best,\n>>>>>>>>>>>> Yurii\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>\n\n", "body_text_quoted": [{"expand": true, "text": "Looking forward to it!\n\nOn Sat, Sep 18, 2021 at 6:22 AM James Urie wrote:"}, {"expand": false, "text": "\n> Scheduled some time next week!\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Sep 16, 2021, at 01:21 PM, Jean Lafleur wrote:\n>\n> Hey James,\n>\n> Do you want to do a quick catch-up on co-marketing this week?\n> Here's a link to my calendar: https://calendly.com/john-lafleur/30min\n>\n> Thanks,\n>\n> John\n>\n> On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:\n>\n>> Hi James,\n>>\n>> Happy to report we finished working on the Close.com source and have\n>> released it in beta just this week! I'll let John handle the co-marketing\n>> efforts ;)\n>>\n>> Best,\n>> Shrif\n>>\n>> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>>\n>>> Hey John! Any updates on this integration thus far?\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>>\n>>> Excellent! Keep us in the loop.\n>>>\n>>> James Urie\n>>> Sr. Account Executive\n>>>\n>>> \n>>>\n>>>\n>>>\n>>>\n>>>\n>>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>>\n>>> Hey Nick and James,\n>>>\n>>> Thanks for the call today.\n>>> Following up on our conversation, we should be able to release the\n>>> integration within 1 or 2 weeks.\n>>>\n>>> Thanks,\n>>>\n>>> John\n>>>\n>>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>>\n>>>> Scheduled for next Tuesday!\n>>>>\n>>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>>\n>>>>> Jean!\n>>>>>\n>>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur \n>>>>> wrote:\n>>>>>\n>>>>>> Nice to meet you, Nick!\n>>>>>> Let's schedule some time together as soon as we got the integration\n>>>>>> working :).\n>>>>>>\n>>>>>> Looking forward to it!\n>>>>>>\n>>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>>\n>>>>>>> Great! :-)\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, Close \n>>>>>>>\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>>\n>>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>>\n>>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>>\n>>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>>\n>>>>>>>>> Cheers,\n>>>>>>>>>\n>>>>>>>>> Nick Persico\n>>>>>>>>> Director of Sales, *Close* \n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Thank you James!\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Extended the trial out until 2023. Just ping us then when you\n>>>>>>>>>> need more time.\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>>> working properly\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>>> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Hi James,\n>>>>>>>>>>>\n>>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Shrif\n>>>>>>>>>>>\n>>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>>> tests wrote:\n>>>>>>>>>>>\n>>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> James Urie\n>>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>>\n>>>>>>>>>>>> \n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>>\n>>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>>\n>>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>>\n>>>>>>>>>>>> We're happy to build and maintain this integration\n>>>>>>>>>>>> independently without support from Kustomer; the only thing we'd need is\n>>>>>>>>>>>> access to an API-enabled Close sandbox account where we can verify that the\n>>>>>>>>>>>> integration we've built is functional.\n>>>>>>>>>>>>\n>>>>>>>>>>>> Best,\n>>>>>>>>>>>> Yurii\n>>>>>>>>>>>>\n>>>>>>>>>>>>\n>>>>>\n\n"}], "body_html": "
    Looking forward to it!

    On Sat, Sep 18, 2021 at 6:22 AM James Urie <james.urie@close.com> wrote:
    Scheduled some time next week!

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Sep 16, 2021, at 01:21 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n\n", "contact_id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "sender": "Jean Lafleur ", "followup_sequence_delay": null, "references": ["", "", "", "", "", "", "<163000492090.2550.4736014357857312201@smtpgw.close.com>", "<163181368546.9449.10929845414794115430@smtpgw.close.com>", "", "", "<163190650280.3618.8817856985357292069@smtpgw.close.com>"], "cc": ["integration-test@airbyte.io", "nick@close.com", "sherif@airbyte.io", "yuri.cherniaiev@airbyte.io"], "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe"}, "emitted_at": 1691417090319} +{"stream": "email_activities", "data": {"body_html_quoted": [{"expand": true, "html": "
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    "}, {"expand": false, "html": "
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n
    "}], "updated_by_name": null, "sequence_name": null, "direction": "outgoing", "date_updated": "2022-11-08T12:35:28.436000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "bulk_email_action_id": null, "_type": "Email", "envelope": {"subject": "Re: Sandbox Account", "in_reply_to": "", "bcc": [], "to": [{"name": "James Urie", "email": "james.urie@close.com"}], "from": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "reply_to": [], "message_id": "", "is_autoreply": false, "sender": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}], "cc": [{"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}, {"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "date": "Fri, 17 Sep 2021 06:21:05 +1100"}, "template_name": null, "status": "sent", "send_attempts": [], "thread_id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "has_reply": false, "body_preview": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif N", "attachments": [], "date_sent": "2021-09-16T19:21:05+00:00", "created_by_name": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "opens": [], "to": ["james.urie@close.com"], "send_as_id": null, "followup_sequence_id": null, "activity_at": "2021-09-16T19:21:05+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "in_reply_to_id": null, "users": [], "sequence_id": null, "subject": "Re: Sandbox Account", "template_id": null, "need_smtp_credentials": false, "date_created": "2021-09-16T19:21:05+00:00", "user_name": "Airbyte Team", "opens_summary": null, "sequence_subscription_id": null, "date_scheduled": null, "id": "acti_3Mapa6JtEWAYWsQU2xF6sPDFz97slwbmFqhye8vDxAV", "message_ids": [""], "updated_by": null, "created_by": null, "bcc": [], "body_text": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:\n\n> Hi James,\n>\n> Happy to report we finished working on the Close.com source and have\n> released it in beta just this week! I'll let John handle the co-marketing\n> efforts ;)\n>\n> Best,\n> Shrif\n>\n> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>\n>> Hey John! Any updates on this integration thus far?\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>\n>> Excellent! Keep us in the loop.\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>\n>> Hey Nick and James,\n>>\n>> Thanks for the call today.\n>> Following up on our conversation, we should be able to release the\n>> integration within 1 or 2 weeks.\n>>\n>> Thanks,\n>>\n>> John\n>>\n>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>\n>>> Scheduled for next Tuesday!\n>>>\n>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>\n>>>> Jean!\n>>>>\n>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>\n>>>> Cheers,\n>>>>\n>>>> Nick Persico\n>>>> Director of Sales, Close \n>>>>\n>>>>\n>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>>\n>>>>> Nice to meet you, Nick!\n>>>>> Let's schedule some time together as soon as we got the integration\n>>>>> working :).\n>>>>>\n>>>>> Looking forward to it!\n>>>>>\n>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>\n>>>>>> Great! :-)\n>>>>>>\n>>>>>> Cheers,\n>>>>>>\n>>>>>> Nick Persico\n>>>>>> Director of Sales, Close \n>>>>>>\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>> wrote:\n>>>>>>\n>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>\n>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>>\n>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>\n>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>\n>>>>>>>> Cheers,\n>>>>>>>>\n>>>>>>>> Nick Persico\n>>>>>>>> Director of Sales, *Close* \n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Thank you James!\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>>> more time.\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>> working properly\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Hi James,\n>>>>>>>>>>\n>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Shrif\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>> tests wrote:\n>>>>>>>>>>\n>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>\n>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>\n>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>\n>>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>>> we've built is functional.\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Yurii\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>\n\n", "body_text_quoted": [{"expand": true, "text": "Hey James,\n\nDo you want to do a quick catch-up on co-marketing this week?\nHere's a link to my calendar: https://calendly.com/john-lafleur/30min\n\nThanks,\n\nJohn\n\nOn Fri, Sep 17, 2021 at 4:49 AM Sherif Nada wrote:"}, {"expand": false, "text": "\n> Hi James,\n>\n> Happy to report we finished working on the Close.com source and have\n> released it in beta just this week! I'll let John handle the co-marketing\n> efforts ;)\n>\n> Best,\n> Shrif\n>\n> On Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n>\n>> Hey John! Any updates on this integration thus far?\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>>\n>> Excellent! Keep us in the loop.\n>>\n>> James Urie\n>> Sr. Account Executive\n>>\n>> \n>>\n>>\n>>\n>>\n>>\n>> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>>\n>> Hey Nick and James,\n>>\n>> Thanks for the call today.\n>> Following up on our conversation, we should be able to release the\n>> integration within 1 or 2 weeks.\n>>\n>> Thanks,\n>>\n>> John\n>>\n>> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>>\n>>> Scheduled for next Tuesday!\n>>>\n>>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>>\n>>>> Jean!\n>>>>\n>>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>>\n>>>> Cheers,\n>>>>\n>>>> Nick Persico\n>>>> Director of Sales, Close \n>>>>\n>>>>\n>>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>>\n>>>>> Nice to meet you, Nick!\n>>>>> Let's schedule some time together as soon as we got the integration\n>>>>> working :).\n>>>>>\n>>>>> Looking forward to it!\n>>>>>\n>>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>>\n>>>>>> Great! :-)\n>>>>>>\n>>>>>> Cheers,\n>>>>>>\n>>>>>> Nick Persico\n>>>>>> Director of Sales, Close \n>>>>>>\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>>> wrote:\n>>>>>>\n>>>>>>> +John, COO, since he would handle marketing.\n>>>>>>>\n>>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>>\n>>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>>\n>>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>>\n>>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>>\n>>>>>>>> Cheers,\n>>>>>>>>\n>>>>>>>> Nick Persico\n>>>>>>>> Director of Sales, *Close* \n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Thank you James!\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>>> more time.\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>>> working properly\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>>> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Hi James,\n>>>>>>>>>>\n>>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Shrif\n>>>>>>>>>>\n>>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration\n>>>>>>>>>> tests wrote:\n>>>>>>>>>>\n>>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> James Urie\n>>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>>\n>>>>>>>>>>> \n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>>\n>>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>>\n>>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>>\n>>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>>\n>>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>>> we've built is functional.\n>>>>>>>>>>>\n>>>>>>>>>>> Best,\n>>>>>>>>>>> Yurii\n>>>>>>>>>>>\n>>>>>>>>>>>\n>>>>\n\n"}], "body_html": "
    Hey James,\u00a0

    Do you want to do a quick catch-up on co-marketing this week?
    Here's a link to my calendar:\u00a0https://calendly.com/john-lafleur/30min

    Thanks,

    John

    On Fri, Sep 17, 2021 at 4:49 AM Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n
    \n\n", "contact_id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "sender": "Jean Lafleur ", "followup_sequence_delay": null, "references": ["", "", "<162629568767.3923.5347313150012573840@smtpgw.close.com>", "", "", "", "", "", "", "<163000492090.2550.4736014357857312201@smtpgw.close.com>", "<163181368546.9449.10929845414794115430@smtpgw.close.com>", ""], "cc": ["integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io", "sherif@airbyte.io"], "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe"}, "emitted_at": 1691417090324} +{"stream": "email_activities", "data": {"body_html_quoted": [{"expand": true, "html": "
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    "}, {"expand": false, "html": "
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    "}], "updated_by_name": null, "sequence_name": null, "direction": "outgoing", "date_updated": "2022-11-08T12:35:28.662000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "bulk_email_action_id": null, "_type": "Email", "envelope": {"subject": "Re: Sandbox Account", "in_reply_to": "<163181368546.9449.10929845414794115430@smtpgw.close.com>", "bcc": [], "to": [{"name": "James Urie", "email": "james.urie@close.com"}], "from": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "reply_to": [], "message_id": "", "is_autoreply": false, "sender": [{"name": "Sherif Nada", "email": "sherif@airbyte.io"}], "cc": [{"name": "Jean Lafleur", "email": "john@airbyte.io"}, {"name": "Airbyte", "email": "integration-test@airbyte.io"}, {"name": "Nick Persico", "email": "nick@close.com"}, {"name": "Yuri Cherniaiev", "email": "yuri.cherniaiev@airbyte.io"}], "date": "Thu, 16 Sep 2021 10:48:53 -0700"}, "template_name": null, "status": "sent", "send_attempts": [], "thread_id": "acti_ca3UUwuIKEStBlyz5WsmniBKV60eVZCxVknub1SSnyL", "has_reply": false, "body_preview": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 ", "attachments": [], "date_sent": "2021-09-16T17:48:53+00:00", "created_by_name": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "opens": [], "to": ["james.urie@close.com"], "send_as_id": null, "followup_sequence_id": null, "activity_at": "2021-09-16T17:48:53+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "in_reply_to_id": null, "users": [], "sequence_id": null, "subject": "Re: Sandbox Account", "template_id": null, "need_smtp_credentials": false, "date_created": "2021-09-16T17:48:53+00:00", "user_name": "Airbyte Team", "opens_summary": null, "sequence_subscription_id": null, "date_scheduled": null, "id": "acti_DvkhzZr7BulOSmgomLFc3htdTwCDnu85eEoDuL8qQ9R", "message_ids": [""], "updated_by": null, "created_by": null, "bcc": [], "body_text": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:\n\n> Hey John! Any updates on this integration thus far?\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>\n> Excellent! Keep us in the loop.\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>\n> Hey Nick and James,\n>\n> Thanks for the call today.\n> Following up on our conversation, we should be able to release the\n> integration within 1 or 2 weeks.\n>\n> Thanks,\n>\n> John\n>\n> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>\n>> Scheduled for next Tuesday!\n>>\n>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>\n>>> Jean!\n>>>\n>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>\n>>> Cheers,\n>>>\n>>> Nick Persico\n>>> Director of Sales, Close \n>>>\n>>>\n>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>\n>>>> Nice to meet you, Nick!\n>>>> Let's schedule some time together as soon as we got the integration\n>>>> working :).\n>>>>\n>>>> Looking forward to it!\n>>>>\n>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>\n>>>>> Great! :-)\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>> wrote:\n>>>>>\n>>>>>> +John, COO, since he would handle marketing.\n>>>>>>\n>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>\n>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>\n>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, *Close* \n>>>>>>>\n>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada wrote:\n>>>>>>>\n>>>>>>> Thank you James!\n>>>>>>>\n>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>> more time.\n>>>>>>>>\n>>>>>>>> James Urie\n>>>>>>>> Sr. Account Executive\n>>>>>>>>\n>>>>>>>> \n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>> working properly\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Hi James,\n>>>>>>>>>\n>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>\n>>>>>>>>> Best,\n>>>>>>>>> Shrif\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests\n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>\n>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>\n>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>\n>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>> we've built is functional.\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Yurii\n>>>>>>>>>>\n>>>>>>>>>>\n>>>\n\n", "body_text_quoted": [{"expand": true, "text": "Hi James,\n\nHappy to report we finished working on the Close.com source and have\nreleased it in beta just this week! I'll let John handle the co-marketing\nefforts ;)\n\nBest,\nShrif\n\nOn Thu, Sep 16, 2021 at 10:34 AM James Urie wrote:"}, {"expand": false, "text": "\n> Hey John! Any updates on this integration thus far?\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 26, 2021, at 01:08 PM, James Urie wrote:\n>\n> Excellent! Keep us in the loop.\n>\n> James Urie\n> Sr. Account Executive\n>\n> \n>\n>\n>\n>\n>\n> On Aug 24, 2021, at 11:00 PM, Jean Lafleur wrote:\n>\n> Hey Nick and James,\n>\n> Thanks for the call today.\n> Following up on our conversation, we should be able to release the\n> integration within 1 or 2 weeks.\n>\n> Thanks,\n>\n> John\n>\n> On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur wrote:\n>\n>> Scheduled for next Tuesday!\n>>\n>> On Tue, Aug 17, 2021 at 7:02 AM Nick Persico wrote:\n>>\n>>> Jean!\n>>>\n>>> Here you go \u2192 https://savvycal.com/nickpersico/chat\n>>>\n>>> Cheers,\n>>>\n>>> Nick Persico\n>>> Director of Sales, Close \n>>>\n>>>\n>>> On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur wrote:\n>>>\n>>>> Nice to meet you, Nick!\n>>>> Let's schedule some time together as soon as we got the integration\n>>>> working :).\n>>>>\n>>>> Looking forward to it!\n>>>>\n>>>> On Thu, Jul 15, 2021 at 8:30 AM Nick Persico wrote:\n>>>>\n>>>>> Great! :-)\n>>>>>\n>>>>> Cheers,\n>>>>>\n>>>>> Nick Persico\n>>>>> Director of Sales, Close \n>>>>>\n>>>>>\n>>>>> On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada \n>>>>> wrote:\n>>>>>\n>>>>>> +John, COO, since he would handle marketing.\n>>>>>>\n>>>>>> Will ping you guys as soon as we have the integration ready to go.\n>>>>>> Currently estimating some time in August, so let's circle back then!\n>>>>>>\n>>>>>> On Wed, Jul 14, 2021 at 1:48 PM Nick Persico wrote:\n>>>>>>\n>>>>>>> Hey Sherif and Airbyte team --\n>>>>>>>\n>>>>>>> Please let me know once you have the integration working. I would\n>>>>>>> love to check it out so we can put some co-marketing resources behind it!\n>>>>>>>\n>>>>>>> Cheers,\n>>>>>>>\n>>>>>>> Nick Persico\n>>>>>>> Director of Sales, *Close* \n>>>>>>>\n>>>>>>> On Jul 13, 2021, at 05:09 PM, Sherif Nada wrote:\n>>>>>>>\n>>>>>>> Thank you James!\n>>>>>>>\n>>>>>>> On Tue, Jul 13, 2021 at 1:25 PM James Urie \n>>>>>>> wrote:\n>>>>>>>\n>>>>>>>> Extended the trial out until 2023. Just ping us then when you need\n>>>>>>>> more time.\n>>>>>>>>\n>>>>>>>> James Urie\n>>>>>>>> Sr. Account Executive\n>>>>>>>>\n>>>>>>>> \n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>>\n>>>>>>>> On Jul 13, 2021, at 02:00 PM, Sherif Nada \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>> Correct. The purpose would be to ensure the integration is\n>>>>>>>> working properly\n>>>>>>>>\n>>>>>>>> On Tue, Jul 13, 2021 at 12:29 PM James Urie \n>>>>>>>> wrote:\n>>>>>>>>\n>>>>>>>>> How long do you need the sandbox account? Indefinitely?\n>>>>>>>>>\n>>>>>>>>> James Urie\n>>>>>>>>> Sr. Account Executive\n>>>>>>>>>\n>>>>>>>>> \n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>>\n>>>>>>>>> On Jul 13, 2021, at 12:35 PM, Sherif Nada \n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>> Hi James,\n>>>>>>>>>\n>>>>>>>>> Yes the trial account was intended for that purpose!\n>>>>>>>>>\n>>>>>>>>> Best,\n>>>>>>>>> Shrif\n>>>>>>>>>\n>>>>>>>>> On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests\n>>>>>>>>> wrote:\n>>>>>>>>>\n>>>>>>>>>> Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial\n>>>>>>>>>> account that you opened today intended for this purpose?\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> James Urie\n>>>>>>>>>> Sr. Account Executive\n>>>>>>>>>>\n>>>>>>>>>> \n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>>\n>>>>>>>>>> On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <\n>>>>>>>>>> yuri.cherniaiev@airbyte.io> wrote:\n>>>>>>>>>>\n>>>>>>>>>> Dear Close Team,\n>>>>>>>>>>\n>>>>>>>>>> We are an Airbyte -- an open source data\n>>>>>>>>>> portability platform which helps users desilo their data across APIs,\n>>>>>>>>>> databases, and warehouses. We're like the open source version of Fivetran.\n>>>>>>>>>>\n>>>>>>>>>> We've received requests to build an integration with Close to\n>>>>>>>>>> allow users to pull their Close data via API into analysis & dashboard\n>>>>>>>>>> tools like Redshift, BigQuery, and Looker.\n>>>>>>>>>>\n>>>>>>>>>> We're happy to build and maintain this integration independently\n>>>>>>>>>> without support from Kustomer; the only thing we'd need is access to an\n>>>>>>>>>> API-enabled Close sandbox account where we can verify that the integration\n>>>>>>>>>> we've built is functional.\n>>>>>>>>>>\n>>>>>>>>>> Best,\n>>>>>>>>>> Yurii\n>>>>>>>>>>\n>>>>>>>>>>\n>>>\n\n"}], "body_html": "
    Hi James,\u00a0

    Happy to report we finished working on the Close.com source and have released it in beta just this week! I'll let John handle the co-marketing efforts ;)\u00a0

    Best,\u00a0
    Shrif

    On Thu, Sep 16, 2021 at 10:34 AM James Urie <james.urie@close.com> wrote:
    Hey John! Any updates on this integration thus far?\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 26, 2021, at 01:08 PM, James Urie <james.urie@close.com> wrote:
    Excellent! Keep us in the loop.\u00a0

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Aug 24, 2021, at 11:00 PM, Jean Lafleur <john@airbyte.io> wrote:
    Hey Nick and James,\u00a0

    Thanks for the call today.\u00a0
    Following up on our conversation, we should be able to release the integration within 1 or 2 weeks.\u00a0

    Thanks,

    John

    On Tue, Aug 17, 2021 at 7:29 AM Jean Lafleur <john@airbyte.io> wrote:
    Scheduled for next Tuesday!

    On Tue, Aug 17, 2021 at 7:02 AM Nick Persico <nick@close.com> wrote:
    \"\"

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 11:17 PM, Jean Lafleur <john@airbyte.io> wrote:
    Nice to meet you, Nick!
    Let's schedule some time together as soon as we got the integration working :).\u00a0

    Looking forward to it!

    On Thu, Jul 15, 2021 at 8:30 AM Nick Persico <nick@close.com> wrote:
    Great! :-)\u00a0

    Cheers,

    Nick Persico
    Director of Sales, Close


    On Wed, Jul 14, 2021 at 5:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    +John, COO, since he would handle marketing.\u00a0

    Will ping you guys as soon as we have the integration ready to go. Currently estimating some time in August, so let's circle back then!

    On Wed, Jul 14, 2021 at 1:48 PM Nick Persico <nick@close.com> wrote:
    Hey Sherif and Airbyte team --

    Please let me know once you have the integration working. I would love to check it out so we can put some co-marketing resources behind it!

    Cheers,

    Nick Persico
    Director of Sales, Close

    On Jul 13, 2021, at 05:09 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Thank you James!

    On Tue, Jul 13, 2021 at 1:25 PM James Urie <james.urie@close.com> wrote:
    Extended the trial out until 2023. Just ping us then when you need more time.

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 02:00 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Correct. The purpose would be to ensure the integration is working\u00a0properly

    On Tue, Jul 13, 2021 at 12:29 PM James Urie <james.urie@close.com> wrote:
    How long do you need the sandbox account? Indefinitely?

    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 12:35 PM, Sherif Nada <sherif@airbyte.io> wrote:
    Hi James,\u00a0

    Yes the trial account was intended for that purpose!

    Best,\u00a0
    Shrif

    On Tue, Jul 13, 2021 at 7:03 AM 'James Urie' via Integration tests <integration-test@airbyte.io> wrote:
    Hey Yuri! Happy to accommodate a sandbox. Was the Airbyte trial account that you opened today intended for this purpose?


    James Urie
    Sr. Account Executive    		\u00a0



    		\u00a0\u00a0



    On Jul 13, 2021, at 06:24 AM, Yuri Cherniaiev <yuri.cherniaiev@airbyte.io> wrote:
    Dear Close Team,

    We are an Airbyte -- an open source data portability platform which helps users desilo their data across APIs, databases, and warehouses. We're like the open source version of Fivetran.\u00a0

    We've received requests to build an integration with Close to allow users to pull their Close data via API into analysis & dashboard tools like Redshift, BigQuery, and Looker.

    We're happy to build and maintain this integration independently without support from Kustomer; the only thing we'd need is access to an API-enabled Close sandbox account where we can verify that the integration we've built is functional.\u00a0

    Best,\u00a0
    Yurii

    \n\n", "contact_id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "sender": "Sherif Nada ", "followup_sequence_delay": null, "references": ["", "", "<162629568767.3923.5347313150012573840@smtpgw.close.com>", "", "", "", "", "", "", "<163000492090.2550.4736014357857312201@smtpgw.close.com>", "<163181368546.9449.10929845414794115430@smtpgw.close.com>"], "cc": ["john@airbyte.io", "integration-test@airbyte.io", "nick@close.com", "yuri.cherniaiev@airbyte.io"], "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe"}, "emitted_at": 1691417090329} +{"stream": "sms_activities", "data": {"local_country_iso": "US", "activity_at": "2021-08-11T18:14:32.750000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 202-555-0186", "sequence_subscription_id": null, "date_scheduled": null, "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "_type": "SMS", "updated_by_name": "Airbyte Team", "local_phone": "+14154445555", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "acti_GIVSys3F0wFeA519lDa5QKRfYOPgskyqKj2aXiCMSEO", "attachments": [], "sequence_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_sent": null, "local_phone_formatted": "+1 415-444-5555", "created_by_name": "Airbyte Team", "date_created": "2021-08-11T18:14:32.750000+00:00", "cost": null, "text": "Hi! This is a reminder that we have a call scheduled for 12pm PT today.", "direction": "outbound", "error_message": null, "date_updated": "2021-08-11T18:14:32.750000+00:00", "source": "Close.io", "template_id": null, "sequence_id": null, "remote_country_iso": "US", "status": "draft", "remote_phone": "+12025550186", "user_name": "Airbyte Team"}, "emitted_at": 1691417093006} +{"stream": "sms_activities", "data": {"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "attachments": [], "remote_phone": "+14156236785", "sequence_id": null, "remote_country_iso": "US", "sequence_subscription_id": null, "cost": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 415-623-6785", "created_by_name": "Airbyte Team", "date_updated": "2022-11-09T14:02:24.943000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outbound", "local_phone_formatted": "+1 415-625-1293", "text": "Hi!", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "date_created": "2022-11-09T14:02:18.756000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "local_phone": "+14156251293", "source": "Close.io", "_type": "SMS", "local_country_iso": "US", "updated_by_name": "Airbyte Team", "date_scheduled": null, "error_message": null, "id": "acti_CXrlrlHc8QpP5MBXqCfcJKpctDqBREuG3Whj0JQODu4", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "activity_at": "2022-11-09T14:02:18.756000+00:00", "user_name": "Airbyte Team", "sequence_name": null, "status": "draft", "date_sent": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417093912} +{"stream": "sms_activities", "data": {"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "attachments": [{"media_id": "media_580CfGelcIvP5BzvIYTlhX", "url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/", "thumbnail_url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/thumbnail/", "content_type": "image/png", "size": 6132, "filename": "Airbyte_logo_75x75.png"}], "remote_phone": "+14156236785", "sequence_id": null, "remote_country_iso": "US", "sequence_subscription_id": null, "cost": "3", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 415-623-6785", "created_by_name": "Airbyte Team", "date_updated": "2022-11-09T12:54:44.405000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outbound", "local_phone_formatted": "+1 415-625-1293", "text": "Hi!", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "date_created": "2022-11-09T12:54:15.456000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "local_phone": "+14156251293", "source": "Close.io", "_type": "SMS", "local_country_iso": "US", "updated_by_name": "Airbyte Team", "date_scheduled": null, "error_message": null, "id": "acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "activity_at": "2022-11-09T12:54:44.404000+00:00", "user_name": "Airbyte Team", "sequence_name": null, "status": "sent", "date_sent": "2022-11-09T12:54:44.404000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417093915} +{"stream": "call_activities", "data": {"local_phone_formatted": null, "voicemail_url": "https://api.close.com/call/acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt/voicemail/", "transferred_to_user_id": null, "forwarded_to": null, "remote_country_iso": "US", "date_answered": null, "voicemail_duration": 28, "_type": "Call", "disposition": "vm-left", "status": "completed", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "updated_by_name": null, "date_created": "2021-07-16T00:00:12.646000+00:00", "recording_expires_at": null, "remote_phone_formatted": "+1 650-517-6539", "cost": null, "phone": "+16505176539", "is_joinable": false, "created_by_name": null, "note": null, "sequence_id": null, "recording_url": null, "local_phone": null, "sequence_subscription_id": null, "transferred_from_user_id": null, "has_recording": false, "source": "Close.io", "transferred_from": null, "direction": "inbound", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "dialer_saved_search_id": null, "created_by": null, "remote_phone": "+16505176539", "updated_by": null, "duration": 0, "call_method": "regular", "id": "acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt", "coach_legs": [], "is_to_group_number": false, "date_updated": "2021-07-16T00:00:12.646000+00:00", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "users": [], "user_name": null, "note_html": null, "is_forwarded": false, "transferred_to": null, "dialer_id": null, "local_country_iso": "", "user_id": null, "activity_at": "2021-07-16T00:00:12.646000+00:00"}, "emitted_at": 1691417095367} +{"stream": "call_activities", "data": {"local_phone_formatted": null, "voicemail_url": null, "transferred_to_user_id": null, "forwarded_to": null, "remote_country_iso": "US", "date_answered": null, "voicemail_duration": 0, "_type": "Call", "disposition": null, "status": "no-answer", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by_name": "Airbyte Team", "date_created": "2021-07-05T11:39:00.850000+00:00", "recording_expires_at": null, "remote_phone_formatted": "+1 202-555-0186", "cost": null, "phone": "+12025550186", "is_joinable": false, "created_by_name": "Airbyte Team", "note": "Gob never answered.", "sequence_id": null, "recording_url": null, "local_phone": null, "sequence_subscription_id": null, "transferred_from_user_id": null, "has_recording": false, "source": "Close.io", "transferred_from": null, "direction": "outbound", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "dialer_saved_search_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone": "+12025550186", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "duration": 0, "call_method": "regular", "id": "acti_wszWUd92D7wNYSbn5gKWXCf55NeyU0jc1Vguv7DfUiH", "coach_legs": [], "is_to_group_number": false, "date_updated": "2021-07-13T11:39:04.536000+00:00", "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "users": [], "user_name": "Airbyte Team", "note_html": "

    Gob never answered.

    ", "is_forwarded": false, "transferred_to": null, "dialer_id": null, "local_country_iso": "", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-07-05T11:39:00.850000+00:00"}, "emitted_at": 1691417095372} +{"stream": "call_activities", "data": {"activity_at": "2022-11-09T13:57:14.751000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_name": null, "users": [], "recording_url": null, "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "transferred_to": null, "remote_phone": "+14156236785", "remote_country_iso": "US", "cost": "2", "_type": "Call", "local_country_iso": "US", "duration": 17, "created_by_name": "Airbyte Team", "direction": "outbound", "has_recording": false, "voicemail_url": null, "transferred_to_user_id": null, "date_answered": "2022-11-09T13:57:22.063000+00:00", "transferred_from": null, "coach_legs": [], "phone": "+14156236785", "dialer_saved_search_id": null, "voicemail_duration": 0, "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "dialer_id": null, "is_forwarded": false, "is_joinable": false, "transferred_from_user_id": null, "local_phone": "+14156251293", "date_updated": "2022-11-09T13:57:40.167000+00:00", "recording_expires_at": null, "forwarded_to": null, "sequence_subscription_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "completed", "source": "Close.io", "date_created": "2022-11-09T13:57:14.751000+00:00", "local_phone_formatted": "+1 415-625-1293", "note_html": "

    ", "is_to_group_number": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "disposition": "answered", "remote_phone_formatted": "+1 415-623-6785", "user_name": "Airbyte Team", "note": "", "call_method": "regular", "id": "acti_ZgqHg31m0XXDZPwaxUVUNOhnEoLSG3fY8rMn9iIS3cN", "sequence_id": null}, "emitted_at": 1691417095961} +{"stream": "meeting_activities", "data": {"starts_at": "2022-11-12T18:00:00+00:00", "user_note": null, "updated_by_name": "Airbyte Team", "notetaker_id": null, "_type": "Meeting", "id": "acti_AlRbqNk15jdt7Eq2phHXuV49qSpkLBB7af3mwvJkk1z", "duration": 3600, "ends_at": "2022-11-12T19:00:00+00:00", "title": "Test meeting 2", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "completed", "activity_at": "2022-11-12T18:00:00+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "connected_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "user_name": "Airbyte Team", "user_note_html": null, "date_updated": "2022-11-12T19:00:03.639000+00:00", "is_recurring": false, "contact_id": null, "attendees": [{"contact_id": null, "is_organizer": false, "email": "irina.grankova@gmail.com", "name": null, "status": "yes", "user_id": null}, {"contact_id": null, "is_organizer": true, "email": "iryna.grankova@airbyte.io", "name": null, "status": "yes", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "calendar_event_link": "https://www.google.com/calendar/event?eid=MWRlcDU1cDZtamY0MGxnYnJ2OXI5ajlocG8gaXJ5bmEuZ3JhbmtvdmFAYWlyYnl0ZS5pbw", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "location": null, "integrations": [], "created_by_name": "Airbyte Team", "source": "calendar", "note": "", "date_created": "2022-11-12T18:00:00+00:00", "calendar_event_uids": ["1dep55p6mjf40lgbrv9r9j9hpo"]}, "emitted_at": 1691417097896} +{"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Potential", "users": [], "activity_at": "2021-08-25T21:15:35.163000+00:00", "date_created": "2021-08-25T21:15:35.163000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_wSO4ltT4XHGI6wkN2oCZpRdCtw1ALhsRttt8osqVjll", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-08-25T21:15:35.163000+00:00", "old_status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099274} +{"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Qualified", "users": [], "activity_at": "2021-08-25T21:15:34.607000+00:00", "date_created": "2021-08-25T21:15:34.607000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_T4XA2LBQYvrqAimflFjU2KXY0I2g1BPbOUn2jHZ59Dj", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "date_updated": "2021-08-25T21:15:34.607000+00:00", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099279} +{"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Qualified", "users": [], "activity_at": "2021-08-25T21:15:34.044000+00:00", "date_created": "2021-08-25T21:15:34.044000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_udWvk5RJ6SFuO9Ra6d1pAJg200Q0Zyq43kEq0FpceIx", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-08-25T21:15:34.044000+00:00", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099283} +{"stream": "opportunity_status_change_activities", "data": {"old_status_label": "Demo Completed", "updated_by_name": "Airbyte Team", "created_by_name": "Airbyte Team", "contact_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-08-18T10:26:44.228000+00:00", "_type": "OpportunityStatusChange", "new_status_type": "active", "opportunity_value_period": "one_time", "users": [], "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "old_status_type": "active", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "opportunity_id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "new_pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy", "id": "acti_wYiwpVt6MU3LVce1p8JHW8TowOW8GlDikXHkIHvl5xM", "new_status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "old_pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy", "opportunity_value_formatted": "$500", "opportunity_value_currency": "USD", "old_pipeline_name": "Sales", "user_name": "Airbyte Team", "activity_at": "2021-08-18T10:26:44.228000+00:00", "date_updated": "2021-08-18T10:26:44.228000+00:00", "new_pipeline_name": "Sales", "opportunity_confidence": 75, "old_status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "opportunity_value": 50000, "new_status_label": "Proposal Sent", "opportunity_date_won": "2021-07-15T00:00:00+00:00"}, "emitted_at": 1691417101187} +{"stream": "task_completed_activities", "data": {"_type": "TaskCompleted", "id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "task_assigned_to_name": "Airbyte Team", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_created": "2021-08-18T10:31:52.002000+00:00", "users": [], "task_text": "Send Steli an email", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-08-18T10:31:52.002000+00:00", "date_updated": "2021-08-18T10:31:52.002000+00:00", "task_id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "task_assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, "emitted_at": 1691417104863} +{"stream": "task_completed_activities", "data": {"created_by_name": "Airbyte Team", "activity_at": "2022-11-09T12:44:26.185000+00:00", "date_created": "2022-11-09T12:44:26.185000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "TaskCompleted", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "task_assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "date_updated": "2022-11-09T12:44:26.185000+00:00", "task_text": "Follow up", "updated_by_name": "Airbyte Team", "task_assigned_to_name": "Airbyte Team", "task_id": "task_KAzmd4tqcQ1dYypqOjeZqmpKVlFZf3LWJfxCSoX29GY", "users": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_tEFyhPH4AKZ3YHaqqFEdm9Lot1oEI4yEyKzEf0ncbGE"}, "emitted_at": 1691417105367} +{"stream": "task_completed_activities", "data": {"created_by_name": "Airbyte Team", "activity_at": "2022-11-09T11:06:11.134000+00:00", "date_created": "2022-11-09T11:06:11.134000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "_type": "TaskCompleted", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "task_assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "date_updated": "2022-11-09T11:06:11.134000+00:00", "task_text": "Follow up", "updated_by_name": "Airbyte Team", "task_assigned_to_name": "Airbyte Team", "task_id": "task_iHHSk7GuvunATuloc0hM2k2btiuJpasL12GkwHnIVjP", "users": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_DJOFYNqKoXSmPB4yJHVRafxAqhnvk5olHmmihw4cqND"}, "emitted_at": 1691417105370} +{"stream": "lead_custom_fields", "data": {"accepts_multiple_values": false, "referenced_custom_type_id": null, "choices": null, "type": "user", "description": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:01.352000+00:00", "date_updated": "2021-07-13T11:39:01.352000+00:00", "id": "cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh", "back_reference_is_visible": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "editable_with_roles": [], "name": "Lead Owner", "is_shared": false}, "emitted_at": 1691417106647} +{"stream": "contact_custom_fields", "data": {"referenced_custom_type_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-08-11T18:05:14.598000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "name": "Birthday", "accepts_multiple_values": false, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "description": null, "choices": null, "back_reference_is_visible": null, "type": "date", "id": "cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu", "date_updated": "2021-08-11T18:05:14.598000+00:00", "is_shared": false}, "emitted_at": 1691417107525} +{"stream": "opportunity_custom_fields", "data": {"referenced_custom_type_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "description": null, "name": "Contract Link (Example)", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_pzwnr4IYZvEqq68ENLlWknEZdVsPk53rd3571eRcP56", "is_shared": false, "date_created": "2021-07-13T11:39:01.714000+00:00", "type": "text", "choices": null, "accepts_multiple_values": false, "back_reference_is_visible": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:01.714000+00:00"}, "emitted_at": 1691417108342} +{"stream": "opportunity_custom_fields", "data": {"referenced_custom_type_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "description": null, "name": "Product (Example)", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_aDbTIccjcJ9GQqM0xqA8uqgjYuEclhLMxnqfG3TAhsx", "is_shared": false, "date_created": "2021-07-13T11:39:01.447000+00:00", "type": "choices", "choices": ["Widget A", "Widget B", "Widget C"], "accepts_multiple_values": true, "back_reference_is_visible": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:01.447000+00:00"}, "emitted_at": 1691417108345} +{"stream": "opportunity_custom_fields", "data": {"referenced_custom_type_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "description": null, "name": "Renewal Date (Example)", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_Z6Vyxe2J0lrRqFpWQQGLkPo8iwhmZ9hzWc69ll2oFox", "is_shared": false, "date_created": "2021-07-13T11:39:01.676000+00:00", "type": "date", "choices": null, "accepts_multiple_values": false, "back_reference_is_visible": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:01.676000+00:00"}, "emitted_at": 1691417108348} +{"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_UftGNTS2rq9XMG9hOHkALp5cgn4Xl8nZqN7gReax3lc", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.212000+00:00", "name": "Contact", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "contact", "date_updated": "2021-07-13T11:39:02.212000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109700} +{"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_3W4n175LyZ3QMfr665jUS19nt2QATHyM7QaxXQC4QqA", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.744000+00:00", "name": "Current Vendor: Other (if applicable)", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "text", "date_updated": "2021-07-13T11:39:02.744000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109703} +{"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_bp93vNo2vxbmM2QhQ7JvdytgcozCCGmOklTG9E8rDYa", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.478000+00:00", "name": "Industry: Other (if applicable)", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "text", "date_updated": "2021-07-13T11:39:02.478000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109705} +{"stream": "users", "data": {"last_used_timezone": "America/New_York", "date_created": "2021-07-13T11:36:04.905000+00:00", "date_updated": "2023-06-29T19:39:02.749000+00:00", "first_name": "Airbyte", "last_name": "Team", "google_profile_image_url": null, "image": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef", "organizations": ["orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"], "email": "integration-test@airbyte.io", "email_verified_at": "2021-07-13T11:37:23.175000+00:00", "id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417111009} +{"stream": "contacts", "data": {"id": "cont_b4h4BcmWn7rKbnsHQ0JfADwXGgndbpc5JlQEdHHyv78", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User1"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "User1", "date_created": "2022-11-11T09:00:52.289000+00:00", "emails": [], "lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "date_updated": "2023-01-30T16:02:41.174000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [{"phone_formatted": "+1 600-000-0001", "country": null, "type": "office", "phone": "+16000000001"}], "urls": [], "title": "Product Manager", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "User1"}, "emitted_at": 1691417111908} +{"stream": "contacts", "data": {"id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User2"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "User2", "date_created": "2022-11-08T15:54:42.381000+00:00", "emails": [{"type": "office", "email": "user2.sample@gmail.com"}], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "date_updated": "2023-01-30T16:03:28.787000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [{"phone_formatted": "+1 415-623-6785", "country": "US", "type": "office", "phone": "+14156236785"}], "urls": [], "title": "Test Lead", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "User2", "custom.cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu": "2022-12-01"}, "emitted_at": 1691417111912} +{"stream": "contacts", "data": {"id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Cooper", "date_created": "2022-07-05T21:01:25.612000+00:00", "emails": [], "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "date_updated": "2022-07-05T21:01:25.612000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [], "urls": [], "title": "", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Cooper"}, "emitted_at": 1691417111916} +{"stream": "roles", "data": {"date_created": "0001-01-01T00:00:00", "date_updated": "0001-01-01T00:00:00", "editable": false, "id": "admin", "name": "Admin", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "permissions": ["bulk_delete", "bulk_edit", "bulk_email", "bulk_import", "bulk_sequence_subscriptions", "call_coach_barge", "call_coach_listen", "calling", "delete_leads", "delete_own_activities", "delete_own_opportunities", "delete_own_tasks", "export", "manage_customizations", "manage_email_sequences", "manage_group_numbers", "manage_organization", "manage_others_activities", "manage_others_opportunities", "manage_others_tasks", "manage_team_email_templates", "manage_team_smart_views", "merge_leads", "view_all_leads"], "visibility_user_lcf_behavior": null, "visibility_user_lcf_ids": []}, "emitted_at": 1691417112946} +{"stream": "roles", "data": {"date_created": "0001-01-01T00:00:00", "date_updated": "0001-01-01T00:00:00", "editable": false, "id": "restricteduser", "name": "Restricted User", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "permissions": ["delete_own_activities", "delete_own_opportunities", "delete_own_tasks", "calling", "view_all_leads"], "visibility_user_lcf_behavior": null, "visibility_user_lcf_ids": []}, "emitted_at": 1691417112949} +{"stream": "roles", "data": {"date_created": "0001-01-01T00:00:00", "date_updated": "0001-01-01T00:00:00", "editable": false, "id": "superuser", "name": "Super User", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "permissions": ["bulk_delete", "bulk_edit", "bulk_email", "bulk_import", "bulk_sequence_subscriptions", "call_coach_barge", "call_coach_listen", "calling", "delete_leads", "delete_own_activities", "delete_own_opportunities", "delete_own_tasks", "export", "manage_customizations", "manage_email_sequences", "manage_group_numbers", "manage_others_activities", "manage_others_opportunities", "manage_others_tasks", "manage_team_email_templates", "manage_team_smart_views", "merge_leads", "view_all_leads"], "visibility_user_lcf_behavior": null, "visibility_user_lcf_ids": []}, "emitted_at": 1691417112951} +{"stream": "lead_statuses", "data": {"id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Potential"}, "emitted_at": 1691417116498} +{"stream": "lead_statuses", "data": {"id": "stat_I73shDEVu3FGzxWmP7Jti3MC6aJEdYH4e036HF1bk7k", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Customer"}, "emitted_at": 1691417116501} +{"stream": "lead_statuses", "data": {"id": "stat_K2bWWhgZqujhdsO1FPg189wAapaKMVH7bi7FwPqjFfy", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Bad Fit"}, "emitted_at": 1691417116503} +{"stream": "opportunity_statuses", "data": {"id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Proposal Sent", "type": "active", "pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy"}, "emitted_at": 1691417118224} +{"stream": "opportunity_statuses", "data": {"id": "stat_CZr5826cyG8wqIg4tD6bbjaqePP4HAYOMSLgOhi1Xbf", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Lost", "type": "lost", "pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy"}, "emitted_at": 1691417118226} +{"stream": "opportunity_statuses", "data": {"id": "stat_ObYTUqjVZW0nTZXjvHhzMGyK999e42WZdIhkaNq12En", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "label": "Contract Sent", "type": "active", "pipeline_id": "pipe_0IAl41rGk9OPls9CdxFpHy"}, "emitted_at": 1691417118228} +{"stream": "pipelines", "data": {"created_by": null, "date_created": "2021-07-13T11:36:04.983404", "date_updated": "2021-07-13T11:36:04.983404", "id": "pipe_0IAl41rGk9OPls9CdxFpHy", "name": "Sales", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "statuses": [{"id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "label": "Demo Completed", "type": "active"}, {"id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "label": "Proposal Sent", "type": "active"}, {"id": "stat_ObYTUqjVZW0nTZXjvHhzMGyK999e42WZdIhkaNq12En", "label": "Contract Sent", "type": "active"}, {"id": "stat_gaqEGSVHIFzrofTfzzg5UfjyBZ1B6KERccIy2MOp8FG", "label": "Won", "type": "won"}, {"id": "stat_CZr5826cyG8wqIg4tD6bbjaqePP4HAYOMSLgOhi1Xbf", "label": "Lost", "type": "lost"}], "updated_by": null}, "emitted_at": 1691417119534} +{"stream": "email_templates", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attachments": [], "body": "Hi {{ contact.first_name }},

    I'm {{ user.first_name }} with {{ organization.name }}. We help companies in the {{ lead.custom.[\"Industry\"] }} space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at {{ lead.display_name }} and show you what we're working on.

    Are you available for a quick call tomorrow afternoon?
    ", "name": "Email 1 - Intro", "id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "is_shared": true, "is_archived": false, "date_created": "2021-07-13T11:39:00.497000+00:00", "subject": "{{ lead.display_name }} + {{ organization.name }}", "date_updated": "2022-08-11T18:11:08.966000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417120444} +{"stream": "email_templates", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attachments": [], "body": "Hi {{ contact.first_name }},

    Friendly follow-up.

    I wanted to show you how {{ organization.name }} can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

    - Wed @ 11AM
    - Thur @ 2PM
    - Fri @ 3PM
    ", "name": "Email 2 - Follow-up #1", "id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "is_shared": true, "is_archived": false, "date_created": "2021-07-13T11:39:00.503000+00:00", "subject": "{{ organization.name }} Follow-up", "date_updated": "2022-08-11T18:11:08.972000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417120447} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "next_billing_on"], "data": {"address_id": null, "bundle_id": null, "carrier": "twilio", "country": "US", "date_created": "2022-11-08T12:35:29.464000+00:00", "date_updated": "2023-08-07T10:04:12.414000+00:00", "forward_to": null, "forward_to_enabled": false, "forward_to_formatted": null, "id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "is_group_number": false, "is_verified": false, "label": null, "last_billed_price": "1.15", "mms_enabled": true, "next_billing_on": "2023-09-07", "number": "+14156251293", "number_formatted": "+1 415-625-1293", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "press_1_to_accept": true, "sms_enabled": true, "supports_mms_to_countries": ["CA", "US"], "supports_sms_to_countries": ["CA", "PR", "US"], "type": "internal", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "voicemail_greeting_url": "https://closeio-voicemail-greetings.s3.amazonaws.com/14bzVblrIaekmXSGQEKM11/undefined.mp3"}, "date_created": "2023-08-07T10:04:12.416000", "date_updated": "2023-08-07T10:04:12.416000", "id": "ev_3o6F0cl3A3CkYyoc9vrVjM", "lead_id": null, "meta": {}, "oauth_client_id": null, "oauth_scope": null, "object_id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "object_type": "phone_number", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-07-07T10:03:30.945000+00:00", "next_billing_on": "2023-08-07"}, "request_id": null, "user_id": null}, "emitted_at": 1691417122379} +{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Potential", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Cooper", "phones": [], "urls": [], "display_name": "Cooper", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-07-05T21:01:25.612000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [], "title": "", "date_updated": "2022-07-05T21:01:25.612000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY"}], "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "addresses": [], "date_created": "2022-07-05T21:01:25.608000+00:00", "custom": {}, "html_url": "https://app.close.com/lead/lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY/", "date_updated": "2022-07-05T21:01:25.658000+00:00", "tasks": [], "name": "Alex", "created_by_name": "Airbyte Team", "display_name": "Alex", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Alex"}], "updated_by_name": "Airbyte Team", "description": "", "url": null, "opportunities": []}, "emitted_at": 1691417125238} +{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Interested", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Gob Bluth", "phones": [{"phone": "+12025550186", "phone_formatted": "+1 202-555-0186", "country": "US", "type": "office"}], "urls": [], "display_name": "Gob Bluth", "id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:04.430000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "bluth@close.com", "type": "office"}], "title": "Magician", "date_updated": "2021-07-13T11:39:04.430000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Gob%20Bluth"}], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Tobias F\u00fcnke", "phones": [], "urls": [], "display_name": "Tobias F\u00fcnke", "id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:04.441000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "tobiasfunke@close.com", "type": "office"}], "title": "Blue Man Group (Understudy)", "date_updated": "2021-07-13T11:39:04.441000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Tobias%20F%C3%BCnke"}], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Point of Contact"]}], "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "addresses": [{"address_1": "100 Bluth Drive", "label": "business", "country": "US", "city": "Los Angeles", "zipcode": "90210", "state": "CA", "address_2": null}], "date_created": "2021-07-05T11:39:00.850000+00:00", "custom": {"Current Vendor/Software": "BiffCo", "Industry": "Real estate", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Google Search"}, "html_url": "https://app.close.com/lead/lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2/", "date_updated": "2022-11-08T16:28:54.174000+00:00", "tasks": [], "name": "Bluth Company (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "display_name": "Bluth Company (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Bluth%20Company%20%28Example%C2%A0Lead%29"}], "updated_by_name": "Airbyte Team", "description": null, "url": null, "opportunities": [{"annualized_expected_value": 225000, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "status_type": "active", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "date_created": "2021-07-13T11:39:04.817000+00:00", "expected_value": 225000, "lead_name": "Bluth Company (Example\u00a0Lead)", "date_lost": null, "date_updated": "2021-07-13T11:39:04.817000+00:00", "note": "Gob's ready to buy a $3,000 suit.", "value_formatted": "$3,000", "status_label": "Demo Completed", "contact_name": null, "created_by_name": "Airbyte Team", "confidence": 75, "contact_id": null, "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "value": 300000, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_won": "2021-07-16", "value_currency": "USD", "updated_by_name": "Airbyte Team", "annualized_value": 300000, "integration_links": [], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2"}], "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "BiffCo", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Real estate", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Google Search"}, "emitted_at": 1691417125245} +{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Interested", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Steli Efti", "phones": [{"phone": "+16505176539", "phone_formatted": "+1 650-517-6539", "country": "US", "type": "office"}], "urls": [], "display_name": "Steli Efti", "id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.354000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "sales@close.com", "type": "office"}], "title": "CEO & Co-Founder", "date_updated": "2021-07-13T11:39:03.354000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Steli%20Efti"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Nick Persico", "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "country": "US", "type": "office"}], "urls": [], "display_name": "Nick Persico", "id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.366000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "nick@close.com", "type": "office"}], "title": "Director of Revenue", "date_updated": "2021-07-13T11:39:03.366000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Nick%20Persico"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper", "Point of Contact"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Customer Success Team", "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "country": "US", "type": "office"}], "urls": [], "display_name": "Customer Success Team", "id": "cont_4cmimyQMTMi61kc72mLAV9XdHdddw6LR1LqzvoNdSuV", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.374000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "success@close.com", "type": "office"}], "title": null, "date_updated": "2021-07-13T11:39:03.374000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Customer%20Success%20Team"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Support", "phones": [], "urls": [], "display_name": "Support", "id": "cont_CI5c6Ekew0cyhSgoFIXeaz2PJVmmtigF2XNIGUDXKnu", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.380000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "support@close.com", "type": "office"}], "title": null, "date_updated": "2021-07-13T11:39:03.380000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Support"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "addresses": [{"address_1": "PO Box 7775 #69574", "label": "mailing", "country": "US", "city": "San Francisco", "zipcode": "94120", "state": "CA", "address_2": null}], "date_created": "2021-07-13T11:39:03.315000+00:00", "custom": {"Current Vendor/Software": "Stark Industries", "Industry": "Software", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Website"}, "html_url": "https://app.close.com/lead/lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz/", "date_updated": "2022-11-09T02:40:26.489000+00:00", "tasks": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "view": "archive", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "date_created": "2021-07-13T11:39:03.520000+00:00", "lead_name": "Close (Example\u00a0Lead)", "is_dateless": null, "date_updated": "2022-11-08T12:21:15.730000+00:00", "text": "Call Steli", "is_complete": true, "contact_name": null, "created_by_name": "Airbyte Team", "date": "2021-07-18", "contact_id": null, "due_date": "2021-07-18", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "_type": "lead", "updated_by_name": "Airbyte Team", "object_type": "taskcompleted", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "view": "archive", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "date_created": "2021-07-13T11:39:03.463000+00:00", "lead_name": "Close (Example\u00a0Lead)", "is_dateless": null, "date_updated": "2021-08-18T10:31:52.081000+00:00", "text": "Send Steli an email", "is_complete": true, "contact_name": null, "created_by_name": "Airbyte Team", "date": "2021-07-16", "contact_id": null, "due_date": "2021-07-16", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "_type": "lead", "updated_by_name": "Airbyte Team", "object_type": "taskcompleted", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "name": "Close (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "display_name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Close%20%28Example%C2%A0Lead%29"}], "updated_by_name": "Airbyte Team", "description": "Visit our blog for high quality sales content, blog.close.com!", "url": "https://close.com", "opportunities": [{"annualized_expected_value": 37500, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Proposal Sent", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "status_type": "active", "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "date_created": "2021-07-13T11:39:04.284000+00:00", "expected_value": 37500, "lead_name": "Close (Example\u00a0Lead)", "date_lost": null, "date_updated": "2021-08-18T10:26:44.306000+00:00", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "value_formatted": "$500", "status_label": "Proposal Sent", "contact_name": "Steli Efti", "created_by_name": "Airbyte Team", "confidence": 75, "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "value": 50000, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_won": "2021-07-15", "value_currency": "USD", "updated_by_name": "Airbyte Team", "annualized_value": 50000, "integration_links": [], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Stark Industries", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Software", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Website"}, "emitted_at": 1691417125251} +{"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 300000, "status_display_name": "Demo Completed", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Demo Completed", "value_currency": "USD", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "lead_name": "Bluth Company (Example\u00a0Lead)", "date_updated": "2021-07-13T11:39:04.817000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "expected_value": 225000, "date_created": "2021-07-13T11:39:04.817000+00:00", "date_won": "2021-07-16", "note": "Gob's ready to buy a $3,000 suit.", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "contact_id": null, "contact_name": null, "annualized_expected_value": 225000, "confidence": 75, "annualized_value": 300000, "updated_by_name": "Airbyte Team", "value_formatted": "$3,000", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127685} +{"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 50000, "status_display_name": "Demo Completed", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Demo Completed", "value_currency": "USD", "id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "date_updated": "2021-07-13T11:39:05.061000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "expected_value": 37500, "date_created": "2021-07-13T11:39:05.061000+00:00", "date_won": "2021-07-15", "note": "Bruce needs new software for the Bat Cave.", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "contact_id": null, "contact_name": null, "annualized_expected_value": 37500, "confidence": 75, "annualized_value": 50000, "updated_by_name": "Airbyte Team", "value_formatted": "$500", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127689} +{"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 50000, "status_display_name": "Proposal Sent", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Proposal Sent", "value_currency": "USD", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "lead_name": "Close (Example\u00a0Lead)", "date_updated": "2021-08-18T10:26:44.306000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "expected_value": 37500, "date_created": "2021-07-13T11:39:04.284000+00:00", "date_won": "2021-07-15", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "contact_name": "Steli Efti", "annualized_expected_value": 37500, "confidence": 75, "annualized_value": 50000, "updated_by_name": "Airbyte Team", "value_formatted": "$500", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127692} +{"stream": "lead_tasks", "data": {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "is_complete": true, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "view": "archive", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "_type": "lead", "assigned_to_name": "Airbyte Team", "contact_name": null, "created_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "is_dateless": null, "due_date": "2021-07-16", "contact_id": null, "date_updated": "2021-08-18T10:31:52.081000+00:00", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "lead_name": "Close (Example\u00a0Lead)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "object_type": "taskcompleted", "date": "2021-07-16", "date_created": "2021-07-13T11:39:03.463000+00:00", "text": "Send Steli an email", "updated_by_name": "Airbyte Team"}, "emitted_at": 1691417130629} +{"stream": "lead_tasks", "data": {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "is_complete": true, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "view": "archive", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "_type": "lead", "assigned_to_name": "Airbyte Team", "contact_name": null, "created_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "is_dateless": null, "due_date": "2021-07-18", "contact_id": null, "date_updated": "2022-11-08T12:21:15.730000+00:00", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "lead_name": "Close (Example\u00a0Lead)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "object_type": "taskcompleted", "date": "2021-07-18", "date_created": "2021-07-13T11:39:03.520000+00:00", "text": "Call Steli", "updated_by_name": "Airbyte Team"}, "emitted_at": 1691417130633} +{"stream": "lead_tasks", "data": {"lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "id": "task_Wgzt6t0ZGRlvZTnfhSfxHoFRWm6zXBMuuecKxRkfYmZ", "contact_name": null, "is_dateless": false, "date": "2022-11-12", "text": "Follow up", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "_type": "lead", "assigned_to_name": "Airbyte Team", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_type": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2022-11-09T11:06:06.145000+00:00", "created_by_name": "Airbyte Team", "object_id": null, "date_created": "2022-11-08T15:10:57.220000+00:00", "updated_by_name": "Airbyte Team", "view": "inbox", "due_date": "2022-11-12", "is_complete": false}, "emitted_at": 1691417131138} +{"stream": "email_followup_tasks", "data": {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "total_emails_count_in_thread": 4, "object_type": "emailthread", "date_updated": "2022-11-08T16:28:54.794000+00:00", "created_by_name": "Airbyte Team", "object_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-11-08T15:44:05.221000+00:00", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "_type": "email_followup", "id": "noti_3I4QaKqYel0nzGyKT8mpDKy4q2KKu4IuIZZ3Gb7gs4x", "is_complete": false, "body_preview": "Test", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "contact_name": "Bruce Wayne", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "email_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "subject": "(no subject)", "view": "inbox", "date": "2022-11-15", "assigned_to_name": "Airbyte Team"}, "emitted_at": 1691417136079} +{"stream": "email_followup_tasks", "data": {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "total_emails_count_in_thread": 1, "object_type": "emailthread", "date_updated": "2022-11-09T12:54:10.330000+00:00", "created_by_name": "Airbyte Team", "object_id": "acti_fFBPZHGPKt60kZG2SzI5ws5OMbysXqCAGJuGlKucDy5", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-11-09T12:54:10.330000+00:00", "lead_name": "Test Lead", "_type": "email_followup", "id": "noti_JU7vkNkhpywznaGu5P0WuBWxsMXtzZmkOSXaCxbxTsN", "is_complete": false, "body_preview": "Hi Iryna, \n \nI'm Jean with Airbyte. We help companies in the space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Test Lead\nand show you what we're working on. \n ", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "contact_name": "User2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "email_id": "acti_a3AsG3P38a8yDx0XgI8ZNZR4Ko8XRUBybauASuT8C2j", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "subject": "Test Lead + Airbyte", "view": "inbox", "date": "2022-11-16", "assigned_to_name": "Airbyte Team"}, "emitted_at": 1691417136083} +{"stream": "opportunity_due_tasks", "data": {"updated_by_name": "Airbyte Team", "contact_name": "Steli Efti", "created_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "opportunity_value_formatted": "$500", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "date_updated": "2021-08-18T10:26:44.323000+00:00", "_type": "opportunity_due", "lead_name": "Close (Example\u00a0Lead)", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date": "2021-07-15", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "noti_ELGehUlDZ1pmlIeibPlLbTyruBTKdACAQ1q1XLiWo6B", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "assigned_to_name": "Airbyte Team", "view": "inbox", "opportunity_value_period": "one_time", "opportunity_note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "date_created": "2021-07-13T11:39:04.316000+00:00", "object_id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "is_complete": false, "opportunity_value": 50000, "object_type": "opportunity"}, "emitted_at": 1691417146979} +{"stream": "opportunity_due_tasks", "data": {"updated_by_name": "Airbyte Team", "contact_name": null, "created_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "opportunity_value_formatted": "$3,000", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "date_updated": "2021-07-13T11:39:04.839000+00:00", "_type": "opportunity_due", "lead_name": "Bluth Company (Example\u00a0Lead)", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date": "2021-07-16", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "noti_wcbYMl0dpbdcypRygf9jYd9axfIzvSsj06gXHmPmE4k", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "assigned_to_name": "Airbyte Team", "view": "inbox", "opportunity_value_period": "one_time", "opportunity_note": "Gob's ready to buy a $3,000 suit.", "date_created": "2021-07-13T11:39:04.839000+00:00", "object_id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "is_complete": false, "opportunity_value": 300000, "object_type": "opportunity"}, "emitted_at": 1691417146983} +{"stream": "opportunity_due_tasks", "data": {"updated_by_name": "Airbyte Team", "contact_name": null, "created_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "opportunity_value_formatted": "$500", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "date_updated": "2021-07-13T11:39:05.083000+00:00", "_type": "opportunity_due", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date": "2021-07-15", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "noti_IPi8EZj1AtqSwbnSMg5OT2DpUqNHCl2RM7luvbx9zsp", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "assigned_to_name": "Airbyte Team", "view": "inbox", "opportunity_value_period": "one_time", "opportunity_note": "Bruce needs new software for the Bat Cave.", "date_created": "2021-07-13T11:39:05.083000+00:00", "object_id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "is_complete": false, "opportunity_value": 50000, "object_type": "opportunity"}, "emitted_at": 1691417146987} +{"stream": "google_connected_accounts", "data": {"latest_receive_error": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "available_calendars": [{"id": "iryna.grankova@airbyte.io", "name": "iryna.grankova@airbyte.io", "status": "ok", "default": true, "synced": true}, {"id": "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com", "name": "Close Tasks: Airbyte Team", "status": "ok", "default": false, "synced": true}], "email": "iryna.grankova@airbyte.io", "date_created": "2022-11-08T12:13:40.319000+00:00", "receive_status": "ok", "available_features": ["calendar_syncing", "email_sending", "email_syncing"], "calendar_receive_status": "ok", "latest_send_error": null, "sync_all_calendars": false, "is_imap_archive_sync_enabled": true, "_type": "google", "send_status": "ok", "receive_attempts_count": 0, "smtp": {"username": "iryna.grankova@airbyte.io", "host": "smtp.gmail.com", "date_updated": "2022-11-08T12:13:40.319000+00:00", "port": 465, "use_ssl": true}, "imap": {"username": "iryna.grankova@airbyte.io", "host": "imap.gmail.com", "date_updated": "2022-11-08T12:13:40.319000+00:00", "port": 993, "use_ssl": true}, "id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "synced_calendars": ["iryna.grankova@airbyte.io", "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com"], "identities": [{"name": "Jean Lafleur", "email": "iryna.grankova@airbyte.io"}], "lead_suggestions_updated_at": "2022-11-08T12:13:45.711000+00:00", "default_identity": {"name": "Jean Lafleur", "email": "iryna.grankova@airbyte.io"}, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "enabled_features": ["calendar_syncing", "email_sending", "email_syncing"], "email_receive_status": "ok"}, "emitted_at": 1691417152839} +{"stream": "google_connected_accounts", "data": {"available_features": ["email_sending", "email_syncing"], "date_created": "2022-11-08T14:43:13.927000+00:00", "email": "integration-test@airbyte.io", "enabled_features": ["email_sending", "email_syncing"], "_type": "custom_email", "calendar_receive_status": null, "default_identity": {"name": "Jean Lafleur", "email": "integration-test@airbyte.io"}, "email_receive_status": "error", "latest_receive_error": null, "latest_send_error": null, "receive_status": "error", "send_status": "ok", "id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "identities": [{"name": "Jean Lafleur", "email": "integration-test@airbyte.io"}], "imap": {"username": "iryna.grankova@globallogic.com", "host": "imap.gmail.com", "date_updated": "2022-11-08T14:43:13.927000+00:00", "port": 993, "use_ssl": true}, "is_imap_archive_sync_enabled": true, "lead_suggestions_updated_at": "2022-11-08T14:43:17.844000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "receive_attempts_count": 0, "smtp": {"username": "iryna.grankova@globallogic.com", "host": "smtp.gmail.com", "date_updated": "2022-11-08T14:43:13.927000+00:00", "port": 465, "use_ssl": true}, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417152842} +{"stream": "google_connected_accounts", "data": {"latest_receive_error": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "email": "integration-test@airbyte.io", "date_created": "2022-11-08T13:13:08.795000+00:00", "receive_status": null, "available_features": [], "calendar_receive_status": null, "latest_send_error": null, "is_imap_archive_sync_enabled": true, "_type": "zoom", "zoom_account_plan": null, "send_status": null, "receive_attempts_count": 0, "smtp": null, "imap": null, "id": "emailacct_pSuPSwRBjmzoTYudYJRn9ENpaR3RAaRp7K2jJ8syPsp", "identities": [], "lead_suggestions_updated_at": null, "default_identity": {"name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "enabled_features": [], "email_receive_status": null}, "emitted_at": 1691417152844} +{"stream": "custom_email_connected_accounts", "data": {"imap": {"use_ssl": true, "username": "iryna.grankova@airbyte.io", "port": 993, "host": "imap.gmail.com", "date_updated": "2022-11-08T12:13:40.319000+00:00"}, "receive_attempts_count": 0, "_type": "google", "smtp": {"use_ssl": true, "username": "iryna.grankova@airbyte.io", "port": 465, "host": "smtp.gmail.com", "date_updated": "2022-11-08T12:13:40.319000+00:00"}, "is_imap_archive_sync_enabled": true, "email_receive_status": "ok", "synced_calendars": ["iryna.grankova@airbyte.io", "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com"], "available_features": ["calendar_syncing", "email_sending", "email_syncing"], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "email": "iryna.grankova@airbyte.io", "available_calendars": [{"id": "iryna.grankova@airbyte.io", "name": "iryna.grankova@airbyte.io", "status": "ok", "default": true, "synced": true}, {"id": "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com", "name": "Close Tasks: Airbyte Team", "status": "ok", "default": false, "synced": true}], "send_status": "ok", "default_identity": {"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}, "date_created": "2022-11-08T12:13:40.319000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "latest_send_error": null, "sync_all_calendars": false, "id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "calendar_receive_status": "ok", "lead_suggestions_updated_at": "2022-11-08T12:13:45.711000+00:00", "latest_receive_error": null, "enabled_features": ["calendar_syncing", "email_sending", "email_syncing"], "receive_status": "ok", "identities": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}]}, "emitted_at": 1691417153709} +{"stream": "custom_email_connected_accounts", "data": {"available_features": ["email_sending", "email_syncing"], "date_created": "2022-11-08T14:43:13.927000+00:00", "email": "integration-test@airbyte.io", "enabled_features": ["email_sending", "email_syncing"], "_type": "custom_email", "calendar_receive_status": null, "default_identity": {"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}, "email_receive_status": "error", "latest_receive_error": null, "latest_send_error": null, "receive_status": "error", "send_status": "ok", "id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "identities": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "imap": {"use_ssl": true, "username": "iryna.grankova@globallogic.com", "port": 993, "host": "imap.gmail.com", "date_updated": "2022-11-08T14:43:13.927000+00:00"}, "is_imap_archive_sync_enabled": true, "lead_suggestions_updated_at": "2022-11-08T14:43:17.844000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "receive_attempts_count": 0, "smtp": {"use_ssl": true, "username": "iryna.grankova@globallogic.com", "port": 465, "host": "smtp.gmail.com", "date_updated": "2022-11-08T14:43:13.927000+00:00"}, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417153712} +{"stream": "custom_email_connected_accounts", "data": {"imap": null, "receive_attempts_count": 0, "_type": "zoom", "smtp": null, "is_imap_archive_sync_enabled": true, "email_receive_status": null, "available_features": [], "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "email": "integration-test@airbyte.io", "send_status": null, "default_identity": {"email": "integration-test@airbyte.io", "name": "Airbyte Team"}, "date_created": "2022-11-08T13:13:08.795000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "latest_send_error": null, "zoom_account_plan": null, "id": "emailacct_pSuPSwRBjmzoTYudYJRn9ENpaR3RAaRp7K2jJ8syPsp", "calendar_receive_status": null, "lead_suggestions_updated_at": null, "latest_receive_error": null, "enabled_features": [], "receive_status": null, "identities": []}, "emitted_at": 1691417153714} +{"stream": "zoom_connected_accounts", "data": {"sync_all_calendars": false, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "default_identity": {"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}, "receive_status": "ok", "imap": {"date_updated": "2022-11-08T12:13:40.319000+00:00", "username": "iryna.grankova@airbyte.io", "host": "imap.gmail.com", "use_ssl": true, "port": 993}, "smtp": {"date_updated": "2022-11-08T12:13:40.319000+00:00", "username": "iryna.grankova@airbyte.io", "host": "smtp.gmail.com", "use_ssl": true, "port": 465}, "calendar_receive_status": "ok", "email_receive_status": "ok", "is_imap_archive_sync_enabled": true, "latest_send_error": null, "available_calendars": [{"id": "iryna.grankova@airbyte.io", "name": "iryna.grankova@airbyte.io", "status": "ok", "default": true, "synced": true}, {"id": "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com", "name": "Close Tasks: Airbyte Team", "status": "ok", "default": false, "synced": true}], "synced_calendars": ["iryna.grankova@airbyte.io", "jf4v33abpv80jp1r8vhdfjonm0bj0sk9@import.calendar.google.com"], "identities": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "send_status": "ok", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "enabled_features": ["calendar_syncing", "email_sending", "email_syncing"], "_type": "google", "lead_suggestions_updated_at": "2022-11-08T12:13:45.711000+00:00", "email": "iryna.grankova@airbyte.io", "id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "date_created": "2022-11-08T12:13:40.319000+00:00", "latest_receive_error": null, "receive_attempts_count": 0, "available_features": ["calendar_syncing", "email_sending", "email_syncing"]}, "emitted_at": 1691417154591} +{"stream": "zoom_connected_accounts", "data": {"available_features": ["email_sending", "email_syncing"], "date_created": "2022-11-08T14:43:13.927000+00:00", "email": "integration-test@airbyte.io", "enabled_features": ["email_sending", "email_syncing"], "_type": "custom_email", "calendar_receive_status": null, "default_identity": {"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}, "email_receive_status": "error", "latest_receive_error": null, "latest_send_error": null, "receive_status": "error", "send_status": "ok", "id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "identities": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "imap": {"date_updated": "2022-11-08T14:43:13.927000+00:00", "username": "iryna.grankova@globallogic.com", "host": "imap.gmail.com", "use_ssl": true, "port": 993}, "is_imap_archive_sync_enabled": true, "lead_suggestions_updated_at": "2022-11-08T14:43:17.844000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "receive_attempts_count": 0, "smtp": {"date_updated": "2022-11-08T14:43:13.927000+00:00", "username": "iryna.grankova@globallogic.com", "host": "smtp.gmail.com", "use_ssl": true, "port": 465}, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417154593} +{"stream": "zoom_connected_accounts", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "default_identity": {"email": "integration-test@airbyte.io", "name": "Airbyte Team"}, "receive_status": null, "imap": null, "smtp": null, "calendar_receive_status": null, "email_receive_status": null, "is_imap_archive_sync_enabled": true, "latest_send_error": null, "identities": [], "send_status": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "enabled_features": [], "_type": "zoom", "lead_suggestions_updated_at": null, "email": "integration-test@airbyte.io", "id": "emailacct_pSuPSwRBjmzoTYudYJRn9ENpaR3RAaRp7K2jJ8syPsp", "date_created": "2022-11-08T13:13:08.795000+00:00", "latest_receive_error": null, "receive_attempts_count": 0, "available_features": [], "zoom_account_plan": null}, "emitted_at": 1691417154596} +{"stream": "email_sequences", "data": {"allow_manual_enrollment": true, "created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:06.184401", "date_updated": "2022-11-08T12:07:06.184405", "id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "name": "Sequence 1", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "schedule_id": "sched_1eL83fHJodv0OtoibuJFAw", "status": "active", "steps": [{"created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:06.191662", "date_updated": "2022-11-08T12:07:06.191665", "delay": 0, "email_template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "id": "seqstep_5lFdCIwj4qVpdVMH3K26iQ", "required": true, "sms_template_id": null, "step_allowed_delay": null, "step_type": "email", "threading": "old_thread", "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, {"created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:06.191692", "date_updated": "2022-11-08T12:07:06.191693", "delay": 259200, "email_template_id": null, "id": "seqstep_4YVJ2uoVccNHYG95qvwDQw", "required": false, "sms_template_id": null, "step_allowed_delay": 3600, "step_type": "call", "threading": null, "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "timezone": "Europe/Kiev", "trigger_query": null, "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417156273} +{"stream": "email_sequences", "data": {"allow_manual_enrollment": true, "created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:48.571475", "date_updated": "2022-11-08T12:07:48.571481", "id": "seq_7gEZ4ByvvLv1rI6szKolhR", "name": "Sequence 2", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "schedule_id": "sched_3MvlPuSqL0NGJyY95MBMpF", "status": "active", "steps": [{"created_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:07:48.623996", "date_updated": "2022-11-08T12:07:48.624000", "delay": 3600, "email_template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "id": "seqstep_71ELVa12hUsLg2ba3K74Yq", "required": true, "sms_template_id": null, "step_allowed_delay": null, "step_type": "email", "threading": "old_thread", "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "timezone": "Europe/Kiev", "trigger_query": null, "updated_by_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417156276} +{"stream": "dialer", "data": {"caller_id": "", "date_created": "2022-11-08T14:27:23.185019", "date_updated": "2022-11-08T14:27:23.309015", "id": "dial_5qE6KYskn4PfT6oQLTe8ez", "is_user_dependent": false, "music_preference": "tone1", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "ring_mode": "more-voicemails", "source_type": "saved-search", "source_value": "save_vL2ENUTpFhqrYHf97tm61Y9Sh6QughnuMURuIH4VDCC", "status": "inactive", "target_type": "lead", "type": "power", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "users": [{"is_active": false, "state": "inactive", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "visibility_mode": "all"}, "emitted_at": 1691417157213} +{"stream": "smart_views", "data": {"date_created": "2022-11-08T15:50:57.032000+00:00", "date_updated": "2022-11-08T15:50:57.032000+00:00", "id": "save_UeQpNDWG0AuXKKunEIABD0qSjHvV62EFaTHzN70B6gN", "is_shared": false, "is_user_dependent": false, "name": "Deleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": null, "s_query": {"query": {"negate": false, "queries": [{"negate": false, "object_type": "lead", "type": "object_type"}, {"negate": false, "queries": [{"negate": false, "queries": [{"condition": {"before": {"range": "today", "type": "start_end_of_predefined_relative_period", "which": "end"}, "on_or_after": {"range": "today", "type": "start_end_of_predefined_relative_period", "which": "start"}, "type": "moment_range"}, "field": {"field_name": "date_created", "object_type": "lead", "type": "regular_field"}, "negate": false, "type": "field_condition"}], "type": "and"}], "type": "and"}], "type": "and"}, "results_limit": null, "sort": []}, "type": "lead", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417158090} +{"stream": "smart_views", "data": {"date_created": "2021-07-13T11:39:00.802000+00:00", "date_updated": "2021-07-13T11:39:00.802000+00:00", "id": "save_vL2ENUTpFhqrYHf97tm61Y9Sh6QughnuMURuIH4VDCC", "is_shared": true, "is_user_dependent": false, "name": "\ud83d\udce3 Untouched Leads", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "times_communicated:0 notes:0 tasks:0 opportunities:0", "s_query": null, "type": "lead", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417158092} +{"stream": "smart_views", "data": {"date_created": "2021-07-13T11:39:00.745000+00:00", "date_updated": "2021-07-13T11:39:00.745000+00:00", "id": "save_dnBmv1ixCRWNZ6P13oydsxGPLLxs94SWGzvBZIEqo3x", "is_shared": true, "is_user_dependent": false, "name": "\u260e\ufe0f Leads to Call", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "has:phone_numbers opportunities:0 calls:0 tasks:0", "s_query": null, "type": "lead", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417158095} +{"stream": "email_bulk_actions", "data": {"id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "date_created": "2022-11-08T12:58:33.566000+00:00", "date_updated": "2022-11-08T12:58:47.020000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "has:active_opportunities last_communication_date > \"7 days ago\"", "s_query": {"negate": false, "type": "match_all"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 1, "n_objects_processed": 1, "n_leads": 1, "n_leads_processed": 1, "send_done_email": true, "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "contact_preference": "lead", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "sender": "Jean Lafleur "}, "emitted_at": 1691417159433} +{"stream": "sequence_subscription_bulk_actions", "data": {"calls_assigned_to": [], "n_leads_processed": 11, "query": "", "date_updated": "2022-11-08T15:10:17.831000+00:00", "s_query": {"negate": false, "queries": [{"negate": false, "object_type": "contact", "type": "object_type"}, {"negate": false, "queries": [], "type": "and"}], "type": "and"}, "sort": [], "n_objects_processed": 11, "sender_email": "integration-test@airbyte.io", "results_limit": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sender_name": "Jean Lafleur", "id": "bulkseqsub_nB54SIHOmtafCyAri59aJc24OfcCWudvQFtIW6k7oJv", "n_objects": 11, "sender_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "from_phone_number_id": null, "n_leads": 11, "action_type": "subscribe", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "done", "contact_preference": "contact", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-11-08T15:09:57.014000+00:00", "send_done_email": true}, "emitted_at": 1691417160379} +{"stream": "sequence_subscription_bulk_actions", "data": {"calls_assigned_to": ["group_2WQ2sEM5fBJ2Ip0lWMyRLg"], "n_leads_processed": 11, "query": "", "date_updated": "2022-11-08T15:09:55.730000+00:00", "s_query": {"negate": false, "queries": [{"negate": false, "object_type": "contact", "type": "object_type"}, {"negate": false, "queries": [], "type": "and"}], "type": "and"}, "sort": [], "n_objects_processed": 11, "sender_email": "iryna.grankova@airbyte.io", "results_limit": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sender_name": "Jean Lafleur", "id": "bulkseqsub_YrswODZaYxUAhKYKcOcIQCWRxzbUjlsIgno8viR55dw", "n_objects": 11, "sender_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "from_phone_number_id": null, "n_leads": 11, "action_type": "subscribe", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "done", "contact_preference": "contact", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-11-08T15:09:38.582000+00:00", "send_done_email": true}, "emitted_at": 1691417160382} +{"stream": "delete_bulk_actions", "data": {"id": "bulkdelete_jfj3OxS8q9ri6uDwvr09Gwxl9Q6Oh3tgJ4cUPAaZq52", "date_created": "2022-11-08T15:49:38.627000+00:00", "date_updated": "2022-11-08T15:49:44.832000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "", "s_query": {"negate": false, "queries": [{"negate": false, "object_type": "lead", "type": "object_type"}, {"negate": false, "queries": [{"negate": false, "queries": [{"condition": {"before": {"range": "today", "which": "end", "type": "start_end_of_predefined_relative_period"}, "on_or_after": {"range": "today", "which": "start", "type": "start_end_of_predefined_relative_period"}, "type": "moment_range"}, "field": {"field_name": "date_created", "object_type": "lead", "type": "regular_field"}, "negate": false, "type": "field_condition"}], "type": "and"}], "type": "and"}], "type": "and"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 3, "n_objects_processed": 3, "n_leads": 3, "n_leads_processed": 3, "send_done_email": true, "bulk_object_type": "lead"}, "emitted_at": 1691417161239} +{"stream": "edit_bulk_actions", "data": {"id": "bulkedit_GMBUd9rUAIfYnKXokMKXV1AHMQUm9yCMGXhh96xYlxr", "date_created": "2021-08-25T21:15:27.477000+00:00", "date_updated": "2021-08-25T21:15:35.868000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "*", "s_query": {"negate": false, "type": "match_all"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 3, "n_objects_processed": 3, "n_leads": 3, "n_leads_processed": 3, "send_done_email": true, "type": "set_lead_status", "custom_field_name": null, "custom_field_value": null, "lead_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q"}, "emitted_at": 1691417162306} +{"stream": "edit_bulk_actions", "data": {"id": "bulkedit_mA19Kcb30egVp18LcMXP2OCTQaLsGJyWikJMXYu32kv", "date_created": "2021-08-25T21:03:43.590000+00:00", "date_updated": "2021-08-25T21:03:54.810000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "query": "*", "s_query": {"negate": false, "type": "match_all"}, "sort": [], "results_limit": null, "status": "done", "n_objects": 3, "n_objects_processed": 3, "n_leads": 3, "n_leads_processed": 3, "send_done_email": true, "type": "set_custom_field", "custom_field_name": "Lead Owner", "custom_field_value": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "lead_status_id": null}, "emitted_at": 1691417162309} +{"stream": "integration_links", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Google Search", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "url": "https://google.com/search?q={{ lead.display_name }}", "date_created": "2021-07-13T11:39:00.832000+00:00", "id": "ilink_CUrmAjZi7dy3T5nodx6INqBAYMIme3HQ9In2Zm4PHI4", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2022-08-11T18:11:08.979000+00:00", "type": "lead"}, "emitted_at": 1691417163428} +{"stream": "integration_links", "data": {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "LinkedIn Search", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "url": "https://www.linkedin.com/search/results/people/?keywords={{ contact.name }}", "date_created": "2021-07-13T11:39:00.840000+00:00", "id": "ilink_UFZJXoAlAiDsIbBMKPaK87FDp0hgjduPjRMcdjomGzS", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2022-08-11T18:11:08.993000+00:00", "type": "contact"}, "emitted_at": 1691417163431} +{"stream": "custom_activities", "data": {"api_create_only": false, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T12:03:27.631229", "date_updated": "2022-11-08T12:03:27.631229", "description": null, "editable_with_roles": [], "fields": [{"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": "Test description", "editable_with_roles": [], "id": "cf_KLkKl1BRUm4o7zknJtTkvp6TMFgff1984UelnlGN9mZ", "is_shared": false, "name": "Test 1", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": "Test activity", "editable_with_roles": [], "id": "cf_aea8B3HHCHNaoNIqzpH6HLvYqvlxCbZNZU8OkhkAuuW", "is_shared": false, "name": "Test Actvity", "referenced_custom_type_id": null, "required": false, "type": "datetime"}], "id": "actitype_4cMlNQsx560XS5dhcUpxp2", "name": "Meeting Activity 1", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417164366} +{"stream": "custom_activities", "data": {"api_create_only": false, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:01.879565", "date_updated": "2021-07-13T11:39:01.879565", "description": null, "editable_with_roles": [], "fields": [{"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_UftGNTS2rq9XMG9hOHkALp5cgn4Xl8nZqN7gReax3lc", "is_shared": false, "name": "Contact", "referenced_custom_type_id": null, "required": false, "type": "contact"}, {"accepts_multiple_values": true, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi", "is_shared": true, "name": "Contact Role", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim", "is_shared": true, "name": "Industry", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_bp93vNo2vxbmM2QhQ7JvdytgcozCCGmOklTG9E8rDYa", "is_shared": false, "name": "Industry: Other (if applicable)", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D", "is_shared": true, "name": "Current Vendor/Software", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_3W4n175LyZ3QMfr665jUS19nt2QATHyM7QaxXQC4QqA", "is_shared": false, "name": "Current Vendor: Other (if applicable)", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_i1zuHB2hra0uMSHQuDvkx3niXapGCORwWL5t9mO6w6Q", "is_shared": false, "name": "Number of Potential Users", "referenced_custom_type_id": null, "required": false, "type": "number"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4", "is_shared": true, "name": "Referral Source", "referenced_custom_type_id": null, "required": false, "type": "choices"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_WKYpud81IJKhD0uFmxm12baU0YcAuTN3nE6TzktrJoT", "is_shared": false, "name": "Referral Source: Other (if applicable)", "referenced_custom_type_id": null, "required": false, "type": "text"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_ldS8XBbJo4PxIpBYkA3XWQkwY3aGqHxJxDRV7sMTsaA", "is_shared": false, "name": "Next Steps", "referenced_custom_type_id": null, "required": false, "type": "textarea"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_jxTmWbZeP5LEVL4vokkPvTW0VPUU2UHobsphKgUB6xK", "is_shared": false, "name": "Notes", "referenced_custom_type_id": null, "required": false, "type": "textarea"}, {"accepts_multiple_values": false, "back_reference_is_visible": null, "converting_to_type": null, "description": null, "editable_with_roles": [], "id": "cf_1cL7txuLyrTH6Cpw15KtPLPT36l9eOdlJLKUdosxQLm", "is_shared": false, "name": "Qualified?", "referenced_custom_type_id": null, "required": false, "type": "choices"}], "id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "name": "Qualification Call", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417164370} diff --git a/airbyte-integrations/connectors/source-close-com/metadata.yaml b/airbyte-integrations/connectors/source-close-com/metadata.yaml index d366c279059d..6cd112e206e1 100644 --- a/airbyte-integrations/connectors/source-close-com/metadata.yaml +++ b/airbyte-integrations/connectors/source-close-com/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: dfffecb7-9a13-43e9-acdc-b92af7997ca9 - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.2 dockerRepository: airbyte/source-close-com githubIssueLabel: source-close-com icon: close.svg @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-close-com/requirements.txt b/airbyte-integrations/connectors/source-close-com/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-close-com/requirements.txt +++ b/airbyte-integrations/connectors/source-close-com/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-close-com/setup.py b/airbyte-integrations/connectors/source-close-com/setup.py index a9a1db64b97b..b9d9aaf53e7d 100644 --- a/airbyte-integrations/connectors/source-close-com/setup.py +++ b/airbyte-integrations/connectors/source-close-com/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum", "requests"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json index c87043fd8b8d..cafcf348dd9a 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "description": { "date_updated": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/answered_detached_call_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/answered_detached_call_tasks.json index d0d1f795f0e5..c46a84d6152d 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/answered_detached_call_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/answered_detached_call_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json index 21e4c832d8c5..886d0d164101 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "source": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json index 2ee569d66f4c..97ed85cb4ca3 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "description": { "date_updated": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contacts.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contacts.json index 28de10e512bb..5f979b8e291a 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contacts.json @@ -1,4 +1,5 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "additionalProperties": true, "properties": { diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json index 62ea600b9f60..e092624b7a2c 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_activities.json index 46983ba54199..ec9c87d33115 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_email_connected_accounts.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_email_connected_accounts.json index 530cb58bddd6..a0780f209f6d 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_email_connected_accounts.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/custom_email_connected_accounts.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "identities": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/delete_bulk_actions.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/delete_bulk_actions.json index e5527d59def8..1f6336967e87 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/delete_bulk_actions.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/delete_bulk_actions.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/dialer.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/dialer.json index 89cbcd944108..9c5fceb06997 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/dialer.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/dialer.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/edit_bulk_actions.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/edit_bulk_actions.json index 2bfb07f0d8ab..525e221b3c5f 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/edit_bulk_actions.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/edit_bulk_actions.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_activities.json index 1d84699d7d2e..d4bdc791fbed 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "cc": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_bulk_actions.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_bulk_actions.json index e1c3e308d6f4..8ae5035c5dd3 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_bulk_actions.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_bulk_actions.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json index ee73e804f19c..35ab577ebc36 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "_type": { "type": ["null", "string"] @@ -69,6 +71,9 @@ }, "updated_by_name ": { "type": ["null", "string"] + }, + "total_emails_count_in_thread ": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_sequences.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_sequences.json index c0256b8103d0..bca153fbe37a 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_sequences.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_sequences.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_templates.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_templates.json index 41726fc761fb..c1d28416ef31 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_templates.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_templates.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "attachments": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_thread_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_thread_activities.json index 9c5f1c5c9a4a..e6207bc19c0e 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_thread_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_thread_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json index 5e180115ad7b..0deeddc49d12 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "action": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/google_connected_accounts.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/google_connected_accounts.json index 0f9fb3080f17..38baf7097942 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/google_connected_accounts.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/google_connected_accounts.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "identities": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_email_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_email_tasks.json index 25c6b1b19ac2..725be5f20ca9 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_email_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_email_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "contact_id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_sms_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_sms_tasks.json index 8c4114b802dd..6e48700c11d9 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_sms_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/incoming_sms_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/integration_links.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/integration_links.json index 990e4c7e8403..4c9d4b417e89 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/integration_links.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/integration_links.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_created": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json index 1c4f2adc550c..19bc94eed623 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "status_id": { "date_updated": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_status_change_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_status_change_activities.json index 1913aa4d6b60..7c53951b488d 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_status_change_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_status_change_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_statuses.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_statuses.json index 0f358276a05f..d527a024e9d7 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_statuses.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_statuses.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "label": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_tasks.json index 41f02866ce4b..98d58881ceff 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/leads.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/leads.json index 37bf96824965..624e0ae5861f 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/leads.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/leads.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "status_id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json index 5da14e0ba5d7..66cb927a51e9 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/missed_call_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/missed_call_tasks.json index 81ec62bb1cc6..8f1604e14fc5 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/missed_call_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/missed_call_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/note_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/note_activities.json index 395f25336467..e43b757c3a82 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/note_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/note_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunities.json index 299d3cce77ed..89d490c44540 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "integration_links": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json index 2ee569d66f4c..97ed85cb4ca3 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "description": { "date_updated": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_due_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_due_tasks.json index 7110a5e10936..99ef973f6f86 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_due_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_due_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "object_id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json index 28a08b07c170..46a659113aa3 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "organization_id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_statuses.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_statuses.json index ba94459a2848..8be8d925b919 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_statuses.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_statuses.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "label": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json index 0737904c88fb..46ec2b161dbd 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "created_by": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/roles.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/roles.json index a910a305d2d6..01da09336885 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/roles.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/roles.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_created": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/send_as.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/send_as.json index 56b7c8fee78e..c62bdb1d4619 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/send_as.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/send_as.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sequence_subscription_bulk_actions.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sequence_subscription_bulk_actions.json index cccb6fad0d9d..832bc36d37f6 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sequence_subscription_bulk_actions.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sequence_subscription_bulk_actions.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/smart_views.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/smart_views.json index 0baaef11b96a..7d1f57cc3220 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/smart_views.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/smart_views.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json index b9b2d0850179..a31ed621a3df 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "date_updated": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json index 281cdab03a31..3c8e3434563d 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/users.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/users.json index ed15a0152d98..1700ac906d86 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/users.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/users.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "email": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/voicemail_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/voicemail_tasks.json index a4139f72256f..d8a272a7e5f0 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/voicemail_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/voicemail_tasks.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "created_by": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/zoom_connected_accounts.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/zoom_connected_accounts.json index 54a423911c33..6aeb1df5b094 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/zoom_connected_accounts.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/zoom_connected_accounts.json @@ -1,5 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "identities": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/spec.json b/airbyte-integrations/connectors/source-close-com/source_close_com/spec.json index 173feb2e30ad..7fd1ceddad2a 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/spec.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/spec.json @@ -8,17 +8,19 @@ "additionalProperties": true, "properties": { "api_key": { + "title": "API Key", "type": "string", "description": "Close.com API key (usually starts with 'api_'; find yours here).", "airbyte_secret": true }, "start_date": { + "title": "Replication Start Date", "type": "string", - "description": "The start date to sync data. Leave blank for full sync. Format: YYYY-MM-DD.", + "description": "The start date to sync data; all data after this date will be replicated. Leave blank to retrieve all the data available in the account. Format: YYYY-MM-DD.", "examples": ["2021-01-01"], "default": "2021-01-01", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "format": "date-time" + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/basic.sql index 95f96bed5111..1e192d718949 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/basic.sql @@ -1,15 +1,283 @@ -CREATE SCHEMA COCKROACHDB_BASIC; -CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); - -CREATE TABLE COCKROACHDB_BASIC.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_11 character,test_column_12 character(8),test_column_13 varchar,test_column_14 character(12),test_column_15 date,test_column_16 float8,test_column_17 float,test_column_19 int,test_column_2 BIT(3),test_column_23 numeric,test_column_24 decimal,test_column_25 smallint,test_column_26 text,test_column_27 time,test_column_28 timetz,test_column_29 timestamp,test_column_3 bigint,test_column_8 boolean ); - -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (1, 'a', '{asb123}', 'a', 'a', '1999-01-08', '123', '123', -2147483648, B'101', '99999', '+inf', -32768, 'a', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54', -9223372036854775808, true); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (2, '*', '{asb12}', 'abc', 'abc', '1999-01-08', '1234567890.1234567', '1234567890.1234567', 2147483647, B'101', '99999', 999, 32767, 'abc', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 9223372036854775807, 'yes'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (3, '*', '{asb12}', 'Миші йдуть на південь, не питай чому;', 'Миші йдуть;', '1999-01-08', 'infinity', 'infinity', 2147483647, B'101', '99999', '-inf', 32767, 'Миші йдуть;', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, '1'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (4, '*', '{asb12}', '櫻花分店', '櫻花分店', '1999-01-08', '+infinity', '+infinity', 2147483647, B'101', '99999', '+infinity', 32767, '櫻花分店', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, false); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (5, '*', '{asb12}', '', '', '1999-01-08', '+inf', '+inf', 2147483647, B'101', '99999', '-infinity', 32767, '', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'no'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (6, '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', 'inf', 'inf', 2147483647, B'101', '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, '0'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (7, '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', '-inf', '-inf', 2147483647, B'101', '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, '0'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (8, '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', '-infinity', '-infinity', 2147483647, B'101', '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, '0'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (9, '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', 'nan', 'nan', 2147483647, B'101', '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, '0'); -INSERT INTO COCKROACHDB_BASIC.TEST_DATASET VALUES (10, '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', 'nan', 'nan', 2147483647, B'101', '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, '0'); +CREATE + SCHEMA COCKROACHDB_BASIC; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TABLE + COCKROACHDB_BASIC.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_11 CHARACTER, + test_column_12 CHARACTER(8), + test_column_13 VARCHAR, + test_column_14 CHARACTER(12), + test_column_15 DATE, + test_column_16 float8, + test_column_17 FLOAT, + test_column_19 INT, + test_column_2 BIT(3), + test_column_23 NUMERIC, + test_column_24 DECIMAL, + test_column_25 SMALLINT, + test_column_26 text, + test_column_27 TIME, + test_column_28 timetz, + test_column_29 TIMESTAMP, + test_column_3 BIGINT, + test_column_8 BOOLEAN + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 1, + 'a', + '{asb123}', + 'a', + 'a', + '1999-01-08', + '123', + '123', + - 2147483648, + B'101', + '99999', + '+inf', + - 32768, + 'a', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54', + - 9223372036854775808, + TRUE + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 2, + '*', + '{asb12}', + 'abc', + 'abc', + '1999-01-08', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + B'101', + '99999', + 999, + 32767, + 'abc', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 9223372036854775807, + 'yes' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 3, + '*', + '{asb12}', + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть;', + '1999-01-08', + 'infinity', + 'infinity', + 2147483647, + B'101', + '99999', + '-inf', + 32767, + 'Миші йдуть;', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + '1' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 4, + '*', + '{asb12}', + '櫻花分店', + '櫻花分店', + '1999-01-08', + '+infinity', + '+infinity', + 2147483647, + B'101', + '99999', + '+infinity', + 32767, + '櫻花分店', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + FALSE + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 5, + '*', + '{asb12}', + '', + '', + '1999-01-08', + '+inf', + '+inf', + 2147483647, + B'101', + '99999', + '-infinity', + 32767, + '', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'no' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 6, + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + 'inf', + 'inf', + 2147483647, + B'101', + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + '0' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 7, + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + '-inf', + '-inf', + 2147483647, + B'101', + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + '0' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 8, + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + '-infinity', + '-infinity', + 2147483647, + B'101', + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + '0' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 9, + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + 'nan', + 'nan', + 2147483647, + B'101', + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + '0' + ); + +INSERT + INTO + COCKROACHDB_BASIC.TEST_DATASET + VALUES( + 10, + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + 'nan', + 'nan', + 2147483647, + B'101', + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + '0' + ); diff --git a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full.sql index 74c63f73ec67..9b62f53b1ca0 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full.sql +++ b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full.sql @@ -1,15 +1,454 @@ -CREATE SCHEMA COCKROACHDB_FULL; -CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); - -CREATE TABLE COCKROACHDB_FULL.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 STRING[],test_column_10 blob,test_column_11 character,test_column_12 character(8),test_column_13 varchar,test_column_14 character(12),test_column_15 date,test_column_16 float8,test_column_17 float,test_column_18 inet,test_column_19 int,test_column_2 BIT(3),test_column_20 interval,test_column_21 json,test_column_22 jsonb,test_column_23 numeric,test_column_24 decimal,test_column_25 smallint,test_column_26 text,test_column_27 time,test_column_28 timetz,test_column_29 timestamp,test_column_3 bigint,test_column_30 uuid,test_column_31 mood,test_column_32 text[],test_column_33 int[],test_column_4 bigserial,test_column_5 serial,test_column_6 smallserial,test_column_7 BIT VARYING(5),test_column_8 boolean,test_column_9 bytea[] ); - -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (1, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), 'a', '{asb123}', 'a', 'a', '1999-01-08', '123', '123', '198.24.10.0/24', null, B'101', null, null, null, '99999', '+inf', null, 'a', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54', -9223372036854775808, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', 1, 1, 1, B'101', true, ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (2, null, null, '*', '{asb12}', 'abc', 'abc', null, '1234567890.1234567', '1234567890.1234567', '198.24.10.0', -2147483648, null, 'P1Y2M3DT4H5M6S', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, null, 999, -32768, 'abc', null, null, TIMESTAMP '2004-10-19 10:23:54.123456', 9223372036854775807, null, null, null, null, 9223372036854775807, 2147483647, 32767, null, 'yes', null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (3, null, null, null, null, 'Миші йдуть на південь, не питай чому;', 'Миші йдуть;', null, null, null, '198.10/8', 2147483647, null, '-178000000', null, null, null, '-inf', 32767, 'Миші йдуть;', null, null, null, 0, null, null, null, null, 0, 0, 0, null, '1', null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (4, null, null, null, null, '櫻花分店', '櫻花分店', null, 'infinity', 'infinity', null, null, null, '178000000', null, null, null, '+infinity', null, '櫻花分店', null, null, null, null, null, null, null, null, -9223372036854775808, -2147483647, -32767, null, false, null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (5, null, null, null, null, '', '', null, '+infinity', '+infinity', null, null, null, null, null, null, null, '-infinity', null, '', null, null, null, null, null, null, null, null, null, null, null, null, 'no', null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (6, null, null, null, null, null, null, null, '+inf', '+inf', null, null, null, null, null, null, null, 'nan', null, null, null, null, null, null, null, null, null, null, null, null, null, null, '0', null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (7, null, null, null, null, '\xF0\x9F\x9A\x80', null, null, 'inf', 'inf', null, null, null, null, null, null, null, null, null, '\xF0\x9F\x9A\x80', null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (8, null, null, null, null, null, null, null, '-inf', '-inf', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (9, null, null, null, null, null, null, null, '-infinity', '-infinity', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO COCKROACHDB_FULL.TEST_DATASET VALUES (10, null, null, null, null, null, null, null, 'nan', 'nan', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); +CREATE + SCHEMA COCKROACHDB_FULL; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TABLE + COCKROACHDB_FULL.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 STRING [], + test_column_10 BLOB, + test_column_11 CHARACTER, + test_column_12 CHARACTER(8), + test_column_13 VARCHAR, + test_column_14 CHARACTER(12), + test_column_15 DATE, + test_column_16 float8, + test_column_17 FLOAT, + test_column_18 inet, + test_column_19 INT, + test_column_2 BIT(3), + test_column_20 INTERVAL, + test_column_21 json, + test_column_22 jsonb, + test_column_23 NUMERIC, + test_column_24 DECIMAL, + test_column_25 SMALLINT, + test_column_26 text, + test_column_27 TIME, + test_column_28 timetz, + test_column_29 TIMESTAMP, + test_column_3 BIGINT, + test_column_30 uuid, + test_column_31 mood, + test_column_32 text [], + test_column_33 INT [], + test_column_4 bigserial, + test_column_5 serial, + test_column_6 smallserial, + test_column_7 BIT VARYING(5), + test_column_8 BOOLEAN, + test_column_9 bytea [] + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 1, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + 'a', + '{asb123}', + 'a', + 'a', + '1999-01-08', + '123', + '123', + '198.24.10.0/24', + NULL, + B'101', + NULL, + NULL, + NULL, + '99999', + '+inf', + NULL, + 'a', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54', + - 9223372036854775808, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + 1, + 1, + 1, + B'101', + TRUE, + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 2, + NULL, + NULL, + '*', + '{asb12}', + 'abc', + 'abc', + NULL, + '1234567890.1234567', + '1234567890.1234567', + '198.24.10.0', + - 2147483648, + NULL, + 'P1Y2M3DT4H5M6S', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + NULL, + 999, + - 32768, + 'abc', + NULL, + NULL, + TIMESTAMP '2004-10-19 10:23:54.123456', + 9223372036854775807, + NULL, + NULL, + NULL, + NULL, + 9223372036854775807, + 2147483647, + 32767, + NULL, + 'yes', + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 3, + NULL, + NULL, + NULL, + NULL, + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть;', + NULL, + NULL, + NULL, + '198.10/8', + 2147483647, + NULL, + '-178000000', + NULL, + NULL, + NULL, + '-inf', + 32767, + 'Миші йдуть;', + NULL, + NULL, + NULL, + 0, + NULL, + NULL, + NULL, + NULL, + 0, + 0, + 0, + NULL, + '1', + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 4, + NULL, + NULL, + NULL, + NULL, + '櫻花分店', + '櫻花分店', + NULL, + 'infinity', + 'infinity', + NULL, + NULL, + NULL, + '178000000', + NULL, + NULL, + NULL, + '+infinity', + NULL, + '櫻花分店', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + - 9223372036854775808, + - 2147483647, + - 32767, + NULL, + FALSE, + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 5, + NULL, + NULL, + NULL, + NULL, + '', + '', + NULL, + '+infinity', + '+infinity', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '-infinity', + NULL, + '', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'no', + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 6, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '+inf', + '+inf', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'nan', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '0', + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 7, + NULL, + NULL, + NULL, + NULL, + '\xF0\x9F\x9A\x80', + NULL, + NULL, + 'inf', + 'inf', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '\xF0\x9F\x9A\x80', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 8, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '-inf', + '-inf', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 9, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '-infinity', + '-infinity', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + COCKROACHDB_FULL.TEST_DATASET + VALUES( + 10, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'nan', + 'nan', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full_without_nulls.sql index d2f65a53a52a..165170c9f99c 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full_without_nulls.sql +++ b/airbyte-integrations/connectors/source-cockroachdb/integration_tests/seed/full_without_nulls.sql @@ -1,15 +1,508 @@ -CREATE SCHEMA COCKROACHDB_FULL_NN; -CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); - -CREATE TABLE COCKROACHDB_FULL_NN.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 STRING[],test_column_10 blob,test_column_11 character,test_column_12 character(8),test_column_13 varchar,test_column_14 character(12),test_column_15 date,test_column_16 float8,test_column_17 float,test_column_18 inet,test_column_19 int,test_column_2 BIT(3),test_column_20 interval,test_column_21 json,test_column_22 jsonb,test_column_23 numeric,test_column_24 decimal,test_column_25 smallint,test_column_26 text,test_column_27 time,test_column_28 timetz,test_column_29 timestamp,test_column_3 bigint,test_column_30 uuid,test_column_31 mood,test_column_32 text[],test_column_33 int[],test_column_4 bigserial,test_column_5 serial,test_column_6 smallserial,test_column_7 BIT VARYING(5),test_column_8 boolean,test_column_9 bytea[] ); - -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (1, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), 'a', '{asb123}', 'a', 'a', '1999-01-08', '123', '123', '198.24.10.0/24', -2147483648, B'101', 'P1Y2M3DT4H5M6S', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', '+inf', -32768, 'a', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54', -9223372036854775808, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', 1, 1, 1, B'101', true, ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (2, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', 'abc', 'abc', '1999-01-08', '1234567890.1234567', '1234567890.1234567', '198.24.10.0', 2147483647, B'101', '-178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', 999, 32767, 'abc', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 9223372036854775807, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', 9223372036854775807, 2147483647, 32767, B'101', 'yes', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (3, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', 'Миші йдуть на південь, не питай чому;', 'Миші йдуть;', '1999-01-08', 'infinity', 'infinity', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', '-inf', 32767, 'Миші йдуть;', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', 0, 0, 0, B'101', '1', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (4, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '櫻花分店', '櫻花分店', '1999-01-08', '+infinity', '+infinity', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', '+infinity', 32767, '櫻花分店', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', false, ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (5, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '', '', '1999-01-08', '+inf', '+inf', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', '-infinity', 32767, '', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', 'no', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (6, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', 'inf', 'inf', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', '0', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (7, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', '-inf', '-inf', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', '0', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (8, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', '-infinity', '-infinity', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', '0', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (9, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', 'nan', 'nan', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', '0', ARRAY['☃'::bytes, 'ї'::bytes]); -INSERT INTO COCKROACHDB_FULL_NN.TEST_DATASET VALUES (10, ARRAY['sky', 'road', 'car'], decode('1234', 'hex'), '*', '{asb12}', '\xF0\x9F\x9A\x80', '', '1999-01-08', 'nan', 'nan', '198.10/8', 2147483647, B'101', '178000000', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '99999', 'nan', 32767, '\xF0\x9F\x9A\x80', '04:05:06', '04:05:06Z', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'happy', '{10000, 10000, 10000, 10000}', '{10000, 10000, 10000, 10000}', -9223372036854775808, -2147483647, -32767, B'101', '0', ARRAY['☃'::bytes, 'ї'::bytes]); +CREATE + SCHEMA COCKROACHDB_FULL_NN; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TABLE + COCKROACHDB_FULL_NN.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 STRING [], + test_column_10 BLOB, + test_column_11 CHARACTER, + test_column_12 CHARACTER(8), + test_column_13 VARCHAR, + test_column_14 CHARACTER(12), + test_column_15 DATE, + test_column_16 float8, + test_column_17 FLOAT, + test_column_18 inet, + test_column_19 INT, + test_column_2 BIT(3), + test_column_20 INTERVAL, + test_column_21 json, + test_column_22 jsonb, + test_column_23 NUMERIC, + test_column_24 DECIMAL, + test_column_25 SMALLINT, + test_column_26 text, + test_column_27 TIME, + test_column_28 timetz, + test_column_29 TIMESTAMP, + test_column_3 BIGINT, + test_column_30 uuid, + test_column_31 mood, + test_column_32 text [], + test_column_33 INT [], + test_column_4 bigserial, + test_column_5 serial, + test_column_6 smallserial, + test_column_7 BIT VARYING(5), + test_column_8 BOOLEAN, + test_column_9 bytea [] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 1, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + 'a', + '{asb123}', + 'a', + 'a', + '1999-01-08', + '123', + '123', + '198.24.10.0/24', + - 2147483648, + B'101', + 'P1Y2M3DT4H5M6S', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + '+inf', + - 32768, + 'a', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54', + - 9223372036854775808, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + 1, + 1, + 1, + B'101', + TRUE, + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 2, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + 'abc', + 'abc', + '1999-01-08', + '1234567890.1234567', + '1234567890.1234567', + '198.24.10.0', + 2147483647, + B'101', + '-178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + 999, + 32767, + 'abc', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 9223372036854775807, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + 9223372036854775807, + 2147483647, + 32767, + B'101', + 'yes', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 3, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть;', + '1999-01-08', + 'infinity', + 'infinity', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + '-inf', + 32767, + 'Миші йдуть;', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + 0, + 0, + 0, + B'101', + '1', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 4, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '櫻花分店', + '櫻花分店', + '1999-01-08', + '+infinity', + '+infinity', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + '+infinity', + 32767, + '櫻花分店', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + FALSE, + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 5, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '', + '', + '1999-01-08', + '+inf', + '+inf', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + '-infinity', + 32767, + '', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + 'no', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 6, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + 'inf', + 'inf', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + '0', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 7, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + '-inf', + '-inf', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + '0', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 8, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + '-infinity', + '-infinity', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + '0', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 9, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + 'nan', + 'nan', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + '0', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); + +INSERT + INTO + COCKROACHDB_FULL_NN.TEST_DATASET + VALUES( + 10, + ARRAY [ 'sky', + 'road', + 'car' ], + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '', + '1999-01-08', + 'nan', + 'nan', + '198.10/8', + 2147483647, + B'101', + '178000000', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '99999', + 'nan', + 32767, + '\xF0\x9F\x9A\x80', + '04:05:06', + '04:05:06Z', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 'happy', + '{10000, 10000, 10000, 10000}', + '{10000, 10000, 10000, 10000}', + - 9223372036854775808, + - 2147483647, + - 32767, + B'101', + '0', + ARRAY [ '☃'::bytes, + 'ї'::bytes ] + ); diff --git a/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml b/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml index 296a1fcda50b..9b2a4df22be9 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml +++ b/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-coda/Dockerfile b/airbyte-integrations/connectors/source-coda/Dockerfile index f8d116a3a924..681122ca5ce2 100644 --- a/airbyte-integrations/connectors/source-coda/Dockerfile +++ b/airbyte-integrations/connectors/source-coda/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_coda ./source_coda ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=1.2.0 LABEL io.airbyte.name=airbyte/source-coda diff --git a/airbyte-integrations/connectors/source-coda/__init__.py b/airbyte-integrations/connectors/source-coda/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml b/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml index a692eb177d65..d31c73aef6b8 100644 --- a/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml @@ -1,10 +1,10 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-coda:dev acceptance_tests: spec: tests: - spec_path: "source_coda/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" connection: tests: - config_path: "secrets/config.json" @@ -14,22 +14,17 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: - name: formulas - bypass_reason: "no records" + bypass_reason: "Sandbox account can't seed the stream" - name: permissions - bypass_reason: "no records" - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json index b6bef3c3fd7f..b021e7fe56c4 100644 --- a/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json @@ -62,6 +62,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "rows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-coda/metadata.yaml b/airbyte-integrations/connectors/source-coda/metadata.yaml index b547db60428d..c16c1748fef9 100644 --- a/airbyte-integrations/connectors/source-coda/metadata.yaml +++ b/airbyte-integrations/connectors/source-coda/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - https://coda.io/ + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 27f910fd-f832-4b2e-bcfd-6ab342e434d8 - dockerImageTag: 0.1.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-coda githubIssueLabel: source-coda icon: coda.svg license: MIT name: Coda - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-19 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/coda tags: - - language:python + - language:low-code + ab_internal: + sl: 100 + ql: 100 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-coda/setup.py b/airbyte-integrations/connectors/source-coda/setup.py index 8d3eae38ff0e..446b5807dcec 100644 --- a/airbyte-integrations/connectors/source-coda/setup.py +++ b/airbyte-integrations/connectors/source-coda/setup.py @@ -6,13 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ - "pytest~=6.2.5", + "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-coda/source_coda/__init__.py b/airbyte-integrations/connectors/source-coda/source_coda/__init__.py old mode 100755 new mode 100644 diff --git a/airbyte-integrations/connectors/source-coda/source_coda/manifest.yaml b/airbyte-integrations/connectors/source-coda/source_coda/manifest.yaml new file mode 100644 index 000000000000..2cae1c1e38f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/manifest.yaml @@ -0,0 +1,180 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] + + requester: + type: HttpRequester + url_base: "https://coda.io/apis/v1/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['auth_token'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + retriever_with_partition: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/docs_stream" + parent_key: "id" + partition_field: "doc_id" + + rows_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/tables_stream" + parent_key: "id" + partition_field: "table_id" + + parent_base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['href'] }}" + page_token_option: + type: "RequestPath" + field_name: "from" + inject_into: "url_base" + + docs_stream: + $ref: "#/definitions/parent_base_stream" + $parameters: + name: "docs" + primary_key: "id" + path: "docs" + retriever: + $ref: "#/definitions/retriever" + paginator: + $ref: "#/definitions/base_paginator" + + categories_stream: + $ref: "#/definitions/parent_base_stream" + $parameters: + name: "categories" + primary_key: "name" + path: "categories" + + permissions_stream: + $parameters: + name: "permissions" + primary_key: "id" + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever_with_partition" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + path: "docs/{{ stream_partition.doc_id }}/acl/permissions" + + pages_stream: + $parameters: + name: "pages" + primary_key: "id" + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever_with_partition" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + path: "docs/{{ stream_partition.doc_id }}/pages" + + tables_stream: + $parameters: + name: "tables" + primary_key: "id" + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever_with_partition" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + path: "docs/{{ stream_partition.doc_id }}/tables" + + formulas_stream: + $parameters: + name: "formulas" + primary_key: "id" + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever_with_partition" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + path: "docs/{{ stream_partition.doc_id }}/formulas" + + controls_stream: + $parameters: + name: "controls" + primary_key: "id" + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever_with_partition" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + path: "docs/{{ stream_partition.doc_id }}/controls" + + rows_stream: + $parameters: + name: "rows" + primary_key: "id" + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever_with_partition" + partition_router: + $ref: "#/definitions/rows_partition_router" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + path: "docs/{{ stream_partition.parent_slice.doc_id }}/tables/{{ stream_partition.table_id }}/rows" + +streams: + - "#/definitions/docs_stream" + - "#/definitions/permissions_stream" + - "#/definitions/categories_stream" + - "#/definitions/pages_stream" + - "#/definitions/tables_stream" + - "#/definitions/formulas_stream" + - "#/definitions/controls_stream" + - "#/definitions/rows_stream" + +check: + type: CheckStream + stream_names: + - "docs" + - "permissions" + - "categories" + - "pages" + - "tables" + - "formulas" + - "controls" + - "rows" diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json index a90ed1fede99..0fef40bebb64 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json index 14fc1e737c60..48afaf5822f2 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json @@ -1,37 +1,38 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "parent": { - "type": "object", + "type": ["null", "object"], "properties": { "type": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json index 3a28e58b0c33..11f1d56627fb 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json @@ -1,153 +1,154 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "icon": { - "type": "object", + "type": ["null", "object"], "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] } } }, "name": { - "type": "string" + "type": ["null", "string"] }, "owner": { - "type": "string" + "type": ["null", "string"] }, - "owner_name": { - "type": "string" + "ownerName": { + "type": ["null", "string"] }, "docSize": { - "type": "object", + "type": ["null", "object"], "properties": { "totalRowCount": { - "type": "integer" + "type": ["null", "integer"] }, "tableViewCount": { - "type": "integer" + "type": ["null", "integer"] }, "pageCount": { - "type": "integer" + "type": ["null", "integer"] }, "overApiSizeLimit": { - "type": "boolean" + "type": ["null", "boolean"] } } }, "sourceDoc": { - "type": "object", + "type": ["null", "object"], "properties": { "totalRowCount": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] } } }, "createdAt": { - "type": "string" + "type": ["null", "string"] }, "updatedAt": { - "type": "string" + "type": ["null", "string"] }, "published": { - "type": "object", + "type": ["null", "object"], "properties": { "description": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "imageLink": { - "type": "string" + "type": ["null", "string"] }, "discoverable": { - "type": "boolean" + "type": ["null", "boolean"] }, "earnCredit": { - "type": "boolean" + "type": ["null", "boolean"] }, "mode": { - "type": "string" + "type": ["null", "string"] }, "categories": { - "type": "string", + "type": ["null", "string"], "items": { - "type": "string" + "type": ["null", "string"] } } } }, "folder": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } }, "workspace": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "organizationId": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } }, "workspaceId": { - "type": "string" + "type": ["null", "string"] }, "folderId": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json index 14fc1e737c60..48afaf5822f2 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json @@ -1,37 +1,38 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "parent": { - "type": "object", + "type": ["null", "object"], "properties": { "type": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json index 79ca344a2b13..cdf35c7d26e6 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": "string" @@ -20,8 +21,54 @@ "subtitle": { "type": "string" }, + "createdBy": { + "type": ["null", "object"], + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "createdAt": { + "type": "string" + }, + "updatedBy": { + "type": ["null", "object"], + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "updatedAt": { + "type": "string" + }, "icon": { - "type": "object", + "type": ["null", "object"], "properties": { "name": { "type": "string" @@ -35,7 +82,7 @@ } }, "image": { - "type": "object", + "type": ["null", "object"], "properties": { "width": { "type": "number" @@ -52,7 +99,7 @@ } }, "parent": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { "type": "string" @@ -74,7 +121,7 @@ "children": { "type": "array", "items": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { "type": "string" @@ -96,7 +143,7 @@ "authors": { "type": "array", "items": { - "type": "object", + "type": ["null", "object"], "properties": { "@context": { "type": "string" @@ -119,7 +166,7 @@ "type": "string" }, "updatedBy": { - "type": "object", + "type": ["null", "object"], "properties": { "@context": { "type": "string" @@ -139,7 +186,7 @@ } }, "createdBy": { - "type": "object", + "type": ["null", "object"], "properties": { "@context": { "type": "string" @@ -159,6 +206,79 @@ } } } + }, + "createdAt": { + "type": ["null", "string"] + }, + "contentType": { + "type": ["null", "string"], + "enum": ["canvas", "embed"] + }, + "createdBy": { + "type": "object", + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "authors": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + } + }, + "updatedBy": { + "type": ["null", "object"], + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "updatedAt": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json index e440e36472a4..fafc921e910f 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json @@ -1,21 +1,22 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "access": { - "type": "string" + "type": ["null", "string"] }, "principal": { - "type": "object", + "type": ["null", "object"], "properties": { "type": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/rows.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/rows.json new file mode 100644 index 000000000000..865871890f00 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/rows.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "index": { + "type": "number" + }, + "createdAt": { + "type": "string" + }, + "updatedAt": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "values": { + "type": "object", + "properties": { + "^c-_": { + "type": ["string", "number", "object", "array", "boolean", "null"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json index 94b46a4170dc..8b07af6e249f 100644 --- a/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json @@ -1,42 +1,43 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "tableType": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "parent": { - "type": "object", + "type": ["null", "object"], "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "href": { - "type": "string" + "type": ["null", "string"] }, "browserLink": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-coda/source_coda/source.py b/airbyte-integrations/connectors/source-coda/source_coda/source.py old mode 100755 new mode 100644 index 08ceb1a32a15..db20659db063 --- a/airbyte-integrations/connectors/source-coda/source_coda/source.py +++ b/airbyte-integrations/connectors/source-coda/source_coda/source.py @@ -2,169 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -BASE_URL = "https://coda.io/apis/v1/" +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class CodaStream(HttpStream, ABC): - - url_base = BASE_URL - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.limit = 25 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return response.json().get("nextPageToken", None) - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - if next_page_token: - return {"pageToken": next_page_token, "limit": self.limit} - else: - return {"limit": self.limit} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json()["items"] - - -class Docs(CodaStream): - - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "docs" - - -class CodaStreamDoc(CodaStream): - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """ - self.authenticator (which should be used as the - authenticator for Users) is object of NoAuth() - - so self._session.auth is used instead - """ - docs_stream = Docs(**{"authenticator": self._authenticator}) - for doc in docs_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"doc_id": doc["id"]} - - -class Permissions(CodaStreamDoc): - - primary_key = "id" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - doc_id = stream_slice["doc_id"] - return f"docs/{doc_id}/acl/permissions" - - -class Categories(CodaStreamDoc): - - primary_key = "name" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "categories" - - -class Pages(CodaStreamDoc): - - primary_key = "id" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - doc_id = stream_slice["doc_id"] - return f"docs/{doc_id}/pages" - - -class Tables(CodaStreamDoc): - - primary_key = "id" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - doc_id = stream_slice["doc_id"] - return f"docs/{doc_id}/tables" - - -class Formulas(CodaStreamDoc): - - primary_key = "id" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - doc_id = stream_slice["doc_id"] - return f"docs/{doc_id}/formulas" - - -class Controls(CodaStreamDoc): - - primary_key = "id" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - doc_id = stream_slice["doc_id"] - return f"docs/{doc_id}/controls" - - -# Source -class SourceCoda(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config.get("auth_token") - headers = {"Authorization": f"Bearer {token}"} - r = requests.get(f"{BASE_URL}whoami", headers=headers) - if r.status_code == 200: - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - stream_args = { - "authenticator": TokenAuthenticator(token=config.get("auth_token")), - } - - return [ - Docs(**stream_args), - Permissions(**stream_args), - Categories(**stream_args), - Pages(**stream_args), - Tables(**stream_args), - Formulas(**stream_args), - Controls(**stream_args), - ] +# Declarative Source +class SourceCoda(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-coda/unit_tests/test_source.py b/airbyte-integrations/connectors/source-coda/unit_tests/test_source.py deleted file mode 100644 index 163a5d37fa21..000000000000 --- a/airbyte-integrations/connectors/source-coda/unit_tests/test_source.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_coda.source import SourceCoda - - -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - def raise_for_status(self): - if self.status_code != 200: - raise Exception("Bad things happened") - - -def mocked_requests_get(fail=False): - def wrapper(*args, **kwargs): - if fail: - return MockResponse(None, 404) - - return MockResponse( - {"name": "John", "loginId": "john@example.com", "type": "user", "href": "https://coda.io/apis/v1/whoami", "tokenName": "as", "scoped": False, "pictureLink": "https://images-coda.io", "workspace":{ - "id": "test-id", - "type": "workspace", - "browserLink": "https://coda.io/link", - "name": "title" - }}, 200 - ) - - return wrapper - - -@patch("requests.get", side_effect=mocked_requests_get()) -def test_check_connection(mocker): - source = SourceCoda() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceCoda() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 7 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py deleted file mode 100644 index 8aec8ea4d02b..000000000000 --- a/airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from source_coda.source import CodaStream - -logger = logging.getLogger() -logger.level = logging.DEBUG - - -authenticator = TokenAuthenticator(token="test_token"), - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(CodaStream, "path", "v0/example_endpoint") - mocker.patch.object(CodaStream, "primary_key", "test_primary_key") - mocker.patch.object(CodaStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = CodaStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"limit": 25} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = CodaStream(authenticator=authenticator) - response = MagicMock() - response.json.return_value = { - "id": "1244fds", - "name": "Test doc" - } - inputs = {"response": response} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = CodaStream(authenticator=authenticator) - - response = MagicMock() - response.json = MagicMock(return_value={'items': [{"id": 101}]}) - - inputs = {"response": response} - expected_parsed_object = response.json()['items'][0] - assert next(iter(stream.parse_response(**inputs))) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = CodaStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = CodaStream(authenticator=authenticator) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = CodaStream(authenticator=authenticator) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = CodaStream(authenticator=authenticator) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-coin-api/metadata.yaml b/airbyte-integrations/connectors/source-coin-api/metadata.yaml index d3b6f0ac53ad..fc65c06c61c2 100644 --- a/airbyte-integrations/connectors/source-coin-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-coin-api/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-coin-api/requirements.txt b/airbyte-integrations/connectors/source-coin-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-coin-api/requirements.txt +++ b/airbyte-integrations/connectors/source-coin-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-coin-api/setup.py b/airbyte-integrations/connectors/source-coin-api/setup.py index 68743c5d5168..b4c098be7797 100644 --- a/airbyte-integrations/connectors/source-coin-api/setup.py +++ b/airbyte-integrations/connectors/source-coin-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml b/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml index 66ed47bf2492..b7619d147910 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml +++ b/airbyte-integrations/connectors/source-coingecko-coins/metadata.yaml @@ -10,7 +10,7 @@ data: name: CoinGecko Coins registries: cloud: - enabled: false + enabled: false # Did not pass acceptance tests oss: enabled: true releaseStage: alpha @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt b/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt +++ b/airbyte-integrations/connectors/source-coingecko-coins/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-coingecko-coins/setup.py b/airbyte-integrations/connectors/source-coingecko-coins/setup.py index a04d072d806c..39a959dcdc27 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/setup.py +++ b/airbyte-integrations/connectors/source-coingecko-coins/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/manifest.yaml b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/manifest.yaml index 3feb8f9d42f3..b67196fcdd6c 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/manifest.yaml +++ b/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/manifest.yaml @@ -83,166 +83,166 @@ definitions: properties: id: type: - - string - - 'null' + - string + - "null" symbol: type: - - string - - 'null' + - string + - "null" name: type: - - string - - 'null' + - string + - "null" localization: type: object properties: en: type: - - string - - 'null' + - string + - "null" de: type: - - string - - 'null' + - string + - "null" es: type: - - string - - 'null' + - string + - "null" fr: type: - - string - - 'null' + - string + - "null" it: type: - - string - - 'null' + - string + - "null" pl: type: - - string - - 'null' + - string + - "null" ro: type: - - string - - 'null' + - string + - "null" hu: type: - - string - - 'null' + - string + - "null" nl: type: - - string - - 'null' + - string + - "null" pt: type: - - string - - 'null' + - string + - "null" sv: type: - - string - - 'null' + - string + - "null" vi: type: - - string - - 'null' + - string + - "null" tr: type: - - string - - 'null' + - string + - "null" ru: type: - - string - - 'null' + - string + - "null" ja: type: - - string - - 'null' + - string + - "null" zh: type: - - string - - 'null' + - string + - "null" zh-tw: type: - - string - - 'null' + - string + - "null" ko: type: - - string - - 'null' + - string + - "null" ar: type: - - string - - 'null' + - string + - "null" th: type: - - string - - 'null' + - string + - "null" id: type: - - string - - 'null' + - string + - "null" cs: type: - - string - - 'null' + - string + - "null" da: type: - - string - - 'null' + - string + - "null" el: type: - - string - - 'null' + - string + - "null" hi: type: - - string - - 'null' - 'no': + - string + - "null" + "no": type: - - string - - 'null' + - string + - "null" sk: type: - - string - - 'null' + - string + - "null" uk: type: - - string - - 'null' + - string + - "null" he: type: - - string - - 'null' + - string + - "null" fi: type: - - string - - 'null' + - string + - "null" bg: type: - - string - - 'null' + - string + - "null" hr: type: - - string - - 'null' + - string + - "null" lt: type: - - string - - 'null' + - string + - "null" sl: type: - - string - - 'null' + - string + - "null" image: type: object properties: thumb: type: - - string - - 'null' + - string + - "null" small: type: - - string - - 'null' + - string + - "null" market_data: type: object properties: @@ -251,742 +251,742 @@ definitions: properties: aed: type: - - number - - 'null' + - number + - "null" ars: type: - - number - - 'null' + - number + - "null" aud: type: - - number - - 'null' + - number + - "null" bch: type: - - number - - 'null' + - number + - "null" bdt: type: - - number - - 'null' + - number + - "null" bhd: type: - - number - - 'null' + - number + - "null" bmd: type: - - number - - 'null' + - number + - "null" bnb: type: - - number - - 'null' + - number + - "null" brl: type: - - number - - 'null' + - number + - "null" btc: type: - - number - - 'null' + - number + - "null" cad: type: - - number - - 'null' + - number + - "null" chf: type: - - number - - 'null' + - number + - "null" clp: type: - - number - - 'null' + - number + - "null" cny: type: - - number - - 'null' + - number + - "null" czk: type: - - number - - 'null' + - number + - "null" dkk: type: - - number - - 'null' + - number + - "null" dot: type: - - number - - 'null' + - number + - "null" eos: type: - - number - - 'null' + - number + - "null" eth: type: - - number - - 'null' + - number + - "null" eur: type: - - number - - 'null' + - number + - "null" gbp: type: - - number - - 'null' + - number + - "null" hkd: type: - - number - - 'null' + - number + - "null" huf: type: - - number - - 'null' + - number + - "null" idr: type: - - number - - 'null' + - number + - "null" ils: type: - - number - - 'null' + - number + - "null" inr: type: - - number - - 'null' + - number + - "null" jpy: type: - - number - - 'null' + - number + - "null" krw: type: - - number - - 'null' + - number + - "null" kwd: type: - - number - - 'null' + - number + - "null" lkr: type: - - number - - 'null' + - number + - "null" ltc: type: - - number - - 'null' + - number + - "null" mmk: type: - - number - - 'null' + - number + - "null" mxn: type: - - number - - 'null' + - number + - "null" myr: type: - - number - - 'null' + - number + - "null" ngn: type: - - number - - 'null' + - number + - "null" nok: type: - - number - - 'null' + - number + - "null" nzd: type: - - number - - 'null' + - number + - "null" php: type: - - number - - 'null' + - number + - "null" pkr: type: - - number - - 'null' + - number + - "null" pln: type: - - number - - 'null' + - number + - "null" rub: type: - - number - - 'null' + - number + - "null" sar: type: - - number - - 'null' + - number + - "null" sek: type: - - number - - 'null' + - number + - "null" sgd: type: - - number - - 'null' + - number + - "null" thb: type: - - number - - 'null' + - number + - "null" try: type: - - number - - 'null' + - number + - "null" twd: type: - - number - - 'null' + - number + - "null" uah: type: - - number - - 'null' + - number + - "null" usd: type: - - number - - 'null' + - number + - "null" vef: type: - - number - - 'null' + - number + - "null" vnd: type: - - number - - 'null' + - number + - "null" xag: type: - - number - - 'null' + - number + - "null" xau: type: - - number - - 'null' + - number + - "null" xdr: type: - - number - - 'null' + - number + - "null" xlm: type: - - number - - 'null' + - number + - "null" xrp: type: - - number - - 'null' + - number + - "null" yfi: type: - - number - - 'null' + - number + - "null" zar: type: - - number - - 'null' + - number + - "null" bits: type: - - number - - 'null' + - number + - "null" link: type: - - number - - 'null' + - number + - "null" sats: type: - - number - - 'null' + - number + - "null" market_cap: type: object properties: aed: type: - - number - - 'null' + - number + - "null" ars: type: - - number - - 'null' + - number + - "null" aud: type: - - number - - 'null' + - number + - "null" bch: type: - - number - - 'null' + - number + - "null" bdt: type: - - number - - 'null' + - number + - "null" bhd: type: - - number - - 'null' + - number + - "null" bmd: type: - - number - - 'null' + - number + - "null" bnb: type: - - number - - 'null' + - number + - "null" brl: type: - - number - - 'null' + - number + - "null" btc: type: - - number - - 'null' + - number + - "null" cad: type: - - number - - 'null' + - number + - "null" chf: type: - - number - - 'null' + - number + - "null" clp: type: - - number - - 'null' + - number + - "null" cny: type: - - number - - 'null' + - number + - "null" czk: type: - - number - - 'null' + - number + - "null" dkk: type: - - number - - 'null' + - number + - "null" dot: type: - - number - - 'null' + - number + - "null" eos: type: - - number - - 'null' + - number + - "null" eth: type: - - number - - 'null' + - number + - "null" eur: type: - - number - - 'null' + - number + - "null" gbp: type: - - number - - 'null' + - number + - "null" hkd: type: - - number - - 'null' + - number + - "null" huf: type: - - number - - 'null' + - number + - "null" idr: type: - - number - - 'null' + - number + - "null" ils: type: - - number - - 'null' + - number + - "null" inr: type: - - number - - 'null' + - number + - "null" jpy: type: - - number - - 'null' + - number + - "null" krw: type: - - number - - 'null' + - number + - "null" kwd: type: - - number - - 'null' + - number + - "null" lkr: type: - - number - - 'null' + - number + - "null" ltc: type: - - number - - 'null' + - number + - "null" mmk: type: - - number - - 'null' + - number + - "null" mxn: type: - - number - - 'null' + - number + - "null" myr: type: - - number - - 'null' + - number + - "null" ngn: type: - - number - - 'null' + - number + - "null" nok: type: - - number - - 'null' + - number + - "null" nzd: type: - - number - - 'null' + - number + - "null" php: type: - - number - - 'null' + - number + - "null" pkr: type: - - number - - 'null' + - number + - "null" pln: type: - - number - - 'null' + - number + - "null" rub: type: - - number - - 'null' + - number + - "null" sar: type: - - number - - 'null' + - number + - "null" sek: type: - - number - - 'null' + - number + - "null" sgd: type: - - number - - 'null' + - number + - "null" thb: type: - - number - - 'null' + - number + - "null" try: type: - - number - - 'null' + - number + - "null" twd: type: - - number - - 'null' + - number + - "null" uah: type: - - number - - 'null' + - number + - "null" usd: type: - - number - - 'null' + - number + - "null" vef: type: - - number - - 'null' + - number + - "null" vnd: type: - - number - - 'null' + - number + - "null" xag: type: - - number - - 'null' + - number + - "null" xau: type: - - number - - 'null' + - number + - "null" xdr: type: - - number - - 'null' + - number + - "null" xlm: type: - - number - - 'null' + - number + - "null" xrp: type: - - number - - 'null' + - number + - "null" yfi: type: - - number - - 'null' + - number + - "null" zar: type: - - number - - 'null' + - number + - "null" bits: type: - - number - - 'null' + - number + - "null" link: type: - - number - - 'null' + - number + - "null" sats: type: - - number - - 'null' + - number + - "null" total_volume: type: object properties: aed: type: - - number - - 'null' + - number + - "null" ars: type: - - number - - 'null' + - number + - "null" aud: type: - - number - - 'null' + - number + - "null" bch: type: - - number - - 'null' + - number + - "null" bdt: type: - - number - - 'null' + - number + - "null" bhd: type: - - number - - 'null' + - number + - "null" bmd: type: - - number - - 'null' + - number + - "null" bnb: type: - - number - - 'null' + - number + - "null" brl: type: - - number - - 'null' + - number + - "null" btc: type: - - number - - 'null' + - number + - "null" cad: type: - - number - - 'null' + - number + - "null" chf: type: - - number - - 'null' + - number + - "null" clp: type: - - number - - 'null' + - number + - "null" cny: type: - - number - - 'null' + - number + - "null" czk: type: - - number - - 'null' + - number + - "null" dkk: type: - - number - - 'null' + - number + - "null" dot: type: - - number - - 'null' + - number + - "null" eos: type: - - number - - 'null' + - number + - "null" eth: type: - - number - - 'null' + - number + - "null" eur: type: - - number - - 'null' + - number + - "null" gbp: type: - - number - - 'null' + - number + - "null" hkd: type: - - number - - 'null' + - number + - "null" huf: type: - - number - - 'null' + - number + - "null" idr: type: - - number - - 'null' + - number + - "null" ils: type: - - number - - 'null' + - number + - "null" inr: type: - - number - - 'null' + - number + - "null" jpy: type: - - number - - 'null' + - number + - "null" krw: type: - - number - - 'null' + - number + - "null" kwd: type: - - number - - 'null' + - number + - "null" lkr: type: - - number - - 'null' + - number + - "null" ltc: type: - - number - - 'null' + - number + - "null" mmk: type: - - number - - 'null' + - number + - "null" mxn: type: - - number - - 'null' + - number + - "null" myr: type: - - number - - 'null' + - number + - "null" ngn: type: - - number - - 'null' + - number + - "null" nok: type: - - number - - 'null' + - number + - "null" nzd: type: - - number - - 'null' + - number + - "null" php: type: - - number - - 'null' + - number + - "null" pkr: type: - - number - - 'null' + - number + - "null" pln: type: - - number - - 'null' + - number + - "null" rub: type: - - number - - 'null' + - number + - "null" sar: type: - - number - - 'null' + - number + - "null" sek: type: - - number - - 'null' + - number + - "null" sgd: type: - - number - - 'null' + - number + - "null" thb: type: - - number - - 'null' + - number + - "null" try: type: - - number - - 'null' + - number + - "null" twd: type: - - number - - 'null' + - number + - "null" uah: type: - - number - - 'null' + - number + - "null" usd: type: - - number - - 'null' + - number + - "null" vef: type: - - number - - 'null' + - number + - "null" vnd: type: - - number - - 'null' + - number + - "null" xag: type: - - number - - 'null' + - number + - "null" xau: type: - - number - - 'null' + - number + - "null" xdr: type: - - number - - 'null' + - number + - "null" xlm: type: - - number - - 'null' + - number + - "null" xrp: type: - - number - - 'null' + - number + - "null" yfi: type: - - number - - 'null' + - number + - "null" zar: type: - - number - - 'null' + - number + - "null" bits: type: - - number - - 'null' + - number + - "null" link: type: - - number - - 'null' + - number + - "null" sats: type: - - number - - 'null' + - number + - "null" community_data: type: object properties: @@ -994,66 +994,66 @@ definitions: twitter_followers: {} reddit_average_posts_48h: type: - - number - - 'null' + - number + - "null" reddit_average_comments_48h: type: - - number - - 'null' + - number + - "null" reddit_subscribers: type: - - number - - 'null' + - number + - "null" reddit_accounts_active_48h: type: - - string - - 'null' + - string + - "null" developer_data: type: object properties: forks: type: - - number - - 'null' + - number + - "null" stars: type: - - number - - 'null' + - number + - "null" subscribers: type: - - number - - 'null' + - number + - "null" total_issues: type: - - number - - 'null' + - number + - "null" closed_issues: type: - - number - - 'null' + - number + - "null" pull_requests_merged: type: - - number - - 'null' + - number + - "null" pull_request_contributors: type: - - number - - 'null' + - number + - "null" code_additions_deletions_4_weeks: type: object properties: additions: type: - - number - - 'null' + - number + - "null" deletions: type: - - number - - 'null' + - number + - "null" commit_count_4_weeks: type: - - number - - 'null' + - number + - "null" public_interest_stats: type: object properties: diff --git a/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml b/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml index 27bdee025ace..a73b7184a363 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml +++ b/airbyte-integrations/connectors/source-coinmarketcap/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-coinmarketcap/requirements.txt b/airbyte-integrations/connectors/source-coinmarketcap/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/requirements.txt +++ b/airbyte-integrations/connectors/source-coinmarketcap/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-coinmarketcap/setup.py b/airbyte-integrations/connectors/source-coinmarketcap/setup.py index a742819f051b..64fd1b41273b 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/setup.py +++ b/airbyte-integrations/connectors/source-coinmarketcap/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-commcare/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-commcare/integration_tests/abnormal_state.json index 6cfc6686b6e0..58287f5763fa 100644 --- a/airbyte-integrations/connectors/source-commcare/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-commcare/integration_tests/abnormal_state.json @@ -2,4 +2,4 @@ "Assess a referred patient": { "indexed_on": "2023-11-25T20:30:30.2423" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-commcare/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-commcare/integration_tests/configured_catalog.json index d34c7a7f43ed..b475172888b7 100644 --- a/airbyte-integrations/connectors/source-commcare/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commcare/integration_tests/configured_catalog.json @@ -4,20 +4,13 @@ "stream": { "name": "Assess a referred patient", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "indexed_on" - ] + "default_cursor_field": ["indexed_on"] }, "sync_mode": "incremental", - "cursor_field": [ - "indexed_on" - ], + "cursor_field": ["indexed_on"], "destination_sync_mode": "append" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-commcare/metadata.yaml b/airbyte-integrations/connectors/source-commcare/metadata.yaml index 77c6811116bb..973903f93f22 100644 --- a/airbyte-integrations/connectors/source-commcare/metadata.yaml +++ b/airbyte-integrations/connectors/source-commcare/metadata.yaml @@ -16,4 +16,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/commcare tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-commcare/requirements.txt b/airbyte-integrations/connectors/source-commcare/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-commcare/requirements.txt +++ b/airbyte-integrations/connectors/source-commcare/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-commcare/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-commcare/sample_files/configured_catalog.json index efb1c1536963..049bdcf1b6d1 100644 --- a/airbyte-integrations/connectors/source-commcare/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commcare/sample_files/configured_catalog.json @@ -1,22 +1,16 @@ { - "type": "CATALOG", - "catalog": { - "streams": [ - { - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": {} - }, - "supported_sync_modes": [ - "incremental", - "full_refresh" - ], - "supported_destination_sync_modes": [ - "overwrite", - "append_dedup" - ] - } - ] - } -} \ No newline at end of file + "type": "CATALOG", + "catalog": { + "streams": [ + { + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "supported_destination_sync_modes": ["overwrite", "append_dedup"] + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-commcare/setup.py b/airbyte-integrations/connectors/source-commcare/setup.py index 87f88f342abc..fd011eec2b1f 100644 --- a/airbyte-integrations/connectors/source-commcare/setup.py +++ b/airbyte-integrations/connectors/source-commcare/setup.py @@ -13,9 +13,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-commcare/source_commcare/spec.yaml b/airbyte-integrations/connectors/source-commcare/source_commcare/spec.yaml index 20e62fa81984..487c25ad6c02 100644 --- a/airbyte-integrations/connectors/source-commcare/source_commcare/spec.yaml +++ b/airbyte-integrations/connectors/source-commcare/source_commcare/spec.yaml @@ -16,7 +16,7 @@ connectionSpecification: Commcare API Key airbyte_secret: true order: 0 - project_space: + project_space: type: string title: Project Space description: >- diff --git a/airbyte-integrations/connectors/source-commercetools/.dockerignore b/airbyte-integrations/connectors/source-commercetools/.dockerignore index f0754479b199..cd52b08c75ea 100644 --- a/airbyte-integrations/connectors/source-commercetools/.dockerignore +++ b/airbyte-integrations/connectors/source-commercetools/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_commercetools !setup.py diff --git a/airbyte-integrations/connectors/source-commercetools/Dockerfile b/airbyte-integrations/connectors/source-commercetools/Dockerfile index f2b6a4948e72..9d3b156bfcb5 100644 --- a/airbyte-integrations/connectors/source-commercetools/Dockerfile +++ b/airbyte-integrations/connectors/source-commercetools/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /airbyte/integration_code # upgrade pip to the latest version RUN apk --no-cache upgrade \ && pip install --upgrade pip \ - && apk --no-cache add tzdata + && apk --no-cache add tzdata build-base COPY setup.py ./ @@ -34,5 +34,5 @@ COPY source_commercetools ./source_commercetools ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-commercetools diff --git a/airbyte-integrations/connectors/source-commercetools/README.md b/airbyte-integrations/connectors/source-commercetools/README.md index af4e8895f827..d79ac18a3c12 100644 --- a/airbyte-integrations/connectors/source-commercetools/README.md +++ b/airbyte-integrations/connectors/source-commercetools/README.md @@ -1,34 +1,10 @@ # Commercetools Source -This is the repository for the Commercetools source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/commercetools). +This is the repository for the Commercetools configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/commercetools). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/commercetools) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commercetools/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/commercetools) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commercetools/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source commercetools test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commercetools:dev disc docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commercetools:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-commercetools/__init__.py b/airbyte-integrations/connectors/source-commercetools/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-commercetools/acceptance-test-config.yml b/airbyte-integrations/connectors/source-commercetools/acceptance-test-config.yml index eb31af453cff..bc5c0e98319c 100644 --- a/airbyte-integrations/connectors/source-commercetools/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-commercetools/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-commercetools:dev tests: spec: - - spec_path: "source_commercetools/spec.json" + - spec_path: "source_commercetools/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/__init__.py b/airbyte-integrations/connectors/source-commercetools/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/catalog.json b/airbyte-integrations/connectors/source-commercetools/integration_tests/catalog.json deleted file mode 100644 index 6aa73d163d56..000000000000 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/catalog.json +++ /dev/null @@ -1,7688 +0,0 @@ -{ - "streams": [ - { - "name": "customers", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "customerNumber": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": " date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": " date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "email": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - }, - "stores": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - } - } - } - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "middleName": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "dateOfBirth": { - "type": ["null", "string"], - "format": "date" - }, - "companyName": { - "type": ["null", "string"] - }, - "vatId": { - "type": ["null", "string"] - }, - "addresses": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "defaultShippingAddressId": { - "type": ["null", "string"] - }, - "shippingAddressIds": { - "type": ["null", "array"] - }, - "defaultBillingAddressId": { - "type": ["null", "string"] - }, - "billingAddressIds": { - "type": ["null", "array"] - }, - "isEmailVerified": { - "type": ["null", "boolean"] - }, - "externalId": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "locale": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "orders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "completedAt": { - "type": ["null", "integer"], - "format": "date-time" - }, - "orderNumber": { - "type": ["null", "string"] - }, - "customerId": { - "type": ["null", "string"] - }, - "customerEmail": { - "type": ["null", "string"] - }, - "anonymousId": { - "type": ["null", "string"] - }, - "store": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - } - } - }, - "lineItems": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "productId": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "productSlug": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "productType": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "variant": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - }, - "totalPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "quantity": { - "type": ["null", "number"] - }, - "addedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "state": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "taxRate": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "includedInPrice": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "subRates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - } - } - }, - "supplyChannel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "distributionChannel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedPricePerQuantity": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "discountedPrice": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "includedDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedAmount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - } - } - }, - "priceMode": { - "type": ["null", "string"] - }, - "lineItemMode": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "shippingDetails": { - "type": ["null", "object"], - "properties": { - "targets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "addressKey": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - }, - "valid": { - "type": ["null", "boolean"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "customLineItems": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "money": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - }, - "totalPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "slug": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "taxCategory": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "taxRate": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "includedInPrice": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "subRates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discountedPricePerQuantity": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "discountedPrice": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "includedDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedAmount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "shippingDetails": { - "type": ["null", "object"], - "properties": { - "targets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "addressKey": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - }, - "valid": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "totalPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "taxPortions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "rate": { - "type": ["null", "number"] - }, - "amount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - }, - "shippingAddress": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "billingAddress": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "taxMode": { - "type": ["null", "string"] - }, - "taxRoundingMode": { - "type": ["null", "string"] - }, - "taxCalculationMode": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "country": { - "type": ["null", "string"] - }, - "orderState": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "shipmentState": { - "type": ["null", "string"] - }, - "paymentState": { - "type": ["null", "string"] - }, - "shippingInfo": { - "type": ["null", "object"], - "properties": { - "shippingMethodName": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "shippingRate": { - "type": ["null", "object"], - "properties": { - "price": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "freeAbove": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "tiers": { - "type": ["null", "array"] - }, - "isMatching": { - "type": ["null", "boolean"] - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - }, - "taxRate": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "includedInPrice": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "subRates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - } - } - }, - "taxCategory": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "shippingMethod": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "deliveries": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - }, - "parcels": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "measurements": { - "type": ["null", "string"] - }, - "trackingData": { - "type": ["null", "object"], - "properties": { - "trackingId": { - "type": ["null", "string"] - }, - "carrier": { - "type": ["null", "string"] - }, - "provider": { - "type": ["null", "string"] - }, - "providerTransaction": { - "type": ["null", "string"] - }, - "isReturn": { - "type": ["null", "boolean"] - } - } - }, - "items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - } - } - } - }, - "address": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - } - } - }, - "discountedPrice": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "includedDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedAmount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - }, - "shippingMethodState": { - "type": ["null", "string"] - } - } - }, - "syncInfo": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "externalId": { - "type": ["null", "string"] - }, - "syncedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "returnInfo": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - }, - "lineItemId": { - "type": ["null", "string"] - }, - "comment": { - "type": ["null", "string"] - }, - "shipmentState": { - "type": ["null", "string"] - }, - "paymentState": { - "type": ["null", "string"] - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "returnTrackingId": { - "type": ["null", "string"] - }, - "returnDate": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "discountCodes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discountCode": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "state": { - "type": ["null", "string"] - } - } - } - }, - "refusedGifts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "lastMessageSequenceNumber": { - "type": ["null", "number"] - }, - "cart": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "paymentInfo": { - "type": ["null", "object"], - "properties": { - "payments": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "locale": { - "type": ["null", "string"] - }, - "inventoryMode": { - "type": ["null", "string"] - }, - "shippingRateInput": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "origin": { - "type": ["null", "string"] - }, - "itemShippingAddresses": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "discount_codes", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "description": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "code": { - "type": ["null", "string"] - }, - "cartDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "groups": { - "type": ["null", "array"] - }, - "isActive": { - "type": ["null", "boolean"] - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "references": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "maxApplications": { - "type": ["null", "number"] - }, - "maxApplicationsPerCustomer": { - "type": ["null", "number"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "payments", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - }, - "interfaceId": { - "type": ["null", "string"] - }, - "amountPlanned": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "paymentMethodInfo": { - "type": ["null", "object"], - "properties": { - "paymentInterface": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - } - } - }, - "paymentStatus": { - "type": ["null", "object"], - "properties": { - "interfaceCode": { - "type": ["null", "string"] - }, - "interfaceText": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "transactions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "interactionId": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - } - }, - "interfaceInteractions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "productType": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "masterData": { - "type": ["null", "object"], - "properties": { - "published": { - "type": ["null", "boolean"] - }, - "current": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "categories": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "categoryOrderHints": { - "type": ["null", "object"] - }, - "description": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "slug": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaTitle": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaDescription": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaKeywords": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "masterVariant": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - }, - "variants": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - } - }, - "searchKeywords": { - "type": ["null", "object"], - "properties": { - "text": { - "type": ["null", "string"] - }, - "suggestTokenizer": { - "type": ["null", "string"] - } - } - } - } - }, - "staged": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "categories": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "categoryOrderHints": { - "type": ["null", "object"] - }, - "description": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "slug": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaTitle": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaDescription": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaKeywords": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "masterVariant": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - }, - "variants": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - } - }, - "searchKeywords": { - "type": ["null", "object"], - "properties": { - "text": { - "type": ["null", "string"] - }, - "suggestTokenizer": { - "type": ["null", "string"] - } - } - } - } - }, - "hasStagedChanges": { - "type": ["null", "boolean"] - } - } - }, - "taxCategory": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "reviewRatingStatistics": { - "type": ["null", "object"], - "properties": { - "averageRating": { - "type": ["null", "number"] - }, - "highestRating": { - "type": ["null", "number"] - }, - "lowestRating": { - "type": ["null", "number"] - }, - "count": { - "type": ["null", "number"] - }, - "ratingsDistribution": { - "type": ["null", "object"] - } - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["lastModifiedAt"], - "source_defined_primary_key": [["id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json index 080c28a73612..819e9f44cf81 100644 --- a/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-commercetools/integration_tests/configured_catalog.json @@ -3,389 +3,7 @@ { "stream": { "name": "customers", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "customerNumber": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": " date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": " date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "email": { - "type": ["null", "string"] - }, - "password": { - "type": ["null", "string"] - }, - "stores": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - } - } - } - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "middleName": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "dateOfBirth": { - "type": ["null", "string"], - "format": "date" - }, - "companyName": { - "type": ["null", "string"] - }, - "vatId": { - "type": ["null", "string"] - }, - "addresses": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "defaultShippingAddressId": { - "type": ["null", "string"] - }, - "shippingAddressIds": { - "type": ["null", "array"] - }, - "defaultBillingAddressId": { - "type": ["null", "string"] - }, - "billingAddressIds": { - "type": ["null", "array"] - }, - "isEmailVerified": { - "type": ["null", "boolean"] - }, - "externalId": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "locale": { - "type": ["null", "string"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["lastModifiedAt"] @@ -397,3181 +15,7 @@ { "stream": { "name": "orders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "completedAt": { - "type": ["null", "integer"], - "format": "date-time" - }, - "orderNumber": { - "type": ["null", "string"] - }, - "customerId": { - "type": ["null", "string"] - }, - "customerEmail": { - "type": ["null", "string"] - }, - "anonymousId": { - "type": ["null", "string"] - }, - "store": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - } - } - }, - "lineItems": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "productId": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "productSlug": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "productType": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "variant": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - }, - "totalPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "quantity": { - "type": ["null", "number"] - }, - "addedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "state": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "taxRate": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "includedInPrice": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "subRates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - } - } - }, - "supplyChannel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "distributionChannel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedPricePerQuantity": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "discountedPrice": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "includedDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedAmount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - } - } - }, - "priceMode": { - "type": ["null", "string"] - }, - "lineItemMode": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "shippingDetails": { - "type": ["null", "object"], - "properties": { - "targets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "addressKey": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - }, - "valid": { - "type": ["null", "boolean"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "customLineItems": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "money": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - }, - "totalPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "slug": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "taxCategory": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "taxRate": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "includedInPrice": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "subRates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discountedPricePerQuantity": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "quantity": { - "type": ["null", "number"] - }, - "discountedPrice": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "includedDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedAmount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "shippingDetails": { - "type": ["null", "object"], - "properties": { - "targets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "addressKey": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - }, - "valid": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "totalPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "taxPortions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "rate": { - "type": ["null", "number"] - }, - "amount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - }, - "shippingAddress": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "billingAddress": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "taxMode": { - "type": ["null", "string"] - }, - "taxRoundingMode": { - "type": ["null", "string"] - }, - "taxCalculationMode": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "country": { - "type": ["null", "string"] - }, - "orderState": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "shipmentState": { - "type": ["null", "string"] - }, - "paymentState": { - "type": ["null", "string"] - }, - "shippingInfo": { - "type": ["null", "object"], - "properties": { - "shippingMethodName": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "shippingRate": { - "type": ["null", "object"], - "properties": { - "price": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "freeAbove": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "tiers": { - "type": ["null", "array"] - }, - "isMatching": { - "type": ["null", "boolean"] - } - } - }, - "taxedPrice": { - "type": ["null", "object"], - "properties": { - "totalNet": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "totalGross": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - }, - "taxRate": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "includedInPrice": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "subRates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - } - } - }, - "taxCategory": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "shippingMethod": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "deliveries": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - }, - "parcels": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "measurements": { - "type": ["null", "string"] - }, - "trackingData": { - "type": ["null", "object"], - "properties": { - "trackingId": { - "type": ["null", "string"] - }, - "carrier": { - "type": ["null", "string"] - }, - "provider": { - "type": ["null", "string"] - }, - "providerTransaction": { - "type": ["null", "string"] - }, - "isReturn": { - "type": ["null", "boolean"] - } - } - }, - "items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - } - } - } - } - } - } - }, - "address": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - } - } - }, - "discountedPrice": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "includedDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "discountedAmount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - } - } - }, - "shippingMethodState": { - "type": ["null", "string"] - } - } - }, - "syncInfo": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "externalId": { - "type": ["null", "string"] - }, - "syncedAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "returnInfo": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - }, - "lineItemId": { - "type": ["null", "string"] - }, - "comment": { - "type": ["null", "string"] - }, - "shipmentState": { - "type": ["null", "string"] - }, - "paymentState": { - "type": ["null", "string"] - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "returnTrackingId": { - "type": ["null", "string"] - }, - "returnDate": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "discountCodes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discountCode": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "state": { - "type": ["null", "string"] - } - } - } - }, - "refusedGifts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "lastMessageSequenceNumber": { - "type": ["null", "number"] - }, - "cart": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - }, - "paymentInfo": { - "type": ["null", "object"], - "properties": { - "payments": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "locale": { - "type": ["null", "string"] - }, - "inventoryMode": { - "type": ["null", "string"] - }, - "shippingRateInput": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "origin": { - "type": ["null", "string"] - }, - "itemShippingAddresses": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "salutation": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "streetName": { - "type": ["null", "string"] - }, - "streetNumber": { - "type": ["null", "string"] - }, - "additionalStreetInfo": { - "type": ["null", "string"] - }, - "postalCode": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "building": { - "type": ["null", "string"] - }, - "apartment": { - "type": ["null", "string"] - }, - "pOBox": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "mobile": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "fax": { - "type": ["null", "string"] - }, - "additionalAddressInfo": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["lastModifiedAt"] @@ -3583,244 +27,7 @@ { "stream": { "name": "discount_codes", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "description": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "code": { - "type": ["null", "string"] - }, - "cartDiscounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "groups": { - "type": ["null", "array"] - }, - "isActive": { - "type": ["null", "boolean"] - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "references": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "maxApplications": { - "type": ["null", "number"] - }, - "maxApplicationsPerCustomer": { - "type": ["null", "number"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["lastModifiedAt"] @@ -3832,355 +39,7 @@ { "stream": { "name": "payments", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - }, - "interfaceId": { - "type": ["null", "string"] - }, - "amountPlanned": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "paymentMethodInfo": { - "type": ["null", "object"], - "properties": { - "paymentInterface": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - } - } - }, - "paymentStatus": { - "type": ["null", "object"], - "properties": { - "interfaceCode": { - "type": ["null", "string"] - }, - "interfaceText": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "transactions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "interactionId": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - } - }, - "interfaceInteractions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["lastModifiedAt"] @@ -4192,3512 +51,7 @@ { "stream": { "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "number"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "createdBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "lastModifiedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastModifiedBy": { - "type": "object", - "properties": { - "clientId": { - "type": ["null", "string"] - }, - "externalUserId": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "anonymousId": { - "type": ["null", "string"] - } - } - }, - "productType": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "masterData": { - "type": ["null", "object"], - "properties": { - "published": { - "type": ["null", "boolean"] - }, - "current": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "categories": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "categoryOrderHints": { - "type": ["null", "object"] - }, - "description": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "slug": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaTitle": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaDescription": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaKeywords": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "masterVariant": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - }, - "variants": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - } - }, - "searchKeywords": { - "type": ["null", "object"], - "properties": { - "text": { - "type": ["null", "string"] - }, - "suggestTokenizer": { - "type": ["null", "string"] - } - } - } - } - }, - "staged": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "categories": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - }, - "categoryOrderHints": { - "type": ["null", "object"] - }, - "description": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "slug": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaTitle": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaDescription": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "metaKeywords": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "masterVariant": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - }, - "variants": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "prices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "attributes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "price": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "tiers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "minimumQuantity": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - } - } - } - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "label": { - "type": ["null", "string"] - } - } - } - }, - "assets": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "sources": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "uri": { - "type": ["null", "string"] - }, - "key": { - "type": ["null", "string"] - }, - "dimensions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "w": { - "type": ["null", "number"] - }, - "h": { - "type": ["null", "number"] - } - } - } - }, - "contentType": { - "type": ["null", "string"] - } - } - } - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "array"] - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - } - }, - "availability": { - "type": ["null", "object"], - "properties": { - "isOnStock": { - "type": ["null", "boolean"] - }, - "restockableInDays": { - "type": ["null", "number"] - }, - "availableQuantity": { - "type": ["null", "number"] - }, - "channels": { - "type": ["null", "object"] - } - } - }, - "isMatchingVariant": { - "type": ["null", "boolean"] - }, - "scopedPrice": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "currentValue": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customerGroup": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "channel": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "validFrom": { - "type": ["null", "string"], - "format": "date-time" - }, - "validUntil": { - "type": ["null", "string"], - "format": "date-time" - }, - "discounted": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "currencyCode": { - "type": ["null", "string"] - }, - "centAmount": { - "type": ["null", "number"] - }, - "fractionDigits": { - "type": ["null", "number"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "fields": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "object"], - "properties": { - "^.*$": { - "type": ["null", "string"] - } - } - }, - "required": { - "type": ["null", "boolean"] - }, - "inputHint": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "scopedPriceDiscounted": { - "type": ["null", "boolean"] - } - } - } - }, - "searchKeywords": { - "type": ["null", "object"], - "properties": { - "text": { - "type": ["null", "string"] - }, - "suggestTokenizer": { - "type": ["null", "string"] - } - } - } - } - }, - "hasStagedChanges": { - "type": ["null", "boolean"] - } - } - }, - "taxCategory": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "state": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "obj": { - "type": ["null", "object"], - "properties": { - "typeId": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } - } - }, - "reviewRatingStatistics": { - "type": ["null", "object"], - "properties": { - "averageRating": { - "type": ["null", "number"] - }, - "highestRating": { - "type": ["null", "number"] - }, - "lowestRating": { - "type": ["null", "number"] - }, - "count": { - "type": ["null", "number"] - }, - "ratingsDistribution": { - "type": ["null", "object"] - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["lastModifiedAt"] diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index 1bb46ca21940..dcbb88d3fafe 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - auth.${region}.${host}.commercetools.com + - api.${region}.${host}.commercetools.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 008b2e26-11a3-11ec-82a8-0242ac130003 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-commercetools githubIssueLabel: source-commercetools icon: commercetools.svg license: MIT name: Commercetools - registries: - cloud: - enabled: false - oss: - enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/commercetools tags: - - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-commercetools/requirements.txt b/airbyte-integrations/connectors/source-commercetools/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-commercetools/requirements.txt +++ b/airbyte-integrations/connectors/source-commercetools/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-commercetools/setup.py b/airbyte-integrations/connectors/source-commercetools/setup.py index 7b8dab56e751..9622d7ad2c24 100644 --- a/airbyte-integrations/connectors/source-commercetools/setup.py +++ b/airbyte-integrations/connectors/source-commercetools/setup.py @@ -5,15 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "connector-acceptance-test", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_commercetools", @@ -22,7 +16,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/__init__.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/__init__.py index 54400e0e1827..da64652a612c 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/__init__.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py new file mode 100644 index 000000000000..f39ab7dd6f28 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/components.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from dataclasses import dataclass + +import backoff +import requests +from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + +logger = logging.getLogger("airbyte") + + +@dataclass +class CommerceToolsOauth2Authenticator(DeclarativeOauth2Authenticator): + @backoff.on_exception( + backoff.expo, + DefaultBackoffException, + on_backoff=lambda details: logger.info( + f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." + ), + max_time=300, + ) + def _get_refresh_access_token_response(self): + region = self.config["region"] + project_key = self.config["project_key"] + host = self.config["host"] + url = f"https://auth.{region}.{host}.commercetools.com/oauth/token?grant_type=client_credentials&scope=manage_project:{project_key}" + try: + response = requests.post(url, auth=(self.config["client_id"], self.config["client_secret"])) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + if e.response.status_code == 429 or e.response.status_code >= 500: + raise DefaultBackoffException(request=e.response.request, response=e.response) + raise + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml new file mode 100644 index 000000000000..8bc1b738f224 --- /dev/null +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/manifest.yaml @@ -0,0 +1,178 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - customers + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: offset + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: OffsetIncrement + page_size: 500 + + requester: + type: HttpRequester + url_base: >- + https://api.{{ config['region'] }}.{{ config['host'] }}.commercetools.com/{{ config['project_key'] }} + path: "{{ parameters.path }}" + http_method: GET + request_parameters: + sort: lastModifiedAt asc + where: lastModifiedAt >= :date + request_headers: {} + authenticator: + type: CustomAuthenticator + class_name: source_commercetools.components.CommerceToolsOauth2Authenticator + client_id: "{{ config['client_id'] }}" + client_secret: "{{ config['client_secret'] }}" + refresh_request_body: + scope: manage_project:{{ config['project_key'] }} + token_refresh_endpoint: https://auth.{{ config['region'] }}.{{ config['host'] }}.commercetools.com/oauth/token + grant_type: client_credentials + access_token_name: access_token + request_body_json: {} + + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + + incremental_sync: + type: DatetimeBasedCursor + cursor_field: lastModifiedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + start_time_option: + inject_into: request_parameter + field_name: var.date + type: RequestOption + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + + base_stream: + type: DeclarativeStream + primary_key: id + retriever: + $ref: "#/definitions/retriever" + incremental_sync: + $ref: "#/definitions/incremental_sync" + + customers_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "customers" + path: "customers" + + discount_codes_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "discount_codes" + path: "discount-codes" + + orders_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "orders" + path: "orders" + + payments_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "payments" + path: "payments" + + products_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "products" + path: "products" + +streams: + - "#/definitions/customers_stream" + - "#/definitions/discount_codes_stream" + - "#/definitions/orders_stream" + - "#/definitions/payments_stream" + - "#/definitions/products_stream" + +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/commercetools + type: Spec + connection_specification: + additionalProperties: true + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - start_date + - client_id + - client_secret + - host + - project_key + - region + properties: + start_date: + type: string + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + order: 0 + client_id: + type: string + title: Client ID + airbyte_secret: true + description: "Id of API Client." + order: 1 + client_secret: + type: string + title: Client secret + airbyte_secret: true + description: "The password of secret of API Client." + order: 2 + host: + title: Host + description: >- + The cloud provider your shop is hosted. See: https://docs.commercetools.com/api/authorization + enum: + - gcp + - aws + type: string + order: 3 + project_key: + title: Project key + description: The project key + type: string + order: 4 + region: + title: Region + description: The region of the platform. + type: string + examples: ["us-central1", "australia-southeast1"] + order: 5 diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/customers.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/customers.json index 6db6a9b5d508..d2e66718b4d7 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/customers.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/customers.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -81,7 +82,10 @@ "type": ["null", "string"] }, "billingAddressIds": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "isEmailVerified": { "type": ["null", "boolean"] @@ -97,6 +101,15 @@ }, "locale": { "type": ["null", "string"] + }, + "authenticationMode": { + "type": ["null", "string"] + }, + "lastMessageSequenceNumber": { + "type": ["null", "integer"] + }, + "versionModifiedAt": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/discount_codes.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/discount_codes.json index b8bfcca75aef..3d0548e60e46 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/discount_codes.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/discount_codes.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -65,6 +66,28 @@ }, "custom": { "$ref": "custom.json" + }, + "applicationVersion": { + "type": ["null", "integer"] + }, + "attributeTypes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "cartFieldTypes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "customLineItemFieldTypes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "lineItemFieldTypes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "versionModifiedAt": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/orders.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/orders.json index 7c2f080ed8e0..9c0356e0e852 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/orders.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/orders.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -483,6 +484,34 @@ "origin": { "type": ["null", "string"] }, + "directDiscounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "shipping": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "shippingMode": { + "type": ["null", "string"] + }, + "taxedShippingPrice": { + "type": ["null", "object"], + "additionalProperties": true + }, + "transactionFee": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "versionModifiedAt": { + "type": ["null", "string"] + }, "itemShippingAddresses": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/payments.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/payments.json index f3111d3624a9..eaa2d94a59f9 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/payments.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/payments.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -39,6 +40,7 @@ }, "paymentMethodInfo": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "paymentInterface": { "type": ["null", "string"] @@ -53,6 +55,7 @@ }, "paymentStatus": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "interfaceCode": { "type": ["null", "string"] @@ -69,6 +72,7 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/products.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/products.json index 75f48e625ca0..236772a41113 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/products.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/products.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -30,6 +31,7 @@ }, "masterData": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "published": { "type": ["null", "boolean"] @@ -53,6 +55,7 @@ }, "reviewRatingStatistics": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "averageRating": { "type": ["null", "number"] @@ -67,9 +70,19 @@ "type": ["null", "number"] }, "ratingsDistribution": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true } } + }, + "lastMessageSequenceNumber": { + "type": ["null", "integer"] + }, + "lastVariantId": { + "type": ["null", "integer"] + }, + "versionModifiedAt": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/address.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/address.json index d8bd7d537f64..0bbc1ab9fe80 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/address.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/address.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/created_by.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/created_by.json index 8ece3d49e7ab..44d5d1f020cf 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/created_by.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/created_by.json @@ -1,5 +1,6 @@ { "type": "object", + "additionalProperties": true, "properties": { "clientId": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/custom.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/custom.json index 25f42a7a4715..af621b58729a 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/custom.json +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/schemas/shared/custom.json @@ -1,11 +1,13 @@ { "type": ["null", "object"], + "additionalProperties": true, "properties": { "type": { "$ref": "reference.json" }, "fields": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "type": { "$ref": "field_type.json" diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py b/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py index 2d6e8776a198..cdcca15f5522 100644 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py +++ b/airbyte-integrations/connectors/source-commercetools/source_commercetools/source.py @@ -2,180 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from base64 import b64encode -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class CommercetoolsStream(HttpStream, ABC): - - # Page size - limit = 500 - # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh - primary_key = "id" - filter_field = "lastModifiedAt" - offset = 0 - - def __init__(self, region: str, project_key: str, start_date: str, client_id: str, client_secret: str, host: str, **kwargs): - super().__init__(**kwargs) - self.start_date = start_date - self.project_key = project_key - self.region = region - self.client_id = client_id - self.client_secret = client_secret - self.host = host - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - json_response = response.json() - total = json_response.get("total", 0) - offset = json_response.get("offset", 0) - if offset + self.limit < total: - return dict(offset=offset + self.limit) - else: - return None - - @property - def url_base(self) -> str: - return f"https://api.{self.region}.{self.host}.commercetools.com/{self.project_key}/" - - def path(self, **kwargs) -> str: - return f"{self.data_field}" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"offset": self.offset, "limit": self.limit} - if next_page_token: - params.update(**next_page_token) - params.update({"sort": "lastModifiedAt asc"}) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - records = json_response.get("results", []) if json_response is not None else [] - yield from records - - -# Basic incremental stream -class IncrementalCommercetoolsStream(CommercetoolsStream, ABC): - @property - def limit(self): - return super().limit - - # Setting the default cursor field for all streams - cursor_field = "lastModifiedAt" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - - def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if not next_page_token: - params["limit"] = self.limit - params["sort"] = "lastModifiedAt asc" - if stream_state: - params["where"] = "lastModifiedAt >= :date" - params["var.date"] = stream_state.get(self.cursor_field) - return params - - # Parse the stream_slice with respect to stream_state for Incremental refresh - # cases where we slice the stream, the endpoints for those classes don't accept any other filtering, - # but they provide us with the updated_at field in most cases, so we used that as incremental filtering during the order slicing. - def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None, records_slice: Mapping[str, Any] = None) -> Iterable: - # Getting records >= state - if stream_state: - for record in records_slice: - if record[self.cursor_field] >= stream_state.get(self.cursor_field): - yield record - else: - yield from records_slice - - -class Customers(IncrementalCommercetoolsStream): - data_field = "customers" - - -class DiscountCodes(IncrementalCommercetoolsStream): - data_field = "discount-codes" - - -class Orders(IncrementalCommercetoolsStream): - data_field = "orders" - - -class Payments(IncrementalCommercetoolsStream): - data_field = "payments" - - -class Products(IncrementalCommercetoolsStream): - data_field = "products" - - -# Source -class SourceCommercetools(AbstractSource): - def _convert_auth_to_token(self, username: str, password: str) -> str: - username = username.encode("latin1") - password = password.encode("latin1") - token = b64encode(b":".join((username, password))).strip().decode("ascii") - return token - - def get_access_token(self, config) -> Tuple[str, any]: - region = config["region"] - project_key = config["project_key"] - host = config["host"] - url = f"https://auth.{region}.{host}.commercetools.com/oauth/token?grant_type=client_credentials&scope=manage_project:{project_key}" - - try: - response = requests.post(url, auth=(config["client_id"], config["client_secret"])) - response.raise_for_status() - json_response = response.json() - return json_response.get("access_token", None), None if json_response is not None else None, None - except requests.exceptions.RequestException as e: - return None, e - - def check_connection(self, logger, config) -> Tuple[bool, any]: - region = config["region"] - project_key = config["project_key"] - host = config["host"] - url = f"https://api.{region}.{host}.commercetools.com/{project_key}" - access_token = self.get_access_token(config) - token_value = access_token[0] - token_exception = access_token[1] - - if token_exception: - return False, token_exception - - if token_value: - auth = TokenAuthenticator(token=token_value).get_auth_header() - try: - response = requests.get(url, headers=auth) - response.raise_for_status() - return True, None - except requests.exceptions.RequestException as e: - return False, e - return False, "Token not found" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - - access_token = self.get_access_token(config) - auth = TokenAuthenticator(token=access_token[0]) - args = { - "authenticator": auth, - "region": config["region"], - "host": config["host"], - "project_key": config["project_key"], - "start_date": config["start_date"], - "client_id": config["client_id"], - "client_secret": config["client_secret"], - } - - return [Customers(**args), Orders(**args), DiscountCodes(**args), Payments(**args), Products(**args)] +# Declarative Source +class SourceCommercetools(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-commercetools/source_commercetools/spec.json b/airbyte-integrations/connectors/source-commercetools/source_commercetools/spec.json deleted file mode 100644 index 446cc77984b8..000000000000 --- a/airbyte-integrations/connectors/source-commercetools/source_commercetools/spec.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/commercetools", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Commercetools Source CDK Specifications", - "type": "object", - "required": [ - "region", - "start_date", - "host", - "project_key", - "client_id", - "client_secret" - ], - "additionalProperties": false, - "properties": { - "region": { - "type": "string", - "description": "The region of the platform.", - "examples": ["us-central1", "australia-southeast1"] - }, - "host": { - "type": "string", - "enum": ["gcp", "aws"], - "description": "The cloud provider your shop is hosted. See: https://docs.commercetools.com/api/authorization" - }, - "start_date": { - "type": "string", - "description": "The date you would like to replicate data. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "project_key": { - "type": "string", - "description": "The project key" - }, - "client_id": { - "type": "string", - "description": "Id of API Client.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "description": "The password of secret of API Client.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-commercetools/unit_tests/__init__.py b/airbyte-integrations/connectors/source-commercetools/unit_tests/__init__.py deleted file mode 100644 index 9db886e0930f..000000000000 --- a/airbyte-integrations/connectors/source-commercetools/unit_tests/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# diff --git a/airbyte-integrations/connectors/source-configcat/metadata.yaml b/airbyte-integrations/connectors/source-configcat/metadata.yaml index c31c8cfad669..b4725144c2f1 100644 --- a/airbyte-integrations/connectors/source-configcat/metadata.yaml +++ b/airbyte-integrations/connectors/source-configcat/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-configcat/requirements.txt b/airbyte-integrations/connectors/source-configcat/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-configcat/requirements.txt +++ b/airbyte-integrations/connectors/source-configcat/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-configcat/setup.py b/airbyte-integrations/connectors/source-configcat/setup.py index 96411643b25b..0bfbad2e7170 100644 --- a/airbyte-integrations/connectors/source-configcat/setup.py +++ b/airbyte-integrations/connectors/source-configcat/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-confluence/.dockerignore b/airbyte-integrations/connectors/source-confluence/.dockerignore index b98d666d0fe7..005263c3f44a 100644 --- a/airbyte-integrations/connectors/source-confluence/.dockerignore +++ b/airbyte-integrations/connectors/source-confluence/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_confluence !setup.py diff --git a/airbyte-integrations/connectors/source-confluence/Dockerfile b/airbyte-integrations/connectors/source-confluence/Dockerfile index 5fe034ed6ea3..cf556f1a8c5d 100644 --- a/airbyte-integrations/connectors/source-confluence/Dockerfile +++ b/airbyte-integrations/connectors/source-confluence/Dockerfile @@ -34,5 +34,5 @@ COPY source_confluence ./source_confluence ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-confluence diff --git a/airbyte-integrations/connectors/source-confluence/README.md b/airbyte-integrations/connectors/source-confluence/README.md index 90be582dab79..a884f5a32a49 100644 --- a/airbyte-integrations/connectors/source-confluence/README.md +++ b/airbyte-integrations/connectors/source-confluence/README.md @@ -1,35 +1,10 @@ # Confluence Source -This is the repository for the Confluence source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/confluence). +This is the repository for the Confluence configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/confluence). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/confluence) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_confluence/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/confluence) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_confluence/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source confluence test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-confluence:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-confluence:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-confluence/__init__.py b/airbyte-integrations/connectors/source-confluence/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-confluence/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml b/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml index 37206897a343..581340ee6eab 100644 --- a/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml @@ -4,7 +4,7 @@ connector_image: airbyte/source-confluence:dev acceptance_tests: spec: tests: - - spec_path: "source_confluence/spec.json" + - spec_path: "source_confluence/spec.yaml" connection: tests: - config_path: "secrets/config.json" @@ -21,6 +21,10 @@ acceptance_tests: expect_records: path: "integration_tests/expected_records.jsonl" fail_on_extra_columns: false + ignored_fields: + pages: + - name: body/view/value + bypass_reason: "Different class order" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/__init__.py b/airbyte-integrations/connectors/source-confluence/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-confluence/integration_tests/configured_catalog.json index 254b2beddfca..4c3d87c62037 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/configured_catalog.json @@ -5,7 +5,7 @@ "name": "audit", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["author"]] + "source_defined_primary_key": [["creationDate"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl index 533b65bb081f..5c8990995fce 100644 --- a/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-confluence/integration_tests/expected_records.jsonl @@ -1,5 +1,13 @@ -{"stream": "audit", "data": {"author": {"type": "user", "displayName": "System", "operations": null, "isExternalCollaborator": false, "accountType": "", "publicName": "Unknown user", "externalCollaborator": false}, "remoteAddress": "", "creationDate": "1682500900997", "summary": "User added to group", "description": "", "category": "Users and groups", "sysAdmin": false, "superAdmin": false, "affectedObject": {"name": "jira-servicemanagement-users-airbyteio:aab99a7c-3ce3-4123-b580-e4e00460754d", "objectType": "Group"}, "changedValues": [], "associatedObjects": [{"name": "Atlassian Assist", "objectType": "User"}]}, "emitted_at": 1683113208533} -{"stream": "pages", "data": {"id": "98487", "type": "page", "status": "current", "title": "Overview", "history": {"lastUpdated": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-02-17T09:08:30.734Z", "friendlyWhen": "Feb 17, 2023", "message": "", "number": 1, "minorEdit": false, "confRev": "confluence$content$98487.1", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/version/1"}}, "latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-02-17T09:08:30.734Z", "contributors": {"_expandable": {"publishers": ""}}, "_expandable": {"lastOwnedBy": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/history"}}, "version": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-02-17T09:08:30.734Z", "friendlyWhen": "Feb 17, 2023", "message": "", "number": 1, "minorEdit": false, "confRev": "confluence$content$98487.1", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/version/1"}}, "descendants": {"comment": {"results": [], "start": 0, "limit": 25, "size": 0, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/descendant/comment"}}, "_expandable": {"attachment": "/rest/api/content/98487/descendant/attachment", "page": "/rest/api/content/98487/descendant/page"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/descendant"}}, "macroRenderedOutput": {}, "body": {"storage": {"value": "\n\n

    Say hello to your colleagues who want to know your name, pronouns, role, team and location (or if you're remote).

    \n     \n\n

    \n \n\n

    \ud83d\udcc4 Recent pages that I've worked on

    \n     \n\n 5\n \n\n 5titles\n \n\n

    \ud83d\udd90 Get in touch

    \n     \n\n

    \u2709\ufe0f Insert your email here

    \n

    \ud83d\udcbc Insert your LinkedIn URL here

    \n     \n\n

    \ud83d\udd17 Insert your Twitter handle here

    \n

    \ud83d\udc64 Insert your Medium profile here

    \n     \n\n

    End with a bang! Some options are: "I am so grateful to be here at <Insert company name> and very excited to get started!" or "Looking forward to meeting all of you!" or "Can't wait to get to know all of you!"

    \n     ", "representation": "storage", "embeddedContent": [], "_expandable": {"content": "/rest/api/content/98487"}}, "view": {"value": "
    \n
    \n
    \n
    \n\n\n

    Say hello to your colleagues who want to know your name, pronouns, role, team and location (or if you're remote).

    \n
    \n
    \n
    \n
    \n\n\n

    \n

    \n
    \n
    \n
    \n
    \n
    \n\n\n

    \ud83d\udcc4 Recent pages that I've worked on

    \n
    \n
    \n
    \n
    \n
    \n
    \n\n\n \n\n
    \n\n

    Recently Updated

    \n
    \n \n
    \n
    \n \n
    \n
    \n\n
    \n
    \n
    \n
    \n\n\n \n \n\n
    \n

    Blog Posts

    \n \n
    \n\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n

    \ud83d\udd90 Get in touch

    \n
    \n
    \n
    \n
    \n
    \n
    \n\n\n

    \u2709\ufe0f

    \n

    \ud83d\udcbc

    \n
    \n
    \n
    \n
    \n\n\n

    \ud83d\udd17

    \n

    \ud83d\udc64

    \n
    \n
    \n
    \n
    \n
    \n
    \n\n\n

    End with a bang! Some options are: "I am so grateful to be here at <Insert company name> and very excited to get started!" or "Looking forward to meeting all of you!" or "Can't wait to get to know all of you!"

    \n
    \n
    \n
    \n
    ", "representation": "view", "_expandable": {"webresource": "", "embeddedContent": "", "mediaToken": "", "content": "/rest/api/content/98487"}}, "_expandable": {"editor": "", "atlas_doc_format": "", "export_view": "", "styled_view": "", "dynamic": "", "editor2": "", "anonymous_export_view": ""}}, "extensions": {"position": 959}, "restrictions": {"read": {"operation": "read", "restrictions": {"user": {"results": [], "start": 0, "limit": 200, "size": 0}, "_expandable": {"group": ""}}, "_expandable": {"content": "/rest/api/content/98487"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/restriction/byOperation/read"}}, "_expandable": {"update": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/restriction/byOperation"}}, "_expandable": {"childTypes": "", "container": "/rest/api/space/~5fc9e78d2730d800760becc4", "schedulePublishInfo": "", "metadata": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/98487/child", "ancestors": "", "space": "/rest/api/space/~5fc9e78d2730d800760becc4"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487", "tinyui": "/x/t4AB", "editui": "/pages/resumedraft.action?draftId=98487", "webui": "/spaces/~5fc9e78d2730d800760becc4/overview"}}, "emitted_at": 1685446909177} -{"stream": "blog_posts", "data": {"id": "1343496", "type": "blogpost", "status": "current", "title": "Blog post integration tests", "history": {"lastUpdated": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-03-06T15:54:15.290Z", "friendlyWhen": "Mar 06, 2023", "message": "", "number": 1, "minorEdit": false, "syncRev": "0.confluence$content$1343496.18", "syncRevSource": "synchrony-ack", "confRev": "confluence$content$1343496.19", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/1343496"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/version/1"}}, "latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-03-06T15:54:14.990Z", "contributors": {"_expandable": {"publishers": ""}}, "_expandable": {"lastOwnedBy": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/history"}}, "version": {"by": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "when": "2023-03-06T15:54:15.290Z", "friendlyWhen": "Mar 06, 2023", "message": "", "number": 1, "minorEdit": false, "syncRev": "0.confluence$content$1343496.18", "syncRevSource": "synchrony-ack", "confRev": "confluence$content$1343496.19", "contentTypeModified": false, "_expandable": {"collaborators": "", "content": "/rest/api/content/1343496"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/version/1"}}, "descendants": {"comment": {"results": [], "start": 0, "limit": 25, "size": 0, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/descendant/comment"}}, "_expandable": {"attachment": "/rest/api/content/1343496/descendant/attachment"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/descendant"}}, "macroRenderedOutput": {}, "body": {"storage": {"value": "

    Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

    ", "representation": "storage", "embeddedContent": [], "_expandable": {"content": "/rest/api/content/1343496"}}, "view": {"value": "

    Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

    ", "representation": "view", "_expandable": {"webresource": "", "embeddedContent": "", "mediaToken": "", "content": "/rest/api/content/1343496"}}, "_expandable": {"editor": "", "atlas_doc_format": "", "export_view": "", "styled_view": "", "dynamic": "", "editor2": "", "anonymous_export_view": ""}}, "restrictions": {"read": {"operation": "read", "restrictions": {"user": {"results": [], "start": 0, "limit": 200, "size": 0}, "_expandable": {"group": ""}}, "_expandable": {"content": "/rest/api/content/1343496"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/restriction/byOperation/read"}}, "_expandable": {"update": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/restriction/byOperation"}}, "_expandable": {"childTypes": "", "container": "/rest/api/space/~5fc9e78d2730d800760becc4", "schedulePublishInfo": "", "metadata": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/1343496/child", "ancestors": "", "space": "/rest/api/space/~5fc9e78d2730d800760becc4"}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496", "tinyui": "/x/CIAU", "editui": "/pages/resumedraft.action?draftId=1343496", "webui": "/spaces/~5fc9e78d2730d800760becc4/blog/2023/03/06/1343496/Blog+post+integration+tests"}}, "emitted_at": 1683113210956} -{"stream": "space", "data": {"id": 196612, "key": "AIRBYTE", "name": "Airbyte", "icon": {"path": "/download/attachments/196611/AIRBYTE-default?version=1&modificationDate=1676624930960&cacheVersion=1&api=v2", "width": 48, "height": 48, "isDefault": false}, "description": {"plain": {"value": "", "representation": "plain", "embeddedContent": []}, "view": {"value": "", "representation": "view", "embeddedContent": []}}, "type": "global", "permissions": [{"id": 196617, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196626, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196631, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196661, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196663, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196678, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196686, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196732, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196735, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229395, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229519, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196630, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196645, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196666, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196675, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196676, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196680, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196697, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196733, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196741, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229389, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229513, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196619, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196621, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196691, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196693, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196695, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196714, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196719, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196729, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196744, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229401, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229525, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196616, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196627, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196651, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196677, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196683, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196710, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196722, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196723, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196746, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229383, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229507, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196644, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196668, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196679, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196699, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196700, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196711, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196720, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196727, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196747, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196628, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196632, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196639, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196649, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196673, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196698, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196702, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196709, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196739, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229413, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229537, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196633, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196652, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196667, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196672, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196687, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196705, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196713, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196721, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196740, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229425, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229549, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196620, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196624, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196641, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196653, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196660, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196685, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196718, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196726, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196734, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229419, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229543, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196640, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196670, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196701, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196704, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196716, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196731, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196736, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196743, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196745, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196637, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196648, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196669, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196671, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196689, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196690, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196712, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196717, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196737, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229407, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229531, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196615, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196625, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196638, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196642, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196646, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196650, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196664, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196688, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196707, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196623, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196634, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196635, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196658, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196662, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196692, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196708, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196725, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229431, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229555, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196647, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196657, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196659, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196681, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196684, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196706, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196724, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196728, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 196742, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229377, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229501, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}], "status": "current", "_expandable": {"settings": "/rest/api/space/AIRBYTE/settings", "metadata": "", "operations": "", "lookAndFeel": "/rest/api/settings/lookandfeel?spaceKey=AIRBYTE", "identifiers": "", "theme": "/rest/api/space/AIRBYTE/theme", "history": "", "homepage": "/rest/api/content/196769"}, "_links": {"webui": "/spaces/AIRBYTE", "self": "https://airbyteio.atlassian.net/wiki/rest/api/space/AIRBYTE"}}, "emitted_at": 1683113211733} -{"stream": "group", "data": {"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}, "emitted_at": 1683113212280} +{"stream": "audit", "data": {"author": {"type": "user", "displayName": "System", "operations": null, "isExternalCollaborator": false, "accountType": "", "publicName": "Unknown user", "externalCollaborator": false}, "remoteAddress": "", "creationDate": 1690104397645, "summary": "User added to group", "description": "", "category": "Users and groups", "sysAdmin": false, "superAdmin": false, "affectedObject": {"name": "atlassian-addons-admin:90b9ffb1-ed26-4b5e-af59-8f684900ce83", "objectType": "Group"}, "changedValues": [], "associatedObjects": [{"name": "Atlas for Jira Cloud", "objectType": "User"}]}, "emitted_at": 1692382392348} +{"stream": "audit", "data": {"author": {"type": "user", "displayName": "System", "operations": null, "isExternalCollaborator": false, "accountType": "", "publicName": "Unknown user", "externalCollaborator": false}, "remoteAddress": "", "creationDate": 1690104397645, "summary": "User added to group", "description": "", "category": "Users and groups", "sysAdmin": false, "superAdmin": false, "affectedObject": {"name": "atlassian-addons:957dc3a7-4f7f-49ac-9945-073810b47fce", "objectType": "Group"}, "changedValues": [], "associatedObjects": [{"name": "Atlas for Jira Cloud", "objectType": "User"}]}, "emitted_at": 1692382392350} +{"stream": "audit", "data": {"author": {"type": "user", "displayName": "System", "operations": null, "isExternalCollaborator": false, "accountType": "", "publicName": "Unknown user", "externalCollaborator": false}, "remoteAddress": "", "creationDate": 1690104397644, "summary": "User added to group", "description": "", "category": "Users and groups", "sysAdmin": false, "superAdmin": false, "affectedObject": {"name": "jira-software-users:4452b254-035d-469a-a422-1f4666dce50e", "objectType": "Group"}, "changedValues": [], "associatedObjects": [{"name": "Atlas for Jira Cloud", "objectType": "User"}]}, "emitted_at": 1692382392352} +{"stream": "pages", "data": {"id": "98487", "type": "page", "status": "current", "title": "Overview", "history": {"latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-02-17T09:08:30.734Z", "_expandable": {"lastUpdated": "", "previousVersion": "", "lastOwnedBy": "", "contributors": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487/history"}}, "macroRenderedOutput": {}, "extensions": {"position": 959}, "_expandable": {"container": "/rest/api/space/~5fc9e78d2730d800760becc4", "metadata": "", "restrictions": "/rest/api/content/98487/restriction/byOperation", "body": "", "version": "", "descendants": "/rest/api/content/98487/descendant", "space": "/rest/api/space/~5fc9e78d2730d800760becc4", "childTypes": "", "schedulePublishInfo": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/98487/child", "ancestors": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/98487", "tinyui": "/x/t4AB", "editui": "/pages/resumedraft.action?draftId=98487", "webui": "/spaces/~5fc9e78d2730d800760becc4/overview"}}, "emitted_at": 1692382392630} +{"stream": "pages", "data": {"id": "196769", "type": "page", "status": "current", "title": "Airbyte", "history": {"latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-02-17T09:08:51.941Z", "_expandable": {"lastUpdated": "", "previousVersion": "", "lastOwnedBy": "", "contributors": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/196769/history"}}, "macroRenderedOutput": {}, "extensions": {"position": 38}, "_expandable": {"container": "/rest/api/space/AIRBYTE", "metadata": "", "restrictions": "/rest/api/content/196769/restriction/byOperation", "body": "", "version": "", "descendants": "/rest/api/content/196769/descendant", "space": "/rest/api/space/AIRBYTE", "childTypes": "", "schedulePublishInfo": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/196769/child", "ancestors": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/196769", "tinyui": "/x/oQAD", "editui": "/pages/resumedraft.action?draftId=196769", "webui": "/spaces/AIRBYTE/overview"}}, "emitted_at": 1692382392635} +{"stream": "pages", "data": {"id": "196828", "type": "page", "status": "current", "title": "Template - Project plan", "history": {"latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-02-17T09:08:57.011Z", "_expandable": {"lastUpdated": "", "previousVersion": "", "lastOwnedBy": "", "contributors": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/196828/history"}}, "macroRenderedOutput": {}, "extensions": {"position": 867}, "_expandable": {"container": "/rest/api/space/AIRBYTE", "metadata": "", "restrictions": "/rest/api/content/196828/restriction/byOperation", "body": "", "version": "", "descendants": "/rest/api/content/196828/descendant", "space": "/rest/api/space/AIRBYTE", "childTypes": "", "schedulePublishInfo": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/196828/child", "ancestors": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/196828", "tinyui": "/x/3AAD", "editui": "/pages/resumedraft.action?draftId=196828", "webui": "/spaces/AIRBYTE/pages/196828/Template+-+Project+plan"}}, "emitted_at": 1692382392639} +{"stream": "blog_posts", "data": {"id": "1343496", "type": "blogpost", "status": "current", "title": "Blog post integration tests", "history": {"latest": true, "createdBy": {"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}, "createdDate": "2023-03-06T15:54:14.990Z", "_expandable": {"lastUpdated": "", "previousVersion": "", "lastOwnedBy": "", "contributors": "", "nextVersion": "", "ownedBy": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496/history"}}, "macroRenderedOutput": {}, "_expandable": {"container": "/rest/api/space/~5fc9e78d2730d800760becc4", "metadata": "", "restrictions": "/rest/api/content/1343496/restriction/byOperation", "body": "", "version": "", "descendants": "/rest/api/content/1343496/descendant", "space": "/rest/api/space/~5fc9e78d2730d800760becc4", "childTypes": "", "schedulePublishInfo": "", "operations": "", "schedulePublishDate": "", "children": "/rest/api/content/1343496/child", "ancestors": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/content/1343496", "tinyui": "/x/CIAU", "editui": "/pages/resumedraft.action?draftId=1343496", "webui": "/spaces/~5fc9e78d2730d800760becc4/blog/2023/03/06/1343496/Blog+post+integration+tests"}}, "emitted_at": 1692382392894} +{"stream": "space", "data": {"id": 98306, "key": "~5fc9e78d2730d800760becc4", "name": "integration test", "type": "personal", "permissions": [{"id": 98354, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98371, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98417, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98438, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98442, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98444, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98462, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98475, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98477, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229455, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229579, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98351, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98363, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98369, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98386, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98391, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98434, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98460, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98472, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98473, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229449, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229573, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98358, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98364, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98365, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98393, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98396, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98413, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98433, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98446, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98450, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229461, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229585, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98373, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98374, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98376, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98390, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98408, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98422, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98441, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98461, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98474, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229443, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229567, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98350, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98352, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98394, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98415, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98419, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98439, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98454, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98456, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98476, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98347, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98366, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98388, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98389, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98424, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98436, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98440, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98449, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98452, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229473, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229597, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98349, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98380, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98382, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98383, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98387, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98405, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98425, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98451, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98468, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229485, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229609, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98360, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98372, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98401, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98402, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98406, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98423, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98432, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98464, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98478, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229479, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229603, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98348, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98385, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98392, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98400, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98414, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98418, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98437, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98443, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98455, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98346, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98375, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98377, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98378, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98404, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98420, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98426, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98457, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98479, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229467, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229591, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98368, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98379, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98407, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98409, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98429, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98435, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98467, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98469, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98471, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98356, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98359, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98381, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98384, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98399, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98403, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98416, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98427, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229491, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229615, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98362, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98367, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98370, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98398, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98421, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98430, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98445, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98458, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 98470, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229437, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 229561, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}], "status": "current", "_expandable": {"settings": "/rest/api/space/~5fc9e78d2730d800760becc4/settings", "metadata": "", "operations": "", "lookAndFeel": "/rest/api/settings/lookandfeel?spaceKey=~5fc9e78d2730d800760becc4", "identifiers": "", "icon": "", "description": "", "theme": "/rest/api/space/~5fc9e78d2730d800760becc4/theme", "history": "", "homepage": "/rest/api/content/98487"}, "_links": {"webui": "/spaces/~5fc9e78d2730d800760becc4", "self": "https://airbyteio.atlassian.net/wiki/rest/api/space/~5fc9e78d2730d800760becc4"}}, "emitted_at": 1692382393289} +{"stream": "space", "data": {"id": 2850818, "key": "TPM", "name": "Test Project Marketing", "type": "global", "permissions": [{"id": 2850833, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "archive", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850822, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850846, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850853, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850857, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850887, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850890, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850912, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850922, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850954, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850978, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851038, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850834, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850852, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850867, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850873, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850878, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850898, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850923, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850933, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850943, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850972, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851032, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850824, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850845, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850862, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850863, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850872, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850895, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850920, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850938, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850940, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850984, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851044, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850821, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850828, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850841, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850859, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850860, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850901, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850904, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850906, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850949, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850966, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851026, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850832, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850864, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850870, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850891, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850892, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850894, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850925, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850930, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850935, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850835, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850843, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850856, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850866, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850884, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850888, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850919, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850944, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850950, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850996, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851056, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850868, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850876, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850879, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850900, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850907, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850916, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850936, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850939, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850942, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851008, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851068, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850823, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850826, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850837, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850844, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850885, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850913, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850928, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850931, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850951, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851002, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851062, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850831, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850842, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850848, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850902, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850905, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850909, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850911, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850932, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850948, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850861, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850875, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850880, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850903, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850914, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850918, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850945, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850946, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850947, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850990, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851050, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850825, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850850, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850865, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850881, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850886, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850896, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850897, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850926, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850929, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850820, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850839, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850851, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850854, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850858, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850899, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850927, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850953, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851014, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851074, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850847, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850855, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850869, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850874, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850883, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850889, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850910, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850915, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850952, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2850960, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2851020, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 2916353, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}], "status": "current", "_expandable": {"settings": "/rest/api/space/TPM/settings", "metadata": "", "operations": "", "lookAndFeel": "/rest/api/settings/lookandfeel?spaceKey=TPM", "identifiers": "", "icon": "", "description": "", "theme": "/rest/api/space/TPM/theme", "history": "", "homepage": "/rest/api/content/2851095"}, "_links": {"webui": "/spaces/TPM", "self": "https://airbyteio.atlassian.net/wiki/rest/api/space/TPM"}}, "emitted_at": 1692382393294} +{"stream": "space", "data": {"id": 4751362, "key": "TTMP2", "name": "Test Team Managed Project 2", "type": "global", "permissions": [{"id": 4751408, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "archive", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751368, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751370, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751387, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751407, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751411, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751423, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751432, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751442, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751444, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751529, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751591, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751369, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751389, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751394, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751420, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751422, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751429, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751460, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751464, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751496, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751523, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751585, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751388, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751391, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751409, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751412, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751433, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751434, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751449, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751453, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751465, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751535, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751597, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751375, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751377, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751380, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751390, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751404, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751436, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751455, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751463, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751467, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751517, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751579, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "create", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751374, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751383, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751415, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751419, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751428, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751441, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751459, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751470, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751482, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "export", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751365, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751381, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751413, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751418, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751446, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751451, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751488, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751489, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751495, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751547, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751609, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "attachment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751379, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751395, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751437, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751452, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751458, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751469, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751472, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751479, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751486, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751559, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751621, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "blogpost"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751373, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751398, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751400, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751430, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751439, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751443, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751474, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751481, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751492, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751553, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751615, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "comment"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751366, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751382, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751403, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751410, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751416, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751440, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751454, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751478, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751498, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751376, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751397, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751447, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751448, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751457, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751475, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751477, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751480, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751483, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751541, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751603, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "delete", "targetType": "page"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751367, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751396, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751399, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751431, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751435, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751466, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751476, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751490, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751493, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "restrict_content", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751378, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751384, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751386, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751417, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751426, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751468, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751473, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751484, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751565, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751627, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "administer", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751371, "subjects": {"group": {"results": [{"type": "group", "name": "site-admins", "id": "76dad095-fc1a-467a-88b4-fde534220985", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/site-admins"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751372, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-users", "id": "38d808e9-113f-45c4-817b-099e953b687a", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751405, "subjects": {"user": {"results": [{"type": "known", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "email": "integration-test@airbyte.io", "publicName": "integration test", "profilePicture": {"path": "/wiki/aa-avatar/5fc9e78d2730d800760becc4", "width": 48, "height": 48, "isDefault": false}, "displayName": "integration test", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5fc9e78d2730d800760becc4"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751406, "subjects": {"group": {"results": [{"type": "group", "name": "jira-users", "id": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-users"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751414, "subjects": {"group": {"results": [{"type": "group", "name": "jira-administrators", "id": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/jira-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751421, "subjects": {"group": {"results": [{"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751424, "subjects": {"group": {"results": [{"type": "group", "name": "administrators", "id": "0ca6e087-7a61-4986-a269-98fe268854a1", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751445, "subjects": {"group": {"results": [{"type": "group", "name": "confluence-admins-airbyteio", "id": "a605fff8-d2cf-4f80-852c-008f5bbddd45", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/confluence-admins-airbyteio"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751491, "subjects": {"group": {"results": [{"type": "group", "name": "system-administrators", "id": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/system-administrators"}}], "size": 1}, "_expandable": {"user": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751511, "subjects": {"user": {"results": [{"type": "known", "accountId": "6035864ce2020c0070b5285b", "accountType": "app", "email": "", "publicName": "Microsoft Teams for Confluence Cloud", "profilePicture": {"path": "/wiki/aa-avatar/6035864ce2020c0070b5285b", "width": 48, "height": 48, "isDefault": false}, "displayName": "Microsoft Teams for Confluence Cloud", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=6035864ce2020c0070b5285b"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}, {"id": 4751573, "subjects": {"user": {"results": [{"type": "known", "accountId": "5b70c8b80fd0ac05d389f5e9", "accountType": "app", "email": "", "publicName": "Chat Notifications", "profilePicture": {"path": "/wiki/aa-avatar/5b70c8b80fd0ac05d389f5e9", "width": 48, "height": 48, "isDefault": false}, "displayName": "Chat Notifications", "isExternalCollaborator": false, "_expandable": {"operations": "", "personalSpace": ""}, "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/api/user?accountId=5b70c8b80fd0ac05d389f5e9"}}], "size": 1}, "_expandable": {"group": ""}}, "operation": {"operation": "read", "targetType": "space"}, "anonymousAccess": false, "unlicensedAccess": false}], "status": "current", "_expandable": {"settings": "/rest/api/space/TTMP2/settings", "metadata": "", "operations": "", "lookAndFeel": "/rest/api/settings/lookandfeel?spaceKey=TTMP2", "identifiers": "", "icon": "", "description": "", "theme": "/rest/api/space/TTMP2/theme", "history": "", "homepage": "/rest/api/content/4751499"}, "_links": {"webui": "/spaces/TTMP2", "self": "https://airbyteio.atlassian.net/wiki/rest/api/space/TTMP2"}}, "emitted_at": 1692382393299} +{"stream": "group", "data": {"type": "group", "name": "test group 19", "id": "3c4fef5d-9721-4f20-9a68-346d222de3cf", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/test%20group%2019"}}, "emitted_at": 1692382393486} +{"stream": "group", "data": {"type": "group", "name": "test group 17", "id": "022bc924-ac57-442d-80c9-df042b73ad87", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/test%20group%2017"}}, "emitted_at": 1692382393488} +{"stream": "group", "data": {"type": "group", "name": "integration-test-group", "id": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "_links": {"self": "https://airbyteio.atlassian.net/wiki/rest/experimental/group/integration-test-group"}}, "emitted_at": 1692382393489} diff --git a/airbyte-integrations/connectors/source-confluence/metadata.yaml b/airbyte-integrations/connectors/source-confluence/metadata.yaml index 035440f653cf..f45628d976b2 100644 --- a/airbyte-integrations/connectors/source-confluence/metadata.yaml +++ b/airbyte-integrations/connectors/source-confluence/metadata.yaml @@ -2,22 +2,27 @@ data: allowedHosts: hosts: - ${subdomain}.atlassian.net + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: cf40a7f8-71f8-45ce-a7fa-fca053e4028c - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-confluence githubIssueLabel: source-confluence icon: confluence.svg license: MIT name: Confluence - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2021-11-05 releaseStage: beta + supportLevel: certified documentationUrl: https://docs.airbyte.com/integrations/sources/confluence tags: - - language:python + - language:low-code + ab_internal: + sl: 200 + ql: 300 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-confluence/requirements.txt b/airbyte-integrations/connectors/source-confluence/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-confluence/requirements.txt +++ b/airbyte-integrations/connectors/source-confluence/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-confluence/setup.py b/airbyte-integrations/connectors/source-confluence/setup.py index 8b68dfa4ae34..aaacdff5bbe3 100644 --- a/airbyte-integrations/connectors/source-confluence/setup.py +++ b/airbyte-integrations/connectors/source-confluence/setup.py @@ -6,24 +6,24 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", ] setup( name="source_confluence", description="Source implementation for Confluence.", - author="Tuan Nguyen", - author_email="anhtuan.nguyen@me.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/__init__.py b/airbyte-integrations/connectors/source-confluence/source_confluence/__init__.py index b73058e73def..70c4df585097 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/__init__.py +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/manifest.yaml b/airbyte-integrations/connectors/source-confluence/source_confluence/manifest.yaml new file mode 100644 index 000000000000..1af4e0e43f9b --- /dev/null +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/manifest.yaml @@ -0,0 +1,159 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - space + +definitions: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_confluence/schemas/{{ parameters.name }}.json" + + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + + offset_increment_paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: start + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: OffsetIncrement + page_size: 25 + + base_retriever: + type: SimpleRetriever + paginator: + $ref: "#/definitions/offset_increment_paginator" + record_selector: + $ref: "#/definitions/selector" + + requester: + type: HttpRequester + url_base: https://{{ config['domain_name'] }}/wiki/rest/api/ + path: "{{ parameters.path }}" + http_method: GET + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['email'] }}" + password: "{{ config['api_token'] }}" + request_body_json: {} + request_headers: {} + + base_stream: + type: DeclarativeStream + schema_loader: + $ref: "#/definitions/schema_loader" + + audit_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: {} + primary_key: "creationDate" + $parameters: + name: "audit" + path: "audit" + + blogposts_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + expand: >- + ["history","history.lastUpdated","history.previousVersion","history.contributors","restrictions.read.restrictions.user","version","descendants.comment","body","body.storage","body.view",] + primary_key: "id" + $parameters: + name: "blog_posts" + path: content?type=blogpost + + group_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: {} + primary_key: "id" + $parameters: + name: "group" + path: "group" + + pages_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + expand: >- + ["history","history.lastUpdated","history.previousVersion","history.contributors","restrictions.read.restrictions.user","version","descendants.comment","body","body.storage","body.view",] + primary_key: "id" + $parameters: + name: "pages" + path: "content?type=page" + + space_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + expand: '["permissions","icon","description.plain","description.view"]' + primary_key: "id" + $parameters: + name: "space" + path: "space" + +streams: + - "#/definitions/audit_stream" + - "#/definitions/blogposts_stream" + - "#/definitions/group_stream" + - "#/definitions/pages_stream" + - "#/definitions/space_stream" + +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/confluence + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - email + - api_token + - domain_name + properties: + email: + type: string + title: Email + description: "Your Confluence login email" + examples: ["abc@example.com"] + order: 0 + api_token: + type: string + title: "API Token" + description: 'Please follow the Jira confluence for generating an API token: generating an API token.' + airbyte_secret: true + order: 1 + domain_name: + title: "Domain name" + description: "Your Confluence domain name" + type: string + order: 2 diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/audit.json b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/audit.json index ee1d7cca3f43..ad66fad449d9 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/audit.json +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/audit.json @@ -1,10 +1,11 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "author": { "type": ["null", "object"] }, "remoteAddress": { "type": ["null", "string"] }, - "creationDate": { "type": ["null", "string"] }, + "creationDate": { "type": ["null", "integer"] }, "summary": { "type": ["null", "string"] }, "description": { "type": ["null", "string"] }, "category": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/blog_posts.json b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/blog_posts.json index 023d04af6716..b688d257f5e4 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/blog_posts.json +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/blog_posts.json @@ -101,6 +101,7 @@ } }, "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string" diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/group.json b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/group.json index e430f4dfaa03..fba8eff62a50 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/group.json +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/group.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/pages.json b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/pages.json index 5aee16977001..2e2e0351c21e 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/pages.json +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/pages.json @@ -101,6 +101,7 @@ } }, "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string" @@ -161,11 +162,11 @@ }, "body": { "type": "object", - "properties": { + "properties": { "storage": { "type": "object", - "properties": { - "value": { + "properties": { + "value": { "type": "string" }, "representation": { @@ -176,7 +177,7 @@ }, "_expandable": { "type": "object", - "properties": { + "properties": { "content": { "type": "string" } @@ -186,8 +187,8 @@ }, "view": { "type": "object", - "properties": { - "value": { + "properties": { + "value": { "type": "string" }, "representation": { @@ -195,8 +196,8 @@ }, "_expandable": { "type": "object", - "properties": { - "webresource": { + "properties": { + "webresource": { "type": "string" } } diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/space.json b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/space.json index a021d235c3ab..81eef4aa9e54 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/space.json +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/schemas/space.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] }, "key": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/source.py b/airbyte-integrations/connectors/source-confluence/source_confluence/source.py index a0d53554b292..2668415967ef 100644 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/source.py +++ b/airbyte-integrations/connectors/source-confluence/source_confluence/source.py @@ -2,166 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from abc import ABC -from base64 import b64encode -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -logger = logging.getLogger("airbyte") +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class ConfluenceStream(HttpStream, ABC): - url_base = "https://{}/wiki/rest/api/" - primary_key = "id" - limit = 50 - start = 0 - expand = [] - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, config: Dict): - super().__init__(authenticator=config["authenticator"]) - self.config = config - self.url_base = self.url_base.format(config["domain_name"]) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - json_response = response.json() - links = json_response.get("_links") - next_link = links.get("next") - if next_link: - self.start += self.limit - return {"start": self.start} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit, "expand": ",".join(self.expand)} - if next_page_token: - params.update({"start": next_page_token["start"]}) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - records = json_response.get("results", []) - yield from records - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.api_name - - -class BaseContentStream(ConfluenceStream, ABC): - api_name = "content" - expand = [ - "history", - "history.lastUpdated", - "history.previousVersion", - "history.contributors", - "restrictions.read.restrictions.user", - "version", - "descendants.comment", - "body", - "body.storage", - "body.view", - ] - limit = 25 - content_type = None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params.update({"type": self.content_type}) - return params - - -class Pages(BaseContentStream): - """ - API documentation: https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get - """ - - content_type = "page" - - -class BlogPosts(BaseContentStream): - """ - API documentation: https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get - """ - - content_type = "blogpost" - - -class Space(ConfluenceStream): - """ - API documentation: https://developer.atlassian.com/cloud/confluence/rest/api-group-space/#api-wiki-rest-api-space-get - """ - - api_name = "space" - expand = ["permissions", "icon", "description.plain", "description.view"] - - -class Group(ConfluenceStream): - """ - API documentation: https://developer.atlassian.com/cloud/confluence/rest/api-group-group/#api-wiki-rest-api-group-get - """ - - api_name = "group" - - -class Audit(ConfluenceStream): - """ - API documentation: https://developer.atlassian.com/cloud/confluence/rest/api-group-audit/#api-wiki-rest-api-audit-get - """ - - primary_key = "author" - api_name = "audit" - limit = 1000 - - -# Source -class HttpBasicAuthenticator(TokenAuthenticator): - def __init__(self, email: str, token: str, auth_method: str = "Basic", **kwargs): - auth_string = f"{email}:{token}".encode("utf8") - b64_encoded = b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) - - -class SourceConfluence(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - auth = HttpBasicAuthenticator(config["email"], config["api_token"], auth_method="Basic").get_auth_header() - url = f"https://{config['domain_name']}/wiki/rest/api/space" - try: - response = requests.get(url, headers=auth) - response.raise_for_status() - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def account_plan_validation(self, config, auth): - # stream Audit requires Premium or Standard Plan - url = f"https://{config['domain_name']}/wiki/rest/api/audit?limit=1" - is_premium_or_standard_plan = False - try: - response = requests.get(url, headers=auth.get_auth_header()) - response.raise_for_status() - is_premium_or_standard_plan = True - except requests.exceptions.RequestException as e: - logger.warning(f"An exception occurred while trying to access Audit stream: {str(e)}. Skipping this stream.") - finally: - return is_premium_or_standard_plan - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = HttpBasicAuthenticator(config["email"], config["api_token"], auth_method="Basic") - config["authenticator"] = auth - streams = [Pages(config), BlogPosts(config), Space(config), Group(config)] - if self.account_plan_validation(config, auth): - streams.append(Audit(config)) - return streams +# Declarative Source +class SourceConfluence(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-confluence/source_confluence/spec.json b/airbyte-integrations/connectors/source-confluence/source_confluence/spec.json deleted file mode 100644 index 8b596ebfe108..000000000000 --- a/airbyte-integrations/connectors/source-confluence/source_confluence/spec.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Confluence Spec", - "type": "object", - "required": ["api_token", "domain_name", "email"], - "additionalProperties": true, - "properties": { - "api_token": { - "title": "API Token", - "type": "string", - "description": "Please follow the Jira confluence for generating an API token: generating an API token.", - "airbyte_secret": true - }, - "domain_name": { - "title": "Domain name", - "type": "string", - "description": "Your Confluence domain name", - "examples": ["example.atlassian.net"] - }, - "email": { - "title": "Email", - "type": "string", - "description": "Your Confluence login email", - "examples": ["abc@example.com"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-confluence/unit_tests/__init__.py b/airbyte-integrations/connectors/source-confluence/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-confluence/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-confluence/unit_tests/conftest.py b/airbyte-integrations/connectors/source-confluence/unit_tests/conftest.py deleted file mode 100644 index f0854145f4ff..000000000000 --- a/airbyte-integrations/connectors/source-confluence/unit_tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest - - -@pytest.fixture(scope="session", name="config") -def config_fixture(): - return {"api_token": "test_api_key", "domain_name": "example.atlassian.net", "email": "test@example.com"} diff --git a/airbyte-integrations/connectors/source-confluence/unit_tests/test_source.py b/airbyte-integrations/connectors/source-confluence/unit_tests/test_source.py deleted file mode 100644 index bdf48d134f5e..000000000000 --- a/airbyte-integrations/connectors/source-confluence/unit_tests/test_source.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_confluence.source import SourceConfluence - - -def setup_responses(): - responses.add( - responses.GET, - "https://example.atlassian.net/wiki/rest/api/space", - json={"access_token": "test_api_key", "expires_in": 3600}, - ) - - -@responses.activate -def test_check_connection(config): - setup_responses() - source = SourceConfluence() - logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == (True, None) - - -def test_check_connection_failed(config): - source = SourceConfluence() - logger_mock = MagicMock() - assert source.check_connection(logger_mock, config)[0] is False - - -@responses.activate -def test_streams_count(config): - responses.add( - responses.GET, - "https://example.atlassian.net/wiki/rest/api/audit", - status=200, - ) - source = SourceConfluence() - streams = source.streams(config) - expected_streams_number = 5 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-confluence/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-confluence/unit_tests/test_streams.py deleted file mode 100644 index 9488966fe7e2..000000000000 --- a/airbyte-integrations/connectors/source-confluence/unit_tests/test_streams.py +++ /dev/null @@ -1,131 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pytest -import requests -from source_confluence.source import Audit, BaseContentStream, BlogPosts, ConfluenceStream, Group, Pages, Space - - -class TestsConfluenceStream: - confluence_stream_class = ConfluenceStream({"authenticator": MagicMock(), "domain_name": "test"}) - - def tests_url_base(self): - assert self.confluence_stream_class.url_base == "https://test/wiki/rest/api/" - - def test_primary_key(self): - assert self.confluence_stream_class.primary_key == "id" - - def test_limit(self): - assert self.confluence_stream_class.limit == 50 - - def test_start(self): - assert self.confluence_stream_class.start == 0 - - def test_expand(self): - assert self.confluence_stream_class.expand == [] - - def test_next_page_token(self, requests_mock): - url = "https://test.atlassian.net/wiki/rest/api/space" - requests_mock.get(url, status_code=200, json={"_links": {"next": "test link"}}) - response = requests.get(url) - assert self.confluence_stream_class.next_page_token(response=response) == {"start": 50} - - @pytest.mark.parametrize( - ("stream_state", "stream_slice", "next_page_token", "expected"), - [ - ({}, {}, {}, {'limit': 50, 'expand': ''}), - ], - ) - def test_request_params(self, stream_state, stream_slice, next_page_token, expected): - assert self.confluence_stream_class.request_params(stream_state, stream_slice, next_page_token) == expected - - def test_parse_response(self, requests_mock): - url = "https://test.atlassian.net/wiki/rest/api/space" - requests_mock.get(url, status_code=200, json={"results": ["test", "test1", "test3"]}) - response = requests.get(url) - assert list(self.confluence_stream_class.parse_response(response=response)) == ['test', 'test1', 'test3'] - - -class TestBaseContentStream: - base_content_stream_class = BaseContentStream({"authenticator": MagicMock(), "domain_name": "test"}) - - def test_path(self): - assert self.base_content_stream_class.path({}, {}, {}) == "content" - - def test_expand(self): - assert self.base_content_stream_class.expand == ["history", - "history.lastUpdated", - "history.previousVersion", - "history.contributors", - "restrictions.read.restrictions.user", - "version", - "descendants.comment", - "body", - "body.storage", - "body.view", - ] - - def test_limit(self): - assert self.base_content_stream_class.limit == 25 - - def test_content_type(self): - assert self.base_content_stream_class.content_type is None - - @pytest.mark.parametrize( - ("stream_state", "stream_slice", "next_page_token", "expected"), - [ - ({}, {}, {}, {'limit': 25, - 'expand': 'history,history.lastUpdated,history.previousVersion,history.contributors,restrictions.' - 'read.restrictions.user,version,descendants.comment,body,body.storage,body.view', - 'type': None}), - ], - ) - def test_request_params(self, stream_state, stream_slice, next_page_token, expected): - assert self.base_content_stream_class.request_params(stream_state, stream_slice, next_page_token) == expected - - -class TestPages: - pages_class = Pages({"authenticator": MagicMock(), "domain_name": "test"}) - - def test_content_type(self): - assert self.pages_class.content_type == "page" - - -class TestBlogPosts: - blog_posts_class = BlogPosts({"authenticator": MagicMock(), "domain_name": "test"}) - - def test_content_type(self): - assert self.blog_posts_class.content_type == "blogpost" - - -class TestSpace: - space_class = Space({"authenticator": MagicMock(), "domain_name": "test"}) - - def test_api_name(self): - assert self.space_class.api_name == "space" - - def test_expand(self): - assert self.space_class.expand == ["permissions", "icon", "description.plain", "description.view"] - - -class TestGroup: - group_class = Group({"authenticator": MagicMock(), "domain_name": "test"}) - - def test_api_name(self): - assert self.group_class.api_name == "group" - - -class TestAudit: - audit_class = Audit({"authenticator": MagicMock(), "domain_name": "test"}) - - def test_api_name(self): - assert self.audit_class.api_name == "audit" - - def test_primary_key(self): - assert self.audit_class.primary_key == "author" - - def test_limit(self): - assert self.audit_class.limit == 1000 diff --git a/airbyte-integrations/connectors/source-convertkit/metadata.yaml b/airbyte-integrations/connectors/source-convertkit/metadata.yaml index 828c0f007faa..66d558a615f6 100644 --- a/airbyte-integrations/connectors/source-convertkit/metadata.yaml +++ b/airbyte-integrations/connectors/source-convertkit/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-convertkit/requirements.txt b/airbyte-integrations/connectors/source-convertkit/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-convertkit/requirements.txt +++ b/airbyte-integrations/connectors/source-convertkit/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-convertkit/setup.py b/airbyte-integrations/connectors/source-convertkit/setup.py index c368b959d73d..a4a3f97a2b7d 100644 --- a/airbyte-integrations/connectors/source-convertkit/setup.py +++ b/airbyte-integrations/connectors/source-convertkit/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-convex/Dockerfile b/airbyte-integrations/connectors/source-convex/Dockerfile index 0bf904d7c506..472b313e8dd7 100644 --- a/airbyte-integrations/connectors/source-convex/Dockerfile +++ b/airbyte-integrations/connectors/source-convex/Dockerfile @@ -34,5 +34,5 @@ COPY source_convex ./source_convex ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-convex diff --git a/airbyte-integrations/connectors/source-convex/acceptance-test-config.yml b/airbyte-integrations/connectors/source-convex/acceptance-test-config.yml index 6b0426a2ba08..4f422a6a0546 100644 --- a/airbyte-integrations/connectors/source-convex/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-convex/acceptance-test-config.yml @@ -15,6 +15,8 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.1" # TODO: Need creds to run integration tests https://github.com/airbytehq/airbyte/issues/25571 # basic_read: # tests: diff --git a/airbyte-integrations/connectors/source-convex/metadata.yaml b/airbyte-integrations/connectors/source-convex/metadata.yaml index 6a4541aa2f26..911f0ab12cf9 100644 --- a/airbyte-integrations/connectors/source-convex/metadata.yaml +++ b/airbyte-integrations/connectors/source-convex/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: c332628c-f55c-4017-8222-378cfafda9b2 - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-convex githubIssueLabel: source-convex icon: convex.svg @@ -10,11 +10,15 @@ data: name: Convex registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/convex tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-convex/requirements.txt b/airbyte-integrations/connectors/source-convex/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-convex/requirements.txt +++ b/airbyte-integrations/connectors/source-convex/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-convex/setup.py b/airbyte-integrations/connectors/source-convex/setup.py index 298823d390a7..22937fa853f1 100644 --- a/airbyte-integrations/connectors/source-convex/setup.py +++ b/airbyte-integrations/connectors/source-convex/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", ] diff --git a/airbyte-integrations/connectors/source-convex/source_convex/source.py b/airbyte-integrations/connectors/source-convex/source_convex/source.py index 5df2576bdbc2..ecf094ff9c01 100644 --- a/airbyte-integrations/connectors/source-convex/source_convex/source.py +++ b/airbyte-integrations/connectors/source-convex/source_convex/source.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, TypedDict import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -30,6 +31,8 @@ }, ) +CONVEX_CLIENT_VERSION = "0.2.0" + # Source class SourceConvex(AbstractSource): @@ -37,7 +40,10 @@ def _json_schemas(self, config: ConvexConfig) -> requests.Response: deployment_url = config["deployment_url"] access_key = config["access_key"] url = f"{deployment_url}/api/json_schemas?deltaSchema=true&format=convex_json" - headers = {"Authorization": f"Convex {access_key}"} + headers = { + "Authorization": f"Convex {access_key}", + "Convex-Client": f"airbyte-export-{CONVEX_CLIENT_VERSION}", + } return requests.get(url, headers=headers) def check_connection(self, logger: Any, config: ConvexConfig) -> Tuple[bool, Any]: @@ -124,8 +130,16 @@ def state(self, value: ConvexState) -> None: self._delta_cursor_value = value["delta_cursor"] def next_page_token(self, response: requests.Response) -> Optional[ConvexState]: - # Inner level of pagination shares the same state as outer, - # and returns None to indicate that we're done. + if response.status_code != 200: + raise Exception(format_http_error("Failed request", response)) + resp_json = response.json() + if self._snapshot_has_more: + self._snapshot_cursor_value = resp_json["cursor"] + self._snapshot_has_more = resp_json["hasMore"] + self._delta_cursor_value = resp_json["snapshot"] + else: + self._delta_cursor_value = resp_json["cursor"] + self._delta_has_more = resp_json["hasMore"] return self.state if self._delta_has_more else None def path( @@ -150,13 +164,6 @@ def parse_response( if response.status_code != 200: raise Exception(format_http_error("Failed request", response)) resp_json = response.json() - if self._snapshot_has_more: - self._snapshot_cursor_value = resp_json["cursor"] - self._snapshot_has_more = resp_json["hasMore"] - self._delta_cursor_value = resp_json["snapshot"] - else: - self._delta_cursor_value = resp_json["cursor"] - self._delta_has_more = resp_json["hasMore"] return list(resp_json["values"]) def request_params( @@ -176,14 +183,28 @@ def request_params( params["cursor"] = self._delta_cursor_value return params + def request_headers( + self, + stream_state: ConvexState, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[ConvexState] = None, + ) -> Dict[str, str]: + """ + Custom headers for each HTTP request, not including Authorization. + """ + return { + "Convex-Client": f"airbyte-export-{CONVEX_CLIENT_VERSION}", + } + def get_updated_state(self, current_stream_state: ConvexState, latest_record: Mapping[str, Any]) -> ConvexState: """ This (deprecated) method is still used by AbstractSource to update state between calls to `read_records`. """ return self.state - def read_records(self, *args: Any, **kwargs: Any) -> Iterator[Any]: - for record in super().read_records(*args, **kwargs): + def read_records(self, sync_mode: SyncMode, *args: Any, **kwargs: Any) -> Iterator[Any]: + self._delta_has_more = sync_mode == SyncMode.incremental + for record in super().read_records(sync_mode, *args, **kwargs): ts_ns = record["_ts"] ts_seconds = ts_ns / 1e9 # convert from nanoseconds. # equivalent of java's `new Timestamp(transactionMillis).toInstant().toString()` @@ -203,5 +224,5 @@ def format_http_error(context: str, resp: requests.Response) -> str: try: err = resp.json() return f"{context}: {resp.status_code}: {err['code']}: {err['message']}" - except JSONDecodeError or KeyError: + except (JSONDecodeError, KeyError): return f"{context}: {resp.text}" diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py index 910f48ad1681..6f7a56c5577c 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py @@ -30,6 +30,7 @@ def test_get_updated_state(patch_incremental_base_class): resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 3000, "hasMore": True} resp.status_code = 200 stream.parse_response(resp, {}) + stream.next_page_token(resp) assert stream.get_updated_state(None, None) == { "snapshot_cursor": 1234, "snapshot_has_more": True, @@ -37,6 +38,7 @@ def test_get_updated_state(patch_incremental_base_class): } resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 1235}], "cursor": 1235, "snapshot": 3000, "hasMore": False} stream.parse_response(resp, {}) + stream.next_page_token(resp) assert stream.get_updated_state(None, None) == { "snapshot_cursor": 1235, "snapshot_has_more": False, @@ -44,6 +46,7 @@ def test_get_updated_state(patch_incremental_base_class): } resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 1235}], "cursor": 8000, "hasMore": True} stream.parse_response(resp, {}) + stream.next_page_token(resp) assert stream.get_updated_state(None, None) == { "snapshot_cursor": 1235, "snapshot_has_more": False, @@ -52,6 +55,7 @@ def test_get_updated_state(patch_incremental_base_class): assert stream._delta_has_more is True resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 1235}], "cursor": 9000, "hasMore": False} stream.parse_response(resp, {}) + stream.next_page_token(resp) assert stream.get_updated_state(None, None) == { "snapshot_cursor": 1235, "snapshot_has_more": False, diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py index c02bcd81f287..267126670024 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py @@ -70,13 +70,12 @@ def test_parse_response(patch_base_class): inputs = {"response": resp, "stream_state": {}} expected_parsed_objects = [{"_id": "my_id", "field": "f", "_ts": 1234}] assert stream.parse_response(**inputs) == expected_parsed_objects - assert stream.state == {"snapshot_cursor": 1234, "snapshot_has_more": True, "delta_cursor": 2000} def test_request_headers(patch_base_class): stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - assert stream.request_headers(**inputs) == {} + assert stream.request_headers(**inputs) == {"Convex-Client": "airbyte-export-0.2.0"} def test_http_method(patch_base_class): diff --git a/airbyte-integrations/connectors/source-copper/Dockerfile b/airbyte-integrations/connectors/source-copper/Dockerfile index 10473861984a..85e6e9425e6c 100644 --- a/airbyte-integrations/connectors/source-copper/Dockerfile +++ b/airbyte-integrations/connectors/source-copper/Dockerfile @@ -34,5 +34,5 @@ COPY source_copper ./source_copper ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-copper diff --git a/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml b/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml index bf675b8d0ec3..9d1a24bc8d47 100644 --- a/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml @@ -14,6 +14,8 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.0" basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-copper/metadata.yaml b/airbyte-integrations/connectors/source-copper/metadata.yaml index a519e4caae95..884ebdff532d 100644 --- a/airbyte-integrations/connectors/source-copper/metadata.yaml +++ b/airbyte-integrations/connectors/source-copper/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 44f3002f-2df9-4f6d-b21c-02cd3b47d0dc - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-copper githubIssueLabel: source-copper icon: copper.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/copper tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-copper/requirements.txt b/airbyte-integrations/connectors/source-copper/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-copper/requirements.txt +++ b/airbyte-integrations/connectors/source-copper/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-copper/setup.py b/airbyte-integrations/connectors/source-copper/setup.py index ad6e5ff73326..0ede569ed344 100644 --- a/airbyte-integrations/connectors/source-copper/setup.py +++ b/airbyte-integrations/connectors/source-copper/setup.py @@ -10,10 +10,10 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.21.0", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/TODO.md b/airbyte-integrations/connectors/source-copper/source_copper/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json index 81e9ce0abf49..e2f1947e515c 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json @@ -1,85 +1,105 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "address": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "street": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "null" + "type": ["null", "string"] }, "postal_code": { - "type": "string" + "type": ["null", "string"] }, "country": { - "type": "null" + "type": ["null", "string"] } } }, "assignee_id": { - "type": "integer" + "type": ["null", "integer"] }, "contact_type_id": { - "type": "integer" + "type": ["null", "integer"] }, "details": { - "type": "string" + "type": ["null", "string"] }, "email_domain": { - "type": "string" + "type": ["null", "string"] }, "socials": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "url": { - "type": "string" + "type": ["null", "string"] }, "category": { - "type": "string" + "type": ["null", "string"] } } } }, "tags": { - "type": "array", - "items": {} + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "websites": { - "type": "array", + "type": ["null", "array"], "items": { "type": ["object", "null"], + "additionalProperties": true, "properties": { "url": { - "type": "string" + "type": ["null", "string"] }, "category": { - "type": "string" + "type": ["null", "string"] } } } }, "interaction_count": { - "type": "integer" + "type": ["null", "integer"] }, "date_created": { - "type": "integer" + "type": ["null", "integer"] }, "date_modified": { - "type": "integer" + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "phone_numbers": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } } } } diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/opportunities.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/opportunities.json index 18091aec45b7..3fa2ead4e515 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/opportunities.json @@ -15,45 +15,47 @@ "close_date": { "type": ["null", "string"] }, - "company_id" : { - "type" : ["null", "string"] - }, - "company_name" : { - "type" : ["null", "string"] - }, - "customer_source_id" : { - "type" : ["null", "string"] - }, - "details" : { - "type" : ["null", "string"] - }, - "loss_reason_id" : { - "type" : ["null", "string"] - }, - "monetary_value" : { - "type" : ["null", "integer"] - }, - "pipeline_id" : { - "type" : ["null", "string"] - }, - "primary_contact_id" : { - "type" : ["null", "string"] - }, - "priority" : { - "type" : ["null", "string"] - }, - "pipeline_stage_id" : { - "type" : ["null", "string"] - }, - "status" : { - "type" : ["null", "string"] + "company_id": { + "type": ["null", "string"] + }, + "company_name": { + "type": ["null", "string"] + }, + "customer_source_id": { + "type": ["null", "string"] + }, + "details": { + "type": ["null", "string"] + }, + "loss_reason_id": { + "type": ["null", "string"] + }, + "monetary_value": { + "type": ["null", "integer"] + }, + "pipeline_id": { + "type": ["null", "string"] + }, + "primary_contact_id": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "string"] + }, + "pipeline_stage_id": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] }, "tags": { "type": ["null", "array"], - "items": {} + "items": { + "type": ["null", "string"] + } }, - "win_probability" : { - "type" : ["null", "number"] + "win_probability": { + "type": ["null", "number"] }, "date_created": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json index 45f6d33a5eee..f7f25b051543 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json @@ -1,33 +1,35 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "prefix": { - "type": "null" + "type": ["null", "string"] }, "first_name": { - "type": "string" + "type": ["null", "string"] }, "middle_name": { - "type": "null" + "type": ["null", "string"] }, "last_name": { - "type": "string" + "type": ["null", "string"] }, "suffix": { - "type": "null" + "type": ["null", "string"] }, "address": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "street": { - "type": "string" + "type": ["string", "null"] }, "city": { "type": ["string", "null"] @@ -44,55 +46,85 @@ } }, "assignee_id": { - "type": "integer" + "type": ["null", "integer"] }, "company_id": { - "type": "integer" + "type": ["null", "integer"] }, "company_name": { - "type": "string" + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "socials": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "leads_converted_from": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } }, "contact_type_id": { - "type": "integer" + "type": ["null", "integer"] }, "details": { - "type": "null" + "type": ["null", "string"] }, "emails": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "category": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] } } } }, "phone_numbers": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "number": { - "type": "string" + "type": ["null", "string"] }, "category": { - "type": "string" + "type": ["null", "string"] } } } }, "title": { - "type": "null" + "type": ["null", "string"] }, "websites": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "url": { "type": "string" @@ -104,7 +136,7 @@ } }, "date_created": { - "type": "integer" + "type": ["null", "integer"] }, "date_modified": { "type": ["integer", "null"] @@ -113,11 +145,10 @@ "type": ["integer", "null"] }, "interaction_count": { - "type": "integer" + "type": ["null", "integer"] }, "date_lead_created": { "type": ["integer", "null"] } - }, - "required": ["id"] + } } diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json index 0cc15febf620..611f3091f803 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json @@ -1,30 +1,44 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "related_resource": { "type": ["string", "null"] }, "assignee_id": { - "type": "integer" + "type": ["null", "integer"] }, "status": { - "type": "string" + "type": ["null", "string"] }, "details": { - "type": "string" + "type": ["null", "string"] }, "date_created": { - "type": "integer" + "type": ["null", "integer"] }, "date_modified": { - "type": "integer" + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-courier/metadata.yaml b/airbyte-integrations/connectors/source-courier/metadata.yaml index 0d0a6165858e..d2badc1d80db 100644 --- a/airbyte-integrations/connectors/source-courier/metadata.yaml +++ b/airbyte-integrations/connectors/source-courier/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-courier/requirements.txt b/airbyte-integrations/connectors/source-courier/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-courier/requirements.txt +++ b/airbyte-integrations/connectors/source-courier/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-courier/setup.py b/airbyte-integrations/connectors/source-courier/setup.py index 1b726126d376..883332ece7a6 100644 --- a/airbyte-integrations/connectors/source-courier/setup.py +++ b/airbyte-integrations/connectors/source-courier/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-datadog/Dockerfile b/airbyte-integrations/connectors/source-datadog/Dockerfile index 31c2458c5e31..37ad70df367c 100644 --- a/airbyte-integrations/connectors/source-datadog/Dockerfile +++ b/airbyte-integrations/connectors/source-datadog/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.2 LABEL io.airbyte.name=airbyte/source-datadog diff --git a/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml index 0d38ac0d8533..b9b95f0cb687 100644 --- a/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml @@ -1,25 +1,36 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-datadog:dev -tests: +acceptance_tests: spec: - - spec_path: "source_datadog/spec.yaml" + tests: + - spec_path: "source_datadog/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - - incident_teams - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/inc_configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "incident_teams" + bypass_reason: "Test account does not have data for this stream" + - name: "logs" + bypass_reason: "Test account does not have data for this stream" + # TODO: logs stream currently has no data. This will need to be added to test incremental. + # incremental: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/inc_configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-datadog/integration_tests/invalid_config.json index 6e27d0fcd606..3a273a156bb4 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/invalid_config.json @@ -1,4 +1,5 @@ { + "site": "", "api_key": "", "application_key": "", "query": "sample query", diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-datadog/integration_tests/sample_config.json index 9be54f53bd05..bb87d9ae80af 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/sample_config.json @@ -1,8 +1,16 @@ { + "site": "datadoghq.com", "api_key": "", "application_key": "", "query": "sample query", "limit": 100, "start_date": "2022-10-10T00:00:00Z", - "end_date": "2022-10-10T00:10:00Z" + "end_date": "2022-10-10T00:10:00Z", + "queries": [ + { + "name": "Query Name", + "data_source": "metrics", + "query": "sample query" + } + ] } diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml index d9f3ed3b5ac8..734c40ef9f90 100644 --- a/airbyte-integrations/connectors/source-datadog/metadata.yaml +++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 1cfc30c7-82db-43f4-9fd7-ac1b42312cda - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.2 dockerRepository: airbyte/source-datadog githubIssueLabel: source-datadog icon: datadog.svg @@ -10,11 +10,15 @@ data: name: Datadog registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/datadog tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-datadog/requirements.txt b/airbyte-integrations/connectors/source-datadog/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-datadog/requirements.txt +++ b/airbyte-integrations/connectors/source-datadog/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-datadog/setup.py b/airbyte-integrations/connectors/source-datadog/setup.py index f45e4bc4f7d8..8d2a7f0da611 100644 --- a/airbyte-integrations/connectors/source-datadog/setup.py +++ b/airbyte-integrations/connectors/source-datadog/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests==2.25.1"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/dashboards.json b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/dashboards.json index c767d3fadc03..14350c585a30 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/dashboards.json +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/dashboards.json @@ -13,6 +13,12 @@ "description": "Creation date of the dashboard.", "readOnly": true }, + "deleted_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "Deletion date of the dashboard.", + "readOnly": true + }, "description": { "type": ["string", "null"], "description": "Description of the dashboard.", diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/downtimes.json b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/downtimes.json index a996968a6ecd..c34ab884fba9 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/downtimes.json +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/downtimes.json @@ -17,6 +17,12 @@ "description": "If a scheduled downtime is canceled.", "readOnly": true }, + "child_id": { + "type": ["integer", "null"] + }, + "created": { + "type": ["integer", "null"] + }, "creator_id": { "type": ["integer", "null"], "description": "User ID of the downtime creator.", @@ -47,6 +53,9 @@ "description": "A message to include with notifications for this downtime.", "readOnly": true }, + "modified": { + "type": ["integer", "null"] + }, "monitor_id": { "type": ["integer", "null"], "description": "A single monitor to which the downtime applies. If not provided, the downtime applies to all monitors.", @@ -65,6 +74,21 @@ "description": "If the first recovery notification during a downtime should be muted.", "readOnly": true }, + "notify_end_states": { + "type": ["array", "null"], + "items": { + "type": "string" + } + }, + "notify_end_types": { + "type": ["array", "null"], + "items": { + "type": "string" + } + }, + "org_id": { + "type": ["integer", "null"] + }, "parent_id": { "type": ["integer", "null"], "description": "ID of the parent Downtime.", @@ -88,6 +112,9 @@ "description": "POSIX timestamp to start the downtime. If not provided, the downtime starts the moment it is created.", "readOnly": true }, + "status": { + "type": ["string", "null"] + }, "timezone": { "type": ["string", "null"], "description": "The timezone in which to display the downtime's start and end times in Datadog applications.", @@ -97,6 +124,9 @@ "type": ["integer", "null"], "description": "ID of the last user that updated the downtime.", "readOnly": true + }, + "uuid": { + "type": ["string", "null"] } }, "additionalProperties": true diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/metrics.json b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/metrics.json index c74e6c5a5915..ea7d6cc8cd1d 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/metrics.json @@ -1,5 +1,15 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": true + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"], + "description": "The metric name for this resource." + }, + "type": { + "type": ["null", "string"], + "description": "The metric resource type." + } + } } diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/synthetic_tests.json b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/synthetic_tests.json index a2b7d0e0d91b..d9e26f003a1f 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/synthetic_tests.json +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/synthetic_tests.json @@ -7,6 +7,12 @@ "description": "Configuration object for a Synthetic test.", "additionalProperties": true }, + "created_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "Creation date of the Synthetic test.", + "readOnly": true + }, "creator": { "type": ["object", "null"], "description": "Object describing the creator of the shared element.", @@ -39,6 +45,12 @@ "description": "Notification message associated with the test.", "readOnly": true }, + "modified_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "Creation date of the Synthetic test.", + "readOnly": true + }, "monitor_id": { "type": ["integer", "null"], "description": "The associated monitor ID.", diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/source.py b/airbyte-integrations/connectors/source-datadog/source_datadog/source.py index 175294ed65d2..90dcba7fb305 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/source.py +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from datetime import datetime from typing import Any, List, Mapping, Optional, Tuple @@ -11,7 +12,18 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from pydantic.datetime_parse import timedelta -from source_datadog.streams import AuditLogs, Dashboards, Downtimes, Incidents, IncidentTeams, Logs, Metrics, SyntheticTests, Users +from source_datadog.streams import ( + AuditLogs, + Dashboards, + Downtimes, + Incidents, + IncidentTeams, + Logs, + Metrics, + SeriesStream, + SyntheticTests, + Users, +) class SourceDatadog(AbstractSource): @@ -31,7 +43,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: args = self.connector_config(config) - return [ + base_streams = [ AuditLogs(**args), Dashboards(**args), Downtimes(**args), @@ -42,14 +54,41 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SyntheticTests(**args), Users(**args), ] + queries = config.get("queries", []) + + # Create a stream for each query in the list + query_streams = [] + for query in queries: + if all(field in query and query[field] for field in ["name", "data_source", "query"]): + name = query["name"] + data_source = query["data_source"] + query_string = query["query"] + + # Create a new stream using the query name, data source, and query string + new_stream = SeriesStream( + name=name, + data_source=data_source, + query_string=query_string, + **args, + ) + query_streams.append(new_stream) + else: + logging.info("Query fields are missing, Streams not created") + + # Combine the base streams and query streams + return base_streams + query_streams def connector_config(self, config: Mapping[str, Any]) -> Mapping[str, Any]: return { + "site": config.get("site", "datadoghq.com"), "authenticator": self._get_authenticator(config), "query": config.get("query", ""), "max_records_per_request": config.get("max_records_per_request", 5000), "start_date": config.get("start_date", datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")), "end_date": config.get("end_date", (datetime.now() + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")), + "query_start_date": config.get("start_date", ""), + "query_end_date": config.get("end_date", ""), + "queries": config.get("queries", []), } diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/spec.yaml b/airbyte-integrations/connectors/source-datadog/source_datadog/spec.yaml index de0e69d3aa9f..897f8e20b54d 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/spec.yaml +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/spec.yaml @@ -25,14 +25,6 @@ connectionSpecification: description: The search query. This just applies to Incremental syncs. If empty, it'll collect all logs. type: string order: 3 - max_records_per_request: - type: integer - title: Max records per requests - default: 5000 - minimum: 1 - maximum: 5000 - description: Maximum number of records to collect per request. - order: 4 start_date: title: Start date pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ @@ -42,6 +34,20 @@ connectionSpecification: type: string examples: - "2022-10-01T00:00:00Z" + order: 4 + site: + title: Site + description: The site where Datadog data resides in. + type: string + enum: + [ + "datadoghq.com", + "us3.datadoghq.com", + "us5.datadoghq.com", + "datadoghq.eu", + "ddog-gov.com", + ] + default: "datadoghq.com" order: 5 end_date: title: End date @@ -54,3 +60,41 @@ connectionSpecification: - "2022-10-01T00:00:00Z" type: string order: 6 + max_records_per_request: + type: integer + title: Max records per requests + default: 5000 + minimum: 1 + maximum: 5000 + description: Maximum number of records to collect per request. + order: 7 + queries: + title: Queries + description: >- + List of queries to be run and used as inputs. + type: array + order: 8 + default: [] + items: + type: object + required: + - name + - data_source + - query + properties: + name: + title: Query Name + description: The variable name for use in queries. + type: string + order: 1 + data_source: + title: Data Source + description: A data source that is powered by the platform. + type: string + enum: ["metrics", "cloud_cost", "logs", "rum"] + order: 2 + query: + title: Query + description: A classic query string. + type: string + order: 3 diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py b/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py index 1b7addc559c3..44e17453678b 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py @@ -3,12 +3,15 @@ # from abc import ABC, abstractmethod +from datetime import datetime from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.http import HttpStream +from dateutil.parser import parse +from pydantic.datetime_parse import timedelta class DatadogStream(HttpStream, ABC): @@ -19,17 +22,32 @@ class DatadogStream(HttpStream, ABC): primary_key: Optional[str] = None parse_response_root: Optional[str] = None - def __init__(self, query: str, max_records_per_request: int, start_date: str, end_date: str, **kwargs): + def __init__( + self, + site: str, + query: str, + max_records_per_request: int, + start_date: str, + end_date: str, + query_start_date: str, + query_end_date: str, + queries: List[Dict[str, str]] = None, + **kwargs, + ): super().__init__(**kwargs) + self.site = site self.query = query self.max_records_per_request = max_records_per_request self.start_date = start_date self.end_date = end_date + self.query_start_date = query_start_date + self.query_end_date = query_end_date + self.queries = queries or [] self._cursor_value = None @property def url_base(self) -> str: - return "https://api.datadoghq.com/api" + return f"https://api.{self.site}/api" def request_headers(self, **kwargs) -> Mapping[str, Any]: return { @@ -111,8 +129,8 @@ class IncrementalSearchableStream(V2ApiDatadogStream, IncrementalMixin, ABC): primary_key: Optional[str] = "id" parse_response_root: Optional[str] = "data" - def __init__(self, query: str, max_records_per_request: int, start_date: str, end_date: str, **kwargs): - super().__init__(query, max_records_per_request, start_date, end_date, **kwargs) + def __init__(self, site: str, query: str, max_records_per_request: int, start_date: str, end_date: str, **kwargs): + super().__init__(site, query, max_records_per_request, start_date, end_date, **kwargs) self._cursor_value = "" @property @@ -286,3 +304,100 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, self.current_page += 1 next_page_token = {"offset": self.current_page} return next_page_token + + +class SeriesStream(IncrementalSearchableStream, ABC): + """ + https://docs.datadoghq.com/api/latest/metrics/?code-lang=curl#query-timeseries-data-across-multiple-products + """ + + primary_key: Optional[str] = None + parse_response_root: Optional[str] = "data" + + def __init__(self, name, data_source, query_string, **kwargs): + super().__init__(**kwargs) + self.name = name + self.data_source = data_source + self.query_string = query_string + + @property + def http_method(self) -> str: + return "POST" + + def path(self, **kwargs) -> str: + return "query/timeseries" + + @property + def name(self) -> str: + return self._name + + @name.setter + def name(self, value): + self._name = value + + @property + def cursor_field(self) -> Union[str, List[str]]: + return "sync_date" + + def request_body_json( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Optional[Mapping]: + + if self.query_end_date: + end_date = int(parse(self.query_end_date).timestamp() * 1000) + else: + end_date = int(datetime.now().timestamp()) * 1000 + + if self.query_start_date: + start_date = int(parse(self.query_start_date).timestamp() * 1000) + elif self._cursor_value: + start_date = int(parse(self._cursor_value).timestamp() * 1000) + else: + start_date = int((datetime.now() - timedelta(hours=24)).timestamp()) * 1000 + + payload = { + "data": { + "type": "timeseries_request", + "attributes": { + "to": end_date, + "from": start_date, + "queries": [ + { + "data_source": self.data_source, + "name": self.name, + } + ], + }, + } + } + + if self.data_source in ["metrics", "cloud_cost"]: + payload["data"]["attributes"]["queries"][0]["query"] = self.query_string + elif self.data_source in ["logs", "rum"]: + payload["data"]["attributes"]["queries"][0]["search"] = {"query": self.query_string} + payload["data"]["attributes"]["queries"][0]["compute"] = {"aggregation": "count"} + print(payload) + return payload + + def get_json_schema(self) -> Mapping[str, Any]: + local_json_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {}, + "additionalProperties": True, + } + return local_json_schema + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data = response.json() + data["stream"] = self.name + data["query"] = self.query_string + data["data_source"] = self.data_source + return [data] + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + self._cursor_value = self.end_date + return None diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py b/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py index 3ec1edd2f150..6526981ddf26 100644 --- a/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py @@ -8,6 +8,20 @@ @fixture(name="config") def config_fixture(): return { + "site": "datadoghq.com", + "api_key": "test_api_key", + "application_key": "test_application_key", + "query": "", + "max_records_per_request": 5000, + "start_date": "2022-10-10T00:00:00Z", + "end_date": "2022-10-10T00:10:00Z", + } + + +@fixture(name="config_eu") +def config_fixture_eu(): + return { + "site": "datadoghq.eu", "api_key": "test_api_key", "application_key": "test_application_key", "query": "", @@ -390,3 +404,54 @@ def _mock_stream(path, response=None): requests_mock.get(url, json=response) return _mock_stream + + +@fixture(name="config_timeseries") +def config_timeseries_fixture(): + return { + "site": "datadoghq.eu", + "api_key": "test_api_key", + "application_key": "test_application_key", + "query": "", + "max_records_per_request": 5000, + "start_date": "2022-10-10T00:00:00Z", + "end_date": "2022-10-10T00:10:00Z", + "queries": [ + { + "name": "NodeCount", + "data_source": "metrics", + "query": "kubernetes_state.node.count{*}" + }, + { + "name": "Resource", + "data_source": "rum", + "query": "@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)" + } + ] + } + + +@fixture(name="config_timeseries_invalid") +def config_timeseries_invalid_fixture(): + return { + "site": "datadoghq.eu", + "api_key": "test_api_key", + "application_key": "test_application_key", + "query": "", + "max_records_per_request": 5000, + "start_date": "2022-10-10T00:00:00Z", + "end_date": "2022-10-10T00:10:00Z", + "queries": [ + { + "data_source": "metrics", + "query": "missing_name_query_string", + }, + { + "query": "missing_name_and_data_source_query_string", + }, + { + "name": "MissingQuery", + "data_source": "metrics", + } + ] + } diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py index 6ecfdf719937..b8c36d194397 100644 --- a/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py @@ -2,11 +2,13 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from unittest.mock import MagicMock from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import ConnectorSpecification from source_datadog.source import SourceDatadog +from source_datadog.streams import AuditLogs, SeriesStream logger = AirbyteLogger() @@ -56,3 +58,29 @@ def test_spec(): spec = SourceDatadog().spec(logger_mock) assert isinstance(spec, ConnectorSpecification) + + +def test_streams_with_valid_queries(config_timeseries): + streams = SourceDatadog().streams(config_timeseries) + + assert len(streams) == 11 + assert isinstance(streams[0], AuditLogs) + assert isinstance(streams[-1], SeriesStream) + assert streams[-1].name == "Resource" + assert streams[-2].name == "NodeCount" + + +def test_streams_with_invalid_queries(config_timeseries_invalid, caplog): + with caplog.at_level(logging.INFO): + streams = SourceDatadog().streams(config_timeseries_invalid) + + assert len(streams) == 9 + assert isinstance(streams[0], AuditLogs) + + invalid_query_names = ["", "MissingQuery"] + invalid_queries_exist = any(isinstance(stream, SeriesStream) and stream.name in invalid_query_names for stream in streams) + assert not invalid_queries_exist + + missing_query_logs = "Query fields are missing, Streams not created" + assert missing_query_logs in caplog.messages + assert len(caplog.records) == 3 diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py index 82d76b6b87d9..45d65c35abc2 100644 --- a/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py @@ -19,6 +19,7 @@ IncidentTeams, Logs, Metrics, + SeriesStream, SyntheticTests, Users, ) @@ -43,15 +44,26 @@ def test_task_stream(requests_mock, stream, config, mock_responses): @patch.multiple(DatadogStream, __abstractmethods__=set()) def test_next_page_token(config): stream = DatadogStream( + site=config["site"], query=config["query"], max_records_per_request=config["max_records_per_request"], start_date=config["start_date"], end_date=config["end_date"], + query_start_date=config["start_date"], + query_end_date=config["end_date"], ) inputs = {"response": MagicMock()} assert stream.next_page_token(**inputs) is None +def test_site_config(config): + assert config['site'] == 'datadoghq.com' + + +def test_site_config_eu(config_eu): + assert config_eu['site'] == 'datadoghq.eu' + + @pytest.mark.parametrize( "stream", [AuditLogs, Dashboards, Downtimes, Incidents, IncidentTeams, Logs, Metrics, SyntheticTests, Users], @@ -98,3 +110,96 @@ def test_next_page_token_paginated(stream, config): response._content = json.dumps(body_content).encode("utf-8") result = instance.next_page_token(response=response) assert result.get("offset") == 999 + + +@patch.multiple(DatadogStream, __abstractmethods__=set()) +def test_site_parameter_is_set(config): + site = "example.com" + stream = DatadogStream( + site=site, + query=config["query"], + max_records_per_request=config["max_records_per_request"], + start_date=config["start_date"], + end_date=config["end_date"], + query_start_date=config["start_date"], + query_end_date=config["end_date"], + ) + url_base = stream.url_base + expected_url_base = f"https://api.{site}/api" + assert url_base == expected_url_base + + +@patch.multiple(DatadogStream, __abstractmethods__=set()) +def test_site_parameter_is_not_set(config): + stream = DatadogStream( + site=config["site"], + query=config["query"], + max_records_per_request=config["max_records_per_request"], + start_date=config["start_date"], + end_date=config["end_date"], + query_start_date=config["start_date"], + query_end_date=config["end_date"], + ) + url_base = stream.url_base + expected_url_base = "https://api.datadoghq.com/api" + assert url_base == expected_url_base + + +@patch.multiple(SeriesStream, __abstractmethods__=set()) +def test_request_body_json(config): + stream = SeriesStream( + site=config["site"], + query=config["query"], + max_records_per_request=config["max_records_per_request"], + start_date=config["start_date"], + end_date=config["end_date"], + query_start_date="2023-01-01T00:00:00Z", + query_end_date="2023-02-01T00:00:00Z", + name="test_stream", + data_source="metrics", + query_string="test_query" + ) + stream_state = { + "stream_state_key": "value" + } + expected_payload = { + "data": { + "type": "timeseries_request", + "attributes": { + "to": 1675209600000, + "from": 1672531200000, + "queries": [ + { + "data_source": "metrics", + "query": "test_query", + "name": "test_stream" + } + ] + } + } + } + payload = stream.request_body_json(stream_state) + assert payload == expected_payload + + +@patch.multiple(SeriesStream, __abstractmethods__=set()) +def test_get_json_schema(config): + stream = SeriesStream( + site=config["site"], + query=config["query"], + max_records_per_request=config["max_records_per_request"], + start_date=config["start_date"], + end_date=config["end_date"], + query_start_date=config["start_date"], + query_end_date=config["end_date"], + name="test_stream", + data_source="metrics", + query_string="test_query" + ) + expected_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {}, + "additionalProperties": True + } + assert stream.get_json_schema() == expected_schema diff --git a/airbyte-integrations/connectors/source-datascope/metadata.yaml b/airbyte-integrations/connectors/source-datascope/metadata.yaml index 14ef5020e0a5..2d61ac39b220 100644 --- a/airbyte-integrations/connectors/source-datascope/metadata.yaml +++ b/airbyte-integrations/connectors/source-datascope/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-datascope/requirements.txt b/airbyte-integrations/connectors/source-datascope/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-datascope/requirements.txt +++ b/airbyte-integrations/connectors/source-datascope/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-datascope/setup.py b/airbyte-integrations/connectors/source-datascope/setup.py index c3d13399c41c..4a3a8864ddf4 100644 --- a/airbyte-integrations/connectors/source-datascope/setup.py +++ b/airbyte-integrations/connectors/source-datascope/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index 5e7be8f336e7..1a70ab09d31a 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.1.20 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index 5a7edbffc579..0cb0e59c15fa 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -30,3 +30,4 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' } + diff --git a/airbyte-integrations/connectors/source-db2/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-db2/integration_tests/seed/basic.sql index a89459e1a2b5..bd4219832cfb 100644 --- a/airbyte-integrations/connectors/source-db2/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-db2/integration_tests/seed/basic.sql @@ -1,17 +1,452 @@ -CREATE SCHEMA DB2_BASIC; - -CREATE TABLE DB2_BASIC.TEST_DATASET(ID INTEGER NOT NULL PRIMARY KEY, TEST_COLUMN_1 SMALLINT,TEST_COLUMN_10 BOOLEAN,TEST_COLUMN_11 CHAR,TEST_COLUMN_12 VARCHAR(256),TEST_COLUMN_13 VARCHAR(128),TEST_COLUMN_14 NCHAR,TEST_COLUMN_15 NVARCHAR(128),TEST_COLUMN_2 INTEGER,TEST_COLUMN_23 DATE,TEST_COLUMN_24 TIME,TEST_COLUMN_25 TIMESTAMP,TEST_COLUMN_3 BIGINT,TEST_COLUMN_4 DECIMAL(31, 0),TEST_COLUMN_5 REAL,TEST_COLUMN_6 DOUBLE,TEST_COLUMN_7 DECFLOAT(16),TEST_COLUMN_8 DECFLOAT(34) ); - -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (1, -32768, 't', 'a', 'тест', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), ' ', ' ', -2147483648, '0001-01-01', '00.00.00', '2018-03-22-12.00.00.123', -9223372036854775808, 1, 0, DOUBLE('-1.7976931348623157E+308'), 0, 0); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (2, 32767, 'true', ' ', '⚡ test ��', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '1:59 PM', '2018-03-22-12.00.00.123456', 9223372036854775807, DECIMAL((-1 + 10E+29), 31, 0), CAST('-3.4028234663852886E38' AS REAL), DOUBLE('-2.2250738585072014E-308'), 1.0E308, DECFLOAT(10E+307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (3, 32767, 'y', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180322125959', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), REAL('-1.1754943508222875e-38'), DOUBLE('2.2250738585072014E-308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (4, 32767, 'yes', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), REAL(1.1754943508222875e-38), DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (5, 32767, 'on', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (6, 32767, '1', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (7, 32767, 'f', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (8, 32767, 'false', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (9, 32767, 'n', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (10, 32767, 'no', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (11, 32767, 'off', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (12, 32767, '0', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); -INSERT INTO DB2_BASIC.TEST_DATASET VALUES (13, 32767, '0', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', 2147483647, '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34)); +CREATE + SCHEMA DB2_BASIC; + +CREATE + TABLE + DB2_BASIC.TEST_DATASET( + ID INTEGER NOT NULL PRIMARY KEY, + TEST_COLUMN_1 SMALLINT, + TEST_COLUMN_10 BOOLEAN, + TEST_COLUMN_11 CHAR, + TEST_COLUMN_12 VARCHAR(256), + TEST_COLUMN_13 VARCHAR(128), + TEST_COLUMN_14 NCHAR, + TEST_COLUMN_15 NVARCHAR(128), + TEST_COLUMN_2 INTEGER, + TEST_COLUMN_23 DATE, + TEST_COLUMN_24 TIME, + TEST_COLUMN_25 TIMESTAMP, + TEST_COLUMN_3 BIGINT, + TEST_COLUMN_4 DECIMAL( + 31, + 0 + ), + TEST_COLUMN_5 REAL, + TEST_COLUMN_6 DOUBLE, + TEST_COLUMN_7 DECFLOAT(16), + TEST_COLUMN_8 DECFLOAT(34) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 1, + - 32768, + 't', + 'a', + 'тест', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + ' ', + ' ', + - 2147483648, + '0001-01-01', + '00.00.00', + '2018-03-22-12.00.00.123', + - 9223372036854775808, + 1, + 0, + DOUBLE('-1.7976931348623157E+308'), + 0, + 0 + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 2, + 32767, + 'true', + ' ', + '⚡ test ��', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '1:59 PM', + '2018-03-22-12.00.00.123456', + 9223372036854775807, + DECIMAL( + ( + - 1 + 10E + 29 + ), + 31, + 0 + ), + CAST( + '-3.4028234663852886E38' AS REAL + ), + DOUBLE('-2.2250738585072014E-308'), + 1.0E308, + DECFLOAT( + 10E + 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 3, + 32767, + 'y', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180322125959', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + REAL('-1.1754943508222875e-38'), + DOUBLE('2.2250738585072014E-308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 4, + 32767, + 'yes', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + REAL( + 1.1754943508222875e - 38 + ), + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 5, + 32767, + 'on', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 6, + 32767, + '1', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 7, + 32767, + 'f', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 8, + 32767, + 'false', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 9, + 32767, + 'n', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 10, + 32767, + 'no', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 11, + 32767, + 'off', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 12, + 32767, + '0', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); + +INSERT + INTO + DB2_BASIC.TEST_DATASET + VALUES( + 13, + 32767, + '0', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + 2147483647, + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ) + ); diff --git a/airbyte-integrations/connectors/source-db2/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-db2/integration_tests/seed/full.sql index 8a78152b7cce..75c2d7a30994 100644 --- a/airbyte-integrations/connectors/source-db2/integration_tests/seed/full.sql +++ b/airbyte-integrations/connectors/source-db2/integration_tests/seed/full.sql @@ -1,17 +1,482 @@ -CREATE SCHEMA DB2_FULL; - -CREATE TABLE DB2_FULL.TEST_DATASET(ID INTEGER NOT NULL PRIMARY KEY, TEST_COLUMN_1 SMALLINT,TEST_COLUMN_10 BOOLEAN,TEST_COLUMN_11 CHAR,TEST_COLUMN_12 VARCHAR(256),TEST_COLUMN_13 VARCHAR(128),TEST_COLUMN_14 NCHAR,TEST_COLUMN_15 NVARCHAR(128),TEST_COLUMN_16 GRAPHIC(8),TEST_COLUMN_17 VARGRAPHIC(8),TEST_COLUMN_18 VARBINARY(32),TEST_COLUMN_19 BLOB,TEST_COLUMN_2 INTEGER,TEST_COLUMN_20 CLOB,TEST_COLUMN_21 NCLOB,TEST_COLUMN_22 XML,TEST_COLUMN_23 DATE,TEST_COLUMN_24 TIME,TEST_COLUMN_25 TIMESTAMP,TEST_COLUMN_3 BIGINT,TEST_COLUMN_4 DECIMAL(31, 0),TEST_COLUMN_5 REAL,TEST_COLUMN_6 DOUBLE,TEST_COLUMN_7 DECFLOAT(16),TEST_COLUMN_8 DECFLOAT(34),TEST_COLUMN_9 DECFLOAT ); - -INSERT INTO DB2_FULL.TEST_DATASET VALUES (1, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, SNaN); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (2, -32768, 't', 'a', 'тест', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), ' ', ' ', ' ', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB(' '), -2147483648, ' ', ' ', XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '0001-01-01', '00.00.00', '2018-03-22-12.00.00.123', -9223372036854775808, 1, 0, DOUBLE('-1.7976931348623157E+308'), 0, 0, NaN); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (3, 32767, 'true', ' ', '⚡ test ��', null, 'テ', 'テスト', '12345678', null, null, BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), null, '9999-12-31', '1:59 PM', '2018-03-22-12.00.00.123456', 9223372036854775807, DECIMAL((-1 + 10E+29), 31, 0), CAST('-3.4028234663852886E38' AS REAL), DOUBLE('-2.2250738585072014E-308'), 1.0E308, DECFLOAT(10E+307, 34), Infinity); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (4, null, 'y', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', null, null, null, null, null, null, null, null, null, null, null, null, '23.59.59', '20180322125959', null, DECIMAL((1 - 10E+29), 31, 0), REAL('-1.1754943508222875e-38'), DOUBLE('2.2250738585072014E-308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (5, null, 'yes', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '20180101 12:00:59 PM', null, null, REAL(1.1754943508222875e-38), DOUBLE('1.7976931348623157E+308'), null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (6, null, 'on', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 3.4028234663852886E38, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (7, null, '1', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (8, null, 'f', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (9, null, 'false', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (10, null, 'n', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (11, null, 'no', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (12, null, 'off', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO DB2_FULL.TEST_DATASET VALUES (13, null, '0', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); +CREATE + SCHEMA DB2_FULL; + +CREATE + TABLE + DB2_FULL.TEST_DATASET( + ID INTEGER NOT NULL PRIMARY KEY, + TEST_COLUMN_1 SMALLINT, + TEST_COLUMN_10 BOOLEAN, + TEST_COLUMN_11 CHAR, + TEST_COLUMN_12 VARCHAR(256), + TEST_COLUMN_13 VARCHAR(128), + TEST_COLUMN_14 NCHAR, + TEST_COLUMN_15 NVARCHAR(128), + TEST_COLUMN_16 GRAPHIC(8), + TEST_COLUMN_17 VARGRAPHIC(8), + TEST_COLUMN_18 VARBINARY(32), + TEST_COLUMN_19 BLOB, + TEST_COLUMN_2 INTEGER, + TEST_COLUMN_20 CLOB, + TEST_COLUMN_21 NCLOB, + TEST_COLUMN_22 XML, + TEST_COLUMN_23 DATE, + TEST_COLUMN_24 TIME, + TEST_COLUMN_25 TIMESTAMP, + TEST_COLUMN_3 BIGINT, + TEST_COLUMN_4 DECIMAL( + 31, + 0 + ), + TEST_COLUMN_5 REAL, + TEST_COLUMN_6 DOUBLE, + TEST_COLUMN_7 DECFLOAT(16), + TEST_COLUMN_8 DECFLOAT(34), + TEST_COLUMN_9 DECFLOAT + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 1, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + SNaN + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 2, + - 32768, + 't', + 'a', + 'тест', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + ' ', + ' ', + ' ', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB(' '), + - 2147483648, + ' ', + ' ', + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '0001-01-01', + '00.00.00', + '2018-03-22-12.00.00.123', + - 9223372036854775808, + 1, + 0, + DOUBLE('-1.7976931348623157E+308'), + 0, + 0, + NaN + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 3, + 32767, + 'true', + ' ', + '⚡ test ��', + NULL, + 'テ', + 'テスト', + '12345678', + NULL, + NULL, + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + NULL, + '9999-12-31', + '1:59 PM', + '2018-03-22-12.00.00.123456', + 9223372036854775807, + DECIMAL( + ( + - 1 + 10E + 29 + ), + 31, + 0 + ), + CAST( + '-3.4028234663852886E38' AS REAL + ), + DOUBLE('-2.2250738585072014E-308'), + 1.0E308, + DECFLOAT( + 10E + 307, + 34 + ), + Infinity + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 4, + NULL, + 'y', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '23.59.59', + '20180322125959', + NULL, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + REAL('-1.1754943508222875e-38'), + DOUBLE('2.2250738585072014E-308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 5, + NULL, + 'yes', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '20180101 12:00:59 PM', + NULL, + NULL, + REAL( + 1.1754943508222875e - 38 + ), + DOUBLE('1.7976931348623157E+308'), + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 6, + NULL, + 'on', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 3.4028234663852886E38, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 7, + NULL, + '1', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 8, + NULL, + 'f', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 9, + NULL, + 'false', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 10, + NULL, + 'n', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 11, + NULL, + 'no', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 12, + NULL, + 'off', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + DB2_FULL.TEST_DATASET + VALUES( + 13, + NULL, + '0', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-db2/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-db2/integration_tests/seed/full_without_nulls.sql index 19d9324242e5..82350806a6e5 100644 --- a/airbyte-integrations/connectors/source-db2/integration_tests/seed/full_without_nulls.sql +++ b/airbyte-integrations/connectors/source-db2/integration_tests/seed/full_without_nulls.sql @@ -1,17 +1,668 @@ -CREATE SCHEMA DB2_FULL_NONULL; - -CREATE TABLE DB2_FULL_NONULL.TEST_DATASET(ID INTEGER NOT NULL PRIMARY KEY, TEST_COLUMN_1 SMALLINT,TEST_COLUMN_10 BOOLEAN,TEST_COLUMN_11 CHAR,TEST_COLUMN_12 VARCHAR(256),TEST_COLUMN_13 VARCHAR(128),TEST_COLUMN_14 NCHAR,TEST_COLUMN_15 NVARCHAR(128),TEST_COLUMN_16 GRAPHIC(8),TEST_COLUMN_17 VARGRAPHIC(8),TEST_COLUMN_18 VARBINARY(32),TEST_COLUMN_19 BLOB,TEST_COLUMN_2 INTEGER,TEST_COLUMN_20 CLOB,TEST_COLUMN_21 NCLOB,TEST_COLUMN_22 XML,TEST_COLUMN_23 DATE,TEST_COLUMN_24 TIME,TEST_COLUMN_25 TIMESTAMP,TEST_COLUMN_3 BIGINT,TEST_COLUMN_4 DECIMAL(31, 0),TEST_COLUMN_5 REAL,TEST_COLUMN_6 DOUBLE,TEST_COLUMN_7 DECFLOAT(16),TEST_COLUMN_8 DECFLOAT(34),TEST_COLUMN_9 DECFLOAT ); - -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (1, -32768, 't', 'a', 'тест', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), ' ', ' ', ' ', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB(' '), -2147483648, ' ', ' ', XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '0001-01-01', '00.00.00', '2018-03-22-12.00.00.123', -9223372036854775808, 1, 0, DOUBLE('-1.7976931348623157E+308'), 0, 0, SNaN); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (2, 32767, 'true', ' ', '⚡ test ��', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '1:59 PM', '2018-03-22-12.00.00.123456', 9223372036854775807, DECIMAL((-1 + 10E+29), 31, 0), CAST('-3.4028234663852886E38' AS REAL), DOUBLE('-2.2250738585072014E-308'), 1.0E308, DECFLOAT(10E+307, 34), NaN); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (3, 32767, 'y', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180322125959', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), REAL('-1.1754943508222875e-38'), DOUBLE('2.2250738585072014E-308'), 1.0E-306, DECFLOAT(10E-307, 34), Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (4, 32767, 'yes', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), REAL(1.1754943508222875e-38), DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (5, 32767, 'on', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (6, 32767, '1', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (7, 32767, 'f', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (8, 32767, 'false', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (9, 32767, 'n', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (10, 32767, 'no', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (11, 32767, 'off', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (12, 32767, '0', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); -INSERT INTO DB2_FULL_NONULL.TEST_DATASET VALUES (13, 32767, '0', '*', '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), 'テ', 'テスト', '12345678', VARGRAPHIC(100500, ','), VARBINARY('test VARBINARY type', 19), BLOB('test BLOB type'), 2147483647, CLOB('test CLOB type'), NCLOB('test NCLOB type'), XMLPARSE (DOCUMENT 'Manual...' PRESERVE WHITESPACE), '9999-12-31', '23.59.59', '20180101 12:00:59 PM', 9223372036854775807, DECIMAL((1 - 10E+29), 31, 0), 3.4028234663852886E38, DOUBLE('1.7976931348623157E+308'), 1.0E-306, DECFLOAT(10E-307, 34), -Infinity); +CREATE + SCHEMA DB2_FULL_NONULL; + +CREATE + TABLE + DB2_FULL_NONULL.TEST_DATASET( + ID INTEGER NOT NULL PRIMARY KEY, + TEST_COLUMN_1 SMALLINT, + TEST_COLUMN_10 BOOLEAN, + TEST_COLUMN_11 CHAR, + TEST_COLUMN_12 VARCHAR(256), + TEST_COLUMN_13 VARCHAR(128), + TEST_COLUMN_14 NCHAR, + TEST_COLUMN_15 NVARCHAR(128), + TEST_COLUMN_16 GRAPHIC(8), + TEST_COLUMN_17 VARGRAPHIC(8), + TEST_COLUMN_18 VARBINARY(32), + TEST_COLUMN_19 BLOB, + TEST_COLUMN_2 INTEGER, + TEST_COLUMN_20 CLOB, + TEST_COLUMN_21 NCLOB, + TEST_COLUMN_22 XML, + TEST_COLUMN_23 DATE, + TEST_COLUMN_24 TIME, + TEST_COLUMN_25 TIMESTAMP, + TEST_COLUMN_3 BIGINT, + TEST_COLUMN_4 DECIMAL( + 31, + 0 + ), + TEST_COLUMN_5 REAL, + TEST_COLUMN_6 DOUBLE, + TEST_COLUMN_7 DECFLOAT(16), + TEST_COLUMN_8 DECFLOAT(34), + TEST_COLUMN_9 DECFLOAT + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 1, + - 32768, + 't', + 'a', + 'тест', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + ' ', + ' ', + ' ', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB(' '), + - 2147483648, + ' ', + ' ', + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '0001-01-01', + '00.00.00', + '2018-03-22-12.00.00.123', + - 9223372036854775808, + 1, + 0, + DOUBLE('-1.7976931348623157E+308'), + 0, + 0, + SNaN + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 2, + 32767, + 'true', + ' ', + '⚡ test ��', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '1:59 PM', + '2018-03-22-12.00.00.123456', + 9223372036854775807, + DECIMAL( + ( + - 1 + 10E + 29 + ), + 31, + 0 + ), + CAST( + '-3.4028234663852886E38' AS REAL + ), + DOUBLE('-2.2250738585072014E-308'), + 1.0E308, + DECFLOAT( + 10E + 307, + 34 + ), + NaN + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 3, + 32767, + 'y', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180322125959', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + REAL('-1.1754943508222875e-38'), + DOUBLE('2.2250738585072014E-308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 4, + 32767, + 'yes', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + REAL( + 1.1754943508222875e - 38 + ), + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 5, + 32767, + 'on', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 6, + 32767, + '1', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 7, + 32767, + 'f', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 8, + 32767, + 'false', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 9, + 32767, + 'n', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 10, + 32767, + 'no', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 11, + 32767, + 'off', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 12, + 32767, + '0', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); + +INSERT + INTO + DB2_FULL_NONULL.TEST_DATASET + VALUES( + 13, + 32767, + '0', + '*', + '!"#$%&\''()*+,-./:;<=>?\@[\]^_\`{|}~', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + 'テ', + 'テスト', + '12345678', + VARGRAPHIC( + 100500, + ',' + ), + VARBINARY( + 'test VARBINARY type', + 19 + ), + BLOB('test BLOB type'), + 2147483647, + CLOB('test CLOB type'), + NCLOB('test NCLOB type'), + XMLPARSE( + DOCUMENT 'Manual...' PRESERVE WHITESPACE + ), + '9999-12-31', + '23.59.59', + '20180101 12:00:59 PM', + 9223372036854775807, + DECIMAL( + ( + 1 - 10E + 29 + ), + 31, + 0 + ), + 3.4028234663852886E38, + DOUBLE('1.7976931348623157E+308'), + 1.0E - 306, + DECFLOAT( + 10E - 307, + 34 + ), + - Infinity + ); diff --git a/airbyte-integrations/connectors/source-db2/metadata.yaml b/airbyte-integrations/connectors/source-db2/metadata.yaml index 489d3a724a61..69cda6038a2f 100644 --- a/airbyte-integrations/connectors/source-db2/metadata.yaml +++ b/airbyte-integrations/connectors/source-db2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 - dockerImageTag: 0.1.19 + dockerImageTag: 0.1.20 dockerRepository: airbyte/source-db2 githubIssueLabel: source-db2 icon: db2.svg @@ -21,4 +21,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml index 565103f29f6e..98877e8ac55c 100644 --- a/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-delighted/acceptance-test-config.yml @@ -23,11 +23,14 @@ acceptance_tests: expect_records: path: "integration_tests/expected_records.jsonl" incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" + bypass_reason: > + "Incrremental tests are disabled until CAT works with cursor data-types directly, + relatated slack thread: https://airbyte-globallogic.slack.com/archives/C02U9R3AF37/p1690810513681859" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-delighted/metadata.yaml b/airbyte-integrations/connectors/source-delighted/metadata.yaml index 73291f6e02de..9b27d60b5d52 100644 --- a/airbyte-integrations/connectors/source-delighted/metadata.yaml +++ b/airbyte-integrations/connectors/source-delighted/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-delighted/requirements.txt b/airbyte-integrations/connectors/source-delighted/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-delighted/requirements.txt +++ b/airbyte-integrations/connectors/source-delighted/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-delighted/setup.py b/airbyte-integrations/connectors/source-delighted/setup.py index b228e186ed84..ca92c54ec6fe 100644 --- a/airbyte-integrations/connectors/source-delighted/setup.py +++ b/airbyte-integrations/connectors/source-delighted/setup.py @@ -10,8 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", "responses~=0.13.3", ] diff --git a/airbyte-integrations/connectors/source-dixa/Dockerfile b/airbyte-integrations/connectors/source-dixa/Dockerfile index 93cb4d948a8e..e902f7d6f7d6 100644 --- a/airbyte-integrations/connectors/source-dixa/Dockerfile +++ b/airbyte-integrations/connectors/source-dixa/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-dixa diff --git a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml index a3a615fb5a34..30ff974f5d74 100644 --- a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml @@ -8,7 +8,7 @@ tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: - config_path: "secrets/config.json" basic_read: diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json index 580eb448f5d6..efe861f2078b 100644 --- a/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-dixa/integration_tests/sample_config.json @@ -1,5 +1,5 @@ { "api_token": "TOKEN", "start_date": "2020-01-01", - "batch_size": "31" + "batch_size": 31 } diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index 40b717ae6a3d..d26ebbd01c91 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7 - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-dixa githubIssueLabel: source-dixa icon: dixa.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/dixa tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-dixa/requirements.txt b/airbyte-integrations/connectors/source-dixa/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-dixa/requirements.txt +++ b/airbyte-integrations/connectors/source-dixa/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-dixa/setup.py b/airbyte-integrations/connectors/source-dixa/setup.py index 72e5ddbe8e3d..65b250fac3ca 100644 --- a/airbyte-integrations/connectors/source-dixa/setup.py +++ b/airbyte-integrations/connectors/source-dixa/setup.py @@ -6,12 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json index bd88556883f5..6ee94ca78602 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json @@ -1,6 +1,8 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Conversation Export", + "additionalProperties": true, "type": ["null", "object"], - "required": ["id", "created_at", "initial_channel", "requester_id"], "properties": { "id": { "type": ["null", "integer"] @@ -145,6 +147,13 @@ "type": ["null", "string"] } }, + "tags_info": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + }, "conversation_wrapup_notes": { "type": ["null", "array"], "items": { @@ -163,6 +172,9 @@ "status": { "type": ["null", "string"] }, + "transferee_number": { + "type": ["null", "string"] + }, "updated_at": { "type": ["null", "integer"] }, @@ -180,6 +192,30 @@ }, "anonymized_at": { "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "attribute_id": { + "type": ["null", "string"] + }, + "identifier": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "array", "string"] + }, + "data_type": { + "type": ["null", "string"] + }, + "archived": { + "type": ["null", "boolean"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py index 63f2cb8a05a0..8f0689672881 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py @@ -109,13 +109,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> """ Check connectivity using one day's worth of data. """ - config["authenticator"] = TokenAuthenticator(token=config["api_token"]) - stream = ConversationExport(config) - # using 1 day batch size for slices. - stream.batch_size = 1 - # use the first slice from stream_slices list - stream_slice = stream.stream_slices()[0] try: + config["authenticator"] = TokenAuthenticator(token=config["api_token"]) + stream = ConversationExport(config) + # using 1 day batch size for slices. + stream.batch_size = 1 + # use the first slice from stream_slices list + stream_slice = stream.stream_slices()[0] list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) return True, None except Exception as e: diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json b/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json index 016b237406a5..d4bfe2b03417 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json @@ -5,7 +5,7 @@ "title": "Dixa Spec", "type": "object", "required": ["api_token", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "api_token": { "type": "string", diff --git a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py index 61ad3b2038cf..cbd78e67e68c 100644 --- a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py @@ -53,8 +53,8 @@ def test_stream_slices_without_state(conversation_export): conversation_export.end_timestamp = 1625259600000 # 2021-07-03 00:00:00 + 1 ms expected_slices = [ - {"updated_after": 1625097600000, "updated_before": 1625184000000}, - {"updated_after": 1625184000000, "updated_before": 1625259600000}, + {'updated_after': 1625097600000, 'updated_before': 1625184000000}, + {'updated_after': 1625184000000, 'updated_before': 1625259600000}, ] actual_slices = conversation_export.stream_slices() diff --git a/airbyte-integrations/connectors/source-dockerhub/Dockerfile b/airbyte-integrations/connectors/source-dockerhub/Dockerfile index 058503c031b4..d40883d645c8 100644 --- a/airbyte-integrations/connectors/source-dockerhub/Dockerfile +++ b/airbyte-integrations/connectors/source-dockerhub/Dockerfile @@ -34,5 +34,5 @@ COPY source_dockerhub ./source_dockerhub ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-dockerhub diff --git a/airbyte-integrations/connectors/source-dockerhub/README.md b/airbyte-integrations/connectors/source-dockerhub/README.md index a6cf6c8b73c7..73055dd9d3ea 100644 --- a/airbyte-integrations/connectors/source-dockerhub/README.md +++ b/airbyte-integrations/connectors/source-dockerhub/README.md @@ -1,35 +1,10 @@ # Dockerhub Source -This is the repository for the Dockerhub source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/dockerhub) (not active yet). +This is the repository for the Dockerhub configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/dockerhub). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,36 +13,15 @@ To build using Gradle, from the Airbyte repository root, run: ./gradlew :airbyte-integrations:connectors:source-dockerhub:build ``` - - -### Locally running the connector -``` -python main.py spec -python main.py check --config sample_files/config.json -python main.py discover --config sample_files/config.json -python main.py read --config sample_files/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -87,37 +41,20 @@ the Dockerfile. Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-dockerhub:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dockerhub:dev check --config /sample_files/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dockerhub:dev discover --config /sample_files/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dockerhub:dev read --config /sample_files/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dockerhub:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dockerhub:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dockerhub:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-dockerhub/__init__.py b/airbyte-integrations/connectors/source-dockerhub/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-dockerhub/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-dockerhub/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dockerhub/acceptance-test-config.yml index e17d56ce9e3a..3d0d5ceef505 100644 --- a/airbyte-integrations/connectors/source-dockerhub/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-dockerhub/acceptance-test-config.yml @@ -4,20 +4,24 @@ connector_image: airbyte/source-dockerhub:dev tests: spec: - spec_path: "source_dockerhub/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" connection: - - config_path: "sample_files/config.json" + - config_path: "secrets/config.json" status: "succeed" # even with an incorrect username the api still returns 200 so just ignoring the invalid config check for now # - config_path: "integration_tests/invalid_config.json" # status: "failed" discovery: - - config_path: "sample_files/config.json" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" basic_read: - - config_path: "sample_files/config.json" + - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] full_refresh: - - config_path: "sample_files/config.json" + - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" # testing sequentially for same results can fail because of pull counts increasing for an image between runs ignored_fields: diff --git a/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-dockerhub/integration_tests/__init__.py b/airbyte-integrations/connectors/source-dockerhub/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-dockerhub/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-dockerhub/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py index dbe8d95312ff..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dockerhub/integration_tests/acceptance.py @@ -3,10 +3,6 @@ # -import os -import pathlib -import shutil - import pytest pytest_plugins = ("connector_acceptance_test.plugin",) @@ -14,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """This source doesn't have any secrets, so this copies the sample_files config into secrets/ for acceptance tests""" - src_folder = pathlib.Path(__file__).parent.parent.resolve() - os.makedirs(f"{src_folder}/secrets", exist_ok=True) - shutil.copy(f"{src_folder}/sample_files/config.json", f"{src_folder}/secrets/") + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-dockerhub/integration_tests/catalog.json b/airbyte-integrations/connectors/source-dockerhub/integration_tests/catalog.json deleted file mode 100644 index 9627353a77b8..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/integration_tests/catalog.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "streams": [ - { - "name": "docker_hub", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "user": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "namespace": { - "type": ["null", "string"] - }, - "repository_type": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "is_private": { - "type": ["null", "boolean"] - }, - "is_automated": { - "type": ["null", "boolean"] - }, - "can_edit": { - "type": ["null", "boolean"] - }, - "star_count": { - "type": ["null", "integer"] - }, - "pull_count": { - "type": ["null", "integer"] - }, - "last_updated": { - "type": ["null", "string"] - }, - "is_migrated": { - "type": ["null", "boolean"] - }, - "collaborator_count": { - "type": ["null", "integer"] - }, - "affiliation": { - "type": ["null", "string"] - }, - "hub_user": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-dockerhub/sample_files/config.json b/airbyte-integrations/connectors/source-dockerhub/integration_tests/sample_config.json similarity index 100% rename from airbyte-integrations/connectors/source-dockerhub/sample_files/config.json rename to airbyte-integrations/connectors/source-dockerhub/integration_tests/sample_config.json diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index 8b40761d7866..3afe52544412 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -1,20 +1,29 @@ data: + allowedHosts: + hosts: + - hub.docker.com + - auth.docker.io + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 72d405a3-56d8-499f-a571-667c03406e43 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-dockerhub githubIssueLabel: source-dockerhub icon: dockerhub.svg license: MIT name: Dockerhub - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2022-05-20 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/dockerhub tags: - - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-dockerhub/requirements.txt b/airbyte-integrations/connectors/source-dockerhub/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-dockerhub/requirements.txt +++ b/airbyte-integrations/connectors/source-dockerhub/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-dockerhub/setup.py b/airbyte-integrations/connectors/source-dockerhub/setup.py index 31ce7e4962e8..ea4b2d5873c8 100644 --- a/airbyte-integrations/connectors/source-dockerhub/setup.py +++ b/airbyte-integrations/connectors/source-dockerhub/setup.py @@ -5,19 +5,21 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests~=2.28.0"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( name="source_dockerhub", description="Source implementation for Dockerhub.", author="Airbyte", - author_email="shawn@airbyte.io", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/__init__.py b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/__init__.py index 4961990cca6c..e0a8dc75b336 100644 --- a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/__init__.py +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml new file mode 100644 index 000000000000..936cd50cda97 --- /dev/null +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml @@ -0,0 +1,78 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - docker_hub + +streams: + - type: DeclarativeStream + name: docker_hub + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://hub.docker.com/v2/ + path: repositories/{{ config['docker_username'] }}/ + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: SessionTokenAuthenticator + login_requester: + type: HttpRequester + url_base: >- + https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/alpine:pull/ + path: "" + authenticator: + type: NoAuth + http_method: GET + request_parameters: {} + request_headers: {} + request_body_json: {} + session_token_path: + - token + request_authentication: + type: ApiKey + inject_into: + type: RequestOption + field_name: Authorization + inject_into: header + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("next", {}) }}' + stop_condition: '{{ not response.get("next", {}) }}' + +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/dockerhub + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - docker_username + properties: + docker_username: + type: string + order: 0 + title: Docker Username + description: >- + Username of DockerHub person or organization (for + https://hub.docker.com/v2/repositories/USERNAME/ API call) + pattern: ^[a-z0-9_\-]+$ + examples: + - airbyte diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/schemas/docker_hub.json b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/schemas/docker_hub.json index f72e7df20c30..8ed7fb3d4229 100644 --- a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/schemas/docker_hub.json +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/schemas/docker_hub.json @@ -35,6 +35,24 @@ "pull_count": { "type": ["null", "integer"] }, + "date_registered": { + "type": ["null", "string"] + }, + "status_description": { + "type": ["null", "string"] + }, + "content_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "media_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "last_updated": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/source.py b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/source.py index 177d573c2700..1ffad6cb0e8c 100644 --- a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/source.py +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/source.py @@ -2,89 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from typing import Any, Iterable, List, Mapping, Optional, Tuple -from urllib.parse import urlparse +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -logger = logging.getLogger("airbyte") +WARNING: Do not modify this file. +""" -class SourceDockerhub(AbstractSource): - jwt = None - - def check_connection(self, logger, config) -> Tuple[bool, any]: - username = config["docker_username"] - - # get JWT - jwt_url = "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/alpine:pull" - response = requests.get(jwt_url) - self.jwt = response.json()["token"] - - # check that jwt is valid and that username is valid - url = f"https://hub.docker.com/v2/repositories/{username}/" - try: - response = requests.get(url, headers={"Authorization": self.jwt}) - response.raise_for_status() - except requests.exceptions.HTTPError as e: - if e.response.status_code == 401: - logger.info(str(e)) - return False, "Invalid JWT received, check if auth.docker.io changed API" - elif e.response.status_code == 404: - logger.info(str(e)) - return False, f"User '{username}' not found, check if hub.docker.com/u/{username} exists" - else: - logger.info(str(e)) - return False, f"Error getting basic user info for Docker user '{username}', unexpected error" - json_response = response.json() - repocount = json_response["count"] - logger.info(f"Connection check for Docker user '{username}' successful: {repocount} repos found") - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [DockerHub(jwt=self.jwt, config=config)] - - -class DockerHub(HttpStream): - url_base = "https://hub.docker.com/v2" - - # Set this as a noop. - primary_key = None - - def __init__(self, jwt: str, config: Mapping[str, Any], **kwargs): - super().__init__() - # Here's where we set the variable from our input to pass it down to the source. - self.jwt = jwt - self.docker_username = config["docker_username"] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - decoded_response = response.json() - if decoded_response["next"] is None: - return None - else: - para = urlparse(decoded_response["next"]).query - return "?" + para - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = "" - ) -> str: - return f"/v2/repositories/{self.docker_username}/" + str(next_page_token or "") - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - return {"Authorization": self.jwt} - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - for repository in response.json().get("results"): - yield repository +# Declarative Source +class SourceDockerhub(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/spec.yaml b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/spec.yaml deleted file mode 100644 index e2de2922d552..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/dockerhub -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Dockerhub Spec - type: object - required: - - docker_username - additionalProperties: false - properties: - docker_username: - type: string - description: Username of DockerHub person or organization (for https://hub.docker.com/v2/repositories/USERNAME/ API call) - pattern: ^[a-z0-9_\-]+$ - examples: - - airbyte diff --git a/airbyte-integrations/connectors/source-dockerhub/unit_tests/__init__.py b/airbyte-integrations/connectors/source-dockerhub/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-dockerhub/unit_tests/test_source.py b/airbyte-integrations/connectors/source-dockerhub/unit_tests/test_source.py deleted file mode 100644 index 48d84ae2edc2..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/unit_tests/test_source.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_dockerhub.source import SourceDockerhub - - -def test_check_connection(): - source = SourceDockerhub() - logger_mock, config_mock = MagicMock(), { - "docker_username": "airbyte" - } # shouldnt actually ping network request in test but we will skip for now - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(): - source = SourceDockerhub() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-dockerhub/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-dockerhub/unit_tests/test_streams.py deleted file mode 100644 index 5a89734689f8..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/unit_tests/test_streams.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import requests -from source_dockerhub.source import DockerHub - - -def test_next_page_token(): - stream = DockerHub(jwt="foo", config={"docker_username": "foo"}) - - # mocking the request with a response that has a next page token - response = requests.Response() - response.url = "https://foo" - response.json = MagicMock() - response.json.return_value = {"next": "https://foo?page=2"} - inputs = {"response": response} - - expected_token = "?page=2" # expected next page token - assert stream.next_page_token(**inputs) == expected_token - - -# cant get this to work - TypeError: 'list' object is not an iterator -# def test_parse_response(patch_base_class, mocker): -# response = mocker.MagicMock() -# response.json.return_value = {"one": 1} -# stream = DockerHub(jwt="foo", config={"docker_username": "foo"}) - -# inputs = { -# "response": response, -# "stream_state": MagicMock(), -# "stream_slice": MagicMock(), -# "next_page_token": MagicMock(), -# } - -# expected_parsed_object = {"one": 1} -# assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(): - stream = DockerHub(jwt="foo", config={"docker_username": "foo"}) - - inputs = { - "stream_state": MagicMock(), - "stream_slice": MagicMock(), - "next_page_token": MagicMock(), - } - - expected_headers = {"Authorization": "foo"} - assert stream.request_headers(**inputs) == expected_headers diff --git a/airbyte-integrations/connectors/source-dremio/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-dremio/integration_tests/invalid_config.json index 0c2626be7f68..dbaa2047dd34 100644 --- a/airbyte-integrations/connectors/source-dremio/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-dremio/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "base_url": "does_not_exist", "api_key": "invalid_api_key" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-dremio/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-dremio/integration_tests/sample_config.json index cd6ed9d2ef06..2af67e549db5 100644 --- a/airbyte-integrations/connectors/source-dremio/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-dremio/integration_tests/sample_config.json @@ -2,5 +2,3 @@ "base_url": "https://app.dremio.cloud/", "api_key": "sample_api_key" } - - diff --git a/airbyte-integrations/connectors/source-dremio/metadata.yaml b/airbyte-integrations/connectors/source-dremio/metadata.yaml index b6e7387c0a54..4315e72f6be3 100644 --- a/airbyte-integrations/connectors/source-dremio/metadata.yaml +++ b/airbyte-integrations/connectors/source-dremio/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-dremio/requirements.txt b/airbyte-integrations/connectors/source-dremio/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-dremio/requirements.txt +++ b/airbyte-integrations/connectors/source-dremio/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-dremio/setup.py b/airbyte-integrations/connectors/source-dremio/setup.py index 26c48401c5f8..290c8fcf4fbe 100644 --- a/airbyte-integrations/connectors/source-dremio/setup.py +++ b/airbyte-integrations/connectors/source-dremio/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-dremio/source_dremio/schemas/catalogs.json b/airbyte-integrations/connectors/source-dremio/source_dremio/schemas/catalogs.json index f330e57f5b10..74633153c10a 100644 --- a/airbyte-integrations/connectors/source-dremio/source_dremio/schemas/catalogs.json +++ b/airbyte-integrations/connectors/source-dremio/source_dremio/schemas/catalogs.json @@ -13,7 +13,7 @@ "path": { "type": "array" }, - "type":{ + "type": { "type": "string" }, "tag": { @@ -28,6 +28,5 @@ } } } - } } diff --git a/airbyte-integrations/connectors/source-drift/Dockerfile b/airbyte-integrations/connectors/source-drift/Dockerfile index 67d7bff67428..8c68336821da 100644 --- a/airbyte-integrations/connectors/source-drift/Dockerfile +++ b/airbyte-integrations/connectors/source-drift/Dockerfile @@ -34,5 +34,5 @@ COPY source_drift ./source_drift ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-drift diff --git a/airbyte-integrations/connectors/source-drift/README.md b/airbyte-integrations/connectors/source-drift/README.md index 7155c2470cea..7bc078e5f95d 100644 --- a/airbyte-integrations/connectors/source-drift/README.md +++ b/airbyte-integrations/connectors/source-drift/README.md @@ -1,64 +1,27 @@ -# Drift Source +# Drift Source -This is the repository for the Drift source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/drift). +This is the repository for the Drift configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/drift). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle -From the Airbyte repository root, run: +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: ``` ./gradlew :airbyte-integrations:connectors:source-drift:build ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/drift) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_drift/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/drift) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_drift/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source drift test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image #### Build @@ -80,31 +43,40 @@ Then run any of the connector commands as follows: docker run --rm airbyte/source-drift:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-drift:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-drift:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-drift:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-drift:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-drift:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-drift:unitTest +``` +To run acceptance and custom integration tests: ``` -docker build . --no-cache -t airbyte/source-drift:dev \ -&& python -m pytest -p connector_acceptance_test.plugin +./gradlew :airbyte-integrations:connectors:source-drift:integrationTest ``` -To run your integration tests with docker ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-drift/__init__.py b/airbyte-integrations/connectors/source-drift/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml index 192379caf094..1a7dd7340b9d 100644 --- a/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml @@ -1,20 +1,42 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-drift:dev -tests: +acceptance_tests: spec: - - spec_path: "source_drift/spec.json" + tests: + - spec_path: "source_drift/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["accounts", ] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: users + bypass_reason: "Sandbox account can't seed this stream" + - name: contacts + bypass_reason: "Sandbox account can't seed this stream" + - name: accounts + bypass_reason: "Sandbox account can't seed this stream" +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py b/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-drift/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py index 82823254d266..d49b55882333 100644 --- a/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-drift/integration_tests/acceptance.py @@ -10,5 +10,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json deleted file mode 100644 index 54a9f62bbcd3..000000000000 --- a/airbyte-integrations/connectors/source-drift/integration_tests/catalog.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "streams": [ - { - "name": "accounts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ownerId": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "accountId": { - "type": "string" - }, - "customProperties": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "name": { - "type": "string" - }, - "value": {}, - "type": { - "type": "string" - } - } - } - }, - "deleted": { - "type": "boolean" - }, - "createDateTime": { - "type": "integer" - }, - "updateDateTime": { - "type": "integer" - }, - "targeted": { - "type": "boolean" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - { - "name": "conversations", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "participants": { - "type": "array", - "items": { - "type": "integer" - } - }, - "status": { - "type": "string", - "enum": ["open", "closed", "pending", "bulk_sent"] - }, - "contactId": { - "type": "integer" - }, - "inboxId": { - "type": "integer" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "relatedPlaybookId": { - "type": ["null", "string"] - }, - "conversationTags": { - "type": "array", - "items": { - "type": "object", - "properties": { - "color": { - "type": "string", - "description": "HEX value" - }, - "name": { - "type": "string" - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - { - "name": "users", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "orgId": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "alias": { - "type": "string" - }, - "email": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "locale": { - "type": "string" - }, - "availability": { - "type": "string" - }, - "role": { - "type": "string" - }, - "timeZone": { - "type": "string" - }, - "avatarUrl": { - "type": "string" - }, - "verified": { - "type": "boolean" - }, - "bot": { - "type": "boolean" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - ] -} diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json index abfe040b2667..fde717b6dfe4 100644 --- a/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-drift/integration_tests/configured_catalog.json @@ -26,6 +26,24 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "messages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index 0cec75b519c2..f17ce487e236 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -1,23 +1,28 @@ data: allowedHosts: hosts: - - driftapi.com + - https://driftapi.com/ + registries: + oss: + enabled: true + cloud: + enabled: false # hide Source Drift https://github.com/airbytehq/airbyte/issues/24270 connectorSubtype: api connectorType: source definitionId: 445831eb-78db-4b1f-8f1f-0d96ad8739e2 - dockerImageTag: 0.2.6 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-drift githubIssueLabel: source-drift icon: drift.svg license: MIT name: Drift - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2023-08-10 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/drift tags: - - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-drift/requirements.txt b/airbyte-integrations/connectors/source-drift/requirements.txt index 9ce85523c234..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-drift/requirements.txt +++ b/airbyte-integrations/connectors/source-drift/requirements.txt @@ -1,3 +1,2 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-drift/sample_files/catalog.json b/airbyte-integrations/connectors/source-drift/sample_files/catalog.json deleted file mode 100644 index a38650255ebc..000000000000 --- a/airbyte-integrations/connectors/source-drift/sample_files/catalog.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "streams": [ - { - "name": "accounts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ownerId": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "accountId": { - "type": "string" - }, - "customProperties": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "name": { - "type": "string" - }, - "value": {}, - "type": { - "type": "string" - } - } - } - }, - "deleted": { - "type": "boolean" - }, - "createDateTime": { - "type": "integer" - }, - "updateDateTime": { - "type": "integer" - }, - "targeted": { - "type": "boolean" - } - } - } - }, - { - "name": "conversations", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "participants": { - "type": "array", - "items": { - "type": "integer" - } - }, - "status": { - "type": "string", - "enum": ["open", "closed", "pending", "bulk_sent"] - }, - "contactId": { - "type": "integer" - }, - "inboxId": { - "type": "integer" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "relatedPlaybookId": { - "type": ["null", "string"] - }, - "conversationTags": { - "type": "array", - "items": { - "type": "object", - "properties": { - "color": { - "type": "string", - "description": "HEX value" - }, - "name": { - "type": "string" - } - } - } - } - } - } - }, - { - "name": "users", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "orgId": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "alias": { - "type": "string" - }, - "email": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "locale": { - "type": "string" - }, - "availability": { - "type": "string" - }, - "role": { - "type": "string" - }, - "timeZone": { - "type": "string" - }, - "avatarUrl": { - "type": "string" - }, - "verified": { - "type": "boolean" - }, - "bot": { - "type": "boolean" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-drift/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-drift/sample_files/configured_catalog.json deleted file mode 100644 index dca6f7252ab3..000000000000 --- a/airbyte-integrations/connectors/source-drift/sample_files/configured_catalog.json +++ /dev/null @@ -1,176 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "accounts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ownerId": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "accountId": { - "type": "string" - }, - "customProperties": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "name": { - "type": "string" - }, - "value": {}, - "type": { - "type": "string" - } - } - } - }, - "deleted": { - "type": "boolean" - }, - "createDateTime": { - "type": "integer" - }, - "updateDateTime": { - "type": "integer" - }, - "targeted": { - "type": "boolean" - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "conversations", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "participants": { - "type": "array", - "items": { - "type": "integer" - } - }, - "status": { - "type": "string", - "enum": ["open", "closed", "pending", "bulk_sent"] - }, - "contactId": { - "type": "integer" - }, - "inboxId": { - "type": "integer" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "relatedPlaybookId": { - "type": ["null", "string"] - }, - "conversationTags": { - "type": "array", - "items": { - "type": "object", - "properties": { - "color": { - "type": "string", - "description": "HEX value" - }, - "name": { - "type": "string" - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "users", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "orgId": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "alias": { - "type": "string" - }, - "email": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "locale": { - "type": "string" - }, - "availability": { - "type": "string" - }, - "role": { - "type": "string" - }, - "timeZone": { - "type": "string" - }, - "avatarUrl": { - "type": "string" - }, - "verified": { - "type": "boolean" - }, - "bot": { - "type": "boolean" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-drift/sample_files/sample_config.json b/airbyte-integrations/connectors/source-drift/sample_files/sample_config.json deleted file mode 100644 index 3a18507ec5ae..000000000000 --- a/airbyte-integrations/connectors/source-drift/sample_files/sample_config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "credentials": { - "access_token": "123412341234sfsdfs" - } -} diff --git a/airbyte-integrations/connectors/source-drift/setup.py b/airbyte-integrations/connectors/source-drift/setup.py index f20267c6589a..5407ab8ae1e5 100644 --- a/airbyte-integrations/connectors/source-drift/setup.py +++ b/airbyte-integrations/connectors/source-drift/setup.py @@ -5,10 +5,12 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests~=2.22"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test", ] @@ -20,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-drift/source_drift/__init__.py b/airbyte-integrations/connectors/source-drift/source_drift/__init__.py index 6152a5b0910b..b55ab15e0129 100644 --- a/airbyte-integrations/connectors/source-drift/source_drift/__init__.py +++ b/airbyte-integrations/connectors/source-drift/source_drift/__init__.py @@ -1,4 +1,8 @@ -from .client import Client +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + from .source import SourceDrift -__all__ = ["SourceDrift", "Client"] +__all__ = ["SourceDrift"] diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/__init__.py b/airbyte-integrations/connectors/source-drift/source_drift/client/__init__.py deleted file mode 100644 index 1a0f8cfb5586..000000000000 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .client import Client -from .common import APIError, AuthError, NotFoundError, ServerError, ValidationError - -__all__ = ["Client", "APIError", "AuthError", "ServerError", "ValidationError", "NotFoundError"] diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/api.py b/airbyte-integrations/connectors/source-drift/source_drift/client/api.py deleted file mode 100644 index 36899145a543..000000000000 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/api.py +++ /dev/null @@ -1,154 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import json -from enum import IntEnum -from functools import partial -from typing import Iterator, List - -import requests - -from .common import _parsed_response, cursor_paginator, next_url_paginator - - -class APIClient: - USER_AGENT = "Airbyte contact@airbyte.io" - BASE_URL = "https://driftapi.com" - - def __init__(self, access_token: str): - self._token = access_token - self._headers = { - "Authorization": f"Bearer {self._token}", - "User-Agent": self.USER_AGENT, - "Content-Type": "application/json", - "Accept": "application/json", - } - - self.accounts = Account(client=self) - self.contacts = Contact(client=self) - self.conversations = Conversation(client=self) - self.messages = Message(client=self) - self.users = User(client=self) - - @_parsed_response - def post(self, url, data, **kwargs): - return requests.post(self.full_url(url), data=json.dumps(data), headers=self._headers, **kwargs) - - @_parsed_response - def patch(self, url, data, **kwargs): - """Used in fixtures.py only""" - return requests.patch(self.full_url(url), data=json.dumps(data), headers=self._headers, **kwargs) - - @_parsed_response - def get(self, url, **kwargs): - return requests.get(self.full_url(url), headers=self._headers, **kwargs) - - def full_url(self, url): - return f"{self.BASE_URL}/{url}" - - def check_token(self, token: str): - return self.post("app/token_info", data={"accessToken": token}) - - -class User: - def __init__(self, client: APIClient): - self.client = client - - def get(self, pk) -> dict: - return self.client.get(f"users/{pk}")["data"] - - def list(self) -> Iterator[dict]: - """Doesn't support pagination and return all users at once""" - yield from self.client.get("users/list")["data"] - - def update(self, pk, **attributes) -> dict: - params = {"userId": pk} - return self.client.patch("users/update", data=attributes, params=params) - - -class Conversation: - pagination = partial(cursor_paginator, per_page=50) - - class Status(IntEnum): - OPEN = 1 - CLOSED = 2 - PENDING = 3 - - def __init__(self, client: APIClient): - self.client = client - - def get(self, pk: int) -> dict: - return self.client.get(f"conversations/{pk}") - - def list(self, statuses: List[Status] = None) -> Iterator[dict]: - """Conversations returned will be ordered by their updatedAt time with the most recently updated at the top of the list.""" - statuses = list(map(int, statuses or [])) - params = {"statusId": statuses} - request = partial(self.client.get, url="conversations/list") - return self.pagination(request, params=params) - - def create(self, **attributes) -> dict: - return self.client.post("conversations/new", data=attributes) - - def update(self, pk: int, **attributes) -> dict: - params = {"userId": pk} - return self.client.patch("conversations/update", data=attributes, params=params) - - -class Message: - pagination = partial(cursor_paginator, per_page=50) - - def __init__(self, client: APIClient): - self.client = client - - def list(self, conversation_id: int) -> Iterator[dict]: - """You have to provide conversation ID to get list of messages""" - request = partial(self.client.get, url=f"conversations/{conversation_id}/messages") - for data in self.pagination(request): - yield from data.get("messages", []) - - def create(self, conversation_id: int, **attributes) -> dict: - return self.client.post(f"conversations/{conversation_id}/messages", data=attributes).get("data") - - -class Account: - pagination = partial(next_url_paginator, per_page=100) - - def __init__(self, client: APIClient): - self.client = client - - def get(self, pk: int) -> dict: - return self.client.get(f"accounts/{pk}") - - def list(self) -> Iterator[dict]: - request = partial(self.client.get, url="accounts") - for data in self.pagination(request): - yield from data.get("accounts", []) - - def create(self, **attributes) -> dict: - return self.client.post("accounts/create", data=attributes).get("data") - - def update(self, pk: int, **attributes) -> dict: - params = {"userId": pk} - return self.client.patch("accounts/update", data=attributes, params=params) - - -class Contact: - def __init__(self, client: APIClient): - self.client = client - - def get(self, pk: int) -> dict: - return self.client.get(f"contacts/{pk}")["data"] - - def list(self, email: str) -> Iterator[dict]: - """List contacts only possible with exact email filter""" - yield from self.client.get("contacts", params={"email": email})["data"] - - def create(self, **attributes) -> dict: - return self.client.post("contacts", data=attributes).get("data") - - def update(self, pk: int, **attributes) -> dict: - params = {"userId": pk} - return self.client.patch("contacts/update", data=attributes, params=params) diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/client.py b/airbyte-integrations/connectors/source-drift/source_drift/client/client.py deleted file mode 100644 index 0df443966092..000000000000 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/client.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from typing import Dict, Iterator, Tuple - -from airbyte_cdk.sources.deprecated.client import BaseClient - -from .api import APIClient -from .common import AuthError, ValidationError - - -class DriftAuthenticator: - def __init__(self, config: Dict): - self.config = config - - def get_token(self) -> str: - access_token = self.config.get("access_token") - if access_token: - return access_token - else: - return self.config.get("credentials").get("access_token") - - -class Client(BaseClient): - def __init__(self, **config: Dict): - super().__init__() - self._client = APIClient(access_token=DriftAuthenticator(config).get_token()) - - def stream__accounts(self, **kwargs) -> Iterator[dict]: - yield from self._client.accounts.list() - - def stream__users(self, **kwargs) -> Iterator[dict]: - yield from self._client.users.list() - - def stream__conversations(self, **kwargs) -> Iterator[dict]: - yield from self._client.conversations.list() - - def health_check(self) -> Tuple[bool, str]: - alive = True - error_msg = None - - try: - # we don't care about response, just checking authorisation - self._client.check_token("definitely_not_a_token") - except ValidationError: # this is ok because `definitely_not_a_token` - pass - except AuthError as error: - alive = False - error_msg = str(error) - - return alive, error_msg diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/common.py b/airbyte-integrations/connectors/source-drift/source_drift/client/common.py deleted file mode 100644 index 639b61160045..000000000000 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/common.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import collections -import functools -from typing import Iterator -from urllib.parse import parse_qs, urlparse - -import requests - - -class ServerError(Exception): - """Server respond with error""" - - -class APIError(Exception): - """Base class for API errors""" "" - - -class ValidationError(APIError): - """Provided data has failed validation""" - - -class AuthError(APIError): - """Token is wrong or expired""" - - -class NotFoundError(APIError): - """Object not found""" - - -class RateLimitError(APIError): - """API calls reached allowed limit""" - - -def cursor_paginator(request, start_index: int = None, per_page: int = 100, params: dict = None) -> Iterator[dict]: - """Paginator that use cursor offset to navigate""" - params = params or {} - index = start_index - while True: - result = request(params={**params, "next": index, "limit": per_page}) - if isinstance(result["data"], collections.abc.Sequence): - yield from result["data"] - else: - yield result["data"] - index = result.get("pagination", {}).get("next") - if not index: - break - - -def next_url_paginator(request, start_index: int = None, per_page: int = 100, params: dict = None) -> Iterator[dict]: - """Paginator that use next url to navigate""" - params = params or {} - size = per_page - index = start_index - while True: - result = request(params={**params, "index": index, "size": size}) - if isinstance(result["data"], collections.abc.Sequence): - yield from result["data"] - else: - yield result["data"] - - next_url = result["data"].get("next") - if not next_url: - break - - # parse url to unify request command - next_url = urlparse(next_url) - next_params = parse_qs(next_url.query) - index = next_params.get("index", [None])[0] - size = next_params.get("size", [None])[0] - - -def exception_from_code(code: int, message: str) -> Exception: - """Map response code to exception class""" - mapping = { - 400: ValidationError, - 401: AuthError, - 403: AuthError, - 429: RateLimitError, - 404: NotFoundError, - 500: ServerError, - 502: ServerError, - 503: ServerError, - 504: ServerError, - } - - return mapping.get(code, APIError)(code, message) - - -def _parsed_response(func): - """Decorator to check response status and parse its body""" - - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - response = func(*args, **kwargs) - result = response.json() if response.text else {} - if not response.ok: - msg = result # fallback to the whole response - if "error" in result: - msg = result["error"].get("message", result) - # multiple errors? grab all of them - elif "errors" in result: - msg = result["errors"] - raise exception_from_code(response.status_code, msg) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as err: - raise ServerError(err.request.status_code, "Connection Error") from err - - return result - - return wrapper diff --git a/airbyte-integrations/connectors/source-drift/source_drift/client/fixture.py b/airbyte-integrations/connectors/source-drift/source_drift/client/fixture.py deleted file mode 100644 index 622ab2c77ee1..000000000000 --- a/airbyte-integrations/connectors/source-drift/source_drift/client/fixture.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import os - -from .api import APIClient - - -class FakeDataFactory: - @staticmethod - def account(seed): - return { - "ownerId": 2000 + seed, - "name": f"Company Name {seed}", - "domain": f"www.domain.n{seed}.com", - "customProperties": [{"label": "My Number", "name": " my number", "value": 1, "type": "NUMBER"}], - "targeted": True, - } - - @staticmethod - def contact(seed): - return {"attributes": {"email": f"airbyte-test-email-{seed}@airbyte.io"}} - - @staticmethod - def conversation(seed, email=None): - return { - "email": email or f"airbyte-test-email-{seed}@airbyte.io", - "message": {"body": f"Test conversation message #{seed}", "attributes": {"integrationSource": "Message from airbyte tests"}}, - } - - @staticmethod - def message(seed): - return { - "type": "chat", - "body": f"Test message #{seed}", - } - - -def main(): - client = APIClient(access_token=os.getenv("DRIFT_TOKEN", "YOUR_TOKEN_HERE")) - - # create 120 accounts and 120 conversation with 120 new contacts - for i in range(120): - client.accounts.create(**FakeDataFactory.account(i + 1)) - conversation = client.conversations.create(**FakeDataFactory.conversation(i)) - # in each conversation create +3 additional messages - for k in range(3): - client.messages.create(conversation_id=conversation["id"], **FakeDataFactory.message(k)) - - -if __name__ == "__main__": - main() diff --git a/airbyte-integrations/connectors/source-drift/source_drift/manifest.yaml b/airbyte-integrations/connectors/source-drift/source_drift/manifest.yaml new file mode 100644 index 000000000000..20f05d3ebed3 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/source_drift/manifest.yaml @@ -0,0 +1,128 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + requester: + type: HttpRequester + url_base: "https://driftapi.com" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "NoPagination" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['next'] }}" + page_token_option: + type: "RequestPath" + field_name: "page_token" + inject_into: "request_parameter" + + accounts_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + paginator: + $ref: "#/definitions/base_paginator" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "accounts"] + name: "accounts" + primary_key: "ownerId" + $parameters: + path: "/accounts" + + conversations_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + paginator: + $ref: "#/definitions/base_paginator" + name: "conversations" + primary_key: "id" + $parameters: + path: "/conversations" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + primary_key: "id" + $parameters: + path: "/users" + + contacts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "contacts" + primary_key: "id" + path: "/contacts" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + email: "{{ config['email'] }}" + + messages_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/conversations_stream" + parent_key: "id" + partition_field: "parent_id" + + messages_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "messages" + primary_key: "id" + path: "/conversations/{{ stream_partition.parent_id }}/messages" + retriever: + $ref: "#/definitions/retriever" + paginator: + $ref: "#/definitions/base_paginator" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "messages"] + partition_router: + $ref: "#/definitions/messages_partition_router" + +streams: + - "#/definitions/accounts_stream" + - "#/definitions/conversations_stream" + - "#/definitions/users_stream" + - "#/definitions/contacts_stream" + - "#/definitions/messages_stream" + +check: + type: CheckStream + stream_names: + - "accounts" + - "conversations" + - "users" + - "contacts" + - "messages" diff --git a/airbyte-integrations/connectors/source-drift/source_drift/source.py b/airbyte-integrations/connectors/source-drift/source_drift/source.py index 25d7db241c88..24dcb151a5e7 100644 --- a/airbyte-integrations/connectors/source-drift/source_drift/source.py +++ b/airbyte-integrations/connectors/source-drift/source_drift/source.py @@ -2,11 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_cdk.sources.deprecated.base_source import BaseSource +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from .client import Client +WARNING: Do not modify this file. +""" -class SourceDrift(BaseSource): - client_class = Client +# Declarative Source +class SourceDrift(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-drift/source_drift/spec.json b/airbyte-integrations/connectors/source-drift/source_drift/spec.json deleted file mode 100644 index 29d76c79575b..000000000000 --- a/airbyte-integrations/connectors/source-drift/source_drift/spec.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/drift", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Drift Spec", - "type": "object", - "required": [], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "client_id", - "client_secret", - "access_token", - "refresh_token" - ], - "properties": { - "credentials": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Drift developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your Drift developer application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token.", - "default": "", - "airbyte_secret": true - } - } - }, - { - "title": "Access Token", - "type": "object", - "required": ["access_token"], - "properties": { - "credentials": { - "type": "string", - "const": "access_token", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Drift Access Token. See the docs for more information on how to generate this key.", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] - } - } -} diff --git a/airbyte-integrations/connectors/source-drift/source_drift/spec.yaml b/airbyte-integrations/connectors/source-drift/source_drift/spec.yaml new file mode 100644 index 000000000000..bbbc429a5ac3 --- /dev/null +++ b/airbyte-integrations/connectors/source-drift/source_drift/spec.yaml @@ -0,0 +1,106 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/drift +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Drift Spec + type: object + required: [] + additionalProperties: true + properties: + email: + type: string + description: Email used as parameter for contacts stream + title: Email parameter for contacts stream + default: "test@test.com" + credentials: + title: Authorization Method + type: object + oneOf: + - type: object + title: OAuth2.0 + required: + - client_id + - client_secret + - access_token + - refresh_token + properties: + credentials: + type: string + const: oauth2.0 + order: 0 + client_id: + type: string + title: Client ID + description: The Client ID of your Drift developer application. + airbyte_secret: true + client_secret: + type: string + title: Client Secret + description: The Client Secret of your Drift developer application. + airbyte_secret: true + access_token: + type: string + title: Access Token + description: Access Token for making authenticated requests. + airbyte_secret: true + refresh_token: + type: string + title: Refresh Token + description: Refresh Token to renew the expired Access Token. + default: "" + airbyte_secret: true + - title: Access Token + type: object + required: + - access_token + properties: + credentials: + type: string + const: access_token + order: 0 + access_token: + type: string + title: Access Token + description: + Drift Access Token. See the docs + for more information on how to generate this key. + airbyte_secret: true +advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - credentials + - credentials + predicate_value: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + properties: + access_token: + type: string + path_in_connector_config: + - credentials + - access_token + refresh_token: + type: string + path_in_connector_config: + - credentials + - refresh_token + complete_oauth_server_input_specification: + type: object + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + properties: + client_id: + type: string + path_in_connector_config: + - credentials + - client_id + client_secret: + type: string + path_in_connector_config: + - credentials + - client_secret diff --git a/airbyte-integrations/connectors/source-drift/unit_tests/test_client.py b/airbyte-integrations/connectors/source-drift/unit_tests/test_client.py deleted file mode 100644 index b7c3d928477f..000000000000 --- a/airbyte-integrations/connectors/source-drift/unit_tests/test_client.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest -from source_drift.client import AuthError, Client - -config = {"credentials": {"access_token": "wrong_key"}} - - -def test__heal_check_with_wrong_token(): - client = Client(**config) - alive, error = client.health_check() - - assert not alive - assert error == "(401, 'The access token is invalid or has expired')" - - -def test__users_with_wrong_token(): - client = Client(**config) - with pytest.raises(AuthError, match="(401, 'The access token is invalid or has expired')"): - next(client.stream__users()) diff --git a/airbyte-integrations/connectors/source-dv-360/metadata.yaml b/airbyte-integrations/connectors/source-dv-360/metadata.yaml index fee60657d6f7..acaa90cec05d 100644 --- a/airbyte-integrations/connectors/source-dv-360/metadata.yaml +++ b/airbyte-integrations/connectors/source-dv-360/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/dv-360 tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-dv-360/requirements.txt b/airbyte-integrations/connectors/source-dv-360/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-dv-360/requirements.txt +++ b/airbyte-integrations/connectors/source-dv-360/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-dv-360/setup.py b/airbyte-integrations/connectors/source-dv-360/setup.py index 1ab33aaab3b0..a8198206559e 100644 --- a/airbyte-integrations/connectors/source-dv-360/setup.py +++ b/airbyte-integrations/connectors/source-dv-360/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-api-python-client"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "pytest-mock"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock"] setup( name="source_dv_360", diff --git a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml index 8484070c094a..e71ffe623c56 100644 --- a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 50401137-8871-4c5a-abb7-1f5fda35545a dockerImageTag: 0.1.2 dockerRepository: airbyte/source-dynamodb + documentationUrl: https://docs.airbyte.com/integrations/sources/dynamodb githubIssueLabel: source-dynamodb icon: dynamodb.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/dynamodb + supportLevel: community tags: - language:java metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml b/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml index 8c4af5f6b849..c6eeee25e73c 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: false releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/sources/e2e-test + documentationUrl: https://docs.airbyte.com/integrations/sources/e2e-test tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-e2e-test-cloud/src/test/resources/expected_spec.json index 4ce7bc37a611..cae1dcdf3f19 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/src/test/resources/expected_spec.json @@ -1,119 +1,106 @@ { - "documentationUrl" : "https://docs.airbyte.com/integrations/sources/e2e-test", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/e2e-test", "protocol_version": "0.2.1", - "connectionSpecification" : { - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "Cloud E2E Test Source Spec", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Cloud E2E Test Source Spec", "additionalProperties": true, - "type" : "object", - "oneOf" : [ + "type": "object", + "oneOf": [ { - "title" : "Continuous Feed", - "type" : "object", - "required" : [ - "type", - "max_messages", - "mock_catalog" - ], - "additionalProperties" : true, - "properties" : { - "type" : { - "type" : "string", - "const" : "CONTINUOUS_FEED", - "default" : "CONTINUOUS_FEED", - "order" : 10 + "title": "Continuous Feed", + "type": "object", + "required": ["type", "max_messages", "mock_catalog"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "const": "CONTINUOUS_FEED", + "default": "CONTINUOUS_FEED", + "order": 10 }, - "max_messages" : { - "title" : "Max Records", - "description" : "Number of records to emit per stream. Min 1. Max 100 billion.", - "type" : "integer", - "default" : 100, - "min" : 1, - "max" : 100000000000, - "order" : 20 + "max_messages": { + "title": "Max Records", + "description": "Number of records to emit per stream. Min 1. Max 100 billion.", + "type": "integer", + "default": 100, + "min": 1, + "max": 100000000000, + "order": 20 }, - "seed" : { - "title" : "Random Seed", - "description" : "When the seed is unspecified, the current time millis will be used as the seed. Range: [0, 1000000].", - "type" : "integer", - "default" : 0, - "examples" : [ - 42 - ], - "min" : 0, - "max" : 1000000, - "order" : 30 + "seed": { + "title": "Random Seed", + "description": "When the seed is unspecified, the current time millis will be used as the seed. Range: [0, 1000000].", + "type": "integer", + "default": 0, + "examples": [42], + "min": 0, + "max": 1000000, + "order": 30 }, - "message_interval_ms" : { - "title" : "Message Interval (ms)", - "description" : "Interval between messages in ms. Min 0 ms. Max 60000 ms (1 minute).", - "type" : "integer", - "min" : 0, - "max" : 60000, - "default" : 0, - "order" : 40 + "message_interval_ms": { + "title": "Message Interval (ms)", + "description": "Interval between messages in ms. Min 0 ms. Max 60000 ms (1 minute).", + "type": "integer", + "min": 0, + "max": 60000, + "default": 0, + "order": 40 }, - "mock_catalog" : { - "title" : "Mock Catalog", - "type" : "object", - "order" : 50, - "oneOf" : [ + "mock_catalog": { + "title": "Mock Catalog", + "type": "object", + "order": 50, + "oneOf": [ { - "title" : "Single Schema", - "description" : "A catalog with one or multiple streams that share the same schema.", + "title": "Single Schema", + "description": "A catalog with one or multiple streams that share the same schema.", "type": "object", - "required" : [ - "type", - "stream_name", - "stream_schema" - ], - "properties" : { - "type" : { - "type" : "string", - "const" : "SINGLE_STREAM", - "default" : "SINGLE_STREAM" + "required": ["type", "stream_name", "stream_schema"], + "properties": { + "type": { + "type": "string", + "const": "SINGLE_STREAM", + "default": "SINGLE_STREAM" }, - "stream_name" : { - "title" : "Stream Name", - "description" : "Name of the data stream.", - "type" : "string", - "default" : "data_stream" + "stream_name": { + "title": "Stream Name", + "description": "Name of the data stream.", + "type": "string", + "default": "data_stream" }, - "stream_schema" : { - "title" : "Stream Schema", - "description" : "A Json schema for the stream. The schema should be compatible with draft-07. See this doc for examples.", - "type" : "string", - "default" : "{ \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } }" + "stream_schema": { + "title": "Stream Schema", + "description": "A Json schema for the stream. The schema should be compatible with draft-07. See this doc for examples.", + "type": "string", + "default": "{ \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } }" }, - "stream_duplication" : { - "title" : "Duplicate the stream N times", - "description" : "Duplicate the stream for easy load testing. Each stream name will have a number suffix. For example, if the stream name is \"ds\", the duplicated streams will be \"ds_0\", \"ds_1\", etc.", - "type" : "integer", - "default" : 1, - "min" : 1, - "max" : 10000 + "stream_duplication": { + "title": "Duplicate the stream N times", + "description": "Duplicate the stream for easy load testing. Each stream name will have a number suffix. For example, if the stream name is \"ds\", the duplicated streams will be \"ds_0\", \"ds_1\", etc.", + "type": "integer", + "default": 1, + "min": 1, + "max": 10000 } } }, { - "title" : "Multi Schema", + "title": "Multi Schema", "type": "object", - "description" : "A catalog with multiple data streams, each with a different schema.", - "required" : [ - "type", - "stream_schemas" - ], - "properties" : { - "type" : { - "type" : "string", - "const" : "MULTI_STREAM", - "default" : "MULTI_STREAM" + "description": "A catalog with multiple data streams, each with a different schema.", + "required": ["type", "stream_schemas"], + "properties": { + "type": { + "type": "string", + "const": "MULTI_STREAM", + "default": "MULTI_STREAM" }, - "stream_schemas" : { - "title" : "Streams and Schemas", - "description" : "A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. The schema should be compatible with draft-07. See this doc for examples.", - "type" : "string", - "default" : "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" } } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"boolean\" } } } }" + "stream_schemas": { + "title": "Streams and Schemas", + "description": "A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. The schema should be compatible with draft-07. See this doc for examples.", + "type": "string", + "default": "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" } } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"boolean\" } } } }" } } } diff --git a/airbyte-integrations/connectors/source-e2e-test/metadata.yaml b/airbyte-integrations/connectors/source-e2e-test/metadata.yaml index 93a90aea39b2..c695e3a3052f 100644 --- a/airbyte-integrations/connectors/source-e2e-test/metadata.yaml +++ b/airbyte-integrations/connectors/source-e2e-test/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/e2e-test tags: - language:java + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json index da020e3be094..306ef0f3f020 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json @@ -8,162 +8,143 @@ "type": "object", "oneOf": [ { - "title" : "Legacy Infinite Feed", - "description" : "This mode is used for Platform acceptance tests. The catalog has one \"data\" stream, which has one string field \"column1\". This mode will emit messages infinitely.", - "required" : [ - "type", - "max_records" - ], + "title": "Legacy Infinite Feed", + "description": "This mode is used for Platform acceptance tests. The catalog has one \"data\" stream, which has one string field \"column1\". This mode will emit messages infinitely.", + "required": ["type", "max_records"], "type": "object", - "additionalProperties" : true, - "properties" : { - "type" : { - "type" : "string", - "const" : "INFINITE_FEED", - "default" : "INFINITE_FEED" + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "const": "INFINITE_FEED", + "default": "INFINITE_FEED" }, - "max_records" : { - "title" : "Max Records", - "description" : "Number of records to emit. If not set, defaults to infinity.", - "type" : "integer" + "max_records": { + "title": "Max Records", + "description": "Number of records to emit. If not set, defaults to infinity.", + "type": "integer" }, - "message_interval" : { - "title" : "Message Interval", - "description" : "Interval between messages in ms.", - "type" : "integer" + "message_interval": { + "title": "Message Interval", + "description": "Interval between messages in ms.", + "type": "integer" } } }, { - "title" : "Legacy Exception After N", - "description" : "This mode is used for Platform acceptance tests. The catalog has one \"data\" stream, which has one string field \"column1\". This mode will throw an exception after N messages.", - "required" : [ - "type", - "throw_after_n_records" - ], + "title": "Legacy Exception After N", + "description": "This mode is used for Platform acceptance tests. The catalog has one \"data\" stream, which has one string field \"column1\". This mode will throw an exception after N messages.", + "required": ["type", "throw_after_n_records"], "type": "object", - "additionalProperties" : true, - "properties" : { - "type" : { - "type" : "string", - "const" : "EXCEPTION_AFTER_N", - "default" : "EXCEPTION_AFTER_N" + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "const": "EXCEPTION_AFTER_N", + "default": "EXCEPTION_AFTER_N" }, - "throw_after_n_records" : { - "title" : "Throw After N Records", - "description" : "Number of records to emit before throwing an exception. Min 1.", - "type" : "integer", - "min" : 1 + "throw_after_n_records": { + "title": "Throw After N Records", + "description": "Number of records to emit before throwing an exception. Min 1.", + "type": "integer", + "min": 1 } } }, { - "title" : "Continuous Feed", - "type" : "object", - "required" : [ - "type", - "max_messages", - "mock_catalog" - ], - "additionalProperties" : true, - "properties" : { - "type" : { - "type" : "string", - "const" : "CONTINUOUS_FEED", - "default" : "CONTINUOUS_FEED", - "order" : 10 + "title": "Continuous Feed", + "type": "object", + "required": ["type", "max_messages", "mock_catalog"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "const": "CONTINUOUS_FEED", + "default": "CONTINUOUS_FEED", + "order": 10 }, - "max_messages" : { - "title" : "Max Records", - "description" : "Number of records to emit per stream. Min 1. Max 100 billion.", - "type" : "integer", - "default" : 100, - "min" : 1, - "max" : 100000000000, - "order" : 20 + "max_messages": { + "title": "Max Records", + "description": "Number of records to emit per stream. Min 1. Max 100 billion.", + "type": "integer", + "default": 100, + "min": 1, + "max": 100000000000, + "order": 20 }, - "seed" : { - "title" : "Random Seed", - "description" : "When the seed is unspecified, the current time millis will be used as the seed. Range: [0, 1000000].", - "type" : "integer", - "default" : 0, - "examples" : [ - 42 - ], - "min" : 0, - "max" : 1000000, - "order" : 30 + "seed": { + "title": "Random Seed", + "description": "When the seed is unspecified, the current time millis will be used as the seed. Range: [0, 1000000].", + "type": "integer", + "default": 0, + "examples": [42], + "min": 0, + "max": 1000000, + "order": 30 }, - "message_interval_ms" : { - "title" : "Message Interval (ms)", - "description" : "Interval between messages in ms. Min 0 ms. Max 60000 ms (1 minute).", - "type" : "integer", - "min" : 0, - "max" : 60000, - "default" : 0, - "order" : 40 + "message_interval_ms": { + "title": "Message Interval (ms)", + "description": "Interval between messages in ms. Min 0 ms. Max 60000 ms (1 minute).", + "type": "integer", + "min": 0, + "max": 60000, + "default": 0, + "order": 40 }, - "mock_catalog" : { - "title" : "Mock Catalog", - "type" : "object", - "order" : 50, - "oneOf" : [ + "mock_catalog": { + "title": "Mock Catalog", + "type": "object", + "order": 50, + "oneOf": [ { - "title" : "Single Schema", - "description" : "A catalog with one or multiple streams that share the same schema.", - "type" : "object", - "required" : [ - "type", - "stream_name", - "stream_schema" - ], - "properties" : { - "type" : { - "type" : "string", - "const" : "SINGLE_STREAM", - "default" : "SINGLE_STREAM" + "title": "Single Schema", + "description": "A catalog with one or multiple streams that share the same schema.", + "type": "object", + "required": ["type", "stream_name", "stream_schema"], + "properties": { + "type": { + "type": "string", + "const": "SINGLE_STREAM", + "default": "SINGLE_STREAM" }, - "stream_name" : { - "title" : "Stream Name", - "description" : "Name of the data stream.", - "type" : "string", - "default" : "data_stream" + "stream_name": { + "title": "Stream Name", + "description": "Name of the data stream.", + "type": "string", + "default": "data_stream" }, - "stream_schema" : { - "title" : "Stream Schema", - "description" : "A Json schema for the stream. The schema should be compatible with draft-07. See this doc for examples.", - "type" : "string", - "default" : "{ \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } }" + "stream_schema": { + "title": "Stream Schema", + "description": "A Json schema for the stream. The schema should be compatible with draft-07. See this doc for examples.", + "type": "string", + "default": "{ \"type\": \"object\", \"properties\": { \"column1\": { \"type\": \"string\" } } }" }, - "stream_duplication" : { - "title" : "Duplicate the stream N times", - "description" : "Duplicate the stream for easy load testing. Each stream name will have a number suffix. For example, if the stream name is \"ds\", the duplicated streams will be \"ds_0\", \"ds_1\", etc.", - "type" : "integer", - "default" : 1, - "min" : 1, - "max" : 10000 + "stream_duplication": { + "title": "Duplicate the stream N times", + "description": "Duplicate the stream for easy load testing. Each stream name will have a number suffix. For example, if the stream name is \"ds\", the duplicated streams will be \"ds_0\", \"ds_1\", etc.", + "type": "integer", + "default": 1, + "min": 1, + "max": 10000 } } }, { - "title" : "Multi Schema", - "type" : "object", - "description" : "A catalog with multiple data streams, each with a different schema.", - "required" : [ - "type", - "stream_schemas" - ], - "properties" : { - "type" : { - "type" : "string", - "const" : "MULTI_STREAM", - "default" : "MULTI_STREAM" + "title": "Multi Schema", + "type": "object", + "description": "A catalog with multiple data streams, each with a different schema.", + "required": ["type", "stream_schemas"], + "properties": { + "type": { + "type": "string", + "const": "MULTI_STREAM", + "default": "MULTI_STREAM" }, - "stream_schemas" : { - "title" : "Streams and Schemas", - "description" : "A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. The schema should be compatible with draft-07. See this doc for examples.", - "type" : "string", - "default" : "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" } } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"boolean\" } } } }" + "stream_schemas": { + "title": "Streams and Schemas", + "description": "A Json object specifying multiple data streams and their schemas. Each key in this object is one stream name. Each value is the schema for that stream. The schema should be compatible with draft-07. See this doc for examples.", + "type": "string", + "default": "{ \"stream1\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"string\" } } }, \"stream2\": { \"type\": \"object\", \"properties\": { \"field1\": { \"type\": \"boolean\" } } } }" } } } @@ -172,28 +153,22 @@ } }, { - "title" : "Benchmark", - "description" : "This mode is used for speed benchmarks. Specifically, it should be used for testing the throughput of the platform and destination. It optimizes for emitting records very quickly, so that it should never be the bottleneck.", + "title": "Benchmark", + "description": "This mode is used for speed benchmarks. Specifically, it should be used for testing the throughput of the platform and destination. It optimizes for emitting records very quickly, so that it should never be the bottleneck.", "type": "object", - "required" : [ - "type", - "schema", - "terminationCondition" - ], - "additionalProperties" : true, - "properties" : { - "type" : { - "type" : "string", - "const" : "BENCHMARK", - "default" : "BENCHMARK" + "required": ["type", "schema", "terminationCondition"], + "additionalProperties": true, + "properties": { + "type": { + "type": "string", + "const": "BENCHMARK", + "default": "BENCHMARK" }, "schema": { "title": "Schema", "description": "schema of the data in the benchmark.", "type": "string", - "enum":[ - "FIVE_STRING_COLUMNS" - ] + "enum": ["FIVE_STRING_COLUMNS"] }, "terminationCondition": { "title": "Termination Condition", @@ -203,12 +178,12 @@ { "title": "max records", "type": "object", - "required": ["type","max"], + "required": ["type", "max"], "properties": { - "type" : { - "type" : "string", - "const" : "MAX_RECORDS", - "default" : "MAX_RECORDS" + "type": { + "type": "string", + "const": "MAX_RECORDS", + "default": "MAX_RECORDS" }, "max": { "type": "number" diff --git a/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml b/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml index 3857ac686125..1c7dad3da8d3 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml +++ b/airbyte-integrations/connectors/source-elasticsearch/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml index d98a2e0da976..6b3d33420107 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml +++ b/airbyte-integrations/connectors/source-emailoctopus/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-emailoctopus/requirements.txt b/airbyte-integrations/connectors/source-emailoctopus/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/requirements.txt +++ b/airbyte-integrations/connectors/source-emailoctopus/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/setup.py b/airbyte-integrations/connectors/source-emailoctopus/setup.py index 46260d8f25a0..9cd42d8b7050 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/setup.py +++ b/airbyte-integrations/connectors/source-emailoctopus/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-everhour/metadata.yaml b/airbyte-integrations/connectors/source-everhour/metadata.yaml index 81926803a11d..a41a9283a3d1 100644 --- a/airbyte-integrations/connectors/source-everhour/metadata.yaml +++ b/airbyte-integrations/connectors/source-everhour/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/everhour tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-everhour/requirements.txt b/airbyte-integrations/connectors/source-everhour/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-everhour/requirements.txt +++ b/airbyte-integrations/connectors/source-everhour/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-everhour/setup.py b/airbyte-integrations/connectors/source-everhour/setup.py index da7860e6ac55..e1838cba9eda 100644 --- a/airbyte-integrations/connectors/source-everhour/setup.py +++ b/airbyte-integrations/connectors/source-everhour/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1"] setup( name="source_everhour", diff --git a/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/tasks.json b/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/tasks.json index 2d74aaf95067..ef33f489b20e 100644 --- a/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/tasks.json @@ -1,63 +1,63 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "type": "string" - }, - "status": { - "type": "string" - }, - "url": { - "type": "string" - }, - "iteration": { - "type": "string" - }, - "projects": { - "type": "array", - "items": {"type": "string"} - }, - "createdAt": { - "type": "string" - }, - "dueOn": { - "type": "string" - }, - "labels": { - "type": "array", - "items": {"type": "string"} - }, - "time": { - "type": "object", - "properties": { - "total": { - "type": "integer" - }, - "users": { - "type": ["null", "object"] - }, - "timerTime": { - "type": "integer" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "status": { + "type": "string" + }, + "url": { + "type": "string" + }, + "iteration": { + "type": "string" + }, + "projects": { + "type": "array", + "items": { "type": "string" } + }, + "createdAt": { + "type": "string" + }, + "dueOn": { + "type": "string" + }, + "labels": { + "type": "array", + "items": { "type": "string" } + }, + "time": { + "type": "object", + "properties": { + "total": { + "type": "integer" + }, + "users": { + "type": ["null", "object"] + }, + "timerTime": { + "type": "integer" } - }, - "completed": { - "type": "boolean" - }, - "completedAt": { - "type": "string" - }, - "assignees": { - "type": "array", - "items": {"type": "string"} } + }, + "completed": { + "type": "boolean" + }, + "completedAt": { + "type": "string" + }, + "assignees": { + "type": "array", + "items": { "type": "string" } } - } \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/time.json b/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/time.json index 4dff2f42bb8f..caf299b6c3c5 100644 --- a/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/time.json +++ b/airbyte-integrations/connectors/source-everhour/source_everhour/schemas/time.json @@ -80,7 +80,7 @@ }, "history": { "type": ["null", "array"], - "items": { + "items": { "type": ["null", "object"], "additionalProperties": true } diff --git a/airbyte-integrations/connectors/source-everhour/source_everhour/spec.yaml b/airbyte-integrations/connectors/source-everhour/source_everhour/spec.yaml index fa9af9ba6019..edd568a21921 100644 --- a/airbyte-integrations/connectors/source-everhour/source_everhour/spec.yaml +++ b/airbyte-integrations/connectors/source-everhour/source_everhour/spec.yaml @@ -10,7 +10,7 @@ connectionSpecification: type: string title: API Key description: >- - Everhour API Key. See the docs - for information on how to generate this key. + Everhour API Key. See the docs + for information on how to generate this key. airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-exchange-rates/.dockerignore b/airbyte-integrations/connectors/source-exchange-rates/.dockerignore new file mode 100644 index 000000000000..e7a736f7a47d --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_exchange_rates +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile index 01d3442be145..2d396799391b 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile +++ b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile @@ -1,20 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -ENV CODE_PATH="source_exchange_rates" -ENV WORKDIR=/airbyte/integration_code -WORKDIR $WORKDIR -COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -COPY main.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code -RUN pip install . +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_exchange_rates ./source_exchange_rates ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.2.8 +LABEL io.airbyte.version=1.3.0 LABEL io.airbyte.name=airbyte/source-exchange-rates diff --git a/airbyte-integrations/connectors/source-exchange-rates/README.md b/airbyte-integrations/connectors/source-exchange-rates/README.md index c96e743d77e2..78969eda4723 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/README.md +++ b/airbyte-integrations/connectors/source-exchange-rates/README.md @@ -1,34 +1,10 @@ # Exchange Rates Source -This is the repository for the Exchange Rates source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/exchangeratesapi). +This is the repository for the Exchange Rates configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/exchange-rates). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,21 +14,13 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -The exchangerates API does not require authentication. - -### Locally running the connector -``` -python main.py spec -python main.py check --config integration_tests/config.json -python main.py discover --config integration_tests/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/exchange-rates) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_exchange_rates/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source exchange-rates test creds` +and place them into `secrets/config.json`. ### Locally running the connector docker image @@ -73,23 +41,42 @@ the Dockerfile. Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-exchange-rates:dev spec -docker run --rm -v $(pwd)/integration_tests:/integration_tests airbyte/source-exchange-rates:dev check --config /integration_tests/config.json -docker run --rm -v $(pwd)/integration_tests:/integration_tests airbyte/source-exchange-rates:dev discover --config /integration_tests/config.json -docker run --rm -v $(pwd)/integration_tests:/integration_tests airbyte/source-exchange-rates:dev read --config /integration_tests/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-exchange-rates:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-exchange-rates:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-exchange-rates:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` +## Testing -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-exchange-rates:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-exchange-rates:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-exchange-rates:integrationTest +``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-exchange-rates/__init__.py b/airbyte-integrations/connectors/source-exchange-rates/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml index 011c2171cac4..325b1b1ea837 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml @@ -1,27 +1,34 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-exchange-rates:dev -tests: +acceptance_tests: spec: - - spec_path: "source_exchange_rates/spec.yaml" + tests: + - spec_path: "source_exchange_rates/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 1.3.0 connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 1.3.0 basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - # exchange_rates stream records are different on each read - # full_refresh: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # there are no state messages with current setup - # incremental: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" - # cursor_paths: - # exchange_rates: ["date"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + # bypass_reason: "This connector does not implement incremental sync" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-docker.sh new file mode 100755 index 000000000000..5797d20fe9a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-docker.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/__init__.py b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py index d49b55882333..82823254d266 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/acceptance.py @@ -10,4 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json index d94bc91739af..5add0beeec9e 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json @@ -6,6 +6,12 @@ "json_schema": { "type": "object", "properties": { + "success": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "integer"] + }, "base": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..534fbcfc52b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/expected_records.jsonl @@ -0,0 +1 @@ +{"stream":"exchange_rates","data":{"success":true,"timestamp":1691602143,"base":"EUR","date":"2023-08-09","rates":{"AED":4.03227,"AFN":92.943648,"ALL":103.350532,"AMD":423.83111,"ANG":1.978785,"AOA":905.578551,"ARS":313.070955,"AUD":1.680022,"AWG":1.978801,"AZN":1.863022,"BAM":1.95628,"BBD":2.216764,"BDT":120.170576,"BGN":1.955314,"BHD":0.41391,"BIF":3110.977374,"BMD":1.097809,"BND":1.477276,"BOB":7.587241,"BRL":5.386838,"BSD":1.097879,"BTC":3.7219745e-05,"BTN":90.949869,"BWP":14.85678,"BYN":2.771205,"BYR":21517.062701,"BZD":2.213154,"CAD":1.4741,"CDF":2678.654917,"CHF":0.962433,"CLF":0.034212,"CLP":943.951693,"CNY":7.915094,"COP":4441.527931,"CRC":591.374708,"CUC":1.097809,"CUP":29.091947,"CVE":110.281005,"CZK":24.276293,"DJF":195.102992,"DKK":7.45092,"DOP":62.178369,"DZD":149.078078,"EGP":33.929006,"ERN":16.46714,"ETB":60.537384,"EUR":1,"FJD":2.457665,"FKP":0.862606,"GBP":0.862504,"GEL":2.859776,"GGP":0.862606,"GHS":12.259622,"GIP":0.862606,"GMD":66.310384,"GNF":9442.402481,"GTQ":8.635551,"GYD":229.887915,"HKD":8.586022,"HNL":27.001624,"HRK":7.381515,"HTG":149.868132,"HUF":388.656384,"IDR":16686.317454,"ILS":4.050028,"IMP":0.862606,"INR":90.954436,"IQD":1449.16872,"IRR":46451.054009,"ISK":144.296093,"JEP":0.862606,"JMD":169.514675,"JOD":0.777357,"JPY":157.665729,"KES":157.528953,"KGS":96.464523,"KHR":4539.154921,"KMF":493.822047,"KPW":988.011736,"KRW":1442.707893,"KWD":0.33773,"KYD":0.914933,"KZT":489.654585,"LAK":21338.303891,"LBP":16480.268701,"LKR":350.299128,"LRD":204.953944,"LSL":20.532542,"LTL":3.241545,"LVL":0.664054,"LYD":5.26029,"MAD":10.675116,"MDL":19.460749,"MGA":4883.442696,"MKD":61.635682,"MMK":2305.586623,"MNT":3798.348961,"MOP":8.841112,"MRO":391.917739,"MUR":50.279702,"MVR":16.851129,"MWK":1190.851809,"MXN":18.733781,"MYR":5.018069,"MZN":69.436786,"NAD":20.528952,"NGN":844.511944,"NIO":40.170221,"NOK":11.21242,"NPR":145.514024,"NZD":1.812175,"OMR":0.422674,"PAB":1.097924,"PEN":4.066301,"PGK":3.938002,"PHP":61.834133,"PKR":315.592598,"PLN":4.465493,"PYG":7969.245435,"QAR":3.997084,"RON":4.945077,"RSD":117.257838,"RUB":106.350259,"RWF":1294.129272,"SAR":4.118864,"SBD":9.187332,"SCR":14.699133,"SDG":660.088465,"SEK":11.73001,"SGD":1.477778,"SHP":1.335759,"SLE":23.021629,"SLL":21681.733803,"SOS":625.197209,"SSP":660.329725,"SRD":42.125123,"STD":22722.436487,"SYP":14381.213261,"SZL":20.855021,"THB":38.511424,"TJS":12.060963,"TMT":3.842333,"TND":3.390858,"TOP":2.598899,"TRY":29.660173,"TTD":7.4462,"TWD":34.841179,"TZS":2744.523271,"UAH":40.547217,"UGX":4013.021054,"USD":1.097809,"UYU":41.800255,"UZS":12807.141929,"VEF":3394943.011329,"VES":33.007388,"VND":26067.482339,"VUV":131.432581,"WST":3.015276,"XAF":656.117963,"XAG":0.048389,"XAU":0.000573,"XCD":2.966885,"XDR":0.821209,"XOF":656.052211,"XPF":119.825742,"YER":274.836836,"ZAR":20.829231,"ZMK":9881.60248,"ZMW":20.724122,"ZWL":353.494154}},"emitted_at":1687166006562} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml index 18202810906f..dc7062891b8f 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml @@ -3,22 +3,24 @@ data: hosts: - ${subdomain}.apilayer.com - apilayer.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: e2b40e36-aa0e-4bed-b41b-bcea6fa348b1 - dockerImageTag: 1.2.8 + dockerImageTag: 1.3.0 dockerRepository: airbyte/source-exchange-rates githubIssueLabel: source-exchange-rates icon: exchangeratesapi.svg license: MIT name: Exchange Rates Api - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-19 releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/exchangeratesapi + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/exchange-rates tags: - - language:python + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-exchange-rates/setup.py b/airbyte-integrations/connectors/source-exchange-rates/setup.py index 895418957150..120437119f72 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-exchange-rates/setup.py @@ -5,12 +5,24 @@ from setuptools import find_packages, setup +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + setup( name="source_exchange_rates", - description="Source implementation for Exchange Rate API.", + description="Source implementation for Exchange Rates.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, - install_requires=["airbyte-cdk~=0.1", "pendulum>=2,<3"], + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, ) diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/__init__.py b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/__init__.py index 7273cd938410..ab0ed6ffb52f 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/__init__.py +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/__init__.py @@ -1,3 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + from .source import SourceExchangeRates __all__ = ["SourceExchangeRates"] diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml new file mode 100644 index 000000000000..82f5825f366a --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml @@ -0,0 +1,48 @@ +version: 0.44.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - exchange_rates +streams: + - type: DeclarativeStream + name: exchange_rates + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.apilayer.com/exchangerates_data/ + path: latest + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['access_key'] }}" + inject_into: + type: RequestOption + field_name: apikey + inject_into: header + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + incremental_sync: + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%d" + cursor_field: date + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" +metadata: + autoImportSchema: + exchange_rates: true diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json index 1c4986139cd3..e68948f2b053 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json @@ -1,15 +1,23 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "required": ["base", "date", "rates"], "properties": { + "success": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "integer"] + }, "base": { - "type": "string" + "type": ["null", "string"] }, "date": { - "type": "string" + "type": ["null", "string"] }, "rates": { - "type": "object", + "type": ["null", "object"], "properties": { "AED": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py index 576b417aadbf..6de5b6b1561a 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py @@ -2,115 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from pendulum import DateTime +WARNING: Do not modify this file. +""" -class ExchangeRates(HttpStream): - - date_field_name = "date" - - # HttpStream related fields - url_base = "https://api.apilayer.com/exchangerates_data/" - cursor_field = date_field_name - primary_key = "" - - def __init__(self, base: Optional[str], start_date: DateTime, access_key: str, ignore_weekends: Optional[bool]): - super().__init__() - self._base = base - self._start_date = start_date - self.access_key = access_key - self.ignore_weekends = ignore_weekends - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return stream_slice[self.date_field_name] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = {} - - if self._base is not None: - params["base"] = self._base - - return params - - def request_headers(self, **kwargs) -> MutableMapping[str, Any]: - headers = {"apikey": self.access_key} - - return headers - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - yield response_json - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - stream_state = stream_state or {} - start_date = pendulum.parse(stream_state.get(self.date_field_name, self._start_date)) - return chunk_date_range(start_date, self.ignore_weekends) - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - current_stream_state = current_stream_state or {} - current_stream_state[self.date_field_name] = max( - latest_record[self.date_field_name], current_stream_state.get(self.date_field_name, self._start_date) - ) - return current_stream_state - - -def chunk_date_range(start_date: DateTime, ignore_weekends: bool) -> Iterable[Mapping[str, Any]]: - """ - Returns a list of each day between the start date and now. Ignore weekends since exchanges don't run on weekends. - The return value is a list of dicts {'date': date_string}. - """ - days = [] - now = pendulum.now() - while start_date < now: - day_of_week = start_date.day_of_week - if day_of_week != pendulum.SATURDAY and day_of_week != pendulum.SUNDAY or not ignore_weekends: - days.append({"date": start_date.to_date_string()}) - start_date = start_date.add(days=1) - - return days - - -class SourceExchangeRates(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - try: - headers = {"apikey": config["access_key"]} - params = {} - base = config.get("base") - if base is not None: - params["base"] = base - - resp = requests.get(f"{ExchangeRates.url_base}{config['start_date']}", params=params, headers=headers) - status = resp.status_code - logger.info(f"Ping response code: {status}") - if status == 200: - return True, None - # When API requests is sent but the requested data is not available or the API call fails - # for some reason, a JSON error is returned. - # https://exchangeratesapi.io/documentation/#errors - error = resp.json().get("error", resp.json()) - code = error.get("code") - message = error.get("message") or error.get("info") - # If code is base_currency_access_restricted, error is caused by switching base currency while using free - # plan - if code == "base_currency_access_restricted": - message = f"{message} (this plan doesn't support selecting the base currency)" - return False, message - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [ExchangeRates(config.get("base"), config["start_date"], config["access_key"], config.get("ignore_weekends", True))] +# Declarative Source +class SourceExchangeRates(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-exchange-rates/unit_tests/conftest.py b/airbyte-integrations/connectors/source-exchange-rates/unit_tests/conftest.py deleted file mode 100644 index 4557bc3816c9..000000000000 --- a/airbyte-integrations/connectors/source-exchange-rates/unit_tests/conftest.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture(name="config") -def config_fixture(requests_mock): - config = {"start_date": "2022-09-08", "base": "USD", "access_key": "KEY"} - - return config - - -@fixture(name="mock_stream") -def mock_stream_fixture(requests_mock): - def _mock_stream(path, response=None, status_code=200): - if response is None: - response = {} - - url = f"https://api.apilayer.com/exchangerates_data/{path}" - requests_mock.get(url, json=response, status_code=status_code) - - return _mock_stream diff --git a/airbyte-integrations/connectors/source-exchange-rates/unit_tests/test_source.py b/airbyte-integrations/connectors/source-exchange-rates/unit_tests/test_source.py deleted file mode 100644 index 36a121ec46dc..000000000000 --- a/airbyte-integrations/connectors/source-exchange-rates/unit_tests/test_source.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging - -from source_exchange_rates.source import SourceExchangeRates - -logger = logging.getLogger("airbyte") - - -def test_check_connection_ok(config, mock_stream): - response = {"success": True, "timestamp": 1662681599, "historical": True, "base": "USD", "date": "2022-09-08", "rates": {"AED": 1}} - mock_stream(config["start_date"], response=response) - ok, error_msg = SourceExchangeRates().check_connection(logger, config=config) - - assert ok - assert not error_msg - - -def test_check_connection_exception(config, mock_stream): - message = ( - "You have exceeded your daily/monthly API rate limit. Please review and upgrade your subscription plan at " - "https://promptapi.com/subscriptions to continue. " - ) - response = {"message": message} - mock_stream(config["start_date"], response=response, status_code=429) - ok, error_msg = SourceExchangeRates().check_connection(logger, config=config) - - assert not ok - assert error_msg == message - - -def test_streams(config): - streams = SourceExchangeRates().streams(config) - - assert len(streams) == 1 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index e9f97da7fab1..a7647a8e0db0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.0 +LABEL io.airbyte.version=1.1.7 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index b7abd3868bbe..6a93f0f9ecbc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -7,7 +7,7 @@ acceptance_tests: tests: - spec_path: "integration_tests/spec.json" backward_compatibility_tests_config: - disable_for_version: 0.3.6 # pattern for Account ID was added + disable_for_version: "0.5.0" connection: tests: - config_path: "secrets/config.json" @@ -17,65 +17,44 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.5.0" basic_read: tests: - config_path: "secrets/config.json" + ignored_fields: + ad_account: + - name: age + bypass_reason: is changeable + - name: amount_spent + bypass_reason: is changeable + - name: capabilities + bypass_reason: is changeable + ad_creatives: + - name: thumbnail_url + bypass_reason: is changeable + images: + - name: permalink_url + bypass_reason: is changeable + - name: url + bypass_reason: is changeable + - name: url_128 + bypass_reason: is changeable + ads_insights_demographics_dma_region: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_dma: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing empty_streams: - - name: "ad_account" - bypass_reason: "Cannot populate" - - name: "ad_sets" - bypass_reason: "Thumbnail urls changes permanently" - - name: "ad_creatives" - bypass_reason: "Thumbnail urls changes permanently" - - name: "ads_insights_action_carousel_card" - bypass_reason: "Data not permanent" - - name: "ads_insights_action_conversion_device" - bypass_reason: "Data not permanent" - name: "ads_insights_action_product_id" bypass_reason: "Data not permanent" - - name: "ads_insights_action_reaction" - bypass_reason: "Data not permanent" - - name: "ads_insights_action_video_sound" - bypass_reason: "Data not permanent" - - name: "ads_insights_action_video_type" - bypass_reason: "Data not permanent" - - name: "ads_insights_action_type" - bypass_reason: "Data not permanent" - - name: "ads_insights_delivery_device" - bypass_reason: "Data not permanent" - - name: "ads_insights_delivery_platform" - bypass_reason: "Data not permanent" - - name: "ads_insights_delivery_platform_and_device_platform" - bypass_reason: "Data not permanent" - - name: "ads_insights_demographics_age" - bypass_reason: "Data not permanent" - - name: "ads_insights_age_and_gender" - bypass_reason: "Data not permanent" - - name: "ads_insights_demographics_country" - bypass_reason: "Data not permanent" - - name: "ads_insights_demographics_dma_region" - bypass_reason: "Data not permanent" - - name: "ads_insights_demographics_gender" - bypass_reason: "Data not permanent" - - name: "ads_insights_dma" - bypass_reason: "Data not permanent" - - name: "ads_insights_country" - bypass_reason: "Data not permanent" - - name: "ads_insights_platform_and_device" - bypass_reason: "Data not permanent" - - name: "ads_insights_region" - bypass_reason: "Data not permanent" - - name: "activities" - bypass_reason: "age field autoupdated" - - name: "custom_conversions" - bypass_reason: "Cannot populate" - - name: "images" - bypass_reason: "Links not permanent" - name: "videos" bypass_reason: "Cannot populate" timeout_seconds: 4800 expect_records: path: "integration_tests/expected_records.jsonl" + extra_records: yes incremental: tests: - config_path: "secrets/config.json" @@ -86,3 +65,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" timeout_seconds: 4800 + ignored_fields: + activities: + - name: extra_data + bypass_reason: "image_url in extra_data is changing frequently" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/ads_insights_configured_catalog.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/ads_insights_configured_catalog.json index 5946b803ea73..213268e4eea0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/ads_insights_configured_catalog.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/ads_insights_configured_catalog.json @@ -4,10 +4,7 @@ "stream": { "name": "ads_insights_action_carousel_card", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -16,10 +13,7 @@ "stream": { "name": "ads_insights_action_conversion_device", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -28,10 +22,7 @@ "stream": { "name": "ads_insights_action_product_id", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -40,10 +31,7 @@ "stream": { "name": "ads_insights_action_reaction", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -52,10 +40,7 @@ "stream": { "name": "ads_insights_action_video_sound", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -64,10 +49,7 @@ "stream": { "name": "ads_insights_action_video_type", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -76,10 +58,7 @@ "stream": { "name": "ads_insights_action_type", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -88,10 +67,7 @@ "stream": { "name": "ads_insights_delivery_device", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -100,10 +76,7 @@ "stream": { "name": "ads_insights_delivery_platform", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -112,10 +85,7 @@ "stream": { "name": "ads_insights_delivery_platform_and_device_platform", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -124,10 +94,7 @@ "stream": { "name": "ads_insights_demographics_age", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -136,10 +103,7 @@ "stream": { "name": "ads_insights_age_and_gender", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -148,10 +112,7 @@ "stream": { "name": "ads_insights_demographics_country", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -160,10 +121,7 @@ "stream": { "name": "ads_insights_demographics_dma_region", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -172,10 +130,7 @@ "stream": { "name": "ads_insights_demographics_gender", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -184,10 +139,7 @@ "stream": { "name": "ads_insights_dma", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -196,10 +148,7 @@ "stream": { "name": "ads_insights_country", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -208,10 +157,7 @@ "stream": { "name": "ads_insights_platform_and_device", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" @@ -220,13 +166,10 @@ "stream": { "name": "ads_insights_region", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", "destination_sync_mode": "append" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json index 99f1a7a70c02..651412dbaf73 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json @@ -1,193 +1,148 @@ { - "streams": [ - { - "stream": { - "name": "ad_account", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ad_sets", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ad_creatives", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights_age_and_gender", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights_country", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights_region", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights_dma", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights_platform_and_device", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ads_insights_action_type", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "campaigns", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "custom_conversions", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "images", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "videos", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "activities", - "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} \ No newline at end of file + "streams": [ + { + "stream": { + "name": "ad_account", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_sets", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_creatives", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights_age_and_gender", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights_country", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights_region", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights_dma", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights_platform_and_device", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads_insights_action_type", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "custom_conversions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "images", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "videos", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "activities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl index 3a40dd4637bd..a75f785fadce 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl @@ -1,17 +1,28 @@ -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666124230398"}, "id": "23853620229650398", "updated_time": "2023-03-21T22:41:46-0700", "created_time": "2023-03-17T08:04:31-0700", "name": "With The Highest Standard for Reliability", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["link_click"], "post": ["662226902575095"], "post.wall": ["112704783733939"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["662226902575095"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047368} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666124200398"}, "id": "23853620217720398", "updated_time": "2023-03-21T22:37:51-0700", "created_time": "2023-03-17T08:04:34-0700", "name": "Reliability Is The Cornerstone", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["662226905908428"]}, {"action.type": ["link_click"], "post": ["662226905908428"], "post.wall": ["112704783733939"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047372} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853630871570398"}, "id": "23853620247100398", "updated_time": "2023-03-21T22:37:44-0700", "created_time": "2023-03-17T08:04:29-0700", "name": "CV1_PP_SQ_Carousel", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["660122832785502"]}, {"action.type": ["link_click"], "post": ["660122832785502"], "post.wall": ["112704783733939"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047374} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666125630398"}, "id": "23853620198790398", "updated_time": "2023-03-21T22:33:56-0700", "created_time": "2023-03-17T08:04:29-0700", "name": "Don't Compromise Between Cost/Relaibility", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["660122622785523", "662226992575086"]}, {"action.type": ["link_click"], "post": ["660122622785523", "662226992575086"], "post.wall": ["112704783733939"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047377} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666125970398"}, "id": "23853619670410398", "updated_time": "2023-03-21T22:25:48-0700", "created_time": "2023-03-17T08:04:31-0700", "name": "The New Standard For Data Movemenet", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["660122656118853", "662226985908420"]}, {"action.type": ["link_click"], "post": ["660122656118853", "662226985908420"], "post.wall": ["112704783733939"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047380} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853498584890398", "adset_id": "23853498584920398", "status": "ACTIVE", "creative": {"id": "23853516655660398"}, "id": "23853498584940398", "updated_time": "2023-03-09T11:51:38-0800", "created_time": "2023-03-08T14:18:46-0800", "name": "New Leads Ad", "targeting": {"age_max": 65, "age_min": 18, "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "1829444163943758", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["2667253716886462"]}, {"action.type": ["link_click"], "post": ["653858936745225"], "post.wall": ["112704783733939"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["653858936745225"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["3735721989788818"]}]}, "emitted_at": 1682686047382} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23846765228240398", "adset_id": "23846765228280398", "status": "ACTIVE", "creative": {"id": "23846784944290398"}, "id": "23846784938030398", "updated_time": "2021-08-27T08:32:19-0700", "created_time": "2021-02-11T18:24:12-0800", "name": "Stock photo ad 2", "targeting": {"age_max": 65, "age_min": 18, "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"]}, "effective_status": "CAMPAIGN_PAUSED", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["dwell"], "creative": ["23846784944290398"]}, {"action.type": ["attention_event"], "creative": ["23846784944290398"]}, {"action.type": ["link_click"], "post": ["243077367363346"], "post.wall": ["112704783733939"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["243077367363346"]}]}, "emitted_at": 1682686047384} -{"stream": "ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23846765228240398", "adset_id": "23846765228280398", "status": "ACTIVE", "creative": {"id": "23846815595220398"}, "id": "23846765228310398", "updated_time": "2021-08-27T08:32:19-0700", "created_time": "2021-02-09T15:04:15-0800", "name": "Airbyte Ad", "targeting": {"age_max": 65, "age_min": 18, "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"]}, "effective_status": "CAMPAIGN_PAUSED", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["2667253716886462"]}, {"action.type": ["attention_event"], "creative": ["23846815595220398"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["244953057175777"]}, {"action.type": ["link_click"], "post": ["244953057175777"], "post.wall": ["112704783733939"]}, {"action.type": ["dwell"], "creative": ["23846815595220398"]}]}, "emitted_at": 1682686047387} -{"stream": "ads_insights", "data": {"account_currency": "USD", "account_id": "212551616838260", "account_name": "Airbyte", "actions": [{"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "link_click", "value": 1.0, "1d_click": 1.0, "7d_click": 1.0, "28d_click": 1.0}, {"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "page_engagement", "value": 1.0, "1d_click": 1.0, "7d_click": 1.0, "28d_click": 1.0}, {"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "post_engagement", "value": 1.0, "1d_click": 1.0, "7d_click": 1.0, "28d_click": 1.0}], "ad_id": "23846765228310398", "ad_name": "Airbyte Ad", "adset_id": "23846765228280398", "adset_name": "Vanilla awareness ad set", "buying_type": "AUCTION", "campaign_id": "23846765228240398", "campaign_name": "Airbyte Awareness Campaign 1 (sherif)", "catalog_segment_value_mobile_purchase_roas": [{"value": 0.0}], "clicks": 1, "conversion_rate_ranking": "UNKNOWN", "cost_per_estimated_ad_recallers": 0.009538, "cost_per_inline_link_click": 1.24, "cost_per_inline_post_engagement": 1.24, "cost_per_unique_click": 1.24, "cost_per_unique_inline_link_click": 1.24, "cpc": 1.24, "cpm": 1.27572, "cpp": 1.341991, "created_time": "2021-02-09", "ctr": 0.102881, "date_start": "2021-02-14", "date_stop": "2021-02-14", "engagement_rate_ranking": "UNKNOWN", "estimated_ad_recall_rate": 14.069264, "estimated_ad_recallers": 130.0, "frequency": 1.051948, "impressions": 972, "inline_link_click_ctr": 0.102881, "inline_link_clicks": 1, "inline_post_engagement": 1, "mobile_app_purchase_roas": [{"value": 0.0}], "objective": "BRAND_AWARENESS", "optimization_goal": "AD_RECALL_LIFT", "outbound_clicks": [{"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "outbound_click", "value": 1.0}], "quality_ranking": "UNKNOWN", "reach": 924, "social_spend": 0.0, "spend": 1.24, "unique_actions": [{"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "link_click", "value": 1.0, "1d_click": 1.0, "7d_click": 1.0, "28d_click": 1.0}, {"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "page_engagement", "value": 1.0, "1d_click": 1.0, "7d_click": 1.0, "28d_click": 1.0}, {"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "post_engagement", "value": 1.0, "1d_click": 1.0, "7d_click": 1.0, "28d_click": 1.0}], "unique_clicks": 1, "unique_ctr": 0.108225, "unique_inline_link_click_ctr": 0.108225, "unique_inline_link_clicks": 1, "unique_link_clicks_ctr": 0.108225, "unique_outbound_clicks": [{"action_destination": "241903844147365", "action_target_id": "241903844147365", "action_type": "outbound_click", "value": 1.0}], "updated_time": "2021-08-27", "website_ctr": [{"action_type": "link_click", "value": 0.102881}], "website_purchase_roas": [{"value": 0.0}], "wish_bid": 0.0}, "emitted_at": 1682686056966} -{"stream": "ads_insights", "data": {"account_currency": "USD", "account_id": "212551616838260", "account_name": "Airbyte", "ad_id": "23846784938030398", "ad_name": "Stock photo ad 2", "adset_id": "23846765228280398", "adset_name": "Vanilla awareness ad set", "buying_type": "AUCTION", "campaign_id": "23846765228240398", "campaign_name": "Airbyte Awareness Campaign 1 (sherif)", "catalog_segment_value_mobile_purchase_roas": [{"value": 0.0}], "clicks": 1, "conversion_rate_ranking": "UNKNOWN", "cost_per_estimated_ad_recallers": 0.008889, "cost_per_unique_click": 0.8, "cpc": 0.8, "cpm": 1.255887, "cpp": 1.296596, "created_time": "2021-02-11", "ctr": 0.156986, "date_start": "2021-02-14", "date_stop": "2021-02-14", "engagement_rate_ranking": "UNKNOWN", "estimated_ad_recall_rate": 14.58671, "estimated_ad_recallers": 90.0, "frequency": 1.032415, "impressions": 637, "inline_link_clicks": 0, "inline_post_engagement": 0, "mobile_app_purchase_roas": [{"value": 0.0}], "objective": "BRAND_AWARENESS", "optimization_goal": "AD_RECALL_LIFT", "quality_ranking": "UNKNOWN", "reach": 617, "social_spend": 0.0, "spend": 0.8, "unique_clicks": 1, "unique_ctr": 0.162075, "unique_inline_link_clicks": 0, "unique_link_clicks_ctr": 0.0, "updated_time": "2021-08-27", "website_purchase_roas": [{"value": 0.0}], "wish_bid": 0.0}, "emitted_at": 1682686056980} -{"stream": "ads_insights", "data": {"account_currency": "USD", "account_id": "212551616838260", "account_name": "Airbyte", "actions": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "link_click", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "page_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "post_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}], "ad_id": "23846765228310398", "ad_name": "Airbyte Ad", "adset_id": "23846765228280398", "adset_name": "Vanilla awareness ad set", "buying_type": "AUCTION", "campaign_id": "23846765228240398", "campaign_name": "Airbyte Awareness Campaign 1 (sherif)", "catalog_segment_value_mobile_purchase_roas": [{"value": 0.0}], "clicks": 3, "conversion_rate_ranking": "UNKNOWN", "cost_per_estimated_ad_recallers": 0.007, "cost_per_inline_link_click": 0.396667, "cost_per_inline_post_engagement": 0.396667, "cost_per_unique_click": 0.396667, "cost_per_unique_inline_link_click": 0.396667, "cpc": 0.396667, "cpm": 0.902199, "cpp": 0.948207, "created_time": "2021-02-09", "ctr": 0.227445, "date_start": "2021-02-15", "date_stop": "2021-02-15", "engagement_rate_ranking": "UNKNOWN", "estimated_ad_recall_rate": 13.545817, "estimated_ad_recallers": 170.0, "frequency": 1.050996, "impressions": 1319, "inline_link_click_ctr": 0.227445, "inline_link_clicks": 3, "inline_post_engagement": 3, "mobile_app_purchase_roas": [{"value": 0.0}], "objective": "BRAND_AWARENESS", "optimization_goal": "AD_RECALL_LIFT", "outbound_clicks": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "outbound_click", "value": 3.0}], "quality_ranking": "UNKNOWN", "reach": 1255, "social_spend": 0.0, "spend": 1.19, "unique_actions": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "link_click", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "page_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "post_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}], "unique_clicks": 3, "unique_ctr": 0.239044, "unique_inline_link_click_ctr": 0.239044, "unique_inline_link_clicks": 3, "unique_link_clicks_ctr": 0.239044, "unique_outbound_clicks": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "outbound_click", "value": 3.0}], "updated_time": "2021-08-27", "website_ctr": [{"action_type": "link_click", "value": 0.227445}], "website_purchase_roas": [{"value": 0.0}], "wish_bid": 0.0}, "emitted_at": 1682686057366} -{"stream": "ads_insights", "data": {"account_currency": "USD", "account_id": "212551616838260", "account_name": "Airbyte", "ad_id": "23846784938030398", "ad_name": "Stock photo ad 2", "adset_id": "23846765228280398", "adset_name": "Vanilla awareness ad set", "buying_type": "AUCTION", "campaign_id": "23846765228240398", "campaign_name": "Airbyte Awareness Campaign 1 (sherif)", "catalog_segment_value_mobile_purchase_roas": [{"value": 0.0}], "clicks": 0, "conversion_rate_ranking": "UNKNOWN", "cost_per_estimated_ad_recallers": 0.0085, "cpm": 1.177285, "cpp": 1.233672, "created_time": "2021-02-11", "ctr": 0.0, "date_start": "2021-02-15", "date_stop": "2021-02-15", "engagement_rate_ranking": "UNKNOWN", "estimated_ad_recall_rate": 14.513788, "estimated_ad_recallers": 100.0, "frequency": 1.047896, "impressions": 722, "inline_link_clicks": 0, "inline_post_engagement": 0, "mobile_app_purchase_roas": [{"value": 0.0}], "objective": "BRAND_AWARENESS", "optimization_goal": "AD_RECALL_LIFT", "quality_ranking": "UNKNOWN", "reach": 689, "social_spend": 0.0, "spend": 0.85, "unique_clicks": 0, "unique_ctr": 0.0, "unique_inline_link_clicks": 0, "unique_link_clicks_ctr": 0.0, "updated_time": "2021-08-27", "website_purchase_roas": [{"value": 0.0}], "wish_bid": 0.0}, "emitted_at": 1682686057381} -{"stream": "campaigns", "data": {"account_id": "212551616838260", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2021-01-18T21:36:42-0800", "effective_status": "PAUSED", "id": "23846542053890398", "name": "Fake Campaign 0", "objective": "MESSAGES", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "1969-12-31T15:59:59-0800", "updated_time": "2021-02-18T01:00:02-0800"}, "emitted_at": 1682686106887} -{"stream": "campaigns", "data": {"account_id": "212551616838260", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2021-01-18T21:36:16-0800", "effective_status": "PAUSED", "id": "23846542048150398", "name": "Fake Campaign 0", "objective": "MESSAGES", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "1969-12-31T15:59:59-0800", "updated_time": "2021-02-18T01:00:04-0800"}, "emitted_at": 1682686106887} -{"stream": "campaigns", "data": {"account_id": "212551616838260", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2021-01-18T21:33:03-0800", "effective_status": "PAUSED", "id": "23846542041640398", "name": "Fake Campaign 0", "objective": "MESSAGES", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "1969-12-31T15:59:59-0800", "updated_time": "2021-02-18T01:00:03-0800"}, "emitted_at": 1682686106888} -{"stream": "campaigns", "data": {"account_id": "212551616838260", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2021-01-18T21:20:51-0800", "effective_status": "PAUSED", "id": "23846541931180398", "name": "Fake Campaign 0", "objective": "MESSAGES", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "1969-12-31T15:59:59-0800", "updated_time": "2021-02-18T01:00:04-0800"}, "emitted_at": 1682686106889} -{"stream": "campaigns", "data": {"account_id": "212551616838260", "bid_strategy": "LOWEST_COST_WITHOUT_CAP", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2020-04-13T18:13:49-0700", "effective_status": "PAUSED", "id": "23844521599700398", "lifetime_budget": 3000.0, "name": "Dataline Test Unblock", "objective": "CONVERSIONS", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "2020-04-13T18:21:59-0700", "stop_time": "2020-04-30T00:00:00-0700", "updated_time": "2021-07-25T00:50:53-0700"}, "emitted_at": 1682686106889} +{"stream":"ad_account","data":{"id":"act_212551616838260","account_id":"212551616838260","account_status":1,"age":1219.3692361111,"amount_spent":"39125","balance":"0","business":{"id":"1506473679510495","name":"Airbyte"},"business_city":"","business_country_code":"US","business_name":"","business_street":"","business_street2":"","can_create_brand_lift_study":false,"capabilities":["CAN_CREATE_CALL_ADS","CAN_SEE_GROWTH_OPPORTUNITY_DATA","ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT","CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE","CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S","ENABLE_BIZ_DISCO_ADS","ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS","ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS","ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS","ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE","ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION","ENABLE_PAC_FOR_BIZ_DISCO_ADS","CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_RECURRING_BUDGET","HAS_VALID_PAYMENT_METHODS","CAN_USE_LINK_CLICK_BILLING_EVENT","CAN_USE_CPA_BILLING_EVENT","CAN_SEE_NEW_CONVERSION_WINDOW_NUX","ADS_INSTREAM_INTERFACE_INTEGRITY","ADS_INSTREAM_LINK_CLICK","ADS_INSTREAM_LINK_CLICK_IMAGE","ADS_IN_OBJECTIVES_DEPRECATION","MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES","CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL","ALLOW_INSTREAM_ONLY_FOR_REACH","ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS","CAN_CREATE_INSTAGRAM_EXPLORE_ADS","ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY","ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN","INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW","ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION","ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY","WHATSAPP_DESTINATION_ADS","CTM_ADS_CREATION_CLICK_TO_DIRECT","CTW_ADS_ENABLE_IG_FEED_PLACEMENT","CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE","CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER","CTW_ADS_TRUSTED_TIER_ADVERTISER","ADS_PLACEMENT_MARKETPLACE","CAN_CHANGE_BILLING_THRESHOLD","CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING","CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING","ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D","ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT","ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET","CAN_SEE_APP_AD_EVENTS","CAN_SEE_NEW_STANDARD_EVENTS_BETA","CAN_SEE_VCK_HOLIDAY_TEMPLATES","ENABLE_DCO_FOR_FB_STORY_ADS","CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT","CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY","CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES","CAN_USE_IG_SEARCH_PLACEMENT","CAN_USE_IG_SEARCH_GRID_ADS","CAN_USE_IG_REELS_PAC_CAROUSEL","CAN_USE_IG_REELS_POSITION","CAN_SEE_CONVERSION_LIFT_SUMMARY","CAN_USE_IG_PROFILE_FEED_POSITION","CAN_USE_IG_PROFILE_FEED_AUTO_PLACEMENT","CAN_USE_IG_PROFILE_FEED_DPA_CREATION","CAN_USE_IG_REELS_AUTO_PLACEMENT","CAN_USE_IG_REELS_REACH_AND_FREQUENCY","CAN_USE_IG_REELS_OVERLAY_POSITION","CAN_USE_IG_REELS_OVERLAY_AUTO_PLACEMENT","CAN_USE_IG_REELS_OVERLAY_PAC","CAN_USE_IG_SHOP_TAB_PAC","CAN_SEE_LEARNING_STAGE","ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS","ENABLE_MESSENGER_INBOX_VIDEO_ADS","ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS","ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS","ENABLE_REACH_FOR_FB_STORY_ADS","CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT","ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW","ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY","ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS","ENABLE_LEAD_GEN_FOR_FB_STORY_ADS","CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH","CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW","CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT","ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS","CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS","CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS","CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION","CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE","CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE","CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT","RF_ALLOW_MARKETPLACE_ACCOUNT","RF_ALLOW_SEARCH_ACCOUNT","VERTICAL_VIDEO_PAC_INSTREAM_UPSELL","IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH","ADS_BM_REQUIREMENTS_OCT_15_RELEASE","ENABLE_POST_ENGAGEMENT_FOR_FB_STORY","ENBABLE_CATALOG_SALES_FOR_FB_STORY","CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS","CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS","IS_NON_TAIL_AD_ACCOUNT","IS_IN_DSA_GK","IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT","IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT","IS_IN_ODAX_EXPERIENCE","IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT","IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT","IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT","CAN_USE_MARKETPLACE_DESKTOP","ADS_MERCHANT_OVERLAYS_DEPRECATION","CONNECTIONS_DEPRECATION_V2","CAN_USE_LIVE_VIDEO_FOR_THRUPLAY","CAN_SEE_HEC_AM_FLOW","CAN_SEE_POLITICAL_FLOW","ADS_INSTREAM_PLACEMENT_CATALOG_SALES","ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS","ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS","ENABLE_REACH_FOR_FB_GROUP_TAB_ADS","CAN_USE_CONVERSATIONS_OPTIMIZATION","ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS","CAN_USE_IG_STORY_POLLS_PAC_CREATION","IOS14_CEO_CAMPAIGN_CREATION","ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS","DIGITAL_CIRCULAR_ADS","CAN_SEE_SAFR_V3_FLOW","CAN_USE_FB_REELS_POSITION","CAN_USE_ADS_ON_FB_REELS_POSITION","CAN_USE_FB_REELS_AUTO_PLACEMENT","ENABLE_FB_REELS_CREATION_PAC_ADS","ENABLE_FB_REELS_CREATION_DCO_ADS","ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS","ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS","RF_CPA_BILLING_DEPRECATION_PHASE_2","ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES","ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION","ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION_L2_NUX","ENABLE_ADS_ON_IG_SHOP_TAB_DEPRECATION_L2_NUX","ADS_RF_FB_REELS_PLACEMENT","ENABLE_ADS_ON_FB_INSTANT_ARTICLE_DEPRECATION_L2_NUX","REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY"],"created_time":"2020-04-13T18:04:59-0700","currency":"USD","disable_reason":0,"end_advertiser":1506473679510495,"end_advertiser_name":"Airbyte","fb_entity":85,"has_migrated_permissions":true,"is_attribution_spec_system_default":true,"is_direct_deals_enabled":false,"is_in_3ds_authorization_enabled_market":false,"is_notifications_enabled":true,"is_personal":0,"is_tax_id_required":false,"min_campaign_group_spend_cap":10000,"min_daily_budget":100,"name":"Airbyte","offsite_pixels_tos_accepted":true,"owner":1506473679510495,"rf_spec":{"min_reach_limits":{"US":200000,"CA":200000,"GB":200000,"AR":200000,"AU":200000,"AT":200000,"BE":200000,"BR":200000,"CL":200000,"CN":200000,"CO":200000,"HR":200000,"DK":200000,"DO":200000,"EG":200000,"FI":200000,"FR":200000,"DE":200000,"GR":200000,"HK":200000,"IN":200000,"ID":200000,"IE":200000,"IL":200000,"IT":200000,"JP":200000,"JO":200000,"KW":200000,"LB":200000,"MY":200000,"MX":200000,"NL":200000,"NZ":200000,"NG":200000,"NO":200000,"PK":200000,"PA":200000,"PE":200000,"PH":200000,"PL":200000,"RU":200000,"SA":200000,"RS":200000,"SG":200000,"ZA":200000,"KR":200000,"ES":200000,"SE":200000,"CH":200000,"TW":200000,"TH":200000,"TR":200000,"AE":200000,"VE":200000,"PT":200000,"LU":200000,"BG":200000,"CZ":200000,"SI":200000,"IS":200000,"SK":200000,"LT":200000,"TT":200000,"BD":200000,"LK":200000,"KE":200000,"HU":200000,"MA":200000,"CY":200000,"JM":200000,"EC":200000,"RO":200000,"BO":200000,"GT":200000,"CR":200000,"QA":200000,"SV":200000,"HN":200000,"NI":200000,"PY":200000,"UY":200000,"PR":200000,"BA":200000,"PS":200000,"TN":200000,"BH":200000,"VN":200000,"GH":200000,"MU":200000,"UA":200000,"MT":200000,"BS":200000,"MV":200000,"OM":200000,"MK":200000,"LV":200000,"EE":200000,"IQ":200000,"DZ":200000,"AL":200000,"NP":200000,"MO":200000,"ME":200000,"SN":200000,"GE":200000,"BN":200000,"UG":200000,"GP":200000,"BB":200000,"AZ":200000,"TZ":200000,"LY":200000,"MQ":200000,"CM":200000,"BW":200000,"ET":200000,"KZ":200000,"NA":200000,"MG":200000,"NC":200000,"MD":200000,"FJ":200000,"BY":200000,"JE":200000,"GU":200000,"YE":200000,"ZM":200000,"IM":200000,"HT":200000,"KH":200000,"AW":200000,"PF":200000,"AF":200000,"BM":200000,"GY":200000,"AM":200000,"MW":200000,"AG":200000,"RW":200000,"GG":200000,"GM":200000,"FO":200000,"LC":200000,"KY":200000,"BJ":200000,"AD":200000,"GD":200000,"VI":200000,"BZ":200000,"VC":200000,"MN":200000,"MZ":200000,"ML":200000,"AO":200000,"GF":200000,"UZ":200000,"DJ":200000,"BF":200000,"MC":200000,"TG":200000,"GL":200000,"GA":200000,"GI":200000,"CD":200000,"KG":200000,"PG":200000,"BT":200000,"KN":200000,"SZ":200000,"LS":200000,"LA":200000,"LI":200000,"MP":200000,"SR":200000,"SC":200000,"VG":200000,"TC":200000,"DM":200000,"MR":200000,"AX":200000,"SM":200000,"SL":200000,"NE":200000,"CG":200000,"AI":200000,"YT":200000,"CV":200000,"GN":200000,"TM":200000,"BI":200000,"TJ":200000,"VU":200000,"SB":200000,"ER":200000,"WS":200000,"AS":200000,"FK":200000,"GQ":200000,"TO":200000,"KM":200000,"PW":200000,"FM":200000,"CF":200000,"SO":200000,"MH":200000,"VA":200000,"TD":200000,"KI":200000,"ST":200000,"TV":200000,"NR":200000,"RE":200000,"LR":200000,"ZW":200000,"CI":200000,"MM":200000,"AN":200000,"AQ":200000,"BQ":200000,"BV":200000,"IO":200000,"CX":200000,"CC":200000,"CK":200000,"CW":200000,"TF":200000,"GW":200000,"HM":200000,"XK":200000,"MS":200000,"NU":200000,"NF":200000,"PN":200000,"BL":200000,"SH":200000,"MF":200000,"PM":200000,"SX":200000,"GS":200000,"SS":200000,"SJ":200000,"TL":200000,"TK":200000,"UM":200000,"WF":200000,"EH":200000},"countries":["US","CA","GB","AR","AU","AT","BE","BR","CL","CN","CO","HR","DK","DO","EG","FI","FR","DE","GR","HK","IN","ID","IE","IL","IT","JP","JO","KW","LB","MY","MX","NL","NZ","NG","NO","PK","PA","PE","PH","PL","RU","SA","RS","SG","ZA","KR","ES","SE","CH","TW","TH","TR","AE","VE","PT","LU","BG","CZ","SI","IS","SK","LT","TT","BD","LK","KE","HU","MA","CY","JM","EC","RO","BO","GT","CR","QA","SV","HN","NI","PY","UY","PR","BA","PS","TN","BH","VN","GH","MU","UA","MT","BS","MV","OM","MK","EE","LV","IQ","DZ","AL","NP","MO","ME","SN","GE","BN","UG","GP","BB","ZW","CI","AZ","TZ","LY","MQ","MM","CM","BW","ET","KZ","NA","MG","NC","MD","FJ","BY","JE","GU","YE","ZM","IM","HT","KH","AW","PF","AF","BM","GY","AM","MW","AG","RW","GG","GM","FO","LC","KY","BJ","AD","GD","VI","BZ","VC","MN","MZ","ML","AO","GF","UZ","DJ","BF","MC","TG","GL","GA","GI","CD","KG","PG","BT","KN","SZ","LS","LA","LI","MP","SR","SC","VG","TC","DM","MR","AX","SM","SL","NE","CG","AI","YT","LR","CV","GN","TM","BI","TJ","VU","SB","ER","WS","AS","FK","GQ","TO","KM","PW","FM","CF","SO","MH","VA","TD","KI","ST","TV","NR","RE","AN","AQ","BQ","BV","IO","CX","CC","CK","CW","TF","GW","HM","XK","MS","NU","NF","PN","BL","SH","MF","PM","SX","GS","SS","SJ","TL","TK","UM","WF","EH"],"min_campaign_duration":{"US":1,"CA":1,"GB":1,"AR":1,"AU":1,"AT":1,"BE":1,"BR":1,"CL":1,"CN":1,"CO":1,"HR":1,"DK":1,"DO":1,"EG":1,"FI":1,"FR":1,"DE":1,"GR":1,"HK":1,"IN":1,"ID":1,"IE":1,"IL":1,"IT":1,"JP":1,"JO":1,"KW":1,"LB":1,"MY":1,"MX":1,"NL":1,"NZ":1,"NG":1,"NO":1,"PK":1,"PA":1,"PE":1,"PH":1,"PL":1,"RU":1,"SA":1,"RS":1,"SG":1,"ZA":1,"KR":1,"ES":1,"SE":1,"CH":1,"TW":1,"TH":1,"TR":1,"AE":1,"VE":1,"PT":1,"LU":1,"BG":1,"CZ":1,"SI":1,"IS":1,"SK":1,"LT":1,"TT":1,"BD":1,"LK":1,"KE":1,"HU":1,"MA":1,"CY":1,"JM":1,"EC":1,"RO":1,"BO":1,"GT":1,"CR":1,"QA":1,"SV":1,"HN":1,"NI":1,"PY":1,"UY":1,"PR":1,"BA":1,"PS":1,"TN":1,"BH":1,"VN":1,"GH":1,"MU":1,"UA":1,"MT":1,"BS":1,"MV":1,"OM":1,"MK":1,"LV":1,"EE":1,"IQ":1,"DZ":1,"AL":1,"NP":1,"MO":1,"ME":1,"SN":1,"GE":1,"BN":1,"UG":1,"GP":1,"BB":1,"AZ":1,"TZ":1,"LY":1,"MQ":1,"CM":1,"BW":1,"ET":1,"KZ":1,"NA":1,"MG":1,"NC":1,"MD":1,"FJ":1,"BY":1,"JE":1,"GU":1,"YE":1,"ZM":1,"IM":1,"HT":1,"KH":1,"AW":1,"PF":1,"AF":1,"BM":1,"GY":1,"AM":1,"MW":1,"AG":1,"RW":1,"GG":1,"GM":1,"FO":1,"LC":1,"KY":1,"BJ":1,"AD":1,"GD":1,"VI":1,"BZ":1,"VC":1,"MN":1,"MZ":1,"ML":1,"AO":1,"GF":1,"UZ":1,"DJ":1,"BF":1,"MC":1,"TG":1,"GL":1,"GA":1,"GI":1,"CD":1,"KG":1,"PG":1,"BT":1,"KN":1,"SZ":1,"LS":1,"LA":1,"LI":1,"MP":1,"SR":1,"SC":1,"VG":1,"TC":1,"DM":1,"MR":1,"AX":1,"SM":1,"SL":1,"NE":1,"CG":1,"AI":1,"YT":1,"CV":1,"GN":1,"TM":1,"BI":1,"TJ":1,"VU":1,"SB":1,"ER":1,"WS":1,"AS":1,"FK":1,"GQ":1,"TO":1,"KM":1,"PW":1,"FM":1,"CF":1,"SO":1,"MH":1,"VA":1,"TD":1,"KI":1,"ST":1,"TV":1,"NR":1,"RE":1,"LR":1,"ZW":1,"CI":1,"MM":1,"AN":1,"AQ":1,"BQ":1,"BV":1,"IO":1,"CX":1,"CC":1,"CK":1,"CW":1,"TF":1,"GW":1,"HM":1,"XK":1,"MS":1,"NU":1,"NF":1,"PN":1,"BL":1,"SH":1,"MF":1,"PM":1,"SX":1,"GS":1,"SS":1,"SJ":1,"TL":1,"TK":1,"UM":1,"WF":1,"EH":1},"max_campaign_duration":{"US":90,"CA":90,"GB":90,"AR":90,"AU":90,"AT":90,"BE":90,"BR":90,"CL":90,"CN":90,"CO":90,"HR":90,"DK":90,"DO":90,"EG":90,"FI":90,"FR":90,"DE":90,"GR":90,"HK":90,"IN":90,"ID":90,"IE":90,"IL":90,"IT":90,"JP":90,"JO":90,"KW":90,"LB":90,"MY":90,"MX":90,"NL":90,"NZ":90,"NG":90,"NO":90,"PK":90,"PA":90,"PE":90,"PH":90,"PL":90,"RU":90,"SA":90,"RS":90,"SG":90,"ZA":90,"KR":90,"ES":90,"SE":90,"CH":90,"TW":90,"TH":90,"TR":90,"AE":90,"VE":90,"PT":90,"LU":90,"BG":90,"CZ":90,"SI":90,"IS":90,"SK":90,"LT":90,"TT":90,"BD":90,"LK":90,"KE":90,"HU":90,"MA":90,"CY":90,"JM":90,"EC":90,"RO":90,"BO":90,"GT":90,"CR":90,"QA":90,"SV":90,"HN":90,"NI":90,"PY":90,"UY":90,"PR":90,"BA":90,"PS":90,"TN":90,"BH":90,"VN":90,"GH":90,"MU":90,"UA":90,"MT":90,"BS":90,"MV":90,"OM":90,"MK":90,"LV":90,"EE":90,"IQ":90,"DZ":90,"AL":90,"NP":90,"MO":90,"ME":90,"SN":90,"GE":90,"BN":90,"UG":90,"GP":90,"BB":90,"AZ":90,"TZ":90,"LY":90,"MQ":90,"CM":90,"BW":90,"ET":90,"KZ":90,"NA":90,"MG":90,"NC":90,"MD":90,"FJ":90,"BY":90,"JE":90,"GU":90,"YE":90,"ZM":90,"IM":90,"HT":90,"KH":90,"AW":90,"PF":90,"AF":90,"BM":90,"GY":90,"AM":90,"MW":90,"AG":90,"RW":90,"GG":90,"GM":90,"FO":90,"LC":90,"KY":90,"BJ":90,"AD":90,"GD":90,"VI":90,"BZ":90,"VC":90,"MN":90,"MZ":90,"ML":90,"AO":90,"GF":90,"UZ":90,"DJ":90,"BF":90,"MC":90,"TG":90,"GL":90,"GA":90,"GI":90,"CD":90,"KG":90,"PG":90,"BT":90,"KN":90,"SZ":90,"LS":90,"LA":90,"LI":90,"MP":90,"SR":90,"SC":90,"VG":90,"TC":90,"DM":90,"MR":90,"AX":90,"SM":90,"SL":90,"NE":90,"CG":90,"AI":90,"YT":90,"CV":90,"GN":90,"TM":90,"BI":90,"TJ":90,"VU":90,"SB":90,"ER":90,"WS":90,"AS":90,"FK":90,"GQ":90,"TO":90,"KM":90,"PW":90,"FM":90,"CF":90,"SO":90,"MH":90,"VA":90,"TD":90,"KI":90,"ST":90,"TV":90,"NR":90,"RE":90,"LR":90,"ZW":90,"CI":90,"MM":90,"AN":90,"AQ":90,"BQ":90,"BV":90,"IO":90,"CX":90,"CC":90,"CK":90,"CW":90,"TF":90,"GW":90,"HM":90,"XK":90,"MS":90,"NU":90,"NF":90,"PN":90,"BL":90,"SH":90,"MF":90,"PM":90,"SX":90,"GS":90,"SS":90,"SJ":90,"TL":90,"TK":90,"UM":90,"WF":90,"EH":90},"max_days_to_finish":{"US":180,"CA":180,"GB":180,"AR":180,"AU":180,"AT":180,"BE":180,"BR":180,"CL":180,"CN":180,"CO":180,"HR":180,"DK":180,"DO":180,"EG":180,"FI":180,"FR":180,"DE":180,"GR":180,"HK":180,"IN":180,"ID":180,"IE":180,"IL":180,"IT":180,"JP":180,"JO":180,"KW":180,"LB":180,"MY":180,"MX":180,"NL":180,"NZ":180,"NG":180,"NO":180,"PK":180,"PA":180,"PE":180,"PH":180,"PL":180,"RU":180,"SA":180,"RS":180,"SG":180,"ZA":180,"KR":180,"ES":180,"SE":180,"CH":180,"TW":180,"TH":180,"TR":180,"AE":180,"VE":180,"PT":180,"LU":180,"BG":180,"CZ":180,"SI":180,"IS":180,"SK":180,"LT":180,"TT":180,"BD":180,"LK":180,"KE":180,"HU":180,"MA":180,"CY":180,"JM":180,"EC":180,"RO":180,"BO":180,"GT":180,"CR":180,"QA":180,"SV":180,"HN":180,"NI":180,"PY":180,"UY":180,"PR":180,"BA":180,"PS":180,"TN":180,"BH":180,"VN":180,"GH":180,"MU":180,"UA":180,"MT":180,"BS":180,"MV":180,"OM":180,"MK":180,"LV":180,"EE":180,"IQ":180,"DZ":180,"AL":180,"NP":180,"MO":180,"ME":180,"SN":180,"GE":180,"BN":180,"UG":180,"GP":180,"BB":180,"AZ":180,"TZ":180,"LY":180,"MQ":180,"CM":180,"BW":180,"ET":180,"KZ":180,"NA":180,"MG":180,"NC":180,"MD":180,"FJ":180,"BY":180,"JE":180,"GU":180,"YE":180,"ZM":180,"IM":180,"HT":180,"KH":180,"AW":180,"PF":180,"AF":180,"BM":180,"GY":180,"AM":180,"MW":180,"AG":180,"RW":180,"GG":180,"GM":180,"FO":180,"LC":180,"KY":180,"BJ":180,"AD":180,"GD":180,"VI":180,"BZ":180,"VC":180,"MN":180,"MZ":180,"ML":180,"AO":180,"GF":180,"UZ":180,"DJ":180,"BF":180,"MC":180,"TG":180,"GL":180,"GA":180,"GI":180,"CD":180,"KG":180,"PG":180,"BT":180,"KN":180,"SZ":180,"LS":180,"LA":180,"LI":180,"MP":180,"SR":180,"SC":180,"VG":180,"TC":180,"DM":180,"MR":180,"AX":180,"SM":180,"SL":180,"NE":180,"CG":180,"AI":180,"YT":180,"CV":180,"GN":180,"TM":180,"BI":180,"TJ":180,"VU":180,"SB":180,"ER":180,"WS":180,"AS":180,"FK":180,"GQ":180,"TO":180,"KM":180,"PW":180,"FM":180,"CF":180,"SO":180,"MH":180,"VA":180,"TD":180,"KI":180,"ST":180,"TV":180,"NR":180,"RE":180,"LR":180,"ZW":180,"CI":180,"MM":180,"AN":180,"AQ":180,"BQ":180,"BV":180,"IO":180,"CX":180,"CC":180,"CK":180,"CW":180,"TF":180,"GW":180,"HM":180,"XK":180,"MS":180,"NU":180,"NF":180,"PN":180,"BL":180,"SH":180,"MF":180,"PM":180,"SX":180,"GS":180,"SS":180,"SJ":180,"TL":180,"TK":180,"UM":180,"WF":180,"EH":180},"global_io_max_campaign_duration":100},"spend_cap":"0","tax_id_status":0,"timezone_id":1,"timezone_name":"America/Los_Angeles","timezone_offset_hours_utc":-7,"tos_accepted":{"web_custom_audience_tos":1},"user_tasks":["DRAFT","ANALYZE"]},"emitted_at":1692180820858} +{"stream":"ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666125630398"}, "id": "23853620198790398", "updated_time": "2023-03-21T22:33:56-0700", "created_time": "2023-03-17T08:04:29-0700", "name": "Don't Compromise Between Cost/Relaibility", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["660122622785523", "662226992575086"]}, {"action.type": ["link_click"], "post": ["660122622785523", "662226992575086"], "post.wall": ["112704783733939"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047377} +{"stream":"ad_sets","data":{"name":"Lookalike audience_Free Connector Program","promoted_object":{"pixel_id":"917042523049733","custom_event_type":"COMPLETE_REGISTRATION"},"id":"23853619670380398","account_id":"212551616838260","updated_time":"2023-03-21T14:20:51-0700","daily_budget":2000,"budget_remaining":2000,"effective_status":"ACTIVE","campaign_id":"23853619670350398","created_time":"2023-03-17T08:04:28-0700","start_time":"2023-03-17T08:04:28-0700","lifetime_budget":0,"targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"bid_strategy":"LOWEST_COST_WITHOUT_CAP"},"emitted_at":1692180821847} +{"stream":"campaigns", "data": {"account_id": "212551616838260", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2021-01-18T21:36:42-0800", "effective_status": "PAUSED", "id": "23846542053890398", "name": "Fake Campaign 0", "objective": "MESSAGES", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "1969-12-31T15:59:59-0800", "updated_time": "2021-02-18T01:00:02-0800"}, "emitted_at": 1682686106887} +{"stream":"custom_audiences","data":{"id":"23853683587660398","account_id":"212551616838260","approximate_count_lower_bound":4700,"approximate_count_upper_bound":5600,"customer_file_source":"PARTNER_PROVIDED_ONLY","data_source":{"type":"UNKNOWN","sub_type":"ANYTHING","creation_params":"[]"},"delivery_status":{"code":200,"description":"This audience is ready for use."},"description":"Custom Audience-Web Traffic [ALL] - _copy","is_value_based":false,"name":"Web Traffic [ALL] - _copy","operation_status":{"code":200,"description":"Normal"},"permission_for_actions":{"can_edit":false,"can_see_insight":"True","can_share":"False","subtype_supports_lookalike":"True","supports_recipient_lookalike":"False"},"retention_days":0,"subtype":"CUSTOM","time_content_updated":1679433484,"time_created":1679433479,"time_updated":1679433484},"emitted_at":1692028917200} +{"stream":"ad_creatives","data":{"id":"23844568440620398","account_id":"212551616838260","actor_id":"112704783733939","asset_feed_spec":{"images":[{"adlabels":[{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"191x100":[[0,411],[589,719]]}},{"adlabels":[{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"100x100":[[12,282],[574,844]]}},{"adlabels":[{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[14,72],[562,1046]]}},{"adlabels":[{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[0,0],[589,1047]]}}],"bodies":[{"adlabels":[{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"}],"text":""}],"call_to_action_types":["LEARN_MORE"],"descriptions":[{"text":"Unmatched attribution, ad performances, and lead conversion, by unlocking your ad-blocked traffic across all your tools."}],"link_urls":[{"adlabels":[{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"}],"website_url":"http://dataline.io/","display_url":""}],"titles":[{"adlabels":[{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"}],"text":"Unblock all your adblocked traffic"}],"ad_formats":["AUTOMATIC_FORMAT"],"asset_customization_rules":[{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["instagram","audience_network","messenger"],"instagram_positions":["story"],"messenger_positions":["story"],"audience_network_positions":["classic"]},"image_label":{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"},"body_label":{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},"link_url_label":{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},"title_label":{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},"priority":1},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["right_hand_column","instant_article","search"]},"image_label":{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"},"body_label":{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},"link_url_label":{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},"title_label":{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},"priority":2},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["story"]},"image_label":{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"},"body_label":{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"},"link_url_label":{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"},"title_label":{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"},"priority":3},{"customization_spec":{"age_max":65,"age_min":13},"image_label":{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"},"body_label":{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},"link_url_label":{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},"title_label":{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},"priority":4}],"optimization_type":"PLACEMENT","additional_data":{"multi_share_end_card":false,"is_click_to_message":false}},"effective_object_story_id":"112704783733939_117519556585795","name":"{{product.name}} 2020-04-21-49cbe5bd90ed9861ea68bb38f7d6fc7c","instagram_actor_id":"3437258706290825","object_story_spec":{"page_id":"112704783733939","instagram_actor_id":"3437258706290825"},"object_type":"SHARE","status":"ACTIVE","thumbnail_url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/93287504_23844521781140398_125048020067680256_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=a3999f&_nc_ohc=-TT4Z0FkPeYAX97qejq&_nc_ht=scontent-dus1-1.xx&edm=AAT1rw8EAAAA&stp=c0.5000x0.5000f_dst-emg0_p64x64_q75&ur=58080a&oh=00_AfBjMrayWFyOLmIgVt8Owtv2fBSJVyCmtNuPLpCQyggdpg&oe=64E18154"},"emitted_at":1692180825964} +{"stream":"activities","data":{"actor_id":"10167035656105444","actor_name":"Ilana Enoukov","application_id":"119211728144504","application_name":"Power Editor","date_time_in_timezone":"03/21/2023 at 2:20 PM","event_time":"2023-03-21T21:20:39+0000","event_type":"update_ad_set_target_spec","extra_data":"{\"old_value\":[{\"content\":\"Custom audience:\",\"children\":[\"Web Traffic [ALL] - _copy\"]},{\"content\":\"Location:\",\"children\":[\"United States\"]},{\"content\":\"Age:\",\"children\":[\"18 - 60\"]},{\"content\":\"Placements:\",\"children\":[\"on pages: Feed on desktop computers, Video feeds on desktop computers, Instagram feed, Instagram Stories, Instagram Profile Feed, Instagram Explore, Instagram Explore home, Instagram Reels, Instagram Shop, Third-party apps and websites on mobile devices, Feed on mobile devices, Video feeds on mobile devices, Right column on desktop computers, Instream video on mobile devices, Instream video on desktop computers, Instant Article, Marketplace on desktop computers, Marketplace on mobile devices, Facebook Stories on mobile devices, Messenger Stories, Marketplace search on desktop devices, Marketplace search on mobile devices, Search on mobile devices, Search on desktop devices, Facebook Reels overlay on mobile devices, Facebook Business Explore on mobile devices, Facebook Reels or Video search on mobile devices\"]},{\"content\":\"Advantage custom audience:\",\"children\":[\"Off\"]}],\"new_value\":[{\"content\":\"Custom audience:\",\"children\":[\"Lookalike (US, 10\\u0025) - Airbyte Cloud Users or Web Traffic [ALL] - _copy\"]},{\"content\":\"Location:\",\"children\":[\"United States\"]},{\"content\":\"Age:\",\"children\":[\"18 - 60\"]},{\"content\":\"Placements:\",\"children\":[\"on pages: Feed on desktop computers, Video feeds on desktop computers, Instagram feed, Instagram Stories, Instagram Profile Feed, Instagram Explore, Instagram Explore home, Instagram Reels, Instagram Shop, Third-party apps and websites on mobile devices, Feed on mobile devices, Video feeds on mobile devices, Right column on desktop computers, Instream video on mobile devices, Instream video on desktop computers, Instant Article, Marketplace on desktop computers, Marketplace on mobile devices, Facebook Stories on mobile devices, Messenger Stories, Marketplace search on desktop devices, Marketplace search on mobile devices, Search on mobile devices, Search on desktop devices, Facebook Reels overlay on mobile devices, Facebook Business Explore on mobile devices, Facebook Reels or Video search on mobile devices\"]},{\"content\":\"Advantage custom audience:\",\"children\":[\"On\"]}],\"type\":\"targets_spec\"}","object_id":"23853619670380398","object_name":"Lookalike audience_Free Connector Program","object_type":"CAMPAIGN","translated_event_type":"Ad set targeting updated"},"emitted_at":1692180829460} +{"stream":"custom_conversions","data":{"id":"694166388077667","account_id":"212551616838260","creation_time":"2020-04-22T01:36:00+0000","custom_event_type":"CONTACT","data_sources":[{"id":"2667253716886462","source_type":"PIXEL","name":"Dataline's Pixel"}],"default_conversion_value":0,"event_source_type":"pixel","is_archived":true,"is_unavailable":false,"name":"SubscribedButtonClick","retention_days":0,"rule":"{\"and\":[{\"event\":{\"eq\":\"PageView\"}},{\"or\":[{\"URL\":{\"i_contains\":\"SubscribedButtonClick\"}}]}]}"},"emitted_at":1692180839174} +{"stream":"images","data":{"id":"212551616838260:c1e94a8768a405f0f212d71fe8336647","account_id":"212551616838260","name":"Audience_1_Ad_3_1200x1200_blue_CTA_arrow.png_105","creatives":["23853630775340398","23853630871360398","23853666124200398"],"original_height":1200,"original_width":1200,"permalink_url":"https://www.facebook.com/ads/image/?d=AQIDNjjLb7VzVJ26jXb_HpudCEUJqbV_lLF2JVsdruDcBxnXQEKfzzd21VVJnkm0B-JLosUXNNg1BH78y7FxnK3AH-0D_lnk7kn39_bIcOMK7Z9HYyFInfsVY__adup3A5zGTIcHC9Y98Je5qK-yD8F6","status":"ACTIVE","url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfDCqQ6viqrgLcfbO3O5-n030Usq7Zyt2c1TmsatqnYf7Q&oe=64E2779A","created_time":"2023-03-16T13:13:17-0700","hash":"c1e94a8768a405f0f212d71fe8336647","url_128":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?stp=dst-png_s128x128&_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfAY50CMpox2s4w_f18IVx7sZuXlg4quF6YNIJJ8D4PZew&oe=64E2779A","is_associated_creatives_in_adgroups":true,"updated_time":"2023-03-17T08:09:56-0700","height":1200,"width":1200},"emitted_at":1692180839582} +{"stream":"ads_insights", "data": {"account_currency": "USD", "account_id": "212551616838260", "account_name": "Airbyte", "actions": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "link_click", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "page_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "post_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}], "ad_id": "23846765228310398", "ad_name": "Airbyte Ad", "adset_id": "23846765228280398", "adset_name": "Vanilla awareness ad set", "buying_type": "AUCTION", "campaign_id": "23846765228240398", "campaign_name": "Airbyte Awareness Campaign 1 (sherif)", "catalog_segment_value_mobile_purchase_roas": [{"value": 0.0}], "clicks": 3, "conversion_rate_ranking": "UNKNOWN", "cost_per_estimated_ad_recallers": 0.007, "cost_per_inline_link_click": 0.396667, "cost_per_inline_post_engagement": 0.396667, "cost_per_unique_click": 0.396667, "cost_per_unique_inline_link_click": 0.396667, "cpc": 0.396667, "cpm": 0.902199, "cpp": 0.948207, "created_time": "2021-02-09", "ctr": 0.227445, "date_start": "2021-02-15", "date_stop": "2021-02-15", "engagement_rate_ranking": "UNKNOWN", "estimated_ad_recall_rate": 13.545817, "estimated_ad_recallers": 170.0, "frequency": 1.050996, "impressions": 1319, "inline_link_click_ctr": 0.227445, "inline_link_clicks": 3, "inline_post_engagement": 3, "mobile_app_purchase_roas": [{"value": 0.0}], "objective": "BRAND_AWARENESS", "optimization_goal": "AD_RECALL_LIFT", "outbound_clicks": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "outbound_click", "value": 3.0}], "quality_ranking": "UNKNOWN", "reach": 1255, "social_spend": 0.0, "spend": 1.19, "unique_actions": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "link_click", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "page_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "post_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}], "unique_clicks": 3, "unique_ctr": 0.239044, "unique_inline_link_click_ctr": 0.239044, "unique_inline_link_clicks": 3, "unique_link_clicks_ctr": 0.239044, "unique_outbound_clicks": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "outbound_click", "value": 3.0}], "updated_time": "2021-08-27", "website_ctr": [{"action_type": "link_click", "value": 0.227445}], "website_purchase_roas": [{"value": 0.0}], "wish_bid": 0.0}, "emitted_at": 1682686057366} +{"stream":"ads_insights_action_carousel_card","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":3,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.007,"cost_per_inline_link_click":0.396667,"cost_per_inline_post_engagement":0.396667,"cost_per_unique_click":0.396667,"cost_per_unique_inline_link_click":0.396667,"cpc":0.396667,"cpm":0.902199,"cpp":0.948207,"created_time":"2021-02-09","ctr":0.227445,"date_start":"2021-02-15","date_stop":"2021-02-15","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":13.545817,"estimated_ad_recallers":170,"frequency":1.050996,"impressions":1319,"inline_link_click_ctr":0.227445,"inline_link_clicks":3,"inline_post_engagement":3,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":1255,"social_spend":0,"spend":1.19,"unique_actions":[{"value":3,"1d_click":3,"7d_click":3,"28d_click":3}],"unique_clicks":3,"unique_ctr":0.239044,"unique_inline_link_click_ctr":0.239044,"unique_inline_link_clicks":3,"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.227445}],"website_purchase_roas":[{"value":0}],"wish_bid":0},"emitted_at":1692180857757} +{"stream":"ads_insights_action_conversion_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":1.470588,"cpp":1.492537,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.925373,"estimated_ad_recallers":10,"frequency":1.014925,"impressions":68,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":67,"spend":0.1,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"device_platform":"desktop"},"emitted_at":1692180864186} +{"stream":"ads_insights_action_reaction","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"conversion_rate_ranking":"UNKNOWN","created_time":"2021-02-09","date_start":"2021-02-08","date_stop":"2021-02-08","engagement_rate_ranking":"UNKNOWN","frequency":0,"impressions":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":0,"social_spend":0,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","wish_bid":0},"emitted_at":1692180909192} +{"stream":"ads_insights_action_type","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":2,"conversion_rate_ranking":"UNKNOWN","cost_per_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_estimated_ad_recallers":0.006571,"cost_per_inline_link_click":0.23,"cost_per_inline_post_engagement":0.23,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cost_per_unique_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_unique_click":0.23,"cost_per_unique_inline_link_click":0.23,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cpc":0.23,"cpm":0.946502,"cpp":0.964361,"created_time":"2021-02-09","ctr":0.411523,"date_start":"2021-02-11","date_stop":"2021-02-11","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":14.675052,"estimated_ad_recallers":70,"frequency":1.018868,"impressions":486,"inline_link_click_ctr":0.411523,"inline_link_clicks":2,"inline_post_engagement":2,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":2}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.411523}],"quality_ranking":"UNKNOWN","reach":477,"social_spend":0,"spend":0.46,"unique_actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"unique_clicks":2,"unique_ctr":0.419287,"unique_inline_link_click_ctr":0.419287,"unique_inline_link_clicks":2,"unique_link_clicks_ctr":0.419287,"unique_outbound_clicks":[{"action_type":"outbound_click","value":2}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.419287}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.411523}],"website_purchase_roas":[{"value":0}],"wish_bid":0},"emitted_at":1692180948892} +{"stream":"ads_insights_action_video_sound","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"conversion_rate_ranking":"UNKNOWN","created_time":"2021-02-09","date_start":"2021-02-08","date_stop":"2021-02-08","engagement_rate_ranking":"UNKNOWN","frequency":0,"impressions":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":0,"social_spend":0,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","wish_bid":0},"emitted_at":1692180958086} +{"stream":"ads_insights_action_video_type","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"conversion_rate_ranking":"UNKNOWN","created_time":"2021-02-09","date_start":"2021-02-08","date_stop":"2021-02-08","engagement_rate_ranking":"UNKNOWN","frequency":0,"impressions":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":0,"social_spend":0,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","wish_bid":0},"emitted_at":1692180966403} +{"stream":"ads_insights_age_and_gender","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":0.714286,"cpp":0.769231,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":7.692308,"estimated_ad_recallers":1,"frequency":1.076923,"gender_targeting":"female","impressions":14,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":13,"spend":0.01,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"age":"18-24","gender":"female"},"emitted_at":1692180975689} +{"stream":"ads_insights_country","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.008667,"cpm":1.293532,"cpp":1.381142,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-12","date_stop":"2021-02-12","estimated_ad_recall_rate":15.936255,"estimated_ad_recallers":120,"frequency":1.067729,"impressions":804,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":753,"spend":1.04,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"country":"US"},"emitted_at":1692180985386} +{"stream":"ads_insights_delivery_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":1.470588,"cpp":1.492537,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.925373,"estimated_ad_recallers":10,"frequency":1.014925,"impressions":68,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":67,"spend":0.1,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"device_platform":"desktop"},"emitted_at":1692180993524} +{"stream":"ads_insights_delivery_platform","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":2,"cost_per_action_type":[{"action_type":"link_click","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"page_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"post_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225}],"cost_per_estimated_ad_recallers":0.0075,"cost_per_inline_link_click":0.225,"cost_per_inline_post_engagement":0.225,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.225}],"cost_per_unique_action_type":[{"action_type":"link_click","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"page_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"post_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225}],"cost_per_unique_click":0.225,"cost_per_unique_inline_link_click":0.225,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.225}],"cpc":0.225,"cpm":1.034483,"cpp":1.056338,"created_time":"2021-02-09","ctr":0.45977,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.084507,"estimated_ad_recallers":60,"frequency":1.021127,"impressions":435,"inline_link_click_ctr":0.45977,"inline_link_clicks":2,"inline_post_engagement":2,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":2}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.45977}],"reach":426,"spend":0.45,"unique_actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"unique_clicks":2,"unique_ctr":0.469484,"unique_inline_link_click_ctr":0.469484,"unique_inline_link_clicks":2,"unique_link_clicks_ctr":0.469484,"unique_outbound_clicks":[{"action_type":"outbound_click","value":2}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.469484}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.45977}],"website_purchase_roas":[{"value":0}],"publisher_platform":"facebook"},"emitted_at":1692181004988} +{"stream":"ads_insights_delivery_platform_and_device_platform","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":1.470588,"cpp":1.492537,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.925373,"estimated_ad_recallers":10,"frequency":1.014925,"impressions":68,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":67,"spend":0.1,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"publisher_platform":"facebook","device_platform":"desktop"},"emitted_at":1692181043593} +{"stream":"ads_insights_demographics_age","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.0045,"cpm":0.633803,"cpp":0.656934,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.59854,"estimated_ad_recallers":20,"frequency":1.036496,"impressions":142,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":137,"spend":0.09,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"age":"18-24"},"emitted_at":1692181051722} +{"stream":"ads_insights_demographics_country","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":2,"cost_per_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_estimated_ad_recallers":0.006571,"cost_per_inline_link_click":0.23,"cost_per_inline_post_engagement":0.23,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cost_per_unique_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_unique_click":0.23,"cost_per_unique_inline_link_click":0.23,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cpc":0.23,"cpm":0.946502,"cpp":0.964361,"created_time":"2021-02-09","ctr":0.411523,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.675052,"estimated_ad_recallers":70,"frequency":1.018868,"impressions":486,"inline_link_click_ctr":0.411523,"inline_link_clicks":2,"inline_post_engagement":2,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":2}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.411523}],"reach":477,"spend":0.46,"unique_actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"unique_clicks":2,"unique_ctr":0.419287,"unique_inline_link_click_ctr":0.419287,"unique_inline_link_clicks":2,"unique_link_clicks_ctr":0.419287,"unique_outbound_clicks":[{"action_type":"outbound_click","value":2}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.419287}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.411523}],"website_purchase_roas":[{"value":0}],"country":"US"},"emitted_at":1692181061009} +{"stream":"ads_insights_demographics_dma_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0,"cpm":0,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-12","date_stop":"2021-02-12","estimated_ad_recallers":1,"frequency":1.333333,"impressions":4,"inline_link_clicks":0,"inline_post_engagement":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks_ctr":[{"value":0}],"reach":3,"spend":0,"unique_clicks":0,"unique_outbound_clicks_ctr":[{"value":0}],"updated_time":"2021-08-27","dma":"Abilene-Sweetwater"},"emitted_at":1692181100592} +{"stream":"ads_insights_demographics_gender","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.008,"cpm":1.333333,"cpp":1.37931,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":17.241379,"estimated_ad_recallers":10,"frequency":1.034483,"gender_targeting":"female","impressions":60,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":58,"spend":0.08,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"gender":"female"},"emitted_at":1692181125390} +{"stream":"ads_insights_dma","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0,"cpm":0,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-12","date_stop":"2021-02-12","estimated_ad_recallers":1,"frequency":1.333333,"impressions":4,"inline_link_clicks":0,"inline_post_engagement":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":3,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","dma":"Abilene-Sweetwater"},"emitted_at":1692181165207} +{"stream":"ads_insights_platform_and_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.007,"cpm":1.627907,"cpp":1.627907,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":23.255814,"estimated_ad_recallers":10,"frequency":1,"impressions":43,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":43,"spend":0.07,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"publisher_platform":"facebook","platform_position":"feed","impression_device":"android_smartphone"},"emitted_at":1692181188895} +{"stream":"ads_insights_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0,"cpm":0,"cpp":0,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":8.333333,"estimated_ad_recallers":1,"frequency":1,"impressions":12,"inline_link_clicks":0,"inline_post_engagement":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":12,"spend":0,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","region":"Alabama"},"emitted_at":1692181230021} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json index cb9b927b0a63..35ac2f8aa38b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json @@ -336,4 +336,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 3b263d3180c9..cea425df779e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -34,7 +34,7 @@ }, "access_token": { "title": "Access Token", - "description": "The value of the generated access token. From your App’s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", "order": 3, "airbyte_secret": true, "type": "string" @@ -82,19 +82,17 @@ "type": "array", "items": { "title": "ValidEnums", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", + "description": "An enumeration.", "enum": [ "account_currency", "account_id", "account_name", "action_values", "actions", - "ad_bid_value", "ad_click_actions", "ad_id", "ad_impression_actions", "ad_name", - "adset_bid_value", "adset_end", "adset_id", "adset_name", @@ -141,6 +139,7 @@ "cpm", "cpp", "created_time", + "creative_media_type", "ctr", "date_start", "date_stop", @@ -161,6 +160,7 @@ "inline_link_click_ctr", "inline_link_clicks", "inline_post_engagement", + "instagram_upcoming_event_reminders_set", "instant_experience_clicks_to_open", "instant_experience_clicks_to_start", "instant_experience_outbound_clicks", @@ -184,6 +184,7 @@ "spend", "total_postbacks", "total_postbacks_detailed", + "total_postbacks_detailed_v4", "unique_actions", "unique_clicks", "unique_conversions", @@ -225,21 +226,24 @@ "type": "array", "items": { "title": "ValidBreakdowns", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", + "description": "An enumeration.", "enum": [ "ad_format_asset", "age", "app_id", "body_asset", "call_to_action_asset", + "coarse_conversion_value", "country", "description_asset", "device_platform", "dma", + "fidelity_type", "frequency_value", "gender", "hourly_stats_aggregated_by_advertiser_time_zone", "hourly_stats_aggregated_by_audience_time_zone", + "hsid", "image_asset", "impression_device", "is_conversion_id_modeled", @@ -247,8 +251,10 @@ "mmm", "place_page_id", "platform_position", + "postback_sequence_index", "product_id", "publisher_platform", + "redownload", "region", "skan_campaign_id", "skan_conversion_id", @@ -264,7 +270,7 @@ "type": "array", "items": { "title": "ValidActionBreakdowns", - "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", + "description": "An enumeration.", "enum": [ "action_canvas_component_name", "action_carousel_card_id", @@ -279,6 +285,13 @@ ] } }, + "action_report_time": { + "title": "Action Report Time", + "description": "Determines the report time of action stats. For example, if a person saw the ad on Jan 1st but converted on Jan 2nd, when you query the API with action_report_time=impression, you see a conversion on Jan 1st. When you query the API with action_report_time=conversion, you see a conversion on Jan 2nd.", + "default": "mixed", + "enum": ["conversion", "impression", "mixed"], + "type": "string" + }, "time_increment": { "title": "Time Increment", "description": "Time window in days by which to aggregate statistics. The sync will be chunked into N day intervals, where N is the number of days you specified. For example, if you set this value to 7, then all statistics will be reported as 7-day aggregates by starting from the start_date. If the start and end dates are October 1st and October 30th, then the connector will output 5 records: 01 - 06, 07 - 13, 14 - 20, 21 - 27, and 28 - 30 (3 days only).", @@ -340,6 +353,7 @@ "default": 50, "order": 9, "exclusiveMinimum": 0, + "maximum": 50, "type": "integer" }, "action_breakdowns_allow_empty": { @@ -348,18 +362,63 @@ "default": true, "airbyte_hidden": true, "type": "boolean" + }, + "client_id": { + "title": "Client Id", + "description": "The Client Id for your OAuth app", + "airbyte_secret": true, + "airbyte_hidden": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The Client Secret for your OAuth app", + "airbyte_secret": true, + "airbyte_hidden": true, + "type": "string" } }, "required": ["account_id", "start_date", "access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [], - "oauthFlowOutputParameters": [["access_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["access_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": true, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["client_secret"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index 60274aba2e43..57c1f108f263 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -5,11 +5,11 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 0.4.0 + dockerImageTag: 1.1.7 dockerRepository: airbyte/source-facebook-marketing githubIssueLabel: source-facebook-marketing icon: facebook.svg - license: MIT + license: ELv2 name: Facebook Marketing registries: cloud: @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt +++ b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index c342bddb8ad6..144e8b73abc7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -8,11 +8,11 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.36", "cached_property==1.5.2", - "facebook_business==16.0.0", + "facebook_business==17.0.0", "pendulum>=2,<3", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8", "freezegun"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8", "freezegun"] setup( name="source_facebook_marketing", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py index bc49ad1a94fc..478ad43960d2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from facebook_business import api diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py index 8897ace2b60e..92a85fb7dd8b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py @@ -34,6 +34,12 @@ class MyFacebookAdsApi(FacebookAdsApi): MAX_RATE, MAX_PAUSE_INTERVAL = (95, pendulum.duration(minutes=10)) MIN_RATE, MIN_PAUSE_INTERVAL = (85, pendulum.duration(minutes=2)) + # see `_should_restore_page_size` method docstring for more info. + # attribute to handle the reduced request limit + request_record_limit_is_reduced: bool = False + # attribute to save the status of last successfull call + last_api_call_is_successful: bool = False + @dataclass class Throttle: """Utilization of call rate in %, from 0 to 100""" @@ -139,6 +145,14 @@ def _update_insights_throttle_limit(self, response: FacebookResponse): per_account=ads_insights_throttle.get("acc_id_util_pct", 0), ) + def _should_restore_default_page_size(self, params): + """ + Track the state of the `request_record_limit_is_reduced` and `last_api_call_is_successfull`, + based on the logic from `@backoff_policy` (common.py > `reduce_request_record_limit` and `revert_request_record_limit`) + """ + params = True if params else False + return params and not self.request_record_limit_is_reduced and self.last_api_call_is_successful + @backoff_policy def call( self, @@ -151,6 +165,8 @@ def call( api_version=None, ): """Makes an API call, delegate actual work to parent class and handles call rates""" + if self._should_restore_default_page_size(params): + params.update(**{"limit": self.default_page_size}) response = super().call(method, path, params, headers, files, url_override, api_version) self._update_insights_throttle_limit(response) self._handle_call_rate_limit(response, params) @@ -160,10 +176,14 @@ def call( class API: """Simple wrapper around Facebook API""" - def __init__(self, account_id: str, access_token: str): + def __init__(self, account_id: str, access_token: str, page_size: int = 100): self._account_id = account_id # design flaw in MyFacebookAdsApi requires such strange set of new default api instance self.api = MyFacebookAdsApi.init(access_token=access_token, crash_log=False) + # adding the default page size from config to the api base class + # reference issue: https://github.com/airbytehq/airbyte/issues/25383 + setattr(self.api, "default_page_size", page_size) + # set the default API client to Facebook lib. FacebookAdsApi.set_default_api(self.api) @cached_property diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json index 220c68f30413..d36fbabbe87c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json @@ -137,9 +137,6 @@ } } }, - "has_advertiser_opted_in_odax": { - "type": ["null", "boolean"] - }, "has_migrated_permissions": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_audiences.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_audiences.json new file mode 100644 index 000000000000..78e9c3c1b000 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_audiences.json @@ -0,0 +1,163 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "approximate_count_lower_bound": { + "type": ["null", "integer"] + }, + "approximate_count_upper_bound": { + "type": ["null", "integer"] + }, + "customer_file_source": { + "type": ["null", "string"] + }, + "data_source": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "sub_type": { + "type": ["null", "string"] + }, + "creation_params": { + "type": ["null", "string"] + } + } + }, + "delivery_status": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "is_value_based": { + "type": ["null", "boolean"] + }, + "lookalike_audience_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "lookalike_spec": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "ratio": { + "type": ["null", "number"] + }, + "type": { + "type": ["null", "string"] + }, + "origin": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + } + } + }, + "name": { + "type": ["null", "string"] + }, + "operation_status": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + } + } + }, + "opt_out_link": { + "type": ["null", "string"] + }, + "page_deletion_marked_delete_time": { + "type": ["null", "integer"] + }, + "permission_for_actions": { + "type": ["null", "object"], + "properties": { + "can_edit": { + "type": ["null", "boolean"] + }, + "can_see_insight": { + "type": ["null", "string"] + }, + "can_share": { + "type": ["null", "string"] + }, + "subtype_supports_lookalike": { + "type": ["null", "string"] + }, + "supports_recipient_lookalike": { + "type": ["null", "string"] + } + } + }, + "pixel_id": { + "type": ["null", "string"] + }, + "retention_days": { + "type": ["null", "integer"] + }, + "rule": { + "type": ["null", "string"] + }, + "rule_aggregation": { + "type": ["null", "string"] + }, + "sharing_status": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + } + } + }, + "subtype": { + "type": ["null", "string"] + }, + "time_content_updated": { + "type": ["null", "integer"] + }, + "time_created": { + "type": ["null", "integer"] + }, + "time_updated": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 2641544131d9..16db2e4f6245 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -8,7 +8,14 @@ import facebook_business import pendulum import requests -from airbyte_cdk.models import AuthSpecification, ConnectorSpecification, DestinationSyncMode, FailureType, OAuth2Specification +from airbyte_cdk.models import ( + AdvancedAuth, + AuthFlowType, + ConnectorSpecification, + DestinationSyncMode, + FailureType, + OAuthConfigSpecification, +) from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException @@ -42,10 +49,12 @@ AdsInsightsPlatformAndDevice, AdsInsightsRegion, Campaigns, + CustomAudiences, CustomConversions, Images, Videos, ) +from source_facebook_marketing.streams.common import AccountTypeException from .utils import validate_end_date, validate_start_date @@ -78,9 +87,19 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.end_date < config.start_date: return False, "end_date must be equal or after start_date." - api = API(account_id=config.account_id, access_token=config.access_token) + api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) logger.info(f"Select account {api.account}") - except (requests.exceptions.RequestException, ValidationError, FacebookAPIException) as e: + + account_info = api.account.api_get(fields=["is_personal"]) + + if account_info.get("is_personal"): + message = ( + "The personal ad account you're currently using is not eligible " + "for this operation. Please switch to a business ad account." + ) + raise AccountTypeException(message) + + except (requests.exceptions.RequestException, ValidationError, FacebookAPIException, AccountTypeException) as e: return False, e # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters @@ -101,7 +120,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) - api = API(account_id=config.account_id, access_token=config.access_token) + api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) insights_args = dict( api=api, start_date=config.start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window @@ -164,6 +183,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: page_size=config.page_size, max_batch_size=config.max_batch_size, ), + CustomAudiences( + api=api, + include_deleted=config.include_deleted, + page_size=config.page_size, + max_batch_size=config.max_batch_size, + ), Images( api=api, start_date=config.start_date, @@ -203,10 +228,30 @@ def spec(self, *args, **kwargs) -> ConnectorSpecification: supportsIncremental=True, supported_destination_sync_modes=[DestinationSyncMode.append], connectionSpecification=ConnectorConfig.schema(), - authSpecification=AuthSpecification( - auth_type="oauth2.0", - oauth2Specification=OAuth2Specification( - rootObject=[], oauthFlowInitParameters=[], oauthFlowOutputParameters=[["access_token"]] + advanced_auth=AdvancedAuth( + auth_flow_type=AuthFlowType.oauth2_0, + oauth_config_specification=OAuthConfigSpecification( + complete_oauth_output_specification={ + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["access_token"], + } + }, + }, + complete_oauth_server_input_specification={ + "type": "object", + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + complete_oauth_server_output_specification={ + "type": "object", + "additionalProperties": True, + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["client_secret"]}, + }, + }, ), ), ) @@ -233,6 +278,7 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List breakdowns=list(set(insight.breakdowns)), action_breakdowns=list(set(insight.action_breakdowns)), action_breakdowns_allow_empty=config.action_breakdowns_allow_empty, + action_report_time=insight.action_report_time, time_increment=insight.time_increment, start_date=insight.start_date or config.start_date, end_date=insight.end_date or config.end_date, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index b9c0f9a51fc6..99340caf954d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -52,6 +52,17 @@ class Config: default=[], ) + action_report_time: str = Field( + title="Action Report Time", + description=( + "Determines the report time of action stats. For example, if a person saw the ad on Jan 1st " + "but converted on Jan 2nd, when you query the API with action_report_time=impression, you see a conversion on Jan 1st. " + "When you query the API with action_report_time=conversion, you see a conversion on Jan 2nd." + ), + default="mixed", + enum=["conversion", "impression", "mixed"], + ) + time_increment: Optional[PositiveInt] = Field( title="Time Increment", description=( @@ -190,7 +201,7 @@ class Config: default=28, ) - max_batch_size: Optional[PositiveInt] = Field( + max_batch_size: Optional[int] = Field( title="Maximum size of Batched Requests", order=9, description=( @@ -198,6 +209,8 @@ class Config: "Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases." ), default=50, + gt=0, + le=50, ) action_breakdowns_allow_empty: bool = Field( @@ -205,3 +218,15 @@ class Config: default=True, airbyte_hidden=True, ) + + client_id: Optional[str] = Field( + description="The Client Id for your OAuth app", + airbyte_secret=True, + airbyte_hidden=True, + ) + + client_secret: Optional[str] = Field( + description="The Client Secret for your OAuth app", + airbyte_secret=True, + airbyte_hidden=True, + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py index 47fa87fb5159..f330e583026c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py @@ -29,6 +29,7 @@ AdsInsightsPlatformAndDevice, AdsInsightsRegion, Campaigns, + CustomAudiences, CustomConversions, Images, Videos, @@ -61,6 +62,7 @@ "AdsInsightsDemographicsGender", "Campaigns", "CustomConversions", + "CustomAudiences", "Images", "Videos", "Activities", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py index 39b509946d92..fe44c18223e8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py @@ -19,6 +19,8 @@ from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse from source_facebook_marketing.streams.common import retry_pattern +from ..utils import validate_start_date + logger = logging.getLogger("airbyte") @@ -189,7 +191,6 @@ class InsightAsyncJob(AsyncJob): job_timeout = pendulum.duration(hours=1) page_size = 100 - INSIGHTS_RETENTION_PERIOD = pendulum.duration(months=37) def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: Mapping[str, Any], **kwargs): """Initialize @@ -242,10 +243,9 @@ def _split_by_edge_class(self, edge_class: Union[Type[Campaign], Type[AdSet], Ty params = dict(copy.deepcopy(self._params)) # get objects from attribution window as well (28 day + 1 current day) new_start = self._interval.start - pendulum.duration(days=28 + 1) - oldest_date = pendulum.today().date() - self.INSIGHTS_RETENTION_PERIOD - new_start = max(new_start, oldest_date) - params.update(fields=[pk_name], level=level) + new_start = validate_start_date(new_start) params["time_range"].update(since=new_start.to_date_string()) + params.update(fields=[pk_name], level=level) params.pop("time_increment") # query all days result = self._edge_object.get_insights(params=params) ids = set(row[pk_name] for row in result) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 8e93929c9c33..5913b8a78eb4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -60,6 +60,7 @@ def __init__( breakdowns: List[str] = None, action_breakdowns: List[str] = None, action_breakdowns_allow_empty: bool = False, + action_report_time: str = "mixed", time_increment: Optional[int] = None, insights_lookback_window: int = None, level: str = "ad", @@ -78,6 +79,7 @@ def __init__( if breakdowns is not None: self.breakdowns = breakdowns self.time_increment = time_increment or self.time_increment + self.action_report_time = action_report_time self._new_class_name = name self._insights_lookback_window = insights_lookback_window self.level = level @@ -276,6 +278,7 @@ def request_params(self, **kwargs) -> MutableMapping[str, Any]: return { "level": self.level, "action_breakdowns": self.action_breakdowns, + "action_report_time": self.action_report_time, "breakdowns": self.breakdowns, "fields": self.fields, "time_increment": self.time_increment, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index c86c4f2a93a3..27011e7e1640 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod from datetime import datetime from functools import partial +from math import ceil from queue import Queue from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional @@ -40,6 +41,8 @@ class FBMarketingStream(Stream, ABC): enable_deleted = True # entity prefix for `include_deleted` filter, it usually matches singular version of stream name entity_prefix = None + # In case of Error 'Too much data was requested in batch' some fields should be removed from request + fields_exceptions = [] @property def availability_strategy(self) -> Optional["AvailabilityStrategy"]: @@ -50,7 +53,7 @@ def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 1 self._api = api self.page_size = page_size if page_size is not None else 100 self._include_deleted = include_deleted if self.enable_deleted else False - self.max_batch_size = max_batch_size if max_batch_size is not None else 50 + self.max_batch_size = self._initial_max_batch_size = max_batch_size if max_batch_size is not None else 50 @cached_property def fields(self) -> List[str]: @@ -72,15 +75,30 @@ def execute_in_batch(self, pending_requests: Iterable[FacebookRequest]) -> Itera requests_q.put(r) def success(response: FacebookResponse): + self.max_batch_size = self._initial_max_batch_size records.append(response.json()) + def reduce_batch_size(request: FacebookRequest): + if self.max_batch_size == 1 and set(self.fields_exceptions) & set(request._fields): + logger.warning( + f"Removing fields from object {self.name} with id={request._node_id} : {set(self.fields_exceptions) & set(request._fields)}" + ) + request._fields = [x for x in request._fields if x not in self.fields_exceptions] + elif self.max_batch_size == 1: + raise RuntimeError("Batch request failed with only 1 request in it") + self.max_batch_size = ceil(self.max_batch_size / 2) + logger.warning(f"Caught retryable error: Too much data was requested in batch. Reducing batch size to {self.max_batch_size}") + def failure(response: FacebookResponse, request: Optional[FacebookRequest] = None): # although it is Optional in the signature for compatibility, we need it always assert request, "Missing a request object" resp_body = response.json() - if not isinstance(resp_body, dict) or resp_body.get("error", {}).get("code") != FACEBOOK_BATCH_ERROR_CODE: - # response body is not a json object or the error code is different + if not isinstance(resp_body, dict): raise RuntimeError(f"Batch request failed with response: {resp_body}") + elif resp_body.get("error", {}).get("message") == "Please reduce the amount of data you're asking for, then retry your request": + reduce_batch_size(request) + elif resp_body.get("error", {}).get("code") != FACEBOOK_BATCH_ERROR_CODE: + raise RuntimeError(f"Batch request failed with response: {resp_body}; unknown error code") requests_q.put(request) api_batch: FacebookAdsApiBatch = self._api.api.new_batch() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py index abe79bbf11df..db1913ffc90e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py @@ -27,6 +27,10 @@ class JobException(Exception): """Scheduled job failed""" +class AccountTypeException(Exception): + """Wrong account type""" + + def retry_pattern(backoff_type, exception, **wait_gen_kwargs): def log_retry_attempt(details): _, exc, _ = sys.exc_info() @@ -35,12 +39,35 @@ def log_retry_attempt(details): def reduce_request_record_limit(details): _, exc, _ = sys.exc_info() + # the list of error patterns to track, + # in order to reduce the requestt page size and retry + error_patterns = [ + "Please reduce the amount of data you're asking for, then retry your request", + "An unknown error occurred", + ] if ( details.get("kwargs", {}).get("params", {}).get("limit") and exc.http_status() == http.client.INTERNAL_SERVER_ERROR - and exc.api_error_message() == "Please reduce the amount of data you're asking for, then retry your request" + and exc.api_error_message() in error_patterns ): + # reduce the existing request `limit` param by a half and retry details["kwargs"]["params"]["limit"] = int(int(details["kwargs"]["params"]["limit"]) / 2) + # set the flag to the api class that the last api call failed + details.get("args")[0].last_api_call_is_successfull = False + # set the flag to the api class that the `limit` param was reduced + details.get("args")[0].request_record_limit_is_reduced = True + + def revert_request_record_limit(details): + """ + This method is triggered `on_success` after successfull retry, + sets the internal class flags to provide the logic to restore the previously reduced + `limit` param. + """ + # reference issue: https://github.com/airbytehq/airbyte/issues/25383 + # set the flag to the api class that the last api call was ssuccessfull + details.get("args")[0].last_api_call_is_successfull = True + # set the flag to the api class that the `limit` param is restored + details.get("args")[0].request_record_limit_is_reduced = False def should_retry_api_error(exc): if isinstance(exc, FacebookRequestError): @@ -68,6 +95,7 @@ def should_retry_api_error(exc): exception, jitter=None, on_backoff=[log_retry_attempt, reduce_request_record_limit], + on_success=[revert_request_record_limit], giveup=lambda exc: not should_retry_api_error(exc), **wait_gen_kwargs, ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 8a35199f8387..2f447af8630e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -82,6 +82,19 @@ def list_objects(self, params: Mapping[str, Any]) -> Iterable: return self._api.account.get_custom_conversions(params=params) +class CustomAudiences(FBMarketingStream): + """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-conversion""" + + entity_prefix = "customaudience" + enable_deleted = False + # The `rule` field is excluded from the list because it caused the error message "Please reduce the amount of data" for certain connections. + # https://github.com/airbytehq/oncall/issues/2765 + fields_exceptions = ["rule"] + + def list_objects(self, params: Mapping[str, Any]) -> Iterable: + return self._api.account.get_custom_audiences(params=params) + + class Ads(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/adgroup""" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py index b0a52abf4a77..d3550ae6d1c8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py @@ -3,9 +3,10 @@ # import logging +from typing import Union import pendulum -from pendulum import DateTime +from pendulum import Date, DateTime logger = logging.getLogger("airbyte") @@ -14,11 +15,20 @@ # HTTP response. # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview DATA_RETENTION_PERIOD = 37 +DateOrDateTime = Union[Date, DateTime] -def validate_start_date(start_date: DateTime) -> DateTime: - now = pendulum.now(tz=start_date.tzinfo) - today = now.replace(microsecond=0, second=0, minute=0, hour=0) +def cast_to_type(input_date: DateOrDateTime, target_date: DateOrDateTime) -> DateOrDateTime: + # casts `target_date` to the type of `input_date` + if type(target_date) == type(input_date): + return target_date + if isinstance(target_date, DateTime): + return target_date.date() + return pendulum.datetime(target_date.year, target_date.month, target_date.day) + + +def validate_start_date(start_date: DateOrDateTime) -> DateOrDateTime: + today = cast_to_type(start_date, pendulum.today()) retention_date = today.subtract(months=DATA_RETENTION_PERIOD) if retention_date.day != today.day: # `.subtract(months=37)` can be erroneous, for instance: @@ -26,20 +36,24 @@ def validate_start_date(start_date: DateTime) -> DateTime: # that's why we're adjusting the date to the 1st day of the next month retention_date = retention_date.replace(month=retention_date.month + 1, day=1) - if start_date > now: + # FB does not provide precise description of how we should calculate the 37 months datetime difference. + # As timezone difference can result in an additional time delta, this is a reassurance we stay inside the 37 months limit. + retention_date = retention_date.add(days=1) + + if start_date > today: message = f"The start date cannot be in the future. Set start date to today's date - {today}." logger.warning(message) - return today + return cast_to_type(start_date, today) elif start_date < retention_date: message = ( f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. Set start date to {retention_date}." ) logger.warning(message) - return retention_date + return cast_to_type(start_date, retention_date) return start_date -def validate_end_date(start_date: DateTime, end_date: DateTime) -> DateTime: +def validate_end_date(start_date: DateOrDateTime, end_date: DateOrDateTime) -> DateOrDateTime: if start_date > end_date: message = f"The end date must be after start date. Set end date to {start_date}." logger.warning(message) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py index 01a4b402103a..ad2454b02ea4 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py @@ -49,7 +49,7 @@ def fb_account_response_fixture(account_id): @fixture(name="api") def api_fixture(some_config, requests_mock, fb_account_response): - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"]) + api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/adaccounts", [fb_account_response]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_id']}/", [fb_account_response]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py index 90544c1d9a15..69941a42501d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py @@ -40,6 +40,7 @@ def job_fixture(api, account): params = { "level": "ad", "action_breakdowns": [], + "action_report_time": "mixed", "breakdowns": [], "fields": ["field1", "field2"], "time_increment": 1, @@ -342,7 +343,10 @@ def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field): "breakdowns": [], "fields": [id_field], "level": next_edge_class.__name__.lower(), - "time_range": {"since": (today - pendulum.duration(months=37)).to_date_string(), "until": end.to_date_string()}, + "time_range": { + "since": (today - pendulum.duration(months=37) + pendulum.duration(days=1)).to_date_string(), + "until": end.to_date_string() + }, } ) assert len(small_jobs) == 3 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 59952d4e915e..c773ad505dec 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -55,6 +55,7 @@ def test_init(self, api): assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] assert stream.name == "ads_insights" assert stream.primary_key == ["date_start", "account_id", "ad_id"] + assert stream.action_report_time == "mixed" def test_init_override(self, api): stream = AdsInsights( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py index 20b520194411..c64f1c778ba8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py @@ -128,6 +128,31 @@ def test_execute_in_batch_with_fails(self, api, batch, mock_batch_responses): assert batch.add_request.call_count == len(requests) assert batch.execute.call_count == 1 + def test_batch_reduce_amount(self, api, batch, mock_batch_responses, caplog): + """Reduce batch size to 1 and finally fail with message""" + + retryable_message = "Please reduce the amount of data you're asking for, then retry your request" + mock_batch_responses( + [ + { + "json": [ + {"body": {"error": {"message": retryable_message}}, "code": 500, "headers": {}}, + ], + } + ] + ) + + stream = SomeTestStream(api=api) + requests = [FacebookRequest("node", "GET", "endpoint")] + with pytest.raises(RuntimeError, match="Batch request failed with only 1 request in..."): + list(stream.execute_in_batch(requests)) + + assert batch.add_request.call_count == 7 + assert batch.execute.call_count == 7 + assert stream.max_batch_size == 1 + for index, expected_batch_size in enumerate(["25", "13", "7", "4", "2", "1"]): + assert expected_batch_size in caplog.messages[index] + def test_execute_in_batch_retry_batch_error(self, api, batch, mock_batch_responses): """Should retry without exception when any request returns 960 error code""" mock_batch_responses( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 74bcec4f315c..310546d5ec19 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -9,7 +9,7 @@ from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError -from source_facebook_marketing.streams import AdAccount, AdCreatives, Campaigns, Videos +from source_facebook_marketing.streams import Activities, AdAccount, AdCreatives, Campaigns, Videos FB_API_VERSION = FacebookAdsApi.API_VERSION @@ -158,6 +158,43 @@ def test_limit_error_retry(self, fb_call_amount_data_response, requests_mock, ap except FacebookRequestError: assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "25", "12", "6"] + def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id): + """Error every time, check limit parameter decreases by 2 times every new call""" + + error = { + "json": { + "error": { + "message": "An unknown error occurred", + "code": 1, + } + }, + "status_code": 500, + } + success = { + "json": { + 'data': [], + "paging": { + "cursors": { + "after": "test", + }, + "next": f"https://graph.facebook.com/{FB_API_VERSION}/act_{account_id}/activities?limit=31&after=test" + } + }, + "status_code": 200, + } + + res = requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/activities", + [error, success, error, success], + ) + + stream = Activities(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + try: + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + except FacebookRequestError: + assert [x.qs.get("limit")[0] for x in res.request_history] == ['100', '50', '100', '50'] + def test_limit_error_retry_next_page(self, fb_call_amount_data_response, requests_mock, api, account_id): """Unlike the previous test, this one tests the API call fail on the second or more page of a request.""" base_url = FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/advideos" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 1b7e472f4f03..0f614bb6ef02 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -10,19 +10,20 @@ from facebook_business import FacebookAdsApi, FacebookSession from source_facebook_marketing import SourceFacebookMarketing from source_facebook_marketing.spec import ConnectorConfig +from source_facebook_marketing.streams.common import AccountTypeException from .utils import command_check @pytest.fixture(name="config") -def config_fixture(): +def config_fixture(requests_mock): config = { "account_id": "123", "access_token": "TOKEN", "start_date": "2019-10-10T00:00:00Z", "end_date": "2020-10-10T00:00:00Z", } - + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {}) return config @@ -49,100 +50,113 @@ def logger_mock_fixture(mocker): return mocker.patch("source_facebook_marketing.source.logger") +@pytest.fixture +def fb_marketing(): + return SourceFacebookMarketing() + + class TestSourceFacebookMarketing: - def test_check_connection_ok(self, api, config, logger_mock): - ok, error_msg = SourceFacebookMarketing().check_connection(logger_mock, config=config) + def test_check_connection_ok(self, config, logger_mock, fb_marketing): + ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) assert ok assert not error_msg - api.assert_called_once_with(account_id="123", access_token="TOKEN") - logger_mock.info.assert_called_once_with(f"Select account {api.return_value.account}") - def test_check_connection_future_date_range(self, api, config, logger_mock): + def test_check_connection_future_date_range(self, api, config, logger_mock, fb_marketing): config["start_date"] = "2219-10-10T00:00:00" config["end_date"] = "2219-10-11T00:00:00" - assert SourceFacebookMarketing().check_connection(logger_mock, config=config) == ( + assert fb_marketing.check_connection(logger_mock, config=config) == ( False, "Date range can not be in the future.", ) - def test_check_connection_end_date_before_start_date(self, api, config, logger_mock): + def test_check_connection_end_date_before_start_date(self, api, config, logger_mock, fb_marketing): config["start_date"] = "2019-10-10T00:00:00" config["end_date"] = "2019-10-09T00:00:00" - assert SourceFacebookMarketing().check_connection(logger_mock, config=config) == ( + assert fb_marketing.check_connection(logger_mock, config=config) == ( False, "end_date must be equal or after start_date.", ) - def test_check_connection_empty_config(self, api, logger_mock): + def test_check_connection_empty_config(self, api, logger_mock, fb_marketing): config = {} - ok, error_msg = SourceFacebookMarketing().check_connection(logger_mock, config=config) + ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) assert not ok assert error_msg - def test_check_connection_invalid_config(self, api, config, logger_mock): + def test_check_connection_invalid_config(self, api, config, logger_mock, fb_marketing): config.pop("start_date") - ok, error_msg = SourceFacebookMarketing().check_connection(logger_mock, config=config) + ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) assert not ok assert error_msg - def test_check_connection_exception(self, api, config, logger_mock): + def test_check_connection_exception(self, api, config, logger_mock, fb_marketing): api.side_effect = RuntimeError("Something went wrong!") with pytest.raises(RuntimeError, match="Something went wrong!"): - SourceFacebookMarketing().check_connection(logger_mock, config=config) + fb_marketing.check_connection(logger_mock, config=config) - def test_streams(self, config, api): - streams = SourceFacebookMarketing().streams(config) + def test_streams(self, config, api, fb_marketing): + streams = fb_marketing.streams(config) - assert len(streams) == 29 + assert len(streams) == 30 - def test_spec(self): - spec = SourceFacebookMarketing().spec() + def test_spec(self, fb_marketing): + spec = fb_marketing.spec() assert isinstance(spec, ConnectorSpecification) - def test_get_custom_insights_streams(self, api, config): + def test_get_custom_insights_streams(self, api, config, fb_marketing): config["custom_insights"] = [ {"name": "test", "fields": ["account_id"], "breakdowns": ["ad_format_asset"], "action_breakdowns": ["action_device"]}, ] config = ConnectorConfig.parse_obj(config) - assert SourceFacebookMarketing().get_custom_insights_streams(api, config) + assert fb_marketing.get_custom_insights_streams(api, config) - def test_get_custom_insights_action_breakdowns_allow_empty(self, api, config): + def test_get_custom_insights_action_breakdowns_allow_empty(self, api, config, fb_marketing): config["custom_insights"] = [ {"name": "test", "fields": ["account_id"], "breakdowns": ["ad_format_asset"], "action_breakdowns": []}, ] config["action_breakdowns_allow_empty"] = False - streams = SourceFacebookMarketing().get_custom_insights_streams(api, ConnectorConfig.parse_obj(config)) + streams = fb_marketing.get_custom_insights_streams(api, ConnectorConfig.parse_obj(config)) assert len(streams) == 1 assert streams[0].breakdowns == ["ad_format_asset"] assert streams[0].action_breakdowns == ["action_type", "action_target_id", "action_destination"] config["action_breakdowns_allow_empty"] = True - streams = SourceFacebookMarketing().get_custom_insights_streams(api, ConnectorConfig.parse_obj(config)) + streams = fb_marketing.get_custom_insights_streams(api, ConnectorConfig.parse_obj(config)) assert len(streams) == 1 assert streams[0].breakdowns == ["ad_format_asset"] assert streams[0].action_breakdowns == [] -def test_check_config(config_gen, requests_mock): +def test_check_config(config_gen, requests_mock, fb_marketing): requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {}) - source = SourceFacebookMarketing() - assert command_check(source, config_gen()) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert command_check(fb_marketing, config_gen()) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) - status = command_check(source, config_gen(start_date="2019-99-10T00:00:00Z")) + status = command_check(fb_marketing, config_gen(start_date="2019-99-10T00:00:00Z")) assert status.status == Status.FAILED - status = command_check(source, config_gen(end_date="2019-99-10T00:00:00Z")) + status = command_check(fb_marketing, config_gen(end_date="2019-99-10T00:00:00Z")) assert status.status == Status.FAILED with pytest.raises(Exception): - assert command_check(source, config_gen(start_date=...)) + assert command_check(fb_marketing, config_gen(start_date=...)) + + assert command_check(fb_marketing, config_gen(end_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert command_check(fb_marketing, config_gen(end_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + + +def test_check_connection_account_type_exception(mocker, fb_marketing, config, logger_mock): + api_mock = mocker.Mock() + api_mock.account.api_get.return_value = {"account": 123, "is_personal": 1} + mocker.patch('source_facebook_marketing.source.API', return_value=api_mock) + + result, error = fb_marketing.check_connection(logger=logger_mock, config=config) - assert command_check(source, config_gen(end_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) - assert command_check(source, config_gen(end_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert not result + assert isinstance(error, AccountTypeException) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py index 5fccdaf5f6e9..13638dbfef97 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py @@ -49,10 +49,12 @@ def test_filter_all_statuses(api, mocker): @pytest.mark.parametrize( - "url", ["https://graph.facebook.com", "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] + "url", ["https://graph.facebook.com", + "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] ) def test_fetch_thumbnail_data_url(url, requests_mock): - requests_mock.get(url, status_code=200, headers={"content-type": "content-type"}, content=b"") + requests_mock.get(url, status_code=200, headers={ + "content-type": "content-type"}, content=b"") assert fetch_thumbnail_data_url(url) == "data:content-type;base64," @@ -61,7 +63,8 @@ def test_parse_call_rate_header(): "x-business-use-case-usage": '{"test":[{"type":"ads_management","call_count":1,"total_cputime":1,' '"total_time":1,"estimated_time_to_regain_access":1}]}' } - assert MyFacebookAdsApi._parse_call_rate_header(headers) == (1, duration(minutes=1)) + assert MyFacebookAdsApi._parse_call_rate_header( + headers) == (1, duration(minutes=1)) @pytest.mark.parametrize( @@ -69,30 +72,53 @@ def test_parse_call_rate_header(): [ [AdsInsights, [], ["action_type", "action_target_id", "action_destination"]], [AdsInsightsActionType, [], ["action_type"]], - [AdsInsightsAgeAndGender, ["age", "gender"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsCountry, ["country"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsDma, ["dma"], ["action_type", "action_target_id", "action_destination"]], - [AdsInsightsPlatformAndDevice, ["publisher_platform", "platform_position", "impression_device"], ["action_type"]], - [AdsInsightsRegion, ["region"], ["action_type", "action_target_id", "action_destination"]], + [AdsInsightsAgeAndGender, ["age", "gender"], [ + "action_type", "action_target_id", "action_destination"]], + [AdsInsightsCountry, ["country"], ["action_type", + "action_target_id", "action_destination"]], + [AdsInsightsDma, ["dma"], ["action_type", + "action_target_id", "action_destination"]], + [AdsInsightsPlatformAndDevice, ["publisher_platform", + "platform_position", "impression_device"], ["action_type"]], + [AdsInsightsRegion, ["region"], ["action_type", + "action_target_id", "action_destination"]], ], ) def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} + kwargs = {"api": None, "start_date": pendulum.now( + ), "end_date": pendulum.now(), "insights_lookback_window": 1} stream = class_name(**kwargs) assert stream.breakdowns == breakdowns assert stream.action_breakdowns == action_breakdowns def test_custom_ads_insights_breakdowns(): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now(), "insights_lookback_window": 1} - stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=["action_destination"], **kwargs) + kwargs = {"api": None, "start_date": pendulum.now( + ), "end_date": pendulum.now(), "insights_lookback_window": 1} + stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=[ + "action_destination"], **kwargs) assert stream.breakdowns == ["mmm"] assert stream.action_breakdowns == ["action_destination"] stream = AdsInsights(breakdowns=[], action_breakdowns=[], **kwargs) assert stream.breakdowns == [] - assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] + assert stream.action_breakdowns == [ + "action_type", "action_target_id", "action_destination"] - stream = AdsInsights(breakdowns=[], action_breakdowns=[], action_breakdowns_allow_empty=True, **kwargs) + stream = AdsInsights(breakdowns=[], action_breakdowns=[], + action_breakdowns_allow_empty=True, **kwargs) assert stream.breakdowns == [] assert stream.action_breakdowns == [] + + +def test_custom_ads_insights_action_report_times(): + kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now( + ), "insights_lookback_window": 1, "action_breakdowns": ["action_destination"], "breakdowns": []} + stream = AdsInsights(**kwargs) + assert stream.action_report_time == "mixed" + + stream = AdsInsights(action_report_time="conversion", **kwargs) + assert stream.action_report_time == "conversion" + + stream = AdsInsights(action_report_time="impression", **kwargs) + assert stream.action_report_time == "impression" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py index 69e82932d763..d50f9f48f5bc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py @@ -22,10 +22,10 @@ ( "start_date", pendulum.local(2019, 1, 1), - pendulum.local(2020, 3, 1), + pendulum.local(2020, 3, 2), [ f"The start date cannot be beyond 37 months from the current date. " - f"Set start date to {pendulum.local(2020, 3, 1)}." + f"Set start date to {pendulum.local(2020, 3, 2)}." ] ), ( diff --git a/airbyte-integrations/connectors/source-facebook-pages/Dockerfile b/airbyte-integrations/connectors/source-facebook-pages/Dockerfile index 9f0d1639d068..e2436d7d20f9 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-pages/Dockerfile @@ -34,5 +34,5 @@ COPY source_facebook_pages ./source_facebook_pages ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-facebook-pages diff --git a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml index 236c2863ca29..510bf9e10a33 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml @@ -5,11 +5,11 @@ data: connectorSubtype: api connectorType: source definitionId: 010eb12f-837b-4685-892d-0a39f76a98f5 - dockerImageTag: 0.2.4 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-facebook-pages githubIssueLabel: source-facebook-pages icon: facebook.svg - license: MIT + license: ELv2 name: Facebook Pages registries: cloud: @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-facebook-pages/requirements.txt b/airbyte-integrations/connectors/source-facebook-pages/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/requirements.txt +++ b/airbyte-integrations/connectors/source-facebook-pages/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-facebook-pages/setup.py b/airbyte-integrations/connectors/source-facebook-pages/setup.py index a49ba9eb84c2..7bce89eaea9d 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/setup.py +++ b/airbyte-integrations/connectors/source-facebook-pages/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "requests-mock"] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"] setup( name="source_facebook_pages", diff --git a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/spec.yaml b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/spec.yaml index 94b96e282802..b0d995592dc5 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/spec.yaml +++ b/airbyte-integrations/connectors/source-facebook-pages/source_facebook_pages/spec.yaml @@ -18,4 +18,22 @@ connectionSpecification: type: string title: Page ID description: Page ID - order: 1 \ No newline at end of file + order: 1 +advanced_auth: + auth_flow_type: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + properties: + access_token: + type: string + path_in_connector_config: + - access_token + complete_oauth_server_input_specification: + type: object + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: {} diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/initial_record.json b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/initial_record.json index ba66ddf06c0f..0193cd73819b 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/initial_record.json +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/initial_record.json @@ -42,7 +42,11 @@ "zip": "94121" }, "merchant_review_status": "default", - "messenger_ads_default_icebreakers": ["Can I learn more about your business?", "Can you tell me more about your ad?", "Is anyone available to chat?"], + "messenger_ads_default_icebreakers": [ + "Can I learn more about your business?", + "Can you tell me more about your ad?", + "Is anyone available to chat?" + ], "messenger_ads_quick_replies_type": "UNKNOWN", "name": "Airbyte", "name_with_location_descriptor": "Airbyte", @@ -76,11 +80,13 @@ "created_time": "2022-05-03T22:39:06+0000", "name": "Timeline photos", "id": "123" - }, { + }, + { "created_time": "2022-05-03T22:39:06+0000", "name": "Profile pictures", "id": "123" - }, { + }, + { "created_time": "2022-05-03T22:39:06+0000", "name": "Cover photos", "id": "123" @@ -99,98 +105,122 @@ "created_time": "2023-01-26T17:42:27+0000", "message": "wow - Airbyte is on the HN front page today! Thanks for your support!\n\n(we're at the bottom right now... but it's just 1hr in!)", "id": "123" - }, { + }, + { "created_time": "2023-01-17T20:24:38+0000", "message": "\u2763\ufe0f Interested in better data quality, ultra reliable data pipelines, and a modern alternative to Airflow?\n\nOur community call with Dagster's Ben Pankow starts in 30 mins\n\nSignup here: \n\nAnd learn all about Dagster and how it works swimmingly with Airbyte! \ud83d\udc19", "id": "123" - }, { + }, + { "created_time": "2023-01-03T13:51:59+0000", "message": "the top 10 good data reads of 2022, from the dbt newsletter: (we're quite happy about #3!)", "id": "123" - }, { + }, + { "created_time": "2022-12-26T01:29:43+0000", "message": "May the magic of Christmas bring you peace, happiness, and abundance in the coming year. Happy Holidays from all of us at Airbyte!", "id": "123" - }, { + }, + { "created_time": "2022-12-21T17:28:01+0000", "message": "\ud83d\udc18 Meet the hottest new Data community: Mastodon!\n\n \n\nYour guide to a federated social media future, and how to join the burgeoning new Data Folks on the platform that *cannot be bought*!", "id": "123" - }, { + }, + { "created_time": "2022-12-19T06:46:17+0000", "id": "123" - }, { + }, + { "created_time": "2022-12-07T17:57:52+0000", "message": "Thousands of attendees are already streaming in for Day 1 of move(data) conf!\n\nJoin us: https://hopin.com/events/movedata\n\nand all these amazing speakers! Starting now!", "id": "123" - }, { + }, + { "created_time": "2022-12-02T16:59:20+0000", "message": "What are the best Data Engineering newsletters?\n\nWe asked ~900 data engineers, and the people have spoken!\n\nPresenting the first preview of the State of Data Engineering survey results, ahead of our Move(data) Conference next week!", "id": "123" - }, { + }, + { "created_time": "2022-11-29T14:14:15+0000", "message": "\ud83d\udc19 The biggest Data Engineering conference of the year starts in 1 week!\n\n\ud83d\uddfa\ufe0f The entire Modern Data Landscape in one conf\n\u270c\ufe0f Dual tracked for maximum pleasure\n\u26a1 Lightning talks only \n\nSee the incredible lineup here:\n\nhttps://airbyte.com/blog/move-data-2022-speakers \n\nHave you registered yet?", "id": "123" - }, { + }, + { "created_time": "2022-11-28T18:00:01+0000", "message": "\ud83c\udf82 It's the 10 year anniversary of #AmazonRedshift today!\n\nWhat do you think were the key highlights of AWS' fastest growing service?\n\nHow did it revolutionize the data engineering industry, and where is it going in the next decade?\n\nWe did a full retrospective + interviewed people inside and out for their opinions:", "id": "123" - }, { + }, + { "created_time": "2022-11-22T16:00:56+0000", "message": "Tristan Handy on the latest Analytics Engineering Roundup: \n\n\"Thalia Barrera writes perhaps the best overview of the state of data engineering since Max B\u2019s canonical post.\" \n\nSo honored for the shoutout!", "id": "123" - }, { + }, + { "created_time": "2022-10-24T17:16:32+0000", "message": "What's that? Airbyte's on the Hacker News front page! \ud83c\udf89 \n\nhttps://news.ycombinator.com/ \n\nWe're discussing @Thalia Barrera's excellent overview of the Past, Present & Future of the Data Engineer role.\n\nIs Data Eng too complex? What's next in adopting Software Eng Best Practices? Join in \ud83d\udc47", "id": "123" - }, { + }, + { "created_time": "2022-10-20T14:16:53+0000", "message": "\ud83e\udd16 Data Nets: The Rise of the AI-first Data Stack\n\n \n\nConfused about the hottest new debate arising from #dbtCoalesce this year? We extracted, loaded, and transformed your favorite thought leadership to explain:\n\n- What are Data Nets?\n- Data Nets vs Data Mesh vs Data Contract\n- When do you need Data Nets?\n- What is a Data Net architecture?\n- What has been dbt and Databricks' response?\n- WHY are we doing this?!?", "id": "123" - }, { + }, + { "created_time": "2022-10-18T15:55:53+0000", "message": "Most surprising result so far from our State of Data Engineering Survey (https://forms.gle/nxZM6XftVYC7yzqW7):\n\n\ud83d\uded1 20% of data teams are in a hiring freeze! \ud83d\uded1\n\nWe're conducting the biggest survey of Data Engineers ever. BigQuery vs Snowflake? Monte Carlo vs Great Expectations? Battle of the Data Engineering Substacks? \n\nCome have your voice heard! (and get special Octavia swag!)", "id": "123" - }, { + }, + { "created_time": "2022-10-13T13:22:00+0000", "message": "It took just 4 months for Get In Data to implement a Modern Data Platform w/ data ingestion (Airbyte), transformation (dbt), orchestration (Amazon Managed Workflows - Apache Airflow), warehouse (Snowflake) https://getindata.com/blog/how-we-built-modern-data-platform-for-fintech-scale-up/", "id": "123" - }, { + }, + { "created_time": "2022-10-11T13:14:00+0000", "message": "Great intro to Airbyte by Kahan Data Solutions: \"Airbyte is incredibly powerful and sufficient for most use cases.\" \ud83d\ude80", "id": "123" - }, { + }, + { "created_time": "2022-09-28T22:56:54+0000", "message": "We've nearly doubled our Sales team over the last several months, and have welcomed onboard @Grace Stern, Growth Account Executive and @Ryan Bernstein, Solutions Engineer.", "id": "123" - }, { + }, + { "created_time": "2022-09-21T12:12:00+0000", "message": "We're excited to share Anecdote's success story about how using Airbyte helped their team grow by freeing up time to focus on building their awesome AI engine \ud83d\ude80 https://airbyte.com/blog/how-airbytes-reliable-ready-to-use-data-pipelines-sped-up-anecdotes-launch-and-growth #OpenSource #DataIntegration #ModernDataStack", "id": "112704783733939_504211088376678" - }, { + }, + { "created_time": "2022-09-07T15:08:27+0000", "message": "Learn how to build a Data Ingestion Pipeline from Microsoft SQL Server to Snowflake as part of your data integration strategy.", "id": "123" - }, { + }, + { "created_time": "2022-08-31T20:14:18+0000", "message": "Consider these key questions before moving data to stay on top of #DataSecurity \n https://airbyte.com/blog/4-questions-data-security-experts-ask-before-moving-data", "id": "123" - }, { + }, + { "created_time": "2022-08-24T19:55:40+0000", "message": "We're proud to announce our inaugural Data Podcast awards:\n\n\ud83c\udfc6 The Best Data Podcasts in 2022!\n\n \n\n(Basically we listened to a ton of podcasts and chose our top 5 favorites each, and why you should check them out!)\n\nDid we miss any? What are your faves?", "id": "123" - }, { + }, + { "created_time": "2022-08-18T18:35:23+0000", "message": "An Airbyte user self-hosting an unsecured instance had their connector credentials stolen. \n\nTransparency is a core value at Airbyte, so we are choosing to highlight this to our community and discuss the steps we will take to improve Airbyte's security defaults.", "id": "123" - }, { + }, + { "created_time": "2022-08-11T15:36:53+0000", "message": "Tuesday was a big day for @dagsterio at the #dagsterday. Huge congrats on the v1.0 release and the General Availability of Dagster Cloud!\n\nWe wrote a short recap about the announcements and newest features, one big called #branchdeployments.\n\nhttps://airbyte.com/blog/dagster-1-0-launch", "id": "123" - }, { + }, + { "created_time": "2022-08-05T15:01:03+0000", "message": "We suggest keeping #Airflow to schedule and monitor #ELT pipelines, but use other #opensource projects that are better suited for the extract, load, and transform steps.", "id": "123" - }, { + }, + { "created_time": "2022-08-04T15:00:37+0000", "message": "Raw data can be very messy as the great saying \"garbage in, garbage out\" \n\n\ud83d\uddd1 But how can we solve it? #SQL #stringfunctions #cleandata", "id": "123" @@ -210,7 +240,8 @@ "description": "Our developer advocate, Abhi Vaidyanatha, will be giving a talk for the upcoming installment of the Data on Kubernetes meetups! He'll go over Airbyte's journey toward bringing the ELT/ETL world to the cloud-native landscape and describe the learnings made along the way.", "updated_time": "2021-11-05T00:44:18+0000", "id": "123" - }, { + }, + { "description": "Michel Tricot\n\nMichel Tricot is the CEO and Co-Founder of Airbyte.", "updated_time": "2022-12-23T03:44:55+0000", "id": "123" @@ -244,16 +275,20 @@ { "created_time": "2022-05-03T22:39:13+0000", "id": "123" - }, { + }, + { "created_time": "2020-11-04T20:11:02+0000", "id": "123" - }, { + }, + { "created_time": "2020-09-19T07:53:49+0000", "id": "123" - }, { + }, + { "created_time": "2020-04-14T04:31:39+0000", "id": "123" - }, { + }, + { "created_time": "2020-04-14T01:24:21+0000", "id": "123" } @@ -278,7 +313,8 @@ "description": "Our developer advocate, Abhi Vaidyanatha, will be giving a talk for the upcoming installment of the Data on Kubernetes meetups! He'll go over Airbyte's journey toward bringing the ELT/ETL world to the cloud-native landscape and describe the learnings made along the way.", "updated_time": "2021-11-05T00:44:18+0000", "id": "123" - }, { + }, + { "description": "Michel Tricot\n\nMichel Tricot is the CEO and Co-Founder of Airbyte.", "updated_time": "2022-12-23T03:44:55+0000", "id": "12344" @@ -291,4 +327,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/transformed_record.json b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/transformed_record.json index 9bcc8136150f..93a6a6a74a67 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/transformed_record.json +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/transformed_record.json @@ -42,7 +42,11 @@ "zip": "94121" }, "merchant_review_status": "default", - "messenger_ads_default_icebreakers": ["Can I learn more about your business?", "Can you tell me more about your ad?", "Is anyone available to chat?"], + "messenger_ads_default_icebreakers": [ + "Can I learn more about your business?", + "Can you tell me more about your ad?", + "Is anyone available to chat?" + ], "messenger_ads_quick_replies_type": "UNKNOWN", "name": "Airbyte", "name_with_location_descriptor": "Airbyte", @@ -76,11 +80,13 @@ "created_time": "2022-05-03T22:39:06+00:00", "name": "Timeline photos", "id": "123" - }, { + }, + { "created_time": "2022-05-03T22:39:06+00:00", "name": "Profile pictures", "id": "123" - }, { + }, + { "created_time": "2022-05-03T22:39:06+00:00", "name": "Cover photos", "id": "123" @@ -99,98 +105,122 @@ "created_time": "2023-01-26T17:42:27+00:00", "message": "wow - Airbyte is on the HN front page today! Thanks for your support!\n\n(we're at the bottom right now... but it's just 1hr in!)", "id": "123" - }, { + }, + { "created_time": "2023-01-17T20:24:38+00:00", "message": "\u2763\ufe0f Interested in better data quality, ultra reliable data pipelines, and a modern alternative to Airflow?\n\nOur community call with Dagster's Ben Pankow starts in 30 mins\n\nSignup here: \n\nAnd learn all about Dagster and how it works swimmingly with Airbyte! \ud83d\udc19", "id": "123" - }, { + }, + { "created_time": "2023-01-03T13:51:59+00:00", "message": "the top 10 good data reads of 2022, from the dbt newsletter: (we're quite happy about #3!)", "id": "123" - }, { + }, + { "created_time": "2022-12-26T01:29:43+00:00", "message": "May the magic of Christmas bring you peace, happiness, and abundance in the coming year. Happy Holidays from all of us at Airbyte!", "id": "123" - }, { + }, + { "created_time": "2022-12-21T17:28:01+00:00", "message": "\ud83d\udc18 Meet the hottest new Data community: Mastodon!\n\n \n\nYour guide to a federated social media future, and how to join the burgeoning new Data Folks on the platform that *cannot be bought*!", "id": "123" - }, { + }, + { "created_time": "2022-12-19T06:46:17+00:00", "id": "123" - }, { + }, + { "created_time": "2022-12-07T17:57:52+00:00", "message": "Thousands of attendees are already streaming in for Day 1 of move(data) conf!\n\nJoin us: https://hopin.com/events/movedata\n\nand all these amazing speakers! Starting now!", "id": "123" - }, { + }, + { "created_time": "2022-12-02T16:59:20+00:00", "message": "What are the best Data Engineering newsletters?\n\nWe asked ~900 data engineers, and the people have spoken!\n\nPresenting the first preview of the State of Data Engineering survey results, ahead of our Move(data) Conference next week!", "id": "123" - }, { + }, + { "created_time": "2022-11-29T14:14:15+00:00", "message": "\ud83d\udc19 The biggest Data Engineering conference of the year starts in 1 week!\n\n\ud83d\uddfa\ufe0f The entire Modern Data Landscape in one conf\n\u270c\ufe0f Dual tracked for maximum pleasure\n\u26a1 Lightning talks only \n\nSee the incredible lineup here:\n\nhttps://airbyte.com/blog/move-data-2022-speakers \n\nHave you registered yet?", "id": "123" - }, { + }, + { "created_time": "2022-11-28T18:00:01+00:00", "message": "\ud83c\udf82 It's the 10 year anniversary of #AmazonRedshift today!\n\nWhat do you think were the key highlights of AWS' fastest growing service?\n\nHow did it revolutionize the data engineering industry, and where is it going in the next decade?\n\nWe did a full retrospective + interviewed people inside and out for their opinions:", "id": "123" - }, { + }, + { "created_time": "2022-11-22T16:00:56+00:00", "message": "Tristan Handy on the latest Analytics Engineering Roundup: \n\n\"Thalia Barrera writes perhaps the best overview of the state of data engineering since Max B\u2019s canonical post.\" \n\nSo honored for the shoutout!", "id": "123" - }, { + }, + { "created_time": "2022-10-24T17:16:32+00:00", "message": "What's that? Airbyte's on the Hacker News front page! \ud83c\udf89 \n\nhttps://news.ycombinator.com/ \n\nWe're discussing @Thalia Barrera's excellent overview of the Past, Present & Future of the Data Engineer role.\n\nIs Data Eng too complex? What's next in adopting Software Eng Best Practices? Join in \ud83d\udc47", "id": "123" - }, { + }, + { "created_time": "2022-10-20T14:16:53+00:00", "message": "\ud83e\udd16 Data Nets: The Rise of the AI-first Data Stack\n\n \n\nConfused about the hottest new debate arising from #dbtCoalesce this year? We extracted, loaded, and transformed your favorite thought leadership to explain:\n\n- What are Data Nets?\n- Data Nets vs Data Mesh vs Data Contract\n- When do you need Data Nets?\n- What is a Data Net architecture?\n- What has been dbt and Databricks' response?\n- WHY are we doing this?!?", "id": "123" - }, { + }, + { "created_time": "2022-10-18T15:55:53+00:00", "message": "Most surprising result so far from our State of Data Engineering Survey (https://forms.gle/nxZM6XftVYC7yzqW7):\n\n\ud83d\uded1 20% of data teams are in a hiring freeze! \ud83d\uded1\n\nWe're conducting the biggest survey of Data Engineers ever. BigQuery vs Snowflake? Monte Carlo vs Great Expectations? Battle of the Data Engineering Substacks? \n\nCome have your voice heard! (and get special Octavia swag!)", "id": "123" - }, { + }, + { "created_time": "2022-10-13T13:22:00+00:00", "message": "It took just 4 months for Get In Data to implement a Modern Data Platform w/ data ingestion (Airbyte), transformation (dbt), orchestration (Amazon Managed Workflows - Apache Airflow), warehouse (Snowflake) https://getindata.com/blog/how-we-built-modern-data-platform-for-fintech-scale-up/", "id": "123" - }, { + }, + { "created_time": "2022-10-11T13:14:00+00:00", "message": "Great intro to Airbyte by Kahan Data Solutions: \"Airbyte is incredibly powerful and sufficient for most use cases.\" \ud83d\ude80", "id": "123" - }, { + }, + { "created_time": "2022-09-28T22:56:54+00:00", "message": "We've nearly doubled our Sales team over the last several months, and have welcomed onboard @Grace Stern, Growth Account Executive and @Ryan Bernstein, Solutions Engineer.", "id": "123" - }, { + }, + { "created_time": "2022-09-21T12:12:00+00:00", "message": "We're excited to share Anecdote's success story about how using Airbyte helped their team grow by freeing up time to focus on building their awesome AI engine \ud83d\ude80 https://airbyte.com/blog/how-airbytes-reliable-ready-to-use-data-pipelines-sped-up-anecdotes-launch-and-growth #OpenSource #DataIntegration #ModernDataStack", "id": "112704783733939_504211088376678" - }, { + }, + { "created_time": "2022-09-07T15:08:27+00:00", "message": "Learn how to build a Data Ingestion Pipeline from Microsoft SQL Server to Snowflake as part of your data integration strategy.", "id": "123" - }, { + }, + { "created_time": "2022-08-31T20:14:18+00:00", "message": "Consider these key questions before moving data to stay on top of #DataSecurity \n https://airbyte.com/blog/4-questions-data-security-experts-ask-before-moving-data", "id": "123" - }, { + }, + { "created_time": "2022-08-24T19:55:40+00:00", "message": "We're proud to announce our inaugural Data Podcast awards:\n\n\ud83c\udfc6 The Best Data Podcasts in 2022!\n\n \n\n(Basically we listened to a ton of podcasts and chose our top 5 favorites each, and why you should check them out!)\n\nDid we miss any? What are your faves?", "id": "123" - }, { + }, + { "created_time": "2022-08-18T18:35:23+00:00", "message": "An Airbyte user self-hosting an unsecured instance had their connector credentials stolen. \n\nTransparency is a core value at Airbyte, so we are choosing to highlight this to our community and discuss the steps we will take to improve Airbyte's security defaults.", "id": "123" - }, { + }, + { "created_time": "2022-08-11T15:36:53+00:00", "message": "Tuesday was a big day for @dagsterio at the #dagsterday. Huge congrats on the v1.0 release and the General Availability of Dagster Cloud!\n\nWe wrote a short recap about the announcements and newest features, one big called #branchdeployments.\n\nhttps://airbyte.com/blog/dagster-1-0-launch", "id": "123" - }, { + }, + { "created_time": "2022-08-05T15:01:03+00:00", "message": "We suggest keeping #Airflow to schedule and monitor #ELT pipelines, but use other #opensource projects that are better suited for the extract, load, and transform steps.", "id": "123" - }, { + }, + { "created_time": "2022-08-04T15:00:37+00:00", "message": "Raw data can be very messy as the great saying \"garbage in, garbage out\" \n\n\ud83d\uddd1 But how can we solve it? #SQL #stringfunctions #cleandata", "id": "123" @@ -210,7 +240,8 @@ "description": "Our developer advocate, Abhi Vaidyanatha, will be giving a talk for the upcoming installment of the Data on Kubernetes meetups! He'll go over Airbyte's journey toward bringing the ELT/ETL world to the cloud-native landscape and describe the learnings made along the way.", "updated_time": "2021-11-05T00:44:18+00:00", "id": "123" - }, { + }, + { "description": "Michel Tricot\n\nMichel Tricot is the CEO and Co-Founder of Airbyte.", "updated_time": "2022-12-23T03:44:55+00:00", "id": "123" @@ -244,16 +275,20 @@ { "created_time": "2022-05-03T22:39:13+00:00", "id": "123" - }, { + }, + { "created_time": "2020-11-04T20:11:02+00:00", "id": "123" - }, { + }, + { "created_time": "2020-09-19T07:53:49+00:00", "id": "123" - }, { + }, + { "created_time": "2020-04-14T04:31:39+00:00", "id": "123" - }, { + }, + { "created_time": "2020-04-14T01:24:21+00:00", "id": "123" } @@ -278,7 +313,8 @@ "description": "Our developer advocate, Abhi Vaidyanatha, will be giving a talk for the upcoming installment of the Data on Kubernetes meetups! He'll go over Airbyte's journey toward bringing the ELT/ETL world to the cloud-native landscape and describe the learnings made along the way.", "updated_time": "2021-11-05T00:44:18+00:00", "id": "123" - }, { + }, + { "description": "Michel Tricot\n\nMichel Tricot is the CEO and Co-Founder of Airbyte.", "updated_time": "2022-12-23T03:44:55+00:00", "id": "12344" @@ -291,4 +327,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index f7cf96ddfc6a..d0648a0212e1 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.1.0 +LABEL io.airbyte.version=5.0.0 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/README.md b/airbyte-integrations/connectors/source-faker/README.md index a99cf816ab13..7864a4b45d53 100644 --- a/airbyte-integrations/connectors/source-faker/README.md +++ b/airbyte-integrations/connectors/source-faker/README.md @@ -164,3 +164,5 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Create a Pull Request. 1. Pat yourself on the back for being an awesome contributor. 1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + +The end diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml index 27aecde8bfaa..9c72cea9eb5f 100644 --- a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml @@ -14,16 +14,48 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: "1.0.0" # We changed the cursor field of the Purchases stream in 2.0.0 + disable_for_version: "4.0.0" # We changed the '*_id' and products.year fields to be integers instead of number basic_read: tests: - config_path: secrets/config.json expect_records: path: integration_tests/expected_records.jsonl + ignored_fields: + users: + - name: updated_at + bypass_reason: "dynamic field" + - name: created_at + bypass_reason: "dynamic field" + products: + - name: updated_at + bypass_reason: "dynamic field" + - name: created_at + bypass_reason: "dynamic field" + purchases: + - name: updated_at + bypass_reason: "dynamic field" + - name: created_at + bypass_reason: "dynamic field" full_refresh: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json + ignored_fields: + users: + - name: updated_at + bypass_reason: "dynamic field" + - name: created_at + bypass_reason: "dynamic field" + products: + - name: updated_at + bypass_reason: "dynamic field" + - name: created_at + bypass_reason: "dynamic field" + purchases: + - name: updated_at + bypass_reason: "dynamic field" + - name: created_at + bypass_reason: "dynamic field" incremental: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-faker/csv_export/README.md b/airbyte-integrations/connectors/source-faker/csv_export/README.md index fdc5245ff308..cc1e11d8d426 100644 --- a/airbyte-integrations/connectors/source-faker/csv_export/README.md +++ b/airbyte-integrations/connectors/source-faker/csv_export/README.md @@ -21,14 +21,13 @@ But let's assume we don't have 1TB of local hard disk. So, we want to make 10 ch ```json { "count": 2039840637, - "seed": 0, - "records_per_sync": 203984064 + "seed": 0 } ``` **`state.json`** -At the end of every sync, increment the `id` in the users stream and the `user_id` in the purchases stream by `203984064`, the `records_per_sync` chunk size +At the end of every sync, increment the `id` in the users stream and the `user_id` in the purchases stream by `203984064` ```json [ diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json index 014e08c0cddd..48ec425863b9 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json @@ -3,7 +3,7 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 11 + "updated_at": 11 }, "stream_descriptor": { "name": "users" @@ -14,7 +14,7 @@ "type": "STREAM", "stream": { "stream_state": { - "user_id": 11 + "updated_at": 11 }, "stream_descriptor": { "name": "purchases" @@ -25,7 +25,7 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 101 + "updated_at": 101 }, "stream_descriptor": { "name": "products" diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json b/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json index 12499ba976f7..afdbad35dc09 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json @@ -6,7 +6,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { "type": "number" }, + "id": { "type": "integer" }, "created_at": { "type": "string", "format": "date-time", diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json index 342b973b47bc..1b6f30efdf19 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json @@ -7,7 +7,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { "type": "number" }, + "id": { "type": "integer" }, "created_at": { "type": "string", "format": "date-time", @@ -45,9 +45,19 @@ "name": "purchases", "json_schema": { "properties": { - "id": { "type": "number" }, - "user_id": { "type": "number" }, - "product_id": { "type": "number" }, + "id": { "type": "integer" }, + "user_id": { "type": "integer" }, + "product_id": { "type": "integer" }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, "added_to_cart_at": { "type": ["null", "string"], "format": "date-time", @@ -77,19 +87,24 @@ "name": "products", "json_schema": { "properties": { - "id": { "type": "number" }, + "id": { "type": "integer" }, "make": { "type": "string" }, "model": { "type": "string" }, - "year": { "type": "number" }, + "year": { "type": "integer" }, "price": { "type": "number" }, "created_at": { "type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } } }, - "supported_sync_modes": ["incremental"], + "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["created_at"] }, diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl index 9a171b604f7b..ac354808125f 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.jsonl @@ -1,9 +1,9 @@ {"stream": "users", "data": {"id": 1, "created_at": "2004-10-28T02:16:07+00:00", "updated_at": "2014-08-21T12:50:13+00:00", "name": "Rudolf", "title": "M.Des", "age": 66, "email": "wisconsin1930+1@yandex.com", "telephone": "(483) 676-2851", "gender": "Fluid", "language": "Arabic", "academic_degree": "Bachelor", "nationality": "Argentinian", "occupation": "Valve Technician", "height": "1.50", "blood_type": "B\u2212", "weight": 81, "address": {"street_number": "276", "street_name": "Yukon", "city": "Wooster", "state": "Wisconsin", "province": "California", "postal_code": "24467", "country_code": "BI"}}, "emitted_at": 1683605758599} {"stream": "users", "data": {"id": 2, "created_at": "2000-12-15T08:46:51+00:00", "updated_at": "2015-01-29T12:27:38+00:00", "name": "Orville", "title": "Miss", "age": 30, "email": "recipes2070+2@yahoo.com", "telephone": "994.991.6727", "gender": "Other", "language": "Montenegrin", "academic_degree": "PhD", "nationality": "Costa Rican", "occupation": "Optical Advisor", "height": "1.64", "blood_type": "AB\u2212", "weight": 70, "address": {"street_number": "1000", "street_name": "Avenue E", "city": "Wilkinsburg", "state": "Missouri", "province": "Nevada", "postal_code": "67628", "country_code": "MT"}}, "emitted_at": 1683605758599} {"stream": "users", "data": {"id": 3, "created_at": "2017-01-31T12:43:13+00:00", "updated_at": "2018-02-11T00:01:01+00:00", "name": "Rachell", "title": "M.A.", "age": 21, "email": "assets1924+3@protonmail.com", "telephone": "+1-(118)-374-3865", "gender": "Female", "language": "Dutch", "academic_degree": "PhD", "nationality": "Danish", "occupation": "Aeronautical Engineer", "height": "1.89", "blood_type": "AB+", "weight": 63, "address": {"street_number": "210", "street_name": "Lysette", "city": "Arlington Heights", "state": "Alaska", "province": "Alaska", "postal_code": "60869", "country_code": "QA"}}, "emitted_at": 1683605758599} -{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "added_to_cart_at": "2003-02-23T11:53:10+00:00", "purchased_at": "2011-03-30T11:53:10+00:00", "returned_at": null}, "emitted_at": 1683605758786} -{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "added_to_cart_at": "2023-05-20T13:40:25+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1683605758786} -{"stream": "purchases", "data": {"id": 3, "product_id": 40, "user_id": 3, "added_to_cart_at": "2013-09-18T00:23:29+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1683605758786} +{"stream": "purchases", "data": {"id": 1, "product_id": 8, "user_id": 1, "created_at": "2001-02-03T11:53:10.771720", "updated_at": "2023-07-11T12:21:28+00:00", "added_to_cart_at": "2005-03-14T11:53:10+00:00", "purchased_at": "2013-04-18T11:53:10+00:00", "returned_at": null}, "emitted_at": 1689067288181} +{"stream": "purchases", "data": {"id": 2, "product_id": 95, "user_id": 2, "created_at": "2018-11-06T13:40:25.842708", "updated_at": "2023-07-11T12:21:28+00:00", "added_to_cart_at": "2023-05-20T13:40:25+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1689067288181} +{"stream": "purchases", "data": {"id": 3, "product_id": 40, "user_id": 3, "created_at": "2008-01-28T00:23:29.977111", "updated_at": "2023-07-11T12:21:28+00:00", "added_to_cart_at": "2013-09-18T00:23:29+00:00", "purchased_at": null, "returned_at": null}, "emitted_at": 1689067288181} {"stream": "products", "data": {"id": 1, "make": "Mazda", "model": "MX-5", "year": 2008, "price": 2869, "created_at": "2022-02-01T17:02:19+00:00"}, "emitted_at": 1682937993845} {"stream": "products", "data": {"id": 2, "make": "Mercedes-Benz", "model": "C-Class", "year": 2009, "price": 42397, "created_at": "2021-01-25T14:31:33+00:00"}, "emitted_at": 1682937993846} {"stream": "products", "data": {"id": 3, "make": "Honda", "model": "Accord Crosstour", "year": 2011, "price": 63293, "created_at": "2021-02-11T05:36:03+00:00"}, "emitted_at": 1682937993846} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json index c3f31c949ebc..b9edd86b582b 100644 --- a/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json @@ -1,6 +1,6 @@ { "count": 10, "seed": 0, - "records_per_sync": 10, - "parallelism": 1 + "parallelism": 1, + "always_updated": false } diff --git a/airbyte-integrations/connectors/source-faker/metadata.yaml b/airbyte-integrations/connectors/source-faker/metadata.yaml index ec8ffefaa30e..1d1a3bfffae2 100644 --- a/airbyte-integrations/connectors/source-faker/metadata.yaml +++ b/airbyte-integrations/connectors/source-faker/metadata.yaml @@ -4,7 +4,7 @@ data: connectorSubtype: api connectorType: source definitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 - dockerImageTag: 2.1.0 + dockerImageTag: 5.0.0 dockerRepository: airbyte/source-faker githubIssueLabel: source-faker icon: faker.svg @@ -30,4 +30,18 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/faker tags: - language:python + releases: + breakingChanges: + 5.0.0: + message: + "ID and products.year fields are changing to be integers instead + of floats." + upgradeDeadline: "2023-08-31" + 4.0.0: + message: "This is a breaking change message" + upgradeDeadline: "2023-07-19" + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-faker/requirements.txt b/airbyte-integrations/connectors/source-faker/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-faker/requirements.txt +++ b/airbyte-integrations/connectors/source-faker/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-faker/setup.py b/airbyte-integrations/connectors/source-faker/setup.py index 16826d62d5b4..1a16ba5ea485 100644 --- a/airbyte-integrations/connectors/source-faker/setup.py +++ b/airbyte-integrations/connectors/source-faker/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "mimesis==6.1.1"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.2", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py index 3668839d6a90..70f166a8a4fb 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/purchase_generator.py @@ -72,6 +72,7 @@ def generate(self, user_id: int) -> List[Dict]: id = user_id + i + 1 - id_offset time_a = dt.datetime() time_b = dt.datetime() + updated_at = format_airbyte_time(datetime.datetime.now()) created_at = time_a if time_a <= time_b else time_b product_id = numeric.integer_number(1, total_products) added_to_cart_at = self.random_date_in_range(created_at) @@ -88,6 +89,8 @@ def generate(self, user_id: int) -> List[Dict]: "id": id, "product_id": product_id, "user_id": user_id + 1, + "created_at": created_at, + "updated_at": updated_at, "added_to_cart_at": format_airbyte_time(added_to_cart_at) if added_to_cart_at is not None else None, "purchased_at": format_airbyte_time(purchased_at) if purchased_at is not None else None, "returned_at": format_airbyte_time(returned_at) if returned_at is not None else None, diff --git a/airbyte-integrations/connectors/source-faker/source_faker/schemas/products.json b/airbyte-integrations/connectors/source-faker/source_faker/schemas/products.json index 04a96566860c..9f86a457d4c8 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/schemas/products.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/schemas/products.json @@ -2,15 +2,20 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { "type": "number" }, + "id": { "type": "integer" }, "make": { "type": "string" }, "model": { "type": "string" }, - "year": { "type": "number" }, + "year": { "type": "integer" }, "price": { "type": "number" }, "created_at": { "type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } } } diff --git a/airbyte-integrations/connectors/source-faker/source_faker/schemas/purchases.json b/airbyte-integrations/connectors/source-faker/source_faker/schemas/purchases.json index c1c032e5ecd9..ab23a9259909 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/schemas/purchases.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/schemas/purchases.json @@ -2,9 +2,19 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { "type": "number" }, - "user_id": { "type": "number" }, - "product_id": { "type": "number" }, + "id": { "type": "integer" }, + "user_id": { "type": "integer" }, + "product_id": { "type": "integer" }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, "added_to_cart_at": { "type": ["null", "string"], "format": "date-time", diff --git a/airbyte-integrations/connectors/source-faker/source_faker/schemas/users.json b/airbyte-integrations/connectors/source-faker/source_faker/schemas/users.json index 720152749a9f..d0c30cd42ee2 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/schemas/users.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/schemas/users.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": { "type": "number" }, + "id": { "type": "integer" }, "created_at": { "type": "string", "format": "date-time", diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py index e67ea8d2faf0..e191687782b2 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/source.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -21,12 +21,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: count: int = config["count"] if "count" in config else 0 seed: int = config["seed"] if "seed" in config else None - records_per_sync: int = config["records_per_sync"] if "records_per_sync" in config else 500 records_per_slice: int = config["records_per_slice"] if "records_per_slice" in config else 100 + always_updated: bool = config["always_updated"] if "always_updated" in config else True parallelism: int = config["parallelism"] if "parallelism" in config else 4 return [ - Products(count, seed, parallelism, records_per_sync, records_per_slice), - Users(count, seed, parallelism, records_per_sync, records_per_slice), - Purchases(count, seed, parallelism, records_per_sync, records_per_slice), + Products(count, seed, parallelism, records_per_slice, always_updated), + Users(count, seed, parallelism, records_per_slice, always_updated), + Purchases(count, seed, parallelism, records_per_slice, always_updated), ] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json index 0d20f791000d..82759547a3a0 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/spec.json +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -22,21 +22,19 @@ "default": -1, "order": 1 }, - "records_per_sync": { - "title": "Records Per Sync", - "description": "How many fake records will be returned for each sync, for each stream? By default, it will take 2 syncs to create the requested 1000 records.", - "type": "integer", - "minimum": 1, - "default": 500, - "order": 2 - }, "records_per_slice": { "title": "Records Per Stream Slice", "description": "How many fake records will be in each page (stream slice), before a state message is emitted?", "type": "integer", "minimum": 1, "default": 1000, - "order": 3 + "order": 2 + }, + "always_updated": { + "title": "Always Updated", + "description": "Should the updated_at values for every record be new each sync? Setting this to false will case the source to stop emitting records after COUNT records have been emitted.", + "type": "boolean", + "default": true }, "parallelism": { "title": "Parallelism", diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index 0a1b1618c06f..ba7d70b7dd2c 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import datetime import os from multiprocessing import Pool from typing import Any, Dict, Iterable, List, Mapping, Optional @@ -10,18 +11,19 @@ from .purchase_generator import PurchaseGenerator from .user_generator import UserGenerator -from .utils import generate_estimate, read_json +from .utils import format_airbyte_time, generate_estimate, read_json class Products(Stream, IncrementalMixin): primary_key = None - cursor_field = "id" + cursor_field = "updated_at" - def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): super().__init__(**kwargs) + self.count = count self.seed = seed - self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice + self.always_updated = always_updated @property def state_checkpoint_interval(self) -> Optional[int]: @@ -32,7 +34,7 @@ def state(self) -> Mapping[str, Any]: if hasattr(self, "_state"): return self._state else: - return {self.cursor_field: 0} + return {} @state.setter def state(self, value: Mapping[str, Any]): @@ -43,33 +45,36 @@ def load_products(self) -> List[Dict]: return read_json(os.path.join(dirname, "record_data", "products.json")) def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: - total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 + if "updated_at" in self.state and not self.always_updated: + return iter([]) + products = self.load_products() + updated_at = "" median_record_byte_size = 180 - rows_to_emit = len(products) - total_records - if rows_to_emit > 0: - yield generate_estimate(self.name, rows_to_emit, median_record_byte_size) + rows_to_emit = len(products) + yield generate_estimate(self.name, rows_to_emit, median_record_byte_size) for product in products: - if product["id"] > total_records: + if product["id"] <= self.count: + updated_at = format_airbyte_time(datetime.datetime.now()) + product["updated_at"] = updated_at yield product - total_records = product["id"] - self.state = {self.cursor_field: total_records, "seed": self.seed} + self.state = {"seed": self.seed, "updated_at": updated_at} class Users(Stream, IncrementalMixin): primary_key = None - cursor_field = "id" + cursor_field = "updated_at" - def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed - self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice self.parallelism = parallelism + self.always_updated = always_updated self.generator = UserGenerator(self.name, self.seed) @property @@ -81,7 +86,7 @@ def state(self) -> Mapping[str, Any]: if hasattr(self, "_state"): return self._state else: - return {self.cursor_field: 0} + return {} @state.setter def state(self, value: Mapping[str, Any]): @@ -93,42 +98,43 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON """ - total_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 - records_in_sync = 0 + if "updated_at" in self.state and not self.always_updated: + return iter([]) + + updated_at = "" median_record_byte_size = 450 - yield generate_estimate(self.name, self.count - total_records, median_record_byte_size) + yield generate_estimate(self.name, self.count, median_record_byte_size) + loop_offset = 0 with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: - while records_in_sync < self.count and records_in_sync < self.records_per_sync: - records_remaining_this_loop = min(self.records_per_slice, (self.count - total_records)) - if records_remaining_this_loop <= 0: - break - users = pool.map(self.generator.generate, range(total_records, total_records + records_remaining_this_loop)) + while loop_offset < self.count: + records_remaining_this_loop = min(self.records_per_slice, (self.count - loop_offset)) + users = pool.map(self.generator.generate, range(loop_offset, loop_offset + records_remaining_this_loop)) for user in users: - total_records += 1 - records_in_sync += 1 + updated_at = user.record.data["updated_at"] + loop_offset += 1 yield user - if records_in_sync >= self.records_per_sync: - break + if records_remaining_this_loop == 0: + break - self.state = {self.cursor_field: total_records, "seed": self.seed} + self.state = {"seed": self.seed, "updated_at": updated_at} - self.state = {self.cursor_field: total_records, "seed": self.seed} + self.state = {"seed": self.seed, "updated_at": updated_at} class Purchases(Stream, IncrementalMixin): primary_key = None - cursor_field = "id" + cursor_field = "updated_at" - def __init__(self, count: int, seed: int, parallelism: int, records_per_sync: int, records_per_slice: int, **kwargs): + def __init__(self, count: int, seed: int, parallelism: int, records_per_slice: int, always_updated: bool, **kwargs): super().__init__(**kwargs) self.count = count self.seed = seed - self.records_per_sync = records_per_sync self.records_per_slice = records_per_slice self.parallelism = parallelism + self.always_updated = always_updated self.generator = PurchaseGenerator(self.name, self.seed) @property @@ -140,7 +146,7 @@ def state(self) -> Mapping[str, Any]: if hasattr(self, "_state"): return self._state else: - return {self.cursor_field: 0} + return {} @state.setter def state(self, value: Mapping[str, Any]): @@ -152,31 +158,28 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: We make N workers (where N is the number of available CPUs) and spread out the CPU-bound work of generating records and serializing them to JSON """ - total_purchase_records = self.state[self.cursor_field] if self.cursor_field in self.state else 0 - total_user_records = self.state["user_id"] if "user_id" in self.state else 0 - user_records_in_sync = 0 + if "updated_at" in self.state and not self.always_updated: + return iter([]) + + updated_at = "" # a fuzzy guess, some users have purchases, some don't median_record_byte_size = 230 - yield generate_estimate(self.name, (self.count - total_user_records) * 1.3, median_record_byte_size) + yield generate_estimate(self.name, (self.count) * 1.3, median_record_byte_size) + loop_offset = 0 with Pool(initializer=self.generator.prepare, processes=self.parallelism) as pool: - while total_user_records < self.count and user_records_in_sync < self.records_per_sync: - records_remaining_this_loop = min(self.records_per_slice, (self.count - user_records_in_sync)) - if records_remaining_this_loop <= 0: - break - carts = pool.map(self.generator.generate, range(total_user_records, total_user_records + records_remaining_this_loop)) + while loop_offset < self.count: + records_remaining_this_loop = min(self.records_per_slice, (self.count - loop_offset)) + carts = pool.map(self.generator.generate, range(loop_offset, loop_offset + records_remaining_this_loop)) for purchases in carts: + loop_offset += 1 for purchase in purchases: - total_purchase_records += 1 + updated_at = purchase.record.data["updated_at"] yield purchase + if records_remaining_this_loop == 0: + break - total_user_records += 1 - user_records_in_sync += 1 - - if user_records_in_sync >= self.records_per_sync: - break - - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + self.state = {"seed": self.seed, "updated_at": updated_at} - self.state = {self.cursor_field: total_purchase_records, "user_id": total_user_records, "seed": self.seed} + self.state = {"seed": self.seed, "updated_at": updated_at} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py index 8d1d72248566..2e8a0b7b2192 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/user_generator.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import datetime from multiprocessing import current_process from airbyte_cdk.models import AirbyteRecordMessage, Type @@ -38,17 +39,14 @@ def prepare(self): dt = Datetime(seed=seed_with_offset) def generate(self, user_id: int): - time_a = dt.datetime() - time_b = dt.datetime() - # faker doesn't always produce unique email addresses, so to enforce uniqueness, we will append the user_id to the prefix email_parts = person.email().split("@") email = f"{email_parts[0]}+{user_id + 1}@{email_parts[1]}" profile = { "id": user_id + 1, - "created_at": format_airbyte_time(time_a if time_a <= time_b else time_b), - "updated_at": format_airbyte_time(time_a if time_a > time_b else time_b), + "created_at": format_airbyte_time(dt.datetime()), + "updated_at": format_airbyte_time(datetime.datetime.now()), "name": person.name(), "title": person.title(), "age": person.age(), @@ -76,8 +74,5 @@ def generate(self, user_id: int): while not profile["created_at"]: profile["created_at"] = format_airbyte_time(dt.datetime()) - if not profile["updated_at"]: - profile["updated_at"] = profile["created_at"] + 1 - record = AirbyteRecordMessage(stream=self.stream_name, data=profile, emitted_at=now_millis()) return AirbyteMessageWithCachedJSON(type=Type.RECORD, record=record) diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py index 0e16824fc6bf..e4fbef60201b 100644 --- a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -9,13 +9,13 @@ class MockLogger: - def debug(a,b, **kwargs): + def debug(a, b, **kwargs): return None - def info(a,b, **kwargs): + def info(a, b, **kwargs): return None - def exception(a,b,**kwargs): + def exception(a, b, **kwargs): print(b) return None @@ -46,7 +46,7 @@ def test_source_streams(): assert len(schemas) == 3 assert schemas[1]["properties"] == { - "id": {"type": "number"}, + "id": {"type": "integer"}, "created_at": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "updated_at": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "name": {"type": "string"}, @@ -62,18 +62,18 @@ def test_source_streams(): "height": {"type": "string"}, "blood_type": {"type": "string"}, "weight": {"type": "integer"}, - 'address': { - 'type': 'object', - 'properties': { - 'city': {'type': 'string'}, - 'country_code': {'type': 'string'}, - 'postal_code': {'type': 'string'}, - 'province': {'type': 'string'}, - 'state': {'type': 'string'}, - 'street_name': {'type': 'string'}, - 'street_number': {'type': 'string'} - } - } + "address": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "country_code": {"type": "string"}, + "postal_code": {"type": "string"}, + "province": {"type": "string"}, + "state": {"type": "string"}, + "street_name": {"type": "string"}, + "street_number": {"type": "string"}, + }, + }, } @@ -95,7 +95,6 @@ def test_read_small_random_data(): estimate_row_count = 0 record_rows_count = 0 state_rows_count = 0 - latest_state = {} for row in iterator: if row.type is Type.TRACE: estimate_row_count = estimate_row_count + 1 @@ -103,17 +102,15 @@ def test_read_small_random_data(): record_rows_count = record_rows_count + 1 if row.type is Type.STATE: state_rows_count = state_rows_count + 1 - latest_state = row assert estimate_row_count == 4 assert record_rows_count == 10 assert state_rows_count == 1 - assert latest_state.state.data == {"users": {"id": 10, "seed": None}} -def test_no_read_limit_hit(): +def test_read_always_updated(): source = SourceFaker() - config = {"count": 10, "parallelism": 1} + config = {"count": 10, "parallelism": 1, "always_updated": False} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -123,62 +120,61 @@ def test_no_read_limit_hit(): } ] ) - state = {"users": {"id": 10}} + state = {} + iterator = source.read(logger, config, catalog, state) + + record_rows_count = 0 + for row in iterator: + if row.type is Type.RECORD: + record_rows_count = record_rows_count + 1 + + assert record_rows_count == 10 + + state = {"users": {"updated_at": "something"}} iterator = source.read(logger, config, catalog, state) record_rows_count = 0 - state_rows_count = 0 - latest_state = {} for row in iterator: if row.type is Type.RECORD: record_rows_count = record_rows_count + 1 - if row.type is Type.STATE: - state_rows_count = state_rows_count + 1 - latest_state = row assert record_rows_count == 0 - assert state_rows_count == 1 - assert latest_state.state.data == {"users": {"id": 10, "seed": None}} -def test_read_big_random_data(): +def test_read_products(): source = SourceFaker() - config = {"count": 1000, "records_per_slice": 100, "records_per_sync": 1000, "parallelism": 1} + config = {"count": 999, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ - { - "stream": {"name": "users", "json_schema": {}, "supported_sync_modes": ["incremental"]}, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - }, { "stream": {"name": "products", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - }, + } ] ) state = {} iterator = source.read(logger, config, catalog, state) + estimate_row_count = 0 record_rows_count = 0 state_rows_count = 0 - latest_state = {} for row in iterator: + if row.type is Type.TRACE: + estimate_row_count = estimate_row_count + 1 if row.type is Type.RECORD: record_rows_count = record_rows_count + 1 if row.type is Type.STATE: state_rows_count = state_rows_count + 1 - latest_state = row - assert record_rows_count == 1000 + 100 # 1000 users, and 100 products - assert latest_state.state.data == {'users': {'seed': None, 'id': 1000}, 'products': {'id': 100, 'seed': None}} - assert state_rows_count == 10 + 1 + 1 + 1 + assert estimate_row_count == 4 + assert record_rows_count == 100 # only 100 products, no matter the count + assert state_rows_count == 2 -def test_with_purchases(): +def test_read_big_random_data(): source = SourceFaker() - config = {"count": 1000, "records_per_sync": 1000, "parallelism": 1} + config = {"count": 1000, "records_per_slice": 100, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { @@ -191,11 +187,6 @@ def test_with_purchases(): "sync_mode": "incremental", "destination_sync_mode": "overwrite", }, - { - "stream": {"name": "purchases", "json_schema": {}, "supported_sync_modes": ["incremental"]}, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - }, ] ) state = {} @@ -203,31 +194,36 @@ def test_with_purchases(): record_rows_count = 0 state_rows_count = 0 - latest_state = {} for row in iterator: if row.type is Type.RECORD: record_rows_count = record_rows_count + 1 if row.type is Type.STATE: state_rows_count = state_rows_count + 1 - latest_state = row - assert record_rows_count > 1000 + 100 # should be greater than 1000 users, and 100 products - assert state_rows_count > 10 + 1 # should be greater than 1000/100, and one state for the products - assert latest_state.state.data["users"] == {"id": 1000, "seed": None} - assert latest_state.state.data["products"] == {'id': 100, 'seed': None} - assert latest_state.state.data["purchases"]["user_id"] > 0 + assert record_rows_count == 1000 + 100 # 1000 users, and 100 products + assert state_rows_count == 10 + 1 + 1 + 1 -def test_sync_ends_with_limit(): +def test_with_purchases(): source = SourceFaker() - config = {"count": 100, "records_per_sync": 5, "parallelism": 1} + config = {"count": 1000, "parallelism": 1} catalog = ConfiguredAirbyteCatalog( streams=[ { "stream": {"name": "users", "json_schema": {}, "supported_sync_modes": ["incremental"]}, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - } + }, + { + "stream": {"name": "products", "json_schema": {}, "supported_sync_modes": ["full_refresh"]}, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + }, + { + "stream": {"name": "purchases", "json_schema": {}, "supported_sync_modes": ["incremental"]}, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + }, ] ) state = {} @@ -235,17 +231,14 @@ def test_sync_ends_with_limit(): record_rows_count = 0 state_rows_count = 0 - latest_state = {} for row in iterator: if row.type is Type.RECORD: record_rows_count = record_rows_count + 1 if row.type is Type.STATE: state_rows_count = state_rows_count + 1 - latest_state = row - assert record_rows_count == 5 - assert state_rows_count == 1 - assert latest_state.state.data == {"users": {"id": 5, "seed": None}} + assert record_rows_count > 1000 + 100 # should be greater than 1000 users, and 100 products + assert state_rows_count > 10 + 1 # should be greater than 1000/100, and one state for the products def test_read_with_seed(): @@ -268,8 +261,8 @@ def test_read_with_seed(): iterator = source.read(logger, config, catalog, state) records = [row for row in iterator if row.type is Type.RECORD] - assert records[0].record.data["occupation"] == "Cartoonist" - assert records[0].record.data["email"] == "reflect1958+1@yahoo.com" + assert records[0].record.data["occupation"] == "Sheriff Principal" + assert records[0].record.data["email"] == "alleged2069+1@example.com" def test_ensure_no_purchases_without_users(): diff --git a/airbyte-integrations/connectors/source-fastbill/Dockerfile b/airbyte-integrations/connectors/source-fastbill/Dockerfile index 796bd1df115e..9dbfa2515b84 100644 --- a/airbyte-integrations/connectors/source-fastbill/Dockerfile +++ b/airbyte-integrations/connectors/source-fastbill/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_fastbill ./source_fastbill ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-fastbill diff --git a/airbyte-integrations/connectors/source-fastbill/README.md b/airbyte-integrations/connectors/source-fastbill/README.md index 8cdfc72fd52d..a6cbffd7fe59 100644 --- a/airbyte-integrations/connectors/source-fastbill/README.md +++ b/airbyte-integrations/connectors/source-fastbill/README.md @@ -1,35 +1,10 @@ # Fastbill Source -This is the repository for the Fastbill source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/fastbill). +This is the repository for the Fastbill configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/fastbill). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,7 +14,7 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/fastbill) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/fastbill) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_fastbill/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,14 +22,6 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source fastbill test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fastbill:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-fastbill:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-fastbill/__init__.py b/airbyte-integrations/connectors/source-fastbill/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-fastbill/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-fastbill/acceptance-test-config.yml b/airbyte-integrations/connectors/source-fastbill/acceptance-test-config.yml index 84429591ceaa..1081a3bd8c19 100644 --- a/airbyte-integrations/connectors/source-fastbill/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-fastbill/acceptance-test-config.yml @@ -1,20 +1,31 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests +# Airbyte doesn't have a Fastbill test account to run tests connector_image: airbyte/source-fastbill:dev -tests: - spec: - - spec_path: "source_fastbill/spec.yaml" +acceptance_tests: + # spec: + # tests: + # - spec_path: "source_fastbill/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["products","recurring_invoices"] - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + # - config_path: "secrets/config.json" + # status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + # discovery: + # tests: + # - config_path: "secrets/config.json" + # basic_read: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # empty_streams: + # - name: products + # - name: recurring_invoices + # - name: revenues + # incremental: + # bypass_reason: "This connector does not implement incremental sync" + # full_refresh: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100755 --- a/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-fastbill/integration_tests/abnormal_state.json index 8890ca08332d..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { - "customers": { - "CUSTOMER_TYPE": 12 + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-fastbill/integration_tests/invalid_config.json index 27bdf1f2dc26..4c66021cda3c 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { - "api_key": "badkeeeey", - "username": "wrong_username" + "api_key": "api_key", + "username": "example@gmail.com" } diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_config.json index ae6a028d1714..4c66021cda3c 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { - "username": "", - "api_key": "" + "api_key": "api_key", + "username": "example@gmail.com" } diff --git a/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_state.json index 7a9ca5814ba8..3587e579822d 100644 --- a/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-fastbill/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "customers": { - "CUSTOMER_TYPE": "strings" + "todo-stream-name": { + "todo-field-name": "value" } } diff --git a/airbyte-integrations/connectors/source-fastbill/metadata.yaml b/airbyte-integrations/connectors/source-fastbill/metadata.yaml index f049c94a93ee..805d8dd61b3c 100644 --- a/airbyte-integrations/connectors/source-fastbill/metadata.yaml +++ b/airbyte-integrations/connectors/source-fastbill/metadata.yaml @@ -1,20 +1,25 @@ data: + allowedHosts: + hosts: + - "*" + registries: + cloud: + enabled: false + oss: + enabled: true connectorSubtype: api connectorType: source definitionId: eb3e9c1c-0467-4eb7-a172-5265e04ccd0a - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-fastbill githubIssueLabel: source-fastbill icon: fastbill.svg license: MIT name: Fastbill - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: "2022-11-08" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/fastbill tags: - - language:python + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-fastbill/requirements.txt b/airbyte-integrations/connectors/source-fastbill/requirements.txt index 91de78ac4144..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-fastbill/requirements.txt +++ b/airbyte-integrations/connectors/source-fastbill/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-fastbill/setup.py b/airbyte-integrations/connectors/source-fastbill/setup.py index a8b6b27944b8..843e992da23a 100644 --- a/airbyte-integrations/connectors/source-fastbill/setup.py +++ b/airbyte-integrations/connectors/source-fastbill/setup.py @@ -5,11 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.4", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "responses~=0.21.0"] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_fastbill", diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/components.py b/airbyte-integrations/connectors/source-fastbill/source_fastbill/components.py new file mode 100644 index 000000000000..002f96dd713c --- /dev/null +++ b/airbyte-integrations/connectors/source-fastbill/source_fastbill/components.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import base64 +from dataclasses import dataclass + +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator + + +@dataclass +class CustomAuthenticator(BasicHttpAuthenticator): + @property + def token(self): + + username = self._username.eval(self.config).encode("latin1") + password = self._password.eval(self.config).encode("latin1") + encoded_credentials = base64.b64encode(b":".join((username, password))).strip() + token = "Basic " + encoded_credentials.decode("utf-8") + return token diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/helpers.py b/airbyte-integrations/connectors/source-fastbill/source_fastbill/helpers.py deleted file mode 100644 index 35354483950e..000000000000 --- a/airbyte-integrations/connectors/source-fastbill/source_fastbill/helpers.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def req_body(offset, endpoint: str): - return {"SERVICE": f"{endpoint}.get", "FILTER": {}, "OFFSET": offset} - - -def get_next_page_token(response, response_key: str, API_OFFSET_LIMIT: int, endpoint: str): - response = response.json() - offset = response["REQUEST"]["OFFSET"] if response["REQUEST"]["OFFSET"] >= 0 else None - if offset is None: - response_request = response["REQUEST"]["OFFSET"] - raise Exception(f"No valid offset value found:{response_request}") - - if len(response["RESPONSE"][response_key]) == API_OFFSET_LIMIT: - return req_body(offset + API_OFFSET_LIMIT, endpoint) - return None - - -def get_request_body_json(next_page_token, endpoint): - if next_page_token: - return next_page_token - else: - return {"SERVICE": f"{endpoint}.get", "FILTER": {}, "OFFSET": 0} diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/manifest.yaml b/airbyte-integrations/connectors/source-fastbill/source_fastbill/manifest.yaml new file mode 100644 index 000000000000..67e0f9622d21 --- /dev/null +++ b/airbyte-integrations/connectors/source-fastbill/source_fastbill/manifest.yaml @@ -0,0 +1,117 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["RESPONSE", "{{ parameters.record_extractor }}"] + requester: + type: HttpRequester + url_base: "https://my.fastbill.com/api/1.0" + http_method: "POST" + authenticator: + class_name: source_fastbill.components.CustomAuthenticator + username: "{{config['username']}}" + password: "{{config['api_key']}}" + request_body_json: + SERVICE: "{{ parameters.endpoint }}.get" + LIMIT: 100 + Content-Type: "application/json" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "OffsetIncrement" + page_size: 100 + page_token_option: + type: "RequestOption" + field_name: "OFFSET" + inject_into: "body_json" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + invoices_stream: + $ref: "#/definitions/base_stream" + name: "invoices" + primary_key: "INVOICE_ID" + $parameters: + path: "/api.php" + endpoint: "invoice" + record_extractor: "INVOICES" + + recurring_invoices_stream: + $ref: "#/definitions/base_stream" + name: "recurring_invoices" + primary_key: "INVOICE_ID" + $parameters: + path: "/api.php" + endpoint: "recurring" + record_extractor: "INVOICES" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + primary_key: "ARTICLE_ID" + $parameters: + path: "/api.php" + endpoint: "article" + record_extractor: "ARTICLES" + + revenues_stream: + $ref: "#/definitions/base_stream" + name: "revenues" + primary_key: "INVOICE_ID" + $parameters: + path: "/api.php" + endpoint: "revenue" + record_extractor: "REVENUES" + + customers_stream: + $ref: "#/definitions/base_stream" + name: "customers" + primary_key: "CUSTOMER_ID" + $parameters: + path: "/api.php" + endpoint: "customer" + record_extractor: "CUSTOMERS" + +streams: + - "#/definitions/invoices_stream" + - "#/definitions/recurring_invoices_stream" + - "#/definitions/products_stream" + - "#/definitions/revenues_stream" + - "#/definitions/customers_stream" + +check: + type: CheckStream + stream_names: + - "invoices" + +spec: + type: Spec + documentationUrl: "https://docs.airbyte.com/integrations/sources/fastbill" + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Fastbill Spec + type: object + required: + - username + - api_key + properties: + username: + title: Username + type: string + description: Username for Fastbill account + api_key: + title: API Key + type: string + description: Fastbill API key + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/source.py b/airbyte-integrations/connectors/source-fastbill/source_fastbill/source.py index 1e1f9242ac50..4c88c647489f 100644 --- a/airbyte-integrations/connectors/source-fastbill/source_fastbill/source.py +++ b/airbyte-integrations/connectors/source-fastbill/source_fastbill/source.py @@ -2,120 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from source_fastbill.helpers import get_next_page_token, get_request_body_json +WARNING: Do not modify this file. +""" -class FastbillStream(HttpStream, ABC): - url_base = " https://my.fastbill.com/api/1.0/api.php" - API_OFFSET_LIMIT = 100 - - def __init__(self, *args, username: str = None, api_key: str = None, **kwargs): - super().__init__(*args, **kwargs) - # self.endpoint = None - self._username = username - self._api_key = api_key - # self.data = None - - @property - def http_method(self) -> str: - return "POST" - - def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return None - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - return {"Content-type": "application/json"} - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Union[Mapping, str]]: - return get_request_body_json(next_page_token, endpoint=self.endpoint) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return get_next_page_token( - response=response, response_key=self.data, API_OFFSET_LIMIT=self.API_OFFSET_LIMIT, endpoint=self.endpoint - ) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json().get("RESPONSE", {}).get(self.data, []) - - -class Invoices(FastbillStream): - primary_key = "INVOICE_ID" - data = "INVOICES" - endpoint = "invoice" - - -class RecurringInvoices(FastbillStream): - primary_key = "INVOICE_ID" - data = "INVOICES" - endpoint = "recurring" - - -class Products(FastbillStream): - primary_key = "ARTICLE_ID" - data = "ARTICLES" - endpoint = "article" - - -class Revenues(FastbillStream): - primary_key = "INVOICE_ID" - data = "REVENUES" - endpoint = "revenue" - - -class Customers(FastbillStream): - primary_key = "CUSTOMER_ID" - data = "CUSTOMERS" - endpoint = "customer" - - -# Source -class SourceFastbill(AbstractSource): - def get_basic_auth(self, config: Mapping[str, Any]) -> requests.auth.HTTPBasicAuth: - return requests.auth.HTTPBasicAuth(config["username"], config["api_key"]) - - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - auth = self.get_basic_auth(config) - records = Customers(auth, **config).read_records(sync_mode=SyncMode.full_refresh) - next(records, None) - return True, None - except Exception as error: - return False, f"Unable to connect to Fastbill API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - - auth = self.get_basic_auth(config) - return [ - Customers(auth, **config), - Invoices(auth, **config), - RecurringInvoices(auth, **config), - Products(auth, **config), - Revenues(auth, **config), - ] +# Declarative Source +class SourceFastbill(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-fastbill/source_fastbill/spec.yaml b/airbyte-integrations/connectors/source-fastbill/source_fastbill/spec.yaml deleted file mode 100644 index 7352591fdcf4..000000000000 --- a/airbyte-integrations/connectors/source-fastbill/source_fastbill/spec.yaml +++ /dev/null @@ -1,18 +0,0 @@ -documentationUrl: "https://docs.airbyte.com/integrations/sources/fastbill" -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Fastbill Spec - type: object - required: - - username - - api_key - properties: - username: - title: Username - type: string - description: Username for Fastbill account - api_key: - title: API Key - type: string - description: Fastbill API key - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-fastbill/unit_tests/test_components.py b/airbyte-integrations/connectors/source-fastbill/unit_tests/test_components.py new file mode 100644 index 000000000000..7e202dec15f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-fastbill/unit_tests/test_components.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_fastbill.components import CustomAuthenticator + + +def test_token_generation(): + + config = {"username": "example@gmail.com", "api_key": "api_key"} + authenticator = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters=None) + token = authenticator.token + expected_token = "Basic ZXhhbXBsZUBnbWFpbC5jb206YXBpX2tleQ==" + assert expected_token == token diff --git a/airbyte-integrations/connectors/source-fastbill/unit_tests/test_source.py b/airbyte-integrations/connectors/source-fastbill/unit_tests/test_source.py deleted file mode 100644 index 13e22552f3fe..000000000000 --- a/airbyte-integrations/connectors/source-fastbill/unit_tests/test_source.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_fastbill.source import SourceFastbill - - -@responses.activate -def test_check_connection(mocker): - url = "https://my.fastbill.com/api/1.0/api.php" - source = SourceFastbill() - logger_mock, config_mock = MagicMock(), MagicMock() - responses.add( - responses.POST, - url, - json={ - "REQUEST": { - "OFFSET": 0, - "FILTER": [], - "LIMIT": 0, - }, - "RESPONSE": { - "CUSTOMERS": "", - }, - }, - ) - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceFastbill() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 5 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-fastbill/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-fastbill/unit_tests/test_streams.py deleted file mode 100644 index 61b1fc8a5ec1..000000000000 --- a/airbyte-integrations/connectors/source-fastbill/unit_tests/test_streams.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_fastbill.source import FastbillStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(FastbillStream, "path", "v0/example_endpoint") - mocker.patch.object(FastbillStream, "primary_key", "test_primary_key") - mocker.patch.object(FastbillStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = FastbillStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = None - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = FastbillStream() - stream.endpoint = "test_endpoint" - stream.data = "test_data" - inputs = {"response": MagicMock()} - - inputs["response"].json.return_value = {"REQUEST": {"OFFSET": 0}, "RESPONSE": {"test_data": []}} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = FastbillStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"Content-type": "application/json"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = FastbillStream() - expected_method = "POST" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = FastbillStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = FastbillStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-fauna/metadata.yaml b/airbyte-integrations/connectors/source-fauna/metadata.yaml index 47c7da8be2d7..602dccd287e4 100644 --- a/airbyte-integrations/connectors/source-fauna/metadata.yaml +++ b/airbyte-integrations/connectors/source-fauna/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/fauna tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-fauna/requirements.txt b/airbyte-integrations/connectors/source-fauna/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-fauna/requirements.txt +++ b/airbyte-integrations/connectors/source-fauna/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-fauna/setup.py b/airbyte-integrations/connectors/source-fauna/setup.py index 2ee8b217e021..552523c1eb37 100644 --- a/airbyte-integrations/connectors/source-fauna/setup.py +++ b/airbyte-integrations/connectors/source-fauna/setup.py @@ -11,8 +11,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index 77b342fbf959..fa3b8daf6d7e 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -1,13 +1,22 @@ -FROM airbyte/source-file:0.3.9 +### WARNING ### +# This Dockerfile will soon be deprecated. +# It is not used to build the connector image we publish to DockerHub. +# The new logic to build the connector image is declared with Dagger here: +# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L771 + +# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. +# Please reach out to the Connectors Operations team if you have any question. +FROM airbyte/source-file:0.3.11 WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure COPY main.py ./ COPY setup.py ./ +ENV DOCKER_BUILD=True RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.9 +LABEL io.airbyte.version=0.3.11 LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file-secure/metadata.yaml b/airbyte-integrations/connectors/source-file-secure/metadata.yaml index d88de8ae1f31..9bccb5d2ef88 100644 --- a/airbyte-integrations/connectors/source-file-secure/metadata.yaml +++ b/airbyte-integrations/connectors/source-file-secure/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.9 + dockerImageTag: 0.3.11 dockerRepository: airbyte/source-file-secure githubIssueLabel: source-file icon: file.svg diff --git a/airbyte-integrations/connectors/source-file-secure/requirements.txt b/airbyte-integrations/connectors/source-file-secure/requirements.txt index add0ec3bd480..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-file-secure/requirements.txt +++ b/airbyte-integrations/connectors/source-file-secure/requirements.txt @@ -1,3 +1 @@ --e ../../bases/connector-acceptance-test --e ../source-file -e . diff --git a/airbyte-integrations/connectors/source-file-secure/setup.py b/airbyte-integrations/connectors/source-file-secure/setup.py index 4fc2474a58c7..f2fb71c1899a 100644 --- a/airbyte-integrations/connectors/source-file-secure/setup.py +++ b/airbyte-integrations/connectors/source-file-secure/setup.py @@ -2,9 +2,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os +from pathlib import Path from setuptools import find_packages, setup + +def local_dependency(name: str) -> str: + """Returns a path to a local package.""" + if os.environ.get("DAGGER_BUILD"): + return f"{name} @ file:///local_dependencies/{name}" + else: + return f"{name} @ file://{Path.cwd().parent / name}" + + MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "gcsfs==2022.7.1", @@ -23,7 +34,10 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["boto3==1.21.21", "pytest==7.1.2", "pytest-docker==1.0.0", "pytest-mock~=3.8.2"] +if not os.environ.get("DOCKER_BUILD"): + MAIN_REQUIREMENTS.append(local_dependency("source-file")) + +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "boto3==1.21.21", "pytest==7.1.2", "pytest-docker==1.0.0", "pytest-mock~=3.8.2"] setup( name="source_file_secure", diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py index 95c8d328d87f..efc59cfbdceb 100644 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py +++ b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py @@ -4,7 +4,8 @@ import json import os -import sys + +import source_file # some integration tests doesn't setup dependences from # requirements.txt file and Python can return a exception. @@ -12,26 +13,10 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import ConnectorSpecification -try: - import source_file.source -except ModuleNotFoundError: - current_dir = os.path.dirname(os.path.abspath(__file__)) - parent_source_local = os.path.join(current_dir, "../../source-file") - if os.path.isdir(parent_source_local): - sys.path.append(parent_source_local) - else: - raise RuntimeError("not found parent source folder") - import source_file.source - -# import original classes of the native Source File -from source_file import SourceFile as ParentSourceFile -from source_file.client import Client -from source_file.client import URLFile as ParentURLFile - LOCAL_STORAGE_NAME = "local" -class URLFileSecure(ParentURLFile): +class URLFileSecure(source_file.client.URLFile): """Updating of default logic: This connector shouldn't work with local files. """ @@ -43,7 +28,7 @@ def __init__(self, url: str, provider: dict, binary=None, encoding=None): super().__init__(url, provider, binary, encoding) -class SourceFileSecure(ParentSourceFile): +class SourceFileSecure(source_file.SourceFile): """Updating of default source logic This connector shouldn't work with local files. The base logic of this connector are implemented in the "source-file" connector. @@ -52,7 +37,7 @@ class SourceFileSecure(ParentSourceFile): @property def client_class(self): # replace a standard class variable to the new one - class ClientSecure(Client): + class ClientSecure(source_file.client.Client): reader_class = URLFileSecure return ClientSecure diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index 480930cc2177..5755c3e6505d 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -17,5 +17,5 @@ COPY source_file ./source_file ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.9 +LABEL io.airbyte.version=0.3.11 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py index 054f3ae3442c..cb8041a4f396 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/conftest.py @@ -6,6 +6,7 @@ import json import os import random +import shutil import socket import string import tempfile @@ -81,7 +82,17 @@ def is_ssh_ready(ip, port): @pytest.fixture(scope="session") -def ssh_service(docker_ip, docker_services): +def move_sample_files_to_tmp(): + """Copy sample files to /tmp so that they can be accessed by the dockerd service in the context of Dagger test runs. + The sample files are mounted to the SSH service from the container under test (container) following instructions of docker-compose.yml in this directory.""" + sample_files = Path(HERE / "sample_files") + shutil.copytree(sample_files, "/tmp/s3_sample_files") + yield True + shutil.rmtree("/tmp/s3_sample_files") + + +@pytest.fixture(scope="session") +def ssh_service(move_sample_files_to_tmp, docker_ip, docker_services): """Ensure that SSH service is up and responsive.""" # `port_for` takes a container port and returns the corresponding host port port = docker_services.port_for("ssh", 22) diff --git a/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml b/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml index 67306a1e102e..60414c5910a5 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml +++ b/airbyte-integrations/connectors/source-file/integration_tests/docker-compose.yml @@ -1,9 +1,9 @@ -version: '3' +version: "3" services: ssh: image: atmoz/sftp ports: - "2222:22" volumes: - - ./sample_files:/home/user1/files + - /tmp/s3_sample_files:/home/user1/files command: user1:abc123@456#:1001 diff --git a/airbyte-integrations/connectors/source-file/metadata.yaml b/airbyte-integrations/connectors/source-file/metadata.yaml index c816ab4309cc..a6a868e7b8c3 100644 --- a/airbyte-integrations/connectors/source-file/metadata.yaml +++ b/airbyte-integrations/connectors/source-file/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.9 + dockerImageTag: 0.3.11 dockerRepository: airbyte/source-file githubIssueLabel: source-file icon: file.svg @@ -14,7 +14,7 @@ data: registries: cloud: dockerRepository: airbyte/source-file-secure - dockerImageTag: 0.3.9 # Dont forget to publish source-file-secure as well when updating this. + dockerImageTag: 0.3.11 # Dont forget to publish source-file-secure as well when updating this. enabled: true oss: enabled: true @@ -22,4 +22,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/file tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-file/requirements.txt b/airbyte-integrations/connectors/source-file/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-file/requirements.txt +++ b/airbyte-integrations/connectors/source-file/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index ee23ed829cb7..d3ae04ed60c6 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -24,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1", "docker-compose"] setup( name="source_file", diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index cf2487ff2bc4..1c6dd8cb7c0a 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -4,6 +4,7 @@ import json +import logging import sys import tempfile import traceback @@ -39,6 +40,9 @@ SSH_TIMEOUT = 60 +# Force the log level of the smart-open logger to ERROR - https://github.com/airbytehq/airbyte/pull/27157 +logging.getLogger("smart_open").setLevel(logging.ERROR) + class ConfigurationError(Exception): """Client mis-configured""" diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py index b15323638627..50bba3cdce94 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import json diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/invalid_config.json index 84e364d32368..381fc12895bb 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/invalid_config.json @@ -1,6 +1,6 @@ { - "database_name": "invalid_database", - "google_application_credentials": "{\"type\":\"service_account\",\"project_id\":\"invalid\",\"private_key_id\":\"invalid_key_id\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"invalid-admin@invalid-project.iam.gserviceaccount.com\",\"client_id\":\"invalid_id\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/invalid-admin%40invalid-project.iam.gserviceaccount.com\"}", - "path": "invalid_path", - "buffer_size": 2 -} \ No newline at end of file + "database_name": "invalid_database", + "google_application_credentials": "{\"type\":\"service_account\",\"project_id\":\"invalid\",\"private_key_id\":\"invalid_key_id\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\ninvalid_private_key_string\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"invalid-admin@invalid-project.iam.gserviceaccount.com\",\"client_id\":\"invalid_id\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/invalid-admin%40invalid-project.iam.gserviceaccount.com\"}", + "path": "invalid_path", + "buffer_size": 2 +} diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/records.json b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/records.json index ee2b720ae464..d5800bff73d6 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/records.json +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/records.json @@ -1,6 +1,6 @@ { - "a": {"a": 1}, - "aa": {"aa": 2}, - "b": {"b": 3}, - "c": {"c": 4} -} \ No newline at end of file + "a": { "a": 1 }, + "aa": { "aa": 2 }, + "b": { "b": 3 }, + "c": { "c": 4 } +} diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/sample_config.json index f30954345bd6..fcb28f6f8beb 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/integration_tests/sample_config.json @@ -1,6 +1,6 @@ { - "database_name": "your-database-name", - "google_application_credentials": "your service_account's credentials JSON", - "path": "path/to/node", - "buffer_size": 10000 + "database_name": "your-database-name", + "google_application_credentials": "your service_account's credentials JSON", + "path": "path/to/node", + "buffer_size": 10000 } diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py index d08b22c8f434..54d63471838e 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/main.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/main.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml b/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml index d506b8797ead..abd8e66c3414 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml @@ -16,7 +16,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/sources/firebase-realtime-database + documentationUrl: https://docs.airbyte.com/integrations/sources/firebase-realtime-database tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py index c0d725e46ea7..1424bb5b1b68 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/setup.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # @@ -11,6 +11,8 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/firebase_rtdb.py b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/firebase_rtdb.py index 2783b3e8d10f..cda280f3edbb 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/firebase_rtdb.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/firebase_rtdb.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import json diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/source.py b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/source.py index ff1378b0edc3..cc06c95433a1 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/source.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/source_firebase_realtime_database/source.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py index 680f95ffff9b..a1370a44d289 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/unit_tests/unit_test.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import string diff --git a/airbyte-integrations/connectors/source-firebolt/Dockerfile b/airbyte-integrations/connectors/source-firebolt/Dockerfile index 2e73ba320e88..7ae26d3abbc0 100644 --- a/airbyte-integrations/connectors/source-firebolt/Dockerfile +++ b/airbyte-integrations/connectors/source-firebolt/Dockerfile @@ -35,5 +35,5 @@ COPY source_firebolt ./source_firebolt ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-firebolt diff --git a/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml index 63d17cc334d0..ff09f0084a8a 100644 --- a/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml @@ -1,29 +1,35 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-firebolt:dev -tests: +acceptance_tests: spec: - - spec_path: "source_firebolt/spec.json" + tests: + - spec_path: "source_firebolt/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - # 0.1.0 contains queries that overwhelm the API server on this test - disable_for_version: "0.1.0" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + # 0.1.0 contains queries that overwhelm the API server on this test + disable_for_version: "0.2.1" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - timeout_seconds: 120 - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: yes - extra_records: no + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + timeout_seconds: 120 + expect_trace_message_on_failure: false + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: yes + extra_records: no full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py index dc65809110a5..c66074e0551b 100644 --- a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py @@ -66,7 +66,7 @@ def table_schema() -> str: # Removed as part of PR 25965 because... it just didn't work? "column5": {"type": ["null", "string"]}, "column6": {"type": "array", "items": {"type": ["null", "integer"]}}, - "column7": {"type": ["null", "integer"]}, + "column7": {"type": ["null", "boolean"]}, }, } return schema diff --git a/airbyte-integrations/connectors/source-firebolt/metadata.yaml b/airbyte-integrations/connectors/source-firebolt/metadata.yaml index d86366aca620..d56f04699b78 100644 --- a/airbyte-integrations/connectors/source-firebolt/metadata.yaml +++ b/airbyte-integrations/connectors/source-firebolt/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: source definitionId: 6f2ac653-8623-43c4-8950-19218c7caf3d - dockerImageTag: 0.2.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-firebolt githubIssueLabel: source-firebolt icon: firebolt.svg @@ -17,4 +17,13 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/firebolt tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community + releases: + breakingChanges: + 1.0.0: + message: "Add new data type columns." + upgradeDeadline: "2023-08-23" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-firebolt/requirements.txt b/airbyte-integrations/connectors/source-firebolt/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-firebolt/requirements.txt +++ b/airbyte-integrations/connectors/source-firebolt/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-firebolt/setup.py b/airbyte-integrations/connectors/source-firebolt/setup.py index af9da9f0bb35..13bb7d102b18 100644 --- a/airbyte-integrations/connectors/source-firebolt/setup.py +++ b/airbyte-integrations/connectors/source-firebolt/setup.py @@ -5,12 +5,13 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "firebolt-sdk>=0.12.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "firebolt-sdk>=0.14.0"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest>=6.2.5", # 6.2.5 has python10 compatibility fixes "pytest-asyncio>=0.18.0", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py index 98481fecbbbe..61a7a71a5792 100644 --- a/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py @@ -29,12 +29,13 @@ def convert_type(fb_type: str, nullable: bool) -> Dict[str, Union[str, Dict]]: "FLOAT": {"type": "number"}, "DOUBLE": {"type": "number"}, "DOUBLE PRECISION": {"type": "number"}, - "BOOLEAN": {"type": "integer"}, + "BOOLEAN": {"type": "boolean"}, # Firebolt bigint is max 8 byte so it fits in Airbyte's "integer" "BIGINT": {"type": "integer"}, "LONG": {"type": "integer"}, "DECIMAL": {"type": "string", "airbyte_type": "big_number"}, "DATE": {"type": "string", "format": "date"}, + "PGDATE": {"type": "string", "format": "date"}, "TIMESTAMP": { "type": "string", "format": "date-time", @@ -45,6 +46,16 @@ def convert_type(fb_type: str, nullable: bool) -> Dict[str, Union[str, Dict]]: "format": "date-time", "airbyte_type": "timestamp_without_timezone", }, + "TIMESTAMPNTZ": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone", + }, + "TIMESTAMPTZ": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_with_timezone", + }, } if fb_type.upper().startswith("ARRAY"): inner_type = fb_type[6:-1] # Strip ARRAY() diff --git a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py index 045b7335e26b..759af4e5cfb2 100644 --- a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py +++ b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py @@ -128,7 +128,26 @@ def test_connection(mock_connection, config, config_no_engine, logger): ("ARRAY(ARRAY(INT NOT NULL))", False, {"type": "array", "items": {"type": "array", "items": {"type": ["null", "integer"]}}}), ("int", True, {"type": ["null", "integer"]}), ("DUMMY", False, {"type": "string"}), - ("boolean", False, {"type": "integer"}), + ("boolean", False, {"type": "boolean"}), + ("pgdate", False, {"type": "string", "format": "date"}), + ( + "TIMESTAMPNTZ", + False, + { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone", + }, + ), + ( + "TIMESTAMPTZ", + False, + { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_with_timezone", + }, + ), ], ) def test_convert_type(type, nullable, result): @@ -143,9 +162,12 @@ def test_convert_type(type, nullable, result): ["a", 1], ), ([datetime.fromisoformat("2019-01-01 20:12:02"), 2], ["2019-01-01T20:12:02", 2]), + ([[date.fromisoformat("0019-01-01"), 2], 0.2214], [["0019-01-01", 2], 0.2214]), ([[date.fromisoformat("2019-01-01"), 2], 0.2214], [["2019-01-01", 2], 0.2214]), ([[None, 2], None], [[None, 2], None]), ([Decimal("1231232.123459999990457054844258706536")], ["1231232.123459999990457054844258706536"]), + ([datetime.fromisoformat("2019-01-01 20:12:02+01:30"), 2], ["2019-01-01T20:12:02+01:30", 2]), + ([True, 2], [True, 2]), ], ) def test_format_fetch_result(data, expected): diff --git a/airbyte-integrations/connectors/source-flexport/Dockerfile b/airbyte-integrations/connectors/source-flexport/Dockerfile index 463534a929bd..9218587fcd0a 100644 --- a/airbyte-integrations/connectors/source-flexport/Dockerfile +++ b/airbyte-integrations/connectors/source-flexport/Dockerfile @@ -34,5 +34,5 @@ COPY source_flexport ./source_flexport ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-flexport diff --git a/airbyte-integrations/connectors/source-flexport/README.md b/airbyte-integrations/connectors/source-flexport/README.md index 0396edd7a0b6..c93479ce2a32 100644 --- a/airbyte-integrations/connectors/source-flexport/README.md +++ b/airbyte-integrations/connectors/source-flexport/README.md @@ -1,35 +1,10 @@ # Flexport Source -This is the repository for the Flexport source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/flexport). +This is the repository for the Flexport configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/flexport). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/flexport) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_flexport/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/flexport) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_flexport/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source flexport test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-flexport:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-flexport:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-flexport/__init__.py b/airbyte-integrations/connectors/source-flexport/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-flexport/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-flexport/acceptance-test-config.yml b/airbyte-integrations/connectors/source-flexport/acceptance-test-config.yml index d2356e9c17b0..6333001736de 100644 --- a/airbyte-integrations/connectors/source-flexport/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-flexport/acceptance-test-config.yml @@ -1,42 +1,61 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-flexport:dev -tests: - spec: - - spec_path: "source_flexport/spec.json" +test_strictness_level: low +acceptance_tests: + # spec: + # tests: + # - spec_path: "source_flexport/manifest.yaml" + # Airbyte doesn't have a test account to test against Flexport API connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_companies.json" - empty_streams: [] - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_locations.json" - empty_streams: [] - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_products.json" - empty_streams: [] - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_shipments.json" - empty_streams: [] - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_invoices.json" - empty_streams: [] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_shipments.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_companies.json" - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_locations.json" - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_products.json" - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_invoices.json" + tests: + # - config_path: "secrets/config.json" + # status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" +# discovery: +# tests: +# - config_path: "secrets/config.json" +# basic_read: +# tests: +# # - config_path: "secrets/config.json" +# # configured_catalog_path: "integration_tests/configured_catalog.json" +# # empty_streams: [] +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_companies.json" +# empty_streams: [] +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_locations.json" +# empty_streams: [] +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_products.json" +# empty_streams: [] +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_shipments.json" +# empty_streams: [] +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_invoices.json" +# empty_streams: [] +# # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# # expect_records: +# # path: "integration_tests/expected_records.jsonl" +# # extra_fields: no +# # exact_order: no +# # extra_records: yes +# incremental: +# bypass_reason: "This connector does not implement incremental sync" +# # TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_shipments.json" +# future_state_path: "integration_tests/abnormal_state.json" +# full_refresh: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_companies.json" +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_locations.json" +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_products.json" +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog_invoices.json" diff --git a/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-flexport/icon.svg b/airbyte-integrations/connectors/source-flexport/icon.svg new file mode 100644 index 000000000000..2eb476e955b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-flexport/icon.svg @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/__init__.py b/airbyte-integrations/connectors/source-flexport/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-flexport/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-flexport/metadata.yaml b/airbyte-integrations/connectors/source-flexport/metadata.yaml index 884d99ccdbe6..deb99f7ca238 100644 --- a/airbyte-integrations/connectors/source-flexport/metadata.yaml +++ b/airbyte-integrations/connectors/source-flexport/metadata.yaml @@ -1,19 +1,29 @@ data: + allowedHosts: + hosts: + - api.flexport.com + - flexport.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: f95337f1-2ad1-4baf-922f-2ca9152de630 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-flexport githubIssueLabel: source-flexport + icon: flexport.svg license: MIT name: Flexport - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2021-12-14 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/flexport tags: - - language:python + - language:low-code + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-flexport/requirements.txt b/airbyte-integrations/connectors/source-flexport/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-flexport/requirements.txt +++ b/airbyte-integrations/connectors/source-flexport/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-flexport/setup.py b/airbyte-integrations/connectors/source-flexport/setup.py index b131a0b4a291..0deaf76a6445 100644 --- a/airbyte-integrations/connectors/source-flexport/setup.py +++ b/airbyte-integrations/connectors/source-flexport/setup.py @@ -10,20 +10,19 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "requests-mock~=1.9.3", - "connector-acceptance-test", ] setup( name="source_flexport", description="Source implementation for Flexport.", - author="Labanoras Tech", - author_email="jv@labanoras.io", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/__init__.py b/airbyte-integrations/connectors/source-flexport/source_flexport/__init__.py index 656ed2e1debf..d54bdab5923c 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/__init__.py +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/manifest.yaml b/airbyte-integrations/connectors/source-flexport/source_flexport/manifest.yaml new file mode 100644 index 000000000000..279cda2ecb8d --- /dev/null +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/manifest.yaml @@ -0,0 +1,157 @@ +version: 0.50.2 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - companies + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: per + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + + requester: + type: HttpRequester + url_base: https://api.flexport.com/ + path: "{{ parameters.path }}" + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_key'] }}" + request_body_json: {} + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + primary_key: + - id + retriever: + $ref: "#/definitions/retriever" + + companies_stream: + $ref: "#/definitions/base_stream" + name: "companies" + $parameters: + path: "network/companies" + + invoices_stream: + $ref: "#/definitions/base_stream" + name: "invoices" + $parameters: + path: "invoices" + + locations_stream: + $ref: "#/definitions/base_stream" + name: "locations" + $parameters: + path: "network/locations" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + $parameters: + path: "products" + + shipments_stream: + type: DeclarativeStream + name: "shipments" + primary_key: + - id + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + requester: + type: HttpRequester + url_base: https://api.flexport.com/ + path: "shipments" + http_method: GET + request_parameters: + sort: updated_at + direction: asc + request_headers: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_key'] }}" + request_body_json: {} + incremental_sync: + type: DatetimeBasedCursor + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_field: updated_at + start_time_option: + inject_into: request_parameter + field_name: f.updated_at.gt + type: RequestOption + end_time_option: + inject_into: request_parameter + field_name: f.updated_at.lt + type: RequestOption + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + +streams: + - "#/definitions/companies_stream" + - "#/definitions/invoices_stream" + - "#/definitions/locations_stream" + - "#/definitions/products_stream" + - "#/definitions/shipments_stream" + +spec: + type: Spec + documentation_url: "https://docs.airbyte.com/integrations/sources/flexport" + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - api_key + - start_date + properties: + api_key: + type: string + title: API Key + airbyte_secret: true + order: 0 + start_date: + type: string + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + order: 1 diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/source.py b/airbyte-integrations/connectors/source-flexport/source_flexport/source.py index ab7040583192..4a9b88b0ed3a 100644 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/source.py +++ b/airbyte-integrations/connectors/source-flexport/source_flexport/source.py @@ -2,40 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_flexport.streams import Companies, FlexportError, FlexportStream, Invoices, Locations, Products, Shipments +WARNING: Do not modify this file. +""" -class SourceFlexport(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - headers = {"Authorization": f"Bearer {config['api_key']}"} - response = requests.get(f"{FlexportStream.url_base}network/companies?page=1&per=1", headers=headers) - - try: - response.raise_for_status() - except Exception as exc: - try: - error = response.json()["errors"][0] - if error: - return False, FlexportError(f"{error['code']}: {error['message']}") - return False, exc - except Exception: - return False, exc - - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = TokenAuthenticator(token=config["api_key"]) - return [ - Companies(authenticator=auth), - Locations(authenticator=auth), - Products(authenticator=auth), - Invoices(authenticator=auth), - Shipments(authenticator=auth, start_date=config["start_date"]), - ] +# Declarative Source +class SourceFlexport(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json b/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json deleted file mode 100644 index d74cccc49727..000000000000 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/spec.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/flexport", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Flexport Spec", - "additionalProperties": true, - "type": "object", - "required": ["api_key", "start_date"], - "properties": { - "api_key": { - "order": 0, - "type": "string", - "title": "API Key", - "airbyte_secret": true - }, - "start_date": { - "order": 1, - "title": "Start Date", - "type": "string", - "format": "date-time" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py b/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py deleted file mode 100644 index b76a8be71b2f..000000000000 --- a/airbyte-integrations/connectors/source-flexport/source_flexport/streams.py +++ /dev/null @@ -1,174 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union -from urllib.parse import parse_qsl, urlparse - -import pendulum -import requests -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator -from requests.auth import AuthBase - - -class FlexportError(Exception): - pass - - -class FlexportStream(HttpStream, ABC): - url_base = "https://api.flexport.com/" - raise_on_http_errors = False - primary_key = "id" - page_size = 100 - - def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, start_date: str = None): - super().__init__(authenticator=authenticator) - - self._authenticator = authenticator - self.start_date = start_date - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - # https://apidocs.flexport.com/v3/tag/Pagination/ - # All list endpoints return paginated responses. The response object contains - # elements of the current page, and links to the previous and next pages. - data = response.json()["data"] - next = data.get("next") - - if next: - url = urlparse(next) - qs = dict(parse_qsl(url.query)) - - return { - "page": qs["page"], - "per": qs["per"], - } - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - if next_page_token: - return next_page_token - - return { - "page": 1, - "per": self.page_size, - } - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # https://apidocs.flexport.com/v3/tag/Response-Semantics - json = response.json() - - http_error = None - try: - response.raise_for_status() - except requests.HTTPError as exc: - http_error = exc - - flexport_error = None - try: - flexport_error = json.get("error") - except AttributeError: - raise FlexportError("Unexpected response") from http_error - - if flexport_error: - try: - if "code" in flexport_error and "message" in flexport_error: - raise FlexportError(f"{flexport_error['code']}: {flexport_error['message']}") from http_error - except TypeError: - pass - - raise FlexportError(f"Unexpected error: {flexport_error}") from http_error - - if http_error: - raise http_error - - yield from json["data"]["data"] - - -class IncrementalFlexportStream(FlexportStream, ABC): - epoch_start = pendulum.from_timestamp(0, tz="UTC").to_iso8601_string() - - @property - def cursor_field(self) -> str: - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - current = current_stream_state.get(self.cursor_field, self.epoch_start) - latest = latest_record.get(self.cursor_field, self.epoch_start) - - return { - self.cursor_field: max(latest, current), - } - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - if not stream_state: - stream_state = {} - - from_date = pendulum.parse(stream_state.get(self.cursor_field, self.start_date)) - end_date = max(from_date, pendulum.tomorrow("UTC")) - - date_diff = end_date - from_date - if date_diff.years > 0: - interval = pendulum.duration(months=1) - elif date_diff.months > 0: - interval = pendulum.duration(weeks=1) - elif date_diff.weeks > 0: - interval = pendulum.duration(days=1) - else: - interval = pendulum.duration(hours=1) - - while True: - to_date = min(from_date + interval, end_date) - yield {"from": from_date.isoformat(), "to": to_date.add(seconds=1).isoformat()} - from_date = to_date - if from_date >= end_date: - break - - -class Companies(FlexportStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "network/companies" - - -class Locations(FlexportStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "network/locations" - - -class Products(FlexportStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "products" - - -class Invoices(FlexportStream): - page_size = 100 - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "invoices" - - -class Shipments(IncrementalFlexportStream): - cursor_field = "updated_at" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "shipments" - - def request_params(self, stream_slice: Mapping[str, any] = None, **kwargs) -> MutableMapping[str, Any]: - return { - **super().request_params(stream_slice=stream_slice, **kwargs), - "sort": self.cursor_field, - "direction": "asc", - "f.updated_at.gt": stream_slice["from"], - "f.updated_at.lt": stream_slice["to"], - } diff --git a/airbyte-integrations/connectors/source-flexport/unit_tests/__init__.py b/airbyte-integrations/connectors/source-flexport/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-flexport/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-flexport/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-flexport/unit_tests/test_incremental_streams.py deleted file mode 100644 index 325a5508b0c2..000000000000 --- a/airbyte-integrations/connectors/source-flexport/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,124 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pendulum -import pytest -from source_flexport.streams import IncrementalFlexportStream, Shipments - - -@pytest.fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalFlexportStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalFlexportStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalFlexportStream, "__abstractmethods__", set()) - mocker.patch.object(IncrementalFlexportStream, "cursor_field", "test_cursor") - - -@pytest.mark.parametrize( - ("stream_class", "cursor"), - [ - (Shipments, "updated_at"), - ], -) -def test_cursor_field(patch_incremental_base_class, stream_class, cursor): - stream = stream_class() - expected_cursor_field = cursor - assert stream.cursor_field == expected_cursor_field - - -@pytest.mark.parametrize( - ("current", "latest", "expected"), - [ - ({"test_cursor": "2021-01-01"}, {}, {"test_cursor": "2021-01-01"}), - ({}, {"test_cursor": "2021-01-01"}, {"test_cursor": "2021-01-01"}), - ({"test_cursor": "2021-01-01"}, {"test_cursor": "2050-01-01"}, {"test_cursor": "2050-01-01"}), - ({"test_cursor": "2050-01-01"}, {"test_cursor": "2021-01-01"}, {"test_cursor": "2050-01-01"}), - ], -) -def test_get_updated_state(patch_incremental_base_class, current, latest, expected): - stream = IncrementalFlexportStream(start_date="2021-01-02") - inputs = {"current_stream_state": current, "latest_record": latest} - assert stream.get_updated_state(**inputs) == expected - - -def date(*args): - return pendulum.datetime(*args).isoformat() - - -@pytest.mark.parametrize( - ("now", "stream_state", "slice_count", "expected_from_date", "expected_to_date"), - [ - (None, None, 24, date(2050, 1, 1), date(2050, 1, 2, 0, 0, 1)), - (date(2050, 1, 2), None, 48, date(2050, 1, 1), date(2050, 1, 3, 0, 0, 1)), - (None, {"test_cursor": date(2050, 1, 4)}, 1, date(2050, 1, 4), date(2050, 1, 4, 0, 0, 1)), - ( - date(2050, 1, 5), - {"test_cursor": date(2050, 1, 4)}, - 48, - date(2050, 1, 4), - date(2050, 1, 6, 0, 0, 1), - ), - ( - # Yearly - date(2052, 1, 1), - {"test_cursor": date(2050, 1, 1)}, - 25, - date(2050, 1, 1), - date(2052, 1, 2, 0, 0, 1), - ), - ( - # Monthly - date(2050, 4, 1), - {"test_cursor": date(2050, 1, 1)}, - 13, - date(2050, 1, 1), - date(2050, 4, 2, 0, 0, 1), - ), - ( - # Weekly - date(2050, 1, 31), - {"test_cursor": date(2050, 1, 1)}, - 5, - date(2050, 1, 1), - date(2050, 2, 1, 0, 0, 1), - ), - ( - # Daily - date(2050, 1, 1, 23, 59, 59), - {"test_cursor": date(2050, 1, 1)}, - 24, - date(2050, 1, 1), - date(2050, 1, 2, 0, 0, 1), - ), - ], -) -def test_stream_slices(patch_incremental_base_class, now, stream_state, slice_count, expected_from_date, expected_to_date): - start_date = date(2050, 1, 1) - pendulum.set_test_now(pendulum.parse(now if now else start_date)) - - stream = IncrementalFlexportStream(start_date=start_date) - stream_slices = list(stream.stream_slices(stream_state)) - - assert len(stream_slices) == slice_count - assert stream_slices[0]["from"] == expected_from_date - assert stream_slices[-1]["to"] == expected_to_date - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalFlexportStream, "cursor_field", "dummy_field") - stream = IncrementalFlexportStream() - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalFlexportStream() - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalFlexportStream() - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-flexport/unit_tests/test_source.py b/airbyte-integrations/connectors/source-flexport/unit_tests/test_source.py deleted file mode 100644 index 6b45978839bb..000000000000 --- a/airbyte-integrations/connectors/source-flexport/unit_tests/test_source.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import re -from unittest.mock import MagicMock - -import pytest -from source_flexport.source import SourceFlexport -from source_flexport.streams import FlexportStream - - -@pytest.mark.parametrize( - ("status_code", "response", "expected"), - [ - (200, {}, (True, None)), - (401, {}, (False, "401 Client Error")), - (401, {"errors": [{"code": "server_error", "message": "Server error"}]}, (False, "server_error: Server error")), - ], -) -def test_check_connection(mocker, requests_mock, status_code, response, expected): - expected_ok, expected_error = expected - requests_mock.get(FlexportStream.url_base + "network/companies?page=1&per=1", status_code=status_code, json=response) - - source = SourceFlexport() - logger_mock, config_mock = MagicMock(), MagicMock() - - ok, error = source.check_connection(logger_mock, config_mock) - assert ok == expected_ok - if isinstance(expected_error, str): - assert re.match(expected_error, str(error)) - else: - assert error == expected_error - - -def test_streams(mocker): - source = SourceFlexport() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 5 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-flexport/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-flexport/unit_tests/test_streams.py deleted file mode 100644 index bb644f93b3c2..000000000000 --- a/airbyte-integrations/connectors/source-flexport/unit_tests/test_streams.py +++ /dev/null @@ -1,87 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from requests.exceptions import HTTPError -from source_flexport.source import FlexportError -from source_flexport.streams import FlexportStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(FlexportStream, "path", "v0/example_endpoint") - mocker.patch.object(FlexportStream, "primary_key", "test_primary_key") - mocker.patch.object(FlexportStream, "__abstractmethods__", set()) - - -@pytest.mark.parametrize( - ("next_page_token", "expected"), - [ - (None, {"page": 1, "per": FlexportStream.page_size}), - ({"page": 2, "per": 50}, {"page": 2, "per": 50}), - ], -) -def test_request_params(patch_base_class, next_page_token, expected): - stream = FlexportStream() - assert stream.request_params(next_page_token=next_page_token) == expected - - -@pytest.mark.parametrize( - ("response", "expected"), - [ - ({"data": {"next": None}}, None), - ({"data": {"next": "/endpoint"}}, KeyError("page")), - ({"data": {"next": "/endpoint?page=2"}}, KeyError("per")), - ({"data": {"next": "/endpoint?page=2&per=42"}}, {"page": "2", "per": "42"}), - ], -) -def test_next_page_token(patch_base_class, requests_mock, response, expected): - url = "http://dummy" - requests_mock.get(url, json=response) - response = requests.get(url) - - stream = FlexportStream() - - if isinstance(expected, Exception): - with pytest.raises(type(expected), match=str(expected)): - stream.next_page_token(response) - else: - assert stream.next_page_token(response) == expected - - -@pytest.mark.parametrize( - ("status_code", "response", "expected"), - [ - (200, None, Exception()), - (400, None, Exception()), - (200, "string_response", FlexportError("Unexpected response")), - (401, "string_response", FlexportError("Unexpected response")), - (200, {"error": None}, KeyError("data")), - (402, {"error": None}, HTTPError("402 Client Error")), - (200, {"error": {}}, KeyError("data")), - (403, {"error": {}}, HTTPError("403 Client Error")), - (200, {"error": "unexpected_error_type"}, FlexportError("Unexpected error: unexpected_error_type")), - (404, {"error": "unexpected_error_type"}, FlexportError("Unexpected error: unexpected_error_type")), - (200, {"error": {"code": "error_code", "message": "Error message"}}, FlexportError("error_code: Error message")), - (405, {"error": {"code": "error_code", "message": "Error message"}}, FlexportError("error_code: Error message")), - (200, {"error": None, "data": "unexpected_data_type"}, TypeError("string indices must be integers")), - (200, {"error": None, "data": {"data": None}}, TypeError("'NoneType' object is not iterable")), - (200, {"error": None, "data": {"data": "hello"}}, ["h", "e", "l", "l", "o"]), - (200, {"error": None, "data": {"data": ["record_1", "record_2"]}}, ["record_1", "record_2"]), - ], -) -def test_parse_response(patch_base_class, requests_mock, status_code, response, expected): - url = "http://dummy" - requests_mock.get(url, status_code=status_code, json=response) - response = requests.get(url) - - stream = FlexportStream() - - if isinstance(expected, Exception): - with pytest.raises(type(expected), match=str(expected)): - list(stream.parse_response(response)) - else: - assert list(stream.parse_response(response)) == expected diff --git a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml index 3948cc029be0..29164f4d589e 100644 --- a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/freshcaller tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshcaller/requirements.txt b/airbyte-integrations/connectors/source-freshcaller/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-freshcaller/requirements.txt +++ b/airbyte-integrations/connectors/source-freshcaller/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-freshcaller/setup.py b/airbyte-integrations/connectors/source-freshcaller/setup.py index ef4196e5ed5c..ec2fdf7dba17 100644 --- a/airbyte-integrations/connectors/source-freshcaller/setup.py +++ b/airbyte-integrations/connectors/source-freshcaller/setup.py @@ -13,7 +13,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock~=1.9.3", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-freshdesk/Dockerfile b/airbyte-integrations/connectors/source-freshdesk/Dockerfile index 7c5266bbd8f1..12a7dad4e74b 100644 --- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile +++ b/airbyte-integrations/connectors/source-freshdesk/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.0.2 +LABEL io.airbyte.version=3.0.4 LABEL io.airbyte.name=airbyte/source-freshdesk diff --git a/airbyte-integrations/connectors/source-freshdesk/metadata.yaml b/airbyte-integrations/connectors/source-freshdesk/metadata.yaml index c12f6505d05b..8a16f346e628 100644 --- a/airbyte-integrations/connectors/source-freshdesk/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshdesk/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 - dockerImageTag: 3.0.2 + dockerImageTag: 3.0.4 dockerRepository: airbyte/source-freshdesk githubIssueLabel: source-freshdesk icon: freshdesk.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/freshdesk tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshdesk/requirements.txt b/airbyte-integrations/connectors/source-freshdesk/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-freshdesk/requirements.txt +++ b/airbyte-integrations/connectors/source-freshdesk/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py index fb0c1520d236..c0d8b408f781 100644 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ b/airbyte-integrations/connectors/source-freshdesk/setup.py @@ -5,18 +5,13 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "backoff==1.10.0", - "requests==2.25.1", - "pendulum==2.1.2", -] +MAIN_REQUIREMENTS = ["airbyte-cdk", "backoff==1.10.0", "requests==2.25.1"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6", "requests_mock~=1.9.3", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py index cc44bf18b185..463cefae8517 100644 --- a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import logging import re from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional @@ -19,6 +19,8 @@ from source_freshdesk.availability_strategy import FreshdeskAvailabilityStrategy from source_freshdesk.utils import CallCredit +logger = logging.getLogger("airbyte") + class FreshdeskStream(HttpStream, ABC): """Basic stream API that allows to iterate over entities""" @@ -98,6 +100,15 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Muta return [] return response.json() or [] + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == requests.codes.FORBIDDEN: + # Issue: https://github.com/airbytehq/airbyte/issues/26717 + # we should skip the stream if subscription level had changed during sync + self.forbidden_stream = True + setattr(self, "raise_on_http_errors", False) + logger.warning(f"Stream `{self.name}` is not available. {response.text}") + return super().should_retry(response) + class IncrementalFreshdeskStream(FreshdeskStream, IncrementalMixin): diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py index 5fdb55192c04..ce3a20a113ca 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py @@ -25,11 +25,9 @@ def test_check_connection_invalid_api_key(requests_mock, config): requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", responses) ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) - - assert not ok and error_msg == "The endpoint to access stream \'settings\' returned 401: Unauthorized. " \ - "This is most likely due to wrong credentials. " \ - "Please visit https://docs.airbyte.com/integrations/sources/freshdesk to learn more. " \ - "You have to be logged in to perform this action." + assert not ok + assert "The endpoint to access stream \'settings\' returned 401: Unauthorized. This is most likely due to wrong credentials. " in error_msg + assert "You have to be logged in to perform this action." in error_msg def test_check_connection_empty_config(config): diff --git a/airbyte-integrations/connectors/source-freshsales/metadata.yaml b/airbyte-integrations/connectors/source-freshsales/metadata.yaml index 4941612dad4f..4b9db49454b2 100644 --- a/airbyte-integrations/connectors/source-freshsales/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshsales/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshsales/requirements.txt b/airbyte-integrations/connectors/source-freshsales/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-freshsales/requirements.txt +++ b/airbyte-integrations/connectors/source-freshsales/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-freshsales/setup.py b/airbyte-integrations/connectors/source-freshsales/setup.py index 5c7bfa3e0f4e..91ef829534e2 100644 --- a/airbyte-integrations/connectors/source-freshsales/setup.py +++ b/airbyte-integrations/connectors/source-freshsales/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest", "pytest-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-freshservice/Dockerfile b/airbyte-integrations/connectors/source-freshservice/Dockerfile index 68de82bef15a..7732b3d3d243 100644 --- a/airbyte-integrations/connectors/source-freshservice/Dockerfile +++ b/airbyte-integrations/connectors/source-freshservice/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_freshservice ./source_freshservice + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_freshservice ./source_freshservice ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.2.0 LABEL io.airbyte.name=airbyte/source-freshservice diff --git a/airbyte-integrations/connectors/source-freshservice/README.md b/airbyte-integrations/connectors/source-freshservice/README.md index 8f50e4975edc..bc19d433edc6 100644 --- a/airbyte-integrations/connectors/source-freshservice/README.md +++ b/airbyte-integrations/connectors/source-freshservice/README.md @@ -1,35 +1,10 @@ # Freshservice Source -This is the repository for the Freshservice source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/freshservice). +This is the repository for the Freshservice configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/freshservice). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshservice) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshservice/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/freshservice) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshservice/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshservice test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshservice:dev disco docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshservice:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-freshservice/__init__.py b/airbyte-integrations/connectors/source-freshservice/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml index ea3b89747601..5f2bc08d045b 100644 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml @@ -1,37 +1,51 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-freshservice:dev -tests: +acceptance_tests: spec: - - spec_path: "source_freshservice/spec.json" + tests: + - spec_path: "source_freshservice/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.1" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "1.1.0" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - [ - "assets", - "releases", - "problems", - "software", - "products", - "changes", - "vendors", - "purchase_orders", - "satisfaction_survey_responses", - ] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: satisfaction_survey_responses + bypass_reason: Test account does not have permissions + - name: releases + bypass_reason: Test account does not have permissions + - name: assets + bypass_reason: Test account does not have permissions + - name: problems + bypass_reason: Test account does not have permissions + - name: software + bypass_reason: Test account does not have permissions + - name: products + bypass_reason: Test account does not have permissions + - name: changes + bypass_reason: Test account does not have permissions + - name: vendors + bypass_reason: Test account does not have permissions + - name: purchase_orders + bypass_reason: Test account does not have permissions + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py b/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json index 294b2030787c..5d54b7054d32 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { - "domain_name": "mydomain.freshservice.com", - "api_key": "", - "start_date": "2123-09-01T00:00:00Z" + "domain_name": "domain.freshservice.com", + "api_key": "invalid_api_key", + "start_date": "2020-10-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json index 1f60922419fd..bfc85382d266 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/sample_config.json @@ -1,5 +1,5 @@ { - "domain_name": "mydomain.freshservice.com", - "api_key": "12345", - "start_date": "2021-09-01T00:00:00Z" + "domain_name": "domain.freshservice.com", + "api_key": "api_key", + "start_date": "2020-10-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-freshservice/metadata.yaml b/airbyte-integrations/connectors/source-freshservice/metadata.yaml index 86b4f28a2fc6..61c73b4c6b10 100644 --- a/airbyte-integrations/connectors/source-freshservice/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshservice/metadata.yaml @@ -1,20 +1,25 @@ data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 9bb85338-ea95-4c93-b267-6be89125b267 - dockerImageTag: 1.1.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-freshservice githubIssueLabel: source-freshservice icon: freshservice.svg license: MIT name: Freshservice - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2021-10-29" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice tags: - - language:python + - language:lowcode metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshservice/requirements.txt b/airbyte-integrations/connectors/source-freshservice/requirements.txt index 91de78ac4144..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-freshservice/requirements.txt +++ b/airbyte-integrations/connectors/source-freshservice/requirements.txt @@ -1,2 +1,2 @@ -e ../../bases/connector-acceptance-test --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json deleted file mode 100644 index 141f41b723cb..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/sample_files/configured_catalog.json +++ /dev/null @@ -1,1046 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "tickets", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "attachments": { - "type": "array" - }, - "cc_emails": { - "type": "array" - }, - "department_id": { - "type": "integer" - }, - "custom_fields": { - "type": "object" - }, - "deleted": { - "type": "boolean" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "due_by": { - "type": "string" - }, - "email": { - "type": "string" - }, - "email_config_id": { - "type": "integer" - }, - "fr_due_by": { - "type": "string" - }, - "fr_escalated": { - "type": "boolean" - }, - "fwd_emails": { - "type": "array" - }, - "group_id": { - "type": "integer" - }, - "id": { - "type": "integer" - }, - "is_escalated": { - "type": "boolean" - }, - "name": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "priority": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "reply_cc_emails": { - "type": "array" - }, - "requester_id": { - "type": "integer" - }, - "responder_id": { - "type": "integer" - }, - "source": { - "type": "integer" - }, - "spam": { - "type": "boolean" - }, - "status": { - "type": "integer" - }, - "subject": { - "type": "string" - }, - "tags": { - "type": "array" - }, - "to_emails": { - "type": "array" - }, - "type": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "urgency": { - "type": "integer" - }, - "impact": { - "type": "integer" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "problems", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "requester_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "priority": { - "type": "integer" - }, - "status": { - "type": "integer" - }, - "impact": { - "type": "integer" - }, - "known_error": { - "type": "boolean" - }, - "subject": { - "type": "string" - }, - "due_by": { - "type": "string" - }, - "department_id": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "associated_change": { - "type": "integer" - }, - "custom_fields": { - "type": "object" - }, - "analysis_fields": { - "type": "object" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "changes", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "requester_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "priority": { - "type": "integer" - }, - "impact": { - "type": "integer" - }, - "status": { - "type": "integer" - }, - "risk": { - "type": "integer" - }, - "change_type": { - "type": "integer" - }, - "approval_status": { - "type": "integer" - }, - "planned_start_date": { - "type": "integer" - }, - "planned_end_date": { - "type": "integer" - }, - "subject": { - "type": "string" - }, - "department_id": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "custom_fields": { - "type": "object" - }, - "maintenance_window": { - "type": "object" - }, - "blackout_window": { - "type": "object" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "releases", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "priority": { - "type": "integer" - }, - "status": { - "type": "integer" - }, - "release_type": { - "type": "integer" - }, - "subject": { - "type": "string" - }, - "description": { - "type": "string" - }, - "planned_start_date": { - "type": "string" - }, - "planned_end_date": { - "type": "string" - }, - "work_start_date": { - "type": "string" - }, - "work_end_date": { - "type": "string" - }, - "department_id": { - "type": "integer" - }, - "category": { - "type": "string" - }, - "sub_category": { - "type": "string" - }, - "item_category": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "associated_assets": { - "type": "array" - }, - "associated_changes": { - "type": "array" - }, - "custom_fields": { - "type": "object" - }, - "planning_fields": { - "type": "object" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "requesters", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "job_title": { - "type": "string" - }, - "primary_email": { - "type": "string" - }, - "secondary_emails": { - "type": "array" - }, - "work_phone_number": { - "type": "string" - }, - "mobile_phone_number": { - "type": "string" - }, - "department_ids": { - "type": "array" - }, - "can_see_all_tickets_from_associated_departments": { - "type": "boolean" - }, - "reporting_manager_id": { - "type": "integer" - }, - "address": { - "type": "string" - }, - "time_zone": { - "type": "string" - }, - "time_format": { - "type": "string" - }, - "language": { - "type": "string" - }, - "location_id": { - "type": "integer" - }, - "background_information": { - "type": "string" - }, - "custom_fields": { - "type": "object" - }, - "active": { - "type": "boolean" - }, - "has_logged_in": { - "type": "boolean" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "is_agent": { - "type": "boolean" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "agents", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "occasional": { - "type": "boolean" - }, - "job_title": { - "type": "string" - }, - "email": { - "type": "string" - }, - "work_phone_number": { - "type": "string" - }, - "mobile_phone_number": { - "type": "string" - }, - "department_ids": { - "type": "array" - }, - "can_see_all_tickets_from_associated_departments": { - "type": "boolean" - }, - "reporting_manager_id": { - "type": "integer" - }, - "address": { - "type": "string" - }, - "time_zone": { - "type": "string" - }, - "time_format": { - "type": "string" - }, - "language": { - "type": "string" - }, - "location_id": { - "type": "integer" - }, - "background_information": { - "type": "string" - }, - "scoreboard_level_id": { - "type": "integer" - }, - "ticket_scope": { - "type": "string" - }, - "problem_scope": { - "type": "string" - }, - "change_scope": { - "type": "string" - }, - "release_scope": { - "type": "string" - }, - "group_ids": { - "type": "array" - }, - "member_of": { - "type": "array" - }, - "observer_of": { - "type": "array" - }, - "role_ids": { - "type": "array" - }, - "roles": { - "type": "array" - }, - "last_login_at": { - "type": "string" - }, - "last_active_at": { - "type": "string" - }, - "custom_fields": { - "type": "object" - }, - "has_logged_in": { - "type": "boolean" - }, - "active": { - "type": "boolean" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "locations", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "parent_location_id": { - "type": "integer" - }, - "primary_contact_id": { - "type": "integer" - }, - "line1": { - "type": "string" - }, - "line2": { - "type": "string" - }, - "city": { - "type": "string" - }, - "state": { - "type": "string" - }, - "country": { - "type": "string" - }, - "zipcode": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "asset_type_id": { - "type": "integer" - }, - "manufacturer": { - "type": "string" - }, - "status": { - "type": "string" - }, - "mode_of_procurement": { - "type": "string" - }, - "depreciation_type_id": { - "type": "integer" - }, - "description": { - "type": "string" - }, - "description_text": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "vendors", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "primary_contact_id": { - "type": "integer" - }, - "line1": { - "type": "string" - }, - "city": { - "type": "string" - }, - "state": { - "type": "string" - }, - "country": { - "type": "string" - }, - "zipcode": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "assets", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "display_id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "asset_type_id": { - "type": "integer" - }, - "asset_tag": { - "type": "string" - }, - "impact": { - "type": "string" - }, - "author_type": { - "type": "string" - }, - "usage_type": { - "type": "string" - }, - "user_id": { - "type": "integer" - }, - "location_id": { - "type": "integer" - }, - "department_id": { - "type": "integer" - }, - "agent_id": { - "type": "integer" - }, - "group_id": { - "type": "integer" - }, - "assigned_on": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "purchase_orders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "vendor_id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "po_number": { - "type": "string" - }, - "vendor_details": { - "type": "string" - }, - "expected_delivery_date": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "created_by": { - "type": "string" - }, - "status": { - "type": "integer" - }, - "shipping_address": { - "type": "string" - }, - "billing_same_as_shipping": { - "type": "boolean" - }, - "billing_address": { - "type": "string" - }, - "currency_code": { - "type": "string" - }, - "conversion_rate": { - "type": "number" - }, - "department_id": { - "type": "integer" - }, - "discount_percentage": { - "type": "integer" - }, - "tax_percentage": { - "type": "integer" - }, - "shipping_cost": { - "type": "integer" - }, - "custom_fields": { - "type": "object" - }, - "purchase_items": { - "type": "array" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "software", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "application_type": { - "type": "string" - }, - "status": { - "type": "string" - }, - "publisher_id": { - "type": "integer" - }, - "managed_by_id": { - "type": "integer" - }, - "notes": { - "type": "string" - }, - "category": { - "type": "string" - }, - "sources": { - "type": "string" - }, - "user_count": { - "type": "integer" - }, - "installation_count": { - "type": "integer" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - } - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshservice/setup.py b/airbyte-integrations/connectors/source-freshservice/setup.py index b6f51d335627..8ccedf94055a 100644 --- a/airbyte-integrations/connectors/source-freshservice/setup.py +++ b/airbyte-integrations/connectors/source-freshservice/setup.py @@ -10,20 +10,19 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "responses~=0.13.3", "connector-acceptance-test", ] setup( name="source_freshservice", description="Source implementation for Freshservice.", - author="Tuan Nguyen", - author_email="anhtuan.nguyen@me.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py index 32d0e324d0dc..f18cc5950ea0 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml new file mode 100644 index 000000000000..fb2cbfe386c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml @@ -0,0 +1,193 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.path_extractor }}"] + requester: + type: HttpRequester + url_base: "https://{{config['domain_name']}}/api/v2/" + http_method: "GET" + authenticator: + type: "BasicHttpAuthenticator" + username: "{{config['api_key']}}" + password: "" + error_handler: + response_filters: + - http_codes: [403] + action: IGNORE + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "PageIncrement" + page_size: 30 + start_from_page: 1 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "updated_at" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + field_name: "updated_since" + inject_into: "request_parameter" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + tickets_stream: + $ref: "#/definitions/base_stream" + name: "tickets" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "tickets" + path: "/tickets" + + satisfaction_survey_responses_stream: + name: "satisfaction_survey_responses" + primary_key: "id" + $parameters: + path_extractor: "csat_response" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "tickets/{{ stream_slice.parent_id }}/csat_response" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/tickets_stream" + parent_key: "id" + partition_field: "parent_id" + + problems_stream: + $ref: "#/definitions/base_stream" + name: "problems" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "problems" + path: "/problems" + + changes_stream: + $ref: "#/definitions/base_stream" + name: "changes" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "changes" + path: "/changes" + + releases_stream: + $ref: "#/definitions/base_stream" + name: "releases" + primary_key: "id" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "releases" + path: "/changes" + + requesters_stream: + $ref: "#/definitions/base_stream" + name: "requesters" + primary_key: "id" + $parameters: + path_extractor: "requesters" + path: "/requesters" + + agents_stream: + $ref: "#/definitions/base_stream" + name: "agents" + primary_key: "id" + $parameters: + path_extractor: "agents" + path: "/agents" + + locations_stream: + $ref: "#/definitions/base_stream" + name: "locations" + primary_key: "id" + $parameters: + path_extractor: "locations" + path: "/locations" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + primary_key: "id" + $parameters: + path_extractor: "products" + path: "/products" + + vendors_stream: + $ref: "#/definitions/base_stream" + name: "vendors" + primary_key: "id" + $parameters: + path_extractor: "vendors" + path: "/vendors" + + assets_stream: + $ref: "#/definitions/base_stream" + name: "assets" + primary_key: "id" + $parameters: + path_extractor: "assets" + path: "/assets" + + software_stream: + $ref: "#/definitions/base_stream" + name: "software" + primary_key: "id" + $parameters: + path_extractor: "applications" + path: "/applications" + + purchase_orders_stream: + $ref: "#/definitions/base_stream" + name: "purchase_orders" + primary_key: "id" + $parameters: + path_extractor: "purchase_orders" + path: "/purchase_orders" + +streams: + - "#/definitions/tickets_stream" + - "#/definitions/satisfaction_survey_responses_stream" + - "#/definitions/problems_stream" + - "#/definitions/changes_stream" + - "#/definitions/releases_stream" + - "#/definitions/requesters_stream" + - "#/definitions/agents_stream" + - "#/definitions/locations_stream" + - "#/definitions/products_stream" + - "#/definitions/vendors_stream" + - "#/definitions/assets_stream" + - "#/definitions/purchase_orders_stream" + - "#/definitions/software_stream" + +check: + type: CheckStream + stream_names: + - "tickets" diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json index ed438d09de03..005af789f286 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json @@ -139,7 +139,13 @@ "type": ["null", "array"] }, "workspace_info": { - "type": ["null", "string"] + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + }, + "work_schedule_id": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json index 0f259a323c3f..e8b0a6816b57 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/assets.json @@ -1,23 +1,65 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "display_id": { "type": ["null", "integer"] }, - "name": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, - "asset_type_id": { "type": ["null", "integer"] }, - "asset_tag": { "type": ["null", "string"] }, - "impact": { "type": ["null", "string"] }, - "author_type": { "type": ["null", "string"] }, - "usage_type": { "type": ["null", "string"] }, - "user_id": { "type": ["null", "integer"] }, - "location_id": { "type": ["null", "integer"] }, - "department_id": { "type": ["null", "integer"] }, - "agent_id": { "type": ["null", "integer"] }, - "group_id": { "type": ["null", "integer"] }, - "assigned_on": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "display_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "asset_type_id": { + "type": ["null", "integer"] + }, + "asset_tag": { + "type": ["null", "string"] + }, + "impact": { + "type": ["null", "string"] + }, + "author_type": { + "type": ["null", "string"] + }, + "usage_type": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "department_id": { + "type": ["null", "integer"] + }, + "agent_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "assigned_on": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "discovery_enabled": { + "type": ["null", "boolean"] + }, + "end_of_life": { + "type": ["null", "string"], + "format": "%Y-%m-%d" + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json index cb9e3aff1c6d..0c8a03caa413 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/changes.json @@ -1,30 +1,111 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "agent_id": { "type": ["null", "integer"] }, - "description": { "type": ["null", "string"] }, - "description_text": { "type": ["null", "string"] }, - "requester_id": { "type": ["null", "integer"] }, - "group_id": { "type": ["null", "integer"] }, - "priority": { "type": ["null", "integer"] }, - "impact": { "type": ["null", "integer"] }, - "status": { "type": ["null", "integer"] }, - "risk": { "type": ["null", "integer"] }, - "change_type": { "type": ["null", "integer"] }, - "approval_status": { "type": ["null", "integer"] }, - "planned_start_date": { "type": ["null", "string"] }, - "planned_end_date": { "type": ["null", "string"] }, - "subject": { "type": ["null", "string"] }, - "department_id": { "type": ["null", "integer"] }, - "category": { "type": ["null", "string"] }, - "sub_category": { "type": ["null", "string"] }, - "item_category": { "type": ["null", "string"] }, - "custom_fields": { "type": ["null", "object"] }, - "maintenance_window": { "type": ["null", "object"] }, - "blackout_window": { "type": ["null", "object"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "agent_id": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "description_text": { + "type": ["null", "string"] + }, + "requester_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "priority": { + "type": ["null", "integer"] + }, + "impact": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "risk": { + "type": ["null", "integer"] + }, + "change_type": { + "type": ["null", "integer"] + }, + "approval_status": { + "type": ["null", "integer"] + }, + "planned_start_date": { + "type": ["null", "string"] + }, + "planned_end_date": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "department_id": { + "type": ["null", "integer"] + }, + "category": { + "type": ["null", "string"] + }, + "sub_category": { + "type": ["null", "string"] + }, + "item_category": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "maintenance_window": { + "type": ["null", "object"], + "additionalProperties": true + }, + "blackout_window": { + "type": ["null", "object"], + "additionalProperties": true + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "planned_effort": { + "type": ["string", "null"] + }, + "attachments": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + }, + "impacted_services": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + }, + "workspace_id": { + "type": ["null", "integer"] + }, + "change_window_id": { + "type": ["null", "integer"] + }, + "assets": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json index 154048ff14e6..748b273350af 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/locations.json @@ -3,21 +3,56 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "name": { "type": ["null", "string"] }, - "parent_location_id": { "type": ["null", "integer"] }, - "primary_contact_id": { "type": ["null", "integer"] }, - "line1": { "type": ["null", "string"] }, - "line2": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "zipcode": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "email": {"type": ["null", "string"] }, - "phone": {"type": ["null", "string"] }, - "address": {"type": ["null", "string"] }, - "contact_name": {"type": ["null", "string"] } + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "parent_location_id": { + "type": ["null", "integer"] + }, + "primary_contact_id": { + "type": ["null", "integer"] + }, + "address": { + "type": "object", + "additionalProperties": true, + "properties": { + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zipcode": { + "type": ["null", "string"] + } + } + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "contact_name": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json index c6bcd36e8291..21f3b6ff8404 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/problems.json @@ -1,27 +1,100 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "agent_id": { "type": ["null", "integer"] }, - "requester_id": { "type": ["null", "integer"] }, - "group_id": { "type": ["null", "integer"] }, - "description": { "type": ["null", "string"] }, - "description_text": { "type": ["null", "string"] }, - "priority": { "type": ["null", "integer"] }, - "status": { "type": ["null", "integer"] }, - "impact": { "type": ["null", "integer"] }, - "known_error": { "type": ["null", "boolean"] }, - "subject": { "type": ["null", "string"] }, - "due_by": { "type": ["null", "string"] }, - "department_id": { "type": ["null", "integer"] }, - "category": { "type": ["null", "string"] }, - "sub_category": { "type": ["null", "string"] }, - "item_category": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "associated_change": { "type": ["null", "integer"] }, - "custom_fields": { "type": ["null", "object"] }, - "analysis_fields": { "type": "object" } + "id": { + "type": "integer" + }, + "agent_id": { + "type": ["null", "integer"] + }, + "requester_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "description_text": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "impact": { + "type": ["null", "integer"] + }, + "known_error": { + "type": ["null", "boolean"] + }, + "subject": { + "type": ["null", "string"] + }, + "due_by": { + "type": ["null", "string"] + }, + "department_id": { + "type": ["null", "integer"] + }, + "category": { + "type": ["null", "string"] + }, + "sub_category": { + "type": ["null", "string"] + }, + "item_category": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "associated_change": { + "type": ["null", "integer"] + }, + "custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "analysis_fields": { + "type": "object", + "additionalProperties": true + }, + "planned_start_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "planned_end_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "planned_effort": { + "type": ["string", "null"] + }, + "attachments": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + }, + "workspace_id": { + "type": ["null", "integer"] + }, + "assets": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json index 3de1cb4eccb1..fd80515db95b 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/purchase_orders.json @@ -1,27 +1,76 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "vendor_id": { "type": ["null", "integer"] }, - "name": { "type": ["null", "string"] }, - "po_number": { "type": ["null", "string"] }, - "vendor_details": { "type": ["null", "string"] }, - "expected_delivery_date": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "created_by": { "type": ["null", "string"] }, - "status": { "type": ["null", "integer"] }, - "shipping_address": { "type": ["null", "string"] }, - "billing_same_as_shipping": { "type": ["null", "integer"] }, - "billing_address": { "type": ["null", "string"] }, - "currency_code": { "type": ["null", "string"] }, - "conversion_rate": { "type": "number" }, - "department_id": { "type": ["null", "integer"] }, - "discount_percentage": { "type": ["null", "integer"] }, - "tax_percentage": { "type": ["null", "integer"] }, - "shipping_cost": { "type": ["null", "integer"] }, - "custom_fields": { "type": ["null", "object"] }, - "purchase_items": { "type": "array" } + "id": { + "type": "integer" + }, + "vendor_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "po_number": { + "type": ["null", "string"] + }, + "vendor_details": { + "type": ["null", "string"] + }, + "expected_delivery_date": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "created_by": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "integer"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "billing_same_as_shipping": { + "type": ["null", "integer"] + }, + "billing_address": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + }, + "conversion_rate": { + "type": "number" + }, + "department_id": { + "type": ["null", "integer"] + }, + "discount_percentage": { + "type": ["null", "integer"] + }, + "tax_percentage": { + "type": ["null", "integer"] + }, + "shipping_cost": { + "type": ["null", "integer"] + }, + "workspace_id": { + "type": ["null", "integer"] + }, + "total_cost": { + "type": ["null", "number"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "purchase_items": { + "type": "array" + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json index e77460367419..73cc035f2a95 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requesters.json @@ -3,37 +3,92 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "first_name": { "type": ["null", "string"] }, - "last_name": { "type": ["null", "string"] }, - "job_title": { "type": ["null", "string"] }, - "primary_email": { "type": ["null", "string"] }, - "secondary_emails": { "type": ["null", "array"] }, - "work_phone_number": { "type": ["null", "string"] }, - "mobile_phone_number": { "type": ["null", "string"] }, - "department_ids": { "type": ["null", "array"] }, + "id": { + "type": "integer" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "job_title": { + "type": ["null", "string"] + }, + "primary_email": { + "type": ["null", "string"] + }, + "secondary_emails": { + "type": ["null", "array"] + }, + "work_phone_number": { + "type": ["null", "string"] + }, + "mobile_phone_number": { + "type": ["null", "string"] + }, + "department_ids": { + "type": ["null", "array"] + }, "can_see_all_tickets_from_associated_departments": { "type": ["null", "boolean"] }, - "reporting_manager_id": { "type": ["null", "integer"] }, - "address": { "type": ["null", "string"] }, - "time_zone": { "type": ["null", "string"] }, - "time_format": { "type": ["null", "string"] }, - "language": { "type": ["null", "string"] }, - "location_id": { "type": ["null", "integer"] }, - "background_information": { "type": ["null", "string"] }, - "custom_fields": { "type": ["null", "object"] }, - "active": { "type": ["null", "boolean"] }, - "has_logged_in": { "type": ["null", "boolean"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] }, - "is_agent": { "type": ["null", "boolean"] }, - "department_names": { "type": ["null", "array"] }, - "vip_user": { "type": ["null", "boolean"] }, - "external_id": { "type": ["null", "string"] }, + "reporting_manager_id": { + "type": ["null", "integer"] + }, + "address": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "time_format": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "background_information": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "active": { + "type": ["null", "boolean"] + }, + "has_logged_in": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "is_agent": { + "type": ["null", "boolean"] + }, + "department_names": { + "type": ["null", "array"] + }, + "vip_user": { + "type": ["null", "boolean"] + }, + "external_id": { + "type": ["null", "string"] + }, "can_see_all_changes_from_associated_departments": { "type": ["null", "boolean"] }, - "location_name": { "type": ["null", "string"] } + "location_name": { + "type": ["null", "string"] + }, + "work_schedule_id": { + "type": ["null", "integer"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json index 6652c10f11a9..2c83436b7bd9 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/software.json @@ -1,20 +1,58 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "name": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, - "application_type": { "type": ["null", "string"] }, - "status": { "type": ["null", "string"] }, - "publisher_id": { "type": ["null", "integer"] }, - "managed_by_id": { "type": ["null", "integer"] }, - "notes": { "type": ["null", "string"] }, - "category": { "type": ["null", "string"] }, - "sources": { "type": ["null", "string"] }, - "user_count": { "type": ["null", "integer"] }, - "installation_count": { "type": ["null", "integer"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "application_type": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "publisher_id": { + "type": ["null", "integer"] + }, + "managed_by_id": { + "type": ["null", "integer"] + }, + "notes": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "sources": { + "type": ["null", "array"], + "items": { + "additionalProperties": true + } + }, + "user_count": { + "type": ["null", "integer"] + }, + "installation_count": { + "type": ["null", "integer"] + }, + "workspace_id": { + "type": ["null", "integer"] + }, + "additional_data": { + "type": ["null", "object"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json index f68696260f7e..94984c8d005d 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/vendors.json @@ -1,17 +1,61 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { - "id": { "type": "integer" }, - "name": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, - "primary_contact_id": { "type": ["null", "integer"] }, - "line1": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "zipcode": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"] } + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "contact_name": { + "type": ["null", "string"] + }, + "mobile": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "integer"] + }, + "primary_contact_id": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"] + }, + "address": { + "type": "object", + "additionalProperties": true, + "properties": { + "line1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zipcode": { + "type": ["null", "string"] + } + } + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py index ec33e8cce3c6..d4cdeed0e652 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/source.py @@ -2,73 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import base64 -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from .streams import ( - Agents, - Assets, - Changes, - Locations, - Problems, - Products, - PurchaseOrders, - Releases, - Requesters, - SatisfactionSurveyResponses, - Software, - Tickets, - Vendors, -) +WARNING: Do not modify this file. +""" -# Source -class HttpBasicAuthenticator(TokenAuthenticator): - def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): - auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") - b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) - - -class SourceFreshservice(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - kwargs = { - "authenticator": HttpBasicAuthenticator((config["api_key"], "")), - "start_date": config["start_date"], - "domain_name": config["domain_name"], - } - try: - tickets = Tickets(**kwargs).read_records(sync_mode=SyncMode.full_refresh) - next(tickets) - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - kwargs = { - "authenticator": HttpBasicAuthenticator((config["api_key"], "")), - "start_date": config["start_date"], - "domain_name": config["domain_name"], - } - return [ - Tickets(**kwargs), - Problems(**kwargs), - Changes(**kwargs), - Releases(**kwargs), - Requesters(**kwargs), - Agents(**kwargs), - Locations(**kwargs), - Products(**kwargs), - Vendors(**kwargs), - Assets(**kwargs), - PurchaseOrders(**kwargs), - SatisfactionSurveyResponses(**kwargs), - Software(**kwargs), - ] +# Declarative Source +class SourceFreshservice(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json deleted file mode 100644 index 1ff611c10e85..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/freshservice", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Freshservice Spec", - "type": "object", - "required": ["domain_name", "api_key", "start_date"], - "additionalProperties": true, - "properties": { - "domain_name": { - "type": "string", - "title": "Domain Name", - "description": "The name of your Freshservice domain", - "examples": ["mydomain.freshservice.com"] - }, - "api_key": { - "title": "API Key", - "type": "string", - "description": "Freshservice API Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "UTC date and time in the format 2020-10-01T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2020-10-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml new file mode 100644 index 000000000000..4eefac53ed51 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/spec.yaml @@ -0,0 +1,33 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice +connectionSpecification: + "$schema": http://json-schema.org/draft-07/schema# + title: Freshservice Spec + type: object + required: + - domain_name + - api_key + - start_date + additionalProperties: true + properties: + domain_name: + type: string + title: Domain Name + description: The name of your Freshservice domain + examples: + - mydomain.freshservice.com + api_key: + title: API Key + type: string + description: + Freshservice API Key. See here. + The key is case sensitive. + airbyte_secret: true + start_date: + title: Start Date + type: string + description: + UTC date and time in the format 2020-10-01T00:00:00Z. Any data + before this date will not be replicated. + examples: + - "2020-10-01T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py b/airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py deleted file mode 100644 index bc5daec9ac34..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/streams.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from http import HTTPStatus -from typing import Any, Iterable, Mapping, MutableMapping, Optional -from urllib.parse import parse_qsl, urlparse - -import requests -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - - -# Basic full refresh stream -class FreshserviceStream(HttpStream, ABC): - primary_key = "id" - order_field = "updated_at" - page_size = 30 - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, start_date: str = None, domain_name: str = None, **kwargs): - super().__init__(**kwargs) - self._start_date = start_date - self.domain_name = domain_name - - @property - def url_base(self) -> str: - return f"https://{self.domain_name}/api/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page = response.links.get("next") - if next_page: - return {"page": dict(parse_qsl(urlparse(next_page.get("url")).query)).get("page")} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"per_page": self.page_size} - - if next_page_token: - params.update(next_page_token) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code == HTTPStatus.NO_CONTENT: - return - json_response = response.json() - records = json_response.get(self.object_name, []) if self.object_name is not None else json_response - yield from records - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.object_name - - -# Basic incremental stream -class IncrementalFreshserviceStream(FreshserviceStream, ABC): - state_checkpoint_interval = 60 - cursor_field = "updated_at" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - - def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if not next_page_token: - params["order_by"] = self.order_field - params["order_type"] = "asc" - if stream_state: - params["updated_since"] = stream_state.get(self.cursor_field) - return params - - -class Tickets(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#view_all_ticket - """ - - object_name = "tickets" - - -class SatisfactionSurveyResponses(IncrementalFreshserviceStream, HttpSubStream): - object_name = "csat_response" - """ - API docs: https://api.freshservice.com/#ticket_csat_attributes - - self.authenticator (which should be used as the - authenticator for Users) is object of NoAuth() - so self._session.auth is used instead - """ - - def __init__(self, **kwargs): - super().__init__(parent=Tickets(**kwargs), **kwargs) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - ticket_id = stream_slice["parent"]["id"] - return f"tickets/{ticket_id}/csat_response" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code == HTTPStatus.NO_CONTENT: - return - json_response = response.json() - record = json_response.get(self.object_name, {}) if self.object_name is not None else json_response - yield record - - -class Problems(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#problems - """ - - object_name = "problems" - - -class Changes(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#changes - """ - - object_name = "changes" - - -class Releases(IncrementalFreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#releases - """ - - object_name = "releases" - - -class Requesters(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#requesters - """ - - object_name = "requesters" - - -class Agents(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#agents - """ - - object_name = "agents" - - -class Locations(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#locations - """ - - object_name = "locations" - - -class Products(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#products - """ - - object_name = "products" - - -class Vendors(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#vendors - """ - - object_name = "vendors" - - -class Assets(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#assets - """ - - object_name = "assets" - - -class PurchaseOrders(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#purchase-order - """ - - object_name = "purchase_orders" - - -class Software(FreshserviceStream): - """ - API docs: https://api.freshservice.com/v2/#software - """ - - object_name = "applications" diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py deleted file mode 100644 index 5bb422d2b7f4..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest - - -@pytest.fixture -def test_config(): - return { - "domain_name": "test.freshservice.com", - "api_key": "test_api_key", - "start_date": "2021-05-07T00:00:00Z", - } diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py deleted file mode 100644 index f540ab794e69..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_source.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest import mock -from unittest.mock import MagicMock - -import responses -from requests.exceptions import HTTPError -from source_freshservice.source import SourceFreshservice - - -def setup_responses(): - responses.add( - responses.GET, - "https://test.freshservice.com/api/v2/tickets", - json={"per_page": 30, "order_by": "updated_at", "order_type": "asc", "updated_since": "2021-05-07T00:00:00Z"}, - ) - - -@mock.patch("source_freshservice.streams.Tickets.read_records", return_value=iter([1])) -def test_check_connection_success(mocker): - source = SourceFreshservice() - logger_mock = MagicMock() - test_config = MagicMock() - assert source.check_connection(logger_mock, test_config) == (True, None) - - -def test_check_connection_failure(mocker, test_config): - source = SourceFreshservice() - logger_mock = MagicMock() - response = source.check_connection(logger_mock, test_config) - - assert response[0] is False - assert type(response[1]) == HTTPError - - -def test_stream_count(mocker): - source = SourceFreshservice() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 13 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py deleted file mode 100644 index 01f17698a14f..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/unit_tests/test_stream.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from source_freshservice.streams import FreshserviceStream, Tickets - - -@pytest.fixture -def patch_base_class(mocker): - mocker.patch.object(FreshserviceStream, "path", "v0/example_endpoint") - # mocker.patch.object(FreshserviceStream, "domain_name", "https://example.example") - - -def test_request_params(patch_base_class): - stream = FreshserviceStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"per_page": 30} - assert stream.request_params(**inputs) == expected_params - - -class Fake(object): - pass - - -def test_next_page_token(patch_base_class): - stream = FreshserviceStream() - response = Fake() - response.links = {'next': {'url': 'https://dummy?page=3'}} - inputs = {'response': response} - expected_token = {'page': '3'} - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_when_no_next_page(patch_base_class): - stream = FreshserviceStream() - response = Fake() - response.links = {} - inputs = {'response': response} - assert stream.next_page_token(**inputs) is None - - -def test_parse_response(patch_base_class, requests_mock): - stream = Tickets() - requests_mock.get('https://dummy', json={ - "tickets": [ - { - "subject": "test", - "group_id": 18000074057 - } - ] - }) - res = requests.get('https://dummy') - inputs = {'response': res} - expected_parsed = {"subject": "test", "group_id": 18000074057} - assert next(stream.parse_response(**inputs)) == expected_parsed diff --git a/airbyte-integrations/connectors/source-fullstory/metadata.yaml b/airbyte-integrations/connectors/source-fullstory/metadata.yaml index b5f6da4626a0..a645935c33eb 100644 --- a/airbyte-integrations/connectors/source-fullstory/metadata.yaml +++ b/airbyte-integrations/connectors/source-fullstory/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python -metadataSpecVersion: '1.0' + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-fullstory/requirements.txt b/airbyte-integrations/connectors/source-fullstory/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-fullstory/requirements.txt +++ b/airbyte-integrations/connectors/source-fullstory/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-fullstory/setup.py b/airbyte-integrations/connectors/source-fullstory/setup.py index 1b74c555ffab..99bc576b124a 100644 --- a/airbyte-integrations/connectors/source-fullstory/setup.py +++ b/airbyte-integrations/connectors/source-fullstory/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-gainsight-px/.dockerignore b/airbyte-integrations/connectors/source-gainsight-px/.dockerignore new file mode 100644 index 000000000000..8f1f9150d35a --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_gainsight_px +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-gainsight-px/Dockerfile b/airbyte-integrations/connectors/source-gainsight-px/Dockerfile new file mode 100644 index 000000000000..b00e69c3e089 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_gainsight_px ./source_gainsight_px + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-gainsight-px diff --git a/airbyte-integrations/connectors/source-gainsight-px/README.md b/airbyte-integrations/connectors/source-gainsight-px/README.md new file mode 100644 index 000000000000..7373411b868a --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/README.md @@ -0,0 +1,82 @@ +# Gainsight Px Source + +This is the repository for the Gainsight Px configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/gainsight-px). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-gainsight-px:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/gainsight-px) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gainsight_px/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source gainsight-px test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-gainsight-px:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-gainsight-px:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-gainsight-px:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gainsight-px:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gainsight-px:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gainsight-px:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-gainsight-px:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-gainsight-px:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-gainsight-px/__init__.py b/airbyte-integrations/connectors/source-gainsight-px/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml new file mode 100644 index 000000000000..bd7e46aae2c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml @@ -0,0 +1,51 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-gainsight-px:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_gainsight_px/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: accounts + bypass_reason: "Sandbox account cannot seed the stream" + - name: articles + bypass_reason: "Sandbox account cannot seed the stream" + - name: feature + bypass_reason: "Sandbox account cannot seed the stream" + - name: segments + bypass_reason: "Sandbox account cannot seed the stream" + - name: kcbot + bypass_reason: "Sandbox account cannot seed the stream" + - name: users + bypass_reason: "Sandbox account cannot seed the stream" +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gainsight-px/build.gradle b/airbyte-integrations/connectors/source-gainsight-px/build.gradle new file mode 100644 index 000000000000..735ea0dbde01 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_gainsight_px' +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/icon.svg b/airbyte-integrations/connectors/source-gainsight-px/icon.svg new file mode 100644 index 000000000000..e46c1d3780f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/icon.svg @@ -0,0 +1 @@ +Asset 2 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/__init__.py b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..c3b26b458302 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/configured_catalog.json @@ -0,0 +1,76 @@ +{ + "streams": [ + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "articles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "admin_attributes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "feature", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "segments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "kcbot", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "user_attributes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/invalid_config.json new file mode 100644 index 000000000000..6016942564e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "wrong-api-key" +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/sample_config.json new file mode 100644 index 000000000000..3fc2c76d0430 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "25yyyxxx-abcd-qwer-asdf-cvbnvbnmpoiu" +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/main.py b/airbyte-integrations/connectors/source-gainsight-px/main.py new file mode 100644 index 000000000000..5ae4980cd0e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gainsight_px import SourceGainsightPx + +if __name__ == "__main__": + source = SourceGainsightPx() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml b/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml new file mode 100644 index 000000000000..2d340cc0adaf --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/metadata.yaml @@ -0,0 +1,28 @@ +data: + allowedHosts: + hosts: + - api.aptrinsic.com/v1 + registries: + cloud: + enabled: true + oss: + enabled: true + connectorSubtype: api + connectorType: source + definitionId: 0da3b186-8879-4e94-8738-55b48762f1e8 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-gainsight-px + githubIssueLabel: source-gainsight-px + icon: gainsight-px.svg + license: MIT + name: Gainsight Px + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/gainsight-px + tags: + - language:low-code + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gainsight-px/requirements.txt b/airbyte-integrations/connectors/source-gainsight-px/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/source-gainsight-px/setup.py b/airbyte-integrations/connectors/source-gainsight-px/setup.py new file mode 100644 index 000000000000..3ba161a38e22 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + +setup( + name="source_gainsight_px", + description="Source implementation for Gainsight Px.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/__init__.py b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/__init__.py new file mode 100644 index 000000000000..c0d02e94d047 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceGainsightPx + +__all__ = ["SourceGainsightPx"] diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/manifest.yaml b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/manifest.yaml new file mode 100644 index 000000000000..c940f484eb36 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/manifest.yaml @@ -0,0 +1,149 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + custom_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractorPath }}"] + + requester: + type: HttpRequester + url_base: "https://api.aptrinsic.com/v1/" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "X-APTRINSIC-API-KEY" + api_token: "{{ config['api_key'] }}" + + retriever: + type: SimpleRetriever + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records[-1]['scrollId'] }}" + page_size: 5 + page_token_option: + type: "RequestPath" + field_name: "scrollId" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/selector" + + custom_retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/custom_selector" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + accounts_stream: + $parameters: + path: "accounts" + extractorPath: "accounts" + type: DeclarativeStream + retriever: + $ref: "#/definitions/custom_retriever" + name: "accounts" + primary_key: "id" + + admin_attributes_stream: + $parameters: + path: "admin/model/account/attributes" + $ref: "#/definitions/base_stream" + name: "admin_attributes" + primary_key: "id" + + articles_stream: + $parameters: + path: "articles" + extractorPath: "articleExternalViewList" + type: DeclarativeStream + retriever: + $ref: "#/definitions/custom_retriever" + name: "articles" + primary_key: "id" + + feature_stream: + $parameters: + path: "feature" + extractorPath: "features" + type: DeclarativeStream + retriever: + $ref: "#/definitions/custom_retriever" + name: "feature" + primary_key: "id" + + segments_stream: + $parameters: + path: "segment" + extractorPath: "segments" + type: DeclarativeStream + retriever: + $ref: "#/definitions/custom_retriever" + name: "segments" + + kcbot_stream: + $parameters: + path: "kcbot" + extractorPath: "kcList" + type: DeclarativeStream + retriever: + $ref: "#/definitions/custom_retriever" + name: "kcbot" + primary_key: "id" + + user_attributes_stream: + $parameters: + path: "admin/model/user/attributes" + $ref: "#/definitions/base_stream" + name: "user_attributes" + primary_key: "id" + + users_stream: + $parameters: + path: "users" + extractorPath: "users" + type: DeclarativeStream + retriever: + $ref: "#/definitions/custom_retriever" + name: "users" + primary_key: "id" + +streams: + - "#/definitions/accounts_stream" + - "#/definitions/admin_attributes_stream" + - "#/definitions/articles_stream" + - "#/definitions/feature_stream" + - "#/definitions/segments_stream" + - "#/definitions/kcbot_stream" + - "#/definitions/user_attributes_stream" + - "#/definitions/users_stream" + +check: + type: CheckStream + stream_names: + - "accounts" + - "admin_attributes" + - "articles" + - "feature" + - "segments" + - "kcbot" + - "user_attributes" + - "users" diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/accounts.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/accounts.json new file mode 100644 index 000000000000..a1a30710c6f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/accounts.json @@ -0,0 +1,71 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Accounts Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "trackedSubscriptionId": { + "type": ["null", "string"] + }, + "sfdcId": { + "type": ["null", "string"] + }, + "lastSeenDate": { + "type": ["null", "string"] + }, + "dunsNumber": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "numberOfEmployees": { + "type": ["null", "string"] + }, + "sicCode": { + "type": ["null", "string"] + }, + "website": { + "type": ["null", "string"] + }, + "naicsCode": { + "type": ["null", "string"] + }, + "plan": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "numberOfUsers": { + "type": ["null", "string"] + }, + "propertyKeys": { + "type": ["null", "array"], + "items": { + "location": { + "type": ["null", "string"] + } + } + }, + "createDate": { + "type": ["null", "string"] + }, + "lastModifiedDate": { + "type": ["null", "string"] + }, + "customAttributes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "parentGroupId": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/admin_attributes.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/admin_attributes.json new file mode 100644 index 000000000000..ac6e73995f9d --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/admin_attributes.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "User Attributes Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "apiName": { + "type": ["null", "string"] + }, + "internalName": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "origin": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "number"] + }, + "modifiedDate": { + "type": ["null", "number"] + }, + "defaultValue": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/articles.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/articles.json new file mode 100644 index 000000000000..1143e451f009 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/articles.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Articles Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "modifiedBy": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "articleKCs": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "kcId": { + "type": ["null", "string"] + }, + "kcName": { + "type": ["null", "string"] + } + } + } + }, + "articleName": { + "type": ["null", "string"] + }, + "author": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "number"] + }, + "modifiedDate": { + "type": ["null", "number"] + }, + "productId": { + "type": ["null", "string"] + }, + "productName": { + "type": ["null", "string"] + }, + "releaseDate": { + "type": ["null", "string"] + }, + "viewType": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/feature.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/feature.json new file mode 100644 index 000000000000..064d1979189a --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/feature.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Feature Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"], + "enum": ["FEATURE", "MODULE"] + }, + "parentFeatureId": { + "type": ["null", "string"] + }, + "propertyKey": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"], + "enum": ["ACTIVATED", "DELETED"] + }, + "featureLabels": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/kcbot.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/kcbot.json new file mode 100644 index 000000000000..13e357d4d841 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/kcbot.json @@ -0,0 +1,65 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "KCBot Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "createdBy": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "createdDate": { + "type": ["null", "number"] + }, + "description": { + "type": ["null", "string"] + }, + "environments": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "id": { + "type": ["null", "string"] + }, + "modifiedBy": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "modifiedDate": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "number"] + }, + "productId": { + "type": ["null", "string"] + }, + "productName": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/segments.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/segments.json new file mode 100644 index 000000000000..1382369c2a86 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/segments.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Segment Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/user_attributes.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/user_attributes.json new file mode 100644 index 000000000000..ac6e73995f9d --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/user_attributes.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "User Attributes Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "apiName": { + "type": ["null", "string"] + }, + "internalName": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "origin": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "number"] + }, + "modifiedDate": { + "type": ["null", "number"] + }, + "defaultValue": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/users.json b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/users.json new file mode 100644 index 000000000000..e2337f049ba4 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/schemas/users.json @@ -0,0 +1,175 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Users Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "aptrinsicId": { + "type": ["null", "string"] + }, + "identifyId": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"], + "enum": ["LEAD", "USER", "VISITOR", "EMPTY_USER_TYPE"] + }, + "gender": { + "type": ["null", "string"], + "enum": ["MALE", "FEMALE", "OTHER", "EMPTY_GENDER"] + }, + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "lastSeenDate": { + "type": ["null", "integer"], + "format": "int64" + }, + "signUpDate": { + "type": ["null", "integer"], + "format": "int64" + }, + "firstVisitDate": { + "type": ["null", "integer"], + "format": "int64" + }, + "title": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "integer"], + "format": "int64" + }, + "role": { + "type": ["null", "string"] + }, + "subscriptionId": { + "type": ["null", "string"] + }, + "accountId": { + "type": ["null", "string"] + }, + "numberOfVisits": { + "type": ["null", "integer"], + "format": "int32" + }, + "location": { + "type": ["null", "string"] + }, + "propertyKeys": { + "type": ["null", "array"], + "description": "Aptrinsic Tag Key, at least one is required", + "items": { + "type": ["null", "string"] + }, + "examples": [["AP-XXXXXXXXXX-2"]] + }, + "createDate": { + "type": ["null", "integer"], + "format": "int64" + }, + "lastModifiedDate": { + "type": ["null", "integer"], + "format": "int64" + }, + "globalUnsubscribe": { + "type": ["null", "boolean"] + }, + "sfdcContactId": { + "type": ["null", "string"] + }, + "lastVisitedUserAgentData": { + "type": ["null", "array"], + "items": { + "propertyKey": { + "type": ["null", "string"] + }, + "userAgent": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "rawUserAgent": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "platformType": { + "type": ["null", "string"] + }, + "platformVersion": { + "type": ["null", "string"] + }, + "browserType": { + "type": ["null", "string"] + }, + "browserVersion": { + "type": ["null", "string"] + } + } + } + } + }, + "id": { + "type": ["null", "string"], + "description": "Synonym for identifyId, output only, not filterable" + }, + "lastInferredLocation": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "countryName": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "stateName": { + "type": ["null", "string"] + }, + "stateCode": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "street": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "continent": { + "type": ["null", "string"] + }, + "regionName": { + "type": ["null", "string"] + }, + "timeZone": { + "type": ["null", "string"] + }, + "coordinates": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/source.py b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/source.py new file mode 100644 index 000000000000..0ea94fe90b48 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceGainsightPx(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/spec.yaml b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/spec.yaml new file mode 100644 index 000000000000..a78bb4fef492 --- /dev/null +++ b/airbyte-integrations/connectors/source-gainsight-px/source_gainsight_px/spec.yaml @@ -0,0 +1,14 @@ +documentation_url: https://docs.airbyte.com/integrations/sources/gainsight-px +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Gainsight Px Spec + type: object + additionalProperties: true + required: + - api_key + properties: + api_key: + title: API Key + type: string + description: The Aptrinsic API Key which is recieved from the dashboard settings (ref - https://app.aptrinsic.com/settings/api-keys) + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-gcs/Dockerfile b/airbyte-integrations/connectors/source-gcs/Dockerfile index 94c99b4ee907..0d9dda5d897c 100644 --- a/airbyte-integrations/connectors/source-gcs/Dockerfile +++ b/airbyte-integrations/connectors/source-gcs/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index 1d6b209b55f6..927b6b006180 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -1,20 +1,24 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-gcs + documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs icon: gcs.svg - license: MIT + license: ELv2 name: GCS - releaseStage: alpha registries: cloud: enabled: true oss: enabled: true - documentationUrl: https://docs.airbyte.com/integrations/sources/gcs + releaseStage: alpha + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gcs/requirements.txt b/airbyte-integrations/connectors/source-gcs/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-gcs/requirements.txt +++ b/airbyte-integrations/connectors/source-gcs/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index 8870475fe637..73669bd2ee21 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "google-cloud-storage==2.5.0", "pandas==1.5.3"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.2", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml index f98c7269ea82..6b042e975071 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml @@ -11,7 +11,7 @@ connectionSpecification: gcs_bucket: type: string title: GCS bucket - description: GCS bucket name + description: GCS bucket name gcs_path: type: string title: GCS Path @@ -23,4 +23,3 @@ connectionSpecification: airbyte_secret: true examples: - '{ "type": "service_account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ... }' - diff --git a/airbyte-integrations/connectors/source-genesys/Dockerfile b/airbyte-integrations/connectors/source-genesys/Dockerfile index 39db0db35fd9..62b8144a1cea 100644 --- a/airbyte-integrations/connectors/source-genesys/Dockerfile +++ b/airbyte-integrations/connectors/source-genesys/Dockerfile @@ -34,5 +34,5 @@ COPY source_genesys ./source_genesys ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-genesys diff --git a/airbyte-integrations/connectors/source-genesys/metadata.yaml b/airbyte-integrations/connectors/source-genesys/metadata.yaml index 655e08fd2b7a..f6522beea79e 100644 --- a/airbyte-integrations/connectors/source-genesys/metadata.yaml +++ b/airbyte-integrations/connectors/source-genesys/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/genesys tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-genesys/requirements.txt b/airbyte-integrations/connectors/source-genesys/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-genesys/requirements.txt +++ b/airbyte-integrations/connectors/source-genesys/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-genesys/setup.py b/airbyte-integrations/connectors/source-genesys/setup.py index 356de3cee16e..1c118e316680 100644 --- a/airbyte-integrations/connectors/source-genesys/setup.py +++ b/airbyte-integrations/connectors/source-genesys/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py index 73ab86a9059d..30d0d1106c43 100644 --- a/airbyte-integrations/connectors/source-genesys/source_genesys/source.py +++ b/airbyte-integrations/connectors/source-genesys/source_genesys/source.py @@ -14,10 +14,16 @@ class GenesysStream(HttpStream, ABC): - url_base = "https://api.mypurecloud.com.au/api/v2/" page_size = 500 - def __init__(self, *args, **kwargs): + @property + def url_base(self): + if self._api_base_url is not None: + return self._api_base_url + "/api/v2/" + return None + + def __init__(self, api_base_url, *args, **kwargs): + self._api_base_url = api_base_url super().__init__(*args, **kwargs) def backoff_time(self, response: requests.Response) -> Optional[int]: @@ -254,21 +260,26 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: - GENESYS_TENANT_ENDPOINT_MAP: Dict = { - "Americas (US East)": "https://login.mypurecloud.com", - "Americas (US East 2)": "https://login.use2.us-gov-pure.cloud", - "Americas (US West)": "https://login.usw2.pure.cloud", - "Americas (Canada)": "https://login.cac1.pure.cloud", - "Americas (São Paulo)": "https://login.sae1.pure.cloud", - "EMEA (Frankfurt)": "https://login.mypurecloud.de", - "EMEA (Dublin)": "https://login.mypurecloud.ie", - "EMEA (London)": "https://login.euw2.pure.cloud", - "Asia Pacific (Mumbai)": "https://login.aps1.pure.cloud", - "Asia Pacific (Seoul)": "https://login.apne2.pure.cloud", - "Asia Pacific (Sydney)": "https://login.mypurecloud.com.au", + GENESYS_REGION_DOMAIN_MAP: Dict[str, str] = { + "Americas (US East)": "mypurecloud.com", + "Americas (US East 2)": "use2.us-gov-pure.cloud", + "Americas (US West)": "usw2.pure.cloud", + "Americas (Canada)": "cac1.pure.cloud", + "Americas (São Paulo)": "sae1.pure.cloud", + "EMEA (Frankfurt)": "mypurecloud.de", + "EMEA (Dublin)": "mypurecloud.ie", + "EMEA (London)": "euw2.pure.cloud", + "Asia Pacific (Mumbai)": "aps1.pure.cloud", + "Asia Pacific (Seoul)": "apne2.pure.cloud", + "Asia Pacific (Sydney)": "mypurecloud.com.au", + } + domain = GENESYS_REGION_DOMAIN_MAP.get(config["tenant_endpoint"]) + base_url = f"https://login.{domain}" + api_base_url = f"https://api.{domain}" + args = { + "api_base_url": api_base_url, + "authenticator": GenesysOAuthAuthenticator(base_url, config["client_id"], config["client_secret"]), } - base_url = GENESYS_TENANT_ENDPOINT_MAP.get(config["tenant_endpoint"]) - args = {"authenticator": GenesysOAuthAuthenticator(base_url, config["client_id"], config["client_secret"])} # response = self.get_connection_response(config) # response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py b/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py index 84e982fe4802..74600da0200e 100644 --- a/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-genesys/unit_tests/test_source.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock +import pytest from source_genesys.source import SourceGenesys @@ -21,3 +22,32 @@ def test_streams(mocker): streams = source.streams(config_mock) expected_streams_number = 16 assert len(streams) == expected_streams_number + + +@pytest.mark.parametrize( + ("tenant_endpoint", "url_base"), + [ + ("Americas (US East)", "https://api.mypurecloud.com/api/v2/"), + ("Americas (US East 2)", "https://api.use2.us-gov-pure.cloud/api/v2/"), + ("Americas (US West)", "https://api.usw2.pure.cloud/api/v2/"), + ("Americas (Canada)", "https://api.cac1.pure.cloud/api/v2/"), + ("Americas (São Paulo)", "https://api.sae1.pure.cloud/api/v2/"), + ("EMEA (Frankfurt)", "https://api.mypurecloud.de/api/v2/"), + ("EMEA (Dublin)", "https://api.mypurecloud.ie/api/v2/"), + ("EMEA (London)", "https://api.euw2.pure.cloud/api/v2/"), + ("Asia Pacific (Mumbai)", "https://api.aps1.pure.cloud/api/v2/"), + ("Asia Pacific (Seoul)", "https://api.apne2.pure.cloud/api/v2/"), + ("Asia Pacific (Sydney)", "https://api.mypurecloud.com.au/api/v2/"), + ], +) +def test_url_base(tenant_endpoint, url_base): + source = SourceGenesys() + config_mock = MagicMock() + config_mock.__getitem__.side_effect = lambda key: tenant_endpoint if key == "tenant_endpoint" else None + SourceGenesys.get_connection_response = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 16 + assert len(streams) == expected_streams_number + + for stream in streams: + assert stream.url_base == url_base diff --git a/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py index 3c16b087c95d..3d098b4aee14 100644 --- a/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-genesys/unit_tests/test_streams.py @@ -18,20 +18,20 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} expected_params = {"pageSize": 500} assert stream.request_params(**inputs) == expected_params def test_request_headers(patch_base_class): - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} assert len(stream.request_headers(**inputs)) == 0 def test_http_method(patch_base_class): - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") expected_method = "GET" assert stream.http_method == expected_method @@ -48,12 +48,18 @@ def test_http_method(patch_base_class): def test_should_retry(patch_base_class, http_status, should_retry): response_mock = MagicMock() response_mock.status_code = http_status - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") assert stream.should_retry(response_mock) == should_retry def test_backoff_time(patch_base_class): response_mock = MagicMock() - stream = GenesysStream() + stream = GenesysStream(api_base_url="https://dummy.url") expected_backoff_time = 1 assert stream.backoff_time(response_mock) == expected_backoff_time + + +def test_url_base(patch_base_class): + api_base_url = "https://dummy.url" + stream = GenesysStream(api_base_url=api_base_url) + assert stream.url_base == api_base_url + "/api/v2/" diff --git a/airbyte-integrations/connectors/source-getlago/metadata.yaml b/airbyte-integrations/connectors/source-getlago/metadata.yaml index a5fcf785302c..539d110e928a 100644 --- a/airbyte-integrations/connectors/source-getlago/metadata.yaml +++ b/airbyte-integrations/connectors/source-getlago/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-getlago/requirements.txt b/airbyte-integrations/connectors/source-getlago/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-getlago/requirements.txt +++ b/airbyte-integrations/connectors/source-getlago/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-getlago/setup.py b/airbyte-integrations/connectors/source-getlago/setup.py index bd576ce9ff5d..37d77842438d 100644 --- a/airbyte-integrations/connectors/source-getlago/setup.py +++ b/airbyte-integrations/connectors/source-getlago/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 34031d113a8c..14a7ee12b273 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.0.4 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json index d622a53fec63..c8bcb9af640c 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json @@ -33,7 +33,9 @@ "type": "STREAM", "stream": { "stream_state": { - "airbytehq/integration-test": { "master": { "created_at": "2121-06-30T10:04:41Z" } } + "airbytehq/integration-test": { + "master": { "created_at": "2121-06-30T10:04:41Z" } + } }, "stream_descriptor": { "name": "commits" } } diff --git a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl index 9ef6af54ab9d..b445361bfe9f 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl @@ -11,8 +11,8 @@ {"stream":"issue_labels","data":{"id":3295756566,"node_id":"MDU6TGFiZWwzMjk1NzU2NTY2","url":"https://api.github.com/repos/airbytehq/integration-test/labels/bug","name":"bug","color":"d73a4a","default":true,"description":"Something isn't working","repository":"airbytehq/integration-test"},"emitted_at":1677668750697} {"stream":"issue_milestones","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/milestones/1","html_url":"https://github.com/airbytehq/integration-test/milestone/1","labels_url":"https://api.github.com/repos/airbytehq/integration-test/milestones/1/labels","id":7097357,"node_id":"MI_kwDOF9hP9c4AbEwN","number":1,"title":"main","description":null,"creator":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"open_issues":3,"closed_issues":1,"state":"open","created_at":"2021-08-27T15:43:44Z","updated_at":"2021-08-27T16:02:49Z","due_on":null,"closed_at":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668751023} {"stream":"issue_reactions","data":{"node_id":"MDEzOklzc3VlUmVhY3Rpb24xMjcwNDg0NTY=","id":127048456,"content":"ROCKET","created_at":"2021-09-06T11:13:32Z","user":{"node_id":"MDQ6VXNlcjM0MTAzMTI1","id":34103125,"login":"yevhenii-ldv","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","html_url":"https://github.com/yevhenii-ldv","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","issue_number":11},"emitted_at":1677668751465} -{"stream":"issues","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/issues/14","repository_url":"https://api.github.com/repos/airbytehq/integration-test","labels_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/labels{/name}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/comments","events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/events","html_url":"https://github.com/airbytehq/integration-test/pull/14","id":1291262400,"node_id":"PR_kwDOF9hP9c46s2Qa","number":14,"title":"New PR from feature/branch_5","user":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"labels":[{"id":3984065862,"node_id":"LA_kwDOF9hP9c7teAVG","url":"https://api.github.com/repos/airbytehq/integration-test/labels/labeler","name":"labeler","color":"ededed","default":false,"description":null}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":0,"created_at":"2022-07-01T11:05:28Z","updated_at":"2022-10-04T17:41:29Z","closed_at":null,"author_association":"MEMBER","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/14","html_url":"https://github.com/airbytehq/integration-test/pull/14","diff_url":"https://github.com/airbytehq/integration-test/pull/14.diff","patch_url":"https://github.com/airbytehq/integration-test/pull/14.patch","merged_at":null},"body":"Signed-off-by: Sergey Chvalyuk ","reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/timeline","performed_via_github_app":null,"state_reason":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668752186} -{"stream":"organizations","data":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","url":"https://api.github.com/orgs/airbytehq","repos_url":"https://api.github.com/orgs/airbytehq/repos","events_url":"https://api.github.com/orgs/airbytehq/events","hooks_url":"https://api.github.com/orgs/airbytehq/hooks","issues_url":"https://api.github.com/orgs/airbytehq/issues","members_url":"https://api.github.com/orgs/airbytehq/members{/member}","public_members_url":"https://api.github.com/orgs/airbytehq/public_members{/member}","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","description":"Simple & extensible open-source data integration","name":"Airbyte","company":null,"blog":"https://airbyte.io","location":"United States of America","email":"contact@airbyte.io","twitter_username":"AirbyteHQ","is_verified":true,"has_organization_projects":true,"has_repository_projects":true,"public_repos":34,"public_gists":0,"followers":199,"following":0,"html_url":"https://github.com/airbytehq","created_at":"2020-01-11T06:27:48Z","updated_at":"2023-02-06T21:57:02Z","type":"Organization","total_private_repos":43,"owned_private_repos":41,"private_gists":null,"disk_usage":null,"collaborators":null,"billing_email":null,"default_repository_permission":null,"members_can_create_repositories":true,"two_factor_requirement_enabled":null,"members_allowed_repository_creation_type":"all","members_can_create_public_repositories":true,"members_can_create_private_repositories":true,"members_can_create_internal_repositories":false,"members_can_create_pages":true,"members_can_fork_private_repositories":false,"web_commit_signoff_required":false,"members_can_create_public_pages":true,"members_can_create_private_pages":true,"plan":{"name":"team","space":976562499,"private_repos":999999,"filled_seats":125,"seats":135}},"emitted_at":1677668752546} +{"stream": "issues", "data": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/14", "repository_url": "https://api.github.com/repos/airbytehq/integration-test", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/labels{/name}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/comments", "events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/events", "html_url": "https://github.com/airbytehq/integration-test/pull/14", "id": 1291262400, "node_id": "PR_kwDOF9hP9c46s2Qa", "number": 14, "title": "New PR from feature/branch_5", "user": {"login": "grubberr", "id": 195743, "node_id": "MDQ6VXNlcjE5NTc0Mw==", "avatar_url": "https://avatars.githubusercontent.com/u/195743?v=4", "gravatar_id": "", "url": "https://api.github.com/users/grubberr", "html_url": "https://github.com/grubberr", "followers_url": "https://api.github.com/users/grubberr/followers", "following_url": "https://api.github.com/users/grubberr/following{/other_user}", "gists_url": "https://api.github.com/users/grubberr/gists{/gist_id}", "starred_url": "https://api.github.com/users/grubberr/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/grubberr/subscriptions", "organizations_url": "https://api.github.com/users/grubberr/orgs", "repos_url": "https://api.github.com/users/grubberr/repos", "events_url": "https://api.github.com/users/grubberr/events{/privacy}", "received_events_url": "https://api.github.com/users/grubberr/received_events", "type": "User", "site_admin": false}, "labels": [{"id": 3984065862, "node_id": "LA_kwDOF9hP9c7teAVG", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/labeler", "name": "labeler", "color": "ededed", "default": false, "description": null}], "state": "open", "locked": false, "assignee": null, "assignees": [], "milestone": null, "comments": 0, "created_at": "2022-07-01T11:05:28Z", "updated_at": "2022-10-04T17:41:29Z", "closed_at": null, "author_association": "FIRST_TIME_CONTRIBUTOR", "active_lock_reason": null, "draft": false, "pull_request": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/14", "html_url": "https://github.com/airbytehq/integration-test/pull/14", "diff_url": "https://github.com/airbytehq/integration-test/pull/14.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/14.patch", "merged_at": null}, "body": "Signed-off-by: Sergey Chvalyuk ", "reactions": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0}, "timeline_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/timeline", "performed_via_github_app": null, "state_reason": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1689868850497} +{"stream": "organizations", "data": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "url": "https://api.github.com/orgs/airbytehq", "repos_url": "https://api.github.com/orgs/airbytehq/repos", "events_url": "https://api.github.com/orgs/airbytehq/events", "hooks_url": "https://api.github.com/orgs/airbytehq/hooks", "issues_url": "https://api.github.com/orgs/airbytehq/issues", "members_url": "https://api.github.com/orgs/airbytehq/members{/member}", "public_members_url": "https://api.github.com/orgs/airbytehq/public_members{/member}", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "description": "Simple & extensible open-source data integration", "name": "Airbyte", "company": null, "blog": "https://airbyte.io", "location": "United States of America", "email": "contact@airbyte.io", "twitter_username": "AirbyteHQ", "is_verified": true, "has_organization_projects": true, "has_repository_projects": true, "public_repos": 46, "public_gists": 0, "followers": 267, "following": 0, "html_url": "https://github.com/airbytehq", "created_at": "2020-01-11T06:27:48Z", "updated_at": "2023-05-03T17:28:12Z", "archived_at": null, "type": "Organization", "total_private_repos": 57, "owned_private_repos": 55, "private_gists": null, "disk_usage": null, "collaborators": null, "billing_email": null, "default_repository_permission": null, "members_can_create_repositories": true, "two_factor_requirement_enabled": null, "members_allowed_repository_creation_type": "all", "members_can_create_public_repositories": true, "members_can_create_private_repositories": true, "members_can_create_internal_repositories": false, "members_can_create_pages": true, "members_can_fork_private_repositories": false, "web_commit_signoff_required": false, "members_can_create_public_pages": true, "members_can_create_private_pages": true, "plan": {"name": "team", "space": 976562499, "private_repos": 999999, "filled_seats": 116, "seats": 118}}, "emitted_at": 1689065014314} {"stream":"project_cards","data":{"url":"https://api.github.com/projects/columns/cards/77859890","project_url":"https://api.github.com/projects/13167124","id":77859890,"node_id":"PRC_lALOF9hP9c4AyOoUzgSkDDI","note":"note_1","archived":false,"creator":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"created_at":"2022-02-17T09:56:51Z","updated_at":"2022-02-17T09:56:51Z","column_url":"https://api.github.com/projects/columns/17807006","repository":"airbytehq/integration-test","project_id":13167124,"column_id":17807006},"emitted_at":1677668754200} {"stream":"project_columns","data":{"url":"https://api.github.com/projects/columns/17807092","project_url":"https://api.github.com/projects/13167124","cards_url":"https://api.github.com/projects/columns/17807092/cards","id":17807092,"node_id":"PC_lATOF9hP9c4AyOoUzgEPtvQ","name":"column_2","created_at":"2022-02-17T09:57:27Z","updated_at":"2022-02-17T09:57:27Z","repository":"airbytehq/integration-test","project_id":13167124},"emitted_at":1677668754456} {"stream":"projects","data":{"owner_url":"https://api.github.com/repos/airbytehq/integration-test","url":"https://api.github.com/projects/13167124","html_url":"https://github.com/airbytehq/integration-test/projects/3","columns_url":"https://api.github.com/projects/13167124/columns","id":13167124,"node_id":"PRO_kwLOF9hP9c4AyOoU","name":"project_3","body":null,"number":3,"state":"open","creator":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"created_at":"2021-08-27T15:43:57Z","updated_at":"2022-02-17T12:16:56Z","repository":"airbytehq/integration-test"},"emitted_at":1677668754468} @@ -21,7 +21,7 @@ {"stream":"pull_request_stats","data":{"node_id":"MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2","id":721435506,"number":5,"updated_at":"2021-08-27T15:53:14Z","changed_files":5,"deletions":0,"additions":5,"merged":false,"mergeable":"MERGEABLE","can_be_rebased":true,"maintainer_can_modify":false,"merge_state_status":"BLOCKED","comments":0,"commits":5,"review_comments":0,"merged_by":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668759962} {"stream": "pull_requests", "data": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5", "id": 721435506, "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "html_url": "https://github.com/airbytehq/integration-test/pull/5", "diff_url": "https://github.com/airbytehq/integration-test/pull/5.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/5.patch", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5", "number": 5, "state": "open", "locked": false, "title": "New PR from feature/branch_4", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "body": null, "created_at": "2021-08-27T15:43:40Z", "updated_at": "2021-08-27T15:53:14Z", "closed_at": null, "merged_at": null, "merge_commit_sha": "191309e3da8b36705156348ae73f4dca836533f9", "assignee": null, "assignees": [], "requested_reviewers": [], "requested_teams": [], "labels": [{"id": 3295756566, "node_id": "MDU6TGFiZWwzMjk1NzU2NTY2", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/bug", "name": "bug", "color": "d73a4a", "default": true, "description": "Something isn't working"}, {"id": 3300346197, "node_id": "MDU6TGFiZWwzMzAwMzQ2MTk3", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/critical", "name": "critical", "color": "ededed", "default": false, "description": null}], "milestone": null, "draft": false, "commits_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits", "review_comments_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments", "review_comment_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "head": {"label": "airbytehq:feature/branch_4", "ref": "feature/branch_4", "sha": "31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo_id": 400052213}, "base": {"label": "airbytehq:master", "ref": "master", "sha": "978753aeb56f7b49872279d1b491411a6235aa90", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments", "created_at": "2021-08-26T05:32:43Z", "updated_at": "2022-07-08T01:27:13Z", "pushed_at": "2023-05-03T16:40:56Z", "git_url": "git://github.com/airbytehq/integration-test.git", "ssh_url": "git@github.com:airbytehq/integration-test.git", "clone_url": "https://github.com/airbytehq/integration-test.git", "svn_url": "https://github.com/airbytehq/integration-test", "homepage": null, "size": 11, "stargazers_count": 4, "watchers_count": 4, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 10, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 2, "open_issues": 10, "watchers": 4, "default_branch": "master"}, "repo_id": null}, "_links": {"self": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5"}, "html": {"href": "https://github.com/airbytehq/integration-test/pull/5"}, "issue": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5"}, "comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments"}, "review_comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments"}, "review_comment": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}"}, "commits": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits"}, "statuses": {"href": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083"}}, "author_association": "CONTRIBUTOR", "auto_merge": null, "active_lock_reason": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1685698519242} {"stream":"releases","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586","assets_url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586/assets","upload_url":"https://uploads.github.com/repos/airbytehq/integration-test/releases/48581586/assets{?name,label}","html_url":"https://github.com/airbytehq/integration-test/releases/tag/dev-0.9","id":48581586,"author":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"node_id":"MDc6UmVsZWFzZTQ4NTgxNTg2","tag_name":"dev-0.9","target_commitish":"master","name":"9 global release","draft":false,"prerelease":false,"created_at":"2021-08-27T07:03:09Z","published_at":"2021-08-27T15:43:53Z","assets":[],"tarball_url":"https://api.github.com/repos/airbytehq/integration-test/tarball/dev-0.9","zipball_url":"https://api.github.com/repos/airbytehq/integration-test/zipball/dev-0.9","body":"","repository":"airbytehq/integration-test"},"emitted_at":1677668760424} -{"stream":"repositories","data":{"id":283046497,"node_id":"MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=","name":"airbyte","full_name":"airbytehq/airbyte","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/airbyte","description":"Data integration platform for ELT pipelines from APIs, databases & files to warehouses & lakes.","fork":false,"url":"https://api.github.com/repos/airbytehq/airbyte","forks_url":"https://api.github.com/repos/airbytehq/airbyte/forks","keys_url":"https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/airbyte/teams","hooks_url":"https://api.github.com/repos/airbytehq/airbyte/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/airbyte/events","assignees_url":"https://api.github.com/repos/airbytehq/airbyte/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/airbyte/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/airbyte/tags","blobs_url":"https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/airbyte/languages","stargazers_url":"https://api.github.com/repos/airbytehq/airbyte/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/airbyte/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/airbyte/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/airbyte/subscription","commits_url":"https://api.github.com/repos/airbytehq/airbyte/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/airbyte/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/airbyte/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/airbyte/merges","archive_url":"https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/airbyte/downloads","issues_url":"https://api.github.com/repos/airbytehq/airbyte/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/airbyte/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/airbyte/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/airbyte/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/airbyte/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/airbyte/deployments","created_at":"2020-07-27T23:55:54Z","updated_at":"2023-03-01T10:52:23Z","pushed_at":"2023-03-01T10:45:54Z","git_url":"git://github.com/airbytehq/airbyte.git","ssh_url":"git@github.com:airbytehq/airbyte.git","clone_url":"https://github.com/airbytehq/airbyte.git","svn_url":"https://github.com/airbytehq/airbyte","homepage":"https://airbyte.com","size":1798499,"stargazers_count":9816,"watchers_count":9816,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":true,"has_discussions":false,"forks_count":2457,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":4426,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"allow_forking":true,"is_template":false,"web_commit_signoff_required":false,"topics":["airbyte","bigquery","change-data-capture","data","data-analysis","data-collection","data-engineering","data-ingestion","data-integration","elt","etl","java","pipeline","python","redshift","snowflake"],"visibility":"public","forks":2457,"open_issues":4426,"watchers":9816,"default_branch":"master","permissions":{"admin":true,"maintain":true,"push":true,"triage":true,"pull":true},"security_and_analysis":{"secret_scanning":{"status":"disabled"},"secret_scanning_push_protection":{"status":"disabled"}},"organization":"airbytehq"},"emitted_at":1677668763835} +{"stream": "repositories", "data": {"id": 283046497, "node_id": "MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=", "name": "airbyte", "full_name": "airbytehq/airbyte", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/airbyte", "description": "Data integration platform for ELT pipelines from APIs, databases & files to warehouses & lakes.", "fork": false, "url": "https://api.github.com/repos/airbytehq/airbyte", "forks_url": "https://api.github.com/repos/airbytehq/airbyte/forks", "keys_url": "https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/airbyte/teams", "hooks_url": "https://api.github.com/repos/airbytehq/airbyte/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/airbyte/events", "assignees_url": "https://api.github.com/repos/airbytehq/airbyte/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/airbyte/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/airbyte/tags", "blobs_url": "https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/airbyte/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/airbyte/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/airbyte/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/airbyte/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/airbyte/subscription", "commits_url": "https://api.github.com/repos/airbytehq/airbyte/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/airbyte/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/airbyte/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/airbyte/merges", "archive_url": "https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/airbyte/downloads", "issues_url": "https://api.github.com/repos/airbytehq/airbyte/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/airbyte/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/airbyte/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/airbyte/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/airbyte/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/airbyte/deployments", "created_at": "2020-07-27T23:55:54Z", "updated_at": "2023-07-11T04:02:27Z", "pushed_at": "2023-07-11T08:47:50Z", "git_url": "git://github.com/airbytehq/airbyte.git", "ssh_url": "git@github.com:airbytehq/airbyte.git", "clone_url": "https://github.com/airbytehq/airbyte.git", "svn_url": "https://github.com/airbytehq/airbyte", "homepage": "https://airbyte.com", "size": 3671239, "stargazers_count": 11137, "watchers_count": 11137, "language": "Python", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": true, "has_discussions": true, "forks_count": 2881, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 4723, "license": {"key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA="}, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": ["airbyte", "bigquery", "change-data-capture", "data", "data-analysis", "data-collection", "data-engineering", "data-ingestion", "data-integration", "elt", "etl", "java", "pipeline", "python", "redshift", "snowflake"], "visibility": "public", "forks": 2881, "open_issues": 4723, "watchers": 11137, "default_branch": "master", "permissions": {"admin": true, "maintain": true, "push": true, "triage": true, "pull": true}, "security_and_analysis": {"secret_scanning": {"status": "disabled"}, "secret_scanning_push_protection": {"status": "disabled"}, "dependabot_security_updates": {"status": "enabled"}}, "organization": "airbytehq"}, "emitted_at": 1689065865611} {"stream":"review_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726","pull_request_review_id":742633128,"id":699253726,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDY5OTI1MzcyNg==","diff_hunk":"@@ -0,0 +1 @@\n+text_for_file_","path":"github_sources/file_1.txt","position":1,"original_position":1,"commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","original_commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"body":"Good point","created_at":"2021-08-31T12:01:15Z","updated_at":"2021-08-31T12:01:15Z","html_url":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726","pull_request_url":"https://api.github.com/repos/airbytehq/integration-test/pulls/4","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726"},"html":{"href":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726"},"pull_request":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/4"}},"reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726/reactions","total_count":1,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"start_line":null,"original_start_line":null,"start_side":null,"line":1,"original_line":1,"side":"RIGHT","subject_type": "line","repository":"airbytehq/integration-test"},"emitted_at":1677668764426} {"stream":"reviews","data":{"node_id":"MDE3OlB1bGxSZXF1ZXN0UmV2aWV3NzQwNjU5Nzk4","id":740659798,"body":"Review commit for branch feature/branch_4","state":"COMMENTED","html_url":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798","author_association":"CONTRIBUTOR","submitted_at":"2021-08-27T15:43:42Z","created_at":"2021-08-27T15:43:42Z","updated_at":"2021-08-27T15:43:42Z","user":{"node_id":"MDQ6VXNlcjc0MzkwMQ==","id":743901,"login":"gaart","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","html_url":"https://github.com/gaart","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","pull_request_url":"https://github.com/airbytehq/integration-test/pull/5","commit_id":"31a3e3f19fefce60fba6bfc69dd2b3fb5195a083","_links":{"html":{"href":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798"},"pull_request":{"href":"https://github.com/airbytehq/integration-test/pull/5"}}},"emitted_at":1677668764954} {"stream":"stargazers","data":{"starred_at":"2021-08-27T16:23:34Z","user":{"login":"VasylLazebnyk","id":68591643,"node_id":"MDQ6VXNlcjY4NTkxNjQz","avatar_url":"https://avatars.githubusercontent.com/u/68591643?v=4","gravatar_id":"","url":"https://api.github.com/users/VasylLazebnyk","html_url":"https://github.com/VasylLazebnyk","followers_url":"https://api.github.com/users/VasylLazebnyk/followers","following_url":"https://api.github.com/users/VasylLazebnyk/following{/other_user}","gists_url":"https://api.github.com/users/VasylLazebnyk/gists{/gist_id}","starred_url":"https://api.github.com/users/VasylLazebnyk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VasylLazebnyk/subscriptions","organizations_url":"https://api.github.com/users/VasylLazebnyk/orgs","repos_url":"https://api.github.com/users/VasylLazebnyk/repos","events_url":"https://api.github.com/users/VasylLazebnyk/events{/privacy}","received_events_url":"https://api.github.com/users/VasylLazebnyk/received_events","type":"User","site_admin":false},"repository":"airbytehq/integration-test","user_id":68591643},"emitted_at":1677668765231} @@ -30,6 +30,6 @@ {"stream":"users","data":{"login":"AirbyteEricksson","id":101604444,"node_id":"U_kgDOBg5cXA","avatar_url":"https://avatars.githubusercontent.com/u/101604444?v=4","gravatar_id":"","url":"https://api.github.com/users/AirbyteEricksson","html_url":"https://github.com/AirbyteEricksson","followers_url":"https://api.github.com/users/AirbyteEricksson/followers","following_url":"https://api.github.com/users/AirbyteEricksson/following{/other_user}","gists_url":"https://api.github.com/users/AirbyteEricksson/gists{/gist_id}","starred_url":"https://api.github.com/users/AirbyteEricksson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AirbyteEricksson/subscriptions","organizations_url":"https://api.github.com/users/AirbyteEricksson/orgs","repos_url":"https://api.github.com/users/AirbyteEricksson/repos","events_url":"https://api.github.com/users/AirbyteEricksson/events{/privacy}","received_events_url":"https://api.github.com/users/AirbyteEricksson/received_events","type":"User","site_admin":false,"organization":"airbytehq"},"emitted_at":1677668766142} {"stream":"workflows","data":{"id":22952989,"node_id":"W_kwDOF9hP9c4BXjwd","name":"Pull Request Labeler","path":".github/workflows/labeler.yml","state":"active","created_at":"2022-03-30T21:30:37.000+02:00","updated_at":"2022-03-30T21:30:37.000+02:00","url":"https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989","html_url":"https://github.com/airbytehq/integration-test/blob/master/.github/workflows/labeler.yml","badge_url":"https://github.com/airbytehq/integration-test/workflows/Pull%20Request%20Labeler/badge.svg","repository":"airbytehq/integration-test"},"emitted_at":1677668766580} {"stream":"workflow_runs","data":{"id":3184250176,"name":"Pull Request Labeler","node_id":"WFR_kwLOF9hP9c69y81A","head_branch":"feature/branch_5","head_sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","path":".github/workflows/labeler.yml","display_title":"New PR from feature/branch_5","run_number":3,"event":"pull_request_target","status":"completed","conclusion":"success","workflow_id":22952989,"check_suite_id":8611635614,"check_suite_node_id":"CS_kwDOF9hP9c8AAAACAUshng","url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176","html_url":"https://github.com/airbytehq/integration-test/actions/runs/3184250176","pull_requests":[{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/14","id":984835098,"number":14,"head":{"ref":"feature/branch_5","sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","repo":{"id":400052213,"url":"https://api.github.com/repos/airbytehq/integration-test","name":"integration-test"}},"base":{"ref":"master","sha":"a12c9379604f7b32e54e5459122aa48473f806ee","repo":{"id":400052213,"url":"https://api.github.com/repos/airbytehq/integration-test","name":"integration-test"}}}],"created_at":"2022-10-04T17:41:18Z","updated_at":"2022-10-04T17:41:32Z","actor":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"run_attempt":1,"referenced_workflows":[],"run_started_at":"2022-10-04T17:41:18Z","triggering_actor":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"jobs_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/jobs","logs_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/logs","check_suite_url":"https://api.github.com/repos/airbytehq/integration-test/check-suites/8611635614","artifacts_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/artifacts","cancel_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/cancel","rerun_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/rerun","previous_attempt_url":null,"workflow_url":"https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989","head_commit":{"id":"f71e5f6894578148d52b487dff07e55804fd9cfd","tree_id":"bb78ec62be8c5c640010e7c897f40932ce59e725","message":"file_5.txt updated\n\nSigned-off-by: Sergey Chvalyuk ","timestamp":"2022-10-04T17:41:08Z","author":{"name":"Sergey Chvalyuk","email":"grubberr@gmail.com"},"committer":{"name":"Sergey Chvalyuk","email":"grubberr@gmail.com"}},"repository":{"id":400052213,"node_id":"MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=","name":"integration-test","full_name":"airbytehq/integration-test","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/integration-test","description":"Used for integration testing the Github source connector","fork":false,"url":"https://api.github.com/repos/airbytehq/integration-test","forks_url":"https://api.github.com/repos/airbytehq/integration-test/forks","keys_url":"https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/integration-test/teams","hooks_url":"https://api.github.com/repos/airbytehq/integration-test/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/integration-test/events","assignees_url":"https://api.github.com/repos/airbytehq/integration-test/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/integration-test/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/integration-test/tags","blobs_url":"https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/integration-test/languages","stargazers_url":"https://api.github.com/repos/airbytehq/integration-test/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/integration-test/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/integration-test/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/integration-test/subscription","commits_url":"https://api.github.com/repos/airbytehq/integration-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/integration-test/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/integration-test/merges","archive_url":"https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/integration-test/downloads","issues_url":"https://api.github.com/repos/airbytehq/integration-test/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/integration-test/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/integration-test/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/integration-test/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/integration-test/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/integration-test/deployments"},"head_repository":{"id":400052213,"node_id":"MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=","name":"integration-test","full_name":"airbytehq/integration-test","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/integration-test","description":"Used for integration testing the Github source connector","fork":false,"url":"https://api.github.com/repos/airbytehq/integration-test","forks_url":"https://api.github.com/repos/airbytehq/integration-test/forks","keys_url":"https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/integration-test/teams","hooks_url":"https://api.github.com/repos/airbytehq/integration-test/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/integration-test/events","assignees_url":"https://api.github.com/repos/airbytehq/integration-test/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/integration-test/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/integration-test/tags","blobs_url":"https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/integration-test/languages","stargazers_url":"https://api.github.com/repos/airbytehq/integration-test/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/integration-test/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/integration-test/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/integration-test/subscription","commits_url":"https://api.github.com/repos/airbytehq/integration-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/integration-test/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/integration-test/merges","archive_url":"https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/integration-test/downloads","issues_url":"https://api.github.com/repos/airbytehq/integration-test/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/integration-test/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/integration-test/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/integration-test/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/integration-test/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/integration-test/deployments"}},"emitted_at":1677668766993} -{"stream":"workflow_jobs","data":{"id":8705992587,"run_id":3184250176,"workflow_name":"Pull Request Labeler","head_branch":"feature/branch_5","run_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176","run_attempt":1,"node_id":"CR_kwDOF9hP9c8AAAACBurniw","head_sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","url":"https://api.github.com/repos/airbytehq/integration-test/actions/jobs/8705992587","html_url":"https://github.com/airbytehq/integration-test/actions/runs/3184250176/jobs/5192436167","status":"completed","conclusion":"success","created_at":"2022-10-04T17:41:20Z","started_at":"2022-10-04T17:41:27Z","completed_at":"2022-10-04T17:41:30Z","name":"triage","steps":[{"name":"Set up job","status":"completed","conclusion":"success","number":1,"started_at":"2022-10-04T10:41:26.000-07:00","completed_at":"2022-10-04T10:41:27.000-07:00"},{"name":"Run actions/labeler@v3","status":"completed","conclusion":"success","number":2,"started_at":"2022-10-04T10:41:27.000-07:00","completed_at":"2022-10-04T10:41:29.000-07:00"},{"name":"Complete job","status":"completed","conclusion":"success","number":3,"started_at":"2022-10-04T10:41:29.000-07:00","completed_at":"2022-10-04T10:41:29.000-07:00"}],"check_run_url":"https://api.github.com/repos/airbytehq/integration-test/check-runs/8705992587","labels":["ubuntu-latest"],"runner_id":1,"runner_name":"Hosted Agent","runner_group_id":2,"runner_group_name":"GitHub Actions","repository":"airbytehq/integration-test"},"emitted_at":1677668767830} +{"stream":"workflow_jobs","data":{"id": 8705992587, "run_id": 3184250176, "workflow_name": "Pull Request Labeler", "head_branch": "feature/branch_5", "run_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176", "run_attempt": 1, "node_id": "CR_kwDOF9hP9c8AAAACBurniw", "head_sha": "f71e5f6894578148d52b487dff07e55804fd9cfd", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/jobs/8705992587", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/3184250176/job/8705992587", "status": "completed", "conclusion": "success", "created_at": "2022-10-04T17:41:20Z", "started_at": "2022-10-04T17:41:27Z", "completed_at": "2022-10-04T17:41:30Z", "name": "triage", "steps": [{"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1, "started_at": "2022-10-04T20:41:26.000+03:00", "completed_at": "2022-10-04T20:41:27.000+03:00"}, {"name": "Run actions/labeler@v3", "status": "completed", "conclusion": "success", "number": 2, "started_at": "2022-10-04T20:41:27.000+03:00", "completed_at": "2022-10-04T20:41:29.000+03:00"}, {"name": "Complete job", "status": "completed", "conclusion": "success", "number": 3, "started_at": "2022-10-04T20:41:29.000+03:00", "completed_at": "2022-10-04T20:41:29.000+03:00"}], "check_run_url": "https://api.github.com/repos/airbytehq/integration-test/check-runs/8705992587", "labels": ["ubuntu-latest"], "runner_id": 1, "runner_name": "Hosted Agent", "runner_group_id": 2, "runner_group_name": "GitHub Actions", "repository": "airbytehq/integration-test"},"emitted_at":1677668767830} {"stream":"team_members","data":{"login":"sherifnada","id":6246757,"node_id":"MDQ6VXNlcjYyNDY3NTc=","avatar_url":"https://avatars.githubusercontent.com/u/6246757?v=4","gravatar_id":"","url":"https://api.github.com/users/sherifnada","html_url":"https://github.com/sherifnada","followers_url":"https://api.github.com/users/sherifnada/followers","following_url":"https://api.github.com/users/sherifnada/following{/other_user}","gists_url":"https://api.github.com/users/sherifnada/gists{/gist_id}","starred_url":"https://api.github.com/users/sherifnada/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sherifnada/subscriptions","organizations_url":"https://api.github.com/users/sherifnada/orgs","repos_url":"https://api.github.com/users/sherifnada/repos","events_url":"https://api.github.com/users/sherifnada/events{/privacy}","received_events_url":"https://api.github.com/users/sherifnada/received_events","type":"User","site_admin":false,"organization":"airbytehq","team_slug":"zazmic"},"emitted_at":1677668768649} {"stream":"team_memberships","data":{"state":"active","role":"maintainer","url":"https://api.github.com/organizations/59758427/team/4432406/memberships/sherifnada","organization":"airbytehq","team_slug":"zazmic","username":"sherifnada"},"emitted_at":1677668779034} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-github/metadata.yaml b/airbyte-integrations/connectors/source-github/metadata.yaml index 0f99717be32b..7684650afb9e 100644 --- a/airbyte-integrations/connectors/source-github/metadata.yaml +++ b/airbyte-integrations/connectors/source-github/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e - dockerImageTag: 1.0.1 + dockerImageTag: 1.0.4 maxSecondsBetweenMessages: 5400 dockerRepository: airbyte/source-github githubIssueLabel: source-github @@ -33,4 +33,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/github tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-github/requirements.txt b/airbyte-integrations/connectors/source-github/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-github/requirements.txt +++ b/airbyte-integrations/connectors/source-github/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py index fc6487368a23..0c618c4eef04 100644 --- a/airbyte-integrations/connectors/source-github/setup.py +++ b/airbyte-integrations/connectors/source-github/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "pendulum~=2.1.2", "sgqlc"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "responses~=0.23.1", "freezegun~=1.2.0"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "responses~=0.23.1", "freezegun~=1.2.0"] setup( name="source_github", diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json b/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json index e0a3f1f73797..cab08b9532ce 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json @@ -88,6 +88,10 @@ "type": "string", "format": "date-time" }, + "archived_at": { + "type": ["null", "string"], + "format": "date-time" + }, "type": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index f7c61f553721..5c2b905915eb 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -29,6 +29,18 @@ "title": "Access Token", "description": "OAuth access token", "airbyte_secret": true + }, + "client_id": { + "type": "string", + "title": "Client Id", + "description": "OAuth Client Id", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client ssecret", + "description": "OAuth Client secret", + "airbyte_secret": true } } }, diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile index c140a0b72dad..67e6a7b6081b 100644 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ b/airbyte-integrations/connectors/source-gitlab/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.2.1 +LABEL io.airbyte.version=1.6.0 LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml index d13110db35aa..df414a0a2876 100644 --- a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml @@ -21,11 +21,11 @@ acceptance_tests: timeout_seconds: 3600 expect_records: path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false ignored_fields: jobs: - name: "user" bypass_reason: "User object contains local_time which will be different each time test is run" + fail_on_extra_columns: false - config_path: "secrets/config_with_ids.json" timeout_seconds: 3600 empty_streams: @@ -35,20 +35,20 @@ acceptance_tests: bypass_reason: "Group in this config does not have epics issues. This stream is tested in the above TC." expect_records: path: "integration_tests/expected_records_with_ids.jsonl" - fail_on_extra_columns: false ignored_fields: jobs: - name: "user" bypass_reason: "User object contains local_time which will be different each time test is run" + fail_on_extra_columns: false - config_path: "secrets/config_oauth.json" timeout_seconds: 3600 expect_records: path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false ignored_fields: jobs: - name: "user" bypass_reason: "User object contains local_time which will be different each time test is run" + fail_on_extra_columns: false incremental: tests: - config_path: "secrets/config_with_ids.json" diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl index 14d6a16769d3..b6607d921ac9 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl @@ -1,56 +1,56 @@ -{"stream": "project_milestones", "data": {"id": 1943563, "iid": 1, "project_id": 25156633, "title": "Sample GitLab Project Milestone 1", "description": "Sample GitLab Project Milestone 1", "state": "active", "created_at": "2021-02-15T16:00:33.349Z", "updated_at": "2021-02-15T16:00:33.349Z", "due_date": "2021-03-12", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/1"}, "emitted_at": 1684343741770} -{"stream": "project_milestones", "data": {"id": 1943564, "iid": 2, "project_id": 25156633, "title": "Sample GitLab Project Milestone 2", "description": "Sample GitLab Project Milestone 2", "state": "active", "created_at": "2021-02-15T16:00:46.934Z", "updated_at": "2021-02-15T16:00:46.934Z", "due_date": "2021-03-19", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/2"}, "emitted_at": 1684343741773} -{"stream": "project_milestones", "data": {"id": 1943565, "iid": 3, "project_id": 25156633, "title": "Sample GitLab Project Milestone 3", "description": "Sample GitLab Project Milestone 3", "state": "active", "created_at": "2021-02-15T16:01:07.153Z", "updated_at": "2021-02-15T16:01:07.153Z", "due_date": "2021-03-26", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/3"}, "emitted_at": 1684343741773} -{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1684343816341} -{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1684343816560} -{"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1684343715047} -{"stream": "merge_requests", "data": {"id": 92098762, "iid": 30, "project_id": 25156633, "title": "Id blanditiis consequatur ut.", "description": "##### Voluptatum\nPorro et quo. Laborum molestias ducimus. Labore dolorum adipisci. Quisquam est quis. Sint accusamus maxime.\n* Veritatis. \n* Eos. \n* Adipisci. \n* Quibusdam. \n* Sint. \n* Consequuntur. \n* Hic. \n* Voluptate. \n* Velit.", "state": "opened", "created_at": "2021-02-15T15:55:38.117Z", "updated_at": "2021-02-15T15:55:38.117Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "laudantium-unde-et-iste-et", "user_notes_count": 15, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["asperiores-ex", "quidem-labore", "sed-consequuntur"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "reference": "!30", "references": {"short": "!30", "relative": "!30", "full": "airbyte.io/ci-test-project!30"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1684343765239} -{"stream": "merge_requests", "data": {"id": 92098761, "iid": 29, "project_id": 25156633, "title": "Fugiat aut voluptatem voluptas.", "description": "###### Unde\nEligendi nemo quam. Veritatis delectus iure. Placeat ut odit. Officiis accusantium sit. Eos sequi cupiditate.\n# Eum", "state": "opened", "created_at": "2021-02-15T15:55:34.534Z", "updated_at": "2021-02-15T15:55:34.534Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ea-dolor-quia-et-sint", "user_notes_count": 4, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["earum-eaque", "nisi-et", "sed-voluptatem"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "reference": "!29", "references": {"short": "!29", "relative": "!29", "full": "airbyte.io/ci-test-project!29"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1684343765241} -{"stream": "merge_requests", "data": {"id": 92098758, "iid": 28, "project_id": 25156633, "title": "Delectus sit quod repellendus.", "description": "###### Corporis\nMolestias eius corrupti. Est maiores ut. Deleniti itaque deserunt. Perspiciatis quis et. Non et quia.\n### Dolorum\nNihil at et. Eligendi recusandae omnis. Eaque ratione dolorem.\n### Quam\nRem ad vel. Officiis sint voluptatem. Asperiores odit non.\n0. Hic. \n1. Labore. \n2. Voluptates. \n3. Dolores. \n4. Laborum. \n5. Non. \n6. Odit.", "state": "opened", "created_at": "2021-02-15T15:55:31.164Z", "updated_at": "2021-02-15T15:55:31.164Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ipsum-consequatur-et-in-et", "user_notes_count": 19, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["nisi-et", "omnis-assumenda", "ut-incidunt"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "reference": "!28", "references": {"short": "!28", "relative": "!28", "full": "airbyte.io/ci-test-project!28"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1684343765242} -{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1684343694766} -{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1684343695028} -{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1684343695306} -{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1684343828046} -{"stream": "issues", "data": {"id": 80940833, "iid": 30, "project_id": 25156633, "title": "Nulla tempore voluptatibus error.", "description": "### Voluptate\nQui quaerat praesentium. Voluptates temporibus quae. Libero aliquid quod. Nihil rerum earum. Inventore et illum.\n`Quod.`", "state": "opened", "created_at": "2021-02-15T15:56:04.924Z", "updated_at": "2021-02-15T15:56:04.924Z", "closed_at": null, "closed_by": null, "labels": ["et-facere", "nisi-et", "suscipit-consectetur"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 20, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/30", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "airbyte.io/ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1684343719358} -{"stream": "issues", "data": {"id": 80940829, "iid": 29, "project_id": 25156633, "title": "Dolores tempora deserunt perspiciatis.", "description": "#### Magnam\nTempora optio eos. Quos quam ut. Accusamus aperiam consequatur. Saepe sit nam. Eaque tenetur qui.\n0. Aut. \n1. Ratione.", "state": "opened", "created_at": "2021-02-15T15:56:04.790Z", "updated_at": "2021-02-15T15:56:04.790Z", "closed_at": null, "closed_by": null, "labels": ["in-qui", "iste-est", "odio-ut"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 18, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/29", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/29/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/29/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#29", "relative": "#29", "full": "airbyte.io/ci-test-project#29"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1684343719361} -{"stream": "issues", "data": {"id": 80940824, "iid": 28, "project_id": 25156633, "title": "Ut quae nesciunt facere.", "description": "##### Error\nIpsum doloremque beatae. Non incidunt ut. Quia esse atque. Quo dolores repudiandae. Sint aliquid et.\n## A", "state": "opened", "created_at": "2021-02-15T15:56:04.658Z", "updated_at": "2021-02-15T15:56:04.658Z", "closed_at": null, "closed_by": null, "labels": ["officia-laborum", "sit-neque", "voluptas-officiis"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 13, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/28", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/28/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/28/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#28", "relative": "#28", "full": "airbyte.io/ci-test-project#28"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1684343719362} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2022-12-02T08:51:28.434Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 41541892}, "emitted_at": 1684343746577} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T14:46:31.550Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25156633}, "emitted_at": 1684343746816} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-10T17:16:54.405Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25032440}, "emitted_at": 1684343747262} -{"stream": "epics", "data": {"id": 678569, "iid": 1, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Source Gitlab: certify to Beta", "description": "Lorem ipsum", "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": "2022-12-11", "start_date_is_fixed": true, "start_date_fixed": "2022-12-11", "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": "2022-12-30", "due_date": "2022-12-30", "due_date_is_fixed": true, "due_date_fixed": "2022-12-30", "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/1", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/1", "references": {"short": "&1", "relative": "&1", "full": "airbyte.io&1"}, "created_at": "2022-12-11T10:50:04.280Z", "updated_at": "2022-12-11T10:50:26.276Z", "closed_at": null, "labels": [], "upvotes": 1, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/1", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/1/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1684343825354} -{"stream": "commits", "data": {"id": "27329d3afac51fbf2762428e12f2635d1137c549", "short_id": "27329d3a", "created_at": "2021-02-15T15:52:52.000+00:00", "parent_ids": ["b362ea7aa65515dc35ff3a93423478b2143e771d"], "title": "Update README.md", "message": "Update README.md", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:52:52.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:52:52.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/27329d3afac51fbf2762428e12f2635d1137c549", "stats": {"additions": 6, "deletions": 0, "total": 6}, "project_id": 25156633}, "emitted_at": 1684343704642} -{"stream": "commits", "data": {"id": "b362ea7aa65515dc35ff3a93423478b2143e771d", "short_id": "b362ea7a", "created_at": "2021-02-15T15:52:03.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:52:03.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:52:03.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/b362ea7aa65515dc35ff3a93423478b2143e771d", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25156633}, "emitted_at": 1684343704643} -{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1684343709130} -{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:15 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1684343735922} -{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:15 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1684343735925} -{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:15 PM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1684343736224} -{"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541892}, "emitted_at": 1684343752156} -{"stream": "project_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541892}, "emitted_at": 1684343752158} -{"stream": "project_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541892}, "emitted_at": 1684343752158} -{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343796221} -{"stream": "projects", "data": {"id": 41541892, "description": "Project description", "name": "Test Project 2", "name_with_namespace": "New Group Airbute / Test Subgroup Airbyte / Test Project 2", "path": "test-project-2", "path_with_namespace": "new-group-airbute/test-subgroup-airbyte/test-project-2", "created_at": "2022-12-02T08:51:27.429Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/test-subgroup-airbyte/test-project-2.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/test-subgroup-airbyte/test-project-2.git", "web_url": "https://gitlab.com/new-group-airbute/test-subgroup-airbyte/test-project-2", "readme_url": "https://gitlab.com/new-group-airbute/test-subgroup-airbyte/test-project-2/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-02T08:51:27.429Z", "namespace": {"id": 61014863, "name": "Test Subgroup Airbyte", "path": "test-subgroup-airbyte", "kind": "group", "full_path": "new-group-airbute/test-subgroup-airbyte", "parent_id": 11329647, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/test-subgroup-airbyte/test-project-2", "_links": {"self": "https://gitlab.com/api/v4/projects/41541892", "issues": "https://gitlab.com/api/v4/projects/41541892/issues", "merge_requests": "https://gitlab.com/api/v4/projects/41541892/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/41541892/repository/branches", "labels": "https://gitlab.com/api/v4/projects/41541892/labels", "events": "https://gitlab.com/api/v4/projects/41541892/events", "members": "https://gitlab.com/api/v4/projects/41541892/members", "cluster_agents": "https://gitlab.com/api/v4/projects/41541892/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2022-12-03T08:51:27.465Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-test-subgroup-airbyte-test-project-2-41541892-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "enabled", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": null, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": "gitlab_project", "import_status": "finished", "import_error": null, "open_issues_count": 0, "description_html": "

    Project description

    ", "updated_at": "2022-12-02T08:51:29.191Z", "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941jzk79ABKrb8iMGhN9aoy", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 1, "storage_size": 2495610, "repository_size": 2495610, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684853245004} -{"stream": "projects", "data": {"id": 25156633, "description": "", "name": "CI Test Project", "name_with_namespace": "airbyte.io / CI Test Project", "path": "ci-test-project", "path_with_namespace": "airbyte.io/ci-test-project", "created_at": "2021-03-15T14:46:27.213Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:airbyte.io/ci-test-project.git", "http_url_to_repo": "https://gitlab.com/airbyte.io/ci-test-project.git", "web_url": "https://gitlab.com/airbyte.io/ci-test-project", "readme_url": "https://gitlab.com/airbyte.io/ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-11T10:50:26.066Z", "namespace": {"id": 11266951, "name": "airbyte.io", "path": "airbyte.io", "kind": "group", "full_path": "airbyte.io", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/airbyte.io"}, "container_registry_image_prefix": "registry.gitlab.com/airbyte.io/ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633", "issues": "https://gitlab.com/api/v4/projects/25156633/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25156633/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25156633/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25156633/labels", "events": "https://gitlab.com/api/v4/projects/25156633/events", "members": "https://gitlab.com/api/v4/projects/25156633/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25156633/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T14:48:00.805Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": false, "jobs_enabled": true, "snippets_enabled": false, "container_registry_enabled": true, "service_desk_enabled": false, "service_desk_address": "contact-project+airbyte-io-ci-test-project-25156633-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "disabled", "builds_access_level": "private", "snippets_access_level": "disabled", "pages_access_level": "enabled", "analytics_access_level": "disabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "disabled", "feature_flags_access_level": "disabled", "infrastructure_access_level": "disabled", "monitor_access_level": "disabled", "emails_disabled": false, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": "gitlab_project", "import_status": "finished", "import_error": null, "open_issues_count": 29, "description_html": "", "updated_at": "2022-12-11T10:50:26.066Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941WLwrsxTacV58yxzWsvB2", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": null, "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 2, "storage_size": 618659, "repository_size": 618659, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "approvals_before_merge": 0, "mirror": false, "external_authorization_classification_label": "", "marked_for_deletion_at": null, "marked_for_deletion_on": null, "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "issues_template": null, "merge_requests_template": null, "merge_pipelines_enabled": false, "merge_trains_enabled": false, "allow_pipeline_trigger_approve_deployment": false, "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684853245873} -{"stream": "projects", "data": {"id": 41541906, "description": "Project description", "name": "Test Public Project 1", "name_with_namespace": "New Group Airbute / Test Public SG / Test Public Project 1", "path": "test-public-project-1", "path_with_namespace": "new-group-airbute/test-public-sg/test-public-project-1", "created_at": "2022-12-02T08:52:11.319Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/test-public-sg/test-public-project-1.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/test-public-sg/test-public-project-1.git", "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-public-project-1", "readme_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-public-project-1/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-02T08:52:11.319Z", "namespace": {"id": 61014902, "name": "Test Public SG", "path": "test-public-sg", "kind": "group", "full_path": "new-group-airbute/test-public-sg", "parent_id": 11329647, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/test-public-sg/test-public-project-1", "_links": {"self": "https://gitlab.com/api/v4/projects/41541906", "issues": "https://gitlab.com/api/v4/projects/41541906/issues", "merge_requests": "https://gitlab.com/api/v4/projects/41541906/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/41541906/repository/branches", "labels": "https://gitlab.com/api/v4/projects/41541906/labels", "events": "https://gitlab.com/api/v4/projects/41541906/events", "members": "https://gitlab.com/api/v4/projects/41541906/members", "cluster_agents": "https://gitlab.com/api/v4/projects/41541906/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "public", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2022-12-03T08:52:11.354Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": false, "service_desk_address": "contact-project+new-group-airbute-test-public-sg-test-public-project-1-41541906-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": null, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": "gitlab_project", "import_status": "finished", "import_error": null, "open_issues_count": 0, "description_html": "

    Project description

    ", "updated_at": "2022-12-02T08:52:13.326Z", "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941Ni12De_sRyL6anY2uhzP", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 1, "storage_size": 209715, "repository_size": 209715, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684853246277} -{"stream": "branches", "data": {"name": "master", "commit": {"id": "a804df14057d4443439499a6152638bd5183fa92", "short_id": "a804df14", "created_at": "2019-03-07T15:30:56.000+01:00", "parent_ids": [], "title": "Initial template creation", "message": "Initial template creation\n", "author_name": "GitLab", "author_email": "root@localhost", "authored_date": "2019-03-07T15:30:56.000+01:00", "committer_name": "Jason Lenny", "committer_email": "jlenny@gitlab.com", "committed_date": "2019-03-07T15:30:56.000+01:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-subgroup-airbyte/test-project-2/-/commit/a804df14057d4443439499a6152638bd5183fa92"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/test-subgroup-airbyte/test-project-2/-/tree/master", "commit_id": "a804df14057d4443439499a6152638bd5183fa92", "project_id": 41541892}, "emitted_at": 1684343700383} -{"stream": "branches", "data": {"name": "at-adipisci-ducimus-qui-nihil", "commit": {"id": "e10493c095260599a73a32def40249a4c389e354", "short_id": "e10493c0", "created_at": "2021-02-15T15:55:06.000+00:00", "parent_ids": ["763258bc3b5803074eb2c23eb069275f9716a2c1"], "title": "Nisi ipsam rem repudiandae.", "message": "Nisi ipsam rem repudiandae.", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:55:06.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:55:06.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/e10493c095260599a73a32def40249a4c389e354"}, "merged": false, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/tree/at-adipisci-ducimus-qui-nihil", "commit_id": "e10493c095260599a73a32def40249a4c389e354", "project_id": 25156633}, "emitted_at": 1684343700668} -{"stream": "branches", "data": {"name": "atque-corrupti-laboriosam-nobis-explicabo", "commit": {"id": "b66e20cbfd9b59275eefdd05ad35d277a0e58866", "short_id": "b66e20cb", "created_at": "2021-02-15T15:54:45.000+00:00", "parent_ids": ["24e81aef412904ef8cb9e9ef3c062d0b8735835d"], "title": "Perferendis in maxime consequatur.", "message": "Perferendis in maxime consequatur.", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:54:45.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:54:45.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/b66e20cbfd9b59275eefdd05ad35d277a0e58866"}, "merged": false, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/tree/atque-corrupti-laboriosam-nobis-explicabo", "commit_id": "b66e20cbfd9b59275eefdd05ad35d277a0e58866", "project_id": 25156633}, "emitted_at": 1684343700669} -{"stream": "merge_request_commits", "data": {"id": 92098762, "iid": 30, "project_id": 25156633, "title": "Id blanditiis consequatur ut.", "description": "##### Voluptatum\nPorro et quo. Laborum molestias ducimus. Labore dolorum adipisci. Quisquam est quis. Sint accusamus maxime.\n* Veritatis. \n* Eos. \n* Adipisci. \n* Quibusdam. \n* Sint. \n* Consequuntur. \n* Hic. \n* Voluptate. \n* Velit.", "state": "opened", "created_at": "2021-02-15T15:55:38.117Z", "updated_at": "2021-02-15T15:55:38.117Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "laudantium-unde-et-iste-et", "user_notes_count": 15, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["asperiores-ex", "quidem-labore", "sed-consequuntur"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "reference": "!30", "references": {"short": "!30", "relative": "!30", "full": "airbyte.io/ci-test-project!30"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "18", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 30}, "emitted_at": 1684343775814} -{"stream": "merge_request_commits", "data": {"id": 92098761, "iid": 29, "project_id": 25156633, "title": "Fugiat aut voluptatem voluptas.", "description": "###### Unde\nEligendi nemo quam. Veritatis delectus iure. Placeat ut odit. Officiis accusantium sit. Eos sequi cupiditate.\n# Eum", "state": "opened", "created_at": "2021-02-15T15:55:34.534Z", "updated_at": "2021-02-15T15:55:34.534Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ea-dolor-quia-et-sint", "user_notes_count": 4, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["earum-eaque", "nisi-et", "sed-voluptatem"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "reference": "!29", "references": {"short": "!29", "relative": "!29", "full": "airbyte.io/ci-test-project!29"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "17", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 29}, "emitted_at": 1684343776091} -{"stream": "merge_request_commits", "data": {"id": 92098758, "iid": 28, "project_id": 25156633, "title": "Delectus sit quod repellendus.", "description": "###### Corporis\nMolestias eius corrupti. Est maiores ut. Deleniti itaque deserunt. Perspiciatis quis et. Non et quia.\n### Dolorum\nNihil at et. Eligendi recusandae omnis. Eaque ratione dolorem.\n### Quam\nRem ad vel. Officiis sint voluptatem. Asperiores odit non.\n0. Hic. \n1. Labore. \n2. Voluptates. \n3. Dolores. \n4. Laborum. \n5. Non. \n6. Odit.", "state": "opened", "created_at": "2021-02-15T15:55:31.164Z", "updated_at": "2021-02-15T15:55:31.164Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ipsum-consequatur-et-in-et", "user_notes_count": 19, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["nisi-et", "omnis-assumenda", "ut-incidunt"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "reference": "!28", "references": {"short": "!28", "relative": "!28", "full": "airbyte.io/ci-test-project!28"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "15", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 28}, "emitted_at": 1684343776409} -{"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1684343744055} -{"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1684343744056} -{"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1684343744056} -{"stream": "pipelines", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "name": null}, "emitted_at": 1684343805542} -{"stream": "pipelines", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "name": null}, "emitted_at": 1684343805543} -{"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1684343757733} -{"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1684343757734} -{"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1684343757735} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1684343821223} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1684343821224} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1684343821446} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1684343748475} -{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1684343748476} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1684343748717} -{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343798247} -{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343798248} -{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343798248} +{"stream": "project_milestones", "data": {"id": 1943563, "iid": 1, "project_id": 25156633, "title": "Sample GitLab Project Milestone 1", "description": "Sample GitLab Project Milestone 1", "state": "active", "created_at": "2021-02-15T16:00:33.349Z", "updated_at": "2021-02-15T16:00:33.349Z", "due_date": "2021-03-12", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/1"}, "emitted_at": 1686568101690} +{"stream": "project_milestones", "data": {"id": 1943564, "iid": 2, "project_id": 25156633, "title": "Sample GitLab Project Milestone 2", "description": "Sample GitLab Project Milestone 2", "state": "active", "created_at": "2021-02-15T16:00:46.934Z", "updated_at": "2021-02-15T16:00:46.934Z", "due_date": "2021-03-19", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/2"}, "emitted_at": 1686568101692} +{"stream": "project_milestones", "data": {"id": 1943565, "iid": 3, "project_id": 25156633, "title": "Sample GitLab Project Milestone 3", "description": "Sample GitLab Project Milestone 3", "state": "active", "created_at": "2021-02-15T16:01:07.153Z", "updated_at": "2021-02-15T16:01:07.153Z", "due_date": "2021-03-26", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/3"}, "emitted_at": 1686568101692} +{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686568215223} +{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686568215999} +{"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1686568061140} +{"stream": "merge_requests", "data": {"id": 92098762, "iid": 30, "project_id": 25156633, "title": "Id blanditiis consequatur ut.", "description": "##### Voluptatum\nPorro et quo. Laborum molestias ducimus. Labore dolorum adipisci. Quisquam est quis. Sint accusamus maxime.\n* Veritatis. \n* Eos. \n* Adipisci. \n* Quibusdam. \n* Sint. \n* Consequuntur. \n* Hic. \n* Voluptate. \n* Velit.", "state": "opened", "created_at": "2021-02-15T15:55:38.117Z", "updated_at": "2021-02-15T15:55:38.117Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "laudantium-unde-et-iste-et", "user_notes_count": 15, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["asperiores-ex", "quidem-labore", "sed-consequuntur"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:38.117Z", "reference": "!30", "references": {"short": "!30", "relative": "!30", "full": "airbyte.io/ci-test-project!30"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686568137434} +{"stream": "merge_requests", "data": {"id": 92098761, "iid": 29, "project_id": 25156633, "title": "Fugiat aut voluptatem voluptas.", "description": "###### Unde\nEligendi nemo quam. Veritatis delectus iure. Placeat ut odit. Officiis accusantium sit. Eos sequi cupiditate.\n# Eum", "state": "opened", "created_at": "2021-02-15T15:55:34.534Z", "updated_at": "2021-02-15T15:55:34.534Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ea-dolor-quia-et-sint", "user_notes_count": 4, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["earum-eaque", "nisi-et", "sed-voluptatem"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:34.534Z", "reference": "!29", "references": {"short": "!29", "relative": "!29", "full": "airbyte.io/ci-test-project!29"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686568137437} +{"stream": "merge_requests", "data": {"id": 92098758, "iid": 28, "project_id": 25156633, "title": "Delectus sit quod repellendus.", "description": "###### Corporis\nMolestias eius corrupti. Est maiores ut. Deleniti itaque deserunt. Perspiciatis quis et. Non et quia.\n### Dolorum\nNihil at et. Eligendi recusandae omnis. Eaque ratione dolorem.\n### Quam\nRem ad vel. Officiis sint voluptatem. Asperiores odit non.\n0. Hic. \n1. Labore. \n2. Voluptates. \n3. Dolores. \n4. Laborum. \n5. Non. \n6. Odit.", "state": "opened", "created_at": "2021-02-15T15:55:31.164Z", "updated_at": "2021-02-15T15:55:31.164Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ipsum-consequatur-et-in-et", "user_notes_count": 19, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["nisi-et", "omnis-assumenda", "ut-incidunt"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:31.164Z", "reference": "!28", "references": {"short": "!28", "relative": "!28", "full": "airbyte.io/ci-test-project!28"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686568137439} +{"stream":"groups","data":{"id":68657749,"web_url":"https://gitlab.com/groups/empty-group4","name":"Empty Group","path":"empty-group4","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"Empty Group","full_path":"empty-group4","created_at":"2023-06-09T13:47:19.446Z","parent_id":null,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941-x4xtBM6_zdFj-ED8QF8","prevent_sharing_groups_outside_hierarchy":false,"shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[]},"emitted_at":1688127739135} +{"stream":"groups","data":{"id":11329647,"web_url":"https://gitlab.com/groups/new-group-airbute","name":"New Group Airbute","path":"new-group-airbute","description":"","visibility":"public","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute","full_path":"new-group-airbute","created_at":"2021-03-15T15:55:53.613Z","parent_id":null,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941-PhosPap-Sf1UxL1g6m4","prevent_sharing_groups_outside_hierarchy":false,"shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[{"id":25157276,"path_with_namespace":"new-group-airbute/new-ci-test-project"}]},"emitted_at":1688127739529} +{"stream":"groups","data":{"id":61014882,"web_url":"https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg","name":"Test Private SG","path":"test-private-sg","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute / Test Subgroup Airbyte / Test Private SG","full_path":"new-group-airbute/test-subgroup-airbyte/test-private-sg","created_at":"2022-12-02T08:46:22.648Z","parent_id":61014863,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941bjUaJQy2zzar-JmNBjfq","shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[]},"emitted_at":1688127739793} +{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1686568231753} +{"stream": "issues", "data": {"id": 80940833, "iid": 30, "project_id": 25156633, "title": "Nulla tempore voluptatibus error.", "description": "### Voluptate\nQui quaerat praesentium. Voluptates temporibus quae. Libero aliquid quod. Nihil rerum earum. Inventore et illum.\n`Quod.`", "state": "opened", "created_at": "2021-02-15T15:56:04.924Z", "updated_at": "2021-02-15T15:56:04.924Z", "closed_at": null, "closed_by": null, "labels": ["et-facere", "nisi-et", "suscipit-consectetur"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 20, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/30", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "airbyte.io/ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686568070306} +{"stream": "issues", "data": {"id": 80940829, "iid": 29, "project_id": 25156633, "title": "Dolores tempora deserunt perspiciatis.", "description": "#### Magnam\nTempora optio eos. Quos quam ut. Accusamus aperiam consequatur. Saepe sit nam. Eaque tenetur qui.\n0. Aut. \n1. Ratione.", "state": "opened", "created_at": "2021-02-15T15:56:04.790Z", "updated_at": "2021-02-15T15:56:04.790Z", "closed_at": null, "closed_by": null, "labels": ["in-qui", "iste-est", "odio-ut"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 18, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/29", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/29/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/29/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#29", "relative": "#29", "full": "airbyte.io/ci-test-project#29"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686568070307} +{"stream": "issues", "data": {"id": 80940824, "iid": 28, "project_id": 25156633, "title": "Ut quae nesciunt facere.", "description": "##### Error\nIpsum doloremque beatae. Non incidunt ut. Quia esse atque. Quo dolores repudiandae. Sint aliquid et.\n## A", "state": "opened", "created_at": "2021-02-15T15:56:04.658Z", "updated_at": "2021-02-15T15:56:04.658Z", "closed_at": null, "closed_by": null, "labels": ["officia-laborum", "sit-neque", "voluptas-officiis"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 13, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/28", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/28/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/28/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#28", "relative": "#28", "full": "airbyte.io/ci-test-project#28"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686568070307} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2022-12-02T08:50:10.348Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 41541858}, "emitted_at": 1686568108823} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T14:46:31.550Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25156633}, "emitted_at": 1686568109879} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-10T17:16:54.405Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25032440}, "emitted_at": 1686568110194} +{"stream": "epics", "data": {"id": 678569, "iid": 1, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Source Gitlab: certify to Beta", "description": "Lorem ipsum", "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": "2022-12-11", "start_date_is_fixed": true, "start_date_fixed": "2022-12-11", "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": "2022-12-30", "due_date": "2022-12-30", "due_date_is_fixed": true, "due_date_fixed": "2022-12-30", "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/1", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/1", "references": {"short": "&1", "relative": "&1", "full": "airbyte.io&1"}, "created_at": "2022-12-11T10:50:04.280Z", "updated_at": "2022-12-11T10:50:26.276Z", "closed_at": null, "labels": [], "upvotes": 1, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/1", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/1/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1686568224925} +{"stream": "commits", "data": {"id": "fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "short_id": "fb24e673", "created_at": "2022-12-02T14:26:55.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2022-12-02T14:26:55.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2022-12-02T14:26:55.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/commit/fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "stats": {"additions": 92, "deletions": 0, "total": 92}, "project_id": 41551658}, "emitted_at": 1686568045674} +{"stream": "commits", "data": {"id": "27329d3afac51fbf2762428e12f2635d1137c549", "short_id": "27329d3a", "created_at": "2021-02-15T15:52:52.000+00:00", "parent_ids": ["b362ea7aa65515dc35ff3a93423478b2143e771d"], "title": "Update README.md", "message": "Update README.md", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:52:52.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:52:52.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/27329d3afac51fbf2762428e12f2635d1137c549", "stats": {"additions": 6, "deletions": 0, "total": 6}, "project_id": 25156633}, "emitted_at": 1686568048416} +{"stream": "commits", "data": {"id": "b362ea7aa65515dc35ff3a93423478b2143e771d", "short_id": "b362ea7a", "created_at": "2021-02-15T15:52:03.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:52:03.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:52:03.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/b362ea7aa65515dc35ff3a93423478b2143e771d", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25156633}, "emitted_at": 1686568048417} +{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686568098000} +{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686568098001} +{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1686568098411} +{"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541858}, "emitted_at": 1686568116129} +{"stream": "project_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541858}, "emitted_at": 1686568116131} +{"stream": "project_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541858}, "emitted_at": 1686568116131} +{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568182190} +{"stream": "projects", "data": {"id": 41541858, "description": "Project description", "name": "Test Project 1", "name_with_namespace": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Project 1", "path": "test-project-1", "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-project-1", "created_at": "2022-12-02T08:50:08.842Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/test-public-sg/test-sg-public-2/test-project-1.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1.git", "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1", "readme_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-02T08:50:08.842Z", "namespace": {"id": 61014943, "name": "Test SG Public 2", "path": "test-sg-public-2", "kind": "group", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2", "parent_id": 61014902, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1", "_links": {"self": "https://gitlab.com/api/v4/projects/41541858", "issues": "https://gitlab.com/api/v4/projects/41541858/issues", "merge_requests": "https://gitlab.com/api/v4/projects/41541858/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/41541858/repository/branches", "labels": "https://gitlab.com/api/v4/projects/41541858/labels", "events": "https://gitlab.com/api/v4/projects/41541858/events", "members": "https://gitlab.com/api/v4/projects/41541858/members", "cluster_agents": "https://gitlab.com/api/v4/projects/41541858/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2022-12-03T08:50:08.883Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-test-public-sg-test-sg-public-2-test-project-41541858-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": "gitlab_project", "import_status": "finished", "import_error": null, "open_issues_count": 0, "description_html": "

    Project description

    ", "updated_at": "2022-12-02T08:50:11.170Z", "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941JLqwDRN64-__uzBXcgc5", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 1, "storage_size": 125829, "repository_size": 125829, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448723961} +{"stream": "projects", "data": {"id": 41551658, "description": null, "name": "Test_project_in_nested_subgroup", "name_with_namespace": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1 / Test_project_in_nested_subgroup", "path": "test_project_in_nested_subgroup", "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup", "created_at": "2022-12-02T14:26:55.282Z", "default_branch": "main", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup.git", "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup", "readme_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/blob/main/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-02T14:26:55.282Z", "namespace": {"id": 61015181, "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "kind": "group", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "parent_id": 61014943, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup", "_links": {"self": "https://gitlab.com/api/v4/projects/41551658", "issues": "https://gitlab.com/api/v4/projects/41551658/issues", "merge_requests": "https://gitlab.com/api/v4/projects/41551658/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/41551658/repository/branches", "labels": "https://gitlab.com/api/v4/projects/41551658/labels", "events": "https://gitlab.com/api/v4/projects/41551658/events", "members": "https://gitlab.com/api/v4/projects/41551658/members", "cluster_agents": "https://gitlab.com/api/v4/projects/41551658/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2022-12-03T14:26:55.314Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-test-public-sg-test-sg-public-2-test-private-41551658-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 0, "description_html": "", "updated_at": "2022-12-02T14:26:56.266Z", "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941hyrJGkPgfF9b5KARxqHr", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 1, "storage_size": 73400, "repository_size": 73400, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": null, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448724778} +{"stream": "projects", "data": {"id": 25032439, "description": null, "name": "documentation", "name_with_namespace": "airbyte.io / documentation", "path": "documentation", "path_with_namespace": "airbyte.io/documentation", "created_at": "2021-03-10T17:16:53.200Z", "default_branch": "main", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:airbyte.io/documentation.git", "http_url_to_repo": "https://gitlab.com/airbyte.io/documentation.git", "web_url": "https://gitlab.com/airbyte.io/documentation", "readme_url": null, "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2021-03-10T17:16:53.200Z", "namespace": {"id": 11266951, "name": "airbyte.io", "path": "airbyte.io", "kind": "group", "full_path": "airbyte.io", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/airbyte.io"}, "container_registry_image_prefix": "registry.gitlab.com/airbyte.io/documentation", "_links": {"self": "https://gitlab.com/api/v4/projects/25032439", "issues": "https://gitlab.com/api/v4/projects/25032439/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25032439/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25032439/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25032439/labels", "events": "https://gitlab.com/api/v4/projects/25032439/events", "members": "https://gitlab.com/api/v4/projects/25032439/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25032439/cluster_agents"}, "packages_enabled": true, "empty_repo": true, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-11T17:16:53.215Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+airbyte-io-documentation-25032439-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 0, "description_html": "", "updated_at": "2022-03-23T13:23:04.923Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941iwELAs9x3hqVbY3Bo_q4", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 0, "storage_size": 0, "repository_size": 0, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "approvals_before_merge": 0, "mirror": false, "external_authorization_classification_label": "", "marked_for_deletion_at": null, "marked_for_deletion_on": null, "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "issues_template": null, "merge_requests_template": null, "merge_pipelines_enabled": false, "merge_trains_enabled": false, "allow_pipeline_trigger_approve_deployment": false, "permissions": {"project_access": null, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448725290} +{"stream": "branches", "data": {"name": "master", "commit": {"id": "bcdfbfd57c8f3cd6cd65998464bb71a562d49948", "short_id": "bcdfbfd5", "created_at": "2019-03-06T09:52:24.000+01:00", "parent_ids": [], "title": "Initial template creation", "message": "Initial template creation\n", "author_name": "GitLab", "author_email": "root@localhost", "authored_date": "2019-03-06T09:52:24.000+01:00", "committer_name": "Jason Lenny", "committer_email": "jlenny@gitlab.com", "committed_date": "2019-03-06T09:52:24.000+01:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1/-/commit/bcdfbfd57c8f3cd6cd65998464bb71a562d49948"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1/-/tree/master", "commit_id": "bcdfbfd57c8f3cd6cd65998464bb71a562d49948", "project_id": 41541858}, "emitted_at": 1686568039222} +{"stream": "branches", "data": {"name": "main", "commit": {"id": "fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "short_id": "fb24e673", "created_at": "2022-12-02T14:26:55.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2022-12-02T14:26:55.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2022-12-02T14:26:55.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/commit/fb24e6736b3a959a59e49b56d2d83a28ea3ae15b"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/tree/main", "commit_id": "fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "project_id": 41551658}, "emitted_at": 1686568039632} +{"stream": "branches", "data": {"name": "at-adipisci-ducimus-qui-nihil", "commit": {"id": "e10493c095260599a73a32def40249a4c389e354", "short_id": "e10493c0", "created_at": "2021-02-15T15:55:06.000+00:00", "parent_ids": ["763258bc3b5803074eb2c23eb069275f9716a2c1"], "title": "Nisi ipsam rem repudiandae.", "message": "Nisi ipsam rem repudiandae.", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:55:06.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:55:06.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/e10493c095260599a73a32def40249a4c389e354"}, "merged": false, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/tree/at-adipisci-ducimus-qui-nihil", "commit_id": "e10493c095260599a73a32def40249a4c389e354", "project_id": 25156633}, "emitted_at": 1686568040451} +{"stream": "merge_request_commits", "data": {"id": 92098762, "iid": 30, "project_id": 25156633, "title": "Id blanditiis consequatur ut.", "description": "##### Voluptatum\nPorro et quo. Laborum molestias ducimus. Labore dolorum adipisci. Quisquam est quis. Sint accusamus maxime.\n* Veritatis. \n* Eos. \n* Adipisci. \n* Quibusdam. \n* Sint. \n* Consequuntur. \n* Hic. \n* Voluptate. \n* Velit.", "state": "opened", "created_at": "2021-02-15T15:55:38.117Z", "updated_at": "2021-02-15T15:55:38.117Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "laudantium-unde-et-iste-et", "user_notes_count": 15, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["asperiores-ex", "quidem-labore", "sed-consequuntur"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:38.117Z", "reference": "!30", "references": {"short": "!30", "relative": "!30", "full": "airbyte.io/ci-test-project!30"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "18", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 30}, "emitted_at": 1686568154022} +{"stream": "merge_request_commits", "data": {"id": 92098761, "iid": 29, "project_id": 25156633, "title": "Fugiat aut voluptatem voluptas.", "description": "###### Unde\nEligendi nemo quam. Veritatis delectus iure. Placeat ut odit. Officiis accusantium sit. Eos sequi cupiditate.\n# Eum", "state": "opened", "created_at": "2021-02-15T15:55:34.534Z", "updated_at": "2021-02-15T15:55:34.534Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ea-dolor-quia-et-sint", "user_notes_count": 4, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["earum-eaque", "nisi-et", "sed-voluptatem"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:34.534Z", "reference": "!29", "references": {"short": "!29", "relative": "!29", "full": "airbyte.io/ci-test-project!29"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "17", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 29}, "emitted_at": 1686568154423} +{"stream": "merge_request_commits", "data": {"id": 92098758, "iid": 28, "project_id": 25156633, "title": "Delectus sit quod repellendus.", "description": "###### Corporis\nMolestias eius corrupti. Est maiores ut. Deleniti itaque deserunt. Perspiciatis quis et. Non et quia.\n### Dolorum\nNihil at et. Eligendi recusandae omnis. Eaque ratione dolorem.\n### Quam\nRem ad vel. Officiis sint voluptatem. Asperiores odit non.\n0. Hic. \n1. Labore. \n2. Voluptates. \n3. Dolores. \n4. Laborum. \n5. Non. \n6. Odit.", "state": "opened", "created_at": "2021-02-15T15:55:31.164Z", "updated_at": "2021-02-15T15:55:31.164Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ipsum-consequatur-et-in-et", "user_notes_count": 19, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["nisi-et", "omnis-assumenda", "ut-incidunt"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:31.164Z", "reference": "!28", "references": {"short": "!28", "relative": "!28", "full": "airbyte.io/ci-test-project!28"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "15", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 28}, "emitted_at": 1686568154935} +{"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1686568104768} +{"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1686568104771} +{"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1686568104771} +{"stream": "pipelines", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "name": null}, "emitted_at": 1686568199114} +{"stream": "pipelines", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "name": null}, "emitted_at": 1686568199117} +{"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123880} +{"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123881} +{"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123881} +{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1686568218271} +{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1686568218274} +{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1686568218612} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2023-06-09T13:47:19.592Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 68657749}, "emitted_at": 1686568111619} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686568111909} +{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686568111910} +{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185586} +{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185588} +{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185590} diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl index 7fce40bb51e8..280b0ec036fa 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl @@ -1,49 +1,49 @@ -{"stream": "pipelines", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "name": null}, "emitted_at": 1684343629136} -{"stream": "pipelines", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "name": null}, "emitted_at": 1684343629137} -{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343628324} -{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:13 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1684343610407} -{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:13 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1684343610409} -{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:13 PM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1684343610678} -{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1684343626362} -{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1684343626640} -{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1684343626941} -{"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1684343612511} -{"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1684343612512} -{"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1684343612513} -{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1684343630715} -{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1684343630931} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1684343632207} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1684343632208} -{"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1684343618726} -{"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1684343618727} -{"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1684343618727} -{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1684343600517} -{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1684343600778} -{"stream": "groups", "data": {"id": 61015181, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": null, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "created_at": "2022-12-02T08:54:42.252Z", "parent_id": 61014943, "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941x8xQf6K-UvnnyJ-bcut4", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 41551658, "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]}, "emitted_at": 1684343601101} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1684343614789} -{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1684343614790} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1684343615131} -{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343603572} -{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1684343603573} -{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343603573} -{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1684343604286} -{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1684343604287} -{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1684343604287} -{"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1684343605773} -{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1684855570331} -{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343628635} -{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343628637} -{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1684343628637} -{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1684343624263} -{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1684343624265} -{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1684343624265} -{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1684343608462} -{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1684343608464} -{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1684343608464} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1684343614534} -{"stream": "project_labels", "data": {"id": 19116944, "name": "Label 1", "description": null, "description_html": "", "text_color": "#1F1E24", "color": "#ffff00", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1684343617358} -{"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 25157276}, "emitted_at": 1684343617359} -{"stream": "project_labels", "data": {"id": 19116954, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#ff00ff", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1684343617359} -{"stream": "project_milestones", "data": {"id": 1943705, "iid": 51, "project_id": 25157276, "title": "Project Milestone 51", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.915Z", "updated_at": "2021-03-15T15:33:16.915Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/51"}, "emitted_at": 1684343611971} -{"stream": "project_milestones", "data": {"id": 1943704, "iid": 50, "project_id": 25157276, "title": "Project Milestone 50", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.329Z", "updated_at": "2021-03-15T15:33:16.329Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/50"}, "emitted_at": 1684343611971} -{"stream": "project_milestones", "data": {"id": 1943703, "iid": 49, "project_id": 25157276, "title": "Project Milestone 49", "description": null, "state": "active", "created_at": "2021-03-15T15:33:15.960Z", "updated_at": "2021-03-15T15:33:15.960Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/49"}, "emitted_at": 1684343611971} +{"stream": "pipelines", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "name": null}, "emitted_at": 1686567225920} +{"stream": "pipelines", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "name": null}, "emitted_at": 1686567225922} +{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567224796} +{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567192490} +{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567192491} +{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1686567192861} +{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1686567221987} +{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1686567222383} +{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1686567223002} +{"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1686567198876} +{"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1686567198878} +{"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1686567198878} +{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686567228326} +{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686567228728} +{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1686567230821} +{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1686567230823} +{"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686567210102} +{"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686567210104} +{"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686567210104} +{"stream":"groups","data":{"id":11329647,"web_url":"https://gitlab.com/groups/new-group-airbute","name":"New Group Airbute","path":"new-group-airbute","description":"","visibility":"public","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute","full_path":"new-group-airbute","created_at":"2021-03-15T15:55:53.613Z","parent_id":null,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941-PhosPap-Sf1UxL1g6m4","prevent_sharing_groups_outside_hierarchy":false,"shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[{"id":25157276,"path_with_namespace":"new-group-airbute/new-ci-test-project"}]},"emitted_at":1688238188152} +{"stream":"groups","data":{"id":61014882,"web_url":"https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg","name":"Test Private SG","path":"test-private-sg","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute / Test Subgroup Airbyte / Test Private SG","full_path":"new-group-airbute/test-subgroup-airbyte/test-private-sg","created_at":"2022-12-02T08:46:22.648Z","parent_id":61014863,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941bjUaJQy2zzar-JmNBjfq","shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[]},"emitted_at":1688238188456} +{"stream":"groups","data":{"id":61015181,"web_url":"https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1","name":"Test Private SubSubG 1","path":"test-private-subsubg-1","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1","full_path":"new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1","created_at":"2022-12-02T08:54:42.252Z","parent_id":61014943,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941x8xQf6K-UvnnyJ-bcut4","shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[{"id":41551658,"path_with_namespace":"new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]},"emitted_at":1688238188865} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686567203646} +{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686567203646} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1686567204049} +{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567183576} +{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567183576} +{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567183577} +{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1686567184540} +{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1686567184541} +{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1686567184541} +{"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1686567186609} +{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448724369} +{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225240} +{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225242} +{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225243} +{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1686567219461} +{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686567219464} +{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686567219466} +{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686567189959} +{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686567189960} +{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686567189960} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1686567202944} +{"stream": "project_labels", "data": {"id": 19116944, "name": "Label 1", "description": null, "description_html": "", "text_color": "#1F1E24", "color": "#ffff00", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1686567207747} +{"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 25157276}, "emitted_at": 1686567207748} +{"stream": "project_labels", "data": {"id": 19116954, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#ff00ff", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1686567207748} +{"stream": "project_milestones", "data": {"id": 1943705, "iid": 51, "project_id": 25157276, "title": "Project Milestone 51", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.915Z", "updated_at": "2021-03-15T15:33:16.915Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/51"}, "emitted_at": 1686567197935} +{"stream": "project_milestones", "data": {"id": 1943704, "iid": 50, "project_id": 25157276, "title": "Project Milestone 50", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.329Z", "updated_at": "2021-03-15T15:33:16.329Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/50"}, "emitted_at": 1686567197937} +{"stream": "project_milestones", "data": {"id": 1943703, "iid": 49, "project_id": 25157276, "title": "Project Milestone 49", "description": null, "state": "active", "created_at": "2021-03-15T15:33:15.960Z", "updated_at": "2021-03-15T15:33:15.960Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/49"}, "emitted_at": 1686567197937} diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json index 4b3afce2d911..905017d6b746 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json @@ -5,7 +5,7 @@ "groups": "new-group-airbute", "projects": "new-group-airbute/new-ci-test-project", "credentials": { - "auth_type": "access_token", - "access_token": "migrated_from_old_config" + "auth_type": "access_token", + "access_token": "migrated_from_old_config" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-gitlab/metadata.yaml b/airbyte-integrations/connectors/source-gitlab/metadata.yaml index 0e4f60551396..10304aeaebbc 100644 --- a/airbyte-integrations/connectors/source-gitlab/metadata.yaml +++ b/airbyte-integrations/connectors/source-gitlab/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 - dockerImageTag: 1.2.1 + dockerImageTag: 1.6.0 dockerRepository: airbyte/source-gitlab githubIssueLabel: source-gitlab icon: gitlab.svg @@ -16,8 +16,12 @@ data: enabled: true oss: enabled: true - releaseStage: beta + releaseStage: generally_available documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gitlab/requirements.txt b/airbyte-integrations/connectors/source-gitlab/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-gitlab/requirements.txt +++ b/airbyte-integrations/connectors/source-gitlab/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gitlab/setup.py b/airbyte-integrations/connectors/source-gitlab/setup.py index 562cc42c530e..682fadb8af03 100644 --- a/airbyte-integrations/connectors/source-gitlab/setup.py +++ b/airbyte-integrations/connectors/source-gitlab/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy==4.1.1"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "requests_mock", "pytest-mock"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "requests_mock", "pytest-mock"] setup( name="source_gitlab", diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/branches.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/branches.json index c4ce9ef40bcf..8fd539d0af14 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/branches.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/branches.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "project_id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json index 42ae0ab2ef00..89a7d2f5ae31 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "project_id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json index 145bfa415bbb..0b27bc602861 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -97,6 +97,152 @@ }, "epic_issue_id": { "type": ["null", "integer"] + }, + "merge_requests_count": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "task_status": { + "type": ["null", "string"] + }, + "moved_to_id": { + "type": ["null", "integer"] + }, + "iteration": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "sequence": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "due_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "web_url": { + "type": ["null", "string"] + } + } + }, + "has_tasks": { + "type": ["null", "boolean"] + }, + "blocking_issues_count": { + "type": ["null", "integer"] + }, + "closed_by": { + "type": ["null", "object"], + "properties": { + "state": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "references": { + "type": ["null", "object"], + "properties": { + "full": { + "type": ["null", "string"] + }, + "relative": { + "type": ["null", "string"] + }, + "short": { + "type": ["null", "string"] + } + } + }, + "epic": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "group_id": { + "type": ["null", "integer"] + } + } + }, + "issue_type": { + "type": ["null", "string"] + }, + "severity": { + "type": ["null", "string"] + }, + "service_desk_reply_to": { + "type": ["null", "string"] + }, + "task_completion_status": { + "type": ["null", "object"], + "properties": { + "count": { + "type": ["null", "integer"] + }, + "completed_count": { + "type": ["null", "integer"] + } + } + }, + "relative_position": { + "type": ["null", "integer"] + }, + "epic_iid": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json index e72352d8ee5e..5be8c292c03e 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -89,6 +89,43 @@ }, "downvotes": { "type": ["null", "integer"] + }, + "parent_iid": { + "type": ["null", "integer"] + }, + "color": { + "type": ["null", "string"] + }, + "text_color": { + "type": ["null", "string"] + }, + "web_edit_url": { + "type": ["null", "string"] + }, + "due_date_from_milestones": { + "type": ["null", "string"], + "format": "date" + }, + "_links": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "epic_issues": { + "type": ["null", "string"] + }, + "group": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "string"] + } + } + }, + "start_date_from_milestones": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json index 4d50781a45cc..2784b61d8c03 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -56,6 +56,20 @@ }, "group_id": { "type": ["null", "integer"] + }, + "group": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_labels.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_labels.json index e16c5a675760..925b2559ceaa 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_labels.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_labels.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "group_id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json index 708edf354459..52705d38be9a 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "group_id": { @@ -17,6 +17,9 @@ "state": { "type": ["null", "string"] }, + "membership_state": { + "type": ["null", "string"] + }, "avatar_url": { "type": ["null", "string"] }, @@ -32,7 +35,27 @@ }, "expires_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date" + }, + "created_by": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_milestones.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_milestones.json index 71f833d9c4b2..2ce56b4eed73 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_milestones.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_milestones.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json index dde18226e3ab..3c8870097537 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "projects": { @@ -106,6 +106,25 @@ }, "prevent_forking_outside_group": { "type": ["null", "boolean"] + }, + "wiki_access_level": { + "type": ["null", "string"] + }, + "marked_for_deletion_on": { + "type": ["null", "string"], + "format": "date" + }, + "prevent_sharing_groups_outside_hierarchy": { + "type": ["null", "boolean"] + }, + "membership_lock": { + "type": ["null", "boolean"] + }, + "ip_restriction_ranges": { + "type": ["null", "string"] + }, + "shared_runners_setting": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json index b84e371216cb..e02aa19fcf1a 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -134,6 +134,74 @@ }, "task_status": { "type": ["null", "string"] + }, + "severity": { + "type": ["null", "string"] + }, + "iteration": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "sequence": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "due_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "web_url": { + "type": ["null", "string"] + } + } + }, + "epic": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "group_id": { + "type": ["null", "integer"] + } + }, + "epic_iid": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json index fc5639b456d1..4c41e56b46a5 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -87,6 +87,21 @@ }, "project_id": { "type": ["null", "integer"] + }, + "erased_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "failure_reason": { + "type": ["null", "string"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "ci_job_token_scope_enabled": { + "type": ["null", "boolean"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json index 694b5d7d729b..0045a1443c16 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -35,6 +35,10 @@ "type": ["null", "string"], "format": "date-time" }, + "prepared_at": { + "type": ["null", "string"], + "format": "date-time" + }, "closed_by": { "type": ["null", "object"] }, @@ -193,6 +197,38 @@ }, "merge_request_iid": { "type": ["null", "integer"] + }, + "draft": { + "type": ["null", "boolean"] + }, + "detailed_merge_status": { + "type": ["null", "string"] + }, + "squash_on_merge": { + "type": ["null", "boolean"] + }, + "merge_user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json index 950daaf36183..ae8197c69bdd 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_requests.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -24,6 +24,10 @@ "type": ["null", "string"], "format": "date-time" }, + "prepared_at": { + "type": ["null", "string"], + "format": "date-time" + }, "updated_at": { "type": ["null", "string"], "format": "date-time" @@ -160,6 +164,38 @@ }, "merged_by_id": { "type": ["null", "integer"] + }, + "draft": { + "type": ["null", "boolean"] + }, + "detailed_merge_status": { + "type": ["null", "string"] + }, + "squash_on_merge": { + "type": ["null", "boolean"] + }, + "merge_user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines.json index a90546bc4c13..aa3fc6b3a840 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines.json @@ -1,16 +1,22 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { "type": ["null", "integer"] }, + "iid": { + "type": ["null", "integer"] + }, "project_id": { "type": ["null", "integer"] }, "sha": { "type": ["null", "string"] }, + "source": { + "type": ["null", "string"] + }, "ref": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json index 2b4bc0b1c5fa..5bd2853879a2 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/pipelines_extended.json @@ -1,16 +1,22 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { "type": ["null", "integer"] }, + "iid": { + "type": ["null", "integer"] + }, "project_id": { "type": ["null", "integer"] }, "sha": { "type": ["null", "string"] }, + "source": { + "type": ["null", "string"] + }, "ref": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_labels.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_labels.json index 000b3ddb6839..4fe31f07a89a 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_labels.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_labels.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "project_id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json index 19ba6b1d0fab..691e9ca74768 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "project_id": { @@ -17,6 +17,9 @@ "state": { "type": ["null", "string"] }, + "membership_state": { + "type": ["null", "string"] + }, "avatar_url": { "type": ["null", "string"] }, @@ -33,6 +36,26 @@ "expires_at": { "type": ["null", "string"], "format": "date-time" + }, + "created_by": { + "avatar_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_milestones.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_milestones.json index fe9f9c1dfb4e..a6ff1d7065e0 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_milestones.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_milestones.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json index 26fbcd9843a0..3e4a9ce4b9ab 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -379,6 +379,98 @@ } } } + }, + "feature_flags_access_level": { + "type": ["null", "string"] + }, + "group_runners_enabled": { + "type": ["null", "boolean"] + }, + "enforce_auth_checks_on_uploads": { + "type": ["null", "boolean"] + }, + "monitor_access_level": { + "type": ["null", "string"] + }, + "container_registry_access_level": { + "type": ["null", "string"] + }, + "import_type": { + "type": ["null", "string"] + }, + "ci_job_token_scope_enabled": { + "type": ["null", "boolean"] + }, + "requirements_access_level": { + "type": ["null", "string"] + }, + "releases_access_level": { + "type": ["null", "string"] + }, + "runner_token_expiration_interval": { + "type": ["null", "string"] + }, + "squash_option": { + "type": ["null", "string"] + }, + "squash_commit_template": { + "type": ["null", "string"] + }, + "issue_branch_template": { + "type": ["null", "string"] + }, + "keep_latest_artifact": { + "type": ["null", "boolean"] + }, + "import_url": { + "type": ["null", "string"] + }, + "ci_separated_caches": { + "type": ["null", "boolean"] + }, + "security_and_compliance_access_level": { + "type": ["null", "string"] + }, + "infrastructure_access_level": { + "type": ["null", "string"] + }, + "merge_commit_template": { + "type": ["null", "string"] + }, + "ci_allow_fork_pipelines_to_run_in_parent_project": { + "type": ["null", "boolean"] + }, + "environments_access_level": { + "type": ["null", "string"] + }, + "approvals_before_merge": { + "type": ["null", "integer"] + }, + "marked_for_deletion_at": { + "type": ["null", "string"], + "format": "date" + }, + "merge_trains_enabled": { + "type": ["null", "boolean"] + }, + "mirror": { + "type": ["null", "boolean"] + }, + "issues_template": { + "type": ["null", "string"] + }, + "merge_pipelines_enabled": { + "type": ["null", "boolean"] + }, + "merge_requests_template": { + "type": ["null", "string"] + }, + "allow_pipeline_trigger_approve_deployment": { + "type": ["null", "boolean"] + }, + "marked_for_deletion_on": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/releases.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/releases.json index 33b89efe81e3..43b311851113 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/releases.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/releases.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "name": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/tags.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/tags.json index 22d22a3b60d1..ae6dc1f038a9 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/tags.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/tags.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "name": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json index 081ebd85cc73..9c6369ce1721 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index 1e1a106bc39c..de3545dc89c8 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -3,8 +3,11 @@ # +import os from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union +import pendulum +from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -40,12 +43,45 @@ Tags, Users, ) +from .utils import parse_url + + +class SingleUseRefreshTokenGitlabOAuth2Authenticator(SingleUseRefreshTokenOauth2Authenticator): + def __init__(self, *args, created_at_name: str = "created_at", **kwargs): + super().__init__(*args, **kwargs) + self._created_at_name = created_at_name + + def get_created_at_name(self) -> str: + return self._created_at_name + + def get_access_token(self) -> str: + if self.token_has_expired(): + new_access_token, access_token_expires_in, access_token_created_at, new_refresh_token = self.refresh_access_token() + new_token_expiry_date = self.get_new_token_expiry_date(access_token_expires_in, access_token_created_at) + self.access_token = new_access_token + self.set_refresh_token(new_refresh_token) + self.set_token_expiry_date(new_token_expiry_date) + emit_configuration_as_airbyte_control_message(self._connector_config) + return self.access_token + + @staticmethod + def get_new_token_expiry_date(access_token_expires_in: int, access_token_created_at: int) -> pendulum.DateTime: + return pendulum.from_timestamp(access_token_created_at + access_token_expires_in) + + def refresh_access_token(self) -> Tuple[str, int, int, str]: + response_json = self._get_refresh_access_token_response() + return ( + response_json[self.get_access_token_name()], + response_json[self.get_expires_in_name()], + response_json[self.get_created_at_name()], + response_json[self.get_refresh_token_name()], + ) def get_authenticator(config: MutableMapping) -> AuthBase: if config["credentials"]["auth_type"] == "access_token": return TokenAuthenticator(token=config["credentials"]["access_token"]) - return SingleUseRefreshTokenOauth2Authenticator(config, token_refresh_endpoint=f"https://{config['api_url']}/oauth/token") + return SingleUseRefreshTokenGitlabOAuth2Authenticator(config, token_refresh_endpoint=f"https://{config['api_url']}/oauth/token") class SourceGitlab(AbstractSource): @@ -55,6 +91,11 @@ def __init__(self, *args, **kwargs): self.__groups_stream: Optional[GitlabStream] = None self.__projects_stream: Optional[GitlabStream] = None + @staticmethod + def _ensure_default_values(config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + config["api_url"] = config.get("api_url") or "gitlab.com" + return config + def _groups_stream(self, config: MutableMapping[str, Any]) -> Groups: if not self.__groups_stream: auth_params = self._auth_params(config) @@ -95,16 +136,28 @@ def _get_group_list(self, config: MutableMapping[str, Any]) -> List[str]: for stream_slice in stream.stream_slices(sync_mode=SyncMode.full_refresh): yield from stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + @staticmethod + def _is_http_allowed() -> bool: + return os.environ.get("DEPLOYMENT_MODE", "").upper() != "CLOUD" + def check_connection(self, logger, config) -> Tuple[bool, any]: + config = self._ensure_default_values(config) + is_valid, scheme, _ = parse_url(config["api_url"]) + if not is_valid: + return False, "Invalid API resource locator." + if scheme == "http" and not self._is_http_allowed(): + return False, "Http scheme is not allowed in this environment. Please use `https` instead." try: projects = self._projects_stream(config) for stream_slice in projects.stream_slices(sync_mode=SyncMode.full_refresh): next(projects.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) return True, None + return True, None # in case there's no projects except Exception as error: return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(error)}" def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: + config = self._ensure_default_values(config) auth_params = self._auth_params(config) groups, projects = self._groups_stream(config), self._projects_stream(config) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json index 0e5dd6ff8daa..9061dadafb07 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source Gitlab Spec", "type": "object", - "required": ["api_url", "start_date", "credentials"], + "required": ["start_date", "credentials"], "additionalProperties": true, "properties": { "credentials": { @@ -73,23 +73,27 @@ } ] }, - "api_url": { - "type": "string", - "examples": ["gitlab.com"], - "title": "API URL", - "default": "gitlab.com", - "description": "Please enter your basic URL from GitLab instance.", - "order": 1 - }, "start_date": { "type": "string", "title": "Start Date", "description": "The date from which you'd like to replicate data for GitLab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", "examples": ["2021-03-01T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 2, + "order": 1, "format": "date-time" }, + "api_url": { + "type": "string", + "examples": [ + "gitlab.com", + "https://gitlab.com", + "https://gitlab.company.org" + ], + "title": "API URL", + "default": "gitlab.com", + "description": "Please enter your basic URL from GitLab instance.", + "order": 2 + }, "groups": { "type": "string", "examples": ["airbyte.io"], diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index 49ddad6ad7ae..de655d03e974 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -5,7 +5,6 @@ import datetime from abc import ABC from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import urlparse import pendulum import requests @@ -14,6 +13,8 @@ from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream +from .utils import parse_url + class GitlabStream(HttpStream, ABC): primary_key = "id" @@ -22,7 +23,7 @@ class GitlabStream(HttpStream, ABC): flatten_id_keys = [] flatten_list_keys = [] per_page = 50 - non_retriable_codes: List[int] = (403,) + non_retriable_codes: List[int] = (403, 404) def __init__(self, api_url: str, **kwargs): super().__init__(**kwargs) @@ -53,12 +54,7 @@ def request_params( @property def url_base(self) -> str: - parse_result = urlparse(self.api_url) - # Default scheme to "https" if URL doesn't contain - scheme = parse_result.scheme if parse_result.scheme else "https" - # hostname without a scheme will result in `path` attribute - # Use path if netloc is not detected - host = parse_result.netloc if parse_result.netloc else parse_result.path + _, scheme, host = parse_url(self.api_url) return f"{scheme}://{host}/api/v4/" @property @@ -311,7 +307,6 @@ class Branches(GitlabChildStream): primary_key = "name" flatten_id_keys = ["commit"] flatten_parent_id = True - non_retriable_codes = (403, 404) class Commits(IncrementalGitlabChildStream): diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/utils.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/utils.py new file mode 100644 index 000000000000..0b5605260a40 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/utils.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Tuple + + +def parse_url(url: str) -> Tuple[bool, str, str]: + parts = url.split("://") + if len(parts) > 1: + scheme, url = parts + else: + scheme = "https" + if scheme not in ("http", "https"): + return False, "", "" + parts = url.split("/", 1) + if len(parts) > 1: + return False, "", "" + host, *_ = parts + return True, scheme, host diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py index f589f2a6b7cf..d250f5489258 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py @@ -5,7 +5,7 @@ import pytest -@pytest.fixture(params=["gitlab.com", "https://gitlab.com", "https://gitlab.com/api/v4"]) +@pytest.fixture(params=["gitlab.com", "http://gitlab.com", "https://gitlab.com"]) def config(request): return { "start_date": "2021-01-01T00:00:00Z", @@ -15,3 +15,17 @@ def config(request): "access_token": "token" } } + + +@pytest.fixture(autouse=True) +def disable_cache(mocker): + mocker.patch( + "source_gitlab.streams.Projects.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_gitlab.streams.Groups.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index 4b120bad78fb..1c2b2a54757b 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -4,6 +4,7 @@ import logging +import pytest from source_gitlab import SourceGitlab from source_gitlab.streams import GitlabStream @@ -19,18 +20,52 @@ def test_streams(config, requests_mock): assert projects.project_ids == [] -def test_connection_success(config, requests_mock): - requests_mock.get("/api/v4/groups", json=[{"id": "g1"}]) - requests_mock.get("/api/v4/groups/g1", json=[{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]) - requests_mock.get("/api/v4/projects/p1", json={"id": "p1"}) +@pytest.mark.parametrize( + "url_mocks", + ( + ( + {"url": "/api/v4/groups", "json": [{"id": "g1"}]}, + {"url": "/api/v4/groups/g1", "json": [{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]}, + {"url": "/api/v4/projects/p1", "json": {"id": "p1"}} + ), + ( + {"url": "/api/v4/groups", "json": []}, + ), + ) +) +def test_connection_success(config, requests_mock, url_mocks): + for url_mock in url_mocks: + requests_mock.get(**url_mock) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config) assert (status, msg) == (True, None) -def test_connection_fail(config, mocker, requests_mock): +def test_connection_fail_due_to_api_error(config, mocker, requests_mock): mocker.patch("time.sleep") requests_mock.get("/api/v4/groups", status_code=500) source = SourceGitlab() status, msg = source.check_connection(logging.getLogger(), config) assert status is False, msg.startswith('Unable to connect to Gitlab API with the provided credentials - "DefaultBackoffException"') + + +@pytest.mark.parametrize( + "api_url, deployment_env, expected_message", + ( + ("http://gitlab.my.company.org", "CLOUD", "Http scheme is not allowed in this environment. Please use `https` instead."), + ("https://gitlab.com/api/v4", "CLOUD", "Invalid API resource locator.") + ) +) +def test_connection_fail_due_to_config_error(mocker, api_url, deployment_env, expected_message): + mocker.patch("os.environ", {"DEPLOYMENT_MODE": deployment_env}) + source = SourceGitlab() + config = { + "start_date": "2021-01-01T00:00:00Z", + "api_url": api_url, + "credentials": { + "auth_type": "access_token", + "access_token": "token" + } + } + status, msg = source.check_connection(logging.getLogger(), config) + assert (status, msg) == (False, expected_message) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py index af266a088596..2ed3aa88ac14 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -9,32 +9,100 @@ from source_gitlab.streams import Branches, Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"} +start_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=14) -start_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=14) -projects = Projects(project_ids=["p_1"], **auth_params) -pipelines = Pipelines(parent_stream=projects, start_date=str(start_date), **auth_params) -merge_requests = MergeRequests(parent_stream=projects, start_date=str(start_date), **auth_params) -tags = Tags(parent_stream=projects, repository_part=True, **auth_params) -releases = Releases(parent_stream=projects, **auth_params) -jobs = Jobs(parent_stream=pipelines, **auth_params) -merge_request_commits = MergeRequestCommits(parent_stream=merge_requests, **auth_params) -branches = Branches(parent_stream=projects, **auth_params) -commits = Commits(parent_stream=projects, repository_part=True, start_date=str(start_date), **auth_params) - - -def test_should_retry(mocker, requests_mock): +@pytest.fixture() +def projects(): + return Projects(project_ids=["p_1"], **auth_params) + + +@pytest.fixture() +def pipelines(projects): + return Pipelines(parent_stream=projects, start_date=str(start_date), **auth_params) + + +@pytest.fixture() +def merge_requests(projects): + return MergeRequests(parent_stream=projects, start_date=str(start_date), **auth_params) + + +@pytest.fixture() +def tags(projects): + return Tags(parent_stream=projects, repository_part=True, **auth_params) + + +@pytest.fixture() +def releases(projects): + return Releases(parent_stream=projects, **auth_params) + + +@pytest.fixture() +def jobs(pipelines): + return Jobs(parent_stream=pipelines, **auth_params) + + +@pytest.fixture() +def merge_request_commits(merge_requests): + return MergeRequestCommits(parent_stream=merge_requests, **auth_params) + + +@pytest.fixture() +def branches(projects): + return Branches(parent_stream=projects, **auth_params) + + +@pytest.fixture() +def commits(projects): + return Commits(parent_stream=projects, repository_part=True, start_date=str(start_date), **auth_params) + + +@pytest.mark.parametrize( + "stream, extra_mocks, expected_call_count", + ( + ( + "projects", + ({"url": "/api/v4/projects/p_1", "status_code": 403},), + 1 + ), + ( + "projects", + ({"url": "/api/v4/projects/p_1", "status_code": 404},), + 1 + ), + ( + "branches", + ( + {"url": "/api/v4/projects/p_1", "json": [{"id": "p_1"}]}, + {"url": "/api/v4/projects/p_1/branches", "status_code": 403} + ), + 2 + ), + ( + "branches", + ( + {"url": "/api/v4/projects/p_1", "json": [{"id": "p_1"}]}, + {"url": "/api/v4/projects/p_1/branches", "status_code": 404} + ), + 2 + ), + ), +) +def test_should_retry(mocker, requests_mock, stream, extra_mocks, expected_call_count, request): mocker.patch("time.sleep") - requests_mock.get("/api/v4/projects/p_1", status_code=403) - for stream_slice in projects.stream_slices(sync_mode="full_refresh"): - records = list(projects.read_records(sync_mode="full_refresh", stream_slice=stream_slice)) + stream = request.getfixturevalue(stream) + for extra_mock in extra_mocks: + requests_mock.get(**extra_mock) + + for stream_slice in stream.stream_slices(sync_mode="full_refresh"): + records = list(stream.read_records(sync_mode="full_refresh", stream_slice=stream_slice)) assert records == [] - assert requests_mock.call_count == 1 + assert requests_mock.call_count == expected_call_count test_cases = ( ( - jobs, + "jobs", ( ("/api/v4/projects/p_1/pipelines", [{"project_id": "p_1", "id": "build_project_p1"}],), ( @@ -47,14 +115,14 @@ def test_should_retry(mocker, requests_mock): [{"commit": {"id": "c_23"}, "commit_id": "c_23", "id": "j_1", "pipeline": {"id": "p_17"}, "pipeline_id": "p_17", "project_id": "p_1", "runner": None, "runner_id": None, "user": {"id": "u_1"}, "user_id": "u_1"}] ), ( - tags, + "tags", ( ("/api/v4/projects/p_1/repository/tags", [{"commit": {"id": "c_1"}, "name": "t_1", "target": "ddc89"}],), ), [{"commit": {"id": "c_1"}, "commit_id": "c_1", "project_id": "p_1", "name": "t_1", "target": "ddc89"}] ), ( - releases, + "releases", ( ( "/api/v4/projects/p_1/releases", @@ -71,7 +139,7 @@ def test_should_retry(mocker, requests_mock): [{"author": {"id": "666", "name": "John"}, "author_id": "666", "commit": {"id": "abcd689"}, "commit_id": "abcd689", "id": "r_1", "milestones": ["m1", "m2"], "project_id": "p_1"}] ), ( - merge_request_commits, + "merge_request_commits", ( ("/api/v4/projects/p_1/merge_requests", [{"id": "mr_1", "iid": "mr_1", "project_id": "p_1"}],), ("/api/v4/projects/p_1/merge_requests/mr_1", [{"id": "mrc_1",}],), @@ -82,7 +150,8 @@ def test_should_retry(mocker, requests_mock): @pytest.mark.parametrize("stream, response_mocks, expected_records", test_cases) -def test_transform(requests_mock, stream, response_mocks, expected_records): +def test_transform(requests_mock, stream, response_mocks, expected_records, request): + stream = request.getfixturevalue(stream) requests_mock.get("/api/v4/projects/p_1", json=[{"id": "p_1"}]) for url, json in response_mocks: @@ -98,50 +167,43 @@ def test_transform(requests_mock, stream, response_mocks, expected_records): "stream, current_state, latest_record, new_state", ( ( - pipelines, + "pipelines", {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}}, {"project_id": "219445", "updated_at": "2022-12-16T00:12:41.005675+02:00"}, {"219445": {"updated_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}} ), ( - pipelines, + "pipelines", {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}}, {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.011+02:00"}, {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}} ), ( - pipelines, + "pipelines", {}, {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.010001+02:00"}, {"211378": {"updated_at": "2021-03-10T23:58:58.010001+02:00"}} ), ( - commits, + "commits", {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}}, {"project_id": "219445", "created_at": "2022-12-16T00:12:41.005675+02:00"}, {"219445": {"created_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}} ), ( - commits, + "commits", {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}}, {"project_id": "211378", "created_at": "2021-03-10T23:58:58.011+02:00"}, {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}} ), ( - commits, + "commits", {}, {"project_id": "211378", "created_at": "2021-03-10T23:58:58.010001+02:00"}, {"211378": {"created_at": "2021-03-10T23:58:58.010001+02:00"}} ) ) ) -def test_updated_state(stream, current_state, latest_record, new_state): +def test_updated_state(stream, current_state, latest_record, new_state, request): + stream = request.getfixturevalue(stream) assert stream.get_updated_state(current_state, latest_record) == new_state - - -def test_blocked_branches(requests_mock): - requests_mock.get("/api/v4/projects/p_1/branches", status_code=404) - for stream_slice in branches.stream_slices(sync_mode="full_refresh"): - records = list(branches.read_records(sync_mode="full_refresh", stream_slice=stream_slice)) - assert records == [] - assert requests_mock.call_count == 1 diff --git a/airbyte-integrations/connectors/source-glassfrog/Dockerfile b/airbyte-integrations/connectors/source-glassfrog/Dockerfile index 80ae0eda635b..6ac45d8e9f95 100644 --- a/airbyte-integrations/connectors/source-glassfrog/Dockerfile +++ b/airbyte-integrations/connectors/source-glassfrog/Dockerfile @@ -34,5 +34,5 @@ COPY source_glassfrog ./source_glassfrog ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-glassfrog diff --git a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml index 989aac2555fb..4e8922bd6d22 100644 --- a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: cf8ff320-6272-4faa-89e6-4402dc17e5d5 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-glassfrog githubIssueLabel: source-glassfrog icon: glassfrog.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/glassfrog tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-glassfrog/requirements.txt b/airbyte-integrations/connectors/source-glassfrog/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-glassfrog/requirements.txt +++ b/airbyte-integrations/connectors/source-glassfrog/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-glassfrog/setup.py b/airbyte-integrations/connectors/source-glassfrog/setup.py index 454f9a5d569f..66e81d3d6700 100644 --- a/airbyte-integrations/connectors/source-glassfrog/setup.py +++ b/airbyte-integrations/connectors/source-glassfrog/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json index 63b4024f04b2..c1ee6ca93121 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -4,47 +4,39 @@ "additionalProperties": true, "properties": { "id": { - "type": "integer", - "default": 0 + "type": ["null", "integer"] }, "tag_names": { - "type": "array", - "default": [], + "type": ["null", "array"], "items": { - "type": "string", - "default": "" + "type": ["null", "string"] } }, "name": { - "type": "string", - "default": "" + "type": ["null", "string"] + }, + "settings": { + "type": ["null", "object"] }, "email": { - "type": "string", - "default": "" + "type": ["null", "string"] }, "external_id": { - "type": ["null", "integer"], - "default": null + "type": ["null", "integer"] }, "links": { - "type": "object", - "default": {}, + "type": ["null", "object"], "properties": { "circles": { - "type": "array", - "default": [], + "type": ["null", "array"], "items": { - "type": ["null", "integer"], - "default": null + "type": ["null", "integer"] } }, "organization_ids": { - "type": "array", - "default": [], + "type": ["null", "array"], "items": { - "type": ["null", "integer"], - "default": null + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-gnews/metadata.yaml b/airbyte-integrations/connectors/source-gnews/metadata.yaml index da23d3a6e264..0f8a7ba6b189 100644 --- a/airbyte-integrations/connectors/source-gnews/metadata.yaml +++ b/airbyte-integrations/connectors/source-gnews/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gnews/requirements.txt b/airbyte-integrations/connectors/source-gnews/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-gnews/requirements.txt +++ b/airbyte-integrations/connectors/source-gnews/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gnews/setup.py b/airbyte-integrations/connectors/source-gnews/setup.py index 520ee9733c07..3e4cf6f2cc28 100644 --- a/airbyte-integrations/connectors/source-gnews/setup.py +++ b/airbyte-integrations/connectors/source-gnews/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-gocardless/metadata.yaml b/airbyte-integrations/connectors/source-gocardless/metadata.yaml index 33718850d690..d49af139e575 100644 --- a/airbyte-integrations/connectors/source-gocardless/metadata.yaml +++ b/airbyte-integrations/connectors/source-gocardless/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gocardless/requirements.txt b/airbyte-integrations/connectors/source-gocardless/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-gocardless/requirements.txt +++ b/airbyte-integrations/connectors/source-gocardless/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gocardless/setup.py b/airbyte-integrations/connectors/source-gocardless/setup.py index 0aabf6de87f5..b49d3b8111e0 100644 --- a/airbyte-integrations/connectors/source-gocardless/setup.py +++ b/airbyte-integrations/connectors/source-gocardless/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-gong/metadata.yaml b/airbyte-integrations/connectors/source-gong/metadata.yaml index 606f9db10b3a..08864fcb3190 100644 --- a/airbyte-integrations/connectors/source-gong/metadata.yaml +++ b/airbyte-integrations/connectors/source-gong/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gong/requirements.txt b/airbyte-integrations/connectors/source-gong/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-gong/requirements.txt +++ b/airbyte-integrations/connectors/source-gong/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gong/setup.py b/airbyte-integrations/connectors/source-gong/setup.py index 24146245720d..e87d4ea56771 100644 --- a/airbyte-integrations/connectors/source-gong/setup.py +++ b/airbyte-integrations/connectors/source-gong/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 25013af67f2d..6727c54ae432 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.22 +LABEL io.airbyte.version=0.7.4 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index f8426aa37913..f0c3f9051b53 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -26,6 +26,10 @@ acceptance_tests: bypass_reason: "Floating data" - name: "display_topics_performance_report" bypass_reason: "Stream not filled yet." + - name: "account_labels" + bypass_reason: "Unable to seed the stream" + - name: "ad_group_criterion_labels" + bypass_reason: "Unable to seed the stream" - name: "click_view" bypass_reason: "Stream not filled yet." - name: "unhappytable" @@ -38,6 +42,10 @@ acceptance_tests: configured_catalog_path: "integration_tests/configured_catalog.json" - config_path: "secrets/config_with_gaql.json" configured_catalog_path: "integration_tests/configured_catalog_with_gaql_only.json" + # This config allows to read from the click_view stream which is empty in other configs. + # It should be tested anyway because it has different date range compared to other streams. + - config_path: "secrets/config_click_view.json" + configured_catalog_path: "integration_tests/configured_catalog_with_click_view.json" incremental: tests: - config_path: "secrets/incremental_config.json" @@ -56,6 +64,7 @@ acceptance_tests: ad_groups: ["4651612872", "segments.date"] accounts: ["4651612872", "segments.date"] campaigns: ["4651612872", "segments.date"] + campaign_budget: ["4651612872", "segments.date"] user_location_report: ["4651612872", "segments.date"] ad_group_ad_report: ["4651612872", "segments.date"] display_keyword_performance_report: ["4651612872", "segments.date"] diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json index 9241aad423b4..939fc0556265 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json @@ -2,92 +2,113 @@ { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "account_performance_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "click_view" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "geographic_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "keyword_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "display_topics_performance_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "shopping_performance_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "ad_group_ads" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "ad_groups" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "accounts" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "campaigns" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "user_location_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "ad_group_ad_report" } } }, { "type": "STREAM", "stream": { - "stream_state": { "4651612872": { "segments.date": "2222-01-01" }}, + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, "stream_descriptor": { "name": "display_keyword_performance_report" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, + "stream_descriptor": { "name": "campaign_budget" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, + "stream_descriptor": { "name": "campaign_bidding_strategies" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, + "stream_descriptor": { "name": "ad_group_bidding_strategies" } + } } ] diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index 06de28d488af..7e19621f3bb4 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -246,6 +246,131 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "audience", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "user_interest", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "labels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["label.id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["label.id"]] + }, + { + "stream": { + "name": "account_labels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["customer_label.resource_name"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["customer_label.resource_name"]] + }, + { + "stream": { + "name": "campaign_bidding_strategies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "ad_group_bidding_strategies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "ad_group_criterions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"]] + }, + { + "stream": { + "name": "ad_listing_group_criterions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"]] + }, + { + "stream": { + "name": "ad_group_criterion_labels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [ + ["ad_group_criterion_label.resource_name"] + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["ad_group_criterion_label.resource_name"]] } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_click_view.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_click_view.json new file mode 100644 index 000000000000..d7edf73ae046 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_click_view.json @@ -0,0 +1,18 @@ +{ + "streams": [ + { + "stream": { + "name": "click_view", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "source_defined_primary_key": [["click_view.gclid"], ["segments.date"]], + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"], + "primary_key": [["click_view.gclid"], ["segments.date"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_gaql_only.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_gaql_only.json index d222571101c3..2804c77d24dc 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_gaql_only.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_with_gaql_only.json @@ -11,4 +11,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl index 1c85b69d1b12..830126123dfc 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl @@ -18,16 +18,16 @@ {"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137051662444, "segments.date": "2022-04-09"}, "emitted_at": 1671617280447} {"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137020701042, "segments.date": "2022-04-10"}, "emitted_at": 1671617280448} {"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-04-10"}, "emitted_at": 1671617280450} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "open source analytics", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 1, "metrics.ctr": 0.0, "segments.date": "2022-02-16", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 1732729676}, "emitted_at": 1671617284427} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "etl pipeline", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 3, "metrics.ctr": 0.0, "segments.date": "2022-02-16", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 297585172071}, "emitted_at": 1671617284428} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "open source etl", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 7, "metrics.ctr": 0.0, "segments.date": "2022-02-16", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 298270671583}, "emitted_at": 1671617284429} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "data connectors", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 5, "metrics.ctr": 0.0, "segments.date": "2022-02-16", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 303376179543}, "emitted_at": 1671617284429} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "open source analytics", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 1, "metrics.ctr": 0.0, "segments.date": "2022-02-17", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 1732729676}, "emitted_at": 1671617284430} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "etl pipeline", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 3, "metrics.ctr": 0.0, "segments.date": "2022-02-17", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 297585172071}, "emitted_at": 1671617284430} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "open source etl", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 7, "metrics.ctr": 0.0, "segments.date": "2022-02-17", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 298270671583}, "emitted_at": 1671617284431} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "data connectors", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 5, "metrics.ctr": 0.0, "segments.date": "2022-02-17", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 303376179543}, "emitted_at": 1671617284431} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "open source analytics", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 1, "metrics.ctr": 0.0, "segments.date": "2022-02-18", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 1732729676}, "emitted_at": 1671617284432} -{"stream": "keyword_report", "data": {"customer.descriptive_name": "", "ad_group.id": 123273719655, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "etl pipeline", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 3, "metrics.ctr": 0.0, "segments.date": "2022-02-18", "campaign.bidding_strategy_type": "TARGET_SPEND", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 0, "ad_group_criterion.criterion_id": 297585172071}, "emitted_at": 1671617284432} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source analytics","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":1,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":1732729676},"emitted_at":1688988446821} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"etl pipeline","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":3,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":297585172071},"emitted_at":1688988446824} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source etl","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":7,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":298270671583},"emitted_at":1688988446825} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"data connectors","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":5,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":303376179543},"emitted_at":1688988446826} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source analytics","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":1,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":1732729676},"emitted_at":1688988446827} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"etl pipeline","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":3,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":297585172071},"emitted_at":1688988446827} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source etl","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":7,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":298270671583},"emitted_at":1688988446828} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"data connectors","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":5,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":303376179543},"emitted_at":1688988446829} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source analytics","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":1,"metrics.ctr":0,"segments.date":"2022-02-17","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":1732729676},"emitted_at":1688988446829} +{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":137020701042,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"airbytes","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":10,"metrics.ctr":0.06666666666666667,"segments.date":"2022-05-14","campaign.bidding_strategy_type":"MAXIMIZE_CONVERSIONS","metrics.clicks":2,"metrics.cost_micros":50000,"metrics.impressions":30,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":2,"metrics.interaction_event_types":["InteractionEventType.CLICK"],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":423065099654},"emitted_at":1688988450597} {"stream": "display_keyword_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 10012000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 1, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10012, "metrics.active_view_measurable_impressions": 1, "metrics.active_view_viewability": 1.0, "ad_group.id": 143992182864, "ad_group.name": "Video Non-skippable - 2022-05-30", "ad_group.status": "ENABLED", "segments.ad_network_type": "YOUTUBE_WATCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 10012000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/143992182864", "campaign.base_campaign": "customers/4651612872/campaigns/17354032686", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "TARGET_CPM", "campaign.id": 17354032686, "campaign.name": "Video Non-skippable - 2022-05-30", "campaign.status": "ENABLED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10012, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.keyword.text": "big data software", "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.day_of_week": "TUESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 26160872903, "metrics.impressions": 1, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: false\n"], "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-30", "segments.year": 2022, "segments.date": "2022-05-31"}, "emitted_at": 1671617298755} {"stream": "campaign_labels", "data": {"campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1671617384908} {"stream": "campaign_labels", "data": {"campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1671617384909} @@ -69,18 +69,32 @@ {"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137051662444,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group.name":"Группа объявлений 1","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":76,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"REMOVED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.2894736842105263,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005471} {"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137020701042,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137020701042","ad_group.name":"Группа объявлений 2","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH_PARTNERS","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137020701042","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED_LIMITED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078631218,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":16,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"ENABLED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.0,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005476} {"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137051662444,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group.name":"Группа объявлений 1","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH_PARTNERS","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":28,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"REMOVED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.0,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005481} -{"stream":"ad_group_ads","data":{"ad_group_ad.ad.added_by_google_ads":false,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"ad_group_ad.ad.app_engagement_ad.descriptions":[],"ad_group_ad.ad.app_engagement_ad.headlines":[],"ad_group_ad.ad.app_engagement_ad.images":[],"ad_group_ad.ad.app_engagement_ad.videos":[],"ad_group_ad.ad.call_ad.business_name":"","ad_group_ad.ad.call_ad.call_tracked":false,"ad_group_ad.ad.call_ad.conversion_action":"","ad_group_ad.ad.call_ad.conversion_reporting_state":"UNSPECIFIED","ad_group_ad.ad.call_ad.country_code":"","ad_group_ad.ad.call_ad.description1":"","ad_group_ad.ad.call_ad.description2":"","ad_group_ad.ad.call_ad.disable_call_conversion":false,"ad_group_ad.ad.call_ad.headline1":"","ad_group_ad.ad.call_ad.headline2":"","ad_group_ad.ad.call_ad.path1":"","ad_group_ad.ad.call_ad.path2":"","ad_group_ad.ad.call_ad.phone_number":"","ad_group_ad.ad.call_ad.phone_number_verification_url":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.display_upload_product_type":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.media_bundle":"","ad_group_ad.ad.display_url":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_dynamic_search_ad.description2":"","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","ad_group_ad.ad.final_app_urls":[],"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_url_suffix":"","ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.hotel_ad":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.preview_image_url":"","ad_group_ad.ad.image_ad.preview_pixel_height":0,"ad_group_ad.ad.image_ad.preview_pixel_width":0,"ad_group_ad.ad.legacy_app_install_ad":"","ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.legacy_responsive_display_ad.description":"","ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.local_ad.call_to_actions":[],"ad_group_ad.ad.local_ad.descriptions":[],"ad_group_ad.ad.local_ad.headlines":[],"ad_group_ad.ad.local_ad.logo_images":[],"ad_group_ad.ad.local_ad.marketing_images":[],"ad_group_ad.ad.local_ad.path1":"","ad_group_ad.ad.local_ad.path2":"","ad_group_ad.ad.local_ad.videos":[],"ad_group_ad.ad.name":"","ad_group_ad.ad.resource_name":"customers/4651612872/ads/592078676857","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements":false,"ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video":false,"ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.shopping_comparison_listing_ad.headline":"","ad_group_ad.ad.shopping_product_ad":"","ad_group_ad.ad.shopping_smart_ad":"","ad_group_ad.ad.smart_campaign_ad.descriptions":[],"ad_group_ad.ad.smart_campaign_ad.headlines":[],"ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","ad_group_ad.ad.url_collections":[],"ad_group_ad.ad.url_custom_parameters":[],"ad_group_ad.ad.video_ad.in_feed.description1":"","ad_group_ad.ad.video_ad.in_feed.description2":"","ad_group_ad.ad.video_ad.in_feed.headline":"","ad_group_ad.ad.video_ad.in_stream.action_button_label":"","ad_group_ad.ad.video_ad.in_stream.action_headline":"","ad_group_ad.ad.video_ad.out_stream.description":"","ad_group_ad.ad.video_ad.out_stream.headline":"","ad_group_ad.ad.video_responsive_ad.call_to_actions":[],"ad_group_ad.ad.video_responsive_ad.companion_banners":[],"ad_group_ad.ad.video_responsive_ad.descriptions":[],"ad_group_ad.ad.video_responsive_ad.headlines":[],"ad_group_ad.ad.video_responsive_ad.long_headlines":[],"ad_group_ad.ad.video_responsive_ad.videos":[],"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group_ad.ad_strength":"POOR","ad_group_ad.labels":[],"ad_group_ad.policy_summary.approval_status":"APPROVED","ad_group_ad.policy_summary.policy_topic_entries":[],"ad_group_ad.policy_summary.review_status":"REVIEWED","ad_group_ad.resource_name":"customers/4651612872/adGroupAds/137051662444~592078676857","ad_group_ad.status":"REMOVED","segments.date":"2022-04-08"},"emitted_at":1679577401999} -{"stream":"ad_group_ads","data":{"ad_group_ad.ad.added_by_google_ads":false,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"ad_group_ad.ad.app_engagement_ad.descriptions":[],"ad_group_ad.ad.app_engagement_ad.headlines":[],"ad_group_ad.ad.app_engagement_ad.images":[],"ad_group_ad.ad.app_engagement_ad.videos":[],"ad_group_ad.ad.call_ad.business_name":"","ad_group_ad.ad.call_ad.call_tracked":false,"ad_group_ad.ad.call_ad.conversion_action":"","ad_group_ad.ad.call_ad.conversion_reporting_state":"UNSPECIFIED","ad_group_ad.ad.call_ad.country_code":"","ad_group_ad.ad.call_ad.description1":"","ad_group_ad.ad.call_ad.description2":"","ad_group_ad.ad.call_ad.disable_call_conversion":false,"ad_group_ad.ad.call_ad.headline1":"","ad_group_ad.ad.call_ad.headline2":"","ad_group_ad.ad.call_ad.path1":"","ad_group_ad.ad.call_ad.path2":"","ad_group_ad.ad.call_ad.phone_number":"","ad_group_ad.ad.call_ad.phone_number_verification_url":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.display_upload_product_type":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.media_bundle":"","ad_group_ad.ad.display_url":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_dynamic_search_ad.description2":"","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","ad_group_ad.ad.final_app_urls":[],"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_url_suffix":"","ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.hotel_ad":"","ad_group_ad.ad.id":592078631218,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.preview_image_url":"","ad_group_ad.ad.image_ad.preview_pixel_height":0,"ad_group_ad.ad.image_ad.preview_pixel_width":0,"ad_group_ad.ad.legacy_app_install_ad":"","ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.legacy_responsive_display_ad.description":"","ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.local_ad.call_to_actions":[],"ad_group_ad.ad.local_ad.descriptions":[],"ad_group_ad.ad.local_ad.headlines":[],"ad_group_ad.ad.local_ad.logo_images":[],"ad_group_ad.ad.local_ad.marketing_images":[],"ad_group_ad.ad.local_ad.path1":"","ad_group_ad.ad.local_ad.path2":"","ad_group_ad.ad.local_ad.videos":[],"ad_group_ad.ad.name":"","ad_group_ad.ad.resource_name":"customers/4651612872/ads/592078631218","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements":false,"ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video":false,"ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.shopping_comparison_listing_ad.headline":"","ad_group_ad.ad.shopping_product_ad":"","ad_group_ad.ad.shopping_smart_ad":"","ad_group_ad.ad.smart_campaign_ad.descriptions":[],"ad_group_ad.ad.smart_campaign_ad.headlines":[],"ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","ad_group_ad.ad.url_collections":[],"ad_group_ad.ad.url_custom_parameters":[],"ad_group_ad.ad.video_ad.in_feed.description1":"","ad_group_ad.ad.video_ad.in_feed.description2":"","ad_group_ad.ad.video_ad.in_feed.headline":"","ad_group_ad.ad.video_ad.in_stream.action_button_label":"","ad_group_ad.ad.video_ad.in_stream.action_headline":"","ad_group_ad.ad.video_ad.out_stream.description":"","ad_group_ad.ad.video_ad.out_stream.headline":"","ad_group_ad.ad.video_responsive_ad.call_to_actions":[],"ad_group_ad.ad.video_responsive_ad.companion_banners":[],"ad_group_ad.ad.video_responsive_ad.descriptions":[],"ad_group_ad.ad.video_responsive_ad.headlines":[],"ad_group_ad.ad.video_responsive_ad.long_headlines":[],"ad_group_ad.ad.video_responsive_ad.videos":[],"ad_group_ad.ad_group":"customers/4651612872/adGroups/137020701042","ad_group_ad.ad_strength":"POOR","ad_group_ad.labels":[],"ad_group_ad.policy_summary.approval_status":"APPROVED_LIMITED","ad_group_ad.policy_summary.policy_topic_entries":["topic: \"TRADEMARKS_IN_AD_TEXT\"\ntype_: LIMITED\nevidences {\n text_list {\n texts: \"airbyte\"\n }\n}\nconstraints {\n reseller_constraint {\n }\n}\n"],"ad_group_ad.policy_summary.review_status":"REVIEWED","ad_group_ad.resource_name":"customers/4651612872/adGroupAds/137020701042~592078631218","ad_group_ad.status":"ENABLED","segments.date":"2022-04-09"},"emitted_at":1679577402006} -{"stream":"ad_group_ads","data":{"ad_group_ad.ad.added_by_google_ads":false,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"ad_group_ad.ad.app_engagement_ad.descriptions":[],"ad_group_ad.ad.app_engagement_ad.headlines":[],"ad_group_ad.ad.app_engagement_ad.images":[],"ad_group_ad.ad.app_engagement_ad.videos":[],"ad_group_ad.ad.call_ad.business_name":"","ad_group_ad.ad.call_ad.call_tracked":false,"ad_group_ad.ad.call_ad.conversion_action":"","ad_group_ad.ad.call_ad.conversion_reporting_state":"UNSPECIFIED","ad_group_ad.ad.call_ad.country_code":"","ad_group_ad.ad.call_ad.description1":"","ad_group_ad.ad.call_ad.description2":"","ad_group_ad.ad.call_ad.disable_call_conversion":false,"ad_group_ad.ad.call_ad.headline1":"","ad_group_ad.ad.call_ad.headline2":"","ad_group_ad.ad.call_ad.path1":"","ad_group_ad.ad.call_ad.path2":"","ad_group_ad.ad.call_ad.phone_number":"","ad_group_ad.ad.call_ad.phone_number_verification_url":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.display_upload_product_type":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.media_bundle":"","ad_group_ad.ad.display_url":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_dynamic_search_ad.description2":"","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","ad_group_ad.ad.final_app_urls":[],"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_url_suffix":"","ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.hotel_ad":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.preview_image_url":"","ad_group_ad.ad.image_ad.preview_pixel_height":0,"ad_group_ad.ad.image_ad.preview_pixel_width":0,"ad_group_ad.ad.legacy_app_install_ad":"","ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.legacy_responsive_display_ad.description":"","ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.local_ad.call_to_actions":[],"ad_group_ad.ad.local_ad.descriptions":[],"ad_group_ad.ad.local_ad.headlines":[],"ad_group_ad.ad.local_ad.logo_images":[],"ad_group_ad.ad.local_ad.marketing_images":[],"ad_group_ad.ad.local_ad.path1":"","ad_group_ad.ad.local_ad.path2":"","ad_group_ad.ad.local_ad.videos":[],"ad_group_ad.ad.name":"","ad_group_ad.ad.resource_name":"customers/4651612872/ads/592078676857","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements":false,"ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video":false,"ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.shopping_comparison_listing_ad.headline":"","ad_group_ad.ad.shopping_product_ad":"","ad_group_ad.ad.shopping_smart_ad":"","ad_group_ad.ad.smart_campaign_ad.descriptions":[],"ad_group_ad.ad.smart_campaign_ad.headlines":[],"ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","ad_group_ad.ad.url_collections":[],"ad_group_ad.ad.url_custom_parameters":[],"ad_group_ad.ad.video_ad.in_feed.description1":"","ad_group_ad.ad.video_ad.in_feed.description2":"","ad_group_ad.ad.video_ad.in_feed.headline":"","ad_group_ad.ad.video_ad.in_stream.action_button_label":"","ad_group_ad.ad.video_ad.in_stream.action_headline":"","ad_group_ad.ad.video_ad.out_stream.description":"","ad_group_ad.ad.video_ad.out_stream.headline":"","ad_group_ad.ad.video_responsive_ad.call_to_actions":[],"ad_group_ad.ad.video_responsive_ad.companion_banners":[],"ad_group_ad.ad.video_responsive_ad.descriptions":[],"ad_group_ad.ad.video_responsive_ad.headlines":[],"ad_group_ad.ad.video_responsive_ad.long_headlines":[],"ad_group_ad.ad.video_responsive_ad.videos":[],"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group_ad.ad_strength":"POOR","ad_group_ad.labels":[],"ad_group_ad.policy_summary.approval_status":"APPROVED","ad_group_ad.policy_summary.policy_topic_entries":[],"ad_group_ad.policy_summary.review_status":"REVIEWED","ad_group_ad.resource_name":"customers/4651612872/adGroupAds/137051662444~592078676857","ad_group_ad.status":"REMOVED","segments.date":"2022-04-09"},"emitted_at":1679577402012} -{"stream":"ad_group_ads","data":{"ad_group_ad.ad.added_by_google_ads":false,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"ad_group_ad.ad.app_engagement_ad.descriptions":[],"ad_group_ad.ad.app_engagement_ad.headlines":[],"ad_group_ad.ad.app_engagement_ad.images":[],"ad_group_ad.ad.app_engagement_ad.videos":[],"ad_group_ad.ad.call_ad.business_name":"","ad_group_ad.ad.call_ad.call_tracked":false,"ad_group_ad.ad.call_ad.conversion_action":"","ad_group_ad.ad.call_ad.conversion_reporting_state":"UNSPECIFIED","ad_group_ad.ad.call_ad.country_code":"","ad_group_ad.ad.call_ad.description1":"","ad_group_ad.ad.call_ad.description2":"","ad_group_ad.ad.call_ad.disable_call_conversion":false,"ad_group_ad.ad.call_ad.headline1":"","ad_group_ad.ad.call_ad.headline2":"","ad_group_ad.ad.call_ad.path1":"","ad_group_ad.ad.call_ad.path2":"","ad_group_ad.ad.call_ad.phone_number":"","ad_group_ad.ad.call_ad.phone_number_verification_url":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.display_upload_product_type":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.media_bundle":"","ad_group_ad.ad.display_url":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_dynamic_search_ad.description2":"","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","ad_group_ad.ad.final_app_urls":[],"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_url_suffix":"","ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.hotel_ad":"","ad_group_ad.ad.id":592078631218,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.preview_image_url":"","ad_group_ad.ad.image_ad.preview_pixel_height":0,"ad_group_ad.ad.image_ad.preview_pixel_width":0,"ad_group_ad.ad.legacy_app_install_ad":"","ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.legacy_responsive_display_ad.description":"","ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.local_ad.call_to_actions":[],"ad_group_ad.ad.local_ad.descriptions":[],"ad_group_ad.ad.local_ad.headlines":[],"ad_group_ad.ad.local_ad.logo_images":[],"ad_group_ad.ad.local_ad.marketing_images":[],"ad_group_ad.ad.local_ad.path1":"","ad_group_ad.ad.local_ad.path2":"","ad_group_ad.ad.local_ad.videos":[],"ad_group_ad.ad.name":"","ad_group_ad.ad.resource_name":"customers/4651612872/ads/592078631218","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements":false,"ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video":false,"ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.shopping_comparison_listing_ad.headline":"","ad_group_ad.ad.shopping_product_ad":"","ad_group_ad.ad.shopping_smart_ad":"","ad_group_ad.ad.smart_campaign_ad.descriptions":[],"ad_group_ad.ad.smart_campaign_ad.headlines":[],"ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","ad_group_ad.ad.url_collections":[],"ad_group_ad.ad.url_custom_parameters":[],"ad_group_ad.ad.video_ad.in_feed.description1":"","ad_group_ad.ad.video_ad.in_feed.description2":"","ad_group_ad.ad.video_ad.in_feed.headline":"","ad_group_ad.ad.video_ad.in_stream.action_button_label":"","ad_group_ad.ad.video_ad.in_stream.action_headline":"","ad_group_ad.ad.video_ad.out_stream.description":"","ad_group_ad.ad.video_ad.out_stream.headline":"","ad_group_ad.ad.video_responsive_ad.call_to_actions":[],"ad_group_ad.ad.video_responsive_ad.companion_banners":[],"ad_group_ad.ad.video_responsive_ad.descriptions":[],"ad_group_ad.ad.video_responsive_ad.headlines":[],"ad_group_ad.ad.video_responsive_ad.long_headlines":[],"ad_group_ad.ad.video_responsive_ad.videos":[],"ad_group_ad.ad_group":"customers/4651612872/adGroups/137020701042","ad_group_ad.ad_strength":"POOR","ad_group_ad.labels":[],"ad_group_ad.policy_summary.approval_status":"APPROVED_LIMITED","ad_group_ad.policy_summary.policy_topic_entries":["topic: \"TRADEMARKS_IN_AD_TEXT\"\ntype_: LIMITED\nevidences {\n text_list {\n texts: \"airbyte\"\n }\n}\nconstraints {\n reseller_constraint {\n }\n}\n"],"ad_group_ad.policy_summary.review_status":"REVIEWED","ad_group_ad.resource_name":"customers/4651612872/adGroupAds/137020701042~592078631218","ad_group_ad.status":"ENABLED","segments.date":"2022-04-10"},"emitted_at":1679577402018} -{"stream":"ad_group_ads","data":{"ad_group_ad.ad.added_by_google_ads":false,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"ad_group_ad.ad.app_engagement_ad.descriptions":[],"ad_group_ad.ad.app_engagement_ad.headlines":[],"ad_group_ad.ad.app_engagement_ad.images":[],"ad_group_ad.ad.app_engagement_ad.videos":[],"ad_group_ad.ad.call_ad.business_name":"","ad_group_ad.ad.call_ad.call_tracked":false,"ad_group_ad.ad.call_ad.conversion_action":"","ad_group_ad.ad.call_ad.conversion_reporting_state":"UNSPECIFIED","ad_group_ad.ad.call_ad.country_code":"","ad_group_ad.ad.call_ad.description1":"","ad_group_ad.ad.call_ad.description2":"","ad_group_ad.ad.call_ad.disable_call_conversion":false,"ad_group_ad.ad.call_ad.headline1":"","ad_group_ad.ad.call_ad.headline2":"","ad_group_ad.ad.call_ad.path1":"","ad_group_ad.ad.call_ad.path2":"","ad_group_ad.ad.call_ad.phone_number":"","ad_group_ad.ad.call_ad.phone_number_verification_url":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.display_upload_product_type":"UNSPECIFIED","ad_group_ad.ad.display_upload_ad.media_bundle":"","ad_group_ad.ad.display_url":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_dynamic_search_ad.description2":"","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","ad_group_ad.ad.final_app_urls":[],"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_url_suffix":"","ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.hotel_ad":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.preview_image_url":"","ad_group_ad.ad.image_ad.preview_pixel_height":0,"ad_group_ad.ad.image_ad.preview_pixel_width":0,"ad_group_ad.ad.legacy_app_install_ad":"","ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.legacy_responsive_display_ad.description":"","ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.local_ad.call_to_actions":[],"ad_group_ad.ad.local_ad.descriptions":[],"ad_group_ad.ad.local_ad.headlines":[],"ad_group_ad.ad.local_ad.logo_images":[],"ad_group_ad.ad.local_ad.marketing_images":[],"ad_group_ad.ad.local_ad.path1":"","ad_group_ad.ad.local_ad.path2":"","ad_group_ad.ad.local_ad.videos":[],"ad_group_ad.ad.name":"","ad_group_ad.ad.resource_name":"customers/4651612872/ads/592078676857","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements":false,"ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video":false,"ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.shopping_comparison_listing_ad.headline":"","ad_group_ad.ad.shopping_product_ad":"","ad_group_ad.ad.shopping_smart_ad":"","ad_group_ad.ad.smart_campaign_ad.descriptions":[],"ad_group_ad.ad.smart_campaign_ad.headlines":[],"ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","ad_group_ad.ad.url_collections":[],"ad_group_ad.ad.url_custom_parameters":[],"ad_group_ad.ad.video_ad.in_feed.description1":"","ad_group_ad.ad.video_ad.in_feed.description2":"","ad_group_ad.ad.video_ad.in_feed.headline":"","ad_group_ad.ad.video_ad.in_stream.action_button_label":"","ad_group_ad.ad.video_ad.in_stream.action_headline":"","ad_group_ad.ad.video_ad.out_stream.description":"","ad_group_ad.ad.video_ad.out_stream.headline":"","ad_group_ad.ad.video_responsive_ad.call_to_actions":[],"ad_group_ad.ad.video_responsive_ad.companion_banners":[],"ad_group_ad.ad.video_responsive_ad.descriptions":[],"ad_group_ad.ad.video_responsive_ad.headlines":[],"ad_group_ad.ad.video_responsive_ad.long_headlines":[],"ad_group_ad.ad.video_responsive_ad.videos":[],"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group_ad.ad_strength":"POOR","ad_group_ad.labels":[],"ad_group_ad.policy_summary.approval_status":"APPROVED","ad_group_ad.policy_summary.policy_topic_entries":[],"ad_group_ad.policy_summary.review_status":"REVIEWED","ad_group_ad.resource_name":"customers/4651612872/adGroupAds/137051662444~592078676857","ad_group_ad.status":"REMOVED","segments.date":"2022-04-10"},"emitted_at":1679577402025} -{"stream":"campaigns","data":{"campaign.accessible_bidding_strategy":"","campaign.ad_serving_optimization_status":"OPTIMIZE","campaign.advertising_channel_sub_type":"UNSPECIFIED","campaign.advertising_channel_type":"SEARCH","campaign.app_campaign_setting.app_id":"","campaign.app_campaign_setting.app_store":"UNSPECIFIED","campaign.app_campaign_setting.bidding_strategy_goal_type":"UNSPECIFIED","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","campaign.bidding_strategy":"","campaign.bidding_strategy_type":"TARGET_SPEND","campaign.campaign_budget":"customers/4651612872/campaignBudgets/10695604507","campaign_budget.amount_micros":750000,"campaign.commission.commission_rate_micros":0,"campaign.dynamic_search_ads_setting.domain_name":"","campaign.dynamic_search_ads_setting.feeds":[],"campaign.dynamic_search_ads_setting.language_code":"","campaign.dynamic_search_ads_setting.use_supplied_urls_only":false,"campaign.end_date":"2037-12-30","campaign.excluded_parent_asset_field_types":[],"campaign.experiment_type":"BASE","campaign.final_url_suffix":"","campaign.frequency_caps":[],"campaign.geo_target_type_setting.negative_geo_target_type":"PRESENCE","campaign.geo_target_type_setting.positive_geo_target_type":"PRESENCE_OR_INTEREST","campaign.hotel_setting.hotel_center_id":0,"campaign.id":16820250687,"campaign.labels":[],"campaign.local_campaign_setting.location_source_type":"UNSPECIFIED","campaign.manual_cpc.enhanced_cpc_enabled":false,"campaign.manual_cpm":"","campaign.manual_cpv":"","campaign.maximize_conversion_value.target_roas":0.0,"campaign.maximize_conversions.target_cpa_micros":0,"campaign.name":"Website traffic-Search-15","campaign.network_settings.target_content_network":true,"campaign.network_settings.target_google_search":true,"campaign.network_settings.target_partner_search_network":false,"campaign.network_settings.target_search_network":true,"campaign.optimization_goal_setting.optimization_goal_types":[],"campaign.optimization_score":0.0,"campaign.payment_mode":"CLICKS","campaign.percent_cpc.cpc_bid_ceiling_micros":0,"campaign.percent_cpc.enhanced_cpc_enabled":false,"campaign.real_time_bidding_setting.opt_in":false,"campaign.resource_name":"customers/4651612872/campaigns/16820250687","campaign.selective_optimization.conversion_actions":[],"campaign.serving_status":"SERVING","campaign.shopping_setting.campaign_priority":0,"campaign.shopping_setting.enable_local":false,"campaign.shopping_setting.merchant_id":0,"campaign.shopping_setting.sales_country":"","campaign.start_date":"2022-04-08","campaign.status":"PAUSED","campaign.target_cpa.cpc_bid_ceiling_micros":0,"campaign.target_cpa.cpc_bid_floor_micros":0,"campaign.target_cpa.target_cpa_micros":0,"campaign.target_cpm.target_frequency_goal.target_count":0,"campaign.target_cpm.target_frequency_goal.time_unit":"UNSPECIFIED","campaign.target_impression_share.cpc_bid_ceiling_micros":0,"campaign.target_impression_share.location":"UNSPECIFIED","campaign.target_impression_share.location_fraction_micros":0,"campaign.target_roas.cpc_bid_ceiling_micros":0,"campaign.target_roas.cpc_bid_floor_micros":0,"campaign.target_roas.target_roas":0.0,"campaign.target_spend.cpc_bid_ceiling_micros":0,"campaign.target_spend.target_spend_micros":0,"campaign.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n"],"campaign.tracking_setting.tracking_url":"","campaign.tracking_url_template":"","campaign.url_custom_parameters":[],"campaign.vanity_pharma.vanity_pharma_display_url_mode":"UNSPECIFIED","campaign.vanity_pharma.vanity_pharma_text":"UNSPECIFIED","campaign.video_brand_safety_suitability":"UNSPECIFIED","metrics.clicks":0,"metrics.ctr":0.0,"metrics.conversions":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":0,"metrics.impressions":2.0,"metrics.video_views":0.0,"metrics.video_quartile_p100_rate":0.0,"segments.date":"2022-04-08","segments.hour":21.0},"emitted_at":1679577498472} -{"stream":"campaigns","data":{"campaign.accessible_bidding_strategy":"","campaign.ad_serving_optimization_status":"OPTIMIZE","campaign.advertising_channel_sub_type":"UNSPECIFIED","campaign.advertising_channel_type":"SEARCH","campaign.app_campaign_setting.app_id":"","campaign.app_campaign_setting.app_store":"UNSPECIFIED","campaign.app_campaign_setting.bidding_strategy_goal_type":"UNSPECIFIED","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","campaign.bidding_strategy":"","campaign.bidding_strategy_type":"TARGET_SPEND","campaign.campaign_budget":"customers/4651612872/campaignBudgets/10695604507","campaign_budget.amount_micros":750000,"campaign.commission.commission_rate_micros":0,"campaign.dynamic_search_ads_setting.domain_name":"","campaign.dynamic_search_ads_setting.feeds":[],"campaign.dynamic_search_ads_setting.language_code":"","campaign.dynamic_search_ads_setting.use_supplied_urls_only":false,"campaign.end_date":"2037-12-30","campaign.excluded_parent_asset_field_types":[],"campaign.experiment_type":"BASE","campaign.final_url_suffix":"","campaign.frequency_caps":[],"campaign.geo_target_type_setting.negative_geo_target_type":"PRESENCE","campaign.geo_target_type_setting.positive_geo_target_type":"PRESENCE_OR_INTEREST","campaign.hotel_setting.hotel_center_id":0,"campaign.id":16820250687,"campaign.labels":[],"campaign.local_campaign_setting.location_source_type":"UNSPECIFIED","campaign.manual_cpc.enhanced_cpc_enabled":false,"campaign.manual_cpm":"","campaign.manual_cpv":"","campaign.maximize_conversion_value.target_roas":0.0,"campaign.maximize_conversions.target_cpa_micros":0,"campaign.name":"Website traffic-Search-15","campaign.network_settings.target_content_network":true,"campaign.network_settings.target_google_search":true,"campaign.network_settings.target_partner_search_network":false,"campaign.network_settings.target_search_network":true,"campaign.optimization_goal_setting.optimization_goal_types":[],"campaign.optimization_score":0.0,"campaign.payment_mode":"CLICKS","campaign.percent_cpc.cpc_bid_ceiling_micros":0,"campaign.percent_cpc.enhanced_cpc_enabled":false,"campaign.real_time_bidding_setting.opt_in":false,"campaign.resource_name":"customers/4651612872/campaigns/16820250687","campaign.selective_optimization.conversion_actions":[],"campaign.serving_status":"SERVING","campaign.shopping_setting.campaign_priority":0,"campaign.shopping_setting.enable_local":false,"campaign.shopping_setting.merchant_id":0,"campaign.shopping_setting.sales_country":"","campaign.start_date":"2022-04-08","campaign.status":"PAUSED","campaign.target_cpa.cpc_bid_ceiling_micros":0,"campaign.target_cpa.cpc_bid_floor_micros":0,"campaign.target_cpa.target_cpa_micros":0,"campaign.target_cpm.target_frequency_goal.target_count":0,"campaign.target_cpm.target_frequency_goal.time_unit":"UNSPECIFIED","campaign.target_impression_share.cpc_bid_ceiling_micros":0,"campaign.target_impression_share.location":"UNSPECIFIED","campaign.target_impression_share.location_fraction_micros":0,"campaign.target_roas.cpc_bid_ceiling_micros":0,"campaign.target_roas.cpc_bid_floor_micros":0,"campaign.target_roas.target_roas":0.0,"campaign.target_spend.cpc_bid_ceiling_micros":0,"campaign.target_spend.target_spend_micros":0,"campaign.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n"],"campaign.tracking_setting.tracking_url":"","campaign.tracking_url_template":"","campaign.url_custom_parameters":[],"campaign.vanity_pharma.vanity_pharma_display_url_mode":"UNSPECIFIED","campaign.vanity_pharma.vanity_pharma_text":"UNSPECIFIED","campaign.video_brand_safety_suitability":"UNSPECIFIED","metrics.clicks":0,"metrics.ctr":0.0,"metrics.conversions":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":0,"metrics.impressions":1.0,"metrics.video_views":0.0,"metrics.video_quartile_p100_rate":0.0,"segments.date":"2022-04-09","segments.hour":2.0},"emitted_at":1679577498477} -{"stream":"campaigns","data":{"campaign.accessible_bidding_strategy":"","campaign.ad_serving_optimization_status":"OPTIMIZE","campaign.advertising_channel_sub_type":"UNSPECIFIED","campaign.advertising_channel_type":"SEARCH","campaign.app_campaign_setting.app_id":"","campaign.app_campaign_setting.app_store":"UNSPECIFIED","campaign.app_campaign_setting.bidding_strategy_goal_type":"UNSPECIFIED","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","campaign.bidding_strategy":"","campaign.bidding_strategy_type":"TARGET_SPEND","campaign.campaign_budget":"customers/4651612872/campaignBudgets/10695604507","campaign_budget.amount_micros":750000,"campaign.commission.commission_rate_micros":0,"campaign.dynamic_search_ads_setting.domain_name":"","campaign.dynamic_search_ads_setting.feeds":[],"campaign.dynamic_search_ads_setting.language_code":"","campaign.dynamic_search_ads_setting.use_supplied_urls_only":false,"campaign.end_date":"2037-12-30","campaign.excluded_parent_asset_field_types":[],"campaign.experiment_type":"BASE","campaign.final_url_suffix":"","campaign.frequency_caps":[],"campaign.geo_target_type_setting.negative_geo_target_type":"PRESENCE","campaign.geo_target_type_setting.positive_geo_target_type":"PRESENCE_OR_INTEREST","campaign.hotel_setting.hotel_center_id":0,"campaign.id":16820250687,"campaign.labels":[],"campaign.local_campaign_setting.location_source_type":"UNSPECIFIED","campaign.manual_cpc.enhanced_cpc_enabled":false,"campaign.manual_cpm":"","campaign.manual_cpv":"","campaign.maximize_conversion_value.target_roas":0.0,"campaign.maximize_conversions.target_cpa_micros":0,"campaign.name":"Website traffic-Search-15","campaign.network_settings.target_content_network":true,"campaign.network_settings.target_google_search":true,"campaign.network_settings.target_partner_search_network":false,"campaign.network_settings.target_search_network":true,"campaign.optimization_goal_setting.optimization_goal_types":[],"campaign.optimization_score":0.0,"campaign.payment_mode":"CLICKS","campaign.percent_cpc.cpc_bid_ceiling_micros":0,"campaign.percent_cpc.enhanced_cpc_enabled":false,"campaign.real_time_bidding_setting.opt_in":false,"campaign.resource_name":"customers/4651612872/campaigns/16820250687","campaign.selective_optimization.conversion_actions":[],"campaign.serving_status":"SERVING","campaign.shopping_setting.campaign_priority":0,"campaign.shopping_setting.enable_local":false,"campaign.shopping_setting.merchant_id":0,"campaign.shopping_setting.sales_country":"","campaign.start_date":"2022-04-08","campaign.status":"PAUSED","campaign.target_cpa.cpc_bid_ceiling_micros":0,"campaign.target_cpa.cpc_bid_floor_micros":0,"campaign.target_cpa.target_cpa_micros":0,"campaign.target_cpm.target_frequency_goal.target_count":0,"campaign.target_cpm.target_frequency_goal.time_unit":"UNSPECIFIED","campaign.target_impression_share.cpc_bid_ceiling_micros":0,"campaign.target_impression_share.location":"UNSPECIFIED","campaign.target_impression_share.location_fraction_micros":0,"campaign.target_roas.cpc_bid_ceiling_micros":0,"campaign.target_roas.cpc_bid_floor_micros":0,"campaign.target_roas.target_roas":0.0,"campaign.target_spend.cpc_bid_ceiling_micros":0,"campaign.target_spend.target_spend_micros":0,"campaign.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n"],"campaign.tracking_setting.tracking_url":"","campaign.tracking_url_template":"","campaign.url_custom_parameters":[],"campaign.vanity_pharma.vanity_pharma_display_url_mode":"UNSPECIFIED","campaign.vanity_pharma.vanity_pharma_text":"UNSPECIFIED","campaign.video_brand_safety_suitability":"UNSPECIFIED","metrics.clicks":0,"metrics.ctr":0.0,"metrics.conversions":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":0,"metrics.impressions":4.0,"metrics.video_views":0.0,"metrics.video_quartile_p100_rate":0.0,"segments.date":"2022-04-09","segments.hour":7.0},"emitted_at":1679577498481} -{"stream":"campaigns","data":{"campaign.accessible_bidding_strategy":"","campaign.ad_serving_optimization_status":"OPTIMIZE","campaign.advertising_channel_sub_type":"UNSPECIFIED","campaign.advertising_channel_type":"SEARCH","campaign.app_campaign_setting.app_id":"","campaign.app_campaign_setting.app_store":"UNSPECIFIED","campaign.app_campaign_setting.bidding_strategy_goal_type":"UNSPECIFIED","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","campaign.bidding_strategy":"","campaign.bidding_strategy_type":"TARGET_SPEND","campaign.campaign_budget":"customers/4651612872/campaignBudgets/10695604507","campaign_budget.amount_micros":750000,"campaign.commission.commission_rate_micros":0,"campaign.dynamic_search_ads_setting.domain_name":"","campaign.dynamic_search_ads_setting.feeds":[],"campaign.dynamic_search_ads_setting.language_code":"","campaign.dynamic_search_ads_setting.use_supplied_urls_only":false,"campaign.end_date":"2037-12-30","campaign.excluded_parent_asset_field_types":[],"campaign.experiment_type":"BASE","campaign.final_url_suffix":"","campaign.frequency_caps":[],"campaign.geo_target_type_setting.negative_geo_target_type":"PRESENCE","campaign.geo_target_type_setting.positive_geo_target_type":"PRESENCE_OR_INTEREST","campaign.hotel_setting.hotel_center_id":0,"campaign.id":16820250687,"campaign.labels":[],"campaign.local_campaign_setting.location_source_type":"UNSPECIFIED","campaign.manual_cpc.enhanced_cpc_enabled":false,"campaign.manual_cpm":"","campaign.manual_cpv":"","campaign.maximize_conversion_value.target_roas":0.0,"campaign.maximize_conversions.target_cpa_micros":0,"campaign.name":"Website traffic-Search-15","campaign.network_settings.target_content_network":true,"campaign.network_settings.target_google_search":true,"campaign.network_settings.target_partner_search_network":false,"campaign.network_settings.target_search_network":true,"campaign.optimization_goal_setting.optimization_goal_types":[],"campaign.optimization_score":0.0,"campaign.payment_mode":"CLICKS","campaign.percent_cpc.cpc_bid_ceiling_micros":0,"campaign.percent_cpc.enhanced_cpc_enabled":false,"campaign.real_time_bidding_setting.opt_in":false,"campaign.resource_name":"customers/4651612872/campaigns/16820250687","campaign.selective_optimization.conversion_actions":[],"campaign.serving_status":"SERVING","campaign.shopping_setting.campaign_priority":0,"campaign.shopping_setting.enable_local":false,"campaign.shopping_setting.merchant_id":0,"campaign.shopping_setting.sales_country":"","campaign.start_date":"2022-04-08","campaign.status":"PAUSED","campaign.target_cpa.cpc_bid_ceiling_micros":0,"campaign.target_cpa.cpc_bid_floor_micros":0,"campaign.target_cpa.target_cpa_micros":0,"campaign.target_cpm.target_frequency_goal.target_count":0,"campaign.target_cpm.target_frequency_goal.time_unit":"UNSPECIFIED","campaign.target_impression_share.cpc_bid_ceiling_micros":0,"campaign.target_impression_share.location":"UNSPECIFIED","campaign.target_impression_share.location_fraction_micros":0,"campaign.target_roas.cpc_bid_ceiling_micros":0,"campaign.target_roas.cpc_bid_floor_micros":0,"campaign.target_roas.target_roas":0.0,"campaign.target_spend.cpc_bid_ceiling_micros":0,"campaign.target_spend.target_spend_micros":0,"campaign.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n"],"campaign.tracking_setting.tracking_url":"","campaign.tracking_url_template":"","campaign.url_custom_parameters":[],"campaign.vanity_pharma.vanity_pharma_display_url_mode":"UNSPECIFIED","campaign.vanity_pharma.vanity_pharma_text":"UNSPECIFIED","campaign.video_brand_safety_suitability":"UNSPECIFIED","metrics.clicks":0,"metrics.ctr":0.0,"metrics.conversions":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":0,"metrics.impressions":1.0,"metrics.video_views":0.0,"metrics.video_quartile_p100_rate":0.0,"segments.date":"2022-04-09","segments.hour":8.0},"emitted_at":1679577498485} -{"stream":"campaigns","data":{"campaign.accessible_bidding_strategy":"","campaign.ad_serving_optimization_status":"OPTIMIZE","campaign.advertising_channel_sub_type":"UNSPECIFIED","campaign.advertising_channel_type":"SEARCH","campaign.app_campaign_setting.app_id":"","campaign.app_campaign_setting.app_store":"UNSPECIFIED","campaign.app_campaign_setting.bidding_strategy_goal_type":"UNSPECIFIED","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","campaign.bidding_strategy":"","campaign.bidding_strategy_type":"TARGET_SPEND","campaign.campaign_budget":"customers/4651612872/campaignBudgets/10695604507","campaign_budget.amount_micros":750000,"campaign.commission.commission_rate_micros":0,"campaign.dynamic_search_ads_setting.domain_name":"","campaign.dynamic_search_ads_setting.feeds":[],"campaign.dynamic_search_ads_setting.language_code":"","campaign.dynamic_search_ads_setting.use_supplied_urls_only":false,"campaign.end_date":"2037-12-30","campaign.excluded_parent_asset_field_types":[],"campaign.experiment_type":"BASE","campaign.final_url_suffix":"","campaign.frequency_caps":[],"campaign.geo_target_type_setting.negative_geo_target_type":"PRESENCE","campaign.geo_target_type_setting.positive_geo_target_type":"PRESENCE_OR_INTEREST","campaign.hotel_setting.hotel_center_id":0,"campaign.id":16820250687,"campaign.labels":[],"campaign.local_campaign_setting.location_source_type":"UNSPECIFIED","campaign.manual_cpc.enhanced_cpc_enabled":false,"campaign.manual_cpm":"","campaign.manual_cpv":"","campaign.maximize_conversion_value.target_roas":0.0,"campaign.maximize_conversions.target_cpa_micros":0,"campaign.name":"Website traffic-Search-15","campaign.network_settings.target_content_network":true,"campaign.network_settings.target_google_search":true,"campaign.network_settings.target_partner_search_network":false,"campaign.network_settings.target_search_network":true,"campaign.optimization_goal_setting.optimization_goal_types":[],"campaign.optimization_score":0.0,"campaign.payment_mode":"CLICKS","campaign.percent_cpc.cpc_bid_ceiling_micros":0,"campaign.percent_cpc.enhanced_cpc_enabled":false,"campaign.real_time_bidding_setting.opt_in":false,"campaign.resource_name":"customers/4651612872/campaigns/16820250687","campaign.selective_optimization.conversion_actions":[],"campaign.serving_status":"SERVING","campaign.shopping_setting.campaign_priority":0,"campaign.shopping_setting.enable_local":false,"campaign.shopping_setting.merchant_id":0,"campaign.shopping_setting.sales_country":"","campaign.start_date":"2022-04-08","campaign.status":"PAUSED","campaign.target_cpa.cpc_bid_ceiling_micros":0,"campaign.target_cpa.cpc_bid_floor_micros":0,"campaign.target_cpa.target_cpa_micros":0,"campaign.target_cpm.target_frequency_goal.target_count":0,"campaign.target_cpm.target_frequency_goal.time_unit":"UNSPECIFIED","campaign.target_impression_share.cpc_bid_ceiling_micros":0,"campaign.target_impression_share.location":"UNSPECIFIED","campaign.target_impression_share.location_fraction_micros":0,"campaign.target_roas.cpc_bid_ceiling_micros":0,"campaign.target_roas.cpc_bid_floor_micros":0,"campaign.target_roas.target_roas":0.0,"campaign.target_spend.cpc_bid_ceiling_micros":0,"campaign.target_spend.target_spend_micros":0,"campaign.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n"],"campaign.tracking_setting.tracking_url":"","campaign.tracking_url_template":"","campaign.url_custom_parameters":[],"campaign.vanity_pharma.vanity_pharma_display_url_mode":"UNSPECIFIED","campaign.vanity_pharma.vanity_pharma_text":"UNSPECIFIED","campaign.video_brand_safety_suitability":"UNSPECIFIED","metrics.clicks":1,"metrics.ctr":0.25,"metrics.conversions":0.0,"metrics.conversions_value":0.0,"metrics.cost_micros":70000,"metrics.impressions":4.0,"metrics.video_views":0.0,"metrics.video_quartile_p100_rate":0.0,"segments.date":"2022-04-09","segments.hour":9.0},"emitted_at":1679577498489} -{"stream":"ad_groups","data":{"ad_group.ad_rotation_mode":"UNSPECIFIED","ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","ad_group.campaign":"customers/4651612872/campaigns/16820250687","ad_group.cpc_bid_micros":10000,"ad_group.cpm_bid_micros":10000,"ad_group.cpv_bid_micros":0,"ad_group.display_custom_bid_dimension":"UNSPECIFIED","ad_group.effective_target_cpa_micros":0,"ad_group.effective_target_cpa_source":"UNSPECIFIED","ad_group.effective_target_roas":0.0,"ad_group.effective_target_roas_source":"UNSPECIFIED","ad_group.excluded_parent_asset_field_types":[],"ad_group.optimized_targeting_enabled":false,"ad_group.final_url_suffix":"","ad_group.id":137051662444,"ad_group.labels":[],"ad_group.name":"Группа объявлений 1","ad_group.percent_cpc_bid_micros":0,"ad_group.resource_name":"customers/4651612872/adGroups/137051662444","ad_group.status":"ENABLED","ad_group.target_cpa_micros":0,"ad_group.target_cpm_micros":10000,"ad_group.target_roas":0.0,"ad_group.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n","targeting_dimension: AGE_RANGE\nbid_only: true\n","targeting_dimension: GENDER\nbid_only: true\n","targeting_dimension: PARENTAL_STATUS\nbid_only: true\n","targeting_dimension: INCOME_RANGE\nbid_only: true\n"],"ad_group.tracking_url_template":"","ad_group.type":"SEARCH_STANDARD","ad_group.url_custom_parameters":[],"segments.date":"2022-04-08"},"emitted_at":1679577823324} -{"stream":"ad_groups","data":{"ad_group.ad_rotation_mode":"UNSPECIFIED","ad_group.base_ad_group":"customers/4651612872/adGroups/137020701042","ad_group.campaign":"customers/4651612872/campaigns/16820250687","ad_group.cpc_bid_micros":10000,"ad_group.cpm_bid_micros":10000,"ad_group.cpv_bid_micros":0,"ad_group.display_custom_bid_dimension":"UNSPECIFIED","ad_group.effective_target_cpa_micros":0,"ad_group.effective_target_cpa_source":"UNSPECIFIED","ad_group.effective_target_roas":0.0,"ad_group.effective_target_roas_source":"UNSPECIFIED","ad_group.excluded_parent_asset_field_types":[],"ad_group.optimized_targeting_enabled":false,"ad_group.final_url_suffix":"","ad_group.id":137020701042,"ad_group.labels":[],"ad_group.name":"Группа объявлений 2","ad_group.percent_cpc_bid_micros":0,"ad_group.resource_name":"customers/4651612872/adGroups/137020701042","ad_group.status":"ENABLED","ad_group.target_cpa_micros":0,"ad_group.target_cpm_micros":10000,"ad_group.target_roas":0.0,"ad_group.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n","targeting_dimension: AGE_RANGE\nbid_only: true\n","targeting_dimension: GENDER\nbid_only: true\n","targeting_dimension: PARENTAL_STATUS\nbid_only: true\n","targeting_dimension: INCOME_RANGE\nbid_only: true\n"],"ad_group.tracking_url_template":"","ad_group.type":"SEARCH_STANDARD","ad_group.url_custom_parameters":[],"segments.date":"2022-04-09"},"emitted_at":1679577823326} -{"stream":"ad_groups","data":{"ad_group.ad_rotation_mode":"UNSPECIFIED","ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","ad_group.campaign":"customers/4651612872/campaigns/16820250687","ad_group.cpc_bid_micros":10000,"ad_group.cpm_bid_micros":10000,"ad_group.cpv_bid_micros":0,"ad_group.display_custom_bid_dimension":"UNSPECIFIED","ad_group.effective_target_cpa_micros":0,"ad_group.effective_target_cpa_source":"UNSPECIFIED","ad_group.effective_target_roas":0.0,"ad_group.effective_target_roas_source":"UNSPECIFIED","ad_group.excluded_parent_asset_field_types":[],"ad_group.optimized_targeting_enabled":false,"ad_group.final_url_suffix":"","ad_group.id":137051662444,"ad_group.labels":[],"ad_group.name":"Группа объявлений 1","ad_group.percent_cpc_bid_micros":0,"ad_group.resource_name":"customers/4651612872/adGroups/137051662444","ad_group.status":"ENABLED","ad_group.target_cpa_micros":0,"ad_group.target_cpm_micros":10000,"ad_group.target_roas":0.0,"ad_group.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n","targeting_dimension: AGE_RANGE\nbid_only: true\n","targeting_dimension: GENDER\nbid_only: true\n","targeting_dimension: PARENTAL_STATUS\nbid_only: true\n","targeting_dimension: INCOME_RANGE\nbid_only: true\n"],"ad_group.tracking_url_template":"","ad_group.type":"SEARCH_STANDARD","ad_group.url_custom_parameters":[],"segments.date":"2022-04-09"},"emitted_at":1679577823327} -{"stream":"ad_groups","data":{"ad_group.ad_rotation_mode":"UNSPECIFIED","ad_group.base_ad_group":"customers/4651612872/adGroups/137020701042","ad_group.campaign":"customers/4651612872/campaigns/16820250687","ad_group.cpc_bid_micros":10000,"ad_group.cpm_bid_micros":10000,"ad_group.cpv_bid_micros":0,"ad_group.display_custom_bid_dimension":"UNSPECIFIED","ad_group.effective_target_cpa_micros":0,"ad_group.effective_target_cpa_source":"UNSPECIFIED","ad_group.effective_target_roas":0.0,"ad_group.effective_target_roas_source":"UNSPECIFIED","ad_group.excluded_parent_asset_field_types":[],"ad_group.optimized_targeting_enabled":false,"ad_group.final_url_suffix":"","ad_group.id":137020701042,"ad_group.labels":[],"ad_group.name":"Группа объявлений 2","ad_group.percent_cpc_bid_micros":0,"ad_group.resource_name":"customers/4651612872/adGroups/137020701042","ad_group.status":"ENABLED","ad_group.target_cpa_micros":0,"ad_group.target_cpm_micros":10000,"ad_group.target_roas":0.0,"ad_group.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n","targeting_dimension: AGE_RANGE\nbid_only: true\n","targeting_dimension: GENDER\nbid_only: true\n","targeting_dimension: PARENTAL_STATUS\nbid_only: true\n","targeting_dimension: INCOME_RANGE\nbid_only: true\n"],"ad_group.tracking_url_template":"","ad_group.type":"SEARCH_STANDARD","ad_group.url_custom_parameters":[],"segments.date":"2022-04-10"},"emitted_at":1679577823328} -{"stream":"ad_groups","data":{"ad_group.ad_rotation_mode":"UNSPECIFIED","ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","ad_group.campaign":"customers/4651612872/campaigns/16820250687","ad_group.cpc_bid_micros":10000,"ad_group.cpm_bid_micros":10000,"ad_group.cpv_bid_micros":0,"ad_group.display_custom_bid_dimension":"UNSPECIFIED","ad_group.effective_target_cpa_micros":0,"ad_group.effective_target_cpa_source":"UNSPECIFIED","ad_group.effective_target_roas":0.0,"ad_group.effective_target_roas_source":"UNSPECIFIED","ad_group.excluded_parent_asset_field_types":[],"ad_group.optimized_targeting_enabled":false,"ad_group.final_url_suffix":"","ad_group.id":137051662444,"ad_group.labels":[],"ad_group.name":"Группа объявлений 1","ad_group.percent_cpc_bid_micros":0,"ad_group.resource_name":"customers/4651612872/adGroups/137051662444","ad_group.status":"ENABLED","ad_group.target_cpa_micros":0,"ad_group.target_cpm_micros":10000,"ad_group.target_roas":0.0,"ad_group.targeting_setting.target_restrictions":["targeting_dimension: AUDIENCE\nbid_only: true\n","targeting_dimension: AGE_RANGE\nbid_only: true\n","targeting_dimension: GENDER\nbid_only: true\n","targeting_dimension: PARENTAL_STATUS\nbid_only: true\n","targeting_dimension: INCOME_RANGE\nbid_only: true\n"],"ad_group.tracking_url_template":"","ad_group.type":"SEARCH_STANDARD","ad_group.url_custom_parameters":[],"segments.date":"2022-04-10"},"emitted_at":1679577823329} +{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078676857, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078676857", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137051662444", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137051662444~592078676857", "ad_group_ad.status": "REMOVED", "segments.date": "2022-04-08"}, "emitted_at": 1692608495121} +{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED_LIMITED", "ad_group_ad.policy_summary.policy_topic_entries": ["topic: \"TRADEMARKS_IN_AD_TEXT\"\ntype_: LIMITED\nevidences {\n text_list {\n texts: \"airbyte\"\n }\n}\nconstraints {\n reseller_constraint {\n }\n}\n"], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-04-09"}, "emitted_at": 1692608495139} +{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078676857, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078676857", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137051662444", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137051662444~592078676857", "ad_group_ad.status": "REMOVED", "segments.date": "2022-04-09"}, "emitted_at": 1692608495148} +{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED_LIMITED", "ad_group_ad.policy_summary.policy_topic_entries": ["topic: \"TRADEMARKS_IN_AD_TEXT\"\ntype_: LIMITED\nevidences {\n text_list {\n texts: \"airbyte\"\n }\n}\nconstraints {\n reseller_constraint {\n }\n}\n"], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-04-10"}, "emitted_at": 1692608495155} +{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078676857, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078676857", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137051662444", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137051662444~592078676857", "ad_group_ad.status": "REMOVED", "segments.date": "2022-04-10"}, "emitted_at": 1692608495162} +{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 16.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908590} +{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 1, "metrics.ctr": 1.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 10000, "metrics.impressions": 1.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 10000.0, "metrics.average_cpc": 10000.0, "metrics.average_cpm": 10000000.0, "metrics.interactions": 1, "metrics.interaction_event_types": "['InteractionEventType.CLICK']", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 18.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908597} +{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 19.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908603} +{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 20.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908610} +{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 22.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908617} +{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137051662444, "ad_group.labels": [], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137051662444", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-08"}, "emitted_at": 1692610032155} +{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-09"}, "emitted_at": 1692610032158} +{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137051662444, "ad_group.labels": [], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137051662444", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-09"}, "emitted_at": 1692610032159} +{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-10"}, "emitted_at": 1692610032161} +{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137051662444, "ad_group.labels": [], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137051662444", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-10"}, "emitted_at": 1692610032162} +{"stream": "audience", "data": {"audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1688510825339} +{"stream": "user_interest", "data": {"user_interest.availabilities": ["channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: SEARCH\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: DISPLAY\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: SHOPPING\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: DISCOVERY\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_SUBSET_SUBTYPES\n advertising_channel_type: VIDEO\n advertising_channel_sub_type: VIDEO_SEQUENCE\n advertising_channel_sub_type: VIDEO_OUTSTREAM\n advertising_channel_sub_type: VIDEO_ACTION\n advertising_channel_sub_type: VIDEO_NON_SKIPPABLE\n advertising_channel_sub_type: VIDEO_REACH_TARGET_FREQUENCY\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n"], "user_interest.launched_to_all": false, "user_interest.name": "Cloud Services Power Users", "user_interest.resource_name": "customers/4651612872/userInterests/92931", "user_interest.taxonomy_type": "AFFINITY", "user_interest.user_interest_id": 92931, "user_interest.user_interest_parent": "customers/4651612872/userInterests/92507"}, "emitted_at": 1688510842265} +{"stream": "campaign_budget", "data": {"campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": ["STANDARD"], "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": ["DAILY"], "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": ["REMOVED"], "campaign_budget.total_amount_micros": 0, "campaign_budget.type": ["STANDARD"], "segments.date": "2022-04-08", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": ["REMOVED"], "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 2, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1692610545110} +{"stream": "campaign_budget", "data": {"campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": ["STANDARD"], "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": ["DAILY"], "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": ["REMOVED"], "campaign_budget.total_amount_micros": 0, "campaign_budget.type": ["STANDARD"], "segments.date": "2022-04-09", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": ["REMOVED"], "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 70000.0, "metrics.average_cpc": 70000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 492957.74647887325, "metrics.average_cpv": 0.0, "metrics.clicks": 1, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 70000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.007042253521126761, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 142, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.007042253521126761, "metrics.interactions": 1, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1692610545113} +{"stream": "campaign_budget", "data": {"campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": ["STANDARD"], "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": ["DAILY"], "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": ["REMOVED"], "campaign_budget.total_amount_micros": 0, "campaign_budget.type": ["STANDARD"], "segments.date": "2022-04-10", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": ["REMOVED"], "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 175000.0, "metrics.average_cpc": 175000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 17721518.987341773, "metrics.average_cpv": 0.0, "metrics.clicks": 8, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 1400000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.10126582278481013, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 79, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.10126582278481013, "metrics.interactions": 8, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1692610545116} +{"stream": "labels", "data": {"label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1689230737253} +{"stream": "campaign_bidding_strategies", "data": {"campaign.id": 17354032686, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-31"}, "emitted_at": 1689230742268} +{"stream": "campaign_bidding_strategies", "data": {"campaign.id": 17324459992, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-06-01"}, "emitted_at": 1689230742269} +{"stream": "ad_group_bidding_strategies", "data": {"ad_group.id": 143992182864, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-31"}, "emitted_at": 1689230746492} +{"stream": "ad_group_bidding_strategies", "data": {"ad_group.id": 138459160713, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-06-01"}, "emitted_at": 1689230746493} +{"stream": "ad_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/143992182864", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 82426333464, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "uservertical::80530", "ad_group_criterion.effective_cpc_bid_micros": 0, "ad_group_criterion.effective_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "UNSPECIFIED", "ad_group_criterion.keyword.text": "", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/143992182864~82426333464", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "USER_INTEREST", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "customers/4651612872/userInterests/80530", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1689230748366} +{"stream": "ad_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/143992182864", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 297422806498, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "api integration", "ad_group_criterion.effective_cpc_bid_micros": 0, "ad_group_criterion.effective_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "api integration", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/143992182864~297422806498", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1689230748368} +{"stream": "ad_listing_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.criterion_id": 82426333464, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1689230749081} +{"stream": "ad_listing_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.criterion_id": 297422806498, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1689230749082} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json index bfe6dbf07f88..12bb06914012 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json @@ -133,6 +133,23 @@ "cursor_field": ["segments.date"], "primary_key": [["campaign.id"], ["segments.date"]] }, + { + "stream": { + "name": "campaign_budget", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"], + "source_defined_primary_key": [ + ["campaign_budget.id"], + ["segments.date"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"], + "primary_key": [["campaign_budget.id"], ["segments.date"]] + }, { "stream": { "name": "user_location_report", @@ -168,6 +185,50 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "campaign_bidding_strategies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "primary_key": [ + ["campaign.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "ad_group_bidding_strategies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "primary_key": [ + ["ad_group.id"], + ["bidding_strategy.id"], + ["segments.date"] + ], + "cursor_field": ["segments.date"] } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml index cd9801386cca..381177418962 100644 --- a/airbyte-integrations/connectors/source-google-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-ads/metadata.yaml @@ -6,11 +6,11 @@ data: connectorSubtype: api connectorType: source definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 - dockerImageTag: 0.2.22 + dockerImageTag: 0.7.4 dockerRepository: airbyte/source-google-ads githubIssueLabel: source-google-ads icon: google-adwords.svg - license: MIT + license: Elv2 name: Google Ads registries: cloud: @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-ads/requirements.txt b/airbyte-integrations/connectors/source-google-ads/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-google-ads/requirements.txt +++ b/airbyte-integrations/connectors/source-google-ads/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py index 39f872c4c095..4e4e5e100d99 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# """ MIT License diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 388672bafb3a..dc67173a5d85 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -8,7 +8,6 @@ from typing import Any, Iterator, List, Mapping, MutableMapping import backoff -import pendulum from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.client import GoogleAdsClient @@ -19,22 +18,33 @@ REPORT_MAPPING = { "accounts": "customer", - "service_accounts": "customer", + "account_labels": "customer_label", + "account_performance_report": "customer", "ad_group_ads": "ad_group_ad", "ad_group_ad_labels": "ad_group_ad_label", + "ad_group_ad_report": "ad_group_ad", "ad_groups": "ad_group", + "ad_group_bidding_strategies": "ad_group", + "ad_group_criterions": "ad_group_criterion", + "ad_group_criterion_labels": "ad_group_criterion_label", "ad_group_labels": "ad_group_label", + "ad_listing_group_criterions": "ad_group_criterion", + "audience": "audience", "campaigns": "campaign", + "campaign_real_time_bidding_settings": "campaign", + "campaign_bidding_strategies": "campaign", + "campaign_budget": "campaign_budget", "campaign_labels": "campaign_label", - "account_performance_report": "customer", - "ad_group_ad_report": "ad_group_ad", + "click_view": "click_view", "display_keyword_performance_report": "display_keyword_view", "display_topics_performance_report": "topic_view", - "shopping_performance_report": "shopping_performance_view", - "user_location_report": "user_location_view", - "click_view": "click_view", "geographic_report": "geographic_view", "keyword_report": "keyword_view", + "labels": "label", + "service_accounts": "customer", + "shopping_performance_report": "shopping_performance_view", + "user_interest": "user_interest", + "user_location_report": "user_location_view", } API_VERSION = "v13" logger = logging.getLogger("airbyte") @@ -107,15 +117,12 @@ def convert_schema_into_query( ) -> str: from_category = REPORT_MAPPING[report_name] fields = GoogleAds.get_fields_from_schema(schema) - fields = ",\n".join(fields) + fields = ", ".join(fields) - query_template = f"SELECT {fields} FROM {from_category} " + query_template = f"SELECT {fields} FROM {from_category}" if cursor_field: - end_date_inclusive = "<=" if (pendulum.parse(to_date) - pendulum.parse(from_date)).days > 1 else "<" - query_template += ( - f"WHERE {cursor_field} >= '{from_date}' AND {cursor_field} {end_date_inclusive} '{to_date}' ORDER BY {cursor_field} ASC" - ) + query_template += f" WHERE {cursor_field} >= '{from_date}' AND {cursor_field} <= '{to_date}' ORDER BY {cursor_field} ASC" return query_template diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json new file mode 100644 index 000000000000..294e82084a59 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer_label.resource_name": { + "type": ["null", "string"] + }, + "customer_label.customer": { + "type": ["null", "string"] + }, + "customer.id": { + "type": ["null", "integer"] + }, + "customer_label.label": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json new file mode 100644 index 000000000000..bc1b798dc61e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.aligned_campaign_budget_id": { + "type": ["null", "integer"] + }, + "bidding_strategy.campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.effective_currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.enhanced_cpc": { + "type": ["null", "string"] + }, + "bidding_strategy.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.name": { + "type": ["null", "string"] + }, + "bidding_strategy.non_removed_campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.resource_name": { + "type": ["null", "string"] + }, + "bidding_strategy.status": { + "type": ["null", "string"] + }, + "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.location": { + "type": ["null", "string"] + }, + "bidding_strategy.target_impression_share.location_fraction_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.target_spend.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_spend.target_spend_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.type": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json new file mode 100644 index 000000000000..16d04dd4720c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group_criterion_label.ad_group_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion_label.label": { + "type": ["null", "string"] + }, + "ad_group_criterion_label.resource_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json new file mode 100644 index 000000000000..bf90272fbb3c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json @@ -0,0 +1,225 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.ad_group": { + "type": ["null", "string"] + }, + "ad_group_criterion.age_range.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.app_payment_model.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.approval_status": { + "type": ["null", "string"] + }, + "ad_group_criterion.audience.audience": { + "type": ["null", "string"] + }, + "ad_group_criterion.bid_modifier": { + "type": ["null", "number"] + }, + "ad_group_criterion.combined_audience.combined_audience": { + "type": ["null", "string"] + }, + "ad_group_criterion.cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.cpv_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.custom_affinity.custom_affinity": { + "type": ["null", "string"] + }, + "ad_group_criterion.custom_audience.custom_audience": { + "type": ["null", "string"] + }, + "ad_group_criterion.custom_intent.custom_intent": { + "type": ["null", "string"] + }, + "ad_group_criterion.disapproval_reasons": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.display_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpc_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpm_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpv_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpv_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_percent_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_percent_cpc_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.final_mobile_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.final_url_suffix": { + "type": ["null", "string"] + }, + "ad_group_criterion.final_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.gender.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.income_range.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.match_type": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.text": { + "type": ["null", "string"] + }, + "ad_group_criterion.labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.mobile_app_category.mobile_app_category_constant": { + "type": ["null", "string"] + }, + "ad_group_criterion.mobile_application.app_id": { + "type": ["null", "string"] + }, + "ad_group_criterion.mobile_application.name": { + "type": ["null", "string"] + }, + "ad_group_criterion.negative": { + "type": ["null", "boolean"] + }, + "ad_group_criterion.parental_status.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.percent_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.placement.url": { + "type": ["null", "string"] + }, + "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.first_page_cpc_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.first_position_cpc_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.top_of_page_cpc_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.quality_info.creative_quality_score": { + "type": ["null", "string"] + }, + "ad_group_criterion.quality_info.post_click_quality_score": { + "type": ["null", "string"] + }, + "ad_group_criterion.quality_info.quality_score": { + "type": ["null", "integer"] + }, + "ad_group_criterion.quality_info.search_predicted_ctr": { + "type": ["null", "string"] + }, + "ad_group_criterion.resource_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.status": { + "type": ["null", "string"] + }, + "ad_group_criterion.system_serving_status": { + "type": ["null", "string"] + }, + "ad_group_criterion.topic.path": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.topic.topic_constant": { + "type": ["null", "string"] + }, + "ad_group_criterion.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group_criterion.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "ad_group_criterion.user_interest.user_interest_category": { + "type": ["null", "string"] + }, + "ad_group_criterion.user_list.user_list": { + "type": ["null", "string"] + }, + "ad_group_criterion.webpage.conditions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.webpage.coverage_percentage": { + "type": ["null", "number"] + }, + "ad_group_criterion.webpage.criterion_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.webpage.sample.sample_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.youtube_channel.channel_id": { + "type": ["null", "string"] + }, + "ad_group_criterion.youtube_video.video_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json new file mode 100644 index 000000000000..bfd193ddef36 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json @@ -0,0 +1,75 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.activity_country.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.activity_id.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.activity_rating.value": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.hotel_class.value": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.hotel_id.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_bidding_category.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.product_bidding_category.level": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_brand.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_channel.channel": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_condition.condition": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_item_id.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_type.level": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_type.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.parent_ad_group_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json new file mode 100644 index 000000000000..c1bd49e8eed2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "audience.description": { + "type": ["null", "string"] + }, + "audience.dimensions": { + "type": ["null", "array"] + }, + "audience.exclusion_dimension": { + "type": ["null", "string"] + }, + "audience.id": { + "type": ["null", "integer"] + }, + "audience.name": { + "type": ["null", "string"] + }, + "audience.resource_name": { + "type": ["null", "string"] + }, + "audience.status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json new file mode 100644 index 000000000000..32493bfd957e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "campaign.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.aligned_campaign_budget_id": { + "type": ["null", "integer"] + }, + "bidding_strategy.campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.effective_currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.enhanced_cpc": { + "type": ["null", "string"] + }, + "bidding_strategy.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.name": { + "type": ["null", "string"] + }, + "bidding_strategy.non_removed_campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.resource_name": { + "type": ["null", "string"] + }, + "bidding_strategy.status": { + "type": ["null", "string"] + }, + "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.location": { + "type": ["null", "string"] + }, + "bidding_strategy.target_impression_share.location_fraction_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.target_spend.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_spend.target_spend_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.type": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json new file mode 100644 index 000000000000..2be876e64cbb --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "campaign_budget.aligned_bidding_strategy_id": { + "type": ["null", "integer"] + }, + "campaign_budget.amount_micros": { + "type": ["null", "integer"] + }, + "campaign_budget.delivery_method": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "campaign_budget.explicitly_shared": { + "type": ["null", "boolean"] + }, + "campaign_budget.has_recommended_budget": { + "type": ["null", "boolean"] + }, + "campaign_budget.id": { + "type": ["null", "integer"] + }, + "campaign_budget.name": { + "type": ["null", "string"] + }, + "campaign_budget.period": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "campaign_budget.recommended_budget_amount_micros": { + "type": ["null", "integer"] + }, + "campaign_budget.recommended_budget_estimated_change_weekly_clicks": { + "type": ["null", "integer"] + }, + "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": { + "type": ["null", "integer"] + }, + "campaign_budget.recommended_budget_estimated_change_weekly_interactions": { + "type": ["null", "integer"] + }, + "campaign_budget.recommended_budget_estimated_change_weekly_views": { + "type": ["null", "integer"] + }, + "campaign_budget.reference_count": { + "type": ["null", "integer"] + }, + "campaign_budget.resource_name": { + "type": ["null", "string"] + }, + "campaign_budget.status": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "campaign_budget.total_amount_micros": { + "type": ["null", "integer"] + }, + "campaign_budget.type": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "segments.budget_campaign_association_status.campaign": { + "type": ["null", "string"] + }, + "segments.budget_campaign_association_status.status": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.all_conversions": { + "type": ["null", "number"] + }, + "metrics.all_conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.all_conversions_value": { + "type": ["null", "number"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpe": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.average_cpv": { + "type": ["null", "number"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.cost_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "metrics.cross_device_conversions": { + "type": ["null", "number"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "metrics.engagement_rate": { + "type": ["null", "number"] + }, + "metrics.engagements": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.interaction_rate": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.value_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "metrics.video_view_rate": { + "type": ["null", "number"] + }, + "metrics.video_views": { + "type": ["null", "integer"] + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json index af0258bdd861..7f7a54c07ce2 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json @@ -269,12 +269,57 @@ "metrics.video_quartile_p100_rate": { "type": ["null", "number"] }, + "metrics.active_view_cpm": { + "type": ["null", "number"] + }, + "metrics.active_view_ctr": { + "type": ["null", "number"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "string"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, "segments.date": { "type": ["null", "string"], "format": "date" }, "segments.hour": { "type": ["null", "number"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json index 4116c2ba57df..6ce4e2a4de4c 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json @@ -80,6 +80,39 @@ "metrics.impressions": { "type": ["null", "integer"] }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + }, "ad_group_criterion.criterion_id": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json new file mode 100644 index 000000000000..1621d5831bc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "label.id": { + "type": ["null", "integer"] + }, + "label.name": { + "type": ["null", "string"] + }, + "label.resource_name": { + "type": ["null", "string"] + }, + "label.status": { + "type": ["null", "string"] + }, + "label.text_label.background_color": { + "type": ["null", "string"] + }, + "label.text_label.description": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json new file mode 100644 index 000000000000..350a40ef5906 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "user_interest.availabilities": { + "type": ["null", "array"] + }, + "user_interest.launched_to_all": { + "type": ["null", "boolean"] + }, + "user_interest.name": { + "type": ["null", "string"] + }, + "user_interest.resource_name": { + "type": ["null", "string"] + }, + "user_interest.taxonomy_type": { + "type": ["null", "string"] + }, + "user_interest.user_interest_id": { + "type": ["null", "integer"] + }, + "user_interest.user_interest_parent": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index 60c42176e0a7..e7c485e08e63 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -12,19 +12,29 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException +from google.ads.googleads.v13.errors.types.authentication_error import AuthenticationErrorEnum +from google.ads.googleads.v13.errors.types.authorization_error import AuthorizationErrorEnum from pendulum import parse, today from .custom_query_stream import CustomQuery, IncrementalCustomQuery from .google_ads import GoogleAds from .models import Customer from .streams import ( + AccountLabels, AccountPerformanceReport, Accounts, AdGroupAdLabels, AdGroupAdReport, AdGroupAds, + AdGroupBiddingStrategies, + AdGroupCriterionLabels, + AdGroupCriterions, AdGroupLabels, AdGroups, + AdListingGroupCriterions, + Audience, + CampaignBiddingStrategies, + CampaignBudget, CampaignLabels, Campaigns, ClickView, @@ -32,8 +42,10 @@ DisplayTopicsPerformanceReport, GeographicReport, KeywordReport, + Labels, ServiceAccounts, ShoppingPerformanceReport, + UserInterest, UserLocationReport, ) from .utils import GAQL @@ -121,6 +133,13 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> pass return True, None except GoogleAdsException as exception: + if AuthorizationErrorEnum.AuthorizationError.USER_PERMISSION_DENIED in ( + x.error_code.authorization_error for x in exception.failure.errors + ) or AuthenticationErrorEnum.AuthenticationError.CUSTOMER_NOT_FOUND in ( + x.error_code.authentication_error for x in exception.failure.errors + ): + message = f"Failed to access the customer '{exception.customer_id}'. Ensure the customer is linked to your manager account or check your permissions to access this customer account." + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) error_messages = ", ".join([error.message for error in exception.failure.errors]) logger.error(traceback.format_exc()) return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}" @@ -137,10 +156,20 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: AdGroupAds(**incremental_config), AdGroupAdLabels(google_api, customers=customers), AdGroups(**incremental_config), + AdGroupBiddingStrategies(**incremental_config), + AdGroupCriterions(google_api, customers=customers), + AdGroupCriterionLabels(google_api, customers=customers), AdGroupLabels(google_api, customers=customers), + AdListingGroupCriterions(google_api, customers=customers), Accounts(**incremental_config), + AccountLabels(google_api, customers=customers), + Audience(google_api, customers=customers), + CampaignBiddingStrategies(**incremental_config), + CampaignBudget(**incremental_config), CampaignLabels(google_api, customers=customers), ClickView(**incremental_config), + Labels(google_api, customers=customers), + UserInterest(google_api, customers=customers), ] # Metrics streams cannot be requested for a manager account. if non_manager_accounts: diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 57d046ba98e2..73e4eca612b0 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -23,34 +23,34 @@ "type": "string", "title": "Developer Token", "order": 0, - "description": "Developer token granted by Google to use their APIs. More instruction on how to find this value in our docs", + "description": "The Developer Token granted by Google to use their APIs. For detailed instructions on finding this value, refer to our documentation.", "airbyte_secret": true }, "client_id": { "type": "string", "title": "Client ID", "order": 1, - "description": "The Client ID of your Google Ads developer application. More instruction on how to find this value in our docs" + "description": "The Client ID of your Google Ads developer application. For detailed instructions on finding this value, refer to our documentation." }, "client_secret": { "type": "string", "title": "Client Secret", "order": 2, - "description": "The Client Secret of your Google Ads developer application. More instruction on how to find this value in our docs", + "description": "The Client Secret of your Google Ads developer application. For detailed instructions on finding this value, refer to our documentation.", "airbyte_secret": true }, "refresh_token": { "type": "string", "title": "Refresh Token", "order": 3, - "description": "The token for obtaining a new access token. More instruction on how to find this value in our docs", + "description": "The token used to obtain a new Access Token. For detailed instructions on finding this value, refer to our documentation.", "airbyte_secret": true }, "access_token": { "type": "string", "title": "Access Token", "order": 4, - "description": "Access Token for making authenticated requests. More instruction on how to find this value in our docs", + "description": "The Access Token for making authenticated requests. For detailed instructions on finding this value, refer to our documentation.", "airbyte_secret": true } } @@ -58,16 +58,16 @@ "customer_id": { "title": "Customer ID(s)", "type": "string", - "description": "Comma separated list of (client) customer IDs. Each customer ID must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs. Metrics streams like AdGroupAdReport cannot be requested for a manager account.", + "description": "Comma-separated list of (client) customer IDs. Each customer ID must be specified as a 10-digit number without dashes. For detailed instructions on finding this value, refer to our documentation.", "pattern": "^[0-9]{10}(,[0-9]{10})*$", - "pattern_descriptor": "^[0-9]{10}(,[0-9]{10})*$ . The customer ID must be 10 digits. Separate multiple customer IDs using commas.", + "pattern_descriptor": "The customer ID must be 10 digits. Separate multiple customer IDs using commas.", "examples": ["6783948572,5839201945"], "order": 1 }, "start_date": { "type": "string", "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25. Any data before this date will not be replicated.", + "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", "examples": ["2017-01-25"], @@ -77,7 +77,7 @@ "end_date": { "type": "string", "title": "End Date", - "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.", + "description": "UTC date in the format YYYY-MM-DD. Any data after this date will not be replicated.", "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", "examples": ["2017-01-30"], @@ -95,8 +95,9 @@ "properties": { "query": { "type": "string", + "multiline": true, "title": "Custom Query", - "description": "A custom defined GAQL query for building the report. Should not contain segments.date expression because it is used by incremental streams. See Google's query builder for more information.", + "description": "A custom defined GAQL query for building the report. Avoid including the segments.date field; wherever possible, Airbyte will automatically include it for incremental syncs. For more information, refer to Google's documentation.", "examples": [ "SELECT segments.ad_destination_type, campaign.advertising_channel_sub_type FROM campaign WHERE campaign.status = 'PAUSED'" ] @@ -104,7 +105,7 @@ "table_name": { "type": "string", "title": "Destination Table Name", - "description": "The table name in your destination database for choosen query." + "description": "The table name in your destination database for the chosen query." } } } @@ -112,7 +113,8 @@ "login_customer_id": { "type": "string", "title": "Login Customer ID for Managed Accounts", - "description": "If your access to the customer account is through a manager account, this field is required and must be set to the customer ID of the manager account (10-digit number without dashes). More information about this field you can see here", + "description": "If your access to the customer account is through a manager account, this field is required, and must be set to the 10-digit customer ID of the manager account. For more information about this field, refer to Google's documentation.", + "pattern_descriptor": ": 10 digits, with no dashes.", "pattern": "^([0-9]{10})?$", "examples": ["7349206847"], "order": 4 @@ -120,7 +122,7 @@ "conversion_window_days": { "title": "Conversion Window", "type": "integer", - "description": "A conversion window is the period of time after an ad interaction (such as an ad click or video view) during which a conversion, such as a purchase, is recorded in Google Ads. For more information, see Google's documentation.", + "description": "A conversion window is the number of days after an ad interaction (such as an ad click or video view) during which a conversion, such as a purchase, is recorded in Google Ads. For more information, see Google's documentation.", "minimum": 0, "maximum": 1095, "default": 14, @@ -129,16 +131,50 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials"], - "oauthFlowInitParameters": [ - ["client_id"], - ["client_secret"], - ["developer_token"] - ], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + }, + "developer_token": { + "type": "string", + "path_in_connector_config": ["credentials", "developer_token"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 6299e78d8596..1cebedf2abd6 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -4,7 +4,7 @@ import logging from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum from airbyte_cdk.models import SyncMode @@ -43,47 +43,27 @@ def parse_dates(stream_slice): return start_date, end_date -def get_date_params(start_date: str, time_zone=None, range_days: int = None, end_date: pendulum.datetime = None) -> Tuple[str, str]: - """ - Returns `start_date` and `end_date` for the given stream_slice. - If (end_date - start_date) is a big date range (>= 1 month), it can take more than 2 hours to process all the records from the given slice. - After 2 hours next page tokens will be expired, finally resulting in page token expired error - Currently this method returns `start_date` and `end_date` with 15 days difference. - """ - - end_date = end_date or pendulum.yesterday(tz=time_zone) - start_date = pendulum.parse(start_date) - if start_date > pendulum.now(): - return start_date.to_date_string(), start_date.add(days=1).to_date_string() - end_date = min(end_date, start_date.add(days=range_days)) - - # Fix issue #4806, start date should always be lower than end date. - if start_date.add(days=1).date() >= end_date.date(): - return start_date.add(days=1).to_date_string(), start_date.add(days=2).to_date_string() - return start_date.add(days=1).to_date_string(), end_date.to_date_string() - - def chunk_date_range( start_date: str, conversion_window: int, - field: str, end_date: str = None, days_of_data_storage: int = None, range_days: int = None, time_zone=None, ) -> Iterable[Optional[MutableMapping[str, any]]]: """ - Passing optional parameter end_date for testing - Returns a list of the beginning and ending timestamps of each `range_days` between the start date and now. - The return value is a list of dicts {'date': str} which can be used directly with the Slack API + Returns `start_date` and `end_date` for the given stream_slice. + If (end_date - start_date) is a big date range (>= 1 month), it can take more than 2 hours to process all the records from the given slice. + After 2 hours next page tokens will be expired, finally resulting in page token expired error + Currently this method returns `start_date` and `end_date` with `range_days` difference which is 15 days in most cases. """ - intervals = [] - end_date = pendulum.parse(end_date) if end_date else pendulum.now() - start_date = pendulum.parse(start_date) + today = pendulum.today(tz=time_zone) + end_date = min(pendulum.parse(end_date, tz=time_zone), today) if end_date else today + start_date = pendulum.parse(start_date, tz=time_zone) # For some metrics we can only get data not older than N days, it is Google Ads policy if days_of_data_storage: - start_date = max(start_date, pendulum.now().subtract(days=days_of_data_storage - conversion_window)) + start_date = max(start_date, pendulum.now(tz=time_zone).subtract(days=days_of_data_storage - conversion_window)) # As in to return some state when state in abnormal if start_date > end_date: @@ -91,17 +71,15 @@ def chunk_date_range( # applying conversion window start_date = start_date.subtract(days=conversion_window) + slice_start = start_date - while start_date < end_date: - start, end = get_date_params(start_date.to_date_string(), time_zone=time_zone, range_days=range_days, end_date=end_date) - intervals.append( - { - "start_date": start, - "end_date": end, - } - ) - start_date = start_date.add(days=range_days) - return intervals + while slice_start.date() <= end_date.date(): + slice_end = min(end_date, slice_start.add(days=range_days - 1)) + yield { + "start_date": slice_start.to_date_string(), + "end_date": slice_end.to_date_string(), + } + slice_start = slice_end.add(days=1) class GoogleAdsStream(Stream, ABC): @@ -136,6 +114,7 @@ def read_records(self, sync_mode, stream_slice: Optional[Mapping[str, Any]] = No for response in response_records: yield from self.parse_response(response) except GoogleAdsException as exc: + exc.customer_id = customer_id if not self.CATCH_API_ERRORS: raise for error in exc.failure.errors: @@ -194,7 +173,6 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite start_date=start_date, end_date=end_date, conversion_window=self.conversion_window_days, - field=self.cursor_field, days_of_data_storage=self.days_of_data_storage, range_days=self.range_days, time_zone=customer.time_zone, @@ -225,7 +203,9 @@ def read_records( date_in_latest_record = pendulum.parse(record[self.cursor_field]) cursor_value = (max(date_in_current_stream, date_in_latest_record)).to_date_string() self.state = {customer_id: {self.cursor_field: cursor_value}} - self.incremental_sieve_logger.info(f"Updated state for customer {customer_id}. Full state is {self.state}.") + # When large amount of data this log produces so much records so the enire log is not usable + # See: https://github.com/airbytehq/oncall/issues/2460 + # self.incremental_sieve_logger.info(f"Updated state for customer {customer_id}. Full state is {self.state}.") yield record continue self.state = {customer_id: {self.cursor_field: record[self.cursor_field]}} @@ -280,6 +260,14 @@ class Accounts(IncrementalGoogleAdsStream): primary_key = ["customer.id", "segments.date"] +class AccountLabels(GoogleAdsStream): + """ + Account Labels stream: https://developers.google.com/google-ads/api/fields/v14/customer_label + """ + + primary_key = ["customer_label.resource_name"] + + class ServiceAccounts(GoogleAdsStream): """ This stream is intended to be used as a service class, not exposed to a user @@ -298,6 +286,24 @@ class Campaigns(IncrementalGoogleAdsStream): primary_key = ["campaign.id", "segments.date", "segments.hour"] +class CampaignBudget(IncrementalGoogleAdsStream): + """ + Campaigns stream: https://developers.google.com/google-ads/api/fields/v13/campaign_budget + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["campaign_budget.id", "segments.date"] + + +class CampaignBiddingStrategies(IncrementalGoogleAdsStream): + """ + Campaign Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v14/campaign + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["campaign.id", "bidding_strategy.id", "segments.date"] + + class CampaignLabels(GoogleAdsStream): """ Campaign labels stream: https://developers.google.com/google-ads/api/fields/v11/campaign_label @@ -324,6 +330,42 @@ class AdGroupLabels(GoogleAdsStream): primary_key = ["ad_group_label.resource_name"] +class AdGroupBiddingStrategies(IncrementalGoogleAdsStream): + """ + Ad Group Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v14/ad_group + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["ad_group.id", "bidding_strategy.id", "segments.date"] + + +class AdGroupCriterions(GoogleAdsStream): + """ + Ad Group Criterions stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["ad_group.id", "ad_group_criterion.criterion_id"] + + +class AdGroupCriterionLabels(GoogleAdsStream): + """ + Ad Group Criterion Labels stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion_label + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["ad_group_criterion_label.resource_name"] + + +class AdListingGroupCriterions(GoogleAdsStream): + """ + Ad Group Criterions stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["ad_group.id", "ad_group_criterion.criterion_id"] + + class AdGroupAds(IncrementalGoogleAdsStream): """ AdGroups stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad @@ -403,3 +445,27 @@ class ClickView(IncrementalGoogleAdsStream): primary_key = ["click_view.gclid", "segments.date", "segments.ad_network_type"] days_of_data_storage = 90 range_days = 1 + + +class UserInterest(GoogleAdsStream): + """ + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + """ + + primary_key = ["user_interest.user_interest_id"] + + +class Audience(GoogleAdsStream): + """ + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + """ + + primary_key = ["audience.id"] + + +class Labels(GoogleAdsStream): + """ + Labels stream: https://developers.google.com/google-ads/api/fields/v14/label + """ + + primary_key = ["label.id"] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/__init__.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index 8bb699d43680..cdd5285a051d 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -4,15 +4,11 @@ from datetime import date -import pendulum import pytest from airbyte_cdk.utils import AirbyteTracedException -from freezegun import freeze_time from google.auth import exceptions -from pendulum.tz.timezone import Timezone from source_google_ads.google_ads import GoogleAds -from source_google_ads.models import Customer -from source_google_ads.streams import IncrementalGoogleAdsStream, chunk_date_range, get_date_params +from source_google_ads.streams import chunk_date_range from .common import MockGoogleAdsClient, MockGoogleAdsService @@ -89,78 +85,76 @@ def test_get_fields_from_schema(): def test_interval_chunking(): mock_intervals = [ - {"start_date": "2021-06-18", "end_date": "2021-06-27"}, - {"start_date": "2021-06-28", "end_date": "2021-07-07"}, - {"start_date": "2021-07-08", "end_date": "2021-07-17"}, - {"start_date": "2021-07-18", "end_date": "2021-07-27"}, - {"start_date": "2021-07-28", "end_date": "2021-08-06"}, - {"start_date": "2021-08-07", "end_date": "2021-08-15"}, + {"start_date": "2021-06-17", "end_date": "2021-06-26"}, + {"start_date": "2021-06-27", "end_date": "2021-07-06"}, + {"start_date": "2021-07-07", "end_date": "2021-07-16"}, + {"start_date": "2021-07-17", "end_date": "2021-07-26"}, + {"start_date": "2021-07-27", "end_date": "2021-08-05"}, + {"start_date": "2021-08-06", "end_date": "2021-08-10"}, ] - intervals = chunk_date_range("2021-07-01", 14, "segments.date", "2021-08-15", range_days=10) - + intervals = list(chunk_date_range("2021-07-01", 14, "2021-08-10", range_days=10, time_zone="UTC")) assert mock_intervals == intervals -def test_get_date_params(customers): - # Please note that this is equal to inputted stream_slice start date + 1 day - mock_start_date = "2021-05-19" - mock_end_date = "2021-06-02" - mock_conversion_window_days = 14 - - incremental_stream_config = dict( - conversion_window_days=mock_conversion_window_days, - start_date=mock_start_date, - api=MockGoogleAdsClient(SAMPLE_CONFIG), - customers=customers, - ) - - stream = IncrementalGoogleAdsStream(**incremental_stream_config) - - for customer in stream.customers: - start_date, end_date = get_date_params( - start_date="2021-05-18", range_days=stream.range_days, time_zone=customer.time_zone, end_date=pendulum.parse("2021-08-15") - ) - - assert mock_start_date == start_date and mock_end_date == end_date - - -@freeze_time("2022-01-30 03:21:34", tz_offset=0) -def test_get_date_params_with_time_zone(): - time_zone_chatham = Timezone("Pacific/Chatham") # UTC+12:45 - customer = Customer(id="id", time_zone=time_zone_chatham, is_manager_account=False) - mock_start_date_chatham = pendulum.today(tz=time_zone_chatham).subtract(days=1).to_date_string() - time_zone_honolulu = Timezone("Pacific/Honolulu") # UTC-10:00 - customer_2 = Customer(id="id_2", time_zone=time_zone_honolulu, is_manager_account=False) - mock_start_date_honolulu = pendulum.today(tz=time_zone_honolulu).subtract(days=1).to_date_string() - - mock_conversion_window_days = 14 - - incremental_stream_config = dict( - conversion_window_days=mock_conversion_window_days, - start_date=mock_start_date_chatham, - api=MockGoogleAdsClient(SAMPLE_CONFIG), - customers=[customer], - ) - stream = IncrementalGoogleAdsStream(**incremental_stream_config) - start_date_chatham, end_date_chatham = get_date_params( - start_date=mock_start_date_chatham, time_zone=customer.time_zone, range_days=stream.range_days - ) - - incremental_stream_config.update({"start_date": mock_start_date_honolulu, "customers": [customer_2]}) - stream_2 = IncrementalGoogleAdsStream(**incremental_stream_config) - - start_date_honolulu, end_date_honolulu = get_date_params( - start_date=mock_start_date_honolulu, time_zone=customer_2.time_zone, range_days=stream_2.range_days - ) - - assert start_date_honolulu != start_date_chatham and end_date_honolulu != end_date_chatham - - -def test_convert_schema_into_query(): - report_name = "ad_group_ad_report" - query = "SELECT segment.date FROM ad_group_ad WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-03-01' ORDER BY segments.date ASC" - response = GoogleAds.convert_schema_into_query(SAMPLE_SCHEMA, report_name, "2020-01-01", "2020-03-01", "segments.date") - assert response == query +generic_schema = {"properties": {"ad_group_id": {}, "segments.date": {}, "campaign_id": {}, "account_id": {}}} + + +@pytest.mark.parametrize( + "stream_schema, report_name, slice_start, slice_end, cursor, expected_sql", + ( + ( + generic_schema, + "ad_group_ads", + "2020-01-01", + "2020-01-10", + "segments.date", + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-10' ORDER BY segments.date ASC" + ), + ( + generic_schema, + "ad_group_ads", + "2020-01-01", + "2020-01-02", + "segments.date", + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-02' ORDER BY segments.date ASC" + ), + ( + generic_schema, + "ad_group_ads", + None, + None, + None, + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad" + ), + ( + generic_schema, + "click_view", + "2020-01-01", + "2020-01-10", + "segments.date", + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-10' ORDER BY segments.date ASC" + ), + ( + generic_schema, + "click_view", + "2020-01-01", + "2020-01-02", + "segments.date", + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-02' ORDER BY segments.date ASC" + ), + ( + generic_schema, + "click_view", + None, + None, + None, + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view" + ), + ), +) +def test_convert_schema_into_query(stream_schema, report_name, slice_start, slice_end, cursor, expected_sql): + query = GoogleAds.convert_schema_into_query(stream_schema, report_name, slice_start, slice_end, cursor) + assert query == expected_sql def test_get_field_value(): @@ -174,67 +168,3 @@ def test_parse_single_result(): date = "2001-01-01" response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date)) assert response == response - - -# Add a sample config with date parameters -SAMPLE_CONFIG_WITH_DATE = { - "credentials": { - "developer_token": "developer_token", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token", - }, - "customer_id": "customer_id", - "start_date": "2021-11-01", - "end_date": "2021-11-15", -} - - -def test_get_date_params_with_date(customers): - # Please note that this is equal to inputted stream_slice start date + 1 day - mock_start_date = SAMPLE_CONFIG_WITH_DATE["start_date"] - mock_end_date = SAMPLE_CONFIG_WITH_DATE["end_date"] - incremental_stream_config = dict( - start_date=mock_start_date, - end_date=mock_end_date, - conversion_window_days=0, - customers=customers, - api=MockGoogleAdsClient(SAMPLE_CONFIG_WITH_DATE), - ) - stream = IncrementalGoogleAdsStream(**incremental_stream_config) - for customer in stream.customers: - start_date, end_date = get_date_params( - start_date="2021-10-31", time_zone=customer.time_zone, range_days=stream.range_days, end_date=pendulum.parse("2021-11-15") - ) - assert mock_start_date == start_date and mock_end_date == end_date - - -SAMPLE_CONFIG_WITHOUT_END_DATE = { - "credentials": { - "developer_token": "developer_token", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token", - }, - "customer_id": "customer_id", - "start_date": "2021-11-01", -} - - -def test_get_date_params_without_end_date(customers): - # Please note that this is equal to inputted stream_slice start date + 1 day - mock_start_date = SAMPLE_CONFIG_WITHOUT_END_DATE["start_date"] - mock_end_date = "2021-11-30" - incremental_stream_config = dict( - start_date=mock_start_date, - end_date=mock_end_date, - conversion_window_days=0, - customers=customers, - api=MockGoogleAdsClient(SAMPLE_CONFIG_WITHOUT_END_DATE), - ) - stream = IncrementalGoogleAdsStream(**incremental_stream_config) - for customer in stream.customers: - start_date, end_date = get_date_params(start_date="2021-10-31", range_days=stream.range_days, time_zone=customer.time_zone) - assert mock_start_date == start_date - # There is a Google limitation where we capture only a 15-day date range - assert end_date == "2021-11-15" diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index e8d731006538..c520140822b7 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -110,43 +110,42 @@ def mock_fields_meta_data(): def test_chunk_date_range_without_end_date(): start_date_str = "2022-01-24" conversion_window = 0 - field = "date" - response = chunk_date_range( - start_date=start_date_str, conversion_window=conversion_window, field=field, end_date=None, days_of_data_storage=None, range_days=1 - ) + slices = list(chunk_date_range( + start_date=start_date_str, conversion_window=conversion_window, end_date=None, days_of_data_storage=None, range_days=1, time_zone="UTC" + )) expected_response = [ - {"start_date": "2022-01-25", "end_date": "2022-01-26"}, - {"start_date": "2022-01-26", "end_date": "2022-01-27"}, - {"start_date": "2022-01-27", "end_date": "2022-01-28"}, - {"start_date": "2022-01-28", "end_date": "2022-01-29"}, - {"start_date": "2022-01-29", "end_date": "2022-01-30"}, - {"start_date": "2022-01-30", "end_date": "2022-01-31"}, + {"start_date": "2022-01-24", "end_date": "2022-01-24"}, + {"start_date": "2022-01-25", "end_date": "2022-01-25"}, + {"start_date": "2022-01-26", "end_date": "2022-01-26"}, + {"start_date": "2022-01-27", "end_date": "2022-01-27"}, + {"start_date": "2022-01-28", "end_date": "2022-01-28"}, + {"start_date": "2022-01-29", "end_date": "2022-01-29"}, + {"start_date": "2022-01-30", "end_date": "2022-01-30"}, ] - assert expected_response == response + assert expected_response == slices def test_chunk_date_range(): start_date = "2021-03-04" end_date = "2021-05-04" conversion_window = 14 - field = "date" - response = chunk_date_range(start_date, conversion_window, field, end_date, range_days=10) + slices = list(chunk_date_range(start_date, conversion_window, end_date, range_days=10, time_zone="UTC")) assert [ - {"start_date": "2021-02-19", "end_date": "2021-02-28"}, - {"start_date": "2021-03-01", "end_date": "2021-03-10"}, - {"start_date": "2021-03-11", "end_date": "2021-03-20"}, - {"start_date": "2021-03-21", "end_date": "2021-03-30"}, - {"start_date": "2021-03-31", "end_date": "2021-04-09"}, - {"start_date": "2021-04-10", "end_date": "2021-04-19"}, - {"start_date": "2021-04-20", "end_date": "2021-04-29"}, - {"start_date": "2021-04-30", "end_date": "2021-05-04"}, - ] == response + {"start_date": "2021-02-18", "end_date": "2021-02-27"}, + {"start_date": "2021-02-28", "end_date": "2021-03-09"}, + {"start_date": "2021-03-10", "end_date": "2021-03-19"}, + {"start_date": "2021-03-20", "end_date": "2021-03-29"}, + {"start_date": "2021-03-30", "end_date": "2021-04-08"}, + {"start_date": "2021-04-09", "end_date": "2021-04-18"}, + {"start_date": "2021-04-19", "end_date": "2021-04-28"}, + {"start_date": "2021-04-29", "end_date": "2021-05-04"}, + ] == slices def test_streams_count(config, mock_account_info): source = SourceGoogleAds() streams = source.streams(config) - expected_streams_number = 19 + expected_streams_number = 29 assert len(streams) == expected_streams_number @@ -528,8 +527,8 @@ def test_stream_slices(config, customers): ) slices = list(stream.stream_slices()) assert slices == [ - {"start_date": "2020-12-19", "end_date": "2021-01-02", "customer_id": "123"}, - {"start_date": "2021-01-03", "end_date": "2021-01-17", "customer_id": "123"}, - {"start_date": "2021-01-18", "end_date": "2021-02-01", "customer_id": "123"}, - {"start_date": "2021-02-02", "end_date": "2021-02-10", "customer_id": "123"}, + {"start_date": "2020-12-18", "end_date": "2021-01-01", "customer_id": "123"}, + {"start_date": "2021-01-02", "end_date": "2021-01-16", "customer_id": "123"}, + {"start_date": "2021-01-17", "end_date": "2021-01-31", "customer_id": "123"}, + {"start_date": "2021-02-01", "end_date": "2021-02-10", "customer_id": "123"}, ] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index 26b38110ddd5..7c0f38fdd34a 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -223,6 +223,7 @@ def test_retry_transient_errors(mocker, config, customers, error_cls): def test_cyclic_sieve(caplog): original_logger = logging.getLogger("test") + original_logger.setLevel(logging.DEBUG) sieve = cyclic_sieve(original_logger, fraction=10) for _ in range(20): sieve.info("Ground Control to Major Tom") diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile index e522f2782acf..6c3b35f5861b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile @@ -28,5 +28,5 @@ COPY source_google_analytics_data_api ./source_google_analytics_data_api ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=1.1.3 LABEL io.airbyte.name=airbyte/source-google-analytics-data-api diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml index b1ff88d51fd7..c3206c26c842 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml @@ -13,7 +13,7 @@ acceptance_tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "exception" + status: "failed" discovery: tests: - config_path: "secrets/config.json" @@ -29,43 +29,21 @@ acceptance_tests: exact_order: no extra_records: yes ignored_fields: - daily_active_users: - - name: uuid - bypass_reason: "dynamically created field" - weekly_active_users: - - name: uuid - bypass_reason: "dynamically created field" - four_weekly_active_users: - - name: uuid - bypass_reason: "dynamically created field" devices: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamic field" locations: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamic field" pages: - - name: uuid - bypass_reason: "dynamically created field" - name: screenPageViews bypass_reason: "dynamically created field" - name: bounceRate bypass_reason: "dynamically created field" website_overview: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamically created field" - cohort_report: - - name: uuid - bypass_reason: "dynamically created field" pivot_report: - - name: uuid - bypass_reason: "dynamically created field" - name: sessions bypass_reason: "volatile data" full_refresh: @@ -73,47 +51,23 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" ignored_fields: - daily_active_users: - - name: uuid - bypass_reason: "dynamically created field" - weekly_active_users: - - name: uuid - bypass_reason: "dynamically created field" - four_weekly_active_users: - - name: uuid - bypass_reason: "dynamically created field" devices: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamic field" locations: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamic field" - pages: - - name: uuid - bypass_reason: "dynamically created field" traffic_sources: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamically created field" website_overview: - - name: uuid - bypass_reason: "dynamically created field" - name: averageSessionDuration bypass_reason: "dynamically created field" - cohort_report: - - name: uuid - bypass_reason: "dynamically created field" - pivot_report: - - name: uuid - bypass_reason: "dynamically created field" incremental: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" + timeout_seconds: 3600 + configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" + threshold_days: 2 diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json index 390b4952d02b..c1da32d744dc 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json @@ -7,10 +7,11 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [["property_id"], ["date"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] }, { "stream": { @@ -19,10 +20,11 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [["property_id"], ["date"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] }, { "stream": { @@ -31,10 +33,11 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [["property_id"], ["date"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] }, { "stream": { @@ -43,10 +46,23 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [ + ["property_id"], + ["date"], + ["deviceCategory"], + ["operatingSystem"], + ["browser"] + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [ + ["property_id"], + ["date"], + ["deviceCategory"], + ["operatingSystem"], + ["browser"] + ] }, { "stream": { @@ -55,10 +71,23 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [ + ["property_id"], + ["region"], + ["country"], + ["city"], + ["date"] + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [ + ["property_id"], + ["region"], + ["country"], + ["city"], + ["date"] + ] }, { "stream": { @@ -67,10 +96,21 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [ + ["property_id"], + ["date"], + ["hostName"], + ["pagePathPlusQueryString"] + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [ + ["property_id"], + ["date"], + ["hostName"], + ["pagePathPlusQueryString"] + ] }, { "stream": { @@ -79,10 +119,21 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [ + ["property_id"], + ["date"], + ["sessionSource"], + ["sessionMedium"] + ] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [ + ["property_id"], + ["date"], + ["sessionSource"], + ["sessionMedium"] + ] }, { "stream": { @@ -91,30 +142,51 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [["property_id"], ["date"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] }, { "stream": { "name": "cohort_report", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [ + ["property_id"], + ["cohort"], + ["cohortNthDay"] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["cohort"], ["cohortNthDay"]] }, { "stream": { "name": "pivot_report", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["uuid"]] + "source_defined_primary_key": [ + ["property_id"], + ["browser"], + ["country"], + ["language"], + ["startDate"], + ["endDate"] + ] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "overwrite", + "primary_key": [ + ["property_id"], + ["browser"], + ["country"], + ["language"], + ["startDate"], + ["endDate"] + ] } ] } diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl index f63d7d13fdfa..861748826431 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl @@ -1,90 +1,90 @@ -{"stream": "daily_active_users", "data": {"uuid": "8958ad05-9217-4232-af3a-842b2ef3066a", "property_id": "314186564", "date": "20230406", "active1DayUsers": 2562}, "emitted_at": 1681405954033} -{"stream": "daily_active_users", "data": {"uuid": "2ededb64-296c-4cd3-8285-258f952e6d01", "property_id": "314186564", "date": "20230403", "active1DayUsers": 2521}, "emitted_at": 1681405954034} -{"stream": "daily_active_users", "data": {"uuid": "aee7c406-9668-4e7d-8f1e-e4b6e944781a", "property_id": "314186564", "date": "20230404", "active1DayUsers": 2386}, "emitted_at": 1681405954034} -{"stream": "daily_active_users", "data": {"uuid": "8fdcf306-c6ce-418d-a8b0-114c3e0853e3", "property_id": "314186564", "date": "20230405", "active1DayUsers": 2318}, "emitted_at": 1681405954035} -{"stream": "daily_active_users", "data": {"uuid": "ddae49fa-6a49-409c-a3d9-972043e1297e", "property_id": "314186564", "date": "20230411", "active1DayUsers": 2248}, "emitted_at": 1681405954035} -{"stream": "daily_active_users", "data": {"uuid": "d3c0fe8d-729b-49d5-ba09-836d11c72b36", "property_id": "314186564", "date": "20230412", "active1DayUsers": 2164}, "emitted_at": 1681405954036} -{"stream": "daily_active_users", "data": {"uuid": "3bed51b3-fe19-4cfe-aa7b-2f834444e4ac", "property_id": "314186564", "date": "20230410", "active1DayUsers": 2021}, "emitted_at": 1681405954036} -{"stream": "daily_active_users", "data": {"uuid": "930dc88c-2176-44d7-b89d-f7a0deed7cb6", "property_id": "314186564", "date": "20230407", "active1DayUsers": 1628}, "emitted_at": 1681405954037} -{"stream": "daily_active_users", "data": {"uuid": "862aee10-bc17-4fcb-9917-c03dcf930b7a", "property_id": "314186564", "date": "20230409", "active1DayUsers": 1009}, "emitted_at": 1681405954037} -{"stream": "daily_active_users", "data": {"uuid": "888e8a03-f908-49ea-bc24-47d3e0c71ce5", "property_id": "314186564", "date": "20230402", "active1DayUsers": 978}, "emitted_at": 1681405954038} -{"stream": "weekly_active_users", "data": {"uuid": "552f657c-894d-42a5-b136-621a27c83f20", "property_id": "314186564", "date": "20230403", "active7DayUsers": 11840}, "emitted_at": 1681405954684} -{"stream": "weekly_active_users", "data": {"uuid": "ab335ba9-16a4-47f7-9475-0cd0c71e13c5", "property_id": "314186564", "date": "20230406", "active7DayUsers": 11828}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "c2249443-a731-454f-a92d-dcabde1e053e", "property_id": "314186564", "date": "20230404", "active7DayUsers": 11812}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "9031b092-0365-45f6-a3fd-a42697fc60e5", "property_id": "314186564", "date": "20230405", "active7DayUsers": 11751}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "686ec0f2-245a-41ee-8b2f-28c6ff58cca3", "property_id": "314186564", "date": "20230408", "active7DayUsers": 11745}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "c230f0bb-e08d-410c-aaf7-92ea55c1eb3c", "property_id": "314186564", "date": "20230409", "active7DayUsers": 11739}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "360eeddb-8e7c-4cf8-ba68-fe0f74bdd4b2", "property_id": "314186564", "date": "20230407", "active7DayUsers": 11637}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "fa08f066-b0fe-4cce-9ff1-aa51533a36a2", "property_id": "314186564", "date": "20230401", "active7DayUsers": 11547}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "99849845-7511-46ee-bb0f-b99fcbe86094", "property_id": "314186564", "date": "20230402", "active7DayUsers": 11521}, "emitted_at": 1681405954685} -{"stream": "weekly_active_users", "data": {"uuid": "1537fbde-6a1e-4676-ad38-3952c6c49517", "property_id": "314186564", "date": "20230410", "active7DayUsers": 11369}, "emitted_at": 1681405954686} -{"stream": "four_weekly_active_users", "data": {"uuid": "3ea1880f-1e1b-42f8-858f-d66c49400592", "property_id": "314186564", "date": "20230401", "active28DayUsers": 48082}, "emitted_at": 1681405955854} -{"stream": "four_weekly_active_users", "data": {"uuid": "e3697089-6a56-466f-893c-57506cce44f4", "property_id": "314186564", "date": "20230402", "active28DayUsers": 47927}, "emitted_at": 1681405955854} -{"stream": "four_weekly_active_users", "data": {"uuid": "2a76a756-b625-4291-9012-09d6449b4460", "property_id": "314186564", "date": "20230403", "active28DayUsers": 44678}, "emitted_at": 1681405955854} -{"stream": "four_weekly_active_users", "data": {"uuid": "30f93662-7a47-4455-bb87-4e23753d7836", "property_id": "314186564", "date": "20230404", "active28DayUsers": 42997}, "emitted_at": 1681405955854} -{"stream": "four_weekly_active_users", "data": {"uuid": "bf5980dd-18bd-4dd3-b6f7-123a8d9bc7ba", "property_id": "314186564", "date": "20230405", "active28DayUsers": 42219}, "emitted_at": 1681405955855} -{"stream": "four_weekly_active_users", "data": {"uuid": "095af0ca-ca67-44a9-acaf-fbe1f04beb8f", "property_id": "314186564", "date": "20230406", "active28DayUsers": 42028}, "emitted_at": 1681405955855} -{"stream": "four_weekly_active_users", "data": {"uuid": "29958f7e-91a1-4377-a5e7-763ee1fc26f7", "property_id": "314186564", "date": "20230407", "active28DayUsers": 41851}, "emitted_at": 1681405955855} -{"stream": "four_weekly_active_users", "data": {"uuid": "b5cd639e-99a4-4fd9-871e-9883a509d35e", "property_id": "314186564", "date": "20230408", "active28DayUsers": 41775}, "emitted_at": 1681405955855} -{"stream": "four_weekly_active_users", "data": {"uuid": "b85e1864-aca2-45ae-8bc2-614da510ac29", "property_id": "314186564", "date": "20230409", "active28DayUsers": 41717}, "emitted_at": 1681405955855} -{"stream": "four_weekly_active_users", "data": {"uuid": "505deea8-b4d0-4d76-9c78-1faaa8e3c725", "property_id": "314186564", "date": "20230410", "active28DayUsers": 41212}, "emitted_at": 1681405955855} -{"stream": "devices", "data": {"uuid": "f43b1120-b565-4174-a974-251de4ddb3e7", "property_id": "314186564", "date": "20230411", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 973, "newUsers": 368, "sessions": 1667, "sessionsPerUser": 2.110126582278481, "averageSessionDuration": 308.8923676994601, "screenPageViews": 5367, "screenPageViewsPerSession": 3.2195560887822436, "bounceRate": 0.498500299940012}, "emitted_at": 1681405958296} -{"stream": "devices", "data": {"uuid": "11fa7956-54f4-42ae-bf50-ccd7f78acb43", "property_id": "314186564", "date": "20230412", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 969, "newUsers": 350, "sessions": 1588, "sessionsPerUser": 2.0025220680958387, "averageSessionDuration": 336.108126070529, "screenPageViews": 4726, "screenPageViewsPerSession": 2.9760705289672544, "bounceRate": 0.5012594458438288}, "emitted_at": 1681405958296} -{"stream": "devices", "data": {"uuid": "3a7a9be6-715f-407d-964d-36a335d1f89c", "property_id": "314186564", "date": "20230404", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 942, "newUsers": 352, "sessions": 1554, "sessionsPerUser": 2.007751937984496, "averageSessionDuration": 328.66656451029604, "screenPageViews": 5217, "screenPageViewsPerSession": 3.357142857142857, "bounceRate": 0.4954954954954955}, "emitted_at": 1681405958296} -{"stream": "devices", "data": {"uuid": "d174c54a-7091-4736-b921-c191bf5dd4b6", "property_id": "314186564", "date": "20230406", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 942, "newUsers": 389, "sessions": 1551, "sessionsPerUser": 1.9783163265306123, "averageSessionDuration": 357.5382107272727, "screenPageViews": 5102, "screenPageViewsPerSession": 3.289490651192779, "bounceRate": 0.49258542875564154}, "emitted_at": 1681405958297} -{"stream": "devices", "data": {"uuid": "889df6a1-f480-483e-8957-d4ca767028f8", "property_id": "314186564", "date": "20230403", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 929, "newUsers": 341, "sessions": 1546, "sessionsPerUser": 2.0558510638297873, "averageSessionDuration": 315.4776974385511, "screenPageViews": 5116, "screenPageViewsPerSession": 3.309184993531695, "bounceRate": 0.5071151358344114}, "emitted_at": 1681405958297} -{"stream": "devices", "data": {"uuid": "7a12fdb1-65bd-4f9d-9b28-f7d4bd9efc45", "property_id": "314186564", "date": "20230405", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 926, "newUsers": 363, "sessions": 1573, "sessionsPerUser": 2.0428571428571427, "averageSessionDuration": 346.09502719898285, "screenPageViews": 5032, "screenPageViewsPerSession": 3.1989828353464715, "bounceRate": 0.4869675778766688}, "emitted_at": 1681405958297} -{"stream": "devices", "data": {"uuid": "36283f3b-12f0-44bf-af7e-fa17790a536a", "property_id": "314186564", "date": "20230410", "deviceCategory": "desktop", "operatingSystem": "Macintosh", "browser": "Chrome", "totalUsers": 920, "newUsers": 374, "sessions": 1524, "sessionsPerUser": 2.0456375838926175, "averageSessionDuration": 255.77025801837266, "screenPageViews": 4025, "screenPageViewsPerSession": 2.641076115485564, "bounceRate": 0.5255905511811023}, "emitted_at": 1681405958297} -{"stream": "devices", "data": {"uuid": "9ed102f9-6670-4883-890a-86d647e3e08a", "property_id": "314186564", "date": "20230403", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Chrome", "totalUsers": 781, "newUsers": 366, "sessions": 1184, "sessionsPerUser": 1.8528951486697967, "averageSessionDuration": 278.84846059881755, "screenPageViews": 2993, "screenPageViewsPerSession": 2.5278716216216215, "bounceRate": 0.5616554054054054}, "emitted_at": 1681405958297} -{"stream": "devices", "data": {"uuid": "a4d1798d-40ec-4653-b3ec-2c4951fa7323", "property_id": "314186564", "date": "20230411", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Chrome", "totalUsers": 760, "newUsers": 365, "sessions": 1155, "sessionsPerUser": 1.896551724137931, "averageSessionDuration": 264.1307251896104, "screenPageViews": 2452, "screenPageViewsPerSession": 2.122943722943723, "bounceRate": 0.5316017316017316}, "emitted_at": 1681405958298} -{"stream": "devices", "data": {"uuid": "0f4cb5c7-703f-4e21-9efd-828da56eb03b", "property_id": "314186564", "date": "20230404", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Chrome", "totalUsers": 727, "newUsers": 345, "sessions": 1137, "sessionsPerUser": 1.8517915309446253, "averageSessionDuration": 252.06245670272648, "screenPageViews": 2601, "screenPageViewsPerSession": 2.287598944591029, "bounceRate": 0.5488126649076517}, "emitted_at": 1681405958298} -{"stream": "locations", "data": {"uuid": "5606f203-9304-4b42-82c8-af3d825cd34a", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230406", "totalUsers": 108, "newUsers": 62, "sessions": 157, "sessionsPerUser": 1.6354166666666667, "averageSessionDuration": 435.44268001273895, "screenPageViews": 534, "screenPageViewsPerSession": 3.4012738853503186, "bounceRate": 0.5031847133757962}, "emitted_at": 1681405962136} -{"stream": "locations", "data": {"uuid": "759886f0-3723-44bf-ba3f-1b994169ed85", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230405", "totalUsers": 95, "newUsers": 54, "sessions": 123, "sessionsPerUser": 1.5769230769230769, "averageSessionDuration": 499.2074986666667, "screenPageViews": 481, "screenPageViewsPerSession": 3.910569105691057, "bounceRate": 0.44715447154471544}, "emitted_at": 1681405962136} -{"stream": "locations", "data": {"uuid": "81c1cd25-8d43-48a9-836b-e10b1991466a", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230403", "totalUsers": 94, "newUsers": 46, "sessions": 126, "sessionsPerUser": 1.68, "averageSessionDuration": 424.00281903174607, "screenPageViews": 499, "screenPageViewsPerSession": 3.9603174603174605, "bounceRate": 0.5238095238095238}, "emitted_at": 1681405962136} -{"stream": "locations", "data": {"uuid": "1a833419-629e-4678-9fb2-f97f10da5d40", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230404", "totalUsers": 85, "newUsers": 47, "sessions": 121, "sessionsPerUser": 1.6575342465753424, "averageSessionDuration": 378.81275640495863, "screenPageViews": 434, "screenPageViewsPerSession": 3.5867768595041323, "bounceRate": 0.48760330578512395}, "emitted_at": 1681405962136} -{"stream": "locations", "data": {"uuid": "577fd852-67a2-43c3-8321-56fa00dac58c", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230412", "totalUsers": 85, "newUsers": 49, "sessions": 131, "sessionsPerUser": 1.8194444444444444, "averageSessionDuration": 379.1322029236641, "screenPageViews": 391, "screenPageViewsPerSession": 2.984732824427481, "bounceRate": 0.5267175572519084}, "emitted_at": 1681405962137} -{"stream": "locations", "data": {"uuid": "bb0f3fa2-dc9c-475d-a47c-ace82c720da2", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230410", "totalUsers": 81, "newUsers": 42, "sessions": 135, "sessionsPerUser": 1.9565217391304348, "averageSessionDuration": 303.13140742962963, "screenPageViews": 376, "screenPageViewsPerSession": 2.785185185185185, "bounceRate": 0.5407407407407407}, "emitted_at": 1681405962137} -{"stream": "locations", "data": {"uuid": "f7057f58-fbbd-4947-ae37-a7a29d6700ac", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230411", "totalUsers": 81, "newUsers": 38, "sessions": 123, "sessionsPerUser": 1.9523809523809523, "averageSessionDuration": 362.51537134146344, "screenPageViews": 312, "screenPageViewsPerSession": 2.5365853658536586, "bounceRate": 0.5934959349593496}, "emitted_at": 1681405962137} -{"stream": "locations", "data": {"uuid": "afb62a0d-5e5a-4f88-8367-3fe45a35d06f", "property_id": "314186564", "region": "Karnataka", "country": "India", "city": "Bengaluru", "date": "20230411", "totalUsers": 76, "newUsers": 52, "sessions": 123, "sessionsPerUser": 1.8636363636363635, "averageSessionDuration": 203.00314456910567, "screenPageViews": 261, "screenPageViewsPerSession": 2.1219512195121952, "bounceRate": 0.4959349593495935}, "emitted_at": 1681405962137} -{"stream": "locations", "data": {"uuid": "ce319e7f-8f8f-4c73-9f45-9a80a08bcf33", "property_id": "314186564", "region": "Karnataka", "country": "India", "city": "Bengaluru", "date": "20230403", "totalUsers": 69, "newUsers": 34, "sessions": 102, "sessionsPerUser": 1.728813559322034, "averageSessionDuration": 256.4942830490196, "screenPageViews": 216, "screenPageViewsPerSession": 2.1176470588235294, "bounceRate": 0.5490196078431373}, "emitted_at": 1681405962137} -{"stream": "locations", "data": {"uuid": "28fa18ea-3a0c-4c09-9ffc-b24d2d5abeaf", "property_id": "314186564", "region": "New York", "country": "United States", "city": "New York", "date": "20230407", "totalUsers": 69, "newUsers": 30, "sessions": 98, "sessionsPerUser": 1.849056603773585, "averageSessionDuration": 489.54009168367344, "screenPageViews": 376, "screenPageViewsPerSession": 3.836734693877551, "bounceRate": 0.4489795918367347}, "emitted_at": 1681405962137} -{"stream": "pages", "data": {"uuid": "1d042583-6008-4bb8-abad-a1d3a171c9c5", "property_id": "314186564", "date": "20230405", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1190, "bounceRate": 0.5516393442622951}, "emitted_at": 1681405967183} -{"stream": "pages", "data": {"uuid": "e24096ca-ad32-49eb-824a-2a65e52b9a08", "property_id": "314186564", "date": "20230411", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1151, "bounceRate": 0.5400641025641025}, "emitted_at": 1681405967184} -{"stream": "pages", "data": {"uuid": "8911f922-c707-4af1-95b5-18a81e31b5af", "property_id": "314186564", "date": "20230404", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1137, "bounceRate": 0.5617232808616405}, "emitted_at": 1681405967184} -{"stream": "pages", "data": {"uuid": "315dad3c-81e5-432f-a47c-134e89a7cf42", "property_id": "314186564", "date": "20230410", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1099, "bounceRate": 0.5416666666666666}, "emitted_at": 1681405967184} -{"stream": "pages", "data": {"uuid": "8ba00398-80cf-4a17-afba-2cf223244cfe", "property_id": "314186564", "date": "20230403", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1092, "bounceRate": 0.5569070373588184}, "emitted_at": 1681405967184} -{"stream": "pages", "data": {"uuid": "0d62e64a-4da9-46b8-8e90-e1180ea9b1ab", "property_id": "314186564", "date": "20230412", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1089, "bounceRate": 0.5690515806988353}, "emitted_at": 1681405967184} -{"stream": "pages", "data": {"uuid": "d5a7b5bf-b8a9-4f29-8a06-0e4bcfee17a7", "property_id": "314186564", "date": "20230406", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 1005, "bounceRate": 0.5516279069767441}, "emitted_at": 1681405967185} -{"stream": "pages", "data": {"uuid": "e77044bf-1cd6-4ef0-a475-ff6255ac1cf8", "property_id": "314186564", "date": "20230407", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 734, "bounceRate": 0.571619812583668}, "emitted_at": 1681405967185} -{"stream": "pages", "data": {"uuid": "2c0db575-cf8a-4cca-955d-baa2032bec54", "property_id": "314186564", "date": "20230403", "hostName": "airbyte.com", "pagePathPlusQueryString": "/blog/data-modeling-unsung-hero-data-engineering-introduction", "screenPageViews": 541, "bounceRate": 0.7192691029900332}, "emitted_at": 1681405967185} -{"stream": "pages", "data": {"uuid": "4d4ea3e6-91ac-4da7-baf6-4e8cd6665b8c", "property_id": "314186564", "date": "20230402", "hostName": "airbyte.com", "pagePathPlusQueryString": "/", "screenPageViews": 529, "bounceRate": 0.5614678899082569}, "emitted_at": 1681405967185} -{"stream": "website_overview", "data": {"uuid": "3175b7c6-8257-4111-b15a-5c97e16437fe", "property_id": "314186564", "date": "20230406", "totalUsers": 3014, "newUsers": 1539, "sessions": 4257, "sessionsPerUser": 1.661592505854801, "averageSessionDuration": 270.9253856281419, "screenPageViews": 10839, "screenPageViewsPerSession": 2.5461592670894997, "bounceRate": 0.5391120507399577}, "emitted_at": 1681405971634} -{"stream": "website_overview", "data": {"uuid": "c376b15d-d38e-4caf-816d-9dc2b8f04d8f", "property_id": "314186564", "date": "20230403", "totalUsers": 2988, "newUsers": 1461, "sessions": 4350, "sessionsPerUser": 1.725505751685839, "averageSessionDuration": 246.36103450390806, "screenPageViews": 10749, "screenPageViewsPerSession": 2.4710344827586206, "bounceRate": 0.5618390804597702}, "emitted_at": 1681405971634} -{"stream": "website_overview", "data": {"uuid": "28294b7a-85ad-4d73-bd11-20e5041c7de9", "property_id": "314186564", "date": "20230404", "totalUsers": 2817, "newUsers": 1367, "sessions": 4153, "sessionsPerUser": 1.7405699916177704, "averageSessionDuration": 259.69049313965803, "screenPageViews": 10653, "screenPageViewsPerSession": 2.5651336383337346, "bounceRate": 0.5379243920057789}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "7cba1969-8d97-4f56-b1c2-07795408ec05", "property_id": "314186564", "date": "20230405", "totalUsers": 2754, "newUsers": 1333, "sessions": 4004, "sessionsPerUser": 1.727351164797239, "averageSessionDuration": 290.08648263536463, "screenPageViews": 10737, "screenPageViewsPerSession": 2.6815684315684316, "bounceRate": 0.5072427572427572}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "19b5c257-3ff5-4f76-af0a-e8d54915fcf5", "property_id": "314186564", "date": "20230411", "totalUsers": 2730, "newUsers": 1273, "sessions": 4006, "sessionsPerUser": 1.7820284697508897, "averageSessionDuration": 256.8832527284074, "screenPageViews": 10073, "screenPageViewsPerSession": 2.514478282576136, "bounceRate": 0.5162256615077384}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "14c67449-e3d5-4e59-af8e-8fc8dd3880db", "property_id": "314186564", "date": "20230412", "totalUsers": 2642, "newUsers": 1215, "sessions": 3940, "sessionsPerUser": 1.820702402957486, "averageSessionDuration": 281.3629124893401, "screenPageViews": 10621, "screenPageViewsPerSession": 2.6956852791878174, "bounceRate": 0.5309644670050762}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "ffa23a73-b5b1-41f6-ae2e-b27a4b6e020e", "property_id": "314186564", "date": "20230410", "totalUsers": 2409, "newUsers": 1173, "sessions": 3602, "sessionsPerUser": 1.7822859970311726, "averageSessionDuration": 252.51497996779568, "screenPageViews": 8973, "screenPageViewsPerSession": 2.491116046640755, "bounceRate": 0.524153248195447}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "e42b5011-0d04-4e34-8b40-03b463f8537c", "property_id": "314186564", "date": "20230407", "totalUsers": 1950, "newUsers": 974, "sessions": 2710, "sessionsPerUser": 1.6646191646191646, "averageSessionDuration": 261.6388968815498, "screenPageViews": 6972, "screenPageViewsPerSession": 2.572693726937269, "bounceRate": 0.5431734317343173}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "93775261-4ea3-4fb4-babb-4110b6843fbb", "property_id": "314186564", "date": "20230409", "totalUsers": 1277, "newUsers": 664, "sessions": 1661, "sessionsPerUser": 1.6461843409316155, "averageSessionDuration": 199.5610062384106, "screenPageViews": 3300, "screenPageViewsPerSession": 1.9867549668874172, "bounceRate": 0.5605057194461168}, "emitted_at": 1681405971635} -{"stream": "website_overview", "data": {"uuid": "4df90187-80fa-433e-8c99-bfa28764c5a3", "property_id": "314186564", "date": "20230402", "totalUsers": 1185, "newUsers": 605, "sessions": 1505, "sessionsPerUser": 1.5388548057259714, "averageSessionDuration": 221.2044838358804, "screenPageViews": 3260, "screenPageViewsPerSession": 2.166112956810631, "bounceRate": 0.5348837209302325}, "emitted_at": 1681405971636} -{"stream": "cohort_report", "data": {"uuid": "f75938fd-3274-4807-8c37-ce7dd0c3a790", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0000", "cohortActiveUsers": 731}, "emitted_at": 1681405973101} -{"stream": "cohort_report", "data": {"uuid": "8f228a9c-57b8-4329-9de7-f5fb56ee18b6", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0001", "cohortActiveUsers": 25}, "emitted_at": 1681405973101} -{"stream": "cohort_report", "data": {"uuid": "f79f8d4d-4166-4fcd-9ca0-0704c3b9af13", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0002", "cohortActiveUsers": 9}, "emitted_at": 1681405973101} -{"stream": "cohort_report", "data": {"uuid": "d47f77ef-5e72-4c66-9360-cee92352deaf", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0003", "cohortActiveUsers": 6}, "emitted_at": 1681405973101} -{"stream": "cohort_report", "data": {"uuid": "d90efbfd-b320-45ea-9b51-672fa86d01bb", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0004", "cohortActiveUsers": 4}, "emitted_at": 1681405973101} -{"stream": "cohort_report", "data": {"uuid": "8b2ded37-ede8-4f27-932f-b6b357c363a8", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0009", "cohortActiveUsers": 4}, "emitted_at": 1681405973102} -{"stream": "cohort_report", "data": {"uuid": "6e8e19a2-a5cb-4a82-b800-23f38a0f5268", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0010", "cohortActiveUsers": 4}, "emitted_at": 1681405973102} -{"stream": "cohort_report", "data": {"uuid": "38fc20ce-d8f1-459c-b880-44851128a923", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0011", "cohortActiveUsers": 4}, "emitted_at": 1681405973102} -{"stream": "cohort_report", "data": {"uuid": "8a77d306-0c5d-431c-b2a7-6b2c87383719", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0013", "cohortActiveUsers": 4}, "emitted_at": 1681405973102} -{"stream": "cohort_report", "data": {"uuid": "105f7916-0d45-4e17-98db-c6ea3e7d5223", "property_id": "314186564", "cohort": "cohort_0", "cohortNthDay": "0025", "cohortActiveUsers": 4}, "emitted_at": 1681405973102} -{"stream": "pivot_report", "data": {"uuid": "a220f05f-5185-41b7-9730-379875cc8866", "property_id": "314186564", "browser": "Chrome", "country": "United States", "language": "English", "sessions": 10441}, "emitted_at": 1681405974686} -{"stream": "pivot_report", "data": {"uuid": "d01ac2b1-8947-4231-89f6-822fc7b08a01", "property_id": "314186564", "browser": "Chrome", "country": "India", "language": "English", "sessions": 4170}, "emitted_at": 1681405974686} -{"stream": "pivot_report", "data": {"uuid": "cd9027aa-8226-418c-9b60-9d6856327a93", "property_id": "314186564", "browser": "Safari", "country": "United States", "language": "English", "sessions": 1798}, "emitted_at": 1681405974686} -{"stream": "pivot_report", "data": {"uuid": "79a69e5c-0ceb-45e7-b789-5a4904ce273d", "property_id": "314186564", "browser": "Chrome", "country": "Canada", "language": "English", "sessions": 961}, "emitted_at": 1681405974686} -{"stream": "pivot_report", "data": {"uuid": "c9d18f96-3762-47e0-b322-72ad7953c645", "property_id": "314186564", "browser": "Chrome", "country": "United Kingdom", "language": "English", "sessions": 740}, "emitted_at": 1681405974686} -{"stream": "pivot_report", "data": {"uuid": "5688b831-a3be-4b5a-ae0b-f464b8098de4", "property_id": "314186564", "browser": "Chrome", "country": "Australia", "language": "English", "sessions": 621}, "emitted_at": 1681405974687} -{"stream": "pivot_report", "data": {"uuid": "fac29ad5-6fcb-42ef-ac2f-31b782c391cd", "property_id": "314186564", "browser": "Chrome", "country": "Brazil", "language": "Portuguese", "sessions": 566}, "emitted_at": 1681405974687} -{"stream": "pivot_report", "data": {"uuid": "a4e75b7f-034f-4511-8491-70c801950e93", "property_id": "314186564", "browser": "Chrome", "country": "Vietnam", "language": "English", "sessions": 565}, "emitted_at": 1681405974687} -{"stream": "pivot_report", "data": {"uuid": "ad237798-ae12-46bd-892d-0d8b28acca8e", "property_id": "314186564", "browser": "Edge", "country": "United States", "language": "English", "sessions": 555}, "emitted_at": 1681405974687} -{"stream": "pivot_report", "data": {"uuid": "8a06cd72-4789-4f36-bf5b-f2c860e1aece", "property_id": "314186564", "browser": "Chrome", "country": "Pakistan", "language": "English", "sessions": 522}, "emitted_at": 1681405974687} \ No newline at end of file +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230406","active1DayUsers":2562},"emitted_at":1681405954033} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230403","active1DayUsers":2521},"emitted_at":1681405954034} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230404","active1DayUsers":2386},"emitted_at":1681405954034} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230405","active1DayUsers":2318},"emitted_at":1681405954035} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230411","active1DayUsers":2248},"emitted_at":1681405954035} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230412","active1DayUsers":2164},"emitted_at":1681405954036} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230410","active1DayUsers":2021},"emitted_at":1681405954036} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230407","active1DayUsers":1628},"emitted_at":1681405954037} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230409","active1DayUsers":1009},"emitted_at":1681405954037} +{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230402","active1DayUsers":978},"emitted_at":1681405954038} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230403","active7DayUsers":11840},"emitted_at":1681405954684} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230406","active7DayUsers":11828},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230404","active7DayUsers":11812},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230405","active7DayUsers":11751},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230408","active7DayUsers":11745},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230409","active7DayUsers":11739},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230407","active7DayUsers":11637},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230401","active7DayUsers":11547},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230402","active7DayUsers":11521},"emitted_at":1681405954685} +{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230410","active7DayUsers":11369},"emitted_at":1681405954686} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230401","active28DayUsers":48082},"emitted_at":1681405955854} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230402","active28DayUsers":47927},"emitted_at":1681405955854} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230403","active28DayUsers":44678},"emitted_at":1681405955854} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230404","active28DayUsers":42997},"emitted_at":1681405955854} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230405","active28DayUsers":42219},"emitted_at":1681405955855} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230406","active28DayUsers":42028},"emitted_at":1681405955855} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230407","active28DayUsers":41851},"emitted_at":1681405955855} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230408","active28DayUsers":41775},"emitted_at":1681405955855} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230409","active28DayUsers":41717},"emitted_at":1681405955855} +{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230410","active28DayUsers":41212},"emitted_at":1681405955855} +{"stream":"devices","data":{"property_id":"314186564","date":"20230411","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":973,"newUsers":368,"sessions":1667,"sessionsPerUser":2.110126582278481,"averageSessionDuration":308.8923676994601,"screenPageViews":5367,"screenPageViewsPerSession":3.2195560887822436,"bounceRate":0.498500299940012},"emitted_at":1681405958296} +{"stream":"devices","data":{"property_id":"314186564","date":"20230412","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":969,"newUsers":350,"sessions":1588,"sessionsPerUser":2.0025220680958387,"averageSessionDuration":336.108126070529,"screenPageViews":4726,"screenPageViewsPerSession":2.9760705289672544,"bounceRate":0.5012594458438288},"emitted_at":1681405958296} +{"stream":"devices","data":{"property_id":"314186564","date":"20230404","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":942,"newUsers":352,"sessions":1554,"sessionsPerUser":2.007751937984496,"averageSessionDuration":328.66656451029604,"screenPageViews":5217,"screenPageViewsPerSession":3.357142857142857,"bounceRate":0.4954954954954955},"emitted_at":1681405958296} +{"stream":"devices","data":{"property_id":"314186564","date":"20230406","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":942,"newUsers":389,"sessions":1551,"sessionsPerUser":1.9783163265306123,"averageSessionDuration":357.5382107272727,"screenPageViews":5102,"screenPageViewsPerSession":3.289490651192779,"bounceRate":0.49258542875564154},"emitted_at":1681405958297} +{"stream":"devices","data":{"property_id":"314186564","date":"20230403","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":929,"newUsers":341,"sessions":1546,"sessionsPerUser":2.0558510638297873,"averageSessionDuration":315.4776974385511,"screenPageViews":5116,"screenPageViewsPerSession":3.309184993531695,"bounceRate":0.5071151358344114},"emitted_at":1681405958297} +{"stream":"devices","data":{"property_id":"314186564","date":"20230405","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":926,"newUsers":363,"sessions":1573,"sessionsPerUser":2.0428571428571427,"averageSessionDuration":346.09502719898285,"screenPageViews":5032,"screenPageViewsPerSession":3.1989828353464715,"bounceRate":0.4869675778766688},"emitted_at":1681405958297} +{"stream":"devices","data":{"property_id":"314186564","date":"20230410","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":920,"newUsers":374,"sessions":1524,"sessionsPerUser":2.0456375838926175,"averageSessionDuration":255.77025801837266,"screenPageViews":4025,"screenPageViewsPerSession":2.641076115485564,"bounceRate":0.5255905511811023},"emitted_at":1681405958297} +{"stream":"devices","data":{"property_id":"314186564","date":"20230403","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":781,"newUsers":366,"sessions":1184,"sessionsPerUser":1.8528951486697967,"averageSessionDuration":278.84846059881755,"screenPageViews":2993,"screenPageViewsPerSession":2.5278716216216215,"bounceRate":0.5616554054054054},"emitted_at":1681405958297} +{"stream":"devices","data":{"property_id":"314186564","date":"20230411","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":760,"newUsers":365,"sessions":1155,"sessionsPerUser":1.896551724137931,"averageSessionDuration":264.1307251896104,"screenPageViews":2452,"screenPageViewsPerSession":2.122943722943723,"bounceRate":0.5316017316017316},"emitted_at":1681405958298} +{"stream":"devices","data":{"property_id":"314186564","date":"20230404","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":727,"newUsers":345,"sessions":1137,"sessionsPerUser":1.8517915309446253,"averageSessionDuration":252.06245670272648,"screenPageViews":2601,"screenPageViewsPerSession":2.287598944591029,"bounceRate":0.5488126649076517},"emitted_at":1681405958298} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230406","totalUsers":108,"newUsers":62,"sessions":157,"sessionsPerUser":1.6354166666666667,"averageSessionDuration":435.44268001273895,"screenPageViews":534,"screenPageViewsPerSession":3.4012738853503186,"bounceRate":0.5031847133757962},"emitted_at":1681405962136} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230405","totalUsers":95,"newUsers":54,"sessions":123,"sessionsPerUser":1.5769230769230769,"averageSessionDuration":499.2074986666667,"screenPageViews":481,"screenPageViewsPerSession":3.910569105691057,"bounceRate":0.44715447154471544},"emitted_at":1681405962136} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230403","totalUsers":94,"newUsers":46,"sessions":126,"sessionsPerUser":1.68,"averageSessionDuration":424.00281903174607,"screenPageViews":499,"screenPageViewsPerSession":3.9603174603174605,"bounceRate":0.5238095238095238},"emitted_at":1681405962136} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230404","totalUsers":85,"newUsers":47,"sessions":121,"sessionsPerUser":1.6575342465753424,"averageSessionDuration":378.81275640495863,"screenPageViews":434,"screenPageViewsPerSession":3.5867768595041323,"bounceRate":0.48760330578512395},"emitted_at":1681405962136} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230412","totalUsers":85,"newUsers":49,"sessions":131,"sessionsPerUser":1.8194444444444444,"averageSessionDuration":379.1322029236641,"screenPageViews":391,"screenPageViewsPerSession":2.984732824427481,"bounceRate":0.5267175572519084},"emitted_at":1681405962137} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230410","totalUsers":81,"newUsers":42,"sessions":135,"sessionsPerUser":1.9565217391304348,"averageSessionDuration":303.13140742962963,"screenPageViews":376,"screenPageViewsPerSession":2.785185185185185,"bounceRate":0.5407407407407407},"emitted_at":1681405962137} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230411","totalUsers":81,"newUsers":38,"sessions":123,"sessionsPerUser":1.9523809523809523,"averageSessionDuration":362.51537134146344,"screenPageViews":312,"screenPageViewsPerSession":2.5365853658536586,"bounceRate":0.5934959349593496},"emitted_at":1681405962137} +{"stream":"locations","data":{"property_id":"314186564","region":"Karnataka","country":"India","city":"Bengaluru","date":"20230411","totalUsers":76,"newUsers":52,"sessions":123,"sessionsPerUser":1.8636363636363635,"averageSessionDuration":203.00314456910567,"screenPageViews":261,"screenPageViewsPerSession":2.1219512195121952,"bounceRate":0.4959349593495935},"emitted_at":1681405962137} +{"stream":"locations","data":{"property_id":"314186564","region":"Karnataka","country":"India","city":"Bengaluru","date":"20230403","totalUsers":69,"newUsers":34,"sessions":102,"sessionsPerUser":1.728813559322034,"averageSessionDuration":256.4942830490196,"screenPageViews":216,"screenPageViewsPerSession":2.1176470588235294,"bounceRate":0.5490196078431373},"emitted_at":1681405962137} +{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230407","totalUsers":69,"newUsers":30,"sessions":98,"sessionsPerUser":1.849056603773585,"averageSessionDuration":489.54009168367344,"screenPageViews":376,"screenPageViewsPerSession":3.836734693877551,"bounceRate":0.4489795918367347},"emitted_at":1681405962137} +{"stream":"pages","data":{"property_id":"314186564","date":"20230405","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1190,"bounceRate":0.5516393442622951},"emitted_at":1681405967183} +{"stream":"pages","data":{"property_id":"314186564","date":"20230411","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1151,"bounceRate":0.5400641025641025},"emitted_at":1681405967184} +{"stream":"pages","data":{"property_id":"314186564","date":"20230404","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1137,"bounceRate":0.5617232808616405},"emitted_at":1681405967184} +{"stream":"pages","data":{"property_id":"314186564","date":"20230410","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1099,"bounceRate":0.5416666666666666},"emitted_at":1681405967184} +{"stream":"pages","data":{"property_id":"314186564","date":"20230403","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1092,"bounceRate":0.5569070373588184},"emitted_at":1681405967184} +{"stream":"pages","data":{"property_id":"314186564","date":"20230412","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1089,"bounceRate":0.5690515806988353},"emitted_at":1681405967184} +{"stream":"pages","data":{"property_id":"314186564","date":"20230406","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1005,"bounceRate":0.5516279069767441},"emitted_at":1681405967185} +{"stream":"pages","data":{"property_id":"314186564","date":"20230407","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":734,"bounceRate":0.571619812583668},"emitted_at":1681405967185} +{"stream":"pages","data":{"property_id":"314186564","date":"20230403","hostName":"airbyte.com","pagePathPlusQueryString":"/blog/data-modeling-unsung-hero-data-engineering-introduction","screenPageViews":541,"bounceRate":0.7192691029900332},"emitted_at":1681405967185} +{"stream":"pages","data":{"property_id":"314186564","date":"20230402","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":529,"bounceRate":0.5614678899082569},"emitted_at":1681405967185} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230406","totalUsers":3014,"newUsers":1539,"sessions":4257,"sessionsPerUser":1.661592505854801,"averageSessionDuration":270.9253856281419,"screenPageViews":10839,"screenPageViewsPerSession":2.5461592670894997,"bounceRate":0.5391120507399577},"emitted_at":1681405971634} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230403","totalUsers":2988,"newUsers":1461,"sessions":4350,"sessionsPerUser":1.725505751685839,"averageSessionDuration":246.36103450390806,"screenPageViews":10749,"screenPageViewsPerSession":2.4710344827586206,"bounceRate":0.5618390804597702},"emitted_at":1681405971634} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230404","totalUsers":2817,"newUsers":1367,"sessions":4153,"sessionsPerUser":1.7405699916177704,"averageSessionDuration":259.69049313965803,"screenPageViews":10653,"screenPageViewsPerSession":2.5651336383337346,"bounceRate":0.5379243920057789},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230405","totalUsers":2754,"newUsers":1333,"sessions":4004,"sessionsPerUser":1.727351164797239,"averageSessionDuration":290.08648263536463,"screenPageViews":10737,"screenPageViewsPerSession":2.6815684315684316,"bounceRate":0.5072427572427572},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230411","totalUsers":2730,"newUsers":1273,"sessions":4006,"sessionsPerUser":1.7820284697508897,"averageSessionDuration":256.8832527284074,"screenPageViews":10073,"screenPageViewsPerSession":2.514478282576136,"bounceRate":0.5162256615077384},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230412","totalUsers":2642,"newUsers":1215,"sessions":3940,"sessionsPerUser":1.820702402957486,"averageSessionDuration":281.3629124893401,"screenPageViews":10621,"screenPageViewsPerSession":2.6956852791878174,"bounceRate":0.5309644670050762},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230410","totalUsers":2409,"newUsers":1173,"sessions":3602,"sessionsPerUser":1.7822859970311726,"averageSessionDuration":252.51497996779568,"screenPageViews":8973,"screenPageViewsPerSession":2.491116046640755,"bounceRate":0.524153248195447},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230407","totalUsers":1950,"newUsers":974,"sessions":2710,"sessionsPerUser":1.6646191646191646,"averageSessionDuration":261.6388968815498,"screenPageViews":6972,"screenPageViewsPerSession":2.572693726937269,"bounceRate":0.5431734317343173},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230409","totalUsers":1277,"newUsers":664,"sessions":1661,"sessionsPerUser":1.6461843409316155,"averageSessionDuration":199.5610062384106,"screenPageViews":3300,"screenPageViewsPerSession":1.9867549668874172,"bounceRate":0.5605057194461168},"emitted_at":1681405971635} +{"stream":"website_overview","data":{"property_id":"314186564","date":"20230402","totalUsers":1185,"newUsers":605,"sessions":1505,"sessionsPerUser":1.5388548057259714,"averageSessionDuration":221.2044838358804,"screenPageViews":3260,"screenPageViewsPerSession":2.166112956810631,"bounceRate":0.5348837209302325},"emitted_at":1681405971636} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0000","cohortActiveUsers":731},"emitted_at":1681405973101} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0001","cohortActiveUsers":25},"emitted_at":1681405973101} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0002","cohortActiveUsers":9},"emitted_at":1681405973101} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0003","cohortActiveUsers":6},"emitted_at":1681405973101} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0004","cohortActiveUsers":4},"emitted_at":1681405973101} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0009","cohortActiveUsers":4},"emitted_at":1681405973102} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0010","cohortActiveUsers":4},"emitted_at":1681405973102} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0011","cohortActiveUsers":4},"emitted_at":1681405973102} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0013","cohortActiveUsers":4},"emitted_at":1681405973102} +{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0025","cohortActiveUsers":4},"emitted_at":1681405973102} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"United States","language":"English","sessions":24293,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261616} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"India","language":"English","sessions":9419,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Safari","country":"United States","language":"English","sessions":3863,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Canada","language":"English","sessions":2560,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"United Kingdom","language":"English","sessions":1964,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Edge","country":"United States","language":"English","sessions":1351,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Australia","language":"English","sessions":1307,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Brazil","language":"Portuguese","sessions":1302,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Vietnam","language":"English","sessions":1196,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} +{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Germany","language":"English","sessions":963,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/incremental_catalog.json deleted file mode 100644 index 0900aa6dbfed..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/incremental_catalog.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "daily_active_users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "weekly_active_users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "four_weekly_active_users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "devices", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "locations", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "pages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "traffic_sources", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "website_overview", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["uuid"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml index 245b87dc8cdb..26825ee071fc 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: api connectorType: source definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a - dockerImageTag: 0.2.4 + dockerImageTag: 1.1.3 dockerRepository: airbyte/source-google-analytics-data-api githubIssueLabel: source-google-analytics-data-api icon: google-analytics.svg - license: MIT + license: Elv2 name: Google Analytics 4 (GA4) registries: cloud: @@ -22,4 +22,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt b/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py index 1b42a7de0691..d93d394a4354 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py @@ -12,7 +12,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py index d10fcfc2cef4..7c476baae98f 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py @@ -9,6 +9,8 @@ import requests +from .utils import API_LIMIT_PER_HOUR + class GoogleAnalyticsApiQuotaBase: # Airbyte Logger @@ -22,8 +24,9 @@ class GoogleAnalyticsApiQuotaBase: should_retry: Optional[bool] = True backoff_time: Optional[int] = None raise_on_http_errors: bool = True - # stop making new slices globaly + # stop making new slices globally stop_iter: bool = False + error_message = None # mapping with scenarios for each quota kind quota_mapping: Mapping[str, Any] = { "concurrentRequests": { @@ -39,6 +42,7 @@ class GoogleAnalyticsApiQuotaBase: "should_retry": True, "raise_on_http_errors": False, "stop_iter": False, + "error_message": API_LIMIT_PER_HOUR, }, "potentiallyThresholdedRequestsPerHour": { "error_pattern": "Exhausted potentially thresholded requests quota.", @@ -46,8 +50,9 @@ class GoogleAnalyticsApiQuotaBase: "should_retry": True, "raise_on_http_errors": False, "stop_iter": False, + "error_message": API_LIMIT_PER_HOUR, }, - # TODO: The next scenarious are commented out for now. + # TODO: The next scenarios are commented out for now. # When we face with one of these at least 1 time, # we should be able to uncomment the one matches the criteria # and fill-in the `error_pattern` to track that quota as well. @@ -103,6 +108,7 @@ def _set_retry_attrs_for_quota(self, quota_name: str) -> None: self.raise_on_http_errors = quota.get("raise_on_http_errors") self.stop_iter = quota.get("stop_iter") self.backoff_time = quota.get("backoff") + self.error_message = quota.get("error_message") def _set_default_retry_attrs(self) -> None: self.should_retry = True @@ -121,9 +127,11 @@ def _check_remaining_quota(self, current_quota: Mapping[str, Any]) -> None: remaining_percent: float = remaining / total_available # make an early stop if we faced with the quota that is going to run out if remaining_percent <= self.treshold: - self.logger.warn(f"The `{quota_name}` quota is running out of tokens. Available {remaining} out of {total_available}.") + self.logger.warning(f"The `{quota_name}` quota is running out of tokens. Available {remaining} out of {total_available}.") self._set_retry_attrs_for_quota(quota_name) return None + elif self.error_message: + self.logger.warning(self.error_message) def _check_for_errors(self, response: requests.Response) -> None: try: @@ -137,7 +145,7 @@ def _check_for_errors(self, response: requests.Response) -> None: self.logger.warn(f"The `{quota_name}` quota is exceeded!") return None except AttributeError as attr_e: - self.logger.warn( + self.logger.warning( f"`GoogleAnalyticsApiQuota._check_for_errors`: Received non JSON response from the API. Full error: {attr_e}. Bypassing." ) pass diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/custom_reports_schema.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/custom_reports_schema.json index 777a8403bafe..a591bd1c7fba 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/custom_reports_schema.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/custom_reports_schema.json @@ -124,8 +124,8 @@ } } }, - "offset": {"type": ["null", "string", "integer"]}, - "limit": {"type": ["string", "integer"]}, + "offset": { "type": ["null", "string", "integer"] }, + "limit": { "type": ["string", "integer"] }, "metricAggregations": { "type": ["null", "string"], "enum": ["COUNT", "TOTAL", "MAXIMUM", "MINIMUM"] diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py index 8c22d3a7f56b..5d7091cf431c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py @@ -11,6 +11,7 @@ from http import HTTPStatus from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple +import dpath import jsonschema import requests from airbyte_cdk.models import FailureType, SyncMode @@ -20,18 +21,24 @@ from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError from source_google_analytics_data_api import utils -from source_google_analytics_data_api.utils import DATE_FORMAT +from source_google_analytics_data_api.utils import DATE_FORMAT, WRONG_DIMENSIONS, WRONG_JSON_SYNTAX, WRONG_METRICS from .api_quota import GoogleAnalyticsApiQuota -from .utils import authenticator_class_map, get_dimensions_type, get_metrics_type, metrics_type_to_python +from .utils import ( + authenticator_class_map, + check_invalid_property_error, + check_no_property_error, + get_dimensions_type, + get_metrics_type, + get_source_defined_primary_key, + metrics_type_to_python, +) # set the quota handler globaly since limitations are the same for all streams # the initial values should be saved once and tracked for each stream, inclusivelly. GoogleAnalyticsQuotaHandler: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() -# set page_size to 100000 due to determination of maximum limit value in official documentation -# https://developers.google.com/analytics/devguides/reporting/data/v1/basics#pagination -PAGE_SIZE = 100000 +LOOKBACK_WINDOW = datetime.timedelta(days=2) class ConfigurationError(Exception): @@ -61,14 +68,26 @@ class GoogleAnalyticsDataApiAbstractStream(HttpStream, ABC): http_method = "POST" raise_on_http_errors = True - def __init__(self, *, config: Mapping[str, Any], **kwargs): + def __init__(self, *, config: Mapping[str, Any], page_size: int = 100_000, **kwargs): super().__init__(**kwargs) self._config = config + self._source_defined_primary_key = get_source_defined_primary_key(self.name) + # default value is 100 000 due to determination of maximum limit value in official documentation + # https://developers.google.com/analytics/devguides/reporting/data/v1/basics#pagination + self._page_size = page_size @property def config(self): return self._config + @property + def page_size(self): + return self._page_size + + @page_size.setter + def page_size(self, value: int): + self._page_size = value + # handle the quota errors with prepared values for: # `should_retry`, `backoff_time`, `raise_on_http_errors`, `stop_iter` based on quota scenario. @GoogleAnalyticsQuotaHandler.handle_quota() @@ -93,22 +112,21 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream): """ _record_date_format = "%Y%m%d" - primary_key = "uuid" offset = 0 metadata = MetadataDescriptor() @property def cursor_field(self) -> Optional[str]: - return "date" if "date" in self.config.get("dimensions", {}) else [] + return "date" if "date" in self.config.get("dimensions", []) else [] - @staticmethod - def add_primary_key() -> dict: - return {"uuid": str(uuid.uuid4())} - - @staticmethod - def add_property_id(property_id): - return {"property_id": property_id} + @property + def primary_key(self): + pk = ["property_id"] + self.config.get("dimensions", []) + if "cohort_spec" not in self.config and "date" not in pk: + pk.append("startDate") + pk.append("endDate") + return pk @staticmethod def add_dimensions(dimensions, row) -> dict: @@ -133,7 +151,6 @@ def get_json_schema(self) -> Mapping[str, Any]: "additionalProperties": True, "properties": { "property_id": {"type": ["string"]}, - "uuid": {"type": ["string"], "description": "Custom unique identifier for each record, to support primary key"}, }, } @@ -143,6 +160,14 @@ def get_json_schema(self) -> Mapping[str, Any]: for d in self.config["dimensions"] } ) + # skipping startDate and endDate fields for cohort stream, because it doesn't support startDate and endDate fields + if "cohort_spec" not in self.config and "date" not in self.config["dimensions"]: + schema["properties"].update( + { + "startDate": {"type": ["null", "string"], "format": "date"}, + "endDate": {"type": ["null", "string"], "format": "date"}, + } + ) schema["properties"].update( { @@ -163,9 +188,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, total_rows = r["rowCount"] if self.offset == 0: - self.offset = PAGE_SIZE + self.offset = self.page_size else: - self.offset += PAGE_SIZE + self.offset += self.page_size if total_rows <= self.offset: self.offset = 0 @@ -193,9 +218,24 @@ def parse_response( metrics_type_map = {h.get("name"): h.get("type") for h in r.get("metricHeaders", [{}])} for row in r.get("rows", []): - yield self.add_primary_key() | self.add_property_id(self.config["property_id"]) | self.add_dimensions( - dimensions, row - ) | self.add_metrics(metrics, metrics_type_map, row) + record = { + "property_id": self.config["property_id"], + **self.add_dimensions(dimensions, row), + **self.add_metrics(metrics, metrics_type_map, row), + } + + # https://github.com/airbytehq/airbyte/pull/26283 + # We pass the uuid field for synchronizations which still have the old + # configured_catalog with the old primary key. We need it to avoid of removal of rows + # in the deduplication process. As soon as the customer press "refresh source schema" + # this part is no longer needed. + if self._source_defined_primary_key == [["uuid"]]: + record["uuid"] = str(uuid.uuid4()) + + if "cohort_spec" not in self.config and "date" not in record: + record["startDate"] = stream_slice["startDate"] + record["endDate"] = stream_slice["endDate"] + yield record def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): updated_state = utils.string_to_date(latest_record[self.cursor_field], self._record_date_format) @@ -219,7 +259,7 @@ def request_body_json( "dateRanges": [stream_slice], "returnPropertyQuota": True, "offset": str(0), - "limit": str(PAGE_SIZE), + "limit": str(self.page_size), } if next_page_token and next_page_token.get("offset") is not None: payload.update({"offset": str(next_page_token["offset"])}) @@ -234,6 +274,7 @@ def stream_slices( start_date = stream_state and stream_state.get(self.cursor_field) if start_date: start_date = utils.string_to_date(start_date, self._record_date_format, old_format=DATE_FORMAT) + start_date -= LOOKBACK_WINDOW start_date = max(start_date, self.config["date_ranges_start_date"]) else: start_date = self.config["date_ranges_start_date"] @@ -322,8 +363,10 @@ def _validate_and_transform(self, config: Mapping[str, Any], report_names: Set[s if isinstance(config["custom_reports"], str): try: config["custom_reports"] = json.loads(config["custom_reports"]) + if not isinstance(config["custom_reports"], list): + raise ValueError except ValueError: - raise ConfigurationError("custom_reports is not valid JSON") + raise ConfigurationError(WRONG_JSON_SYNTAX) else: config["custom_reports"] = [] @@ -331,6 +374,12 @@ def _validate_and_transform(self, config: Mapping[str, Any], report_names: Set[s try: jsonschema.validate(instance=config["custom_reports"], schema=schema) except jsonschema.ValidationError as e: + if message := check_no_property_error(e): + raise ConfigurationError(message) + if message := check_invalid_property_error(e): + report_name = dpath.util.get(config["custom_reports"], str(e.absolute_path[0])).get("name") + raise ConfigurationError(message.format(fields=e.message, report_name=report_name)) + key_path = "custom_reports" if e.path: key_path += "." + ".".join(map(str, e.path)) @@ -371,7 +420,9 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> return False, str(e) config["authenticator"] = self.get_authenticator(config) + metadata = None try: + # explicitly setting small page size for the check operation not to cause OOM issues stream = GoogleAnalyticsDataApiMetadataStream(config=config, authenticator=config["authenticator"]) metadata = next(stream.read_records(sync_mode=SyncMode.full_refresh), None) except HTTPError as e: @@ -398,12 +449,12 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> invalid_dimensions = set(report["dimensions"]) - dimensions if invalid_dimensions: invalid_dimensions = ", ".join(invalid_dimensions) - return False, f"custom_reports: invalid dimension(s): {invalid_dimensions} for the custom report: {report['name']}" + return False, WRONG_DIMENSIONS.format(fields=invalid_dimensions, report_name=report["name"]) invalid_metrics = set(report["metrics"]) - metrics if invalid_metrics: invalid_metrics = ", ".join(invalid_metrics) - return False, f"custom_reports: invalid metric(s): {invalid_metrics} for the custom report: {report['name']}" - report_stream = self.instantiate_report_class(report, config) + return False, WRONG_METRICS.format(fields=invalid_metrics, report_name=report["name"]) + report_stream = self.instantiate_report_class(report, config, page_size=100) # check if custom_report dimensions + metrics can be combined and report generated stream_slice = next(report_stream.stream_slices(sync_mode=SyncMode.full_refresh)) next(report_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), None) @@ -416,10 +467,14 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [self.instantiate_report_class(report, config) for report in reports + config["custom_reports"]] @staticmethod - def instantiate_report_class(report: dict, config: Mapping[str, Any]) -> GoogleAnalyticsDataApiBaseStream: + def instantiate_report_class(report: dict, config: Mapping[str, Any], **extra_kwargs) -> GoogleAnalyticsDataApiBaseStream: cohort_spec = report.get("cohortSpec") pivots = report.get("pivots") - stream_config = {"metrics": report["metrics"], "dimensions": report["dimensions"], **config} + stream_config = { + "metrics": report["metrics"], + "dimensions": report["dimensions"], + **config, + } report_class_tuple = (GoogleAnalyticsDataApiBaseStream,) if pivots: stream_config["pivots"] = pivots @@ -427,4 +482,4 @@ def instantiate_report_class(report: dict, config: Mapping[str, Any]) -> GoogleA if cohort_spec: stream_config["cohort_spec"] = cohort_spec report_class_tuple = (CohortReportMixin, *report_class_tuple) - return type(report["name"], report_class_tuple, {})(config=stream_config, authenticator=config["authenticator"]) + return type(report["name"], report_class_tuple, {})(config=stream_config, authenticator=config["authenticator"], **extra_kwargs) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json index fa001dc23db1..051338214db1 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json @@ -65,7 +65,7 @@ "credentials_json": { "title": "Service Account JSON Key", "type": "string", - "description": "The JSON key of the service account to use for authorization", + "description": "The JSON key linked to the service account used for authorization. For steps on obtaining this key, refer to the setup guide.", "examples": [ "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" ], @@ -79,9 +79,10 @@ "property_id": { "type": "string", "title": "Property ID", - "description": "A Google Analytics GA4 property identifier whose events are tracked. Specified in the URL path and not the body such as \"123...\". See the docs for more details.", + "description": "The Property ID is a unique number assigned to each property in Google Analytics, found in your GA4 property URL. This ID allows the connector to track the specific events associated with your property. Refer to the Google Analytics documentation to locate your property ID.", "pattern": "^[0-9]*$", - "pattern_descriptor": "such as \"123...\"", + "pattern_descriptor": "123...", + "examples": ["1738294", "5729978930"], "order": 1 }, "date_ranges_start_date": { @@ -89,18 +90,21 @@ "title": "Start Date", "description": "The start date from which to replicate report data in the format YYYY-MM-DD. Data generated before this date will not be included in the report. Not applied to custom Cohort reports.", "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2021-01-01"], "order": 2 }, "custom_reports": { "order": 3, "type": "string", "title": "Custom Reports", - "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the docs for more information about the exact format you can use to fill out this field." + "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the documentation for more information about the exact format you can use to fill out this field." }, "window_in_days": { "type": "integer", - "title": "Data request time increment in days", - "description": "The time increment used by the connector when requesting data from the Google Analytics API. More information is available in the the docs. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. Not applied to custom Cohort reports.", + "title": "Data Request Interval (Days)", + "description": "The interval in days for each data request made to the Google Analytics API. A larger value speeds up data sync, but increases the chance of data sampling, which may result in inaccuracies. We recommend a value of 1 to minimize sampling, unless speed is an absolute priority over accuracy. Acceptable values range from 1 to 364. Does not apply to custom Cohort reports. More information is available in the documentation.", "examples": [30, 60, 90, 120, 200, 364], "minimum": 1, "maximum": 364, @@ -111,10 +115,7 @@ }, "advanced_auth": { "auth_flow_type": "oauth2.0", - "predicate_key": [ - "credentials", - "auth_type" - ], + "predicate_key": ["credentials", "auth_type"], "predicate_value": "Client", "oauth_config_specification": { "complete_oauth_output_specification": { @@ -122,17 +123,11 @@ "properties": { "access_token": { "type": "string", - "path_in_connector_config": [ - "credentials", - "access_token" - ] + "path_in_connector_config": ["credentials", "access_token"] }, "refresh_token": { "type": "string", - "path_in_connector_config": [ - "credentials", - "refresh_token" - ] + "path_in_connector_config": ["credentials", "refresh_token"] } } }, @@ -152,17 +147,11 @@ "properties": { "client_id": { "type": "string", - "path_in_connector_config": [ - "credentials", - "client_id" - ] + "path_in_connector_config": ["credentials", "client_id"] }, "client_secret": { "type": "string", - "path_in_connector_config": [ - "credentials", - "client_secret" - ] + "path_in_connector_config": ["credentials", "client_secret"] } } } diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py index bce85dd46983..c4336453ca06 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py @@ -3,10 +3,14 @@ # +import argparse import calendar import datetime +import json +import sys from typing import Dict +import jsonschema from airbyte_cdk.sources.streams.http import auth from source_google_analytics_data_api.authenticator import GoogleServiceKeyAuthenticator @@ -58,6 +62,15 @@ ), } +WRONG_JSON_SYNTAX = "The custom report entered is not in a JSON array format. Check the entered format follows the syntax in our docs: https://docs.airbyte.com/integrations/sources/google-analytics-data-api/" +NO_NAME = "The custom report entered does not contain a name, which is required. Check the entered format follows the syntax in our docs: https://docs.airbyte.com/integrations/sources/google-analytics-data-api/" +NO_DIMENSIONS = "The custom report entered does not contain dimensions, which is required. Check the entered format follows the syntax in our docs (https://docs.airbyte.com/integrations/sources/google-analytics-data-api/) and validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)." +NO_METRICS = "The custom report entered does not contain metrics, which is required. Check the entered format follows the syntax in our docs (https://docs.airbyte.com/integrations/sources/google-analytics-data-api/) and validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)." +WRONG_DIMENSIONS = "The custom report {report_name} entered contains invalid dimensions: {fields}. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)." +WRONG_METRICS = "The custom report {report_name} entered contains invalid metrics: {fields}. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)." +WRONG_PIVOTS = "The custom report {report_name} entered contains invalid pivots: {fields}. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot)." +API_LIMIT_PER_HOUR = "Your API key has reached its limit for the hour. Wait until the quota refreshes in an hour to retry." + def datetime_to_secs(dt: datetime.datetime) -> int: return calendar.timegm(dt.utctimetuple()) @@ -88,3 +101,37 @@ def metrics_type_to_python(t: str) -> type: def get_dimensions_type(d: str) -> str: return "string" + + +def check_no_property_error(exc: jsonschema.ValidationError) -> str: + mapper = { + "'name' is a required property": NO_NAME, + "'dimensions' is a required property": NO_DIMENSIONS, + "'metrics' is a required property": NO_METRICS, + } + return mapper.get(exc.message) + + +def check_invalid_property_error(exc: jsonschema.ValidationError) -> str: + mapper = {"dimensions": WRONG_DIMENSIONS, "metrics": WRONG_METRICS, "pivots": WRONG_PIVOTS} + for property in mapper: + if property in exc.schema_path: + return mapper[property] + + +def get_source_defined_primary_key(stream): + """ + https://github.com/airbytehq/airbyte/pull/26283 + It's not a very elegant way to get source_defined_primary_key inside the stream. + It's used only for a smooth transition to the new primary key. + As soon as the transition will complete we can remove this function. + """ + if len(sys.argv) > 1 and "read" == sys.argv[1]: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + read_subparser = subparsers.add_parser("read") + read_subparser.add_argument("--catalog", type=str, required=True) + args, unknown = parser.parse_known_args() + catalog = json.loads(open(args.catalog).read()) + res = {s["stream"]["name"]: s["stream"].get("source_defined_primary_key") for s in catalog["streams"]} + return res.get(stream) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py index 21dfdcfe2810..e2ca394d1147 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py @@ -11,6 +11,7 @@ from airbyte_cdk.models import AirbyteConnectionStatus, FailureType, Status from airbyte_cdk.utils import AirbyteTracedException from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi +from source_google_analytics_data_api.utils import NO_DIMENSIONS, NO_METRICS, NO_NAME, WRONG_JSON_SYNTAX json_credentials = """ { @@ -65,26 +66,25 @@ def inner(**kwargs): @pytest.mark.parametrize( - "config_values, status, message", + "config_values, is_successful, message", [ ({}, Status.SUCCEEDED, None), ({"custom_reports": ...}, Status.SUCCEEDED, None), ({"custom_reports": "[]"}, Status.SUCCEEDED, None), - ({"custom_reports": "invalid"}, Status.FAILED, "'custom_reports is not valid JSON'"), - ({"custom_reports": "{}"}, Status.FAILED, '"custom_reports: {} is not of type \'array\'"'), - ({"custom_reports": "[{}]"}, Status.FAILED, '"custom_reports.0: \'name\' is a required property"'), - ({"custom_reports": "[{\"name\": \"name\"}]"}, Status.FAILED, '"custom_reports.0: \'dimensions\' is a required property"'), + ({"custom_reports": "invalid"}, Status.FAILED, f"'{WRONG_JSON_SYNTAX}'"), + ({"custom_reports": "{}"}, Status.FAILED, f"'{WRONG_JSON_SYNTAX}'"), + ({"custom_reports": "[{}]"}, Status.FAILED, f"'{NO_NAME}'"), + ({"custom_reports": "[{\"name\": \"name\"}]"}, Status.FAILED, f"'{NO_DIMENSIONS}'"), + ({"custom_reports": "[{\"name\": \"daily_active_users\", \"dimensions\": [\"date\"]}]"}, Status.FAILED, f"'{NO_METRICS}'"), + ({"custom_reports": "[{\"name\": \"daily_active_users\", \"metrics\": [\"totalUsers\"], \"dimensions\": [{\"name\": \"city\"}]}]"}, Status.FAILED, '"The custom report daily_active_users entered contains invalid dimensions: {\'name\': \'city\'} is not of type \'string\'. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)."'), ({"date_ranges_start_date": "2022-20-20"}, Status.FAILED, '"time data \'2022-20-20\' does not match format \'%Y-%m-%d\'"'), ({"credentials": {"auth_type": "Service", "credentials_json": "invalid"}}, Status.FAILED, "'credentials.credentials_json is not valid JSON'"), - ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [], \"metrics\": []}]"}, - Status.FAILED, "'custom_reports.0.dimensions: [] is too short'"), - ({"custom_reports": "[{\"name\": \"daily_active_users\", \"dimensions\": [\"date\"], \"metrics\": [\"totalUsers\"]}]"}, - Status.FAILED, "'custom_reports: daily_active_users already exist as a default report(s).'"), + ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [], \"metrics\": []}]"}, Status.FAILED, "'The custom report name entered contains invalid dimensions: [] is too short. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'"), + ({"custom_reports": "[{\"name\": \"daily_active_users\", \"dimensions\": [\"date\"], \"metrics\": [\"totalUsers\"]}]"}, Status.FAILED, "'custom_reports: daily_active_users already exist as a default report(s).'"), ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [\"unknown\"], \"metrics\": [\"totalUsers\"]}]"}, - Status.FAILED, "'custom_reports: invalid dimension(s): unknown for the custom report: name'"), - ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [\"date\"], \"metrics\": [\"unknown\"]}]"}, - Status.FAILED, "'custom_reports: invalid metric(s): unknown for the custom report: name'"), + Status.FAILED, "'The custom report name entered contains invalid dimensions: unknown. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'"), + ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [\"date\"], \"metrics\": [\"unknown\"]}]"}, Status.FAILED, "'The custom report name entered contains invalid metrics: unknown. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'"), ({"custom_reports": "[{\"name\": \"cohort_report\", \"dimensions\": [\"cohort\", \"cohortNthDay\"], \"metrics\": " "[\"cohortActiveUsers\"], \"cohortSpec\": {\"cohorts\": [{\"dimension\": \"firstSessionDate\", \"dateRange\": " "{\"startDate\": \"2023-01-01\", \"endDate\": \"2023-01-01\"}}], \"cohortsRange\": {\"endOffset\": 100}}}]"}, @@ -92,10 +92,10 @@ def inner(**kwargs): ({"custom_reports": "[{\"name\": \"pivot_report\", \"dateRanges\": [{ \"startDate\": \"2020-09-01\", \"endDate\": " "\"2020-09-15\" }], \"dimensions\": [\"browser\", \"country\", \"language\"], \"metrics\": [\"sessions\"], " "\"pivots\": {}}]"}, - Status.FAILED, '"custom_reports.0.pivots: {} is not of type \'null\', \'array\'"'), + Status.FAILED, '"The custom report pivot_report entered contains invalid pivots: {} is not of type \'null\', \'array\'. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot)."'), ], ) -def test_check(requests_mock, config_gen, config_values, status, message): +def test_check(requests_mock, config_gen, config_values, is_successful, message): requests_mock.register_uri("POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"}) @@ -114,12 +114,11 @@ def test_check(requests_mock, config_gen, config_values, status, message): source = SourceGoogleAnalyticsDataApi() logger = MagicMock() - - assert source.check(logger, config_gen(**config_values)) == AirbyteConnectionStatus(status=status, message=message) - - with pytest.raises(AirbyteTracedException) as e: - source.check(logger, config_gen(property_id="UA-11111111")) - assert e.value.failure_type == FailureType.config_error + assert source.check(logger, config_gen(**config_values)) == AirbyteConnectionStatus(status=is_successful, message=message) + if not is_successful: + with pytest.raises(AirbyteTracedException) as e: + source.check(logger, config_gen(property_id="UA-11111111")) + assert e.value.failure_type == FailureType.config_error def test_streams(mocker, patch_base_class): diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 8a774994334a..4654981e70bb 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -10,11 +10,10 @@ import pytest from freezegun import freeze_time -from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream, PAGE_SIZE +from source_google_analytics_data_api.source import GoogleAnalyticsDataApiBaseStream from .utils import read_incremental - json_credentials = """ { "type": "service_account", @@ -90,13 +89,23 @@ def test_request_body_json(patch_base_class): "dateRanges": [request_body_params["stream_slice"]], "returnPropertyQuota": True, "offset": str(0), - "limit": str(PAGE_SIZE) + "limit": "100000", } - request_body_json = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]).request_body_json(**request_body_params) + request_body_json = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]).request_body_json( + **request_body_params + ) assert request_body_json == expected_body_json +def test_changed_page_size(patch_base_class): + request_body_params = {"stream_state": MagicMock(), "stream_slice": MagicMock(), "next_page_token": None} + stream = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]) + stream.page_size = 100 + request_body_json = stream.request_body_json(**request_body_params) + assert request_body_json["limit"] == "100" + + def test_next_page_token_equal_chunk(patch_base_class): stream = GoogleAnalyticsDataApiBaseStream(authenticator=MagicMock(), config=patch_base_class["config"]) response = MagicMock() @@ -226,8 +235,6 @@ def test_parse_response(patch_base_class): response.json.return_value = response_data inputs = {"response": response, "stream_state": {}} actual_records: Mapping[str, Any] = list(stream.parse_response(**inputs)) - for record in actual_records: - del record["uuid"] assert actual_records == expected_data @@ -315,38 +322,51 @@ def test_read_incremental(requests_mock): "dimensionHeaders": [{"name": "date"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], "rows": [{"dimensionValues": [{"value": "20221229"}], "metricValues": [{"value": "100"}]}], - "rowCount": 1 + "rowCount": 1, }, { "dimensionHeaders": [{"name": "date"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], "rows": [{"dimensionValues": [{"value": "20221230"}], "metricValues": [{"value": "110"}]}], - "rowCount": 1 + "rowCount": 1, }, { "dimensionHeaders": [{"name": "date"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], "rows": [{"dimensionValues": [{"value": "20221231"}], "metricValues": [{"value": "120"}]}], - "rowCount": 1 + "rowCount": 1, }, { "dimensionHeaders": [{"name": "date"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], "rows": [{"dimensionValues": [{"value": "20230101"}], "metricValues": [{"value": "130"}]}], + "rowCount": 1, + }, + # 2-nd incremental read + { + "dimensionHeaders": [{"name": "date"}], + "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], + "rows": [{"dimensionValues": [{"value": "20221230"}], "metricValues": [{"value": "112"}]}], "rowCount": 1 }, { "dimensionHeaders": [{"name": "date"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20230101"}], "metricValues": [{"value": "140"}]}], + "rows": [{"dimensionValues": [{"value": "20221231"}], "metricValues": [{"value": "125"}]}], "rowCount": 1 }, + { + "dimensionHeaders": [{"name": "date"}], + "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], + "rows": [{"dimensionValues": [{"value": "20230101"}], "metricValues": [{"value": "140"}]}], + "rowCount": 1, + }, { "dimensionHeaders": [{"name": "date"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], "rows": [{"dimensionValues": [{"value": "20230102"}], "metricValues": [{"value": "150"}]}], - "rowCount": 1 - } + "rowCount": 1, + }, ] requests_mock.register_uri( @@ -358,9 +378,6 @@ def test_read_incremental(requests_mock): with freeze_time("2023-01-01 12:00:00"): records = list(read_incremental(stream, stream_state)) - for record in records: - del record["uuid"] - assert records == [ {"date": "20221229", "totalUsers": 100, "property_id": 123}, {"date": "20221230", "totalUsers": 110, "property_id": 123}, @@ -373,10 +390,9 @@ def test_read_incremental(requests_mock): with freeze_time("2023-01-02 12:00:00"): records = list(read_incremental(stream, stream_state)) - for record in records: - del record["uuid"] - assert records == [ + {"date": "20221230", "totalUsers": 112, "property_id": 123}, + {"date": "20221231", "totalUsers": 125, "property_id": 123}, {"date": "20230101", "totalUsers": 140, "property_id": 123}, {"date": "20230102", "totalUsers": 150, "property_id": 123}, ] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index b514787d3b96..076145b76938 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.34 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml index 74e68d5e1d72..f14f9a0d4cf7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml @@ -25,6 +25,7 @@ acceptance_tests: tests: - config_path: secrets/service_config.json configured_catalog_path: integration_tests/configured_catalog.json + timeout_seconds: 2400 future_state: future_state_path: integration_tests/abnormal_state.json threshold_days: 2 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml index 33db4eb56b28..4b17e0c7619f 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml @@ -8,11 +8,11 @@ data: connectorSubtype: api connectorType: source definitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 - dockerImageTag: 0.1.34 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-google-analytics-v4 githubIssueLabel: source-google-analytics-v4 icon: google-analytics.svg - license: MIT + license: Elv2 name: Google Analytics (Universal Analytics) registries: cloud: @@ -23,4 +23,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4 tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt b/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt +++ b/airbyte-integrations/connectors/source-google-analytics-v4/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py index d7b95a251f49..c37ee40da749 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py @@ -12,7 +12,6 @@ "requests-mock", "pytest-mock", "freezegun", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py index ec9300f0f3ea..9e6d232996e6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py @@ -7,6 +7,9 @@ from typing import Dict, List, Optional, Union from pydantic import BaseModel, Field, ValidationError, validator +from pydantic.class_validators import root_validator + +ERROR_MSG_MISSING_SEGMENT_DIMENSION = "errors: `ga:segment` is required" class Model(BaseModel): @@ -17,6 +20,7 @@ class Config: dimensions: list[str] metrics: list[str] filter: Optional[str] + segments: Optional[list[str]] @validator("dimensions", "metrics") def check_field_reference_forrmat(cls, value): @@ -28,6 +32,15 @@ def check_field_reference_forrmat(cls, value): if "ga:" not in v: raise ValueError(v) + @classmethod + @root_validator(pre=True) + def check_segment_included_in_dimension(cls, values): + dimensions = values.get("dimensions") + segments = values.get("segments") + if segments and "ga:segment" not in dimensions: + raise ValueError(ERROR_MSG_MISSING_SEGMENT_DIMENSION) + return values + class Explainer: """ diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index b4240980df8d..33fefa97a7ca 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -81,7 +81,9 @@ "type": "string", "title": "Replication Start Date", "description": "The date in the format YYYY-MM-DD. Any data before this date will not be replicated.", - "examples": ["2020-06-01"] + "examples": ["2020-06-01"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$|^$|[\\s\\S]+$", + "format": "date" }, "view_id": { "order": 2, @@ -105,12 +107,48 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"], ["refresh_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py index fa64aac258b9..f724857256fe 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py @@ -10,12 +10,19 @@ @pytest.mark.parametrize( "custom_reports, expected", ( - ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"), - ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"), - ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"), - ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"), + ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"), + ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"), + ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"), + ([{"name": "missing_segment_dimension", "dimensions": ["ga:test"], "segments": ["another_segment"],"metrics": ["ga:test"]}], "errors: `ga:segment` is required"), + ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"), ), - ids=["incorrrect field reference", "type_error", "not_permitted", "missing"], + ids=[ + "incorrrect field reference", + "type_error", + "not_permitted", + "missing", + "missing_segment_dimension" + ] ) def test_custom_reports_validator(custom_reports, expected): try: diff --git a/airbyte-integrations/connectors/source-google-directory/Dockerfile b/airbyte-integrations/connectors/source-google-directory/Dockerfile index d286e5830056..3f7aa557e8fc 100644 --- a/airbyte-integrations/connectors/source-google-directory/Dockerfile +++ b/airbyte-integrations/connectors/source-google-directory/Dockerfile @@ -34,5 +34,5 @@ COPY source_google_directory ./source_google_directory ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-google-directory diff --git a/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml index 291c559f0534..88358cb96123 100644 --- a/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-directory/acceptance-test-config.yml @@ -7,19 +7,13 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" - - config_path: "integration_tests/invalid_config_oauth.json" - status: "failed" discovery: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-directory/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-directory/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-google-directory/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-google-directory/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-google-directory/metadata.yaml b/airbyte-integrations/connectors/source-google-directory/metadata.yaml index fdd6bb7f8e06..9db4b72e67e8 100644 --- a/airbyte-integrations/connectors/source-google-directory/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-directory/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: d19ae824-e289-4b14-995a-0632eb46d246 - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-google-directory githubIssueLabel: source-google-directory icon: googledirectory.svg @@ -10,7 +10,7 @@ data: name: Google Directory registries: cloud: - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.1 enabled: true oss: enabled: true @@ -18,4 +18,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-directory tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-directory/requirements.txt b/airbyte-integrations/connectors/source-google-directory/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-google-directory/requirements.txt +++ b/airbyte-integrations/connectors/source-google-directory/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-directory/setup.py b/airbyte-integrations/connectors/source-google-directory/setup.py index 6c6924ce5e33..a4dbf5967f65 100644 --- a/airbyte-integrations/connectors/source-google-directory/setup.py +++ b/airbyte-integrations/connectors/source-google-directory/setup.py @@ -14,9 +14,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/__init__.py b/airbyte-integrations/connectors/source-google-directory/source_google_directory/__init__.py index ee5bc8f8b2f4..17d7060dde9a 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/__init__.py +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from .source import SourceGoogleDirectory __all__ = ["SourceGoogleDirectory"] diff --git a/airbyte-integrations/connectors/source-google-directory/source_google_directory/schemas/groups.json b/airbyte-integrations/connectors/source-google-directory/source_google_directory/schemas/groups.json index cb3be010dc41..a0aca45f8fa3 100644 --- a/airbyte-integrations/connectors/source-google-directory/source_google_directory/schemas/groups.json +++ b/airbyte-integrations/connectors/source-google-directory/source_google_directory/schemas/groups.json @@ -26,13 +26,13 @@ "adminCreated": { "type": ["null", "boolean"] }, - "aliases": { + "aliases": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, - "nonEditableAliases": { + "nonEditableAliases": { "type": ["null", "array"], "items": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml b/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml index 3fe61d821464..68f96b0db8e7 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/requirements.txt b/airbyte-integrations/connectors/source-google-pagespeed-insights/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/requirements.txt +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py b/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py index a56af79f8744..07d5d4a738ff 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile index 3fd5fa50ce49..9e9b1b2ca66b 100755 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ b/airbyte-integrations/connectors/source-google-search-console/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.3.1 LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml index ec849b29728e..de661af6035d 100755 --- a/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml @@ -25,7 +25,20 @@ acceptance_tests: extra_fields: yes exact_order: yes extra_records: no - timeout_seconds: 1800 + timeout_seconds: 3600 + empty_streams: + - name: search_analytics_page_report + bypass_reason: "Fast changing data" + - name: search_analytics_keyword_site_report_by_page + bypass_reason: "Fast changing data" + - name: search_analytics_keyword_site_report_by_site + bypass_reason: "Fast changing data" + - name: search_analytics_keyword_page_report + bypass_reason: "Fast changing data" + - name: search_analytics_site_report_by_page + bypass_reason: "Fast changing data" + - name: search_analytics_site_report_by_site + bypass_reason: "Fast changing data" ignored_fields: sitemaps: - name: lastDownloaded @@ -133,11 +146,11 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/catalog.json" - timeout_seconds: 1800 + timeout_seconds: 3600 incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_incremental.json" - timeout_seconds: 1800 + timeout_seconds: 3600 future_state: future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py index 1b60a7644218..1174b079d6b3 100755 --- a/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/credentials/setup.py @@ -11,7 +11,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json index 00a2163baf26..883a3f90e368 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/abnormal_state.json @@ -4,10 +4,12 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" }, + "discover": { "date": "2050-08-28" }, + "googleNews": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "search_analytics_by_date" } @@ -18,10 +20,12 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" }, + "discover": { "date": "2050-08-28" }, + "googleNews": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "search_analytics_by_country" } @@ -32,10 +36,11 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" }, + "googleNews": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "search_analytics_by_device" } @@ -46,10 +51,12 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" }, + "discover": { "date": "2050-08-28" }, + "googleNews": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "search_analytics_by_page" } @@ -60,10 +67,10 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "search_analytics_by_query" } @@ -74,10 +81,10 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "search_analytics_all_fields" } @@ -88,13 +95,103 @@ "stream": { "stream_state": { "https://airbyte.io/": { - "web": { "date": "2023-08-28" }, - "news": { "date": "2023-08-28" }, - "image": { "date": "2023-08-28" }, - "video": { "date": "2023-08-28" } + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } } }, "stream_descriptor": { "name": "custom_dimensions" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" }, + "googleNews": { "date": "2050-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_site_report_by_page" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } + } + }, + "stream_descriptor": { + "name": "search_analytics_keyword_site_report_by_page" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } + } + }, + "stream_descriptor": { + "name": "search_analytics_keyword_site_report_by_site" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_keyword_page_report" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" }, + "googleNews": { "date": "2050-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_page_report" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "https://airbyte.io/": { + "web": { "date": "2050-08-28" }, + "news": { "date": "2050-08-28" }, + "image": { "date": "2050-08-28" }, + "video": { "date": "2050-08-28" } + } + }, + "stream_descriptor": { "name": "search_analytics_site_report_by_site" } + } } ] diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json index d407ec074692..ecc5ed329e58 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog.json @@ -90,6 +90,78 @@ "cursor_field": ["date"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "search_analytics_keyword_page_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_keyword_site_report_by_page", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_keyword_site_report_by_site", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_page_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_site_report_by_page", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_site_report_by_site", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, { "stream": { "name": "custom_dimensions", diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog_incremental.json index 55fefa3ee0d4..bab112df8b49 100755 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/configured_catalog_incremental.json @@ -71,6 +71,78 @@ "sync_mode": "incremental", "cursor_field": ["date"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_keyword_page_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_keyword_site_report_by_page", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_keyword_site_report_by_site", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_page_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_site_report_by_page", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_analytics_site_report_by_site", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"] + }, + "sync_mode": "incremental", + "cursor_field": ["date"], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-google-search-console/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-search-console/integration_tests/expected_records.jsonl index 418dadab5947..4b75a033c505 100644 --- a/airbyte-integrations/connectors/source-google-search-console/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-search-console/integration_tests/expected_records.jsonl @@ -16,3 +16,4 @@ {"stream": "search_analytics_all_fields", "data": {"clicks": 18, "impressions": 33, "ctr": 0.5454545454545454, "position": 1, "site_url": "https://airbyte.io/", "search_type": "web", "date": "2021-10-19", "country": "fra", "device": "DESKTOP", "page": "https://airbyte.io/", "query": "airbyte"}, "emitted_at": 1677799225649} {"stream": "custom_dimensions", "data": {"clicks": 169, "impressions": 4869, "ctr": 0.03470938591086466, "position": 29.429657013760526, "site_url": "https://airbyte.io/", "search_type": "web", "date": "2021-10-19", "country": "usa", "device": "DESKTOP"}, "emitted_at": 1677799232181} {"stream": "custom_dimensions", "data": {"clicks": 56, "impressions": 2729, "ctr": 0.020520337119824112, "position": 33.29351410773177, "site_url": "https://airbyte.io/", "search_type": "web", "date": "2021-10-19", "country": "ind", "device": "DESKTOP"}, "emitted_at": 1677799232181} +{"stream": "search_analytics_keyword_page_report", "data": {"clicks": 1, "impressions": 1, "ctr": 1, "position": 4, "site_url": "https://airbyte.io/", "search_type": "web", "date": "2022-02-14", "country": "nld", "device": "DESKTOP", "query": "airbyte s3 destination", "page": "https://airbyte.io/connections/Progress-to-S3"}, "emitted_at": 1688032519825} diff --git a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml index a0ba45503b7f..4a132d7d3cf6 100644 --- a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml @@ -5,11 +5,11 @@ data: connectorSubtype: api connectorType: source definitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 - dockerImageTag: 1.0.1 + dockerImageTag: 1.3.1 dockerRepository: airbyte/source-google-search-console githubIssueLabel: source-google-search-console icon: googlesearchconsole.svg - license: MIT + license: Elv2 name: Google Search Console registries: cloud: @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-search-console/requirements.txt b/airbyte-integrations/connectors/source-google-search-console/requirements.txt index 9ce85523c234..7b9114ed5867 100755 --- a/airbyte-integrations/connectors/source-google-search-console/requirements.txt +++ b/airbyte-integrations/connectors/source-google-search-console/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-search-console/setup.py b/airbyte-integrations/connectors/source-google-search-console/setup.py index ad1a26e7ec72..fd73d6450d37 100755 --- a/airbyte-integrations/connectors/source-google-search-console/setup.py +++ b/airbyte-integrations/connectors/source-google-search-console/setup.py @@ -12,9 +12,10 @@ ] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", + "pytest-lazy-fixture", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py index 7de3fee7ef67..77dc53148762 100644 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py @@ -3,5 +3,28 @@ # +from typing import Any, List, Set, Union + + class InvalidSiteURLValidationError(Exception): - pass + def __init__(self, invalid_site_url: Union[Set, List]) -> None: + message = f'The following URLs are not permitted: {", ".join(invalid_site_url)}' + super().__init__(message) + + +class UnauthorizedOauthError(Exception): + def __init__(self): + message = "Unable to connect with privided OAuth credentials. The `access token` or `refresh token` is expired. Please re-authrenticate using valid account credenials." + super().__init__(message) + + +class UnauthorizedServiceAccountError(Exception): + def __init__(self): + message = "Unable to connect with privided Service Account credentials. Make sure the `sevice account crdentials` povided is valid." + super().__init__(message) + + +class UnidentifiedError(Exception): + def __init__(self, error_body: Any): + message = f"Unable to connect to Google Search Console API - {error_body}" + super().__init__(message) diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_page_report.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_page_report.json new file mode 100755 index 000000000000..ff62135f12c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_page_report.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "site_url": { + "type": ["null", "string"] + }, + "search_type": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date" + }, + "country": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "query": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "ctr": { + "type": ["null", "number"], + "multipleOf": 1e-25 + }, + "position": { + "type": ["null", "number"], + "multipleOf": 1e-25 + } + } +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json new file mode 100755 index 000000000000..d5a5fda941d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_site_report_by_page.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "site_url": { + "type": ["null", "string"] + }, + "search_type": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date" + }, + "country": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "query": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "ctr": { + "type": ["null", "number"], + "multipleOf": 1e-25 + }, + "position": { + "type": ["null", "number"], + "multipleOf": 1e-25 + } + } +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json new file mode 100755 index 000000000000..d5a5fda941d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_keyword_site_report_by_site.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "site_url": { + "type": ["null", "string"] + }, + "search_type": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date" + }, + "country": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "query": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "ctr": { + "type": ["null", "number"], + "multipleOf": 1e-25 + }, + "position": { + "type": ["null", "number"], + "multipleOf": 1e-25 + } + } +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_page_report.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_page_report.json new file mode 100755 index 000000000000..19725cdafcdf --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_page_report.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "site_url": { + "type": ["null", "string"] + }, + "search_type": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date" + }, + "country": { + "type": ["null", "string"] + }, + "page": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "ctr": { + "type": ["null", "number"], + "multipleOf": 1e-25 + }, + "position": { + "type": ["null", "number"], + "multipleOf": 1e-25 + } + } +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_site_report_by_page.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_site_report_by_page.json new file mode 100755 index 000000000000..32a663aaf7b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_site_report_by_page.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "site_url": { + "type": ["null", "string"] + }, + "search_type": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date" + }, + "country": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "ctr": { + "type": ["null", "number"], + "multipleOf": 1e-25 + }, + "position": { + "type": ["null", "number"], + "multipleOf": 1e-25 + } + } +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_site_report_by_site.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_site_report_by_site.json new file mode 100755 index 000000000000..32a663aaf7b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/schemas/search_analytics_site_report_by_site.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "site_url": { + "type": ["null", "string"] + }, + "search_type": { + "type": ["null", "string"] + }, + "date": { + "type": ["null", "string"], + "format": "date" + }, + "country": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "clicks": { + "type": ["null", "integer"] + }, + "impressions": { + "type": ["null", "integer"] + }, + "ctr": { + "type": ["null", "number"], + "multipleOf": 1e-25 + }, + "position": { + "type": ["null", "number"], + "multipleOf": 1e-25 + } + } +} diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py index 7c4375776163..decbb448107f 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py @@ -6,6 +6,7 @@ from google.auth.transport.requests import Request from google.oauth2.service_account import Credentials from requests.auth import AuthBase +from source_google_search_console.exceptions import UnauthorizedServiceAccountError DEFAULT_SCOPES = ["https://www.googleapis.com/auth/webmasters.readonly"] @@ -16,8 +17,11 @@ def __init__(self, service_account_info: str, email: str, scopes=None): self.credentials: Credentials = Credentials.from_service_account_info(service_account_info, scopes=self.scopes).with_subject(email) def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: - if not self.credentials.valid: - # We pass a dummy request because the refresh iface requires it - self.credentials.refresh(Request()) - self.credentials.apply(request.headers) - return request + try: + if not self.credentials.valid: + # We pass a dummy request because the refresh iface requires it + self.credentials.refresh(Request()) + self.credentials.apply(request.headers) + return request + except Exception: + raise UnauthorizedServiceAccountError diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index b38f57feacf3..29e9d7a894d1 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -3,7 +3,7 @@ # import json -from typing import Any, List, Mapping, Optional, Tuple +from typing import Any, List, Mapping, Optional, Tuple, Union from urllib.parse import urlparse import jsonschema @@ -14,7 +14,12 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from source_google_search_console.exceptions import InvalidSiteURLValidationError +from source_google_search_console.exceptions import ( + InvalidSiteURLValidationError, + UnauthorizedOauthError, + UnauthorizedServiceAccountError, + UnidentifiedError, +) from source_google_search_console.service_account_authenticator import ServiceAccountAuthenticator from source_google_search_console.streams import ( SearchAnalyticsAllFields, @@ -24,6 +29,12 @@ SearchAnalyticsByDevice, SearchAnalyticsByPage, SearchAnalyticsByQuery, + SearchAnalyticsKeywordPageReport, + SearchAnalyticsKeywordSiteReportByPage, + SearchAnalyticsKeywordSiteReportBySite, + SearchAnalyticsPageReport, + SearchAnalyticsSiteReportByPage, + SearchAnalyticsSiteReportBySite, Sitemaps, Sites, ) @@ -76,7 +87,7 @@ def _validate_and_transform(self, config: Mapping[str, Any]): config["site_urls"] = [self.normalize_url(url) for url in config["site_urls"]] - config["data_state"] = config.get("date_state", "final") + config["data_state"] = config.get("data_state", "final") return config def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: @@ -94,31 +105,45 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> next(sites_gen) return True, None - except (InvalidSiteURLValidationError, jsonschema.ValidationError) as e: + except (InvalidSiteURLValidationError, UnauthorizedOauthError, UnauthorizedServiceAccountError, jsonschema.ValidationError) as e: return False, repr(e) - except Exception as error: + except (Exception, UnidentifiedError) as error: return ( False, - f"Unable to connect to Google Search Console API with the provided credentials - {repr(error)}", + f"Unable to check connectivity to Google Search Console API - {repr(error)}", ) - @staticmethod - def validate_site_urls(site_urls, auth): + def validate_site_urls(self, site_urls: List[str], auth: Union[ServiceAccountAuthenticator, Oauth2Authenticator]): if isinstance(auth, ServiceAccountAuthenticator): request = auth(requests.Request(method="GET", url="https://www.googleapis.com/webmasters/v3/sites")) with requests.Session() as s: response = s.send(s.prepare_request(request)) + # the exceptions for `service account` are handled in `service_account_authenticator.py` else: - response = requests.get("https://www.googleapis.com/webmasters/v3/sites", headers=auth.get_auth_header()) - response_data = response.json() - + # catch the error while refreshing the access token + auth_header = self.get_client_auth_header(auth) + if "error" in auth_header: + if auth_header.get("code", 0) in [400, 401]: + raise UnauthorizedOauthError + # validate site urls with provided authenticator + response = requests.get("https://www.googleapis.com/webmasters/v3/sites", headers=auth_header) + # validate the status of the response, if it was successfull if response.status_code != 200: - raise Exception(f"Unable to connect to Google Search Console API - {response_data}") + raise UnidentifiedError(response.json()) - remote_site_urls = {s["siteUrl"] for s in response_data["siteEntry"]} + remote_site_urls = {s["siteUrl"] for s in response.json()["siteEntry"]} invalid_site_url = set(site_urls) - remote_site_urls if invalid_site_url: - raise InvalidSiteURLValidationError(f'The following URLs are not permitted: {", ".join(invalid_site_url)}') + raise InvalidSiteURLValidationError(invalid_site_url) + + def get_client_auth_header(self, auth: Oauth2Authenticator) -> Mapping[str, Any]: + try: + return auth.get_auth_header() + except requests.exceptions.HTTPError as e: + return { + "code": e.response.status_code, + "error": e.response.json(), + } def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ @@ -136,6 +161,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SearchAnalyticsByQuery(**stream_config), SearchAnalyticsByPage(**stream_config), SearchAnalyticsAllFields(**stream_config), + SearchAnalyticsKeywordPageReport(**stream_config), + SearchAnalyticsPageReport(**stream_config), + SearchAnalyticsSiteReportBySite(**stream_config), + SearchAnalyticsSiteReportByPage(**stream_config), + SearchAnalyticsKeywordSiteReportByPage(**stream_config), + SearchAnalyticsKeywordSiteReportBySite(**stream_config), ] streams = streams + self.get_custom_reports(config=config, stream_config=stream_config) diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json index 1fc489c122d9..f46daf10ea02 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json @@ -12,25 +12,27 @@ "type": "string" }, "title": "Website URL Property", - "description": "The URLs of the website property attached to your GSC account. Read more here.", + "description": "The URLs of the website property attached to your GSC account. Learn more about properties here.", "examples": ["https://example1.com/", "sc-domain:example2.com"], "order": 0 }, "start_date": { "type": "string", "title": "Start Date", - "description": "UTC date in the format 2017-01-25. Any data before this date will not be replicated.", + "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated.", "examples": ["2021-01-01"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", "order": 1, "format": "date" }, "end_date": { "type": "string", "title": "End Date", - "description": "UTC date in the format 2017-01-25. Any data after this date will not be replicated. Must be greater or equal to the start date field.", + "description": "UTC date in the format YYYY-MM-DD. Any data created after this date will not be replicated. Must be greater or equal to the start date field. Leaving this field blank will replicate all data from the start date onward.", "examples": ["2021-12-12"], "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", "order": 2, "format": "date" }, @@ -113,14 +115,14 @@ "order": 4, "type": "string", "title": "Custom Reports", - "description": "A JSON array describing the custom reports you want to sync from Google Search Console. See the docs for more information about the exact format you can use to fill out this field." + "description": "A JSON array describing the custom reports you want to sync from Google Search Console. See our documentation for more information on formulating custom reports." }, "data_state": { "type": "string", - "title": "Data State", + "title": "Data Freshness", "enum": ["final", "all"], - "description": "If \"final\" or if this parameter is omitted, the returned data will include only finalized data. Setting this parameter to \"all\" should not be used with Incremental Sync mode as it may cause data loss. If \"all\", data will include fresh data.", - "examples": ["final"], + "description": "If set to 'final', the returned data will include only finalized, stable data. If set to 'all', fresh data will be included. When using Incremental sync mode, we do not recommend setting this parameter to 'all' as it may cause data loss. More information can be found in our full documentation.", + "examples": ["final", "all"], "default": "final", "order": 5 } diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index d01638318f29..f707e527f8ff 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -3,6 +3,7 @@ # from abc import ABC +from enum import Enum from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import quote_plus, unquote_plus @@ -16,10 +17,17 @@ ROW_LIMIT = 25000 +class QueryAggregationType(Enum): + auto = "auto" + by_page = "byPage" + by_property = "byProperty" + + class GoogleSearchConsole(HttpStream, ABC): url_base = BASE_URL primary_key = None data_field = "" + raise_on_http_errors = True def __init__( self, @@ -57,6 +65,17 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp for record in records: yield record + def should_retry(self, response: requests.Response) -> bool: + response_json = response.json() + if "error" in response_json: + error = response_json.get("error", {}) + # handle the `HTTP-403` - insufficient permissions + if error.get("code", 0) == 403: + self.logger.error(f"Stream {self.name}. {error.get('message')}. Skipping.") + setattr(self, "raise_on_http_errors", False) + return False + return super().should_retry(response) + class Sites(GoogleSearchConsole): """ @@ -94,6 +113,7 @@ class SearchAnalytics(GoogleSearchConsole, ABC): """ data_field = "rows" + aggregation_type = QueryAggregationType.auto start_row = 0 dimensions = [] search_types = ["web", "news", "image", "video"] @@ -121,7 +141,8 @@ def stream_slices( """ The `stream_slices` implements iterator functionality for `site_urls` and `searchType`. The user can pass many `site_url`, and we have to process all of them, we can also pass the` searchType` parameter in the `request body` to get data using some` - searchType` value from [` web`, `news `,` image`, `video`]. It's just a double nested loop with a yield statement. + searchType` value from [` web`, `news `,` image`, `video`, `discover`, `googleNews`]. + It's just a double nested loop with a yield statement. """ for site_url in self._site_urls: @@ -176,7 +197,8 @@ def request_body_json( 2. The `endDate` is retrieved from the `config.json`. 3. The `sizes` parameter is used to group the result by some dimension. The following dimensions are available: `date`, `country`, `page`, `device`, `query`. - 4. For the `searchType` check the paragraph stream_slices method. + 4. For the `type` check the paragraph stream_slices method. + Filter results to the following type ["web", "news", "image", "video", "discover", "googleNews"] 5. For the `startRow` and `rowLimit` check next_page_token method. """ @@ -184,8 +206,8 @@ def request_body_json( "startDate": stream_slice["start_date"], "endDate": stream_slice["end_date"], "dimensions": self.dimensions, - "searchType": stream_slice.get("search_type"), - "aggregationType": "auto", + "type": stream_slice.get("search_type"), + "aggregationType": self.aggregation_type.value, "startRow": self.start_row, "rowLimit": ROW_LIMIT, "dataState": stream_slice.get("data_state"), @@ -278,18 +300,22 @@ def get_updated_state( class SearchAnalyticsByDate(SearchAnalytics): + search_types = ["web", "news", "image", "video", "discover", "googleNews"] dimensions = ["date"] class SearchAnalyticsByCountry(SearchAnalytics): + search_types = ["web", "news", "image", "video", "discover", "googleNews"] dimensions = ["date", "country"] class SearchAnalyticsByDevice(SearchAnalytics): + search_types = ["web", "news", "image", "video", "googleNews"] dimensions = ["date", "device"] class SearchAnalyticsByPage(SearchAnalytics): + search_types = ["web", "news", "image", "video", "discover", "googleNews"] dimensions = ["date", "page"] @@ -301,6 +327,73 @@ class SearchAnalyticsAllFields(SearchAnalytics): dimensions = ["date", "country", "device", "page", "query"] +class SearchAppearance(SearchAnalytics): + """ + Dimension searchAppearance can't be used with other dimension. + search appearance data (AMP, blue link, rich result, and so on) must be queried using a two-step process. + https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data + """ + + dimensions = ["searchAppearance"] + + +class SearchByKeyword(SearchAnalytics): + """ + Adds searchAppearance value to dimensionFilterGroups in json body + https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data + """ + + def request_body_json( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Optional[Union[Dict[str, Any], str]]: + data = super().request_body_json(stream_state, stream_slice, next_page_token) + + stream = SearchAppearance(self.authenticator, self._site_urls, self._start_date, self._end_date) + keywords_records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_state=stream_state, stream_slice=stream_slice) + keywords = {record["searchAppearance"] for record in keywords_records} + + filters = [] + for keyword in keywords: + filters.append({"dimension": "searchAppearance", "operator": "equals", "expression": keyword}) + + data["dimensionFilterGroups"] = [{"filters": filters}] + + return data + + +class SearchAnalyticsKeywordPageReport(SearchByKeyword): + dimensions = ["date", "country", "device", "query", "page"] + + +class SearchAnalyticsKeywordSiteReportByPage(SearchByKeyword): + dimensions = ["date", "country", "device", "query"] + aggregation_type = QueryAggregationType.by_page + + +class SearchAnalyticsKeywordSiteReportBySite(SearchByKeyword): + dimensions = ["date", "country", "device", "query"] + aggregation_type = QueryAggregationType.by_property + + +class SearchAnalyticsSiteReportBySite(SearchAnalytics): + dimensions = ["date", "country", "device"] + aggregation_type = QueryAggregationType.by_property + + +class SearchAnalyticsSiteReportByPage(SearchAnalytics): + search_types = ["web", "news", "image", "video", "googleNews"] + dimensions = ["date", "country", "device"] + aggregation_type = QueryAggregationType.by_page + + +class SearchAnalyticsPageReport(SearchAnalytics): + search_types = ["web", "news", "image", "video", "googleNews"] + dimensions = ["date", "country", "device", "page"] + + class SearchAnalyticsByCustomDimensions(SearchAnalytics): dimension_to_property_schema_map = { "country": [{"country": {"type": ["null", "string"]}}], @@ -319,7 +412,7 @@ def get_json_schema(self) -> Mapping[str, Any]: return super(SearchAnalyticsByCustomDimensions, self).get_json_schema() except FileNotFoundError: schema: Mapping[str, Any] = { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "additionalProperties": True, "properties": { diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py index d4e27c0d95c7..3c201316fcaf 100644 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py @@ -23,6 +23,38 @@ def config_fixture(requests_mock): } +@fixture(name="service_account_config") +def config_service_account_fixture(requests_mock): + return { + "site_urls": ["https://example.com/"], + "start_date": "2022-01-01", + "end_date": "2022-02-01", + "authorization": { + "auth_type": "Service", + "service_account_info": "{\n \"type\": \"service_account\",\n \"project_id\": \"test\",\n \"private_key_id\": \"123\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BBQEFAASCBKcwggSjAgEAAoIBAQDCJPUvep0vXeWb\\nqiwnDxWdd8D75FWJBaYB3rjZvBBhZiY3sA7DmEOj+NJHl4PiPzP8tDZl9MyLBWEc\\neTFSmHSBYSqxax9AOLzWXfLzUezjediIRsGC/Eq9Ue0rkDdMcdcfzQ5J9RDDI1DF\\n1UBxVHFOf7DOSOU7meNPFjAO68aITErvnTh/XL1wWC28PYL351hs57WwLSQTuW0e\\ncUw9XUOE977+qJ4Cs3ZM5c10eid5DDWS4heFG/9hEkobXy34BNdeDodfe9xGSJxD\\nFoAhADj6jMn1z7YgsUG7zpsyW8yh2LtnYdT+fMqIl0FeB4dt0kB3uU1f6vqgo97p\\ndibK6DQ3AgMBAAECggEADWZPz+eZreyNDbnokzrDMw7SMIof1AKIsZKMPDPAE1Pu\\ndlbkWS0LGhs92p6DOUKqFWdHMkrh/tEvuy94L1G1CAr+gqe4mY4KjPPuC7I1wRuM\\n50ovWtlliGL9SIDxkbw+IB4SJIBrS3SgCg+AA6WgezQ5lHtLUXPh6ivHXfhGLlKR\\nI+Gow93UklbxcT57ezeDZVn0U3iUG1H7NkE0livyTTGEMm6GxUqxje7axA4ZVfRL\\nRVrNAHQTihPTThmN/p47Wbh6C8m7o1/cutYDk52CuCjuifxNINlak1ZimSEJ7mcY\\nSIglnTmndQImwiyeDbITtJ3gyYiJerjHnMAYH+VInQKBgQD5HH1tKBxZouozdweu\\n6lpTyko+TBa/3Eo2pgFxbJrKe3pBhkNWVLrCukZxWDFkKSbC+5xaSNGnh/lP/6FX\\nWHWBuBL8R5os9bfNQ9xnArZX7OhzN+aIh8aK5gmEPJE1JaepPyC0X8vaTBqFiQlK\\n6aRB89RqOUlB86B9vzJca7p7LQKBgQDHg1h9A6X9EkWewW5cSOuScw4FElK8N62v\\n5oVByBZZb/Ys9zP04m0yG7VdRSjk8xyCH5+GDS5m9jTxJdctON2AOPL7de8KOtga\\nJSHivUdDLkt7wSmvblc/JYnNs5+B783gTOpdBrXhV6Wo+QpVw1Pcx15b10WLAs8l\\nMzk7LG27cwKBgDJPorVNCIzB7nL+czrMcfnCPURfsaiGISbwWBJEUO7cCVD6gNcK\\nvb1eSaPSoAcOmJmAn49MbatcNuoFQtyVLQZJ2uvAuk6iQcdfF8BmN9WCL2A1xgWF\\nBoA+/WULpngJZtczvLMxNcac4C5gAtRyY44+ZIQflcAQKDW9S7qGt17xAoGBAJ37\\npLtBg1PU/yoJ81DCMT/DOYvMiZUe5bsO5+BCB2iE3sOWcB7umRb/l+qmVA6Pb7ie\\nP9yPXXoMZbm6hBv8FnFtJwL1zPYlyG9TjfSUevR4mS8CsvaGhjGvkOJA5QKoGDcP\\n0Nke8jDhDX2yzntA84w0lsRUv22nKM5FNIFl2fJ/AoGAOAVtlKRPPi2YrjUqqy6F\\nYr9RXwDZIaHQv9RKzkhPN346zXrYOuAGoL7V7F/MyUH3nX3pzHJDns71+S4Ms5qq\\n6ZjMCu/ic/RsCIoCH5IQsubLpI5bnSsHVt8wLMNR9LwQ/lbRJPWF4LmMnDNJCuC0\\nqJd/bEiNrFhu8IgD6NCT7dQ=\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"search-console-integration-tes@airbyte.iam.gserviceaccount.com\",\n \"client_id\": \"123\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/search-console-integration-test%40dairbyte.iam.gserviceaccount.com\"\n}", + "email": "test@example.com" + }, + "custom_reports": "[{\"name\": \"custom_dimensions\", \"dimensions\": [\"date\", \"country\", \"device\"]}]" + } + + +@fixture(name="forbidden_error_message_json") +def forbidden_error_message_json(requests_mock): + return { + "error": { + "code": 403, + "message": "User does not have sufficient permission for site 'https://test-site-test.com/'. See also: https://support.google.com/webmasters/answer/9999999.", + "errors": [ + { + "message": "User does not have sufficient permission for site 'https://test-site-test.com/'. See also: https://support.google.com/webmasters/answer/9999999.", + "domain": "global", + "reason": "forbidden" + } + ] + } + } + + @fixture def config_gen(config): def inner(**kwargs): diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py index 603fc1fec75c..11436196c644 100755 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py @@ -7,9 +7,17 @@ from urllib.parse import quote_plus import pytest +import requests from airbyte_cdk.models import AirbyteConnectionStatus, Status, SyncMode +from pytest_lazyfixture import lazy_fixture from source_google_search_console.source import SourceGoogleSearchConsole -from source_google_search_console.streams import ROW_LIMIT, GoogleSearchConsole, SearchAnalyticsByCustomDimensions, SearchAnalyticsByDate +from source_google_search_console.streams import ( + ROW_LIMIT, + GoogleSearchConsole, + SearchAnalyticsByCustomDimensions, + SearchAnalyticsByDate, + Sites, +) from utils import command_check logger = logging.getLogger("airbyte") @@ -112,6 +120,16 @@ def test_updated_state(): } +def test_forbidden_should_retry(requests_mock, forbidden_error_message_json): + stream = Sites(None, ["https://domain1.com"], "2023-01-01", "2023-01-01") + slice = list(stream.stream_slices(None))[0] + url = stream.url_base + stream.path(None, slice) + requests_mock.get(url, status_code=403, json=forbidden_error_message_json) + test_response = requests.get(url) + assert stream.should_retry(test_response) is False + assert stream.raise_on_http_errors is False + + @pytest.mark.parametrize( "stream_class, expected", [ @@ -136,7 +154,7 @@ def test_parse_response(stream_class, expected): assert record == expected -def test_check_connection(config_gen, mocker, requests_mock): +def test_check_connection(config_gen, config, mocker, requests_mock): requests_mock.get("https://www.googleapis.com/webmasters/v3/sites/https%3A%2F%2Fexample.com%2F", json={}) requests_mock.get("https://www.googleapis.com/webmasters/v3/sites", json={"siteEntry": [{"siteUrl": "https://example.com/"}]}) requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": "token", "expires_in": 10}) @@ -160,7 +178,7 @@ def test_check_connection(config_gen, mocker, requests_mock): assert command_check(source, config_gen(start_date="start_date")) assert command_check(source, config_gen(start_date="2022-99-99")) == AirbyteConnectionStatus( status=Status.FAILED, - message="\"Unable to connect to Google Search Console API with the provided credentials - ParserError('Unable to parse string [2022-99-99]')\"", + message="\"Unable to check connectivity to Google Search Console API - ParserError('Unable to parse string [2022-99-99]')\"", ) # test end_date @@ -170,25 +188,43 @@ def test_check_connection(config_gen, mocker, requests_mock): assert command_check(source, config_gen(end_date="end_date")) assert command_check(source, config_gen(end_date="2022-99-99")) == AirbyteConnectionStatus( status=Status.FAILED, - message="\"Unable to connect to Google Search Console API with the provided credentials - ParserError('Unable to parse string [2022-99-99]')\"", + message="\"Unable to check connectivity to Google Search Console API - ParserError('Unable to parse string [2022-99-99]')\"", ) # test custom_reports assert command_check(source, config_gen(custom_reports="")) == AirbyteConnectionStatus( status=Status.FAILED, - message="\"Unable to connect to Google Search Console API with the provided credentials - Exception('custom_reports is not valid JSON')\"", + message="\"Unable to check connectivity to Google Search Console API - Exception('custom_reports is not valid JSON')\"", ) assert command_check(source, config_gen(custom_reports="{}")) == AirbyteConnectionStatus( status=Status.FAILED, message="''" ) +@pytest.mark.parametrize( + "test_config, expected", + [ + ( + lazy_fixture("config"), + (False, "UnauthorizedOauthError('Unable to connect with privided OAuth credentials. The `access token` or `refresh token` is expired. Please re-authrenticate using valid account credenials.')")), + ( + lazy_fixture("service_account_config"), + (False, "UnauthorizedServiceAccountError('Unable to connect with privided Service Account credentials. Make sure the `sevice account crdentials` povided is valid.')")) + ], +) +def test_unauthorized_creds_exceptions(test_config, expected, requests_mock): + source = SourceGoogleSearchConsole() + requests_mock.post("https://oauth2.googleapis.com/token", status_code=401, json={}) + actual = source.check_connection(logger, test_config) + assert actual == expected + + def test_streams(config_gen): source = SourceGoogleSearchConsole() streams = source.streams(config_gen()) - assert len(streams) == 9 + assert len(streams) == 15 streams = source.streams(config_gen(custom_reports=...)) - assert len(streams) == 8 + assert len(streams) == 14 def test_get_start_date(): diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 722701c713b5..647779e246e1 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.11-alpine3.15 as base +FROM python:3.9.16-alpine3.18 as base # build and load all requirements FROM base as builder @@ -25,7 +25,9 @@ COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN echo "Etc/UTC" > /etc/timezone # bash is installed for more convenient debugging. -RUN apk --no-cache add bash +RUN apk --no-cache add bash && \ + # upgrading openssl due to https://nvd.nist.gov/vuln/detail/CVE-2023-2650 + apk upgrade # copy payload code only COPY main.py ./ @@ -34,5 +36,5 @@ COPY source_google_sheets ./source_google_sheets ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.39 +LABEL io.airbyte.version=0.3.7 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml index 583adf2855ce..6bdbb7516264 100644 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml @@ -28,4 +28,6 @@ acceptance_tests: spec: tests: - spec_path: source_google_sheets/spec.yaml + backward_compatibility_tests_config: + disable_for_version: "0.3.6" diff --git a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml index 6996a910ab7c..8e7c2e79be2d 100644 --- a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml @@ -5,15 +5,14 @@ data: connectorSubtype: file connectorType: source definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 - dockerImageTag: 0.2.39 + dockerImageTag: 0.3.7 dockerRepository: airbyte/source-google-sheets githubIssueLabel: source-google-sheets icon: google-sheets.svg - license: MIT + license: Elv2 name: Google Sheets registries: cloud: - dockerImageTag: 0.2.21 enabled: true oss: enabled: true @@ -21,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-sheets/requirements.txt b/airbyte-integrations/connectors/source-google-sheets/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-google-sheets/requirements.txt +++ b/airbyte-integrations/connectors/source-google-sheets/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 5d2f45a40ba9..1dd377a9357e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -6,19 +6,20 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", "backoff", "requests", "google-auth-httplib2", "google-api-python-client", - "PyYAML==5.4", + "PyYAML~=6.0", "pydantic~=1.9.2", "Unidecode", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py index ab71a6017f22..cbc988ca70ba 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import logging from typing import Dict, List import backoff @@ -11,29 +11,44 @@ from .helpers import SCOPES, Helpers - -def give_up(error): - code = error.resp.status - # Stop retrying if it's not a problem with the rate limit or on the server end - return not (code == status_codes.TOO_MANY_REQUESTS or 500 <= code < 600) +logger = logging.getLogger("airbyte") class GoogleSheetsClient: + class Backoff: + row_batch_size = 200 + + @classmethod + def increase_row_batch_size(cls, details): + if details["exception"].status_code == status_codes.TOO_MANY_REQUESTS and cls.row_batch_size < 1000: + cls.row_batch_size = cls.row_batch_size + 10 + logger.info(f"Increasing number of records fetching due to rate limits. Current value: {cls.row_batch_size}") + + @staticmethod + def give_up(error): + code = error.resp.status + # Stop retrying if it's not a problem with the rate limit or on the server end + return not (code == status_codes.TOO_MANY_REQUESTS or 500 <= code < 600) + def __init__(self, credentials: Dict[str, str], scopes: List[str] = SCOPES): self.client = Helpers.get_authenticated_sheets_client(credentials, scopes) - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) + def create_range(self, sheet, row_cursor): + range = f"{sheet}!{row_cursor}:{row_cursor + self.Backoff.row_batch_size}" + return range + + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size) def get(self, **kwargs): return self.client.get(**kwargs).execute() - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size) def create(self, **kwargs): return self.client.create(**kwargs).execute() - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size) def get_values(self, **kwargs): return self.client.values().batchGet(**kwargs).execute() - @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=give_up) + @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size) def update_values(self, **kwargs): return self.client.values().batchUpdate(**kwargs).execute() diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py index e99f6952fc56..d8f367baddb6 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py @@ -7,7 +7,7 @@ import re from collections import defaultdict from datetime import datetime -from typing import Dict, FrozenSet, Iterable, List +from typing import Dict, FrozenSet, Iterable, List, Tuple from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, SyncMode @@ -218,3 +218,11 @@ def get_spreadsheet_id(id_or_url: str) -> str: return m.group(2) else: return id_or_url + + @staticmethod + def check_sheet_is_valid(client, spreadsheet_id: str, sheet_name: str) -> Tuple[bool, str]: + try: + Helpers.get_first_row(client, spreadsheet_id, sheet_name) + return True, "" + except Exception as e: + return False, str(e) diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py index 638b6d4dc1b1..cf7416a48276 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py @@ -28,10 +28,8 @@ from .helpers import Helpers from .models.spreadsheet import Spreadsheet from .models.spreadsheet_values import SpreadsheetValues -from .utils import safe_name_conversion +from .utils import exception_description_by_status_code, safe_name_conversion -# set default batch read size -ROW_BATCH_SIZE = 200 # override default socket timeout to be 10 mins instead of 60 sec. # on behalf of https://github.com/airbytehq/oncall/issues/242 DEFAULT_SOCKET_TIMEOUT: int = 600 @@ -131,12 +129,18 @@ def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: return AirbyteCatalog(streams=streams) except errors.HttpError as err: - reason = str(err) - if err.resp.status == status_codes.NOT_FOUND: - reason = "Requested spreadsheet was not found." - raise Exception(f"Could not run discovery: {reason}") - - def read( + error_description = exception_description_by_status_code(err.resp.status, spreadsheet_id) + config_error_status_codes = [status_codes.NOT_FOUND, status_codes.FORBIDDEN] + if err.resp.status in config_error_status_codes: + message = f"{error_description}. {err.reason}." + raise AirbyteTracedException( + message=message, + internal_message=message, + failure_type=FailureType.config_error, + ) from err + raise Exception(f"Could not discover the schema of your spreadsheet. {error_description}. {err.reason}.") + + def _read( self, logger: AirbyteLogger, config: json, @@ -148,7 +152,6 @@ def read( sheet_to_column_name = Helpers.parse_sheet_and_column_names_from_catalog(catalog) spreadsheet_id = Helpers.get_spreadsheet_id(config["spreadsheet_id"]) - row_batch_size = config.get("row_batch_size", ROW_BATCH_SIZE) logger.info(f"Starting syncing spreadsheet {spreadsheet_id}") # For each sheet in the spreadsheet, get a batch of rows, and as long as there hasn't been # a blank row, emit the row batch @@ -159,33 +162,65 @@ def read( logger.info(f"Row counts: {sheet_row_counts}") for sheet in sheet_to_column_index_to_name.keys(): logger.info(f"Syncing sheet {sheet}") - column_index_to_name = sheet_to_column_index_to_name[sheet] - row_cursor = 2 # we start syncing past the header row - # For the loop, it is necessary that the initial row exists when we send a request to the API, - # if the last row of the interval goes outside the sheet - this is normal, we will return - # only the real data of the sheet and in the next iteration we will loop out. - while row_cursor <= sheet_row_counts[sheet]: - range = f"{sheet}!{row_cursor}:{row_cursor + row_batch_size}" - logger.info(f"Fetching range {range}") - row_batch = SpreadsheetValues.parse_obj( - client.get_values(spreadsheetId=spreadsheet_id, ranges=range, majorDimension="ROWS") - ) - - row_cursor += row_batch_size + 1 - # there should always be one range since we requested only one - value_ranges = row_batch.valueRanges[0] - - if not value_ranges.values: - break - - row_values = value_ranges.values - if len(row_values) == 0: - break - - for row in row_values: - if not Helpers.is_row_empty(row) and Helpers.row_contains_relevant_data(row, column_index_to_name.keys()): - yield AirbyteMessage(type=Type.RECORD, record=Helpers.row_data_to_record_message(sheet, row, column_index_to_name)) - logger.info(f"Finished syncing spreadsheet {spreadsheet_id}") + # We revalidate the sheet here to avoid errors in case the sheet was changed after the sync started + is_valid, reason = Helpers.check_sheet_is_valid(client, spreadsheet_id, sheet) + if is_valid: + column_index_to_name = sheet_to_column_index_to_name[sheet] + row_cursor = 2 # we start syncing past the header row + # For the loop, it is necessary that the initial row exists when we send a request to the API, + # if the last row of the interval goes outside the sheet - this is normal, we will return + # only the real data of the sheet and in the next iteration we will loop out. + while row_cursor <= sheet_row_counts[sheet]: + range = client.create_range(sheet, row_cursor) + logger.info(f"Fetching range {range}") + row_batch = SpreadsheetValues.parse_obj( + client.get_values(spreadsheetId=spreadsheet_id, ranges=range, majorDimension="ROWS") + ) + + row_cursor += client.Backoff.row_batch_size + 1 + # there should always be one range since we requested only one + value_ranges = row_batch.valueRanges[0] + + if not value_ranges.values: + break + + row_values = value_ranges.values + if len(row_values) == 0: + break + + for row in row_values: + if not Helpers.is_row_empty(row) and Helpers.row_contains_relevant_data(row, column_index_to_name.keys()): + yield AirbyteMessage( + type=Type.RECORD, record=Helpers.row_data_to_record_message(sheet, row, column_index_to_name) + ) + else: + logger.info(f"Skipping syncing sheet {sheet}: {reason}") + + def read( + self, + logger: AirbyteLogger, + config: json, + catalog: ConfiguredAirbyteCatalog, + state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, + ) -> Generator[AirbyteMessage, None, None]: + spreadsheet_id = Helpers.get_spreadsheet_id(config["spreadsheet_id"]) + try: + yield from self._read(logger, config, catalog, state) + except errors.HttpError as e: + error_description = exception_description_by_status_code(e.status_code, spreadsheet_id) + + if e.status_code == status_codes.FORBIDDEN: + raise AirbyteTracedException( + message=f"Stopped syncing process. {error_description}", + internal_message=error_description, + failure_type=FailureType.config_error, + ) from e + if e.status_code == status_codes.TOO_MANY_REQUESTS: + logger.info(f"Stopped syncing process due to rate limits. {error_description}") + else: + logger.info(f"{e.status_code}: {e.reason}. {error_description}") + finally: + logger.info(f"Finished syncing spreadsheet {spreadsheet_id}") @staticmethod def get_credentials(config): diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/spec.yaml b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/spec.yaml index c802aa4364d6..5a2541b0d1a5 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/spec.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/spec.yaml @@ -12,18 +12,13 @@ connectionSpecification: type: string title: Spreadsheet Link description: >- - Enter the link to the Google spreadsheet you want to sync + Enter the link to the Google spreadsheet you want to sync. To copy the link, click the 'Share' button in the top-right corner of the spreadsheet, then click 'Copy link'. examples: - https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit - row_batch_size: - type: integer - title: Row Batch Size - description: Number of rows fetched when making a Google Sheet API call. Defaults to 200. - default: 200 names_conversion: type: boolean - title: Columns Name Conversion - description: Columns name conversion using a set of rules, for example, 'My Name' -> 'my-name'. + title: Convert Column Names to SQL-Compliant Format + description: Enables the conversion of column names to a standardized, SQL-compliant format. For example, 'My Name' -> 'my_name'. Enable this option if your destination is SQL-based. default: false credentials: type: object @@ -45,17 +40,17 @@ connectionSpecification: client_id: title: Client ID type: string - description: "Enter your Google application's Client ID" + description: "Enter your Google application's Client ID. See Google's documentation for more information." airbyte_secret: true client_secret: title: Client Secret type: string - description: "Enter your Google application's Client Secret" + description: "Enter your Google application's Client Secret. See Google's documentation for more information." airbyte_secret: true refresh_token: title: Refresh Token type: string - description: "Enter your Google application's refresh token" + description: "Enter your Google application's refresh token. See Google's documentation for more information." airbyte_secret: true - title: Service Account Key Authentication type: object @@ -69,7 +64,7 @@ connectionSpecification: service_account_info: type: string title: Service Account Information. - description: 'Enter your Google Cloud service account key in JSON format' + description: 'The JSON key of the service account to use for authorization. Read more here.' airbyte_secret: true examples: - '{ "type": "service_account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ... }' diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py index 9a39e9f7e33a..689d9856a8d7 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py @@ -6,6 +6,7 @@ import re import unidecode +from requests.status_codes import codes as status_codes TOKEN_PATTERN = re.compile(r"[A-Z]+[a-z]*|[a-z]+|\d+|(?P[^a-zA-Z\d]+)") DEFAULT_SEPARATOR = "_" @@ -40,3 +41,26 @@ def safe_name_conversion(text): if not new: raise Exception(f"initial string '{text}' converted to empty") return new + + +def exception_description_by_status_code(code: int, spreadsheet_id) -> str: + if code in [status_codes.INTERNAL_SERVER_ERROR, status_codes.BAD_GATEWAY, status_codes.SERVICE_UNAVAILABLE]: + return ( + "There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side." + " Please try again. If this issue persists, contact support" + ) + if code == status_codes.FORBIDDEN: + return ( + f"The authenticated Google Sheets user does not have permissions to view the spreadsheet with id {spreadsheet_id}. " + "Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" + ) + if code == status_codes.NOT_FOUND: + return ( + f"The requested Google Sheets spreadsheet with id {spreadsheet_id} does not exist. " + f"Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support" + ) + + if code == status_codes.TOO_MANY_REQUESTS: + return "Rate limit has been reached. Please try later or request a higher quota for your account." + + return "" diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py new file mode 100644 index 000000000000..425559f8edb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +import requests +from source_google_sheets.client import GoogleSheetsClient + + +@pytest.mark.parametrize( + "status, need_give_up", + [ + (429, False), (500, False), (404, True) + ] +) +def test_backoff_give_up(status, need_give_up, mocker): + e = requests.HTTPError('error') + e.resp = mocker.Mock(status=status) + assert need_give_up is GoogleSheetsClient.Backoff.give_up(e) + + +def test_backoff_increase_row_batch_size(mocker): + assert GoogleSheetsClient.Backoff.row_batch_size == 200 + e = requests.HTTPError('error') + e.status_code = 429 + GoogleSheetsClient.Backoff.increase_row_batch_size({"exception": e}) + assert GoogleSheetsClient.Backoff.row_batch_size == 210 + GoogleSheetsClient.Backoff.row_batch_size = 1000 + GoogleSheetsClient.Backoff.increase_row_batch_size({"exception": e}) + assert GoogleSheetsClient.Backoff.row_batch_size == 1000 diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index d886b9d59cc0..3743f3e46513 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -34,6 +34,18 @@ def google_sheet_client(row_data, spreadsheet_id, client): return sheet_client +def google_sheet_invalid_client(spreadsheet_id, client): + fake_response = Spreadsheet( + spreadsheetId=spreadsheet_id, + sheets=[Sheet(data=[])], + ) + client.get.return_value.execute.return_value = fake_response + with patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes: None): + sheet_client = GoogleSheetsClient({"fake": "credentials"}, ["auth_scopes"]) + sheet_client.client = client + return sheet_client + + class TestHelpers(unittest.TestCase): def test_headers_to_airbyte_stream(self): sheet_name = "sheet1" @@ -190,6 +202,26 @@ def test_get_first_row_empty_sheet(self): self.assertEqual(Helpers.get_first_row(sheet_client, spreadsheet_id, sheet), []) client.get.assert_called_with(spreadsheetId=spreadsheet_id, includeGridData=True, ranges=f"{sheet}!1:1") + def test_check_sheet_is_valid(self): + spreadsheet_id = "123" + sheet = "s1" + expected_first_row = ["1", "2", "3", "4"] + row_data = [RowData(values=[CellData(formattedValue=v) for v in expected_first_row])] + client = Mock() + sheet_client = google_sheet_client(row_data, spreadsheet_id, client) + is_valid, reason = Helpers.check_sheet_is_valid(sheet_client, spreadsheet_id, sheet) + self.assertTrue(is_valid) + self.assertEqual(reason, "") + + def test_check_sheet_is_valid_empty(self): + spreadsheet_id = "123" + sheet = "s1" + client = Mock() + sheet_client = google_sheet_invalid_client(spreadsheet_id, client) + is_valid, reason = Helpers.check_sheet_is_valid(sheet_client, spreadsheet_id, sheet) + self.assertFalse(is_valid) + self.assertEqual(reason, "Expected data for exactly one range for sheet s1") + def test_get_sheets_in_spreadsheet(self): spreadsheet_id = "id1" expected_sheets = ["s1", "s2"] diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py index a219f40a3323..372b2e7d4dc6 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py @@ -2,13 +2,44 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + import pytest import requests +from airbyte_cdk.models.airbyte_protocol import ( + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.utils import AirbyteTracedException from apiclient import errors from source_google_sheets import SourceGoogleSheets from source_google_sheets.client import GoogleSheetsClient -from source_google_sheets.helpers import SCOPES +from source_google_sheets.helpers import SCOPES, Helpers +from source_google_sheets.models import CellData, GridData, RowData, Sheet, SheetProperties, Spreadsheet + + +def set_http_error_for_google_sheets_client(mocker, resp): + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", side_effect=errors.HttpError(resp=resp, content=b'')) + + +def set_resp_http_error(status_code, error_message=None): + resp = requests.Response() + resp.status = status_code + if error_message: + resp.reason = error_message + return resp + + +def set_sheets_type_grid(sheet_first_row): + data = [ + GridData(rowData=[RowData(values=[CellData(formattedValue=v) for v in sheet_first_row]) + ])] + sheet = Sheet(properties=SheetProperties(title='sheet1', gridProperties='true', sheetType="GRID"), data=data) + return sheet def test_invalid_credentials_error_message(invalid_config): @@ -20,11 +51,255 @@ def test_invalid_credentials_error_message(invalid_config): def test_invalid_link_error_message(mocker, invalid_config): source = SourceGoogleSheets() - resp = requests.Response() - resp.status = 404 - mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) - mocker.patch.object(GoogleSheetsClient, "get", side_effect=errors.HttpError(resp=resp, content=b'')) + set_http_error_for_google_sheets_client(mocker, set_resp_http_error(404)) with pytest.raises(AirbyteTracedException) as e: source.check(logger=None, config=invalid_config) expected_message = 'Config error: The spreadsheet link is not valid. Enter the URL of the Google spreadsheet you want to sync.' assert e.value.args[0] == expected_message + + +def test_discover_404_error(mocker, invalid_config): + source = SourceGoogleSheets() + set_http_error_for_google_sheets_client(mocker, set_resp_http_error(404, "Requested entity was not found")) + + with pytest.raises(AirbyteTracedException) as e: + source.discover(logger=mocker.MagicMock(), config=invalid_config) + expected_message = ("The requested Google Sheets spreadsheet with id invalid_spreadsheet_id does not exist." + " Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support. Requested entity was not found.") + assert e.value.args[0] == expected_message + + +def test_discover_403_error(mocker, invalid_config): + source = SourceGoogleSheets() + set_http_error_for_google_sheets_client(mocker, set_resp_http_error(403, "The caller does not have right permissions")) + + with pytest.raises(AirbyteTracedException) as e: + source.discover(logger=mocker.MagicMock(), config=invalid_config) + expected_message = ("The authenticated Google Sheets user does not have permissions to view the " + "spreadsheet with id invalid_spreadsheet_id. Please ensure the authenticated user has access" + " to the Spreadsheet and reauthenticate. If the issue persists, contact support. " + "The caller does not have right permissions.") + assert e.value.args[0] == expected_message + + +def test_check_invalid_creds_json_file(invalid_config): + source = SourceGoogleSheets() + res = source.check(logger=None, config={""}) + assert 'Please use valid credentials json file' in res.message + + +def test_check_access_expired(mocker, invalid_config): + source = SourceGoogleSheets() + set_http_error_for_google_sheets_client(mocker, set_resp_http_error(403)) + expected_message = 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + with pytest.raises(AirbyteTracedException): + res = source.check(logger=None, config=invalid_config) + assert res.message == expected_message + + +def test_check_expected_to_read_data_from_1_sheet(mocker, invalid_config, caplog): + source = SourceGoogleSheets() + spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "2"]), set_sheets_type_grid(["3", "4"])]) + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) + res = source.check(logger=logging.getLogger('airbyte'), config=invalid_config) + assert str(res.status) == "Status.FAILED" + assert "Unexpected return result: Sheet sheet1 was expected to contain data on exactly 1 sheet." in caplog.text + + +def test_check_duplicated_headers(invalid_config, mocker, caplog): + source = SourceGoogleSheets() + spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "1", "3", "4"])]) + source = SourceGoogleSheets() + expected_message = "The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:sheet1, headers:['1']]" + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) + res = source.check(logger=logging.getLogger('airbyte'), config=invalid_config) + assert str(res.status) == "Status.FAILED" + assert expected_message in res.message + + +def test_check_status_succeeded(mocker, invalid_config): + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( + spreadsheetId='spreadsheet_id', sheets=[Sheet(properties=SheetProperties(title=t)) for t in ["1", "2", "3", "4"]] + )) + + res = source.check(logger=None, config=invalid_config) + assert str(res.status) == "Status.SUCCEEDED" + + +def test_discover_with_non_grid_sheets(mocker, invalid_config): + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( + spreadsheetId='spreadsheet_id', sheets=[Sheet(properties=SheetProperties(title=t)) for t in ["1", "2", "3", "4"]] + )) + res = source.discover(logger=mocker.MagicMock(), config=invalid_config) + assert res.streams == [] + + +def test_discover(mocker, invalid_config): + source = SourceGoogleSheets() + spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "2", "3", "4"])]) + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) + res = source.discover(logger=mocker.MagicMock(), config=invalid_config) + assert len(res.streams) == 1 + + +def test_discover_with_names_conversion(mocker, invalid_config): + invalid_config["names_conversion"] = True + spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1 тест", "2", "3", "4"])]) + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) + res = source.discover(logger=mocker.MagicMock(), config=invalid_config) + assert len(res.streams) == 1 + assert "_1_test" in res.streams[0].json_schema["properties"].keys() + + +def test_discover_incorrect_spreadsheet_name(mocker, invalid_config): + spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "2", "3", "4"])]) + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) + res = source.discover(logger=mocker.MagicMock(), config=invalid_config) + assert len(res.streams) == 1 + + +def test_discover_could_not_run_discover(mocker, invalid_config): + source = SourceGoogleSheets() + set_http_error_for_google_sheets_client(mocker, set_resp_http_error(500, "Interval Server error")) + + with pytest.raises(Exception) as e: + source.discover(logger=mocker.MagicMock(), config=invalid_config) + expected_message = ("Could not discover the schema of your spreadsheet. There was an issue with the Google Sheets API." + " This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support. Interval Server error.") + assert e.value.args[0] == expected_message + + +def test_get_credentials(invalid_config): + expected_config = { + 'auth_type': 'Client', 'client_id': 'fake_client_id', + 'client_secret': 'fake_client_secret', 'refresh_token': 'fake_refresh_token' + } + assert expected_config == SourceGoogleSheets.get_credentials(invalid_config) + + +def test_get_credentials_old_style(): + old_style_config = { + "credentials_json": "some old style data" + } + expected_config = {'auth_type': 'Service', 'service_account_info': 'some old style data'} + assert expected_config == SourceGoogleSheets.get_credentials(old_style_config) + + +def test_read_429_error(mocker, invalid_config, caplog): + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=mocker.Mock) + mocker.patch.object(Helpers, "get_sheets_in_spreadsheet", side_effect=errors.HttpError(resp=set_resp_http_error(429, "Request a higher quota limit"), content=b'')) + + sheet1 = "soccer_team" + sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) + sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) + assert [] == records + assert "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account" in caplog.text + + +def test_read_403_error(mocker, invalid_config, caplog): + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + mocker.patch.object(GoogleSheetsClient, "get", return_value=mocker.Mock) + mocker.patch.object(Helpers, "get_sheets_in_spreadsheet", side_effect=errors.HttpError(resp=set_resp_http_error(403, "Permission denied"), content=b'')) + + sheet1 = "soccer_team" + sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) + sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + with pytest.raises(AirbyteTracedException) as e: + next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) + assert str(e.value) == "The authenticated Google Sheets user does not have permissions to view the spreadsheet with id invalid_spreadsheet_id. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" + + +def test_read_expected_data_on_1_sheet(invalid_config, mocker, caplog): + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + sheet1 = "soccer_team" + sheet2 = "soccer_team2" + mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( + spreadsheetId='spreadsheet_id', sheets=[Sheet(properties=SheetProperties(title=t)) for t in [sheet1, sheet2]] + )) + + sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) + sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet2, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + + with pytest.raises(Exception) as e: + next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) + assert "Unexpected return result: Sheet soccer_team was expected to contain data on exactly 1 sheet." in str(e.value) + + +def test_read_emply_sheet(invalid_config, mocker, caplog): + source = SourceGoogleSheets() + mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) + sheet1 = "soccer_team" + sheet2 = "soccer_team2" + mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( + spreadsheetId=invalid_config["spreadsheet_id"], + sheets=[Sheet(properties=SheetProperties(title=t), data=[{"test1": "12", "test2": "123"},]) for t in [sheet1, ]] + )) + + sheet1_columns = frozenset(["arsenal", "chelsea"]) + sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet1, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ConfiguredAirbyteStream( + stream=AirbyteStream(name=sheet2, json_schema=sheet1_schema, supported_sync_modes=["full_refresh"]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + records = list(source.read(logger=logging.getLogger("airbyte"), catalog=catalog,config=invalid_config)) + assert records == [] + assert "The sheet soccer_team (ID invalid_spreadsheet_id) is empty!" in caplog.text diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py index 0787e1adba32..8e44ce2b3e37 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py @@ -3,7 +3,7 @@ # import pytest -from source_google_sheets.utils import name_conversion, safe_name_conversion +from source_google_sheets.utils import exception_description_by_status_code, name_conversion, safe_name_conversion def test_name_conversion(): @@ -24,3 +24,16 @@ def test_safe_name_conversion(): with pytest.raises(Exception) as exc_info: safe_name_conversion("*****") assert exc_info.value.args[0] == "initial string '*****' converted to empty" + + +@pytest.mark.parametrize( + "status_code, expected_message", + [ + (404, "The requested Google Sheets spreadsheet with id spreadsheet_id does not exist. Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support"), + (429, "Rate limit has been reached. Please try later or request a higher quota for your account."), + (500, "There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support"), + (403, "The authenticated Google Sheets user does not have permissions to view the spreadsheet with id spreadsheet_id. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support"), + ] +) +def test_exception_description_by_status_code(status_code, expected_message): + assert expected_message == exception_description_by_status_code(status_code, "spreadsheet_id") diff --git a/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml b/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml index ba8c7c474856..58d9d4bd572f 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-webfonts/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-webfonts/requirements.txt b/airbyte-integrations/connectors/source-google-webfonts/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/requirements.txt +++ b/airbyte-integrations/connectors/source-google-webfonts/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-webfonts/setup.py b/airbyte-integrations/connectors/source-google-webfonts/setup.py index 17f17855f877..863003482ff1 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/setup.py +++ b/airbyte-integrations/connectors/source-google-webfonts/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml b/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml index 126a55045584..a0b5a4ca2007 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/metadata.yaml @@ -18,4 +18,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/google-workspace-admin-reports tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt b/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py index ac4bf1b5aab0..7170103bf496 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/setup.py @@ -15,9 +15,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile index ec6cae2a94d3..a6c69afd9724 100644 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-greenhouse/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.0 +LABEL io.airbyte.version=0.4.2 LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml index 71f170151611..41a387794e0e 100644 --- a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml @@ -35,6 +35,9 @@ acceptance_tests: users: - name: updated_at bypass_reason: "Updated when you login to account" + job_posts: + - name: updated_at + bypass_reason: "Updated when you try edit without editing" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json index ca00786c22ee..0c293724d9ba 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json @@ -3,70 +3,70 @@ "type": "STREAM", "stream": { "stream_descriptor": { "name": "candidates" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "demographics_answers" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "users" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "scorecards" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "offers" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "job_stages" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "job_posts" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "interviews" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "jobs" }, - "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "updated_at": "2222-01-21T00:00:00.000Z" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "applications" }, - "stream_state": { "applied_at": "2222-01-21T00:00:00.000Z"} + "stream_state": { "applied_at": "2222-01-21T00:00:00.000Z" } } }, { @@ -113,4 +113,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl index b101bd61de51..ed7fa8cd13b4 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl @@ -1,273 +1,89 @@ -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}, "emitted_at": 1674121346826} -{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}, "emitted_at": 1674121346830} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:24:37.050Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test", "last_activity": "2020-11-24T23:24:37.049Z", "is_private": false, "id": 17130511003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:24:37.018Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}], "application_ids": [19214950003], "addresses": []}, "emitted_at": 1674121896848} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:25:13.806Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test2", "last_activity": "2020-11-24T23:25:13.804Z", "is_private": false, "id": 17130554003, "first_name": "Test2", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:25:13.777Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}], "application_ids": [19214993003], "addresses": []}, "emitted_at": 1674121896849} -{"stream": "close_reasons", "data": {"id": 4010635003, "name": "Not Filling"}, "emitted_at": 1671104667975} -{"stream": "close_reasons", "data": {"id": 4010634003, "name": "On Hold"}, "emitted_at": 1671104667977} -{"stream": "close_reasons", "data": {"id": 4010633003, "name": "Hire - New Headcount"}, "emitted_at": 1671104667978} -{"stream": "close_reasons", "data": {"id": 4010632003, "name": "Hire - Backfill"}, "emitted_at": 1671104667980} -{"stream": "degrees", "data": {"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, "emitted_at": 1671104668454} -{"stream": "degrees", "data": {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, "emitted_at": 1671104668455} -{"stream": "degrees", "data": {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, "emitted_at": 1671104668457} -{"stream": "degrees", "data": {"id": 10848290003, "name": "Master's Degree", "priority": 3, "external_id": null}, "emitted_at": 1671104668458} -{"stream": "departments", "data": {"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}, "emitted_at": 1671104680251} -{"stream": "departments", "data": {"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}, "emitted_at": 1671104680253} -{"stream": "job_posts", "data": {"id": 4252332003, "active": true, "live": false, "first_published_at": null, "title": "Test job", "location": {"id": 4219721003, "name": "test", "office_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4177046003, "content": "

    Test description

    ", "internal_content": null, "updated_at": "2021-04-02T17:38:54.835Z", "created_at": "2020-11-24T23:29:24.315Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_5125927003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_5125928003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1671104680729} -{"stream": "job_posts", "data": {"id": 4751597003, "active": true, "live": false, "first_published_at": null, "title": "Test Job 2", "location": {"id": 4700649003, "name": "US", "office_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4177048003, "content": "

    Job post content

    ", "internal_content": null, "updated_at": "2021-10-07T18:46:59.032Z", "created_at": "2021-10-07T18:46:58.846Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_7911674003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_7911675003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1671104680732} -{"stream": "jobs", "data": {"id": 4177046003, "name": "Test job", "requisition_id": "3", "notes": null, "confidential": false, "is_template": true, "copied_from_id": null, "status": "open", "created_at": "2020-11-24T23:27:11.699Z", "opened_at": "2020-11-24T23:27:11.878Z", "closed_at": null, "updated_at": "2021-04-21T17:48:24.779Z", "departments": [{"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4320015003, "opening_id": "3-1", "status": "open", "opened_at": "2020-11-24T23:27:11.723Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": null}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": null}}}, "emitted_at": 1671104681275} -{"stream": "jobs", "data": {"id": 4177048003, "name": "Test Job 2", "requisition_id": "4", "notes": null, "confidential": false, "is_template": false, "copied_from_id": null, "status": "open", "created_at": "2020-11-24T23:27:45.634Z", "opened_at": "2020-11-24T23:27:45.878Z", "closed_at": null, "updated_at": "2021-10-07T18:46:58.982Z", "departments": [{"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": null}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": null}}}, "emitted_at": 1671104681278} -{"stream": "offers", "data": {"id": 4154100003, "version": 1, "application_id": 19215333003, "created_at": "2020-11-24T23:32:25.760Z", "updated_at": "2020-11-24T23:32:25.772Z", "sent_at": null, "resolved_at": null, "starts_at": "2020-12-04", "status": "unresolved", "job_id": 4177048003, "candidate_id": 17130848003, "opening": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "custom_fields": {"employment_type": "Contract"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Contract"}}}, "emitted_at": 1671104681765} -{"stream": "scorecards", "data": {"id": 5253031003, "updated_at": "2020-11-24T23:33:10.440Z", "created_at": "2020-11-24T23:33:10.440Z", "interview": "Application Review", "interview_step": {"id": 5628634003, "name": "Application Review"}, "candidate_id": 17130848003, "application_id": 19215333003, "interviewed_at": "2020-11-25T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "submitted_at": "2020-11-24T23:33:10.440Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "no_decision"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no_decision"}], "ratings": {"definitely_not": [], "no": [], "mixed": [], "yes": [], "strong_yes": []}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": ""}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1674122180016} -{"stream": "scorecards", "data": {"id": 9664505003, "updated_at": "2021-09-29T17:23:11.468Z", "created_at": "2021-09-29T17:23:11.468Z", "interview": "Preliminary Screening Call", "interview_step": {"id": 5628615003, "name": "Preliminary Screening Call"}, "candidate_id": 40517966003, "application_id": 44937562003, "interviewed_at": "2021-09-29T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "submitted_at": "2021-09-29T17:23:11.468Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "yes"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "mixed"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "yes"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "mixed"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "yes"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "strong_yes"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "mixed"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no"}], "ratings": {"definitely_not": [], "no": ["Communication"], "mixed": ["Three to five years of experience", "Passionate", "Currently based locally"], "yes": ["Willing to do required travel", "Personable", "Organizational Skills", "Manage competing priorities", "Fits our salary range"], "strong_yes": ["Empathetic"]}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": "test"}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1674122180017} -{"stream": "users", "data": {"id": 4218085003, "name": "Greenhouse Admin", "first_name": "Greenhouse", "last_name": "Admin", "primary_email_address": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "updated_at": "2020-11-18T14:09:08.401Z", "created_at": "2020-11-18T14:09:08.401Z", "disabled": false, "site_admin": true, "emails": ["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1674122410599} -{"stream": "users", "data": {"id": 4218086003, "name": "Airbyte Team", "first_name": "Airbyte", "last_name": "Team", "primary_email_address": "integration-test@airbyte.io", "updated_at": "2023-02-21T22:22:17.287Z", "created_at": "2020-11-18T14:09:08.481Z", "disabled": false, "site_admin": true, "emails": ["integration-test@airbyte.io"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1674122410605} -{"stream": "custom_fields", "data": {"id": 7431124003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 0, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1678905640586} -{"stream": "custom_fields", "data": {"id": 4680905003, "name": "Relationship", "active": true, "field_type": "referral_question", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "relationship", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845806003, "name": "Coworker", "priority": 0, "external_id": null}, {"id": 10845807003, "name": "School", "priority": 1, "external_id": null}, {"id": 10845808003, "name": "Manager", "priority": 2, "external_id": null}, {"id": 10845809003, "name": "Reported", "priority": 3, "external_id": null}, {"id": 10845810003, "name": "Friend", "priority": 4, "external_id": null}, {"id": 10845811003, "name": "Do not know", "priority": 5, "external_id": null}]}, "emitted_at": 1678905640590} -{"stream": "demographics_question_sets", "data": {"title": "Test Question Set 1", "id": 4000197003, "description": "

    Test Question Set 1 description

    ", "active": true}, "emitted_at": 1671104684218} -{"stream": "demographics_question_sets", "data": {"title": "Test Question Set 2", "id": 4000198003, "description": "

    Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

    ", "active": true}, "emitted_at": 1671104684219} -{"stream": "demographics_questions", "data": {"translations": [{"name": "q1", "language": "en"}], "required": false, "name": "q1", "id": 4000714003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1671104684819} -{"stream": "demographics_questions", "data": {"translations": [{"name": "q2", "language": "en"}], "required": false, "name": "q2", "id": 4000715003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1671104684822} -{"stream": "demographics_questions", "data": {"translations": [{"name": "question1", "language": "en"}], "required": false, "name": "question1", "id": 4000716003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1671104684823} -{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a1", "language": "en"}], "name": "a1", "id": 4004258003, "free_form": false, "demographic_question_id": 4000714003, "active": true}, "emitted_at": 1671104685443} -{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a2", "language": "en"}], "name": "a2", "id": 4004259003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1671104685445} -{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a3", "language": "en"}], "name": "a3", "id": 4004260003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1671104685446} -{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.248Z", "id": 9308815003, "free_form_text": null, "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004262003, "created_at": "2021-11-03T19:56:07.248Z", "application_id": 47459993003}, "emitted_at": 1671104686128} -{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.252Z", "id": 9308816003, "free_form_text": "custom answer", "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004263003, "created_at": "2021-11-03T19:56:07.252Z", "application_id": 47459993003}, "emitted_at": 1671104686131} -{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.259Z", "id": 9308817003, "free_form_text": null, "demographic_question_id": 4000717003, "demographic_answer_option_id": 4004266003, "created_at": "2021-11-03T19:56:07.259Z", "application_id": 47459993003}, "emitted_at": 1671104686133} -{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.248Z", "id": 9308815003, "free_form_text": null, "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004262003, "created_at": "2021-11-03T19:56:07.248Z", "application_id": 47459993003}, "emitted_at": 1671104699294} -{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.252Z", "id": 9308816003, "free_form_text": "custom answer", "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004263003, "created_at": "2021-11-03T19:56:07.252Z", "application_id": 47459993003}, "emitted_at": 1671104699296} -{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.259Z", "id": 9308817003, "free_form_text": null, "demographic_question_id": 4000717003, "demographic_answer_option_id": 4004266003, "created_at": "2021-11-03T19:56:07.259Z", "application_id": 47459993003}, "emitted_at": 1671104699298} -{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a1", "language": "en"}], "name": "a1", "id": 4004258003, "free_form": false, "demographic_question_id": 4000714003, "active": true}, "emitted_at": 1671104702277} -{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a2", "language": "en"}], "name": "a2", "id": 4004259003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1671104702597} -{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a3", "language": "en"}], "name": "a3", "id": 4004260003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1671104702600} -{"stream": "interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1671104717302} -{"stream": "interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1671104717305} -{"stream": "interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1671104717307} -{"stream": "applications_interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1671104719202} -{"stream": "applications_interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1671104719205} -{"stream": "applications_interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1671104719207} -{"stream": "sources", "data": {"id": 4000000003, "name": "Recurse", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1671104720043} -{"stream": "sources", "data": {"id": 4000001003, "name": "cliquify", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1671104720044} -{"stream": "sources", "data": {"id": 4000002003, "name": "ContactOut", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1671104720046} -{"stream": "sources", "data": {"id": 4000003003, "name": "Crosschq", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1671104720047} -{"stream": "sources", "data": {"id": 4000004003, "name": "Talentpair", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1671104720048} -{"stream": "rejection_reasons", "data": {"id": 4014678003, "name": "reason1", "type": {"id": 4000000003, "name": "We rejected them"}}, "emitted_at": 1671104721477} -{"stream": "rejection_reasons", "data": {"id": 4014679003, "name": "reason2", "type": {"id": 4000001003, "name": "They rejected us"}}, "emitted_at": 1671104721479} -{"stream": "rejection_reasons", "data": {"id": 4014680003, "name": "reason3", "type": {"id": 4000002003, "name": "None specified"}}, "emitted_at": 1671104721480} -{"stream": "jobs_openings", "data": {"id": 4320015003, "opening_id": "3-1", "status": "open", "opened_at": "2020-11-24T23:27:11.723Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1671104722472} -{"stream": "jobs_openings", "data": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1671104722888} -{"stream": "jobs_openings", "data": {"id": 4926182003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1671104723277} -{"stream": "job_stages", "data": {"id": 5245803003, "name": "Application Review", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 0, "interviews": [{"id": 5628614003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628609003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1671104735884} -{"stream": "job_stages", "data": {"id": 5245804003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 1, "interviews": [{"id": 5628615003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628610003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1671104735886} -{"stream": "job_stages", "data": {"id": 5245805003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 2, "interviews": [{"id": 5628616003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628611003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1671104735888} -{"stream": "jobs_stages", "data": {"id": 5245803003, "name": "Application Review", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 0, "interviews": [{"id": 5628614003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628609003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1671104737182} -{"stream": "jobs_stages", "data": {"id": 5245804003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 1, "interviews": [{"id": 5628615003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628610003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1671104737185} -{"stream": "jobs_stages", "data": {"id": 5245805003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 2, "interviews": [{"id": 5628616003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628611003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1671104737187} -{"stream":"demographics_question_sets_questions","data":{"translations":[{"name":"q1","language":"en"}],"required":false,"name":"q1","id":4000714003,"demographic_question_set_id":4000197003,"answer_type":"multi_value_single_select","active":true},"emitted_at":1674591295418} -{"stream": "approvals", "data": {"id": 5217303003, "offer_id": null, "sequential": true, "version": 1, "approval_type": "open_job", "approval_status": "approved", "job_id": 4177048003, "requested_by_user_id": null, "approver_groups": []}, "emitted_at": 1681933619451} -{"stream": "approvals", "data": {"id": 5217304003, "offer_id": null, "sequential": true, "version": 1, "approval_type": "offer_job", "approval_status": "approved", "job_id": 4177048003, "requested_by_user_id": null, "approver_groups": []}, "emitted_at": 1681933619454} -{"stream": "approvals", "data": {"id": 5217305003, "offer_id": null, "sequential": true, "version": 1, "approval_type": "offer_candidate", "approval_status": "approved", "job_id": 4177048003, "requested_by_user_id": null, "approver_groups": []}, "emitted_at": 1681933619457} -{"stream": "schools", "data": {"id": 10845822003, "name": "Abraham Baldwin Agricultural College", "priority": 0, "external_id": null}, "emitted_at": 1681933683603} -{"stream": "schools", "data": {"id": 10845823003, "name": "Academy of Art University", "priority": 1, "external_id": null}, "emitted_at": 1681933683605} -{"stream": "schools", "data": {"id": 10845824003, "name": "Acadia University", "priority": 2, "external_id": null}, "emitted_at": 1681933683607} -{"stream": "schools", "data": {"id": 10845825003, "name": "Adams State University", "priority": 3, "external_id": null}, "emitted_at": 1681933683608} -{"stream": "schools", "data": {"id": 10845826003, "name": "Adelphi University", "priority": 4, "external_id": null}, "emitted_at": 1681933683610} -{"stream": "schools", "data": {"id": 10845827003, "name": "Adrian College", "priority": 5, "external_id": null}, "emitted_at": 1681933683611} -{"stream": "schools", "data": {"id": 10845828003, "name": "Adventist University of Health Sciences", "priority": 6, "external_id": null}, "emitted_at": 1681933683612} -{"stream": "schools", "data": {"id": 10845829003, "name": "Agnes Scott College", "priority": 7, "external_id": null}, "emitted_at": 1681933683613} -{"stream": "schools", "data": {"id": 10845830003, "name": "AIB College of Business", "priority": 8, "external_id": null}, "emitted_at": 1681933683614} -{"stream": "schools", "data": {"id": 10845831003, "name": "Alaska Pacific University", "priority": 9, "external_id": null}, "emitted_at": 1681933683615} -{"stream": "schools", "data": {"id": 10845832003, "name": "Albany College of Pharmacy and Health Sciences", "priority": 10, "external_id": null}, "emitted_at": 1681933683617} -{"stream": "schools", "data": {"id": 10845833003, "name": "Albany State University", "priority": 11, "external_id": null}, "emitted_at": 1681933683618} -{"stream": "schools", "data": {"id": 10845834003, "name": "Albertus Magnus College", "priority": 12, "external_id": null}, "emitted_at": 1681933683619} -{"stream": "schools", "data": {"id": 10845835003, "name": "Albion College", "priority": 13, "external_id": null}, "emitted_at": 1681933683620} -{"stream": "schools", "data": {"id": 10845836003, "name": "Albright College", "priority": 14, "external_id": null}, "emitted_at": 1681933683621} -{"stream": "schools", "data": {"id": 10845837003, "name": "Alderson Broaddus University", "priority": 15, "external_id": null}, "emitted_at": 1681933683622} -{"stream": "schools", "data": {"id": 10845838003, "name": "Alfred University", "priority": 16, "external_id": null}, "emitted_at": 1681933683624} -{"stream": "schools", "data": {"id": 10845839003, "name": "Alice Lloyd College", "priority": 17, "external_id": null}, "emitted_at": 1681933683625} -{"stream": "schools", "data": {"id": 10845840003, "name": "Allegheny College", "priority": 18, "external_id": null}, "emitted_at": 1681933683626} -{"stream": "schools", "data": {"id": 10845841003, "name": "Allen College", "priority": 19, "external_id": null}, "emitted_at": 1681933683627} -{"stream": "schools", "data": {"id": 10845842003, "name": "Allen University", "priority": 20, "external_id": null}, "emitted_at": 1681933683628} -{"stream": "schools", "data": {"id": 10845843003, "name": "Alliant International University", "priority": 21, "external_id": null}, "emitted_at": 1681933683629} -{"stream": "schools", "data": {"id": 10845844003, "name": "Alma College", "priority": 22, "external_id": null}, "emitted_at": 1681933683630} -{"stream": "schools", "data": {"id": 10845845003, "name": "Alvernia University", "priority": 23, "external_id": null}, "emitted_at": 1681933683632} -{"stream": "schools", "data": {"id": 10845846003, "name": "Alverno College", "priority": 24, "external_id": null}, "emitted_at": 1681933683633} -{"stream": "schools", "data": {"id": 10845847003, "name": "Amberton University", "priority": 25, "external_id": null}, "emitted_at": 1681933683634} -{"stream": "schools", "data": {"id": 10845848003, "name": "American Academy of Art", "priority": 26, "external_id": null}, "emitted_at": 1681933683635} -{"stream": "schools", "data": {"id": 10845849003, "name": "American Indian College of the Assemblies of God", "priority": 27, "external_id": null}, "emitted_at": 1681933683636} -{"stream": "schools", "data": {"id": 10845850003, "name": "American InterContinental University", "priority": 28, "external_id": null}, "emitted_at": 1681933683637} -{"stream": "schools", "data": {"id": 10845851003, "name": "American International College", "priority": 29, "external_id": null}, "emitted_at": 1681933683638} -{"stream": "schools", "data": {"id": 10845852003, "name": "American Jewish University", "priority": 30, "external_id": null}, "emitted_at": 1681933683639} -{"stream": "schools", "data": {"id": 10845853003, "name": "American Public University System", "priority": 31, "external_id": null}, "emitted_at": 1681933683640} -{"stream": "schools", "data": {"id": 10845854003, "name": "American University", "priority": 32, "external_id": null}, "emitted_at": 1681933683641} -{"stream": "schools", "data": {"id": 10845855003, "name": "American University in Bulgaria", "priority": 33, "external_id": null}, "emitted_at": 1681933683643} -{"stream": "schools", "data": {"id": 10845856003, "name": "American University in Cairo", "priority": 34, "external_id": null}, "emitted_at": 1681933683644} -{"stream": "schools", "data": {"id": 10845857003, "name": "American University of Beirut", "priority": 35, "external_id": null}, "emitted_at": 1681933683645} -{"stream": "schools", "data": {"id": 10845858003, "name": "American University of Paris", "priority": 36, "external_id": null}, "emitted_at": 1681933683646} -{"stream": "schools", "data": {"id": 10845859003, "name": "American University of Puerto Rico", "priority": 37, "external_id": null}, "emitted_at": 1681933683647} -{"stream": "schools", "data": {"id": 10845860003, "name": "Amherst College", "priority": 38, "external_id": null}, "emitted_at": 1681933683648} -{"stream": "schools", "data": {"id": 10845861003, "name": "Amridge University", "priority": 39, "external_id": null}, "emitted_at": 1681933683649} -{"stream": "schools", "data": {"id": 10845862003, "name": "Anderson University", "priority": 40, "external_id": null}, "emitted_at": 1681933683650} -{"stream": "schools", "data": {"id": 10845863003, "name": "Andrews University", "priority": 41, "external_id": null}, "emitted_at": 1681933683651} -{"stream": "schools", "data": {"id": 10845864003, "name": "Angelo State University", "priority": 42, "external_id": null}, "emitted_at": 1681933683652} -{"stream": "schools", "data": {"id": 10845865003, "name": "Anna Maria College", "priority": 43, "external_id": null}, "emitted_at": 1681933683653} -{"stream": "schools", "data": {"id": 10845866003, "name": "Antioch University", "priority": 44, "external_id": null}, "emitted_at": 1681933683653} -{"stream": "schools", "data": {"id": 10845867003, "name": "Appalachian Bible College", "priority": 45, "external_id": null}, "emitted_at": 1681933683654} -{"stream": "schools", "data": {"id": 10845868003, "name": "Aquinas College", "priority": 46, "external_id": null}, "emitted_at": 1681933683655} -{"stream": "schools", "data": {"id": 10845869003, "name": "Arcadia University", "priority": 47, "external_id": null}, "emitted_at": 1681933683656} -{"stream": "schools", "data": {"id": 10845870003, "name": "Argosy University", "priority": 48, "external_id": null}, "emitted_at": 1681933683657} -{"stream": "schools", "data": {"id": 10845871003, "name": "Arizona Christian University", "priority": 49, "external_id": null}, "emitted_at": 1681933683658} -{"stream": "schools", "data": {"id": 10845872003, "name": "Arizona State University - West", "priority": 50, "external_id": null}, "emitted_at": 1681933683659} -{"stream": "schools", "data": {"id": 10845873003, "name": "Arkansas Baptist College", "priority": 51, "external_id": null}, "emitted_at": 1681933683659} -{"stream": "schools", "data": {"id": 10845874003, "name": "Arkansas Tech University", "priority": 52, "external_id": null}, "emitted_at": 1681933683660} -{"stream": "schools", "data": {"id": 10845875003, "name": "Armstrong Atlantic State University", "priority": 53, "external_id": null}, "emitted_at": 1681933683661} -{"stream": "schools", "data": {"id": 10845876003, "name": "Art Academy of Cincinnati", "priority": 54, "external_id": null}, "emitted_at": 1681933683662} -{"stream": "schools", "data": {"id": 10845877003, "name": "Art Center College of Design", "priority": 55, "external_id": null}, "emitted_at": 1681933683663} -{"stream": "schools", "data": {"id": 10845878003, "name": "Art Institute of Atlanta", "priority": 56, "external_id": null}, "emitted_at": 1681933683664} -{"stream": "schools", "data": {"id": 10845879003, "name": "Art Institute of Colorado", "priority": 57, "external_id": null}, "emitted_at": 1681933683665} -{"stream": "schools", "data": {"id": 10845880003, "name": "Art Institute of Houston", "priority": 58, "external_id": null}, "emitted_at": 1681933683665} -{"stream": "schools", "data": {"id": 10845881003, "name": "Art Institute of Pittsburgh", "priority": 59, "external_id": null}, "emitted_at": 1681933683666} -{"stream": "schools", "data": {"id": 10845882003, "name": "Art Institute of Portland", "priority": 60, "external_id": null}, "emitted_at": 1681933683667} -{"stream": "schools", "data": {"id": 10845883003, "name": "Art Institute of Seattle", "priority": 61, "external_id": null}, "emitted_at": 1681933683668} -{"stream": "schools", "data": {"id": 10845884003, "name": "Asbury University", "priority": 62, "external_id": null}, "emitted_at": 1681933683669} -{"stream": "schools", "data": {"id": 10845885003, "name": "Ashford University", "priority": 63, "external_id": null}, "emitted_at": 1681933683670} -{"stream": "schools", "data": {"id": 10845886003, "name": "Ashland University", "priority": 64, "external_id": null}, "emitted_at": 1681933683670} -{"stream": "schools", "data": {"id": 10845887003, "name": "Assumption College", "priority": 65, "external_id": null}, "emitted_at": 1681933683671} -{"stream": "schools", "data": {"id": 10845888003, "name": "Athens State University", "priority": 66, "external_id": null}, "emitted_at": 1681933683672} -{"stream": "schools", "data": {"id": 10845889003, "name": "Auburn University - Montgomery", "priority": 67, "external_id": null}, "emitted_at": 1681933683673} -{"stream": "schools", "data": {"id": 10845890003, "name": "Augsburg College", "priority": 68, "external_id": null}, "emitted_at": 1681933683674} -{"stream": "schools", "data": {"id": 10845891003, "name": "Augustana College", "priority": 69, "external_id": null}, "emitted_at": 1681933683675} -{"stream": "schools", "data": {"id": 10845892003, "name": "Aurora University", "priority": 70, "external_id": null}, "emitted_at": 1681933683676} -{"stream": "schools", "data": {"id": 10845893003, "name": "Austin College", "priority": 71, "external_id": null}, "emitted_at": 1681933683676} -{"stream": "schools", "data": {"id": 10845894003, "name": "Alcorn State University", "priority": 72, "external_id": null}, "emitted_at": 1681933683677} -{"stream": "schools", "data": {"id": 10845895003, "name": "Ave Maria University", "priority": 73, "external_id": null}, "emitted_at": 1681933683678} -{"stream": "schools", "data": {"id": 10845896003, "name": "Averett University", "priority": 74, "external_id": null}, "emitted_at": 1681933683679} -{"stream": "schools", "data": {"id": 10845897003, "name": "Avila University", "priority": 75, "external_id": null}, "emitted_at": 1681933683680} -{"stream": "schools", "data": {"id": 10845898003, "name": "Azusa Pacific University", "priority": 76, "external_id": null}, "emitted_at": 1681933683681} -{"stream": "schools", "data": {"id": 10845899003, "name": "Babson College", "priority": 77, "external_id": null}, "emitted_at": 1681933683681} -{"stream": "schools", "data": {"id": 10845900003, "name": "Bacone College", "priority": 78, "external_id": null}, "emitted_at": 1681933683682} -{"stream": "schools", "data": {"id": 10845901003, "name": "Baker College of Flint", "priority": 79, "external_id": null}, "emitted_at": 1681933683683} -{"stream": "schools", "data": {"id": 10845902003, "name": "Baker University", "priority": 80, "external_id": null}, "emitted_at": 1681933683684} -{"stream": "schools", "data": {"id": 10845903003, "name": "Baldwin Wallace University", "priority": 81, "external_id": null}, "emitted_at": 1681933683685} -{"stream": "schools", "data": {"id": 10845904003, "name": "Christian Brothers University", "priority": 82, "external_id": null}, "emitted_at": 1681933683686} -{"stream": "schools", "data": {"id": 10845905003, "name": "Abilene Christian University", "priority": 83, "external_id": null}, "emitted_at": 1681933683687} -{"stream": "schools", "data": {"id": 10845906003, "name": "Arizona State University", "priority": 84, "external_id": null}, "emitted_at": 1681933683687} -{"stream": "schools", "data": {"id": 10845907003, "name": "Auburn University", "priority": 85, "external_id": null}, "emitted_at": 1681933683688} -{"stream": "schools", "data": {"id": 10845908003, "name": "Alabama A&M University", "priority": 86, "external_id": null}, "emitted_at": 1681933683689} -{"stream": "schools", "data": {"id": 10845909003, "name": "Alabama State University", "priority": 87, "external_id": null}, "emitted_at": 1681933683690} -{"stream": "schools", "data": {"id": 10845910003, "name": "Arkansas State University", "priority": 88, "external_id": null}, "emitted_at": 1681933683691} -{"stream": "schools", "data": {"id": 10845911003, "name": "Baptist Bible College", "priority": 89, "external_id": null}, "emitted_at": 1681933683692} -{"stream": "schools", "data": {"id": 10845912003, "name": "Baptist Bible College and Seminary", "priority": 90, "external_id": null}, "emitted_at": 1681933683692} -{"stream": "schools", "data": {"id": 10845913003, "name": "Baptist College of Florida", "priority": 91, "external_id": null}, "emitted_at": 1681933683693} -{"stream": "schools", "data": {"id": 10845914003, "name": "Baptist Memorial College of Health Sciences", "priority": 92, "external_id": null}, "emitted_at": 1681933683694} -{"stream": "schools", "data": {"id": 10845915003, "name": "Baptist Missionary Association Theological Seminary", "priority": 93, "external_id": null}, "emitted_at": 1681933683695} -{"stream": "schools", "data": {"id": 10845916003, "name": "Bard College", "priority": 94, "external_id": null}, "emitted_at": 1681933683696} -{"stream": "schools", "data": {"id": 10845917003, "name": "Bard College at Simon's Rock", "priority": 95, "external_id": null}, "emitted_at": 1681933683697} -{"stream": "schools", "data": {"id": 10845918003, "name": "Barnard College", "priority": 96, "external_id": null}, "emitted_at": 1681933683698} -{"stream": "schools", "data": {"id": 10845919003, "name": "Barry University", "priority": 97, "external_id": null}, "emitted_at": 1681933683698} -{"stream": "schools", "data": {"id": 10845920003, "name": "Barton College", "priority": 98, "external_id": null}, "emitted_at": 1681933683699} -{"stream": "schools", "data": {"id": 10845921003, "name": "Bastyr University", "priority": 99, "external_id": null}, "emitted_at": 1681933683700} -{"stream": "offices", "data": {"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}, "emitted_at": 1681933973520} -{"stream": "disciplines", "data": {"id": 10848297003, "name": "Accounting", "priority": 0, "external_id": null}, "emitted_at": 1681934694438} -{"stream": "disciplines", "data": {"id": 10848298003, "name": "African Studies", "priority": 1, "external_id": null}, "emitted_at": 1681934694442} -{"stream": "disciplines", "data": {"id": 10848299003, "name": "Agriculture", "priority": 2, "external_id": null}, "emitted_at": 1681934694445} -{"stream": "disciplines", "data": {"id": 10848300003, "name": "Anthropology", "priority": 3, "external_id": null}, "emitted_at": 1681934694447} -{"stream": "disciplines", "data": {"id": 10848301003, "name": "Applied Health Services", "priority": 4, "external_id": null}, "emitted_at": 1681934694449} -{"stream": "disciplines", "data": {"id": 10848302003, "name": "Architecture", "priority": 5, "external_id": null}, "emitted_at": 1681934694452} -{"stream": "disciplines", "data": {"id": 10848303003, "name": "Art", "priority": 6, "external_id": null}, "emitted_at": 1681934694454} -{"stream": "disciplines", "data": {"id": 10848304003, "name": "Asian Studies", "priority": 7, "external_id": null}, "emitted_at": 1681934694457} -{"stream": "disciplines", "data": {"id": 10848305003, "name": "Biology", "priority": 8, "external_id": null}, "emitted_at": 1681934694458} -{"stream": "disciplines", "data": {"id": 10848306003, "name": "Business", "priority": 9, "external_id": null}, "emitted_at": 1681934694461} -{"stream": "disciplines", "data": {"id": 10848307003, "name": "Business Administration", "priority": 10, "external_id": null}, "emitted_at": 1681934694462} -{"stream": "disciplines", "data": {"id": 10848308003, "name": "Chemistry", "priority": 11, "external_id": null}, "emitted_at": 1681934694464} -{"stream": "disciplines", "data": {"id": 10848309003, "name": "Classical Languages", "priority": 12, "external_id": null}, "emitted_at": 1681934694467} -{"stream": "disciplines", "data": {"id": 10848310003, "name": "Communications & Film", "priority": 13, "external_id": null}, "emitted_at": 1681934694469} -{"stream": "disciplines", "data": {"id": 10848311003, "name": "Computer Science", "priority": 14, "external_id": null}, "emitted_at": 1681934694472} -{"stream": "disciplines", "data": {"id": 10848312003, "name": "Dentistry", "priority": 15, "external_id": null}, "emitted_at": 1681934694475} -{"stream": "disciplines", "data": {"id": 10848313003, "name": "Developing Nations", "priority": 16, "external_id": null}, "emitted_at": 1681934694477} -{"stream": "disciplines", "data": {"id": 10848314003, "name": "Discipline Unknown", "priority": 17, "external_id": null}, "emitted_at": 1681934694480} -{"stream": "disciplines", "data": {"id": 10848315003, "name": "Earth Sciences", "priority": 18, "external_id": null}, "emitted_at": 1681934694481} -{"stream": "disciplines", "data": {"id": 10848316003, "name": "Economics", "priority": 19, "external_id": null}, "emitted_at": 1681934694483} -{"stream": "disciplines", "data": {"id": 10848317003, "name": "Education", "priority": 20, "external_id": null}, "emitted_at": 1681934694485} -{"stream": "disciplines", "data": {"id": 10848318003, "name": "Electronics", "priority": 21, "external_id": null}, "emitted_at": 1681934694487} -{"stream": "disciplines", "data": {"id": 10848319003, "name": "Engineering", "priority": 22, "external_id": null}, "emitted_at": 1681934694489} -{"stream": "disciplines", "data": {"id": 10848320003, "name": "English Studies", "priority": 23, "external_id": null}, "emitted_at": 1681934694492} -{"stream": "disciplines", "data": {"id": 10848321003, "name": "Environmental Studies", "priority": 24, "external_id": null}, "emitted_at": 1681934694493} -{"stream": "disciplines", "data": {"id": 10848322003, "name": "European Studies", "priority": 25, "external_id": null}, "emitted_at": 1681934694495} -{"stream": "disciplines", "data": {"id": 10848323003, "name": "Fashion", "priority": 26, "external_id": null}, "emitted_at": 1681934694496} -{"stream": "disciplines", "data": {"id": 10848324003, "name": "Finance", "priority": 27, "external_id": null}, "emitted_at": 1681934694498} -{"stream": "disciplines", "data": {"id": 10848325003, "name": "Fine Arts", "priority": 28, "external_id": null}, "emitted_at": 1681934694499} -{"stream": "disciplines", "data": {"id": 10848326003, "name": "General Studies", "priority": 29, "external_id": null}, "emitted_at": 1681934694501} -{"stream": "disciplines", "data": {"id": 10848327003, "name": "Health Services", "priority": 30, "external_id": null}, "emitted_at": 1681934694503} -{"stream": "disciplines", "data": {"id": 10848328003, "name": "History", "priority": 31, "external_id": null}, "emitted_at": 1681934694504} -{"stream": "disciplines", "data": {"id": 10848329003, "name": "Human Resources Management", "priority": 32, "external_id": null}, "emitted_at": 1681934694506} -{"stream": "disciplines", "data": {"id": 10848330003, "name": "Humanities", "priority": 33, "external_id": null}, "emitted_at": 1681934694508} -{"stream": "disciplines", "data": {"id": 10848331003, "name": "Industrial Arts & Carpentry", "priority": 34, "external_id": null}, "emitted_at": 1681934694510} -{"stream": "disciplines", "data": {"id": 10848332003, "name": "Information Systems", "priority": 35, "external_id": null}, "emitted_at": 1681934694511} -{"stream": "disciplines", "data": {"id": 10848333003, "name": "International Relations", "priority": 36, "external_id": null}, "emitted_at": 1681934694512} -{"stream": "disciplines", "data": {"id": 10848334003, "name": "Journalism", "priority": 37, "external_id": null}, "emitted_at": 1681934694514} -{"stream": "disciplines", "data": {"id": 10848335003, "name": "Languages", "priority": 38, "external_id": null}, "emitted_at": 1681934694515} -{"stream": "disciplines", "data": {"id": 10848336003, "name": "Latin American Studies", "priority": 39, "external_id": null}, "emitted_at": 1681934694516} -{"stream": "disciplines", "data": {"id": 10848337003, "name": "Law", "priority": 40, "external_id": null}, "emitted_at": 1681934694517} -{"stream": "disciplines", "data": {"id": 10848338003, "name": "Linguistics", "priority": 41, "external_id": null}, "emitted_at": 1681934694519} -{"stream": "disciplines", "data": {"id": 10848339003, "name": "Manufacturing & Mechanics", "priority": 42, "external_id": null}, "emitted_at": 1681934694521} -{"stream": "disciplines", "data": {"id": 10848340003, "name": "Mathematics", "priority": 43, "external_id": null}, "emitted_at": 1681934694523} -{"stream": "disciplines", "data": {"id": 10848341003, "name": "Medicine", "priority": 44, "external_id": null}, "emitted_at": 1681934694525} -{"stream": "disciplines", "data": {"id": 10848342003, "name": "Middle Eastern Studies", "priority": 45, "external_id": null}, "emitted_at": 1681934694526} -{"stream": "disciplines", "data": {"id": 10848343003, "name": "Naval Science", "priority": 46, "external_id": null}, "emitted_at": 1681934694527} -{"stream": "disciplines", "data": {"id": 10848344003, "name": "North American Studies", "priority": 47, "external_id": null}, "emitted_at": 1681934694529} -{"stream": "disciplines", "data": {"id": 10848345003, "name": "Nuclear Technics", "priority": 48, "external_id": null}, "emitted_at": 1681934694530} -{"stream": "disciplines", "data": {"id": 10848346003, "name": "Operations Research & Strategy", "priority": 49, "external_id": null}, "emitted_at": 1681934694531} -{"stream": "disciplines", "data": {"id": 10848347003, "name": "Organizational Theory", "priority": 50, "external_id": null}, "emitted_at": 1681934694533} -{"stream": "disciplines", "data": {"id": 10848348003, "name": "Philosophy", "priority": 51, "external_id": null}, "emitted_at": 1681934694534} -{"stream": "disciplines", "data": {"id": 10848349003, "name": "Physical Education", "priority": 52, "external_id": null}, "emitted_at": 1681934694536} -{"stream": "disciplines", "data": {"id": 10848350003, "name": "Physical Sciences", "priority": 53, "external_id": null}, "emitted_at": 1681934694537} -{"stream": "disciplines", "data": {"id": 10848351003, "name": "Physics", "priority": 54, "external_id": null}, "emitted_at": 1681934694540} -{"stream": "disciplines", "data": {"id": 10848352003, "name": "Political Science", "priority": 55, "external_id": null}, "emitted_at": 1681934694541} -{"stream": "disciplines", "data": {"id": 10848353003, "name": "Psychology", "priority": 56, "external_id": null}, "emitted_at": 1681934694543} -{"stream": "disciplines", "data": {"id": 10848354003, "name": "Public Policy", "priority": 57, "external_id": null}, "emitted_at": 1681934694544} -{"stream": "disciplines", "data": {"id": 10848355003, "name": "Public Service", "priority": 58, "external_id": null}, "emitted_at": 1681934694545} -{"stream": "disciplines", "data": {"id": 10848356003, "name": "Religious Studies", "priority": 59, "external_id": null}, "emitted_at": 1681934694547} -{"stream": "disciplines", "data": {"id": 10848357003, "name": "Russian & Soviet Studies", "priority": 60, "external_id": null}, "emitted_at": 1681934694548} -{"stream": "disciplines", "data": {"id": 10848358003, "name": "Scandinavian Studies", "priority": 61, "external_id": null}, "emitted_at": 1681934694549} -{"stream": "disciplines", "data": {"id": 10848359003, "name": "Science", "priority": 62, "external_id": null}, "emitted_at": 1681934694550} -{"stream": "disciplines", "data": {"id": 10848360003, "name": "Slavic Studies", "priority": 63, "external_id": null}, "emitted_at": 1681934694552} -{"stream": "disciplines", "data": {"id": 10848361003, "name": "Social Science", "priority": 64, "external_id": null}, "emitted_at": 1681934694553} -{"stream": "disciplines", "data": {"id": 10848362003, "name": "Social Sciences", "priority": 65, "external_id": null}, "emitted_at": 1681934694554} -{"stream": "disciplines", "data": {"id": 10848363003, "name": "Sociology", "priority": 66, "external_id": null}, "emitted_at": 1681934694556} -{"stream": "disciplines", "data": {"id": 10848364003, "name": "Speech", "priority": 67, "external_id": null}, "emitted_at": 1681934694557} -{"stream": "disciplines", "data": {"id": 10848365003, "name": "Statistics & Decision Theory", "priority": 68, "external_id": null}, "emitted_at": 1681934694559} -{"stream": "disciplines", "data": {"id": 10848366003, "name": "Urban Studies", "priority": 69, "external_id": null}, "emitted_at": 1681934694560} -{"stream": "disciplines", "data": {"id": 10848367003, "name": "Veterinary Medicine", "priority": 70, "external_id": null}, "emitted_at": 1681934694561} -{"stream": "disciplines", "data": {"id": 10848368003, "name": "Other", "priority": 71, "external_id": null}, "emitted_at": 1681934694562} -{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68322855003, "created_at": "2020-11-24T23:24:37.047Z", "subject": null, "body": "John Lafleur added Test Test through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936729321} -{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68323000003, "created_at": "2020-11-24T23:25:13.802Z", "subject": null, "body": "John Lafleur added Test2 Test2 through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936729557} -{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68323687003, "created_at": "2020-11-24T23:28:19.777Z", "subject": null, "body": "John Lafleur added Name Lastname through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936729844} -{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68324733003, "created_at": "2020-11-24T23:32:25.778Z", "subject": null, "body": "John Lafleur created an offer for Jack D.", "user": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}}, {"id": 68324702003, "created_at": "2020-11-24T23:32:12.116Z", "subject": null, "body": "Jack D was moved into Offer for Test Job 2 by John Lafleur.", "user": null}, {"id": 68324227003, "created_at": "2020-11-24T23:30:14.622Z", "subject": null, "body": "John Lafleur added Jack D through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936730097} -{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 188328519003, "created_at": "2021-09-29T16:38:03.617Z", "subject": "Rejected from Test job by John Lafleur", "body": "Reason: Other (add notes below)\n\nRejection reasons", "user": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}}, {"id": 188328222003, "created_at": "2021-09-29T16:37:27.657Z", "subject": null, "body": "John Lafleur added Test User through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936730375} -{"stream": "activity_feed", "data": {"emails": [{"id": 191580377003, "created_at": "2021-10-08T08:18:36.301Z", "subject": "Information Request for Airbyte Integration Sandbox", "body": "Test,\n\nThanks for your interest in the Test job position at Airbyte Integration Sandbox. Since we do not have a formal application from you on file, we have a few outstanding questions for you.\n\nPlease use this link to submit additional information: https://grnh.se/r/bc1d8b994d7e9895d733d77ce40483us\n\nRegards,\nJohn", "to": "vadym.hevlich@zazmic.com", "from": "\"John Lafleur\" ", "cc": [], "user": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}}], "notes": [], "activities": [{"id": 192035490003, "created_at": "2021-10-10T16:21:44.211Z", "subject": null, "body": "Test Scheduled Interview was moved into Preliminary Phone Screen for Test job by Greenhouse Admin.", "user": null}, {"id": 188353359003, "created_at": "2021-09-29T17:20:36.122Z", "subject": null, "body": "John Lafleur added Test Scheduled Interview through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936730635} -{"stream": "activity_feed", "data": {"emails": [{"id": 200928610003, "created_at": "2021-11-03T19:53:26.769Z", "subject": "Information Request for Airbyte Integration Sandbox", "body": "Test,\n\nThanks for your interest in the Test job 3 position at Airbyte Integration Sandbox. Since we do not have a formal application from you on file, we have a few outstanding questions for you.\n\nPlease use this link to submit additional information: https://grnh.se/r/b5d6c1c2aeefc20290006449efc093us\n\nRegards,\nJohn", "to": "vadym.hevlich@zazmic.com", "from": "\"John Lafleur\" ", "cc": [], "user": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}}], "notes": [], "activities": [{"id": 200932536003, "created_at": "2021-11-03T19:56:07.399Z", "subject": null, "body": "Test Candidate submitted a job post for Test job 3.", "user": null}, {"id": 200925541003, "created_at": "2021-11-03T19:51:14.699Z", "subject": null, "body": "John Lafleur added Test Candidate through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936730891} -{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 207104666003, "created_at": "2021-11-22T08:41:55.711Z", "subject": null, "body": "John Lafleur added Yurii Cherniaev through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1681936731132} -{"stream": "email_templates", "data": {"id": 4071597003, "name": "Default Eeoc Data Request", "default": true, "updated_at": "2021-12-20T18:37:50.451Z", "created_at": "2020-11-18T14:09:07.631Z", "description": "This template is used for requesting EEOC data from the candidate.", "type": "eeoc_data_request", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Thank you for your interest in {{JOB_NAME}} at {{COMPANY}}.

    \n\n

    For government reporting purposes, we ask candidates to respond to the below self-identification survey.\nCompletion of the form is entirely voluntary. Whatever your decision, it will not be considered in the hiring\nprocess or thereafter. Any information that you do provide will be recorded and maintained in a\nconfidential file.

    \n\n

    As set forth in {{COMPANY}}\u2019s Equal Employment Opportunity policy,\nwe do not discriminate on the basis of any protected group status under any applicable law.

    \n\n

    Thank You

    ", "user": null}, "emitted_at": 1681936906593} -{"stream": "email_templates", "data": {"id": 4112339003, "name": "Default Candidate Self-Schedule request", "default": true, "updated_at": "2021-07-15T20:09:31.345Z", "created_at": "2021-07-15T20:09:31.345Z", "description": "This template is used when sending a request for a candidate to self-schedule.", "type": "candidate_self_schedule_request", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    Hi {{CANDIDATE_FIRST_NAME}},

    \n\n

    Please use this link to schedule your interview: {{INTERVIEW_REQUEST_LINK}}

    \n\n

    \n Regards,\n
    \n {{MY_FIRST_NAME}}\n

    ", "user": null}, "emitted_at": 1681936906595} -{"stream": "email_templates", "data": {"id": 4071603003, "name": "Default Agency Job Assignment Notification", "default": true, "updated_at": "2020-11-18T14:09:07.667Z", "created_at": "2020-11-18T14:09:07.667Z", "description": "This template is used when an agency recruiter is added to a job", "type": "agency_recruiter_assigned", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Hi,

    \n\n

    {{MY_FULL_NAME}} from {{COMPANY}} has assigned you to the {{JOB_NAME}} job. You can now view this job and submit candidates to it via the agency portal.

    \n\n

    \n Log in here to get started: app.greenhouse.io\n

    \n\n

    \n The Greenhouse Team
    \n app.greenhouse.io\n

    \n\n

    \n \u00a9 2020 Greenhouse / 18 West 18th Street, 11th Floor, New York, NY 10011, USA\n

    ", "user": null}, "emitted_at": 1681936906596} -{"stream": "email_templates", "data": {"id": 4071602003, "name": "Default GDPR Consent Extension", "default": true, "updated_at": "2020-11-18T14:09:07.661Z", "created_at": "2020-11-18T14:09:07.661Z", "description": "This template is used when requesting consent to extend data retention", "type": "gdpr_consent_extension", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Use this template to draft language that will automatically be sent to candidates whose data you would like to keep for an additional retention period. Consider consulting with your legal department on this copy.

    ", "user": null}, "emitted_at": 1681936906598} -{"stream": "email_templates", "data": {"id": 4071601003, "name": "Default Request Interview with Calendly", "default": true, "updated_at": "2020-11-18T14:09:07.655Z", "created_at": "2020-11-18T14:09:07.655Z", "description": "This template is used when sending a Calendly link to a candidate.", "type": "calendly_request", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    \n {{CANDIDATE_FIRST_NAME}},\n

    \n\n

    \n Thanks for your interest in the {{JOB_NAME}} position at {{COMPANY}}. We're excited to move forward with\n the interview process.\n

    \n\n

    \n To help us schedule your next interview(s), please select a time through the Calendly link below.\n

    \n\n

    \n Regards,\n
    \n {{MY_FIRST_NAME}}\n

    ", "user": null}, "emitted_at": 1681936906600} -{"stream": "email_templates", "data": {"id": 4071600003, "name": "Default Rejection Notification For Followers", "default": true, "updated_at": "2020-11-18T14:09:07.649Z", "created_at": "2020-11-18T14:09:07.649Z", "description": "This template is sent to all followers of a candidate when they are marked as rejected.", "type": "rejection_for_followers", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    This is an automated message to alert you to a status change of {{CANDIDATE_NAME}},\na candidate whom you are following in Greenhouse. Please know that the hiring team\nhas decided not to move forward with this candidate's application for {{JOB_NAME}}.\nPlease be mindful that the candidate may not have been notified yet or could still\nbe active on other roles. We recommend reaching out to the hiring team with any\nquestions.

    ", "user": null}, "emitted_at": 1681936906602} -{"stream": "email_templates", "data": {"id": 4071599003, "name": "Default Stage Change Notification For Followers", "default": true, "updated_at": "2020-11-18T14:09:07.643Z", "created_at": "2020-11-18T14:09:07.643Z", "description": "This template is sent to all followers of a candidate when they are moved to a different stage in the\ninterview process.\n", "type": "stage_change_for_followers", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_NAME}}, whom you are following in Greenhouse, was moved into {{STAGE_NAME}}\nfor {{JOB_NAME}} by {{ACTING_USER}}.

    ", "user": null}, "emitted_at": 1681936906604} -{"stream": "email_templates", "data": {"id": 4071598003, "name": "Default Event Prospect Auto Reply", "default": true, "updated_at": "2020-11-18T14:09:07.637Z", "created_at": "2020-11-18T14:09:07.637Z", "description": "This template is sent automatically to prospects after your recruiting event.", "type": "event_prospect_auto_reply", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_FIRST_NAME}},

    \n\n

    Thank you for attending {{EVENT_NAME}}. If you haven't already submitted a resume or would like to upload a high resolution version of your resume, you can do so here: {{RESUME_UPLOAD_LINK}}.

    \n\n

    Regards,
    \n{{COMPANY}}

    \n\n

    ** Please note: Do not reply to this email. This email is sent from an unattended mailbox. Replies will not be read.

    ", "user": null}, "emitted_at": 1681936906605} -{"stream": "email_templates", "data": {"id": 4071596003, "name": "Default Candidate Interview Confirmation Message", "default": true, "updated_at": "2020-11-18T14:09:07.624Z", "created_at": "2020-11-18T14:09:07.624Z", "description": "This template is used for confirming candidate interviews.", "type": "candidate_availability_confirmation", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_FIRST_NAME}},

    \n\n

    Thanks for submitting your availability for the {{JOB_NAME}} position.

    \n\n

    You're confirmed for your interview on:

    \n\n{{INTERVIEW_SCHEDULE}}\n\n

    Let us know if you have any other questions before your interview.

    \n\n

    Thanks,
    \n{{MY_FIRST_NAME}}

    ", "user": null}, "emitted_at": 1681936906607} -{"stream": "email_templates", "data": {"id": 4071595003, "name": "Default GDPR Notification", "default": true, "updated_at": "2020-11-18T14:09:07.619Z", "created_at": "2020-11-18T14:09:07.619Z", "description": "This template is used when sending a GDPR notification.", "type": "gdpr_notification", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Use this template to draft language that will automatically be sent to candidates whose data you are collecting from other sources (not the candidates themselves) to comply with Art. 14 of GDPR. For more information, please visit our help center: https://support.greenhouse.io/hc/en-us/articles/360002038211

    ", "user": null}, "emitted_at": 1681936906609} -{"stream": "email_templates", "data": {"id": 4071594003, "name": "Default Job Post Request", "default": true, "updated_at": "2020-11-18T14:09:07.613Z", "created_at": "2020-11-18T14:09:07.613Z", "description": "This template is used when sending a job post for a candidate to complete.", "type": "job_post_request", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_FIRST_NAME}},

    \n

    Thanks for your interest in the {{JOB_NAME}} position at {{COMPANY}}. Since we do not have a formal application from you on file, we have a few outstanding questions for you.

    \n\n

    Please use this link to submit additional information: {{SUBMISSION_LINK}}

    \n\n

    Regards,
    \n{{MY_FIRST_NAME}}

    ", "user": null}, "emitted_at": 1681936906611} -{"stream": "email_templates", "data": {"id": 4071593003, "name": "Default Candidate Availability Request Message", "default": true, "updated_at": "2020-11-18T14:09:07.607Z", "created_at": "2020-11-18T14:09:07.607Z", "description": "This template is used for requesting interview availability from the candidate.", "type": "candidate_availability_request", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_FIRST_NAME}},

    \n\n

    Thanks for your interest in the {{JOB_NAME}} position at {{COMPANY}}. We're excited to move forward with the interview process.

    \n\n

    To help us schedule your next interview(s), please let us know when you're available by selecting the online calendar link below.

    \n\n

    We'll coordinate with our team and confirm a time with you.

    \n\n

    Regards,
    \n{{MY_FIRST_NAME}}

    ", "user": null}, "emitted_at": 1681936906612} -{"stream": "email_templates", "data": {"id": 4071592003, "name": "Default Prospect Referral Receipt", "default": true, "updated_at": "2020-11-18T14:09:07.601Z", "created_at": "2020-11-18T14:09:07.601Z", "description": "This template is used for letting users know their referral has been received.", "type": "prospect_referral_receipt", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_NAME}} has been submitted as a prospect.

    \n

    {{DUPLICATE_WARNING}}

    \n

    Click here to see all your referrals: {{MY_REFERRALS_URL}}

    ", "user": null}, "emitted_at": 1681936906614} -{"stream": "email_templates", "data": {"id": 4071591003, "name": "Default Candidate Referral Receipt", "default": true, "updated_at": "2020-11-18T14:09:07.595Z", "created_at": "2020-11-18T14:09:07.595Z", "description": "This template is used for letting users know their referral has been received.", "type": "candidate_referral_receipt", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_NAME}} has been submitted to {{JOB_NAME}}.

    \n

    \nRecruiter: {{RECRUITER}}
    \nCoordinator: {{COORDINATOR}}\n

    \n

    {{DUPLICATE_WARNING}}

    \n

    Click here to see all your referrals: {{MY_REFERRALS_URL}}

    ", "user": null}, "emitted_at": 1681936906616} -{"stream": "email_templates", "data": {"id": 4071590003, "name": "Default Non Admin Welcome Message", "default": true, "updated_at": "2020-11-18T14:09:07.589Z", "created_at": "2020-11-18T14:09:07.589Z", "description": "This template is used for inviting new users to Greenhouse.", "type": "non_admin_welcome", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Hi,

    \n

    {{MY_FULL_NAME}} has invited you to join Greenhouse!

    \n\n

    \n Greenhouse is the new recruiting software that {{COMPANY}} is using to be great at hiring.\n\n Specifically, we'll help prepare you for your interviews and make sure they're quick and easy.\n

    \n\n

    Click here to get started: {{INVITATION_LINK}}

    \n\n

    \n The Greenhouse Team
    \n app.greenhouse.io\n

    \n\n

    \n Questions? Email us at support@greenhouse.io
    \n \u00a9 2020 Greenhouse / 18 West 18th Street, 11th Floor, New York, NY 10011, USA\n

    ", "user": null}, "emitted_at": 1681936906618} -{"stream": "email_templates", "data": {"id": 4071589003, "name": "Default Job Admin Welcome Message", "default": true, "updated_at": "2020-11-18T14:09:07.582Z", "created_at": "2020-11-18T14:09:07.582Z", "description": "This template is used for inviting new users to Greenhouse.", "type": "job_admin_welcome", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Hi,

    \n

    {{MY_FULL_NAME}} has invited you to join Greenhouse!

    \n\n

    \n Greenhouse is the new recruiting software that {{COMPANY}} is using to be great at hiring.\n\n Namely, we'll help you find and hire better people more quickly.\n

    \n\n

    Click here to get started: {{INVITATION_LINK}}

    \n\n

    \n The Greenhouse Team
    \n app.greenhouse.io\n

    \n\n

    \n Questions? Email us at support@greenhouse.io
    \n \u00a9 2020 Greenhouse / 18 West 18th Street, 11th Floor, New York, NY 10011, USA\n

    ", "user": null}, "emitted_at": 1681936906620} -{"stream": "email_templates", "data": {"id": 4071588003, "name": "Default Site Admin Welcome Message", "default": true, "updated_at": "2020-11-18T14:09:07.576Z", "created_at": "2020-11-18T14:09:07.576Z", "description": "This template is used for inviting new users to Greenhouse.", "type": "site_admin_welcome", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Hi,

    \n

    {{MY_FULL_NAME}} has invited you to join Greenhouse!

    \n\n

    \n Greenhouse is the new recruiting software that {{COMPANY}} is using to be great at hiring.\n\n We'll make it easy for you to build and execute a world-class hiring plan.\n

    \n\n

    Click here to get started: {{INVITATION_LINK}}

    \n\n

    Sincerely,

    \n\n

    \n The Greenhouse Team
    \n app.greenhouse.io\n

    \n\n

    \n Questions? Email us at support@greenhouse.io
    \n \u00a9 2020 Greenhouse / 18 West 18th Street, 11th Floor, New York, NY 10011, USA\n

    ", "user": null}, "emitted_at": 1681936906622} -{"stream": "email_templates", "data": {"id": 4071587003, "name": "Default Interviewer Invite", "default": true, "updated_at": "2020-11-18T14:09:07.570Z", "created_at": "2020-11-18T14:09:07.570Z", "description": "This template is sent to interviewers when you schedule an interview.", "type": "interviewer_invite", "from": null, "cc": [], "body": "Please interview {{CANDIDATE_NAME}}.\n\nInterview type: {{INTERVIEW_NAME}}\n\nHere is a link to the interview kit, where you can view instructions and a scorecard:\n\n{{INTERVIEW_KIT_LINK}}", "html_body": null, "user": null}, "emitted_at": 1681936906624} -{"stream": "email_templates", "data": {"id": 4071586003, "name": "Default Scorecard Reminder", "default": true, "updated_at": "2020-11-18T14:09:07.563Z", "created_at": "2020-11-18T14:09:07.563Z", "description": "This template is sent to users when they have a scorecard due.", "type": "scorecard_reminder", "from": null, "cc": [], "body": null, "html_body": "

    Hello,

    \n\n

    It's been {{TIME_SINCE_INTERVIEWED}} since you completed your interview with {{CANDIDATE_NAME}}.

    \n\n

    We know that feedback quality gets worse the longer you wait to submit. In fact, one hour after the event, memory recall declines to 44%.

    \n\n

    Submit your feedback now to help us make the right hiring decision \u2013 {{SCORECARD_LINK}}

    \n\n

    Your feedback is important! Thanks for your help.

    ", "user": null}, "emitted_at": 1681936906625} -{"stream": "email_templates", "data": {"id": 4071585003, "name": "Default Candidate Rejection", "default": true, "updated_at": "2020-11-18T14:09:07.557Z", "created_at": "2020-11-18T14:09:07.557Z", "description": "This template is sent to candidates when you reject them from a job. You will always have the ability to preview, edit and customize this email before sending, or choose not to send it at all.", "type": "candidate_rejection", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_FIRST_NAME}},

    \n\n

    Unfortunately, we have decided not to proceed with your candidacy for the current opening at {{COMPANY}}.

    \n\n

    We received many qualified applicants and have decided to move ahead with another candidate who we feel is a better match for this particular position.

    \n\n

    Thanks again for your interest in {{COMPANY}} and we wish you luck in your search.

    \n\n

    Regards,
    \n{{COMPANY}}

    ", "user": null}, "emitted_at": 1681936906627} -{"stream": "email_templates", "data": {"id": 4071584003, "name": "Default Candidate Auto Reply", "default": true, "updated_at": "2020-11-18T14:09:07.549Z", "created_at": "2020-11-18T14:09:07.549Z", "description": "This template is sent automatically to candidates when they complete your online job application, letting them know that you've received their application.", "type": "candidate_auto_reply", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    {{CANDIDATE_FIRST_NAME}},

    \n\n

    Thanks for applying to {{COMPANY}}. Your application has been received and we will review it right away.

    \n\n

    If your application seems like a good fit for the position we will contact you soon.

    \n\n

    Regards,
    \n{{COMPANY}}

    \n\n

    ** Please note: Do not reply to this email. This email is sent from an unattended mailbox. Replies will not be read.

    ", "user": null}, "emitted_at": 1681936906629} -{"stream": "user_roles", "data": {"id": 4011623003, "type": "job_admin", "name": "Private"}, "emitted_at": 1681937618735} -{"stream": "user_roles", "data": {"id": 4011622003, "type": "job_admin", "name": "Standard"}, "emitted_at": 1681937618737} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}, "emitted_at": 1691572561703} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}, "emitted_at": 1691572561707} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}, "emitted_at": 1691572561710} +{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.248Z", "id": 9308815003, "free_form_text": null, "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004262003, "created_at": "2021-11-03T19:56:07.248Z", "application_id": 47459993003}, "emitted_at": 1691572563683} +{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.252Z", "id": 9308816003, "free_form_text": "custom answer", "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004263003, "created_at": "2021-11-03T19:56:07.252Z", "application_id": 47459993003}, "emitted_at": 1691572563685} +{"stream": "applications_demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.259Z", "id": 9308817003, "free_form_text": null, "demographic_question_id": 4000717003, "demographic_answer_option_id": 4004266003, "created_at": "2021-11-03T19:56:07.259Z", "application_id": 47459993003}, "emitted_at": 1691572563686} +{"stream": "applications_interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572565721} +{"stream": "applications_interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572565725} +{"stream": "applications_interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572565728} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:24:37.050Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test", "last_activity": "2020-11-24T23:24:37.049Z", "is_private": false, "id": 17130511003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:24:37.018Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}], "application_ids": [19214950003], "addresses": []}, "emitted_at": 1691572566540} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:25:13.806Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test2", "last_activity": "2020-11-24T23:25:13.804Z", "is_private": false, "id": 17130554003, "first_name": "Test2", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:25:13.777Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}], "application_ids": [19214993003], "addresses": []}, "emitted_at": 1691572566544} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:28:19.781Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Lastname", "last_activity": "2020-11-24T23:28:19.779Z", "is_private": false, "id": 17130732003, "first_name": "Name", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:28:19.710Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}], "application_ids": [19215172003], "addresses": []}, "emitted_at": 1691572566548} +{"stream": "close_reasons", "data": {"id": 4010635003, "name": "Not Filling"}, "emitted_at": 1691572567002} +{"stream": "close_reasons", "data": {"id": 4010634003, "name": "On Hold"}, "emitted_at": 1691572567004} +{"stream": "close_reasons", "data": {"id": 4010633003, "name": "Hire - New Headcount"}, "emitted_at": 1691572567006} +{"stream": "custom_fields", "data": {"id": 4680898003, "name": "School Name", "active": true, "field_type": "candidate", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "school_name", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845822003, "name": "Abraham Baldwin Agricultural College", "priority": 0, "external_id": null}, {"id": 10845823003, "name": "Academy of Art University", "priority": 1, "external_id": null}, {"id": 10845824003, "name": "Acadia University", "priority": 2, "external_id": null}, {"id": 10845825003, "name": "Adams State University", "priority": 3, "external_id": null}, {"id": 10845826003, "name": "Adelphi University", "priority": 4, "external_id": null}, {"id": 10845827003, "name": "Adrian College", "priority": 5, "external_id": null}, {"id": 10845828003, "name": "Adventist University of Health Sciences", "priority": 6, "external_id": null}, {"id": 10845829003, "name": "Agnes Scott College", "priority": 7, "external_id": null}, {"id": 10845830003, "name": "AIB College of Business", "priority": 8, "external_id": null}, {"id": 10845831003, "name": "Alaska Pacific University", "priority": 9, "external_id": null}, {"id": 10845832003, "name": "Albany College of Pharmacy and Health Sciences", "priority": 10, "external_id": null}, {"id": 10845833003, "name": "Albany State University", "priority": 11, "external_id": null}, {"id": 10845834003, "name": "Albertus Magnus College", "priority": 12, "external_id": null}, {"id": 10845835003, "name": "Albion College", "priority": 13, "external_id": null}, {"id": 10845836003, "name": "Albright College", "priority": 14, "external_id": null}, {"id": 10845837003, "name": "Alderson Broaddus University", "priority": 15, "external_id": null}, {"id": 10845838003, "name": "Alfred University", "priority": 16, "external_id": null}, {"id": 10845839003, "name": "Alice Lloyd College", "priority": 17, "external_id": null}, {"id": 10845840003, "name": "Allegheny College", "priority": 18, "external_id": null}, {"id": 10845841003, "name": "Allen College", "priority": 19, "external_id": null}, {"id": 10845842003, "name": "Allen University", "priority": 20, "external_id": null}, {"id": 10845843003, "name": "Alliant International University", "priority": 21, "external_id": null}, {"id": 10845844003, "name": "Alma College", "priority": 22, "external_id": null}, {"id": 10845845003, "name": "Alvernia University", "priority": 23, "external_id": null}, {"id": 10845846003, "name": "Alverno College", "priority": 24, "external_id": null}, {"id": 10845847003, "name": "Amberton University", "priority": 25, "external_id": null}, {"id": 10845848003, "name": "American Academy of Art", "priority": 26, "external_id": null}, {"id": 10845849003, "name": "American Indian College of the Assemblies of God", "priority": 27, "external_id": null}, {"id": 10845850003, "name": "American InterContinental University", "priority": 28, "external_id": null}, {"id": 10845851003, "name": "American International College", "priority": 29, "external_id": null}, {"id": 10845852003, "name": "American Jewish University", "priority": 30, "external_id": null}, {"id": 10845853003, "name": "American Public University System", "priority": 31, "external_id": null}, {"id": 10845854003, "name": "American University", "priority": 32, "external_id": null}, {"id": 10845855003, "name": "American University in Bulgaria", "priority": 33, "external_id": null}, {"id": 10845856003, "name": "American University in Cairo", "priority": 34, "external_id": null}, {"id": 10845857003, "name": "American University of Beirut", "priority": 35, "external_id": null}, {"id": 10845858003, "name": "American University of Paris", "priority": 36, "external_id": null}, {"id": 10845859003, "name": "American University of Puerto Rico", "priority": 37, "external_id": null}, {"id": 10845860003, "name": "Amherst College", "priority": 38, "external_id": null}, {"id": 10845861003, "name": "Amridge University", "priority": 39, "external_id": null}, {"id": 10845862003, "name": "Anderson University", "priority": 40, "external_id": null}, {"id": 10845863003, "name": "Andrews University", "priority": 41, "external_id": null}, {"id": 10845864003, "name": "Angelo State University", "priority": 42, "external_id": null}, {"id": 10845865003, "name": "Anna Maria College", "priority": 43, "external_id": null}, {"id": 10845866003, "name": "Antioch University", "priority": 44, "external_id": null}, {"id": 10845867003, "name": "Appalachian Bible College", "priority": 45, "external_id": null}, {"id": 10845868003, "name": "Aquinas College", "priority": 46, "external_id": null}, {"id": 10845869003, "name": "Arcadia University", "priority": 47, "external_id": null}, {"id": 10845870003, "name": "Argosy University", "priority": 48, "external_id": null}, {"id": 10845871003, "name": "Arizona Christian University", "priority": 49, "external_id": null}, {"id": 10845872003, "name": "Arizona State University - West", "priority": 50, "external_id": null}, {"id": 10845873003, "name": "Arkansas Baptist College", "priority": 51, "external_id": null}, {"id": 10845874003, "name": "Arkansas Tech University", "priority": 52, "external_id": null}, {"id": 10845875003, "name": "Armstrong Atlantic State University", "priority": 53, "external_id": null}, {"id": 10845876003, "name": "Art Academy of Cincinnati", "priority": 54, "external_id": null}, {"id": 10845877003, "name": "Art Center College of Design", "priority": 55, "external_id": null}, {"id": 10845878003, "name": "Art Institute of Atlanta", "priority": 56, "external_id": null}, {"id": 10845879003, "name": "Art Institute of Colorado", "priority": 57, "external_id": null}, {"id": 10845880003, "name": "Art Institute of Houston", "priority": 58, "external_id": null}, {"id": 10845881003, "name": "Art Institute of Pittsburgh", "priority": 59, "external_id": null}, {"id": 10845882003, "name": "Art Institute of Portland", "priority": 60, "external_id": null}, {"id": 10845883003, "name": "Art Institute of Seattle", "priority": 61, "external_id": null}, {"id": 10845884003, "name": "Asbury University", "priority": 62, "external_id": null}, {"id": 10845885003, "name": "Ashford University", "priority": 63, "external_id": null}, {"id": 10845886003, "name": "Ashland University", "priority": 64, "external_id": null}, {"id": 10845887003, "name": "Assumption College", "priority": 65, "external_id": null}, {"id": 10845888003, "name": "Athens State University", "priority": 66, "external_id": null}, {"id": 10845889003, "name": "Auburn University - Montgomery", "priority": 67, "external_id": null}, {"id": 10845890003, "name": "Augsburg College", "priority": 68, "external_id": null}, {"id": 10845891003, "name": "Augustana College", "priority": 69, "external_id": null}, {"id": 10845892003, "name": "Aurora University", "priority": 70, "external_id": null}, {"id": 10845893003, "name": "Austin College", "priority": 71, "external_id": null}, {"id": 10845894003, "name": "Alcorn State University", "priority": 72, "external_id": null}, {"id": 10845895003, "name": "Ave Maria University", "priority": 73, "external_id": null}, {"id": 10845896003, "name": "Averett University", "priority": 74, "external_id": null}, {"id": 10845897003, "name": "Avila University", "priority": 75, "external_id": null}, {"id": 10845898003, "name": "Azusa Pacific University", "priority": 76, "external_id": null}, {"id": 10845899003, "name": "Babson College", "priority": 77, "external_id": null}, {"id": 10845900003, "name": "Bacone College", "priority": 78, "external_id": null}, {"id": 10845901003, "name": "Baker College of Flint", "priority": 79, "external_id": null}, {"id": 10845902003, "name": "Baker University", "priority": 80, "external_id": null}, {"id": 10845903003, "name": "Baldwin Wallace University", "priority": 81, "external_id": null}, {"id": 10845904003, "name": "Christian Brothers University", "priority": 82, "external_id": null}, {"id": 10845905003, "name": "Abilene Christian University", "priority": 83, "external_id": null}, {"id": 10845906003, "name": "Arizona State University", "priority": 84, "external_id": null}, {"id": 10845907003, "name": "Auburn University", "priority": 85, "external_id": null}, {"id": 10845908003, "name": "Alabama A&M University", "priority": 86, "external_id": null}, {"id": 10845909003, "name": "Alabama State University", "priority": 87, "external_id": null}, {"id": 10845910003, "name": "Arkansas State University", "priority": 88, "external_id": null}, {"id": 10845911003, "name": "Baptist Bible College", "priority": 89, "external_id": null}, {"id": 10845912003, "name": "Baptist Bible College and Seminary", "priority": 90, "external_id": null}, {"id": 10845913003, "name": "Baptist College of Florida", "priority": 91, "external_id": null}, {"id": 10845914003, "name": "Baptist Memorial College of Health Sciences", "priority": 92, "external_id": null}, {"id": 10845915003, "name": "Baptist Missionary Association Theological Seminary", "priority": 93, "external_id": null}, {"id": 10845916003, "name": "Bard College", "priority": 94, "external_id": null}, {"id": 10845917003, "name": "Bard College at Simon's Rock", "priority": 95, "external_id": null}, {"id": 10845918003, "name": "Barnard College", "priority": 96, "external_id": null}, {"id": 10845919003, "name": "Barry University", "priority": 97, "external_id": null}, {"id": 10845920003, "name": "Barton College", "priority": 98, "external_id": null}, {"id": 10845921003, "name": "Bastyr University", "priority": 99, "external_id": null}, {"id": 10845922003, "name": "Bates College", "priority": 100, "external_id": null}, {"id": 10845923003, "name": "Bauder College", "priority": 101, "external_id": null}, {"id": 10845924003, "name": "Bay Path College", "priority": 102, "external_id": null}, {"id": 10845925003, "name": "Bay State College", "priority": 103, "external_id": null}, {"id": 10845926003, "name": "Bayamon Central University", "priority": 104, "external_id": null}, {"id": 10845927003, "name": "Beacon College", "priority": 105, "external_id": null}, {"id": 10845928003, "name": "Becker College", "priority": 106, "external_id": null}, {"id": 10845929003, "name": "Belhaven University", "priority": 107, "external_id": null}, {"id": 10845930003, "name": "Bellarmine University", "priority": 108, "external_id": null}, {"id": 10845931003, "name": "Bellevue College", "priority": 109, "external_id": null}, {"id": 10845932003, "name": "Bellevue University", "priority": 110, "external_id": null}, {"id": 10845933003, "name": "Bellin College", "priority": 111, "external_id": null}, {"id": 10845934003, "name": "Belmont Abbey College", "priority": 112, "external_id": null}, {"id": 10845935003, "name": "Belmont University", "priority": 113, "external_id": null}, {"id": 10845936003, "name": "Beloit College", "priority": 114, "external_id": null}, {"id": 10845937003, "name": "Bemidji State University", "priority": 115, "external_id": null}, {"id": 10845938003, "name": "Benedict College", "priority": 116, "external_id": null}, {"id": 10845939003, "name": "Benedictine College", "priority": 117, "external_id": null}, {"id": 10845940003, "name": "Benedictine University", "priority": 118, "external_id": null}, {"id": 10845941003, "name": "Benjamin Franklin Institute of Technology", "priority": 119, "external_id": null}, {"id": 10845942003, "name": "Bennett College", "priority": 120, "external_id": null}, {"id": 10845943003, "name": "Bennington College", "priority": 121, "external_id": null}, {"id": 10845944003, "name": "Bentley University", "priority": 122, "external_id": null}, {"id": 10845945003, "name": "Berea College", "priority": 123, "external_id": null}, {"id": 10845946003, "name": "Berkeley College", "priority": 124, "external_id": null}, {"id": 10845947003, "name": "Berklee College of Music", "priority": 125, "external_id": null}, {"id": 10845948003, "name": "Berry College", "priority": 126, "external_id": null}, {"id": 10845949003, "name": "Bethany College", "priority": 127, "external_id": null}, {"id": 10845950003, "name": "Bethany Lutheran College", "priority": 128, "external_id": null}, {"id": 10845951003, "name": "Bethel College", "priority": 129, "external_id": null}, {"id": 10845952003, "name": "Bethel University", "priority": 130, "external_id": null}, {"id": 10845953003, "name": "BI Norwegian Business School", "priority": 131, "external_id": null}, {"id": 10845954003, "name": "Binghamton University - SUNY", "priority": 132, "external_id": null}, {"id": 10845955003, "name": "Biola University", "priority": 133, "external_id": null}, {"id": 10845956003, "name": "Birmingham-Southern College", "priority": 134, "external_id": null}, {"id": 10845957003, "name": "Bismarck State College", "priority": 135, "external_id": null}, {"id": 10845958003, "name": "Black Hills State University", "priority": 136, "external_id": null}, {"id": 10845959003, "name": "Blackburn College", "priority": 137, "external_id": null}, {"id": 10845960003, "name": "Blessing-Rieman College of Nursing", "priority": 138, "external_id": null}, {"id": 10845961003, "name": "Bloomfield College", "priority": 139, "external_id": null}, {"id": 10845962003, "name": "Bloomsburg University of Pennsylvania", "priority": 140, "external_id": null}, {"id": 10845963003, "name": "Blue Mountain College", "priority": 141, "external_id": null}, {"id": 10845964003, "name": "Bluefield College", "priority": 142, "external_id": null}, {"id": 10845965003, "name": "Bluefield State College", "priority": 143, "external_id": null}, {"id": 10845966003, "name": "Bluffton University", "priority": 144, "external_id": null}, {"id": 10845967003, "name": "Boricua College", "priority": 145, "external_id": null}, {"id": 10845968003, "name": "Boston Architectural College", "priority": 146, "external_id": null}, {"id": 10845969003, "name": "Boston Conservatory", "priority": 147, "external_id": null}, {"id": 10845970003, "name": "Boston University", "priority": 148, "external_id": null}, {"id": 10845971003, "name": "Bowdoin College", "priority": 149, "external_id": null}, {"id": 10845972003, "name": "Bowie State University", "priority": 150, "external_id": null}, {"id": 10845973003, "name": "Bradley University", "priority": 151, "external_id": null}, {"id": 10845974003, "name": "Brandeis University", "priority": 152, "external_id": null}, {"id": 10845975003, "name": "Brandman University", "priority": 153, "external_id": null}, {"id": 10845976003, "name": "Brazosport College", "priority": 154, "external_id": null}, {"id": 10845977003, "name": "Brenau University", "priority": 155, "external_id": null}, {"id": 10845978003, "name": "Brescia University", "priority": 156, "external_id": null}, {"id": 10845979003, "name": "Brevard College", "priority": 157, "external_id": null}, {"id": 10845980003, "name": "Brewton-Parker College", "priority": 158, "external_id": null}, {"id": 10845981003, "name": "Briar Cliff University", "priority": 159, "external_id": null}, {"id": 10845982003, "name": "Briarcliffe College", "priority": 160, "external_id": null}, {"id": 10845983003, "name": "Bridgewater College", "priority": 161, "external_id": null}, {"id": 10845984003, "name": "Bridgewater State University", "priority": 162, "external_id": null}, {"id": 10845985003, "name": "Brigham Young University - Hawaii", "priority": 163, "external_id": null}, {"id": 10845986003, "name": "Brigham Young University - Idaho", "priority": 164, "external_id": null}, {"id": 10845987003, "name": "Brock University", "priority": 165, "external_id": null}, {"id": 10845988003, "name": "Bryan College", "priority": 166, "external_id": null}, {"id": 10845989003, "name": "Bryn Athyn College of the New Church", "priority": 167, "external_id": null}, {"id": 10845990003, "name": "Bryn Mawr College", "priority": 168, "external_id": null}, {"id": 10845991003, "name": "Boston College", "priority": 169, "external_id": null}, {"id": 10845992003, "name": "Buena Vista University", "priority": 170, "external_id": null}, {"id": 10845993003, "name": "Burlington College", "priority": 171, "external_id": null}, {"id": 10845994003, "name": "Bowling Green State University", "priority": 172, "external_id": null}, {"id": 10845995003, "name": "Brown University", "priority": 173, "external_id": null}, {"id": 10845996003, "name": "Appalachian State University", "priority": 174, "external_id": null}, {"id": 10845997003, "name": "Brigham Young University - Provo", "priority": 175, "external_id": null}, {"id": 10845998003, "name": "Boise State University", "priority": 176, "external_id": null}, {"id": 10845999003, "name": "Bethune-Cookman University", "priority": 177, "external_id": null}, {"id": 10846000003, "name": "Bryant University", "priority": 178, "external_id": null}, {"id": 10846001003, "name": "Cabarrus College of Health Sciences", "priority": 179, "external_id": null}, {"id": 10846002003, "name": "Cabrini College", "priority": 180, "external_id": null}, {"id": 10846003003, "name": "Cairn University", "priority": 181, "external_id": null}, {"id": 10846004003, "name": "Caldwell College", "priority": 182, "external_id": null}, {"id": 10846005003, "name": "California Baptist University", "priority": 183, "external_id": null}, {"id": 10846006003, "name": "California College of the Arts", "priority": 184, "external_id": null}, {"id": 10846007003, "name": "California Institute of Integral Studies", "priority": 185, "external_id": null}, {"id": 10846008003, "name": "California Institute of Technology", "priority": 186, "external_id": null}, {"id": 10846009003, "name": "California Institute of the Arts", "priority": 187, "external_id": null}, {"id": 10846010003, "name": "California Lutheran University", "priority": 188, "external_id": null}, {"id": 10846011003, "name": "California Maritime Academy", "priority": 189, "external_id": null}, {"id": 10846012003, "name": "California State Polytechnic University - Pomona", "priority": 190, "external_id": null}, {"id": 10846013003, "name": "California State University - Bakersfield", "priority": 191, "external_id": null}, {"id": 10846014003, "name": "California State University - Channel Islands", "priority": 192, "external_id": null}, {"id": 10846015003, "name": "California State University - Chico", "priority": 193, "external_id": null}, {"id": 10846016003, "name": "California State University - Dominguez Hills", "priority": 194, "external_id": null}, {"id": 10846017003, "name": "California State University - East Bay", "priority": 195, "external_id": null}, {"id": 10846018003, "name": "California State University - Fullerton", "priority": 196, "external_id": null}, {"id": 10846019003, "name": "California State University - Los Angeles", "priority": 197, "external_id": null}, {"id": 10846020003, "name": "California State University - Monterey Bay", "priority": 198, "external_id": null}, {"id": 10846021003, "name": "California State University - Northridge", "priority": 199, "external_id": null}, {"id": 10846022003, "name": "California State University - San Bernardino", "priority": 200, "external_id": null}, {"id": 10846023003, "name": "California State University - San Marcos", "priority": 201, "external_id": null}, {"id": 10846024003, "name": "California State University - Stanislaus", "priority": 202, "external_id": null}, {"id": 10846025003, "name": "California University of Pennsylvania", "priority": 203, "external_id": null}, {"id": 10846026003, "name": "Calumet College of St. Joseph", "priority": 204, "external_id": null}, {"id": 10846027003, "name": "Calvary Bible College and Theological Seminary", "priority": 205, "external_id": null}, {"id": 10846028003, "name": "Calvin College", "priority": 206, "external_id": null}, {"id": 10846029003, "name": "Cambridge College", "priority": 207, "external_id": null}, {"id": 10846030003, "name": "Cameron University", "priority": 208, "external_id": null}, {"id": 10846031003, "name": "Campbellsville University", "priority": 209, "external_id": null}, {"id": 10846032003, "name": "Canisius College", "priority": 210, "external_id": null}, {"id": 10846033003, "name": "Capella University", "priority": 211, "external_id": null}, {"id": 10846034003, "name": "Capital University", "priority": 212, "external_id": null}, {"id": 10846035003, "name": "Capitol College", "priority": 213, "external_id": null}, {"id": 10846036003, "name": "Cardinal Stritch University", "priority": 214, "external_id": null}, {"id": 10846037003, "name": "Caribbean University", "priority": 215, "external_id": null}, {"id": 10846038003, "name": "Carleton College", "priority": 216, "external_id": null}, {"id": 10846039003, "name": "Carleton University", "priority": 217, "external_id": null}, {"id": 10846040003, "name": "Carlos Albizu University", "priority": 218, "external_id": null}, {"id": 10846041003, "name": "Carlow University", "priority": 219, "external_id": null}, {"id": 10846042003, "name": "Carnegie Mellon University", "priority": 220, "external_id": null}, {"id": 10846043003, "name": "Carroll College", "priority": 221, "external_id": null}, {"id": 10846044003, "name": "Carroll University", "priority": 222, "external_id": null}, {"id": 10846045003, "name": "Carson-Newman University", "priority": 223, "external_id": null}, {"id": 10846046003, "name": "Carthage College", "priority": 224, "external_id": null}, {"id": 10846047003, "name": "Case Western Reserve University", "priority": 225, "external_id": null}, {"id": 10846048003, "name": "Castleton State College", "priority": 226, "external_id": null}, {"id": 10846049003, "name": "Catawba College", "priority": 227, "external_id": null}, {"id": 10846050003, "name": "Cazenovia College", "priority": 228, "external_id": null}, {"id": 10846051003, "name": "Cedar Crest College", "priority": 229, "external_id": null}, {"id": 10846052003, "name": "Cedarville University", "priority": 230, "external_id": null}, {"id": 10846053003, "name": "Centenary College", "priority": 231, "external_id": null}, {"id": 10846054003, "name": "Centenary College of Louisiana", "priority": 232, "external_id": null}, {"id": 10846055003, "name": "Central Baptist College", "priority": 233, "external_id": null}, {"id": 10846056003, "name": "Central Bible College", "priority": 234, "external_id": null}, {"id": 10846057003, "name": "Central Christian College", "priority": 235, "external_id": null}, {"id": 10846058003, "name": "Central College", "priority": 236, "external_id": null}, {"id": 10846059003, "name": "Central Methodist University", "priority": 237, "external_id": null}, {"id": 10846060003, "name": "Central Penn College", "priority": 238, "external_id": null}, {"id": 10846061003, "name": "Central State University", "priority": 239, "external_id": null}, {"id": 10846062003, "name": "Central Washington University", "priority": 240, "external_id": null}, {"id": 10846063003, "name": "Centre College", "priority": 241, "external_id": null}, {"id": 10846064003, "name": "Chadron State College", "priority": 242, "external_id": null}, {"id": 10846065003, "name": "Chamberlain College of Nursing", "priority": 243, "external_id": null}, {"id": 10846066003, "name": "Chaminade University of Honolulu", "priority": 244, "external_id": null}, {"id": 10846067003, "name": "Champlain College", "priority": 245, "external_id": null}, {"id": 10846068003, "name": "Chancellor University", "priority": 246, "external_id": null}, {"id": 10846069003, "name": "Chapman University", "priority": 247, "external_id": null}, {"id": 10846070003, "name": "Charles R. Drew University of Medicine and Science", "priority": 248, "external_id": null}, {"id": 10846071003, "name": "Charter Oak State College", "priority": 249, "external_id": null}, {"id": 10846072003, "name": "Chatham University", "priority": 250, "external_id": null}, {"id": 10846073003, "name": "Chestnut Hill College", "priority": 251, "external_id": null}, {"id": 10846074003, "name": "Cheyney University of Pennsylvania", "priority": 252, "external_id": null}, {"id": 10846075003, "name": "Chicago State University", "priority": 253, "external_id": null}, {"id": 10846076003, "name": "Chipola College", "priority": 254, "external_id": null}, {"id": 10846077003, "name": "Chowan University", "priority": 255, "external_id": null}, {"id": 10846078003, "name": "Christendom College", "priority": 256, "external_id": null}, {"id": 10846079003, "name": "Baylor University", "priority": 257, "external_id": null}, {"id": 10846080003, "name": "Central Connecticut State University", "priority": 258, "external_id": null}, {"id": 10846081003, "name": "Central Michigan University", "priority": 259, "external_id": null}, {"id": 10846082003, "name": "Charleston Southern University", "priority": 260, "external_id": null}, {"id": 10846083003, "name": "California State University - Sacramento", "priority": 261, "external_id": null}, {"id": 10846084003, "name": "California State University - Fresno", "priority": 262, "external_id": null}, {"id": 10846085003, "name": "Campbell University", "priority": 263, "external_id": null}, {"id": 10846086003, "name": "Christopher Newport University", "priority": 264, "external_id": null}, {"id": 10846087003, "name": "Cincinnati Christian University", "priority": 265, "external_id": null}, {"id": 10846088003, "name": "Cincinnati College of Mortuary Science", "priority": 266, "external_id": null}, {"id": 10846089003, "name": "City University of Seattle", "priority": 267, "external_id": null}, {"id": 10846090003, "name": "Claflin University", "priority": 268, "external_id": null}, {"id": 10846091003, "name": "Claremont McKenna College", "priority": 269, "external_id": null}, {"id": 10846092003, "name": "Clarion University of Pennsylvania", "priority": 270, "external_id": null}, {"id": 10846093003, "name": "Clark Atlanta University", "priority": 271, "external_id": null}, {"id": 10846094003, "name": "Clark University", "priority": 272, "external_id": null}, {"id": 10846095003, "name": "Clarke University", "priority": 273, "external_id": null}, {"id": 10846096003, "name": "Clarkson College", "priority": 274, "external_id": null}, {"id": 10846097003, "name": "Clarkson University", "priority": 275, "external_id": null}, {"id": 10846098003, "name": "Clayton State University", "priority": 276, "external_id": null}, {"id": 10846099003, "name": "Clear Creek Baptist Bible College", "priority": 277, "external_id": null}, {"id": 10846100003, "name": "Clearwater Christian College", "priority": 278, "external_id": null}, {"id": 10846101003, "name": "Cleary University", "priority": 279, "external_id": null}, {"id": 10846102003, "name": "College of William and Mary", "priority": 280, "external_id": null}, {"id": 10846103003, "name": "Cleveland Chiropractic College", "priority": 281, "external_id": null}, {"id": 10846104003, "name": "Cleveland Institute of Art", "priority": 282, "external_id": null}, {"id": 10846105003, "name": "Cleveland Institute of Music", "priority": 283, "external_id": null}, {"id": 10846106003, "name": "Cleveland State University", "priority": 284, "external_id": null}, {"id": 10846107003, "name": "Coe College", "priority": 285, "external_id": null}, {"id": 10846108003, "name": "Cogswell Polytechnical College", "priority": 286, "external_id": null}, {"id": 10846109003, "name": "Coker College", "priority": 287, "external_id": null}, {"id": 10846110003, "name": "Colby College", "priority": 288, "external_id": null}, {"id": 10846111003, "name": "Colby-Sawyer College", "priority": 289, "external_id": null}, {"id": 10846112003, "name": "College at Brockport - SUNY", "priority": 290, "external_id": null}, {"id": 10846113003, "name": "College for Creative Studies", "priority": 291, "external_id": null}, {"id": 10846114003, "name": "College of Charleston", "priority": 292, "external_id": null}, {"id": 10846115003, "name": "College of Idaho", "priority": 293, "external_id": null}, {"id": 10846116003, "name": "College of Mount St. Joseph", "priority": 294, "external_id": null}, {"id": 10846117003, "name": "College of Mount St. Vincent", "priority": 295, "external_id": null}, {"id": 10846118003, "name": "College of New Jersey", "priority": 296, "external_id": null}, {"id": 10846119003, "name": "College of New Rochelle", "priority": 297, "external_id": null}, {"id": 10846120003, "name": "College of Our Lady of the Elms", "priority": 298, "external_id": null}, {"id": 10846121003, "name": "College of Saints John Fisher & Thomas More", "priority": 299, "external_id": null}, {"id": 10846122003, "name": "College of Southern Nevada", "priority": 300, "external_id": null}, {"id": 10846123003, "name": "College of St. Benedict", "priority": 301, "external_id": null}, {"id": 10846124003, "name": "College of St. Elizabeth", "priority": 302, "external_id": null}, {"id": 10846125003, "name": "College of St. Joseph", "priority": 303, "external_id": null}, {"id": 10846126003, "name": "College of St. Mary", "priority": 304, "external_id": null}, {"id": 10846127003, "name": "College of St. Rose", "priority": 305, "external_id": null}, {"id": 10846128003, "name": "College of St. Scholastica", "priority": 306, "external_id": null}, {"id": 10846129003, "name": "College of the Atlantic", "priority": 307, "external_id": null}, {"id": 10846130003, "name": "College of the Holy Cross", "priority": 308, "external_id": null}, {"id": 10846131003, "name": "College of the Ozarks", "priority": 309, "external_id": null}, {"id": 10846132003, "name": "College of Wooster", "priority": 310, "external_id": null}, {"id": 10846133003, "name": "Colorado Christian University", "priority": 311, "external_id": null}, {"id": 10846134003, "name": "Colorado College", "priority": 312, "external_id": null}, {"id": 10846135003, "name": "Colorado Mesa University", "priority": 313, "external_id": null}, {"id": 10846136003, "name": "Colorado School of Mines", "priority": 314, "external_id": null}, {"id": 10846137003, "name": "Colorado State University - Pueblo", "priority": 315, "external_id": null}, {"id": 10846138003, "name": "Colorado Technical University", "priority": 316, "external_id": null}, {"id": 10846139003, "name": "Columbia College", "priority": 317, "external_id": null}, {"id": 10846140003, "name": "Columbia College Chicago", "priority": 318, "external_id": null}, {"id": 10846141003, "name": "Columbia College of Nursing", "priority": 319, "external_id": null}, {"id": 10846142003, "name": "Columbia International University", "priority": 320, "external_id": null}, {"id": 10846143003, "name": "Columbus College of Art and Design", "priority": 321, "external_id": null}, {"id": 10846144003, "name": "Columbus State University", "priority": 322, "external_id": null}, {"id": 10846145003, "name": "Conception Seminary College", "priority": 323, "external_id": null}, {"id": 10846146003, "name": "Concord University", "priority": 324, "external_id": null}, {"id": 10846147003, "name": "Concordia College", "priority": 325, "external_id": null}, {"id": 10846148003, "name": "Concordia College - Moorhead", "priority": 326, "external_id": null}, {"id": 10846149003, "name": "Concordia University", "priority": 327, "external_id": null}, {"id": 10846150003, "name": "Concordia University Chicago", "priority": 328, "external_id": null}, {"id": 10846151003, "name": "Concordia University Texas", "priority": 329, "external_id": null}, {"id": 10846152003, "name": "Concordia University Wisconsin", "priority": 330, "external_id": null}, {"id": 10846153003, "name": "Concordia University - St. Paul", "priority": 331, "external_id": null}, {"id": 10846154003, "name": "Connecticut College", "priority": 332, "external_id": null}, {"id": 10846155003, "name": "Converse College", "priority": 333, "external_id": null}, {"id": 10846156003, "name": "Cooper Union", "priority": 334, "external_id": null}, {"id": 10846157003, "name": "Coppin State University", "priority": 335, "external_id": null}, {"id": 10846158003, "name": "Corban University", "priority": 336, "external_id": null}, {"id": 10846159003, "name": "Corcoran College of Art and Design", "priority": 337, "external_id": null}, {"id": 10846160003, "name": "Cornell College", "priority": 338, "external_id": null}, {"id": 10846161003, "name": "Cornerstone University", "priority": 339, "external_id": null}, {"id": 10846162003, "name": "Cornish College of the Arts", "priority": 340, "external_id": null}, {"id": 10846163003, "name": "Covenant College", "priority": 341, "external_id": null}, {"id": 10846164003, "name": "Cox College", "priority": 342, "external_id": null}, {"id": 10846165003, "name": "Creighton University", "priority": 343, "external_id": null}, {"id": 10846166003, "name": "Criswell College", "priority": 344, "external_id": null}, {"id": 10846167003, "name": "Crown College", "priority": 345, "external_id": null}, {"id": 10846168003, "name": "Culinary Institute of America", "priority": 346, "external_id": null}, {"id": 10846169003, "name": "Culver-Stockton College", "priority": 347, "external_id": null}, {"id": 10846170003, "name": "Cumberland University", "priority": 348, "external_id": null}, {"id": 10846171003, "name": "Columbia University", "priority": 349, "external_id": null}, {"id": 10846172003, "name": "Cornell University", "priority": 350, "external_id": null}, {"id": 10846173003, "name": "Colorado State University", "priority": 351, "external_id": null}, {"id": 10846174003, "name": "University of Virginia", "priority": 352, "external_id": null}, {"id": 10846175003, "name": "Colgate University", "priority": 353, "external_id": null}, {"id": 10846176003, "name": "CUNY - Baruch College", "priority": 354, "external_id": null}, {"id": 10846177003, "name": "CUNY - Brooklyn College", "priority": 355, "external_id": null}, {"id": 10846178003, "name": "CUNY - City College", "priority": 356, "external_id": null}, {"id": 10846179003, "name": "CUNY - College of Staten Island", "priority": 357, "external_id": null}, {"id": 10846180003, "name": "CUNY - Hunter College", "priority": 358, "external_id": null}, {"id": 10846181003, "name": "CUNY - John Jay College of Criminal Justice", "priority": 359, "external_id": null}, {"id": 10846182003, "name": "CUNY - Lehman College", "priority": 360, "external_id": null}, {"id": 10846183003, "name": "CUNY - Medgar Evers College", "priority": 361, "external_id": null}, {"id": 10846184003, "name": "CUNY - New York City College of Technology", "priority": 362, "external_id": null}, {"id": 10846185003, "name": "CUNY - Queens College", "priority": 363, "external_id": null}, {"id": 10846186003, "name": "CUNY - York College", "priority": 364, "external_id": null}, {"id": 10846187003, "name": "Curry College", "priority": 365, "external_id": null}, {"id": 10846188003, "name": "Curtis Institute of Music", "priority": 366, "external_id": null}, {"id": 10846189003, "name": "D'Youville College", "priority": 367, "external_id": null}, {"id": 10846190003, "name": "Daemen College", "priority": 368, "external_id": null}, {"id": 10846191003, "name": "Dakota State University", "priority": 369, "external_id": null}, {"id": 10846192003, "name": "Dakota Wesleyan University", "priority": 370, "external_id": null}, {"id": 10846193003, "name": "Dalhousie University", "priority": 371, "external_id": null}, {"id": 10846194003, "name": "Dallas Baptist University", "priority": 372, "external_id": null}, {"id": 10846195003, "name": "Dallas Christian College", "priority": 373, "external_id": null}, {"id": 10846196003, "name": "Dalton State College", "priority": 374, "external_id": null}, {"id": 10846197003, "name": "Daniel Webster College", "priority": 375, "external_id": null}, {"id": 10846198003, "name": "Davenport University", "priority": 376, "external_id": null}, {"id": 10846199003, "name": "Davis and Elkins College", "priority": 377, "external_id": null}, {"id": 10846200003, "name": "Davis College", "priority": 378, "external_id": null}, {"id": 10846201003, "name": "Daytona State College", "priority": 379, "external_id": null}, {"id": 10846202003, "name": "Dean College", "priority": 380, "external_id": null}, {"id": 10846203003, "name": "Defiance College", "priority": 381, "external_id": null}, {"id": 10846204003, "name": "Delaware Valley College", "priority": 382, "external_id": null}, {"id": 10846205003, "name": "Delta State University", "priority": 383, "external_id": null}, {"id": 10846206003, "name": "Denison University", "priority": 384, "external_id": null}, {"id": 10846207003, "name": "DePaul University", "priority": 385, "external_id": null}, {"id": 10846208003, "name": "DePauw University", "priority": 386, "external_id": null}, {"id": 10846209003, "name": "DEREE - The American College of Greece", "priority": 387, "external_id": null}, {"id": 10846210003, "name": "DeSales University", "priority": 388, "external_id": null}, {"id": 10846211003, "name": "DeVry University", "priority": 389, "external_id": null}, {"id": 10846212003, "name": "Dickinson College", "priority": 390, "external_id": null}, {"id": 10846213003, "name": "Dickinson State University", "priority": 391, "external_id": null}, {"id": 10846214003, "name": "Dillard University", "priority": 392, "external_id": null}, {"id": 10846215003, "name": "Divine Word College", "priority": 393, "external_id": null}, {"id": 10846216003, "name": "Dixie State College of Utah", "priority": 394, "external_id": null}, {"id": 10846217003, "name": "Doane College", "priority": 395, "external_id": null}, {"id": 10846218003, "name": "Dominican College", "priority": 396, "external_id": null}, {"id": 10846219003, "name": "Dominican University", "priority": 397, "external_id": null}, {"id": 10846220003, "name": "Dominican University of California", "priority": 398, "external_id": null}, {"id": 10846221003, "name": "Donnelly College", "priority": 399, "external_id": null}, {"id": 10846222003, "name": "Dordt College", "priority": 400, "external_id": null}, {"id": 10846223003, "name": "Dowling College", "priority": 401, "external_id": null}, {"id": 10846224003, "name": "Drew University", "priority": 402, "external_id": null}, {"id": 10846225003, "name": "Drexel University", "priority": 403, "external_id": null}, {"id": 10846226003, "name": "Drury University", "priority": 404, "external_id": null}, {"id": 10846227003, "name": "Dunwoody College of Technology", "priority": 405, "external_id": null}, {"id": 10846228003, "name": "Earlham College", "priority": 406, "external_id": null}, {"id": 10846229003, "name": "Drake University", "priority": 407, "external_id": null}, {"id": 10846230003, "name": "East Central University", "priority": 408, "external_id": null}, {"id": 10846231003, "name": "East Stroudsburg University of Pennsylvania", "priority": 409, "external_id": null}, {"id": 10846232003, "name": "East Tennessee State University", "priority": 410, "external_id": null}, {"id": 10846233003, "name": "East Texas Baptist University", "priority": 411, "external_id": null}, {"id": 10846234003, "name": "East-West University", "priority": 412, "external_id": null}, {"id": 10846235003, "name": "Eastern Connecticut State University", "priority": 413, "external_id": null}, {"id": 10846236003, "name": "Eastern Mennonite University", "priority": 414, "external_id": null}, {"id": 10846237003, "name": "Eastern Nazarene College", "priority": 415, "external_id": null}, {"id": 10846238003, "name": "Eastern New Mexico University", "priority": 416, "external_id": null}, {"id": 10846239003, "name": "Eastern Oregon University", "priority": 417, "external_id": null}, {"id": 10846240003, "name": "Eastern University", "priority": 418, "external_id": null}, {"id": 10846241003, "name": "Eckerd College", "priority": 419, "external_id": null}, {"id": 10846242003, "name": "ECPI University", "priority": 420, "external_id": null}, {"id": 10846243003, "name": "Edgewood College", "priority": 421, "external_id": null}, {"id": 10846244003, "name": "Edinboro University of Pennsylvania", "priority": 422, "external_id": null}, {"id": 10846245003, "name": "Edison State College", "priority": 423, "external_id": null}, {"id": 10846246003, "name": "Edward Waters College", "priority": 424, "external_id": null}, {"id": 10846247003, "name": "Elizabeth City State University", "priority": 425, "external_id": null}, {"id": 10846248003, "name": "Elizabethtown College", "priority": 426, "external_id": null}, {"id": 10846249003, "name": "Elmhurst College", "priority": 427, "external_id": null}, {"id": 10846250003, "name": "Elmira College", "priority": 428, "external_id": null}, {"id": 10846251003, "name": "Embry-Riddle Aeronautical University", "priority": 429, "external_id": null}, {"id": 10846252003, "name": "Embry-Riddle Aeronautical University - Prescott", "priority": 430, "external_id": null}, {"id": 10846253003, "name": "Emerson College", "priority": 431, "external_id": null}, {"id": 10846254003, "name": "Duquesne University", "priority": 432, "external_id": null}, {"id": 10846255003, "name": "Eastern Washington University", "priority": 433, "external_id": null}, {"id": 10846256003, "name": "Eastern Illinois University", "priority": 434, "external_id": null}, {"id": 10846257003, "name": "Eastern Kentucky University", "priority": 435, "external_id": null}, {"id": 10846258003, "name": "Eastern Michigan University", "priority": 436, "external_id": null}, {"id": 10846259003, "name": "Elon University", "priority": 437, "external_id": null}, {"id": 10846260003, "name": "Delaware State University", "priority": 438, "external_id": null}, {"id": 10846261003, "name": "Duke University", "priority": 439, "external_id": null}, {"id": 10846262003, "name": "California Polytechnic State University - San Luis Obispo", "priority": 440, "external_id": null}, {"id": 10846263003, "name": "Emmanuel College", "priority": 441, "external_id": null}, {"id": 10846264003, "name": "Emmaus Bible College", "priority": 442, "external_id": null}, {"id": 10846265003, "name": "Emory and Henry College", "priority": 443, "external_id": null}, {"id": 10846266003, "name": "Emory University", "priority": 444, "external_id": null}, {"id": 10846267003, "name": "Emporia State University", "priority": 445, "external_id": null}, {"id": 10846268003, "name": "Endicott College", "priority": 446, "external_id": null}, {"id": 10846269003, "name": "Erskine College", "priority": 447, "external_id": null}, {"id": 10846270003, "name": "Escuela de Artes Plasticas de Puerto Rico", "priority": 448, "external_id": null}, {"id": 10846271003, "name": "Eureka College", "priority": 449, "external_id": null}, {"id": 10846272003, "name": "Evangel University", "priority": 450, "external_id": null}, {"id": 10846273003, "name": "Everest College - Phoenix", "priority": 451, "external_id": null}, {"id": 10846274003, "name": "Everglades University", "priority": 452, "external_id": null}, {"id": 10846275003, "name": "Evergreen State College", "priority": 453, "external_id": null}, {"id": 10846276003, "name": "Excelsior College", "priority": 454, "external_id": null}, {"id": 10846277003, "name": "Fairfield University", "priority": 455, "external_id": null}, {"id": 10846278003, "name": "Fairleigh Dickinson University", "priority": 456, "external_id": null}, {"id": 10846279003, "name": "Fairmont State University", "priority": 457, "external_id": null}, {"id": 10846280003, "name": "Faith Baptist Bible College and Theological Seminary", "priority": 458, "external_id": null}, {"id": 10846281003, "name": "Farmingdale State College - SUNY", "priority": 459, "external_id": null}, {"id": 10846282003, "name": "Fashion Institute of Technology", "priority": 460, "external_id": null}, {"id": 10846283003, "name": "Faulkner University", "priority": 461, "external_id": null}, {"id": 10846284003, "name": "Fayetteville State University", "priority": 462, "external_id": null}, {"id": 10846285003, "name": "Felician College", "priority": 463, "external_id": null}, {"id": 10846286003, "name": "Ferris State University", "priority": 464, "external_id": null}, {"id": 10846287003, "name": "Ferrum College", "priority": 465, "external_id": null}, {"id": 10846288003, "name": "Finlandia University", "priority": 466, "external_id": null}, {"id": 10846289003, "name": "Fisher College", "priority": 467, "external_id": null}, {"id": 10846290003, "name": "Fisk University", "priority": 468, "external_id": null}, {"id": 10846291003, "name": "Fitchburg State University", "priority": 469, "external_id": null}, {"id": 10846292003, "name": "Five Towns College", "priority": 470, "external_id": null}, {"id": 10846293003, "name": "Flagler College", "priority": 471, "external_id": null}, {"id": 10846294003, "name": "Florida Christian College", "priority": 472, "external_id": null}, {"id": 10846295003, "name": "Florida College", "priority": 473, "external_id": null}, {"id": 10846296003, "name": "Florida Gulf Coast University", "priority": 474, "external_id": null}, {"id": 10846297003, "name": "Florida Institute of Technology", "priority": 475, "external_id": null}, {"id": 10846298003, "name": "Florida Memorial University", "priority": 476, "external_id": null}, {"id": 10846299003, "name": "Florida Southern College", "priority": 477, "external_id": null}, {"id": 10846300003, "name": "Florida State College - Jacksonville", "priority": 478, "external_id": null}, {"id": 10846301003, "name": "Fontbonne University", "priority": 479, "external_id": null}, {"id": 10846302003, "name": "Fort Hays State University", "priority": 480, "external_id": null}, {"id": 10846303003, "name": "Fort Lewis College", "priority": 481, "external_id": null}, {"id": 10846304003, "name": "Fort Valley State University", "priority": 482, "external_id": null}, {"id": 10846305003, "name": "Framingham State University", "priority": 483, "external_id": null}, {"id": 10846306003, "name": "Francis Marion University", "priority": 484, "external_id": null}, {"id": 10846307003, "name": "Franciscan University of Steubenville", "priority": 485, "external_id": null}, {"id": 10846308003, "name": "Frank Lloyd Wright School of Architecture", "priority": 486, "external_id": null}, {"id": 10846309003, "name": "Franklin and Marshall College", "priority": 487, "external_id": null}, {"id": 10846310003, "name": "Franklin College", "priority": 488, "external_id": null}, {"id": 10846311003, "name": "Franklin College Switzerland", "priority": 489, "external_id": null}, {"id": 10846312003, "name": "Franklin Pierce University", "priority": 490, "external_id": null}, {"id": 10846313003, "name": "Franklin University", "priority": 491, "external_id": null}, {"id": 10846314003, "name": "Franklin W. Olin College of Engineering", "priority": 492, "external_id": null}, {"id": 10846315003, "name": "Freed-Hardeman University", "priority": 493, "external_id": null}, {"id": 10846316003, "name": "Fresno Pacific University", "priority": 494, "external_id": null}, {"id": 10846317003, "name": "Friends University", "priority": 495, "external_id": null}, {"id": 10846318003, "name": "Frostburg State University", "priority": 496, "external_id": null}, {"id": 10846319003, "name": "Gallaudet University", "priority": 497, "external_id": null}, {"id": 10846320003, "name": "Gannon University", "priority": 498, "external_id": null}, {"id": 10846321003, "name": "Geneva College", "priority": 499, "external_id": null}, {"id": 10846322003, "name": "George Fox University", "priority": 500, "external_id": null}, {"id": 10846323003, "name": "George Mason University", "priority": 501, "external_id": null}, {"id": 10846324003, "name": "George Washington University", "priority": 502, "external_id": null}, {"id": 10846325003, "name": "Georgetown College", "priority": 503, "external_id": null}, {"id": 10846326003, "name": "Georgia College & State University", "priority": 504, "external_id": null}, {"id": 10846327003, "name": "Georgia Gwinnett College", "priority": 505, "external_id": null}, {"id": 10846328003, "name": "Georgia Regents University", "priority": 506, "external_id": null}, {"id": 10846329003, "name": "Georgia Southwestern State University", "priority": 507, "external_id": null}, {"id": 10846330003, "name": "Georgian Court University", "priority": 508, "external_id": null}, {"id": 10846331003, "name": "Gettysburg College", "priority": 509, "external_id": null}, {"id": 10846332003, "name": "Glenville State College", "priority": 510, "external_id": null}, {"id": 10846333003, "name": "God's Bible School and College", "priority": 511, "external_id": null}, {"id": 10846334003, "name": "Goddard College", "priority": 512, "external_id": null}, {"id": 10846335003, "name": "Golden Gate University", "priority": 513, "external_id": null}, {"id": 10846336003, "name": "Goldey-Beacom College", "priority": 514, "external_id": null}, {"id": 10846337003, "name": "Goldfarb School of Nursing at Barnes-Jewish College", "priority": 515, "external_id": null}, {"id": 10846338003, "name": "Gonzaga University", "priority": 516, "external_id": null}, {"id": 10846339003, "name": "Gordon College", "priority": 517, "external_id": null}, {"id": 10846340003, "name": "Fordham University", "priority": 518, "external_id": null}, {"id": 10846341003, "name": "Georgia Institute of Technology", "priority": 519, "external_id": null}, {"id": 10846342003, "name": "Gardner-Webb University", "priority": 520, "external_id": null}, {"id": 10846343003, "name": "Georgia Southern University", "priority": 521, "external_id": null}, {"id": 10846344003, "name": "Georgia State University", "priority": 522, "external_id": null}, {"id": 10846345003, "name": "Florida State University", "priority": 523, "external_id": null}, {"id": 10846346003, "name": "Dartmouth College", "priority": 524, "external_id": null}, {"id": 10846347003, "name": "Florida International University", "priority": 525, "external_id": null}, {"id": 10846348003, "name": "Georgetown University", "priority": 526, "external_id": null}, {"id": 10846349003, "name": "Furman University", "priority": 527, "external_id": null}, {"id": 10846350003, "name": "Gordon State College", "priority": 528, "external_id": null}, {"id": 10846351003, "name": "Goshen College", "priority": 529, "external_id": null}, {"id": 10846352003, "name": "Goucher College", "priority": 530, "external_id": null}, {"id": 10846353003, "name": "Governors State University", "priority": 531, "external_id": null}, {"id": 10846354003, "name": "Grace Bible College", "priority": 532, "external_id": null}, {"id": 10846355003, "name": "Grace College and Seminary", "priority": 533, "external_id": null}, {"id": 10846356003, "name": "Grace University", "priority": 534, "external_id": null}, {"id": 10846357003, "name": "Graceland University", "priority": 535, "external_id": null}, {"id": 10846358003, "name": "Grand Canyon University", "priority": 536, "external_id": null}, {"id": 10846359003, "name": "Grand Valley State University", "priority": 537, "external_id": null}, {"id": 10846360003, "name": "Grand View University", "priority": 538, "external_id": null}, {"id": 10846361003, "name": "Granite State College", "priority": 539, "external_id": null}, {"id": 10846362003, "name": "Gratz College", "priority": 540, "external_id": null}, {"id": 10846363003, "name": "Great Basin College", "priority": 541, "external_id": null}, {"id": 10846364003, "name": "Great Lakes Christian College", "priority": 542, "external_id": null}, {"id": 10846365003, "name": "Green Mountain College", "priority": 543, "external_id": null}, {"id": 10846366003, "name": "Greensboro College", "priority": 544, "external_id": null}, {"id": 10846367003, "name": "Greenville College", "priority": 545, "external_id": null}, {"id": 10846368003, "name": "Grinnell College", "priority": 546, "external_id": null}, {"id": 10846369003, "name": "Grove City College", "priority": 547, "external_id": null}, {"id": 10846370003, "name": "Guilford College", "priority": 548, "external_id": null}, {"id": 10846371003, "name": "Gustavus Adolphus College", "priority": 549, "external_id": null}, {"id": 10846372003, "name": "Gwynedd-Mercy College", "priority": 550, "external_id": null}, {"id": 10846373003, "name": "Hamilton College", "priority": 551, "external_id": null}, {"id": 10846374003, "name": "Hamline University", "priority": 552, "external_id": null}, {"id": 10846375003, "name": "Hampden-Sydney College", "priority": 553, "external_id": null}, {"id": 10846376003, "name": "Hampshire College", "priority": 554, "external_id": null}, {"id": 10846377003, "name": "Hannibal-LaGrange University", "priority": 555, "external_id": null}, {"id": 10846378003, "name": "Hanover College", "priority": 556, "external_id": null}, {"id": 10846379003, "name": "Hardin-Simmons University", "priority": 557, "external_id": null}, {"id": 10846380003, "name": "Harding University", "priority": 558, "external_id": null}, {"id": 10846381003, "name": "Harrington College of Design", "priority": 559, "external_id": null}, {"id": 10846382003, "name": "Harris-Stowe State University", "priority": 560, "external_id": null}, {"id": 10846383003, "name": "Harrisburg University of Science and Technology", "priority": 561, "external_id": null}, {"id": 10846384003, "name": "Hartwick College", "priority": 562, "external_id": null}, {"id": 10846385003, "name": "Harvey Mudd College", "priority": 563, "external_id": null}, {"id": 10846386003, "name": "Haskell Indian Nations University", "priority": 564, "external_id": null}, {"id": 10846387003, "name": "Hastings College", "priority": 565, "external_id": null}, {"id": 10846388003, "name": "Haverford College", "priority": 566, "external_id": null}, {"id": 10846389003, "name": "Hawaii Pacific University", "priority": 567, "external_id": null}, {"id": 10846390003, "name": "Hebrew Theological College", "priority": 568, "external_id": null}, {"id": 10846391003, "name": "Heidelberg University", "priority": 569, "external_id": null}, {"id": 10846392003, "name": "Hellenic College", "priority": 570, "external_id": null}, {"id": 10846393003, "name": "Henderson State University", "priority": 571, "external_id": null}, {"id": 10846394003, "name": "Hendrix College", "priority": 572, "external_id": null}, {"id": 10846395003, "name": "Heritage University", "priority": 573, "external_id": null}, {"id": 10846396003, "name": "Herzing University", "priority": 574, "external_id": null}, {"id": 10846397003, "name": "Hesser College", "priority": 575, "external_id": null}, {"id": 10846398003, "name": "High Point University", "priority": 576, "external_id": null}, {"id": 10846399003, "name": "Hilbert College", "priority": 577, "external_id": null}, {"id": 10846400003, "name": "Hillsdale College", "priority": 578, "external_id": null}, {"id": 10846401003, "name": "Hiram College", "priority": 579, "external_id": null}, {"id": 10846402003, "name": "Hobart and William Smith Colleges", "priority": 580, "external_id": null}, {"id": 10846403003, "name": "Hodges University", "priority": 581, "external_id": null}, {"id": 10846404003, "name": "Hofstra University", "priority": 582, "external_id": null}, {"id": 10846405003, "name": "Hollins University", "priority": 583, "external_id": null}, {"id": 10846406003, "name": "Holy Apostles College and Seminary", "priority": 584, "external_id": null}, {"id": 10846407003, "name": "Indiana State University", "priority": 585, "external_id": null}, {"id": 10846408003, "name": "Holy Family University", "priority": 586, "external_id": null}, {"id": 10846409003, "name": "Holy Names University", "priority": 587, "external_id": null}, {"id": 10846410003, "name": "Hood College", "priority": 588, "external_id": null}, {"id": 10846411003, "name": "Hope College", "priority": 589, "external_id": null}, {"id": 10846412003, "name": "Hope International University", "priority": 590, "external_id": null}, {"id": 10846413003, "name": "Houghton College", "priority": 591, "external_id": null}, {"id": 10846414003, "name": "Howard Payne University", "priority": 592, "external_id": null}, {"id": 10846415003, "name": "Hult International Business School", "priority": 593, "external_id": null}, {"id": 10846416003, "name": "Humboldt State University", "priority": 594, "external_id": null}, {"id": 10846417003, "name": "Humphreys College", "priority": 595, "external_id": null}, {"id": 10846418003, "name": "Huntingdon College", "priority": 596, "external_id": null}, {"id": 10846419003, "name": "Huntington University", "priority": 597, "external_id": null}, {"id": 10846420003, "name": "Husson University", "priority": 598, "external_id": null}, {"id": 10846421003, "name": "Huston-Tillotson University", "priority": 599, "external_id": null}, {"id": 10846422003, "name": "Illinois College", "priority": 600, "external_id": null}, {"id": 10846423003, "name": "Illinois Institute of Art at Chicago", "priority": 601, "external_id": null}, {"id": 10846424003, "name": "Illinois Institute of Technology", "priority": 602, "external_id": null}, {"id": 10846425003, "name": "Illinois Wesleyan University", "priority": 603, "external_id": null}, {"id": 10846426003, "name": "Immaculata University", "priority": 604, "external_id": null}, {"id": 10846427003, "name": "Indian River State College", "priority": 605, "external_id": null}, {"id": 10846428003, "name": "Indiana Institute of Technology", "priority": 606, "external_id": null}, {"id": 10846429003, "name": "Indiana University East", "priority": 607, "external_id": null}, {"id": 10846430003, "name": "Indiana University Northwest", "priority": 608, "external_id": null}, {"id": 10846431003, "name": "Indiana University of Pennsylvania", "priority": 609, "external_id": null}, {"id": 10846432003, "name": "Indiana University Southeast", "priority": 610, "external_id": null}, {"id": 10846433003, "name": "Illinois State University", "priority": 611, "external_id": null}, {"id": 10846434003, "name": "Indiana University - Bloomington", "priority": 612, "external_id": null}, {"id": 10846435003, "name": "Davidson College", "priority": 613, "external_id": null}, {"id": 10846436003, "name": "Idaho State University", "priority": 614, "external_id": null}, {"id": 10846437003, "name": "Harvard University", "priority": 615, "external_id": null}, {"id": 10846438003, "name": "Howard University", "priority": 616, "external_id": null}, {"id": 10846439003, "name": "Houston Baptist University", "priority": 617, "external_id": null}, {"id": 10846440003, "name": "Indiana University - Kokomo", "priority": 618, "external_id": null}, {"id": 10846441003, "name": "Indiana University - South Bend", "priority": 619, "external_id": null}, {"id": 10846442003, "name": "Indiana University-Purdue University - Fort Wayne", "priority": 620, "external_id": null}, {"id": 10846443003, "name": "Indiana University-Purdue University - Indianapolis", "priority": 621, "external_id": null}, {"id": 10846444003, "name": "Indiana Wesleyan University", "priority": 622, "external_id": null}, {"id": 10846445003, "name": "Institute of American Indian and Alaska Native Culture and Arts Development", "priority": 623, "external_id": null}, {"id": 10846446003, "name": "Inter American University of Puerto Rico - Aguadilla", "priority": 624, "external_id": null}, {"id": 10846447003, "name": "Inter American University of Puerto Rico - Arecibo", "priority": 625, "external_id": null}, {"id": 10846448003, "name": "Inter American University of Puerto Rico - Barranquitas", "priority": 626, "external_id": null}, {"id": 10846449003, "name": "Inter American University of Puerto Rico - Bayamon", "priority": 627, "external_id": null}, {"id": 10846450003, "name": "Inter American University of Puerto Rico - Fajardo", "priority": 628, "external_id": null}, {"id": 10846451003, "name": "Inter American University of Puerto Rico - Guayama", "priority": 629, "external_id": null}, {"id": 10846452003, "name": "Inter American University of Puerto Rico - Metropolitan Campus", "priority": 630, "external_id": null}, {"id": 10846453003, "name": "Inter American University of Puerto Rico - Ponce", "priority": 631, "external_id": null}, {"id": 10846454003, "name": "Inter American University of Puerto Rico - San German", "priority": 632, "external_id": null}, {"id": 10846455003, "name": "International College of the Cayman Islands", "priority": 633, "external_id": null}, {"id": 10846456003, "name": "Iona College", "priority": 634, "external_id": null}, {"id": 10846457003, "name": "Iowa Wesleyan College", "priority": 635, "external_id": null}, {"id": 10846458003, "name": "Ithaca College", "priority": 636, "external_id": null}, {"id": 10846459003, "name": "Jarvis Christian College", "priority": 637, "external_id": null}, {"id": 10846460003, "name": "Jewish Theological Seminary of America", "priority": 638, "external_id": null}, {"id": 10846461003, "name": "John Brown University", "priority": 639, "external_id": null}, {"id": 10846462003, "name": "John Carroll University", "priority": 640, "external_id": null}, {"id": 10846463003, "name": "John F. Kennedy University", "priority": 641, "external_id": null}, {"id": 10846464003, "name": "Johns Hopkins University", "priority": 642, "external_id": null}, {"id": 10846465003, "name": "Johnson & Wales University", "priority": 643, "external_id": null}, {"id": 10846466003, "name": "Johnson C. Smith University", "priority": 644, "external_id": null}, {"id": 10846467003, "name": "Johnson State College", "priority": 645, "external_id": null}, {"id": 10846468003, "name": "Johnson University", "priority": 646, "external_id": null}, {"id": 10846469003, "name": "Jones International University", "priority": 647, "external_id": null}, {"id": 10846470003, "name": "Judson College", "priority": 648, "external_id": null}, {"id": 10846471003, "name": "Judson University", "priority": 649, "external_id": null}, {"id": 10846472003, "name": "Juilliard School", "priority": 650, "external_id": null}, {"id": 10846473003, "name": "Juniata College", "priority": 651, "external_id": null}, {"id": 10846474003, "name": "Kalamazoo College", "priority": 652, "external_id": null}, {"id": 10846475003, "name": "Kansas City Art Institute", "priority": 653, "external_id": null}, {"id": 10846476003, "name": "Kansas Wesleyan University", "priority": 654, "external_id": null}, {"id": 10846477003, "name": "Kaplan University", "priority": 655, "external_id": null}, {"id": 10846478003, "name": "Kean University", "priority": 656, "external_id": null}, {"id": 10846479003, "name": "Keene State College", "priority": 657, "external_id": null}, {"id": 10846480003, "name": "Keiser University", "priority": 658, "external_id": null}, {"id": 10846481003, "name": "Kendall College", "priority": 659, "external_id": null}, {"id": 10846482003, "name": "Kennesaw State University", "priority": 660, "external_id": null}, {"id": 10846483003, "name": "Kentucky Christian University", "priority": 661, "external_id": null}, {"id": 10846484003, "name": "Kentucky State University", "priority": 662, "external_id": null}, {"id": 10846485003, "name": "Kentucky Wesleyan College", "priority": 663, "external_id": null}, {"id": 10846486003, "name": "Kenyon College", "priority": 664, "external_id": null}, {"id": 10846487003, "name": "Kettering College", "priority": 665, "external_id": null}, {"id": 10846488003, "name": "Kettering University", "priority": 666, "external_id": null}, {"id": 10846489003, "name": "Keuka College", "priority": 667, "external_id": null}, {"id": 10846490003, "name": "Keystone College", "priority": 668, "external_id": null}, {"id": 10846491003, "name": "King University", "priority": 669, "external_id": null}, {"id": 10846492003, "name": "King's College", "priority": 670, "external_id": null}, {"id": 10846493003, "name": "Knox College", "priority": 671, "external_id": null}, {"id": 10846494003, "name": "Kutztown University of Pennsylvania", "priority": 672, "external_id": null}, {"id": 10846495003, "name": "Kuyper College", "priority": 673, "external_id": null}, {"id": 10846496003, "name": "La Roche College", "priority": 674, "external_id": null}, {"id": 10846497003, "name": "La Salle University", "priority": 675, "external_id": null}, {"id": 10846498003, "name": "La Sierra University", "priority": 676, "external_id": null}, {"id": 10846499003, "name": "LaGrange College", "priority": 677, "external_id": null}, {"id": 10846500003, "name": "Laguna College of Art and Design", "priority": 678, "external_id": null}, {"id": 10846501003, "name": "Lake Erie College", "priority": 679, "external_id": null}, {"id": 10846502003, "name": "Lake Forest College", "priority": 680, "external_id": null}, {"id": 10846503003, "name": "Lake Superior State University", "priority": 681, "external_id": null}, {"id": 10846504003, "name": "Lakeland College", "priority": 682, "external_id": null}, {"id": 10846505003, "name": "Lakeview College of Nursing", "priority": 683, "external_id": null}, {"id": 10846506003, "name": "Lancaster Bible College", "priority": 684, "external_id": null}, {"id": 10846507003, "name": "Lander University", "priority": 685, "external_id": null}, {"id": 10846508003, "name": "Lane College", "priority": 686, "external_id": null}, {"id": 10846509003, "name": "Langston University", "priority": 687, "external_id": null}, {"id": 10846510003, "name": "Lasell College", "priority": 688, "external_id": null}, {"id": 10846511003, "name": "Lawrence Technological University", "priority": 689, "external_id": null}, {"id": 10846512003, "name": "Lawrence University", "priority": 690, "external_id": null}, {"id": 10846513003, "name": "Le Moyne College", "priority": 691, "external_id": null}, {"id": 10846514003, "name": "Lebanon Valley College", "priority": 692, "external_id": null}, {"id": 10846515003, "name": "Lee University", "priority": 693, "external_id": null}, {"id": 10846516003, "name": "Lees-McRae College", "priority": 694, "external_id": null}, {"id": 10846517003, "name": "Kansas State University", "priority": 695, "external_id": null}, {"id": 10846518003, "name": "James Madison University", "priority": 696, "external_id": null}, {"id": 10846519003, "name": "Lafayette College", "priority": 697, "external_id": null}, {"id": 10846520003, "name": "Jacksonville University", "priority": 698, "external_id": null}, {"id": 10846521003, "name": "Kent State University", "priority": 699, "external_id": null}, {"id": 10846522003, "name": "Lamar University", "priority": 700, "external_id": null}, {"id": 10846523003, "name": "Jackson State University", "priority": 701, "external_id": null}, {"id": 10846524003, "name": "Lehigh University", "priority": 702, "external_id": null}, {"id": 10846525003, "name": "Jacksonville State University", "priority": 703, "external_id": null}, {"id": 10846526003, "name": "LeMoyne-Owen College", "priority": 704, "external_id": null}, {"id": 10846527003, "name": "Lenoir-Rhyne University", "priority": 705, "external_id": null}, {"id": 10846528003, "name": "Lesley University", "priority": 706, "external_id": null}, {"id": 10846529003, "name": "LeTourneau University", "priority": 707, "external_id": null}, {"id": 10846530003, "name": "Lewis & Clark College", "priority": 708, "external_id": null}, {"id": 10846531003, "name": "Lewis University", "priority": 709, "external_id": null}, {"id": 10846532003, "name": "Lewis-Clark State College", "priority": 710, "external_id": null}, {"id": 10846533003, "name": "Lexington College", "priority": 711, "external_id": null}, {"id": 10846534003, "name": "Life Pacific College", "priority": 712, "external_id": null}, {"id": 10846535003, "name": "Life University", "priority": 713, "external_id": null}, {"id": 10846536003, "name": "LIM College", "priority": 714, "external_id": null}, {"id": 10846537003, "name": "Limestone College", "priority": 715, "external_id": null}, {"id": 10846538003, "name": "Lincoln Christian University", "priority": 716, "external_id": null}, {"id": 10846539003, "name": "Lincoln College", "priority": 717, "external_id": null}, {"id": 10846540003, "name": "Lincoln Memorial University", "priority": 718, "external_id": null}, {"id": 10846541003, "name": "Lincoln University", "priority": 719, "external_id": null}, {"id": 10846542003, "name": "Lindenwood University", "priority": 720, "external_id": null}, {"id": 10846543003, "name": "Lindsey Wilson College", "priority": 721, "external_id": null}, {"id": 10846544003, "name": "Linfield College", "priority": 722, "external_id": null}, {"id": 10846545003, "name": "Lipscomb University", "priority": 723, "external_id": null}, {"id": 10846546003, "name": "LIU Post", "priority": 724, "external_id": null}, {"id": 10846547003, "name": "Livingstone College", "priority": 725, "external_id": null}, {"id": 10846548003, "name": "Lock Haven University of Pennsylvania", "priority": 726, "external_id": null}, {"id": 10846549003, "name": "Loma Linda University", "priority": 727, "external_id": null}, {"id": 10846550003, "name": "Longwood University", "priority": 728, "external_id": null}, {"id": 10846551003, "name": "Loras College", "priority": 729, "external_id": null}, {"id": 10846552003, "name": "Louisiana College", "priority": 730, "external_id": null}, {"id": 10846553003, "name": "Louisiana State University Health Sciences Center", "priority": 731, "external_id": null}, {"id": 10846554003, "name": "Louisiana State University - Alexandria", "priority": 732, "external_id": null}, {"id": 10846555003, "name": "Louisiana State University - Shreveport", "priority": 733, "external_id": null}, {"id": 10846556003, "name": "Lourdes University", "priority": 734, "external_id": null}, {"id": 10846557003, "name": "Loyola Marymount University", "priority": 735, "external_id": null}, {"id": 10846558003, "name": "Loyola University Chicago", "priority": 736, "external_id": null}, {"id": 10846559003, "name": "Loyola University Maryland", "priority": 737, "external_id": null}, {"id": 10846560003, "name": "Loyola University New Orleans", "priority": 738, "external_id": null}, {"id": 10846561003, "name": "Lubbock Christian University", "priority": 739, "external_id": null}, {"id": 10846562003, "name": "Luther College", "priority": 740, "external_id": null}, {"id": 10846563003, "name": "Lycoming College", "priority": 741, "external_id": null}, {"id": 10846564003, "name": "Lyme Academy College of Fine Arts", "priority": 742, "external_id": null}, {"id": 10846565003, "name": "Lynchburg College", "priority": 743, "external_id": null}, {"id": 10846566003, "name": "Lyndon State College", "priority": 744, "external_id": null}, {"id": 10846567003, "name": "Lynn University", "priority": 745, "external_id": null}, {"id": 10846568003, "name": "Lyon College", "priority": 746, "external_id": null}, {"id": 10846569003, "name": "Macalester College", "priority": 747, "external_id": null}, {"id": 10846570003, "name": "MacMurray College", "priority": 748, "external_id": null}, {"id": 10846571003, "name": "Madonna University", "priority": 749, "external_id": null}, {"id": 10846572003, "name": "Maharishi University of Management", "priority": 750, "external_id": null}, {"id": 10846573003, "name": "Maine College of Art", "priority": 751, "external_id": null}, {"id": 10846574003, "name": "Maine Maritime Academy", "priority": 752, "external_id": null}, {"id": 10846575003, "name": "Malone University", "priority": 753, "external_id": null}, {"id": 10846576003, "name": "Manchester University", "priority": 754, "external_id": null}, {"id": 10846577003, "name": "Manhattan Christian College", "priority": 755, "external_id": null}, {"id": 10846578003, "name": "Manhattan College", "priority": 756, "external_id": null}, {"id": 10846579003, "name": "Manhattan School of Music", "priority": 757, "external_id": null}, {"id": 10846580003, "name": "Manhattanville College", "priority": 758, "external_id": null}, {"id": 10846581003, "name": "Mansfield University of Pennsylvania", "priority": 759, "external_id": null}, {"id": 10846582003, "name": "Maranatha Baptist Bible College", "priority": 760, "external_id": null}, {"id": 10846583003, "name": "Marian University", "priority": 761, "external_id": null}, {"id": 10846584003, "name": "Marietta College", "priority": 762, "external_id": null}, {"id": 10846585003, "name": "Marlboro College", "priority": 763, "external_id": null}, {"id": 10846586003, "name": "Marquette University", "priority": 764, "external_id": null}, {"id": 10846587003, "name": "Mars Hill University", "priority": 765, "external_id": null}, {"id": 10846588003, "name": "Martin Luther College", "priority": 766, "external_id": null}, {"id": 10846589003, "name": "Martin Methodist College", "priority": 767, "external_id": null}, {"id": 10846590003, "name": "Martin University", "priority": 768, "external_id": null}, {"id": 10846591003, "name": "Mary Baldwin College", "priority": 769, "external_id": null}, {"id": 10846592003, "name": "Marygrove College", "priority": 770, "external_id": null}, {"id": 10846593003, "name": "Maryland Institute College of Art", "priority": 771, "external_id": null}, {"id": 10846594003, "name": "Marylhurst University", "priority": 772, "external_id": null}, {"id": 10846595003, "name": "Marymount Manhattan College", "priority": 773, "external_id": null}, {"id": 10846596003, "name": "Marymount University", "priority": 774, "external_id": null}, {"id": 10846597003, "name": "Maryville College", "priority": 775, "external_id": null}, {"id": 10846598003, "name": "Maryville University of St. Louis", "priority": 776, "external_id": null}, {"id": 10846599003, "name": "Marywood University", "priority": 777, "external_id": null}, {"id": 10846600003, "name": "Massachusetts College of Art and Design", "priority": 778, "external_id": null}, {"id": 10846601003, "name": "Massachusetts College of Liberal Arts", "priority": 779, "external_id": null}, {"id": 10846602003, "name": "Massachusetts College of Pharmacy and Health Sciences", "priority": 780, "external_id": null}, {"id": 10846603003, "name": "Massachusetts Institute of Technology", "priority": 781, "external_id": null}, {"id": 10846604003, "name": "Massachusetts Maritime Academy", "priority": 782, "external_id": null}, {"id": 10846605003, "name": "Master's College and Seminary", "priority": 783, "external_id": null}, {"id": 10846606003, "name": "Mayville State University", "priority": 784, "external_id": null}, {"id": 10846607003, "name": "McDaniel College", "priority": 785, "external_id": null}, {"id": 10846608003, "name": "McGill University", "priority": 786, "external_id": null}, {"id": 10846609003, "name": "McKendree University", "priority": 787, "external_id": null}, {"id": 10846610003, "name": "McMurry University", "priority": 788, "external_id": null}, {"id": 10846611003, "name": "McPherson College", "priority": 789, "external_id": null}, {"id": 10846612003, "name": "Medaille College", "priority": 790, "external_id": null}, {"id": 10846613003, "name": "Marist College", "priority": 791, "external_id": null}, {"id": 10846614003, "name": "McNeese State University", "priority": 792, "external_id": null}, {"id": 10846615003, "name": "Louisiana Tech University", "priority": 793, "external_id": null}, {"id": 10846616003, "name": "Marshall University", "priority": 794, "external_id": null}, {"id": 10846617003, "name": "Medical University of South Carolina", "priority": 795, "external_id": null}, {"id": 10846618003, "name": "Memorial University of Newfoundland", "priority": 796, "external_id": null}, {"id": 10846619003, "name": "Memphis College of Art", "priority": 797, "external_id": null}, {"id": 10846620003, "name": "Menlo College", "priority": 798, "external_id": null}, {"id": 10846621003, "name": "Mercy College", "priority": 799, "external_id": null}, {"id": 10846622003, "name": "Mercy College of Health Sciences", "priority": 800, "external_id": null}, {"id": 10846623003, "name": "Mercy College of Ohio", "priority": 801, "external_id": null}, {"id": 10846624003, "name": "Mercyhurst University", "priority": 802, "external_id": null}, {"id": 10846625003, "name": "Meredith College", "priority": 803, "external_id": null}, {"id": 10846626003, "name": "Merrimack College", "priority": 804, "external_id": null}, {"id": 10846627003, "name": "Messiah College", "priority": 805, "external_id": null}, {"id": 10846628003, "name": "Methodist University", "priority": 806, "external_id": null}, {"id": 10846629003, "name": "Metropolitan College of New York", "priority": 807, "external_id": null}, {"id": 10846630003, "name": "Metropolitan State University", "priority": 808, "external_id": null}, {"id": 10846631003, "name": "Metropolitan State University of Denver", "priority": 809, "external_id": null}, {"id": 10846632003, "name": "Miami Dade College", "priority": 810, "external_id": null}, {"id": 10846633003, "name": "Miami International University of Art & Design", "priority": 811, "external_id": null}, {"id": 10846634003, "name": "Michigan Technological University", "priority": 812, "external_id": null}, {"id": 10846635003, "name": "Mid-America Christian University", "priority": 813, "external_id": null}, {"id": 10846636003, "name": "Mid-Atlantic Christian University", "priority": 814, "external_id": null}, {"id": 10846637003, "name": "Mid-Continent University", "priority": 815, "external_id": null}, {"id": 10846638003, "name": "MidAmerica Nazarene University", "priority": 816, "external_id": null}, {"id": 10846639003, "name": "Middle Georgia State College", "priority": 817, "external_id": null}, {"id": 10846640003, "name": "Middlebury College", "priority": 818, "external_id": null}, {"id": 10846641003, "name": "Midland College", "priority": 819, "external_id": null}, {"id": 10846642003, "name": "Midland University", "priority": 820, "external_id": null}, {"id": 10846643003, "name": "Midstate College", "priority": 821, "external_id": null}, {"id": 10846644003, "name": "Midway College", "priority": 822, "external_id": null}, {"id": 10846645003, "name": "Midwestern State University", "priority": 823, "external_id": null}, {"id": 10846646003, "name": "Miles College", "priority": 824, "external_id": null}, {"id": 10846647003, "name": "Millersville University of Pennsylvania", "priority": 825, "external_id": null}, {"id": 10846648003, "name": "Milligan College", "priority": 826, "external_id": null}, {"id": 10846649003, "name": "Millikin University", "priority": 827, "external_id": null}, {"id": 10846650003, "name": "Mills College", "priority": 828, "external_id": null}, {"id": 10846651003, "name": "Millsaps College", "priority": 829, "external_id": null}, {"id": 10846652003, "name": "Milwaukee Institute of Art and Design", "priority": 830, "external_id": null}, {"id": 10846653003, "name": "Milwaukee School of Engineering", "priority": 831, "external_id": null}, {"id": 10846654003, "name": "Minneapolis College of Art and Design", "priority": 832, "external_id": null}, {"id": 10846655003, "name": "Minnesota State University - Mankato", "priority": 833, "external_id": null}, {"id": 10846656003, "name": "Minnesota State University - Moorhead", "priority": 834, "external_id": null}, {"id": 10846657003, "name": "Minot State University", "priority": 835, "external_id": null}, {"id": 10846658003, "name": "Misericordia University", "priority": 836, "external_id": null}, {"id": 10846659003, "name": "Mississippi College", "priority": 837, "external_id": null}, {"id": 10846660003, "name": "Mississippi University for Women", "priority": 838, "external_id": null}, {"id": 10846661003, "name": "Missouri Baptist University", "priority": 839, "external_id": null}, {"id": 10846662003, "name": "Missouri Southern State University", "priority": 840, "external_id": null}, {"id": 10846663003, "name": "Missouri University of Science & Technology", "priority": 841, "external_id": null}, {"id": 10846664003, "name": "Missouri Valley College", "priority": 842, "external_id": null}, {"id": 10846665003, "name": "Missouri Western State University", "priority": 843, "external_id": null}, {"id": 10846666003, "name": "Mitchell College", "priority": 844, "external_id": null}, {"id": 10846667003, "name": "Molloy College", "priority": 845, "external_id": null}, {"id": 10846668003, "name": "Monmouth College", "priority": 846, "external_id": null}, {"id": 10846669003, "name": "Monroe College", "priority": 847, "external_id": null}, {"id": 10846670003, "name": "Montana State University - Billings", "priority": 848, "external_id": null}, {"id": 10846671003, "name": "Montana State University - Northern", "priority": 849, "external_id": null}, {"id": 10846672003, "name": "Montana Tech of the University of Montana", "priority": 850, "external_id": null}, {"id": 10846673003, "name": "Montclair State University", "priority": 851, "external_id": null}, {"id": 10846674003, "name": "Monterrey Institute of Technology and Higher Education - Monterrey", "priority": 852, "external_id": null}, {"id": 10846675003, "name": "Montreat College", "priority": 853, "external_id": null}, {"id": 10846676003, "name": "Montserrat College of Art", "priority": 854, "external_id": null}, {"id": 10846677003, "name": "Moody Bible Institute", "priority": 855, "external_id": null}, {"id": 10846678003, "name": "Moore College of Art & Design", "priority": 856, "external_id": null}, {"id": 10846679003, "name": "Moravian College", "priority": 857, "external_id": null}, {"id": 10846680003, "name": "Morehouse College", "priority": 858, "external_id": null}, {"id": 10846681003, "name": "Morningside College", "priority": 859, "external_id": null}, {"id": 10846682003, "name": "Morris College", "priority": 860, "external_id": null}, {"id": 10846683003, "name": "Morrisville State College", "priority": 861, "external_id": null}, {"id": 10846684003, "name": "Mount Aloysius College", "priority": 862, "external_id": null}, {"id": 10846685003, "name": "Mount Angel Seminary", "priority": 863, "external_id": null}, {"id": 10846686003, "name": "Mount Carmel College of Nursing", "priority": 864, "external_id": null}, {"id": 10846687003, "name": "Mount Holyoke College", "priority": 865, "external_id": null}, {"id": 10846688003, "name": "Mount Ida College", "priority": 866, "external_id": null}, {"id": 10846689003, "name": "Mount Marty College", "priority": 867, "external_id": null}, {"id": 10846690003, "name": "Mount Mary University", "priority": 868, "external_id": null}, {"id": 10846691003, "name": "Mount Mercy University", "priority": 869, "external_id": null}, {"id": 10846692003, "name": "Mount Olive College", "priority": 870, "external_id": null}, {"id": 10846693003, "name": "Mississippi State University", "priority": 871, "external_id": null}, {"id": 10846694003, "name": "Montana State University", "priority": 872, "external_id": null}, {"id": 10846695003, "name": "Mississippi Valley State University", "priority": 873, "external_id": null}, {"id": 10846696003, "name": "Monmouth University", "priority": 874, "external_id": null}, {"id": 10846697003, "name": "Morehead State University", "priority": 875, "external_id": null}, {"id": 10846698003, "name": "Miami University - Oxford", "priority": 876, "external_id": null}, {"id": 10846699003, "name": "Morgan State University", "priority": 877, "external_id": null}, {"id": 10846700003, "name": "Missouri State University", "priority": 878, "external_id": null}, {"id": 10846701003, "name": "Michigan State University", "priority": 879, "external_id": null}, {"id": 10846702003, "name": "Mount St. Mary College", "priority": 880, "external_id": null}, {"id": 10846703003, "name": "Mount St. Mary's College", "priority": 881, "external_id": null}, {"id": 10846704003, "name": "Mount St. Mary's University", "priority": 882, "external_id": null}, {"id": 10846705003, "name": "Mount Vernon Nazarene University", "priority": 883, "external_id": null}, {"id": 10846706003, "name": "Muhlenberg College", "priority": 884, "external_id": null}, {"id": 10846707003, "name": "Multnomah University", "priority": 885, "external_id": null}, {"id": 10846708003, "name": "Muskingum University", "priority": 886, "external_id": null}, {"id": 10846709003, "name": "Naropa University", "priority": 887, "external_id": null}, {"id": 10846710003, "name": "National American University", "priority": 888, "external_id": null}, {"id": 10846711003, "name": "National Graduate School of Quality Management", "priority": 889, "external_id": null}, {"id": 10846712003, "name": "National Hispanic University", "priority": 890, "external_id": null}, {"id": 10846713003, "name": "National Labor College", "priority": 891, "external_id": null}, {"id": 10846714003, "name": "National University", "priority": 892, "external_id": null}, {"id": 10846715003, "name": "National-Louis University", "priority": 893, "external_id": null}, {"id": 10846716003, "name": "Nazarene Bible College", "priority": 894, "external_id": null}, {"id": 10846717003, "name": "Nazareth College", "priority": 895, "external_id": null}, {"id": 10846718003, "name": "Nebraska Methodist College", "priority": 896, "external_id": null}, {"id": 10846719003, "name": "Nebraska Wesleyan University", "priority": 897, "external_id": null}, {"id": 10846720003, "name": "Neumann University", "priority": 898, "external_id": null}, {"id": 10846721003, "name": "Nevada State College", "priority": 899, "external_id": null}, {"id": 10846722003, "name": "New College of Florida", "priority": 900, "external_id": null}, {"id": 10846723003, "name": "New England College", "priority": 901, "external_id": null}, {"id": 10846724003, "name": "New England Conservatory of Music", "priority": 902, "external_id": null}, {"id": 10846725003, "name": "New England Institute of Art", "priority": 903, "external_id": null}, {"id": 10846726003, "name": "New England Institute of Technology", "priority": 904, "external_id": null}, {"id": 10846727003, "name": "New Jersey City University", "priority": 905, "external_id": null}, {"id": 10846728003, "name": "New Jersey Institute of Technology", "priority": 906, "external_id": null}, {"id": 10846729003, "name": "New Mexico Highlands University", "priority": 907, "external_id": null}, {"id": 10846730003, "name": "New Mexico Institute of Mining and Technology", "priority": 908, "external_id": null}, {"id": 10846731003, "name": "New Orleans Baptist Theological Seminary", "priority": 909, "external_id": null}, {"id": 10846732003, "name": "New School", "priority": 910, "external_id": null}, {"id": 10846733003, "name": "New York Institute of Technology", "priority": 911, "external_id": null}, {"id": 10846734003, "name": "New York University", "priority": 912, "external_id": null}, {"id": 10846735003, "name": "Newberry College", "priority": 913, "external_id": null}, {"id": 10846736003, "name": "Newbury College", "priority": 914, "external_id": null}, {"id": 10846737003, "name": "Newman University", "priority": 915, "external_id": null}, {"id": 10846738003, "name": "Niagara University", "priority": 916, "external_id": null}, {"id": 10846739003, "name": "Nichols College", "priority": 917, "external_id": null}, {"id": 10846740003, "name": "North Carolina Wesleyan College", "priority": 918, "external_id": null}, {"id": 10846741003, "name": "North Central College", "priority": 919, "external_id": null}, {"id": 10846742003, "name": "North Central University", "priority": 920, "external_id": null}, {"id": 10846743003, "name": "North Greenville University", "priority": 921, "external_id": null}, {"id": 10846744003, "name": "North Park University", "priority": 922, "external_id": null}, {"id": 10846745003, "name": "Northcentral University", "priority": 923, "external_id": null}, {"id": 10846746003, "name": "Northeastern Illinois University", "priority": 924, "external_id": null}, {"id": 10846747003, "name": "Northeastern State University", "priority": 925, "external_id": null}, {"id": 10846748003, "name": "Northeastern University", "priority": 926, "external_id": null}, {"id": 10846749003, "name": "Northern Kentucky University", "priority": 927, "external_id": null}, {"id": 10846750003, "name": "Northern Michigan University", "priority": 928, "external_id": null}, {"id": 10846751003, "name": "Northern New Mexico College", "priority": 929, "external_id": null}, {"id": 10846752003, "name": "Northern State University", "priority": 930, "external_id": null}, {"id": 10846753003, "name": "Northland College", "priority": 931, "external_id": null}, {"id": 10846754003, "name": "Northwest Christian University", "priority": 932, "external_id": null}, {"id": 10846755003, "name": "Northwest Florida State College", "priority": 933, "external_id": null}, {"id": 10846756003, "name": "Northwest Missouri State University", "priority": 934, "external_id": null}, {"id": 10846757003, "name": "Northwest Nazarene University", "priority": 935, "external_id": null}, {"id": 10846758003, "name": "Northwest University", "priority": 936, "external_id": null}, {"id": 10846759003, "name": "Northwestern College", "priority": 937, "external_id": null}, {"id": 10846760003, "name": "Northwestern Health Sciences University", "priority": 938, "external_id": null}, {"id": 10846761003, "name": "Northwestern Oklahoma State University", "priority": 939, "external_id": null}, {"id": 10846762003, "name": "Northwood University", "priority": 940, "external_id": null}, {"id": 10846763003, "name": "Norwich University", "priority": 941, "external_id": null}, {"id": 10846764003, "name": "Notre Dame College of Ohio", "priority": 942, "external_id": null}, {"id": 10846765003, "name": "Notre Dame de Namur University", "priority": 943, "external_id": null}, {"id": 10846766003, "name": "Notre Dame of Maryland University", "priority": 944, "external_id": null}, {"id": 10846767003, "name": "Nova Scotia College of Art and Design", "priority": 945, "external_id": null}, {"id": 10846768003, "name": "Nova Southeastern University", "priority": 946, "external_id": null}, {"id": 10846769003, "name": "Nyack College", "priority": 947, "external_id": null}, {"id": 10846770003, "name": "Oakland City University", "priority": 948, "external_id": null}, {"id": 10846771003, "name": "Oakland University", "priority": 949, "external_id": null}, {"id": 10846772003, "name": "Oakwood University", "priority": 950, "external_id": null}, {"id": 10846773003, "name": "Oberlin College", "priority": 951, "external_id": null}, {"id": 10846774003, "name": "Occidental College", "priority": 952, "external_id": null}, {"id": 10846775003, "name": "Oglala Lakota College", "priority": 953, "external_id": null}, {"id": 10846776003, "name": "North Carolina A&T State University", "priority": 954, "external_id": null}, {"id": 10846777003, "name": "Northern Illinois University", "priority": 955, "external_id": null}, {"id": 10846778003, "name": "North Dakota State University", "priority": 956, "external_id": null}, {"id": 10846779003, "name": "Nicholls State University", "priority": 957, "external_id": null}, {"id": 10846780003, "name": "North Carolina Central University", "priority": 958, "external_id": null}, {"id": 10846781003, "name": "Norfolk State University", "priority": 959, "external_id": null}, {"id": 10846782003, "name": "Northwestern State University of Louisiana", "priority": 960, "external_id": null}, {"id": 10846783003, "name": "Northern Arizona University", "priority": 961, "external_id": null}, {"id": 10846784003, "name": "North Carolina State University - Raleigh", "priority": 962, "external_id": null}, {"id": 10846785003, "name": "Northwestern University", "priority": 963, "external_id": null}, {"id": 10846786003, "name": "Oglethorpe University", "priority": 964, "external_id": null}, {"id": 10846787003, "name": "Ohio Christian University", "priority": 965, "external_id": null}, {"id": 10846788003, "name": "Ohio Dominican University", "priority": 966, "external_id": null}, {"id": 10846789003, "name": "Ohio Northern University", "priority": 967, "external_id": null}, {"id": 10846790003, "name": "Ohio Valley University", "priority": 968, "external_id": null}, {"id": 10846791003, "name": "Ohio Wesleyan University", "priority": 969, "external_id": null}, {"id": 10846792003, "name": "Oklahoma Baptist University", "priority": 970, "external_id": null}, {"id": 10846793003, "name": "Oklahoma Christian University", "priority": 971, "external_id": null}, {"id": 10846794003, "name": "Oklahoma City University", "priority": 972, "external_id": null}, {"id": 10846795003, "name": "Oklahoma Panhandle State University", "priority": 973, "external_id": null}, {"id": 10846796003, "name": "Oklahoma State University Institute of Technology - Okmulgee", "priority": 974, "external_id": null}, {"id": 10846797003, "name": "Oklahoma State University - Oklahoma City", "priority": 975, "external_id": null}, {"id": 10846798003, "name": "Oklahoma Wesleyan University", "priority": 976, "external_id": null}, {"id": 10846799003, "name": "Olivet College", "priority": 977, "external_id": null}, {"id": 10846800003, "name": "Olivet Nazarene University", "priority": 978, "external_id": null}, {"id": 10846801003, "name": "Olympic College", "priority": 979, "external_id": null}, {"id": 10846802003, "name": "Oral Roberts University", "priority": 980, "external_id": null}, {"id": 10846803003, "name": "Oregon College of Art and Craft", "priority": 981, "external_id": null}, {"id": 10846804003, "name": "Oregon Health and Science University", "priority": 982, "external_id": null}, {"id": 10846805003, "name": "Oregon Institute of Technology", "priority": 983, "external_id": null}, {"id": 10846806003, "name": "Otis College of Art and Design", "priority": 984, "external_id": null}, {"id": 10846807003, "name": "Ottawa University", "priority": 985, "external_id": null}, {"id": 10846808003, "name": "Otterbein University", "priority": 986, "external_id": null}, {"id": 10846809003, "name": "Ouachita Baptist University", "priority": 987, "external_id": null}, {"id": 10846810003, "name": "Our Lady of Holy Cross College", "priority": 988, "external_id": null}, {"id": 10846811003, "name": "Our Lady of the Lake College", "priority": 989, "external_id": null}, {"id": 10846812003, "name": "Our Lady of the Lake University", "priority": 990, "external_id": null}, {"id": 10846813003, "name": "Pace University", "priority": 991, "external_id": null}, {"id": 10846814003, "name": "Pacific Lutheran University", "priority": 992, "external_id": null}, {"id": 10846815003, "name": "Pacific Northwest College of Art", "priority": 993, "external_id": null}, {"id": 10846816003, "name": "Pacific Oaks College", "priority": 994, "external_id": null}, {"id": 10846817003, "name": "Pacific Union College", "priority": 995, "external_id": null}, {"id": 10846818003, "name": "Pacific University", "priority": 996, "external_id": null}, {"id": 10846819003, "name": "Paine College", "priority": 997, "external_id": null}, {"id": 10846820003, "name": "Palm Beach Atlantic University", "priority": 998, "external_id": null}, {"id": 10846821003, "name": "Palmer College of Chiropractic", "priority": 999, "external_id": null}, {"id": 10846822003, "name": "Park University", "priority": 1000, "external_id": null}, {"id": 10846823003, "name": "Parker University", "priority": 1001, "external_id": null}, {"id": 10846824003, "name": "Patten University", "priority": 1002, "external_id": null}, {"id": 10846825003, "name": "Paul Smith's College", "priority": 1003, "external_id": null}, {"id": 10846826003, "name": "Peirce College", "priority": 1004, "external_id": null}, {"id": 10846827003, "name": "Peninsula College", "priority": 1005, "external_id": null}, {"id": 10846828003, "name": "Pennsylvania College of Art and Design", "priority": 1006, "external_id": null}, {"id": 10846829003, "name": "Pennsylvania College of Technology", "priority": 1007, "external_id": null}, {"id": 10846830003, "name": "Pennsylvania State University - Erie, The Behrend College", "priority": 1008, "external_id": null}, {"id": 10846831003, "name": "Pennsylvania State University - Harrisburg", "priority": 1009, "external_id": null}, {"id": 10846832003, "name": "Pepperdine University", "priority": 1010, "external_id": null}, {"id": 10846833003, "name": "Peru State College", "priority": 1011, "external_id": null}, {"id": 10846834003, "name": "Pfeiffer University", "priority": 1012, "external_id": null}, {"id": 10846835003, "name": "Philadelphia University", "priority": 1013, "external_id": null}, {"id": 10846836003, "name": "Philander Smith College", "priority": 1014, "external_id": null}, {"id": 10846837003, "name": "Piedmont College", "priority": 1015, "external_id": null}, {"id": 10846838003, "name": "Pine Manor College", "priority": 1016, "external_id": null}, {"id": 10846839003, "name": "Pittsburg State University", "priority": 1017, "external_id": null}, {"id": 10846840003, "name": "Pitzer College", "priority": 1018, "external_id": null}, {"id": 10846841003, "name": "Plaza College", "priority": 1019, "external_id": null}, {"id": 10846842003, "name": "Plymouth State University", "priority": 1020, "external_id": null}, {"id": 10846843003, "name": "Point Loma Nazarene University", "priority": 1021, "external_id": null}, {"id": 10846844003, "name": "Point Park University", "priority": 1022, "external_id": null}, {"id": 10846845003, "name": "Point University", "priority": 1023, "external_id": null}, {"id": 10846846003, "name": "Polytechnic Institute of New York University", "priority": 1024, "external_id": null}, {"id": 10846847003, "name": "Pomona College", "priority": 1025, "external_id": null}, {"id": 10846848003, "name": "Pontifical Catholic University of Puerto Rico", "priority": 1026, "external_id": null}, {"id": 10846849003, "name": "Pontifical College Josephinum", "priority": 1027, "external_id": null}, {"id": 10846850003, "name": "Post University", "priority": 1028, "external_id": null}, {"id": 10846851003, "name": "Potomac College", "priority": 1029, "external_id": null}, {"id": 10846852003, "name": "Pratt Institute", "priority": 1030, "external_id": null}, {"id": 10846853003, "name": "Prescott College", "priority": 1031, "external_id": null}, {"id": 10846854003, "name": "Presentation College", "priority": 1032, "external_id": null}, {"id": 10846855003, "name": "Principia College", "priority": 1033, "external_id": null}, {"id": 10846856003, "name": "Providence College", "priority": 1034, "external_id": null}, {"id": 10846857003, "name": "Puerto Rico Conservatory of Music", "priority": 1035, "external_id": null}, {"id": 10846858003, "name": "Purchase College - SUNY", "priority": 1036, "external_id": null}, {"id": 10846859003, "name": "Purdue University - Calumet", "priority": 1037, "external_id": null}, {"id": 10846860003, "name": "Purdue University - North Central", "priority": 1038, "external_id": null}, {"id": 10846861003, "name": "Queens University of Charlotte", "priority": 1039, "external_id": null}, {"id": 10846862003, "name": "Oklahoma State University", "priority": 1040, "external_id": null}, {"id": 10846863003, "name": "Oregon State University", "priority": 1041, "external_id": null}, {"id": 10846864003, "name": "Portland State University", "priority": 1042, "external_id": null}, {"id": 10846865003, "name": "Old Dominion University", "priority": 1043, "external_id": null}, {"id": 10846866003, "name": "Prairie View A&M University", "priority": 1044, "external_id": null}, {"id": 10846867003, "name": "Presbyterian College", "priority": 1045, "external_id": null}, {"id": 10846868003, "name": "Purdue University - West Lafayette", "priority": 1046, "external_id": null}, {"id": 10846869003, "name": "Ohio University", "priority": 1047, "external_id": null}, {"id": 10846870003, "name": "Princeton University", "priority": 1048, "external_id": null}, {"id": 10846871003, "name": "Quincy University", "priority": 1049, "external_id": null}, {"id": 10846872003, "name": "Quinnipiac University", "priority": 1050, "external_id": null}, {"id": 10846873003, "name": "Radford University", "priority": 1051, "external_id": null}, {"id": 10846874003, "name": "Ramapo College of New Jersey", "priority": 1052, "external_id": null}, {"id": 10846875003, "name": "Randolph College", "priority": 1053, "external_id": null}, {"id": 10846876003, "name": "Randolph-Macon College", "priority": 1054, "external_id": null}, {"id": 10846877003, "name": "Ranken Technical College", "priority": 1055, "external_id": null}, {"id": 10846878003, "name": "Reed College", "priority": 1056, "external_id": null}, {"id": 10846879003, "name": "Regent University", "priority": 1057, "external_id": null}, {"id": 10846880003, "name": "Regent's American College London", "priority": 1058, "external_id": null}, {"id": 10846881003, "name": "Regis College", "priority": 1059, "external_id": null}, {"id": 10846882003, "name": "Regis University", "priority": 1060, "external_id": null}, {"id": 10846883003, "name": "Reinhardt University", "priority": 1061, "external_id": null}, {"id": 10846884003, "name": "Rensselaer Polytechnic Institute", "priority": 1062, "external_id": null}, {"id": 10846885003, "name": "Research College of Nursing", "priority": 1063, "external_id": null}, {"id": 10846886003, "name": "Resurrection University", "priority": 1064, "external_id": null}, {"id": 10846887003, "name": "Rhode Island College", "priority": 1065, "external_id": null}, {"id": 10846888003, "name": "Rhode Island School of Design", "priority": 1066, "external_id": null}, {"id": 10846889003, "name": "Rhodes College", "priority": 1067, "external_id": null}, {"id": 10846890003, "name": "Richard Stockton College of New Jersey", "priority": 1068, "external_id": null}, {"id": 10846891003, "name": "Richmond - The American International University in London", "priority": 1069, "external_id": null}, {"id": 10846892003, "name": "Rider University", "priority": 1070, "external_id": null}, {"id": 10846893003, "name": "Ringling College of Art and Design", "priority": 1071, "external_id": null}, {"id": 10846894003, "name": "Ripon College", "priority": 1072, "external_id": null}, {"id": 10846895003, "name": "Rivier University", "priority": 1073, "external_id": null}, {"id": 10846896003, "name": "Roanoke College", "priority": 1074, "external_id": null}, {"id": 10846897003, "name": "Robert B. Miller College", "priority": 1075, "external_id": null}, {"id": 10846898003, "name": "Roberts Wesleyan College", "priority": 1076, "external_id": null}, {"id": 10846899003, "name": "Rochester College", "priority": 1077, "external_id": null}, {"id": 10846900003, "name": "Rochester Institute of Technology", "priority": 1078, "external_id": null}, {"id": 10846901003, "name": "Rockford University", "priority": 1079, "external_id": null}, {"id": 10846902003, "name": "Rockhurst University", "priority": 1080, "external_id": null}, {"id": 10846903003, "name": "Rocky Mountain College", "priority": 1081, "external_id": null}, {"id": 10846904003, "name": "Rocky Mountain College of Art and Design", "priority": 1082, "external_id": null}, {"id": 10846905003, "name": "Roger Williams University", "priority": 1083, "external_id": null}, {"id": 10846906003, "name": "Rogers State University", "priority": 1084, "external_id": null}, {"id": 10846907003, "name": "Rollins College", "priority": 1085, "external_id": null}, {"id": 10846908003, "name": "Roosevelt University", "priority": 1086, "external_id": null}, {"id": 10846909003, "name": "Rosalind Franklin University of Medicine and Science", "priority": 1087, "external_id": null}, {"id": 10846910003, "name": "Rose-Hulman Institute of Technology", "priority": 1088, "external_id": null}, {"id": 10846911003, "name": "Rosemont College", "priority": 1089, "external_id": null}, {"id": 10846912003, "name": "Rowan University", "priority": 1090, "external_id": null}, {"id": 10846913003, "name": "Rush University", "priority": 1091, "external_id": null}, {"id": 10846914003, "name": "Rust College", "priority": 1092, "external_id": null}, {"id": 10846915003, "name": "Rutgers, the State University of New Jersey - Camden", "priority": 1093, "external_id": null}, {"id": 10846916003, "name": "Rutgers, the State University of New Jersey - Newark", "priority": 1094, "external_id": null}, {"id": 10846917003, "name": "Ryerson University", "priority": 1095, "external_id": null}, {"id": 10846918003, "name": "Sacred Heart Major Seminary", "priority": 1096, "external_id": null}, {"id": 10846919003, "name": "Saginaw Valley State University", "priority": 1097, "external_id": null}, {"id": 10846920003, "name": "Salem College", "priority": 1098, "external_id": null}, {"id": 10846921003, "name": "Salem International University", "priority": 1099, "external_id": null}, {"id": 10846922003, "name": "Salem State University", "priority": 1100, "external_id": null}, {"id": 10846923003, "name": "Salisbury University", "priority": 1101, "external_id": null}, {"id": 10846924003, "name": "Salish Kootenai College", "priority": 1102, "external_id": null}, {"id": 10846925003, "name": "Salve Regina University", "priority": 1103, "external_id": null}, {"id": 10846926003, "name": "Samuel Merritt University", "priority": 1104, "external_id": null}, {"id": 10846927003, "name": "San Diego Christian College", "priority": 1105, "external_id": null}, {"id": 10846928003, "name": "San Francisco Art Institute", "priority": 1106, "external_id": null}, {"id": 10846929003, "name": "San Francisco Conservatory of Music", "priority": 1107, "external_id": null}, {"id": 10846930003, "name": "San Francisco State University", "priority": 1108, "external_id": null}, {"id": 10846931003, "name": "Sanford College of Nursing", "priority": 1109, "external_id": null}, {"id": 10846932003, "name": "Santa Clara University", "priority": 1110, "external_id": null}, {"id": 10846933003, "name": "Santa Fe University of Art and Design", "priority": 1111, "external_id": null}, {"id": 10846934003, "name": "Sarah Lawrence College", "priority": 1112, "external_id": null}, {"id": 10846935003, "name": "Savannah College of Art and Design", "priority": 1113, "external_id": null}, {"id": 10846936003, "name": "School of the Art Institute of Chicago", "priority": 1114, "external_id": null}, {"id": 10846937003, "name": "School of Visual Arts", "priority": 1115, "external_id": null}, {"id": 10846938003, "name": "Schreiner University", "priority": 1116, "external_id": null}, {"id": 10846939003, "name": "Scripps College", "priority": 1117, "external_id": null}, {"id": 10846940003, "name": "Seattle Pacific University", "priority": 1118, "external_id": null}, {"id": 10846941003, "name": "Seattle University", "priority": 1119, "external_id": null}, {"id": 10846942003, "name": "Seton Hall University", "priority": 1120, "external_id": null}, {"id": 10846943003, "name": "Seton Hill University", "priority": 1121, "external_id": null}, {"id": 10846944003, "name": "Sewanee - University of the South", "priority": 1122, "external_id": null}, {"id": 10846945003, "name": "Shaw University", "priority": 1123, "external_id": null}, {"id": 10846946003, "name": "Shawnee State University", "priority": 1124, "external_id": null}, {"id": 10846947003, "name": "Shenandoah University", "priority": 1125, "external_id": null}, {"id": 10846948003, "name": "Shepherd University", "priority": 1126, "external_id": null}, {"id": 10846949003, "name": "Shimer College", "priority": 1127, "external_id": null}, {"id": 10846950003, "name": "Sacred Heart University", "priority": 1128, "external_id": null}, {"id": 10846951003, "name": "Robert Morris University", "priority": 1129, "external_id": null}, {"id": 10846952003, "name": "Sam Houston State University", "priority": 1130, "external_id": null}, {"id": 10846953003, "name": "Samford University", "priority": 1131, "external_id": null}, {"id": 10846954003, "name": "Savannah State University", "priority": 1132, "external_id": null}, {"id": 10846955003, "name": "San Jose State University", "priority": 1133, "external_id": null}, {"id": 10846956003, "name": "Rutgers, the State University of New Jersey - New Brunswick", "priority": 1134, "external_id": null}, {"id": 10846957003, "name": "San Diego State University", "priority": 1135, "external_id": null}, {"id": 10846958003, "name": "Shippensburg University of Pennsylvania", "priority": 1136, "external_id": null}, {"id": 10846959003, "name": "Shorter University", "priority": 1137, "external_id": null}, {"id": 10846960003, "name": "Siena College", "priority": 1138, "external_id": null}, {"id": 10846961003, "name": "Siena Heights University", "priority": 1139, "external_id": null}, {"id": 10846962003, "name": "Sierra Nevada College", "priority": 1140, "external_id": null}, {"id": 10846963003, "name": "Silver Lake College", "priority": 1141, "external_id": null}, {"id": 10846964003, "name": "Simmons College", "priority": 1142, "external_id": null}, {"id": 10846965003, "name": "Simon Fraser University", "priority": 1143, "external_id": null}, {"id": 10846966003, "name": "Simpson College", "priority": 1144, "external_id": null}, {"id": 10846967003, "name": "Simpson University", "priority": 1145, "external_id": null}, {"id": 10846968003, "name": "Sinte Gleska University", "priority": 1146, "external_id": null}, {"id": 10846969003, "name": "Sitting Bull College", "priority": 1147, "external_id": null}, {"id": 10846970003, "name": "Skidmore College", "priority": 1148, "external_id": null}, {"id": 10846971003, "name": "Slippery Rock University of Pennsylvania", "priority": 1149, "external_id": null}, {"id": 10846972003, "name": "Smith College", "priority": 1150, "external_id": null}, {"id": 10846973003, "name": "Sojourner-Douglass College", "priority": 1151, "external_id": null}, {"id": 10846974003, "name": "Soka University of America", "priority": 1152, "external_id": null}, {"id": 10846975003, "name": "Sonoma State University", "priority": 1153, "external_id": null}, {"id": 10846976003, "name": "South College", "priority": 1154, "external_id": null}, {"id": 10846977003, "name": "South Dakota School of Mines and Technology", "priority": 1155, "external_id": null}, {"id": 10846978003, "name": "South Seattle Community College", "priority": 1156, "external_id": null}, {"id": 10846979003, "name": "South Texas College", "priority": 1157, "external_id": null}, {"id": 10846980003, "name": "South University", "priority": 1158, "external_id": null}, {"id": 10846981003, "name": "Southeastern Oklahoma State University", "priority": 1159, "external_id": null}, {"id": 10846982003, "name": "Southeastern University", "priority": 1160, "external_id": null}, {"id": 10846983003, "name": "Southern Adventist University", "priority": 1161, "external_id": null}, {"id": 10846984003, "name": "Southern Arkansas University", "priority": 1162, "external_id": null}, {"id": 10846985003, "name": "Southern Baptist Theological Seminary", "priority": 1163, "external_id": null}, {"id": 10846986003, "name": "Southern California Institute of Architecture", "priority": 1164, "external_id": null}, {"id": 10846987003, "name": "Southern Connecticut State University", "priority": 1165, "external_id": null}, {"id": 10846988003, "name": "Southern Illinois University - Edwardsville", "priority": 1166, "external_id": null}, {"id": 10846989003, "name": "Southern Nazarene University", "priority": 1167, "external_id": null}, {"id": 10846990003, "name": "Southern New Hampshire University", "priority": 1168, "external_id": null}, {"id": 10846991003, "name": "Southern Oregon University", "priority": 1169, "external_id": null}, {"id": 10846992003, "name": "Southern Polytechnic State University", "priority": 1170, "external_id": null}, {"id": 10846993003, "name": "Southern University - New Orleans", "priority": 1171, "external_id": null}, {"id": 10846994003, "name": "Southern Vermont College", "priority": 1172, "external_id": null}, {"id": 10846995003, "name": "Southern Wesleyan University", "priority": 1173, "external_id": null}, {"id": 10846996003, "name": "Southwest Baptist University", "priority": 1174, "external_id": null}, {"id": 10846997003, "name": "Southwest Minnesota State University", "priority": 1175, "external_id": null}, {"id": 10846998003, "name": "Southwest University of Visual Arts", "priority": 1176, "external_id": null}, {"id": 10846999003, "name": "Southwestern Adventist University", "priority": 1177, "external_id": null}, {"id": 10847000003, "name": "Southwestern Assemblies of God University", "priority": 1178, "external_id": null}, {"id": 10847001003, "name": "Southwestern Christian College", "priority": 1179, "external_id": null}, {"id": 10847002003, "name": "Southwestern Christian University", "priority": 1180, "external_id": null}, {"id": 10847003003, "name": "Southwestern College", "priority": 1181, "external_id": null}, {"id": 10847004003, "name": "Southwestern Oklahoma State University", "priority": 1182, "external_id": null}, {"id": 10847005003, "name": "Southwestern University", "priority": 1183, "external_id": null}, {"id": 10847006003, "name": "Spalding University", "priority": 1184, "external_id": null}, {"id": 10847007003, "name": "Spelman College", "priority": 1185, "external_id": null}, {"id": 10847008003, "name": "Spring Arbor University", "priority": 1186, "external_id": null}, {"id": 10847009003, "name": "Spring Hill College", "priority": 1187, "external_id": null}, {"id": 10847010003, "name": "Springfield College", "priority": 1188, "external_id": null}, {"id": 10847011003, "name": "St. Ambrose University", "priority": 1189, "external_id": null}, {"id": 10847012003, "name": "St. Anselm College", "priority": 1190, "external_id": null}, {"id": 10847013003, "name": "St. Anthony College of Nursing", "priority": 1191, "external_id": null}, {"id": 10847014003, "name": "St. Augustine College", "priority": 1192, "external_id": null}, {"id": 10847015003, "name": "St. Augustine's University", "priority": 1193, "external_id": null}, {"id": 10847016003, "name": "St. Bonaventure University", "priority": 1194, "external_id": null}, {"id": 10847017003, "name": "St. Catharine College", "priority": 1195, "external_id": null}, {"id": 10847018003, "name": "St. Catherine University", "priority": 1196, "external_id": null}, {"id": 10847019003, "name": "St. Charles Borromeo Seminary", "priority": 1197, "external_id": null}, {"id": 10847020003, "name": "St. Cloud State University", "priority": 1198, "external_id": null}, {"id": 10847021003, "name": "St. Edward's University", "priority": 1199, "external_id": null}, {"id": 10847022003, "name": "St. Francis College", "priority": 1200, "external_id": null}, {"id": 10847023003, "name": "St. Francis Medical Center College of Nursing", "priority": 1201, "external_id": null}, {"id": 10847024003, "name": "St. Gregory's University", "priority": 1202, "external_id": null}, {"id": 10847025003, "name": "St. John Fisher College", "priority": 1203, "external_id": null}, {"id": 10847026003, "name": "St. John Vianney College Seminary", "priority": 1204, "external_id": null}, {"id": 10847027003, "name": "St. John's College", "priority": 1205, "external_id": null}, {"id": 10847028003, "name": "St. John's University", "priority": 1206, "external_id": null}, {"id": 10847029003, "name": "St. Joseph Seminary College", "priority": 1207, "external_id": null}, {"id": 10847030003, "name": "St. Joseph's College", "priority": 1208, "external_id": null}, {"id": 10847031003, "name": "St. Joseph's College New York", "priority": 1209, "external_id": null}, {"id": 10847032003, "name": "St. Joseph's University", "priority": 1210, "external_id": null}, {"id": 10847033003, "name": "St. Lawrence University", "priority": 1211, "external_id": null}, {"id": 10847034003, "name": "St. Leo University", "priority": 1212, "external_id": null}, {"id": 10847035003, "name": "Southern University and A&M College", "priority": 1213, "external_id": null}, {"id": 10847036003, "name": "Southern Methodist University", "priority": 1214, "external_id": null}, {"id": 10847037003, "name": "Southeast Missouri State University", "priority": 1215, "external_id": null}, {"id": 10847038003, "name": "Southern Utah University", "priority": 1216, "external_id": null}, {"id": 10847039003, "name": "South Dakota State University", "priority": 1217, "external_id": null}, {"id": 10847040003, "name": "St. Francis University", "priority": 1218, "external_id": null}, {"id": 10847041003, "name": "Southeastern Louisiana University", "priority": 1219, "external_id": null}, {"id": 10847042003, "name": "Southern Illinois University - Carbondale", "priority": 1220, "external_id": null}, {"id": 10847043003, "name": "St. Louis College of Pharmacy", "priority": 1221, "external_id": null}, {"id": 10847044003, "name": "St. Louis University", "priority": 1222, "external_id": null}, {"id": 10847045003, "name": "St. Luke's College of Health Sciences", "priority": 1223, "external_id": null}, {"id": 10847046003, "name": "St. Martin's University", "priority": 1224, "external_id": null}, {"id": 10847047003, "name": "St. Mary's College", "priority": 1225, "external_id": null}, {"id": 10847048003, "name": "St. Mary's College of California", "priority": 1226, "external_id": null}, {"id": 10847049003, "name": "St. Mary's College of Maryland", "priority": 1227, "external_id": null}, {"id": 10847050003, "name": "St. Mary's Seminary and University", "priority": 1228, "external_id": null}, {"id": 10847051003, "name": "St. Mary's University of Minnesota", "priority": 1229, "external_id": null}, {"id": 10847052003, "name": "St. Mary's University of San Antonio", "priority": 1230, "external_id": null}, {"id": 10847053003, "name": "St. Mary-of-the-Woods College", "priority": 1231, "external_id": null}, {"id": 10847054003, "name": "St. Michael's College", "priority": 1232, "external_id": null}, {"id": 10847055003, "name": "St. Norbert College", "priority": 1233, "external_id": null}, {"id": 10847056003, "name": "St. Olaf College", "priority": 1234, "external_id": null}, {"id": 10847057003, "name": "St. Paul's College", "priority": 1235, "external_id": null}, {"id": 10847058003, "name": "St. Peter's University", "priority": 1236, "external_id": null}, {"id": 10847059003, "name": "St. Petersburg College", "priority": 1237, "external_id": null}, {"id": 10847060003, "name": "St. Thomas Aquinas College", "priority": 1238, "external_id": null}, {"id": 10847061003, "name": "St. Thomas University", "priority": 1239, "external_id": null}, {"id": 10847062003, "name": "St. Vincent College", "priority": 1240, "external_id": null}, {"id": 10847063003, "name": "St. Xavier University", "priority": 1241, "external_id": null}, {"id": 10847064003, "name": "Stephens College", "priority": 1242, "external_id": null}, {"id": 10847065003, "name": "Sterling College", "priority": 1243, "external_id": null}, {"id": 10847066003, "name": "Stevens Institute of Technology", "priority": 1244, "external_id": null}, {"id": 10847067003, "name": "Stevenson University", "priority": 1245, "external_id": null}, {"id": 10847068003, "name": "Stillman College", "priority": 1246, "external_id": null}, {"id": 10847069003, "name": "Stonehill College", "priority": 1247, "external_id": null}, {"id": 10847070003, "name": "Strayer University", "priority": 1248, "external_id": null}, {"id": 10847071003, "name": "Suffolk University", "priority": 1249, "external_id": null}, {"id": 10847072003, "name": "Sul Ross State University", "priority": 1250, "external_id": null}, {"id": 10847073003, "name": "Sullivan University", "priority": 1251, "external_id": null}, {"id": 10847074003, "name": "SUNY Buffalo State", "priority": 1252, "external_id": null}, {"id": 10847075003, "name": "SUNY College of Agriculture and Technology - Cobleskill", "priority": 1253, "external_id": null}, {"id": 10847076003, "name": "SUNY College of Environmental Science and Forestry", "priority": 1254, "external_id": null}, {"id": 10847077003, "name": "SUNY College of Technology - Alfred", "priority": 1255, "external_id": null}, {"id": 10847078003, "name": "SUNY College of Technology - Canton", "priority": 1256, "external_id": null}, {"id": 10847079003, "name": "SUNY College of Technology - Delhi", "priority": 1257, "external_id": null}, {"id": 10847080003, "name": "SUNY College - Cortland", "priority": 1258, "external_id": null}, {"id": 10847081003, "name": "SUNY College - Old Westbury", "priority": 1259, "external_id": null}, {"id": 10847082003, "name": "SUNY College - Oneonta", "priority": 1260, "external_id": null}, {"id": 10847083003, "name": "SUNY College - Potsdam", "priority": 1261, "external_id": null}, {"id": 10847084003, "name": "SUNY Downstate Medical Center", "priority": 1262, "external_id": null}, {"id": 10847085003, "name": "SUNY Empire State College", "priority": 1263, "external_id": null}, {"id": 10847086003, "name": "SUNY Institute of Technology - Utica/Rome", "priority": 1264, "external_id": null}, {"id": 10847087003, "name": "SUNY Maritime College", "priority": 1265, "external_id": null}, {"id": 10847088003, "name": "SUNY Upstate Medical University", "priority": 1266, "external_id": null}, {"id": 10847089003, "name": "SUNY - Fredonia", "priority": 1267, "external_id": null}, {"id": 10847090003, "name": "SUNY - Geneseo", "priority": 1268, "external_id": null}, {"id": 10847091003, "name": "SUNY - New Paltz", "priority": 1269, "external_id": null}, {"id": 10847092003, "name": "SUNY - Oswego", "priority": 1270, "external_id": null}, {"id": 10847093003, "name": "SUNY - Plattsburgh", "priority": 1271, "external_id": null}, {"id": 10847094003, "name": "Swarthmore College", "priority": 1272, "external_id": null}, {"id": 10847095003, "name": "Sweet Briar College", "priority": 1273, "external_id": null}, {"id": 10847096003, "name": "Tabor College", "priority": 1274, "external_id": null}, {"id": 10847097003, "name": "Talladega College", "priority": 1275, "external_id": null}, {"id": 10847098003, "name": "Tarleton State University", "priority": 1276, "external_id": null}, {"id": 10847099003, "name": "Taylor University", "priority": 1277, "external_id": null}, {"id": 10847100003, "name": "Tennessee Wesleyan College", "priority": 1278, "external_id": null}, {"id": 10847101003, "name": "Texas A&M International University", "priority": 1279, "external_id": null}, {"id": 10847102003, "name": "Texas A&M University - Commerce", "priority": 1280, "external_id": null}, {"id": 10847103003, "name": "Texas A&M University - Corpus Christi", "priority": 1281, "external_id": null}, {"id": 10847104003, "name": "Texas A&M University - Galveston", "priority": 1282, "external_id": null}, {"id": 10847105003, "name": "Texas A&M University - Kingsville", "priority": 1283, "external_id": null}, {"id": 10847106003, "name": "Texas A&M University - Texarkana", "priority": 1284, "external_id": null}, {"id": 10847107003, "name": "Texas College", "priority": 1285, "external_id": null}, {"id": 10847108003, "name": "Texas Lutheran University", "priority": 1286, "external_id": null}, {"id": 10847109003, "name": "Bucknell University", "priority": 1287, "external_id": null}, {"id": 10847110003, "name": "Butler University", "priority": 1288, "external_id": null}, {"id": 10847111003, "name": "Stephen F. Austin State University", "priority": 1289, "external_id": null}, {"id": 10847112003, "name": "Texas A&M University - College Station", "priority": 1290, "external_id": null}, {"id": 10847113003, "name": "Stanford University", "priority": 1291, "external_id": null}, {"id": 10847114003, "name": "Stetson University", "priority": 1292, "external_id": null}, {"id": 10847115003, "name": "Stony Brook University - SUNY", "priority": 1293, "external_id": null}, {"id": 10847116003, "name": "Syracuse University", "priority": 1294, "external_id": null}, {"id": 10847117003, "name": "Texas Christian University", "priority": 1295, "external_id": null}, {"id": 10847118003, "name": "Temple University", "priority": 1296, "external_id": null}, {"id": 10847119003, "name": "Clemson University", "priority": 1297, "external_id": null}, {"id": 10847120003, "name": "Texas Southern University", "priority": 1298, "external_id": null}, {"id": 10847121003, "name": "Austin Peay State University", "priority": 1299, "external_id": null}, {"id": 10847122003, "name": "Tennessee State University", "priority": 1300, "external_id": null}, {"id": 10847123003, "name": "Ball State University", "priority": 1301, "external_id": null}, {"id": 10847124003, "name": "Texas Tech University Health Sciences Center", "priority": 1302, "external_id": null}, {"id": 10847125003, "name": "Texas Wesleyan University", "priority": 1303, "external_id": null}, {"id": 10847126003, "name": "Texas Woman's University", "priority": 1304, "external_id": null}, {"id": 10847127003, "name": "The Catholic University of America", "priority": 1305, "external_id": null}, {"id": 10847128003, "name": "The Sage Colleges", "priority": 1306, "external_id": null}, {"id": 10847129003, "name": "Thiel College", "priority": 1307, "external_id": null}, {"id": 10847130003, "name": "Thomas Aquinas College", "priority": 1308, "external_id": null}, {"id": 10847131003, "name": "Thomas College", "priority": 1309, "external_id": null}, {"id": 10847132003, "name": "Thomas Edison State College", "priority": 1310, "external_id": null}, {"id": 10847133003, "name": "Thomas Jefferson University", "priority": 1311, "external_id": null}, {"id": 10847134003, "name": "Thomas More College", "priority": 1312, "external_id": null}, {"id": 10847135003, "name": "Thomas More College of Liberal Arts", "priority": 1313, "external_id": null}, {"id": 10847136003, "name": "Thomas University", "priority": 1314, "external_id": null}, {"id": 10847137003, "name": "Tiffin University", "priority": 1315, "external_id": null}, {"id": 10847138003, "name": "Tilburg University", "priority": 1316, "external_id": null}, {"id": 10847139003, "name": "Toccoa Falls College", "priority": 1317, "external_id": null}, {"id": 10847140003, "name": "Tougaloo College", "priority": 1318, "external_id": null}, {"id": 10847141003, "name": "Touro College", "priority": 1319, "external_id": null}, {"id": 10847142003, "name": "Transylvania University", "priority": 1320, "external_id": null}, {"id": 10847143003, "name": "Trent University", "priority": 1321, "external_id": null}, {"id": 10847144003, "name": "Trevecca Nazarene University", "priority": 1322, "external_id": null}, {"id": 10847145003, "name": "Trident University International", "priority": 1323, "external_id": null}, {"id": 10847146003, "name": "Trine University", "priority": 1324, "external_id": null}, {"id": 10847147003, "name": "Trinity Christian College", "priority": 1325, "external_id": null}, {"id": 10847148003, "name": "Trinity College", "priority": 1326, "external_id": null}, {"id": 10847149003, "name": "Trinity College of Nursing & Health Sciences", "priority": 1327, "external_id": null}, {"id": 10847150003, "name": "Trinity International University", "priority": 1328, "external_id": null}, {"id": 10847151003, "name": "Trinity Lutheran College", "priority": 1329, "external_id": null}, {"id": 10847152003, "name": "Trinity University", "priority": 1330, "external_id": null}, {"id": 10847153003, "name": "Trinity Western University", "priority": 1331, "external_id": null}, {"id": 10847154003, "name": "Truett McConnell College", "priority": 1332, "external_id": null}, {"id": 10847155003, "name": "Truman State University", "priority": 1333, "external_id": null}, {"id": 10847156003, "name": "Tufts University", "priority": 1334, "external_id": null}, {"id": 10847157003, "name": "Tusculum College", "priority": 1335, "external_id": null}, {"id": 10847158003, "name": "Tuskegee University", "priority": 1336, "external_id": null}, {"id": 10847159003, "name": "Union College", "priority": 1337, "external_id": null}, {"id": 10847160003, "name": "Union Institute and University", "priority": 1338, "external_id": null}, {"id": 10847161003, "name": "Union University", "priority": 1339, "external_id": null}, {"id": 10847162003, "name": "United States Coast Guard Academy", "priority": 1340, "external_id": null}, {"id": 10847163003, "name": "United States International University - Kenya", "priority": 1341, "external_id": null}, {"id": 10847164003, "name": "United States Merchant Marine Academy", "priority": 1342, "external_id": null}, {"id": 10847165003, "name": "United States Sports Academy", "priority": 1343, "external_id": null}, {"id": 10847166003, "name": "Unity College", "priority": 1344, "external_id": null}, {"id": 10847167003, "name": "Universidad Adventista de las Antillas", "priority": 1345, "external_id": null}, {"id": 10847168003, "name": "Universidad del Este", "priority": 1346, "external_id": null}, {"id": 10847169003, "name": "Universidad del Turabo", "priority": 1347, "external_id": null}, {"id": 10847170003, "name": "Universidad Metropolitana", "priority": 1348, "external_id": null}, {"id": 10847171003, "name": "Universidad Politecnica De Puerto Rico", "priority": 1349, "external_id": null}, {"id": 10847172003, "name": "University of Advancing Technology", "priority": 1350, "external_id": null}, {"id": 10847173003, "name": "University of Alabama - Huntsville", "priority": 1351, "external_id": null}, {"id": 10847174003, "name": "University of Alaska - Anchorage", "priority": 1352, "external_id": null}, {"id": 10847175003, "name": "University of Alaska - Fairbanks", "priority": 1353, "external_id": null}, {"id": 10847176003, "name": "University of Alaska - Southeast", "priority": 1354, "external_id": null}, {"id": 10847177003, "name": "University of Alberta", "priority": 1355, "external_id": null}, {"id": 10847178003, "name": "University of Arkansas for Medical Sciences", "priority": 1356, "external_id": null}, {"id": 10847179003, "name": "University of Arkansas - Fort Smith", "priority": 1357, "external_id": null}, {"id": 10847180003, "name": "University of Arkansas - Little Rock", "priority": 1358, "external_id": null}, {"id": 10847181003, "name": "University of Arkansas - Monticello", "priority": 1359, "external_id": null}, {"id": 10847182003, "name": "University of Baltimore", "priority": 1360, "external_id": null}, {"id": 10847183003, "name": "University of Bridgeport", "priority": 1361, "external_id": null}, {"id": 10847184003, "name": "University of British Columbia", "priority": 1362, "external_id": null}, {"id": 10847185003, "name": "University of Calgary", "priority": 1363, "external_id": null}, {"id": 10847186003, "name": "University of California - Riverside", "priority": 1364, "external_id": null}, {"id": 10847187003, "name": "Holy Cross College", "priority": 1365, "external_id": null}, {"id": 10847188003, "name": "Towson University", "priority": 1366, "external_id": null}, {"id": 10847189003, "name": "United States Military Academy", "priority": 1367, "external_id": null}, {"id": 10847190003, "name": "The Citadel", "priority": 1368, "external_id": null}, {"id": 10847191003, "name": "Troy University", "priority": 1369, "external_id": null}, {"id": 10847192003, "name": "University of California - Davis", "priority": 1370, "external_id": null}, {"id": 10847193003, "name": "Grambling State University", "priority": 1371, "external_id": null}, {"id": 10847194003, "name": "University at Albany - SUNY", "priority": 1372, "external_id": null}, {"id": 10847195003, "name": "University at Buffalo - SUNY", "priority": 1373, "external_id": null}, {"id": 10847196003, "name": "United States Naval Academy", "priority": 1374, "external_id": null}, {"id": 10847197003, "name": "University of Arizona", "priority": 1375, "external_id": null}, {"id": 10847198003, "name": "University of California - Los Angeles", "priority": 1376, "external_id": null}, {"id": 10847199003, "name": "Florida A&M University", "priority": 1377, "external_id": null}, {"id": 10847200003, "name": "Texas State University", "priority": 1378, "external_id": null}, {"id": 10847201003, "name": "University of Alabama - Birmingham", "priority": 1379, "external_id": null}, {"id": 10847202003, "name": "University of California - Santa Cruz", "priority": 1380, "external_id": null}, {"id": 10847203003, "name": "University of Central Missouri", "priority": 1381, "external_id": null}, {"id": 10847204003, "name": "University of Central Oklahoma", "priority": 1382, "external_id": null}, {"id": 10847205003, "name": "University of Charleston", "priority": 1383, "external_id": null}, {"id": 10847206003, "name": "University of Chicago", "priority": 1384, "external_id": null}, {"id": 10847207003, "name": "University of Cincinnati - UC Blue Ash College", "priority": 1385, "external_id": null}, {"id": 10847208003, "name": "University of Colorado - Colorado Springs", "priority": 1386, "external_id": null}, {"id": 10847209003, "name": "University of Colorado - Denver", "priority": 1387, "external_id": null}, {"id": 10847210003, "name": "University of Dallas", "priority": 1388, "external_id": null}, {"id": 10847211003, "name": "University of Denver", "priority": 1389, "external_id": null}, {"id": 10847212003, "name": "University of Detroit Mercy", "priority": 1390, "external_id": null}, {"id": 10847213003, "name": "University of Dubuque", "priority": 1391, "external_id": null}, {"id": 10847214003, "name": "University of Evansville", "priority": 1392, "external_id": null}, {"id": 10847215003, "name": "University of Findlay", "priority": 1393, "external_id": null}, {"id": 10847216003, "name": "University of Great Falls", "priority": 1394, "external_id": null}, {"id": 10847217003, "name": "University of Guam", "priority": 1395, "external_id": null}, {"id": 10847218003, "name": "University of Guelph", "priority": 1396, "external_id": null}, {"id": 10847219003, "name": "University of Hartford", "priority": 1397, "external_id": null}, {"id": 10847220003, "name": "University of Hawaii - Hilo", "priority": 1398, "external_id": null}, {"id": 10847221003, "name": "University of Hawaii - Maui College", "priority": 1399, "external_id": null}, {"id": 10847222003, "name": "University of Hawaii - West Oahu", "priority": 1400, "external_id": null}, {"id": 10847223003, "name": "University of Houston - Clear Lake", "priority": 1401, "external_id": null}, {"id": 10847224003, "name": "University of Houston - Downtown", "priority": 1402, "external_id": null}, {"id": 10847225003, "name": "University of Houston - Victoria", "priority": 1403, "external_id": null}, {"id": 10847226003, "name": "University of Illinois - Chicago", "priority": 1404, "external_id": null}, {"id": 10847227003, "name": "University of Illinois - Springfield", "priority": 1405, "external_id": null}, {"id": 10847228003, "name": "University of Indianapolis", "priority": 1406, "external_id": null}, {"id": 10847229003, "name": "University of Jamestown", "priority": 1407, "external_id": null}, {"id": 10847230003, "name": "University of La Verne", "priority": 1408, "external_id": null}, {"id": 10847231003, "name": "University of Maine - Augusta", "priority": 1409, "external_id": null}, {"id": 10847232003, "name": "University of Maine - Farmington", "priority": 1410, "external_id": null}, {"id": 10847233003, "name": "University of Maine - Fort Kent", "priority": 1411, "external_id": null}, {"id": 10847234003, "name": "University of Maine - Machias", "priority": 1412, "external_id": null}, {"id": 10847235003, "name": "University of Maine - Presque Isle", "priority": 1413, "external_id": null}, {"id": 10847236003, "name": "University of Mary", "priority": 1414, "external_id": null}, {"id": 10847237003, "name": "University of Mary Hardin-Baylor", "priority": 1415, "external_id": null}, {"id": 10847238003, "name": "University of Mary Washington", "priority": 1416, "external_id": null}, {"id": 10847239003, "name": "University of Maryland - Baltimore", "priority": 1417, "external_id": null}, {"id": 10847240003, "name": "University of Maryland - Baltimore County", "priority": 1418, "external_id": null}, {"id": 10847241003, "name": "University of Maryland - Eastern Shore", "priority": 1419, "external_id": null}, {"id": 10847242003, "name": "University of Maryland - University College", "priority": 1420, "external_id": null}, {"id": 10847243003, "name": "University of Massachusetts - Boston", "priority": 1421, "external_id": null}, {"id": 10847244003, "name": "University of Massachusetts - Dartmouth", "priority": 1422, "external_id": null}, {"id": 10847245003, "name": "University of Massachusetts - Lowell", "priority": 1423, "external_id": null}, {"id": 10847246003, "name": "University of Medicine and Dentistry of New Jersey", "priority": 1424, "external_id": null}, {"id": 10847247003, "name": "University of Michigan - Dearborn", "priority": 1425, "external_id": null}, {"id": 10847248003, "name": "University of Michigan - Flint", "priority": 1426, "external_id": null}, {"id": 10847249003, "name": "University of Minnesota - Crookston", "priority": 1427, "external_id": null}, {"id": 10847250003, "name": "University of Minnesota - Duluth", "priority": 1428, "external_id": null}, {"id": 10847251003, "name": "University of Minnesota - Morris", "priority": 1429, "external_id": null}, {"id": 10847252003, "name": "University of Mississippi Medical Center", "priority": 1430, "external_id": null}, {"id": 10847253003, "name": "University of Missouri - Kansas City", "priority": 1431, "external_id": null}, {"id": 10847254003, "name": "University of Missouri - St. Louis", "priority": 1432, "external_id": null}, {"id": 10847255003, "name": "University of Mobile", "priority": 1433, "external_id": null}, {"id": 10847256003, "name": "University of Montana - Western", "priority": 1434, "external_id": null}, {"id": 10847257003, "name": "University of Montevallo", "priority": 1435, "external_id": null}, {"id": 10847258003, "name": "University of Mount Union", "priority": 1436, "external_id": null}, {"id": 10847259003, "name": "University of Nebraska Medical Center", "priority": 1437, "external_id": null}, {"id": 10847260003, "name": "University of Nebraska - Kearney", "priority": 1438, "external_id": null}, {"id": 10847261003, "name": "University of Dayton", "priority": 1439, "external_id": null}, {"id": 10847262003, "name": "University of Delaware", "priority": 1440, "external_id": null}, {"id": 10847263003, "name": "University of Florida", "priority": 1441, "external_id": null}, {"id": 10847264003, "name": "University of Iowa", "priority": 1442, "external_id": null}, {"id": 10847265003, "name": "University of Idaho", "priority": 1443, "external_id": null}, {"id": 10847266003, "name": "University of Kentucky", "priority": 1444, "external_id": null}, {"id": 10847267003, "name": "University of Massachusetts - Amherst", "priority": 1445, "external_id": null}, {"id": 10847268003, "name": "University of Maine", "priority": 1446, "external_id": null}, {"id": 10847269003, "name": "University of Michigan - Ann Arbor", "priority": 1447, "external_id": null}, {"id": 10847270003, "name": "University of Cincinnati", "priority": 1448, "external_id": null}, {"id": 10847271003, "name": "University of Miami", "priority": 1449, "external_id": null}, {"id": 10847272003, "name": "University of Louisiana - Monroe", "priority": 1450, "external_id": null}, {"id": 10847273003, "name": "University of Missouri", "priority": 1451, "external_id": null}, {"id": 10847274003, "name": "University of Mississippi", "priority": 1452, "external_id": null}, {"id": 10847275003, "name": "University of Memphis", "priority": 1453, "external_id": null}, {"id": 10847276003, "name": "University of Houston", "priority": 1454, "external_id": null}, {"id": 10847277003, "name": "University of Colorado - Boulder", "priority": 1455, "external_id": null}, {"id": 10847278003, "name": "University of Nebraska - Omaha", "priority": 1456, "external_id": null}, {"id": 10847279003, "name": "University of New Brunswick", "priority": 1457, "external_id": null}, {"id": 10847280003, "name": "University of New England", "priority": 1458, "external_id": null}, {"id": 10847281003, "name": "University of New Haven", "priority": 1459, "external_id": null}, {"id": 10847282003, "name": "University of New Orleans", "priority": 1460, "external_id": null}, {"id": 10847283003, "name": "University of North Alabama", "priority": 1461, "external_id": null}, {"id": 10847284003, "name": "University of North Carolina School of the Arts", "priority": 1462, "external_id": null}, {"id": 10847285003, "name": "University of North Carolina - Asheville", "priority": 1463, "external_id": null}, {"id": 10847286003, "name": "University of North Carolina - Greensboro", "priority": 1464, "external_id": null}, {"id": 10847287003, "name": "University of North Carolina - Pembroke", "priority": 1465, "external_id": null}, {"id": 10847288003, "name": "University of North Carolina - Wilmington", "priority": 1466, "external_id": null}, {"id": 10847289003, "name": "University of North Florida", "priority": 1467, "external_id": null}, {"id": 10847290003, "name": "University of North Georgia", "priority": 1468, "external_id": null}, {"id": 10847291003, "name": "University of Northwestern Ohio", "priority": 1469, "external_id": null}, {"id": 10847292003, "name": "University of Northwestern - St. Paul", "priority": 1470, "external_id": null}, {"id": 10847293003, "name": "University of Ottawa", "priority": 1471, "external_id": null}, {"id": 10847294003, "name": "University of Phoenix", "priority": 1472, "external_id": null}, {"id": 10847295003, "name": "University of Pikeville", "priority": 1473, "external_id": null}, {"id": 10847296003, "name": "University of Portland", "priority": 1474, "external_id": null}, {"id": 10847297003, "name": "University of Prince Edward Island", "priority": 1475, "external_id": null}, {"id": 10847298003, "name": "University of Puerto Rico - Aguadilla", "priority": 1476, "external_id": null}, {"id": 10847299003, "name": "University of Puerto Rico - Arecibo", "priority": 1477, "external_id": null}, {"id": 10847300003, "name": "University of Puerto Rico - Bayamon", "priority": 1478, "external_id": null}, {"id": 10847301003, "name": "University of Puerto Rico - Cayey", "priority": 1479, "external_id": null}, {"id": 10847302003, "name": "University of Puerto Rico - Humacao", "priority": 1480, "external_id": null}, {"id": 10847303003, "name": "University of Puerto Rico - Mayaguez", "priority": 1481, "external_id": null}, {"id": 10847304003, "name": "University of Puerto Rico - Medical Sciences Campus", "priority": 1482, "external_id": null}, {"id": 10847305003, "name": "University of Puerto Rico - Ponce", "priority": 1483, "external_id": null}, {"id": 10847306003, "name": "University of Puerto Rico - Rio Piedras", "priority": 1484, "external_id": null}, {"id": 10847307003, "name": "University of Puget Sound", "priority": 1485, "external_id": null}, {"id": 10847308003, "name": "University of Redlands", "priority": 1486, "external_id": null}, {"id": 10847309003, "name": "University of Regina", "priority": 1487, "external_id": null}, {"id": 10847310003, "name": "University of Rio Grande", "priority": 1488, "external_id": null}, {"id": 10847311003, "name": "University of Rochester", "priority": 1489, "external_id": null}, {"id": 10847312003, "name": "University of San Francisco", "priority": 1490, "external_id": null}, {"id": 10847313003, "name": "University of Saskatchewan", "priority": 1491, "external_id": null}, {"id": 10847314003, "name": "University of Science and Arts of Oklahoma", "priority": 1492, "external_id": null}, {"id": 10847315003, "name": "University of Scranton", "priority": 1493, "external_id": null}, {"id": 10847316003, "name": "University of Sioux Falls", "priority": 1494, "external_id": null}, {"id": 10847317003, "name": "University of South Carolina - Aiken", "priority": 1495, "external_id": null}, {"id": 10847318003, "name": "University of South Carolina - Beaufort", "priority": 1496, "external_id": null}, {"id": 10847319003, "name": "University of South Carolina - Upstate", "priority": 1497, "external_id": null}, {"id": 10847320003, "name": "University of South Florida - St. Petersburg", "priority": 1498, "external_id": null}, {"id": 10847321003, "name": "University of Southern Indiana", "priority": 1499, "external_id": null}, {"id": 10847322003, "name": "University of Southern Maine", "priority": 1500, "external_id": null}, {"id": 10847323003, "name": "University of St. Francis", "priority": 1501, "external_id": null}, {"id": 10847324003, "name": "University of St. Joseph", "priority": 1502, "external_id": null}, {"id": 10847325003, "name": "University of St. Mary", "priority": 1503, "external_id": null}, {"id": 10847326003, "name": "University of St. Thomas", "priority": 1504, "external_id": null}, {"id": 10847327003, "name": "University of Tampa", "priority": 1505, "external_id": null}, {"id": 10847328003, "name": "University of Texas Health Science Center - Houston", "priority": 1506, "external_id": null}, {"id": 10847329003, "name": "University of Texas Health Science Center - San Antonio", "priority": 1507, "external_id": null}, {"id": 10847330003, "name": "University of Texas Medical Branch - Galveston", "priority": 1508, "external_id": null}, {"id": 10847331003, "name": "University of Texas of the Permian Basin", "priority": 1509, "external_id": null}, {"id": 10847332003, "name": "University of Texas - Arlington", "priority": 1510, "external_id": null}, {"id": 10847333003, "name": "University of Texas - Brownsville", "priority": 1511, "external_id": null}, {"id": 10847334003, "name": "University of Texas - Pan American", "priority": 1512, "external_id": null}, {"id": 10847335003, "name": "University of Oregon", "priority": 1513, "external_id": null}, {"id": 10847336003, "name": "University of New Mexico", "priority": 1514, "external_id": null}, {"id": 10847337003, "name": "University of Pennsylvania", "priority": 1515, "external_id": null}, {"id": 10847338003, "name": "University of North Dakota", "priority": 1516, "external_id": null}, {"id": 10847339003, "name": "University of Nevada - Reno", "priority": 1517, "external_id": null}, {"id": 10847340003, "name": "University of New Hampshire", "priority": 1518, "external_id": null}, {"id": 10847341003, "name": "University of Texas - Austin", "priority": 1519, "external_id": null}, {"id": 10847342003, "name": "University of Southern Mississippi", "priority": 1520, "external_id": null}, {"id": 10847343003, "name": "University of Rhode Island", "priority": 1521, "external_id": null}, {"id": 10847344003, "name": "University of South Dakota", "priority": 1522, "external_id": null}, {"id": 10847345003, "name": "University of Tennessee", "priority": 1523, "external_id": null}, {"id": 10847346003, "name": "University of North Texas", "priority": 1524, "external_id": null}, {"id": 10847347003, "name": "University of North Carolina - Charlotte", "priority": 1525, "external_id": null}, {"id": 10847348003, "name": "University of Texas - San Antonio", "priority": 1526, "external_id": null}, {"id": 10847349003, "name": "University of Notre Dame", "priority": 1527, "external_id": null}, {"id": 10847350003, "name": "University of Southern California", "priority": 1528, "external_id": null}, {"id": 10847351003, "name": "University of Texas - Tyler", "priority": 1529, "external_id": null}, {"id": 10847352003, "name": "University of the Arts", "priority": 1530, "external_id": null}, {"id": 10847353003, "name": "University of the Cumberlands", "priority": 1531, "external_id": null}, {"id": 10847354003, "name": "University of the District of Columbia", "priority": 1532, "external_id": null}, {"id": 10847355003, "name": "University of the Ozarks", "priority": 1533, "external_id": null}, {"id": 10847356003, "name": "University of the Pacific", "priority": 1534, "external_id": null}, {"id": 10847357003, "name": "University of the Sacred Heart", "priority": 1535, "external_id": null}, {"id": 10847358003, "name": "University of the Sciences", "priority": 1536, "external_id": null}, {"id": 10847359003, "name": "University of the Southwest", "priority": 1537, "external_id": null}, {"id": 10847360003, "name": "University of the Virgin Islands", "priority": 1538, "external_id": null}, {"id": 10847361003, "name": "University of the West", "priority": 1539, "external_id": null}, {"id": 10847362003, "name": "University of Toronto", "priority": 1540, "external_id": null}, {"id": 10847363003, "name": "University of Vermont", "priority": 1541, "external_id": null}, {"id": 10847364003, "name": "University of Victoria", "priority": 1542, "external_id": null}, {"id": 10847365003, "name": "University of Virginia - Wise", "priority": 1543, "external_id": null}, {"id": 10847366003, "name": "University of Waterloo", "priority": 1544, "external_id": null}, {"id": 10847367003, "name": "University of West Alabama", "priority": 1545, "external_id": null}, {"id": 10847368003, "name": "University of West Florida", "priority": 1546, "external_id": null}, {"id": 10847369003, "name": "University of West Georgia", "priority": 1547, "external_id": null}, {"id": 10847370003, "name": "University of Windsor", "priority": 1548, "external_id": null}, {"id": 10847371003, "name": "University of Winnipeg", "priority": 1549, "external_id": null}, {"id": 10847372003, "name": "University of Wisconsin - Eau Claire", "priority": 1550, "external_id": null}, {"id": 10847373003, "name": "University of Wisconsin - Green Bay", "priority": 1551, "external_id": null}, {"id": 10847374003, "name": "University of Wisconsin - La Crosse", "priority": 1552, "external_id": null}, {"id": 10847375003, "name": "University of Wisconsin - Milwaukee", "priority": 1553, "external_id": null}, {"id": 10847376003, "name": "University of Wisconsin - Oshkosh", "priority": 1554, "external_id": null}, {"id": 10847377003, "name": "University of Wisconsin - Parkside", "priority": 1555, "external_id": null}, {"id": 10847378003, "name": "University of Wisconsin - Platteville", "priority": 1556, "external_id": null}, {"id": 10847379003, "name": "University of Wisconsin - River Falls", "priority": 1557, "external_id": null}, {"id": 10847380003, "name": "University of Wisconsin - Stevens Point", "priority": 1558, "external_id": null}, {"id": 10847381003, "name": "University of Wisconsin - Stout", "priority": 1559, "external_id": null}, {"id": 10847382003, "name": "University of Wisconsin - Superior", "priority": 1560, "external_id": null}, {"id": 10847383003, "name": "University of Wisconsin - Whitewater", "priority": 1561, "external_id": null}, {"id": 10847384003, "name": "Upper Iowa University", "priority": 1562, "external_id": null}, {"id": 10847385003, "name": "Urbana University", "priority": 1563, "external_id": null}, {"id": 10847386003, "name": "Ursinus College", "priority": 1564, "external_id": null}, {"id": 10847387003, "name": "Ursuline College", "priority": 1565, "external_id": null}, {"id": 10847388003, "name": "Utah Valley University", "priority": 1566, "external_id": null}, {"id": 10847389003, "name": "Utica College", "priority": 1567, "external_id": null}, {"id": 10847390003, "name": "Valdosta State University", "priority": 1568, "external_id": null}, {"id": 10847391003, "name": "Valley City State University", "priority": 1569, "external_id": null}, {"id": 10847392003, "name": "Valley Forge Christian College", "priority": 1570, "external_id": null}, {"id": 10847393003, "name": "VanderCook College of Music", "priority": 1571, "external_id": null}, {"id": 10847394003, "name": "Vanguard University of Southern California", "priority": 1572, "external_id": null}, {"id": 10847395003, "name": "Vassar College", "priority": 1573, "external_id": null}, {"id": 10847396003, "name": "Vaughn College of Aeronautics and Technology", "priority": 1574, "external_id": null}, {"id": 10847397003, "name": "Vermont Technical College", "priority": 1575, "external_id": null}, {"id": 10847398003, "name": "Victory University", "priority": 1576, "external_id": null}, {"id": 10847399003, "name": "Vincennes University", "priority": 1577, "external_id": null}, {"id": 10847400003, "name": "Virginia Commonwealth University", "priority": 1578, "external_id": null}, {"id": 10847401003, "name": "Virginia Intermont College", "priority": 1579, "external_id": null}, {"id": 10847402003, "name": "Virginia State University", "priority": 1580, "external_id": null}, {"id": 10847403003, "name": "Virginia Union University", "priority": 1581, "external_id": null}, {"id": 10847404003, "name": "Virginia Wesleyan College", "priority": 1582, "external_id": null}, {"id": 10847405003, "name": "Viterbo University", "priority": 1583, "external_id": null}, {"id": 10847406003, "name": "Voorhees College", "priority": 1584, "external_id": null}, {"id": 10847407003, "name": "Wabash College", "priority": 1585, "external_id": null}, {"id": 10847408003, "name": "Walden University", "priority": 1586, "external_id": null}, {"id": 10847409003, "name": "Waldorf College", "priority": 1587, "external_id": null}, {"id": 10847410003, "name": "Walla Walla University", "priority": 1588, "external_id": null}, {"id": 10847411003, "name": "Walsh College of Accountancy and Business Administration", "priority": 1589, "external_id": null}, {"id": 10847412003, "name": "Walsh University", "priority": 1590, "external_id": null}, {"id": 10847413003, "name": "Warner Pacific College", "priority": 1591, "external_id": null}, {"id": 10847414003, "name": "Warner University", "priority": 1592, "external_id": null}, {"id": 10847415003, "name": "Warren Wilson College", "priority": 1593, "external_id": null}, {"id": 10847416003, "name": "Wartburg College", "priority": 1594, "external_id": null}, {"id": 10847417003, "name": "Washburn University", "priority": 1595, "external_id": null}, {"id": 10847418003, "name": "Washington Adventist University", "priority": 1596, "external_id": null}, {"id": 10847419003, "name": "Washington and Jefferson College", "priority": 1597, "external_id": null}, {"id": 10847420003, "name": "Washington and Lee University", "priority": 1598, "external_id": null}, {"id": 10847421003, "name": "Washington College", "priority": 1599, "external_id": null}, {"id": 10847422003, "name": "Washington University in St. Louis", "priority": 1600, "external_id": null}, {"id": 10847423003, "name": "Watkins College of Art, Design & Film", "priority": 1601, "external_id": null}, {"id": 10847424003, "name": "Wayland Baptist University", "priority": 1602, "external_id": null}, {"id": 10847425003, "name": "Wayne State College", "priority": 1603, "external_id": null}, {"id": 10847426003, "name": "Wayne State University", "priority": 1604, "external_id": null}, {"id": 10847427003, "name": "Waynesburg University", "priority": 1605, "external_id": null}, {"id": 10847428003, "name": "Valparaiso University", "priority": 1606, "external_id": null}, {"id": 10847429003, "name": "Villanova University", "priority": 1607, "external_id": null}, {"id": 10847430003, "name": "Virginia Tech", "priority": 1608, "external_id": null}, {"id": 10847431003, "name": "Washington State University", "priority": 1609, "external_id": null}, {"id": 10847432003, "name": "University of Toledo", "priority": 1610, "external_id": null}, {"id": 10847433003, "name": "Wagner College", "priority": 1611, "external_id": null}, {"id": 10847434003, "name": "University of Wyoming", "priority": 1612, "external_id": null}, {"id": 10847435003, "name": "University of Wisconsin - Madison", "priority": 1613, "external_id": null}, {"id": 10847436003, "name": "University of Tulsa", "priority": 1614, "external_id": null}, {"id": 10847437003, "name": "Webb Institute", "priority": 1615, "external_id": null}, {"id": 10847438003, "name": "Webber International University", "priority": 1616, "external_id": null}, {"id": 10847439003, "name": "Webster University", "priority": 1617, "external_id": null}, {"id": 10847440003, "name": "Welch College", "priority": 1618, "external_id": null}, {"id": 10847441003, "name": "Wellesley College", "priority": 1619, "external_id": null}, {"id": 10847442003, "name": "Wells College", "priority": 1620, "external_id": null}, {"id": 10847443003, "name": "Wentworth Institute of Technology", "priority": 1621, "external_id": null}, {"id": 10847444003, "name": "Wesley College", "priority": 1622, "external_id": null}, {"id": 10847445003, "name": "Wesleyan College", "priority": 1623, "external_id": null}, {"id": 10847446003, "name": "Wesleyan University", "priority": 1624, "external_id": null}, {"id": 10847447003, "name": "West Chester University of Pennsylvania", "priority": 1625, "external_id": null}, {"id": 10847448003, "name": "West Liberty University", "priority": 1626, "external_id": null}, {"id": 10847449003, "name": "West Texas A&M University", "priority": 1627, "external_id": null}, {"id": 10847450003, "name": "West Virginia State University", "priority": 1628, "external_id": null}, {"id": 10847451003, "name": "West Virginia University Institute of Technology", "priority": 1629, "external_id": null}, {"id": 10847452003, "name": "West Virginia University - Parkersburg", "priority": 1630, "external_id": null}, {"id": 10847453003, "name": "West Virginia Wesleyan College", "priority": 1631, "external_id": null}, {"id": 10847454003, "name": "Western Connecticut State University", "priority": 1632, "external_id": null}, {"id": 10847455003, "name": "Western Governors University", "priority": 1633, "external_id": null}, {"id": 10847456003, "name": "Western International University", "priority": 1634, "external_id": null}, {"id": 10847457003, "name": "Western Nevada College", "priority": 1635, "external_id": null}, {"id": 10847458003, "name": "Western New England University", "priority": 1636, "external_id": null}, {"id": 10847459003, "name": "Western New Mexico University", "priority": 1637, "external_id": null}, {"id": 10847460003, "name": "Western Oregon University", "priority": 1638, "external_id": null}, {"id": 10847461003, "name": "Western State Colorado University", "priority": 1639, "external_id": null}, {"id": 10847462003, "name": "Western University", "priority": 1640, "external_id": null}, {"id": 10847463003, "name": "Western Washington University", "priority": 1641, "external_id": null}, {"id": 10847464003, "name": "Westfield State University", "priority": 1642, "external_id": null}, {"id": 10847465003, "name": "Westminster College", "priority": 1643, "external_id": null}, {"id": 10847466003, "name": "Westmont College", "priority": 1644, "external_id": null}, {"id": 10847467003, "name": "Wheaton College", "priority": 1645, "external_id": null}, {"id": 10847468003, "name": "Wheeling Jesuit University", "priority": 1646, "external_id": null}, {"id": 10847469003, "name": "Wheelock College", "priority": 1647, "external_id": null}, {"id": 10847470003, "name": "Whitman College", "priority": 1648, "external_id": null}, {"id": 10847471003, "name": "Whittier College", "priority": 1649, "external_id": null}, {"id": 10847472003, "name": "Whitworth University", "priority": 1650, "external_id": null}, {"id": 10847473003, "name": "Wichita State University", "priority": 1651, "external_id": null}, {"id": 10847474003, "name": "Widener University", "priority": 1652, "external_id": null}, {"id": 10847475003, "name": "Wilberforce University", "priority": 1653, "external_id": null}, {"id": 10847476003, "name": "Wiley College", "priority": 1654, "external_id": null}, {"id": 10847477003, "name": "Wilkes University", "priority": 1655, "external_id": null}, {"id": 10847478003, "name": "Willamette University", "priority": 1656, "external_id": null}, {"id": 10847479003, "name": "William Carey University", "priority": 1657, "external_id": null}, {"id": 10847480003, "name": "William Jessup University", "priority": 1658, "external_id": null}, {"id": 10847481003, "name": "William Jewell College", "priority": 1659, "external_id": null}, {"id": 10847482003, "name": "William Paterson University of New Jersey", "priority": 1660, "external_id": null}, {"id": 10847483003, "name": "William Peace University", "priority": 1661, "external_id": null}, {"id": 10847484003, "name": "William Penn University", "priority": 1662, "external_id": null}, {"id": 10847485003, "name": "William Woods University", "priority": 1663, "external_id": null}, {"id": 10847486003, "name": "Williams Baptist College", "priority": 1664, "external_id": null}, {"id": 10847487003, "name": "Williams College", "priority": 1665, "external_id": null}, {"id": 10847488003, "name": "Wilmington College", "priority": 1666, "external_id": null}, {"id": 10847489003, "name": "Wilmington University", "priority": 1667, "external_id": null}, {"id": 10847490003, "name": "Wilson College", "priority": 1668, "external_id": null}, {"id": 10847491003, "name": "Wingate University", "priority": 1669, "external_id": null}, {"id": 10847492003, "name": "Winona State University", "priority": 1670, "external_id": null}, {"id": 10847493003, "name": "Winston-Salem State University", "priority": 1671, "external_id": null}, {"id": 10847494003, "name": "Winthrop University", "priority": 1672, "external_id": null}, {"id": 10847495003, "name": "Wisconsin Lutheran College", "priority": 1673, "external_id": null}, {"id": 10847496003, "name": "Wittenberg University", "priority": 1674, "external_id": null}, {"id": 10847497003, "name": "Woodbury University", "priority": 1675, "external_id": null}, {"id": 10847498003, "name": "Worcester Polytechnic Institute", "priority": 1676, "external_id": null}, {"id": 10847499003, "name": "Worcester State University", "priority": 1677, "external_id": null}, {"id": 10847500003, "name": "Wright State University", "priority": 1678, "external_id": null}, {"id": 10847501003, "name": "Xavier University", "priority": 1679, "external_id": null}, {"id": 10847502003, "name": "Xavier University of Louisiana", "priority": 1680, "external_id": null}, {"id": 10847503003, "name": "Yeshiva University", "priority": 1681, "external_id": null}, {"id": 10847504003, "name": "York College", "priority": 1682, "external_id": null}, {"id": 10847505003, "name": "York College of Pennsylvania", "priority": 1683, "external_id": null}, {"id": 10847506003, "name": "York University", "priority": 1684, "external_id": null}, {"id": 10847507003, "name": "University of Cambridge", "priority": 1685, "external_id": null}, {"id": 10847508003, "name": "UCL (University College London)", "priority": 1686, "external_id": null}, {"id": 10847509003, "name": "Imperial College London", "priority": 1687, "external_id": null}, {"id": 10847510003, "name": "University of Oxford", "priority": 1688, "external_id": null}, {"id": 10847511003, "name": "ETH Zurich (Swiss Federal Institute of Technology)", "priority": 1689, "external_id": null}, {"id": 10847512003, "name": "University of Edinburgh", "priority": 1690, "external_id": null}, {"id": 10847513003, "name": "Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne", "priority": 1691, "external_id": null}, {"id": 10847514003, "name": "King's College London (KCL)", "priority": 1692, "external_id": null}, {"id": 10847515003, "name": "National University of Singapore (NUS)", "priority": 1693, "external_id": null}, {"id": 10847516003, "name": "University of Hong Kong", "priority": 1694, "external_id": null}, {"id": 10847517003, "name": "Australian National University", "priority": 1695, "external_id": null}, {"id": 10847518003, "name": "Ecole normale sup\u00e9rieure, Paris", "priority": 1696, "external_id": null}, {"id": 10847519003, "name": "University of Bristol", "priority": 1697, "external_id": null}, {"id": 10847520003, "name": "The University of Melbourne", "priority": 1698, "external_id": null}, {"id": 10847521003, "name": "The University of Tokyo", "priority": 1699, "external_id": null}, {"id": 10847522003, "name": "The University of Manchester", "priority": 1700, "external_id": null}, {"id": 10847523003, "name": "Western Illinois University", "priority": 1701, "external_id": null}, {"id": 10847524003, "name": "Wofford College", "priority": 1702, "external_id": null}, {"id": 10847525003, "name": "Western Carolina University", "priority": 1703, "external_id": null}, {"id": 10847526003, "name": "West Virginia University", "priority": 1704, "external_id": null}, {"id": 10847527003, "name": "Yale University", "priority": 1705, "external_id": null}, {"id": 10847528003, "name": "The Hong Kong University of Science and Technology", "priority": 1706, "external_id": null}, {"id": 10847529003, "name": "Kyoto University", "priority": 1707, "external_id": null}, {"id": 10847530003, "name": "Seoul National University", "priority": 1708, "external_id": null}, {"id": 10847531003, "name": "The University of Sydney", "priority": 1709, "external_id": null}, {"id": 10847532003, "name": "The Chinese University of Hong Kong", "priority": 1710, "external_id": null}, {"id": 10847533003, "name": "Ecole Polytechnique", "priority": 1711, "external_id": null}, {"id": 10847534003, "name": "Nanyang Technological University (NTU)", "priority": 1712, "external_id": null}, {"id": 10847535003, "name": "The University of Queensland", "priority": 1713, "external_id": null}, {"id": 10847536003, "name": "University of Copenhagen", "priority": 1714, "external_id": null}, {"id": 10847537003, "name": "Peking University", "priority": 1715, "external_id": null}, {"id": 10847538003, "name": "Tsinghua University", "priority": 1716, "external_id": null}, {"id": 10847539003, "name": "Ruprecht-Karls-Universit\u00e4t Heidelberg", "priority": 1717, "external_id": null}, {"id": 10847540003, "name": "University of Glasgow", "priority": 1718, "external_id": null}, {"id": 10847541003, "name": "The University of New South Wales", "priority": 1719, "external_id": null}, {"id": 10847542003, "name": "Technische Universit\u00e4t M\u00fcnchen", "priority": 1720, "external_id": null}, {"id": 10847543003, "name": "Osaka University", "priority": 1721, "external_id": null}, {"id": 10847544003, "name": "University of Amsterdam", "priority": 1722, "external_id": null}, {"id": 10847545003, "name": "KAIST - Korea Advanced Institute of Science & Technology", "priority": 1723, "external_id": null}, {"id": 10847546003, "name": "Trinity College Dublin", "priority": 1724, "external_id": null}, {"id": 10847547003, "name": "University of Birmingham", "priority": 1725, "external_id": null}, {"id": 10847548003, "name": "The University of Warwick", "priority": 1726, "external_id": null}, {"id": 10847549003, "name": "Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen", "priority": 1727, "external_id": null}, {"id": 10847550003, "name": "Tokyo Institute of Technology", "priority": 1728, "external_id": null}, {"id": 10847551003, "name": "Lund University", "priority": 1729, "external_id": null}, {"id": 10847552003, "name": "London School of Economics and Political Science (LSE)", "priority": 1730, "external_id": null}, {"id": 10847553003, "name": "Monash University", "priority": 1731, "external_id": null}, {"id": 10847554003, "name": "University of Helsinki", "priority": 1732, "external_id": null}, {"id": 10847555003, "name": "The University of Sheffield", "priority": 1733, "external_id": null}, {"id": 10847556003, "name": "University of Geneva", "priority": 1734, "external_id": null}, {"id": 10847557003, "name": "Leiden University", "priority": 1735, "external_id": null}, {"id": 10847558003, "name": "The University of Nottingham", "priority": 1736, "external_id": null}, {"id": 10847559003, "name": "Tohoku University", "priority": 1737, "external_id": null}, {"id": 10847560003, "name": "KU Leuven", "priority": 1738, "external_id": null}, {"id": 10847561003, "name": "University of Zurich", "priority": 1739, "external_id": null}, {"id": 10847562003, "name": "Uppsala University", "priority": 1740, "external_id": null}, {"id": 10847563003, "name": "Utrecht University", "priority": 1741, "external_id": null}, {"id": 10847564003, "name": "National Taiwan University (NTU)", "priority": 1742, "external_id": null}, {"id": 10847565003, "name": "University of St Andrews", "priority": 1743, "external_id": null}, {"id": 10847566003, "name": "The University of Western Australia", "priority": 1744, "external_id": null}, {"id": 10847567003, "name": "University of Southampton", "priority": 1745, "external_id": null}, {"id": 10847568003, "name": "Fudan University", "priority": 1746, "external_id": null}, {"id": 10847569003, "name": "University of Oslo", "priority": 1747, "external_id": null}, {"id": 10847570003, "name": "Durham University", "priority": 1748, "external_id": null}, {"id": 10847571003, "name": "Aarhus University", "priority": 1749, "external_id": null}, {"id": 10847572003, "name": "Erasmus University Rotterdam", "priority": 1750, "external_id": null}, {"id": 10847573003, "name": "Universit\u00e9 de Montr\u00e9al", "priority": 1751, "external_id": null}, {"id": 10847574003, "name": "The University of Auckland", "priority": 1752, "external_id": null}, {"id": 10847575003, "name": "Delft University of Technology", "priority": 1753, "external_id": null}, {"id": 10847576003, "name": "University of Groningen", "priority": 1754, "external_id": null}, {"id": 10847577003, "name": "University of Leeds", "priority": 1755, "external_id": null}, {"id": 10847578003, "name": "Nagoya University", "priority": 1756, "external_id": null}, {"id": 10847579003, "name": "Universit\u00e4t Freiburg", "priority": 1757, "external_id": null}, {"id": 10847580003, "name": "City University of Hong Kong", "priority": 1758, "external_id": null}, {"id": 10847581003, "name": "The University of Adelaide", "priority": 1759, "external_id": null}, {"id": 10847582003, "name": "Pohang University of Science And Technology (POSTECH)", "priority": 1760, "external_id": null}, {"id": 10847583003, "name": "Freie Universit\u00e4t Berlin", "priority": 1761, "external_id": null}, {"id": 10847584003, "name": "University of Basel", "priority": 1762, "external_id": null}, {"id": 10847585003, "name": "University of Lausanne", "priority": 1763, "external_id": null}, {"id": 10847586003, "name": "Universit\u00e9 Pierre et Marie Curie (UPMC)", "priority": 1764, "external_id": null}, {"id": 10847587003, "name": "Yonsei University", "priority": 1765, "external_id": null}, {"id": 10847588003, "name": "University of York", "priority": 1766, "external_id": null}, {"id": 10847589003, "name": "Queen Mary, University of London (QMUL)", "priority": 1767, "external_id": null}, {"id": 10847590003, "name": "Karlsruhe Institute of Technology (KIT)", "priority": 1768, "external_id": null}, {"id": 10847591003, "name": "KTH, Royal Institute of Technology", "priority": 1769, "external_id": null}, {"id": 10847592003, "name": "Lomonosov Moscow State University", "priority": 1770, "external_id": null}, {"id": 10847593003, "name": "Maastricht University", "priority": 1771, "external_id": null}, {"id": 10847594003, "name": "University of Ghent", "priority": 1772, "external_id": null}, {"id": 10847595003, "name": "Shanghai Jiao Tong University", "priority": 1773, "external_id": null}, {"id": 10847596003, "name": "Humboldt-Universit\u00e4t zu Berlin", "priority": 1774, "external_id": null}, {"id": 10847597003, "name": "Universidade de S\u00e3o Paulo (USP)", "priority": 1775, "external_id": null}, {"id": 10847598003, "name": "Georg-August-Universit\u00e4t G\u00f6ttingen", "priority": 1776, "external_id": null}, {"id": 10847599003, "name": "Newcastle University", "priority": 1777, "external_id": null}, {"id": 10847600003, "name": "University of Liverpool", "priority": 1778, "external_id": null}, {"id": 10847601003, "name": "Kyushu University", "priority": 1779, "external_id": null}, {"id": 10847602003, "name": "Eberhard Karls Universit\u00e4t T\u00fcbingen", "priority": 1780, "external_id": null}, {"id": 10847603003, "name": "Technical University of Denmark", "priority": 1781, "external_id": null}, {"id": 10847604003, "name": "Cardiff University", "priority": 1782, "external_id": null}, {"id": 10847605003, "name": "Universit\u00e9 Catholique de Louvain (UCL)", "priority": 1783, "external_id": null}, {"id": 10847606003, "name": "University College Dublin", "priority": 1784, "external_id": null}, {"id": 10847607003, "name": "McMaster University", "priority": 1785, "external_id": null}, {"id": 10847608003, "name": "Hebrew University of Jerusalem", "priority": 1786, "external_id": null}, {"id": 10847609003, "name": "Radboud University Nijmegen", "priority": 1787, "external_id": null}, {"id": 10847610003, "name": "Hokkaido University", "priority": 1788, "external_id": null}, {"id": 10847611003, "name": "Korea University", "priority": 1789, "external_id": null}, {"id": 10847612003, "name": "University of Cape Town", "priority": 1790, "external_id": null}, {"id": 10847613003, "name": "Rheinisch-Westf\u00e4lische Technische Hochschule Aachen", "priority": 1791, "external_id": null}, {"id": 10847614003, "name": "University of Aberdeen", "priority": 1792, "external_id": null}, {"id": 10847615003, "name": "Wageningen University", "priority": 1793, "external_id": null}, {"id": 10847616003, "name": "University of Bergen", "priority": 1794, "external_id": null}, {"id": 10847617003, "name": "University of Bern", "priority": 1795, "external_id": null}, {"id": 10847618003, "name": "University of Otago", "priority": 1796, "external_id": null}, {"id": 10847619003, "name": "Lancaster University", "priority": 1797, "external_id": null}, {"id": 10847620003, "name": "Eindhoven University of Technology", "priority": 1798, "external_id": null}, {"id": 10847621003, "name": "Ecole Normale Sup\u00e9rieure de Lyon", "priority": 1799, "external_id": null}, {"id": 10847622003, "name": "University of Vienna", "priority": 1800, "external_id": null}, {"id": 10847623003, "name": "The Hong Kong Polytechnic University", "priority": 1801, "external_id": null}, {"id": 10847624003, "name": "Sungkyunkwan University", "priority": 1802, "external_id": null}, {"id": 10847625003, "name": "Rheinische Friedrich-Wilhelms-Universit\u00e4t Bonn", "priority": 1803, "external_id": null}, {"id": 10847626003, "name": "Universidad Nacional Aut\u00f3noma de M\u00e9xico (UNAM)", "priority": 1804, "external_id": null}, {"id": 10847627003, "name": "Zhejiang University", "priority": 1805, "external_id": null}, {"id": 10847628003, "name": "Pontificia Universidad Cat\u00f3lica de Chile", "priority": 1806, "external_id": null}, {"id": 10847629003, "name": "Universiti Malaya (UM)", "priority": 1807, "external_id": null}, {"id": 10847630003, "name": "Universit\u00e9 Libre de Bruxelles (ULB)", "priority": 1808, "external_id": null}, {"id": 10847631003, "name": "University of Exeter", "priority": 1809, "external_id": null}, {"id": 10847632003, "name": "Stockholm University", "priority": 1810, "external_id": null}, {"id": 10847633003, "name": "Queen's University of Belfast", "priority": 1811, "external_id": null}, {"id": 10847634003, "name": "Vrije Universiteit Brussel (VUB)", "priority": 1812, "external_id": null}, {"id": 10847635003, "name": "University of Science and Technology of China", "priority": 1813, "external_id": null}, {"id": 10847636003, "name": "Nanjing University", "priority": 1814, "external_id": null}, {"id": 10847637003, "name": "Universitat Aut\u00f3noma de Barcelona", "priority": 1815, "external_id": null}, {"id": 10847638003, "name": "University of Barcelona", "priority": 1816, "external_id": null}, {"id": 10847639003, "name": "VU University Amsterdam", "priority": 1817, "external_id": null}, {"id": 10847640003, "name": "Technion - Israel Institute of Technology", "priority": 1818, "external_id": null}, {"id": 10847641003, "name": "Technische Universit\u00e4t Berlin", "priority": 1819, "external_id": null}, {"id": 10847642003, "name": "University of Antwerp", "priority": 1820, "external_id": null}, {"id": 10847643003, "name": "Universit\u00e4t Hamburg", "priority": 1821, "external_id": null}, {"id": 10847644003, "name": "University of Bath", "priority": 1822, "external_id": null}, {"id": 10847645003, "name": "University of Bologna", "priority": 1823, "external_id": null}, {"id": 10847646003, "name": "Queen's University, Ontario", "priority": 1824, "external_id": null}, {"id": 10847647003, "name": "Universit\u00e9 Paris-Sud 11", "priority": 1825, "external_id": null}, {"id": 10847648003, "name": "Keio University", "priority": 1826, "external_id": null}, {"id": 10847649003, "name": "University of Sussex", "priority": 1827, "external_id": null}, {"id": 10847650003, "name": "Universidad Aut\u00f3noma de Madrid", "priority": 1828, "external_id": null}, {"id": 10847651003, "name": "Aalto University", "priority": 1829, "external_id": null}, {"id": 10847652003, "name": "Sapienza University of Rome", "priority": 1830, "external_id": null}, {"id": 10847653003, "name": "Tel Aviv University", "priority": 1831, "external_id": null}, {"id": 10847654003, "name": "National Tsing Hua University", "priority": 1832, "external_id": null}, {"id": 10847655003, "name": "Chalmers University of Technology", "priority": 1833, "external_id": null}, {"id": 10847656003, "name": "University of Leicester", "priority": 1834, "external_id": null}, {"id": 10847657003, "name": "Universit\u00e9 Paris Diderot - Paris 7", "priority": 1835, "external_id": null}, {"id": 10847658003, "name": "University of Gothenburg", "priority": 1836, "external_id": null}, {"id": 10847659003, "name": "University of Turku", "priority": 1837, "external_id": null}, {"id": 10847660003, "name": "Universit\u00e4t Frankfurt am Main", "priority": 1838, "external_id": null}, {"id": 10847661003, "name": "Universidad de Buenos Aires", "priority": 1839, "external_id": null}, {"id": 10847662003, "name": "University College Cork", "priority": 1840, "external_id": null}, {"id": 10847663003, "name": "University of Tsukuba", "priority": 1841, "external_id": null}, {"id": 10847664003, "name": "University of Reading", "priority": 1842, "external_id": null}, {"id": 10847665003, "name": "Sciences Po Paris", "priority": 1843, "external_id": null}, {"id": 10847666003, "name": "Universidade Estadual de Campinas", "priority": 1844, "external_id": null}, {"id": 10847667003, "name": "King Fahd University of Petroleum & Minerals", "priority": 1845, "external_id": null}, {"id": 10847668003, "name": "University Complutense Madrid", "priority": 1846, "external_id": null}, {"id": 10847669003, "name": "Universit\u00e9 Paris-Sorbonne (Paris IV)", "priority": 1847, "external_id": null}, {"id": 10847670003, "name": "University of Dundee", "priority": 1848, "external_id": null}, {"id": 10847671003, "name": "Universit\u00e9 Joseph Fourier - Grenoble 1", "priority": 1849, "external_id": null}, {"id": 10847672003, "name": "Waseda University", "priority": 1850, "external_id": null}, {"id": 10847673003, "name": "Indian Institute of Technology Delhi (IITD)", "priority": 1851, "external_id": null}, {"id": 10847674003, "name": "Universidad de Chile", "priority": 1852, "external_id": null}, {"id": 10847675003, "name": "Universit\u00e9 Paris 1 Panth\u00e9on-Sorbonne", "priority": 1853, "external_id": null}, {"id": 10847676003, "name": "Universit\u00e9 de Strasbourg", "priority": 1854, "external_id": null}, {"id": 10847677003, "name": "University of Twente", "priority": 1855, "external_id": null}, {"id": 10847678003, "name": "University of East Anglia (UEA)", "priority": 1856, "external_id": null}, {"id": 10847679003, "name": "National Chiao Tung University", "priority": 1857, "external_id": null}, {"id": 10847680003, "name": "Politecnico di Milano", "priority": 1858, "external_id": null}, {"id": 10847681003, "name": "Charles University", "priority": 1859, "external_id": null}, {"id": 10847682003, "name": "Indian Institute of Technology Bombay (IITB)", "priority": 1860, "external_id": null}, {"id": 10847683003, "name": "University of Milano", "priority": 1861, "external_id": null}, {"id": 10847684003, "name": "Westf\u00e4lische Wilhelms-Universit\u00e4t M\u00fcnster", "priority": 1862, "external_id": null}, {"id": 10847685003, "name": "University of Canterbury", "priority": 1863, "external_id": null}, {"id": 10847686003, "name": "Chulalongkorn University", "priority": 1864, "external_id": null}, {"id": 10847687003, "name": "Saint-Petersburg State University", "priority": 1865, "external_id": null}, {"id": 10847688003, "name": "University of Liege", "priority": 1866, "external_id": null}, {"id": 10847689003, "name": "Universit\u00e4t zu K\u00f6ln", "priority": 1867, "external_id": null}, {"id": 10847690003, "name": "Loughborough University", "priority": 1868, "external_id": null}, {"id": 10847691003, "name": "National Cheng Kung University", "priority": 1869, "external_id": null}, {"id": 10847692003, "name": "Universit\u00e4t Stuttgart", "priority": 1870, "external_id": null}, {"id": 10847693003, "name": "Hanyang University", "priority": 1871, "external_id": null}, {"id": 10847694003, "name": "American University of Beirut (AUB)", "priority": 1872, "external_id": null}, {"id": 10847695003, "name": "Norwegian University of Science And Technology", "priority": 1873, "external_id": null}, {"id": 10847696003, "name": "Beijing Normal University", "priority": 1874, "external_id": null}, {"id": 10847697003, "name": "King Saud University", "priority": 1875, "external_id": null}, {"id": 10847698003, "name": "University of Oulu", "priority": 1876, "external_id": null}, {"id": 10847699003, "name": "Kyung Hee University", "priority": 1877, "external_id": null}, {"id": 10847700003, "name": "University of Strathclyde", "priority": 1878, "external_id": null}, {"id": 10847701003, "name": "Universit\u00e4t Ulm", "priority": 1879, "external_id": null}, {"id": 10847702003, "name": "University of Pisa", "priority": 1880, "external_id": null}, {"id": 10847703003, "name": "Technische Universit\u00e4t Darmstadt", "priority": 1881, "external_id": null}, {"id": 10847704003, "name": "Technische Universit\u00e4t Dresden", "priority": 1882, "external_id": null}, {"id": 10847705003, "name": "Macquarie University", "priority": 1883, "external_id": null}, {"id": 10847706003, "name": "Vienna University of Technology", "priority": 1884, "external_id": null}, {"id": 10847707003, "name": "Royal Holloway University of London", "priority": 1885, "external_id": null}, {"id": 10847708003, "name": "Victoria University of Wellington", "priority": 1886, "external_id": null}, {"id": 10847709003, "name": "University of Padua", "priority": 1887, "external_id": null}, {"id": 10847710003, "name": "Universiti Kebangsaan Malaysia (UKM)", "priority": 1888, "external_id": null}, {"id": 10847711003, "name": "University of Technology, Sydney", "priority": 1889, "external_id": null}, {"id": 10847712003, "name": "Universit\u00e4t Konstanz", "priority": 1890, "external_id": null}, {"id": 10847713003, "name": "Universidad de Los Andes Colombia", "priority": 1891, "external_id": null}, {"id": 10847714003, "name": "Universit\u00e9 Paris Descartes", "priority": 1892, "external_id": null}, {"id": 10847715003, "name": "Tokyo Medical and Dental University", "priority": 1893, "external_id": null}, {"id": 10847716003, "name": "University of Wollongong", "priority": 1894, "external_id": null}, {"id": 10847717003, "name": "Universit\u00e4t Erlangen-N\u00fcrnberg", "priority": 1895, "external_id": null}, {"id": 10847718003, "name": "Queensland University of Technology", "priority": 1896, "external_id": null}, {"id": 10847719003, "name": "Tecnol\u00f3gico de Monterrey (ITESM)", "priority": 1897, "external_id": null}, {"id": 10847720003, "name": "Universit\u00e4t Mannheim", "priority": 1898, "external_id": null}, {"id": 10847721003, "name": "Universitat Pompeu Fabra", "priority": 1899, "external_id": null}, {"id": 10847722003, "name": "Mahidol University", "priority": 1900, "external_id": null}, {"id": 10847723003, "name": "Curtin University", "priority": 1901, "external_id": null}, {"id": 10847724003, "name": "National University of Ireland, Galway", "priority": 1902, "external_id": null}, {"id": 10847725003, "name": "Universidade Federal do Rio de Janeiro", "priority": 1903, "external_id": null}, {"id": 10847726003, "name": "University of Surrey", "priority": 1904, "external_id": null}, {"id": 10847727003, "name": "Hong Kong Baptist University", "priority": 1905, "external_id": null}, {"id": 10847728003, "name": "Ume\u00e5 University", "priority": 1906, "external_id": null}, {"id": 10847729003, "name": "Universit\u00e4t Innsbruck", "priority": 1907, "external_id": null}, {"id": 10847730003, "name": "RMIT University", "priority": 1908, "external_id": null}, {"id": 10847731003, "name": "University of Eastern Finland", "priority": 1909, "external_id": null}, {"id": 10847732003, "name": "Christian-Albrechts-Universit\u00e4t zu Kiel", "priority": 1910, "external_id": null}, {"id": 10847733003, "name": "Indian Institute of Technology Kanpur (IITK)", "priority": 1911, "external_id": null}, {"id": 10847734003, "name": "National Yang Ming University", "priority": 1912, "external_id": null}, {"id": 10847735003, "name": "Johannes Gutenberg Universit\u00e4t Mainz", "priority": 1913, "external_id": null}, {"id": 10847736003, "name": "The University of Newcastle", "priority": 1914, "external_id": null}, {"id": 10847737003, "name": "Al-Farabi Kazakh National University", "priority": 1915, "external_id": null}, {"id": 10847738003, "name": "\u00c9cole des Ponts ParisTech", "priority": 1916, "external_id": null}, {"id": 10847739003, "name": "University of Jyv\u00e4skyl\u00e4", "priority": 1917, "external_id": null}, {"id": 10847740003, "name": "L.N. Gumilyov Eurasian National University", "priority": 1918, "external_id": null}, {"id": 10847741003, "name": "Kobe University", "priority": 1919, "external_id": null}, {"id": 10847742003, "name": "University of Tromso", "priority": 1920, "external_id": null}, {"id": 10847743003, "name": "Hiroshima University", "priority": 1921, "external_id": null}, {"id": 10847744003, "name": "Universit\u00e9 Bordeaux 1, Sciences Technologies", "priority": 1922, "external_id": null}, {"id": 10847745003, "name": "University of Indonesia", "priority": 1923, "external_id": null}, {"id": 10847746003, "name": "Universit\u00e4t Leipzig", "priority": 1924, "external_id": null}, {"id": 10847747003, "name": "University of Southern Denmark", "priority": 1925, "external_id": null}, {"id": 10847748003, "name": "Indian Institute of Technology Madras (IITM)", "priority": 1926, "external_id": null}, {"id": 10847749003, "name": "University of The Witwatersrand", "priority": 1927, "external_id": null}, {"id": 10847750003, "name": "University of Navarra", "priority": 1928, "external_id": null}, {"id": 10847751003, "name": "Universidad Austral - Argentina", "priority": 1929, "external_id": null}, {"id": 10847752003, "name": "Universidad Carlos III de Madrid", "priority": 1930, "external_id": null}, {"id": 10847753003, "name": "Universit\u00e0\u00a1 degli Studi di Roma - Tor Vergata", "priority": 1931, "external_id": null}, {"id": 10847754003, "name": "Pontificia Universidad Cat\u00f3lica Argentina Santa Mar\u00eda de los Buenos Aires", "priority": 1932, "external_id": null}, {"id": 10847755003, "name": "UCA", "priority": 1933, "external_id": null}, {"id": 10847756003, "name": "Julius-Maximilians-Universit\u00e4t W\u00fcrzburg", "priority": 1934, "external_id": null}, {"id": 10847757003, "name": "Universidad Nacional de Colombia", "priority": 1935, "external_id": null}, {"id": 10847758003, "name": "Laval University", "priority": 1936, "external_id": null}, {"id": 10847759003, "name": "Ben Gurion University of The Negev", "priority": 1937, "external_id": null}, {"id": 10847760003, "name": "Link\u00f6ping University", "priority": 1938, "external_id": null}, {"id": 10847761003, "name": "Aalborg University", "priority": 1939, "external_id": null}, {"id": 10847762003, "name": "Bauman Moscow State Technical University", "priority": 1940, "external_id": null}, {"id": 10847763003, "name": "Ecole Normale Sup\u00e9rieure de Cachan", "priority": 1941, "external_id": null}, {"id": 10847764003, "name": "SOAS - School of Oriental and African Studies, University of London", "priority": 1942, "external_id": null}, {"id": 10847765003, "name": "University of Essex", "priority": 1943, "external_id": null}, {"id": 10847766003, "name": "University of Warsaw", "priority": 1944, "external_id": null}, {"id": 10847767003, "name": "Griffith University", "priority": 1945, "external_id": null}, {"id": 10847768003, "name": "University of South Australia", "priority": 1946, "external_id": null}, {"id": 10847769003, "name": "Massey University", "priority": 1947, "external_id": null}, {"id": 10847770003, "name": "University of Porto", "priority": 1948, "external_id": null}, {"id": 10847771003, "name": "Universitat Polit\u00e8cnica de Catalunya", "priority": 1949, "external_id": null}, {"id": 10847772003, "name": "Indian Institute of Technology Kharagpur (IITKGP)", "priority": 1950, "external_id": null}, {"id": 10847773003, "name": "City University London", "priority": 1951, "external_id": null}, {"id": 10847774003, "name": "Dublin City University", "priority": 1952, "external_id": null}, {"id": 10847775003, "name": "Pontificia Universidad Javeriana", "priority": 1953, "external_id": null}, {"id": 10847776003, "name": "James Cook University", "priority": 1954, "external_id": null}, {"id": 10847777003, "name": "Novosibirsk State University", "priority": 1955, "external_id": null}, {"id": 10847778003, "name": "Universidade Nova de Lisboa", "priority": 1956, "external_id": null}, {"id": 10847779003, "name": "Universit\u00e9 Aix-Marseille", "priority": 1957, "external_id": null}, {"id": 10847780003, "name": "Universiti Sains Malaysia (USM)", "priority": 1958, "external_id": null}, {"id": 10847781003, "name": "Universiti Teknologi Malaysia (UTM)", "priority": 1959, "external_id": null}, {"id": 10847782003, "name": "Universit\u00e9 Paris Dauphine", "priority": 1960, "external_id": null}, {"id": 10847783003, "name": "University of Coimbra", "priority": 1961, "external_id": null}, {"id": 10847784003, "name": "Brunel University", "priority": 1962, "external_id": null}, {"id": 10847785003, "name": "King Abdul Aziz University (KAU)", "priority": 1963, "external_id": null}, {"id": 10847786003, "name": "Ewha Womans University", "priority": 1964, "external_id": null}, {"id": 10847787003, "name": "Nankai University", "priority": 1965, "external_id": null}, {"id": 10847788003, "name": "Taipei Medical University", "priority": 1966, "external_id": null}, {"id": 10847789003, "name": "Universit\u00e4t Jena", "priority": 1967, "external_id": null}, {"id": 10847790003, "name": "Ruhr-Universit\u00e4t Bochum", "priority": 1968, "external_id": null}, {"id": 10847791003, "name": "Heriot-Watt University", "priority": 1969, "external_id": null}, {"id": 10847792003, "name": "Politecnico di Torino", "priority": 1970, "external_id": null}, {"id": 10847793003, "name": "Universit\u00e4t Bremen", "priority": 1971, "external_id": null}, {"id": 10847794003, "name": "Xi'an Jiaotong University", "priority": 1972, "external_id": null}, {"id": 10847795003, "name": "Birkbeck College, University of London", "priority": 1973, "external_id": null}, {"id": 10847796003, "name": "Oxford Brookes University", "priority": 1974, "external_id": null}, {"id": 10847797003, "name": "Jagiellonian University", "priority": 1975, "external_id": null}, {"id": 10847798003, "name": "University of Tampere", "priority": 1976, "external_id": null}, {"id": 10847799003, "name": "University of Florence", "priority": 1977, "external_id": null}, {"id": 10847800003, "name": "Deakin University", "priority": 1978, "external_id": null}, {"id": 10847801003, "name": "University of the Philippines", "priority": 1979, "external_id": null}, {"id": 10847802003, "name": "Universitat Polit\u00e8cnica de Val\u00e8ncia", "priority": 1980, "external_id": null}, {"id": 10847803003, "name": "Sun Yat-sen University", "priority": 1981, "external_id": null}, {"id": 10847804003, "name": "Universit\u00e9 Montpellier 2, Sciences et Techniques du Languedoc", "priority": 1982, "external_id": null}, {"id": 10847805003, "name": "Moscow State Institute of International Relations (MGIMO-University)", "priority": 1983, "external_id": null}, {"id": 10847806003, "name": "Stellenbosch University", "priority": 1984, "external_id": null}, {"id": 10847807003, "name": "Polit\u00e9cnica de Madrid", "priority": 1985, "external_id": null}, {"id": 10847808003, "name": "Instituto Tecnol\u00f3gico de Buenos Aires (ITBA)", "priority": 1986, "external_id": null}, {"id": 10847809003, "name": "La Trobe University", "priority": 1987, "external_id": null}, {"id": 10847810003, "name": "Universit\u00e9 Paul Sabatier Toulouse III", "priority": 1988, "external_id": null}, {"id": 10847811003, "name": "Karl-Franzens-Universit\u00e4t Graz", "priority": 1989, "external_id": null}, {"id": 10847812003, "name": "Universit\u00e4t D\u00fcsseldorf", "priority": 1990, "external_id": null}, {"id": 10847813003, "name": "University of Naples - Federico Ii", "priority": 1991, "external_id": null}, {"id": 10847814003, "name": "Aston University", "priority": 1992, "external_id": null}, {"id": 10847815003, "name": "University of Turin", "priority": 1993, "external_id": null}, {"id": 10847816003, "name": "Beihang University (former BUAA)", "priority": 1994, "external_id": null}, {"id": 10847817003, "name": "Indian Institute of Technology Roorkee (IITR)", "priority": 1995, "external_id": null}, {"id": 10847818003, "name": "National Central University", "priority": 1996, "external_id": null}, {"id": 10847819003, "name": "Sogang University", "priority": 1997, "external_id": null}, {"id": 10847820003, "name": "Universit\u00e4t Regensburg", "priority": 1998, "external_id": null}, {"id": 10847821003, "name": "Universit\u00e9 Lille 1, Sciences et Technologie", "priority": 1999, "external_id": null}, {"id": 10847822003, "name": "University of Tasmania", "priority": 2000, "external_id": null}, {"id": 10847823003, "name": "University of Waikato", "priority": 2001, "external_id": null}, {"id": 10847824003, "name": "Wuhan University", "priority": 2002, "external_id": null}, {"id": 10847825003, "name": "National Taiwan University of Science And Technology", "priority": 2003, "external_id": null}, {"id": 10847826003, "name": "Universidade Federal de S\u00e3o Paulo (UNIFESP)", "priority": 2004, "external_id": null}, {"id": 10847827003, "name": "Universit\u00e0 degli Studi di Pavia", "priority": 2005, "external_id": null}, {"id": 10847828003, "name": "Universit\u00e4t Bayreuth", "priority": 2006, "external_id": null}, {"id": 10847829003, "name": "Universit\u00e9 Claude Bernard Lyon 1", "priority": 2007, "external_id": null}, {"id": 10847830003, "name": "Universit\u00e9 du Qu\u00e9bec", "priority": 2008, "external_id": null}, {"id": 10847831003, "name": "Universiti Putra Malaysia (UPM)", "priority": 2009, "external_id": null}, {"id": 10847832003, "name": "University of Kent", "priority": 2010, "external_id": null}, {"id": 10847833003, "name": "University of St Gallen (HSG)", "priority": 2011, "external_id": null}, {"id": 10847834003, "name": "Bond University", "priority": 2012, "external_id": null}, {"id": 10847835003, "name": "United Arab Emirates University", "priority": 2013, "external_id": null}, {"id": 10847836003, "name": "Universidad de San Andr\u00c3\u00a9s", "priority": 2014, "external_id": null}, {"id": 10847837003, "name": "Universidad Nacional de La Plata", "priority": 2015, "external_id": null}, {"id": 10847838003, "name": "Universit\u00e4t des Saarlandes", "priority": 2016, "external_id": null}, {"id": 10847839003, "name": "American University of Sharjah (AUS)", "priority": 2017, "external_id": null}, {"id": 10847840003, "name": "Bilkent University", "priority": 2018, "external_id": null}, {"id": 10847841003, "name": "Flinders University", "priority": 2019, "external_id": null}, {"id": 10847842003, "name": "Hankuk (Korea) University of Foreign Studies", "priority": 2020, "external_id": null}, {"id": 10847843003, "name": "Middle East Technical University", "priority": 2021, "external_id": null}, {"id": 10847844003, "name": "Philipps-Universit\u00e4t Marburg", "priority": 2022, "external_id": null}, {"id": 10847845003, "name": "Swansea University", "priority": 2023, "external_id": null}, {"id": 10847846003, "name": "Tampere University of Technology", "priority": 2024, "external_id": null}, {"id": 10847847003, "name": "Universit\u00e4t Bielefeld", "priority": 2025, "external_id": null}, {"id": 10847848003, "name": "University of Manitoba", "priority": 2026, "external_id": null}, {"id": 10847849003, "name": "Chiba University", "priority": 2027, "external_id": null}, {"id": 10847850003, "name": "Moscow Institute of Physics and Technology State University", "priority": 2028, "external_id": null}, {"id": 10847851003, "name": "Tallinn University of Technology", "priority": 2029, "external_id": null}, {"id": 10847852003, "name": "Taras Shevchenko National University of Kyiv", "priority": 2030, "external_id": null}, {"id": 10847853003, "name": "Tokyo University of Science", "priority": 2031, "external_id": null}, {"id": 10847854003, "name": "University of Salamanca", "priority": 2032, "external_id": null}, {"id": 10847855003, "name": "University of Trento", "priority": 2033, "external_id": null}, {"id": 10847856003, "name": "Universit\u00e9 de Sherbrooke", "priority": 2034, "external_id": null}, {"id": 10847857003, "name": "Universit\u00e9 Panth\u00e9on-Assas (Paris 2)", "priority": 2035, "external_id": null}, {"id": 10847858003, "name": "University of Delhi", "priority": 2036, "external_id": null}, {"id": 10847859003, "name": "Abo Akademi University", "priority": 2037, "external_id": null}, {"id": 10847860003, "name": "Czech Technical University In Prague", "priority": 2038, "external_id": null}, {"id": 10847861003, "name": "Leibniz Universit\u00e4t Hannover", "priority": 2039, "external_id": null}, {"id": 10847862003, "name": "Pusan National University", "priority": 2040, "external_id": null}, {"id": 10847863003, "name": "Shanghai University", "priority": 2041, "external_id": null}, {"id": 10847864003, "name": "St. Petersburg State Politechnical University", "priority": 2042, "external_id": null}, {"id": 10847865003, "name": "Universit\u00e0 Cattolica del Sacro Cuore", "priority": 2043, "external_id": null}, {"id": 10847866003, "name": "University of Genoa", "priority": 2044, "external_id": null}, {"id": 10847867003, "name": "Bandung Institute of Technology (ITB)", "priority": 2045, "external_id": null}, {"id": 10847868003, "name": "Bogazici University", "priority": 2046, "external_id": null}, {"id": 10847869003, "name": "Goldsmiths, University of London", "priority": 2047, "external_id": null}, {"id": 10847870003, "name": "National Sun Yat-sen University", "priority": 2048, "external_id": null}, {"id": 10847871003, "name": "Renmin (People\u2019s) University of China", "priority": 2049, "external_id": null}, {"id": 10847872003, "name": "Universidad de Costa Rica", "priority": 2050, "external_id": null}, {"id": 10847873003, "name": "Universidad de Santiago de Chile - USACH", "priority": 2051, "external_id": null}, {"id": 10847874003, "name": "University of Tartu", "priority": 2052, "external_id": null}, {"id": 10847875003, "name": "Aristotle University of Thessaloniki", "priority": 2053, "external_id": null}, {"id": 10847876003, "name": "Auckland University of Technology", "priority": 2054, "external_id": null}, {"id": 10847877003, "name": "Bangor University", "priority": 2055, "external_id": null}, {"id": 10847878003, "name": "Charles Darwin University", "priority": 2056, "external_id": null}, {"id": 10847879003, "name": "Kingston University, London", "priority": 2057, "external_id": null}, {"id": 10847880003, "name": "Universitat de Valencia", "priority": 2058, "external_id": null}, {"id": 10847881003, "name": "Universit\u00e9 Montpellier 1", "priority": 2059, "external_id": null}, {"id": 10847882003, "name": "University of Pretoria", "priority": 2060, "external_id": null}, {"id": 10847883003, "name": "Lincoln University", "priority": 2061, "external_id": null}, {"id": 10847884003, "name": "National Taiwan Normal University", "priority": 2062, "external_id": null}, {"id": 10847885003, "name": "National University of Sciences And Technology (NUST) Islamabad", "priority": 2063, "external_id": null}, {"id": 10847886003, "name": "Swinburne University of Technology", "priority": 2064, "external_id": null}, {"id": 10847887003, "name": "Tongji University", "priority": 2065, "external_id": null}, {"id": 10847888003, "name": "Universidad de Zaragoza", "priority": 2066, "external_id": null}, {"id": 10847889003, "name": "Universidade Federal de Minas Gerais", "priority": 2067, "external_id": null}, {"id": 10847890003, "name": "Universit\u00e4t Duisburg-Essen", "priority": 2068, "external_id": null}, {"id": 10847891003, "name": "Al-Imam Mohamed Ibn Saud Islamic University", "priority": 2069, "external_id": null}, {"id": 10847892003, "name": "Harbin Institute of Technology", "priority": 2070, "external_id": null}, {"id": 10847893003, "name": "People's Friendship University of Russia", "priority": 2071, "external_id": null}, {"id": 10847894003, "name": "Universidade Estadual PaulistaJ\u00falio de Mesquita Filho' (UNESP)", "priority": 2072, "external_id": null}, {"id": 10847895003, "name": "Universit\u00e9 Nice Sophia-Antipolis", "priority": 2073, "external_id": null}, {"id": 10847896003, "name": "University of Crete", "priority": 2074, "external_id": null}, {"id": 10847897003, "name": "University of Milano-Bicocca", "priority": 2075, "external_id": null}, {"id": 10847898003, "name": "Ateneo de Manila University", "priority": 2076, "external_id": null}, {"id": 10847899003, "name": "Beijing Institute of Technology", "priority": 2077, "external_id": null}, {"id": 10847900003, "name": "Chang Gung University", "priority": 2078, "external_id": null}, {"id": 10847901003, "name": "hung-Ang University", "priority": 2079, "external_id": null}, {"id": 10847902003, "name": "Dublin Institute of Technology", "priority": 2080, "external_id": null}, {"id": 10847903003, "name": "Huazhong University of Science and Technology", "priority": 2081, "external_id": null}, {"id": 10847904003, "name": "International Islamic University Malaysia (IIUM)", "priority": 2082, "external_id": null}, {"id": 10847905003, "name": "Johannes Kepler University Linz", "priority": 2083, "external_id": null}, {"id": 10847906003, "name": "Justus-Liebig-Universit\u00e4t Gie\u00dfen", "priority": 2084, "external_id": null}, {"id": 10847907003, "name": "Kanazawa University", "priority": 2085, "external_id": null}, {"id": 10847908003, "name": "Keele University", "priority": 2086, "external_id": null}, {"id": 10847909003, "name": "Koc University", "priority": 2087, "external_id": null}, {"id": 10847910003, "name": "National and Kapodistrian University of Athens", "priority": 2088, "external_id": null}, {"id": 10847911003, "name": "National Research University \u2013 Higher School of Economics (HSE)", "priority": 2089, "external_id": null}, {"id": 10847912003, "name": "National Technical University of Athens", "priority": 2090, "external_id": null}, {"id": 10847913003, "name": "Okayama University", "priority": 2091, "external_id": null}, {"id": 10847914003, "name": "Sabanci University", "priority": 2092, "external_id": null}, {"id": 10847915003, "name": "Southeast University", "priority": 2093, "external_id": null}, {"id": 10847916003, "name": "Sultan Qaboos University", "priority": 2094, "external_id": null}, {"id": 10847917003, "name": "Technische Universit\u00e4t Braunschweig", "priority": 2095, "external_id": null}, {"id": 10847918003, "name": "Technische Universit\u00e4t Dortmund", "priority": 2096, "external_id": null}, {"id": 10847919003, "name": "The Catholic University of Korea", "priority": 2097, "external_id": null}, {"id": 10847920003, "name": "Tianjin University", "priority": 2098, "external_id": null}, {"id": 10847921003, "name": "Tokyo Metropolitan University", "priority": 2099, "external_id": null}, {"id": 10847922003, "name": "Universidad de Antioquia", "priority": 2100, "external_id": null}, {"id": 10847923003, "name": "University of Granada", "priority": 2101, "external_id": null}, {"id": 10847924003, "name": "Universidad de Palermo", "priority": 2102, "external_id": null}, {"id": 10847925003, "name": "Universidad Nacional de C\u00f3rdoba", "priority": 2103, "external_id": null}, {"id": 10847926003, "name": "Universidade de Santiago de Compostela", "priority": 2104, "external_id": null}, {"id": 10847927003, "name": "Universidade Federal do Rio Grande Do Sul", "priority": 2105, "external_id": null}, {"id": 10847928003, "name": "University of Siena", "priority": 2106, "external_id": null}, {"id": 10847929003, "name": "University of Trieste", "priority": 2107, "external_id": null}, {"id": 10847930003, "name": "Universitas Gadjah Mada", "priority": 2108, "external_id": null}, {"id": 10847931003, "name": "Universit\u00e9 de Lorraine", "priority": 2109, "external_id": null}, {"id": 10847932003, "name": "Universit\u00e9 de Rennes 1", "priority": 2110, "external_id": null}, {"id": 10847933003, "name": "University of Bradford", "priority": 2111, "external_id": null}, {"id": 10847934003, "name": "University of Hull", "priority": 2112, "external_id": null}, {"id": 10847935003, "name": "University of Kwazulu-Natal", "priority": 2113, "external_id": null}, {"id": 10847936003, "name": "University of Limerick", "priority": 2114, "external_id": null}, {"id": 10847937003, "name": "University of Stirling", "priority": 2115, "external_id": null}, {"id": 10847938003, "name": "University of Szeged", "priority": 2116, "external_id": null}, {"id": 10847939003, "name": "Ural Federal University", "priority": 2117, "external_id": null}, {"id": 10847940003, "name": "Xiamen University", "priority": 2118, "external_id": null}, {"id": 10847941003, "name": "Yokohama City University", "priority": 2119, "external_id": null}, {"id": 10847942003, "name": "Aberystwyth University", "priority": 2120, "external_id": null}, {"id": 10847943003, "name": "Belarus State University", "priority": 2121, "external_id": null}, {"id": 10847944003, "name": "Cairo University", "priority": 2122, "external_id": null}, {"id": 10847945003, "name": "Chiang Mai University", "priority": 2123, "external_id": null}, {"id": 10847946003, "name": "Chonbuk National University", "priority": 2124, "external_id": null}, {"id": 10847947003, "name": "E\u00f6tv\u00f6s Lor\u00e1nd University", "priority": 2125, "external_id": null}, {"id": 10847948003, "name": "Inha University", "priority": 2126, "external_id": null}, {"id": 10847949003, "name": "Instituto Polit\u00e9cnico Nacional (IPN)", "priority": 2127, "external_id": null}, {"id": 10847950003, "name": "Istanbul Technical University", "priority": 2128, "external_id": null}, {"id": 10847951003, "name": "Kumamoto University", "priority": 2129, "external_id": null}, {"id": 10847952003, "name": "Kyungpook National University", "priority": 2130, "external_id": null}, {"id": 10847953003, "name": "Lingnan University (Hong Kong)", "priority": 2131, "external_id": null}, {"id": 10847954003, "name": "Masaryk University", "priority": 2132, "external_id": null}, {"id": 10847955003, "name": "Murdoch University", "priority": 2133, "external_id": null}, {"id": 10847956003, "name": "Nagasaki University", "priority": 2134, "external_id": null}, {"id": 10847957003, "name": "National Chung Hsing University", "priority": 2135, "external_id": null}, {"id": 10847958003, "name": "National Taipei University of Technology", "priority": 2136, "external_id": null}, {"id": 10847959003, "name": "National University of Ireland Maynooth", "priority": 2137, "external_id": null}, {"id": 10847960003, "name": "Osaka City University", "priority": 2138, "external_id": null}, {"id": 10847961003, "name": "Pontificia Universidad Cat\u00f3lica del Per\u00fa", "priority": 2139, "external_id": null}, {"id": 10847962003, "name": "Pontificia Universidade Cat\u00f3lica de S\u00e3o Paulo (PUC -SP)", "priority": 2140, "external_id": null}, {"id": 10847963003, "name": "Pontificia Universidade Cat\u00f3lica do Rio de Janeiro (PUC - Rio)", "priority": 2141, "external_id": null}, {"id": 10847964003, "name": "Qatar University", "priority": 2142, "external_id": null}, {"id": 10847965003, "name": "Rhodes University", "priority": 2143, "external_id": null}, {"id": 10847966003, "name": "Tokyo University of Agriculture and Technology", "priority": 2144, "external_id": null}, {"id": 10847967003, "name": "Tomsk Polytechnic University", "priority": 2145, "external_id": null}, {"id": 10847968003, "name": "Tomsk State University", "priority": 2146, "external_id": null}, {"id": 10847969003, "name": "Umm Al-Qura University", "priority": 2147, "external_id": null}, {"id": 10847970003, "name": "Universidad Cat\u00f3lica Andr\u00e9s Bello - UCAB", "priority": 2148, "external_id": null}, {"id": 10847971003, "name": "Universidad Central de Venezuela - UCV", "priority": 2149, "external_id": null}, {"id": 10847972003, "name": "Universidad de Belgrano", "priority": 2150, "external_id": null}, {"id": 10847973003, "name": "Universidad de Concepci\u00f3n", "priority": 2151, "external_id": null}, {"id": 10847974003, "name": "Universidad de Sevilla", "priority": 2152, "external_id": null}, {"id": 10847975003, "name": "Universidade Catolica Portuguesa, Lisboa", "priority": 2153, "external_id": null}, {"id": 10847976003, "name": "Universidade de Brasilia (UnB)", "priority": 2154, "external_id": null}, {"id": 10847977003, "name": "University of Lisbon", "priority": 2155, "external_id": null}, {"id": 10847978003, "name": "University of Ljubljana", "priority": 2156, "external_id": null}, {"id": 10847979003, "name": "University of Seoul", "priority": 2157, "external_id": null}, {"id": 10847980003, "name": "Abu Dhabi University", "priority": 2158, "external_id": null}, {"id": 10847981003, "name": "Ain Shams University", "priority": 2159, "external_id": null}, {"id": 10847982003, "name": "Ajou University", "priority": 2160, "external_id": null}, {"id": 10847983003, "name": "De La Salle University", "priority": 2161, "external_id": null}, {"id": 10847984003, "name": "Dongguk University", "priority": 2162, "external_id": null}, {"id": 10847985003, "name": "Gifu University", "priority": 2163, "external_id": null}, {"id": 10847986003, "name": "Hacettepe University", "priority": 2164, "external_id": null}, {"id": 10847987003, "name": "Indian Institute of Technology Guwahati (IITG)", "priority": 2165, "external_id": null}, {"id": 10847988003, "name": "Jilin University", "priority": 2166, "external_id": null}, {"id": 10847989003, "name": "Kazan Federal University", "priority": 2167, "external_id": null}, {"id": 10847990003, "name": "King Khalid University", "priority": 2168, "external_id": null}, {"id": 10847991003, "name": "Martin-Luther-Universit\u00e4t Halle-Wittenberg", "priority": 2169, "external_id": null}, {"id": 10847992003, "name": "National Chengchi University", "priority": 2170, "external_id": null}, {"id": 10847993003, "name": "National Technical University of UkraineKyiv Polytechnic Institute'", "priority": 2171, "external_id": null}, {"id": 10847994003, "name": "Niigata University", "priority": 2172, "external_id": null}, {"id": 10847995003, "name": "Osaka Prefecture University", "priority": 2173, "external_id": null}, {"id": 10847996003, "name": "Paris Lodron University of Salzburg", "priority": 2174, "external_id": null}, {"id": 10847997003, "name": "Sharif University of Technology", "priority": 2175, "external_id": null}, {"id": 10847998003, "name": "Southern Federal University", "priority": 2176, "external_id": null}, {"id": 10847999003, "name": "Thammasat University", "priority": 2177, "external_id": null}, {"id": 10848000003, "name": "Universidad de Guadalajara (UDG)", "priority": 2178, "external_id": null}, {"id": 10848001003, "name": "Universidad de la Rep\u00fablica (UdelaR)", "priority": 2179, "external_id": null}, {"id": 10848002003, "name": "Universidad Iberoamericana (UIA)", "priority": 2180, "external_id": null}, {"id": 10848003003, "name": "Universidad Torcuato Di Tella", "priority": 2181, "external_id": null}, {"id": 10848004003, "name": "Universidade Federal da Bahia", "priority": 2182, "external_id": null}, {"id": 10848005003, "name": "Universidade Federal de S\u00e3o Carlos", "priority": 2183, "external_id": null}, {"id": 10848006003, "name": "Universidade Federal de Vi\u00e7osa", "priority": 2184, "external_id": null}, {"id": 10848007003, "name": "Perugia University", "priority": 2185, "external_id": null}, {"id": 10848008003, "name": "Universit\u00e9 de Nantes", "priority": 2186, "external_id": null}, {"id": 10848009003, "name": "Universit\u00e9 Saint-Joseph de Beyrouth", "priority": 2187, "external_id": null}, {"id": 10848010003, "name": "University of Canberra", "priority": 2188, "external_id": null}, {"id": 10848011003, "name": "University of Debrecen", "priority": 2189, "external_id": null}, {"id": 10848012003, "name": "University of Johannesburg", "priority": 2190, "external_id": null}, {"id": 10848013003, "name": "University of Mumbai", "priority": 2191, "external_id": null}, {"id": 10848014003, "name": "University of Patras", "priority": 2192, "external_id": null}, {"id": 10848015003, "name": "University of Tehran", "priority": 2193, "external_id": null}, {"id": 10848016003, "name": "University of Ulsan", "priority": 2194, "external_id": null}, {"id": 10848017003, "name": "University of Ulster", "priority": 2195, "external_id": null}, {"id": 10848018003, "name": "University of Zagreb", "priority": 2196, "external_id": null}, {"id": 10848019003, "name": "Vilnius University", "priority": 2197, "external_id": null}, {"id": 10848020003, "name": "Warsaw University of Technology", "priority": 2198, "external_id": null}, {"id": 10848021003, "name": "Al Azhar University", "priority": 2199, "external_id": null}, {"id": 10848022003, "name": "Bar-Ilan University", "priority": 2200, "external_id": null}, {"id": 10848023003, "name": "Brno University of Technology", "priority": 2201, "external_id": null}, {"id": 10848024003, "name": "Chonnam National University", "priority": 2202, "external_id": null}, {"id": 10848025003, "name": "Chungnam National University", "priority": 2203, "external_id": null}, {"id": 10848026003, "name": "Corvinus University of Budapest", "priority": 2204, "external_id": null}, {"id": 10848027003, "name": "Gunma University", "priority": 2205, "external_id": null}, {"id": 10848028003, "name": "Hallym University", "priority": 2206, "external_id": null}, {"id": 10848029003, "name": "Instituto Tecnol\u00f3gico Autonomo de M\u00e9xico (ITAM)", "priority": 2207, "external_id": null}, {"id": 10848030003, "name": "Istanbul University", "priority": 2208, "external_id": null}, {"id": 10848031003, "name": "Jordan University of Science & Technology", "priority": 2209, "external_id": null}, {"id": 10848032003, "name": "Kasetsart University", "priority": 2210, "external_id": null}, {"id": 10848033003, "name": "Kazakh-British Technical University", "priority": 2211, "external_id": null}, {"id": 10848034003, "name": "Khazar University", "priority": 2212, "external_id": null}, {"id": 10848035003, "name": "London Metropolitan University", "priority": 2213, "external_id": null}, {"id": 10848036003, "name": "Middlesex University", "priority": 2214, "external_id": null}, {"id": 10848037003, "name": "Universidad Industrial de Santander", "priority": 2215, "external_id": null}, {"id": 10848038003, "name": "Pontificia Universidad Cat\u00f3lica de Valpara\u00edso", "priority": 2216, "external_id": null}, {"id": 10848039003, "name": "Pontificia Universidade Cat\u00f3lica do Rio Grande do Sul", "priority": 2217, "external_id": null}, {"id": 10848040003, "name": "Qafqaz University", "priority": 2218, "external_id": null}, {"id": 10848041003, "name": "Ritsumeikan University", "priority": 2219, "external_id": null}, {"id": 10848042003, "name": "Shandong University", "priority": 2220, "external_id": null}, {"id": 10848043003, "name": "University of St. Kliment Ohridski", "priority": 2221, "external_id": null}, {"id": 10848044003, "name": "South Kazakhstan State University (SKSU)", "priority": 2222, "external_id": null}, {"id": 10848045003, "name": "Universidad Adolfo Ib\u00e1\u00f1ez", "priority": 2223, "external_id": null}, {"id": 10848046003, "name": "Universidad Aut\u00f3noma del Estado de M\u00e9xico", "priority": 2224, "external_id": null}, {"id": 10848047003, "name": "Universidad Aut\u00f3noma Metropolitana (UAM)", "priority": 2225, "external_id": null}, {"id": 10848048003, "name": "Universidad de Alcal\u00e1", "priority": 2226, "external_id": null}, {"id": 10848049003, "name": "Universidad Nacional Costa Rica", "priority": 2227, "external_id": null}, {"id": 10848050003, "name": "Universidad Nacional de Mar del Plata", "priority": 2228, "external_id": null}, {"id": 10848051003, "name": "Universidad Peruana Cayetano Heredia", "priority": 2229, "external_id": null}, {"id": 10848052003, "name": "Universidad Sim\u00f3n Bol\u00edvar Venezuela", "priority": 2230, "external_id": null}, {"id": 10848053003, "name": "Universidade Federal de Santa Catarina", "priority": 2231, "external_id": null}, {"id": 10848054003, "name": "Universidade Federal do Paran\u00e1 (UFPR)", "priority": 2232, "external_id": null}, {"id": 10848055003, "name": "Universidade Federal Fluminense", "priority": 2233, "external_id": null}, {"id": 10848056003, "name": "University of Modena", "priority": 2234, "external_id": null}, {"id": 10848057003, "name": "Universit\u00e9 Lumi\u00e8re Lyon 2", "priority": 2235, "external_id": null}, {"id": 10848058003, "name": "Universit\u00e9 Toulouse 1, Capitole", "priority": 2236, "external_id": null}, {"id": 10848059003, "name": "University of Economics Prague", "priority": 2237, "external_id": null}, {"id": 10848060003, "name": "University of Hertfordshire", "priority": 2238, "external_id": null}, {"id": 10848061003, "name": "University of Plymouth", "priority": 2239, "external_id": null}, {"id": 10848062003, "name": "University of Salford", "priority": 2240, "external_id": null}, {"id": 10848063003, "name": "University of Science and Technology Beijing", "priority": 2241, "external_id": null}, {"id": 10848064003, "name": "University of Western Sydney", "priority": 2242, "external_id": null}, {"id": 10848065003, "name": "Yamaguchi University", "priority": 2243, "external_id": null}, {"id": 10848066003, "name": "Yokohama National University", "priority": 2244, "external_id": null}, {"id": 10848067003, "name": "Airlangga University", "priority": 2245, "external_id": null}, {"id": 10848068003, "name": "Alexandria University", "priority": 2246, "external_id": null}, {"id": 10848069003, "name": "Alexandru Ioan Cuza University", "priority": 2247, "external_id": null}, {"id": 10848070003, "name": "Alpen-Adria-Universit\u00e4t Klagenfurt", "priority": 2248, "external_id": null}, {"id": 10848071003, "name": "Aoyama Gakuin University", "priority": 2249, "external_id": null}, {"id": 10848072003, "name": "Athens University of Economy And Business", "priority": 2250, "external_id": null}, {"id": 10848073003, "name": "Babes-Bolyai University", "priority": 2251, "external_id": null}, {"id": 10848074003, "name": "Baku State University", "priority": 2252, "external_id": null}, {"id": 10848075003, "name": "Belarusian National Technical University", "priority": 2253, "external_id": null}, {"id": 10848076003, "name": "Benem\u00e9rita Universidad Aut\u00f3noma de Puebla", "priority": 2254, "external_id": null}, {"id": 10848077003, "name": "Bogor Agricultural University", "priority": 2255, "external_id": null}, {"id": 10848078003, "name": "Coventry University", "priority": 2256, "external_id": null}, {"id": 10848079003, "name": "Cukurova University", "priority": 2257, "external_id": null}, {"id": 10848080003, "name": "Diponegoro University", "priority": 2258, "external_id": null}, {"id": 10848081003, "name": "Donetsk National University", "priority": 2259, "external_id": null}, {"id": 10848082003, "name": "Doshisha University", "priority": 2260, "external_id": null}, {"id": 10848083003, "name": "E.A.Buketov Karaganda State University", "priority": 2261, "external_id": null}, {"id": 10848084003, "name": "Far Eastern Federal University", "priority": 2262, "external_id": null}, {"id": 10848085003, "name": "Fu Jen Catholic University", "priority": 2263, "external_id": null}, {"id": 10848086003, "name": "Kagoshima University", "priority": 2264, "external_id": null}, {"id": 10848087003, "name": "Kaunas University of Technology", "priority": 2265, "external_id": null}, {"id": 10848088003, "name": "Kazakh Ablai khan University of International Relations and World Languages", "priority": 2266, "external_id": null}, {"id": 10848089003, "name": "Kazakh National Pedagogical University Abai", "priority": 2267, "external_id": null}, {"id": 10848090003, "name": "Kazakh National Technical University", "priority": 2268, "external_id": null}, {"id": 10848091003, "name": "Khon Kaen University", "priority": 2269, "external_id": null}, {"id": 10848092003, "name": "King Faisal University", "priority": 2270, "external_id": null}, {"id": 10848093003, "name": "King Mongkut''s University of Technology Thonburi", "priority": 2271, "external_id": null}, {"id": 10848094003, "name": "Kuwait University", "priority": 2272, "external_id": null}, {"id": 10848095003, "name": "Lodz University", "priority": 2273, "external_id": null}, {"id": 10848096003, "name": "Manchester Metropolitan University", "priority": 2274, "external_id": null}, {"id": 10848097003, "name": "Lobachevsky State University of Nizhni Novgorod", "priority": 2275, "external_id": null}, {"id": 10848098003, "name": "National Technical UniversityKharkiv Polytechnic Institute'", "priority": 2276, "external_id": null}, {"id": 10848099003, "name": "Nicolaus Copernicus University", "priority": 2277, "external_id": null}, {"id": 10848100003, "name": "Northumbria University at Newcastle", "priority": 2278, "external_id": null}, {"id": 10848101003, "name": "Nottingham Trent University", "priority": 2279, "external_id": null}, {"id": 10848102003, "name": "Ochanomizu University", "priority": 2280, "external_id": null}, {"id": 10848103003, "name": "Plekhanov Russian University of Economics", "priority": 2281, "external_id": null}, {"id": 10848104003, "name": "Pontificia Universidad Catolica del Ecuador", "priority": 2282, "external_id": null}, {"id": 10848105003, "name": "Prince of Songkla University", "priority": 2283, "external_id": null}, {"id": 10848106003, "name": "S.Seifullin Kazakh Agro Technical University", "priority": 2284, "external_id": null}, {"id": 10848107003, "name": "Saitama University", "priority": 2285, "external_id": null}, {"id": 10848108003, "name": "Sepuluh Nopember Institute of Technology", "priority": 2286, "external_id": null}, {"id": 10848109003, "name": "Shinshu University", "priority": 2287, "external_id": null}, {"id": 10848110003, "name": "The Robert Gordon University", "priority": 2288, "external_id": null}, {"id": 10848111003, "name": "Tokai University", "priority": 2289, "external_id": null}, {"id": 10848112003, "name": "Universidad ANAHUAC", "priority": 2290, "external_id": null}, {"id": 10848113003, "name": "Universidad Austral de Chile", "priority": 2291, "external_id": null}, {"id": 10848114003, "name": "University Aut\u00f3noma de Nuevo Le\u00f3n (UANL)", "priority": 2292, "external_id": null}, {"id": 10848115003, "name": "Universidad de la Habana", "priority": 2293, "external_id": null}, {"id": 10848116003, "name": "Universidad de La Sabana", "priority": 2294, "external_id": null}, {"id": 10848117003, "name": "Universidad de las Am\u00e9ricas Puebla (UDLAP)", "priority": 2295, "external_id": null}, {"id": 10848118003, "name": "Universidad de los Andes M\u00e9rida", "priority": 2296, "external_id": null}, {"id": 10848119003, "name": "University of Murcia", "priority": 2297, "external_id": null}, {"id": 10848120003, "name": "Universidad de Puerto Rico", "priority": 2298, "external_id": null}, {"id": 10848121003, "name": "Universidad de San Francisco de Quito", "priority": 2299, "external_id": null}, {"id": 10848122003, "name": "Universidad de Talca", "priority": 2300, "external_id": null}, {"id": 10848123003, "name": "Universidad del Norte", "priority": 2301, "external_id": null}, {"id": 10848124003, "name": "Universidad del Rosario", "priority": 2302, "external_id": null}, {"id": 10848125003, "name": "Universidad del Valle", "priority": 2303, "external_id": null}, {"id": 10848126003, "name": "Universidad Nacional de Cuyo", "priority": 2304, "external_id": null}, {"id": 10848127003, "name": "Universidad Nacional de Rosario", "priority": 2305, "external_id": null}, {"id": 10848128003, "name": "Universidad Nacional de Tucum\u00e1n", "priority": 2306, "external_id": null}, {"id": 10848129003, "name": "Universidad Nacional del Sur", "priority": 2307, "external_id": null}, {"id": 10848130003, "name": "Universidad Nacional Mayor de San Marcos", "priority": 2308, "external_id": null}, {"id": 10848131003, "name": "Universidad T\u00e9cnica Federico Santa Mar\u00eda", "priority": 2309, "external_id": null}, {"id": 10848132003, "name": "Universidad Tecnol\u00f3gica Nacional (UTN)", "priority": 2310, "external_id": null}, {"id": 10848133003, "name": "Universidade do Estado do Rio de Janeiro (UERJ)", "priority": 2311, "external_id": null}, {"id": 10848134003, "name": "Universidade Estadual de Londrina (UEL)", "priority": 2312, "external_id": null}, {"id": 10848135003, "name": "Universidade Federal de Santa Maria", "priority": 2313, "external_id": null}, {"id": 10848136003, "name": "Universidade Federal do Cear\u00e1 (UFC)", "priority": 2314, "external_id": null}, {"id": 10848137003, "name": "Universidade Federal do Pernambuco", "priority": 2315, "external_id": null}, {"id": 10848138003, "name": "Universit\u00e0 Ca'' Foscari Venezia", "priority": 2316, "external_id": null}, {"id": 10848139003, "name": "Catania University", "priority": 2317, "external_id": null}, {"id": 10848140003, "name": "Universit\u00e0 degli Studi Roma Tre", "priority": 2318, "external_id": null}, {"id": 10848141003, "name": "Universit\u00e9 Charles-de-Gaulle Lille 3", "priority": 2319, "external_id": null}, {"id": 10848142003, "name": "Universit\u00e9 de Caen Basse-Normandie", "priority": 2320, "external_id": null}, {"id": 10848143003, "name": "Universit\u00e9 de Cergy-Pontoise", "priority": 2321, "external_id": null}, {"id": 10848144003, "name": "Universit\u00e9 de Poitiers", "priority": 2322, "external_id": null}, {"id": 10848145003, "name": "Universit\u00e9 Jean Moulin Lyon 3", "priority": 2323, "external_id": null}, {"id": 10848146003, "name": "Universit\u00e9 Lille 2 Droit et Sant\u00e9", "priority": 2324, "external_id": null}, {"id": 10848147003, "name": "Universit\u00e9 Paris Ouest Nanterre La D\u00e9fense", "priority": 2325, "external_id": null}, {"id": 10848148003, "name": "Universit\u00e9 Paul-Val\u00e9ry Montpellier 3", "priority": 2326, "external_id": null}, {"id": 10848149003, "name": "Universit\u00e9 Pierre Mend\u00e8s France - Grenoble 2", "priority": 2327, "external_id": null}, {"id": 10848150003, "name": "Universit\u00e9 Stendhal Grenoble 3", "priority": 2328, "external_id": null}, {"id": 10848151003, "name": "Universit\u00e9 Toulouse II, Le Mirail", "priority": 2329, "external_id": null}, {"id": 10848152003, "name": "Universiti Teknologi MARA - UiTM", "priority": 2330, "external_id": null}, {"id": 10848153003, "name": "University of Baghdad", "priority": 2331, "external_id": null}, {"id": 10848154003, "name": "University of Bahrain", "priority": 2332, "external_id": null}, {"id": 10848155003, "name": "University of Bari", "priority": 2333, "external_id": null}, {"id": 10848156003, "name": "University of Belgrade", "priority": 2334, "external_id": null}, {"id": 10848157003, "name": "University of Brawijaya", "priority": 2335, "external_id": null}, {"id": 10848158003, "name": "University of Brescia", "priority": 2336, "external_id": null}, {"id": 10848159003, "name": "University of Bucharest", "priority": 2337, "external_id": null}, {"id": 10848160003, "name": "University of Calcutta", "priority": 2338, "external_id": null}, {"id": 10848161003, "name": "University of Central Lancashire", "priority": 2339, "external_id": null}, {"id": 10848162003, "name": "University of Colombo", "priority": 2340, "external_id": null}, {"id": 10848163003, "name": "University of Dhaka", "priority": 2341, "external_id": null}, {"id": 10848164003, "name": "University of East London", "priority": 2342, "external_id": null}, {"id": 10848165003, "name": "University of Engineering & Technology (UET) Lahore", "priority": 2343, "external_id": null}, {"id": 10848166003, "name": "University of Greenwich", "priority": 2344, "external_id": null}, {"id": 10848167003, "name": "University of Jordan", "priority": 2345, "external_id": null}, {"id": 10848168003, "name": "University of Karachi", "priority": 2346, "external_id": null}, {"id": 10848169003, "name": "University of Lahore", "priority": 2347, "external_id": null}, {"id": 10848170003, "name": "University of Latvia", "priority": 2348, "external_id": null}, {"id": 10848171003, "name": "University of New England", "priority": 2349, "external_id": null}, {"id": 10848172003, "name": "University of Pune", "priority": 2350, "external_id": null}, {"id": 10848173003, "name": "University of Santo Tomas", "priority": 2351, "external_id": null}, {"id": 10848174003, "name": "University of Southern Queensland", "priority": 2352, "external_id": null}, {"id": 10848175003, "name": "University of Wroclaw", "priority": 2353, "external_id": null}, {"id": 10848176003, "name": "Verona University", "priority": 2354, "external_id": null}, {"id": 10848177003, "name": "Victoria University", "priority": 2355, "external_id": null}, {"id": 10848178003, "name": "Vilnius Gediminas Technical University", "priority": 2356, "external_id": null}, {"id": 10848179003, "name": "Voronezh State University", "priority": 2357, "external_id": null}, {"id": 10848180003, "name": "Vytautas Magnus University", "priority": 2358, "external_id": null}, {"id": 10848181003, "name": "West University of Timisoara", "priority": 2359, "external_id": null}, {"id": 10848182003, "name": "University of South Alabama", "priority": 2360, "external_id": null}, {"id": 10848183003, "name": "University of Arkansas", "priority": 2361, "external_id": null}, {"id": 10848184003, "name": "University of California - Berkeley", "priority": 2362, "external_id": null}, {"id": 10848185003, "name": "University of Connecticut", "priority": 2363, "external_id": null}, {"id": 10848186003, "name": "University of South Florida", "priority": 2364, "external_id": null}, {"id": 10848187003, "name": "University of Georgia", "priority": 2365, "external_id": null}, {"id": 10848188003, "name": "University of Hawaii - Manoa", "priority": 2366, "external_id": null}, {"id": 10848189003, "name": "Iowa State University", "priority": 2367, "external_id": null}, {"id": 10848190003, "name": "Murray State University", "priority": 2368, "external_id": null}, {"id": 10848191003, "name": "University of Louisville", "priority": 2369, "external_id": null}, {"id": 10848192003, "name": "Western Kentucky University", "priority": 2370, "external_id": null}, {"id": 10848193003, "name": "Louisiana State University - Baton Rouge", "priority": 2371, "external_id": null}, {"id": 10848194003, "name": "University of Maryland - College Park", "priority": 2372, "external_id": null}, {"id": 10848195003, "name": "University of Minnesota - Twin Cities", "priority": 2373, "external_id": null}, {"id": 10848196003, "name": "University of Montana", "priority": 2374, "external_id": null}, {"id": 10848197003, "name": "East Carolina University", "priority": 2375, "external_id": null}, {"id": 10848198003, "name": "University of North Carolina - Chapel Hill", "priority": 2376, "external_id": null}, {"id": 10848199003, "name": "Wake Forest University", "priority": 2377, "external_id": null}, {"id": 10848200003, "name": "University of Nebraska - Lincoln", "priority": 2378, "external_id": null}, {"id": 10848201003, "name": "New Mexico State University", "priority": 2379, "external_id": null}, {"id": 10848202003, "name": "Ohio State University - Columbus", "priority": 2380, "external_id": null}, {"id": 10848203003, "name": "University of Oklahoma", "priority": 2381, "external_id": null}, {"id": 10848204003, "name": "Pennsylvania State University - University Park", "priority": 2382, "external_id": null}, {"id": 10848205003, "name": "University of Pittsburgh", "priority": 2383, "external_id": null}, {"id": 10848206003, "name": "University of Tennessee - Chattanooga", "priority": 2384, "external_id": null}, {"id": 10848207003, "name": "Vanderbilt University", "priority": 2385, "external_id": null}, {"id": 10848208003, "name": "Rice University", "priority": 2386, "external_id": null}, {"id": 10848209003, "name": "University of Utah", "priority": 2387, "external_id": null}, {"id": 10848210003, "name": "University of Richmond", "priority": 2388, "external_id": null}, {"id": 10848211003, "name": "University of Arkansas - Pine Bluff", "priority": 2389, "external_id": null}, {"id": 10848212003, "name": "University of Central Florida", "priority": 2390, "external_id": null}, {"id": 10848213003, "name": "Florida Atlantic University", "priority": 2391, "external_id": null}, {"id": 10848214003, "name": "Hampton University", "priority": 2392, "external_id": null}, {"id": 10848215003, "name": "Liberty University", "priority": 2393, "external_id": null}, {"id": 10848216003, "name": "Mercer University", "priority": 2394, "external_id": null}, {"id": 10848217003, "name": "Middle Tennessee State University", "priority": 2395, "external_id": null}, {"id": 10848218003, "name": "University of Nevada - Las Vegas", "priority": 2396, "external_id": null}, {"id": 10848219003, "name": "South Carolina State University", "priority": 2397, "external_id": null}, {"id": 10848220003, "name": "University of Tennessee - Martin", "priority": 2398, "external_id": null}, {"id": 10848221003, "name": "Weber State University", "priority": 2399, "external_id": null}, {"id": 10848222003, "name": "Youngstown State University", "priority": 2400, "external_id": null}, {"id": 10848223003, "name": "University of the Incarnate Word", "priority": 2401, "external_id": null}, {"id": 10848224003, "name": "University of Washington", "priority": 2402, "external_id": null}, {"id": 10848225003, "name": "University of Louisiana - Lafayette", "priority": 2403, "external_id": null}, {"id": 10848226003, "name": "Coastal Carolina University", "priority": 2404, "external_id": null}, {"id": 10848227003, "name": "Utah State University", "priority": 2405, "external_id": null}, {"id": 10848228003, "name": "University of Alabama", "priority": 2406, "external_id": null}, {"id": 10848229003, "name": "University of Illinois - Urbana-Champaign", "priority": 2407, "external_id": null}, {"id": 10848230003, "name": "United States Air Force Academy", "priority": 2408, "external_id": null}, {"id": 10848231003, "name": "University of Akron", "priority": 2409, "external_id": null}, {"id": 10848232003, "name": "University of Central Arkansas", "priority": 2410, "external_id": null}, {"id": 10848233003, "name": "University of Kansas", "priority": 2411, "external_id": null}, {"id": 10848234003, "name": "University of Northern Colorado", "priority": 2412, "external_id": null}, {"id": 10848235003, "name": "University of Northern Iowa", "priority": 2413, "external_id": null}, {"id": 10848236003, "name": "University of South Carolina", "priority": 2414, "external_id": null}, {"id": 10848237003, "name": "Tennessee Technological University", "priority": 2415, "external_id": null}, {"id": 10848238003, "name": "University of Texas - El Paso", "priority": 2416, "external_id": null}, {"id": 10848239003, "name": "Texas Tech University", "priority": 2417, "external_id": null}, {"id": 10848240003, "name": "Tulane University", "priority": 2418, "external_id": null}, {"id": 10848241003, "name": "Virginia Military Institute", "priority": 2419, "external_id": null}, {"id": 10848242003, "name": "Western Michigan University", "priority": 2420, "external_id": null}, {"id": 10848243003, "name": "Wilfrid Laurier University", "priority": 2421, "external_id": null}, {"id": 10848244003, "name": "University of San Diego", "priority": 2422, "external_id": null}, {"id": 10848245003, "name": "University of California - San Diego", "priority": 2423, "external_id": null}, {"id": 10848246003, "name": "Brooks Institute of Photography", "priority": 2424, "external_id": null}, {"id": 10848247003, "name": "Acupuncture and Integrative Medicine College - Berkeley", "priority": 2425, "external_id": null}, {"id": 10848248003, "name": "Southern Alberta Institute of Technology", "priority": 2426, "external_id": null}, {"id": 10848249003, "name": "Susquehanna University", "priority": 2427, "external_id": null}, {"id": 10848250003, "name": "University of Texas - Dallas", "priority": 2428, "external_id": null}, {"id": 10848251003, "name": "Thunderbird School of Global Management", "priority": 2429, "external_id": null}, {"id": 10848252003, "name": "Presidio Graduate School", "priority": 2430, "external_id": null}, {"id": 10848253003, "name": "\u00c9cole sup\u00e9rieure de commerce de Dijon", "priority": 2431, "external_id": null}, {"id": 10848254003, "name": "University of California - San Francisco", "priority": 2432, "external_id": null}, {"id": 10848255003, "name": "Hack Reactor", "priority": 2433, "external_id": null}, {"id": 10848256003, "name": "St. Mary''s College of California", "priority": 2434, "external_id": null}, {"id": 10848257003, "name": "New England Law", "priority": 2435, "external_id": null}, {"id": 10848258003, "name": "University of California, Merced", "priority": 2436, "external_id": null}, {"id": 10848259003, "name": "University of California, Hastings College of the Law", "priority": 2437, "external_id": null}, {"id": 10848260003, "name": "V.N. Karazin Kharkiv National University", "priority": 2438, "external_id": null}, {"id": 10848261003, "name": "SIM University (UniSIM)", "priority": 2439, "external_id": null}, {"id": 10848262003, "name": "Singapore Management University (SMU)", "priority": 2440, "external_id": null}, {"id": 10848263003, "name": "Singapore University of Technology and Design (SUTD)", "priority": 2441, "external_id": null}, {"id": 10848264003, "name": "Singapore Institute of Technology (SIT)", "priority": 2442, "external_id": null}, {"id": 10848265003, "name": "Nanyang Polytechnic (NYP)", "priority": 2443, "external_id": null}, {"id": 10848266003, "name": "Ngee Ann Polytechnic (NP)", "priority": 2444, "external_id": null}, {"id": 10848267003, "name": "Republic Polytechnic (RP)", "priority": 2445, "external_id": null}, {"id": 10848268003, "name": "Singapore Polytechnic (SP)", "priority": 2446, "external_id": null}, {"id": 10848269003, "name": "Temasek Polytechnic (TP)", "priority": 2447, "external_id": null}, {"id": 10848270003, "name": "INSEAD", "priority": 2448, "external_id": null}, {"id": 10848271003, "name": "Funda\u00e7\u00e3o Get\u00falio Vargas", "priority": 2449, "external_id": null}, {"id": 10848272003, "name": "Acharya Nagarjuna University", "priority": 2450, "external_id": null}, {"id": 10848273003, "name": "University of California - Santa Barbara", "priority": 2451, "external_id": null}, {"id": 10848274003, "name": "University of California - Irvine", "priority": 2452, "external_id": null}, {"id": 10848275003, "name": "California State University - Long Beach", "priority": 2453, "external_id": null}, {"id": 10848276003, "name": "Robert Morris University Illinois", "priority": 2454, "external_id": null}, {"id": 10848277003, "name": "Harold Washington College - City Colleges of Chicago", "priority": 2455, "external_id": null}, {"id": 10848278003, "name": "Harry S Truman College - City Colleges of Chicago", "priority": 2456, "external_id": null}, {"id": 10848279003, "name": "Kennedy-King College - City Colleges of Chicago", "priority": 2457, "external_id": null}, {"id": 10848280003, "name": "Malcolm X College - City Colleges of Chicago", "priority": 2458, "external_id": null}, {"id": 10848281003, "name": "Olive-Harvey College - City Colleges of Chicago", "priority": 2459, "external_id": null}, {"id": 10848282003, "name": "Richard J Daley College - City Colleges of Chicago", "priority": 2460, "external_id": null}, {"id": 10848283003, "name": "Wilbur Wright College - City Colleges of Chicago", "priority": 2461, "external_id": null}, {"id": 10848284003, "name": "Abertay University", "priority": 2462, "external_id": null}, {"id": 10848285003, "name": "Pontif\u00edcia Universidade Cat\u00f3lica de Minas Gerais", "priority": 2463, "external_id": null}, {"id": 10848286003, "name": "Other", "priority": 2464, "external_id": null}, {"id": 19126655003, "name": "Atlanta College of Arts", "priority": 2465, "external_id": null}]}, "emitted_at": 1691572567819} +{"stream": "custom_fields", "data": {"id": 4680899003, "name": "Degree", "active": true, "field_type": "candidate", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "degree", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, {"id": 10848290003, "name": "Master's Degree", "priority": 3, "external_id": null}, {"id": 10848291003, "name": "Master of Business Administration (M.B.A.)", "priority": 4, "external_id": null}, {"id": 10848292003, "name": "Juris Doctor (J.D.)", "priority": 5, "external_id": null}, {"id": 10848293003, "name": "Doctor of Medicine (M.D.)", "priority": 6, "external_id": null}, {"id": 10848294003, "name": "Doctor of Philosophy (Ph.D.)", "priority": 7, "external_id": null}, {"id": 10848295003, "name": "Engineer's Degree", "priority": 8, "external_id": null}, {"id": 10848296003, "name": "Other", "priority": 9, "external_id": null}]}, "emitted_at": 1691572567849} +{"stream": "custom_fields", "data": {"id": 4680900003, "name": "Discipline", "active": true, "field_type": "candidate", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "discipline", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848297003, "name": "Accounting", "priority": 0, "external_id": null}, {"id": 10848298003, "name": "African Studies", "priority": 1, "external_id": null}, {"id": 10848299003, "name": "Agriculture", "priority": 2, "external_id": null}, {"id": 10848300003, "name": "Anthropology", "priority": 3, "external_id": null}, {"id": 10848301003, "name": "Applied Health Services", "priority": 4, "external_id": null}, {"id": 10848302003, "name": "Architecture", "priority": 5, "external_id": null}, {"id": 10848303003, "name": "Art", "priority": 6, "external_id": null}, {"id": 10848304003, "name": "Asian Studies", "priority": 7, "external_id": null}, {"id": 10848305003, "name": "Biology", "priority": 8, "external_id": null}, {"id": 10848306003, "name": "Business", "priority": 9, "external_id": null}, {"id": 10848307003, "name": "Business Administration", "priority": 10, "external_id": null}, {"id": 10848308003, "name": "Chemistry", "priority": 11, "external_id": null}, {"id": 10848309003, "name": "Classical Languages", "priority": 12, "external_id": null}, {"id": 10848310003, "name": "Communications & Film", "priority": 13, "external_id": null}, {"id": 10848311003, "name": "Computer Science", "priority": 14, "external_id": null}, {"id": 10848312003, "name": "Dentistry", "priority": 15, "external_id": null}, {"id": 10848313003, "name": "Developing Nations", "priority": 16, "external_id": null}, {"id": 10848314003, "name": "Discipline Unknown", "priority": 17, "external_id": null}, {"id": 10848315003, "name": "Earth Sciences", "priority": 18, "external_id": null}, {"id": 10848316003, "name": "Economics", "priority": 19, "external_id": null}, {"id": 10848317003, "name": "Education", "priority": 20, "external_id": null}, {"id": 10848318003, "name": "Electronics", "priority": 21, "external_id": null}, {"id": 10848319003, "name": "Engineering", "priority": 22, "external_id": null}, {"id": 10848320003, "name": "English Studies", "priority": 23, "external_id": null}, {"id": 10848321003, "name": "Environmental Studies", "priority": 24, "external_id": null}, {"id": 10848322003, "name": "European Studies", "priority": 25, "external_id": null}, {"id": 10848323003, "name": "Fashion", "priority": 26, "external_id": null}, {"id": 10848324003, "name": "Finance", "priority": 27, "external_id": null}, {"id": 10848325003, "name": "Fine Arts", "priority": 28, "external_id": null}, {"id": 10848326003, "name": "General Studies", "priority": 29, "external_id": null}, {"id": 10848327003, "name": "Health Services", "priority": 30, "external_id": null}, {"id": 10848328003, "name": "History", "priority": 31, "external_id": null}, {"id": 10848329003, "name": "Human Resources Management", "priority": 32, "external_id": null}, {"id": 10848330003, "name": "Humanities", "priority": 33, "external_id": null}, {"id": 10848331003, "name": "Industrial Arts & Carpentry", "priority": 34, "external_id": null}, {"id": 10848332003, "name": "Information Systems", "priority": 35, "external_id": null}, {"id": 10848333003, "name": "International Relations", "priority": 36, "external_id": null}, {"id": 10848334003, "name": "Journalism", "priority": 37, "external_id": null}, {"id": 10848335003, "name": "Languages", "priority": 38, "external_id": null}, {"id": 10848336003, "name": "Latin American Studies", "priority": 39, "external_id": null}, {"id": 10848337003, "name": "Law", "priority": 40, "external_id": null}, {"id": 10848338003, "name": "Linguistics", "priority": 41, "external_id": null}, {"id": 10848339003, "name": "Manufacturing & Mechanics", "priority": 42, "external_id": null}, {"id": 10848340003, "name": "Mathematics", "priority": 43, "external_id": null}, {"id": 10848341003, "name": "Medicine", "priority": 44, "external_id": null}, {"id": 10848342003, "name": "Middle Eastern Studies", "priority": 45, "external_id": null}, {"id": 10848343003, "name": "Naval Science", "priority": 46, "external_id": null}, {"id": 10848344003, "name": "North American Studies", "priority": 47, "external_id": null}, {"id": 10848345003, "name": "Nuclear Technics", "priority": 48, "external_id": null}, {"id": 10848346003, "name": "Operations Research & Strategy", "priority": 49, "external_id": null}, {"id": 10848347003, "name": "Organizational Theory", "priority": 50, "external_id": null}, {"id": 10848348003, "name": "Philosophy", "priority": 51, "external_id": null}, {"id": 10848349003, "name": "Physical Education", "priority": 52, "external_id": null}, {"id": 10848350003, "name": "Physical Sciences", "priority": 53, "external_id": null}, {"id": 10848351003, "name": "Physics", "priority": 54, "external_id": null}, {"id": 10848352003, "name": "Political Science", "priority": 55, "external_id": null}, {"id": 10848353003, "name": "Psychology", "priority": 56, "external_id": null}, {"id": 10848354003, "name": "Public Policy", "priority": 57, "external_id": null}, {"id": 10848355003, "name": "Public Service", "priority": 58, "external_id": null}, {"id": 10848356003, "name": "Religious Studies", "priority": 59, "external_id": null}, {"id": 10848357003, "name": "Russian & Soviet Studies", "priority": 60, "external_id": null}, {"id": 10848358003, "name": "Scandinavian Studies", "priority": 61, "external_id": null}, {"id": 10848359003, "name": "Science", "priority": 62, "external_id": null}, {"id": 10848360003, "name": "Slavic Studies", "priority": 63, "external_id": null}, {"id": 10848361003, "name": "Social Science", "priority": 64, "external_id": null}, {"id": 10848362003, "name": "Social Sciences", "priority": 65, "external_id": null}, {"id": 10848363003, "name": "Sociology", "priority": 66, "external_id": null}, {"id": 10848364003, "name": "Speech", "priority": 67, "external_id": null}, {"id": 10848365003, "name": "Statistics & Decision Theory", "priority": 68, "external_id": null}, {"id": 10848366003, "name": "Urban Studies", "priority": 69, "external_id": null}, {"id": 10848367003, "name": "Veterinary Medicine", "priority": 70, "external_id": null}, {"id": 10848368003, "name": "Other", "priority": 71, "external_id": null}]}, "emitted_at": 1691572567852} +{"stream": "degrees", "data": {"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, "emitted_at": 1691572568468} +{"stream": "degrees", "data": {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, "emitted_at": 1691572568471} +{"stream": "degrees", "data": {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, "emitted_at": 1691572568473} +{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.248Z", "id": 9308815003, "free_form_text": null, "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004262003, "created_at": "2021-11-03T19:56:07.248Z", "application_id": 47459993003}, "emitted_at": 1691572569084} +{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.252Z", "id": 9308816003, "free_form_text": "custom answer", "demographic_question_id": 4000716003, "demographic_answer_option_id": 4004263003, "created_at": "2021-11-03T19:56:07.252Z", "application_id": 47459993003}, "emitted_at": 1691572569087} +{"stream": "demographics_answers", "data": {"updated_at": "2021-11-03T19:56:07.259Z", "id": 9308817003, "free_form_text": null, "demographic_question_id": 4000717003, "demographic_answer_option_id": 4004266003, "created_at": "2021-11-03T19:56:07.259Z", "application_id": 47459993003}, "emitted_at": 1691572569089} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a1", "language": "en"}], "name": "a1", "id": 4004258003, "free_form": false, "demographic_question_id": 4000714003, "active": true}, "emitted_at": 1691572569704} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a2", "language": "en"}], "name": "a2", "id": 4004259003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1691572569707} +{"stream": "demographics_answer_options", "data": {"translations": [{"name": "a3", "language": "en"}], "name": "a3", "id": 4004260003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1691572569709} +{"stream": "demographics_questions", "data": {"translations": [{"name": "q1", "language": "en"}], "required": false, "name": "q1", "id": 4000714003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1691572570395} +{"stream": "demographics_questions", "data": {"translations": [{"name": "q2", "language": "en"}], "required": false, "name": "q2", "id": 4000715003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1691572570398} +{"stream": "demographics_questions", "data": {"translations": [{"name": "question1", "language": "en"}], "required": false, "name": "question1", "id": 4000716003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1691572570401} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a1", "language": "en"}], "name": "a1", "id": 4004258003, "free_form": false, "demographic_question_id": 4000714003, "active": true}, "emitted_at": 1691572571510} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a2", "language": "en"}], "name": "a2", "id": 4004259003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1691572571858} +{"stream": "demographics_answers_answer_options", "data": {"translations": [{"name": "a3", "language": "en"}], "name": "a3", "id": 4004260003, "free_form": false, "demographic_question_id": 4000715003, "active": true}, "emitted_at": 1691572571860} +{"stream": "demographics_question_sets", "data": {"title": "Test Question Set 1", "id": 4000197003, "description": "

    Test Question Set 1 description

    ", "active": true}, "emitted_at": 1691572575338} +{"stream": "demographics_question_sets", "data": {"title": "Test Question Set 2", "id": 4000198003, "description": "

    Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

    ", "active": true}, "emitted_at": 1691572575341} +{"stream": "demographics_question_sets", "data": {"title": "U.S. Standard Demographic Questions", "id": 4002702003, "description": "We invite applicants to share their demographic background. If you choose to complete this survey, your responses may be used to identify areas of improvement in our hiring process.", "active": true}, "emitted_at": 1691572575343} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "q1", "language": "en"}], "required": false, "name": "q1", "id": 4000714003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_single_select", "active": true}, "emitted_at": 1691572576350} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "q2", "language": "en"}], "required": false, "name": "q2", "id": 4000715003, "demographic_question_set_id": 4000197003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1691572576353} +{"stream": "demographics_question_sets_questions", "data": {"translations": [{"name": "question1", "language": "en"}], "required": false, "name": "question1", "id": 4000716003, "demographic_question_set_id": 4000198003, "answer_type": "multi_value_multi_select", "active": true}, "emitted_at": 1691572576675} +{"stream": "departments", "data": {"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}, "emitted_at": 1691572577771} +{"stream": "departments", "data": {"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}, "emitted_at": 1691572577774} +{"stream": "jobs", "data": {"id": 4177046003, "name": "Test job", "requisition_id": "3", "notes": null, "confidential": false, "is_template": true, "copied_from_id": null, "status": "open", "created_at": "2020-11-24T23:27:11.699Z", "opened_at": "2020-11-24T23:27:11.878Z", "closed_at": null, "updated_at": "2021-04-21T17:48:24.779Z", "departments": [{"id": 4028122003, "name": "Test dep1", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4320015003, "opening_id": "3-1", "status": "open", "opened_at": "2020-11-24T23:27:11.723Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": null}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": null}}}, "emitted_at": 1691572578315} +{"stream": "jobs", "data": {"id": 4177048003, "name": "Test Job 2", "requisition_id": "4", "notes": null, "confidential": false, "is_template": false, "copied_from_id": null, "status": "open", "created_at": "2020-11-24T23:27:45.634Z", "opened_at": "2020-11-24T23:27:45.878Z", "closed_at": null, "updated_at": "2021-10-07T18:46:58.982Z", "departments": [{"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": null}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": null}}}, "emitted_at": 1691572578318} +{"stream": "jobs", "data": {"id": 4446240003, "name": "Copy of Test Job 2", "requisition_id": "5", "notes": null, "confidential": false, "is_template": false, "copied_from_id": 4177048003, "status": "open", "created_at": "2021-10-08T08:19:42.383Z", "opened_at": "2021-10-08T08:19:42.818Z", "closed_at": null, "updated_at": "2021-10-08T08:19:42.821Z", "departments": [{"id": 4028123003, "name": "test dep 2", "parent_id": null, "parent_department_external_id": null, "child_ids": [], "child_department_external_ids": [], "external_id": null}], "offices": [{"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}], "hiring_team": {"hiring_managers": [], "recruiters": [], "coordinators": [], "sourcers": []}, "openings": [{"id": 4928188003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:24.949Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4928187003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-10T16:39:08.365Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4928186003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-10T16:38:57.407Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4926182003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}, {"id": 4926183003, "opening_id": "5-2", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}], "custom_fields": {"employment_type": "Full-time"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Full-time"}}}, "emitted_at": 1691572578321} +{"stream": "jobs_openings", "data": {"id": 4320015003, "opening_id": "3-1", "status": "open", "opened_at": "2020-11-24T23:27:11.723Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1691572579250} +{"stream": "jobs_openings", "data": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1691572579565} +{"stream": "jobs_openings", "data": {"id": 4926182003, "opening_id": "5-1", "status": "open", "opened_at": "2021-10-08T08:19:42.457Z", "closed_at": null, "application_id": null, "close_reason": null}, "emitted_at": 1691572579917} +{"stream": "interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572580878} +{"stream": "interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572580882} +{"stream": "interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572580884} +{"stream": "job_posts", "data": {"id": 4252332003, "active": true, "live": false, "first_published_at": null, "title": "Test job", "location": {"id": 4219721003, "name": "test", "office_id": null, "job_post_custom_location_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4177046003, "content": "

    Test description

    ", "internal_content": null, "updated_at": "2023-06-02T13:47:05.880Z", "created_at": "2020-11-24T23:29:24.315Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_5125927003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_5125928003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1691572581344} +{"stream": "job_posts", "data": {"id": 4751597003, "active": true, "live": false, "first_published_at": null, "title": "Test Job 2", "location": {"id": 4700649003, "name": "US", "office_id": null, "job_post_custom_location_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4177048003, "content": "

    Job post content

    ", "internal_content": null, "updated_at": "2023-06-02T13:47:05.880Z", "created_at": "2021-10-07T18:46:58.846Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_7911674003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_7911675003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1691572581348} +{"stream": "job_posts", "data": {"id": 4752433003, "active": true, "live": false, "first_published_at": null, "title": "Test Job 2", "location": {"id": 4701484003, "name": "US", "office_id": null, "job_post_custom_location_id": null, "job_post_location_type": {"id": 4000000003, "name": "Free Text"}}, "internal": false, "external": true, "job_id": 4446240003, "content": "

    Job post content

    ", "internal_content": null, "updated_at": "2023-06-02T13:47:05.880Z", "created_at": "2021-10-08T08:19:42.720Z", "demographic_question_set_id": null, "questions": [{"required": true, "private": false, "label": "First Name", "name": "first_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Last Name", "name": "last_name", "type": "short_text", "values": [], "description": null}, {"required": true, "private": false, "label": "Email", "name": "email", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Phone", "name": "phone", "type": "short_text", "values": [], "description": null}, {"required": false, "private": false, "label": "Resume", "name": "resume", "type": "attachment", "values": [], "description": null}, {"required": false, "private": false, "label": "Cover Letter", "name": "cover_letter", "type": "attachment", "values": [], "description": null}, {"required": null, "private": false, "label": "LinkedIn Profile", "name": "question_7918434003", "type": "short_text", "values": [], "description": null}, {"required": null, "private": false, "label": "Website", "name": "question_7918435003", "type": "short_text", "values": [], "description": null}]}, "emitted_at": 1691572581350} +{"stream": "job_stages", "data": {"id": 5245803003, "name": "Application Review", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 0, "interviews": [{"id": 5628614003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628609003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1691572581897} +{"stream": "job_stages", "data": {"id": 5245804003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 1, "interviews": [{"id": 5628615003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628610003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1691572581900} +{"stream": "job_stages", "data": {"id": 5245805003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 2, "interviews": [{"id": 5628616003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628611003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1691572581902} +{"stream": "jobs_stages", "data": {"id": 5245803003, "name": "Application Review", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 0, "interviews": [{"id": 5628614003, "name": "Application Review", "schedulable": false, "interview_kit": {"id": 5628609003, "content": null, "questions": []}, "estimated_minutes": 1, "default_interviewer_users": []}]}, "emitted_at": 1691572582909} +{"stream": "jobs_stages", "data": {"id": 5245804003, "name": "Preliminary Phone Screen", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 1, "interviews": [{"id": 5628615003, "name": "Preliminary Screening Call", "schedulable": true, "interview_kit": {"id": 5628610003, "content": null, "questions": []}, "estimated_minutes": 20, "default_interviewer_users": []}]}, "emitted_at": 1691572582912} +{"stream": "jobs_stages", "data": {"id": 5245805003, "name": "Phone Interview", "created_at": "2020-11-24T23:27:11.756Z", "updated_at": "2020-11-24T23:27:11.756Z", "active": true, "job_id": 4177046003, "priority": 2, "interviews": [{"id": 5628616003, "name": "Behavioral Phone Interview", "schedulable": true, "interview_kit": {"id": 5628611003, "content": null, "questions": []}, "estimated_minutes": 30, "default_interviewer_users": []}]}, "emitted_at": 1691572582916} +{"stream": "offers", "data": {"id": 4154100003, "version": 1, "application_id": 19215333003, "created_at": "2020-11-24T23:32:25.760Z", "updated_at": "2020-11-24T23:32:25.772Z", "sent_at": null, "resolved_at": null, "starts_at": "2020-12-04", "status": "unresolved", "job_id": 4177048003, "candidate_id": 17130848003, "opening": {"id": 4320018003, "opening_id": "4-1", "status": "open", "opened_at": "2020-11-24T23:27:45.665Z", "closed_at": null, "application_id": null, "close_reason": null}, "custom_fields": {"employment_type": "Contract"}, "keyed_custom_fields": {"employment_type": {"name": "Employment Type", "type": "single_select", "value": "Contract"}}}, "emitted_at": 1691572583986} +{"stream": "rejection_reasons", "data": {"id": 4014678003, "name": "reason1", "type": {"id": 4000000003, "name": "We rejected them"}}, "emitted_at": 1691572584415} +{"stream": "rejection_reasons", "data": {"id": 4014679003, "name": "reason2", "type": {"id": 4000001003, "name": "They rejected us"}}, "emitted_at": 1691572584418} +{"stream": "rejection_reasons", "data": {"id": 4014680003, "name": "reason3", "type": {"id": 4000002003, "name": "None specified"}}, "emitted_at": 1691572584419} +{"stream": "scorecards", "data": {"id": 5253031003, "updated_at": "2020-11-24T23:33:10.440Z", "created_at": "2020-11-24T23:33:10.440Z", "interview": "Application Review", "interview_step": {"id": 5628634003, "name": "Application Review"}, "candidate_id": 17130848003, "application_id": 19215333003, "interviewed_at": "2020-11-25T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "submitted_at": "2020-11-24T23:33:10.440Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "no_decision"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "no_decision"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "no_decision"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "no_decision"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no_decision"}], "ratings": {"definitely_not": [], "no": [], "mixed": [], "yes": [], "strong_yes": []}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": ""}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1691572584876} +{"stream": "scorecards", "data": {"id": 9664505003, "updated_at": "2021-09-29T17:23:11.468Z", "created_at": "2021-09-29T17:23:11.468Z", "interview": "Preliminary Screening Call", "interview_step": {"id": 5628615003, "name": "Preliminary Screening Call"}, "candidate_id": 40517966003, "application_id": 44937562003, "interviewed_at": "2021-09-29T01:00:00.000Z", "submitted_by": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "interviewer": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "submitted_at": "2021-09-29T17:23:11.468Z", "overall_recommendation": "no_decision", "attributes": [{"name": "Willing to do required travel", "type": "Details", "note": null, "rating": "yes"}, {"name": "Three to five years of experience", "type": "Qualifications", "note": null, "rating": "mixed"}, {"name": "Personable", "type": "Personality Traits", "note": null, "rating": "yes"}, {"name": "Passionate", "type": "Personality Traits", "note": null, "rating": "mixed"}, {"name": "Organizational Skills", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Manage competing priorities", "type": "Skills", "note": null, "rating": "yes"}, {"name": "Fits our salary range", "type": "Details", "note": null, "rating": "yes"}, {"name": "Empathetic", "type": "Personality Traits", "note": null, "rating": "strong_yes"}, {"name": "Currently based locally", "type": "Details", "note": null, "rating": "mixed"}, {"name": "Communication", "type": "Skills", "note": null, "rating": "no"}], "ratings": {"definitely_not": [], "no": ["Communication"], "mixed": ["Three to five years of experience", "Passionate", "Currently based locally"], "yes": ["Willing to do required travel", "Personable", "Organizational Skills", "Manage competing priorities", "Fits our salary range"], "strong_yes": ["Empathetic"]}, "questions": [{"id": null, "question": "Key Take-Aways", "answer": "test"}, {"id": null, "question": "Private Notes", "answer": ""}]}, "emitted_at": 1691572584880} +{"stream": "sources", "data": {"id": 4000000003, "name": "Recurse", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1691572585322} +{"stream": "sources", "data": {"id": 4000001003, "name": "cliquify", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1691572585324} +{"stream": "sources", "data": {"id": 4000002003, "name": "ContactOut", "type": {"id": 4000003003, "name": "Prospecting"}}, "emitted_at": 1691572585326} +{"stream": "users", "data": {"id": 4218085003, "name": "Greenhouse Admin", "first_name": "Greenhouse", "last_name": "Admin", "primary_email_address": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "updated_at": "2020-11-18T14:09:08.401Z", "created_at": "2020-11-18T14:09:08.401Z", "disabled": false, "site_admin": true, "emails": ["scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1691572586823} +{"stream": "users", "data": {"id": 4218086003, "name": "Airbyte Team", "first_name": "Airbyte", "last_name": "Team", "primary_email_address": "integration-test@airbyte.io", "updated_at": "2023-05-24T21:42:47.510Z", "created_at": "2020-11-18T14:09:08.481Z", "disabled": false, "site_admin": true, "emails": ["integration-test@airbyte.io"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1691572586827} +{"stream": "users", "data": {"id": 4218087003, "name": "emily.brooks+airbyte_integration@greenhouse.io", "first_name": null, "last_name": null, "primary_email_address": "emily.brooks+airbyte_integration@greenhouse.io", "updated_at": "2020-11-18T14:09:08.991Z", "created_at": "2020-11-18T14:09:08.809Z", "disabled": false, "site_admin": true, "emails": ["emily.brooks+airbyte_integration@greenhouse.io"], "employee_id": null, "linked_candidate_ids": [], "offices": [], "departments": []}, "emitted_at": 1691572586830} +{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68322855003, "created_at": "2020-11-24T23:24:37.047Z", "subject": null, "body": "John Lafleur added Test Test through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1691572587744} +{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68323000003, "created_at": "2020-11-24T23:25:13.802Z", "subject": null, "body": "John Lafleur added Test2 Test2 through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1691572587927} +{"stream": "activity_feed", "data": {"emails": [], "notes": [], "activities": [{"id": 68323687003, "created_at": "2020-11-24T23:28:19.777Z", "subject": null, "body": "John Lafleur added Name Lastname through Greenhouse.", "user": null}], "linkedin_notes": [], "linkedin_inmails": []}, "emitted_at": 1691572588092} +{"stream": "approvals", "data": {"id": 5217303003, "offer_id": null, "sequential": true, "version": 1, "approval_type": "open_job", "approval_status": "approved", "job_id": 4177048003, "requested_by_user_id": null, "approver_groups": []}, "emitted_at": 1691572590079} +{"stream": "approvals", "data": {"id": 5217304003, "offer_id": null, "sequential": true, "version": 1, "approval_type": "offer_job", "approval_status": "approved", "job_id": 4177048003, "requested_by_user_id": null, "approver_groups": []}, "emitted_at": 1691572590082} +{"stream": "approvals", "data": {"id": 5217305003, "offer_id": null, "sequential": true, "version": 1, "approval_type": "offer_candidate", "approval_status": "approved", "job_id": 4177048003, "requested_by_user_id": null, "approver_groups": []}, "emitted_at": 1691572590084} +{"stream": "disciplines", "data": {"id": 10848297003, "name": "Accounting", "priority": 0, "external_id": null}, "emitted_at": 1691572590852} +{"stream": "disciplines", "data": {"id": 10848298003, "name": "African Studies", "priority": 1, "external_id": null}, "emitted_at": 1691572590854} +{"stream": "disciplines", "data": {"id": 10848299003, "name": "Agriculture", "priority": 2, "external_id": null}, "emitted_at": 1691572590856} +{"stream": "schools", "data": {"id": 10845822003, "name": "Abraham Baldwin Agricultural College", "priority": 0, "external_id": null}, "emitted_at": 1691572591399} +{"stream": "schools", "data": {"id": 10845823003, "name": "Academy of Art University", "priority": 1, "external_id": null}, "emitted_at": 1691572591403} +{"stream": "schools", "data": {"id": 10845824003, "name": "Acadia University", "priority": 2, "external_id": null}, "emitted_at": 1691572591405} +{"stream": "email_templates", "data": {"id": 4071597003, "name": "Default Eeoc Data Request", "default": true, "updated_at": "2021-12-20T18:37:50.451Z", "created_at": "2020-11-18T14:09:07.631Z", "description": "This template is used for requesting EEOC data from the candidate.", "type": "eeoc_data_request", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Thank you for your interest in {{JOB_NAME}} at {{COMPANY}}.

    \n\n

    For government reporting purposes, we ask candidates to respond to the below self-identification survey.\nCompletion of the form is entirely voluntary. Whatever your decision, it will not be considered in the hiring\nprocess or thereafter. Any information that you do provide will be recorded and maintained in a\nconfidential file.

    \n\n

    As set forth in {{COMPANY}}\u2019s Equal Employment Opportunity policy,\nwe do not discriminate on the basis of any protected group status under any applicable law.

    \n\n

    Thank You

    ", "user": null}, "emitted_at": 1691572593892} +{"stream": "email_templates", "data": {"id": 4112339003, "name": "Default Candidate Self-Schedule request", "default": true, "updated_at": "2021-07-15T20:09:31.345Z", "created_at": "2021-07-15T20:09:31.345Z", "description": "This template is used when sending a request for a candidate to self-schedule.", "type": "candidate_self_schedule_request", "from": "{{MY_EMAIL_ADDRESS}}", "cc": [], "body": null, "html_body": "

    Hi {{CANDIDATE_FIRST_NAME}},

    \n\n

    Please use this link to schedule your interview: {{INTERVIEW_REQUEST_LINK}}

    \n\n

    \n Regards,\n
    \n {{MY_FIRST_NAME}}\n

    ", "user": null}, "emitted_at": 1691572593895} +{"stream": "email_templates", "data": {"id": 4071603003, "name": "Default Agency Job Assignment Notification", "default": true, "updated_at": "2020-11-18T14:09:07.667Z", "created_at": "2020-11-18T14:09:07.667Z", "description": "This template is used when an agency recruiter is added to a job", "type": "agency_recruiter_assigned", "from": "no-reply@us.greenhouse-mail.io", "cc": [], "body": null, "html_body": "

    Hi,

    \n\n

    {{MY_FULL_NAME}} from {{COMPANY}} has assigned you to the {{JOB_NAME}} job. You can now view this job and submit candidates to it via the agency portal.

    \n\n

    \n Log in here to get started: app.greenhouse.io\n

    \n\n

    \n The Greenhouse Team
    \n app.greenhouse.io\n

    \n\n

    \n \u00a9 2020 Greenhouse / 18 West 18th Street, 11th Floor, New York, NY 10011, USA\n

    ", "user": null}, "emitted_at": 1691572593898} +{"stream": "offices", "data": {"id": 4019854003, "name": "Test office", "location": {"name": null}, "primary_contact_user_id": 4218086003, "parent_id": null, "parent_office_external_id": null, "child_ids": [], "child_office_external_ids": [], "external_id": null}, "emitted_at": 1691572594362} +{"stream": "user_roles", "data": {"id": 4011623003, "type": "job_admin", "name": "Private"}, "emitted_at": 1691572595642} +{"stream": "user_roles", "data": {"id": 4011622003, "type": "job_admin", "name": "Standard"}, "emitted_at": 1691572595644} diff --git a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml index a470dca75e82..bf5927079078 100644 --- a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 - dockerImageTag: 0.4.0 + dockerImageTag: 0.4.2 dockerRepository: airbyte/source-greenhouse githubIssueLabel: source-greenhouse icon: greenhouse.svg @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-greenhouse/requirements.txt b/airbyte-integrations/connectors/source-greenhouse/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-greenhouse/requirements.txt +++ b/airbyte-integrations/connectors/source-greenhouse/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index 320b5c10eea0..73945258c42b 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -6,6 +6,7 @@ from setuptools import find_packages, setup TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", ] @@ -16,7 +17,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk", "dataclasses-jsonschema==2.15.1"], + install_requires=["airbyte-cdk>=0.44.1", "dataclasses-jsonschema==2.15.1"], package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/__init__.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/__init__.py index f8a5744233ad..015341650840 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/__init__.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from .source import SourceGreenhouse __all__ = ["SourceGreenhouse"] diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py index 7de00d74193c..7eeba4b7ed8d 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py @@ -7,13 +7,13 @@ from typing import Any, ClassVar, Iterable, Mapping, MutableMapping, Optional, Union from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer +from airbyte_cdk.sources.declarative.incremental import Cursor from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from airbyte_cdk.sources.streams.core import Stream @dataclass -class GreenHouseSlicer(StreamSlicer): +class GreenHouseSlicer(Cursor): parameters: InitVar[Mapping[str, Any]] cursor_field: str request_cursor_field: str @@ -24,8 +24,8 @@ class GreenHouseSlicer(StreamSlicer): def __post_init__(self, parameters: Mapping[str, Any]): self._state = {} - def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, **kwargs) -> Iterable[StreamSlice]: - yield {self.request_cursor_field: stream_state.get(self.cursor_field, self.START_DATETIME)} + def stream_slices(self) -> Iterable[StreamSlice]: + yield {self.request_cursor_field: self._state.get(self.cursor_field, self.START_DATETIME)} def _max_dt_str(self, *args: str) -> Optional[str]: new_state_candidates = list(map(lambda x: datetime.datetime.strptime(x, self.DATETIME_FORMAT), filter(None, args))) @@ -36,16 +36,43 @@ def _max_dt_str(self, *args: str) -> Optional[str]: (dt, micro) = max_dt.strftime(self.DATETIME_FORMAT).split(".") return "%s.%03dZ" % (dt, int(micro[:-1:]) / 1000) - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - # stream_state can be passed in as a stream_slice parameter - it's a framework flaw, so we have to workaround it - slice_state = stream_slice.get(self.cursor_field) + def set_initial_state(self, stream_state: StreamState) -> None: + cursor_value = stream_state.get(self.cursor_field) + if cursor_value: + self._state[self.cursor_field] = cursor_value + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + stream_slice_value = stream_slice.get(self.cursor_field) current_state = self._state.get(self.cursor_field) - last_cursor = last_record and last_record[self.cursor_field] - max_dt = self._max_dt_str(slice_state, current_state, last_cursor) + record_cursor_value = most_recent_record and most_recent_record[self.cursor_field] + max_dt = self._max_dt_str(stream_slice_value, current_state, record_cursor_value) if not max_dt: return self._state[self.cursor_field] = max_dt + def should_be_synced(self, record: Record) -> bool: + """ + As of 2023-06-28, the expectation is that this method will only be used for semi-incremental and data feed and therefore the + implementation is irrelevant for greenhouse + """ + return True + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice + """ + first_cursor_value = first.get(self.cursor_field) + second_cursor_value = second.get(self.cursor_field) + if first_cursor_value and second_cursor_value: + return first_cursor_value >= second_cursor_value + elif first_cursor_value: + return True + else: + return False + + def _parse_to_datetime(self, datetime_str: str) -> datetime.datetime: + return datetime.datetime.strptime(datetime_str, self.DATETIME_FORMAT) + def get_stream_state(self) -> StreamState: return self._state @@ -74,37 +101,46 @@ class GreenHouseSubstreamSlicer(GreenHouseSlicer): stream_slice_field: str parent_key: str - def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]: - for parent_stream_slice in self.parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=None, stream_state=stream_state): + def stream_slices(self) -> Iterable[StreamSlice]: + for parent_stream_slice in self.parent_stream.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=None, stream_state=self.get_stream_state() + ): for parent_record in self.parent_stream.read_records( sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None ): parent_state_value = parent_record.get(self.parent_key) yield { self.stream_slice_field: parent_state_value, - self.request_cursor_field: stream_state.get(str(parent_state_value), {}).get(self.cursor_field, self.START_DATETIME), + self.request_cursor_field: self._state.get(str(parent_state_value), {}).get(self.cursor_field, self.START_DATETIME), } - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - if last_record: - # stream_slice is really a stream slice - substream_id = str(stream_slice[self.stream_slice_field]) - current_state = self._state.get(substream_id, {}).get(self.cursor_field) - last_state = last_record[self.cursor_field] - max_dt = self._max_dt_str(last_state, current_state) - self._state[substream_id] = {self.cursor_field: max_dt} - return - # stream_slice here may be a stream slice or a state - if self.stream_slice_field in stream_slice: + def set_initial_state(self, stream_state: StreamState) -> None: + if self.stream_slice_field in stream_state: return - substream_ids = map(lambda x: str(x), set(stream_slice.keys()) | set(self._state.keys())) + substream_ids = map(lambda x: str(x), set(stream_state.keys()) | set(self._state.keys())) for id_ in substream_ids: self._state[id_] = { self.cursor_field: self._max_dt_str( - stream_slice.get(id_, {}).get(self.cursor_field), self._state.get(id_, {}).get(self.cursor_field) + stream_state.get(id_, {}).get(self.cursor_field), self._state.get(id_, {}).get(self.cursor_field) ) } + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + if most_recent_record: + substream_id = str(stream_slice[self.stream_slice_field]) + current_state = self._state.get(substream_id, {}).get(self.cursor_field) + last_state = most_recent_record[self.cursor_field] + max_dt = self._max_dt_str(last_state, current_state) + self._state[substream_id] = {self.cursor_field: max_dt} + return + + def should_be_synced(self, record: Record) -> bool: + """ + As of 2023-06-28, the expectation is that this method will only be used for semi-incremental and data feed and therefore the + implementation is irrelevant for greenhouse + """ + return True + def get_request_params( self, *, diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py index 79a6b9615a0a..0e9d1b5bb96a 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py @@ -3,10 +3,10 @@ # -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock import pytest -from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer @@ -14,7 +14,7 @@ def test_slicer(): date_time = "2022-09-05T10:10:10.000000Z" date_time_dict = {date_time: date_time} slicer = GreenHouseSlicer(cursor_field=date_time, parameters={}, request_cursor_field=None) - slicer.update_cursor(stream_slice=date_time_dict, last_record=date_time_dict) + slicer.close_slice(date_time_dict, date_time_dict) assert slicer.get_stream_state() == {date_time: "2022-09-05T10:10:10.000Z"} assert slicer.get_request_headers() == {} assert slicer.get_request_body_data() == {} @@ -34,16 +34,17 @@ def test_slicer(): ) def test_sub_slicer(last_record, expected, records): date_time = "2022-09-05T10:10:10.000000Z" - parent_slicer = GreenHouseSlicer(cursor_field=date_time, parameters={}, request_cursor_field=None) - GreenHouseSlicer.read_records = MagicMock(return_value=records) + parent_stream = Mock(spec=Stream) + parent_stream.stream_slices.return_value = [{"a slice": "value"}] + parent_stream.read_records = MagicMock(return_value=records) slicer = GreenHouseSubstreamSlicer( cursor_field=date_time, parameters={}, request_cursor_field=None, - parent_stream=parent_slicer, + parent_stream=parent_stream, stream_slice_field=date_time, parent_key="parent_key", ) - stream_slice = next(slicer.stream_slices(SyncMode, {})) if records else {} - slicer.update_cursor(stream_slice=stream_slice, last_record=last_record) + stream_slice = next(slicer.stream_slices()) if records else {} + slicer.close_slice(stream_slice, last_record) assert slicer.get_stream_state() == expected diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py index 7c86d38ecc25..a7a9adaf7202 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_streams.py @@ -28,28 +28,28 @@ def create_response(headers): def test_next_page_token_has_next(applications_stream): headers = {"link": '; rel="next"'} response = create_response(headers) - next_page_token = applications_stream.retriever.next_page_token(response=response) + next_page_token = applications_stream.retriever._next_page_token(response=response) assert next_page_token == {"next_page_token": "https://harvest.greenhouse.io/v1/applications?per_page=100&since_id=123456789"} def test_next_page_token_has_not_next(applications_stream): response = create_response({}) - next_page_token = applications_stream.retriever.next_page_token(response=response) + next_page_token = applications_stream.retriever._next_page_token(response=response) assert next_page_token is None def test_request_params_next_page_token_is_not_none(applications_stream): response = create_response({"link": f'; rel="next"'}) - next_page_token = applications_stream.retriever.next_page_token(response=response) - request_params = applications_stream.retriever.request_params(next_page_token=next_page_token, stream_state={}) - path = applications_stream.retriever.path(next_page_token=next_page_token, stream_state={}) + next_page_token = applications_stream.retriever._next_page_token(response=response) + request_params = applications_stream.retriever._request_params(next_page_token=next_page_token, stream_state={}) + path = applications_stream.retriever._paginator_path() assert "applications?per_page=100&since_id=123456789" == path assert request_params == {"per_page": 100} def test_request_params_next_page_token_is_none(applications_stream): - request_params = applications_stream.retriever.request_params(stream_state={}) + request_params = applications_stream.retriever._request_params(stream_state={}) assert request_params == {"per_page": 100} @@ -138,8 +138,8 @@ def test_parse_response_expected_response(applications_stream): ] """ response._content = response_content - parsed_response = applications_stream.retriever.parse_response(response, stream_state={}) - records = [record for record in parsed_response] + parsed_response = applications_stream.retriever._parse_response(response, stream_state={}) + records = [dict(record) for record in parsed_response] assert records == json.loads(response_content) @@ -148,7 +148,7 @@ def test_parse_response_empty_content(applications_stream): response = requests.Response() response.status_code = 200 response._content = b"[]" - parsed_response = applications_stream.retriever.parse_response(response, stream_state={}) + parsed_response = applications_stream.retriever._parse_response(response, stream_state={}) records = [record for record in parsed_response] assert records == [] @@ -164,7 +164,7 @@ def test_ignore_403(applications_stream): response = requests.Response() response.status_code = 403 response._content = b"" - parsed_response = applications_stream.retriever.parse_response(response, stream_state={}) + parsed_response = applications_stream.retriever._parse_response(response, stream_state={}) records = [record for record in parsed_response] assert records == [] @@ -173,5 +173,5 @@ def test_retry_429(applications_stream): response = requests.Response() response.status_code = 429 response._content = b"{}" - should_retry = applications_stream.retriever.should_retry(response) + should_retry = applications_stream.retriever.requester._should_retry(response) assert should_retry is True diff --git a/airbyte-integrations/connectors/source-gridly/metadata.yaml b/airbyte-integrations/connectors/source-gridly/metadata.yaml index d5a03c6ededb..1c6747986c8c 100644 --- a/airbyte-integrations/connectors/source-gridly/metadata.yaml +++ b/airbyte-integrations/connectors/source-gridly/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/gridly tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gridly/requirements.txt b/airbyte-integrations/connectors/source-gridly/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-gridly/requirements.txt +++ b/airbyte-integrations/connectors/source-gridly/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gridly/setup.py b/airbyte-integrations/connectors/source-gridly/setup.py index 7815120a9680..dc08caa562c5 100644 --- a/airbyte-integrations/connectors/source-gridly/setup.py +++ b/airbyte-integrations/connectors/source-gridly/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-gutendex/metadata.yaml b/airbyte-integrations/connectors/source-gutendex/metadata.yaml index 13d9503516d8..01e6a71bbaed 100644 --- a/airbyte-integrations/connectors/source-gutendex/metadata.yaml +++ b/airbyte-integrations/connectors/source-gutendex/metadata.yaml @@ -17,4 +17,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gutendex/requirements.txt b/airbyte-integrations/connectors/source-gutendex/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-gutendex/requirements.txt +++ b/airbyte-integrations/connectors/source-gutendex/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-gutendex/setup.py b/airbyte-integrations/connectors/source-gutendex/setup.py index eff233be2f2b..b96103135cdf 100644 --- a/airbyte-integrations/connectors/source-gutendex/setup.py +++ b/airbyte-integrations/connectors/source-gutendex/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl index c6109ec549df..249c503bd2c4 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl @@ -1,110 +1,81 @@ -{"stream": "clients", "data": {"id": 10749825, "name": "First client", "is_active": true, "address": null, "statement_key": "48d746ca9125fe984b3bd800747669cc", "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-25T16:16:52Z", "currency": "USD"}, "emitted_at": 1682938830992} -{"stream": "clients", "data": {"id": 10748673, "name": "Users", "is_active": true, "address": null, "statement_key": "6b70f27acd3bf496daba22316cb750d3", "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "currency": "USD"}, "emitted_at": 1682938830992} -{"stream": "clients", "data": {"id": 10748671, "name": "[SAMPLE] Client B", "is_active": true, "address": null, "statement_key": "61faacd69e255af516cd8d772073afd4", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "currency": "USD"}, "emitted_at": 1682938830993} -{"stream": "clients", "data": {"id": 10748670, "name": "[SAMPLE] Client A", "is_active": true, "address": null, "statement_key": "1f2a8709628bb49a3b673dfcf1d09319", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-25T16:17:55Z", "currency": "USD"}, "emitted_at": 1682938830993} -{"stream": "contacts", "data": {"id": 8468604, "title": null, "first_name": "[SAMPLE] Morgan", "last_name": "Minute", "email": "morgan@harvestsample.com", "phone_office": "", "phone_mobile": "", "fax": "", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "client": {"id": 10748671, "name": "[SAMPLE] Client B"}}, "emitted_at": 1682938831461} -{"stream": "contacts", "data": {"id": 8468603, "title": null, "first_name": "[SAMPLE] Sofia", "last_name": "Stopwatch", "email": "sofia@harvestsample.com", "phone_office": "", "phone_mobile": "", "fax": "", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}}, "emitted_at": 1682938831462} -{"stream": "company", "data": {"base_uri": "https://airbyte.harvestapp.com", "full_domain": "airbyte.harvestapp.com", "name": "Airbyte", "is_active": true, "week_start_day": "Monday", "wants_timestamp_timers": false, "time_format": "hours_minutes", "date_format": "%m/%d/%Y", "plan_type": "simple-v4", "expense_feature": true, "invoice_feature": true, "estimate_feature": true, "team_feature": true, "weekly_capacity": 144000, "approval_feature": true, "clock": "12h", "currency": "USD", "currency_code_display": "iso_code_none", "currency_symbol_display": "symbol_before", "decimal_symbol": ".", "thousands_separator": ",", "color_scheme": "orange"}, "emitted_at": 1682938831931} -{"stream": "invoices", "data": {"id": 28174545, "client_key": "489645d5b2becebe06f7a696a4d0db6a8a1c8ff1", "number": "2", "purchase_order": "", "amount": 22000.0, "due_amount": 21500.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "Subj", "notes": "", "state": "draft", "period_start": null, "period_end": null, "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": null, "paid_at": null, "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:17:55Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": null, "currency": "USD", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632435, "kind": "Service", "description": "[SAMPLE] Fixed Fee Project", "quantity": 1.0, "unit_price": 21900.0, "amount": 21900.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}, {"id": 132632436, "kind": "Product", "description": "", "quantity": 1.0, "unit_price": 100.0, "amount": 100.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}]}, "emitted_at": 1682938832494} -{"stream": "invoices", "data": {"id": 28174531, "client_key": "1a3a59c71a8dd22b3a341807456c754220dc202c", "number": "1", "purchase_order": "", "amount": 76.9, "due_amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": 4.0, "discount_amount": 3.2, "subject": "", "notes": "Note", "state": "paid", "period_start": "2021-05-05", "period_end": "2021-05-05", "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": "2021-05-25T16:46:28Z", "paid_at": "2021-05-25T00:00:00Z", "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:16:51Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "currency": "USD", "client": {"id": 10749825, "name": "First client"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632398, "kind": "Service", "description": "[FP] First project: Design (05/05/2021 - 05/05/2021)", "quantity": 0.01, "unit_price": 10.0, "amount": 0.1, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}, {"id": 132632399, "kind": "Service", "description": "[FP] First project: Programming (05/05/2021 - 05/05/2021)", "quantity": 8.0, "unit_price": 10.0, "amount": 80.0, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}]}, "emitted_at": 1682938832495} -{"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1682938834647} -{"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1682938834647} -{"stream": "invoice_payments", "data": {"id": 21857618, "amount": 500.0, "paid_at": "2021-05-26T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "", "transaction_id": null, "created_at": "2021-05-26T09:07:06Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": "2021-05-26", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174545}, "emitted_at": 1682938835711} -{"stream": "invoice_item_categories", "data": {"id": 2732435, "name": "Product", "use_as_service": false, "use_as_expense": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938836454} -{"stream": "invoice_item_categories", "data": {"id": 2732434, "name": "Service", "use_as_service": true, "use_as_expense": false, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938836455} -{"stream": "estimates", "data": {"id": 2695071, "client_key": "de25b9eb3e82c0d5032777559e8ac0cfdfbf82b1", "number": "1", "purchase_order": "", "amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "", "notes": "", "state": "sent", "issue_date": "2021-05-27", "sent_at": "2021-05-27T18:12:42Z", "created_at": "2021-05-27T18:12:30Z", "updated_at": "2021-05-27T18:12:42Z", "accepted_at": null, "declined_at": null, "currency": "USD", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": []}, "emitted_at": 1682938836990} -{"stream": "estimate_messages", "data": {"id": 4857940, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "send_me_a_copy": true, "created_at": "2021-05-27T18:12:42Z", "updated_at": "2021-05-27T18:12:42Z", "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "event_type": null, "subject": "Estimate #1 from Airbyte", "body": "---------------------------------------------\r\nEstimate Summary\r\n---------------------------------------------\r\nEstimate ID: 1\r\nEstimate Date: 05/27/2021\r\nClient: [SAMPLE] Client A\r\nP.O. Number: \r\nAmount: $0.00\r\n\r\nYou can view the estimate here:\r\n\r\n%estimate_url%\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 2695071}, "emitted_at": 1682938838664} -{"stream": "estimate_item_categories", "data": {"id": 2614512, "name": "Product", "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938839173} -{"stream": "estimate_item_categories", "data": {"id": 2614511, "name": "Service", "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938839174} -{"stream": "expenses", "data": {"id": 31751921, "spent_date": "2021-04-27", "notes": "This is a sample expense entry.", "total_cost": 51.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758382, "name": "[SAMPLE] Hiromi Hourglass"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}, "expense_category": {"id": 7892982, "name": "Meals", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326464, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1682938839774} -{"stream": "expenses", "data": {"id": 31751926, "spent_date": "2021-04-25", "notes": "This is a sample expense entry.", "total_cost": 142.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758381, "name": "[SAMPLE] Kiran Kronological"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "expense_category": {"id": 7892984, "name": "Transportation", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326479, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1682938839774} -{"stream": "expenses", "data": {"id": 31751920, "spent_date": "2021-04-20", "notes": "This is a sample expense entry.", "total_cost": 30.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758381, "name": "[SAMPLE] Kiran Kronological"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}, "expense_category": {"id": 7892982, "name": "Meals", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326463, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1682938839775} -{"stream": "expenses", "data": {"id": 31751924, "spent_date": "2021-04-18", "notes": "This is a sample expense entry.", "total_cost": 58.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758382, "name": "[SAMPLE] Hiromi Hourglass"}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "project": {"id": 28671448, "name": "Monthly Retainer", "code": "SAMPLE"}, "expense_category": {"id": 7892981, "name": "Entertainment", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326475, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1682938839775} -{"stream": "expenses", "data": {"id": 31751927, "spent_date": "2021-04-16", "notes": "This is a sample expense entry.", "total_cost": 84.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}, "expense_category": {"id": 7892984, "name": "Transportation", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326466, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 175.0}, "invoice": null}, "emitted_at": 1682938839776} -{"stream": "expenses", "data": {"id": 31751922, "spent_date": "2021-04-14", "notes": "This is a sample expense entry.", "total_cost": 23.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758383, "name": "[SAMPLE] Tamara Timekeeper"}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "project": {"id": 28671447, "name": "Time & Materials Project", "code": "SAMPLE"}, "expense_category": {"id": 7892984, "name": "Transportation", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326470, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": 35.0, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": 175.0}, "invoice": null}, "emitted_at": 1682938839777} -{"stream": "expenses", "data": {"id": 31751923, "spent_date": "2021-04-10", "notes": "This is a sample expense entry.", "total_cost": 200.0, "units": 1.0, "billable": false, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758381, "name": "[SAMPLE] Kiran Kronological"}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "project": {"id": 28671447, "name": "Time & Materials Project", "code": "SAMPLE"}, "expense_category": {"id": 7892983, "name": "Lodging", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326468, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": 33.0, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1682938839777} -{"stream": "expenses", "data": {"id": 31751928, "spent_date": "2021-04-07", "notes": "This is a sample expense entry.", "total_cost": 174.0, "units": 1.0, "billable": false, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}, "expense_category": {"id": 7892984, "name": "Transportation", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326466, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 175.0}, "invoice": null}, "emitted_at": 1682938839778} -{"stream": "expenses", "data": {"id": 31751925, "spent_date": "2021-04-07", "notes": "This is a sample expense entry.", "total_cost": 180.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "project": {"id": 28671448, "name": "Monthly Retainer", "code": "SAMPLE"}, "expense_category": {"id": 7892982, "name": "Meals", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326477, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0}, "invoice": null}, "emitted_at": 1682938839778} -{"stream": "expense_categories", "data": {"id": 7892986, "name": "Mileage", "unit_name": "mile", "unit_price": 0.575, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840329} -{"stream": "expense_categories", "data": {"id": 7892985, "name": "Other", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840330} -{"stream": "expense_categories", "data": {"id": 7892984, "name": "Transportation", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840330} -{"stream": "expense_categories", "data": {"id": 7892983, "name": "Lodging", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840330} -{"stream": "expense_categories", "data": {"id": 7892982, "name": "Meals", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840330} -{"stream": "expense_categories", "data": {"id": 7892981, "name": "Entertainment", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840330} -{"stream": "tasks", "data": {"id": 16575211, "name": "Vacation", "billable_by_default": false, "default_hourly_rate": null, "is_default": false, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840829} -{"stream": "tasks", "data": {"id": 16575210, "name": "Business Development", "billable_by_default": false, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840830} -{"stream": "tasks", "data": {"id": 16575209, "name": "Project Management", "billable_by_default": true, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840830} -{"stream": "tasks", "data": {"id": 16575208, "name": "Marketing", "billable_by_default": true, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840830} -{"stream": "tasks", "data": {"id": 16575207, "name": "Programming", "billable_by_default": true, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840831} -{"stream": "tasks", "data": {"id": 16575206, "name": "Design", "billable_by_default": true, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1682938840831} -{"stream": "time_entries", "data": {"id": 1494415019, "spent_date": "2021-05-05", "hours": 0.01, "hours_without_timer": 0.01, "rounded_hours": 0.01, "notes": "", "is_locked": true, "locked_reason": "Item Invoiced and Locked for this Time Period", "is_closed": false, "is_billed": true, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": true, "budgeted": false, "billable_rate": 10.0, "cost_rate": null, "created_at": "2021-05-05T12:53:38Z", "updated_at": "2021-05-25T16:16:52Z", "user": {"id": 3758380, "name": "Airbyte Developer"}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575206, "name": "Design"}, "user_assignment": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0}, "task_assignment": {"id": 307640132, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null}, "invoice": {"id": 28174531, "number": "1"}, "external_reference": null}, "emitted_at": 1682938841523} -{"stream": "time_entries", "data": {"id": 1494414737, "spent_date": "2021-05-05", "hours": 8.0, "hours_without_timer": 8.0, "rounded_hours": 8.0, "notes": "", "is_locked": true, "locked_reason": "Item Invoiced and Locked for this Time Period", "is_closed": false, "is_billed": true, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": true, "budgeted": false, "billable_rate": 10.0, "cost_rate": null, "created_at": "2021-05-05T12:53:23Z", "updated_at": "2021-05-25T16:16:52Z", "user": {"id": 3758380, "name": "Airbyte Developer"}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575207, "name": "Programming"}, "user_assignment": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0}, "task_assignment": {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null}, "invoice": {"id": 28174531, "number": "1"}, "external_reference": null}, "emitted_at": 1682938841524} -{"stream": "time_entries", "data": {"id": 1494238685, "spent_date": "2021-05-05", "hours": 0.71, "hours_without_timer": 0.71, "rounded_hours": 0.71, "notes": "This is a sample time entry.", "is_locked": false, "locked_reason": null, "is_closed": false, "is_billed": false, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": false, "budgeted": true, "billable_rate": null, "cost_rate": 60.0, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "task": {"id": 16575208, "name": "Marketing"}, "user_assignment": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0}, "task_assignment": {"id": 307607000, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null}, "invoice": null, "external_reference": null}, "emitted_at": 1682938841524} -{"stream": "time_entries", "data": {"id": 1494238684, "spent_date": "2021-05-05", "hours": 2.62, "hours_without_timer": 2.62, "rounded_hours": 2.62, "notes": "This is a sample time entry.", "is_locked": false, "locked_reason": null, "is_closed": false, "is_billed": false, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": false, "budgeted": true, "billable_rate": null, "cost_rate": 60.0, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "project": {"id": 28671447, "name": "Time & Materials Project", "code": "SAMPLE"}, "task": {"id": 16575210, "name": "Business Development"}, "user_assignment": {"id": 286326471, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": 33.0, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": 175.0}, "task_assignment": {"id": 307606988, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null}, "invoice": null, "external_reference": null}, "emitted_at": 1682938841524} -{"stream": "time_entries", "data": {"id": 1494238683, "spent_date": "2021-05-05", "hours": 1.42, "hours_without_timer": 1.42, "rounded_hours": 1.42, "notes": "This is a sample time entry.", "is_locked": false, "locked_reason": null, "is_closed": false, "is_billed": false, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": false, "budgeted": true, "billable_rate": null, "cost_rate": 60.0, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "task": {"id": 16575209, "name": "Project Management"}, "user_assignment": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0}, "task_assignment": {"id": 307607002, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null}, "invoice": null, "external_reference": null}, "emitted_at": 1682938841525} -{"stream": "user_assignments", "data": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1682938844316} -{"stream": "user_assignments", "data": {"id": 286326492, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": null, "project": {"id": 28671451, "name": "Airbyte", "code": null}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1682938844317} -{"stream": "user_assignments", "data": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}}, "emitted_at": 1682938844317} -{"stream": "task_assignments", "data": {"id": 307640135, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575209, "name": "Project Management"}}, "emitted_at": 1682938845521} -{"stream": "task_assignments", "data": {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575207, "name": "Programming"}}, "emitted_at": 1682938845522} -{"stream": "task_assignments", "data": {"id": 307640133, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575208, "name": "Marketing"}}, "emitted_at": 1682938845522} -{"stream": "task_assignments", "data": {"id": 307640132, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575206, "name": "Design"}}, "emitted_at": 1682938845522} -{"stream": "projects", "data": {"id": 28674500, "name": "First project", "code": "FP", "is_active": true, "is_billable": true, "is_fixed_fee": false, "bill_by": "People", "budget": null, "budget_by": "none", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": "Some notes", "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10749825, "name": "First client", "currency": "USD"}}, "emitted_at": 1682938846235} -{"stream": "projects", "data": {"id": 28671451, "name": "Airbyte", "code": null, "is_active": true, "is_billable": true, "is_fixed_fee": false, "bill_by": "none", "budget": null, "budget_by": "none", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": null, "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10748673, "name": "Users", "currency": "USD"}}, "emitted_at": 1682938846236} -{"stream": "projects", "data": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE", "is_active": true, "is_billable": false, "is_fixed_fee": false, "bill_by": "none", "budget": 160.0, "budget_by": "project", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": "Non-billable projects are perfect for tracking time you don\u2019t want to invoice for. You can use them to track internal projects, vacation/sick time, or pro bono work.", "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}}, "emitted_at": 1682938846236} -{"stream": "roles", "data": {"id": 763939, "name": "Sample Role", "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "user_ids": [3758381, 3758382, 3758383, 3758384]}, "emitted_at": 1682938847667} -{"stream": "users", "data": {"id": 3758384, "first_name": "[SAMPLE] Warrin", "last_name": "Wristwatch", "email": "warrin@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": false, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["member"], "permissions_claims": ["expenses:read:own", "expenses:write:own", "timers:read:own", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_4.png"}, "emitted_at": 1682938848246} -{"stream": "users", "data": {"id": 3758383, "first_name": "[SAMPLE] Tamara", "last_name": "Timekeeper", "email": "tamara@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:all", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_3.png"}, "emitted_at": 1682938848247} -{"stream": "users", "data": {"id": 3758382, "first_name": "[SAMPLE] Hiromi", "last_name": "Hourglass", "email": "hiromi@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_2.png"}, "emitted_at": 1682938848247} -{"stream": "users", "data": {"id": 3758381, "first_name": "[SAMPLE] Kiran", "last_name": "Kronological", "email": "kiran@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_1.png"}, "emitted_at": 1682938848247} -{"stream": "users", "data": {"id": 3758380, "first_name": "Airbyte", "last_name": "Developer", "email": "integration-test@airbyte.io", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": null, "cost_rate": null, "roles": [], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:all", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://d3s3969qhosaug.cloudfront.net/v2/default-avatars/4144.png"}, "emitted_at": 1682938848247} -{"stream": "billable_rates", "data": {"id": 2164495, "amount": 175.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "parent_id": 3758384}, "emitted_at": 1682938850125} -{"stream": "billable_rates", "data": {"id": 2164494, "amount": 175.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758383}, "emitted_at": 1682938850354} -{"stream": "billable_rates", "data": {"id": 2164493, "amount": 125.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758382}, "emitted_at": 1682938850545} -{"stream": "billable_rates", "data": {"id": 2164492, "amount": 125.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758381}, "emitted_at": 1682938850833} -{"stream": "cost_rates", "data": {"id": 1181742, "amount": 60.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "parent_id": 3758384}, "emitted_at": 1682938852389} -{"stream": "cost_rates", "data": {"id": 1181741, "amount": 60.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758383}, "emitted_at": 1682938852596} -{"stream": "cost_rates", "data": {"id": 1181740, "amount": 40.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758382}, "emitted_at": 1682938852801} -{"stream": "cost_rates", "data": {"id": 1181739, "amount": 40.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758381}, "emitted_at": 1682938853002} -{"stream": "project_assignments", "data": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE", "is_billable": false}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "task_assignments": [{"id": 307606998, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606999, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307607000, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307607001, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307607002, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1682938854549} -{"stream": "project_assignments", "data": {"id": 286326477, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671448, "name": "Monthly Retainer", "code": "SAMPLE", "is_billable": true}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "task_assignments": [{"id": 307606993, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606994, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307606995, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307606996, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307606997, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1682938854550} -{"stream": "project_assignments", "data": {"id": 286326471, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": 33.0, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": 175.0, "project": {"id": 28671447, "name": "Time & Materials Project", "code": "SAMPLE", "is_billable": true}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "task_assignments": [{"id": 307606988, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606989, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307606990, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307606991, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307606992, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1682938854551} -{"stream": "project_assignments", "data": {"id": 286326466, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 175.0, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE", "is_billable": true}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "task_assignments": [{"id": 307606983, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606984, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307606985, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307606986, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307606987, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1682938854551} -{"stream": "project_assignments", "data": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0, "project": {"id": 28674500, "name": "First project", "code": "FP", "is_billable": true}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "task_assignments": [{"id": 307640131, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307640132, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307640133, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307640135, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758380}, "emitted_at": 1682938855381} -{"stream": "expenses_clients", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_amount": 481.0, "billable_amount": 307.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938856813} -{"stream": "expenses_clients", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_amount": 461.0, "billable_amount": 261.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938856814} -{"stream": "expenses_projects", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "project_id": 28671446, "project_name": "[SAMPLE] Fixed Fee Project", "total_amount": 339.0, "billable_amount": 165.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938857738} -{"stream": "expenses_projects", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671448, "project_name": "[SAMPLE] Monthly Retainer", "total_amount": 238.0, "billable_amount": 238.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938857738} -{"stream": "expenses_projects", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "project_id": 28671449, "project_name": "[SAMPLE] Non-Billable Project", "total_amount": 142.0, "billable_amount": 142.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938857738} -{"stream": "expenses_projects", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671447, "project_name": "[SAMPLE] Time & Materials Project", "total_amount": 223.0, "billable_amount": 23.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938857738} -{"stream": "expenses_categories", "data": {"expense_category_id": 7892981, "expense_category_name": "Entertainment", "total_amount": 58.0, "billable_amount": 58.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938858653} -{"stream": "expenses_categories", "data": {"expense_category_id": 7892983, "expense_category_name": "Lodging", "total_amount": 200.0, "billable_amount": 0.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938858654} -{"stream": "expenses_categories", "data": {"expense_category_id": 7892982, "expense_category_name": "Meals", "total_amount": 261.0, "billable_amount": 261.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938858654} -{"stream": "expenses_categories", "data": {"expense_category_id": 7892984, "expense_category_name": "Transportation", "total_amount": 423.0, "billable_amount": 249.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938858654} -{"stream": "expenses_team", "data": {"user_id": 3758382, "user_name": "[SAMPLE] Hiromi Hourglass", "is_contractor": false, "total_amount": 109.0, "billable_amount": 109.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938859544} -{"stream": "expenses_team", "data": {"user_id": 3758381, "user_name": "[SAMPLE] Kiran Kronological", "is_contractor": false, "total_amount": 372.0, "billable_amount": 172.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938859545} -{"stream": "expenses_team", "data": {"user_id": 3758383, "user_name": "[SAMPLE] Tamara Timekeeper", "is_contractor": false, "total_amount": 23.0, "billable_amount": 23.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938859545} -{"stream": "expenses_team", "data": {"user_id": 3758384, "user_name": "[SAMPLE] Warrin Wristwatch", "is_contractor": false, "total_amount": 438.0, "billable_amount": 264.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1682938859545} -{"stream": "uninvoiced", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "project_id": 28671446, "project_name": "Fixed Fee Project", "currency": "USD", "total_hours": 165.98, "uninvoiced_hours": 130.1, "uninvoiced_expenses": 165.0, "uninvoiced_amount": -100.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938860517} -{"stream": "uninvoiced", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671448, "project_name": "Monthly Retainer", "currency": "USD", "total_hours": 160.41, "uninvoiced_hours": 142.99, "uninvoiced_expenses": 238.0, "uninvoiced_amount": 20700.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938860518} -{"stream": "uninvoiced", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671447, "project_name": "Time & Materials Project", "currency": "USD", "total_hours": 166.21, "uninvoiced_hours": 142.7, "uninvoiced_expenses": 23.0, "uninvoiced_amount": 20454.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938860518} -{"stream": "time_clients", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_hours": 306.95, "billable_hours": 130.1, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938861628} -{"stream": "time_clients", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_hours": 326.62, "billable_hours": 285.69, "currency": "USD", "billable_amount": 40893.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938861628} -{"stream": "time_clients", "data": {"client_id": 10749825, "client_name": "First client", "total_hours": 8.01, "billable_hours": 8.01, "currency": "USD", "billable_amount": 80.1, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938861628} -{"stream": "time_projects", "data": {"project_id": 28674500, "project_name": "[FP] First project", "client_id": 10749825, "client_name": "First client", "total_hours": 8.01, "billable_hours": 8.01, "currency": "USD", "billable_amount": 80.1, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938862548} -{"stream": "time_projects", "data": {"project_id": 28671446, "project_name": "[SAMPLE] Fixed Fee Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_hours": 165.98, "billable_hours": 130.1, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938862549} -{"stream": "time_projects", "data": {"project_id": 28671448, "project_name": "[SAMPLE] Monthly Retainer", "client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_hours": 160.41, "billable_hours": 142.99, "currency": "USD", "billable_amount": 20462.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938862549} -{"stream": "time_projects", "data": {"project_id": 28671449, "project_name": "[SAMPLE] Non-Billable Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_hours": 140.97, "billable_hours": 0.0, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938862549} -{"stream": "time_projects", "data": {"project_id": 28671447, "project_name": "[SAMPLE] Time & Materials Project", "client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_hours": 166.21, "billable_hours": 142.7, "currency": "USD", "billable_amount": 20431.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938862549} -{"stream": "time_tasks", "data": {"task_id": 16575210, "task_name": "Business Development", "total_hours": 112.6, "billable_hours": 0.0, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938863622} -{"stream": "time_tasks", "data": {"task_id": 16575206, "task_name": "Design", "total_hours": 48.13, "billable_hours": 46.16, "currency": "USD", "billable_amount": 3596.35, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938863622} -{"stream": "time_tasks", "data": {"task_id": 16575208, "task_name": "Marketing", "total_hours": 234.46, "billable_hours": 187.2, "currency": "USD", "billable_amount": 19424.75, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938863623} -{"stream": "time_tasks", "data": {"task_id": 16575207, "task_name": "Programming", "total_hours": 62.72, "billable_hours": 51.37, "currency": "USD", "billable_amount": 3913.75, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938863623} -{"stream": "time_tasks", "data": {"task_id": 16575209, "task_name": "Project Management", "total_hours": 183.67, "billable_hours": 139.07, "currency": "USD", "billable_amount": 14038.5, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938863623} -{"stream": "time_team", "data": {"user_id": 3758382, "user_name": "[SAMPLE] Hiromi Hourglass", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_2.png", "total_hours": 162.23, "billable_hours": 129.77, "currency": "USD", "billable_amount": 11227.5, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938864562} -{"stream": "time_team", "data": {"user_id": 3758381, "user_name": "[SAMPLE] Kiran Kronological", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_1.png", "total_hours": 156.68, "billable_hours": 131.12, "currency": "USD", "billable_amount": 11528.75, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938864562} -{"stream": "time_team", "data": {"user_id": 3758383, "user_name": "[SAMPLE] Tamara Timekeeper", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_3.png", "total_hours": 154.59, "billable_hours": 76.22, "currency": "USD", "billable_amount": 9329.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938864562} -{"stream": "time_team", "data": {"user_id": 3758384, "user_name": "[SAMPLE] Warrin Wristwatch", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_4.png", "total_hours": 160.07, "billable_hours": 78.68, "currency": "USD", "billable_amount": 8807.75, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938864562} -{"stream": "time_team", "data": {"user_id": 3758380, "user_name": "Airbyte Developer", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://d3s3969qhosaug.cloudfront.net/v2/default-avatars/4144.png", "total_hours": 8.01, "billable_hours": 8.01, "currency": "USD", "billable_amount": 80.1, "from": "20210101", "to": "20220101"}, "emitted_at": 1682938864563} -{"stream": "project_budget", "data": {"project_id": 28671446, "project_name": "Fixed Fee Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "budget_is_monthly": false, "budget_by": "project_cost", "is_active": true, "budget": 21900.0, "budget_spent": 19164.5, "budget_remaining": 2735.5}, "emitted_at": 1682938865474} -{"stream": "project_budget", "data": {"project_id": 28671449, "project_name": "Non-Billable Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "budget_is_monthly": false, "budget_by": "project", "is_active": true, "budget": 160.0, "budget_spent": 140.97, "budget_remaining": 19.03}, "emitted_at": 1682938865474} +{"stream": "clients", "data": {"id": 10749825, "name": "First client", "is_active": true, "address": null, "statement_key": "48d746ca9125fe984b3bd800747669cc", "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-25T16:16:52Z", "currency": "USD"}, "emitted_at": 1690884270553} +{"stream": "clients", "data": {"id": 10748673, "name": "Users", "is_active": true, "address": null, "statement_key": "6b70f27acd3bf496daba22316cb750d3", "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "currency": "USD"}, "emitted_at": 1690884270554} +{"stream": "clients", "data": {"id": 10748671, "name": "[SAMPLE] Client B", "is_active": true, "address": null, "statement_key": "61faacd69e255af516cd8d772073afd4", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "currency": "USD"}, "emitted_at": 1690884270554} +{"stream": "contacts", "data": {"id": 8468604, "title": null, "first_name": "[SAMPLE] Morgan", "last_name": "Minute", "email": "morgan@harvestsample.com", "phone_office": "", "phone_mobile": "", "fax": "", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "client": {"id": 10748671, "name": "[SAMPLE] Client B"}}, "emitted_at": 1690884271043} +{"stream": "contacts", "data": {"id": 8468603, "title": null, "first_name": "[SAMPLE] Sofia", "last_name": "Stopwatch", "email": "sofia@harvestsample.com", "phone_office": "", "phone_mobile": "", "fax": "", "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}}, "emitted_at": 1690884271044} +{"stream": "company", "data": {"base_uri": "https://airbyte.harvestapp.com", "full_domain": "airbyte.harvestapp.com", "name": "Airbyte", "is_active": true, "week_start_day": "Monday", "wants_timestamp_timers": false, "time_format": "hours_minutes", "date_format": "%m/%d/%Y", "plan_type": "simple-v4", "expense_feature": true, "invoice_feature": true, "estimate_feature": true, "team_feature": true, "weekly_capacity": 144000, "approval_feature": true, "clock": "12h", "currency": "USD", "currency_code_display": "iso_code_none", "currency_symbol_display": "symbol_before", "decimal_symbol": ".", "thousands_separator": ",", "color_scheme": "orange"}, "emitted_at": 1690884271497} +{"stream": "invoices", "data": {"id": 28174545, "client_key": "489645d5b2becebe06f7a696a4d0db6a8a1c8ff1", "number": "2", "purchase_order": "", "amount": 22000.0, "due_amount": 21500.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "Subj", "notes": "", "state": "draft", "period_start": null, "period_end": null, "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": null, "paid_at": null, "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:17:55Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": null, "currency": "USD", "payment_options": [], "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632435, "kind": "Service", "description": "[SAMPLE] Fixed Fee Project", "quantity": 1.0, "unit_price": 21900.0, "amount": 21900.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}, {"id": 132632436, "kind": "Product", "description": "", "quantity": 1.0, "unit_price": 100.0, "amount": 100.0, "taxed": false, "taxed2": false, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}}]}, "emitted_at": 1690884271995} +{"stream": "invoices", "data": {"id": 28174531, "client_key": "1a3a59c71a8dd22b3a341807456c754220dc202c", "number": "1", "purchase_order": "", "amount": 76.9, "due_amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": 4.0, "discount_amount": 3.2, "subject": "", "notes": "Note", "state": "paid", "period_start": "2021-05-05", "period_end": "2021-05-05", "issue_date": "2021-05-25", "due_date": "2021-05-25", "payment_term": "upon receipt", "sent_at": "2021-05-25T16:46:28Z", "paid_at": "2021-05-25T00:00:00Z", "closed_at": null, "recurring_invoice_id": null, "created_at": "2021-05-25T16:16:51Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "currency": "USD", "payment_options": [], "client": {"id": 10749825, "name": "First client"}, "estimate": null, "retainer": null, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": [{"id": 132632398, "kind": "Service", "description": "[FP] First project: Design (05/05/2021 - 05/05/2021)", "quantity": 0.01, "unit_price": 10.0, "amount": 0.1, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}, {"id": 132632399, "kind": "Service", "description": "[FP] First project: Programming (05/05/2021 - 05/05/2021)", "quantity": 8.0, "unit_price": 10.0, "amount": 80.0, "taxed": false, "taxed2": false, "project": {"id": 28674500, "name": "First project", "code": "FP"}}]}, "emitted_at": 1690884271995} +{"stream": "invoice_messages", "data": {"id": 57176997, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:46:28Z", "updated_at": "2021-05-25T16:46:28Z", "attach_pdf": false, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1690884273321} +{"stream": "invoice_messages", "data": {"id": 57176927, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "include_link_to_client_invoice": false, "send_me_a_copy": true, "thank_you": false, "reminder": false, "send_reminder_on": null, "created_at": "2021-05-25T16:43:30Z", "updated_at": "2021-05-25T16:43:30Z", "attach_pdf": true, "event_type": null, "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "subject": "Invoice #1 from Airbyte", "body": "---------------------------------------------\r\nInvoice Summary\r\n---------------------------------------------\r\nInvoice ID: 1\r\nIssue Date: 05/25/2021\r\nClient: First client\r\nP.O. Number: \r\nAmount: $76.90\r\nDue: 05/25/2021 (upon receipt)\r\n\r\nThe detailed invoice is attached as a PDF.\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 28174531}, "emitted_at": 1690884273322} +{"stream": "invoice_payments", "data": {"id": 21857618, "amount": 500.0, "paid_at": "2021-05-26T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "", "transaction_id": null, "created_at": "2021-05-26T09:07:06Z", "updated_at": "2021-05-26T09:07:06Z", "paid_date": "2021-05-26", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174545}, "emitted_at": 1690884275279} +{"stream": "invoice_payments", "data": {"id": 21857615, "amount": 76.9, "paid_at": "2021-05-25T00:00:00Z", "recorded_by": "Airbyte Developer", "recorded_by_email": "integration-test@airbyte.io", "notes": "Payed", "transaction_id": null, "created_at": "2021-05-26T09:06:37Z", "updated_at": "2021-05-26T09:06:37Z", "paid_date": "2021-05-25", "payment_gateway": {"id": null, "name": null}, "parent_id": 28174531}, "emitted_at": 1690884276439} +{"stream": "invoice_item_categories", "data": {"id": 2732435, "name": "Product", "use_as_service": false, "use_as_expense": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884276919} +{"stream": "invoice_item_categories", "data": {"id": 2732434, "name": "Service", "use_as_service": true, "use_as_expense": false, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884276920} +{"stream": "estimates", "data": {"id": 2695071, "client_key": "de25b9eb3e82c0d5032777559e8ac0cfdfbf82b1", "number": "1", "purchase_order": "", "amount": 0.0, "tax": null, "tax_amount": 0.0, "tax2": null, "tax2_amount": 0.0, "discount": null, "discount_amount": 0.0, "subject": "", "notes": "", "state": "sent", "issue_date": "2021-05-27", "sent_at": "2021-05-27T18:12:42Z", "created_at": "2021-05-27T18:12:30Z", "updated_at": "2021-05-27T18:12:42Z", "accepted_at": null, "declined_at": null, "currency": "USD", "client": {"id": 10748670, "name": "[SAMPLE] Client A"}, "creator": {"id": 3758380, "name": "Airbyte Developer"}, "line_items": []}, "emitted_at": 1690884277393} +{"stream": "estimate_messages", "data": {"id": 4857940, "sent_by": "Airbyte Developer", "sent_by_email": "integration-test@airbyte.io", "sent_from": "Airbyte Developer", "sent_from_email": "integration-test@airbyte.io", "send_me_a_copy": true, "created_at": "2021-05-27T18:12:42Z", "updated_at": "2021-05-27T18:12:42Z", "recipients": [{"name": "Airbyte Developer", "email": "integration-test@airbyte.io"}], "event_type": null, "subject": "Estimate #1 from Airbyte", "body": "---------------------------------------------\r\nEstimate Summary\r\n---------------------------------------------\r\nEstimate ID: 1\r\nEstimate Date: 05/27/2021\r\nClient: [SAMPLE] Client A\r\nP.O. Number: \r\nAmount: $0.00\r\n\r\nYou can view the estimate here:\r\n\r\n%estimate_url%\r\n\r\nThank you!\r\n---------------------------------------------", "parent_id": 2695071}, "emitted_at": 1690884278421} +{"stream": "estimate_item_categories", "data": {"id": 2614512, "name": "Product", "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884279411} +{"stream": "estimate_item_categories", "data": {"id": 2614511, "name": "Service", "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884279412} +{"stream": "expenses", "data": {"id": 31751921, "spent_date": "2021-04-27", "notes": "This is a sample expense entry.", "total_cost": 51.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758382, "name": "[SAMPLE] Hiromi Hourglass"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}, "expense_category": {"id": 7892982, "name": "Meals", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326464, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1690884279963} +{"stream": "expenses", "data": {"id": 31751926, "spent_date": "2021-04-25", "notes": "This is a sample expense entry.", "total_cost": 142.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758381, "name": "[SAMPLE] Kiran Kronological"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "expense_category": {"id": 7892984, "name": "Transportation", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326479, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1690884279964} +{"stream": "expenses", "data": {"id": 31751920, "spent_date": "2021-04-20", "notes": "This is a sample expense entry.", "total_cost": 30.0, "units": 1.0, "billable": true, "receipt": null, "is_closed": false, "is_locked": false, "is_billed": false, "locked_reason": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758381, "name": "[SAMPLE] Kiran Kronological"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671446, "name": "Fixed Fee Project", "code": "SAMPLE"}, "expense_category": {"id": 7892982, "name": "Meals", "unit_price": null, "unit_name": null}, "user_assignment": {"id": 286326463, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": 125.0}, "invoice": null}, "emitted_at": 1690884279964} +{"stream": "expense_categories", "data": {"id": 7892986, "name": "Mileage", "unit_name": "mile", "unit_price": 0.575, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884280458} +{"stream": "expense_categories", "data": {"id": 7892985, "name": "Other", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884280459} +{"stream": "expense_categories", "data": {"id": 7892984, "name": "Transportation", "unit_name": null, "unit_price": null, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884280459} +{"stream": "tasks", "data": {"id": 16575211, "name": "Vacation", "billable_by_default": false, "default_hourly_rate": null, "is_default": false, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884282019} +{"stream": "tasks", "data": {"id": 16575210, "name": "Business Development", "billable_by_default": false, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884282020} +{"stream": "tasks", "data": {"id": 16575209, "name": "Project Management", "billable_by_default": true, "default_hourly_rate": null, "is_default": true, "is_active": true, "created_at": "2021-05-05T08:17:57Z", "updated_at": "2021-05-05T08:17:57Z"}, "emitted_at": 1690884282020} +{"stream": "time_entries", "data": {"id": 1494415019, "spent_date": "2021-05-05", "hours": 0.01, "hours_without_timer": 0.01, "rounded_hours": 0.01, "notes": "", "is_locked": true, "locked_reason": "Item Invoiced and Locked for this Time Period", "is_closed": false, "is_billed": true, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": true, "budgeted": false, "billable_rate": 10.0, "cost_rate": null, "created_at": "2021-05-05T12:53:38Z", "updated_at": "2021-05-25T16:16:52Z", "user": {"id": 3758380, "name": "Airbyte Developer"}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575206, "name": "Design"}, "user_assignment": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0}, "task_assignment": {"id": 307640132, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null}, "invoice": {"id": 28174531, "number": "1"}, "external_reference": null}, "emitted_at": 1690884282692} +{"stream": "time_entries", "data": {"id": 1494414737, "spent_date": "2021-05-05", "hours": 8.0, "hours_without_timer": 8.0, "rounded_hours": 8.0, "notes": "", "is_locked": true, "locked_reason": "Item Invoiced and Locked for this Time Period", "is_closed": false, "is_billed": true, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": true, "budgeted": false, "billable_rate": 10.0, "cost_rate": null, "created_at": "2021-05-05T12:53:23Z", "updated_at": "2021-05-25T16:16:52Z", "user": {"id": 3758380, "name": "Airbyte Developer"}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575207, "name": "Programming"}, "user_assignment": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0}, "task_assignment": {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null}, "invoice": {"id": 28174531, "number": "1"}, "external_reference": null}, "emitted_at": 1690884282693} +{"stream": "time_entries", "data": {"id": 1494238685, "spent_date": "2021-05-05", "hours": 0.71, "hours_without_timer": 0.71, "rounded_hours": 0.71, "notes": "This is a sample time entry.", "is_locked": false, "locked_reason": null, "is_closed": false, "is_billed": false, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": false, "budgeted": true, "billable_rate": null, "cost_rate": 60.0, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "task": {"id": 16575208, "name": "Marketing"}, "user_assignment": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0}, "task_assignment": {"id": 307607000, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null}, "invoice": null, "external_reference": null}, "emitted_at": 1690884282694} +{"stream": "user_assignments", "data": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1690884285723} +{"stream": "user_assignments", "data": {"id": 286326492, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": null, "project": {"id": 28671451, "name": "Airbyte", "code": null}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1690884285748} +{"stream": "user_assignments", "data": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}}, "emitted_at": 1690884285748} +{"stream": "task_assignments", "data": {"id": 307640135, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575209, "name": "Project Management"}}, "emitted_at": 1690884286243} +{"stream": "task_assignments", "data": {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575207, "name": "Programming"}}, "emitted_at": 1690884286244} +{"stream": "task_assignments", "data": {"id": 307640133, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575208, "name": "Marketing"}}, "emitted_at": 1690884286244} +{"stream": "projects", "data": {"id": 28674500, "name": "First project", "code": "FP", "is_active": true, "is_billable": true, "is_fixed_fee": false, "bill_by": "People", "budget": null, "budget_by": "none", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": "Some notes", "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10749825, "name": "First client", "currency": "USD"}}, "emitted_at": 1690884286734} +{"stream": "projects", "data": {"id": 28671451, "name": "Airbyte", "code": null, "is_active": true, "is_billable": true, "is_fixed_fee": false, "bill_by": "none", "budget": null, "budget_by": "none", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": null, "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10748673, "name": "Users", "currency": "USD"}}, "emitted_at": 1690884286734} +{"stream": "projects", "data": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE", "is_active": true, "is_billable": false, "is_fixed_fee": false, "bill_by": "none", "budget": 160.0, "budget_by": "project", "budget_is_monthly": false, "notify_when_over_budget": false, "over_budget_notification_percentage": 80.0, "show_budget_to_all": false, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "starts_on": null, "ends_on": null, "over_budget_notification_date": null, "notes": "Non-billable projects are perfect for tracking time you don\u2019t want to invoice for. You can use them to track internal projects, vacation/sick time, or pro bono work.", "cost_budget": null, "cost_budget_include_expenses": false, "hourly_rate": null, "fee": null, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}}, "emitted_at": 1690884286734} +{"stream": "roles", "data": {"id": 763939, "name": "Sample Role", "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "user_ids": [3758381, 3758382, 3758383, 3758384]}, "emitted_at": 1690884287208} +{"stream": "users", "data": {"id": 3758384, "first_name": "[SAMPLE] Warrin", "last_name": "Wristwatch", "email": "warrin@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": false, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["member"], "permissions_claims": ["expenses:read:own", "expenses:write:own", "timers:read:own", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_4.png"}, "emitted_at": 1690884287699} +{"stream": "users", "data": {"id": 3758383, "first_name": "[SAMPLE] Tamara", "last_name": "Timekeeper", "email": "tamara@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:47Z", "can_create_projects": true, "default_hourly_rate": 175.0, "cost_rate": 60.0, "roles": ["Sample Role"], "access_roles": ["administrator"], "permissions_claims": ["billable_rates:read:all", "billable_rates:write:all", "billing:read:own", "billing:write:own", "clients:read:all", "clients:write:all", "company:read:own", "company:write:own", "cost_rates:read:all", "cost_rates:write:all", "estimates:read:all", "estimates:write:all", "expenses:read:all", "expenses:write:all", "invoices:read:all", "invoices:write:all", "projects:read:all", "projects:write:all", "saved_reports:read:inactive", "saved_reports:write:inactive", "tasks:read:all", "tasks:write:all", "timers:read:all", "timers:write:all", "users:read:all", "users:write:all"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_3.png"}, "emitted_at": 1690884287699} +{"stream": "users", "data": {"id": 3758382, "first_name": "[SAMPLE] Hiromi", "last_name": "Hourglass", "email": "hiromi@harvestsample.com", "telephone": "", "timezone": "Kyiv", "weekly_capacity": 144000, "has_access_to_all_future_projects": false, "is_contractor": false, "is_active": true, "calendar_integration_enabled": false, "calendar_integration_source": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2022-04-21T09:59:46Z", "can_create_projects": false, "default_hourly_rate": 125.0, "cost_rate": 40.0, "roles": ["Sample Role"], "access_roles": ["manager"], "permissions_claims": ["expenses:read:managed", "expenses:write:own", "projects:read:managed", "timers:read:managed", "timers:write:own"], "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_2.png"}, "emitted_at": 1690884287700} +{"stream": "billable_rates", "data": {"id": 2164495, "amount": 175.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "parent_id": 3758384}, "emitted_at": 1690884288862} +{"stream": "billable_rates", "data": {"id": 2164494, "amount": 175.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758383}, "emitted_at": 1690884289068} +{"stream": "billable_rates", "data": {"id": 2164493, "amount": 125.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758382}, "emitted_at": 1690884289278} +{"stream": "cost_rates", "data": {"id": 1181742, "amount": 60.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "parent_id": 3758384}, "emitted_at": 1690884290743} +{"stream": "cost_rates", "data": {"id": 1181741, "amount": 60.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758383}, "emitted_at": 1690884290945} +{"stream": "cost_rates", "data": {"id": 1181740, "amount": 40.0, "start_date": null, "end_date": null, "created_at": "2021-05-05T08:19:31Z", "updated_at": "2021-05-05T08:19:31Z", "parent_id": 3758382}, "emitted_at": 1690884291149} +{"stream": "project_assignments", "data": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE", "is_billable": false}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "task_assignments": [{"id": 307606998, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606999, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307607000, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307607001, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307607002, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1690884293424} +{"stream": "project_assignments", "data": {"id": 286326477, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671448, "name": "Monthly Retainer", "code": "SAMPLE", "is_billable": true}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "task_assignments": [{"id": 307606993, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606994, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307606995, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307606996, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307606997, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1690884293424} +{"stream": "project_assignments", "data": {"id": 286326471, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": 33.0, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": 175.0, "project": {"id": 28671447, "name": "Time & Materials Project", "code": "SAMPLE", "is_billable": true}, "client": {"id": 10748671, "name": "[SAMPLE] Client B", "currency": "USD"}, "task_assignments": [{"id": 307606988, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575210, "name": "Business Development"}}, {"id": 307606989, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575206, "name": "Design"}}, {"id": 307606990, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575208, "name": "Marketing"}}, {"id": 307606991, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575207, "name": "Programming"}}, {"id": 307606992, "billable": true, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null, "task": {"id": 16575209, "name": "Project Management"}}], "parent_id": 3758384}, "emitted_at": 1690884293425} +{"stream": "expenses_clients", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_amount": 481.0, "billable_amount": 307.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884294719} +{"stream": "expenses_clients", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_amount": 461.0, "billable_amount": 261.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884294720} +{"stream": "expenses_projects", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "project_id": 28671446, "project_name": "[SAMPLE] Fixed Fee Project", "total_amount": 339.0, "billable_amount": 165.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884295555} +{"stream": "expenses_projects", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671448, "project_name": "[SAMPLE] Monthly Retainer", "total_amount": 238.0, "billable_amount": 238.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884295555} +{"stream": "expenses_projects", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "project_id": 28671449, "project_name": "[SAMPLE] Non-Billable Project", "total_amount": 142.0, "billable_amount": 142.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884295555} +{"stream": "expenses_categories", "data": {"expense_category_id": 7892981, "expense_category_name": "Entertainment", "total_amount": 58.0, "billable_amount": 58.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884296818} +{"stream": "expenses_categories", "data": {"expense_category_id": 7892983, "expense_category_name": "Lodging", "total_amount": 200.0, "billable_amount": 0.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884296819} +{"stream": "expenses_categories", "data": {"expense_category_id": 7892982, "expense_category_name": "Meals", "total_amount": 261.0, "billable_amount": 261.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884296819} +{"stream": "expenses_team", "data": {"user_id": 3758382, "user_name": "[SAMPLE] Hiromi Hourglass", "is_contractor": false, "total_amount": 109.0, "billable_amount": 109.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884297758} +{"stream": "expenses_team", "data": {"user_id": 3758381, "user_name": "[SAMPLE] Kiran Kronological", "is_contractor": false, "total_amount": 372.0, "billable_amount": 172.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884297758} +{"stream": "expenses_team", "data": {"user_id": 3758383, "user_name": "[SAMPLE] Tamara Timekeeper", "is_contractor": false, "total_amount": 23.0, "billable_amount": 23.0, "currency": "USD", "from": "20210101", "to": "20220101"}, "emitted_at": 1690884297759} +{"stream": "uninvoiced", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "project_id": 28671446, "project_name": "Fixed Fee Project", "currency": "USD", "total_hours": 165.98, "uninvoiced_hours": 130.1, "uninvoiced_expenses": 165.0, "uninvoiced_amount": -100.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884299583} +{"stream": "uninvoiced", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671448, "project_name": "Monthly Retainer", "currency": "USD", "total_hours": 160.41, "uninvoiced_hours": 142.99, "uninvoiced_expenses": 238.0, "uninvoiced_amount": 20700.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884299584} +{"stream": "uninvoiced", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "project_id": 28671447, "project_name": "Time & Materials Project", "currency": "USD", "total_hours": 166.21, "uninvoiced_hours": 142.7, "uninvoiced_expenses": 23.0, "uninvoiced_amount": 20454.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884299584} +{"stream": "time_clients", "data": {"client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_hours": 306.95, "billable_hours": 130.1, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884300446} +{"stream": "time_clients", "data": {"client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_hours": 326.62, "billable_hours": 285.69, "currency": "USD", "billable_amount": 40893.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884300447} +{"stream": "time_clients", "data": {"client_id": 10749825, "client_name": "First client", "total_hours": 8.01, "billable_hours": 8.01, "currency": "USD", "billable_amount": 80.1, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884300447} +{"stream": "time_projects", "data": {"project_id": 28674500, "project_name": "[FP] First project", "client_id": 10749825, "client_name": "First client", "total_hours": 8.01, "billable_hours": 8.01, "currency": "USD", "billable_amount": 80.1, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884301381} +{"stream": "time_projects", "data": {"project_id": 28671446, "project_name": "[SAMPLE] Fixed Fee Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "total_hours": 165.98, "billable_hours": 130.1, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884301382} +{"stream": "time_projects", "data": {"project_id": 28671448, "project_name": "[SAMPLE] Monthly Retainer", "client_id": 10748671, "client_name": "[SAMPLE] Client B", "total_hours": 160.41, "billable_hours": 142.99, "currency": "USD", "billable_amount": 20462.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884301382} +{"stream": "time_tasks", "data": {"task_id": 16575210, "task_name": "Business Development", "total_hours": 112.6, "billable_hours": 0.0, "currency": "USD", "billable_amount": 0.0, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884302301} +{"stream": "time_tasks", "data": {"task_id": 16575206, "task_name": "Design", "total_hours": 48.13, "billable_hours": 46.16, "currency": "USD", "billable_amount": 3596.35, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884302302} +{"stream": "time_tasks", "data": {"task_id": 16575208, "task_name": "Marketing", "total_hours": 234.46, "billable_hours": 187.2, "currency": "USD", "billable_amount": 19424.75, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884302302} +{"stream": "time_team", "data": {"user_id": 3758382, "user_name": "[SAMPLE] Hiromi Hourglass", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_2.png", "total_hours": 162.23, "billable_hours": 129.77, "currency": "USD", "billable_amount": 11227.5, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884303272} +{"stream": "time_team", "data": {"user_id": 3758381, "user_name": "[SAMPLE] Kiran Kronological", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_1.png", "total_hours": 156.68, "billable_hours": 131.12, "currency": "USD", "billable_amount": 11528.75, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884303273} +{"stream": "time_team", "data": {"user_id": 3758383, "user_name": "[SAMPLE] Tamara Timekeeper", "is_contractor": false, "weekly_capacity": 144000, "avatar_url": "https://cache.harvestapp.com/assets/avatars/sample_avatar_3.png", "total_hours": 154.59, "billable_hours": 76.22, "currency": "USD", "billable_amount": 9329.25, "from": "20210101", "to": "20220101"}, "emitted_at": 1690884303273} +{"stream": "project_budget", "data": {"project_id": 28671446, "project_name": "Fixed Fee Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "budget_is_monthly": false, "budget_by": "project_cost", "is_active": true, "budget": 21900.0, "budget_spent": 19164.5, "budget_remaining": 2735.5}, "emitted_at": 1690884304186} +{"stream": "project_budget", "data": {"project_id": 28671449, "project_name": "Non-Billable Project", "client_id": 10748670, "client_name": "[SAMPLE] Client A", "budget_is_monthly": false, "budget_by": "project", "is_active": true, "budget": 160.0, "budget_spent": 140.97, "budget_remaining": 19.03}, "emitted_at": 1690884304187} +{"stream": "project_budget", "data": {"project_id": 28671448, "project_name": "Monthly Retainer", "client_id": 10748671, "client_name": "[SAMPLE] Client B", "budget_is_monthly": true, "budget_by": "project_cost", "is_active": true, "budget": 21910.0, "budget_spent": 0.0, "budget_remaining": 21910.0}, "emitted_at": 1690884304188} diff --git a/airbyte-integrations/connectors/source-harvest/metadata.yaml b/airbyte-integrations/connectors/source-harvest/metadata.yaml index 3b0c124988ca..6930f5cc5963 100644 --- a/airbyte-integrations/connectors/source-harvest/metadata.yaml +++ b/airbyte-integrations/connectors/source-harvest/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/harvest tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-harvest/requirements.txt b/airbyte-integrations/connectors/source-harvest/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-harvest/requirements.txt +++ b/airbyte-integrations/connectors/source-harvest/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-harvest/setup.py b/airbyte-integrations/connectors/source-harvest/setup.py index 880a43c6eef8..a208606a02a8 100644 --- a/airbyte-integrations/connectors/source-harvest/setup.py +++ b/airbyte-integrations/connectors/source-harvest/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json index c8a20cab052d..0ff2ad3449b8 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json @@ -162,6 +162,9 @@ } } } + }, + "payment_options": { + "type": ["null", "array"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/.dockerignore b/airbyte-integrations/connectors/source-hellobaton/.dockerignore index 2e018bfa0708..28b18340cf6c 100644 --- a/airbyte-integrations/connectors/source-hellobaton/.dockerignore +++ b/airbyte-integrations/connectors/source-hellobaton/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_hellobaton !setup.py diff --git a/airbyte-integrations/connectors/source-hellobaton/Dockerfile b/airbyte-integrations/connectors/source-hellobaton/Dockerfile index 146576b35cb7..67d8edc2eb01 100644 --- a/airbyte-integrations/connectors/source-hellobaton/Dockerfile +++ b/airbyte-integrations/connectors/source-hellobaton/Dockerfile @@ -34,5 +34,5 @@ COPY source_hellobaton ./source_hellobaton ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-hellobaton diff --git a/airbyte-integrations/connectors/source-hellobaton/README.md b/airbyte-integrations/connectors/source-hellobaton/README.md index 772a10e7de9b..e626764601dd 100644 --- a/airbyte-integrations/connectors/source-hellobaton/README.md +++ b/airbyte-integrations/connectors/source-hellobaton/README.md @@ -1,35 +1,10 @@ # Hellobaton Source -This is the repository for the Hellobaton source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hellobaton). +This is the repository for the Hellobaton configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/hellobaton). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hellobaton) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hellobaton/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/hellobaton) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hellobaton/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hellobaton test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hellobaton:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hellobaton:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-hellobaton/__init__.py b/airbyte-integrations/connectors/source-hellobaton/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml index b3016f30f2a4..915422c84fb7 100644 --- a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml @@ -1,20 +1,29 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-hellobaton:dev -tests: +acceptance_tests: spec: - - spec_path: "source_hellobaton/spec.json" + tests: + - spec_path: "source_hellobaton/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["templates", "time_entries"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "templates" + - name: "time_entries" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-hellobaton/metadata.yaml b/airbyte-integrations/connectors/source-hellobaton/metadata.yaml index 0889b0453170..f01eb8e06ffe 100644 --- a/airbyte-integrations/connectors/source-hellobaton/metadata.yaml +++ b/airbyte-integrations/connectors/source-hellobaton/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - ${company}.hellobaton.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 492b56d1-937c-462e-8076-21ad2031e784 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-hellobaton githubIssueLabel: source-hellobaton icon: hellobaton.svg license: MIT name: Hellobaton - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2022-01-14 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/hellobaton tags: - - language:python + - language:low-code + ab_internal: + sl: 100 + ql: 100 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-hellobaton/requirements.txt b/airbyte-integrations/connectors/source-hellobaton/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-hellobaton/requirements.txt +++ b/airbyte-integrations/connectors/source-hellobaton/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-hellobaton/setup.py b/airbyte-integrations/connectors/source-hellobaton/setup.py index fe664ca447f4..91ef2d7cc5f0 100644 --- a/airbyte-integrations/connectors/source-hellobaton/setup.py +++ b/airbyte-integrations/connectors/source-hellobaton/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( @@ -22,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py index 9137ca503a67..16eec72f437f 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml new file mode 100644 index 000000000000..aec61695eca8 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml @@ -0,0 +1,171 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - activity + +definitions: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_hellobaton/schemas/{{ parameters.path }}.json" + + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: page_size + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("next", {}) }}' + stop_condition: '{{ not response.get("next", {}) }}' + + requester: + type: HttpRequester + url_base: https://{{ config['company'] }}.hellobaton.com/api/ + path: "{{ parameters.path }}" + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + inject_into: request_parameter + field_name: api_key + request_body_json: {} + + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + + base_stream: + type: DeclarativeStream + # name: '{{ parameters.name }}' + primary_key: + - id + schema_loader: + $ref: "#/definitions/schema_loader" + retriever: + $ref: "#/definitions/retriever" + + activity_stream: + $ref: "#/definitions/base_stream" + name: "activity" + $parameters: + path: "activity" + + companies_stream: + $ref: "#/definitions/base_stream" + name: "companies" + $parameters: + path: "companies" + + milestones_stream: + $ref: "#/definitions/base_stream" + name: "milestones" + $parameters: + path: "milestones" + + phases_stream: + $ref: "#/definitions/base_stream" + name: "phases" + $parameters: + path: "phases" + + project_attachments_stream: + $ref: "#/definitions/base_stream" + name: "project_attachments" + $parameters: + path: "project_attachments" + + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + $parameters: + path: "projects" + + tasks_stream: + $ref: "#/definitions/base_stream" + name: "tasks" + $parameters: + path: "tasks" + + task_attachments_stream: + $ref: "#/definitions/base_stream" + name: "task_attachments" + $parameters: + path: "task_attachments" + + templates_stream: + $ref: "#/definitions/base_stream" + name: "templates" + $parameters: + path: "templates" + + time_entries_stream: + $ref: "#/definitions/base_stream" + name: "time_entries" + $parameters: + path: "time_entries" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + $parameters: + path: "users" + +streams: + - "#/definitions/activity_stream" + - "#/definitions/companies_stream" + - "#/definitions/milestones_stream" + - "#/definitions/phases_stream" + - "#/definitions/project_attachments_stream" + - "#/definitions/projects_stream" + - "#/definitions/tasks_stream" + - "#/definitions/task_attachments_stream" + - "#/definitions/templates_stream" + - "#/definitions/time_entries_stream" + - "#/definitions/users_stream" + +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/hellobaton + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - api_key + - company + properties: + api_key: + type: string + title: API Key + description: "authentication key required to access the api endpoints" + airbyte_secret: true + order: 0 + company: + title: company + description: Company name that generates your base api url + examples: ["google", "facebook", "microsoft"] + type: string + order: 1 diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json index cc4a631b4d81..1c56c2670dc7 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -3,46 +3,18 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "type": { - "type": ["string", "null"] - }, - "group": { - "type": "string" - }, - "parent": { - "type": ["string", "null"] - }, - "child": { - "type": ["string", "null"] - }, - "actor": { - "type": ["string", "null"] - }, - "project": { - "type": ["string", "null"] - }, - "parent_type": { - "type": ["string", "null"] - }, - "child_type": { - "type": ["string", "null"] - }, - "meta": { - "type": ["object", "null"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "group": { "type": ["string", "null"] }, + "parent": { "type": ["string", "null"] }, + "child": { "type": ["string", "null"] }, + "actor": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "parent_type": { "type": ["string", "null"] }, + "child_type": { "type": ["string", "null"] }, + "meta": { "type": ["object", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json index ebbb25699ee3..dcd834dc03c2 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json @@ -3,30 +3,11 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - - "_self": { - "type": "string" - }, - - "name": { - "type": "string" - }, - - "type": { - "type": "string" - }, - - "created": { - "type": "string", - "format": "date-time" - }, - - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json index 16e0f0060b47..1e5a6479cffa 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json @@ -3,93 +3,42 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": ["string", "null"] - }, - "project": { - "type": "string" - }, - "task_list": { - "type": "string" - }, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "task_list": { "type": ["string", "null"] }, "phase": { "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } }, - "deadline_fixed": { - "type": "boolean" - }, - "deadline_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, + "deadline_fixed": { "type": ["boolean", "null"] }, + "deadline_datetime": { "type": ["string", "null"], "format": "date-time" }, "risk_profiles": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "risk_level": { - "type": "string" - }, - "formula": { - "type": "string" - }, - "over_run": { - "type": "integer" - } + "id": { "type": ["integer", "null"] }, + "risk_level": { "type": ["string", "null"] }, + "formula": { "type": ["string", "null"] }, + "over_run": { "type": ["integer", "null"] } } } }, - "start_datetime": { - "type": "string" - }, - "finish_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "created_from": { - "type": ["string", "null"] - }, - "duration": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "start_datetime": { "type": ["string", "null"] }, + "finish_datetime": { "type": ["string", "null"], "format": "date-time" }, + "created_from": { "type": ["string", "null"] }, + "duration": { "type": ["integer", "null"] }, + "feedback_list": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json index 476437ea6774..e7ab1ae41dfd 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json @@ -3,25 +3,11 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": ["string", "null"] - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "order": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json index 0eb0977f53f2..5aacde353566 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json @@ -3,40 +3,16 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "project": { - "type": "string" - }, - "url": { - "type": "string" - }, - "label": { - "type": ["string", "null"] - }, - "created_by": { - "type": "string" - }, - "type": { - "type": "string" - }, - "is_sow": { - "type": "boolean" - }, - "original_filename": { - "type": ["string", "null"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] }, + "label": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "is_sow": { "type": ["boolean", "null"] }, + "original_filename": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json index e52493d79752..c36ca3de4a0e 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json @@ -3,109 +3,54 @@ "type": "object", "additionalProperties": true, "properties": { - "_self": { - "type": "string" - }, - "annual_contract_value": { - "type": "string" - }, - "attachment_list": { - "type": "string" - }, - "client_systems": { - "type": ["string", "null"] - }, + "_self": { "type": ["string", "null"] }, + "annual_contract_value": { "type": ["string", "null"] }, + "archived": { "type": ["boolean", "null"] }, + "attachment_list": { "type": ["string", "null"] }, + "client_systems": { "type": ["string", "null"] }, "companies": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, - "completed_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "cost": { - "type": ["integer", "null"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "created_from": { - "type": ["string", "null"] - }, - "created_from_template": { - "type": ["string", "null"] - }, - "creator": { - "type": ["string", "null"] - }, - "deadline_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "estimated_duration": { - "type": ["integer", "null"] - }, - "id": { - "type": "integer" - }, - "implementation_budget": { - "type": "string" - }, - "milestone_list": { - "type": "string" - }, - "modified": { - "type": ["string", "null"], - "format": "date-time" - }, + "completed_datetime": { "type": ["string", "null"], "format": "date-time" }, + "cost": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "created_from": { "type": ["string", "null"] }, + "created_from_template": { "type": ["string", "null"] }, + "creator": { "type": ["string", "null"] }, + "custom_field_list": { "type": ["string", "null"] }, + "deadline_datetime": { "type": ["string", "null"], "format": "date-time" }, + "estimated_duration": { "type": ["integer", "null"] }, + "id": { "type": ["integer", "null"] }, + "implementation_budget": { "type": ["string", "null"] }, + "milestone_feedback_list": { "type": ["string", "null"] }, + "milestone_list": { "type": ["string", "null"] }, + "modified": { "type": ["string", "null"], "format": "date-time" }, "phase": { "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": ["string", "null"] - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "order": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } }, + "project_phases_list": { "type": ["string", "null"] }, + "project_user_list": { "type": ["string", "null"] }, "risk_profiles": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "risk_score": { - "type": "number" - }, - "level": { - "type": "string" - }, - "variance": { - "type": "integer" - }, - "formula": { - "type": "string" - }, + "id": { "type": ["integer", "null"] }, + "risk_score": { "type": ["number", "null"] }, + "level": { "type": ["string", "null"] }, + "variance": { "type": ["integer", "null"] }, + "formula": { "type": ["string", "null"] }, "projected_golive_datetime": { "type": ["string", "null"], "format": "date-time" @@ -113,25 +58,11 @@ } } }, - "start_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "started_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "status": { - "type": "string" - }, - "task_list": { - "type": "string" - }, - "time_entry_list": { - "type": "string" - }, - "title": { - "type": "string" - } + "start_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_datetime": { "type": ["string", "null"], "format": "date-time" }, + "status": { "type": ["string", "null"] }, + "task_list": { "type": ["string", "null"] }, + "time_entry_list": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json index 90da99ec9d51..0c41e829e195 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json @@ -3,47 +3,19 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "task": { - "type": "string" - }, - "url": { - "type": "string" - }, - "type": { - "type": "string" - }, - "label": { - "type": ["string", "null"] - }, - "deliverable": { - "type": "boolean" - }, - "requires_approval": { - "type": "boolean" - }, - "approved": { - "type": ["boolean", "null"] - }, - "revision_task": { - "type": ["string", "null"] - }, - "original_filename": { - "type": ["string", "null"] - }, - "created_by": { - "type": ["string", "null"] - }, - "created": { - "type": "string" - }, - "modified": { - "type": "string" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "task": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "label": { "type": ["string", "null"] }, + "deliverable": { "type": ["boolean", "null"] }, + "requires_approval": { "type": ["boolean", "null"] }, + "approved": { "type": ["boolean", "null"] }, + "revision_task": { "type": ["string", "null"] }, + "original_filename": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"] }, + "modified": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json index 980c82d5f5ce..b2ff5e9c16d5 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json @@ -3,43 +3,22 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": ["string", "null"] - }, - "project": { - "type": "string" - }, - "status": { - "type": "string" - }, - "dependency": { - "type": ["string", "null"] - }, - "start_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "due_datetime": { - "type": "string", - "format": "date-time" - }, - "started_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "finished_datetime": { - "type": ["string", "null"], - "format": "date-time" + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] }, + "dependencies": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } }, + "start_datetime": { "type": ["string", "null"], "format": "date-time" }, + "due_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_datetime": { "type": ["string", "null"], "format": "date-time" }, + "finished_datetime": { "type": ["string", "null"], "format": "date-time" }, "started_overridden_datetime": { "type": ["string", "null"], "format": "date-time" @@ -48,69 +27,31 @@ "type": ["string", "null"], "format": "date-time" }, - "estimated_duration": { - "type": ["integer", "null"] - }, - "milestone": { - "type": "string" - }, - "created_by": { - "type": ["string", "null"] - }, - "assigned_to": { - "type": ["string", "null"] - }, - "created_from": { - "type": ["string", "null"] - }, + "estimated_duration": { "type": ["integer", "null"] }, + "milestone": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "assigned_to": { "type": ["string", "null"] }, + "created_from": { "type": ["string", "null"] }, "risk_profiles": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "risk_level": { - "type": "string" - }, - "formula": { - "type": "string" - }, - "over_run": { - "type": "integer" - }, - "task_variance": { - "type": "integer" - }, - "cool_down": { - "type": "integer" - }, - "reason": { - "type": "integer" - }, - "duration": { - "type": "integer" - }, - "estimated_duration": { - "type": "integer" - } + "id": { "type": ["integer", "null"] }, + "risk_level": { "type": ["string", "null"] }, + "formula": { "type": ["string", "null"] }, + "over_run": { "type": ["integer", "null"] }, + "task_variance": { "type": ["integer", "null"] }, + "cool_down": { "type": ["integer", "null"] }, + "reason": { "type": ["integer", "null"] }, + "duration": { "type": ["integer", "null"] }, + "estimated_duration": { "type": ["integer", "null"] } } } }, - "time_entry_list": { - "type": "string" - }, - "attachment_list": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "time_entry_list": { "type": ["string", "null"] }, + "attachment_list": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json index e2d02743d883..e364f7ddb2bf 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json @@ -3,82 +3,32 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "status": { - "type": "string" - }, - "cost": { - "type": ["string", "null"] - }, - "annual_contract_value": { - "type": ["string", "null"] - }, - "implementation_budget": { - "type": ["string", "null"] - }, - "estimated_duration": { - "type": ["integer", "null"] - }, - "created_from_template": { - "type": ["string", "null"] - }, - "created_from": { - "type": ["string", "null"] - }, - "start_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "started_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "deadline_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "completed_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "client_systems": { - "type": ["string", "null"] - }, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] }, + "cost": { "type": ["string", "null"] }, + "annual_contract_value": { "type": ["string", "null"] }, + "implementation_budget": { "type": ["string", "null"] }, + "estimated_duration": { "type": ["integer", "null"] }, + "created_from_template": { "type": ["string", "null"] }, + "created_from": { "type": ["string", "null"] }, + "start_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_datetime": { "type": ["string", "null"], "format": "date-time" }, + "deadline_datetime": { "type": ["string", "null"], "format": "date-time" }, + "completed_datetime": { "type": ["string", "null"], "format": "date-time" }, + "client_systems": { "type": ["string", "null"] }, "phase": { "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "order": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } }, - "creator": { - "type": ["string", "null"] - }, - "task_list": { - "type": "string" - } + "creator": { "type": ["string", "null"] }, + "task_list": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json index 7ea2f5553be2..b75a95128fc1 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json @@ -3,58 +3,25 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "user": { - "type": "string" - }, - "created_by": { - "type": "string" - }, - "project": { - "type": "string" - }, - "task": { - "type": ["string", "null"] - }, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "user": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "task": { "type": ["string", "null"] }, "rate": { - "type": "object", + "type": ["object", "null"], "properties": { - "id": { - "type": "integer" - }, - "hourly_rate": { - "type": "string" - } + "id": { "type": ["integer", "null"] }, + "hourly_rate": { "type": ["string", "null"] } } }, - "started_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "ended_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "reference_date": { - "type": "string", - "format": "date-time" - }, - "billable": { - "type": "boolean" - }, - "calculated_duration": { - "type": ["integer", "null"] - }, - "inputted_duration": { - "type": ["integer", "null"] - }, - "notes": { - "type": "string" - } + "started_at": { "type": ["string", "null"], "format": "date-time" }, + "ended_at": { "type": ["string", "null"], "format": "date-time" }, + "reference_date": { "type": ["string", "null"], "format": "date-time" }, + "billable": { "type": ["boolean", "null"] }, + "calculated_duration": { "type": ["integer", "null"] }, + "inputted_duration": { "type": ["integer", "null"] }, + "notes": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json index 3e6d3d73a603..4ce6c1eed02c 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json @@ -3,47 +3,20 @@ "type": "object", "additionalProperties": true, "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "account_type": { - "type": "string" - }, - "job_title": { - "type": "string" - }, - "company": { - "type": "string" - }, - "avatar_url": { - "type": ["string", "null"] - }, - "created_by": { - "type": ["string", "null"] - }, - "signed_up_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "first_name": { "type": ["string", "null"] }, + "last_name": { "type": ["string", "null"] }, + "capacity": { "type": ["integer", "null"] }, + "department": { "type": ["object", "null"] }, + "email": { "type": ["string", "null"] }, + "account_type": { "type": ["string", "null"] }, + "job_title": { "type": ["string", "null"] }, + "company": { "type": ["string", "null"] }, + "avatar_url": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "signed_up_at": { "type": ["string", "null"], "format": "date-time" }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py index 295a4a10b53c..30da755b57fa 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py @@ -2,58 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream +WARNING: Do not modify this file. +""" -from .streams import ( - Activity, - Companies, - Milestones, - Phases, - ProjectAttachments, - Projects, - TaskAttachments, - Tasks, - Templates, - TimeEntries, - Users, -) -STREAMS = [Activity, Companies, Milestones, Projects, Phases, ProjectAttachments, Tasks, TaskAttachments, Templates, TimeEntries, Users] - - -# Source -class SourceHellobaton(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, any]) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - url_template = "https://{company}.hellobaton.com/api/" - try: - params = { - "api_key": config["api_key"], - } - base_url = url_template.format(company=config["company"]) - # This is just going to return a mapping of available endpoints - response = requests.get(base_url, params=params) - status_code = response.status_code - logger.info(f"Status code: {status_code}") - if status_code == 200: - return True, None - - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - return [stream(company=config["company"], api_key=config["api_key"]) for stream in STREAMS] +# Declarative Source +class SourceHellobaton(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json deleted file mode 100644 index 50793f64fae2..000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Hellobaton Spec", - "type": "object", - "required": ["api_key", "company"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "authentication key required to access the api endpoints", - "airbyte_secret": true - }, - "company": { - "type": "string", - "description": "Company name that generates your base api url", - "examples": ["google", "facebook", "microsoft"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py deleted file mode 100644 index 654cd2548401..000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py +++ /dev/null @@ -1,204 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional -from urllib.parse import parse_qs, urlparse - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -# Basic full refresh stream -class HellobatonStream(HttpStream, ABC): - """ - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - """ - - page_size: int = 100 - primary_key: str = "id" - - def __init__(self, company: str, api_key: str, **kwargs): - super().__init__(**kwargs) - self.api_key = api_key - self.company = company - - @property - def url_base(self) -> str: - """ - Using this method instead of class init to dynamically generate base url based on config - """ - company = self.company - return f"https://{company}.hellobaton.com/api/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Logic to generate next page token based on the response - """ - - payload = response.json() - result_count = payload["count"] - - if result_count > self.page_size: - query_string = urlparse(payload["next"]).query - next_page_token = parse_qs(query_string).get("page", None) - - else: - next_page_token = None - - return next_page_token - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - API request params which expect an api key for auth and any pagination is done using defined in the next_page_token method - """ - - params = {"api_key": self.api_key, "page_size": self.page_size, "page": next_page_token} - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - May want to add logic here to unpack foreign keys from urls but tbd - For now each response record is accessed through the results key in the JSON payload - """ - for results in response.json()["results"]: - yield results - - -class Activity(HellobatonStream): - """ - Activity stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "activity" - - -class Companies(HellobatonStream): - """ - Companies stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "companies" - - -class Milestones(HellobatonStream): - """ - Milestones stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "milestones" - - -class Phases(HellobatonStream): - """ - Phases stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "phases" - - -class ProjectAttachments(HellobatonStream): - """ - Project attachments stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "project_attachments" - - -class Projects(HellobatonStream): - """ - Projects stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "projects" - - -class Tasks(HellobatonStream): - """ - Tasks stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "tasks" - - -class TaskAttachments(HellobatonStream): - """ - Task attachments stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "task_attachments" - - -class Templates(HellobatonStream): - """ - Templates stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "templates" - - -class TimeEntries(HellobatonStream): - """ - Time entries stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "time_entries" - - -class Users(HellobatonStream): - """ - Users stream class - """ - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - - return "users" diff --git a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml index 36c7d886540d..8b0907752bb1 100644 --- a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/hubplanner tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-hubplanner/requirements.txt b/airbyte-integrations/connectors/source-hubplanner/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-hubplanner/requirements.txt +++ b/airbyte-integrations/connectors/source-hubplanner/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py index 8f0343407d3e..0aa78f28fe34 100644 --- a/airbyte-integrations/connectors/source-hubplanner/setup.py +++ b/airbyte-integrations/connectors/source-hubplanner/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 1c7d1301fbe0..ac381183e833 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.8.4 +LABEL io.airbyte.version=1.4.1 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index e308b7d1c39a..fba382b6bc7b 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -36,6 +36,10 @@ The primary key for the following streams is `pipelineId`: - deal_pipelines +The primary key for the following streams is `vid-to-merge`: + +- contacts_merged_audit + The following streams do not have a primary key: - contact_lists (The primary key could potentially be a composite key (portalId, listId) - https://legacydocs.hubspot.com/docs/methods/lists/get_lists) diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index 7b83a2790ca0..94fa9c282e4c 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -39,6 +39,10 @@ acceptance_tests: bypass_reason: Unable to populate - name: quotes bypass_reason: Unable to populate + - name: deals_archived + bypass_reason: Unable to populate + - name: owners_archived + bypass_reason: unable to populate ignored_fields: contact_lists: - name: ilsFilterBranch @@ -46,15 +50,25 @@ acceptance_tests: companies: - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties/hs_was_imported + bypass_reason: attribute is not stable contacts: - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties/hs_time_in_subscriber + bypass_reason: Hubspot time depend on current time - name: properties/hs_latest_source_timestamp bypass_reason: Hubspot time depend on current time - name: properties/hs_predictivescoringtier bypass_reason: Hubspot prediction changes - name: properties/lastmodifieddate bypass_reason: Hubspot time depend on current time + - name: properties/hs_time_in_lead + bypass_reason: Hubspot time depend on current time + - name: properties/hs_time_in_opportunity + bypass_reason: Hubspot time depend on current time + - name: properties/hs_was_imported + bypass_reason: attribute is not stable - name: updatedAt bypass_reason: Hubspot time depend on current time deals: @@ -89,7 +103,12 @@ acceptance_tests: goals: - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time - fail_on_extra_columns: false + - name: properties/hs_lastmodifieddate + bypass_reason: Hubspot time depend on current time + - name: properties/hs_kpi_value_last_calculated_at + bypass_reason: Hubspot time depend on current time + - name: updatedAt + bypass_reason: field changes too often full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json index 060ce30ce4d0..5ad10ada31e8 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json @@ -43,6 +43,17 @@ } } }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "contacts_merged_audit" + }, + "stream_state": { + "updatedAt": "2221-10-12T13:37:56.412000+00:00" + } + } + }, { "type": "STREAM", "stream": { diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl index f1e98eb05e38..643ca48f835d 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl @@ -1,49 +1,52 @@ -{"stream": "campaigns", "data": {"id": 243851494, "lastUpdatedTime": 1675121674226, "appId": 113, "appName": "Batch", "contentId": 100523515217, "subject": "test", "name": "test", "counters": {"dropped": 1}, "lastProcessingFinishedAt": 1675121674000, "lastProcessingStartedAt": 1675121671000, "lastProcessingStateChangeAt": 1675121674000, "numIncluded": 1, "processingState": "DONE", "type": "BATCH_EMAIL"}, "emitted_at": 1685387171525} -{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 63794924339, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false}, "emitted_at": 1685387173357} -{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-04-04T15:12:52.778000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 4766715220, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-04-04T15:12:52.778Z", "archived": false, "contacts": ["151", "151"]}, "emitted_at": 1685387173358} -{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false}, "emitted_at": 1685387173359} +{"stream": "campaigns", "data": {"id": 243851494, "lastUpdatedTime": 1675121674226, "appId": 113, "appName": "Batch", "contentId": 100523515217, "subject": "test", "name": "test", "counters": {"dropped": 1}, "lastProcessingFinishedAt": 1675121674000, "lastProcessingStartedAt": 1675121671000, "lastProcessingStateChangeAt": 1675121674000, "numIncluded": 1, "processingState": "DONE", "type": "BATCH_EMAIL"}, "emitted_at": 1688723569670} +{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 68102534447, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false}, "emitted_at": 1689694783559} +{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-04-04T15:12:52.778000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 9074325326, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-04-04T15:12:52.778Z", "archived": false, "contacts": ["151", "151"]}, "emitted_at": 1689694783560} +{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false}, "emitted_at": 1689694783560} {"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 166, "createdAt": 1675120756833, "updatedAt": 1675120852460, "name": "Test", "listType": "DYNAMIC", "authorId": 12282590, "filters": [], "metaData": {"size": 3, "lastSizeChangeAt": 1675257270514, "processing": "DONE", "lastProcessingStateChangeAt": 1675120853286, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"createdate\",\"operation\":{\"propertyType\":\"datetime\",\"operator\":\"IS_AFTER\",\"timestamp\":1669957199999,\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"requiresTimeZoneConversion\":true,\"operationType\":\"datetime\",\"operatorName\":\"IS_AFTER\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "internal": false, "limitExempt": false, "dynamic": true}, "emitted_at": 1685387174847} -{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "sh", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 77736986078, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:28:17.125000+00:00", "lastname": "na", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-03-21T19:28:17.125Z", "archived": false, "companies": ["5000526215", "5000526215"]}, "emitted_at": 1685387176169} -{"stream": "contacts", "data": {"id": "201", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": null, "createdate": "2021-01-14T14:26:17.014000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "201", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-01-14T14:26:17.014000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-01-14T14:26:17.014000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-01-14T14:26:17.081000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-01-14T14:26:17.014000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 201, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_3", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 74752799179, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:30:56.518000+00:00", "lastname": "testerson", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-01-14T14:26:17.014Z", "updatedAt": "2023-04-03T20:30:56.518Z", "archived": false, "companies": ["5170561229", "5170561229"]}, "emitted_at": 1685387176170} -{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 71384466249, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"]}, "emitted_at": 1685387176171} +{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "sh", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 82747504126, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:28:17.125000+00:00", "lastname": "na", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-03-21T19:28:17.125Z", "archived": false, "companies": ["5000526215", "5000526215"]}, "emitted_at": 1690397694270} +{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 76394984297, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"]}, "emitted_at": 1690397694271} +{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 76286658061, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false}, "emitted_at": 1690397694271} +{"stream": "contacts", "data": {"id": "601", "properties": {"address": "0 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot Test", "company_size": null, "country": null, "createdate": "2021-10-12T13:22:50.930000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingapicontact_0@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test contact 0", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "601", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-10-12T13:22:50.930000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-10-12T13:22:50.930000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-10-12T13:22:51.107000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-10-12T13:22:50.930000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 601, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.31, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_2", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 56352723312, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:29:57.224000+00:00", "lastname": "testerson number 0", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-10-12T13:22:50.930Z", "updatedAt": "2023-04-03T20:29:57.224Z", "archived": false}, "emitted_at": 1690397694272} +{"stream": "contacts", "data": {"id": "651", "properties": {"address": "1 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot Test", "company_size": null, "country": null, "createdate": "2021-01-14T14:26:17.014000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingapicontact_1@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test contact 1-1", "gender": null, "graduation_date": null, "hs_additional_emails": "testingapis@hubspot.com", "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "201;651", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-01-14T14:26:17.014000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": "201:1688758327178", "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-10-12T13:23:01.830000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-01-14T14:26:17.081000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-10-12T13:23:01.830000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": "201", "hs_object_id": 651, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.39, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_2", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 56352712412, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-07-07T19:33:25.177000+00:00", "lastname": "testerson number 1", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-01-14T14:26:17.014Z", "updatedAt": "2023-07-07T19:33:25.177Z", "archived": false, "companies": ["5170561229", "5170561229"]}, "emitted_at": 1690397694272} +{"stream": "contacts", "data": {"id": "2501", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2023-01-30T23:17:09.904000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "test@test.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "2501", "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2023-01-30T23:17:09.904000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+555555555555", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "BR", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-01-30T23:17:09.904000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2023-01-31T00:31:07.832000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": "2023-01-31T00:31:07.832000+00:00", "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "test.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CRM_UI", "hs_latest_source_data_2": "userId:12282590", "hs_latest_source_timestamp": "2023-01-30T23:17:10.053000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": "Legitimate interest \u2013 prospect/lead", "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2023-01-30T23:17:09.904000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": "2023-01-31T00:31:07.832000+00:00", "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 2501, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 2.93, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_1", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5555555555", "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 4437928, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 15272626409, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-04T15:11:47.143000+00:00", "lastname": "test", "lifecyclestage": "opportunity", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "+555555555555", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2023-01-30T23:17:09.904Z", "updatedAt": "2023-04-04T15:11:47.143Z", "archived": false}, "emitted_at": 1690397694273} +{"stream": "contacts", "data": {"id": "2551", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2023-01-31T00:11:58.499000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "test-integration-test-user-4@testmail.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": null, "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "2551", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2023-01-31T00:11:58.499000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-01-31T00:11:58.499000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "testmail.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CRM_UI", "hs_latest_source_data_2": "userId:12282590", "hs_latest_source_timestamp": "2023-01-31T00:11:58.582000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": "Legitimate interest \u2013 prospect/lead", "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2023-01-31T00:11:58.499000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 2551, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 2.93, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_1", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 15273775742, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2023-01-31T00:20:40.680000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:29:54.560000+00:00", "lastname": null, "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2023-01-31T00:11:58.499Z", "updatedAt": "2023-04-03T20:29:54.560Z", "archived": false}, "emitted_at": 1690397694273} +{"stream": "contacts", "data": {"id": "2601", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2023-02-01T13:08:49.766000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "test.person@airbyte.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "TEST", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "2601", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2023-02-01T13:08:49.735000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-02-01T13:08:49.735000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "airbyte.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CRM_UI", "hs_latest_source_data_2": "userId:12282590", "hs_latest_source_timestamp": "2023-02-01T13:08:49.863000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": "Performance of a contract", "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2023-02-01T13:08:49.735000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": "12282590", "hs_marketable_reason_type": "USER_SET", "hs_marketable_status": "true", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 2601, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 3.15, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_1", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 15140764507, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2023-02-01T13:08:49.735000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:29:58.691000+00:00", "lastname": "TEST", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2023-02-01T13:08:49.766Z", "updatedAt": "2023-04-03T20:29:58.691Z", "archived": false}, "emitted_at": 1690397694274} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1675124235515, "vid": 2501, "is-member": true}, "emitted_at": 1685387177140} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 61, "internal-list-id": 2147483643, "timestamp": 1675124259228, "vid": 2501, "is-member": true}, "emitted_at": 1685387177141} {"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 166, "internal-list-id": 2147483643, "timestamp": 1675120848102, "vid": 2501, "is-member": true}, "emitted_at": 1685387177141} {"stream": "deal_pipelines", "data": {"label": "New Business Pipeline", "displayOrder": 3, "active": true, "stages": [{"label": "Initial Qualification", "displayOrder": 0, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "9567448", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Success! Closed Won", "displayOrder": 2, "metadata": {"isClosed": "true", "probability": "1.0"}, "stageId": "customclosedwonstage", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Negotiation", "displayOrder": 1, "metadata": {"isClosed": "false", "probability": "0.5"}, "stageId": "9567449", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Closed Lost", "displayOrder": 3, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "66894120", "createdAt": 1680620354263, "updatedAt": 1680620354263, "active": true}], "objectType": "DEAL", "objectTypeId": "0-3", "pipelineId": "b9152945-a594-4835-9676-a6f405fecd71", "createdAt": 1610635973956, "updatedAt": 1680620354263, "default": false}, "emitted_at": 1685387177933} -{"stream": "deals", "data": {"id": "4280411910", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-02-22T14:01:11.762000+00:00", "days_to_close": 0, "dealname": "Test Deal 332", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": "Test deal", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-02-22T14:01:11.762000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-22T14:01:11.762000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_lastmodifieddate": "2023-04-04T21:28:37.824000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": 0, "hs_object_id": 4280411910, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 95.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 71384707397, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2021-02-22T14:01:11.762000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-02-22T14:01:11.762Z", "updatedAt": "2023-04-04T21:28:37.824Z", "archived": false, "companies": ["5183409178", "5183409178"], "line_items": ["5153237390"]}, "emitted_at": 1685387179141} -{"stream": "deals", "data": {"id": "4315375411", "properties": {"amount": 10, "amount_in_home_currency": 10, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-02-28T20:20:10.826000+00:00", "createdate": "2021-02-23T20:20:10.826000+00:00", "days_to_close": 5, "dealname": "Test deal 2", "dealstage": "appointmentscheduled", "dealtype": null, "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:21:32.862000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-23T20:21:32.862000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 10, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_lastmodifieddate": "2023-01-30T23:10:56.577000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": null, "hs_object_id": 4315375411, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 2.0, "hs_projected_amount_in_home_currency": 2.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 10.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 71275486299, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2021-02-23T20:21:32.862000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2021-02-26T06:00:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": 0, "num_notes": 2, "pipeline": "default"}, "createdAt": "2021-02-23T20:20:10.826Z", "updatedAt": "2023-01-30T23:10:56.577Z", "archived": false, "line_items": ["1188257165"]}, "emitted_at": 1685387179143} -{"stream": "deals", "data": {"id": "5313445525", "properties": {"amount": 60, "amount_in_home_currency": 60, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-05-31T10:21:28.593000+00:00", "createdate": "2021-05-21T10:21:28.593000+00:00", "days_to_close": 10, "dealname": "Test Deal AAAA", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": 60.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-05-21T10:22:40.228000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-05-21T10:22:40.228000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": 60, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_lastmodifieddate": "2023-01-23T15:35:59.701000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": 0, "hs_object_id": 5313445525, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": "medium", "hs_projected_amount": 12.0, "hs_projected_amount_in_home_currency": 12.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 60.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 63794618933, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2021-05-21T10:22:40.228000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-05-21T10:21:28.593Z", "updatedAt": "2023-01-23T15:35:59.701Z", "archived": false, "companies": ["5438025334", "5438025334"], "line_items": ["1510167477"]}, "emitted_at": 1685387179143} -{"stream": "deals_archived", "data": {"id": "11936210032", "properties": {"amount": 34, "amount_in_home_currency": 34, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2023-01-30T23:41:04.079000+00:00", "createdate": "2023-01-30T23:41:12.865000+00:00", "days_to_close": 0, "dealname": "test", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "OFFLINE", "hs_analytics_latest_source_data_1": "CRM_UI", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "CRM_UI", "hs_analytics_latest_source_data_2": "userId:12282590", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "userId:12282590", "hs_analytics_latest_source_timestamp": "2023-01-30T23:17:10.053000+00:00", "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": "2023-01-30T23:17:10.053000+00:00", "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:12.865000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2023-01-30T23:41:12.865000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": 34, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_lastmodifieddate": "2023-04-04T15:07:16.048000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0, "hs_num_of_associated_line_items": null, "hs_num_target_accounts": 0, "hs_object_id": 11936210032, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 6.800000000000001, "hs_projected_amount_in_home_currency": 6.800000000000001, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 34.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 10265107836, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:41:12.865000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2023-02-03T07:00:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 1, "num_contacted_notes": 0, "num_notes": 2, "pipeline": "default"}, "createdAt": "2023-01-30T23:41:12.865Z", "updatedAt": "2023-04-04T15:07:16.048Z", "archived": true, "archivedAt": "2023-04-04T15:11:44.712Z"}, "emitted_at": 1685387180679} -{"stream": "deals_archived", "data": {"id": "4307833946", "properties": {"amount": 123, "amount_in_home_currency": 123, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2023-04-04T15:08:40.475000+00:00", "createdate": "2021-02-23T20:20:10.826000+00:00", "days_to_close": 769, "dealname": "Test deal", "dealstage": "closedwon", "dealtype": "existingbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 123.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 123, "hs_closed_amount_in_home_currency": 123, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:21:05.293000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-23T20:21:05.293000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": "2023-04-04T15:08:40.710000+00:00", "hs_date_entered_contractsent": "2023-04-04T15:08:40.710000+00:00", "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": "2023-04-04T15:08:40.710000+00:00", "hs_date_entered_presentationscheduled": "2023-04-04T15:08:40.710000+00:00", "hs_date_entered_qualifiedtobuy": "2023-04-04T15:08:40.710000+00:00", "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": "2023-04-04T15:08:40.710000+00:00", "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": "2023-04-04T15:08:40.710000+00:00", "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": "2023-04-04T15:08:40.710000+00:00", "hs_date_exited_presentationscheduled": "2023-04-04T15:08:40.710000+00:00", "hs_date_exited_qualifiedtobuy": "2023-04-04T15:08:40.710000+00:00", "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 1, "hs_deal_stage_probability_shadow": 1, "hs_exchange_rate": null, "hs_forecast_amount": 123, "hs_forecast_probability": null, "hs_is_closed": true, "hs_is_closed_won": true, "hs_is_deal_split": false, "hs_lastmodifieddate": "2023-04-04T15:08:41.191000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": null, "hs_num_target_accounts": 0, "hs_object_id": 4307833946, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 123, "hs_projected_amount_in_home_currency": 123, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 123.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 66509255417, "hs_time_in_closedlost": null, "hs_time_in_closedwon": 4766259991, "hs_time_in_contractsent": 0, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": 0, "hs_time_in_presentationscheduled": 0, "hs_time_in_qualifiedtobuy": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2021-02-23T20:21:05.293000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-02-23T20:20:10.826Z", "updatedAt": "2023-04-04T15:08:41.191Z", "archived": true, "archivedAt": "2023-04-04T15:12:51.602Z"}, "emitted_at": 1685387180681} -{"stream": "email_events", "data": {"id": "cd276838-3925-4649-9a38-2b61761362c4", "source": "SOURCE_HUBSPOT_CUSTOMER", "subscriptions": [{"id": 23704464, "status": "SUBSCRIBED", "legalBasisChange": {"legalBasisType": "LEGITIMATE_INTEREST_CLIENT", "legalBasisExplanation": "test", "optState": "OPT_IN"}}], "recipient": "testingapicontact_0@hubspot.com", "created": 1675123491624, "sourceId": "Self Service Resubscription", "type": "STATUSCHANGE", "portalId": 8727216, "appId": 0, "emailCampaignId": 0}, "emitted_at": 1685387181597} -{"stream": "email_events", "data": {"appName": "Batch", "id": "4dcb9481-77e6-42d8-ab46-c1e3ed9395db", "emailCampaignId": 243851494, "recipient": "test@test.com", "portalId": 8727216, "appId": 113, "created": 1675121674226, "dropMessage": "Email not allowed to be sent to the specified recipient in a sandbox portal", "dropReason": "VALIDATION_FAILED", "from": "\"Team Airbyte\" ", "cc": [], "bcc": [], "replyTo": ["integration-test@airbyte.io"], "subject": "test", "type": "DROPPED", "sentBy": {"id": "4dcb9481-77e6-42d8-ab46-c1e3ed9395db", "created": 1675121674226}, "smtpId": null}, "emitted_at": 1685387181598} +{"stream": "deals", "data": {"id": "4280411910", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-02-22T14:01:11.762000+00:00", "days_to_close": 0, "dealname": "Test Deal 332", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": "Test deal", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-02-22T14:01:11.762000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-22T14:01:11.762000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2023-04-04T21:28:37.824000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": 0, "hs_object_id": 4280411910, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 95.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 77505247423, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-22T14:01:11.762000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-02-22T14:01:11.762Z", "updatedAt": "2023-04-04T21:28:37.824Z", "archived": false, "companies": ["5183409178", "5183409178"], "line_items": ["5153237390"]}, "emitted_at": 1691507719264} +{"stream": "deals", "data": {"id": "4315375411", "properties": {"amount": 10, "amount_in_home_currency": 10, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-02-28T20:20:10.826000+00:00", "createdate": "2021-02-23T20:20:10.826000+00:00", "days_to_close": 5, "dealname": "Test deal 2", "dealstage": "appointmentscheduled", "dealtype": null, "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:21:32.862000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-23T20:21:32.862000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 5, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 10, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2023-01-30T23:10:56.577000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": null, "hs_object_id": 4315375411, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 2.0, "hs_projected_amount_in_home_currency": 2.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 10.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 77396026323, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:21:32.862000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2021-02-26T06:00:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": 0, "num_notes": 2, "pipeline": "default"}, "createdAt": "2021-02-23T20:20:10.826Z", "updatedAt": "2023-01-30T23:10:56.577Z", "archived": false, "line_items": ["1188257165"]}, "emitted_at": 1691507719265} +{"stream": "deals", "data": {"id": "5313445525", "properties": {"amount": 60, "amount_in_home_currency": 60, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-05-31T10:21:28.593000+00:00", "createdate": "2021-05-21T10:21:28.593000+00:00", "days_to_close": 10, "dealname": "Test Deal AAAA", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": 60.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-05-21T10:22:40.228000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-05-21T10:22:40.228000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 10, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": 60, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2023-01-23T15:35:59.701000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": 0, "hs_object_id": 5313445525, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": "medium", "hs_projected_amount": 12.0, "hs_projected_amount_in_home_currency": 12.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 60.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 69915158957, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-05-21T10:22:40.228000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-05-21T10:21:28.593Z", "updatedAt": "2023-01-23T15:35:59.701Z", "archived": false, "companies": ["5438025334", "5438025334"], "line_items": ["1510167477"]}, "emitted_at": 1691507719265} +{"stream": "email_events", "data": {"id": "cd276838-3925-4649-9a38-2b61761362c4", "source": "SOURCE_HUBSPOT_CUSTOMER", "recipient": "testingapicontact_0@hubspot.com", "subscriptions": [{"id": 23704464, "status": "SUBSCRIBED", "legalBasisChange": {"legalBasisType": "LEGITIMATE_INTEREST_CLIENT", "legalBasisExplanation": "test", "optState": "OPT_IN"}}], "sourceId": "Self Service Resubscription", "created": 1675123491624, "type": "STATUSCHANGE", "portalId": 8727216, "appId": 0, "emailCampaignId": 0}, "emitted_at": 1688977609453} {"stream": "email_subscriptions", "data": {"id": 23704464, "portalId": 8727216, "name": "Test sub", "description": "Test sub", "active": true, "internal": false, "category": "Marketing", "channel": "Email", "businessUnitId": 0}, "emitted_at": 1685387183303} {"stream": "email_subscriptions", "data": {"id": 10798197, "portalId": 8727216, "name": "DONT USE ME", "description": "Receive feedback requests and customer service information.", "active": true, "internal": true, "category": "Service", "channel": "Email", "order": 0, "internalName": "SERVICE_HUB_FEEDBACK", "businessUnitId": 0}, "emitted_at": 1685387183304} {"stream": "email_subscriptions", "data": {"id": 11890603, "portalId": 8727216, "name": "DONT USE ME ", "description": "TTTT", "active": true, "internal": false, "category": "", "channel": "", "order": 1, "businessUnitId": 0}, "emitted_at": 1685387183304} {"stream": "engagements", "data": {"id": 11257289597, "portalId": 8727216, "active": true, "createdAt": 1614111907503, "lastUpdated": 1681915963485, "createdBy": 12282590, "modifiedBy": 12282590, "ownerId": 52550153, "type": "TASK", "timestamp": 1614319200000, "allAccessibleTeamIds": [], "bodyPreview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [4315375411], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [], "scheduledTasks": [{"engagementId": 11257289597, "portalId": 8727216, "engagementType": "TASK", "taskType": "REMINDER", "timestamp": 1614319200000, "uuid": "TASK:e41fd851-f7c7-4381-85fa-796d076163aa"}], "metadata": {"body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "status": "NOT_STARTED", "forObjectType": "OWNER", "subject": "Follow up on Test deal 2", "taskType": "TODO", "reminders": [1614319200000], "priority": "NONE", "isAllDay": false}}, "emitted_at": 1685387185031} {"stream": "engagements", "data": {"id": 30652596616, "portalId": 8727216, "active": true, "createdAt": 1675122083198, "lastUpdated": 1675122083198, "createdBy": 12282590, "modifiedBy": 12282590, "ownerId": 52550153, "type": "NOTE", "timestamp": 1675122083198, "source": "CRM_UI", "allAccessibleTeamIds": [], "bodyPreview": "test", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n
    \n

    test

    \n
    \n \n", "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [], "metadata": {"body": "

    test

    "}}, "emitted_at": 1685387185033} {"stream": "engagements", "data": {"id": 30652597343, "portalId": 8727216, "active": true, "createdAt": 1675122108834, "lastUpdated": 1680621107231, "createdBy": 12282590, "modifiedBy": 12282590, "ownerId": 52550153, "type": "TASK", "timestamp": 1675407600000, "source": "CRM_UI", "allAccessibleTeamIds": [], "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [], "scheduledTasks": [], "metadata": {"status": "NOT_STARTED", "forObjectType": "OWNER", "subject": "test", "taskType": "TODO", "reminders": [], "sendDefaultReminder": false, "priority": "NONE", "isAllDay": false}}, "emitted_at": 1685387185033} -{"stream": "engagements_notes", "data": {"id": "30652596616", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test", "hs_body_preview_html": "\n \n \n
    \n

    test

    \n
    \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:23.198000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:41:23.198000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_note_body": "

    test

    ", "hs_object_id": 30652596616, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2023-01-30T23:41:23.198000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:41:23.198000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:23.198Z", "updatedAt": "2023-01-30T23:41:23.198Z", "archived": false}, "emitted_at": 1685387193275} -{"stream": "engagements_notes", "data": {"id": "30652613125", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test", "hs_body_preview_html": "\n \n \n
    \n

    test

    \n
    \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:47.542000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:47.542000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_note_body": "

    test

    ", "hs_object_id": 30652613125, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2023-01-30T23:51:47.542000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:51:47.542000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:47.542Z", "updatedAt": "2023-01-30T23:51:47.542Z", "archived": false, "companies": ["11481383026"]}, "emitted_at": 1685387193276} -{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 1, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 11257289597, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"]}, "emitted_at": 1685387194316} -{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 0, "hs_num_associated_tickets": 0, "hs_object_id": 30652597343, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false}, "emitted_at": 1685387194317} -{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 1, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 30652613208, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"]}, "emitted_at": 1685387194317} +{"stream": "engagements_notes", "data": {"id": "30652596616", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test", "hs_body_preview_html": "\n \n \n
    \n

    test

    \n
    \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:23.198000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:41:23.198000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_note_body": "

    test

    ", "hs_object_id": 30652596616, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2023-01-30T23:41:23.198000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:23.198000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:23.198Z", "updatedAt": "2023-01-30T23:41:23.198Z", "archived": false}, "emitted_at": 1689697216801} +{"stream": "engagements_notes", "data": {"id": "30652613125", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test", "hs_body_preview_html": "\n \n \n
    \n

    test

    \n
    \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:47.542000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:47.542000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_note_body": "

    test

    ", "hs_object_id": 30652613125, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2023-01-30T23:51:47.542000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:47.542000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:47.542Z", "updatedAt": "2023-01-30T23:51:47.542Z", "archived": false, "companies": ["11481383026"]}, "emitted_at": 1689697216801} +{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 1, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 11257289597, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"]}, "emitted_at": 1689697218307} +{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 0, "hs_num_associated_tickets": 0, "hs_object_id": 30652597343, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false}, "emitted_at": 1689697218308} +{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 1, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 30652613208, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"]}, "emitted_at": 1689697218309} {"stream": "forms", "data": {"id": "03e69987-1dcb-4d55-9cb6-d3812ac00ee6", "name": "New form 93", "createdAt": "2023-02-13T16:56:33.108Z", "updatedAt": "2023-02-13T16:56:33.108Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195424} {"stream": "forms", "data": {"id": "0a7fd84f-471e-444a-a4e0-ca36d39f8af7", "name": "New form 27", "createdAt": "2023-02-13T16:45:22.640Z", "updatedAt": "2023-02-13T16:45:22.640Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195425} {"stream": "forms", "data": {"id": "0bf0c00f-e68d-4de2-8cd9-d9b04e41072f", "name": "New form 55", "createdAt": "2023-02-13T16:50:27.345Z", "updatedAt": "2023-02-13T16:50:27.345Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195425} -{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"},{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": null, "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:24.463000+00:00", "hs_lastmodifieddate": "2023-05-18T21:29:49.662000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-05-18T21:29:49.662Z", "archived": false}, "emitted_at": 1685387216587} -{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"},{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": null, "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:22.345000+00:00", "hs_lastmodifieddate": "2023-05-18T21:29:49.662000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-05-18T21:29:49.662Z", "archived": false}, "emitted_at": 1685387216588} -{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"},{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": null, "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:20.800000+00:00", "hs_lastmodifieddate": "2023-05-18T21:29:49.662000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-05-18T21:29:49.662Z", "archived": false}, "emitted_at": 1685387216589} -{"stream": "line_items", "data": {"id": "4617680695", "properties": {"amount": 34.0, "createdate": "2023-01-31T00:31:29.812000+00:00", "description": null, "discount": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:31:29.812000+00:00", "hs_line_item_currency_code": null, "hs_margin": 34.0, "hs_margin_acv": 34.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 34.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 4617680695, "hs_position_on_quote": 0, "hs_pre_discount_amount": 34.0, "hs_product_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 34.0, "hs_term_in_months": null, "hs_total_discount": 0.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 34, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:31:29.812Z", "updatedAt": "2023-01-31T00:31:29.812Z", "archived": false}, "emitted_at": 1685387217713} -{"stream": "line_items", "data": {"id": "5153237390", "properties": {"amount": 95.0, "createdate": "2023-04-04T21:28:36.663000+00:00", "description": "Baseball hat, medium", "discount": 5, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-04-04T21:28:36.663000+00:00", "hs_line_item_currency_code": null, "hs_margin": 90.0, "hs_margin_acv": 90.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 90.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 5153237390, "hs_position_on_quote": 0, "hs_pre_discount_amount": 100.0, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 95.0, "hs_term_in_months": null, "hs_total_discount": 5.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 100, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-04-04T21:28:36.663Z", "updatedAt": "2023-04-04T21:28:36.663Z", "archived": false}, "emitted_at": 1685387217714} -{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "allEmailCampaignIds": [243851494], "analyticsPageId": "100523515217", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1675121582718, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-0", "widgets": ["module-2-0-0"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1675121645993, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 100523515217, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "test", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "nlkwziGL", "primaryEmailCampaignId": 243851494, "processingStatus": "PUBLISHED", "publishDate": 1675121645997, "publishImmediately": true, "publishedAt": 1675121646297, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0, "open": 0, "delivered": 0, "bounce": 0, "unsubscribed": 0, "click": 0, "reply": 0, "dropped": 1, "selected": 1, "spamreport": 0, "suppressed": 0, "hardbounced": 0, "softbounced": 0, "pending": 0, "contactslost": 0, "notsent": 1}, "deviceBreakdown": {"open_device_type": {"computer": 0, "mobile": 0, "unknown": 0}, "click_device_type": {"computer": 0, "mobile": 0, "unknown": 0}}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 100.0}}, "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "batch", "subject": "test", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1675121702583, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [2501], "visibleToAll": true}, "emitted_at": 1685387218890} -{"stream": "owners", "data": {"id": "52550153", "email": "integration-test@airbyte.io", "firstName": "Team-1", "lastName": "Airbyte", "userId": 12282590, "createdAt": "2020-10-28T21:17:56.082Z", "updatedAt": "2023-01-31T00:25:34.448Z", "archived": false}, "emitted_at": 1685387219734} -{"stream": "products", "data": {"id": "1783898388", "properties": {"amount": null, "createdate": "2023-01-31T00:08:27.149000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:28:58.829000+00:00", "hs_merged_object_ids": null, "hs_object_id": 1783898388, "hs_product_type": "inventory", "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 1, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:08:27.149Z", "updatedAt": "2023-01-31T00:28:58.829Z", "archived": false}, "emitted_at": 1685387221089} +{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-08-18T14:59:25.726000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-18T14:59:25.726Z", "archived": false}, "emitted_at": 1692530575531} +{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:22.345000+00:00", "hs_lastmodifieddate": "2023-08-18T14:59:25.726000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-18T14:59:25.726Z", "archived": false}, "emitted_at": 1692530575532} +{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-19T22:11:13.040000+00:00", "hs_lastmodifieddate": "2023-08-19T22:11:13.080000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "in_progress", "hs_status_display_order": 1, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-19T22:11:13.080Z", "archived": false}, "emitted_at": 1692530575532} +{"stream": "line_items", "data": {"id": "4617680695", "properties": {"amount": 34.0, "createdate": "2023-01-31T00:31:29.812000+00:00", "description": null, "discount": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:31:29.812000+00:00", "hs_line_item_currency_code": null, "hs_margin": 34.0, "hs_margin_acv": 34.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 34.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 4617680695, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 34.0, "hs_product_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 34.0, "hs_term_in_months": null, "hs_total_discount": 0.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 34, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:31:29.812Z", "updatedAt": "2023-01-31T00:31:29.812Z", "archived": false}, "emitted_at": 1689697250135} +{"stream": "line_items", "data": {"id": "5153237390", "properties": {"amount": 95.0, "createdate": "2023-04-04T21:28:36.663000+00:00", "description": "Baseball hat, medium", "discount": 5, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-04-04T21:28:36.663000+00:00", "hs_line_item_currency_code": null, "hs_margin": 90.0, "hs_margin_acv": 90.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 90.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 5153237390, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 100.0, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 95.0, "hs_term_in_months": null, "hs_total_discount": 5.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 100, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-04-04T21:28:36.663Z", "updatedAt": "2023-04-04T21:28:36.663Z", "archived": false}, "emitted_at": 1689697250136} +{"stream":"marketing_emails","data":{"ab":false,"abHoursToWait":4,"abSampleSizeDefault":null,"abSamplingDefault":null,"abSuccessMetric":null,"abTestPercentage":50,"abVariation":false,"absoluteUrl":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","allEmailCampaignIds":[243851494],"analyticsPageId":"100523515217","analyticsPageType":"email","archivedAt":0,"archivedInDashboard":false,"audienceAccess":"PUBLIC","author":"integration-test@airbyte.io","authorName":"Team-1 Airbyte","blogRssSettings":null,"canSpamSettingsId":36765207029,"categoryId":2,"contentAccessRuleIds":[],"contentAccessRuleTypes":[],"contentTypeCategory":2,"createPage":false,"created":1675121582718,"createdById":12282590,"currentState":"PUBLISHED","currentlyPublished":true,"customReplyTo":"","customReplyToEnabled":false,"domain":"","emailBody":"{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}","emailNote":"","emailTemplateMode":"DRAG_AND_DROP","emailType":"BATCH_EMAIL","emailbodyPlaintext":"","feedbackSurveyId":null,"flexAreas":{"main":{"boxed":false,"isSingleColumnFullWidth":false,"sections":[{"columns":[{"id":"column-0-0","widgets":["module-0-0-0"],"width":12}],"id":"section-0","style":{"backgroundColor":"#eaf0f6","backgroundType":"CONTENT","paddingBottom":"10px","paddingTop":"10px"}},{"columns":[{"id":"column-1-0","widgets":["module-1-0-0"],"width":12}],"id":"section-1","style":{"backgroundType":"CONTENT","paddingBottom":"30px","paddingTop":"30px"}},{"columns":[{"id":"column-2-0","widgets":["module-2-0-0"],"width":12}],"id":"section-2","style":{"backgroundColor":"","backgroundType":"CONTENT","paddingBottom":"20px","paddingTop":"20px"}}]}},"freezeDate":1675121645993,"fromName":"Team Airbyte","hasContentAccessRules":false,"htmlTitle":"","id":100523515217,"isCreatedFomSandboxSync":false,"isGraymailSuppressionEnabled":true,"isInstanceLayoutPage":false,"isPublished":true,"isRecipientFatigueSuppressionEnabled":null,"language":"en","layoutSections":{},"liveDomain":"integrationtest-dev-8727216-8727216.hs-sites.com","mailingListsExcluded":[],"mailingListsIncluded":[],"maxRssEntries":5,"metaDescription":"","name":"test","pageExpiryEnabled":false,"pageRedirected":false,"pastMabExperimentIds":[],"portalId":8727216,"previewKey":"nlkwziGL","primaryEmailCampaignId":243851494,"processingStatus":"PUBLISHED","publishDate":1675121645997,"publishImmediately":true,"publishedAt":1675121646297,"publishedByEmail":"integration-test@airbyte.io","publishedById":12282590,"publishedByName":"Team-1 Airbyte","publishedUrl":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","replyTo":"integration-test@airbyte.io","resolvedDomain":"integrationtest-dev-8727216-8727216.hs-sites.com","rssEmailByText":"By","rssEmailClickThroughText":"Read more »","rssEmailCommentText":"Comment »","rssEmailEntryTemplateEnabled":false,"rssEmailImageMaxWidth":0,"rssEmailUrl":"","sections":{},"securityState":"NONE","selected":0,"slug":"-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","smartEmailFields":{},"state":"PUBLISHED","stats":{"counters":{"sent":0,"open":0,"delivered":0,"bounce":0,"unsubscribed":0,"click":0,"reply":0,"dropped":1,"selected":1,"spamreport":0,"suppressed":0,"hardbounced":0,"softbounced":0,"pending":0,"contactslost":0,"notsent":1},"deviceBreakdown":{"open_device_type":{"computer":0,"mobile":0,"unknown":0},"click_device_type":{"computer":0,"mobile":0,"unknown":0}},"failedToLoad":false,"qualifierStats":{},"ratios":{"clickratio":0,"clickthroughratio":0,"deliveredratio":0,"openratio":0,"replyratio":0,"unsubscribedratio":0,"spamreportratio":0,"bounceratio":0,"hardbounceratio":0,"softbounceratio":0,"contactslostratio":0,"pendingratio":0,"notsentratio":100}},"styleSettings":{"background_color":"#EAF0F6","background_image":null,"background_image_type":null,"body_border_color":"#EAF0F6","body_border_color_choice":"BORDER_MANUAL","body_border_width":"1","body_color":"#ffffff","color_picker_favorite1":null,"color_picker_favorite2":null,"color_picker_favorite3":null,"color_picker_favorite4":null,"color_picker_favorite5":null,"color_picker_favorite6":null,"email_body_padding":null,"email_body_width":null,"heading_one_font":{"bold":null,"color":null,"font":null,"font_style":{},"italic":null,"size":"28","underline":null},"heading_two_font":{"bold":null,"color":null,"font":null,"font_style":{},"italic":null,"size":"22","underline":null},"links_font":{"bold":false,"color":"#00a4bd","font":null,"font_style":{},"italic":false,"size":null,"underline":true},"primary_accent_color":null,"primary_font":"Arial, sans-serif","primary_font_color":"#23496d","primary_font_line_height":null,"primary_font_size":"15","secondary_accent_color":null,"secondary_font":"Arial, sans-serif","secondary_font_color":"#23496d","secondary_font_line_height":null,"secondary_font_size":"12","use_email_client_default_settings":false,"user_module_defaults":{"button_email":{"background_color":"#00a4bd","corner_radius":8,"font":"Arial, sans-serif","font_color":"#ffffff","font_size":16,"font_style":{"color":"#ffffff","font":"Arial, sans-serif","size":{"units":"px","value":16},"styles":{"bold":false,"italic":false,"underline":false}}},"email_divider":{"color":{"color":"#23496d","opacity":100},"height":1,"line_type":"solid"}}},"subcategory":"batch","subject":"test","subscription":23704464,"subscriptionName":"Test sub","teamPerms":[],"templatePath":"@hubspot/email/dnd/welcome.html","transactional":false,"translations":{},"unpublishedAt":0,"updated":1675121702583,"updatedById":12282590,"url":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","useRssHeadlineAsSubject":false,"userPerms":[],"vidsExcluded":[],"vidsIncluded":[2501],"visibleToAll":true},"emitted_at":1688060624527} +{"stream":"owners", "data": {"id": "52550153", "email": "integration-test@airbyte.io", "firstName": "Team-1", "lastName": "Airbyte", "userId": 12282590, "createdAt": "2020-10-28T21:17:56.082Z", "updatedAt": "2023-01-31T00:25:34.448Z", "archived": false}, "emitted_at": 1685387219734} +{"stream": "products", "data": {"id": "1783898388", "properties": {"amount": null, "createdate": "2023-01-31T00:08:27.149000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:28:58.829000+00:00", "hs_merged_object_ids": null, "hs_object_id": 1783898388, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_type": "inventory", "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 1, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:08:27.149Z", "updatedAt": "2023-01-31T00:28:58.829Z", "archived": false}, "emitted_at": 1689697253487} {"stream": "property_history", "data": {"value": "sh", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1673944973763, "selected": false, "property": "firstname", "vid": 151}, "emitted_at": 1685387222765} {"stream": "subscription_changes", "data": {"timestamp": 1675123491624, "recipient": "testingapicontact_0@hubspot.com", "portalId": 8727216, "normalizedEmailId": "6b59e963-cabc-4bf8-baec-feab401bdd98", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1675123491624, "change": "SUBSCRIBED", "portalId": 8727216, "subscriptionId": 23704464, "causedByEvent": {"id": "cd276838-3925-4649-9a38-2b61761362c4", "created": 1675123491624}, "changeType": "SUBSCRIPTION_STATUS"}]}, "emitted_at": 1685387223625} -{"stream": "tickets", "data": {"id": "1401690016", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2023-01-30T23:52:42.464000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2023-01-30T23:52:42.464000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2023-01-30T23:52:42.464000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2023-01-30T23:52:43.939000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_times_contacted": 0, "hs_object_id": 1401690016, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 1401690016, "hs_ticket_priority": null, "hs_time_in_1": 10264463094, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hubspot_owner_assigneddate": "2023-01-30T23:52:42.464000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": null, "subject": "test", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2023-01-30T23:52:42.464Z", "updatedAt": "2023-01-30T23:52:43.939Z", "archived": false}, "emitted_at": 1685387225514} +{"stream": "tickets", "data": {"id": "1401690016", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2023-01-30T23:52:42.464000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2023-01-30T23:52:42.464000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2023-01-30T23:52:42.464000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2023-01-30T23:52:43.939000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_times_contacted": 0, "hs_object_id": 1401690016, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 1401690016, "hs_ticket_priority": null, "hs_time_in_1": 17411822718, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:52:42.464000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": null, "subject": "test", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2023-01-30T23:52:42.464Z", "updatedAt": "2023-01-30T23:52:43.939Z", "archived": false}, "emitted_at": 1692534585461} {"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 40032127, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 321690519}, "name": "Unnamed workflow - Mon Mar 15 2021 12:58:03 GMT+0200 (cloned)", "id": 40032127, "type": "DRIP_DELAY", "enabled": true, "creationSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "createdByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "clonedFromWorkflowId": 23314874, "createdAt": 1675124258186}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "updatedByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "updatedAt": 1675124308226}, "originalAuthorUserId": 12282590, "contactListIds": {"enrolled": 167, "active": 168, "completed": 169, "succeeded": 170}, "personaTagIds": [], "lastUpdatedByUserId": 12282590, "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1675124258190, "updatedAt": 1675124308226, "description": ""}, "emitted_at": 1685387227678} -{"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false}, "emitted_at": 1685387228789} -{"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false}, "emitted_at": 1685387228789} -{"stream": "cars", "data": {"id": "5938880072", "properties": {"car_id": 1, "car_name": 3232324, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:15.836000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880072, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:15.836Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false}, "emitted_at": 1685387229829} -{"stream": "cars", "data": {"id": "5938880073", "properties": {"car_id": 2, "car_name": 23232, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:20.583000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880073, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:20.583Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false}, "emitted_at": 1685387229830} +{"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false}, "emitted_at": 1689697266624} +{"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false}, "emitted_at": 1689697266625} +{"stream": "cars", "data": {"id": "5938880072", "properties": {"car_id": 1, "car_name": 3232324, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:15.836000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880072, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:15.836Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false}, "emitted_at": 1689697267882} +{"stream": "cars", "data": {"id": "5938880073", "properties": {"car_id": 2, "car_name": 23232, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:20.583000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880073, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:20.583Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false}, "emitted_at": 1689697267883} +{"stream": "contacts_merged_audit", "data": {"canonical-vid": 651, "vid-to-merge": 201, "timestamp": 1688758327178, "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", "user-id": 12282590, "num-properties-moved": 45, "merged_from_email": {"value": "testingapis@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1610634377014, "selected": false}, "merged_to_email": {"value": "testingapicontact_1@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1634044981830, "selected": false}, "first-name": "test", "last-name": "testerson"}, "emitted_at": 1688758844966} diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index c1c4fbe8f45c..4507f326b015 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -5,19 +5,24 @@ data: connectorSubtype: api connectorType: source definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c - dockerImageTag: 0.8.4 + dockerImageTag: 1.4.1 dockerRepository: airbyte/source-hubspot githubIssueLabel: source-hubspot icon: hubspot.svg - license: MIT + license: ELv2 name: HubSpot registries: cloud: enabled: true + dockerImageTag: 1.4.1 oss: enabled: true releaseStage: generally_available documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-hubspot/requirements.txt b/airbyte-integrations/connectors/source-hubspot/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-hubspot/requirements.txt +++ b/airbyte-integrations/connectors/source-hubspot/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json index 35c11c5716bc..90d38d0e6d12 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json @@ -222,6 +222,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "owners_archived", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "products", @@ -304,6 +313,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts_merged_audit", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_oauth_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_oauth_catalog.json index d662c94894c1..dbee7d9e770d 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_oauth_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_oauth_catalog.json @@ -208,6 +208,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts_merged_audit", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json index e4945dd782af..48948a9969c5 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_catalog.json @@ -45,6 +45,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "contacts_merged_audit", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "deal_pipelines", diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_oauth_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_oauth_catalog.json index 6a6ea8d34cf2..9323d14ff76d 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_oauth_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/full_refresh_oauth_catalog.json @@ -197,6 +197,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts_merged_audit", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 3647b298862b..8adc9bbbdf2e 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -17,7 +17,6 @@ "pytest~=6.2", "pytest-mock~=3.6", "requests-mock~=1.9.3", - "connector-acceptance-test", ] setup( @@ -27,7 +26,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*,yaml", "schemas/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py index bc78501c6482..91222c77fa92 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from .source import SourceHubspot __all__ = ["SourceHubspot"] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py index d138e5004791..e73313f80e3f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py @@ -5,17 +5,31 @@ from typing import Any +import requests +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError -class HubspotError(HTTPError): +class HubspotError(AirbyteTracedException): """ Base error class. Subclassing HTTPError to avoid breaking existing code that expects only HTTPErrors. """ + def __init__( + self, + internal_message: str = None, + message: str = None, + failure_type: FailureType = FailureType.system_error, + exception: BaseException = None, + response: requests.Response = None, + ): + super().__init__(internal_message, message, failure_type, exception) + self.response = response -class HubspotTimeout(HubspotError): + +class HubspotTimeout(HTTPError): """502/504 HubSpot has processing limits in place to prevent a single client from causing degraded performance, and these responses indicate that those limits have been hit. You'll normally only see these timeout responses when making a large number of requests over a sustained period. If you get one of these responses, @@ -31,10 +45,14 @@ class HubspotAccessDenied(HubspotError): """403 Forbidden""" -class HubspotRateLimited(HubspotError): +class HubspotRateLimited(HTTPError): """429 Rate Limit Reached""" +class HubspotBadRequest(HubspotError): + """400 Bad Request""" + + class InvalidStartDateConfigError(Exception): """Raises when the User inputs wrong or invalid `start_date` in inout configuration""" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json index ea20c0503800..7b2b881ae805 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json @@ -67,6 +67,18 @@ "id": { "type": ["null", "integer"] }, + "lastProcessingFinishedAt": { + "type": ["null", "integer"] + }, + "lastProcessingStateChangeAt": { + "type": ["null", "integer"] + }, + "lastProcessingStartedAt": { + "type": ["null", "integer"] + }, + "processingState": { + "type": ["null", "string"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json index cdb936ba1d9f..500d1460eda3 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json @@ -63,6 +63,21 @@ } } }, + "ilsFilterBranch": { + "type": ["null", "string"] + }, + "internal": { + "type": ["null", "boolean"] + }, + "authorId": { + "type": ["null", "integer"] + }, + "limitExempt": { + "type": ["null", "boolean"] + }, + "teamIds": { + "type": ["null", "array"] + }, "portalId": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json new file mode 100644 index 000000000000..f3c66139aef9 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "canonical-vid": { + "type": ["null", "integer"] + }, + "vid-to-merge": { + "type": ["null", "integer"] + }, + "timestamp": { + "type": ["null", "integer"] + }, + "entity-id": { + "type": ["null", "string"] + }, + "user-id": { + "type": ["null", "integer"] + }, + "num-properties-moved": { + "type": ["null", "integer"] + }, + "merged_from_email": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "source-vids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "updated-by-user-id": { + "type": ["null", "integer"] + }, + "source-label": { + "type": ["null", "string"] + }, + "source-type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "source-id": { + "type": ["null", "string"] + }, + "selected": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "integer"] + } + } + }, + "merged_to_email": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "updated-by-user-id": { + "type": ["null", "integer"] + }, + "source-label": { + "type": ["null", "string"] + }, + "source-type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "source-id": { + "type": ["null", "string"] + }, + "selected": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "integer"] + } + } + }, + "first-name": { + "type": ["null", "string"] + }, + "last-name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json index b6573f853fe8..f77970630564 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json @@ -8,6 +8,12 @@ "appName": { "type": ["null", "string"] }, + "bcc": { + "type": ["null", "array"] + }, + "cc": { + "type": ["null", "array"] + }, "attempt": { "type": ["null", "integer"] }, @@ -36,7 +42,10 @@ "type": ["null", "string"] }, "version": { - "type": ["null", "string"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } }, @@ -174,6 +183,34 @@ "sourceId": { "type": ["null", "string"] }, + "subscriptions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "legalBasisChange": { + "type": ["null", "object"], + "properties": { + "legalBasisExplanation": { + "type": ["null", "string"] + }, + "legalBasisType": { + "type": ["null", "string"] + }, + "optState": { + "type": ["null", "string"] + } + } + }, + "status": { + "type": ["null", "string"] + } + } + } + }, "status": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_subscriptions.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_subscriptions.json index 014bad920f5b..ca2870b3e479 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_subscriptions.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_subscriptions.json @@ -1,22 +1,40 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "active": { - "type": ["null", "boolean"] - }, - "portalId":{ - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string" ] - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "order": { + "type": ["null", "integer"] + }, + "businessUnitId": { + "type": ["null", "integer"] + }, + "internal": { + "type": ["null", "boolean"] + }, + "internalName": { + "type": ["null", "string"] + }, + "category": { + "type": ["null", "string"] + }, + "channel": { + "type": ["null", "string"] } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json index d00ba5d6a342..07212bcbf3e4 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json @@ -14,6 +14,12 @@ "portalId": { "type": ["null", "integer"] }, + "queueMembershipIds": { + "type": ["null", "array"] + }, + "scheduledTasks": { + "type": ["null", "array"] + }, "active": { "type": ["null", "boolean"] }, @@ -257,7 +263,7 @@ } }, "threadId": { - "type": ["null", "string", "integer"] + "type": ["null", "string"] }, "messageId": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json index de55f229a1c7..89a39011f6fc 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json @@ -8,6 +8,9 @@ "name": { "type": ["null", "string"] }, + "formType": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json index 1eb0b900f70c..0a6e4a507111 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json @@ -30,6 +30,10 @@ "type": ["null", "string"], "format": "date-time" }, + "hs_kpi_value_last_calculated_at": { + "type": ["null", "string"], + "format": "date-time" + }, "hs_object_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json index b2cdd4573c6f..735b16fc8d82 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json @@ -397,6 +397,312 @@ "items": { "type": ["null", "integer"] } + }, + "publishedByEmail": { + "type": ["null", "string"] + }, + "sections": { + "type": ["null", "object"] + }, + "author": { + "type": ["null", "string"] + }, + "isCreatedFomSandboxSync": { + "type": ["null", "boolean"] + }, + "rssEmailUrl": { + "type": ["null", "string"] + }, + "teamPerms": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "securityState": { + "type": ["null", "string"] + }, + "isInstanceLayoutPage": { + "type": ["null", "boolean"] + }, + "audienceAccess": { + "type": ["null", "string"] + }, + "campaignUtm": { + "type": ["null", "string"] + }, + "contentAccessRuleTypes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "pastMabExperimentIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "rssEmailClickThroughText": { + "type": ["null", "string"] + }, + "rssEmailImageMaxWidth": { + "type": ["null", "integer"] + }, + "flexAreas": { + "type": ["null", "object"], + "properties": { + "properties": { + "main": { + "type": ["null", "object"], + "properties": { + "boxed": { + "type": ["null", "boolean"] + }, + "isSingleColumnFullWidth": { + "type": ["null", "boolean"] + }, + "sections": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "object"], + "properties": { + "columns": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "widgets": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "string"] + } + ] + }, + "width": { + "type": ["null", "integer"] + } + } + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "style": { + "type": ["null", "object"], + "properties": { + "backgroundColor": { + "type": ["null", "string"] + }, + "backgroundType": { + "type": ["null", "string"] + }, + "paddingBottom": { + "type": ["null", "string"] + }, + "paddingTop": { + "type": ["null", "string"] + } + } + } + } + }, + { + "type": ["null", "object"], + "properties": { + "columns": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "widgets": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "string"] + } + ] + }, + "width": { + "type": ["null", "integer"] + } + } + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "style": { + "type": ["null", "object"], + "properties": { + "backgroundType": { + "type": ["null", "string"] + }, + "paddingBottom": { + "type": ["null", "string"] + }, + "paddingTop": { + "type": ["null", "string"] + } + } + } + } + }, + { + "type": ["null", "object"], + "properties": { + "columns": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "widgets": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "string"] + } + ] + }, + "width": { + "type": ["null", "integer"] + } + } + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "style": { + "type": ["null", "object"], + "properties": { + "backgroundColor": { + "type": ["null", "string"] + }, + "backgroundType": { + "type": ["null", "string"] + }, + "paddingBottom": { + "type": ["null", "string"] + }, + "paddingTop": { + "type": ["null", "string"] + } + } + } + } + } + ] + } + } + } + } + } + }, + "emailCampaignGroupId": { + "type": ["null", "integer"] + }, + "layoutSections": { + "type": ["null", "object"] + }, + "blogRssSettings": { + "type": ["null", "string"] + }, + "archivedInDashboard": { + "type": ["null", "boolean"] + }, + "publishedAt": { + "type": ["null", "integer"] + }, + "lastEditUpdateId": { + "type": ["null", "integer"] + }, + "lastEditSessionId": { + "type": ["null", "integer"] + }, + "styleSettings": { + "type": ["null", "object"], + "properties": { + "background_color": { "type": ["null", "string"] }, + "background_image": { "type": ["null", "string"] }, + "background_image_type": { "type": ["null", "string"] }, + "body_border_color": { "type": ["null", "string"] }, + "body_border_color_choice": { "type": ["null", "string"] }, + "body_border_width": { "type": ["null", "string"] }, + "body_color": { "type": ["null", "string"] }, + "color_picker_favorite1": { "type": ["null", "string"] }, + "color_picker_favorite2": { "type": ["null", "string"] }, + "color_picker_favorite3": { "type": ["null", "string"] }, + "color_picker_favorite4": { "type": ["null", "string"] }, + "color_picker_favorite5": { "type": ["null", "string"] }, + "color_picker_favorite6": { "type": ["null", "string"] }, + "email_body_padding": { "type": ["null", "string"] }, + "email_body_width": { "type": ["null", "string"] }, + "heading_one_font": { + "properties": { + "bold": { "type": ["null", "string"] }, + "color": { "type": ["null", "string"] }, + "font": { "type": ["null", "string"] }, + "font_style": { "type": ["null", "object"] }, + "italic": { "type": ["null", "string"] }, + "size": { "type": ["null", "string"] }, + "underline": { "type": ["null", "string"] } + } + } + } + }, + "visibleToAll": { + "type": ["null", "boolean"] + }, + "language": { + "type": ["null", "string"] + }, + "rssEmailByText": { + "type": ["null", "string"] + }, + "rssEmailCommentText": { + "type": ["null", "string"] + }, + "hasContentAccessRules": { + "type": ["null", "boolean"] + }, + "archivedAt": { + "type": ["null", "integer"] + }, + "translations": { + "type": ["null", "object"] + }, + "userPerms": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "contentAccessRuleIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "rssEmailEntryTemplateEnabled": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json new file mode 100644 index 000000000000..15150ec585d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners_archived.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "userId": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "membership": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json index 97fd5c1e20db..de0f9bbe9c86 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json @@ -28,6 +28,12 @@ }, "vid": { "type": ["null", "integer"] + }, + "source-vids": { + "type": ["array", "null"], + "items": { + "type": ["null", "integer"] + } } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json index 3e3fd857cba3..7d8da5c6f09c 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json @@ -11,6 +11,9 @@ "recipient": { "type": ["null", "string"] }, + "normalizedEmailId": { + "type": ["null", "string"] + }, "changes": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json index e74dc6c7d192..4d89d65055cb 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json @@ -42,6 +42,99 @@ } } } + }, + "lastUpdatedByUserId": { + "type": ["null", "integer"] + }, + "contactCounts": { + "type": ["null", "object"], + "properties": { + "active": { + "type": ["null", "integer"] + }, + "enrolled": { + "type": ["null", "integer"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "originalAuthorUserId": { + "type": ["null", "integer"] + }, + "migrationStatus": { + "type": ["null", "object"], + "properties": { + "enrollmentMigrationStatus": { + "type": ["null", "string"] + }, + "enrollmentMigrationTimestamp": { + "type": ["null", "integer"] + }, + "flowId": { + "type": ["null", "integer"] + }, + "lastSuccessfulMigrationTimestamp": { + "type": ["null", "integer"] + }, + "migrationStatus": { + "type": ["null", "string"] + }, + "platformOwnsActions": { + "type": ["null", "boolean"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "workflowId": { + "type": ["null", "integer"] + } + } + }, + "updateSource": { + "type": ["null", "object"], + "properties": { + "sourceApplication": { + "properties": { + "serviceName": { "type": ["null", "string"] }, + "source": { "type": ["null", "string"] } + } + }, + "updatedAt": { "type": ["null", "integer"] }, + "updatedByUser": { + "properties": { + "userEmail": { "type": ["null", "string"] }, + "userId": { "type": ["null", "integer"] } + } + } + } + }, + "creationSource": { + "type": ["null", "object"], + "properties": { + "clonedFromWorkflowId": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "integer"] + }, + "createdByUser": { + "properties": { + "userEmail": { "type": ["null", "string"] }, + "userId": { "type": ["null", "integer"] } + } + }, + "sourceApplication": { + "properties": { + "serviceName": { "type": ["null", "string"] }, + "source": { "type": ["null", "string"] } + } + } + } + }, + "portalId": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index f4e8233bf09f..2d1c6a642dca 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -3,6 +3,7 @@ # import logging +from http import HTTPStatus from itertools import chain from typing import Any, List, Mapping, Optional, Tuple @@ -11,6 +12,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from requests import HTTPError +from source_hubspot.errors import HubspotInvalidAuth from source_hubspot.streams import ( API, Campaigns, @@ -18,6 +20,7 @@ ContactLists, Contacts, ContactsListMemberships, + ContactsMergedAudit, CustomObject, DealPipelines, Deals, @@ -36,6 +39,7 @@ LineItems, MarketingEmails, Owners, + OwnersArchived, Products, PropertyHistory, SubscriptionChanges, @@ -59,6 +63,12 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> except HTTPError as error: alive = False error_msg = repr(error) + if error.response.status_code == HTTPStatus.BAD_REQUEST: + response_json = error.response.json() + error_msg = f"400 Bad Request: {response_json['message']}, please check if provided credentials are valid." + except HubspotInvalidAuth as e: + alive = False + error_msg = repr(e) return alive, error_msg def get_granted_scopes(self, authenticator): @@ -93,6 +103,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ContactLists(**common_params), Contacts(**common_params), ContactsListMemberships(**common_params), + ContactsMergedAudit(**common_params), DealPipelines(**common_params), Deals(**common_params), DealsArchived(**common_params), @@ -110,6 +121,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: LineItems(**common_params), MarketingEmails(**common_params), Owners(**common_params), + OwnersArchived(**common_params), Products(**common_params), PropertyHistory(**common_params), SubscriptionChanges(**common_params), @@ -142,8 +154,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return available_streams def get_custom_object_streams(self, api: API, common_params: Mapping[str, Any]): - schemas = api.get_custom_object_schemas() - streams = [] - for entity, schema in schemas.items(): - streams.append(CustomObject(entity=entity, schema=schema, **common_params)) - return streams + for (entity, fully_qualified_name, schema) in api.get_custom_objects_metadata(): + yield CustomObject(entity=entity, schema=schema, fully_qualified_name=fully_qualified_name, **common_params) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index ba6e8bb667e3..d272bff24316 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -7,6 +7,7 @@ import sys import time from abc import ABC, abstractmethod +from datetime import timedelta from functools import cached_property, lru_cache from http import HTTPStatus from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union @@ -15,7 +16,8 @@ import pendulum as pendulum import requests from airbyte_cdk.entrypoint import logger -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import FailureType +from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy @@ -24,6 +26,7 @@ from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError, codes from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout, InvalidStartDateConfigError @@ -61,6 +64,16 @@ CUSTOM_FIELD_VALUE_TO_TYPE = {v: k for k, v in CUSTOM_FIELD_TYPE_TO_VALUE.items()} +def retry_token_expired_handler(**kwargs): + """Retry helper when token expired""" + + return backoff.on_exception( + backoff.expo, + HubspotInvalidAuth, + **kwargs, + ) + + def retry_connection_handler(**kwargs): """Retry helper, log each attempt""" @@ -122,7 +135,7 @@ def check_availability(self, stream: Stream, logger: logging.Logger, source: Opt class API: - """HubSpot API interface, authorize, retrieve and post, supports backoff logic""" + """HubSpot API interface, authorize, retrieve and post, supports backoff logic.""" BASE_URL = "https://api.hubapi.com" USER_AGENT = "Airbyte" @@ -172,20 +185,26 @@ def _parse_and_handle_errors(response) -> Union[MutableMapping[str, Any], List[M if response.headers.get("content-type") == "application/json;charset=utf-8" and response.status_code != HTTPStatus.OK: message = response.json().get("message") - if response.status_code == HTTPStatus.FORBIDDEN: - """Once hit the forbidden endpoint, we return the error message from response.""" - pass + if response.status_code == HTTPStatus.BAD_REQUEST: + message = f"Request to {response.url} didn't succeed. Please verify your credentials and try again.\nError message from Hubspot API: {message}" + logger.warning(message) + elif response.status_code == HTTPStatus.FORBIDDEN: + message = f"The authenticated user does not have permissions to access the URL {response.url}. Verify your permissions to access this endpoint." + logger.warning(message) elif response.status_code in (HTTPStatus.UNAUTHORIZED, CLOUDFLARE_ORIGIN_DNS_ERROR): - raise HubspotInvalidAuth(message, response=response) + message = ( + "The user cannot be authorized with provided credentials. Please verify that your credentails are valid and try again." + ) + raise HubspotInvalidAuth(internal_message=message, failure_type=FailureType.config_error, response=response) elif response.status_code == HTTPStatus.TOO_MANY_REQUESTS: retry_after = response.headers.get("Retry-After") + message = f"You have reached your Hubspot API limit. We will resume replication once after {retry_after} seconds.\nSee https://developers.hubspot.com/docs/api/usage-details" raise HubspotRateLimited( - f"429 Rate Limit Exceeded: API rate-limit has been reached until {retry_after} seconds." - " See https://developers.hubspot.com/docs/api/usage-details", + message, response=response, ) elif response.status_code in (HTTPStatus.BAD_GATEWAY, HTTPStatus.SERVICE_UNAVAILABLE): - raise HubspotTimeout(message, response=response) + raise HubspotTimeout(message, response) else: response.raise_for_status() @@ -205,17 +224,15 @@ def post( response = self._session.post(self.BASE_URL + url, params=params, json=data) return self._parse_and_handle_errors(response), response - def get_custom_object_schemas(self) -> Mapping[str, Any]: + def get_custom_objects_metadata(self) -> Iterable[Tuple[str, str, Mapping[str, Any]]]: data, response = self.get("/crm/v3/schemas", {}) - schemas = {} - if response.ok and "results" in data: - for raw_schema in data["results"]: - schema = self.generate_schema(raw_schema=raw_schema) - schemas[raw_schema["name"]] = schema - else: + if not response.ok or "results" not in data: self.logger.warn(self._parse_and_handle_errors(response)) + return () - return schemas + return ( + (metadata["name"], metadata["fullyQualifiedName"], self.generate_schema(raw_schema=metadata)) for metadata in data["results"] + ) def generate_schema(self, raw_schema: Mapping[str, Any]) -> Mapping[str, Any]: properties = {} @@ -328,6 +345,12 @@ def __init__(self, api: API, start_date: Union[str, pendulum.datetime], credenti if creds_title in (OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS): self._authenticator = api.get_authenticator() + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == HTTPStatus.UNAUTHORIZED: + message = response.json().get("message") + raise HubspotInvalidAuth(message, response=response) + return super().should_retry(response) + def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == codes.too_many_requests: return float(response.headers.get("Retry-After", 3)) @@ -346,6 +369,7 @@ def get_json_schema(self) -> Mapping[str, Any]: json_schema["properties"]["properties"] = {"type": "object", "properties": self.properties} return json_schema + @retry_token_expired_handler(max_tries=5) def handle_request( self, stream_slice: Mapping[str, Any] = None, @@ -446,7 +470,11 @@ def read_records( # Always return an empty generator just in case no records were ever yielded yield from [] except requests.exceptions.HTTPError as e: - raise e + response = e.response + if response.status_code == HTTPStatus.UNAUTHORIZED: + raise AirbyteTracedException("The authentication to HubSpot has expired. Re-authenticate to restore access to HubSpot.") + else: + raise e def parse_response_error_message(self, response: requests.Response) -> Optional[str]: try: @@ -984,10 +1012,12 @@ class CRMSearchStream(IncrementalStream, ABC): updated_at_field = "updatedAt" last_modified_field: str = None associations: List[str] = None + fully_qualified_name: str = None @property def url(self): - return f"/crm/v3/objects/{self.entity}/search" if self.state else f"/crm/v3/objects/{self.entity}" + object_type_id = self.fully_qualified_name or self.entity + return f"/crm/v3/objects/{object_type_id}/search" if self.state else f"/crm/v3/objects/{object_type_id}" def __init__( self, @@ -1253,6 +1283,7 @@ class ContactLists(IncrementalStream): updated_at_field = "updatedAt" created_at_field = "createdAt" limit_field = "count" + primary_key = "listId" need_chunk = False scopes = {"crm.lists.read"} @@ -1392,25 +1423,13 @@ class EmailEvents(IncrementalStream): scopes = {"content"} -class Engagements(IncrementalStream): - """Engagements, API v1 - Docs: https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements - https://legacydocs.hubspot.com/docs/methods/engagements/get-recent-engagements - """ - - url = "/engagements/v1/engagements/paged" +class EngagementsABC(Stream, ABC): more_key = "hasMore" updated_at_field = "lastUpdated" created_at_field = "createdAt" primary_key = "id" scopes = {"crm.objects.companies.read", "crm.objects.contacts.read", "crm.objects.deals.read", "tickets", "e-commerce"} - @property - def url(self): - if self.state: - return "/engagements/v1/engagements/recent/modified" - return "/engagements/v1/engagements/paged" - def _transform(self, records: Iterable) -> Iterable: yield from super()._transform({**record.pop("engagement"), **record} for record in records) @@ -1423,16 +1442,109 @@ def request_params( params = {"count": 250} if next_page_token: params["offset"] = next_page_token["offset"] - if self.state: - params.update({"since": int(self._state.timestamp() * 1000), "count": 100}) return params + +class EngagementsAll(EngagementsABC): + """All Engagements API: + https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements + + Note: Returns all engagements records ordered by 'createdAt' (not 'lastUpdated') field + """ + + @property + def url(self): + return "/engagements/v1/engagements/paged" + + +class EngagementsRecentError(Exception): + pass + + +class EngagementsRecent(EngagementsABC): + """Recent Engagements API: + https://legacydocs.hubspot.com/docs/methods/engagements/get-recent-engagements + + Get the most recently created or updated engagements in a portal, sorted by when they were last updated, + with the most recently updated engagements first. + + Important: This endpoint returns only last 10k most recently updated records in the last 30 days. + """ + + total_records_limit = 10000 + last_days_limit = 29 + + @property + def url(self): + return "/engagements/v1/engagements/recent/modified" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self._start_date < pendulum.now() - timedelta(days=self.last_days_limit): + raise EngagementsRecentError( + '"Recent engagements" API returns records updated in the last 30 days only. ' + f'Start date {self._start_date} is older so "All engagements" API should be used' + ) + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params.update( + { + "since": int(self._start_date.timestamp() * 1000), + "count": 100, + } + ) + return params + + def parse_response( + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + # Check if "Recent engagements" API is applicable for use + response_info = response.json() + if response_info: + total = response_info.get("total") + if total > self.total_records_limit: + yield from [] + raise EngagementsRecentError( + '"Recent engagements" API returns only 10k most recently updated records. ' + 'API response indicates that there are more records so "All engagements" API should be used' + ) + yield from super().parse_response(response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + + +class Engagements(EngagementsABC, IncrementalStream): + """Engagements stream does not send requests directly, instead it uses: + - EngagementsRecent if start_date/state is less than 30 days and API is able to return all records (<10k), or + - EngagementsAll which extracts all records, but supports filter on connector side + """ + + @property + def url(self): + return "/engagements/v1/engagements/paged" + def stream_slices( self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: self.set_sync(sync_mode) return [None] + def process_records(self, records: Iterable[Mapping[str, Any]]) -> Iterable[Mapping[str, Any]]: + """Process each record to find latest cursor value""" + for record in records: + cursor = self._field_to_datetime(record[self.updated_at_field]) + self.latest_cursor = max(cursor, self.latest_cursor) if self.latest_cursor else cursor + yield record + def read_records( self, sync_mode: SyncMode, @@ -1440,40 +1552,39 @@ def read_records( stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - pagination_complete = False + self.latest_cursor = None - next_page_token = None - latest_cursor = None - - while not pagination_complete: - response = self.handle_request(stream_slice=stream_slice, stream_state=stream_state, next_page_token=next_page_token) - records = self._transform(self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)) - - if self.filter_old_records: - records = self._filter_old_records(records) - - for record in records: - cursor = self._field_to_datetime(record[self.updated_at_field]) - latest_cursor = max(cursor, latest_cursor) if latest_cursor else cursor - yield record - - next_page_token = self.next_page_token(response) - if self.state and next_page_token and next_page_token["offset"] >= 10000: - # As per Hubspot documentation, the recent engagements endpoint will only return the 10K - # most recently updated engagements. Since they are returned sorted by `lastUpdated` in - # descending order, we stop getting records if we have already reached 10,000. Attempting - # to get more than 10K will result in a HTTP 400 error. - # https://legacydocs.hubspot.com/docs/methods/engagements/get-recent-engagements - next_page_token = None - - if not next_page_token: - pagination_complete = True + # The date we need records since + since_date = self._start_date + if stream_state: + since_date_timestamp = stream_state.get(self.updated_at_field) + if since_date_timestamp: + since_date = pendulum.from_timestamp(int(since_date_timestamp) / 1000) + + stream_params = { + "api": self._api, + "start_date": since_date, + "credentials": self._credentials, + } - # Always return an empty generator just in case no records were ever yielded - yield from [] + try: + # Try 'Recent' API first, since it is more efficient + records = EngagementsRecent(**stream_params).read_records(sync_mode.full_refresh, cursor_field) + yield from self.process_records(records) + except EngagementsRecentError as e: + # if 'Recent' API in not applicable and raises the error + # then use 'All' API which returns all records, which are filtered on connector side + self.logger.info(e) + records = EngagementsAll(**stream_params).read_records(sync_mode.full_refresh, cursor_field) + yield from self.process_records(records) + + # State should be updated only once at the end of the sync + # because records are not ordered in ascending by 'lastUpdated' field + self._update_state(latest_cursor=self.latest_cursor, is_last_record=True) - self._update_state(latest_cursor=latest_cursor, is_last_record=True) + def _transform(self, records: Iterable) -> Iterable: + # transformation is not needed, because it was done in a substream + yield from records class Forms(ClientSideIncrementalStream): @@ -1581,6 +1692,27 @@ class Owners(ClientSideIncrementalStream): scopes = {"crm.objects.owners.read"} +class OwnersArchived(ClientSideIncrementalStream): + """Archived Owners, API v3""" + + url = "/crm/v3/owners" + updated_at_field = "updatedAt" + created_at_field = "createdAt" + cursor_field_datetime_format = "YYYY-MM-DDTHH:mm:ss.SSSSSSZ" + primary_key = "id" + scopes = {"crm.objects.owners.read"} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params["archived"] = "true" + return params + + class PropertyHistory(Stream): """Contacts Endpoint, API v1 Is used to get all Contacts and the history of their respective @@ -1596,6 +1728,7 @@ class PropertyHistory(Stream): data_field = "contacts" page_field = "vid-offset" page_filter = "vidOffset" + primary_key = "vid" denormalize_records = True limit_field = "count" limit = 100 @@ -1676,6 +1809,72 @@ class Contacts(CRMSearchStream): scopes = {"crm.objects.contacts.read"} +class ContactsMergedAudit(Stream): + url = "/contacts/v1/contact/vids/batch/" + updated_at_field = "timestamp" + scopes = {"crm.objects.contacts.read"} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.config = kwargs + + def get_json_schema(self) -> Mapping[str, Any]: + """Override get_json_schema defined in Stream class + Final object does not have properties field + We return JSON schema as defined in : + source_hubspot/schemas/contacts_merged_audit.json + """ + return super(Stream, self).get_json_schema() + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + slices = [] + + # we can query a max of 100 contacts at a time + max_contacts = 100 + slices = [] + contact_batch = [] + + contacts = Contacts(**self.config) + contacts._sync_mode = SyncMode.full_refresh + contacts.filter_old_records = False + + for contact in contacts.read_records(sync_mode=SyncMode.full_refresh): + if contact["properties"].get("hs_merged_object_ids"): + contact_batch.append(contact["id"]) + + if len(contact_batch) == max_contacts: + slices.append({"vid": contact_batch}) + contact_batch = [] + + if contact_batch: + slices.append({"vid": contact_batch}) + + return slices + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {"vid": stream_slice["vid"]} + + def parse_response( + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + response = self._parse_response(response) + if response.get("status", None) == "error": + self.logger.warning(f"Stream `{self.name}` cannot be procced. {response.get('message')}") + return + + for contact_id in list(response.keys()): + yield from response[contact_id]["merge-audits"] + + class EngagementsCalls(CRMSearchStream): entity = "calls" last_modified_field = "hs_lastmodifieddate" @@ -1757,20 +1956,24 @@ class CustomObject(CRMSearchStream, ABC): primary_key = "id" scopes = {"crm.schemas.custom.read", "crm.objects.custom.read"} - def __init__(self, entity: str, schema: Mapping[str, Any], **kwargs): + def __init__(self, entity: str, schema: Mapping[str, Any], fully_qualified_name: str, **kwargs): super().__init__(**kwargs) self.entity = entity self.schema = schema + self.fully_qualified_name = fully_qualified_name @property def name(self) -> str: return self.entity def get_json_schema(self) -> Mapping[str, Any]: - if not self.schema: - self.schema = self._api.get_custom_object_schemas()[self.entity] return self.schema + @property + def properties(self) -> Mapping[str, Any]: + # do not make extra api queries + return self.get_json_schema()["properties"]["properties"]["properties"] + class EmailSubscriptions(Stream): """EMAIL SUBSCRIPTION, API v1 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index a4cde6bb62eb..0c4ff9c8f613 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -32,6 +32,11 @@ def common_params_fixture(config): return common_params +@pytest.fixture(name="config_invalid_client_id") +def config_invalid_client_id_fixture(): + return {"start_date": "2021-01-10T00:00:00Z", "credentials": {"credentials_title": "OAuth Credentials", "client_id": "invalid_client_id", "client_secret": "invalid_client_secret", "access_token": "test_access_token", "refresh_token": "test_refresh_token"}} + + @pytest.fixture(name="config") def config_fixture(): return {"start_date": "2021-01-10T00:00:00Z", "credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"}} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_handle_retry_token_expired.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_handle_retry_token_expired.py new file mode 100644 index 000000000000..1b6d2cc6c4ea --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_handle_retry_token_expired.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from unittest.mock import patch + +import pytest +import requests +from source_hubspot.errors import HubspotInvalidAuth +from source_hubspot.streams import Stream + + +# Define a mock function to be used with backoff.on_exception +def mock_retry_func(*args, **kwargs): + # Define the error message + error_message = "Token expired" + + # Create a mock response with a 401 status code + response = requests.Response() + response.status_code = 401 + response._content = json.dumps({"message": error_message}).encode() + + # Raise the exception with the defined message + raise HubspotInvalidAuth(error_message, response=response) + + +@patch.multiple(Stream, __abstractmethods__=set()) +def test_handle_request_with_retry(common_params): + # Create a mock instance of the Stream class + stream_instance = Stream(**common_params) + + # Create a mock response + mock_response = requests.Response() + mock_response.status_code = 200 + mock_response._content = json.dumps({"data": "Mocked response"}).encode() + + # Mock the _send_request method of the Stream class to return the mock response + with patch.object(stream_instance, "_send_request", return_value=mock_response): + response = stream_instance.handle_request() + + assert response.status_code == 200 + assert response.json() == {"data": "Mocked response"} + + +@patch.multiple(Stream, __abstractmethods__=set()) +def test_handle_request_with_retry_token_expired(common_params): + # Create a mock instance of the Stream class + stream_instance = Stream(**common_params) + + # Mock the _send_request method of the Stream class to raise HubspotInvalidAuth exception + with patch.object(stream_instance, "_send_request", side_effect=mock_retry_func) as mocked_send_request: + with pytest.raises(HubspotInvalidAuth): + stream_instance.handle_request() + + assert mocked_send_request.call_count == 5 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 56c2fbf6f110..14a5737ab6c4 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -4,6 +4,7 @@ import logging +from datetime import timedelta from http import HTTPStatus from unittest.mock import MagicMock @@ -62,6 +63,16 @@ def test_check_connection_exception(config): assert error_msg +def test_check_connection_bad_request_exception(requests_mock, config_invalid_client_id): + responses = [ + {"json": {"message": "invalid client_id"}, "status_code": 400}, + ] + requests_mock.register_uri("POST", "/oauth/v1/token", responses) + ok, error_msg = SourceHubspot().check_connection(logger, config=config_invalid_client_id) + assert not ok + assert error_msg + + def test_check_connection_invalid_start_date_exception(config_invalid_date): with pytest.raises(InvalidStartDateConfigError): ok, error_msg = SourceHubspot().check_connection(logger, config=config_invalid_date) @@ -74,7 +85,7 @@ def test_streams(requests_mock, config): streams = SourceHubspot().streams(config) - assert len(streams) == 28 + assert len(streams) == 30 def test_check_credential_title_exception(config): @@ -170,6 +181,7 @@ def test_stream_forbidden(requests_mock, config, caplog): records = list(SourceHubspot().read(logger, config, catalog, {})) assert json["message"] in caplog.text + assert "The authenticated user does not have permissions to access the URL" in caplog.text records = [r for r in records if r.type == Type.RECORD] assert not records @@ -210,6 +222,7 @@ def test_parent_stream_forbidden(requests_mock, config, caplog, fake_properties_ records = list(SourceHubspot().read(logger, config, catalog, {})) assert json["message"] in caplog.text + assert "The authenticated user does not have permissions to access the URL" in caplog.text records = [r for r in records if r.type == Type.RECORD] assert not records @@ -525,36 +538,95 @@ def test_engagements_stream_pagination_works(requests_mock, common_params): test_stream = Engagements(**common_params) records, _ = read_incremental(test_stream, {}) # The stream should handle pagination correctly and output 250 records. - assert len(records) == 250 + assert len(records) == 100 assert test_stream.state["lastUpdated"] == int(test_stream._init_sync.timestamp() * 1000) -def test_incremental_engagements_stream_stops_at_10K_records(requests_mock, common_params, fake_properties_list): +def test_engagements_stream_since_old_date(requests_mock, common_params, fake_properties_list): """ - If there are more than 10,000 engagements that would be returned by the Hubspot recent engagements endpoint, - the Engagements instance should stop at the 10Kth record. + Connector should use 'All Engagements' API for old dates (more than 30 days) """ - + old_date = 1614038400000 # Tuesday, 23 February 2021 г., 0:00:00 responses = [ { "json": { - "results": [{"engagement": {"id": f"{y}", "lastUpdated": 1641234595252}} for y in range(100)], - "hasMore": True, - "offset": x * 100, + "results": [{"engagement": {"id": f"{y}", "lastUpdated": old_date}} for y in range(100)], + "hasMore": False, + "offset": 0, + "total": 100 }, "status_code": 200, } - for x in range(1, 102) ] # Create test_stream instance with some state test_stream = Engagements(**common_params) - test_stream.state = {"lastUpdated": 1641234595251} + test_stream.state = {"lastUpdated": old_date} # Mocking Request - requests_mock.register_uri("GET", "/engagements/v1/engagements/recent/modified?count=100", responses) + requests_mock.register_uri("GET", "/engagements/v1/engagements/paged?count=250", responses) records, _ = read_incremental(test_stream, {}) # The stream should not attempt to get more than 10K records. - assert len(records) == 10000 + assert len(records) == 100 + assert test_stream.state["lastUpdated"] == int(test_stream._init_sync.timestamp() * 1000) + + +def test_engagements_stream_since_recent_date(requests_mock, common_params, fake_properties_list): + """ + Connector should use 'Recent Engagements' API for recent dates (less than 30 days) + """ + recent_date = pendulum.now() - timedelta(days=10) # 10 days ago + recent_date = int(recent_date.timestamp() * 1000) + responses = [ + { + "json": { + "results": [{"engagement": {"id": f"{y}", "lastUpdated": recent_date}} for y in range(100)], + "hasMore": False, + "offset": 0, + "total": 100 + }, + "status_code": 200, + } + ] + + # Create test_stream instance with some state + test_stream = Engagements(**common_params) + test_stream.state = {"lastUpdated": recent_date} + # Mocking Request + requests_mock.register_uri("GET", f"/engagements/v1/engagements/recent/modified?count=100&since={recent_date}", responses) + records, _ = read_incremental(test_stream, {"lastUpdated": recent_date}) + # The stream should not attempt to get more than 10K records. + assert len(records) == 100 + assert test_stream.state["lastUpdated"] == int(test_stream._init_sync.timestamp() * 1000) + + +def test_engagements_stream_since_recent_date_more_than_10k(requests_mock, common_params, fake_properties_list): + """ + Connector should use 'Recent Engagements' API for recent dates (less than 30 days). + If response from 'Recent Engagements' API returns 10k records, it means that there more records, + so 'All Engagements' API should be used. + """ + recent_date = pendulum.now() - timedelta(days=10) # 10 days ago + recent_date = int(recent_date.timestamp() * 1000) + responses = [ + { + "json": { + "results": [{"engagement": {"id": f"{y}", "lastUpdated": recent_date}} for y in range(100)], + "hasMore": False, + "offset": 0, + "total": 10001 + }, + "status_code": 200, + } + ] + + # Create test_stream instance with some state + test_stream = Engagements(**common_params) + test_stream.state = {"lastUpdated": recent_date} + # Mocking Request + requests_mock.register_uri("GET", f"/engagements/v1/engagements/recent/modified?count=100&since={recent_date}", responses) + requests_mock.register_uri("GET", "/engagements/v1/engagements/paged?count=250", responses) + records, _ = read_incremental(test_stream, {"lastUpdated": recent_date}) + assert len(records) == 100 assert test_stream.state["lastUpdated"] == int(test_stream._init_sync.timestamp() * 1000) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 3b9a775906cd..bc7a150474cb 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -10,6 +10,7 @@ Companies, ContactLists, Contacts, + ContactsMergedAudit, CustomObject, DealPipelines, Deals, @@ -27,6 +28,7 @@ LineItems, MarketingEmails, Owners, + OwnersArchived, Products, TicketPipelines, Tickets, @@ -60,7 +62,8 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p properties_response = [ { "json": [ - {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} + {"name": property_name, "type": "string", + "updatedAt": 1571085954360, "createdAt": 1565059306048} for property_name in fake_properties_list ], "status_code": 200, @@ -68,11 +71,13 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p ] requests_mock.register_uri("GET", stream.url, responses) - requests_mock.register_uri("GET", "/properties/v2/contact/properties", properties_response) + requests_mock.register_uri( + "GET", "/properties/v2/contact/properties", properties_response) _, stream_state = read_incremental(stream, {}) - expected = int(pendulum.parse(common_params["start_date"]).timestamp() * 1000) + expected = int(pendulum.parse( + common_params["start_date"]).timestamp() * 1000) assert stream_state[stream.updated_at_field] == expected @@ -84,6 +89,8 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p (Companies, "company", {"updatedAt": "2022-02-25T16:43:11Z"}), (ContactLists, "contact", {"updatedAt": "2022-02-25T16:43:11Z"}), (Contacts, "contact", {"updatedAt": "2022-02-25T16:43:11Z"}), + (ContactsMergedAudit, "contact", { + "updatedAt": "2022-02-25T16:43:11Z"}), (Deals, "deal", {"updatedAt": "2022-02-25T16:43:11Z"}), (DealsArchived, "deal", {"archivedAt": "2022-02-25T16:43:11Z"}), (DealPipelines, "deal", {"updatedAt": 1675121674226}), @@ -91,7 +98,8 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p (EmailSubscriptions, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsCalls, "calls", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsEmails, "emails", {"updatedAt": "2022-02-25T16:43:11Z"}), - (EngagementsMeetings, "meetings", {"updatedAt": "2022-02-25T16:43:11Z"}), + (EngagementsMeetings, "meetings", { + "updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsNotes, "notes", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsTasks, "tasks", {"updatedAt": "2022-02-25T16:43:11Z"}), (Forms, "form", {"updatedAt": "2022-02-25T16:43:11Z"}), @@ -100,6 +108,7 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p (LineItems, "line_item", {"updatedAt": "2022-02-25T16:43:11Z"}), (MarketingEmails, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (Owners, "", {"updatedAt": "2022-02-25T16:43:11Z"}), + (OwnersArchived, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (Products, "product", {"updatedAt": "2022-02-25T16:43:11Z"}), (TicketPipelines, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (Tickets, "ticket", {"updatedAt": "2022-02-25T16:43:11Z"}), @@ -124,21 +133,61 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para properties_response = [ { "json": [ - {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} + {"name": property_name, "type": "string", + "updatedAt": 1571085954360, "createdAt": 1565059306048} for property_name in fake_properties_list ], "status_code": 200, } ] + contact_reponse = [ + { + "json": { + stream.data_field: [ + { + "id": "test_id", + "created": "2022-06-25T16:43:11Z", + "properties": { + "hs_merged_object_ids": "test_id" + } + } + | cursor_value + ], + } + } + ] + read_batch_contact_v1_response = [ + { + "json": { + "test_id": { + "vid": "test_id", + 'merge-audits': [ + { + 'canonical-vid': 2, + 'vid-to-merge': 5608, + 'timestamp': 1653322839932 + } + ] + } + }, + "status_code": 200, + } + ] is_form_submission = isinstance(stream, FormSubmissions) stream._sync_mode = SyncMode.full_refresh stream_url = stream.url + "/test_id" if is_form_submission else stream.url stream._sync_mode = None requests_mock.register_uri("GET", stream_url, responses) + requests_mock.register_uri( + "GET", "/crm/v3/objects/contact", contact_reponse) requests_mock.register_uri("GET", "/marketing/v3/forms", responses) - requests_mock.register_uri("GET", "/email/public/v1/campaigns/test_id", responses) - requests_mock.register_uri("GET", f"/properties/v2/{endpoint}/properties", properties_response) + requests_mock.register_uri( + "GET", "/email/public/v1/campaigns/test_id", responses) + requests_mock.register_uri( + "GET", f"/properties/v2/{endpoint}/properties", properties_response) + requests_mock.register_uri( + "GET", "/contacts/v1/contact/vids/batch/", read_batch_contact_v1_response) records = read_full_refresh(stream) assert records @@ -155,7 +204,8 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para def test_common_error_retry(error_response, requests_mock, common_params, fake_properties_list): """Error once, check that we retry and not fail""" properties_response = [ - {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} + {"name": property_name, "type": "string", + "updatedAt": 1571085954360, "createdAt": 1565059306048} for property_name in fake_properties_list ] responses = [ @@ -178,7 +228,8 @@ def test_common_error_retry(error_response, requests_mock, common_params, fake_p } ], } - requests_mock.register_uri("GET", "/properties/v2/company/properties", responses) + requests_mock.register_uri( + "GET", "/properties/v2/company/properties", responses) stream._sync_mode = SyncMode.full_refresh stream_url = stream.url stream._sync_mode = None @@ -231,9 +282,12 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope { "json": { stream.data_field: [ - {"id": "test_id_1", "createdAt": "2022-03-25T16:43:11Z", "updatedAt": "2023-01-30T23:46:36.287Z"}, - {"id": "test_id_2", "createdAt": "2022-03-25T16:43:11Z", "updatedAt": latest_cursor_value}, - {"id": "test_id_3", "createdAt": "2022-03-25T16:43:11Z", "updatedAt": "2023-02-20T23:46:36.287Z"}, + {"id": "test_id_1", "createdAt": "2022-03-25T16:43:11Z", + "updatedAt": "2023-01-30T23:46:36.287Z"}, + {"id": "test_id_2", "createdAt": "2022-03-25T16:43:11Z", + "updatedAt": latest_cursor_value}, + {"id": "test_id_3", "createdAt": "2022-03-25T16:43:11Z", + "updatedAt": "2023-02-20T23:46:36.287Z"}, ], } } @@ -241,7 +295,8 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope properties_response = [ { "json": [ - {"name": property_name, "type": "string", "createdAt": "2023-01-30T23:46:24.355Z", "updatedAt": "2023-01-30T23:46:36.287Z"} + {"name": property_name, "type": "string", "createdAt": "2023-01-30T23:46:24.355Z", + "updatedAt": "2023-01-30T23:46:36.287Z"} for property_name in fake_properties_list ], "status_code": 200, @@ -249,17 +304,21 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope ] requests_mock.register_uri("GET", stream.url, responses) - requests_mock.register_uri("GET", "/properties/v2/form/properties", properties_response) + requests_mock.register_uri( + "GET", "/properties/v2/form/properties", properties_response) list(stream.read_records(SyncMode.incremental)) - assert stream.state == {stream.cursor_field: pendulum.parse(latest_cursor_value).to_rfc3339_string()} + assert stream.state == {stream.cursor_field: pendulum.parse( + latest_cursor_value).to_rfc3339_string()} @pytest.mark.parametrize( "state, record, expected", [ - ({"updatedAt": ""}, {"id": "test_id_1", "updatedAt": "2023-01-30T23:46:36.287Z"}, (True, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})), - ({"updatedAt": "2023-01-30T23:46:36.287000+00:00"}, {"id": "test_id_1", "updatedAt": "2023-01-29T01:02:03.123Z"}, (False, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})), + ({"updatedAt": ""}, {"id": "test_id_1", "updatedAt": "2023-01-30T23:46:36.287Z"}, + (True, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})), + ({"updatedAt": "2023-01-30T23:46:36.287000+00:00"}, {"id": "test_id_1", + "updatedAt": "2023-01-29T01:02:03.123Z"}, (False, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})), ], ids=[ "Empty Sting in state + new record", @@ -274,14 +333,16 @@ def test_empty_string_in_state(state, record, expected, requests_mock, common_pa properties_response = [ { "json": [ - {"name": property_name, "type": "string", "CreatedAt": "2023-01-30T23:46:24.355Z", "updatedAt": "2023-01-30T23:46:36.287Z"} + {"name": property_name, "type": "string", "CreatedAt": "2023-01-30T23:46:24.355Z", + "updatedAt": "2023-01-30T23:46:36.287Z"} for property_name in fake_properties_list ], "status_code": 200, } ] requests_mock.register_uri("GET", stream.url, json=record) - requests_mock.register_uri("GET", "/properties/v2/form/properties", properties_response) + requests_mock.register_uri( + "GET", "/properties/v2/form/properties", properties_response) # end of mocking `availability strategy` result = stream.filter_by_state(stream.state, record) @@ -348,21 +409,44 @@ def expected_custom_object_json_schema(): def test_custom_object_stream_doesnt_call_hubspot_to_get_json_schema_if_available( requests_mock, custom_object_schema, expected_custom_object_json_schema, common_params ): - stream = CustomObject(entity="animals", schema=expected_custom_object_json_schema, **common_params) + stream = CustomObject(entity="animals", schema=expected_custom_object_json_schema, + fully_qualified_name="p123_animals", **common_params) - adapter = requests_mock.register_uri("GET", "/crm/v3/schemas", [{"json": {"results": [custom_object_schema]}}]) + adapter = requests_mock.register_uri( + "GET", "/crm/v3/schemas", [{"json": {"results": [custom_object_schema]}}]) json_schema = stream.get_json_schema() assert json_schema == expected_custom_object_json_schema assert not adapter.called -def test_custom_object_stream_calls_hubspot_to_get_json_schema( - requests_mock, custom_object_schema, expected_custom_object_json_schema, common_params -): - stream = CustomObject(entity="animals", schema=None, **common_params) +def test_contacts_merged_audit_stream_doesnt_call_hubspot_to_get_json_schema(requests_mock, common_params): + stream = ContactsMergedAudit(**common_params) - adapter = requests_mock.register_uri("GET", "/crm/v3/schemas", [{"json": {"results": [custom_object_schema]}}]) - json_schema = stream.get_json_schema() - assert json_schema == expected_custom_object_json_schema - assert adapter.called + adapter = requests_mock.register_uri( + "GET", + f"/properties/v2/{stream.entity}/properties", + [ + { + "json": [ + { + 'name': 'hs_object_id', + 'label': 'Record ID', + 'type': 'number', + } + ] + } + ] + ) + _ = stream.get_json_schema() + + assert not adapter.called + + +def test_get_custom_objects_metadata_success(requests_mock, custom_object_schema, expected_custom_object_json_schema, api): + requests_mock.register_uri( + "GET", "/crm/v3/schemas", json={"results": [custom_object_schema]}) + for (entity, fully_qualified_name, schema) in api.get_custom_objects_metadata(): + assert entity == "animals" + assert fully_qualified_name == "p19936848_Animal" + assert schema == expected_custom_object_json_schema diff --git a/airbyte-integrations/connectors/source-insightly/metadata.yaml b/airbyte-integrations/connectors/source-insightly/metadata.yaml index 41086effbf1b..06202981ce95 100644 --- a/airbyte-integrations/connectors/source-insightly/metadata.yaml +++ b/airbyte-integrations/connectors/source-insightly/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/insightly tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-insightly/requirements.txt b/airbyte-integrations/connectors/source-insightly/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-insightly/requirements.txt +++ b/airbyte-integrations/connectors/source-insightly/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index ca2e69bed983..a8c4637c1342 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -11,9 +11,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile index 534ae8018452..529b56cc22fb 100644 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ b/airbyte-integrations/connectors/source-instagram/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.8 +LABEL io.airbyte.version=1.0.11 LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml index ed9bfddc65aa..295efd229c13 100644 --- a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml @@ -59,6 +59,13 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + user_lifetime_insights: + - name: value + bypass_reason: Floating values from sync-to-sync, due to live updating info. + user_insights: + - name: profile_views + bypass_reason: Floating values from sync-to-sync, due to live updating info. incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json index 522a05ab1cae..81d9934b99b8 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json @@ -18,6 +18,20 @@ "description": "The value of the access token generated with instagram_basic, instagram_manage_insights, pages_show_list, pages_read_engagement, Instagram Public Content Access permissions. See the docs for more information", "airbyte_secret": true, "type": "string" + }, + "client_id": { + "title": "Client Id", + "description": "The Client ID for your Oauth application", + "airbyte_secret": true, + "airbyte_hidden": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The Client Secret for your Oauth application", + "airbyte_secret": true, + "airbyte_hidden": true, + "type": "string" } }, "required": ["start_date", "access_token"] @@ -32,9 +46,7 @@ "properties": { "access_token": { "type": "string", - "path_in_connector_config": [ - "access_token" - ] + "path_in_connector_config": ["access_token"] } } }, @@ -54,15 +66,11 @@ "properties": { "client_id": { "type": "string", - "path_in_connector_config": [ - "client_id" - ] + "path_in_connector_config": ["client_id"] }, "client_secret": { "type": "string", - "path_in_connector_config": [ - "client_secret" - ] + "path_in_connector_config": ["client_secret"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 69c082ce3904..d15bd17a275f 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.8 + dockerImageTag: 1.0.11 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/instagram tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-instagram/requirements.txt b/airbyte-integrations/connectors/source-instagram/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-instagram/requirements.txt +++ b/airbyte-integrations/connectors/source-instagram/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-instagram/sample_files/sample_state.json b/airbyte-integrations/connectors/source-instagram/sample_files/sample_state.json new file mode 100644 index 000000000000..0a523d00ca64 --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/sample_files/sample_state.json @@ -0,0 +1,15 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "17841400000000000": { + "date": "2023-06-28T07:00:00+00:00" + } + }, + "stream_descriptor": { + "name": "user_insights" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-instagram/setup.py b/airbyte-integrations/connectors/source-instagram/setup.py index 9d2bc14a535f..a15d5f7ab25e 100644 --- a/airbyte-integrations/connectors/source-instagram/setup.py +++ b/airbyte-integrations/connectors/source-instagram/setup.py @@ -12,6 +12,7 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8", diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index c52faab11197..6f78da2614c7 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -3,7 +3,7 @@ # from datetime import datetime -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, Optional, Tuple from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, DestinationSyncMode, OAuthConfigSpecification from airbyte_cdk.sources import AbstractSource @@ -34,6 +34,18 @@ class Config: airbyte_secret=True, ) + client_id: Optional[str] = Field( + description=("The Client ID for your Oauth application"), + airbyte_secret=True, + airbyte_hidden=True, + ) + + client_secret: Optional[str] = Field( + description=("The Client Secret for your Oauth application"), + airbyte_secret=True, + airbyte_hidden=True, + ) + class SourceInstagram(AbstractSource): def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any]: diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 0a90d9be9360..72239476831f 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -10,7 +10,7 @@ import pendulum from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams import IncrementalMixin, Stream from cached_property import cached_property from facebook_business.adobjects.igmedia import IGMedia from facebook_business.exceptions import FacebookRequestError @@ -76,12 +76,32 @@ def _clear_url(record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: return record -class InstagramIncrementalStream(InstagramStream, ABC): +class InstagramIncrementalStream(InstagramStream, IncrementalMixin): """Base class for incremental streams""" def __init__(self, start_date: datetime, **kwargs): super().__init__(**kwargs) self._start_date = pendulum.instance(start_date) + self._state = {} + + @property + def state(self): + return self._state + + @state.setter + def state(self, value: Mapping[str, Any]): + self._state.update(**value) + + def _update_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + """Update stream state from latest record""" + # if there is no `end_date` value take the `start_date` + record_value = latest_record.get(self.cursor_field) or self._start_date.to_iso8601_string() + account_id = latest_record.get("business_account_id") + state_value = current_stream_state.get(account_id, {}).get(self.cursor_field) or record_value + max_cursor = max(pendulum.parse(state_value), pendulum.parse(record_value)) + new_stream_state = copy.deepcopy(current_stream_state) + new_stream_state[account_id] = {self.cursor_field: str(max_cursor)} + return new_stream_state class Users(InstagramStream): @@ -122,8 +142,8 @@ def read_records( "page_id": account["page_id"], "business_account_id": ig_account.get("id"), "metric": insight["name"], - "date": insight["values"][0]["end_time"], - "value": insight["values"][0]["value"], + "date": insight["values"][0].get("end_time"), + "value": insight["values"][0].get("value"), } def request_params( @@ -218,8 +238,15 @@ def read_records( f"temporarily unavailable data." ) self.should_exit_gracefully = True + yield from complete_records + # update state using IncrementalMixin + # reference issue: https://github.com/airbytehq/airbyte/issues/24697 + if sync_mode == SyncMode.incremental and complete_records: + for record in complete_records: + self.state = self._update_state(self.state, record) + def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: @@ -267,20 +294,6 @@ def _state_has_legacy_format(self, state: Mapping[str, Any]) -> bool: return True return False - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - """Update stream state from latest record""" - record_value = latest_record[self.cursor_field] - account_id = latest_record.get("business_account_id") - state_value = current_stream_state.get(account_id, {}).get(self.cursor_field) or record_value - max_cursor = max(pendulum.parse(state_value), pendulum.parse(record_value)) - - new_stream_state = copy.deepcopy(current_stream_state) - new_stream_state[account_id] = { - self.cursor_field: str(max_cursor), - } - - return new_stream_state - class Media(InstagramStream): """Children objects can only be of the media_type == "CAROUSEL_ALBUM". diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index c54b1ed96826..cbe45c836813 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import List + from facebook_business import FacebookAdsApi, FacebookSession from pytest import fixture from source_instagram.api import InstagramAPI as API @@ -92,6 +94,72 @@ def user_insight_data_fixture(): } +@fixture(name="user_lifetime_insights") +def user_lifetime_insights(): + class UserLiftimeInsightEntityMock: + def __new__(cls, values: dict): + cls.insights = [ + # No `end_time` Key, reference to this issue: + # https://github.com/airbytehq/airbyte/issues/22929 + { + "description": "Test_no_end_time", + "id": "18/insights/audience_gender_age/lifetime", + "name": "audience_gender_age", + "period": "lifetime", + "title": "Gender and Age", + "values": [values], + }, + ] + + @classmethod + def get(cls, element): + for insight in cls.insights: + return insight[element] + + @classmethod + def get_insights(cls, **kwargs) -> List[dict]: + return cls.insights + + return UserLiftimeInsightEntityMock + + +@fixture(name="user_insights") +def user_insights(): + class UserInsightEntityMock: + # reference Issue: + # https://github.com/airbytehq/airbyte/issues/24697 + class UserInsight: + + def __init__(self, values: dict): + self.data = { + "description": "test_insight", + "id": "123", + "name": "test_insight_metric", + "period": "day", + "title": "Test Insight", + "values": [values], + } + + def __dict__(self): + return self.data + + def export_all_data(self): + return self.__dict__() + + def __new__(cls, values: dict): + cls.insights = [cls.UserInsight(values)] + + @classmethod + def get(cls, element): + return cls.insights[0].__dict__()[element] + + @classmethod + def get_insights(cls, **kwargs) -> List[dict]: + return cls.insights + + return UserInsightEntityMock + + @fixture(name="user_stories_data") def user_stories_data_fixture(): return {"id": "test_id"} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 81bdff81e4d8..7d016c67f483 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import pytest +from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession from source_instagram.streams import ( InstagramStream, @@ -187,10 +188,92 @@ def test_user_lifetime_insights_read(api, config, user_insight_data, requests_mo ] +@pytest.mark.parametrize( + "values,expected", + [ + ({"end_time": "test_end_time", "value": "test_value"}, {"date": "test_end_time", "value": "test_value"}), + ({"value": "test_value"}, {"date": None, "value": "test_value"}), + ({"end_time": "test_end_time"}, {"date": "test_end_time", "value": None}), + ({}, {"date": None, "value": None}), + ], + ids=[ + "`end_time` and `value` are present", + "no `end_time`, but `value` is present", + "`end_time` is present, but no `value`", + "no `end_time` and no `value`", + ] +) +def test_user_lifetime_insights_read_with_missing_keys(api, user_lifetime_insights, values, expected): + """ + This tests shows the behaviour of the `read_records` when either `end_time` or `value` key is not present in the data. + """ + stream = UserLifetimeInsights(api=api) + user_lifetime_insights(values) + test_slice = {"account": {"page_id": 1, "instagram_business_account": user_lifetime_insights}} + for insight in stream.read_records(sync_mode=None, stream_slice=test_slice): + assert insight["date"] == expected.get("date") + assert insight["value"] == expected.get("value") + + +@pytest.mark.parametrize( + "values,slice_dates,expected", + [ + # the state is updated to the value of `end_time` + ( + {"end_time": "2023-01-01 00:01:00", "value": {"one": 1}}, + {"since": "2023-01-01 00:01:00", "until": "2023-01-02 00:01:00"}, + {"123": {"date": "2023-01-01T00:01:00+00:00"}}, + ), + # the state is taken from `start_date` + ( + {"end_time": None, "value": {"two": 2}}, + {"since": "2023-01-01 00:00:00", "until": "2023-01-02 00:01:00"}, + # state from `start_date` is expected + {"123": {"date": "2023-01-01T01:01:01+00:00"}}, + ), + # the state is updated to the value of `end_time` + ( + {"end_time": "2023-02-02 00:02:00", "value": None}, + {"since": "2023-06-01 21:00:00", "until": "2023-06-02 22:00:00"}, + {"123": {"date": "2023-02-02T00:02:00+00:00"}}, + ), + # the state is taken from `start_date` + ( + {"end_time": None, "value": None}, + {"since": "2023-06-01 21:00:00", "until": "2023-06-02 22:00:00"}, + {"123": {"date": "2023-01-01T01:01:01+00:00"}}, + ), + ], + ids=[ + "Normal state flow", + "No `end_time` value in record", + "No `value` in record", + "No `end_time` and no `value` in record", + ] +) +def test_user_insights_state(api, user_insights, values, slice_dates, expected): + """ + This test shows how `STATE` is managed based on the scenario for Incremental Read. + """ + import pendulum + + # UserInsights stream + stream = UserInsights(api=api, start_date=pendulum.parse("2023-01-01T01:01:01Z")) + # Populate the fixute with `values` + user_insights(values) + # simulate `read_recods` generator job + list( + stream.read_records( + sync_mode=SyncMode.incremental, + stream_slice={"account": {"page_id": 1, "instagram_business_account": user_insights}, **slice_dates}, + ) + ) + assert stream.state == expected + + def test_stories_read(api, requests_mock, user_stories_data): test_id = "test_id" stream = Stories(api=api) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/stories", [{"json": user_stories_data}]) records = read_full_refresh(stream) diff --git a/airbyte-integrations/connectors/source-instatus/metadata.yaml b/airbyte-integrations/connectors/source-instatus/metadata.yaml index f60b0803ba29..67980e6c3fb3 100644 --- a/airbyte-integrations/connectors/source-instatus/metadata.yaml +++ b/airbyte-integrations/connectors/source-instatus/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-instatus/requirements.txt b/airbyte-integrations/connectors/source-instatus/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-instatus/requirements.txt +++ b/airbyte-integrations/connectors/source-instatus/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-instatus/setup.py b/airbyte-integrations/connectors/source-instatus/setup.py index 30f26a8fad32..0a0586841809 100644 --- a/airbyte-integrations/connectors/source-instatus/setup.py +++ b/airbyte-integrations/connectors/source-instatus/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incident_updates.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incident_updates.json index 543f436c5904..7b602e1726b7 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incident_updates.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incident_updates.json @@ -2,45 +2,51 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": {"type": ["null", "string"]}, - "message": {"type": ["null", "string"]}, - "messageHtml": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "notify": {"type": ["null", "boolean"]}, - "started": {"type": ["null", "string"], "format": "date-time"}, + "id": { "type": ["null", "string"] }, + "message": { "type": ["null", "string"] }, + "messageHtml": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "notify": { "type": ["null", "boolean"] }, + "started": { "type": ["null", "string"], "format": "date-time" }, "incident": { "type": "object", "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "started": {"type": ["null", "string"], "format": "date-time"}, - "status": {"type": ["null", "string"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "started": { "type": ["null", "string"], "format": "date-time" }, + "status": { "type": ["null", "string"] }, "components": { "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "showUptime": {"type": ["null", "boolean"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] }, "site": { "type": ["null", "object"], "properties": { - "subdomain": {"type": ["null", "string"]}, - "logoUrl": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "slackIntegrations": {"type": ["null", "array"]}, - "subscribers": {"type": ["null", "array"], "items" : {"type" : ["null", "object"], "properties" : { - "name": {"type" : ["null", "string"]} - }}} + "subdomain": { "type": ["null", "string"] }, + "logoUrl": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "slackIntegrations": { "type": ["null", "array"] }, + "subscribers": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] } + } + } + } } }, "subscribers": { "type": ["null", "array"], "items": { "type": ["null", "object"], - "properties": {"name": {"type": ["null", "object"]}} + "properties": { "name": { "type": ["null", "object"] } } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incidents.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incidents.json index 66a798b8c42d..6534ad05d76e 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incidents.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/incidents.json @@ -27,40 +27,40 @@ "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "message": {"type": ["null", "string"]}, - "messageHtml": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "notify": {"type": ["null", "boolean"]}, - "started": {"type": ["null", "string"], "format": "date-time"}, - "ended": {"type": ["null", "string"], "format": "date-time"}, - "duration": {"type": ["null", "integer"]}, - "createdAt": {"type": ["null", "string"], "format": "date-time"} - } + "id": { "type": ["null", "string"] }, + "message": { "type": ["null", "string"] }, + "messageHtml": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "notify": { "type": ["null", "boolean"] }, + "started": { "type": ["null", "string"], "format": "date-time" }, + "ended": { "type": ["null", "string"], "format": "date-time" }, + "duration": { "type": ["null", "integer"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" } + } } }, "components": { - "type": ["null", "array"], + "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "showUptime": {"type": ["null", "boolean"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] }, "site": { - "type": ["null", "object"], - "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "subdomain": {"type": ["null", "string"]}, - "color": {"type": ["null", "boolean"]}, - "logoUrl": {"type": ["null", "string"]}, - "publicEmail": {"type": ["null", "string"]} + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "subdomain": { "type": ["null", "string"] }, + "color": { "type": ["null", "boolean"] }, + "logoUrl": { "type": ["null", "string"] }, + "publicEmail": { "type": ["null", "string"] } + } } } - } - } + } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenance_updates.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenance_updates.json index d53848835106..46de56425439 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenance_updates.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenance_updates.json @@ -15,7 +15,8 @@ "type": ["null", "boolean"] }, "started": { - "type": ["null", "string"], "format": "date-time" + "type": ["null", "string"], + "format": "date-time" }, "status": { "type": ["null", "string"] @@ -23,28 +24,28 @@ "maintenance": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "start": {"type": ["null", "string"], "format": "date-time"}, - "status": {"type": ["null", "string"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "start": { "type": ["null", "string"], "format": "date-time" }, + "status": { "type": ["null", "string"] }, "components": { "type": ["null", "array"], "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "showUptime": {"type": ["null", "boolean"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] }, "site": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "subdomain": {"type": ["null", "string"]}, - "color": {"type": ["null", "string"]}, - "logoUrl": {"type": ["null", "string"]}, - "publicEmail": {"type": ["null", "string"]} + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "subdomain": { "type": ["null", "string"] }, + "color": { "type": ["null", "string"] }, + "logoUrl": { "type": ["null", "string"] }, + "publicEmail": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenances.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenances.json index 2881f38506ff..a11f481425af 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenances.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/maintenances.json @@ -12,7 +12,8 @@ "type": ["null", "string"] }, "start": { - "type": ["null", "string"], "format": "date-time" + "type": ["null", "string"], + "format": "date-time" }, "duration": { "type": ["null", "integer"] @@ -40,15 +41,15 @@ "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "message": {"type": ["null", "string"]}, - "messageHtml": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "notify": {"type": ["null", "boolean"]}, - "started": {"type": ["null", "string"], "format": "date-time"}, - "ended": {"type": ["null", "string"], "format": "date-time"}, - "duration": {"type": ["null", "integer"]}, - "createdAt": {"type": ["null", "string"], "format": "date-time"} + "id": { "type": ["null", "string"] }, + "message": { "type": ["null", "string"] }, + "messageHtml": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "notify": { "type": ["null", "boolean"] }, + "started": { "type": ["null", "string"], "format": "date-time" }, + "ended": { "type": ["null", "string"], "format": "date-time" }, + "duration": { "type": ["null", "integer"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" } } } }, @@ -57,19 +58,19 @@ "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "showUptime": {"type": ["null", "boolean"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] }, "site": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "subdomain": {"type": ["null", "string"]}, - "color": {"type": ["null", "boolean"]}, - "logoUrl": {"type": ["null", "string"]}, - "publicEmail": {"type": ["null", "string"]} + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "subdomain": { "type": ["null", "string"] }, + "color": { "type": ["null", "boolean"] }, + "logoUrl": { "type": ["null", "string"] }, + "publicEmail": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/metrics.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/metrics.json index 8668924fc235..4c5450170ab0 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/metrics.json @@ -18,19 +18,16 @@ "type": ["null", "string"] }, "data": { - "type" : [ - "null", - "array" - ], - "items": { - "type": ["null", "object"], - "properties": { - "properties": { - "timestamp": {"type": ["null", "integer"]}, - "value": {"type": ["null", "number"]} - } - } - } + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "properties": { + "timestamp": { "type": ["null", "integer"] }, + "value": { "type": ["null", "number"] } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/page_components.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/page_components.json index cf7b10b2747a..390479e6b4a8 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/page_components.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/page_components.json @@ -26,26 +26,26 @@ "group": { "type": ["null", "object"], "properties": { - "closed": {"type": ["null", "boolean"]}, - "createdAt": {"type": ["null", "string"], "format": "date-time"}, - "description": {"type": ["null", "string"]}, - "descriptionHtml": {"type": ["null", "string"]}, - "descriptionTranslationId": {"type": ["null", "string"]}, - "id": {"type": ["null", "string"]}, - "importedFromStatuspage": {"type": ["null", "boolean"]}, - "name": {"type": ["null", "string"]}, - "nameHtml": {"type": ["null", "string"]}, - "nameHtmlTranslationId": {"type": ["null", "string"]}, - "order": {"type": ["null", "integer"]}, - "showUptime": {"type": ["null", "boolean"]}, - "siteId": {"type": ["null", "string"]}, - "titled": {"type": ["null", "boolean"]}, - "updatedAt": {"type": ["null", "string"], "format": "date-time"}, + "closed": { "type": ["null", "boolean"] }, + "createdAt": { "type": ["null", "string"], "format": "date-time" }, + "description": { "type": ["null", "string"] }, + "descriptionHtml": { "type": ["null", "string"] }, + "descriptionTranslationId": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "importedFromStatuspage": { "type": ["null", "boolean"] }, + "name": { "type": ["null", "string"] }, + "nameHtml": { "type": ["null", "string"] }, + "nameHtmlTranslationId": { "type": ["null", "string"] }, + "order": { "type": ["null", "integer"] }, + "showUptime": { "type": ["null", "boolean"] }, + "siteId": { "type": ["null", "string"] }, + "titled": { "type": ["null", "boolean"] }, + "updatedAt": { "type": ["null", "string"], "format": "date-time" }, "translations": { "type": ["null", "object"], "properties": { - "name": {"type": ["null", "string"]}, - "description": {"type": ["null", "string"]} + "name": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] } } } } @@ -55,18 +55,18 @@ "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "started": {"type": ["null", "string"], "format": "date-time"}, - "status": {"type": ["null", "string"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "started": { "type": ["null", "string"], "format": "date-time" }, + "status": { "type": ["null", "string"] }, "components": { "items": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "showUptime": {"type": ["null", "boolean"]} + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] } } } } @@ -76,11 +76,11 @@ "updates": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "message": {"type": ["null", "string"]}, - "messageHtml": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "started": {"type": ["null", "string"], "format": "date-time"} + "id": { "type": ["null", "string"] }, + "message": { "type": ["null", "string"] }, + "messageHtml": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "started": { "type": ["null", "string"], "format": "date-time" } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/public_data.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/public_data.json index 31a89704c14c..87b6d2e46317 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/public_data.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/public_data.json @@ -5,9 +5,9 @@ "page": { "type": ["null", "object"], "properties": { - "name": {"type": ["null", "string"]}, - "url": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]} + "name": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] } } }, "activeIncidents": { @@ -15,11 +15,11 @@ "items": { "type": ["null", "object"], "properties": { - "name": {"type": ["null", "string"]}, - "started": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "impact": {"type": ["null", "string"]}, - "url": {"type": ["null", "string"]} + "name": { "type": ["null", "string"] }, + "started": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "impact": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } } } }, @@ -28,11 +28,11 @@ "items": { "type": ["null", "object"], "properties": { - "name": {"type": ["null", "string"]}, - "start": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "duration": {"type": ["null", "string"]}, - "url": {"type": ["null", "string"]} + "name": { "type": ["null", "string"] }, + "start": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "duration": { "type": ["null", "string"] }, + "url": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/status_pages.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/status_pages.json index 2dca86385d16..1478ac984866 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/status_pages.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/status_pages.json @@ -106,4 +106,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/subscribers.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/subscribers.json index 35f4d36c1d84..e433f68cbc3f 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/subscribers.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/subscribers.json @@ -26,19 +26,19 @@ "components": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "status": {"type": ["null", "string"]}, - "showUptime": {"type": ["null", "boolean"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] }, "site": { "type": ["null", "object"], "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "subdomain": {"type": ["null", "string"]}, - "color": {"type": ["null", "boolean"]}, - "logoUrl": {"type": ["null", "boolean"]}, - "publicEmail": {"type": ["null", "boolean"]} + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "subdomain": { "type": ["null", "string"] }, + "color": { "type": ["null", "boolean"] }, + "logoUrl": { "type": ["null", "boolean"] }, + "publicEmail": { "type": ["null", "boolean"] } } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/team.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/team.json index 27e2a55f6a4d..85223a9cddf6 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/team.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/team.json @@ -2,9 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id": {"type": ["null", "string"]}, - "name": {"type": ["null", "string"]}, - "email": {"type": ["null", "string"]}, - "avatar": {"type": ["null", "string"]} + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "avatar": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/templates.json b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/templates.json index 33f755b7afe0..f24b5fddc812 100644 --- a/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/templates.json +++ b/airbyte-integrations/connectors/source-instatus/source_instatus/schemas/templates.json @@ -51,29 +51,29 @@ "items": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "string"]}, - "name": { "type": ["null", "string"]}, - "status": { "type": ["null", "string"]}, - "showUptime": { "type": ["null", "boolean"]}, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "status": { "type": ["null", "string"] }, + "showUptime": { "type": ["null", "boolean"] }, "site": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "string"]}, - "name": { "type": ["null", "string"]}, - "subdomain": { "type": ["null", "string"]}, - "color": { "type": ["null", "string"]}, - "logoUrl": { "type": ["null", "string"]}, - "publicEmail": { "type": ["null", "string"]} - } - } - } - } + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "subdomain": { "type": ["null", "string"] }, + "color": { "type": ["null", "string"] }, + "logoUrl": { "type": ["null", "string"] }, + "publicEmail": { "type": ["null", "string"] } + } + } + } + } }, "translations": { "type": ["null", "object"], "properties": { - "name": {"type": ["null", "string"]}, - "message": {"type": ["null", "string"]} + "name": { "type": ["null", "string"] }, + "message": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index 55474004433f..d1ffa9e1739c 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -34,5 +34,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml index 07f16d6d3d3c..2bcc47ce0751 100644 --- a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml @@ -6,6 +6,9 @@ acceptance_tests: spec: tests: - spec_path: "source_intercom/spec.json" + # Spec fix: advanced auth configuration contain `client_id` and `client_secret` fields but they were missing in spec. + backward_compatibility_tests_config: + disable_for_version: "0.2.1" connection: tests: - config_path: "secrets/config.json" @@ -15,12 +18,14 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + # Schema fix: update schemas with undeclared fields which is not breaking change + backward_compatibility_tests_config: + disable_for_version: "0.2.1" basic_read: tests: - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json index 12988440ebc6..2bd1cb003b2c 100755 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -1,77 +1,77 @@ [ { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "segments" - }, - "stream_state": { - "updated_at": 7626086649 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "segments" + }, + "stream_state": { + "updated_at": 7626086649 } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "companies" - }, - "stream_state": { - "updated_at": 7626086649 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "companies" + }, + "stream_state": { + "updated_at": 7626086649 } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "company_segments" - }, - "stream_state": { - "updated_at": 7626086649, - "companies": { - "updated_at": 7626086649 - } - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "company_segments" + }, + "stream_state": { + "updated_at": 7626086649, + "companies": { + "updated_at": 7626086649 + } } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "conversations" - }, - "stream_state": { - "updated_at": 7626086649, - "conversations": { - "updated_at": 7626086649 - } - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "conversations" + }, + "stream_state": { + "updated_at": 7626086649, + "conversations": { + "updated_at": 7626086649 + } } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "conversation_parts" - }, - "stream_state": { - "updated_at": 7626086649, - "conversations": { - "updated_at": 7626086649 - } - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "conversation_parts" + }, + "stream_state": { + "updated_at": 7626086649, + "conversations": { + "updated_at": 7626086649 + } } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "contacts" - }, - "stream_state": { - "updated_at": 7626086649 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "contacts" + }, + "stream_state": { + "updated_at": 7626086649 } + } } ] diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl index d76b7e947357..6823ccd4dbb4 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl @@ -11,18 +11,17 @@ {"stream": "admins", "data": {"type": "admin", "email": "user8.sample.airbyte@outlook.com", "id": "6407155", "name": "User8 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407167, 6407174], "team_priority_level": {"primary_team_ids": [6407167, 6407174]}}, "emitted_at": 1680518975305} {"stream": "admins", "data": {"type": "admin", "email": "user9.sample.airbyte@outlook.com", "id": "6407156", "name": "User9 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407168, 6407170, 6407173], "team_priority_level": {"primary_team_ids": [6407168, 6407170, 6407173]}}, "emitted_at": 1680518975306} {"stream": "admins", "data": {"type": "admin", "email": "user10.sample.airbyte@outlook.com", "id": "6407160", "name": "User10 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407168, 6407175], "team_priority_level": {"primary_team_ids": [6407168, 6407175]}}, "emitted_at": 1680518975308} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc46a811f1737ded479ef-qualification-company", "id": "63ecc46a811f1737ded479ee", "app_id": "wjw5eps7", "name": "Test Company 4", "created_at": 1676461162, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 150, "website": "www.company4.com", "industry": "Software", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976223} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5731d460cdc137c906d-qualification-company", "id": "63ecc5731d460cdc137c906c", "app_id": "wjw5eps7", "name": "Test Company 8", "created_at": 1676461427, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 49, "website": "www.company8.com", "industry": "Manufacturing", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976229} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc4e99a2c64721f435a23-qualification-company", "id": "63ecc4e99a2c64721f435a22", "app_id": "wjw5eps7", "name": "Test Company 6", "created_at": 1676461289, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 55, "website": "www.company6.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976233} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc7afb3789118eb91306b-qualification-company", "id": "63ecc7afb3789118eb91306a", "app_id": "wjw5eps7", "name": "Test Company 11", "created_at": 1676461999, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 9, "website": "www.company11.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976238} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc61266325d8ebd24ed11-qualification-company", "id": "63ecc61266325d8ebd24ed10", "app_id": "wjw5eps7", "name": "Test Company 10", "created_at": 1676461586, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 38, "website": "www.company10.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976243} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc3d60e3c81baaad9f9ef-qualification-company", "id": "63ecc3d60e3c81baaad9f9ee", "app_id": "wjw5eps7", "name": "Company 1", "created_at": 1676461015, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 25, "website": "www.company1.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976247} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc41866325d2e90b0d3c6-qualification-company", "id": "63ecc41866325d2e90b0d3c5", "app_id": "wjw5eps7", "name": "Test Company 3", "created_at": 1676461080, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 50, "website": "www.company3.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976250} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfef66325dc8a0ac006f-qualification-company", "id": "63ecbfef66325dc8a0ac006e", "app_id": "wjw5eps7", "name": "Test Company 2", "created_at": 1676460015, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT 123", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976254} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc52f00fc87e58e8fb1f2-qualification-company", "id": "63ecc52f00fc87e58e8fb1f1", "app_id": "wjw5eps7", "name": "Test Company 7", "created_at": 1676461359, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 23, "website": "www.company7.com", "industry": "Production", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976257} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfccb064f24a4941d219-qualification-company", "id": "63ecbfccb064f24a4941d218", "app_id": "wjw5eps7", "name": "Test Company", "created_at": 1676459980, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799571", "name": "Tag1"}, {"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976261} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5d32059cdacf4ac6171-qualification-company", "id": "63ecc5d32059cdacf4ac6170", "app_id": "wjw5eps7", "name": "Test Company 9", "created_at": 1676461523, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 75, "website": "www.company9.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976264} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecc4af3c8e034aa0ce74e2-qualification-company", "id": "63ecc4af3c8e034aa0ce74e1", "app_id": "wjw5eps7", "name": "Test Company 5", "created_at": 1676461231, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 98, "website": "www.company5.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799633", "name": "Tag5"}, {"type": "tag", "id": "7799634", "name": "Tag6"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1680518976268} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5731d460cdc137c906d-qualification-company", "id": "63ecc5731d460cdc137c906c", "app_id": "wjw5eps7", "name": "Test Company 8", "created_at": 1676461427, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 49, "website": "www.company8.com", "industry": "Manufacturing", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867526} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc52f00fc87e58e8fb1f2-qualification-company", "id": "63ecc52f00fc87e58e8fb1f1", "app_id": "wjw5eps7", "name": "Test Company 7", "created_at": 1676461359, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 23, "website": "www.company7.com", "industry": "Production", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867529} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc46a811f1737ded479ef-qualification-company", "id": "63ecc46a811f1737ded479ee", "app_id": "wjw5eps7", "name": "Test Company 4", "created_at": 1676461162, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 150, "website": "www.company4.com", "industry": "Software", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867531} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc5d32059cdacf4ac6171-qualification-company", "id": "63ecc5d32059cdacf4ac6170", "app_id": "wjw5eps7", "name": "Test Company 9", "created_at": 1676461523, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 75, "website": "www.company9.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867536} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc61266325d8ebd24ed11-qualification-company", "id": "63ecc61266325d8ebd24ed10", "app_id": "wjw5eps7", "name": "Test Company 10", "created_at": 1676461586, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 38, "website": "www.company10.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867538} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfccb064f24a4941d219-qualification-company", "id": "63ecbfccb064f24a4941d218", "app_id": "wjw5eps7", "name": "Test Company", "created_at": 1676459980, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799571", "name": "Tag1"}, {"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867533} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfef66325dc8a0ac006f-qualification-company", "id": "63ecbfef66325dc8a0ac006e", "app_id": "wjw5eps7", "name": "Test Company 2", "created_at": 1676460015, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT 123", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867540} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc41866325d2e90b0d3c6-qualification-company", "id": "63ecc41866325d2e90b0d3c5", "app_id": "wjw5eps7", "name": "Test Company 3", "created_at": 1676461080, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 50, "website": "www.company3.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867542} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc3d60e3c81baaad9f9ef-qualification-company", "id": "63ecc3d60e3c81baaad9f9ee", "app_id": "wjw5eps7", "name": "Company 1", "created_at": 1676461015, "updated_at": 1689068298, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 25, "website": "www.company1.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867548} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc4e99a2c64721f435a23-qualification-company", "id": "63ecc4e99a2c64721f435a22", "app_id": "wjw5eps7", "name": "Test Company 6", "created_at": 1676461289, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 55, "website": "www.company6.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867544} +{"stream": "companies", "data": {"type": "company", "company_id": "63ecc7afb3789118eb91306b-qualification-company", "id": "63ecc7afb3789118eb91306a", "app_id": "wjw5eps7", "name": "Test Company 11", "created_at": 1676461999, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 9, "website": "www.company11.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867546} {"stream": "company_attributes", "data": {"type": "data_attribute", "name": "name", "full_name": "name", "label": "Company name", "description": "The name of a company", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977060} {"stream": "company_attributes", "data": {"type": "data_attribute", "name": "company_id", "full_name": "company_id", "label": "Company ID", "description": "A number identifying a company", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977062} {"stream": "company_attributes", "data": {"type": "data_attribute", "name": "last_request_at", "full_name": "last_request_at", "label": "Company last seen", "description": "The last day anyone from a company visited your site or app", "data_type": "date", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "company"}, "emitted_at": 1680518977064} @@ -51,35 +50,33 @@ {"stream": "company_segments", "data": {"type": "segment", "id": "63ea1a43d9c86cceefd8796e", "name": "Revenue", "created_at": 1676286531, "updated_at": 1676462321, "person_type": "user"}, "emitted_at": 1680518982259} {"stream": "company_segments", "data": {"type": "segment", "id": "63ecc7f36d40e8184b5d47a6", "name": "Sales", "created_at": 1676462067, "updated_at": 1676462069, "person_type": "user"}, "emitted_at": 1680518982262} {"stream": "company_segments", "data": {"type": "segment", "id": "6241a4b8c8b709894fa54df1", "name": "Test_1", "created_at": 1648469176, "updated_at": 1676462341, "person_type": "user"}, "emitted_at": 1680518982266} -{"stream": "conversations", "data": {"type": "conversation", "id": "1", "created_at": 1607553243, "updated_at": 1626346673, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "701718739", "delivered_as": "customer_initiated", "subject": "", "body": "

    hey there

    ", "author": {"type": "lead", "id": "5fd150d50697b6d0bbc4a2c2", "name": null, "email": ""}, "attachments": [], "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d", "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": {"created_at": 1607553243, "type": "conversation", "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d"}, "admin_assignee_id": null, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": 4317957, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": 4317954, "first_contact_reply_at": 1607553243, "first_assignment_at": null, "first_admin_reply_at": 1625654131, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": 1607553246, "last_admin_reply_at": 1625656000, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 7}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": null, "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983339} -{"stream": "conversations", "data": {"type": "conversation", "id": "59", "created_at": 1676460979, "updated_at": 1676460980, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952658", "delivered_as": "automated", "subject": "", "body": "

    Test 1

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea418c0931f79d99a197ff"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983347} -{"stream": "conversations", "data": {"type": "conversation", "id": "60", "created_at": 1676461133, "updated_at": 1676461134, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952871", "delivered_as": "automated", "subject": "", "body": "

    Test 3

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a0eddb9b625fb712c9"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test3", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983355} -{"stream": "conversations", "data": {"type": "conversation", "id": "61", "created_at": 1676461196, "updated_at": 1676461197, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952963", "delivered_as": "automated", "subject": "", "body": "

    Test 4

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1b0e17c53248c7956"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 4", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983362} -{"stream": "conversations", "data": {"type": "conversation", "id": "62", "created_at": 1676461263, "updated_at": 1676461264, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953065", "delivered_as": "automated", "subject": "", "body": "

    Test 5

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1eddb9b625fb712cf"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 5", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983368} -{"stream": "conversations", "data": {"type": "conversation", "id": "63", "created_at": 1676461327, "updated_at": 1676461328, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953153", "delivered_as": "automated", "subject": "", "body": "

    Test 6

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2b2d44e63848146e7"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 6", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983374} -{"stream": "conversations", "data": {"type": "conversation", "id": "64", "created_at": 1676461395, "updated_at": 1676461396, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953262", "delivered_as": "automated", "subject": "", "body": "

    Test 7

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2c340f850172f2905"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 7", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983379} -{"stream": "conversations", "data": {"type": "conversation", "id": "65", "created_at": 1676461499, "updated_at": 1676461499, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953436", "delivered_as": "automated", "subject": "", "body": "

    Test Lead 1

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983385} -{"stream": "conversations", "data": {"type": "conversation", "id": "66", "created_at": 1676461563, "updated_at": 1676461564, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953541", "delivered_as": "automated", "subject": "", "body": "

    Test 9

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a3b0e17c505e52044d"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 9", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983390} -{"stream": "conversations", "data": {"type": "conversation", "id": "67", "created_at": 1676461636, "updated_at": 1676461637, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953649", "delivered_as": "automated", "subject": "", "body": "

    Test 10

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a7b0e17c5039fbb824"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 10", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983396} -{"stream": "conversations", "data": {"type": "conversation", "id": "68", "created_at": 1676461800, "updated_at": 1676461800, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953852", "delivered_as": "automated", "subject": "", "body": "

    Test Lead 5001

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ecc6c2811f17873ed2d007"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 5001", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983401} -{"stream": "conversations", "data": {"type": "conversation", "id": "69", "created_at": 1676462031, "updated_at": 1676462031, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51954139", "delivered_as": "automated", "subject": "", "body": "

    Test 11

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a80931f79b6998e89f"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 11", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1680518983407} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288120839", "part_type": "comment", "body": "

    is this showing up

    ", "created_at": 1607553246, "updated_at": 1607553246, "notified_at": 1607553246, "assigned_to": null, "author": {"id": "5fd150d50697b6d0bbc4a2c2", "type": "user", "name": null, "email": ""}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1680518985032} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121348", "part_type": "comment", "body": "

    Airbyte [DEV] will reply as soon as they can.

    ", "created_at": 1607553249, "updated_at": 1607553249, "notified_at": 1607553249, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1680518985038} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121392", "part_type": "comment", "body": "

    Give the team a way to reach you:

    ", "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1680518985043} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121429", "part_type": "comment", "body": null, "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1680518985048} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "9852986065", "part_type": "comment", "body": "

    This message was deleted

    ", "created_at": 1625654131, "updated_at": 1626346672, "notified_at": 1625654131, "assigned_to": null, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": true, "conversation_id": "1"}, "emitted_at": 1680518985053} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "9853397844", "part_type": "comment", "body": "

    This message was deleted

    ", "created_at": 1625656000, "updated_at": 1626346669, "notified_at": 1625656000, "assigned_to": null, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": true, "conversation_id": "1"}, "emitted_at": 1680518985059} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19759948453", "part_type": "assignment", "body": null, "created_at": 1676460980, "updated_at": 1676460980, "notified_at": 1676460980, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "59"}, "emitted_at": 1680518985401} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19759993209", "part_type": "assignment", "body": null, "created_at": 1676461134, "updated_at": 1676461134, "notified_at": 1676461134, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "60"}, "emitted_at": 1680518985752} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760015015", "part_type": "assignment", "body": null, "created_at": 1676461197, "updated_at": 1676461197, "notified_at": 1676461197, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "61"}, "emitted_at": 1680518986090} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760037718", "part_type": "assignment", "body": null, "created_at": 1676461264, "updated_at": 1676461264, "notified_at": 1676461264, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "62"}, "emitted_at": 1680518986436} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760059085", "part_type": "assignment", "body": null, "created_at": 1676461328, "updated_at": 1676461328, "notified_at": 1676461328, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "63"}, "emitted_at": 1680518986773} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760081807", "part_type": "assignment", "body": null, "created_at": 1676461396, "updated_at": 1676461396, "notified_at": 1676461396, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "64"}, "emitted_at": 1680518987088} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760115881", "part_type": "assignment", "body": null, "created_at": 1676461499, "updated_at": 1676461499, "notified_at": 1676461499, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "65"}, "emitted_at": 1680518987404} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760137018", "part_type": "assignment", "body": null, "created_at": 1676461564, "updated_at": 1676461564, "notified_at": 1676461564, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "66"}, "emitted_at": 1680518987729} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760160934", "part_type": "assignment", "body": null, "created_at": 1676461637, "updated_at": 1676461637, "notified_at": 1676461637, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "67"}, "emitted_at": 1680518988107} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760216689", "part_type": "assignment", "body": null, "created_at": 1676461800, "updated_at": 1676461800, "notified_at": 1676461800, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "68"}, "emitted_at": 1680518988444} -{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760294809", "part_type": "assignment", "body": null, "created_at": 1676462031, "updated_at": 1676462031, "notified_at": 1676462031, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "69"}, "emitted_at": 1680518988798} +{"stream": "conversations", "data": {"type": "conversation", "id": "1", "created_at": 1607553243, "updated_at": 1626346673, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "701718739", "delivered_as": "customer_initiated", "subject": "", "body": "

    hey there

    ", "author": {"type": "lead", "id": "5fd150d50697b6d0bbc4a2c2", "name": null, "email": ""}, "attachments": [], "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d", "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": {"created_at": 1607553243, "type": "conversation", "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d"}, "admin_assignee_id": null, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": 4317957, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": 4317954, "first_contact_reply_at": 1607553243, "first_assignment_at": null, "first_admin_reply_at": 1625654131, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": 1607553246, "last_admin_reply_at": 1625656000, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 7}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": null, "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694977} +{"stream": "conversations", "data": {"type": "conversation", "id": "59", "created_at": 1676460979, "updated_at": 1689068230, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952658", "delivered_as": "automated", "subject": "", "body": "

    Test 1

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea418c0931f79d99a197ff"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": false, "state": "closed", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 3}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695018} +{"stream": "conversations", "data": {"type": "conversation", "id": "60", "created_at": 1676461133, "updated_at": 1676461134, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952871", "delivered_as": "automated", "subject": "", "body": "

    Test 3

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a0eddb9b625fb712c9"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test3", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694982} +{"stream": "conversations", "data": {"type": "conversation", "id": "61", "created_at": 1676461196, "updated_at": 1676461197, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952963", "delivered_as": "automated", "subject": "", "body": "

    Test 4

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1b0e17c53248c7956"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 4", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694985} +{"stream": "conversations", "data": {"type": "conversation", "id": "63", "created_at": 1676461327, "updated_at": 1676461328, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953153", "delivered_as": "automated", "subject": "", "body": "

    Test 6

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2b2d44e63848146e7"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 6", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694989} +{"stream": "conversations", "data": {"type": "conversation", "id": "64", "created_at": 1676461395, "updated_at": 1676461396, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953262", "delivered_as": "automated", "subject": "", "body": "

    Test 7

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2c340f850172f2905"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 7", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694994} +{"stream": "conversations", "data": {"type": "conversation", "id": "65", "created_at": 1676461499, "updated_at": 1676461499, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953436", "delivered_as": "automated", "subject": "", "body": "

    Test Lead 1

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694998} +{"stream": "conversations", "data": {"type": "conversation", "id": "66", "created_at": 1676461563, "updated_at": 1676461564, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953541", "delivered_as": "automated", "subject": "", "body": "

    Test 9

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a3b0e17c505e52044d"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 9", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695003} +{"stream": "conversations", "data": {"type": "conversation", "id": "67", "created_at": 1676461636, "updated_at": 1676461637, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953649", "delivered_as": "automated", "subject": "", "body": "

    Test 10

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a7b0e17c5039fbb824"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 10", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695007} +{"stream": "conversations", "data": {"type": "conversation", "id": "68", "created_at": 1676461800, "updated_at": 1676461800, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953852", "delivered_as": "automated", "subject": "", "body": "

    Test Lead 5001

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ecc6c2811f17873ed2d007"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 5001", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695011} +{"stream": "conversations", "data": {"type": "conversation", "id": "69", "created_at": 1676462031, "updated_at": 1676462031, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51954139", "delivered_as": "automated", "subject": "", "body": "

    Test 11

    ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a80931f79b6998e89f"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 11", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695015} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288120839", "part_type": "comment", "body": "

    is this showing up

    ", "created_at": 1607553246, "updated_at": 1607553246, "notified_at": 1607553246, "assigned_to": null, "author": {"id": "5fd150d50697b6d0bbc4a2c2", "type": "user", "name": null, "email": ""}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241806} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121348", "part_type": "comment", "body": "

    Airbyte [DEV] will reply as soon as they can.

    ", "created_at": 1607553249, "updated_at": 1607553249, "notified_at": 1607553249, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241811} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121392", "part_type": "comment", "body": "

    Give the team a way to reach you:

    ", "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241815} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121429", "part_type": "comment", "body": null, "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241819} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "9852986065", "part_type": "comment", "body": "

    This message was deleted

    ", "created_at": 1625654131, "updated_at": 1626346672, "notified_at": 1625654131, "assigned_to": null, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": true, "conversation_id": "1"}, "emitted_at": 1688632241822} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "9853397844", "part_type": "comment", "body": "

    This message was deleted

    ", "created_at": 1625656000, "updated_at": 1626346669, "notified_at": 1625656000, "assigned_to": null, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": true, "conversation_id": "1"}, "emitted_at": 1688632241825} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19759948453", "part_type": "assignment", "body": null, "created_at": 1676460980, "updated_at": 1676460980, "notified_at": 1676460980, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "59"}, "emitted_at": 1688632242153} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19759993209", "part_type": "assignment", "body": null, "created_at": 1676461134, "updated_at": 1676461134, "notified_at": 1676461134, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "60"}, "emitted_at": 1688632242481} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760015015", "part_type": "assignment", "body": null, "created_at": 1676461197, "updated_at": 1676461197, "notified_at": 1676461197, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "61"}, "emitted_at": 1688632242922} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760059085", "part_type": "assignment", "body": null, "created_at": 1676461328, "updated_at": 1676461328, "notified_at": 1676461328, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "63"}, "emitted_at": 1688632243464} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760081807", "part_type": "assignment", "body": null, "created_at": 1676461396, "updated_at": 1676461396, "notified_at": 1676461396, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "64"}, "emitted_at": 1688632243779} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760115881", "part_type": "assignment", "body": null, "created_at": 1676461499, "updated_at": 1676461499, "notified_at": 1676461499, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "65"}, "emitted_at": 1688632244129} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760137018", "part_type": "assignment", "body": null, "created_at": 1676461564, "updated_at": 1676461564, "notified_at": 1676461564, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "66"}, "emitted_at": 1688632244457} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760160934", "part_type": "assignment", "body": null, "created_at": 1676461637, "updated_at": 1676461637, "notified_at": 1676461637, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "67"}, "emitted_at": 1688632244747} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760216689", "part_type": "assignment", "body": null, "created_at": 1676461800, "updated_at": 1676461800, "notified_at": 1676461800, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "68"}, "emitted_at": 1688632245084} +{"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "19760294809", "part_type": "assignment", "body": null, "created_at": 1676462031, "updated_at": 1676462031, "notified_at": 1676462031, "assigned_to": {"type": "admin", "id": "4423433"}, "author": {"id": "4423433", "type": "admin", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "69"}, "emitted_at": 1688632245506} {"stream": "contact_attributes", "data": {"id": 9182660, "type": "data_attribute", "name": "Company", "full_name": "custom_attributes.Company", "label": "Company", "data_type": "string", "api_writable": true, "ui_writable": true, "custom": true, "archived": false, "created_at": 1676295702, "updated_at": 1676295702, "model": "contact"}, "emitted_at": 1680518989376} {"stream": "contact_attributes", "data": {"id": 9182661, "type": "data_attribute", "name": "Tag", "full_name": "custom_attributes.Tag", "label": "Tag", "data_type": "string", "api_writable": true, "ui_writable": false, "custom": true, "archived": false, "created_at": 1676295702, "updated_at": 1676295702, "model": "contact"}, "emitted_at": 1680518989378} {"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "id", "full_name": "id", "label": "ID", "description": "The Intercom defined id representing the user", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989379} @@ -90,26 +87,9 @@ {"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "android_sdk_version", "full_name": "android_sdk_version", "label": "Android SDK version", "description": "The version of the Android SDK a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989386} {"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "ios_app_name", "full_name": "ios_app_name", "label": "iOS App name", "description": "The name of iOS app a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989387} {"stream": "contact_attributes", "data": {"type": "data_attribute", "name": "ios_sdk_version", "full_name": "ios_sdk_version", "label": "iOS SDK version", "description": "The version of the iOS SDK a person is using", "data_type": "string", "api_writable": false, "ui_writable": false, "custom": false, "archived": false, "model": "contact"}, "emitted_at": 1680518989389} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41aaeddb9b627ce9b882", "workspace_id": "wjw5eps7", "external_id": "20033080", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33080", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296619, "updated_at": 1676296619, "signed_up_at": 2328134400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41aaeddb9b627ce9b882/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990376} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1b0e17c51fa4eb704", "workspace_id": "wjw5eps7", "external_id": "20037835", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User37835", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2738966400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1b0e17c51fa4eb704/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990383} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1c340f84dffe8cddc", "workspace_id": "wjw5eps7", "external_id": "20033579", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33579", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2371248000, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1c340f84dffe8cddc/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990407} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1eddb9b607f593949", "workspace_id": "wjw5eps7", "external_id": "20035970", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User35970", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2577830400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1eddb9b607f593949/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1eddb9b607f593949/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1eddb9b607f593949/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1eddb9b607f593949/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990414} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1eddb9b625fb7130a", "workspace_id": "wjw5eps7", "external_id": "20042431", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User42431", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 3136060800, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1eddb9b625fb7130a/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1eddb9b625fb7130a/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1eddb9b625fb7130a/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1eddb9b625fb7130a/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990421} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41e6ab35564e6f32f4a6", "workspace_id": "wjw5eps7", "external_id": "20049782", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User49782", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296678, "updated_at": 1676296678, "signed_up_at": 3771187200, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41e6ab35564e6f32f4a6/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41e6ab35564e6f32f4a6/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41e6ab35564e6f32f4a6/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41e6ab35564e6f32f4a6/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990428} -{"stream": "contacts", "data": {"type": "contact", "id": "63ecbef231c68598171f7cc2", "workspace_id": "wjw5eps7", "external_id": "123456789", "role": "user", "email": "test@test.com", "phone": null, "name": "Test User", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676459763, "updated_at": 1676459763, "signed_up_at": null, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [], "url": "/contacts/63ecbef231c68598171f7cc2/tags", "total_count": 0, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ecbef231c68598171f7cc2/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ecbef231c68598171f7cc2/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ecbef231c68598171f7cc2/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990435} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea419cb0e17c50f294c724", "workspace_id": "wjw5eps7", "external_id": "20033712", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33712", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296604, "updated_at": 1676460037, "signed_up_at": 2382739200, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea419cb0e17c50f294c724/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea419cb0e17c50f294c724/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecbfef66325dc8a0ac006e", "type": "company", "url": "/companies/63ecbfef66325dc8a0ac006e"}], "url": "/contacts/63ea419cb0e17c50f294c724/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea419cb0e17c50f294c724/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990443} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a0eddb9b625fb712c9", "workspace_id": "wjw5eps7", "external_id": "20030866", "role": "user", "email": "user20.sample@gmail.com", "phone": "+188844477723", "name": "User30866", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296608, "updated_at": 1676461134, "signed_up_at": 2136844800, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461134, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a0eddb9b625fb712c9/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a0eddb9b625fb712c9/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc41866325d2e90b0d3c5", "type": "company", "url": "/companies/63ecc41866325d2e90b0d3c5"}], "url": "/contacts/63ea41a0eddb9b625fb712c9/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a0eddb9b625fb712c9/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990450} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a1b0e17c53248c7956", "workspace_id": "wjw5eps7", "external_id": "20031178", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13888333888", "name": "User31178", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296609, "updated_at": 1676461197, "signed_up_at": 2163801600, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461197, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a1b0e17c53248c7956/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a1b0e17c53248c7956/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc46a811f1737ded479ee", "type": "company", "url": "/companies/63ecc46a811f1737ded479ee"}], "url": "/contacts/63ea41a1b0e17c53248c7956/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a1b0e17c53248c7956/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990457} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a1eddb9b625fb712cf", "workspace_id": "wjw5eps7", "external_id": "20030872", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13888434888", "name": "User30872", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296609, "updated_at": 1676461264, "signed_up_at": 2137363200, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461264, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a1eddb9b625fb712cf/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a1eddb9b625fb712cf/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc4af3c8e034aa0ce74e1", "type": "company", "url": "/companies/63ecc4af3c8e034aa0ce74e1"}], "url": "/contacts/63ea41a1eddb9b625fb712cf/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a1eddb9b625fb712cf/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990463} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a2b2d44e63848146e7", "workspace_id": "wjw5eps7", "external_id": "20028832", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13335556789", "name": "User28832", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296610, "updated_at": 1676461327, "signed_up_at": 1961107200, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461327, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a2b2d44e63848146e7/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a2b2d44e63848146e7/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc4e99a2c64721f435a22", "type": "company", "url": "/companies/63ecc4e99a2c64721f435a22"}], "url": "/contacts/63ea41a2b2d44e63848146e7/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a2b2d44e63848146e7/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990469} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a2c340f850172f2905", "workspace_id": "wjw5eps7", "external_id": "20032071", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13888333888", "name": "User32071", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296610, "updated_at": 1676461395, "signed_up_at": 2240956800, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461395, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a2c340f850172f2905/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a2c340f850172f2905/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc52f00fc87e58e8fb1f1", "type": "company", "url": "/companies/63ecc52f00fc87e58e8fb1f1"}], "url": "/contacts/63ea41a2c340f850172f2905/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a2c340f850172f2905/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990474} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a30931f79c14406608", "workspace_id": "wjw5eps7", "external_id": "20032315", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13844463883", "name": "User32315", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296611, "updated_at": 1676461454, "signed_up_at": 2262038400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a30931f79c14406608/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a30931f79c14406608/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc5731d460cdc137c906c", "type": "company", "url": "/companies/63ecc5731d460cdc137c906c"}], "url": "/contacts/63ea41a30931f79c14406608/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a30931f79c14406608/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990479} -{"stream": "contacts", "data": {"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2", "workspace_id": "wjw5eps7", "external_id": "56f1e303-24f7-4f0a-ae18-0bafd587c894", "role": "lead", "email": null, "phone": null, "name": null, "avatar": "https://static.intercomassets.com/app/pseudonym_avatars_2019/yellow-guitar.png", "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1607553237, "updated_at": 1676461499, "signed_up_at": null, "last_seen_at": 1607553243, "last_replied_at": 1607553246, "last_contacted_at": 1676461499, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": "chrome", "browser_version": "87.0.4280.88", "browser_language": "en", "os": "OS X 10.15.6", "location": {"type": "location", "country": "United States", "region": "California", "city": "Daly City", "country_code": "USA", "continent_code": "NA"}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [], "url": "/contacts/5fd150d50697b6d0bbc4a2c2/tags", "total_count": 0, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/5fd150d50697b6d0bbc4a2c2/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/5fd150d50697b6d0bbc4a2c2/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/5fd150d50697b6d0bbc4a2c2/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990484} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a3b0e17c505e52044d", "workspace_id": "wjw5eps7", "external_id": "20032233", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13823333883", "name": "User32233", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296611, "updated_at": 1676461564, "signed_up_at": 2254953600, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461564, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a3b0e17c505e52044d/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a3b0e17c505e52044d/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc5d32059cdacf4ac6170", "type": "company", "url": "/companies/63ecc5d32059cdacf4ac6170"}], "url": "/contacts/63ea41a3b0e17c505e52044d/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a3b0e17c505e52044d/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990489} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a7b0e17c5039fbb824", "workspace_id": "wjw5eps7", "external_id": "20036235", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13888463888", "name": "User36235", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296615, "updated_at": 1676461637, "signed_up_at": 2600726400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461637, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a7b0e17c5039fbb824/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a7b0e17c5039fbb824/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc61266325d8ebd24ed10", "type": "company", "url": "/companies/63ecc61266325d8ebd24ed10"}], "url": "/contacts/63ea41a7b0e17c5039fbb824/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a7b0e17c5039fbb824/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990494} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea418c0931f79d99a197ff", "workspace_id": "wjw5eps7", "external_id": "20026950", "role": "user", "email": "user20.sample@gmail.com", "phone": "+133344455567", "name": "User26950", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296589, "updated_at": 1676461686, "signed_up_at": 1798502400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676460980, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {"Company": null}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}, {"id": "7799571", "type": "tag", "url": "/tags/7799571"}], "url": "/contacts/63ea418c0931f79d99a197ff/tags", "total_count": 2, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea418c0931f79d99a197ff/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc3d60e3c81baaad9f9ee", "type": "company", "url": "/companies/63ecc3d60e3c81baaad9f9ee"}], "url": "/contacts/63ea418c0931f79d99a197ff/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea418c0931f79d99a197ff/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990499} -{"stream": "contacts", "data": {"type": "contact", "id": "63ecc6c2811f17873ed2d007", "workspace_id": "wjw5eps7", "external_id": "347ff2b4-93a0-4d78-b0e2-487c3855a3fd", "role": "lead", "email": "user5001.sample.airbyte@gmail.com", "phone": "+13335556789", "name": "User5001", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676461762, "updated_at": 1676461800, "signed_up_at": null, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676461800, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [], "url": "/contacts/63ecc6c2811f17873ed2d007/tags", "total_count": 0, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ecc6c2811f17873ed2d007/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecbfccb064f24a4941d218", "type": "company", "url": "/companies/63ecbfccb064f24a4941d218"}], "url": "/contacts/63ecc6c2811f17873ed2d007/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ecc6c2811f17873ed2d007/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990505} -{"stream": "contacts", "data": {"type": "contact", "id": "63ea41a80931f79b6998e89f", "workspace_id": "wjw5eps7", "external_id": "20033328", "role": "user", "email": "user20.sample@gmail.com", "phone": "+13888463888", "name": "User33328", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296616, "updated_at": 1676462031, "signed_up_at": 2349561600, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": 1676462031, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41a80931f79b6998e89f/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41a80931f79b6998e89f/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [{"id": "63ecc7afb3789118eb91306a", "type": "company", "url": "/companies/63ecc7afb3789118eb91306a"}], "url": "/contacts/63ea41a80931f79b6998e89f/companies", "total_count": 1, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41a80931f79b6998e89f/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1680518990510} +{"stream": "contacts", "data": {"type": "contact", "id": "63ea41aaeddb9b627ce9b882", "workspace_id": "wjw5eps7", "external_id": "20033080", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33080", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296619, "updated_at": 1676296619, "signed_up_at": 2328134400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41aaeddb9b627ce9b882/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41aaeddb9b627ce9b882/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1689154147400} +{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1b0e17c51fa4eb704", "workspace_id": "wjw5eps7", "external_id": "20037835", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User37835", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2738966400, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1b0e17c51fa4eb704/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1b0e17c51fa4eb704/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1689154147403} +{"stream": "contacts", "data": {"type": "contact", "id": "63ea41b1c340f84dffe8cddc", "workspace_id": "wjw5eps7", "external_id": "20033579", "role": "user", "email": "user20.sample@gmail.com", "phone": null, "name": "User33579", "avatar": null, "owner_id": null, "social_profiles": {"type": "list", "data": []}, "has_hard_bounced": false, "marked_email_as_spam": false, "unsubscribed_from_emails": false, "created_at": 1676296625, "updated_at": 1676296625, "signed_up_at": 2371248000, "last_seen_at": null, "last_replied_at": null, "last_contacted_at": null, "last_email_opened_at": null, "last_email_clicked_at": null, "language_override": null, "browser": null, "browser_version": null, "browser_language": null, "os": null, "location": {"type": "location", "country": null, "region": null, "city": null, "country_code": null, "continent_code": null}, "android_app_name": null, "android_app_version": null, "android_device": null, "android_os_version": null, "android_sdk_version": null, "android_last_seen_at": null, "ios_app_name": null, "ios_app_version": null, "ios_device": null, "ios_os_version": null, "ios_sdk_version": null, "ios_last_seen_at": null, "custom_attributes": {}, "tags": {"type": "list", "data": [{"id": "7800292", "type": "tag", "url": "/tags/7800292"}], "url": "/contacts/63ea41b1c340f84dffe8cddc/tags", "total_count": 1, "has_more": false}, "notes": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/notes", "total_count": 0, "has_more": false}, "companies": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/companies", "total_count": 0, "has_more": false}, "opted_out_subscription_types": {"type": "list", "data": [], "url": "/contacts/63ea41b1c340f84dffe8cddc/subscriptions", "total_count": 0, "has_more": false}, "utm_campaign": null, "utm_content": null, "utm_medium": null, "utm_source": null, "utm_term": null, "referrer": null, "sms_consent": false, "unsubscribed_from_sms": false}, "emitted_at": 1689154147406} {"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f238", "name": "Active", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1680518991074} {"stream": "segments", "data": {"type": "segment", "id": "63ea19b36ea882cf2f785a45", "name": "Country", "created_at": 1676286387, "updated_at": 1676286387, "person_type": "user"}, "emitted_at": 1680518991078} {"stream": "segments", "data": {"type": "segment", "id": "5f8d1c6caee76458e332f237", "name": "New", "created_at": 1603083372, "updated_at": 1603083372, "person_type": "user"}, "emitted_at": 1680518991082} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json index dad2bcddc87e..6745627f060d 100755 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -1,74 +1,74 @@ [ { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "segments" - }, - "stream_state": { - "updated_at": 1676297667 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "segments" + }, + "stream_state": { + "updated_at": 1676297667 } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "companies" - }, - "stream_state": { - "updated_at": 1676517777 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "companies" + }, + "stream_state": { + "updated_at": 1676517777 } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "company_segments" - }, - "stream_state": { - "updated_at": 1676462345, - "companies": { - "updated_at": 1676517777 - } - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "company_segments" + }, + "stream_state": { + "updated_at": 1676462345, + "companies": { + "updated_at": 1676517777 + } } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "conversations" - }, - "stream_state": { - "updated_at": 1676462031 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "conversations" + }, + "stream_state": { + "updated_at": 1676462031 } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "conversation_parts" - }, - "stream_state": { - "updated_at": 1676462031, - "conversations": { - "updated_at": 1676462031 - } - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "conversation_parts" + }, + "stream_state": { + "updated_at": 1676462031, + "conversations": { + "updated_at": 1676462031 + } } + } }, { - "type": "STREAM", - "stream": { - "stream_descriptor": { - "name": "contacts" - }, - "stream_state": { - "updated_at": 1676462031 - } + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "contacts" + }, + "stream_state": { + "updated_at": 1676462031 } + } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-intercom/metadata.yaml b/airbyte-integrations/connectors/source-intercom/metadata.yaml index 3f8f50e98941..c9feadaca378 100644 --- a/airbyte-integrations/connectors/source-intercom/metadata.yaml +++ b/airbyte-integrations/connectors/source-intercom/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a - dockerImageTag: 0.2.1 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-intercom githubIssueLabel: source-intercom icon: intercom.svg @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-intercom/requirements.txt b/airbyte-integrations/connectors/source-intercom/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-intercom/requirements.txt +++ b/airbyte-integrations/connectors/source-intercom/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-intercom/setup.py b/airbyte-integrations/connectors/source-intercom/setup.py index 0b0c8cb0e1c7..f5fce35eb718 100644 --- a/airbyte-integrations/connectors/source-intercom/setup.py +++ b/airbyte-integrations/connectors/source-intercom/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk==0.29.0", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest", "pytest-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/components.py b/airbyte-integrations/connectors/source-intercom/source_intercom/components.py index e837a0c7a82c..6e87c9b9e8a1 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/components.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/components.py @@ -2,25 +2,23 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import os from dataclasses import InitVar, dataclass, field -from functools import lru_cache, wraps +from functools import wraps from time import sleep from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union -import dpath.util import requests -from airbyte_cdk.models import AirbyteMessage, SyncMode, Type -from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.incremental.cursor import Cursor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig -from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler -from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.response_status import ResponseStatus +from airbyte_cdk.sources.declarative.requesters.http_requester import HttpRequester from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_nested_request_input_provider import ( + InterpolatedNestedRequestInputProvider, +) from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider -from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester -from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState from airbyte_cdk.sources.streams.core import Stream @@ -28,15 +26,13 @@ @dataclass -class IncrementalSingleSlice(StreamSlicer): - +class IncrementalSingleSliceCursor(Cursor): cursor_field: Union[InterpolatedString, str] config: Config parameters: InitVar[Mapping[str, Any]] - _cursor: dict = field(default_factory=dict) - initial_state: dict = field(default_factory=dict) def __post_init__(self, parameters: Mapping[str, Any]): + self._state = {} self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) def get_request_params( @@ -45,7 +41,8 @@ def get_request_params( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + # Current implementation does not provide any options to update request params. + # Returns empty dict return self._get_request_option(RequestOptionType.request_parameter, stream_slice) def get_request_headers( @@ -54,7 +51,8 @@ def get_request_headers( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + # Current implementation does not provide any options to update request headers. + # Returns empty dict return self._get_request_option(RequestOptionType.header, stream_slice) def get_request_body_data( @@ -63,7 +61,8 @@ def get_request_body_data( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Mapping[str, Any]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + # Current implementation does not provide any options to update body data. + # Returns empty dict return self._get_request_option(RequestOptionType.body_data, stream_slice) def get_request_body_json( @@ -72,100 +71,65 @@ def get_request_body_json( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Mapping]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + # Current implementation does not provide any options to update body json. + # Returns empty dict return self._get_request_option(RequestOptionType.body_json, stream_slice) def _get_request_option(self, option_type: RequestOptionType, stream_slice: StreamSlice): return {} def get_stream_state(self) -> StreamState: - return self._cursor if self._cursor else {} - - def _get_max_state_value( - self, - current_state_value: Optional[Union[int, str]], - last_record_value: Optional[Union[int, str]], - ) -> Optional[Union[int, str]]: - if current_state_value and last_record_value: - cursor = max(current_state_value, last_record_value) - elif current_state_value: - cursor = current_state_value - else: - cursor = last_record_value - return cursor - - def _set_initial_state(self, stream_slice: StreamSlice): - self.initial_state = stream_slice if not self.initial_state else self.initial_state - - def _update_cursor_with_prior_state(self): - self._cursor["prior_state"] = {self.cursor_field.eval(self.config): self.initial_state.get(self.cursor_field.eval(self.config))} - - def _get_current_state(self, stream_slice: StreamSlice) -> Union[str, float, int]: - return stream_slice.get(self.cursor_field.eval(self.config)) - - def _get_last_record_value(self, last_record: Optional[Record] = None) -> Union[str, float, int]: - return last_record.get(self.cursor_field.eval(self.config)) if last_record else None - - def _get_current_cursor_value(self) -> Union[str, float, int]: - return self._cursor.get(self.cursor_field.eval(self.config)) if self._cursor else None - - def _update_current_cursor( - self, - current_cursor_value: Optional[Union[str, float, int]] = None, - updated_cursor_value: Optional[Union[str, float, int]] = None, - ): - if current_cursor_value and updated_cursor_value: - self._cursor.update(**{self.cursor_field.eval(self.config): max(updated_cursor_value, current_cursor_value)}) - elif updated_cursor_value: - self._cursor.update(**{self.cursor_field.eval(self.config): updated_cursor_value}) - - def _update_stream_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - self._update_current_cursor( - self._get_current_cursor_value(), - self._get_max_state_value( - self._get_current_state(stream_slice), - self._get_last_record_value(last_record), - ), - ) - - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - # freeze initial state - self._set_initial_state(stream_slice) - # update the state of the child stream cursor_field value from previous sync, - # and freeze it to have an ability to compare the record vs state - self._update_cursor_with_prior_state() - self._update_stream_cursor(stream_slice, last_record) - - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: + return self._state + + def set_initial_state(self, stream_state: StreamState): + cursor_field = self.cursor_field.eval(self.config) + cursor_value = stream_state.get(cursor_field) + if cursor_value: + self._state[cursor_field] = cursor_value + self._state["prior_state"] = self._state.copy() + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + latest_record = self._state if self.is_greater_than_or_equal(self._state, most_recent_record) else most_recent_record + if latest_record: + cursor_field = self.cursor_field.eval(self.config) + self._state[cursor_field] = latest_record[cursor_field] + + def stream_slices(self) -> Iterable[Mapping[str, Any]]: yield {} + def should_be_synced(self, record: Record) -> bool: + """ + Evaluating if a record should be synced allows for filtering and stop condition on pagination + """ + record_cursor_value = record.get(self.cursor_field.eval(self.config)) + return bool(record_cursor_value) -@dataclass -class IncrementalSubstreamSlicer(StreamSlicer): - """ - Like SubstreamSlicer, but works incrementaly with both parent and substream. - - Input Arguments: - - :: cursor_field: srt - substream cursor_field value - :: parent_complete_fetch: bool - If `True`, all slices is fetched into a list first, then yield. - If `False`, substream emits records on each parernt slice yield. - :: parent_stream_configs: ParentStreamConfig - Describes how to create a stream slice from a parent stream. + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice + """ + cursor_field = self.cursor_field.eval(self.config) + first_cursor_value = first.get(cursor_field) if first else None + second_cursor_value = second.get(cursor_field) if second else None + if first_cursor_value and second_cursor_value: + return first_cursor_value > second_cursor_value + elif first_cursor_value: + return True + else: + return False - """ - config: Config - parameters: InitVar[Mapping[str, Any]] - cursor_field: Union[InterpolatedString, str] +@dataclass +class IncrementalSubstreamSlicerCursor(IncrementalSingleSliceCursor): parent_stream_configs: List[ParentStreamConfig] parent_complete_fetch: bool = field(default=False) - _cursor: dict = field(default_factory=dict) - initial_state: dict = field(default_factory=dict) def __post_init__(self, parameters: Mapping[str, Any]): + super().__post_init__(parameters) + if not self.parent_stream_configs: raise ValueError("IncrementalSubstreamSlicer needs at least 1 parent stream") - self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) + # parent stream parts self.parent_config: ParentStreamConfig = self.parent_stream_configs[0] self.parent_stream: Stream = self.parent_config.stream @@ -175,186 +139,50 @@ def __post_init__(self, parameters: Mapping[str, Any]): self.substream_slice_field: str = self.parent_stream_configs[0].partition_field.eval(self.config) self.parent_field: str = self.parent_stream_configs[0].parent_key.eval(self.config) - def get_request_params( - self, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response - return self._get_request_option(RequestOptionType.request_parameter, stream_slice) - - def get_request_headers( - self, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response - return self._get_request_option(RequestOptionType.header, stream_slice) - - def get_request_body_data( - self, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response - return self._get_request_option(RequestOptionType.body_data, stream_slice) - - def get_request_body_json( - self, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Mapping]: - # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response - return self._get_request_option(RequestOptionType.body_json, stream_slice) - - def _get_request_option(self, option_type: RequestOptionType, stream_slice: StreamSlice): - return {} - - def _get_max_state_value( - self, current_state_value: Optional[Union[int, str]], last_record_value: Optional[Union[int, str]] - ) -> Optional[Union[int, str]]: - if current_state_value and last_record_value: - cursor = max(current_state_value, last_record_value) - elif current_state_value: - cursor = current_state_value - else: - cursor = last_record_value - return cursor - - def _set_initial_state(self, stream_slice: StreamSlice): - self.initial_state = stream_slice if not self.initial_state else self.initial_state - - def _get_last_record_value(self, last_record: Optional[Record] = None, parent: Optional[bool] = False) -> Union[str, float, int]: - if parent: - return last_record.get(self.parent_cursor_field) if last_record else None - else: - return last_record.get(self.cursor_field.eval(self.config)) if last_record else None - - def _get_current_cursor_value(self, parent: Optional[bool] = False) -> Union[str, float, int]: - if parent: - return self._cursor.get(self.parent_stream_name, {}).get(self.parent_cursor_field) if self._cursor else None - else: - return self._cursor.get(self.cursor_field.eval(self.config)) if self._cursor else None - - def _get_current_state(self, stream_slice: StreamSlice, parent: Optional[bool] = False) -> Union[str, float, int]: - if parent: - return stream_slice.get(self.parent_stream_name, {}).get(self.parent_cursor_field) - else: - return stream_slice.get(self.cursor_field.eval(self.config)) - - def _update_current_cursor( - self, - current_cursor_value: Optional[Union[str, float, int]] = None, - updated_cursor_value: Optional[Union[str, float, int]] = None, - parent: Optional[bool] = False, - ): - if current_cursor_value and updated_cursor_value: - if parent: - self._cursor.update( - **{self.parent_stream_name: {self.parent_cursor_field: max(updated_cursor_value, current_cursor_value)}} - ) - else: - self._cursor.update(**{self.cursor_field.eval(self.config): max(updated_cursor_value, current_cursor_value)}) - elif updated_cursor_value: - if parent: - self._cursor.update(**{self.parent_stream_name: {self.parent_cursor_field: updated_cursor_value}}) - else: - self._cursor.update(**{self.cursor_field.eval(self.config): updated_cursor_value}) - - def _update_substream_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - self._update_current_cursor( - self._get_current_cursor_value(), - self._get_max_state_value( - self._get_current_state(stream_slice), - self._get_last_record_value(last_record), - ), - ) - - def _update_parent_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - if self.parent_cursor_field: - self._update_current_cursor( - self._get_current_cursor_value(parent=True), - self._get_max_state_value( - self._get_current_state(stream_slice, parent=True), - self._get_last_record_value(last_record, parent=True), - ), - ) - - def _update_cursor_with_prior_state(self): - self._cursor["prior_state"] = { - self.cursor_field.eval(self.config): self.initial_state.get(self.cursor_field.eval(self.config)), - self.parent_stream_name: { - self.parent_cursor_field: self.initial_state.get(self.parent_stream_name, {}).get(self.parent_cursor_field) - }, - } - - def get_stream_state(self) -> StreamState: - return self._cursor if self._cursor else {} - - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - # freeze initial state - self._set_initial_state(stream_slice) - # update the state of the child stream cursor_field value from previous sync, - # and freeze it to have an ability to compare the record vs state - self._update_cursor_with_prior_state() - # we focus on updating the substream's cursor in this method, - # the parent's cursor is updated while reading parent stream - self._update_substream_cursor(stream_slice, last_record) + def set_initial_state(self, stream_state: StreamState): + super().set_initial_state(stream_state=stream_state) + if self.parent_stream_name in stream_state and stream_state.get(self.parent_stream_name, {}).get(self.parent_cursor_field): + parent_stream_state = {self.parent_cursor_field: stream_state[self.parent_stream_name][self.parent_cursor_field]} + self._state[self.parent_stream_name] = parent_stream_state + if "prior_state" in self._state: + self._state["prior_state"][self.parent_stream_name] = parent_stream_state + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + super().close_slice(stream_slice=stream_slice, most_recent_record=most_recent_record) + if self.parent_stream: + self._state[self.parent_stream_name] = self.parent_stream.state + + def stream_slices(self) -> Iterable[Mapping[str, Any]]: + parent_state = (self._state or {}).get(self.parent_stream_name, {}) + slices_generator = self.read_parent_stream(self.parent_sync_mode, self.parent_cursor_field, parent_state) + yield from [slice for slice in slices_generator] if self.parent_complete_fetch else slices_generator def read_parent_stream( self, sync_mode: SyncMode, cursor_field: Optional[str], stream_state: Mapping[str, Any] ) -> Iterable[Mapping[str, Any]]: + self.parent_stream.state = stream_state - for parent_slice in self.parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state): - empty_parent_slice = True + parent_stream_slices_gen = self.parent_stream.stream_slices( + sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state + ) - # update slice with parent state, to pass the initial parent state to the parent instance - # stream_state is being replaced by empty object, since the parent stream is not directly initiated - parent_prior_state = self._cursor.get("prior_state", {}).get(self.parent_stream_name, {}).get(self.parent_cursor_field) - parent_slice.update({"prior_state": {self.parent_cursor_field: parent_prior_state}}) + for parent_slice in parent_stream_slices_gen: - for parent_record in self.parent_stream.read_records( + parent_records_gen = self.parent_stream.read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=parent_slice, stream_state=stream_state - ): - # Skip non-records (eg AirbyteLogMessage) - if isinstance(parent_record, AirbyteMessage): - if parent_record.type == Type.RECORD: - parent_record = parent_record.record.data - - try: - substream_slice = dpath.util.get(parent_record, self.parent_field) - except KeyError: - pass - else: - empty_parent_slice = False - slice = { - self.substream_slice_field: substream_slice, - self.cursor_field.eval(self.config): self._cursor.get(self.cursor_field.eval(self.config)), + ) + + for parent_record in parent_records_gen: + substream_slice_value = parent_record.get(self.parent_field) + if substream_slice_value: + cursor_field = self.cursor_field.eval(self.config) + yield { + self.substream_slice_field: substream_slice_value, + cursor_field: self._state.get(cursor_field), self.parent_stream_name: { - self.parent_cursor_field: self._cursor.get(self.parent_stream_name, {}).get(self.parent_cursor_field) + self.parent_cursor_field: self._state.get(self.parent_stream_name, {}).get(self.parent_cursor_field) }, } - # track and update the parent cursor - self._update_parent_cursor(slice, parent_record) - yield slice - - # If the parent slice contains no records, - if empty_parent_slice: - yield from [] - - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: - stream_state = self.initial_state or {} - parent_state = stream_state.get(self.parent_stream_name, {}) - parent_state.update(**{"prior_state": self._cursor.get("prior_state", {}).get(self.parent_stream_name, {})}) - slices_generator = self.read_parent_stream(self.parent_sync_mode, self.parent_cursor_field, parent_state) - if self.parent_complete_fetch: - yield from [slice for slice in slices_generator] - else: - yield from slices_generator @dataclass @@ -479,104 +307,42 @@ def wrapper_balance_rate_limit(*args, **kwargs): return decorator -@dataclass -class HttpRequesterWithRateLimiter(Requester): +@dataclass(eq=False) +class HttpRequesterWithRateLimiter(HttpRequester): """ The difference between the built-in `HttpRequester` and this one is the custom decorator, applied on top of `interpret_response_status` to preserve the api calls for a defined amount of time, calculated using the rate limit headers and not use the custom backoff strategy, since we deal with Response.status_code == 200, the default requester's logic doesn't allow to handle the status of 200 with `should_retry()`. - - Attributes: - name (str): Name of the stream. Only used for request/response caching - url_base (Union[InterpolatedString, str]): Base url to send requests to - path (Union[InterpolatedString, str]): Path to send requests to - http_method (Union[str, HttpMethod]): HTTP method to use when sending requests - request_options_provider (Optional[InterpolatedRequestOptionsProvider]): request option provider defining the options to set on outgoing requests - authenticator (DeclarativeAuthenticator): Authenticator defining how to authenticate to the source - error_handler (Optional[ErrorHandler]): Error handler defining how to detect and handle errors - config (Config): The user-provided configuration as specified by the source's spec """ - name: str - url_base: Union[InterpolatedString, str] - path: Union[InterpolatedString, str] - config: Config - parameters: InitVar[Mapping[str, Any]] - http_method: Union[str, HttpMethod] = HttpMethod.GET - request_parameters: Optional[RequestInput] = None - request_headers: Optional[RequestInput] = None - request_body_data: Optional[RequestInput] = None request_body_json: Optional[RequestInput] = None - authenticator: DeclarativeAuthenticator = None - error_handler: Optional[ErrorHandler] = None - - def __post_init__(self, parameters: Mapping[str, Any]): - self.url_base = InterpolatedString.create(self.url_base, parameters=parameters) - self.path = InterpolatedString.create(self.path, parameters=parameters) + request_headers: Optional[RequestInput] = None + request_parameters: Optional[RequestInput] = None - self.authenticator = self.authenticator or NoAuth(parameters=parameters) - if type(self.http_method) == str: - self.http_method = HttpMethod[self.http_method] - self._method = self.http_method - self.error_handler = self.error_handler or DefaultErrorHandler(parameters=parameters, config=self.config) - self._parameters = parameters + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__post_init__(parameters) self.request_parameters = self.request_parameters if self.request_parameters else {} self.request_headers = self.request_headers if self.request_headers else {} - self.request_body_data = self.request_body_data if self.request_body_data else {} self.request_body_json = self.request_body_json if self.request_body_json else {} - if self.request_body_json and self.request_body_data: - raise ValueError("RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both") - self._parameter_interpolator = InterpolatedRequestInputProvider( config=self.config, request_inputs=self.request_parameters, parameters=parameters ) self._headers_interpolator = InterpolatedRequestInputProvider( config=self.config, request_inputs=self.request_headers, parameters=parameters ) - self._body_data_interpolator = InterpolatedRequestInputProvider( - config=self.config, request_inputs=self.request_body_data, parameters=parameters - ) - self._body_json_interpolator = InterpolatedRequestInputProvider( + self._body_json_interpolator = InterpolatedNestedRequestInputProvider( config=self.config, request_inputs=self.request_body_json, parameters=parameters ) - @property - def cache_filename(self) -> str: - return f"{self.name}.yml" - - @property - def use_cache(self) -> bool: - return False - - def __hash__(self): - return hash(tuple(self.__dict__)) - - def get_authenticator(self): - return self.authenticator - - def get_url_base(self): - return os.path.join(self.url_base.eval(self.config), "") - - def get_path( - self, *, stream_state: Optional[StreamState], stream_slice: Optional[StreamSlice], next_page_token: Optional[Mapping[str, Any]] - ) -> str: - kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - path = self.path.eval(self.config, **kwargs) - return path.strip("/") - - def get_method(self): - return self._method - # The RateLimiter is applied to balance the api requests. - @lru_cache(maxsize=10) @IntercomRateLimiter.balance_rate_limit() def interpret_response_status(self, response: requests.Response) -> ResponseStatus: # Check for response.headers to define the backoff time before the next api call - return self.error_handler.interpret_response(response) + return super().interpret_response_status(response) def get_request_params( self, @@ -599,15 +365,6 @@ def get_request_headers( ) -> Mapping[str, Any]: return self._headers_interpolator.eval_request_inputs(stream_state, stream_slice, next_page_token) - def get_request_body_data( - self, - *, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Union[Mapping, str]]: - return self._body_data_interpolator.eval_request_inputs(stream_state, stream_slice, next_page_token) - def get_request_body_json( self, *, @@ -616,12 +373,3 @@ def get_request_body_json( next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Mapping]: return self._body_json_interpolator.eval_request_inputs(stream_state, stream_slice, next_page_token) - - def request_kwargs( - self, - *, - stream_state: Optional[StreamState] = None, - stream_slice: Optional[StreamSlice] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: - return {} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml b/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml index a957ff309d9c..a87842f5ef04 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml @@ -1,7 +1,6 @@ -version: "0.1.0" +version: "0.50.2" definitions: - ## bases schema_loader: type: JsonFileSchemaLoader @@ -10,7 +9,7 @@ definitions: description: "Base records selector for Full Refresh streams" extractor: type: DpathExtractor - field_path: ["{{ parameters.get('data_field', 'data')}}"] + field_path: ["{{ parameters.get('data_field', 'data')}}"] requester: description: "Base Requester for Full Refresh streams" type: CustomRequester @@ -21,8 +20,10 @@ definitions: type: BearerAuthenticator api_token: "{{ config['access_token'] }}" request_headers: - Intercom-Version: '2.5' # ATTENTION: API version change is possible here - Accept: 'application/json' + Intercom-Version: "2.5" # ATTENTION: API version change is possible here + Accept: "application/json" + error_handler: + type: "DefaultErrorHandler" retriever: description: "Base Retriever for Full Refresh streams" record_selector: @@ -41,33 +42,33 @@ definitions: inject_into: request_parameter field_name: per_page page_token_option: - type: RequestPath + type: RequestPath requester_incremental_search: $ref: "#/definitions/requester" http_method: "POST" request_body_json: query: "{ - 'operator': 'OR', - 'value': [ - { - 'field': 'updated_at', - 'operator': '>', - 'value': {{ stream_slice.get('prior_state', stream_state.get('prior_state', {})).get('updated_at') or format_datetime(config['start_date'], '%s') }} - }, - { - 'field': 'updated_at', - 'operator': '=', - 'value': {{ stream_slice.get('prior_state', stream_state.get('prior_state', {})).get('updated_at') or format_datetime(config['start_date'], '%s') }} - }, - ], - }" + 'operator': 'OR', + 'value': [ + { + 'field': 'updated_at', + 'operator': '>', + 'value': {{ stream_slice.get('prior_state', stream_state.get('prior_state', {})).get('updated_at') or format_datetime(config['start_date'], '%s') }} + }, + { + 'field': 'updated_at', + 'operator': '=', + 'value': {{ stream_slice.get('prior_state', stream_state.get('prior_state', {})).get('updated_at') or format_datetime(config['start_date'], '%s') }} + }, + ], + }" sort: "{'field': 'updated_at', 'order': 'ascending'}" pagination: "{ 'per_page': {{ parameters.get('page_size') }}, 'page': {{ next_page_token.get('next_page_token').get('page') }}, 'starting_after': '{{ next_page_token.get('next_page_token').get('starting_after') }}' - }" - + }" + ## streams # full-refresh stream_full_refresh: @@ -98,7 +99,7 @@ definitions: primary_key: "name" path: "teams" data_field: "teams" - + stream_data_attributes: description: "https://developers.intercom.com/intercom-api-reference/reference#list-data-attributes" $ref: "#/definitions/stream_full_refresh" @@ -124,15 +125,15 @@ definitions: primary_key: "name" path: "data_attributes" model: "contact" - - # semi-incremental + + # semi-incremental # (full-refresh and emit records >= *prior state) # (prior state - frozen state from previous sync, it automatically updates with next sync) stream_semi_incremental: $ref: "#/definitions/stream_full_refresh" incremental_sync: type: CustomIncrementalSync - class_name: source_intercom.components.IncrementalSingleSlice + class_name: source_intercom.components.IncrementalSingleSliceCursor cursor_field: "updated_at" retriever: $ref: "#/definitions/stream_full_refresh/retriever" @@ -183,7 +184,7 @@ definitions: 500 - server-side error, should retry after 60 sec. " response_filters: - - http_codes: [ 400, 500 ] + - http_codes: [400, 500] action: RETRY backoff_strategies: - type: ConstantBackoffStrategy @@ -191,15 +192,15 @@ definitions: - type: DefaultErrorHandler description: "404 - scroll_param is expired or not found while requesting, ignore" response_filters: - - http_codes: [ 404 ] + - http_codes: [404] action: IGNORE - + # semi-incremental substreams substream_semi_incremental: $ref: "#/definitions/stream_full_refresh" incremental_sync: type: CustomIncrementalSync - class_name: source_intercom.components.IncrementalSubstreamSlicer + class_name: source_intercom.components.IncrementalSubstreamSlicerCursor cursor_field: "updated_at" retriever: $ref: "#/definitions/stream_full_refresh/retriever" @@ -256,7 +257,7 @@ definitions: $ref: "#/definitions/stream_full_refresh" incremental_sync: type: CustomIncrementalSync - class_name: source_intercom.components.IncrementalSingleSlice + class_name: source_intercom.components.IncrementalSingleSliceCursor cursor_field: "updated_at" retriever: $ref: "#/definitions/stream_full_refresh/retriever" @@ -287,7 +288,7 @@ definitions: path: "conversations/search" data_field: "conversations" page_size: 150 - + streams: - "#/definitions/admins" - "#/definitions/tags" @@ -302,5 +303,5 @@ streams: - "#/definitions/company_segments" check: - stream_names: + stream_names: - "tags" diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json index 160befaed83b..d283d90b232f 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json @@ -58,6 +58,23 @@ }, "type": { "type": ["null", "string"] + }, + "team_priority_level": { + "type": ["null", "object"], + "properties": { + "primary_team_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "secondary_team_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json index 9008e0612cee..21a989d14e26 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json @@ -1,6 +1,9 @@ { "type": "object", "properties": { + "id": { + "type": ["null", "integer"] + }, "admin_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json index c31356bdb928..97187685769b 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json @@ -1,6 +1,9 @@ { "type": ["null", "object"], "properties": { + "id": { + "type": ["null", "integer"] + }, "type": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json index cf56585f6629..e669fa2f0fee 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -304,6 +304,58 @@ "type": ["null", "boolean"] } } + }, + "opted_in_subscription_types": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + }, + "url": { + "type": ["null", "string"] + }, + "total_count": { + "type": ["null", "integer"] + }, + "has_more": { + "type": ["null", "boolean"] + } + } + }, + "utm_content": { + "type": ["null", "string"] + }, + "utm_campaign": { + "type": ["null", "string"] + }, + "utm_source": { + "type": ["null", "string"] + }, + "referrer": { + "type": ["null", "string"] + }, + "utm_term": { + "type": ["null", "string"] + }, + "utm_medium": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json index 0a041ca940c3..1123a72ee24a 100755 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json @@ -428,6 +428,34 @@ }, "redacted": { "type": ["null", "boolean"] + }, + "topics": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "topics": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + }, + "total_count": { + "type": ["null", "integer"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json index 54a064248367..0567220eb6b0 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json @@ -20,6 +20,18 @@ "type": "string", "description": "Access token for making authenticated requests. See the Intercom docs for more information.", "airbyte_secret": true + }, + "client_id": { + "title": "Client Id", + "type": "string", + "description": "Client Id for your Intercom application.", + "airbyte_secret": true + }, + "client_secret": { + "title": "Client Secret", + "type": "string", + "description": "Client Secret for your Intercom application.", + "airbyte_secret": true } } }, diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py new file mode 100644 index 000000000000..455793c8e7de --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, Mock + +import pytest +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.streams import Stream +from source_intercom.components import IncrementalSingleSliceCursor, IncrementalSubstreamSlicerCursor + + +def test_slicer(): + date_time_dict = {"updated_at": 1662459010} + slicer = IncrementalSingleSliceCursor(config={}, parameters={}, cursor_field="updated_at") + slicer.close_slice(date_time_dict, date_time_dict) + assert slicer.get_stream_state() == date_time_dict + assert slicer.get_request_headers() == {} + assert slicer.get_request_body_data() == {} + assert slicer.get_request_body_json() == {} + + +@pytest.mark.parametrize( + "last_record, expected, records", + [ + ( + {"first_stream_cursor": 1662459010}, + {'first_stream_cursor': 1662459010, 'prior_state': {'first_stream_cursor': 1662459010, 'parent_stream_name': {'parent_cursor_field': 1662459010}}, 'parent_stream_name': {'parent_cursor_field': 1662459010}}, + [{"first_stream_cursor": 1662459010}], + ) + ], +) +def test_sub_slicer(last_record, expected, records): + parent_stream = Mock(spec=Stream) + parent_stream.name = "parent_stream_name" + parent_stream.cursor_field = "parent_cursor_field" + parent_stream.state = {"parent_stream_name": {"parent_cursor_field": 1662459010}} + parent_stream.stream_slices.return_value = [{"a slice": "value"}] + parent_stream.read_records = MagicMock(return_value=records) + + parent_config = ParentStreamConfig( + stream=parent_stream, + parent_key="first_stream_cursor", + partition_field="first_stream_id", + parameters={}, + config={}, + ) + + slicer = IncrementalSubstreamSlicerCursor( + config={}, parameters={}, cursor_field="first_stream_cursor", parent_stream_configs=[parent_config], parent_complete_fetch=True + ) + slicer.set_initial_state(expected) + stream_slice = next(slicer.stream_slices()) if records else {} + slicer.close_slice(stream_slice, last_record) + assert slicer.get_stream_state() == expected diff --git a/airbyte-integrations/connectors/source-intruder/metadata.yaml b/airbyte-integrations/connectors/source-intruder/metadata.yaml index 86d17ed12d49..a70265c693ff 100644 --- a/airbyte-integrations/connectors/source-intruder/metadata.yaml +++ b/airbyte-integrations/connectors/source-intruder/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-intruder/requirements.txt b/airbyte-integrations/connectors/source-intruder/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-intruder/requirements.txt +++ b/airbyte-integrations/connectors/source-intruder/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-intruder/setup.py b/airbyte-integrations/connectors/source-intruder/setup.py index 65ceaa4fe994..67cb265463db 100644 --- a/airbyte-integrations/connectors/source-intruder/setup.py +++ b/airbyte-integrations/connectors/source-intruder/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-ip2whois/metadata.yaml b/airbyte-integrations/connectors/source-ip2whois/metadata.yaml index c63ce49fb9b0..a14ed278b363 100644 --- a/airbyte-integrations/connectors/source-ip2whois/metadata.yaml +++ b/airbyte-integrations/connectors/source-ip2whois/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-ip2whois/requirements.txt b/airbyte-integrations/connectors/source-ip2whois/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-ip2whois/requirements.txt +++ b/airbyte-integrations/connectors/source-ip2whois/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-ip2whois/setup.py b/airbyte-integrations/connectors/source-ip2whois/setup.py index a57756c47152..92303a3f0ea4 100644 --- a/airbyte-integrations/connectors/source-ip2whois/setup.py +++ b/airbyte-integrations/connectors/source-ip2whois/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index ed6f0f878f59..d05453734efe 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.28 +LABEL io.airbyte.version=0.1.30 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/metadata.yaml b/airbyte-integrations/connectors/source-iterable/metadata.yaml index 79d9870e77be..3df40f4c8d7b 100644 --- a/airbyte-integrations/connectors/source-iterable/metadata.yaml +++ b/airbyte-integrations/connectors/source-iterable/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 - dockerImageTag: 0.1.28 + dockerImageTag: 0.1.30 dockerRepository: airbyte/source-iterable githubIssueLabel: source-iterable icon: iterable.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/iterable tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-iterable/requirements.txt b/airbyte-integrations/connectors/source-iterable/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-iterable/requirements.txt +++ b/airbyte-integrations/connectors/source-iterable/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-iterable/setup.py b/airbyte-integrations/connectors/source-iterable/setup.py index 00a468461135..5d2e499d31c7 100644 --- a/airbyte-integrations/connectors/source-iterable/setup.py +++ b/airbyte-integrations/connectors/source-iterable/setup.py @@ -12,7 +12,7 @@ "requests~=2.25", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "responses==0.23.1", "freezegun==1.1.0"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "responses==0.23.1", "freezegun==1.1.0"] setup( diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py b/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py index f6892bc0f4ec..7ef95649a827 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/slice_generators.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import dataclasses import math from dataclasses import dataclass from typing import Iterable, List, Optional, Tuple @@ -15,6 +16,9 @@ class StreamSlice: start_date: DateTime end_date: DateTime + def __iter__(self): + return ((field.name, getattr(self, field.name)) for field in dataclasses.fields(self)) + class SliceGenerator: """ diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py index 8cc9bc44506d..177c7987dd38 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -88,6 +88,18 @@ def check_generic_error(self, response: requests.Response) -> bool: if response_json.get("code") in codes and msg_pattern in response_json.get("msg", ""): return True + def request_kwargs( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Mapping[str, Any]: + """ + https://requests.readthedocs.io/en/latest/user/advanced/#timeouts + https://github.com/airbytehq/oncall/issues/1985#issuecomment-1559276465 + """ + return {"timeout": (60, 300)} + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() or {} records = response_json.get(self.data_field, []) @@ -216,7 +228,10 @@ def request_kwargs( Passing stream=True argument to requests.session.send method to avoid loading whole analytics report content into memory. """ - return {"stream": True} + return { + **super().request_kwargs(stream_state, stream_slice, next_page_token), + "stream": True, + } def get_start_date(self, stream_state: Mapping[str, Any]) -> DateTime: stream_state = stream_state or {} diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py index e38365bcc412..cbf7c611059b 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py @@ -3,6 +3,7 @@ # import json +from unittest.mock import MagicMock import pendulum import pytest @@ -249,3 +250,19 @@ def get_body(emails): records = list(read_full_refresh(stream)) assert [r["email"] for r in records] == ['user1', 'user2', 'user3', 'user5'] assert m.call_count == 3 + + +def test_retry_read_timeout(): + stream = Lists(authenticator=None) + stream._session.send = MagicMock(side_effect=requests.exceptions.ReadTimeout) + with pytest.raises(requests.exceptions.ReadTimeout): + list(read_full_refresh(stream)) + stream._session.send.call_args[1] == {'timeout': (60, 300)} + assert stream._session.send.call_count == stream.max_retries + 1 + + stream = Campaigns(authenticator=None) + stream._session.send = MagicMock(side_effect=requests.exceptions.ConnectionError) + with pytest.raises(requests.exceptions.ConnectionError): + list(read_full_refresh(stream)) + stream._session.send.call_args[1] == {'timeout': (60, 300)} + assert stream._session.send.call_count == stream.max_retries + 1 diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 7db253375288..aa067a77a587 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -21,12 +21,14 @@ import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_TYPE_NAME; import static io.airbyte.db.jdbc.JdbcConstants.JDBC_DECIMAL_DIGITS; import static io.airbyte.db.jdbc.JdbcConstants.JDBC_IS_NULLABLE; +import static io.airbyte.db.jdbc.JdbcConstants.KEY_SEQ; import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; @@ -59,10 +61,10 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -110,7 +112,8 @@ protected AutoCloseableIterator queryTableFullRefresh(final JdbcDataba final SyncMode syncMode, final Optional cursorField) { LOGGER.info("Queueing query for table: {}", tableName); - // This corresponds to the initial sync for in INCREMENTAL_MODE, where the ordering of the records matters + // This corresponds to the initial sync for in INCREMENTAL_MODE, where the ordering of the records + // matters // as intermediate state messages are emitted (if the connector emits intermediate state). if (syncMode.equals(SyncMode.INCREMENTAL) && getStateEmissionFrequency() > 0) { final String quotedCursorField = enquoteIdentifier(cursorField.get(), getQuoteString()); @@ -145,13 +148,14 @@ protected List> getCheckOperations(fina * * @return a map by StreamName to associated list of primary keys */ - private static Map> aggregatePrimateKeys(final List> entries) { + @VisibleForTesting + public static Map> aggregatePrimateKeys(final List entries) { final Map> result = new HashMap<>(); - entries.forEach(entry -> { - if (!result.containsKey(entry.getKey())) { - result.put(entry.getKey(), new ArrayList<>()); + entries.stream().sorted(Comparator.comparingInt(PrimaryKeyAttributesFromDb::keySequence)).forEach(entry -> { + if (!result.containsKey(entry.streamName())) { + result.put(entry.streamName(), new ArrayList<>()); } - result.get(entry.getKey()).add(entry.getValue()); + result.get(entry.streamName()).add(entry.primaryKey()); }); return result; } @@ -260,6 +264,13 @@ public JsonSchemaType getAirbyteType(final Datatype columnType) { return sourceOperations.getAirbyteType(columnType); } + @VisibleForTesting + public record PrimaryKeyAttributesFromDb(String streamName, + String primaryKey, + int keySequence) { + + } + @Override protected Map> discoverPrimaryKeys(final JdbcDatabase database, final List>> tableInfos) { @@ -274,7 +285,8 @@ protected Map> discoverPrimaryKeys(final JdbcDatabase datab r.getObject(JDBC_COLUMN_SCHEMA_NAME) != null ? r.getString(JDBC_COLUMN_SCHEMA_NAME) : r.getString(JDBC_COLUMN_DATABASE_NAME); final String streamName = JdbcUtils.getFullyQualifiedTableName(schemaName, r.getString(JDBC_COLUMN_TABLE_NAME)); final String primaryKey = r.getString(JDBC_COLUMN_COLUMN_NAME); - return new SimpleImmutableEntry<>(streamName, primaryKey); + final int keySeq = r.getInt(KEY_SEQ); + return new PrimaryKeyAttributesFromDb(streamName, primaryKey, keySeq); })); if (!tablePrimaryKeys.isEmpty()) { return tablePrimaryKeys; @@ -291,7 +303,7 @@ protected Map> discoverPrimaryKeys(final JdbcDatabase datab try { final Map> primaryKeys = aggregatePrimateKeys(database.bufferedResultSetQuery( connection -> connection.getMetaData().getPrimaryKeys(getCatalog(database), tableInfo.getNameSpace(), tableInfo.getName()), - r -> new SimpleImmutableEntry<>(streamName, r.getString(JDBC_COLUMN_COLUMN_NAME)))); + r -> new PrimaryKeyAttributesFromDb(streamName, r.getString(JDBC_COLUMN_COLUMN_NAME), r.getInt(KEY_SEQ)))); return primaryKeys.getOrDefault(streamName, Collections.emptyList()); } catch (final SQLException e) { LOGGER.error(String.format("Could not retrieve primary keys for %s: %s", streamName, e)); @@ -476,6 +488,7 @@ protected List identifyStreamsToSnapshot(final Configur final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) .map(Jsons::clone) .collect(Collectors.toList()); diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index bba9eca3db41..3f4c420cac77 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -944,7 +944,7 @@ public void testIncrementalWithConcurrentInsertion() throws Exception { } - private JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { + protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { for (final JsonNode stream : airbyteMessage.getState().getData().get("streams")) { if (stream.get("stream_name").asText().equals(streamName)) { return stream; @@ -965,13 +965,13 @@ private void incrementalCursorCheck( getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0)); } - private void incrementalCursorCheck( - final String initialCursorField, - final String cursorField, - final String initialCursorValue, - final String endCursorValue, - final List expectedRecordMessages, - final ConfiguredAirbyteStream airbyteStream) + protected void incrementalCursorCheck( + final String initialCursorField, + final String cursorField, + final String initialCursorValue, + final String endCursorValue, + final List expectedRecordMessages, + final ConfiguredAirbyteStream airbyteStream) throws Exception { airbyteStream.setSyncMode(SyncMode.INCREMENTAL); airbyteStream.setCursorField(List.of(cursorField)); @@ -980,25 +980,15 @@ private void incrementalCursorCheck( final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog() .withStreams(List.of(airbyteStream)); - final DbStreamState dbStreamState = new DbStreamState() - .withStreamName(airbyteStream.getStream().getName()) - .withStreamNamespace(airbyteStream.getStream().getNamespace()) - .withCursorField(List.of(initialCursorField)) - .withCursor(initialCursorValue) - .withCursorRecordCount(1L); + final DbStreamState dbStreamState = buildStreamState(airbyteStream, initialCursorField, initialCursorValue); final List actualMessages = MoreIterators .toList(source.read(config, configuredCatalog, Jsons.jsonNode(createState(List.of(dbStreamState))))); setEmittedAtToNull(actualMessages); - final List expectedStreams = List.of( - new DbStreamState() - .withStreamName(airbyteStream.getStream().getName()) - .withStreamNamespace(airbyteStream.getStream().getNamespace()) - .withCursorField(List.of(cursorField)) - .withCursor(endCursorValue) - .withCursorRecordCount(1L)); + final List expectedStreams = List.of(buildStreamState(airbyteStream, cursorField, endCursorValue)); + final List expectedMessages = new ArrayList<>(expectedRecordMessages); expectedMessages.addAll(createExpectedTestMessages(expectedStreams)); @@ -1007,6 +997,17 @@ private void incrementalCursorCheck( assertTrue(actualMessages.containsAll(expectedMessages)); } + protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configuredAirbyteStream, + final String cursorField, + final String cursorValue) { + return new DbStreamState() + .withStreamName(configuredAirbyteStream.getStream().getName()) + .withStreamNamespace(configuredAirbyteStream.getStream().getNamespace()) + .withCursorField(List.of(cursorField)) + .withCursor(cursorValue) + .withCursorRecordCount(1L); + } + // get catalog and perform a defensive copy. protected ConfiguredAirbyteCatalog getConfiguredCatalogWithOneStream(final String defaultNamespace) { final ConfiguredAirbyteCatalog catalog = CatalogHelpers.toDefaultConfiguredCatalog(getCatalog(defaultNamespace)); diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile b/airbyte-integrations/connectors/source-jira/Dockerfile index b101ad3f513f..5a183bdb9b25 100644 --- a/airbyte-integrations/connectors/source-jira/Dockerfile +++ b/airbyte-integrations/connectors/source-jira/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.11 +LABEL io.airbyte.version=0.3.12 LABEL io.airbyte.name=airbyte/source-jira diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json index 2c71bed4f0a5..41150ae12b29 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json @@ -4,9 +4,7 @@ "stream": { "name": "application_roles", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["key"]] }, "sync_mode": "full_refresh", @@ -16,9 +14,7 @@ "stream": { "name": "avatars", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -28,9 +24,7 @@ "stream": { "name": "boards", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -40,14 +34,9 @@ "stream": { "name": "board_issues", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated" - ], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", @@ -57,9 +46,7 @@ "stream": { "name": "dashboards", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -69,9 +56,7 @@ "stream": { "name": "filters", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -81,9 +66,7 @@ "stream": { "name": "filter_sharing", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -93,9 +76,7 @@ "stream": { "name": "groups", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["groupId"]] }, "sync_mode": "full_refresh", @@ -105,14 +86,9 @@ "stream": { "name": "issues", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated" - ], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", @@ -122,14 +98,9 @@ "stream": { "name": "issue_comments", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated" - ], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", @@ -139,9 +110,7 @@ "stream": { "name": "issue_fields", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -151,9 +120,7 @@ "stream": { "name": "issue_field_configurations", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -163,9 +130,7 @@ "stream": { "name": "issue_custom_field_contexts", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -175,9 +140,7 @@ "stream": { "name": "issue_link_types", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -187,9 +150,7 @@ "stream": { "name": "issue_navigator_settings", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append" @@ -198,9 +159,7 @@ "stream": { "name": "issue_notification_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -210,9 +169,7 @@ "stream": { "name": "issue_priorities", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -222,9 +179,7 @@ "stream": { "name": "issue_properties", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["key"]] }, "sync_mode": "full_refresh", @@ -234,9 +189,7 @@ "stream": { "name": "issue_remote_links", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -246,9 +199,7 @@ "stream": { "name": "issue_resolutions", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -258,9 +209,7 @@ "stream": { "name": "issue_security_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -270,9 +219,7 @@ "stream": { "name": "issue_type_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -282,9 +229,7 @@ "stream": { "name": "issue_type_screen_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -294,9 +239,7 @@ "stream": { "name": "issue_votes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append" @@ -305,9 +248,7 @@ "stream": { "name": "issue_watchers", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append" @@ -316,14 +257,9 @@ "stream": { "name": "issue_worklogs", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated" - ], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", @@ -333,9 +269,7 @@ "stream": { "name": "jira_settings", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -345,9 +279,7 @@ "stream": { "name": "labels", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["label"]] }, "sync_mode": "full_refresh", @@ -357,9 +289,7 @@ "stream": { "name": "permissions", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["key"]] }, "sync_mode": "full_refresh", @@ -369,9 +299,7 @@ "stream": { "name": "permission_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -381,9 +309,7 @@ "stream": { "name": "projects", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -393,9 +319,7 @@ "stream": { "name": "project_avatars", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -405,9 +329,7 @@ "stream": { "name": "project_categories", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -417,9 +339,7 @@ "stream": { "name": "project_components", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -429,9 +349,7 @@ "stream": { "name": "project_email", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["projectId"]] }, "sync_mode": "full_refresh", @@ -441,9 +359,7 @@ "stream": { "name": "project_permission_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -453,9 +369,7 @@ "stream": { "name": "project_types", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append" @@ -464,9 +378,7 @@ "stream": { "name": "project_versions", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -476,9 +388,7 @@ "stream": { "name": "screens", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -488,9 +398,7 @@ "stream": { "name": "screen_tabs", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -500,9 +408,7 @@ "stream": { "name": "screen_tab_fields", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -512,9 +418,7 @@ "stream": { "name": "screen_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -524,9 +428,7 @@ "stream": { "name": "sprints", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -536,14 +438,9 @@ "stream": { "name": "sprint_issues", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated" - ], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", @@ -553,9 +450,7 @@ "stream": { "name": "time_tracking", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["key"]] }, "sync_mode": "full_refresh", @@ -565,9 +460,7 @@ "stream": { "name": "users", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["accountId"]] }, "sync_mode": "full_refresh", @@ -577,9 +470,7 @@ "stream": { "name": "users_groups_detailed", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["accountId"]] }, "sync_mode": "full_refresh", @@ -589,9 +480,7 @@ "stream": { "name": "workflows", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -601,9 +490,7 @@ "stream": { "name": "workflow_schemes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -613,9 +500,7 @@ "stream": { "name": "workflow_statuses", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -625,9 +510,7 @@ "stream": { "name": "workflow_status_categories", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl index ec0f7ed48f0f..8b1db037ebac 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl @@ -5,7 +5,6 @@ {"stream": "avatars", "data": {"id": "10304", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10304", "24x24": "/secure/useravatar?size=small&avatarId=10304", "32x32": "/secure/useravatar?size=medium&avatarId=10304", "48x48": "/secure/useravatar?avatarId=10304"}}, "emitted_at": 1685112891086} {"stream": "boards", "data": {"id": 1, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/1", "name": "IT board", "type": "scrum", "location": {"projectId": 10000, "displayName": "integration-tests (IT)", "projectName": "integration-tests", "projectKey": "IT", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10424?size=small", "name": "integration-tests (IT)"}, "projectId": "10000", "projectKey": "IT"}, "emitted_at": 1685112893105} {"stream": "boards", "data": {"id": 17, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/17", "name": "TESTKEY13 board", "type": "scrum", "location": {"projectId": 10016, "displayName": "Test project 13 (TESTKEY13)", "projectName": "Test project 13", "projectKey": "TESTKEY13", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10425?size=small", "name": "Test project 13 (TESTKEY13)"}, "projectId": "10016", "projectKey": "TESTKEY13"}, "emitted_at": 1685112893106} -{"stream": "boards", "data": {"id": 58, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/58", "name": "TTMP2 board", "type": "simple", "location": {"projectId": 10064, "displayName": "Test Team Managed Project 2 (TTMP2)", "projectName": "Test Team Managed Project 2", "projectKey": "TTMP2", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10412?size=small", "name": "Test Team Managed Project 2 (TTMP2)"}, "projectId": "10064", "projectKey": "TTMP2"}, "emitted_at": 1685112893453} {"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10012", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10012", "key": "IT-6", "fields": {"updated": "2022-05-17T04:26:21.613-0700", "created": "2021-03-11T06:14:18.085-0800"}, "boardId": 1, "created": "2021-03-11T06:14:18.085-0800", "updated": "2022-05-17T04:26:21.613-0700"}, "emitted_at": 1685112894918} {"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10019", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10019", "key": "IT-9", "fields": {"updated": "2023-04-05T04:57:18.118-0700", "created": "2021-03-11T06:14:24.791-0800"}, "boardId": 1, "created": "2021-03-11T06:14:24.791-0800", "updated": "2023-04-05T04:57:18.118-0700"}, "emitted_at": 1685112894919} {"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10000", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10000", "key": "IT-1", "fields": {"updated": "2022-05-17T04:26:28.885-0700", "created": "2020-12-07T06:12:17.863-0800"}, "boardId": 1, "created": "2020-12-07T06:12:17.863-0800", "updated": "2022-05-17T04:26:28.885-0700"}, "emitted_at": 1685112894919} @@ -21,9 +20,8 @@ {"stream": "groups", "data": {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87"}, "emitted_at": 1685112927902} {"stream": "groups", "data": {"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1"}, "emitted_at": 1685112927903} {"stream": "groups", "data": {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, "emitted_at": 1685112927903} -{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10632", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10632", "key": "TTMP2-1", "fields": {"statuscategorychangedate": "2023-05-04T03:00:07.588-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10019", "id": "10019", "description": "Stories track functionality or features expressed as user goals.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10315?size=medium", "name": "Story", "subtask": false, "avatarId": 10315, "entityId": "bcca552a-8575-4810-bb17-af1c361551ae", "hierarchyLevel": 0}, "timespent": 12000, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10064", "id": "10064", "key": "TTMP2", "name": "Test Team Managed Project 2", "projectTypeKey": "software", "simplified": true, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "description": "Category 1", "name": "Category 1"}}, "fixVersions": [], "aggregatetimespent": 12000, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10029": null, "resolutiondate": null, "workratio": -1, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": true}, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TTMP2-1/watchers", "watchCount": 1, "isWatching": true}, "lastViewed": "2023-05-11T02:37:05.241-0700", "customfield_10181": null, "created": "2023-05-04T03:00:07.323-0700", "customfield_10020": [{"id": 12, "name": "TTMP2 Sprint 1", "state": "active", "boardId": 58, "goal": "Test", "startDate": "2023-05-04T16:18:15.222Z", "endDate": "2023-05-18T16:18:04.000Z"}], "customfield_10021": null, "customfield_10022": null, "customfield_10220": null, "customfield_10221": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/5", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/lowest.svg", "name": "Lowest", "id": "5"}, "customfield_10023": null, "customfield_10024": null, "customfield_10222": null, "customfield_10223": null, "customfield_10025": null, "customfield_10224": null, "labels": ["Label8"], "customfield_10016": null, "customfield_10214": null, "customfield_10017": null, "customfield_10215": null, "customfield_10216": null, "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i0078f:", "customfield_10218": null, "timeestimate": 0, "aggregatetimeoriginalestimate": null, "versions": [], "customfield_10219": null, "issuelinks": [], "assignee": null, "updated": "2023-05-08T03:04:45.139-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10017", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10017", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test description"}]}]}, "customfield_10010": null, "customfield_10210": null, "customfield_10211": null, "customfield_10014": null, "customfield_10212": null, "customfield_10015": null, "customfield_10213": null, "timetracking": {"remainingEstimate": "0m", "timeSpent": "3h 20m", "remainingEstimateSeconds": 0, "timeSpentSeconds": 12000}, "customfield_10005": null, "customfield_10006": null, "security": null, "customfield_10007": null, "customfield_10008": null, "customfield_10009": null, "attachment": [], "aggregatetimeestimate": 0, "customfield_10209": null, "summary": "Sandbox account access", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 12000, "total": 12000, "percent": 100}, "customfield_10001": null, "customfield_10002": null, "customfield_10003": null, "customfield_10047": null, "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 12000, "total": 12000, "percent": 100}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TTMP2-1/votes", "votes": 0, "hasVoted": false}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10632/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10632/worklog/11822", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"content": [{"content": [{"text": "I did some work here.", "type": "text"}], "type": "paragraph"}], "type": "doc", "version": 1}, "created": "2023-05-08T03:04:45.056-0700", "updated": "2023-05-08T03:04:45.056-0700", "started": "2021-01-17T04:34:00.000-0800", "timeSpent": "3h 20m", "timeSpentSeconds": 12000, "id": "11822", "issueId": "10632"}]}}, "projectId": "10064", "projectKey": "TTMP2", "created": "2023-05-04T03:00:07.323-0700", "updated": "2023-05-08T03:04:45.139-0700"}, "emitted_at": 1685112931676} -{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10627", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627", "key": "TESTKEY13-1", "fields": {"statuscategorychangedate": "2022-06-09T16:29:32.382-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": null, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "name": "Test project 13", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "description": "Category 1", "name": "Category 1"}}, "fixVersions": [{"self": "https://airbyteio.atlassian.net/rest/api/3/version/10066", "id": "10066", "description": "An excellent version", "name": "New Version 1", "archived": false, "released": true, "releaseDate": "2010-07-06"}], "aggregatetimespent": null, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10029": null, "resolutiondate": null, "workratio": -1, "lastViewed": "2023-04-05T01:34:42.647-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "customfield_10181": null, "created": "2022-06-09T16:29:31.871-0700", "customfield_10020": [{"id": 2, "name": "IT Sprint 1", "state": "active", "boardId": 1, "goal": "Deliver results", "startDate": "2022-05-17T11:25:59.072Z", "endDate": "2022-05-31T11:25:00.000Z"}], "customfield_10021": null, "customfield_10220": null, "customfield_10022": null, "customfield_10221": null, "customfield_10023": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "customfield_10222": null, "customfield_10024": null, "customfield_10223": null, "customfield_10025": null, "labels": ["test"], "customfield_10224": null, "customfield_10026": 3.0, "customfield_10016": null, "customfield_10214": null, "customfield_10215": null, "customfield_10017": "dark_orange", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10216": null, "customfield_10019": "0|i0077b:", "customfield_10217": [], "timeestimate": null, "customfield_10218": null, "aggregatetimeoriginalestimate": null, "customfield_10219": null, "versions": [], "issuelinks": [], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-04-04T04:36:21.195-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10065", "id": "10065", "name": "Component 0", "description": "This is a Jira component"}], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test issue"}]}]}, "customfield_10010": null, "customfield_10011": "EPIC NAME TEXT", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10210": null, "customfield_10013": "ghx-label-14", "customfield_10211": null, "customfield_10014": null, "customfield_10212": null, "customfield_10015": null, "customfield_10213": null, "timetracking": {}, "customfield_10005": null, "customfield_10006": null, "customfield_10007": null, "security": null, "customfield_10008": null, "customfield_10009": "2022-12-09T00:00:00.000-0800", "aggregatetimeestimate": null, "attachment": [], "customfield_10209": null, "summary": "My Summary", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "customfield_10001": null, "customfield_10002": null, "customfield_10003": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}], "customfield_10047": null, "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 0, "total": 0}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10016", "projectKey": "TESTKEY13", "created": "2022-06-09T16:29:31.871-0700", "updated": "2023-04-04T04:36:21.195-0700"}, "emitted_at": 1685112931677} -{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10626", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "key": "IT-26", "fields": {"statuscategorychangedate": "2022-05-17T04:28:19.775-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": 28800, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 28800, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10029": null, "resolutiondate": null, "workratio": -1, "lastViewed": "2023-05-04T09:52:20.143-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "customfield_10181": null, "created": "2022-05-17T04:28:19.523-0700", "customfield_10020": null, "customfield_10021": null, "customfield_10022": null, "customfield_10220": null, "customfield_10023": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "customfield_10221": null, "customfield_10024": null, "customfield_10222": null, "customfield_10025": null, "customfield_10223": null, "customfield_10026": null, "customfield_10224": null, "labels": [], "customfield_10016": null, "customfield_10214": null, "customfield_10215": null, "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10216": null, "customfield_10217": [], "customfield_10019": "0|i00773:", "aggregatetimeoriginalestimate": null, "timeestimate": null, "customfield_10218": null, "versions": [], "customfield_10219": null, "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "outwardIssue": {"id": "10625", "key": "IT-25", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "fields": {"summary": "Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-04-05T05:08:50.112-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10010": null, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10210": null, "customfield_10013": "ghx-label-2", "customfield_10211": null, "customfield_10212": null, "customfield_10014": null, "customfield_10015": null, "timetracking": {"timeSpent": "1d", "timeSpentSeconds": 28800}, "customfield_10213": null, "customfield_10005": null, "customfield_10006": null, "customfield_10007": null, "security": null, "customfield_10008": null, "attachment": [], "customfield_10009": null, "aggregatetimeestimate": null, "customfield_10209": null, "summary": "CLONE - Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 28800, "total": 28800, "percent": 100}, "customfield_10001": null, "customfield_10002": null, "customfield_10003": null, "customfield_10047": null, "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 28800, "total": 28800, "percent": 100}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:28:19.523-0700", "updated": "2023-04-05T05:08:50.112-0700"}, "emitted_at": 1685112931678} +{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10627", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627", "key": "TESTKEY13-1", "fields": {"statuscategorychangedate": "2022-06-09T16:29:32.382-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": null, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "name": "Test project 13", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "description": "Category 1", "name": "Category 1"}}, "fixVersions": [{"self": "https://airbyteio.atlassian.net/rest/api/3/version/10066", "id": "10066", "description": "An excellent version", "name": "New Version 1", "archived": false, "released": true, "releaseDate": "2010-07-06"}], "aggregatetimespent": null, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10227": null, "customfield_10029": null, "customfield_10228": null, "resolutiondate": null, "workratio": -1, "lastViewed": "2023-07-05T12:49:36.121-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "customfield_10181": null, "created": "2022-06-09T16:29:31.871-0700", "customfield_10020": [{"id": 2, "name": "IT Sprint 1", "state": "active", "boardId": 1, "goal": "Deliver results", "startDate": "2022-05-17T11:25:59.072Z", "endDate": "2022-05-31T11:25:00.000Z"}], "customfield_10021": null, "customfield_10022": null, "customfield_10220": null, "customfield_10221": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "customfield_10023": null, "customfield_10222": null, "customfield_10024": null, "customfield_10025": null, "customfield_10223": null, "labels": ["test"], "customfield_10026": 3.0, "customfield_10224": null, "customfield_10016": null, "customfield_10214": null, "customfield_10017": "dark_orange", "customfield_10215": null, "customfield_10216": null, "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i0077b:", "customfield_10218": null, "timeestimate": null, "aggregatetimeoriginalestimate": null, "versions": [], "customfield_10219": null, "issuelinks": [], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-04-04T04:36:21.195-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10065", "id": "10065", "name": "Component 0", "description": "This is a Jira component"}], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test issue"}]}]}, "customfield_10010": null, "customfield_10011": "EPIC NAME TEXT", "customfield_10210": null, "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-14", "customfield_10211": null, "customfield_10014": null, "customfield_10212": null, "customfield_10015": null, "customfield_10213": null, "timetracking": {}, "customfield_10005": null, "customfield_10006": null, "customfield_10007": null, "security": null, "customfield_10008": null, "attachment": [], "aggregatetimeestimate": null, "customfield_10009": "2022-12-09T00:00:00.000-0800", "customfield_10209": null, "summary": "My Summary", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "customfield_10001": null, "customfield_10002": null, "customfield_10047": null, "customfield_10003": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}], "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 0, "total": 0}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10016", "projectKey": "TESTKEY13", "created": "2022-06-09T16:29:31.871-0700", "updated": "2023-04-04T04:36:21.195-0700"}, "emitted_at": 1690193760166} +{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10625", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "key": "IT-25", "fields": {"statuscategorychangedate": "2022-05-17T04:06:24.675-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": null, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": null, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10029": null, "customfield_10227": null, "customfield_10228": null, "resolutiondate": null, "workratio": -1, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "lastViewed": null, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/watchers", "watchCount": 1, "isWatching": true}, "customfield_10181": null, "created": "2022-05-17T04:06:24.048-0700", "customfield_10020": null, "customfield_10021": null, "customfield_10220": null, "customfield_10022": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "customfield_10221": null, "customfield_10023": null, "customfield_10024": null, "customfield_10222": null, "customfield_10223": null, "customfield_10025": null, "customfield_10224": null, "labels": [], "customfield_10026": null, "customfield_10016": null, "customfield_10214": null, "customfield_10017": "dark_yellow", "customfield_10215": null, "customfield_10216": null, "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i0076v:", "customfield_10217": [], "customfield_10218": null, "aggregatetimeoriginalestimate": null, "timeestimate": null, "versions": [], "customfield_10219": null, "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "inwardIssue": {"id": "10626", "key": "IT-26", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "fields": {"summary": "CLONE - Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2022-05-17T04:28:19.876-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10010": null, "customfield_10011": "Test 2", "customfield_10210": null, "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10211": null, "customfield_10013": "ghx-label-2", "customfield_10212": null, "customfield_10014": null, "customfield_10015": null, "customfield_10213": null, "timetracking": {}, "customfield_10005": null, "customfield_10006": null, "customfield_10007": null, "security": null, "customfield_10008": null, "attachment": [], "customfield_10009": null, "aggregatetimeestimate": null, "customfield_10209": null, "summary": "Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "customfield_10001": null, "customfield_10002": null, "customfield_10047": null, "customfield_10003": null, "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 0, "total": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/votes", "votes": 0, "hasVoted": false}, "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:06:24.048-0700", "updated": "2022-05-17T04:28:19.876-0700"}, "emitted_at": 1690193759636} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}, "emitted_at": 1685112937324} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10521", "id": "10521", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque eget venenatis elit. Duis eu justo eget augue iaculis fermentum. Sed semper quam laoreet nisi egestas at posuere augue semper.", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:32:43.099-0700", "updated": "2021-04-14T14:32:43.099-0700", "jsdPublic": true}, "emitted_at": 1685112937947} {"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10639", "id": "10639", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Linked related issue!", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.998-0700", "updated": "2021-04-15T00:08:48.998-0700", "jsdPublic": true}, "emitted_at": 1685112937947} @@ -66,15 +64,16 @@ {"stream": "issue_type_screen_schemes", "data": {"id": "1", "name": "Default Issue Type Screen Scheme", "description": "The default issue type screen scheme"}, "emitted_at": 1685113000857} {"stream": "issue_type_screen_schemes", "data": {"id": "10000", "name": "IT: Scrum Issue Type Screen Scheme", "description": ""}, "emitted_at": 1685113000858} {"stream": "issue_type_screen_schemes", "data": {"id": "10001", "name": "P2: Scrum Issue Type Screen Scheme", "description": ""}, "emitted_at": 1685113000859} -{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TTMP2-1/votes", "votes": 0, "hasVoted": false, "voters": []}, "emitted_at": 1685113005352} -{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/votes", "votes": 0, "hasVoted": false, "voters": []}, "emitted_at": 1685113005642} -{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false, "voters": []}, "emitted_at": 1685113005933} -{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TTMP2-1/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1685113017332} -{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1685113017708} -{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1685113018000} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10632/worklog/11822", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"content": [{"content": [{"text": "I did some work here.", "type": "text"}], "type": "paragraph"}], "type": "doc", "version": 1}, "created": "2023-05-08T03:04:45.056-0700", "updated": "2023-05-08T03:04:45.056-0700", "started": "2021-01-17T04:34:00.000-0800", "timeSpent": "3h 20m", "timeSpentSeconds": 12000, "id": "11822", "issueId": "10632"}, "emitted_at": 1685113032143} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}, "emitted_at": 1685113032862} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574-0700", "updated": "2021-04-15T11:39:46.574-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, "emitted_at": 1685113033429} +{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-2/votes", "votes": 1, "hasVoted": true, "voters": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}]}, "emitted_at": 1686151528794} +{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-1/votes", "votes": 1, "hasVoted": true, "voters": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}]}, "emitted_at": 1686151529331} +{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/votes", "votes": 0, "hasVoted": false, "voters": []}, "emitted_at": 1686151530871} +{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-2/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1686152488275} +{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-1/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1686152488658} +{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1686152489518} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}, "emitted_at": 1686153001542} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, "emitted_at": 1686153002291} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/worklog/11711", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:48.447-0700", "updated": "2021-04-15T11:39:48.447-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "1h 28m", "timeSpentSeconds": 5280, "id": "11711", "issueId": "10075"}, "emitted_at": 1686153002726} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/worklog/11714", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 3", "type": "text"}]}]}, "created": "2021-04-15T11:39:50.205-0700", "updated": "2021-04-15T11:39:50.205-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "1h 23m", "timeSpentSeconds": 4980, "id": "11714", "issueId": "10075"}, "emitted_at": 1686153002728} {"stream": "jira_settings", "data": {"id": "jira.issuenav.criteria.autoupdate", "key": "jira.issuenav.criteria.autoupdate", "value": "true", "name": "Auto Update Criteria", "desc": "Turn on to update search results automatically", "type": "boolean"}, "emitted_at": 1685113041270} {"stream": "jira_settings", "data": {"id": "jira.clone.prefix", "key": "jira.clone.prefix", "value": "CLONE -", "name": "The prefix added to the Summary field of cloned issues", "type": "string"}, "emitted_at": 1685113041271} {"stream": "jira_settings", "data": {"id": "jira.date.picker.java.format", "key": "jira.date.picker.java.format", "value": "d/MMM/yy", "name": "Date Picker Format (Java)", "desc": "This part is only for the Java (server side) generated dates. Note that this should correspond to the javascript date picker format (jira.date.picker.javascript.format) setting.", "type": "string"}, "emitted_at": 1685113041271} @@ -87,9 +86,8 @@ {"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 10056, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056", "name": "CAW software permission scheme", "description": "The permission scheme for Jira Software Free. In Free, any registered user can access and administer this project.", "permissions": [{"id": 14200, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14200", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 14201, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14201", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14202, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14202", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14203, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14203", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14204, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14204", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14205, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14205", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14206, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14206", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14207, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14207", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 14208, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14208", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14209, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14209", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14210, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14210", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14211, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14211", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 14212, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14212", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14213, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14213", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14214, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14214", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14215, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14215", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14216, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14216", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14217, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14217", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 14218, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14218", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14219, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14219", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14220, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14220", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 14221, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14221", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14222, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14222", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14223, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14223", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14224, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14224", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 14225, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14225", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14226, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14226", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14227, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14227", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14228, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14228", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14229, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14229", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14230, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14230", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14231, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14231", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14232, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14232", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 14233, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14233", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 14234, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14234", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 14235, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14235", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14236, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14236", "holder": {"type": "applicationRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14237, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14237", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14238, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14238", "holder": {"type": "applicationRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14239, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14239", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14240, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14240", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14241, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14241", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14242, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14242", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14243, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14243", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 14244, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14244", "holder": {"type": "applicationRole"}, "permission": "DELETE_ISSUES"}, {"id": 14245, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14245", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 14246, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14246", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 14247, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14247", "holder": {"type": "applicationRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14248, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14248", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 14249, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14249", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14250, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14250", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14251, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14251", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14252, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14252", "holder": {"type": "applicationRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14253, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14253", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14254, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14254", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 14255, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14255", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14256, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14256", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14257, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14257", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14258, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14258", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14259, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14259", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14260, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14260", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14261, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14261", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14262, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14262", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14263, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14263", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14264, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14264", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14265, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14265", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14266, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14266", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14267, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14267", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14404, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14404", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 14481, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14481", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14542, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14542", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14714, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14714", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14715, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14715", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14836, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14836", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14837, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14837", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15117, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/15117", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1685113044013} {"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 10055, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055", "name": "CLK software permission scheme", "description": "The permission scheme for Jira Software Free. In Free, any registered user can access and administer this project.", "permissions": [{"id": 14132, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14132", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 14133, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14133", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14134, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14134", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14135, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14135", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14136, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14136", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14137, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14137", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14138, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14138", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14139, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14139", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 14140, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14140", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14141, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14141", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14142, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14142", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14143, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14143", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 14144, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14144", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14145, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14145", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14146, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14146", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14147, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14147", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14148, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14148", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14149, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14149", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 14150, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14150", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14151, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14151", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14152, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14152", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 14153, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14153", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14154, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14154", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14155, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14155", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14156, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14156", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 14157, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14157", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14158, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14158", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14159, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14159", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14160, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14160", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14161, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14161", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14162, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14162", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14163, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14163", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14164, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14164", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 14165, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14165", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 14166, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14166", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 14167, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14167", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14168, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14168", "holder": {"type": "applicationRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14169, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14169", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14170, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14170", "holder": {"type": "applicationRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14171, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14171", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14172, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14172", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14173, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14173", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14174, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14174", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14175, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14175", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 14176, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14176", "holder": {"type": "applicationRole"}, "permission": "DELETE_ISSUES"}, {"id": 14177, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14177", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 14178, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14178", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 14179, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14179", "holder": {"type": "applicationRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14180, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14180", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 14181, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14181", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14182, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14182", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14183, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14183", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14184, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14184", "holder": {"type": "applicationRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14185, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14185", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14186, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14186", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 14187, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14187", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14188, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14188", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14189, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14189", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14190, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14190", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14191, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14191", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14192, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14192", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14193, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14193", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14194, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14194", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14195, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14195", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14196, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14196", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14197, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14197", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14198, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14198", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14199, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14199", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14405, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14405", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 14482, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14482", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14543, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14543", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14712, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14712", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14713, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14713", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14834, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14834", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14835, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14835", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15118, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/15118", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1685113044015} {"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0", "name": "Default Permission Scheme", "description": "This is the default Permission Scheme. Any new projects that are created will be assigned this scheme.", "permissions": [{"id": 10004, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10004", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 10005, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10005", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 10006, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10006", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 10007, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10007", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 10008, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10008", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 10009, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10009", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 10010, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10010", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 10011, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10011", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 10012, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10012", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 10013, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10013", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 10014, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10014", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 10015, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10015", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 10016, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10016", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 10017, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10017", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 10018, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10018", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 10019, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10019", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 10020, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10020", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 10021, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10021", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 10022, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10022", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 10023, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10023", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 10024, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10024", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 10025, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10025", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 10026, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10026", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 10027, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10027", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 10028, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10028", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 10029, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10029", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 10030, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10030", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 10031, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10031", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 10033, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10033", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 10200, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10200", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 10300, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10300", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 10301, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10301", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 10302, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10302", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 10303, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10303", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 10304, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10304", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 10305, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10305", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 10306, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10306", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 10307, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10307", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 10308, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10308", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 10309, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10309", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 10310, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10310", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 10311, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10311", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 10312, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10312", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 10313, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10313", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 10314, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10314", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 10315, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10315", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 10316, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10316", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 10317, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10317", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 10318, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10318", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 10319, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10319", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 10320, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10320", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 10321, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10321", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 10322, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10322", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 10323, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10323", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 10324, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10324", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 10325, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10325", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 10326, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10326", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 10327, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10327", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 10328, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10328", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 10329, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10329", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 10330, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10330", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 10331, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10331", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 10332, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10332", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 10333, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10333", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 10464, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10464", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 10465, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10465", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 10466, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10466", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 10467, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10467", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14538, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14538", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14599, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14599", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14600, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14600", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14601, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14601", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14722, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14722", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14723, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14723", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15119, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/15119", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1685113044017} -{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "description": "Test", "name": "integration-tests", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "name": "Test category 2", "description": "Test Project Category 2"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1685113044196} -{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "description": "Test project 13 description", "name": "Test project 13", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "name": "Category 1", "description": "Category 1"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1685113044197} -{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10064", "id": "10064", "key": "TTMP2", "description": "", "name": "Test Team Managed Project 2", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "name": "Category 1", "description": "Category 1"}, "projectTypeKey": "software", "simplified": true, "style": "next-gen", "isPrivate": false, "properties": {}, "entityId": "6fc48839-dfa5-487d-ad8f-8b540f1748d7", "uuid": "6fc48839-dfa5-487d-ad8f-8b540f1748d7"}, "emitted_at": 1685113044199} +{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "description": "Test", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "integration-tests", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "name": "Test category 2", "description": "Test Project Category 2"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1686153767914} +{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "description": "Test project 13 description", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Test project 13", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "name": "Category 1", "description": "Category 1"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1686153767915} {"stream": "project_avatars", "data": {"id": "10400", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10400&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10400&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10400&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10400&avatarType=project"}}, "emitted_at": 1685113044864} {"stream": "project_avatars", "data": {"id": "10401", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10401&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10401&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10401&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10401&avatarType=project"}}, "emitted_at": 1685113044864} {"stream": "project_avatars", "data": {"id": "10402", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10402&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10402&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10402&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10402&avatarType=project"}}, "emitted_at": 1685113044865} @@ -101,7 +99,6 @@ {"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10048", "id": "10048", "name": "Component 2", "description": "This is a Jira component", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_LEAD", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_LEAD", "description": "This is a Jira component", "name": "Component 2", "id": "10048", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "self": "https://airbyteio.atlassian.net/rest/api/3/component/10048"}, "emitted_at": 1685113046993} {"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10000"}, "emitted_at": 1685113048530} {"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10016"}, "emitted_at": 1685113049055} -{"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10064"}, "emitted_at": 1685113049588} {"stream": "project_types", "data": {"key": "product_discovery", "formattedKey": "Product Discovery", "descriptionI18nKey": "jira.project.type.polaris.description", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8cGF0aCBzdHlsZT0iZmlsbDojRjc5MjMyOyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkgTTEzNi42NjcsMTc4LjMzM0wxMjUsMTkwbC00MS42NjctNDBMOTUsMTM4LjMzM2wzMC0zMEwxMzYuNjY3LDEyMGwtMzAsMzBMMTM2LjY2NywxNzguMzMzeiBNMjA1LDE2MS42NjdsLTMwLDMwTDE2My4zMzMsMTgwDQoJCWwzMC0zMGwtMzAtMzBMMTc1LDEwOC4zMzNMMjE2LjY2NywxNTBMMjA1LDE2MS42Njd6Ii8+DQo8L2c+DQo8Zz4NCgk8cG9seWdvbiBzdHlsZT0iZmlsbDojRkZGRkZGOyIgcG9pbnRzPSIxNzUsMTkxLjY2NyAyMDUsMTYxLjY2NyAyMTYuNjY3LDE1MCAxNzUsMTA4LjMzMyAxNjMuMzMzLDEyMCAxOTMuMzMzLDE1MCAxNjMuMzMzLDE4MCAJIi8+DQoJPHBvbHlnb24gc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIHBvaW50cz0iMTI1LDEwOC4zMzMgOTUsMTM4LjMzMyA4My4zMzMsMTUwIDEyNSwxOTAgMTM2LjY2NywxNzguMzMzIDEwNi42NjcsMTUwIDEzNi42NjcsMTIwIAkiLz4NCjwvZz4NCjwvc3ZnPg0K", "color": "#F5A623"}, "emitted_at": 1685113053300} {"stream": "project_types", "data": {"key": "software", "formattedKey": "Software", "descriptionI18nKey": "jira.project.type.software.description", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8cGF0aCBzdHlsZT0iZmlsbDojRjc5MjMyOyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkgTTEzNi42NjcsMTc4LjMzM0wxMjUsMTkwbC00MS42NjctNDBMOTUsMTM4LjMzM2wzMC0zMEwxMzYuNjY3LDEyMGwtMzAsMzBMMTM2LjY2NywxNzguMzMzeiBNMjA1LDE2MS42NjdsLTMwLDMwTDE2My4zMzMsMTgwDQoJCWwzMC0zMGwtMzAtMzBMMTc1LDEwOC4zMzNMMjE2LjY2NywxNTBMMjA1LDE2MS42Njd6Ii8+DQo8L2c+DQo8Zz4NCgk8cG9seWdvbiBzdHlsZT0iZmlsbDojRkZGRkZGOyIgcG9pbnRzPSIxNzUsMTkxLjY2NyAyMDUsMTYxLjY2NyAyMTYuNjY3LDE1MCAxNzUsMTA4LjMzMyAxNjMuMzMzLDEyMCAxOTMuMzMzLDE1MCAxNjMuMzMzLDE4MCAJIi8+DQoJPHBvbHlnb24gc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIHBvaW50cz0iMTI1LDEwOC4zMzMgOTUsMTM4LjMzMyA4My4zMzMsMTUwIDEyNSwxOTAgMTM2LjY2NywxNzguMzMzIDEwNi42NjcsMTUwIDEzNi42NjcsMTIwIAkiLz4NCjwvZz4NCjwvc3ZnPg0K", "color": "#F5A623"}, "emitted_at": 1685113053300} {"stream": "project_types", "data": {"key": "service_desk", "formattedKey": "Service Desk", "descriptionI18nKey": "jira.project.type.servicedesk.description.jsm", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8Zz4NCgkJPHJlY3QgeD0iMTAwIiB5PSIxMDAiIHN0eWxlPSJmaWxsOiM2N0FCNDk7IiB3aWR0aD0iMTAwIiBoZWlnaHQ9IjY2LjY2NyIvPg0KCQk8cGF0aCBzdHlsZT0iZmlsbDojNjdBQjQ5OyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkJIE0yMTYuNjY3LDEwMHY2Ni42Njd2MTYuNjY3aC01MFYyMDBIMjAwdjE2LjY2N0gxMDBWMjAwaDMzLjMzM3YtMTYuNjY3aC01MHYtMTYuNjY3VjEwMFY4My4zMzNoMTMzLjMzM1YxMDB6Ii8+DQoJPC9nPg0KPC9nPg0KPHBhdGggc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIGQ9Ik0yMTYuNjY3LDE4My4zMzN2LTE2LjY2N1YxMDBWODMuMzMzSDgzLjMzM1YxMDB2NjYuNjY3djE2LjY2N2g1MFYyMDBIMTAwdjE2LjY2N2gxMDBWMjAwaC0zMy4zMzMNCgl2LTE2LjY2N0gyMTYuNjY3eiBNMTAwLDE2Ni42NjdWMTAwaDEwMHY2Ni42NjdIMTAweiIvPg0KPC9zdmc+DQo=", "color": "#67AB49"}, "emitted_at": 1685113053300} @@ -131,12 +128,12 @@ {"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "locale": "en_US"}, "emitted_at": 1685113170166} {"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true}, "emitted_at": 1685113170167} {"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true}, "emitted_at": 1685113170167} -{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 29, "items": [{"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "confluence-users", "groupId": "38d808e9-113f-45c4-817b-099e953b687a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=38d808e9-113f-45c4-817b-099e953b687a"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}, {"name": "Test group 11", "groupId": "b0e6d76f-701a-4208-a88d-4478f242edde", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=b0e6d76f-701a-4208-a88d-4478f242edde"}, {"name": "Test group 12", "groupId": "dddc24a0-ef00-407e-abef-5a660b6f55cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dddc24a0-ef00-407e-abef-5a660b6f55cf"}, {"name": "Test group 13", "groupId": "dbe4af74-8387-4b08-843b-86af78dd738e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dbe4af74-8387-4b08-843b-86af78dd738e"}, {"name": "Test group 14", "groupId": "d4570a20-38d8-44cc-a63b-0924d0d0d0ff", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=d4570a20-38d8-44cc-a63b-0924d0d0d0ff"}, {"name": "Test group 15", "groupId": "87bde5c0-7231-44a7-88b5-421da2ab8052", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=87bde5c0-7231-44a7-88b5-421da2ab8052"}, {"name": "Test group 16", "groupId": "538b6aa2-bf57-402f-93c0-c2e2d68b7155", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=538b6aa2-bf57-402f-93c0-c2e2d68b7155"}, {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=022bc924-ac57-442d-80c9-df042b73ad87"}, {"name": "Test group 18", "groupId": "bbfc6fc9-96db-4e66-88f4-c55b08298272", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bbfc6fc9-96db-4e66-88f4-c55b08298272"}, {"name": "Test group 19", "groupId": "3c4fef5d-9721-4f20-9a68-346d222de3cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3c4fef5d-9721-4f20-9a68-346d222de3cf"}, {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}, {"name": "Test group 3", "groupId": "638aa1ad-8707-4d56-9361-f5959b6c4785", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=638aa1ad-8707-4d56-9361-f5959b6c4785"}, {"name": "Test group 4", "groupId": "532554e0-43be-4eca-9186-b417dcf38547", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=532554e0-43be-4eca-9186-b417dcf38547"}, {"name": "Test group 5", "groupId": "6b663734-85b6-4185-8fb2-9ac27709b3aa", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=6b663734-85b6-4185-8fb2-9ac27709b3aa"}, {"name": "Test group 6", "groupId": "2d4af5cf-cd34-4e78-9445-abc000cdd5cc", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d4af5cf-cd34-4e78-9445-abc000cdd5cc"}, {"name": "Test group 7", "groupId": "e8a97909-d807-4f79-8548-1f2c156ae6f0", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e8a97909-d807-4f79-8548-1f2c156ae6f0"}, {"name": "Test group 8", "groupId": "3ee851e7-6688-495a-a6f6-737e85a23878", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3ee851e7-6688-495a-a6f6-737e85a23878"}, {"name": "Test group 9", "groupId": "af27d0b1-4378-443f-9a6d-f878848b144a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=af27d0b1-4378-443f-9a6d-f878848b144a"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1685113171084} -{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 2, "items": [{"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1685113171411} -{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 1, "items": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1685113171720} -{"stream": "workflows", "data": {"id": {"name": "Builds Workflow", "entityId": "Builds Workflow"}, "description": "Builds Workflow", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800"}, "emitted_at": 1685113185890} -{"stream": "workflows", "data": {"id": {"name": "classic default workflow", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4"}, "description": "The classic JIRA default workflow", "created": "2020-12-03T23:41:38.951-0800", "updated": "2020-12-03T23:41:57.343-0800"}, "emitted_at": 1685113185891} -{"stream": "workflows", "data": {"id": {"name": "jira", "entityId": "jira"}, "description": "The default Jira workflow.", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800"}, "emitted_at": 1685113185891} +{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 30, "items": [{"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "confluence-users", "groupId": "38d808e9-113f-45c4-817b-099e953b687a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=38d808e9-113f-45c4-817b-099e953b687a"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=125680d3-7e85-41ad-a662-892b6590272e"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}, {"name": "Test group 11", "groupId": "b0e6d76f-701a-4208-a88d-4478f242edde", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=b0e6d76f-701a-4208-a88d-4478f242edde"}, {"name": "Test group 12", "groupId": "dddc24a0-ef00-407e-abef-5a660b6f55cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dddc24a0-ef00-407e-abef-5a660b6f55cf"}, {"name": "Test group 13", "groupId": "dbe4af74-8387-4b08-843b-86af78dd738e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dbe4af74-8387-4b08-843b-86af78dd738e"}, {"name": "Test group 14", "groupId": "d4570a20-38d8-44cc-a63b-0924d0d0d0ff", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=d4570a20-38d8-44cc-a63b-0924d0d0d0ff"}, {"name": "Test group 15", "groupId": "87bde5c0-7231-44a7-88b5-421da2ab8052", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=87bde5c0-7231-44a7-88b5-421da2ab8052"}, {"name": "Test group 16", "groupId": "538b6aa2-bf57-402f-93c0-c2e2d68b7155", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=538b6aa2-bf57-402f-93c0-c2e2d68b7155"}, {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=022bc924-ac57-442d-80c9-df042b73ad87"}, {"name": "Test group 18", "groupId": "bbfc6fc9-96db-4e66-88f4-c55b08298272", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bbfc6fc9-96db-4e66-88f4-c55b08298272"}, {"name": "Test group 19", "groupId": "3c4fef5d-9721-4f20-9a68-346d222de3cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3c4fef5d-9721-4f20-9a68-346d222de3cf"}, {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}, {"name": "Test group 3", "groupId": "638aa1ad-8707-4d56-9361-f5959b6c4785", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=638aa1ad-8707-4d56-9361-f5959b6c4785"}, {"name": "Test group 4", "groupId": "532554e0-43be-4eca-9186-b417dcf38547", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=532554e0-43be-4eca-9186-b417dcf38547"}, {"name": "Test group 5", "groupId": "6b663734-85b6-4185-8fb2-9ac27709b3aa", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=6b663734-85b6-4185-8fb2-9ac27709b3aa"}, {"name": "Test group 6", "groupId": "2d4af5cf-cd34-4e78-9445-abc000cdd5cc", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d4af5cf-cd34-4e78-9445-abc000cdd5cc"}, {"name": "Test group 7", "groupId": "e8a97909-d807-4f79-8548-1f2c156ae6f0", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e8a97909-d807-4f79-8548-1f2c156ae6f0"}, {"name": "Test group 8", "groupId": "3ee851e7-6688-495a-a6f6-737e85a23878", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3ee851e7-6688-495a-a6f6-737e85a23878"}, {"name": "Test group 9", "groupId": "af27d0b1-4378-443f-9a6d-f878848b144a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=af27d0b1-4378-443f-9a6d-f878848b144a"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1688723244956} +{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 3, "items": [{"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1688723245241} +{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 1, "items": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1688723245508} +{"stream": "workflows", "data": {"id": {"name": "Builds Workflow", "entityId": "Builds Workflow"}, "description": "Builds Workflow", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800"}, "emitted_at": 1688485038533} +{"stream": "workflows", "data": {"id": {"name": "classic default workflow", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4"}, "description": "The classic JIRA default workflow", "created": "2020-12-03T23:41:38.951-0800", "updated": "2023-06-30T02:33:48.808-0700"}, "emitted_at": 1688485038534} +{"stream": "workflows", "data": {"id": {"name": "jira", "entityId": "jira"}, "description": "The default Jira workflow.", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800"}, "emitted_at": 1688485038534} {"stream": "workflow_schemes", "data": {"id": 10000, "name": "classic", "description": "classic", "defaultWorkflow": "classic default workflow", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10000"}, "emitted_at": 1685113186843} {"stream": "workflow_schemes", "data": {"id": 10001, "name": "IT: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project IT", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10001"}, "emitted_at": 1685113186844} {"stream": "workflow_schemes", "data": {"id": 10002, "name": "P2: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project P2", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10002"}, "emitted_at": 1685113186844} diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index 7f6b05ad5961..ad0753c1bea0 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 - dockerImageTag: 0.3.11 + dockerImageTag: 0.3.12 maxSecondsBetweenMessages: 21600 dockerRepository: airbyte/source-jira githubIssueLabel: source-jira @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/jira tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-jira/requirements.txt b/airbyte-integrations/connectors/source-jira/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-jira/requirements.txt +++ b/airbyte-integrations/connectors/source-jira/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py index d32505c66770..e481ac92aa12 100644 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -8,8 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "requests==2.25.1", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest==6.2.5", - "connector-acceptance-test", "responses~=0.22.0", ] diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json index 0814d2a73be9..f9a2b9d0a2ef 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json @@ -180,7 +180,7 @@ }, "48x48": { "type": "string", - "description": "The URL of the item's 48x48 pixel avatar." + "description": "The URL of the item's 48x48 pixel avatar." } } }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filters.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filters.json index 8f367eff04b8..989078b38a16 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filters.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filters.json @@ -31,7 +31,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -124,7 +124,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -252,7 +252,7 @@ "viewUrl": { "type": "string", "description": "A URL to view the filter results in Jira, using the ID of the filter. For example, *https://your-domain.atlassian.net/issues/?filter=10100*.", - + "readOnly": true }, "searchUrl": { @@ -425,7 +425,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -555,7 +555,7 @@ "self": { "type": "string", "description": "The URL of the component.", - + "readOnly": true }, "id": { @@ -579,7 +579,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -672,7 +672,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -820,7 +820,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -913,7 +913,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -1052,7 +1052,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -1468,7 +1468,7 @@ "self": { "type": "string", "description": "The URL of the version.", - + "readOnly": true }, "id": { @@ -1632,7 +1632,7 @@ "self": { "type": "string", "description": "The URL of the project category.", - + "readOnly": true }, "id": { @@ -1788,7 +1788,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -1881,7 +1881,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -2020,7 +2020,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -2113,7 +2113,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -2242,7 +2242,7 @@ "self": { "type": "string", "description": "The URL the project role details.", - + "readOnly": true }, "name": { @@ -2293,7 +2293,7 @@ "avatarUrl": { "type": "string", "description": "The avatar of the role actor.", - + "readOnly": true }, "actorUser": { @@ -2461,7 +2461,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -2492,7 +2492,7 @@ "self": { "type": "string", "description": "The URL of the user.", - + "readOnly": true }, "key": { @@ -2585,7 +2585,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } @@ -2717,7 +2717,7 @@ "self": { "type": "string", "description": "The URL for these group details.", - + "readOnly": true } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json index ec98a590f5b8..3fa5fe792cb1 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/spec.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/spec.json @@ -17,7 +17,11 @@ "domain": { "type": "string", "title": "Domain", - "examples": [".atlassian.net", ".jira.com", "jira..com"], + "examples": [ + ".atlassian.net", + ".jira.com", + "jira..com" + ], "description": "The Domain for your Jira account, e.g. airbyteio.atlassian.net, airbyteio.jira.com, jira.your-domain.com", "order": 1 }, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index 6d0dbf6d296b..cddb6fb1e54e 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -808,7 +808,7 @@ def path(self, **kwargs) -> str: def request_params(self, **kwargs): params = super().request_params(**kwargs) - params["expand"] = "description" + params["expand"] = "description,lead" return params def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py index 672c98a78a7d..7c5ff589790b 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py @@ -266,7 +266,7 @@ def mock_projects_responses(config, projects_response): Projects.use_cache = False responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description", + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", json=projects_response, ) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/application_role.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/application_role.json index 59278724ab6a..5c4d2f26bff4 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/application_role.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/application_role.json @@ -36,9 +36,7 @@ } ], "name": "Jira Software", - "defaultGroups": [ - "jira-software-users" - ], + "defaultGroups": ["jira-software-users"], "defaultGroupsDetails": [ { "name": "jira-software-users", @@ -55,4 +53,4 @@ "hasUnlimitedSeats": false, "platform": false } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/avatars.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/avatars.json index d2bcac330da4..f82f80db8d73 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/avatars.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/avatars.json @@ -25,4 +25,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/board.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/board.json index ebe2fdd85ed1..bf7696692d45 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/board.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/board.json @@ -52,4 +52,4 @@ "projectKey": "Project1" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/dashboard.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/dashboard.json index d1772124ec94..5aadf0a7edac 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/dashboard.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/dashboard.json @@ -33,4 +33,4 @@ "systemDashboard": true } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter.json index 6a1267d7c3ab..ff41ebb62bc3 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter.json @@ -50,4 +50,4 @@ "subscriptions": [] } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter_sharing.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter_sharing.json index 9b52e1686979..cfbd0edafb78 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter_sharing.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/filter_sharing.json @@ -21,4 +21,4 @@ "properties": {} } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/groups.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/groups.json index f9bfc52ecc5e..65a892829a92 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/groups.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/groups.json @@ -17,4 +17,4 @@ "groupId": "test4" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_comments.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_comments.json index 469f09add69c..752c7bc42a14 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_comments.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_comments.json @@ -105,4 +105,4 @@ "jsdPublic": true } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json index 26379e87c7cf..ead4a03913ba 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json @@ -15,4 +15,4 @@ "isAnyIssueType": true } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json index 610e30b832c7..05a312fc78a8 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json @@ -7,9 +7,7 @@ "orderable": false, "navigable": true, "searchable": true, - "clauseNames": [ - "statusCategoryChangedDate" - ], + "clauseNames": ["statusCategoryChangedDate"], "schema": { "type": "datetime", "system": "statuscategorychangedate" @@ -23,10 +21,7 @@ "orderable": true, "navigable": true, "searchable": true, - "clauseNames": [ - "issuetype", - "type" - ], + "clauseNames": ["issuetype", "type"], "schema": { "type": "issuetype", "system": "issuetype" @@ -40,8 +35,6 @@ "orderable": false, "navigable": true, "searchable": false, - "clauseNames": [ - "parent" - ] + "clauseNames": ["parent"] } ] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_notification_schemas.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_notification_schemas.json index 83ae82f4e70a..e1dcac1490a2 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_notification_schemas.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_notification_schemas.json @@ -16,4 +16,4 @@ "isDefault": false } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_properties.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_properties.json index 35bdfe3294fa..d59d0c538dca 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_properties.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_properties.json @@ -28,4 +28,4 @@ "isDefault": false } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_remote_links.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_remote_links.json index 6dbf2bc1576d..199f3dfa04f0 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_remote_links.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_remote_links.json @@ -53,4 +53,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_resolutions.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_resolutions.json index 05d099d0a5c4..86917f5dfb93 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_resolutions.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_resolutions.json @@ -22,4 +22,4 @@ "isDefault": false } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_security_schemes.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_security_schemes.json index 2fff5eb10a66..ac5c1ef97c1a 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_security_schemes.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_security_schemes.json @@ -14,4 +14,4 @@ "defaultSecurityLevelId": 10002 } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_type.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_type.json index fe00415a627b..a1fcb2576617 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_type.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_type.json @@ -17,4 +17,4 @@ "defaultIssueTypeId": "10001" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_votes.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_votes.json index f9e403dfd42b..18beb6cc1216 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_votes.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_votes.json @@ -19,4 +19,4 @@ "active": false } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_watchers.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_watchers.json index 831072d27fec..e8a8f1d851f0 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_watchers.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_watchers.json @@ -10,4 +10,4 @@ "active": false } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_worklogs.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_worklogs.json index b298cef96fb6..8574cdf3f049 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_worklogs.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_worklogs.json @@ -45,4 +45,4 @@ "issueId": "10002" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json index a970c050c7be..612f3b35a262 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json @@ -63,9 +63,7 @@ }, "customfield_10024": null, "customfield_10025": null, - "labels": [ - "test" - ], + "labels": ["test"], "customfield_10026": null, "customfield_10016": null, "customfield_10017": "dark_orange", @@ -249,4 +247,4 @@ "updated": "2022-12-08T02:22:18.889-0800" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_field_configurations.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_field_configurations.json index 836741367613..f2cb17d7b163 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_field_configurations.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_field_configurations.json @@ -7,4 +7,4 @@ "isDefault": true } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_link_types.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_link_types.json index 4d155274f9fb..be9628042722 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_link_types.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_link_types.json @@ -22,4 +22,4 @@ "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_navigator_settings.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_navigator_settings.json index b1e458116430..42d78c8bcc98 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_navigator_settings.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues_navigator_settings.json @@ -11,4 +11,4 @@ "label": "Summary", "value": "summary" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/jira_settings.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/jira_settings.json index 9a4fd5009807..3d0613ba7d6b 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/jira_settings.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/jira_settings.json @@ -14,4 +14,4 @@ "name": "The prefix added to the Summary field of cloned issues", "type": "string" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/labels.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/labels.json index 251696ac53c8..e1b73c8f1fe7 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/labels.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/labels.json @@ -3,8 +3,5 @@ "startAt": 0, "total": 100, "isLast": false, - "values": [ - "performance", - "security" - ] -} \ No newline at end of file + "values": ["performance", "security"] +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/permissions.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/permissions.json index c9a0e2b6b2f2..c3fb75ea4cea 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/permissions.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/permissions.json @@ -7,4 +7,4 @@ "description": "Ability to modify a collection of issues at once. For example, resolve multiple issues in one step." } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_components.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_components.json index b81cdb7cd4db..81dbdba9a852 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_components.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_components.json @@ -143,4 +143,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_email.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_email.json index a16798f2ccf2..758bae5649cb 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_email.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_email.json @@ -7,4 +7,4 @@ "emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10016" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_permissions.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_permissions.json index a5c2cedad686..4525be814a49 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_permissions.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/project_permissions.json @@ -13,4 +13,4 @@ "name": "Staff Only" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json index b155807cea6c..a463e6c679dd 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json @@ -39,4 +39,4 @@ "properties": {} } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_avatars.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_avatars.json index 0c0ead56df9b..ea9494ea7ba1 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_avatars.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_avatars.json @@ -27,4 +27,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_categories.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_categories.json index d9a1eb92b67b..f6aefa5f57bf 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_categories.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_categories.json @@ -11,4 +11,4 @@ "description": "Category 2", "name": "Category 2" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_versions.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_versions.json index ae4f8c424a07..0d52c5940043 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_versions.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects_versions.json @@ -35,4 +35,4 @@ } } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/screen_tabs.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/screen_tabs.json index ec0a5ee79505..cfb2d551f4b7 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/screen_tabs.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/screen_tabs.json @@ -7,4 +7,4 @@ "id": 10001, "name": "Field Tab" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/screens.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/screens.json index b7ae1067088a..f815a364111c 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/screens.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/screens.json @@ -11,4 +11,4 @@ "description": "This screen is used in the workflow and enables you to assign issues" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprint_issues.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprint_issues.json index e6f405b04089..9ea627a016a6 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprint_issues.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprint_issues.json @@ -30,4 +30,4 @@ "updated": "2022-05-17T04:26:21.613-0700" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprints.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprints.json index e254a83380e4..2e74e7aae62f 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprints.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/sprints.json @@ -11,4 +11,4 @@ "goal": "Deliver results" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/time_tracking.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/time_tracking.json index e003b94aacb5..bd2bcf442aa8 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/time_tracking.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/time_tracking.json @@ -3,4 +3,4 @@ "key": "JIRA", "name": "JIRA provided time tracking" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/users.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/users.json index b526d3ca43ab..79beb79a4c80 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/users.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/users.json @@ -23,4 +23,4 @@ "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png" } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/users_groups_detailed.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/users_groups_detailed.json index 42f10c5fde79..5e4a17cce7b3 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/users_groups_detailed.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/users_groups_detailed.json @@ -205,4 +205,4 @@ }, "expand": "groups,applicationRoles" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_schemas.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_schemas.json index 71b07527775f..268b2e8e23f0 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_schemas.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_schemas.json @@ -17,4 +17,4 @@ "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10001" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_status_categories.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_status_categories.json index 6099fcd99446..49ae287b85bf 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_status_categories.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_status_categories.json @@ -13,4 +13,4 @@ "colorName": "blue-gray", "name": "To Do" } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_statuses.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_statuses.json index 210811cb20cb..11e4bdb4be80 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_statuses.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflow_statuses.json @@ -29,4 +29,4 @@ "name": "To Do" } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflows.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflows.json index 04828d7d8926..457d022f16e3 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflows.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/workflows.json @@ -19,4 +19,4 @@ "updated": "2020-12-03T23:41:57.343-0800" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py index 0abeb53115bb..cb9ba9330782 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py @@ -20,7 +20,7 @@ def test_streams(config): def test_check_connection(config, projects_response, labels_response): responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description", + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", json=projects_response, ) responses.add( diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index 44e4cb3ec4db..654b49b72715 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -376,7 +376,7 @@ def test_filter_sharing_stream(config, filter_sharing_response): def test_projects_stream(config, projects_response): responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description", + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", json=projects_response, ) @@ -651,7 +651,7 @@ def test_issues_stream(config, projects_response, mock_issues_responses, issues_ projects_response['values'].append({"id": "3", "key": "Project1"}) responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description", + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", json=projects_response, ) responses.add( diff --git a/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml b/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml index 26293a1ca818..f62b94eacbf6 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-k6-cloud/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-k6-cloud/requirements.txt b/airbyte-integrations/connectors/source-k6-cloud/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/requirements.txt +++ b/airbyte-integrations/connectors/source-k6-cloud/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-k6-cloud/setup.py b/airbyte-integrations/connectors/source-k6-cloud/setup.py index 5bb3b0a6b6ee..05908924fd5b 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/setup.py +++ b/airbyte-integrations/connectors/source-k6-cloud/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-kafka/metadata.yaml b/airbyte-integrations/connectors/source-kafka/metadata.yaml index 7be9d9571c23..72575793c636 100644 --- a/airbyte-integrations/connectors/source-kafka/metadata.yaml +++ b/airbyte-integrations/connectors/source-kafka/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-klarna/metadata.yaml b/airbyte-integrations/connectors/source-klarna/metadata.yaml index fdf8ed1d0485..6dc2f5b3bdce 100644 --- a/airbyte-integrations/connectors/source-klarna/metadata.yaml +++ b/airbyte-integrations/connectors/source-klarna/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/klarna tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-klarna/requirements.txt b/airbyte-integrations/connectors/source-klarna/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-klarna/requirements.txt +++ b/airbyte-integrations/connectors/source-klarna/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-klarna/setup.py b/airbyte-integrations/connectors/source-klarna/setup.py index 9038ca259be6..046558436d0f 100644 --- a/airbyte-integrations/connectors/source-klarna/setup.py +++ b/airbyte-integrations/connectors/source-klarna/setup.py @@ -8,10 +8,10 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", ""] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.22.0", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile index 5e5b79d17f7b..e36a1a15073e 100644 --- a/airbyte-integrations/connectors/source-klaviyo/Dockerfile +++ b/airbyte-integrations/connectors/source-klaviyo/Dockerfile @@ -34,5 +34,5 @@ COPY source_klaviyo ./source_klaviyo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.3.2 LABEL io.airbyte.name=airbyte/source-klaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml index 681fc36bc03f..955fd79d0665 100644 --- a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml @@ -8,6 +8,10 @@ acceptance_tests: expect_records: path: integration_tests/expected_records.jsonl extra_records: true + ignored_fields: + email_templates: + - name: html + bypass_reason: unstable data connection: tests: - config_path: secrets/config.json @@ -17,8 +21,6 @@ acceptance_tests: discovery: tests: - config_path: secrets/config.json - backward_compatibility_tests_config: - disable_for_version: "0.1.11" full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/__init__.py b/airbyte-integrations/connectors/source-klaviyo/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl index 2399647c9f68..1cbe453f08c0 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl @@ -37,41 +37,34 @@ {"stream": "lists", "data": {"object": "list", "id": "X7UeXn", "name": "Test5___", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:34+00:00", "updated": "2021-11-16T14:24:34+00:00", "person_count": 1}, "emitted_at": 1663367161890} {"stream": "lists", "data": {"object": "list", "id": "S8nmQ9", "name": "Test AAAB", "list_type": "list", "folder": null, "created": "2021-11-16T15:02:51+00:00", "updated": "2021-11-16T15:02:51+00:00", "person_count": 1}, "emitted_at": 1663367161890} {"stream": "lists", "data": {"object": "list", "id": "SBYgiK", "name": "SMS Subscribers", "list_type": "list", "folder": null, "created": "2022-05-31T06:52:26+00:00", "updated": "2022-05-31T06:52:26+00:00", "person_count": 0}, "emitted_at": 1663367161890} -{"stream": "metrics", "data": {"object": "metric", "id": "RUQ6YQ", "name": "Active on Site", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487216} -{"stream": "metrics", "data": {"object": "metric", "id": "Xi7Kwh", "name": "Bounced Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487221} -{"stream": "metrics", "data": {"object": "metric", "id": "VePdj9", "name": "Cancelled Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487222} -{"stream": "metrics", "data": {"object": "metric", "id": "SPnhc3", "name": "Checkout Started", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487223} -{"stream": "metrics", "data": {"object": "metric", "id": "SxR9Bt", "name": "Clicked Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487223} -{"stream": "metrics", "data": {"object": "metric", "id": "VFFb4u", "name": "Clicked Email", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2021-05-17T23:43:50+00:00", "updated": "2021-05-17T23:43:50+00:00"}, "emitted_at": 1673610487223} -{"stream": "metrics", "data": {"object": "metric", "id": "Y5TbbA", "name": "Clicked SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487224} -{"stream": "metrics", "data": {"object": "metric", "id": "VEsf4u", "name": "Consented to Receive SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487224} -{"stream": "metrics", "data": {"object": "metric", "id": "RhP4nd", "name": "Dropped Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487225} -{"stream": "metrics", "data": {"object": "metric", "id": "TeZiVn", "name": "Failed to deliver Automated Response SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487225} -{"stream": "metrics", "data": {"object": "metric", "id": "S5Au3w", "name": "Failed to Deliver SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487225} -{"stream": "metrics", "data": {"object": "metric", "id": "X3f6PC", "name": "Fulfilled Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487226} -{"stream": "metrics", "data": {"object": "metric", "id": "RcjEmN", "name": "Fulfilled Partial Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:45:47+00:00", "updated": "2022-05-31T06:45:47+00:00"}, "emitted_at": 1673610487226} -{"stream": "metrics", "data": {"object": "metric", "id": "THfYvj", "name": "Marked Email as Spam", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487226} -{"stream": "metrics", "data": {"object": "metric", "id": "Yy9QKx", "name": "Opened Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487227} -{"stream": "metrics", "data": {"object": "metric", "id": "RDXsib", "name": "Ordered Product", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487227} -{"stream": "metrics", "data": {"object": "metric", "id": "TspjNE", "name": "Placed Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487228} -{"stream": "metrics", "data": {"object": "metric", "id": "RszrqT", "name": "Received Automated Response SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487228} -{"stream": "metrics", "data": {"object": "metric", "id": "WKHXf4", "name": "Received Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487228} -{"stream": "metrics", "data": {"object": "metric", "id": "WhthF7", "name": "Received SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487229} -{"stream": "metrics", "data": {"object": "metric", "id": "R2WpFy", "name": "Refunded Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487229} -{"stream": "metrics", "data": {"object": "metric", "id": "XsS8yX", "name": "Sent SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487229} -{"stream": "metrics", "data": {"object": "metric", "id": "UBNaGw", "name": "Subscribed to Back in Stock", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487230} -{"stream": "metrics", "data": {"object": "metric", "id": "Tp8t7d", "name": "Subscribed to List", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-11-16T15:05:22+00:00", "updated": "2021-11-16T15:05:22+00:00"}, "emitted_at": 1673610487230} -{"stream": "metrics", "data": {"object": "metric", "id": "VvFRZN", "name": "Unsubscribed", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1673610487231} -{"stream": "metrics", "data": {"object": "metric", "id": "TS2mxZ", "name": "Unsubscribed from SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1673610487231} -{"stream": "metrics", "data": {"object": "metric", "id": "YcDVHu", "name": "Viewed Product", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1673610487232} -{"stream": "email_templates", "data": {"object": "email-template", "id": "RdbN2P", "name": "Newsletter #1 (Images & Text)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    This template starts with images.

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Everyone loves pictures. They're more engaging that text by itself and the images in this template will neatly stack on mobile devices for the best viewing experience.

    \n

    Use the text area below to add additional content or add more images to create a larger image gallery. You can drag blocks from the left sidebar to add content to your template. You can customize this colors, fonts and styling of this template to match your brand by clicking the \"Styles\" button to the left.

    \n

    Happy emailing!

    \n

    The Klaviyo Team

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:37+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1679665066638} -{"stream": "email_templates", "data": {"object": "email-template", "id": "WAtsyg", "name": "Newsletter #2 (Image Gallery)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    This template is all images, baby.

    \n

    Even though these images are less than 200px wide, we recommend using larger images so that your images look good on mobile! On mobile, columns will stack, so you'll have more room and the images can be bigger... and more awesome.

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:38+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1679665066639} -{"stream": "email_templates", "data": {"object": "email-template", "id": "YnTk4K", "name": "Newsletter #3 (Image Gallery with Captions)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    This template has images and captions.

    \n

    Perfect for showing off your products or features. Use the captions to give additional information and link to your website.

    \n

    Notice that each image is 450px wide. The reason we recommend larger images is because of mobile! On mobile, columns will stack, so the images can expand to be larger.

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \nImage Caption\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:39+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1679665066639} -{"stream": "email_templates", "data": {"object": "email-template", "id": "XKShs8", "name": "Newsletter #4 (Story Boxes)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    What's a story box? An image and some text.

    \n

    Story boxes are great if you have stories or announcements. Choose a graphic or image and then add a short description. Check out how this template looks on mobile!

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Story Headline
    \nAdd a line or two about what this story is about. Read more.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Story Headline
    \nAdd a line or two about what this story is about. Read more.

    \n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Story Headline
    \nAdd a line or two about what this story is about. Read more.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Story Headline
    \nAdd a line or two about what this story is about. Read more.

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:40+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1679665066640} -{"stream": "email_templates", "data": {"object": "email-template", "id": "RKzAi4", "name": "Newsletter #5 (List of Text Headlines)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Extra! Extra! Here are the headlines!

    \n

    If you curate a list of interesting or helpful links daily, weekly or monthly, this template is easy to edit and easy to read.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Headline goes here and links to the full article

    \n

    If you have the first few sentences or a teaser of the article, add it here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Headline goes here and links to the full article

    \n

    If you have the first few sentences or a teaser of the article, add it here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Headline goes here and links to the full article

    \n

    If you have the first few sentences or a teaser of the article, add it here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Headline goes here and links to the full article

    \n

    If you have the first few sentences or a teaser of the article, add it here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    You got to the end! We saved the best for last...

    \n

    If you have the first few sentences or a teaser of the article, add it here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n
    \n\n\n
    \n
    \n\n\n\n\n\n\n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:40+00:00", "updated": "2022-05-31T06:36:46+00:00"}, "emitted_at": 1679665066640} -{"stream": "email_templates", "data": {"object": "email-template", "id": "TxZtNx", "name": "Newsletter #6 (List of Icons & Headlines)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Headlines with icons.

    \n

    Want to dress up a text only email? Adding icons or small pictures can help people visually undestand what each item is about.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n\n\n\n\n\n\n
    \n

    Headline with a link to the article

    \n

    Insert article teaser here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n\n\n\n\n\n\n
    \n

    Headline with a link to the article

    \n

    Insert article teaser here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n\n\n\n\n\n\n
    \n

    Headline with a link to the article

    \n

    Insert article teaser here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n\n\n\n\n\n\n
    \n

    Headline with a link to the article

    \n

    Insert article teaser here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n\n\n\n\n\n\n
    \n

    We saved the best for last...

    \n

    Insert article teaser here. You could also some information about the link such as a category tag or the website domain it's on, such as klaviyo.com.

    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:41+00:00", "updated": "2022-05-31T06:36:46+00:00"}, "emitted_at": 1679665066641} -{"stream": "email_templates", "data": {"object": "email-template", "id": "Vkrmj2", "name": "Newsletter #7 (Snack)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    When you have one thing to say, a snack email is great way to do it.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    A snack email has four components: a headline, an image, a short description and a button people can link on to learn more or keep going. This is a great option when you have one announcement to make.

    \n

    Snack emails are easy to read, so your customers and users will appreciate you making it easy for them to decide whether this email is relevant to them.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n\n\n\n
    \nKeep Reading\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n
    \n
    \n\n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n
    \n\n\n
    \n
    \n\n\n\n\n\n\n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:42+00:00", "updated": "2022-05-31T06:36:46+00:00"}, "emitted_at": 1679665066642} -{"stream": "email_templates", "data": {"object": "email-template", "id": "R8yxFr", "name": "Newsletter #8 (Snack w/ Recommendations)", "html": "\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n\n\n\n\n\n\n
    \nSHOP NOW\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    When you have one thing to say, a snack email is great way to do it.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    A snack email has four components: a headline, an image, a short description and a button people can link on to learn more or keep going. This is a great option when you have one announcement to make.

    \n

    Snack emails are easy to read, so your customers and users will appreciate you making it easy for them to decide whether this email is relevant to them.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n\n\n\n
    \nKeep Reading\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Just for you...

    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Product Title
    \nThis is a short description of this product.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n\n\n\n
    \nView Product\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Product Title
    \nThis is a short description of this product.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n\n\n\n
    \nView Product\n
    \n
    \n    		\n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"\"\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Product Title
    \nThis is a short description of this product.

    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n\n\n\n
    \nView Product\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\"Facebook\"\n
    \n\n\n\n\n\n\n\n
    \n\"Twitter\"\n
    \n\n\n\n\n\n\n\n
    \n\"LinkedIn\"\n
    \n\n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n
    \n\n
    \n
    \n
    \u200c
    \n\n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:42+00:00", "updated": "2022-05-31T06:36:46+00:00"}, "emitted_at": 1679665066642} +{"stream": "metrics", "data": {"object": "metric", "id": "RUQ6YQ", "name": "Active on Site", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105477} +{"stream": "metrics", "data": {"object": "metric", "id": "Xi7Kwh", "name": "Bounced Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105478} +{"stream": "metrics", "data": {"object": "metric", "id": "VePdj9", "name": "Cancelled Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105478} +{"stream": "metrics", "data": {"object": "metric", "id": "SPnhc3", "name": "Checkout Started", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105478} +{"stream": "metrics", "data": {"object": "metric", "id": "SxR9Bt", "name": "Clicked Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105478} +{"stream": "metrics", "data": {"object": "metric", "id": "VFFb4u", "name": "Clicked Email", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2021-05-17T23:43:50+00:00", "updated": "2021-05-17T23:43:50+00:00"}, "emitted_at": 1688724105479} +{"stream": "metrics", "data": {"object": "metric", "id": "Y5TbbA", "name": "Clicked SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105479} +{"stream": "metrics", "data": {"object": "metric", "id": "RhP4nd", "name": "Dropped Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105479} +{"stream": "metrics", "data": {"object": "metric", "id": "TeZiVn", "name": "Failed to deliver Automated Response SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105479} +{"stream": "metrics", "data": {"object": "metric", "id": "S5Au3w", "name": "Failed to Deliver SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105479} +{"stream": "metrics", "data": {"object": "metric", "id": "X3f6PC", "name": "Fulfilled Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "RcjEmN", "name": "Fulfilled Partial Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:45:47+00:00", "updated": "2022-05-31T06:45:47+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "THfYvj", "name": "Marked Email as Spam", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "Yy9QKx", "name": "Opened Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "RDXsib", "name": "Ordered Product", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "TspjNE", "name": "Placed Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "RszrqT", "name": "Received Automated Response SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "WKHXf4", "name": "Received Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105480} +{"stream": "metrics", "data": {"object": "metric", "id": "WhthF7", "name": "Received SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "R2WpFy", "name": "Refunded Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "XsS8yX", "name": "Sent SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "UBNaGw", "name": "Subscribed to Back in Stock", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "Tp8t7d", "name": "Subscribed to List", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-11-16T15:05:22+00:00", "updated": "2021-11-16T15:05:22+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "VEsf4u", "name": "Subscribed to SMS Marketing", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "VvFRZN", "name": "Unsubscribed", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "TS2mxZ", "name": "Unsubscribed from SMS Marketing", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} +{"stream": "metrics", "data": {"object": "metric", "id": "YcDVHu", "name": "Viewed Product", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105482} +{"stream": "email_templates", "data": {"object": "email-template", "id": "RdbN2P", "name": "Newsletter #1 (Images & Text)", "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n
    \n\n\n\n\n\n\n
    \n\n
    \n\n\n\n\n\n\n
    \n\n
    \n\n
    \n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n\n\n\n\n
    \n\n\n\n
    \n\n
    \n
    \n\n\n\n\n
    \n\n\n\n\n
    \n\n SHOP NOW\n \n
    \n
    \n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    \n

    \n\n
    \n
    \n
    \n
    \n\n
    \n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    This template starts with images.

    \n
    \n
    \n
    \n
    \n\n
    \n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n\n
    \n
    \n
    \n
    \n
    \n
    \n\n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n
    \n\n
    \n
    \n
    \n
    \n
    \n
    \n\n
    \n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n

    Everyone loves pictures. They're more engaging that text by itself and the images in this template will neatly stack on mobile devices for the best viewing experience.

    \n

    Use the text area below to add additional content or add more images to create a larger image gallery. You can drag blocks from the left sidebar to add content to your template. You can customize this colors, fonts and styling of this template to match your brand by clicking the \"Styles\" button to the left.

    \n

    Happy emailing!

    \n

    The Klaviyo Team

    \n
    \n
    \n
    \n
    \n\n
    \n\n\n
    \n\n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n
    \n\n
    \n\n
    \n\n\"Facebook\"\n\n
    \n\n
    \n
    \n\n
    \n\n\"Twitter\"\n\n
    \n\n
    \n
    \n\n
    \n\n\"LinkedIn\"\n\n
    \n\n
    \n\n
    \n
    \n
    \n
    \n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n
    No longer want to receive these emails? {% unsubscribe %}.
    {{ organization.name }} {{ organization.full_address }}
    \n
    \n
    \n
    \n
    \n\n
    \n\n
    \n\n
    \n
    \n\n
    \n
    \n\n
    \n\n
    \n\n\n\n\n\n\n
    \n\n
    \n\n\n\n\n\n\n
    \n\n\n\n\n\n\n
    \n\n\"Powered\n\n
    \n
    \n
    \n\n
    \n
    \n\n
    \n
    \n\n", "is_writeable": true, "created": "2021-03-31T10:50:37+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1686259883909} {"stream": "profiles", "data": {"type": "profile", "id": "01F5VTP8THZD8CGS2AKNE63370", "attributes": {"email": "some.email.that.dont.exist@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name", "last_name": "Last Name", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:12:55+00:00", "updated": "2021-05-17T00:12:55+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/segments/"}}}, "updated": "2021-05-17T00:12:55+00:00"}, "emitted_at": 1679533540462} {"stream": "profiles", "data": {"type": "profile", "id": "01F5VTQ44548K2TBCG1EWPZEDN", "attributes": {"email": "some.email.that.dont.exist2@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "Strange Name1", "last_name": "Funny Name1", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:13:23+00:00", "updated": "2021-05-17T00:16:44+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/segments/"}}}, "updated": "2021-05-17T00:16:44+00:00"}, "emitted_at": 1679533540462} {"stream": "profiles", "data": {"type": "profile", "id": "01F5VTX8KP49GGQ4BG77HZ9FRH", "attributes": {"email": "some.email.that.dont.exist3@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "Strange Name2", "last_name": "Funny Name2", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:16:44+00:00", "updated": "2021-05-17T00:16:44+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": null, "country": null, "latitude": null, "longitude": null, "region": null, "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/segments/"}}}, "updated": "2021-05-17T00:16:44+00:00"}, "emitted_at": 1679533540463} diff --git a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml index a0aed604e565..c062944c3905 100644 --- a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.2 dockerRepository: airbyte/source-klaviyo githubIssueLabel: source-klaviyo icon: klaviyo.svg @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/klaviyo tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-klaviyo/requirements.txt b/airbyte-integrations/connectors/source-klaviyo/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-klaviyo/requirements.txt +++ b/airbyte-integrations/connectors/source-klaviyo/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-klaviyo/setup.py b/airbyte-integrations/connectors/source-klaviyo/setup.py index 0c71eecd134e..3603d9d4885b 100644 --- a/airbyte-integrations/connectors/source-klaviyo/setup.py +++ b/airbyte-integrations/connectors/source-klaviyo/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "connector-acceptance-test", "requests_mock~=1.8"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock~=1.8"] setup( name="source_klaviyo", diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/__init__.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/__init__.py index f9ad82ab7107..53739124631f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/__init__.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from .source import SourceKlaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json index 09a5cba42c4b..da46b6550410 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json @@ -1,27 +1,29 @@ { - "type": "object", - "additionalProperties": true, - "properties": { - "object": { - "type": "string" - }, - "id": { - "type": "string" - }, - "name": { - "type": ["null", "string"] - }, - "html": { - "type": ["null", "string"] - }, - "is_writeable": { - "type": ["null", "boolean"] - }, - "created": { - "type": "string", "format": "date-time" - }, - "updated": { - "type": "string", "format": "date-time" - } + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": ["null", "string"] + }, + "html": { + "type": ["null", "string"] + }, + "is_writeable": { + "type": ["null", "boolean"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "updated": { + "type": "string", + "format": "date-time" } + } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json index a7ca8550707d..21f80313fd0a 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/profiles.json @@ -26,7 +26,7 @@ } }, "links": { "type": ["null", "object"] }, - "relationships": { "type": ["null", "object"] }, - "segments": { "type": ["null", "object"] } + "relationships": { "type": ["null", "object"] }, + "segments": { "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py index 3b18c19d05bb..4843cd213aa1 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import re from typing import Any, List, Mapping, Tuple from airbyte_cdk.models import SyncMode @@ -21,7 +22,15 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any # we use metrics endpoint because it never returns an error _ = list(Metrics(api_key=config["api_key"]).read_records(sync_mode=SyncMode.full_refresh)) except Exception as e: - return False, repr(e) + original_error_message = repr(e) + + # Regular expression pattern to match the API key + pattern = r"api_key=\b\w+\b" + + # Remove the API key from the error message + error_message = re.sub(pattern, "api_key=***", original_error_message) + + return False, error_message return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index 64016f355beb..f2b1b085449f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -198,6 +198,7 @@ class IncrementalKlaviyoStreamV1(KlaviyoStreamV1, ABC): def __init__(self, start_date: str, **kwargs): super().__init__(**kwargs) self._start_ts = int(pendulum.parse(start_date).timestamp()) + self._start_sync = int(pendulum.now().timestamp()) @property @abstractmethod @@ -232,7 +233,9 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late latest_record = datetime.datetime.strptime(latest_record, "%Y-%m-%d %H:%M:%S") latest_record = datetime.datetime.timestamp(latest_record) - return {self.cursor_field: max(latest_record, state_ts)} + new_value = max(latest_record, state_ts) + new_value = min(new_value, self._start_sync) + return {self.cursor_field: new_value} def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py index 5a8549abc6b5..bc90b7154d08 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py @@ -14,13 +14,13 @@ 400, "Bad request", False, - "HTTPError('400 Client Error: None for url: https://a.klaviyo.com/api/v1/metrics?api_key=api_key&count=100')", + "HTTPError('400 Client Error: None for url: https://a.klaviyo.com/api/v1/metrics?api_key=***&count=100')", ), ( 403, "Forbidden", False, - "HTTPError('403 Client Error: None for url: https://a.klaviyo.com/api/v1/metrics?api_key=api_key&count=100')", + "HTTPError('403 Client Error: None for url: https://a.klaviyo.com/api/v1/metrics?api_key=***&count=100')", ), ), ) diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py index e2f02306858c..fcee95efe92f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py @@ -136,13 +136,15 @@ def test_request_params(self, next_page_token, stream_state, expected_params): {"updated_at": "2021-04-03 17:15:12", "some_field": 100}, {"updated_at": datetime.strptime("2021-04-03 17:15:12", "%Y-%m-%d %H:%M:%S").timestamp()}, ), + ({"updated_at": 12}, {"updated_at": 7998603215}, None), + ({"updated_at": 7998603215}, {"updated_at": 12}, None), ], ) def test_get_updated_state(self, current_state, latest_record, expected_state): stream = SomeIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) result = stream.get_updated_state(current_stream_state=current_state, latest_record=latest_record) - assert result == expected_state + assert result == (expected_state if expected_state else {stream.cursor_field: stream._start_sync}) @pytest.mark.parametrize( ["response_json", "next_page_token"], diff --git a/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml b/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml index a0df722980b1..ede0b9624645 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-kustomer-singer/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/kustomer + documentationUrl: https://docs.airbyte.com/integrations/sources/kustomer-singer tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-kustomer-singer/requirements.txt b/airbyte-integrations/connectors/source-kustomer-singer/requirements.txt index 8f1f1d5c6b27..16f70161a28f 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-kustomer-singer/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -# -e ../../bases/connector-acceptance-test --e . +# -e . diff --git a/airbyte-integrations/connectors/source-kustomer-singer/setup.py b/airbyte-integrations/connectors/source-kustomer-singer/setup.py index 2a0d6bcebf6c..5e14ee6e2e1d 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/setup.py +++ b/airbyte-integrations/connectors/source-kustomer-singer/setup.py @@ -50,7 +50,7 @@ def run(self): MAIN_REQUIREMENTS = ["airbyte-cdk", "tap-kustomer==1.0.2"] -TEST_REQUIREMENTS = ["pytest~=6.1"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1"] setup( name="source_kustomer_singer", diff --git a/airbyte-integrations/connectors/source-kyriba/metadata.yaml b/airbyte-integrations/connectors/source-kyriba/metadata.yaml index 2c7d2a099608..347d7b22f2fa 100644 --- a/airbyte-integrations/connectors/source-kyriba/metadata.yaml +++ b/airbyte-integrations/connectors/source-kyriba/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/kyriba tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-kyriba/requirements.txt b/airbyte-integrations/connectors/source-kyriba/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-kyriba/requirements.txt +++ b/airbyte-integrations/connectors/source-kyriba/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-kyriba/setup.py b/airbyte-integrations/connectors/source-kyriba/setup.py index e45e6b44413e..c81f4d4e1e0d 100644 --- a/airbyte-integrations/connectors/source-kyriba/setup.py +++ b/airbyte-integrations/connectors/source-kyriba/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-kyve/.dockerignore b/airbyte-integrations/connectors/source-kyve/.dockerignore new file mode 100644 index 000000000000..12f6f20c1ce3 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_kyve +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-kyve/Dockerfile b/airbyte-integrations/connectors/source-kyve/Dockerfile new file mode 100644 index 000000000000..13c92bb1b460 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_kyve ./source_kyve + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-kyve diff --git a/airbyte-integrations/connectors/source-kyve/README.md b/airbyte-integrations/connectors/source-kyve/README.md new file mode 100644 index 000000000000..9c864ea7d8f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/README.md @@ -0,0 +1,141 @@ +# Kyve Source + +This is the repository for the Kyve source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/kyve). + +## Local development + +### Prerequisites + +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies + +From this connector directory, create a virtual environment: + +```sh +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: + +```sh +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` + +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle + +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: + +```sh +./gradlew :airbyte-integrations:connectors:source-kyve:build +``` + +### Locally running the connector + +```sh +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build + +First, make sure you build the latest Docker image: + +```sh +docker build . -t airbyte/source-kyve:dev +``` + +You can also build the connector image via Gradle: + +```sh +./gradlew :airbyte-integrations:connectors:source-kyve:airbyteDocker +``` + +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run + +Then run any of the connector commands as follows: + +```sh +docker run --rm airbyte/source-kyve:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kyve:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kyve:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-kyve:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing + +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: + +```sh +pip install .[tests] +``` + +### Unit Tests + +To run unit tests locally, from the connector directory run: + +```sh +python -m pytest unit_tests +``` + +### Integration Tests + +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). + +#### Custom Integration tests + +Place custom tests inside `integration_tests/` folder, then, from the connector root, run + +```sh +python -m pytest integration_tests +``` + +#### Acceptance Tests + +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run + +```sh +python -m pytest integration_tests -p integration_tests.acceptance +``` + +To run your integration tests with docker + +### Using gradle to run tests + +All commands should be run from airbyte project root. +To run unit tests: + +```sh +./gradlew :airbyte-integrations:connectors:source-kyve:unitTest +``` + +To run acceptance and custom integration tests: + +```sh +./gradlew :airbyte-integrations:connectors:source-kyve:integrationTest +``` diff --git a/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml b/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml new file mode 100644 index 000000000000..d4daf4793898 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-kyve:dev +test_strictness_level: low +acceptance_tests: + spec: + tests: + - spec_path: "source_kyve/spec.yaml" + config_path: "secrets/config.json" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_multiple_pools.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + bypass_reason: "Schema tests fail by default as the offset is not present in the data but important for the pagination." + basic_read: + tests: + - config_path: "secrets/config.json" + timeout_seconds: 60 + - config_path: "secrets/config_multiple_pools.json" + timeout_seconds: 60 + full_refresh: + tests: + - config_path: "secrets/config.json" + incremental: + bypass_reason: "Schema tests fail by default as as the offset is not present in the data but important for the pagination." diff --git a/airbyte-integrations/connectors/source-kyve/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-kyve/acceptance-test-docker.sh new file mode 100755 index 000000000000..5797d20fe9a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/acceptance-test-docker.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-kyve/build.gradle b/airbyte-integrations/connectors/source-kyve/build.gradle new file mode 100644 index 000000000000..a14286e26086 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_kyve' +} diff --git a/airbyte-integrations/connectors/source-kyve/icon.svg b/airbyte-integrations/connectors/source-kyve/icon.svg new file mode 100644 index 000000000000..7701b6ab547a --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/__init__.py b/airbyte-integrations/connectors/source-kyve/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-kyve/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..bb06d5557438 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "pool_0": { + "offset": "1000000000" + } +} diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py new file mode 100644 index 000000000000..ca7e3877a08f --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +# TODO: check with Airbyte-team if this is the right place to have this +def pytest_collection_modifyitems(config, items): + skip_cursor = pytest.mark.skip(reason="MANUALLY SKIPPED: Cursor never in schema") + for item in items: + if "test_defined_cursors_exist_in_schema" in item.name or "test_read_sequential_slices" in item.name or "test_two_sequential_reads" in item.name: + item.add_marker(skip_cursor) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/config.json b/airbyte-integrations/connectors/source-kyve/integration_tests/config.json new file mode 100644 index 000000000000..396fb936447e --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/config.json @@ -0,0 +1,7 @@ +{ + "pool_ids": "0", + "start_ids": "0", + "url_base": "https://api.korellia.kyve.network", + "page_size": 1, + "max_pages": 2 +} diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-kyve/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..35069bd2fe17 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "pool_0", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-kyve/integration_tests/incremental_catalog.json new file mode 100644 index 000000000000..4816982dd2a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/incremental_catalog.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "pool_0", + "json_schema": {}, + "source_defined_cursor": true, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "default_cursor_field": ["offset"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-kyve/integration_tests/invalid_config.json new file mode 100644 index 000000000000..77e5b6c98836 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "pool_ids": 2 +} diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json b/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json new file mode 100644 index 000000000000..408a9dd4d401 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json @@ -0,0 +1,7 @@ +{ + "pool_ids": "0,1", + "start_ids": "0,0", + "url_base": "https://api.korellia.kyve.network", + "page_size": 1, + "max_pages": 2 +} diff --git a/airbyte-integrations/connectors/source-kyve/main.py b/airbyte-integrations/connectors/source-kyve/main.py new file mode 100644 index 000000000000..f4055d71e80e --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_kyve import SourceKyve + +if __name__ == "__main__": + source = SourceKyve() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-kyve/metadata.yaml b/airbyte-integrations/connectors/source-kyve/metadata.yaml new file mode 100644 index 000000000000..f3240f90b9af --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/metadata.yaml @@ -0,0 +1,25 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: 60a1efcc-c31c-4c63-b508-5b48b6a9f4a6 + dockerImageTag: 0.1.0 + maxSecondsBetweenMessages: 7200 + dockerRepository: airbyte/source-kyve + githubIssueLabel: source-kyve + icon: kyve.svg + license: MIT + name: KYVE + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/kyve + tags: + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-kyve/requirements.txt b/airbyte-integrations/connectors/source-kyve/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/source-kyve/setup.py b/airbyte-integrations/connectors/source-kyve/setup.py new file mode 100644 index 000000000000..0ed10d850cea --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + +setup( + name="source_kyve", + description="Source implementation for Kyve.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/__init__.py b/airbyte-integrations/connectors/source-kyve/source_kyve/__init__.py new file mode 100644 index 000000000000..d22f602a893c --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceKyve + +__all__ = ["SourceKyve"] diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/block.json new file mode 100644 index 000000000000..735b6990dc29 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/block.json @@ -0,0 +1,84 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "hash": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + }, + "version": { + "type": ["null", "integer"] + }, + "versionHex": { + "type": ["null", "string"] + }, + "merkleroot": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "integer"] + }, + "mediantime": { + "type": ["null", "integer"] + }, + "nonce": { + "type": ["null", "integer"] + }, + "bits": { + "type": ["null", "string"] + }, + "difficulty": { + "type": ["null", "number"] + }, + "chainwork": { + "type": ["null", "string"] + }, + "nTx": { + "type": ["null", "integer"] + }, + "previousblockhash": { + "type": ["null", "string"] + }, + "nextblockhash": { + "type": ["null", "string"] + }, + "strippedsize": { + "type": ["null", "integer"] + }, + "size": { + "type": ["null", "integer"] + }, + "weight": { + "type": ["null", "integer"] + }, + "tx": { + "type": ["null", "array"], + "items": { + "$ref": "bitcoin/transaction.json" + } + } + }, + "required": [ + "hash", + "height", + "version", + "versionHex", + "merkleroot", + "time", + "mediantime", + "nonce", + "bits", + "difficulty", + "chainwork", + "nTx", + "previousblockhash", + "nextblockhash", + "strippedsize", + "size", + "weight", + "tx" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/transaction.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/transaction.json new file mode 100644 index 000000000000..817aab822182 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/transaction.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "txid": { + "type": ["null", "string"] + }, + "hash": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "integer"] + }, + "size": { + "type": ["null", "integer"] + }, + "vsize": { + "type": ["null", "integer"] + }, + "weight": { + "type": ["null", "integer"] + }, + "locktime": { + "type": ["null", "integer"] + }, + "vin": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "txid": { + "type": ["null", "string"] + }, + "vout": { + "type": ["null", "integer"] + }, + "scriptSig": { + "type": ["null", "object"], + "properties": { + "asm": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": ["asm", "hex"] + }, + "sequence": { + "type": ["null", "integer"] + } + }, + "required": ["txid", "vout", "scriptSig", "sequence"] + } + }, + "vout": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "number"] + }, + "n": { + "type": ["null", "integer"] + }, + "scriptPubKey": { + "type": ["null", "object"], + "properties": { + "asm": { + "type": ["null", "string"] + }, + "desc": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["asm", "desc", "hex", "address", "type"] + } + }, + "required": ["value", "n", "scriptPubKey"] + } + }, + "fee": { + "type": ["null", "number"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": [ + "txid", + "hash", + "version", + "size", + "vsize", + "weight", + "locktime", + "vin", + "vout", + "fee", + "hex" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/block.json new file mode 100644 index 000000000000..a7aa400be894 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/block.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "hash": { + "type": ["null", "string"] + }, + "parentHash": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "timestamp": { + "type": ["null", "integer"] + }, + "nonce": { + "type": ["null", "string"] + }, + "difficulty": { + "type": ["null", "integer"] + }, + "gasLimit": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": ["type", "hex"] + }, + "gasUsed": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": ["type", "hex"] + }, + "miner": { + "type": ["null", "string"] + }, + "transactions": { + "type": ["null", "array"], + "items": { + "$ref": "celo/transaction.json" + } + }, + "_difficulty": { + "type": "null" + } + }, + "required": [ + "hash", + "parentHash", + "number", + "timestamp", + "nonce", + "difficulty", + "gasLimit", + "gasUsed", + "miner", + "transactions", + "_difficulty" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/transaction.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/transaction.json new file mode 100644 index 000000000000..6a48246b53af --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/transaction.json @@ -0,0 +1,108 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "hash": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "integer"] + }, + "accessList": { + "type": "null" + }, + "blockHash": { + "type": ["null", "string"] + }, + "blockNumber": { + "type": ["null", "integer"] + }, + "transactionIndex": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "gasPrice": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": ["type", "hex"] + }, + "gasLimit": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": ["type", "hex"] + }, + "to": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "hex": { + "type": ["null", "string"] + } + }, + "required": ["type", "hex"] + }, + "nonce": { + "type": ["null", "integer"] + }, + "data": { + "type": ["null", "string"] + }, + "r": { + "type": ["null", "string"] + }, + "s": { + "type": ["null", "string"] + }, + "v": { + "type": ["null", "integer"] + }, + "creates": { + "type": "null" + }, + "chainId": { + "type": ["null", "integer"] + } + }, + "required": [ + "hash", + "type", + "accessList", + "blockHash", + "blockNumber", + "transactionIndex", + "from", + "gasPrice", + "gasLimit", + "to", + "value", + "nonce", + "data", + "r", + "s", + "v", + "creates", + "chainId" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/block.json new file mode 100644 index 000000000000..9e64350abfe9 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/block.json @@ -0,0 +1,232 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "block_id": { + "type": ["null", "object"], + "properties": { + "hash": { + "type": ["null", "string"] + }, + "part_set_header": { + "type": ["null", "object"], + "properties": { + "total": { + "type": ["null", "integer"] + }, + "hash": { + "type": ["null", "string"] + } + }, + "required": ["total", "hash"] + } + }, + "required": ["hash", "part_set_header"] + }, + "block": { + "type": ["null", "object"], + "properties": { + "header": { + "type": ["null", "object"], + "properties": { + "version": { + "type": ["null", "object"], + "properties": { + "block": { + "type": ["null", "string"] + }, + "app": { + "type": ["null", "string"] + } + }, + "required": ["block", "app"] + }, + "chain_id": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "string"] + }, + "time": { + "type": ["null", "string"] + }, + "last_block_id": { + "type": ["null", "object"], + "properties": { + "hash": { + "type": ["null", "string"] + }, + "part_set_header": { + "type": ["null", "object"], + "properties": { + "total": { + "type": ["null", "integer"] + }, + "hash": { + "type": ["null", "string"] + } + }, + "required": ["total", "hash"] + } + }, + "required": ["hash", "part_set_header"] + }, + "last_commit_hash": { + "type": ["null", "string"] + }, + "data_hash": { + "type": ["null", "string"] + }, + "validators_hash": { + "type": ["null", "string"] + }, + "next_validators_hash": { + "type": ["null", "string"] + }, + "consensus_hash": { + "type": ["null", "string"] + }, + "app_hash": { + "type": ["null", "string"] + }, + "last_results_hash": { + "type": ["null", "string"] + }, + "evidence_hash": { + "type": ["null", "string"] + }, + "proposer_address": { + "type": ["null", "string"] + } + }, + "required": [ + "version", + "chain_id", + "height", + "time", + "last_block_id", + "last_commit_hash", + "data_hash", + "validators_hash", + "next_validators_hash", + "consensus_hash", + "app_hash", + "last_results_hash", + "evidence_hash", + "proposer_address" + ] + }, + "data": { + "type": ["null", "object"], + "properties": { + "txs": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + }, + "required": ["txs"] + }, + "evidence": { + "type": ["null", "object"], + "properties": { + "evidence": { + "type": ["null", "array"], + "items": { + "items": {} + } + } + }, + "required": ["evidence"] + }, + "last_commit": { + "type": ["null", "object"], + "properties": { + "height": { + "type": ["null", "string"] + }, + "round": { + "type": ["null", "integer"] + }, + "block_id": { + "type": ["null", "object"], + "properties": { + "hash": { + "type": ["null", "string"] + }, + "part_set_header": { + "type": ["null", "object"], + "properties": { + "total": { + "type": ["null", "integer"] + }, + "hash": { + "type": ["null", "string"] + } + }, + "required": ["total", "hash"] + } + }, + "required": ["hash", "part_set_header"] + }, + "signatures": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "block_id_flag": { + "type": ["null", "string"] + }, + "validator_address": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"] + }, + "signature": { + "type": ["null", "string"] + } + }, + "required": [ + "block_id_flag", + "validator_address", + "timestamp", + "signature" + ] + } + } + }, + "required": ["height", "round", "block_id", "signatures"] + } + }, + "required": ["header", "data", "evidence", "last_commit"] + }, + "__kyve": { + "type": ["null", "object"], + "properties": { + "block": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "object"], + "properties": { + "parsed_txs": { + "type": ["null", "array"], + "items": { + "$ref": "cosmos/parsedtxs.json" + } + } + }, + "required": ["parsed_txs"] + } + }, + "required": ["data"] + } + }, + "required": ["block"] + } + }, + "required": ["block_id", "block", "__kyve"] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/parsedtxs.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/parsedtxs.json new file mode 100644 index 000000000000..44584e37b678 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/parsedtxs.json @@ -0,0 +1,132 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "height": { + "type": ["null", "string"] + }, + "txhash": { + "type": ["null", "string"] + }, + "codespace": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "integer"] + }, + "data": { + "type": ["null", "string"] + }, + "raw_log": { + "type": ["null", "string"] + }, + "logs": { + "type": ["null", "array"], + "items": {} + }, + "info": { + "type": ["null", "string"] + }, + "gas_wanted": { + "type": ["null", "string"] + }, + "gas_used": { + "type": ["null", "string"] + }, + "tx": { + "type": ["null", "object"], + "properties": { + "@type": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "object"], + "properties": { + "messages": { + "type": ["null", "array"], + "items": {} + }, + "memo": { + "type": ["null", "string"] + }, + "timeout_height": { + "type": ["null", "string"] + }, + "extension_options": { + "type": ["null", "array"], + "items": {} + }, + "non_critical_extension_options": { + "type": ["null", "array"], + "items": {} + } + }, + "required": [ + "messages", + "memo", + "timeout_height", + "extension_options", + "non_critical_extension_options" + ] + }, + "auth_info": { + "type": ["null", "object"], + "properties": { + "signer_infos": { + "type": ["null", "array"], + "items": {} + }, + "fee": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "array"], + "items": {} + }, + "gas_limit": { + "type": ["null", "string"] + }, + "payer": { + "type": ["null", "string"] + }, + "granter": { + "type": ["null", "string"] + } + }, + "required": ["amount", "gas_limit", "payer", "granter"] + } + }, + "required": ["signer_infos", "fee"] + }, + "signatures": { + "type": "array", + "items": {} + } + }, + "required": ["@type", "body", "auth_info", "signatures"] + }, + "timestamp": { + "type": ["null", "string"] + }, + "events": { + "type": ["null", "array"], + "items": {} + } + }, + "required": [ + "height", + "txhash", + "codespace", + "code", + "data", + "raw_log", + "logs", + "info", + "gas_wanted", + "gas_used", + "tx", + "timestamp", + "events" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/block.json new file mode 100644 index 000000000000..e3d4b5365385 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/block.json @@ -0,0 +1,83 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "hash": { + "type": ["null", "string"] + }, + "miner": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "gasUsed": { + "type": ["null", "object"], + "properties": { + "hex": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["hex", "type"] + }, + "gasLimit": { + "type": ["null", "object"], + "properties": { + "hex": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["hex", "type"] + }, + "extraData": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "integer"] + }, + "difficulty": { + "type": ["null", "integer"] + }, + "parentHash": { + "type": ["null", "string"] + }, + "_difficulty": { + "type": ["null", "object"], + "properties": { + "hex": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["hex", "type"] + }, + "transactions": { + "type": ["null", "array"], + "items": { + "$ref": "evm/transaction.json" + } + } + }, + "required": [ + "hash", + "miner", + "number", + "gasUsed", + "gasLimit", + "extraData", + "timestamp", + "difficulty", + "parentHash", + "_difficulty", + "transactions" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/transaction.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/transaction.json new file mode 100644 index 000000000000..b0c818fe7240 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/transaction.json @@ -0,0 +1,111 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "r": { + "type": ["null", "string"] + }, + "s": { + "type": ["null", "string"] + }, + "v": { + "type": ["null", "integer"] + }, + "to": { + "type": ["string", "null"] + }, + "raw": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "hash": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "integer"] + }, + "nonce": { + "type": ["null", "integer"] + }, + "value": { + "type": ["null", "object"], + "properties": { + "hex": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["hex", "type"] + }, + "chainId": { + "type": ["null", "integer"] + }, + "creates": { + "type": ["null", "string"] + }, + "gasLimit": { + "type": ["null", "object"], + "properties": { + "hex": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["hex", "type"] + }, + "gasPrice": { + "type": ["null", "object"], + "properties": { + "hex": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "required": ["hex", "type"] + }, + "blockHash": { + "type": ["null", "string"] + }, + "accessList": { + "type": "null" + }, + "blockNumber": { + "type": ["null", "integer"] + }, + "transactionIndex": { + "type": ["null", "integer"] + } + }, + "required": [ + "r", + "s", + "v", + "to", + "data", + "from", + "hash", + "type", + "nonce", + "value", + "chainId", + "creates", + "gasLimit", + "gasPrice", + "blockHash", + "accessList", + "blockNumber", + "transactionIndex" + ] +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/uniswap/event.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/uniswap/event.json new file mode 100644 index 000000000000..9d945a06b04f --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/uniswap/event.json @@ -0,0 +1,55 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "contains": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "blockNumber": { + "type": ["null", "integer"] + }, + "blockHash": { + "type": ["null", "string"] + }, + "transactionIndex": { + "type": ["null", "integer"] + }, + "removed": { + "type": ["null", "boolean"] + }, + "address": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "string"] + }, + "topics": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "transactionHash": { + "type": ["null", "string"] + }, + "logIndex": { + "type": ["null", "integer"] + }, + "parsedEvent": { + "type": ["null", "object"] + } + }, + "required": [ + "blockNumber", + "blockHash", + "transactionIndex", + "removed", + "address", + "data", + "topics", + "transactionHash", + "logIndex", + "parsedEvent" + ] + } +} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/source.py b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py new file mode 100644 index 000000000000..2ec43d5c80db --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/source.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from copy import deepcopy +from typing import Any, List, Mapping, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream + +from .stream import KYVEStream + + +class SourceKyve(AbstractSource): + def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # check that pools and bundles are the same length + pools = config.get("pool_ids").split(",") + start_ids = config.get("start_ids").split(",") + + if not len(pools) == len(start_ids): + return False, "Please add a start_id for every pool" + + for pool_id in pools: + try: + # check if endpoint is available and returns valid data + response = requests.get(f"{config['url_base']}/kyve/query/v1beta1/pool/{pool_id}") + if not response.ok: + # todo improve error handling for cases like pool not found + return False, response.json() + except Exception as e: + return False, e + + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + streams: List[Stream] = [] + + pools = config.get("pool_ids").split(",") + start_ids = config.get("start_ids").split(",") + + for (pool_id, start_id) in zip(pools, start_ids): + response = requests.get(f"{config['url_base']}/kyve/query/v1beta1/pool/{pool_id}") + pool_data = response.json().get("pool").get("data") + + config_copy = dict(deepcopy(config)) + config_copy["start_ids"] = int(start_id) + # add a new stream based on the pool_data + streams.append(KYVEStream(config=config_copy, pool_data=pool_data)) + + return streams diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml b/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml new file mode 100644 index 000000000000..dae37e3ca58c --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml @@ -0,0 +1,51 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/kyve +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: KYVE Spec + type: object + required: + - pool_ids + - start_ids + properties: + pool_ids: + type: string + title: Pool-IDs + description: The IDs of the KYVE storage pool you want to archive. (Comma separated) + order: 0 + examples: + - "0" + - "0,1" + start_ids: + type: string + title: Bundle-Start-IDs + description: + The start-id defines, from which bundle id the pipeline should + start to extract the data (Comma separated) + order: 1 + examples: + - "0" + - "0,0" + url_base: + type: string + title: KYVE-API URL Base + description: URL to the KYVE Chain API. + default: https://api.korellia.kyve.network + order: 2 + examples: + - https://api.korellia.kyve.network/ + - https://api.beta.kyve.network/ + max_pages: + type: integer + description: + The maximum amount of pages to go trough. Set to 'null' for all + pages. + order: 3 + airbyte_hidden: true + page_size: + type: integer + description: + The pagesize for pagination, smaller numbers are used in integration + tests. + default: 100 + order: 4 + airbyte_hidden: true diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py new file mode 100644 index 000000000000..31184963c9ac --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py @@ -0,0 +1,162 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import gzip +import json +import logging +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.sources.streams import IncrementalMixin +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.streams.http import HttpStream +from source_kyve.util import CustomResourceSchemaLoader + +logger = logging.getLogger("airbyte") + +# this mapping handles the schema to runtime relation +# this needs to be updated whenever a new schema is integrated +runtime_to_root_file_mapping = { + "@kyvejs/bitcoin": "bitcoin/block", + "@kyvejs/celo": "celo/block", + "@kyvejs/cosmos": "cosmos/block", + "@kyvejs/evm": "evm/block", + "@kyvejs/uniswap": "uniswap/event", +} + + +class KYVEStream(HttpStream, IncrementalMixin): + url_base = None + + cursor_field = "offset" + page_size = 100 + + # Set this as a noop. + primary_key = None + + name = None + + def __init__(self, config: Mapping[str, Any], pool_data: Mapping[str, Any], **kwargs): + super().__init__(**kwargs) + # Here's where we set the variable from our input to pass it down to the source. + self.pool_id = pool_data.get("id") + + self.name = f"pool_{self.pool_id}" + self.runtime = pool_data.get("runtime") + + self.url_base = config["url_base"] + # this is an ugly solution but has to parsed by source to be a single item + self._offset = int(config["start_ids"]) + + self.page_size = config["page_size"] + self.max_pages = config.get("max_pages", None) + # For incremental querying + self._cursor_value = None + + def get_json_schema(self) -> Mapping[str, Any]: + # this is KYVE's default schema, if a root_schema is defined + # the ResourceSchemaLoader automatically resolves the dependency + schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": {"key": {"type": "integer"}, "value": {"type": "object"}}, + "required": ["key", "value"], + } + # in case we have defined a schema file, we can get it from the mapping + schema_root_file = runtime_to_root_file_mapping.get(self.runtime, None) + + # we update the default schema in case there is a root_file + if schema_root_file: + inlay_schema = CustomResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(schema_root_file) + schema["properties"]["value"] = inlay_schema + return schema + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"/kyve/query/v1beta1/finalized_bundles/{self.pool_id}" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + # Set the pagesize in the request parameters + params = {"pagination.limit": self.page_size} + + # Handle pagination by inserting the next page's token in the request parameters + if next_page_token: + params["next_page_token"] = next_page_token + + # In case we use incremental streaming, we start with the stored _offset + offset = stream_state.get(self.cursor_field, self._offset) or 0 + + params["pagination.offset"] = offset + + return params + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + try: + # set the state to store the latest bundle_id + bundles = response.json().get("finalized_bundles") + latest_bundle = bundles[-1] + + self._cursor_value = latest_bundle.get("id") + except IndexError: + bundles = [] + + for bundle in bundles: + storage_id = bundle.get("storage_id") + # retrieve file from Arweave + response_from_arweave = requests.get(f"https://arweave.net/{storage_id}") + + if not response.ok: + logger.error(f"Reading bundle {storage_id} with status code {response.status_code}") + # todo future: this is a temporary fix until the bugs with Arweave are solved + continue + try: + decompressed = gzip.decompress(response_from_arweave.content) + except gzip.BadGzipFile as e: + logger.error(f"Decompressing bundle {storage_id} failed with '{e}'") + # todo future: this is a temporary fix until the bugs with Arweave are solved + # todo future: usually this exception should fail + continue + + # todo future: fail on incorrect hash, enabled after regenesis + # bundle_hash = bundle.get("bundle_hash") + # local_hash = hmac.new(b"", msg=decompressed, digestmod=hashlib.sha256).digest().hex() + # assert local_hash == bundle_hash, print("HASHES DO NOT MATCH") + decompressed_as_json = json.loads(decompressed) + + # extract the value from the key -> value mapping + yield from decompressed_as_json + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + # in case we set a max_pages parameter we need to abort + if self.max_pages and self._offset >= self.max_pages * self.page_size: + return + + json_response = response.json() + next_key = json_response.get("pagination", {}).get("next_key") + if next_key: + self._offset += self.page_size + return {"pagination.offset": self._offset} + + @property + def state(self) -> Mapping[str, Any]: + if self._cursor_value: + return {self.cursor_field: self._cursor_value} + else: + return {self.cursor_field: self._offset} + + @state.setter + def state(self, value: Mapping[str, Any]): + self._cursor_value = value[self.cursor_field] diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/util.py b/airbyte-integrations/connectors/source-kyve/source_kyve/util.py new file mode 100644 index 000000000000..23ee80e078dd --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/util.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import importlib +import os + +import jsonref +from airbyte_cdk.sources.utils.schema_helpers import JsonFileLoader, ResourceSchemaLoader, resolve_ref_links + + +# We custom implemented the class to remove the requirement that $ref's have to be resolved from the 'shared folder' +class CustomResourceSchemaLoader(ResourceSchemaLoader): + """JSONSchema loader from package resources""" + + def _resolve_schema_references(self, raw_schema: dict) -> dict: + """ + Resolve links to external references and move it to local "definitions" map. + + :param raw_schema jsonschema to lookup for external links. + :return JSON serializable object with references without external dependencies. + """ + + package = importlib.import_module(self.package_name) + base = os.path.dirname(package.__file__) + "/" + resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/"), base_uri=base) + resolved = resolve_ref_links(resolved) + return resolved diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/__init__.py b/airbyte-integrations/connectors/source-kyve/unit_tests/__init__.py new file mode 100644 index 000000000000..e5cdaf070e65 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/__init__.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +config = { + "pool_ids": "0", + "start_ids": "0", + "url_base": "https://api.korellia.kyve.network", + "page_size": 100 +} + +pool_data = { + "id": "0", + "name": "Moonbeam", + "runtime": "@kyvejs/evm", + "logo": "ar://9FJDam56yBbmvn8rlamEucATH5UcYqSBw468rlCXn8E", + "config": "ar://DgdB-2hLrxjhyEEbCML__dgZN5_uS7T6Z5XDkaFh3P0", + "start_key": "1188653", + "current_key": "3003437", + "current_summary": "0x143ab09916916a7e9a3c1bf2fc18d1a9dece46c7d351bba21bcd64674de4e080", + "current_index": "3002988", + "total_bundles": "115947", + "upload_interval": "120", + "operating_cost": "2500000000", + "min_delegation": "100000000000000", + "max_bundle_size": "100", + "disabled": False, + "funders": [ + { + "address": "kyve1hfvhl7vf635xta2l4y5p4myj23pp7sg08f5rew", + "amount": "360622257494263" + } + ], + "total_funds": "360622257494263", + "protocol": { + "version": "1.0.0-beta.6", + "binaries": "{\"kyve-linux-arm64\":\"https://github.com/KYVENetwork/kyvejs/releases/download/%40kyvejs%2Fevm%401.0.0-beta.6/kyve-linux-arm64.zip\",\"kyve-linux-x64\":\"https://github.com/KYVENetwork/kyvejs/releases/download/%40kyvejs%2Fevm%401.0.0-beta.6/kyve-linux-x64.zip\",\"kyve-macos-x64\":\"https://github.com/KYVENetwork/kyvejs/releases/download/%40kyvejs%2Fevm%401.0.0-beta.6/kyve-macos-x64.zip\"}", + "last_upgrade": "1675501187" + }, + "upgrade_plan": { + "version": "", + "binaries": "", + "scheduled_at": "0", + "duration": "0" + }, + "current_storage_provider_id": 1, + "current_compression_id": 1 +} diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py new file mode 100644 index 000000000000..102469c49b3c --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +MOCK_RESPONSE_BINARY = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xed}]\x8fd7r\xe5\x7f\xe9g=\x90\xc1 \x19\x9c\xb7\xcc\xac\xca\xb5\x81\x85\xe1\x97}2\x8c\x05\xc9 =\x82G\x9a\x81Z\xb3\xb0a\xcc\x7f\xdfCf\xb5\xa4\xce\xbe\xb2>"\xe5I\t\xaaBw\xe5\xc7\xbd\xbc\xfc8\x87\x11\xc1\x88 \xff\xe5\xbf\xde\xfd\xfb\xf8\xcfw\x7fx\xe7]\xe2\x1c\xc9\xbd\xfb\xec\xdd\xff\xab\x7f\xfa\xebx\xf7\x87\xffz\xf7\xc7\xfa\xfe\x8f\xf8\xca\xfdG\xa8n\xe4\xe0\xcb\xa4\x16)\xc8\x0c\xa3uWcW\xf6\xa5\x17j\x93\xbaw\x8e\xe3\xf0U\xe6\xa8\x83\x89h\xf6!\xa5IK$\x19\x85\xfe\xa5~5\xbe\xfc\xfa\x1f>\x94\xe8F\xcbS\x1c.\xe8\xadi\xee3\x87\x1aS\x8a<)y\x1e\xc1\x87T\x9d\x06.\xa9\xd7^gvEk#\xad\xd1\xc7\x11\x93L\x94\xf8\xe5_\xbfh\xe3\xabw\x7f\xf8P\xf3\xcf\xde}\xfd\xf9\x17\xe3\xfd\xd7\xf5\x8b\xbf\xe0\xc3\x94c\xcc)\x14\xc1\x85\x7f\xfe\xb2\x8f\xdbS\xef~P\x8a~>\xe7\xe7\xfd\xaf\x7f\xfa\x1a\x9d\x80"\xfe\xad\xbe\xff\xdf\x9f\x7f\xf1\xf9\xd7\xab\xf9_\xff\xe7_\xd6m\xe7\xcf\xff\xed\x9fn\x8f\xfa\xec\xdd\x1f\xc7\x7f\xdc\n\xa2D\xb1\xa2\x80\xbf\xed[\xfe\xcf\xfb\xa1?t\xc7\xbe\xf6\x8b\xcf\xbf\\U\xc6{-\xd2\x82\x8b\xb5(\x870]"e\xee\xfe%2u\xed\x14_/\x17wu\x95W\x15\xc7\x7f|\xfdU}\xa9_\xd7}#>\xc0\xdb/\xdf\xd7\xfe\xf5\xe7\x7f\xfe\xf2\xfd\xbb?\xfc\xcb\xbf~\xf6\xae\xd5\xf7\xe3:\xc6?\x8f\xaf\xfeW}\xffC5\xe1*>w\xb9\xd5\xe8\xff~\xb7\x03~\xb0\x05\x7f\xfb\xdbgw\x80\xf1\x87\x80\xe1\xe45\x8b\xf0djap)\r\xcd\x95)\xc9\xb7>\xda\x18\xd2Z\xf3h\x9e\'J\x03#\x9e<\xad7\x82\x11v\x94\xeb\xf8\x140\x0f\x80\xe0\x1d`\xfc1`\xca\xb3\x00&U4\xf9c\xd0\xa0\x03\xbc\x9bE$\xc5\x1az\x0c\xaf\xfe\xc5\x15\x96V\\o.\x05\x96\xd7\xe4\xc0\x8f\xf8\xec\xa0\xa1C\xd0Hs\xc49\xf5\x18eP\x9e#\x15\x8dqt\xaa\xa3\xb8\xd4\xc9\x8d\xc1\xb1u\xe9\x92\x9d\x14\x99\xbe\x0e\x9a\xa4\x14\x93\x0fcr?\x98e\x1e\x00\xc3;\xd0\xd0\x11h\xd8\xa5g\x01\xcd\xdd,\xc3%\xbc\xcc:\xa3\xa4<\xaf\'w\n\xe7\x97v\xe9\xe9\xd2\xea9\x8c\xd7\xa9\xf3\x94}\x10\x96g\x07L8\x04\x8co:\xb4\xb0\xe6\xc8Rf\xe7\x16ZeI\x01S\xa8\x96X\xc6(\xdd\x0f\xe7\x00\x8e\x92\xd3L@U\x8d\xade\xc1\x17\x1c]\xf5\x9f\x02\xe6\x01\x10\xbc\x03L8\x04\x8c\xa7g\x01L\x8aC\xe2/\x06\x9a\xef\x8a\x84\x9c\xfc\xf4K\xba\xd7>\x8bw\xcdq.\xc9\x85YH]\x9c\x144\xc5\xde\x83\x8c\xac\x1e\x9d\x8a\t\xbc\x84\xa0\x9a\x0b.s\x99\x17\x17o\xed@w\xd4\xde\xc7{\xf4\xc8{t\xc8\x97\x7f\xfd\xd3\x9f\x00\xcd?\xfd\xb9\xff\xfb?<\x10\x18\xbb\xc0\x7f\xfat,\xbfm\xdf?~\xa9\xab\x13Q\x9d\xf9\xd5\x9f\xbf\xb8\t\xf5D\x1eH\xf1\xd7\x94{\xca\x04\xf5&\x9e\x01\xa0k\xc8W\x06\x96:\xf9p\x1d|zy\xb7\x87\xe7\x9f\xbf\xfa\xbc\x8f\x1f\x1c\x9fU\x0e4\x85\x0fc\xfa\xa3`\xd0C\xbc\xdd\xf0\xf5\x9for\x83\x9c\x0b\xb1y\xf6y\x96Y\xf4\xf5\xc4\xae\xe4\x16\xb3hp\x97\x90\xc3\xe5\x05]\xed\xf8\xbb,\xfb\xc1Y\xe6\r\xbf\xec\x1d\x07\xc8R\xfd\x80\x826\xbb\xc6\x9c\xfd=\xaa\x7f\xfc\x0fI\x87\x1c\xfe\xf9\xf7\xaf\x1f\xb1\xdd\xee\x86\xedv\xdf\x8d\xcf\x87L\x89!$\n%\x94\xb4\xfeJ\x82r\x82O\xe2\xfa\x0c\xa2*@G\x96\xf5]p\xf8\xce\xa7%\xbe(\xe0\xd3%\xc6B\xc4\xb7\xb8*$|\x06\xcd p`\xdc\x19\xf0y\xc2?\xc8\xc5\xc4\xbbD\x94\xb9_A\xf7\xc6O\xd8\xdf`6[/m\xf5\xefn\x95\xb2\xeb\x1f\xf1\x84\xb0\x9f\xebnOD\xbdW\xfd#Z\x98W+\xd6\xd3\xf09Z\xb0\xbf[\x17\xad\xda\xfa\xf5!\xfew\xab\xcd\xbb\xee\xb4\xda\x85O\xd6u\xab\x05\xb4{\x88\xf75\xab\xed\x98bP"\xfe\xae\xa7\x86\xf5}\x0e\xb7:0\xee\xa7\xfd\xdd\xea\xbb\xd5;\xb4\xcb\xa4\xb4J-\xab\x07\xf0\xad\xa0\x84\xd5\x9fe?\x1d\xa5\xe2\xfd\xea\xb5Ug\xd4\x065F-WK\xf6\xd8@\xe1\n\xab\xa6\xbb\xe7W\xc3p}\xd8O\r\xbb\x9c\xbc\xfa\x1c\x1fg\xfc\xc6\xd5\xf3k\xd4\xf0\xbb\xfa\x7f\x95C\xebI\xab\x0c|\xb7\xda\xbd\xeaGh\x15\xe4\xc3\xea\xb3\xb4\xeb\xba[\xb7\xfa\xeb\xd6#\x82\xefp\xc5\xea\xc97L\xc8\x1a\xf3\xd5\xb7a\xbd\xf6\xbbgW9\x06\xfe\xdd\xc6o\xa3k\xf54\xfa\x00\xaf\x1d\x9e{\xc3\x1b\xda\x92V\xff\xad\xb6\xa3Q\xabf\xbb^\xab6\x84\x7fy\xf7\t\xda\xb4\xc6f\xdd\x89\xef\xd2\xee\xc9\xb4\xdb\xba\xfas!\x93w/\xc9\x1e\x99\xb4K\x96\x85\xe4\xdd\xd65\xeey\xf7\x9el\xe4\xd2\xae\xc1\xe2\xc4\r!kT6\xf2Q\x98\xbf\xdd\xb3\xc7m\x8d\xbf\xdb\xbd!\xbbWd\xf7\xe3\xfa\x9f\xd6\xd3V\r\xf0\xca\xdfz9\xad^\xe6\x1bo\xd6\xa7\xbbN\xb2_e\\\xb9z\x94\x17\xe7v\xbd\xd0\xb2\xf5\xa47\\\xfap\xab[\xde\xe3\xe77\xe2\xfcn\xed\xdb\xc8\xef_\xd9\xc8\x977\xec\xf0\x1e\xad\xdb\xc8\xf3\xc6\xcb*e\xa1\x847\x87\xc3\x1aw\xb4a\x11\x11\xa5c\xc2\xbd\xc9\xe3Yz-3\xeb\xc8\xb3\xe6\x86);NN\x10`Qjh\r/f\xd5\x12\xa8\xe6\x9e\x99f(\x9a\xdb\xac5W\x82\xae\x82\x81\\\xd2\xec\xfd\x9bX\x9c\x01\xdam\xef\xb5\x01\x83\x90\xc3\x0e\xeaoOP\x82{\x18\xb5M\xc8\xae\xdaU\xa4D\xa1\xe4|\xa1\xc9\x19\xf2q\xe9\x02\x9aZ\xd8\xa6\x1e\x84\xa1\x80\x16\x9f\xbd\xeb_\x8d\xfa\xf5x\xffA\x00\xf7?\xd6\xcf\xbf\xfcG\xa8\x1b\x98\xb7\xfd\xdf\xfe\xae\xca"\x1f*\x8baB2\xb7\xe2=\x86\xb2\xb6\x04-\x8c\xc6\x9a#b\xed\xb1\xad!@g\xf9.\x15\x8d/\xc3\xc3\xd8\x8c\x13\x16\x80\xeb,\x1e}\xdf\x0e\xac\x8b\x07h\x19w\xca"\x1f+\x8b\xfc,\xca\xe2\x9du\xe1\x1d\xf5\xe0\xd8cZ\xd0tu\xe7\xb4\x0cP\x85\x19\x1a\x08f7E\x7fm\x10\xe3\xa1>\xbbu\x11\x0f\x01\x83\xd2\xc1*P\x00\x8a\x91rI\x99u\xaa\xa4\xd2XK\xea\xa9\x005\x91\x93KN|\xac\x83\xc7\xa8C@R\x97G\x04O\xbd;X\xc3\xb0C\xf0\x0e0\xf1\x180\xcfj\x8e.\xf95_24\xff\xe4T\xd3\x85_N\xb3\xa6$\x9cZ>\x9f\x13\xd15\xd0\x05z\xe9\xb3\x03&\x1d\x02\x06#\xef=7\x95\x9cc%\x187U1\x80\xc2\xd5\xcf0%F\xcc\xd0iv\x9f\xb4z\xd8\x8f1\xd5\xecR\x1d]\x99Q-\x1f\xda\xa7\x80y\x00\x04\xef\x00\x93\x8e\x01\x93\x9f\x140\xe9\x14h)\xa3\xf9\xf5E|u\x90M\x1c\xaf]]>\xc7>2\xc5R+\x95kx\xfaU\xd2|\xbc~\xa1\xd4\xe6\xcc\r\x14\x00D\\\x9f\xa0G\x81\xd0\xf0{\xb5\xa1A\xeb\xa1\xea\x92NGJa\xe4\x9c\xa7K%@\xd0$\x98\xcc\xb9\xf5\x83eu;\x04\xef\x00\x93\x0f\x01C\xcf*\x92~+\x80\x91C\xc0\xcc0\x96\xe6?\x16\xeb\x13\x94R\xdf#\x0c\x1f(z\xe8t\xaa\x05\xd3(\xf4g\x9d\x98\t\x82#\x08\x107\x9b\xa2\xbd.\xd7\xd6d\xa6p\xa0\xc3\xd8!x\x07\x189\x06L|R\xc0\x9c\x8b\x933\xb1\xf4\x90O\xa7\xe5[r\x97y!\x8a3B\xd1\xf7\xe5\x02j\xf8\xc6\xee\xe9\x01S\x8eg\x989\xe7\x08\xa1\x972aB@h\xb4N\xadB\xf9\x84\xae\x9fz\r\x93\xa9\x0f\x14\x00)\x8c7\xb5\x177a\x02\xf4\x06Sc(\x0c\x8bO\x01\xf3\x00\x08\xde\x01\xa6\x1c\x03\xe6YE\x92`\xdc\x8a\x17\xaf\xc5\xbd\xf8(\xed\xdcjO\xcc\x97\x97+\xe6\x98\xf3j\xb2\x10\x80\x93\x9e\x1c0\xe1\xd8\xd3;\xe3X\x8b\x0f\x89\xca\xa4\x8c\x86\xf5\xea\xa4\xa3\xb5S\x134{\x8c $\xc2Z\xde\xe8\x13\x86\xce\xcc\xb1\xa9\xd3Qk\xf7\xaa\x82;\x0eD\xd2\x03 \xf81`\xc2\xa1\xa7\x97\xe9i\x1cwyiew\x8ao,\xd3\x05j\xa1\xce\x0c\xe3OZ\t\xe7s\xa9I\xdc+\xbf\xf4SP\xf1P\x82\x9d>;h\x8e\xbd\xbd\x82Y\x80\x07\x9a3\x97i\x93\xfa\x18NJ\xf6}\x84\x1a\x849\rjk\xb1%\x07L\xaf0\x85a$\x13l\x9a\xa8\xbe7J\x18\xba\x83Y\xc6\x0e\xc3;\xd0\x1cz{\xf1\x8cg\x01\xcd\xbd\xa5\xf4\x1b\x01\xcc\xb1\xa77\xe4\x0c\x15m\xa4\xe8hd\x073\x981\xdc\xea\'\xd4W\x99!\x8e\xde\n\x95\x10\x1d\x0fR\x0c-\xf49\xdf2*E\xa1S\xd1D\x07\x8e;;\x04\xef\x00s\xec\xe9\rO#\x96|\xe5\xd1>\x06M+\\|AWq\xea\x91\xdc\xb5\xbf@>\x8350\xba\xa3\x0f\xa7\x1e[\xac\x12\xb6\xf7\xfd\xa9As\xec\xed\x05"rL=\xf4\xe5\x89\xcbn\xd6\xa5x@msA\xd9\xade\x94&\xb5Bq\xad1\xc1\xfc\x95\xc25\xf9\xc1\x12\x9a\x13W\x81\x9d\x83\xf5\x18;\x0c\xef@s\xec\xed\rO#\x9a\xeef\x19\xf7B/\x189\xea\x11}{n)\x9cj\xb8\xa2+\xd1z~q\x058*\xcb7{}v\xc0\x1c\xaf\xf8z\x17j\x89>Ch0\xe6\x85\x04Y\xe2&\xde@`\xb8\x06\x08\x15\x98\xca\x1e\xedKn*Cp@C\x99m\xf8\x06\x83\xb8\xb2K\x07\xca\xef\x03 x\x07\x98\xe3\x15_\xf6O\n\x98tr\xd4\xeb\x84\xa5\xe8]&\x89\x0c\xf3)\x90\\\x840\x8b\xe6$\xc5g\xbe\x005\xcf\x0e\x98\xe3\x15_P\xbf\xb8\xea\xa0wj\x95\xa5_D\xc0\x03\xea\x9c\x1b\x8d\xb9\x00/\x19\xf3@qP\xfd5I\x8f\xd9\x0bl\xe3BPO\x94\x14\xaa\xec\x81\xf2k\x87\xe0\x1d`\x8eW|9<)`\xf8\x82\x0e\x8a\xa7Q\xda\x0b,\xc5kX\x9e\xa5rE\x9f\xe4\x1e\xe5\x12j\xf3\xd2\xae\xe9\xf5\xd9\xcd\xebp\xbc\xe2\xdb:\xcf\x99\x9c\'\xe0\xc5\xd7\xd2\x12Z#m\xc5!B\x94P\xcd\xc2c\xf6\xa5\xb5z*1\xc2\xbc\x11\xc0\xa1-?7\xb4\x93vd^?\x00\x82w\x809^\xf1\xe5\xa7Q|\x03Fh~\x0c\x9a\xeb\xf2oc.\xbd\xc4\x17\x91\x13\xb0\x92\xf0:\x87\xcc\xf2Z\\k\xc5\x9f\xbc\x7f\r\xa5=;h\x8eW}\xdb\xf4%+\x0c]\x1e\t\xa6K\xaf\xe83\x9d\x98\x0b\x8a\x06\xe9\xa3E(\xf9#f\xd5\x00zH[\xeb\t\xd3w\xd7\xa4\x17\x92\xd1\x0e\xf4\x98\x07\xc0\xf0\x0e4\xc7\xab\xbe\xfc\xbb_\xe9\x97\x05\xcc\xf1\xaa/\xa5\xe0\x8a0&\xce6k\x86\xd4 \xc7\xb0\x83\'Io\x10\x12]c\xa3<\x80\x16\x11\xf6,0}\x86/K\xff\x80\x19P\xa7;\x02\x8c\x1d\x82w\x809^\xf5\x8dO#\x96\x0e\xc2\x1c\x1f\x08\x9a\xefz\x8d#\xc6&B\xd6\xe7\x811\xc1\xe4\xdc\x0b\xec\x08\xd6\xees\x9b3\x86\xca\xdc\xc0P\x8c\x14A\xc2c\xf6\x1b-\xc7\x1c[\x9d\x03S\xfb\\\xbe\xf2\x1f\x1d\xe6\xf8\x00`\x1c\x849\xee\xb1\xfc=\xcc\xf1(\xcc\x91\x1e\x1f\xe6X\x7f\xfe\xfd\xeb\xe7\xd7\x1f\xe6\xb8\xc3\xa7h\x87\xc6\xdd\x02\xe4h\x87\xd5\xa5\x1d\xf8\x96w\x90T\xd8\xa1~\xb7\x10\xb9\xb2\xc3\xe9\xfc\x0e\x98\xdaar+\xc4o\x87\xfb\xc9\x0eV\xe3\x1d\xc6\xe7V\x99;\x10\xd0\xa5[t!\xefp\xb4\x15\xd6\xb5B\x1d\xfd\x0e\x04\\AY\xf60G\xda\x01\x95i\x05\xab\xed0\xb3\x15\x02\x98v\xa8\\\xdc5\x0e;`m\x85\x1b\xde\x02\xc6V;\xca\n\x94\xdbW\xc7\x1d\x00\xc7o\x81\x8be\x87\x06\xba]\xd6-@Lv\x18\xe5\xae\xff\n\x85[\xa1\x87\xcb\xbd\xb9\xc2\x0c\xd3\xad\xec]\xf2\n\x98[%\xeeP\xb4\xb8[\xfe\x16\xee\xb8\xc3\x0e\xe9\x16B\x1an\xe1\x8bn\x07\xbd\xed\x9e]\x01\x83+\x90\xee\xd6\x8a]\xe7U\x17\xda\xe5\xa7\xb7\xf0D\xde\xbd\x98n\xc1x\xeb]X!\x94\xbc\xbf\x95\xb7~\xbd\x8d\x9a\xdf\xe1\x88\xb4[\x99w;V\x00fz\x0b\xa9\xf3\xbb\x07V\xd0\xe3-TQ\xc2\x87 \xc9\xb2\xc3:Wp\xa0\xec\xf0\xc7\x15\x9a\x17w\xcbn\xbf\xb7\xd0O~k\xcf\nr\r\xbb\xf5\x14\xc8<~\xbf\x87\xa9\xfe]\xc3T\xdf}\x08s\xf4\tJ,W\x14\x19Y0\xb7\xc3\xa6\x19\x9e\xdb\x8a\xfd\x0fI`/7\x18\x00\xaet\xbc\x05\x9cp\x01\x94\x80\t\r&\xcf\xd6g\xd8)"\xb70\xc70"\x8d\x99\x96\x93\xa9\xc3\x90bR\x9e\xa1\xf5\xd8G]\x05\xa7F\xa8\xcc\xc8Y2\xfb\x15x\x05I\x88B\xab#\xd7\x1b\x86m{R\x9f?\xcc1\x1c{|\xb3\xa3\xd00\xaf\xc7\xee#,\xef*\t\xfa[\xa2RC\xeaiy\xb2\xd9\x8b\x0c\t=\x08\xf9\x9c3\xfad%\xb0,\x87J\x84$?\xc8\xbc{\x80\x96q\xa7,\x1e{|\xe3\xb3\xc6\x94\xc4\xe8\xc1\xef\xad&:w\xc1\xff0\xb0\xda9_}\x17>\xd5\xd8\xf4\xec\xd1\x95\xfc\xec\xd6\x05\x1f{|\x05z}\x8f\x03@\x01%JJ\xae\x0f\xb7r)\xf3\xf0u\xc6\xd1z\xaf2+\xb1\xcc\xb54\xc1i\xa6Q}\xa5\xde\'\xec\xcdR\xd3\xa7\x80y\x00\x04?\x06\x0c\x1f{|\xe3\xb3\x9a\xa3\xbf\x91\xdc^>\xf6\xf6j#\xc2\\I\x98J3\xb4k\xd6\\\xdd\x04z\x94s\x9d\xe2\x94z\xa3*\xb1H\xee\xa5`N/IF!O\rV\xa7\xabZ\x0e\x9cwv\x08\xde\x01\xe6\xd8\xdb\x9b\x9e\xc6\x1c\xfd\x8d\x02\xe6\xd8\xdb\xeb\xa8\xc6\x9a\xaa\xd7\nU\x05\xd2\xba\xb5\x019\xab\xa1\xadx3\x97\x1b\x85\xce\xbe\x8e\xa4\x1d\x06q\x0b*\xb3\xe7\x08k+c\xf6\xc8e\x1e\x84\x07<\x00\x82w\x809\xf6\xf6\xa6g\x15I\'9\x9f\xce\x0e\xfa\x9d\xb2\xf3\xe8\xc1\xd7\xc4+oeB\xb9z\xc9\x1e\xb6\xf8)\x87\xab\xdb+{O\r\x98cO/l\x87\xde\xa6\xf3>\x92j,P1\x82\x962)\xa6\x91|.\xb09\xeb\xf4\x156G\x969\xa1EB\x90T\x1f\xbb\xc4\x04\x9b\x0e\n\xc8\xa7\x80y\x00\x04\xef\x00s\xec\xe9MO#\x92\x98K\xbc\x03M\x893\x00\xe9\xd1\xa3\xd3\xca\na\xe3\x96\xe5\xcc\xa7\xe6ctY.\xab]\xa9\x96gw\xde\xf1\xf7x{y4\xd70n\x90\ra\x9bF\xd0\xef\x05\xff\xf0~{\r89\xcc\x14\xda\x1aT\x90\xd4\xf2(0\x04::\t\x9a\x1d\x8c\x85\x83U\xd2\x07\xc0\xf0\x0e4\xc7\xde\xde\xf4\xac{\x94\xfcFv\x0f\xe0coo\t\xb0\x0b\xfb$\x07\xb1[\xbc\x0b\\{\x8b\x98RG\xcdU\xa7B\xac\x94\n\x8b\'\xc5^3\xacP\xdf\xcb\xd0\x1c\xc7,\xc3ii)\x1ex{\xed\x10\xbc\x03\xcc\xb1\xb77\xbb\'\x05\xcco$6\x96\x8f\xbd\xbd:\xbd\xf4\xe24\xf4\x91\xfb\xec\x92i\x80\xf53v\x0e\xe2\xfc\xda\\d\xad5\x04/\xc3\xc5\xc6\x1e\xaa\x9bV)J-3gv\x07\xb1\xb1\x0f\x80\xe0\x1d`\x8e\xbd\xbd\xf9i\xe2I\x0e\xfc0NN\xa1]r\x9a\xe2Oa\x14\x19z\x8e\xd7\x93\xf7/!\xe2\xf2J\xbe4\'\xd9\xff\x98,\xc2\xefn7\x11\x98\x08\xe6\x82$\x17\xa7\x8c^)\xe8t\xa1y\x9f\xa3J\x87\xc8\x1f$\xb9R\xf6\xa3t\x015\xfbL\xd0\xbc\x87_\x89\x0eu\xafZ\xfch?\xcc\x03\x80q\xe0\x87\xd9c\xf9\xbb\x1f\xe6\xc8\x0f\x13\x1e\xef\x87i?\xff\xfe\xf5\xf3\xeb\xf7\xc3\xec$\xfe\xf8\xcd*\xfe\x07\x8fF\\k\xfb{\x1dx\xa5\xf1\xfb\xb7\x14\xff\xe5\x85\xe0\xb7uf\xb7\xd7\x94\xe9m\xb5\xdc\xefd\xff\xf4\xb6\xee\xcb\xdb3P\xf6\x06\x14y{-\xcaN\xdc\xe7\xbd\x12\xbe\xbc\x07\xb7\x04xZ\x08\xb2\xfc\xecu\xfc\xb5Z\xce7_\xc3^#\xa7\xbd\x02\x8d\xb2\xdf\xb6\x0f\xf0{\xe3\x81\xb5$\x9e\xb6\x7fh\'\xff\xef\xabW;\xd6\'e\xaf\xb3\xcb.+\xbdmi\xe0\xb7\x87\xc0/_\xd1\xf6S\xc5\xede\xd9k\xe3\xdbs\xb1\xd7\xf3\xd7\xfa\xfb\xf2S\xbcmC@{\xab\x81\x0f+\xf8\xe9\xe6wIkM>\xec\r9V-\xdd^\x87\xa7\xed-\xda\xbe\x81\xed3(\xbb7n\xf5\x8b\xdbw\x95\xb7\xb7\x83\xf7/\xbd\xf9\x1f\xe2\xf6\x11\xdcV\xfc\xcb\xaeu\xda\xeb\xfcq\xf7h\xda[~\x84\xbd\xa1\xc2\xcd\xfbp{\xeb\xf6\x16\x05y\xfbS\xde\xbaa\xf7\xc2\xad\x867\x0fL\xd8\xeb\xf4\xe5\xe6\x83\n\xb76\xde\xfe\xf2\x1e\xd3\xbc=\x00o\xde\xa9t\xeb\xcd\xe5\x19`\xf3\xf8\xfd\xeeG\xfb\xbb\xfa\xd1\xbe\xf1\xc3\x10%Y\x8bp\x83|j\x10\x9eN!]\xda\xda\xef\xc3\x03\xa4\xbe\xf6\\g\nc\x14J\x0e\xd2\x9aK\x8ea(z\xcc\xc3\x04\xbae\xfb\xdf\xfc0YZ\xf0\x93\x1bt\x16\xe90\xa3\xb9\xc1>Z\x9b\xd7\xb5\x91\xa0$\x97\x02\x11\xbc\xac\xa6\xd0\xa7@\xd7\x91\xae~\xe8\xe4\xd2\xd0g\xeaR\xfe\xd6\x0f#O\xec\x87\xe1\xe3(/\xe6\xd9`d\x87\x1aG\xa9\x03\xafym\xdb!hj\x86\xb8\x84\xb2\xe1\xb4\x0ffG\xb3\x06\xd7:\xd49\x88\x80\x94\xb4\xc1\x1aXI.\x07\x8b^v-\xe3NY<\x8e\xf2\xca\xcf\xbaJ\xca,\xd7\xfe\x9a\x84\xd2\x85\xaf\xee%\xceX^\xf3\xf5\\\xb3.)\x84\xc9\xf4t\x02\x8b\xf3\xd3\xfba\xbe\'\xb7w2\xe5\xe54[\x89\xa7u6\x85.<2gt\x1a\xa5V\xa2\xc4\x1ce\xedH\x17;5p&\xf9\x11h\xac\xcdPG%\xe2\x03\xc0<\x00\x82w\x809\x8e\xf2\xca\xcf\x9a\xde\x90\xc79\x9e\n\xda#A\x93H\xf4\xa9d\x82\xcd\xf5\xda\xd1\xcd\xe4\x06\xe8\x92\xfc\xcb\xf3\xaf_|\x8f\xa7\xb7\xb9$uE\x93\xa9\x00\x19T\x1cE\xae\xb1\xc2\xf0\xc9\xc5\xf5\x00\x82w\x809\xf6\xf6\x96g\x15K\xbf\x15\xc0\x1c{{\xd3^\xf5\xa8\xe4\xa3\x96\x18\xd9c$Kw\xd4\xa1\xaaB\xcf\x88.z\xa8\x1f\xb3\xeb\x8c\x11\xe6a\xce\xd5;\xf1~\xf4\x90\xfaPr\x07\x1b\xf3=\x00\x82w\x809\xf6\xf6\x96\xa7\x11K\xf7\x96R\x0c\x1a\xaaBy\x93x\x12zq\xd7\xf3\xd9c\x8a%\x98\xe3+h\x02\xbdJq\x9c\xce\xcf\x0e\x98cO\xefpDE|\xcc\xcd\x8b\x0f\xb9\r\\\xad\xa5\x8bw\xcbY\xb9R\n+e\x0c$W\x80\t\xaa,\x04N\x87q\x1c{\nk\xe9\xe1`\xdb\xef\x07@\xf0\x0e0\xc7\x9e\xde\xe79\\\xe0\xb7\x99\xa6\x19\xbf\xc7\xd3+\xa9\xb5RIk\xc5eCF\n\x8eG$u\xbe\x15\x07\xfchS\x85\x12"\x1aK\x0f\xb5\x04?<\xb4\x90\x96\xeb\x8aP\x9e\xc4\x18\x90\xf1\xeb\x8f\'\xd9\xd1\x16i\x1f\xc4p\xcb\xbb\xf5;\xf6!\xbc\xc5\\\xd0>h!\xedx\x91[4\xc2\xce\xed\xddQ\x14\xb7\xdc\xc1\xedkO\xf4\x8d\xc7:oO}\xda\x19\x87\xb7\xa8\x8e\xdb1\x0cn\x97\xb4\xa3\xf5\xdf\x0e\x96X^\xf2G\x1c\x7f\xc1;> \xa5\x9b\xbf~\x1f\xb7\xb2s-\xd3\xdb\x01\x10;\x06c\xbf\x8ao1\x00y\xc7\x06\xf0\xce\xd9t\xfb`\x87\xdd\x82\x9d\x03\xca\xb7\xa8\x8a[\x9e\xf3\x8a\\\xd8\x99\x9b\xb7L\xcax\xcbj\xdeY\xca~\xdf\xb9=\xfe\xbb\xddn=g\x97@;~c\x7f\xb6{\xedvHEN\xb78\x89[d\x89\xdf\xbd\xb7\xa3\x13\xc2-\x1b\xd5\xed\xa7\xa5\xb7\xc8\x9bU\xfa\xb7\x07q\xf8}\xdd-\x92\xa2\xec\xa8\x86\x1du\xb0\x9f\x16o1<;\x0eb\x1d\x80q\x8b\x17\x88\x1fb)\xc2\xed\xd0\x8a\xd5\xe6\xbcG\xf2vP\x08\xbf\x8d\xd1-B!|\x88\xf6y\x8b\xb5\xe1\x1d]\x91v\xbf\xf8\x1d\'\xc3{\xbcn\xd1A\xe1\x96\xd3\xbd3N\x1f\x91\xd7\xfb{<\xd0\xdf1\x1e\xe8\xdd7\xc7\x97\xf0\xdaf<\xb8\xdcZ\xa6ud\xc6\xf2\x1c\xca\x9c.s]\x11 kU\x8eP\rn\x90\xd0sh\x19K\xc5H=B\x99\xd1\xc2K\x9a\xdd\xe2I\\\x85\xe6R\x13\xf0\xab\xec\xb2\xf3\xb5R\x9fN0\x83g\'\xb5s\xa69\'\x14\xc6\xd4\xa0HH\x1bRC\xf4B\r\xcf\xe2\xd2w<\xfe\xaf \x9e$\x1e{{\xc3\xcesv0\x08\xc4uhx\x95J\xca\xbe\x02\xa2=OYk\xda\xc1)\xb4\r C\'\xf4\x13\nM\xd9\xf3l-\x8eX\x8f\xb6\xfe~\x80\x96q\xa7,\x1ez{\xa3\x7fZ\x93\xf4q\x8a\xe2\xdf\x130\xe9\xd8\xdb\x0bU)Q\x17n%\xcc\x91\xa0\xbb\xe1\xda\x8e\x1f\xe5\xb5lI\xcb\x8c\xea\xebh\xd4Rs\x89\xb1\x05\x85|\xd0^X\xab\xf7+\xc3\xf9S\xc0<\x00\x82\x1f\x03&\x1dz{\xa3\x7f\x1a\x93\xf4\xb7\x99\xde\x90\xbe\xe7\xcc^\xc1\xb4\xcc.\x8a\x9f\\\x19\xbav\xaa\xac2\x01\x8d\xbd\xe9\xa5N\xe7sq\x03B\xac\x16\xcdy\xc1\x8aW\n\xf7Z\xd4Z[\x18\x1e\x84\x07\xd8!x\x07\x98COo\xfc\xfd\xbc\x9b_\x180\xc7\x9e^B[0\x84\\ZJM\xc8\x8dR\x0b\xc42D|WJe\xf6\x99\xb2+~\xf4\x9a\xa20\xec\xe5\xb0\xeca\xd1\x0c\x8ep\xf5G\xe7\xf5\xda!x\x07\x98CO\xef>\xc9\xf39\x00\x93\xab\xa7_\x0e4\x1fm\xb9\x1d\x08h(\x1e\xa6\xb3\'\xc9\\j\x84m^\xc8\x85\x1aE \xf5!\xfa\x07\xcf4}_\xb1\x16D\x05w\xb8\xd4C\xab\xaez\xf9)\xeb\x17\x0f\x00\xc6\xc1\xfa\xc5\x1e\xcb\xffv\xfd\xa2\x05i\x1c0\x11\x97\x93\xa43f\x10\x88\xa6S\x89a\\\xa1\x00\xbe\xb6\xec%\xc5\x93\xcc\xf3/\xbd~1\x99\xf8\xbb\xeb\x17\xd1kPth\x8a\xe7F\x973\xa1\xf3\xa7\x94t\x16B\xab\xc6\xb9\xbed\xf1\xd4\xf6"\xffO^\xbf\x80\x86+\xdfY\xbd\xe8\x1c\x86R7.\x1f\x18\x8d\xaf\xff\xe6\xfeY\x03\xb0\x05\xeb\xac4L\xf9\xa0w\xa3\x99G\x1f~\x8c\xd08\x81\xdd\x05\xcaX\xfa\xd6z\xc8\xb9\xc3`\x10\x8aa\x9fp\x0e#\x82]\x9d<\xfc\xca\xe7\xae\xebh\xbde\'\xa4\x98\xf3\x8c\xd9\xad\x8dS\xa9L\xc8f\x86\x80\x16\xc8\x95o\xac\x07R\x1f\xaa\x1fubL\x89\x92\xce\xb1\xb6\xe2\xc3\x1cS\x98\x9d@4A\xf3\r\xa2\x04\x83Fr\xd3V(\xc2\x04\x89\xab\x1c\x12\xd1\xf9\xeeW\xb1+P:\x8e\xb1h\xbd\xc1\xa8\xd7\x08U\xbfS^\xce\xba\xb0\xcf\x88L\x8dy\x1d#\xb06\xd8\xd0\x19\xfa\xc0\xec\x95G-c\xf8\xb9\\\xdf\xdd\xadc\x80\xe4@\x19|\x00\xc7\xef\xa6\xea\xc3\x18\x8b\xf8<\x07\x07\xe5\x1aK\xfdM\xee\xdb\x91\x8e\xe3,\xd2\x14\xc207\x87\x81\x9d\xb3\x89\xf7\xb0\xcf\xbd\x14\xd8\x01\xa3\x8c\x99\xdb\x94u\xd8F\rc\x9d\xdc2j\x96\x918@Z,\xff%\xd4\xbe\x03\x93\xd3\x0e\xc3;\xd0\x1c\xc6Y\xecc\xee\x9e\x034\x1e\x8d\x84$\xfd\xc5\xf2\xea?\xdaf\xa5\x8c\x90gJ\x1e*v\x93\x81~\x0573\xcc0Z\xdbw\xc5\x11 \xe2\xfdR\x94J(.\x94\xa9\xa3\x90\xc22s\xa3\xc6\xe8\xfbO\x91\xf0\x0f\x80\xc6\x91\x84\xe7\x1f\x92\xf0i\xa9\x86/\x99\xce\xde]\xd3\xa9^%r\x91L\xc0\xc6)\xbc\xf6\xf6R\xe4|\xbe\xc0\xae\xff\t\x12\xfe#"\xfd\x0c \xbc\xc9\xf8\xf2\xda\xfd%\xbc\xf4+\xba|B\xa9\x8a\xd73\x0cC\x1ag\xa6\x93\x8f)M\x8d\xact\xce?G\xc6\xfb%r\xbe\x15\xf1\x14\xc5\xc1\x14\xb5\x89\xe8d\xba{\xeb\x92\x96\xfb\x83QC\xf1\xc6\xe3\xe1W\xa8\x8e\xed~\x9b\x8e\xd4\xc5\xb6\xf1(8d\xbc\xdf\x96p\xdb\xc5\x06\xa0.\xd9x\xbf\r@]l\x00\xeab\xdb\xb8\xb7\x8b-\xe1\xbc\x8b\xcdC\x07\xd5\xcdx\xbf\xcd\xc3\xd8e\xda\xee\xb7\xf2\xbf\x18\xf9_\x8c\xfc/F\xfe\x17#\xff\x8b\x91\xff\xc5\xc8\xffb\xe4\x7f1\xf2\xbf\x18\xf9o\xdc\xb8\xbb\x1b7\x9c\xe8\xc6\x00\x83^\x8c\xfc/F\xfe\x17#\xff\x8d\xfb\xa6\xf7j\xe4\x7f5\xf2\xbf\x1a\xf9_\x8d\xfc\xafF\xfeW#\xff\xab\x91\xff\xd5\xc8\xffj\xe4\x7f5\xf2\xbf\x1a\xf9_\x8d\xfc\xafF\xfeW#\xff\xab\x91\xff\xc6\xfdzz3\xf2\xbf\x19\xf9\xdf\x8c\xfcoF\xfe7#\xff\x9b\x91\xff\xcd\xc8\xfff\xe4\x7f3\xf2\xbf\x19\xf9\xdf\x8c\xfcoF\xfe7#\xff\x9b\x91\xff\xcd\xc8\x7fc\x80b\xefF\xfew#\xff\xbb\x91\xff\xdd\xc8\xffn\xe4\x7f7\xf2\xbf\x1b\xf9otqu\xa3\x01\xd9\x8d\nh7\n\xb0n$@\xefF\xfew#\xff\xbb\x91\xff\xb6\xea\xe3~#\xff\xd5\xc8\x7f5\xf2_\x8d\xfcW#\xff\xd5\xc8\x7f5\xf2_\x8d\xfcW#\xff\xd5\xc8\x7f5\xf2_\x8d\xfcW#\xff\xd5\xc8\x7f5\xf2\xdf\x98\xe0\xd0\x87\x91\xff\xc3\xc8\xffa\xe4\xff0\xf2\x7f\x18\xf9?\x8c\xfc\x1fF\xfe\x0f#\xff\x87\x91\xff\xc3\xc8\xffa\xe4\xff0\xf2\x7f\x18\xf9?\x8c\xfc\x1fF\xfe\xdbn\xc7\xfdF\xfeO#\xff\xa7\x91\xff\xd3\xc8\xffi\xe4\xff4\xf2\x7f\x1a\xf9?\x8d\xfc\x9fF\xfeO#\xff\xa7\x91\xff\xd3\xc8\xffi\xe4\xff4\xf2\x7f\xda\x08lT\xffq\xbf\x8d\xffj\x8c\x91Tg\xe3\xbf\x1a\xe3?\xd4\xd9\xf8\xaf\xc6\x00\x12u6\xfe\xab1CV\x9d\x8d\xff\xeal\xfcWg\xe3\xbf:\x1b\xff\xd5\xc8 5f\x18\xab3\xf2\xdf\x98\xdf\xab\xde\xc8\x7fo\xe4\xbf7\xf2\xdf\x1b\xf9\xef\x8d\xfc\xf7F\xfe{#\xff\xbd\x91\xff\xc6\x002\xf5F\xfe{#\xff\x8d\x19\xfa\xea\x8d\xfc\xf7F\xfe{#\xff\x8d)\nJF\xfe\x93\x91\xffd\xe4?\x19\xf9OF\xfe\x93\x91\xffd\xe4\xbf1\x00U\xc9\xc8\x7f2\xf2\x9f\x8c\xfc\'#\xff\xc9\xc8\x7f2\xf2\x9f\x8c\xfc\xb7\xd1\x07\xf7\x1b\xf9o\xdc\xa0B\x83\x91\xff\xc6\x03W\xd4x\xe0\x8e\x06#\xff\x83\x91\xff\xc6\x00r\rF\xfe\x07#\xff\x83\x91\xff\xc1\xc8\xff`\xe4\x7f0\xf2?\x18\xf9oL\x7fP6\xf2\x9f\x8d\xfcg#\xff\xd9\xc8\x7f6\xf2\x9f\x8d\xfcg#\xff\xd9\xc8\x7f6\xf2\x9f\x8d\xfcg#\xff\xd9\xc8\x7f6\xf2\x9f\x8d\xfcg#\xff\x8d\xe7\xc5i4\xf2?\x1a\xf9\x1f\x8d\xfc\x8fF\xfeG#\xff\xa3\x91\xff\xd1\xc8\xffh\xe4\x7f4\xf2?\x1a\xf9\x1f\x8d\xfc\x8fF\xfeG#\xff\xa3\x91\xff\xd1\xc8\x7f\x1b\xfcp\xbf\x91\xff\xc9\xc8\xffd\xe4\x7f2\xf2?\x19\xf9\x9f\x8c\xfcOF\xfe\'#\xff\x93\x91\xff\xc9\xc8\xffd\xe4\x7f2\xf2?\x19\xf9\x9f\x8c\xfcOF\xfe\xdb\xe0\x83\xfb\x8d\xfc\xcfF\xfeg#\xff\xb3\x91\xff\xd9\xc8\xffl\xe4\x7f6\xf2?\x1b\xf9\x9f\x8d\xfc\xcfF\xfeg#\xff\xb3\x91\xff\xd9\xc8\xffl\xe4\x7f6\xf2\xdf\xb8\x7f\x80\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xff7\x86o\xe3~#\xff\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7fc\xfa\x05\xee7\xf2\xdf\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xff7\xa6O\xe1~#\xff\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7fc\xfa#\xee7\xf2\xdf\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\x81^\x8d\t\xb8jL\xe03\xa6/\xe3~#\xff\x8d\x01\xc8j\x0c`Tc\x00\x94\x1a\x03(\xd4\xe8\x80U\xa3\x03G\x8d\x0b\xc0j\\@R\xa3\x01\xaaF\x05V\x8d\x02P\x8d\x04Rc\xfe\xbf\x1a\xf3\xff\x8d\x8f\xc7\xfdF\xfe\x1b\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xff\xc6\xe9\x03\xf7\x1b\xf9o\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\x1b\xc5?\xee\xb7\xf1\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\x198\x8c\xf9\xffF\xf5\xdd\xb9\xce\x9d\x1d\x97\xf0\xfdv\xd4\xdab\xaf\xf3L>\x97XB\x90\xc1yr\x0cu\x1d\x93\\\xa8A\x81\xa83\x1a\x97a\x9d\xf5\x18\x17\xc7\x9c\xc4\xc7u\xe6\xf7\xf7]\xb1\xcf\xcc\xd1L\xcd\xbb\xb9\x8eL\xfb\xf6\xcc\x9c\x1aFo\xeb\x0c\xab\xd6\xa3q=\xdf\x19\xf3\xd9\xff\xe7\x7f\xbcQ\x02\xb8o\x8e\xa8[\xc7\xa9\xfbB^\x96W2\x0e\xf6\xd4\\\xe6\xee\x8aw\xa5\xfbH\xbc\xce\xe1\xc6\xdf2K\xd3\\I\xf3\xa8\xde\x87(\xb5T\xc75~sD]jn\xf6\xe9*y?;\xe5\x91\x99\x12\xe5()\x10\x8d\xc8L\x12\xa5\xf7\x02\x0c\xb2\xafcm\xa1\xd98\xd4\x1a\x94{\xed\xad\x95_\xc7\x01\xd7)\x1e\x1fp\x9d\x1d\x07\t\xda:\x9eU\xbb\x97\x8a92\xe5\x01\x826U\x91\x92\x04:\x0fP\x16\xf7\xe1A3d\xef"\xfar\xce\x9e{\x8b\xf3\xd3\xd3\xc6\x1epH\xd5\xddic\xf1\xf8\xb41y\x96\xd3\xc6\xeeN\x12M\xe1\xac\xa7\x92IB\xa6K\xeb\x92\xeb\x85\xc2I}\x8cte\xe6\xd7\x8b\xab\x85n\xa7\xbf=\xfax\xba\xa8 \x04\x86/=\x020\xe9\x100}5l\xc2\x0e\nUjL.\xb6X\xa97J\xd5\xe7\xc0^\xb9d\xed\xbc\xce\xa6\x9eIi\x0c\xe1\xb1v\xfd\xed\x14f\xd4\xb4\x8f\x83\xbf?\xe0\xda\x0e\xc1;\xc0\xa4C\xc0\x04z\x16\xc0\xa48$\xde\x9d\x8a\xfe\xea\x13\xc7(\xe7q\t\xb10&\x1fN=D\xaa\xe3\xd4S\x08\xce\x0f\xe2\\\xe4\xf2\xd3\x0e\xa7\xa3\xee\x1d\t\xe62\x1f\x05\xe2\xb4*aB\xa4\xc2 d\x86\x80\r\x1a\xf6,\xa9\xd2+\xa6\xbe\x15\xdcF\x81\x92\xe4\xd2\xc0\xd3\\\xdcO9\x9c\xee\x01\xc08:\x9c.\xfd\xd0\xe1t\x9a\xc8\x8f\xe2\xfc5\xe5\xbe\x8e\x8d--\x9e5\xc6k\xc8W\x1e\xa9t\xf2\xe1:\xf8\xf4\xf2K\x1f?\xdbWn\xffw\x8e\xa6[\xa9\n!6\xcf>O\xc8 }=1\xa6\xc4\x16\xb3hp\x97\x90\xc3\xe5%\xa8:\xfe9G\xd3\xb1\x07e\xe2wN\xa7k\xb3k\xcc\x96\xec\n\x92\x1e\x8c\xd1iF\xe3\xc0l^\x1a7\xe7YZap\xa1$N>x\xbc\xe2P0/a\xa6\xc6\xdc\x91\xd2\xfa$\xe2\x93E\x98\xbc(\xb9\x94?|SRH\xb4\xbe\x83\x86\n\x0e\xe3\x15\x855\x9f\x11\xee\xd9W@O\xc9\xb8\x9b1\xe4\x1cp#\xcap\xb8r]\x8d\xfb\xf0O\xf0\x1aO\x00\xe3\x8dji\x87\n\x92\xf6\xd3Q\xe7\x84\x1aA\xdd\xd95G\x8dw\x1d K\xf1\x1aj\xeb\xae\xf3j\x17\x87\xf5*\x07\xd4\xe7\x9b+W\x8d\xf65\x98\x0b\x96\xdaMx\xb5\xda\x00u*\xacr\x05W\xa3\x19\xbbE\x1e\xf7\xa0=\xb8c\xb5p\x95\x82\xb6\xadV\xee\x1a\xac\xcb\x1c\xca\r\xb8\x12\xea;\xbe\xa1]\x8b\xdd\xb3(\x9aV\xb9\xf8\xb7\xee[\xfd\x92q\xf5\xea\xa5\xfd\xbc\xb0\xff\xa0\xff\xd6\x9c\xb4JD\xdd\xf1\'\xee\x1e\xdd\xf7\xecrP?\xd4\x1a\xcf\xdb\xa5\xf8u\xdd.MV\xc9\xfb\xeeu\xc5\xea\xe7\xd5\xc7n\xf5\xfb\xae\xf9\x1aM\xbf\xeb\x8f\xf6\xeek\xca\xad\\\x94\xb9\xeeY\xfdA{\xc4\xdd\x1e\xf3\xf8\xd6jA\x1fPZ}LK\xa6\xe0\xdb5\xea\x19\xdf\x8bqw\x8e=~\x1fp\xb2\xfb\x19m\xc2\xff\xab\x06y\xf7\xc0m|\xc2\xdb\xabU\xff\x80\x7fy\xd7\x7f\xd5j\xa3\x12\xb5\xa1\xddg\xbc\x91X6\x06\xc3n\xf7j\xd9\xc2\xf1\xfan\x8dg\xde#\xc8{d\xdc\xc6\xc3\x1a\r\xde\xa3\x96w\x9f\xd2\xbe\x83o\x9f\xa5\x1b\xf6\xd3\xae\xcf\xc2\xb2l\xf4\xb8\x8d{\xde\xbd\xe66Z\xe2FA\xd9c\x1ev\x1d7?\xc2\xfaf\xbd[\xd7\x95\xdd\xb2\xdb\xf8\x01i\xbbD\xda,\xa2=v\xab?\x17Fp\xfdF\x9e,\xce,\xcem\xac-<\xaf\xe7\x95\xdd\x0eN\x9bm\xbb\xcf\\\x08o\xe8\xdc}\xb1F{\xb3-\xed~A\xad\xf6\xe8\xa5\xcd\x93\xbc{Sv\xbb\xd6H\xd27\xd6\x85\x0fK\x11\x99\xd5W\x8a(\x85\xeb\xb2\x08\xa0\xc4j\xad\x92b\x0b\x91%+\xb4\x10_2D\x19\xfa\xb7\xe1Q\x95{\x9c\xe4D\xf6y\xd07\xeb\x02B{F\x82\x81\xe1\xd3\x0c\xcb*i\xebX\xdd\x01!\xcbnLMu\xcc\x1a\xf0\xae7?[\xa3A\xad\xe1\xfe\xa9\x03\x9aO\xf2\xcc\xbf\xdc\x01\xd8\xd1+\x0c\xd4Y\xe5\x11\xcab>T\x16gV\xef\x84Xg\x97\xa8\n\xeb))S-\xe8\xa7\x1dJ\x92\xa2\x1b9\xf6u\xaeC\x1a\xcb\xa6JM`#\x14\x1d>\'\xa1\x03\xeb\xe2\x01Z\xc6\x9d\xb2\x98\x8f\x95E~\x16e\xf1\xce\xba\x18i\xbc\xc4<\xcf\xc0\xf1\xb5\x9d\x8a\x82\x8e\x97A3^\xaf\'\xcf\xd4\xcf\x94\xca:\x80x>\xfb\xe1\xd7r\x08\x18\xf5I\x8a\x8f^9h\xb2;^\xc3\x98\x05\x8f\xa80\xbfB\x19\x80\x08d\xae\xce\xc1\xbe\xc3\xe6U\x10\xa4+\x8c\xe4\x88Y`\x84\xccY!oj\xd2\xbcVm\xd5\xcb(\xfdS\xd0<\x00\x86\x1f\x83&\xbbc\xd0\xc4g\x01\xcd\xdd,C5\xca\x89\xc3\xe9\x02e\x8a\x95e\xce\xd6e\x86\x8c\x8f\x8b\x1f\xce\x17q\x04+=<;`\xfc\xf1*)\xb3#\xe7}\xed\xd2d\xba\xbc\xc6\x8d\xc4\x95\xe20A\x90\x8fMZ\x0f\x98h\xa1S\xc5\x8cf\x0e.\xb5\xc2$\xa7\xd0\x82\x9b\x85\x0e\xf4\x18;\x04\xef\x00\xe3\x8f\x01\x93\x9f\x140\xeca\xee\xcck\x1a\x98V]}\x15\xa8\xf8\x97\\__<\xecN\'\xbe\xb8H\xe3\n1\xfd\xec\x80\xa1C\xc0$u\xdb\xf4\xa8\xbdaz\x05\x080\x16\x15\xda\xc5(\xb16h#-\xf92\x0b\xda\xd9G\x1f\xa9\xb5\x16 n\x1cc~\xe0QZ>\x10K\x0f\x80\xe0\x1d`\xe8\x180\xe5Y\x00\x13\x18\x16\xca\xc7\xa0q\x97\x1a\xe4\xea\xc2\xa5\x15\x7fy\xcd\xe7\x97\x9c\xdd\xda\xd2\xb4.\xc9t:\x9f\xf8\x1a]\x7f\xa9\xe9\xd9A\x13\x0eA\xe3\xa2\x0e\xf2\xa3\xb5Z{\x84~0\\\xca5e\xc2\x10:/\x14\x01"I\x98G!P\x12A7\x01G`<\x83@\\0=lG\xd4\xbd/\xc6\x0e\xc3;\xd0\x84C\xd0\xc4g\x15K\xbfV\xc0\xfc\xeb\xff\x07`L\xf1{"\x0c\x01\x00' diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..ea824656df58 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_incremental_streams.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_kyve.source import KYVEStream as IncrementalKyveStream + +from . import config, pool_data + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalKyveStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalKyveStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalKyveStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalKyveStream(config, pool_data) + # TODO: replace this with your expected cursor field + expected_cursor_field = "offset" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalKyveStream(config, pool_data) + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalKyveStream(config, pool_data) + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalKyveStream, "cursor_field", "dummy_field") + stream = IncrementalKyveStream(config, pool_data) + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalKyveStream(config, pool_data) + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalKyveStream(config, pool_data) + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_source.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_source.py new file mode 100644 index 000000000000..0a1cab4847b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_kyve.source import SourceKyve + +from . import config + + +def test_check_connection(mocker): + source = SourceKyve() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceKyve() + streams = source.streams(config) + expected_streams_number = 1 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py new file mode 100644 index 000000000000..b294187d9112 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py @@ -0,0 +1,172 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +import requests +from source_kyve.source import KYVEStream as KyveStream + +from . import config, pool_data +from .test_data import MOCK_RESPONSE_BINARY + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(KyveStream, "path", "v0/example_endpoint") + mocker.patch.object(KyveStream, "primary_key", "test_primary_key") + mocker.patch.object(KyveStream, "__abstractmethods__", set()) + + +@pytest.mark.parametrize( + 'stream_offset,stream_offset_context,next_page_token_value', + [ + (None, None, 'next_page_token'), + (None, 200, 'next_page_token'), + (None, None, None), + (None, 200, None), + (100, None, None), + (100, 200, None), + (100, None, 'next_page_token'), + (100, 200, 'next_page_token'), + ] +) +def test_request_params(patch_base_class, stream_offset, stream_offset_context,next_page_token_value): + stream = KyveStream(config, pool_data) + if stream_offset: + stream._offset = 100 + + expected_params = { + 'pagination.limit': 100, + 'pagination.offset': stream_offset_context or stream_offset or 0 + } + + inputs = { + "stream_slice": None, + "stream_state": {'offset': stream_offset_context} if stream_offset_context else {}, + "next_page_token": next_page_token_value + } + + if next_page_token_value: + expected_params["next_page_token"] = next_page_token_value + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token_max_pages_set(patch_base_class): + stream = KyveStream(config, pool_data) + stream.max_pages = 20 + stream._offset = 2100 + inputs = {"response": MagicMock()} + + assert stream.next_page_token(**inputs) is None + + +def test_next_page_token(patch_base_class): + stream = KyveStream(config, pool_data) + inputs = {"response": MagicMock()} + expected_token = {'pagination.offset': 100} + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class, monkeypatch): + stream = KyveStream(config, pool_data) + + input_request = requests.Response() + inputs = {"response": input_request, "stream_state": {}} + + mock_input_request_response_json = {'finalized_bundles': [{"storage_id": 10}], 'pagination': {'next_key': None, 'total': '0'}} + + mock_finalized_bundles_request = MagicMock(return_value=mock_input_request_response_json) + monkeypatch.setattr('requests.Response.json', mock_finalized_bundles_request) + + class _MockContentResponse: + def __init__(self): + self.content = MOCK_RESPONSE_BINARY + mock_get_content = MagicMock(return_value=_MockContentResponse()) + monkeypatch.setattr('requests.get', mock_get_content) + mock_response_ok = MagicMock(return_value=True) + monkeypatch.setattr('requests.Response.ok', mock_response_ok) + + expected_parsed_object = { + 'key': '10647520', + 'value': { + 'hash': '0x3a0e7319f2b5238f3ebc0a5cd419c92bf2c10045e1a8feae4222fce89b8b6287', + 'parentHash': '0x0eb7f809b8cbbd7cf73a56654f2614e3136a0d3496cacaf709dab2da515e568f', + 'number': 10647520, + 'timestamp': 1675576398, + 'nonce': '0x0000000000000000', + 'difficulty': 0, + 'gasLimit': { + 'type': 'BigNumber', + 'hex': '0x02625a00' + }, + 'gasUsed': { + 'type': 'BigNumber', + 'hex': '0x00' + }, + 'miner': '0xd98b305a9d433f062d44c1D542cdc25ECC0F0a40', + 'extraData': '0x', + 'transactions': [], + 'baseFeePerGas': { + 'type': 'BigNumber', + 'hex': '0x04a817c800' + }, + '_difficulty': { + 'type': 'BigNumber', + 'hex': '0x00' + } + } + } + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_parse_response_error_on_finalized_bundle_fetching(patch_base_class, monkeypatch): + stream = KyveStream(config, pool_data) + + input_request = requests.Response() + inputs = {"response": input_request, "stream_state": {}} + + mock_finalized_bundles_request = MagicMock(side_effect=IndexError) + monkeypatch.setattr('requests.Response.json', mock_finalized_bundles_request) + + with pytest.raises(StopIteration): + next(stream.parse_response(**inputs)) + + +def test_request_headers(patch_base_class): + stream = KyveStream(config, pool_data) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = KyveStream(config, pool_data) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = KyveStream(config, pool_data) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = KyveStream(config, pool_data) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_util.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_util.py new file mode 100644 index 000000000000..a5e5bcb8e0f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_util.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_kyve.util import CustomResourceSchemaLoader + + +def test_custom_loader(): + custom_loader = CustomResourceSchemaLoader("source_kyve") + schema = custom_loader.get_schema("evm/block") + + assert schema == { + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": True, + "properties": { + "_difficulty": { + "properties": { + "hex": {"type": ["null", "string"]}, + "type": {"type": ["null", "string"]}, + }, + "required": ["hex", "type"], + "type": ["null", "object"], + }, + "difficulty": {"type": ["null", "integer"]}, + "extraData": {"type": ["null", "string"]}, + "gasLimit": { + "properties": { + "hex": {"type": ["null", "string"]}, + "type": {"type": ["null", "string"]}, + }, + "required": ["hex", "type"], + "type": ["null", "object"], + }, + "gasUsed": { + "properties": { + "hex": {"type": ["null", "string"]}, + "type": {"type": ["null", "string"]}, + }, + "required": ["hex", "type"], + "type": ["null", "object"], + }, + "hash": {"type": ["null", "string"]}, + "miner": {"type": ["null", "string"]}, + "number": {"type": ["null", "integer"]}, + "parentHash": {"type": ["null", "string"]}, + "timestamp": {"type": ["null", "integer"]}, + "transactions": { + "items": { + "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": True, + "properties": { + "accessList": {"type": "null"}, + "blockHash": {"type": ["null", "string"]}, + "blockNumber": {"type": ["null", "integer"]}, + "chainId": {"type": ["null", "integer"]}, + "creates": {"type": ["null", "string"]}, + "data": {"type": ["null", "string"]}, + "from": {"type": ["null", "string"]}, + "gasLimit": { + "properties": { + "hex": {"type": ["null", "string"]}, + "type": {"type": ["null", "string"]}, + }, + "required": ["hex", "type"], + "type": ["null", "object"], + }, + "gasPrice": { + "properties": { + "hex": {"type": ["null", "string"]}, + "type": {"type": ["null", "string"]}, + }, + "required": ["hex", "type"], + "type": ["null", "object"], + }, + "hash": {"type": ["null", "string"]}, + "nonce": {"type": ["null", "integer"]}, + "r": {"type": ["null", "string"]}, + "raw": {"type": ["null", "string"]}, + "s": {"type": ["null", "string"]}, + "to": {"type": ["string", "null"]}, + "transactionIndex": {"type": ["null", "integer"]}, + "type": {"type": ["null", "integer"]}, + "v": {"type": ["null", "integer"]}, + "value": { + "properties": { + "hex": {"type": ["null", "string"]}, + "type": {"type": ["null", "string"]}, + }, + "required": ["hex", "type"], + "type": ["null", "object"], + }, + }, + "required": [ + "r", + "s", + "v", + "to", + "data", + "from", + "hash", + "type", + "nonce", + "value", + "chainId", + "creates", + "gasLimit", + "gasPrice", + "blockHash", + "accessList", + "blockNumber", + "transactionIndex", + ], + "type": "object", + }, + "type": ["null", "array"], + }, + }, + "required": [ + "hash", + "miner", + "number", + "gasUsed", + "gasLimit", + "extraData", + "timestamp", + "difficulty", + "parentHash", + "_difficulty", + "transactions", + ], + "type": "object", + } diff --git a/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml b/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml index ad9d6a598474..d55516006941 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml +++ b/airbyte-integrations/connectors/source-launchdarkly/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-launchdarkly/requirements.txt b/airbyte-integrations/connectors/source-launchdarkly/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/requirements.txt +++ b/airbyte-integrations/connectors/source-launchdarkly/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-launchdarkly/setup.py b/airbyte-integrations/connectors/source-launchdarkly/setup.py index 036eb01d57e7..07d48e22e847 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/setup.py +++ b/airbyte-integrations/connectors/source-launchdarkly/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-lemlist/metadata.yaml b/airbyte-integrations/connectors/source-lemlist/metadata.yaml index c9c0d8179113..5585511fa412 100644 --- a/airbyte-integrations/connectors/source-lemlist/metadata.yaml +++ b/airbyte-integrations/connectors/source-lemlist/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/lemlist tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-lemlist/requirements.txt b/airbyte-integrations/connectors/source-lemlist/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-lemlist/requirements.txt +++ b/airbyte-integrations/connectors/source-lemlist/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-lemlist/setup.py b/airbyte-integrations/connectors/source-lemlist/setup.py index f686243ddf83..56192197bbfd 100644 --- a/airbyte-integrations/connectors/source-lemlist/setup.py +++ b/airbyte-integrations/connectors/source-lemlist/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.2.5", "pytest-mock~=3.6.1", "connector-acceptance-test", "responses~=0.14.0"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2.5", "pytest-mock~=3.6.1", "responses~=0.14.0"] setup( name="source_lemlist", diff --git a/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml b/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml index 778c1ca59ea6..9a49336207cb 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml +++ b/airbyte-integrations/connectors/source-lever-hiring/metadata.yaml @@ -10,11 +10,15 @@ data: name: Lever Hiring registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/lever-hiring tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-lever-hiring/requirements.txt b/airbyte-integrations/connectors/source-lever-hiring/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/requirements.txt +++ b/airbyte-integrations/connectors/source-lever-hiring/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-lever-hiring/setup.py b/airbyte-integrations/connectors/source-lever-hiring/setup.py index b06cafa294a0..3ceb890c8e93 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/setup.py +++ b/airbyte-integrations/connectors/source-lever-hiring/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index 6e60c44f7842..6790451bf0b1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.6.1 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml index bb289e9e3131..b303c49e4115 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml @@ -6,6 +6,7 @@ acceptance_tests: spec: tests: - spec_path: "source_linkedin_ads/spec.json" + config_path: "secrets/config_oauth.json" connection: tests: - config_path: "secrets/config_oauth.json" @@ -13,22 +14,44 @@ acceptance_tests: timeout_seconds: 60 - config_path: "integration_tests/invalid_config.json" status: "failed" + - config_path: "integration_tests/invalid_config_custom_report.json" + status: "failed" discovery: tests: - config_path: "secrets/config_oauth.json" - backward_compatibility_tests_config: - disable_for_version: 0.1.16 # migration to May 2023 Api Version; schema changes; stream removed timeout_seconds: 60 basic_read: tests: - config_path: "secrets/config_oauth.json" expect_records: path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + fail_on_extra_columns: true + timeout_seconds: 3600 + ignored_fields: + campaign_groups: + - name: "lastModified" + bypass_reason: "Volatile data" + empty_streams: + - name: ad_member_company_size_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_country_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_job_function_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_job_title_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_industry_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_seniority_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_region_analytics + bypass_reason: "Empty stream; Retention period is 2y" + - name: ad_member_company_analytics + bypass_reason: "Empty stream; Retention period is 2y" incremental: tests: - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" missing_streams: diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json index 4e06df98d127..7a407fd0ece1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/abnormal_state.json @@ -30,22 +30,78 @@ { "type": "STREAM", "stream": { - "stream_state": { "lastModified": "2050-01-01" }, - "stream_descriptor": { "name": "ad_direct_sponsored_contents" } + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_campaign_analytics" } } }, { "type": "STREAM", "stream": { "stream_state": { "end_date": "2050-01-01" }, - "stream_descriptor": { "name": "ad_campaign_analytics" } + "stream_descriptor": { "name": "ad_creative_analytics" } } }, { "type": "STREAM", "stream": { "stream_state": { "end_date": "2050-01-01" }, - "stream_descriptor": { "name": "ad_creative_analytics" } + "stream_descriptor": { "name": "ad_impression_device_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_company_size_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_country_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_job_function_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_job_title_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_industry_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_seniority_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_region_analytics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "end_date": "2050-01-01" }, + "stream_descriptor": { "name": "ad_member_company_analytics" } } } ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json index 1864e6eb744f..0713a1762b94 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/configured_catalog.json @@ -60,6 +60,18 @@ "cursor_field": ["lastModifiedAt"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "conversions", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, { "stream": { "name": "ad_campaign_analytics", @@ -83,6 +95,114 @@ "sync_mode": "incremental", "cursor_field": ["end_date"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_impression_device_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_company_size_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_country_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_job_function_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_job_title_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_industry_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_seniority_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_region_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_member_company_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl index 84c08b1f5a1f..9a55a6a2d76c 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl @@ -1,37 +1,50 @@ -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["RUNNABLE"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"6"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Jean’s Ad Account","currency":"USD","id":508720451,"status":"ACTIVE","created":"2021-06-14 10:09:22","lastModified":"2021-08-27 23:40:10"},"emitted_at":1684840476614} -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"4"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Test Account 2","currency":"USD","id":508774356,"status":"ACTIVE","created":"2021-08-21 21:28:19","lastModified":"2021-08-25 10:52:25"},"emitted_at":1684840476615} -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"4"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Test Account 1","currency":"USD","id":508777244,"status":"ACTIVE","created":"2021-08-21 21:27:55","lastModified":"2021-08-22 20:35:44"},"emitted_at":1684840476615} -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"3"},"reference":"urn:li:person:HRnXB4kIO7","notifiedOnCreativeApproval":false,"name":"Test Account 3","currency":"NOK","id":510426150,"status":"ACTIVE","created":"2022-10-07 16:41:09","lastModified":"2022-10-07 16:41:09"},"emitted_at":1684840476615} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508720451","created":"2021-06-14 10:09:22","lastModified":"2021-06-14 10:09:22"},"emitted_at":1684840478573} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508774356","created":"2021-08-21 21:28:19","lastModified":"2021-08-21 21:28:19"},"emitted_at":1684840478863} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508777244","created":"2021-08-21 21:27:55","lastModified":"2021-08-21 21:27:55"},"emitted_at":1684840479156} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:510426150","created":"2022-10-07 16:41:09","lastModified":"2022-10-07 16:41:09"},"emitted_at":1684840479404} -{"stream":"campaign_groups","data":{"runSchedule":{"start":1623665362312},"test":false,"name":"Default Campaign Group","servingStatuses":["RUNNABLE"],"backfilled":true,"id":615492066,"account":"urn:li:sponsoredAccount:508720451","status":"ACTIVE","created":"2021-06-14 10:09:22","lastModified":"2021-06-14 10:09:22"},"emitted_at":1684840481331} -{"stream":"campaign_groups","data":{"runSchedule":{"start":1628229693058,"end":1630971900000},"test":false,"totalBudget":{"currencyCode":"USD","amount":"100"},"name":"Airbyte Test","servingStatuses":["CAMPAIGN_GROUP_END_DATE_HOLD","CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"],"backfilled":false,"id":616471656,"account":"urn:li:sponsoredAccount:508720451","status":"ACTIVE","created":"2021-08-06 06:01:33","lastModified":"2021-09-06 23:35:14"},"emitted_at":1684840481332} -{"stream":"campaign_groups","data":{"runSchedule":{"start":1629581299760},"test":false,"name":"Test Campaign Group 2","servingStatuses":["STOPPED","BILLING_HOLD"],"backfilled":false,"id":616749096,"account":"urn:li:sponsoredAccount:508774356","status":"PAUSED","created":"2021-08-21 21:28:19","lastModified":"2021-08-21 21:29:27"},"emitted_at":1684840481623} -{"stream":"campaign_groups","data":{"runSchedule":{"start":1629581275652},"test":false,"name":"Test Campaign Group 1","servingStatuses":["STOPPED","BILLING_HOLD"],"backfilled":false,"id":616749086,"account":"urn:li:sponsoredAccount:508777244","status":"PAUSED","created":"2021-08-21 21:27:55","lastModified":"2021-08-22 20:29:09"},"emitted_at":1684840481905} -{"stream":"campaign_groups","data":{"runSchedule":{"start":1665160869034},"test":false,"name":"New Campaign Group","servingStatuses":["BILLING_HOLD"],"backfilled":false,"id":628297234,"account":"urn:li:sponsoredAccount:510426150","status":"ACTIVE","created":"2022-10-07 16:41:09","lastModified":"2022-10-07 19:16:09"},"emitted_at":1684840482130} -{"stream":"campaigns","data":{"storyDeliveryEnabled":false,"targetingCriteria":{"include":{"and":[{"type":"urn:li:adTargetingFacet:interfaceLocales","values":["urn:li:locale:en_US"]},{"type":"urn:li:adTargetingFacet:locations","values":["urn:li:geo:103644278"]}]}},"pacingStrategy":"LIFETIME","locale":{"country":"US","language":"en"},"type":"SPONSORED_UPDATES","optimizationTargetType":"MAX_REACH","runSchedule":{"start":1628230144426,"end":1630971900000},"costType":"CPM","creativeSelection":"OPTIMIZED","offsiteDeliveryEnabled":true,"id":168387646,"audienceExpansionEnabled":true,"test":false,"format":"STANDARD_UPDATE","servingStatuses":["CAMPAIGN_END_DATE_HOLD","STOPPED","CAMPAIGN_GROUP_END_DATE_HOLD","CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"],"version":{"versionTag":"5"},"objectiveType":"BRAND_AWARENESS","associatedEntity":"urn:li:organization:64265083","offsitePreferences":{"iabCategories":{"exclude":[]},"publisherRestrictionFiles":{"include":[],"exclude":[]}},"campaignGroup":"urn:li:sponsoredCampaignGroup:616471656","dailyBudget":{"currencyCode":"USD","amount":"10"},"unitCost":{"currencyCode":"USD","amount":"62.73"},"name":"Brand awareness - Aug 6, 2021","account":"urn:li:sponsoredAccount:508720451","status":"COMPLETED","created":"2021-08-06 06:03:52","lastModified":"2021-11-17 06:44:23"},"emitted_at":1684840484115} -{"stream":"campaigns","data":{"storyDeliveryEnabled":false,"targetingCriteria":{"include":{"and":[{"type":"urn:li:adTargetingFacet:titles","values":["urn:li:title:100","urn:li:title:10326","urn:li:title:10457","urn:li:title:10738","urn:li:title:10966","urn:li:title:11349","urn:li:title:1159","urn:li:title:11622","urn:li:title:1176","urn:li:title:11886","urn:li:title:1211","urn:li:title:12490","urn:li:title:13499","urn:li:title:1359","urn:li:title:1399","urn:li:title:1414","urn:li:title:14642","urn:li:title:14893","urn:li:title:1586","urn:li:title:160","urn:li:title:16432","urn:li:title:1685","urn:li:title:17134","urn:li:title:17265","urn:li:title:1845","urn:li:title:189","urn:li:title:1890","urn:li:title:18930","urn:li:title:1897","urn:li:title:191","urn:li:title:2105","urn:li:title:2189","urn:li:title:219","urn:li:title:23347","urn:li:title:23484","urn:li:title:24","urn:li:title:25166","urn:li:title:25169","urn:li:title:25170","urn:li:title:25194","urn:li:title:25201","urn:li:title:25203","urn:li:title:25204","urn:li:title:253","urn:li:title:266","urn:li:title:2740","urn:li:title:3172","urn:li:title:318","urn:li:title:328","urn:li:title:332","urn:li:title:3516","urn:li:title:3549","urn:li:title:3598","urn:li:title:39","urn:li:title:3927","urn:li:title:424","urn:li:title:4327","urn:li:title:4384","urn:li:title:4403","urn:li:title:4484","urn:li:title:4677","urn:li:title:4691","urn:li:title:5316","urn:li:title:539","urn:li:title:556","urn:li:title:5762","urn:li:title:599","urn:li:title:6058","urn:li:title:607","urn:li:title:659","urn:li:title:661","urn:li:title:67","urn:li:title:7000","urn:li:title:7110","urn:li:title:7176","urn:li:title:7555","urn:li:title:761","urn:li:title:7732","urn:li:title:9","urn:li:title:932","urn:li:title:940","urn:li:title:9540","urn:li:title:9633","urn:li:title:971","urn:li:title:9715","urn:li:title:9763"]},{"type":"urn:li:adTargetingFacet:locations","values":["urn:li:geo:103644278"]},{"type":"urn:li:adTargetingFacet:interfaceLocales","values":["urn:li:locale:en_US"]}]}},"pacingStrategy":"LIFETIME","locale":{"country":"US","language":"en"},"type":"SPONSORED_UPDATES","optimizationTargetType":"MAX_CLICK","runSchedule":{"start":1629849600000},"costType":"CPM","creativeSelection":"OPTIMIZED","offsiteDeliveryEnabled":true,"id":169185036,"audienceExpansionEnabled":true,"test":false,"format":"STANDARD_UPDATE","servingStatuses":["STOPPED","ACCOUNT_SERVING_HOLD","CAMPAIGN_GROUP_STATUS_HOLD"],"version":{"versionTag":"3"},"objectiveType":"WEBSITE_VISIT","associatedEntity":"urn:li:organization:64265083","offsitePreferences":{"iabCategories":{"exclude":[]},"publisherRestrictionFiles":{"include":[],"exclude":[]}},"campaignGroup":"urn:li:sponsoredCampaignGroup:616749096","dailyBudget":{"currencyCode":"USD","amount":"75"},"unitCost":{"currencyCode":"USD","amount":"16.41"},"name":"Website visits - Aug 25, 2021","account":"urn:li:sponsoredAccount:508774356","status":"DRAFT","created":"2021-08-25 10:52:29","lastModified":"2021-11-07 12:41:09"},"emitted_at":1684840484417} -{"stream":"campaigns","data":{"storyDeliveryEnabled":false,"targetingCriteria":{"include":{"and":[{"type":"urn:li:adTargetingFacet:interfaceLocales","values":["urn:li:locale:en_US"]},{"type":"urn:li:adTargetingFacet:locations","values":["urn:li:geo:103644278"]}]}},"pacingStrategy":"LIFETIME","locale":{"country":"US","language":"en"},"type":"SPONSORED_UPDATES","optimizationTargetType":"MAX_REACH","runSchedule":{"start":1629590400000},"costType":"CPM","creativeSelection":"OPTIMIZED","offsiteDeliveryEnabled":true,"id":169037246,"audienceExpansionEnabled":true,"test":false,"format":"SINGLE_VIDEO","servingStatuses":["STOPPED","ACCOUNT_SERVING_HOLD","CAMPAIGN_GROUP_STATUS_HOLD"],"version":{"versionTag":"3"},"objectiveType":"BRAND_AWARENESS","associatedEntity":"urn:li:organization:64265083","offsitePreferences":{"iabCategories":{"exclude":[]},"publisherRestrictionFiles":{"include":[],"exclude":[]}},"campaignGroup":"urn:li:sponsoredCampaignGroup:616749086","dailyBudget":{"currencyCode":"USD","amount":"100"},"unitCost":{"currencyCode":"USD","amount":"61.02"},"name":"Brand awareness - Aug 22, 2021","account":"urn:li:sponsoredAccount:508777244","status":"DRAFT","created":"2021-08-22 20:37:17","lastModified":"2021-11-07 12:20:05"},"emitted_at":1684840484720} -{"stream":"creatives","data":{"servingHoldReasons":["CAMPAIGN_END_DATE_HOLD","CAMPAIGN_STOPPED","CAMPAIGN_GROUP_END_DATE_HOLD","CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"],"lastModifiedAt":1656599327000,"lastModifiedBy":"urn:li:system:0","content":{"reference":"urn:li:share:6823991265126957056"},"createdAt":1628229937000,"isTest":false,"createdBy":"urn:li:person:HRnXB4kIO7","review":{"status":"APPROVED"},"isServing":false,"campaign":"urn:li:sponsoredCampaign:168387646","id":"urn:li:sponsoredCreative:133813726","intendedStatus":"ACTIVE","account":"urn:li:sponsoredAccount:508720451"},"emitted_at":1684840486978} -{"stream":"creatives","data":{"servingHoldReasons":["UNDER_REVIEW","CAMPAIGN_STOPPED","ACCOUNT_SERVING_HOLD","CAMPAIGN_GROUP_STATUS_HOLD"],"lastModifiedAt":1656631421000,"lastModifiedBy":"urn:li:system:0","content":{"reference":"urn:li:share:6836249289476456448"},"createdAt":1629888842000,"isTest":false,"createdBy":"urn:li:person:HRnXB4kIO7","review":{"status":"PENDING"},"isServing":false,"campaign":"urn:li:sponsoredCampaign:169185036","id":"urn:li:sponsoredCreative:136324456","intendedStatus":"ACTIVE","account":"urn:li:sponsoredAccount:508774356"},"emitted_at":1684840487277} -{"stream":"creatives","data":{"servingHoldReasons":["UNDER_REVIEW","CAMPAIGN_STOPPED","ACCOUNT_SERVING_HOLD","CAMPAIGN_GROUP_STATUS_HOLD"],"lastModifiedAt":1631289063000,"lastModifiedBy":"urn:li:person:HRnXB4kIO7","content":{"reference":"urn:li:ugcPost:6835311566041284608"},"createdAt":1629665365000,"isTest":false,"createdBy":"urn:li:person:HRnXB4kIO7","review":{"status":"PENDING"},"isServing":false,"campaign":"urn:li:sponsoredCampaign:169037246","id":"urn:li:sponsoredCreative:135841046","intendedStatus":"ACTIVE","account":"urn:li:sponsoredAccount:508777244"},"emitted_at":1684840487612} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.67,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.67,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":830,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":887,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732138} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.230000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.230000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"approximateUniqueImpressions":552,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":552,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732141} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.470000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.470000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":902,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":994,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732143} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":15.000000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":15.000000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":9,"cardClicks":0,"approximateUniqueImpressions":1546,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":9,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1560,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":9,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732145} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.39,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.39,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1241,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1241,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732146} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.460000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.460000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":5,"cardClicks":0,"approximateUniqueImpressions":371,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":5,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":403,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732148} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.360000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.360000000000001,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1017,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1017,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732149} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.999999999999998,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.999999999999998,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":1279,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1606,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732150} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.4199999999999997,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.4199999999999997,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":116,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":116,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684920732151} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.999999999999998,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.999999999999998,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":1279,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1606,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907513} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.230000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.230000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"approximateUniqueImpressions":552,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":552,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907516} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.67,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.67,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":830,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":887,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907517} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.360000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.360000000000001,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1017,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1017,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907518} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":15.000000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":15.000000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":9,"cardClicks":0,"approximateUniqueImpressions":1546,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":9,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1560,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":9,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907519} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.4199999999999997,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.4199999999999997,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":116,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":116,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907520} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.39,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.39,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1241,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1241,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907521} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.460000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.460000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":5,"cardClicks":0,"approximateUniqueImpressions":371,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":5,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":403,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907522} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.470000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.470000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":902,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":994,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1684922907523} \ No newline at end of file +{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["RUNNABLE"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"6"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Jean’s Ad Account","currency":"USD","id":508720451,"status":"ACTIVE","created":"2021-06-14T10:09:22+00:00","lastModified":"2021-08-27T23:40:10+00:00"},"emitted_at":1691579980301} +{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"4"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Test Account 2","currency":"USD","id":508774356,"status":"ACTIVE","created":"2021-08-21T21:28:19+00:00","lastModified":"2021-08-25T10:52:25+00:00"},"emitted_at":1691579980306} +{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"4"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Test Account 1","currency":"USD","id":508777244,"status":"ACTIVE","created":"2021-08-21T21:27:55+00:00","lastModified":"2021-08-22T20:35:44+00:00"},"emitted_at":1691579980310} +{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"3"},"reference":"urn:li:person:HRnXB4kIO7","notifiedOnCreativeApproval":false,"name":"Test Account 3","currency":"NOK","id":510426150,"status":"ACTIVE","created":"2022-10-07T16:41:09+00:00","lastModified":"2022-10-07T16:41:09+00:00"},"emitted_at":1691579980314} +{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508720451","created":"2021-06-14T10:09:22+00:00","lastModified":"2021-06-14T10:09:22+00:00"},"emitted_at":1691579983649} +{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508774356","created":"2021-08-21T21:28:19+00:00","lastModified":"2021-08-21T21:28:19+00:00"},"emitted_at":1691579983951} +{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508777244","created":"2021-08-21T21:27:55+00:00","lastModified":"2021-08-21T21:27:55+00:00"},"emitted_at":1691579984340} +{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:510426150","created":"2022-10-07T16:41:09+00:00","lastModified":"2022-10-07T16:41:09+00:00"},"emitted_at":1691579984764} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1623665362312}, "test": false, "name": "Default Campaign Group", "servingStatuses": ["RUNNABLE"], "backfilled": true, "id": 615492066, "account": "urn:li:sponsoredAccount:508720451", "status": "ACTIVE", "created": "2021-06-14T10:09:22+00:00", "lastModified": "2021-06-14T10:09:22+00:00"}, "emitted_at": 1692699331827} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1628229693058, "end": 1695253500000}, "test": false, "totalBudget": {"currencyCode": "USD", "amount": "200"}, "name": "Airbyte Test", "servingStatuses": ["RUNNABLE"], "backfilled": false, "id": 616471656, "account": "urn:li:sponsoredAccount:508720451", "status": "ACTIVE", "created": "2021-08-06T06:01:33+00:00", "lastModified": "2023-08-21T10:08:54+00:00"}, "emitted_at": 1692699331830} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1629581299760}, "test": false, "name": "Test Campaign Group 2", "servingStatuses": ["STOPPED", "BILLING_HOLD"], "backfilled": false, "id": 616749096, "account": "urn:li:sponsoredAccount:508774356", "status": "PAUSED", "created": "2021-08-21T21:28:19+00:00", "lastModified": "2021-08-21T21:29:27+00:00"}, "emitted_at": 1692699332065} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1629581275652}, "test": false, "name": "Test Campaign Group 1", "servingStatuses": ["STOPPED", "BILLING_HOLD"], "backfilled": false, "id": 616749086, "account": "urn:li:sponsoredAccount:508777244", "status": "PAUSED", "created": "2021-08-21T21:27:55+00:00", "lastModified": "2021-08-22T20:29:09+00:00"}, "emitted_at": 1692699332293} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1665160869034}, "test": false, "name": "New Campaign Group", "servingStatuses": ["BILLING_HOLD"], "backfilled": false, "id": 628297234, "account": "urn:li:sponsoredAccount:510426150", "status": "ACTIVE", "created": "2022-10-07T16:41:09+00:00", "lastModified": "2022-10-07T19:16:09+00:00"}, "emitted_at": 1692699332617} +{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_REACH", "runSchedule": {"start": 1628230144426, "end": 1630971900000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 168387646, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["CAMPAIGN_END_DATE_HOLD", "STOPPED"], "version": {"versionTag": "6"}, "objectiveType": "BRAND_AWARENESS", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616471656", "dailyBudget": {"currencyCode": "USD", "amount": "10"}, "unitCost": {"currencyCode": "USD", "amount": "62.73"}, "name": "Brand awareness - Aug 6, 2021", "account": "urn:li:sponsoredAccount:508720451", "status": "COMPLETED", "created": "2021-08-06T06:03:52+00:00", "lastModified": "2023-08-21T10:08:58+00:00"}, "emitted_at": 1692704799180} +{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:titles", "values": ["urn:li:title:100", "urn:li:title:10326", "urn:li:title:10457", "urn:li:title:10738", "urn:li:title:10966", "urn:li:title:11349", "urn:li:title:1159", "urn:li:title:11622", "urn:li:title:1176", "urn:li:title:11886", "urn:li:title:1211", "urn:li:title:12490", "urn:li:title:13499", "urn:li:title:1359", "urn:li:title:1399", "urn:li:title:1414", "urn:li:title:14642", "urn:li:title:14893", "urn:li:title:1586", "urn:li:title:160", "urn:li:title:16432", "urn:li:title:1685", "urn:li:title:17134", "urn:li:title:17265", "urn:li:title:1845", "urn:li:title:189", "urn:li:title:1890", "urn:li:title:18930", "urn:li:title:1897", "urn:li:title:191", "urn:li:title:2105", "urn:li:title:2189", "urn:li:title:219", "urn:li:title:23347", "urn:li:title:23484", "urn:li:title:24", "urn:li:title:25166", "urn:li:title:25169", "urn:li:title:25170", "urn:li:title:25194", "urn:li:title:25201", "urn:li:title:25203", "urn:li:title:25204", "urn:li:title:253", "urn:li:title:266", "urn:li:title:2740", "urn:li:title:3172", "urn:li:title:318", "urn:li:title:328", "urn:li:title:332", "urn:li:title:3516", "urn:li:title:3549", "urn:li:title:3598", "urn:li:title:39", "urn:li:title:3927", "urn:li:title:424", "urn:li:title:4327", "urn:li:title:4384", "urn:li:title:4403", "urn:li:title:4484", "urn:li:title:4677", "urn:li:title:4691", "urn:li:title:5316", "urn:li:title:539", "urn:li:title:556", "urn:li:title:5762", "urn:li:title:599", "urn:li:title:6058", "urn:li:title:607", "urn:li:title:659", "urn:li:title:661", "urn:li:title:67", "urn:li:title:7000", "urn:li:title:7110", "urn:li:title:7176", "urn:li:title:7555", "urn:li:title:761", "urn:li:title:7732", "urn:li:title:9", "urn:li:title:932", "urn:li:title:940", "urn:li:title:9540", "urn:li:title:9633", "urn:li:title:971", "urn:li:title:9715", "urn:li:title:9763"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}, {"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_CLICK", "runSchedule": {"start": 1629849600000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 169185036, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "version": {"versionTag": "3"}, "objectiveType": "WEBSITE_VISIT", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616749096", "dailyBudget": {"currencyCode": "USD", "amount": "75"}, "unitCost": {"currencyCode": "USD", "amount": "16.41"}, "name": "Website visits - Aug 25, 2021", "account": "urn:li:sponsoredAccount:508774356", "status": "DRAFT", "created": "2021-08-25T10:52:29+00:00", "lastModified": "2021-11-07T12:41:09+00:00"}, "emitted_at": 1692704799423} +{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_REACH", "runSchedule": {"start": 1629590400000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 169037246, "audienceExpansionEnabled": true, "test": false, "format": "SINGLE_VIDEO", "servingStatuses": ["STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "version": {"versionTag": "3"}, "objectiveType": "BRAND_AWARENESS", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616749086", "dailyBudget": {"currencyCode": "USD", "amount": "100"}, "unitCost": {"currencyCode": "USD", "amount": "61.02"}, "name": "Brand awareness - Aug 22, 2021", "account": "urn:li:sponsoredAccount:508777244", "status": "DRAFT", "created": "2021-08-22T20:37:17+00:00", "lastModified": "2021-11-07T12:20:05+00:00"}, "emitted_at": 1692704799664} +{"stream": "creatives", "data": {"servingHoldReasons": ["CAMPAIGN_END_DATE_HOLD", "CAMPAIGN_STOPPED"], "lastModifiedAt": 1656599327000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6823991265126957056"}, "createdAt": 1628229937000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "APPROVED"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:168387646", "id": "urn:li:sponsoredCreative:133813726", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508720451"}, "emitted_at": 1692707187268} +{"stream": "creatives", "data": {"servingHoldReasons": ["UNDER_REVIEW", "CAMPAIGN_STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "lastModifiedAt": 1656631421000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6836249289476456448"}, "createdAt": 1629888842000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "PENDING"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:169185036", "id": "urn:li:sponsoredCreative:136324456", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508774356"}, "emitted_at": 1692707187573} +{"stream": "creatives", "data": {"servingHoldReasons": ["UNDER_REVIEW", "CAMPAIGN_STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "lastModifiedAt": 1631289063000, "lastModifiedBy": "urn:li:person:HRnXB4kIO7", "content": {"reference": "urn:li:ugcPost:6835311566041284608"}, "createdAt": 1629665365000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "PENDING"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:169037246", "id": "urn:li:sponsoredCreative:135841046", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508777244"}, "emitted_at": 1692707188048} +{"stream":"conversions","data":{"postClickAttributionWindowSize":30,"viewThroughAttributionWindowSize":7,"created":1692168056678,"imagePixelTag":"\"\"","type":"AD_CLICK","enabled":true,"associatedCampaigns":[{"associatedAt":1692609636804,"campaign":"urn:li:sponsoredCampaign:252074216","conversion":"urn:lla:llaPartnerConversion:13703588"},{"associatedAt":1692168067977,"campaign":"urn:li:sponsoredCampaign:251861596","conversion":"urn:lla:llaPartnerConversion:13703588"}],"campaigns":["urn:li:sponsoredCampaign:252074216","urn:li:sponsoredCampaign:251861596"],"name":"Airbyte","urlMatchRuleExpression":[[{"matchValue":"https://airbyte.com/","matchType":"STARTS_WITH"}]],"lastModified":1692168056678,"id":13703588,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[{"type":"STARTS_WITH","matchValue":"https://airbyte.com/"}],"value":{"currencyCode":"USD","amount":"1"},"account":"urn:li:sponsoredAccount:508720451"},"emitted_at":1692726263419} +{"stream":"conversions","data":{"postClickAttributionWindowSize":30,"viewThroughAttributionWindowSize":7,"created":1629376903467,"imagePixelTag":"\"\"","type":"AD_VIEW","enabled":true,"associatedCampaigns":[{"associatedAt":1692167555159,"campaign":"urn:li:sponsoredCampaign:251861596","conversion":"urn:lla:llaPartnerConversion:4677476"},{"associatedAt":1629376986791,"campaign":"urn:li:sponsoredCampaign:168387646","conversion":"urn:lla:llaPartnerConversion:4677476"}],"campaigns":["urn:li:sponsoredCampaign:251861596","urn:li:sponsoredCampaign:168387646"],"name":"Test Conversion","urlMatchRuleExpression":[[{"matchValue":"www.aibyte.io","matchType":"STARTS_WITH"}]],"lastModified":1629380909048,"id":4677476,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[],"value":{"currencyCode":"USD","amount":"0"},"account":"urn:li:sponsoredAccount:508720451"},"emitted_at":1692726263420} +{"stream":"conversions","data":{"postClickAttributionWindowSize":1,"viewThroughAttributionWindowSize":1,"created":1629888666093,"imagePixelTag":"\"\"","type":"SIGN_UP","enabled":true,"associatedCampaigns":[{"associatedAt":1629888749778,"campaign":"urn:li:sponsoredCampaign:169185036","conversion":"urn:lla:llaPartnerConversion:4620028"}],"campaigns":["urn:li:sponsoredCampaign:169185036"],"name":"Test Conversion 3","urlMatchRuleExpression":[[{"matchValue":"https://airbyte.io","matchType":"STARTS_WITH"}]],"lastModified":1629888698401,"id":4620028,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[],"value":{"currencyCode":"USD","amount":"15"},"account":"urn:li:sponsoredAccount:508774356"},"emitted_at":1692726264005} +{"stream":"conversions","data":{"postClickAttributionWindowSize":1,"viewThroughAttributionWindowSize":1,"created":1629664605296,"imagePixelTag":"\"\"","type":"KEY_PAGE_VIEW","enabled":true,"associatedCampaigns":[{"associatedAt":1629664638873,"campaign":"urn:li:sponsoredCampaign:169037246","conversion":"urn:lla:llaPartnerConversion:4604364"}],"campaigns":["urn:li:sponsoredCampaign:169037246"],"name":"Test Conversion 2","urlMatchRuleExpression":[[{"matchValue":"https://airbyte.io","matchType":"STARTS_WITH"}]],"lastModified":1629664630274,"id":4604364,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[],"value":{"currencyCode":"USD","amount":"15"},"account":"urn:li:sponsoredAccount:508777244"},"emitted_at":1692726264514} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.67,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.67,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":830,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":887,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070057} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.230000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.230000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"approximateUniqueImpressions":552,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":552,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070089} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.470000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.470000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":902,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":994,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070112} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":15.000000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":15.000000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":9,"cardClicks":0,"approximateUniqueImpressions":1546,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":9,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1560,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":9,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070136} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.39,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.39,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1241,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1241,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070162} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.460000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.460000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":5,"cardClicks":0,"approximateUniqueImpressions":371,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":5,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":403,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070188} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.360000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.360000000000001,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1017,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1017,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070211} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.999999999999998,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.999999999999998,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":1279,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1606,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070250} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.4199999999999997,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.4199999999999997,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":116,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":116,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070269} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.999999999999998,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.999999999999998,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":1279,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1606,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220592} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.230000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.230000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"approximateUniqueImpressions":552,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":552,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220619} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.67,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.67,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":830,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":887,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220638} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.360000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.360000000000001,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1017,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1017,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220656} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":15.000000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":15.000000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":9,"cardClicks":0,"approximateUniqueImpressions":1546,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":9,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1560,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":9,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220677} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.4199999999999997,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.4199999999999997,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":116,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":116,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220697} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.39,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.39,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1241,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1241,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220720} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.460000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.460000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":5,"cardClicks":0,"approximateUniqueImpressions":371,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":5,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":403,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220745} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.470000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.470000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":902,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":994,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220787} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.48000000000000004,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.48000000000000004,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":36,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377453} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":3.69,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":3.69,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":371,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"videoViews":0},"emitted_at":1691580377480} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":11.11,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":11.11,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":674,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["DESKTOP_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377509} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.550000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.550000000000001,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":393,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["DESKTOP_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377539} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.76,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.76,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":75,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"videoViews":0},"emitted_at":1691580377562} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.75,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.75,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":62,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377591} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.760000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.760000000000002,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":797,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["DESKTOP_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"videoViews":0},"emitted_at":1691580377616} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.09,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.09,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":25,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377643} +{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.57,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.57,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":35,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377665} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/incremental_catalog.json new file mode 100644 index 000000000000..bedbea8ff4bd --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/incremental_catalog.json @@ -0,0 +1,100 @@ +{ + "streams": [ + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "full_refresh", + "cursor_field": [], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_users", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_groups", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModified"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModified"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "creatives", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastModifiedAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastModifiedAt"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_campaign_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_creative_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_impression_device_analytics", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["end_date"] + }, + "sync_mode": "incremental", + "cursor_field": ["end_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/invalid_config_custom_report.json b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/invalid_config_custom_report.json new file mode 100644 index 000000000000..533159258ad7 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/invalid_config_custom_report.json @@ -0,0 +1,17 @@ +{ + "start_date": "2021-01-01", + "account_ids": [], + "credentials": { + "auth_method": "oAuth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "ad_analytics_reports": [ + { + "name": "CreativeAdByMonth", + "pivot_by": "not_in_enum", + "time_granularity": "MONTHLY" + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml index 5b7d97d7bcdb..b74e5c81af1d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml @@ -1,12 +1,13 @@ data: allowedHosts: hosts: + - linkedin.com - api.linkedin.com connectorSubtype: api connectorType: source definitionId: 137ece28-5434-455c-8f34-69dc3782f451 maxSecondsBetweenMessages: 21600 - dockerImageTag: 0.2.1 + dockerImageTag: 0.6.1 dockerRepository: airbyte/source-linkedin-ads githubIssueLabel: source-linkedin-ads icon: linkedin.svg @@ -18,7 +19,20 @@ data: oss: enabled: true releaseStage: generally_available + suggestedStreams: + streams: + - accounts + - account_users + - ad_campaign_analytics + - ad_creative_analytics + - campaigns + - campaign_groups + - creatives documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt b/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt +++ b/airbyte-integrations/connectors/source-linkedin-ads/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py index 41311725b70b..1c15f41abf5f 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/setup.py @@ -6,12 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.37", + "airbyte-cdk~=0.50", ] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_creative_analytics.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_analytics.json similarity index 100% rename from airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_creative_analytics.json rename to airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_analytics.json diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_campaign_analytics.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_campaign_analytics.json deleted file mode 100644 index 93fbeb99e11a..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/ad_campaign_analytics.json +++ /dev/null @@ -1,304 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": ["null", "object"], - "title": "Ad Campaign Analytics", - "additionalProperties": true, - "properties": { - "actionClicks": { - "type": ["null", "number"] - }, - "adUnitClicks": { - "type": ["null", "number"] - }, - "approximateUniqueImpressions": { - "type": ["null", "number"] - }, - "cardClicks": { - "type": ["null", "number"] - }, - "cardImpressions": { - "type": ["null", "number"] - }, - "clicks": { - "type": ["null", "number"] - }, - "commentLikes": { - "type": ["null", "number"] - }, - "comments": { - "type": ["null", "number"] - }, - "companyPageClicks": { - "type": ["null", "number"] - }, - "conversionValueInLocalCurrency": { - "type": ["null", "number"] - }, - "costInLocalCurrency": { - "type": ["null", "number"] - }, - "costInUsd": { - "type": ["null", "number"] - }, - "documentCompletions": { - "type": ["null", "number"] - }, - "documentFirstQuartileCompletions": { - "type": ["null", "number"] - }, - "documentMidpointCompletions": { - "type": ["null", "number"] - }, - "documentThirdQuartileCompletions": { - "type": ["null", "number"] - }, - "downloadClicks": { - "type": ["null", "number"] - }, - "end_date": { - "type": ["null", "string"], - "format": "date" - }, - "externalWebsiteConversions": { - "type": ["null", "number"] - }, - "externalWebsitePostClickConversions": { - "type": ["null", "number"] - }, - "externalWebsitePostViewConversions": { - "type": ["null", "number"] - }, - "follows": { - "type": ["null", "number"] - }, - "fullScreenPlays": { - "type": ["null", "number"] - }, - "impressions": { - "type": ["null", "number"] - }, - "jobApplications": { - "type": ["null", "number"] - }, - "jobApplyClicks": { - "type": ["null", "number"] - }, - "landingPageClicks": { - "type": ["null", "number"] - }, - "leadGenerationMailContactInfoShares": { - "type": ["null", "number"] - }, - "leadGenerationMailInterestedClicks": { - "type": ["null", "number"] - }, - "likes": { - "type": ["null", "number"] - }, - "oneClickLeadFormOpens": { - "type": ["null", "number"] - }, - "oneClickLeads": { - "type": ["null", "number"] - }, - "opens": { - "type": ["null", "number"] - }, - "otherEngagements": { - "type": ["null", "number"] - }, - "pivotValue": { - "type": ["null", "string"] - }, - "pivotValues": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "postClickJobApplications": { - "type": ["null", "number"] - }, - "postClickJobApplyClicks": { - "type": ["null", "number"] - }, - "postClickRegistrations": { - "type": ["null", "number"] - }, - "postViewJobApplications": { - "type": ["null", "number"] - }, - "postViewJobApplyClicks": { - "type": ["null", "number"] - }, - "postViewRegistrations": { - "type": ["null", "number"] - }, - "reactions": { - "type": ["null", "number"] - }, - "registrations": { - "type": ["null", "number"] - }, - "sends": { - "type": ["null", "number"] - }, - "shares": { - "type": ["null", "number"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date" - }, - "talentLeads": { - "type": ["null", "number"] - }, - "textUrlClicks": { - "type": ["null", "number"] - }, - "totalEngagements": { - "type": ["null", "number"] - }, - "validWorkEmailLeads": { - "type": ["null", "number"] - }, - "videoCompletions": { - "type": ["null", "number"] - }, - "videoFirstQuartileCompletions": { - "type": ["null", "number"] - }, - "videoMidpointCompletions": { - "type": ["null", "number"] - }, - "videoStarts": { - "type": ["null", "number"] - }, - "videoThirdQuartileCompletions": { - "type": ["null", "number"] - }, - "videoViews": { - "type": ["null", "number"] - }, - "viralCardClicks": { - "type": ["null", "number"] - }, - "viralCardImpressions": { - "type": ["null", "number"] - }, - "viralClicks": { - "type": ["null", "number"] - }, - "viralCommentLikes": { - "type": ["null", "number"] - }, - "viralComments": { - "type": ["null", "number"] - }, - "viralCompanyPageClicks": { - "type": ["null", "number"] - }, - "viralDocumentCompletions": { - "type": ["null", "number"] - }, - "viralDocumentFirstQuartileCompletions": { - "type": ["null", "number"] - }, - "viralDocumentMidpointCompletions": { - "type": ["null", "number"] - }, - "viralDocumentThirdQuartileCompletions": { - "type": ["null", "number"] - }, - "viralDownloadClicks": { - "type": ["null", "number"] - }, - "viralExternalWebsiteConversions": { - "type": ["null", "number"] - }, - "viralExternalWebsitePostClickConversions": { - "type": ["null", "number"] - }, - "viralExternalWebsitePostViewConversions": { - "type": ["null", "number"] - }, - "viralFollows": { - "type": ["null", "number"] - }, - "viralFullScreenPlays": { - "type": ["null", "number"] - }, - "viralImpressions": { - "type": ["null", "number"] - }, - "viralJobApplications": { - "type": ["null", "number"] - }, - "viralJobApplyClicks": { - "type": ["null", "number"] - }, - "viralLandingPageClicks": { - "type": ["null", "number"] - }, - "viralLikes": { - "type": ["null", "number"] - }, - "viralOneClickLeadFormOpens": { - "type": ["null", "number"] - }, - "viralOneClickLeads": { - "type": ["null", "number"] - }, - "viralOtherEngagements": { - "type": ["null", "number"] - }, - "viralPostClickJobApplications": { - "type": ["null", "number"] - }, - "viralPostClickJobApplyClicks": { - "type": ["null", "number"] - }, - "viralPostClickRegistrations": { - "type": ["null", "number"] - }, - "viralPostViewJobApplications": { - "type": ["null", "number"] - }, - "viralPostViewJobApplyClicks": { - "type": ["null", "number"] - }, - "viralPostViewRegistrations": { - "type": ["null", "number"] - }, - "viralReactions": { - "type": ["null", "number"] - }, - "viralRegistrations": { - "type": ["null", "number"] - }, - "viralShares": { - "type": ["null", "number"] - }, - "viralTotalEngagements": { - "type": ["null", "number"] - }, - "viralVideoCompletions": { - "type": ["null", "number"] - }, - "viralVideoFirstQuartileCompletions": { - "type": ["null", "number"] - }, - "viralVideoMidpointCompletions": { - "type": ["null", "number"] - }, - "viralVideoStarts": { - "type": ["null", "number"] - }, - "viralVideoThirdQuartileCompletions": { - "type": ["null", "number"] - }, - "viralVideoViews": { - "type": ["null", "number"] - } - } -} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json index 42c95aadf622..7e0380a14b08 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaign_groups.json @@ -29,6 +29,17 @@ "test": { "type": ["null", "boolean"] }, + "totalBudget": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + }, "servingStatuses": { "type": ["null", "array"], "items": { @@ -46,6 +57,12 @@ }, "status": { "type": ["null", "string"] + }, + "allowedCampaignTypes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json index 8b2bacd997b8..a149e7dc4e37 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/campaigns.json @@ -120,6 +120,17 @@ } } }, + "totalBudget": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + }, "unitCost": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/conversions.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/conversions.json new file mode 100644 index 000000000000..d46ad7129950 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/conversions.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "title": "Conversions", + "properties": { + "attributionType": { + "type": ["null", "string"] + }, + "account": { + "type": ["null", "string"] + }, + "campaigns": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "created": { + "type": ["null", "integer"] + }, + "enabled": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "imagePixelTag": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "latestFirstPartyCallbackAt": { + "type": ["null", "integer"] + }, + "postClickAttributionWindowSize": { + "type": ["null", "integer"] + }, + "viewThroughAttributionWindowSize": { + "type": ["null", "integer"] + }, + "lastCallbackAt": { + "type": ["null", "integer"] + }, + "lastModified": { + "type": ["null", "integer"] + }, + "value": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + } + }, + "associatedCampaigns": { + "type": ["null", "array"] + }, + "urlMatchRuleExpression": { + "type": ["null", "array"] + }, + "urlRules": { + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json index adb0ba0c20b0..c310b738652a 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/schemas/creatives.json @@ -27,7 +27,15 @@ "type": ["null", "string"] }, "review": { - "type": ["null", "object"] + "type": ["null", "object"], + "properties": { + "status": { + "type": ["null", "string"] + }, + "rejectionReasons": { + "type": ["null", "array"] + } + } }, "isServing": { "type": ["null", "boolean"] @@ -43,6 +51,17 @@ }, "account": { "type": ["null", "string"] + }, + "leadgenCallToAction": { + "type": ["null", "object"], + "properties": { + "destination": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 6892afbaf9b3..8a89a468eaab 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -6,54 +6,34 @@ import logging from typing import Any, List, Mapping, Optional, Tuple, Union -import backoff -import requests -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType from source_linkedin_ads.streams import ( Accounts, AccountUsers, AdCampaignAnalytics, AdCreativeAnalytics, + AdImpressionDeviceAnalytics, + AdMemberCompanyAnalytics, + AdMemberCompanySizeAnalytics, + AdMemberCountryAnalytics, + AdMemberIndustryAnalytics, + AdMemberJobFunctionAnalytics, + AdMemberJobTitleAnalytics, + AdMemberRegionAnalytics, + AdMemberSeniorityAnalytics, CampaignGroups, Campaigns, + Conversions, Creatives, ) logger = logging.getLogger("airbyte") -class LinkedinAdsOAuth2Authenticator(Oauth2Authenticator): - @backoff.on_exception( - backoff.expo, - DefaultBackoffException, - on_backoff=lambda details: logger.info( - f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." - ), - max_time=300, - ) - def refresh_access_token(self) -> Tuple[str, int]: - try: - response = requests.request( - method="POST", - url=self.token_refresh_endpoint, - data=self.get_refresh_request_body(), - headers=self.get_refresh_access_token_headers(), - ) - response.raise_for_status() - response_json = response.json() - return response_json["access_token"], response_json["expires_in"] - except requests.exceptions.RequestException as e: - if e.response.status_code == 429 or e.response.status_code >= 500: - raise DefaultBackoffException(request=e.response.request, response=e.response) - raise - except Exception as e: - raise Exception(f"Error while refreshing access token: {e}") from e - - class SourceLinkedinAds(AbstractSource): """ Abstract Source inheritance, provides: @@ -62,7 +42,7 @@ class SourceLinkedinAds(AbstractSource): """ @classmethod - def get_authenticator(cls, config: Mapping[str, Any]) -> Union[TokenAuthenticator, LinkedinAdsOAuth2Authenticator]: + def get_authenticator(cls, config: Mapping[str, Any]) -> Union[TokenAuthenticator, Oauth2Authenticator]: """ Validate input parameters and generate a necessary Authentication object This connectors support 2 auth methods: @@ -76,7 +56,7 @@ def get_authenticator(cls, config: Mapping[str, Any]) -> Union[TokenAuthenticato access_token = config["credentials"]["access_token"] if auth_method else config["access_token"] return TokenAuthenticator(token=access_token) elif auth_method == "oAuth2.0": - return LinkedinAdsOAuth2Authenticator( + return Oauth2Authenticator( token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", client_id=config["credentials"]["client_id"], client_secret=config["credentials"]["client_secret"], @@ -90,14 +70,11 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> :: for this check method the Customer must have the "r_liteprofile" scope enabled. :: more info: https://docs.microsoft.com/linkedin/consumer/integrations/self-serve/sign-in-with-linkedin """ - + self._validate_ad_analytics_reports(config) config["authenticator"] = self.get_authenticator(config) stream = Accounts(config) - # need to load the first item only - stream.records_limit = 1 try: - next(stream.read_records(sync_mode=SyncMode.full_refresh), None) - return True, None + return stream.check_availability(logger) except Exception as e: return False, e @@ -106,13 +83,47 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Mapping a input config of the user input configuration as defined in the connector spec. Passing config to the streams. """ + self._validate_ad_analytics_reports(config) config["authenticator"] = self.get_authenticator(config) - return [ + streams = [ Accounts(config), AccountUsers(config), - AdCampaignAnalytics(config), - AdCreativeAnalytics(config), - CampaignGroups(config), - Campaigns(config), - Creatives(config), + AdCampaignAnalytics(config=config), + AdCreativeAnalytics(config=config), + AdImpressionDeviceAnalytics(config=config), + AdMemberCompanySizeAnalytics(config=config), + AdMemberCountryAnalytics(config=config), + AdMemberJobFunctionAnalytics(config=config), + AdMemberJobTitleAnalytics(config=config), + AdMemberIndustryAnalytics(config=config), + AdMemberSeniorityAnalytics(config=config), + AdMemberRegionAnalytics(config=config), + AdMemberCompanyAnalytics(config=config), + CampaignGroups(config=config), + Campaigns(config=config), + Creatives(config=config), + Conversions(config=config), ] + + return streams + self.get_custom_ad_analytics_reports(config) + + def get_custom_ad_analytics_reports(self, config: Mapping[str, Any]) -> List[Stream]: + streams = [] + + for ad_report in config.get("ad_analytics_reports", []): + stream = AdCampaignAnalytics( + name=f"Custom{ad_report.get('name')}", + pivot_by=ad_report.get("pivot_by"), + time_granularity=ad_report.get("time_granularity"), + config=config, + ) + streams.append(stream) + + return streams + + def _validate_ad_analytics_reports(self, config: Mapping[str, Any]) -> None: + report_names = [x["name"] for x in config.get("ad_analytics_reports", [])] + if len(report_names) != len(set(report_names)): + report_names = [x["name"] for x in config.get("ad_analytics_reports")] + message = f"Stream names for Custom Ad Analytics reports should be unique, duplicated streams: {set(name for name in report_names if report_names.count(name) > 1)}" + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json index 877902499ed0..e130c0a073de 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json @@ -23,25 +23,25 @@ "client_id": { "type": "string", "title": "Client ID", - "description": "The client ID of the LinkedIn Ads developer application.", + "description": "The client ID of your developer application. Refer to our documentation for more information.", "airbyte_secret": true }, "client_secret": { "type": "string", - "title": "Client secret", - "description": "The client secret the LinkedIn Ads developer application.", + "title": "Client Secret", + "description": "The client secret of your developer application. Refer to our documentation for more information.", "airbyte_secret": true }, "refresh_token": { "type": "string", - "title": "Refresh token", - "description": "The key to refresh the expired access token.", + "title": "Refresh Token", + "description": "The key to refresh the expired access token. Refer to our documentation for more information.", "airbyte_secret": true } } }, { - "title": "Access token", + "title": "Access Token", "type": "object", "required": ["access_token"], "properties": { @@ -51,8 +51,8 @@ }, "access_token": { "type": "string", - "title": "Access token", - "description": "The token value generated using the authentication code. See the docs to obtain yours.", + "title": "Access Token", + "description": "The access token generated for your developer application. Refer to our documentation for more information.", "airbyte_secret": true } } @@ -61,19 +61,73 @@ }, "start_date": { "type": "string", - "title": "Start date", + "title": "Start Date", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "description": "UTC date in the format 2020-09-17. Any data before this date will not be replicated.", + "pattern_descriptor": "YYYY-MM-DD", + "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated.", "examples": ["2021-05-17"], "format": "date" }, "account_ids": { "title": "Account IDs", "type": "array", - "description": "Specify the account IDs separated by a space, to pull the data from. Leave empty, if you want to pull the data from all associated accounts. See the LinkedIn Ads docs for more info.", + "description": "Specify the account IDs to pull data from, separated by a space. Leave this field empty if you want to pull the data from all accounts accessible by the authenticated user. See the LinkedIn docs to locate these IDs.", "items": { "type": "integer" }, + "examples": ["123456789"], + "default": [] + }, + "ad_analytics_reports": { + "title": "Custom Ad Analytics Reports", + "type": "array", + "items": { + "type": "object", + "title": "Ad Analytics Report Configuration", + "description": "Config for custom ad Analytics Report", + "required": ["name", "pivot_by", "time_granularity"], + "properties": { + "name": { + "title": "Report Name", + "description": "The name for the custom report.", + "type": "string" + }, + "pivot_by": { + "title": "Pivot Category", + "description": "Choose a category to pivot your analytics report around. This selection will organize your data based on the chosen attribute, allowing you to analyze trends and performance from different perspectives.", + "type": "string", + "enum": [ + "COMPANY", + "ACCOUNT", + "SHARE", + "CAMPAIGN", + "CREATIVE", + "CAMPAIGN_GROUP", + "CONVERSION", + "CONVERSATION_NODE", + "CONVERSATION_NODE_OPTION_INDEX", + "SERVING_LOCATION", + "CARD_INDEX", + "MEMBER_COMPANY_SIZE", + "MEMBER_INDUSTRY", + "MEMBER_SENIORITY", + "MEMBER_JOB_TITLE ", + "MEMBER_JOB_FUNCTION ", + "MEMBER_COUNTRY_V2 ", + "MEMBER_REGION_V2", + "MEMBER_COMPANY", + "PLACEMENT_NAME", + "IMPRESSION_DEVICE_TYPE" + ] + }, + "time_granularity": { + "title": "Time Granularity", + "description": "Choose how to group the data in your report by time. The options are:
    - 'ALL': A single result summarizing the entire time range.
    - 'DAILY': Group results by each day.
    - 'MONTHLY': Group results by each month.
    - 'YEARLY': Group results by each year.
    Selecting a time grouping helps you analyze trends and patterns over different time periods.", + "type": "string", + "enum": ["ALL", "DAILY", "MONTHLY", "YEARLY"] + } + } + }, "default": [] } } diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index 78b24e09ea4c..08b0418ab808 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -10,7 +10,10 @@ import pendulum import requests +from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.utils import casing +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from .analytics import make_analytics_slices, merge_chunks, update_analytics_params @@ -35,11 +38,19 @@ class LinkedinAdsStream(HttpStream, ABC): def __init__(self, config: Dict): super().__init__(authenticator=config.get("authenticator")) self.config = config + self.date_time_fields = self._get_date_time_items_from_schema() + + def _get_date_time_items_from_schema(self): + """ + Get all properties from schema with format: 'date-time' + """ + schema = self.get_json_schema() + return [k for k, v in schema["properties"].items() if v.get("format") == "date-time"] @property def accounts(self): """Property to return the list of the user Account Ids from input""" - return ",".join(map(str, self.config.get("account_ids"))) + return ",".join(map(str, self.config.get("account_ids", []))) def path( self, @@ -83,7 +94,17 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp """ We need to get out the nested complex data structures for further normalisation, so the transform_data method is applied. """ - yield from transform_data(response.json().get("elements")) + for record in transform_data(response.json().get("elements")): + yield self._date_time_to_rfc3339(record) + + def _date_time_to_rfc3339(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Transform 'date-time' items to RFC3339 format + """ + for item in record: + if item in self.date_time_fields and record[item]: + record[item] = pendulum.parse(record[item]).to_rfc3339_string() + return record def should_retry(self, response: requests.Response) -> bool: if response.status_code == 429: @@ -154,7 +175,7 @@ def parent_stream(self) -> object: @property def state_checkpoint_interval(self) -> Optional[int]: """Define the checkpoint from the records output size.""" - return super().records_limit + return 100 def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: current_stream_state = {self.cursor_field: self.config.get("start_date")} if not current_stream_state else current_stream_state @@ -172,7 +193,6 @@ class LinkedInAdsStreamSlicing(IncrementalLinkedinAdsStream): parent_stream = Accounts parent_values_map = {"account_id": "id"} - # define default additional request params def filter_records_newer_than_state( self, stream_state: Mapping[str, Any] = None, records_slice: Iterable[Mapping[str, Any]] = None @@ -256,6 +276,7 @@ class Campaigns(LinkedInAdsStreamSlicing): """ endpoint = "adCampaigns" + use_cache = True def path( self, @@ -332,6 +353,43 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_record.get(self.cursor_field), int(current_stream_state.get(self.cursor_field)))} +class Conversions(LinkedInAdsStreamSlicing): + """ + Get Conversions data using `account_id` slicing. + https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversion-tracking?view=li-lms-2023-05&tabs=curl#find-conversions-by-ad-account + """ + + endpoint = "conversions" + search_param = "account" + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + headers = super().request_headers(stream_state, stream_slice, next_page_token) + headers.update({"X-Restli-Protocol-Version": "2.0.0"}) + return headers + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params["q"] = self.search_param + params["account"] = f"urn%3Ali%3AsponsoredAccount%3A{stream_slice.get('account_id')}" + + return urlencode(params, safe="():,%") + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = ( + {self.cursor_field: pendulum.parse(self.config.get("start_date")).format("x")} + if not current_stream_state + else current_stream_state + ) + return {self.cursor_field: max(latest_record.get(self.cursor_field), int(current_stream_state.get(self.cursor_field)))} + + class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC): """ AdAnalytics Streams more info: @@ -343,10 +401,27 @@ class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC): primary_key = ["pivotValue", "end_date"] cursor_field = "end_date" + def get_json_schema(self) -> Mapping[str, Any]: + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics") + + def __init__(self, name: str = None, pivot_by: str = None, time_granularity: str = None, **kwargs): + self.user_stream_name = name + if pivot_by: + self.pivot_by = pivot_by + if time_granularity: + self.time_granularity = time_granularity + super().__init__(**kwargs) + + @property + def name(self) -> str: + """We override stream name to let the user change it via configuration.""" + name = self.user_stream_name or self.__class__.__name__ + return casing.camel_to_snake(name) + @property def base_analytics_params(self) -> MutableMapping[str, Any]: """Define the base parameters for analytics streams""" - return {"q": "analytics", "pivot": self.pivot_by, "timeGranularity": "(value:DAILY)"} + return {"q": "analytics", "pivot": f"(value:{self.pivot_by})", "timeGranularity": f"(value:{self.time_granularity})"} def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -401,7 +476,8 @@ class AdCampaignAnalytics(LinkedInAdsAnalyticsStream): parent_values_map = {"campaign_id": "id"} search_param = "campaigns" search_param_value = "sponsoredCampaign" - pivot_by = "(value:CAMPAIGN)" + pivot_by = "CAMPAIGN" + time_granularity = "DAILY" class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): @@ -413,8 +489,45 @@ class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): parent_values_map = {"creative_id": "id"} search_param = "creatives" search_param_value = "sponsoredCreative" - pivot_by = "(value:CREATIVE)" + pivot_by = "CREATIVE" + time_granularity = "DAILY" def get_primary_key_from_slice(self, stream_slice) -> str: creative_id = stream_slice.get(self.primary_slice_key).split(":")[-1] return creative_id + + +class AdImpressionDeviceAnalytics(AdCampaignAnalytics): + pivot_by = "IMPRESSION_DEVICE_TYPE" + + +class AdMemberCompanySizeAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COMPANY_SIZE" + + +class AdMemberIndustryAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_INDUSTRY" + + +class AdMemberSeniorityAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_SENIORITY" + + +class AdMemberJobTitleAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_JOB_TITLE" + + +class AdMemberJobFunctionAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_JOB_FUNCTION" + + +class AdMemberCountryAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COUNTRY_V2" + + +class AdMemberRegionAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_REGION_V2" + + +class AdMemberCompanyAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COMPANY" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py index 3d13ceb8b14b..f000f88f343a 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -38,7 +38,6 @@ def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: def transform_change_audit_stamps( record: Dict, dict_key: str = "changeAuditStamps", props: List = ["created", "lastModified"], fields: List = ["time"] ) -> Mapping[str, Any]: - """ :: EXAMPLE `changeAuditStamps` input structure: { @@ -96,7 +95,6 @@ def transform_date_range( props: List = ["start", "end"], fields: List = ["year", "month", "day"], ) -> Mapping[str, Any]: - """ :: EXAMPLE `dateRange` input structure in Analytics streams: { @@ -320,7 +318,6 @@ def transform_data(records: List) -> Iterable[Mapping]: to be properly normalised in the destination. """ for record in records: - if "changeAuditStamps" in record: record = transform_change_audit_stamps(record) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py index f7806b19ba5d..adda86e5f619 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py @@ -3,12 +3,10 @@ # -from unittest.mock import patch - import pytest import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.utils import AirbyteTracedException from source_linkedin_ads.source import ( Accounts, AccountUsers, @@ -17,7 +15,6 @@ CampaignGroups, Campaigns, Creatives, - LinkedinAdsOAuth2Authenticator, SourceLinkedinAds, ) from source_linkedin_ads.streams import LINKEDIN_VERSION_API @@ -43,6 +40,29 @@ }, } +TEST_CONFIG_DUPLICATE_CUSTOM_AD_ANALYTICS_REPORTS: dict = { + "start_date": "2021-01-01", + "account_ids": [], + "credentials": { + "auth_method": "oAuth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "ad_analytics_reports": [ + { + "name": "ShareAdByMonth", + "pivot_by": "COMPANY", + "time_granularity": "MONTHLY" + }, + { + "name": "ShareAdByMonth", + "pivot_by": "COMPANY", + "time_granularity": "MONTHLY" + } + ] +} + class TestAllStreams: _instance: SourceLinkedinAds = SourceLinkedinAds() @@ -62,20 +82,6 @@ def test_get_authenticator(self, config: dict): test = self._instance.get_authenticator(config) assert isinstance(test, (Oauth2Authenticator, TokenAuthenticator)) - @pytest.mark.parametrize( - "response, check_passed", - [ - (iter({"id": 123}), True), - (requests.HTTPError(), False), - ], - ids=["Success", "Fail"], - ) - def test_check(self, response, check_passed): - with patch.object(Accounts, "read_records", return_value=response) as mock_method: - result = self._instance.check_connection(logger=AirbyteLogger, config=TEST_CONFIG) - mock_method.assert_called() - assert check_passed == result[0] - @pytest.mark.parametrize( "stream_cls", [ @@ -125,7 +131,7 @@ def test_streams(self, stream_cls): ], ) def test_path(self, stream_cls, stream_slice, expected): - stream = stream_cls(TEST_CONFIG) + stream = stream_cls(config=TEST_CONFIG) result = stream.path(stream_slice=stream_slice) assert result == expected @@ -175,7 +181,7 @@ class TestAccountUsers: stream: AccountUsers = AccountUsers(TEST_CONFIG) def test_state_checkpoint_interval(self): - assert self.stream.state_checkpoint_interval == 500 + assert self.stream.state_checkpoint_interval == 100 def test_get_updated_state(self): state = self.stream.get_updated_state( @@ -250,7 +256,7 @@ class TestLinkedInAdsAnalyticsStream: ], ) def test_base_analytics_params(self, stream_cls, expected): - stream = stream_cls(TEST_CONFIG) + stream = stream_cls(config=TEST_CONFIG) result = stream.base_analytics_params assert result == expected @@ -291,7 +297,7 @@ def test_base_analytics_params(self, stream_cls, expected): ], ) def test_request_params(self, stream_cls, slice, expected): - stream = stream_cls(TEST_CONFIG) + stream = stream_cls(config=TEST_CONFIG) result = stream.request_params(stream_state={}, stream_slice=slice) assert expected == result @@ -302,7 +308,7 @@ def test_retry_get_access_token(requests_mock): "https://www.linkedin.com/oauth/v2/accessToken", [{"status_code": 429}, {"status_code": 429}, {"status_code": 200, "json": {"access_token": "token", "expires_in": 3600}}], ) - auth = LinkedinAdsOAuth2Authenticator( + auth = Oauth2Authenticator( token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", client_id="client_id", client_secret="client_secret", @@ -311,3 +317,26 @@ def test_retry_get_access_token(requests_mock): token = auth.get_access_token() assert len(requests_mock.request_history) == 3 assert token == "token" + + +@pytest.mark.parametrize( + "record, expected", + [ + ({}, {}), + ({"lastModified": "2021-05-27 11:59:53.710000"}, {"lastModified": "2021-05-27T11:59:53.710000+00:00"}), + ({"lastModified": None}, {"lastModified": None}), + ({"lastModified": ""}, {"lastModified": ""}), + ], + ids=["empty_record", "transformed_record", "null_value", "empty_value"], +) +def test_date_time_to_rfc3339(record, expected): + stream = Accounts(TEST_CONFIG) + result = stream._date_time_to_rfc3339(record) + assert result == expected + + +def test_duplicated_custom_ad_analytics_report(): + with pytest.raises(AirbyteTracedException) as e: + SourceLinkedinAds().streams(TEST_CONFIG_DUPLICATE_CUSTOM_AD_ANALYTICS_REPORTS) + expected_message = "Stream names for Custom Ad Analytics reports should be unique, duplicated streams: {'ShareAdByMonth'}" + assert e.value.message == expected_message diff --git a/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml index 3f4d5136134e..f2c1338be532 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-pages/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: af54297c-e8f8-4d63-a00d-a94695acc9d3 dockerImageTag: 1.0.2 dockerRepository: airbyte/source-linkedin-pages + documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-pages githubIssueLabel: source-linkedin-pages icon: linkedin.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-pages + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt +++ b/airbyte-integrations/connectors/source-linkedin-pages/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-linkedin-pages/setup.py b/airbyte-integrations/connectors/source-linkedin-pages/setup.py index 4f15e598ba51..f690842da795 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-pages/setup.py @@ -11,9 +11,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-linnworks/metadata.yaml b/airbyte-integrations/connectors/source-linnworks/metadata.yaml index a764fd84eaa1..326dbedaa270 100644 --- a/airbyte-integrations/connectors/source-linnworks/metadata.yaml +++ b/airbyte-integrations/connectors/source-linnworks/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/linnworks tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-linnworks/requirements.txt b/airbyte-integrations/connectors/source-linnworks/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-linnworks/requirements.txt +++ b/airbyte-integrations/connectors/source-linnworks/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-linnworks/setup.py b/airbyte-integrations/connectors/source-linnworks/setup.py index a9bda180884b..f0a487191567 100644 --- a/airbyte-integrations/connectors/source-linnworks/setup.py +++ b/airbyte-integrations/connectors/source-linnworks/setup.py @@ -13,7 +13,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock~=1.9.3", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-lokalise/metadata.yaml b/airbyte-integrations/connectors/source-lokalise/metadata.yaml index 5605e2ef7527..079f05bcb4f6 100644 --- a/airbyte-integrations/connectors/source-lokalise/metadata.yaml +++ b/airbyte-integrations/connectors/source-lokalise/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-lokalise/requirements.txt b/airbyte-integrations/connectors/source-lokalise/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-lokalise/requirements.txt +++ b/airbyte-integrations/connectors/source-lokalise/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-lokalise/setup.py b/airbyte-integrations/connectors/source-lokalise/setup.py index c2055a99946d..69d13c0b79f0 100644 --- a/airbyte-integrations/connectors/source-lokalise/setup.py +++ b/airbyte-integrations/connectors/source-lokalise/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-looker/metadata.yaml b/airbyte-integrations/connectors/source-looker/metadata.yaml index ec25a424201c..10e3a4cfe079 100644 --- a/airbyte-integrations/connectors/source-looker/metadata.yaml +++ b/airbyte-integrations/connectors/source-looker/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/looker tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-looker/requirements.txt b/airbyte-integrations/connectors/source-looker/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-looker/requirements.txt +++ b/airbyte-integrations/connectors/source-looker/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index 7cac3ecbeaa6..f38fc2a1095e 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -13,9 +13,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", ] diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 33cfff16b5be..c99eeee394a8 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailchimp/requirements.txt b/airbyte-integrations/connectors/source-mailchimp/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-mailchimp/requirements.txt +++ b/airbyte-integrations/connectors/source-mailchimp/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py index 15d8343ca9c5..f2973669a61e 100644 --- a/airbyte-integrations/connectors/source-mailchimp/setup.py +++ b/airbyte-integrations/connectors/source-mailchimp/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "responses~=0.19.0", "requests-mock~=1.9.3"] +TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "responses~=0.19.0", "requests-mock~=1.9.3"] setup( diff --git a/airbyte-integrations/connectors/source-mailerlite/metadata.yaml b/airbyte-integrations/connectors/source-mailerlite/metadata.yaml index 939048c97d24..fae3ac67fb61 100644 --- a/airbyte-integrations/connectors/source-mailerlite/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailerlite/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailerlite/requirements.txt b/airbyte-integrations/connectors/source-mailerlite/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-mailerlite/requirements.txt +++ b/airbyte-integrations/connectors/source-mailerlite/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-mailerlite/setup.py b/airbyte-integrations/connectors/source-mailerlite/setup.py index 63f23091bbed..5b49cb1a5d31 100644 --- a/airbyte-integrations/connectors/source-mailerlite/setup.py +++ b/airbyte-integrations/connectors/source-mailerlite/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-mailersend/metadata.yaml b/airbyte-integrations/connectors/source-mailersend/metadata.yaml index 5cf2c441569a..436e6a8514ed 100644 --- a/airbyte-integrations/connectors/source-mailersend/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailersend/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailersend/requirements.txt b/airbyte-integrations/connectors/source-mailersend/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-mailersend/requirements.txt +++ b/airbyte-integrations/connectors/source-mailersend/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-mailersend/setup.py b/airbyte-integrations/connectors/source-mailersend/setup.py index ffe2256b03bb..8fc9c7d2bdbc 100644 --- a/airbyte-integrations/connectors/source-mailersend/setup.py +++ b/airbyte-integrations/connectors/source-mailersend/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-mailgun/Dockerfile b/airbyte-integrations/connectors/source-mailgun/Dockerfile index dd36d7ee4fa9..0f9bc9f710fd 100644 --- a/airbyte-integrations/connectors/source-mailgun/Dockerfile +++ b/airbyte-integrations/connectors/source-mailgun/Dockerfile @@ -34,5 +34,5 @@ COPY source_mailgun ./source_mailgun ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-mailgun diff --git a/airbyte-integrations/connectors/source-mailgun/README.md b/airbyte-integrations/connectors/source-mailgun/README.md index a49614c72d35..c3198f50139a 100644 --- a/airbyte-integrations/connectors/source-mailgun/README.md +++ b/airbyte-integrations/connectors/source-mailgun/README.md @@ -1,35 +1,10 @@ # Mailgun Source -This is the repository for the Mailgun source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/mailgun). +This is the repository for the Mailgun configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/mailgun). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -39,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailgun) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailgun/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/mailgun) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailgun/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source mailgun test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -79,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailgun:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailgun:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-mailgun/__init__.py b/airbyte-integrations/connectors/source-mailgun/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailgun/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml index 354740b32bbf..3d87f163752c 100644 --- a/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml @@ -1,24 +1,41 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-mailgun:dev -tests: +acceptance_tests: spec: - - spec_path: "source_mailgun/spec.json" + tests: + - spec_path: "source_mailgun/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 0.1.1 connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 0.1.1 basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: events + bypass_reason: "Sandbox account can't seed this stream" +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailgun/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-mailgun/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py index 9e7ab9483d5f..d49b55882333 100644 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-mailgun/integration_tests/acceptance.py @@ -2,59 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime -import json -import os.path -from typing import Any, Dict, Iterable import pytest -from .fill_data import DataFiller - pytest_plugins = ("connector_acceptance_test.plugin",) -class Config: - - messages_per_sender: int = 5 - min_update_interval: int = datetime.timedelta(2).total_seconds() # 2 days - - recipients: Iterable[str] = [ - "integration-test@airbyte.io", - ] - senders: Iterable[str] = [ - "Ann", - "Bob", - "Carl", - ] - - test_data_file_path: str = os.path.join(os.path.dirname(__file__), "test_data.json") - - @classmethod - def get_test_data(cls) -> Dict[str, Any]: - with open(cls.test_data_file_path) as tdf: - return json.load(tdf) - - @classmethod - def update_timestamp(cls, timestamp: float): - test_data = cls.get_test_data() - test_data.update({"last_update_timestamp": timestamp}) - with open(cls.test_data_file_path, "w") as tdf: - json.dump(test_data, tdf) - - @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ - Fill the sandbox with fresh data by sending simple text-only messages. - """ - config = Config - now = datetime.datetime.now().timestamp() - if config.get_test_data()["last_update_timestamp"] + config.min_update_interval <= now: - data_filler: DataFiller = DataFiller( - from_emails=config.senders, to=config.recipients, number_of_messages=config.messages_per_sender - ) - data_filler.fill() - config.update_timestamp(now) - yield diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/fill_data.py b/airbyte-integrations/connectors/source-mailgun/integration_tests/fill_data.py deleted file mode 100644 index f16eaef7702c..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/fill_data.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import argparse -import concurrent.futures -import json -import logging -import os -from typing import Dict, List, Sequence, Tuple, Union -from urllib.parse import urljoin - -import requests as requests - -# TODO: unit-tests -# TODO: combine URL according to a region from config -API_URL = "https://api.mailgun.net/v3/" - - -# TODO: add more fields for sending messages: cc, bcc, o:*, h:*, v:*, etc. - - -class DataFiller: - def __init__(self, from_emails, to, number_of_messages, logger=None, **kwargs): - config_file = kwargs.get("config_file", os.path.join(os.path.dirname(__file__), os.pardir, "secrets", "config.json")) - with open(config_file) as fp: - config = json.load(fp) - self.api_key = config["private_key"] - self.domains = self.get_domains() - self.senders = from_emails - self.recipients = to - self.number = number_of_messages - - self.logger = logger or logging.getLogger("Data Filler") - - def fill(self): - self.send_messages(self.domains, self.senders, self.recipients) - - def get_domains(self) -> Tuple[str]: - domains_url = urljoin(API_URL, "domains") - response = requests.get(domains_url, auth=("api", self.api_key)) - domains: Tuple[str] = tuple(item["name"] for item in response.json()["items"]) # type: ignore[assignment] - return domains - - def post_request(self, url, data): - return requests.post(url, auth=("api", self.api_key), data=data) - - def send_messages(self, domains: Sequence[str], senders: Sequence[str], recipients: Sequence[str]): - with concurrent.futures.ThreadPoolExecutor(max_workers=len(domains) * len(senders) * self.number) as executor: - future_responses = {} - for domain in domains: - url: str = urljoin(urljoin(API_URL, f"{domain}/"), "messages") - for sender in senders: - for i in range(self.number): - data: Dict[str, Union[str, List[str]]] = { - "from": f"{sender} <{sender}@{domain}>", - "to": recipients, - "subject": f"Test message #{i} from {sender}", - "text": f"Hello! It is an amazing test message #{i}!", # TODO: create full messages - } - future_responses[executor.submit(self.post_request, url, data)] = (sender, recipients) - for future in concurrent.futures.as_completed(future_responses): - sender, recipients = future_responses[future] - response = future.result() - if response.ok: - self.logger.info(f"Message #{i} from {sender} to {recipients} was successfully sent") - else: - self.logger.warning(response.reason) # TODO: Make more informative output - - -if __name__ == "__main__": - parser = argparse.ArgumentParser("Populate test data for Mailgun sandbox") - - parser.add_argument("-f", "--from-emails", action="append", help="List of senders' names", required=True) - parser.add_argument( - "-t", - "--to", - action="append", - help="List of recipients (must be defined as 'Authorized recipients' in the Mailgun sandbox", - required=True, - ) - parser.add_argument( - "-n", "--number-of-messages", type=int, default=5, help="Number of messages to send for every sender-recipient pair" - ) - - args = parser.parse_args() - - logging.basicConfig(level=logging.INFO) - - data_filler = DataFiller(**vars(args)) - data_filler.fill() diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_config.json index e1a434006576..84757bb38760 100644 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_config.json @@ -1,3 +1,5 @@ { - "private_key": "test-api-key" + "private_key": "xxx17ec8cf955c450b20d02454d04xxxxxxx34xxxxxxx9dxxx", + "domain_region": "us", + "start_date": "2023-08-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-mailgun/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/state.json b/airbyte-integrations/connectors/source-mailgun/integration_tests/state.json deleted file mode 100644 index d91c02e18ada..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "events": { - "timestamp": 1636236000.0 - } -} diff --git a/airbyte-integrations/connectors/source-mailgun/integration_tests/test_data.json b/airbyte-integrations/connectors/source-mailgun/integration_tests/test_data.json deleted file mode 100644 index a3909e35484d..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/integration_tests/test_data.json +++ /dev/null @@ -1 +0,0 @@ -{"last_update_timestamp": 1677763543.5336} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mailgun/metadata.yaml b/airbyte-integrations/connectors/source-mailgun/metadata.yaml index 2bc605a8f83f..e83c02b1cee6 100644 --- a/airbyte-integrations/connectors/source-mailgun/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailgun/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - https://api.mailgun.net/ + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 5b9cb09e-1003-4f9c-983d-5779d1b2cd51 - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-mailgun githubIssueLabel: source-mailgun icon: mailgun.svg license: MIT name: Mailgun - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-10 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/mailgun tags: - - language:python + - language:low-code + ab_internal: + sl: 200 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailgun/setup.py b/airbyte-integrations/connectors/source-mailgun/setup.py index afb3d6bb08bf..3ede43d33d0b 100644 --- a/airbyte-integrations/connectors/source-mailgun/setup.py +++ b/airbyte-integrations/connectors/source-mailgun/setup.py @@ -6,13 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "responses~=0.16.0", "connector-acceptance-test", ] @@ -23,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/__init__.py b/airbyte-integrations/connectors/source-mailgun/source_mailgun/__init__.py index de9a71510677..333333942b12 100644 --- a/airbyte-integrations/connectors/source-mailgun/source_mailgun/__init__.py +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml b/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml new file mode 100644 index 000000000000..92ef0a7abfed --- /dev/null +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml @@ -0,0 +1,95 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] + + requester: + type: HttpRequester + url_base: | + {{ + 'https://api.mailgun.net/v3' + if config['domain_region'] == 'US' + else 'https://api.eu.mailgun.net/v3' + }} + http_method: "GET" + authenticator: + type: "BasicHttpAuthenticator" + username: "api" + password: "{{ config['private_key'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['paging', 'next'] }}" + page_token_option: + type: "RequestPath" + field_name: "from" + inject_into: "url_base" + + incremental_sync_base: + type: DatetimeBasedCursor + cursor_field: "{{ parameters.incremental_cursor }}" + datetime_format: "%s" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + + domains_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "domains" + primary_key: "id" + path: "/domains" + retriever: + $ref: "#/definitions/retriever" + paginator: + $ref: "#/definitions/base_paginator" + + events_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "events" + primary_key: "id" + path: "/events" + incremental_cursor: "timestamp" + retriever: + $ref: "#/definitions/retriever" + paginator: + $ref: "#/definitions/base_paginator" + incremental_sync: + $ref: "#/definitions/incremental_sync_base" + +streams: + - "#/definitions/domains_stream" + - "#/definitions/events_stream" + +check: + type: CheckStream + stream_names: + - "domains" + - "events" diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/domains.json b/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/domains.json index c5e92c09ce2f..336fd5d815ae 100644 --- a/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/domains.json +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/domains.json @@ -1,11 +1,13 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "dkim_key_size": { - "type": "integer" + "type": ["null", "integer"] }, "force_dkim_authority": { - "type": "boolean" + "type": ["null", "boolean"] }, "ips": { "type": ["null", "array"], @@ -14,7 +16,7 @@ } }, "name": { - "type": "string" + "type": ["null", "string"] }, "pool": { "type": ["null", "string"] @@ -26,10 +28,37 @@ "type": ["null", "string"] }, "web_scheme": { - "type": "string" + "type": ["null", "string"] }, "wildcard": { - "type": "boolean" + "type": ["null", "boolean"] + }, + "state": { + "type": ["null", "string"] + }, + "skip_verification": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "require_tls": { + "type": ["null", "boolean"] + }, + "is_disabled": { + "type": ["null", "boolean"] + }, + "smtp_login": { + "type": ["null", "string"] + }, + "web_prefix": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/events.json b/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/events.json index c3f3305fb200..13242085b486 100644 --- a/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/events.json +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/schemas/events.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "campaigns": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/source.py b/airbyte-integrations/connectors/source-mailgun/source_mailgun/source.py index 49bcfc202998..da49a6bc6984 100644 --- a/airbyte-integrations/connectors/source-mailgun/source_mailgun/source.py +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/source.py @@ -2,202 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime -import json -import logging -import time -from abc import ABC -from numbers import Number -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import urljoin +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from pydantic import HttpUrl -from requests.auth import HTTPBasicAuth +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -DOMAIN_BY_REGION = {"EU": "https://api.eu.mailgun.net/", "US": "https://api.mailgun.net/"} +WARNING: Do not modify this file. +""" -class MailgunStream(HttpStream, ABC): - """ - Base class for Mailgun streams. - Provides common streams' functionality. - """ - - primary_key: str = None - - def __init__(self, config: Mapping[str, Any], *args, **kwargs): - super().__init__(*args, **kwargs) - region = config.get("domain_region", "US") - self._url_base: HttpUrl = urljoin(DOMAIN_BY_REGION[region], "v3/") - - @property - def url_base(self) -> HttpUrl: - return self._url_base - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, HttpUrl]]: - """ - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - next_page: Optional[HttpUrl] = response.json().get("paging", {}).get("next") - return {"url": next_page} if next_page and self._pre_parse_response(response) else None - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Optional[Mapping[str, HttpUrl]] = None, - ) -> str: - return next_page_token["url"] if next_page_token else "" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - yield from self._pre_parse_response(response) - - @staticmethod - def _pre_parse_response(response: requests.Response) -> List: - return response.json()["items"] - - -class Domains(MailgunStream): - """ - "Domains" stream. - API reference is here: https://documentation.mailgun.com/en/latest/api-domains.html - """ - - primary_key: str = "name" - - def path(self, *args, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return super().path(*args, next_page_token=next_page_token, **kwargs) or "domains" - - -class IncrementalMailgunStream(MailgunStream, ABC): - """ - Base class for incremental Mailgun streams. - Provides common functionality for incremental streams. - """ - - # Messages are stored for 3 days, so it prevents occasional attempt to read from the start of the Epoch - default_shift: datetime.datetime = pendulum.duration(3) - - def __init__(self, config: Mapping[str, Any], *args, **kwargs): - super().__init__(*args, config=config, **kwargs) - - try: - if "start_date" in config: - start_date = pendulum.parse(config["start_date"]) - else: - start_date = pendulum.now() - self.default_shift - - except pendulum.parsing.exceptions.ParserError as e: - raise ValueError(f"Unrecognized date format. {e}") - - self.start_timestamp: Number = start_date.timestamp() - - @staticmethod - def chunk_timestamps_range(start_timestamp: Number, interval: Number = 60 * 60 * 24) -> Iterable[Tuple[Number]]: - """ - Yield a tuple of beginning and ending timestamps of each day between the start timestamp and end timestamp. - """ - end: Number = time.time() - if start_timestamp > end: - yield start_timestamp, start_timestamp - - while start_timestamp <= end: - end_timestamp = start_timestamp + interval - yield start_timestamp, end_timestamp - start_timestamp = end_timestamp - - -class Events(IncrementalMailgunStream): - """ - "Events" stream. - API reference is here: https://documentation.mailgun.com/en/latest/api-events.html - """ - - # TODO: Event Polling. See https://documentation.mailgun.com/en/latest/api-events.html#event-polling - - cursor_field: str = "timestamp" - - primary_key: str = "id" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_timestamp = latest_record.get(self.cursor_field, self.start_timestamp) - if current_stream_state and self.cursor_field in current_stream_state: - latest_timestamp = max(latest_timestamp, current_stream_state[self.cursor_field]) - - return {self.cursor_field: latest_timestamp} - - def path(self, *args, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return super().path(*args, next_page_token=next_page_token, **kwargs) or "events" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params: MutableMapping[str, Any] = super().request_params( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ) - params.update(stream_slice) - if stream_state: - params["begin"] = stream_state[self.cursor_field] - - # If "end" parameter is not provided, it's required to define a search direction. - # See https://documentation.mailgun.com/en/latest/api-events.html#time-range - if "end" not in params: - params["ascending"] = "yes" - return params - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, float]]]: - """ - Provide a generator of date_slices in such format: - {"begin": 1636411500.0, "end": 1636497900.0} - """ - stream_state = stream_state or {} - start_date = stream_state.get(self.cursor_field, self.start_timestamp) - for period in self.chunk_timestamps_range(start_date): - yield {"begin": period[0], "end": period[1]} - - -class SourceMailgun(AbstractSource): - def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - region = config.get("domain_region", "US") - try: - url = urljoin(DOMAIN_BY_REGION[region], "v3/domains") - except KeyError: - return False, f"'domain_region' has to be one of {list(DOMAIN_BY_REGION)} or to be omitted" - - response = requests.get(url, auth=("api", config["private_key"])) - if response.status_code == 200: - return True, None - else: - message = "Connection check failed. " - try: - message += response.json()["message"] - except json.JSONDecodeError: - message += f"Unexpected response format from the server. It returns:\n{response.text}" - finally: - return False, message - except requests.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = HTTPBasicAuth("api", config["private_key"]) - - return [Domains(config=config, authenticator=auth), Events(config=config, authenticator=auth)] +# Declarative Source +class SourceMailgun(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/spec.json b/airbyte-integrations/connectors/source-mailgun/source_mailgun/spec.json deleted file mode 100644 index 3b95758c2b6c..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/source_mailgun/spec.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/mailgun", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Mailgun Spec", - "type": "object", - "required": ["private_key"], - "additionalProperties": true, - "properties": { - "private_key": { - "type": "string", - "airbyte_secret": true, - "description": "Primary account API key to access your Mailgun data.", - "title": "Private API Key" - }, - "domain_region": { - "type": "string", - "description": "Domain region code. 'EU' or 'US' are possible values. The default is 'US'.", - "title": "Domain Region Code" - }, - "start_date": { - "title": "Replication Start Date", - "description": "UTC date and time in the format 2020-10-01 00:00:00. Any data before this date will not be replicated. If omitted, defaults to 3 days ago.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}$", - "examples": ["2020-10-01 00:00:00"], - "type": "string", - "format": "date-time" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/spec.yaml b/airbyte-integrations/connectors/source-mailgun/source_mailgun/spec.yaml new file mode 100644 index 000000000000..b7b6334f2652 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/spec.yaml @@ -0,0 +1,31 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/mailgun +connectionSpecification: + "$schema": http://json-schema.org/draft-07/schema# + title: Source Mailgun Spec + type: object + required: + - private_key + additionalProperties: true + properties: + private_key: + type: string + airbyte_secret: true + description: Primary account API key to access your Mailgun data. + title: Private API Key + domain_region: + type: string + description: + Domain region code. 'EU' or 'US' are possible values. The default + is 'US'. + title: Domain Region Code + default: "US" + start_date: + title: Replication Start Date + description: + UTC date and time in the format 2020-10-01 00:00:00. Any data before + this date will not be replicated. If omitted, defaults to 3 days ago. + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2023-08-01T00:00:00Z" + type: string + format: date-time diff --git a/airbyte-integrations/connectors/source-mailgun/unit_tests/__init__.py b/airbyte-integrations/connectors/source-mailgun/unit_tests/__init__.py deleted file mode 100644 index 5e0b70e2a192..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/unit_tests/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# -TEST_CONFIG = { - "private_key": "test_private_key", -} diff --git a/airbyte-integrations/connectors/source-mailgun/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mailgun/unit_tests/conftest.py deleted file mode 100644 index 0714ab06b39f..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/unit_tests/conftest.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from base64 import b64encode -from unittest.mock import MagicMock - -import pytest -import responses - -from . import TEST_CONFIG - - -@pytest.fixture -def test_config(): - return TEST_CONFIG.copy() - - -@pytest.fixture -def mocked_responses(): - with responses.RequestsMock() as r: - yield r - - -@pytest.fixture -def auth_header(test_config): - encoded_auth = b64encode(f"api:{test_config['private_key']}".encode()).decode() - return f"Basic {encoded_auth}" - - -@pytest.fixture -def next_page_url(): - return "next-page-url" - - -@pytest.fixture -def test_records(): - return [ - {"name": "Item 1"}, - {"name": "Item 2"}, - ] - - -@pytest.fixture -def normal_response(next_page_url, test_records): - response = MagicMock() - response.json = MagicMock( - return_value={ - "items": test_records, - "paging": { - "next": next_page_url, - }, - } - ) - - return response diff --git a/airbyte-integrations/connectors/source-mailgun/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-mailgun/unit_tests/test_incremental_streams.py deleted file mode 100644 index dad88e265d96..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import time - -import pytest as pytest -from airbyte_cdk.models import SyncMode -from source_mailgun.source import Domains, Events, IncrementalMailgunStream - -from . import TEST_CONFIG - - -@pytest.mark.parametrize( - "stream, cursor_field", - [ - (IncrementalMailgunStream(TEST_CONFIG), []), - (Domains(TEST_CONFIG), []), - (Events(TEST_CONFIG), "timestamp"), - ], -) -def test_cursor_field(stream, cursor_field): - assert stream.cursor_field == cursor_field - - -@pytest.mark.parametrize( - "stream, current_stream_state, latest_record, expected_state", - [ - (IncrementalMailgunStream(TEST_CONFIG), None, None, {}), - (Events(TEST_CONFIG), {"timestamp": 1000}, {"timestamp": 2000}, {"timestamp": 2000}), - (Events(TEST_CONFIG), {"timestamp": 2000}, {"timestamp": 1000}, {"timestamp": 2000}), - ], -) -def test_get_updated_state(stream, current_stream_state, latest_record, expected_state): - inputs = {"current_stream_state": current_stream_state, "latest_record": latest_record} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_get_updated_state_events_default_timestamp(): - state = Events(TEST_CONFIG).get_updated_state(current_stream_state={}, latest_record={}) - assert state["timestamp"] == pytest.approx(time.time(), 60 * 60) - - -@pytest.mark.parametrize( - "stream, inputs, expected_stream_slice", - [ - (IncrementalMailgunStream(TEST_CONFIG), {"sync_mode": SyncMode.incremental}, None), - (Events(TEST_CONFIG), {"stream_state": {"timestamp": 1000000}}, {"begin": 1000000, "end": 1086400}), - ], -) -def test_stream_slices(stream, inputs, expected_stream_slice): - slices = iter(stream.stream_slices(**inputs)) - assert next(slices) == expected_stream_slice - - -@pytest.mark.parametrize( - "stream, support_incremental", - [ - (Domains(TEST_CONFIG), False), - (Events(TEST_CONFIG), True), - ], -) -def test_supports_incremental(stream, support_incremental): - assert stream.supports_incremental == support_incremental - - -@pytest.mark.parametrize( - "stream, source_defined_cursor", - [ - (Domains(TEST_CONFIG), True), - (Events(TEST_CONFIG), True), - ], -) -def test_source_defined_cursor(stream, source_defined_cursor): - assert stream.source_defined_cursor == source_defined_cursor - - -@pytest.mark.parametrize( - "stream, state_checkpoint_interval", - [ - (Domains(TEST_CONFIG), None), - (Events(TEST_CONFIG), None), - ], -) -def test_stream_checkpoint_interval(stream, state_checkpoint_interval): - assert stream.state_checkpoint_interval == state_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-mailgun/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mailgun/unit_tests/test_source.py deleted file mode 100644 index be59f9c29766..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/unit_tests/test_source.py +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -from unittest.mock import MagicMock - -import pytest -import requests as requests -import responses -from source_mailgun.source import SourceMailgun - -from . import TEST_CONFIG - - -@pytest.fixture -def check_connection_url(): - return "https://api.mailgun.net/v3/domains" - - -@pytest.fixture -def source_mailgun(test_config): - source = SourceMailgun() - yield source - del source - - -@pytest.mark.parametrize( - "config", - [ - TEST_CONFIG, - dict(**TEST_CONFIG, **{"start_date": "2021-01-01T00:00:00Z"}), - dict(**TEST_CONFIG, **{"start_date": "2021-01-01T00:00:00Z", "end_date": "2021-12-31T23:59:59Z"}), - ], -) -def test_check_connection(mocked_responses, source_mailgun, auth_header, check_connection_url, config): - bad_config = config.copy() - bad_config["private_key"] = bad_config["private_key"][-1::-1] - bad_key_message = "Bad key message" - bad_key_body = json.dumps({"message": bad_key_message}) - - def request_callback(request): - if request.headers.get("Authorization") == auth_header: - return 200, {}, "" - else: - return 401, {}, bad_key_body - - mocked_responses.add_callback(responses.GET, check_connection_url, callback=request_callback, content_type="application/json") - - logger_mock = MagicMock() - - assert source_mailgun.check_connection(logger_mock, config) == (True, None) - - check_result = source_mailgun.check_connection(logger_mock, bad_config) - assert not check_result[0] - assert bad_key_message in check_result[1] - - -def test_check_connection_config_region_error(mocked_responses, source_mailgun, check_connection_url, test_config): - test_config["domain_region"] = "WRONG_REGION" - check_result = source_mailgun.check_connection(MagicMock(), test_config) - assert not check_result[0] - assert "domain_region" in check_result[1] - - -def test_check_connection_request_error(mocked_responses, source_mailgun, check_connection_url, test_config): - custom_exception = requests.RequestException() - mocked_responses.add(responses.GET, check_connection_url, body=custom_exception) - assert source_mailgun.check_connection(MagicMock(), test_config) == (False, custom_exception) - - -@pytest.mark.parametrize( - "config, error", - [ - (TEST_CONFIG, None), - (dict(**TEST_CONFIG, **{"start_date": "2021-01-01T00:00:00Z"}), None), - (dict(**TEST_CONFIG, **{"start_date": "wrong format"}), "date format"), - ], -) -def test_streams(config, error): - source = SourceMailgun() - expected_streams_number = 2 - if error is None: - streams = source.streams(config) - assert len(streams) == expected_streams_number - else: - with pytest.raises(ValueError) as exc_info: - source.streams(config) - assert error in str(exc_info.value) diff --git a/airbyte-integrations/connectors/source-mailgun/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mailgun/unit_tests/test_streams.py deleted file mode 100644 index eaa00a3fdee5..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/unit_tests/test_streams.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_mailgun.source import Domains, Events, MailgunStream - -from . import TEST_CONFIG - -DATES_RANGE = {"start_timestamp": 10, "end_timestamp": 20} - - -@pytest.mark.parametrize( - "stream,next_page_token,expected", - [ - (MailgunStream(TEST_CONFIG), None, ""), - (MailgunStream(TEST_CONFIG), {"url": "next-page-token"}, "next-page-token"), - (Domains(TEST_CONFIG), None, "domains"), - (Domains(TEST_CONFIG), {"url": "https://some-url/path/token"}, "https://some-url/path/token"), - (Events(TEST_CONFIG), None, "events"), - (Events(TEST_CONFIG), {"url": "https://some-url/path/token"}, "https://some-url/path/token"), - ], -) -def test_path(stream, next_page_token, expected): - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": next_page_token} - assert stream.path(**inputs) == expected - - -@pytest.mark.parametrize( - "stream,stream_slice,stream_state,next_page_token,expected", - [ - (Domains(TEST_CONFIG), {}, None, None, {}), - (Events(TEST_CONFIG), {}, {"timestamp": 1609452000.0}, None, {"ascending": "yes", "begin": 1609452000.0}), - (Events(TEST_CONFIG), {"begin": 10, "end": 20}, {"timestamp": 15}, None, {"begin": 15, "end": 20}), - (Events(dict(**TEST_CONFIG, **DATES_RANGE)), {"begin": 10, "end": 20}, {}, None, {"begin": 10, "end": 20}), - ], -) -def test_request_params(stream, stream_slice, stream_state, next_page_token, expected): - inputs = {"stream_slice": stream_slice, "stream_state": stream_state, "next_page_token": next_page_token} - assert stream.request_params(**inputs) == expected - - -@pytest.mark.parametrize( - "stream", - [ - MailgunStream(TEST_CONFIG), - Domains(TEST_CONFIG), - Events(TEST_CONFIG), - ], -) -class TestStreams: - def test_next_page_token(self, stream, normal_response, next_page_url): - inputs = {"response": normal_response} - assert stream.next_page_token(**inputs) == {"url": next_page_url} - - def test_next_page_token_last_page(self, stream): - response = MagicMock() - response.json = MagicMock(return_value={"items": [], "paging": {"next": "some-url"}}) - inputs = {"response": response} - assert stream.next_page_token(**inputs) is None - - def test_parse_response(self, stream, normal_response, test_records): - inputs = {"response": normal_response} - expected_parsed_object = test_records[0] - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - def test_request_headers(self, stream): - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - def test_http_method(self, stream): - expected_method = "GET" - assert stream.http_method == expected_method - - def test_backoff_time(self, stream): - response_mock = MagicMock() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - @pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], - ) - def test_should_retry(self, stream, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - assert stream.should_retry(response_mock) == should_retry - - -@pytest.mark.parametrize( - "stream_class", - [ - MailgunStream, - Domains, - Events, - ], -) -def test_domain_region(stream_class, test_config): - assert stream_class(config=test_config).url_base == "https://api.mailgun.net/v3/" - - test_config["domain_region"] = "EU" - assert stream_class(config=test_config).url_base == "https://api.eu.mailgun.net/v3/" - - test_config["domain_region"] = "US" - assert stream_class(config=test_config).url_base == "https://api.mailgun.net/v3/" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/test_catalog.json b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/test_catalog.json index 826e7ab35994..c3675b314d6b 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/test_catalog.json +++ b/airbyte-integrations/connectors/source-mailjet-mail/integration_tests/test_catalog.json @@ -4,9 +4,7 @@ "stream": { "name": "listrecipient", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml index 233811321f10..9a5b49f50b99 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt b/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt +++ b/airbyte-integrations/connectors/source-mailjet-mail/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-mailjet-mail/setup.py b/airbyte-integrations/connectors/source-mailjet-mail/setup.py index e26fe7d4e3cc..5f6daf3cdc2b 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/setup.py +++ b/airbyte-integrations/connectors/source-mailjet-mail/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml index 79d4be34d7a8..a68dbed823e2 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-sms/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt b/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt +++ b/airbyte-integrations/connectors/source-mailjet-sms/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-mailjet-sms/setup.py b/airbyte-integrations/connectors/source-mailjet-sms/setup.py index d30a85eb557c..7edfd5e35e8e 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/setup.py +++ b/airbyte-integrations/connectors/source-mailjet-sms/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index c90637292dc8..d4276e9ecc21 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.2.0 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml index d5c6d2a8aa5f..f5fa3a1faf4a 100644 --- a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml @@ -81,6 +81,14 @@ acceptance_tests: bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" - name: "activities_respondedtoapollinwebinar" bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_reached_dialogue_goal" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_interactedwith_documentin_dialogue" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwitha_dialogue" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_scheduled_meetingin_dialogue" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" # 52 streams, most of them use BULK API therefore it takes much time to run a sync timeout_seconds: 9000 fail_on_extra_columns: false diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.jsonl index 9fcff1ee1645..e4df100a9059 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.jsonl @@ -1,181 +1,72 @@ -{"stream": "activity_types", "data": {"id": 1, "name": "Visit Webpage", "description": "User visits a web page", "primaryAttribute": {"name": "Webpage ID", "dataType": "integer"}, "attributes": [{"name": "Client IP Address", "dataType": "string"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "Search Engine", "dataType": "string"}, {"name": "Search Query", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webpage URL", "dataType": "string"}]}, "emitted_at": 1682445125832} -{"stream": "activity_types", "data": {"id": 2, "name": "Fill Out Form", "description": "User fills out and submits a form on web page", "primaryAttribute": {"name": "Webform ID", "dataType": "integer"}, "attributes": [{"name": "Client IP Address", "dataType": "string"}, {"name": "Form Fields", "dataType": "text"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1682445125832} -{"stream": "activity_types", "data": {"id": 3, "name": "Click Link", "description": "User clicks link on a page", "primaryAttribute": {"name": "Link ID", "dataType": "integer"}, "attributes": [{"name": "Client IP Address", "dataType": "string"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1682445125832} -{"stream": "activity_types", "data": {"id": 6, "name": "Send Email", "description": "Send Marketo Email to a person", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1682445125833} -{"stream": "activity_types", "data": {"id": 7, "name": "Email Delivered", "description": "Marketo Email is delivered to a lead/contact", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1682445125833} -{"stream": "activity_types", "data": {"id": 8, "name": "Email Bounced", "description": "Marketo Email is bounced for a lead", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Category", "dataType": "string"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Details", "dataType": "string"}, {"name": "Email", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Subcategory", "dataType": "string"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1682445125833} -{"stream": "activity_types", "data": {"id": 9, "name": "Unsubscribe Email", "description": "Person unsubscribed from Marketo Emails", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "Integer"}, {"name": "Client IP Address", "dataType": "string"}, {"name": "Form Fields", "dataType": "text"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "Test Variant", "dataType": "integer"}, {"name": "User Agent", "dataType": "string"}, {"name": "Webform ID", "dataType": "integer"}, {"name": "Webpage ID", "dataType": "integer"}]}, "emitted_at": 1682445125833} -{"stream": "activity_types", "data": {"id": 10, "name": "Open Email", "description": "User opens Marketo Email", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Bot Activity Pattern", "dataType": "string"}, {"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Device", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Is Bot Activity", "dataType": "boolean"}, {"name": "Is Mobile Device", "dataType": "boolean"}, {"name": "Platform", "dataType": "string"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}, {"name": "User Agent", "dataType": "string"}]}, "emitted_at": 1682445125834} -{"stream": "activity_types", "data": {"id": 11, "name": "Click Email", "description": "User clicks on a link in a Marketo Email", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Bot Activity Pattern", "dataType": "string"}, {"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Device", "dataType": "string"}, {"name": "Is Bot Activity", "dataType": "boolean"}, {"name": "Is Mobile Device", "dataType": "boolean"}, {"name": "Is Predictive", "dataType": "boolean"}, {"name": "Link", "dataType": "string"}, {"name": "Link ID", "dataType": "string"}, {"name": "Platform", "dataType": "string"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Test Variant", "dataType": "integer"}, {"name": "User Agent", "dataType": "string"}]}, "emitted_at": 1682445125834} -{"stream": "activity_types", "data": {"id": 12, "name": "New Lead", "description": "New person/record is added to the lead database", "attributes": [{"name": "Created Date", "dataType": "date"}, {"name": "Form Name", "dataType": "string"}, {"name": "Lead Source", "dataType": "string"}, {"name": "List Name", "dataType": "string"}, {"name": "SFDC Type", "dataType": "string"}, {"name": "Source Type", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1682445125835} -{"stream": "activity_types", "data": {"id": 13, "name": "Change Data Value", "description": "Changed attribute value for a person/record", "primaryAttribute": {"name": "Attribute Name", "dataType": "integer"}, "attributes": [{"name": "New Value", "dataType": "string"}, {"name": "Old Value", "dataType": "string"}, {"name": "Reason", "dataType": "string"}, {"name": "Source", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1682445125835} -{"stream": "activity_types", "data": {"id": 22, "name": "Change Score", "description": "Modify the value of a score field", "primaryAttribute": {"name": "Score Name", "dataType": "integer"}, "attributes": [{"name": "Change Value", "dataType": "string"}, {"name": "New Value", "dataType": "integer"}, {"name": "Old Value", "dataType": "integer"}, {"name": "Reason", "dataType": "string"}]}, "emitted_at": 1682445125835} -{"stream": "activity_types", "data": {"id": 24, "name": "Add to List", "description": "Add a person/record to a list", "primaryAttribute": {"name": "List ID", "dataType": "integer"}, "attributes": [{"name": "Source", "dataType": "string"}]}, "emitted_at": 1682445125835} -{"stream": "activity_types", "data": {"id": 25, "name": "Remove from List", "description": "Remove this lead from a list", "primaryAttribute": {"name": "List ID", "dataType": "integer"}, "attributes": [{"name": "Source", "dataType": "string"}]}, "emitted_at": 1682445125836} -{"stream": "activity_types", "data": {"id": 27, "name": "Email Bounced Soft", "description": "Campaign Email is bounced soft for a lead", "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, "attributes": [{"name": "Campaign Run ID", "dataType": "integer"}, {"name": "Category", "dataType": "string"}, {"name": "Choice Number", "dataType": "integer"}, {"name": "Details", "dataType": "string"}, {"name": "Email", "dataType": "string"}, {"name": "Has Predictive", "dataType": "boolean"}, {"name": "Step ID", "dataType": "integer"}, {"name": "Subcategory", "dataType": "string"}, {"name": "Test Variant", "dataType": "integer"}]}, "emitted_at": 1682445125836} -{"stream": "activity_types", "data": {"id": 32, "name": "Merge Leads", "description": "Merge two or more leads into a single record", "primaryAttribute": {"name": "Lead ID", "dataType": "integer"}, "attributes": [{"name": "Merge IDs", "dataType": "array"}, {"name": "Master Updated", "dataType": "boolean"}, {"name": "Merged in Sales", "dataType": "boolean"}, {"name": "Merge Source", "dataType": "string"}, {"name": "API Method Name", "dataType": "string"}, {"name": "Modifying User", "dataType": "string"}, {"name": "Request Id", "dataType": "string"}]}, "emitted_at": 1682445125836} -{"stream": "activity_types", "data": {"id": 34, "name": "Add to Opportunity", "description": "Add to an Opportunity", "primaryAttribute": {"name": "Oppty ID", "dataType": "integer"}, "attributes": [{"name": "Is Primary", "dataType": "boolean"}, {"name": "Role", "dataType": "string"}]}, "emitted_at": 1682445125837} -{"stream": "activity_types", "data": {"id": 35, "name": "Remove from Opportunity", "description": "Remove from an Opportunity", "primaryAttribute": {"name": "Oppty ID", "dataType": "integer"}, "attributes": [{"name": "Is Primary", "dataType": "boolean"}, {"name": "Role", "dataType": "string"}]}, "emitted_at": 1682445125837} -{"stream": "campaigns", "data": {"id": 1053, "name": "Test Campaign 2", "description": "Description of the Test Campaign 2", "type": "batch", "workspaceName": "Default", "createdAt": "2023-01-17T10:53:35Z", "updatedAt": "2023-01-17T10:56:52Z", "active": false}, "emitted_at": 1682445127130} -{"stream": "campaigns", "data": {"id": 1054, "name": "Test Campaign 3", "description": "Description of the Test Campaign 3", "type": "batch", "workspaceName": "Default", "createdAt": "2023-01-17T10:54:04Z", "updatedAt": "2023-01-17T10:57:12Z", "active": false}, "emitted_at": 1682445127130} -{"stream": "campaigns", "data": {"id": 1055, "name": "Test Campaign 4", "description": "Description of the Test Campaign 4", "type": "batch", "workspaceName": "Default", "createdAt": "2023-01-17T10:54:24Z", "updatedAt": "2023-01-17T10:53:37Z", "active": false}, "emitted_at": 1682445127130} -{"stream": "campaigns", "data": {"id": 1056, "name": "Test Campaign 5", "description": "Description of the Test Campaign 5", "type": "trigger", "workspaceName": "Default", "createdAt": "2023-01-17T10:54:45Z", "updatedAt": "2023-01-20T16:25:04Z", "active": false}, "emitted_at": 1682445127131} -{"stream": "campaigns", "data": {"id": 1124, "name": "Test Smart Campaign", "description": "Test", "type": "batch", "programName": "API Test Program", "programId": 1003, "workspaceName": "Default", "createdAt": "2023-01-20T12:46:51Z", "updatedAt": "2023-01-23T08:24:16Z", "active": false}, "emitted_at": 1682445127131} -{"stream": "campaigns", "data": {"id": 1130, "name": "01-Process Registrations", "description": "This campaign must be activated BEFORE the registration URL is published. It sets the program status to \"Registered\", which makes the registrant's information available to the Marketo's mobile event app. Then the registrant gets a confirmation email with a calendar link.", "type": "trigger", "programName": "BP-LE-YYYY-MM-DD Live Event v3", "programId": 1026, "workspaceName": "Default", "createdAt": "2023-01-20T13:08:56Z", "updatedAt": "2023-01-20T13:09:53Z", "active": false}, "emitted_at": 1682445127131} -{"stream": "campaigns", "data": {"id": 1131, "name": "02a-Send Invitation", "description": "This campaign sends the initial invitation for the event. It should only include marketable leads, which have checked the proper subscription option.", "type": "batch", "programName": "BP-LE-YYYY-MM-DD Live Event v3", "programId": 1026, "workspaceName": "Default", "createdAt": "2023-01-20T13:08:56Z", "updatedAt": "2023-01-20T13:08:58Z", "active": false}, "emitted_at": 1682445127131} -{"stream": "campaigns", "data": {"id": 1132, "name": "04-Send Follow-up Emails", "description": "This campaign assumes that all registrants have been updated to a new program status of \"Attended\" or \"No Show\". You can do this by using the Marketo mobile event app for checking-in attendees, or you can import lists of attendees and no shows under the Program's Members tab. This campaign will then send follow-up emails that you can use to further engage your audience.", "type": "batch", "programName": "BP-LE-YYYY-MM-DD Live Event v3", "programId": 1026, "workspaceName": "Default", "createdAt": "2023-01-20T13:08:56Z", "updatedAt": "2023-01-20T13:08:58Z", "active": false}, "emitted_at": 1682445127131} -{"stream": "campaigns", "data": {"id": 1133, "name": "02b-Send Invitation Reminder", "description": "This campaign sends a reminder to people who received the original invitation, but have not registered, yet.", "type": "batch", "programName": "BP-LE-YYYY-MM-DD Live Event v3", "programId": 1026, "workspaceName": "Default", "createdAt": "2023-01-20T13:08:56Z", "updatedAt": "2023-01-20T13:08:57Z", "active": false}, "emitted_at": 1682445127132} -{"stream": "campaigns", "data": {"id": 1134, "name": "02c-Send Invitation Last Chance", "description": "This campaign sends a last reminder to people who received the original invitation, but have not registered, yet.", "type": "batch", "programName": "BP-LE-YYYY-MM-DD Live Event v3", "programId": 1026, "workspaceName": "Default", "createdAt": "2023-01-20T13:08:56Z", "updatedAt": "2023-01-20T13:08:57Z", "active": false}, "emitted_at": 1682445127132} -{"stream": "campaigns", "data": {"id": 1135, "name": "03-Send Reminder to Attend", "description": "This campaign sends a reminder to all registrants, including a calendar link. This campaign should be scheduled close to the event date - typically the day prior.", "type": "batch", "programName": "BP-LE-YYYY-MM-DD Live Event v3", "programId": 1026, "workspaceName": "Default", "createdAt": "2023-01-20T13:08:56Z", "updatedAt": "2023-01-20T13:08:57Z", "active": false}, "emitted_at": 1682445127132} -{"stream": "campaigns", "data": {"id": 1145, "name": "01-Influenced (Program Success)", "description": "Activate this Campaign to record when a lead successfully completed all steps of the Program and was influenced by the information it offered.", "type": "trigger", "programName": "BP-ES-YYYY-MM-DD Email Send v3", "programId": 1028, "workspaceName": "Default", "createdAt": "2023-01-23T08:02:31Z", "updatedAt": "2023-01-23T08:03:12Z", "active": false}, "emitted_at": 1682445127132} -{"stream": "campaigns", "data": {"id": 1161, "name": "Test Smart Campaign 12", "description": "Test", "type": "batch", "programName": "Test Nurture", "programId": 1025, "workspaceName": "Default", "createdAt": "2023-01-23T08:26:33Z", "updatedAt": "2023-01-23T08:26:33Z", "active": false}, "emitted_at": 1682445127132} -{"stream": "leads", "data": {"company": "Test Company", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 885, "mktoName": "Test Person 1 All People", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Test Person 1", "middleName": null, "lastName": "All People", "email": "0M996mK4pH@ttt.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "89", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-17T11:00:58Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "885", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "885", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192205} -{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": "airbyte.io", "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 888, "mktoName": "[Sample]Name Surname", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "[Sample]Name", "middleName": null, "lastName": "Surname", "email": "i.g@example.io", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "90", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": "Asia/Riyadh", "originalSourceType": "Web form fillout", "originalSourceInfo": "Form: API Test Program.123.airbyte", "registrationSourceType": "Web form fillout", "registrationSourceInfo": "Form: API Test Program.123.airbyte", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": "85.209.47.207", "inferredCompany": "Arabic Computer System Co.", "inferredCountry": "Saudi Arabia", "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-17T12:29:38Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "888", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "888", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192206} -{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": "globallogic.com", "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 889, "mktoName": "[Sample] sample", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "[Sample]", "middleName": null, "lastName": "sample", "email": "i.g@example.io", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "91", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": "Asia/Riyadh", "originalSourceType": "Web form fillout", "originalSourceInfo": "Form: API Test Program.123.airbyte", "registrationSourceType": "Web form fillout", "registrationSourceInfo": "Form: API Test Program.123.airbyte", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": "85.209.47.207", "inferredCompany": "Arabic Computer System Co.", "inferredCountry": "Saudi Arabia", "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-17T12:30:09Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": "_mch-mktoweb.com-1673954921699-88079", "externalSalesPersonId": null, "leadPerson": "889", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "889", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192206} -{"stream": "leads", "data": {"company": "Test Company Org 5", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 890, "mktoName": "Test Person 10 Test 10", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Test Person 10", "middleName": null, "lastName": "Test 10", "email": "name10@example.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "92", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": "US", "postalCode": null, "personTimeZone": "America/Chicago", "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-19T15:07:51Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "890", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "890", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192207} -{"stream": "leads", "data": {"company": "Test Company Org 11", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 891, "mktoName": "Name11 Surname11", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Name11", "middleName": null, "lastName": "Surname11", "email": "name11@example.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "93", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": "US", "postalCode": null, "personTimeZone": "America/Chicago", "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-19T15:25:17Z", "updatedAt": "2023-01-19T15:27:44Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "891", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "891", "leadPartitionId": "3", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192208} -{"stream": "leads", "data": {"company": "Test Company Org 2", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 895, "mktoName": "Test 2", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Test", "middleName": null, "lastName": "2", "email": "name5@example.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "94", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-20T11:56:00Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "895", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "895", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192209} -{"stream": "leads", "data": {"company": "Test Company Org 5", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 897, "mktoName": "Test test", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Test", "middleName": null, "lastName": "test", "email": "name8@example.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "95", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-23T08:09:55Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "897", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "897", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192210} -{"stream": "leads", "data": {"company": "Test Company Org 5", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 898, "mktoName": "Person 1 Person 1", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Person 1", "middleName": null, "lastName": "Person 1", "email": "name100@example.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "96", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-23T08:20:01Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "898", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "898", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192211} -{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 901, "mktoName": null, "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": null, "middleName": null, "lastName": null, "email": null, "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "97", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": "America/New_York", "originalSourceType": "Web form fillout", "originalSourceInfo": "Form: API Test Program.123.airbyte", "registrationSourceType": "Web form fillout", "registrationSourceInfo": "Form: API Test Program.123.airbyte", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": "63.117.14.132", "inferredCompany": "Verizon Business", "inferredCountry": "United States", "inferredCity": "Bronx", "inferredStateRegion": "NY", "inferredPostalCode": "10472", "inferredMetropolitanArea": "New York NY (MRC-501)", "inferredPhoneAreaCode": "718", "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-25T00:21:35Z", "updatedAt": "2023-01-25T00:21:37Z", "cookies": "_mch-mktoweb.com-1674606091542-53000", "externalSalesPersonId": null, "leadPerson": "901", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "901", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1682445192212} -{"stream": "lists", "data": {"id": 1016, "name": "List 1", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:43:09Z", "updatedAt": "2023-01-20T13:11:03Z"}, "emitted_at": 1682445193305} -{"stream": "lists", "data": {"id": 1017, "name": "List 2", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:43:41Z", "updatedAt": "2023-01-17T12:43:43Z"}, "emitted_at": 1682445193306} -{"stream": "lists", "data": {"id": 1018, "name": "List 3", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:43:59Z", "updatedAt": "2023-01-17T12:44:00Z"}, "emitted_at": 1682445193306} -{"stream": "lists", "data": {"id": 1019, "name": "List 4", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:44:15Z", "updatedAt": "2023-01-17T12:44:16Z"}, "emitted_at": 1682445193306} -{"stream": "lists", "data": {"id": 1020, "name": "List 5", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:44:28Z", "updatedAt": "2023-01-17T12:44:30Z"}, "emitted_at": 1682445193306} -{"stream": "programs", "data": {"id": 1016, "name": "123", "description": "", "createdAt": "2021-09-01T16:02:30Z", "updatedAt": "2023-01-17T12:36:46Z", "url": "https://app-sj32.marketo.com/#EBP1016A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "completed", "workspace": "Default", "headStart": false}, "emitted_at": 1682445194496} -{"stream": "programs", "data": {"id": 1017, "name": "air", "description": "", "createdAt": "2021-09-01T16:09:23Z", "updatedAt": "2023-01-17T12:31:23Z", "url": "https://app-sj32.marketo.com/#EBP1017A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "completed", "workspace": "Default", "headStart": false}, "emitted_at": 1682445194496} -{"stream": "programs", "data": {"id": 1003, "name": "API Test Program", "description": "Sample API Program", "createdAt": "2021-01-18T13:55:44Z", "updatedAt": "2023-01-20T13:02:44Z", "url": "https://app-sj32.marketo.com/#PG1003A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194496} -{"stream": "programs", "data": {"id": 1004, "name": "API Test Program 1", "description": "Sample API Program 1", "createdAt": "2021-01-18T13:57:37Z", "updatedAt": "2023-01-24T11:20:05Z", "url": "https://app-sj32.marketo.com/#PG1004A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194497} -{"stream": "programs", "data": {"id": 1013, "name": "API Test Program 10", "description": "Sample API Program 10", "createdAt": "2021-01-18T13:57:46Z", "updatedAt": "2023-01-20T16:27:11Z", "url": "https://app-sj32.marketo.com/#PG1013A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194497} -{"stream": "programs", "data": {"id": 1012, "name": "API Test Program 9", "description": "Sample API Program 9", "createdAt": "2021-01-18T13:57:45Z", "updatedAt": "2023-01-20T16:14:21Z", "url": "https://app-sj32.marketo.com/#PG1012A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 50, "folderName": "Nurture"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194497} -{"stream": "programs", "data": {"id": 1028, "name": "BP-ES-YYYY-MM-DD Email Send v3", "description": "Quick and easy Email Send Program for sending one single email, with or without A/B test. Channel: \"Email Send\". \nNOTE: Don't forget to activate the \"01-Influenced\" Campaign for tracking Success!", "createdAt": "2023-01-23T08:02:31Z", "updatedAt": "2023-01-23T08:03:04Z", "url": "https://app-sj32.marketo.com/#EBP1028A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Folder", "value": 54, "folderName": "Data Management"}, "status": "unlocked", "workspace": "Default", "headStart": false}, "emitted_at": 1682445194498} -{"stream": "programs", "data": {"id": 1026, "name": "BP-LE-YYYY-MM-DD Live Event v3", "description": "Live Event Program, with registration page, three invitation emails, and follow-up emails. This is suitable for all events where you require registrations, including roadshows, lunches, dinners, or presentations at trade show events. Use the Marketo Events App for check-in and attendance tracking on your iPad or Android tablet.\nNOTE: Before the event, download the registration information to your iPad/Adroid Tablet. Then check-in visitors, and synchronize the attendance information back to Marketo. If you don't have a tablet, you can import attendees after under the Program's \"Members\" tab before sending your follow-up emails.", "createdAt": "2023-01-20T13:08:36Z", "updatedAt": "2023-01-20T16:32:34Z", "url": "https://app-sj32.marketo.com/#ME1026A1", "type": "Event", "channel": "Live Event", "folder": {"type": "Folder", "value": 48, "folderName": "Emails"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194498} -{"stream": "programs", "data": {"id": 1027, "name": "BP-LI-YYYY-MM-DD List Import v3", "description": "Program to manage imports of new target lists and record Acquisition Program at the same time. Go to the \"Members\" tab, and click on \"Import Members\". Follow the on-screen instructions, In Step 3, assign the Member Status \"On List\". \nNOTE: List Import Programs do not track \"Influence\", or \"Success\", because they do not communicate with anyone. Their sole purpose is to record Acquisition Program for First-Touch Attribution and other Analytics.", "createdAt": "2023-01-23T07:58:14Z", "updatedAt": "2023-01-23T07:58:59Z", "url": "https://app-sj32.marketo.com/#PG1027A1", "type": "Default", "channel": "List Import", "folder": {"type": "Folder", "value": 51, "folderName": "Web Content"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194498} -{"stream": "programs", "data": {"id": 1018, "name": "John Doe", "description": "", "createdAt": "2021-09-08T12:49:49Z", "updatedAt": "2023-01-17T12:29:10Z", "url": "https://app-sj32.marketo.com/#PG1018A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194499} -{"stream": "programs", "data": {"id": 1023, "name": "My Test Program 1", "description": "Test", "createdAt": "2023-01-17T12:38:35Z", "updatedAt": "2023-01-17T12:39:38Z", "url": "https://app-sj32.marketo.com/#EBP1023A1", "type": "Email", "channel": "Newsletter", "folder": {"type": "Folder", "value": 49, "folderName": "Newsletters"}, "status": "unlocked", "workspace": "Default"}, "emitted_at": 1682445194499} -{"stream": "programs", "data": {"id": 1024, "name": "Test Event 1", "description": "Test", "createdAt": "2023-01-20T09:29:38Z", "updatedAt": "2023-01-20T09:30:59Z", "url": "https://app-sj32.marketo.com/#ME1024A1", "type": "Event", "channel": "Live Event", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "", "workspace": "Default"}, "emitted_at": 1682445194499} -{"stream": "programs", "data": {"id": 1025, "name": "Test Nurture", "description": "Test", "createdAt": "2023-01-20T09:51:39Z", "updatedAt": "2023-01-23T08:27:01Z", "url": "https://app-sj32.marketo.com/#NP1025A1", "type": "Engagement", "channel": "Nurture", "folder": {"type": "Folder", "value": 50, "folderName": "Nurture"}, "status": "on", "workspace": "Default"}, "emitted_at": 1682445194500} -{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2125", "leadId": 864, "activityDate": "2023-01-17T11:29:20Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "10", "primaryAttributeValue": "Landing-Page-1", "webpage_url": "/lp/602-EUO-598/Landing-Page-1.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "https://602-euo-598.mktoweb.com/"}, "emitted_at": 1682445321184} -{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2132", "leadId": 888, "activityDate": "2023-01-17T12:29:28Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "11", "primaryAttributeValue": "Landing-Page-2", "webpage_url": "/lp/602-EUO-598/Landing-Page-2.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1682445321185} -{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2133", "leadId": 888, "activityDate": "2023-01-17T12:29:51Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "12", "primaryAttributeValue": "Landing-Page-3", "webpage_url": "/lp/602-EUO-598/Landing-Page-3.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1682445321185} -{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2199", "leadId": 864, "activityDate": "2023-01-20T12:40:22Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "1", "primaryAttributeValue": "UnsubscribePage", "webpage_url": "/lp/datalineaedev/UnsubscribePage.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg", "referrer_url": "http://mkto-sj320154.com/"}, "emitted_at": 1682445321186} -{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2200", "leadId": 864, "activityDate": "2023-01-20T12:40:36Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "2", "primaryAttributeValue": "UnsubscribeConfirm", "webpage_url": "/lp/602-EUO-598/UnsubscribeConfirm.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", "query_parameters": "aliId=eyJpIjoiT2hyTkpVMUJTSUZRajZIZyIsInQiOiJKXC9xOUlRU3ZRR0czdlBTWnkxS3dzQT09In0%3D", "referrer_url": "http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1"}, "emitted_at": 1682445321186} -{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2293", "leadId": 901, "activityDate": "2023-01-25T00:21:34Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "12", "primaryAttributeValue": "Landing-Page-3", "webpage_url": "/lp/602-EUO-598/Landing-Page-3.html", "client_ip_address": "63.117.14.132", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1682445321187} -{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2129", "leadId": 864, "activityDate": "2023-01-17T11:29:43Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:7:\"Segment\";s:8:\"LastName\";s:1:\"5\";s:5:\"Email\";s:27:\"integration-test@airbyte.io\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1005\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-1.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1673954921699-88079\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:66:\"https://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-1.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"774b4fee27ce1a145370b5ffebe8f4e300c1404cfd86a2067763c65b063d1b66\";s:25:\"formServiceRequestId31337\";s:17:\"17915#185bf7ded1f\";}", "client_ip_address": "85.209.47.207", "webpage_id": 10, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "https://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-1.html"}, "emitted_at": 1682445447915} -{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2131", "leadId": 888, "activityDate": "2023-01-17T12:29:40Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:12:\"[Sample]Name\";s:8:\"LastName\";s:7:\"Surname\";s:5:\"Email\";s:25:\"iryna.grankova@airbyte.io\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1007\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1673954921699-88079\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:65:\"http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"31c81f3245d5080d0bb93a07f67ae67a0c4667adcce1400cee8857206f656981\";s:25:\"formServiceRequestId31337\";s:17:\"161b7#185bfb4d8a8\";}", "client_ip_address": "85.209.47.207", "webpage_id": 11, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html"}, "emitted_at": 1682445447915} -{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2135", "leadId": 889, "activityDate": "2023-01-17T12:30:10Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:8:\"[Sample]\";s:8:\"LastName\";s:6:\"sample\";s:5:\"Email\";s:30:\"iryna.grankova@globallogic.com\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1009\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1673954921699-88079\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:65:\"http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"c7f8c6af1167c273c27bc3b441b5be79672d5a6d61f66398eba04782ca972195\";s:25:\"formServiceRequestId31337\";s:17:\"16fe4#185bfb55182\";}", "client_ip_address": "85.209.47.207", "webpage_id": 12, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html"}, "emitted_at": 1682445447916} -{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2198", "leadId": 864, "activityDate": "2023-01-20T12:40:36Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1", "primaryAttributeValue": "Email Unsubscribe Form", "form_fields": "a:20:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:5:\"Email\";s:27:\"integration-test@airbyte.io\";s:12:\"Unsubscribed\";s:3:\"Yes\";s:6:\"formid\";s:1:\"1\";s:4:\"lpId\";s:1:\"1\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:93:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/UnsubscribePage.html?cr={creative}&kw={keyword}\";s:12:\"followupLpId\";s:1:\"2\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1674218381862-27778\";s:7:\"formVid\";s:1:\"1\";s:7:\"mkt_tok\";s:102:\"NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg\";s:13:\"_mktoReferrer\";s:197:\"http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg\";s:14:\"checksumFields\";s:113:\"Email,Unsubscribed,formid,lpId,subId,munchkinId,lpurl,followupLpId,cr,kw,q,_mkt_trk,formVid,mkt_tok,_mktoReferrer\";s:8:\"checksum\";s:64:\"b959482d2aff7c190b46cc567e79fce9c99a6342fbe1eb3fd863ec9a6ac1159a\";s:25:\"formServiceRequestId31337\";s:16:\"7e0e#185cf31f199\";}", "client_ip_address": "85.209.47.207", "webpage_id": 1, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg", "referrer_url": "http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg"}, "emitted_at": 1682445447916} -{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2292", "leadId": 901, "activityDate": "2023-01-25T00:21:37Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:0:\"\";s:8:\"LastName\";s:0:\"\";s:5:\"Email\";s:0:\"\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1009\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1674606091542-53000\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:65:\"http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"ff09fb4acb61768982a6d6dd287c60f63d79cbf618e7a95d45343818da3c5a59\";s:25:\"formServiceRequestId31337\";s:16:\"157d#185e64d2c2b\";}", "client_ip_address": "63.117.14.132", "webpage_id": 12, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html"}, "emitted_at": 1682445447916} -{"stream": "activities_send_email", "data": {"marketoGUID": "2185", "leadId": 109, "activityDate": "2023-01-20T11:45:57Z", "activityTypeId": 6, "campaignId": 1109, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 66, "step_id": 305, "choice_number": 0}, "emitted_at": 1682445574498} -{"stream": "activities_send_email", "data": {"marketoGUID": "2206", "leadId": 876, "activityDate": "2023-01-20T12:49:27Z", "activityTypeId": 6, "campaignId": 1126, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 86, "step_id": 364, "choice_number": 0}, "emitted_at": 1682445574499} -{"stream": "activities_send_email", "data": {"marketoGUID": "2239", "leadId": 99, "activityDate": "2023-01-20T16:21:28Z", "activityTypeId": 6, "campaignId": 1138, "primaryAttributeValueId": "1013", "primaryAttributeValue": "BP-LE-YYYY-MM-DD Live Event v3.EMAIL-Invitation", "campaign_run_id": 92, "step_id": 417, "choice_number": 0}, "emitted_at": 1682445574499} -{"stream": "activities_send_email", "data": {"marketoGUID": "2250", "leadId": 99, "activityDate": "2023-01-22T11:57:56Z", "activityTypeId": 6, "campaignId": 1100, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 95, "step_id": 245, "choice_number": 0}, "emitted_at": 1682445574499} -{"stream": "activities_send_email", "data": {"marketoGUID": "2251", "leadId": 379, "activityDate": "2023-01-22T11:57:56Z", "activityTypeId": 6, "campaignId": 1100, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 95, "step_id": 245, "choice_number": 0}, "emitted_at": 1682445574500} -{"stream": "activities_send_email", "data": {"marketoGUID": "2252", "leadId": 800, "activityDate": "2023-01-22T11:57:56Z", "activityTypeId": 6, "campaignId": 1100, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 95, "step_id": 245, "choice_number": 0}, "emitted_at": 1682445574500} -{"stream": "activities_send_email", "data": {"marketoGUID": "2271", "leadId": 880, "activityDate": "2023-01-23T08:25:26Z", "activityTypeId": 6, "campaignId": 1159, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 112, "step_id": 489, "choice_number": 0}, "emitted_at": 1682445574500} -{"stream": "activities_email_delivered", "data": {"marketoGUID": "2186", "leadId": 109, "activityDate": "2023-01-20T11:45:58Z", "activityTypeId": 7, "campaignId": 1109, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 66, "step_id": 305, "choice_number": 407}, "emitted_at": 1682445700861} -{"stream": "activities_email_delivered", "data": {"marketoGUID": "2208", "leadId": 876, "activityDate": "2023-01-20T12:49:29Z", "activityTypeId": 7, "campaignId": 1126, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 86, "step_id": 364, "choice_number": 470}, "emitted_at": 1682445700861} -{"stream": "activities_email_bounced", "data": {"marketoGUID": "2273", "leadId": 880, "activityDate": "2023-01-23T08:25:26Z", "activityTypeId": 8, "campaignId": 1159, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 112, "category": "2", "email": "alastor@moody.com", "details": "550 5.4.1 Recipient address rejected: Access denied. AS(201806281) [SN1NAM02FT0043.eop-nam02.prod.protection.outlook.com]", "subcategory": "2003", "step_id": 489, "choice_number": 623}, "emitted_at": 1682445827491} -{"stream": "activities_unsubscribe_email", "data": {"marketoGUID": "2201", "leadId": 864, "activityDate": "2023-01-20T12:40:36Z", "activityTypeId": 9, "campaignId": null, "primaryAttributeValueId": "1010", "primaryAttributeValue": "My Email Program - RF.Test Email 1", "webform_id": 1, "client_ip_address": "85.209.47.207", "form_fields": "a:20:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:5:\"Email\";s:27:\"integration-test@airbyte.io\";s:12:\"Unsubscribed\";s:3:\"Yes\";s:6:\"formid\";s:1:\"1\";s:4:\"lpId\";s:1:\"1\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:93:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/UnsubscribePage.html?cr={creative}&kw={keyword}\";s:12:\"followupLpId\";s:1:\"2\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1674218381862-27778\";s:7:\"formVid\";s:1:\"1\";s:7:\"mkt_tok\";s:102:\"NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg\";s:13:\"_mktoReferrer\";s:197:\"http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg\";s:14:\"checksumFields\";s:113:\"Email,Unsubscribed,formid,lpId,subId,munchkinId,lpurl,followupLpId,cr,kw,q,_mkt_trk,formVid,mkt_tok,_mktoReferrer\";s:8:\"checksum\";s:64:\"b959482d2aff7c190b46cc567e79fce9c99a6342fbe1eb3fd863ec9a6ac1159a\";s:25:\"formServiceRequestId31337\";s:16:\"7e0e#185cf31f199\";}", "webpage_id": 1, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg", "referrer_url": "http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg"}, "emitted_at": 1682445954184} -{"stream": "activities_click_email", "data": {"marketoGUID": "2213", "leadId": 876, "activityDate": "2023-01-20T12:57:10Z", "activityTypeId": 11, "campaignId": 1126, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 86, "platform": "Win7", "is_mobile_device": false, "step_id": 364, "device": "Windows Desktop", "choice_number": 470, "is_bot_activity": false, "user_agent": "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", "bot_activity_pattern": "N/A", "link": "http://mylink"}, "emitted_at": 1682446080915} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2108", "leadId": 885, "activityDate": "2023-01-17T11:00:59Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "885", "primaryAttributeValue": "Test Person 1 All People", "created_date": "2023-01-17", "source_type": "New lead"}, "emitted_at": 1682446207156} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2130", "leadId": 888, "activityDate": "2023-01-17T12:29:39Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "888", "primaryAttributeValue": "[Sample]Name Surname", "form_name": "API Test Program.123.airbyte", "created_date": "2023-01-17", "source_type": "Web form fillout"}, "emitted_at": 1682446207156} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2134", "leadId": 889, "activityDate": "2023-01-17T12:30:09Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "889", "primaryAttributeValue": "[Sample] sample", "form_name": "API Test Program.123.airbyte", "created_date": "2023-01-17", "source_type": "Web form fillout"}, "emitted_at": 1682446207157} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2142", "leadId": 890, "activityDate": "2023-01-19T15:07:51Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "890", "primaryAttributeValue": "Test Person 10 Test 10", "created_date": "2023-01-19", "source_type": "New lead"}, "emitted_at": 1682446207157} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2147", "leadId": 891, "activityDate": "2023-01-19T15:25:17Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "891", "primaryAttributeValue": "Name11 Surname11", "created_date": "2023-01-19", "source_type": "New lead"}, "emitted_at": 1682446207158} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2192", "leadId": 895, "activityDate": "2023-01-20T11:56:00Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "895", "primaryAttributeValue": "Test 2", "created_date": "2023-01-20", "source_type": "New lead"}, "emitted_at": 1682446207158} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2263", "leadId": 897, "activityDate": "2023-01-23T08:09:56Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "897", "primaryAttributeValue": "Test test", "created_date": "2023-01-23", "source_type": "New lead"}, "emitted_at": 1682446207158} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2265", "leadId": 898, "activityDate": "2023-01-23T08:20:01Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "898", "primaryAttributeValue": "Person 1 Person 1", "created_date": "2023-01-23", "source_type": "New lead"}, "emitted_at": 1682446207159} -{"stream": "activities_new_lead", "data": {"marketoGUID": "2291", "leadId": 901, "activityDate": "2023-01-25T00:21:36Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "901", "primaryAttributeValue": null, "form_name": "API Test Program.123.airbyte", "created_date": "2023-01-24", "source_type": "Web form fillout"}, "emitted_at": 1682446207159} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2243", "leadId": 800, "activityDate": "2023-01-20T16:28:59Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "4", "primaryAttributeValue": "Company Name", "old_value": null, "new_value": "Test Company Org 5", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333994} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2244", "leadId": 800, "activityDate": "2023-01-20T16:28:59Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "12", "primaryAttributeValue": "Main Phone", "old_value": null, "new_value": "+13888335555", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333994} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2259", "leadId": 99, "activityDate": "2023-01-23T08:06:52Z", "activityTypeId": 13, "campaignId": 1149, "primaryAttributeValueId": "602", "primaryAttributeValue": "Acquisition Date", "old_value": null, "new_value": "2023-01-26 14:00:00", "reason": "Changed by Smart Campaign Run Action Change Data Value 2023-01-23 12:06:45 am action Change Data Value", "source": "Marketo Flow Action"}, "emitted_at": 1682446333995} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2274", "leadId": 880, "activityDate": "2023-01-23T08:25:38Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "77", "primaryAttributeValue": "Email Invalid", "old_value": "False", "new_value": "True", "reason": "System flow action sysActionEmailBounced for lead 880"}, "emitted_at": 1682446333995} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2275", "leadId": 880, "activityDate": "2023-01-23T08:25:38Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "78", "primaryAttributeValue": "Email Invalid Cause", "old_value": null, "new_value": "550 5.4.1 Recipient address rejected: Access denied. AS(201806281) [SN1NAM02FT0043.eop-nam02.prod.protection.outlook.com]", "reason": "System flow action sysActionEmailBounced for lead 880"}, "emitted_at": 1682446333996} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2285", "leadId": 824, "activityDate": "2023-01-23T14:25:01Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "56", "primaryAttributeValue": "Email Address", "old_value": "y.kurochkin@zazmic.com", "new_value": "y.k@example.com", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333996} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2286", "leadId": 867, "activityDate": "2023-01-23T14:26:11Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "56", "primaryAttributeValue": "Email Address", "old_value": "yurii.cherniaiev@globallogic.com", "new_value": "y.c@example.com", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333996} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2287", "leadId": 868, "activityDate": "2023-01-23T14:26:47Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "56", "primaryAttributeValue": "Email Address", "old_value": "yurii.cherniaiev@globallogic.com", "new_value": "y.c@example.com", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333997} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2288", "leadId": 869, "activityDate": "2023-01-23T14:26:57Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "56", "primaryAttributeValue": "Email Address", "old_value": "yurii.chenriaiev@globallogic.com", "new_value": "y.c@example.com", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333997} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2289", "leadId": 888, "activityDate": "2023-01-23T14:27:16Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "56", "primaryAttributeValue": "Email Address", "old_value": "iryna.grankova@airbyte.io", "new_value": "i.g@example.io", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333998} -{"stream": "activities_change_data_value", "data": {"marketoGUID": "2290", "leadId": 889, "activityDate": "2023-01-23T14:27:19Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "56", "primaryAttributeValue": "Email Address", "old_value": "iryna.grankova@globallogic.com", "new_value": "i.g@example.io", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1682446333998} -{"stream": "activities_change_score", "data": {"marketoGUID": "2118", "leadId": 140, "activityDate": "2023-01-17T11:09:23Z", "activityTypeId": 22, "campaignId": 1064, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "change_value": "+10", "old_value": null, "new_value": 10, "reason": "Changed by Smart Campaign Run Action Change Score 2023-01-17 03:09:02 am action Change Score"}, "emitted_at": 1682446460678} -{"stream": "activities_change_score", "data": {"marketoGUID": "2209", "leadId": 143, "activityDate": "2023-01-20T12:55:59Z", "activityTypeId": 22, "campaignId": null, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "change_value": "+20", "old_value": null, "new_value": 20, "reason": "Manual person edit"}, "emitted_at": 1682446460678} -{"stream": "activities_change_score", "data": {"marketoGUID": "2211", "leadId": 362, "activityDate": "2023-01-20T12:56:46Z", "activityTypeId": 22, "campaignId": null, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "change_value": "+100", "old_value": null, "new_value": 100, "reason": "Manual person edit"}, "emitted_at": 1682446460679} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2110", "leadId": 100, "activityDate": "2023-01-17T11:07:23Z", "activityTypeId": 24, "campaignId": 1059, "primaryAttributeValueId": "1002", "primaryAttributeValue": "Test list number 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587196} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2111", "leadId": 100, "activityDate": "2023-01-17T11:07:24Z", "activityTypeId": 24, "campaignId": 1058, "primaryAttributeValueId": "1001", "primaryAttributeValue": "Test list", "source": "Marketo Flow Action"}, "emitted_at": 1682446587196} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2116", "leadId": 140, "activityDate": "2023-01-17T11:08:53Z", "activityTypeId": 24, "campaignId": 1063, "primaryAttributeValueId": "1004", "primaryAttributeValue": "Test list number 3", "source": "Marketo Flow Action"}, "emitted_at": 1682446587196} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2117", "leadId": 140, "activityDate": "2023-01-17T11:08:54Z", "activityTypeId": 24, "campaignId": 1062, "primaryAttributeValueId": "1003", "primaryAttributeValue": "Test list number 2", "source": "Marketo Flow Action"}, "emitted_at": 1682446587196} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2120", "leadId": 807, "activityDate": "2023-01-17T11:25:53Z", "activityTypeId": 24, "campaignId": 1067, "primaryAttributeValueId": "1006", "primaryAttributeValue": "Test list number 5", "source": "Marketo Flow Action"}, "emitted_at": 1682446587197} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2123", "leadId": 816, "activityDate": "2023-01-17T11:26:53Z", "activityTypeId": 24, "campaignId": 1069, "primaryAttributeValueId": "1007", "primaryAttributeValue": "Test list number 6", "source": "Marketo Flow Action"}, "emitted_at": 1682446587197} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2136", "leadId": 100, "activityDate": "2023-01-17T14:14:53Z", "activityTypeId": 24, "campaignId": 1072, "primaryAttributeValueId": "1001", "primaryAttributeValue": "Test list", "source": "Marketo Flow Action"}, "emitted_at": 1682446587197} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2154", "leadId": 105, "activityDate": "2023-01-20T09:32:23Z", "activityTypeId": 24, "campaignId": 1082, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587198} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2162", "leadId": 110, "activityDate": "2023-01-20T09:45:23Z", "activityTypeId": 24, "campaignId": 1090, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587198} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2163", "leadId": 112, "activityDate": "2023-01-20T09:45:53Z", "activityTypeId": 24, "campaignId": 1091, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587198} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2164", "leadId": 140, "activityDate": "2023-01-20T09:45:53Z", "activityTypeId": 24, "campaignId": 1092, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587198} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2187", "leadId": 105, "activityDate": "2023-01-20T11:50:53Z", "activityTypeId": 24, "campaignId": 1110, "primaryAttributeValueId": "1011", "primaryAttributeValue": "Test list number 10", "source": "Marketo Flow Action"}, "emitted_at": 1682446587199} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2194", "leadId": 110, "activityDate": "2023-01-20T12:34:23Z", "activityTypeId": 24, "campaignId": 1115, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587199} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2195", "leadId": 110, "activityDate": "2023-01-20T12:36:23Z", "activityTypeId": 24, "campaignId": 1116, "primaryAttributeValueId": "1002", "primaryAttributeValue": "Test list number 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446587199} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2261", "leadId": 99, "activityDate": "2023-01-23T08:07:52Z", "activityTypeId": 24, "campaignId": 1151, "primaryAttributeValueId": "1012", "primaryAttributeValue": "EM - Auteur - v1.airbyte", "source": "Marketo Flow Action"}, "emitted_at": 1682446587199} -{"stream": "activities_addto_list", "data": {"marketoGUID": "2266", "leadId": 898, "activityDate": "2023-01-23T08:20:02Z", "activityTypeId": 24, "campaignId": null, "primaryAttributeValueId": "1010", "primaryAttributeValue": "Test list number 9"}, "emitted_at": 1682446587200} -{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2165", "leadId": 379, "activityDate": "2023-01-20T09:46:23Z", "activityTypeId": 25, "campaignId": 1094, "primaryAttributeValueId": "1017", "primaryAttributeValue": "API Test Program.List 2", "source": "Marketo Flow Action"}, "emitted_at": 1682446714343} -{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2166", "leadId": 104, "activityDate": "2023-01-20T09:46:53Z", "activityTypeId": 25, "campaignId": 1095, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1682446714343} -{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2167", "leadId": 801, "activityDate": "2023-01-20T09:47:23Z", "activityTypeId": 25, "campaignId": 1096, "primaryAttributeValueId": "1017", "primaryAttributeValue": "API Test Program.List 2", "source": "Marketo Flow Action"}, "emitted_at": 1682446714343} -{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2168", "leadId": 802, "activityDate": "2023-01-20T09:48:23Z", "activityTypeId": 25, "campaignId": 1097, "primaryAttributeValueId": "1017", "primaryAttributeValue": "API Test Program.List 2", "source": "Marketo Flow Action"}, "emitted_at": 1682446714344} -{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2137", "leadId": 807, "activityDate": "2023-01-18T11:25:57Z", "activityTypeId": 27, "campaignId": 1066, "primaryAttributeValueId": "1002", "primaryAttributeValue": "EM - Auteur - v1.EM - Auteur - v1", "campaign_run_id": 28, "category": "4", "email": "kjashaedd-13@klooblept.com", "details": "554 5.4.7 [internal] (last transfail: 454 4.4.4 [internal] no MX or A for domain)", "subcategory": "4003", "step_id": 110, "choice_number": 118}, "emitted_at": 1682446841225} -{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2138", "leadId": 816, "activityDate": "2023-01-18T11:26:56Z", "activityTypeId": 27, "campaignId": 1070, "primaryAttributeValueId": "1002", "primaryAttributeValue": "EM - Auteur - v1.EM - Auteur - v1", "campaign_run_id": 30, "category": "4", "email": "kjashaedd-37@klooblept.com", "details": "554 5.4.7 [internal] (last transfail: 454 4.4.4 [internal] no MX or A for domain)", "subcategory": "4003", "step_id": 126, "choice_number": 134}, "emitted_at": 1682446841225} -{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2240", "leadId": 99, "activityDate": "2023-01-20T16:21:27Z", "activityTypeId": 27, "campaignId": 1138, "primaryAttributeValueId": "1013", "primaryAttributeValue": "BP-LE-YYYY-MM-DD Live Event v3.EMAIL-Invitation", "campaign_run_id": 92, "category": "4", "email": "name2@example.com", "details": "551 5.7.0 [internal] recipient blackholed", "subcategory": "4006", "step_id": 417, "choice_number": 544}, "emitted_at": 1682446841226} -{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2256", "leadId": 99, "activityDate": "2023-01-22T11:57:55Z", "activityTypeId": 27, "campaignId": 1100, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 95, "category": "4", "email": "name2@example.com", "details": "551 5.7.0 [internal] recipient blackholed", "subcategory": "4006", "step_id": 245, "choice_number": 260}, "emitted_at": 1682446841226} -{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2257", "leadId": 379, "activityDate": "2023-01-22T11:57:56Z", "activityTypeId": 27, "campaignId": 1100, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 95, "category": "3", "email": "testemail@gmail.com", "details": "552-5.2.2 The email account that you tried to reach is over quota and inactive.\r\n552-5.2.2 Please direct the recipient to\r\n552 5.2.2 https://support.google.com/mail/?p=OverQuotaPerm c1-20020a62e801000000b0058df8aad481si11279798pfi.352 - gsmtp", "subcategory": "3001", "step_id": 245, "choice_number": 260}, "emitted_at": 1682446841226} -{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2284", "leadId": 800, "activityDate": "2023-01-23T11:58:00Z", "activityTypeId": 27, "campaignId": 1100, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 95, "category": "4", "email": "kjashaedd-1@klooblept.com", "details": "554 5.4.7 [internal] (last transfail: 454 4.4.4 [internal] no MX or A for domain)", "subcategory": "4003", "step_id": 245, "choice_number": 260}, "emitted_at": 1682446841227} -{"stream": "activities_merge_leads", "data": {"marketoGUID": "2126", "leadId": 864, "activityDate": "2023-01-17T11:29:43Z", "activityTypeId": 32, "campaignId": null, "primaryAttributeValueId": "864", "primaryAttributeValue": "yuriiyurii", "merge_ids": [887]}, "emitted_at": 1682446967868} -{"stream": "activities_merge_leads", "data": {"marketoGUID": "2184", "leadId": 809, "activityDate": "2023-01-20T11:44:22Z", "activityTypeId": 32, "campaignId": null, "primaryAttributeValueId": "809", "primaryAttributeValue": "Kataldar-30", "merge_ids": [811], "mergedin_sales": false, "merge_source": "leaddb", "master_updated": false}, "emitted_at": 1682446967868} -{"stream": "activities_send_alert", "data": {"marketoGUID": "2179", "leadId": 109, "activityDate": "2023-01-20T11:39:27Z", "activityTypeId": 38, "campaignId": 1107, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "send_to_owner": "Lead Owner", "send_to_list": "integration-tgest@airbyte.io"}, "emitted_at": 1682447094591} -{"stream": "activities_request_campaign", "data": {"marketoGUID": "2193", "leadId": 104, "activityDate": "2023-01-20T12:01:23Z", "activityTypeId": 47, "campaignId": 1113, "primaryAttributeValueId": "1056", "primaryAttributeValue": "Test Campaign 5", "source": "Marketo Flow Action"}, "emitted_at": 1682447221209} -{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2150", "leadId": 891, "activityDate": "2023-01-19T15:45:54Z", "activityTypeId": 100, "campaignId": 1077, "primaryAttributeValueId": "3", "primaryAttributeValue": "Test Partition 2", "old_partition_id": 2, "reason": "Lead Action"}, "emitted_at": 1682447347992} -{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2151", "leadId": 800, "activityDate": "2023-01-19T15:46:23Z", "activityTypeId": 100, "campaignId": 1079, "primaryAttributeValueId": "2", "primaryAttributeValue": "Test Partition", "old_partition_id": 1, "reason": "Lead Action"}, "emitted_at": 1682447347992} -{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2152", "leadId": 112, "activityDate": "2023-01-19T15:46:23Z", "activityTypeId": 100, "campaignId": 1078, "primaryAttributeValueId": "2", "primaryAttributeValue": "Test Partition", "old_partition_id": 1, "reason": "Lead Action"}, "emitted_at": 1682447347993} -{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2153", "leadId": 807, "activityDate": "2023-01-19T15:46:53Z", "activityTypeId": 100, "campaignId": 1080, "primaryAttributeValueId": "2", "primaryAttributeValue": "Test Partition", "old_partition_id": 1, "reason": "Lead Action"}, "emitted_at": 1682447347993} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2114", "leadId": 100, "activityDate": "2023-01-17T11:07:53Z", "activityTypeId": 104, "campaignId": 1061, "primaryAttributeValueId": "1013", "primaryAttributeValue": "API Test Program 10", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 120, "success": false}, "emitted_at": 1682447474741} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2122", "leadId": 807, "activityDate": "2023-01-17T11:26:23Z", "activityTypeId": 104, "campaignId": 1068, "primaryAttributeValueId": "1008", "primaryAttributeValue": "API Test Program 5", "old_status_id": 9, "new_status_id": 11, "acquired_by": false, "old_status": "Not in Program", "new_status": "Influenced", "program_member_id": 121, "success": true}, "emitted_at": 1682447474741} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2141", "leadId": 23, "activityDate": "2023-01-19T14:48:24Z", "activityTypeId": 104, "campaignId": 1075, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 122, "success": false}, "emitted_at": 1682447474742} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2156", "leadId": 105, "activityDate": "2023-01-20T09:41:23Z", "activityTypeId": 104, "campaignId": 1083, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 123, "success": false}, "emitted_at": 1682447474742} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2157", "leadId": 23, "activityDate": "2023-01-20T09:42:23Z", "activityTypeId": 104, "campaignId": 1084, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program 3", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 124, "success": false}, "emitted_at": 1682447474743} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2158", "leadId": 100, "activityDate": "2023-01-20T09:42:53Z", "activityTypeId": 104, "campaignId": 1086, "primaryAttributeValueId": "1008", "primaryAttributeValue": "API Test Program 5", "old_status_id": 9, "new_status_id": 11, "acquired_by": false, "old_status": "Not in Program", "new_status": "Influenced", "program_member_id": 125, "success": true}, "emitted_at": 1682447474743} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2159", "leadId": 99, "activityDate": "2023-01-20T09:42:53Z", "activityTypeId": 104, "campaignId": 1085, "primaryAttributeValueId": "1013", "primaryAttributeValue": "API Test Program 10", "old_status_id": 9, "new_status_id": 11, "acquired_by": false, "old_status": "Not in Program", "new_status": "Influenced", "program_member_id": 126, "success": true}, "emitted_at": 1682447474744} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2160", "leadId": 104, "activityDate": "2023-01-20T09:43:23Z", "activityTypeId": 104, "campaignId": 1088, "primaryAttributeValueId": "1010", "primaryAttributeValue": "API Test Program 7", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 127, "success": false}, "emitted_at": 1682447474744} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2161", "leadId": 105, "activityDate": "2023-01-20T09:43:53Z", "activityTypeId": 104, "campaignId": 1089, "primaryAttributeValueId": "1011", "primaryAttributeValue": "API Test Program 8", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 128, "success": false}, "emitted_at": 1682447474745} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2171", "leadId": 99, "activityDate": "2023-01-20T09:57:53Z", "activityTypeId": 104, "campaignId": 1102, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "old_status_id": 30, "new_status_id": 31, "acquired_by": false, "old_status": "Not in Program", "new_status": "Member", "program_member_id": 129, "success": false}, "emitted_at": 1682447474745} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2174", "leadId": 800, "activityDate": "2023-01-20T09:59:53Z", "activityTypeId": 104, "campaignId": 1103, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "old_status_id": 30, "new_status_id": 31, "acquired_by": false, "old_status": "Not in Program", "new_status": "Member", "program_member_id": 130, "success": false}, "emitted_at": 1682447474745} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2191", "leadId": 105, "activityDate": "2023-01-20T11:53:23Z", "activityTypeId": 104, "campaignId": 1112, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "old_status_id": 30, "new_status_id": 31, "acquired_by": false, "old_status": "Not in Program", "new_status": "Member", "program_member_id": 131, "success": false}, "emitted_at": 1682447474746} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2204", "leadId": 876, "activityDate": "2023-01-20T12:48:53Z", "activityTypeId": 104, "campaignId": 1125, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "old_status_id": 30, "new_status_id": 31, "acquired_by": false, "old_status": "Not in Program", "new_status": "Member", "program_member_id": 132, "success": false}, "emitted_at": 1682447474746} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2220", "leadId": 379, "activityDate": "2023-01-20T13:00:24Z", "activityTypeId": 104, "campaignId": 1127, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "old_status_id": 30, "new_status_id": 31, "acquired_by": false, "old_status": "Not in Program", "new_status": "Member", "program_member_id": 133, "success": false}, "emitted_at": 1682447474747} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2221", "leadId": 110, "activityDate": "2023-01-20T13:06:23Z", "activityTypeId": 104, "campaignId": 1128, "primaryAttributeValueId": "1003", "primaryAttributeValue": "API Test Program", "old_status_id": 49, "new_status_id": 51, "acquired_by": false, "old_status": "Not in Program", "new_status": "Opened", "program_member_id": 134, "success": false}, "emitted_at": 1682447474747} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2222", "leadId": 110, "activityDate": "2023-01-20T13:08:23Z", "activityTypeId": 104, "campaignId": 1129, "primaryAttributeValueId": "1003", "primaryAttributeValue": "API Test Program", "old_status_id": 51, "new_status_id": 52, "acquired_by": false, "old_status": "Opened", "new_status": "Clicked", "program_member_id": 134, "success": false}, "emitted_at": 1682447474747} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2241", "leadId": 99, "activityDate": "2023-01-20T16:27:53Z", "activityTypeId": 104, "campaignId": 1139, "primaryAttributeValueId": "1013", "primaryAttributeValue": "API Test Program 10", "old_status_id": 11, "new_status_id": 10, "acquired_by": false, "old_status": "Influenced", "new_status": "Filled-out Form", "program_member_id": 126, "success": false}, "emitted_at": 1682447474748} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2264", "leadId": 100, "activityDate": "2023-01-23T08:14:22Z", "activityTypeId": 104, "campaignId": 1155, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "old_status_id": 9, "new_status_id": 11, "acquired_by": false, "old_status": "Not in Program", "new_status": "Influenced", "program_member_id": 135, "success": true}, "emitted_at": 1682447474748} -{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2269", "leadId": 880, "activityDate": "2023-01-23T08:24:51Z", "activityTypeId": 104, "campaignId": 1158, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "old_status_id": 30, "new_status_id": 31, "acquired_by": false, "old_status": "Not in Program", "new_status": "Member", "program_member_id": 136, "success": false}, "emitted_at": 1682447474749} -{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2169", "leadId": 99, "activityDate": "2023-01-20T09:57:53Z", "activityTypeId": 113, "campaignId": 1102, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1682447601643} -{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2172", "leadId": 800, "activityDate": "2023-01-20T09:59:53Z", "activityTypeId": 113, "campaignId": 1103, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1682447601643} -{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2189", "leadId": 105, "activityDate": "2023-01-20T11:53:23Z", "activityTypeId": 113, "campaignId": 1112, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1682447601643} -{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2202", "leadId": 876, "activityDate": "2023-01-20T12:48:53Z", "activityTypeId": 113, "campaignId": 1125, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1682447601643} -{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2218", "leadId": 379, "activityDate": "2023-01-20T13:00:24Z", "activityTypeId": 113, "campaignId": 1127, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1682447601644} -{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2267", "leadId": 880, "activityDate": "2023-01-23T08:24:51Z", "activityTypeId": 113, "campaignId": 1158, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1682447601644} -{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2205", "leadId": 876, "activityDate": "2023-01-20T12:48:53Z", "activityTypeId": 114, "campaignId": 1125, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_name": "Stream 1", "new_track_id": 1}, "emitted_at": 1682447728227} -{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2260", "leadId": 99, "activityDate": "2023-01-23T08:07:22Z", "activityTypeId": 114, "campaignId": 1150, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "previous_track_id": 1, "previous_track_name": "Stream 1", "track_name": "Stream 2", "new_track_id": 2}, "emitted_at": 1682447728227} -{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2270", "leadId": 880, "activityDate": "2023-01-23T08:24:52Z", "activityTypeId": 114, "campaignId": 1158, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_name": "Stream 1", "new_track_id": 1}, "emitted_at": 1682447728227} -{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2276", "leadId": 880, "activityDate": "2023-01-23T08:26:21Z", "activityTypeId": 114, "campaignId": 1158, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "previous_track_id": 1, "previous_track_name": "Stream 1", "track_name": "Stream 1", "new_track_id": 1}, "emitted_at": 1682447728228} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2170", "leadId": 99, "activityDate": "2023-01-20T09:57:53Z", "activityTypeId": 115, "campaignId": 1102, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1682447854893} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2173", "leadId": 800, "activityDate": "2023-01-20T09:59:53Z", "activityTypeId": 115, "campaignId": 1103, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1682447854894} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2190", "leadId": 105, "activityDate": "2023-01-20T11:53:23Z", "activityTypeId": 115, "campaignId": 1112, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1682447854894} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2203", "leadId": 876, "activityDate": "2023-01-20T12:48:53Z", "activityTypeId": 115, "campaignId": 1125, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1682447854894} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2219", "leadId": 379, "activityDate": "2023-01-20T13:00:24Z", "activityTypeId": 115, "campaignId": 1127, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1682447854895} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2262", "leadId": 99, "activityDate": "2023-01-23T08:08:21Z", "activityTypeId": 115, "campaignId": 1152, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm", "previous_nurture_cadence": "Norm"}, "emitted_at": 1682447854895} -{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2268", "leadId": 880, "activityDate": "2023-01-23T08:24:51Z", "activityTypeId": 115, "campaignId": 1158, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1682447854896} -{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2188", "leadId": 105, "activityDate": "2023-01-20T11:52:53Z", "activityTypeId": 123, "campaignId": 1111, "primaryAttributeValueId": "1013", "primaryAttributeValue": "API Test Program 10", "source": "Marketo Flow Action"}, "emitted_at": 1682447981328} -{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2228", "leadId": 110, "activityDate": "2023-01-20T16:04:24Z", "activityTypeId": 123, "campaignId": 1136, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "source": "Marketo Flow Action"}, "emitted_at": 1682447981329} -{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2229", "leadId": 99, "activityDate": "2023-01-20T16:05:23Z", "activityTypeId": 123, "campaignId": 1137, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "source": "Marketo Flow Action"}, "emitted_at": 1682447981329} -{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2242", "leadId": 800, "activityDate": "2023-01-20T16:28:53Z", "activityTypeId": 123, "campaignId": 1140, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "source": "Marketo Flow Action"}, "emitted_at": 1682447981330} -{"stream": "activities_execute_campaign", "data": {"marketoGUID": "2178", "leadId": 105, "activityDate": "2023-01-20T11:36:54Z", "activityTypeId": 155, "campaignId": 1106, "primaryAttributeValueId": "1105", "primaryAttributeValue": "Test Campaign 10", "used_parent_campaign_token_context": false, "qualified": false}, "emitted_at": 1682448107771} -{"stream": "segmentations", "data": {"id": 1001, "name": "Smart List1", "description": "Description of the Smart List1", "createdAt": "2023-01-17T11:04:34Z+0000", "updatedAt": "2023-01-20T09:17:13Z+0000", "url": "https://app-sj32.marketo.com/#SG1001A1", "folder": {"type": "Program", "value": 16, "folderName": null}, "status": "draft", "workspace": "Default"}, "emitted_at": 1681839965388} -{"stream": "segmentations", "data": {"id": 1002, "name": "Segment 5", "description": "Description of the Segment 5", "createdAt": "2023-01-17T11:10:22Z+0000", "updatedAt": "2023-01-17T11:10:22Z+0000", "url": "https://app-sj32.marketo.com/#SG1002A1", "folder": {"type": "Program", "value": 16, "folderName": null}, "status": "draft", "workspace": "Default"}, "emitted_at": 1681839965389} +{"stream": "activity_types", "data": {"id": 1, "name": "Visit Webpage", "description": "User visits a web page", "primaryAttribute": {"name": "Webpage ID", "dataType": "integer"}, "attributes": [{"name": "Webpage URL", "dataType": "string"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Client IP Address", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}, {"name": "Search Engine", "dataType": "string"}, {"name": "Search Query", "dataType": "string"}]}, "emitted_at": 1691573642575} +{"stream": "activity_types", "data": {"id": 2, "name": "Fill Out Form", "description": "User fills out and submits a form on web page", "primaryAttribute": {"name": "Webform ID", "dataType": "integer"}, "attributes": [{"name": "Form Fields", "dataType": "text"}, {"name": "Webpage ID", "dataType": "integer"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Client IP Address", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}]}, "emitted_at": 1691573642576} +{"stream": "activity_types", "data": {"id": 3, "name": "Click Link", "description": "User clicks link on a page", "primaryAttribute": {"name": "Link ID", "dataType": "integer"}, "attributes": [{"name": "Webpage ID", "dataType": "integer"}, {"name": "Query Parameters", "dataType": "string"}, {"name": "Client IP Address", "dataType": "string"}, {"name": "Referrer URL", "dataType": "string"}, {"name": "User Agent", "dataType": "string"}]}, "emitted_at": 1691573642576} +{"stream": "segmentations", "data": {"id": 1001, "name": "Smart List1", "description": "Description of the Smart List1", "createdAt": "2023-01-17T11:04:34Z+0000", "updatedAt": "2023-01-20T09:17:13Z+0000", "url": "https://app-sj32.marketo.com/#SG1001A1", "folder": {"type": "Program", "value": 16, "folderName": null}, "status": "draft", "workspace": "Default"}, "emitted_at": 1691573643535} +{"stream": "segmentations", "data": {"id": 1002, "name": "Segment 5", "description": "Description of the Segment 5", "createdAt": "2023-01-17T11:10:22Z+0000", "updatedAt": "2023-01-17T11:10:22Z+0000", "url": "https://app-sj32.marketo.com/#SG1002A1", "folder": {"type": "Program", "value": 16, "folderName": null}, "status": "draft", "workspace": "Default"}, "emitted_at": 1691573643536} +{"stream": "campaigns", "data": {"id": 1053, "name": "Test Campaign 2", "description": "Description of the Test Campaign 2", "type": "batch", "workspaceName": "Default", "createdAt": "2023-01-17T10:53:35Z", "updatedAt": "2023-01-17T10:56:52Z", "active": false}, "emitted_at": 1691573644721} +{"stream": "campaigns", "data": {"id": 1054, "name": "Test Campaign 3", "description": "Description of the Test Campaign 3", "type": "batch", "workspaceName": "Default", "createdAt": "2023-01-17T10:54:04Z", "updatedAt": "2023-01-17T10:57:12Z", "active": false}, "emitted_at": 1691573644721} +{"stream": "campaigns", "data": {"id": 1055, "name": "Test Campaign 4", "description": "Description of the Test Campaign 4", "type": "batch", "workspaceName": "Default", "createdAt": "2023-01-17T10:54:24Z", "updatedAt": "2023-01-17T10:53:37Z", "active": false}, "emitted_at": 1691573644722} +{"stream": "leads", "data": {"company": "Test Company", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 885, "mktoName": "Test Person 1 All People", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Test Person 1", "middleName": null, "lastName": "All People", "email": "0M996mK4pH@ttt.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "89", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-17T11:00:58Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "885", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "885", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1691573710449} +{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": "airbyte.io", "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 888, "mktoName": "[Sample]Name Surname", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "[Sample]Name", "middleName": null, "lastName": "Surname", "email": "i.g@example.io", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "90", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": "Asia/Riyadh", "originalSourceType": "Web form fillout", "originalSourceInfo": "Form: API Test Program.123.airbyte", "registrationSourceType": "Web form fillout", "registrationSourceInfo": "Form: API Test Program.123.airbyte", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": "85.209.47.207", "inferredCompany": "Arabic Computer System Co.", "inferredCountry": "Saudi Arabia", "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-17T12:29:38Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "888", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "888", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1691573710451} +{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": "globallogic.com", "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 889, "mktoName": "[Sample] sample", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "[Sample]", "middleName": null, "lastName": "sample", "email": "i.g@example.io", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "91", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": "Asia/Riyadh", "originalSourceType": "Web form fillout", "originalSourceInfo": "Form: API Test Program.123.airbyte", "registrationSourceType": "Web form fillout", "registrationSourceInfo": "Form: API Test Program.123.airbyte", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": "85.209.47.207", "inferredCompany": "Arabic Computer System Co.", "inferredCountry": "Saudi Arabia", "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2023-01-17T12:30:09Z", "updatedAt": "2023-01-24T11:10:51Z", "cookies": "_mch-mktoweb.com-1673954921699-88079", "externalSalesPersonId": null, "leadPerson": "889", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "889", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1691573710452} +{"stream": "lists", "data": {"id": 1016, "name": "List 1", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:43:09Z", "updatedAt": "2023-01-20T13:11:03Z"}, "emitted_at": 1691573711748} +{"stream": "lists", "data": {"id": 1017, "name": "List 2", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:43:41Z", "updatedAt": "2023-01-17T12:43:43Z"}, "emitted_at": 1691573711749} +{"stream": "lists", "data": {"id": 1018, "name": "List 3", "description": "Test", "programName": "API Test Program", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2023-01-17T12:43:59Z", "updatedAt": "2023-01-17T12:44:00Z"}, "emitted_at": 1691573711749} +{"stream": "programs", "data": {"id": 1016, "name": "123", "description": "", "createdAt": "2021-09-01T16:02:30Z", "updatedAt": "2023-01-17T12:36:46Z", "url": "https://app-sj32.marketo.com/#EBP1016A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "completed", "workspace": "Default", "headStart": false}, "emitted_at": 1691573713032} +{"stream": "programs", "data": {"id": 1017, "name": "air", "description": "", "createdAt": "2021-09-01T16:09:23Z", "updatedAt": "2023-01-17T12:31:23Z", "url": "https://app-sj32.marketo.com/#EBP1017A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "completed", "workspace": "Default", "headStart": false}, "emitted_at": 1691573713033} +{"stream": "programs", "data": {"id": 1003, "name": "API Test Program", "description": "Sample API Program", "createdAt": "2021-01-18T13:55:44Z", "updatedAt": "2023-01-20T13:02:44Z", "url": "https://app-sj32.marketo.com/#PG1003A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default"}, "emitted_at": 1691573713033} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2125", "leadId": 864, "activityDate": "2023-01-17T11:29:20Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "10", "primaryAttributeValue": "Landing-Page-1", "webpage_url": "/lp/602-EUO-598/Landing-Page-1.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "https://602-euo-598.mktoweb.com/"}, "emitted_at": 1691573840265} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2132", "leadId": 888, "activityDate": "2023-01-17T12:29:28Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "11", "primaryAttributeValue": "Landing-Page-2", "webpage_url": "/lp/602-EUO-598/Landing-Page-2.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1691573840267} +{"stream": "activities_visit_webpage", "data": {"marketoGUID": "2133", "leadId": 888, "activityDate": "2023-01-17T12:29:51Z", "activityTypeId": 1, "campaignId": null, "primaryAttributeValueId": "12", "primaryAttributeValue": "Landing-Page-3", "webpage_url": "/lp/602-EUO-598/Landing-Page-3.html", "client_ip_address": "85.209.47.207", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/"}, "emitted_at": 1691573840267} +{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2129", "leadId": 864, "activityDate": "2023-01-17T11:29:43Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:7:\"Segment\";s:8:\"LastName\";s:1:\"5\";s:5:\"Email\";s:27:\"integration-test@airbyte.io\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1005\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-1.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1673954921699-88079\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:66:\"https://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-1.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"774b4fee27ce1a145370b5ffebe8f4e300c1404cfd86a2067763c65b063d1b66\";s:25:\"formServiceRequestId31337\";s:17:\"17915#185bf7ded1f\";}", "client_ip_address": "85.209.47.207", "webpage_id": 10, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "https://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-1.html"}, "emitted_at": 1691573969384} +{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2131", "leadId": 888, "activityDate": "2023-01-17T12:29:40Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:12:\"[Sample]Name\";s:8:\"LastName\";s:7:\"Surname\";s:5:\"Email\";s:25:\"iryna.grankova@airbyte.io\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1007\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1673954921699-88079\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:65:\"http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"31c81f3245d5080d0bb93a07f67ae67a0c4667adcce1400cee8857206f656981\";s:25:\"formServiceRequestId31337\";s:17:\"161b7#185bfb4d8a8\";}", "client_ip_address": "85.209.47.207", "webpage_id": 11, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-2.html"}, "emitted_at": 1691573969385} +{"stream": "activities_fill_out_form", "data": {"marketoGUID": "2135", "leadId": 889, "activityDate": "2023-01-17T12:30:10Z", "activityTypeId": 2, "campaignId": null, "primaryAttributeValueId": "1006", "primaryAttributeValue": "API Test Program.123.airbyte", "form_fields": "a:19:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:9:\"FirstName\";s:8:\"[Sample]\";s:8:\"LastName\";s:6:\"sample\";s:5:\"Email\";s:30:\"iryna.grankova@globallogic.com\";s:6:\"formid\";s:4:\"1006\";s:4:\"lpId\";s:4:\"1009\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:92:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html?cr={creative}&kw={keyword}\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1673954921699-88079\";s:7:\"formVid\";s:4:\"1006\";s:13:\"_mktoReferrer\";s:65:\"http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html\";s:14:\"checksumFields\";s:98:\"FirstName,LastName,Email,formid,lpId,subId,munchkinId,lpurl,cr,kw,q,_mkt_trk,formVid,_mktoReferrer\";s:8:\"checksum\";s:64:\"c7f8c6af1167c273c27bc3b441b5be79672d5a6d61f66398eba04782ca972195\";s:25:\"formServiceRequestId31337\";s:17:\"16fe4#185bfb55182\";}", "client_ip_address": "85.209.47.207", "webpage_id": 12, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 Edg/109.0.1518.52", "query_parameters": null, "referrer_url": "http://602-euo-598.mktoweb.com/lp/602-EUO-598/Landing-Page-3.html"}, "emitted_at": 1691573969386} +{"stream": "activities_send_email", "data": {"marketoGUID": "2185", "leadId": 109, "activityDate": "2023-01-20T11:45:57Z", "activityTypeId": 6, "campaignId": 1109, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 66, "step_id": 305, "choice_number": 0}, "emitted_at": 1691574223778} +{"stream": "activities_send_email", "data": {"marketoGUID": "2206", "leadId": 876, "activityDate": "2023-01-20T12:49:27Z", "activityTypeId": 6, "campaignId": 1126, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 86, "step_id": 364, "choice_number": 0}, "emitted_at": 1691574223779} +{"stream": "activities_send_email", "data": {"marketoGUID": "2239", "leadId": 99, "activityDate": "2023-01-20T16:21:28Z", "activityTypeId": 6, "campaignId": 1138, "primaryAttributeValueId": "1013", "primaryAttributeValue": "BP-LE-YYYY-MM-DD Live Event v3.EMAIL-Invitation", "campaign_run_id": 92, "step_id": 417, "choice_number": 0}, "emitted_at": 1691574223780} +{"stream": "activities_email_delivered", "data": {"marketoGUID": "2186", "leadId": 109, "activityDate": "2023-01-20T11:45:58Z", "activityTypeId": 7, "campaignId": 1109, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 66, "step_id": 305, "choice_number": 407}, "emitted_at": 1691574350613} +{"stream": "activities_email_delivered", "data": {"marketoGUID": "2208", "leadId": 876, "activityDate": "2023-01-20T12:49:29Z", "activityTypeId": 7, "campaignId": 1126, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 86, "step_id": 364, "choice_number": 470}, "emitted_at": 1691574350615} +{"stream": "activities_email_bounced", "data": {"marketoGUID": "2273", "leadId": 880, "activityDate": "2023-01-23T08:25:26Z", "activityTypeId": 8, "campaignId": 1159, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 112, "category": "2", "email": "alastor@moody.com", "details": "550 5.4.1 Recipient address rejected: Access denied. AS(201806281) [SN1NAM02FT0043.eop-nam02.prod.protection.outlook.com]", "subcategory": "2003", "step_id": 489, "choice_number": 623}, "emitted_at": 1691574477148} +{"stream": "activities_unsubscribe_email", "data": {"marketoGUID": "2201", "leadId": 864, "activityDate": "2023-01-20T12:40:36Z", "activityTypeId": 9, "campaignId": null, "primaryAttributeValueId": "1010", "primaryAttributeValue": "My Email Program - RF.Test Email 1", "webform_id": 1, "client_ip_address": "85.209.47.207", "form_fields": "a:20:{s:6:\"module\";s:11:\"leadCapture\";s:6:\"action\";s:5:\"save2\";s:5:\"Email\";s:27:\"integration-test@airbyte.io\";s:12:\"Unsubscribed\";s:3:\"Yes\";s:6:\"formid\";s:1:\"1\";s:4:\"lpId\";s:1:\"1\";s:5:\"subId\";s:3:\"154\";s:10:\"munchkinId\";s:11:\"602-EUO-598\";s:5:\"lpurl\";s:93:\"http://602-EUO-598.mktoweb.com/lp/602-EUO-598/UnsubscribePage.html?cr={creative}&kw={keyword}\";s:12:\"followupLpId\";s:1:\"2\";s:2:\"cr\";s:0:\"\";s:2:\"kw\";s:0:\"\";s:1:\"q\";s:0:\"\";s:8:\"_mkt_trk\";s:57:\"id:602-EUO-598&token:_mch-mktoweb.com-1674218381862-27778\";s:7:\"formVid\";s:1:\"1\";s:7:\"mkt_tok\";s:102:\"NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg\";s:13:\"_mktoReferrer\";s:197:\"http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg\";s:14:\"checksumFields\";s:113:\"Email,Unsubscribed,formid,lpId,subId,munchkinId,lpurl,followupLpId,cr,kw,q,_mkt_trk,formVid,mkt_tok,_mktoReferrer\";s:8:\"checksum\";s:64:\"b959482d2aff7c190b46cc567e79fce9c99a6342fbe1eb3fd863ec9a6ac1159a\";s:25:\"formServiceRequestId31337\";s:16:\"7e0e#185cf31f199\";}", "webpage_id": 1, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", "query_parameters": "mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg", "referrer_url": "http://602-euo-598.mktoweb.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAGJbdICA9DCAo37JqD5ozxyzAlhl8ryjBs-YDMZdHsGTFeOe-C6dFcD19gwXxmu7jXj3HO-IA2VTSiA7upNlg"}, "emitted_at": 1691574603915} +{"stream": "activities_click_email", "data": {"marketoGUID": "2213", "leadId": 876, "activityDate": "2023-01-20T12:57:10Z", "activityTypeId": 11, "campaignId": 1126, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "campaign_run_id": 86, "platform": "Win7", "is_mobile_device": false, "step_id": 364, "device": "Windows Desktop", "choice_number": 470, "is_bot_activity": false, "user_agent": "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", "bot_activity_pattern": "N/A", "link": "http://mylink"}, "emitted_at": 1691574858008} +{"stream": "activities_new_lead", "data": {"marketoGUID": "2108", "leadId": 885, "activityDate": "2023-01-17T11:00:59Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "885", "primaryAttributeValue": "Test Person 1 All People", "created_date": "2023-01-17", "source_type": "New lead"}, "emitted_at": 1691574984967} +{"stream": "activities_new_lead", "data": {"marketoGUID": "2130", "leadId": 888, "activityDate": "2023-01-17T12:29:39Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "888", "primaryAttributeValue": "[Sample]Name Surname", "form_name": "API Test Program.123.airbyte", "created_date": "2023-01-17", "source_type": "Web form fillout"}, "emitted_at": 1691574984968} +{"stream": "activities_new_lead", "data": {"marketoGUID": "2134", "leadId": 889, "activityDate": "2023-01-17T12:30:09Z", "activityTypeId": 12, "campaignId": null, "primaryAttributeValueId": "889", "primaryAttributeValue": "[Sample] sample", "form_name": "API Test Program.123.airbyte", "created_date": "2023-01-17", "source_type": "Web form fillout"}, "emitted_at": 1691574984968} +{"stream": "activities_change_data_value", "data": {"marketoGUID": "2109", "leadId": 100, "activityDate": "2023-01-17T11:06:30Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "50", "primaryAttributeValue": "Is Customer", "old_value": "False", "new_value": "True", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1691575112456} +{"stream": "activities_change_data_value", "data": {"marketoGUID": "2115", "leadId": 140, "activityDate": "2023-01-17T11:08:10Z", "activityTypeId": 13, "campaignId": null, "primaryAttributeValueId": "48", "primaryAttributeValue": "Is Partner", "old_value": "False", "new_value": "True", "reason": "Manual person edit", "source": "Lead update"}, "emitted_at": 1691575112457} +{"stream": "activities_change_data_value", "data": {"marketoGUID": "2119", "leadId": 140, "activityDate": "2023-01-17T11:09:23Z", "activityTypeId": 13, "campaignId": 1064, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "old_value": null, "new_value": "10", "reason": "Changed by Smart Campaign Run Action Change Score 2023-01-17 03:09:02 am action Change Score", "source": "Marketo Flow Action"}, "emitted_at": 1691575112458} +{"stream": "activities_change_score", "data": {"marketoGUID": "2118", "leadId": 140, "activityDate": "2023-01-17T11:09:23Z", "activityTypeId": 22, "campaignId": 1064, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "change_value": "+10", "old_value": null, "new_value": 10, "reason": "Changed by Smart Campaign Run Action Change Score 2023-01-17 03:09:02 am action Change Score"}, "emitted_at": 1691575239887} +{"stream": "activities_change_score", "data": {"marketoGUID": "2209", "leadId": 143, "activityDate": "2023-01-20T12:55:59Z", "activityTypeId": 22, "campaignId": null, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "change_value": "+20", "old_value": null, "new_value": 20, "reason": "Manual person edit"}, "emitted_at": 1691575239887} +{"stream": "activities_change_score", "data": {"marketoGUID": "2211", "leadId": 362, "activityDate": "2023-01-20T12:56:46Z", "activityTypeId": 22, "campaignId": null, "primaryAttributeValueId": "146", "primaryAttributeValue": "Lead Score", "change_value": "+100", "old_value": null, "new_value": 100, "reason": "Manual person edit"}, "emitted_at": 1691575239888} +{"stream": "activities_addto_list", "data": {"marketoGUID": "2110", "leadId": 100, "activityDate": "2023-01-17T11:07:23Z", "activityTypeId": 24, "campaignId": 1059, "primaryAttributeValueId": "1002", "primaryAttributeValue": "Test list number 1", "source": "Marketo Flow Action"}, "emitted_at": 1691575367894} +{"stream": "activities_addto_list", "data": {"marketoGUID": "2111", "leadId": 100, "activityDate": "2023-01-17T11:07:24Z", "activityTypeId": 24, "campaignId": 1058, "primaryAttributeValueId": "1001", "primaryAttributeValue": "Test list", "source": "Marketo Flow Action"}, "emitted_at": 1691575367895} +{"stream": "activities_addto_list", "data": {"marketoGUID": "2116", "leadId": 140, "activityDate": "2023-01-17T11:08:53Z", "activityTypeId": 24, "campaignId": 1063, "primaryAttributeValueId": "1004", "primaryAttributeValue": "Test list number 3", "source": "Marketo Flow Action"}, "emitted_at": 1691575367895} +{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2165", "leadId": 379, "activityDate": "2023-01-20T09:46:23Z", "activityTypeId": 25, "campaignId": 1094, "primaryAttributeValueId": "1017", "primaryAttributeValue": "API Test Program.List 2", "source": "Marketo Flow Action"}, "emitted_at": 1691575497844} +{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2166", "leadId": 104, "activityDate": "2023-01-20T09:46:53Z", "activityTypeId": 25, "campaignId": 1095, "primaryAttributeValueId": "1016", "primaryAttributeValue": "API Test Program.List 1", "source": "Marketo Flow Action"}, "emitted_at": 1691575497845} +{"stream": "activities_removefrom_list", "data": {"marketoGUID": "2167", "leadId": 801, "activityDate": "2023-01-20T09:47:23Z", "activityTypeId": 25, "campaignId": 1096, "primaryAttributeValueId": "1017", "primaryAttributeValue": "API Test Program.List 2", "source": "Marketo Flow Action"}, "emitted_at": 1691575497845} +{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2137", "leadId": 807, "activityDate": "2023-01-18T11:25:57Z", "activityTypeId": 27, "campaignId": 1066, "primaryAttributeValueId": "1002", "primaryAttributeValue": "EM - Auteur - v1.EM - Auteur - v1", "campaign_run_id": 28, "category": "4", "email": "kjashaedd-13@klooblept.com", "details": "554 5.4.7 [internal] (last transfail: 454 4.4.4 [internal] no MX or A for domain)", "subcategory": "4003", "step_id": 110, "choice_number": 118}, "emitted_at": 1691575628465} +{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2138", "leadId": 816, "activityDate": "2023-01-18T11:26:56Z", "activityTypeId": 27, "campaignId": 1070, "primaryAttributeValueId": "1002", "primaryAttributeValue": "EM - Auteur - v1.EM - Auteur - v1", "campaign_run_id": 30, "category": "4", "email": "kjashaedd-37@klooblept.com", "details": "554 5.4.7 [internal] (last transfail: 454 4.4.4 [internal] no MX or A for domain)", "subcategory": "4003", "step_id": 126, "choice_number": 134}, "emitted_at": 1691575628466} +{"stream": "activities_email_bounced_soft", "data": {"marketoGUID": "2240", "leadId": 99, "activityDate": "2023-01-20T16:21:27Z", "activityTypeId": 27, "campaignId": 1138, "primaryAttributeValueId": "1013", "primaryAttributeValue": "BP-LE-YYYY-MM-DD Live Event v3.EMAIL-Invitation", "campaign_run_id": 92, "category": "4", "email": "name2@example.com", "details": "551 5.7.0 [internal] recipient blackholed", "subcategory": "4006", "step_id": 417, "choice_number": 544}, "emitted_at": 1691575628466} +{"stream": "activities_merge_leads", "data": {"marketoGUID": "2126", "leadId": 864, "activityDate": "2023-01-17T11:29:43Z", "activityTypeId": 32, "campaignId": null, "primaryAttributeValueId": "864", "primaryAttributeValue": "yuriiyurii", "merge_ids": [887]}, "emitted_at": 1691575755576} +{"stream": "activities_merge_leads", "data": {"marketoGUID": "2184", "leadId": 809, "activityDate": "2023-01-20T11:44:22Z", "activityTypeId": 32, "campaignId": null, "primaryAttributeValueId": "809", "primaryAttributeValue": "Kataldar-30", "merge_ids": [811], "mergedin_sales": false, "merge_source": "leaddb", "master_updated": false}, "emitted_at": 1691575755577} +{"stream": "activities_send_alert", "data": {"marketoGUID": "2179", "leadId": 109, "activityDate": "2023-01-20T11:39:27Z", "activityTypeId": 38, "campaignId": 1107, "primaryAttributeValueId": "1009", "primaryAttributeValue": "Test Nurture.integration-tgest@airbyte.io", "send_to_owner": "Lead Owner", "send_to_list": "integration-tgest@airbyte.io"}, "emitted_at": 1691576390592} +{"stream": "activities_request_campaign", "data": {"marketoGUID": "2193", "leadId": 104, "activityDate": "2023-01-20T12:01:23Z", "activityTypeId": 47, "campaignId": 1113, "primaryAttributeValueId": "1056", "primaryAttributeValue": "Test Campaign 5", "source": "Marketo Flow Action"}, "emitted_at": 1691577025384} +{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2150", "leadId": 891, "activityDate": "2023-01-19T15:45:54Z", "activityTypeId": 100, "campaignId": 1077, "primaryAttributeValueId": "3", "primaryAttributeValue": "Test Partition 2", "old_partition_id": 2, "reason": "Lead Action"}, "emitted_at": 1691577279987} +{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2151", "leadId": 800, "activityDate": "2023-01-19T15:46:23Z", "activityTypeId": 100, "campaignId": 1079, "primaryAttributeValueId": "2", "primaryAttributeValue": "Test Partition", "old_partition_id": 1, "reason": "Lead Action"}, "emitted_at": 1691577279989} +{"stream": "activities_change_lead_partition", "data": {"marketoGUID": "2152", "leadId": 112, "activityDate": "2023-01-19T15:46:23Z", "activityTypeId": 100, "campaignId": 1078, "primaryAttributeValueId": "2", "primaryAttributeValue": "Test Partition", "old_partition_id": 1, "reason": "Lead Action"}, "emitted_at": 1691577279989} +{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2114", "leadId": 100, "activityDate": "2023-01-17T11:07:53Z", "activityTypeId": 104, "campaignId": 1061, "primaryAttributeValueId": "1013", "primaryAttributeValue": "API Test Program 10", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 120, "success": false}, "emitted_at": 1691577660166} +{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2122", "leadId": 807, "activityDate": "2023-01-17T11:26:23Z", "activityTypeId": 104, "campaignId": 1068, "primaryAttributeValueId": "1008", "primaryAttributeValue": "API Test Program 5", "old_status_id": 9, "new_status_id": 11, "acquired_by": false, "old_status": "Not in Program", "new_status": "Influenced", "program_member_id": 121, "success": true}, "emitted_at": 1691577660167} +{"stream": "activities_change_statusin_progression", "data": {"marketoGUID": "2141", "leadId": 23, "activityDate": "2023-01-19T14:48:24Z", "activityTypeId": 104, "campaignId": 1075, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "old_status_id": 9, "new_status_id": 10, "acquired_by": false, "old_status": "Not in Program", "new_status": "Filled-out Form", "program_member_id": 122, "success": false}, "emitted_at": 1691577660168} +{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2169", "leadId": 99, "activityDate": "2023-01-20T09:57:53Z", "activityTypeId": 113, "campaignId": 1102, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1691578293441} +{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2172", "leadId": 800, "activityDate": "2023-01-20T09:59:53Z", "activityTypeId": 113, "campaignId": 1103, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1691578293442} +{"stream": "activities_addto_nurture", "data": {"marketoGUID": "2189", "leadId": 105, "activityDate": "2023-01-20T11:53:23Z", "activityTypeId": 113, "campaignId": 1112, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_id": 1, "track_name": "Stream 1"}, "emitted_at": 1691578293443} +{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2205", "leadId": 876, "activityDate": "2023-01-20T12:48:53Z", "activityTypeId": 114, "campaignId": 1125, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_name": "Stream 1", "new_track_id": 1}, "emitted_at": 1691578419995} +{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2260", "leadId": 99, "activityDate": "2023-01-23T08:07:22Z", "activityTypeId": 114, "campaignId": 1150, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "previous_track_id": 1, "previous_track_name": "Stream 1", "track_name": "Stream 2", "new_track_id": 2}, "emitted_at": 1691578419996} +{"stream": "activities_change_nurture_track", "data": {"marketoGUID": "2270", "leadId": 880, "activityDate": "2023-01-23T08:24:52Z", "activityTypeId": 114, "campaignId": 1158, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "track_name": "Stream 1", "new_track_id": 1}, "emitted_at": 1691578419997} +{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2170", "leadId": 99, "activityDate": "2023-01-20T09:57:53Z", "activityTypeId": 115, "campaignId": 1102, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1691578546892} +{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2173", "leadId": 800, "activityDate": "2023-01-20T09:59:53Z", "activityTypeId": 115, "campaignId": 1103, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1691578546893} +{"stream": "activities_change_nurture_cadence", "data": {"marketoGUID": "2190", "leadId": 105, "activityDate": "2023-01-20T11:53:23Z", "activityTypeId": 115, "campaignId": 1112, "primaryAttributeValueId": "1025", "primaryAttributeValue": "Test Nurture", "new_nurture_cadence": "Norm"}, "emitted_at": 1691578546894} +{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2188", "leadId": 105, "activityDate": "2023-01-20T11:52:53Z", "activityTypeId": 123, "campaignId": 1111, "primaryAttributeValueId": "1013", "primaryAttributeValue": "API Test Program 10", "source": "Marketo Flow Action"}, "emitted_at": 1691578674088} +{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2228", "leadId": 110, "activityDate": "2023-01-20T16:04:24Z", "activityTypeId": 123, "campaignId": 1136, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "source": "Marketo Flow Action"}, "emitted_at": 1691578674089} +{"stream": "activities_change_program_member_data", "data": {"marketoGUID": "2229", "leadId": 99, "activityDate": "2023-01-20T16:05:23Z", "activityTypeId": 123, "campaignId": 1137, "primaryAttributeValueId": "1004", "primaryAttributeValue": "API Test Program 1", "source": "Marketo Flow Action"}, "emitted_at": 1691578674089} +{"stream": "activities_execute_campaign", "data": {"marketoGUID": "2178", "leadId": 105, "activityDate": "2023-01-20T11:36:54Z", "activityTypeId": 155, "campaignId": 1106, "primaryAttributeValueId": "1105", "primaryAttributeValue": "Test Campaign 10", "used_parent_campaign_token_context": false, "qualified": false}, "emitted_at": 1691578927568} diff --git a/airbyte-integrations/connectors/source-marketo/metadata.yaml b/airbyte-integrations/connectors/source-marketo/metadata.yaml index 93cff41dad6d..d0fc6429c308 100644 --- a/airbyte-integrations/connectors/source-marketo/metadata.yaml +++ b/airbyte-integrations/connectors/source-marketo/metadata.yaml @@ -5,11 +5,11 @@ data: connectorSubtype: api connectorType: source definitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 - dockerImageTag: 1.1.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-marketo githubIssueLabel: source-marketo icon: marketo.svg - license: MIT + license: ELv2 name: Marketo registries: cloud: @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/marketo tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-marketo/requirements.txt b/airbyte-integrations/connectors/source-marketo/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-marketo/requirements.txt +++ b/airbyte-integrations/connectors/source-marketo/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-marketo/setup.py b/airbyte-integrations/connectors/source-marketo/setup.py index dfb65a09e7a8..1588bd2fc2a5 100644 --- a/airbyte-integrations/connectors/source-marketo/setup.py +++ b/airbyte-integrations/connectors/source-marketo/setup.py @@ -14,7 +14,6 @@ "pytest-faker==2.0.0", "pytest-mock~=3.6.1", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-merge/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-merge/integration_tests/abnormal_state.json index bebf411648b1..30d9fb07be20 100644 --- a/airbyte-integrations/connectors/source-merge/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-merge/integration_tests/abnormal_state.json @@ -6,4 +6,4 @@ "stream_descriptor": { "name": "users" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-merge/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-merge/integration_tests/sample_state.json index 8073ef8dd23d..76e1c16d8ed5 100644 --- a/airbyte-integrations/connectors/source-merge/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-merge/integration_tests/sample_state.json @@ -6,4 +6,4 @@ "stream_descriptor": { "name": "users" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-merge/metadata.yaml b/airbyte-integrations/connectors/source-merge/metadata.yaml index b4af81b734e8..13c6df4feb03 100644 --- a/airbyte-integrations/connectors/source-merge/metadata.yaml +++ b/airbyte-integrations/connectors/source-merge/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python -metadataSpecVersion: '1.0' + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-merge/requirements.txt b/airbyte-integrations/connectors/source-merge/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-merge/requirements.txt +++ b/airbyte-integrations/connectors/source-merge/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-merge/setup.py b/airbyte-integrations/connectors/source-merge/setup.py index 89109be06adb..aa45d3bdc5e2 100644 --- a/airbyte-integrations/connectors/source-merge/setup.py +++ b/airbyte-integrations/connectors/source-merge/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-metabase/Dockerfile b/airbyte-integrations/connectors/source-metabase/Dockerfile index 08c142ea73cd..0cff57643073 100644 --- a/airbyte-integrations/connectors/source-metabase/Dockerfile +++ b/airbyte-integrations/connectors/source-metabase/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-metabase diff --git a/airbyte-integrations/connectors/source-metabase/acceptance-test-config.yml b/airbyte-integrations/connectors/source-metabase/acceptance-test-config.yml index e0347f6fdd58..327271ee1ef0 100644 --- a/airbyte-integrations/connectors/source-metabase/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-metabase/acceptance-test-config.yml @@ -2,37 +2,37 @@ connector_image: airbyte/source-metabase:dev acceptance_tests: spec: tests: - - spec_path: "source_metabase/spec.yaml" + - spec_path: "source_metabase/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/config_http_url.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/config_http_url.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.1" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - - name: activity - bypass_reason: "data changes very fast" - - name: cards - bypass_reason: "data changes very fast" - - name: collections - bypass_reason: "data changes very fast" - - name: dashboards - bypass_reason: "data changes very fast" - - name: users - bypass_reason: "data changes very fast" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: cards + bypass_reason: "data changes very fast" + - name: collections + bypass_reason: "data changes very fast" + - name: dashboards + bypass_reason: "data changes very fast" + - name: users + bypass_reason: "data changes very fast" + fail_on_extra_columns: false full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" incremental: bypass_reason: "This connector does not implement incremental sync" diff --git a/airbyte-integrations/connectors/source-metabase/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-metabase/integration_tests/configured_catalog.json index 9bd45b123957..3bfb3cea5fc9 100644 --- a/airbyte-integrations/connectors/source-metabase/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-metabase/integration_tests/configured_catalog.json @@ -1,15 +1,5 @@ { "streams": [ - { - "stream": { - "name": "activity", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "collections", diff --git a/airbyte-integrations/connectors/source-metabase/metadata.yaml b/airbyte-integrations/connectors/source-metabase/metadata.yaml index 6ea4b009941d..3da269c3938c 100644 --- a/airbyte-integrations/connectors/source-metabase/metadata.yaml +++ b/airbyte-integrations/connectors/source-metabase/metadata.yaml @@ -1,8 +1,11 @@ data: + allowedHosts: + hosts: + - "*" connectorSubtype: api connectorType: source definitionId: c7cb421b-942e-4468-99ee-e369bcabaec5 - dockerImageTag: 0.3.1 + dockerImageTag: 1.0.1 dockerRepository: airbyte/source-metabase githubIssueLabel: source-metabase icon: metabase.svg @@ -18,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-metabase/requirements.txt b/airbyte-integrations/connectors/source-metabase/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-metabase/requirements.txt +++ b/airbyte-integrations/connectors/source-metabase/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json index 79723e036be2..c5d9714a6ea6 100644 --- a/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json @@ -1,179 +1,5 @@ { "streams": [ - { - "stream": { - "name": "activity", - "json_schema": { - "type": ["null", "object"], - "properties": { - "table_id": { - "type": ["null", "integer"] - }, - "table": { - "type": ["null", "string"] - }, - "database_id": { - "type": ["null", "integer"] - }, - "model_exists": { - "type": ["null", "boolean"] - }, - "topic": { - "type": ["null", "string"] - }, - "custom_id": { - "type": ["null", "string"] - }, - "details": { - "type": ["null", "object"], - "properties": { - "description": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "dashcards": { - "type": ["null", "array"], - "items": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "card_id": { - "type": ["null", "integer"] - }, - "exists": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "model_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "database": { - "type": ["null", "object"], - "properties": { - "description": { - "type": ["null", "string"] - }, - "features": { - "type": ["null", "array"] - }, - "cache_field_values_schedule": { - "type": ["null", "string"] - }, - "timezone": { - "type": ["null", "string"] - }, - "auto_run_queries": { - "type": ["null", "boolean"] - }, - "metadata_sync_schedule": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "caveats": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "is_full_sync": { - "type": ["null", "boolean"] - }, - "updated_at": { - "type": ["null", "string"] - }, - "cache_ttl": { - "type": ["null", "integer"] - }, - "is_sample": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_on_demand": { - "type": ["null", "boolean"] - }, - "options": { - "type": ["null", "string"] - }, - "engine": { - "type": ["null", "string"] - }, - "initial_sync_status": { - "type": ["null", "string"] - }, - "refingerprint": { - "type": ["null", "boolean"] - }, - "created_at": { - "type": ["null", "string"] - }, - "points_of_interest": { - "type": ["null", "string"] - } - } - }, - "user_id": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "user": { - "type": ["null", "object"], - "properties": { - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_login": { - "type": ["null", "string"] - }, - "is_qbnewb": { - "type": ["null", "boolean"] - }, - "is_superuser": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "last_name": { - "type": ["null", "string"] - }, - "date_joined": { - "type": ["null", "string"] - }, - "common_name": { - "type": ["null", "string"] - } - } - }, - "model": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "cards", diff --git a/airbyte-integrations/connectors/source-metabase/setup.py b/airbyte-integrations/connectors/source-metabase/setup.py index 848b1aecc765..228d443e50e2 100644 --- a/airbyte-integrations/connectors/source-metabase/setup.py +++ b/airbyte-integrations/connectors/source-metabase/setup.py @@ -8,6 +8,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "requests>=2.28.0", "types-requests>=2.27.30"] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock", "requests_mock~=1.8", diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml b/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml index 878f47fd7005..6147da2559aa 100644 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml @@ -13,7 +13,7 @@ definitions: url_base: "{{ config['instance_api_url'] }}" http_method: "GET" authenticator: - type: "SessionTokenAuthenticator" + type: "LegacySessionTokenAuthenticator" username: "{{ config['username'] }}" password: "{{ config['password'] }}" header: "X-Metabase-Session" @@ -39,11 +39,6 @@ definitions: primary_key: "id" retriever: $ref: "#/definitions/retriever" - activity_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "activity" - path: "activity" cards_stream: $ref: "#/definitions/base_stream" $parameters: @@ -67,7 +62,6 @@ definitions: name: "users" path: "user" streams: - - "#/definitions/activity_stream" - "#/definitions/cards_stream" - "#/definitions/collections_stream" - "#/definitions/dashboards_stream" @@ -75,4 +69,4 @@ streams: check: stream_names: - - "activity" + - "dashboards" diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/activity.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/activity.json deleted file mode 100644 index e7afabe5080a..000000000000 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/activity.json +++ /dev/null @@ -1,183 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "table_id": { - "type": ["null", "integer"] - }, - "table": { - "type": ["null", "object"], - "properties": { - "active": { "type": "boolean" }, - "caveats": { "type": ["null", "string"] }, - "created_at": { "type": ["null", "string"] }, - "db_id": { "type": "integer" }, - "description": { "type": ["null", "string"] }, - "display_name": { "type": ["null", "string"] }, - "entity_type": { "type": ["null", "string"] }, - "field_order": { "type": ["null", "string"] }, - "id": { "type": "integer" }, - "initial_sync_status": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, - "points_of_interest": { "type": ["null", "array"] }, - "schema": { "type": ["null", "string"] }, - "show_in_getting_started": { "type": "boolean" }, - "updated_at": { "type": ["null", "string"] }, - "visibility_type": { "type": ["null", "string"] } - } - }, - "database_id": { - "type": ["null", "integer"] - }, - "model_exists": { - "type": ["null", "boolean"] - }, - "topic": { - "type": ["null", "string"] - }, - "custom_id": { - "type": ["null", "string"] - }, - "details": { - "type": ["null", "object"], - "properties": { - "description": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "dashcards": { - "type": ["null", "array"], - "items": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "card_id": { - "type": ["null", "integer"] - }, - "exists": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "model_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "database": { - "type": ["null", "object"], - "properties": { - "description": { - "type": ["null", "string"] - }, - "features": { - "type": ["null", "array"] - }, - "cache_field_values_schedule": { - "type": ["null", "string"] - }, - "timezone": { - "type": ["null", "string"] - }, - "auto_run_queries": { - "type": ["null", "boolean"] - }, - "metadata_sync_schedule": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "caveats": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "is_full_sync": { - "type": ["null", "boolean"] - }, - "updated_at": { - "type": ["null", "string"] - }, - "cache_ttl": { - "type": ["null", "integer"] - }, - "is_sample": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "is_on_demand": { - "type": ["null", "boolean"] - }, - "options": { - "type": ["null", "string"] - }, - "engine": { - "type": ["null", "string"] - }, - "initial_sync_status": { - "type": ["null", "string"] - }, - "refingerprint": { - "type": ["null", "boolean"] - }, - "created_at": { - "type": ["null", "string"] - }, - "points_of_interest": { - "type": ["null", "string"] - } - } - }, - "user_id": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "user": { - "type": ["null", "object"], - "properties": { - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_login": { - "type": ["null", "string"] - }, - "is_qbnewb": { - "type": ["null", "boolean"] - }, - "is_superuser": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer"] - }, - "last_name": { - "type": ["null", "string"] - }, - "date_joined": { - "type": ["null", "string"] - }, - "common_name": { - "type": ["null", "string"] - } - } - }, - "model": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/streams.py b/airbyte-integrations/connectors/source-metabase/source_metabase/streams.py deleted file mode 100644 index a655648c84f3..000000000000 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/streams.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Any, Iterable, Mapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class MetabaseStream(HttpStream, ABC): - def __init__(self, instance_api_url: str, **kwargs): - super().__init__(**kwargs) - self.instance_api_url = instance_api_url - - primary_key = "id" - response_entity = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - @property - def url_base(self) -> str: - return self.instance_api_url - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - if self.response_entity: - result = response_json.get(self.response_entity, []) - else: - result = response_json - yield from result - - -class Activity(MetabaseStream): - def path(self, **kwargs) -> str: - return "activity" - - -class Cards(MetabaseStream): - def path(self, **kwargs) -> str: - return "card" - - -class Collections(MetabaseStream): - def path(self, **kwargs) -> str: - return "collection" - - -class Dashboards(MetabaseStream): - def path(self, **kwargs) -> str: - return "dashboard" - - -class Users(MetabaseStream): - - response_entity = "data" - - def path(self, **kwargs) -> str: - return "user" diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/Dockerfile b/airbyte-integrations/connectors/source-microsoft-dataverse/Dockerfile index bab4b96ea1f3..14f8609fa175 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/Dockerfile +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/Dockerfile @@ -34,5 +34,5 @@ COPY source_microsoft_dataverse ./source_microsoft_dataverse ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-microsoft-dataverse diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml index 78fac6748f4f..b91c74f4767e 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 9220e3de-3b60-4bb2-a46f-046d59ea235a - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-microsoft-dataverse githubIssueLabel: source-microsoft-dataverse icon: microsoftdataverse.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-dataverse tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/requirements.txt b/airbyte-integrations/connectors/source-microsoft-dataverse/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/requirements.txt +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py b/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py index 263302314df7..e86306c30138 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/source.py b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/source.py index 5902f83e1d79..3cec34b51a6f 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/source.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/source_microsoft_dataverse/source.py @@ -39,8 +39,8 @@ def discover(self, logger: logging.Logger, config: Mapping[str, Any]) -> Airbyte stream = AirbyteStream( name=entity["LogicalName"], json_schema=schema, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] ) - stream.source_defined_cursor = True if "modifiedon" in schema["properties"]: + stream.source_defined_cursor = True stream.default_cursor_field = ["modifiedon"] else: stream = AirbyteStream(name=entity["LogicalName"], json_schema=schema, supported_sync_modes=[SyncMode.full_refresh]) diff --git a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml index bdab1841c701..cf90d7d46549 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-teams tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt b/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt +++ b/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-microsoft-teams/setup.py b/airbyte-integrations/connectors/source-microsoft-teams/setup.py index 8d8c788e92a7..1867013845c2 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/setup.py @@ -13,9 +13,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index fa7a0493dbfb..4c1d31e45794 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -4,14 +4,14 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* WORKDIR /airbyte/integration_code -COPY source_mixpanel ./source_mixpanel -COPY main.py ./ COPY setup.py ./ RUN pip install . +COPY source_mixpanel ./source_mixpanel +COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.33 +LABEL io.airbyte.version=0.1.37 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml index 1ffdbd0aac3d..141b8e19e4b0 100644 --- a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml @@ -1,6 +1,10 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-mixpanel:dev +custom_environment_variables: + REQS_PER_HOUR_LIMIT: 0 + AVAILABLE_TESTING_RANGE_DAYS: 10 + PATCH_FUNNEL_SLICES: yes test_strictness_level: "high" acceptance_tests: spec: @@ -18,8 +22,8 @@ acceptance_tests: status: "failed" discovery: tests: - - config_path: "secrets/config_old.json" - timeout_seconds: 60 + - config_path: "secrets/config_incremental.json" + timeout_seconds: 900 basic_read: tests: - config_path: "secrets/config.json" @@ -29,6 +33,18 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes + empty_streams: + - name: export + bypass_reason: "Data expired too often" + - name: annotations + bypass_reason: "Data expired too often" + ignored_fields: + funnels: + - name: date + bypass_reason: "Data changes too often" + revenue: + - name: date + bypass_reason: "Data changes too often" full_refresh: tests: - config_path: "secrets/config_old.json" @@ -36,14 +52,15 @@ acceptance_tests: timeout_seconds: 9000 incremental: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config_incremental.json" configured_catalog_path: "integration_tests/configured_catalog_incremental.json" future_state: future_state_path: "integration_tests/abnormal_state.json" cursor_paths: cohorts: ["created"] export: ["time"] - funnels: ["36152117", "date"] + funnels: ["41833532", "date"] revenue: ["date"] - cohort_members": ["last_seen"] + engage: [ "last_seen" ] + cohort_members: [ "last_seen" ] timeout_seconds: 9000 diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json index 0d4d97a4e1ae..89a95990ac33 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json @@ -2,7 +2,7 @@ { "type": "STREAM", "stream": { - "stream_state": { "36152117": { "date": "2030-01-01" } }, + "stream_state": { "41833532": { "date": "2030-01-01" } }, "stream_descriptor": { "name": "funnels" } } }, diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_annotations.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_annotations.json deleted file mode 100644 index 0e3c10404216..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_annotations.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "annotations", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_cohort_members.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_cohort_members.json deleted file mode 100644 index 42147041b8e9..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_cohort_members.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "cohort_members", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_cohorts.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_cohorts.json deleted file mode 100644 index 1660128017c0..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_cohorts.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "cohorts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_engage.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_engage.json deleted file mode 100644 index 54e3681b8b03..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_engage.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "engage", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_export.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_export.json deleted file mode 100644 index 22831b7dbfeb..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_export.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "export", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "default_cursor_field": ["time"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_funnels.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_funnels.json deleted file mode 100644 index 00de5e7066c6..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_funnels.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "funnels", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "default_cursor_field": ["date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_revenue.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_revenue.json deleted file mode 100644 index 837ba1b12a0d..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_revenue.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "revenue", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "default_cursor_field": ["date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl index 7daf145e9d95..02e36881e9c1 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl @@ -1,16 +1,12 @@ -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-05-29", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1 }, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0 } ], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1 } }, "emitted_at": 1684508037955} -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-05-27", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1 }, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0 } ], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1 } }, "emitted_at": 1684508037956} -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-05-28", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1 }, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0 } ], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1 } }, "emitted_at": 1684508037956} -{"stream": "engage", "data": {"distinct_id": "123@gmail.com", "email": "123@gmail.com", "name": "123", "123": "123456", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1684508042343} -{"stream": "engage", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1684508042345} -{"stream": "engage", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1684508042346} -{"stream": "annotations", "data": {"date": "2023-01-15T12:00:00+01:00", "description": "test", "id": 1138193, "project_id": 2529987, "user": {"id": 3440095, "first_name": "", "last_name": ""}}, "emitted_at": 1684508044902} -{"stream": "annotations", "data": {"date": "2023-01-13T12:00:00+01:00", "description": "test123", "id": 1138196, "project_id": 2529987, "user": {"id": 3440095, "first_name": "", "last_name": ""}}, "emitted_at": 1684508044904} -{"stream": "annotations", "data": {"date": "2023-01-13T12:00:00+01:00", "description": "test121233", "id": 1138197, "project_id": 2529987, "user": {"id": 3440095, "first_name": "", "last_name": ""}}, "emitted_at": 1684508044904} -{"stream": "export", "data": {"event": "Signed up", "import": "True", "insert_id": "29fc2962-6d9c-455d-95ad-95b84f09b9e4", "mp_api_endpoint": "api.mixpanel.com", "mp_api_timestamp_ms": "1685362729000", "distinct_id": "91304156-cafc-4673-a237-623d1129c801", "mp_processing_time_ms": "1685362729276", "time": "2023-05-29T14:17:52Z"}, "emitted_at": 1685362954820} -{"stream": "cohorts", "data": {"id": 1478097, "project_id": 2529987, "name": "Cohort1", "description": "", "data_group_id": null, "count": 2, "is_visible": 1, "created": "2021-09-14 15:57:43"}, "emitted_at": 1684508052373} -{"stream": "cohort_members", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1684508059432} -{"stream": "cohort_members", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1684508059434} -{"stream": "revenue", "data": {"date": "2023-05-27", "amount": 0.0, "count": 3, "paid_count": 0 }, "emitted_at": 1684508063120} -{"stream": "revenue", "data": {"date": "2023-05-28", "amount": 0.0, "count": 3, "paid_count": 0 }, "emitted_at": 1684508063121} -{"stream": "revenue", "data": {"date": "2023-05-29", "amount": 0.0, "count": 3, "paid_count": 0 }, "emitted_at": 1684508063121} +{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-06-25", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1687889775303} +{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-06-26", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1687889775303} +{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-06-27", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1687889775303} +{"stream": "engage", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1687889778985} +{"stream": "engage", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1687889778988} +{"stream": "engage", "data": {"distinct_id": "123@gmail.com", "email": "123@gmail.com", "name": "123", "123": "123456", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1687889778988} +{"stream": "cohorts", "data": {"id": 1478097, "project_id": 2529987, "name": "Cohort1", "description": "", "data_group_id": null, "count": 2, "is_visible": 1, "created": "2021-09-14 15:57:43"}, "emitted_at": 1687889787689} +{"stream": "cohort_members", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1687889914154} +{"stream": "cohort_members", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1687889914156} +{"stream": "revenue", "data": {"date": "2023-06-25", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1687889918052} +{"stream": "revenue", "data": {"date": "2023-06-26", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1687889918052} +{"stream": "revenue", "data": {"date": "2023-06-27", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1687889918052} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json deleted file mode 100644 index 2cb9e7ea0d9f..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/sample_state.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "8901755": { "date": "2021-07-13" }, - "10463655": { "date": "2021-07-13" } - }, - "stream_descriptor": { "name": "funnels" } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { "date": "2021-07-01" }, - "stream_descriptor": { "name": "revenue" } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { "date": "2021-06-16T12:00:00" }, - "stream_descriptor": { "name": "export" } - } - } -] diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index bc9005da2c96..4a49e272acc5 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a - dockerImageTag: 0.1.33 + dockerImageTag: 0.1.37 dockerRepository: airbyte/source-mixpanel githubIssueLabel: source-mixpanel icon: mixpanel.svg @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/mixpanel tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mixpanel/requirements.txt b/airbyte-integrations/connectors/source-mixpanel/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-mixpanel/requirements.txt +++ b/airbyte-integrations/connectors/source-mixpanel/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-mixpanel/setup.py b/airbyte-integrations/connectors/source-mixpanel/setup.py index c16b4b1f7860..574a1130d6f9 100644 --- a/airbyte-integrations/connectors/source-mixpanel/setup.py +++ b/airbyte-integrations/connectors/source-mixpanel/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk~=0.2", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "pytest-mock~=3.6", "requests_mock~=1.8"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] setup( name="source_mixpanel", diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/annotations.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/annotations.json index f597a9d399af..caf311c987c5 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/annotations.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/annotations.json @@ -22,10 +22,10 @@ "type": ["null", "integer"] }, "first_name": { - "type" : ["null", "string"] + "type": ["null", "string"] }, "last_name": { - "type" : ["null", "string"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/export.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/export.json index b98a2b0f03e1..e33706bd25c2 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/export.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/schemas/export.json @@ -8,6 +8,9 @@ "distinct_id": { "type": ["null", "string"] }, + "insert_id": { + "type": ["null", "string"] + }, "time": { "type": ["null", "string"], "format": "date-time" @@ -234,9 +237,6 @@ "URL": { "type": ["null", "string"] }, - "insert_id": { - "type": ["null", "string"] - }, "mp_api_timestamp_ms": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index 6d5063ead948..e6f1dc6edf58 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -3,7 +3,9 @@ # import base64 +import json import logging +import os from typing import Any, List, Mapping, Tuple import pendulum @@ -13,7 +15,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator, TokenAuthenticator -from .streams import Annotations, CohortMembers, Cohorts, Engage, Export, Funnels, FunnelsList, Revenue +from .streams import Annotations, CohortMembers, Cohorts, Engage, Export, Funnels, Revenue from .testing import adapt_streams_if_testing, adapt_validate_if_testing from .utils import read_full_refresh @@ -25,6 +27,8 @@ def __init__(self, token: str): class SourceMixpanel(AbstractSource): + STREAMS = [Cohorts, CohortMembers, Funnels, Revenue, Export, Annotations, Engage] + def get_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: credentials = config.get("credentials") if credentials: @@ -79,16 +83,29 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> try: config = self._validate_and_transform(config) auth = self.get_authenticator(config) - FunnelsList.max_retries = 0 - funnels = FunnelsList(authenticator=auth, **config) - funnels.reqs_per_hour_limit = 0 - next(read_full_refresh(funnels), None) - except requests.HTTPError as e: - return False, e.response.json()["error"] except Exception as e: return False, e - return True, None + # https://github.com/airbytehq/airbyte/pull/27252#discussion_r1228356872 + # temporary solution, testing access for all streams to avoid 402 error + stream_kwargs = {"authenticator": auth, "reqs_per_hour_limit": 0, **config} + reason = None + for stream_class in self.STREAMS: + try: + stream = stream_class(**stream_kwargs) + next(read_full_refresh(stream), None) + return True, None + except requests.HTTPError as e: + try: + reason = e.response.json()["error"] + except json.JSONDecoder: + reason = e.response.content + if e.response.status_code != 402: + return False, reason + logger.info(f"Stream {stream_class.__name__}: {e.response.json()['error']}") + except Exception as e: + return False, str(e) + return False, reason @adapt_streams_if_testing def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -100,26 +117,21 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: logger.info(f"Using start_date: {config['start_date']}, end_date: {config['end_date']}") auth = self.get_authenticator(config) - streams = [ - Annotations(authenticator=auth, **config), - Cohorts(authenticator=auth, **config), - Funnels(authenticator=auth, **config), - Revenue(authenticator=auth, **config), - ] - - # streams with dynamically generated schema - for stream in [ - CohortMembers(authenticator=auth, **config), - Engage(authenticator=auth, **config), - Export(authenticator=auth, **config), - ]: + stream_kwargs = {"authenticator": auth, "reqs_per_hour_limit": 0, **config} + streams = [] + for stream_cls in self.STREAMS: + stream = stream_cls(**stream_kwargs) try: stream.get_json_schema() + next(read_full_refresh(stream), None) except requests.HTTPError as e: if e.response.status_code != 402: raise e logger.warning("Stream '%s' - is disabled, reason: 402 Payment Required", stream.name) else: + reqs_per_hour_limit = int(os.environ.get("REQS_PER_HOUR_LIMIT", stream.DEFAULT_REQS_PER_HOUR_LIMIT)) + # We preserve sleeping between requests in case this is not a running acceptance test. + # Otherwise, we do not want to wait as each API call is followed by sleeping ~60 seconds. + stream.reqs_per_hour_limit = reqs_per_hour_limit streams.append(stream) - return streams diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json index 9bf0f4a0601c..7e2ea3f591bf 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json @@ -92,7 +92,7 @@ "description": "The date in the format YYYY-MM-DD. Any data before this date will not be replicated. If this option is not set, the connector will replicate data from up to one year ago by default.", "examples": ["2021-11-16"], "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$", - "format": "date-time" + "format": "date" }, "end_date": { "order": 6, @@ -101,7 +101,7 @@ "description": "The date in the format YYYY-MM-DD. Any data after this date will not be replicated. Left empty to always sync to most recent date", "examples": ["2021-11-16"], "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$", - "format": "date-time" + "format": "date" }, "region": { "order": 7, diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index 8028948fdab2..3ea5d4cd41b8 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import time from abc import ABC from datetime import timedelta from typing import Any, Iterable, List, Mapping, MutableMapping, Optional @@ -20,14 +21,21 @@ class MixpanelStream(HttpStream, ABC): 60 queries per hour. """ + DEFAULT_REQS_PER_HOUR_LIMIT = 60 + @property def url_base(self): prefix = "eu." if self.region == "EU" else "" return f"https://{prefix}mixpanel.com/api/2.0/" - # https://help.mixpanel.com/hc/en-us/articles/115004602563-Rate-Limits-for-Export-API-Endpoints#api-export-endpoint-rate-limits - reqs_per_hour_limit: int = 60 # 1 query per minute - retries: int = 0 + @property + def reqs_per_hour_limit(self): + # https://help.mixpanel.com/hc/en-us/articles/115004602563-Rate-Limits-for-Export-API-Endpoints#api-export-endpoint-rate-limits + return self._reqs_per_hour_limit + + @reqs_per_hour_limit.setter + def reqs_per_hour_limit(self, value): + self._reqs_per_hour_limit = value def __init__( self, @@ -40,6 +48,7 @@ def __init__( attribution_window: int = 0, # in days select_properties_by_default: bool = True, project_id: int = None, + reqs_per_hour_limit: int = DEFAULT_REQS_PER_HOUR_LIMIT, **kwargs, ): self.start_date = start_date @@ -50,6 +59,8 @@ def __init__( self.region = region self.project_timezone = project_timezone self.project_id = project_id + self.retries = 0 + self._reqs_per_hour_limit = reqs_per_hour_limit super().__init__(authenticator=authenticator) @@ -62,15 +73,6 @@ def request_headers( ) -> Mapping[str, Any]: return {"Accept": "application/json"} - def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: - try: - return super()._send_request(request, request_kwargs) - except requests.exceptions.HTTPError as e: - error_message = e.response.text - if error_message: - self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") - raise e - def process_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() if self.data_field is not None: @@ -93,6 +95,12 @@ def parse_response( # parse the whole response yield from self.process_response(response, stream_state=stream_state, **kwargs) + if self.reqs_per_hour_limit > 0: + # we skip this block, if self.reqs_per_hour_limit = 0, + # in all other cases wait for X seconds to match API limitations + self.logger.info(f"Sleep for {3600 / self.reqs_per_hour_limit} seconds to match API limitations after reading from {self.name}") + time.sleep(3600 / self.reqs_per_hour_limit) + def backoff_time(self, response: requests.Response) -> float: """ Some API endpoints do not return "Retry-After" header. @@ -101,16 +109,28 @@ def backoff_time(self, response: requests.Response) -> float: retry_after = response.headers.get("Retry-After") if retry_after: + self.logger.debug(f"API responded with `Retry-After` header: {retry_after}") return float(retry_after) self.retries += 1 return 2**self.retries * 60 + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 402: + self.logger.warning(f"Unable to perform a request. Payment Required: {response.json()['error']}") + return False + return super().should_retry(response) + def get_stream_params(self) -> Mapping[str, Any]: """ Fetch required parameters in a given stream. Used to create sub-streams """ - params = {"authenticator": self.authenticator, "region": self.region, "project_timezone": self.project_timezone} + params = { + "authenticator": self.authenticator, + "region": self.region, + "project_timezone": self.project_timezone, + "reqs_per_hour_limit": self.reqs_per_hour_limit, + } if self.project_id: params["project_id"] = self.project_id return params @@ -130,13 +150,14 @@ class DateSlicesMixin: def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - date_slices: list = [] - # use the latest date between self.start_date and stream_state start_date = self.start_date + cursor_value = None + if stream_state and self.cursor_field and self.cursor_field in stream_state: # Remove time part from state because API accept 'from_date' param in date format only ('YYYY-MM-DD') # It also means that sync returns duplicated entries for the date from the state (date range is inclusive) + cursor_value = stream_state[self.cursor_field] stream_state_date = pendulum.parse(stream_state[self.cursor_field]).date() start_date = max(start_date, stream_state_date) @@ -148,17 +169,16 @@ def stream_slices( while start_date <= end_date: current_end_date = start_date + timedelta(days=self.date_window_size - 1) # -1 is needed because dates are inclusive - date_slices.append( - { - "start_date": str(start_date), - "end_date": str(min(current_end_date, end_date)), - } - ) + stream_slice = { + "start_date": str(start_date), + "end_date": str(min(current_end_date, end_date)), + } + if cursor_value: + stream_slice[self.cursor_field] = cursor_value + yield stream_slice # add 1 additional day because date range is inclusive start_date = current_end_date + timedelta(days=1) - return date_slices - def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohort_members.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohort_members.py index 4921d6ee6659..62e7570e9b52 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohort_members.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohort_members.py @@ -29,14 +29,11 @@ def stream_slices( if sync_mode == SyncMode.incremental: self.set_cursor(cursor_field) - stream_slices = [] # full refresh is needed because even though some cohorts might already have been read # they can still have new members added cohorts = Cohorts(**self.get_stream_params()).read_records(SyncMode.full_refresh) for cohort in cohorts: - stream_slices.append({"id": cohort["id"]}) - - return stream_slices + yield {"id": cohort["id"]} def process_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: records = super().process_response(response, **kwargs) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohorts.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohorts.py index 173d8d521252..e3433d5db964 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohorts.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/cohorts.py @@ -40,6 +40,7 @@ class Cohorts(IncrementalMixpanelStream): primary_key: str = "id" cursor_field = "created" + use_cache = True def path(self, **kwargs) -> str: return "cohorts/list" diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py index a3d6c9bae204..fe297fef6828 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/export.py @@ -186,11 +186,13 @@ def get_json_schema(self) -> Mapping[str, Any]: def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - mapping = super().request_params(stream_state, stream_slice, next_page_token) - if stream_state and "date" in stream_state: - timestamp = int(pendulum.parse(stream_state["date"]).timestamp()) - mapping["where"] = f'properties["$time"]>=datetime({timestamp})' - return mapping + params = super().request_params(stream_state, stream_slice, next_page_token) + # additional filter by timestamp because required start date and end date only allow to filter by date + cursor_param = stream_slice.get(self.cursor_field) + if cursor_param: + timestamp = int(pendulum.parse(cursor_param).timestamp()) + params["where"] = f'properties["$time"]>=datetime({timestamp})' + return params def request_kwargs( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/funnels.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/funnels.py index 8a0d69d37168..baabbd78d4af 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/funnels.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/funnels.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional from urllib.parse import parse_qs, urlparse import requests @@ -34,19 +34,16 @@ class Funnels(DateSlicesMixin, IncrementalMixpanelStream): data_field: str = "data" cursor_field: str = "date" min_date: str = "90" # days + funnels = {} def path(self, **kwargs) -> str: return "funnels" - def get_funnel_slices(self, sync_mode) -> List[dict]: + def get_funnel_slices(self, sync_mode) -> Iterator[dict]: stream = FunnelsList(**self.get_stream_params()) - funnel_slices = list(read_full_refresh(stream)) # [{'funnel_id': , 'name': }, {...}] + return read_full_refresh(stream) # [{'funnel_id': , 'name': }, {...}] - # save all funnels in dict(:, ...) - self.funnels = {funnel["funnel_id"]: funnel["name"] for funnel in funnel_slices} - return funnel_slices - - def funnel_slices(self, sync_mode) -> List[dict]: + def funnel_slices(self, sync_mode) -> Iterator[dict]: return self.get_funnel_slices(sync_mode) def stream_slices( @@ -80,17 +77,16 @@ def stream_slices( stream_state: Dict = stream_state or {} # One stream slice is a combination of all funnel_slices and date_slices - stream_slices: List = [] funnel_slices = self.funnel_slices(sync_mode) for funnel_slice in funnel_slices: # get single funnel state + # save all funnels in dict(:, ...) + self.funnels[funnel_slice["funnel_id"]] = funnel_slice["name"] funnel_id = str(funnel_slice["funnel_id"]) funnel_state = stream_state.get(funnel_id) date_slices = super().stream_slices(sync_mode, cursor_field=cursor_field, stream_state=funnel_state) for date_slice in date_slices: - stream_slices.append({**funnel_slice, **date_slice}) - - return stream_slices + yield {**funnel_slice, **date_slice} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/testing.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/testing.py index 57ae047d672d..598d0f96c117 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/testing.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/testing.py @@ -3,64 +3,47 @@ # import logging +import os from functools import wraps from .streams import Funnels -AVAILABLE_TESTING_RANGE_DAYS = 10 - def funnel_slices_patched(self: Funnels, sync_mode): """ Return only first result from funnels """ funnel_slices_values = self.get_funnel_slices(sync_mode) - return [funnel_slices_values[0]] if funnel_slices_values else funnel_slices_values + single_slice = next(funnel_slices_values, None) + return [single_slice] if single_slice else [] def adapt_streams_if_testing(func): - """ - Due to API limitations (60 requests per hour) there is unavailable to make acceptance tests in normal mode, - so we're reducing amount of requests by, if `_testing` flag is set in config: - - 1. Patch Funnels, so we download data only for one Funnel entity - 2. Removing RPS limit for faster testing - """ - + # Patch Funnels, so we download data only for one Funnel entity @wraps(func) def wrapper(self, config): - if not config.get("_testing"): - return func(self, config) - - # Patch Funnels, so we download data only for one Funnel entity - Funnels.funnel_slices = funnel_slices_patched - - streams = func(self, config) - - for stream in streams: - stream.reqs_per_hour_limit = 0 - return streams + if bool(os.environ.get("PATCH_FUNNEL_SLICES", "")): + Funnels.funnel_slices = funnel_slices_patched + return func(self, config) return wrapper def adapt_validate_if_testing(func): """ - Due to API limitations (60 requests per hour) there is unavailable to make acceptance tests in normal mode, - so we're reducing amount of requests by, if `_testing` flag is set in config: - - 1. Take time range in only 1 month + Due to API limitations (60 requests per hour) it is impossible to run acceptance tests in normal mode, + so we're reducing amount of requests by aligning start date if `AVAILABLE_TESTING_RANGE_DAYS` flag is set in env variables. """ @wraps(func) def wrapper(self, config): config = func(self, config) - if config.get("_testing"): + available_testing_range_days = int(os.environ.get("AVAILABLE_TESTING_RANGE_DAYS", 0)) + if available_testing_range_days: logger = logging.getLogger("airbyte") logger.info("SOURCE IN TESTING MODE, DO NOT USE IN PRODUCTION!") - # Take time range in only 1 month - if (config["end_date"] - config["start_date"]).days > AVAILABLE_TESTING_RANGE_DAYS: - config["start_date"] = config["end_date"].subtract(days=AVAILABLE_TESTING_RANGE_DAYS) + if (config["end_date"] - config["start_date"]).days > available_testing_range_days: + config["start_date"] = config["end_date"].subtract(days=available_testing_range_days) return config return wrapper diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py index 8513e07a9dc1..a1814f08f501 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py @@ -32,3 +32,17 @@ def config_raw(config): "start_date": str(config["start_date"]), "end_date": str(config["end_date"]), } + + +@pytest.fixture(autouse=True) +def patch_time(mocker): + mocker.patch("time.sleep") + + +@pytest.fixture(autouse=True) +def disable_cache(mocker): + mocker.patch( + "source_mixpanel.streams.cohorts.Cohorts.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py index c2ad5b3aad07..0cafc3ce2d62 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py @@ -8,7 +8,7 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import AirbyteConnectionStatus, Status from source_mixpanel.source import SourceMixpanel, TokenAuthenticatorBase64 -from source_mixpanel.streams import FunnelsList +from source_mixpanel.streams import Annotations, CohortMembers, Cohorts, Engage, Export, Funnels, FunnelsList, Revenue from .utils import command_check, get_url_to_mock, setup_response @@ -18,16 +18,15 @@ @pytest.fixture def check_connection_url(config): auth = TokenAuthenticatorBase64(token=config["api_secret"]) - funnel_list = FunnelsList(authenticator=auth, **config) - return get_url_to_mock(funnel_list) + annotations = Cohorts(authenticator=auth, **config) + return get_url_to_mock(annotations) @pytest.mark.parametrize( "response_code,expect_success,response_json", [ (200, True, {}), - (400, False, {"error": "Request error"}), - (500, False, {"error": "Server error"}), + (400, False, {"error": "Request error"}) ], ) def test_check_connection(requests_mock, check_connection_url, config_raw, response_code, expect_success, response_json): @@ -39,6 +38,31 @@ def test_check_connection(requests_mock, check_connection_url, config_raw, respo assert error == expected_error +def test_check_connection_all_streams_402_error(requests_mock, check_connection_url, config_raw, config): + auth = TokenAuthenticatorBase64(token=config["api_secret"]) + requests_mock.register_uri("GET", get_url_to_mock(Cohorts(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("GET", get_url_to_mock(Annotations(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("POST", get_url_to_mock(Engage(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("GET", get_url_to_mock(Export(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("GET", get_url_to_mock(Revenue(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("GET", get_url_to_mock(Funnels(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("GET", get_url_to_mock(FunnelsList(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri("GET", get_url_to_mock(CohortMembers(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + + ok, error = SourceMixpanel().check_connection(logger, config_raw) + assert ok is False and error == "Payment required" + + +def test_check_connection_402_error_on_first_stream(requests_mock, check_connection_url, config, config_raw): + auth = TokenAuthenticatorBase64(token=config["api_secret"]) + requests_mock.register_uri("GET", get_url_to_mock(Cohorts(authenticator=auth, **config)), setup_response(200, {})) + requests_mock.register_uri("GET", get_url_to_mock(Annotations(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + + ok, error = SourceMixpanel().check_connection(logger, config_raw) + # assert ok is True + assert error is None + + def test_check_connection_bad_config(): config = {} source = SourceMixpanel() @@ -52,8 +76,22 @@ def test_check_connection_incomplete(config_raw): def test_streams(requests_mock, config_raw): + requests_mock.register_uri("POST", "https://mixpanel.com/api/2.0/engage?page_size=1000", setup_response(200, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/properties", setup_response(200, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/annotations", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/cohorts/list", setup_response(200, {"id": 123})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels", setup_response(200, {})) + requests_mock.register_uri( + "GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(200, {"funnel_id": 123, "name": "name"}) + ) + requests_mock.register_uri( + "GET", "https://data.mixpanel.com/api/2.0/export", + setup_response(200, {"event": "some event", "properties": {"event": 124, "time": 124124}}) + ) + streams = SourceMixpanel().streams(config_raw) assert len(streams) == 7 @@ -61,15 +99,32 @@ def test_streams(requests_mock, config_raw): def test_streams_string_date(requests_mock, config_raw): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/properties", setup_response(200, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/annotations", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/cohorts/list", setup_response(200, {"id": 123})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", setup_response(200, {})) + requests_mock.register_uri("POST", "https://mixpanel.com/api/2.0/engage", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri( + "GET", "https://data.mixpanel.com/api/2.0/export", + setup_response(200, {"event": "some event", "properties": {"event": 124, "time": 124124}}) + ) config = copy.deepcopy(config_raw) config["start_date"] = "2020-01-01" config["end_date"] = "2020-01-02" streams = SourceMixpanel().streams(config) - assert len(streams) == 7 + assert len(streams) == 6 def test_streams_disabled_402(requests_mock, config_raw): - requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/properties", setup_response(402, {})) - requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(402, {})) + json_response = {"error": "Your plan does not allow API calls. Upgrade at mixpanel.com/pricing"} + requests_mock.register_uri("POST", "https://mixpanel.com/api/2.0/engage?page_size=1000", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/properties", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/events/properties/top", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/annotations", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/cohorts/list", setup_response(402, json_response)) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(402, json_response)) + requests_mock.register_uri("GET", "https://data.mixpanel.com/api/2.0/export?from_date=2017-01-20&to_date=2017-02-18", setup_response(402, json_response)) streams = SourceMixpanel().streams(config_raw) - assert {s.name for s in streams} == {"funnels", "revenue", "annotations", "cohorts"} + assert {s.name for s in streams} == {'annotations', 'engage', 'revenue'} diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index a44bd95af013..d519bc8d124b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -439,15 +439,12 @@ def test_export_stream(requests_mock, export_response, config): def test_export_stream_request_params(config): stream = Export(authenticator=MagicMock(), **config) stream_slice = {"start_date": "2017-01-25T00:00:00Z", "end_date": "2017-02-25T00:00:00Z"} - stream_state = {"date": "2021-06-16T17:00:00"} - - request_params = stream.request_params(stream_state=None, stream_slice=stream_slice) - assert "where" not in request_params request_params = stream.request_params(stream_state={}, stream_slice=stream_slice) assert "where" not in request_params - request_params = stream.request_params(stream_state=stream_state, stream_slice=stream_slice) + stream_slice["time"] = "2021-06-16T17:00:00" + request_params = stream.request_params(stream_state={}, stream_slice=stream_slice) assert "where" in request_params timestamp = int(pendulum.parse("2021-06-16T17:00:00Z").timestamp()) assert request_params.get("where") == f'properties["$time"]>=datetime({timestamp})' @@ -468,3 +465,20 @@ def test_export_iter_dicts(config): assert list(stream.iter_dicts([record_string, record_string[:2], record_string[2:], record_string])) == [record, record, record] # drop record parts because they are not standing nearby assert list(stream.iter_dicts([record_string, record_string[:2], record_string, record_string[2:]])) == [record, record] + + +@pytest.mark.parametrize( + ("http_status_code", "should_retry", "log_message"), + [ + (402, False, "Unable to perform a request. Payment Required: "), + ], +) +def test_should_retry_payment_required(http_status_code, should_retry, log_message, config, caplog): + response_mock = MagicMock() + response_mock.status_code = http_status_code + response_mock.json = MagicMock(return_value={"error": "Your plan does not allow API calls. Upgrade at mixpanel.com/pricing"}) + streams = [Annotations, CohortMembers, Cohorts, Engage, EngageSchema, Export, ExportSchema, Funnels, FunnelsList, Revenue] + for stream_class in streams: + stream = stream_class(authenticator=MagicMock(), **config) + assert stream.should_retry(response_mock) == should_retry + assert log_message in caplog.text diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py index cd867ffd80f6..65ffdad8b74b 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py @@ -16,7 +16,7 @@ def test_date_slices(): stream_slices = Annotations( authenticator=NoAuth(), start_date=now, end_date=now, date_window_size=1, region="EU", project_timezone="US/Pacific" ).stream_slices(sync_mode="any") - assert 1 == len(stream_slices) + assert 1 == len(list(stream_slices)) stream_slices = Annotations( authenticator=NoAuth(), @@ -26,7 +26,7 @@ def test_date_slices(): region="US", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert 2 == len(stream_slices) + assert 2 == len(list(stream_slices)) stream_slices = Annotations( authenticator=NoAuth(), @@ -36,7 +36,7 @@ def test_date_slices(): date_window_size=1, project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert 3 == len(stream_slices) + assert 3 == len(list(stream_slices)) stream_slices = Annotations( authenticator=NoAuth(), @@ -46,7 +46,7 @@ def test_date_slices(): date_window_size=10, project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert 1 == len(stream_slices) + assert 1 == len(list(stream_slices)) # test with attribution_window stream_slices = Annotations( @@ -58,7 +58,7 @@ def test_date_slices(): region="US", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert 8 == len(stream_slices) + assert 8 == len(list(stream_slices)) # Test with start_date end_date range stream_slices = Annotations( @@ -69,7 +69,7 @@ def test_date_slices(): region="US", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}] == stream_slices + assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}] == list(stream_slices) stream_slices = Annotations( authenticator=NoAuth(), @@ -79,7 +79,7 @@ def test_date_slices(): region="EU", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}, {"start_date": "2021-07-02", "end_date": "2021-07-02"}] == stream_slices + assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}, {"start_date": "2021-07-02", "end_date": "2021-07-02"}] == list(stream_slices) stream_slices = Annotations( authenticator=NoAuth(), @@ -93,7 +93,7 @@ def test_date_slices(): {"start_date": "2021-07-01", "end_date": "2021-07-01"}, {"start_date": "2021-07-02", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}, - ] == stream_slices + ] == list(stream_slices) stream_slices = Annotations( authenticator=NoAuth(), @@ -103,7 +103,7 @@ def test_date_slices(): region="US", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert [{"start_date": "2021-07-01", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == stream_slices + assert [{"start_date": "2021-07-01", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == list(stream_slices) # test with stream_state stream_slices = Export( @@ -114,4 +114,7 @@ def test_date_slices(): region="US", project_timezone="US/Pacific", ).stream_slices(sync_mode="any", stream_state={"time": "2021-07-02T00:00:00Z"}) - assert [{"start_date": "2021-07-02", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == stream_slices + assert [ + {"start_date": "2021-07-02", "end_date": "2021-07-02", "time": "2021-07-02T00:00:00Z"}, + {"start_date": "2021-07-03", "end_date": "2021-07-03", "time": "2021-07-02T00:00:00Z"} + ] == list(stream_slices) diff --git a/airbyte-integrations/connectors/source-monday/Dockerfile b/airbyte-integrations/connectors/source-monday/Dockerfile index 2782b1f5f25b..592a1a6d20c7 100644 --- a/airbyte-integrations/connectors/source-monday/Dockerfile +++ b/airbyte-integrations/connectors/source-monday/Dockerfile @@ -34,5 +34,5 @@ COPY source_monday ./source_monday ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=1.1.2 LABEL io.airbyte.name=airbyte/source-monday diff --git a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml index f401faf6dfdf..16ea6d61a9d4 100644 --- a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-monday:dev +test_strictness_level: "high" acceptance_tests: spec: tests: @@ -18,14 +19,16 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + # `boards`, `items`, `updates` streams schemas were modified. PR: https://github.com/airbytehq/airbyte/pull/27410 + # Changes applies to all configs backward_compatibility_tests_config: - disable_for_version: "0.1.4" + disable_for_version: "0.2.6" - config_path: "secrets/config_api_token.json" backward_compatibility_tests_config: - disable_for_version: "0.1.4" + disable_for_version: "0.2.6" - config_path: "secrets/config_oauth.json" backward_compatibility_tests_config: - disable_for_version: "0.1.4" + disable_for_version: "0.2.6" basic_read: tests: - config_path: "secrets/config_api_token.json" @@ -37,6 +40,13 @@ acceptance_tests: empty_streams: - name: teams bypass_reason: "unable to populate" + ignored_fields: + items: + - name: assets/*/public_url + bypass_reason: "Unstable data" + updates: + - name: assets/*/public_url + bypass_reason: "Unstable data" - config_path: "secrets/config_oauth.json" expect_records: path: "integration_tests/expected_records.jsonl" @@ -46,8 +56,23 @@ acceptance_tests: empty_streams: - name: teams bypass_reason: "unable to populate" + ignored_fields: + items: + - name: assets/*/public_url + bypass_reason: "Unstable data" + updates: + - name: assets/*/public_url + bypass_reason: "Unstable data" full_refresh: tests: - config_path: "secrets/config_api_token.json" incremental: - bypass_reason: "Incremental syncs are not supported on this connector." + tests: + - config_path: "secrets/config_api_token.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + items: [ "updated_at_int" ] + boards: [ "updated_at_int" ] + activity_logs: [ "created_at_int" ] diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/__init__.py b/airbyte-integrations/connectors/source-monday/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-monday/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-monday/integration_tests/abnormal_state.json index 52b0f2c2118f..bb56aed4e603 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-monday/integration_tests/abnormal_state.json @@ -1,5 +1,41 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "items" + }, + "stream_state": { + "updated_at": 1699041749, + "activity_logs": { + "created_at_int": 1699041749 + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "boards" + }, + "stream_state": { + "updated_at_int": 1699041749, + "activity_logs": { + "created_at_int": 1699041749 + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "activity_logs" + }, + "stream_state": { + "created_at_int": 1699041749 + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-monday/integration_tests/configured_catalog.json index a44855e5ae74..91fa7d7b5c01 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-monday/integration_tests/configured_catalog.json @@ -4,6 +4,34 @@ "stream": { "name": "items", "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at_int"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at_int"], + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "boards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at_int"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at_int"], + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "teams", + "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, @@ -12,7 +40,7 @@ }, { "stream": { - "name": "boards", + "name": "updates", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] @@ -22,7 +50,7 @@ }, { "stream": { - "name": "teams", + "name": "users", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] @@ -32,7 +60,7 @@ }, { "stream": { - "name": "updates", + "name": "workspaces", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] @@ -42,13 +70,27 @@ }, { "stream": { - "name": "users", + "name": "tags", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activity_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at_int"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at_int"], + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] } ] } diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl index c2443f941ff7..319adbe5ebcb 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl @@ -1,76 +1,17 @@ -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2019-04-11\",\"icon\":null,\"changed_at\":\"2019-04-10 08:06:40 UTC\"}", "additional_info": null, "text": "2019-04-11", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-07-22T06:28:36.561Z\"}", "additional_info": "{\"label\":\"Evaluating\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-07-22T06:28:36.561Z\"}", "text": "Evaluating", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:09:58.545Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:09:58.545Z\"}", "text": "Approved", "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:11:20.855Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:11:20.855Z\"}", "text": "Approved", "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:14:00.506Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:14:00.506Z\"}", "text": "Approved", "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2019-05-15T14:34:59.145Z\"}", "additional_info": "{\"label\":\"Declined\",\"color\":\"#e2445c\",\"changed_at\":\"2019-05-15T14:34:59.145Z\"}", "text": "Declined", "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2019-04-10T08:17:41.900Z\"}", "additional_info": "{\"label\":\"On Hold\",\"color\":\"#fdab3d\",\"changed_at\":\"2019-04-10T08:17:41.900Z\"}", "text": "On Hold", "title": "Legal approval", "type": "color"}, {"id": "file", "value": "{\"files\":null}", "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2019-04-30T15:43:35.438Z\"}", "additional_info": "{\"label\":\"Declined\",\"color\":\"#e2445c\",\"changed_at\":\"2019-04-30T15:43:35.438Z\"}", "text": "Declined", "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:51 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:51Z", "creator_id": "25479561", "group": {"id": "topics"}, "id": "3555407934", "name": "Zendesk", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:51Z", "updates": []}, "emitted_at": 1670509826492} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2019-04-11\",\"changed_at\":\"2019-04-02T07:03:23.375Z\"}", "additional_info": null, "text": "2019-04-11", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":11,\"post_id\":null,\"changed_at\":\"2020-06-25T11:41:22.421Z\"}", "additional_info": "{\"label\":\"On hold\",\"color\":\"#BB3354\",\"changed_at\":\"2020-06-25T11:41:22.421Z\"}", "text": "On hold", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "text": "Approved", "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2019-04-10T08:11:26.186Z\"}", "additional_info": "{\"label\":\"Declined\",\"color\":\"#e2445c\",\"changed_at\":\"2019-04-10T08:11:26.186Z\"}", "text": "Declined", "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": "{\"files\":null}", "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-06-25T06:36:38.993Z\"}", "additional_info": "{\"label\":\"On Hold\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-06-25T06:36:38.993Z\"}", "text": "On Hold", "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:52 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "25479561", "group": {"id": "topics"}, "id": "3555407944", "name": "Salesforce", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509826493} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2019-04-17\",\"changed_at\":\"2019-04-02T07:03:25.251Z\"}", "additional_info": null, "text": "2019-04-17", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":14,\"post_id\":null,\"changed_at\":\"2020-07-22T06:28:39.711Z\"}", "additional_info": "{\"label\":\"Waiting for vendor\",\"color\":\"#784BD1\",\"changed_at\":\"2020-07-22T06:28:39.711Z\"}", "text": "Waiting for vendor", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:10:00.038Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:10:00.038Z\"}", "text": "Approved", "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:11:23.942Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:11:23.942Z\"}", "text": "Approved", "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:14:02.186Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:14:02.186Z\"}", "text": "Approved", "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:16:19.222Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:16:19.222Z\"}", "text": "Approved", "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:17:46.022Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:17:46.022Z\"}", "text": "Approved", "title": "Legal approval", "type": "color"}, {"id": "file", "value": "{\"files\":null}", "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-25T06:36:40.961Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-25T06:36:40.961Z\"}", "text": "Approved", "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:52 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "25479561", "group": {"id": "topics"}, "id": "3555407952", "name": "YouCanBookMe", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509826494} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-25T11:41:48.118Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-25T11:41:48.118Z\"}", "text": "Done", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": null, "additional_info": null, "text": null, "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": null, "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": null, "additional_info": null, "text": null, "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408067", "name": "Box", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826495} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":108,\"post_id\":null,\"changed_at\":\"2020-07-22T06:27:41.551Z\"}", "additional_info": "{\"label\":\"Approved for use\",\"color\":\"#4eccc6\",\"changed_at\":\"2020-07-22T06:27:41.551Z\"}", "text": "Approved for use", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": null, "additional_info": null, "text": null, "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": null, "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": null, "additional_info": null, "text": null, "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408077", "name": "Slack", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826495} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":108,\"post_id\":null,\"changed_at\":\"2020-07-22T06:26:53.835Z\"}", "additional_info": "{\"label\":\"Approved for use\",\"color\":\"#4eccc6\",\"changed_at\":\"2020-07-22T06:26:53.835Z\"}", "text": "Approved for use", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": null, "additional_info": null, "text": null, "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": null, "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": null, "additional_info": null, "text": null, "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408088", "name": "HelpJuice", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826496} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":108,\"post_id\":null,\"changed_at\":\"2020-07-22T06:28:31.709Z\"}", "additional_info": "{\"label\":\"Approved for use\",\"color\":\"#4eccc6\",\"changed_at\":\"2020-07-22T06:28:31.709Z\"}", "text": "Approved for use", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": null, "additional_info": null, "text": null, "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": null, "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": null, "additional_info": null, "text": null, "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408094", "name": "LucidChart", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826497} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2019-04-11\",\"changed_at\":\"2019-04-02T07:03:23.375Z\"}", "additional_info": null, "text": "2019-04-11", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-07-22T06:28:25.645Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-07-22T06:28:25.645Z\"}", "text": "Done", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "text": "Approved", "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-05-15T12:27:06.894Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-05-15T12:27:06.894Z\"}", "text": "Approved", "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-05-15T12:27:08.700Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-05-15T12:27:08.700Z\"}", "text": "Approved", "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-05-15T12:27:10.209Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-05-15T12:27:10.209Z\"}", "text": "Approved", "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-05-15T12:27:11.909Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-05-15T12:27:11.909Z\"}", "text": "Approved", "title": "Legal approval", "type": "color"}, {"id": "file", "value": "{\"files\":null}", "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-05-15T12:27:15.385Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-05-15T12:27:15.385Z\"}", "text": "Approved", "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "25479561", "group": {"id": "new_group"}, "id": "3555408048", "name": "Aircall", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826498} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2019-04-17\",\"changed_at\":\"2019-04-02T07:03:25.251Z\"}", "additional_info": null, "text": "2019-04-17", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":108,\"post_id\":null,\"changed_at\":\"2020-07-22T06:28:29.177Z\"}", "additional_info": "{\"label\":\"Approved for use\",\"color\":\"#4eccc6\",\"changed_at\":\"2020-07-22T06:28:29.177Z\"}", "text": "Approved for use", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:10:00.038Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:10:00.038Z\"}", "text": "Approved", "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:11:23.942Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:11:23.942Z\"}", "text": "Approved", "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:14:02.186Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:14:02.186Z\"}", "text": "Approved", "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:16:19.222Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:16:19.222Z\"}", "text": "Approved", "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-04-10T08:17:46.022Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-04-10T08:17:46.022Z\"}", "text": "Approved", "title": "Legal approval", "type": "color"}, {"id": "file", "value": "{\"files\":null}", "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-05-15T12:27:17.250Z\"}", "additional_info": "{\"label\":\"Approved\",\"color\":\"#00c875\",\"changed_at\":\"2019-05-15T12:27:17.250Z\"}", "text": "Approved", "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "25479561", "group": {"id": "new_group"}, "id": "3555408057", "name": "Zoom", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826499} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":3,\"post_id\":null,\"changed_at\":\"2020-07-22T06:27:17.793Z\"}", "additional_info": "{\"label\":\"Waiting for legal\",\"color\":\"#0086c0\",\"changed_at\":\"2020-07-22T06:27:17.793Z\"}", "text": "Waiting for legal", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": null, "additional_info": null, "text": null, "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": null, "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": null, "additional_info": null, "text": null, "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group2816"}, "id": "3555408102", "name": "Gaviti", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826500} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407826"}, "column_values": [{"id": "manager1", "value": null, "additional_info": null, "text": "", "title": "Owner", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Request date", "type": "date"}, {"id": "status1", "value": "{\"index\":15,\"post_id\":null,\"changed_at\":\"2020-07-22T06:27:22.578Z\"}", "additional_info": "{\"label\":\"Negotiation\",\"color\":\"#9CD326\",\"changed_at\":\"2020-07-22T06:27:22.578Z\"}", "text": "Negotiation", "title": "Procurement status", "type": "color"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Manager", "type": "multiple-person"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Manager approval", "type": "color"}, {"id": "budget_owner", "value": null, "additional_info": null, "text": "", "title": "POC owner", "type": "multiple-person"}, {"id": "budget_owner_approval4", "value": null, "additional_info": null, "text": null, "title": "POC status", "type": "color"}, {"id": "manager", "value": null, "additional_info": null, "text": "", "title": "Budget owner", "type": "multiple-person"}, {"id": "status4", "value": null, "additional_info": null, "text": null, "title": "Budget owner approval", "type": "color"}, {"id": "people", "value": null, "additional_info": null, "text": "", "title": "Procurement team", "type": "multiple-person"}, {"id": "budget_owner_approval", "value": null, "additional_info": null, "text": null, "title": "Procurement approval", "type": "color"}, {"id": "procurement_team", "value": null, "additional_info": null, "text": "", "title": "Finance", "type": "multiple-person"}, {"id": "procurement_approval", "value": null, "additional_info": null, "text": null, "title": "Finance approval", "type": "color"}, {"id": "finance", "value": null, "additional_info": null, "text": "", "title": "Legal", "type": "multiple-person"}, {"id": "finance_approval", "value": null, "additional_info": null, "text": null, "title": "Legal approval", "type": "color"}, {"id": "file", "value": null, "additional_info": null, "text": "", "title": "File", "type": "file"}, {"id": "legal", "value": null, "additional_info": null, "text": "", "title": "Security", "type": "multiple-person"}, {"id": "legal_approval", "value": null, "additional_info": null, "text": null, "title": "Security approval", "type": "color"}, {"id": "date", "value": null, "additional_info": null, "text": "", "title": "Renewal date", "type": "date"}, {"id": "last_updated", "value": null, "additional_info": null, "text": "2022-11-21 14:36:53 UTC", "title": "Last updated", "type": "pulse-updated"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group2816"}, "id": "3555408118", "name": "Priority", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509826501} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN56456\"", "additional_info": null, "text": "SN56456", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2020-06-22T08:37:41.248Z\"}", "additional_info": "{\"label\":\"Needs replacement\",\"color\":\"#e2445c\",\"changed_at\":\"2020-06-22T08:37:41.248Z\"}", "text": "Needs replacement", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-05-14\",\"changed_at\":\"2020-06-22T11:25:44.457Z\"}", "additional_info": null, "text": "2020-05-14", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Zakariah Macleod\"", "additional_info": null, "text": "Zakariah Macleod", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-10\",\"changed_at\":\"2020-06-24T10:59:53.938Z\"}", "additional_info": null, "text": "2020-06-10", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "duplicate_of_tvs___projectors"}, "id": "3555407991", "name": "Samsung", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828048} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN 94-34-AS-GT-66\"", "additional_info": null, "text": "SN 94-34-AS-GT-66", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-06-22T08:37:42.971Z\"}", "additional_info": "{\"label\":\"Out for repair\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-06-22T08:37:42.971Z\"}", "text": "Out for repair", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-10\",\"changed_at\":\"2020-06-22T11:25:46.599Z\"}", "additional_info": null, "text": "2020-06-10", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Corbin Blackburn\"", "additional_info": null, "text": "Corbin Blackburn", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-11\",\"changed_at\":\"2020-06-24T10:59:55.752Z\"}", "additional_info": null, "text": "2020-06-11", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "duplicate_of_tvs___projectors"}, "id": "3555407997", "name": "Sonos One", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828051} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"P2219G\"", "additional_info": null, "text": "P2219G", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-06-22T08:37:44.792Z\"}", "additional_info": "{\"label\":\"Out for repair\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-06-22T08:37:44.792Z\"}", "text": "Out for repair", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-02\",\"changed_at\":\"2020-06-22T11:25:48.402Z\"}", "additional_info": null, "text": "2020-06-02", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Jorge Mcgowan\"", "additional_info": null, "text": "Jorge Mcgowan", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-28\",\"changed_at\":\"2020-06-24T10:59:57.460Z\"}", "additional_info": null, "text": "2020-06-28", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "duplicate_of_tvs___projectors"}, "id": "3555408009", "name": "Dell", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828055} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN FVFYVE57L21Y\"", "additional_info": null, "text": "SN FVFYVE57L21Y", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-03\",\"changed_at\":\"2020-06-22T11:25:52.834Z\"}", "additional_info": null, "text": "2020-06-03", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Rylee Pham\"", "additional_info": null, "text": "Rylee Pham", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-03-11\",\"changed_at\":\"2020-06-24T11:00:01.660Z\"}", "additional_info": null, "text": "2020-03-11", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:51Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407931", "name": "Macbook Pro", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:51Z", "updates": []}, "emitted_at": 1670509828057} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN FVFYVE57L22Y\"", "additional_info": null, "text": "SN FVFYVE57L22Y", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-03\",\"changed_at\":\"2020-06-22T11:25:56.327Z\"}", "additional_info": null, "text": "2020-06-03", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Kaydon Gamble\"", "additional_info": null, "text": "Kaydon Gamble", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-02-19\",\"changed_at\":\"2020-06-24T11:00:06.248Z\"}", "additional_info": null, "text": "2020-02-19", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407941", "name": "Macbook Pro", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828060} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN FVFYVE57L23W\"", "additional_info": null, "text": "SN FVFYVE57L23W", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-04\",\"changed_at\":\"2020-06-22T11:25:58.156Z\"}", "additional_info": null, "text": "2020-06-04", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Eli Reyes\"", "additional_info": null, "text": "Eli Reyes", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-03-26\",\"changed_at\":\"2020-06-24T11:00:13.482Z\"}", "additional_info": null, "text": "2020-03-26", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407947", "name": "Macbook Pro", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828062} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN FVFYVE57L41V\"", "additional_info": null, "text": "SN FVFYVE57L41V", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-11\",\"changed_at\":\"2020-06-22T11:26:00.305Z\"}", "additional_info": null, "text": "2020-06-11", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Finley Hilton\"", "additional_info": null, "text": "Finley Hilton", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-03-12\",\"changed_at\":\"2020-06-24T11:00:09.820Z\"}", "additional_info": null, "text": "2020-03-12", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407961", "name": "Macbook Pro", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828063} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN FVFYVE67L21Y\"", "additional_info": null, "text": "SN FVFYVE67L21Y", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-17\",\"changed_at\":\"2020-06-22T11:26:02.141Z\"}", "additional_info": null, "text": "2020-06-17", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Fabien Morton\"", "additional_info": null, "text": "Fabien Morton", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-04-21\",\"changed_at\":\"2020-06-24T11:00:17.305Z\"}", "additional_info": null, "text": "2020-04-21", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407968", "name": "Macbook Pro", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828065} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN FVFYVE57L28Y\"", "additional_info": null, "text": "SN FVFYVE57L28Y", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-18\",\"changed_at\":\"2020-06-22T11:26:04.062Z\"}", "additional_info": null, "text": "2020-06-18", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Amelia-Mae Flower\"", "additional_info": null, "text": "Amelia-Mae Flower", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-04\",\"changed_at\":\"2020-06-24T11:00:21.416Z\"}", "additional_info": null, "text": "2020-06-04", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407977", "name": "Macbook Pro", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828067} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"3sBeKstD\"", "additional_info": null, "text": "3sBeKstD", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-03\",\"changed_at\":\"2020-06-22T11:26:06.624Z\"}", "additional_info": null, "text": "2020-06-03", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Masuma Carver\"", "additional_info": null, "text": "Masuma Carver", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-04-07\",\"changed_at\":\"2020-06-24T11:00:35.389Z\"}", "additional_info": null, "text": "2020-04-07", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555408016", "name": "Dell - U2417H", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828068} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"eqK2M67W\"", "additional_info": null, "text": "eqK2M67W", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-10\",\"changed_at\":\"2020-06-22T11:26:08.798Z\"}", "additional_info": null, "text": "2020-06-10", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Tadhg Hensley\"", "additional_info": null, "text": "Tadhg Hensley", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-04\",\"changed_at\":\"2020-06-24T11:00:23.245Z\"}", "additional_info": null, "text": "2020-06-04", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555408058", "name": "Dell - U2418H", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828069} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"39QTALuj\"", "additional_info": null, "text": "39QTALuj", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-09\",\"changed_at\":\"2020-06-22T11:26:10.906Z\"}", "additional_info": null, "text": "2020-06-09", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Aanya Booth\"", "additional_info": null, "text": "Aanya Booth", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-05\",\"changed_at\":\"2020-06-24T11:00:25.468Z\"}", "additional_info": null, "text": "2020-06-05", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555408070", "name": "Dell - U2416H", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828071} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"LTgqT9cY\"", "additional_info": null, "text": "LTgqT9cY", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-03\",\"changed_at\":\"2020-06-22T11:26:12.701Z\"}", "additional_info": null, "text": "2020-06-03", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Kiana Burnett\"", "additional_info": null, "text": "Kiana Burnett", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-20\",\"changed_at\":\"2020-06-24T11:00:27.706Z\"}", "additional_info": null, "text": "2020-06-20", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555408083", "name": "Dell - U2419HX", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828072} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"FjE7nsrs\"", "additional_info": null, "text": "FjE7nsrs", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": "{\"date\":\"2020-06-10\",\"changed_at\":\"2020-06-22T11:26:17.156Z\"}", "additional_info": null, "text": "2020-06-10", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": "\"Roxie Forbes\"", "additional_info": null, "text": "Roxie Forbes", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-05-06\",\"changed_at\":\"2020-06-24T11:00:31.519Z\"}", "additional_info": null, "text": "2020-05-06", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555408091", "name": "Dell - P2219H", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828073} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN32456\"", "additional_info": null, "text": "SN32456", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-05-05\",\"changed_at\":\"2020-06-24T11:00:39.214Z\"}", "additional_info": null, "text": "2020-05-05", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408021", "name": "Samsung", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828074} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN32457\"", "additional_info": null, "text": "SN32457", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-05-07\",\"changed_at\":\"2020-06-24T11:00:42.752Z\"}", "additional_info": null, "text": "2020-05-07", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408033", "name": "Samsung", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828075} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN32458\"", "additional_info": null, "text": "SN32458", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-05-21\",\"changed_at\":\"2020-06-24T11:00:46.170Z\"}", "additional_info": null, "text": "2020-05-21", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408041", "name": "Samsung", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828077} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407785"}, "column_values": [{"id": "text4", "value": "\"SN32458\"", "additional_info": null, "text": "SN32458", "title": "SN", "type": "text"}, {"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "additional_info": "{\"label\":\"Working well\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T08:41:28.806Z\"}", "text": "Working well", "title": "Status", "type": "color"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Date given to current owner", "type": "date"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Current owner", "type": "text"}, {"id": "date_given_to_current_owner", "value": "{\"date\":\"2020-06-03\",\"changed_at\":\"2020-06-24T11:00:48.979Z\"}", "additional_info": null, "text": "2020-06-03", "title": "Last checked", "type": "date"}], "created_at": "2022-11-21T14:36:53Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555408052", "name": "Samsung", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:53Z", "updates": []}, "emitted_at": 1670509828079} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Start date", "type": "date"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Team", "type": "color"}, {"id": "status8", "value": null, "additional_info": null, "text": null, "title": "Site", "type": "color"}, {"id": "status1", "value": null, "additional_info": null, "text": null, "title": "Computer type", "type": "color"}, {"id": "status2", "value": null, "additional_info": null, "text": null, "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": null, "additional_info": null, "text": null, "title": "Google account", "type": "color"}, {"id": "google_account", "value": null, "additional_info": null, "text": null, "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": null, "additional_info": null, "text": null, "title": "365 account", "type": "color"}, {"id": "365_account3", "value": null, "additional_info": null, "text": null, "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": null, "additional_info": null, "text": null, "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": null, "additional_info": null, "text": "", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "25479561", "group": {"id": "airbyte_group"}, "id": "3555408019", "name": "new item", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:56Z", "updates": [{"id": "1825289531"}]}, "emitted_at": 1670509828865} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": null, "additional_info": null, "text": "", "title": "Start date", "type": "date"}, {"id": "status", "value": null, "additional_info": null, "text": null, "title": "Team", "type": "color"}, {"id": "status8", "value": null, "additional_info": null, "text": null, "title": "Site", "type": "color"}, {"id": "status1", "value": null, "additional_info": null, "text": null, "title": "Computer type", "type": "color"}, {"id": "status2", "value": null, "additional_info": null, "text": null, "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": null, "additional_info": null, "text": null, "title": "Google account", "type": "color"}, {"id": "google_account", "value": null, "additional_info": null, "text": null, "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": null, "additional_info": null, "text": null, "title": "365 account", "type": "color"}, {"id": "365_account3", "value": null, "additional_info": null, "text": null, "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": null, "additional_info": null, "text": null, "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": null, "additional_info": null, "text": "", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555407986", "name": "Hi there! \ud83d\udc4b Click here for more information \u27a1\ufe0f", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": [{"id": "1825289518"}]}, "emitted_at": 1670509828869} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2020-06-23\"}", "additional_info": null, "text": "2020-06-23", "title": "Start date", "type": "date"}, {"id": "status", "value": "{\"index\":7,\"post_id\":null}", "additional_info": "{\"label\":\"Finance\",\"color\":\"#579bfc\",\"changed_at\":null}", "text": "Finance", "title": "Team", "type": "color"}, {"id": "status8", "value": "{\"index\":1,\"post_id\":null}", "additional_info": "{\"label\":\"Denver\",\"color\":\"#225091\",\"changed_at\":null}", "text": "Denver", "title": "Site", "type": "color"}, {"id": "status1", "value": "{\"index\":14,\"post_id\":null}", "additional_info": "{\"label\":\"PC\",\"color\":\"#784BD1\",\"changed_at\":null}", "text": "PC", "title": "Computer type", "type": "color"}, {"id": "status2", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-06-21T13:46:33.895Z\"}", "additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-06-21T13:46:33.895Z\"}", "text": "Working on it", "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": null, "additional_info": null, "text": null, "title": "Google account", "type": "color"}, {"id": "google_account", "value": null, "additional_info": null, "text": null, "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": null, "additional_info": null, "text": null, "title": "365 account", "type": "color"}, {"id": "365_account3", "value": null, "additional_info": null, "text": null, "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": null, "additional_info": null, "text": null, "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": "{\"text\":\"bjornk@yahoo.com\",\"email\":\"bjornk@yahoo.com\"}", "additional_info": null, "text": "bjornk@yahoo.com", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:51Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407939", "name": "Employee name 3", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828873} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2020-06-19\"}", "additional_info": null, "text": "2020-06-19", "title": "Start date", "type": "date"}, {"id": "status", "value": "{\"index\":4,\"post_id\":null}", "additional_info": "{\"label\":\"Sales\",\"color\":\"#a25ddc\",\"changed_at\":null}", "text": "Sales", "title": "Team", "type": "color"}, {"id": "status8", "value": "{\"index\":1,\"post_id\":null}", "additional_info": "{\"label\":\"Denver\",\"color\":\"#225091\",\"changed_at\":null}", "text": "Denver", "title": "Site", "type": "color"}, {"id": "status1", "value": "{\"index\":14,\"post_id\":null}", "additional_info": "{\"label\":\"PC\",\"color\":\"#784BD1\",\"changed_at\":null}", "text": "PC", "title": "Computer type", "type": "color"}, {"id": "status2", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:46:39.331Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:46:39.331Z\"}", "text": "Done", "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:46:40.460Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:46:40.460Z\"}", "text": "Done", "title": "Google account", "type": "color"}, {"id": "google_account", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-06-21T13:46:41.571Z\"}", "additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-06-21T13:46:41.571Z\"}", "text": "Working on it", "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:15.506Z\"}", "additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2020-06-21T13:49:15.506Z\"}", "text": "Working on it", "title": "365 account", "type": "color"}, {"id": "365_account3", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T06:24:56.006Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T06:24:56.006Z\"}", "text": "Done", "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-22T06:24:57.281Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-22T06:24:57.281Z\"}", "text": "Done", "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": "{\"text\":\"fangorn@att.net\",\"email\":\"fangorn@att.net\"}", "additional_info": null, "text": "fangorn@att.net", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555407955", "name": "Employee name 5", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828876} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2020-05-15\"}", "additional_info": null, "text": "2020-05-15", "title": "Start date", "type": "date"}, {"id": "status", "value": "{\"index\":6,\"post_id\":null}", "additional_info": "{\"label\":\"Partners\",\"color\":\"#037f4c\",\"changed_at\":null}", "text": "Partners", "title": "Team", "type": "color"}, {"id": "status8", "value": "{\"index\":2,\"post_id\":null}", "additional_info": "{\"label\":\"Florida\",\"color\":\"#FF642E\",\"changed_at\":null}", "text": "Florida", "title": "Site", "type": "color"}, {"id": "status1", "value": "{\"index\":1,\"post_id\":null}", "additional_info": "{\"label\":\"Mac\",\"color\":\"#00c875\",\"changed_at\":null}", "text": "Mac", "title": "Computer type", "type": "color"}, {"id": "status2", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:51.414Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:51.414Z\"}", "text": "Done", "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:52.581Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:52.581Z\"}", "text": "Done", "title": "Google account", "type": "color"}, {"id": "google_account", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:54.106Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:54.106Z\"}", "text": "Done", "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:55.348Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:55.348Z\"}", "text": "Done", "title": "365 account", "type": "color"}, {"id": "365_account3", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:55.348Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:55.348Z\"}", "text": "Done", "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:55.348Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:55.348Z\"}", "text": "Done", "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": "{\"text\":\"mdielmann@me.com\",\"email\":\"mdielmann@me.com\"}", "additional_info": null, "text": "mdielmann@me.com", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555407946", "name": "Employee name 4", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828879} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2020-04-16\"}", "additional_info": null, "text": "2020-04-16", "title": "Start date", "type": "date"}, {"id": "status", "value": "{\"index\":3,\"post_id\":null}", "additional_info": "{\"label\":\"R&D\",\"color\":\"#0086c0\",\"changed_at\":null}", "text": "R&D", "title": "Team", "type": "color"}, {"id": "status8", "value": "{\"index\":14,\"post_id\":null}", "additional_info": "{\"label\":\"New York\",\"color\":\"#784BD1\",\"changed_at\":null}", "text": "New York", "title": "Site", "type": "color"}, {"id": "status1", "value": "{\"index\":1,\"post_id\":null}", "additional_info": "{\"label\":\"Mac\",\"color\":\"#00c875\",\"changed_at\":null}", "text": "Mac", "title": "Computer type", "type": "color"}, {"id": "status2", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:57.121Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:57.121Z\"}", "text": "Done", "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:59.755Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:59.755Z\"}", "text": "Done", "title": "Google account", "type": "color"}, {"id": "google_account", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:02.766Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:02.766Z\"}", "text": "Done", "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:06.494Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:06.494Z\"}", "text": "Done", "title": "365 account", "type": "color"}, {"id": "365_account3", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:06.494Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:06.494Z\"}", "text": "Done", "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:06.494Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:06.494Z\"}", "text": "Done", "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": "{\"text\":\"bjornk@outlook.com\",\"email\":\"bjornk@outlook.com\"}", "additional_info": null, "text": "bjornk@outlook.com", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "duplicate_of_new_hires___6_25_"}, "id": "3555407966", "name": "Employee name 6", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828881} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555407698"}, "column_values": [{"id": "people", "value": null, "additional_info": null, "text": "", "title": "IT owner", "type": "multiple-person"}, {"id": "person", "value": null, "additional_info": null, "text": "", "title": "Responsible HR", "type": "multiple-person"}, {"id": "date4", "value": "{\"date\":\"2020-04-29\"}", "additional_info": null, "text": "2020-04-29", "title": "Start date", "type": "date"}, {"id": "status", "value": "{\"index\":4,\"post_id\":null}", "additional_info": "{\"label\":\"Sales\",\"color\":\"#a25ddc\",\"changed_at\":null}", "text": "Sales", "title": "Team", "type": "color"}, {"id": "status8", "value": "{\"index\":1,\"post_id\":null}", "additional_info": "{\"label\":\"Denver\",\"color\":\"#225091\",\"changed_at\":null}", "text": "Denver", "title": "Site", "type": "color"}, {"id": "status1", "value": "{\"index\":1,\"post_id\":null}", "additional_info": "{\"label\":\"Mac\",\"color\":\"#00c875\",\"changed_at\":null}", "text": "Mac", "title": "Computer type", "type": "color"}, {"id": "status2", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:48:58.600Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:48:58.600Z\"}", "text": "Done", "title": "Computer setup", "type": "color"}, {"id": "computer_setup", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:01.489Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:01.489Z\"}", "text": "Done", "title": "Google account", "type": "color"}, {"id": "google_account", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:04.009Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:04.009Z\"}", "text": "Done", "title": "Zoom account", "type": "color"}, {"id": "zoom_account", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:05.324Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:05.324Z\"}", "text": "Done", "title": "365 account", "type": "color"}, {"id": "365_account3", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:05.324Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:05.324Z\"}", "text": "Done", "title": "Setup desk monitor", "type": "color"}, {"id": "set_up_desk_monitor", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2020-06-21T13:49:05.324Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2020-06-21T13:49:05.324Z\"}", "text": "Done", "title": "Setup entrance tag", "type": "color"}, {"id": "email", "value": "{\"text\":\"mcmillan@gmail.com\",\"email\":\"mcmillan@gmail.com\"}", "additional_info": null, "text": "mcmillan@gmail.com", "title": "Email", "type": "email"}], "created_at": "2022-11-21T14:36:52Z", "creator_id": "36694549", "group": {"id": "duplicate_of_new_hires___6_25_"}, "id": "3555407969", "name": "Employee name 7", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:36:52Z", "updates": []}, "emitted_at": 1670509828883} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": null, "additional_info": null, "text": null, "title": "Status", "type": "color"}, {"id": "link", "value": "{\"url\":\"https://drive.google.com/file/d/1wC5EQFsISkGAYLTKtcw1sG3ppHNLwESl/view\",\"text\":\"Link\",\"changed_at\":\"2022-06-08T12:54:27.555Z\"}", "additional_info": null, "text": "Link - https://drive.google.com/file/d/1wC5EQFsISkGAYLTKtcw1sG3ppHNLwESl/view", "title": "Link", "type": "link"}, {"id": "label", "value": null, "additional_info": null, "text": null, "title": "Label", "type": "color"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:04:41Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179394", "name": "API session", "parent_item": {"id": "3555179341"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:41Z", "updates": []}, "emitted_at": 1670509829849} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": null, "additional_info": null, "text": null, "title": "Status", "type": "color"}, {"id": "link", "value": "{\"url\":\"https://drive.google.com/file/d/1BbEblAXGDOxTiTIDIcqOzfhAh43IyuTu/view\",\"text\":\"Link\",\"changed_at\":\"2022-06-08T12:54:31.968Z\"}", "additional_info": null, "text": "Link - https://drive.google.com/file/d/1BbEblAXGDOxTiTIDIcqOzfhAh43IyuTu/view", "title": "Link", "type": "link"}, {"id": "label", "value": null, "additional_info": null, "text": null, "title": "Label", "type": "color"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:04:41Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179405", "name": "Build a view", "parent_item": {"id": "3555179341"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:41Z", "updates": []}, "emitted_at": 1670509829853} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": null, "additional_info": null, "text": null, "title": "Status", "type": "color"}, {"id": "link", "value": "{\"url\":\"https://drive.google.com/file/d/1ByMoeo2yhQcZfEOLhPP_iTjhutsLHipO/view\",\"text\":\"Link\",\"changed_at\":\"2022-06-08T12:54:38.402Z\"}", "additional_info": null, "text": "Link - https://drive.google.com/file/d/1ByMoeo2yhQcZfEOLhPP_iTjhutsLHipO/view", "title": "Link", "type": "link"}, {"id": "label", "value": null, "additional_info": null, "text": null, "title": "Label", "type": "color"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:04:41Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179418", "name": "Build an integration", "parent_item": {"id": "3555179341"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:41Z", "updates": []}, "emitted_at": 1670509829857} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": null, "additional_info": null, "text": null, "title": "Status", "type": "color"}, {"id": "link", "value": "{\"url\":\"https://drive.google.com/file/d/1MbhkWgJY8piKmH0uivIHaKnwYPyNr3rS/view\",\"text\":\"Link\",\"changed_at\":\"2022-06-08T12:54:48.148Z\"}", "additional_info": null, "text": "Link - https://drive.google.com/file/d/1MbhkWgJY8piKmH0uivIHaKnwYPyNr3rS/view", "title": "Link", "type": "link"}, {"id": "label", "value": null, "additional_info": null, "text": null, "title": "Label", "type": "color"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:04:41Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179422", "name": "Authentication", "parent_item": {"id": "3555179341"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:41Z", "updates": []}, "emitted_at": 1670509829860} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": null, "additional_info": null, "text": null, "title": "Status", "type": "color"}, {"id": "link", "value": "{\"url\":\"https://drive.google.com/file/d/1ZUwXi9VA7MfD3JZJgvVXmpdvaosHJIKm/view\",\"text\":\"Link\",\"changed_at\":\"2022-06-08T12:54:53.319Z\"}", "additional_info": null, "text": "Link - https://drive.google.com/file/d/1ZUwXi9VA7MfD3JZJgvVXmpdvaosHJIKm/view", "title": "Link", "type": "link"}, {"id": "label", "value": null, "additional_info": null, "text": null, "title": "Label", "type": "color"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:04:41Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179431", "name": "Build a Workspace template", "parent_item": {"id": "3555179341"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:41Z", "updates": []}, "emitted_at": 1670509829863} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2022-11-21T14:40:58.550Z\"}", "additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2022-11-21T14:40:58.550Z\"}", "text": "Done", "title": "Status", "type": "color"}, {"id": "link", "value": "{\"url\":\"https://airbyte.com/\",\"text\":\"Airbyte\",\"changed_at\":\"2022-11-21T14:40:42.184Z\"}", "additional_info": null, "text": "Airbyte - https://airbyte.com/", "title": "Link", "type": "link"}, {"id": "label", "value": "{\"index\":156,\"post_id\":null,\"changed_at\":\"2022-11-21T14:41:45.550Z\"}", "additional_info": "{\"label\":\"Label 3\",\"color\":\"#9D99B9\",\"changed_at\":\"2022-11-21T14:41:45.550Z\"}", "text": "Label 3", "title": "Label", "type": "color"}, {"id": "text", "value": "\"Test test test\"", "additional_info": null, "text": "Test test test", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:40:34Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555433784", "name": "Test", "parent_item": {"id": "3555179351"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:42:01Z", "updates": []}, "emitted_at": 1670509829866} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179105"}, "column_values": [{"id": "status", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-11-21T14:41:32.359Z\"}", "additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2022-11-21T14:41:32.359Z\"}", "text": "Working on it", "title": "Status", "type": "color"}, {"id": "link", "value": null, "additional_info": null, "text": "", "title": "Link", "type": "link"}, {"id": "label", "value": "{\"index\":105,\"post_id\":null,\"changed_at\":\"2022-11-21T14:41:43.450Z\"}", "additional_info": "{\"label\":\"Label 1\",\"color\":\"#9AADBD\",\"changed_at\":\"2022-11-21T14:41:43.450Z\"}", "text": "Label 1", "title": "Label", "type": "color"}, {"id": "text", "value": "\"one two three #!!\"", "additional_info": null, "text": "one two three #!!", "title": "Text", "type": "text"}], "created_at": "2022-11-21T14:41:12Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555437747", "name": "Test1", "parent_item": {"id": "3555179351"}, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:42:09Z", "updates": [{"id": "1825302913"}]}, "emitted_at": 1670509829868} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": null, "additional_info": null, "text": "", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2022-06-07T11:29:38.019Z\"}", "additional_info": "{\"label\":\"Done!\",\"color\":\"#00c875\",\"changed_at\":\"2022-06-07T11:29:38.019Z\"}", "text": "Done!", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179259", "name": "Create a dev account", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831250} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": "{\"linkedPulseIds\":[{\"linkedPulseId\":3555433784},{\"linkedPulseId\":3555437747}]}", "additional_info": null, "text": "Test, Test1", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": null, "additional_info": null, "text": "", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:29:19.711Z\"}", "additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2022-06-07T11:29:19.711Z\"}", "text": "Working on it", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:40Z", "creator_id": "29814928", "group": {"id": "topics"}, "id": "3555179351", "name": "Click to read this update \ud83e\udd29", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:41:13Z", "updates": [{"id": "1825206780"}]}, "emitted_at": 1670509831253} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://www.youtube.com/watch?v=nUMK6d1JcCY\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:16:13.208Z\"}", "additional_info": null, "text": "Link - https://www.youtube.com/watch?v=nUMK6d1JcCY", "title": "Link", "type": "link"}, {"id": "text", "value": "\"You can write you notes here\"", "additional_info": null, "text": "You can write you notes here", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2022-06-07T11:16:08.728Z\"}", "additional_info": "{\"label\":\"Video\",\"color\":\"#e2445c\",\"changed_at\":\"2022-06-07T11:16:08.728Z\"}", "text": "Video", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "14293832", "group": {"id": "topics"}, "id": "3555179247", "name": "What is monday - 2 min video \ud83c\udfa5 (Very cool)", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831256} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": null, "additional_info": null, "text": "", "title": "Link", "type": "link"}, {"id": "text", "value": "\"Notes\"", "additional_info": null, "text": "Notes", "title": "My notes", "type": "text"}, {"id": "status_1", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-11-21T14:42:28.383Z\"}", "additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2022-11-21T14:42:28.383Z\"}", "text": "Working on it", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2022-11-21T14:42:31.122Z\"}", "additional_info": "{\"label\":\"Documentation\",\"color\":\"#175A63\",\"changed_at\":\"2022-11-21T14:42:31.122Z\"}", "text": "Documentation", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:42:29Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555446655", "name": "Item 1", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:42:42Z", "updates": []}, "emitted_at": 1670509831257} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": null, "additional_info": null, "text": "", "title": "Link", "type": "link"}, {"id": "text", "value": "\"Note 2\"", "additional_info": null, "text": "Note 2", "title": "My notes", "type": "text"}, {"id": "status_1", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2022-11-21T14:42:44.903Z\"}", "additional_info": "{\"label\":\"Stuck\",\"color\":\"#e2445c\",\"changed_at\":\"2022-11-21T14:42:44.903Z\"}", "text": "Stuck", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-11-21T14:42:47.315Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-11-21T14:42:47.315Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:42:48Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "3555448801", "name": "Item 2", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:42:58Z", "updates": []}, "emitted_at": 1670509831260} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://support.monday.com/hc/en-us/articles/360001267945-The-board-views\",\"text\":\"LInk\",\"changed_at\":\"2022-06-07T11:18:26.141Z\"}", "additional_info": null, "text": "LInk - https://support.monday.com/hc/en-us/articles/360001267945-The-board-views", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:17:57.718Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T11:17:57.718Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "36694549", "group": {"id": "new_group37570"}, "id": "3555179253", "name": "Board views", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831262} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://support.monday.com/hc/en-us/articles/360017143959-The-Item-Card-\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:57:42.870Z\"}", "additional_info": null, "text": "Link - https://support.monday.com/hc/en-us/articles/360017143959-The-Item-Card-", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:22:29.156Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T11:22:29.156Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "29814928", "group": {"id": "new_group37570"}, "id": "3555179292", "name": "Item views", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831263} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://support.monday.com/hc/en-us/articles/360003445540-monday-com-Integrations\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:51:35.374Z\"}", "additional_info": null, "text": "Link - https://support.monday.com/hc/en-us/articles/360003445540-monday-com-Integrations", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:22:29.156Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T11:22:29.156Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "29814928", "group": {"id": "new_group37570"}, "id": "3555179262", "name": "Integrations", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831265} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://support.monday.com/hc/en-us/articles/360002187819-The-Dashboards\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:54:50.218Z\"}", "additional_info": null, "text": "Link - https://support.monday.com/hc/en-us/articles/360002187819-The-Dashboards", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:22:29.156Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T11:22:29.156Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "29814928", "group": {"id": "new_group37570"}, "id": "3555179270", "name": "Dashboard widgets", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831266} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://support.monday.com/hc/en-us/articles/360001362625-monday-com-board-templates\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:56:20.627Z\"}", "additional_info": null, "text": "Link - https://support.monday.com/hc/en-us/articles/360001362625-monday-com-board-templates", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:56:23.316Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T11:56:23.316Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "29814928", "group": {"id": "new_group37570"}, "id": "3555179288", "name": "Workspace template", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831268} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://monday.com/marketplace\",\"text\":\"Apps marketplace\",\"changed_at\":\"2022-04-12T13:55:17.553Z\"}", "additional_info": null, "text": "Apps marketplace - https://monday.com/marketplace", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:38Z", "creator_id": "14293832", "group": {"id": "group_title"}, "id": "3555179185", "name": "Check out our marketplace - The puzzle icon on the left pane", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:38Z", "updates": []}, "emitted_at": 1670509831269} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://apps.developer.monday.com/docs/monday-app-development-process\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:58:53.115Z\"}", "additional_info": null, "text": "Link - https://apps.developer.monday.com/docs/monday-app-development-process", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T11:58:38.720Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T11:58:38.720Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "14293832", "group": {"id": "group_title"}, "id": "3555179305", "name": "Plan your app", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831270} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://monday.com/developers/apps/intro\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:59:07.145Z\"}", "additional_info": null, "text": "Link - https://monday.com/developers/apps/intro", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2022-06-07T11:58:40.733Z\"}", "additional_info": "{\"label\":\"Documentation\",\"color\":\"#175A63\",\"changed_at\":\"2022-06-07T11:58:40.733Z\"}", "text": "Documentation", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555179310", "name": "Check out our monday apps documentation", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:40Z", "updates": []}, "emitted_at": 1670509831271} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://apps.developer.monday.com/docs/workspace-templates\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T11:59:30.527Z\"}", "additional_info": null, "text": "Link - https://apps.developer.monday.com/docs/workspace-templates", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2022-06-07T11:58:43.818Z\"}", "additional_info": "{\"label\":\"Documentation\",\"color\":\"#175A63\",\"changed_at\":\"2022-06-07T11:58:43.818Z\"}", "text": "Documentation", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:40Z", "creator_id": "36694549", "group": {"id": "group_title"}, "id": "3555179318", "name": "Bundling templates with your app", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:40Z", "updates": []}, "emitted_at": 1670509831272} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": "{\"linkedPulseIds\":[{\"linkedPulseId\":3555179394},{\"linkedPulseId\":3555179405},{\"linkedPulseId\":3555179418},{\"linkedPulseId\":3555179422},{\"linkedPulseId\":3555179431}]}", "additional_info": null, "text": "API session, Build a view, Build an integration, Authentication, Build a Workspace template", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": null, "additional_info": null, "text": "", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":2,\"post_id\":null,\"changed_at\":\"2022-06-07T11:59:35.220Z\"}", "additional_info": "{\"label\":\"Video\",\"color\":\"#e2445c\",\"changed_at\":\"2022-06-07T11:59:35.220Z\"}", "text": "Video", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:40Z", "creator_id": "29814928", "group": {"id": "group_title"}, "id": "3555179341", "name": "Sessions recordings - See the framework in action (Subitems)", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:42Z", "updates": []}, "emitted_at": 1670509831273} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://apps.developer.monday.com/docs/submit-your-app\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T12:06:37.152Z\"}", "additional_info": null, "text": "Link - https://apps.developer.monday.com/docs/submit-your-app", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2022-06-07T12:06:30.006Z\"}", "additional_info": "{\"label\":\"Article\",\"color\":\"#66CCFF\",\"changed_at\":\"2022-06-07T12:06:30.006Z\"}", "text": "Article", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:40Z", "creator_id": "36694549", "group": {"id": "new_group45036"}, "id": "3555179324", "name": "Prepare for marketplace review", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:40Z", "updates": []}, "emitted_at": 1670509831275} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://view.monday.com/2586384041-f47a2dae812b0a295f0a974bfc39b42d?r=use1\",\"text\":\"Link to board\",\"changed_at\":\"2022-04-25T09:16:11.585Z\"}", "additional_info": null, "text": "Link to board - https://view.monday.com/2586384041-f47a2dae812b0a295f0a974bfc39b42d?r=use1", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "29814928", "group": {"id": "new_group45036"}, "id": "3555179218", "name": "App review template board", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831276} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://forms.monday.com/forms/c390831d51274ee8882bf4fc96a6fb17\",\"text\":\"Submission form\",\"changed_at\":\"2022-04-12T14:04:58.308Z\"}", "additional_info": null, "text": "Submission form - https://forms.monday.com/forms/c390831d51274ee8882bf4fc96a6fb17", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "36694549", "group": {"id": "new_group45036"}, "id": "3555179230", "name": "Submit your app to review", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831277} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://community.monday.com/c/developers/8\",\"text\":\"Link \",\"changed_at\":\"2022-04-12T14:02:46.916Z\"}", "additional_info": null, "text": "Link - https://community.monday.com/c/developers/8", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:38Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555179200", "name": "Developers community \ud83d\udc68\u200d\ud83d\udc68\u200d\ud83d\udc66\u200d\ud83d\udc66", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:38Z", "updates": []}, "emitted_at": 1670509831278} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://style.monday.com/?path=/docs/welcome--page\",\"text\":\"Link\",\"changed_at\":\"2022-04-12T14:03:00.031Z\"}", "additional_info": null, "text": "Link - https://style.monday.com/?path=/docs/welcome--page", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555179207", "name": "Design kit \ud83d\udc69\ud83c\udffb\u200d\ud83c\udfa8", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831279} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://apps.developer.monday.com/docs/monetization\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T12:07:37.151Z\"}", "additional_info": null, "text": "Link - https://apps.developer.monday.com/docs/monetization", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2022-06-07T12:07:24.046Z\"}", "additional_info": "{\"label\":\"Documentation\",\"color\":\"#175A63\",\"changed_at\":\"2022-06-07T12:07:24.046Z\"}", "text": "Documentation", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:40Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555179327", "name": "monday apps monetization \ud83d\udcb0", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:40Z", "updates": []}, "emitted_at": 1670509831280} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"mailto:appsupport@monday.com/\",\"text\":\"appsupport@monday.com\",\"changed_at\":\"2022-04-13T20:54:57.089Z\"}", "additional_info": null, "text": "appsupport@monday.com - mailto:appsupport@monday.com/", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:39Z", "creator_id": "36694549", "group": {"id": "new_group"}, "id": "3555179211", "name": "Technical support team \ud83e\udd70", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:39Z", "updates": []}, "emitted_at": 1670509831281} -{"stream": "items", "data": {"assets": [], "board": {"id": "3555179067"}, "column_values": [{"id": "subitems", "value": null, "additional_info": null, "text": "", "title": "Subitems", "type": "subtasks"}, {"id": "link", "value": "{\"url\":\"https://mondayclimatechallenge.devpost.com/\",\"text\":\"Link\",\"changed_at\":\"2022-06-07T13:40:18.751Z\"}", "additional_info": null, "text": "Link - https://mondayclimatechallenge.devpost.com/", "title": "Link", "type": "link"}, {"id": "text", "value": null, "additional_info": null, "text": "", "title": "My notes", "type": "text"}, {"id": "status_1", "value": null, "additional_info": null, "text": "Need to review", "title": "Status", "type": "color"}, {"id": "status_10", "value": null, "additional_info": null, "text": "Other", "title": "Type", "type": "color"}], "created_at": "2022-11-21T14:04:40Z", "creator_id": "29814928", "group": {"id": "new_group"}, "id": "3555179334", "name": "Make sure you saw to see challenge post (awesome prizes included)", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2022-11-21T14:04:40Z", "updates": []}, "emitted_at": 1670509831281} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 380}, {"archived": false, "id": "manager1", "settings_str": "{}", "title": "Owner", "type": "multiple-person", "width": 80}, {"archived": false, "id": "date4", "settings_str": "{}", "title": "Request date", "type": "date", "width": null}, {"archived": false, "id": "status1", "settings_str": "{\"labels\":{\"0\":\"Evaluating\",\"1\":\"Done\",\"2\":\"Denied\",\"3\":\"Waiting for legal\",\"6\":\"Approved for POC\",\"11\":\"On hold\",\"14\":\"Waiting for vendor\",\"15\":\"Negotiation\",\"108\":\"Approved for use\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":7,\"3\":8,\"5\":9,\"6\":3,\"11\":6,\"14\":5,\"15\":4,\"108\":2},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"11\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"},\"15\":{\"color\":\"#9CD326\",\"border\":\"#89B921\",\"var_name\":\"lime-green\"},\"108\":{\"color\":\"#4eccc6\",\"border\":\"#4eccc6\",\"var_name\":\"australia\"}}}", "title": "Procurement status", "type": "color", "width": null}, {"archived": false, "id": "person", "settings_str": "{}", "title": "Manager", "type": "multiple-person", "width": 80}, {"archived": false, "id": "status", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Manager approval", "type": "color", "width": null}, {"archived": false, "id": "budget_owner", "settings_str": "{}", "title": "POC owner", "type": "multiple-person", "width": 80}, {"archived": false, "id": "budget_owner_approval4", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "POC status", "type": "color", "width": null}, {"archived": false, "id": "manager", "settings_str": "{}", "title": "Budget owner", "type": "multiple-person", "width": 80}, {"archived": false, "id": "status4", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Budget owner approval", "type": "color", "width": 185}, {"archived": false, "id": "people", "settings_str": "{}", "title": "Procurement team", "type": "multiple-person", "width": null}, {"archived": false, "id": "budget_owner_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Procurement approval", "type": "color", "width": null}, {"archived": false, "id": "procurement_team", "settings_str": "{}", "title": "Finance", "type": "multiple-person", "width": null}, {"archived": false, "id": "procurement_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Finance approval", "type": "color", "width": null}, {"archived": false, "id": "finance", "settings_str": "{}", "title": "Legal", "type": "multiple-person", "width": null}, {"archived": false, "id": "finance_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Redlines\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Legal approval", "type": "color", "width": null}, {"archived": false, "id": "file", "settings_str": "{}", "title": "File", "type": "file", "width": null}, {"archived": false, "id": "legal", "settings_str": "{}", "title": "Security", "type": "multiple-person", "width": null}, {"archived": false, "id": "legal_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Security approval", "type": "color", "width": null}, {"archived": false, "id": "date", "settings_str": "{\"hide_footer\":false}", "title": "Renewal date", "type": "date", "width": null}, {"archived": false, "id": "last_updated", "settings_str": "{}", "title": "Last updated", "type": "pulse-updated", "width": 129}], "communication": null, "description": "Many IT departments need to handle the procurement process for new services. The essence of this board is to streamline this process by providing an intuitive structure that supports collaboration and efficiency.", "groups": [{"archived": null, "color": "#579bfc", "deleted": null, "id": "topics", "position": "65536", "title": "Reviewing"}, {"archived": null, "color": "#FF642E", "deleted": null, "id": "new_group", "position": "98304.0", "title": "Corporate IT"}, {"archived": null, "color": "#037f4c", "deleted": null, "id": "new_group2816", "position": "114688.0", "title": "Finance"}], "id": "3555407826", "name": "Procurement process", "owner": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:36:50Z", "updates": [], "views": [{"id": "80969928"}], "workspace": null}, "emitted_at": 1670509832350} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 523}, {"archived": false, "id": "text4", "settings_str": "{}", "title": "SN", "type": "text", "width": null}, {"archived": false, "id": "status", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Out for repair\",\"1\":\"Working well\",\"2\":\"Needs replacement\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Status", "type": "color", "width": null}, {"archived": false, "id": "date4", "settings_str": "{}", "title": "Date given to current owner", "type": "date", "width": 204}, {"archived": false, "id": "text", "settings_str": "{}", "title": "Current owner", "type": "text", "width": null}, {"archived": false, "id": "date_given_to_current_owner", "settings_str": "{}", "title": "Last checked", "type": "date", "width": 129}], "communication": null, "description": "Welcome to your inventory management board. This is the place to track and manage all of your IT equipment inventory.", "groups": [{"archived": null, "color": "#BB3354", "deleted": null, "id": "duplicate_of_tvs___projectors", "position": "65408", "title": "Out of service"}, {"archived": null, "color": "#579bfc", "deleted": null, "id": "topics", "position": "65536", "title": "Laptops"}, {"archived": null, "color": "#a25ddc", "deleted": null, "id": "group_title", "position": "98304", "title": "Monitors"}, {"archived": null, "color": "#037f4c", "deleted": null, "id": "new_group", "position": "163840.0", "title": "TVs & projectors"}], "id": "3555407785", "name": "Inventory management", "owner": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "duplicate_of_tvs___projectors"}, "updated_at": "2022-11-21T14:36:49Z", "updates": [], "views": [], "workspace": null}, "emitted_at": 1670509832352} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 347}, {"archived": false, "id": "people", "settings_str": "{}", "title": "IT owner", "type": "multiple-person", "width": 98}, {"archived": false, "id": "person", "settings_str": "{}", "title": "Responsible HR", "type": "multiple-person", "width": 112}, {"archived": false, "id": "date4", "settings_str": "{}", "title": "Start date", "type": "date", "width": 114}, {"archived": false, "id": "status", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"color_mapping\":{\"0\":16,\"1\":11,\"11\":1,\"16\":0},\"labels\":{\"0\":\"Product\",\"1\":\"Design\",\"2\":\"HR\",\"3\":\"R\\u0026D\",\"4\":\"Sales\",\"6\":\"Partners\",\"7\":\"Finance\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"3\":3,\"4\":4,\"5\":7,\"6\":5,\"7\":6},\"labels_colors\":{\"0\":{\"color\":\"#66CCFF\",\"border\":\"#5AB3E0\",\"var_name\":\"turquoise\"},\"1\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"4\":{\"color\":\"#a25ddc\",\"border\":\"#9238AF\",\"var_name\":\"purple\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"7\":{\"color\":\"#579bfc\",\"border\":\"#4387E8\",\"var_name\":\"bright-blue\"}}}", "title": "Team", "type": "color", "width": 103}, {"archived": false, "id": "status8", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"color_mapping\":{\"1\":107,\"2\":19,\"3\":1,\"19\":2,\"107\":3},\"labels\":{\"1\":\"Denver\",\"2\":\"Florida\",\"14\":\"New York\"},\"labels_positions_v2\":{\"1\":2,\"2\":0,\"5\":3,\"14\":1},\"labels_colors\":{\"1\":{\"color\":\"#225091\",\"border\":\"#225091\",\"var_name\":\"navy\"},\"2\":{\"color\":\"#FF642E\",\"border\":\"#E05828\",\"var_name\":\"dark-orange\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"}}}", "title": "Site", "type": "color", "width": 80}, {"archived": false, "id": "status1", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"1\":\"Mac\",\"14\":\"PC\"},\"labels_positions_v2\":{\"1\":0,\"5\":2,\"14\":1},\"labels_colors\":{\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"}}}", "title": "Computer type", "type": "color", "width": 107}, {"archived": false, "id": "status2", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Computer setup", "type": "color", "width": 116}, {"archived": false, "id": "computer_setup", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Google account", "type": "color", "width": 110}, {"archived": false, "id": "google_account", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Zoom account", "type": "color", "width": 104}, {"archived": false, "id": "zoom_account", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "365 account", "type": "color", "width": 102}, {"archived": false, "id": "365_account3", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Setup desk monitor", "type": "color", "width": 132}, {"archived": false, "id": "set_up_desk_monitor", "settings_str": "{\"done_colors\":[1],\"hide_footer\":true,\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Setup entrance tag", "type": "color", "width": null}, {"archived": false, "id": "email", "settings_str": "{}", "title": "Email", "type": "email", "width": null}], "communication": null, "description": "This is an IT onboarding process board. The essence of this board is to track the IT onboarding process of new employees.", "groups": [{"archived": null, "color": "#037f4c", "deleted": null, "id": "airbyte_group27398", "position": "16352.0", "title": "Airbyte group"}, {"archived": null, "color": "#a25ddc", "deleted": null, "id": "airbyte_group", "position": "32704.0", "title": "Airbyte group"}, {"archived": null, "color": "#FF642E", "deleted": null, "id": "new_group", "position": "65408", "title": "More information about this template:"}, {"archived": null, "color": "#579bfc", "deleted": null, "id": "topics", "position": "65536", "title": "New Hires - June"}, {"archived": null, "color": "#a25ddc", "deleted": null, "id": "group_title", "position": "98304", "title": "New Hires - May"}, {"archived": null, "color": "#037f4c", "deleted": null, "id": "duplicate_of_new_hires___6_25_", "position": "196608.0", "title": "New Hires - April"}], "id": "3555407698", "name": "IT Onboarding", "owner": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "airbyte_group27398"}, "updated_at": "2022-11-21T14:36:49Z", "updates": [{"id": "1825289531"}], "views": [{"id": "80969927"}, {"id": "80969929"}], "workspace": null}, "emitted_at": 1670509832353} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 414}, {"archived": false, "id": "status", "settings_str": "{\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Status", "type": "color", "width": null}, {"archived": false, "id": "link", "settings_str": "{}", "title": "Link", "type": "link", "width": null}, {"archived": false, "id": "label", "settings_str": "{\"done_colors\":[1],\"labels\":{\"3\":\"Label 2\",\"105\":\"Label 1\",\"156\":\"Label 3\"},\"labels_positions_v2\":{\"3\":1,\"5\":3,\"105\":0,\"156\":2},\"labels_colors\":{\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"105\":{\"color\":\"#9AADBD\",\"border\":\"#9AADBD\",\"var_name\":\"winter\"},\"156\":{\"color\":\"#9D99B9\",\"border\":\"#9D99B9\",\"var_name\":\"purple_gray\"}}}", "title": "Label", "type": "color", "width": null}, {"archived": false, "id": "text", "settings_str": "{}", "title": "Text", "type": "text", "width": null}], "communication": null, "description": null, "groups": [{"archived": null, "color": "#579bfc", "deleted": null, "id": "topics", "position": "65536", "title": "Subitems"}], "id": "3555179105", "name": "Subitems of Welcome to your monday dev account \ud83d\ude0d", "owner": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:42:06Z", "updates": [{"id": "1825302913"}], "views": [], "workspace": null}, "emitted_at": 1670509832355} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 596}, {"archived": false, "id": "subitems", "settings_str": "{\"allowMultipleItems\":true,\"itemTypeName\":\"column.subtasks.title\",\"displayType\":\"BOARD_INLINE\",\"boardIds\":[3555179105]}", "title": "Subitems", "type": "subtasks", "width": null}, {"archived": false, "id": "link", "settings_str": "{}", "title": "Link", "type": "link", "width": 168}, {"archived": false, "id": "text", "settings_str": "{}", "title": "My notes", "type": "text", "width": 262}, {"archived": false, "id": "status_1", "settings_str": "{\"labels\":{\"0\":\"Working on it\",\"1\":\"Done!\",\"2\":\"Stuck\",\"5\":\"Need to review\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":2,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"5\":{\"color\":\"#c4c4c4\",\"border\":\"#B0B0B0\",\"var_name\":\"grey\"}}}", "title": "Status", "type": "color", "width": null}, {"archived": false, "id": "status_10", "settings_str": "{\"done_colors\":[1],\"color_mapping\":{\"0\":16,\"1\":160,\"16\":0,\"160\":1},\"labels\":{\"0\":\"Article\",\"1\":\"Documentation\",\"2\":\"Video\",\"5\":\"Other\"},\"labels_colors\":{\"0\":{\"color\":\"#66CCFF\",\"border\":\"#5AB3E0\",\"var_name\":\"turquoise\"},\"1\":{\"color\":\"#175A63\",\"border\":\"#175A63\",\"var_name\":\"eden\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"5\":{\"color\":\"#c4c4c4\",\"border\":\"#B0B0B0\",\"var_name\":\"grey\"}}}", "title": "Type", "type": "color", "width": null}], "communication": null, "description": null, "groups": [{"archived": null, "color": "#579bfc", "deleted": null, "id": "topics", "position": "81920", "title": "Get to know monday.com"}, {"archived": null, "color": "#FF158A", "deleted": null, "id": "new_group37570", "position": "90112", "title": "What can be developed on monday.com"}, {"archived": null, "color": "#a25ddc", "deleted": null, "id": "group_title", "position": "98304", "title": "Build your monday app"}, {"archived": null, "color": "#fdab3d", "deleted": null, "id": "new_group45036", "position": "131072.0", "title": "Prepare for app submission"}, {"archived": null, "color": "#0086c0", "deleted": null, "id": "new_group", "position": "163840.0", "title": "Helpful resources"}], "id": "3555179067", "name": "Welcome to your monday dev account \ud83d\ude0d", "owner": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:04:38Z", "updates": [{"id": "1825206780"}], "views": [{"id": "80965788"}], "workspace": null}, "emitted_at": 1670509832356} -{"stream": "updates", "data": {"assets": [], "body": "

    \ufeffTest

    ", "created_at": "2022-11-21T14:41:21Z", "creator_id": "36694549", "id": "1825302913", "item_id": "3555437747", "replies": [{"id": "1825303266", "creator_id": "36694549", "created_at": "2022-11-21T14:41:29Z", "text_body": "Test test", "updated_at": "2022-11-21T14:41:29Z", "body": "

    \ufeffTest test

    "}], "text_body": "Test", "updated_at": "2022-11-21T14:41:29Z"}, "emitted_at": 1676683112518} -{"stream": "updates", "data": {"assets": [], "body": "

    \ufeffHey there \ud83d\udc4b

    \ufeffThis is an update, we usually use this to...update \ud83d\ude04

    \ufeffWe love to communicate with the context of a specific item.

    \ufeff

    \ufeffRight above this box, there are tabs for different item views which can also be used for apps.

    ", "created_at": "2022-06-08T12:53:39Z", "creator_id": "-7", "id": "1825206780", "item_id": "3555179351", "replies": [], "text_body": "Hey there \ud83d\udc4b\n\nThis is an update, we usually use this to...update \ud83d\ude04\n\nWe love to communicate with the context of a specific item.\n\n\n\nRight above this box, there are tabs for different item views which can also be used for apps.", "updated_at": "2022-11-21T14:04:40Z"}, "emitted_at": 1676683112520} -{"stream": "updates", "data": {"assets": [], "body": "

    @Airbyte Testin \ufeffhi\ufeff

    ", "created_at": "2021-10-22T17:02:22Z", "creator_id": "-7", "id": "1825289531", "item_id": "3555408019", "replies": [], "text_body": "@Airbyte Testin hi", "updated_at": "2022-11-21T14:36:53Z"}, "emitted_at": 1676683112522} -{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:03:00Z", "join_date": null, "email": "integration-test@airbyte.io", "enabled": true, "id": 36694549, "is_admin": true, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Airbyte Team", "phone": "", "photo_original": "https://files.monday.com/use1/photos/36694549/original/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_small": "https://files.monday.com/use1/photos/36694549/small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb": "https://files.monday.com/use1/photos/36694549/thumb/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb_small": "https://files.monday.com/use1/photos/36694549/thumb_small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_tiny": "https://files.monday.com/use1/photos/36694549/tiny/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "time_zone_identifier": "Europe/Kiev", "title": "Airbyte Developer Account", "url": "https://airbyte-unit.monday.com/users/36694549", "utc_hours_diff": 3}, "emitted_at": 1670509895766} -{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:33:18Z", "join_date": null, "email": "iryna.grankova@airbyte.io", "enabled": true, "id": 36695702, "is_admin": false, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Iryna Grankova", "phone": null, "photo_original": "https://files.monday.com/use1/photos/36695702/original/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_small": "https://files.monday.com/use1/photos/36695702/small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb": "https://files.monday.com/use1/photos/36695702/thumb/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb_small": "https://files.monday.com/use1/photos/36695702/thumb_small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_tiny": "https://files.monday.com/use1/photos/36695702/tiny/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "time_zone_identifier": "Europe/Athens", "title": null, "url": "https://airbyte-unit.monday.com/users/36695702", "utc_hours_diff": 3}, "emitted_at": 1670509895770} \ No newline at end of file +{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2019-03-01T17:24:57.321Z\"}", "description": null, "id": "status", "text": "Working on it", "title": "Status", "type": "color", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2019-03-01T17:24:57.321Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-11", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "open", "title": "Tags", "type": "tag", "value": "{\"tag_ids\":[19038090]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211945", "name": "Item 1", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-15T16:19:37Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "updated_at_int": 1686845977}, "emitted_at": 1690884054247} +{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "description": null, "id": "status", "text": "Done", "title": "Status", "type": "color", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-03-01T17:28:23.178Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-11", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "closed", "title": "Tags", "type": "tag", "value": "{\"tag_ids\":[19038091]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211964", "name": "Item 2", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-13T13:59:36Z", "updates": [], "updated_at_int": 1686664776}, "emitted_at": 1690884054254} +{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":null,\"color\":\"#c4c4c4\",\"changed_at\":\"2019-03-01T17:25:02.248Z\"}", "description": null, "id": "status", "text": null, "title": "Status", "type": "color", "value": "{\"index\":5,\"post_id\":null,\"changed_at\":\"2019-03-01T17:25:02.248Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-13", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-13\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:26.291Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "", "title": "Tags", "type": "tag", "value": null}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211977", "name": "Item 3", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-13T13:58:26Z", "updates": [], "updated_at_int": 1686664706}, "emitted_at": 1690884054258} +{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 400}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Person", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "tags", "settings_str": "{\"hide_footer\":false}", "title": "Tags", "type": "tag", "width": null}], "communication": null, "description": null, "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Group Title"}, {"archived": false, "color": "#a25ddc", "deleted": false, "id": "group_title", "position": "98304", "title": "Group Title"}, {"archived": false, "color": "#808080", "deleted": false, "id": "new_group", "position": "163840.0", "title": "New Group unit board"}], "id": "4635211873", "name": "New Board", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-06-20T12:12:46Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "views": [], "workspace": {"id": 2845647, "name": "Test workspace", "kind": "open", "description": null}, "updated_at_int": 1687263166}, "emitted_at": 1690884065399} +{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 400}, {"archived": false, "description": null, "id": "files", "settings_str": "{\"hide_footer\":false}", "title": "Files", "type": "file", "width": null}], "communication": null, "description": null, "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Group Title"}], "id": "4634950289", "name": "test doc", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-06-13T13:28:31Z", "updates": [], "views": [{"id": "103920755", "name": "Table", "settings_str": "{}", "type": "FeatureBoardView", "view_specific_data_str": "{}"}], "workspace": {"id": 2845647, "name": "Test workspace", "kind": "open", "description": null}, "updated_at_int": 1686662911}, "emitted_at": 1690884065405} +{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 380}, {"archived": false, "description": null, "id": "manager1", "settings_str": "{}", "title": "Owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Request date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "status1", "settings_str": "{\"labels\":{\"0\":\"Evaluating\",\"1\":\"Done\",\"2\":\"Denied\",\"3\":\"Waiting for legal\",\"6\":\"Approved for POC\",\"11\":\"On hold\",\"14\":\"Waiting for vendor\",\"15\":\"Negotiation\",\"108\":\"Approved for use\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":7,\"3\":8,\"5\":9,\"6\":3,\"11\":6,\"14\":5,\"15\":4,\"108\":2},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"11\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"},\"15\":{\"color\":\"#9CD326\",\"border\":\"#89B921\",\"var_name\":\"lime-green\"},\"108\":{\"color\":\"#4eccc6\",\"border\":\"#4eccc6\",\"var_name\":\"australia\"}}}", "title": "Procurement status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Manager", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Manager approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "budget_owner", "settings_str": "{}", "title": "POC owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "budget_owner_approval4", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "POC status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "manager", "settings_str": "{}", "title": "Budget owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "status4", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Budget owner approval", "type": "color", "width": 185}, {"archived": false, "description": null, "id": "people", "settings_str": "{}", "title": "Procurement team", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "budget_owner_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Procurement approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "procurement_team", "settings_str": "{}", "title": "Finance", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "procurement_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Finance approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "finance", "settings_str": "{}", "title": "Legal", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "finance_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Redlines\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Legal approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "file", "settings_str": "{}", "title": "File", "type": "file", "width": null}, {"archived": false, "description": null, "id": "legal", "settings_str": "{}", "title": "Security", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "legal_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Security approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "date", "settings_str": "{\"hide_footer\":false}", "title": "Renewal date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "last_updated", "settings_str": "{}", "title": "Last updated", "type": "pulse-updated", "width": 129}], "communication": null, "description": "Many IT departments need to handle the procurement process for new services. The essence of this board is to streamline this process by providing an intuitive structure that supports collaboration and efficiency.", "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Reviewing"}, {"archived": false, "color": "#FF642E", "deleted": false, "id": "new_group", "position": "98304.0", "title": "Corporate IT"}, {"archived": false, "color": "#037f4c", "deleted": false, "id": "new_group2816", "position": "114688.0", "title": "Finance"}], "id": "3555407826", "name": "Procurement process", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:36:50Z", "updates": [], "views": [{"id": "80969928", "name": "Chart", "settings_str": "{\"x_axis_columns\":{\"status1\":true},\"y_axis_columns\":{\"default-label-count\":true},\"z_axis_columns\":{},\"guideline_base\":{},\"graph_type\":\"column\",\"empty_values\":false,\"group_by\":\"month\"}", "type": "GraphBoardView", "view_specific_data_str": "{}"}], "workspace": null, "updated_at_int": 1669041410}, "emitted_at": 1690884065408} +{"stream": "tags", "data": {"color": "#00c875", "id": 19038090, "name": "open"}, "emitted_at": 1690884065804} +{"stream": "tags", "data": {"color": "#fdab3d", "id": 19038091, "name": "closed"}, "emitted_at": 1690884065806} +{"stream": "updates", "data": {"assets": [{"created_at": "2023-06-15T16:19:31Z", "file_extension": ".jpg", "file_size": 116107, "id": "919077184", "name": "black_cat.jpg", "original_geometry": "473x600", "public_url": "https://files-monday-com.s3.amazonaws.com/14202902/resources/919077184/black_cat.jpg?response-content-disposition=attachment&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA4MPVJMFXGWGLJTLY%2F20230801%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230801T100107Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=5d2d3ca95375589e620f89630d58ff0f7417f1ddd8968ceb57af854657718564", "uploaded_by": {"id": 36694549}, "url": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/black_cat.jpg", "url_thumbnail": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/thumb_small-black_cat.jpg"}], "body": "", "created_at": "2023-06-15T16:19:36Z", "creator_id": "36694549", "id": "2223820299", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:19:36Z"}, "emitted_at": 1690884067025} +{"stream": "updates", "data": {"assets": [], "body": "



    ", "created_at": "2023-06-15T16:18:50Z", "creator_id": "36694549", "id": "2223818363", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:18:50Z"}, "emitted_at": 1690884067027} +{"stream": "updates", "data": {"assets": [], "body": "

    \ufeffTest

    ", "created_at": "2022-11-21T14:41:21Z", "creator_id": "36694549", "id": "1825302913", "item_id": "3555437747", "replies": [{"id": "1825303266", "creator_id": "36694549", "created_at": "2022-11-21T14:41:29Z", "text_body": "Test test", "updated_at": "2022-11-21T14:41:29Z", "body": "

    \ufeffTest test

    "}, {"id": "2223806079", "creator_id": "36694549", "created_at": "2023-06-15T16:14:13Z", "text_body": "", "updated_at": "2023-06-15T16:14:13Z", "body": "



    "}], "text_body": "Test", "updated_at": "2023-06-15T16:14:13Z"}, "emitted_at": 1690884067029} +{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:03:00Z", "join_date": null, "email": "integration-test@airbyte.io", "enabled": true, "id": 36694549, "is_admin": true, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Airbyte Team", "phone": "", "photo_original": "https://files.monday.com/use1/photos/36694549/original/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_small": "https://files.monday.com/use1/photos/36694549/small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb": "https://files.monday.com/use1/photos/36694549/thumb/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb_small": "https://files.monday.com/use1/photos/36694549/thumb_small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_tiny": "https://files.monday.com/use1/photos/36694549/tiny/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "time_zone_identifier": "Europe/Kiev", "title": "Airbyte Developer Account", "url": "https://airbyte-unit.monday.com/users/36694549", "utc_hours_diff": 3}, "emitted_at": 1690884067354} +{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:33:18Z", "join_date": null, "email": "iryna.grankova@airbyte.io", "enabled": true, "id": 36695702, "is_admin": false, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Iryna Grankova", "phone": null, "photo_original": "https://files.monday.com/use1/photos/36695702/original/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_small": "https://files.monday.com/use1/photos/36695702/small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb": "https://files.monday.com/use1/photos/36695702/thumb/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb_small": "https://files.monday.com/use1/photos/36695702/thumb_small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_tiny": "https://files.monday.com/use1/photos/36695702/tiny/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "time_zone_identifier": "Europe/Athens", "title": null, "url": "https://airbyte-unit.monday.com/users/36695702", "utc_hours_diff": 3}, "emitted_at": 1690884067356} +{"stream": "workspaces", "data": {"created_at": "2023-06-08T11:26:44Z", "description": null, "id": 2845647, "kind": "open", "name": "Test workspace", "state": "active", "account_product": {"id": 2248222, "kind": "core"}, "owners_subscribers": [{"id": 36694549}], "settings": {"icon": {"color": "#FDAB3D", "image": null}}, "team_owners_subscribers": [], "teams_subscribers": [], "users_subscribers": [{"id": 36694549}]}, "emitted_at": 1690884067856} +{"stream": "activity_logs", "data": {"id": "81d07d4d-414d-458e-b44c-fef36e44c424", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"new_group\",\"group_name\":\"New Group unit board\",\"group_color\":\"#808080\",\"is_top_group\":false,\"pulse_id\":4672924165,\"pulse_name\":\"Item 7\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631837419768", "created_at_int": 1687263183, "pulse_id": 4672924165}, "emitted_at": 1690884068262} +{"stream": "activity_logs", "data": {"id": "c0aa4bab-d3a4-4f13-8942-934178c0238a", "event": "update_column_value", "data": "{\"board_id\":4635211873,\"group_id\":\"group_title\",\"is_top_group\":false,\"pulse_id\":4672922929,\"pulse_name\":\"Item 6\",\"column_id\":\"status\",\"column_type\":\"color\",\"column_title\":\"Status\",\"value\":{\"label\":{\"index\":1,\"text\":\"Done\",\"style\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"is_done\":true},\"post_id\":null},\"previous_value\":null,\"is_column_with_hide_permissions\":false}", "entity": "pulse", "created_at": "16872631743009674", "created_at_int": 1687263174, "pulse_id": 4672922929}, "emitted_at": 1690884068266} +{"stream": "activity_logs", "data": {"id": "4e8f926c-1b4e-43d8-a3de-09af876ccb9e", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"group_title\",\"group_name\":\"Group Title\",\"group_color\":\"#a25ddc\",\"is_top_group\":false,\"pulse_id\":4672922929,\"pulse_name\":\"Item 6\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631712788820", "created_at_int": 1687263171, "pulse_id": 4672922929}, "emitted_at": 1690884068269} diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-monday/integration_tests/incremental_catalog.json new file mode 100644 index 000000000000..dbe78bcc0d69 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/integration_tests/incremental_catalog.json @@ -0,0 +1,46 @@ +{ + "streams": [ + { + "stream": { + "name": "activity_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at_int"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at_int"], + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at_int"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at_int"], + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "boards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at_int"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at_int"], + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + } + ] +} diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-monday/integration_tests/sample_state.json index 3587e579822d..f66e60dd4c2d 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-monday/integration_tests/sample_state.json @@ -1,5 +1,41 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "items" + }, + "stream_state": { + "updated_at_int": 1677263687, + "activity_logs": { + "created_at_int": 1677263687 + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "boards" + }, + "stream_state": { + "updated_at_int": 1677263687, + "activity_logs": { + "created_at_int": 1677263687 + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "activity_logs" + }, + "stream_state": { + "created_at_int": 1677263687 + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-monday/metadata.yaml b/airbyte-integrations/connectors/source-monday/metadata.yaml index cb9d187df5b3..2c85a0d98294 100644 --- a/airbyte-integrations/connectors/source-monday/metadata.yaml +++ b/airbyte-integrations/connectors/source-monday/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 80a54ea2-9959-4040-aac1-eee42423ec9b - dockerImageTag: 0.2.5 + dockerImageTag: 1.1.2 dockerRepository: airbyte/source-monday githubIssueLabel: source-monday icon: monday.svg @@ -16,9 +16,13 @@ data: enabled: true oss: enabled: true - releaseStage: beta + releaseStage: generally_available documentationUrl: https://docs.airbyte.com/integrations/sources/monday tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-monday/requirements.txt b/airbyte-integrations/connectors/source-monday/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-monday/requirements.txt +++ b/airbyte-integrations/connectors/source-monday/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-monday/setup.py b/airbyte-integrations/connectors/source-monday/setup.py index 4d2beea9a22c..4af4bd4dac1c 100644 --- a/airbyte-integrations/connectors/source-monday/setup.py +++ b/airbyte-integrations/connectors/source-monday/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk>=0.44.1", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-monday/source_monday/__init__.py b/airbyte-integrations/connectors/source-monday/source_monday/__init__.py index 8e3109b2c13e..ec982cff1605 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/__init__.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from .graphql_requester import MondayGraphqlRequester diff --git a/airbyte-integrations/connectors/source-monday/source_monday/components.py b/airbyte-integrations/connectors/source-monday/source_monday/components.py new file mode 100644 index 000000000000..fde4f2f3b958 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/components.py @@ -0,0 +1,217 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass, field +from typing import Any, Iterable, List, Mapping, Optional, Union + +import dpath.util +from airbyte_cdk.models import AirbyteMessage, SyncMode, Type +from airbyte_cdk.sources.declarative.incremental import Cursor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.streams.core import Stream + +RequestInput = Union[str, Mapping[str, str]] + + +@dataclass +class IncrementalSingleSlice(Cursor): + cursor_field: Union[InterpolatedString, str] + config: Config + parameters: InitVar[Mapping[str, Any]] + + def __post_init__(self, parameters: Mapping[str, Any]): + self._state = {} + self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) + + def get_request_params( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.request_parameter, stream_slice) + + def get_request_headers( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.header, stream_slice) + + def get_request_body_data( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.body_data, stream_slice) + + def get_request_body_json( + self, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Optional[Mapping]: + # Pass the stream_slice from the argument, not the cursor because the cursor is updated after processing the response + return self._get_request_option(RequestOptionType.body_json, stream_slice) + + def _get_request_option(self, option_type: RequestOptionType, stream_slice: StreamSlice): + return {} + + def get_stream_state(self) -> StreamState: + return self._state + + def set_initial_state(self, stream_state: StreamState): + cursor_value = stream_state.get(self.cursor_field.eval(self.config)) + if cursor_value: + self._state[self.cursor_field.eval(self.config)] = cursor_value + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + latest_record = self._state if self.is_greater_than_or_equal(self._state, most_recent_record) else most_recent_record + + if not latest_record: + return + self._state[self.cursor_field.eval(self.config)] = latest_record[self.cursor_field.eval(self.config)] + + def stream_slices(self) -> Iterable[Mapping[str, Any]]: + yield {} + + def should_be_synced(self, record: Record) -> bool: + """ + As of 2023-06-28, the expectation is that this method will only be used for semi-incremental and data feed and therefore the + implementation is irrelevant for greenhouse + """ + return True + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice + """ + first_cursor_value = first.get(self.cursor_field.eval(self.config)) if first else None + second_cursor_value = second.get(self.cursor_field.eval(self.config)) if second else None + if first_cursor_value and second_cursor_value: + return first_cursor_value > second_cursor_value + elif first_cursor_value: + return True + else: + return False + + +@dataclass +class IncrementalSubstreamSlicer(IncrementalSingleSlice): + """ + Like SubstreamSlicer, but works incrementaly with both parent and substream. + + Input Arguments: + + :: cursor_field: srt - substream cursor_field value + :: parent_complete_fetch: bool - If `True`, all slices is fetched into a list first, then yield. + If `False`, substream emits records on each parernt slice yield. + :: parent_stream_configs: ParentStreamConfig - Describes how to create a stream slice from a parent stream. + + """ + + config: Config + parameters: InitVar[Mapping[str, Any]] + cursor_field: Union[InterpolatedString, str] + parent_stream_configs: List[ParentStreamConfig] + nested_items_per_page: int + parent_complete_fetch: bool = field(default=False) + + def __post_init__(self, parameters: Mapping[str, Any]): + super().__post_init__(parameters) + if not self.parent_stream_configs: + raise ValueError("IncrementalSubstreamSlicer needs at least 1 parent stream") + self.cursor_field = InterpolatedString.create(self.cursor_field, parameters=parameters) + # parent stream parts + self.parent_config: ParentStreamConfig = self.parent_stream_configs[0] + self.parent_stream: Stream = self.parent_config.stream + self.parent_stream_name: str = self.parent_stream.name + self.parent_cursor_field: str = self.parent_stream.cursor_field + self.parent_sync_mode: SyncMode = SyncMode.incremental if self.parent_stream.supports_incremental is True else SyncMode.full_refresh + self.substream_slice_field: str = self.parent_stream_configs[0].partition_field.eval(self.config) + self.parent_field: str = self.parent_stream_configs[0].parent_key.eval(self.config) + + def set_initial_state(self, stream_state: StreamState): + cursor_value = stream_state.get(self.cursor_field.eval(self.config)) + if cursor_value: + self._state[self.cursor_field.eval(self.config)] = cursor_value + if self.parent_stream_name in stream_state and stream_state.get(self.parent_stream_name, {}).get(self.parent_cursor_field): + self._state[self.parent_stream_name] = { + self.parent_cursor_field: stream_state[self.parent_stream_name][self.parent_cursor_field] + } + + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: + latest_record = self._state if self.is_greater_than_or_equal(self._state, most_recent_record) else most_recent_record + + if not latest_record: + return + + max_state = latest_record[self.cursor_field.eval(self.config)] + self._state[self.cursor_field.eval(self.config)] = max_state + + if self.parent_stream: + parent_state = self.parent_stream.state or {self.parent_cursor_field: max_state} + self._state[self.parent_stream_name] = parent_state + + def read_parent_stream( + self, sync_mode: SyncMode, cursor_field: Optional[str], stream_state: Mapping[str, Any] + ) -> Iterable[Mapping[str, Any]]: + self.parent_stream.state = stream_state + + # check if state is empty -> + if not stream_state.get(self.parent_cursor_field): + # yield empty slice for complete fetch of items stream + yield {} + return + + all_ids = set() + slice_ids = list() + empty_parent_slice = True + + for parent_slice in self.parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state): + for parent_record in self.parent_stream.read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=parent_slice, stream_state=stream_state + ): + # Skip non-records (eg AirbyteLogMessage) + if isinstance(parent_record, AirbyteMessage): + if parent_record.type == Type.RECORD: + parent_record = parent_record.record.data + + try: + substream_slice = dpath.util.get(parent_record, self.parent_field) + except KeyError: + pass + else: + empty_parent_slice = False + + # check if record with this id was already processed + if substream_slice not in all_ids: + all_ids.add(substream_slice) + slice_ids.append(substream_slice) + + # yield slice with desired number of ids + if self.nested_items_per_page == len(slice_ids): + yield {self.substream_slice_field: slice_ids} + slice_ids = list() + # yield leftover ids if any left + if slice_ids: + yield {self.substream_slice_field: slice_ids} + + # If the parent slice contains no records + if empty_parent_slice: + yield from [] + + def stream_slices(self) -> Iterable[Mapping[str, Any]]: + parent_state = (self._state or {}).get(self.parent_stream_name, {}) + + slices_generator = self.read_parent_stream(self.parent_sync_mode, self.parent_cursor_field, parent_state) + yield from [slice for slice in slices_generator] if self.parent_complete_fetch else slices_generator diff --git a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py new file mode 100644 index 000000000000..bd8524044024 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from dataclasses import InitVar, dataclass, field +from datetime import datetime +from typing import Any, List, Mapping, Union + +import dpath.util +import requests +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config, Record + +logger = logging.getLogger("airbyte") + + +@dataclass +class MondayActivityExtractor(RecordExtractor): + """ + Record extractor that extracts record of the form from activity logs stream: + + { "list": { "ID_1": record_1, "ID_2": record_2, ... } } + + Attributes: + parameters (Mapping[str, Any]): Additional runtime parameters to be used for string interpolation + decoder (Decoder): The decoder responsible to transfom the response in a Mapping + """ + + parameters: InitVar[Mapping[str, Any]] + decoder: Decoder = JsonDecoder(parameters={}) + + def extract_records(self, response: requests.Response) -> List[Record]: + response_body = self.decoder.decode(response) + result = [] + if not response_body["data"]["boards"]: + return result + + for board_data in response_body["data"]["boards"]: + if not isinstance(board_data, dict): + continue + + for record in board_data.get("activity_logs", []): + json_data = json.loads(record["data"]) + new_record = record + if record.get("created_at"): + new_record.update({"created_at_int": int(record.get("created_at", 0)) // 10_000_000}) + else: + continue + + if record.get("entity") == "pulse" and json_data.get("pulse_id"): + new_record.update({"pulse_id": json_data.get("pulse_id")}) + + if record.get("entity") == "board" and json_data.get("board_id"): + new_record.update({"board_id": json_data.get("board_id")}) + + result.append(new_record) + + return result + + +@dataclass +class MondayIncrementalItemsExtractor(RecordExtractor): + """ + Record extractor that searches a decoded response over a path defined as an array of fields. + """ + + field_path: List[Union[InterpolatedString, str]] + config: Config + parameters: InitVar[Mapping[str, Any]] + additional_field_path: List[Union[InterpolatedString, str]] = field(default_factory=list) + decoder: Decoder = JsonDecoder(parameters={}) + + def __post_init__(self, parameters: Mapping[str, Any]): + for field_list in (self.field_path, self.additional_field_path): + for path_index in range(len(field_list)): + if isinstance(field_list[path_index], str): + field_list[path_index] = InterpolatedString.create(field_list[path_index], parameters=parameters) + + def try_extract_records(self, response: requests.Response, field_path: List[Union[InterpolatedString, str]]) -> List[Record]: + response_body = self.decoder.decode(response) + + path = [path.eval(self.config) for path in field_path] + extracted = dpath.util.values(response_body, path) if path else response_body + + pattern_path = "*" in path + if not pattern_path: + extracted = dpath.util.get(response_body, path, default=[]) + + if extracted: + if isinstance(extracted, list) and None in extracted: + logger.warning(f"Record with null value received; errors: {response_body.get('errors')}") + return [x for x in extracted if x] + return extracted if isinstance(extracted, list) else [extracted] + return [] + + def extract_records(self, response: requests.Response) -> List[Record]: + result = self.try_extract_records(response, field_path=self.field_path) + if not result and self.additional_field_path: + result = self.try_extract_records(response, self.additional_field_path) + + for item_index in range(len(result)): + if "updated_at" in result[item_index]: + result[item_index]["updated_at_int"] = int( + datetime.strptime(result[item_index]["updated_at"], "%Y-%m-%dT%H:%M:%S%z").timestamp() + ) + return result diff --git a/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py b/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py index 7432a583b380..0e5fd049583c 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py @@ -3,6 +3,7 @@ # from dataclasses import dataclass +from datetime import datetime from functools import partial from typing import Any, Mapping, MutableMapping, Optional, Type, Union @@ -15,14 +16,15 @@ @dataclass class MondayGraphqlRequester(HttpRequester): NEXT_PAGE_TOKEN_FIELD_NAME = "next_page_token" - NESTED_OBJECTS_LIMIT_MAX_VALUE = 100 limit: Union[InterpolatedString, str, int] = None + nested_limit: Union[InterpolatedString, str, int] = None def __post_init__(self, parameters: Mapping[str, Any]): super(MondayGraphqlRequester, self).__post_init__(parameters) self.limit = InterpolatedString.create(self.limit, parameters=parameters) + self.nested_limit = InterpolatedString.create(self.nested_limit, parameters=parameters) self.name = parameters.get("name", "").lower() def _ensure_type(self, t: Type, o: Any): @@ -34,11 +36,26 @@ def _ensure_type(self, t: Type, o: Any): def _get_schema_root_properties(self): schema_loader = JsonFileSchemaLoader(config=self.config, parameters={"name": self.name}) - schema = schema_loader.get_json_schema() - return schema["properties"] + schema = schema_loader.get_json_schema()["properties"] + + # delete fields that will be created by extractor + delete_fields = ["updated_at_int", "created_at_int", "pulse_id"] + if self.name == "activity_logs": + delete_fields.append("board_id") + for field in delete_fields: + if field in schema: + schema.pop(field) + + return schema def _get_object_arguments(self, **object_arguments) -> str: - return ",".join([f"{argument}:{value}" for argument, value in object_arguments.items() if value is not None]) + return ",".join( + [ + f"{argument}:{value}" if argument != "fromt" else f'from:"{value}"' + for argument, value in object_arguments.items() + if value is not None + ] + ) def _build_query(self, object_name: str, field_schema: dict, **object_arguments) -> str: """ @@ -69,14 +86,27 @@ def _build_items_query(self, object_name: str, field_schema: dict, sub_page: Opt Special optimization needed for items stream. Starting October 3rd, 2022 items can only be reached through boards. See https://developer.monday.com/api-reference/docs/items-queries#items-queries """ - query = self._build_query(object_name, field_schema, limit=self.NESTED_OBJECTS_LIMIT_MAX_VALUE, page=sub_page) + nested_limit = self.nested_limit.eval(self.config) + + query = self._build_query("items", field_schema, limit=nested_limit, page=sub_page) arguments = self._get_object_arguments(**object_arguments) return f"boards({arguments}){{{query}}}" + def _build_items_incremental_query(self, object_name: str, field_schema: dict, stream_slice: dict, **object_arguments) -> str: + """ + Special optimization needed for items stream. Starting October 3rd, 2022 items can only be reached through boards. + See https://developer.monday.com/api-reference/docs/items-queries#items-queries + """ + nested_limit = self.nested_limit.eval(self.config) + + object_arguments["limit"] = nested_limit + object_arguments["ids"] = stream_slice["ids"] + return self._build_query("items", field_schema, **object_arguments) + def _build_teams_query(self, object_name: str, field_schema: dict, **object_arguments) -> str: """ Special optimization needed for tests to pass successfully because of rate limits. - It makes a query cost less points and should not be used to production + It makes a query cost less points, but it is never used in production """ teams_limit = self.config.get("teams_limit") if teams_limit: @@ -86,6 +116,23 @@ def _build_teams_query(self, object_name: str, field_schema: dict, **object_argu return f"{object_name}({arguments}){query}" return self._build_query(object_name=object_name, field_schema=field_schema, **object_arguments) + def _build_activity_query(self, object_name: str, field_schema: dict, sub_page: Optional[int], **object_arguments) -> str: + """ + Special optimization needed for items stream. Starting October 3rd, 2022 items can only be reached through boards. + See https://developer.monday.com/api-reference/docs/items-queries#items-queries + """ + nested_limit = self.nested_limit.eval(self.config) + + created_at = (object_arguments.get("stream_state", dict()) or dict()).get("created_at_int") + object_arguments.pop("stream_state") + + if created_at: + created_at = datetime.fromtimestamp(created_at).strftime("%Y-%m-%dT%H:%M:%SZ") + + query = self._build_query(object_name, field_schema, limit=nested_limit, page=sub_page, fromt=created_at) + arguments = self._get_object_arguments(**object_arguments) + return f"boards({arguments}){{{query}}}" + def get_request_params( self, *, @@ -97,13 +144,22 @@ def get_request_params( Combines queries to a single GraphQL query. """ limit = self.limit.eval(self.config) + page = next_page_token and next_page_token[self.NEXT_PAGE_TOKEN_FIELD_NAME] - if self.name == "items": + if self.name == "boards" and stream_slice: + query_builder = partial(self._build_query, **stream_slice) + elif self.name == "items": # `items` stream use a separate pagination strategy where first level pages are across `boards` and sub-pages are across `items` page, sub_page = page if page else (None, None) - query_builder = partial(self._build_items_query, sub_page=sub_page) + if not stream_slice: + query_builder = partial(self._build_items_query, sub_page=sub_page) + else: + query_builder = partial(self._build_items_incremental_query, stream_slice=stream_slice) elif self.name == "teams": query_builder = self._build_teams_query + elif self.name == "activity_logs": + page, sub_page = page if page else (None, None) + query_builder = partial(self._build_activity_query, sub_page=sub_page, stream_state=stream_state) else: query_builder = self._build_query query = query_builder( diff --git a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py index a46ab8a18e09..5b18cb4b37b7 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py @@ -39,7 +39,7 @@ def next_page_token(self, response, last_records: List[Mapping[str, Any]]) -> Op self._sub_page += 1 else: self._sub_page = self.start_from_page - if len(response.json()["data"]["boards"]): + if response.json()["data"].get("boards"): self._page += 1 else: return None diff --git a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml index 12a85bc7f42d..dc482bae0f68 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml +++ b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml @@ -18,16 +18,37 @@ definitions: http_method: "GET" authenticator: type: BearerAuthenticator - api_token: "{{ config['credentials']['api_token'] if config['credentials']['auth_type'] == 'api_token' else config['credentials']['access_token'] if config['credentials']['auth_type'] == 'oauth2.0' else config.get('api_token', '') }}" + api_token: "{{ config.get('credentials', {}).get('api_token') if config.get('credentials', {}).get('auth_type') == 'api_token' else config.get('credentials', {}).get('access_token') if config.get('credentials', {}).get('auth_type') == 'oauth2.0' else config.get('api_token', '') }}" limit: "{{ parameters['items_per_page'] }}" + nested_limit: "{{ parameters.get('nested_items_per_page', 1) }}" error_handler: - type: "DefaultErrorHandler" - response_filters: - - predicate: "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" - action: RETRY - backoff_strategies: - - type: ConstantBackoffStrategy - backoff_time_in_seconds: 60 + type: CompositeErrorHandler + error_handlers: + - type: "DefaultErrorHandler" + response_filters: + - predicate: "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" + action: RETRY + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 60 + - type: "DefaultErrorHandler" + description: " + Ignore the slice when there is no access to the requested entity - 'code: 403, message: None'. + https://github.com/airbytehq/alpha-beta-issues/issues/846 + " + response_filters: + - http_codes: [403] + action: IGNORE + - type: "DefaultErrorHandler" + description: " + Retry when `Internal Server Error occures - `code: 500, message: Internal server error`. + https://github.com/airbytehq/alpha-beta-issues/issues/245 + " + response_filters: + - http_codes: [500, 502] + action: RETRY + backoff_strategies: + - type: ExponentialBackoffStrategy default_paginator: type: "DefaultPaginator" pagination_strategy: @@ -52,30 +73,11 @@ definitions: $ref: "#/definitions/retriever" paginator: type: NoPagination - items_stream: - $ref: "#/definitions/base_stream" - retriever: - $ref: "#/definitions/retriever" - record_selector: - type: RecordSelector - extractor: - field_path: ["data", "boards", "*", "items", "*"] - paginator: - $ref: "#/definitions/default_paginator" - pagination_strategy: - class_name: "source_monday.item_pagination_strategy.ItemPaginationStrategy" - type: "CustomPaginationStrategy" - page_size: 100 - $parameters: - name: "items" - path: "" - items_per_page: 1 - boards_stream: - $ref: "#/definitions/base_stream" + tags_stream: + $ref: "#/definitions/base_nopagination_stream" $parameters: - name: "boards" + name: "tags" path: "" - items_per_page: 100 teams_stream: $ref: "#/definitions/base_nopagination_stream" $parameters: @@ -91,13 +93,106 @@ definitions: $parameters: name: "users" path: "" + workspaces_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "workspaces" + path: "" + + double_paginator: + $ref: "#/definitions/default_paginator" + pagination_strategy: + class_name: "source_monday.item_pagination_strategy.ItemPaginationStrategy" + type: "CustomPaginationStrategy" + + activity_logs_stream: + description: "https://developers.intercom.com/intercom-api-reference/reference/scroll-over-all-companies" + incremental_sync: + type: CustomIncrementalSync + class_name: source_monday.components.IncrementalSingleSlice + cursor_field: "created_at_int" + retriever: + $ref: "#/definitions/base_stream/retriever" + record_selector: + $ref: "#/definitions/selector" + extractor: + class_name: "source_monday.extractor.MondayActivityExtractor" + paginator: + $ref: "#/definitions/double_paginator" + $parameters: + name: "activity_logs" + primary_key: "id" + path: "" + items_per_page: 1 + page_size: 50 + nested_items_per_page: 50 + + # semi-incremental substreams + substream_semi_incremental: + $ref: "#/definitions/base_stream" + incremental_sync: + type: CustomIncrementalSync + class_name: source_monday.components.IncrementalSubstreamSlicer + cursor_field: "updated_at_int" + parent_complete_fetch: true + parent_stream_configs: + - type: ParentStreamConfig + stream: "#/definitions/activity_logs_stream" + partition_field: "ids" + incremental_extractor: + class_name: "source_monday.extractor.MondayIncrementalItemsExtractor" + + boards_stream: + $ref: "#/definitions/substream_semi_incremental" + $parameters: + name: "boards" + path: "" + parent_key: "board_id" + items_per_page: 10 + nested_items_per_page: 10 + field_path: ["data", "boards", "*"] + retriever: + $ref: "#/definitions/base_stream/retriever" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "PageIncrement" + start_from_page: 1 + page_size: 10 + record_selector: + $ref: "#/definitions/selector" + extractor: + $ref: "#/definitions/incremental_extractor" + + items_stream: + $ref: "#/definitions/substream_semi_incremental" + $parameters: + name: "items" + path: "" + items_per_page: 1 + page_size: 20 + nested_items_per_page: 20 + parent_key: "pulse_id" + field_path: ["data", "items", "*"] + additional_field_path: ["data", "boards", "*", "items", "*"] + retriever: + $ref: "#/definitions/base_stream/retriever" + paginator: + $ref: "#/definitions/double_paginator" + record_selector: + $ref: "#/definitions/selector" + extractor: + $ref: "#/definitions/incremental_extractor" streams: - "#/definitions/items_stream" - "#/definitions/boards_stream" + - "#/definitions/tags_stream" - "#/definitions/teams_stream" - "#/definitions/updates_stream" - "#/definitions/users_stream" + - "#/definitions/workspaces_stream" + - "#/definitions/activity_logs_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/activity_logs.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/activity_logs.json new file mode 100644 index 000000000000..9b5af113d9af --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/activity_logs.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": ["null", "string"] }, + "event": { "type": ["null", "string"] }, + "data": { "type": ["null", "string"] }, + "entity": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "created_at_int": { "type": ["null", "integer"] }, + "pulse_id": { "type": ["null", "integer"] }, + "board_id": { "type": ["null", "integer"] } + } +} diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json index 69f57548890b..e976ea5e54f4 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json @@ -5,31 +5,50 @@ "board_kind": { "type": ["null", "string"] }, "columns": { "type": ["null", "array"], - "properties": { - "archived": { "type": ["null", "boolean"] }, - "id": { "type": ["null", "string"] }, - "settings_str": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, - "type": { "type": ["null", "string"] }, - "width": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "archived": { "type": ["null", "boolean"] }, + "description": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "settings_str": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "width": { "type": ["null", "integer"] } + } } }, - "communication": { "type": ["null", "object"] }, + "communication": { "type": ["null", "string"] }, "description": { "type": ["null", "string"] }, "groups": { "type": ["null", "array"], - "properties": { - "archived": { "type": ["null", "boolean"] }, - "color": { "type": ["null", "string"] }, - "deleted": { "type": ["null", "boolean"] }, - "id": { "type": ["null", "string"] }, - "position": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "archived": { "type": ["null", "boolean"] }, + "color": { "type": ["null", "string"] }, + "deleted": { "type": ["null", "boolean"] }, + "id": { "type": ["null", "string"] }, + "position": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } } }, "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, - "owner": { + "owners": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } + } + }, + "creator": { "type": ["null", "object"], "properties": { "id": { "type": ["null", "integer"] } @@ -40,14 +59,22 @@ "state": { "type": ["null", "string"] }, "subscribers": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } }, "tags": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } }, "top_group": { @@ -57,16 +84,29 @@ } }, "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "updated_at_int": { "type": ["null", "integer"] }, "updates": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } }, "views": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "settings_str": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "view_specific_data_str": { "type": ["null", "string"] } + } } }, "workspace": { diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json index 66763c3e21c2..27f02d52354f 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json @@ -4,8 +4,26 @@ "properties": { "assets": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "file_extension": { "type": ["null", "string"] }, + "file_size": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "original_geometry": { "type": ["null", "string"] }, + "public_url": { "type": ["null", "string"] }, + "uploaded_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] } + } + }, + "url": { "type": ["null", "string"] }, + "url_thumbnail": { "type": ["null", "string"] } + } } }, "board": { @@ -16,13 +34,18 @@ }, "column_values": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] }, - "value": { "type": ["null", "object"] }, - "additional_info": { "type": ["null", "object"] }, - "text": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, - "type": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "additional_info": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] }, + "text": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "value": { "type": ["null", "string"] } + } } }, "created_at": { "type": ["null", "string"], "format": "date-time" }, @@ -44,15 +67,24 @@ "state": { "type": ["null", "string"] }, "subscribers": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } }, "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "updated_at_int": { "type": ["null", "integer"] }, "updates": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] } + } } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json new file mode 100644 index 000000000000..c96a58d1d42a --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "color": { "type": ["null", "string"] }, + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] } + } +} diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json index c409d05b6bbd..16cb865fcc92 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json @@ -7,8 +7,12 @@ "picture_url": { "type": ["null", "string"] }, "users": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json index c35d55986103..d0e004a69982 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json @@ -4,8 +4,26 @@ "properties": { "assets": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "file_extension": { "type": ["null", "string"] }, + "file_size": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "original_geometry": { "type": ["null", "string"] }, + "public_url": { "type": ["null", "string"] }, + "uploaded_by": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] } + } + }, + "url": { "type": ["null", "string"] }, + "url_thumbnail": { "type": ["null", "string"] } + } } }, "body": { "type": ["null", "string"] }, @@ -15,13 +33,17 @@ "item_id": { "type": ["null", "string"] }, "replies": { "type": ["null", "array"], - "properties": { - "id": { "type": ["null", "integer"] }, - "creator_id": { "type": ["null", "integer"] }, - "created_at": { "type": ["null", "string"], "format": "date-time" }, - "text_body": { "type": ["null", "string"] }, - "updated_at": { "type": ["null", "string"], "format": "date-time" }, - "body": { "type": ["null", "string"] } + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "creator_id": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "text_body": { "type": ["null", "string"] }, + "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "body": { "type": ["null", "string"] } + } } }, "text_body": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json new file mode 100644 index 000000000000..af7bf79b2d97 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json @@ -0,0 +1,73 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "description": { "type": ["null", "string"] }, + "id": { "type": ["null", "integer"] }, + "kind": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "account_product": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "kind": { "type": ["null", "string"] } + } + }, + "owners_subscribers": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } + } + }, + "settings": { + "type": ["null", "object"], + "properties": { + "icon": { + "type": ["null", "object"], + "properties": { + "color": { "type": ["null", "string"] }, + "image": { "type": ["null", "string"] } + } + } + } + }, + "team_owners_subscribers": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] } + } + } + }, + "teams_subscribers": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] } + } + } + }, + "users_subscribers": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/__init__.py b/airbyte-integrations/connectors/source-monday/unit_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py new file mode 100644 index 000000000000..8b6c19fb6617 --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from typing import Any +from unittest.mock import MagicMock, Mock + +import pytest +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from airbyte_cdk.sources.streams import Stream +from requests import Response +from source_monday.components import IncrementalSingleSlice, IncrementalSubstreamSlicer +from source_monday.extractor import MondayIncrementalItemsExtractor + + +def _create_response(content: Any) -> Response: + response = Response() + response._content = json.dumps(content).encode("utf-8") + return response + + +def test_slicer(): + date_time_dict = {"updated_at": 1662459010} + slicer = IncrementalSingleSlice(config={}, parameters={}, cursor_field="updated_at") + slicer.close_slice(date_time_dict, date_time_dict) + assert slicer.get_stream_state() == date_time_dict + assert slicer.get_request_headers() == {} + assert slicer.get_request_body_data() == {} + assert slicer.get_request_body_json() == {} + + +@pytest.mark.parametrize( + "last_record, expected, records", + [ + ( + {"first_stream_cursor": 1662459010}, + {'parent_stream_name': {'parent_cursor_field': 1662459010}, 'first_stream_cursor': 1662459010}, + [{"first_stream_cursor": 1662459010}], + ), + (None, {}, []), + ], +) +def test_sub_slicer(last_record, expected, records): + parent_stream = Mock(spec=Stream) + parent_stream.name = "parent_stream_name" + parent_stream.cursor_field = "parent_cursor_field" + parent_stream.stream_slices.return_value = [{"a slice": "value"}] + parent_stream.read_records = MagicMock(return_value=records) + + parent_config = ParentStreamConfig( + stream=parent_stream, + parent_key="id", + partition_field="first_stream_id", + parameters={}, + config={}, + ) + + slicer = IncrementalSubstreamSlicer( + config={}, parameters={}, cursor_field="first_stream_cursor", parent_stream_configs=[parent_config], nested_items_per_page=10 + ) + stream_slice = next(slicer.stream_slices()) if records else {} + slicer.close_slice(stream_slice, last_record) + assert slicer.get_stream_state() == expected + + +def test_null_records(caplog): + extractor = MondayIncrementalItemsExtractor( + field_path=["data", "boards", "*"], + config={}, + parameters={}, + ) + content = { + "data": + {"boards": [ + {"board_kind": "private", "id": "1234561", "updated_at": "2023-08-15T10:30:49Z"}, + {"board_kind": "private", "id": "1234562", "updated_at": "2023-08-15T10:30:50Z"}, + {"board_kind": "private", "id": "1234563", "updated_at": "2023-08-15T10:30:51Z"}, + {"board_kind": "private", "id": "1234564", "updated_at": "2023-08-15T10:30:52Z"}, + {"board_kind": "private", "id": "1234565", "updated_at": "2023-08-15T10:30:43Z"}, + {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z"}, + None, + None + ]}, + "errors": [{"message": "Cannot return null for non-nullable field Board.creator"}], + "account_id": 123456} + response = _create_response(content) + records = extractor.extract_records(response) + warning_message = "Record with null value received; errors: [{'message': 'Cannot return null for non-nullable field Board.creator'}]" + assert warning_message in caplog.messages + expected_records = [ + {"board_kind": "private", "id": "1234561", "updated_at": "2023-08-15T10:30:49Z", "updated_at_int": 1692095449}, + {"board_kind": "private", "id": "1234562", "updated_at": "2023-08-15T10:30:50Z", "updated_at_int": 1692095450}, + {"board_kind": "private", "id": "1234563", "updated_at": "2023-08-15T10:30:51Z", "updated_at_int": 1692095451}, + {"board_kind": "private", "id": "1234564", "updated_at": "2023-08-15T10:30:52Z", "updated_at_int": 1692095452}, + {"board_kind": "private", "id": "1234565", "updated_at": "2023-08-15T10:30:43Z", "updated_at_int": 1692095443}, + {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z", "updated_at_int": 1692095454} + ] + assert records == expected_records diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py index 19c94851f706..ed79a1bce6bb 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py @@ -98,7 +98,8 @@ def test_get_request_params(mocker, input_schema, graphql_query, stream_name, co authenticator=MagicMock(), error_handler=MagicMock(), limit="{{ parameters['items_per_page'] }}", - parameters={"name": stream_name, "items_per_page": 100}, + nested_limit="{{ parameters.get('nested_items_per_page', 1) }}", + parameters={"name": stream_name, "items_per_page": 100, "nested_items_per_page": 100}, config=config ) assert requester.get_request_params( diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/.dockerignore b/airbyte-integrations/connectors/source-mongodb-internal-poc/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/Dockerfile b/airbyte-integrations/connectors/source-mongodb-internal-poc/Dockerfile new file mode 100644 index 000000000000..7b1aec14b1eb --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/Dockerfile @@ -0,0 +1,28 @@ +### WARNING ### +# The Java connector Dockerfiles will soon be deprecated. +# This Dockerfile is not used to build the connector image we publish to DockerHub. +# The new logic to build the connector image is declared with Dagger here: +# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 + +# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. +# Please reach out to the Connectors Operations team if you have any question. +FROM airbyte/integration-base-java:dev AS build + +WORKDIR /airbyte + +ENV APPLICATION source-mongodb-internal-poc + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar + +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-mongodb-internal-poc + +COPY --from=build /airbyte /airbyte + +LABEL io.airbyte.version=0.0.1 +LABEL io.airbyte.name=airbyte/source-mongodb-internal-poc diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/README.md b/airbyte-integrations/connectors/source-mongodb-internal-poc/README.md new file mode 100644 index 000000000000..8ec72f9f4466 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/README.md @@ -0,0 +1,60 @@ +# MongoDb Source (Internal POC) + +## Documentation +This is the repository for the MongoDb source connector in Java. +For information about how to use this connector within Airbyte, see [User Documentation](https://docs.airbyte.io/integrations/sources/mongodb-internal-poc) + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:build +``` + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +## Testing +We use `JUnit` for Java tests. + +### Test Configuration + +No specific configuration needed for testing Standalone MongoDb instance, MongoDb Test Container is used. +In order to test the MongoDb Atlas or Replica set, you need to provide configuration parameters. + +## Community Contributor + +As a community contributor, you will need to have an Atlas cluster to test MongoDb source. + +1. Create `secrets/credentials.json` file + 1. Insert below json to the file with your configuration + ``` + { + "database": "database_name", + "user": "username", + "password": "password", + "connection_string": "mongodb+srv://cluster0.abcd1.mongodb.net/", + "replica_set": "atlas-abcdefg-shard-0", + "auth_source": "auth_database" + } + ``` + +## Airbyte Employee + +1. Access the `MONGODB_TEST_CREDS` secret on LastPass +1. Create a file with the contents at `secrets/credentials.json` + + +#### Acceptance Tests +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:integrationTest +``` diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-config.yml new file mode 100644 index 000000000000..5db8fa612d87 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-config.yml @@ -0,0 +1,38 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-mongodb-internal-poc:dev +acceptance_tests: + spec: + tests: + - spec_path: "integration_tests/expected_spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.0.1" + config_path: "secrets/credentials.json" + timeout_seconds: 60 + connection: + tests: + - config_path: "secrets/credentials.json" + status: "succeed" + timeout_seconds: 60 + discovery: + bypass_reason: "The first version of this connector returns null when discovery is called, which causes the test to fail. We can stop bypassing this test once a new version of the connector is released." + #TODO: remove bypass_reason once a version that supports discovery is released + uncomment the lines below +# tests: +# - config_path: "secrets/credentials.json" +# backward_compatibility_tests_config: +# disable_for_version: "0.0.1" +# timeout_seconds: 60 + basic_read: + bypass_reason: "Full refresh syncs are not supported on this connector." + full_refresh: + bypass_reason: "Full refresh syncs are not supported on this connector." + incremental: + bypass_reason: "Incremental syncs are not yet supported by this connector." + #TODO: remove bypass_reason once a version that supports incremental syncs is released + uncomment the lines below +# tests: +# - config_path: "secrets/credentials.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# cursor_paths: +# listingsAndReviews: ["id"] +# timeout_seconds: 180 + diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-docker.sh new file mode 100755 index 000000000000..5797d20fe9a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-docker.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/build.gradle b/airbyte-integrations/connectors/source-mongodb-internal-poc/build.gradle new file mode 100644 index 000000000000..29bc652075ba --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/build.gradle @@ -0,0 +1,68 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' + id 'airbyte-connector-acceptance-test' + id 'org.jetbrains.kotlin.jvm' version '1.9.0' +} + +application { + mainClass = 'io.airbyte.integrations.source.mongodb.internal.MongoDbSource' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +} + +dependencies { + implementation libs.slf4j.api + implementation libs.jackson.databind + implementation project(':airbyte-db:db-lib') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-integrations:bases:debezium') + implementation libs.airbyte.protocol + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + implementation 'org.mongodb:mongodb-driver-sync:4.10.2' + + testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) + testImplementation "org.jetbrains.kotlinx:kotlinx-cli:0.3.5" + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-internal-poc') + integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) +} + +/* + * Executes the script that generates test data and inserts it into the provided database/collection. + * + * To execute this task, use the following command: + * + * ./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:generateTestData -PconnectionString= -PreplicaSet= -PdatabaseName= -PcollectionName= -Pusername= + * + * Optionally, you can provide -PnumberOfDocuments to change the number of generated documents from the default (10,000). + */ +tasks.register('generateTestData', JavaExec) { + def arguments = [] + + if(project.hasProperty('collectionName')) { + arguments.addAll(['--collection-name', collectionName]) + } + if(project.hasProperty('connectionString')) { + arguments.addAll(['--connection-string', connectionString]) + } + if(project.hasProperty('databaseName')) { + arguments.addAll(['--database-name', databaseName]) + } + if (project.hasProperty('numberOfDocuments')) { + arguments.addAll(['--number', numberOfDocuments]) + } + if(project.hasProperty('replicaSet')) { + arguments.addAll(['--replica-set', replicaSet]) + } + if(project.hasProperty('username')) { + arguments.addAll(['--username', username]) + } + + classpath = sourceSets.test.runtimeClasspath + main 'io.airbyte.integrations.source.mongodb.internal.MongoDbInsertClient' + standardInput = System.in + args arguments +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/icon.svg b/airbyte-integrations/connectors/source-mongodb-internal-poc/icon.svg new file mode 100644 index 000000000000..66b68e75556d --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..32f5f5691b81 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/configured_catalog.json @@ -0,0 +1,281 @@ +{ + "streams": [ + { + "stream": { + "name": "listingsAndReviews", + "json_schema": { + "type": "object", + "properties": { + "amenities": { + "type": "array" + }, + "notes": { + "type": "string" + }, + "access": { + "type": "string" + }, + "house_rules": { + "type": "string" + }, + "first_review": { + "type": "string" + }, + "calendar_last_scraped": { + "type": "string" + }, + "description": { + "type": "string" + }, + "neighborhood_overview": { + "type": "string" + }, + "_id_aibyte_transform": { + "type": "string" + }, + "availability": { + "type": "object", + "properties": { + "availability_365": { + "type": "number" + }, + "availability_30": { + "type": "number" + }, + "availability_60": { + "type": "number" + }, + "availability_90": { + "type": "number" + } + } + }, + "number_of_reviews": { + "type": "number" + }, + "space": { + "type": "string" + }, + "review_scores": { + "type": "object", + "properties": { + "review_scores_checkin": { + "type": "number" + }, + "review_scores_communication": { + "type": "number" + }, + "review_scores_rating": { + "type": "number" + }, + "review_scores_accuracy": { + "type": "number" + }, + "review_scores_location": { + "type": "number" + }, + "review_scores_value": { + "type": "number" + }, + "review_scores_cleanliness": { + "type": "number" + } + } + }, + "cleaning_fee": { + "type": "number" + }, + "reviews": { + "type": "array" + }, + "price": { + "type": "number" + }, + "reviews_per_month": { + "type": "number" + }, + "host": { + "type": "object", + "properties": { + "host_verifications": { + "type": "array" + }, + "host_url": { + "type": "string" + }, + "host_response_time": { + "type": "string" + }, + "host_has_profile_pic": { + "type": "boolean" + }, + "host_about": { + "type": "string" + }, + "host_picture_url": { + "type": "string" + }, + "host_id": { + "type": "string" + }, + "host_listings_count": { + "type": "number" + }, + "host_total_listings_count": { + "type": "number" + }, + "host_location": { + "type": "string" + }, + "host_is_superhost": { + "type": "boolean" + }, + "host_neighbourhood": { + "type": "string" + }, + "host_thumbnail_url": { + "type": "string" + }, + "host_response_rate": { + "type": "number" + }, + "host_name": { + "type": "string" + }, + "host_identity_verified": { + "type": "boolean" + } + } + }, + "property_type": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "monthly_price": { + "type": "number" + }, + "security_deposit": { + "type": "number" + }, + "images": { + "type": "object", + "properties": { + "picture_url": { + "type": "string" + }, + "xl_picture_url": { + "type": "string" + }, + "medium_url": { + "type": "string" + }, + "thumbnail_url": { + "type": "string" + } + } + }, + "address": { + "type": "object", + "properties": { + "market": { + "type": "string" + }, + "country_code": { + "type": "string" + }, + "country": { + "type": "string" + }, + "street": { + "type": "string" + }, + "suburb": { + "type": "string" + }, + "location": { + "type": "object", + "properties": { + "coordinates": { + "type": "array" + }, + "type": { + "type": "string" + }, + "is_location_exact": { + "type": "boolean" + } + } + }, + "government_area": { + "type": "string" + } + } + }, + "weekly_price": { + "type": "number" + }, + "bed_type": { + "type": "string" + }, + "listing_url": { + "type": "string" + }, + "guests_included": { + "type": "number" + }, + "maximum_nights": { + "type": "string" + }, + "bathrooms": { + "type": "number" + }, + "extra_people": { + "type": "number" + }, + "bedrooms": { + "type": "number" + }, + "minimum_nights": { + "type": "string" + }, + "last_review": { + "type": "string" + }, + "transit": { + "type": "string" + }, + "accommodates": { + "type": "number" + }, + "interaction": { + "type": "string" + }, + "name": { + "type": "string" + }, + "cancellation_policy": { + "type": "string" + }, + "beds": { + "type": "number" + }, + "last_scraped": { + "type": "string" + }, + "room_type": { + "type": "string" + } + } + }, + "supported_sync_modes": ["incremental"], + "default_cursor_field": [], + "source_defined_primary_key": [], + "namespace": "sample_airbnb" + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/expected_spec.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/expected_spec.json new file mode 100644 index 000000000000..4ad8a5adc2f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/expected_spec.json @@ -0,0 +1,58 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", + "changelogUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MongoDb Source Spec", + "type": "object", + "required": ["connection_string", "database", "replica_set"], + "additionalProperties": true, + "properties": { + "connection_string": { + "title": "Connection String", + "type": "string", + "description": "The connection string of the database that you want to replicate..", + "examples": [ + "mongodb+srv://example.mongodb.net/", + "mongodb://example1.host.com:27017,example2.host.com:27017,example3.host.com:27017/", + "mongodb://example.host.com:27017/" + ], + "order": 1 + }, + "database": { + "title": "Database Name", + "type": "string", + "description": "The database you want to replicate.", + "order": 2 + }, + "user": { + "title": "User", + "type": "string", + "description": "The username which is used to access the database.", + "order": 3 + }, + "password": { + "title": "Password", + "type": "string", + "description": "The password associated with this username.", + "airbyte_secret": true, + "order": 4 + }, + "auth_source": { + "title": "Authentication Source", + "type": "string", + "description": "The authentication source where the user information is stored.", + "default": "admin", + "examples": ["admin"], + "order": 5 + }, + "replica_set": { + "title": "Replica Set", + "type": "string", + "description": "The name of the replica set to be replicated.", + "order": 6 + } + } + }, + "supported_destination_sync_modes": [] +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-internal-poc/metadata.yaml new file mode 100644 index 000000000000..fdcc68d8f2ab --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/metadata.yaml @@ -0,0 +1,20 @@ +data: + connectorSubtype: database + connectorType: source + definitionId: 5ac5a7e5-43f5-4e7a-bf53-70961b0307bc + dockerImageTag: 0.0.1 + dockerRepository: airbyte/source-mongodb-internal-poc + githubIssueLabel: source-mongodb-internal-poc + icon: mongodb.svg + license: ELv2 + name: MongoDb POC + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 + tags: + - language:java +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelper.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelper.java new file mode 100644 index 000000000000..de8626fd4b41 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelper.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.List; + +/** + * Collection of utility methods for generating the {@link AirbyteCatalog}. + */ +public class MongoCatalogHelper { + + /** + * The default cursor field name. + */ + public static final String DEFAULT_CURSOR_FIELD = "_id"; + + /** + * The list of supported sync modes for a given stream. + */ + public static final List SUPPORTED_SYNC_MODES = List.of(SyncMode.INCREMENTAL); + + /** + * Builds an {@link AirbyteStream} with the correct configuration for this source. + * + * @param streamName The name of the stream. + * @param streamNamespace The namespace of the stream. + * @param fields The fields associated with the stream. + * @return The configured {@link AirbyteStream} for this source. + */ + public static AirbyteStream buildAirbyteStream(final String streamName, final String streamNamespace, final List fields) { + return CatalogHelpers.createAirbyteStream(streamName, streamNamespace, addCdcMetadataColumns(fields)) + .withSupportedSyncModes(SUPPORTED_SYNC_MODES) + .withSourceDefinedCursor(true) + .withDefaultCursorField(List.of(DEFAULT_CURSOR_FIELD)) + .withSourceDefinedPrimaryKey(List.of(List.of(DEFAULT_CURSOR_FIELD))); + } + + /** + * Adds the metadata columns required to use CDC to the list of discovered fields. + * + * @param fields The list of discovered fields. + * @return The modified list of discovered fields that includes the required CDC metadata columns. + */ + public static List addCdcMetadataColumns(final List fields) { + final List modifiedFields = new ArrayList<>(fields); + modifiedFields.add(new Field(DebeziumEventUtils.CDC_LSN, JsonSchemaType.NUMBER)); + modifiedFields.add(new Field(DebeziumEventUtils.CDC_UPDATED_AT, JsonSchemaType.STRING)); + modifiedFields.add(new Field(DebeziumEventUtils.CDC_DELETED_AT, JsonSchemaType.STRING)); + return modifiedFields; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConnectionUtils.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConnectionUtils.java new file mode 100644 index 000000000000..b5f387e3a712 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConnectionUtils.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.REPLICA_SET_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.USER_CONFIGURATION_KEY; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.MongoCredential; +import com.mongodb.MongoDriverInformation; +import com.mongodb.ReadPreference; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; + +/** + * Helper utility for building a {@link MongoClient}. + */ +public class MongoConnectionUtils { + + /** + * Creates a new {@link MongoClient} from the source configuration. + * + * @param config The source's configuration. + * @return The configured {@link MongoClient}. + */ + public static MongoClient createMongoClient(final JsonNode config) { + final String authSource = config.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(); + + final ConnectionString mongoConnectionString = new ConnectionString(buildConnectionString(config)); + + final MongoDriverInformation mongoDriverInformation = MongoDriverInformation.builder() + .driverName("Airbyte") + .build(); + + final MongoClientSettings.Builder mongoClientSettingsBuilder = MongoClientSettings.builder() + .applyConnectionString(mongoConnectionString) + .readPreference(ReadPreference.secondaryPreferred()); + + if (config.has(USER_CONFIGURATION_KEY) && config.has(PASSWORD_CONFIGURATION_KEY)) { + final String user = config.get(USER_CONFIGURATION_KEY).asText(); + final String password = config.get(PASSWORD_CONFIGURATION_KEY).asText(); + mongoClientSettingsBuilder.credential(MongoCredential.createCredential(user, authSource, password.toCharArray())); + } + + return MongoClients.create(mongoClientSettingsBuilder.build(), mongoDriverInformation); + } + + private static String buildConnectionString(final JsonNode config) { + final String connectionString = config.get(CONNECTION_STRING_CONFIGURATION_KEY).asText(); + final String replicaSet = config.get(REPLICA_SET_CONFIGURATION_KEY).asText(); + final StringBuilder builder = new StringBuilder(); + builder.append(connectionString); + builder.append("?replicaSet="); + builder.append(replicaSet); + builder.append("&retryWrites=false"); + builder.append("&provider=airbyte"); + builder.append("&tls=true"); + return builder.toString(); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConstants.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConstants.java new file mode 100644 index 000000000000..fd0e1f485407 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConstants.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +public class MongoConstants { + + public static final String AUTH_SOURCE_CONFIGURATION_KEY = "auth_source"; + public static final Integer CHECKPOINT_INTERVAL = 1000; + public static final String CONNECTION_STRING_CONFIGURATION_KEY = "connection_string"; + public static final String DATABASE_CONFIGURATION_KEY = "database"; + public static final String ID_FIELD = "_id"; + public static final String PASSWORD_CONFIGURATION_KEY = "password"; + public static final String REPLICA_SET_CONFIGURATION_KEY = "replica_set"; + public static final String USER_CONFIGURATION_KEY = "user"; + + private MongoConstants() {} + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSource.java new file mode 100644 index 000000000000..47f371d539c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSource.java @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.CHECKPOINT_INTERVAL; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.ID_FIELD; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Sorts; +import com.mongodb.connection.ClusterType; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.BaseConnector; +import io.airbyte.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.Source; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; +import org.bson.BsonDocument; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoDbSource extends BaseConnector implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbSource.class); + + /** Helper class for holding a collection-name and stream state together */ + private record CollectionNameState(Optional name, Optional state) {} + + public static void main(final String[] args) throws Exception { + final Source source = new MongoDbSource(); + LOGGER.info("starting source: {}", MongoDbSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", MongoDbSource.class); + } + + @Override + public AirbyteConnectionStatus check(final JsonNode config) { + try (final MongoClient mongoClient = createMongoClient(config)) { + final String databaseName = config.get(DATABASE_CONFIGURATION_KEY).asText(); + + /* + * Perform the authorized collections check before the cluster type check. The MongoDB Java driver + * needs to actually execute a command in order to fetch the cluster description. Querying for the + * authorized collections guarantees that the cluster description will be available to the driver. + */ + if (MongoUtil.getAuthorizedCollections(mongoClient, databaseName).isEmpty()) { + return new AirbyteConnectionStatus() + .withMessage("Target MongoDB database does not contain any authorized collections.") + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } + if (!ClusterType.REPLICA_SET.equals(mongoClient.getClusterDescription().getType())) { + return new AirbyteConnectionStatus() + .withMessage("Target MongoDB instance is not a replica set cluster.") + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } + } catch (final Exception e) { + LOGGER.error("Unable to perform source check operation.", e); + return new AirbyteConnectionStatus() + .withMessage(e.getMessage()) + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } + + LOGGER.info("The source passed the check operation test!"); + return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); + } + + @Override + public AirbyteCatalog discover(final JsonNode config) { + try (final MongoClient mongoClient = createMongoClient(config)) { + final String databaseName = config.get(DATABASE_CONFIGURATION_KEY).asText(); + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName); + return new AirbyteCatalog().withStreams(streams); + } + } + + @Override + public AutoCloseableIterator read(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final JsonNode state) { + final var databaseName = config.get(DATABASE_CONFIGURATION_KEY).asText(); + final var emittedAt = Instant.now(); + + final var states = convertState(state); + final MongoClient mongoClient = MongoConnectionUtils.createMongoClient(config); + + try { + final var database = mongoClient.getDatabase(databaseName); + // TODO treat INCREMENTAL and FULL_REFRESH differently? + return AutoCloseableIterators.appendOnClose(AutoCloseableIterators.concatWithEagerClose( + convertCatalogToIterators(catalog, states, database, emittedAt), + AirbyteTraceMessageUtility::emitStreamStatusTrace), + mongoClient::close); + } catch (final Exception e) { + mongoClient.close(); + throw e; + } + } + + /** + * Converts the JsonNode into a map of mongodb collection names to stream states. + */ + @VisibleForTesting + protected Map convertState(final JsonNode state) { + // I'm unsure if the JsonNode data is going to be a singular AirbyteStateMessage or an array of + // AirbyteStateMessages. + // So this currently handles both cases, converting the singular message into a list of messages, + // leaving the list of messages + // as a list of messages, or returning an empty list. + final List states = Jsons.tryObject(state, AirbyteStateMessage.class) + .map(List::of) + .orElseGet(() -> Jsons.tryObject(state, AirbyteStateMessage[].class) + .map(Arrays::asList) + .orElse(List.of())); + + // TODO add namespace support? + return states.stream() + .filter(s -> s.getType() == AirbyteStateType.STREAM) + .map(s -> new CollectionNameState( + Optional.ofNullable(s.getStream().getStreamDescriptor()).map(StreamDescriptor::getName), + Jsons.tryObject(s.getStream().getStreamState(), MongodbStreamState.class))) + // only keep states that could be parsed + .filter(p -> p.name.isPresent() && p.state.isPresent()) + .collect(Collectors.toMap( + p -> p.name.orElseThrow(), + p -> p.state.orElseThrow())); + } + + /** + * Converts the streams in the catalog into a list of AutoCloseableIterators. + */ + private List> convertCatalogToIterators( + final ConfiguredAirbyteCatalog catalog, + final Map states, + final MongoDatabase database, + final Instant emittedAt) { + return catalog.getStreams() + .stream() + .peek(airbyteStream -> { + if (!airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) + LOGGER.warn("Stream {} configured with unsupported sync mode: {}", airbyteStream.getStream().getName(), airbyteStream.getSyncMode()); + }) + .filter(airbyteStream -> airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) + .map(airbyteStream -> { + final var collectionName = airbyteStream.getStream().getName(); + final var collection = database.getCollection(collectionName); + // TODO verify that if all fields are selected that all fields are returned here + // (or should this check and ignore them if all fields are selected) + final var fields = Projections.fields(Projections.include(CatalogHelpers.getTopLevelFieldNames(airbyteStream).stream().toList())); + + // find the existing state, if there is one, for this steam + final Optional existingState = states.entrySet().stream() + // look only for states that match this stream's name + // TODO add namespace support + .filter(state -> state.getKey().equals(airbyteStream.getStream().getName())) + .map(Entry::getValue) + .findFirst(); + + // The filter determines the starting point of this iterator based on the state of this collection. + // If a state exists, it will use that state to create a query akin to + // "where _id > [last saved state] order by _id ASC". + // If no state exists, it will create a query akin to "where 1=1 order by _id ASC" + final Bson filter = existingState + // TODO add type support here when we add support for _id fields that are not ObjectId types + .map(state -> Filters.gt(ID_FIELD, new ObjectId(state.id()))) + // if nothing was found, return a new BsonDocument + .orElseGet(BsonDocument::new); + + final var cursor = collection.find() + .filter(filter) + .projection(fields) + .sort(Sorts.ascending(ID_FIELD)) + .cursor(); + + final var stateIterator = new MongoDbStateIterator(cursor, airbyteStream, existingState, emittedAt, CHECKPOINT_INTERVAL); + return AutoCloseableIterators.fromIterator(stateIterator, cursor::close, null); + }) + .toList(); + } + + protected MongoClient createMongoClient(final JsonNode config) { + return MongoConnectionUtils.createMongoClient(config); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIterator.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIterator.java new file mode 100644 index 000000000000..4dce1c39aadf --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIterator.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import com.mongodb.MongoException; +import com.mongodb.client.MongoCursor; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.mongodb.MongoUtils; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.time.Instant; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A state-emitting iterator that emits a state message every checkpointInterval messages when + * iterating over a MongoCursor. + * + * Will also output a state message as the last message after the wrapper iterator has completed. + */ +class MongoDbStateIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbStateIterator.class); + + private final MongoCursor iter; + private final ConfiguredAirbyteStream stream; + private final List fields; + private final Instant emittedAt; + private final Integer checkpointInterval; + + /** + * Counts the number of records seen in this batch, resets when a state-message has been generated. + */ + private int count = 0; + + /** + * Pointer to the last document _id seen by this iterator, necessary to track for state messages. + */ + private String lastId = null; + + /** + * This iterator outputs a final state when the wrapped `iter` has concluded. When this is true, the + * final message will be returned. + */ + private boolean finalStateNext = false; + + /** + * Constructor. + * + * @param iter MongoCursor that iterates over Mongo documents + * @param stream the stream that this iterator represents + * @param state the initial state of this stream + * @param emittedAt when this iterator was started + * @param checkpointInterval how often a state message should be emitted. + */ + MongoDbStateIterator(final MongoCursor iter, + final ConfiguredAirbyteStream stream, + Optional state, + final Instant emittedAt, + final int checkpointInterval) { + this.iter = iter; + this.stream = stream; + this.checkpointInterval = checkpointInterval; + this.emittedAt = emittedAt; + fields = CatalogHelpers.getTopLevelFieldNames(stream).stream().toList(); + lastId = state.map(MongodbStreamState::id).orElse(null); + } + + @Override + public boolean hasNext() { + try { + if (iter.hasNext()) { + return true; + } + } catch (MongoException e) { + // If hasNext throws an exception, log it and then treat it as if hasNext returned false. + LOGGER.info("hasNext threw an exception: {}", e.getMessage(), e); + } + + if (!finalStateNext) { + finalStateNext = true; + return true; + } + + return false; + } + + @Override + public AirbyteMessage next() { + if ((count > 0 && count % checkpointInterval == 0) || finalStateNext) { + count = 0; + + final var streamState = new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor() + .withName(stream.getStream().getName()) + .withNamespace(stream.getStream().getNamespace())); + if (lastId != null) { + // TODO add type support in here once more than ObjectId fields are supported + streamState.withStreamState(Jsons.jsonNode(new MongodbStreamState(lastId))); + } + + final var stateMessage = new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(streamState); + + return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); + } + + count++; + final var document = iter.next(); + final var jsonNode = MongoUtils.toJsonNode(document, fields); + + lastId = document.getObjectId("_id").toString(); + + return new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(stream.getStream().getName()) + .withNamespace(stream.getStream().getNamespace()) + .withEmittedAt(emittedAt.toEpochMilli()) + .withData(jsonNode)); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoField.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoField.java new file mode 100644 index 000000000000..a63f50a976f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoField.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import java.util.Objects; + +/** + * Custom implementation of {@link Field} that only uses the name of the field for equality. This is + * to support MongoDB's unstructured documents which may contain more than one document with the + * same field name, but different data type. + */ +public class MongoField extends Field { + + public MongoField(String name, JsonSchemaType type) { + super(name, type); + } + + public boolean equals(Object o) { + if (this == o) { + return true; + } else if (o != null && this.getClass() == o.getClass()) { + final MongoField field = (MongoField) o; + return this.getName().equals(field.getName()); + } else { + return false; + } + } + + public int hashCode() { + return Objects.hash(new Object[] {this.getName()}); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoUtil.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoUtil.java new file mode 100644 index 000000000000..21c326be72b5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoUtil.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import com.mongodb.MongoCommandException; +import com.mongodb.MongoException; +import com.mongodb.MongoSecurityException; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.model.Aggregates; +import io.airbyte.commons.exceptions.ConnectionErrorException; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.bson.Document; +import org.bson.conversions.Bson; + +public class MongoUtil { + + /** + * The maximum number of documents to sample when attempting to discover the unique keys/types for a + * collection. Inspired by the + * sampling method + * utilized by the MongoDB Compass client. + */ + private static final Integer DISCOVERY_SAMPLE_SIZE = 1000; + + /** + * Set of collection prefixes that should be ignored when performing operations, such as discover to + * avoid access issues. + */ + private static final Set IGNORED_COLLECTIONS = Set.of("system.", "replset.", "oplog."); + + /** + * Returns the set of collections that the current credentials are authorized to access. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server for authorized + * collections. + * @param databaseName The name of the database to query for authorized collections. + * @return The set of authorized collection names (may be empty). + * @throws ConnectionErrorException if unable to perform the authorized collection query. + */ + public static Set getAuthorizedCollections(final MongoClient mongoClient, final String databaseName) { + /* + * db.runCommand ({listCollections: 1.0, authorizedCollections: true, nameOnly: true }) the command + * returns only those collections for which the user has privileges. For example, if a user has find + * action on specific collections, the command returns only those collections; or, if a user has + * find or any other action, on the database resource, the command lists all collections in the + * database. + */ + try { + final Document document = mongoClient.getDatabase(databaseName).runCommand(new Document("listCollections", 1) + .append("authorizedCollections", true) + .append("nameOnly", true)) + .append("filter", "{ 'type': 'collection' }"); + return document.toBsonDocument() + .get("cursor").asDocument() + .getArray("firstBatch") + .stream() + .map(bsonValue -> bsonValue.asDocument().getString("name").getValue()) + .filter(MongoUtil::isSupportedCollection) + .collect(Collectors.toSet()); + } catch (final MongoSecurityException e) { + final MongoCommandException exception = (MongoCommandException) e.getCause(); + throw new ConnectionErrorException(String.valueOf(exception.getCode()), e); + } catch (final MongoException e) { + throw new ConnectionErrorException(String.valueOf(e.getCode()), e); + } + } + + /** + * Retrieves the {@link AirbyteStream}s available to the source by querying the MongoDB server. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server. + * @param databaseName The name of the database to query for collections. + * @return The list of {@link AirbyteStream}s that map to the available collections in the provided + * database. + */ + public static List getAirbyteStreams(final MongoClient mongoClient, final String databaseName) { + final Set authorizedCollections = getAuthorizedCollections(mongoClient, databaseName); + return authorizedCollections.parallelStream().map(collectionName -> { + /* + * Fetch the keys/types from the first N documents and the last N documents from the collection. + * This is an attempt to "survey" the documents in the collection for variance in the schema keys. + */ + final Set discoveredFields = new HashSet<>(); + final MongoCollection mongoCollection = mongoClient.getDatabase(databaseName).getCollection(collectionName); + discoveredFields.addAll(getFieldsInCollection(mongoCollection)); + return createAirbyteStream(collectionName, databaseName, new ArrayList<>(discoveredFields)); + }).collect(Collectors.toList()); + } + + private static AirbyteStream createAirbyteStream(final String collectionName, final String databaseName, final List fields) { + return MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, fields); + } + + private static Set getFieldsInCollection(final MongoCollection collection) { + final Set discoveredFields = new HashSet<>(); + final Map fieldsMap = Map.of("input", Map.of("$objectToArray", "$$ROOT"), + "as", "each", + "in", Map.of("k", "$$each.k", "v", Map.of("$type", "$$each.v"))); + + final Document mapFunction = new Document("$map", fieldsMap); + final Document arrayToObjectAggregation = new Document("$arrayToObject", mapFunction); + + final Map groupMap = new HashMap<>(); + groupMap.put("_id", null); + groupMap.put("fields", Map.of("$addToSet", "$fields")); + + final List aggregateList = new ArrayList<>(); + aggregateList.add(Aggregates.sample(DISCOVERY_SAMPLE_SIZE)); + aggregateList.add(Aggregates.project(new Document("fields", arrayToObjectAggregation))); + aggregateList.add(Aggregates.unwind("$fields")); + aggregateList.add(new Document("$group", groupMap)); + + final AggregateIterable output = collection.aggregate(aggregateList); + + try (final MongoCursor cursor = output.cursor()) { + while (cursor.hasNext()) { + final Map fields = ((List>) cursor.next().get("fields")).get(0); + discoveredFields.addAll(fields.entrySet().stream() + .map(e -> new MongoField(e.getKey(), convertToSchemaType(e.getValue()))) + .collect(Collectors.toSet())); + } + } + + return discoveredFields; + } + + private static JsonSchemaType convertToSchemaType(final String type) { + return switch (type) { + case "boolean" -> JsonSchemaType.BOOLEAN; + case "int", "long", "double", "decimal" -> JsonSchemaType.NUMBER; + case "array" -> JsonSchemaType.ARRAY; + case "object", "javascriptWithScope" -> JsonSchemaType.OBJECT; + default -> JsonSchemaType.STRING; + }; + } + + private static boolean isSupportedCollection(final String collectionName) { + return !IGNORED_COLLECTIONS.stream().anyMatch(s -> collectionName.startsWith(s)); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongodbStreamState.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongodbStreamState.java new file mode 100644 index 000000000000..12b8dad1f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongodbStreamState.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +/* + * TODO replace `isObjectId` with _id enum (ObjectId, String, etc.) + */ +public record MongodbStreamState(String id) { // , boolean isObjectId) { + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/resources/spec.json new file mode 100644 index 000000000000..4a38d5ee89d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/resources/spec.json @@ -0,0 +1,57 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", + "changelogUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MongoDb Source Spec", + "type": "object", + "required": ["connection_string", "database", "replica_set"], + "additionalProperties": true, + "properties": { + "connection_string": { + "title": "Connection String", + "type": "string", + "description": "The connection string of the database that you want to replicate..", + "examples": [ + "mongodb+srv://example.mongodb.net/", + "mongodb://example1.host.com:27017,example2.host.com:27017,example3.host.com:27017/", + "mongodb://example.host.com:27017/" + ], + "order": 1 + }, + "database": { + "title": "Database Name", + "type": "string", + "description": "The database you want to replicate.", + "order": 2 + }, + "user": { + "title": "User", + "type": "string", + "description": "The username which is used to access the database.", + "order": 3 + }, + "password": { + "title": "Password", + "type": "string", + "description": "The password associated with this username.", + "airbyte_secret": true, + "order": 4 + }, + "auth_source": { + "title": "Authentication Source", + "type": "string", + "description": "The authentication source where the user information is stored.", + "default": "admin", + "examples": ["admin"], + "order": 5 + }, + "replica_set": { + "title": "Replica Set", + "type": "string", + "description": "The name of the replica set to be replicated.", + "order": 6 + } + } + } +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test-integration/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test-integration/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceAcceptanceTest.java new file mode 100644 index 000000000000..71f1b0ec4646 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test-integration/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceAcceptanceTest.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static io.airbyte.integrations.source.mongodb.internal.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; +import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.DATABASE_CONFIGURATION_KEY; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import org.bson.BsonArray; +import org.bson.BsonString; +import org.bson.Document; +import org.bson.types.ObjectId; + +public class MongoDbSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final String DATABASE_NAME = "test"; + private static final String COLLECTION_NAME = "acceptance_test1"; + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); + + private JsonNode config; + private MongoClient mongoClient; + + @Override + protected void setupEnvironment(final TestDestinationEnv testEnv) throws IOException { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH + + ". Override by setting setting path with the CREDENTIALS_PATH constant."); + } + + config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); + ((ObjectNode) config).put(DATABASE_CONFIGURATION_KEY, DATABASE_NAME); + + mongoClient = MongoConnectionUtils.createMongoClient(config); + + insertTestData(mongoClient); + } + + private void insertTestData(final MongoClient mongoClient) { + mongoClient.getDatabase(DATABASE_NAME).createCollection(COLLECTION_NAME); + final MongoCollection collection = mongoClient.getDatabase(DATABASE_NAME).getCollection(COLLECTION_NAME); + final var objectDocument = new Document("testObject", new Document("name", "subName").append("testField1", "testField1").append("testInt", 10) + .append("thirdLevelDocument", new Document("data", "someData").append("intData", 1))); + + final var doc1 = new Document("_id", new ObjectId("64c0029d95ad260d69ef28a0")) + .append("id", "0001").append("name", "Test") + .append("test", 10).append("test_array", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) + .append("double_test", 100.12).append("int_test", 100).append("object_test", objectDocument); + + final var doc2 = new Document("_id", new ObjectId("64c0029d95ad260d69ef28a1")) + .append("id", "0002").append("name", "Mongo").append("test", "test_value").append("int_test", 201).append("object_test", objectDocument); + + final var doc3 = new Document("_id", new ObjectId("64c0029d95ad260d69ef28a2")) + .append("id", "0003").append("name", "Source").append("test", null) + .append("double_test", 212.11).append("int_test", 302).append("object_test", objectDocument); + + collection.insertMany(List.of(doc1, doc2, doc3)); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + mongoClient.getDatabase(DATABASE_NAME).getCollection(COLLECTION_NAME).drop(); + mongoClient.close(); + } + + @Override + protected String getImageName() { + return "airbyte/source-mongodb-internal-poc:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + final List fields = List.of( + Field.of(DEFAULT_CURSOR_FIELD, JsonSchemaType.STRING), + Field.of("id", JsonSchemaType.STRING), + Field.of("name", JsonSchemaType.STRING), + Field.of("test", JsonSchemaType.STRING), + Field.of("test_array", JsonSchemaType.ARRAY), + Field.of("empty_test", JsonSchemaType.STRING), + Field.of("double_test", JsonSchemaType.NUMBER), + Field.of("int_test", JsonSchemaType.NUMBER), + Field.of("object_test", JsonSchemaType.OBJECT)); + + final AirbyteStream airbyteStream = MongoCatalogHelper.buildAirbyteStream(COLLECTION_NAME, DATABASE_NAME, fields); + final ConfiguredAirbyteStream configuredIncrementalAirbyteStream = convertToConfiguredAirbyteStream(airbyteStream, SyncMode.INCREMENTAL); + + return new ConfiguredAirbyteCatalog().withStreams(List.of(configuredIncrementalAirbyteStream)); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + + private ConfiguredAirbyteStream convertToConfiguredAirbyteStream(final AirbyteStream airbyteStream, final SyncMode syncMode) { + return new ConfiguredAirbyteStream() + .withSyncMode(syncMode) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withCursorField(List.of(DEFAULT_CURSOR_FIELD)) + .withStream(airbyteStream); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelperTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelperTest.java new file mode 100644 index 000000000000..f67e0e7f1645 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelperTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static io.airbyte.integrations.source.mongodb.internal.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; +import static io.airbyte.integrations.source.mongodb.internal.MongoCatalogHelper.SUPPORTED_SYNC_MODES; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MongoCatalogHelperTest { + + @Test + void testBuildingAirbyteStream() { + final String streamName = "name"; + final String streamNamespace = "namespace"; + final List discoveredFields = List.of(new Field("field1", JsonSchemaType.STRING), + new Field("field2", JsonSchemaType.NUMBER)); + + final AirbyteStream airbyteStream = MongoCatalogHelper.buildAirbyteStream(streamName, streamNamespace, discoveredFields); + + assertNotNull(airbyteStream); + assertEquals(streamNamespace, airbyteStream.getNamespace()); + assertEquals(streamName, airbyteStream.getName()); + assertEquals(List.of(DEFAULT_CURSOR_FIELD), airbyteStream.getDefaultCursorField()); + assertEquals(true, airbyteStream.getSourceDefinedCursor()); + assertEquals(List.of(List.of(DEFAULT_CURSOR_FIELD)), airbyteStream.getSourceDefinedPrimaryKey()); + assertEquals(SUPPORTED_SYNC_MODES, airbyteStream.getSupportedSyncModes()); + assertEquals(5, airbyteStream.getJsonSchema().get("properties").size()); + + discoveredFields.forEach(f -> assertTrue(airbyteStream.getJsonSchema().get("properties").has(f.getName()))); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_LSN)); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_LSN).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_DELETED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_UPDATED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceTest.java new file mode 100644 index 000000000000..c4a02f426d83 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceTest.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.connection.ClusterDescription; +import com.mongodb.connection.ClusterType; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbSourceTest { + + private static final String DB_NAME = "airbyte_test"; + + private JsonNode airbyteSourceConfig; + private MongoClient mongoClient; + private MongoDbSource source; + + @BeforeEach + void setup() { + airbyteSourceConfig = createConfiguration(Optional.empty(), Optional.empty()); + mongoClient = mock(MongoClient.class); + source = spy(new MongoDbSource()); + doReturn(mongoClient).when(source).createMongoClient(airbyteSourceConfig); + } + + @Test + void testCheckOperation() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, airbyteConnectionStatus.getStatus()); + } + + @Test + void testCheckOperationNoAuthorizedCollections() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("no_authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals("Target MongoDB database does not contain any authorized collections.", airbyteConnectionStatus.getMessage()); + } + + @Test + void testCheckOperationInvalidClusterType() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.STANDALONE); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals("Target MongoDB instance is not a replica set cluster.", airbyteConnectionStatus.getMessage()); + } + + @Test + void testCheckOperationUnexpectedException() { + final String expectedMessage = "This is just a test failure."; + when(mongoClient.getDatabase(any())).thenThrow(new IllegalArgumentException(expectedMessage)); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals(expectedMessage, airbyteConnectionStatus.getMessage()); + } + + @Test + void testDiscoverOperation() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(s -> new Document(s)).collect(Collectors.toList()); + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoCursor cursor = mock(MongoCursor.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + + final AirbyteCatalog airbyteCatalog = source.discover(airbyteSourceConfig); + + assertNotNull(airbyteCatalog); + assertEquals(1, airbyteCatalog.getStreams().size()); + + final Optional stream = airbyteCatalog.getStreams().stream().findFirst(); + assertTrue(stream.isPresent()); + assertEquals(DB_NAME, stream.get().getNamespace()); + assertEquals("testCollection", stream.get().getName()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("_id").get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("name").get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("last_updated").get("type").asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("total").get("type").asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("price").get("type").asText()); + assertEquals(JsonSchemaType.ARRAY.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("items").get("type").asText()); + assertEquals(JsonSchemaType.OBJECT.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("owners").get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("other").get("type").asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_LSN).get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + assertEquals(true, stream.get().getSourceDefinedCursor()); + assertEquals(List.of(MongoCatalogHelper.DEFAULT_CURSOR_FIELD), stream.get().getDefaultCursorField()); + assertEquals(List.of(List.of(MongoCatalogHelper.DEFAULT_CURSOR_FIELD)), stream.get().getSourceDefinedPrimaryKey()); + assertEquals(MongoCatalogHelper.SUPPORTED_SYNC_MODES, stream.get().getSupportedSyncModes()); + } + + @Test + void testDiscoverOperationWithUnexpectedFailure() throws IOException { + final String expectedMessage = "This is just a test failure."; + when(mongoClient.getDatabase(any())).thenThrow(new IllegalArgumentException(expectedMessage)); + + assertThrows(IllegalArgumentException.class, () -> source.discover(airbyteSourceConfig)); + } + + @Test + void testFullRefresh() throws Exception { + // TODO implement + } + + @Test + void testIncrementalRefresh() throws Exception { + // TODO implement + } + + @Test + void testConvertState() { + final var state1 = Jsons.deserialize( + "[{\"type\":\"STREAM\",\"stream\":{\"stream_descriptor\":{\"name\":\"test.acceptance_test1\"},\"stream_state\":{\"id\":\"64c0029d95ad260d69ef28a2\"}}}]"); + final var actual = source.convertState(state1); + assertTrue(actual.containsKey("test.acceptance_test1"), "missing test.acceptance_test1"); + assertEquals("64c0029d95ad260d69ef28a2", actual.get("test.acceptance_test1").id(), "id value does not match"); + + } + + private static JsonNode createConfiguration(final Optional username, final Optional password) { + final Map config = new HashMap<>(); + final Map baseConfig = Map.of( + MongoConstants.DATABASE_CONFIGURATION_KEY, DB_NAME, + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://localhost:27017/", + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, "admin", + MongoConstants.REPLICA_SET_CONFIGURATION_KEY, "replica-set"); + + config.putAll(baseConfig); + username.ifPresent(u -> config.put(MongoConstants.USER_CONFIGURATION_KEY, u)); + password.ifPresent(p -> config.put(MongoConstants.PASSWORD_CONFIGURATION_KEY, p)); + return Jsons.deserialize(Jsons.serialize(config)); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIteratorTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIteratorTest.java new file mode 100644 index 000000000000..040081f2b8b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIteratorTest.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +import com.mongodb.MongoException; +import com.mongodb.client.MongoCursor; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +class MongoDbStateIteratorTest { + + private static final int CHECKPOINT_INTERVAL = 2; + @Mock + private MongoCursor mongoCursor; + private AutoCloseable closeable; + + @BeforeEach + public void setup() { + closeable = MockitoAnnotations.openMocks(this); + } + + @AfterEach + public void teardown() throws Exception { + closeable.close(); + } + + @Test + void happyPath() { + final var docs = docs(); + + when(mongoCursor.hasNext()).thenAnswer(new Answer() { + + private int count = 0; + + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + count++; + // hasNext will be called for each doc plus for each state message + return count <= (docs.size() + (docs.size() % CHECKPOINT_INTERVAL)); + } + + }); + + when(mongoCursor.next()).thenAnswer(new Answer() { + + private int offset = 0; + + @Override + public Document answer(InvocationOnMock invocation) throws Throwable { + final var doc = docs.get(offset); + offset++; + return doc; + } + + }); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + + final var iter = new MongoDbStateIterator(mongoCursor, stream, Optional.empty(), Instant.now(), CHECKPOINT_INTERVAL); + + // with a batch size of 2, the MongoDbStateIterator should return the following after each + // `hasNext`/`next` call: + // true, record Air Force Blue + // true, record Alice Blue + // true, state (with Alice Blue as the state) + // true, record Alizarin Crimson + // true, state (with Alizarin Crimson) + // false + AirbyteMessage message; + assertTrue(iter.hasNext(), "air force blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "alice blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(1).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(1).get("_id").toString(), + message.getState().getStream().getStreamState().get("id").asText(), + "state id should match last record id"); + + assertTrue(iter.hasNext(), "alizarin crimson should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(2).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(2).get("_id").toString(), + message.getState().getStream().getStreamState().get("id").asText(), + "state id should match last record id"); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + @Test + void treatHasNextExceptionAsFalse() { + final var docs = docs(); + + // on the second hasNext call, throw an exception + when(mongoCursor.hasNext()) + .thenReturn(true) + .thenThrow(new MongoException("test exception")); + + when(mongoCursor.next()).thenReturn(docs.get(0)); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + + final var iter = new MongoDbStateIterator(mongoCursor, stream, Optional.empty(), Instant.now(), CHECKPOINT_INTERVAL); + + // with a batch size of 2, the MongoDbStateIterator should return the following after each + // `hasNext`/`next` call: + // true, record Air Force Blue + // true (exception thrown), state (with Air Force Blue as the state) + // false + AirbyteMessage message; + assertTrue(iter.hasNext(), "air force blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(0).get("_id").toString(), + message.getState().getStream().getStreamState().get("id").asText(), + "state id should match last record id"); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + @Test + void initialStateIsReturnedIfUnderlyingIteratorIsEmpty() { + final var docs = docs(); + + // on the second hasNext call, throw an exception + when(mongoCursor.hasNext()).thenReturn(false); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + final var objectId = "64dfb6a7bb3c3458c30801f4"; + final var iter = new MongoDbStateIterator(mongoCursor, stream, Optional.of(new MongodbStreamState(objectId)), Instant.now(), CHECKPOINT_INTERVAL); + + // the MongoDbStateIterator should return the following after each + // `hasNext`/`next` call: + // false + // then the generated state message should have the same id as the initial state + assertTrue(iter.hasNext(), "state should be next"); + + final AirbyteMessage message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + objectId, + message.getState().getStream().getStreamState().get("id").asText(), + "state id should match initial state "); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + private ConfiguredAirbyteCatalog catalog() { + return new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("_id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withCursorField(List.of("_id")) + .withStream(CatalogHelpers.createAirbyteStream( + "test.unit", + Field.of("_id", JsonSchemaType.STRING), + Field.of("name", JsonSchemaType.STRING), + Field.of("hex", JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.INCREMENTAL)) + .withDefaultCursorField(List.of("_id"))))); + } + + private List docs() { + return List.of( + new Document("_id", new ObjectId("64c0029d95ad260d69ef28a0")) + .append("name", "Air Force Blue").append("hex", "#5d8aa8"), + new Document("_id", new ObjectId("64c0029d95ad260d69ef28a1")) + .append("name", "Alice Blue").append("hex", "#f0f8ff"), + new Document("_id", new ObjectId("64c0029d95ad260d69ef28a2")) + .append("name", "Alizarin Crimson").append("hex", "#e32636")); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoUtilTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoUtilTest.java new file mode 100644 index 000000000000..de2e80e82e4f --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoUtilTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.bson.Document; +import org.junit.jupiter.api.Test; + +public class MongoUtilTest { + + @Test + void testGetAirbyteStreams() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(s -> new Document(s)).collect(Collectors.toList()); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName); + assertNotNull(streams); + assertEquals(1, streams.size()); + assertEquals(11, streams.get(0).getJsonSchema().get("properties").size()); + } + + @Test + void testGetAirbyteStreamsDifferentDataTypes() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response_different_datatypes.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(s -> new Document(s)).collect(Collectors.toList()); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName); + assertNotNull(streams); + assertEquals(1, streams.size()); + assertEquals(11, streams.get(0).getJsonSchema().get("properties").size()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + streams.get(0).getJsonSchema().get("properties").get("total").get("type").asText()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/kotlin/MongoDbInsertClient.kt b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/kotlin/MongoDbInsertClient.kt new file mode 100644 index 000000000000..a944983fa008 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/kotlin/MongoDbInsertClient.kt @@ -0,0 +1,52 @@ +package io.airbyte.integrations.source.mongodb.internal + +import io.airbyte.commons.json.Jsons +import kotlinx.cli.ArgParser +import kotlinx.cli.ArgType +import kotlinx.cli.default +import kotlinx.cli.required +import org.bson.BsonTimestamp +import org.bson.Document +import java.lang.System.currentTimeMillis + +object MongoDbInsertClient { + + @JvmStatic + fun main(args: Array) { + val parser = ArgParser("MongoDb Insert Client") + val connectionString by parser.option(ArgType.String, fullName = "connection-string", shortName = "cs", description = "MongoDb Connection String").required() + val databaseName by parser.option(ArgType.String, fullName = "database-name", shortName = "d", description = "Database Name").required() + val collectionName by parser.option(ArgType.String, fullName = "collection-name", shortName = "cn", description = "Collection Name").required() + val replicaSet by parser.option(ArgType.String, fullName = "replica-set", shortName = "r", description = "Replica Set").required() + val username by parser.option(ArgType.String, fullName = "username", shortName = "u", description = "Username").required() + val numberOfDocuments by parser.option(ArgType.Int, fullName = "number", shortName = "n", description = "Number of documents to generate").default(10000) + + parser.parse(args) + + println("Enter password: ") + val password = readln() + + var config = mapOf(MongoConstants.DATABASE_CONFIGURATION_KEY to databaseName, + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY to connectionString, + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY to "admin", + MongoConstants.REPLICA_SET_CONFIGURATION_KEY to replicaSet, + MongoConstants.USER_CONFIGURATION_KEY to username, + MongoConstants.PASSWORD_CONFIGURATION_KEY to password) + + MongoConnectionUtils.createMongoClient(Jsons.deserialize(Jsons.serialize(config))).use { mongoClient -> + val documents = mutableListOf() + for (i in 0..numberOfDocuments) { + documents += Document().append("name", "Document $i") + .append("description", "This is document #$i") + .append("doubleField", i.toDouble()) + .append("intField", i) + .append("objectField", mapOf("key" to "value")) + .append("timestamp", BsonTimestamp(currentTimeMillis())) + } + + mongoClient.getDatabase(databaseName).getCollection(collectionName).insertMany(documents) + } + + println("Inserted $numberOfDocuments document(s) to $databaseName.$collectionName") + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/authorized_collections_response.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/authorized_collections_response.json new file mode 100644 index 000000000000..89369adef455 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/authorized_collections_response.json @@ -0,0 +1,24 @@ +{ + "cursor": { + "id": 0, + "ns": "sample_airbnb.$cmd.listCollections", + "firstBatch": [ + { + "name": "testCollection", + "type": "collection", + "options": {}, + "info": { + "readOnly": false, + "uuid": "68fdfd7d-7cbf-41c2-aa65-277a6cdc478e" + }, + "idIndex": { + "v": 2, + "key": { + "_id": 1 + }, + "name": "_id_" + } + } + ] + } +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/no_authorized_collections_response.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/no_authorized_collections_response.json new file mode 100644 index 000000000000..d4cb8acaa5a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/no_authorized_collections_response.json @@ -0,0 +1,7 @@ +{ + "cursor": { + "id": 0, + "ns": "sample_airbnb.$cmd.listCollections", + "firstBatch": [] + } +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response.json new file mode 100644 index 000000000000..55c665524244 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response.json @@ -0,0 +1,31 @@ +[ + { + "_id": null, + "fields": [ + { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "int", + "price": "decimal", + "items": "array", + "owners": "object" + } + ] + }, + { + "_id": null, + "fields": [ + { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "int", + "price": "decimal", + "items": "array", + "owners": "object", + "other": "string" + } + ] + } +] diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response_different_datatypes.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response_different_datatypes.json new file mode 100644 index 000000000000..f487d4d80404 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response_different_datatypes.json @@ -0,0 +1,31 @@ +[ + { + "_id": null, + "fields": [ + { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "int", + "price": "decimal", + "items": "array", + "owners": "object" + } + ] + }, + { + "_id": null, + "fields": [ + { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "string", + "price": "decimal", + "items": "array", + "owners": "object", + "other": "string" + } + ] + } +] diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile index 8a68ed4c0522..437a991592c6 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml index 3fbfb6e87846..791eed372111 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml @@ -7,11 +7,11 @@ data: connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 0.1.19 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-mongodb-strict-encrypt githubIssueLabel: source-mongodb-v2 icon: mongodb.svg - license: MIT + license: ELv2 name: MongoDb releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java index 062c075dc988..048a5ff9e59d 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java @@ -66,27 +66,13 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc + ". Override by setting setting path with the CREDENTIALS_PATH constant."); } - final String credentialsJsonString = Files.readString(CREDENTIALS_PATH); - final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString); + config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); + ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", MongoInstanceType.ATLAS.getType()) - .put("cluster_url", credentialsJson.get("cluster_url").asText()) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("user", credentialsJson.get("user").asText()) - .put(JdbcUtils.PASSWORD_KEY, credentialsJson.get(JdbcUtils.PASSWORD_KEY).asText()) - .put(INSTANCE_TYPE, instanceConfig) - .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME) - .put("auth_source", "admin") - .build()); - - final var credentials = String.format("%s:%s@", config.get("user").asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText()); - final String connectionString = String.format("mongodb+srv://%s%s/%s?retryWrites=true&w=majority&tls=true", - credentials, - config.get(INSTANCE_TYPE).get("cluster_url").asText(), + final String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&tls=true", + config.get("user").asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + config.get("instance_type").get("cluster_url").asText(), config.get(JdbcUtils.DATABASE_KEY).asText()); database = new MongoDatabase(connectionString, DATABASE_NAME); diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json index 2b5821b4f975..48be1e68bb2f 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json @@ -20,8 +20,7 @@ "properties": { "instance": { "type": "string", - "enum": ["standalone"], - "default": "standalone" + "const": "standalone" }, "host": { "title": "Host", @@ -47,8 +46,7 @@ "properties": { "instance": { "type": "string", - "enum": ["replica"], - "default": "replica" + "const": "replica" }, "server_addresses": { "title": "Server Addresses", @@ -67,13 +65,12 @@ }, { "title": "MongoDB Atlas", - "additionalProperties": false, + "additionalProperties": true, "required": ["instance", "cluster_url"], "properties": { "instance": { "type": "string", - "enum": ["atlas"], - "default": "atlas" + "const": "atlas" }, "cluster_url": { "title": "Cluster URL", diff --git a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile index 8cbd44ac9c45..c59d6f6d0d1f 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile +++ b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-mongodb-v2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index c3ccc1da8c38..b489335c356c 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -1,22 +1,26 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 0.1.19 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-mongodb-v2 + documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 icon: mongodb.svg - license: MIT + license: ELv2 name: MongoDb registries: cloud: - dockerImageTag: 0.1.7 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-mongodb-strict-encrypt enabled: true oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java index 3ff135af8310..6cae561d5527 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java @@ -4,13 +4,10 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.db.mongodb.MongoUtils.MongoInstanceType.ATLAS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCollection; import io.airbyte.commons.json.Jsons; import io.airbyte.db.jdbc.JdbcUtils; @@ -62,21 +59,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc + ". Override by setting setting path with the CREDENTIALS_PATH constant."); } - final String credentialsJsonString = Files.readString(CREDENTIALS_PATH); - final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString); - - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", ATLAS.getType()) - .put("cluster_url", credentialsJson.get("cluster_url").asText()) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("user", credentialsJson.get("user").asText()) - .put(JdbcUtils.PASSWORD_KEY, credentialsJson.get(JdbcUtils.PASSWORD_KEY).asText()) - .put("instance_type", instanceConfig) - .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME) - .put("auth_source", "admin") - .build()); + config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); + ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); final String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&tls=true", config.get("user").asText(), @@ -84,7 +68,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get("instance_type").get("cluster_url").asText(), config.get(JdbcUtils.DATABASE_KEY).asText()); - database = new MongoDatabase(connectionString, DATABASE_NAME); + database = new MongoDatabase(connectionString, config.get(JdbcUtils.DATABASE_KEY).asText()); final MongoCollection collection = database.createCollection(COLLECTION_NAME); final var objectDocument = new Document("testObject", new Document("name", "subName").append("testField1", "testField1").append("testInt", 10) @@ -134,11 +118,11 @@ public void testCheckIncorrectPassword() throws Exception { @Test public void testCheckIncorrectCluster() throws Exception { - ((ObjectNode) config).with("instance_type") - .put("cluster_url", "cluster0.iqgf8.mongodb.netfail"); + final String badClusterUrl = "cluster0.iqgf8.mongodb.netfail"; + config.withObject("/instance_type").put("cluster_url", badClusterUrl); final AirbyteConnectionStatus status = new MongoDbSource().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: -4")); + assertTrue(status.getMessage().matches("State code: -\\d+.*")); } @Test diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 0b00a76e2832..b9c0668add68 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.17 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle index 82d2b6f983df..75f3de23eb74 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle @@ -29,3 +29,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml index 4a01ddc10bda..fbdd8bc54b41 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml @@ -11,14 +11,19 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 1.0.17 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-mssql-strict-encrypt githubIssueLabel: source-mssql icon: mssql.svg - license: MIT + license: ELv2 name: Microsoft SQL Server (MSSQL) releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/mssql tags: - language:java + releases: + breakingChanges: + 2.0.0: + message: "Add default cursor for cdc" + upgradeDeadline: "2023-08-23" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 27a10869e785..790035a77ffb 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.17 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index ca6dee1792e8..050978a5e7ad 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -39,3 +39,4 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-mssql/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-mssql/integration_tests/seed/basic.sql index fca7de8274d2..616bc1b2e897 100644 --- a/airbyte-integrations/connectors/source-mssql/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-mssql/integration_tests/seed/basic.sql @@ -1,12 +1,228 @@ -CREATE DATABASE MSSQL_BASIC; +CREATE + DATABASE MSSQL_BASIC; + USE MSSQL_BASIC; -CREATE TABLE dbo.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bigint,test_column_10 float,test_column_11 real,test_column_12 date,test_column_13 smalldatetime,test_column_14 datetime,test_column_15 datetime2,test_column_16 time,test_column_18 char,test_column_2 int,test_column_20 text,test_column_21 nchar,test_column_22 nvarchar(max),test_column_23 ntext,test_column_25 varbinary(3),test_column_3 smallint,test_column_4 tinyint,test_column_6 DECIMAL(5,2),test_column_7 numeric ); +CREATE + TABLE + dbo.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 FLOAT, + test_column_11 REAL, + test_column_12 DATE, + test_column_13 smalldatetime, + test_column_14 datetime, + test_column_15 datetime2, + test_column_16 TIME, + test_column_18 CHAR, + test_column_2 INT, + test_column_20 text, + test_column_21 nchar, + test_column_22 nvarchar(MAX), + test_column_23 ntext, + test_column_25 VARBINARY(3), + test_column_3 SMALLINT, + test_column_4 tinyint, + test_column_6 DECIMAL( + 5, + 2 + ), + test_column_7 NUMERIC + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + '123', + '123', + '0001-01-01', + '1900-01-01', + '1753-01-01', + '0001-01-01', + '13:00:01', + 'a', + - 2147483648, + 'a', + 'a', + 'a', + 'a', + CAST( + 'ABC' AS VARBINARY + ), + - 32768, + 0, + 999.33, + '99999' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '1234567890.1234567', + '1234567890.1234567', + '9999-12-31', + '2079-06-06', + '9999-12-31', + '9999-12-31', + '13:00:04Z', + '*', + 2147483647, + 'abc', + '*', + 'abc', + 'abc', + CAST( + 'ABC' AS VARBINARY + ), + 32767, + 255, + 999.33, + '99999' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 3, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '*', + 2147483647, + 'Some test text 123$%^&*()_', + N'ї', + N'Миші йдуть на південь, не питай чому;', + N'Миші йдуть на південь, не питай чому;', + CAST( + 'ABC' AS VARBINARY + ), + 32767, + 255, + 999.33, + '99999' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 4, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '*', + 2147483647, + '', + N'ї', + N'櫻花分店', + N'櫻花分店', + CAST( + 'ABC' AS VARBINARY + ), + 32767, + 255, + 999.33, + '99999' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 5, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '*', + 2147483647, + '', + N'ї', + '', + '', + CAST( + 'ABC' AS VARBINARY + ), + 32767, + 255, + 999.33, + '99999' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 6, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '*', + 2147483647, + '', + N'ї', + N'\xF0\x9F\x9A\x80', + N'\xF0\x9F\x9A\x80', + CAST( + 'ABC' AS VARBINARY + ), + 32767, + 255, + 999.33, + '99999' + ); -INSERT INTO dbo.TEST_DATASET VALUES (1, -9223372036854775808, '123', '123', '0001-01-01', '1900-01-01', '1753-01-01', '0001-01-01', '13:00:01', 'a', -2147483648, 'a', 'a', 'a', 'a', CAST( 'ABC' AS VARBINARY), -32768, 0, 999.33, '99999'); -INSERT INTO dbo.TEST_DATASET VALUES (2, 9223372036854775807, '1234567890.1234567', '1234567890.1234567', '9999-12-31', '2079-06-06', '9999-12-31', '9999-12-31', '13:00:04Z', '*', 2147483647, 'abc', '*', 'abc', 'abc', CAST( 'ABC' AS VARBINARY), 32767, 255, 999.33, '99999'); -INSERT INTO dbo.TEST_DATASET VALUES (3, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '*', 2147483647, 'Some test text 123$%^&*()_', N'ї', N'Миші йдуть на південь, не питай чому;', N'Миші йдуть на південь, не питай чому;', CAST( 'ABC' AS VARBINARY), 32767, 255, 999.33, '99999'); -INSERT INTO dbo.TEST_DATASET VALUES (4, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '*', 2147483647, '', N'ї', N'櫻花分店', N'櫻花分店', CAST( 'ABC' AS VARBINARY), 32767, 255, 999.33, '99999'); -INSERT INTO dbo.TEST_DATASET VALUES (5, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '*', 2147483647, '', N'ї', '', '', CAST( 'ABC' AS VARBINARY), 32767, 255, 999.33, '99999'); -INSERT INTO dbo.TEST_DATASET VALUES (6, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '*', 2147483647, '', N'ї', N'\xF0\x9F\x9A\x80', N'\xF0\x9F\x9A\x80', CAST( 'ABC' AS VARBINARY), 32767, 255, 999.33, '99999'); -INSERT INTO dbo.TEST_DATASET VALUES (7, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '*', 2147483647, '', N'ї', N'\xF0\x9F\x9A\x80', N'\xF0\x9F\x9A\x80', CAST( 'ABC' AS VARBINARY), 32767, 255, 999.33, '99999'); +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 7, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '*', + 2147483647, + '', + N'ї', + N'\xF0\x9F\x9A\x80', + N'\xF0\x9F\x9A\x80', + CAST( + 'ABC' AS VARBINARY + ), + 32767, + 255, + 999.33, + '99999' + ); diff --git a/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full.sql index b24fa0e70e45..9d7a8a920429 100644 --- a/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full.sql +++ b/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full.sql @@ -1,12 +1,320 @@ -CREATE DATABASE MSSQL_FULL; +CREATE + DATABASE MSSQL_FULL; + USE MSSQL_FULL; -CREATE TABLE dbo.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bigint,test_column_10 float,test_column_11 real,test_column_12 date,test_column_13 smalldatetime,test_column_14 datetime,test_column_15 datetime2,test_column_16 time,test_column_17 datetimeoffset,test_column_18 char,test_column_19 varchar(max) COLLATE Latin1_General_100_CI_AI_SC_UTF8,test_column_2 int,test_column_20 text,test_column_21 nchar,test_column_22 nvarchar(max),test_column_23 ntext,test_column_24 binary,test_column_25 varbinary(3),test_column_26 geometry,test_column_27 uniqueidentifier,test_column_28 xml,test_column_29 geography,test_column_3 smallint,test_column_30 hierarchyid,test_column_31 sql_variant,test_column_4 tinyint,test_column_5 bit,test_column_6 DECIMAL(5,2),test_column_7 numeric,test_column_8 money,test_column_9 smallmoney ); +CREATE + TABLE + dbo.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 FLOAT, + test_column_11 REAL, + test_column_12 DATE, + test_column_13 smalldatetime, + test_column_14 datetime, + test_column_15 datetime2, + test_column_16 TIME, + test_column_17 datetimeoffset, + test_column_18 CHAR, + test_column_19 VARCHAR(MAX) COLLATE Latin1_General_100_CI_AI_SC_UTF8, + test_column_2 INT, + test_column_20 text, + test_column_21 nchar, + test_column_22 nvarchar(MAX), + test_column_23 ntext, + test_column_24 BINARY, + test_column_25 VARBINARY(3), + test_column_26 geometry, + test_column_27 uniqueidentifier, + test_column_28 xml, + test_column_29 geography, + test_column_3 SMALLINT, + test_column_30 hierarchyid, + test_column_31 sql_variant, + test_column_4 tinyint, + test_column_5 bit, + test_column_6 DECIMAL( + 5, + 2 + ), + test_column_7 NUMERIC, + test_column_8 money, + test_column_9 smallmoney + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + '123', + '123', + '0001-01-01', + '1900-01-01', + '1753-01-01', + '0001-01-01', + NULL, + '0001-01-10 00:00:00 +01:00', + 'a', + 'a', + NULL, + 'a', + 'a', + 'a', + 'a', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '1', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + NULL, + '/1/1/', + 'a', + NULL, + NULL, + 999.33, + '99999', + NULL, + NULL + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '1234567890.1234567', + '1234567890.1234567', + '9999-12-31', + '2079-06-06', + '9999-12-31', + '9999-12-31', + '13:00:01', + '9999-01-10 00:00:00 +01:00', + '*', + 'abc', + - 2147483648, + 'abc', + '*', + 'abc', + 'abc', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + - 32768, + NULL, + 'abc', + 0, + 0, + NULL, + NULL, + '9990000.3647', + '-214748.3648' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 3, + 0, + NULL, + NULL, + '1999-01-08', + NULL, + '9999-12-31T13:00:04Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04Z', + NULL, + NULL, + N'Миші йдуть на південь, не питай чому;', + 2147483647, + 'Some test text 123$%^&*()_', + N'ї', + N'Миші йдуть на південь, не питай чому;', + N'Миші йдуть на південь, не питай чому;', + NULL, + NULL, + NULL, + NULL, + '', + NULL, + 32767, + NULL, + N'Миші йдуть на південь, не питай чому;', + 255, + 1, + NULL, + NULL, + NULL, + 214748.3647 + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 4, + NULL, + NULL, + NULL, + NULL, + NULL, + '9999-12-31T13:00:04.123Z', + NULL, + '13:00:04.123456Z', + NULL, + NULL, + N'櫻花分店', + NULL, + '', + NULL, + N'櫻花分店', + N'櫻花分店', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + N'櫻花分店', + NULL, + 'true', + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 5, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '', + NULL, + NULL, + NULL, + '', + '', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '', + NULL, + 'false', + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 6, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); -INSERT INTO dbo.TEST_DATASET VALUES (1, -9223372036854775808, '123', '123', '0001-01-01', '1900-01-01', '1753-01-01', '0001-01-01', null, '0001-01-10 00:00:00 +01:00', 'a', 'a', null, 'a', 'a', 'a', 'a', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '1', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), null, '/1/1/', 'a', null, null, 999.33, '99999', null, null); -INSERT INTO dbo.TEST_DATASET VALUES (2, 9223372036854775807, '1234567890.1234567', '1234567890.1234567', '9999-12-31', '2079-06-06', '9999-12-31', '9999-12-31', '13:00:01', '9999-01-10 00:00:00 +01:00', '*', 'abc', -2147483648, 'abc', '*', 'abc', 'abc', null, null, null, null, null, null, -32768, null, 'abc', 0, 0, null, null, '9990000.3647', '-214748.3648'); -INSERT INTO dbo.TEST_DATASET VALUES (3, 0, null, null, '1999-01-08', null, '9999-12-31T13:00:04Z', '9999-12-31T13:00:04.123456Z', '13:00:04Z', null, null, N'Миші йдуть на південь, не питай чому;', 2147483647, 'Some test text 123$%^&*()_', N'ї', N'Миші йдуть на південь, не питай чому;', N'Миші йдуть на південь, не питай чому;', null, null, null, null, '', null, 32767, null, N'Миші йдуть на південь, не питай чому;', 255, 1, null, null, null, 214748.3647); -INSERT INTO dbo.TEST_DATASET VALUES (4, null, null, null, null, null, '9999-12-31T13:00:04.123Z', null, '13:00:04.123456Z', null, null, N'櫻花分店', null, '', null, N'櫻花分店', N'櫻花分店', null, null, null, null, null, null, null, null, N'櫻花分店', null, 'true', null, null, null, null); -INSERT INTO dbo.TEST_DATASET VALUES (5, null, null, null, null, null, null, null, null, null, null, '', null, null, null, '', '', null, null, null, null, null, null, null, null, '', null, 'false', null, null, null, null); -INSERT INTO dbo.TEST_DATASET VALUES (6, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO dbo.TEST_DATASET VALUES (7, null, null, null, null, null, null, null, null, null, null, N'\xF0\x9F\x9A\x80', null, null, null, N'\xF0\x9F\x9A\x80', N'\xF0\x9F\x9A\x80', null, null, null, null, null, null, null, null, N'\xF0\x9F\x9A\x80', null, null, null, null, null, null); +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 7, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + N'\xF0\x9F\x9A\x80', + NULL, + NULL, + NULL, + N'\xF0\x9F\x9A\x80', + N'\xF0\x9F\x9A\x80', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + N'\xF0\x9F\x9A\x80', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full_without_nulls.sql index 6992bc60f61a..2b6483f9e569 100644 --- a/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full_without_nulls.sql +++ b/airbyte-integrations/connectors/source-mssql/integration_tests/seed/full_without_nulls.sql @@ -1,12 +1,380 @@ -CREATE DATABASE MSSQL_FULL_NN; +CREATE + DATABASE MSSQL_FULL_NN; + USE MSSQL_FULL_NN; -CREATE TABLE dbo.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bigint,test_column_10 float,test_column_11 real,test_column_12 date,test_column_13 smalldatetime,test_column_14 datetime,test_column_15 datetime2,test_column_16 time,test_column_17 datetimeoffset,test_column_18 char,test_column_19 varchar(max) COLLATE Latin1_General_100_CI_AI_SC_UTF8,test_column_2 int,test_column_20 text,test_column_21 nchar,test_column_22 nvarchar(max),test_column_23 ntext,test_column_24 binary,test_column_25 varbinary(3),test_column_26 geometry,test_column_27 uniqueidentifier,test_column_28 xml,test_column_29 geography,test_column_3 smallint,test_column_30 hierarchyid,test_column_31 sql_variant,test_column_4 tinyint,test_column_5 bit,test_column_6 DECIMAL(5,2),test_column_7 numeric,test_column_8 money,test_column_9 smallmoney ); +CREATE + TABLE + dbo.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 FLOAT, + test_column_11 REAL, + test_column_12 DATE, + test_column_13 smalldatetime, + test_column_14 datetime, + test_column_15 datetime2, + test_column_16 TIME, + test_column_17 datetimeoffset, + test_column_18 CHAR, + test_column_19 VARCHAR(MAX) COLLATE Latin1_General_100_CI_AI_SC_UTF8, + test_column_2 INT, + test_column_20 text, + test_column_21 nchar, + test_column_22 nvarchar(MAX), + test_column_23 ntext, + test_column_24 BINARY, + test_column_25 VARBINARY(3), + test_column_26 geometry, + test_column_27 uniqueidentifier, + test_column_28 xml, + test_column_29 geography, + test_column_3 SMALLINT, + test_column_30 hierarchyid, + test_column_31 sql_variant, + test_column_4 tinyint, + test_column_5 bit, + test_column_6 DECIMAL( + 5, + 2 + ), + test_column_7 NUMERIC, + test_column_8 money, + test_column_9 smallmoney + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + '123', + '123', + '0001-01-01', + '1900-01-01', + '1753-01-01', + '0001-01-01', + '13:00:01', + '0001-01-10 00:00:00 +01:00', + 'a', + 'a', + - 2147483648, + 'a', + 'a', + 'a', + 'a', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '1', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + - 32768, + '/1/1/', + 'a', + 0, + 0, + 999.33, + '99999', + '9990000.3647', + '-214748.3648' + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '1234567890.1234567', + '1234567890.1234567', + '9999-12-31', + '2079-06-06', + '9999-12-31', + '9999-12-31', + '13:00:04Z', + '9999-01-10 00:00:00 +01:00', + '*', + 'abc', + 2147483647, + 'abc', + '*', + 'abc', + 'abc', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + 32767, + '/1/1/', + 'abc', + 255, + 1, + 999.33, + '99999', + '9990000.3647', + 214748.3647 + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 3, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '9999-01-10 00:00:00 +01:00', + '*', + N'Миші йдуть на південь, не питай чому;', + 2147483647, + 'Some test text 123$%^&*()_', + N'ї', + N'Миші йдуть на південь, не питай чому;', + N'Миші йдуть на південь, не питай чому;', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + 32767, + '/1/1/', + N'Миші йдуть на південь, не питай чому;', + 255, + 'true', + 999.33, + '99999', + '9990000.3647', + 214748.3647 + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 4, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '9999-01-10 00:00:00 +01:00', + '*', + N'櫻花分店', + 2147483647, + '', + N'ї', + N'櫻花分店', + N'櫻花分店', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + 32767, + '/1/1/', + N'櫻花分店', + 255, + 'false', + 999.33, + '99999', + '9990000.3647', + 214748.3647 + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 5, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '9999-01-10 00:00:00 +01:00', + '*', + '', + 2147483647, + '', + N'ї', + '', + '', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + 32767, + '/1/1/', + '', + 255, + 'false', + 999.33, + '99999', + '9990000.3647', + 214748.3647 + ); + +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 6, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '9999-01-10 00:00:00 +01:00', + '*', + N'\xF0\x9F\x9A\x80', + 2147483647, + '', + N'ї', + N'\xF0\x9F\x9A\x80', + N'\xF0\x9F\x9A\x80', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + 32767, + '/1/1/', + N'\xF0\x9F\x9A\x80', + 255, + 'false', + 999.33, + '99999', + '9990000.3647', + 214748.3647 + ); -INSERT INTO dbo.TEST_DATASET VALUES (1, -9223372036854775808, '123', '123', '0001-01-01', '1900-01-01', '1753-01-01', '0001-01-01', '13:00:01', '0001-01-10 00:00:00 +01:00', 'a', 'a', -2147483648, 'a', 'a', 'a', 'a', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '1', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), -32768, '/1/1/', 'a', 0, 0, 999.33, '99999', '9990000.3647', '-214748.3648'); -INSERT INTO dbo.TEST_DATASET VALUES (2, 9223372036854775807, '1234567890.1234567', '1234567890.1234567', '9999-12-31', '2079-06-06', '9999-12-31', '9999-12-31', '13:00:04Z', '9999-01-10 00:00:00 +01:00', '*', 'abc', 2147483647, 'abc', '*', 'abc', 'abc', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), 32767, '/1/1/', 'abc', 255, 1, 999.33, '99999', '9990000.3647', 214748.3647); -INSERT INTO dbo.TEST_DATASET VALUES (3, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '9999-01-10 00:00:00 +01:00', '*', N'Миші йдуть на південь, не питай чому;', 2147483647, 'Some test text 123$%^&*()_', N'ї', N'Миші йдуть на південь, не питай чому;', N'Миші йдуть на південь, не питай чому;', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), 32767, '/1/1/', N'Миші йдуть на південь, не питай чому;', 255, 'true', 999.33, '99999', '9990000.3647', 214748.3647); -INSERT INTO dbo.TEST_DATASET VALUES (4, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '9999-01-10 00:00:00 +01:00', '*', N'櫻花分店', 2147483647, '', N'ї', N'櫻花分店', N'櫻花分店', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), 32767, '/1/1/', N'櫻花分店', 255, 'false', 999.33, '99999', '9990000.3647', 214748.3647); -INSERT INTO dbo.TEST_DATASET VALUES (5, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '9999-01-10 00:00:00 +01:00', '*', '', 2147483647, '', N'ї', '', '', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), 32767, '/1/1/', '', 255, 'false', 999.33, '99999', '9990000.3647', 214748.3647); -INSERT INTO dbo.TEST_DATASET VALUES (6, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '9999-01-10 00:00:00 +01:00', '*', N'\xF0\x9F\x9A\x80', 2147483647, '', N'ї', N'\xF0\x9F\x9A\x80', N'\xF0\x9F\x9A\x80', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), 32767, '/1/1/', N'\xF0\x9F\x9A\x80', 255, 'false', 999.33, '99999', '9990000.3647', 214748.3647); -INSERT INTO dbo.TEST_DATASET VALUES (7, 0, '1234567890.1234567', '1234567890.1234567', '1999-01-08', '2079-06-06', '9999-12-31T13:00:04.123Z', '9999-12-31T13:00:04.123456Z', '13:00:04.123456Z', '9999-01-10 00:00:00 +01:00', '*', N'\xF0\x9F\x9A\x80', 2147483647, '', N'ї', N'\xF0\x9F\x9A\x80', N'\xF0\x9F\x9A\x80', CAST( 'A' AS BINARY(1)), CAST( 'ABC' AS VARBINARY), geometry::STGeomFromText('LINESTRING (100 100, 20 180, 180 180)', 0), '375CFC44-CAE3-4E43-8083-821D2DF0E626', '', geography::STGeomFromText('LINESTRING(-122.360 47.656, -122.343 47.656 )', 4326), 32767, '/1/1/', N'\xF0\x9F\x9A\x80', 255, 'false', 999.33, '99999', '9990000.3647', 214748.3647); +INSERT + INTO + dbo.TEST_DATASET + VALUES( + 7, + 0, + '1234567890.1234567', + '1234567890.1234567', + '1999-01-08', + '2079-06-06', + '9999-12-31T13:00:04.123Z', + '9999-12-31T13:00:04.123456Z', + '13:00:04.123456Z', + '9999-01-10 00:00:00 +01:00', + '*', + N'\xF0\x9F\x9A\x80', + 2147483647, + '', + N'ї', + N'\xF0\x9F\x9A\x80', + N'\xF0\x9F\x9A\x80', + CAST( + 'A' AS BINARY(1) + ), + CAST( + 'ABC' AS VARBINARY + ), + geometry::STGeomFromText( + 'LINESTRING (100 100, 20 180, 180 180)', + 0 + ), + '375CFC44-CAE3-4E43-8083-821D2DF0E626', + '', + geography::STGeomFromText( + 'LINESTRING(-122.360 47.656, -122.343 47.656 )', + 4326 + ), + 32767, + '/1/1/', + N'\xF0\x9F\x9A\x80', + 255, + 'false', + 999.33, + '99999', + '9990000.3647', + 214748.3647 + ); diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index 65f5cca207ea..5c3d7d163211 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 200 + sl: 100 allowedHosts: hosts: - ${host} @@ -6,11 +9,12 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 1.0.17 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-mssql + documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql icon: mssql.svg - license: MIT + license: ELv2 name: Microsoft SQL Server (MSSQL) registries: cloud: @@ -19,8 +23,13 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/mssql + supportLevel: community tags: - language:java - language:python + releases: + breakingChanges: + 2.0.0: + message: "Add default cursor for cdc" + upgradeDeadline: "2023-08-23" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java index e3f18dc45ee1..be19a7df5d9a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java @@ -4,14 +4,36 @@ package io.airbyte.integrations.source.mssql; +import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_EVENT_SERIAL_NO; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_LSN; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.integrations.debezium.CdcMetadataInjector; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; -public class MssqlCdcConnectorMetadataInjector implements CdcMetadataInjector { +public class MssqlCdcConnectorMetadataInjector implements CdcMetadataInjector { + + private final long emittedAtConverted; + + // This now makes this class stateful. Please make sure to use the same instance within a sync + private final AtomicLong recordCounter = new AtomicLong(1); + private static final long ONE_HUNDRED_MILLION = 100_000_000; + private static MssqlCdcConnectorMetadataInjector mssqlCdcConnectorMetadataInjector; + + private MssqlCdcConnectorMetadataInjector(final Instant emittedAt) { + this.emittedAtConverted = emittedAt.getEpochSecond() * ONE_HUNDRED_MILLION; + } + + public static MssqlCdcConnectorMetadataInjector getInstance(final Instant emittedAt) { + if (mssqlCdcConnectorMetadataInjector == null) { + mssqlCdcConnectorMetadataInjector = new MssqlCdcConnectorMetadataInjector(emittedAt); + } + + return mssqlCdcConnectorMetadataInjector; + } @Override public void addMetaData(final ObjectNode event, final JsonNode source) { @@ -19,6 +41,7 @@ public void addMetaData(final ObjectNode event, final JsonNode source) { final String eventSerialNo = source.get("event_serial_no").asText(); event.put(CDC_LSN, commitLsn); event.put(CDC_EVENT_SERIAL_NO, eventSerialNo); + event.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); } @Override @@ -26,4 +49,8 @@ public String namespace(final JsonNode source) { return source.get("schema").asText(); } + private Long getCdcDefaultCursor() { + return this.emittedAtConverted + this.recordCounter.getAndIncrement(); + } + } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java index 4d3a50fefaa8..bc15f1bcb8c7 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java @@ -120,8 +120,8 @@ static boolean isCdc(final JsonNode config) { @VisibleForTesting static SnapshotIsolation getSnapshotIsolationConfig(final JsonNode config) { // new replication method config since version 0.4.0 - if (config.hasNonNull(REPLICATION_FIELD)) { - final JsonNode replicationConfig = config.get(REPLICATION_FIELD); + if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { + final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); final JsonNode snapshotIsolation = replicationConfig.get(CDC_SNAPSHOT_ISOLATION_FIELD); return SnapshotIsolation.from(snapshotIsolation.asText()); } @@ -131,8 +131,8 @@ static SnapshotIsolation getSnapshotIsolationConfig(final JsonNode config) { @VisibleForTesting static DataToSync getDataToSyncConfig(final JsonNode config) { // new replication method config since version 0.4.0 - if (config.hasNonNull(REPLICATION_FIELD)) { - final JsonNode replicationConfig = config.get(REPLICATION_FIELD); + if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { + final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); final JsonNode dataToSync = replicationConfig.get(CDC_DATA_TO_SYNC_FIELD); return DataToSync.from(dataToSync.asText()); } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 9dc1e67a3b5f..2c72ba028abe 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; @@ -83,6 +84,7 @@ SELECT CAST(IIF(EXISTS(SELECT TOP 1 1 FROM "%s"."%s" WHERE "%s" IS NULL), 1, 0) public static final String CDC_EVENT_SERIAL_NO = "_ab_cdc_event_serial_no"; private static final String HIERARCHYID = "hierarchyid"; private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; + public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; private List schemas; public static Source sshWrappedSource() { @@ -240,6 +242,7 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { .map(MssqlSource::overrideSyncModes) .map(MssqlSource::removeIncrementalWithoutPk) .map(MssqlSource::setIncrementalToSourceDefined) + .map(MssqlSource::setDefaultCursorFieldForCdc) .map(MssqlSource::addCdcMetadataColumns) .collect(toList()); @@ -453,10 +456,12 @@ public List> getIncrementalIterators( MssqlCdcTargetPosition.getTargetPosition(database, sourceConfig.get(JdbcUtils.DATABASE_KEY).asText()), true, firstRecordWaitTime, OptionalInt.empty()); + final MssqlCdcConnectorMetadataInjector mssqlCdcConnectorMetadataInjector = MssqlCdcConnectorMetadataInjector.getInstance(emittedAt); + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, new MssqlCdcSavedInfoFetcher(stateManager.getCdcStateManager().getCdcState()), new MssqlCdcStateHandler(stateManager), - new MssqlCdcConnectorMetadataInjector(), + mssqlCdcConnectorMetadataInjector, MssqlCdcHelper.getDebeziumProperties(database, catalog), emittedAt, true); @@ -494,17 +499,30 @@ private static AirbyteStream setIncrementalToSourceDefined(final AirbyteStream s return stream; } + /* + * To prepare for Destination v2, cdc streams must have a default cursor field Cursor format: the + * airbyte [emittedAt] + [sync wide record counter] + */ + private static AirbyteStream setDefaultCursorFieldForCdc(final AirbyteStream stream) { + if (stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)) { + stream.setDefaultCursorField(ImmutableList.of(CDC_DEFAULT_CURSOR)); + } + return stream; + } + // Note: in place mutation. private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); final ObjectNode properties = (ObjectNode) jsonSchema.get("properties"); + final JsonNode airbyteIntegerType = Jsons.jsonNode(ImmutableMap.of("type", "number", "airbyte_type", "integer")); final JsonNode stringType = Jsons.jsonNode(ImmutableMap.of("type", "string")); properties.set(CDC_LSN, stringType); properties.set(CDC_UPDATED_AT, stringType); properties.set(CDC_DELETED_AT, stringType); properties.set(CDC_EVENT_SERIAL_NO, stringType); + properties.set(CDC_DEFAULT_CURSOR, airbyteIntegerType); return stream; } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java index b12183fe125b..e27ab427338a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java @@ -6,6 +6,7 @@ import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_EVENT_SERIAL_NO; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_LSN; import static io.airbyte.integrations.source.mssql.MssqlSource.DRIVER_CLASS; @@ -21,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -38,6 +40,7 @@ import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; import io.debezium.connector.sqlserver.Lsn; import java.sql.SQLException; import java.util.List; @@ -181,7 +184,7 @@ public String createSchemaQuery(final String schemaName) { // TODO : Delete this Override when MSSQL supports individual table snapshot @Override - public void newTableSnapshotTest() throws Exception { + public void newTableSnapshotTest() { // Do nothing } @@ -314,7 +317,7 @@ void testAssertSnapshotIsolationDisabled() { // set snapshot_isolation level to "Read Committed" to disable snapshot .put("snapshot_isolation", "Read Committed") .build()); - Jsons.replaceNestedValue(config, List.of("replication"), replicationConfig); + Jsons.replaceNestedValue(config, List.of("replication_method"), replicationConfig); assertDoesNotThrow(() -> source.assertSnapshotIsolationAllowed(config, testJdbcDatabase)); switchSnapshotIsolation(false, dbName); assertDoesNotThrow(() -> source.assertSnapshotIsolationAllowed(config, testJdbcDatabase)); @@ -350,7 +353,7 @@ void testCdcCheckOperations() throws Exception { void testCdcCheckOperationsWithDot() throws Exception { // assertCdcEnabledInDb and validate escape with special character switchCdcOnDatabase(true, dbNamewithDot); - AirbyteConnectionStatus status = getSource().check(getConfig()); + final AirbyteConnectionStatus status = getSource().check(getConfig()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.SUCCEEDED); } @@ -374,6 +377,7 @@ protected void removeCDCColumns(final ObjectNode data) { data.remove(CDC_UPDATED_AT); data.remove(CDC_DELETED_AT); data.remove(CDC_EVENT_SERIAL_NO); + data.remove(CDC_DEFAULT_CURSOR); } @Override @@ -409,6 +413,7 @@ protected void assertNullCdcMetaData(final JsonNode data) { assertNull(data.get(CDC_UPDATED_AT)); assertNull(data.get(CDC_DELETED_AT)); assertNull(data.get(CDC_EVENT_SERIAL_NO)); + assertNull(data.get(CDC_DEFAULT_CURSOR)); } @Override @@ -416,6 +421,7 @@ protected void assertCdcMetaData(final JsonNode data, final boolean deletedAtNul assertNotNull(data.get(CDC_LSN)); assertNotNull(data.get(CDC_EVENT_SERIAL_NO)); assertNotNull(data.get(CDC_UPDATED_AT)); + assertNotNull(data.get(CDC_DEFAULT_CURSOR)); if (deletedAtNull) { assertTrue(data.get(CDC_DELETED_AT).isNull()); } else { @@ -428,12 +434,21 @@ protected void addCdcMetadataColumns(final AirbyteStream stream) { final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); final ObjectNode properties = (ObjectNode) jsonSchema.get("properties"); + final JsonNode airbyteIntegerType = Jsons.jsonNode(ImmutableMap.of("type", "number", "airbyte_type", "integer")); final JsonNode stringType = Jsons.jsonNode(ImmutableMap.of("type", "string")); properties.set(CDC_LSN, stringType); properties.set(CDC_UPDATED_AT, stringType); properties.set(CDC_DELETED_AT, stringType); properties.set(CDC_EVENT_SERIAL_NO, stringType); + properties.set(CDC_DEFAULT_CURSOR, airbyteIntegerType); + + } + @Override + protected void addCdcDefaultCursorField(final AirbyteStream stream) { + if (stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)) { + stream.setDefaultCursorField(ImmutableList.of(CDC_DEFAULT_CURSOR)); + } } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java index 304a1252efcf..a2f29d5064a7 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java @@ -62,33 +62,33 @@ public void testGetSnapshotIsolation() { assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(LEGACY_CDC_CONFIG)); // new replication method config since version 0.4.0 - final JsonNode newCdcNonSnapshot = Jsons.jsonNode(Map.of("replication", + final JsonNode newCdcNonSnapshot = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Read Committed")))); assertEquals(SnapshotIsolation.READ_COMMITTED, MssqlCdcHelper.getSnapshotIsolationConfig(newCdcNonSnapshot)); - final JsonNode newCdcSnapshot = Jsons.jsonNode(Map.of("replication", + final JsonNode newCdcSnapshot = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")))); assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(newCdcSnapshot)); // migration from legacy to new config final JsonNode mixCdcNonSnapshot = Jsons.jsonNode(Map.of( - "replication_method", "Standard", - "replication", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "replication", "Standard", + "replication_method", Jsons.jsonNode(Map.of( + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Read Committed")))); assertEquals(SnapshotIsolation.READ_COMMITTED, MssqlCdcHelper.getSnapshotIsolationConfig(mixCdcNonSnapshot)); final JsonNode mixCdcSnapshot = Jsons.jsonNode(Map.of( - "replication_method", "Standard", - "replication", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "replication", "Standard", + "replication_method", Jsons.jsonNode(Map.of( + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")))); assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(mixCdcSnapshot)); @@ -100,33 +100,33 @@ public void testGetDataToSyncConfig() { assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(LEGACY_CDC_CONFIG)); // new replication method config since version 0.4.0 - final JsonNode newCdcExistingAndNew = Jsons.jsonNode(Map.of("replication", + final JsonNode newCdcExistingAndNew = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Read Committed")))); assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(newCdcExistingAndNew)); - final JsonNode newCdcNewOnly = Jsons.jsonNode(Map.of("replication", + final JsonNode newCdcNewOnly = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "New Changes Only", "snapshot_isolation", "Snapshot")))); assertEquals(DataToSync.NEW_CHANGES_ONLY, MssqlCdcHelper.getDataToSyncConfig(newCdcNewOnly)); final JsonNode mixCdcExistingAndNew = Jsons.jsonNode(Map.of( - "replication_method", "Standard", - "replication", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "replication", "Standard", + "replication_method", Jsons.jsonNode(Map.of( + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Read Committed")))); assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(mixCdcExistingAndNew)); final JsonNode mixCdcNewOnly = Jsons.jsonNode(Map.of( - "replication_method", "Standard", - "replication", + "replication", "Standard", + "replication_method", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "New Changes Only", "snapshot_isolation", "Snapshot")))); assertEquals(DataToSync.NEW_CHANGES_ONLY, MssqlCdcHelper.getDataToSyncConfig(mixCdcNewOnly)); diff --git a/airbyte-integrations/connectors/source-my-hours/metadata.yaml b/airbyte-integrations/connectors/source-my-hours/metadata.yaml index 14520bfeccf8..68328da187e8 100644 --- a/airbyte-integrations/connectors/source-my-hours/metadata.yaml +++ b/airbyte-integrations/connectors/source-my-hours/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/my-hours tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-my-hours/requirements.txt b/airbyte-integrations/connectors/source-my-hours/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-my-hours/requirements.txt +++ b/airbyte-integrations/connectors/source-my-hours/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-my-hours/setup.py b/airbyte-integrations/connectors/source-my-hours/setup.py index db0a937579b7..cd096054986c 100644 --- a/airbyte-integrations/connectors/source-my-hours/setup.py +++ b/airbyte-integrations/connectors/source-my-hours/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "requests_mock==1.8.0", "responses~=0.16.0", ] diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index fe26c070409f..d7b6d236fda8 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -24,6 +24,6 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.24 +LABEL io.airbyte.version=3.0.1 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle index ddb4516e6365..c305939e20bf 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle @@ -29,3 +29,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml index d60d4d750741..ea210bc1997d 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml @@ -11,14 +11,19 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 2.0.24 + dockerImageTag: 3.0.1 dockerRepository: airbyte/source-mysql-strict-encrypt githubIssueLabel: source-mysql icon: mysql.svg - license: MIT + license: ELv2 name: MySQL releaseStage: beta documentationUrl: https://docs.airbyte.com/integrations/sources/mysql tags: - language:java + releases: + breakingChanges: + 3.0.0: + message: "Add default cursor for cdc" + upgradeDeadline: "2023-08-17" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index 65be62dec454..10e6728338b2 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -39,7 +39,8 @@ "title": "Password", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 4, + "always_show": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", @@ -172,25 +173,14 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "Replication method to use for extracting data from the database.", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", "order": 8, + "default": "CDC", "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", "required": ["method"], "properties": { "method": { @@ -205,13 +195,27 @@ "default": 300, "min": 120, "max": 1200, - "order": 1 + "order": 1, + "always_show": true }, "server_time_zone": { "type": "string", "title": "Configured server timezone for the MySQL source (Advanced)", "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", - "order": 2 + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 } } } diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 21f94bd5485c..76a9db7cc650 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -24,6 +24,6 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.24 +LABEL io.airbyte.version=3.0.1 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 189ad748b4f7..8bacfd89490a 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -1,9 +1,12 @@ +import org.jsonschema2pojo.SourceType + plugins { id 'application' id 'airbyte-docker' id 'airbyte-integration-test-java' id 'airbyte-performance-test-java' id 'airbyte-connector-acceptance-test' + id 'org.jsonschema2pojo' version '1.2.1' } application { @@ -39,3 +42,18 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + +jsonSchema2Pojo { + sourceType = SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/internal_models") + targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') + removeOldOutput = true + + targetPackage = 'io.airbyte.integrations.source.mysql.internal.models' + + useLongIntegers = true + generateBuilders = true + includeConstructors = false + includeSetters = true +} + diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/basic.sql index ec787b1c8eb3..17be356eebd3 100644 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/basic.sql @@ -1,14 +1,153 @@ -CREATE DATABASE MYSQL_BASIC; +CREATE + DATABASE MYSQL_BASIC; + USE MYSQL_BASIC; -SET @@sql_mode=''; +SET +@@sql_mode = ''; + +CREATE + TABLE + test.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 bit, + test_column_10 SMALLINT, + test_column_12 SMALLINT unsigned, + test_column_13 mediumint, + test_column_15 INT, + test_column_16 INT unsigned, + test_column_18 BIGINT, + test_column_19 FLOAT, + test_column_2 bit(1), + test_column_20 DOUBLE, + test_column_21 DECIMAL( + 10, + 3 + ), + test_column_22 DECIMAL( + 19, + 2 + ), + test_column_24 DATE, + test_column_25 datetime NOT NULL DEFAULT now(), + test_column_26 datetime, + test_column_27 TIMESTAMP, + test_column_29 TIME, + test_column_3 bit(7), + test_column_30 YEAR, + test_column_31 VARCHAR(63), + test_column_4 tinyint, + test_column_5 tinyint(1), + test_column_6 tinyint(1) unsigned, + test_column_7 tinyint(2), + test_column_8 BOOL, + test_column_9 BOOLEAN + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 1, + 1, + - 32768, + 0, + - 8388608, + - 2147483648, + 3428724653, + 9223372036854775807, + 10.5, + 1, + POWER( 10, 308 ), + 0.188, + 1700000.01, + '1999-01-08', + '2005-10-10 23:22:21', + '2005-10-10 23:22:21', + '2021-01-00', + '-22:59:59', + b'1000001', + '1997', + 'Airbyte', + - 128, + 1, + 0, + - 128, + 1, + 1 + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 2, + 0, + 32767, + 65535, + 8388607, + 2147483647, + 3428724653, + 9223372036854775807, + 10.5, + 0, + 1 / POWER( 10, 45 ), + 0.188, + 1700000.01, + '2021-01-01', + '2013-09-05T10:10:02', + '2013-09-05T10:10:02', + '2021-00-00', + '23:59:59', + b'1000001', + '0', + '!"#$%&\'()*+, + -./:; -CREATE TABLE test.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bit,test_column_10 smallint,test_column_12 smallint unsigned,test_column_13 mediumint,test_column_15 int,test_column_16 int unsigned,test_column_18 bigint,test_column_19 float,test_column_2 bit(1),test_column_20 double,test_column_21 decimal(10,3),test_column_22 decimal(19,2),test_column_24 date,test_column_25 datetime not null default now(),test_column_26 datetime,test_column_27 timestamp,test_column_29 time,test_column_3 bit(7),test_column_30 year,test_column_31 VARCHAR(63),test_column_4 tinyint,test_column_5 tinyint(1),test_column_6 tinyint(1) unsigned,test_column_7 tinyint(2),test_column_8 BOOL,test_column_9 BOOLEAN ); +<=>? \@ [ \ ] ^_\ ` { | } ~ ', 127, 0, 1, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (3, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, ' 2021 - 01 - 01 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2013 - 09 - 06 T10:10:02 ', ' 0000 - 00 - 00 ', ' 00:00:00 ', b' 1000001 ', ' 50 ', ' ! "#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 2, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (4, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', b'1000001', '70', '!" #$ %& \'()*+,-./:;<=>?\@[\]^_\`{|}~', +127, +0, +3, +127, +0, +0 + ); -INSERT INTO test.TEST_DATASET VALUES (1, 1, -32768, 0, -8388608, -2147483648, 3428724653, 9223372036854775807, 10.5, 1, power(10, 308), 0.188, 1700000.01, '1999-01-08', '2005-10-10 23:22:21', '2005-10-10 23:22:21', '2021-01-00', '-22:59:59', b'1000001', '1997', 'Airbyte', -128, 1, 0, -128, 1, 1); -INSERT INTO test.TEST_DATASET VALUES (2, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 1/power(10, 45), 0.188, 1700000.01, '2021-01-01', '2013-09-05T10:10:02', '2013-09-05T10:10:02', '2021-00-00', '23:59:59', b'1000001', '0', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 1, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (3, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '0000-00-00', '00:00:00', b'1000001', '50', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 2, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (4, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', b'1000001', '70', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 3, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (5, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', b'1000001', '80', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 3, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (6, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', b'1000001', '99', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 3, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (7, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', b'1000001', '99', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 3, 127, 0, 0); +INSERT + INTO + test.TEST_DATASET + VALUES( + 5, + 0, + 32767, + 65535, + 8388607, + 2147483647, + 3428724653, + 9223372036854775807, + 10.5, + 0, + 10.5, + 0.188, + 1700000.01, + '2021-01-01', + '2013-09-06T10:10:02', + '2013-09-06T10:10:02', + '2022-08-09T10:17:16.161342Z', + '00:00:00', + b'1000001', + '80', + '!"#$%&\'()*+, + -./:; +<=>? \@ [ \ ] ^_\ ` { | } ~ ', 127, 0, 3, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (6, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, ' 2021 - 01 - 01 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2022 - 08 - 09 T10:17:16.161342 Z', ' 00:00:00 ', b' 1000001 ', ' 99 ', ' ! "#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 127, 0, 3, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (7, 0, 32767, 65535, 8388607, 2147483647, 3428724653, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', b'1000001', '99', '!" #$ %& \'()*+,-./:;<=>?\@[\]^_\`{|}~', +127, +0, +3, +127, +0, +0 + ); diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full.sql index 3200791ce79e..a6499c150184 100644 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full.sql +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full.sql @@ -1,13 +1,528 @@ -CREATE DATABASE MYSQL_FULL; +CREATE + DATABASE MYSQL_FULL; + USE MYSQL_FULL; -SET @@sql_mode=''; +SET +@@sql_mode = ''; + +CREATE + TABLE + test.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 bit, + test_column_10 SMALLINT, + test_column_11 SMALLINT zerofill, + test_column_12 SMALLINT unsigned, + test_column_13 mediumint, + test_column_14 mediumint zerofill, + test_column_15 INT, + test_column_16 INT unsigned, + test_column_17 INT zerofill, + test_column_18 BIGINT, + test_column_19 FLOAT, + test_column_2 bit(1), + test_column_20 DOUBLE, + test_column_21 DECIMAL( + 10, + 3 + ), + test_column_22 DECIMAL( + 19, + 2 + ), + test_column_23 DATE NOT NULL DEFAULT '0000-00-00', + test_column_24 DATE, + test_column_25 datetime NOT NULL DEFAULT now(), + test_column_26 datetime, + test_column_27 TIMESTAMP, + test_column_28 TIME NOT NULL DEFAULT '00:00:00', + test_column_29 TIME, + test_column_3 bit(7), + test_column_30 YEAR, + test_column_31 VARCHAR(63), + test_column_32 VARCHAR(63) CHARACTER + SET + utf16, + test_column_33 VARCHAR(63) CHARACTER + SET + cp1251, + test_column_34 VARCHAR(7) CHARACTER + SET + BINARY, + test_column_35 CHAR(63), + test_column_36 CHAR(63) CHARACTER + SET + utf16, + test_column_37 CHAR(63) CHARACTER + SET + cp1251, + test_column_38 CHAR(7) CHARACTER + SET + BINARY, + test_column_39 BLOB, + test_column_4 tinyint, + test_column_40 TINYBLOB, + test_column_5 tinyint(1), + test_column_51 json, + test_column_52 ENUM( + 'xs', + 's', + 'm', + 'l', + 'xl' + ), + test_column_53 + SET + ( + 'xs', + 's', + 'm', + 'l', + 'xl' + ), + test_column_6 tinyint(1) unsigned, + test_column_7 tinyint(2), + test_column_8 BOOL, + test_column_9 BOOLEAN + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 1, + NULL, + NULL, + 1, + NULL, + NULL, + 1, + NULL, + 3428724653, + 1, + NULL, + NULL, + NULL, + NULL, + 0.188, + 1700000.01, + '1999-01-08', + '1999-01-08', + '2005-10-10 23:22:21', + '2005-10-10 23:22:21', + NULL, + '-22:59:59', + '-22:59:59', + NULL, + NULL, + NULL, + 0 xfffd, + 'тест', + NULL, + NULL, + 0 xfffd, + 'тест', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + concat( + lpad( + '0', + 262144, + '0' + ), + lpad( + '0', + 262144, + '0' + ), + lpad( + '0', + 262144, + '0' + ), + lpad( + '0', + 261568, + '0' + ) + ), + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 2, + 1, + - 32768, + NULL, + 0, + - 8388608, + NULL, + - 2147483648, + NULL, + NULL, + 9223372036854775807, + 10.5, + 1, + POWER( 10, 308 ), + NULL, + NULL, + '2021-01-01', + '2021-01-01', + '2013-09-05T10:10:02', + '2013-09-05T10:10:02', + '2021-01-00', + '23:59:59', + '23:59:59', + b'1000001', + '1997', + 'Airbyte', + NULL, + NULL, + 'Airbyte', + 'Airbyte', + NULL, + NULL, + 'Airbyte', + 'Airbyte', + - 128, + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'test', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 1, + 'test', + '{"a": 10, "b": 15}', + 'xs', + 'xs,s', + 0, + - 128, + 1, + 1 + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 3, + 0, + 32767, + NULL, + 65535, + 8388607, + NULL, + 2147483647, + NULL, + NULL, + NULL, + NULL, + 0, + 1 / POWER( 10, 45 ), + NULL, + NULL, + NULL, + NULL, + '2013-09-06T10:10:02', + '2013-09-06T10:10:02', + '2021-00-00', + '00:00:00', + '00:00:00', + NULL, + '0', + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + NULL, + NULL, + NULL, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + NULL, + NULL, + NULL, + NULL, + 127, + NULL, + NULL, + NULL, + NULL, + NULL, + 'тест', + 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', + 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', + 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', + 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', + 0, + NULL, + '{"fóo": "bär"}', + 'm', + 'm,xl', + 1, + 127, + 0, + 0 + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 4, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 10.5, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '0000-00-00', + NULL, + NULL, + NULL, + '50', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '{"春江潮水连海平": "海上明月共潮生"}', + NULL, + NULL, + 2, + NULL, + NULL, + NULL + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 5, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '2022-08-09T10:17:16.161342Z', + NULL, + NULL, + NULL, + '70', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 3, + NULL, + NULL, + NULL + ); -CREATE TABLE test.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bit,test_column_10 smallint,test_column_11 smallint zerofill,test_column_12 smallint unsigned,test_column_13 mediumint,test_column_14 mediumint zerofill,test_column_15 int,test_column_16 int unsigned,test_column_17 int zerofill,test_column_18 bigint,test_column_19 float,test_column_2 bit(1),test_column_20 double,test_column_21 decimal(10,3),test_column_22 decimal(19,2),test_column_23 date not null default '0000-00-00',test_column_24 date,test_column_25 datetime not null default now(),test_column_26 datetime,test_column_27 timestamp,test_column_28 time not null default '00:00:00',test_column_29 time,test_column_3 bit(7),test_column_30 year,test_column_31 VARCHAR(63),test_column_32 VARCHAR(63) character set utf16,test_column_33 VARCHAR(63) character set cp1251,test_column_34 VARCHAR(7) character set binary,test_column_35 CHAR(63),test_column_36 CHAR(63) character set utf16,test_column_37 CHAR(63) character set cp1251,test_column_38 CHAR(7) character set binary,test_column_39 BLOB,test_column_4 tinyint,test_column_40 TINYBLOB,test_column_5 tinyint(1),test_column_51 json,test_column_52 ENUM('xs', 's', 'm', 'l', 'xl'),test_column_53 SET('xs', 's', 'm', 'l', 'xl'),test_column_6 tinyint(1) unsigned,test_column_7 tinyint(2),test_column_8 BOOL,test_column_9 BOOLEAN ); +INSERT + INTO + test.TEST_DATASET + VALUES( + 6, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '80', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); -INSERT INTO test.TEST_DATASET VALUES (1, null, null, 1, null, null, 1, null, 3428724653, 1, null, null, null, null, 0.188, 1700000.01, '1999-01-08', '1999-01-08', '2005-10-10 23:22:21', '2005-10-10 23:22:21', null, '-22:59:59', '-22:59:59', null, null, null, 0xfffd, 'тест', null, null, 0xfffd, 'тест', null, null, null, null, null, null, null, null, null, null, null, null, null, null, concat(lpad('0', 262144, '0'),lpad('0', 262144, '0'),lpad('0', 262144, '0'),lpad('0', 261568, '0')), null, null, null, null, null, null, null); -INSERT INTO test.TEST_DATASET VALUES (2, 1, -32768, null, 0, -8388608, null, -2147483648, null, null, 9223372036854775807, 10.5, 1, power(10, 308), null, null, '2021-01-01', '2021-01-01', '2013-09-05T10:10:02', '2013-09-05T10:10:02', '2021-01-00', '23:59:59', '23:59:59', b'1000001', '1997', 'Airbyte', null, null, 'Airbyte', 'Airbyte', null, null, 'Airbyte', 'Airbyte', -128, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'test', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 1, 'test', '{"a": 10, "b": 15}', 'xs', 'xs,s', 0, -128, 1, 1); -INSERT INTO test.TEST_DATASET VALUES (3, 0, 32767, null, 65535, 8388607, null, 2147483647, null, null, null, null, 0, 1/power(10, 45), null, null, null, null, '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2021-00-00', '00:00:00', '00:00:00', null, '0', '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', null, null, null, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', null, null, null, null, 127, null, null, null, null, null, 'тест', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, null, '{"fóo": "bär"}', 'm', 'm,xl', 1, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (4, null, null, null, null, null, null, null, null, null, null, null, null, 10.5, null, null, null, null, null, null, '0000-00-00', null, null, null, '50', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), null, null, null, null, null, null, '{"春江潮水连海平": "海上明月共潮生"}', null, null, 2, null, null, null); -INSERT INTO test.TEST_DATASET VALUES (5, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '2022-08-09T10:17:16.161342Z', null, null, null, '70', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 3, null, null, null); -INSERT INTO test.TEST_DATASET VALUES (6, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '80', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO test.TEST_DATASET VALUES (7, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '99', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); +INSERT + INTO + test.TEST_DATASET + VALUES( + 7, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '99', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full_without_nulls.sql index dbe873011d6f..b4d6ff698342 100644 --- a/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full_without_nulls.sql +++ b/airbyte-integrations/connectors/source-mysql/integration_tests/seed/full_without_nulls.sql @@ -1,13 +1,306 @@ -CREATE DATABASE MYSQL_FULL_NN; +CREATE + DATABASE MYSQL_FULL_NN; + USE MYSQL_FULL_NN; -SET @@sql_mode=''; +SET +@@sql_mode = ''; + +CREATE + TABLE + test.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 bit, + test_column_10 SMALLINT, + test_column_11 SMALLINT zerofill, + test_column_12 SMALLINT unsigned, + test_column_13 mediumint, + test_column_14 mediumint zerofill, + test_column_15 INT, + test_column_16 INT unsigned, + test_column_17 INT zerofill, + test_column_18 BIGINT, + test_column_19 FLOAT, + test_column_2 bit(1), + test_column_20 DOUBLE, + test_column_21 DECIMAL( + 10, + 3 + ), + test_column_22 DECIMAL( + 19, + 2 + ), + test_column_23 DATE NOT NULL DEFAULT '0000-00-00', + test_column_24 DATE, + test_column_25 datetime NOT NULL DEFAULT now(), + test_column_26 datetime, + test_column_27 TIMESTAMP, + test_column_28 TIME NOT NULL DEFAULT '00:00:00', + test_column_29 TIME, + test_column_3 bit(7), + test_column_30 YEAR, + test_column_31 VARCHAR(63), + test_column_32 VARCHAR(63) CHARACTER + SET + utf16, + test_column_33 VARCHAR(63) CHARACTER + SET + cp1251, + test_column_34 VARCHAR(7) CHARACTER + SET + BINARY, + test_column_35 CHAR(63), + test_column_36 CHAR(63) CHARACTER + SET + utf16, + test_column_37 CHAR(63) CHARACTER + SET + cp1251, + test_column_38 CHAR(7) CHARACTER + SET + BINARY, + test_column_39 BLOB, + test_column_4 tinyint, + test_column_40 TINYBLOB, + test_column_5 tinyint(1), + test_column_51 json, + test_column_52 ENUM( + 'xs', + 's', + 'm', + 'l', + 'xl' + ), + test_column_53 + SET + ( + 'xs', + 's', + 'm', + 'l', + 'xl' + ), + test_column_6 tinyint(1) unsigned, + test_column_7 tinyint(2), + test_column_8 BOOL, + test_column_9 BOOLEAN + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 1, + 1, + - 32768, + 1, + 0, + - 8388608, + 1, + - 2147483648, + 3428724653, + 1, + 9223372036854775807, + 10.5, + 1, + POWER( 10, 308 ), + 0.188, + 1700000.01, + '1999-01-08', + '1999-01-08', + '2005-10-10 23:22:21', + '2005-10-10 23:22:21', + '2021-01-00', + '-22:59:59', + '-22:59:59', + b'1000001', + '1997', + 'Airbyte', + 0 xfffd, + 'тест', + 'Airbyte', + 'Airbyte', + 0 xfffd, + 'тест', + 'Airbyte', + 'Airbyte', + - 128, + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'test', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 'Airbyte', + 1, + concat( + lpad( + '0', + 262144, + '0' + ), + lpad( + '0', + 262144, + '0' + ), + lpad( + '0', + 262144, + '0' + ), + lpad( + '0', + 261568, + '0' + ) + ), + '{"a": 10, "b": 15}', + 'xs', + 'xs,s', + 0, + - 128, + 1, + 1 + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 2, + 0, + 32767, + 1, + 65535, + 8388607, + 1, + 2147483647, + 3428724653, + 1, + 9223372036854775807, + 10.5, + 0, + 1 / POWER( 10, 45 ), + 0.188, + 1700000.01, + '2021-01-01', + '2021-01-01', + '2013-09-05T10:10:02', + '2013-09-05T10:10:02', + '2021-00-00', + '23:59:59', + '23:59:59', + b'1000001', + '0', + '!"#$%&\'()*+, + -./:; + +<=>? \@ [ \ ] ^_\ ` { | } ~ ', 0xfffd, ' тест', ' Airbyte', ' ! "#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'тест', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{" fóo": " bär"}', 'm', 'm,xl', 1, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (3, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '0000-00-00', '00:00:00', '00:00:00', b'1000001', '50', '!" #$ %& \'()*+,-./:;<=>?\@[\]^_\`{|}~', +0 xfffd, +'тест', +'Airbyte', +'!"#$%&\'()*+, +-./:; + +<=>? \@ [ \ ] ^_\ ` { | } ~ ', 0xfffd, ' тест', ' Airbyte', ' Airbyte', 127, ' Airbyte', ' Airbyte', ' Airbyte', ' Airbyte', ' Airbyte', FROM_BASE64(' iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve / Vta93e985996qlu1k3vzUJiq / d + xS13LrVtX9zrf9 / 9 / 5 jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc / az / y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc + B + ENH / 1 Z79z / a + Oz / 8 KjX8IQBBKwQMPG22Q8e / Hrvuni + e6Z / MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso + ySc3CO1TF3MjL / cG + 62 oHJCazYYH77v + C0W8TQbJVqxbAbvfm9 / 1 nvv2kf / cO5ub + r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG + qUd9lx5utTh7zRHkxdHFb / wj / F4 / JsP3vizabwNRl0L + OsX9 / mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH / weciajcbG8NnmePw / xiK + px + 88 aUs6ngwAYuo0 / GRvn1l0YRhl / MfP7z48mdGps9 + KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg ++ QouXDyE5rYgZNOH5FJh3cTs9JfG5xY +/ fs / v7YVdT7qVsBkne2AobxUKVU + nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff / LcJzXU77DrVsD / 49 RvxZ9KPfFHo3Pj / bd0vyd0 + 8 CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk + TIQsyFFMNT4zNfzRx / Mm /+ E8 / v7URdTrqUsD7Satmpi7 + 2 sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V / bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv / bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX +/ mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno + 4E PNPZuGEhO / 1e kBk1pQLWxq + CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL /+ iK9dJzlmHOJRd5M4CIiShQ + OqdgyEgVU / dseq6wAXU26krAtr1PHE8Nvz + XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF + IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA /+ TFc + unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK / O5w + hRu7L4FQTFKQIhF + bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98 + cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch / nkPFTZxyNvwST / blodk + mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO / x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI + fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM + cQc3tX + a7A1C + n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw / esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2 + Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH + 1 pm1vf03Yn5zAwFVhUedLGZwHLonF3AM + dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8 + 1 KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd / 8 km1WMr1MhDCpCuuyQG0 + LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT +/ dWMokMM00lCyB5fpzhmApqgR / RIHcqkH1Oabb8fekxbITgTONLxvlno88 + 5 G6IB / qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU / ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR + B2VYAi97UsOmePWZ7tBHJOxCYrDkHB + OWyXm5pj + bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk + iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00 // kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj + ok / GqxoFU5KBBEGQErFF / D2We7zo5lZM + 1 lULdUidcudKJ6ACSDxpXKZZtTBqAsBL + Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI / 3 dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu + cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv + 2 PGp3l + XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv / GAg + AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G / wxbGjehLAvirvXvx8lwyjue + dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC + lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4 + 9 DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L / fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG / bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8 / Hf4R + 3 zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW + dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb +/ YiojVz + FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG / VWPEga4ewQ3 / 21 LeeVqF8NFkeb64oha0 / vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK + f3Ft8pI / KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp + V + MAThunavCml / l6YbQ7p1RNxpOk7VpmETalXoZtKll / L706git5rHwUTdfm7x / 6 Qk + j1LznzvZd1xRDF4TX8qd + m / kt5uaIiaHgyG / 4 NP98KBv74dMXvvZoj9n +/ Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v / Q9nFh8Adc27sVdLZthj / 8 Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop + AEp2lSoayOJy94k30SgtY + OA3tl87m57 + qwVpdoM2fR5NQRVGj48D + xXD5pFwNluRm + PrOnQ787GF0Gg4mt / 67 D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg / 0 bpmwx4ZwjFKuLL554u / xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9 + 5 dvP56bE / t0VjQ0tzBBm / QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9 / r92p / 4 je / sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT + S + 6 kTIdFgz64fNZGM5l8ZWszc1tv + ScR / QyLvcvnxySd39Znizwgvor3v + ysSIm5vGxLzf8 + dO / tXNycfqPBVG / qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3 /+ W2f + 4 zVvll7dtmqHqEk + mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql + FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL + QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf + 0 NAASH6rLJRJFOpE8rvJ + 51 oknBfMSEavqQSWdXn0sd / rUfn30w9Ppzx + 3 OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t / awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc / 23 cqyIgGPBrmhMiV + 3 rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu / AT1jweunAaJ1IZXlEpkm31SwGaPAZ / TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9 + ha0R3sIONEdjqIOxmUL + OFT + 9 S / eeazv7060Bv5t2s + hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1 + CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7 + 7 XIlNK4pnM3dnTdhKHFk65LsKDGxLc / Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti / dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd / GiQCDBC5QdNSgNjoZk0v91aJq1Eh + Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP / ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT / 6 VN8XVq0HgO7Gmkg0O4 + DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON / t + 2 jWMzO4eDkS + QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw + mAYxkSMuVMy3z6NF + De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0 + Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK / ip9d + BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB / ORLmallmsXwDNNwNJhepwsloD + 0 BT3 + DfCCJdv1p0zIHKFik0KviFk90eKdf0 // vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX + ariowLngZl8zBXIatzosmOr291OUH + ATwwj58Mj5YXz70MO4kBtjq / 2 rkbloi3XR / vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463 + gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45 / U00TByAMf8K / 4 lWtV5LqL6HSYkR + UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL / fj3moKQc / N + q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4 / hyRcexFh2BG2 + XrT6u1CgC9oYaEV / cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0 / t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN + yYqBhlV + O380b1lV0NkhFRzuvloS6UbDIq / ysIj5Od / sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh + flm1CHAJQiba / x / IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE / EmAr5MkEG1gX2kw + 8 jdwnqLeheI8xwi9VQkk5Uu + U0c2bEuiaGGZBrp0L5ZXc / DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V / 8 bbO7agvE0 + VpZdX + LXYviPKDE + 3 xJNBuS / rd5HixIDQzAYELtDnRWQQS + 4 Nqu5TPsOXbRGbLVlruAzYQpH55 + BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog / bFL + Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0 + Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X + bJR55y2m7 + 6 HtEFLoRlWkukPIJKCKsa + ilC1zmypQpBtAS6mK / nFKIsyTBUA8eP / l8YI / LDZOtTq6sU3S / huoZqWY / NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK / Bq7FB4b1eJ270CpptHeMWR / sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs / iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6 + XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb / gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96 +/ n9dj7Wy7hcxwmtdhcTfCIE / 6 FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT + BWOdn2JasJ + PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO + 8 tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46 / lxwKyy563E + j88LqT4EfFlpkiJJJQ / mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi + gYNVmzV4GbeM71VYt4Ngqp0n / u6Vz / tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR + ZOqscE2 + 7 vwLQ + RS9W6LfK2NS + DhfypzlWLdnS21 / AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6 + ms + NUBhcKTBOEXZ1lZ9H5SiC / EYNVuU8 / zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO / szIvkpDjAplJOdz / fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv + FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7 + LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0 / KQrVKKuqzXz1PvlVmybGiRYNh + N + 6 PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg / 3 RAQL0A7yFUaPWRnjvAAladxZTk3SDxB / LrskUX2dVvvjKJyOKrDQTRQG / 5 qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp + 927 ai9eHX0O8WCELWV1UDBRvCwT / fFDH1f + 9 MX73 / I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA + XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg / 4 MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV + fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp / P59PY2rgL9679MMaS57CYT / GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx + dWAQhUm7Gw + zLRet7aIsezwrXCQjWy915womKbs1veC + z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf / lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb + ipkl / USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ / vRH + 5 zGSo3WXbKw5xo2eWHqyYctedd5 + Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX / pBwdv2p7XjX9vSuX / 9 tdn3 / e // 83 Q +/ 8 g5rd / x6 / 5 rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy / QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t + YNpXtIn545oeIZU8hY + dJ7D7M56c2LxaG ++ n0p73PiQYj5umJI / jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF / 06E KNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT + HYEhx0j / 5 jYDM68cXT + 4 dOG6c + 8 RUYu5 / W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8 / I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX + xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7 +/ pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW / PCZYaiZJFpnkq0XRk / cniol34u3YFyWgDsVf17zaaN019CLzg8 / M / oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0 + P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb + M0 / Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz / sz74i / vv0LL5id / J6vP3qKWo // bGj6E / tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7 / mJr + 7E Co / LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9 + 8 iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0 // yk9G9 +/ 1 PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb / oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd +++ pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt /+ vSFb / eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs / hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7 + hwdf + Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd / DS5lVcPDuK1l36Cbz77f + O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I /+ I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV + cfc7R / q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw / 4 SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185 /+ QX / uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua + uYCzliJfsr9r9OLvNslxwVkYqlm + 31 Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl + Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0 + 0 k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw + YmY1xQsLNIEobS / Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl + QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT / E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl ++ ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy + yIaujc8HAj6jVKHZmClDDRLgQd8 + NUf5MJldgTXbzpr0Ywt48eJTiFNk + qtrfxMbmjagOdiGxWwBDSRYH5nf + cUyIU86WlpV3vSskC + sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO / 4 Q68nFD3KreTlmzjQ + 6 mtAM6FpLPgC + 2 Z + 35 ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH / YFtj6wVW41xYqvDoOBgIQ8 + SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL / x6D9KFPDra / IgEGEBSUYl + lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV + GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb + zuc1Rj0GyVBNMPWG2lNtnFXvlzq0 + mV / tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD + P2cH9yvn5Y +/ ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK + y6cimdJBd + Enc / r1I3 + Prz / 5 V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS + 5 FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY / sohewY9vS8kyxNgiNczETbfv8bBPyZp877C + V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg / KBR0NGmlGKIYfkf8ZpZncqlm8D + USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN + JEMB + CWVbn63rKbK / AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y / jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy + WRcRM + RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64 / ldxe / 8 WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB / K5XzkenUaBtWYKyYgH / z3n2pmzZf / 43 ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG / pOKxocfx3Cv / HefPfI3AjgKlXWFMzZegp6NEKGzggc + q6GqKrDM80KoSEJYnKMuBFj1 + kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y + uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66 / ets / naBI9 + kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa +/ H3vX7UWCzGeRomCxVoDlprsEHoga3 / zKdLV7uRK7pFL1L5aR / Q7Z4EwQm3cWINAjTumO6hTEU6hn + kOK / frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d + u6u0 + 8e TYRfyPA + dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC + fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh / Bng88Bbk5tvrxFI / SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd / DWEhVL53XiIQ5kmHxZKovMne4EjpWw + cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf / KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr / YBoOE6AtI6GrsI59Mgp / MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH / OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh + VlrSw4ms3MI + iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE + hKO + Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM + JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu / dWshZNvCaMmW62l6FsK + 6 m0U465G0gzXL5iBil + KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85 / qrr / 8 LuqRm0WS7eRZhE3gxNH0Eh4aO4ta + jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp / rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw + NwB9DRtKGIFxlvW0v8P9z7Itl / 9 i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf / fEeWzviODs9BLssAJ / gx9 + wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS / zxsamU5FBtdI08Gb / WIITw8 / gT + 45 bPIzJ + F6RfRVOqggChHpHuwKjwObPB + ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1 / xLl2jY / nP7tafO / NVan4 + OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8 + cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs / UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR + hc6ncZ7Ngq7 + hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6 / OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV + 95e uiGkwmyUxVigzwMFgIAz + xTwJFy3kCPTKU52bnkzg / fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1 + b0nCAPA5gQn51MIL78hUUgM + 1 ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi + KopjECoxfyq4rD + x4gKE + 579 IF + WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN / WRBBmz + RkcfvmbiOYUDE4sEbOkcT / Nz2MuAzpc0z + hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0 + 5 JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG / 9 KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh + zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a + atbcFyhQO0p4qxTMwu4SGbXq + 2 q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH / ty / 8510 HJh / 9 k5ge2zlVmBYiFD + spdToidNPUSagIUg +// yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr / Kvsm7b3xD7N0UZ / 4 x5 / se + HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL + 4 / x0vzf7wvuHpY / dMzSTW7Qq + Ezsa9qCN8teDB48gQ5N12 / YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE / rADZ39 + HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9 / MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0 / Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf + Nfv / Sfnp2fXwwJusrbEp9IvIrflO + ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo + xkSziPXGIxH2XJ8 / jd3fejb9 / 4 QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ / cJvERljG7f2r // iw8bN0Rbz144uHp6abI / lUmtn8 / P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U + wJzq6tU + ITBsustJcbu + diyp4MB1Ei / x / xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N / UiDAFWSdeW4RCFGdHtw / jh2ZhpRP4eek7 + PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV + TWdnfe + e9YnDzObqxK4sTcLv / f7f / iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29 / Q5tb + aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY / xglDTy5OEfxK2HXAj43b4xD0Ck6 / TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc / SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy / JAuNcuCntj / egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd + kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI + INBKhQft / YGoBOwtVNgmhDnac ++ M4PPrNSBXds1MXmlGxYNuuJIfCdy / g + wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy / hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF / znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP + wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP / Rn + H65i245ar1 + E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc + zv7wTrikui6iX + 2 bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6 + 159 Iej2NP5Phw + Mozh / AX0DgQxczbBV3k0t1HAuETZASFn / R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs / GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6 / ZovMG5SCdo + hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q + lZRFOLBoU + fHJoBhem96N1YQN + coow5gYiU85S2qRTXt3mw / rQWkwU55Ep60vb1l7 /+ e1rb17ACo + 6E XBZr + iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE / PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR + sCznXTsBd + un6YtNt3cT + Ojx / dXUD2yaghSDDtDnJX + uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7 + 3 xopAOvb7UxU16N7x9 + lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN + Ey0WkfptV + FbLz1Kk6Tn8J5rbvvrvXt / bwn4BFZ6XLECZgD + CEZU0gN7EzbpH / 7 yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6 / GU8OPw9 / E1hZXUIvMakEX99kFA0sEWqSIOPinM4 / iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO / xxC0D + y79 /+ 0 R9uWbflAt6aDg5XtAZLWExvkvRk / 8 nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6 / Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w / JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b + pDLzqEx1Gbvufb2xZZIeIMhCyznz + MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl + D6YrtKplsCLs1zl6VFE9kJ / ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc + 258 iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho / PJzr31Oo6V3JcsQJ2sepF9 / byh / 5 yd +/ 0 xaEtjOZjRXa24dY3mw7 + zP2tyyx42mq7eDX / j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA / 79 ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd + LdO9 + 8 p179rzl7RDrJw + mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I / MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6 + 4 D7btKplvmV55j1g1t83 + BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c + 9 P / cHwP21fc / XLv3v / n6X + WPjqW2KK / 7 lRPwKWJMPLW / lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4 + 8 P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3 / BA /+ qnb9r57h + 87 + YHRhx381Pgg / ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq + Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt / GvOIT68cEi603Dl / mIXqQMwVUwF2cWmI / 1 WCCDryFCppBFqVDEeza8G / vPfwu3r9uO5HSefKVO75OqGn / J / gxs2LVOeNXqDnfyNITb51f39b / W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6 + xBWi / GH4Jn + j6HXxox2 + hR + 3 HQ8d + DLNiuqv4RdeXe + dEtVjAFmuRtO2afAZ1bN2468t /+ fvf / Jzzpn / ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym + s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX / aRvDK2XUjw + WZMtdVrYM + IcLcniFdcsAD6t2Y + 2 XxhMXcfyFh3hR + lJm2uljadf8Ldz54wVoy5Gsatgr8I0 / bFb1WS + jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL + Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw + ASQ3mTjkCt11I + J5rsgeUFtraGZ5W2PY9m1GixuynFJ / spNa0MQBt0USaj1kbZrYIiwrIyH / Q2qwSr06R3KYBACSf5 / Aa / 0 kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6 / ds11Y7WG7K4 / lt0d0Cr7boTKhtqcOKcCl77I53I + qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0 / 0 chQbA9H7Z1bUGAyHunatItuHPLddiq + oDiduHhgnfMu + gWDdQiZM8X21gTGUBvYy + KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK + 1 UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4 / 3 ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4 / 8 hhBj2n + RHdDO7b1bCPivoCAFsB1 / duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I + NoE5G3Qh4V / OmIxv7r / 3 vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze + F +/ axDbTWKJI2FjW2a5G / 6 H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv / Yv3XrPzIOpk1I2A2XolVfQfX9u1 + dHGSMtB2xKzXud3VIvcLUqDZJyaOo + 54 SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw + Y9 + afTE3FN / eOTk6VWPPvPtPz4 / NX4XpU6GaZiq7WLFTFA + nx + vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz /+ b2z71f9x5ww0jn / vcl68otuh / NupKwG6dVv7Qoa + cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs / M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d + sLG / q0 / IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp + OYTX9rxnSe +/ vlwJL6wa / Puw0fPHNkyPvvi / SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb / z5JFzL2wrFlPN99 / 6 +// 1 fbd9qG587ZsNNl3fFgI + dOiQMpE91yko / sTeG / dmT4y92PDlb /+ f / y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9 / jk2Nxv7P0P / 3 XXxl1Lr776k0hGMBpv33k3W + VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII = '), ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, ' test', ' {"春江潮水连海平":"海上明月共潮生" }', ' m', ' m, +xl', 2, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (4, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, ' 2021 - 01 - 01 ', ' 2021 - 01 - 01 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2022 - 08 - 09 T10:17:16.161342 Z', ' 00:00:00 ', ' 00:00:00 ', b' 1000001 ', ' 70 ', ' ! "#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!" #$ %& \'()*+,-./:;<=>?\@[\]^_\`{|}~', +0 xfffd, +'тест', +'Airbyte', +'Airbyte', +127, +'Airbyte', +'Airbyte', +'Airbyte', +'Airbyte', +'Airbyte', +FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +0, +'test', +'{"春江潮水连海平": "海上明月共潮生"}', +'m', +'m,xl', +3, +127, +0, +0 + ); + +INSERT + INTO + test.TEST_DATASET + VALUES( + 5, + 0, + 32767, + 1, + 65535, + 8388607, + 1, + 2147483647, + 3428724653, + 1, + 9223372036854775807, + 10.5, + 0, + 10.5, + 0.188, + 1700000.01, + '2021-01-01', + '2021-01-01', + '2013-09-06T10:10:02', + '2013-09-06T10:10:02', + '2022-08-09T10:17:16.161342Z', + '00:00:00', + '00:00:00', + b'1000001', + '80', + '!"#$%&\'()*+, + -./:; -CREATE TABLE test.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bit,test_column_10 smallint,test_column_11 smallint zerofill,test_column_12 smallint unsigned,test_column_13 mediumint,test_column_14 mediumint zerofill,test_column_15 int,test_column_16 int unsigned,test_column_17 int zerofill,test_column_18 bigint,test_column_19 float,test_column_2 bit(1),test_column_20 double,test_column_21 decimal(10,3),test_column_22 decimal(19,2),test_column_23 date not null default '0000-00-00',test_column_24 date,test_column_25 datetime not null default now(),test_column_26 datetime,test_column_27 timestamp,test_column_28 time not null default '00:00:00',test_column_29 time,test_column_3 bit(7),test_column_30 year,test_column_31 VARCHAR(63),test_column_32 VARCHAR(63) character set utf16,test_column_33 VARCHAR(63) character set cp1251,test_column_34 VARCHAR(7) character set binary,test_column_35 CHAR(63),test_column_36 CHAR(63) character set utf16,test_column_37 CHAR(63) character set cp1251,test_column_38 CHAR(7) character set binary,test_column_39 BLOB,test_column_4 tinyint,test_column_40 TINYBLOB,test_column_5 tinyint(1),test_column_51 json,test_column_52 ENUM('xs', 's', 'm', 'l', 'xl'),test_column_53 SET('xs', 's', 'm', 'l', 'xl'),test_column_6 tinyint(1) unsigned,test_column_7 tinyint(2),test_column_8 BOOL,test_column_9 BOOLEAN ); +<=>? \@ [ \ ] ^_\ ` { | } ~ ', 0xfffd, ' тест', ' Airbyte', ' ! "#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{" 春江潮水连海平": " 海上明月共潮生"}', 'm', 'm,xl', 3, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (6, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', '00:00:00', b'1000001', '99', '!" #$ %& \'()*+,-./:;<=>?\@[\]^_\`{|}~', +0 xfffd, +'тест', +'Airbyte', +'!"#$%&\'()*+, +-./:; -INSERT INTO test.TEST_DATASET VALUES (1, 1, -32768, 1, 0, -8388608, 1, -2147483648, 3428724653, 1, 9223372036854775807, 10.5, 1, power(10, 308), 0.188, 1700000.01, '1999-01-08', '1999-01-08', '2005-10-10 23:22:21', '2005-10-10 23:22:21', '2021-01-00', '-22:59:59', '-22:59:59', b'1000001', '1997', 'Airbyte', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 0xfffd, 'тест', 'Airbyte', 'Airbyte', -128, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'test', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 1, concat(lpad('0', 262144, '0'),lpad('0', 262144, '0'),lpad('0', 262144, '0'),lpad('0', 261568, '0')), '{"a": 10, "b": 15}', 'xs', 'xs,s', 0, -128, 1, 1); -INSERT INTO test.TEST_DATASET VALUES (2, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 1/power(10, 45), 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-05T10:10:02', '2013-09-05T10:10:02', '2021-00-00', '23:59:59', '23:59:59', b'1000001', '0', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'тест', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{"fóo": "bär"}', 'm', 'm,xl', 1, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (3, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '0000-00-00', '00:00:00', '00:00:00', b'1000001', '50', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{"春江潮水连海平": "海上明月共潮生"}', 'm', 'm,xl', 2, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (4, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', '00:00:00', b'1000001', '70', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{"春江潮水连海平": "海上明月共潮生"}', 'm', 'm,xl', 3, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (5, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', '00:00:00', b'1000001', '80', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{"春江潮水连海平": "海上明月共潮生"}', 'm', 'm,xl', 3, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (6, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', '00:00:00', b'1000001', '99', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{"春江潮水连海平": "海上明月共潮生"}', 'm', 'm,xl', 3, 127, 0, 0); -INSERT INTO test.TEST_DATASET VALUES (7, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, '2021-01-01', '2021-01-01', '2013-09-06T10:10:02', '2013-09-06T10:10:02', '2022-08-09T10:17:16.161342Z', '00:00:00', '00:00:00', b'1000001', '99', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', 'Airbyte', 127, 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', 'Airbyte', FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, 'test', '{"春江潮水连海平": "海上明月共潮生"}', 'm', 'm,xl', 3, 127, 0, 0); +<=>? \@ [ \ ] ^_\ ` { | } ~ ', 0xfffd, ' тест', ' Airbyte', ' Airbyte', 127, ' Airbyte', ' Airbyte', ' Airbyte', ' Airbyte', ' Airbyte', FROM_BASE64(' iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve / Vta93e985996qlu1k3vzUJiq / d + xS13LrVtX9zrf9 / 9 / 5 jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc / az / y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc + B + ENH / 1 Z79z / a + Oz / 8 KjX8IQBBKwQMPG22Q8e / Hrvuni + e6Z / MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso + ySc3CO1TF3MjL / cG + 62 oHJCazYYH77v + C0W8TQbJVqxbAbvfm9 / 1 nvv2kf / cO5ub + r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG + qUd9lx5utTh7zRHkxdHFb / wj / F4 / JsP3vizabwNRl0L + OsX9 / mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH / weciajcbG8NnmePw / xiK + px + 88 aUs6ngwAYuo0 / GRvn1l0YRhl / MfP7z48mdGps9 + KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg ++ QouXDyE5rYgZNOH5FJh3cTs9JfG5xY +/ fs / v7YVdT7qVsBkne2AobxUKVU + nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff / LcJzXU77DrVsD / 49 RvxZ9KPfFHo3Pj / bd0vyd0 + 8 CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk + TIQsyFFMNT4zNfzRx / Mm /+ E8 / v7URdTrqUsD7Satmpi7 + 2 sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V / bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv / bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX +/ mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno + 4E PNPZuGEhO / 1e kBk1pQLWxq + CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL /+ iK9dJzlmHOJRd5M4CIiShQ + OqdgyEgVU / dseq6wAXU26krAtr1PHE8Nvz + XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF + IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA /+ TFc + unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK / O5w + hRu7L4FQTFKQIhF + bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98 + cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch / nkPFTZxyNvwST / blodk + mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO / x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI + fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM + cQc3tX + a7A1C + n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw / esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2 + Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH + 1 pm1vf03Yn5zAwFVhUedLGZwHLonF3AM + dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8 + 1 KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd / 8 km1WMr1MhDCpCuuyQG0 + LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT +/ dWMokMM00lCyB5fpzhmApqgR / RIHcqkH1Oabb8fekxbITgTONLxvlno88 + 5 G6IB / qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU / ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR + B2VYAi97UsOmePWZ7tBHJOxCYrDkHB + OWyXm5pj + bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk + iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00 // kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj + ok / GqxoFU5KBBEGQErFF / D2We7zo5lZM + 1 lULdUidcudKJ6ACSDxpXKZZtTBqAsBL + Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI / 3 dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu + cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv + 2 PGp3l + XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv / GAg + AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G / wxbGjehLAvirvXvx8lwyjue + dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC + lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4 + 9 DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L / fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG / bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8 / Hf4R + 3 zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW + dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb +/ YiojVz + FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG / VWPEga4ewQ3 / 21 LeeVqF8NFkeb64oha0 / vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK + f3Ft8pI / KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp + V + MAThunavCml / l6YbQ7p1RNxpOk7VpmETalXoZtKll / L706git5rHwUTdfm7x / 6 Qk + j1LznzvZd1xRDF4TX8qd + m / kt5uaIiaHgyG / 4 NP98KBv74dMXvvZoj9n +/ Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v / Q9nFh8Adc27sVdLZthj / 8 Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop + AEp2lSoayOJy94k30SgtY + OA3tl87m57 + qwVpdoM2fR5NQRVGj48D + xXD5pFwNluRm + PrOnQ787GF0Gg4mt / 67 D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg / 0 bpmwx4ZwjFKuLL554u / xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9 + 5 dvP56bE / t0VjQ0tzBBm / QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9 / r92p / 4 je / sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT + S + 6 kTIdFgz64fNZGM5l8ZWszc1tv + ScR / QyLvcvnxySd39Znizwgvor3v + ysSIm5vGxLzf8 + dO / tXNycfqPBVG / qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3 /+ W2f + 4 zVvll7dtmqHqEk + mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql + FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL + QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf + 0 NAASH6rLJRJFOpE8rvJ + 51 oknBfMSEavqQSWdXn0sd / rUfn30w9Ppzx + 3 OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t / awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc / 23 cqyIgGPBrmhMiV + 3 rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu / AT1jweunAaJ1IZXlEpkm31SwGaPAZ / TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9 + ha0R3sIONEdjqIOxmUL + OFT + 9 S / eeazv7060Bv5t2s + hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1 + CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7 + 7 XIlNK4pnM3dnTdhKHFk65LsKDGxLc / Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti / dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd / GiQCDBC5QdNSgNjoZk0v91aJq1Eh + Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP / ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT / 6 VN8XVq0HgO7Gmkg0O4 + DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON / t + 2 jWMzO4eDkS + QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw + mAYxkSMuVMy3z6NF + De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0 + Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK / ip9d + BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB / ORLmallmsXwDNNwNJhepwsloD + 0 BT3 + DfCCJdv1p0zIHKFik0KviFk90eKdf0 // vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX + ariowLngZl8zBXIatzosmOr291OUH + ATwwj58Mj5YXz70MO4kBtjq / 2 rkbloi3XR / vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463 + gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45 / U00TByAMf8K / 4 lWtV5LqL6HSYkR + UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL / fj3moKQc / N + q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4 / hyRcexFh2BG2 + XrT6u1CgC9oYaEV / cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0 / t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN + yYqBhlV + O380b1lV0NkhFRzuvloS6UbDIq / ysIj5Od / sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh + flm1CHAJQiba / x / IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE / EmAr5MkEG1gX2kw + 8 jdwnqLeheI8xwi9VQkk5Uu + U0c2bEuiaGGZBrp0L5ZXc / DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V / 8 bbO7agvE0 + VpZdX + LXYviPKDE + 3 xJNBuS / rd5HixIDQzAYELtDnRWQQS + 4 Nqu5TPsOXbRGbLVlruAzYQpH55 + BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog / bFL + Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0 + Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X + bJR55y2m7 + 6 HtEFLoRlWkukPIJKCKsa + ilC1zmypQpBtAS6mK / nFKIsyTBUA8eP / l8YI / LDZOtTq6sU3S / huoZqWY / NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK / Bq7FB4b1eJ270CpptHeMWR / sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs / iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6 + XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb / gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96 +/ n9dj7Wy7hcxwmtdhcTfCIE / 6 FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT + BWOdn2JasJ + PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO + 8 tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46 / lxwKyy563E + j88LqT4EfFlpkiJJJQ / mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi + gYNVmzV4GbeM71VYt4Ngqp0n / u6Vz / tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR + ZOqscE2 + 7 vwLQ + RS9W6LfK2NS + DhfypzlWLdnS21 / AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6 + ms + NUBhcKTBOEXZ1lZ9H5SiC / EYNVuU8 / zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO / szIvkpDjAplJOdz / fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv + FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7 + LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0 / KQrVKKuqzXz1PvlVmybGiRYNh + N + 6 PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg / 3 RAQL0A7yFUaPWRnjvAAladxZTk3SDxB / LrskUX2dVvvjKJyOKrDQTRQG / 5 qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp + 927 ai9eHX0O8WCELWV1UDBRvCwT / fFDH1f + 9 MX73 / I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA + XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg / 4 MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV + fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp / P59PY2rgL9679MMaS57CYT / GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx + dWAQhUm7Gw + zLRet7aIsezwrXCQjWy915womKbs1veC + z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf / lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb + ipkl / USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ / vRH + 5 zGSo3WXbKw5xo2eWHqyYctedd5 + Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX / pBwdv2p7XjX9vSuX / 9 tdn3 / e // 83 Q +/ 8 g5rd / x6 / 5 rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy / QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t + YNpXtIn545oeIZU8hY + dJ7D7M56c2LxaG ++ n0p73PiQYj5umJI / jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF / 06E KNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT + HYEhx0j / 5 jYDM68cXT + 4 dOG6c + 8 RUYu5 / W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8 / I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX + xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7 +/ pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW / PCZYaiZJFpnkq0XRk / cniol34u3YFyWgDsVf17zaaN019CLzg8 / M / oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0 + P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb + M0 / Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz / sz74i / vv0LL5id / J6vP3qKWo // bGj6E / tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7 / mJr + 7E Co / LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9 + 8 iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0 // yk9G9 +/ 1 PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb / oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd +++ pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt /+ vSFb / eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs / hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7 + hwdf + Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd / DS5lVcPDuK1l36Cbz77f + O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I /+ I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV + cfc7R / q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw / 4 SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185 /+ QX / uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua + uYCzliJfsr9r9OLvNslxwVkYqlm + 31 Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl + Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0 + 0 k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw + YmY1xQsLNIEobS / Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl + QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT / E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl ++ ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy + yIaujc8HAj6jVKHZmClDDRLgQd8 + NUf5MJldgTXbzpr0Ywt48eJTiFNk + qtrfxMbmjagOdiGxWwBDSRYH5nf + cUyIU86WlpV3vSskC + sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO / 4 Q68nFD3KreTlmzjQ + 6 mtAM6FpLPgC + 2 Z + 35 ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH / YFtj6wVW41xYqvDoOBgIQ8 + SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL / x6D9KFPDra / IgEGEBSUYl + lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV + GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb + zuc1Rj0GyVBNMPWG2lNtnFXvlzq0 + mV / tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD + P2cH9yvn5Y +/ ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK + y6cimdJBd + Enc / r1I3 + Prz / 5 V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS + 5 FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY / sohewY9vS8kyxNgiNczETbfv8bBPyZp877C + V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg / KBR0NGmlGKIYfkf8ZpZncqlm8D + USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN + JEMB + CWVbn63rKbK / AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y / jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy + WRcRM + RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64 / ldxe / 8 WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB / K5XzkenUaBtWYKyYgH / z3n2pmzZf / 43 ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG / pOKxocfx3Cv / HefPfI3AjgKlXWFMzZegp6NEKGzggc + q6GqKrDM80KoSEJYnKMuBFj1 + kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y + uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66 / ets / naBI9 + kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa +/ H3vX7UWCzGeRomCxVoDlprsEHoga3 / zKdLV7uRK7pFL1L5aR / Q7Z4EwQm3cWINAjTumO6hTEU6hn + kOK / frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d + u6u0 + 8e TYRfyPA + dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC + fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh / Bng88Bbk5tvrxFI / SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd / DWEhVL53XiIQ5kmHxZKovMne4EjpWw + cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf / KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr / YBoOE6AtI6GrsI59Mgp / MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH / OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh + VlrSw4ms3MI + iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE + hKO + Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM + JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu / dWshZNvCaMmW62l6FsK + 6 m0U465G0gzXL5iBil + KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85 / qrr / 8 LuqRm0WS7eRZhE3gxNH0Eh4aO4ta + jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp / rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw + NwB9DRtKGIFxlvW0v8P9z7Itl / 9 i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf / fEeWzviODs9BLssAJ / gx9 + wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS / zxsamU5FBtdI08Gb / WIITw8 / gT + 45 bPIzJ + F6RfRVOqggChHpHuwKjwObPB + ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1 / xLl2jY / nP7tafO / NVan4 + OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8 + cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs / UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR + hc6ncZ7Ngq7 + hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6 / OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV + 95e uiGkwmyUxVigzwMFgIAz + xTwJFy3kCPTKU52bnkzg / fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1 + b0nCAPA5gQn51MIL78hUUgM + 1 ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi + KopjECoxfyq4rD + x4gKE + 579 IF + WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN / WRBBmz + RkcfvmbiOYUDE4sEbOkcT / Nz2MuAzpc0z + hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0 + 5 JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG / 9 KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh + zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a + atbcFyhQO0p4qxTMwu4SGbXq + 2 q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH / ty / 8510 HJh / 9 k5ge2zlVmBYiFD + spdToidNPUSagIUg +// yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr / Kvsm7b3xD7N0UZ / 4 x5 / se + HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL + 4 / x0vzf7wvuHpY / dMzSTW7Qq + Ezsa9qCN8teDB48gQ5N12 / YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE / rADZ39 + HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9 / MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0 / Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf + Nfv / Sfnp2fXwwJusrbEp9IvIrflO + ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo + xkSziPXGIxH2XJ8 / jd3fejb9 / 4 QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ / cJvERljG7f2r // iw8bN0Rbz144uHp6abI / lUmtn8 / P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U + wJzq6tU + ITBsustJcbu + diyp4MB1Ei / x / xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N / UiDAFWSdeW4RCFGdHtw / jh2ZhpRP4eek7 + PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV + TWdnfe + e9YnDzObqxK4sTcLv / f7f / iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29 / Q5tb + aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY / xglDTy5OEfxK2HXAj43b4xD0Ck6 / TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc / SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy / JAuNcuCntj / egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd + kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI + INBKhQft / YGoBOwtVNgmhDnac ++ M4PPrNSBXds1MXmlGxYNuuJIfCdy / g + wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy / hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF / znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP + wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP / Rn + H65i245ar1 + E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc + zv7wTrikui6iX + 2 bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6 + 159 Iej2NP5Phw + Mozh / AX0DgQxczbBV3k0t1HAuETZASFn / R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs / GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6 / ZovMG5SCdo + hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q + lZRFOLBoU + fHJoBhem96N1YQN + coow5gYiU85S2qRTXt3mw / rQWkwU55Ep60vb1l7 /+ e1rb17ACo + 6E XBZr + iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE / PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR + sCznXTsBd + un6YtNt3cT + Ojx / dXUD2yaghSDDtDnJX + uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7 + 3 xopAOvb7UxU16N7x9 + lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN + Ey0WkfptV + FbLz1Kk6Tn8J5rbvvrvXt / bwn4BFZ6XLECZgD + CEZU0gN7EzbpH / 7 yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6 / GU8OPw9 / E1hZXUIvMakEX99kFA0sEWqSIOPinM4 / iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO / xxC0D + y79 /+ 0 R9uWbflAt6aDg5XtAZLWExvkvRk / 8 nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6 / Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w / JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b + pDLzqEx1Gbvufb2xZZIeIMhCyznz + MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl + D6YrtKplsCLs1zl6VFE9kJ / ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc + 258 iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho / PJzr31Oo6V3JcsQJ2sepF9 / byh / 5 yd +/ 0 xaEtjOZjRXa24dY3mw7 + zP2tyyx42mq7eDX / j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA / 79 ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd + LdO9 + 8 p179rzl7RDrJw + mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I / MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6 + 4 D7btKplvmV55j1g1t83 + BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c + 9 P / cHwP21fc / XLv3v / n6X + WPjqW2KK / 7 lRPwKWJMPLW / lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4 + 8 P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3 / BA /+ qnb9r57h + 87 + YHRhx381Pgg / ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq + Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt / GvOIT68cEi603Dl / mIXqQMwVUwF2cWmI / 1 WCCDryFCppBFqVDEeza8G / vPfwu3r9uO5HSefKVO75OqGn / J / gxs2LVOeNXqDnfyNITb51f39b / W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6 + xBWi / GH4Jn + j6HXxox2 + hR + 3 HQ8d + DLNiuqv4RdeXe + dEtVjAFmuRtO2afAZ1bN2468t /+ fvf / Jzzpn / ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym + s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX / aRvDK2XUjw + WZMtdVrYM + IcLcniFdcsAD6t2Y + 2 XxhMXcfyFh3hR + lJm2uljadf8Ldz54wVoy5Gsatgr8I0 / bFb1WS + jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL + Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw + ASQ3mTjkCt11I + J5rsgeUFtraGZ5W2PY9m1GixuynFJ / spNa0MQBt0USaj1kbZrYIiwrIyH / Q2qwSr06R3KYBACSf5 / Aa / 0 kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6 / ds11Y7WG7K4 / lt0d0Cr7boTKhtqcOKcCl77I53I + qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0 / 0 chQbA9H7Z1bUGAyHunatItuHPLddiq + oDiduHhgnfMu + gWDdQiZM8X21gTGUBvYy + KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK + 1 UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4 / 3 ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4 / 8 hhBj2n + RHdDO7b1bCPivoCAFsB1 / duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I + NoE5G3Qh4V / OmIxv7r / 3 vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze + F +/ axDbTWKJI2FjW2a5G / 6 H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv / Yv3XrPzIOpk1I2A2XolVfQfX9u1 + dHGSMtB2xKzXud3VIvcLUqDZJyaOo + 54 SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw + Y9 + afTE3FN / eOTk6VWPPvPtPz4 / NX4XpU6GaZiq7WLFTFA + nx + vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz /+ b2z71f9x5ww0jn / vcl68otuh / NupKwG6dVv7Qoa + cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs / M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d + sLG / q0 / IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp + OYTX9rxnSe +/ vlwJL6wa / Puw0fPHNkyPvvi / SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb / z5JFzL2wrFlPN99 / 6 +// 1 fbd9qG587ZsNNl3fFgI + dOiQMpE91yko / sTeG / dmT4y92PDlb /+ f / y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9 / jk2Nxv7P0P / 3 XXxl1Lr776k0hGMBpv33k3W + VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII = '), ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', ' enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', 0, ' test', ' {"春江潮水连海平":"海上明月共潮生" }', ' m', ' m, +xl', 3, 127, 0, 0); +INSERT INTO test.TEST_DATASET VALUES (7, 0, 32767, 1, 65535, 8388607, 1, 2147483647, 3428724653, 1, 9223372036854775807, 10.5, 0, 10.5, 0.188, 1700000.01, ' 2021 - 01 - 01 ', ' 2021 - 01 - 01 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2013 - 09 - 06 T10:10:02 ', ' 2022 - 08 - 09 T10:17:16.161342 Z', ' 00:00:00 ', ' 00:00:00 ', b' 1000001 ', ' 99 ', ' ! "#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 0xfffd, 'тест', 'Airbyte', '!" #$ %& \'()*+,-./:;<=>?\@[\]^_\`{|}~', +0 xfffd, +'тест', +'Airbyte', +'Airbyte', +127, +'Airbyte', +'Airbyte', +'Airbyte', +'Airbyte', +'Airbyte', +FROM_BASE64('iVBORw0KGgoAAAANSUhEUgAAAHgAAAB4CAYAAAA5ZDbSAAAACXBIWXMAACE4AAAhOAFFljFgAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAEIPSURBVHgB7b0JnFxXeSf6v3vtVV29791q7bIkS8iy5U3IC2DHNnIAJ2MYQl4AZ4HJQB6TmcybQQz5ZfHkDXmQkEcCgQDBwWb1IrDBu4wX7Vu31GpJve/Vta93e985996qlu1k3vzUJiq/d+xS13LrVtX9zrf9/9/5jmDbtgTAQl2Mzwn83zc8vfzBPn777Gc/az/y8qd9iaAULvoSoQYEVdGW1FLFtCuo6AFR0EoFcWkhX1xoX6uK5YnSr5fE4v7PXPPI3Oc+B+ENH/1Z79z/a+Oz/8KjX8IQBBKwQMPG22Q8e/Hrvuni+e6Z/MjWRD6xOVcsblRDaJMEKeRXA2pZL6NoFso+ySc3CO1TF3MjL/cG+62oHJCazYYH77v+C0W8TQbJVqxbAbvfm9/1nvv2kf/cO5ub+r2hpeO3SIF8d3bJipiirQX8klAsmYhFJdJdoFQxIKoG+qUd9lx5utTh7zRHkxdHFb/wj/F4/JsP3vizabwNRl0L+OsX9/mkhH63rAVeKKbThaHcqVv8duDP5tPT61PSFIKKhsSChfYOGZm0iWIZdF9EuWwjlbIQiZsIlltwXWQt5spFfH/weciajcbG8NnmePw/xiK+px+88aUs6ngwAYuo0/GRvn1l0YRhl/MfP7z48mdGps9+KSI1rNfJw8ZCKpKLJjRVhKaJKOVtiORWRUGERDejIsCqkDbLSxg++QouXDyE5rYgZNOH5FJh3cTs9JfG5xY+/fs/v7YVdT7qVsBkne2AobxUKVU+nEjPf6ot3NbdqDXBktIwSFuNiohgQIRtAqYuQBIFLmSZ5rRgC6gUBWhBCSdVHRdME9GwhEhMhmgpqJTQNbew9AcLudQff/LcJzXU77DrVsD/49RvxZ9KPfFHo3Pj/bd0vyd0+8CdmEmPQdJ0FAqOIMn3ks8VINB9SWIazAQtck02dHqORB5t89F7NHosoKlJhk+TIQsyFFMNT4zNfzRx/Mm/+E8/v7URdTrqUsD7Satmpi7+2sTs5Icmkxel7koRPUYOaWMMgqTCIuExASuyCL3CfqTIBUui5pos0V/bEkCKS5OAjiWB6yWaFAppcViGYJL2B2Vogg9zi6n7p3ILv/bFc3fUpSbLqMPx2tjYuuml2Y9WimK8pCxg7MgzSPiDKHUWYCsKLNMm4Qpca5kQBf4fuGlmkbdIQrbovmGQgDUJqmLRRLDofQLCIYkHYYJoIxwRkMyUGtLZ1EeHRuQDdIoTqLNRdxr88MMfkJbKyX+/mExu7wy3CLYUwvdm0vju2DiskEzaS8IzBKgq99P0GJegFlzQgiNsg46TScs1P2k03SffyzVaIW1nViBA51MFhVKs4rZ8qfzpD9gfkFBno+4EPNPZuGEhO/1ekBk1pQLWxq+CNtCI8NY4FEWGadiUHgASaTBlgKSpcCXK3i04Nxv8GJM0XaL/+iK9dJzlmHOJRd5M4CIiShQ+OqdgyEgVU/dseq6wAXU26krAtr1PHE8Nvz+XL8RVUUPJKqKk69i16h0IR8kcWxKa1Q4IFvlV2RUqEzBJ2WaStmuoiM3vCCgZJTSHmtAR6kWhrPO4U1UF+IQgOoJ9kFUToqnSxKk0jGem3s8sCOpo1JWA/+TFc+unc6N7YSiUz5IQpABOz59El7oKm1p2kHBU9Ac3U0RsgWX4lOg7grRdbSaNtZiwXZttsRSK/O5w+hRu7L4FQTFKQIhF+bOAgdha0mgy1z5HyyWTgJP8wr0nmzLrUUejbgS8b98+cWppYpuulzuhUypDAZRBwtINHacWjmKTdguujl8Pnx1BhaBIFmBxSXKttbmwmW9m0bMjdGcCEAGBVDmBUjmHa9puQJne61f9GGhch/nkPFTZxyNvwST/blodk+mFbey7oE5G3XzR7l8vBHPl9LWkdTEWGbNImEfIZI5Hc8MYnziNO/x90IqTBG5QFOz6WNs1ylybBfD3sKcsemzQcRbxaJrow7GZw9gavwZhqREdsQ60BtuRyCTJUkg83bINiVIOMVYs5q7133YkiDoZdZMmnZsYaa6US9vMsihblN4ITMAWS38oy6WI+fjCY4icUTFMM8Ai6TJBsuNIhnRjYha4MJnkTQqoVMpxo4qfhJyFT1YxsjiM+cQc3tX+a7A1C+n5DArFHILwQaUMuJRh75FlA8bVk5lcC52pLnDqutHgXCbZXjb1XstyzC4LjG2XxVboQTHmw/esLA6WS8uDZRK0ibZQuxNvuT7ZMA2sb7qK3qdApwnBoA9m6l8ZewUdUhu2aHHMpYd5rsysgMzUgO6LFICZhtGXz6XbUSejbgRs2+Y6EkKzaYluLuv6UvoFpMykxZS79kYgNwYpRRId6dKtTMltR6iHTG4XSLdJmw00BVqxunE9JrMTXLgEZJGflXBudhCzg99A6eLjkDPDzEzwScRSLvYZLFc2TLPJFsy6CbTqRsBls7LVNHSfTZmMIpJKSY6PZZrMI2YBDgxJwZWkOpPAoNdFst8XE6PY0bKH+1pm1vf03Yn5zAwFVhUedLGZwHLonF3AM+dO4NDRQxim1yVJ4lrMcWz6DF1nGm35KqaxBXUy6kPAJEfdNPqYUrK8tyfWSxBjkAdKXLKuNnMTDF6nwn8Ze45FyfPZWQSFCK5qvh5doTVYE9mIwZlBsFya5cfMk8s0YRSfgsG4D8+1KFiKyPziML8tkAYrCglXdxLpimH0A29S1nMFjroQ8Bd/8km1WMr1MhDCpCuuyQG0+LsoWrb5ZWYC5RbZFbJ36W0X2SgSRHVubhC7YjtxT+/dWMokMM00lCyB5fpzhmApqgR/RIHcqkH1Oabb8fekxbITgTONLxvlno88+5G6IB/qQ4NDoUjBKLUw2SmChmRhAX2BDRQQy9w3Mg00bU/ADqEgeCgWDYmkc3HmWRhDj6Bp9hiOjnyPgiudR+B2VYAi97UsOmePWZ7tBHJOxCYrDkHB+OWyXm5pj+bDqINRF2mSikozYUlRJpKyUUHOSCKuNhEs2YnpwggJxVcNqrjGcWBDqGoz86Epo4zXBk+iOTyMEQq0JGaTBds9hgRMAhQMoYpuitIy00//kTvmxzKSgiLpSChXaabDFnCFj7oQ8FIl0RhVor5NTVuRD5tYKJ9HhXDoTS3bcHFkiI7wuWiVIwzBkQ3XalYQzIj+ok/GqxoFU5KBBEGQErFF/D2We7zo5lZM+1lULdUidcudKJ6ACSDxpXKZZtTBqAsBL+Tn4xO5cbFIZMA1TTfhzoE70YgZ9GpLeIHSJoNBjlYtL7axrIZQcLRSIh8rxxT6xaStCzoHQNiRpu0d5ppkOKZekpY9ZjZBqPl327LEpXKmAXUw6kLAAdUfYFUYS6UkHh16FBdmT2EHZbVILRBAYTooFWpWmmue4DJGEr0qOVEX11HRI/3dZ5gph1DVevYUO5fInbtHPzkns90PsemOYhNIXQejLoIsyxRjzExKFnGzRAnO2GN4sjSJR5ZyKBHMJLjpkWXVTK5DIjkskjMurQx2BOpMBCZiLtDqrOAYB2pxms35YnZjE8Qg9Ktimqu+cujjCq7wURcaXCBzKAquFgqsWkOBSTpdIdpQEfQqW2QvW4DD7jv+2PGp3l+XPXQKAfhxbu4MR6jV2eHhnQDHrttCHUgpCRSsRbRF2jHQMBCbyQ5f8fXk9aDBQkUwNEdKIv/GAg+AGU8LBl9hWeL7etlUzS7XVNHRbp5Y0R0H1nYnjlQ9hYedVKtAdIq6G/wxbGjehLAvirvXvx8lwyjue+dzJq7wseICvmhf9Nk1u3jZ42H7YdHUjbBnUr2aKsffuigF4Aq3pn38K7D6ZyIRdNtwyu74oRbPZdc3beC+lP3HymqdhFpAdY2Hd0dwrMBsbg6bWrfhroH7UdEtHJl9LbLvuXde8dUdKy7gzNx4+9DMgb2zs7MrwpkOPvKIVDFLmuBKlftVuKAGP6L2L/fBtlDVPpHUcnXDGvRFVpGiqyBRM7IAmWIWzYFObG/bRdBnCc7U8KJm582iW8fF4zRCvOZz82jLDOKGiB8/Hf4R+3zf6YX5K94CrvgX1FuDk5VysUUXFt9DmqwNzTzbh8sY8VCbQEGN5GivUI2VLGuZOYaHYqEW+dLxBmnuTHYGcV8jbu29Czd23IMmfw8v1TkzPYRb+/YiojVz+FN4U5MuVIMxdufw0PN44vi3sFCcpbRMltpClRWzVG/VWPEga4ewQ3/21LeeVqF8NFkeb64oha0/vLDv2YpVqJALtSq6oKtiMKWa2tixdcXZfcK+f3Ft8pI/KxAEfUm0aguUpkgqYmoIGXuharMFr0RHsFxEi3Bos4ij0wcxlj6DXe034v6BTxAmPYXB6aPoJvp+V+MAThunavCml/l6YbQ7p1RNxpOk7VpmETalXoZtKll/L706git5rHwUTdfm7x/6Qk+j1LznzvZd1xRDF4TX8qd+m/kt5uaIiaHgyG/4NP98KBv74dMXvvZoj9n+/Jo1d5bf7HQRLU9MEs0JTvoyzJnyXpJ4WI0gGG7C4tIkCSPgfHS12Aq1AmgeDkhIG2k8v/Q9nFh8Adc27sVdLZthj/8Y15dnMEHRVp4Fa2xiCF7k7dw80ISHckEJop+AEp2lSoayOJy94k30SgtY+OA3tl87m57+qwVpdoM2fR5NQRVGj48D+xXD5pFwNluRm+PrOnQ787GF0Gg4mt/67D93wsxiUTBtQ2SXuSXYDMUoUgS7QLRfFLIVgkcecUiRf4NloIbgEg/0bpmwx4ZwjFKuLL554u/xC6IP90Z0zJEACx0qLllnadeAjyoAIgq10mqR5eamhM4rfxnxigr4o9+5dvP56bE/t0VjQ0tzBBm/QsSAjLCkcD5VrlgMtkdFLWGV0AJVGVAPzO9/r92p/4je/sM3Oye7hJZg8fA2S8K5rucmjFpH4FPCEMqaK1yhFk3Yy6BKHmQT+S+6kTIdFgz64fNZGM5l8ZWszc1tv+ScR/QyLvcvnxySd39Znizwgvor3v+ysSIm5vGxLzf8+dO/tXNycfqPBVG/qa2hCbGoD6YgIxLT3HSGKAEf4cYkq4bGIE6ffhzHzvwY87Pp6NHFn3/+W2f+4zVvll7dtmqHqEk+mb2SLmfw3MhzuLX5HtxMOSmMPBfgcuzZ9vJX70zM5FYF5kCQql+FP6DBChH3G9Uga5JTICDWrghHubx0a1ng5f0WBn5XUtYVL+QVEfAaeQNRqdrnTal4y0DLgNAajfGVBSZf+0NAASH6rLJRJFOpE8rvJ+51oknBfMSEavqQSWdXn0sd/rUfn30w9Ppzx+3OoCr5Gi2KdFmpDsOjXzz9AwgXfoEyYdIQnfjLsm039629t/awJgfmV2Ui9hW26IwYJo1ubFVhNUpnuDbDmimt6gp3UWJlOni1wIrvZLRH2vgicmsFc/23cqyIgGPBrmhMiV+3rm1tcFvbtcjbCU7GM7JcIBPIAizdsHiNlE73TRJyIKJBIsiRr8DPUXBaTL77TPrkqtef26eEpJgck1kNM3sfu/AT1jweunAaJ1IZXlEpkm31SwGaPAZ/TxWd4jehWoXJgQxWX8VuLsHPiH5Pwz1wg93ViXdeHVuPgBTk582Xc9i9+ha0R3sIONEdjqIOxmUL+OFT+9S/eeazv7060Bv5t2s+hemlObqA7AI59S5suUhFN2GQgCvkg8t04xE1aTOLqH1+CeW8gEwhvcGolO72znt89sngoUOHlFCwRWj2tQpMg5kHZ2AGNAWzTUGUaZLwGilTR2ewGz0RuvhW2Y2mXR63KgbBBTOc6JgJWZBc4VYPcScGTZiSVUKpZGB7+7XIlNK4pnM3dnTdhKHFk65LsKDGxLc/Fn1samjt0Mih22IzRzAxehhnFk8TvRfgzU4c8IFMoB3ktF6ZaTIJSndXHlR0MtsszKPwt1wypYyRvpmgSYn5YlnU1kpti/dKsBtjvmZLslWHzXENoyo6VZReffTw0llsbd/GiQCDBC5QdNSgNjoZk0v91aJq1Eh+Nx1iEVY1kILz3ERiFFuar8XG6A24a9XH8NS5HyNFLkKURNRHiHWZAmZrdC6On7tmdGGh96vP/ACPHH8IBsrcRDJhssGsZrPWwbWPpUmsMpGZW6YFTJN5XM0WalNumcovbtYuGk2s6082XThfrKTXpSrT/6VN8XVq0HgO7Gmkg0O4+DH9jJJexrnFs7htzR0kMwqixCBWRda7KbEbJS0TilCNlr1zOPcdxpBZGBnT6Sn00ON/t+2jWMzO4eDkS+QiFO8LWJjBFT8uS8DlG481LCYTewqmFTvZJmIpJIIZLbZyvlJxsMNsoYSo1EyBCUXTlDKxSoqKZXO6jvtkw+mAYxkSMuVMy3z6NF+De92aOzNlQf3S4viB2eTx73XLyTxdfKlamuMhTB6eoZJQT06fRClfxK9f9RF0+Tbwmi2LJkWNRbKrsKPnbZ2Hzpn4Kgb6T2PltDSDysUF6GPfgaK/ip9d+BvKCpw2EDbnj2H6G6S3t4men5lrKJX11UwB/ORLmallmsXwDNNwNJhepwsloD+0BT3+DfCCJdv1p0zIHKFik0KviFk90eKdf0//vSndCP7t4UThUIJsg1QjjuBppHeFOfdLQd2z559GuxjBPZ07IOpJdyWhXaX+ariowLngZl8zBXIatzosmOr291OUH+ATwwj58Mj5YXz70MO4kBtjq/2rkbloi3XR/vGyBJwspZpK5UoHC1iclXu2uyjMFRwYNElmlczb3ubdmEkvIEAX30eRKTuWvcbWBrHByAP2WDClSzraDLZHz463+gd1f43BYwIT3FjZkZ3TA4uZ1bnSIl45/U00TByAMf8K/4lWtV5LqL6HSYkR+UE5hu7gGpTMAiFjGjFPa5EsLnFcWw1IOOgT8Hw5D4NpL/fj3moKQc/N+q54IV9ekGVgTblstPAVALazes9rfGLx5ic2D3ZGp4/hyRcexFh2BG2+XrT6u1CgC9oYaEV/cC2Z7IoDNVqsSL1wSb3x6cGHmboY7Ly1paBwIuplnC0DMFgts0a56mulBfz1mefxQi7JAzGvEseDGkWPVKAn54jn7fddw9cO9wZX0/t9yJl5nnqxq6MQOKNqzn37Eo8u6KE29Yo30ZcFVZoVq4fSH02hC8CnsuWu27W9khmBR5yjGqVBUoHkRB7ODCCqBdCkdWF9fBfOFA6wJkduyY1NOLN+yYqBhlV+O380b1lV0NkhFRzuvloS6UbDIq/ysIj5Od/sp1SHXkpZ1fdU0yfHk3PAIl1KQM8b2BrZjcZoB1GBY5DJEjh+flm1CHAJQiba/x/IgyuG3kQCFllkyYANy3KCGNaOyAuA2D9aQIE/EmAr5MkEG1gX2kw+8jdwnqLeheI8xwi9VQkk5Uu+U0c2bEuiaGGZBrp0L5ZXc/DKSfchy2J8hFZJYg1mdKBGp85ZJexZoaCMN2IRTJxPnsD7V/8bbO7agvE0+VpZdX+LXYviPKDE+3xJNBuS/rd5HixIDQzAYELtDnRWQQS+4Nqu5TPsOXbRGbLVlruAzYQpH55+BUfmjkAyFUfjPc0Q5Ev82saFZpo6km55BtqqMUVerToP3VlttItSCV5OC6fo3XJ1lrdVog/bFL+Kt2ZgX1HTNFw0BoHCGCLCBRQqC9yiwJtwQg0NWz4kQdKTDave3j6YctqoSBchXy4gphHB4I86F5JrpF2rsoDjB9lykSOJaTx0+Bs4nHoRiqw5DnGZdjIjvvwzBgc32Yqilt2X+bJR55y2m7+6HtEFLoRlWkukPIJKCKsa+ilC1zmypQpBtAS6mK/nFKIsyTBUA8eP/l8YI/LDZOtTq6sU3S/huoZqWY/NQZbypsFH3uYabEPjESxd1FyhiN74KjSq7WjUWuni6vAyVWeJCAUsBD9Nh2UcDDpBEfOBPAiyHY0EX4urXkL8E5hi2QZK/Bq7FB4b1eJ270CpptHeMWR/sZCb50tcOiKrkNUz6PVTIEVCNuxKFfvQiGx4Qs/iH8ankePrRWvOVvQQL3cqVX86hQv79l35nfIvS8CO4RXgI02cWJjA1qZr8O6+XydSvcBfXw4F83iIhOALEMpEPDE3oyJcSg7uMn1W86zpb/gcRTCWrQp1nvOiYaH2bTiR4LoDjx1ine8OT7yK96+/n9dj7Wy7hcxwmtdhcTfCIE/6FmY8iArlvZIkumnYcqRMcBml2vel8xuog3FZAib9sxwxSxSx5hDPXUSvlECimKYrITlNT+BWOdn2JasJ+PAqF5flt6IgvaHWWJFEY3nQ5pzLNcvLNM07DReO+8tYADieGsNschQf6f8QOtUGzOVnl388PMrXYZk8Hy46/lxwKyy563E+j88LqT4EfFlpkiJJJQ/mkyg1OXD85xhJqDCDPt7eiKU9kofzehdHWJZNVoXNLqQTPcmi+gYNVmzV4GbeM71VYt4Ngqp0n/u6Vz/tCpmBoQdHH8ceeSd8kWGkc8MsmKue37aWR+ZOqscE2+7vwLQ+RS9W6LfK2NS+DhfypzlWLdnS21/AkqSkuBmkCySTf32mXEIzUX8NsQAYFM04YElwyH6+ms+NUBhcKTBOEXZ1lZ9H5SiC/EYNVuU8/zyBdbdja5Ec0yC6rBC3EIKDEnsm23YlJikMHxcwhxweO/szIvkpDjAplJOdz/fMh2OURRelEnknnpARxobmzTiTeQ13rPkATl64iFKlxIv+FE3JoQ7GZZloikqzXFAkJUach0IKAqTJ3nWzDMcJx8gs8ooLi1UjMtLe7+LKQhXBcpqGMi194zYDqqwU2PX3qRo6Ix1VYr9qol0/KQrVKKuqzXz1PvlVmybGiRYNh+N+6PSdRXt5tYfgfgHvSZE3QJvJzGBrw424vv1OdKjrcWz6NW7y3eLLMupgXJaANVFNy0TVsN5VDLFiQrYMZ40lJx1IwBbRg/3RAQL0A7yFUaPWRnjvAAladxZTk3SDxB/LrskUX2dVvvjKJyOKrDQTRQG/5qcUpxmGbjglrZ4QmeKKtY47ooSqv3fWMjmdcjSfwrvpiLWlD9V1v3wRWjUkt3nbh6XSElrp+927ai9eHX0O8WCELWV1UDBRvCwT/fFDH1f+9MX73/I1xpcl4EAgNCXLss6YI8FyfKRecYMR1jafNNswKgiJjWgL9iGuteCWgdtxLjnCu8uxC5st5rEq3ofuSA+XSskoX7LkJU7UQ2e4s7M13IqwFkHYF6kFbEBt9QFvmGVfgnR5QTYTMOuU46VPHt3oAVWOFaHvzzy9FxlaTqB18dg/4MzxL2CqMIT1TVsIvavwDvL5YrGCyxjtxoRfV+fv2HforgDewnGZZIM0LctKngcprPuNopB2ma6AWaWrwp/P59PY2rgL9679MMaS57CYT/GotzdCxAORD4ocIs3s4ILJVZbiy5t9fmjNlzKJYurAzo6dhF2vc0ykx+dWAQhUm7Gw+zLRet7aIsezwrXCQjWy915womKbs1veC+z7M8TL79ewv5TFI3ND2N51I4qVMipmhft0glf/lwTMtgT48on7GzwYXZAlSsXFVQHLirDH9ltUhnt5ebCMeb+ipkl/USkaaA41oyvah0Ipj4iPEKTYKueCpAdxY4MKozKPw7OH6GiCJ4luSpbmCZfegj1tN6CLtJNpzVJxseeuj911yer5dClXmkyOYysF2EpmhFdSem6WV0fZNUDFJ/vRH+5zGSo3WXbKw5xo2eWHqyYctedd5+Iea3EQxmzUEGzqx1VNu3Bx6RyPvtl5NVX5f2WiWX/pBwdv2p7XjX9vSuX/9tdn3/e//83Q+/8g5rd/x6/5rrXN4OYvvHzvhgeP3vOWtEe8rCi6s6VvcXJ2cjaTy/QZJR1LwhLuXv2reHbph9jafDXlnlMctpwmjvbxn34JrxKGIShB7t+YNpXtIn545oeIZU8hY+dJ7D7M56c2LxaG++n0p73PiQYj5umJI/jZ7CimkgaUtiDXUObnw0oELcEo5o0E96e95O81csKsvEdwF/06EKNHNbo9tZwHVZPOA0IWaLF2w0zgpu0K2sa1zbchkU5gIT+HYEhx0j/5jYDM68cXT+4dOG6c+8RUYu5/W9vUH6kQsjaTP0eZgIgcxREd4gCCdvQaUxT2R0T8F7wF47I0uD3WkVckdY61BzQqFvnTFOYWF8gUfxAD4QECGKZoBklIxTR8O6MjzcthHFSoQn6bbYgR8wfx8/I4XqX3SoaMvJFrPz738tVs1y7vc3xSwDDp0SHypRMBCsjcWirmI1nX9y7ilJuJYy4bBgZi65AsLBGvqy4DVxxtFUWhSht63QAEYZnQeTRv8TSJlX+xGCKstGFTbDuOTb3Cg0bTaVlLiLn4Ly7+/pOhvY0L1sSfBYSmB5r90UhTZSPidAuKFGwW/PCZYaiZJFpnkq0XRk/cniol34u3YFyWgDsVf17zaaN019CLzg8/M/oy3tPQQb6yiAwFUOUCCdIvI9QcYkutyT9aXEsqZOBYKqoS4C8oPviCGvQy74MRH0+P3PniyCPVyo6QGK5Q3mqpdB7GPQNV94uyWSK8eQLv6bmXC7qdiIQ0Rb+M0/Uap9ieM4YLny6r8HBycdb6HTz9alKbEZGilMMb5HNLuL5xHfzkbudLM9jRvQsbm65CSA3aiiz/sz74i/vv0LL5id/J6vP3qKWo//bGj6E/tJ20tZEmtEYgiYIGvw8JmixDxjmYcrZrMTWy7/mJr+7ECo/LEvB7byjmGyKNxwi2y7B11CqZxoSVxo9+8iAOj30PPjJnuZwJq2wgEJSJwWFVFxbHfvn2CGQ22f5ERZocskLvJ9DfzEOaz0//yk9G9+/1PicUjZdIq8zlvK5BEvLTxGhUGwldGiYTOoIHb/oIhMoc8lahinPzH7ksomajBm3Y5BY0xKVWvhBcJhexrfE6pAlqLVF6F03quEddQJs4gd+++pMUL6zGVGKSRf5mQPWV3uya7LP3ifPx3N0Vcel3hVxYnSMq8vzIEWylQBOLh8HairMKUkIBUSSTra2KIhwKUwRvNL948Vt/+vSFb/eyc6xUl4TLC7KEfdaq1u4LPk1K89ortmohJONvzs/hdCKPcEQhTaDUJ68TSMFyUY0CsArlkiwtEZEtEVrkIy0z6ceWTDREibojnqJUKEVHFk7+hwdf+Mx29jmqJfMtNRhcyKJoizX9JgF3RztJm0Jc4EfG9yN74XuYHP0pPTZrDJUXTXtSXkbwsbsG5earQ5s5jdhLAmSNwtN6GgJd/DS5lVcPDuK1l36Cbz77f+O7xx4ma7HI0iRLfRPEjY3gL843Fc3sr8qG0mJVFErtAjiROoz9I/+I00vD3M3rvPjf5pgBWwyg030fxSbT87NXjyV+cfc7R/q3fuHlT69Im6bLLnzvaF97MRaKTjPzVskZBCbQzw/4SKgmfBpr7SuhkDdgl8vY2L4JIbmbfmCWm2bNbuBUfJwmRb5AP1i0SMhkqgmYzBbSq185/+QX/uDxD7wjFozlKLCxGLq0rXsrN6UM0OhvHECeLcqmc82T8P786Kt4ZmHWhSHZqIZT7hCqgZMTd4sos51bjBxu6bob13behInUKFkimaNz5aAfX01l8c3EEI6nTzslRywFo3ua+uYCzliJfsr9r9OLvNslxwVkYqlm+31Yiiq8TJhBuIWSw2alycKx51S6FrpuRVOl+Y8HhcjtXZO7LivP9sZlCzg02j29rmfts6ago0yCFEi7fGEVubRJwQSBHA0+0k7S2HwFF9NnsLFlBzY33g6RfOe2tp0olYLw+YmY1xQsLNIEobS/Ma7R8SKSmfTNJ0ZPPPzcyecfkE2lnC5mkCql8L6rf53y5m60B7uRKae5uVOCKiYb2HIWv9tn0vl+QvVfwWlPCCda9gAOFtEfnTuMW9vWYWtLBGdmT/E2wwy8UVjH2a4wLII3FQra1jb2c19NCJyhioE3FYCuV24ulko9hbzlnN8MkhFRuNYyQeZLNl++ky1YPC1LZQyQasDKBSlVvFoKon1tQA5P33fffSvSweeyBcy+yIaujc8HAj6jVKHZmClDDRLgQd8+NUf5MJldgTXbzpr0Ywt48eJTiFNk+qtrfxMbmjagOdiGxWwBDSRYH5nf+cUyIU86WlpV3vSskC+sOjp68BOlrBFULA2nZwYp2hbxgdXvIdNdIQ2sQHDXovgUZ79Crw1wbRWEI3AGYdrLYGcPFCkYRQy99iWcO/4Q68nFD3KreTlmzjQ+6mtAM6FpLPgC+2Z+35ti0cVycaNhWJKui2Bb3cblTr5tbZldG93RXjZ52GM22YpMKUgMFxKj2Brbgk69VxwcO9WJFRorsrqwqaH/YFtj6wVW41xYqvDoOBgIQ8+SOTN0vpIwVyRTxdYiUUT63ePfgZzNYHe8jFVKCamCQCS8jqaYSZFwL/x6D9KFPDra/IgEGEBSUYl+lC1D5MtSXjqzH82Jw6hMPUdmXawiWE4xnFBtV1htyGLV+GPwVRW1kkgme5ny0q8Pnsc3Tp5iWzJxgVqmA3swJqxBimNb+zuc1Rj0GyVBNMPWG2lNtnFXvlzq0+mV/tB6skpRyGR67YpaXYBn8r2bTLJcvK6YMgebFx0uUcaRzD+P2cH9yvn5Y+/ACo0VEfB9Ox5Idzf1PqQEZCOT01Eiv9XR0IrVLRswM5pANEK+y6cimdJBd+Enc/r1I3+Prz/5V5hauICA7KMfZxMVR5PAN49NrRtwa9dvIJcidiooYMtAH2l4ADqBKXq2gtlSGn839CoeHSdMW2FrhWyOQ1dX4PPhthS+5FfW9kyCWzrrdIUXkGmJIhsLccLEEbBzY/sohewY9vS8kyxNgiNczETbfv8bBPyZp877C+V8O3vP8NwwtvvejR2hjVBIiKWSY6KLrGN9eBVNWgktvh4YJYEvxpNJ4w8sjOGhc4cwkRzdfGjq0Ipg1CsiYDY0NfTTxlDjxb72AVTSpFV0cVa1rMLSUgF2sYwIQX4lSg/KBR0NGmlGKIYfkf8ZpZncqlm8D+USWcd0sYjTC08SL5zA7133R1gXvAXv6n0fOprjaGqmqDSo8H0XJsN+JEMB+CWVbn63rKbK/AOvWwwOd1WClwPbbnWmkyuzrj3OYvVqC2HLbedP51kqJJHPTlHckOMlwYTOGZKovQGqXNM44NdNPSLYEjKFHF4Y/jGGDn6fXNUkp0otvjzHoHMY2NJwAyJWF1m2LCy+WRcRM+RDpKYQoVyp5n5fWxwrMFZMwNubNp5RFOXpdL5ovGvLXSTEVrRSGkMOGOn5HPyUGqkUTabJFwtl8k0hiqL9AWQrjGIkHDtI2svy6DRpE5nzQ3NPYDr1Ah64/ldxe/8WpGaKSKUqlDcLWNfWwzeNNAW2SHuABOyDZS2rf6vSSB575Arcc8wOn1t7aNeowhq75LZiosg9ayQxPfhdaIU0vcabdhhhFii8bmjIRwnRCzFBUiKFpJ3G94lcmWGd9ixnvRZbBjuWGkNzhlK55Az5aZsv1mPfSSczrpB/K5XzkenUaBtWYKyYgH/z3n2pmzZf/43ZzMLZV44exG39m7AmCjKtDUiSX85PphBtUCD5fJhbMiAVC2j0U6KvEf6csciEVYiooFw4oCCbFTG/pOKxocfx3Cv/HefPfI3AjgKlXWFMzZegp6NEKGzggc+q6GqKrDM80KoSEJYnKMuBFj1+kGmRbbsL36rFHG4xgnOzvD7SrO2EYfOemMxnfu21szg4lySolZlwUT9y+uwbKipLphjSbVNx9sG0WdUHAs0ESYZUvlaLCZLt48SqRb577Ac4ljgOTaC00HSWxQbIX7PvWCjryoW5U01YgbFiAmZjx469x66/ets/naBI9+kDj2DsxENQNdJYVUNiQYe1ROBHTCUySEV6SUKTEEEDaa5GsF0yJ2BhiXw0kTSdjRL8NJMzOQ0Pj47iT08cgkmutrtJwKaBCLIYRa+/H3vX7UWCzGeRomCxVoDlprsEHoga3/zKdLV7uRK7pFL1L5aR/Q7Z4EwQm3cWINAjTumO6hTEU6hn+kOK/frfnytkw7ZpqiLXcieaZ5OHJ0QE7LDAjQufFTo0RiCQObZIa9lnsdrgJqWdofW88wGheSuy6ceKCvjONXeWb9103d+u6u0+8eTYRfyPA+dgMf8bU3jJzPxEGnoqjViTRpbbj7DcizwhXi0Btts2oVymjJlEBcVsiVISG90tCuWiGtIk3bklYGZRRyqtE7iRhZkbxC1hC+fnjvNVhU6ZkNviwXLua2S6r2reSuR8obp9ju0eV6vkpP8Nh/Bng88Bbk5tvrxFI/SNoWYa24vJ9c8EuJhEILxBgzOFjJ9t1RSUw2jzd/DWEhVL53XiIQ5kmHxZKovMne4EjpWw+cpKQvjEGE3KAAWblBuXzBWRzYoKmI0Pv3vf/KZ1Wz4VbGg8m5Nke3EyDztXgj9GWulrgr/YBoOE6AtI6GrsI59Mgp/MIlgpoCXCut8EyOTKmJgnQeeLaA4Y6CFTH/OzHUdVCtRkErKI16aG8JdPfhVzxQThyc5yh+VlrSw4ms3MI+iLYHPzduTyeSeq5qbS2aGy1sHdrpIQzrC5D95EWYAM1dnk0vZcOueYrXz5jeuSSkIlyNIoduy2zu18j6du4sSvjr0DE5kJXl4cFuOEw5OlUmMICTESusEnJE+hKO+Pa7303h2IBRqxEmPFBczG2sZdL28c2PgFSVXnGdyTmi1ALBIB76sQsN6IXb3vQn9gNWlpA1pbGwloIGHMlVGczyIqltEUloh39ZPp9WM+JRCiVaEou0QXp0zaYKAlSuZbVXFMDXBzD5cCNGsSc3wyCfTViwewu/dWshZNvCaMmW62l6FsK+6m0U465G0gzXL5iBil+KCNr9RIE40puBOGB10202zNiDQF3yDgkBKJ2PQZRb1In9eIq5tvxjWR3Tg4fJDn5uxzmtCLuK8P6xrWIpvLV9fasX2MUUni3vU7ETJaMJedvTKQrDcbn77v08VNa7Z85/qrr/8LuqRm0WS7eRZhE3gxNH0Eh4aO4ta+jVgdTKNVKBOcqXH8mlw0FmgyFBPEHJeXsKYpjlWtlCuS5iZJqxNZm4RNFzBX5giTX5OqZtMrhne4XcFJfQgZm8vMEpCyhNv67uJ8sSYFcHX7NZyarDYvtVx82nY0NygGaVLsRlAJM2Sqil97C9KIazbWp/rfYKINq9LKivtYKlQhduvDPbdg4vw4Jgvj3NyXyAwTB4kPdN6F6dkZZPJZ7n9ZZGeSe8rNHsfCySdw+NwB9DRtKGIFxlvW0v8P9z7Itl/9i0995f7SgSMvfKpQyfXZi2UxSKZ5vHwGf/fEeWzviODs9BLssAJ/gx9+wqCLRDqoZMZCVhC5GQtre7uwaJ0nx0aRKAUgZQJDMumi2yKJqxS/zxsamU5FBtdI08Gb/WIITw8/gT+45bPIzJ+F6RfRVOqggChHpHuwKjwObPB+ITIWaUK8u6cHp2eHiei23UIAZ5WERamZpqqV1/xLl2jY/nP7tafO/NVan4+OKYgYPP4MKqcO4fn0EqL9AfpdDJokUqVyGs8+cxrDUo6IGq0awbPFzKcpwPz5oSOItDZW1rduTmAFxluiwctHd8OOr21Zs/UzrfHO54sWA9mJOqQLmaDI5qm5NLIUYecJxUpN51GezRDDVAGFUjzF2Nm3nbhYGUkWXM3lUFzIQswXSMPMqjbxHJfp0jICH5ZdjZYlSmvmcrM4NXMEt7Tfhd09N6JUzLHNrZwSX1f7vePZRMkTKX3x3PdhzR+hc6ncZ7Ngq7+hl08qVdb0z75zN6YIbfJ423DAaM3byQ2ySMdThnyYgqvv6/OoEN8dotSPMWqM756j4Pup9BzCZLUYAcFoMYZ7M9ZJ8UuwCJoNB6KZsK91Disw3nIBM3O98533PLZz846PrV+95euiGkwmyUxVigzwMFgIAz+xTwJFy3kCPTKU52bnkzg/fhQvDz2G46MvQYv4WJcXlMlvzqdMZBnM7wVH3lpk26vWEKo5rrdGmXXMeW30J2gsncMaWafPeI0fZ5q13NehEO0qCfHY8eN4bHAIPpkAFcpbI2oUbcFm3q5JsBRxId1+b0nCAPA5gQn51MIL78hUUgM+1ccj8lgnsVDhAAWTlAmoNnI5izd9Mwi2lOm3BMIyClmdrwhh7oK7FcNpKRUJNi+KopjECoxfyq4rD+x4gKE+579if+WB9HeHvnZ46OBHxqYvvKdcKreIhaLqU0scn2YNQm1Z4amDQWzLuUQG1qLJa7dUv0JAhwQxokIXrKpgHb8p8Lps5j8Fp5kGzzltN/WRBBmz+RkcfvmbiOYUDE4sEbOkcT/Nz2MuAzpc0z+hkYZRcN4gOrVZjWozIW0BQtE2473r7xkwc6VishNnWNHD0xN9O4dTxz4jKkZrMSWTpq9CRUljZmIGXT0+5JPEehlsC0aKIyby0EirG4iImE2W4GsVUaLYIhwKoZRz0rnWePtQZ2dnASswfqnb6jwgcEG/9KMDf37ipcHTP744NXx7Ort4Uz6f3VRMFVUlUyIoUnW6wRKoYPk07lZNyh+zaQrXFNL6sA9O4Gyj2rbD02RmuZkvFe1a+atbcFyhQO0p4qxTMwu4SGbXq+2q9u5YnhfbbLmMs4yUVYuIFOk205Xa3b8RG8vrCIDJD7V1bn3CwmH/ty/8510HJh/9k5ge2zlVmBYiFD+spdToidNPUSagIUg+//yFAqLxEJ3HxCJlC5H2CNoDrRiVSMChMtaIGzCRGsf4ZJKlgnbU1ziIFRr/Kvsm7b3xD7N0UZ/4x5/se+HQxROdJID3LKWXbplLTF27lFpstjNFusAsbybzpjndYCUSiE58ryXztmkkSIunGF6J66WmuqaNnC3k2RNxsVGiLdvDEGYNpygAqFZYLjfpXmsCTjmyLrk0caK5k2hItOEbg4No6F5VXL+4/x0vzf7wvuHpY/dMzSTW7Qq+Ezsa9qCN8teDB48gQ5N12/YWTJ9f4jFG31UBJMeTBMlSXBInjrwUQE/rADZ39+HksWEsFOdQyZsIRWJZihuOYoXGv9rGWIJTqJxxb0MP2w9/MXg003Vx7PzNZ8ePXzudmLx6KbPUVSyUo2alFBRYDNIcoKBZdaJa0/Gbgl1jiDjXyxrASMtMtAdNsnjGtp1tY3lvaJcztpxVDPzbeL7YRBUE4RQymf3HzyzgxanvI9nsQ0suf+Nfv/Sfnp2fXwwJusrbEp9IvIrflO+ERXTmhew5bNgQw8XBRSwsVDCwtQVmKo+xkSziPXGIxH2XJ8/jd3fejb9/4QgOjL6IliYfh1SbIvHhqBo7hRUaV8zOZ/cJvERljG7f2r//iw8bN0Rbz144uHp6abI/lUmtn8/P3VcU0j1eduQ05nYCJIHVaVo1YNnxp8AlDRY8U+wJzq6tU+ITBsustJcbu+diyp4MB1Ei/x/xSSBrEyZrzPcyZpMjNVXEVCqJxyYeQSQSgRYukXCLlOfa6N/UiDAFWSdeW4RCFGdHtw/jh2ZhpRP4eek7+PnQFJnsAMo5ky11NYNa9Pg7tl43jxUaV+TWdnfe+e9YnDzObqxK4sTcLv/f7f/iqhNjB3t4WTWlIIrF4EmZsFsVXaFWTFZm3IDL3XLHrKVLTqRs83W9VWLBxa29/Q5tb+aYzgGcaPCayTDigrTUT3ApKyEwycwuTZUp6q1ApVihgVKeliY/xglDTy5OEfxK2HXAj43b4xD0Ck6/TOmSJGHLhlaAsL25WcIwumPYn5yHQC6IlRrlc/SbNDXd3bzuwMyZQgYrNN7yNOlyB4tSt7a9Oy/JAuNcuCntj/egwd9EUGaFfF47utR2vujNq8KwvAgby6NjOH7bex6vE77nh4HaY49StKrr051JQd+kgbSxsz2I1kaKttMVjJ9J4eL5POEVPvSTIDdujaAj24HRwzqKkojrb1iHLeJGDB1OoHVVI+INBKhQft/YGoBOwtVNgmhDnac++M4PPrNSBXds1MXmlGxYNuuJIfCdy/g+wgTmX0csRNiSiWm64Gqt4PpnL0Vy82D3LieaeFhOj2Ub1ZUPy/hgLBM2vAniGW9XyS3dwtJ0kRfRsdIt1msrFAugNaoSuUGgVCaPU2eSWBOO4F2bb8ZF/znc0XQbHnniKVgRC80xIlMGE5DpYImw6zyBP+wr7N7yroc3brxxEis46kbAdHENyy13nUnNYP/Rn+H65i245ar1+E5pZlkVhlMRyctzqjgzLiEU4EbXHnlguc+zv7wTrikui6iX+2bnHGwdlhSQ0dIZIJ8cpLSsDJ3Am8xCklOaxYqAcBNF6+159Iej2NP5Phw+Mozh/AX0DgQxczbBV3k0t1HAuETZASFn/R3rRm676VceFnizkpUbdSNgSZLLvJURz2spXVIs/GziFYxNHYNOvLFAKZDlUoA88rUsl7S3uWnmUbfh0IVc8Kxiw6xpN6/ZovMG5SCdo+hlSnx4ZAQPuglU8fvYjVKttIC43omfnn4ebdFOdDT2Q+lZRFOLBoU+fHJoBhem96N1YQN+coow5gYiU85S2qRTXt3mw/rQWkwU55Ep60vb1l7/+e1rb17ACo+6EXBZr+iMAuT1ymwHFEpztGAQ58j3NpMwVc7KOCkO70Tr9mHjaBV3nrVct7q3MFDNl026se4BXcE2jOWHalyxe0yNL7aRTuiYGS2iKRbG9g2d2H31bVgTb8R46ThFxzrmzuQo2CqjXBbxgn8JQ4mXURENFGZpohFh0RBXsSGyEU1aDE/PHClvGXjH97Z1btiPt2DUjYDThYTudLK13ZX7DhDhI0CEQR+sCznXTsBd+un6YtNt3cT+Ojx/dXUD2yaghSDDtDnJX+uPEsRYyjjrxV0Uq4ZROzk3q8Zga4Q7+3xopAOvb7UxU16N7x9+lLcm1sv0fTQiDJqiaCe82SyUMLmQJ7KBInHCpgN+Ey0WkfptV+FbLz1Kk6Tn8J5rbvvrvXt/bwn4BFZ6XLECZgD+CEZU0gN7EzbpH/7yzaZfC7OlIVXzaduez3XVi5thp480K5L3NK6aOjHe1dVklgeXzCK6tS1YUAtIKymsi6/GU8OPw9/E1hZXUIvMakEX99kFA0sEWqSIOPinM4/iOLEfRpA0s7EBbQ0KoXAEjeZLSI9lUCjQJFPoNTLbNhEVVjKOvbfcjqdO/xxC0D+y79/+0R9uWbflAt6aDg5XtAZLWExvkvRk/8nyuHl13zX9yYUJjM3PYWThLFSWo7KjvPYL1UBKcJo5m65g3JDl0notpxiOEfP5ch5bYlshWzN0k5Ar5xCwWb3b6/Bp1D6nRIIVdAUiQahHyZE3dAURDMj8w/JzGSwSNcgsiOIjYKSJtWwyiVDIwyAo8t2b+pDLzqEx1Gbvufb2xZZIeIMhCyznz+MtGFeygE2L0kvoBqN9Nvf4mld1Boh1UlM4Y9jVtUNe8lpjl+D6YrtKplsCLs1zl6VFE9kJ/ErrZqzufAcOD71KRoDVoDhwKNz3WdVqDuc+258iEpN48KaznUgJ4FhMsEZPROmTedZCfoquybwThVTMFmlCkBZTLtwXENBRPoNiwkpfu373j1taOr9dELInho/PJzr31Oo6V3JcsQJ2sepF9/byh/5yd+/0xaEtjOZjRXa24dY3mw7+zP2tyyx42mq7eDX/j1VmUPpjGk4ebPNOaCLmCgs4N38Wd0SDOJA/79ROGbXZYFdRL7g5so0y4c1zxAqx5YpMizXSVFZUqKqEb5NGl4tlpJJst3GBt1CWFIF8u2Fkhfh5Mb715b72Nd+LdO9+8p179rzl7RDrJw+mq5kXbb5g2ydWIYwa7uyEuzWBuFF1FZZ082TLXQDGmsoyQVuihFPTr6I/MYj5hTSRtkJ1pwAI1iVCdgrm6V9RpIDJRyyX6GxdyyqfizqyWabRXj8Q1k3LoAllQtXC081t7Y9t3XjNP63e865De666Lwf8JX4Zo24ErIiqxcpvvFyYC9Zdf2ItQ6+4D7btKplvmV55j1g1t83+BlRYaZBVgkppy2S5gAdPzxCdGEA0FHEnhpNIeQvU2LBdV6CKxC8RmlYgrxnxhZGs5CETjcloTZXytUq5RCkS7HhDy2J7c+9P/cHwP21fc/XLv3v/n6X+WPjqW2KK/7lRPwKWJMPLW/lwNapaj7WcQvReJ4kG5ADPgyusNwTBmqxmanPnVXh17KCzrpidj8xoMRbh5TV8DbGLdNlWNSKDt7s4+8P7i1RsdDZ0Ik4sU2ZukC9Ctw2xRMnQXFtL3/BA/+qnb9r57h+87+YHRhx381Pgg/ilj7oRsChKptu2uQpWcG32AqBlHK7HKtl8b6YKblq7GwenDhI8mEVHrBcx0uB8pYSg2ziVmVSFFRXItb0Qq6bZAw7t2k2UDKIKTYq+Kd5u7CkrfpXiY3FU0QL7V3Wve2H1xo2D0nTr3Pt3M9Lgt/GvOIT68cEi603Dl/mIXqQMwVUwF2cWmI/1WCCDryFCppBFqVDEeza8G/vPfwu3r9uO5HSefKVO75OqGn/J/gxs2LVOeNXqDnfyNITb51f39b/W39Y81L1q65m1pjktqdb02i1bRnZ03r0itVQrNepGwDJUa3leCi8d8oIs06qZa5fcZ6+xBWi/GH4Jn+j6HXxox2+hR+3HQ8d+DLNiuqv4RdeXe+dEtVjAFmuRtO2afAZ1bN2468t/+fvf/Jzzpn/ElTyueD7YG9WNz0zP73r0HqrIlrfO1ym+s6r5MFvR8Mzxx3FjKIoGYwELqUlUm3XYTtrkjFpVZS3xrUq42pNasMX/aRvDK2XUjw+WZMtdVrYM+IcLcniFdcsAD6t2Y+2XxhMXcfyFh3hR+lJm2uljadf8Ldz54wVoy5Gsatgr8I0/bFb1WS+jjvJgwduRDryyw2ll567t8dgeN0Uyl9VkWdwIoyDL+Ie5aSQXKkgKBJYoQs1fc813fCzPc71JtFzCnOCw+ASQ3mTjkCt11I+J5rsgeUFtraGZ5W2PY9m1GixuynFJ/spNa0MQBt0USaj1kbZrYIiwrIyH/Q2qwSr06R3KYBACSf5/Aa/0kBTye6ILVbk3tpiabQfA9g32Fn1X2wBfkr86OS1lQVCX7evgmXqv4M52AzQWfLEtCPrjfTCYTa8GdyyatlhLxbrYcYWNuhEwXVddpHCXt1RwI2fWlKw92oZN7VchqkZ5kxPL6/ds11Y7WG7K4/lt0d0Cr7boTKhtqcOKcCl77I53I+qPUYCmw20X7xwvSJYoiivSZvCXMepGwIZFebAoehA0/0chQbA9H7Z1bUGAyHunatItuHPLddiq+oDiduHhgnfMu+gWDdQiZM8X21gTGUBvYy+KlRyvHBFcK8AMO8GldsXQ6yaKrh8TLfLAxq615XfACYM1NtVTyFUKfC9Ehi2zTSmZyFgzk554F3GvMd7A1FxWK+1UdtRW7bPB0uFSpYw10V6sb16HxcwS3w3Grn6g4/3ZTmyok1E3AhYNy7Z0Z2PiKolPf3P5HJ4/8hhBj2n+RHdDO7b1bCPivoCAFsB1/duxmHW2bPcCL09zncZoHu7sCJhNgvncDHqkBQhGprovhPMWzkQJtinbqJNRNwKWlfhIyN96hNfSLIMT80THncyVYcJJdcYTk9jcsRYbu7egM9qBBmJ7EukkReFCrcuZq43VvYhRO58mKTi3cApnTz0Oo5TltVxemiQKcqUp3HMk4I+NoE5G3Qh4V/OmIxv7r/3vxaI5VyoXq9ExC5YUlQEPzvayTHMPDL6Iuze+F+/axDbTWKJI2FjW2a5G/6H2VDWXZtvnLdHx3xyZw3iuwCNv5r9LRAGWSubc7qtv/Yv3XrPzIOpk1I2A2XolVfQfX9u1+dHGSMtB2xKzXud3VIvcLUqDZJyaOo+54SdxfbCIsYmjfEdvwYU5vc2wPAjSXYIGwcuF2Oko0S1pqmu6TQq0lFx7vOfQpp6djwd8kWNr1txZF9vasVFHSBbw+Y9+afTE3FN/eOTk6VWPPvPtPz4/NX4XpU6GaZiq7WLFTFA+nx+vjBxCbvoUhucyfAcWPuxqoFQzzba7dNR7jk15t4BPlJSybtjq2u7Vz/+b2z71f9x5ww0jn/vcl68otuh/NupKwG6dVv7Qoa+cOd698W9j0Z6DTaFoYnL2zPsnF0Z254umVCyVmcbhQiiEs/M5ZA2Bl85Y7mJxvv8zr5M2yKQrTk013YqUbhGJb9uijz1rru3d+sLG/q0/IA65MR6LH73zhr2n6fPrJj3yBm8gIni7RtXp+OYTX9rxnSe+/vlwJL6wa/Puw0fPHNkyPvvi/SQ3NZeTE5oUWzKRbopF7HhJN5AtyEsBf3xRENPxgN9ozKTNytqeXQ9tXb/z5JFzL2wrFlPN99/6+//1fbd9qG587ZsNNl3fFgI+dOiQMpE91yko/sTeG/dmT4y92PDlb/+f/y2VS7UNdG55dKBn7enXjr90zezSiQcYi9jWtOUru7becHBifnjThakT9/jk2Nxv7P0P/3XXxl1Lr776k0hGMBpv33k3W+VXdxr7usEF7O1w8LYaT778ZEOmPKM2o29xz549BmtUduqVA5tE0xI23nDzqTtW31F57rnnpALGmjQzqt96694l1Of4l5ZECP8PAPgFhqae3ywAAAAASUVORK5CYII='), +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +'enQPSYCmDJBWtMJlV3kHBq4m2OQaTf5SbOH6eSGUqotmtAwWzw', +0, +'test', +'{"春江潮水连海平": "海上明月共潮生"}', +'m', +'m,xl', +3, +127, +0, +0 + ); diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index 37a2b20a7c68..ae7200b1af02 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -6,11 +6,11 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 2.0.24 + dockerImageTag: 3.0.1 dockerRepository: airbyte/source-mysql githubIssueLabel: source-mysql icon: mysql.svg - license: MIT + license: ELv2 name: MySQL registries: cloud: @@ -23,4 +23,13 @@ data: tags: - language:java - language:python + releases: + breakingChanges: + 3.0.0: + message: "Add default cursor for cdc" + upgradeDeadline: "2023-08-17" + ab_internal: + sl: 300 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java index 662cfaffd817..c161cfbb8e21 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java @@ -4,19 +4,56 @@ package io.airbyte.integrations.source.mysql; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_FILE; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_POS; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.integrations.debezium.CdcMetadataInjector; +import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; -public class MySqlCdcConnectorMetadataInjector implements CdcMetadataInjector { +public class MySqlCdcConnectorMetadataInjector implements CdcMetadataInjector { + + private final long emittedAtConverted; + + // This now makes this class stateful. Please make sure to use the same instance within a sync + private final AtomicLong recordCounter = new AtomicLong(1); + private static final long ONE_HUNDRED_MILLION = 100_000_000; + private static MySqlCdcConnectorMetadataInjector mySqlCdcConnectorMetadataInjector; + + private MySqlCdcConnectorMetadataInjector(final Instant emittedAt) { + this.emittedAtConverted = emittedAt.getEpochSecond() * ONE_HUNDRED_MILLION; + } + + public static MySqlCdcConnectorMetadataInjector getInstance(final Instant emittedAt) { + if (mySqlCdcConnectorMetadataInjector == null) { + mySqlCdcConnectorMetadataInjector = new MySqlCdcConnectorMetadataInjector(emittedAt); + } + + return mySqlCdcConnectorMetadataInjector; + } @Override public void addMetaData(final ObjectNode event, final JsonNode source) { event.put(CDC_LOG_FILE, source.get("file").asText()); event.put(CDC_LOG_POS, source.get("pos").asLong()); + event.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); + } + + @Override + public void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, + final String transactionTimestamp, + final MysqlDebeziumStateAttributes debeziumStateAttributes) { + record.put(CDC_UPDATED_AT, transactionTimestamp); + record.put(CDC_LOG_FILE, debeziumStateAttributes.binlogFilename()); + record.put(CDC_LOG_POS, debeziumStateAttributes.binlogPosition()); + record.put(CDC_DELETED_AT, (String) null); + record.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); } @Override @@ -24,4 +61,8 @@ public String namespace(final JsonNode source) { return source.get("db").asText(); } + private Long getCdcDefaultCursor() { + return this.emittedAtConverted + this.recordCounter.getAndIncrement(); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java index 3385812126c9..96e871915da4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java @@ -28,7 +28,7 @@ public class MySqlCdcProperties { private static final Logger LOGGER = LoggerFactory.getLogger(MySqlCdcProperties.class); private static final Duration HEARTBEAT_FREQUENCY = Duration.ofSeconds(10); - static Properties getDebeziumProperties(final JdbcDatabase database) { + public static Properties getDebeziumProperties(final JdbcDatabase database) { final JsonNode sourceConfig = database.getSourceConfig(); final Properties props = commonProperties(database); // snapshot config @@ -122,10 +122,10 @@ static Properties getSnapshotProperties(final JdbcDatabase database) { } private static int generateServerID() { - int min = 5400; - int max = 6400; + final int min = 5400; + final int max = 6400; - int serverId = (int) Math.floor(Math.random() * (max - min + 1) + min); + final int serverId = (int) Math.floor(Math.random() * (max - min + 1) + min); LOGGER.info("Randomly generated Server ID : " + serverId); return serverId; } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java index 76050c614eb9..e99ff2776482 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.mysql.MySqlSource.MYSQL_CDC_OFFSET; -import static io.airbyte.integrations.source.mysql.MySqlSource.MYSQL_DB_HISTORY; +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_DB_HISTORY; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; @@ -17,7 +17,7 @@ public class MySqlCdcSavedInfoFetcher implements CdcSavedInfoFetcher { private final JsonNode savedOffset; private final JsonNode savedSchemaHistory; - protected MySqlCdcSavedInfoFetcher(final CdcState savedState) { + public MySqlCdcSavedInfoFetcher(final CdcState savedState) { final boolean savedStatePresent = savedState != null && savedState.getState() != null; this.savedOffset = savedStatePresent ? savedState.getState().get(MYSQL_CDC_OFFSET) : null; this.savedSchemaHistory = savedStatePresent ? savedState.getState().get(MYSQL_DB_HISTORY) : null; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java index 74e7a3f49f30..1f3ea8d84c8c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.mysql.MySqlSource.MYSQL_CDC_OFFSET; -import static io.airbyte.integrations.source.mysql.MySqlSource.MYSQL_DB_HISTORY; +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_DB_HISTORY; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; @@ -31,6 +31,11 @@ public MySqlCdcStateHandler(final StateManager stateManager) { this.stateManager = stateManager; } + @Override + public boolean isCdcCheckpointEnabled() { + return true; + } + @Override public AirbyteMessage saveState(final Map offset, final String dbHistory) { final Map state = new HashMap<>(); diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java new file mode 100644 index 000000000000..f5c536ad00de --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlQueryUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlQueryUtils.class); + + public record TableSizeInfo(Long tableSize, Long avgRowLength) {} + + public static final String TABLE_ESTIMATE_QUERY = + """ + SELECT + (data_length + index_length) as %s, + AVG_ROW_LENGTH as %s + FROM + information_schema.tables + WHERE + table_schema = '%s' AND table_name = '%s'; + """; + + public static final String MAX_PK_VALUE_QUERY = + """ + SELECT MAX(%s) as %s FROM %s; + """; + + public static final String MAX_PK_COL = "max_pk"; + public static final String TABLE_SIZE_BYTES_COL = "TotalSizeBytes"; + public static final String AVG_ROW_LENGTH = "AVG_ROW_LENGTH"; + + public static String getMaxPkValueForStream(final JdbcDatabase database, + final ConfiguredAirbyteStream stream, + final String pkFieldName, + final String quoteString) { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(namespace, name, quoteString); + final String maxPkQuery = String.format(MAX_PK_VALUE_QUERY, + pkFieldName, + MAX_PK_COL, + fullTableName); + LOGGER.info("Querying for max pk value: {}", maxPkQuery); + try { + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(maxPkQuery).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + if (jsonNodes.get(0).get(MAX_PK_COL) == null) { + LOGGER.info("Max PK is null for table {} - this could indicate an empty table", fullTableName); + return null; + } + return jsonNodes.get(0).get(MAX_PK_COL).asText(); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + public static Map getTableSizeInfoForStreams(final JdbcDatabase database, + final List streams, + final String quoteString) { + final Map tableSizeInfoMap = new HashMap<>(); + streams.forEach(stream -> { + try { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(name, namespace, quoteString); + final List tableEstimateResult = getTableEstimate(database, namespace, name); + Preconditions.checkState(tableEstimateResult.size() == 1); + final long tableEstimateBytes = tableEstimateResult.get(0).get(TABLE_SIZE_BYTES_COL).asLong(); + final long avgTableRowSizeBytes = tableEstimateResult.get(0).get(AVG_ROW_LENGTH).asLong(); + LOGGER.info("Stream {} size estimate is {}, average row size estimate is {}", fullTableName, tableEstimateBytes, avgTableRowSizeBytes); + final TableSizeInfo tableSizeInfo = new TableSizeInfo(tableEstimateBytes, avgTableRowSizeBytes); + final AirbyteStreamNameNamespacePair namespacePair = + new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + tableSizeInfoMap.put(namespacePair, tableSizeInfo); + } catch (final SQLException e) { + LOGGER.warn("Error occurred while attempting to estimate sync size", e); + } + }); + return tableSizeInfoMap; + } + + private static List getTableEstimate(final JdbcDatabase database, final String namespace, final String name) + throws SQLException { + // Construct the table estimate query. + final String tableEstimateQuery = + String.format(TABLE_ESTIMATE_QUERY, TABLE_SIZE_BYTES_COL, AVG_ROW_LENGTH, namespace, name); + LOGGER.info("table estimate query: {}", tableEstimateQuery); + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery(tableEstimateQuery), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + return jsonNodes; + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index b8c8799a09ac..515e87d848b8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.mysql.cj.MysqlType; @@ -45,9 +46,14 @@ import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils; import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; import io.airbyte.integrations.source.mysql.helpers.CdcConfigurationHelper; +import io.airbyte.integrations.source.mysql.initialsync.MySqlFeatureFlags; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil; +import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils; import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -61,6 +67,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -69,8 +76,10 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,10 +106,9 @@ public class MySqlSource extends AbstractJdbcSource implements Source """; public static final String DRIVER_CLASS = DatabaseDriver.MYSQL.getDriverClassName(); - public static final String MYSQL_CDC_OFFSET = "mysql_cdc_offset"; - public static final String MYSQL_DB_HISTORY = "mysql_db_history"; public static final String CDC_LOG_FILE = "_ab_cdc_log_file"; public static final String CDC_LOG_POS = "_ab_cdc_log_pos"; + public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; public static final List SSL_PARAMETERS = List.of( "useSSL=true", "requireSSL=true"); @@ -136,18 +144,30 @@ private static AirbyteStream setIncrementalToSourceDefined(final AirbyteStream s return stream; } + /* + * To prepare for Destination v2, cdc streams must have a default cursor field Cursor format: the + * airbyte [emittedAt(converted to nano seconds)] + [sync wide record counter] + */ + private static AirbyteStream setDefaultCursorFieldForCdc(final AirbyteStream stream) { + if (stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)) { + stream.setDefaultCursorField(ImmutableList.of(CDC_DEFAULT_CURSOR)); + } + return stream; + } + // Note: in place mutation. private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { - final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); final ObjectNode properties = (ObjectNode) jsonSchema.get("properties"); final JsonNode numberType = Jsons.jsonNode(ImmutableMap.of("type", "number")); + final JsonNode airbyteIntegerType = Jsons.jsonNode(ImmutableMap.of("type", "number", "airbyte_type", "integer")); final JsonNode stringType = Jsons.jsonNode(ImmutableMap.of("type", "string")); properties.set(CDC_LOG_FILE, stringType); properties.set(CDC_LOG_POS, numberType); properties.set(CDC_UPDATED_AT, stringType); properties.set(CDC_DELETED_AT, stringType); + properties.set(CDC_DEFAULT_CURSOR, airbyteIntegerType); return stream; } @@ -175,6 +195,7 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { .map(MySqlSource::overrideSyncModes) .map(MySqlSource::removeIncrementalWithoutPk) .map(MySqlSource::setIncrementalToSourceDefined) + .map(MySqlSource::setDefaultCursorFieldForCdc) .map(MySqlSource::addCdcMetadataColumns) .collect(toList()); @@ -184,6 +205,46 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { return catalog; } + @Override + public Collection> readStreams(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final JsonNode state) + throws Exception { + final AirbyteStateType supportedStateType = getSupportedStateType(config); + final StateManager stateManager = + StateManagerFactory.createStateManager(supportedStateType, + StateGeneratorUtils.deserializeInitialState(state, featureFlags.useStreamCapableState(), supportedStateType), catalog); + final Instant emittedAt = Instant.now(); + + final JdbcDatabase database = createDatabase(config); + + logPreSyncDebugData(database, catalog); + + final Map>> fullyQualifiedTableNameToInfo = + discoverWithoutSystemTables(database) + .stream() + .collect(Collectors.toMap(t -> String.format("%s.%s", t.getNameSpace(), t.getName()), + Function + .identity())); + + validateCursorFieldForIncrementalTables(fullyQualifiedTableNameToInfo, catalog, database); + + DbSourceDiscoverUtil.logSourceSchemaChange(fullyQualifiedTableNameToInfo, catalog, this::getAirbyteType); + + final List> incrementalIterators = + getIncrementalIterators(database, catalog, fullyQualifiedTableNameToInfo, stateManager, + emittedAt); + final List> fullRefreshIterators = + getFullRefreshIterators(database, catalog, fullyQualifiedTableNameToInfo, stateManager, + emittedAt); + final List> iteratorList = Stream + .of(incrementalIterators, fullRefreshIterators) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + return iteratorList; + } + @Override public JsonNode toDatabaseConfig(final JsonNode config) { final String encodedDatabaseName = HostPortResolver.encodeValue(config.get(JdbcUtils.DATABASE_KEY).asText()); @@ -271,14 +332,19 @@ public List> getIncrementalIterators(final final StateManager stateManager, final Instant emittedAt) { final JsonNode sourceConfig = database.getSourceConfig(); + final MySqlFeatureFlags featureFlags = new MySqlFeatureFlags(sourceConfig); if (isCdc(sourceConfig) && shouldUseCDC(catalog)) { + if (featureFlags.isCdcInitialSyncViaPkEnabled()) { + LOGGER.info("Using PK + CDC"); + return MySqlInitialReadUtil.getCdcReadIterators(database, catalog, tableNameToTable, stateManager, emittedAt, getQuoteString()); + } final Duration firstRecordWaitTime = FirstRecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(sourceConfig, MySqlCdcTargetPosition.targetPosition(database), true, firstRecordWaitTime, OptionalInt.empty()); final MySqlCdcStateHandler mySqlCdcStateHandler = new MySqlCdcStateHandler(stateManager); - final MySqlCdcConnectorMetadataInjector mySqlCdcConnectorMetadataInjector = new MySqlCdcConnectorMetadataInjector(); + final MySqlCdcConnectorMetadataInjector mySqlCdcConnectorMetadataInjector = MySqlCdcConnectorMetadataInjector.getInstance(emittedAt); final List streamsToSnapshot = identifyStreamsToSnapshot(catalog, stateManager); final Optional cdcState = Optional.ofNullable(stateManager.getCdcStateManager().getCdcState()); @@ -286,7 +352,7 @@ public List> getIncrementalIterators(final final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, new MySqlCdcSavedInfoFetcher(cdcState.orElse(null)), new MySqlCdcStateHandler(stateManager), - new MySqlCdcConnectorMetadataInjector(), + mySqlCdcConnectorMetadataInjector, MySqlCdcProperties.getDebeziumProperties(database), emittedAt, false); diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlFeatureFlags.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlFeatureFlags.java new file mode 100644 index 000000000000..752406db8c78 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlFeatureFlags.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; + +// Feature flags to gate new primary key load features. +public class MySqlFeatureFlags { + + public static final String CDC_VIA_PK = "cdc_via_pk"; + private final JsonNode sourceConfig; + + public MySqlFeatureFlags(final JsonNode sourceConfig) { + this.sourceConfig = sourceConfig; + } + + public boolean isCdcInitialSyncViaPkEnabled() { + return getFlagValue(CDC_VIA_PK); + } + + private boolean getFlagValue(final String flag) { + return sourceConfig.has(flag) && sourceConfig.get(flag).asBoolean(); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java new file mode 100644 index 000000000000..0b27f40956f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.InitialLoadStreams; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.PrimaryKeyInfo; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class MySqlInitialLoadGlobalStateManager implements MySqlInitialLoadStateManager { + + private final Map pairToPrimaryKeyLoadStatus; + // Map of pair to the primary key info (field name & data type) associated with it. + private final Map pairToPrimaryKeyInfo; + private final CdcState cdcState; + + // Only one global state is emitted, which is fanned out into many entries in the DB by platform. As + // a result, we need to keep track of streams that + // have completed the snapshot. + private final Set streamsThatHaveCompletedSnapshot; + + MySqlInitialLoadGlobalStateManager(final InitialLoadStreams initialLoadStreams, + final Map pairToPrimaryKeyInfo, + final CdcState cdcState, + final ConfiguredAirbyteCatalog catalog) { + this.cdcState = cdcState; + this.pairToPrimaryKeyLoadStatus = initPairToPrimaryKeyLoadStatusMap(initialLoadStreams.pairToInitialLoadStatus()); + this.pairToPrimaryKeyInfo = pairToPrimaryKeyInfo; + this.streamsThatHaveCompletedSnapshot = initStreamsCompletedSnapshot(initialLoadStreams, catalog); + } + + private static Set initStreamsCompletedSnapshot(final InitialLoadStreams initialLoadStreams, + final ConfiguredAirbyteCatalog catalog) { + final Set streamsThatHaveCompletedSnapshot = new HashSet<>(); + catalog.getStreams().forEach(configuredAirbyteStream -> { + if (!initialLoadStreams.streamsForInitialLoad().contains(configuredAirbyteStream) + && configuredAirbyteStream.getSyncMode() == SyncMode.INCREMENTAL) { + streamsThatHaveCompletedSnapshot.add( + new AirbyteStreamNameNamespacePair(configuredAirbyteStream.getStream().getName(), configuredAirbyteStream.getStream().getNamespace())); + } + }); + return streamsThatHaveCompletedSnapshot; + } + + private static Map initPairToPrimaryKeyLoadStatusMap( + final Map pairToPkStatus) { + final Map map = new HashMap<>(); + pairToPkStatus.forEach((pair, pkStatus) -> { + final AirbyteStreamNameNamespacePair updatedPair = new AirbyteStreamNameNamespacePair(pair.getName(), pair.getNamespace()); + map.put(updatedPair, pkStatus); + }); + return map; + } + + public AirbyteStateMessage createIntermediateStateMessage(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus) { + final List streamStates = new ArrayList<>(); + streamsThatHaveCompletedSnapshot.forEach(stream -> { + final DbStreamState state = getFinalState(stream); + streamStates.add(getAirbyteStreamState(stream, Jsons.jsonNode(state))); + + }); + streamStates.add(getAirbyteStreamState(pair, (Jsons.jsonNode(pkLoadStatus)))); + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(cdcState)); + globalState.setStreamStates(streamStates); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + @Override + public void updatePrimaryKeyLoadState(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus) { + pairToPrimaryKeyLoadStatus.put(pair, pkLoadStatus); + } + + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + streamsThatHaveCompletedSnapshot.add(pair); + final List streamStates = new ArrayList<>(); + streamsThatHaveCompletedSnapshot.forEach(stream -> { + final DbStreamState state = getFinalState(stream); + streamStates.add(getAirbyteStreamState(stream, Jsons.jsonNode(state))); + }); + + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(cdcState)); + globalState.setStreamStates(streamStates); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + @Override + public PrimaryKeyLoadStatus getPrimaryKeyLoadStatus(final AirbyteStreamNameNamespacePair pair) { + return pairToPrimaryKeyLoadStatus.get(pair); + } + + @Override + public PrimaryKeyInfo getPrimaryKeyInfo(final AirbyteStreamNameNamespacePair pair) { + return pairToPrimaryKeyInfo.get(pair); + } + + private AirbyteStreamState getAirbyteStreamState(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, final JsonNode stateData) { + assert Objects.nonNull(pair); + assert Objects.nonNull(pair.getName()); + assert Objects.nonNull(pair.getNamespace()); + + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(pair.getName()).withNamespace(pair.getNamespace())) + .withStreamState(stateData); + } + + private DbStreamState getFinalState(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair) { + assert Objects.nonNull(pair); + assert Objects.nonNull(pair.getName()); + assert Objects.nonNull(pair.getNamespace()); + + return new DbStreamState() + .withStreamName(pair.getName()) + .withStreamNamespace(pair.getNamespace()) + .withCursorField(Collections.emptyList()) + .withCursor(null); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java new file mode 100644 index 000000000000..28541d2bca59 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.mysql.cj.MysqlType; +import io.airbyte.commons.stream.AirbyteStreamUtils; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.source.mysql.MySqlQueryUtils.TableSizeInfo; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlInitialLoadHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialLoadHandler.class); + + private static final long RECORD_LOGGING_SAMPLE_RATE = 1_000_000; + private final JsonNode config; + private final JdbcDatabase database; + private final MySqlInitialLoadSourceOperations sourceOperations; + private final String quoteString; + private final MySqlInitialLoadStateManager initialLoadStateManager; + private final Function streamStateForIncrementalRunSupplier; + + private static final long QUERY_TARGET_SIZE_GB = 1_073_741_824; + private static final long DEFAULT_CHUNK_SIZE = 1_000_000; + final Map tableSizeInfoMap; + + public MySqlInitialLoadHandler(final JsonNode config, + final JdbcDatabase database, + final MySqlInitialLoadSourceOperations sourceOperations, + final String quoteString, + final MySqlInitialLoadStateManager initialLoadStateManager, + final Function streamStateForIncrementalRunSupplier, + final Map tableSizeInfoMap) { + this.config = config; + this.database = database; + this.sourceOperations = sourceOperations; + this.quoteString = quoteString; + this.initialLoadStateManager = initialLoadStateManager; + this.streamStateForIncrementalRunSupplier = streamStateForIncrementalRunSupplier; + this.tableSizeInfoMap = tableSizeInfoMap; + } + + public List> getIncrementalIterators( + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final Instant emittedAt) { + final List> iteratorList = new ArrayList<>(); + for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { + final AirbyteStream stream = airbyteStream.getStream(); + final String streamName = stream.getName(); + final String namespace = stream.getNamespace(); + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamName, namespace); + final String fullyQualifiedTableName = DbSourceDiscoverUtil.getFullyQualifiedTableName(namespace, streamName); + if (!tableNameToTable.containsKey(fullyQualifiedTableName)) { + LOGGER.info("Skipping stream {} because it is not in the source", fullyQualifiedTableName); + continue; + } + if (airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) { + // Grab the selected fields to sync + final TableInfo> table = tableNameToTable + .get(fullyQualifiedTableName); + final List selectedDatabaseFields = table.getFields() + .stream() + .map(CommonField::getName) + .filter(CatalogHelpers.getTopLevelFieldNames(airbyteStream)::contains) + .collect(Collectors.toList()); + final AutoCloseableIterator queryStream = + new MySqlInitialLoadRecordIterator(database, sourceOperations, quoteString, initialLoadStateManager, selectedDatabaseFields, pair, + calculateChunkSize(tableSizeInfoMap.get(pair), pair), isCompositePrimaryKey(airbyteStream)); + final AutoCloseableIterator recordIterator = + getRecordIterator(queryStream, streamName, namespace, emittedAt.toEpochMilli()); + final AutoCloseableIterator recordAndMessageIterator = augmentWithState(recordIterator, pair); + + iteratorList.add(augmentWithLogs(recordAndMessageIterator, pair, streamName)); + + } + } + return iteratorList; + } + + private static boolean isCompositePrimaryKey(final ConfiguredAirbyteStream stream) { + return stream.getStream().getSourceDefinedPrimaryKey().size() > 1; + } + + // Calculates the number of rows to fetch per query. + @VisibleForTesting + public static long calculateChunkSize(final TableSizeInfo tableSizeInfo, final AirbyteStreamNameNamespacePair pair) { + // If table size info could not be calculated, a default chunk size will be provided. + if (tableSizeInfo == null || tableSizeInfo.tableSize() == 0 || tableSizeInfo.avgRowLength() == 0) { + LOGGER.info("Chunk size could not be determined for pair: {}, defaulting to {} rows", pair, DEFAULT_CHUNK_SIZE); + return DEFAULT_CHUNK_SIZE; + } + final long avgRowLength = tableSizeInfo.avgRowLength(); + final long chunkSize = QUERY_TARGET_SIZE_GB / avgRowLength; + LOGGER.info("Chunk size determined for pair: {}, is {}", pair, chunkSize); + return chunkSize; + } + + // Transforms the given iterator to create an {@link AirbyteRecordMessage} + private AutoCloseableIterator getRecordIterator( + final AutoCloseableIterator recordIterator, + final String streamName, + final String namespace, + final long emittedAt) { + return AutoCloseableIterators.transform(recordIterator, r -> new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(streamName) + .withNamespace(namespace) + .withEmittedAt(emittedAt) + .withData(r))); + } + + // Augments the given iterator with record count logs. + private AutoCloseableIterator augmentWithLogs(final AutoCloseableIterator iterator, + final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, + final String streamName) { + final AtomicLong recordCount = new AtomicLong(); + return AutoCloseableIterators.transform(iterator, + AirbyteStreamUtils.convertFromNameAndNamespace(pair.getName(), pair.getNamespace()), + r -> { + final long count = recordCount.incrementAndGet(); + if (count % RECORD_LOGGING_SAMPLE_RATE == 0) { + LOGGER.info("Reading stream {}. Records read: {}", streamName, count); + } + return r; + }); + } + + private AutoCloseableIterator augmentWithState(final AutoCloseableIterator recordIterator, + final AirbyteStreamNameNamespacePair pair) { + + final PrimaryKeyLoadStatus currentPkLoadStatus = initialLoadStateManager.getPrimaryKeyLoadStatus(pair); + final JsonNode incrementalState = + (currentPkLoadStatus == null || currentPkLoadStatus.getIncrementalState() == null) ? streamStateForIncrementalRunSupplier.apply(pair) + : currentPkLoadStatus.getIncrementalState(); + + final Duration syncCheckpointDuration = + config.get("sync_checkpoint_seconds") != null ? Duration.ofSeconds(config.get("sync_checkpoint_seconds").asLong()) + : MySqlInitialSyncStateIterator.SYNC_CHECKPOINT_DURATION; + final Long syncCheckpointRecords = config.get("sync_checkpoint_records") != null ? config.get("sync_checkpoint_records").asLong() + : MySqlInitialSyncStateIterator.SYNC_CHECKPOINT_RECORDS; + + return AutoCloseableIterators.transformIterator( + r -> new MySqlInitialSyncStateIterator(r, pair, initialLoadStateManager, incrementalState, + syncCheckpointDuration, syncCheckpointRecords), + recordIterator, pair); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java new file mode 100644 index 000000000000..85069edfcfa5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; + +import autovalue.shaded.com.google.common.collect.AbstractIterator; +import com.fasterxml.jackson.databind.JsonNode; +import com.mysql.cj.MysqlType; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.PrimaryKeyInfo; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This record iterator operates over a single stream. It continuously reads data from a table via + * multiple queries with the configured chunk size until the entire table is processed. The next + * query uses the highest watermark of the primary key seen in the previous subquery. Consider a + * table with chunk size = 1,000,000, and 3,500,000 records. The series of queries executed are : + * Query 1 : select * from table order by pk limit 1,800,000, pk_max = pk_max_1 Query 2 : select * + * from table where pk > pk_max_1 order by pk limit 1,800,000, pk_max = pk_max_2 Query 3 : select * + * from table where pk > pk_max_2 order by pk limit 1,800,000, pk_max = pk_max_3 Query 4 : select * + * from table where pk > pk_max_3 order by pk limit 1,800,000, pk_max = pk_max_4 Query 5 : select * + * from table where pk > pk_max_4 order by pk limit 1,800,000. Final query, since there are zero + * records processed here. + */ +public class MySqlInitialLoadRecordIterator extends AbstractIterator + implements AutoCloseableIterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialLoadRecordIterator.class); + + private final MySqlInitialLoadSourceOperations sourceOperations; + + private final String quoteString; + private final MySqlInitialLoadStateManager initialLoadStateManager; + private final List columnNames; + private final AirbyteStreamNameNamespacePair pair; + private final JdbcDatabase database; + // Represents the number of rows to get with each query. + private final long chunkSize; + private final PrimaryKeyInfo pkInfo; + private final boolean isCompositeKeyLoad; + private int numSubqueries = 0; + private AutoCloseableIterator currentIterator; + + MySqlInitialLoadRecordIterator( + final JdbcDatabase database, + final MySqlInitialLoadSourceOperations sourceOperations, + final String quoteString, + final MySqlInitialLoadStateManager initialLoadStateManager, + final List columnNames, + final AirbyteStreamNameNamespacePair pair, + final long chunkSize, + final boolean isCompositeKeyLoad) { + this.database = database; + this.sourceOperations = sourceOperations; + this.quoteString = quoteString; + this.initialLoadStateManager = initialLoadStateManager; + this.columnNames = columnNames; + this.pair = pair; + this.chunkSize = chunkSize; + this.pkInfo = initialLoadStateManager.getPrimaryKeyInfo(pair); + this.isCompositeKeyLoad = isCompositeKeyLoad; + } + + @CheckForNull + @Override + protected JsonNode computeNext() { + if (shouldBuildNextSubquery()) { + try { + // We will only issue one query for a composite key load. If we have already processed all the data + // associated with this + // query, we should indicate that we are done processing for the given stream. + if (isCompositeKeyLoad && numSubqueries >= 1) { + return endOfData(); + } + // Previous stream (and connection) must be manually closed in this iterator. + if (currentIterator != null) { + currentIterator.close(); + } + + LOGGER.info("Subquery number : {}", numSubqueries); + final Stream stream = database.unsafeQuery( + this::getPkPreparedStatement, sourceOperations::rowToJson); + + currentIterator = AutoCloseableIterators.fromStream(stream, pair); + numSubqueries++; + // If the current subquery has no records associated with it, the entire stream has been read. + if (!currentIterator.hasNext()) { + return endOfData(); + } + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + return currentIterator.next(); + } + + private boolean shouldBuildNextSubquery() { + // The next sub-query should be built if (i) it is the first subquery in the sequence. (ii) the + // previous subquery has finished. + return (currentIterator == null || !currentIterator.hasNext()); + } + + private PreparedStatement getPkPreparedStatement(final Connection connection) { + try { + final String tableName = pair.getName(); + final String schemaName = pair.getNamespace(); + LOGGER.info("Preparing query for table: {}", tableName); + final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, + quoteString); + + final String wrappedColumnNames = RelationalDbQueryUtils.enquoteIdentifierList(columnNames, quoteString); + + final PrimaryKeyLoadStatus pkLoadStatus = initialLoadStateManager.getPrimaryKeyLoadStatus(pair); + + if (pkLoadStatus == null) { + LOGGER.info("pkLoadStatus is null"); + final String quotedCursorField = enquoteIdentifier(pkInfo.pkFieldName(), quoteString); + final String sql; + // We cannot load in chunks for a composite key load, since each field might not have distinct + // values. + if (isCompositeKeyLoad) { + sql = String.format("SELECT %s FROM %s ORDER BY %s", wrappedColumnNames, fullTableName, + quotedCursorField); + } else { + sql = String.format("SELECT %s FROM %s ORDER BY %s LIMIT %s", wrappedColumnNames, fullTableName, + quotedCursorField, chunkSize); + } + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); + return preparedStatement; + } else { + LOGGER.info("pkLoadStatus value is : {}", pkLoadStatus.getPkVal()); + final String quotedCursorField = enquoteIdentifier(pkLoadStatus.getPkName(), quoteString); + final String sql; + // We cannot load in chunks for a composite key load, since each field might not have distinct + // values. Furthermore, we have to issue a >= + // query since we may not have processed all of the data associated with the last saved primary key + // value. + if (isCompositeKeyLoad) { + sql = String.format("SELECT %s FROM %s WHERE %s >= ? ORDER BY %s", wrappedColumnNames, fullTableName, + quotedCursorField, quotedCursorField); + } else { + // The pk max value could be null - this can happen in the case of empty tables. In this case, we + // can just issue a query + // without any chunking. + if (pkInfo.pkMaxValue() != null) { + sql = String.format("SELECT %s FROM %s WHERE %s > ? AND %s <= ? ORDER BY %s LIMIT %s", wrappedColumnNames, fullTableName, + quotedCursorField, quotedCursorField, quotedCursorField, chunkSize); + } else { + sql = String.format("SELECT %s FROM %s WHERE %s > ? ORDER BY %s", wrappedColumnNames, fullTableName, + quotedCursorField, quotedCursorField); + } + } + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + final MysqlType cursorFieldType = pkInfo.fieldType(); + sourceOperations.setCursorField(preparedStatement, 1, cursorFieldType, pkLoadStatus.getPkVal()); + if (!isCompositeKeyLoad && pkInfo.pkMaxValue() != null) { + sourceOperations.setCursorField(preparedStatement, 2, cursorFieldType, pkInfo.pkMaxValue()); + } + LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); + return preparedStatement; + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + if (currentIterator != null) { + currentIterator.close(); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java new file mode 100644 index 000000000000..69eef78e7515 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.integrations.source.mysql.MySqlCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mysql.MySqlSourceOperations; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Optional; + +public class MySqlInitialLoadSourceOperations extends MySqlSourceOperations { + + private final Optional metadataInjector; + + public MySqlInitialLoadSourceOperations(final Optional metadataInjector) { + super(); + this.metadataInjector = metadataInjector; + } + + @Override + public JsonNode rowToJson(final ResultSet queryContext) throws SQLException { + if (metadataInjector.isPresent()) { + // the first call communicates with the database. after that the result is cached. + final ResultSetMetaData metadata = queryContext.getMetaData(); + final int columnCount = metadata.getColumnCount(); + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + for (int i = 1; i <= columnCount; i++) { + // attempt to access the column. this allows us to know if it is null before we do type-specific + // parsing. if it is null, we can move on. while awkward, this seems to be the agreed upon way of + // checking for null values with jdbc. + queryContext.getObject(i); + if (queryContext.wasNull()) { + continue; + } + + // convert to java types that will convert into reasonable json. + copyToJsonField(queryContext, i, jsonNode); + } + + metadataInjector.get().inject(jsonNode); + return jsonNode; + } else { + return super.rowToJson(queryContext); + } + } + + public static class CdcMetadataInjector { + + private final String transactionTimestamp; + private final MysqlDebeziumStateAttributes stateAttributes; + private final MySqlCdcConnectorMetadataInjector metadataInjector; + + public CdcMetadataInjector(final String transactionTimestamp, + final MysqlDebeziumStateAttributes stateAttributes, + final MySqlCdcConnectorMetadataInjector metadataInjector) { + this.transactionTimestamp = transactionTimestamp; + this.stateAttributes = stateAttributes; + this.metadataInjector = metadataInjector; + } + + private void inject(final ObjectNode record) { + metadataInjector.addMetaDataToRowsFetchedOutsideDebezium(record, transactionTimestamp, stateAttributes); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java new file mode 100644 index 000000000000..f65cc7b270aa --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.PrimaryKeyInfo; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; + +public interface MySqlInitialLoadStateManager { + + long MYSQL_STATUS_VERSION = 2; + String STATE_TYPE_KEY = "state_type"; + String PRIMARY_KEY_STATE_TYPE = "primary_key"; + + // Returns an intermediate state message for the initial sync. + AirbyteStateMessage createIntermediateStateMessage(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus); + + // Updates the {@link PrimaryKeyLoadStatus} for the state associated with the given pair + void updatePrimaryKeyLoadState(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus); + + // Returns the final state message for the initial sync. + AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun); + + // Returns the previous state emitted, represented as a {@link PrimaryKeyLoadStatus} associated with + // the stream. + PrimaryKeyLoadStatus getPrimaryKeyLoadStatus(final AirbyteStreamNameNamespacePair pair); + + // Returns the current {@PrimaryKeyInfo}, associated with the stream. This includes the data type & + // the column name associated with the stream. + PrimaryKeyInfo getPrimaryKeyInfo(final AirbyteStreamNameNamespacePair pair); + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java new file mode 100644 index 000000000000..de3d7376e300 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; +import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.getTableSizeInfoForStreams; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadGlobalStateManager.STATE_TYPE_KEY; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Sets; +import com.mysql.cj.MysqlType; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.integrations.debezium.internals.FirstRecordWaitTimeUtil; +import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcPosition; +import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; +import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; +import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.integrations.source.mysql.MySqlCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mysql.MySqlCdcProperties; +import io.airbyte.integrations.source.mysql.MySqlCdcSavedInfoFetcher; +import io.airbyte.integrations.source.mysql.MySqlCdcStateHandler; +import io.airbyte.integrations.source.mysql.MySqlQueryUtils; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadSourceOperations.CdcMetadataInjector; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlInitialReadUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialReadUtil.class); + + /* + * Returns the read iterators associated with : 1. Initial cdc read snapshot via primary key + * queries. 2. Incremental cdc reads via debezium. + * + * The initial load iterators need to always be run before the incremental cdc iterators. This is to + * prevent advancing the binlog offset in the state before all streams have snapshotted. Otherwise, + * there could be data loss. + */ + public static List> getCdcReadIterators(final JdbcDatabase database, + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final StateManager stateManager, + final Instant emittedAt, + final String quoteString) { + final JsonNode sourceConfig = database.getSourceConfig(); + final Duration firstRecordWaitTime = FirstRecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); + LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); + // Determine the streams that need to be loaded via primary key sync. + final List> initialLoadIterator = new ArrayList<>(); + + // Construct the initial state for MySQL. If there is already existing state, we use that instead + // since that is associated with the debezium + // state associated with the initial sync. + final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); + final JsonNode initialDebeziumState = mySqlDebeziumStateUtil.constructInitialDebeziumState( + MySqlCdcProperties.getDebeziumProperties(database), catalog, database); + + final JsonNode state = + (stateManager.getCdcStateManager().getCdcState() == null || stateManager.getCdcStateManager().getCdcState().getState() == null) + ? initialDebeziumState + : Jsons.clone(stateManager.getCdcStateManager().getCdcState().getState()); + + final Optional savedOffset = mySqlDebeziumStateUtil.savedOffset( + MySqlCdcProperties.getDebeziumProperties(database), catalog, state.get(MYSQL_CDC_OFFSET), sourceConfig); + + final boolean savedOffsetStillPresentOnServer = + savedOffset.isPresent() && mySqlDebeziumStateUtil.savedOffsetStillPresentOnServer(database, savedOffset.get()); + + if (!savedOffsetStillPresentOnServer) { + LOGGER.warn("Saved offset no longer present on the server, Airbtye is going to trigger a sync from scratch"); + } + + final InitialLoadStreams initialLoadStreams = streamsForInitialPrimaryKeyLoad(stateManager.getCdcStateManager(), catalog, + savedOffsetStillPresentOnServer); + + final CdcState stateToBeUsed = (!savedOffsetStillPresentOnServer || (stateManager.getCdcStateManager().getCdcState() == null + || stateManager.getCdcStateManager().getCdcState().getState() == null)) ? new CdcState().withState(initialDebeziumState) + : stateManager.getCdcStateManager().getCdcState(); + + final MySqlCdcConnectorMetadataInjector metadataInjector = MySqlCdcConnectorMetadataInjector.getInstance(emittedAt); + + // If there are streams to sync via primary key load, build the relevant iterators. + if (!initialLoadStreams.streamsForInitialLoad().isEmpty()) { + LOGGER.info("Streams to be synced via primary key : {}", initialLoadStreams.streamsForInitialLoad().size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(initialLoadStreams.streamsForInitialLoad())); + final MySqlInitialLoadStateManager initialLoadStateManager = + new MySqlInitialLoadGlobalStateManager(initialLoadStreams, + initPairToPrimaryKeyInfoMap(database, initialLoadStreams, tableNameToTable, quoteString), + stateToBeUsed, catalog); + final MysqlDebeziumStateAttributes stateAttributes = MySqlDebeziumStateUtil.getStateAttributesFromDB(database); + + final MySqlInitialLoadSourceOperations sourceOperations = + new MySqlInitialLoadSourceOperations( + Optional.of(new CdcMetadataInjector(emittedAt.toString(), stateAttributes, metadataInjector))); + final MySqlInitialLoadHandler initialLoadHandler = new MySqlInitialLoadHandler(sourceConfig, database, + sourceOperations, + quoteString, + initialLoadStateManager, + namespacePair -> Jsons.emptyObject(), + getTableSizeInfoForStreams(database, catalog.getStreams(), quoteString)); + + initialLoadIterator.addAll(initialLoadHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(initialLoadStreams.streamsForInitialLoad()), + tableNameToTable, + emittedAt)); + } else { + LOGGER.info("No streams will be synced via primary key"); + } + + // Build the incremental CDC iterators. + final AirbyteDebeziumHandler handler = + new AirbyteDebeziumHandler<>(sourceConfig, MySqlCdcTargetPosition.targetPosition(database), true, firstRecordWaitTime, OptionalInt.empty()); + + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, + new MySqlCdcSavedInfoFetcher(stateToBeUsed), + new MySqlCdcStateHandler(stateManager), + metadataInjector, + MySqlCdcProperties.getDebeziumProperties(database), + emittedAt, + false); + + // This starts processing the binglogs as soon as initial sync is complete, this is a bit different + // from the current cdc syncs. + // We finish the current CDC once the initial snapshot is complete and the next sync starts + // processing the binlogs + return Collections.singletonList( + AutoCloseableIterators.concatWithEagerClose( + Stream + .of(initialLoadIterator, Collections.singletonList(AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null))) + .flatMap(Collection::stream) + .collect(Collectors.toList()), + AirbyteTraceMessageUtility::emitStreamStatusTrace)); + } + + /** + * Determines the streams to sync for initial primary key load. These include streams that are (i) + * currently in primary key load (ii) newly added incremental streams. + */ + public static InitialLoadStreams streamsForInitialPrimaryKeyLoad(final CdcStateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog, + final boolean savedOffsetStillPresentOnServer) { + if (!savedOffsetStillPresentOnServer) { + return new InitialLoadStreams( + fullCatalog.getStreams() + .stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .collect(Collectors.toList()), + new HashMap<>()); + } + + final AirbyteStateMessage airbyteStateMessage = stateManager.getRawStateMessage(); + final Set streamsStillinPkSync = new HashSet<>(); + + // Build a map of stream <-> initial load status for streams that currently have an initial primary + // key load in progress. + final Map pairToInitialLoadStatus = new HashMap<>(); + if (airbyteStateMessage != null && airbyteStateMessage.getGlobal() != null && airbyteStateMessage.getGlobal().getStreamStates() != null) { + airbyteStateMessage.getGlobal().getStreamStates().forEach(stateMessage -> { + final JsonNode streamState = stateMessage.getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase(PRIMARY_KEY_STATE_TYPE)) { + final PrimaryKeyLoadStatus primaryKeyLoadStatus = Jsons.object(streamState, PrimaryKeyLoadStatus.class); + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + pairToInitialLoadStatus.put(pair, primaryKeyLoadStatus); + streamsStillinPkSync.add(pair); + } + } + }); + } + + final List streamsForPkSync = new ArrayList<>(); + fullCatalog.getStreams().stream() + .filter(stream -> streamsStillinPkSync.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .forEach(streamsForPkSync::add); + final List newlyAddedStreams = identifyStreamsToSnapshot(fullCatalog, stateManager.getInitialStreamsSynced()); + streamsForPkSync.addAll(newlyAddedStreams); + + return new InitialLoadStreams(streamsForPkSync, pairToInitialLoadStatus); + } + + private static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, + final Set alreadySyncedStreams) { + final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog); + final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); + return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))).map(Jsons::clone) + .collect(Collectors.toList()); + } + + // Build a map of stream <-> primary key info (primary key field name + datatype) for all streams + // currently undergoing initial primary key syncs. + private static Map initPairToPrimaryKeyInfoMap( + final JdbcDatabase database, + final InitialLoadStreams initialLoadStreams, + final Map>> tableNameToTable, + final String quoteString) { + final Map pairToPkInfoMap = new HashMap<>(); + // For every stream that is in primary initial key sync, we want to maintain information about the + // current primary key info associated with the + // stream + initialLoadStreams.streamsForInitialLoad().forEach(stream -> { + final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair = + new io.airbyte.protocol.models.AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + final PrimaryKeyInfo pkInfo = getPrimaryKeyInfo(database, stream, tableNameToTable, quoteString); + pairToPkInfoMap.put(pair, pkInfo); + }); + return pairToPkInfoMap; + } + + // Returns the primary key info associated with the stream. + private static PrimaryKeyInfo getPrimaryKeyInfo(final JdbcDatabase database, + final ConfiguredAirbyteStream stream, + final Map>> tableNameToTable, + final String quoteString) { + // For cursor-based syncs, we cannot always assume a primary key field exists. We need to handle the + // case where it does not exist when we support + // cursor-based syncs. + if (stream.getStream().getSourceDefinedPrimaryKey().size() > 1) { + LOGGER.info("Composite primary key detected for {namespace, stream} : {}, {}", stream.getStream().getNamespace(), stream.getStream().getName()); + } + final String pkFieldName = stream.getStream().getSourceDefinedPrimaryKey().get(0).get(0); + final String fullyQualifiedTableName = + DbSourceDiscoverUtil.getFullyQualifiedTableName(stream.getStream().getNamespace(), (stream.getStream().getName())); + final TableInfo> table = tableNameToTable + .get(fullyQualifiedTableName); + final MysqlType pkFieldType = table.getFields().stream() + .filter(field -> field.getName().equals(pkFieldName)) + .findFirst().get().getType(); + + final String pkMaxValue = MySqlQueryUtils.getMaxPkValueForStream(database, stream, pkFieldName, quoteString); + return new PrimaryKeyInfo(pkFieldName, pkFieldType, pkMaxValue); + } + + public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { + return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); + } + + public record InitialLoadStreams(List streamsForInitialLoad, + Map pairToInitialLoadStatus) { + + } + + public record PrimaryKeyInfo(String pkFieldName, MysqlType fieldType, String pkMaxValue) {} + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java new file mode 100644 index 000000000000..25cc9f72329e --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.MYSQL_STATUS_VERSION; + +import autovalue.shaded.com.google.common.collect.AbstractIterator; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Iterator; +import java.util.Objects; +import javax.annotation.CheckForNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlInitialSyncStateIterator extends AbstractIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialSyncStateIterator.class); + public static final Duration SYNC_CHECKPOINT_DURATION = Duration.ofMinutes(15); + public static final Integer SYNC_CHECKPOINT_RECORDS = 100_000; + + private final Iterator messageIterator; + private final AirbyteStreamNameNamespacePair pair; + private boolean hasEmittedFinalState = false; + private PrimaryKeyLoadStatus pkStatus; + private final JsonNode streamStateForIncrementalRun; + private final MySqlInitialLoadStateManager stateManager; + private long recordCount = 0L; + private Instant lastCheckpoint = Instant.now(); + private final Duration syncCheckpointDuration; + private final Long syncCheckpointRecords; + private final String pkFieldName; + + public MySqlInitialSyncStateIterator(final Iterator messageIterator, + final AirbyteStreamNameNamespacePair pair, + final MySqlInitialLoadStateManager stateManager, + final JsonNode streamStateForIncrementalRun, + final Duration checkpointDuration, + final Long checkpointRecords) { + this.messageIterator = messageIterator; + this.pair = pair; + this.stateManager = stateManager; + this.streamStateForIncrementalRun = streamStateForIncrementalRun; + this.syncCheckpointDuration = checkpointDuration; + this.syncCheckpointRecords = checkpointRecords; + this.pkFieldName = stateManager.getPrimaryKeyInfo(pair).pkFieldName(); + this.pkStatus = stateManager.getPrimaryKeyLoadStatus(pair); + } + + @CheckForNull + @Override + protected AirbyteMessage computeNext() { + if (messageIterator.hasNext()) { + if ((recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) + && Objects.nonNull(pkStatus)) { + LOGGER.info("Emitting initial sync pk state for stream {}, state is {}", pair, pkStatus); + recordCount = 0L; + lastCheckpoint = Instant.now(); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateManager.createIntermediateStateMessage(pair, pkStatus)); + } + // Use try-catch to catch Exception that could occur when connection to the database fails + try { + final AirbyteMessage message = messageIterator.next(); + if (Objects.nonNull(message)) { + final String lastPk = message.getRecord().getData().get(pkFieldName).asText(); + pkStatus = new PrimaryKeyLoadStatus() + .withVersion(MYSQL_STATUS_VERSION) + .withStateType(StateType.PRIMARY_KEY) + .withPkName(pkFieldName) + .withPkVal(lastPk) + .withIncrementalState(streamStateForIncrementalRun); + stateManager.updatePrimaryKeyLoadState(pair, pkStatus); + } + recordCount++; + return message; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } else if (!hasEmittedFinalState) { + hasEmittedFinalState = true; + final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); + LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(finalStateMessage); + } else { + return endOfData(); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml b/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml new file mode 100644 index 000000000000..748d2a8f54c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml @@ -0,0 +1,48 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +title: MySQL Models +type: object +description: MySQL Models +properties: + state_type: + "$ref": "#/definitions/StateType" + primary_key_state: + "$ref": "#/definitions/PrimaryKeyLoadStatus" + cursor_based_state: + "$ref": "#/definitions/CursorBasedStatus" +definitions: + StateType: + description: Enum to define the sync mode of state. + type: string + enum: + - cursor_based + - primary_key + CursorBasedStatus: + type: object + extends: + type: object + existingJavaType: "io.airbyte.integrations.source.relationaldb.models.DbStreamState" + properties: + state_type: + "$ref": "#/definitions/StateType" + version: + description: Version of state. + type: integer + PrimaryKeyLoadStatus: + type: object + properties: + version: + description: Version of state. + type: integer + state_type: + "$ref": "#/definitions/StateType" + pk_name: + description: primary key name + type: string + pk_val: + description: primary key watermark + type: string + incremental_state: + description: State to switch to after completion of primary key initial sync + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index cc949a89b20b..4fada5c46e56 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -39,7 +39,8 @@ "title": "Password", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 4, + "always_show": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", @@ -178,25 +179,14 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "Replication method to use for extracting data from the database.", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", "order": 8, + "default": "CDC", "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", "required": ["method"], "properties": { "method": { @@ -211,13 +201,27 @@ "default": 300, "min": 120, "max": 1200, - "order": 1 + "order": 1, + "always_show": true }, "server_time_zone": { "type": "string", "title": "Configured server timezone for the MySQL source (Advanced)", "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", - "order": 2 + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json index 06a5e3a7e27b..584333ddb9ae 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json @@ -39,7 +39,8 @@ "title": "Password", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 4, + "always_show": true }, "jdbc_url_params": { "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", @@ -178,25 +179,14 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "Replication method to use for extracting data from the database.", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", "order": 8, + "default": "CDC", "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", "required": ["method"], "properties": { "method": { @@ -211,13 +201,27 @@ "default": 300, "min": 120, "max": 1200, - "order": 1 + "order": 1, + "always_show": true }, "server_time_zone": { "type": "string", "title": "Configured server timezone for the MySQL source (Advanced)", "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", - "order": 2 + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 9f9f00c53004..869aa507db16 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -6,11 +6,12 @@ import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; +import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_DB_HISTORY; +import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_FILE; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_POS; import static io.airbyte.integrations.source.mysql.MySqlSource.DRIVER_CLASS; -import static io.airbyte.integrations.source.mysql.MySqlSource.MYSQL_CDC_OFFSET; -import static io.airbyte.integrations.source.mysql.MySqlSource.MYSQL_DB_HISTORY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -19,6 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; @@ -38,6 +40,7 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; import java.sql.SQLException; import java.util.Collections; import java.util.List; @@ -100,6 +103,7 @@ private void init() { .put("username", container.getUsername()) .put("password", container.getPassword()) .put("replication_method", replicationMethod) + .put("sync_checkpoint_records", 1) .put("is_test", true) .build()); } @@ -116,7 +120,7 @@ private void grantCorrectPermissions() { executeQuery("GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " + container.getUsername() + "@'%';"); } - private void purgeAllBinaryLogs() { + protected void purgeAllBinaryLogs() { executeQuery("RESET MASTER;"); } @@ -155,6 +159,7 @@ protected void assertNullCdcMetaData(final JsonNode data) { assertNull(data.get(CDC_LOG_POS)); assertNull(data.get(CDC_UPDATED_AT)); assertNull(data.get(CDC_DELETED_AT)); + assertNull(data.get(CDC_DEFAULT_CURSOR)); } @Override @@ -162,6 +167,7 @@ protected void assertCdcMetaData(final JsonNode data, final boolean deletedAtNul assertNotNull(data.get(CDC_LOG_FILE)); assertNotNull(data.get(CDC_LOG_POS)); assertNotNull(data.get(CDC_UPDATED_AT)); + assertNotNull(data.get(CDC_DEFAULT_CURSOR)); if (deletedAtNull) { assertTrue(data.get(CDC_DELETED_AT).isNull()); } else { @@ -175,6 +181,7 @@ protected void removeCDCColumns(final ObjectNode data) { data.remove(CDC_LOG_POS); data.remove(CDC_UPDATED_AT); data.remove(CDC_DELETED_AT); + data.remove(CDC_DEFAULT_CURSOR); } @Override @@ -182,13 +189,21 @@ protected void addCdcMetadataColumns(final AirbyteStream stream) { final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); final ObjectNode properties = (ObjectNode) jsonSchema.get("properties"); + final JsonNode airbyteIntegerType = Jsons.jsonNode(ImmutableMap.of("type", "number", "airbyte_type", "integer")); final JsonNode numberType = Jsons.jsonNode(ImmutableMap.of("type", "number")); - final JsonNode stringType = Jsons.jsonNode(ImmutableMap.of("type", "string")); properties.set(CDC_LOG_FILE, stringType); properties.set(CDC_LOG_POS, numberType); properties.set(CDC_UPDATED_AT, stringType); properties.set(CDC_DELETED_AT, stringType); + properties.set(CDC_DEFAULT_CURSOR, airbyteIntegerType); + } + + @Override + protected void addCdcDefaultCursorField(final AirbyteStream stream) { + if (stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)) { + stream.setDefaultCursorField(ImmutableList.of(CDC_DEFAULT_CURSOR)); + } } @Override @@ -207,7 +222,9 @@ protected Database getDatabase() { } @Override - public void assertExpectedStateMessages(final List stateMessages) { + protected void assertExpectedStateMessages(final List stateMessages) { + assertEquals(1, stateMessages.size()); + assertNotNull(stateMessages.get(0).getData()); for (final AirbyteStateMessage stateMessage : stateMessages) { assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MYSQL_CDC_OFFSET)); assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MYSQL_DB_HISTORY)); @@ -247,9 +264,7 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); - assertEquals(1, stateAfterFirstBatch.size()); - assertNotNull(stateAfterFirstBatch.get(0).getData()); - assertExpectedStateMessages(stateAfterFirstBatch); + assertStateForSyncShouldHandlePurgedLogsGracefully(stateAfterFirstBatch, 1); final Set recordsFromFirstBatch = extractRecordMessages( dataFromFirstBatch); @@ -274,16 +289,14 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { purgeAllBinaryLogs(); - final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); final AutoCloseableIterator secondBatchIterator = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); - assertEquals(1, stateAfterSecondBatch.size()); - assertNotNull(stateAfterSecondBatch.get(0).getData()); - assertExpectedStateMessages(stateAfterSecondBatch); + assertStateForSyncShouldHandlePurgedLogsGracefully(stateAfterSecondBatch, 2); final Set recordsFromSecondBatch = extractRecordMessages( dataFromSecondBatch); @@ -291,4 +304,49 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { "Expected 46 records to be replicated in the second sync."); } + /** + * This test verifies that multiple states are sent during the CDC process based on number of + * records. We can ensure that more than one `STATE` type of message is sent, but we are not able to + * assert the exact number of messages sent as depends on Debezium. + * + * @throws Exception Exception happening in the test. + */ + @Test + protected void verifyCheckpointStatesByRecords() throws Exception { + // We require a huge amount of records, otherwise Debezium will notify directly the last offset. + final int recordsToCreate = 20000; + + final AutoCloseableIterator firstBatchIterator = getSource() + .read(getConfig(), CONFIGURED_CATALOG, null); + final List dataFromFirstBatch = AutoCloseableIterators + .toListAndClose(firstBatchIterator); + final List stateMessages = extractStateMessages(dataFromFirstBatch); + + // As first `read` operation is from snapshot, it would generate only one state message at the end + // of the process. + assertExpectedStateMessages(stateMessages); + + for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { + final JsonNode record = + Jsons.jsonNode(ImmutableMap + .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, + "F-" + recordsCreated)); + writeModelRecord(record); + } + + final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateMessages.get(stateMessages.size() - 1))); + final AutoCloseableIterator secondBatchIterator = getSource() + .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); + final List dataFromSecondBatch = AutoCloseableIterators + .toListAndClose(secondBatchIterator); + assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); + final List stateMessagesCDC = extractStateMessages(dataFromSecondBatch); + assertTrue(stateMessagesCDC.size() > 1, "Generated only the final state."); + assertEquals(stateMessagesCDC.size(), stateMessagesCDC.stream().distinct().count(), "There are duplicated states."); + } + + protected void assertStateForSyncShouldHandlePurgedLogsGracefully(final List stateMessages, final int syncNumber) { + assertExpectedStateMessages(stateMessages); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/InitialPkLoadEnabledCdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/InitialPkLoadEnabledCdcMysqlSourceTest.java new file mode 100644 index 000000000000..15cd6915e05f --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/InitialPkLoadEnabledCdcMysqlSourceTest.java @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.STATE_TYPE_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Streams; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mysql.initialsync.MySqlFeatureFlags; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +public class InitialPkLoadEnabledCdcMysqlSourceTest extends CdcMysqlSourceTest { + + @Override + protected JsonNode getConfig() { + final JsonNode config = super.getConfig(); + ((ObjectNode) config).put(MySqlFeatureFlags.CDC_VIA_PK, true); + return config; + } + + @Override + protected void assertExpectedStateMessages(final List stateMessages) { + assertEquals(7, stateMessages.size()); + assertStateTypes(stateMessages, 4); + } + + @Override + protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { + super.assertExpectedStateMessages(stateMessages); + } + + @Override + protected void assertStateForSyncShouldHandlePurgedLogsGracefully(final List stateMessages, final int syncNumber) { + if (syncNumber == 1) { + assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(stateMessages); + } else if (syncNumber == 2) { + // Sync number 2 uses the state from sync number 1 but before we trigger the sync 2 we purge the + // binary logs and as a result the validation of + // logs present on the server fails, and we trigger a sync from scratch + assertEquals(47, stateMessages.size()); + assertStateTypes(stateMessages, 44); + } else { + throw new RuntimeException("Unknown sync number"); + } + + } + + @Override + protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(final List stateAfterFirstBatch) { + assertEquals(27, stateAfterFirstBatch.size()); + assertStateTypes(stateAfterFirstBatch, 24); + } + + @Override + protected void assertExpectedStateMessagesForNoData(final List stateMessages) { + assertEquals(2, stateMessages.size()); + } + + private void assertStateTypes(final List stateMessages, final int indexTillWhichExpectPkState) { + JsonNode sharedState = null; + for (int i = 0; i < stateMessages.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + if (i <= indexTillWhichExpectPkState) { + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else { + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } + } + } + + @Override + protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, + final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion) { + assertEquals(7, stateMessages.size()); + for (int i = 0; i <= 4; i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessage.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessage.getGlobal().getSharedState()); + final Set streamsInSnapshotState = stateMessage.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + + stateMessage.getGlobal().getStreamStates().forEach(s -> { + final JsonNode streamState = s.getStreamState(); + if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))) { + assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.get(STATE_TYPE_KEY).asText()); + } else if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))) { + assertFalse(streamState.has(STATE_TYPE_KEY)); + } else { + throw new RuntimeException("Unknown stream"); + } + }); + } + + final AirbyteStateMessage secondLastSateMessage = stateMessages.get(5); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, secondLastSateMessage.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + secondLastSateMessage.getGlobal().getSharedState()); + final Set streamsInSnapshotState = secondLastSateMessage.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + secondLastSateMessage.getGlobal().getStreamStates().forEach(s -> { + final JsonNode streamState = s.getStreamState(); + assertFalse(streamState.has(STATE_TYPE_KEY)); + }); + + final AirbyteStateMessage stateMessageEmittedAfterSecondSyncCompletion = stateMessages.get(6); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSecondSyncCompletion.getType()); + assertNotEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); + final Set streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSyncCompletionState.contains( + new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSyncCompletionState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + assertNotNull(stateMessageEmittedAfterSecondSyncCompletion.getData()); + } + + @Test + public void testCompositeIndexInitialLoad() throws Exception { + // Simulate adding a composite index by modifying the catalog. + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(CONFIGURED_CATALOG); + final List> primaryKeys = configuredCatalog.getStreams().get(0).getStream().getSourceDefinedPrimaryKey(); + primaryKeys.add(List.of("make_id")); + + final AutoCloseableIterator read1 = getSource() + .read(getConfig(), configuredCatalog, null); + + final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); + + final Set recordMessages1 = extractRecordMessages(actualRecords1); + final List stateMessages1 = extractStateMessages(actualRecords1); + assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages1); + assertExpectedStateMessages(stateMessages1); + + // Re-run the sync with state associated with record w/ id = 15 (second to last record). + // We expect to read 2 records, since in the case of a composite PK we issue a >= query. + // We also expect 3 state records. One associated with the pk state, one to signify end of initial + // load, and + // the last one indicating the cdc position we have synced until. + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(4))); + final AutoCloseableIterator read2 = getSource() + .read(getConfig(), configuredCatalog, state); + + final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); + final Set recordMessages2 = extractRecordMessages(actualRecords2); + final List stateMessages2 = extractStateMessages(actualRecords2); + + assertExpectedRecords(new HashSet<>(MODEL_RECORDS.subList(4, 6)), recordMessages2); + assertEquals(3, stateMessages2.size()); + assertStateTypes(stateMessages2, 0); + } + + @Test + public void testTwoStreamSync() throws Exception { + // Add another stream models_2 and read that one as well. + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(CONFIGURED_CATALOG); + + final List MODEL_RECORDS_2 = ImmutableList.of( + Jsons.jsonNode(ImmutableMap.of(COL_ID, 110, COL_MAKE_ID, 1, COL_MODEL, "Fiesta-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 120, COL_MAKE_ID, 1, COL_MODEL, "Focus-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 130, COL_MAKE_ID, 1, COL_MODEL, "Ranger-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 140, COL_MAKE_ID, 2, COL_MODEL, "GLA-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 150, COL_MAKE_ID, 2, COL_MODEL, "A 220-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 160, COL_MAKE_ID, 2, COL_MODEL, "E 350-2"))); + + createTable(MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", + columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); + + for (final JsonNode recordJson : MODEL_RECORDS_2) { + writeRecords(recordJson, MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", COL_ID, + COL_MAKE_ID, COL_MODEL); + } + + final ConfiguredAirbyteStream airbyteStream = new ConfiguredAirbyteStream() + .withStream(CatalogHelpers.createAirbyteStream( + MODELS_STREAM_NAME + "_2", + MODELS_SCHEMA, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), + Field.of(COL_MODEL, JsonSchemaType.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID)))); + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + + final List streams = configuredCatalog.getStreams(); + streams.add(airbyteStream); + configuredCatalog.withStreams(streams); + + final AutoCloseableIterator read1 = getSource() + .read(getConfig(), configuredCatalog, null); + final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); + + final Set recordMessages1 = extractRecordMessages(actualRecords1); + final List stateMessages1 = extractStateMessages(actualRecords1); + assertEquals(13, stateMessages1.size()); + JsonNode sharedState = null; + StreamDescriptor firstStreamInState = null; + for (int i = 0; i < stateMessages1.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages1.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + + if (Objects.isNull(firstStreamInState)) { + assertEquals(1, global.getStreamStates().size()); + firstStreamInState = global.getStreamStates().get(0).getStreamDescriptor(); + } + + if (i <= 4) { + // First 4 state messages are pk state + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else if (i == 5) { + // 5th state message is the final state message emitted for the stream + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } else if (i <= 10) { + // 6th to 10th is the primary_key state message for the 2nd stream but final state message for 1st + // stream + assertEquals(2, global.getStreamStates().size()); + final StreamDescriptor finalFirstStreamInState = firstStreamInState; + global.getStreamStates().forEach(c -> { + if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { + assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); + } else { + assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, c.getStreamState().get(STATE_TYPE_KEY).asText()); + } + }); + } else { + // last 2 state messages don't contain primary_key info cause primary_key sync should be complete + assertEquals(2, global.getStreamStates().size()); + global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); + } + } + + final Set names = new HashSet<>(STREAM_NAMES); + names.add(MODELS_STREAM_NAME + "_2"); + assertExpectedRecords(Streams.concat(MODEL_RECORDS_2.stream(), MODEL_RECORDS.stream()) + .collect(Collectors.toSet()), + recordMessages1, + names, + names, + MODELS_SCHEMA); + + assertEquals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA), firstStreamInState); + + // Triggering a sync with a primary_key state for 1 stream and complete state for other stream + final AutoCloseableIterator read2 = getSource() + .read(getConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateMessages1.get(6)))); + final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); + + final List stateMessages2 = extractStateMessages(actualRecords2); + + assertEquals(6, stateMessages2.size()); + for (int i = 0; i < stateMessages2.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages2.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + assertEquals(2, global.getStreamStates().size()); + + if (i <= 3) { + final StreamDescriptor finalFirstStreamInState = firstStreamInState; + global.getStreamStates().forEach(c -> { + // First 4 state messages are primary_key state for the stream that didn't complete primary_key sync + // the first time + if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { + assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); + } else { + assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, c.getStreamState().get(STATE_TYPE_KEY).asText()); + } + }); + } else { + // last 2 state messages don't contain primary_key info cause primary_key sync should be complete + global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); + } + } + + final Set recordMessages2 = extractRecordMessages(actualRecords2); + assertEquals(5, recordMessages2.size()); + assertExpectedRecords(new HashSet<>(MODEL_RECORDS_2.subList(1, MODEL_RECORDS_2.size())), + recordMessages2, + names, + names, + MODELS_SCHEMA); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlInitialLoadHandlerTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlInitialLoadHandlerTest.java new file mode 100644 index 000000000000..7f09a54fcffb --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlInitialLoadHandlerTest.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.integrations.source.mysql.MySqlQueryUtils.TableSizeInfo; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadHandler; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import org.junit.jupiter.api.Test; + +public class MySqlInitialLoadHandlerTest { + + private static final long ONE_GB = 1_073_741_824; + private static final long ONE_MB = 1_048_576; + + @Test + void testInvalidOrNullTableSizeInfo() { + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair("table_name", "schema_name"); + assertEquals(MySqlInitialLoadHandler.calculateChunkSize(null, pair), 1_000_000L); + final TableSizeInfo invalidRowLengthInfo = new TableSizeInfo(ONE_GB, 0L); + assertEquals(MySqlInitialLoadHandler.calculateChunkSize(invalidRowLengthInfo, pair), 1_000_000L); + final TableSizeInfo invalidTableSizeInfo = new TableSizeInfo(0L, 0L); + assertEquals(MySqlInitialLoadHandler.calculateChunkSize(invalidTableSizeInfo, pair), 1_000_000L); + } + + @Test + void testTableSizeInfo() { + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair("table_name", "schema_name"); + assertEquals(MySqlInitialLoadHandler.calculateChunkSize(new TableSizeInfo(ONE_GB, 2 * ONE_MB), pair), 512L); + assertEquals(MySqlInitialLoadHandler.calculateChunkSize(new TableSizeInfo(ONE_GB, 200L), pair), 5368709L); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 29d1021f9161..09bea61fd2ba 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -18,6 +18,8 @@ import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource.PrimaryKeyAttributesFromDb; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -29,6 +31,8 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -230,4 +234,25 @@ public void testJDBCSessionVariable() throws Exception { } } + @Test + public void testPrimaryKeyOrder() { + final List primaryKeys = new ArrayList<>(); + primaryKeys.add(new PrimaryKeyAttributesFromDb("stream-a", "col1", 3)); + primaryKeys.add(new PrimaryKeyAttributesFromDb("stream-b", "xcol1", 3)); + primaryKeys.add(new PrimaryKeyAttributesFromDb("stream-a", "col2", 2)); + primaryKeys.add(new PrimaryKeyAttributesFromDb("stream-b", "xcol2", 2)); + primaryKeys.add(new PrimaryKeyAttributesFromDb("stream-a", "col3", 1)); + primaryKeys.add(new PrimaryKeyAttributesFromDb("stream-b", "xcol3", 1)); + + final Map> result = AbstractJdbcSource.aggregatePrimateKeys(primaryKeys); + assertEquals(2, result.size()); + assertTrue(result.containsKey("stream-a")); + assertEquals(3, result.get("stream-a").size()); + assertEquals(Arrays.asList("col3", "col2", "col1"), result.get("stream-a")); + + assertTrue(result.containsKey("stream-b")); + assertEquals(3, result.get("stream-b").size()); + assertEquals(Arrays.asList("xcol3", "xcol2", "xcol1"), result.get("stream-b")); + } + } diff --git a/airbyte-integrations/connectors/source-n8n/metadata.yaml b/airbyte-integrations/connectors/source-n8n/metadata.yaml index f757d3268c3d..f34328d9ee75 100644 --- a/airbyte-integrations/connectors/source-n8n/metadata.yaml +++ b/airbyte-integrations/connectors/source-n8n/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-n8n/requirements.txt b/airbyte-integrations/connectors/source-n8n/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-n8n/requirements.txt +++ b/airbyte-integrations/connectors/source-n8n/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-n8n/setup.py b/airbyte-integrations/connectors/source-n8n/setup.py index dcdaf24e3948..82cdff7f8589 100644 --- a/airbyte-integrations/connectors/source-n8n/setup.py +++ b/airbyte-integrations/connectors/source-n8n/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-nasa/metadata.yaml b/airbyte-integrations/connectors/source-nasa/metadata.yaml index 46d403c78544..ebc388c7b2e1 100644 --- a/airbyte-integrations/connectors/source-nasa/metadata.yaml +++ b/airbyte-integrations/connectors/source-nasa/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/nasa tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-nasa/requirements.txt b/airbyte-integrations/connectors/source-nasa/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-nasa/requirements.txt +++ b/airbyte-integrations/connectors/source-nasa/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-nasa/setup.py b/airbyte-integrations/connectors/source-nasa/setup.py index 04c3da58a72e..eb9ee522c041 100644 --- a/airbyte-integrations/connectors/source-nasa/setup.py +++ b/airbyte-integrations/connectors/source-nasa/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-netsuite/metadata.yaml b/airbyte-integrations/connectors/source-netsuite/metadata.yaml index 69c4a5d0b8a3..c2451ba1bfa3 100644 --- a/airbyte-integrations/connectors/source-netsuite/metadata.yaml +++ b/airbyte-integrations/connectors/source-netsuite/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 4f2f093d-ce44-4121-8118-9d13b7bfccd0 dockerImageTag: 0.1.3 dockerRepository: airbyte/source-netsuite + documentationUrl: https://docs.airbyte.com/integrations/sources/netsuite githubIssueLabel: source-netsuite icon: netsuite.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/netsuite + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-netsuite/requirements.txt b/airbyte-integrations/connectors/source-netsuite/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-netsuite/requirements.txt +++ b/airbyte-integrations/connectors/source-netsuite/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-netsuite/setup.py b/airbyte-integrations/connectors/source-netsuite/setup.py index b6b1e8ae3032..42288908347c 100644 --- a/airbyte-integrations/connectors/source-netsuite/setup.py +++ b/airbyte-integrations/connectors/source-netsuite/setup.py @@ -13,7 +13,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-news-api/metadata.yaml b/airbyte-integrations/connectors/source-news-api/metadata.yaml index 042813369483..d33ea74b3d18 100644 --- a/airbyte-integrations/connectors/source-news-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-news-api/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-news-api/requirements.txt b/airbyte-integrations/connectors/source-news-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-news-api/requirements.txt +++ b/airbyte-integrations/connectors/source-news-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-news-api/setup.py b/airbyte-integrations/connectors/source-news-api/setup.py index 8bcca857d65f..733fc131e9ab 100644 --- a/airbyte-integrations/connectors/source-news-api/setup.py +++ b/airbyte-integrations/connectors/source-news-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-news-api/source_news_api/manifest.yaml b/airbyte-integrations/connectors/source-news-api/source_news_api/manifest.yaml index 6b2a23606a7e..d6f8f563e61b 100644 --- a/airbyte-integrations/connectors/source-news-api/source_news_api/manifest.yaml +++ b/airbyte-integrations/connectors/source-news-api/source_news_api/manifest.yaml @@ -62,32 +62,32 @@ definitions: properties: title: type: - - 'null' - - string + - "null" + - string author: type: - - 'null' - - string + - "null" + - string publishedAt: type: - - 'null' - - string + - "null" + - string format: date-time source: type: object properties: Name: type: - - 'null' - - string + - "null" + - string Id: type: - - 'null' - - string + - "null" + - string url: type: - - 'null' - - string + - "null" + - string top_headlines_stream: $ref: "#/definitions/base_stream" $parameters: @@ -104,32 +104,32 @@ definitions: properties: title: type: - - 'null' - - string + - "null" + - string author: type: - - 'null' - - string + - "null" + - string publishedAt: type: - - 'null' - - string + - "null" + - string format: date-time source: type: object properties: Name: type: - - 'null' - - string + - "null" + - string Id: type: - - 'null' - - string + - "null" + - string url: type: - - 'null' - - string + - "null" + - string streams: - "#/definitions/everything_stream" @@ -332,4 +332,4 @@ spec: - popularity - publishedAt default: publishedAt - order: 11 \ No newline at end of file + order: 11 diff --git a/airbyte-integrations/connectors/source-newsdata/metadata.yaml b/airbyte-integrations/connectors/source-newsdata/metadata.yaml index 09f85d532431..e27f67670526 100644 --- a/airbyte-integrations/connectors/source-newsdata/metadata.yaml +++ b/airbyte-integrations/connectors/source-newsdata/metadata.yaml @@ -17,4 +17,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-newsdata/requirements.txt b/airbyte-integrations/connectors/source-newsdata/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-newsdata/requirements.txt +++ b/airbyte-integrations/connectors/source-newsdata/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-newsdata/setup.py b/airbyte-integrations/connectors/source-newsdata/setup.py index 4134511954a2..a7bc19daa4ba 100644 --- a/airbyte-integrations/connectors/source-newsdata/setup.py +++ b/airbyte-integrations/connectors/source-newsdata/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile index a164ee0b7efb..3a69375ac3c8 100644 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -34,5 +34,5 @@ COPY source_notion ./source_notion ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.6 +LABEL io.airbyte.version=1.1.1 LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/README.md b/airbyte-integrations/connectors/source-notion/README.md index bbfcf4c969e1..5f65c1302bfb 100644 --- a/airbyte-integrations/connectors/source-notion/README.md +++ b/airbyte-integrations/connectors/source-notion/README.md @@ -127,7 +127,7 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? 1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +3. Create a Pull Request. +4. Pat yourself on the back for being an awesome contributor. +5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl index 032c324115f1..335e3021715c 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl @@ -1,22 +1,10 @@ -{"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1671456048322} -{"stream": "users", "data": {"object": "user", "id": "b14821de-846b-4719-8a94-2453eeafa609", "name": "Iryna Grankova", "avatar_url": null, "type": "person", "person": {"email": "iryna.grankova@airbyte.io"}}, "emitted_at": 1671456048322} -{"stream": "users", "data": {"object": "user", "id": "1dec424e-8fc2-4265-b82f-731a90dffcd0", "name": "Airbyte", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/daf02971-a282-49e6-810d-860fcb98ad1d/Screenshot_2021-10-19_at_16.42.29.png", "type": "bot", "bot": {"owner": {"type": "workspace", "workspace": true}, "workspace_name": "Airbyte Testing"}}, "emitted_at": 1671456048322} -{"stream": "users", "data": {"object": "user", "id": "7513291b-9767-4197-a879-262e289d887b", "name": "Airbyte", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/d40e67fa-0500-4795-8847-035ad01fb99a/Screenshot_2021-10-19_at_16.42.29.png", "type": "bot", "bot": {}}, "emitted_at": 1671456048322} -{"stream": "pages", "data": {"object": "page", "id": "00074690-3420-4861-84ae-b1c7498b67f1", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "a1298679-9f79-48a8-a991-834cd72eca17"}, "archived": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "type": "people", "people": []}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "type": "relation", "relation": [], "has_more": false}}, {"name": "Type", "value": {"id": "9dB%5E", "type": "select", "select": null}}, {"name": "Sprint", "value": {"id": "Jz.%40", "type": "multi_select", "multi_select": []}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "type": "relation", "relation": [], "has_more": false}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "type": "date", "date": null}}, {"name": "Created", "value": {"id": "iwS0", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "type": "people", "people": []}}, {"name": "Priority", "value": {"id": "%7BMEq", "type": "select", "select": null}}, {"name": "Status", "value": {"id": "%7CF4-", "type": "select", "select": null}}, {"name": "Projects", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Project Spec \ud83d\uddfa", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Project Spec \ud83d\uddfa", "href": null}]}}], "url": "https://www.notion.so/Project-Spec-000746903420486184aeb1c7498b67f1"}, "emitted_at": 1671456051191} -{"stream": "pages", "data": {"object": "page", "id": "249f3796-7e81-47b0-9075-00ed2d06439d", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "a1298679-9f79-48a8-a991-834cd72eca17"}, "archived": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "type": "people", "people": []}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "type": "relation", "relation": [], "has_more": false}}, {"name": "Type", "value": {"id": "9dB%5E", "type": "select", "select": {"id": "1497e06a-abf3-4c81-a619-debfa0c70621", "name": "Bug \ud83d\udc1e", "color": "red"}}}, {"name": "Sprint", "value": {"id": "Jz.%40", "type": "multi_select", "multi_select": [{"id": "8d033c95-5515-4662-b8f3-60cb7d86487a", "name": "Sprint 21", "color": "default"}]}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "type": "relation", "relation": [], "has_more": false}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "type": "date", "date": null}}, {"name": "Created", "value": {"id": "iwS0", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "type": "people", "people": []}}, {"name": "Priority", "value": {"id": "%7BMEq", "type": "select", "select": {"id": "09fy", "name": "P1 \ud83d\udd25", "color": "red"}}}, {"name": "Status", "value": {"id": "%7CF4-", "type": "select", "select": {"id": "ab7c2b08-ed87-4c04-b30f-fa62440f75d5", "name": "In Progress", "color": "yellow"}}}, {"name": "Projects", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "New Emojis Don't Render", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "New Emojis Don't Render", "href": null}]}}], "url": "https://www.notion.so/New-Emojis-Don-t-Render-249f37967e8147b0907500ed2d06439d"}, "emitted_at": 1671456051191} -{"stream": "pages", "data": {"object": "page", "id": "29299296-ef3f-4aff-aef5-02d651a59be3", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\udd0c"}, "parent": {"type": "database_id", "database_id": "a1298679-9f79-48a8-a991-834cd72eca17"}, "archived": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "type": "people", "people": []}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "type": "relation", "relation": [], "has_more": false}}, {"name": "Type", "value": {"id": "9dB%5E", "type": "select", "select": {"id": "ca62f85e-a4ac-474f-b493-82d2df005dff", "name": "Epic \u26f0\ufe0f", "color": "green"}}}, {"name": "Sprint", "value": {"id": "Jz.%40", "type": "multi_select", "multi_select": []}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "type": "relation", "relation": [], "has_more": false}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "type": "date", "date": {"start": "2019-10-17", "end": "2019-10-05", "time_zone": null}}}, {"name": "Created", "value": {"id": "iwS0", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "type": "people", "people": []}}, {"name": "Priority", "value": {"id": "%7BMEq", "type": "select", "select": null}}, {"name": "Status", "value": {"id": "%7CF4-", "type": "select", "select": {"id": "ab7c2b08-ed87-4c04-b30f-fa62440f75d5", "name": "In Progress", "color": "yellow"}}}, {"name": "Projects", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Improve Third Party Integrations", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Improve Third Party Integrations", "href": null}]}}], "url": "https://www.notion.so/Improve-Third-Party-Integrations-29299296ef3f4affaef502d651a59be3"}, "emitted_at": 1671456051192} -{"stream": "pages", "data": {"object": "page", "id": "3f6921c0-a42f-4d28-a169-f99ede14d8b5", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "a1298679-9f79-48a8-a991-834cd72eca17"}, "archived": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "type": "people", "people": []}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "type": "relation", "relation": [], "has_more": false}}, {"name": "Type", "value": {"id": "9dB%5E", "type": "select", "select": {"id": "1497e06a-abf3-4c81-a619-debfa0c70621", "name": "Bug \ud83d\udc1e", "color": "red"}}}, {"name": "Sprint", "value": {"id": "Jz.%40", "type": "multi_select", "multi_select": []}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "type": "relation", "relation": [], "has_more": false}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "type": "date", "date": null}}, {"name": "Created", "value": {"id": "iwS0", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "type": "people", "people": []}}, {"name": "Priority", "value": {"id": "%7BMEq", "type": "select", "select": null}}, {"name": "Status", "value": {"id": "%7CF4-", "type": "select", "select": null}}, {"name": "Projects", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Bug \ud83d\udc1e", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Bug \ud83d\udc1e", "href": null}]}}], "url": "https://www.notion.so/Bug-3f6921c0a42f4d28a169f99ede14d8b5"}, "emitted_at": 1671456051192} -{"stream": "pages", "data": {"object": "page", "id": "6c7fe685-c1a1-44b7-8fde-32ababced458", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "a1298679-9f79-48a8-a991-834cd72eca17"}, "archived": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "type": "people", "people": []}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "type": "relation", "relation": [], "has_more": false}}, {"name": "Type", "value": {"id": "9dB%5E", "type": "select", "select": null}}, {"name": "Sprint", "value": {"id": "Jz.%40", "type": "multi_select", "multi_select": []}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "type": "relation", "relation": [], "has_more": false}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "type": "date", "date": null}}, {"name": "Created", "value": {"id": "iwS0", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "type": "people", "people": []}}, {"name": "Priority", "value": {"id": "%7BMEq", "type": "select", "select": null}}, {"name": "Status", "value": {"id": "%7CF4-", "type": "select", "select": null}}, {"name": "Projects", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Task \ud83d\udd28", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Task \ud83d\udd28", "href": null}]}}], "url": "https://www.notion.so/Task-6c7fe685c1a144b78fde32ababced458"}, "emitted_at": 1671456051192} -{"stream": "pages", "data": {"object": "page", "id": "8fbb79d0-f858-4fe7-9e88-6b4019c72365", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "a1298679-9f79-48a8-a991-834cd72eca17"}, "archived": false, "properties": [{"name": "Engineers", "value": {"id": "%24v1Q", "type": "people", "people": []}}, {"name": "Tasks", "value": {"id": "6%3Dyp", "type": "relation", "relation": [], "has_more": false}}, {"name": "Type", "value": {"id": "9dB%5E", "type": "select", "select": {"id": "3f806034-9c48-4519-871e-60c9c32d73d8", "name": "Task \ud83d\udd28", "color": "yellow"}}}, {"name": "Sprint", "value": {"id": "Jz.%40", "type": "multi_select", "multi_select": [{"id": "bf3fcc55-aefc-43a8-82a0-2d4ac1e74d30", "name": "Sprint 22", "color": "default"}, {"id": "7d78a5e4-28ef-4b21-8495-998fa6655014", "name": "Sprint 23", "color": "default"}, {"id": "fbdb3f96-7979-4027-a461-aab8abda1ca8", "name": "Sprint 24", "color": "default"}]}}, {"name": "Epic", "value": {"id": "L%5BK%3C", "type": "relation", "relation": [], "has_more": false}}, {"name": "Timeline", "value": {"id": "_G%2Bl", "type": "date", "date": null}}, {"name": "Created", "value": {"id": "iwS0", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Product Manager", "value": {"id": "ma%3AW", "type": "people", "people": []}}, {"name": "Priority", "value": {"id": "%7BMEq", "type": "select", "select": {"id": "09fy", "name": "P1 \ud83d\udd25", "color": "red"}}}, {"name": "Status", "value": {"id": "%7CF4-", "type": "select", "select": {"id": "c224a5a5-c284-431e-a65d-90a71712bcac", "name": "Not Started", "color": "red"}}}, {"name": "Projects", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Rewrite Query Caching Logic", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Rewrite Query Caching Logic", "href": null}]}}], "url": "https://www.notion.so/Rewrite-Query-Caching-Logic-8fbb79d0f8584fe79e886b4019c72365"}, "emitted_at": 1671456051193} -{"stream": "pages", "data": {"object": "page", "id": "30597108-b046-4c09-a42b-cb78f6cc3972", "created_time": "2021-10-19T15:11:00.000Z", "last_edited_time": "2022-12-19T12:53:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": {"type": "external", "external": {"url": "https://www.notion.so/images/page-cover/nasa_bruce_mccandless_spacewalk.jpg"}}, "icon": {"type": "emoji", "emoji": "\ud83c\udfb8"}, "parent": {"type": "workspace", "workspace": true}, "archived": false, "properties": [{"name": "title", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Airbyte Home Page", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Airbyte Home Page", "href": null}]}}], "url": "https://www.notion.so/Airbyte-Home-Page-30597108b0464c09a42bcb78f6cc3972"}, "emitted_at": 1671456051193} -{"stream":"databases","data":{"object":"database","id":"a1298679-9f79-48a8-a991-834cd72eca17","cover":null,"icon":{"type":"emoji","emoji":"🚘"},"created_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_time":"2022-09-26T17:38:00.000Z","title":[{"type":"text","text":{"content":"Roadmap","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Roadmap","href":null}],"description":[{"type":"text","text":{"content":"Use this template to track all of your project work.\n\n⛰ ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Use this template to track all of your project work.\n\n⛰ ","href":null},{"type":"text","text":{"content":"Epics","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Epics","href":null},{"type":"text","text":{"content":" are large overarching initiatives.\n🏃‍♂️ ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are large overarching initiatives.\n🏃‍♂️ ","href":null},{"type":"text","text":{"content":"Sprints","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Sprints","href":null},{"type":"text","text":{"content":" are time-bounded pushes to complete a set of tasks.\n🔨 ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are time-bounded pushes to complete a set of tasks.\n🔨 ","href":null},{"type":"text","text":{"content":"Tasks","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Tasks","href":null},{"type":"text","text":{"content":" are the actions that make up epics.\n🐞 ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are the actions that make up epics.\n🐞 ","href":null},{"type":"text","text":{"content":"Bugs","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Bugs","href":null},{"type":"text","text":{"content":" are tasks to fix things.\n\n","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are tasks to fix things.\n\n","href":null},{"type":"text","text":{"content":"↓","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"↓","href":null},{"type":"text","text":{"content":" Click ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" Click ","href":null},{"type":"text","text":{"content":"By Status","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":true,"color":"default"},"plain_text":"By Status","href":null},{"type":"text","text":{"content":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","href":null}],"is_inline":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","name":"Engineers","type":"people","people":{}}},{"name":"Tasks","value":{"id":"6%3Dyp","name":"Tasks","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Epic","synced_property_id":"L%5BK%3C"}}}},{"name":"Type","value":{"id":"9dB%5E","name":"Type","type":"select","select":{"options":[{"id":"ca62f85e-a4ac-474f-b493-82d2df005dff","name":"Epic ⛰️","color":"green"},{"id":"3f806034-9c48-4519-871e-60c9c32d73d8","name":"Task 🔨","color":"yellow"},{"id":"1497e06a-abf3-4c81-a619-debfa0c70621","name":"Bug 🐞","color":"red"}]}}},{"name":"Sprint","value":{"id":"Jz.%40","name":"Sprint","type":"multi_select","multi_select":{"options":[{"id":"8d033c95-5515-4662-b8f3-60cb7d86487a","name":"Sprint 21","color":"default"},{"id":"bf3fcc55-aefc-43a8-82a0-2d4ac1e74d30","name":"Sprint 22","color":"default"},{"id":"7d78a5e4-28ef-4b21-8495-998fa6655014","name":"Sprint 23","color":"default"},{"id":"257e46d2-4a27-4298-b8c7-9b9bfb603bbd","name":"Sprint 20","color":"default"},{"id":"fbdb3f96-7979-4027-a461-aab8abda1ca8","name":"Sprint 24","color":"default"}]}}},{"name":"Epic","value":{"id":"L%5BK%3C","name":"Epic","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Tasks","synced_property_id":"6%3Dyp"}}}},{"name":"Timeline","value":{"id":"_G%2Bl","name":"Timeline","type":"date","date":{}}},{"name":"Created","value":{"id":"iwS0","name":"Created","type":"created_time","created_time":{}}},{"name":"Product Manager","value":{"id":"ma%3AW","name":"Product Manager","type":"people","people":{}}},{"name":"Priority","value":{"id":"%7BMEq","name":"Priority","type":"select","select":{"options":[{"id":"09fy","name":"P1 🔥","color":"red"},{"id":"e1b2f058-4989-4dee-a873-4e88f58d4d0a","name":"P2","color":"orange"},{"id":"0bb46e0b-be4f-4b9c-87c2-b990868e9f92","name":"P3","color":"yellow"},{"id":"1a5512c5-39ad-4fb7-959f-68f596849eeb","name":"P4","color":"green"},{"id":"28c7cce9-7163-4407-9569-3f070da82ad1","name":"P5","color":"blue"}]}}},{"name":"Status","value":{"id":"%7CF4-","name":"Status","type":"select","select":{"options":[{"id":"c224a5a5-c284-431e-a65d-90a71712bcac","name":"Not Started","color":"red"},{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"},{"id":"c410e525-9a47-4ed2-9e72-299abee65575","name":"Complete 🙌","color":"green"}]}}},{"name":"Projects","value":{"id":"title","name":"Projects","type":"title","title":{}}}],"parent":{"type":"workspace","workspace":true},"url":"https://www.notion.so/a12986799f7948a8a991834cd72eca17","archived":false},"emitted_at":1681213098588} -{"stream": "blocks", "data": {"object": "block", "id": "af0bd3c7-9704-44ab-a0af-1a94462f762e", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"rich_text": [{"type": "text", "text": {"content": "Overview", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Overview", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1684458657507} -{"stream": "blocks", "data": {"object": "block", "id": "391f45c4-6d75-4e9e-90db-7457d640f75e", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_2", "heading_2": {"rich_text": [{"type": "text", "text": {"content": "Problem statement", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Problem statement", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1684458657508} -{"stream": "blocks", "data": {"object": "block", "id": "d9698116-183d-48c4-81e2-79a882236000", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"rich_text": [{"type": "text", "text": {"content": "Describe the problem we're trying to solve by doing this work.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Describe the problem we're trying to solve by doing this work.", "href": null}], "color": "gray"}}, "emitted_at": 1684458657508} -{"stream": "blocks", "data": {"object": "block", "id": "09015c19-6cda-49ae-9685-0f8ce37ecae9", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_2", "heading_2": {"rich_text": [{"type": "text", "text": {"content": "Proposed work", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Proposed work", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1684458657508} -{"stream": "blocks", "data": {"object": "block", "id": "0a94c7c3-4e94-4f91-97c4-b564554cdd66", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"rich_text": [{"type": "text", "text": {"content": "High-level overview of what we're building and why we think it'll solve the problem.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "High-level overview of what we're building and why we think it'll solve the problem.", "href": null}], "color": "gray"}}, "emitted_at": 1684458657508} -{"stream": "blocks", "data": {"object": "block", "id": "91eabc7a-f7d9-45e0-a5d8-f32d76db1b95", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"rich_text": [{"type": "text", "text": {"content": "Dependencies & risks", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Dependencies & risks", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1684458657509} -{"stream": "blocks", "data": {"object": "block", "id": "b3646071-aaa0-4ccc-968c-1315e5d741ad", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_2", "heading_2": {"rich_text": [{"type": "text", "text": {"content": "Stakeholders", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Stakeholders", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1684458657509} -{"stream": "blocks", "data": {"object": "block", "id": "65071502-e150-4e70-9b72-b4f75fa4c0b7", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"rich_text": [{"type": "text", "text": {"content": "Who are the critical stakeholders? Regarding what topics?", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Who are the critical stakeholders? Regarding what topics?", "href": null}], "color": "gray"}}, "emitted_at": 1684458657509} -{"stream": "blocks", "data": {"object": "block", "id": "dfe425f7-4b35-42e8-ae70-e7b2e30ef3c9", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "bulleted_list_item", "bulleted_list_item": {"rich_text": [], "color": "default"}}, "emitted_at": 1684458657509} -{"stream": "blocks", "data": {"object": "block", "id": "340a25b2-67eb-408c-af37-9630dcfd2f66", "parent": {"type": "page_id", "page_id": "00074690-3420-4861-84ae-b1c7498b67f1"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_2", "heading_2": {"rich_text": [{"type": "text", "text": {"content": "Risks", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Risks", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1684458657509} +{"stream":"pages","data":{"object":"page","id":"00074690-3420-4861-84ae-b1c7498b67f1","created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"cover":null,"icon":null,"parent":{"type":"database_id","database_id":"a1298679-9f79-48a8-a991-834cd72eca17"},"archived":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","type":"people","people":[]}},{"name":"Tasks","value":{"id":"6%3Dyp","type":"relation","relation":[],"has_more":false}},{"name":"Type","value":{"id":"9dB%5E","type":"select","select":null}},{"name":"Sprint","value":{"id":"Jz.%40","type":"multi_select","multi_select":[]}},{"name":"Epic","value":{"id":"L%5BK%3C","type":"relation","relation":[],"has_more":false}},{"name":"Timeline","value":{"id":"_G%2Bl","type":"date","date":null}},{"name":"Created","value":{"id":"iwS0","type":"created_time","created_time":"2021-10-19T13:33:00.000Z"}},{"name":"Product Manager","value":{"id":"ma%3AW","type":"people","people":[]}},{"name":"Priority","value":{"id":"%7BMEq","type":"select","select":null}},{"name":"Status","value":{"id":"%7CF4-","type":"select","select":null}},{"name":"Projects","value":{"id":"title","type":"title","title":[{"type":"text","text":{"content":"Project Spec 🗺","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Project Spec 🗺","href":null}]}}],"url":"https://www.notion.so/Project-Spec-000746903420486184aeb1c7498b67f1","public_url":null},"emitted_at":1687166006562} +{"stream":"pages","data":{"object":"page","id":"249f3796-7e81-47b0-9075-00ed2d06439d","created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"cover":null,"icon":null,"parent":{"type":"database_id","database_id":"a1298679-9f79-48a8-a991-834cd72eca17"},"archived":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","type":"people","people":[]}},{"name":"Tasks","value":{"id":"6%3Dyp","type":"relation","relation":[],"has_more":false}},{"name":"Type","value":{"id":"9dB%5E","type":"select","select":{"id":"1497e06a-abf3-4c81-a619-debfa0c70621","name":"Bug 🐞","color":"red"}}},{"name":"Sprint","value":{"id":"Jz.%40","type":"multi_select","multi_select":[{"id":"8d033c95-5515-4662-b8f3-60cb7d86487a","name":"Sprint 21","color":"default"}]}},{"name":"Epic","value":{"id":"L%5BK%3C","type":"relation","relation":[],"has_more":false}},{"name":"Timeline","value":{"id":"_G%2Bl","type":"date","date":null}},{"name":"Created","value":{"id":"iwS0","type":"created_time","created_time":"2021-10-19T13:33:00.000Z"}},{"name":"Product Manager","value":{"id":"ma%3AW","type":"people","people":[]}},{"name":"Priority","value":{"id":"%7BMEq","type":"select","select":{"id":"09fy","name":"P1 🔥","color":"red"}}},{"name":"Status","value":{"id":"%7CF4-","type":"select","select":{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"}}},{"name":"Projects","value":{"id":"title","type":"title","title":[{"type":"text","text":{"content":"New Emojis Don't Render","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"New Emojis Don't Render","href":null}]}}],"url":"https://www.notion.so/New-Emojis-Don-t-Render-249f37967e8147b0907500ed2d06439d","public_url":null},"emitted_at":1687166006563} +{"stream":"pages","data":{"object":"page","id":"29299296-ef3f-4aff-aef5-02d651a59be3","created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"cover":null,"icon":{"type":"emoji","emoji":"🔌"},"parent":{"type":"database_id","database_id":"a1298679-9f79-48a8-a991-834cd72eca17"},"archived":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","type":"people","people":[]}},{"name":"Tasks","value":{"id":"6%3Dyp","type":"relation","relation":[],"has_more":false}},{"name":"Type","value":{"id":"9dB%5E","type":"select","select":{"id":"ca62f85e-a4ac-474f-b493-82d2df005dff","name":"Epic ⛰️","color":"green"}}},{"name":"Sprint","value":{"id":"Jz.%40","type":"multi_select","multi_select":[]}},{"name":"Epic","value":{"id":"L%5BK%3C","type":"relation","relation":[],"has_more":false}},{"name":"Timeline","value":{"id":"_G%2Bl","type":"date","date":{"start":"2019-10-17","end":"2019-10-05","time_zone":null}}},{"name":"Created","value":{"id":"iwS0","type":"created_time","created_time":"2021-10-19T13:33:00.000Z"}},{"name":"Product Manager","value":{"id":"ma%3AW","type":"people","people":[]}},{"name":"Priority","value":{"id":"%7BMEq","type":"select","select":null}},{"name":"Status","value":{"id":"%7CF4-","type":"select","select":{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"}}},{"name":"Projects","value":{"id":"title","type":"title","title":[{"type":"text","text":{"content":"Improve Third Party Integrations","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Improve Third Party Integrations","href":null}]}}],"url":"https://www.notion.so/Improve-Third-Party-Integrations-29299296ef3f4affaef502d651a59be3","public_url":null},"emitted_at":1687166006564} +{"stream":"blocks","data":{"object":"block","id":"af0bd3c7-9704-44ab-a0af-1a94462f762e","parent":{"type":"page_id","page_id":"00074690-3420-4861-84ae-b1c7498b67f1"},"created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"has_children":false,"archived":false,"type":"heading_1","heading_1":{"rich_text":[{"type":"text","text":{"content":"Overview","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Overview","href":null}],"is_toggleable":false,"color":"default"}},"emitted_at":1687166008901} +{"stream":"blocks","data":{"object":"block","id":"391f45c4-6d75-4e9e-90db-7457d640f75e","parent":{"type":"page_id","page_id":"00074690-3420-4861-84ae-b1c7498b67f1"},"created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"has_children":false,"archived":false,"type":"heading_2","heading_2":{"rich_text":[{"type":"text","text":{"content":"Problem statement","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Problem statement","href":null}],"is_toggleable":false,"color":"default"}},"emitted_at":1687166008902} +{"stream":"blocks","data":{"object":"block","id":"d9698116-183d-48c4-81e2-79a882236000","parent":{"type":"page_id","page_id":"00074690-3420-4861-84ae-b1c7498b67f1"},"created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"has_children":false,"archived":false,"type":"paragraph","paragraph":{"rich_text":[{"type":"text","text":{"content":"Describe the problem we're trying to solve by doing this work.","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Describe the problem we're trying to solve by doing this work.","href":null}],"color":"gray"}},"emitted_at":1687166008903} +{"stream":"users","data":{"object":"user","id":"5612c094-99ec-4ba3-ac7f-df8d84c8d6be","name":"Sherif Nada","avatar_url":"https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png","type":"person","person":{"email":"sherif@airbyte.io"}},"emitted_at":1687166004972} +{"stream":"users","data":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a","name":"Airyte","avatar_url":null,"type":"person","person":{"email":"integration-test@airbyte.io"}},"emitted_at":1687166004973} +{"stream":"users","data":{"object":"user","id":"c1ff0160-b2af-497a-aab7-8b61e625e4e3","name":"Gil Cho","avatar_url":"https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100","type":"person","person":{"email":"gil@airbyte.io"}},"emitted_at":1687166004973} +{"stream":"databases","data":{"object":"database","id":"a1298679-9f79-48a8-a991-834cd72eca17","cover":null,"icon":{"type":"emoji","emoji":"🚘"},"created_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_time":"2023-06-15T09:18:00.000Z","title":[{"type":"text","text":{"content":"Roadmap","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Roadmap","href":null}],"description":[{"type":"text","text":{"content":"Use this template to track all of your project work.\n\n⛰ ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Use this template to track all of your project work.\n\n⛰ ","href":null},{"type":"text","text":{"content":"Epics","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Epics","href":null},{"type":"text","text":{"content":" are large overarching initiatives.\n🏃‍♂️ ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are large overarching initiatives.\n🏃‍♂️ ","href":null},{"type":"text","text":{"content":"Sprints","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Sprints","href":null},{"type":"text","text":{"content":" are time-bounded pushes to complete a set of tasks.\n🔨 ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are time-bounded pushes to complete a set of tasks.\n🔨 ","href":null},{"type":"text","text":{"content":"Tasks","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Tasks","href":null},{"type":"text","text":{"content":" are the actions that make up epics.\n🐞 ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are the actions that make up epics.\n🐞 ","href":null},{"type":"text","text":{"content":"Bugs","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Bugs","href":null},{"type":"text","text":{"content":" are tasks to fix things.\n\n","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are tasks to fix things.\n\n","href":null},{"type":"text","text":{"content":"↓","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"↓","href":null},{"type":"text","text":{"content":" Click ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" Click ","href":null},{"type":"text","text":{"content":"By Status","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":true,"color":"default"},"plain_text":"By Status","href":null},{"type":"text","text":{"content":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","href":null}],"is_inline":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","name":"Engineers","type":"people","people":{}}},{"name":"Tasks","value":{"id":"6%3Dyp","name":"Tasks","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Epic","synced_property_id":"L%5BK%3C"}}}},{"name":"Type","value":{"id":"9dB%5E","name":"Type","type":"select","select":{"options":[{"id":"ca62f85e-a4ac-474f-b493-82d2df005dff","name":"Epic ⛰️","color":"green"},{"id":"3f806034-9c48-4519-871e-60c9c32d73d8","name":"Task 🔨","color":"yellow"},{"id":"1497e06a-abf3-4c81-a619-debfa0c70621","name":"Bug 🐞","color":"red"}]}}},{"name":"Sprint","value":{"id":"Jz.%40","name":"Sprint","type":"multi_select","multi_select":{"options":[{"id":"8d033c95-5515-4662-b8f3-60cb7d86487a","name":"Sprint 21","color":"default"},{"id":"bf3fcc55-aefc-43a8-82a0-2d4ac1e74d30","name":"Sprint 22","color":"default"},{"id":"7d78a5e4-28ef-4b21-8495-998fa6655014","name":"Sprint 23","color":"default"},{"id":"257e46d2-4a27-4298-b8c7-9b9bfb603bbd","name":"Sprint 20","color":"default"},{"id":"fbdb3f96-7979-4027-a461-aab8abda1ca8","name":"Sprint 24","color":"default"}]}}},{"name":"Epic","value":{"id":"L%5BK%3C","name":"Epic","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Tasks","synced_property_id":"6%3Dyp"}}}},{"name":"Timeline","value":{"id":"_G%2Bl","name":"Timeline","type":"date","date":{}}},{"name":"Created","value":{"id":"iwS0","name":"Created","type":"created_time","created_time":{}}},{"name":"Product Manager","value":{"id":"ma%3AW","name":"Product Manager","type":"people","people":{}}},{"name":"Priority","value":{"id":"%7BMEq","name":"Priority","type":"select","select":{"options":[{"id":"09fy","name":"P1 🔥","color":"red"},{"id":"e1b2f058-4989-4dee-a873-4e88f58d4d0a","name":"P2","color":"orange"},{"id":"0bb46e0b-be4f-4b9c-87c2-b990868e9f92","name":"P3","color":"yellow"},{"id":"1a5512c5-39ad-4fb7-959f-68f596849eeb","name":"P4","color":"green"},{"id":"28c7cce9-7163-4407-9569-3f070da82ad1","name":"P5","color":"blue"}]}}},{"name":"Status","value":{"id":"%7CF4-","name":"Status","type":"select","select":{"options":[{"id":"c224a5a5-c284-431e-a65d-90a71712bcac","name":"Not Started","color":"red"},{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"},{"id":"c410e525-9a47-4ed2-9e72-299abee65575","name":"Complete 🙌","color":"green"}]}}},{"name":"Projects","value":{"id":"title","name":"Projects","type":"title","title":{}}}],"parent":{"type":"workspace","workspace":true},"url":"https://www.notion.so/a12986799f7948a8a991834cd72eca17","public_url":null,"archived":false},"emitted_at":1687166005743} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index 37b9d0b9bb7e..e744b10f59b3 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 1.0.6 + dockerImageTag: 1.1.1 dockerRepository: airbyte/source-notion githubIssueLabel: source-notion icon: notion.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/notion tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-notion/requirements.txt b/airbyte-integrations/connectors/source-notion/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-notion/requirements.txt +++ b/airbyte-integrations/connectors/source-notion/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-notion/setup.py b/airbyte-integrations/connectors/source-notion/setup.py index 7b5240850f12..102f3ab3947a 100644 --- a/airbyte-integrations/connectors/source-notion/setup.py +++ b/airbyte-integrations/connectors/source-notion/setup.py @@ -12,7 +12,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 61b86dc466a8..797a17611b39 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -165,7 +165,7 @@ } }, "image": { "$ref": "file.json" }, - "vidoe": { "$ref": "file.json" }, + "video": { "$ref": "file.json" }, "file": { "$ref": "file.json" }, "pdf": { "$ref": "file.json" }, "bookmark": { diff --git a/airbyte-integrations/connectors/source-notion/source_notion/spec.json b/airbyte-integrations/connectors/source-notion/source_notion/spec.json index 5906db297c4f..b237e7691e2b 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/spec.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/spec.json @@ -75,12 +75,44 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["access_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "OAuth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index d3a31263649b..061fa5c3912d 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -7,11 +7,10 @@ import pydantic import requests -from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException -from airbyte_cdk.utils import AirbyteTracedException from .utils import transform_properties @@ -37,7 +36,8 @@ def __init__(self, config: Mapping[str, Any], **kwargs): def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None - def check_invalid_start_cursor(self, response: requests.Response): + @staticmethod + def check_invalid_start_cursor(response: requests.Response): if response.status_code == 400: message = response.json().get("message", "") if message.startswith("The start_cursor provided is invalid: "): @@ -77,7 +77,8 @@ def next_page_token( return {"next_cursor": next_cursor} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - data = response.json().get("results") + # sometimes notion api returns response without results object + data = response.json().get("results", []) yield from data @@ -146,7 +147,8 @@ def read_records(self, sync_mode: SyncMode, stream_state: Mapping[str, Any] = No except UserDefinedBackoffException as e: message = self.check_invalid_start_cursor(e.response) if message: - raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + self.logger.error(f"Skipping stream {self.name}, error message: {message}") + return raise e def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: @@ -260,7 +262,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: - # if reached recursive limit, don't read any more + # if reached recursive limit, don't read anymore if len(self.block_id_stack) > MAX_BLOCK_DEPTH: return @@ -275,6 +277,15 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: self.block_id_stack.pop() def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 404: + setattr(self, "raise_on_http_errors", False) + self.logger.error( + f"Stream {self.name}: {response.json().get('message')}. 404 HTTP response returns if the block specified by id doesn't" + " exist, or if the integration doesn't have access to the block." + "See more in docs: https://developers.notion.com/reference/get-block-children" + ) + return False + if response.status_code == 400: error_code = response.json().get("code") error_msg = response.json().get("message") diff --git a/airbyte-integrations/connectors/source-notion/source_notion/utils.py b/airbyte-integrations/connectors/source-notion/source_notion/utils.py index 5cf3137405b9..6103f00fb6b9 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/utils.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/utils.py @@ -8,7 +8,7 @@ def transform_properties(record: Mapping[str, Any], dict_key: str = "properties") -> Mapping[str, Any]: """ - Transfrom nested `properties` object. + Transform nested `properties` object. Move unique named entities into `name`, `value` to handle normalization. EXAMPLE INPUT: { diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py index 718996f5bf6f..90a54129f311 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from airbyte_cdk.models import SyncMode from pytest import fixture @@ -189,3 +189,17 @@ def test_recursive_read(blocks, requests_mock): inputs = {"sync_mode": SyncMode.incremental} stream.block_id_stack = [root] assert list(stream.read_records(**inputs)) == [record3, record2, record1, record4] + + +def test_invalid_start_cursor(parent, requests_mock, caplog): + stream = parent + error_message = "The start_cursor provided is invalid: wrong_start_cursor" + search_endpoint = requests_mock.post("https://api.notion.com/v1/search", status_code=400, + json={"object": "error", "status": 400, "code": "validation_error", + "message": error_message}) + + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + with patch.object(stream, "backoff_time", return_value=0.1): + list(stream.read_records(**inputs)) + assert search_endpoint.call_count == 6 + assert f"Skipping stream pages, error message: {error_message}" in caplog.messages diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index c5a9713e0795..0a441050829d 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -86,6 +86,30 @@ def test_should_not_retry_with_ai_block(requests_mock): assert not stream.should_retry(test_response) +def test_should_not_retry_with_not_found_block(requests_mock): + stream = Blocks(parent=None, config=MagicMock()) + json_response = { + "object": "error", + "status": 404, + "message": "Not Found for url: https://api.notion.com/v1/blocks/123/children?page_size=100", + } + requests_mock.get("https://api.notion.com/v1/blocks/123", json=json_response, status_code=404) + test_response = requests.get("https://api.notion.com/v1/blocks/123") + assert not stream.should_retry(test_response) + + +def test_empty_blocks_results(requests_mock): + stream = Blocks(parent=None, config=MagicMock()) + requests_mock.get( + "https://api.notion.com/v1/blocks/aaa/children", + json={ + "next_cursor": None, + }, + ) + stream.block_id_stack = ["aaa"] + assert list(stream.read_records(sync_mode=SyncMode.incremental, stream_slice=[])) == [] + + def test_backoff_time(patch_base_class): response_mock = MagicMock(headers={"retry-after": "10"}) stream = NotionStream(config=MagicMock()) @@ -106,7 +130,7 @@ def test_users_request_params(patch_base_class): assert stream.request_params(**inputs) == expected_params -def test_user_stream_handles_pagination_correclty(requests_mock): +def test_user_stream_handles_pagination_correctly(requests_mock): """ Test shows that Users stream uses pagination as per Notion API docs. """ diff --git a/airbyte-integrations/connectors/source-nytimes/metadata.yaml b/airbyte-integrations/connectors/source-nytimes/metadata.yaml index 4ee33494d644..fa67687b7293 100644 --- a/airbyte-integrations/connectors/source-nytimes/metadata.yaml +++ b/airbyte-integrations/connectors/source-nytimes/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-nytimes/requirements.txt b/airbyte-integrations/connectors/source-nytimes/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-nytimes/requirements.txt +++ b/airbyte-integrations/connectors/source-nytimes/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-nytimes/setup.py b/airbyte-integrations/connectors/source-nytimes/setup.py index 7a270d44be8c..ae57a1144c9f 100644 --- a/airbyte-integrations/connectors/source-nytimes/setup.py +++ b/airbyte-integrations/connectors/source-nytimes/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-okta/Dockerfile b/airbyte-integrations/connectors/source-okta/Dockerfile index 6a2b75318b3c..bf21b93a93f6 100644 --- a/airbyte-integrations/connectors/source-okta/Dockerfile +++ b/airbyte-integrations/connectors/source-okta/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-okta diff --git a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml index 1017b717c4e1..e7a9d92bdad0 100644 --- a/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-okta/acceptance-test-config.yml @@ -8,10 +8,6 @@ acceptance_tests: tests: - config_path: "secrets/config.json" status: "succeed" - - config_path: "secrets/config_api_token.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/__init__.py b/airbyte-integrations/connectors/source-okta/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-okta/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-okta/integration_tests/expected_records.jsonl index 5229b7478927..f61349197f63 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-okta/integration_tests/expected_records.jsonl @@ -1,32 +1,120 @@ -{"stream": "users", "data": {"id": "00u1o8nqx8yJbdIpH5d7", "status": "PASSWORD_EXPIRED", "created": "2021-09-08T07:04:28.000Z", "activated": "2021-09-08T07:04:28.000Z", "statusChanged": "2021-09-08T07:04:28.000Z", "lastLogin": null, "lastUpdated": "2021-09-08T07:04:28.000Z", "passwordChanged": "2021-09-08T07:04:28.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "dima", "lastName": "dima", "mobilePhone": null, "secondEmail": null, "login": "alala@alalal.com", "email": "alala@alalal.com"}, "credentials": {"password": {}, "emails": [{"value": "alala@alalal.com", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u1o8nqx8yJbdIpH5d7"}}}, "emitted_at": 1672948623804} -{"stream": "users", "data": {"id": "00u5vhvnz7GI7C1Ly5d7", "status": "STAGED", "created": "2022-07-21T20:47:01.000Z", "activated": null, "statusChanged": null, "lastLogin": null, "lastUpdated": "2022-07-21T20:47:01.000Z", "passwordChanged": null, "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Test", "lastName": "DEPROVISIONED", "mobilePhone": null, "secondEmail": null, "login": "test@rrrrr.com", "email": "test@rrrrr.com"}, "credentials": {"emails": [{"value": "test@rrrrr.com", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u5vhvnz7GI7C1Ly5d7"}}}, "emitted_at": 1672948623804} -{"stream": "users", "data": {"id": "00u5vhynsalJZ2eMg5d7", "status": "DEPROVISIONED", "created": "2022-07-21T20:46:27.000Z", "activated": "2022-07-21T20:47:13.000Z", "statusChanged": "2022-07-21T20:49:13.000Z", "lastLogin": null, "lastUpdated": "2022-07-21T20:49:13.000Z", "passwordChanged": "2022-07-21T20:47:35.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Test", "lastName": "DEPROVISIONED", "mobilePhone": null, "secondEmail": null, "login": "test@atata.co", "email": "test@atata.co"}, "credentials": {"emails": [{"value": "test@atata.co", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u5vhynsalJZ2eMg5d7"}}}, "emitted_at": 1672948623805} -{"stream": "users", "data": {"id": "00umj7d1bEH95JjIE5d6", "status": "ACTIVE", "created": "2021-04-21T21:04:03.000Z", "activated": null, "statusChanged": "2021-04-21T21:41:18.000Z", "lastLogin": "2023-01-05T19:35:17.000Z", "lastUpdated": "2021-11-03T13:45:55.000Z", "passwordChanged": "2021-04-21T21:41:18.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Shrif", "lastName": "Nada", "mobilePhone": "+1\u202a (415) 623-6785", "secondEmail": null, "login": "integration-test+2@airbyte.io", "email": "integration-test+2@airbyte.io"}, "credentials": {"password": {}, "emails": [{"value": "integration-test+2@airbyte.io", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00umj7d1bEH95JjIE5d6"}}}, "emitted_at": 1672948623805} -{"stream": "groups", "data": {"id": "00gmj7cvt3E2YKzGZ5d6", "created": "2021-04-21T21:03:55.000Z", "lastUpdated": "2021-04-21T21:03:55.000Z", "lastMembershipUpdated": "2022-07-21T20:47:01.000Z", "objectClass": ["okta:user_group"], "type": "BUILT_IN", "profile": {"name": "Everyone", "description": "All users in your organization"}, "_links": {"logo": [{"name": "medium", "href": "https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-medium.1a5ebe44c4244fb796c235d86b47e3bb.png", "type": "image/png"}, {"name": "large", "href": "https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-large.d9cfbd8a00a4feac1aa5612ba02e99c0.png", "type": "image/png"}], "users": {"href": "https://dev-01177082.okta.com/api/v1/groups/00gmj7cvt3E2YKzGZ5d6/users"}, "apps": {"href": "https://dev-01177082.okta.com/api/v1/groups/00gmj7cvt3E2YKzGZ5d6/apps"}}}, "emitted_at": 1672948624728} -{"stream": "groups", "data": {"id": "00g5tqi4p3h31MjL05d7", "created": "2022-07-18T07:58:11.000Z", "lastUpdated": "2022-07-18T07:58:11.000Z", "lastMembershipUpdated": "2022-07-18T07:58:11.000Z", "objectClass": ["okta:user_group"], "type": "OKTA_GROUP", "profile": {"name": "test-runner", "description": null}, "_links": {"logo": [{"name": "medium", "href": "https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-medium.1a5ebe44c4244fb796c235d86b47e3bb.png", "type": "image/png"}, {"name": "large", "href": "https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-large.d9cfbd8a00a4feac1aa5612ba02e99c0.png", "type": "image/png"}], "users": {"href": "https://dev-01177082.okta.com/api/v1/groups/00g5tqi4p3h31MjL05d7/users"}, "apps": {"href": "https://dev-01177082.okta.com/api/v1/groups/00g5tqi4p3h31MjL05d7/apps"}}}, "emitted_at": 1672948624728} -{"stream": "group_members", "data": {"id": "00u1o8nqx8yJbdIpH5d7", "status": "PASSWORD_EXPIRED", "created": "2021-09-08T07:04:28.000Z", "activated": "2021-09-08T07:04:28.000Z", "statusChanged": "2021-09-08T07:04:28.000Z", "lastLogin": null, "lastUpdated": "2021-09-08T07:04:28.000Z", "passwordChanged": "2021-09-08T07:04:28.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "dima", "lastName": "dima", "mobilePhone": null, "secondEmail": null, "login": "alala@alalal.com", "email": "alala@alalal.com"}, "credentials": {"password": {}, "emails": [{"value": "alala@alalal.com", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u1o8nqx8yJbdIpH5d7"}}}, "emitted_at": 1672948626544} -{"stream": "group_members", "data": {"id": "00u5vhvnz7GI7C1Ly5d7", "status": "STAGED", "created": "2022-07-21T20:47:01.000Z", "activated": null, "statusChanged": null, "lastLogin": null, "lastUpdated": "2022-07-21T20:47:01.000Z", "passwordChanged": null, "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Test", "lastName": "DEPROVISIONED", "mobilePhone": null, "secondEmail": null, "login": "test@rrrrr.com", "email": "test@rrrrr.com"}, "credentials": {"emails": [{"value": "test@rrrrr.com", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u5vhvnz7GI7C1Ly5d7"}}}, "emitted_at": 1672948626544} -{"stream": "group_members", "data": {"id": "00u5vhynsalJZ2eMg5d7", "status": "DEPROVISIONED", "created": "2022-07-21T20:46:27.000Z", "activated": "2022-07-21T20:47:13.000Z", "statusChanged": "2022-07-21T20:49:13.000Z", "lastLogin": null, "lastUpdated": "2022-07-21T20:49:13.000Z", "passwordChanged": "2022-07-21T20:47:35.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Test", "lastName": "DEPROVISIONED", "mobilePhone": null, "secondEmail": null, "login": "test@atata.co", "email": "test@atata.co"}, "credentials": {"emails": [{"value": "test@atata.co", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00u5vhynsalJZ2eMg5d7"}}}, "emitted_at": 1672948626544} -{"stream": "group_members", "data": {"id": "00umj7d1bEH95JjIE5d6", "status": "ACTIVE", "created": "2021-04-21T21:04:03.000Z", "activated": null, "statusChanged": "2021-04-21T21:41:18.000Z", "lastLogin": "2023-01-05T19:35:17.000Z", "lastUpdated": "2021-11-03T13:45:55.000Z", "passwordChanged": "2021-04-21T21:41:18.000Z", "type": {"id": "otymj7cw2AJFAHvOO5d6"}, "profile": {"firstName": "Shrif", "lastName": "Nada", "mobilePhone": "+1\u202a (415) 623-6785", "secondEmail": null, "login": "integration-test+2@airbyte.io", "email": "integration-test+2@airbyte.io"}, "credentials": {"password": {}, "emails": [{"value": "integration-test+2@airbyte.io", "status": "VERIFIED", "type": "PRIMARY"}], "provider": {"type": "OKTA", "name": "OKTA"}}, "_links": {"self": {"href": "https://dev-01177082.okta.com/api/v1/users/00umj7d1bEH95JjIE5d6"}}}, "emitted_at": 1672948626545} -{"stream": "resource_sets", "data": {"id": "iam64yqn54IMhp6ng5d7", "label": "test resource set", "description": "test resource set", "created": "2022-08-11T16:08:47.000Z", "lastUpdated": "2022-08-11T16:08:47.000Z", "_links": {"bindings": {"href": "https://dev-01177082.okta.com/api/v1/iam/resource-sets/iam64yqn54IMhp6ng5d7/bindings"}, "self": {"href": "https://dev-01177082.okta.com/api/v1/iam/resource-sets/iam64yqn54IMhp6ng5d7"}, "resources": {"href": "https://dev-01177082.okta.com/api/v1/iam/resource-sets/iam64yqn54IMhp6ng5d7/resources"}}}, "emitted_at": 1672948627684} -{"stream": "custom_roles", "data": {"id": "cr05rkz69f72I0GyP5d7", "label": "custom role for test", "description": "custom role for test", "created": "2022-07-13T07:54:31.000Z", "lastUpdated": "2022-07-13T07:54:31.000Z", "_links": {"permissions": {"href": "https://dev-01177082-admin.okta.com/api/v1/iam/roles/cr05rkz69f72I0GyP5d7/permissions"}, "self": {"href": "https://dev-01177082-admin.okta.com/api/v1/iam/roles/cr05rkz69f72I0GyP5d7"}}}, "emitted_at": 1672948628598} -{"stream": "user_role_assignments", "data": {"id": "ra1mj7d1dHmRrP5hO5d6", "label": "Super Organization Administrator", "type": "SUPER_ADMIN", "status": "ACTIVE", "created": "2021-04-21T21:04:03.000Z", "lastUpdated": "2021-04-21T21:04:03.000Z", "assignmentType": "USER", "_links": {"assignee": {"href": "https://dev-01177082.okta.com/api/v1/users/00umj7d1bEH95JjIE5d6"}}}, "emitted_at": 1672948631676} -{"stream": "group_role_assignments", "data": {"id": "gra5tqjhsatVjkWe75d7", "label": "Read-only Administrator", "type": "READ_ONLY_ADMIN", "status": "ACTIVE", "created": "2022-07-18T07:58:55.000Z", "lastUpdated": "2022-07-18T07:58:55.000Z", "assignmentType": "GROUP", "_links": {"assignee": {"href": "https://dev-01177082.okta.com/api/v1/groups/00g5tqi4p3h31MjL05d7"}}}, "emitted_at": 1672948633338} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "PostmanRuntime/7.28.4", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC refresh token is granted", "eventType": "app.oauth2.token.grant.refresh_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:34:06Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "redirectUri": "http://localhost:3000/auth_flow", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "authCode": "Zmixsgr1yvJhnmWnWdtzZg", "responseTime": "451", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "url": "/oauth2/v1/token?", "requestId": "Y7cmLn8_30-ptEljyPgpHQAACok", "dtHash": "2db76f321a459020aa185b7bdb6f4d5bcd116262589b0daef72afad73b1c4adb", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "threatSuspected": "false", "grantType": "authorization_code"}}, "legacyEventType": "app.oauth2.token.grant.refresh_token_success", "transaction": {"type": "WEB", "id": "Y7cmLn8_30-ptEljyPgpHQAACok", "detail": {}}, "uuid": "eb54b604-8d2f-11ed-a0ee-39336a7d5ce0", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "oartyx5v1m890hLVz5d6", "type": "refresh_token", "alternateId": null, "displayName": "Refresh Token", "detailEntry": {"expires": "2023-04-05T19:34:06.000Z", "subject": "00umj7d1bEH95JjIE5d6", "refreshtokentype": "persistent", "hash": "1YgqrsSb6posp4+oeCNPpqJGHYBnS0KHBqoNTo+bma0="}}]}, "emitted_at": 1672949289576} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "PostmanRuntime/7.28.4", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:34:06Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "redirectUri": "http://localhost:3000/auth_flow", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "authCode": "Zmixsgr1yvJhnmWnWdtzZg", "responseTime": "450", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "url": "/oauth2/v1/token?", "requestId": "Y7cmLn8_30-ptEljyPgpHQAACok", "dtHash": "2db76f321a459020aa185b7bdb6f4d5bcd116262589b0daef72afad73b1c4adb", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "threatSuspected": "false", "grantType": "authorization_code"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7cmLn8_30-ptEljyPgpHQAACok", "detail": {}}, "uuid": "eb548ef3-8d2f-11ed-a0ee-39336a7d5ce0", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.oXk_YrPyOqxQiJnBjRiu5xeBrfD7U5g9d7LB61t7gDY.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T20:34:06.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "mrd_FSNq67RMsg-L7KND_g"}}]}, "emitted_at": 1672949289576} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "python-requests/2.28.1", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:34:33Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "requestId": "Y7cmSQXgLp7AZga3sy4sFQAADdE", "responseTime": "531", "dtHash": "faf1d76af4686bf4c84bde82e2d4e90c4d35084dce35ba82837c6f20ae5047ad", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "refresh_token", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7cmSQXgLp7AZga3sy4sFQAADdE", "detail": {}}, "uuid": "fb717740-8d2f-11ed-9a96-2f2f12a7c2b2", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.34j6yuBIm1YNMwbwDnOGkMfr1SV2ze6-sj48bdHdDdc.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T20:34:33.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "zKw6gcxsynGGWJssIkKUtg"}}]}, "emitted_at": 1672949289576} -{"stream": "logs", "data": {"actor": {"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", "os": "Mac OS X", "browser": "SAFARI"}, "zone": "null", "device": "Computer", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "102S_P1ziE8QgCkqauwntbfrQ"}, "displayMessage": "User logout from Okta", "eventType": "user.session.end", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:17Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"authnRequestId": "Y7cjs3WiF2_5BT47ViWyGAAADbY", "deviceFingerprint": "bdeecfaec6af9a02f3512dc00d7649ab", "requestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "dtHash": "d07d7b76331614ccf6c0bca46b98374cf3ae1659a3eac37664f7269a4d22433f", "requestUri": "/api/v1/authn", "threatSuspected": "false", "url": "/api/v1/authn?"}}, "legacyEventType": "core.user_auth.logout_success", "transaction": {"type": "WEB", "id": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "detail": {}}, "uuid": "1561a269-8d30-11ed-bab1-d99d533f3518", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": null}, "emitted_at": 1672949289577} -{"stream": "logs", "data": {"actor": {"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", "os": "Mac OS X", "browser": "SAFARI"}, "zone": "null", "device": "Computer", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "102TmjdPsraQTil2JAN-R5zyw"}, "displayMessage": "Evaluation of sign-on policy", "eventType": "policy.evaluate_sign_on", "outcome": {"result": "ALLOW", "reason": "Sign-on policy evaluation resulted in ALLOW"}, "published": "2023-01-05T19:35:17Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"authnRequestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "deviceFingerprint": "bdeecfaec6af9a02f3512dc00d7649ab", "requestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "dtHash": "d07d7b76331614ccf6c0bca46b98374cf3ae1659a3eac37664f7269a4d22433f", "requestUri": "/api/v1/authn", "threatSuspected": "false", "url": "/api/v1/authn?"}}, "legacyEventType": null, "transaction": {"type": "WEB", "id": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "detail": {}}, "uuid": "157d19b4-8d30-11ed-bab1-d99d533f3518", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00pmj7cvuaHmm7loL5d6", "type": "PolicyEntity", "alternateId": "unknown", "displayName": "Default Policy", "detailEntry": {"policyType": "OktaSignOn"}}, {"id": "0prmj7cvvHXvozbc85d6", "type": "PolicyRule", "alternateId": "00pmj7cvuaHmm7loL5d6", "displayName": "Default Rule", "detailEntry": null}]}, "emitted_at": 1672949289577} -{"stream": "logs", "data": {"actor": {"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", "os": "Mac OS X", "browser": "SAFARI"}, "zone": "null", "device": "Computer", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "102TmjdPsraQTil2JAN-R5zyw"}, "displayMessage": "User login to Okta", "eventType": "user.session.start", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:17Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"deviceFingerprint": "bdeecfaec6af9a02f3512dc00d7649ab", "requestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "dtHash": "d07d7b76331614ccf6c0bca46b98374cf3ae1659a3eac37664f7269a4d22433f", "origin": "https://dev-01177082.okta.com", "requestUri": "/api/v1/authn", "threatSuspected": "false", "url": "/api/v1/authn?"}}, "legacyEventType": "core.user_auth.login_success", "transaction": {"type": "WEB", "id": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "detail": {}}, "uuid": "157689fd-8d30-11ed-bab1-d99d533f3518", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": null}, "emitted_at": 1672949289578} -{"stream": "logs", "data": {"actor": {"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", "os": "Mac OS X", "browser": "SAFARI"}, "zone": "null", "device": "Computer", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "102TmjdPsraQTil2JAN-R5zyw"}, "displayMessage": "Verify user identity", "eventType": "user.authentication.verify", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:17Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"authnRequestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "deviceFingerprint": "bdeecfaec6af9a02f3512dc00d7649ab", "requestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "dtHash": "d07d7b76331614ccf6c0bca46b98374cf3ae1659a3eac37664f7269a4d22433f", "requestUri": "/api/v1/authn", "threatSuspected": "false", "url": "/api/v1/authn?"}}, "legacyEventType": null, "transaction": {"type": "WEB", "id": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "detail": {}}, "uuid": "157e2b25-8d30-11ed-bab1-d99d533f3518", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": null}, "emitted_at": 1672949289578} -{"stream": "logs", "data": {"actor": {"id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "type": "PublicClientApp", "alternateId": "0oamj7cvhHyoTPQzl5d6", "displayName": "Okta Admin Console", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", "os": "Mac OS X", "browser": "SAFARI"}, "zone": "null", "device": "Computer", "id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "102QMYoaJ_QQ6iZkA9QDpUpHA"}, "displayMessage": "OIDC authorization code request", "eventType": "app.oauth2.authorize.code", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:19Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"redirectUri": "https://dev-01177082-admin.okta.com/admin/sso/callback", "grantedScopes": "openid", "responseMode": "query", "requestUri": "/oauth2/v1/authorize", "requestedScopes": "openid", "userId": "00umj7d1bEH95JjIE5d6", "url": "/oauth2/v1/authorize?response_type=code&response_mode=query&client_id=okta.b58d5b75-07d4-5f25-bf59-368a1261a405&redirect_uri=https%3A%2F%2Fdev-01177082-admin.okta.com%2Fadmin%2Fsso%2Fcallback&scope=openid&state=i4OjVzbComZbiutdGRpaknKiHy4DGI2A&nonce=xqZ8nCrlLZXOd1RCe9wMQfrilVLxu9lw&code_challenge=SbphS8jWNUmlqPMuPndqOC3tAqM8M1hCDM5tgQhEFoc&code_challenge_method=S256", "responseType": "code", "authnRequestId": "Y7cmdbdpW9qU8xRY5atKKgAAC9I", "requestId": "Y7cmd3-Jl_pU6YMK82QLfQAADhc", "dtHash": "d07d7b76331614ccf6c0bca46b98374cf3ae1659a3eac37664f7269a4d22433f", "state": "i4OjVzbComZbiutdGRpaknKiHy4DGI2A", "threatSuspected": "false", "grantType": "authorization_code"}}, "legacyEventType": "app.oauth2.authorize.code_success", "transaction": {"type": "WEB", "id": "Y7cmd3-Jl_pU6YMK82QLfQAADhc", "detail": {}}, "uuid": "16c131f4-8d30-11ed-b292-e57f389c86d2", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "Dh6mB6lfiLeAkR58E-t-cg", "type": "code", "alternateId": null, "displayName": "Authorization Code", "detailEntry": null}]}, "emitted_at": 1672949289579} -{"stream": "logs", "data": {"actor": {"id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "type": "PublicClientApp", "alternateId": "0oamj7cvhHyoTPQzl5d6", "displayName": "Okta Admin Console", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Okta-Integrations", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "ipAddress": "54.189.2.171", "geographicalContext": {"city": "Boardman", "state": "Oregon", "country": "United States", "postalCode": "97818", "geolocation": {"lat": 45.8234, "lon": -119.7257}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC id token is granted", "eventType": "app.oauth2.token.grant.id_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:20Z", "securityContext": {"asNumber": 16509, "asOrg": "amazon.com inc.", "isp": "amazon.com inc", "domain": "amazonaws.com", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "none", "redirectUri": "https://dev-01177082-admin.okta.com/admin/sso/callback", "grantedScopes": "openid", "authCode": "Dh6mB6lfiLeAkR58E-t-cg", "requestId": "Y7cmeIUq_4Wa2ezT7EEfOwAAB4A", "responseTime": "298", "dtHash": "da01fae28d3209a295e7db7eb65f7c5eb1506c163266d076c4698f0f0d71268b", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "authorization_code", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.id_token_success", "transaction": {"type": "WEB", "id": "Y7cmeIUq_4Wa2ezT7EEfOwAAB4A", "detail": {}}, "uuid": "175227e5-8d30-11ed-8c3c-b7917839f9f6", "version": "0", "request": {"ipChain": [{"ip": "54.189.2.171", "geographicalContext": {"city": "Boardman", "state": "Oregon", "country": "United States", "postalCode": "97818", "geolocation": {"lat": 45.8234, "lon": -119.7257}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "ID.fPTp_vITkdGrmfvUEmY5Q7AQBEKUDI0IkUkYjXMmaGw", "type": "id_token", "alternateId": null, "displayName": "ID Token", "detailEntry": {"audience": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "expires": "2023-01-05T20:35:20.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "lcz8MsRPriv-3kDWxY7m5g"}}]}, "emitted_at": 1672949289579} -{"stream": "logs", "data": {"actor": {"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Okta-Integrations", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "ipAddress": "54.189.2.171", "geographicalContext": {"city": "Boardman", "state": "Oregon", "country": "United States", "postalCode": "97818", "geolocation": {"lat": 45.8234, "lon": -119.7257}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "User single sign on to app", "eventType": "user.authentication.sso", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:20Z", "securityContext": {"asNumber": 16509, "asOrg": "amazon.com inc.", "isp": "amazon.com inc", "domain": "amazonaws.com", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"initiationType": "NA", "redirectUri": "https://dev-01177082-admin.okta.com/admin/sso/callback", "requestId": "Y7cmeIUq_4Wa2ezT7EEfOwAAB4A", "dtHash": "da01fae28d3209a295e7db7eb65f7c5eb1506c163266d076c4698f0f0d71268b", "signOnMode": "OpenID Connect", "requestUri": "/oauth2/v1/token", "threatSuspected": "false", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.auth.sso", "transaction": {"type": "WEB", "id": "Y7cmeIUq_4Wa2ezT7EEfOwAAB4A", "detail": {}}, "uuid": "17527607-8d30-11ed-8c3c-b7917839f9f6", "version": "0", "request": {"ipChain": [{"ip": "54.189.2.171", "geographicalContext": {"city": "Boardman", "state": "Oregon", "country": "United States", "postalCode": "97818", "geolocation": {"lat": 45.8234, "lon": -119.7257}}, "version": "V4", "source": null}]}, "target": [{"id": "0oamj7cvhHyoTPQzl5d6", "type": "AppInstance", "alternateId": "Okta Admin Console", "displayName": "Okta Admin Console", "detailEntry": {"signOnModeType": "OPENID_CONNECT"}}, {"id": "0uamj7d1fmnf6eOBG5d6", "type": "AppUser", "alternateId": "unknown", "displayName": "Shrif Nada", "detailEntry": null}]}, "emitted_at": 1672949289579} -{"stream": "logs", "data": {"actor": {"id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "type": "PublicClientApp", "alternateId": "0oamj7cvhHyoTPQzl5d6", "displayName": "Okta Admin Console", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Okta-Integrations", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": "okta.b58d5b75-07d4-5f25-bf59-368a1261a405", "ipAddress": "54.189.2.171", "geographicalContext": {"city": "Boardman", "state": "Oregon", "country": "United States", "postalCode": "97818", "geolocation": {"lat": 45.8234, "lon": -119.7257}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:20Z", "securityContext": {"asNumber": 16509, "asOrg": "amazon.com inc.", "isp": "amazon.com inc", "domain": "amazonaws.com", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "none", "redirectUri": "https://dev-01177082-admin.okta.com/admin/sso/callback", "grantedScopes": "openid", "authCode": "Dh6mB6lfiLeAkR58E-t-cg", "requestId": "Y7cmeIUq_4Wa2ezT7EEfOwAAB4A", "responseTime": "301", "dtHash": "da01fae28d3209a295e7db7eb65f7c5eb1506c163266d076c4698f0f0d71268b", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "authorization_code", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7cmeIUq_4Wa2ezT7EEfOwAAB4A", "detail": {}}, "uuid": "17529d18-8d30-11ed-8c3c-b7917839f9f6", "version": "0", "request": {"ipChain": [{"ip": "54.189.2.171", "geographicalContext": {"city": "Boardman", "state": "Oregon", "country": "United States", "postalCode": "97818", "geolocation": {"lat": 45.8234, "lon": -119.7257}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.hXVGiDg30Ofjky9wTiLdIcrPNKDKG5cSnUnXIwrkO5g", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T20:35:20.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "7y2gFbQ_1vkpCvF5hjJU9Q"}}]}, "emitted_at": 1672949289580} -{"stream": "logs", "data": {"actor": {"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", "os": "Mac OS X", "browser": "SAFARI"}, "zone": "null", "device": "Computer", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "102YkmwjQnTSlSiTKylRstjPw"}, "displayMessage": "User accessing Okta admin app", "eventType": "user.session.access_admin_app", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:35:20Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"requestId": "Y7cmePYjLlH5hi0Wiamg3wAABrw", "dtHash": "59cd878bb2d187ef936878ed75430369d82158f30504e7f14e0abaaa7af8e550", "requestUri": "/admin/sso/callback", "url": "/admin/sso/callback?code=******&state=i4OjVzbComZbiutdGRpaknKiHy4DGI2A"}}, "legacyEventType": "app.admin.sso.login.success", "transaction": {"type": "WEB", "id": "Y7cmePYjLlH5hi0Wiamg3wAABrw", "detail": {}}, "uuid": "175e8422-8d30-11ed-8c5e-672e18a769b2", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "AppUser", "alternateId": "integration-test+2@airbyte.io", "displayName": "Shrif Nada", "detailEntry": null}]}, "emitted_at": 1672949289580} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "python-requests/2.28.1", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:41:15Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "requestId": "Y7cn28ZRfnCmWolCkXXerwAABRA", "responseTime": "460", "dtHash": "42f002f488b28cc514d0f002c7f23fe083672e992e150b0b548f2bba20f07300", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "refresh_token", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7cn28ZRfnCmWolCkXXerwAABRA", "detail": {}}, "uuid": "eae339f6-8d30-11ed-b25d-7f218337b768", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.YzpwyCGIBbg2c0Bue5UlR9_YpYVUOtjQxl5jh1Eu8TM.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T20:41:15.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "cjNfwi62mP1tiA8lwddN3A"}}]}, "emitted_at": 1672949289913} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "python-requests/2.28.1", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:54:48Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "requestId": "Y7crCB8MyLZZp1dedGdseQAABxo", "responseTime": "419", "dtHash": "8ee1d59adfc656fdc0dfff081a5d5b249ed92265aa332d16d0aed242fac5a1f6", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "refresh_token", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7crCB8MyLZZp1dedGdseQAABxo", "detail": {}}, "uuid": "cf890ce0-8d32-11ed-8bf2-3de560facad9", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.RTebvRgNm8swCWP1h_5mt7JgMowEYzLNSGpmEkgrysw.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T20:54:48.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "s1L02smdlH0iQihWL77XQw"}}]}, "emitted_at": 1672949289914} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "python-requests/2.28.1", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T19:55:45Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "requestId": "Y7crQQisLz8CG1FALj4ntQAACAU", "responseTime": "444", "dtHash": "d6d30b9569f9fff60d410c966db219a071698b53d74a1821f1b4927b4464af6c", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "refresh_token", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7crQQisLz8CG1FALj4ntQAACAU", "detail": {}}, "uuid": "f19cce91-8d32-11ed-b280-b59db9ed6496", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.Gj-OEEnjLQ4pN_Qfgq7vgunp50CkiooAyQ_EsIHc-hY.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T20:55:45.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "yf2_KYwDslJrReb0vyXeQA"}}]}, "emitted_at": 1672949289915} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "python-requests/2.28.1", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T20:04:01Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "requestId": "Y7ctMfYmGFpsx_QMnMSCKAAADK8", "responseTime": "328", "dtHash": "26b88d1b7ecf35b8aa19215d8c12f061ffaa6a58b4b99a968d381712b373aca8", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "refresh_token", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7ctMfYmGFpsx_QMnMSCKAAADK8", "detail": {}}, "uuid": "18e4d955-8d34-11ed-9900-0f7c52e3ecb5", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.xW_UxYUommqmGKRDsspLj8ZNJrRJrcOu87wXMTvvgwg.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T21:04:01.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "HUYue6Lc3vPfXvADUlhHag"}}]}, "emitted_at": 1672949289917} -{"stream": "logs", "data": {"actor": {"id": "0oamjyo4lacEQkO5m5d6", "type": "PublicClientApp", "alternateId": "0oamjyo4lacEQkO5m5d6", "displayName": "Airbyte", "detailEntry": null}, "client": {"userAgent": {"rawUserAgent": "python-requests/2.28.1", "os": "Unknown", "browser": "UNKNOWN"}, "zone": "null", "device": "Unknown", "id": null, "ipAddress": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}}, "device": null, "authenticationContext": {"authenticationProvider": null, "credentialProvider": null, "credentialType": null, "issuer": null, "interface": null, "authenticationStep": 0, "externalSessionId": "unknown"}, "displayMessage": "OIDC access token is granted", "eventType": "app.oauth2.token.grant.access_token", "outcome": {"result": "SUCCESS", "reason": null}, "published": "2023-01-05T20:06:37Z", "securityContext": {"asNumber": 3215, "asOrg": "pop nic", "isp": "orange s.a.", "domain": "wanadoo.fr", "isProxy": false}, "severity": "INFO", "debugContext": {"debugData": {"clientAuthType": "client_secret_basic", "grantedScopes": "okta.users.read, okta.logs.read, okta.groups.read, okta.roles.read, offline_access", "requestId": "Y7ctzDvHK7KFn6xtZcb6QAAABng", "responseTime": "454", "dtHash": "5bd02afa139a0addafb0f921aba6829cc9744bdbd6c6ae067248ca926aaac513", "clientSecret": "9Z5i19Z8cdzhBpZWWvEMOQ", "requestUri": "/oauth2/v1/token", "requestedScopes": "", "threatSuspected": "false", "grantType": "refresh_token", "url": "/oauth2/v1/token?"}}, "legacyEventType": "app.oauth2.token.grant.access_token_success", "transaction": {"type": "WEB", "id": "Y7ctzDvHK7KFn6xtZcb6QAAABng", "detail": {}}, "uuid": "75b6bfc2-8d34-11ed-b203-b96596f847f0", "version": "0", "request": {"ipChain": [{"ip": "90.8.134.29", "geographicalContext": {"city": "Cannes", "state": "Provence-Alpes-C\u00f4te d'Azur", "country": "France", "postalCode": "06400", "geolocation": {"lat": 43.5504, "lon": 7.0131}}, "version": "V4", "source": null}]}, "target": [{"id": "00umj7d1bEH95JjIE5d6", "type": "User", "alternateId": null, "displayName": null, "detailEntry": null}, {"id": "AT.7U5XsvO6goLDMXVo2Q_q3mFxbIev15ejOAubkv4rnQc.oartyx5v1m890hLVz5d6", "type": "access_token", "alternateId": null, "displayName": "Access Token", "detailEntry": {"expires": "2023-01-05T21:06:37.000Z", "subject": "00umj7d1bEH95JjIE5d6", "hash": "jF_56RaYykuFkPJ0IaBqnQ"}}]}, "emitted_at": 1672949289918} -{"stream": "permissions", "data": {"label": "okta.users.lifecycle.suspend", "created": "2022-07-13T07:54:31.000Z", "lastUpdated": "2022-07-13T07:54:31.000Z", "conditions": null, "_links": {"role": {"href": "https://dev-01177082-admin.okta.com/api/v1/iam/roles/cr05rkz69f72I0GyP5d7"}, "self": {"href": "https://dev-01177082-admin.okta.com/api/v1/iam/roles/cr05rkz69f72I0GyP5d7/permissions/okta.users.lifecycle.suspend"}}}, "emitted_at": 1672949384541} +{"stream":"users","data":{"id":"00ua65e9m11opLyFx5d7","status":"ACTIVE","created":"2023-06-28T15:10:45.000Z","activated":null,"statusChanged":"2023-06-28T17:00:30.000Z","lastLogin":"2023-06-28T17:00:38.000Z","lastUpdated":"2023-06-28T17:00:30.000Z","passwordChanged":"2023-06-28T17:00:30.000Z","type":{"id":"otya65adpkOaHlmwo5d7"},"profile":{"firstName":"Sherif","lastName":"nada","mobilePhone":null,"secondEmail":null,"login":"integration-test@daxtarity.com","email":"integration-test@daxtarity.com"},"credentials":{"password":{},"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://dev-33855097.okta.com/api/v1/users/00ua65e9m11opLyFx5d7"}}},"emitted_at":1687974332246} +{"stream":"users","data":{"id":"00ua67q26u6ozRG5T5d7","status":"ACTIVE","created":"2023-06-28T17:09:06.000Z","activated":"2023-06-28T17:09:06.000Z","statusChanged":"2023-06-28T17:09:46.000Z","lastLogin":"2023-06-28T17:09:50.000Z","lastUpdated":"2023-06-28T17:09:46.000Z","passwordChanged":"2023-06-28T17:09:46.000Z","type":{"id":"otya65adpkOaHlmwo5d7"},"profile":{"firstName":"integration","lastName":"test1","mobilePhone":null,"secondEmail":null,"login":"integration-test+1@daxtarity.com","email":"integration-test+1@daxtarity.com"},"credentials":{"password":{},"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://dev-33855097.okta.com/api/v1/users/00ua67q26u6ozRG5T5d7"}}},"emitted_at":1687974332247} +{"stream":"groups","data":{"id":"00ga65adpbQSn1nX15d7","created":"2023-06-28T15:10:40.000Z","lastUpdated":"2023-06-28T15:10:40.000Z","lastMembershipUpdated":"2023-06-28T17:09:06.000Z","objectClass":["okta:user_group"],"type":"BUILT_IN","profile":{"name":"Everyone","description":"All users in your organization"},"_links":{"logo":[{"name":"medium","href":"https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-medium.1a5ebe44c4244fb796c235d86b47e3bb.png","type":"image/png"},{"name":"large","href":"https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-large.d9cfbd8a00a4feac1aa5612ba02e99c0.png","type":"image/png"}],"users":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga65adpbQSn1nX15d7/users"},"apps":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga65adpbQSn1nX15d7/apps"}}},"emitted_at":1687974332902} +{"stream":"groups","data":{"id":"00ga65e9kf1YwttUp5d7","created":"2023-06-28T15:10:44.000Z","lastUpdated":"2023-06-28T15:10:44.000Z","lastMembershipUpdated":"2023-06-28T15:10:44.000Z","objectClass":["okta:user_group"],"type":"BUILT_IN","profile":{"name":"Okta Administrators","description":"Okta manages this group, which contains all administrators in your organization."},"_links":{"logo":[{"name":"medium","href":"https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-medium.1a5ebe44c4244fb796c235d86b47e3bb.png","type":"image/png"},{"name":"large","href":"https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-large.d9cfbd8a00a4feac1aa5612ba02e99c0.png","type":"image/png"}],"users":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga65e9kf1YwttUp5d7/users"},"apps":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga65e9kf1YwttUp5d7/apps"}}},"emitted_at":1687974332904} +{"stream":"groups","data":{"id":"00ga67rapdEXY9PVo5d7","created":"2023-06-28T17:08:11.000Z","lastUpdated":"2023-06-28T17:08:11.000Z","lastMembershipUpdated":"2023-06-28T17:09:06.000Z","objectClass":["okta:user_group"],"type":"OKTA_GROUP","profile":{"name":"test-group-1","description":null},"_links":{"logo":[{"name":"medium","href":"https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-medium.1a5ebe44c4244fb796c235d86b47e3bb.png","type":"image/png"},{"name":"large","href":"https://ok12static.oktacdn.com/assets/img/logos/groups/odyssey/okta-large.d9cfbd8a00a4feac1aa5612ba02e99c0.png","type":"image/png"}],"users":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga67rapdEXY9PVo5d7/users"},"apps":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga67rapdEXY9PVo5d7/apps"}}},"emitted_at":1687974332904} +{"stream":"group_members","data":{"id":"00ua65e9m11opLyFx5d7","status":"ACTIVE","created":"2023-06-28T15:10:45.000Z","activated":null,"statusChanged":"2023-06-28T17:00:30.000Z","lastLogin":"2023-06-28T17:00:38.000Z","lastUpdated":"2023-06-28T17:00:30.000Z","passwordChanged":"2023-06-28T17:00:30.000Z","type":{"id":"otya65adpkOaHlmwo5d7"},"profile":{"firstName":"Sherif","lastName":"nada","mobilePhone":null,"secondEmail":null,"login":"integration-test@daxtarity.com","email":"integration-test@daxtarity.com"},"credentials":{"password":{},"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://dev-33855097.okta.com/api/v1/users/00ua65e9m11opLyFx5d7"}},"groupId":"00ga65adpbQSn1nX15d7"},"emitted_at":1687974334544} +{"stream":"group_members","data":{"id":"00ua67q26u6ozRG5T5d7","status":"ACTIVE","created":"2023-06-28T17:09:06.000Z","activated":"2023-06-28T17:09:06.000Z","statusChanged":"2023-06-28T17:09:46.000Z","lastLogin":"2023-06-28T17:09:50.000Z","lastUpdated":"2023-06-28T17:09:46.000Z","passwordChanged":"2023-06-28T17:09:46.000Z","type":{"id":"otya65adpkOaHlmwo5d7"},"profile":{"firstName":"integration","lastName":"test1","mobilePhone":null,"secondEmail":null,"login":"integration-test+1@daxtarity.com","email":"integration-test+1@daxtarity.com"},"credentials":{"password":{},"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://dev-33855097.okta.com/api/v1/users/00ua67q26u6ozRG5T5d7"}},"groupId":"00ga65adpbQSn1nX15d7"},"emitted_at":1687974334545} +{"stream":"group_members","data":{"id":"00ua67q26u6ozRG5T5d7","status":"ACTIVE","created":"2023-06-28T17:09:06.000Z","activated":"2023-06-28T17:09:06.000Z","statusChanged":"2023-06-28T17:09:46.000Z","lastLogin":"2023-06-28T17:09:50.000Z","lastUpdated":"2023-06-28T17:09:46.000Z","passwordChanged":"2023-06-28T17:09:46.000Z","type":{"id":"otya65adpkOaHlmwo5d7"},"profile":{"firstName":"integration","lastName":"test1","mobilePhone":null,"secondEmail":null,"login":"integration-test+1@daxtarity.com","email":"integration-test+1@daxtarity.com"},"credentials":{"password":{},"provider":{"type":"OKTA","name":"OKTA"}},"_links":{"self":{"href":"https://dev-33855097.okta.com/api/v1/users/00ua67q26u6ozRG5T5d7"}},"groupId":"00ga67rapdEXY9PVo5d7"},"emitted_at":1687974334883} +{"stream":"resource_sets","data":{"id":"iama688hw0VLl1JrA5d7","label":"test-resource-set1","description":"test-resource-set1","created":"2023-06-28T17:39:57.000Z","lastUpdated":"2023-06-28T17:39:57.000Z","_links":{"bindings":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7/bindings"},"self":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7"},"resources":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7/resources"}}},"emitted_at":1687974335507} +{"stream":"custom_roles","data":{"id":"cr0a688qm6u91Un0m5d7","label":"user-viewer-role","description":"user-viewer-role","created":"2023-06-28T17:39:34.000Z","lastUpdated":"2023-06-28T17:39:34.000Z","_links":{"permissions":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7/permissions"},"self":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7"}}},"emitted_at":1687974336117} +{"stream":"user_role_assignments","data":{"id":"ra1a65e9m3XfWHtTL5d7","label":"Super Administrator","type":"SUPER_ADMIN","status":"ACTIVE","created":"2023-06-28T15:10:46.000Z","lastUpdated":"2023-06-28T15:10:46.000Z","assignmentType":"USER","_links":{"assignee":{"href":"https://dev-33855097.okta.com/api/v1/users/00ua65e9m11opLyFx5d7"}},"userId":"00ua65e9m11opLyFx5d7"},"emitted_at":1687974337567} +{"stream":"user_role_assignments","data":{"id":"irba68bi9rIDU5CrC5d7","label":"user-viewer-role","type":"CUSTOM","status":"ACTIVE","created":"2023-06-28T17:45:26.000Z","lastUpdated":"2023-06-28T17:45:26.000Z","assignmentType":"GROUP","resource-set":"iama688hw0VLl1JrA5d7","role":"cr0a688qm6u91Un0m5d7","_links":{"role":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7"},"resource-set":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7"},"permissions":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7/permissions"},"member":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7/bindings/cr0a688qm6u91Un0m5d7/members/irba68bi9rIDU5CrC5d7"},"assignee":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga67rapdEXY9PVo5d7"}},"userId":"00ua67q26u6ozRG5T5d7"},"emitted_at":1687974337756} +{"stream":"group_role_assignments","data":{"id":"irba68bi9rIDU5CrC5d7","label":"user-viewer-role","type":"CUSTOM","status":"ACTIVE","created":"2023-06-28T17:45:26.000Z","lastUpdated":"2023-06-28T17:45:26.000Z","assignmentType":"GROUP","resource-set":"iama688hw0VLl1JrA5d7","role":"cr0a688qm6u91Un0m5d7","_links":{"role":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7"},"resource-set":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7"},"permissions":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7/permissions"},"member":{"href":"https://dev-33855097.okta.com/api/v1/iam/resource-sets/iama688hw0VLl1JrA5d7/bindings/cr0a688qm6u91Un0m5d7/members/irba68bi9rIDU5CrC5d7"},"assignee":{"href":"https://dev-33855097.okta.com/api/v1/groups/00ga67rapdEXY9PVo5d7"}},"groupId":"00ga67rapdEXY9PVo5d7"},"emitted_at":1687974339505} +{"stream":"permissions","data":{"label":"okta.users.read","created":"2023-06-28T17:39:34.000Z","lastUpdated":"2023-06-28T17:39:34.000Z","conditions":null,"_links":{"role":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7"},"self":{"href":"https://dev-33855097-admin.okta.com/api/v1/iam/roles/cr0a688qm6u91Un0m5d7/permissions/okta.users.read"}}},"emitted_at":1687974340829} +{"stream":"logs","data":{"actor":{"id":"00u4exiwtOxp3yrE35d5","type":"User","alternateId":"web@okta.com","displayName":"Developer Free","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Java/11.0.19","os":"Unknown","browser":"UNKNOWN"},"zone":"OFF_NETWORK","device":"Unknown","id":"capfx6SYTRNSWSMAOWNS","ipAddress":"34.202.149.224","geographicalContext":{"city":"Ashburn","state":"Virginia","country":"United States","postalCode":"20149","geolocation":{"lat":39.0469,"lon":-77.4903}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsF6rJgAWPTR282mSDMs0rzA"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxNbs2IKJWqWx3TDGU36AAAAEE","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf","requestUri":"/api/internal/orgs","url":"/api/internal/orgs?"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":"WEB","id":"ZJxNbs2IKJWqWx3TDGU36AAAAEE","detail":{"requestApiTokenId":"00T1596qn2A3qlzpq5d7"}},"uuid":"f18fae84-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[{"ip":"34.202.149.224","geographicalContext":{"city":"Ashburn","state":"Virginia","country":"United States","postalCode":"20149","geolocation":{"lat":39.0469,"lon":-77.4903}},"version":"V4","source":null}]},"target":[{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":null}]},"emitted_at":1687974341857} +{"stream":"logs","data":{"actor":{"id":"00u4exiwtOxp3yrE35d5","type":"User","alternateId":"web@okta.com","displayName":"Developer Free","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Java/11.0.19","os":"Unknown","browser":"UNKNOWN"},"zone":"OFF_NETWORK","device":"Unknown","id":"capfx6SYTRNSWSMAOWNS","ipAddress":"34.202.149.224","geographicalContext":{"city":"Ashburn","state":"Virginia","country":"United States","postalCode":"20149","geolocation":{"lat":39.0469,"lon":-77.4903}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsF6rJgAWPTR282mSDMs0rzA"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxNbs2IKJWqWx3TDGU36AAAAEE","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf","requestUri":"/api/internal/orgs","url":"/api/internal/orgs?"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":"WEB","id":"ZJxNbs2IKJWqWx3TDGU36AAAAEE","detail":{"requestApiTokenId":"00T1596qn2A3qlzpq5d7"}},"uuid":"f1a24c2e-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[{"ip":"34.202.149.224","geographicalContext":{"city":"Ashburn","state":"Virginia","country":"United States","postalCode":"20149","geolocation":{"lat":39.0469,"lon":-77.4903}},"version":"V4","source":null}]},"target":[{"id":"0oaa65adp1JPNziXH5d7","type":"AppInstance","alternateId":"Okta Support User","displayName":"Okta Support User","detailEntry":null}]},"emitted_at":1687974341858} +{"stream":"logs","data":{"actor":{"id":"00u4exiwtOxp3yrE35d5","type":"User","alternateId":"web@okta.com","displayName":"Developer Free","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Java/11.0.19","os":"Unknown","browser":"UNKNOWN"},"zone":"OFF_NETWORK","device":"Unknown","id":"capfx6SYTRNSWSMAOWNS","ipAddress":"34.202.149.224","geographicalContext":{"city":"Ashburn","state":"Virginia","country":"United States","postalCode":"20149","geolocation":{"lat":39.0469,"lon":-77.4903}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsF6rJgAWPTR282mSDMs0rzA"},"displayMessage":"Brand was created","eventType":"system.brand.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxNbs2IKJWqWx3TDGU36AAAAEE","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf","requestUri":"/api/internal/orgs","url":"/api/internal/orgs?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxNbs2IKJWqWx3TDGU36AAAAEE","detail":{"requestApiTokenId":"00T1596qn2A3qlzpq5d7"}},"uuid":"f1aecf52-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[{"ip":"34.202.149.224","geographicalContext":{"city":"Ashburn","state":"Virginia","country":"United States","postalCode":"20149","geolocation":{"lat":39.0469,"lon":-77.4903}},"version":"V4","source":null}]},"target":[{"id":"bnda65adp9OBUXHVz5d7","type":"Brand","alternateId":null,"displayName":"bnda65adp9OBUXHVz5d7","detailEntry":{"removePoweredByOkta":"false","brandId":"bnda65adp9OBUXHVz5d7"}}]},"emitted_at":1687974341859} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trspQRD5Y1CRjqnuy8mw5iEUQ"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f1b55f03-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65adpcdlL2iue5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"OktaSignOn"}}]},"emitted_at":1687974341859} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trspQRD5Y1CRjqnuy8mw5iEUQ"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f1bb7984-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65adpcdlL2iue5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"OktaSignOn"}},{"id":"0pra65adpda3WSCrJ5d7","type":"PolicyRule","alternateId":"00pa65adpcdlL2iue5d7","displayName":"Default Rule","detailEntry":null}]},"emitted_at":1687974341859} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs0TlhJY7uQzGeczepOhCiGw"},"displayMessage":"Bootstrap universal directory user profile","eventType":"directory.user_profile.bootstrap","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"cvd.user_profile_bootstrapped","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f1cc1b60-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"otya65adpkOaHlmwo5d7","type":"Schema","alternateId":"user","displayName":"User","detailEntry":null}]},"emitted_at":1687974341860} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsLMaUD5oNRaqqg5rnVsxkNQ"},"displayMessage":"Update universal directory mappings","eventType":"directory.mapping.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"attributesModified":"","attributesAdded":"{Attribute : login,Expression : user.email}","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf","attributesDeleted":""}},"legacyEventType":"cvd.mappings_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f1de43d4-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"prma65adppeINWNBN5d7","type":"ProfileMapping","alternateId":"unknown","displayName":"otya65adpkOaHlmwo5d72otya65adpkOaHlmwo5d7","detailEntry":{"targetProfile":"otya65adpkOaHlmwo5d7","direction":"Okta to App","sourceProfile":"otya65adpkOaHlmwo5d7"}}]},"emitted_at":1687974341860} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"A ThreatInsight configuration was updated","eventType":"security.threat.configuration.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"security.threat.configuration.update","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f1f43cda-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":null},"emitted_at":1687974341860} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsBy-zvvMSSU2rS-zeLoL4iw"},"displayMessage":"MFA factor enabled","eventType":"system.mfa.factor.activate","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f2070193-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00oa65adon5tfl0a05d7","type":"MFA Factor","alternateId":null,"displayName":null,"detailEntry":{"factorType":"PASSWORD_AS_FACTOR"}}]},"emitted_at":1687974341861} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs9zteQq18RgWxuKfBvuA5XA"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:40Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f20ec9c4-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65adpu5ZCmPaK5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"group rule default policy","detailEntry":{"policyType":"group_rule"}}]},"emitted_at":1687974341861} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs0efT00mKQHqIZmCpJu-CzA"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:41Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f27bbc87-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9ip7UdKkka5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"OAuthAuthzPolicy"}},{"id":"0pra65e9iqWJbin9V5d7","type":"PolicyRule","alternateId":"00pa65e9ip7UdKkka5d7","displayName":"Default Policy Rule","detailEntry":null}]},"emitted_at":1687974341861} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs0efT00mKQHqIZmCpJu-CzA"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:41Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f273f454-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9ip7UdKkka5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"OAuthAuthzPolicy"}}]},"emitted_at":1687974341862} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs0efT00mKQHqIZmCpJu-CzA"},"displayMessage":"Custom Authorization Server token signing key rolled over","eventType":"app.oauth2.as.key.rollover","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:41Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authorizationServer":"ausa65adpvhWi5CMk5d7","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf","kid":"XfQr9p3JwOGPdmYU3vBGecm4cuVZVHvycCN4DuJrkWQ","defaultAuthorizationServer":"true"}},"legacyEventType":"app.oauth2.as.key.rollover.legacy","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f26fae90-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"ausa65adpvhWi5CMk5d7","type":"AuthorizationServer","alternateId":null,"displayName":"default","detailEntry":null}]},"emitted_at":1687974341862} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs0efT00mKQHqIZmCpJu-CzA"},"displayMessage":"Authorization server is created.","eventType":"oauth2.as.created","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:41Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"api.oauth2.as.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f270c001-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"ausa65adpvhWi5CMk5d7","type":"AuthorizationServerEntity","alternateId":"unknown","displayName":"default","detailEntry":{"authorizationserverdescription":"Default Authorization Server for your Applications"}}]},"emitted_at":1687974341862} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsg-LN0-C3QvGNrTAqFnJNMg"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:42Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f2d06c5d-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":null}]},"emitted_at":1687974341863} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsHZLPJvGeRcO9huIw_timaw"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:42Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f2f7a37c-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9jp7O8BP0s5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"OktaMfaEnroll"}}]},"emitted_at":1687974341863} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsHZLPJvGeRcO9huIw_timaw"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:42Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f2fc857d-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9jp7O8BP0s5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"OktaMfaEnroll"}},{"id":"0pra65e9jqg1cpnPr5d7","type":"PolicyRule","alternateId":"00pa65e9jp7O8BP0s5d7","displayName":"Default Rule","detailEntry":null}]},"emitted_at":1687974341863} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trshbwpZRvaRLCblYCxCa1Vfg"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:42Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f332feeb-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9jvVs7Emde5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"Password"}}]},"emitted_at":1687974341863} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trshbwpZRvaRLCblYCxCa1Vfg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:42Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f33792cc-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9jvVs7Emde5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"Password"}},{"id":"0pra65e9jwKKLR16z5d7","type":"PolicyRule","alternateId":"00pa65e9jvVs7Emde5d7","displayName":"Default Rule","detailEntry":null}]},"emitted_at":1687974341864} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsOyIuUz1-QSGPqT8DLkvW9Q"},"displayMessage":"Network zone create","eventType":"zone.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:44Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"zoneData":"{\"type\":\"IP\",\"gateways\":[],\"proxies\":[]}","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"zone.create","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f41e48a9-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"nzoa65e9keckT6yEt5d7","type":"NetworkZoneEntity","alternateId":"unknown","displayName":"BlockedIpZone","detailEntry":null}]},"emitted_at":1687974341864} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsOyIuUz1-QSGPqT8DLkvW9Q"},"displayMessage":"Network zone create","eventType":"zone.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:44Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"zoneData":"{\"type\":\"IP\",\"gateways\":[],\"proxies\":[]}","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"zone.create","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f41d3738-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"nzoa65e9kdPIYigIu5d7","type":"NetworkZoneEntity","alternateId":"unknown","displayName":"LegacyIpZone","detailEntry":null}]},"emitted_at":1687974341864} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsXGSMOvStTPqd33ldAyQYzQ"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:44Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f456e4eb-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9kgRHu8Wpl5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Idp Discovery Policy","detailEntry":{"policyType":"IdpDiscovery"}}]},"emitted_at":1687974341865} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsXGSMOvStTPqd33ldAyQYzQ"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:44Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f45ab57e-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00pa65e9kgRHu8Wpl5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Idp Discovery Policy","detailEntry":{"policyType":"IdpDiscovery"}},{"id":"0pra65e9khVrNR5RG5d7","type":"PolicyRule","alternateId":"00pa65e9kgRHu8Wpl5d7","displayName":"Default Rule","detailEntry":null}]},"emitted_at":1687974341865} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsHbdv_gpyRuGvpGc9Mp0gyA"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:45Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f48fcf4c-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"0oaa65e9knCbOow6Y5d7","type":"AppInstance","alternateId":"Okta Browser Plugin","displayName":"Okta Browser Plugin","detailEntry":null}]},"emitted_at":1687974341865} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsHbdv_gpyRuGvpGc9Mp0gyA"},"displayMessage":"Administrator consent granted.","eventType":"app.oauth2.admin.consent.grant","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:45Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.oauth2.admin.consent.grant_success","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f49acbcf-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"oag159d7fuimYAdzU5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.enduser.dashboard.manage","detailEntry":{"publicclientapp":"okta.ee074f99-1b5b-513e-8ea6-f8beeab8dbb9"}},{"id":"oag159d7fvkgZFA105d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.internal.enduser.read","detailEntry":{"publicclientapp":"okta.ee074f99-1b5b-513e-8ea6-f8beeab8dbb9"}},{"id":"oag159d7fwiIEmTid5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.internal.enduser.manage","detailEntry":{"publicclientapp":"okta.ee074f99-1b5b-513e-8ea6-f8beeab8dbb9"}},{"id":"oag159d7fxa0c2IrR5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.users.read.self","detailEntry":{"publicclientapp":"okta.ee074f99-1b5b-513e-8ea6-f8beeab8dbb9"}},{"id":"oag159d7fyUxXwWVc5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.enduser.dashboard.read","detailEntry":{"publicclientapp":"okta.ee074f99-1b5b-513e-8ea6-f8beeab8dbb9"}}]},"emitted_at":1687974341865} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trstdTiNxAmQgGbSU9CRk_zHA"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:45Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f4b8b414-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":null}]},"emitted_at":1687974341866} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trstdTiNxAmQgGbSU9CRk_zHA"},"displayMessage":"Administrator consent granted.","eventType":"app.oauth2.admin.consent.grant","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:45Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.oauth2.admin.consent.grant_success","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f4c3d7aa-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"oag159d7fzWElso1O5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.enduser.dashboard.manage","detailEntry":{"publicclientapp":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26"}},{"id":"oag159d7g0GfzWPrZ5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.internal.enduser.read","detailEntry":{"publicclientapp":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26"}},{"id":"oag159d7g1d3zCiPg5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.internal.enduser.manage","detailEntry":{"publicclientapp":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26"}},{"id":"oag159d7g2YYwbklX5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.users.manage.self","detailEntry":{"publicclientapp":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26"}},{"id":"oag159d7g3V3mkrso5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.users.read.self","detailEntry":{"publicclientapp":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26"}},{"id":"oag159d7g4vmvv36p5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.enduser.dashboard.read","detailEntry":{"publicclientapp":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26"}}]},"emitted_at":1687974341866} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":"34.202.149.224","geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsG8dDGSGgSqmGSCQGdMZiTA"},"displayMessage":"Create okta user","eventType":"user.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:45Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"core.user.config.user_creation.success","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f506ad30-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"placeholder@okta.com","displayName":"Placeholder Admin","detailEntry":null}]},"emitted_at":1687974341866} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsG8dDGSGgSqmGSCQGdMZiTA"},"displayMessage":"Activate Okta user","eventType":"user.lifecycle.activate","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:46Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"core.user.config.user_activated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f5266a39-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"placeholder@okta.com","displayName":"Placeholder Admin","detailEntry":null}]},"emitted_at":1687974341867} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsG8dDGSGgSqmGSCQGdMZiTA"},"displayMessage":"Add user to application membership","eventType":"application.user_membership.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:46Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"appname":"saasure","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.generic.provision.assign_user_to_app","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f52b4c3a-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"0uaa65e9m588OBEHu5d7","type":"AppUser","alternateId":"unknown","displayName":"Placeholder Admin","detailEntry":null},{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":null},{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"placeholder@okta.com","displayName":"Placeholder Admin","detailEntry":null}]},"emitted_at":1687974341867} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsapI5_zsKRcqWCTDutG7x1w"},"displayMessage":"Org Authorization Server token signing key rolled over","eventType":"app.oauth2.key.rollover","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:46Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf","kid":"BufyoxzYUBl2C0zPhvTS8wEIVBZpAs-wnfH-Zszdt7c"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f554cd42-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"00oa65adon5tfl0a05d7","type":"Org","alternateId":null,"displayName":null,"detailEntry":null}]},"emitted_at":1687974341867} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsqYLyXJH1SDmqNbn15yyFiQ"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:46Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f5754da8-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"0oaa65e9mdDSZM97T5d7","type":"AppInstance","alternateId":"Okta ISV Portal App","displayName":"Okta ISV Portal App","detailEntry":null}]},"emitted_at":1687974341867} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsqYLyXJH1SDmqNbn15yyFiQ"},"displayMessage":"Administrator consent granted.","eventType":"app.oauth2.admin.consent.grant","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:46Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.oauth2.admin.consent.grant_success","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f587eb49-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"oag159d7g9nsywL975d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.apps.manage","detailEntry":{"publicclientapp":"okta.9305de5a-67d7-49a9-8dc2-f149ae5aadfa"}}]},"emitted_at":1687974341868} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsqYLyXJH1SDmqNbn15yyFiQ"},"displayMessage":"Add user to application membership","eventType":"application.user_membership.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:46Z","securityContext":{"asNumber":14618,"asOrg":"amazon technologies inc.","isp":"amazon.com inc.","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"appname":"okta_isv_portal","dtHash":"9daa13f3139775f20a0c7b5af644d4bbf9e1660c387c459c926934e1ddbfd1bf"}},"legacyEventType":"app.generic.provision.assign_user_to_app","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"f593840e-15c5-11ee-b293-a1499fcf9071","version":"0","request":{"ipChain":[]},"target":[{"id":"0uaa65e9mxTXfiUBc5d7","type":"AppUser","alternateId":"placeholder@okta.com","displayName":"Placeholder Admin","detailEntry":null},{"id":"0oaa65e9mdDSZM97T5d7","type":"AppInstance","alternateId":"Okta ISV Portal App","displayName":"Okta ISV Portal App","detailEntry":null},{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"placeholder@okta.com","displayName":"Placeholder Admin","detailEntry":null}]},"emitted_at":1687974341868} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsL9qgNz-mQzeeFI4a0t99Gg"},"displayMessage":"Update universal directory mappings","eventType":"directory.mapping.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:48Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"INFO","debugContext":{"debugData":{"attributesModified":"","attributesAdded":"{Attribute : login,Expression : appuser.email},{Attribute : displayName,Expression : appuser.displayName},{Attribute : nickName,Expression : appuser.nickname},{Attribute : firstName,Expression : appuser.firstName},{Attribute : middleName,Expression : appuser.middleName},{Attribute : lastName,Expression : appuser.lastName},{Attribute : email,Expression : appuser.email},{Attribute : profileUrl,Expression : appuser.profile},{Attribute : timezone,Expression : appuser.zoneinfo == null ? \"America/Los_Angeles\" : appuser.zoneinfo},{Attribute : primaryPhone,Expression : appuser.phoneNumber},{Attribute : streetAddress,Expression : appuser.street_address},{Attribute : city,Expression : appuser.locality},{Attribute : state,Expression : appuser.region},{Attribute : zipCode,Expression : appuser.postalCode},{Attribute : countryCode,Expression : appuser.country}","attributesDeleted":""}},"legacyEventType":"cvd.mappings_updated","transaction":{"type":"JOB","id":"doca65cujkpyPnfNo5d7","detail":{}},"uuid":"f64e4807-15c5-11ee-a037-118e56108949","version":"0","request":{"ipChain":[]},"target":[{"id":"prma65ig0nC1Et9Xk5d7","type":"ProfileMapping","alternateId":"unknown","displayName":"otya65icx5eUXQQBM5d72otya65adpkOaHlmwo5d7","detailEntry":{"targetProfile":"otya65adpkOaHlmwo5d7","direction":"IDP to Okta","sourceProfile":"otya65icx5eUXQQBM5d7"}}]},"emitted_at":1687974341868} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsL9qgNz-mQzeeFI4a0t99Gg"},"displayMessage":"Update universal directory mappings","eventType":"directory.mapping.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T15:10:48Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"INFO","debugContext":{"debugData":{"attributesModified":"","attributesAdded":"{Attribute : firstName,Expression : user.firstName},{Attribute : lastName,Expression : user.lastName},{Attribute : email,Expression : user.email}","attributesDeleted":""}},"legacyEventType":"cvd.mappings_updated","transaction":{"type":"JOB","id":"doca65cujkpyPnfNo5d7","detail":{}},"uuid":"f65525d9-15c5-11ee-a037-118e56108949","version":"0","request":{"ipChain":[]},"target":[{"id":"prma65ig1637yriOK5d7","type":"ProfileMapping","alternateId":"unknown","displayName":"otya65adpkOaHlmwo5d72otya65icx5eUXQQBM5d7","detailEntry":{"targetProfile":"otya65icx5eUXQQBM5d7","direction":"Okta to IDP","sourceProfile":"otya65adpkOaHlmwo5d7"}}]},"emitted_at":1687974341868} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:26Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"478fd640-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79uM67VLqE5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Browser Plugin","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"false"}},{"id":"0oaa65e9knCbOow6Y5d7","type":"AppInstance","alternateId":"Okta Browser Plugin","displayName":"Okta Browser Plugin","detailEntry":null}]},"emitted_at":1687974341869} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:26Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"479617d4-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79xLw5QD2v5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Dashboard","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"false"}},{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":null}]},"emitted_at":1687974341869} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:26Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"47828fc6-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79oKDr6G2H5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Admin Console","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"false"}},{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":null}]},"emitted_at":1687974341869} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsgsWDBk9PSKOxCf1JTlw66A"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:27Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"481cac1c-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"0oaa67k7abD1ZgGrh5d7","type":"AppInstance","alternateId":"Okta Authenticator","displayName":"Okta Authenticator","detailEntry":null}]},"emitted_at":1687974341870} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsgsWDBk9PSKOxCf1JTlw66A"},"displayMessage":"Update application","eventType":"application.lifecycle.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:27Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"app.generic.config.app_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48281dd5-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"0oaa67k7abD1ZgGrh5d7","type":"AppInstance","alternateId":"Okta Authenticator","displayName":"Okta Authenticator","detailEntry":null}]},"emitted_at":1687974341870} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsgsWDBk9PSKOxCf1JTlw66A"},"displayMessage":"Administrator consent granted.","eventType":"app.oauth2.admin.consent.grant","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:27Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"app.oauth2.admin.consent.grant_success","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48316caf-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"oag159oivqIIPmp5M5d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.authenticators.manage.self","detailEntry":{"publicclientapp":"okta.63c081db-1f13-5084-882f-e79e1e5e2da7"}},{"id":"oag159oivrl64Ee375d7","type":"ConsentGrant","alternateId":null,"displayName":"okta.authenticators.read","detailEntry":{"publicclientapp":"okta.63c081db-1f13-5084-882f-e79e1e5e2da7"}}]},"emitted_at":1687974341870} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsVsvqPhsBQ6GY3JzGf1DOwA"},"displayMessage":"Bootstrap application user profile","eventType":"directory.app_user_profile.bootstrap","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:28Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"cvd.appuser_profile_bootstrapped","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"485a9fa8-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"otya67k7bcY620xKb5d7","type":"Schema","alternateId":"okta_authenticator","displayName":"Okta Authenticator User","detailEntry":null},{"id":"0oaa67k7abD1ZgGrh5d7","type":"AppInstance","alternateId":"Okta Authenticator","displayName":"Okta Authenticator","detailEntry":{"appEventIsPersonal":"false"}}]},"emitted_at":1687974341870} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsVsvqPhsBQ6GY3JzGf1DOwA"},"displayMessage":"Update universal directory mappings","eventType":"directory.mapping.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:28Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"attributesModified":"","attributesAdded":"","dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","attributesDeleted":"","threatSuspected":"false"}},"legacyEventType":"cvd.mappings_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"486fd560-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"prma67k7bhm8CPLoB5d7","type":"ProfileMapping","alternateId":"unknown","displayName":"otya65adpkOaHlmwo5d72otya67k7bcY620xKb5d7","detailEntry":{"targetProfile":"otya67k7bcY620xKb5d7","direction":"Okta to App","sourceProfile":"otya65adpkOaHlmwo5d7"}}]},"emitted_at":1687974341871} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsVsvqPhsBQ6GY3JzGf1DOwA"},"displayMessage":"Update universal directory mappings","eventType":"directory.mapping.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:28Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"attributesModified":"","attributesAdded":"","dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","attributesDeleted":"","threatSuspected":"false"}},"legacyEventType":"cvd.mappings_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"486b688e-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"prma67k7bgG6HyNKn5d7","type":"ProfileMapping","alternateId":"unknown","displayName":"otya67k7bcY620xKb5d72otya65adpkOaHlmwo5d7","detailEntry":{"targetProfile":"otya65adpkOaHlmwo5d7","direction":"App to Okta","sourceProfile":"otya67k7bcY620xKb5d7"}}]},"emitted_at":1687974341871} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsOpLGnuQBRsWBultDslNgXQ"},"displayMessage":"User profile linked object is created","eventType":"directory.linked_object.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:28Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"linkedObjectAdded":"users","dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48501854-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"otya67k7b9lIY2aCi5d7","type":"Schema","alternateId":"okta_device_profile_v1","displayName":"Okta Device Profile","detailEntry":null}]},"emitted_at":1687974341871} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsBAdHUYVZSNyUPg4ZO1AFYA"},"displayMessage":"Update universal directory mappings","eventType":"directory.mapping.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:28Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"attributesModified":"","attributesAdded":"","dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","attributesDeleted":"{Attribute : login,Expression : user.email}","threatSuspected":"false"}},"legacyEventType":"cvd.mappings_updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48acde6a-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"prma65adppeINWNBN5d7","type":"ProfileMapping","alternateId":"unknown","displayName":"otya65adpkOaHlmwo5d72otya65adpkOaHlmwo5d7","detailEntry":{"targetProfile":"otya65adpkOaHlmwo5d7","direction":"Okta to App","sourceProfile":"otya65adpkOaHlmwo5d7"}}]},"emitted_at":1687974341871} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"Update policy rule","eventType":"policy.rule.update","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:28Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"INFO","debugContext":{"debugData":{}},"legacyEventType":"policy.rule.updated","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48dec3f2-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7b4yzob1A55d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"Okta:ProfileEnrollment"}},{"id":"rula67k7b5DSyQ2jG5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341872} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"49232019-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7c6I9f53rB5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Password only","detailEntry":{"policyType":"Okta:SignOn"}}]},"emitted_at":1687974341872} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48fafe92-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7b4yzob1A55d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"Okta:ProfileEnrollment","previousPolicy":"false"}},{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":null}]},"emitted_at":1687974341872} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Delete policy rule","eventType":"policy.rule.delete","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.deleted","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4921c087-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79xLw5QD2v5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Dashboard","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k79yzWEFOUz5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341872} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"49256a0c-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7c86oOPtp85d7","type":"PolicyEntity","alternateId":"unknown","displayName":"One factor access","detailEntry":{"policyType":"Okta:SignOn"}}]},"emitted_at":1687974341872} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"492e43b6-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7caOHkTdny5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on risk context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7cd0xHyOO85d7","type":"PolicyRule","alternateId":"unknown","displayName":"High risk","detailEntry":null}]},"emitted_at":1687974341873} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4934fa81-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7cfAPAaiDp5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on network context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7cgL6kWwhF5d7","type":"PolicyRule","alternateId":"unknown","displayName":"In network","detailEntry":null}]},"emitted_at":1687974341873} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Delete policy","eventType":"policy.lifecycle.delete","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.deleted","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"491f4f82-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79uM67VLqE5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Browser Plugin","detailEntry":{"policyType":"Okta:SignOn"}}]},"emitted_at":1687974341873} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"49278cee-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7c86oOPtp85d7","type":"PolicyEntity","alternateId":"unknown","displayName":"One factor access","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7c9wlOFrvi5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341873} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4924cdcb-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7c6I9f53rB5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Password only","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7c7sDaLnFr5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341874} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4930dbcc-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7cfAPAaiDp5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on network context","detailEntry":{"policyType":"Okta:SignOn"}}]},"emitted_at":1687974341874} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4912cc5a-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7b2PPGRaLl5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Classic Migrated","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"false"}},{"id":"rsta67k79uM67VLqE5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Browser Plugin","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"true"}},{"id":"0oaa65e9knCbOow6Y5d7","type":"AppInstance","alternateId":"Okta Browser Plugin","displayName":"Okta Browser Plugin","detailEntry":null}]},"emitted_at":1687974341874} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"491a466e-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7b2PPGRaLl5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Classic Migrated","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"false"}},{"id":"rsta67k79xLw5QD2v5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Dashboard","detailEntry":{"policyType":"Okta:SignOn","previousPolicy":"true"}},{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":null}]},"emitted_at":1687974341874} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Create policy","eventType":"policy.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.created","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4928292f-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7caOHkTdny5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on risk context","detailEntry":{"policyType":"Okta:SignOn"}}]},"emitted_at":1687974341875} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"49371d63-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7cfAPAaiDp5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on network context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7ci65nkQvO5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341875} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4935e4e2-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7cfAPAaiDp5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on network context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7chorJvFXu5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Off network","detailEntry":null}]},"emitted_at":1687974341875} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Delete policy rule","eventType":"policy.rule.delete","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.deleted","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"491e8c31-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79uM67VLqE5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Browser Plugin","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k79vTu8PkuI5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341875} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"492bf9c1-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7caOHkTdny5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on risk context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7cbhOj03nD5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Low risk","detailEntry":null}]},"emitted_at":1687974341876} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48f72e01-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7b4yzob1A55d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"Okta:ProfileEnrollment","previousPolicy":"false"}},{"id":"0oaa65e9knCbOow6Y5d7","type":"AppInstance","alternateId":"Okta Browser Plugin","displayName":"Okta Browser Plugin","detailEntry":null}]},"emitted_at":1687974341876} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":null},"displayMessage":"Create policy mapping","eventType":"policy.mapping.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":null,"transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"48efdafe-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7b4yzob1A55d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Default Policy","detailEntry":{"policyType":"Okta:ProfileEnrollment","previousPolicy":"false"}},{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":null}]},"emitted_at":1687974341876} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Delete policy","eventType":"policy.lifecycle.delete","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.deleted","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"49225cc8-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k79xLw5QD2v5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Okta Dashboard","detailEntry":{"policyType":"Okta:SignOn"}}]},"emitted_at":1687974341876} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"492f7c38-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7caOHkTdny5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on risk context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7cerutwfpB5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341877} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs8I3WbDceQaO1BiZJK3tOLg"},"displayMessage":"Add policy rule","eventType":"policy.rule.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:29Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"policy.rule.added","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"492d0b32-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"rsta67k7caOHkTdny5d7","type":"PolicyEntity","alternateId":"unknown","displayName":"Seamless access based on risk context","detailEntry":{"policyType":"Okta:SignOn"}},{"id":"rula67k7ccRwq3Nek5d7","type":"PolicyRule","alternateId":"unknown","displayName":"Med risk","detailEntry":null}]},"emitted_at":1687974341877} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"placeholder@okta.com","displayName":"Placeholder Admin","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs1knynrpyT4uvhiNirg1_Yg"},"displayMessage":"User update password for Okta","eventType":"user.account.update_password","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:30Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"core.user.config.password_update.success","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"499c6eff-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null}]},"emitted_at":1687974341877} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs1knynrpyT4uvhiNirg1_Yg"},"displayMessage":"Reset factor for user","eventType":"user.mfa.factor.deactivate","outcome":{"result":"SUCCESS","reason":"User reset EMAIL_FACTOR factor"},"published":"2023-06-28T17:00:30Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"core.user.factor.deactivate","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"4997b40c-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null}]},"emitted_at":1687974341877} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trs1knynrpyT4uvhiNirg1_Yg"},"displayMessage":"Activate factor for user","eventType":"user.mfa.factor.activate","outcome":{"result":"SUCCESS","reason":"User set up EMAIL_FACTOR factor"},"published":"2023-06-28T17:00:30Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"dtHash":"10acfcc59d14f0ac4ca7615e95971a8ca689682780ce825c6ac6af5127ee6aa7","threatSuspected":"false"}},"legacyEventType":"core.user.factor.activate","transaction":{"type":null,"id":"unknown","detail":{}},"uuid":"49993aad-15d5-11ee-9810-c922c2b7dda6","version":"0","request":{"ipChain":[]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null}]},"emitted_at":1687974341877} +{"stream":"logs","data":{"actor":{"id":"spra65adoodAYMmER5d7","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":"SOCIAL","credentialProvider":null,"credentialType":"ASSERTION","issuer":{"id":"OIDC","type":null},"interface":"OIDC","authenticationStep":0,"externalSessionId":"idxPlzZVpyQT9eaUCwB5tkNtw"},"displayMessage":"Authenticate user with social login","eventType":"user.authentication.auth_via_social","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:35Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxnMA9nQjNl5EId95NC7wAAC7Q","requestId":"ZJxnMgM02SEJlzJJu8naWQAABYA","dtHash":"9b21b9e5015f3408488786e7e10f91c0db48e97f3eeaec560efcd9c190d90416","requestUri":"/idp/idx/introspect","threatSuspected":"false","url":"/idp/idx/introspect?"}},"legacyEventType":"core.user_auth.idp.social.login_success","transaction":{"type":"WEB","id":"ZJxnMgM02SEJlzJJu8naWQAABYA","detail":{}},"uuid":"4ca6ef4b-15d5-11ee-9a4b-710d97404f02","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65icwnbIQth5i5d7","type":"AppInstance","alternateId":"Developer Registration SSO","displayName":"OpenID Connect IdP","detailEntry":null},{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null}]},"emitted_at":1687974341878} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxPlzZVpyQT9eaUCwB5tkNtw"},"displayMessage":"User login to Okta","eventType":"user.session.start","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:35Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxnMA9nQjNl5EId95NC7wAAC7Q","requestId":"ZJxnMgM02SEJlzJJu8naWQAABYA","dtHash":"9b21b9e5015f3408488786e7e10f91c0db48e97f3eeaec560efcd9c190d90416","origin":"https://dev-33855097.okta.com","requestUri":"/idp/idx/introspect","threatSuspected":"false","url":"/idp/idx/introspect?"}},"legacyEventType":"core.user_auth.login_success","transaction":{"type":"WEB","id":"ZJxnMgM02SEJlzJJu8naWQAABYA","detail":{}},"uuid":"4cb87b8b-15d5-11ee-9a4b-710d97404f02","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":null}]},"emitted_at":1687974341878} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxPlzZVpyQT9eaUCwB5tkNtw"},"displayMessage":"Verify user identity","eventType":"user.authentication.verify","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:35Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxnMA9nQjNl5EId95NC7wAAC7Q","requestId":"ZJxnMgM02SEJlzJJu8naWQAABYA","dtHash":"9b21b9e5015f3408488786e7e10f91c0db48e97f3eeaec560efcd9c190d90416","requestUri":"/idp/idx/introspect","threatSuspected":"false","url":"/idp/idx/introspect?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxnMgM02SEJlzJJu8naWQAABYA","detail":{}},"uuid":"4cc37810-15d5-11ee-9a4b-710d97404f02","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":null},"emitted_at":1687974341878} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxPlzZVpyQT9eaUCwB5tkNtw"},"displayMessage":"Evaluation of sign-on policy","eventType":"policy.evaluate_sign_on","outcome":{"result":"ALLOW","reason":"Sign-on policy evaluation resulted in AUTHENTICATED"},"published":"2023-06-28T17:00:35Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxnMA9nQjNl5EId95NC7wAAC7Q","requestId":"ZJxnMgM02SEJlzJJu8naWQAABYA","authMethodFirstVerificationTime":"2023-06-28T17:00:35.336Z","dtHash":"9b21b9e5015f3408488786e7e10f91c0db48e97f3eeaec560efcd9c190d90416","authMethodFirstType":"SSO_SP_INIT","authMethodFirstEnrollment":"0oaa65icwnbIQth5i5d7","requestUri":"/idp/idx/introspect","threatSuspected":"false","url":"/idp/idx/introspect?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxnMgM02SEJlzJJu8naWQAABYA","detail":{}},"uuid":"4cb8547a-15d5-11ee-9a4b-710d97404f02","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":{"signOnModeType":"OPENID_CONNECT","signOnModeEvaluationResult":"AUTHENTICATED"}},{"id":"0pra65adpda3WSCrJ5d7","type":"Rule","alternateId":"unknown","displayName":"Default Rule","detailEntry":null},{"id":"rula67k7b3eKaojXn5d7","type":"Rule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341878} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxPlzZVpyQT9eaUCwB5tkNtw"},"displayMessage":"Evaluation of sign-on policy","eventType":"policy.evaluate_sign_on","outcome":{"result":"ALLOW","reason":"Sign-on policy evaluation resulted in AUTHENTICATED"},"published":"2023-06-28T17:00:38Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"redirectUri":"https://dev-33855097-admin.okta.com/admin/sso/callback","authnRequestId":"ZJxnNW7Sm0yWs5HZanbtegAADww","requestId":"ZJxnNW7Sm0yWs5HZanbtegAADww","authMethodFirstVerificationTime":"2023-06-28T17:00:35.336Z","dtHash":"9b21b9e5015f3408488786e7e10f91c0db48e97f3eeaec560efcd9c190d90416","authMethodFirstType":"SSO_SP_INIT","authMethodFirstEnrollment":"0oaa65icwnbIQth5i5d7","requestUri":"/oauth2/v1/authorize","threatSuspected":"false","url":"/oauth2/v1/authorize?response_type=code&response_mode=query&client_id=okta.b58d5b75-07d4-5f25-bf59-368a1261a405&redirect_uri=https%3A%2F%2Fdev-33855097-admin.okta.com%2Fadmin%2Fsso%2Fcallback&scope=openid&state=7aDReRR8YWxXc6v_cEwFaLcOp_5nD3o2&nonce=blxIQEqpcYCbmcUhdUtWTfBEYscsGxpO&code_challenge=hmHCgLVX5wxrxUXRkqO2oo7FdwZUIFoOP_OqCFMSjoo&code_challenge_method=S256"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxnNW7Sm0yWs5HZanbtegAADww","detail":{}},"uuid":"4e411718-15d5-11ee-8bd1-b1062595cad2","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":{"signOnModeType":"OPENID_CONNECT","signOnModeEvaluationResult":"AUTHENTICATED"}},{"id":"rula67k79pvxsWSVq5d7","type":"Rule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341879} +{"stream":"logs","data":{"actor":{"id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","type":"PublicClientApp","alternateId":"0oaa65adoxfVQdKKA5d7","displayName":"Okta Admin Console","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxPlzZVpyQT9eaUCwB5tkNtw"},"displayMessage":"OIDC authorization code request","eventType":"app.oauth2.authorize.code","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:38Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"redirectUri":"https://dev-33855097-admin.okta.com/admin/sso/callback","grantedScopes":"openid","responseMode":"query","requestUri":"/oauth2/v1/authorize","requestedScopes":"openid","userId":"00ua65e9m11opLyFx5d7","url":"/oauth2/v1/authorize?response_type=code&response_mode=query&client_id=okta.b58d5b75-07d4-5f25-bf59-368a1261a405&redirect_uri=https%3A%2F%2Fdev-33855097-admin.okta.com%2Fadmin%2Fsso%2Fcallback&scope=openid&state=7aDReRR8YWxXc6v_cEwFaLcOp_5nD3o2&nonce=blxIQEqpcYCbmcUhdUtWTfBEYscsGxpO&code_challenge=hmHCgLVX5wxrxUXRkqO2oo7FdwZUIFoOP_OqCFMSjoo&code_challenge_method=S256","responseType":"code","authnRequestId":"ZJxnNW7Sm0yWs5HZanbtegAADww","requestId":"ZJxnNW7Sm0yWs5HZanbtegAADww","dtHash":"9b21b9e5015f3408488786e7e10f91c0db48e97f3eeaec560efcd9c190d90416","state":"7aDReRR8YWxXc6v_cEwFaLcOp_5nD3o2","threatSuspected":"false","grantType":"authorization_code"}},"legacyEventType":"app.oauth2.authorize.code_success","transaction":{"type":"WEB","id":"ZJxnNW7Sm0yWs5HZanbtegAADww","detail":{}},"uuid":"4e50324f-15d5-11ee-8bd1-b1062595cad2","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":null,"displayName":null,"detailEntry":null},{"id":"kdwcjJX6dfsZbeQXZ8aUhg","type":"code","alternateId":null,"displayName":"Authorization Code","detailEntry":null}]},"emitted_at":1687974341879} +{"stream":"logs","data":{"actor":{"id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","type":"PublicClientApp","alternateId":"0oaa65adoxfVQdKKA5d7","displayName":"Okta Admin Console","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Okta-Integrations","os":"Unknown","browser":"UNKNOWN"},"zone":"null","device":"Unknown","id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","ipAddress":"54.189.2.171","geographicalContext":{"city":"Boardman","state":"Oregon","country":"United States","postalCode":"97818","geolocation":{"lat":45.8234,"lon":-119.7257}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"OIDC access token is granted","eventType":"app.oauth2.token.grant.access_token","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:38Z","securityContext":{"asNumber":16509,"asOrg":"amazon.com inc.","isp":"amazon.com inc","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"clientAuthType":"none","redirectUri":"https://dev-33855097-admin.okta.com/admin/sso/callback","grantedScopes":"openid","authCode":"kdwcjJX6dfsZbeQXZ8aUhg","requestId":"ZJxnNhQZPf1OaTNPsbg_eQAADGs","responseTime":"407","dtHash":"4bb3d570e1bf49120b770ff806c3884ec615a4c635d79d47e2f716c4dd18e62d","requestUri":"/oauth2/v1/token","requestedScopes":"","threatSuspected":"false","grantType":"authorization_code","url":"/oauth2/v1/token?"}},"legacyEventType":"app.oauth2.token.grant.access_token_success","transaction":{"type":"WEB","id":"ZJxnNhQZPf1OaTNPsbg_eQAADGs","detail":{}},"uuid":"4ec47876-15d5-11ee-9b7d-3348eeca6465","version":"0","request":{"ipChain":[{"ip":"54.189.2.171","geographicalContext":{"city":"Boardman","state":"Oregon","country":"United States","postalCode":"97818","geolocation":{"lat":45.8234,"lon":-119.7257}},"version":"V4","source":null}]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":null,"displayName":null,"detailEntry":null},{"id":"AT.-WfV3QSKhqV2RkPYs6Bju2D68grMj9aaTMLO-C0-eoY","type":"access_token","alternateId":null,"displayName":"Access Token","detailEntry":{"expires":"2023-06-28T18:00:38.000Z","subject":"00ua65e9m11opLyFx5d7","hash":"ZQFJPFflk82N5_8e5eZ2dw"}}]},"emitted_at":1687974341879} +{"stream":"logs","data":{"actor":{"id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","type":"PublicClientApp","alternateId":"0oaa65adoxfVQdKKA5d7","displayName":"Okta Admin Console","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Okta-Integrations","os":"Unknown","browser":"UNKNOWN"},"zone":"null","device":"Unknown","id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","ipAddress":"54.189.2.171","geographicalContext":{"city":"Boardman","state":"Oregon","country":"United States","postalCode":"97818","geolocation":{"lat":45.8234,"lon":-119.7257}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"OIDC id token is granted","eventType":"app.oauth2.token.grant.id_token","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:38Z","securityContext":{"asNumber":16509,"asOrg":"amazon.com inc.","isp":"amazon.com inc","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"clientAuthType":"none","redirectUri":"https://dev-33855097-admin.okta.com/admin/sso/callback","grantedScopes":"openid","authCode":"kdwcjJX6dfsZbeQXZ8aUhg","requestId":"ZJxnNhQZPf1OaTNPsbg_eQAADGs","responseTime":"405","dtHash":"4bb3d570e1bf49120b770ff806c3884ec615a4c635d79d47e2f716c4dd18e62d","requestUri":"/oauth2/v1/token","requestedScopes":"","threatSuspected":"false","grantType":"authorization_code","url":"/oauth2/v1/token?"}},"legacyEventType":"app.oauth2.token.grant.id_token_success","transaction":{"type":"WEB","id":"ZJxnNhQZPf1OaTNPsbg_eQAADGs","detail":{}},"uuid":"4ec42a54-15d5-11ee-9b7d-3348eeca6465","version":"0","request":{"ipChain":[{"ip":"54.189.2.171","geographicalContext":{"city":"Boardman","state":"Oregon","country":"United States","postalCode":"97818","geolocation":{"lat":45.8234,"lon":-119.7257}},"version":"V4","source":null}]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":null,"displayName":null,"detailEntry":null},{"id":"ID.n4qmgCEY7qHbnryKXajJjwlXIpWf1IMGmJDZgtYUjaE","type":"id_token","alternateId":null,"displayName":"ID Token","detailEntry":{"audience":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","expires":"2023-06-28T18:00:38.000Z","subject":"00ua65e9m11opLyFx5d7","hash":"XVzwSxB1qa-9ZtH4eynVpQ"}}]},"emitted_at":1687974341879} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Okta-Integrations","os":"Unknown","browser":"UNKNOWN"},"zone":"null","device":"Unknown","id":"okta.b58d5b75-07d4-5f25-bf59-368a1261a405","ipAddress":"54.189.2.171","geographicalContext":{"city":"Boardman","state":"Oregon","country":"United States","postalCode":"97818","geolocation":{"lat":45.8234,"lon":-119.7257}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"User single sign on to app","eventType":"user.authentication.sso","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:38Z","securityContext":{"asNumber":16509,"asOrg":"amazon.com inc.","isp":"amazon.com inc","domain":"amazonaws.com","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"initiationType":"NA","redirectUri":"https://dev-33855097-admin.okta.com/admin/sso/callback","requestId":"ZJxnNhQZPf1OaTNPsbg_eQAADGs","dtHash":"4bb3d570e1bf49120b770ff806c3884ec615a4c635d79d47e2f716c4dd18e62d","signOnMode":"OpenID Connect","requestUri":"/oauth2/v1/token","threatSuspected":"false","url":"/oauth2/v1/token?"}},"legacyEventType":"app.auth.sso","transaction":{"type":"WEB","id":"ZJxnNhQZPf1OaTNPsbg_eQAADGs","detail":{}},"uuid":"4ec45165-15d5-11ee-9b7d-3348eeca6465","version":"0","request":{"ipChain":[{"ip":"54.189.2.171","geographicalContext":{"city":"Boardman","state":"Oregon","country":"United States","postalCode":"97818","geolocation":{"lat":45.8234,"lon":-119.7257}},"version":"V4","source":null}]},"target":[{"id":"0oaa65adoxfVQdKKA5d7","type":"AppInstance","alternateId":"Okta Admin Console","displayName":"Okta Admin Console","detailEntry":{"signOnModeType":"OPENID_CONNECT"}},{"id":"0uaa65e9m588OBEHu5d7","type":"AppUser","alternateId":"unknown","displayName":"Sherif nada","detailEntry":null}]},"emitted_at":1687974341880} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"User accessing Okta admin app","eventType":"user.session.access_admin_app","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:00:38Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxnNrtrAPRWNWOcqksQ5AAADyo","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/admin/sso/callback","url":"/admin/sso/callback?code=******&state=7aDReRR8YWxXc6v_cEwFaLcOp_5nD3o2"}},"legacyEventType":"app.admin.sso.login.success","transaction":{"type":"WEB","id":"ZJxnNrtrAPRWNWOcqksQ5AAADyo","detail":{}},"uuid":"4ed197be-15d5-11ee-8809-d7aec7422f96","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua65e9m11opLyFx5d7","type":"AppUser","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null}]},"emitted_at":1687974341880} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Create API token","eventType":"system.api_token.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:05:38Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"concurrencyPercentage":"50","requestId":"ZJxoYgSJ0THbgswl5_LNZQAADoA","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","rateLimitPercentage":"50","requestUri":"/api/internal/tokens","url":"/api/internal/tokens?expand=user"}},"legacyEventType":"api.token.create","transaction":{"type":"WEB","id":"ZJxoYgSJ0THbgswl5_LNZQAADoA","detail":{}},"uuid":"01315a08-15d6-11ee-bceb-81ca998ea143","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00T159py03CikoIKe5d7","type":"Token","alternateId":"unknown","displayName":"Sandbox","detailEntry":null}]},"emitted_at":1687974341880} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Create okta group","eventType":"group.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:08:11Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxo-5zjIR0nvsZU_bXH_QAADfU","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/groups","url":"/api/v1/groups?"}},"legacyEventType":"group.lifecycle.create","transaction":{"type":"WEB","id":"ZJxo-5zjIR0nvsZU_bXH_QAADfU","detail":{}},"uuid":"5c9d6ea6-15d6-11ee-8bd1-b1062595cad2","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ga67rapdEXY9PVo5d7","type":"UserGroup","alternateId":"unknown","displayName":"test-group-1","detailEntry":null}]},"emitted_at":1687974341881} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Create okta user","eventType":"user.lifecycle.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:06Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/users","url":"/api/v1/users?activate=true"}},"legacyEventType":"core.user.config.user_creation.success","transaction":{"type":"WEB","id":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","detail":{}},"uuid":"7d674b29-15d6-11ee-8f12-91efa33480e0","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341881} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Activate factor for user","eventType":"user.mfa.factor.activate","outcome":{"result":"SUCCESS","reason":"User set up EMAIL_FACTOR factor"},"published":"2023-06-28T17:09:06Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/users","url":"/api/v1/users?activate=true"}},"legacyEventType":"core.user.factor.activate","transaction":{"type":"WEB","id":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","detail":{}},"uuid":"7d65c486-15d6-11ee-8f12-91efa33480e0","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341881} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Add user to group membership","eventType":"group.user_membership.add","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:06Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/users","url":"/api/v1/users?activate=true"}},"legacyEventType":"core.user_group_member.user_add","transaction":{"type":"WEB","id":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","detail":{}},"uuid":"7d672417-15d6-11ee-8f12-91efa33480e0","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},{"id":"00ga67rapdEXY9PVo5d7","type":"UserGroup","alternateId":"unknown","displayName":"test-group-1","detailEntry":null}]},"emitted_at":1687974341882} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Activate Okta user","eventType":"user.lifecycle.activate","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:06Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/users","url":"/api/v1/users?activate=true"}},"legacyEventType":"core.user.config.user_activated","transaction":{"type":"WEB","id":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U","detail":{}},"uuid":"7d847022-15d6-11ee-8f12-91efa33480e0","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341882} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"Evaluation of sign-on policy","eventType":"policy.evaluate_sign_on","outcome":{"result":"CHALLENGE","reason":"Sign-on policy evaluation resulted in ENROLL"},"published":"2023-06-28T17:09:24Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"redirectUri":"https://dev-33855097.okta.com/enduser/callback","authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","requestUri":"/oauth2/v1/authorize","threatSuspected":"false","url":"/oauth2/v1/authorize?client_id=okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26&code_challenge=l5huxrHnTz3XnPDd4TuJMMOQbZNTN0CMaN6nVolnrQg&code_challenge_method=S256&nonce=Wjhl0s0xiVnTfYlNNwRvJpGvSWOLSXuF10o50QTbOZDIxE0PGEhDe476N9hQfKRz&redirect_uri=https%3A%2F%2Fdev-33855097.okta.com%2Fenduser%2Fcallback&response_type=code&state=p6V5Fr0RphawgVxNOs8r2jMvJJPqc6ttjFIcWefTjKkM5CyUAfLCg6ZzLtn3e0uM&scope=openid+profile+email+okta.users.read.self+okta.users.manage.self+okta.internal.enduser.read+okta.internal.enduser.manage+okta.enduser.dashboard.read+okta.enduser.dashboard.manage"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","detail":{}},"uuid":"87f66352-15d6-11ee-9500-5b3272fdf952","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":{"signOnModeType":"OPENID_CONNECT","signOnModeEvaluationResult":"CHALLENGE"}},{"id":"0pra65adpda3WSCrJ5d7","type":"Rule","alternateId":"unknown","displayName":"Default Rule","detailEntry":null},{"id":"rula67k7b3eKaojXn5d7","type":"Rule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341882} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"Fired when the user's Okta password is reset","eventType":"user.account.reset_password","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:46Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","requestUri":"/idp/idx/challenge/answer","threatSuspected":"false","url":"/idp/idx/challenge/answer?"}},"legacyEventType":"core.user.config.user_status.password_reset","transaction":{"type":"WEB","id":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","detail":{}},"uuid":"95665471-15d6-11ee-ac29-216e0139dfde","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341882} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"User update password for Okta","eventType":"user.account.update_password","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:46Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","requestUri":"/idp/idx/challenge/answer","threatSuspected":"false","url":"/idp/idx/challenge/answer?"}},"legacyEventType":"core.user.config.password_update.success","transaction":{"type":"WEB","id":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","detail":{}},"uuid":"9562d1fe-15d6-11ee-ac29-216e0139dfde","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341883} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"Activate factor for user","eventType":"user.mfa.factor.activate","outcome":{"result":"SUCCESS","reason":"User set up PASSWORD_AS_FACTOR factor"},"published":"2023-06-28T17:09:46Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","requestUri":"/idp/idx/challenge/answer","threatSuspected":"false","url":"/idp/idx/challenge/answer?"}},"legacyEventType":"core.user.factor.activate","transaction":{"type":"WEB","id":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","detail":{}},"uuid":"9566c9a2-15d6-11ee-ac29-216e0139dfde","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341883} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"User login to Okta","eventType":"user.session.start","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:46Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","origin":"https://dev-33855097.okta.com","requestUri":"/idp/idx/challenge/answer","threatSuspected":"false","url":"/idp/idx/challenge/answer?"}},"legacyEventType":"core.user_auth.login_success","transaction":{"type":"WEB","id":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","detail":{}},"uuid":"95756fa8-15d6-11ee-ac29-216e0139dfde","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"lae159ppwdjvLJv4r5d7","type":"AuthenticatorEnrollment","alternateId":"unknown","displayName":"Password","detailEntry":null},{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":null}]},"emitted_at":1687974341883} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"Evaluation of sign-on policy","eventType":"policy.evaluate_sign_on","outcome":{"result":"ALLOW","reason":"Sign-on policy evaluation resulted in AUTHENTICATED"},"published":"2023-06-28T17:09:46Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","authMethodFirstVerificationTime":"2023-06-28T17:09:46.908Z","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","authMethodFirstType":"okta_password:password:auta67k7a2sFIKG5f5d7","authMethodFirstEnrollment":"lae159ppwdjvLJv4r5d7","requestUri":"/idp/idx/challenge/answer","threatSuspected":"false","url":"/idp/idx/challenge/answer?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxpWpzjIR0nvsZU_bXMFgAADUg","detail":{}},"uuid":"95754897-15d6-11ee-ac29-216e0139dfde","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":{"signOnModeType":"OPENID_CONNECT","signOnModeEvaluationResult":"AUTHENTICATED"}},{"id":"0pra65adpda3WSCrJ5d7","type":"Rule","alternateId":"unknown","displayName":"Default Rule","detailEntry":null},{"id":"rula67k7b3eKaojXn5d7","type":"Rule","alternateId":"unknown","displayName":"Catch-all Rule","detailEntry":null}]},"emitted_at":1687974341883} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"Verify user identity","eventType":"user.authentication.verify","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:50Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpXdMS_hEOdUq2kuCkIAAADOA","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","requestUri":"/idp/idx/skip","threatSuspected":"false","url":"/idp/idx/skip?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxpXdMS_hEOdUq2kuCkIAAADOA","detail":{}},"uuid":"976fb87b-15d6-11ee-9eaf-b91961435c97","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":null},"emitted_at":1687974341884} +{"stream":"logs","data":{"actor":{"id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","type":"PublicClientApp","alternateId":"0oaa65e9l9OO7WnUi5d7","displayName":"Okta Dashboard","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"idxz5zZFpdRQyK1V_59BbGb1g"},"displayMessage":"OIDC authorization code request","eventType":"app.oauth2.authorize.code","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:50Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"redirectUri":"https://dev-33855097.okta.com/enduser/callback","grantedScopes":"openid, profile, email, okta.users.read.self, okta.users.manage.self, okta.internal.enduser.read, okta.internal.enduser.manage, okta.enduser.dashboard.read, okta.enduser.dashboard.manage","responseMode":"query","requestUri":"/login/token/redirect","requestedScopes":"openid, profile, email, okta.users.read.self, okta.users.manage.self, okta.internal.enduser.read, okta.internal.enduser.manage, okta.enduser.dashboard.read, okta.enduser.dashboard.manage","userId":"00ua67q26u6ozRG5T5d7","url":"/login/token/redirect?stateToken=02.id.QkpU5lPEbO5BxuZPoy_VYTS-DIImlpOmMvlitInC","responseType":"code","authnRequestId":"ZJxpQ5zjIR0nvsZU_bXLLQAADUg","requestId":"ZJxpXm8gTiKzUr7sEzq9RwAAAnE","dtHash":"b94009bd8cd0aa8ab2a708d5c999589988be2899fe8e311e5ca02c04491b616a","state":"p6V5Fr0RphawgVxNOs8r2jMvJJPqc6ttjFIcWefTjKkM5CyUAfLCg6ZzLtn3e0uM","threatSuspected":"false","grantType":"authorization_code"}},"legacyEventType":"app.oauth2.authorize.code_success","transaction":{"type":"WEB","id":"ZJxpXm8gTiKzUr7sEzq9RwAAAnE","detail":{}},"uuid":"97d29955-15d6-11ee-9cdf-4584c76be0a6","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":null,"displayName":null,"detailEntry":null},{"id":"clfSVuVkZIYutqIiIg1R8g","type":"code","alternateId":null,"displayName":"Authorization Code","detailEntry":null}]},"emitted_at":1687974341884} +{"stream":"logs","data":{"actor":{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"User single sign on to app","eventType":"user.authentication.sso","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:52Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"initiationType":"NA","redirectUri":"https://dev-33855097.okta.com/enduser/callback","requestId":"ZJxpX90Kxoie1BdGxh-9kgAAAEA","dtHash":"1fb541e6679bd3c23b60f744494acf958509855ef589847959170f752b98550e","signOnMode":"OpenID Connect","requestUri":"/oauth2/v1/token","threatSuspected":"false","url":"/oauth2/v1/token?"}},"legacyEventType":"app.auth.sso","transaction":{"type":"WEB","id":"ZJxpX90Kxoie1BdGxh-9kgAAAEA","detail":{}},"uuid":"98aaf79d-15d6-11ee-9247-addfd94b48e5","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"0oaa65e9l9OO7WnUi5d7","type":"AppInstance","alternateId":"Okta Dashboard","displayName":"Okta Dashboard","detailEntry":{"signOnModeType":"OPENID_CONNECT"}},{"id":"0uaa67nrx5MksUfnu5d7","type":"AppUser","alternateId":"integration-test+1@daxtarity.com","displayName":"integration test1","detailEntry":null}]},"emitted_at":1687974341884} +{"stream":"logs","data":{"actor":{"id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","type":"PublicClientApp","alternateId":"0oaa65e9l9OO7WnUi5d7","displayName":"Okta Dashboard","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"OIDC id token is granted","eventType":"app.oauth2.token.grant.id_token","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:52Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"clientAuthType":"none","redirectUri":"https://dev-33855097.okta.com/enduser/callback","grantedScopes":"openid, profile, email, okta.users.read.self, okta.users.manage.self, okta.internal.enduser.read, okta.internal.enduser.manage, okta.enduser.dashboard.read, okta.enduser.dashboard.manage","authCode":"clfSVuVkZIYutqIiIg1R8g","requestId":"ZJxpX90Kxoie1BdGxh-9kgAAAEA","responseTime":"419","dtHash":"1fb541e6679bd3c23b60f744494acf958509855ef589847959170f752b98550e","requestUri":"/oauth2/v1/token","requestedScopes":"","threatSuspected":"false","grantType":"authorization_code","url":"/oauth2/v1/token?"}},"legacyEventType":"app.oauth2.token.grant.id_token_success","transaction":{"type":"WEB","id":"ZJxpX90Kxoie1BdGxh-9kgAAAEA","detail":{}},"uuid":"98aad08c-15d6-11ee-9247-addfd94b48e5","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":null,"displayName":null,"detailEntry":null},{"id":"ID.a3SCrYn3t1S2GXvKp5Z7rUGotH6pSKI3WFQ2kTFN3zk","type":"id_token","alternateId":null,"displayName":"ID Token","detailEntry":{"audience":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","expires":"2023-06-28T18:09:52.000Z","subject":"00ua67q26u6ozRG5T5d7","hash":"YKR-p5_EqeRwJta2d1ykbQ"}}]},"emitted_at":1687974341884} +{"stream":"logs","data":{"actor":{"id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","type":"PublicClientApp","alternateId":"0oaa65e9l9OO7WnUi5d7","displayName":"Okta Dashboard","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Safari/605.1.15","os":"Mac OS X","browser":"SAFARI"},"zone":"null","device":"Computer","id":"okta.2b1959c8-bcc0-56eb-a589-cfcfb7422f26","ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"unknown"},"displayMessage":"OIDC access token is granted","eventType":"app.oauth2.token.grant.access_token","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:09:52Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"clientAuthType":"none","redirectUri":"https://dev-33855097.okta.com/enduser/callback","grantedScopes":"openid, profile, email, okta.users.read.self, okta.users.manage.self, okta.internal.enduser.read, okta.internal.enduser.manage, okta.enduser.dashboard.read, okta.enduser.dashboard.manage","authCode":"clfSVuVkZIYutqIiIg1R8g","requestId":"ZJxpX90Kxoie1BdGxh-9kgAAAEA","responseTime":"421","dtHash":"1fb541e6679bd3c23b60f744494acf958509855ef589847959170f752b98550e","requestUri":"/oauth2/v1/token","requestedScopes":"","threatSuspected":"false","grantType":"authorization_code","url":"/oauth2/v1/token?"}},"legacyEventType":"app.oauth2.token.grant.access_token_success","transaction":{"type":"WEB","id":"ZJxpX90Kxoie1BdGxh-9kgAAAEA","detail":{}},"uuid":"98ab1eae-15d6-11ee-9247-addfd94b48e5","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"00ua67q26u6ozRG5T5d7","type":"User","alternateId":null,"displayName":null,"detailEntry":null},{"id":"AT.M0HSRBZ5ujSj4vEdfkk4OollCr1KuNZE_kUlwWyij0g","type":"access_token","alternateId":null,"displayName":"Access Token","detailEntry":{"expires":"2023-06-28T18:09:51.000Z","subject":"00ua67q26u6ozRG5T5d7","hash":"tEw6BNhWVaaeVsUKRh8wYw"}}]},"emitted_at":1687974341885} +{"stream":"logs","data":{"actor":{"id":"unknown","type":"SystemPrincipal","alternateId":"system@okta.com","displayName":"Okta System","detailEntry":null},"client":{"userAgent":null,"zone":null,"device":null,"id":null,"ipAddress":null,"geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsj5c_HKi0QUONgLplV8VJog"},"displayMessage":"Email delivery","eventType":"system.email.delivery","outcome":{"result":"SUCCESS","reason":"delivered"},"published":"2023-06-28T17:10:05Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"INFO","debugContext":{"debugData":{"appContextName":"None","emailProvider":"sendgrid","category":"email.welcome","userId":"","emailRequestId":"ZJxpMiG2_rzg4ODYnbBwZQAAC6U"}},"legacyEventType":null,"transaction":{"type":"JOB","id":"ok12-jobecs02c.auw2-ok12.internal16879036778761687903702785","detail":{}},"uuid":"a07c9da5-15d6-11ee-be29-6f36b9971d20","version":"0","request":{"ipChain":[]},"target":[{"id":"integration-test+1@daxtarity.com","type":"email","alternateId":null,"displayName":"integration-test+1@daxtarity.com","detailEntry":null}]},"emitted_at":1687974341885} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"python-requests/2.31.0","os":"Unknown","browser":"UNKNOWN"},"zone":"null","device":"Unknown","id":null,"ipAddress":"107.115.45.34","geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsW1Uz2Px-TsSAosoTWzzHFA"},"displayMessage":"This API token has made too many requests","eventType":"system.operation.rate_limit.violation","outcome":null,"published":"2023-06-28T17:18:42Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"WARN","debugContext":{"debugData":{"operationRateLimitSubtype":"ssws_token","operationRateLimitTimeUnit":"MINUTES","operationRateLimitScopeType":"token","operationRateLimitSecondsToReset":"36","requestId":"ZJxrcjnBHN2CsYaHyf_WUAAAASg","dtHash":"ca286cee3ce2cc6a85e97ad17242ef2ca198cee2fdfa4991c0f65df26f7a444f","operationRateLimitThreshold":"10","operationRateLimitTimeSpan":"1","requestUri":"/api/v1/logs","operationRateLimitType":"web_request","url":"/api/v1/logs?limit=200&since=2023-06-28T17%3A10%3A05Z"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxrcjnBHN2CsYaHyf_WUAAAASg","detail":{"requestApiTokenId":"00T159py03CikoIKe5d7"}},"uuid":"d485e2ca-15d7-11ee-b270-0500528109a7","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":null,"version":"V4","source":null}]},"target":[{"id":"b192d91c-b242-36da-9332-d97a5579f865","type":"Bucket UUID","alternateId":null,"displayName":null,"detailEntry":null},{"id":"00T159py03CikoIKe5d7","type":"Token","alternateId":"unknown","displayName":"Sandbox","detailEntry":null}]},"emitted_at":1687974341885} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"python-requests/2.31.0","os":"Unknown","browser":"UNKNOWN"},"zone":"null","device":"Unknown","id":null,"ipAddress":"107.115.45.34","geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsH33-aJLRQOevPwEvkXjKow"},"displayMessage":"This API token has made too many requests","eventType":"system.operation.rate_limit.violation","outcome":null,"published":"2023-06-28T17:19:42Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"WARN","debugContext":{"debugData":{"operationRateLimitSubtype":"ssws_token","operationRateLimitTimeUnit":"MINUTES","operationRateLimitScopeType":"token","operationRateLimitSecondsToReset":"41","requestId":"ZJxrrq5enWGYMHaiHXuD1AAADKY","dtHash":"ca286cee3ce2cc6a85e97ad17242ef2ca198cee2fdfa4991c0f65df26f7a444f","operationRateLimitThreshold":"10","operationRateLimitTimeSpan":"1","requestUri":"/api/v1/logs","operationRateLimitType":"web_request","url":"/api/v1/logs?limit=200&since=2023-06-21T20%3A49%3A13%2B00%3A00&after=1687972205543_1"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxrrq5enWGYMHaiHXuD1AAADKY","detail":{"requestApiTokenId":"00T159py03CikoIKe5d7"}},"uuid":"f849ec40-15d7-11ee-b43b-b18cf4dad6cb","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":null,"version":"V4","source":null}]},"target":[{"id":"b192d91c-b242-36da-9332-d97a5579f865","type":"Bucket UUID","alternateId":null,"displayName":null,"detailEntry":null},{"id":"00T159py03CikoIKe5d7","type":"Token","alternateId":"unknown","displayName":"Sandbox","detailEntry":null}]},"emitted_at":1687974341885} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"python-requests/2.31.0","os":"Unknown","browser":"UNKNOWN"},"zone":"null","device":"Unknown","id":null,"ipAddress":"107.115.45.34","geographicalContext":null},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"trsGJgqKQYtT7W-5bW2RGApYw"},"displayMessage":"This API token has made too many requests","eventType":"system.operation.rate_limit.violation","outcome":null,"published":"2023-06-28T17:20:47Z","securityContext":{"asNumber":null,"asOrg":null,"isp":null,"domain":null,"isProxy":null},"severity":"WARN","debugContext":{"debugData":{"operationRateLimitSubtype":"ssws_token","operationRateLimitTimeUnit":"MINUTES","operationRateLimitScopeType":"token","operationRateLimitSecondsToReset":"41","requestId":"ZJxr7yUlXOCBVe_VfeMAfQAAB5w","dtHash":"ca286cee3ce2cc6a85e97ad17242ef2ca198cee2fdfa4991c0f65df26f7a444f","operationRateLimitThreshold":"10","operationRateLimitTimeSpan":"1","requestUri":"/api/v1/logs","operationRateLimitType":"web_request","url":"/api/v1/logs?limit=200&since=2023-06-28T17%3A10%3A05Z&after=1687972782382_1"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxr7yUlXOCBVe_VfeMAfQAAB5w","detail":{"requestApiTokenId":"00T159py03CikoIKe5d7"}},"uuid":"1ef5fa17-15d8-11ee-982b-0159ee9d4860","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":null,"version":"V4","source":null}]},"target":[{"id":"b192d91c-b242-36da-9332-d97a5579f865","type":"Bucket UUID","alternateId":null,"displayName":null,"detailEntry":null},{"id":"00T159py03CikoIKe5d7","type":"Token","alternateId":"unknown","displayName":"Sandbox","detailEntry":null}]},"emitted_at":1687974341886} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Role created","eventType":"iam.role.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:39:34Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxwVpPigb6USF3bbX6xIgAACCE","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/iam/roles","url":"/api/v1/iam/roles?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxwVpPigb6USF3bbX6xIgAACCE","detail":{}},"uuid":"bec5ed30-15da-11ee-b89b-8136c54c7096","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"cr0a688qm6u91Un0m5d7","type":"Role","alternateId":"unknown","displayName":"user-viewer-role","detailEntry":null},{"id":"okta.users.read","type":"Permission","alternateId":"unknown","displayName":"okta.users.read","detailEntry":null}]},"emitted_at":1687974341886} +{"stream":"logs","data":{"actor":{"id":"00ua65e9m11opLyFx5d7","type":"User","alternateId":"integration-test@daxtarity.com","displayName":"Sherif nada","detailEntry":null},"client":{"userAgent":{"rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36","os":"Mac OS X","browser":"CHROME"},"zone":"null","device":"Computer","id":null,"ipAddress":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}}},"device":null,"authenticationContext":{"authenticationProvider":null,"credentialProvider":null,"credentialType":null,"issuer":null,"interface":null,"authenticationStep":0,"externalSessionId":"102CALG0t2iSlySu5Bvt2htFg"},"displayMessage":"Resource set created","eventType":"iam.resourceset.create","outcome":{"result":"SUCCESS","reason":null},"published":"2023-06-28T17:39:57Z","securityContext":{"asNumber":7018,"asOrg":"at&t mobility llc","isp":"att services inc","domain":".","isProxy":false},"severity":"INFO","debugContext":{"debugData":{"requestId":"ZJxwbQSL5n4Ain-wBU92KQAABfA","dtHash":"e6d02235b0ae0a1ddbf68aa2d6acfb39e4e570ca150b932e7180031a755782fd","requestUri":"/api/v1/iam/resource-sets","url":"/api/v1/iam/resource-sets?"}},"legacyEventType":null,"transaction":{"type":"WEB","id":"ZJxwbQSL5n4Ain-wBU92KQAABfA","detail":{}},"uuid":"cce5f39c-15da-11ee-b171-b9afd2e39b0a","version":"0","request":{"ipChain":[{"ip":"107.115.45.34","geographicalContext":{"city":"Las Vegas","state":"Nevada","country":"United States","postalCode":"89110","geolocation":{"lat":36.1724,"lon":-115.0677}},"version":"V4","source":null}]},"target":[{"id":"iama688hw0VLl1JrA5d7","type":"ResourceSet","alternateId":"unknown","displayName":"test-resource-set1","detailEntry":null},{"id":"00ga67rapdEXY9PVo5d7","type":"UserGroup","alternateId":null,"displayName":"All users within group test-group-1","detailEntry":null}]},"emitted_at":1687974341886} diff --git a/airbyte-integrations/connectors/source-okta/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-okta/integration_tests/incremental_configured_catalog.json index 3a00736d94d0..476da832a756 100644 --- a/airbyte-integrations/connectors/source-okta/integration_tests/incremental_configured_catalog.json +++ b/airbyte-integrations/connectors/source-okta/integration_tests/incremental_configured_catalog.json @@ -22,28 +22,6 @@ "cursor_field": ["lastUpdated"], "primary_key": [["id"]] }, - { - "stream": { - "name": "group_members", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["id"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "resource_sets", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["id"], - "primary_key": [["id"]] - }, { "stream": { "name": "logs", diff --git a/airbyte-integrations/connectors/source-okta/metadata.yaml b/airbyte-integrations/connectors/source-okta/metadata.yaml index 16106095f1c3..011fb07a3b8b 100644 --- a/airbyte-integrations/connectors/source-okta/metadata.yaml +++ b/airbyte-integrations/connectors/source-okta/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 1d4fdb25-64fc-4569-92da-fcdca79a8372 - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.16 dockerRepository: airbyte/source-okta + documentationUrl: https://docs.airbyte.com/integrations/sources/okta githubIssueLabel: source-okta icon: okta.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/okta + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-okta/requirements.txt b/airbyte-integrations/connectors/source-okta/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-okta/requirements.txt +++ b/airbyte-integrations/connectors/source-okta/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-okta/setup.py b/airbyte-integrations/connectors/source-okta/setup.py index 630b04d8b0e9..a3fac533b39e 100644 --- a/airbyte-integrations/connectors/source-okta/setup.py +++ b/airbyte-integrations/connectors/source-okta/setup.py @@ -12,7 +12,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", - "connector-acceptance-test", "pytest-mock~=3.6.1", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-okta/source_okta/__init__.py b/airbyte-integrations/connectors/source-okta/source_okta/__init__.py index c0977ddc873c..7b52aaf00071 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/__init__.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# """ MIT License diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/custom_roles.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/custom_roles.json index f696a1139aa4..0f1a9ad700fe 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/custom_roles.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/custom_roles.json @@ -6,6 +6,9 @@ "id": { "type": ["null", "string"] }, + "description": { + "type": ["null", "string"] + }, "label": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json index c96ece1029ea..e9e26b10714e 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_members.json @@ -78,6 +78,9 @@ } } }, + "groupId": { + "type": ["null", "string"] + }, "id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json index c55529918cf5..8a80731d4524 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/group_role_assignments.json @@ -1,6 +1,9 @@ { "type": ["null", "object"], "properties": { + "groupId": { + "type": ["null", "string"] + }, "id": { "type": ["null", "string"] }, @@ -27,6 +30,9 @@ "resource-set": { "type": ["string", "null"] }, + "role": { + "type": ["null", "string"] + }, "_links": { "type": ["object", "null"], "properties": { diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json index ea0b25e816ab..2070c9f6f7aa 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/logs.json @@ -121,6 +121,9 @@ }, "type": ["object", "null"] }, + "device": { + "type": ["string", "null"] + }, "displayMessage": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/permissions.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/permissions.json index e3c68176645c..a69acbf0d047 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/permissions.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/permissions.json @@ -39,6 +39,23 @@ "type": "string", "description": "Type of permissions" }, + "conditions": { + "type": ["null", "object"], + "properties": { + "included": { + "type": "array", + "items": { + "type": "object" + } + }, + "excluded": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, "created": { "format": "date-time", "type": "string" diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/resource_sets.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/resource_sets.json index 0741df0fedf9..a4bcd284f2e5 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/resource_sets.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/resource_sets.json @@ -9,6 +9,14 @@ "description": { "type": "string" }, + "created": { + "format": "date-time", + "type": ["string", "null"] + }, + "lastUpdated": { + "format": "date-time", + "type": ["string", "null"] + }, "_links": { "properties": { "assignee": { diff --git a/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json b/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json index c55529918cf5..40abf57fc9cd 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json +++ b/airbyte-integrations/connectors/source-okta/source_okta/schemas/user_role_assignments.json @@ -1,6 +1,9 @@ { "type": ["null", "object"], "properties": { + "userId": { + "type": ["null", "string"] + }, "id": { "type": ["null", "string"] }, @@ -27,6 +30,9 @@ "resource-set": { "type": ["string", "null"] }, + "role": { + "type": ["string", "null"] + }, "_links": { "type": ["object", "null"], "properties": { diff --git a/airbyte-integrations/connectors/source-okta/source_okta/source.py b/airbyte-integrations/connectors/source-okta/source_okta/source.py index b80f3dbbed51..8afd364da703 100644 --- a/airbyte-integrations/connectors/source-okta/source_okta/source.py +++ b/airbyte-integrations/connectors/source-okta/source_okta/source.py @@ -60,12 +60,16 @@ def request_params( **(next_page_token or {}), } - def parse_response( - self, - response: requests.Response, - **kwargs, - ) -> Iterable[Mapping]: - yield from response.json() + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + if isinstance(response_json, list): + for record in response_json: + yield self.transform(record=record, **kwargs) + else: + yield self.transform(record=response_json, **kwargs) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + return record def backoff_time(self, response: requests.Response) -> Optional[float]: # The rate limit resets on the timestamp indicated @@ -115,15 +119,17 @@ def path(self, **kwargs) -> str: return "groups" -class GroupMembers(IncrementalOktaStream): +class GroupMembers(OktaStream): cursor_field = "id" - primary_key = "id" + primary_key = ["groupId", "id"] use_cache = True + reset_token = False min_id = "00u00000000000000000" def stream_slices(self, **kwargs): group_stream = Groups(authenticator=self.authenticator, url_base=self.url_base, start_date=self.start_date) for group in group_stream.read_records(sync_mode=SyncMode.full_refresh): + self.reset_token = True yield {"group_id": group["id"]} def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -136,14 +142,23 @@ def request_params( stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super(IncrementalOktaStream, self).request_params(stream_state, stream_slice, next_page_token) + params = {"limit": self.page_size} latest_entry = stream_state.get(self.cursor_field) if stream_state else self.min_id + if next_page_token: + latest_entry = next_page_token.get("after") + if self.reset_token: + latest_entry = self.min_id + self.reset_token = False params["after"] = latest_entry return params + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["groupId"] = stream_slice["group_id"] + return record + class GroupRoleAssignments(OktaStream): - primary_key = "id" + primary_key = ["groupId", "id"] use_cache = True def stream_slices(self, **kwargs): @@ -155,6 +170,10 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: group_id = stream_slice["group_id"] return f"groups/{group_id}/roles" + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["groupId"] = stream_slice["group_id"] + return record + class Logs(IncrementalOktaStream): @@ -240,8 +259,7 @@ def request_params( return params -class ResourceSets(IncrementalOktaStream): - cursor_field = "id" +class ResourceSets(OktaStream): primary_key = "id" min_id = "iam00000000000000000" @@ -266,18 +284,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return None - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - latest_entry = stream_state.get(self.cursor_field) - if latest_entry: - params["after"] = latest_entry - return params - class CustomRoles(OktaStream): # https://developer.okta.com/docs/reference/api/roles/#list-roles @@ -295,7 +301,7 @@ def parse_response( class UserRoleAssignments(OktaStream): - primary_key = "id" + primary_key = ["userId", "id"] use_cache = True def stream_slices(self, **kwargs): @@ -307,6 +313,10 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: user_id = stream_slice["user_id"] return f"users/{user_id}/roles" + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["userId"] = stream_slice["user_id"] + return record + class Permissions(OktaStream): # https://developer.okta.com/docs/reference/api/roles/#list-permissions diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/conftest.py b/airbyte-integrations/connectors/source-okta/unit_tests/conftest.py index 61bcae95b867..c20ab91a7b6c 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/conftest.py @@ -194,6 +194,7 @@ def group_members_instance(api_url): """ _id = "test_user_id" return { + "groupId": "test_group_id", "id": _id, "status": "ACTIVE", "created": "2021-04-21T21:04:03.000Z", @@ -226,6 +227,7 @@ def group_role_assignments_instance(): Group Role Assignment instance object response """ return { + "groupId": "test_group_id", "actor": { "id": "test_user_id", "type": "User", @@ -276,7 +278,8 @@ def user_role_assignments_instance(api_url): """ _user_id = "test_user_id" return { - "id": _user_id, + "userId": _user_id, + "id": "test_role_id", "label": "Super Organization Administrator", "type": "SUPER_ADMIN", "status": "ACTIVE", diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py index 35b8b8fd6375..3b8a09298211 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py @@ -67,7 +67,7 @@ def test_okta_stream_parse_response(self, patch_base_class, requests_mock, url_b stream = OktaStream(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[{"a": 123}, {"b": "xx"}]) resp = requests.get(f"{api_url}") - inputs = {"response": resp, "stream_state": MagicMock()} + inputs = {"response": resp, "stream_state": MagicMock(), "stream_slice": {}} expected_parsed_object = [{"a": 123}, {"b": "xx"}] assert list(stream.parse_response(**inputs)) == expected_parsed_object @@ -87,7 +87,7 @@ def test_incremental_okta_stream_parse_response(self, patch_base_class, requests stream = IncrementalOktaStream(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[{"a": 123}, {"b": "xx"}]) resp = requests.get(f"{api_url}") - inputs = {"response": resp, "stream_state": MagicMock()} + inputs = {"response": resp, "stream_state": MagicMock(), "stream_slice": {}} expected_parsed_object = [{"a": 123}, {"b": "xx"}] assert list(stream.parse_response(**inputs)) == expected_parsed_object @@ -199,7 +199,8 @@ def test_users_source_request_params_have_latest_entry(self, patch_base_class, u def test_users_source_parse_response(self, requests_mock, patch_base_class, users_instance, url_base, api_url, start_date): stream = Users(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[users_instance]) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [users_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {}} + assert list(stream.parse_response(**inputs)) == [users_instance] class TestStreamCustomRoles: @@ -214,7 +215,8 @@ def test_custom_roles_parse_response(self, requests_mock, patch_base_class, cust stream = CustomRoles(url_base=url_base, start_date=start_date) record = {"roles": [custom_role_instance]} requests_mock.get(f"{api_url}", json=record) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [custom_role_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {}} + assert list(stream.parse_response(**inputs)) == [custom_role_instance] class TestStreamPermissions: @@ -230,7 +232,8 @@ def test_permissions_parse_response(self, requests_mock, patch_base_class, permi stream = Permissions(url_base=url_base, start_date=start_date) record = {"permissions": [permission_instance]} requests_mock.get(f"{api_url}", json=record) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [permission_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {}} + assert list(stream.parse_response(**inputs)) == [permission_instance] class TestStreamGroups: @@ -243,7 +246,8 @@ def test_groups(self, requests_mock, patch_base_class, groups_instance, url_base def test_groups_parse_response(self, requests_mock, patch_base_class, groups_instance, url_base, api_url, start_date): stream = Groups(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[groups_instance]) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [groups_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {}} + assert list(stream.parse_response(**inputs)) == [groups_instance] class TestStreamGroupMembers: @@ -251,24 +255,23 @@ def test_group_members(self, requests_mock, patch_base_class, group_members_inst stream = GroupMembers(url_base=url_base, start_date=start_date) group_id = "test_group_id" requests_mock.get(f"{api_url}/groups/{group_id}/users?limit=200", json=[group_members_instance]) - inputs = {"sync_mode": SyncMode.incremental, "stream_state": {}, "stream_slice": {"group_id": group_id}} + inputs = {"sync_mode": SyncMode.full_refresh, "stream_state": {}, "stream_slice": {"group_id": group_id}} assert list(stream.read_records(**inputs)) == [group_members_instance] def test_group_members_parse_response(self, requests_mock, patch_base_class, group_members_instance, url_base, api_url, start_date): stream = GroupMembers(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[group_members_instance]) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [group_members_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {"group_id": "test_group_id"}} + assert list(stream.parse_response(**inputs)) == [group_members_instance] - def test_group_members_request_params_with_latest_entry(self, patch_base_class, group_members_instance, url_base, start_date): + def test_group_members_request_params_with_latest_entry(self, patch_base_class, group_members_instance, url_base, api_url, start_date): stream = GroupMembers(url_base=url_base, start_date=start_date) inputs = { "stream_slice": {"group_id": "some_group"}, "stream_state": {"id": "some_test_id"}, - "next_page_token": {"next_cursor": "123"}, } assert stream.request_params(**inputs) == { "limit": 200, - "next_cursor": "123", "after": "some_test_id", } @@ -279,13 +282,6 @@ def test_group_members_slice_stream( requests_mock.get(f"{api_url}/groups?limit=200", json=[groups_instance]) assert list(stream.stream_slices()) == [{"group_id": "test_group_id"}] - def test_group_member_request_get_update_state(self, latest_record_instance, url_base, start_date): - stream = GroupMembers(url_base=url_base, start_date=start_date) - stream._cursor_field = "id" - current_stream_state = {"id": "test_user_group_id"} - update_state = stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record_instance) - assert update_state == {"id": "test_user_group_id"} - class TestStreamGroupRoleAssignment: def test_group_role_assignments(self, requests_mock, patch_base_class, group_role_assignments_instance, url_base, api_url, start_date): @@ -301,7 +297,8 @@ def test_group_role_assignments_parse_response( ): stream = GroupRoleAssignments(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[group_role_assignments_instance]) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [group_role_assignments_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {"group_id": "test_group_id"}} + assert list(stream.parse_response(**inputs)) == [group_role_assignments_instance] def test_group_role_assignments_slice_stream( self, requests_mock, patch_base_class, group_members_instance, groups_instance, url_base, api_url, start_date @@ -321,7 +318,8 @@ def test_logs(self, requests_mock, patch_base_class, logs_instance, url_base, ap def test_logs_parse_response(self, requests_mock, patch_base_class, logs_instance, url_base, api_url, start_date): stream = Logs(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}/logs?limit=200", json=[logs_instance]) - assert list(stream.parse_response(response=requests.get(f"{api_url}/logs?limit=200"))) == [logs_instance] + inputs = {"response": requests.get(f"{api_url}/logs?limit=200"), "stream_state": MagicMock(), "stream_slice": {}} + assert list(stream.parse_response(**inputs)) == [logs_instance] def test_logs_request_params_for_since(self, patch_base_class, logs_instance, url_base, start_date): stream = Logs(url_base=url_base, start_date=start_date) @@ -352,7 +350,8 @@ def test_user_role_assignments_parse_response( ): stream = UserRoleAssignments(url_base=url_base, start_date=start_date) requests_mock.get(f"{api_url}", json=[user_role_assignments_instance]) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [user_role_assignments_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {"user_id": "test_user_id"}} + assert list(stream.parse_response(**inputs)) == [user_role_assignments_instance] def test_user_role_assignments_slice_stream( self, requests_mock, patch_base_class, group_members_instance, users_instance, url_base, api_url, start_date @@ -374,7 +373,8 @@ def test_resource_sets_parse_response(self, requests_mock, patch_base_class, res stream = ResourceSets(url_base=url_base, start_date=start_date) record = {"resource-sets": [resource_set_instance]} requests_mock.get(f"{api_url}", json=record) - assert list(stream.parse_response(response=requests.get(f"{api_url}"))) == [resource_set_instance] + inputs = {"response": requests.get(f"{api_url}"), "stream_state": MagicMock(), "stream_slice": {}} + assert list(stream.parse_response(**inputs)) == [resource_set_instance] def test_resource_sets_next_page_token(self, requests_mock, patch_base_class, resource_set_instance, url_base, api_url, start_date): stream = ResourceSets(url_base=url_base, start_date=start_date) @@ -394,6 +394,6 @@ def test_resource_sets_next_page_token(self, requests_mock, patch_base_class, re def test_resource_sets_request_params(self, requests_mock, patch_base_class, resource_set_instance, url_base, api_url, start_date): stream = ResourceSets(url_base=url_base, start_date=start_date) cursor = "iam5cursorFybecursor" - inputs = {"stream_slice": None, "stream_state": {"id": cursor}, "next_page_token": None} - expected_params = {"limit": 200, "after": "iam5cursorFybecursor", "filter": 'id gt "iam5cursorFybecursor"'} + inputs = {"stream_slice": None, "stream_state": {"id": cursor}, "next_page_token": {'after': cursor}} + expected_params = {"limit": 200, "after": "iam5cursorFybecursor"} assert stream.request_params(**inputs) == expected_params diff --git a/airbyte-integrations/connectors/source-omnisend/metadata.yaml b/airbyte-integrations/connectors/source-omnisend/metadata.yaml index dcac1ee12dca..1f350fea734d 100644 --- a/airbyte-integrations/connectors/source-omnisend/metadata.yaml +++ b/airbyte-integrations/connectors/source-omnisend/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-omnisend/requirements.txt b/airbyte-integrations/connectors/source-omnisend/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-omnisend/requirements.txt +++ b/airbyte-integrations/connectors/source-omnisend/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-omnisend/setup.py b/airbyte-integrations/connectors/source-omnisend/setup.py index ab4ab7c2878a..653728e512b5 100644 --- a/airbyte-integrations/connectors/source-omnisend/setup.py +++ b/airbyte-integrations/connectors/source-omnisend/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-onesignal/Dockerfile b/airbyte-integrations/connectors/source-onesignal/Dockerfile index ed6756ca667d..43a620a289db 100644 --- a/airbyte-integrations/connectors/source-onesignal/Dockerfile +++ b/airbyte-integrations/connectors/source-onesignal/Dockerfile @@ -34,5 +34,5 @@ COPY source_onesignal ./source_onesignal ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-onesignal diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json index 0c2a8beed641..b57dff4d6232 100644 --- a/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json @@ -2,15 +2,19 @@ { "type": "STREAM", "stream": { - "stream_state": { "8d466489-38af-4067-a6b8-2645ad83f4c2": { "last_active": 2598826148 }}, + "stream_state": { + "8d466489-38af-4067-a6b8-2645ad83f4c2": { "last_active": 2598826148 } + }, "stream_descriptor": { "name": "devices" } } }, { "type": "STREAM", "stream": { - "stream_state": { "8d466489-38af-4067-a6b8-2645ad83f4c2": { "last_active": 2598826148 }}, + "stream_state": { + "8d466489-38af-4067-a6b8-2645ad83f4c2": { "last_active": 2598826148 } + }, "stream_descriptor": { "name": "notifications" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-onesignal/integration_tests/sample_state.json index 74b34a5ec922..5a2ae9f6d585 100644 --- a/airbyte-integrations/connectors/source-onesignal/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/sample_state.json @@ -2,14 +2,14 @@ { "type": "STREAM", "stream": { - "stream_state": { "fake_app_id": { "last_active": 1598826147 }}, + "stream_state": { "fake_app_id": { "last_active": 1598826147 } }, "stream_descriptor": { "name": "devices" } } }, { "type": "STREAM", "stream": { - "stream_state": { "fake_app_id": { "last_active": 1598826147 }}, + "stream_state": { "fake_app_id": { "last_active": 1598826147 } }, "stream_descriptor": { "name": "notifications" } } } diff --git a/airbyte-integrations/connectors/source-onesignal/metadata.yaml b/airbyte-integrations/connectors/source-onesignal/metadata.yaml index 9cd34bca03b4..26a4f8e5051d 100644 --- a/airbyte-integrations/connectors/source-onesignal/metadata.yaml +++ b/airbyte-integrations/connectors/source-onesignal/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: bb6afd81-87d5-47e3-97c4-e2c2901b1cf8 - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.1 dockerRepository: airbyte/source-onesignal + documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal githubIssueLabel: source-onesignal icon: onesignal.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-onesignal/requirements.txt b/airbyte-integrations/connectors/source-onesignal/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-onesignal/requirements.txt +++ b/airbyte-integrations/connectors/source-onesignal/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-onesignal/setup.py b/airbyte-integrations/connectors/source-onesignal/setup.py index 84746caaf488..f0f2b7829a99 100644 --- a/airbyte-integrations/connectors/source-onesignal/setup.py +++ b/airbyte-integrations/connectors/source-onesignal/setup.py @@ -12,7 +12,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json index f637529e83e7..8d66d7546299 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json @@ -1,6 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -81,6 +82,21 @@ }, "additional_data_is_root_payload": { "type": ["null", "boolean"] + }, + "apns_key_id": { + "type": ["null", "string"] + }, + "apns_p8": { + "type": ["null", "string"] + }, + "apns_team_id": { + "type": ["null", "string"] + }, + "fcm_v1_service_account_json": { + "type": ["null", "string"] + }, + "apns_bundle_id": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/devices.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/devices.json index 7de3a1664c47..fb99daae9012 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/devices.json +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/devices.json @@ -1,6 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string" diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/notifications.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/notifications.json index 55dbb822b4b1..a33cec41b845 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/notifications.json +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/notifications.json @@ -1,6 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "adm_big_picture": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/outcomes.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/outcomes.json index 54ca46fef0d4..e5cde1806a1c 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/outcomes.json +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/outcomes.json @@ -1,6 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string" diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json index 266f48499472..1cf4e4f21bac 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json @@ -4,7 +4,12 @@ "$schema": "https://json-schema.org/draft-07/schema#", "title": "OneSignal Source Spec", "type": "object", - "required": ["user_auth_key", "start_date", "outcome_names", "applications"], + "required": [ + "user_auth_key", + "start_date", + "outcome_names", + "applications" + ], "additionalProperties": true, "properties": { "user_auth_key": { diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json index 2bc897cc6993..5bc2570a65a6 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json @@ -16,1043 +16,526 @@ "type": "object", "properties": { "AED": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AFN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ALL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AMD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ANG": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AOA": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ARS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AUD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AWG": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AZN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BAM": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BBD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BDT": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BGN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BHD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BIF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BMD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BND": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BOB": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BRL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BSD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BTC": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BTN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BWP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BYN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BZD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CAD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CDF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CHF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CLF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CLP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CNH": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CNY": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "COP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CRC": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CUC": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CUP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CVE": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CZK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DJF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DKK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DOP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DZD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "EGP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ERN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ETB": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "EUR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "FJD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "FKP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GBP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GEL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GGP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GHS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GIP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GMD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GNF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GTQ": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "GYD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HKD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HNL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HRK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HTG": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HUF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "IDR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ILS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "IMP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "INR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "IQD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "IRR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ISK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "JEP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "JMD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "JOD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "JPY": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KES": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KGS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KHR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KMF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KPW": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KRW": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KWD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KYD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "KZT": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LAK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LBP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LKR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LRD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LSL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LYD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MAD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MDL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MGA": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MKD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MMK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MNT": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MOP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MRO": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MRU": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MUR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MVR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MWK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MXN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MYR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MZN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "NAD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "NGN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "NIO": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "NOK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "NPR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "NZD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "OMR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PAB": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PEN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PGK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PHP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PKR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PLN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PYG": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "QAR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "RON": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "RSD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "RUB": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "RWF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SAR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SBD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SCR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SDG": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SEK": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SGD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SHP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SLL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SOS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SRD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SSP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "STD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "STN": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SVC": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SYP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SZL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "THB": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TJS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TMT": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TND": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TOP": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TRY": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TTD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TWD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TZS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "UAH": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "UGX": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "USD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "UYU": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "UZS": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "VES": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "VND": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "VUV": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "WST": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XAF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XAG": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XAU": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XCD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XDR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XOF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XPD": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XPF": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "XPT": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "YER": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ZAR": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ZMW": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ZWL": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] } } } } }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "timestamp" - ] + "default_cursor_field": ["timestamp"] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": [ - "timestamp" - ] + "cursor_field": ["timestamp"] } ] } diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml index 24bf190e81cc..adc7e74a580d 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/open-exchange-rates tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt b/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt +++ b/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py index e24f64e3fdbf..8e52d07a0702 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py @@ -11,9 +11,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json index de969d6465a7..92c17f7e56aa 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json @@ -1,1040 +1,527 @@ { - "type": "object", - "required": [ - "base", - "rates" - ], - "properties": { - "base": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "rates": { - "type": "object", - "properties": { - "AED": { - "type": [ - "null", - "number" - ] - }, - "AFN": { - "type": [ - "null", - "number" - ] - }, - "ALL": { - "type": [ - "null", - "number" - ] - }, - "AMD": { - "type": [ - "null", - "number" - ] - }, - "ANG": { - "type": [ - "null", - "number" - ] - }, - "AOA": { - "type": [ - "null", - "number" - ] - }, - "ARS": { - "type": [ - "null", - "number" - ] - }, - "AUD": { - "type": [ - "null", - "number" - ] - }, - "AWG": { - "type": [ - "null", - "number" - ] - }, - "AZN": { - "type": [ - "null", - "number" - ] - }, - "BAM": { - "type": [ - "null", - "number" - ] - }, - "BBD": { - "type": [ - "null", - "number" - ] - }, - "BDT": { - "type": [ - "null", - "number" - ] - }, - "BGN": { - "type": [ - "null", - "number" - ] - }, - "BHD": { - "type": [ - "null", - "number" - ] - }, - "BIF": { - "type": [ - "null", - "number" - ] - }, - "BMD": { - "type": [ - "null", - "number" - ] - }, - "BND": { - "type": [ - "null", - "number" - ] - }, - "BOB": { - "type": [ - "null", - "number" - ] - }, - "BRL": { - "type": [ - "null", - "number" - ] - }, - "BSD": { - "type": [ - "null", - "number" - ] - }, - "BTC": { - "type": [ - "null", - "number" - ] - }, - "BTN": { - "type": [ - "null", - "number" - ] - }, - "BWP": { - "type": [ - "null", - "number" - ] - }, - "BYN": { - "type": [ - "null", - "number" - ] - }, - "BZD": { - "type": [ - "null", - "number" - ] - }, - "CAD": { - "type": [ - "null", - "number" - ] - }, - "CDF": { - "type": [ - "null", - "number" - ] - }, - "CHF": { - "type": [ - "null", - "number" - ] - }, - "CLF": { - "type": [ - "null", - "number" - ] - }, - "CLP": { - "type": [ - "null", - "number" - ] - }, - "CNH": { - "type": [ - "null", - "number" - ] - }, - "CNY": { - "type": [ - "null", - "number" - ] - }, - "COP": { - "type": [ - "null", - "number" - ] - }, - "CRC": { - "type": [ - "null", - "number" - ] - }, - "CUC": { - "type": [ - "null", - "number" - ] - }, - "CUP": { - "type": [ - "null", - "number" - ] - }, - "CVE": { - "type": [ - "null", - "number" - ] - }, - "CZK": { - "type": [ - "null", - "number" - ] - }, - "DJF": { - "type": [ - "null", - "number" - ] - }, - "DKK": { - "type": [ - "null", - "number" - ] - }, - "DOP": { - "type": [ - "null", - "number" - ] - }, - "DZD": { - "type": [ - "null", - "number" - ] - }, - "EGP": { - "type": [ - "null", - "number" - ] - }, - "ERN": { - "type": [ - "null", - "number" - ] - }, - "ETB": { - "type": [ - "null", - "number" - ] - }, - "EUR": { - "type": [ - "null", - "number" - ] - }, - "FJD": { - "type": [ - "null", - "number" - ] - }, - "FKP": { - "type": [ - "null", - "number" - ] - }, - "GBP": { - "type": [ - "null", - "number" - ] - }, - "GEL": { - "type": [ - "null", - "number" - ] - }, - "GGP": { - "type": [ - "null", - "number" - ] - }, - "GHS": { - "type": [ - "null", - "number" - ] - }, - "GIP": { - "type": [ - "null", - "number" - ] - }, - "GMD": { - "type": [ - "null", - "number" - ] - }, - "GNF": { - "type": [ - "null", - "number" - ] - }, - "GTQ": { - "type": [ - "null", - "number" - ] - }, - "GYD": { - "type": [ - "null", - "number" - ] - }, - "HKD": { - "type": [ - "null", - "number" - ] - }, - "HNL": { - "type": [ - "null", - "number" - ] - }, - "HRK": { - "type": [ - "null", - "number" - ] - }, - "HTG": { - "type": [ - "null", - "number" - ] - }, - "HUF": { - "type": [ - "null", - "number" - ] - }, - "IDR": { - "type": [ - "null", - "number" - ] - }, - "ILS": { - "type": [ - "null", - "number" - ] - }, - "IMP": { - "type": [ - "null", - "number" - ] - }, - "INR": { - "type": [ - "null", - "number" - ] - }, - "IQD": { - "type": [ - "null", - "number" - ] - }, - "IRR": { - "type": [ - "null", - "number" - ] - }, - "ISK": { - "type": [ - "null", - "number" - ] - }, - "JEP": { - "type": [ - "null", - "number" - ] - }, - "JMD": { - "type": [ - "null", - "number" - ] - }, - "JOD": { - "type": [ - "null", - "number" - ] - }, - "JPY": { - "type": [ - "null", - "number" - ] - }, - "KES": { - "type": [ - "null", - "number" - ] - }, - "KGS": { - "type": [ - "null", - "number" - ] - }, - "KHR": { - "type": [ - "null", - "number" - ] - }, - "KMF": { - "type": [ - "null", - "number" - ] - }, - "KPW": { - "type": [ - "null", - "number" - ] - }, - "KRW": { - "type": [ - "null", - "number" - ] - }, - "KWD": { - "type": [ - "null", - "number" - ] - }, - "KYD": { - "type": [ - "null", - "number" - ] - }, - "KZT": { - "type": [ - "null", - "number" - ] - }, - "LAK": { - "type": [ - "null", - "number" - ] - }, - "LBP": { - "type": [ - "null", - "number" - ] - }, - "LKR": { - "type": [ - "null", - "number" - ] - }, - "LRD": { - "type": [ - "null", - "number" - ] - }, - "LSL": { - "type": [ - "null", - "number" - ] - }, - "LYD": { - "type": [ - "null", - "number" - ] - }, - "MAD": { - "type": [ - "null", - "number" - ] - }, - "MDL": { - "type": [ - "null", - "number" - ] - }, - "MGA": { - "type": [ - "null", - "number" - ] - }, - "MKD": { - "type": [ - "null", - "number" - ] - }, - "MMK": { - "type": [ - "null", - "number" - ] - }, - "MNT": { - "type": [ - "null", - "number" - ] - }, - "MOP": { - "type": [ - "null", - "number" - ] - }, - "MRO": { - "type": [ - "null", - "number" - ] - }, - "MRU": { - "type": [ - "null", - "number" - ] - }, - "MUR": { - "type": [ - "null", - "number" - ] - }, - "MVR": { - "type": [ - "null", - "number" - ] - }, - "MWK": { - "type": [ - "null", - "number" - ] - }, - "MXN": { - "type": [ - "null", - "number" - ] - }, - "MYR": { - "type": [ - "null", - "number" - ] - }, - "MZN": { - "type": [ - "null", - "number" - ] - }, - "NAD": { - "type": [ - "null", - "number" - ] - }, - "NGN": { - "type": [ - "null", - "number" - ] - }, - "NIO": { - "type": [ - "null", - "number" - ] - }, - "NOK": { - "type": [ - "null", - "number" - ] - }, - "NPR": { - "type": [ - "null", - "number" - ] - }, - "NZD": { - "type": [ - "null", - "number" - ] - }, - "OMR": { - "type": [ - "null", - "number" - ] - }, - "PAB": { - "type": [ - "null", - "number" - ] - }, - "PEN": { - "type": [ - "null", - "number" - ] - }, - "PGK": { - "type": [ - "null", - "number" - ] - }, - "PHP": { - "type": [ - "null", - "number" - ] - }, - "PKR": { - "type": [ - "null", - "number" - ] - }, - "PLN": { - "type": [ - "null", - "number" - ] - }, - "PYG": { - "type": [ - "null", - "number" - ] - }, - "QAR": { - "type": [ - "null", - "number" - ] - }, - "RON": { - "type": [ - "null", - "number" - ] - }, - "RSD": { - "type": [ - "null", - "number" - ] - }, - "RUB": { - "type": [ - "null", - "number" - ] - }, - "RWF": { - "type": [ - "null", - "number" - ] - }, - "SAR": { - "type": [ - "null", - "number" - ] - }, - "SBD": { - "type": [ - "null", - "number" - ] - }, - "SCR": { - "type": [ - "null", - "number" - ] - }, - "SDG": { - "type": [ - "null", - "number" - ] - }, - "SEK": { - "type": [ - "null", - "number" - ] - }, - "SGD": { - "type": [ - "null", - "number" - ] - }, - "SHP": { - "type": [ - "null", - "number" - ] - }, - "SLL": { - "type": [ - "null", - "number" - ] - }, - "SOS": { - "type": [ - "null", - "number" - ] - }, - "SRD": { - "type": [ - "null", - "number" - ] - }, - "SSP": { - "type": [ - "null", - "number" - ] - }, - "STD": { - "type": [ - "null", - "number" - ] - }, - "STN": { - "type": [ - "null", - "number" - ] - }, - "SVC": { - "type": [ - "null", - "number" - ] - }, - "SYP": { - "type": [ - "null", - "number" - ] - }, - "SZL": { - "type": [ - "null", - "number" - ] - }, - "THB": { - "type": [ - "null", - "number" - ] - }, - "TJS": { - "type": [ - "null", - "number" - ] - }, - "TMT": { - "type": [ - "null", - "number" - ] - }, - "TND": { - "type": [ - "null", - "number" - ] - }, - "TOP": { - "type": [ - "null", - "number" - ] - }, - "TRY": { - "type": [ - "null", - "number" - ] - }, - "TTD": { - "type": [ - "null", - "number" - ] - }, - "TWD": { - "type": [ - "null", - "number" - ] - }, - "TZS": { - "type": [ - "null", - "number" - ] - }, - "UAH": { - "type": [ - "null", - "number" - ] - }, - "UGX": { - "type": [ - "null", - "number" - ] - }, - "USD": { - "type": [ - "null", - "number" - ] - }, - "UYU": { - "type": [ - "null", - "number" - ] - }, - "UZS": { - "type": [ - "null", - "number" - ] - }, - "VES": { - "type": [ - "null", - "number" - ] - }, - "VND": { - "type": [ - "null", - "number" - ] - }, - "VUV": { - "type": [ - "null", - "number" - ] - }, - "WST": { - "type": [ - "null", - "number" - ] - }, - "XAF": { - "type": [ - "null", - "number" - ] - }, - "XAG": { - "type": [ - "null", - "number" - ] - }, - "XAU": { - "type": [ - "null", - "number" - ] - }, - "XCD": { - "type": [ - "null", - "number" - ] - }, - "XDR": { - "type": [ - "null", - "number" - ] - }, - "XOF": { - "type": [ - "null", - "number" - ] - }, - "XPD": { - "type": [ - "null", - "number" - ] - }, - "XPF": { - "type": [ - "null", - "number" - ] - }, - "XPT": { - "type": [ - "null", - "number" - ] - }, - "YER": { - "type": [ - "null", - "number" - ] - }, - "ZAR": { - "type": [ - "null", - "number" - ] - }, - "ZMW": { - "type": [ - "null", - "number" - ] - }, - "ZWL": { - "type": [ - "null", - "number" - ] - } - } + "type": "object", + "required": ["base", "rates"], + "properties": { + "base": { + "type": "string" + }, + "timestamp": { + "type": "integer" + }, + "rates": { + "type": "object", + "properties": { + "AED": { + "type": ["null", "number"] + }, + "AFN": { + "type": ["null", "number"] + }, + "ALL": { + "type": ["null", "number"] + }, + "AMD": { + "type": ["null", "number"] + }, + "ANG": { + "type": ["null", "number"] + }, + "AOA": { + "type": ["null", "number"] + }, + "ARS": { + "type": ["null", "number"] + }, + "AUD": { + "type": ["null", "number"] + }, + "AWG": { + "type": ["null", "number"] + }, + "AZN": { + "type": ["null", "number"] + }, + "BAM": { + "type": ["null", "number"] + }, + "BBD": { + "type": ["null", "number"] + }, + "BDT": { + "type": ["null", "number"] + }, + "BGN": { + "type": ["null", "number"] + }, + "BHD": { + "type": ["null", "number"] + }, + "BIF": { + "type": ["null", "number"] + }, + "BMD": { + "type": ["null", "number"] + }, + "BND": { + "type": ["null", "number"] + }, + "BOB": { + "type": ["null", "number"] + }, + "BRL": { + "type": ["null", "number"] + }, + "BSD": { + "type": ["null", "number"] + }, + "BTC": { + "type": ["null", "number"] + }, + "BTN": { + "type": ["null", "number"] + }, + "BWP": { + "type": ["null", "number"] + }, + "BYN": { + "type": ["null", "number"] + }, + "BZD": { + "type": ["null", "number"] + }, + "CAD": { + "type": ["null", "number"] + }, + "CDF": { + "type": ["null", "number"] + }, + "CHF": { + "type": ["null", "number"] + }, + "CLF": { + "type": ["null", "number"] + }, + "CLP": { + "type": ["null", "number"] + }, + "CNH": { + "type": ["null", "number"] + }, + "CNY": { + "type": ["null", "number"] + }, + "COP": { + "type": ["null", "number"] + }, + "CRC": { + "type": ["null", "number"] + }, + "CUC": { + "type": ["null", "number"] + }, + "CUP": { + "type": ["null", "number"] + }, + "CVE": { + "type": ["null", "number"] + }, + "CZK": { + "type": ["null", "number"] + }, + "DJF": { + "type": ["null", "number"] + }, + "DKK": { + "type": ["null", "number"] + }, + "DOP": { + "type": ["null", "number"] + }, + "DZD": { + "type": ["null", "number"] + }, + "EGP": { + "type": ["null", "number"] + }, + "ERN": { + "type": ["null", "number"] + }, + "ETB": { + "type": ["null", "number"] + }, + "EUR": { + "type": ["null", "number"] + }, + "FJD": { + "type": ["null", "number"] + }, + "FKP": { + "type": ["null", "number"] + }, + "GBP": { + "type": ["null", "number"] + }, + "GEL": { + "type": ["null", "number"] + }, + "GGP": { + "type": ["null", "number"] + }, + "GHS": { + "type": ["null", "number"] + }, + "GIP": { + "type": ["null", "number"] + }, + "GMD": { + "type": ["null", "number"] + }, + "GNF": { + "type": ["null", "number"] + }, + "GTQ": { + "type": ["null", "number"] + }, + "GYD": { + "type": ["null", "number"] + }, + "HKD": { + "type": ["null", "number"] + }, + "HNL": { + "type": ["null", "number"] + }, + "HRK": { + "type": ["null", "number"] + }, + "HTG": { + "type": ["null", "number"] + }, + "HUF": { + "type": ["null", "number"] + }, + "IDR": { + "type": ["null", "number"] + }, + "ILS": { + "type": ["null", "number"] + }, + "IMP": { + "type": ["null", "number"] + }, + "INR": { + "type": ["null", "number"] + }, + "IQD": { + "type": ["null", "number"] + }, + "IRR": { + "type": ["null", "number"] + }, + "ISK": { + "type": ["null", "number"] + }, + "JEP": { + "type": ["null", "number"] + }, + "JMD": { + "type": ["null", "number"] + }, + "JOD": { + "type": ["null", "number"] + }, + "JPY": { + "type": ["null", "number"] + }, + "KES": { + "type": ["null", "number"] + }, + "KGS": { + "type": ["null", "number"] + }, + "KHR": { + "type": ["null", "number"] + }, + "KMF": { + "type": ["null", "number"] + }, + "KPW": { + "type": ["null", "number"] + }, + "KRW": { + "type": ["null", "number"] + }, + "KWD": { + "type": ["null", "number"] + }, + "KYD": { + "type": ["null", "number"] + }, + "KZT": { + "type": ["null", "number"] + }, + "LAK": { + "type": ["null", "number"] + }, + "LBP": { + "type": ["null", "number"] + }, + "LKR": { + "type": ["null", "number"] + }, + "LRD": { + "type": ["null", "number"] + }, + "LSL": { + "type": ["null", "number"] + }, + "LYD": { + "type": ["null", "number"] + }, + "MAD": { + "type": ["null", "number"] + }, + "MDL": { + "type": ["null", "number"] + }, + "MGA": { + "type": ["null", "number"] + }, + "MKD": { + "type": ["null", "number"] + }, + "MMK": { + "type": ["null", "number"] + }, + "MNT": { + "type": ["null", "number"] + }, + "MOP": { + "type": ["null", "number"] + }, + "MRO": { + "type": ["null", "number"] + }, + "MRU": { + "type": ["null", "number"] + }, + "MUR": { + "type": ["null", "number"] + }, + "MVR": { + "type": ["null", "number"] + }, + "MWK": { + "type": ["null", "number"] + }, + "MXN": { + "type": ["null", "number"] + }, + "MYR": { + "type": ["null", "number"] + }, + "MZN": { + "type": ["null", "number"] + }, + "NAD": { + "type": ["null", "number"] + }, + "NGN": { + "type": ["null", "number"] + }, + "NIO": { + "type": ["null", "number"] + }, + "NOK": { + "type": ["null", "number"] + }, + "NPR": { + "type": ["null", "number"] + }, + "NZD": { + "type": ["null", "number"] + }, + "OMR": { + "type": ["null", "number"] + }, + "PAB": { + "type": ["null", "number"] + }, + "PEN": { + "type": ["null", "number"] + }, + "PGK": { + "type": ["null", "number"] + }, + "PHP": { + "type": ["null", "number"] + }, + "PKR": { + "type": ["null", "number"] + }, + "PLN": { + "type": ["null", "number"] + }, + "PYG": { + "type": ["null", "number"] + }, + "QAR": { + "type": ["null", "number"] + }, + "RON": { + "type": ["null", "number"] + }, + "RSD": { + "type": ["null", "number"] + }, + "RUB": { + "type": ["null", "number"] + }, + "RWF": { + "type": ["null", "number"] + }, + "SAR": { + "type": ["null", "number"] + }, + "SBD": { + "type": ["null", "number"] + }, + "SCR": { + "type": ["null", "number"] + }, + "SDG": { + "type": ["null", "number"] + }, + "SEK": { + "type": ["null", "number"] + }, + "SGD": { + "type": ["null", "number"] + }, + "SHP": { + "type": ["null", "number"] + }, + "SLL": { + "type": ["null", "number"] + }, + "SOS": { + "type": ["null", "number"] + }, + "SRD": { + "type": ["null", "number"] + }, + "SSP": { + "type": ["null", "number"] + }, + "STD": { + "type": ["null", "number"] + }, + "STN": { + "type": ["null", "number"] + }, + "SVC": { + "type": ["null", "number"] + }, + "SYP": { + "type": ["null", "number"] + }, + "SZL": { + "type": ["null", "number"] + }, + "THB": { + "type": ["null", "number"] + }, + "TJS": { + "type": ["null", "number"] + }, + "TMT": { + "type": ["null", "number"] + }, + "TND": { + "type": ["null", "number"] + }, + "TOP": { + "type": ["null", "number"] + }, + "TRY": { + "type": ["null", "number"] + }, + "TTD": { + "type": ["null", "number"] + }, + "TWD": { + "type": ["null", "number"] + }, + "TZS": { + "type": ["null", "number"] + }, + "UAH": { + "type": ["null", "number"] + }, + "UGX": { + "type": ["null", "number"] + }, + "USD": { + "type": ["null", "number"] + }, + "UYU": { + "type": ["null", "number"] + }, + "UZS": { + "type": ["null", "number"] + }, + "VES": { + "type": ["null", "number"] + }, + "VND": { + "type": ["null", "number"] + }, + "VUV": { + "type": ["null", "number"] + }, + "WST": { + "type": ["null", "number"] + }, + "XAF": { + "type": ["null", "number"] + }, + "XAG": { + "type": ["null", "number"] + }, + "XAU": { + "type": ["null", "number"] + }, + "XCD": { + "type": ["null", "number"] + }, + "XDR": { + "type": ["null", "number"] + }, + "XOF": { + "type": ["null", "number"] + }, + "XPD": { + "type": ["null", "number"] + }, + "XPF": { + "type": ["null", "number"] + }, + "XPT": { + "type": ["null", "number"] + }, + "YER": { + "type": ["null", "number"] + }, + "ZAR": { + "type": ["null", "number"] + }, + "ZMW": { + "type": ["null", "number"] + }, + "ZWL": { + "type": ["null", "number"] } + } } + } } diff --git a/airbyte-integrations/connectors/source-openweather/metadata.yaml b/airbyte-integrations/connectors/source-openweather/metadata.yaml index 78af8ceca107..7276d353f847 100644 --- a/airbyte-integrations/connectors/source-openweather/metadata.yaml +++ b/airbyte-integrations/connectors/source-openweather/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/openweather tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-openweather/requirements.txt b/airbyte-integrations/connectors/source-openweather/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-openweather/requirements.txt +++ b/airbyte-integrations/connectors/source-openweather/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-openweather/setup.py b/airbyte-integrations/connectors/source-openweather/setup.py index b0c755b06b82..16f473b254ab 100644 --- a/airbyte-integrations/connectors/source-openweather/setup.py +++ b/airbyte-integrations/connectors/source-openweather/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-opsgenie/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-opsgenie/integration_tests/abnormal_state.json index 40eec45c4700..8376d13cbce1 100644 --- a/airbyte-integrations/connectors/source-opsgenie/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-opsgenie/integration_tests/abnormal_state.json @@ -1,11 +1,11 @@ { "alerts": { - "updatedAt":"3022-01-01T00:00:00.000000+00:00" + "updatedAt": "3022-01-01T00:00:00.000000+00:00" }, "alert_recipients": { - "updatedAt":"3022-01-01T00:00:00.000000+00:00" + "updatedAt": "3022-01-01T00:00:00.000000+00:00" }, "incidents": { - "updatedAt":"3022-01-01T00:00:00.000000+00:00" + "updatedAt": "3022-01-01T00:00:00.000000+00:00" } } diff --git a/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py index 950b53b59d41..37dfff891923 100644 --- a/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-opsgenie/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-opsgenie/integration_tests/sample_state.json index 017598c67900..16e8bc137492 100644 --- a/airbyte-integrations/connectors/source-opsgenie/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-opsgenie/integration_tests/sample_state.json @@ -1,11 +1,11 @@ { "alerts": { - "updatedAt":"2022-09-13T07:06:51.871000+00:00" + "updatedAt": "2022-09-13T07:06:51.871000+00:00" }, "alert_recipients": { - "updatedAt":"2022-01-12T08:12:33.243000+00:00" + "updatedAt": "2022-01-12T08:12:33.243000+00:00" }, "incidents": { - "updatedAt":"2022-01-14T10:18:34.954000+00:00" + "updatedAt": "2022-01-14T10:18:34.954000+00:00" } } diff --git a/airbyte-integrations/connectors/source-opsgenie/main.py b/airbyte-integrations/connectors/source-opsgenie/main.py index 229a88efe135..0acd6dee136d 100644 --- a/airbyte-integrations/connectors/source-opsgenie/main.py +++ b/airbyte-integrations/connectors/source-opsgenie/main.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml index abc4bb11a678..44359e08afed 100644 --- a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml @@ -16,4 +16,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/opsgenie tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-opsgenie/setup.py b/airbyte-integrations/connectors/source-opsgenie/setup.py index 739dc16aa419..2b029a855e75 100644 --- a/airbyte-integrations/connectors/source-opsgenie/setup.py +++ b/airbyte-integrations/connectors/source-opsgenie/setup.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # @@ -10,6 +10,7 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/alerts.json b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/alerts.json index a82b2788dbb6..af49055e4573 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/alerts.json +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/alerts.json @@ -148,7 +148,7 @@ }, "details": { "type": ["null", "object"], - "properties": { }, + "properties": {}, "additionalProperties": true }, "ownerTeamId": { diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/incidents.json b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/incidents.json index 7567dd0c5ba8..31aa413bd7b2 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/incidents.json +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/incidents.json @@ -62,7 +62,7 @@ }, "details": { "type": ["null", "object"], - "properties": { }, + "properties": {}, "additionalProperties": true }, "notifyStakeholders": { diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py index 39417113602b..743694d15b54 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py index 3b8b09261b04..6812e8427474 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import time diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py index 03055c0056e3..32b939c7a213 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from airbyte_cdk.models import SyncMode diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py index 3b8989004f29..dfb27745a5c7 100644 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import unittest diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py index 8b132004c5bf..32003b46a58c 100644 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import unittest diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py index a331cab6199c..9042065c8925 100644 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import unittest diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index 110680e5a3d3..9c613f848705 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -25,5 +25,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.24 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml index dca3a866d2bb..1903adbfff84 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml @@ -11,11 +11,11 @@ data: connectorSubtype: database connectorType: source definitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 - dockerImageTag: 0.3.24 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-oracle-strict-encrypt githubIssueLabel: source-oracle icon: oracle.svg - license: MIT + license: ELv2 name: Oracle DB releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/oracle diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json index 61f6a599e4c9..a187609590a5 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test/resources/expected_spec.json @@ -140,4 +140,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 368c5b62f716..ef85f0ec58d7 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -16,5 +16,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.24 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index 9bf6b68b10b8..5ffae2d188c4 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -36,3 +36,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-oracle/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-oracle/integration_tests/seed/basic.sql index 0e91378d7952..bb5bc32e35f0 100644 --- a/airbyte-integrations/connectors/source-oracle/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-oracle/integration_tests/seed/basic.sql @@ -1,9 +1,124 @@ -CREATE USER ORACLE_BASIC IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS; +CREATE + USER ORACLE_BASIC IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON + USERS; -CREATE TABLE TEST.TEST_DATASET(ID INTEGER PRIMARY KEY, TEST_COLUMN_1 CHAR(3 CHAR),TEST_COLUMN_11 DATE,TEST_COLUMN_12 TIMESTAMP,TEST_COLUMN_13 TIMESTAMP WITH TIME ZONE,TEST_COLUMN_14 TIMESTAMP WITH LOCAL TIME ZONE,TEST_COLUMN_2 VARCHAR2(256),TEST_COLUMN_3 VARCHAR2(256),TEST_COLUMN_4 NVARCHAR2(3),TEST_COLUMN_5 NUMBER,TEST_COLUMN_6 NUMBER(6,-2),TEST_COLUMN_7 FLOAT(5),TEST_COLUMN_8 FLOAT ); +CREATE + TABLE + TEST.TEST_DATASET( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 CHAR( + 3 CHAR + ), + TEST_COLUMN_11 DATE, + TEST_COLUMN_12 TIMESTAMP, + TEST_COLUMN_13 TIMESTAMP WITH TIME ZONE, + TEST_COLUMN_14 TIMESTAMP WITH LOCAL TIME ZONE, + TEST_COLUMN_2 VARCHAR2(256), + TEST_COLUMN_3 VARCHAR2(256), + TEST_COLUMN_4 NVARCHAR2(3), + TEST_COLUMN_5 NUMBER, + TEST_COLUMN_6 NUMBER( + 6, + - 2 + ), + TEST_COLUMN_7 FLOAT(5), + TEST_COLUMN_8 FLOAT + ); -INSERT INTO TEST.TEST_DATASET VALUES (1, 'a', to_date('-4700/01/01','syyyy/mm/dd'), to_timestamp('2020-06-10 06:14:00.742', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00 EST', 'DD-MON-YYYY HH24:MI:SS TZR'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), 'тест', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 1, 123.89, 1.34, 126.45); -INSERT INTO TEST.TEST_DATASET VALUES (2, 'ab', to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 EST', 'DD-MON-YYYY HH24:MI:SS.FF TZR'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), '⚡ test ��', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 123.45, 123.89, 126.45, 126); -INSERT INTO TEST.TEST_DATASET VALUES (3, 'abc', to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), q'[{|}!"#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', power(10, -130), 123.89, 126.45, 126); -INSERT INTO TEST.TEST_DATASET VALUES (4, 'abc', to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), q'[{|}!"#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 9.99999999999999999999 * power(10, 125), 123.89, 126.45, 126); -INSERT INTO TEST.TEST_DATASET VALUES (5, 'abc', to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), q'[{|}!"#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 9.99999999999999999999 * power(10, 125), 123.89, 126.45, 126); +INSERT + INTO + TEST.TEST_DATASET + VALUES( + 1, + 'a', + to_date( + '-4700/01/01', + 'syyyy/mm/dd' + ), + to_timestamp( + '2020-06-10 06:14:00.742', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00 EST', + 'DD-MON-YYYY HH24:MI:SS TZR' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + 'тест', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + N'テスト', + 1, + 123.89, + 1.34, + 126.45 + ); + +INSERT + INTO + TEST.TEST_DATASET + VALUES( + 2, + 'ab', + to_date( + '9999/12/31 23:59:59', + 'yyyy/mm/dd hh24:mi:ss' + ), + to_timestamp( + '2020-06-10 06:14:00.742123', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.123456 EST', + 'DD-MON-YYYY HH24:MI:SS.FF TZR' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + '⚡ test ��', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + N'テスト', + 123.45, + 123.89, + 126.45, + 126 + ); + +INSERT + INTO + TEST.TEST_DATASET + VALUES( + 3, + 'abc', + to_date( + '9999/12/31 23:59:59', + 'yyyy/mm/dd hh24:mi:ss' + ), + to_timestamp( + '2020-06-10 06:14:00.742123', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00 -5:00', + 'DD-MON-YYYY HH24:MI:SS TZH:TZM' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + q'[{|}!"#$%&'()*+, + -./:; + +<=>? @ [] ^_ `~] ', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N' テスト', power(10, -130), 123.89, 126.45, 126); +INSERT INTO TEST.TEST_DATASET VALUES (4, ' abc', to_date(' 9999 / 12 / 31 23:59:59 ',' yyyy / mm / dd hh24:mi:ss'), to_timestamp(' 2020 - 06 - 10 06:14:00.742123 ', ' YYYY - MM - DD HH24:MI:SS.FF'), to_timestamp_tz(' 21 - FEB - 2009 18:00:00.123456 - 5:00 ', ' DD - MON - YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz(' 21 - FEB - 2009 18:00:00.000456 ', ' DD - MON - YYYY HH24:MI:SS.FF'), q' [ { | } ! "#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 9.99999999999999999999 * power(10, 125), 123.89, 126.45, 126); +INSERT INTO TEST.TEST_DATASET VALUES (5, 'abc', to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), q'[{|}!" #$ %& '()*+,-./:;<=>?@[]^_`~]', +chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), +N'テスト', +9.99999999999999999999 * POWER( 10, 125 ), +123.89, +126.45, +126 + ); diff --git a/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full.sql index 4135cba2fa21..91e0cb907922 100644 --- a/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full.sql +++ b/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full.sql @@ -1,13 +1,194 @@ -CREATE USER ORACLE_FULL IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS; +CREATE + USER ORACLE_FULL IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON + USERS; -CREATE TABLE ORACLE_FULL.TEST_DATASET(ID INTEGER PRIMARY KEY, TEST_COLUMN_1 CHAR(3 CHAR),TEST_COLUMN_10 BINARY_DOUBLE,TEST_COLUMN_11 DATE,TEST_COLUMN_12 TIMESTAMP,TEST_COLUMN_13 TIMESTAMP WITH TIME ZONE,TEST_COLUMN_14 TIMESTAMP WITH LOCAL TIME ZONE,TEST_COLUMN_15 INTERVAL YEAR TO MONTH,TEST_COLUMN_16 BLOB,TEST_COLUMN_17 CLOB,TEST_COLUMN_18 RAW(200),TEST_COLUMN_19 XMLTYPE,TEST_COLUMN_2 VARCHAR2(256),TEST_COLUMN_3 VARCHAR2(256),TEST_COLUMN_4 NVARCHAR2(3),TEST_COLUMN_5 NUMBER,TEST_COLUMN_6 NUMBER(6,-2),TEST_COLUMN_7 FLOAT(5),TEST_COLUMN_8 FLOAT,TEST_COLUMN_9 BINARY_FLOAT ); +CREATE + TABLE + ORACLE_FULL.TEST_DATASET( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 CHAR( + 3 CHAR + ), + TEST_COLUMN_10 BINARY_DOUBLE, + TEST_COLUMN_11 DATE, + TEST_COLUMN_12 TIMESTAMP, + TEST_COLUMN_13 TIMESTAMP WITH TIME ZONE, + TEST_COLUMN_14 TIMESTAMP WITH LOCAL TIME ZONE, + TEST_COLUMN_15 INTERVAL YEAR TO MONTH, + TEST_COLUMN_16 BLOB, + TEST_COLUMN_17 CLOB, + TEST_COLUMN_18 RAW(200), + TEST_COLUMN_19 XMLTYPE, + TEST_COLUMN_2 VARCHAR2(256), + TEST_COLUMN_3 VARCHAR2(256), + TEST_COLUMN_4 NVARCHAR2(3), + TEST_COLUMN_5 NUMBER, + TEST_COLUMN_6 NUMBER( + 6, + - 2 + ), + TEST_COLUMN_7 FLOAT(5), + TEST_COLUMN_8 FLOAT, + TEST_COLUMN_9 BINARY_FLOAT + ); -INSERT INTO ORACLE_FULL.TEST_DATASET VALUES (1, null, 126.45d, to_date('-4700/01/01','syyyy/mm/dd'), to_timestamp('2020-06-10 06:14:00.742', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00 EST', 'DD-MON-YYYY HH24:MI:SS TZR'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '10-2' YEAR TO MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' +INSERT + INTO + ORACLE_FULL.TEST_DATASET + VALUES( + 1, + NULL, + 126.45 d, + to_date( + '-4700/01/01', + 'syyyy/mm/dd' + ), + to_timestamp( + '2020-06-10 06:14:00.742', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00 EST', + 'DD-MON-YYYY HH24:MI:SS TZR' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + INTERVAL '10-2' YEAR TO MONTH, + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + xmltype(' 1 2 -'), null, chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), null, null, 123.89, 1.34, 126.45, 126.45f); -INSERT INTO ORACLE_FULL.TEST_DATASET VALUES (2, 'a', 2.22507485850720E-308, to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 EST', 'DD-MON-YYYY HH24:MI:SS.FF TZR'), null, INTERVAL '9' MONTH, null, null, null, null, 'тест', null, N'テスト', 1, null, 126.45, 126, 1.17549E-38f); -INSERT INTO ORACLE_FULL.TEST_DATASET VALUES (3, 'ab', 1.79769313486231E+308d, null, null, to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM'), null, null, null, null, null, null, '⚡ test ��', null, null, 123.45, null, null, null, 3.40282347E+038f); -INSERT INTO ORACLE_FULL.TEST_DATASET VALUES (4, 'abc', BINARY_DOUBLE_INFINITY, null, null, to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), null, null, null, null, null, null, q'[{|}!"#$%&''()*+,-./:;<=>?@[]^_`~]', null, null, power(10, -130), null, null, null, BINARY_FLOAT_INFINITY); -INSERT INTO ORACLE_FULL.TEST_DATASET VALUES (5, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 9.99999999999999999999 * power(10, 125), null, null, null, null); +'), + NULL, + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + NULL, + NULL, + 123.89, + 1.34, + 126.45, + 126.45 f + ); + +INSERT + INTO + ORACLE_FULL.TEST_DATASET + VALUES( + 2, + 'a', + 2.22507485850720E - 308, + to_date( + '9999/12/31 23:59:59', + 'yyyy/mm/dd hh24:mi:ss' + ), + to_timestamp( + '2020-06-10 06:14:00.742123', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.123456 EST', + 'DD-MON-YYYY HH24:MI:SS.FF TZR' + ), + NULL, + INTERVAL '9' MONTH, + NULL, + NULL, + NULL, + NULL, + 'тест', + NULL, + N'テスト', + 1, + NULL, + 126.45, + 126, + 1.17549E - 38 f + ); + +INSERT + INTO + ORACLE_FULL.TEST_DATASET + VALUES( + 3, + 'ab', + 1.79769313486231E + 308 d, + NULL, + NULL, + to_timestamp_tz( + '21-FEB-2009 18:00:00 -5:00', + 'DD-MON-YYYY HH24:MI:SS TZH:TZM' + ), + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '⚡ test ��', + NULL, + NULL, + 123.45, + NULL, + NULL, + NULL, + 3.40282347E + 038 f + ); + +INSERT + INTO + ORACLE_FULL.TEST_DATASET + VALUES( + 4, + 'abc', + BINARY_DOUBLE_INFINITY, + NULL, + NULL, + to_timestamp_tz( + '21-FEB-2009 18:00:00.123456 -5:00', + 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM' + ), + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + q'[{|}!"#$%&''()*+,-./:;<=>?@[]^_`~]', + NULL, + NULL, + POWER( 10,- 130 ), + NULL, + NULL, + NULL, + BINARY_FLOAT_INFINITY + ); + +INSERT + INTO + ORACLE_FULL.TEST_DATASET + VALUES( + 5, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 9.99999999999999999999 * POWER( 10, 125 ), + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full_without_nulls.sql index 3335a6a2c75b..2f13344d04d9 100644 --- a/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full_without_nulls.sql +++ b/airbyte-integrations/connectors/source-oracle/integration_tests/seed/full_without_nulls.sql @@ -1,29 +1,168 @@ -CREATE USER ORACLE_FULL_NN IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS; +CREATE + USER ORACLE_FULL_NN IDENTIFIED BY TEST DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON + USERS; -CREATE TABLE ORACLE_FULL_NN.TEST_DATASET(ID INTEGER PRIMARY KEY, TEST_COLUMN_1 CHAR(3 CHAR),TEST_COLUMN_10 BINARY_DOUBLE,TEST_COLUMN_11 DATE,TEST_COLUMN_12 TIMESTAMP,TEST_COLUMN_13 TIMESTAMP WITH TIME ZONE,TEST_COLUMN_14 TIMESTAMP WITH LOCAL TIME ZONE,TEST_COLUMN_15 INTERVAL YEAR TO MONTH,TEST_COLUMN_16 BLOB,TEST_COLUMN_17 CLOB,TEST_COLUMN_18 RAW(200),TEST_COLUMN_19 XMLTYPE,TEST_COLUMN_2 VARCHAR2(256),TEST_COLUMN_3 VARCHAR2(256),TEST_COLUMN_4 NVARCHAR2(3),TEST_COLUMN_5 NUMBER,TEST_COLUMN_6 NUMBER(6,-2),TEST_COLUMN_7 FLOAT(5),TEST_COLUMN_8 FLOAT,TEST_COLUMN_9 BINARY_FLOAT ); +CREATE + TABLE + ORACLE_FULL_NN.TEST_DATASET( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 CHAR( + 3 CHAR + ), + TEST_COLUMN_10 BINARY_DOUBLE, + TEST_COLUMN_11 DATE, + TEST_COLUMN_12 TIMESTAMP, + TEST_COLUMN_13 TIMESTAMP WITH TIME ZONE, + TEST_COLUMN_14 TIMESTAMP WITH LOCAL TIME ZONE, + TEST_COLUMN_15 INTERVAL YEAR TO MONTH, + TEST_COLUMN_16 BLOB, + TEST_COLUMN_17 CLOB, + TEST_COLUMN_18 RAW(200), + TEST_COLUMN_19 XMLTYPE, + TEST_COLUMN_2 VARCHAR2(256), + TEST_COLUMN_3 VARCHAR2(256), + TEST_COLUMN_4 NVARCHAR2(3), + TEST_COLUMN_5 NUMBER, + TEST_COLUMN_6 NUMBER( + 6, + - 2 + ), + TEST_COLUMN_7 FLOAT(5), + TEST_COLUMN_8 FLOAT, + TEST_COLUMN_9 BINARY_FLOAT + ); -INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (1, 'a', 126.45d, to_date('-4700/01/01','syyyy/mm/dd'), to_timestamp('2020-06-10 06:14:00.742', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00 EST', 'DD-MON-YYYY HH24:MI:SS TZR'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '10-2' YEAR TO MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' +INSERT + INTO + ORACLE_FULL_NN.TEST_DATASET + VALUES( + 1, + 'a', + 126.45 d, + to_date( + '-4700/01/01', + 'syyyy/mm/dd' + ), + to_timestamp( + '2020-06-10 06:14:00.742', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00 EST', + 'DD-MON-YYYY HH24:MI:SS TZR' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + INTERVAL '10-2' YEAR TO MONTH, + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + xmltype(' 1 2 -'), 'тест', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 1, 123.89, 1.34, 126.45, 126.45f); -INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (2, 'ab', 2.22507485850720E-308, to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 EST', 'DD-MON-YYYY HH24:MI:SS.FF TZR'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '9' MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' - -1 -2 -'), '⚡ test ��', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 123.45, 123.89, 126.45, 126, 1.17549E-38f); -INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (3, 'abc', 1.79769313486231E+308d, to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00 -5:00', 'DD-MON-YYYY HH24:MI:SS TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '9' MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' +'), + 'тест', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + N'テスト', + 1, + 123.89, + 1.34, + 126.45, + 126.45 f + ); + +INSERT + INTO + ORACLE_FULL_NN.TEST_DATASET + VALUES( + 2, + 'ab', + 2.22507485850720E - 308, + to_date( + '9999/12/31 23:59:59', + 'yyyy/mm/dd hh24:mi:ss' + ), + to_timestamp( + '2020-06-10 06:14:00.742123', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.123456 EST', + 'DD-MON-YYYY HH24:MI:SS.FF TZR' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + INTERVAL '9' MONTH, + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + xmltype(' 1 2 -'), q'[{|}!"#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', power(10, -130), 123.89, 126.45, 126, 3.40282347E+038f); -INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (4, 'abc', BINARY_DOUBLE_INFINITY, to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '9' MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' +'), + '⚡ test ��', + chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), + N'テスト', + 123.45, + 123.89, + 126.45, + 126, + 1.17549E - 38 f + ); + +INSERT + INTO + ORACLE_FULL_NN.TEST_DATASET + VALUES( + 3, + 'abc', + 1.79769313486231E + 308 d, + to_date( + '9999/12/31 23:59:59', + 'yyyy/mm/dd hh24:mi:ss' + ), + to_timestamp( + '2020-06-10 06:14:00.742123', + 'YYYY-MM-DD HH24:MI:SS.FF' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00 -5:00', + 'DD-MON-YYYY HH24:MI:SS TZH:TZM' + ), + to_timestamp_tz( + '21-FEB-2009 18:00:00.000456', + 'DD-MON-YYYY HH24:MI:SS.FF' + ), + INTERVAL '9' MONTH, + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + utl_raw.cast_to_raw('some content here'), + xmltype(' 1 2 -'), q'[{|}!"#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 9.99999999999999999999 * power(10, 125), 123.89, 126.45, 126, BINARY_FLOAT_INFINITY); -INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (5, 'abc', BINARY_DOUBLE_INFINITY, to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '9' MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' +'), + q'[{|}!"#$%&'()*+, + -./:; + +<=>? @ [] ^_ `~] ', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N' テスト', power(10, -130), 123.89, 126.45, 126, 3.40282347E+038f); +INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (4, ' abc', BINARY_DOUBLE_INFINITY, to_date(' 9999 / 12 / 31 23:59:59 ',' yyyy / mm / dd hh24:mi:ss'), to_timestamp(' 2020 - 06 - 10 06:14:00.742123 ', ' YYYY - MM - DD HH24:MI:SS.FF'), to_timestamp_tz(' 21 - FEB - 2009 18:00:00.123456 - 5:00 ', ' DD - MON - YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz(' 21 - FEB - 2009 18:00:00.000456 ', ' DD - MON - YYYY HH24:MI:SS.FF'), INTERVAL ' 9 ' MONTH, utl_raw.cast_to_raw(' SOME content here'), utl_raw.cast_to_raw(' SOME content here'), utl_raw.cast_to_raw(' SOME content here'), xmltype(' 1 2 '), q' [ { | } ! "#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 9.99999999999999999999 * power(10, 125), 123.89, 126.45, 126, BINARY_FLOAT_INFINITY); +INSERT INTO ORACLE_FULL_NN.TEST_DATASET VALUES (5, 'abc', BINARY_DOUBLE_INFINITY, to_date('9999/12/31 23:59:59','yyyy/mm/dd hh24:mi:ss'), to_timestamp('2020-06-10 06:14:00.742123', 'YYYY-MM-DD HH24:MI:SS.FF'), to_timestamp_tz('21-FEB-2009 18:00:00.123456 -5:00', 'DD-MON-YYYY HH24:MI:SS.FF TZH:TZM'), to_timestamp_tz('21-FEB-2009 18:00:00.000456', 'DD-MON-YYYY HH24:MI:SS.FF'), INTERVAL '9' MONTH, utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), utl_raw.cast_to_raw('some content here'), xmltype(' 1 2 -'), q'[{|}!"#$%&'()*+,-./:;<=>?@[]^_`~]', chr(33) || chr(34) || chr(35) || chr(36) || chr(37) || chr(38) || chr(39) || chr(40) || chr(41), N'テスト', 9.99999999999999999999 * power(10, 125), 123.89, 126.45, 126, BINARY_FLOAT_INFINITY); +'), q'[{|}!" #$ %& '()*+,-./:;<=>?@[]^_`~]', +chr(33)|| chr(34)|| chr(35)|| chr(36)|| chr(37)|| chr(38)|| chr(39)|| chr(40)|| chr(41), +N'テスト', +9.99999999999999999999 * POWER( 10, 125 ), +123.89, +126.45, +126, +BINARY_FLOAT_INFINITY + ); diff --git a/airbyte-integrations/connectors/source-oracle/metadata.yaml b/airbyte-integrations/connectors/source-oracle/metadata.yaml index 70cfb0f522b9..c12f2a06ba72 100644 --- a/airbyte-integrations/connectors/source-oracle/metadata.yaml +++ b/airbyte-integrations/connectors/source-oracle/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 200 + sl: 100 allowedHosts: hosts: - ${host} @@ -6,11 +9,12 @@ data: connectorSubtype: database connectorType: source definitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 - dockerImageTag: 0.3.24 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-oracle + documentationUrl: https://docs.airbyte.com/integrations/sources/oracle githubIssueLabel: source-oracle icon: oracle.svg - license: MIT + license: ELv2 name: Oracle DB registries: cloud: @@ -20,7 +24,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/oracle + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-orb/metadata.yaml b/airbyte-integrations/connectors/source-orb/metadata.yaml index 8e7eec065a3b..16fbdbb5ab0a 100644 --- a/airbyte-integrations/connectors/source-orb/metadata.yaml +++ b/airbyte-integrations/connectors/source-orb/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/orb tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-orb/requirements.txt b/airbyte-integrations/connectors/source-orb/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-orb/requirements.txt +++ b/airbyte-integrations/connectors/source-orb/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-orb/setup.py b/airbyte-integrations/connectors/source-orb/setup.py index 056a2089c34e..38e64fa19ee1 100644 --- a/airbyte-integrations/connectors/source-orb/setup.py +++ b/airbyte-integrations/connectors/source-orb/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "pendulum==2.1.2"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "responses~=0.13.3", "pendulum==2.1.2"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.13.3", "pendulum==2.1.2"] setup( name="source_orb", diff --git a/airbyte-integrations/connectors/source-orb/source_orb/schemas/invoices.json b/airbyte-integrations/connectors/source-orb/source_orb/schemas/invoices.json index 0417de31822d..1f00340a3e67 100644 --- a/airbyte-integrations/connectors/source-orb/source_orb/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-orb/source_orb/schemas/invoices.json @@ -1,108 +1,116 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": ["null", "object"], - "properties": { - "id": { - "type": "string" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "invoice_date": { - "type": ["string"], - "format": "date-time" - }, - "due_date": { - "type": ["string"], - "format": "date-time" - }, - "invoice_pdf": { - "type": ["null", "string"] - }, - "subtotal": { - "type": ["string"] - }, - "total": { - "type": ["string"] - }, - "amount_due": { - "type": ["string"] - }, - "status": { - "type": ["string"] - }, - "memo": { - "type": ["null", "string"] - }, - "issue_failed_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "sync_failed_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "payment_failed_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "payment_started_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "voided_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "paid_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "issued_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "hosted_invoice_url": { - "type": ["null", "string"] - }, - "line_items": { - "type": ["array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "quantity": { - "type": "number" - }, - "amount": { - "type": "string" - }, - "name": { - "type": "string" - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - }, - "end_date": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - }, - "subscription": { - "type": ["object", "null"], + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "id": { + "type": "string" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "invoice_date": { + "type": ["string"], + "format": "date-time" + }, + "due_date": { + "type": ["string"], + "format": "date-time" + }, + "invoice_pdf": { + "type": ["null", "string"] + }, + "subtotal": { + "type": ["string"] + }, + "total": { + "type": ["string"] + }, + "amount_due": { + "type": ["string"] + }, + "status": { + "type": ["string"] + }, + "memo": { + "type": ["null", "string"] + }, + "issue_failed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "sync_failed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "payment_failed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "payment_started_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "voided_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "paid_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "issued_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hosted_invoice_url": { + "type": ["null", "string"] + }, + "line_items": { + "type": ["array"], + "items": { + "type": "object", "properties": { "id": { "type": "string" + }, + "quantity": { + "type": "number" + }, + "amount": { + "type": "string" + }, + "name": { + "type": "string" + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "end_date": { + "type": ["null", "string"], + "format": "date-time" } } } }, - "required": ["id", "created_at", "invoice_date", "due_date", "subtotal", "total", "amount_due", "status"] - } - \ No newline at end of file + "subscription": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "string" + } + } + } + }, + "required": [ + "id", + "created_at", + "invoice_date", + "due_date", + "subtotal", + "total", + "amount_due", + "status" + ] +} diff --git a/airbyte-integrations/connectors/source-orbit/metadata.yaml b/airbyte-integrations/connectors/source-orbit/metadata.yaml index 65bab83e63bc..60494a01a6e3 100644 --- a/airbyte-integrations/connectors/source-orbit/metadata.yaml +++ b/airbyte-integrations/connectors/source-orbit/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/orbit tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-orbit/requirements.txt b/airbyte-integrations/connectors/source-orbit/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-orbit/requirements.txt +++ b/airbyte-integrations/connectors/source-orbit/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-orbit/setup.py b/airbyte-integrations/connectors/source-orbit/setup.py index 63ced0f2d6d6..c16dff7df89a 100644 --- a/airbyte-integrations/connectors/source-orbit/setup.py +++ b/airbyte-integrations/connectors/source-orbit/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-oura/metadata.yaml b/airbyte-integrations/connectors/source-oura/metadata.yaml index 5b805fb7c102..01992bf8de55 100644 --- a/airbyte-integrations/connectors/source-oura/metadata.yaml +++ b/airbyte-integrations/connectors/source-oura/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-oura/requirements.txt b/airbyte-integrations/connectors/source-oura/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-oura/requirements.txt +++ b/airbyte-integrations/connectors/source-oura/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-oura/setup.py b/airbyte-integrations/connectors/source-oura/setup.py index e83a35a84faf..b78b256c87fb 100644 --- a/airbyte-integrations/connectors/source-oura/setup.py +++ b/airbyte-integrations/connectors/source-oura/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/.dockerignore b/airbyte-integrations/connectors/source-outbrain-amplify/.dockerignore new file mode 100644 index 000000000000..d5c4f64017de --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_outbrain_amplify +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/Dockerfile b/airbyte-integrations/connectors/source-outbrain-amplify/Dockerfile new file mode 100644 index 000000000000..ed3c74805a95 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_outbrain_amplify ./source_outbrain_amplify + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.name=airbyte/source-outbrain-amplify diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/README.md b/airbyte-integrations/connectors/source-outbrain-amplify/README.md new file mode 100644 index 000000000000..dafe820e594b --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/README.md @@ -0,0 +1,138 @@ +# Outbrain Amplify Source + +This is the repository for the Outbrain Amplify source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/outbrain-amplify). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/outbrain-amplify) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_outbrain_amplify/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source outbrain-amplify test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-outbrain-amplify:dev +``` + +If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: +```bash +CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +``` + + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-outbrain-amplify:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outbrain-amplify:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outbrain-amplify:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-outbrain-amplify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml new file mode 100644 index 000000000000..5411d3cede72 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml @@ -0,0 +1,72 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-outbrain-amplify:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_outbrain_amplify/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 2400 + empty_streams: + - name: campaigns + bypass_reason: "no records" + - name: campaigns_geo_location + bypass_reason: "no records" + - name: budgets + bypass_reason: "no records" + - name: promoted_links + bypass_reason: "no records" + - name: promoted_links_sequence_for_campaigns + bypass_reason: "no records" + - name: performance_report_campaigns_by_marketers + bypass_reason: "no records" + - name: performance_report_periodic_by_marketers + bypass_reason: "no records" + - name: performance_report_periodic_by_marketers_campaign + bypass_reason: "no records" + - name: performance_promoted_links + bypass_reason: "no records" + - name: performance_report_marketers_by_publisher + bypass_reason: "no records" + - name: performance_report_publishers_by_campaigns + bypass_reason: "no records" + - name: performance_report_marketers_by_platforms + bypass_reason: "no records" + - name: performance_report_marketers_campaigns_by_platforms + bypass_reason: "no records" + - name: performance_report_marketers_by_geo_performance + bypass_reason: "no records" + - name: performance_report_marketers_campaigns_by_geo + bypass_reason: "no records" + - name: performance_report_marketers_by_interest + bypass_reason: "no records" +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes +# incremental: +# bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/bootstrap.md b/airbyte-integrations/connectors/source-outbrain-amplify/bootstrap.md new file mode 100644 index 000000000000..7bb8b128a213 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/bootstrap.md @@ -0,0 +1,24 @@ +The (Outbrain Amplify Source is [a REST based API](https://www.outbrain.com//). +Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). + +## Outbrain-Amplify api stream +Outbrain Amplify is a content discovery and advertising platform that helps businesses and publishers promote their content to a wider audience. Customers can use Outbrain Amplify to promote their content across a range of premium publishers, including some of the biggest names in media. They can create custom campaigns, set specific targeting criteria, and monitor the performance of their campaigns in real-time. The platform also offers a range of tools and features to help customers optimize their campaigns and improve their ROI. +Offers a powerful way for businesses and publishers to reach new audiences and drive more traffic to their content. With its advanced targeting capabilities and robust reporting tools, the platform can help customers achieve their marketing goals and grow their businesses. +## Endpoints +* marketers stream --> Non-Non-Incremental +* campaigns by marketers stream. --> Non-Non-Incremental +* campaigns geo location stream. --> Non-Incremental +* promoted links for campaigns stream. --> Non-Incremental +* promoted links sequence for campaigns stream. --> Non-Incremental +* budgets for marketers stream. --> Non-Incremental +* performance report campaigns by marketers stream. --> Non-Incremental +* performance report periodic by marketers stream. --> Non-Incremental +* performance report periodic by marketers campaign stream. --> Non-Incremental +* performance report periodic content by promoted links campaign stream. --> Non-Incremental +* performance report marketers by publisher stream. --> Non-Incremental +* performance report publishers by campaigns stream. --> Non-Incremental +* performance report marketers by platforms stream. --> Non-Incremental +* performance report marketers campaigns by platforms stream. --> Non-Incremental +* performance report marketers by geo performance stream. --> Non-Incremental +* performance report marketers campaigns by geo stream. --> Non-Incremental +* performance report marketers by Interest stream. --> Non-Incremental \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/build.gradle b/airbyte-integrations/connectors/source-outbrain-amplify/build.gradle new file mode 100644 index 000000000000..51126f788062 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_outbrain_amplify' +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/icon.svg b/airbyte-integrations/connectors/source-outbrain-amplify/icon.svg new file mode 100644 index 000000000000..4232f45ecba9 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/__init__.py b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..0cf2e27733c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/abnormal_state.json @@ -0,0 +1,14 @@ +{ + "marketers": { + "id": "$#@!1" + }, + "campaigns": { + "id": "$#@!1" + }, + "promoted_links": { + "id": "$#@!1" + }, + "budgets": { + "id": "$#@!1" + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..e26c5c2902f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/configured_catalog.json @@ -0,0 +1,438 @@ +{ + "streams": [ + { + "stream": { + "name": "marketers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "campaigns_geo_location", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_campaigns_by_marketers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_periodic_by_marketers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_periodic_by_marketers_campaign", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_promoted_links", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "fromDate": { + "type": ["string", "null"] + }, + "toDate": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["integer", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + }, + "campaign_id": { + "type": ["string", "null"] + }, + "promoted_link_id": { + "type": ["string", "null"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_marketers_by_publisher", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_publishers_by_campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_marketers_by_platforms", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_marketers_campaigns_by_platforms", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_marketers_by_geo_performance", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_marketers_campaigns_by_geo", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "performance_report_marketers_by_interest", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "campaigns", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "cpc": { + "type": ["number", "null"] + }, + "autoArchived": { + "type": ["boolean", "null"] + }, + "minimumCpc": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "targeting": { + "type": "object", + "properties": { + "platform": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "language": { + "type": ["string", "null"] + }, + "excludeAdBlockUsers": { + "type": ["boolean", "null"] + }, + "nativePlacements": { + "type": "object", + "properties": { + "enabled": { + "type": ["boolean", "null"] + } + } + }, + "includeCellularNetwork": { + "type": ["boolean", "null"] + }, + "nativePlacementsEnabled": { + "type": ["boolean", "null"] + }, + "locationsVersion": { + "type": ["string", "null"] + } + } + }, + "marketerId": { + "type": ["string", "null"] + }, + "autoExpirationOfAds": { + "type": ["integer", "null"] + }, + "contentType": { + "type": ["string", "null"] + }, + "budget": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "shared": { + "type": ["boolean", "null"] + }, + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "startDate": { + "type": ["string", "null"] + }, + "endDate": { + "type": ["string", "null"] + }, + "runForever": { + "type": ["boolean", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "pacing": { + "type": ["string", "null"] + } + } + }, + "prefixTrackingCode": { + "type": "object", + "properties": { + "prefix": { + "type": ["string", "null"] + }, + "encode": { + "type": ["boolean", "null"] + } + } + }, + "liveStatus": { + "type": "object", + "properties": { + "onAirReason": { + "type": ["string", "null"] + }, + "campaignOnAir": { + "type": ["boolean", "null"] + }, + "amountSpent": { + "type": ["number", "null"] + }, + "onAirModificationTime": { + "type": ["string", "null"] + } + } + }, + "readonly": { + "type": ["boolean", "null"] + }, + "startHour": { + "type": ["string", "null"] + }, + "trackingPixels": { + "type": "object", + "properties": { + "enabled": { + "type": ["boolean", "null"] + }, + "urls": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "pixels": { + "type": "object", + "properties": { + "impressionPixels": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "trackingPixels": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "onAirType": { + "type": ["string", "null"] + }, + "objective": { + "type": ["string", "null"] + }, + "creativeFormat": { + "type": ["string", "null"] + }, + "dynamicRetargeting": { + "type": ["boolean", "null"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + } + ] +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/invalid_config.json new file mode 100644 index 000000000000..046e083fbb99 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/invalid_config.json @@ -0,0 +1,10 @@ +{ + "credentials": { + "type": "access_token", + "access_token": "12345" + }, + "report_granularity": "daily", + "geo_location_breakdown": "country", + "start_date": "20220429", + "end_date": "20220430" +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/sample_config.json new file mode 100644 index 000000000000..fdb5be756f12 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/sample_config.json @@ -0,0 +1,10 @@ +{ + "credentials": { + "type": "access_token", + "access_token": "123" + }, + "report_granularity": "daily", + "geo_location_breakdown": "country", + "start_date": "2022-04-29", + "end_date": "2022-04-30" +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/sample_state.json new file mode 100644 index 000000000000..e154dd208426 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/integration_tests/sample_state.json @@ -0,0 +1,27 @@ +{ + "marketers": { + "id": "$#@!1" + }, + "campaigns": { + "id": "$#@!1" + }, + "campaigns_geo_location": {}, + "promoted_links": { + "id": "$#@!1" + }, + "promoted_links_sequence_for_campaigns": {}, + "budgets": { + "id": "$#@!1" + }, + "performance_report_campaigns_by_marketers": {}, + "performance_report_periodic_by_marketers": {}, + "performance_report_periodic_by_marketers_campaign": {}, + "performance_promoted_links": {}, + "performance_report_marketers_by_publisher": {}, + "performance_report_publishers_by_campaigns": {}, + "performance_report_marketers_by_platforms": {}, + "performance_report_marketers_campaigns_by_platforms": {}, + "performance_report_marketers_by_geo_performance": {}, + "performance_report_marketers_campaigns_by_geo": {}, + "performance_report_marketers_by_interest": {} +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/main.py b/airbyte-integrations/connectors/source-outbrain-amplify/main.py new file mode 100644 index 000000000000..2f2acbc0b627 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_outbrain_amplify import SourceOutbrainAmplify + +if __name__ == "__main__": + source = SourceOutbrainAmplify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml new file mode 100644 index 000000000000..0e7bd169237f --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml @@ -0,0 +1,25 @@ +data: + registries: + cloud: + enabled: true + oss: + enabled: true + connectorSubtype: api + connectorType: source + definitionId: 4fe962d0-a70e-4516-aa99-c551abf46352 + dockerImageTag: 0.1.2 + dockerRepository: airbyte/source-outbrain-amplify + githubIssueLabel: source-outbrain-amplify + icon: icon.svg + license: MIT + name: Outbrain Amplify + releaseStage: alpha + supportUrl: https://docs.airbyte.com/integrations/sources/outbrain-amplify + documentationUrl: https://docs.airbyte.com/integrations/sources/outbrain-amplify + tags: + - language:low-code + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/requirements.txt b/airbyte-integrations/connectors/source-outbrain-amplify/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/setup.py b/airbyte-integrations/connectors/source-outbrain-amplify/setup.py new file mode 100644 index 000000000000..d8b5563f3dce --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_outbrain_amplify", + description="Source implementation for Outbrain Amplify.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/__init__.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/__init__.py new file mode 100644 index 000000000000..bea9bf3d4430 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceOutbrainAmplify + +__all__ = ["SourceOutbrainAmplify"] diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py new file mode 100644 index 000000000000..18248b027d46 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/auth.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping + +import requests +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from requests.auth import HTTPBasicAuth + + +class OutbrainAmplifyAuthenticator(TokenAuthenticator): + def __init__(self, config, url_base): + self.config = config + self.url_auth = url_base + "login" + self.token = "" + + def generate_cache_token( + self, + ): + r = requests.get( + self.url_auth, + auth=HTTPBasicAuth(self.config.get("credentials").get("username"), self.config.get("credentials").get("password")), + ) + if r.status_code == 200: + self.token = r.json().get("OB-TOKEN-V1") + else: + raise ConnectionError(r.json().get("message")) + + def get_auth_header(self) -> Mapping[dict, Any]: + if self.config.get("credentials").get("type") == "access_token": + self.token = self.config.get("credentials").get("access_token") + return {"OB-TOKEN-V1": "{}".format(self.token)} + else: + if self.token: + return {"OB-TOKEN-V1": "{}".format(self.token)} + else: + self.generate_cache_token() + return {"OB-TOKEN-V1": "{}".format(self.token)} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/budgets.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/budgets.json new file mode 100644 index 000000000000..ff1c155147f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/budgets.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "shared": { + "type": ["boolean", "null"] + }, + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "amountRemaining": { + "type": ["number", "null"] + }, + "amountSpent": { + "type": ["number", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "startDate": { + "type": ["string", "null"] + }, + "endDate": { + "type": ["string", "null"] + }, + "runForever": { + "type": ["boolean", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "pacing": { + "type": ["string", "null"] + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/campaigns.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/campaigns.json new file mode 100644 index 000000000000..3b1221895230 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/campaigns.json @@ -0,0 +1,195 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "cpc": { + "type": ["number", "null"] + }, + "autoArchived": { + "type": ["boolean", "null"] + }, + "minimumCpc": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "targeting": { + "type": "object", + "properties": { + "platform": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "language": { + "type": ["string", "null"] + }, + "excludeAdBlockUsers": { + "type": ["boolean", "null"] + }, + "nativePlacements": { + "type": "object", + "properties": { + "enabled": { + "type": ["boolean", "null"] + } + } + }, + "includeCellularNetwork": { + "type": ["boolean", "null"] + }, + "nativePlacementsEnabled": { + "type": ["boolean", "null"] + }, + "locationsVersion": { + "type": ["string", "null"] + } + } + }, + "marketerId": { + "type": ["string", "null"] + }, + "autoExpirationOfAds": { + "type": ["integer", "null"] + }, + "contentType": { + "type": ["string", "null"] + }, + "budget": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "shared": { + "type": ["boolean", "null"] + }, + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "startDate": { + "type": ["string", "null"] + }, + "endDate": { + "type": ["string", "null"] + }, + "runForever": { + "type": ["boolean", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "pacing": { + "type": ["string", "null"] + } + } + }, + "prefixTrackingCode": { + "type": "object", + "properties": { + "prefix": { + "type": ["string", "null"] + }, + "encode": { + "type": ["boolean", "null"] + } + } + }, + "liveStatus": { + "type": "object", + "properties": { + "onAirReason": { + "type": ["string", "null"] + }, + "campaignOnAir": { + "type": ["boolean", "null"] + }, + "amountSpent": { + "type": ["number", "null"] + }, + "onAirModificationTime": { + "type": ["string", "null"] + } + } + }, + "readonly": { + "type": ["boolean", "null"] + }, + "startHour": { + "type": ["string", "null"] + }, + "trackingPixels": { + "type": "object", + "properties": { + "enabled": { + "type": ["boolean", "null"] + }, + "urls": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "pixels": { + "type": "object", + "properties": { + "impressionPixels": { + "type": "array", + "items": { + "type": ["string", "null"] + } + }, + "trackingPixels": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } + }, + "onAirType": { + "type": ["string", "null"] + }, + "objective": { + "type": ["string", "null"] + }, + "creativeFormat": { + "type": ["string", "null"] + }, + "dynamicRetargeting": { + "type": ["boolean", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/campaigns_geo_location.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/campaigns_geo_location.json new file mode 100644 index 000000000000..9aee2f5e4686 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/campaigns_geo_location.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "geoType": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "canonicalName": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + }, + "parent": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "geoType": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "canonicalName": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + } + } + }, + "campaign_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/marketers.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/marketers.json new file mode 100644 index 000000000000..4f2b9666a2ac --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/marketers.json @@ -0,0 +1,101 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "campaignDefaults": { + "type": "object", + "properties": { + "language": { + "type": ["string", "null"] + } + } + }, + "blockedSites": { + "type": "object", + "properties": { + "blockedPublishers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "modifiedBy": { + "type": ["string", "null"] + } + } + } + }, + "blockedSections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "publisher": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + } + }, + "creationTime": { + "type": ["string", "null"] + }, + "modifiedBy": { + "type": ["string", "null"] + } + } + } + } + } + }, + "useFirstPartyCookie": { + "type": ["boolean", "null"] + }, + "role": { + "type": ["string", "null"] + }, + "permissions": { + "type": "array", + "items": { + "type": ["string", "null"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_promoted_links.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_promoted_links.json new file mode 100644 index 000000000000..0efc9c209c25 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_promoted_links.json @@ -0,0 +1,116 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "fromDate": { + "type": ["string", "null"] + }, + "toDate": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["integer", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + }, + "campaign_id": { + "type": ["string", "null"] + }, + "promoted_link_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_campaigns_by_marketers.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_campaigns_by_marketers.json new file mode 100644 index 000000000000..7e948ef3c447 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_campaigns_by_marketers.json @@ -0,0 +1,169 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "campaignOnAir": { + "type": ["boolean", "null"] + }, + "onAirReason": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "budget": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "shared": { + "type": ["boolean", "null"] + }, + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + }, + "startDate": { + "type": ["string", "null"] + }, + "endDate": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "pacing": { + "type": ["string", "null"] + } + } + }, + "isVideo": { + "type": ["boolean", "null"] + }, + "cpc": { + "type": ["number", "null"] + }, + "targetedSegments": { + "type": ["integer", "null"] + }, + "optimizationExperimentStatus": { + "type": ["string", "null"] + }, + "creativeFormat": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_geo_performance.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_geo_performance.json new file mode 100644 index 000000000000..d5b55678ea96 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_geo_performance.json @@ -0,0 +1,119 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_interest.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_interest.json new file mode 100644 index 000000000000..b243c30463b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_interest.json @@ -0,0 +1,89 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "targetingStatus": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["integer", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["integer", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["integer", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_platforms.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_platforms.json new file mode 100644 index 000000000000..f2a0407e0a8c --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_platforms.json @@ -0,0 +1,116 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "platform": { + "type": ["string", "null"] + }, + "breakdown": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["integer", "null"] + }, + "clicks": { + "type": ["integer", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_publisher.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_publisher.json new file mode 100644 index 000000000000..8e737a761272 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_by_publisher.json @@ -0,0 +1,139 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "blockStatus": { + "type": ["string", "null"] + }, + "blockTypes": { + "type": "object", + "properties": { + "marketerPublisher": { + "type": ["boolean", "null"] + }, + "marketerSection": { + "type": ["boolean", "null"] + }, + "campaignPublisher": { + "type": ["boolean", "null"] + }, + "campaignSection": { + "type": ["boolean", "null"] + } + } + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_campaigns_by_geo.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_campaigns_by_geo.json new file mode 100644 index 000000000000..7e1437989fc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_campaigns_by_geo.json @@ -0,0 +1,122 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "code": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["integer", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + }, + "campaign_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_campaigns_by_platforms.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_campaigns_by_platforms.json new file mode 100644 index 000000000000..867e03185c31 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_marketers_campaigns_by_platforms.json @@ -0,0 +1,119 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "platform": { + "type": ["string", "null"] + }, + "breakdown": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["integer", "null"] + }, + "clicks": { + "type": ["integer", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["integer", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + }, + "campaign_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_periodic_by_marketers.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_periodic_by_marketers.json new file mode 100644 index 000000000000..0755d6176b26 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_periodic_by_marketers.json @@ -0,0 +1,110 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "fromDate": { + "type": ["string", "null"] + }, + "toDate": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["integer", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_periodic_by_marketers_campaign.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_periodic_by_marketers_campaign.json new file mode 100644 index 000000000000..6b849edc26d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_periodic_by_marketers_campaign.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "fromDate": { + "type": ["string", "null"] + }, + "toDate": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["number", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["integer", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + }, + "campaign_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_publishers_by_campaigns.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_publishers_by_campaigns.json new file mode 100644 index 000000000000..86241e8e168a --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/performance_report_publishers_by_campaigns.json @@ -0,0 +1,122 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + } + } + }, + "metrics": { + "type": "object", + "properties": { + "impressions": { + "type": ["number", "null"] + }, + "clicks": { + "type": ["number", "null"] + }, + "totalConversions": { + "type": ["number", "null"] + }, + "conversions": { + "type": ["number", "null"] + }, + "viewConversions": { + "type": ["number", "null"] + }, + "spend": { + "type": ["number", "null"] + }, + "ecpc": { + "type": ["number", "null"] + }, + "ctr": { + "type": ["number", "null"] + }, + "conversionRate": { + "type": ["number", "null"] + }, + "viewConversionRate": { + "type": ["integer", "null"] + }, + "cpa": { + "type": ["number", "null"] + }, + "totalCpa": { + "type": ["number", "null"] + }, + "totalSumValue": { + "type": ["number", "null"] + }, + "sumValue": { + "type": ["number", "null"] + }, + "viewSumValue": { + "type": ["number", "null"] + }, + "totalAverageValue": { + "type": ["number", "null"] + }, + "averageValue": { + "type": ["number", "null"] + }, + "viewAverageValue": { + "type": ["number", "null"] + }, + "totalRoas": { + "type": ["number", "null"] + }, + "roas": { + "type": ["number", "null"] + }, + "videoReachedFirstQ": { + "type": ["integer", "null"] + }, + "videoReachedSecondQ": { + "type": ["integer", "null"] + }, + "videoReachedThirdQ": { + "type": ["integer", "null"] + }, + "videoReachedCompletion": { + "type": ["integer", "null"] + }, + "videoViewDuration": { + "type": ["integer", "null"] + }, + "videoAvgViewDuration": { + "type": ["number", "null"] + }, + "videoPlays": { + "type": ["integer", "null"] + }, + "clicksOnVideo": { + "type": ["integer", "null"] + }, + "videoActiveCompletions": { + "type": ["integer", "null"] + }, + "videoActiveCompletionsPercentage": { + "type": ["integer", "null"] + } + } + }, + "marketer_id": { + "type": ["string", "null"] + }, + "campaign_id": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/promoted_links.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/promoted_links.json new file mode 100644 index 000000000000..ce8178a9ebfc --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/promoted_links.json @@ -0,0 +1,94 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "text": { + "type": ["string", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "siteName": { + "type": ["string", "null"] + }, + "sectionName": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "campaignId": { + "type": ["string", "null"] + }, + "archived": { + "type": ["boolean", "null"] + }, + "documentLanguage": { + "type": ["string", "null"] + }, + "onAirStatus": { + "type": "object", + "properties": { + "onAir": { + "type": ["boolean", "null"] + }, + "reason": { + "type": ["string", "null"] + } + } + }, + "baseUrl": { + "type": ["string", "null"] + }, + "documentId": { + "type": ["string", "null"] + }, + "metaData": { + "type": ["string", "null"] + }, + "approvalStatus": { + "type": "object", + "properties": { + "status": { + "type": ["string", "null"] + }, + "isEditable": { + "type": ["boolean", "null"] + } + } + }, + "description": { + "type": ["string", "null"] + }, + "callToAction": { + "type": "object", + "properties": { + "type": { + "type": ["string", "null"] + }, + "value": { + "type": ["string", "null"] + } + } + }, + "imageType": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/promoted_links_sequence_for_campaigns.json b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/promoted_links_sequence_for_campaigns.json new file mode 100644 index 000000000000..405e7422a36d --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/schemas/promoted_links_sequence_for_campaigns.json @@ -0,0 +1,148 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "sequences": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "campaignId": { + "type": ["string", "null"] + }, + "title": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "onAirStatus": { + "type": "object", + "properties": { + "onAir": { + "type": ["boolean", "null"] + }, + "reason": { + "type": ["string", "null"] + } + } + }, + "cachedLogoUrl": { + "type": ["string", "null"] + }, + "sponsor": { + "type": ["string", "null"] + }, + "promotedLinks": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "text": { + "type": ["string", "null"] + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "siteName": { + "type": ["string", "null"] + }, + "sectionName": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "enabled": { + "type": ["boolean", "null"] + }, + "cachedImageUrl": { + "type": ["string", "null"] + }, + "campaignId": { + "type": ["string", "null"] + }, + "archived": { + "type": ["boolean", "null"] + }, + "documentLanguage": { + "type": ["string", "null"] + }, + "onAirStatus": { + "type": "object", + "properties": { + "onAir": { + "type": ["boolean", "null"] + }, + "reason": { + "type": ["string", "null"] + } + } + }, + "baseUrl": { + "type": ["string", "null"] + }, + "documentId": { + "type": ["string", "null"] + }, + "sequenceSettings": { + "type": "object", + "properties": { + "sequenceId": { + "type": ["string", "null"] + }, + "sequenceOrder": { + "type": ["integer", "null"] + } + } + }, + "approvalStatus": { + "type": "object", + "properties": { + "status": { + "type": ["string", "null"] + }, + "isEditable": { + "type": ["boolean", "null"] + } + } + }, + "imageType": { + "type": ["string", "null"] + }, + "language": { + "type": ["string", "null"] + } + } + } + }, + "creationTime": { + "type": ["string", "null"] + }, + "lastModified": { + "type": ["string", "null"] + } + } + } + }, + "totalCount": { + "type": ["integer", "null"] + }, + "count": { + "type": ["integer", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py new file mode 100644 index 000000000000..ae0a7b4d6c70 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/source.py @@ -0,0 +1,1168 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream + +from .auth import OutbrainAmplifyAuthenticator + +DEFAULT_END_DATE = pendulum.now() +DEFAULT_GEO_LOCATION_BREAKDOWN = "region" +DEFAULT_REPORT_GRANULARITY = "daily" + + +# Basic full refresh stream +class OutbrainAmplifyStream(HttpStream, ABC): + url_base = "https://api.outbrain.com/amplify/v0.1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + if response.json(): + total_pages = response.json().get("totalCount") + current_page = response.json().get("count") + if current_page < total_pages - 1: + diff = (total_pages - current_page) - 1 + if diff < current_page + 1: + return {"offset": current_page + 1} + else: + return None + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(next_page_token=next_page_token, stream_state=stream_state, **kwargs) + if next_page_token: + params.update(next_page_token) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield {response.json()} + + @staticmethod + def _get_time_interval( + start_date: Union[pendulum.datetime, str], ending_date: Union[pendulum.datetime, str] + ) -> Iterable[Tuple[pendulum.datetime, pendulum.datetime]]: + if isinstance(start_date, str): + start_date = pendulum.parse(start_date) + end_date = pendulum.parse(ending_date) if ending_date else DEFAULT_END_DATE + if end_date < start_date: + raise ValueError(f"Specified start date: {start_date} is later than the end date: {end_date}") + return start_date, end_date + + +class Marketers(OutbrainAmplifyStream): + primary_key = "id" + + def __init__(self, authenticator, config, **kwargs): + super().__init__(**kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + @property + def use_cache(self) -> bool: + return True + + @property + def cache_filename(self): + return "marketers.yml" + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("marketers"): + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "marketers/" + + +class CampaignsByMarketers(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + @property + def use_cache(self) -> bool: + return True + + @property + def cache_filename(self): + return "campaigns.yml" + + @property + def name(self) -> str: + return "campaigns" + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("campaigns"): + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"marketers/{stream_slice['marketer_id']}/campaigns" + + +# Retrieve Campaign GeoLocations. +# A new endpoint has been added which returns all targeted and excluded locations of a given campaign. It can be called in order to retrieve a campaign's geotargeting. +class CampaignsGeoLocation(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: CampaignsByMarketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"campaign_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("geoLocations"): + x["campaign_id"] = stream_slice["campaign_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"campaigns/{stream_slice['campaign_id']}/locations" + + +# List PromotedLinks for Campaign. +# Collection of all PromotedLinks for the specified Campaign. +class PromotedLinksForCampaigns(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: CampaignsByMarketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + @property + def name(self) -> str: + return "promoted_links" + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"campaign_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("promotedLinks"): + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"campaigns/{stream_slice['campaign_id']}/promotedLinks" + + +# List PromotedLinksSequences for Campaign. +# Collection of all PromotedLinksSequences for the specified Campaign. +class PromotedLinksSequenceForCampaigns(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: CampaignsByMarketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"campaign_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("sequences"): + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"campaigns/{stream_slice['campaign_id']}/promotedLinksSequences" + + +# List Budgets for a Marketer. +# Retrieve a collection of all Budgets for the specified Marketer. +class BudgetsForMarketers(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + @property + def name(self) -> str: + return "budgets" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("budgets"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"marketers/{stream_slice['marketer_id']}/budgets" + + +# Retrieve campaigns with performance statistics for a Marketer. +# The API in this sub-section allows retrieving marketer campaigns data with performance statistics. +class PerformanceReportCampaignsByMarketers(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/campaigns?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve periodic performance statistics for a Marketer. +# The API in this sub-section allows retrieving performance statistics by periodic breakdown at different levels: marketer, budget, campaign and promoted link. +class PerformanceReportPeriodicByMarketers(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/periodic?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&breakdown=" + + str(self.config.get("report_granularity", DEFAULT_REPORT_GRANULARITY)) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for all marketer campaigns by periodic breakdown. +# A special endpoint for retrieving periodic data by campaign breakdown. Now supports all breakdowns: daily, weekly, monthly, hourOfDay, dayOfWeek and dayOfWeekByHour. +class PerformanceReportPeriodicByMarketersCampaign(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for results in response.json().get("campaignResults"): + for x in results.get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + x["campaign_id"] = results.get("campaignId") + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/campaigns/periodic?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&breakdown=" + + str(self.config.get("report_granularity", DEFAULT_REPORT_GRANULARITY)) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve periodic performance statistics by promoted link for a campaign. HERE +# A special endpoint for retrieving periodic data by promoted link breakdown for a given campaign. +class PerformanceReportPeriodicContentByPromotedLinksCampaign(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: CampaignsByMarketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + @property + def name(self) -> str: + return "performance_promoted_links" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("marketerId"), "campaign_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for results in response.json().get("promotedLinkResults"): + for x in results.get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + x["campaign_id"] = stream_slice["campaign_id"] + x["promoted_link_id"] = results.get("promotedLinkId") + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/campaigns/{stream_slice['campaign_id']}/periodicContent?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&breakdown=" + + str(self.config.get("report_granularity", DEFAULT_REPORT_GRANULARITY)) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for a Marketer by publisher. +# Marketer performance statistics with breakdown by publisher with publishers that are already blocked at the marketer or campaign level. +class PerformanceReportMarketersByPublisher(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/publishers?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for all marketer campaigns by publisher +# A special endpoint for retrieving publishers data by campaign breakdown. +class PerformanceReportPublishersByCampaigns(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for fetched in response.json().get("campaignResults"): + for x in fetched.get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + x["campaign_id"] = fetched.get("campaignId") + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/campaigns/publishers?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for a Marketer by platform. +# The API in this sub-section allows retrieving performance statistics by platform at different levels: marketer, budget, and campaign. +class PerformanceReportMarketersByPlatforms(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/platforms?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for all marketer campaigns by platform. +# A special endpoint for retrieving platforms data by campaign breakdown. +class PerformanceReportMarketersCampaignsByPlatforms(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for fetched in response.json().get("campaignResults"): + for x in fetched.get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + x["campaign_id"] = fetched.get("campaignId") + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/campaigns/platforms?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve geo performance statistics for a Marketer. +# The API in this sub-section allows retrieving performance statistics by geographic breakdown at different levels: country, region, and subregion. +class PerformanceReportMarketersByGeoPerformance(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/geo?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&breakdown=" + + str(self.config.get("geo_location_breakdown", DEFAULT_GEO_LOCATION_BREAKDOWN)) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for all marketer campaigns by geo. +# A special endpoint for retrieving geo data by campaign breakdown. +class PerformanceReportMarketersCampaignsByGeo(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for fetched in response.json().get("campaignResults"): + for x in fetched.get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + x["campaign_id"] = fetched.get("campaignId") + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/campaigns/geo?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&breakdown=" + + str(self.config.get("geo_location_breakdown", DEFAULT_GEO_LOCATION_BREAKDOWN)) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +# Retrieve performance statistics for a Marketer by interest. +# The API in this sub-section allows retrieving performance statistics by interest at different levels: marketer and campaign. +class PerformanceReportMarketersByInterest(OutbrainAmplifyStream, HttpSubStream): + primary_key = None + + def __init__(self, authenticator, config, parent: Marketers, **kwargs): + super().__init__(parent=parent, **kwargs) + self.config = config + self._authenticator = authenticator + self._session = requests.sessions.Session() + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def stream_slices( + self, sync_mode: SyncMode.full_refresh, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + for record in parent_records: + yield {"marketer_id": record.get("id")} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.json(): + for x in response.json().get("results"): + x["marketer_id"] = stream_slice["marketer_id"] + yield x + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + stream_start, stream_end = self._get_time_interval(self.config.get("start_date"), self.config.get("end_date")) + return ( + f"reports/marketers/{stream_slice['marketer_id']}/interests?from=" + + str(stream_start.date()) + + "&to=" + + str(stream_end.date()) + + "&limit=500" + + "&includeVideoStats=true" + ) + + +class IncrementalOutbrainAmplifyStream(OutbrainAmplifyStream, ABC): + + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + return {} + + +# Source +class SourceOutbrainAmplify(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + url_base = OutbrainAmplifyStream.url_base + auth = OutbrainAmplifyAuthenticator(url_base=url_base, config=config) + try: + auth.get_auth_header() + marketer_stream = Marketers(authenticator=auth, config=config) + next(marketer_stream.read_records(SyncMode.full_refresh)) + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + url_base = OutbrainAmplifyStream.url_base + auth = OutbrainAmplifyAuthenticator(url_base=url_base, config=config) + # Basic stream marketing + # 1. All Marketing streams + stream = [Marketers(authenticator=auth, config=config)] + + # Campaigns Streams. + # 1. Campaigns by marketers (implemented). + # 2. Camapings Geo Location (implemented). + stream.extend( + [ + CampaignsByMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config)), + CampaignsGeoLocation( + authenticator=auth, + config=config, + parent=CampaignsByMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config)), + ), + ] + ) + + # Budget for Marketers stream. + # 1. Budget stream based on marketers id. + stream.extend([BudgetsForMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config))]), + + # Promoted Links stream. + # 1. Promoted Links stream for campaigns. + stream.extend( + [ + PromotedLinksForCampaigns( + authenticator=auth, + config=config, + parent=CampaignsByMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config)), + ) + ] + ) + + # Promoted Links Sequences stream. + # 1. Promoted Links Sequences stream for campaigns. + stream.extend( + [ + PromotedLinksSequenceForCampaigns( + authenticator=auth, + config=config, + parent=CampaignsByMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config)), + ) + ] + ) + + # Performance Reporting. + # 1. Streams to retrieve performance statistics. + stream.extend( + [ + PerformanceReportCampaignsByMarketers( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportPeriodicByMarketers( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportPeriodicByMarketersCampaign( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportPeriodicContentByPromotedLinksCampaign( + authenticator=auth, + config=config, + parent=CampaignsByMarketers(authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config)), + ), + PerformanceReportMarketersByPublisher( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportPublishersByCampaigns( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportMarketersByPlatforms( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportMarketersCampaignsByPlatforms( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportMarketersByGeoPerformance( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportMarketersCampaignsByGeo( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + PerformanceReportMarketersByInterest( + authenticator=auth, config=config, parent=Marketers(authenticator=auth, config=config) + ), + ] + ) + return stream diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/spec.yaml b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/spec.yaml new file mode 100644 index 000000000000..4da9c42eb738 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/source_outbrain_amplify/spec.yaml @@ -0,0 +1,84 @@ +documentationUrl: "https://docs.airbyte.com/integrations/sources/outbrain-amplify" +connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: Outbrain Amplify Spec + type: object + required: ["credentials", "start_date"] + properties: + credentials: + title: Authentication Method + description: >- + Credentials for making authenticated requests requires either + username/password or access_token. + default: {} + order: 0 + type: object + oneOf: + - title: Access token + type: object + properties: + type: + title: Access token is required for authentication requests. + const: access_token + type: string + access_token: + type: string + description: Access Token for making authenticated requests. + airbyte_secret: true + required: + - type + - access_token + - title: Username Password + type: object + properties: + type: + title: Both username and password is required for authentication request. + const: username_password + type: string + username: + type: string + description: >- + Add Username for authentication. + password: + type: string + description: >- + Add Password for authentication. + airbyte_secret: true + required: + - type + - username + - password + report_granularity: + title: Granularity for periodic reports. + description: >- + The granularity used for periodic data in reports. See the + docs. + enum: + - daily + - weekly + - monthly + order: 1 + type: string + geo_location_breakdown: + title: Granularity for geo-location region. + description: >- + The granularity used for geo location data in reports. + enum: + - country + - region + - subregion + order: 2 + type: string + start_date: + type: string + order: 3 + description: >- + Date in the format YYYY-MM-DD eg. 2017-01-25. Any data before this date will not be replicated. + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + end_date: + type: string + order: 4 + description: >- + Date in the format YYYY-MM-DD. + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/__init__.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..5f06ad6891b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_outbrain_amplify.source import IncrementalOutbrainAmplifyStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalOutbrainAmplifyStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalOutbrainAmplifyStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalOutbrainAmplifyStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalOutbrainAmplifyStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalOutbrainAmplifyStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalOutbrainAmplifyStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalOutbrainAmplifyStream, "cursor_field", "dummy_field") + stream = IncrementalOutbrainAmplifyStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalOutbrainAmplifyStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalOutbrainAmplifyStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py new file mode 100644 index 000000000000..c80a9bc4ef26 --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_outbrain_amplify.source import SourceOutbrainAmplify + + +def test_check_connection(mocker): + config = { + "credentials": + { + "type": "access_token", + "access_token" : "MTY1OTUyO" + }, + "report_granularity": "daily", + "geo_location_breakdown": "region", + "start_date" : "2022-04-01", + "end_date" : "2022-04-30" + } + source = SourceOutbrainAmplify() + cond = source.check_connection(True, config)[0] + assert cond is False + + +def test_streams(mocker): + source = SourceOutbrainAmplify() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 17 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py new file mode 100644 index 000000000000..1fa51bef5b6f --- /dev/null +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import inspect +import json +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +import requests +from source_outbrain_amplify.source import OutbrainAmplifyStream + + +@pytest.fixture +def patch_base_class(mocker): + mocker.patch.object(OutbrainAmplifyStream, "path", "v0/example_endpoint") + mocker.patch.object(OutbrainAmplifyStream, "primary_key", "test_primary_key") + mocker.patch.object(OutbrainAmplifyStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = OutbrainAmplifyStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = OutbrainAmplifyStream() + inputs = {"totalCount": 100, "count": 50} + pagination_token = json.dumps(inputs) + response = requests.Response() + response.status_code = 200 + response.headers['content-type'] = 'application/json' + response._content = pagination_token.encode('utf-8') + expected_token = {'offset': 51} + assert stream.next_page_token(response) == expected_token + + +def test_parse_response(patch_base_class): + stream = OutbrainAmplifyStream() + mock_response = { + "campaigns": [], + "totalCount": 5, + "count": 0 + } + mock_response = json.dumps(mock_response) + response = requests.Response() + response.status_code = 200 + response.headers['Content-Type'] = 'application/json' + response._content = mock_response.encode('utf-8') + result = stream.parse_response(response) + expected_result = True + assert inspect.isgenerator(result) == expected_result + + +def test_request_headers(patch_base_class): + stream = OutbrainAmplifyStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = OutbrainAmplifyStream() + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = OutbrainAmplifyStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = OutbrainAmplifyStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time + + +def test_get_time_interval(patch_base_class): + stream = OutbrainAmplifyStream() + start_date = "2022-08-03 00:00:00" + ending_date = "2022-08-02 00:00:00" + try: + stream._get_time_interval(start_date, ending_date) + except ValueError as e: + assert e diff --git a/airbyte-integrations/connectors/source-outreach/Dockerfile b/airbyte-integrations/connectors/source-outreach/Dockerfile index dc3e12ab01c5..58ceb06732fb 100644 --- a/airbyte-integrations/connectors/source-outreach/Dockerfile +++ b/airbyte-integrations/connectors/source-outreach/Dockerfile @@ -34,5 +34,5 @@ COPY source_outreach ./source_outreach ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-outreach diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json index 8d09458dece6..aecfa1a5a17b 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json @@ -31,5 +31,17 @@ }, "calls": { "updatedAt": "2040-11-16T00:00:00Z" + }, + "users": { + "updatedAt": "2040-11-16T00:00:00Z" + }, + "tasks": { + "updatedAt": "2040-11-16T00:00:00Z" + }, + "templates": { + "updatedAt": "2040-11-16T00:00:00Z" + }, + "snippets": { + "updatedAt": "2040-11-16T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json index a07c9004d581..1139b289868d 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json @@ -142,6 +142,58 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "tasks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "templates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "snippets", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["updatedAt"] } ] } diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json index ed1e313ef9c4..d11f3710bb3c 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json @@ -31,5 +31,17 @@ }, "calls": { "updatedAt": "2021-06-28T10:10:20Z" + }, + "users": { + "updatedAt": "2021-06-28T10:10:20Z" + }, + "tasks": { + "updatedAt": "2021-06-28T10:10:20Z" + }, + "templates": { + "updatedAt": "2021-06-28T10:10:20Z" + }, + "snippets": { + "updatedAt": "2021-06-28T10:10:20Z" } } diff --git a/airbyte-integrations/connectors/source-outreach/metadata.yaml b/airbyte-integrations/connectors/source-outreach/metadata.yaml index 6df687ec90d1..278b8bad0daa 100644 --- a/airbyte-integrations/connectors/source-outreach/metadata.yaml +++ b/airbyte-integrations/connectors/source-outreach/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 3490c201-5d95-4783-b600-eaf07a4c7787 - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-outreach + documentationUrl: https://docs.airbyte.com/integrations/sources/outreach githubIssueLabel: source-outreach icon: outreach.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/outreach + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-outreach/requirements.txt b/airbyte-integrations/connectors/source-outreach/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-outreach/requirements.txt +++ b/airbyte-integrations/connectors/source-outreach/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-outreach/setup.py b/airbyte-integrations/connectors/source-outreach/setup.py index 0f9222c19cd6..2367148b8d73 100644 --- a/airbyte-integrations/connectors/source-outreach/setup.py +++ b/airbyte-integrations/connectors/source-outreach/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json index 61c7e0960a02..0518e59d4c10 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json @@ -472,7 +472,7 @@ "description": { "type": ["null", "string"] }, - "domain":{ + "domain": { "type": ["null", "string"] }, "externalSource": { @@ -564,4 +564,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json index f040ce419718..2a4666dac51b 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json @@ -7,269 +7,140 @@ "type": "integer" }, "externalVendor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "direction": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "from": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "note": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "outcome": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "recordingUrl": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sequenceAction": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "state": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "shouldRecordCall": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "to": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "uid": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "userCallType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "vendorCallId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "voicemailRecordingUrl": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "answeredAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "completedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "createdAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "updatedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "dialedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "returnedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "stateChangedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "tags": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, "callDisposition": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "callPurpose": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "opportunity": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "phoneNumber": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "prospect": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "sequence": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "sequenceState": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "sequenceStep": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "task": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, "user": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json index 0ee21b2af6c5..4285c3ea9271 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json @@ -7,107 +7,56 @@ "type": "integer" }, "authId": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "createdAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "editable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "email": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "emailProvider": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "emailSignature": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ewsEndpoint": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ewsSslVerifyMode": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "exchangeVersion": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "imapHost": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "imapPort": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "imapSsl": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "maxEmailsPerDay": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "maxMailingsPerDay": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "maxMailingsPerWeek": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "optOutMessage": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "optOutSignature": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "prospectEmailExclusions": { "type": ["null", "array"], @@ -116,180 +65,96 @@ } }, "providerId": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "providerType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sendDisabled": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sendErroredAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "sendMaxRetries": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "sendMethod": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sendPeriod": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "sendPermanentErrorAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "sendRequiresSync": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sendSuccessAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "sendThreshold": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "sendgridWebhookUrl": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "smtpHost": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "smtpPort": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "smtpSsl": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "smtpUsername": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "syncActiveFrequency": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "syncDisabled": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "syncErroredAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "syncFinishedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "syncMethod": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "syncOutreachFolder": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "syncPassiveFrequency": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "syncPermanentErrorAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "syncSuccessAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "updatedAt": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "userId": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "username": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "creator": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json index 05af6007641a..29b36cddd5b7 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json @@ -193,4 +193,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json index 2d4341671560..997c02324c55 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json @@ -17,7 +17,7 @@ "type": ["null", "string"], "format": "date-time" }, - "currencyType":{ + "currencyType": { "type": ["null", "string"] }, "custom1": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/personas.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/personas.json index 2bc559085015..c9f263fad813 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/personas.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/personas.json @@ -27,4 +27,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/snippets.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/snippets.json new file mode 100644 index 000000000000..d957962eb75e --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/snippets.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "bodyHtml": { + "type": ["null", "string"] + }, + "bodyText": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "shareType": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "contentCategoryMemberships": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "owner": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "updater": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json index c7d21e60340d..291fa583b220 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json @@ -42,4 +42,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json new file mode 100644 index 000000000000..af7241eedc96 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json @@ -0,0 +1,167 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "action": { + "type": ["null", "string"] + }, + "autoskipAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "compiledSequenceTemplateHtml": { + "type": ["null", "string"] + }, + "completed": { + "type": ["null", "boolean"] + }, + "completedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "dueAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "opportunityAssociation": { + "type": ["null", "string"] + }, + "scheduledAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "state": { + "type": ["null", "string"] + }, + "stateChangedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "taskType": { + "type": ["null", "string"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "account": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "call": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "calls": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "completer": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "mailing": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "mailings": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "opportunity": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "owner": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "prospect": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sequence": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sequenceSequenceSteps": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sequenceState": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sequenceStep": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sequenceTemplate": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "subject": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "taskPriority": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "template": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/templates.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/templates.json new file mode 100644 index 000000000000..455d4201cf33 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/templates.json @@ -0,0 +1,131 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "archived": { + "type": ["null", "boolean"] + }, + "archivedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "bccRecipients": { + "type": ["null", "string"] + }, + "bodyHtml": { + "type": ["null", "string"] + }, + "bodyText": { + "type": ["null", "string"] + }, + "bounceCount": { + "type": ["null", "integer"] + }, + "ccRecipients": { + "type": ["null", "string"] + }, + "clickCount": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "deliverCount": { + "type": ["null", "integer"] + }, + "failureCount": { + "type": ["null", "integer"] + }, + "lastUsedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "negativeReplyCount": { + "type": ["null", "integer"] + }, + "neutralReplyCount": { + "type": ["null", "integer"] + }, + "openCount": { + "type": ["null", "integer"] + }, + "optOutCount": { + "type": ["null", "integer"] + }, + "positiveReplyCount": { + "type": ["null", "integer"] + }, + "replyCount": { + "type": ["null", "integer"] + }, + "scheduleCount": { + "type": ["null", "integer"] + }, + "shareType": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "toRecipients": { + "type": ["null", "string"] + }, + "trackLinks": { + "type": ["null", "boolean"] + }, + "trackOpens": { + "type": ["null", "boolean"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "contentCategoryMemberships": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "owner": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "recipients": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "sequenceTemplates": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "updater": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json new file mode 100644 index 000000000000..d3d1290b9da3 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json @@ -0,0 +1,299 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "accountsViewId": { + "type": ["null", "integer"] + }, + "activityNotificationsDisabled": { + "type": ["null", "boolean"] + }, + "bounceWarningEmailEnabled": { + "type": ["null", "boolean"] + }, + "bridgePhone": { + "type": ["null", "string"] + }, + "bridgePhoneExtension": { + "type": ["null", "string"] + }, + "callsViewId": { + "type": ["null", "integer"] + }, + "controlledTabDefault": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "currentSignInAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "custom1": { + "type": ["null", "string"] + }, + "custom2": { + "type": ["null", "string"] + }, + "custom3": { + "type": ["null", "string"] + }, + "custom4": { + "type": ["null", "string"] + }, + "custom5": { + "type": ["null", "string"] + }, + "dailyDigestEmailEnabled": { + "type": ["null", "boolean"] + }, + "defaultRulesetId": { + "type": ["null", "integer"] + }, + "duties": { + "items": { + "properties": { + "duty_type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "type": ["null", "array"] + }, + "email": { + "type": ["null", "string"] + }, + "enableVoiceRecordings": { + "type": ["null", "boolean"] + }, + "engagementEmailsEnabled": { + "type": ["null", "boolean"] + }, + "firstName": { + "type": ["null", "string"] + }, + "inboundBridgePhone": { + "type": ["null", "string"] + }, + "inboundBridgePhoneExtension": { + "type": ["null", "string"] + }, + "inboundCallBehavior": { + "type": ["null", "string"] + }, + "inboundPhoneType": { + "type": ["null", "string"] + }, + "inboundVoicemailCustomMessageText": { + "type": ["null", "string"] + }, + "inboundVoicemailMessageTextVoice": { + "type": ["null", "string"] + }, + "inboundVoicemailPromptType": { + "type": ["null", "string"] + }, + "kaiaRecordingsViewId": { + "type": ["null", "integer"] + }, + "keepBridgePhoneConnected": { + "type": ["null", "boolean"] + }, + "lastName": { + "type": ["null", "string"] + }, + "lastSignInAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "locked": { + "type": ["null", "boolean"] + }, + "mailboxErrorEmailEnabled": { + "type": ["null", "boolean"] + }, + "meetingEngagementNotificationEnabled": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "notificationsEnabled": { + "type": ["null", "boolean"] + }, + "oceClickToDialEverywhere": { + "type": ["null", "boolean"] + }, + "oceGmailToolbar": { + "type": ["null", "boolean"] + }, + "oceGmailTrackingState": { + "type": ["null", "string"] + }, + "oceSalesforceEmailDecorating": { + "type": ["null", "boolean"] + }, + "oceSalesforcePhoneDecorating": { + "type": ["null", "boolean"] + }, + "oceUniversalTaskFlow": { + "type": ["null", "boolean"] + }, + "oceWindowMode": { + "type": ["null", "boolean"] + }, + "opportunitiesViewId": { + "type": ["null", "integer"] + }, + "passwordExpiresAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "phoneCountryCode": { + "type": ["null", "string"] + }, + "phoneNumber": { + "type": ["null", "string"] + }, + "phoneType": { + "type": ["null", "string"] + }, + "pluginAlertNotificationEnabled": { + "type": ["null", "boolean"] + }, + "preferredVoiceRegion": { + "type": ["null", "string"] + }, + "prefersLocalPresence": { + "type": ["null", "boolean"] + }, + "primaryTimezone": { + "type": ["null", "string"] + }, + "prospectsViewId": { + "type": ["null", "integer"] + }, + "reportsTeamPerfViewId": { + "type": ["null", "integer"] + }, + "reportsViewId": { + "type": ["null", "integer"] + }, + "scimExternalId": { + "type": ["null", "string"] + }, + "scimSource": { + "type": ["null", "string"] + }, + "secondaryTimezone": { + "type": ["null", "string"] + }, + "senderNotificationsExcluded": { + "type": ["null", "boolean"] + }, + "tasksViewId": { + "type": ["null", "integer"] + }, + "teamsViewId": { + "type": ["null", "integer"] + }, + "tertiaryTimezone": { + "type": ["null", "string"] + }, + "textingEmailNotifications": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "unknownReplyEmailEnabled": { + "type": ["null", "boolean"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "userGuid": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "usersViewId": { + "type": ["null", "integer"] + }, + "voicemailNotificationEnabled": { + "type": ["null", "boolean"] + }, + "weeklyDigestEmailEnabled": { + "type": ["null", "boolean"] + }, + "contentCategories": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "mailbox": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "mailboxes": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "profile": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "recipients": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "role": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "updater": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index f26ace5db2da..002b27ab7e96 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -208,6 +208,46 @@ def path(self, **kwargs) -> str: return "calls" +class Users(IncrementalOutreachStream): + """ + Users stream. Yields data from the GET /users endpoint. + See https://api.outreach.io/api/v2/docs#user + """ + + def path(self, **kwargs) -> str: + return "users" + + +class Tasks(IncrementalOutreachStream): + """ + Tasks stream. Yields data from the GET /tasts endpoint. + See https://api.outreach.io/api/v2/docs#task + """ + + def path(self, **kwargs) -> str: + return "tasks" + + +class Templates(IncrementalOutreachStream): + """ + Templates stream. Yields data from the GET /templates endpoint. + See https://api.outreach.io/api/v2/docs#template + """ + + def path(self, **kwargs) -> str: + return "templates" + + +class Snippets(IncrementalOutreachStream): + """ + Snippets stream. Yields data from the GET /snippets endpoint. + See https://api.outreach.io/api/v2/docs#snippet + """ + + def path(self, **kwargs) -> str: + return "snippets" + + class OutreachAuthenticator(Oauth2Authenticator): def __init__(self, redirect_uri: str, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str): super().__init__( @@ -256,4 +296,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Mailboxes(authenticator=auth, **config), Stages(authenticator=auth, **config), Calls(authenticator=auth, **config), + Users(authenticator=auth, **config), + Tasks(authenticator=auth, **config), + Templates(authenticator=auth, **config), + Snippets(authenticator=auth, **config), ] diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py index 6510dec7c202..5be9a07247fe 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py @@ -11,5 +11,5 @@ def test_streams(mocker): source = SourceOutreach() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 11 + expected_streams_number = 15 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-pardot/metadata.yaml b/airbyte-integrations/connectors/source-pardot/metadata.yaml index 9cf9be76616b..6a0ab9f2d1ff 100644 --- a/airbyte-integrations/connectors/source-pardot/metadata.yaml +++ b/airbyte-integrations/connectors/source-pardot/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pardot tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pardot/requirements.txt b/airbyte-integrations/connectors/source-pardot/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pardot/requirements.txt +++ b/airbyte-integrations/connectors/source-pardot/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pardot/setup.py b/airbyte-integrations/connectors/source-pardot/setup.py index 2aadd9ca8496..c04454b0bd05 100644 --- a/airbyte-integrations/connectors/source-pardot/setup.py +++ b/airbyte-integrations/connectors/source-pardot/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-partnerstack/metadata.yaml b/airbyte-integrations/connectors/source-partnerstack/metadata.yaml index 3faf20ba1fd6..726c61a4f52d 100644 --- a/airbyte-integrations/connectors/source-partnerstack/metadata.yaml +++ b/airbyte-integrations/connectors/source-partnerstack/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-partnerstack/requirements.txt b/airbyte-integrations/connectors/source-partnerstack/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-partnerstack/requirements.txt +++ b/airbyte-integrations/connectors/source-partnerstack/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-partnerstack/setup.py b/airbyte-integrations/connectors/source-partnerstack/setup.py index a058c78ab0fd..5e6542bbc4d6 100644 --- a/airbyte-integrations/connectors/source-partnerstack/setup.py +++ b/airbyte-integrations/connectors/source-partnerstack/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile index bcb408dbf2e8..282bb021a007 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile +++ b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=2.0.0 LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index 50cd23c2ddf5..38e6b7be325e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -4,41 +4,61 @@ acceptance_tests: spec: tests: - spec_path: source_paypal_transaction/spec.json + config_path: secrets/config_oauth.json + backward_compatibility_tests_config: + disable_for_version: "0.1.13" connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: exception - config_path: secrets/config_oauth.json status: succeed + - config_path: secrets/config_oauth_sandbox.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed - config_path: integration_tests/invalid_config_oauth.json - status: exception + status: failed discovery: tests: - - config_path: secrets/config.json + - config_path: secrets/config_oauth.json + backward_compatibility_tests_config: + disable_for_version: "1.0.0" # Balances schema changed basic_read: tests: - - config_path: secrets/config.json + - config_path: secrets/config_oauth.json + ignored_fields: + balances: + - name: last_refresh_time + bypass_reason: "field changes during every read" empty_streams: - - name: balances - bypass_reason: "value of 'last_refresh_time' field changes during every read" + - name: transactions + bypass_reason: "can not populate" timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no exact_order: no extra_records: yes + - config_path: secrets/config_oauth_sandbox.json + ignored_fields: + balances: + - name: last_refresh_time + bypass_reason: "field changes during every read" + timeout_seconds: 1200 + expect_records: + path: "integration_tests/expected_records_sandbox.jsonl" + extra_fields: no + exact_order: no + extra_records: yes incremental: tests: - - config_path: secrets/config.json + - config_path: secrets/config_oauth.json configured_catalog_path: integration_tests/configured_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json cursor_paths: - transactions: ["date"] - balances: ["date"] + transactions: [ "date" ] + balances: [ "date" ] full_refresh: tests: - - config_path: secrets/config.json + - config_path: secrets/config_oauth.json configured_catalog_path: integration_tests/configured_catalog.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl index 5be7b634a755..7841d256e49d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl @@ -1,41 +1 @@ -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "23N61105X92314351", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-04T17:13:23+0000", "transaction_updated_date": "2021-07-04T17:13:23+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "202.58"}, "available_balance": {"currency_code": "USD", "value": "202.58"}, "invoice_id": "48787580055", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "48787580055"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "48787580055"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-04T17:13:23+0000", "transaction_id": "23N61105X92314351"}, "emitted_at": 1673959656444} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "1FN09943JY662130R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T22:56:54+0000", "transaction_updated_date": "2021-07-05T22:56:54+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "231.52"}, "available_balance": {"currency_code": "USD", "value": "231.52"}, "invoice_id": "65095789448", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "65095789448"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "65095789448"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T22:56:54+0000", "transaction_id": "1FN09943JY662130R"}, "emitted_at": 1673959656446} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0M443597T0019954R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:01:13+0000", "transaction_updated_date": "2021-07-05T23:01:13+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "260.46"}, "available_balance": {"currency_code": "USD", "value": "260.46"}, "invoice_id": "41468340464", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "41468340464"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "41468340464"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:01:13+0000", "transaction_id": "0M443597T0019954R"}, "emitted_at": 1673959656448} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "19C257131E850262B", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:02:46+0000", "transaction_updated_date": "2021-07-05T23:02:46+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "289.40"}, "available_balance": {"currency_code": "USD", "value": "289.40"}, "invoice_id": "23749371955", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "23749371955"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "23749371955"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:02:46+0000", "transaction_id": "19C257131E850262B"}, "emitted_at": 1673959656450} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "6S892278N6406494Y", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:06:12+0000", "transaction_updated_date": "2021-07-05T23:06:12+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "318.34"}, "available_balance": {"currency_code": "USD", "value": "318.34"}, "invoice_id": "62173333941", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "62173333941"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "62173333941"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:06:12+0000", "transaction_id": "6S892278N6406494Y"}, "emitted_at": 1673959656453} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0T320567TS5587836", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:09:04+0000", "transaction_updated_date": "2021-07-05T23:09:04+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "347.28"}, "available_balance": {"currency_code": "USD", "value": "347.28"}, "invoice_id": "56028534885", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "56028534885"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "56028534885"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:09:04+0000", "transaction_id": "0T320567TS5587836"}, "emitted_at": 1673959656455} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3DF69605L9958744R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:12:40+0000", "transaction_updated_date": "2021-07-05T23:12:40+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "376.22"}, "available_balance": {"currency_code": "USD", "value": "376.22"}, "invoice_id": "31766547902", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "31766547902"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "31766547902"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:12:40+0000", "transaction_id": "3DF69605L9958744R"}, "emitted_at": 1673959656457} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2F535603PS249601F", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:12:57+0000", "transaction_updated_date": "2021-07-05T23:12:57+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "405.16"}, "available_balance": {"currency_code": "USD", "value": "405.16"}, "invoice_id": "32577611997", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "32577611997"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "32577611997"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:12:57+0000", "transaction_id": "2F535603PS249601F"}, "emitted_at": 1673959656459} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "243514451L952570P", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:02+0000", "transaction_updated_date": "2021-07-05T23:14:02+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "434.10"}, "available_balance": {"currency_code": "USD", "value": "434.10"}, "invoice_id": "23612058730", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "23612058730"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "23612058730"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:14:02+0000", "transaction_id": "243514451L952570P"}, "emitted_at": 1673959656460} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "27881589Y9461861H", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:19+0000", "transaction_updated_date": "2021-07-05T23:14:19+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "463.04"}, "available_balance": {"currency_code": "USD", "value": "463.04"}, "invoice_id": "53296156982", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "53296156982"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "53296156982"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:14:19+0000", "transaction_id": "27881589Y9461861H"}, "emitted_at": 1673959656462} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3MG39755337297727", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:36+0000", "transaction_updated_date": "2021-07-05T23:14:36+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "491.98"}, "available_balance": {"currency_code": "USD", "value": "491.98"}, "invoice_id": "53235397043", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "53235397043"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "53235397043"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:14:36+0000", "transaction_id": "3MG39755337297727"}, "emitted_at": 1673959656463} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "32J59182JY5989507", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:14:52+0000", "transaction_updated_date": "2021-07-05T23:14:52+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "520.92"}, "available_balance": {"currency_code": "USD", "value": "520.92"}, "invoice_id": "18208641465", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "18208641465"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "18208641465"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:14:52+0000", "transaction_id": "32J59182JY5989507"}, "emitted_at": 1673959656464} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "52795774C7828234R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:09+0000", "transaction_updated_date": "2021-07-05T23:15:09+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "549.86"}, "available_balance": {"currency_code": "USD", "value": "549.86"}, "invoice_id": "32274344746", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "32274344746"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "32274344746"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:15:09+0000", "transaction_id": "52795774C7828234R"}, "emitted_at": 1673959656465} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "19B82038T92822940", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:26+0000", "transaction_updated_date": "2021-07-05T23:15:26+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "578.80"}, "available_balance": {"currency_code": "USD", "value": "578.80"}, "invoice_id": "36419288277", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "36419288277"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "36419288277"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:15:26+0000", "transaction_id": "19B82038T92822940"}, "emitted_at": 1673959656467} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "61G749036D552760G", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:42+0000", "transaction_updated_date": "2021-07-05T23:15:42+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "607.74"}, "available_balance": {"currency_code": "USD", "value": "607.74"}, "invoice_id": "88092228645", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "88092228645"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "88092228645"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:15:42+0000", "transaction_id": "61G749036D552760G"}, "emitted_at": 1673959656468} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5EL311302L108363J", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:15:58+0000", "transaction_updated_date": "2021-07-05T23:15:58+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "636.68"}, "available_balance": {"currency_code": "USD", "value": "636.68"}, "invoice_id": "25494061224", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "25494061224"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "25494061224"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:15:58+0000", "transaction_id": "5EL311302L108363J"}, "emitted_at": 1673959656470} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3VP82838NP358133N", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:16:15+0000", "transaction_updated_date": "2021-07-05T23:16:15+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "665.62"}, "available_balance": {"currency_code": "USD", "value": "665.62"}, "invoice_id": "82173600275", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "82173600275"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "82173600275"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:16:15+0000", "transaction_id": "3VP82838NP358133N"}, "emitted_at": 1673959656471} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2N796839EY2539153", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:16:32+0000", "transaction_updated_date": "2021-07-05T23:16:32+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "694.56"}, "available_balance": {"currency_code": "USD", "value": "694.56"}, "invoice_id": "10442581967", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "10442581967"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "10442581967"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:16:32+0000", "transaction_id": "2N796839EY2539153"}, "emitted_at": 1673959656472} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5WX252723D093564T", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:23:29+0000", "transaction_updated_date": "2021-07-05T23:23:29+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "723.50"}, "available_balance": {"currency_code": "USD", "value": "723.50"}, "invoice_id": "71987080514", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "71987080514"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "71987080514"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:23:29+0000", "transaction_id": "5WX252723D093564T"}, "emitted_at": 1673959656473} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "4PW76195NN227720S", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:23:40+0000", "transaction_updated_date": "2021-07-05T23:23:40+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "752.44"}, "available_balance": {"currency_code": "USD", "value": "752.44"}, "invoice_id": "93025400757", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "93025400757"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "93025400757"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:23:40+0000", "transaction_id": "4PW76195NN227720S"}, "emitted_at": 1673959656474} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0VE851712U5895412", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:23:51+0000", "transaction_updated_date": "2021-07-05T23:23:51+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "781.38"}, "available_balance": {"currency_code": "USD", "value": "781.38"}, "invoice_id": "46225965444", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "46225965444"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "46225965444"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:23:51+0000", "transaction_id": "0VE851712U5895412"}, "emitted_at": 1673959656475} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "63U003588S1135607", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:29:26+0000", "transaction_updated_date": "2021-07-05T23:29:26+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "810.32"}, "available_balance": {"currency_code": "USD", "value": "810.32"}, "invoice_id": "34635559567", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "34635559567"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "34635559567"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:29:26+0000", "transaction_id": "63U003588S1135607"}, "emitted_at": 1673959656476} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2AJ081444T051123A", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:29:37+0000", "transaction_updated_date": "2021-07-05T23:29:37+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "839.26"}, "available_balance": {"currency_code": "USD", "value": "839.26"}, "invoice_id": "92544485996", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "92544485996"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "92544485996"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:29:37+0000", "transaction_id": "2AJ081444T051123A"}, "emitted_at": 1673959656477} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "2KU13114TJ604181E", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:29:48+0000", "transaction_updated_date": "2021-07-05T23:29:48+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "868.20"}, "available_balance": {"currency_code": "USD", "value": "868.20"}, "invoice_id": "10184574713", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "10184574713"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "10184574713"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:29:48+0000", "transaction_id": "2KU13114TJ604181E"}, "emitted_at": 1673959656478} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "1ST090036H2235215", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:31:35+0000", "transaction_updated_date": "2021-07-05T23:31:35+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "897.14"}, "available_balance": {"currency_code": "USD", "value": "897.14"}, "invoice_id": "50350860865", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "50350860865"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "50350860865"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:31:35+0000", "transaction_id": "1ST090036H2235215"}, "emitted_at": 1673959656479} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5BJ418934Y425901G", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:31:46+0000", "transaction_updated_date": "2021-07-05T23:31:46+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "926.08"}, "available_balance": {"currency_code": "USD", "value": "926.08"}, "invoice_id": "12278283055", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "12278283055"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "12278283055"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:31:46+0000", "transaction_id": "5BJ418934Y425901G"}, "emitted_at": 1673959656480} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0SD21997LN026020M", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:31:56+0000", "transaction_updated_date": "2021-07-05T23:31:56+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "955.02"}, "available_balance": {"currency_code": "USD", "value": "955.02"}, "invoice_id": "52396214250", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "52396214250"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "52396214250"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:31:56+0000", "transaction_id": "0SD21997LN026020M"}, "emitted_at": 1673959656481} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "3BH630398E562901G", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:42:41+0000", "transaction_updated_date": "2021-07-05T23:42:41+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "983.96"}, "available_balance": {"currency_code": "USD", "value": "983.96"}, "invoice_id": "18793521512", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "18793521512"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "18793521512"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:42:41+0000", "transaction_id": "3BH630398E562901G"}, "emitted_at": 1673959656482} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "03D88325GF8461705", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:42:52+0000", "transaction_updated_date": "2021-07-05T23:42:52+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1012.90"}, "available_balance": {"currency_code": "USD", "value": "1012.90"}, "invoice_id": "71793513892", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "71793513892"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "71793513892"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:42:52+0000", "transaction_id": "03D88325GF8461705"}, "emitted_at": 1673959656484} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "51852852PL0100404", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:43:03+0000", "transaction_updated_date": "2021-07-05T23:43:03+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1041.84"}, "available_balance": {"currency_code": "USD", "value": "1041.84"}, "invoice_id": "98653187889", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "98653187889"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "98653187889"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:43:03+0000", "transaction_id": "51852852PL0100404"}, "emitted_at": 1673959656485} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "8MF4324694292993B", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:21+0000", "transaction_updated_date": "2021-07-05T23:44:21+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1070.78"}, "available_balance": {"currency_code": "USD", "value": "1070.78"}, "invoice_id": "12489150471", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "12489150471"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "12489150471"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:44:21+0000", "transaction_id": "8MF4324694292993B"}, "emitted_at": 1673959656486} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "87S73342AS6001233", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:32+0000", "transaction_updated_date": "2021-07-05T23:44:32+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1099.72"}, "available_balance": {"currency_code": "USD", "value": "1099.72"}, "invoice_id": "99595079917", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "99595079917"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "99595079917"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:44:32+0000", "transaction_id": "87S73342AS6001233"}, "emitted_at": 1673959656487} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "112146346A741221U", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:44+0000", "transaction_updated_date": "2021-07-05T23:44:44+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1128.66"}, "available_balance": {"currency_code": "USD", "value": "1128.66"}, "invoice_id": "93286331651", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "93286331651"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "93286331651"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:44:44+0000", "transaction_id": "112146346A741221U"}, "emitted_at": 1673959656488} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0N2242037Y9449344", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:44:54+0000", "transaction_updated_date": "2021-07-05T23:44:54+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1157.60"}, "available_balance": {"currency_code": "USD", "value": "1157.60"}, "invoice_id": "71349988314", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "71349988314"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "71349988314"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:44:54+0000", "transaction_id": "0N2242037Y9449344"}, "emitted_at": 1673959656489} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "9NH78349H0388780F", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:05+0000", "transaction_updated_date": "2021-07-05T23:45:05+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1186.54"}, "available_balance": {"currency_code": "USD", "value": "1186.54"}, "invoice_id": "83951023481", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "83951023481"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "83951023481"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:45:05+0000", "transaction_id": "9NH78349H0388780F"}, "emitted_at": 1673959656490} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "10S137566E4828249", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:16+0000", "transaction_updated_date": "2021-07-05T23:45:16+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1215.48"}, "available_balance": {"currency_code": "USD", "value": "1215.48"}, "invoice_id": "88168198250", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "88168198250"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "88168198250"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:45:16+0000", "transaction_id": "10S137566E4828249"}, "emitted_at": 1673959656491} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "7N749695W59419057", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:27+0000", "transaction_updated_date": "2021-07-05T23:45:27+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1244.42"}, "available_balance": {"currency_code": "USD", "value": "1244.42"}, "invoice_id": "38296993497", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "38296993497"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "38296993497"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:45:27+0000", "transaction_id": "7N749695W59419057"}, "emitted_at": 1673959656492} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "43X058357A257931N", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:39+0000", "transaction_updated_date": "2021-07-05T23:45:39+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1273.36"}, "available_balance": {"currency_code": "USD", "value": "1273.36"}, "invoice_id": "33391419042", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "33391419042"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "33391419042"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:45:39+0000", "transaction_id": "43X058357A257931N"}, "emitted_at": 1673959656493} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "5WL82051VY277550S", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:45:50+0000", "transaction_updated_date": "2021-07-05T23:45:50+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1302.30"}, "available_balance": {"currency_code": "USD", "value": "1302.30"}, "invoice_id": "69341308548", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "69341308548"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "69341308548"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:45:50+0000", "transaction_id": "5WL82051VY277550S"}, "emitted_at": 1673959656494} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "9CG36572NK0728016", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:46:01+0000", "transaction_updated_date": "2021-07-05T23:46:01+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1331.24"}, "available_balance": {"currency_code": "USD", "value": "1331.24"}, "invoice_id": "70491310163", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "70491310163"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "70491310163"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:46:01+0000", "transaction_id": "9CG36572NK0728016"}, "emitted_at": 1673959656495} -{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "9K759703FU663194K", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:46:43+0000", "transaction_updated_date": "2021-07-05T23:46:43+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "1360.18"}, "available_balance": {"currency_code": "USD", "value": "1360.18"}, "invoice_id": "44794712899", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "44794712899"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "44794712899"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_initiation_date": "2021-07-05T23:46:43+0000", "transaction_id": "9K759703FU663194K"}, "emitted_at": 1673959656495} \ No newline at end of file +{"stream":"balances","data":{"balances":[{"currency":"USD","primary":true,"total_balance":{"currency_code":"USD","value":"0.00"},"available_balance":{"currency_code":"USD","value":"0.00"},"withheld_balance":{"currency_code":"USD","value":"0.00"}}],"account_id":"QJQSC8WXYCA2L","as_of_time":"2021-07-03T00:00:00+00:00","last_refresh_time":"2023-07-04T07:29:59Z"},"emitted_at":1688463837632} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl new file mode 100644 index 000000000000..3d60f02eb69d --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl @@ -0,0 +1,2 @@ +{"stream":"transactions","data":{"transaction_info":{"paypal_account_id":"ZE5533HZPGMC6","transaction_id":"23N61105X92314351","transaction_event_code":"T0006","transaction_initiation_date":"2021-07-04T17:13:23+0000","transaction_updated_date":"2021-07-04T17:13:23+0000","transaction_amount":{"currency_code":"USD","value":"30.11"},"fee_amount":{"currency_code":"USD","value":"-1.17"},"insurance_amount":{"currency_code":"USD","value":"0.01"},"shipping_amount":{"currency_code":"USD","value":"1.03"},"shipping_discount_amount":{"currency_code":"USD","value":"1.00"},"transaction_status":"S","transaction_subject":"This is the payment transaction description.","ending_balance":{"currency_code":"USD","value":"202.58"},"available_balance":{"currency_code":"USD","value":"202.58"},"invoice_id":"48787580055","custom_field":"EBAY_EMS_90048630020055","protection_eligibility":"01"},"payer_info":{"account_id":"ZE5533HZPGMC6","email_address":"integration-test-buyer@airbyte.io","address_status":"Y","payer_status":"Y","payer_name":{"given_name":"test","surname":"buyer","alternate_full_name":"test buyer"},"country_code":"US"},"shipping_info":{"name":"Hello World","address":{"line1":"4thFloor","line2":"unit#34","city":"SAn Jose","state":"CA","country_code":"US","postal_code":"95131"}},"cart_info":{"item_details":[{"item_code":"1","item_name":"hat","item_description":"Brown color hat","item_quantity":"5","item_unit_price":{"currency_code":"USD","value":"3.00"},"item_amount":{"currency_code":"USD","value":"15.00"},"tax_amounts":[{"tax_amount":{"currency_code":"USD","value":"0.05"}}],"total_item_amount":{"currency_code":"USD","value":"15.05"},"invoice_number":"48787580055"},{"item_code":"product34","item_name":"handbag","item_description":"Black color hand bag","item_quantity":"1","item_unit_price":{"currency_code":"USD","value":"15.00"},"item_amount":{"currency_code":"USD","value":"15.00"},"tax_amounts":[{"tax_amount":{"currency_code":"USD","value":"0.02"}}],"total_item_amount":{"currency_code":"USD","value":"15.02"},"invoice_number":"48787580055"}]},"store_info":{},"auction_info":{},"incentive_info":{},"transaction_initiation_date":"2021-07-04T17:13:23+0000","transaction_id":"23N61105X92314351"},"emitted_at":1688463620839} +{"stream":"balances","data":{"balances":[{"currency":"USD","primary":true,"total_balance":{"currency_code":"USD","value":"173.64"},"available_balance":{"currency_code":"USD","value":"173.64"},"withheld_balance":{"currency_code":"USD","value":"0.00"}}],"account_id":"MDXWPD67GEP5W","as_of_time":"2021-07-03T00:00:00+00:00","last_refresh_time":"2023-07-04T04:59:59Z"},"emitted_at":1688463625694} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml index 8a062829a794..9966b577ccbd 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 - dockerImageTag: 0.1.13 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-paypal-transaction githubIssueLabel: source-paypal-transaction icon: paypal.svg @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/paypal-transaction tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt +++ b/airbyte-integrations/connectors/source-paypal-transaction/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py index ac08efaaf930..402f69640e62 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -13,7 +13,6 @@ "pytest~=6.1", "pytest-mock~=3.6", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index cfed43f13e47..5ec4c8f3ac3d 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -1,46 +1,49 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "balance": { - "type": ["null", "object"], - "properties": { - "currency": { - "type": ["null", "string"] - }, - "primary": { - "type": ["null", "boolean"] - }, - "total_balance": { - "type": ["null", "object"], - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] + "balances": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "currency": { + "type": ["null", "string"] + }, + "primary": { + "type": ["null", "boolean"] + }, + "total_balance": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } } - } - }, - "available_balance": { - "type": ["null", "object"], - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] + }, + "available_balance": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } } - } - }, - "withheld_balance": { - "type": ["null", "object"], - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] + }, + "withheld_balance": { + "type": ["null", "object"], + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json index 9734cd4dc0bd..8dcdd401518f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Paypal Transaction Search", "type": "object", - "required": ["start_date", "is_sandbox"], + "required": ["client_id", "client_secret", "start_date", "is_sandbox"], "additionalProperties": true, "properties": { "client_id": { diff --git a/airbyte-integrations/connectors/source-paystack/metadata.yaml b/airbyte-integrations/connectors/source-paystack/metadata.yaml index 7bcf632336fc..d65d615b940b 100644 --- a/airbyte-integrations/connectors/source-paystack/metadata.yaml +++ b/airbyte-integrations/connectors/source-paystack/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/paystack tags: - language:python + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-paystack/requirements.txt b/airbyte-integrations/connectors/source-paystack/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-paystack/requirements.txt +++ b/airbyte-integrations/connectors/source-paystack/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-paystack/setup.py b/airbyte-integrations/connectors/source-paystack/setup.py index 6258314a412e..946504314000 100644 --- a/airbyte-integrations/connectors/source-paystack/setup.py +++ b/airbyte-integrations/connectors/source-paystack/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "requests-mock"] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock"] setup( name="source_paystack", diff --git a/airbyte-integrations/connectors/source-pendo/metadata.yaml b/airbyte-integrations/connectors/source-pendo/metadata.yaml index 140b97e87601..c3f0ee28127a 100644 --- a/airbyte-integrations/connectors/source-pendo/metadata.yaml +++ b/airbyte-integrations/connectors/source-pendo/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pendo/requirements.txt b/airbyte-integrations/connectors/source-pendo/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pendo/requirements.txt +++ b/airbyte-integrations/connectors/source-pendo/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pendo/setup.py b/airbyte-integrations/connectors/source-pendo/setup.py index 74d5908cd397..8aa15183ac48 100644 --- a/airbyte-integrations/connectors/source-pendo/setup.py +++ b/airbyte-integrations/connectors/source-pendo/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-pendo/source_pendo/manifest.yaml b/airbyte-integrations/connectors/source-pendo/source_pendo/manifest.yaml index ab4e726e7cde..cdfce7956510 100644 --- a/airbyte-integrations/connectors/source-pendo/source_pendo/manifest.yaml +++ b/airbyte-integrations/connectors/source-pendo/source_pendo/manifest.yaml @@ -41,7 +41,7 @@ streams: items: type: - array - - 'null' + - "null" length: type: integer createdAt: @@ -194,7 +194,7 @@ streams: authenticator: type: ApiKeyAuthenticator header: x-pendo-integration-key - api_token: '{{ config[''api_key''] }}' + api_token: "{{ config['api_key'] }}" record_selector: type: RecordSelector extractor: @@ -238,7 +238,7 @@ streams: items: type: - array - - 'null' + - "null" length: type: integer createdAt: @@ -409,7 +409,7 @@ streams: authenticator: type: ApiKeyAuthenticator header: x-pendo-integration-key - api_token: '{{ config[''api_key''] }}' + api_token: "{{ config['api_key'] }}" record_selector: type: RecordSelector extractor: @@ -569,7 +569,7 @@ streams: type: type: string secondaryDateRange: - type: 'null' + type: "null" useSecondaryDateRange: type: boolean timeSeries: @@ -591,7 +591,7 @@ streams: period: type: string secondaryTimeSeries: - type: 'null' + type: "null" singleSource: type: object properties: @@ -666,7 +666,7 @@ streams: properties: count: type: - - 'null' + - "null" - string limit: type: integer @@ -734,7 +734,7 @@ streams: type: object properties: cat: - type: 'null' + type: "null" eval: type: object properties: @@ -888,7 +888,7 @@ streams: items: type: object properties: - '==': + "==": type: integer value: type: string @@ -1072,7 +1072,7 @@ streams: authenticator: type: ApiKeyAuthenticator header: x-pendo-integration-key - api_token: '{{ config[''api_key''] }}' + api_token: "{{ config['api_key'] }}" record_selector: type: RecordSelector extractor: @@ -1247,7 +1247,7 @@ streams: type: string advanceActions: anyOf: - - type: 'null' + - type: "null" - type: object properties: elementClick: @@ -1314,7 +1314,7 @@ streams: properties: visitors: anyOf: - - type: 'null' + - type: "null" - type: object properties: identified: @@ -1357,7 +1357,7 @@ streams: type: string badge: anyOf: - - type: 'null' + - type: "null" - type: object properties: name: @@ -1439,7 +1439,7 @@ streams: actions: type: array layoutId: - type: 'null' + type: "null" properties: type: array items: @@ -1466,13 +1466,13 @@ streams: defaultActions: type: object templateName: - type: 'null' + type: "null" widget: type: string actions: type: array layoutId: - type: 'null' + type: "null" properties: type: array items: @@ -1499,7 +1499,7 @@ streams: defaultActions: type: object templateName: - type: 'null' + type: "null" height: type: integer domJson: @@ -1723,7 +1723,7 @@ streams: type: integer activation: anyOf: - - type: 'null' + - type: "null" - type: object properties: event: @@ -1938,7 +1938,7 @@ streams: authenticator: type: ApiKeyAuthenticator header: x-pendo-integration-key - api_token: '{{ config[''api_key''] }}' + api_token: "{{ config['api_key'] }}" record_selector: type: RecordSelector extractor: @@ -2235,7 +2235,7 @@ streams: authenticator: type: ApiKeyAuthenticator header: x-pendo-integration-key - api_token: '{{ config[''api_key''] }}' + api_token: "{{ config['api_key'] }}" record_selector: type: RecordSelector extractor: @@ -2951,7 +2951,7 @@ streams: authenticator: type: ApiKeyAuthenticator header: x-pendo-integration-key - api_token: '{{ config[''api_key''] }}' + api_token: "{{ config['api_key'] }}" record_selector: type: RecordSelector extractor: diff --git a/airbyte-integrations/connectors/source-pendo/source_pendo/spec.yaml b/airbyte-integrations/connectors/source-pendo/source_pendo/spec.yaml index 7557fd01284f..2e348e837ddd 100644 --- a/airbyte-integrations/connectors/source-pendo/source_pendo/spec.yaml +++ b/airbyte-integrations/connectors/source-pendo/source_pendo/spec.yaml @@ -10,5 +10,3 @@ connectionSpecification: title: API Key airbyte_secret: true additionalProperties: true - - diff --git a/airbyte-integrations/connectors/source-persistiq/metadata.yaml b/airbyte-integrations/connectors/source-persistiq/metadata.yaml index 42585265bd77..b518b28eb03c 100644 --- a/airbyte-integrations/connectors/source-persistiq/metadata.yaml +++ b/airbyte-integrations/connectors/source-persistiq/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/persistiq tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-persistiq/requirements.txt b/airbyte-integrations/connectors/source-persistiq/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-persistiq/requirements.txt +++ b/airbyte-integrations/connectors/source-persistiq/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-persistiq/setup.py b/airbyte-integrations/connectors/source-persistiq/setup.py index 792701cecfb8..0c77971c1a87 100644 --- a/airbyte-integrations/connectors/source-persistiq/setup.py +++ b/airbyte-integrations/connectors/source-persistiq/setup.py @@ -10,10 +10,10 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "requests_mock==1.8.0", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-pexels-api/metadata.yaml b/airbyte-integrations/connectors/source-pexels-api/metadata.yaml index a3c1bd15b1b0..5a90b9a93769 100644 --- a/airbyte-integrations/connectors/source-pexels-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-pexels-api/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pexels-api/requirements.txt b/airbyte-integrations/connectors/source-pexels-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pexels-api/requirements.txt +++ b/airbyte-integrations/connectors/source-pexels-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pexels-api/setup.py b/airbyte-integrations/connectors/source-pexels-api/setup.py index 0e4d2dc0a00d..e45f600af446 100644 --- a/airbyte-integrations/connectors/source-pexels-api/setup.py +++ b/airbyte-integrations/connectors/source-pexels-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile index 87d0ef629808..a058bae4ec48 100644 --- a/airbyte-integrations/connectors/source-pinterest/Dockerfile +++ b/airbyte-integrations/connectors/source-pinterest/Dockerfile @@ -34,5 +34,5 @@ COPY source_pinterest ./source_pinterest ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.5.2 +LABEL io.airbyte.version=0.6.0 LABEL io.airbyte.name=airbyte/source-pinterest diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/abnormal_state.json index da6985d40d87..f545fe4f24ba 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/abnormal_state.json @@ -86,5 +86,16 @@ "name": "ad_analytics" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "DATE": "3021-06-09" + }, + "stream_descriptor": { + "name": "campaign_analytics_report" + } + } } ] diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog.json index 39da41eca065..687d314c3b5a 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog.json @@ -131,6 +131,17 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_analytics_report", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl index 8436b9b88293..1e2f686ffc27 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl @@ -1,3 +1,4 @@ -{"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1680356913507} +{"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375", "id": "666744057242074926"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1688461289470} {"stream": "boards", "data": {"media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "owner": {"username": "integrationtest0375"}, "created_at": "2021-06-08T09:37:18", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "id": "666743988523388559", "collaborator_count": 0, "follower_count": 2, "pin_count": 1, "privacy": "PUBLIC", "name": "business", "description": ""}, "emitted_at": 1680356853019} -{"stream": "board_pins", "data": {"id": "666743919837294988", "board_id": "666743988523388559", "alt_text": null, "note": "", "board_section_id": "5195034916661798218", "is_owner": true, "board_owner": {"username": "integrationtest0375"}, "description": "Data Integration", "media": {"media_type": "image", "images": {"150x150": {"width": 150, "height": 150, "url": "https://i.pinimg.com/150x150/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "400x300": {"width": 400, "height": 300, "url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "600x": {"width": 600, "height": 359, "url": "https://i.pinimg.com/600x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "1200x": {"width": 1200, "height": 718, "url": "https://i.pinimg.com/1200x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "originals": {"width": 1270, "height": 760, "url": "https://i.pinimg.com/originals/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}}}, "product_tags": [], "created_at": "2021-06-08T09:37:30", "dominant_color": "#cacafe", "parent_pin_id": null, "is_standard": true, "creative_type": "REGULAR", "title": "Airbyte", "link": "http://airbyte.io/"}, "emitted_at": 1684325353891} \ No newline at end of file +{"stream":"board_pins","data":{"link":"http://airbyte.io/","dominant_color":"#cacafe","media":{"media_type":"image","images":{"150x150":{"width":150,"height":150,"url":"https://i.pinimg.com/150x150/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"},"400x300":{"width":400,"height":300,"url":"https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"},"600x":{"width":600,"height":359,"url":"https://i.pinimg.com/600x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"},"1200x":{"width":1200,"height":718,"url":"https://i.pinimg.com/1200x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}}},"is_standard":true,"creative_type":"REGULAR","is_owner":true,"board_section_id":"5195034916661798218","id":"666743919837294988","description":"Data Integration","has_been_promoted":true,"created_at":"2021-06-08T09:37:30","note":"","product_tags":[],"alt_text":null,"title":"Airbyte","board_owner":{"username":"integrationtest0375"},"parent_pin_id":null,"board_id":"666743988523388559"},"emitted_at":1688054568572} +{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 750000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301} diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/sample_state.json index 04e756f960c1..b2d86bd409a9 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/sample_state.json @@ -86,5 +86,16 @@ "name": "ad_analytics" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "DATE": "2021-06-09" + }, + "stream_descriptor": { + "name": "campaign_analytics_report" + } + } } ] diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index 3e2fd2112437..cd0e897b6339 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 - dockerImageTag: 0.5.2 + dockerImageTag: 0.6.0 dockerRepository: airbyte/source-pinterest githubIssueLabel: source-pinterest icon: pinterest.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pinterest tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pinterest/requirements.txt b/airbyte-integrations/connectors/source-pinterest/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pinterest/requirements.txt +++ b/airbyte-integrations/connectors/source-pinterest/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py index 6cd581d5fff9..eac9cebacb4b 100644 --- a/airbyte-integrations/connectors/source-pinterest/setup.py +++ b/airbyte-integrations/connectors/source-pinterest/setup.py @@ -10,7 +10,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/__init__.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/__init__.py new file mode 100644 index 000000000000..d5baecddc2b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from .reports import CampaignAnalyticsReport + +__all__ = ["CampaignAnalyticsReport"] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/errors.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/errors.py new file mode 100644 index 000000000000..6033975cd0f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/errors.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +class RetryableException(Exception): + """Custom Exception Class for Retryable Exception""" + + pass + + +class ReportGenerationFailure(RetryableException): + """Custom Exception Class for Report Generation Failure""" + + pass + + +class ReportGenerationInProgress(RetryableException): + """Custom Exception Class for Report Generation In Progress""" + + pass + + +class ReportStatusError(RetryableException): + """Custom Exception Class for Report Status Error""" + + pass diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/models.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/models.py new file mode 100644 index 000000000000..23c1c2568d19 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/models.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel + + +class ReportStatus(str, Enum): + """Enum Class to define the possible status of a report""" + + DOES_NOT_EXIST = "DOES_NOT_EXIST" + EXPIRED = "EXPIRED" + FAILED = "FAILED" + CANCELLED = "CANCELLED" + FINISHED = "FINISHED" + IN_PROGRESS = "IN_PROGRESS" + + +class ReportStatusDetails(BaseModel): + """Model to capture details of the report status""" + + report_status: ReportStatus + url: Optional[str] + size: Optional[int] + + +class ReportInfo(BaseModel): + """Model to capture details of the report info""" + + report_status: ReportStatus + token: str + message: Optional[str] + metrics: Optional[List[dict]] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py new file mode 100644 index 000000000000..fcf5437cedd6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py @@ -0,0 +1,171 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from abc import abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from urllib.parse import urljoin + +import backoff +import requests +from airbyte_cdk.models import SyncMode +from source_pinterest.streams import PinterestAnalyticsStream +from source_pinterest.utils import get_analytics_columns + +from .errors import ReportGenerationFailure, ReportGenerationInProgress, ReportStatusError, RetryableException +from .models import ReportInfo, ReportStatus, ReportStatusDetails + + +class PinterestAnalyticsReportStream(PinterestAnalyticsStream): + """Class defining the stream of Pinterest Analytics Report + Details - https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report""" + + http_method = "POST" + report_wait_timeout = 180 + report_generation_maximum_retries = 5 + + @property + def window_in_days(self): + return 185 # Set window_in_days to 186 days date range + + @property + @abstractmethod + def level(self): + """:return: level on which report should be run""" + + @staticmethod + def _build_api_path(account_id: str) -> str: + """Build the API path for the given account id.""" + return f"ad_accounts/{account_id}/reports" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + """Get the path (i.e. URL) for the stream.""" + return self._build_api_path(stream_slice["parent"]["id"]) + + def _construct_request_body(self, start_date: str, end_date: str, granularity: str, columns: str) -> dict: + """Construct the body of the API request.""" + return { + "start_date": start_date, + "end_date": end_date, + "granularity": granularity, + "columns": columns.split(","), + "level": self.level, + } + + def request_body_json(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: + """Return the body of the API request in JSON format.""" + return self._construct_request_body(stream_slice["start_date"], stream_slice["end_date"], self.granularity, get_analytics_columns()) + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """Return the request parameters.""" + return {} + + def backoff_max_time(func): + def wrapped(self, *args, **kwargs): + return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout * 60, interval=10)(func)( + self, *args, **kwargs + ) + + return wrapped + + def backoff_max_tries(func): + def wrapped(self, *args, **kwargs): + return backoff.on_exception(backoff.expo, ReportGenerationFailure, max_tries=self.report_generation_maximum_retries)(func)( + self, *args, **kwargs + ) + + return wrapped + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + """Read the records from the stream.""" + report_infos = self._init_reports(super().read_records(sync_mode, cursor_field, stream_slice, stream_state)) + self._try_read_records(report_infos, stream_slice) + + for report_info in report_infos: + metrics = report_info.metrics + for campaign_id, records in metrics.items(): + self.logger.info(f"Reports for campaign id: {campaign_id}:") + yield from records + + @backoff_max_time + def _try_read_records(self, report_infos, stream_slice): + """Try to read the records and raise appropriate exceptions in case of failure or in-progress status.""" + incomplete_report_infos = self._incomplete_report_infos(report_infos) + for report_info in incomplete_report_infos: + report_status, report_url = self._verify_report_status(report_info, stream_slice) + report_info.report_status = report_status + if report_status in {ReportStatus.DOES_NOT_EXIST, ReportStatus.EXPIRED, ReportStatus.FAILED, ReportStatus.CANCELLED}: + message = "Report generation failed." + raise ReportGenerationFailure(message) + elif report_status == ReportStatus.FINISHED: + try: + report_info.metrics = self._fetch_report_data(report_url) + except requests.HTTPError as error: + raise ReportGenerationFailure(error) + + pending_report_status = [report_info for report_info in report_infos if report_info.report_status != ReportStatus.FINISHED] + + if len(pending_report_status) > 0: + message = "Report generation in progress." + raise ReportGenerationInProgress(message) + + def _incomplete_report_infos(self, report_infos): + """Return the report infos which are not yet finished.""" + return [r for r in report_infos if r.report_status != ReportStatus.FINISHED] + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + """Parse the API response.""" + yield response.json() + + @backoff_max_tries + def _init_reports(self, init_reports) -> List[ReportInfo]: + """Initialize the reports and return them as a list.""" + report_infos = [] + for init_report in init_reports: + status = ReportInfo.parse_raw(json.dumps(init_report)) + report_infos.append( + ReportInfo( + token=status.token, + report_status=ReportStatus.IN_PROGRESS, + metrics=[], + ) + ) + self.logger.info("Initiated successfully.") + return report_infos + + def _http_get(self, url, params=None, headers=None): + """Make a GET request to the given URL and return the response as a JSON.""" + response = self._session.get(url, params=params, headers=headers) + response.raise_for_status() + return response.json() + + def _verify_report_status(self, report: dict, stream_slice: Mapping[str, Any]) -> tuple: + """Verify the report status and return it along with the report URL.""" + api_path = self._build_api_path(stream_slice["parent"]["id"]) + response_data = self._http_get( + urljoin(self.url_base, api_path), params={"token": report.token}, headers=self.authenticator.get_auth_header() + ) + try: + report_status = ReportStatusDetails.parse_raw(json.dumps(response_data)) + except ValueError as error: + raise ReportStatusError(error) + return report_status.report_status, report_status.url + + def _fetch_report_data(self, url: str) -> dict: + """Fetch the report data from the given URL.""" + return self._http_get(url) + + +class CampaignAnalyticsReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "CAMPAIGN" diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json index 1c0e3c7893f9..a996db16e4ec 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json @@ -12,6 +12,9 @@ "owner": { "type": ["null", "object"], "properties": { + "id": { + "type": ["null", "string"] + }, "username": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json index 98554dd1c23a..5feb51438a6f 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json @@ -11,7 +11,14 @@ }, "creative_type": { "type": ["null", "string"], - "enum": ["REGULAR", "VIDEO", "CAROUSEL", "MAX_VIDEO", "SHOP_THE_PIN", "IDEA"] + "enum": [ + "REGULAR", + "VIDEO", + "CAROUSEL", + "MAX_VIDEO", + "SHOP_THE_PIN", + "IDEA" + ] }, "is_standard": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json new file mode 100644 index 000000000000..18eec20efafe --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json @@ -0,0 +1,346 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "DATE": { + "type": ["null", "string"], + "format": "date" + }, + "ADVERTISER_ID": { + "type": ["null", "number"] + }, + "AD_ACCOUNT_ID": { + "type": ["string"] + }, + "AD_ID": { + "type": ["null", "string"] + }, + "AD_GROUP_ENTITY_STATUS": { + "type": ["null", "string"] + }, + "AD_GROUP_ID": { + "type": ["null", "string"] + }, + "CAMPAIGN_DAILY_SPEND_CAP": { + "type": ["null", "number"] + }, + "CAMPAIGN_ENTITY_STATUS": { + "type": ["null", "string"] + }, + "CAMPAIGN_ID": { + "type": ["null", "number"] + }, + "CAMPAIGN_LIFETIME_SPEND_CAP": { + "type": ["null", "number"] + }, + "CAMPAIGN_NAME": { + "type": ["null", "string"] + }, + "CHECKOUT_ROAS": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_1": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_1_GROSS": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_2": { + "type": ["null", "number"] + }, + "CPC_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "CPM_IN_DOLLAR": { + "type": ["null", "number"] + }, + "CPM_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "CTR": { + "type": ["null", "number"] + }, + "CTR_2": { + "type": ["null", "number"] + }, + "ECPCV_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPCV_P95_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPC_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPC_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "ECPE_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPM_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "ECPV_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECTR": { + "type": ["null", "number"] + }, + "EENGAGEMENT_RATE": { + "type": ["null", "number"] + }, + "ENGAGEMENT_1": { + "type": ["null", "number"] + }, + "ENGAGEMENT_2": { + "type": ["null", "number"] + }, + "ENGAGEMENT_RATE": { + "type": ["null", "number"] + }, + "IDEA_PIN_PRODUCT_TAG_VISIT_1": { + "type": ["null", "number"] + }, + "IDEA_PIN_PRODUCT_TAG_VISIT_2": { + "type": ["null", "number"] + }, + "IMPRESSION_1": { + "type": ["null", "number"] + }, + "IMPRESSION_1_GROSS": { + "type": ["null", "number"] + }, + "IMPRESSION_2": { + "type": ["null", "number"] + }, + "INAPP_CHECKOUT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "OUTBOUND_CLICK_1": { + "type": ["null", "number"] + }, + "OUTBOUND_CLICK_2": { + "type": ["null", "number"] + }, + "PAGE_VISIT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "PAGE_VISIT_ROAS": { + "type": ["null", "number"] + }, + "PAID_IMPRESSION": { + "type": ["null", "number"] + }, + "PIN_ID": { + "type": ["null", "number"] + }, + "PIN_PROMOTION_ID": { + "type": ["null", "number"] + }, + "REPIN_1": { + "type": ["null", "number"] + }, + "REPIN_2": { + "type": ["null", "number"] + }, + "REPIN_RATE": { + "type": ["null", "number"] + }, + "SPEND_IN_DOLLAR": { + "type": ["null", "number"] + }, + "SPEND_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CLICKTHROUGH": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_ADD_TO_CART": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CONVERSIONS": { + "type": ["null", "number"] + }, + "TOTAL_CUSTOM": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_IDEA_PIN_PRODUCT_TAG_VISIT": { + "type": ["null", "number"] + }, + "TOTAL_IMPRESSION_FREQUENCY": { + "type": ["null", "number"] + }, + "TOTAL_IMPRESSION_USER": { + "type": ["null", "number"] + }, + "TOTAL_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_OFFLINE_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_PAGE_VISIT": { + "type": ["null", "number"] + }, + "TOTAL_REPIN_RATE": { + "type": ["null", "number"] + }, + "TOTAL_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_3SEC_VIEWS": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_AVG_WATCHTIME_IN_SECOND": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_MRC_VIEWS": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P0_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P100_COMPLETE": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P25_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P50_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P75_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P95_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_ADD_TO_CART": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CLICK_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_ENGAGEMENT_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_SESSIONS": { + "type": ["null", "number"] + }, + "TOTAL_WEB_VIEW_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "VIDEO_3SEC_VIEWS_2": { + "type": ["null", "number"] + }, + "VIDEO_LENGTH": { + "type": ["null", "number"] + }, + "VIDEO_MRC_VIEWS_2": { + "type": ["null", "number"] + }, + "VIDEO_P0_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P100_COMPLETE_2": { + "type": ["null", "number"] + }, + "VIDEO_P25_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P50_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P75_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P95_COMBINED_2": { + "type": ["null", "number"] + }, + "WEB_CHECKOUT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "WEB_CHECKOUT_ROAS": { + "type": ["null", "number"] + }, + "WEB_SESSIONS_1": { + "type": ["null", "number"] + }, + "WEB_SESSIONS_2": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index 994fae6448cd..e110f15f339b 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -2,346 +2,42 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -from abc import ABC +import copy from base64 import standard_b64encode -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, List, Mapping, Tuple import pendulum import requests -from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.models import FailureType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException - -from .utils import get_analytics_columns, to_datetime_str - -# For Pinterest analytics streams rate limit is 300 calls per day / per user. -# once hit - response would contain `code` property with int. -MAX_RATE_LIMIT_CODE = 8 - - -class PinterestStream(HttpStream, ABC): - url_base = "https://api.pinterest.com/v5/" - primary_key = "id" - data_fields = ["items"] - raise_on_http_errors = True - max_rate_limit_exceeded = False - transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, config: Mapping[str, Any]): - super().__init__(authenticator=config["authenticator"]) - self.config = config - - @property - def start_date(self): - return self.config["start_date"] - - @property - def window_in_days(self): - return 30 # Set window_in_days to 30 days date range - - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page = response.json().get("bookmark", {}) if self.data_fields else {} - - if next_page: - return {"bookmark": next_page} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return next_page_token or {} - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - """ - Parsing response data with respect to Rate Limits. - """ - data = response.json() - - if not self.max_rate_limit_exceeded: - for data_field in self.data_fields: - data = data.get(data_field, []) - - for record in data: - yield record - - def should_retry(self, response: requests.Response) -> bool: - if isinstance(response.json(), dict): - self.max_rate_limit_exceeded = response.json().get("code", 0) == MAX_RATE_LIMIT_CODE - # when max rate limit exceeded, we should skip the stream. - if response.status_code == requests.codes.too_many_requests and self.max_rate_limit_exceeded: - self.logger.error(f"For stream {self.name} Max Rate Limit exceeded.") - setattr(self, "raise_on_http_errors", False) - return 500 <= response.status_code < 600 - - def backoff_time(self, response: requests.Response) -> Optional[float]: - if response.status_code == requests.codes.too_many_requests: - self.logger.error(f"For stream {self.name} rate limit exceeded.") - return float(response.headers.get("X-RateLimit-Reset", 0)) - - -class PinterestSubStream(HttpSubStream): - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream_slices = self.parent.stream_slices( - sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state - ) - # iterate over all parent stream_slices - for stream_slice in parent_stream_slices: - parent_records = self.parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) - - # iterate over all parent records with current stream_slice - for record in parent_records: - yield {"parent": record, "sub_parent": stream_slice} - - -class Boards(PinterestStream): - use_cache = True - - def path(self, **kwargs) -> str: - return "boards" - - -class AdAccounts(PinterestStream): - use_cache = True - - def path(self, **kwargs) -> str: - return "ad_accounts" - - -class BoardSections(PinterestSubStream, PinterestStream): - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['parent']['id']}/sections" - - -class BoardPins(PinterestSubStream, PinterestStream): - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['parent']['id']}/pins" - - -class BoardSectionPins(PinterestSubStream, PinterestStream): - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['sub_parent']['parent']['id']}/sections/{stream_slice['parent']['id']}/pins" - - -class IncrementalPinterestStream(PinterestStream, ABC): - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - default_value = self.start_date.format("YYYY-MM-DD") - latest_state = latest_record.get(self.cursor_field, default_value) - current_state = current_stream_state.get(self.cursor_field, default_value) - latest_state_is_numeric = isinstance(latest_state, int) or isinstance(latest_state, float) - - if latest_state_is_numeric and isinstance(current_state, str): - current_state = datetime.strptime(current_state, "%Y-%m-%d").timestamp() - - return {self.cursor_field: max(latest_state, current_state)} - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, any]]]: - """ - Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. - Returns list of dict, example: [{ - "start_date": "2020-01-01", - "end_date": "2021-01-02" - }, - { - "start_date": "2020-01-03", - "end_date": "2021-01-04" - }, - ...] - """ - - start_date = self.start_date - end_date = pendulum.now() - - # determine stream_state, if no stream_state we use start_date - if stream_state: - state = stream_state.get(self.cursor_field) - - state_is_timestamp = isinstance(state, int) or isinstance(state, float) - if state_is_timestamp: - state = str(datetime.fromtimestamp(state).date()) - - start_date = pendulum.parse(state) - - # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future - start_date = min(start_date, end_date) - date_slices = [] - - while start_date < end_date: - # the amount of days for each data-chunk beginning from start_date - end_date_slice = start_date.add(days=self.window_in_days) - date_slices.append({"start_date": to_datetime_str(start_date), "end_date": to_datetime_str(end_date_slice)}) - - # add 1 day for start next slice from next day and not duplicate data from previous slice end date. - start_date = end_date_slice.add(days=1) - - return date_slices - - -class IncrementalPinterestSubStream(IncrementalPinterestStream): - cursor_field = "updated_time" - - def __init__(self, parent: HttpStream, with_data_slices: bool = True, **kwargs): - super().__init__(**kwargs) - self.parent = parent - self.with_data_slices = with_data_slices - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - date_slices = super().stream_slices(sync_mode, cursor_field, stream_state) if self.with_data_slices else [{}] - parents_slices = PinterestSubStream.stream_slices(self, sync_mode, cursor_field, stream_state) if self.parent else [{}] - - for parents_slice in parents_slices: - for date_slice in date_slices: - parents_slice.update(date_slice) - - yield parents_slice - - -class PinterestAnalyticsStream(IncrementalPinterestSubStream): - primary_key = None - cursor_field = "DATE" - data_fields = [] - granularity = "DAY" - analytics_target_ids = None - - def lookback_date_limt_reached(self, response: requests.Response) -> bool: - """ - After few consecutive requests analytics API return bad request error - with 'You can only get data from the last 90 days' error message. - But with next request all working good. So, we wait 1 sec and - request again if we get this issue. - """ - - if isinstance(response.json(), dict): - return response.json().get("code", 0) and response.status_code == 400 - return False - - def should_retry(self, response: requests.Response) -> bool: - return super().should_retry(response) or self.lookback_date_limt_reached(response) - - def backoff_time(self, response: requests.Response) -> Optional[float]: - if self.lookback_date_limt_reached(response): - return 1 - return super().backoff_time(response) - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params.update( - { - "start_date": stream_slice["start_date"], - "end_date": stream_slice["end_date"], - "granularity": self.granularity, - "columns": get_analytics_columns(), - } - ) - - if self.analytics_target_ids: - params.update({self.analytics_target_ids: stream_slice["parent"]["id"]}) - - return params - - -class ServerSideFilterStream(IncrementalPinterestSubStream): - def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable: - """ - Endpoint does not provide query filtering params, but they provide us - cursor field in most cases, so we used that as incremental filtering - during the parsing. - """ - - if not stream_state or record[self.cursor_field] >= stream_state.get(self.cursor_field): - yield record - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - for record in super().parse_response(response, stream_state, **kwargs): - yield from self.filter_by_state(stream_state=stream_state, record=record) - - -class UserAccountAnalytics(PinterestAnalyticsStream): - data_fields = ["all", "daily_metrics"] - cursor_field = "date" - - def path(self, **kwargs) -> str: - return "user_account/analytics" - - -class AdAccountAnalytics(PinterestAnalyticsStream): - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"ad_accounts/{stream_slice['parent']['id']}/analytics" - - -class Campaigns(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): - super().__init__(parent, with_data_slices, **kwargs) - self.status_filter = status_filter - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" - return f"ad_accounts/{stream_slice['parent']['id']}/campaigns{params}" - - -class CampaignAnalytics(PinterestAnalyticsStream): - analytics_target_ids = "campaign_ids" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"ad_accounts/{stream_slice['sub_parent']['parent']['id']}/campaigns/analytics" - - -class AdGroups(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): - super().__init__(parent, with_data_slices, **kwargs) - self.status_filter = status_filter - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" - return f"ad_accounts/{stream_slice['parent']['id']}/ad_groups{params}" - - -class AdGroupAnalytics(PinterestAnalyticsStream): - analytics_target_ids = "ad_group_ids" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"ad_accounts/{stream_slice['sub_parent']['parent']['id']}/ad_groups/analytics" - - -class Ads(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): - super().__init__(parent, with_data_slices, **kwargs) - self.status_filter = status_filter - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" - return f"ad_accounts/{stream_slice['parent']['id']}/ads{params}" - - -class AdAnalytics(PinterestAnalyticsStream): - analytics_target_ids = "ad_ids" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"ad_accounts/{stream_slice['sub_parent']['parent']['id']}/ads/analytics" +from source_pinterest.reports import CampaignAnalyticsReport + +from .streams import ( + AdAccountAnalytics, + AdAccounts, + AdAnalytics, + AdGroupAnalytics, + AdGroups, + Ads, + BoardPins, + Boards, + BoardSectionPins, + BoardSections, + CampaignAnalytics, + Campaigns, + PinterestStream, + UserAccountAnalytics, +) class SourcePinterest(AbstractSource): - def _validate_and_transform(self, config: Mapping[str, Any]): + def _validate_and_transform(self, config: Mapping[str, Any], amount_of_days_allowed_for_lookup: int = 89): + config = copy.deepcopy(config) today = pendulum.today() - AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP = 89 - latest_date_allowed_by_api = today.subtract(days=AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP) + latest_date_allowed_by_api = today.subtract(days=amount_of_days_allowed_for_lookup) start_date = config["start_date"] if not start_date: @@ -356,7 +52,7 @@ def _validate_and_transform(self, config: Mapping[str, Any]): internal_message=message, failure_type=FailureType.config_error, ) - if (today - config["start_date"]).days > AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP: + if (today - config["start_date"]).days > amount_of_days_allowed_for_lookup: config["start_date"] = latest_date_allowed_by_api return config @@ -389,8 +85,9 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - config = self._validate_and_transform(config) config["authenticator"] = self.get_authenticator(config) + report_config = self._validate_and_transform(config, amount_of_days_allowed_for_lookup=913) + config = self._validate_and_transform(config) status = ",".join(config.get("status")) if config.get("status") else None return [ AdAccountAnalytics(AdAccounts(config), config=config), @@ -404,6 +101,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: BoardSections(Boards(config), config=config), Boards(config), CampaignAnalytics(Campaigns(AdAccounts(config), with_data_slices=False, config=config), config=config), + CampaignAnalyticsReport(AdAccounts(report_config), config=report_config), Campaigns(AdAccounts(config), status_filter=status, config=config), UserAccountAnalytics(None, config=config), ] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py new file mode 100644 index 000000000000..74add04ca3df --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py @@ -0,0 +1,333 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from datetime import datetime +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer + +from .utils import get_analytics_columns, to_datetime_str + +# For Pinterest analytics streams rate limit is 300 calls per day / per user. +# once hit - response would contain `code` property with int. +MAX_RATE_LIMIT_CODE = 8 + + +class PinterestStream(HttpStream, ABC): + url_base = "https://api.pinterest.com/v5/" + primary_key = "id" + data_fields = ["items"] + raise_on_http_errors = True + max_rate_limit_exceeded = False + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + + def __init__(self, config: Mapping[str, Any]): + super().__init__(authenticator=config["authenticator"]) + self.config = config + + @property + def start_date(self): + return self.config["start_date"] + + @property + def window_in_days(self): + return 30 # Set window_in_days to 30 days date range + + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return None + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page = response.json().get("bookmark", {}) if self.data_fields else {} + + if next_page: + return {"bookmark": next_page} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return next_page_token or {} + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + """ + Parsing response data with respect to Rate Limits. + """ + data = response.json() + + if not self.max_rate_limit_exceeded: + for data_field in self.data_fields: + data = data.get(data_field, []) + + for record in data: + yield record + + def should_retry(self, response: requests.Response) -> bool: + if isinstance(response.json(), dict): + self.max_rate_limit_exceeded = response.json().get("code", 0) == MAX_RATE_LIMIT_CODE + # when max rate limit exceeded, we should skip the stream. + if response.status_code == requests.codes.too_many_requests and self.max_rate_limit_exceeded: + self.logger.error(f"For stream {self.name} Max Rate Limit exceeded.") + setattr(self, "raise_on_http_errors", False) + return 500 <= response.status_code < 600 + + def backoff_time(self, response: requests.Response) -> Optional[float]: + if response.status_code == requests.codes.too_many_requests: + self.logger.error(f"For stream {self.name} rate limit exceeded.") + return float(response.headers.get("X-RateLimit-Reset", 0)) + + +class PinterestSubStream(HttpSubStream): + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + # iterate over all parent stream_slices + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + + # iterate over all parent records with current stream_slice + for record in parent_records: + yield {"parent": record, "sub_parent": stream_slice} + + +class Boards(PinterestStream): + use_cache = True + + def path(self, **kwargs) -> str: + return "boards" + + +class AdAccounts(PinterestStream): + use_cache = True + + def path(self, **kwargs) -> str: + return "ad_accounts" + + +class BoardSections(PinterestSubStream, PinterestStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"boards/{stream_slice['parent']['id']}/sections" + + +class BoardPins(PinterestSubStream, PinterestStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"boards/{stream_slice['parent']['id']}/pins" + + +class BoardSectionPins(PinterestSubStream, PinterestStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"boards/{stream_slice['sub_parent']['parent']['id']}/sections/{stream_slice['parent']['id']}/pins" + + +class IncrementalPinterestStream(PinterestStream, ABC): + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + default_value = self.start_date.format("YYYY-MM-DD") + latest_state = latest_record.get(self.cursor_field, default_value) + current_state = current_stream_state.get(self.cursor_field, default_value) + latest_state_is_numeric = isinstance(latest_state, int) or isinstance(latest_state, float) + + if latest_state_is_numeric and isinstance(current_state, str): + current_state = datetime.strptime(current_state, "%Y-%m-%d").timestamp() + + return {self.cursor_field: max(latest_state, current_state)} + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, any]]]: + """ + Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. + Returns list of dict, example: [{ + "start_date": "2020-01-01", + "end_date": "2021-01-02" + }, + { + "start_date": "2020-01-03", + "end_date": "2021-01-04" + }, + ...] + """ + + start_date = self.start_date + end_date = pendulum.now() + + # determine stream_state, if no stream_state we use start_date + if stream_state: + state = stream_state.get(self.cursor_field) + + state_is_timestamp = isinstance(state, int) or isinstance(state, float) + if state_is_timestamp: + state = str(datetime.fromtimestamp(state).date()) + + start_date = pendulum.parse(state) + + # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future + start_date = min(start_date, end_date) + date_slices = [] + + while start_date < end_date: + # the amount of days for each data-chunk beginning from start_date + end_date_slice = ( + end_date if end_date.subtract(days=self.window_in_days) < start_date else start_date.add(days=self.window_in_days) + ) + date_slices.append({"start_date": to_datetime_str(start_date), "end_date": to_datetime_str(end_date_slice)}) + + # add 1 day for start next slice from next day and not duplicate data from previous slice end date. + start_date = end_date_slice.add(days=1) + + return date_slices + + +class IncrementalPinterestSubStream(IncrementalPinterestStream): + cursor_field = "updated_time" + + def __init__(self, parent: HttpStream, with_data_slices: bool = True, **kwargs): + super().__init__(**kwargs) + self.parent = parent + self.with_data_slices = with_data_slices + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + date_slices = super().stream_slices(sync_mode, cursor_field, stream_state) if self.with_data_slices else [{}] + parents_slices = PinterestSubStream.stream_slices(self, sync_mode, cursor_field, stream_state) if self.parent else [{}] + + for parents_slice in parents_slices: + for date_slice in date_slices: + parents_slice.update(date_slice) + + yield parents_slice + + +class PinterestAnalyticsStream(IncrementalPinterestSubStream): + primary_key = None + cursor_field = "DATE" + data_fields = [] + granularity = "DAY" + analytics_target_ids = None + + def lookback_date_limt_reached(self, response: requests.Response) -> bool: + """ + After few consecutive requests analytics API return bad request error + with 'You can only get data from the last 90 days' error message. + But with next request all working good. So, we wait 1 sec and + request again if we get this issue. + """ + + if isinstance(response.json(), dict): + return response.json().get("code", 0) and response.status_code == 400 + return False + + def should_retry(self, response: requests.Response) -> bool: + return super().should_retry(response) or self.lookback_date_limt_reached(response) + + def backoff_time(self, response: requests.Response) -> Optional[float]: + if self.lookback_date_limt_reached(response): + return 1 + return super().backoff_time(response) + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params.update( + { + "start_date": stream_slice["start_date"], + "end_date": stream_slice["end_date"], + "granularity": self.granularity, + "columns": get_analytics_columns(), + } + ) + + if self.analytics_target_ids: + params.update({self.analytics_target_ids: stream_slice["parent"]["id"]}) + + return params + + +class ServerSideFilterStream(IncrementalPinterestSubStream): + def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable: + """ + Endpoint does not provide query filtering params, but they provide us + cursor field in most cases, so we used that as incremental filtering + during the parsing. + """ + + if not stream_state or record[self.cursor_field] >= stream_state.get(self.cursor_field): + yield record + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response, stream_state, **kwargs): + yield from self.filter_by_state(stream_state=stream_state, record=record) + + +class UserAccountAnalytics(PinterestAnalyticsStream): + data_fields = ["all", "daily_metrics"] + cursor_field = "date" + + def path(self, **kwargs) -> str: + return "user_account/analytics" + + +class AdAccountAnalytics(PinterestAnalyticsStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/analytics" + + +class Campaigns(ServerSideFilterStream): + def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + super().__init__(parent, with_data_slices, **kwargs) + self.status_filter = status_filter + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" + return f"ad_accounts/{stream_slice['parent']['id']}/campaigns{params}" + + +class CampaignAnalytics(PinterestAnalyticsStream): + analytics_target_ids = "campaign_ids" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['sub_parent']['parent']['id']}/campaigns/analytics" + + +class AdGroups(ServerSideFilterStream): + def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + super().__init__(parent, with_data_slices, **kwargs) + self.status_filter = status_filter + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" + return f"ad_accounts/{stream_slice['parent']['id']}/ad_groups{params}" + + +class AdGroupAnalytics(PinterestAnalyticsStream): + analytics_target_ids = "ad_group_ids" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['sub_parent']['parent']['id']}/ad_groups/analytics" + + +class Ads(ServerSideFilterStream): + def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + super().__init__(parent, with_data_slices, **kwargs) + self.status_filter = status_filter + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" + return f"ad_accounts/{stream_slice['parent']['id']}/ads{params}" + + +class AdAnalytics(PinterestAnalyticsStream): + analytics_target_ids = "ad_ids" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['sub_parent']['parent']['id']}/ads/analytics" diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py index acb21c0c364f..b929d7e18be0 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock from pytest import fixture +from source_pinterest.reports import CampaignAnalyticsReport @fixture @@ -62,3 +63,17 @@ def test_response_filter(test_record_filter): response = MagicMock() response.json.return_value = test_record_filter return response + + +@fixture +def analytics_report_stream(): + return CampaignAnalyticsReport(parent=None, config=MagicMock()) + + +@fixture +def date_range(): + return { + 'start_date': '2023-01-01', + 'end_date': '2023-01-31', + 'parent': {'id': '123'} + } diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py index 42a1ffb339f8..ca34553b28b6 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_incremental_streams.py @@ -9,7 +9,7 @@ import pytest from airbyte_cdk.models import SyncMode from pytest import fixture -from source_pinterest.source import AdAccountAnalytics, Campaigns, IncrementalPinterestSubStream +from source_pinterest.streams import AdAccountAnalytics, Campaigns, IncrementalPinterestSubStream @fixture diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py new file mode 100644 index 000000000000..75d716c2fc03 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import responses +from source_pinterest.utils import get_analytics_columns + + +@responses.activate +def test_request_body_json(analytics_report_stream, date_range): + granularity = 'DAY' + columns = get_analytics_columns() + + expected_body = { + 'start_date': date_range['start_date'], + 'end_date': date_range['end_date'], + 'granularity': granularity, + 'columns': columns.split(','), + 'level': analytics_report_stream.level, + } + + body = analytics_report_stream.request_body_json(date_range) + assert body == expected_body + + +@responses.activate +def test_read_records(analytics_report_stream, date_range): + report_download_url = 'https://download.report' + report_request_url = 'https://api.pinterest.com/v5/ad_accounts/123/reports' + + final_report_status = { + 'report_status': 'FINISHED', + 'url': report_download_url + } + + initial_response = { + 'report_status': "IN_PROGRESS", + 'token': 'token', + 'message': '' + } + + final_response = {"campaign_id": [{"metric": 1}]} + + responses.add(responses.POST, report_request_url, json=initial_response) + responses.add(responses.GET, report_request_url, json=final_report_status, status=200) + responses.add(responses.GET, report_download_url, json=final_response, status=200) + + sync_mode = 'full_refresh' + cursor_field = ['last_updated'] + stream_state = { + 'start_date': '2023-01-01', + 'end_date': '2023-01-31', + } + + records = analytics_report_stream.read_records(sync_mode, cursor_field, date_range, stream_state) + expected_record = {"metric": 1} + + assert next(records) == expected_record + assert len(responses.calls) == 3 + assert responses.calls[0].request.url == report_request_url diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index 0a474669afec..56b16f6cca40 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -44,7 +44,7 @@ def test_streams(test_config): setup_responses() source = SourcePinterest() streams = source.streams(test_config) - expected_streams_number = 13 + expected_streams_number = 14 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index 004482465128..8c26fffe401e 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -7,7 +7,7 @@ import pytest import requests -from source_pinterest.source import ( +from source_pinterest.streams import ( AdAccountAnalytics, AdAccounts, AdAnalytics, diff --git a/airbyte-integrations/connectors/source-pipedrive/Dockerfile b/airbyte-integrations/connectors/source-pipedrive/Dockerfile index f15258ee4f2e..9ef2f33510dd 100644 --- a/airbyte-integrations/connectors/source-pipedrive/Dockerfile +++ b/airbyte-integrations/connectors/source-pipedrive/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-pipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml index 35d9c5d142c6..928821e53d54 100644 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -3,41 +3,71 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_pipedrive/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.14" + - spec_path: "source_pipedrive/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/old_config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/old_config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.13" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 0.1.19 basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - ignored_fields: - users: - - name: modified - bypass_reason: "constantly increasing date-time field" - - name: last_login - bypass_reason: "constantly increasing date-time field" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + ignored_fields: + users: + - name: modified + bypass_reason: "constantly increasing date-time field" + - name: last_login + bypass_reason: "constantly increasing date-time field" + deal_fields: + - name: show_in_pipelines + bypass_reason: "Unstable data" + - name: important_flag + bypass_reason: "Unstable data" + - name: pipeline_ids + bypass_reason: "Unstable data" + - name: update_time + bypass_reason: "Unstable data" + - name: last_updated_by_user_id + bypass_reason: "Unstable data" + organization_fields: + - name: update_time + bypass_reason: "Unstable data" + - name: important_flag + bypass_reason: "Unstable data" + - name: last_updated_by_user_id + bypass_reason: "Unstable data" + person_fields: + - name: update_time + bypass_reason: "Unstable data" + - name: important_flag + bypass_reason: "Unstable data" + - name: last_updated_by_user_id + bypass_reason: "Unstable data" + product_fields: + - name: update_time + bypass_reason: "Unstable data" + - name: important_flag + bypass_reason: "Unstable data" + - name: last_updated_by_user_id + bypass_reason: "Unstable data" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl index 1d16950607e4..c593d1cc0ba5 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl @@ -1,62 +1,62 @@ -{"stream": "activities", "data": {"id": 1, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "task", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-06", "due_time": "", "duration": "", "busy_flag": true, "add_time": "2021-07-06 15:02:04", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Task1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 10, "lead_id": null, "active_flag": true, "update_time": "2021-07-06 15:02:03", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy)", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal10@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Task", "lead": null}, "emitted_at": 1683115291428} -{"stream": "activities", "data": {"id": 3, "company_id": 7780468, "user_id": 11884360, "done": true, "type": "deadline", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-28", "due_time": "15:15", "duration": "00:30", "busy_flag": true, "add_time": "2021-07-06 15:03:31", "marked_as_done_time": "2023-02-22 08:25:45", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Deadline1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 1, "lead_id": null, "active_flag": true, "update_time": "2023-02-22 08:25:49", "update_user_id": 11884360, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "New note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal1@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Deadline", "lead": null}, "emitted_at": 1683115291429} -{"stream": "activities", "data": {"id": 7, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "call", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2023-02-22", "due_time": "12:30", "duration": "01:00", "busy_flag": true, "add_time": "2023-02-22 08:52:52", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": null, "subject": "Call", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 3, "person_id": 7, "deal_id": null, "lead_id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "active_flag": true, "update_time": "2023-02-22 08:52:52", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Test call
    ", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 7, "primary_flag": true}, {"person_id": 9, "primary_flag": false}], "series": null, "is_recurring": null, "org_name": "Test Organization 3", "person_name": "User6 Sample", "deal_title": null, "lead_title": "Test Organization 3 lead", "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": null, "assigned_to_user_id": 11884360, "type_name": "Call", "lead": {"id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "title": "Test Organization 3 lead", "labels": "aecece60-c069-11eb-93bf-b59c4f1731e6", "source": "Manually created", "owner_id": 11884360, "creator_user_id": 11884360, "deal_id": null, "related_person_id": 4, "related_org_id": 2, "person_name": null, "person_phone": null, "person_email": null, "org_name": null, "org_address": null, "active_flag": true, "add_time": "2023-02-22 08:52:06.047", "update_time": "2023-02-22 11:44:31.718", "archive_time": null, "seen": true, "deal_value": 1200, "deal_currency": "USD", "next_activity_id": 7, "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "visible_to": 3, "source_reference_id": null, "user": null, "deal_expected_close_date": "2023-05-15", "next_activity_status": null, "next_activity_datetime": null, "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null}}, "emitted_at": 1683115291429} -{"stream": "activity_fields", "data": {"id": 1, "key": "id", "name": "ID", "order_nr": 1, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115292598} -{"stream": "activity_fields", "data": {"id": 2, "key": "subject", "name": "Subject", "order_nr": 2, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115292599} -{"stream": "activity_fields", "data": {"id": 3, "key": "type", "name": "Type", "order_nr": 3, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "options": [{"id": "call", "label": "Call"}, {"id": "meeting", "label": "Meeting"}, {"id": "task", "label": "Task"}, {"id": "deadline", "label": "Deadline"}, {"id": "email", "label": "Email"}, {"id": "lunch", "label": "Lunch"}, {"id": "test_1", "label": "Test 1"}], "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115292600} -{"stream": "activity_types", "data": {"id": 7, "order_nr": 7, "name": "Test 1", "key_string": "test_1", "icon_key": "car", "active_flag": true, "color": null, "is_custom_flag": true, "add_time": "2023-02-22 11:13:54", "update_time": "2023-02-22 11:13:54"}, "emitted_at": 1683115293714} -{"stream": "currencies", "data": {"id": 2, "code": "AFN", "name": "Afghanistan Afghani", "symbol": "AFN", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1683115294881} -{"stream": "currencies", "data": {"id": 3, "code": "ALL", "name": "Albanian Lek", "symbol": "ALL", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1683115294882} -{"stream": "currencies", "data": {"id": 41, "code": "DZD", "name": "Algerian Dinar", "symbol": "DZD", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1683115294882} -{"stream": "deals", "data": {"id": 11, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 10, "org_id": 7, "stage_id": 1, "title": "Test Organization 2 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:27:53", "update_time": "2023-02-22 08:27:54", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-02-28", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User9 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal11@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1683115296218} -{"stream": "deals", "data": {"id": 13, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 8, "org_id": 7, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1200, "currency": "USD", "add_time": "2023-02-22 08:53:17", "update_time": "2023-02-22 08:53:18", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-03-31", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User7 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,200", "weighted_value": 1200, "formatted_weighted_value": "$1,200", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal13@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1683115296219} -{"stream": "deals", "data": {"id": 14, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 5, "org_id": 1, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:54:48", "update_time": "2023-02-22 08:54:49", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-04-30", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User2 Sample", "org_name": "Test Organization1", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal14@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1683115296220} -{"stream": "deal_products", "data": {"id": 11, "deal_id": 17, "product_id": 2, "product_variation_id": null, "name": "Item 1", "order_nr": 1, "item_price": 100, "quantity": 6, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 600, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 08:59:11", "last_edit": "2023-02-22 08:59:11", "comments": null, "tax": 0, "quantity_formatted": "6", "sum_formatted": "$600", "product": null}, "emitted_at": 1683115299113} -{"stream": "deal_products", "data": {"id": 12, "deal_id": 20, "product_id": 7, "product_variation_id": null, "name": "Item 6", "order_nr": 1, "item_price": 100, "quantity": 30, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 3000, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:24:57", "last_edit": "2023-02-22 10:24:57", "comments": null, "tax": 0, "quantity_formatted": "30", "sum_formatted": "$3,000", "product": null}, "emitted_at": 1683115299816} -{"stream": "deal_products", "data": {"id": 15, "deal_id": 23, "product_id": 11, "product_variation_id": null, "name": "Item 10", "order_nr": 1, "item_price": 20, "quantity": 120, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 2400, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:30:01", "last_edit": "2023-02-22 10:30:01", "comments": null, "tax": 0.01, "quantity_formatted": "120", "sum_formatted": "$2,400", "product": null}, "emitted_at": 1683115300490} -{"stream": "deal_fields", "data": {"id": 12477, "key": "id", "name": "ID", "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1683115304258} -{"stream": "deal_fields", "data": {"id": 12453, "key": "title", "name": "Title", "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "use_field": "id", "link": "/deal/", "mandatory_flag": true}, "emitted_at": 1683115304259} -{"stream": "deal_fields", "data": {"id": 12454, "key": "creator_user_id", "name": "Creator", "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1683115304260} -{"stream": "files", "data": {"id": 1, "user_id": 11884360, "log_id": null, "add_time": "2023-02-22 10:52:23", "update_time": "2023-02-22 10:52:23", "file_name": "bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "file_size": 6132, "active_flag": true, "inline_flag": false, "remote_location": "s3", "remote_id": "company/7780468/user/11884360/files/bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "s3_bucket": null, "url": "https://app.pipedrive.com/api/v1/files/1/download", "name": "Airbyte logo 75x75.png", "description": null, "deal_id": null, "lead_id": null, "person_id": null, "org_id": 1, "product_id": null, "activity_id": null, "deal_name": null, "lead_name": null, "person_name": null, "people_name": null, "org_name": "Test Organization1", "product_name": null, "mail_message_id": null, "mail_template_id": null, "cid": null, "file_type": "img"}, "emitted_at": 1683115305431} -{"stream": "filters", "data": {"id": 29, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1683115306604} -{"stream": "filters", "data": {"id": 28, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1683115306605} -{"stream": "lead_labels", "data": {"id": "aecece60-c069-11eb-93bf-b59c4f1731e6", "name": "Hot", "color": "red", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1683115307686} -{"stream": "lead_labels", "data": {"id": "aecece61-c069-11eb-93bf-b59c4f1731e6", "name": "Warm", "color": "yellow", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1683115307687} -{"stream": "lead_labels", "data": {"id": "aecece62-c069-11eb-93bf-b59c4f1731e6", "name": "Cold", "color": "blue", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1683115307687} -{"stream": "leads", "data": {"id": "7220c7f0-de6b-11eb-a544-5391b24dc0cf", "title": "Airbyte1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 10000, "currency": "USD"}, "expected_close_date": "2023-05-16", "person_id": 2, "organization_id": null, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 24, "add_time": "2021-07-06T15:04:22.255Z", "update_time": "2023-02-22T11:46:49.844Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadf6oUYVtZxhWEEyLegJqsRe@pipedrivemail.com"}, "emitted_at": 1683115309071} -{"stream": "leads", "data": {"id": "918d8510-de6b-11eb-8042-15d11dd01d20", "title": "Test Organization1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece61-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 3333, "currency": "USD"}, "expected_close_date": "2023-04-29", "person_id": 3, "organization_id": 1, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 25, "add_time": "2021-07-06T15:05:14.977Z", "update_time": "2023-02-22T11:47:18.002Z", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": 10, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": "Test", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadiYsMXTdfCQEKU896kY5xDA@pipedrivemail.com"}, "emitted_at": 1683115309072} -{"stream": "leads", "data": {"id": "bcfcf9c0-b28a-11ed-a6b1-df90b19da35a", "title": "Test Organization 2 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 15000, "currency": "USD"}, "expected_close_date": "2023-04-06", "person_id": 9, "organization_id": 9, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 26, "add_time": "2023-02-22T08:27:26.428Z", "update_time": "2023-02-22T11:47:36.779Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadpkxTNcD2sbWtbcQKvFKfu3@pipedrivemail.com"}, "emitted_at": 1683115309072} -{"stream": "notes", "data": {"id": 1, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 1, "lead_id": null, "content": "Test Note 1
    ", "add_time": "2023-02-22 08:24:35", "update_time": "2023-02-22 08:24:35", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization1"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1683115310379} -{"stream": "notes", "data": {"id": 2, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 2, "lead_id": null, "content": "Test note 2
    ", "add_time": "2023-02-22 08:31:47", "update_time": "2023-02-22 08:31:47", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 2"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1683115310380} -{"stream": "notes", "data": {"id": 3, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 5, "lead_id": null, "content": "Test Note
    ", "add_time": "2023-02-22 08:59:20", "update_time": "2023-02-22 08:59:20", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 5"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1683115310380} -{"stream": "organizations", "data": {"id": 12, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 7", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:21:58", "delete_time": null, "add_time": "2023-02-22 08:18:16", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "DY Patil College, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Pimpri Colony", "address_locality": "Pimpri-Chinchwad", "address_admin_area_level_1": "Maharashtra", "address_admin_area_level_2": "Pune Division", "address_country": "India", "address_postal_code": "411018", "address_formatted_address": "DY Patil College, DR. D Y PATIL MEDICAL COLLEGE, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra 411018, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1683115311467} -{"stream": "organizations", "data": {"id": 13, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 6", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:23:17", "delete_time": null, "add_time": "2023-02-22 08:18:33", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "Anand Vihar Railway Station, Block D, Anand Vihar, Delhi, Uttar Pradesh, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Anand Vihar", "address_locality": "Delhi", "address_admin_area_level_1": "Uttar Pradesh", "address_admin_area_level_2": "Delhi Division", "address_country": "India", "address_postal_code": "261205", "address_formatted_address": "J8X8+F33, Block D, Anand Vihar, Delhi, Uttar Pradesh 261205, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1683115311468} -{"stream": "organizations", "data": {"id": 3, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 3", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 2, "done_activities_count": 0, "undone_activities_count": 2, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:56:24", "delete_time": null, "add_time": "2023-02-22 08:07:28", "visible_to": "3", "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "next_activity_id": 7, "last_activity_id": null, "last_activity_date": null, "label": 6, "address": "Strasbourg, France", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "", "address_locality": "Strasbourg", "address_admin_area_level_1": "Grand Est", "address_admin_area_level_2": "Bas-Rhin", "address_country": "France", "address_postal_code": "", "address_formatted_address": "Strasbourg, France", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1683115311468} -{"stream": "organization_fields", "data": {"id": 4002, "key": "name", "name": "Name", "order_nr": 1, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "link": "/organization/", "use_field": "id", "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115312756} -{"stream": "organization_fields", "data": {"id": 4003, "key": "label", "name": "Label", "order_nr": 2, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": false, "options": [{"color": "green", "label": "Customer", "id": 5}, {"color": "red", "label": "Hot lead", "id": 6}, {"color": "yellow", "label": "Warm lead", "id": 7}, {"color": "blue", "label": "Cold lead", "id": 8}], "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115312757} -{"stream": "organization_fields", "data": {"id": 4004, "key": "owner_id", "name": "Owner", "order_nr": 3, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "use_field": "owner_id", "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115312758} -{"stream": "permission_sets", "data": {"id": "79b42bf0-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals admin", "assignment_count": 1, "app": "sales", "type": "admin"}, "emitted_at": 1683115313946} -{"stream": "permission_sets", "data": {"id": "79b42bf1-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals regular user", "assignment_count": 4, "app": "sales", "type": "regular"}, "emitted_at": 1683115313947} -{"stream": "permission_sets", "data": {"id": "fa17fff0-db56-11ec-93f1-e9cfc58fcd59", "name": "Global admin", "assignment_count": 1, "app": "global", "type": "admin"}, "emitted_at": 1683115313947} -{"stream": "persons", "data": {"id": 5, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User2 Sample", "first_name": "User2", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+14443332266", "primary": true}], "email": [{"label": "work", "value": "user2.sample.airbyte@gmail.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:44:49", "delete_time": null, "add_time": "2023-02-22 08:13:09", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 3, "picture_128_url": null, "org_name": "Test Organization1", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1683115315035} -{"stream": "persons", "data": {"id": 6, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User3 Sample", "first_name": "User3", "last_name": "Sample", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+15554442233", "primary": true}], "email": [{"label": "work", "value": "user3.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:45:52", "delete_time": null, "add_time": "2023-02-22 08:13:53", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 2, "picture_128_url": null, "org_name": "Test Organization1", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1683115315037} -{"stream": "persons", "data": {"id": 2, "company_id": 7780468, "owner_id": 11884360, "org_id": 11, "name": "User4 Sample", "first_name": "User4", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 1, "done_activities_count": 0, "undone_activities_count": 1, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+16665552233", "primary": true}], "email": [{"label": "work", "value": "user4.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 11:46:49", "delete_time": null, "add_time": "2021-07-06 15:04:21", "visible_to": "3", "picture_id": null, "next_activity_date": "2023-03-15", "next_activity_time": "11:30:00", "next_activity_id": 24, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 1, "picture_128_url": null, "org_name": "Test Organization 4", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1683115315038} -{"stream": "person_fields", "data": {"id": 9039, "key": "name", "name": "Name", "order_nr": 1, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "link": "/person/", "use_field": "id", "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115316226} -{"stream": "person_fields", "data": {"id": 9040, "key": "label", "name": "Label", "order_nr": 2, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": false, "options": [{"color": "green", "label": "Customer", "id": 1}, {"color": "red", "label": "Hot lead", "id": 2}, {"color": "yellow", "label": "Warm lead", "id": 3}, {"color": "blue", "label": "Cold lead", "id": 4}], "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115316227} -{"stream": "person_fields", "data": {"id": 9064, "key": "first_name", "name": "First name", "order_nr": 2, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:49", "update_time": "2020-12-10 07:23:49", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": false, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115316227} -{"stream": "pipelines", "data": {"id": 2, "name": "New pipeline", "url_title": "New-pipeline", "order_nr": 2, "active": true, "deal_probability": true, "add_time": "2021-07-06 15:24:11", "update_time": "2021-07-06 15:24:11"}, "emitted_at": 1683115317258} -{"stream": "pipelines", "data": {"id": 3, "name": "New pipeline 2", "url_title": "New-pipeline-2", "order_nr": 3, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:42:31", "update_time": "2023-02-22 10:42:31"}, "emitted_at": 1683115317259} -{"stream": "pipelines", "data": {"id": 4, "name": "New pipeline 3", "url_title": "New-pipeline-3", "order_nr": 4, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:43:12", "update_time": "2023-02-22 10:43:12"}, "emitted_at": 1683115317260} -{"stream": "product_fields", "data": {"id": 17, "key": "name", "name": "Name", "order_nr": 1, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "link": "/api/v1/products/", "use_field": "id", "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115318345} -{"stream": "product_fields", "data": {"id": 18, "key": "owner_id", "name": "Owner", "order_nr": 2, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "use_field": "owner_id", "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115318346} -{"stream": "product_fields", "data": {"id": 19, "key": "code", "name": "Product code", "order_nr": 3, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": false, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1683115318346} -{"stream": "products", "data": {"id": 1, "name": "Test Product", "code": "12345", "description": null, "unit": "Kg", "tax": 5, "category": "12", "active_flag": true, "selectable": true, "first_char": "t", "visible_to": "3", "owner_id": 11884360, "files_count": null, "followers_count": 0, "add_time": "2021-11-09 11:32:16", "update_time": "2021-11-09 11:32:31", "prices": [{"id": 1, "product_id": 1, "price": 10, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1683115319454} -{"stream": "products", "data": {"id": 2, "name": "Item 1", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "followers_count": 0, "add_time": "2023-02-22 08:31:13", "update_time": "2023-02-22 08:31:13", "prices": [{"id": 2, "product_id": 2, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1683115319455} -{"stream": "products", "data": {"id": 3, "name": "Item 2", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "followers_count": 0, "add_time": "2023-02-22 08:31:14", "update_time": "2023-02-22 08:31:14", "prices": [{"id": 3, "product_id": 3, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1683115319456} -{"stream": "roles", "data": {"id": 1, "parent_role_id": null, "name": "(Unassigned users)", "active_flag": true, "assignment_count": "5", "sub_role_count": "0", "level": 1, "description": "This is the default group for managing your visibility settings. New users are added automatically unless you change their group when you invite them."}, "emitted_at": 1683115320947} -{"stream": "stages", "data": {"id": 11, "order_nr": 5, "name": "Negotiations Started", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1683115322062} -{"stream": "stages", "data": {"id": 10, "order_nr": 4, "name": "Proposal Made", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1683115322063} -{"stream": "stages", "data": {"id": 9, "order_nr": 3, "name": "Demo Scheduled", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1683115322064} -{"stream": "users", "data": {"id": 11884360, "name": "Team Airbyte", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "integration-test@airbyte.io", "phone": null, "created": "2020-12-10 07:23:44", "modified": "2023-03-22 14:21:29", "lang": 1, "active_flag": true, "is_admin": 1, "last_login": "2023-03-22 14:21:29", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": true}, "emitted_at": 1683115324157} -{"stream": "users", "data": {"id": 18276123, "name": "User3 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user3.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:43", "modified": "2023-03-22 14:34:12", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:34:12", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1683115324158} -{"stream": "users", "data": {"id": 18276134, "name": "User4 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user4.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:39:38", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:39:38", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1683115324158} +{"stream": "activities", "data": {"id": 1, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "task", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-06", "due_time": "", "duration": "", "busy_flag": true, "add_time": "2021-07-06 15:02:04", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Task1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 10, "lead_id": null, "active_flag": true, "update_time": "2021-07-06 15:02:03", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy)", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal10@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Task", "lead": null}, "emitted_at": 1690801204738} +{"stream": "activities", "data": {"id": 3, "company_id": 7780468, "user_id": 11884360, "done": true, "type": "deadline", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-28", "due_time": "15:15", "duration": "00:30", "busy_flag": true, "add_time": "2021-07-06 15:03:31", "marked_as_done_time": "2023-02-22 08:25:45", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Deadline1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 1, "lead_id": null, "active_flag": true, "update_time": "2023-02-22 08:25:49", "update_user_id": 11884360, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "New note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal1@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Deadline", "lead": null}, "emitted_at": 1690801204739} +{"stream": "activities", "data": {"id": 7, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "call", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2023-02-22", "due_time": "12:30", "duration": "01:00", "busy_flag": true, "add_time": "2023-02-22 08:52:52", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": null, "subject": "Call", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 3, "person_id": 7, "deal_id": null, "lead_id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "active_flag": true, "update_time": "2023-02-22 08:52:52", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Test call
    ", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 7, "primary_flag": true}, {"person_id": 9, "primary_flag": false}], "series": null, "is_recurring": null, "org_name": "Test Organization 3", "person_name": "User6 Sample", "deal_title": null, "lead_title": "Test Organization 3 lead", "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": null, "assigned_to_user_id": 11884360, "type_name": "Call", "lead": {"id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "title": "Test Organization 3 lead", "labels": "aecece60-c069-11eb-93bf-b59c4f1731e6", "source": "Manually created", "owner_id": 11884360, "creator_user_id": 11884360, "deal_id": null, "related_person_id": 4, "related_org_id": 2, "person_name": null, "person_phone": null, "person_email": null, "org_name": null, "org_address": null, "active_flag": true, "add_time": "2023-02-22 08:52:06.047", "update_time": "2023-02-22 11:44:31.718", "archive_time": null, "seen": true, "deal_value": 1200, "deal_currency": "USD", "next_activity_id": 7, "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "visible_to": 3, "source_reference_id": null, "user": null, "deal_expected_close_date": "2023-05-15", "next_activity_status": null, "next_activity_datetime": null, "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null}}, "emitted_at": 1690801204739} +{"stream": "activity_fields", "data": {"id": 1, "key": "id", "name": "ID", "order_nr": 1, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205420} +{"stream": "activity_fields", "data": {"id": 2, "key": "subject", "name": "Subject", "order_nr": 2, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205420} +{"stream": "activity_fields", "data": {"id": 3, "key": "type", "name": "Type", "order_nr": 3, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "options": [{"id": "call", "label": "Call"}, {"id": "meeting", "label": "Meeting"}, {"id": "task", "label": "Task"}, {"id": "deadline", "label": "Deadline"}, {"id": "email", "label": "Email"}, {"id": "lunch", "label": "Lunch"}, {"id": "test_1", "label": "Test 1"}], "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205421} +{"stream": "activity_types", "data": {"id": 7, "order_nr": 7, "name": "Test 1", "key_string": "test_1", "icon_key": "car", "active_flag": true, "color": null, "is_custom_flag": true, "add_time": "2023-02-22 11:13:54", "update_time": "2023-02-22 11:13:54"}, "emitted_at": 1690801206161} +{"stream": "currencies", "data": {"id": 2, "code": "AFN", "name": "Afghanistan Afghani", "symbol": "AFN", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206834} +{"stream": "currencies", "data": {"id": 3, "code": "ALL", "name": "Albanian Lek", "symbol": "ALL", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206834} +{"stream": "currencies", "data": {"id": 41, "code": "DZD", "name": "Algerian Dinar", "symbol": "DZD", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206835} +{"stream": "deals", "data": {"id": 11, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 10, "org_id": 7, "stage_id": 1, "title": "Test Organization 2 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:27:53", "update_time": "2023-02-22 08:27:54", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-02-28", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User9 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal11@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207614} +{"stream": "deals", "data": {"id": 13, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 8, "org_id": 7, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1200, "currency": "USD", "add_time": "2023-02-22 08:53:17", "update_time": "2023-02-22 08:53:18", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-03-31", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User7 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,200", "weighted_value": 1200, "formatted_weighted_value": "$1,200", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal13@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207615} +{"stream": "deals", "data": {"id": 14, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 5, "org_id": 1, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:54:48", "update_time": "2023-02-22 08:54:49", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-04-30", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User2 Sample", "org_name": "Test Organization1", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal14@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207615} +{"stream": "deal_products", "data": {"id": 11, "deal_id": 17, "product_id": 2, "product_variation_id": null, "name": "Item 1", "order_nr": 1, "item_price": 100, "quantity": 6, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 600, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 08:59:11", "last_edit": "2023-02-22 08:59:11", "comments": null, "tax": 0, "quantity_formatted": "6", "sum_formatted": "$600", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801209362} +{"stream": "deal_products", "data": {"id": 12, "deal_id": 20, "product_id": 7, "product_variation_id": null, "name": "Item 6", "order_nr": 1, "item_price": 100, "quantity": 30, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 3000, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:24:57", "last_edit": "2023-02-22 10:24:57", "comments": null, "tax": 0, "quantity_formatted": "30", "sum_formatted": "$3,000", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801209809} +{"stream": "deal_products", "data": {"id": 15, "deal_id": 23, "product_id": 11, "product_variation_id": null, "name": "Item 10", "order_nr": 1, "item_price": 20, "quantity": 120, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 2400, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:30:01", "last_edit": "2023-02-22 10:30:01", "comments": null, "tax": 0.01, "quantity_formatted": "120", "sum_formatted": "$2,400", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801210358} +{"stream": "deal_fields", "data": {"id": 12477, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1690801212684} +{"stream": "deal_fields", "data": {"id": 12453, "key": "title", "name": "Title", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:02", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "use_field": "id", "link": "/deal/", "mandatory_flag": true}, "emitted_at": 1690801212685} +{"stream": "deal_fields", "data": {"id": 12454, "key": "creator_user_id", "name": "Creator", "group_id": null, "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1690801212685} +{"stream": "files", "data": {"id": 1, "user_id": 11884360, "log_id": null, "add_time": "2023-02-22 10:52:23", "update_time": "2023-02-22 10:52:23", "file_name": "bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "file_size": 6132, "active_flag": true, "inline_flag": false, "remote_location": "s3", "remote_id": "company/7780468/user/11884360/files/bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "s3_bucket": null, "url": "https://app.pipedrive.com/api/v1/files/1/download", "name": "Airbyte logo 75x75.png", "description": null, "deal_id": null, "lead_id": null, "person_id": null, "org_id": 1, "product_id": null, "activity_id": null, "deal_name": null, "lead_name": null, "person_name": null, "people_name": null, "org_name": "Test Organization1", "product_name": null, "mail_message_id": null, "mail_template_id": null, "cid": null, "file_type": "img"}, "emitted_at": 1690801213534} +{"stream": "filters", "data": {"id": 29, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1690801214182} +{"stream": "filters", "data": {"id": 28, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1690801214183} +{"stream": "lead_labels", "data": {"id": "aecece60-c069-11eb-93bf-b59c4f1731e6", "name": "Hot", "color": "red", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214865} +{"stream": "lead_labels", "data": {"id": "aecece61-c069-11eb-93bf-b59c4f1731e6", "name": "Warm", "color": "yellow", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214866} +{"stream": "lead_labels", "data": {"id": "aecece62-c069-11eb-93bf-b59c4f1731e6", "name": "Cold", "color": "blue", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214866} +{"stream": "leads", "data": {"id": "7220c7f0-de6b-11eb-a544-5391b24dc0cf", "title": "Airbyte1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 10000, "currency": "USD"}, "expected_close_date": "2023-05-16", "person_id": 2, "organization_id": null, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 24, "add_time": "2021-07-06T15:04:22.255Z", "update_time": "2023-02-22T11:46:49.844Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadf6oUYVtZxhWEEyLegJqsRe@pipedrivemail.com"}, "emitted_at": 1690801215787} +{"stream": "leads", "data": {"id": "918d8510-de6b-11eb-8042-15d11dd01d20", "title": "Test Organization1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece61-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 3333, "currency": "USD"}, "expected_close_date": "2023-04-29", "person_id": 3, "organization_id": 1, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 25, "add_time": "2021-07-06T15:05:14.977Z", "update_time": "2023-02-22T11:47:18.002Z", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": 10, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": "Test", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadiYsMXTdfCQEKU896kY5xDA@pipedrivemail.com"}, "emitted_at": 1690801215788} +{"stream": "leads", "data": {"id": "bcfcf9c0-b28a-11ed-a6b1-df90b19da35a", "title": "Test Organization 2 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 15000, "currency": "USD"}, "expected_close_date": "2023-04-06", "person_id": 9, "organization_id": 9, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 26, "add_time": "2023-02-22T08:27:26.428Z", "update_time": "2023-02-22T11:47:36.779Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadpkxTNcD2sbWtbcQKvFKfu3@pipedrivemail.com"}, "emitted_at": 1690801215788} +{"stream": "notes", "data": {"id": 1, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 1, "lead_id": null, "content": "Test Note 1
    ", "add_time": "2023-02-22 08:24:35", "update_time": "2023-02-22 08:24:35", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization1"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216928} +{"stream": "notes", "data": {"id": 2, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 2, "lead_id": null, "content": "Test note 2
    ", "add_time": "2023-02-22 08:31:47", "update_time": "2023-02-22 08:31:47", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 2"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216929} +{"stream": "notes", "data": {"id": 3, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 5, "lead_id": null, "content": "Test Note
    ", "add_time": "2023-02-22 08:59:20", "update_time": "2023-02-22 08:59:20", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 5"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216929} +{"stream": "organizations", "data": {"id": 12, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 7", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:21:58", "delete_time": null, "add_time": "2023-02-22 08:18:16", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "DY Patil College, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Pimpri Colony", "address_locality": "Pimpri-Chinchwad", "address_admin_area_level_1": "Maharashtra", "address_admin_area_level_2": "Pune Division", "address_country": "India", "address_postal_code": "411018", "address_formatted_address": "DY Patil College, DR. D Y PATIL MEDICAL COLLEGE, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra 411018, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217744} +{"stream": "organizations", "data": {"id": 13, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 6", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:23:17", "delete_time": null, "add_time": "2023-02-22 08:18:33", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "Anand Vihar Railway Station, Block D, Anand Vihar, Delhi, Uttar Pradesh, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Anand Vihar", "address_locality": "Delhi", "address_admin_area_level_1": "Uttar Pradesh", "address_admin_area_level_2": "Delhi Division", "address_country": "India", "address_postal_code": "261205", "address_formatted_address": "J8X8+F33, Block D, Anand Vihar, Delhi, Uttar Pradesh 261205, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217746} +{"stream": "organizations", "data": {"id": 3, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 3", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 2, "done_activities_count": 0, "undone_activities_count": 2, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:56:24", "delete_time": null, "add_time": "2023-02-22 08:07:28", "visible_to": "3", "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "next_activity_id": 7, "last_activity_id": null, "last_activity_date": null, "label": 6, "address": "Strasbourg, France", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "", "address_locality": "Strasbourg", "address_admin_area_level_1": "Grand Est", "address_admin_area_level_2": "Bas-Rhin", "address_country": "France", "address_postal_code": "", "address_formatted_address": "Strasbourg, France", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217746} +{"stream": "organization_fields", "data": {"id": 4012, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801218387} +{"stream": "organization_fields", "data": {"id": 4002, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:05", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/organization/", "mandatory_flag": true}, "emitted_at": 1690801218387} +{"stream": "organization_fields", "data": {"id": 4003, "key": "label", "name": "Label", "group_id": null, "order_nr": 0, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:06", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "options": [{"id": 5, "label": "Customer", "color": "green"}, {"id": 6, "label": "Hot lead", "color": "red"}, {"id": 7, "label": "Warm lead", "color": "yellow"}, {"id": 8, "label": "Cold lead", "color": "blue"}], "mandatory_flag": false}, "emitted_at": 1690801218388} +{"stream": "permission_sets", "data": {"id": "79b42bf0-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals admin", "assignment_count": 1, "app": "sales", "type": "admin"}, "emitted_at": 1690801219068} +{"stream": "permission_sets", "data": {"id": "79b42bf1-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals regular user", "assignment_count": 4, "app": "sales", "type": "regular"}, "emitted_at": 1690801219069} +{"stream": "permission_sets", "data": {"id": "fa17fff0-db56-11ec-93f1-e9cfc58fcd59", "name": "Global admin", "assignment_count": 1, "app": "global", "type": "admin"}, "emitted_at": 1690801219069} +{"stream": "persons", "data": {"id": 5, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User2 Sample", "first_name": "User2", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+14443332266", "primary": true}], "email": [{"label": "work", "value": "user2.sample.airbyte@gmail.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:44:49", "delete_time": null, "add_time": "2023-02-22 08:13:09", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 3, "picture_128_url": null, "org_name": "Test Organization1", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219881} +{"stream": "persons", "data": {"id": 6, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User3 Sample", "first_name": "User3", "last_name": "Sample", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+15554442233", "primary": true}], "email": [{"label": "work", "value": "user3.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:45:52", "delete_time": null, "add_time": "2023-02-22 08:13:53", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 2, "picture_128_url": null, "org_name": "Test Organization1", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219883} +{"stream": "persons", "data": {"id": 2, "company_id": 7780468, "owner_id": 11884360, "org_id": 11, "name": "User4 Sample", "first_name": "User4", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 1, "done_activities_count": 0, "undone_activities_count": 1, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+16665552233", "primary": true}], "email": [{"label": "work", "value": "user4.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 11:46:49", "delete_time": null, "add_time": "2021-07-06 15:04:21", "visible_to": "3", "picture_id": null, "next_activity_date": "2023-03-15", "next_activity_time": "11:30:00", "next_activity_id": 24, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 1, "picture_128_url": null, "org_name": "Test Organization 4", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219883} +{"stream": "person_fields", "data": {"id": 9051, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801220805} +{"stream": "person_fields", "data": {"id": 9039, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:04", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/person/", "mandatory_flag": true}, "emitted_at": 1690801220806} +{"stream": "person_fields", "data": {"id": 9040, "key": "label", "name": "Label", "group_id": null, "order_nr": 0, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:04", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "options": [{"id": 1, "label": "Customer", "color": "green"}, {"id": 2, "label": "Hot lead", "color": "red"}, {"id": 3, "label": "Warm lead", "color": "yellow"}, {"id": 4, "label": "Cold lead", "color": "blue"}], "mandatory_flag": false}, "emitted_at": 1690801220806} +{"stream": "pipelines", "data": {"id": 2, "name": "New pipeline", "url_title": "New-pipeline", "order_nr": 2, "active": true, "deal_probability": true, "add_time": "2021-07-06 15:24:11", "update_time": "2021-07-06 15:24:11"}, "emitted_at": 1690801221622} +{"stream": "pipelines", "data": {"id": 3, "name": "New pipeline 2", "url_title": "New-pipeline-2", "order_nr": 3, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:42:31", "update_time": "2023-02-22 10:42:31"}, "emitted_at": 1690801221623} +{"stream": "pipelines", "data": {"id": 4, "name": "New pipeline 3", "url_title": "New-pipeline-3", "order_nr": 4, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:43:12", "update_time": "2023-02-22 10:43:12"}, "emitted_at": 1690801221623} +{"stream": "product_fields", "data": {"id": 23, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801222343} +{"stream": "product_fields", "data": {"id": 17, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:07", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/api/v1/products/", "mandatory_flag": true}, "emitted_at": 1690801222345} +{"stream": "product_fields", "data": {"id": 18, "key": "owner_id", "name": "Owner", "group_id": null, "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:07", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "owner_id", "display_field": "owner_name", "mandatory_flag": true}, "emitted_at": 1690801222345} +{"stream": "products", "data": {"id": 1, "name": "Test Product", "code": "12345", "description": null, "unit": "Kg", "tax": 5, "category": "12", "active_flag": true, "selectable": true, "first_char": "t", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2021-11-09 11:32:16", "update_time": "2021-11-09 11:32:31", "prices": [{"id": 1, "product_id": 1, "price": 10, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223125} +{"stream": "products", "data": {"id": 2, "name": "Item 1", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2023-02-22 08:31:13", "update_time": "2023-02-22 08:31:13", "prices": [{"id": 2, "product_id": 2, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223126} +{"stream": "products", "data": {"id": 3, "name": "Item 2", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2023-02-22 08:31:14", "update_time": "2023-02-22 08:31:14", "prices": [{"id": 3, "product_id": 3, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223126} +{"stream": "roles", "data": {"id": 1, "parent_role_id": null, "name": "(Unassigned users)", "active_flag": true, "assignment_count": "5", "sub_role_count": "0", "level": 1, "description": "This is the default group for managing your visibility settings. New users are added automatically unless you change their group when you invite them."}, "emitted_at": 1690801224387} +{"stream": "stages", "data": {"id": 11, "order_nr": 5, "name": "Negotiations Started", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} +{"stream": "stages", "data": {"id": 10, "order_nr": 4, "name": "Proposal Made", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} +{"stream": "stages", "data": {"id": 9, "order_nr": 3, "name": "Demo Scheduled", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} +{"stream": "users", "data": {"id": 18276123, "name": "User3 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user3.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:43", "modified": "2023-03-22 14:34:12", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:34:12", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226031} +{"stream": "users", "data": {"id": 18276134, "name": "User4 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user4.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:39:38", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:39:38", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226032} +{"stream": "users", "data": {"id": 18276145, "name": "User5 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user5.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:44:35", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:44:35", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226032} diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index a8a565f83d16..c3c4a3126288 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 0.1.18 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-pipedrive githubIssueLabel: source-pipedrive icon: pipedrive.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pipedrive/requirements.txt b/airbyte-integrations/connectors/source-pipedrive/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pipedrive/requirements.txt +++ b/airbyte-integrations/connectors/source-pipedrive/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pipedrive/setup.py b/airbyte-integrations/connectors/source-pipedrive/setup.py index d332b07996f1..7108dd89b957 100644 --- a/airbyte-integrations/connectors/source-pipedrive/setup.py +++ b/airbyte-integrations/connectors/source-pipedrive/setup.py @@ -11,7 +11,7 @@ "requests~=2.25", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] setup( name="source_pipedrive", diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json index 7926e50d1019..257c0b7c54c7 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json @@ -74,6 +74,9 @@ }, "mandatory_flag": { "type": ["null", "boolean"] + }, + "display_name": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json index 85be9a47cefb..5d6642fe0e3f 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json @@ -135,7 +135,6 @@ "type": ["null", "string"], "format": "date-time", "airbyte_type": "timestamp_without_timezone" - }, "add_time": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json index f80223c46f2f..811da2f9dcc9 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json @@ -59,6 +59,9 @@ "use_field": { "type": ["null", "string"] }, + "display_field": { + "type": ["null", "string"] + }, "active_flag": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json index 1d9a1f29a298..f59204de4e5f 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json @@ -41,9 +41,6 @@ "files_count": { "type": ["null", "integer"] }, - "followers_count": { - "type": ["null", "integer"] - }, "add_time": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml index a8700942d892..f3b9c5bb49ef 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml +++ b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pivotal-tracker tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/requirements.txt b/airbyte-integrations/connectors/source-pivotal-tracker/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/requirements.txt +++ b/airbyte-integrations/connectors/source-pivotal-tracker/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py index 74511e154a23..63de73cbd36f 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/setup.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", ] diff --git a/airbyte-integrations/connectors/source-plaid/Dockerfile b/airbyte-integrations/connectors/source-plaid/Dockerfile index dca1963cb8e2..5d29cf267550 100644 --- a/airbyte-integrations/connectors/source-plaid/Dockerfile +++ b/airbyte-integrations/connectors/source-plaid/Dockerfile @@ -34,5 +34,5 @@ COPY source_plaid ./source_plaid ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-plaid diff --git a/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml index ba1479d67b7b..6375b20c3b4a 100644 --- a/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-plaid/acceptance-test-config.yml @@ -1,24 +1,31 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-plaid:dev -tests: +acceptance_tests: spec: - - spec_path: "source_plaid/spec.json" + tests: + - spec_path: "source_plaid/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json new file mode 100644 index 000000000000..2effb0c1b089 --- /dev/null +++ b/airbyte-integrations/connectors/source-plaid/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,107 @@ +{ + "streams": [ + { + "stream": { + "name": "balance", + "supported_sync_modes": ["full_refresh"], + "json_schema": { + "required": ["account_id", "current"], + "type": "object", + "properties": { + "account_id": { + "type": "string" + }, + "available": { + "type": ["number", "null"] + }, + "current": { + "type": "number" + }, + "iso_currency_code": { + "type": ["string", "null"] + }, + "limit": { + "type": ["number", "null"] + }, + "unofficial_currency_code": { + "type": ["string", "null"] + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "transaction", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "json_schema": { + "type": "object", + "required": [ + "account_id", + "amount", + "iso_currency_code", + "name", + "transaction_id", + "category", + "date", + "transaction_type" + ], + "properties": { + "account_id": { "type": "string" }, + "amount": { "type": "number" }, + "category": { "type": "array", "items": { "type": "string" } }, + "category_id": { "type": ["string", "null"] }, + "date": { "type": "string" }, + "iso_currency_code": { "type": "string" }, + "name": { "type": "string" }, + "payment_channel": { "type": ["string", "null"] }, + "pending": { "type": ["boolean", "null"] }, + "transaction_id": { "type": "string" }, + "transaction_type": { "type": "string" }, + "location": { + "type": ["object", "null"], + "properties": { + "address": { "type": ["string", "null"] }, + "city": { "type": ["string", "null"] }, + "country": { "type": ["string", "null"] }, + "lat": { "type": ["string", "null"] }, + "lon": { "type": ["string", "null"] }, + "postal_code": { "type": ["string", "null"] }, + "region": { "type": ["string", "null"] }, + "store_number": { "type": ["string", "null"] } + } + }, + "payment_meta": { + "type": ["object", "null"], + "properties": { + "by_order_of": { "type": ["string", "null"] }, + "payee": { "type": ["string", "null"] }, + "payer": { "type": ["string", "null"] }, + "payment_method": { "type": ["string", "null"] }, + "payment_processor": { "type": ["string", "null"] }, + "ppd_id": { "type": ["string", "null"] }, + "reason": { "type": ["string", "null"] }, + "reference_number": { "type": ["string", "null"] } + } + }, + "account_owner": { "type": ["string", "null"] }, + "authorized_date": { "type": ["string", "null"] }, + "authorized_datetime": { "type": ["string", "null"] }, + "check_number": { "type": ["string", "null"] }, + "datetime": { "type": ["string", "null"] }, + "merchant_name": { "type": ["string", "null"] }, + "pending_transaction_id": { "type": ["string", "null"] }, + "personal_finance_category": { "type": ["string", "null"] }, + "transaction_code": { "type": ["string", "null"] }, + "unofficial_currency_code": { "type": ["string", "null"] } + } + } + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-plaid/metadata.yaml b/airbyte-integrations/connectors/source-plaid/metadata.yaml index 8423019b7aff..e05ecb3ad445 100644 --- a/airbyte-integrations/connectors/source-plaid/metadata.yaml +++ b/airbyte-integrations/connectors/source-plaid/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: ed799e2b-2158-4c66-8da4-b40fe63bc72a - dockerImageTag: 0.3.2 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-plaid githubIssueLabel: source-plaid icon: plaid.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/plaid tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-plaid/requirements.txt b/airbyte-integrations/connectors/source-plaid/requirements.txt index 9ce85523c234..5b8864c417d3 100644 --- a/airbyte-integrations/connectors/source-plaid/requirements.txt +++ b/airbyte-integrations/connectors/source-plaid/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/connector-acceptance-test --e . diff --git a/airbyte-integrations/connectors/source-plaid/setup.py b/airbyte-integrations/connectors/source-plaid/setup.py index c344b3b06cc2..bcbf0b34a80c 100644 --- a/airbyte-integrations/connectors/source-plaid/setup.py +++ b/airbyte-integrations/connectors/source-plaid/setup.py @@ -5,10 +5,14 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "plaid-python"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "requests-mock~=1.9.3", "connector-acceptance-test", ] @@ -19,7 +23,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py b/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py index 8ff627c962f9..2891fc126707 100644 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml b/airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml new file mode 100644 index 000000000000..b84077619b84 --- /dev/null +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/manifest.yaml @@ -0,0 +1,374 @@ +version: 0.50.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - balance +streams: + - type: DeclarativeStream + name: balance + primary_key: + - account_id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + account_id: + type: string + available: + type: + - "null" + - number + current: + type: number + iso_currency_code: + type: + - "null" + - string + limit: + type: + - "null" + - number + unofficial_currency_code: + type: + - "null" + - string + required: + - account_id + - current + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{config['plaid_env']}}.plaid.com + path: /accounts/balance/get + http_method: POST + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: + secret: "{{config['api_key']}}" + options: + min_last_updated_datetime: "{{format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ')}}" + client_id: "{{config['client_id']}}" + access_token: "{{config['access_token']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - accounts + - "*" + paginator: + type: NoPagination + transformations: + - type: AddFields + fields: + - path: + - available + value: "{{record['balances']['available']}}" + - type: AddFields + fields: + - path: + - current + value: "{{record['balances']['current']}}" + - type: AddFields + fields: + - path: + - iso_currency_code + value: "{{record['balances']['iso_currency_code']}}" + - type: AddFields + fields: + - path: + - limit + value: "{{record['balances']['limit']}}" + - type: AddFields + fields: + - path: + - unofficial_currency_code + value: "{{record['balances']['unofficial_currency_code']}}" + - type: RemoveFields + field_pointers: + - - balances + - type: RemoveFields + field_pointers: + - - mask + - type: RemoveFields + field_pointers: + - - name + - type: RemoveFields + field_pointers: + - - official_name + - type: RemoveFields + field_pointers: + - - subtype + - type: RemoveFields + field_pointers: + - - type + - type: DeclarativeStream + name: transaction + primary_key: + - transaction_id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + account_id: + type: string + account_owner: + type: + - string + - "null" + amount: + type: number + authorized_date: + type: + - string + - "null" + authorized_datetime: + type: + - string + - "null" + category: + items: + type: string + type: array + category_id: + type: + - string + - "null" + check_number: + type: + - string + - "null" + date: + type: string + datetime: + type: + - string + - "null" + iso_currency_code: + type: string + location: + properties: + address: + type: + - string + - "null" + city: + type: + - string + - "null" + country: + type: + - string + - "null" + lat: + type: + - string + - "null" + lon: + type: + - string + - "null" + postal_code: + type: + - string + - "null" + region: + type: + - string + - "null" + store_number: + type: + - string + - "null" + type: + - object + - "null" + merchant_name: + type: + - string + - "null" + name: + type: string + payment_channel: + type: + - string + - "null" + payment_meta: + properties: + by_order_of: + type: + - string + - "null" + payee: + type: + - string + - "null" + payer: + type: + - string + - "null" + payment_method: + type: + - string + - "null" + payment_processor: + type: + - string + - "null" + ppd_id: + type: + - string + - "null" + reason: + type: + - string + - "null" + reference_number: + type: + - string + - "null" + type: + - object + - "null" + pending: + type: + - boolean + - "null" + pending_transaction_id: + type: + - string + - "null" + personal_finance_category: + type: + - string + - "null" + transaction_code: + type: + - string + - "null" + transaction_id: + type: string + transaction_type: + type: string + unofficial_currency_code: + type: + - string + - "null" + required: + - account_id + - amount + - iso_currency_code + - name + - transaction_id + - category + - date + - transaction_type + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://{{config['plaid_env']}}.plaid.com + path: /transactions/get + http_method: POST + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: + secret: "{{config['api_key']}}" + options: + offset: "{{ next_page_token['next_page_token'] }}" + client_id: "{{config['client_id']}}" + access_token: "{{config['access_token']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - transactions + - "*" + paginator: + type: DefaultPaginator + pagination_strategy: + type: OffsetIncrement + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + field_name: start_date + inject_into: body_json + end_time_option: + type: RequestOption + field_name: end_date + inject_into: body_json + start_datetime: + type: MinMaxDatetime + datetime: "{{config['start_date']}}" + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" +spec: + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - access_token + - api_key + - client_id + - plaid_env + properties: + access_token: + type: string + order: 0 + title: Access Token + description: The end-user's Link access token. + airbyte_secret: true + api_key: + type: string + order: 1 + title: API Key + description: The Plaid API key to use to hit the API. + airbyte_secret: true + client_id: + type: string + order: 2 + title: Client ID + description: The Plaid client id. + plaid_env: + enum: + - sandbox + - development + - production + type: string + order: 3 + title: Plaid Environment + description: The Plaid environment. + start_date: + type: string + order: 4 + title: Start Date + format: date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + description: >- + The date from which you'd like to replicate data for Plaid in the + format YYYY-MM-DD. All data generated after this date will be + replicated. + additionalProperties: true + documentation_url: https://example.org + type: Spec +metadata: + autoImportSchema: + balance: false + transaction: false diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json b/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json deleted file mode 100644 index e0b4885ca3bc..000000000000 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/balance.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["account_id", "current"], - "properties": { - "account_id": { "type": "string" }, - "available": { "type": ["number", "null"] }, - "current": { "type": "number" }, - "iso_currency_code": { "type": ["string", "null"] }, - "limit": { "type": ["number", "null"] }, - "unofficial_currency_code": { "type": ["string", "null"] } - } -} diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json b/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json deleted file mode 100644 index dba0a472d568..000000000000 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/schemas/transaction.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "account_id", - "amount", - "iso_currency_code", - "name", - "transaction_id", - "category", - "date", - "transaction_type" - ], - "properties": { - "account_id": { "type": "string" }, - "amount": { "type": "number" }, - "category": { "type": "array", "items": { "type": "string" } }, - "category_id": { "type": ["string", "null"] }, - "date": { "type": "string" }, - "iso_currency_code": { "type": "string" }, - "name": { "type": "string" }, - "payment_channel": { "type": ["string", "null"] }, - "pending": { "type": ["boolean", "null"] }, - "transaction_id": { "type": "string" }, - "transaction_type": { "type": "string" }, - "location": { - "type": ["object", "null"], - "properties": { - "address": { "type": ["string", "null"] }, - "city": { "type": ["string", "null"] }, - "country": { "type": ["string", "null"] }, - "lat": { "type": ["string", "null"] }, - "lon": { "type": ["string", "null"] }, - "postal_code": { "type": ["string", "null"] }, - "region": { "type": ["string", "null"] }, - "store_number": { "type": ["string", "null"] } - } - }, - "payment_meta": { - "type": ["object", "null"], - "properties": { - "by_order_of": { "type": ["string", "null"] }, - "payee": { "type": ["string", "null"] }, - "payer": { "type": ["string", "null"] }, - "payment_method": { "type": ["string", "null"] }, - "payment_processor": { "type": ["string", "null"] }, - "ppd_id": { "type": ["string", "null"] }, - "reason": { "type": ["string", "null"] }, - "reference_number": { "type": ["string", "null"] } - } - }, - "account_owner": { "type": ["string", "null"] }, - "authorized_date": { "type": ["string", "null"] }, - "authorized_datetime": { "type": ["string", "null"] }, - "check_number": { "type": ["string", "null"] }, - "datetime": { "type": ["string", "null"] }, - "merchant_name": { "type": ["string", "null"] }, - "pending_transaction_id": { "type": ["string", "null"] }, - "personal_finance_category": { "type": ["string", "null"] }, - "transaction_code": { "type": ["string", "null"] }, - "unofficial_currency_code": { "type": ["string", "null"] } - } -} diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/source.py b/airbyte-integrations/connectors/source-plaid/source_plaid/source.py index 29961fd57c0a..6da533a6f6e0 100644 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/source.py +++ b/airbyte-integrations/connectors/source-plaid/source_plaid/source.py @@ -2,145 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime -import json -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import plaid -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from plaid.api import plaid_api -from plaid.model.accounts_balance_get_request import AccountsBalanceGetRequest -from plaid.model.accounts_balance_get_request_options import AccountsBalanceGetRequestOptions -from plaid.model.transactions_get_request import TransactionsGetRequest -from plaid.model.transactions_get_request_options import TransactionsGetRequestOptions +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -SPEC_ENV_TO_PLAID_ENV = { - "production": plaid.Environment.Production, - "development": plaid.Environment.Development, - "sandbox": plaid.Environment.Sandbox, -} +WARNING: Do not modify this file. +""" -class PlaidStream(Stream): - def __init__(self, config: Mapping[str, Any]): - plaid_config = plaid.Configuration( - host=SPEC_ENV_TO_PLAID_ENV[config["plaid_env"]], api_key={"clientId": config["client_id"], "secret": config["api_key"]} - ) - api_client = plaid.ApiClient(plaid_config) - self.client = plaid_api.PlaidApi(api_client) - self.access_token = config["access_token"] - self.start_date = datetime.datetime.strptime(config.get("start_date"), "%Y-%m-%d").date() if config.get("start_date") else None - - -class BalanceStream(PlaidStream): - @property - def name(self): - return "balance" - - @property - def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: - return "account_id" - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - min_last_updated_datetime = datetime.datetime.strptime( - datetime.datetime.strftime(self.start_date, "%y-%m-%dT%H:%M:%SZ"), - "%y-%m-%dT%H:%M:%S%z", - ) - options = AccountsBalanceGetRequestOptions(min_last_updated_datetime=min_last_updated_datetime) - getRequest = AccountsBalanceGetRequest(access_token=self.access_token, options=options) - balance_response = self.client.accounts_balance_get(getRequest) - for balance in balance_response["accounts"]: - message_dict = balance["balances"].to_dict() - message_dict["account_id"] = balance["account_id"] - yield message_dict - - -class IncrementalTransactionStream(PlaidStream): - @property - def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: - return "transaction_id" - - @property - def name(self): - return "transaction" - - @property - def source_defined_cursor(self) -> bool: - return True - - @property - def cursor_field(self) -> Union[str, List[str]]: - return "date" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - return {"date": latest_record.get("date")} - - def _get_transactions_response(self, start_date, end_date=datetime.datetime.utcnow().date(), offset=0): - options = TransactionsGetRequestOptions() - options.offset = offset - - return self.client.transactions_get( - TransactionsGetRequest(access_token=self.access_token, start_date=start_date, end_date=end_date, options=options) - ) - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - date = stream_state.get("date") - all_transactions = [] - - if not date: - date = datetime.date.fromtimestamp(0) - else: - date = datetime.date.fromisoformat(date) - if date >= datetime.datetime.utcnow().date(): - return - - if self.start_date: - date = max(self.start_date, date) - - response = self._get_transactions_response(date) - all_transactions.extend(response.transactions) - num_total_transactions = response.total_transactions - - while len(all_transactions) < num_total_transactions: - response = self._get_transactions_response(date, offset=len(all_transactions)) - all_transactions.extend(response.transactions) - - yield from map(lambda x: x.to_dict(), sorted(all_transactions, key=lambda t: t["date"])) - - -class SourcePlaid(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: - try: - plaid_config = plaid.Configuration( - host=SPEC_ENV_TO_PLAID_ENV[config["plaid_env"]], api_key={"clientId": config["client_id"], "secret": config["api_key"]} - ) - api_client = plaid.ApiClient(plaid_config) - client = plaid_api.PlaidApi(api_client) - try: - request = AccountsBalanceGetRequest(access_token=config["access_token"]) - client.accounts_balance_get(request) - return True, None - except plaid.ApiException as e: - response = json.loads(e.body) - return False, response - except Exception as error: - return False, error - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [BalanceStream(config), IncrementalTransactionStream(config)] +# Declarative Source +class SourcePlaid(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-plaid/source_plaid/spec.json b/airbyte-integrations/connectors/source-plaid/source_plaid/spec.json deleted file mode 100644 index 09605304314f..000000000000 --- a/airbyte-integrations/connectors/source-plaid/source_plaid/spec.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "documentationUrl": "https://plaid.com/docs/api/", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": ["access_token", "api_key", "client_id", "plaid_env"], - "additionalProperties": true, - "properties": { - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The end-user's Link access token." - }, - "api_key": { - "title": "API Key", - "type": "string", - "description": "The Plaid API key to use to hit the API.", - "airbyte_secret": true - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Plaid client id" - }, - "plaid_env": { - "title": "Plaid Environment", - "type": "string", - "enum": ["sandbox", "development", "production"], - "description": "The Plaid environment" - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate data for Plaid in the format YYYY-MM-DD. All data generated after this date will be replicated.", - "examples": ["2021-03-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-plausible/metadata.yaml b/airbyte-integrations/connectors/source-plausible/metadata.yaml index 8ba5e295fb2d..042fa88dd49a 100644 --- a/airbyte-integrations/connectors/source-plausible/metadata.yaml +++ b/airbyte-integrations/connectors/source-plausible/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-plausible/requirements.txt b/airbyte-integrations/connectors/source-plausible/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-plausible/requirements.txt +++ b/airbyte-integrations/connectors/source-plausible/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-plausible/setup.py b/airbyte-integrations/connectors/source-plausible/setup.py index 35c3167d1842..d222145d848e 100644 --- a/airbyte-integrations/connectors/source-plausible/setup.py +++ b/airbyte-integrations/connectors/source-plausible/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-pocket/metadata.yaml b/airbyte-integrations/connectors/source-pocket/metadata.yaml index f29f12a6b97a..145bf1b927e4 100644 --- a/airbyte-integrations/connectors/source-pocket/metadata.yaml +++ b/airbyte-integrations/connectors/source-pocket/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pocket/requirements.txt b/airbyte-integrations/connectors/source-pocket/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pocket/requirements.txt +++ b/airbyte-integrations/connectors/source-pocket/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pocket/setup.py b/airbyte-integrations/connectors/source-pocket/setup.py index 0b430b433378..bdbf2b6549ab 100644 --- a/airbyte-integrations/connectors/source-pocket/setup.py +++ b/airbyte-integrations/connectors/source-pocket/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index cde8e0ce4f07..dec58781459c 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pokeapi/requirements.txt b/airbyte-integrations/connectors/source-pokeapi/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pokeapi/requirements.txt +++ b/airbyte-integrations/connectors/source-pokeapi/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index 6e2c4efa3230..e5cdf3627e09 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1"] setup( name="source_pokeapi", diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml b/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml index 3fffa06974bf..9be804fef482 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-polygon-stock-api/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/requirements.txt b/airbyte-integrations/connectors/source-polygon-stock-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/requirements.txt +++ b/airbyte-integrations/connectors/source-polygon-stock-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/setup.py b/airbyte-integrations/connectors/source-polygon-stock-api/setup.py index 265defd869ab..ddd7463c9292 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/setup.py +++ b/airbyte-integrations/connectors/source-polygon-stock-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 1650c048f3f6..6dbaaab0c232 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.33 +LABEL io.airbyte.version=3.1.5 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml index f497d002a406..f60e772766af 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml @@ -1,6 +1,50 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-postgres-strict-encrypt:dev -tests: +custom_environment_variables: + USE_STREAM_CAPABLE_STATE: true +acceptance_tests: spec: - - spec_path: "src/test/resources/expected_spec.json" + tests: + - spec_path: "src/test-integration/resources/expected_strict_encrypt_spec.json" + config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "1.0.52" + - spec_path: "src/test-integration/resources/expected_strict_encrypt_spec.json" + config_path: "secrets/config_cdc.json" + backward_compatibility_tests_config: + disable_for_version: "1.0.52" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_cdc.json" + status: "succeed" + discovery: + tests: + - config_path: "secrets/config.json" + - config_path: "secrets/config_cdc.json" + backward_compatibility_tests_config: + disable_for_version: "2.1.1" + basic_read: + tests: + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.txt" + - config_path: "secrets/config_cdc.json" + expect_records: + path: "integration_tests/expected_records.txt" + full_refresh: + tests: + - config_path: "secrets/config.json" + - config_path: "secrets/config_cdc.json" +# incremental: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/incremental_configured_catalog.json" +# future_state: +# bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" +# - config_path: "secrets/config_cdc.json" +# configured_catalog_path: "integration_tests/incremental_configured_catalog.json" +# future_state: +# bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle index f657b3794952..484e154ad89f 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle @@ -34,3 +34,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/README.md b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/README.md new file mode 100644 index 000000000000..45e74b238d3c --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/README.md @@ -0,0 +1,5 @@ +This directory contains files used to run Connector Acceptance Tests. +* `abnormal_state.json` describes a connector state with a non-existing cursor value. +* `expected_records.txt` lists all the records expected as the output of the basic read operation. +* `incremental_configured_catalog.json` is a configured catalog used as an input of the `incremental` test. +* `seed.sql` is the query we manually ran on a test postgres instance to seed it with test data and enable CDC. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..c3e6b23a2b0d --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/abnormal_state.json @@ -0,0 +1,18 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "stream_name": "id_and_name", + "stream_namespace": "public", + "cursor_field": ["id"], + "cursor": "4", + "cursor_record_count": 1 + }, + "stream_descriptor": { + "name": "id_and_name", + "namespace": "public" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/expected_records.txt new file mode 100644 index 000000000000..d886fbe3fe03 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/expected_records.txt @@ -0,0 +1,3 @@ +{"stream": "id_and_name", "data": {"id": 1, "name": "picard"}, "emitted_at": 999999} +{"stream": "id_and_name", "data": {"id": 2, "name": "crusher"}, "emitted_at": 999999} +{"stream": "id_and_name", "data": {"id": 3, "name": "vash"}, "emitted_at": 999999} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/incremental_configured_catalog.json new file mode 100644 index 000000000000..648876fb50e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/incremental_configured_catalog.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "stream": { + "name": "id_and_name", + "json_schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "id": { + "type": "number", + "airbyte_type": "integer" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": [], + "source_defined_primary_key": [], + "namespace": "public" + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"], + "user_defined_primary_key": ["id"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed.sql new file mode 100644 index 000000000000..48910082f86f --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed.sql @@ -0,0 +1,36 @@ +ALTER ROLE postgres WITH REPLICATION; + +CREATE + TABLE + id_and_name( + id INTEGER, + name VARCHAR(200) + ); + +INSERT + INTO + id_and_name( + id, + name + ) + VALUES( + 1, + 'picard' + ), + ( + 2, + 'crusher' + ), + ( + 3, + 'vash' + ); + +SELECT + pg_create_logical_replication_slot( + 'debezium_slot', + 'pgoutput' + ); + +CREATE + PUBLICATION publication FOR ALL TABLES; diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/basic.sql new file mode 100644 index 000000000000..d7cb4fda899e --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/basic.sql @@ -0,0 +1,334 @@ +CREATE + SCHEMA POSTGRES_BASIC; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TYPE inventory_item AS( + name text, + supplier_id INTEGER, + price NUMERIC + ); +SET +lc_monetary TO 'en_US.utf8'; +SET +TIMEZONE TO 'MST'; + +CREATE + EXTENSION hstore; + +CREATE + TABLE + POSTGRES_BASIC.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_11 CHAR, + test_column_12 CHAR(8), + test_column_13 CHARACTER, + test_column_14 CHARACTER(8), + test_column_15 text, + test_column_16 VARCHAR, + test_column_20 DATE NOT NULL DEFAULT now(), + test_column_21 DATE, + test_column_23 FLOAT, + test_column_24 DOUBLE PRECISION, + test_column_27 INT, + test_column_28 INTEGER, + test_column_3 BIT(1), + test_column_4 BIT(3), + test_column_44 REAL, + test_column_46 SMALLINT, + test_column_51 TIME WITHOUT TIME ZONE, + test_column_52 TIME, + test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_54 TIMESTAMP, + test_column_55 TIMESTAMP WITHOUT TIME ZONE, + test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + test_column_57 TIMESTAMP, + test_column_58 TIMESTAMP WITHOUT TIME ZONE, + test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_60 TIMESTAMP WITH TIME ZONE, + test_column_61 timestamptz, + test_column_7 bool, + test_column_70 TIME WITH TIME ZONE, + test_column_71 timetz, + test_column_8 BOOLEAN + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + 'a', + '{asb123}', + 'a', + '{asb123}', + 'a', + 'a', + '1999-01-08', + '1999-01-08', + '123', + '123', + 1001, + 1001, + B'0', + B'101', + 3.4145, + - 32768, + '13:00:01', + '13:00:01', + '13:00:01', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TRUE, + '13:00:01', + '13:00:01', + TRUE + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '*', + '{asb12}', + '*', + '{asb12}', + 'abc', + 'abc', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + - 2147483648, + - 2147483648, + B'0', + B'101', + 3.4145, + 32767, + '13:00:02+8', + '13:00:02+8', + '13:00:02+8', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + 'yes', + '13:00:00+8', + '13:00:00+8', + 'yes' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 3, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть на південь, не питай чому;', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:03-8', + '13:00:03-8', + '13:00:03-8', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + '1', + '13:00:03-8', + '13:00:03-8', + '1' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 4, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '櫻花分店', + '櫻花分店', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:04Z', + '13:00:04Z', + '13:00:04Z', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + FALSE, + '13:00:04Z', + '13:00:04Z', + FALSE + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 5, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '', + '', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'no', + '13:00:05.012345Z+8', + '13:00:05.012345Z+8', + 'no' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 6, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:00Z-8', + '13:00:00Z-8', + '13:00:00Z-8', + 'epoch', + 'epoch', + 'epoch', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '0' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 7, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '24:00:00', + '24:00:00', + '24:00:00', + 'epoch', + 'epoch', + 'epoch', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '0' + ); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full.sql new file mode 100644 index 000000000000..26edf7749a9f --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full.sql @@ -0,0 +1,825 @@ +CREATE + SCHEMA POSTGRES_FULL; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TYPE inventory_item AS( + name text, + supplier_id INTEGER, + price NUMERIC + ); +SET +lc_monetary TO 'en_US.utf8'; +SET +TIMEZONE TO 'MST'; + +CREATE + EXTENSION hstore; + +CREATE + TABLE + POSTGRES_FULL.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 bytea, + test_column_11 CHAR, + test_column_12 CHAR(8), + test_column_13 CHARACTER, + test_column_14 CHARACTER(8), + test_column_15 text, + test_column_16 VARCHAR, + test_column_17 CHARACTER VARYING(10), + test_column_18 cidr, + test_column_19 circle, + test_column_2 bigserial, + test_column_20 DATE NOT NULL DEFAULT now(), + test_column_21 DATE, + test_column_22 float8, + test_column_23 FLOAT, + test_column_24 DOUBLE PRECISION, + test_column_25 inet, + test_column_26 int4, + test_column_27 INT, + test_column_28 INTEGER, + test_column_29 INTERVAL, + test_column_3 BIT(1), + test_column_30 json, + test_column_31 jsonb, + test_column_32 line, + test_column_33 lseg, + test_column_34 macaddr, + test_column_35 macaddr8, + test_column_36 money, + test_column_37 DECIMAL, + test_column_38 NUMERIC, + test_column_39 PATH, + test_column_4 BIT(3), + test_column_40 pg_lsn, + test_column_41 point, + test_column_42 polygon, + test_column_43 float4, + test_column_44 REAL, + test_column_45 int2, + test_column_46 SMALLINT, + test_column_47 serial2, + test_column_48 smallserial, + test_column_49 serial4, + test_column_5 BIT VARYING(5), + test_column_51 TIME WITHOUT TIME ZONE, + test_column_52 TIME, + test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_54 TIMESTAMP, + test_column_55 TIMESTAMP WITHOUT TIME ZONE, + test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + test_column_57 TIMESTAMP, + test_column_58 TIMESTAMP WITHOUT TIME ZONE, + test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_6 BIT VARYING(5), + test_column_60 TIMESTAMP WITH TIME ZONE, + test_column_61 timestamptz, + test_column_62 tsquery, + test_column_63 tsvector, + test_column_64 uuid, + test_column_65 xml, + test_column_66 mood, + test_column_67 tsrange, + test_column_68 inventory_item, + test_column_69 hstore, + test_column_7 bool, + test_column_70 TIME WITH TIME ZONE, + test_column_71 timetz, + test_column_72 INT2 [], + test_column_73 INT4 [], + test_column_74 INT8 [], + test_column_75 OID [], + test_column_76 VARCHAR [], + test_column_77 CHAR(1)[], + test_column_78 BPCHAR(2)[], + test_column_79 TEXT [], + test_column_8 BOOLEAN, + test_column_80 NAME [], + test_column_81 NUMERIC [], + test_column_82 DECIMAL [], + test_column_83 FLOAT4 [], + test_column_84 FLOAT8 [], + test_column_85 MONEY [], + test_column_86 BOOL [], + test_column_87 BIT [], + test_column_88 BYTEA [], + test_column_89 DATE [], + test_column_9 box, + test_column_90 TIME(6)[], + test_column_91 TIMETZ [], + test_column_92 TIMESTAMPTZ [], + test_column_93 TIMESTAMP [] + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + NULL, + 'a', + '{asb123}', + 'a', + '{asb123}', + 'a', + 'a', + '{asb123}', + NULL, + '(5,7),10', + 1, + '1999-01-08', + '1999-01-08', + '123', + '123', + '123', + '198.24.10.0/24', + NULL, + NULL, + NULL, + NULL, + B'0', + NULL, + NULL, + '{4,5,6}', + '((3,7),(15,18))', + NULL, + NULL, + NULL, + '123', + '123', + '((3,7),(15,18))', + B'101', + '7/A25801C8'::pg_lsn, + '(3,7)', + '((3,7),(15,18))', + NULL, + NULL, + NULL, + NULL, + 1, + 1, + 1, + B'101', + '13:00:01', + '13:00:01', + '13:00:01', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + 'infinity', + 'infinity', + 'infinity', + B'101', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + NULL, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + XMLPARSE( + DOCUMENT 'Manual...' + ), + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + TRUE, + NULL, + NULL, + '{1,2,3}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + TRUE, + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((3,7),(15,18))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '*', + '{asb12}', + 'abc', + 'abc', + '{asb12}', + '192.168.100.128/25', + '(0,0),0', + 9223372036854775807, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.24.10.0', + 1001, + 1001, + 1001, + 'P1Y2M3DT4H5M6S', + NULL, + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08:00:2b:01:02:03', + '08:00:2b:01:02:03:04:05', + '999.99', + NULL, + NULL, + '((0,0),(0,0))', + NULL, + '0/0'::pg_lsn, + '(0,0)', + '((0,0),(0,0))', + 3.4145, + 3.4145, + - 32768, + - 32768, + 32767, + 32767, + 2147483647, + NULL, + '13:00:02+8', + '13:00:02+8', + '13:00:02+8', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + '-infinity', + '-infinity', + '-infinity', + NULL, + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + 'fat & (rat | cat)'::tsquery, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'yes', + '13:00:01', + '13:00:01', + '{4,5,6}', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'yes', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '((0,0),(0,0))', + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 3, + 0, + '1234', + NULL, + NULL, + NULL, + NULL, + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть на південь, не питай чому;', + NULL, + '192.168/24', + '(-10,-4),10', + 0, + NULL, + NULL, + NULL, + NULL, + NULL, + '198.10/8', + - 2147483648, + - 2147483648, + - 2147483648, + '-178000000', + NULL, + NULL, + NULL, + NULL, + NULL, + '08-00-2b-01-02-04', + '08-00-2b-01-02-03-04-06', + '1,001.01', + '1234567890.1234567', + '1234567890.1234567', + NULL, + NULL, + NULL, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + NULL, + NULL, + 32767, + 32767, + 0, + 0, + 0, + NULL, + '13:00:03-8', + '13:00:03-8', + '13:00:03-8', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + NULL, + NULL, + NULL, + NULL, + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + 'fat:ab & cat'::tsquery, + NULL, + NULL, + '', + NULL, + NULL, + NULL, + NULL, + '1', + '13:00:00+8', + '13:00:00+8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '1', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 4, + NULL, + 'abcd', + NULL, + NULL, + NULL, + NULL, + '櫻花分店', + '櫻花分店', + NULL, + '192.168.1', + NULL, + - 9223372036854775808, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 2147483647, + 2147483647, + 2147483647, + '178000000', + NULL, + NULL, + NULL, + NULL, + NULL, + '08002b:010205', + '08002b:0102030407', + '-1,000', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + - 32767, + - 32767, + - 2147483647, + NULL, + '13:00:04Z', + '13:00:04Z', + '13:00:04Z', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + NULL, + NULL, + NULL, + NULL, + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + FALSE, + '13:00:03-8', + '13:00:03-8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + FALSE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 5, + NULL, + '\xabcd', + NULL, + NULL, + NULL, + NULL, + '', + '', + NULL, + '128.1', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '$999.99', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + NULL, + NULL, + NULL, + NULL, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'no', + '13:00:04Z', + '13:00:04Z', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'no', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 6, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '2001:4f8:3:ba::/64', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '$1001.01', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '13:00:00Z-8', + '13:00:00Z-8', + '13:00:00Z-8', + 'epoch', + 'epoch', + 'epoch', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '0', + '13:00:05.012345Z+8', + '13:00:05.012345Z+8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '0', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 7, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '-$1,000', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '24:00:00', + '24:00:00', + '24:00:00', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full_without_nulls.sql new file mode 100644 index 000000000000..f101c9b7a3fa --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full_without_nulls.sql @@ -0,0 +1,861 @@ +CREATE + SCHEMA POSTGRES_FULL_NN; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TYPE inventory_item AS( + name text, + supplier_id INTEGER, + price NUMERIC + ); +SET +lc_monetary TO 'en_US.utf8'; +SET +TIMEZONE TO 'MST'; + +CREATE + EXTENSION hstore; + +CREATE + TABLE + POSTGRES_FULL_NN.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 bytea, + test_column_11 CHAR, + test_column_12 CHAR(8), + test_column_13 CHARACTER, + test_column_14 CHARACTER(8), + test_column_15 text, + test_column_16 VARCHAR, + test_column_17 CHARACTER VARYING(10), + test_column_18 cidr, + test_column_19 circle, + test_column_2 bigserial, + test_column_20 DATE NOT NULL DEFAULT now(), + test_column_21 DATE, + test_column_22 float8, + test_column_23 FLOAT, + test_column_24 DOUBLE PRECISION, + test_column_25 inet, + test_column_26 int4, + test_column_27 INT, + test_column_28 INTEGER, + test_column_29 INTERVAL, + test_column_3 BIT(1), + test_column_30 json, + test_column_31 jsonb, + test_column_32 line, + test_column_33 lseg, + test_column_34 macaddr, + test_column_35 macaddr8, + test_column_36 money, + test_column_37 DECIMAL, + test_column_38 NUMERIC, + test_column_39 PATH, + test_column_4 BIT(3), + test_column_40 pg_lsn, + test_column_41 point, + test_column_42 polygon, + test_column_43 float4, + test_column_44 REAL, + test_column_45 int2, + test_column_46 SMALLINT, + test_column_47 serial2, + test_column_48 smallserial, + test_column_49 serial4, + test_column_5 BIT VARYING(5), + test_column_51 TIME WITHOUT TIME ZONE, + test_column_52 TIME, + test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_54 TIMESTAMP, + test_column_55 TIMESTAMP WITHOUT TIME ZONE, + test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + test_column_57 TIMESTAMP, + test_column_58 TIMESTAMP WITHOUT TIME ZONE, + test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_6 BIT VARYING(5), + test_column_60 TIMESTAMP WITH TIME ZONE, + test_column_61 timestamptz, + test_column_62 tsquery, + test_column_63 tsvector, + test_column_64 uuid, + test_column_65 xml, + test_column_66 mood, + test_column_67 tsrange, + test_column_68 inventory_item, + test_column_69 hstore, + test_column_7 bool, + test_column_70 TIME WITH TIME ZONE, + test_column_71 timetz, + test_column_72 INT2 [], + test_column_73 INT4 [], + test_column_74 INT8 [], + test_column_75 OID [], + test_column_76 VARCHAR [], + test_column_77 CHAR(1)[], + test_column_78 BPCHAR(2)[], + test_column_79 TEXT [], + test_column_8 BOOLEAN, + test_column_80 NAME [], + test_column_81 NUMERIC [], + test_column_82 DECIMAL [], + test_column_83 FLOAT4 [], + test_column_84 FLOAT8 [], + test_column_85 MONEY [], + test_column_86 BOOL [], + test_column_87 BIT [], + test_column_88 BYTEA [], + test_column_89 DATE [], + test_column_9 box, + test_column_90 TIME(6)[], + test_column_91 TIMETZ [], + test_column_92 TIMESTAMPTZ [], + test_column_93 TIMESTAMP [] + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + decode( + '1234', + 'hex' + ), + 'a', + '{asb123}', + 'a', + '{asb123}', + 'a', + 'a', + '{asb123}', + '192.168.100.128/25', + '(5,7),10', + 1, + '1999-01-08', + '1999-01-08', + '123', + '123', + '123', + '198.24.10.0/24', + 1001, + 1001, + 1001, + 'P1Y2M3DT4H5M6S', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{4,5,6}', + '((3,7),(15,18))', + '08:00:2b:01:02:03', + '08:00:2b:01:02:03:04:05', + '999.99', + '123', + '123', + '((3,7),(15,18))', + B'101', + '7/A25801C8'::pg_lsn, + '(3,7)', + '((3,7),(15,18))', + 3.4145, + 3.4145, + - 32768, + - 32768, + 1, + 1, + 1, + B'101', + '13:00:01', + '13:00:01', + '13:00:01', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + 'infinity', + 'infinity', + 'infinity', + B'101', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + 'fat & (rat | cat)'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + XMLPARSE( + DOCUMENT 'Manual...' + ), + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + TRUE, + '13:00:01', + '13:00:01', + '{1,2,3}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + TRUE, + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((3,7),(15,18))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '1234', + '*', + '{asb12}', + '*', + '{asb12}', + 'abc', + 'abc', + '{asb12}', + '192.168/24', + '(0,0),0', + 9223372036854775807, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.24.10.0', + - 2147483648, + - 2147483648, + - 2147483648, + '-178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08-00-2b-01-02-04', + '08-00-2b-01-02-03-04-06', + '1,001.01', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(0,0)', + '((0,0),(0,0))', + 3.4145, + 3.4145, + 32767, + 32767, + 32767, + 32767, + 2147483647, + B'101', + '13:00:02+8', + '13:00:02+8', + '13:00:02+8', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + 'yes', + '13:00:00+8', + '13:00:00+8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + 'yes', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 3, + 0, + 'abcd', + '*', + '{asb12}', + '*', + '{asb12}', + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть на південь, не питай чому;', + '{asb12}', + '192.168.1', + '(-10,-4),10', + 0, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '-1,000', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + 0, + 0, + 0, + B'101', + '13:00:03-8', + '13:00:03-8', + '13:00:03-8', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + '1', + '13:00:03-8', + '13:00:03-8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + '1', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 4, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '櫻花分店', + '櫻花分店', + '{asb12}', + '128.1', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '$999.99', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '13:00:04Z', + '13:00:04Z', + '13:00:04Z', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + FALSE, + '13:00:04Z', + '13:00:04Z', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + FALSE, + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 5, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '', + '', + '{asb12}', + '2001:4f8:3:ba::/64', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '$1001.01', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + 'no', + '13:00:05.012345Z+8', + '13:00:05.012345Z+8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + 'no', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 6, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '{asb12}', + '2001:4f8:3:ba::/64', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '-$1,000', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '13:00:00Z-8', + '13:00:00Z-8', + '13:00:00Z-8', + 'epoch', + 'epoch', + 'epoch', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + '0', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 7, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '{asb12}', + '2001:4f8:3:ba::/64', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '-$1,000', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '24:00:00', + '24:00:00', + '24:00:00', + 'epoch', + 'epoch', + 'epoch', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", +"language" => "English","ISBN-13" => "978-1449370000", +"weight" => "11.2 ounces"', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + '0', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml index 82acfa892470..a16927b98478 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml @@ -12,11 +12,11 @@ data: connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 maxSecondsBetweenMessages: 7200 - dockerImageTag: 2.0.33 + dockerImageTag: 3.1.5 dockerRepository: airbyte/source-postgres-strict-encrypt githubIssueLabel: source-postgres icon: postgresql.svg - license: MIT + license: ELv2 name: Postgres releaseStage: generally_available documentationUrl: https://docs.airbyte.com/integrations/sources/postgres diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json index add556c58b41..44639f1f5636 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json @@ -230,13 +230,13 @@ "group": "advanced", "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "title": "Standard (Xmin)", + "description": "Xmin replication requires no setup on the DB side but will not be able to represent deletions incrementally.", "required": ["method"], "properties": { "method": { "type": "string", - "const": "Standard", + "const": "Xmin", "order": 0 } } @@ -302,6 +302,18 @@ "order": 7 } } + }, + { + "title": "Standard", + "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Standard", + "order": 8 + } + } } ] }, @@ -309,6 +321,7 @@ "type": "object", "title": "SSH Tunnel Method", "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "group": "security", "oneOf": [ { "title": "No Tunnel", diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index dafba4335e03..768f54c2bed4 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.33 +LABEL io.airbyte.version=3.1.5 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index df176559bffd..02b32443345c 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-postgres:dev +# test_strictness_level: high # Uncomment this line to enable high strictness level, requires: https://github.com/airbytehq/airbyte/pull/27872 custom_environment_variables: USE_STREAM_CAPABLE_STATE: true acceptance_tests: @@ -24,6 +25,8 @@ acceptance_tests: tests: - config_path: "secrets/config.json" - config_path: "secrets/config_cdc.json" + backward_compatibility_tests_config: + disable_for_version: "2.1.1" basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index be730c71e81a..d0997a9ff1be 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -15,6 +15,8 @@ application { } dependencies { + implementation 'io.airbyte:airbyte-cdk:0.0.2' + implementation project(':airbyte-db:db-lib') implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:bases:debezium') diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json index ec1eec43d6a7..c3e6b23a2b0d 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/abnormal_state.json @@ -7,7 +7,7 @@ "stream_namespace": "public", "cursor_field": ["id"], "cursor": "4", - "cursor_record_count":1 + "cursor_record_count": 1 }, "stream_descriptor": { "name": "id_and_name", diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json index fe8fa5fc2372..648876fb50e1 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/incremental_configured_catalog.json @@ -1,32 +1,29 @@ { - "streams": [ - { - "stream": { - "name": "id_and_name", - "json_schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "id": { - "type": "number", - "airbyte_type": "integer" - } - } - }, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "default_cursor_field": [], - "source_defined_primary_key": [], - "namespace": "public" + "streams": [ + { + "stream": { + "name": "id_and_name", + "json_schema": { + "type": "object", + "properties": { + "name": { + "type": "string" }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["id"], - "user_defined_primary_key": ["id"] - } - ] + "id": { + "type": "number", + "airbyte_type": "integer" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": [], + "source_defined_primary_key": [], + "namespace": "public" + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["id"], + "user_defined_primary_key": ["id"] + } + ] } diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/basic.sql index 3adc7ca0b57a..d7cb4fda899e 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/basic.sql @@ -1,17 +1,334 @@ -CREATE SCHEMA POSTGRES_BASIC; - -CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); -CREATE TYPE inventory_item AS (name text, supplier_id integer, price numeric); -SET lc_monetary TO 'en_US.utf8'; -SET TIMEZONE TO 'MST'; -CREATE EXTENSION hstore; - -CREATE TABLE POSTGRES_BASIC.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bigint,test_column_11 char,test_column_12 char(8),test_column_13 character,test_column_14 character(8),test_column_15 text,test_column_16 varchar,test_column_20 date not null default now(),test_column_21 date,test_column_23 float,test_column_24 double precision,test_column_27 int,test_column_28 integer,test_column_3 BIT(1),test_column_4 BIT(3),test_column_44 real,test_column_46 smallint,test_column_51 time without time zone,test_column_52 time,test_column_53 time without time zone not null default now(),test_column_54 timestamp,test_column_55 timestamp without time zone,test_column_56 timestamp without time zone default now(),test_column_57 timestamp,test_column_58 timestamp without time zone,test_column_59 timestamp without time zone not null default now(),test_column_60 timestamp with time zone,test_column_61 timestamptz,test_column_7 bool,test_column_70 time with time zone,test_column_71 timetz,test_column_8 boolean ); - -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (1, -9223372036854775808, 'a', '{asb123}', 'a', '{asb123}', 'a', 'a', '1999-01-08', '1999-01-08', '123', '123', 1001, 1001, B'0', B'101', 3.4145, -32768, '13:00:01', '13:00:01', '13:00:01', TIMESTAMP '2004-10-19 10:23:00', TIMESTAMP '2004-10-19 10:23:00', TIMESTAMP '2004-10-19 10:23:00', 0, 0, 0, TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', true, '13:00:01', '13:00:01', true); -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (2, 9223372036854775807, '*', '{asb12}', '*', '{asb12}', 'abc', 'abc', '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', -2147483648, -2147483648, B'0', B'101', 3.4145, 32767, '13:00:02+8', '13:00:02+8', '13:00:02+8', TIMESTAMP '2004-10-19 10:23:54.123456', TIMESTAMP '2004-10-19 10:23:54.123456', TIMESTAMP '2004-10-19 10:23:54.123456', 0, 0, 0, TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', 'yes', '13:00:00+8', '13:00:00+8', 'yes'); -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (3, 0, '*', '{asb12}', '*', '{asb12}', 'Миші йдуть на південь, не питай чому;', 'Миші йдуть на південь, не питай чому;', '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', 2147483647, 2147483647, B'0', B'101', 3.4145, 32767, '13:00:03-8', '13:00:03-8', '13:00:03-8', TIMESTAMP '3004-10-19 10:23:54.123456 BC', TIMESTAMP '3004-10-19 10:23:54.123456 BC', TIMESTAMP '3004-10-19 10:23:54.123456 BC', 0, 0, 0, TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', '1', '13:00:03-8', '13:00:03-8', '1'); -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (4, 0, '*', '{asb12}', '*', '{asb12}', '櫻花分店', '櫻花分店', '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', 2147483647, 2147483647, B'0', B'101', 3.4145, 32767, '13:00:04Z', '13:00:04Z', '13:00:04Z', TIMESTAMP '0001-01-01 00:00:00.000000', TIMESTAMP '0001-01-01 00:00:00.000000', TIMESTAMP '0001-01-01 00:00:00.000000', 0, 0, 0, TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', false, '13:00:04Z', '13:00:04Z', false); -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (5, 0, '*', '{asb12}', '*', '{asb12}', '', '', '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', 2147483647, 2147483647, B'0', B'101', 3.4145, 32767, '13:00:05.01234Z+8', '13:00:05.01234Z+8', '13:00:05.01234Z+8', TIMESTAMP '0001-12-31 23:59:59.999999 BC', TIMESTAMP '0001-12-31 23:59:59.999999 BC', TIMESTAMP '0001-12-31 23:59:59.999999 BC', 0, 0, 0, TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', 'no', '13:00:05.012345Z+8', '13:00:05.012345Z+8', 'no'); -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (6, 0, '*', '{asb12}', '*', '{asb12}', '\xF0\x9F\x9A\x80', '\xF0\x9F\x9A\x80', '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', 2147483647, 2147483647, B'0', B'101', 3.4145, 32767, '13:00:00Z-8', '13:00:00Z-8', '13:00:00Z-8', 'epoch', 'epoch', 'epoch', 0, 0, 0, TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', '0', '13:00:06.00000Z-8', '13:00:06.00000Z-8', '0'); -INSERT INTO POSTGRES_BASIC.TEST_DATASET VALUES (7, 0, '*', '{asb12}', '*', '{asb12}', '\xF0\x9F\x9A\x80', '\xF0\x9F\x9A\x80', '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', 2147483647, 2147483647, B'0', B'101', 3.4145, 32767, '24:00:00', '24:00:00', '24:00:00', 'epoch', 'epoch', 'epoch', 0, 0, 0, TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', '0', '13:00:06.00000Z-8', '13:00:06.00000Z-8', '0'); +CREATE + SCHEMA POSTGRES_BASIC; + +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); + +CREATE + TYPE inventory_item AS( + name text, + supplier_id INTEGER, + price NUMERIC + ); +SET +lc_monetary TO 'en_US.utf8'; +SET +TIMEZONE TO 'MST'; + +CREATE + EXTENSION hstore; + +CREATE + TABLE + POSTGRES_BASIC.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_11 CHAR, + test_column_12 CHAR(8), + test_column_13 CHARACTER, + test_column_14 CHARACTER(8), + test_column_15 text, + test_column_16 VARCHAR, + test_column_20 DATE NOT NULL DEFAULT now(), + test_column_21 DATE, + test_column_23 FLOAT, + test_column_24 DOUBLE PRECISION, + test_column_27 INT, + test_column_28 INTEGER, + test_column_3 BIT(1), + test_column_4 BIT(3), + test_column_44 REAL, + test_column_46 SMALLINT, + test_column_51 TIME WITHOUT TIME ZONE, + test_column_52 TIME, + test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_54 TIMESTAMP, + test_column_55 TIMESTAMP WITHOUT TIME ZONE, + test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + test_column_57 TIMESTAMP, + test_column_58 TIMESTAMP WITHOUT TIME ZONE, + test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_60 TIMESTAMP WITH TIME ZONE, + test_column_61 timestamptz, + test_column_7 bool, + test_column_70 TIME WITH TIME ZONE, + test_column_71 timetz, + test_column_8 BOOLEAN + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + 'a', + '{asb123}', + 'a', + '{asb123}', + 'a', + 'a', + '1999-01-08', + '1999-01-08', + '123', + '123', + 1001, + 1001, + B'0', + B'101', + 3.4145, + - 32768, + '13:00:01', + '13:00:01', + '13:00:01', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TRUE, + '13:00:01', + '13:00:01', + TRUE + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '*', + '{asb12}', + '*', + '{asb12}', + 'abc', + 'abc', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + - 2147483648, + - 2147483648, + B'0', + B'101', + 3.4145, + 32767, + '13:00:02+8', + '13:00:02+8', + '13:00:02+8', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + 'yes', + '13:00:00+8', + '13:00:00+8', + 'yes' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 3, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть на південь, не питай чому;', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:03-8', + '13:00:03-8', + '13:00:03-8', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + '1', + '13:00:03-8', + '13:00:03-8', + '1' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 4, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '櫻花分店', + '櫻花分店', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:04Z', + '13:00:04Z', + '13:00:04Z', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + FALSE, + '13:00:04Z', + '13:00:04Z', + FALSE + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 5, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '', + '', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'no', + '13:00:05.012345Z+8', + '13:00:05.012345Z+8', + 'no' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 6, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '13:00:00Z-8', + '13:00:00Z-8', + '13:00:00Z-8', + 'epoch', + 'epoch', + 'epoch', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '0' + ); + +INSERT + INTO + POSTGRES_BASIC.TEST_DATASET + VALUES( + 7, + 0, + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + 2147483647, + 2147483647, + B'0', + B'101', + 3.4145, + 32767, + '24:00:00', + '24:00:00', + '24:00:00', + 'epoch', + 'epoch', + 'epoch', + 0, + 0, + 0, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '0' + ); diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full.sql index 7947393a2987..26edf7749a9f 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full.sql +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full.sql @@ -1,20 +1,825 @@ -CREATE SCHEMA POSTGRES_FULL; +CREATE + SCHEMA POSTGRES_FULL; -CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); -CREATE TYPE inventory_item AS (name text, supplier_id integer, price numeric); -SET lc_monetary TO 'en_US.utf8'; -SET TIMEZONE TO 'MST'; -CREATE EXTENSION hstore; +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); -CREATE TABLE POSTGRES_FULL.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bigint,test_column_10 bytea,test_column_11 char,test_column_12 char(8),test_column_13 character,test_column_14 character(8),test_column_15 text,test_column_16 varchar,test_column_17 character varying(10),test_column_18 cidr,test_column_19 circle,test_column_2 bigserial,test_column_20 date not null default now(),test_column_21 date,test_column_22 float8,test_column_23 float,test_column_24 double precision,test_column_25 inet,test_column_26 int4,test_column_27 int,test_column_28 integer,test_column_29 interval,test_column_3 BIT(1),test_column_30 json,test_column_31 jsonb,test_column_32 line,test_column_33 lseg,test_column_34 macaddr,test_column_35 macaddr8,test_column_36 money,test_column_37 decimal,test_column_38 numeric,test_column_39 path,test_column_4 BIT(3),test_column_40 pg_lsn,test_column_41 point,test_column_42 polygon,test_column_43 float4,test_column_44 real,test_column_45 int2,test_column_46 smallint,test_column_47 serial2,test_column_48 smallserial,test_column_49 serial4,test_column_5 BIT VARYING(5),test_column_51 time without time zone,test_column_52 time,test_column_53 time without time zone not null default now(),test_column_54 timestamp,test_column_55 timestamp without time zone,test_column_56 timestamp without time zone default now(),test_column_57 timestamp,test_column_58 timestamp without time zone,test_column_59 timestamp without time zone not null default now(),test_column_6 BIT VARYING(5),test_column_60 timestamp with time zone,test_column_61 timestamptz,test_column_62 tsquery,test_column_63 tsvector,test_column_64 uuid,test_column_65 xml,test_column_66 mood,test_column_67 tsrange,test_column_68 inventory_item,test_column_69 hstore,test_column_7 bool,test_column_70 time with time zone,test_column_71 timetz,test_column_72 INT2[],test_column_73 INT4[],test_column_74 INT8[],test_column_75 OID[],test_column_76 VARCHAR[],test_column_77 CHAR(1)[],test_column_78 BPCHAR(2)[],test_column_79 TEXT[],test_column_8 boolean,test_column_80 NAME[],test_column_81 NUMERIC[],test_column_82 DECIMAL[],test_column_83 FLOAT4[],test_column_84 FLOAT8[],test_column_85 MONEY[],test_column_86 BOOL[],test_column_87 BIT[],test_column_88 BYTEA[],test_column_89 DATE[],test_column_9 box,test_column_90 TIME(6)[],test_column_91 TIMETZ[],test_column_92 TIMESTAMPTZ[],test_column_93 TIMESTAMP[] ); +CREATE + TYPE inventory_item AS( + name text, + supplier_id INTEGER, + price NUMERIC + ); +SET +lc_monetary TO 'en_US.utf8'; +SET +TIMEZONE TO 'MST'; -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (1, -9223372036854775808, null, 'a', '{asb123}', 'a', '{asb123}', 'a', 'a', '{asb123}', null, '(5,7),10', 1, '1999-01-08', '1999-01-08', '123', '123', '123', '198.24.10.0/24', null, null, null, null, B'0', null, null, '{4,5,6}', '((3,7),(15,18))', null, null, null, '123', '123', '((3,7),(15,18))', B'101', '7/A25801C8'::pg_lsn, '(3,7)', '((3,7),(15,18))', null, null, null, null, 1, 1, 1, B'101', '13:00:01', '13:00:01', '13:00:01', TIMESTAMP '2004-10-19 10:23:00', TIMESTAMP '2004-10-19 10:23:00', TIMESTAMP '2004-10-19 10:23:00', 'infinity', 'infinity', 'infinity', B'101', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', null, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', XMLPARSE (DOCUMENT 'Manual...'), 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +CREATE + EXTENSION hstore; + +CREATE + TABLE + POSTGRES_FULL.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 bytea, + test_column_11 CHAR, + test_column_12 CHAR(8), + test_column_13 CHARACTER, + test_column_14 CHARACTER(8), + test_column_15 text, + test_column_16 VARCHAR, + test_column_17 CHARACTER VARYING(10), + test_column_18 cidr, + test_column_19 circle, + test_column_2 bigserial, + test_column_20 DATE NOT NULL DEFAULT now(), + test_column_21 DATE, + test_column_22 float8, + test_column_23 FLOAT, + test_column_24 DOUBLE PRECISION, + test_column_25 inet, + test_column_26 int4, + test_column_27 INT, + test_column_28 INTEGER, + test_column_29 INTERVAL, + test_column_3 BIT(1), + test_column_30 json, + test_column_31 jsonb, + test_column_32 line, + test_column_33 lseg, + test_column_34 macaddr, + test_column_35 macaddr8, + test_column_36 money, + test_column_37 DECIMAL, + test_column_38 NUMERIC, + test_column_39 PATH, + test_column_4 BIT(3), + test_column_40 pg_lsn, + test_column_41 point, + test_column_42 polygon, + test_column_43 float4, + test_column_44 REAL, + test_column_45 int2, + test_column_46 SMALLINT, + test_column_47 serial2, + test_column_48 smallserial, + test_column_49 serial4, + test_column_5 BIT VARYING(5), + test_column_51 TIME WITHOUT TIME ZONE, + test_column_52 TIME, + test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_54 TIMESTAMP, + test_column_55 TIMESTAMP WITHOUT TIME ZONE, + test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + test_column_57 TIMESTAMP, + test_column_58 TIMESTAMP WITHOUT TIME ZONE, + test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_6 BIT VARYING(5), + test_column_60 TIMESTAMP WITH TIME ZONE, + test_column_61 timestamptz, + test_column_62 tsquery, + test_column_63 tsvector, + test_column_64 uuid, + test_column_65 xml, + test_column_66 mood, + test_column_67 tsrange, + test_column_68 inventory_item, + test_column_69 hstore, + test_column_7 bool, + test_column_70 TIME WITH TIME ZONE, + test_column_71 timetz, + test_column_72 INT2 [], + test_column_73 INT4 [], + test_column_74 INT8 [], + test_column_75 OID [], + test_column_76 VARCHAR [], + test_column_77 CHAR(1)[], + test_column_78 BPCHAR(2)[], + test_column_79 TEXT [], + test_column_8 BOOLEAN, + test_column_80 NAME [], + test_column_81 NUMERIC [], + test_column_82 DECIMAL [], + test_column_83 FLOAT4 [], + test_column_84 FLOAT8 [], + test_column_85 MONEY [], + test_column_86 BOOL [], + test_column_87 BIT [], + test_column_88 BYTEA [], + test_column_89 DATE [], + test_column_9 box, + test_column_90 TIME(6)[], + test_column_91 TIMETZ [], + test_column_92 TIMESTAMPTZ [], + test_column_93 TIMESTAMP [] + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + NULL, + 'a', + '{asb123}', + 'a', + '{asb123}', + 'a', + 'a', + '{asb123}', + NULL, + '(5,7),10', + 1, + '1999-01-08', + '1999-01-08', + '123', + '123', + '123', + '198.24.10.0/24', + NULL, + NULL, + NULL, + NULL, + B'0', + NULL, + NULL, + '{4,5,6}', + '((3,7),(15,18))', + NULL, + NULL, + NULL, + '123', + '123', + '((3,7),(15,18))', + B'101', + '7/A25801C8'::pg_lsn, + '(3,7)', + '((3,7),(15,18))', + NULL, + NULL, + NULL, + NULL, + 1, + 1, + 1, + B'101', + '13:00:01', + '13:00:01', + '13:00:01', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + 'infinity', + 'infinity', + 'infinity', + B'101', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + NULL, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + XMLPARSE( + DOCUMENT 'Manual...' + ), + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, true, null, null, '{1,2,3}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', true, '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((3,7),(15,18))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (2, 9223372036854775807, decode('1234', 'hex'), '*', '{asb12}', '*', '{asb12}', 'abc', 'abc', '{asb12}', '192.168.100.128/25', '(0,0),0', 9223372036854775807, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.24.10.0', 1001, 1001, 1001, 'P1Y2M3DT4H5M6S', null, '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08:00:2b:01:02:03', '08:00:2b:01:02:03:04:05', '999.99', null, null, '((0,0),(0,0))', null, '0/0'::pg_lsn, '(0,0)', '((0,0),(0,0))', 3.4145, 3.4145, -32768, -32768, 32767, 32767, 2147483647, null, '13:00:02+8', '13:00:02+8', '13:00:02+8', TIMESTAMP '2004-10-19 10:23:54.123456', TIMESTAMP '2004-10-19 10:23:54.123456', TIMESTAMP '2004-10-19 10:23:54.123456', '-infinity', '-infinity', '-infinity', null, TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', 'fat & (rat | cat)'::tsquery, null, null, null, null, null, null, null, 'yes', '13:00:01', '13:00:01', '{4,5,6}', null, null, null, null, null, null, null, 'yes', null, null, null, null, null, null, null, null, null, null, '((0,0),(0,0))', null, null, null, null); -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (3, 0, '1234', null, null, null, null, 'Миші йдуть на південь, не питай чому;', 'Миші йдуть на південь, не питай чому;', null, '192.168/24', '(-10,-4),10', 0, null, null, null, null, null, '198.10/8', -2147483648, -2147483648, -2147483648, '-178000000', null, null, null, null, null, '08-00-2b-01-02-04', '08-00-2b-01-02-03-04-06', '1,001.01', '1234567890.1234567', '1234567890.1234567', null, null, null, '(999999999999999999999999,0)', '((0,0),(999999999999999999999999,0))', null, null, 32767, 32767, 0, 0, 0, null, '13:00:03-8', '13:00:03-8', '13:00:03-8', TIMESTAMP '3004-10-19 10:23:54.123456 BC', TIMESTAMP '3004-10-19 10:23:54.123456 BC', TIMESTAMP '3004-10-19 10:23:54.123456 BC', null, null, null, null, TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', 'fat:ab & cat'::tsquery, null, null, '', null, null, null, null, '1', '13:00:00+8', '13:00:00+8', null, null, null, null, null, null, null, null, '1', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (4, null, 'abcd', null, null, null, null, '櫻花分店', '櫻花分店', null, '192.168.1', null, -9223372036854775808, null, null, null, null, null, null, 2147483647, 2147483647, 2147483647, '178000000', null, null, null, null, null, '08002b:010205', '08002b:0102030407', '-1,000', null, null, null, null, null, null, null, null, null, null, null, -32767, -32767, -2147483647, null, '13:00:04Z', '13:00:04Z', '13:00:04Z', TIMESTAMP '0001-01-01 00:00:00.000000', TIMESTAMP '0001-01-01 00:00:00.000000', TIMESTAMP '0001-01-01 00:00:00.000000', null, null, null, null, TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', null, null, null, null, null, null, null, null, false, '13:00:03-8', '13:00:03-8', null, null, null, null, null, null, null, null, false, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (5, null, '\xabcd', null, null, null, null, '', '', null, '128.1', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '$999.99', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '13:00:05.01234Z+8', '13:00:05.01234Z+8', '13:00:05.01234Z+8', TIMESTAMP '0001-12-31 23:59:59.999999 BC', TIMESTAMP '0001-12-31 23:59:59.999999 BC', TIMESTAMP '0001-12-31 23:59:59.999999 BC', null, null, null, null, TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', null, null, null, null, null, null, null, null, 'no', '13:00:04Z', '13:00:04Z', null, null, null, null, null, null, null, null, 'no', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (6, null, null, null, null, null, null, null, null, null, '2001:4f8:3:ba::/64', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '$1001.01', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '13:00:00Z-8', '13:00:00Z-8', '13:00:00Z-8', 'epoch', 'epoch', 'epoch', null, null, null, null, null, null, null, null, null, null, null, null, null, null, '0', '13:00:05.012345Z+8', '13:00:05.012345Z+8', null, null, null, null, null, null, null, null, '0', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); -INSERT INTO POSTGRES_FULL.TEST_DATASET VALUES (7, null, null, null, null, null, null, '\xF0\x9F\x9A\x80', '\xF0\x9F\x9A\x80', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '-$1,000', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '24:00:00', '24:00:00', '24:00:00', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, '13:00:06.00000Z-8', '13:00:06.00000Z-8', null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null); +"weight" => "11.2 ounces"', + TRUE, + NULL, + NULL, + '{1,2,3}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + TRUE, + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((3,7),(15,18))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + decode( + '1234', + 'hex' + ), + '*', + '{asb12}', + '*', + '{asb12}', + 'abc', + 'abc', + '{asb12}', + '192.168.100.128/25', + '(0,0),0', + 9223372036854775807, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.24.10.0', + 1001, + 1001, + 1001, + 'P1Y2M3DT4H5M6S', + NULL, + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08:00:2b:01:02:03', + '08:00:2b:01:02:03:04:05', + '999.99', + NULL, + NULL, + '((0,0),(0,0))', + NULL, + '0/0'::pg_lsn, + '(0,0)', + '((0,0),(0,0))', + 3.4145, + 3.4145, + - 32768, + - 32768, + 32767, + 32767, + 2147483647, + NULL, + '13:00:02+8', + '13:00:02+8', + '13:00:02+8', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + '-infinity', + '-infinity', + '-infinity', + NULL, + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + 'fat & (rat | cat)'::tsquery, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'yes', + '13:00:01', + '13:00:01', + '{4,5,6}', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'yes', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '((0,0),(0,0))', + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 3, + 0, + '1234', + NULL, + NULL, + NULL, + NULL, + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть на південь, не питай чому;', + NULL, + '192.168/24', + '(-10,-4),10', + 0, + NULL, + NULL, + NULL, + NULL, + NULL, + '198.10/8', + - 2147483648, + - 2147483648, + - 2147483648, + '-178000000', + NULL, + NULL, + NULL, + NULL, + NULL, + '08-00-2b-01-02-04', + '08-00-2b-01-02-03-04-06', + '1,001.01', + '1234567890.1234567', + '1234567890.1234567', + NULL, + NULL, + NULL, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + NULL, + NULL, + 32767, + 32767, + 0, + 0, + 0, + NULL, + '13:00:03-8', + '13:00:03-8', + '13:00:03-8', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + NULL, + NULL, + NULL, + NULL, + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + 'fat:ab & cat'::tsquery, + NULL, + NULL, + '', + NULL, + NULL, + NULL, + NULL, + '1', + '13:00:00+8', + '13:00:00+8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '1', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 4, + NULL, + 'abcd', + NULL, + NULL, + NULL, + NULL, + '櫻花分店', + '櫻花分店', + NULL, + '192.168.1', + NULL, + - 9223372036854775808, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 2147483647, + 2147483647, + 2147483647, + '178000000', + NULL, + NULL, + NULL, + NULL, + NULL, + '08002b:010205', + '08002b:0102030407', + '-1,000', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + - 32767, + - 32767, + - 2147483647, + NULL, + '13:00:04Z', + '13:00:04Z', + '13:00:04Z', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + NULL, + NULL, + NULL, + NULL, + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + FALSE, + '13:00:03-8', + '13:00:03-8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + FALSE, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 5, + NULL, + '\xabcd', + NULL, + NULL, + NULL, + NULL, + '', + '', + NULL, + '128.1', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '$999.99', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + NULL, + NULL, + NULL, + NULL, + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'no', + '13:00:04Z', + '13:00:04Z', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + 'no', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 6, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '2001:4f8:3:ba::/64', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '$1001.01', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '13:00:00Z-8', + '13:00:00Z-8', + '13:00:00Z-8', + 'epoch', + 'epoch', + 'epoch', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '0', + '13:00:05.012345Z+8', + '13:00:05.012345Z+8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '0', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); + +INSERT + INTO + POSTGRES_FULL.TEST_DATASET + VALUES( + 7, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '-$1,000', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '24:00:00', + '24:00:00', + '24:00:00', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ); diff --git a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full_without_nulls.sql index 154b7c9fe637..f101c9b7a3fa 100644 --- a/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full_without_nulls.sql +++ b/airbyte-integrations/connectors/source-postgres/integration_tests/seed/full_without_nulls.sql @@ -1,38 +1,861 @@ -CREATE SCHEMA POSTGRES_FULL_NN; +CREATE + SCHEMA POSTGRES_FULL_NN; -CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy'); -CREATE TYPE inventory_item AS (name text, supplier_id integer, price numeric); -SET lc_monetary TO 'en_US.utf8'; -SET TIMEZONE TO 'MST'; -CREATE EXTENSION hstore; +CREATE + TYPE mood AS ENUM( + 'sad', + 'ok', + 'happy' + ); -CREATE TABLE POSTGRES_FULL_NN.TEST_DATASET(id INTEGER PRIMARY KEY, test_column_1 bigint,test_column_10 bytea,test_column_11 char,test_column_12 char(8),test_column_13 character,test_column_14 character(8),test_column_15 text,test_column_16 varchar,test_column_17 character varying(10),test_column_18 cidr,test_column_19 circle,test_column_2 bigserial,test_column_20 date not null default now(),test_column_21 date,test_column_22 float8,test_column_23 float,test_column_24 double precision,test_column_25 inet,test_column_26 int4,test_column_27 int,test_column_28 integer,test_column_29 interval,test_column_3 BIT(1),test_column_30 json,test_column_31 jsonb,test_column_32 line,test_column_33 lseg,test_column_34 macaddr,test_column_35 macaddr8,test_column_36 money,test_column_37 decimal,test_column_38 numeric,test_column_39 path,test_column_4 BIT(3),test_column_40 pg_lsn,test_column_41 point,test_column_42 polygon,test_column_43 float4,test_column_44 real,test_column_45 int2,test_column_46 smallint,test_column_47 serial2,test_column_48 smallserial,test_column_49 serial4,test_column_5 BIT VARYING(5),test_column_51 time without time zone,test_column_52 time,test_column_53 time without time zone not null default now(),test_column_54 timestamp,test_column_55 timestamp without time zone,test_column_56 timestamp without time zone default now(),test_column_57 timestamp,test_column_58 timestamp without time zone,test_column_59 timestamp without time zone not null default now(),test_column_6 BIT VARYING(5),test_column_60 timestamp with time zone,test_column_61 timestamptz,test_column_62 tsquery,test_column_63 tsvector,test_column_64 uuid,test_column_65 xml,test_column_66 mood,test_column_67 tsrange,test_column_68 inventory_item,test_column_69 hstore,test_column_7 bool,test_column_70 time with time zone,test_column_71 timetz,test_column_72 INT2[],test_column_73 INT4[],test_column_74 INT8[],test_column_75 OID[],test_column_76 VARCHAR[],test_column_77 CHAR(1)[],test_column_78 BPCHAR(2)[],test_column_79 TEXT[],test_column_8 boolean,test_column_80 NAME[],test_column_81 NUMERIC[],test_column_82 DECIMAL[],test_column_83 FLOAT4[],test_column_84 FLOAT8[],test_column_85 MONEY[],test_column_86 BOOL[],test_column_87 BIT[],test_column_88 BYTEA[],test_column_89 DATE[],test_column_9 box,test_column_90 TIME(6)[],test_column_91 TIMETZ[],test_column_92 TIMESTAMPTZ[],test_column_93 TIMESTAMP[] ); +CREATE + TYPE inventory_item AS( + name text, + supplier_id INTEGER, + price NUMERIC + ); +SET +lc_monetary TO 'en_US.utf8'; +SET +TIMEZONE TO 'MST'; -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (1, -9223372036854775808, decode('1234', 'hex'), 'a', '{asb123}', 'a', '{asb123}', 'a', 'a', '{asb123}', '192.168.100.128/25', '(5,7),10', 1, '1999-01-08', '1999-01-08', '123', '123', '123', '198.24.10.0/24', 1001, 1001, 1001, 'P1Y2M3DT4H5M6S', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{4,5,6}', '((3,7),(15,18))', '08:00:2b:01:02:03', '08:00:2b:01:02:03:04:05', '999.99', '123', '123', '((3,7),(15,18))', B'101', '7/A25801C8'::pg_lsn, '(3,7)', '((3,7),(15,18))', 3.4145, 3.4145, -32768, -32768, 1, 1, 1, B'101', '13:00:01', '13:00:01', '13:00:01', TIMESTAMP '2004-10-19 10:23:00', TIMESTAMP '2004-10-19 10:23:00', TIMESTAMP '2004-10-19 10:23:00', 'infinity', 'infinity', 'infinity', B'101', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', 'fat & (rat | cat)'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', XMLPARSE (DOCUMENT 'Manual...'), 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +CREATE + EXTENSION hstore; + +CREATE + TABLE + POSTGRES_FULL_NN.TEST_DATASET( + id INTEGER PRIMARY KEY, + test_column_1 BIGINT, + test_column_10 bytea, + test_column_11 CHAR, + test_column_12 CHAR(8), + test_column_13 CHARACTER, + test_column_14 CHARACTER(8), + test_column_15 text, + test_column_16 VARCHAR, + test_column_17 CHARACTER VARYING(10), + test_column_18 cidr, + test_column_19 circle, + test_column_2 bigserial, + test_column_20 DATE NOT NULL DEFAULT now(), + test_column_21 DATE, + test_column_22 float8, + test_column_23 FLOAT, + test_column_24 DOUBLE PRECISION, + test_column_25 inet, + test_column_26 int4, + test_column_27 INT, + test_column_28 INTEGER, + test_column_29 INTERVAL, + test_column_3 BIT(1), + test_column_30 json, + test_column_31 jsonb, + test_column_32 line, + test_column_33 lseg, + test_column_34 macaddr, + test_column_35 macaddr8, + test_column_36 money, + test_column_37 DECIMAL, + test_column_38 NUMERIC, + test_column_39 PATH, + test_column_4 BIT(3), + test_column_40 pg_lsn, + test_column_41 point, + test_column_42 polygon, + test_column_43 float4, + test_column_44 REAL, + test_column_45 int2, + test_column_46 SMALLINT, + test_column_47 serial2, + test_column_48 smallserial, + test_column_49 serial4, + test_column_5 BIT VARYING(5), + test_column_51 TIME WITHOUT TIME ZONE, + test_column_52 TIME, + test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_54 TIMESTAMP, + test_column_55 TIMESTAMP WITHOUT TIME ZONE, + test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), + test_column_57 TIMESTAMP, + test_column_58 TIMESTAMP WITHOUT TIME ZONE, + test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), + test_column_6 BIT VARYING(5), + test_column_60 TIMESTAMP WITH TIME ZONE, + test_column_61 timestamptz, + test_column_62 tsquery, + test_column_63 tsvector, + test_column_64 uuid, + test_column_65 xml, + test_column_66 mood, + test_column_67 tsrange, + test_column_68 inventory_item, + test_column_69 hstore, + test_column_7 bool, + test_column_70 TIME WITH TIME ZONE, + test_column_71 timetz, + test_column_72 INT2 [], + test_column_73 INT4 [], + test_column_74 INT8 [], + test_column_75 OID [], + test_column_76 VARCHAR [], + test_column_77 CHAR(1)[], + test_column_78 BPCHAR(2)[], + test_column_79 TEXT [], + test_column_8 BOOLEAN, + test_column_80 NAME [], + test_column_81 NUMERIC [], + test_column_82 DECIMAL [], + test_column_83 FLOAT4 [], + test_column_84 FLOAT8 [], + test_column_85 MONEY [], + test_column_86 BOOL [], + test_column_87 BIT [], + test_column_88 BYTEA [], + test_column_89 DATE [], + test_column_9 box, + test_column_90 TIME(6)[], + test_column_91 TIMETZ [], + test_column_92 TIMESTAMPTZ [], + test_column_93 TIMESTAMP [] + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 1, + - 9223372036854775808, + decode( + '1234', + 'hex' + ), + 'a', + '{asb123}', + 'a', + '{asb123}', + 'a', + 'a', + '{asb123}', + '192.168.100.128/25', + '(5,7),10', + 1, + '1999-01-08', + '1999-01-08', + '123', + '123', + '123', + '198.24.10.0/24', + 1001, + 1001, + 1001, + 'P1Y2M3DT4H5M6S', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{4,5,6}', + '((3,7),(15,18))', + '08:00:2b:01:02:03', + '08:00:2b:01:02:03:04:05', + '999.99', + '123', + '123', + '((3,7),(15,18))', + B'101', + '7/A25801C8'::pg_lsn, + '(3,7)', + '((3,7),(15,18))', + 3.4145, + 3.4145, + - 32768, + - 32768, + 1, + 1, + 1, + B'101', + '13:00:01', + '13:00:01', + '13:00:01', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + TIMESTAMP '2004-10-19 10:23:00', + 'infinity', + 'infinity', + 'infinity', + B'101', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', + 'fat & (rat | cat)'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + XMLPARSE( + DOCUMENT 'Manual...' + ), + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, true, '13:00:01', '13:00:01', '{1,2,3}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', true, '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((3,7),(15,18))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (2, 9223372036854775807, '1234', '*', '{asb12}', '*', '{asb12}', 'abc', 'abc', '{asb12}', '192.168/24', '(0,0),0', 9223372036854775807, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.24.10.0', -2147483648, -2147483648, -2147483648, '-178000000', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08-00-2b-01-02-04', '08-00-2b-01-02-03-04-06', '1,001.01', '1234567890.1234567', '1234567890.1234567', '((0,0),(0,0))', B'101', '0/0'::pg_lsn, '(0,0)', '((0,0),(0,0))', 3.4145, 3.4145, 32767, 32767, 32767, 32767, 2147483647, B'101', '13:00:02+8', '13:00:02+8', '13:00:02+8', TIMESTAMP '2004-10-19 10:23:54.123456', TIMESTAMP '2004-10-19 10:23:54.123456', TIMESTAMP '2004-10-19 10:23:54.123456', '-infinity', '-infinity', '-infinity', B'101', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', 'fat:ab & cat'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '', 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +"weight" => "11.2 ounces"', + TRUE, + '13:00:01', + '13:00:01', + '{1,2,3}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + TRUE, + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((3,7),(15,18))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 2, + 9223372036854775807, + '1234', + '*', + '{asb12}', + '*', + '{asb12}', + 'abc', + 'abc', + '{asb12}', + '192.168/24', + '(0,0),0', + 9223372036854775807, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.24.10.0', + - 2147483648, + - 2147483648, + - 2147483648, + '-178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08-00-2b-01-02-04', + '08-00-2b-01-02-03-04-06', + '1,001.01', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(0,0)', + '((0,0),(0,0))', + 3.4145, + 3.4145, + 32767, + 32767, + 32767, + 32767, + 2147483647, + B'101', + '13:00:02+8', + '13:00:02+8', + '13:00:02+8', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + TIMESTAMP '2004-10-19 10:23:54.123456', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, 'yes', '13:00:00+8', '13:00:00+8', '{4,5,6}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', 'yes', '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((0,0),(0,0))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (3, 0, 'abcd', '*', '{asb12}', '*', '{asb12}', 'Миші йдуть на південь, не питай чому;', 'Миші йдуть на південь, не питай чому;', '{asb12}', '192.168.1', '(-10,-4),10', 0, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.10/8', 2147483647, 2147483647, 2147483647, '178000000', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08002b:010205', '08002b:0102030407', '-1,000', '1234567890.1234567', '1234567890.1234567', '((0,0),(0,0))', B'101', '0/0'::pg_lsn, '(999999999999999999999999,0)', '((0,0),(999999999999999999999999,0))', 3.4145, 3.4145, 32767, 32767, 0, 0, 0, B'101', '13:00:03-8', '13:00:03-8', '13:00:03-8', TIMESTAMP '3004-10-19 10:23:54.123456 BC', TIMESTAMP '3004-10-19 10:23:54.123456 BC', TIMESTAMP '3004-10-19 10:23:54.123456 BC', '-infinity', '-infinity', '-infinity', B'101', TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', 'fat:ab & cat'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '', 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +"weight" => "11.2 ounces"', + 'yes', + '13:00:00+8', + '13:00:00+8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + 'yes', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 3, + 0, + 'abcd', + '*', + '{asb12}', + '*', + '{asb12}', + 'Миші йдуть на південь, не питай чому;', + 'Миші йдуть на південь, не питай чому;', + '{asb12}', + '192.168.1', + '(-10,-4),10', + 0, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '-1,000', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + 0, + 0, + 0, + B'101', + '13:00:03-8', + '13:00:03-8', + '13:00:03-8', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + TIMESTAMP '3004-10-19 10:23:54.123456 BC', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, '1', '13:00:03-8', '13:00:03-8', '{4,5,6}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', '1', '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((0,0),(0,0))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (4, 0, '\xabcd', '*', '{asb12}', '*', '{asb12}', '櫻花分店', '櫻花分店', '{asb12}', '128.1', '(-10,-4),10', -9223372036854775808, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.10/8', 2147483647, 2147483647, 2147483647, '178000000', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08002b:010205', '08002b:0102030407', '$999.99', '1234567890.1234567', '1234567890.1234567', '((0,0),(0,0))', B'101', '0/0'::pg_lsn, '(999999999999999999999999,0)', '((0,0),(999999999999999999999999,0))', 3.4145, 3.4145, 32767, 32767, -32767, -32767, -2147483647, B'101', '13:00:04Z', '13:00:04Z', '13:00:04Z', TIMESTAMP '0001-01-01 00:00:00.000000', TIMESTAMP '0001-01-01 00:00:00.000000', TIMESTAMP '0001-01-01 00:00:00.000000', '-infinity', '-infinity', '-infinity', B'101', TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', 'fat:ab & cat'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '', 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +"weight" => "11.2 ounces"', + '1', + '13:00:03-8', + '13:00:03-8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + '1', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 4, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '櫻花分店', + '櫻花分店', + '{asb12}', + '128.1', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '$999.99', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '13:00:04Z', + '13:00:04Z', + '13:00:04Z', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + TIMESTAMP '0001-01-01 00:00:00.000000', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, false, '13:00:04Z', '13:00:04Z', '{4,5,6}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', false, '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((0,0),(0,0))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (5, 0, '\xabcd', '*', '{asb12}', '*', '{asb12}', '', '', '{asb12}', '2001:4f8:3:ba::/64', '(-10,-4),10', -9223372036854775808, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.10/8', 2147483647, 2147483647, 2147483647, '178000000', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08002b:010205', '08002b:0102030407', '$1001.01', '1234567890.1234567', '1234567890.1234567', '((0,0),(0,0))', B'101', '0/0'::pg_lsn, '(999999999999999999999999,0)', '((0,0),(999999999999999999999999,0))', 3.4145, 3.4145, 32767, 32767, -32767, -32767, -2147483647, B'101', '13:00:05.01234Z+8', '13:00:05.01234Z+8', '13:00:05.01234Z+8', TIMESTAMP '0001-12-31 23:59:59.999999 BC', TIMESTAMP '0001-12-31 23:59:59.999999 BC', TIMESTAMP '0001-12-31 23:59:59.999999 BC', '-infinity', '-infinity', '-infinity', B'101', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', 'fat:ab & cat'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '', 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +"weight" => "11.2 ounces"', + FALSE, + '13:00:04Z', + '13:00:04Z', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + FALSE, + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 5, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '', + '', + '{asb12}', + '2001:4f8:3:ba::/64', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '$1001.01', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + '13:00:05.01234Z+8', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + TIMESTAMP '0001-12-31 23:59:59.999999 BC', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, 'no', '13:00:05.012345Z+8', '13:00:05.012345Z+8', '{4,5,6}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', 'no', '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((0,0),(0,0))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (6, 0, '\xabcd', '*', '{asb12}', '*', '{asb12}', '\xF0\x9F\x9A\x80', '\xF0\x9F\x9A\x80', '{asb12}', '2001:4f8:3:ba::/64', '(-10,-4),10', -9223372036854775808, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.10/8', 2147483647, 2147483647, 2147483647, '178000000', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08002b:010205', '08002b:0102030407', '-$1,000', '1234567890.1234567', '1234567890.1234567', '((0,0),(0,0))', B'101', '0/0'::pg_lsn, '(999999999999999999999999,0)', '((0,0),(999999999999999999999999,0))', 3.4145, 3.4145, 32767, 32767, -32767, -32767, -2147483647, B'101', '13:00:00Z-8', '13:00:00Z-8', '13:00:00Z-8', 'epoch', 'epoch', 'epoch', '-infinity', '-infinity', '-infinity', B'101', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', 'fat:ab & cat'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '', 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +"weight" => "11.2 ounces"', + 'no', + '13:00:05.012345Z+8', + '13:00:05.012345Z+8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + 'no', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 6, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '{asb12}', + '2001:4f8:3:ba::/64', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '-$1,000', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '13:00:00Z-8', + '13:00:00Z-8', + '13:00:00Z-8', + 'epoch', + 'epoch', + 'epoch', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, '0', '13:00:06.00000Z-8', '13:00:06.00000Z-8', '{4,5,6}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', '0', '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((0,0),(0,0))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); -INSERT INTO POSTGRES_FULL_NN.TEST_DATASET VALUES (7, 0, '\xabcd', '*', '{asb12}', '*', '{asb12}', '\xF0\x9F\x9A\x80', '\xF0\x9F\x9A\x80', '{asb12}', '2001:4f8:3:ba::/64', '(-10,-4),10', -9223372036854775808, '1991-02-10 BC', '1991-02-10 BC', '1234567890.1234567', '1234567890.1234567', '1234567890.1234567', '198.10/8', 2147483647, 2147483647, 2147483647, '178000000', B'0', '{"a": 10, "b": 15}', '[1, 2, 3]'::jsonb, '{0,1,0}', '((0,0),(0,0))', '08002b:010205', '08002b:0102030407', '-$1,000', '1234567890.1234567', '1234567890.1234567', '((0,0),(0,0))', B'101', '0/0'::pg_lsn, '(999999999999999999999999,0)', '((0,0),(999999999999999999999999,0))', 3.4145, 3.4145, 32767, 32767, -32767, -32767, -2147483647, B'101', '24:00:00', '24:00:00', '24:00:00', 'epoch', 'epoch', 'epoch', '-infinity', '-infinity', '-infinity', B'101', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', 'fat:ab & cat'::tsquery, to_tsvector('The quick brown fox jumped over the lazy dog.'), 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '', 'happy', '(2010-01-01 14:30, 2010-01-01 15:30)', ROW('fuzzy dice', 42, 1.99), '"paperback" => "243","publisher" => "postgresqltutorial.com", +"weight" => "11.2 ounces"', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + '0', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); + +INSERT + INTO + POSTGRES_FULL_NN.TEST_DATASET + VALUES( + 7, + 0, + '\xabcd', + '*', + '{asb12}', + '*', + '{asb12}', + '\xF0\x9F\x9A\x80', + '\xF0\x9F\x9A\x80', + '{asb12}', + '2001:4f8:3:ba::/64', + '(-10,-4),10', + - 9223372036854775808, + '1991-02-10 BC', + '1991-02-10 BC', + '1234567890.1234567', + '1234567890.1234567', + '1234567890.1234567', + '198.10/8', + 2147483647, + 2147483647, + 2147483647, + '178000000', + B'0', + '{"a": 10, "b": 15}', + '[1, 2, 3]'::jsonb, + '{0,1,0}', + '((0,0),(0,0))', + '08002b:010205', + '08002b:0102030407', + '-$1,000', + '1234567890.1234567', + '1234567890.1234567', + '((0,0),(0,0))', + B'101', + '0/0'::pg_lsn, + '(999999999999999999999999,0)', + '((0,0),(999999999999999999999999,0))', + 3.4145, + 3.4145, + 32767, + 32767, + - 32767, + - 32767, + - 2147483647, + B'101', + '24:00:00', + '24:00:00', + '24:00:00', + 'epoch', + 'epoch', + 'epoch', + '-infinity', + '-infinity', + '-infinity', + B'101', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', + 'fat:ab & cat'::tsquery, + to_tsvector('The quick brown fox jumped over the lazy dog.'), + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + '', + 'happy', + '(2010-01-01 14:30, 2010-01-01 15:30)', + ROW( + 'fuzzy dice', + 42, + 1.99 + ), + '"paperback" => "243","publisher" => "postgresqltutorial.com", "language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"' -, '0', '13:00:06.00000Z-8', '13:00:06.00000Z-8', '{4,5,6}', '{-2147483648,2147483646}', '{-9223372036854775808,9223372036854775801}', '{564182,234181}', '{lorem ipsum,dolor sit,amet}', '{l,d,a}', '{l,d,a}', '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', '0', '{object,integer}', '{131070.23,231072.476596593}', '{131070.23,231072.476596593}', '{131070.237689,231072.476596593}', '{131070.237689,231072.476596593}', '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', '{true,yes,1,false,no,0,null}', '{null,1,0}', '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', '{1999-01-08,1991-02-10 BC}', '((0,0),(0,0))', '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}'); +"weight" => "11.2 ounces"', + '0', + '13:00:06.00000Z-8', + '13:00:06.00000Z-8', + '{4,5,6}', + '{-2147483648,2147483646}', + '{-9223372036854775808,9223372036854775801}', + '{564182,234181}', + '{lorem ipsum,dolor sit,amet}', + '{l,d,a}', + '{l,d,a}', + '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', + '0', + '{object,integer}', + '{131070.23,231072.476596593}', + '{131070.23,231072.476596593}', + '{131070.237689,231072.476596593}', + '{131070.237689,231072.476596593}', + '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', + '{true,yes,1,false,no,0,null}', + '{null,1,0}', + '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', + '{1999-01-08,1991-02-10 BC}', + '((0,0),(0,0))', + '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', + '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', + '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', + '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' + ); diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index c83052b5e149..dd20d7d2411d 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -6,12 +6,12 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 2.0.33 + dockerImageTag: 3.1.5 maxSecondsBetweenMessages: 7200 dockerRepository: airbyte/source-postgres githubIssueLabel: source-postgres icon: postgresql.svg - license: MIT + license: ELv2 name: Postgres registries: cloud: @@ -24,4 +24,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java index 538616a34265..a65ec1f6f081 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java @@ -4,8 +4,11 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; @@ -122,4 +125,13 @@ public static Set getPublicizedTables(final Jdbc return publicizedTables; } + /* + * To prepare for Destination v2, cdc streams must have a default cursor field this defaults to lsn + * as a cursor as it is monotonically increasing and unique + */ + public static AirbyteStream setDefaultCursorFieldForCdc(final AirbyteStream stream) { + stream.setDefaultCursorField(ImmutableList.of(CDC_LSN)); + return stream; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcConnectorMetadataInjector.java deleted file mode 100644 index e72682671b8f..000000000000 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcConnectorMetadataInjector.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.postgres; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.integrations.debezium.CdcMetadataInjector; -import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; - -public class PostgresCdcConnectorMetadataInjector implements CdcMetadataInjector { - - @Override - public void addMetaData(final ObjectNode event, final JsonNode source) { - final long lsn = source.get("lsn").asLong(); - event.put(DebeziumEventUtils.CDC_LSN, lsn); - } - - @Override - public String namespace(final JsonNode source) { - return source.get("schema").asText(); - } - -} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java index 7043037bd70f..60c3f2615fc5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java @@ -4,16 +4,30 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.integrations.source.postgres.xmin.XminStateManager.XMIN_STATE_VERSION; import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.CtidStreams; +import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +36,10 @@ */ public class PostgresQueryUtils { + public record TableBlockSize(Long tableSize, Long blockSize) {} + + public record ResultWithFailed (T result, List failed) {} + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresQueryUtils.class); public static final String NULL_CURSOR_VALUE_WITH_SCHEMA_QUERY = @@ -29,14 +47,14 @@ public class PostgresQueryUtils { SELECT (EXISTS (SELECT FROM information_schema.columns WHERE table_schema = '%s' AND table_name = '%s' AND is_nullable = 'YES' AND column_name = '%s')) AND - (EXISTS (SELECT from \"%s\".\"%s\" where \"%s\" IS NULL LIMIT 1)) AS %s + (EXISTS (SELECT from "%s"."%s" where "%s" IS NULL LIMIT 1)) AS %s """; public static final String NULL_CURSOR_VALUE_NO_SCHEMA_QUERY = """ SELECT (EXISTS (SELECT FROM information_schema.columns WHERE table_name = '%s' AND is_nullable = 'YES' AND column_name = '%s')) AND - (EXISTS (SELECT from \"%s\" where \"%s\" IS NULL LIMIT 1)) AS %s + (EXISTS (SELECT from "%s" where "%s" IS NULL LIMIT 1)) AS %s """; public static final String TABLE_ESTIMATE_QUERY = @@ -54,6 +72,10 @@ public class PostgresQueryUtils { (txid_snapshot_xmin(txid_current_snapshot()) % (2^32)::bigint) AS xmin_xid_value, txid_snapshot_xmin(txid_current_snapshot()) AS xmin_raw_value; """; + public static final String MAX_CURSOR_VALUE_QUERY = + """ + SELECT "%s" FROM %s WHERE "%s" = (SELECT MAX("%s") FROM %s); + """; public static final String CTID_FULL_VACUUM_IN_PROGRESS_QUERY = """ @@ -73,6 +95,16 @@ SELECT pg_relation_filenode('%s') public static final String TOTAL_BYTES_RESULT_COL = "totalbytes"; + /** + * Query returns the size table data takes on DB server disk (not incling any index or other + * metadata) And the size of each page used in (page, tuple) ctid. This helps us evaluate how many + * pages we need to read to traverse the entire table. + */ + public static final String CTID_TABLE_BLOCK_SIZE = + """ + WITH block_sz AS (SELECT current_setting('block_size')::int), rel_sz AS (select pg_relation_size('%s')) SELECT * from block_sz, rel_sz + """; + /** * Logs the current xmin status : 1. The number of wraparounds the source DB has undergone. (These * are the epoch bits in the xmin snapshot). 2. The 32-bit xmin value associated with the xmin @@ -89,36 +121,191 @@ public static XminStatus getXminStatus(final JdbcDatabase database) throws SQLEx return new XminStatus() .withNumWraparound(result.get(NUM_WRAPAROUND_COL).asLong()) .withXminXidValue(result.get(XMIN_XID_VALUE_COL).asLong()) - .withXminRawValue(result.get(XMIN_RAW_VALUE_COL).asLong()); + .withXminRawValue(result.get(XMIN_RAW_VALUE_COL).asLong()) + .withVersion(XMIN_STATE_VERSION) + .withStateType(StateType.XMIN); + } + + /** + * Iterates through each stream and find the max cursor value and the record count which has that + * value based on each cursor field provided by the customer per stream This information is saved in + * a Hashmap with the mapping being the AirbyteStreamNameNamespacepair -> CursorBasedStatus + * + * @param database the source db + * @param streams streams to be synced + * @param stateManager stream stateManager + * @return Map of streams to statuses + */ + public static Map getCursorBasedSyncStatusForStreams(final JdbcDatabase database, + final List streams, + final StateManager stateManager, + final String quoteString) { + + final Map cursorBasedStatusMap = new HashMap<>(); + streams.forEach(stream -> { + try { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(namespace, name, quoteString); + + final Optional cursorInfoOptional = + stateManager.getCursorInfo(new io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair(name, namespace)); + if (cursorInfoOptional.isEmpty()) { + throw new RuntimeException(String.format("Stream %s was not provided with an appropriate cursor", stream.getStream().getName())); + } + + LOGGER.info("Querying max cursor value for {}.{}", namespace, name); + final String cursorField = cursorInfoOptional.get().getCursorField(); + final String cursorBasedSyncStatusQuery = String.format(MAX_CURSOR_VALUE_QUERY, + cursorField, + fullTableName, + cursorField, + cursorField, + fullTableName); + LOGGER.debug("Querying for max cursor value: {}", cursorBasedSyncStatusQuery); + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(cursorBasedSyncStatusQuery).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + final CursorBasedStatus cursorBasedStatus = new CursorBasedStatus(); + cursorBasedStatus.setStateType(StateType.CURSOR_BASED); + cursorBasedStatus.setVersion(2L); + cursorBasedStatus.setStreamName(name); + cursorBasedStatus.setStreamNamespace(namespace); + cursorBasedStatus.setCursorField(ImmutableList.of(cursorField)); + + if (!jsonNodes.isEmpty()) { + final JsonNode result = jsonNodes.get(0); + cursorBasedStatus.setCursor(result.get(cursorField).asText()); + cursorBasedStatus.setCursorRecordCount((long) jsonNodes.size()); + } + + cursorBasedStatusMap.put(new AirbyteStreamNameNamespacePair(name, namespace), cursorBasedStatus); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + }); + + return cursorBasedStatusMap; + } + + public static ResultWithFailed> fileNodeForStreams(final JdbcDatabase database, + final List streams, + final String quoteString) { + final Map fileNodes = new HashMap<>(); + final List failedToQuery = new ArrayList<>(); + streams.forEach(stream -> { + try { + final AirbyteStreamNameNamespacePair namespacePair = + new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + final Optional fileNode = fileNodeForStreams(database, namespacePair, quoteString); + fileNode.ifPresentOrElse( + l -> fileNodes.put(namespacePair, l), + () -> failedToQuery.add(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream))); + } catch (final Exception e) { + LOGGER.warn("Failed to fetch relation node for {}.{} .", stream.getStream().getNamespace(), stream.getStream().getName(), e); + failedToQuery.add(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream)); + } + }); + return new ResultWithFailed<>(fileNodes, failedToQuery); + } + + public static Optional fileNodeForStreams(final JdbcDatabase database, final AirbyteStreamNameNamespacePair stream, final String quoteString) + throws SQLException { + final String streamName = stream.getName(); + final String schemaName = stream.getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(schemaName, streamName, quoteString); + final List jsonNodes = database.bufferedResultSetQuery( + conn -> conn.prepareStatement(CTID_FULL_VACUUM_REL_FILENODE_QUERY.formatted(fullTableName)).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + Long relationFilenode = null; + if (!jsonNodes.get(0).isEmpty()) { + relationFilenode = jsonNodes.get(0).get("pg_relation_filenode").asLong(); + LOGGER.info("Relation filenode is for stream {} is {}", fullTableName, relationFilenode); + } else { + LOGGER.debug("No filenode found for {}", fullTableName); + } + return Optional.ofNullable(relationFilenode); } - public static void logFullVacuumStatus(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final String quoteString) { - catalog.getStreams().forEach(stream -> { + public static ResultWithFailed> streamsUnderVacuum(final JdbcDatabase database, + final List streams, + final String quoteString) { + final List streamsUnderVacuuming = new ArrayList<>(); + final List failedToQuery = new ArrayList<>(); + streams.forEach(stream -> { final String streamName = stream.getStream().getName(); final String schemaName = stream.getStream().getNamespace(); final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, streamName, quoteString); - LOGGER.info("Full Vacuum information for {}", fullTableName); try { - List jsonNodes = database.bufferedResultSetQuery( - conn -> conn.prepareStatement(CTID_FULL_VACUUM_REL_FILENODE_QUERY.formatted(fullTableName)).executeQuery(), + final List jsonNodes = database.bufferedResultSetQuery( + conn -> conn.prepareStatement(CTID_FULL_VACUUM_IN_PROGRESS_QUERY.formatted(fullTableName)).executeQuery(), resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); - Preconditions.checkState(jsonNodes.size() == 1); - LOGGER.info("Relation filenode is {}", jsonNodes.get(0).get("pg_relation_filenode")); - - jsonNodes = - database.bufferedResultSetQuery(conn -> conn.prepareStatement(CTID_FULL_VACUUM_IN_PROGRESS_QUERY.formatted(fullTableName)).executeQuery(), - resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); - if (jsonNodes.size() == 0) { - LOGGER.info("No full vacuum currently in progress"); - } else { + if (jsonNodes.size() != 0) { Preconditions.checkState(jsonNodes.size() == 1); - LOGGER.info("Full Vacuum currently in progress in {} phase", jsonNodes.get(0).get("phase")); + LOGGER.warn("Full Vacuum currently in progress for table {} in {} phase, the table will be skipped from syncing data", fullTableName, + jsonNodes.get(0).get("phase")); + streamsUnderVacuuming.add(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream)); } - } catch (SQLException e) { - LOGGER.warn("Failed to log full vacuum in progress. This warning shouldn't affect the sync and can be ignored", e); + } catch (final Exception e) { + // Assume it's safe to progress and skip relation node and vaccuum validation + LOGGER.warn("Failed to fetch vacuum for table {} info", fullTableName, e); + failedToQuery.add(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream)); } }); + return new ResultWithFailed<>(streamsUnderVacuuming, failedToQuery); + } + + public static Map getTableBlockSizeForStreams(final JdbcDatabase database, + final List streams, + final String quoteString) { + final Map tableBlockSizes = new HashMap<>(); + streams.forEach(stream -> { + final AirbyteStreamNameNamespacePair namespacePair = + new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + final TableBlockSize sz = getTableBlockSizeForStream(database, namespacePair, quoteString); + tableBlockSizes.put(namespacePair, sz); + }); + return tableBlockSizes; + + } + + public static TableBlockSize getTableBlockSizeForStream(final JdbcDatabase database, + final AirbyteStreamNameNamespacePair stream, + final String quoteString) { + try { + final String streamName = stream.getName(); + final String schemaName = stream.getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(schemaName, streamName, quoteString); + final List jsonNodes = database.bufferedResultSetQuery( + conn -> conn.prepareStatement(CTID_TABLE_BLOCK_SIZE.formatted(fullTableName)).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + final long relationSize = jsonNodes.get(0).get("pg_relation_size").asLong(); + final long blockSize = jsonNodes.get(0).get("current_setting").asLong(); + LOGGER.info("Stream {} relation size is {}. block size {}", fullTableName, relationSize, blockSize); + return new TableBlockSize(relationSize, blockSize); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * Filter out streams that are currently under vacuum from being synced via Ctid + * + * @param streamsUnderVacuum streams that are currently under vacuum + * @param ctidStreams preliminary streams to be synced via Ctid + * @return ctid streams that are not under vacuum + */ + public static List filterStreamsUnderVacuumForCtidSync(final List streamsUnderVacuum, + final CtidStreams ctidStreams) { + return streamsUnderVacuum.isEmpty() ? List.copyOf(ctidStreams.streamsForCtidSync()) + : ctidStreams.streamsForCtidSync().stream() + .filter(c -> !streamsUnderVacuum.contains(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(c))) + .toList(); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index d7fd2e43e2d5..041a2ff5780c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -11,6 +11,7 @@ import static io.airbyte.db.jdbc.JdbcUtils.EQUALS; import static io.airbyte.db.jdbc.JdbcUtils.PLATFORM_DATA_INCREASE_FACTOR; import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; +import static io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER; import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.PARAM_CA_CERTIFICATE; @@ -20,7 +21,16 @@ import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.ROW_COUNT_RESULT_COL; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.TABLE_ESTIMATE_QUERY; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.TOTAL_BYTES_RESULT_COL; -import static io.airbyte.integrations.source.postgres.PostgresUtils.isIncrementalSyncMode; +import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.filterStreamsUnderVacuumForCtidSync; +import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.getCursorBasedSyncStatusForStreams; +import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.streamsUnderVacuum; +import static io.airbyte.integrations.source.postgres.PostgresUtils.isAnyStreamIncrementalSyncMode; +import static io.airbyte.integrations.source.postgres.PostgresUtils.prettyPrintConfiguredAirbyteStreamList; +import static io.airbyte.integrations.source.postgres.cdc.PostgresCdcCtidInitializer.cdcCtidIteratorsCombined; +import static io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.categoriseStreams; +import static io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.reclassifyCategorisedCtidStreams; +import static io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.categoriseStreams; +import static io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.reclassifyCategorisedCtidStreams; import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import static io.airbyte.integrations.util.PostgresSslConnectionUtils.PARAM_SSL_MODE; import static java.util.stream.Collectors.toList; @@ -38,26 +48,37 @@ import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; -import io.airbyte.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; import io.airbyte.integrations.debezium.internals.postgres.PostgresReplicationConnection; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils; import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils; import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils.ResultWithFailed; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; +import io.airbyte.integrations.source.postgres.ctid.CtidPerStreamStateManager; +import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations; +import io.airbyte.integrations.source.postgres.ctid.CtidStateManager; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; +import io.airbyte.integrations.source.postgres.ctid.PostgresCtidHandler; +import io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.CursorBasedStreams; +import io.airbyte.integrations.source.postgres.cursor_based.PostgresCursorBasedStateManager; +import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; import io.airbyte.integrations.source.postgres.xmin.PostgresXminHandler; +import io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.XminStreams; import io.airbyte.integrations.source.postgres.xmin.XminStateManager; import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.integrations.source.relationaldb.state.StateManager; @@ -80,18 +101,18 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.OptionalInt; -import java.util.OptionalLong; +import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.sql.DataSource; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -167,7 +188,7 @@ public JsonNode toDatabaseConfig(final JsonNode config) { additionalParameters.forEach(x -> jdbcUrl.append(x).append("&")); jdbcUrl.append(toJDBCQueryParams(sslParameters)); - LOGGER.debug("jdbc url: {}", jdbcUrl.toString()); + LOGGER.debug("jdbc url: {}", jdbcUrl); final ImmutableMap.Builder configBuilder = ImmutableMap.builder() .put(JdbcUtils.USERNAME_KEY, config.get(JdbcUtils.USERNAME_KEY).asText()) .put(JdbcUtils.JDBC_URL_KEY, jdbcUrl.toString()); @@ -186,14 +207,13 @@ public String toJDBCQueryParams(final Map sslParams) { .stream() .map((entry) -> { try { - final String result = switch (entry.getKey()) { + return switch (entry.getKey()) { case JdbcSSLConnectionUtils.SSL_MODE -> PARAM_SSLMODE + EQUALS + toSslJdbcParam(SslMode.valueOf(entry.getValue())); case CA_CERTIFICATE_PATH -> SSL_ROOT_CERT + EQUALS + entry.getValue(); case CLIENT_KEY_STORE_URL -> SSL_KEY + EQUALS + Path.of(new URI(entry.getValue())); case CLIENT_KEY_STORE_PASS -> SSL_PASSWORD + EQUALS + entry.getValue(); default -> ""; }; - return result; } catch (final URISyntaxException e) { throw new IllegalArgumentException("unable to convert to URI", e); } @@ -216,7 +236,6 @@ protected Set getExcludedViews() { protected void logPreSyncDebugData(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog) throws SQLException { super.logPreSyncDebugData(database, catalog); - PostgresQueryUtils.logFullVacuumStatus(database, catalog, getQuoteString()); for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { final String streamName = stream.getStream().getName(); final String schemaName = stream.getStream().getNamespace(); @@ -252,6 +271,7 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { .map(PostgresCatalogHelper::overrideSyncModes) .map(PostgresCatalogHelper::removeIncrementalWithoutPk) .map(PostgresCatalogHelper::setIncrementalToSourceDefined) + .map(PostgresCatalogHelper::setDefaultCursorFieldForCdc) .map(PostgresCatalogHelper::addCdcMetadataColumns) // If we're in CDC mode and a stream is not in the publication, the user should only be able to sync // this in FULL_REFRESH mode @@ -263,6 +283,7 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { // Xmin replication has a source-defined cursor (the xmin column). This is done to prevent the user // from being able to pick their own cursor. final List streams = catalog.getStreams().stream() + .map(PostgresCatalogHelper::overrideSyncModes) .map(PostgresCatalogHelper::setIncrementalToSourceDefined) .collect(toList()); @@ -273,12 +294,62 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { } @Override - public JdbcDatabase createDatabase(final JsonNode config) throws SQLException { - final JdbcDatabase database = super.createDatabase(config); + public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { + final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + // Create the data source + final DataSource dataSource = DataSourceFactory.create( + jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, + jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + driverClass, + jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), + getConnectionProperties(sourceConfig)); + // Record the data source so that it can be closed. + dataSources.add(dataSource); + + final JdbcDatabase database = new StreamingJdbcDatabase( + dataSource, + sourceOperations, + streamingQueryConfigProvider); + + quoteString = (quoteString == null ? database.getMetaData().getIdentifierQuoteString() : quoteString); + database.setSourceConfig(sourceConfig); + database.setDatabaseConfig(jdbcConfig); + this.publicizedTablesInCdc = PostgresCatalogHelper.getPublicizedTables(database); + return database; } + public static Map getConnectionProperties(final JsonNode config) { + final Map customProperties = + config.has(JdbcUtils.JDBC_URL_PARAMS_KEY) + ? parseJdbcParameters(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText(), DEFAULT_JDBC_PARAMETERS_DELIMITER) + : new HashMap<>(); + final Map defaultProperties = JdbcDataSourceUtils.getDefaultConnectionProperties(config); + JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); + return MoreMaps.merge(customProperties, defaultProperties); + } + + public static Map parseJdbcParameters(final String jdbcPropertiesString, final String delimiter) { + final Map parameters = new HashMap<>(); + if (!jdbcPropertiesString.isBlank()) { + final String[] keyValuePairs = jdbcPropertiesString.split(delimiter); + for (final String kv : keyValuePairs) { + final String[] split = kv.split("="); + if (split.length == 2) { + parameters.put(split[0], split[1]); + } else if (split.length > 2 && "options".equals(split[0])) { + parameters.put(split[0], kv.substring(split[0].length() + delimiter.length())); + } else { + throw new ConfigErrorException( + "jdbc_url_params must be formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). Got " + + jdbcPropertiesString); + } + } + } + return parameters; + } + @Override public List>> discoverInternal(final JdbcDatabase database) throws Exception { return discoverRawTables(database); @@ -357,13 +428,9 @@ public List> getCheckOperations(final J }); - checkOperations.add(database -> { - PostgresUtils.checkFirstRecordWaitTime(config); - }); + checkOperations.add(database -> PostgresUtils.checkFirstRecordWaitTime(config)); - checkOperations.add(database -> { - PostgresUtils.checkQueueSize(config); - }); + checkOperations.add(database -> PostgresUtils.checkQueueSize(config)); // Verify that a CDC connection can be created checkOperations.add(database -> { @@ -389,75 +456,153 @@ public List> getIncrementalIterators(final final Instant emittedAt) { final JsonNode sourceConfig = database.getSourceConfig(); if (PostgresUtils.isCdc(sourceConfig) && shouldUseCDC(catalog)) { - final Duration firstRecordWaitTime = PostgresUtils.getFirstRecordWaitTime(sourceConfig); - final OptionalInt queueSize = OptionalInt.of(PostgresUtils.getQueueSize(sourceConfig)); - LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); - LOGGER.info("Queue size: {}", queueSize.getAsInt()); - - final PostgresDebeziumStateUtil postgresDebeziumStateUtil = new PostgresDebeziumStateUtil(); - final JsonNode state = - (stateManager.getCdcStateManager().getCdcState() == null || stateManager.getCdcStateManager().getCdcState().getState() == null) ? null - : Jsons.clone(stateManager.getCdcStateManager().getCdcState().getState()); - - final OptionalLong savedOffset = postgresDebeziumStateUtil.savedOffset( - Jsons.clone(PostgresCdcProperties.getDebeziumDefaultProperties(database)), - catalog, - state, - sourceConfig); - - // We should always be able to extract offset out of state if it's not null - if (state != null && savedOffset.isEmpty()) { - throw new RuntimeException( - "Unable extract the offset out of state, State mutation might not be working. " + state.asText()); + LOGGER.info("Using ctid + CDC"); + return cdcCtidIteratorsCombined(database, catalog, tableNameToTable, stateManager, emittedAt, getQuoteString(), + getReplicationSlot(database, sourceConfig).get(0)); + } + + if (isAnyStreamIncrementalSyncMode(catalog) && PostgresUtils.isXmin(sourceConfig)) { + final StreamsCategorised streamsCategorised = categoriseStreams(stateManager, catalog, xminStatus); + final ResultWithFailed> streamsUnderVacuum = streamsUnderVacuum(database, + streamsCategorised.ctidStreams().streamsForCtidSync(), + getQuoteString()); + + // Streams we failed to query for Vacuum - such as in the case of an unsupported postgres server + // are reclassified as xmin since we cannot guarantee that ctid will be possible. + reclassifyCategorisedCtidStreams(streamsCategorised, streamsUnderVacuum.failed()); + + List finalListOfStreamsToBeSyncedViaCtid = + filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); + final ResultWithFailed> fileNodes = + PostgresQueryUtils.fileNodeForStreams(database, + finalListOfStreamsToBeSyncedViaCtid, + getQuoteString()); + + // In case we failed to query for fileNode, streams will get reclassified as xmin + if (!fileNodes.failed().isEmpty()) { + reclassifyCategorisedCtidStreams(streamsCategorised, fileNodes.failed()); + finalListOfStreamsToBeSyncedViaCtid = + filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); } - final boolean savedOffsetAfterReplicationSlotLSN = postgresDebeziumStateUtil.isSavedOffsetAfterReplicationSlotLSN( - // We can assume that there will be only 1 replication slot cause before the sync starts for - // Postgres CDC, - // we run all the check operations and one of the check validates that the replication slot exists - // and has only 1 entry - getReplicationSlot(database, sourceConfig).get(0), - savedOffset); - - if (!savedOffsetAfterReplicationSlotLSN) { - LOGGER.warn("Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch"); - } else if (PostgresUtils.shouldFlushAfterSync(sourceConfig)) { - postgresDebeziumStateUtil.commitLSNToPostgresDatabase(database.getDatabaseConfig(), - savedOffset, - sourceConfig.get("replication_method").get("replication_slot").asText(), - sourceConfig.get("replication_method").get("publication").asText(), - PostgresUtils.getPluginValue(sourceConfig.get("replication_method"))); + final CtidStateManager ctidStateManager = + new CtidPerStreamStateManager(streamsCategorised.ctidStreams().statesFromCtidSync(), fileNodes.result()); + final Map tableBlockSizes = + PostgresQueryUtils.getTableBlockSizeForStreams( + database, + finalListOfStreamsToBeSyncedViaCtid, + getQuoteString()); + + if (!streamsCategorised.ctidStreams().streamsForCtidSync().isEmpty()) { + LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); + } else { + LOGGER.info("No Streams will be synced via ctid."); } - final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(sourceConfig, - PostgresCdcTargetPosition.targetPosition(database), false, firstRecordWaitTime, queueSize); - final PostgresCdcStateHandler postgresCdcStateHandler = new PostgresCdcStateHandler(stateManager); - final List streamsToSnapshot = identifyStreamsToSnapshot(catalog, stateManager); - final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, - new PostgresCdcSavedInfoFetcher(savedOffsetAfterReplicationSlotLSN ? stateManager.getCdcStateManager().getCdcState() : null), - postgresCdcStateHandler, - new PostgresCdcConnectorMetadataInjector(), - PostgresCdcProperties.getDebeziumDefaultProperties(database), - emittedAt, - false); - if (!savedOffsetAfterReplicationSlotLSN || streamsToSnapshot.isEmpty()) { - return Collections.singletonList(incrementalIteratorSupplier.get()); + if (!streamsCategorised.remainingStreams().streamsForXminSync().isEmpty()) { + LOGGER.info("Streams to be synced via xmin : {}", streamsCategorised.remainingStreams().streamsForXminSync().size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(streamsCategorised.remainingStreams().streamsForXminSync())); + } else { + LOGGER.info("No Streams will be synced via xmin."); } - final AutoCloseableIterator snapshotIterator = handler.getSnapshotIterators( - new ConfiguredAirbyteCatalog().withStreams(streamsToSnapshot), new PostgresCdcConnectorMetadataInjector(), - PostgresCdcProperties.getSnapshotProperties(database), postgresCdcStateHandler, emittedAt); - return Collections.singletonList( - AutoCloseableIterators.concatWithEagerClose(AirbyteTraceMessageUtility::emitStreamStatusTrace, snapshotIterator, - AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null))); - - } else if (PostgresUtils.isXmin(sourceConfig) && isIncrementalSyncMode(catalog)) { - final XminStateManager xminStateManager = new XminStateManager(stateManager.getRawStateMessages()); - final PostgresXminHandler handler = new PostgresXminHandler(database, sourceOperations, getQuoteString(), xminStatus, xminStateManager); - return handler.getIncrementalIterators(catalog, tableNameToTable, emittedAt); - } else { - return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, emittedAt); + final XminStateManager xminStateManager = new XminStateManager(streamsCategorised.remainingStreams().statesFromXminSync()); + final PostgresXminHandler xminHandler = new PostgresXminHandler(database, sourceOperations, getQuoteString(), xminStatus, xminStateManager); + + final PostgresCtidHandler ctidHandler = + new PostgresCtidHandler(sourceConfig, database, new CtidPostgresSourceOperations(Optional.empty()), getQuoteString(), + fileNodes.result(), tableBlockSizes, ctidStateManager, + namespacePair -> Jsons.jsonNode(xminStatus)); + + final List> ctidIterators = new ArrayList<>(ctidHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), tableNameToTable, emittedAt)); + final List> xminIterators = new ArrayList<>(xminHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(streamsCategorised.remainingStreams().streamsForXminSync()), tableNameToTable, emittedAt)); + + return Stream + .of(ctidIterators, xminIterators) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + } else if (isAnyStreamIncrementalSyncMode(catalog)) { + final PostgresCursorBasedStateManager postgresCursorBasedStateManager = + new PostgresCursorBasedStateManager(stateManager.getRawStateMessages(), catalog); + final StreamsCategorised streamsCategorised = categoriseStreams(postgresCursorBasedStateManager, catalog); + final ResultWithFailed> streamsUnderVacuum = streamsUnderVacuum(database, + streamsCategorised.ctidStreams().streamsForCtidSync(), + getQuoteString()); + + // Streams we failed to query for Vacuum - such as in the case of an unsupported postgres server + // are reclassified as standard since we cannot guarantee that ctid will be possible. + reclassifyCategorisedCtidStreams(streamsCategorised, streamsUnderVacuum.failed()); + + List finalListOfStreamsToBeSyncedViaCtid = + filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); + final ResultWithFailed> fileNodes = + PostgresQueryUtils.fileNodeForStreams(database, + finalListOfStreamsToBeSyncedViaCtid, + getQuoteString()); + + // Streams we failed to query for fileNode - such as in the case of Views are reclassified as + // standard + if (!fileNodes.failed().isEmpty()) { + reclassifyCategorisedCtidStreams(streamsCategorised, fileNodes.failed()); + finalListOfStreamsToBeSyncedViaCtid = + filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); + } + final CtidStateManager ctidStateManager = + new CtidPerStreamStateManager(streamsCategorised.ctidStreams().statesFromCtidSync(), fileNodes.result()); + final Map tableBlockSizes = + PostgresQueryUtils.getTableBlockSizeForStreams( + database, + finalListOfStreamsToBeSyncedViaCtid, + getQuoteString()); + if (finalListOfStreamsToBeSyncedViaCtid.isEmpty()) { + LOGGER.info("No Streams will be synced via ctid."); + } else { + LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); + } + + if (!streamsCategorised.remainingStreams().streamsForCursorBasedSync().isEmpty()) { + LOGGER.info("Streams to be synced via cursor : {}", streamsCategorised.remainingStreams().streamsForCursorBasedSync().size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(streamsCategorised.remainingStreams().streamsForCursorBasedSync())); + } else { + LOGGER.info("No streams to be synced via cursor"); + } + + final Map cursorBasedStatusMap = + getCursorBasedSyncStatusForStreams(database, finalListOfStreamsToBeSyncedViaCtid, postgresCursorBasedStateManager, getQuoteString()); + + final PostgresCtidHandler cursorBasedCtidHandler = + new PostgresCtidHandler(sourceConfig, + database, + new CtidPostgresSourceOperations(Optional.empty()), + getQuoteString(), + fileNodes.result(), + tableBlockSizes, + ctidStateManager, + namespacePair -> Jsons.jsonNode(cursorBasedStatusMap.get(namespacePair))); + + final List> ctidIterators = new ArrayList<>( + cursorBasedCtidHandler.getIncrementalIterators(new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), + tableNameToTable, + emittedAt)); + final List> cursorBasedIterators = new ArrayList<>(super.getIncrementalIterators(database, + new ConfiguredAirbyteCatalog().withStreams( + streamsCategorised.remainingStreams() + .streamsForCursorBasedSync()), + tableNameToTable, + postgresCursorBasedStateManager, emittedAt)); + + return Stream + .of(ctidIterators, cursorBasedIterators) + .flatMap(Collection::stream) + .collect(Collectors.toList()); } + + return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, emittedAt); } @Override @@ -561,7 +706,7 @@ protected String toSslJdbcParam(final SslMode sslMode) { return toSslJdbcParamInternal(sslMode); } - protected static String toSslJdbcParamInternal(final SslMode sslMode) { + public static String toSslJdbcParamInternal(final SslMode sslMode) { final var result = switch (sslMode) { case DISABLED -> org.postgresql.jdbc.SslMode.DISABLE.value; case ALLOWED -> org.postgresql.jdbc.SslMode.ALLOW.value; @@ -647,4 +792,5 @@ private List getFullTableEstimate(final JdbcDatabase database, Preconditions.checkState(jsonNodes.size() == 1); return jsonNodes; } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index d82537c7fa98..e6356462fc2f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -361,7 +361,7 @@ private void putDoubleArray(final ObjectNode node, final String columnName, fina final ArrayNode arrayNode = Jsons.arrayNode(); final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); while (arrayResultSet.next()) { - arrayNode.add(DataTypeUtils.returnNullIfInvalid(() -> arrayResultSet.getDouble(colIndex), Double::isFinite)); + arrayNode.add(DataTypeUtils.returnNullIfInvalid(() -> arrayResultSet.getDouble(2), Double::isFinite)); } node.set(columnName, arrayNode); } @@ -370,7 +370,7 @@ private void putMoneyArray(final ObjectNode node, final String columnName, final final ArrayNode arrayNode = Jsons.arrayNode(); final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); while (arrayResultSet.next()) { - final String moneyValue = parseMoneyValue(arrayResultSet.getString(colIndex)); + final String moneyValue = parseMoneyValue(arrayResultSet.getString(2)); arrayNode.add(DataTypeUtils.returnNullIfInvalid(() -> DataTypeUtils.returnNullIfInvalid(() -> Double.valueOf(moneyValue), Double::isFinite))); } node.set(columnName, arrayNode); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java index 69fffb0248bc..e2e8d16b9221 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java @@ -27,9 +27,11 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -164,9 +166,13 @@ public static boolean isXmin(final JsonNode config) { return isXmin; } - public static boolean isIncrementalSyncMode(final ConfiguredAirbyteCatalog catalog) { + public static boolean isAnyStreamIncrementalSyncMode(final ConfiguredAirbyteCatalog catalog) { return catalog.getStreams().stream().map(ConfiguredAirbyteStream::getSyncMode) .anyMatch(syncMode -> syncMode == SyncMode.INCREMENTAL); } + public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { + return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java new file mode 100644 index 000000000000..ba64c8a55728 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.cdc; + +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.integrations.debezium.CdcMetadataInjector; + +public class PostgresCdcConnectorMetadataInjector implements CdcMetadataInjector { + + @Override + public void addMetaData(final ObjectNode event, final JsonNode source) { + final long lsn = source.get("lsn").asLong(); + event.put(CDC_LSN, lsn); + } + + @Override + public void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, final String transactionTimestamp, final Long lsn) { + record.put(CDC_UPDATED_AT, transactionTimestamp); + record.put(CDC_LSN, lsn); + record.put(CDC_DELETED_AT, (String) null); + } + + @Override + public String namespace(final JsonNode source) { + return source.get("schema").asText(); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java new file mode 100644 index 000000000000..9b894ba0b5cb --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.cdc; + +import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.streamsUnderVacuum; +import static io.airbyte.integrations.source.postgres.PostgresUtils.prettyPrintConfiguredAirbyteStreamList; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; +import io.airbyte.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils.ResultWithFailed; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; +import io.airbyte.integrations.source.postgres.PostgresType; +import io.airbyte.integrations.source.postgres.PostgresUtils; +import io.airbyte.integrations.source.postgres.cdc.PostgresCdcCtidUtils.CtidStreams; +import io.airbyte.integrations.source.postgres.ctid.CtidGlobalStateManager; +import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations; +import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations.CdcMetadataInjector; +import io.airbyte.integrations.source.postgres.ctid.CtidStateManager; +import io.airbyte.integrations.source.postgres.ctid.PostgresCtidHandler; +import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.OptionalLong; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgresCdcCtidInitializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresCdcCtidInitializer.class); + + public static List> cdcCtidIteratorsCombined(final JdbcDatabase database, + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final StateManager stateManager, + final Instant emittedAt, + final String quoteString, + final JsonNode replicationSlot) { + try { + final JsonNode sourceConfig = database.getSourceConfig(); + final Duration firstRecordWaitTime = PostgresUtils.getFirstRecordWaitTime(sourceConfig); + final OptionalInt queueSize = OptionalInt.of(PostgresUtils.getQueueSize(sourceConfig)); + LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); + LOGGER.info("Queue size: {}", queueSize.getAsInt()); + + final PostgresDebeziumStateUtil postgresDebeziumStateUtil = new PostgresDebeziumStateUtil(); + + final JsonNode initialDebeziumState = postgresDebeziumStateUtil.constructInitialDebeziumState(database, + sourceConfig.get(JdbcUtils.DATABASE_KEY).asText()); + + final JsonNode state = + (stateManager.getCdcStateManager().getCdcState() == null || stateManager.getCdcStateManager().getCdcState().getState() == null) + ? initialDebeziumState + : Jsons.clone(stateManager.getCdcStateManager().getCdcState().getState()); + + final OptionalLong savedOffset = postgresDebeziumStateUtil.savedOffset( + Jsons.clone(PostgresCdcProperties.getDebeziumDefaultProperties(database)), + catalog, + state, + sourceConfig); + + // We should always be able to extract offset out of state if it's not null + if (state != null && savedOffset.isEmpty()) { + throw new RuntimeException( + "Unable extract the offset out of state, State mutation might not be working. " + state.asText()); + } + + final boolean savedOffsetAfterReplicationSlotLSN = postgresDebeziumStateUtil.isSavedOffsetAfterReplicationSlotLSN( + // We can assume that there will be only 1 replication slot cause before the sync starts for + // Postgres CDC, + // we run all the check operations and one of the check validates that the replication slot exists + // and has only 1 entry + replicationSlot, + savedOffset); + + if (!savedOffsetAfterReplicationSlotLSN) { + LOGGER.warn("Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch"); + } else if (PostgresUtils.shouldFlushAfterSync(sourceConfig)) { + postgresDebeziumStateUtil.commitLSNToPostgresDatabase(database.getDatabaseConfig(), + savedOffset, + sourceConfig.get("replication_method").get("replication_slot").asText(), + sourceConfig.get("replication_method").get("publication").asText(), + PostgresUtils.getPluginValue(sourceConfig.get("replication_method"))); + } + final CdcState stateToBeUsed = (!savedOffsetAfterReplicationSlotLSN || stateManager.getCdcStateManager().getCdcState() == null + || stateManager.getCdcStateManager().getCdcState().getState() == null) ? new CdcState().withState(initialDebeziumState) + : stateManager.getCdcStateManager().getCdcState(); + final CtidStreams ctidStreams = PostgresCdcCtidUtils.streamsToSyncViaCtid(stateManager.getCdcStateManager(), catalog, + savedOffsetAfterReplicationSlotLSN); + final List> ctidIterator = new ArrayList<>(); + final List streamsUnderVacuum = new ArrayList<>(); + if (!ctidStreams.streamsForCtidSync().isEmpty()) { + streamsUnderVacuum.addAll(streamsUnderVacuum(database, + ctidStreams.streamsForCtidSync(), quoteString).result()); + + final List finalListOfStreamsToBeSyncedViaCtid = + streamsUnderVacuum.isEmpty() ? ctidStreams.streamsForCtidSync() + : ctidStreams.streamsForCtidSync().stream() + .filter(c -> !streamsUnderVacuum.contains(AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(c))) + .toList(); + LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); + final ResultWithFailed> fileNodes = + PostgresQueryUtils.fileNodeForStreams(database, + finalListOfStreamsToBeSyncedViaCtid, + quoteString); + final CtidStateManager ctidStateManager = new CtidGlobalStateManager(ctidStreams, fileNodes.result(), stateToBeUsed, catalog); + final CtidPostgresSourceOperations ctidPostgresSourceOperations = new CtidPostgresSourceOperations( + Optional.of(new CdcMetadataInjector( + emittedAt.toString(), io.airbyte.db.PostgresUtils.getLsn(database).asLong(), new PostgresCdcConnectorMetadataInjector()))); + final Map tableBlockSizes = + PostgresQueryUtils.getTableBlockSizeForStreams( + database, + finalListOfStreamsToBeSyncedViaCtid, + quoteString); + final PostgresCtidHandler ctidHandler = new PostgresCtidHandler(sourceConfig, database, + ctidPostgresSourceOperations, + quoteString, + fileNodes.result(), + tableBlockSizes, + ctidStateManager, + namespacePair -> Jsons.emptyObject()); + + ctidIterator.addAll(ctidHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), tableNameToTable, emittedAt)); + } else { + LOGGER.info("No streams will be synced via ctid"); + } + + final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(sourceConfig, + PostgresCdcTargetPosition.targetPosition(database), false, firstRecordWaitTime, queueSize); + final PostgresCdcStateHandler postgresCdcStateHandler = new PostgresCdcStateHandler(stateManager); + + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, + new PostgresCdcSavedInfoFetcher(stateToBeUsed), + postgresCdcStateHandler, + new PostgresCdcConnectorMetadataInjector(), + PostgresCdcProperties.getDebeziumDefaultProperties(database), + emittedAt, + false); + + if (ctidIterator.isEmpty()) { + return Collections.singletonList(incrementalIteratorSupplier.get()); + } + + if (streamsUnderVacuum.isEmpty()) { + // This starts processing the WAL as soon as initial sync is complete, this is a bit different from + // the current cdc syncs. + // We finish the current CDC once the initial snapshot is complete and the next sync starts + // processing the WAL + return Stream + .of(ctidIterator, Collections.singletonList(AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null))) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + } else { + LOGGER.warn("Streams are under vacuuming, not going to process WAL"); + return ctidIterator; + } + + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java new file mode 100644 index 000000000000..da07156cde17 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.cdc; + +import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Sets; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class PostgresCdcCtidUtils { + + public static CtidStreams streamsToSyncViaCtid(final CdcStateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog, + final boolean savedOffsetAfterReplicationSlotLSN) { + if (!savedOffsetAfterReplicationSlotLSN) { + return new CtidStreams( + fullCatalog.getStreams() + .stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .collect(Collectors.toList()), + new HashMap<>()); + } + + final AirbyteStateMessage airbyteStateMessage = stateManager.getRawStateMessage(); + final Set streamsStillInCtidSync = new HashSet<>(); + final Map pairToCtidStatus = new HashMap<>(); + if (airbyteStateMessage != null && airbyteStateMessage.getGlobal() != null && airbyteStateMessage.getGlobal().getStreamStates() != null) { + airbyteStateMessage.getGlobal().getStreamStates().forEach(stateMessage -> { + final JsonNode streamState = stateMessage.getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase("ctid")) { + final CtidStatus ctidStatus = Jsons.object(streamState, CtidStatus.class); + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + pairToCtidStatus.put(pair, ctidStatus); + streamsStillInCtidSync.add(pair); + } + } + }); + } + + final List streamsForCtidSync = new ArrayList<>(); + fullCatalog.getStreams().stream() + .filter(stream -> streamsStillInCtidSync.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .forEach(streamsForCtidSync::add); + final List newlyAddedStreams = identifyStreamsToSnapshot(fullCatalog, stateManager.getInitialStreamsSynced()); + streamsForCtidSync.addAll(newlyAddedStreams); + + return new CtidStreams(streamsForCtidSync, pairToCtidStatus); + } + + private static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, + final Set alreadySyncedStreams) { + final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog); + final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); + return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))).map(Jsons::clone) + .collect(Collectors.toList()); + } + + public record CtidStreams(List streamsForCtidSync, + Map pairToCtidStatus) { + + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java similarity index 93% rename from airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java index a83a260e5719..b1789f792cd6 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcProperties.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.postgres; +package io.airbyte.integrations.source.postgres.cdc; import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; @@ -14,6 +14,8 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.debezium.internals.postgres.PostgresConverter; import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.integrations.source.postgres.PostgresSource; +import io.airbyte.integrations.source.postgres.PostgresUtils; import java.net.URI; import java.nio.file.Path; import java.time.Duration; @@ -26,7 +28,7 @@ public class PostgresCdcProperties { private static final int HEARTBEAT_FREQUENCY_SEC = 10; private static final Logger LOGGER = LoggerFactory.getLogger(PostgresCdcProperties.class); - static Properties getDebeziumDefaultProperties(final JdbcDatabase database) { + public static Properties getDebeziumDefaultProperties(final JdbcDatabase database) { final JsonNode sourceConfig = database.getSourceConfig(); final Properties props = commonProperties(database); props.setProperty("plugin.name", PostgresUtils.getPluginValue(sourceConfig.get("replication_method"))); @@ -93,7 +95,7 @@ private static Properties commonProperties(final JdbcDatabase database) { return props; } - static Properties getSnapshotProperties(final JdbcDatabase database) { + public static Properties getSnapshotProperties(final JdbcDatabase database) { final Properties props = commonProperties(database); props.setProperty("snapshot.mode", "initial_only"); return props; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java similarity index 93% rename from airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcSavedInfoFetcher.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java index 55403f5c3fd4..8e5ad5408bd8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.postgres; +package io.airbyte.integrations.source.postgres.cdc; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java similarity index 98% rename from airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java rename to airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java index ea8d3aa5c720..264db2764da8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.postgres; +package io.airbyte.integrations.source.postgres.cdc; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/AirbyteMessageWithCtid.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/AirbyteMessageWithCtid.java new file mode 100644 index 000000000000..cbadc1af7044 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/AirbyteMessageWithCtid.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import io.airbyte.protocol.models.v0.AirbyteMessage; + +/** + * ctid of rows is queried as part of our sync and is used to checkpoint to be able to restart + * failed sync from a known last point. Since we never want to emit a ctid it is kept in a different + * field, to save us an expensive JsonNode.remove() operation. + * + * @param recordMessage row fields to emit + * @param ctid ctid + */ +public record AirbyteMessageWithCtid(AirbyteMessage recordMessage, String ctid) { + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java new file mode 100644 index 000000000000..11f4ba0d09c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Class represents a postgres ctid record in the form of "(number,number)" Used to simplify code + * dealing with ctid calculations. + */ +public class Ctid { + + final Long page; + final Long tuple; + + public static Ctid of(final long page, final long tuple) { + return new Ctid(page, tuple); + } + + public static Ctid of(final String ctid) { + return new Ctid(ctid); + } + + Ctid(final long page, final long tuple) { + this.page = page; + this.tuple = tuple; + } + + Ctid(final String ctid) { + final Pattern p = Pattern.compile("\\d+"); + final Matcher m = p.matcher(ctid); + if (!m.find()) { + throw new IllegalArgumentException("Invalid ctid format"); + } + final String ctidPageStr = m.group(); + this.page = Long.parseLong(ctidPageStr); + + if (!m.find()) { + throw new IllegalArgumentException("Invalid ctid format"); + } + final String ctidTupleStr = m.group(); + this.tuple = Long.parseLong(ctidTupleStr); + + Objects.requireNonNull(this.page); + Objects.requireNonNull(this.tuple); + } + + @Override + public String toString() { + return "(%d,%d)".formatted(page, tuple); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Ctid ctid = (Ctid) o; + return Objects.equals(page, ctid.page) && Objects.equals(tuple, ctid.tuple); + } + + @Override + public int hashCode() { + return Objects.hash(page, tuple); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java new file mode 100644 index 000000000000..9a24253caa30 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.cdc.PostgresCdcCtidUtils.CtidStreams; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CtidGlobalStateManager extends CtidStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(CtidGlobalStateManager.class); + + private final CdcState cdcState; + private final Set streamsThatHaveCompletedSnapshot; + + public CtidGlobalStateManager(final CtidStreams ctidStreams, + final Map fileNodes, + final CdcState cdcState, + final ConfiguredAirbyteCatalog catalog) { + super(filterOutExpiredFileNodes(ctidStreams.pairToCtidStatus(), fileNodes)); + this.cdcState = cdcState; + this.streamsThatHaveCompletedSnapshot = initStreamsCompletedSnapshot(ctidStreams, catalog); + } + + private static Set initStreamsCompletedSnapshot(final CtidStreams ctidStreams, + final ConfiguredAirbyteCatalog catalog) { + final Set streamsThatHaveCompletedSnapshot = new HashSet<>(); + catalog.getStreams().forEach(configuredAirbyteStream -> { + if (ctidStreams.streamsForCtidSync().contains(configuredAirbyteStream) || configuredAirbyteStream.getSyncMode() != SyncMode.INCREMENTAL) { + return; + } + streamsThatHaveCompletedSnapshot.add( + new AirbyteStreamNameNamespacePair(configuredAirbyteStream.getStream().getName(), configuredAirbyteStream.getStream().getNamespace())); + }); + return streamsThatHaveCompletedSnapshot; + } + + private static Map filterOutExpiredFileNodes( + final Map pairToCtidStatus, + final Map fileNodes) { + final Map filteredMap = new HashMap<>(); + pairToCtidStatus.forEach((pair, ctidStatus) -> { + final AirbyteStreamNameNamespacePair updatedPair = new AirbyteStreamNameNamespacePair(pair.getName(), pair.getNamespace()); + if (validateRelationFileNode(ctidStatus, updatedPair, fileNodes)) { + filteredMap.put(updatedPair, ctidStatus); + } else { + LOGGER.warn( + "The relation file node for table in source db {} is not equal to the saved ctid state, a full sync from scratch will be triggered.", + pair); + } + }); + return filteredMap; + } + + @Override + public AirbyteStateMessage createCtidStateMessage(final AirbyteStreamNameNamespacePair pair, final CtidStatus ctidStatus) { + final List streamStates = new ArrayList<>(); + streamsThatHaveCompletedSnapshot.forEach(stream -> { + final DbStreamState state = getFinalState(stream); + streamStates.add(getAirbyteStreamState(stream, Jsons.jsonNode(state))); + + }); + streamStates.add(getAirbyteStreamState(pair, (Jsons.jsonNode(ctidStatus)))); + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(cdcState)); + globalState.setStreamStates(streamStates); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + @Override + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + streamsThatHaveCompletedSnapshot.add(pair); + final List streamStates = new ArrayList<>(); + streamsThatHaveCompletedSnapshot.forEach(stream -> { + final DbStreamState state = getFinalState(stream); + streamStates.add(getAirbyteStreamState(stream, Jsons.jsonNode(state))); + }); + + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(cdcState)); + globalState.setStreamStates(streamStates); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + private AirbyteStreamState getAirbyteStreamState(final AirbyteStreamNameNamespacePair pair, final JsonNode stateData) { + assert Objects.nonNull(pair); + assert Objects.nonNull(pair.getName()); + assert Objects.nonNull(pair.getNamespace()); + + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(pair.getName()).withNamespace(pair.getNamespace())) + .withStreamState(stateData); + } + + private DbStreamState getFinalState(final AirbyteStreamNameNamespacePair pair) { + assert Objects.nonNull(pair); + assert Objects.nonNull(pair.getName()); + assert Objects.nonNull(pair.getNamespace()); + + return new DbStreamState() + .withStreamName(pair.getName()) + .withStreamNamespace(pair.getNamespace()) + .withCursorField(Collections.emptyList()) + .withCursor(null); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java new file mode 100644 index 000000000000..425ea6ea4fc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.postgres.internal.models.XminStatus; +import io.airbyte.integrations.source.postgres.xmin.XminStateManager; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CtidPerStreamStateManager extends CtidStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(CtidPerStreamStateManager.class); + private final static AirbyteStateMessage EMPTY_STATE = new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState()); + + public CtidPerStreamStateManager(final List stateMessages, final Map fileNodes) { + super(createPairToCtidStatusMap(stateMessages, fileNodes)); + } + + private static Map createPairToCtidStatusMap(final List stateMessages, + final Map fileNodes) { + final Map localMap = new HashMap<>(); + if (stateMessages != null) { + for (final AirbyteStateMessage stateMessage : stateMessages) { + if (stateMessage.getType() == AirbyteStateType.STREAM && !stateMessage.equals(EMPTY_STATE)) { + LOGGER.info("State message: " + stateMessage); + final StreamDescriptor streamDescriptor = stateMessage.getStream().getStreamDescriptor(); + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), streamDescriptor.getNamespace()); + final CtidStatus ctidStatus; + try { + ctidStatus = Jsons.object(stateMessage.getStream().getStreamState(), CtidStatus.class); + assert (ctidStatus.getVersion() == CTID_STATUS_VERSION); + assert (ctidStatus.getStateType().equals(StateType.CTID)); + } catch (final IllegalArgumentException e) { + throw new ConfigErrorException("Invalid per-stream state"); + } + if (validateRelationFileNode(ctidStatus, pair, fileNodes)) { + localMap.put(pair, ctidStatus); + } else { + LOGGER.warn( + "The relation file node for table in source db {} is not equal to the saved ctid state, a full sync from scratch will be triggered.", + pair); + } + } + } + } + return localMap; + } + + @Override + public AirbyteStateMessage createCtidStateMessage(final AirbyteStreamNameNamespacePair pair, final CtidStatus ctidStatus) { + final AirbyteStreamState airbyteStreamState = + new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor() + .withName(pair.getName()) + .withNamespace(pair.getNamespace())) + .withStreamState(Jsons.jsonNode(ctidStatus)); + + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(airbyteStreamState); + } + + @Override + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + return XminStateManager.getAirbyteStateMessage(pair, Jsons.object(streamStateForIncrementalRun, XminStatus.class)); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPostgresSourceOperations.java new file mode 100644 index 000000000000..3a032319295e --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPostgresSourceOperations.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.PostgresSourceOperations; +import io.airbyte.integrations.source.postgres.cdc.PostgresCdcConnectorMetadataInjector; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; + +public class CtidPostgresSourceOperations extends PostgresSourceOperations { + + private final Optional cdcMetadataInjector; + + public CtidPostgresSourceOperations(final Optional cdcMetadataInjector) { + super(); + this.cdcMetadataInjector = cdcMetadataInjector; + } + + private static final String CTID = "ctid"; + + public RowDataWithCtid recordWithCtid(final ResultSet queryContext) throws SQLException { + // the first call communicates with the database. after that the result is cached. + final ResultSetMetaData metadata = queryContext.getMetaData(); + final int columnCount = metadata.getColumnCount(); + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + String ctid = null; + for (int i = 1; i <= columnCount; i++) { + final String columnName = metadata.getColumnName(i); + if (columnName.equalsIgnoreCase(CTID)) { + ctid = queryContext.getString(i); + continue; + } + + // convert to java types that will convert into reasonable json. + copyToJsonField(queryContext, i, jsonNode); + } + + if (Objects.nonNull(cdcMetadataInjector) && cdcMetadataInjector.isPresent()) { + cdcMetadataInjector.get().inject(jsonNode); + } + + assert Objects.nonNull(ctid); + return new RowDataWithCtid(jsonNode, ctid); + } + + public record RowDataWithCtid(JsonNode data, String ctid) { + + } + + public static class CdcMetadataInjector { + + private final String transactionTimestamp; + private final long lsn; + private final PostgresCdcConnectorMetadataInjector metadataInjector; + + public CdcMetadataInjector(final String transactionTimestamp, + final long lsn, + final PostgresCdcConnectorMetadataInjector metadataInjector) { + this.transactionTimestamp = transactionTimestamp; + this.lsn = lsn; + this.metadataInjector = metadataInjector; + } + + private void inject(final ObjectNode record) { + metadataInjector.addMetaDataToRowsFetchedOutsideDebezium(record, transactionTimestamp, lsn); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java new file mode 100644 index 000000000000..60c392f19ade --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.CTID_STATUS_VERSION; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.AbstractIterator; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Iterator; +import java.util.Objects; +import javax.annotation.CheckForNull; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CtidStateIterator extends AbstractIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(CtidStateIterator.class); + public static final Duration SYNC_CHECKPOINT_DURATION = Duration.ofMinutes(15); + public static final Integer SYNC_CHECKPOINT_RECORDS = 10_000; + + private final Iterator messageIterator; + private final AirbyteStreamNameNamespacePair pair; + private boolean hasEmittedFinalState; + private String lastCtid; + private final JsonNode streamStateForIncrementalRun; + private final long relationFileNode; + private final CtidStateManager stateManager; + private long recordCount = 0L; + private Instant lastCheckpoint = Instant.now(); + private final Duration syncCheckpointDuration; + private final Long syncCheckpointRecords; + + public CtidStateIterator(final Iterator messageIterator, + final AirbyteStreamNameNamespacePair pair, + final long relationFileNode, + final CtidStateManager stateManager, + final JsonNode streamStateForIncrementalRun, + final Duration checkpointDuration, + final Long checkpointRecords) { + this.messageIterator = messageIterator; + this.pair = pair; + this.relationFileNode = relationFileNode; + this.stateManager = stateManager; + this.streamStateForIncrementalRun = streamStateForIncrementalRun; + this.syncCheckpointDuration = checkpointDuration; + this.syncCheckpointRecords = checkpointRecords; + } + + @CheckForNull + @Override + protected AirbyteMessage computeNext() { + if (messageIterator.hasNext()) { + if ((recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) + && Objects.nonNull(lastCtid) + && StringUtils.isNotBlank(lastCtid)) { + final CtidStatus ctidStatus = new CtidStatus() + .withVersion(CTID_STATUS_VERSION) + .withStateType(StateType.CTID) + .withCtid(lastCtid) + .withIncrementalState(streamStateForIncrementalRun) + .withRelationFilenode(relationFileNode); + LOGGER.info("Emitting ctid state for stream {}, state is {}", pair, ctidStatus); + recordCount = 0L; + lastCheckpoint = Instant.now(); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateManager.createCtidStateMessage(pair, ctidStatus)); + } + // Use try-catch to catch Exception that could occur when connection to the database fails + try { + final AirbyteMessageWithCtid message = messageIterator.next(); + if (Objects.nonNull(message.ctid())) { + this.lastCtid = message.ctid(); + } + recordCount++; + return message.recordMessage(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } else if (!hasEmittedFinalState) { + hasEmittedFinalState = true; + final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); + LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(finalStateMessage); + } else { + return endOfData(); + } + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java new file mode 100644 index 000000000000..86915a90be16 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.util.Map; +import java.util.Objects; + +public abstract class CtidStateManager { + + public static final long CTID_STATUS_VERSION = 2; + public static final String STATE_TYPE_KEY = "state_type"; + public static final String STATE_VER_KEY = "version"; + + private final Map pairToCtidStatus; + + protected CtidStateManager(final Map pairToCtidStatus) { + this.pairToCtidStatus = pairToCtidStatus; + } + + public CtidStatus getCtidStatus(final AirbyteStreamNameNamespacePair pair) { + return pairToCtidStatus.get(pair); + } + + public static boolean validateRelationFileNode(final CtidStatus ctidstatus, + final AirbyteStreamNameNamespacePair pair, + final Map fileNodes) { + + if (fileNodes.containsKey(pair)) { + final Long fileNode = fileNodes.get(pair); + return Objects.equals(ctidstatus.getRelationFilenode(), fileNode); + } + return true; + } + + public abstract AirbyteStateMessage createCtidStateMessage(final AirbyteStreamNameNamespacePair pair, final CtidStatus ctidStatus); + + public abstract AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun); + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java new file mode 100644 index 000000000000..ce4dc2f4ec64 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import com.google.common.collect.Sets; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class CtidUtils { + + public static List identifyNewlyAddedStreams(final ConfiguredAirbyteCatalog fullCatalog, + final Set alreadySeenStreams, + final SyncMode syncMode) { + final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(fullCatalog); + + final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySeenStreams)); + + return fullCatalog.getStreams().stream() + .filter(stream -> stream.getSyncMode() == syncMode) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .collect(Collectors.toList()); + } + + public static List getStreamsFromStreamPairs(final ConfiguredAirbyteCatalog catalog, + final Set streamPairs, + final SyncMode syncMode) { + + return catalog.getStreams().stream() + .filter(stream -> stream.getSyncMode() == syncMode) + .filter(stream -> streamPairs.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .collect(Collectors.toList()); + } + + public record CtidStreams(List streamsForCtidSync, + List statesFromCtidSync) { + + } + + public record StreamsCategorised (CtidStreams ctidStreams, + T remainingStreams) { + + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java new file mode 100644 index 000000000000..59e975351247 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.stream.AirbyteStreamUtils; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; +import io.airbyte.integrations.source.postgres.PostgresType; +import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations.RowDataWithCtid; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.CommonField; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PostgresCtidHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresCtidHandler.class); + + private final JsonNode config; + private final JdbcDatabase database; + private final CtidPostgresSourceOperations sourceOperations; + private final String quoteString; + private final CtidStateManager ctidStateManager; + private final Map fileNodes; + final Map tableBlockSizes; + private final Function streamStateForIncrementalRunSupplier; + private static final int QUERY_TARGET_SIZE_GB = 1; + public static final double MEGABYTE = Math.pow(1024, 2); + public static final double GIGABYTE = MEGABYTE * 1024; + + public PostgresCtidHandler(final JsonNode config, + final JdbcDatabase database, + final CtidPostgresSourceOperations sourceOperations, + final String quoteString, + final Map fileNodes, + final Map tableBlockSizes, + final CtidStateManager ctidStateManager, + final Function streamStateForIncrementalRunSupplier) { + this.config = config; + this.database = database; + this.sourceOperations = sourceOperations; + this.quoteString = quoteString; + this.fileNodes = fileNodes; + this.tableBlockSizes = tableBlockSizes; + this.ctidStateManager = ctidStateManager; + this.streamStateForIncrementalRunSupplier = streamStateForIncrementalRunSupplier; + } + + public List> getIncrementalIterators( + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final Instant emmitedAt) { + final List> iteratorList = new ArrayList<>(); + for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { + final AirbyteStream stream = airbyteStream.getStream(); + final String streamName = stream.getName(); + final String namespace = stream.getNamespace(); + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamName, namespace); + final String fullyQualifiedTableName = DbSourceDiscoverUtil.getFullyQualifiedTableName(namespace, streamName); + if (!tableNameToTable.containsKey(fullyQualifiedTableName)) { + LOGGER.info("Skipping stream {} because it is not in the source", fullyQualifiedTableName); + continue; + } + if (airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) { + // Grab the selected fields to sync + final TableInfo> table = tableNameToTable + .get(fullyQualifiedTableName); + final List selectedDatabaseFields = table.getFields() + .stream() + .map(CommonField::getName) + .filter(CatalogHelpers.getTopLevelFieldNames(airbyteStream)::contains) + .toList(); + final AutoCloseableIterator queryStream = queryTableCtid( + selectedDatabaseFields, + table.getNameSpace(), + table.getName(), + tableBlockSizes.get(pair).tableSize(), + tableBlockSizes.get(pair).blockSize()); + final AutoCloseableIterator recordIterator = + getRecordIterator(queryStream, streamName, namespace, emmitedAt.toEpochMilli()); + final AutoCloseableIterator recordAndMessageIterator = augmentWithState(recordIterator, pair); + final AutoCloseableIterator logAugmented = augmentWithLogs(recordAndMessageIterator, pair, streamName); + iteratorList.add(logAugmented); + + } + } + return iteratorList; + } + + /** + * Builds a plan for subqueries. Each query returning an approximate amount of data. Using + * information about a table size and block (page) size. + * + * @param startCtid starting point + * @param relationSize table size + * @param blockSize page size + * @param chunkSizeGB required amount of data in each partition + * @return a list of ctid that can be used to generate queries. + */ + @VisibleForTesting + static List> ctidQueryPlan(final Ctid startCtid, final long relationSize, final long blockSize, final int chunkSizeGB) { + final List> chunks = new ArrayList<>(); + long lowerBound = startCtid.page; + long upperBound; + final double oneGigaPages = GIGABYTE / blockSize; + final long eachStep = (long) oneGigaPages * chunkSizeGB; + LOGGER.info("Will read {} pages to get {}GB", eachStep, chunkSizeGB); + final long theoreticalLastPage = relationSize / blockSize; + LOGGER.debug("Theoretical last page {}", theoreticalLastPage); + upperBound = lowerBound + eachStep; + + if (upperBound > theoreticalLastPage) { + chunks.add(Pair.of(startCtid, null)); + } else { + chunks.add(Pair.of(Ctid.of(lowerBound, startCtid.tuple), Ctid.of(upperBound, 0))); + while (upperBound < theoreticalLastPage) { + lowerBound = upperBound; + upperBound += eachStep; + chunks.add(Pair.of(Ctid.of(lowerBound, 0), upperBound > theoreticalLastPage ? null : Ctid.of(upperBound, 0))); + } + } + // The last pair is (x,y) -> null to indicate an unbounded "WHERE ctid > (x,y)" query. + // The actual last page is approximated. The last subquery will go until the end of table. + return chunks; + } + + private AutoCloseableIterator queryTableCtid( + final List columnNames, + final String schemaName, + final String tableName, + final long tableSize, + final long blockSize) { + + LOGGER.info("Queueing query for table: {}", tableName); + final AirbyteStreamNameNamespacePair airbyteStream = + AirbyteStreamUtils.convertFromNameAndNamespace(tableName, schemaName); + + final CtidStatus currentCtidStatus = ctidStateManager.getCtidStatus(airbyteStream); + + // Rather than trying to read an entire table with a "WHERE ctid > (0,0)" query, + // We are creating a list of lazy iterators each holding a subquery according to the plan. + // All subqueries are then composed in a single composite iterator. + // Because list consists of lazy iterators, the query is only executing when needed one after the + // other. + final List> subQueriesPlan = + ctidQueryPlan((currentCtidStatus == null) ? Ctid.of(0, 0) : Ctid.of(currentCtidStatus.getCtid()), tableSize, blockSize, QUERY_TARGET_SIZE_GB); + final List> subQueriesIterators = new ArrayList<>(); + subQueriesPlan.forEach(p -> subQueriesIterators.add(AutoCloseableIterators.lazyIterator(() -> { + try { + final Stream stream = database.unsafeQuery( + connection -> createCtidQueryStatement(connection, columnNames, schemaName, tableName, p.getLeft(), p.getRight()), + sourceOperations::recordWithCtid); + + return AutoCloseableIterators.fromStream(stream, airbyteStream); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + }, airbyteStream))); + return AutoCloseableIterators.concatWithEagerClose(subQueriesIterators); + } + + private PreparedStatement createCtidQueryStatement( + final Connection connection, + final List columnNames, + final String schemaName, + final String tableName, + final Ctid lowerBound, + final Ctid upperBound) { + try { + LOGGER.info("Preparing query for table: {}", tableName); + final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, + quoteString); + final String wrappedColumnNames = RelationalDbQueryUtils.enquoteIdentifierList(columnNames, quoteString); + if (upperBound != null) { + final String sql = "SELECT ctid::text, %s FROM %s WHERE ctid > ?::tid AND ctid <= ?::tid".formatted(wrappedColumnNames, fullTableName); + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + preparedStatement.setObject(1, lowerBound.toString()); + preparedStatement.setObject(2, upperBound.toString()); + LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); + return preparedStatement; + } else { + final String sql = "SELECT ctid::text, %s FROM %s WHERE ctid > ?::tid".formatted(wrappedColumnNames, fullTableName); + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + preparedStatement.setObject(1, lowerBound.toString()); + LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); + return preparedStatement; + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + // Transforms the given iterator to create an {@link AirbyteRecordMessage} + private AutoCloseableIterator getRecordIterator( + final AutoCloseableIterator recordIterator, + final String streamName, + final String namespace, + final long emittedAt) { + return AutoCloseableIterators.transform(recordIterator, r -> new AirbyteMessageWithCtid(new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(streamName) + .withNamespace(namespace) + .withEmittedAt(emittedAt) + .withData(r.data())), + r.ctid())); + } + + // Augments the given iterator with record count logs. + private AutoCloseableIterator augmentWithLogs(final AutoCloseableIterator iterator, + final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, + final String streamName) { + final AtomicLong recordCount = new AtomicLong(); + return AutoCloseableIterators.transform(iterator, + AirbyteStreamUtils.convertFromNameAndNamespace(pair.getName(), pair.getNamespace()), + r -> { + final long count = recordCount.incrementAndGet(); + if (count % 1_000_000 == 0) { + LOGGER.info("Reading stream {}. Records read: {}", streamName, count); + } + return r; + }); + } + + private AutoCloseableIterator augmentWithState(final AutoCloseableIterator recordIterator, + final AirbyteStreamNameNamespacePair pair) { + + final CtidStatus currentCtidStatus = ctidStateManager.getCtidStatus(pair); + final JsonNode incrementalState = + (currentCtidStatus == null || currentCtidStatus.getIncrementalState() == null) ? streamStateForIncrementalRunSupplier.apply(pair) + : currentCtidStatus.getIncrementalState(); + final Long latestFileNode = fileNodes.get(pair); + assert latestFileNode != null; + + final Duration syncCheckpointDuration = + config.get("sync_checkpoint_seconds") != null ? Duration.ofSeconds(config.get("sync_checkpoint_seconds").asLong()) + : CtidStateIterator.SYNC_CHECKPOINT_DURATION; + final Long syncCheckpointRecords = config.get("sync_checkpoint_records") != null ? config.get("sync_checkpoint_records").asLong() + : CtidStateIterator.SYNC_CHECKPOINT_RECORDS; + + return AutoCloseableIterators.transformIterator( + r -> new CtidStateIterator(r, pair, latestFileNode, ctidStateManager, incrementalState, + syncCheckpointDuration, syncCheckpointRecords), + recordIterator, pair); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java new file mode 100644 index 000000000000..2b8b65e5994d --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.cursor_based; + +import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; +import static io.airbyte.integrations.source.postgres.ctid.CtidUtils.getStreamsFromStreamPairs; +import static io.airbyte.integrations.source.postgres.ctid.CtidUtils.identifyNewlyAddedStreams; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.CtidStreams; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The class mainly categorises the streams based on the state type into two categories : 1. Streams + * that need to be synced via ctid iterator: These are streams that are either newly added or did + * not complete their initial sync. 2. Streams that need to be synced via cursor-based iterator: + * These are streams that have completed their initial sync and are not syncing data incrementally. + */ +public class CursorBasedCtidUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(CursorBasedCtidUtils.class); + + public static StreamsCategorised categoriseStreams(final StateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog) { + final List rawStateMessages = stateManager.getRawStateMessages(); + final List statesFromCtidSync = new ArrayList<>(); + final Set alreadySeenStreamPairs = new HashSet<>(); + final Set stillInCtidStreamPairs = new HashSet<>(); + + final List statesFromCursorBasedSync = new ArrayList<>(); + final Set cursorBasedSyncStreamPairs = new HashSet<>(); + + if (rawStateMessages != null) { + rawStateMessages.forEach(stateMessage -> { + final JsonNode streamState = stateMessage.getStream().getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStream().getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase(StateType.CTID.value())) { + statesFromCtidSync.add(stateMessage); + stillInCtidStreamPairs.add(pair); + } else if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase(StateType.CURSOR_BASED.value())) { + cursorBasedSyncStreamPairs.add(pair); + statesFromCursorBasedSync.add(stateMessage); + } else { + throw new RuntimeException("Unknown state type: " + streamState.get(STATE_TYPE_KEY).asText()); + } + } else { + LOGGER.info("State type not present, syncing stream {} via cursor", streamDescriptor.getName()); + cursorBasedSyncStreamPairs.add(pair); + statesFromCursorBasedSync.add(stateMessage); + } + alreadySeenStreamPairs.add(new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), streamDescriptor.getNamespace())); + }); + } + + final List newlyAddedIncrementalStreams = + identifyNewlyAddedStreams(fullCatalog, alreadySeenStreamPairs, SyncMode.INCREMENTAL); + final List streamsForCtidSync = getStreamsFromStreamPairs(fullCatalog, stillInCtidStreamPairs, SyncMode.INCREMENTAL); + final List streamsForCursorBasedSync = + getStreamsFromStreamPairs(fullCatalog, cursorBasedSyncStreamPairs, SyncMode.INCREMENTAL); + streamsForCtidSync.addAll(newlyAddedIncrementalStreams); + + return new StreamsCategorised<>(new CtidStreams(streamsForCtidSync, statesFromCtidSync), + new CursorBasedStreams(streamsForCursorBasedSync, statesFromCursorBasedSync)); + } + + public record CursorBasedStreams(List streamsForCursorBasedSync, + List statesFromCursorBasedSync) {} + + /** + * Reclassifies previously categorised ctid stream into standard category. Used in case we identify + * ctid is not possible such as a View + * + * @param categorisedStreams categorised streams + * @param streamPair stream to reclassify + */ + public static void reclassifyCategorisedCtidStream(final StreamsCategorised categorisedStreams, + AirbyteStreamNameNamespacePair streamPair) { + final Optional foundStream = categorisedStreams + .ctidStreams() + .streamsForCtidSync().stream().filter(c -> Objects.equals( + streamPair, + new AirbyteStreamNameNamespacePair(c.getStream().getName(), c.getStream().getNamespace()))) + .findFirst(); + foundStream.ifPresent(c -> { + categorisedStreams.remainingStreams().streamsForCursorBasedSync().add(c); + categorisedStreams.ctidStreams().streamsForCtidSync().remove(c); + LOGGER.info("Reclassified {}.{} as standard stream", c.getStream().getNamespace(), c.getStream().getName()); + }); + + // Should there ever be a matching ctid state when ctid is not possible? + final Optional foundStateMessage = categorisedStreams + .ctidStreams() + .statesFromCtidSync().stream().filter(m -> Objects.equals(streamPair, + new AirbyteStreamNameNamespacePair( + m.getStream().getStreamDescriptor().getName(), + m.getStream().getStreamDescriptor().getNamespace()))) + .findFirst(); + foundStateMessage.ifPresent(m -> { + categorisedStreams.remainingStreams().statesFromCursorBasedSync().add(m); + categorisedStreams.ctidStreams().statesFromCtidSync().remove(m); + }); + } + + public static void reclassifyCategorisedCtidStreams(final StreamsCategorised categorisedStreams, + List streamPairs) { + streamPairs.forEach(c -> reclassifyCategorisedCtidStream(categorisedStreams, c)); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java new file mode 100644 index 000000000000..f490d51cce5d --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.cursor_based; + +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.state.StreamStateManager; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This state manager extends the StreamStateManager to enable writing the state_type and version + * keys to the stream state when they're going through the iterator Once we have verified that + * expanding StreamStateManager itself to include this functionality, this class will be removed + */ +public class PostgresCursorBasedStateManager extends StreamStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamStateManager.class); + + public PostgresCursorBasedStateManager(final List airbyteStateMessages, final ConfiguredAirbyteCatalog catalog) { + super(airbyteStateMessages, catalog); + } + + @Override + public AirbyteStateMessage toState(final Optional pair) { + if (pair.isPresent()) { + final Map pairToCursorInfoMap = getPairToCursorInfoMap(); + final Optional cursorInfo = Optional.ofNullable(pairToCursorInfoMap.get(pair.get())); + + if (cursorInfo.isPresent()) { + LOGGER.debug("Generating state message for {}...", pair); + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + // Temporarily include legacy state for backwards compatibility with the platform + .withData(Jsons.jsonNode(generateDbState(pairToCursorInfoMap))) + .withStream(generateStreamState(pair.get(), cursorInfo.get())); + } else { + LOGGER.warn("Cursor information could not be located in state for stream {}. Returning a new, empty state message...", pair); + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } else { + LOGGER.warn("Stream not provided. Returning a new, empty state message..."); + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } + + /** + * Generates the stream state for the given stream and cursor information. + * + * @param airbyteStreamNameNamespacePair The stream. + * @param cursorInfo The current cursor. + * @return The {@link AirbyteStreamState} representing the current state of the stream. + */ + private AirbyteStreamState generateStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(airbyteStreamNameNamespacePair.getName()).withNamespace(airbyteStreamNameNamespacePair.getNamespace())) + .withStreamState(Jsons.jsonNode(generateDbStreamState(airbyteStreamNameNamespacePair, cursorInfo))); + } + + private CursorBasedStatus generateDbStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + final CursorBasedStatus state = new CursorBasedStatus(); + state.setStateType(StateType.CURSOR_BASED); + state.setVersion(2L); + state.setStreamName(airbyteStreamNameNamespacePair.getName()); + state.setStreamNamespace(airbyteStreamNameNamespacePair.getNamespace()); + state.setCursorField(cursorInfo.getCursorField() == null ? Collections.emptyList() : Lists.newArrayList(cursorInfo.getCursorField())); + state.setCursor(cursorInfo.getCursor()); + if (cursorInfo.getCursorRecordCount() > 0L) { + state.setCursorRecordCount(cursorInfo.getCursorRecordCount()); + } + return state; + } + + /** + * Generates the legacy global state for backwards compatibility. + * + * @param pairToCursorInfoMap The map of stream name/namespace tuple to the current cursor + * information for that stream + * @return The legacy {@link DbState}. + */ + private DbState generateDbState(final Map pairToCursorInfoMap) { + return new DbState() + .withCdc(false) + .withStreams(pairToCursorInfoMap.entrySet().stream() + .sorted(Entry.comparingByKey()) // sort by stream name then namespace for sanity. + .map(e -> generateDbStreamState(e.getKey(), e.getValue())) + .collect(Collectors.toList())); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java index eac9cdd380b1..5c2a48c5a74f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java @@ -7,6 +7,7 @@ import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; @@ -45,7 +46,7 @@ public class PostgresXminHandler { private final JdbcCompatibleSourceOperations sourceOperations; private final JdbcDatabase database; private final String quoteString; - private final XminStatus xminStatus; + private final XminStatus currentXminStatus; private final XminStateManager xminStateManager; private static final Logger LOGGER = LoggerFactory.getLogger(PostgresXminHandler.class); @@ -58,7 +59,7 @@ public PostgresXminHandler(final JdbcDatabase database, this.database = database; this.sourceOperations = sourceOperations; this.quoteString = quoteString; - this.xminStatus = xminStatus; + this.currentXminStatus = xminStatus; this.xminStateManager = xminStateManager; } @@ -138,27 +139,68 @@ private PreparedStatement createXminQueryStatement( quoteString); final String wrappedColumnNames = RelationalDbQueryUtils.enquoteIdentifierList(columnNames, quoteString); + + // Get the xmin status associated with the previous run + final XminStatus previousRunXminStatus = xminStateManager.getXminStatus(airbyteStream); + final PreparedStatement xminPreparedStatement = + getXminPreparedStatement(connection, wrappedColumnNames, fullTableName, previousRunXminStatus, currentXminStatus); + LOGGER.info("Executing query for table {}: {}", tableName, xminPreparedStatement); + return xminPreparedStatement; + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + private PreparedStatement getXminPreparedStatement(final Connection connection, + final String wrappedColumnNames, + final String fullTableName, + final XminStatus prevRunXminStatus, + final XminStatus currentXminStatus) + throws SQLException { + + if (isSingleWraparound(prevRunXminStatus, currentXminStatus)) { // The xmin state that we save represents the lowest XID that is still in progress. To make sure we - // don't miss - // data associated with the current transaction, we have to issue an >= + // don't miss data associated with the current transaction, we have to issue an >=. Because of the + // wraparound, the changes prior to the + // end xmin xid value must also be captured. + LOGGER.info("Detect a single wraparound for {}", fullTableName); + final String sql = String.format("SELECT %s FROM %s WHERE xmin::text::bigint >= ? OR xmin::text::bigint < ?", + wrappedColumnNames, fullTableName); + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + preparedStatement.setLong(1, prevRunXminStatus.getXminXidValue()); + preparedStatement.setLong(2, currentXminStatus.getXminXidValue()); + + return preparedStatement; + } else { + // The xmin state that we save represents the lowest XID that is still in progress. To make sure we + // don't miss data associated with the current transaction, we have to issue an >= final String sql = String.format("SELECT %s FROM %s WHERE xmin::text::bigint >= ?", wrappedColumnNames, fullTableName); final PreparedStatement preparedStatement = connection.prepareStatement(sql.toString()); - - final XminStatus currentStreamXminStatus = xminStateManager.getXminStatus(airbyteStream); - if (currentStreamXminStatus != null) { - preparedStatement.setLong(1, currentStreamXminStatus.getXminXidValue()); + if (prevRunXminStatus != null) { + preparedStatement.setLong(1, prevRunXminStatus.getXminXidValue()); } else { + // In case ctid sync is not possible we will do the initial load using "WHERE xmin >= 0" preparedStatement.setLong(1, 0L); } - LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); + return preparedStatement; - } catch (final SQLException e) { - throw new RuntimeException(e); } } + @VisibleForTesting + static boolean isSingleWraparound(final XminStatus prevRunXminStatus, final XminStatus currentXminStatus) { + // Detect whether the source Postgres DB has undergone a single wraparound event. + return prevRunXminStatus != null && currentXminStatus != null + && currentXminStatus.getNumWraparound() - prevRunXminStatus.getNumWraparound() == 1; + } + + public static boolean shouldPerformFullSync(final XminStatus currentXminStatus, final JsonNode streamState) { + // Detects whether source Postgres DB has undergone multiple wraparound events between syncs. + return streamState.has("num_wraparound") && (currentXminStatus.getNumWraparound() - streamState.get("num_wraparound").asLong() >= 2); + } + // Transforms the given iterator to create an {@link AirbyteRecordMessage} private static AutoCloseableIterator getRecordIterator( final AutoCloseableIterator recordIterator, @@ -196,7 +238,7 @@ private AutoCloseableIterator augmentWithState(final AutoCloseab autoCloseableIterator -> new XminStateIterator( autoCloseableIterator, pair, - xminStatus), + currentXminStatus), recordIterator, AirbyteStreamUtils.convertFromNameAndNamespace(pair.getName(), pair.getNamespace())); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java new file mode 100644 index 000000000000..9316376a5bbe --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.xmin; + +import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; +import static io.airbyte.integrations.source.postgres.ctid.CtidUtils.identifyNewlyAddedStreams; +import static io.airbyte.integrations.source.postgres.xmin.PostgresXminHandler.shouldPerformFullSync; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.CtidStreams; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; +import io.airbyte.integrations.source.postgres.internal.models.XminStatus; +import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The class mainly categorises the streams based on the state type into two categories: 1. Streams + * that need to be synced via ctid iterator: These are streams that are either newly added or did + * not complete their initial sync. 2. Streams that need to be synced via xmin iterator: These are + * streams that have completed their initial sync and are not syncing data incrementally. + */ +public class XminCtidUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(XminCtidUtils.class); + + public static StreamsCategorised categoriseStreams(final StateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog, + final XminStatus currentXminStatus) { + final List rawStateMessages = stateManager.getRawStateMessages(); + final List statesFromCtidSync = new ArrayList<>(); + final List statesFromXminSync = new ArrayList<>(); + + final Set alreadySeenStreams = new HashSet<>(); + final Set streamsStillInCtidSync = new HashSet<>(); + + if (rawStateMessages != null) { + rawStateMessages.forEach(stateMessage -> { + final JsonNode streamState = stateMessage.getStream().getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStream().getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase("ctid")) { + statesFromCtidSync.add(stateMessage); + streamsStillInCtidSync.add(new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), streamDescriptor.getNamespace())); + } else if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase("xmin")) { + if (shouldPerformFullSync(currentXminStatus, streamState)) { + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + LOGGER.info("Detected multiple wraparounds. Will perform a full sync for {}", pair); + streamsStillInCtidSync.add(pair); + } else { + statesFromXminSync.add(stateMessage); + } + } else { + throw new RuntimeException("Unknown state type: " + streamState.get(STATE_TYPE_KEY).asText()); + } + } else { + throw new RuntimeException("State type not present"); + } + alreadySeenStreams.add(new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), streamDescriptor.getNamespace())); + }); + } + + final List newlyAddedIncrementalStreams = + identifyNewlyAddedStreams(fullCatalog, alreadySeenStreams, SyncMode.INCREMENTAL); + final List streamsForCtidSync = new ArrayList<>(); + fullCatalog.getStreams().stream() + .filter(stream -> streamsStillInCtidSync.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .forEach(streamsForCtidSync::add); + + streamsForCtidSync.addAll(newlyAddedIncrementalStreams); + + final List streamsForXminSync = fullCatalog.getStreams().stream() + .filter(stream -> stream.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> !streamsForCtidSync.contains(stream)) + .map(Jsons::clone) + .collect(Collectors.toList()); + + return new StreamsCategorised<>(new CtidStreams(streamsForCtidSync, statesFromCtidSync), new XminStreams(streamsForXminSync, statesFromXminSync)); + } + + public record XminStreams(List streamsForXminSync, + List statesFromXminSync) { + + } + + public static void reclassifyCategorisedCtidStream(final StreamsCategorised categorisedStreams, + AirbyteStreamNameNamespacePair streamPair) { + final Optional foundStream = categorisedStreams + .ctidStreams() + .streamsForCtidSync().stream().filter(c -> Objects.equals( + streamPair, + new AirbyteStreamNameNamespacePair(c.getStream().getName(), c.getStream().getNamespace()))) + .findFirst(); + foundStream.ifPresent(c -> { + categorisedStreams.remainingStreams().streamsForXminSync().add(c); + categorisedStreams.ctidStreams().streamsForCtidSync().remove(c); + LOGGER.info("Reclassified {}.{} as xmin stream", c.getStream().getNamespace(), c.getStream().getName()); + }); + + // Should there ever be a matching ctid state when ctid is not possible? + final Optional foundStateMessage = categorisedStreams + .ctidStreams() + .statesFromCtidSync().stream().filter(m -> Objects.equals(streamPair, + new AirbyteStreamNameNamespacePair( + m.getStream().getStreamDescriptor().getName(), + m.getStream().getStreamDescriptor().getNamespace()))) + .findFirst(); + foundStateMessage.ifPresent(m -> { + categorisedStreams.remainingStreams().statesFromXminSync().add(m); + categorisedStreams.ctidStreams().statesFromCtidSync().remove(m); + }); + } + + public static void reclassifyCategorisedCtidStreams(final StreamsCategorised categorisedStreams, + List streamPairs) { + streamPairs.forEach(c -> reclassifyCategorisedCtidStream(categorisedStreams, c)); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateManager.java index 5459027251a6..8ec2138a5880 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateManager.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateManager.java @@ -4,7 +4,6 @@ package io.airbyte.integrations.source.postgres.xmin; -import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; @@ -27,6 +26,7 @@ public class XminStateManager { private static final Logger LOGGER = LoggerFactory.getLogger(XminStateManager.class); + public static final long XMIN_STATE_VERSION = 2L; private final Map pairToXminStatus; @@ -73,23 +73,26 @@ public XminStatus getXminStatus(final AirbyteStreamNameNamespacePair pair) { * @return AirbyteMessage which includes information on state of records read so far */ public static AirbyteMessage createStateMessage(final AirbyteStreamNameNamespacePair pair, final XminStatus xminStatus) { + final AirbyteStateMessage stateMessage = getAirbyteStateMessage(pair, xminStatus); + + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateMessage); + } + + public static AirbyteStateMessage getAirbyteStateMessage(final AirbyteStreamNameNamespacePair pair, final XminStatus xminStatus) { final AirbyteStreamState airbyteStreamState = new AirbyteStreamState() .withStreamDescriptor( new StreamDescriptor() .withName(pair.getName()) .withNamespace(pair.getNamespace())) - .withStreamState(new ObjectMapper().valueToTree(xminStatus)); + .withStreamState(Jsons.jsonNode(xminStatus)); // Set state - final AirbyteStateMessage stateMessage = - new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(airbyteStreamState); - - return new AirbyteMessage() - .withType(Type.STATE) - .withState(stateMessage); + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(airbyteStreamState); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml b/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml index 8455adb4dead..d46c74c24f4d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml @@ -4,12 +4,41 @@ title: Postgres Models type: object description: Postgres Models properties: - state: + state_type: + "$ref": "#/definitions/StateType" + xmin_state: "$ref": "#/definitions/XminStatus" + ctid_state: + "$ref": "#/definitions/CtidStatus" + cursor_based_state: + "$ref": "#/definitions/CursorBasedStatus" definitions: + StateType: + description: Enum to define the sync mode of state. + type: string + enum: + - cursor_based + - xmin + - ctid + CursorBasedStatus: + type: object + extends: + type: object + existingJavaType: "io.airbyte.integrations.source.relationaldb.models.DbStreamState" + properties: + state_type: + "$ref": "#/definitions/StateType" + version: + description: Version of state. + type: integer XminStatus: type: object properties: + version: + description: Version of state. + type: integer + state_type: + "$ref": "#/definitions/StateType" num_wraparound: description: Number of times the Xmin value has wrapped around. type: integer @@ -19,3 +48,20 @@ definitions: xmin_raw_value: description: The raw value of the xmin snapshot. If no wraparound has occurred, this should be the same as 2. type: integer + CtidStatus: + type: object + properties: + version: + description: Version of state. + type: integer + state_type: + "$ref": "#/definitions/StateType" + ctid: + description: ctid bookmark + type: string + incremental_state: + description: State to switch to after completion of ctid initial sync + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode + relation_filenode: + type: integer diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index 944c03a47a73..bf749864f76c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -229,13 +229,13 @@ "group": "advanced", "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "title": "Standard (Xmin)", + "description": "Xmin replication requires no setup on the DB side but will not be able to represent deletions incrementally.", "required": ["method"], "properties": { "method": { "type": "string", - "const": "Standard", + "const": "Xmin", "order": 0 } } @@ -301,6 +301,18 @@ "order": 7 } } + }, + { + "title": "Standard", + "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Standard", + "order": 8 + } + } } ] } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java index 9e7c756a5fe2..d7dbe4e02ad8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java @@ -588,6 +588,13 @@ protected void initTests() { .addExpectedValues("(\"fuzzy dice\",42,1.99)", null) .build()); + addHstoreTest(); + addTimeWithTimeZoneTest(); + addArraysTestData(); + addMoneyTest(); + } + + protected void addHstoreTest() { addDataTypeTestData( TestDataHolder.builder() .sourceType("hstore") @@ -602,10 +609,6 @@ protected void initTests() { {"ISBN-13":"978-1449370000","weight":"11.2 ounces","paperback":"243","publisher":"postgresqltutorial.com","language":"English"}""", null) .build()); - - addTimeWithTimeZoneTest(); - addArraysTestData(); - addMoneyTest(); } protected void addMoneyTest() { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java index 0c5bd6e5f3e3..da0ce367531d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java @@ -6,18 +6,26 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.standardtest.source.TestDataHolder; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.protocol.models.JsonSchemaType; import java.util.List; import org.jooq.SQLDialect; +import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +@ExtendWith(SystemStubsExtension.class) public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { private static final String SCHEMA_NAME = "test"; @@ -25,9 +33,12 @@ public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgr private static final String PUBLICATION = "publication"; private static final int INITIAL_WAITING_SECONDS = 30; + @SystemStub + private EnvironmentVariables environmentVariables; + @Override protected Database setupDatabase() throws Exception { - + environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); container = new PostgreSQLContainer<>("postgres:14-alpine") .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") @@ -55,7 +66,6 @@ protected Database setupDatabase() throws Exception { .put("replication_method", replicationMethod) .put("is_test", true) .put(JdbcUtils.SSL_KEY, false) - .put("snapshot_mode", "initial_only") .build()); dslContext = DSLContextFactory.create( @@ -99,4 +109,22 @@ public boolean testCatalog() { return true; } + @Override + protected void addHstoreTest() { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("hstore") + .airbyteType(JsonSchemaType.STRING) + .addInsertValues(""" + '"paperback" => "243","publisher" => "postgresqltutorial.com", + "language" => "English","ISBN-13" => "978-1449370000", + "weight" => "11.2 ounces"' + """, null) + .addExpectedValues( + // + "\"weight\"=>\"11.2 ounces\", \"ISBN-13\"=>\"978-1449370000\", \"language\"=>\"English\", \"paperback\"=>\"243\", \"publisher\"=>\"postgresqltutorial.com\"", + null) + .build()); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index db6ac4a5fb0f..ef78c1188f9e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.factory.DSLContextFactory; @@ -31,13 +32,19 @@ import java.util.stream.Collectors; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; // todo (cgardens) - Sanity check that when configured for CDC that postgres performs like any other // incremental source. As we have more sources support CDC we will find a more reusable way of doing // this, but for now this is a solid sanity check. +@ExtendWith(SystemStubsExtension.class) public class CdcPostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { protected static final String SLOT_NAME_BASE = "debezium_slot"; @@ -50,6 +57,14 @@ public class CdcPostgresSourceAcceptanceTest extends AbstractPostgresSourceAccep protected PostgreSQLContainer container; protected JsonNode config; + @SystemStub + private EnvironmentVariables environmentVariables; + + @BeforeEach + void setup() { + environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); + } + @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { container = new PostgreSQLContainer<>("postgres:13-alpine") diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java index b591065733b8..4884a4f59a9c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.factory.DSLContextFactory; @@ -19,12 +20,18 @@ import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.Collections; import java.util.List; import java.util.Set; import org.jooq.SQLDialect; +import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +@ExtendWith(SystemStubsExtension.class) public class CdcWalLogsPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { private static final String SCHEMA_NAME = "test"; @@ -33,6 +40,9 @@ public class CdcWalLogsPostgresSourceDatatypeTest extends AbstractPostgresSource private static final int INITIAL_WAITING_SECONDS = 30; private JsonNode stateAfterFirstSync; + @SystemStub + private EnvironmentVariables environmentVariables; + @Override protected List runRead(final ConfiguredAirbyteCatalog configuredCatalog) throws Exception { if (stateAfterFirstSync == null) { @@ -57,14 +67,11 @@ protected void postSetup() throws Exception { catalog.getStreams().add(dummyTableWithData); final List allMessages = super.runRead(catalog); - if (allMessages.size() != 2) { - throw new RuntimeException("First sync should only generate 2 records"); - } final List stateAfterFirstBatch = extractStateMessages(allMessages); if (stateAfterFirstBatch == null || stateAfterFirstBatch.isEmpty()) { throw new RuntimeException("stateAfterFirstBatch should not be null or empty"); } - stateAfterFirstSync = Jsons.jsonNode(stateAfterFirstBatch); + stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); if (stateAfterFirstSync == null) { throw new RuntimeException("stateAfterFirstSync should not be null"); } @@ -78,7 +85,7 @@ protected void postSetup() throws Exception { @Override protected Database setupDatabase() throws Exception { - + environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); container = new PostgreSQLContainer<>("postgres:14-alpine") .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java index cfb1e967bd93..8ce8007a3f6a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.factory.DSLContextFactory; @@ -15,13 +16,22 @@ import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; import org.jooq.SQLDialect; +import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +@ExtendWith(SystemStubsExtension.class) public class PostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { + @SystemStub + private EnvironmentVariables environmentVariables; + @Override protected Database setupDatabase() throws SQLException { + environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); container = new PostgreSQLContainer<>("postgres:14-alpine") .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java new file mode 100644 index 000000000000..d45fcc7f2bc1 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.HashMap; +import java.util.List; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.containers.PostgreSQLContainer; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; + +@ExtendWith(SystemStubsExtension.class) +public class XminPostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { + + private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME2 = "starships"; + private static final String STREAM_NAME_MATERIALIZED_VIEW = "testview"; + private static final String SCHEMA_NAME = "public"; + @SystemStub + private EnvironmentVariables environmentVariables; + + private PostgreSQLContainer container; + private JsonNode config; + private Database database; + private ConfiguredAirbyteCatalog configCatalog; + + @Override + protected JsonNode getConfig() throws Exception { + return config; + } + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); + + container = new PostgreSQLContainer<>("postgres:13-alpine"); + container.start(); + final String username = container.getUsername(); + final String password = container.getPassword(); + final List schemas = List.of("public"); + config = getXminConfig(username, password, schemas); + try (final DSLContext dslContext = DSLContextFactory.create( + config.get(JdbcUtils.USERNAME_KEY).asText(), + config.get(JdbcUtils.PASSWORD_KEY).asText(), + DatabaseDriver.POSTGRESQL.getDriverClassName(), + String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + container.getHost(), + container.getFirstMappedPort(), + config.get(JdbcUtils.DATABASE_KEY).asText()), + SQLDialect.POSTGRES)) { + database = new Database(dslContext); + + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); + return null; + }); + configCatalog = getXminCatalog(); + } + } + + private JsonNode getXminConfig(final String username, final String password, final List schemas) { + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "Xmin") + .build()); + return Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) + .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) + .put(JdbcUtils.SCHEMAS_KEY, Jsons.jsonNode(schemas)) + .put(JdbcUtils.USERNAME_KEY, username) + .put(JdbcUtils.PASSWORD_KEY, password) + .put(JdbcUtils.SSL_KEY, false) + .put("replication_method", replicationMethod) + .build()); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) throws Exception { + container.close(); + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { + return configCatalog; + } + + @Override + protected JsonNode getState() throws Exception { + return Jsons.jsonNode(new HashMap<>()); + } + + @Override + protected boolean supportsPerStream() { + return true; + } + + private ConfiguredAirbyteCatalog getXminCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME, SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME2, SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME_MATERIALIZED_VIEW, SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))))); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json index d03e067bb550..a691f1e7ee09 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json @@ -229,13 +229,13 @@ "group": "advanced", "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "title": "Standard (Xmin)", + "description": "Xmin replication requires no setup on the DB side but will not be able to represent deletions incrementally.", "required": ["method"], "properties": { "method": { "type": "string", - "const": "Standard", + "const": "Xmin", "order": 0 } } @@ -301,6 +301,18 @@ "order": 7 } } + }, + { + "title": "Standard", + "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Standard", + "order": 8 + } + } } ] }, diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 0bf12801b9e4..9ab452375bc8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -7,8 +7,10 @@ import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -18,6 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Streams; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -41,23 +44,33 @@ import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import javax.sql.DataSource; import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; @@ -138,7 +151,7 @@ private JsonNode getConfig(final String dbName, final String userName, final Str .put(JdbcUtils.SSL_KEY, false) .put("is_test", true) .put("replication_method", replicationMethod) - .put("sync_checkpoint_records", 2) + .put("sync_checkpoint_records", 1) .build()); } @@ -215,7 +228,7 @@ void testCheckWithoutReplicationPermission() throws Exception { @Test void testCheckWithoutPublication() throws Exception { database.query(ctx -> ctx.execute("DROP PUBLICATION " + PUBLICATION + ";")); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source.check(getConfig()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); } @@ -224,12 +237,259 @@ void testCheckWithoutReplicationSlot() throws Exception { final String fullReplicationSlot = SLOT_NAME_BASE + "_" + dbName; database.query(ctx -> ctx.execute("SELECT pg_drop_replication_slot('" + fullReplicationSlot + "');")); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source.check(getConfig()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); } @Override protected void assertExpectedStateMessages(final List stateMessages) { + assertEquals(7, stateMessages.size()); + assertStateTypes(stateMessages, 4); + } + + @Override + protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(final List stateAfterFirstBatch) { + assertEquals(27, stateAfterFirstBatch.size()); + assertStateTypes(stateAfterFirstBatch, 24); + } + + private void assertStateTypes(final List stateMessages, final int indexTillWhichExpectCtidState) { + JsonNode sharedState = null; + for (int i = 0; i < stateMessages.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + if (i <= indexTillWhichExpectCtidState) { + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals("ctid", streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else { + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } + } + } + + @Override + protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, + final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion) { + assertEquals(7, stateMessages.size()); + for (int i = 0; i <= 4; i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessage.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessage.getGlobal().getSharedState()); + final Set streamsInSnapshotState = stateMessage.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + + stateMessage.getGlobal().getStreamStates().forEach(s -> { + final JsonNode streamState = s.getStreamState(); + if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))) { + assertEquals("ctid", streamState.get(STATE_TYPE_KEY).asText()); + } else if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))) { + assertFalse(streamState.has(STATE_TYPE_KEY)); + } else { + throw new RuntimeException("Unknown stream"); + } + }); + } + + final AirbyteStateMessage secondLastSateMessage = stateMessages.get(5); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, secondLastSateMessage.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + secondLastSateMessage.getGlobal().getSharedState()); + final Set streamsInSnapshotState = secondLastSateMessage.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + secondLastSateMessage.getGlobal().getStreamStates().forEach(s -> { + final JsonNode streamState = s.getStreamState(); + assertFalse(streamState.has(STATE_TYPE_KEY)); + }); + + final AirbyteStateMessage stateMessageEmittedAfterSecondSyncCompletion = stateMessages.get(6); + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSecondSyncCompletion.getType()); + assertNotEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); + final Set streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSyncCompletionState.contains( + new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); + assertTrue(streamsInSyncCompletionState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + assertNotNull(stateMessageEmittedAfterSecondSyncCompletion.getData()); + } + + @Test + public void testTwoStreamSync() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(CONFIGURED_CATALOG); + + final List MODEL_RECORDS_2 = ImmutableList.of( + Jsons.jsonNode(ImmutableMap.of(COL_ID, 110, COL_MAKE_ID, 1, COL_MODEL, "Fiesta-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 120, COL_MAKE_ID, 1, COL_MODEL, "Focus-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 130, COL_MAKE_ID, 1, COL_MODEL, "Ranger-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 140, COL_MAKE_ID, 2, COL_MODEL, "GLA-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 150, COL_MAKE_ID, 2, COL_MODEL, "A 220-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 160, COL_MAKE_ID, 2, COL_MODEL, "E 350-2"))); + + createTable(MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", + columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); + + for (final JsonNode recordJson : MODEL_RECORDS_2) { + writeRecords(recordJson, MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", COL_ID, + COL_MAKE_ID, COL_MODEL); + } + + final ConfiguredAirbyteStream airbyteStream = new ConfiguredAirbyteStream() + .withStream(CatalogHelpers.createAirbyteStream( + MODELS_STREAM_NAME + "_2", + MODELS_SCHEMA, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), + Field.of(COL_MODEL, JsonSchemaType.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID)))); + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + + final List streams = configuredCatalog.getStreams(); + streams.add(airbyteStream); + configuredCatalog.withStreams(streams); + + final AutoCloseableIterator read1 = getSource() + .read(getConfig(), configuredCatalog, null); + final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); + + final Set recordMessages1 = extractRecordMessages(actualRecords1); + final List stateMessages1 = extractStateMessages(actualRecords1); + assertEquals(13, stateMessages1.size()); + JsonNode sharedState = null; + StreamDescriptor firstStreamInState = null; + for (int i = 0; i < stateMessages1.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages1.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + + if (Objects.isNull(firstStreamInState)) { + assertEquals(1, global.getStreamStates().size()); + firstStreamInState = global.getStreamStates().get(0).getStreamDescriptor(); + } + + if (i <= 4) { + // First 4 state messages are ctid state + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals("ctid", streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else if (i == 5) { + // 5th state message is the final state message emitted for the stream + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } else if (i <= 10) { + // 6th to 10th is the ctid state message for the 2nd stream but final state message for 1st stream + assertEquals(2, global.getStreamStates().size()); + final StreamDescriptor finalFirstStreamInState = firstStreamInState; + global.getStreamStates().forEach(c -> { + if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { + assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); + } else { + assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); + assertEquals("ctid", c.getStreamState().get(STATE_TYPE_KEY).asText()); + } + }); + } else { + // last 2 state messages don't contain ctid info cause ctid sync should be complete + assertEquals(2, global.getStreamStates().size()); + global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); + } + } + + final Set names = new HashSet<>(STREAM_NAMES); + names.add(MODELS_STREAM_NAME + "_2"); + assertExpectedRecords(Streams.concat(MODEL_RECORDS_2.stream(), MODEL_RECORDS.stream()) + .collect(Collectors.toSet()), + recordMessages1, + names, + names, + MODELS_SCHEMA); + + assertEquals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA), firstStreamInState); + + // Triggering a sync with a ctid state for 1 stream and complete state for other stream + final AutoCloseableIterator read2 = getSource() + .read(getConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateMessages1.get(6)))); + final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); + + final List stateMessages2 = extractStateMessages(actualRecords2); + + assertEquals(6, stateMessages2.size()); + for (int i = 0; i < stateMessages2.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages2.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + assertEquals(2, global.getStreamStates().size()); + + if (i <= 3) { + final StreamDescriptor finalFirstStreamInState = firstStreamInState; + global.getStreamStates().forEach(c -> { + // First 4 state messages are ctid state for the stream that didn't complete ctid sync the first + // time + if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { + assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); + } else { + assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); + assertEquals("ctid", c.getStreamState().get(STATE_TYPE_KEY).asText()); + } + }); + } else { + // last 2 state messages don't contain ctid info cause ctid sync should be complete + global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); + } + } + + final Set recordMessages2 = extractRecordMessages(actualRecords2); + assertEquals(5, recordMessages2.size()); + assertExpectedRecords(new HashSet<>(MODEL_RECORDS_2.subList(1, MODEL_RECORDS_2.size())), + recordMessages2, + names, + names, + MODELS_SCHEMA); + } + + @Override + protected void assertExpectedStateMessagesForNoData(final List stateMessages) { + assertEquals(2, stateMessages.size()); + } + + @Override + protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { assertEquals(1, stateMessages.size()); assertNotNull(stateMessages.get(0).getData()); } @@ -292,6 +552,11 @@ protected void addCdcMetadataColumns(final AirbyteStream stream) { } + @Override + protected void addCdcDefaultCursorField(final AirbyteStream stream) { + stream.setDefaultCursorField(ImmutableList.of(CDC_LSN)); + } + @Override protected Source getSource() { return source; @@ -329,7 +594,7 @@ void testDiscoverFiltersNonPublication() throws Exception { database.query(ctx -> ctx.execute("DROP PUBLICATION " + PUBLICATION + ";")); database.query(ctx -> ctx.execute(String.format("CREATE PUBLICATION " + PUBLICATION + " FOR TABLE %s.%s", MODELS_SCHEMA, "models"))); - final AirbyteCatalog catalog = source.discover(config); + final AirbyteCatalog catalog = source.discover(getConfig()); assertEquals(catalog.getStreams().size(), 2); final AirbyteStream streamInPublication = catalog.getStreams().stream().filter(stream -> stream.getName().equals("models")).findFirst().get(); @@ -369,8 +634,6 @@ public void testTableWithTimestampColDefault() throws Exception { final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); - assertEquals(1, stateAfterFirstBatch.size()); - assertNotNull(stateAfterFirstBatch.get(0).getData()); assertExpectedStateMessages(stateAfterFirstBatch); final Set recordsFromFirstBatch = extractRecordMessages( dataFromFirstBatch); @@ -421,7 +684,7 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); - + assertExpectedStateMessages(stateAfterFirstBatch); // second batch of records again 20 being created for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { final JsonNode record = @@ -431,13 +694,14 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { writeModelRecord(record); } - final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); + // Extract the last state message + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); final AutoCloseableIterator secondBatchIterator = getSource() .read(config, CONFIGURED_CATALOG, state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); - assertExpectedStateMessages(stateAfterSecondBatch); + assertExpectedStateMessagesFromIncrementalSync(stateAfterSecondBatch); for (int recordsCreated = 0; recordsCreated < 1; recordsCreated++) { final JsonNode record = @@ -456,13 +720,18 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { .toListAndClose(thirdBatchIterator); final List stateAfterThirdBatch = extractStateMessages(dataFromThirdBatch); - assertExpectedStateMessages(stateAfterThirdBatch); + assertStateForSyncShouldHandlePurgedLogsGracefully(stateAfterThirdBatch); final Set recordsFromThirdBatch = extractRecordMessages( dataFromThirdBatch); assertEquals(MODEL_RECORDS.size() + recordsToCreate + 1, recordsFromThirdBatch.size()); } + protected void assertStateForSyncShouldHandlePurgedLogsGracefully(final List stateMessages) { + assertEquals(28, stateMessages.size()); + assertStateTypes(stateMessages, 25); + } + @Test void testReachedTargetPosition() { final PostgresCdcTargetPosition ctp = cdcLatestTargetPosition(); @@ -489,19 +758,19 @@ protected void syncShouldIncrementLSN() throws Exception { final JdbcDatabase defaultJdbcDatabase = new DefaultJdbcDatabase(dataSource); final Long replicationSlotAtTheBeginning = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, config).get(0).get("confirmed_flush_lsn").asText()).asLong(); + source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); final AutoCloseableIterator firstBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, null); + .read(getConfig(), CONFIGURED_CATALOG, null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); final Long replicationSlotAfterFirstSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, config).get(0).get("confirmed_flush_lsn").asText()).asLong(); + source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // First sync should not make any change to the replication slot status - assertEquals(replicationSlotAtTheBeginning, replicationSlotAfterFirstSync); + assertLsnPositionForSyncShouldIncrementLSN(replicationSlotAtTheBeginning, replicationSlotAfterFirstSync, 1); // second batch of records again 20 being created for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { @@ -512,19 +781,19 @@ protected void syncShouldIncrementLSN() throws Exception { writeModelRecord(record); } - final JsonNode stateAfterFirstSync = Jsons.jsonNode(stateAfterFirstBatch); + final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); final AutoCloseableIterator secondBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, stateAfterFirstSync); + .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); - assertExpectedStateMessages(stateAfterSecondBatch); + assertExpectedStateMessagesFromIncrementalSync(stateAfterSecondBatch); final Long replicationSlotAfterSecondSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, config).get(0).get("confirmed_flush_lsn").asText()).asLong(); + source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // Second sync should move the replication slot ahead - assertEquals(1, replicationSlotAfterSecondSync.compareTo(replicationSlotAfterFirstSync)); + assertLsnPositionForSyncShouldIncrementLSN(replicationSlotAfterFirstSync, replicationSlotAfterSecondSync, 2); for (int recordsCreated = 0; recordsCreated < 1; recordsCreated++) { final JsonNode record = @@ -537,17 +806,17 @@ protected void syncShouldIncrementLSN() throws Exception { // Triggering sync with the first sync's state only which would mimic a scenario that the second // sync failed on destination end, and we didn't save state final AutoCloseableIterator thirdBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, stateAfterFirstSync); + .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); final List dataFromThirdBatch = AutoCloseableIterators .toListAndClose(thirdBatchIterator); final List stateAfterThirdBatch = extractStateMessages(dataFromThirdBatch); - assertExpectedStateMessages(stateAfterThirdBatch); + assertExpectedStateMessagesFromIncrementalSync(stateAfterThirdBatch); final Set recordsFromThirdBatch = extractRecordMessages( dataFromThirdBatch); final Long replicationSlotAfterThirdSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, config).get(0).get("confirmed_flush_lsn").asText()).asLong(); + source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // Since we used the state, no change should happen to the replication slot assertEquals(replicationSlotAfterSecondSync, replicationSlotAfterThirdSync); @@ -562,27 +831,39 @@ protected void syncShouldIncrementLSN() throws Exception { } final AutoCloseableIterator fourthBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, Jsons.jsonNode(stateAfterThirdBatch)); + .read(getConfig(), CONFIGURED_CATALOG, Jsons.jsonNode(Collections.singletonList(stateAfterThirdBatch.get(stateAfterThirdBatch.size() - 1)))); final List dataFromFourthBatch = AutoCloseableIterators .toListAndClose(fourthBatchIterator); final List stateAfterFourthBatch = extractStateMessages(dataFromFourthBatch); - assertExpectedStateMessages(stateAfterFourthBatch); + assertExpectedStateMessagesFromIncrementalSync(stateAfterFourthBatch); final Set recordsFromFourthBatch = extractRecordMessages( dataFromFourthBatch); final Long replicationSlotAfterFourthSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, config).get(0).get("confirmed_flush_lsn").asText()).asLong(); + source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // Fourth sync should again move the replication slot ahead assertEquals(1, replicationSlotAfterFourthSync.compareTo(replicationSlotAfterThirdSync)); assertEquals(1, recordsFromFourthBatch.size()); } + protected void assertLsnPositionForSyncShouldIncrementLSN(final Long lsnPosition1, + final Long lsnPosition2, + final int syncNumber) { + if (syncNumber == 1) { + assertEquals(1, lsnPosition2.compareTo(lsnPosition1)); + } else if (syncNumber == 2) { + assertEquals(0, lsnPosition2.compareTo(lsnPosition1)); + } else { + throw new RuntimeException("Unknown sync number " + syncNumber); + } + } + /** - * This test verify that multiple states are sent during the CDC process based on number of records. - * We can ensure that more than one `STATE` type of message is sent, but we are not able to assert - * the exact number of messages sent as depends on Debezium. + * This test verifies that multiple states are sent during the CDC process based on number of + * records. We can ensure that more than one `STATE` type of message is sent, but we are not able to + * assert the exact number of messages sent as depends on Debezium. * * @throws Exception Exception happening in the test. */ @@ -591,17 +872,15 @@ protected void verifyCheckpointStatesByRecords() throws Exception { // We require a huge amount of records, otherwise Debezium will notify directly the last offset. final int recordsToCreate = 20000; - ((ObjectNode) config).put("sync_checkpoint_records", 100); - final AutoCloseableIterator firstBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, null); + .read(getConfig(), CONFIGURED_CATALOG, null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateMessages = extractStateMessages(dataFromFirstBatch); // As first `read` operation is from snapshot, it would generate only one state message at the end // of the process. - assertEquals(1, stateMessages.size()); + assertExpectedStateMessages(stateMessages); for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { final JsonNode record = @@ -611,9 +890,9 @@ protected void verifyCheckpointStatesByRecords() throws Exception { writeModelRecord(record); } - final JsonNode stateAfterFirstSync = Jsons.jsonNode(stateMessages); + final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateMessages.get(stateMessages.size() - 1))); final AutoCloseableIterator secondBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, stateAfterFirstSync); + .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); @@ -623,28 +902,27 @@ protected void verifyCheckpointStatesByRecords() throws Exception { } /** - * This test verify that multiple states are sent during the CDC process based on time ranges. We + * This test verifies that multiple states are sent during the CDC process based on time ranges. We * can ensure that more than one `STATE` type of message is sent, but we are not able to assert the * exact number of messages sent as depends on Debezium. * * @throws Exception Exception happening in the test. */ + @Disabled("Disabled 'verifyCheckpointStatesBySeconds' test as flaky. https://github.com/airbytehq/airbyte/issues/29411") @Test protected void verifyCheckpointStatesBySeconds() throws Exception { // We require a huge amount of records, otherwise Debezium will notify directly the last offset. final int recordsToCreate = 20000; - ((ObjectNode) config).put("sync_checkpoint_seconds", 1); - final AutoCloseableIterator firstBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, null); + .read(getConfig(), CONFIGURED_CATALOG, null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateMessages = extractStateMessages(dataFromFirstBatch); // As first `read` operation is from snapshot, it would generate only one state message at the end // of the process. - assertEquals(1, stateMessages.size()); + assertExpectedStateMessages(stateMessages); for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { final JsonNode record = @@ -653,8 +931,11 @@ protected void verifyCheckpointStatesBySeconds() throws Exception { "F-" + recordsCreated)); writeModelRecord(record); } + final JsonNode config = getConfig(); + ((ObjectNode) config).put("sync_checkpoint_seconds", 1); + ((ObjectNode) config).put("sync_checkpoint_records", 100_000); - final JsonNode stateAfterFirstSync = Jsons.jsonNode(stateMessages); + final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateMessages.get(stateMessages.size() - 1))); final AutoCloseableIterator secondBatchIterator = getSource() .read(config, CONFIGURED_CATALOG, stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java new file mode 100644 index 000000000000..2e6d71c5667b --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.cdk.CDKConstants; +import org.junit.jupiter.api.Test; + +class CdkImportTest { + + /** + * This test ensures that the CDK is able to be imported and that its version number matches the + * expected pinned version. + * + * This test can be removed once pinned CDK version reaches v0.1, at which point the CDK will be + * used for base-java imports, and this test will no longer be necessary. + */ + @Test + void cdkVersionShouldMatch() { + // Should fail in unit test phase: + assertEquals("0.0.2", CDKConstants.VERSION.replace("-SNAPSHOT", "")); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java index 6d269d216843..599601b1518b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -16,6 +17,7 @@ import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; class PostgresCatalogHelperTest { @@ -65,6 +67,17 @@ public void testSetIncrementalToSourceDefined() { .getSourceDefinedCursor()); } + @Test + public void testSetDefaultCursorFieldForCdc() { + final AirbyteStream cdcIncrementalStream = new AirbyteStream() + .withSourceDefinedCursor(true) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH)); + PostgresCatalogHelper.setDefaultCursorFieldForCdc(cdcIncrementalStream); + + assertTrue(cdcIncrementalStream.getSourceDefinedCursor()); + assertEquals(cdcIncrementalStream.getDefaultCursorField(), ImmutableList.of(CDC_LSN)); + } + @Test public void testAddCdcMetadataColumns() { final AirbyteStream before = new AirbyteStream() diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java index 504c9fcc92ca..99e402077d30 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java @@ -4,6 +4,14 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.createRecord; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractSpecificFieldFromCombinedMessages; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractStateMessage; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.filterRecords; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.map; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,12 +25,15 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; +import io.airbyte.commons.util.MoreIterators; import io.airbyte.db.factory.DataSourceFactory; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.StreamingJdbcDatabase; import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; @@ -30,16 +41,22 @@ import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -303,6 +320,10 @@ void testSpec() throws Exception { @Override protected List getTestMessages() { + return getTestMessages(streamName); + } + + protected List getTestMessages(final String streamName) { return Lists.newArrayList( new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) @@ -422,37 +443,6 @@ void incrementalTimestampCheck() throws Exception { getTestMessages().get(2))); } - @Override - protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { - final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19", - COL_WAKEUP_AT, "12:12:12.123456-05:00", - COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", - COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19", - COL_WAKEUP_AT, "12:12:12.123456-05:00", - COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", - COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))))); - final DbStreamState state = new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)) - .withCursor("5") - .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); - return expectedMessages; - } - @Override protected boolean supportsPerStream() { return true; @@ -523,4 +513,295 @@ public void testUserHasNoPermissionToDataBase() throws Exception { assertTrue(status.getMessage().contains("State code: 42501;")); } + @Test + void testReadMultipleTablesIncrementally() throws Exception { + ((ObjectNode) config).put("sync_checkpoint_records", 1); + final String namespace = getDefaultNamespace(); + final String streamOneName = TABLE_NAME + "one"; + // Create a fresh first table + database.execute(connection -> { + connection.createStatement().execute( + createTableQuery(getFullyQualifiedTableName(streamOneName), COLUMN_CLAUSE_WITH_PK, + primaryKeyClause(Collections.singletonList("id")))); + connection.createStatement().execute( + String.format( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','10:10:10.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName))); + connection.createStatement().execute( + String.format( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName))); + connection.createStatement().execute( + String.format( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName))); + }); + + // Create a fresh second table + final String streamTwoName = TABLE_NAME + "two"; + final String streamTwoFullyQualifiedName = getFullyQualifiedTableName(streamTwoName); + // Insert records into second table + database.execute(ctx -> { + ctx.createStatement().execute( + createTableQuery(streamTwoFullyQualifiedName, COLUMN_CLAUSE_WITH_PK, "")); + ctx.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (40,'Jean Luc','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName)); + ctx.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (41, 'Groot', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName)); + ctx.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (42, 'Thanos','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName)); + }); + // Create records list that we expect to see in the state message + final List streamTwoExpectedRecords = Arrays.asList( + createRecord(streamTwoName, namespace, map( + COL_ID, 40, + COL_NAME, "Jean Luc", + COL_UPDATED_AT, "2006-10-19", + COL_WAKEUP_AT, "12:12:12.123456-05:00", + COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", + COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456")), + createRecord(streamTwoName, namespace, map( + COL_ID, 41, + COL_NAME, "Groot", + COL_UPDATED_AT, "2006-10-19", + COL_WAKEUP_AT, "12:12:12.123456-05:00", + COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", + COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456")), + createRecord(streamTwoName, namespace, map( + COL_ID, 42, + COL_NAME, "Thanos", + COL_UPDATED_AT, "2006-10-19", + COL_WAKEUP_AT, "12:12:12.123456-05:00", + COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", + COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))); + + // Prep and create a configured catalog to perform sync + final AirbyteStream streamOne = getAirbyteStream(streamOneName, namespace); + final AirbyteStream streamTwo = getAirbyteStream(streamTwoName, namespace); + + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( + new AirbyteCatalog().withStreams(List.of(streamOne, streamTwo))); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + airbyteStream.withPrimaryKey(List.of(List.of(COL_ID))); + }); + + // Perform initial sync + final List messagesFromFirstSync = MoreIterators + .toList(source.read(config, configuredCatalog, null)); + + final List recordsFromFirstSync = filterRecords(messagesFromFirstSync); + + setEmittedAtToNull(messagesFromFirstSync); + // All records in the 2 configured streams should be present + assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( + Stream.concat(getTestMessages(streamOneName).stream().parallel(), + streamTwoExpectedRecords.stream().parallel()).collect(toList())); + + final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); + // Since we are emitting a state message after each record, we should have 1 state for each record - + // 3 from stream1 and 3 from stream2 + assertEquals(6, actualFirstSyncState.size()); + + // The expected state type should be 2 ctid's and the last one being standard + final List expectedStateTypesFromFirstSync = List.of("ctid", "ctid", "cursor_based"); + final List stateTypeOfStreamOneStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, STATE_TYPE_KEY); + final List stateTypeOfStreamTwoStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamTwoName, STATE_TYPE_KEY); + // It should be the same for stream1 and stream2 + assertEquals(stateTypeOfStreamOneStatesFromFirstSync, expectedStateTypesFromFirstSync); + assertEquals(stateTypeOfStreamTwoStatesFromFirstSync, expectedStateTypesFromFirstSync); + + // Create the expected ctids that we should see + final List expectedCtidsFromFirstSync = List.of("(0,1)", "(0,2)"); + final List ctidFromStreamOneStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, "ctid"); + final List ctidFromStreamTwoStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, "ctid"); + + // Verifying each element and its index to match. + // Only checking the first 2 elements since we have verified that the last state_type is + // "cursor_based" + assertEquals(ctidFromStreamOneStatesFromFirstSync.get(0), expectedCtidsFromFirstSync.get(0)); + assertEquals(ctidFromStreamOneStatesFromFirstSync.get(1), expectedCtidsFromFirstSync.get(1)); + assertEquals(ctidFromStreamTwoStatesFromFirstSync.get(0), expectedCtidsFromFirstSync.get(0)); + assertEquals(ctidFromStreamTwoStatesFromFirstSync.get(1), expectedCtidsFromFirstSync.get(1)); + + // Extract only state messages for each stream + final List streamOneStateMessagesFromFirstSync = extractStateMessage(messagesFromFirstSync, streamOneName); + final List streamTwoStateMessagesFromFirstSync = extractStateMessage(messagesFromFirstSync, streamTwoName); + // Extract the incremental states of each stream's first and second state message + final List streamOneIncrementalStatesFromFirstSync = + List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); + + final List streamTwoIncrementalStatesFromFirstSync = + List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); + + // The incremental_state of each stream's first and second incremental states is expected + // to be identical to the stream_state of the final state message for each stream + assertEquals(streamOneIncrementalStatesFromFirstSync.get(0), streamOneFinalStreamStateFromFirstSync); + assertEquals(streamOneIncrementalStatesFromFirstSync.get(1), streamOneFinalStreamStateFromFirstSync); + assertEquals(streamTwoIncrementalStatesFromFirstSync.get(0), streamTwoFinalStreamStateFromFirstSync); + assertEquals(streamTwoIncrementalStatesFromFirstSync.get(1), streamTwoFinalStreamStateFromFirstSync); + + // Sync should work with a ctid state AND a cursor-based state from each stream + // Forcing a sync with + // - stream one state still being the first record read via CTID. + // - stream two state being the CTID state before the final emitted state before the cursor switch + final List messagesFromSecondSyncWithMixedStates = MoreIterators + .toList(source.read(config, configuredCatalog, + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); + + // Extract only state messages for each stream after second sync + final List streamOneStateMessagesFromSecondSync = + extractStateMessage(messagesFromSecondSyncWithMixedStates, streamOneName); + final List stateTypeOfStreamOneStatesFromSecondSync = + extractSpecificFieldFromCombinedMessages(messagesFromSecondSyncWithMixedStates, streamOneName, STATE_TYPE_KEY); + + final List streamTwoStateMessagesFromSecondSync = + extractStateMessage(messagesFromSecondSyncWithMixedStates, streamTwoName); + final List stateTypeOfStreamTwoStatesFromSecondSync = + extractSpecificFieldFromCombinedMessages(messagesFromSecondSyncWithMixedStates, streamTwoName, STATE_TYPE_KEY); + + // Stream One states after the second sync are expected to have 2 stream states + // - 1 with Ctid state_type and 1 state that is of cursorBased state type + assertEquals(2, streamOneStateMessagesFromSecondSync.size()); + assertEquals(List.of("ctid", "cursor_based"), stateTypeOfStreamOneStatesFromSecondSync); + + // Stream Two states after the second sync are expected to have 1 stream state + // - The state that is of cursorBased state type + assertEquals(1, streamTwoStateMessagesFromSecondSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamTwoStatesFromSecondSync); + + // Add some data to each table and perform a third read. + // Expect to see all records be synced via cursorBased method and not ctid + + database.execute(ctx -> { + ctx.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (4,'Hooper','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName))); + ctx.createStatement().execute( + String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (43, 'Iron Man', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName)); + }); + + final List messagesFromThirdSync = MoreIterators + .toList(source.read(config, configuredCatalog, + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); + + // Extract only state messages, state type, and cursor for each stream after second sync + final List streamOneStateMessagesFromThirdSync = + extractStateMessage(messagesFromThirdSync, streamOneName); + final List stateTypeOfStreamOneStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamOneName, STATE_TYPE_KEY); + final List cursorOfStreamOneStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamOneName, "cursor"); + + final List streamTwoStateMessagesFromThirdSync = + extractStateMessage(messagesFromThirdSync, streamTwoName); + final List stateTypeOfStreamTwoStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamTwoName, STATE_TYPE_KEY); + final List cursorOfStreamTwoStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamTwoName, "cursor"); + + // Both streams should now be synced via standard cursor and have updated max cursor values + // cursor: 4 for stream one + // cursor: 43 for stream two + assertEquals(1, streamOneStateMessagesFromThirdSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamOneStatesFromThirdSync); + assertEquals(List.of("4"), cursorOfStreamOneStatesFromThirdSync); + + assertEquals(1, streamTwoStateMessagesFromThirdSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamTwoStatesFromThirdSync); + assertEquals(List.of("43"), cursorOfStreamTwoStatesFromThirdSync); + } + + @Override + protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configuredAirbyteStream, + final String cursorField, + final String cursorValue) { + return new CursorBasedStatus().withStateType(StateType.CURSOR_BASED).withVersion(2L) + .withStreamName(configuredAirbyteStream.getStream().getName()) + .withStreamNamespace(configuredAirbyteStream.getStream().getNamespace()) + .withCursorField(List.of(cursorField)) + .withCursor(cursorValue) + .withCursorRecordCount(1L); + } + + @Override + protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { + final List expectedMessages = new ArrayList<>(); + expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19", + COL_WAKEUP_AT, "12:12:12.123456-05:00", + COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", + COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))))); + expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19", + COL_WAKEUP_AT, "12:12:12.123456-05:00", + COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", + COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))))); + final DbStreamState state = new CursorBasedStatus() + .withStateType(StateType.CURSOR_BASED) + .withVersion(2L) + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L); + + expectedMessages.addAll(createExpectedTestMessages(List.of(state))); + return expectedMessages; + } + + private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { + return CatalogHelpers.createAirbyteStream( + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE), + Field.of(COL_WAKEUP_AT, JsonSchemaType.STRING_TIME_WITH_TIMEZONE), + Field.of(COL_LAST_VISITED_AT, JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE), + Field.of(COL_LAST_COMMENT_AT, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); + } + + // + @Override + protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { + final JsonNode streamState = airbyteMessage.getState().getStream().getStreamState(); + if (streamState.get("stream_name").asText().equals(streamName)) { + return streamState; + } + + throw new IllegalArgumentException("Stream not found in state message: " + streamName); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 00034f96f8b6..6ef7d6a0454c 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -19,6 +19,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; @@ -62,11 +63,19 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +@ExtendWith(SystemStubsExtension.class) class PostgresSourceTest { + @SystemStub + private EnvironmentVariables environmentVariables; + private static final String SCHEMA_NAME = "public"; private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME_PRIVILEGES_TEST_CASE = "id_and_name_3"; @@ -146,6 +155,7 @@ static void init() { @BeforeEach void setup() throws Exception { + environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); final String initScriptName = "init_" + dbName.concat(".sql"); @@ -161,7 +171,7 @@ void setup() throws Exception { "CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));"); ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); ctx.fetch( - "INSERT INTO id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1), (1,'goku', 'Infinity'), ('NaN', 'piccolo', '-Infinity');"); + "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);"); ctx.fetch( @@ -518,6 +528,7 @@ void testReadIncrementalSuccess() throws Exception { ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); return null; }); + final ConfiguredAirbyteCatalog configuredCatalog = CONFIGURED_INCR_CATALOG .withStreams(CONFIGURED_INCR_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( @@ -537,7 +548,7 @@ void testReadIncrementalSuccess() throws Exception { createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("3.0"), "name", "vegeta", "power", 222.1))); // Assert that the correct number of messages are emitted. - assertEquals(actualMessages.size(), expectedOutput.size() + 2); + assertEquals(actualMessages.size(), expectedOutput.size() + 1); assertThat(actualMessages.contains(expectedOutput)); // Assert that the Postgres source is emitting records & state messages in the correct order. assertCorrectRecordOrderForIncrementalSync(actualMessages, "id", JsonSchemaPrimitive.NUMBER, configuredCatalog, @@ -585,7 +596,7 @@ private void assertCorrectRecordOrderForIncrementalSync(final List cursorInfoOptional = stateManager.getCursorInfo(pair); final String cursorCandidate = cursorInfoOptional.get().getCursor(); assertThat(IncrementalUtils.compareCursors(prevStateCursorValue, cursorCandidate, cursorType)).isLessThanOrEqualTo(0); @@ -790,4 +801,55 @@ private static ConfiguredAirbyteStream toIncrementalConfiguredStream(final Airby .withPrimaryKey(new ArrayList<>()); } + @Test + void testParseJdbcParameters() { + final String jdbcPropertiesString = "foo=bar&options=-c%20search_path=test,public,pg_catalog%20-c%20statement_timeout=90000&baz=quux"; + Map parameters = PostgresSource.parseJdbcParameters(jdbcPropertiesString, "&"); + assertEquals("-c%20search_path=test,public,pg_catalog%20-c%20statement_timeout=90000", parameters.get("options")); + assertEquals("bar", parameters.get("foo")); + assertEquals("quux", parameters.get("baz")); + } + + @Test + public void testJdbcOptionsParameter() throws Exception { + try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { + db.start(); + + // Populate DB. + final JsonNode dbConfig = getConfig(db); + try (final DSLContext dslContext = getDslContext(dbConfig)) { + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_bytes (id INTEGER, bytes BYTEA);"); + ctx.fetch("INSERT INTO id_and_bytes (id, bytes) VALUES (1, decode('DEADBEEF', 'hex'));"); + return null; + }); + } + + // Read the table contents using the non-default 'escape' format for bytea values. + final JsonNode sourceConfig = Jsons.jsonNode(ImmutableMap.builder() + .putAll(Jsons.flatten(dbConfig)) + .put(JdbcUtils.JDBC_URL_PARAMS_KEY, "options=-c%20statement_timeout=90000%20-c%20bytea_output=escape") + .build()); + final AirbyteStream airbyteStream = CatalogHelpers.createAirbyteStream( + "id_and_bytes", + SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("bytes", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))); + final AirbyteCatalog airbyteCatalog = new AirbyteCatalog().withStreams(List.of(airbyteStream)); + final Set actualMessages = + MoreIterators.toSet(new PostgresSource().read(sourceConfig, CatalogHelpers.toDefaultConfiguredCatalog(airbyteCatalog), null)); + setEmittedAtToNull(actualMessages); + + // Check that the 'options' JDBC URL parameter was parsed correctly + // and that the bytea value is not in the default 'hex' format. + assertEquals(1, actualMessages.size()); + final AirbyteMessage actualMessage = actualMessages.stream().findFirst().get(); + assertTrue(actualMessage.getRecord().getData().has("bytes")); + assertEquals("\\336\\255\\276\\357", actualMessage.getRecord().getData().get("bytes").asText()); + } + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java index ee1ec29ae5fd..bba095875aff 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java @@ -5,16 +5,18 @@ package io.airbyte.integrations.source.postgres; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.createRecord; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractStateMessage; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.filterRecords; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.map; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.setEmittedAtToNull; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; @@ -38,9 +40,12 @@ import io.airbyte.protocol.models.v0.SyncMode; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.jooq.DSLContext; import org.jooq.SQLDialect; @@ -61,7 +66,7 @@ class XminPostgresSourceTest { @SystemStub private EnvironmentVariables environmentVariables; private static final String SCHEMA_NAME = "public"; - private static final String STREAM_NAME = "id_and_name"; + protected static final String STREAM_NAME = "id_and_name"; private static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( STREAM_NAME, @@ -90,19 +95,19 @@ class XminPostgresSourceTest { .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("first_name"), List.of("last_name"))))); - private static final ConfiguredAirbyteCatalog CONFIGURED_XMIN_CATALOG = toConfiguredXminCatalog(CATALOG); + protected static final ConfiguredAirbyteCatalog CONFIGURED_XMIN_CATALOG = toConfiguredXminCatalog(CATALOG); - private static final Set INITIAL_RECORD_MESSAGES = Sets.newHashSet( + protected static final List INITIAL_RECORD_MESSAGES = Arrays.asList( createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("1.0"), "name", "goku", "power", null)), createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("2.0"), "name", "vegeta", "power", 9000.1)), createRecord(STREAM_NAME, SCHEMA_NAME, map("id", null, "name", "piccolo", "power", null))); - private static final Set NEXT_RECORD_MESSAGES = Sets.newHashSet( + protected static final List NEXT_RECORD_MESSAGES = Arrays.asList( createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("3.0"), "name", "gohan", "power", 222.1))); - private static PostgreSQLContainer PSQL_DB; + protected static PostgreSQLContainer PSQL_DB; - private String dbName; + protected String dbName; @BeforeAll static void init() { @@ -128,7 +133,7 @@ void setup() throws Exception { "CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));"); ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); ctx.fetch( - "INSERT INTO id_and_name (id, name, power) VALUES (2, 'vegeta', 9000.1), (1,'goku', 'Infinity'), ('NaN', 'piccolo', '-Infinity');"); + "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);"); ctx.fetch( @@ -143,11 +148,11 @@ void setup() throws Exception { } } - private static Database getDatabase(final DSLContext dslContext) { + protected static Database getDatabase(final DSLContext dslContext) { return new Database(dslContext); } - private static DSLContext getDslContext(final JsonNode config) { + protected static DSLContext getDslContext(final JsonNode config) { return DSLContextFactory.create( config.get(JdbcUtils.USERNAME_KEY).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText(), @@ -159,7 +164,7 @@ private static DSLContext getDslContext(final JsonNode config) { SQLDialect.POSTGRES); } - private JsonNode getXminConfig(final PostgreSQLContainer psqlDb, final String dbName) { + protected JsonNode getXminConfig(final PostgreSQLContainer psqlDb, final String dbName) { return Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) .put(JdbcUtils.PORT_KEY, psqlDb.getFirstMappedPort()) @@ -169,6 +174,7 @@ private JsonNode getXminConfig(final PostgreSQLContainer psqlDb, final String .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) .put(JdbcUtils.SSL_KEY, false) .put("replication_method", getReplicationMethod()) + .put("sync_checkpoint_records", 1) .build()); } @@ -202,30 +208,88 @@ void testReadSuccess() throws Exception { CONFIGURED_XMIN_CATALOG .withStreams(CONFIGURED_XMIN_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( Collectors.toList())); - final List actualMessages = + final List recordsFromFirstSync = MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, null)); - setEmittedAtToNull(actualMessages); - assertThat(filterRecords(actualMessages)).containsExactlyInAnyOrderElementsOf(INITIAL_RECORD_MESSAGES); + setEmittedAtToNull(recordsFromFirstSync); + assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf(INITIAL_RECORD_MESSAGES); // Extract the state message and assert that it exists. It contains the xmin value, so validating // the actual value isn't useful right now. - final List stateAfterFirstBatch = extractStateMessage(actualMessages); - assertThat(stateAfterFirstBatch.size()).isEqualTo(1); - JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); + final List stateAfterFirstBatch = extractStateMessage(recordsFromFirstSync); + // We should have 3 state messages because we have set state emission frequency after each record in + // the test + assertEquals(3, stateAfterFirstBatch.size()); + + final AirbyteStateMessage firstStateMessage = stateAfterFirstBatch.get(0); + final String stateTypeFromFirstStateMessage = firstStateMessage.getStream().getStreamState().get("state_type").asText(); + final String ctidFromFirstStateMessage = firstStateMessage.getStream().getStreamState().get("ctid").asText(); + final JsonNode incrementalStateFromFirstStateMessage = firstStateMessage.getStream().getStreamState().get("incremental_state"); + + final AirbyteStateMessage secondStateMessage = stateAfterFirstBatch.get(1); + final String stateTypeFromSecondStateMessage = secondStateMessage.getStream().getStreamState().get("state_type").asText(); + final String ctidFromSecondStateMessage = secondStateMessage.getStream().getStreamState().get("ctid").asText(); + final JsonNode incrementalStateFromSecondStateMessage = secondStateMessage.getStream().getStreamState().get("incremental_state"); + + final AirbyteStateMessage thirdStateMessage = stateAfterFirstBatch.get(2); + final String stateTypeFromThirdStateMessage = thirdStateMessage.getStream().getStreamState().get("state_type").asText(); + + // First two state messages should be of ctid type + assertEquals("ctid", stateTypeFromFirstStateMessage); + assertEquals("ctid", stateTypeFromSecondStateMessage); + + // Since the third state message would be the final, it should be of xmin type + assertEquals("xmin", stateTypeFromThirdStateMessage); + + // The ctid value from second state message should be bigger than first state message + assertEquals(1, ctidFromSecondStateMessage.compareTo(ctidFromFirstStateMessage)); + + // The incremental state value from first and second state message should be the same + assertNotNull(incrementalStateFromFirstStateMessage); + assertNotNull(incrementalStateFromSecondStateMessage); + assertEquals(incrementalStateFromFirstStateMessage, incrementalStateFromSecondStateMessage); + + // The third state message should be equal to incremental_state of first two state messages + assertEquals(incrementalStateFromFirstStateMessage, thirdStateMessage.getStream().getStreamState()); // Assert that the last message in the sequence is a state message - assertMessageSequence(actualMessages); + assertMessageSequence(recordsFromFirstSync); + + // Sync should work with a ctid state + final List recordsFromSyncRunningWithACtidState = + MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + Jsons.jsonNode(Collections.singletonList(firstStateMessage)))); + setEmittedAtToNull(recordsFromSyncRunningWithACtidState); + final List expectedDataFromSyncUsingFirstCtidState = new ArrayList<>(2); + final AtomicBoolean skippedFirstRecord = new AtomicBoolean(false); + INITIAL_RECORD_MESSAGES.forEach(c -> { + if (!skippedFirstRecord.get()) { + skippedFirstRecord.set(true); + return; + } + expectedDataFromSyncUsingFirstCtidState.add(c); + }); + assertThat(filterRecords(recordsFromSyncRunningWithACtidState)).containsExactlyElementsOf(expectedDataFromSyncUsingFirstCtidState); + + final List stateAfterSyncWithCtidState = extractStateMessage(recordsFromSyncRunningWithACtidState); + // Since only 2 records should be emitted so 2 state messages are expected + assertEquals(2, stateAfterSyncWithCtidState.size()); + assertEquals(secondStateMessage, stateAfterSyncWithCtidState.get(0)); + assertEquals(thirdStateMessage, stateAfterSyncWithCtidState.get(1)); + + assertMessageSequence(recordsFromSyncRunningWithACtidState); - // Second read, should return no data - final List nextMessages = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, state)); - setEmittedAtToNull(nextMessages); - assertThat(filterRecords(nextMessages)).isEmpty(); + // Read with the final xmin state message should return no data + final List syncWithXminStateType = + MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + Jsons.jsonNode(Collections.singletonList(thirdStateMessage)))); + setEmittedAtToNull(syncWithXminStateType); + assertEquals(0, filterRecords(syncWithXminStateType).size()); // Even though no records were emitted, a state message is still expected - final List stateAfterSecondBatch = extractStateMessage(nextMessages); - assertThat(stateAfterFirstBatch.size()).isEqualTo(1); - state = Jsons.jsonNode(stateAfterSecondBatch); + final List stateAfterXminSync = extractStateMessage(syncWithXminStateType); + assertEquals(1, stateAfterXminSync.size()); + // Since no records were returned so the state should be the same as before + assertEquals(thirdStateMessage, stateAfterXminSync.get(0)); // We add some data and perform a third read. We should verify that (i) a delete is not captured and // (ii) the new record that is inserted into the @@ -239,31 +303,27 @@ void testReadSuccess() throws Exception { }); } - final List lastMessages = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, state)); - setEmittedAtToNull(lastMessages); - assertThat(filterRecords(lastMessages)).containsExactlyInAnyOrderElementsOf(NEXT_RECORD_MESSAGES); - assertMessageSequence(lastMessages); + final List recordsAfterLastSync = + MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + Jsons.jsonNode(Collections.singletonList(stateAfterXminSync.get(0))))); + setEmittedAtToNull(recordsAfterLastSync); + assertThat(filterRecords(recordsAfterLastSync)).containsExactlyElementsOf(NEXT_RECORD_MESSAGES); + assertMessageSequence(recordsAfterLastSync); + final List stateAfterLastSync = extractStateMessage(recordsAfterLastSync); + assertEquals(1, stateAfterLastSync.size()); + + final AirbyteStateMessage finalStateMesssage = stateAfterLastSync.get(0); + final String stateTypeFromFinalStateMessage = finalStateMesssage.getStream().getStreamState().get("state_type").asText(); + assertEquals("xmin", stateTypeFromFinalStateMessage); + assertTrue(finalStateMesssage.getStream().getStreamState().get("xmin_xid_value").asLong() > thirdStateMessage.getStream().getStreamState() + .get("xmin_xid_value").asLong()); + assertTrue(finalStateMesssage.getStream().getStreamState().get("xmin_raw_value").asLong() > thirdStateMessage.getStream().getStreamState() + .get("xmin_raw_value").asLong()); } // Assert that the state message is the last message to be emitted. - private static void assertMessageSequence(final List messages) { - for (int i = 0; i < messages.size(); i++) { - final AirbyteMessage message = messages.get(i); - if (message.getType().equals(Type.STATE)) { - assertThat(i).isEqualTo(messages.size() - 1); - } - } - } - - private static List extractStateMessage(final List messages) { - return messages.stream().filter(r -> r.getType() == Type.STATE).map(AirbyteMessage::getState) - .collect(Collectors.toList()); - } - - private static List filterRecords(final List messages) { - return messages.stream().filter(r -> r.getType() == Type.RECORD) - .collect(Collectors.toList()); + protected static void assertMessageSequence(final List messages) { + assertEquals(Type.STATE, messages.get(messages.size() - 1).getType()); } private static ConfiguredAirbyteCatalog toConfiguredXminCatalog(final AirbyteCatalog catalog) { diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java new file mode 100644 index 000000000000..3d3f5d81baca --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractStateMessage; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.filterRecords; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.setEmittedAtToNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.db.Database; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.PostgreSQLContainer; + +public class XminPostgresWithOldServerSourceTest extends XminPostgresSourceTest { + + @BeforeAll + static void init() { + PSQL_DB = new PostgreSQLContainer<>("postgres:9-alpine"); + PSQL_DB.start(); + } + + @Test + @Override + void testReadSuccess() throws Exception { + // Perform an initial sync with the configured catalog, which is set up to use xmin_replication. + // All of the records in the configured stream should be emitted. + final ConfiguredAirbyteCatalog configuredCatalog = + CONFIGURED_XMIN_CATALOG + .withStreams(CONFIGURED_XMIN_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( + Collectors.toList())); + final List recordsFromFirstSync = + MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, null)); + setEmittedAtToNull(recordsFromFirstSync); + assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf(INITIAL_RECORD_MESSAGES); + + // Extract the state message and assert that it exists. It contains the xmin value, so validating + // the actual value isn't useful right now. + final List stateAfterFirstBatch = extractStateMessage(recordsFromFirstSync); + assertEquals(1, stateAfterFirstBatch.size()); + + final AirbyteStateMessage firstSyncStateMessage = stateAfterFirstBatch.get(0); + final String stateTypeFromFirstStateMessage = firstSyncStateMessage.getStream().getStreamState().get("state_type").asText(); + + // Since the flow reclassified the stream to do initial load using xmin + // It should only contain a single final state of xmin type. + assertEquals("xmin", stateTypeFromFirstStateMessage); + assertFalse(firstSyncStateMessage.getStream().getStreamState().has("ctid")); + assertFalse(firstSyncStateMessage.getStream().getStreamState().has("incremental_state")); + + // Assert that the last message in the sequence is a state message + assertMessageSequence(recordsFromFirstSync); + + // Read with the final xmin state message should return no data + final List syncWithXminStateType = + MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + Jsons.jsonNode(Collections.singletonList(firstSyncStateMessage)))); + setEmittedAtToNull(syncWithXminStateType); + assertEquals(0, filterRecords(syncWithXminStateType).size()); + + // Even though no records were emitted, a state message is still expected + final List stateAfterXminSync = extractStateMessage(syncWithXminStateType); + assertEquals(1, stateAfterXminSync.size()); + // Since no records were returned so the state should be the same as before + assertEquals(firstSyncStateMessage, stateAfterXminSync.get(0)); + + // We add some data and perform a third read. We should verify that (i) a delete is not captured and + // (ii) the new record that is inserted into the + // table is read. + try (final DSLContext dslContext = getDslContext(getXminConfig(PSQL_DB, dbName))) { + final Database database = getDatabase(dslContext); + database.query(ctx -> { + ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); + ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); + return null; + }); + } + + final List recordsAfterLastSync = + MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + Jsons.jsonNode(Collections.singletonList(stateAfterXminSync.get(0))))); + setEmittedAtToNull(recordsAfterLastSync); + assertThat(filterRecords(recordsAfterLastSync)).containsExactlyElementsOf(NEXT_RECORD_MESSAGES); + assertMessageSequence(recordsAfterLastSync); + final List stateAfterLastSync = extractStateMessage(recordsAfterLastSync); + assertEquals(1, stateAfterLastSync.size()); + + final AirbyteStateMessage finalStateMesssage = stateAfterLastSync.get(0); + final String stateTypeFromFinalStateMessage = finalStateMesssage.getStream().getStreamState().get("state_type").asText(); + assertEquals("xmin", stateTypeFromFinalStateMessage); + assertTrue(finalStateMesssage.getStream().getStreamState().get("xmin_xid_value").asLong() > firstSyncStateMessage.getStream().getStreamState() + .get("xmin_xid_value").asLong()); + assertTrue(finalStateMesssage.getStream().getStreamState().get("xmin_raw_value").asLong() > firstSyncStateMessage.getStream().getStreamState() + .get("xmin_raw_value").asLong()); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandlerTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandlerTest.java new file mode 100644 index 000000000000..caa95f561b70 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandlerTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; + +public class PostgresCtidHandlerTest { + + @Test + void testCtidQueryBounds() { + var chunks = PostgresCtidHandler.ctidQueryPlan(new Ctid(0, 0), 380545032192L, 8192L, 50); + var expected = List.of( + Pair.of(Ctid.of(0, 0), Ctid.of(6553600, 0)), + Pair.of(Ctid.of(6553600, 0), Ctid.of(13107200, 0)), + Pair.of(Ctid.of(13107200, 0), Ctid.of(19660800, 0)), + Pair.of(Ctid.of(19660800, 0), Ctid.of(26214400, 0)), + Pair.of(Ctid.of(26214400, 0), Ctid.of(32768000, 0)), + Pair.of(Ctid.of(32768000, 0), Ctid.of(39321600, 0)), + Pair.of(Ctid.of(39321600, 0), Ctid.of(45875200, 0)), + Pair.of(Ctid.of(45875200, 0), null)); + assertEquals(expected, chunks); + + chunks = PostgresCtidHandler.ctidQueryPlan(new Ctid("(23000000,123)"), 380545032192L, 8192L, 45); + expected = List.of( + Pair.of(Ctid.of("(23000000,123)"), Ctid.of(28898240, 0)), + Pair.of(Ctid.of(28898240, 0), Ctid.of("(34796480,0)")), + Pair.of(Ctid.of("(34796480,0)"), Ctid.of(40694720, 0)), + Pair.of(Ctid.of(40694720, 0), null)); + assertEquals(expected, chunks); + + chunks = PostgresCtidHandler.ctidQueryPlan(Ctid.of(0, 0), 380545L, 8192L, 45); + expected = List.of( + Pair.of(Ctid.of(0, 0), null)); + assertEquals(expected, chunks); + + chunks = PostgresCtidHandler.ctidQueryPlan(Ctid.of(9876, 5432), 380545L, 8192L, 45); + expected = List.of( + Pair.of(Ctid.of(9876, 5432), null)); + assertEquals(expected, chunks); + + chunks = PostgresCtidHandler.ctidQueryPlan(Ctid.of(0, 0), 4096L, 8192L, 45); + expected = List.of( + Pair.of(Ctid.of(0, 0), null)); + assertEquals(expected, chunks); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java new file mode 100644 index 000000000000..5bcb8a4b30f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.cursor_based; + +import static io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.categoriseStreams; +import static io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.reclassifyCategorisedCtidStreams; +import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.generateStateMessage; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; +import io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.CursorBasedStreams; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.relationaldb.state.StreamStateManager; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class CursorBasedCtidUtilsTest { + + @Test + public void emptyStateTest() { + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(STREAM_1, STREAM_2)); + final StreamStateManager streamStateManager = new StreamStateManager(Collections.emptyList(), configuredCatalog); + final StreamsCategorised streamsCategorised = categoriseStreams(streamStateManager, configuredCatalog); + + assertEquals(2, streamsCategorised.ctidStreams().streamsForCtidSync().size()); + assertEquals(0, streamsCategorised.remainingStreams().streamsForCursorBasedSync().size()); + assertTrue(streamsCategorised.remainingStreams().streamsForCursorBasedSync().isEmpty()); + assertThat(streamsCategorised.ctidStreams().streamsForCtidSync()).containsExactlyInAnyOrder(STREAM_1, STREAM_2); + } + + @Test + public void correctOneCtidOneCursorBasedTest() { + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(STREAM_1, STREAM_2)); + final JsonNode stream1CtidStatus = Jsons.jsonNode(new CtidStatus() + .withStateType(StateType.CTID) + .withCtid("(0,0)") + .withRelationFilenode(456L)); + + final JsonNode stream2CursorBased = Jsons.jsonNode(new CursorBasedStatus() + .withStateType(StateType.CURSOR_BASED) + .withStreamName(STREAM_2.getStream().getName()) + .withStreamNamespace(STREAM_2.getStream().getNamespace()) + .withCursorField(List.of("COL_ID")) + .withCursor("1") + .withCursorRecordCount(1L)); + + final JsonNode stream2CursorBasedJson = Jsons.jsonNode(stream2CursorBased); + final AirbyteStateMessage stream1CtidState = generateStateMessage(STREAM_1.getStream().getName(), STREAM_1.getStream().getNamespace(), + stream1CtidStatus); + final AirbyteStateMessage stream2StandardState = generateStateMessage(STREAM_2.getStream().getName(), STREAM_2.getStream().getNamespace(), + stream2CursorBasedJson); + final StreamStateManager streamStateManager = new StreamStateManager(List.of(stream1CtidState, stream2StandardState), configuredCatalog); + final StreamsCategorised streamsCategorised = categoriseStreams(streamStateManager, configuredCatalog); + + assertEquals(streamsCategorised.ctidStreams().streamsForCtidSync().size(), 1); + assertEquals(streamsCategorised.remainingStreams().streamsForCursorBasedSync().size(), 1); + assertEquals(streamsCategorised.ctidStreams().streamsForCtidSync().stream().findFirst().get(), STREAM_1); + assertEquals(streamsCategorised.remainingStreams().streamsForCursorBasedSync().stream().findFirst().get(), STREAM_2); + } + + @Test + public void correctEmptyCtidTest() { + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(STREAM_1, STREAM_2)); + final JsonNode standardStatus = Jsons.jsonNode(new CursorBasedStatus() + .withStateType(StateType.CURSOR_BASED) + .withStreamName(STREAM_2.getStream().getName()) + .withStreamNamespace(STREAM_2.getStream().getNamespace()) + .withCursorField(List.of("COL_ID")) + .withCursor("1") + .withCursorRecordCount(1L)); + + final AirbyteStateMessage stream1CtidState = generateStateMessage(STREAM_1.getStream().getName(), STREAM_1.getStream().getNamespace(), + standardStatus); + final AirbyteStateMessage stream2StandardState = generateStateMessage(STREAM_2.getStream().getName(), STREAM_2.getStream().getNamespace(), + standardStatus); + final StreamStateManager streamStateManager = new StreamStateManager(List.of(stream1CtidState, stream2StandardState), configuredCatalog); + final StreamsCategorised streamsCategorised = categoriseStreams(streamStateManager, configuredCatalog); + + assertEquals(streamsCategorised.ctidStreams().streamsForCtidSync().size(), 0); + assertEquals(streamsCategorised.remainingStreams().streamsForCursorBasedSync().size(), 2); + assertThat(streamsCategorised.remainingStreams().streamsForCursorBasedSync()).containsExactlyInAnyOrder(STREAM_1, STREAM_2); + + } + + @Test + public void reclassifyCategorisedCtidStreamTest() { + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(STREAM_1, STREAM_2)); + final StreamStateManager streamStateManager = new StreamStateManager(Collections.emptyList(), configuredCatalog); + final StreamsCategorised streamsCategorised = categoriseStreams(streamStateManager, configuredCatalog); + + List reclassify = + Collections.singletonList(new AirbyteStreamNameNamespacePair(STREAM_1.getStream().getName(), STREAM_1.getStream().getNamespace())); + reclassifyCategorisedCtidStreams(streamsCategorised, reclassify); + assertEquals(1, streamsCategorised.ctidStreams().streamsForCtidSync().size()); + assertEquals(1, streamsCategorised.remainingStreams().streamsForCursorBasedSync().size()); + assertFalse(streamsCategorised.remainingStreams().streamsForCursorBasedSync().isEmpty()); + assertThat(streamsCategorised.ctidStreams().streamsForCtidSync()).containsExactlyInAnyOrder(STREAM_2); + assertThat(streamsCategorised.remainingStreams().streamsForCursorBasedSync()).containsExactlyInAnyOrder(STREAM_1); + } + + @Test + public void fullRefreshStreamCategorisationTest() { + final ConfiguredAirbyteStream STREAM_3_FULL_REFRESH = CatalogHelpers.toDefaultConfiguredStream(CatalogHelpers.createAirbyteStream( + "STREAM_3", + "SCHEMA", + Field.of("COL_ID", JsonSchemaType.INTEGER), + Field.of("COL_MAKE_ID", JsonSchemaType.INTEGER), + Field.of("COL_MODEL", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("COL_ID")))); + + final ConfiguredAirbyteCatalog configuredCatalog = + new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(STREAM_1, STREAM_2, STREAM_3_FULL_REFRESH)); + final JsonNode stream1CtidStatus = Jsons.jsonNode(new CtidStatus() + .withStateType(StateType.CTID) + .withCtid("(0,0)") + .withRelationFilenode(456L)); + + final JsonNode stream2CursorBased = Jsons.jsonNode(new CursorBasedStatus() + .withStateType(StateType.CURSOR_BASED) + .withStreamName(STREAM_2.getStream().getName()) + .withStreamNamespace(STREAM_2.getStream().getNamespace()) + .withCursorField(List.of("COL_ID")) + .withCursor("1") + .withCursorRecordCount(1L)); + + final JsonNode stream2CursorBasedJson = Jsons.jsonNode(stream2CursorBased); + final AirbyteStateMessage stream1CtidState = generateStateMessage(STREAM_1.getStream().getName(), STREAM_1.getStream().getNamespace(), + stream1CtidStatus); + final AirbyteStateMessage stream2StandardState = generateStateMessage(STREAM_2.getStream().getName(), STREAM_2.getStream().getNamespace(), + stream2CursorBasedJson); + final StreamStateManager streamStateManager = new StreamStateManager(List.of(stream1CtidState, stream2StandardState), configuredCatalog); + final StreamsCategorised streamsCategorised = categoriseStreams(streamStateManager, configuredCatalog); + + assertEquals(streamsCategorised.ctidStreams().streamsForCtidSync().size(), 1); + assertEquals(streamsCategorised.remainingStreams().streamsForCursorBasedSync().size(), 1); + assertEquals(streamsCategorised.ctidStreams().streamsForCtidSync().stream().findFirst().get(), STREAM_1); + assertTrue(streamsCategorised.remainingStreams().streamsForCursorBasedSync().contains(STREAM_2)); + assertFalse(streamsCategorised.remainingStreams().streamsForCursorBasedSync().contains(STREAM_3_FULL_REFRESH)); + } + + private static final ConfiguredAirbyteStream STREAM_1 = CatalogHelpers.toDefaultConfiguredStream(CatalogHelpers.createAirbyteStream( + "STREAM_1", + "SCHEMA", + Field.of("COL_ID", JsonSchemaType.INTEGER), + Field.of("COL_MAKE_ID", JsonSchemaType.INTEGER), + Field.of("COL_MODEL", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("COL_ID")))).withSyncMode(SyncMode.INCREMENTAL); + + private static final ConfiguredAirbyteStream STREAM_2 = CatalogHelpers.toDefaultConfiguredStream(CatalogHelpers.createAirbyteStream( + "STREAM_2", + "SCHEMA", + Field.of("COL_ID", JsonSchemaType.INTEGER), + Field.of("COL_MAKE_ID", JsonSchemaType.INTEGER), + Field.of("COL_MODEL", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("COL_ID")))).withSyncMode(SyncMode.INCREMENTAL); + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/utils/PostgresUnitTestsUtil.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/utils/PostgresUnitTestsUtil.java index 88d77cc1bd31..189705298cbd 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/utils/PostgresUnitTestsUtil.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/utils/PostgresUnitTestsUtil.java @@ -4,12 +4,19 @@ package io.airbyte.integrations.source.postgres.utils; +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class PostgresUnitTestsUtil { @@ -47,4 +54,38 @@ public static Map map(final Object... entries) { }; } + public static AirbyteStateMessage generateStateMessage(final String streamName, final String namespace, final JsonNode stateData) { + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor() + .withName(streamName) + .withNamespace(namespace)) + .withStreamState(stateData)); + } + + public static List extractStateMessage(final List messages) { + return messages.stream().filter(r -> r.getType() == Type.STATE).map(AirbyteMessage::getState) + .collect(Collectors.toList()); + } + + public static List extractStateMessage(final List messages, final String streamName) { + return messages.stream().filter(r -> r.getType() == Type.STATE && + r.getState().getStream().getStreamDescriptor().getName().equals(streamName)).map(AirbyteMessage::getState) + .collect(Collectors.toList()); + } + + public static List filterRecords(final List messages) { + return messages.stream().filter(r -> r.getType() == Type.RECORD) + .collect(Collectors.toList()); + } + + public static List extractSpecificFieldFromCombinedMessages(final List messages, + final String streamName, + final String field) { + return extractStateMessage(messages).stream() + .filter(s -> s.getStream().getStreamDescriptor().getName().equals(streamName)) + .map(s -> s.getStream().getStreamState().get(field) != null ? s.getStream().getStreamState().get(field).asText() : "").toList(); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandlerTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandlerTest.java new file mode 100644 index 000000000000..25c2aad834a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandlerTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.xmin; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.internal.models.XminStatus; +import org.junit.jupiter.api.Test; + +public class PostgresXminHandlerTest { + + @Test + void testWraparound() { + final XminStatus initialStatus = + new XminStatus() + .withNumWraparound(0L) + .withXminRawValue(5555L) + .withXminRawValue(5555L); + final JsonNode initialStatusAsJson = Jsons.jsonNode(initialStatus); + + final XminStatus noWrapAroundStatus = + new XminStatus() + .withNumWraparound(0L) + .withXminRawValue(5588L) + .withXminRawValue(5588L); + assertFalse(PostgresXminHandler.isSingleWraparound(initialStatus, noWrapAroundStatus)); + assertFalse(PostgresXminHandler.shouldPerformFullSync(noWrapAroundStatus, initialStatusAsJson)); + + final XminStatus singleWrapAroundStatus = + new XminStatus() + .withNumWraparound(1L) + .withXminRawValue(5588L) + .withXminRawValue(4294972884L); + + assertTrue(PostgresXminHandler.isSingleWraparound(initialStatus, singleWrapAroundStatus)); + assertFalse(PostgresXminHandler.shouldPerformFullSync(singleWrapAroundStatus, initialStatusAsJson)); + + final XminStatus doubleWrapAroundStatus = + new XminStatus() + .withNumWraparound(2L) + .withXminRawValue(5588L) + .withXminRawValue(8589940180L); + + assertFalse(PostgresXminHandler.isSingleWraparound(initialStatus, doubleWrapAroundStatus)); + assertTrue(PostgresXminHandler.shouldPerformFullSync(doubleWrapAroundStatus, initialStatusAsJson)); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java new file mode 100644 index 000000000000..8923a7a932f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.xmin; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.postgres.internal.models.XminStatus; +import io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.XminStreams; +import io.airbyte.integrations.source.relationaldb.state.StreamStateManager; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class XminCtidUtilsTest { + + private static final ConfiguredAirbyteStream MODELS_STREAM = CatalogHelpers.toDefaultConfiguredStream(CatalogHelpers.createAirbyteStream( + "MODELS_STREAM_NAME", + "MODELS_SCHEMA", + Field.of("COL_ID", JsonSchemaType.INTEGER), + Field.of("COL_MAKE_ID", JsonSchemaType.INTEGER), + Field.of("COL_MODEL", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("COL_ID")))) + .withSyncMode(SyncMode.INCREMENTAL); + + private static final ConfiguredAirbyteStream MODELS_STREAM_2 = CatalogHelpers.toDefaultConfiguredStream(CatalogHelpers.createAirbyteStream( + "MODELS_STREAM_NAME_2", + "MODELS_SCHEMA", + Field.of("COL_ID", JsonSchemaType.INTEGER), + Field.of("COL_MAKE_ID", JsonSchemaType.INTEGER), + Field.of("COL_MODEL", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("COL_ID")))) + .withSyncMode(SyncMode.INCREMENTAL); + + @Test + public void emptyStateTest() { + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(MODELS_STREAM, MODELS_STREAM_2)); + final XminStatus xminStatus = new XminStatus().withStateType(StateType.XMIN).withVersion(2L).withXminXidValue(9L).withXminRawValue(9L) + .withNumWraparound(1L); + final StreamStateManager streamStateManager = new StreamStateManager(Collections.emptyList(), configuredCatalog); + final StreamsCategorised streamsCategorised = XminCtidUtils.categoriseStreams(streamStateManager, configuredCatalog, xminStatus); + + assertTrue(streamsCategorised.remainingStreams().streamsForXminSync().isEmpty()); + assertTrue(streamsCategorised.remainingStreams().statesFromXminSync().isEmpty()); + + assertEquals(2, streamsCategorised.ctidStreams().streamsForCtidSync().size()); + assertThat(streamsCategorised.ctidStreams().streamsForCtidSync()).containsExactlyInAnyOrder(MODELS_STREAM, MODELS_STREAM_2); + assertTrue(streamsCategorised.ctidStreams().statesFromCtidSync().isEmpty()); + } + + @Test + public void correctCategorisationTest() { + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(MODELS_STREAM, MODELS_STREAM_2)); + final XminStatus xminStatus = new XminStatus().withStateType(StateType.XMIN).withVersion(2L).withXminXidValue(9L).withXminRawValue(9L) + .withNumWraparound(1L); + final JsonNode xminStatusAsJson = Jsons.jsonNode(xminStatus); + final AirbyteStateMessage xminState = generateStateMessage(xminStatusAsJson, + new StreamDescriptor().withName(MODELS_STREAM.getStream().getName()).withNamespace(MODELS_STREAM.getStream().getNamespace())); + + final CtidStatus ctidStatus = new CtidStatus().withStateType(StateType.CTID).withVersion(2L).withCtid("123").withRelationFilenode(456L) + .withIncrementalState(xminStatusAsJson); + final JsonNode ctidStatusAsJson = Jsons.jsonNode(ctidStatus); + final AirbyteStateMessage ctidState = generateStateMessage(ctidStatusAsJson, + new StreamDescriptor().withName(MODELS_STREAM_2.getStream().getName()).withNamespace(MODELS_STREAM_2.getStream().getNamespace())); + + final StreamStateManager streamStateManager = new StreamStateManager(Arrays.asList(xminState, ctidState), configuredCatalog); + final StreamsCategorised streamsCategorised = XminCtidUtils.categoriseStreams(streamStateManager, configuredCatalog, xminStatus); + + assertEquals(1, streamsCategorised.remainingStreams().streamsForXminSync().size()); + assertEquals(MODELS_STREAM, streamsCategorised.remainingStreams().streamsForXminSync().get(0)); + assertEquals(1, streamsCategorised.remainingStreams().statesFromXminSync().size()); + assertEquals(xminState, streamsCategorised.remainingStreams().statesFromXminSync().get(0)); + + assertEquals(1, streamsCategorised.ctidStreams().streamsForCtidSync().size()); + assertEquals(MODELS_STREAM_2, streamsCategorised.ctidStreams().streamsForCtidSync().get(0)); + assertEquals(1, streamsCategorised.ctidStreams().statesFromCtidSync().size()); + assertEquals(ctidState, streamsCategorised.ctidStreams().statesFromCtidSync().get(0)); + } + + @Test + public void fullRefreshStreamCategorisationTest() { + final ConfiguredAirbyteStream MODELS_STREAM_3_FULL_REFRESH = CatalogHelpers.toDefaultConfiguredStream(CatalogHelpers.createAirbyteStream( + "MODELS_STREAM_NAME_3", + "MODELS_SCHEMA", + Field.of("COL_ID", JsonSchemaType.INTEGER), + Field.of("COL_MAKE_ID", JsonSchemaType.INTEGER), + Field.of("COL_MODEL", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("COL_ID")))); + + final ConfiguredAirbyteCatalog configuredCatalog = + new ConfiguredAirbyteCatalog().withStreams(Arrays.asList(MODELS_STREAM, MODELS_STREAM_2, MODELS_STREAM_3_FULL_REFRESH)); + final XminStatus xminStatus = new XminStatus().withStateType(StateType.XMIN).withVersion(2L).withXminXidValue(9L).withXminRawValue(9L) + .withNumWraparound(1L); + final JsonNode xminStatusAsJson = Jsons.jsonNode(xminStatus); + final AirbyteStateMessage xminState = generateStateMessage(xminStatusAsJson, + new StreamDescriptor().withName(MODELS_STREAM.getStream().getName()).withNamespace(MODELS_STREAM.getStream().getNamespace())); + + final CtidStatus ctidStatus = new CtidStatus().withStateType(StateType.CTID).withVersion(2L).withCtid("123").withRelationFilenode(456L) + .withIncrementalState(xminStatusAsJson); + final JsonNode ctidStatusAsJson = Jsons.jsonNode(ctidStatus); + final AirbyteStateMessage ctidState = generateStateMessage(ctidStatusAsJson, + new StreamDescriptor().withName(MODELS_STREAM_2.getStream().getName()).withNamespace(MODELS_STREAM_2.getStream().getNamespace())); + + final StreamStateManager streamStateManager = new StreamStateManager(Arrays.asList(xminState, ctidState), configuredCatalog); + final StreamsCategorised streamsCategorised = XminCtidUtils.categoriseStreams(streamStateManager, configuredCatalog, xminStatus); + + assertEquals(1, streamsCategorised.remainingStreams().streamsForXminSync().size()); + assertEquals(MODELS_STREAM, streamsCategorised.remainingStreams().streamsForXminSync().get(0)); + assertEquals(1, streamsCategorised.remainingStreams().statesFromXminSync().size()); + assertEquals(xminState, streamsCategorised.remainingStreams().statesFromXminSync().get(0)); + + assertEquals(1, streamsCategorised.ctidStreams().streamsForCtidSync().size()); + assertEquals(MODELS_STREAM_2, streamsCategorised.ctidStreams().streamsForCtidSync().get(0)); + assertEquals(1, streamsCategorised.ctidStreams().statesFromCtidSync().size()); + assertEquals(ctidState, streamsCategorised.ctidStreams().statesFromCtidSync().get(0)); + + } + + private AirbyteStateMessage generateStateMessage(final JsonNode stateData, final StreamDescriptor streamDescriptor) { + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(streamDescriptor).withStreamState(stateData)); + } + +} diff --git a/airbyte-integrations/connectors/source-posthog/Dockerfile b/airbyte-integrations/connectors/source-posthog/Dockerfile index aa97a7a550de..06c2c48b86f2 100644 --- a/airbyte-integrations/connectors/source-posthog/Dockerfile +++ b/airbyte-integrations/connectors/source-posthog/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-posthog diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py b/airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/future_state.json b/airbyte-integrations/connectors/source-posthog/integration_tests/future_state.json index 79e5bae57f3d..5f2f4180d0ab 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/future_state.json @@ -4,4 +4,4 @@ "timestamp": "2221-12-10T10:21:35.003000+00:00" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/future_state_old.json b/airbyte-integrations/connectors/source-posthog/integration_tests/future_state_old.json index a209b41e5311..4ff6f043fb10 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/future_state_old.json +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/future_state_old.json @@ -1,5 +1,5 @@ { "events": { - "timestamp": "2221-12-10T10:21:35.003000+00:00" + "timestamp": "2221-12-10T10:21:35.003000+00:00" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-posthog/metadata.yaml b/airbyte-integrations/connectors/source-posthog/metadata.yaml index faea08497d4f..3d2fcba2975d 100644 --- a/airbyte-integrations/connectors/source-posthog/metadata.yaml +++ b/airbyte-integrations/connectors/source-posthog/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: af6d50ee-dddf-4126-a8ee-7faee990774f - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.13 dockerRepository: airbyte/source-posthog githubIssueLabel: source-posthog icon: posthog.svg @@ -22,4 +22,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-posthog/requirements.txt b/airbyte-integrations/connectors/source-posthog/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-posthog/requirements.txt +++ b/airbyte-integrations/connectors/source-posthog/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-posthog/sample_files/state.json b/airbyte-integrations/connectors/source-posthog/sample_files/state.json index f5d946108473..6829a71911bb 100644 --- a/airbyte-integrations/connectors/source-posthog/sample_files/state.json +++ b/airbyte-integrations/connectors/source-posthog/sample_files/state.json @@ -4,4 +4,4 @@ "timestamp": "2021-12-10T10:21:35.003000+00:00" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-posthog/setup.py b/airbyte-integrations/connectors/source-posthog/setup.py index 734a82d203a7..75fb93d9ac2c 100644 --- a/airbyte-integrations/connectors/source-posthog/setup.py +++ b/airbyte-integrations/connectors/source-posthog/setup.py @@ -5,11 +5,12 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk"] +MAIN_REQUIREMENTS = ["airbyte-cdk>=0.44.1"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py b/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py index efdacd2cff39..42e8df028a3f 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from .source import SourcePosthog __all__ = ["SourcePosthog"] diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/components.py b/airbyte-integrations/connectors/source-posthog/source_posthog/components.py index bc38ed309e76..326210e7721a 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/components.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/components.py @@ -5,14 +5,18 @@ from dataclasses import dataclass from typing import Any, Iterable, Mapping, MutableMapping, Optional -from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.incremental import Cursor from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.stream_slicers import CartesianProductStreamSlicer -from airbyte_cdk.sources.declarative.types import Record, StreamSlice +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState @dataclass class EventsSimpleRetriever(SimpleRetriever): + def __post_init__(self, parameters: Mapping[str, Any]): + super().__post_init__(parameters) + self.cursor = self.stream_slicer if isinstance(self.stream_slicer, Cursor) else None + def request_params( self, stream_state: StreamSlice, @@ -47,11 +51,12 @@ def request_params( self.requester.get_request_params, self.paginator.get_request_params, self.stream_slicer.get_request_params, + self.requester.get_authenticator().get_request_body_json, ) @dataclass -class EventsCartesianProductStreamSlicer(CartesianProductStreamSlicer): +class EventsCartesianProductStreamSlicer(Cursor, CartesianProductStreamSlicer): """Connector requires support of nested state - each project should have own timestamp value, like: { "project_id1": { @@ -76,21 +81,18 @@ def __post_init__(self, parameters: Mapping[str, Any]): def get_stream_state(self) -> Mapping[str, Any]: return self._cursor or {} - def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): - - if not last_record: - # this is actually initial stream state from CLI - self._cursor = stream_slice - return + def set_initial_state(self, stream_state: StreamState) -> None: + self._cursor = stream_state + def close_slice(self, stream_slice: StreamSlice, most_recent_record: Optional[Record]) -> None: project_id = str(stream_slice.get("project_id", "")) - if project_id: + if project_id and most_recent_record: current_cursor_value = self._cursor.get(project_id, {}).get("timestamp", "") - new_cursor_value = last_record.get("timestamp", "") + new_cursor_value = most_recent_record.get("timestamp", "") self._cursor[project_id] = {"timestamp": max(current_cursor_value, new_cursor_value)} - def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: + def stream_slices(self) -> Iterable[StreamSlice]: """Since each project has its own state, then we need to have a separate datetime slices for each project """ @@ -100,16 +102,17 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> project_slicer, datetime_slicer = self.stream_slicers # support of old style state: it contains only a single 'timestamp' field - old_style_state = stream_state if "timestamp" in stream_state else {} + old_style_state = self._cursor if "timestamp" in self._cursor else {} - for project_slice in project_slicer.stream_slices(sync_mode, stream_state): + for project_slice in project_slicer.stream_slices(): project_id = str(project_slice.get("project_id", "")) # use old_style_state if state does not contain states for each project - project_state = stream_state.get(project_id, {}) or old_style_state + project_state = self._cursor.get(project_id, {}) or old_style_state # Each project should have own datetime slices depends on its state - project_datetime_slices = datetime_slicer.stream_slices(sync_mode, project_state) + datetime_slicer.set_initial_state(project_state) + project_datetime_slices = datetime_slicer.stream_slices() # fix date ranges: start_time of next slice must be equal to end_time of previous slice if project_datetime_slices and project_state: @@ -124,3 +127,23 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> slices.extend(project_datetime_slices) return slices + + def should_be_synced(self, record: Record) -> bool: + """ + As of 2023-06-28, the expectation is that this method will only be used for semi-incremental and data feed and therefore the + implementation is irrelevant for posthog + """ + return True + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + """ + Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice + """ + first_cursor_value = first.get("timestamp") + second_cursor_value = second.get("timestamp") + if first_cursor_value and second_cursor_value: + return first_cursor_value >= second_cursor_value + elif first_cursor_value: + return True + else: + return False diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml b/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml index f25382675d17..96c9d9ac178f 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml @@ -129,7 +129,13 @@ definitions: schema_loader: $ref: "#/definitions/schema_loader" retriever: - class_name: source_posthog.components.EventsSimpleRetriever + transformations: + - class_name: "source_posthog.components.EventsSimpleRetriever" + fields: + - path: ["name"] + value: "{{ parameters['name'] }}" + - path: ["primary_key"] + value: "{{ parameters['primary_key'] }}" name: "{{ parameters['name'] }}" primary_key: "{{ parameters['primary_key'] }}" record_selector: diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json index e8802e74354d..21fe09cffdc5 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json @@ -38,7 +38,6 @@ "type": ["boolean", "null"] }, - "last_refresh": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py b/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py index ae7b1b8ba333..a7c40deb21fe 100644 --- a/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-posthog/unit_tests/test_components.py @@ -3,7 +3,6 @@ # import pytest as pytest -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.incremental.datetime_based_cursor import DatetimeBasedCursor from airbyte_cdk.sources.declarative.partition_routers.list_partition_router import ListPartitionRouter @@ -71,10 +70,10 @@ def test_update_cursor(test_name, initial_state, stream_slice, last_record, expected_state): slicer = EventsCartesianProductStreamSlicer(stream_slicers=stream_slicers, parameters={}) # set initial state - slicer.update_cursor(initial_state, None) + slicer.set_initial_state(initial_state) if last_record: - slicer.update_cursor(stream_slice, last_record) + slicer.close_slice(stream_slice, last_record) updated_state = slicer.get_stream_state() assert updated_state == expected_state @@ -122,5 +121,6 @@ def test_update_cursor(test_name, initial_state, stream_slice, last_record, expe ) def test_stream_slices(test_name, stream_state, expected_stream_slices): slicer = EventsCartesianProductStreamSlicer(stream_slicers=stream_slicers, parameters={}) - stream_slices = slicer.stream_slices(SyncMode.incremental, stream_state=stream_state) + slicer.set_initial_state(stream_state) + stream_slices = slicer.stream_slices() assert list(stream_slices) == expected_stream_slices diff --git a/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml b/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml index 3780d1c56d9e..bac99f12668c 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-postmarkapp/requirements.txt b/airbyte-integrations/connectors/source-postmarkapp/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/requirements.txt +++ b/airbyte-integrations/connectors/source-postmarkapp/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-postmarkapp/setup.py b/airbyte-integrations/connectors/source-postmarkapp/setup.py index 8487f0b31c62..08a989882942 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/setup.py +++ b/airbyte-integrations/connectors/source-postmarkapp/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-prestashop/Dockerfile b/airbyte-integrations/connectors/source-prestashop/Dockerfile index 31277e20ff21..3438cdd4b1bb 100644 --- a/airbyte-integrations/connectors/source-prestashop/Dockerfile +++ b/airbyte-integrations/connectors/source-prestashop/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-prestashop diff --git a/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml b/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml index 033e39171769..eddbf16f18e8 100644 --- a/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml @@ -7,8 +7,6 @@ acceptance_tests: - spec_path: "source_prestashop/spec.yaml" # unfortunately timeout plugin takes into account setup code as well (docker setup) timeout_seconds: 300 - backward_compatibility_tests_config: - disable_for_version: "0.2.0" connection: tests: - config_path: "secrets/config.json" @@ -18,10 +16,15 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.1" basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "messages" + bypass_reason: "Can not populate" expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no diff --git a/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py index 25ce2fdba1ae..51f9b12de27d 100644 --- a/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-prestashop/integration_tests/acceptance.py @@ -2,48 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json -import os -import subprocess -import sys -import time -from pathlib import Path - import pytest -HERE = Path(__file__).parent.absolute() pytest_plugins = ("connector_acceptance_test.plugin",) -@pytest.fixture(name="create_config", scope="session") -def create_config_fixture(): - secrets_path = HERE.parent / "secrets" - secrets_path.mkdir(exist_ok=True) - config_filename = str(secrets_path / "config.json") - - config = { - "url": "http://localhost:8080", - "_allow_http": True, - "access_key": "59662QEPFNCJ3KFL3VCT5VNQ4NHVUF4Y", - "start_date": "2021-05-25", - } - - with open(config_filename, "w+") as fp: - json.dump(obj=config, fp=fp) - - @pytest.fixture(scope="session", autouse=True) -def connector_setup(create_config): +def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" - filename = str(HERE / "docker-compose.yaml") - subprocess.check_call([sys.executable, "-m", "pip", "install", "docker-compose"], stdout=subprocess.DEVNULL) - - env = None - AIRBYTE_SAT_CONNECTOR_DIR = os.environ.get("AIRBYTE_SAT_CONNECTOR_DIR") - if AIRBYTE_SAT_CONNECTOR_DIR: - env = {**os.environ, "INTEGRATION_TESTS_DIR": os.path.join(AIRBYTE_SAT_CONNECTOR_DIR, "integration_tests")} - - subprocess.check_call(["docker-compose", "-f", filename, "up", "-d"], env=env) - time.sleep(5) yield - subprocess.check_call(["docker-compose", "-f", filename, "down", "-v"]) diff --git a/airbyte-integrations/connectors/source-prestashop/integration_tests/docker-compose.yaml b/airbyte-integrations/connectors/source-prestashop/integration_tests/docker-compose.yaml deleted file mode 100644 index 794516e942df..000000000000 --- a/airbyte-integrations/connectors/source-prestashop/integration_tests/docker-compose.yaml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.3" -services: - db: - image: mariadb - ports: - - "3307:3306" - environment: - - MYSQL_ROOT_PASSWORD=admin - volumes: - - ${INTEGRATION_TESTS_DIR:-.}:/docker-entrypoint-initdb.d/ - networks: - - prestashop-net - - presta_shop: - image: eugenekulak/airbyte-prestashop:0.1.0 - ports: - - "8080:80" - environment: - - DB_SERVER=db - networks: - - prestashop-net - -networks: - prestashop-net: {} diff --git a/airbyte-integrations/connectors/source-prestashop/integration_tests/dump.sql.gz b/airbyte-integrations/connectors/source-prestashop/integration_tests/dump.sql.gz deleted file mode 100644 index 5a0fbd79b7a2..000000000000 Binary files a/airbyte-integrations/connectors/source-prestashop/integration_tests/dump.sql.gz and /dev/null differ diff --git a/airbyte-integrations/connectors/source-prestashop/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-prestashop/integration_tests/expected_records.jsonl index bf44ef8554cf..f862a1bba127 100644 --- a/airbyte-integrations/connectors/source-prestashop/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-prestashop/integration_tests/expected_records.jsonl @@ -1,56 +1,55 @@ -{"stream": "addresses", "data": {"id": 1, "id_customer": "1", "id_manufacturer": "0", "id_supplier": "0", "id_warehouse": "0", "id_country": "21", "id_state": "0", "alias": "Anonymous", "company": "Anonymous", "lastname": "Anonymous", "firstname": "Anonymous", "vat_number": "0000", "address1": "Anonymous", "address2": "", "postcode": "00000", "city": "Anonymous", "other": "", "phone": "0000000000", "phone_mobile": "0000000000", "dni": "0000", "deleted": "0", "date_add": "2021-07-23 23:18:07", "date_upd": "2021-07-23 23:18:07"}, "emitted_at": 1667903102665} -{"stream": "carriers", "data": {"id": 1, "deleted": "0", "is_module": "0", "id_tax_rules_group": "1", "id_reference": "1", "name": "Airbyte", "active": "1", "is_free": "1", "url": "", "shipping_handling": "0", "shipping_external": "0", "range_behavior": "0", "shipping_method": 1, "max_width": "0", "max_height": "0", "max_depth": "0", "max_weight": "0.000000", "grade": "0", "external_module_name": "", "need_range": "0", "position": "0", "delay": "Pick up in-store"}, "emitted_at": 1667903102711} -{"stream": "cart_rules", "data": {"id": 1, "id_customer": "0", "date_from": "2021-07-24 20:00:00", "date_to": "2021-08-24 20:00:00", "description": "Some test descriptions for cart rule", "quantity": "4", "quantity_per_user": "10", "priority": "1", "partial_use": "1", "code": "E4639C5C", "minimum_amount": "100.000000", "minimum_amount_tax": "0", "minimum_amount_currency": "1", "minimum_amount_shipping": "0", "country_restriction": "0", "carrier_restriction": "0", "group_restriction": "0", "cart_rule_restriction": "0", "product_restriction": "1", "shop_restriction": "0", "free_shipping": "1", "reduction_percent": "15.00", "reduction_amount": "0.000000", "reduction_tax": "0", "reduction_currency": "1", "reduction_product": "-1", "reduction_exclude_special": "1", "gift_product": "13", "gift_product_attribute": "0", "highlight": "1", "active": "1", "date_add": "2021-07-24 20:31:17", "date_upd": "2021-07-24 20:31:17", "name": "This is cart rule"}, "emitted_at": 1667903102754} -{"stream": "carts", "data": {"id": 1, "id_address_delivery": "5", "id_address_invoice": "5", "id_currency": "1", "id_customer": "2", "id_guest": "1", "id_lang": "1", "id_shop_group": "1", "id_shop": "1", "id_carrier": "2", "recyclable": "0", "gift": "0", "gift_message": "", "mobile_theme": "0", "delivery_option": "{\"3\":\"2,\"}", "secure_key": "b44a6d9efd7a0076a0fbce6b15eaf3b1", "allow_seperated_package": "0", "date_add": "2021-07-23 23:18:42", "date_upd": "2021-07-23 23:18:42", "associations": {"cart_rows": [{"id_product": "1", "id_product_attribute": "1", "id_address_delivery": "3", "id_customization": "0", "quantity": "1"}, {"id_product": "2", "id_product_attribute": "9", "id_address_delivery": "3", "id_customization": "0", "quantity": "1"}]}}, "emitted_at": 1667903102798} -{"stream": "categories", "data": {"id": 1, "id_parent": "0", "level_depth": "0", "nb_products_recursive": "19", "active": "1", "id_shop_default": "1", "is_root_category": "0", "position": "0", "date_add": "2021-07-23 23:17:27", "date_upd": "2021-07-23 23:17:27", "name": "Root", "link_rewrite": "root", "description": "", "meta_title": "", "meta_description": "", "meta_keywords": "", "associations": {"categories": [{"id": "2"}]}}, "emitted_at": 1667903102866} -{"stream": "combinations", "data": {"id": 1, "id_product": "1", "location": "", "ean13": "", "isbn": "", "upc": "", "mpn": "", "quantity": "300", "reference": "demo_1", "supplier_reference": "", "wholesale_price": "0.000000", "price": "0.000000", "ecotax": "0.000000", "weight": "0.000000", "unit_price_impact": "0.000000", "minimal_quantity": "1", "low_stock_threshold": null, "low_stock_alert": "0", "default_on": "1", "available_date": "2022-01-01", "associations": {"product_option_values": [{"id": "1"}, {"id": "8"}], "images": [{"id": "2"}]}}, "emitted_at": 1667903102939} -{"stream": "configurations", "data": {"id": 354, "value": "1", "name": "GF_INSTALL_CALC", "id_shop_group": "", "id_shop": "", "date_add": "2021-07-23 23:18:07", "date_upd": "2021-07-23 18:07:10"}, "emitted_at": 1667903102994} -{"stream": "contacts", "data": {"id": 1, "email": "integration-test@airbyte.io", "customer_service": "1", "name": "Webmaster", "description": "If a technical problem occurs on this website"}, "emitted_at": 1667903103626} -{"stream": "content_management_system", "data": {"id": 1, "id_cms_category": "1", "position": "0", "indexation": "0", "active": "1", "meta_description": "Our terms and conditions of delivery", "meta_keywords": "conditions, delivery, delay, shipment, pack", "meta_title": "Delivery", "head_seo_title": "", "link_rewrite": "delivery", "content": "

    Shipments and returns

    Your pack shipment

    Packages are generally dispatched within 2 days after receipt of payment and are shipped via UPS with tracking and drop-off without signature. If you prefer delivery by UPS Extra with required signature, an additional cost will be applied, so please contact us before choosing this method. Whichever shipment choice you make, we will provide you with a link to track your package online.

    Shipping fees include handling and packing fees as well as postage costs. Handling fees are fixed, whereas transport fees vary according to total weight of the shipment. We advise you to group your items in one order. We cannot group two distinct orders placed separately, and shipping fees will apply to each of them. Your package will be dispatched at your own risk, but special care is taken to protect fragile objects.

    Boxes are amply sized and your items are well-protected.

    "}, "emitted_at": 1667903103655} -{"stream": "countries", "data": {"id": 1, "id_zone": "1", "id_currency": "0", "call_prefix": "49", "iso_code": "DE", "active": "0", "contains_states": "0", "need_identification_number": "0", "need_zip_code": "1", "zip_code_format": "NNNNN", "display_tax_label": "1", "name": "Germany"}, "emitted_at": 1667903103703} -{"stream": "currencies", "data": {"id": 1, "names": "US Dollar", "name": "US Dollar", "symbol": "$", "iso_code": "USD", "numeric_iso_code": "840", "precision": "2", "conversion_rate": "1.000000", "deleted": "0", "active": "1", "unofficial": "0", "modified": "0", "pattern": ""}, "emitted_at": 1667903103948} -{"stream": "customer_messages", "data": {"id": 1, "id_employee": "1", "id_customer_thread": "1", "ip_address": "100.1.1.0", "message": "Hello, how are you? It is me Eugene.", "file_name": null, "user_agent": "SOme user Agent2", "private": "0", "date_add": "2020-07-25 02:42:14", "date_upd": "2021-07-02 02:42:22", "read": "1"}, "emitted_at": 1667903103972} -{"stream": "customer_threads", "data": {"id": 1, "id_lang": "1", "id_shop": "1", "id_customer": "1", "id_order": "1", "id_product": "1", "id_contact": "1", "email": "some@mail.com", "token": null, "status": "open", "date_add": "2021-02-25 02:39:00", "date_upd": "2021-06-25 02:39:07", "associations": {"customer_messages": [{"id": "1"}, {"id": "2"}]}}, "emitted_at": 1667903104001} -{"stream": "customers", "data": {"id": 1, "id_default_group": "3", "id_lang": "1", "newsletter_date_add": "0000-00-00 00:00:00", "ip_registration_newsletter": "", "last_passwd_gen": "2021-07-23 17:18:07", "secure_key": "490f551646c7f55281a5be6e9d70f121", "deleted": "0", "passwd": "prestashop", "lastname": "Anonymous", "firstname": "Anonymous", "email": "anonymous@psgdpr.com", "id_gender": "1", "birthday": "0000-00-00", "newsletter": "0", "optin": "1", "website": "", "company": "", "siret": "", "ape": "", "outstanding_allow_amount": "0.000000", "show_public_prices": "0", "id_risk": "0", "max_payment_days": "0", "active": "0", "note": "", "is_guest": "0", "id_shop": "1", "id_shop_group": "1", "date_add": "2021-07-23 23:18:07", "date_upd": "2021-07-23 23:18:07", "reset_password_token": "", "reset_password_validity": "2025-01-01 10:30:00", "associations": {"groups": [{"id": "3"}]}}, "emitted_at": 1667903104030} -{"stream": "deliveries", "data": {"id": 1, "id_carrier": "2", "id_range_price": "0", "id_range_weight": "1", "id_zone": "1", "id_shop": "", "id_shop_group": "", "price": "5.000000"}, "emitted_at": 1667903104066} -{"stream": "employees", "data": {"id": 1, "id_lang": "1", "last_passwd_gen": "2021-07-23 17:17:28", "stats_date_from": "2021-06-23", "stats_date_to": "2021-07-23", "stats_compare_from": "2021-06-23", "stats_compare_to": "2021-07-23", "passwd": "$2y$10$BOkpSr6te9uvu97I05XI3u8hMMowukHd0PWNTR6qjTcA/YOlYc2ja", "lastname": "Kulak", "firstname": "Eugene", "email": "integration-test@airbyte.io", "active": "1", "id_profile": "1", "bo_color": null, "default_tab": "1", "bo_theme": "default", "bo_css": "theme.css", "bo_width": "0", "bo_menu": "1", "stats_compare_option": "1", "preselect_date_range": null, "id_last_order": "0", "id_last_customer_message": "0", "id_last_customer": "0", "reset_password_token": null, "reset_password_validity": "2025-01-01 10:30:00"}, "emitted_at": 1667903104106} -{"stream": "groups", "data": {"id": 1, "reduction": "0.00", "price_display_method": "1", "show_prices": "1", "date_add": "2021-07-23 23:17:27", "date_upd": "2021-07-23 23:17:28", "name": "Visitor"}, "emitted_at": 1667903104146} -{"stream": "guests", "data": {"id": 2, "id_customer": "0", "id_operating_system": "0", "id_web_browser": "0", "javascript": "0", "screen_resolution_x": "0", "screen_resolution_y": "0", "screen_color": "0", "sun_java": "0", "adobe_flash": "0", "adobe_director": "0", "apple_quicktime": "0", "real_player": "0", "windows_media": "0", "accept_language": "", "mobile_theme": "0"}, "emitted_at": 1667903104191} -{"stream": "image_types", "data": {"id": 1, "name": "cart_default", "width": "125", "height": "125", "categories": "0", "products": "1", "manufacturers": "0", "suppliers": "0", "stores": "0"}, "emitted_at": 1667903104232} -{"stream": "languages", "data": {"id": 1, "name": "English (English)", "iso_code": "en", "locale": "en-US", "language_code": "en-us", "active": "1", "is_rtl": "0", "date_format_lite": "m/d/Y", "date_format_full": "m/d/Y H:i:s"}, "emitted_at": 1667903104272} -{"stream": "manufacturers", "data": {"id": 1, "active": "1", "link_rewrite": "studio-design", "name": "Studio Design", "date_add": "2021-07-23 23:18:41", "date_upd": "2021-07-23 23:18:41", "description": "

    Studio Design offers a range of items from ready-to-wear collections to contemporary objects. The brand has been presenting new ideas and trends since its creation in 2012.

    ", "short_description": "", "meta_title": "", "meta_description": "", "meta_keywords": "", "associations": {"addresses": [{"id": "4"}]}}, "emitted_at": 1667903104313} -{"stream": "messages", "data": {"id": 1, "id_cart": "1", "id_order": "1", "id_customer": "2", "id_employee": "1", "message": "This message is about order ", "private": "1", "date_add": "2021-06-25 04:20:13"}, "emitted_at": 1667903104358} -{"stream": "order_carriers", "data": {"id": 1, "id_order": "1", "id_carrier": "2", "id_order_invoice": "0", "weight": "0.000000", "shipping_cost_tax_excl": "2.000000", "shipping_cost_tax_incl": "2.000000", "tracking_number": "", "date_add": "2021-07-23 23:18:42"}, "emitted_at": 1667903104401} -{"stream": "order_details", "data": {"id": 1, "id_order": "1", "product_id": "1", "product_attribute_id": "1", "product_quantity_reinjected": "0", "group_reduction": "0.00", "discount_quantity_applied": "0", "download_hash": "", "download_deadline": "0000-00-00 00:00:00", "id_order_invoice": "0", "id_warehouse": "0", "id_shop": "1", "id_customization": "0", "product_name": "Hummingbird printed t-shirt - Color : White, Size : S", "product_quantity": "1", "product_quantity_in_stock": "1", "product_quantity_return": "0", "product_quantity_refunded": "0", "product_price": "23.900000", "reduction_percent": "0.00", "reduction_amount": "0.000000", "reduction_amount_tax_incl": "0.000000", "reduction_amount_tax_excl": "0.000000", "product_quantity_discount": "0.000000", "product_ean13": "", "product_isbn": "", "product_upc": "", "product_mpn": "", "product_reference": "demo_1", "product_supplier_reference": "", "product_weight": "0.000000", "tax_computation_method": "0", "id_tax_rules_group": "0", "ecotax": "0.000000", "ecotax_tax_rate": "0.000", "download_nb": "0", "unit_price_tax_incl": "23.900000", "unit_price_tax_excl": "23.900000", "total_price_tax_incl": "23.900000", "total_price_tax_excl": "23.900000", "total_shipping_price_tax_excl": "0.000000", "total_shipping_price_tax_incl": "0.000000", "purchase_supplier_price": "0.000000", "original_product_price": "23.900000", "original_wholesale_price": "0.000000", "total_refunded_tax_excl": "0.000000", "total_refunded_tax_incl": "0.000000"}, "emitted_at": 1667903104450} -{"stream": "order_histories", "data": {"id": 1, "id_employee": "0", "id_order_state": "1", "id_order": "1", "date_add": "2021-07-23 23:18:42"}, "emitted_at": 1667903104490} -{"stream": "order_invoices", "data": {"id": 1, "id_order": "1", "number": "1000", "delivery_number": "1", "delivery_date": "2021-08-25 03:08:28", "total_discount_tax_excl": "23.000000", "total_discount_tax_incl": "25.000000", "total_paid_tax_excl": "11.000000", "total_paid_tax_incl": "33.000000", "total_products": "1.000000", "total_products_wt": "0.000000", "total_shipping_tax_excl": "0.000000", "total_shipping_tax_incl": "0.000000", "shipping_tax_computation_method": "1", "total_wrapping_tax_excl": "0.000000", "total_wrapping_tax_incl": "0.000000", "shop_address": "Test address", "note": "some note", "date_add": "2021-05-25 03:09:23"}, "emitted_at": 1667903104525} -{"stream": "order_payments", "data": {"id": 1, "order_reference": "1000", "id_currency": "1", "amount": "30.000000", "payment_method": "1", "conversion_rate": "1.000000", "transaction_id": "1", "card_number": "12343243432434", "card_brand": "Visa", "card_expiration": "10-22", "card_holder": "Customer", "date_add": "2021-07-09 04:22:37"}, "emitted_at": 1667903104553} -{"stream": "order_slip", "data": {"id": 1, "id_customer": "1", "id_order": "1", "conversion_rate": "1.000000", "total_products_tax_excl": "30.000000", "total_products_tax_incl": "20.000000", "total_shipping_tax_excl": "11.000000", "total_shipping_tax_incl": "12.000000", "amount": "10.000000", "shipping_cost": "0", "shipping_cost_amount": "15.000000", "partial": "1", "date_add": "2021-05-25 04:25:48", "date_upd": "2021-06-25 04:25:53", "order_slip_type": "0"}, "emitted_at": 1667903104580} -{"stream": "order_states", "data": {"id": 2, "unremovable": "1", "delivery": "0", "hidden": "0", "send_email": "1", "module_name": "", "invoice": "1", "color": "#3498D8", "logable": "1", "shipped": "0", "paid": "1", "pdf_delivery": "0", "pdf_invoice": "1", "deleted": "0", "name": "Payment accepted", "template": "payment"}, "emitted_at": 1667903104608} -{"stream": "orders", "data": {"id": 1, "id_address_delivery": "5", "id_address_invoice": "5", "id_cart": "1", "id_currency": "1", "id_lang": "1", "id_customer": "2", "id_carrier": "2", "current_state": "6", "module": "ps_checkpayment", "invoice_number": "0", "invoice_date": "0000-00-00 00:00:00", "delivery_number": "0", "delivery_date": "0000-00-00 00:00:00", "valid": "0", "date_add": "2021-07-23 23:18:42", "date_upd": "2021-07-23 23:18:42", "shipping_number": "", "id_shop_group": "1", "id_shop": "1", "secure_key": "b44a6d9efd7a0076a0fbce6b15eaf3b1", "payment": "Payment by check", "recyclable": "0", "gift": "0", "gift_message": "", "mobile_theme": "0", "total_discounts": "0.000000", "total_discounts_tax_incl": "0.000000", "total_discounts_tax_excl": "0.000000", "total_paid": "61.800000", "total_paid_tax_incl": "61.800000", "total_paid_tax_excl": "61.800000", "total_paid_real": "0.000000", "total_products": "59.800000", "total_products_wt": "59.800000", "total_shipping": "2.000000", "total_shipping_tax_incl": "2.000000", "total_shipping_tax_excl": "2.000000", "carrier_tax_rate": "0.000", "total_wrapping": "0.000000", "total_wrapping_tax_incl": "0.000000", "total_wrapping_tax_excl": "0.000000", "round_mode": "0", "round_type": "0", "conversion_rate": "1.000000", "reference": "XKBKNABJK", "associations": {"order_rows": [{"id": "1", "product_id": "1", "product_attribute_id": "1", "product_quantity": "1", "product_name": "Hummingbird printed t-shirt - Color : White, Size : S", "product_reference": "demo_1", "product_ean13": "", "product_isbn": "", "product_upc": "", "product_price": "23.900000", "id_customization": "0", "unit_price_tax_incl": "23.900000", "unit_price_tax_excl": "23.900000"}, {"id": "2", "product_id": "2", "product_attribute_id": "9", "product_quantity": "1", "product_name": "Hummingbird printed sweater - Color : White, Size : S", "product_reference": "demo_3", "product_ean13": "", "product_isbn": "", "product_upc": "", "product_price": "35.900000", "id_customization": "0", "unit_price_tax_incl": "35.900000", "unit_price_tax_excl": "35.900000"}]}}, "emitted_at": 1667903104644} -{"stream": "price_ranges", "data": {"id": 1, "id_carrier": "2", "delimiter1": "0.000000", "delimiter2": "10000.000000"}, "emitted_at": 1667903104689} -{"stream": "product_customization_fields", "data": {"id": 1, "id_product": "19", "type": "1", "required": "1", "is_module": "0", "is_deleted": "0", "name": "Type your text here"}, "emitted_at": 1667903104729} -{"stream": "product_feature_values", "data": {"id": 1, "id_feature": "1", "custom": "0", "value": "Polyester"}, "emitted_at": 1667903104770} -{"stream": "product_features", "data": {"id": 1, "position": "0", "name": "Composition"}, "emitted_at": 1667903104811} -{"stream": "product_option_values", "data": {"id": 1, "id_attribute_group": "1", "color": "", "position": "0", "name": "S"}, "emitted_at": 1667903104852} -{"stream": "product_options", "data": {"id": 1, "is_color_group": "0", "group_type": "select", "position": "0", "name": "Size", "public_name": "Size", "associations": {"product_option_values": [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}]}}, "emitted_at": 1667903104883} -{"stream": "product_suppliers", "data": {"id": 1, "id_product": "19", "id_product_attribute": "0", "id_supplier": "1", "id_currency": "1", "product_supplier_reference": "", "product_supplier_price_te": "0.000000"}, "emitted_at": 1667903104920} -{"stream": "products", "data": {"id": 1, "id_manufacturer": "1", "id_supplier": "0", "id_category_default": "4", "new": null, "cache_default_attribute": "1", "id_default_image": "1", "id_default_combination": "1", "id_tax_rules_group": "9", "position_in_category": "1", "manufacturer_name": "Studio Design", "quantity": "0", "type": "simple", "id_shop_default": "1", "reference": "demo_1", "supplier_reference": "", "location": "", "width": "0.000000", "height": "0.000000", "depth": "0.000000", "weight": "0.300000", "quantity_discount": "0", "ean13": "", "isbn": "", "upc": "", "mpn": "", "cache_is_pack": "0", "cache_has_attachments": "0", "is_virtual": "0", "state": "1", "additional_delivery_times": "1", "delivery_in_stock": "", "delivery_out_stock": "", "on_sale": "0", "online_only": "0", "ecotax": "0.000000", "minimal_quantity": "1", "low_stock_threshold": null, "low_stock_alert": "0", "price": "23.900000", "wholesale_price": "0.000000", "unity": "", "unit_price_ratio": "0.000000", "additional_shipping_cost": "0.000000", "customizable": "0", "text_fields": "0", "uploadable_files": "0", "active": "1", "redirect_type": "301-category", "id_type_redirected": "0", "available_for_order": "1", "available_date": "0000-00-00", "show_condition": "0", "condition": "new", "show_price": "1", "indexed": "1", "visibility": "both", "advanced_stock_management": "0", "date_add": "2021-07-23 23:18:42", "date_upd": "2021-07-23 23:18:42", "pack_stock_type": "3", "meta_description": "", "meta_keywords": "", "meta_title": "", "link_rewrite": "hummingbird-printed-t-shirt", "name": "Hummingbird printed t-shirt", "description": "

    Symbol of lightness and delicacy, the hummingbird evokes curiosity and joy. Studio Design' PolyFaune collection features classic products with colorful patterns, inspired by the traditional japanese origamis. To wear with a chino or jeans. The sublimation textile printing process provides an exceptional color rendering and a color, guaranteed overtime.

    ", "description_short": "

    Regular fit, round neckline, short sleeves. Made of extra long staple pima cotton.

    \r\n

    ", "available_now": "", "available_later": "", "associations": {"categories": [{"id": "2"}, {"id": "3"}, {"id": "4"}], "images": [{"id": "1"}, {"id": "2"}], "combinations": [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}, {"id": "5"}, {"id": "6"}, {"id": "7"}, {"id": "8"}], "product_option_values": [{"id": "1"}, {"id": "8"}, {"id": "11"}, {"id": "2"}, {"id": "3"}, {"id": "4"}], "product_features": [{"id": "1", "id_feature_value": "4"}, {"id": "2", "id_feature_value": "8"}], "stock_availables": [{"id": "1", "id_product_attribute": "0"}, {"id": "20", "id_product_attribute": "1"}, {"id": "21", "id_product_attribute": "2"}, {"id": "22", "id_product_attribute": "3"}, {"id": "23", "id_product_attribute": "4"}, {"id": "24", "id_product_attribute": "5"}, {"id": "25", "id_product_attribute": "6"}, {"id": "26", "id_product_attribute": "7"}, {"id": "27", "id_product_attribute": "8"}]}}, "emitted_at": 1667903105015} -{"stream": "shop_groups", "data": {"id": 1, "name": "Default", "share_customer": "0", "share_order": "0", "share_stock": "0", "active": "1", "deleted": "0"}, "emitted_at": 1667903105082} -{"stream": "shop_urls", "data": {"id": 1, "id_shop": "1", "active": "1", "main": "1", "domain": "localhost:8080", "domain_ssl": "localhost:8080", "physical_uri": "/", "virtual_uri": ""}, "emitted_at": 1667903105120} -{"stream": "shops", "data": {"id": 1, "id_shop_group": "1", "id_category": "2", "active": "1", "deleted": "0", "name": "Airbyte", "theme_name": "classic"}, "emitted_at": 1667903105158} -{"stream": "specific_price_rules", "data": {"id": 2, "id_shop": "1", "id_country": "0", "id_currency": "0", "id_group": "0", "name": "Test price rule 2", "from_quantity": "1", "price": "-1.000000", "reduction": "1.000000", "reduction_tax": "0", "reduction_type": "amount", "from": "0000-00-00 00:00:00", "to": "0000-00-00 00:00:00"}, "emitted_at": 1667903105199} -{"stream": "specific_prices", "data": {"id": 1, "id_shop_group": "0", "id_shop": "0", "id_cart": "0", "id_product": "1", "id_product_attribute": "0", "id_currency": "0", "id_country": "0", "id_group": "0", "id_customer": "0", "id_specific_price_rule": "0", "price": "-1.000000", "from_quantity": "1", "reduction": "0.200000", "reduction_tax": "1", "reduction_type": "percentage", "from": "0000-00-00 00:00:00", "to": "0000-00-00 00:00:00"}, "emitted_at": 1667903105241} -{"stream": "states", "data": {"id": 89, "id_zone": "2", "id_country": "4", "iso_code": "ON", "name": "Ontario", "active": "1"}, "emitted_at": 1667903105781} -{"stream": "stock_availables", "data": {"id": 1, "id_product": "1", "id_product_attribute": "0", "id_shop": "1", "id_shop_group": "0", "quantity": "2958", "depends_on_stock": "0", "out_of_stock": "2", "location": ""}, "emitted_at": 1667903106787} -{"stream": "stock_movement_reasons", "data": {"id": 1, "sign": "1", "deleted": "0", "date_add": "2021-07-23 23:17:27", "date_upd": "2021-07-23 23:17:27", "name": "Increase"}, "emitted_at": 1667903106900} -{"stream": "stock_movements", "data": {"id": 1, "id_product": "", "id_product_attribute": "", "id_warehouse": "", "id_currency": "", "management_type": null, "id_employee": "1", "id_stock": "20", "id_stock_mvt_reason": "11", "id_order": "", "id_supply_order": "", "product_name": false, "ean13": null, "upc": null, "reference": null, "mpn": null, "physical_quantity": "500", "sign": "1", "last_wa": "0.000000", "current_wa": "0.000000", "price_te": "0.000000", "date_add": "2021-07-24 20:28:51"}, "emitted_at": 1667903107797} -{"stream": "stores", "data": {"id": 1, "id_country": "21", "id_state": "12", "hours": " [[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"10:00AM - 04:00PM\"],[\"10:00AM - 04:00PM\"]]", "postcode": "33135", "city": "Miami", "latitude": "25.76500500", "longitude": "-80.24379700", "phone": "", "fax": "", "email": "", "active": "1", "date_add": "2021-07-23 23:18:43", "date_upd": "2021-07-23 23:18:43", "name": "Dade County", "address1": "3030 SW 8th St Miami", "address2": "", "note": ""}, "emitted_at": 1667903107872} -{"stream": "suppliers", "data": {"id": 1, "link_rewrite": "some-cool-supplier", "name": "Some cool supplier", "active": "1", "date_add": "2021-07-24 20:26:22", "date_upd": "2021-07-24 20:26:22", "description": "

    This is a reach text.

    \n

    bold colored

    \n

    italic

    \n

    underscored

    \n

    strikeout

    \n
    \n

    test quotes

    \n

    another line

    \n
    \n

    HEADER 1

    \n

    header 2

    ", "meta_title": "", "meta_description": "", "meta_keywords": "Tag1, Tag2, Tag3, Tag4, Tag4, Tag5, Tag6, Tag7, Tag8, Tag9, Tag10, Tag11"}, "emitted_at": 1667903107922} -{"stream": "tags", "data": {"id": 1, "id_lang": "1", "name": "Crazy"}, "emitted_at": 1667903107957} -{"stream": "tax_rule_groups", "data": {"id": 1, "name": "US-AL Rate (4%)", "active": "1", "deleted": "0", "date_add": "2021-07-23 23:17:28", "date_upd": "2021-07-23 23:17:28"}, "emitted_at": 1667903108794} -{"stream": "tax_rules", "data": {"id": 1, "id_tax_rules_group": "1", "id_state": "4", "id_country": "21", "zipcode_from": "0", "zipcode_to": "0", "id_tax": "1", "behavior": "1", "description": ""}, "emitted_at": 1667903108948} -{"stream": "taxes", "data": {"id": 1, "rate": "4.000", "active": "1", "deleted": "0", "name": "Sales-taxes US-AL 4%"}, "emitted_at": 1667903109059} -{"stream": "translated_configurations", "data": {"id": 38, "value": "#IN", "date_add": "2021-07-25 20:30:30", "date_upd": null, "name": "PS_INVOICE_PREFIX", "id_shop_group": "", "id_shop": ""}, "emitted_at": 1667903109162} -{"stream": "weight_ranges", "data": {"id": 1, "id_carrier": "2", "delimiter1": "0.000000", "delimiter2": "10000.000000"}, "emitted_at": 1667903109211} -{"stream": "zones", "data": {"id": 1, "name": "Europe", "active": "1"}, "emitted_at": 1667903109268} +{"stream":"addresses","data":{"id":1,"id_customer":"1","id_manufacturer":"0","id_supplier":"0","id_warehouse":"0","id_country":"21","id_state":"0","alias":"Anonymous","company":"Anonymous","lastname":"Anonymous","firstname":"Anonymous","vat_number":"0000","address1":"Anonymous","address2":"","postcode":"00000","city":"Anonymous","other":"","phone":"0000000000","phone_mobile":"0000000000","dni":"0000","deleted":"0","date_add":"2022-05-17 17:49:52","date_upd":"2022-05-17 17:49:52"},"emitted_at":1687800491899} +{"stream":"carriers","data":{"id":1,"deleted":"0","is_module":"0","id_tax_rules_group":"1","id_reference":"1","name":"Integration Test","active":"1","is_free":"1","url":"","shipping_handling":"0","shipping_external":"0","range_behavior":"0","shipping_method":1,"max_width":"0","max_height":"0","max_depth":"0","max_weight":"0.000000","grade":"0","external_module_name":"","need_range":"0","position":"0","delay":"Pick up in-store"},"emitted_at":1687800493295} +{"stream":"cart_rules","data":{"id":1,"id_customer":"0","date_from":"2023-06-26 20:00:00","date_to":"2023-07-26 20:00:00","description":"20% OFF","quantity":"1","quantity_per_user":"1","priority":"1","partial_use":"1","code":"20OFF","minimum_amount":"100.000000","minimum_amount_tax":"0","minimum_amount_currency":"1","minimum_amount_shipping":"0","country_restriction":"0","carrier_restriction":"0","group_restriction":"0","cart_rule_restriction":"0","product_restriction":"0","shop_restriction":"0","free_shipping":"0","reduction_percent":"20.00","reduction_amount":"0.000000","reduction_tax":"0","reduction_currency":"1","reduction_product":"0","reduction_exclude_special":"0","gift_product":"0","gift_product_attribute":"0","highlight":"0","active":"1","date_add":"2023-06-26 20:11:01","date_upd":"2023-06-26 20:11:01","name":"20OFF"},"emitted_at":1687811469437} +{"stream":"carts","data":{"id":1,"id_address_delivery":"5","id_address_invoice":"5","id_currency":"1","id_customer":"2","id_guest":"1","id_lang":"1","id_shop_group":"1","id_shop":"1","id_carrier":"2","recyclable":"0","gift":"0","gift_message":"","mobile_theme":"0","delivery_option":"{\"3\":\"2,\"}","secure_key":"b44a6d9efd7a0076a0fbce6b15eaf3b1","allow_seperated_package":"0","date_add":"2022-05-17 17:50:34","date_upd":"2022-05-17 17:50:34","associations":{"cart_rows":[{"id_product":"1","id_product_attribute":"1","id_address_delivery":"3","id_customization":"0","quantity":"1"},{"id_product":"2","id_product_attribute":"9","id_address_delivery":"3","id_customization":"0","quantity":"1"}]}},"emitted_at":1687800495375} +{"stream":"categories","data":{"id":1,"id_parent":"0","level_depth":"0","nb_products_recursive":"20","active":"1","id_shop_default":"1","is_root_category":"0","position":"0","date_add":"2022-05-17 17:48:55","date_upd":"2022-05-17 17:48:55","name":"Root","link_rewrite":"root","description":"","meta_title":"","meta_description":"","meta_keywords":"","associations":{"categories":[{"id":"2"}]}},"emitted_at":1687800496905} +{"stream":"combinations","data":{"id":1,"id_product":"1","location":"","ean13":"","isbn":"","upc":"","mpn":"","quantity":310,"reference":"demo_1","supplier_reference":"","wholesale_price":"0.000000","price":"0.000000","ecotax":"0.000000","weight":"0.000000","unit_price_impact":"0.000000","minimal_quantity":"1","low_stock_threshold":null,"low_stock_alert":"0","default_on":"1","available_date":null,"associations":{"product_option_values":[{"id":"1"},{"id":"8"}],"images":[{"id":"2"}]}},"emitted_at":1687871907372} +{"stream":"configurations","data":{"id":354,"value":null,"name":"BLOCKSOCIAL_INSTAGRAM","id_shop_group":"","id_shop":"","date_add":"2022-05-17 17:49:06","date_upd":"2022-05-17 17:49:06"},"emitted_at":1687800499544} +{"stream":"contacts","data":{"id":1,"email":"ryan@airbyte.io","customer_service":"1","name":"Webmaster","description":"If a technical problem occurs on this website"},"emitted_at":1687800501641} +{"stream":"content_management_system","data":{"id":1,"id_cms_category":"1","position":"0","indexation":"0","active":"1","meta_description":"Our terms and conditions of delivery","meta_keywords":"conditions, delivery, delay, shipment, pack","meta_title":"Delivery","head_seo_title":"","link_rewrite":"delivery","content":"

    Shipments and returns

    Your pack shipment

    Packages are generally dispatched within 2 days after receipt of payment and are shipped via UPS with tracking and drop-off without signature. If you prefer delivery by UPS Extra with required signature, an additional cost will be applied, so please contact us before choosing this method. Whichever shipment choice you make, we will provide you with a link to track your package online.

    Shipping fees include handling and packing fees as well as postage costs. Handling fees are fixed, whereas transport fees vary according to total weight of the shipment. We advise you to group your items in one order. We cannot group two distinct orders placed separately, and shipping fees will apply to each of them. Your package will be dispatched at your own risk, but special care is taken to protect fragile objects.

    Boxes are amply sized and your items are well-protected.

    "},"emitted_at":1687800502421} +{"stream":"countries","data":{"id":1,"id_zone":"1","id_currency":"0","call_prefix":"49","iso_code":"DE","active":"0","contains_states":"0","need_identification_number":"0","need_zip_code":"1","zip_code_format":"NNNNN","display_tax_label":"1","name":"Germany"},"emitted_at":1687800503199} +{"stream":"currencies","data":{"id":1,"names":"US Dollar","name":"US Dollar","symbol":"$","iso_code":"USD","numeric_iso_code":"840","precision":"2","conversion_rate":"1.000000","deleted":"0","active":"1","unofficial":"0","modified":"0","pattern":""},"emitted_at":1687800505481} +{"stream":"customer_messages","data":{"id":1,"id_employee":"0","id_customer_thread":"1","ip_address":"0","message":"Hey,\r\n\r\nYour website's design is absolutely brilliant. The visuals really enhance your message and the content compels action. I've forwarded it to a few of my contacts who I think could benefit from your services.\r\n\r\nWhen I was looking at your site, though, I noticed some mistakes that you've made re: search engine optimization (SEO) which may be leading to a decline in your organic SEO results. Would you like to fix it so that you can get maximum exposure/presence on Google, Bing, Yahoo and web traffic to your website?\r\n\r\nIt's a relatively simple fix. If this is a priority.\r\n\r\nPlease share your “Mobile Number\" and a suitable time to talk, so I can help you in that.\r\n\r\n\r\nBest Regards\r\nHarry Williams\r\nDigital Marketing Team\r\nCall Us: 1-620-765-4699","file_name":"","user_agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36","private":"0","date_add":"2022-07-27 21:33:43","date_upd":"2022-07-27 21:33:43","read":"0"},"emitted_at":1687800506630} +{"stream":"customer_threads","data":{"id":1,"id_lang":"1","id_shop":"1","id_customer":"0","id_order":"0","id_product":"0","id_contact":"1","email":"harrywilliams.websolution02@gmail.com","token":"1WxZqrTyu33z","status":"closed","date_add":"2022-07-27 21:33:43","date_upd":"2023-06-27 11:19:44","associations":{"customer_messages":[{"id":"1"},{"id":"49"}]}},"emitted_at":1687873500479} +{"stream":"customers","data":{"id":1,"id_default_group":"3","id_lang":"1","newsletter_date_add":"0000-00-00 00:00:00","ip_registration_newsletter":"","last_passwd_gen":"2022-05-17 11:49:52","secure_key":"3afae1637af7b9e48272a78400a5f8c5","deleted":"0","passwd":"prestashop","lastname":"Anonymous","firstname":"Anonymous","email":"anonymous@psgdpr.com","id_gender":"1","birthday":"0000-00-00","newsletter":"0","optin":"1","website":"","company":"","siret":"","ape":"","outstanding_allow_amount":"0.000000","show_public_prices":"0","id_risk":"0","max_payment_days":"0","active":"0","note":"","is_guest":"0","id_shop":"1","id_shop_group":"1","date_add":"2022-05-17 17:49:52","date_upd":"2022-05-17 17:49:52","reset_password_token":"","reset_password_validity":null,"associations":{"groups":[{"id":"3"}]}},"emitted_at":1687873501233} +{"stream":"deliveries","data":{"id":1,"id_carrier":"2","id_range_price":"0","id_range_weight":"1","id_zone":"1","id_shop":"","id_shop_group":"","price":"5.000000"},"emitted_at":1687800510532} +{"stream":"employees","data":{"id":1,"id_lang":"1","last_passwd_gen":"2022-05-17 11:48:58","stats_date_from":"2022-04-17","stats_date_to":"2022-05-17","stats_compare_from":null,"stats_compare_to":null,"passwd":"$2y$10$ls8Eq/JxVseLRiQ2g0S11u.0HOdd2/C.UUhuU8dVteYAPSGQcYR.u","lastname":"Owner","firstname":"Site","email":"ryan@airbyte.io","active":"1","id_profile":"1","bo_color":"","default_tab":"1","bo_theme":"default","bo_css":"theme.css","bo_width":"0","bo_menu":"1","stats_compare_option":"1","preselect_date_range":"","id_last_order":"5","id_last_customer_message":"0","id_last_customer":"0","reset_password_token":"43190584ab3117d2685939aa87325eb5a746c64b","reset_password_validity":"2022-07-19 19:25:07","has_enabled_gravatar":"0"},"emitted_at":1687800511327} +{"stream":"groups","data":{"id":1,"reduction":"0.00","price_display_method":"1","show_prices":"1","date_add":"2022-05-17 17:48:55","date_upd":"2022-05-17 17:48:58","name":"Visitor"},"emitted_at":1687800512052} +{"stream":"guests","data":{"id":2,"id_customer":"0","id_operating_system":"0","id_web_browser":"0","javascript":"0","screen_resolution_x":"0","screen_resolution_y":"0","screen_color":"0","sun_java":"0","adobe_flash":"0","adobe_director":"0","apple_quicktime":"0","real_player":"0","windows_media":"0","accept_language":"","mobile_theme":"0"},"emitted_at":1687800513448} +{"stream":"image_types","data":{"id":1,"name":"cart_default","width":"125","height":"125","categories":"0","products":"1","manufacturers":"0","suppliers":"0","stores":"0"},"emitted_at":1687800707301} +{"stream":"languages","data":{"id":1,"name":"English (English)","iso_code":"en","locale":"en-US","language_code":"en-us","active":"1","is_rtl":"0","date_format_lite":"m/d/Y","date_format_full":"m/d/Y H:i:s"},"emitted_at":1687800708042} +{"stream":"manufacturers","data":{"id":1,"active":"1","link_rewrite":"studio-design","name":"Studio Design","date_add":"2022-05-17 17:50:33","date_upd":"2022-05-17 17:50:33","description":"

    Studio Design offers a range of items from ready-to-wear collections to contemporary objects. The brand has been presenting new ideas and trends since its creation in 2012.

    ","short_description":"","meta_title":"","meta_description":"","meta_keywords":"","associations":{"addresses":[{"id":"4"}]}},"emitted_at":1687800708785} +{"stream":"order_carriers","data":{"id":1,"id_order":"1","id_carrier":"2","id_order_invoice":"0","weight":"0.000000","shipping_cost_tax_excl":"7.000000","shipping_cost_tax_incl":"8.400000","tracking_number":"","date_add":"2022-05-17 17:50:34"},"emitted_at":1687800711563} +{"stream":"order_details","data":{"id":1,"id_order":"1","product_id":"1","product_attribute_id":"1","product_quantity_reinjected":"0","group_reduction":"0.00","discount_quantity_applied":"0","download_hash":"","download_deadline":"0000-00-00 00:00:00","id_order_invoice":"0","id_warehouse":"0","id_shop":"1","id_customization":"0","product_name":"Hummingbird printed t-shirt - Color : White, Size : S","product_quantity":"1","product_quantity_in_stock":"1","product_quantity_return":"0","product_quantity_refunded":"0","product_price":"23.900000","reduction_percent":"0.00","reduction_amount":"0.000000","reduction_amount_tax_incl":"0.000000","reduction_amount_tax_excl":"0.000000","product_quantity_discount":"0.000000","product_ean13":"","product_isbn":"","product_upc":"","product_mpn":"","product_reference":"demo_1","product_supplier_reference":"","product_weight":"0.000000","tax_computation_method":"0","id_tax_rules_group":"0","ecotax":"0.000000","ecotax_tax_rate":"0.000","download_nb":"0","unit_price_tax_incl":"23.900000","unit_price_tax_excl":"23.900000","total_price_tax_incl":"23.900000","total_price_tax_excl":"23.900000","total_shipping_price_tax_excl":"0.000000","total_shipping_price_tax_incl":"0.000000","purchase_supplier_price":"0.000000","original_product_price":"23.900000","original_wholesale_price":"0.000000","total_refunded_tax_excl":"0.000000","total_refunded_tax_incl":"0.000000"},"emitted_at":1687800713016} +{"stream":"order_histories","data":{"id":1,"id_employee":"0","id_order_state":"1","id_order":"1","date_add":"2022-05-17 17:50:34"},"emitted_at":1687800713819} +{"stream":"order_invoices","data":{"id":1,"id_order":"6","number":"1","delivery_number":"1","delivery_date":"2023-06-26 20:30:53","total_discount_tax_excl":"0.000000","total_discount_tax_incl":"0.000000","total_paid_tax_excl":"35.720000","total_paid_tax_incl":"35.720000","total_products":"28.720000","total_products_wt":"28.720000","total_shipping_tax_excl":"7.000000","total_shipping_tax_incl":"7.000000","shipping_tax_computation_method":"0","total_wrapping_tax_excl":"0.000000","total_wrapping_tax_incl":"0.000000","shop_address":"Integration Test
    2261 Market Street 4381
    San Francisco, 94114
    United States
    14156236785","note":"Test note","date_add":"2023-04-24 10:16:20"},"emitted_at":1687875414084} +{"stream":"order_payments","data":{"id":1,"order_reference":"ZQXARYQUL","id_currency":"1","amount":"35.720000","payment_method":"Payments by check","conversion_rate":"1.000000","transaction_id":"","card_number":"","card_brand":"","card_expiration":"","card_holder":"","date_add":"2023-04-24 10:16:59"},"emitted_at":1687800716963} +{"stream":"order_slip","data":{"id":1,"id_customer":"4","id_order":"7","conversion_rate":"1.000000","total_products_tax_excl":"57.440000","total_products_tax_incl":"57.440000","total_shipping_tax_excl":"0.000000","total_shipping_tax_incl":"0.000000","amount":"57.440000","shipping_cost":"0","shipping_cost_amount":"0.000000","partial":"0","date_add":"2023-06-27 10:21:21","date_upd":"2023-06-27 10:21:21","order_slip_type":"0","associations":{"order_slip_details":[{"id":"1","id_order_detail":"9","product_quantity":"2","amount_tax_excl":"57.440000","amount_tax_incl":"57.440000"}]}},"emitted_at":1687868307035} +{"stream":"order_states","data":{"id":2,"unremovable":"1","delivery":"0","hidden":"0","send_email":"1","module_name":"","invoice":"1","color":"#3498D8","logable":"1","shipped":"0","paid":"1","pdf_delivery":"0","pdf_invoice":"1","deleted":"0","name":"Payment accepted","template":"payment"},"emitted_at":1687800719394} +{"stream":"orders","data":{"id":1,"id_address_delivery":"5","id_address_invoice":"5","id_cart":"1","id_currency":"1","id_lang":"1","id_customer":"2","id_carrier":"2","current_state":"6","module":"ps_checkpayment","invoice_number":"0","invoice_date":"0000-00-00 00:00:00","delivery_number":"0","delivery_date":"0000-00-00 00:00:00","valid":"0","date_add":"2022-05-17 17:50:34","date_upd":"2022-05-17 17:50:34","shipping_number":"","note":"Test","id_shop_group":"1","id_shop":"1","secure_key":"b44a6d9efd7a0076a0fbce6b15eaf3b1","payment":"Payment by check","recyclable":"0","gift":"0","gift_message":"","mobile_theme":"0","total_discounts":"0.000000","total_discounts_tax_incl":"0.000000","total_discounts_tax_excl":"0.000000","total_paid":"61.800000","total_paid_tax_incl":"68.200000","total_paid_tax_excl":"66.800000","total_paid_real":"0.000000","total_products":"59.800000","total_products_wt":"59.800000","total_shipping":"7.000000","total_shipping_tax_incl":"8.400000","total_shipping_tax_excl":"7.000000","carrier_tax_rate":"0.000","total_wrapping":"0.000000","total_wrapping_tax_incl":"0.000000","total_wrapping_tax_excl":"0.000000","round_mode":"0","round_type":"0","conversion_rate":"1.000000","reference":"XKBKNABJK","associations":{"order_rows":[{"id":"1","product_id":"1","product_attribute_id":"1","product_quantity":"1","product_name":"Hummingbird printed t-shirt - Color : White, Size : S","product_reference":"demo_1","product_ean13":"","product_isbn":"","product_upc":"","product_price":"23.900000","id_customization":"0","unit_price_tax_incl":"23.900000","unit_price_tax_excl":"23.900000"},{"id":"2","product_id":"2","product_attribute_id":"9","product_quantity":"1","product_name":"Hummingbird printed sweater - Color : White, Size : S","product_reference":"demo_3","product_ean13":"","product_isbn":"","product_upc":"","product_price":"35.900000","id_customization":"0","unit_price_tax_incl":"35.900000","unit_price_tax_excl":"35.900000"}]}},"emitted_at":1687800720186} +{"stream":"price_ranges","data":{"id":1,"id_carrier":"2","delimiter1":"0.000000","delimiter2":"10000.000000"},"emitted_at":1687800721581} +{"stream":"product_customization_fields","data":{"id":1,"id_product":"19","type":"1","required":"1","is_module":"0","is_deleted":"0","name":"Type your text here"},"emitted_at":1687800722377} +{"stream":"product_feature_values","data":{"id":1,"id_feature":"1","custom":"0","value":"Polyester"},"emitted_at":1687800723135} +{"stream":"product_features","data":{"id":1,"position":"0","name":"Composition"},"emitted_at":1687800723917} +{"stream":"product_option_values","data":{"id":1,"id_attribute_group":"1","color":"","position":"0","name":"S"},"emitted_at":1687800724685} +{"stream":"product_options","data":{"id":1,"is_color_group":"0","group_type":"select","position":"0","name":"Size","public_name":"Size","associations":{"product_option_values":[{"id":"1"},{"id":"2"},{"id":"3"},{"id":"4"}]}},"emitted_at":1687800725445} +{"stream":"product_suppliers","data":{"id":1,"id_product":"6","id_product_attribute":"0","id_supplier":"2","id_currency":"0","product_supplier_reference":"demo_11","product_supplier_price_te":"5.490000"},"emitted_at":1687800726422} +{"stream":"products","data":{"id":1,"id_manufacturer":"1","id_supplier":"1","id_category_default":"4","new":null,"cache_default_attribute":"1","id_default_image":"1","id_default_combination":"1","id_tax_rules_group":"9","position_in_category":"1","manufacturer_name":"Studio Design","quantity":"0","type":"simple","id_shop_default":"1","reference":"demo_1","supplier_reference":"","location":"","width":"0.000000","height":"0.000000","depth":"0.000000","weight":"0.300000","quantity_discount":"0","ean13":"","isbn":"","upc":"","mpn":"","cache_is_pack":"0","cache_has_attachments":"0","is_virtual":"0","state":"1","additional_delivery_times":"1","delivery_in_stock":"","delivery_out_stock":"","product_type":"combinations","on_sale":"0","online_only":"0","ecotax":"0.000000","minimal_quantity":"1","low_stock_threshold":null,"low_stock_alert":"0","price":"23.900000","wholesale_price":"0.000000","unity":"","unit_price_ratio":"0.000000","additional_shipping_cost":"0.000000","customizable":"0","text_fields":"0","uploadable_files":"0","active":"1","redirect_type":"301-category","id_type_redirected":"0","available_for_order":"1","available_date":"0000-00-00","show_condition":"0","condition":"new","show_price":"1","indexed":"1","visibility":"both","advanced_stock_management":"0","date_add":"2022-05-17 17:50:34","date_upd":"2022-05-17 17:50:34","pack_stock_type":"3","meta_description":"","meta_keywords":"","meta_title":"","link_rewrite":"hummingbird-printed-t-shirt","name":"Hummingbird printed t-shirt","description":"

    Symbol of lightness and delicacy, the hummingbird evokes curiosity and joy. Studio Design' PolyFaune collection features classic products with colorful patterns, inspired by the traditional japanese origamis. To wear with a chino or jeans. The sublimation textile printing process provides an exceptional color rendering and a color, guaranteed overtime.

    ","description_short":"

    Regular fit, round neckline, short sleeves. Made of extra long staple pima cotton.

    \r\n

    ","available_now":"","available_later":"","associations":{"categories":[{"id":"2"},{"id":"3"},{"id":"4"}],"images":[{"id":"1"},{"id":"2"}],"combinations":[{"id":"2"},{"id":"3"},{"id":"4"},{"id":"5"},{"id":"6"},{"id":"7"},{"id":"8"},{"id":"1"}],"product_option_values":[{"id":"1"},{"id":"11"},{"id":"2"},{"id":"8"},{"id":"3"},{"id":"4"}],"product_features":[{"id":"1","id_feature_value":"4"},{"id":"2","id_feature_value":"8"}],"stock_availables":[{"id":"1","id_product_attribute":"0"},{"id":"20","id_product_attribute":"1"},{"id":"21","id_product_attribute":"2"},{"id":"22","id_product_attribute":"3"},{"id":"23","id_product_attribute":"4"},{"id":"24","id_product_attribute":"5"},{"id":"25","id_product_attribute":"6"},{"id":"26","id_product_attribute":"7"},{"id":"27","id_product_attribute":"8"}]}},"emitted_at":1687800727800} +{"stream":"shop_groups","data":{"id":1,"name":"Default","color":"","share_customer":"0","share_order":"0","share_stock":"0","active":"1","deleted":"0"},"emitted_at":1687800729226} +{"stream":"shop_urls","data":{"id":1,"id_shop":"1","active":"1","main":"1","domain":"integration-test.club","domain_ssl":"integration-test.club","physical_uri":"/","virtual_uri":""},"emitted_at":1687800729961} +{"stream":"shops","data":{"id":1,"id_shop_group":"1","id_category":"2","active":"1","deleted":"0","name":"My shop","color":"","theme_name":"classic"},"emitted_at":1687800730700} +{"stream":"specific_price_rules","data":{"id":1,"id_shop":"1","id_country":"0","id_currency":"1","id_group":"3","name":"5+1","from_quantity":"5","price":"-1.000000","reduction":"0.250000","reduction_tax":"0","reduction_type":"percentage","from":"0000-00-00 00:00:00","to":"0000-00-00 00:00:00"},"emitted_at":1687811472883} +{"stream":"specific_prices","data":{"id":1,"id_shop_group":"0","id_shop":"0","id_cart":"0","id_product":"1","id_product_attribute":"0","id_currency":"0","id_country":"0","id_group":"0","id_customer":"0","id_specific_price_rule":"0","price":"-1.000000","from_quantity":"1","reduction":"0.200000","reduction_tax":"1","reduction_type":"percentage","from":"0000-00-00 00:00:00","to":"0000-00-00 00:00:00"},"emitted_at":1687800732322} +{"stream":"states","data":{"id":89,"id_zone":"2","id_country":"4","iso_code":"ON","name":"Ontario","active":"1"},"emitted_at":1687800733101} +{"stream":"stock_availables","data":{"id":1,"id_product":"1","id_product_attribute":"0","id_shop":"1","id_shop_group":"0","quantity":"2410","depends_on_stock":"0","out_of_stock":"2","location":""},"emitted_at":1687875433731} +{"stream":"stock_movement_reasons","data":{"id":1,"sign":"1","deleted":"0","date_add":"2022-05-17 17:48:55","date_upd":"2022-05-17 17:48:55","name":"Increase"},"emitted_at":1687800737473} +{"stream":"stock_movements","data":{"id":1,"id_product":"","id_product_attribute":"","id_warehouse":"","id_currency":"","management_type":null,"id_employee":"2","id_stock":"20","id_stock_mvt_reason":"11","id_order":"","id_supply_order":"","product_name":false,"ean13":null,"upc":null,"reference":null,"mpn":null,"physical_quantity":"10","sign":"1","last_wa":"0.000000","current_wa":"0.000000","price_te":"0.000000","date_add":"2023-06-26 20:13:19"},"emitted_at":1687811474230} +{"stream":"stores","data":{"id":1,"id_country":"21","id_state":"12","hours":" [[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"09:00AM - 07:00PM\"],[\"10:00AM - 04:00PM\"],[\"10:00AM - 04:00PM\"]]","postcode":"33135","city":"Miami","latitude":"25.76500500","longitude":"-80.24379700","phone":"","fax":"","email":"","active":"1","date_add":"2022-05-17 17:50:34","date_upd":"2022-05-17 17:50:34","name":"Dade County","address1":"3030 SW 8th St Miami","address2":"","note":""},"emitted_at":1687800740284} +{"stream":"suppliers","data":{"id":1,"link_rewrite":"fashion-supplier","name":"Fashion supplier","active":"1","date_add":"2022-05-17 17:50:33","date_upd":"2022-05-17 17:50:33","description":"","meta_title":"","meta_description":"","meta_keywords":""},"emitted_at":1687800741658} +{"stream":"tags","data":{"id":1,"id_lang":"1","name":"test_tag"},"emitted_at":1687811475012} +{"stream":"tax_rule_groups","data":{"id":1,"name":"US-AL Rate (4%)","active":"1","deleted":"0","date_add":"2022-05-17 17:48:57","date_upd":"2022-05-17 17:48:57"},"emitted_at":1687800743831} +{"stream":"tax_rules","data":{"id":1,"id_tax_rules_group":"1","id_state":"4","id_country":"21","zipcode_from":"0","zipcode_to":"0","id_tax":"1","behavior":"1","description":""},"emitted_at":1687800745586} +{"stream":"taxes","data":{"id":1,"rate":"4.000","active":"1","deleted":"0","name":"Sales-taxes US-AL 4%"},"emitted_at":1687800746697} +{"stream":"translated_configurations","data":{"id":38,"value":"#IN","date_add":null,"date_upd":null,"name":"PS_INVOICE_PREFIX","id_shop_group":"","id_shop":""},"emitted_at":1687800747833} +{"stream":"weight_ranges","data":{"id":1,"id_carrier":"2","delimiter1":"0.000000","delimiter2":"10000.000000"},"emitted_at":1687800748625} +{"stream":"zones","data":{"id":1,"name":"Europe","active":"1"},"emitted_at":1687800749386} diff --git a/airbyte-integrations/connectors/source-prestashop/metadata.yaml b/airbyte-integrations/connectors/source-prestashop/metadata.yaml index 6aa1cc2ff4d7..a3a5215f6727 100644 --- a/airbyte-integrations/connectors/source-prestashop/metadata.yaml +++ b/airbyte-integrations/connectors/source-prestashop/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: d60a46d4-709f-4092-a6b7-2457f7d455f5 - dockerImageTag: 0.3.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-prestashop githubIssueLabel: source-prestashop icon: prestashop.svg @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-prestashop/requirements.txt b/airbyte-integrations/connectors/source-prestashop/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-prestashop/requirements.txt +++ b/airbyte-integrations/connectors/source-prestashop/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-prestashop/setup.py b/airbyte-integrations/connectors/source-prestashop/setup.py index c35790e1431a..4762aa8a2eaf 100644 --- a/airbyte-integrations/connectors/source-prestashop/setup.py +++ b/airbyte-integrations/connectors/source-prestashop/setup.py @@ -10,8 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py b/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py new file mode 100644 index 000000000000..0175750fc814 --- /dev/null +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/components.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Optional, Tuple + +import pendulum +from airbyte_cdk.sources.declarative.schema import JsonFileSchemaLoader +from airbyte_cdk.sources.declarative.transformations import RecordTransformation +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from pendulum.parsing.exceptions import ParserError + + +@dataclass +class CustomFieldTransformation(RecordTransformation): + """ + Remove all "empty" (e.g. '0000-00-00', '0000-00-00 00:00:00') 'date' and 'date-time' fields from record + """ + + config: Config + parameters: InitVar[Mapping[str, Any]] + + def __post_init__(self, parameters: Mapping[str, Any]): + self.name = parameters.get("name") + self._schema = self._get_schema_root_properties() + self._date_and_date_time_fields = self._get_fields_with_property_formats_from_schema(("date", "date-time")) + + def _get_schema_root_properties(self): + schema_loader = JsonFileSchemaLoader(config=self.config, parameters={"name": self.name}) + schema = schema_loader.get_json_schema() + return schema["properties"] + + def _get_fields_with_property_formats_from_schema(self, property_formats: Tuple[str, ...]) -> List[str]: + """ + Get all properties from schema within property_formats + """ + return [k for k, v in self._schema.items() if v.get("format") in property_formats] + + def transform( + self, + record: Record, + config: Optional[Config] = None, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Record: + for item in record: + if item in self._date_and_date_time_fields and record.get(item): + try: + pendulum.parse(record[item]) + except ParserError: + record[item] = None + return record diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/manifest.yaml b/airbyte-integrations/connectors/source-prestashop/source_prestashop/manifest.yaml index de997da899d8..0a05ae1c4eaf 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/manifest.yaml +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/manifest.yaml @@ -28,6 +28,9 @@ definitions: base_stream: retriever: $ref: "#/definitions/retriever" + transformations: + - type: CustomTransformation + class_name: source_prestashop.components.CustomFieldTransformation base_incremental_stream: $ref: "#/definitions/base_stream" incremental_sync: diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/addresses.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/addresses.json index d3ca5954dec6..f56af2120f95 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/addresses.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/addresses.json @@ -1,9 +1,9 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_customer": { "type": ["null", "string"] @@ -18,28 +18,28 @@ "type": ["null", "string"] }, "id_country": { - "type": "string" + "type": ["null", "string"] }, "id_state": { "type": ["null", "string"] }, "alias": { - "type": "string" + "type": ["null", "string"] }, "company": { "type": ["null", "string"] }, "lastname": { - "type": "string" + "type": ["null", "string"] }, "firstname": { - "type": "string" + "type": ["null", "string"] }, "vat_number": { "type": ["null", "string"] }, "address1": { - "type": "string" + "type": ["null", "string"] }, "address2": { "type": ["null", "string"] @@ -48,7 +48,7 @@ "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "other": { "type": ["null", "string"] @@ -63,14 +63,14 @@ "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carriers.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carriers.json index 94c266d71b38..775a5276726d 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carriers.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carriers.json @@ -1,72 +1,72 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "is_module": { - "type": "string" + "type": ["null", "string"] }, "id_tax_rules_group": { - "type": "string" + "type": ["null", "string"] }, "id_reference": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "is_free": { - "type": "string" + "type": ["null", "string"] }, "url": { - "type": "string" + "type": ["null", "string"] }, "shipping_handling": { - "type": "string" + "type": ["null", "string"] }, "shipping_external": { - "type": "string" + "type": ["null", "string"] }, "range_behavior": { - "type": "string" + "type": ["null", "string"] }, "shipping_method": { "type": ["string", "integer"] }, "max_width": { - "type": "string" + "type": ["null", "string"] }, "max_height": { - "type": "string" + "type": ["null", "string"] }, "max_depth": { - "type": "string" + "type": ["null", "string"] }, "max_weight": { - "type": "string" + "type": ["null", "string"] }, "grade": { - "type": "string" + "type": ["null", "string"] }, "external_module_name": { - "type": "string" + "type": ["null", "string"] }, "need_range": { - "type": "string" + "type": ["null", "string"] }, "position": { - "type": "string" + "type": ["null", "string"] }, "delay": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/cart_rules.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/cart_rules.json index 29f48f8d868f..7047ed41cf65 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/cart_rules.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/cart_rules.json @@ -1,112 +1,112 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "date_from": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_to": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "description": { - "type": "string" + "type": ["null", "string"] }, "quantity": { - "type": "string" + "type": ["null", "string"] }, "quantity_per_user": { - "type": "string" + "type": ["null", "string"] }, "priority": { - "type": "string" + "type": ["null", "string"] }, "partial_use": { - "type": "string" + "type": ["null", "string"] }, "code": { - "type": "string" + "type": ["null", "string"] }, "minimum_amount": { - "type": "string" + "type": ["null", "string"] }, "minimum_amount_tax": { - "type": "string" + "type": ["null", "string"] }, "minimum_amount_currency": { - "type": "string" + "type": ["null", "string"] }, "minimum_amount_shipping": { - "type": "string" + "type": ["null", "string"] }, "country_restriction": { - "type": "string" + "type": ["null", "string"] }, "carrier_restriction": { - "type": "string" + "type": ["null", "string"] }, "group_restriction": { - "type": "string" + "type": ["null", "string"] }, "cart_rule_restriction": { - "type": "string" + "type": ["null", "string"] }, "product_restriction": { - "type": "string" + "type": ["null", "string"] }, "shop_restriction": { - "type": "string" + "type": ["null", "string"] }, "free_shipping": { - "type": "string" + "type": ["null", "string"] }, "reduction_percent": { - "type": "string" + "type": ["null", "string"] }, "reduction_amount": { - "type": "string" + "type": ["null", "string"] }, "reduction_tax": { - "type": "string" + "type": ["null", "string"] }, "reduction_currency": { - "type": "string" + "type": ["null", "string"] }, "reduction_product": { - "type": "string" + "type": ["null", "string"] }, "reduction_exclude_special": { - "type": "string" + "type": ["null", "string"] }, "gift_product": { - "type": "string" + "type": ["null", "string"] }, "gift_product_attribute": { - "type": "string" + "type": ["null", "string"] }, "highlight": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carts.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carts.json index de20450a1c8a..063c778fa50e 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carts.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/carts.json @@ -1,64 +1,64 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_address_delivery": { - "type": "string" + "type": ["null", "string"] }, "id_address_invoice": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_guest": { - "type": "string" + "type": ["null", "string"] }, "id_lang": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_carrier": { - "type": "string" + "type": ["null", "string"] }, "recyclable": { - "type": "string" + "type": ["null", "string"] }, "gift": { - "type": "string" + "type": ["null", "string"] }, "gift_message": { - "type": "string" + "type": ["null", "string"] }, "mobile_theme": { - "type": "string" + "type": ["null", "string"] }, "delivery_option": { - "type": "string" + "type": ["null", "string"] }, "secure_key": { - "type": "string" + "type": ["null", "string"] }, "allow_seperated_package": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "associations": { @@ -70,19 +70,19 @@ "type": "object", "properties": { "id_product": { - "type": "string" + "type": ["null", "string"] }, "id_product_attribute": { - "type": "string" + "type": ["null", "string"] }, "id_address_delivery": { - "type": "string" + "type": ["null", "string"] }, "id_customization": { - "type": "string" + "type": ["null", "string"] }, "quantity": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/categories.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/categories.json index 98f3a643df65..393bb770bb96 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/categories.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/categories.json @@ -1,56 +1,56 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_parent": { - "type": "string" + "type": ["null", "string"] }, "level_depth": { - "type": "string" + "type": ["null", "string"] }, "nb_products_recursive": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "id_shop_default": { - "type": "string" + "type": ["null", "string"] }, "is_root_category": { - "type": "string" + "type": ["null", "string"] }, "position": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "name": { - "type": "string" + "type": ["null", "string"] }, "link_rewrite": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "meta_title": { - "type": "string" + "type": ["null", "string"] }, "meta_description": { - "type": "string" + "type": ["null", "string"] }, "meta_keywords": { - "type": "string" + "type": ["null", "string"] }, "associations": { "type": "object", @@ -61,7 +61,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/combinations.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/combinations.json index a51729d46244..804f9722a19f 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/combinations.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/combinations.json @@ -1,60 +1,60 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "location": { - "type": "string" + "type": ["null", "string"] }, "ean13": { - "type": "string" + "type": ["null", "string"] }, "isbn": { - "type": "string" + "type": ["null", "string"] }, "upc": { - "type": "string" + "type": ["null", "string"] }, "mpn": { - "type": "string" + "type": ["null", "string"] }, "quantity": { - "type": "string" + "type": ["null", "integer"] }, "reference": { - "type": "string" + "type": ["null", "string"] }, "supplier_reference": { - "type": "string" + "type": ["null", "string"] }, "wholesale_price": { - "type": "string" + "type": ["null", "string"] }, "price": { - "type": "string" + "type": ["null", "string"] }, "ecotax": { - "type": "string" + "type": ["null", "string"] }, "weight": { - "type": "string" + "type": ["null", "string"] }, "unit_price_impact": { - "type": "string" + "type": ["null", "string"] }, "minimal_quantity": { - "type": "string" + "type": ["null", "string"] }, "low_stock_threshold": { "type": ["null", "string"] }, "low_stock_alert": { - "type": "string" + "type": ["null", "string"] }, "default_on": { "type": ["null", "string"] @@ -72,7 +72,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } @@ -83,7 +83,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/configurations.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/configurations.json index d567cecdfdb1..3247ae08f27f 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/configurations.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/configurations.json @@ -1,15 +1,15 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "value": { "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { "type": ["null", "string"] @@ -18,11 +18,11 @@ "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/contacts.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/contacts.json index 953b43006e6e..2c32aa05b550 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/contacts.json @@ -1,21 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "customer_service": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/content_management_system.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/content_management_system.json index c035dfd274e2..ce69250f4379 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/content_management_system.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/content_management_system.json @@ -1,39 +1,39 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_cms_category": { - "type": "string" + "type": ["null", "string"] }, "position": { - "type": "string" + "type": ["null", "string"] }, "indexation": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "meta_description": { - "type": "string" + "type": ["null", "string"] }, "meta_keywords": { - "type": "string" + "type": ["null", "string"] }, "meta_title": { - "type": "string" + "type": ["null", "string"] }, "head_seo_title": { - "type": "string" + "type": ["null", "string"] }, "link_rewrite": { - "type": "string" + "type": ["null", "string"] }, "content": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/countries.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/countries.json index 29506878f493..31f49b5efad3 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/countries.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/countries.json @@ -1,42 +1,42 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_zone": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "call_prefix": { - "type": "string" + "type": ["null", "string"] }, "iso_code": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "contains_states": { - "type": "string" + "type": ["null", "string"] }, "need_identification_number": { - "type": "string" + "type": ["null", "string"] }, "need_zip_code": { - "type": "string" + "type": ["null", "string"] }, "zip_code_format": { - "type": "string" + "type": ["null", "string"] }, "display_tax_label": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/currencies.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/currencies.json index d055402fd576..b5254604887d 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/currencies.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/currencies.json @@ -1,45 +1,45 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "names": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "symbol": { - "type": "string" + "type": ["null", "string"] }, "iso_code": { - "type": "string" + "type": ["null", "string"] }, "numeric_iso_code": { - "type": "string" + "type": ["null", "string"] }, "precision": { - "type": "string" + "type": ["null", "string"] }, "conversion_rate": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "unofficial": { - "type": "string" + "type": ["null", "string"] }, "modified": { - "type": "string" + "type": ["null", "string"] }, "pattern": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_messages.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_messages.json index be6d7656a76e..2170b4fa55c8 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_messages.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_messages.json @@ -1,21 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_employee": { - "type": "string" + "type": ["null", "string"] }, "id_customer_thread": { - "type": "string" + "type": ["null", "string"] }, "ip_address": { "type": ["null", "string"] }, "message": { - "type": "string" + "type": ["null", "string"] }, "file_name": { "type": ["null", "string"] @@ -27,11 +27,11 @@ "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "read": { diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_threads.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_threads.json index 8cd2eafdf9b7..867c484bd0e1 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_threads.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customer_threads.json @@ -1,43 +1,43 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_lang": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "id_contact": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "token": { "type": ["null", "string"] }, "status": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "associations": { @@ -49,7 +49,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customers.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customers.json index d94aad307f81..c5dd56155c6b 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customers.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/customers.json @@ -1,42 +1,42 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_default_group": { - "type": "string" + "type": ["null", "string"] }, "id_lang": { - "type": "string" + "type": ["null", "string"] }, "newsletter_date_add": { - "type": "string" + "type": ["null", "string"] }, "ip_registration_newsletter": { "type": ["null", "string"] }, "last_passwd_gen": { - "type": "string" + "type": ["null", "string"] }, "secure_key": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "passwd": { - "type": "string" + "type": ["null", "string"] }, "lastname": { - "type": "string" + "type": ["null", "string"] }, "firstname": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "id_gender": { "type": ["null", "string"] @@ -63,38 +63,38 @@ "type": ["null", "string"] }, "outstanding_allow_amount": { - "type": "string" + "type": ["null", "string"] }, "show_public_prices": { - "type": "string" + "type": ["null", "string"] }, "id_risk": { - "type": "string" + "type": ["null", "string"] }, "max_payment_days": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "note": { - "type": "string" + "type": ["null", "string"] }, "is_guest": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "reset_password_token": { @@ -113,7 +113,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/deliveries.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/deliveries.json index d7bbd0436f7b..d1cb5f1066ca 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/deliveries.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/deliveries.json @@ -1,30 +1,30 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_carrier": { - "type": "string" + "type": ["null", "string"] }, "id_range_price": { - "type": "string" + "type": ["null", "string"] }, "id_range_weight": { - "type": "string" + "type": ["null", "string"] }, "id_zone": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "price": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/employees.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/employees.json index 862e09e825ed..9236848c947a 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/employees.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/employees.json @@ -1,15 +1,15 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_lang": { - "type": "string" + "type": ["null", "string"] }, "last_passwd_gen": { - "type": "string" + "type": ["null", "string"] }, "stats_date_from": { "type": ["null", "string"], @@ -28,28 +28,28 @@ "format": "date" }, "passwd": { - "type": "string" + "type": ["null", "string"] }, "lastname": { - "type": "string" + "type": ["null", "string"] }, "firstname": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "id_profile": { - "type": "string" + "type": ["null", "string"] }, "bo_color": { "type": ["null", "string"] }, "default_tab": { - "type": "string" + "type": ["null", "string"] }, "bo_theme": { "type": ["null", "string"] @@ -58,25 +58,25 @@ "type": ["null", "string"] }, "bo_width": { - "type": "string" + "type": ["null", "string"] }, "bo_menu": { - "type": "string" + "type": ["null", "string"] }, "stats_compare_option": { - "type": "string" + "type": ["null", "string"] }, "preselect_date_range": { "type": ["null", "string"] }, "id_last_order": { - "type": "string" + "type": ["null", "string"] }, "id_last_customer_message": { - "type": "string" + "type": ["null", "string"] }, "id_last_customer": { - "type": "string" + "type": ["null", "string"] }, "reset_password_token": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/groups.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/groups.json index 23dc310ca431..82038c262486 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/groups.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/groups.json @@ -1,29 +1,29 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "reduction": { - "type": "string" + "type": ["null", "string"] }, "price_display_method": { - "type": "string" + "type": ["null", "string"] }, "show_prices": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/guests.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/guests.json index 6b3fe54e8922..08bdfc747914 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/guests.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/guests.json @@ -1,54 +1,54 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_operating_system": { - "type": "string" + "type": ["null", "string"] }, "id_web_browser": { - "type": "string" + "type": ["null", "string"] }, "javascript": { - "type": "string" + "type": ["null", "string"] }, "screen_resolution_x": { - "type": "string" + "type": ["null", "string"] }, "screen_resolution_y": { - "type": "string" + "type": ["null", "string"] }, "screen_color": { - "type": "string" + "type": ["null", "string"] }, "sun_java": { - "type": "string" + "type": ["null", "string"] }, "adobe_flash": { - "type": "string" + "type": ["null", "string"] }, "adobe_director": { - "type": "string" + "type": ["null", "string"] }, "apple_quicktime": { - "type": "string" + "type": ["null", "string"] }, "real_player": { - "type": "string" + "type": ["null", "string"] }, "windows_media": { - "type": "string" + "type": ["null", "string"] }, "accept_language": { - "type": "string" + "type": ["null", "string"] }, "mobile_theme": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/image_types.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/image_types.json index 37b3e69eb181..1cc0f02ddee6 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/image_types.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/image_types.json @@ -1,33 +1,33 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "width": { - "type": "string" + "type": ["null", "string"] }, "height": { - "type": "string" + "type": ["null", "string"] }, "categories": { - "type": "string" + "type": ["null", "string"] }, "products": { - "type": "string" + "type": ["null", "string"] }, "manufacturers": { - "type": "string" + "type": ["null", "string"] }, "suppliers": { - "type": "string" + "type": ["null", "string"] }, "stores": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/languages.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/languages.json index 4dfa3461c288..67d4d4ecdedc 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/languages.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/languages.json @@ -1,33 +1,33 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "iso_code": { - "type": "string" + "type": ["null", "string"] }, "locale": { - "type": "string" + "type": ["null", "string"] }, "language_code": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "is_rtl": { - "type": "string" + "type": ["null", "string"] }, "date_format_lite": { - "type": "string" + "type": ["null", "string"] }, "date_format_full": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/manufacturers.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/manufacturers.json index 53329b0784be..6dec616d7ad5 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/manufacturers.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/manufacturers.json @@ -1,41 +1,41 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "link_rewrite": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "description": { - "type": "string" + "type": ["null", "string"] }, "short_description": { - "type": "string" + "type": ["null", "string"] }, "meta_title": { - "type": "string" + "type": ["null", "string"] }, "meta_description": { - "type": "string" + "type": ["null", "string"] }, "meta_keywords": { - "type": "string" + "type": ["null", "string"] }, "associations": { "type": "object", @@ -46,7 +46,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/messages.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/messages.json index 4ded1eb64073..e8128388d81e 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/messages.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/messages.json @@ -1,30 +1,30 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_cart": { - "type": "string" + "type": ["null", "string"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_employee": { - "type": "string" + "type": ["null", "string"] }, "message": { - "type": "string" + "type": ["null", "string"] }, "private": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_carriers.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_carriers.json index fb24f5aadcce..7d3f4d93d06e 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_carriers.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_carriers.json @@ -1,33 +1,33 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "id_carrier": { - "type": "string" + "type": ["null", "string"] }, "id_order_invoice": { - "type": "string" + "type": ["null", "string"] }, "weight": { - "type": "string" + "type": ["null", "string"] }, "shipping_cost_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "shipping_cost_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "tracking_number": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_details.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_details.json index f33f904016e7..9e64c68e1bac 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_details.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_details.json @@ -1,147 +1,147 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "product_id": { - "type": "string" + "type": ["null", "string"] }, "product_attribute_id": { - "type": "string" + "type": ["null", "string"] }, "product_quantity_reinjected": { - "type": "string" + "type": ["null", "string"] }, "group_reduction": { - "type": "string" + "type": ["null", "string"] }, "discount_quantity_applied": { - "type": "string" + "type": ["null", "string"] }, "download_hash": { - "type": "string" + "type": ["null", "string"] }, "download_deadline": { - "type": "string" + "type": ["null", "string"] }, "id_order_invoice": { - "type": "string" + "type": ["null", "string"] }, "id_warehouse": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_customization": { - "type": "string" + "type": ["null", "string"] }, "product_name": { - "type": "string" + "type": ["null", "string"] }, "product_quantity": { - "type": "string" + "type": ["null", "string"] }, "product_quantity_in_stock": { - "type": "string" + "type": ["null", "string"] }, "product_quantity_return": { - "type": "string" + "type": ["null", "string"] }, "product_quantity_refunded": { - "type": "string" + "type": ["null", "string"] }, "product_price": { - "type": "string" + "type": ["null", "string"] }, "reduction_percent": { - "type": "string" + "type": ["null", "string"] }, "reduction_amount": { - "type": "string" + "type": ["null", "string"] }, "reduction_amount_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "reduction_amount_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "product_quantity_discount": { - "type": "string" + "type": ["null", "string"] }, "product_ean13": { - "type": "string" + "type": ["null", "string"] }, "product_isbn": { - "type": "string" + "type": ["null", "string"] }, "product_upc": { - "type": "string" + "type": ["null", "string"] }, "product_mpn": { - "type": "string" + "type": ["null", "string"] }, "product_reference": { - "type": "string" + "type": ["null", "string"] }, "product_supplier_reference": { - "type": "string" + "type": ["null", "string"] }, "product_weight": { - "type": "string" + "type": ["null", "string"] }, "tax_computation_method": { - "type": "string" + "type": ["null", "string"] }, "id_tax_rules_group": { - "type": "string" + "type": ["null", "string"] }, "ecotax": { - "type": "string" + "type": ["null", "string"] }, "ecotax_tax_rate": { - "type": "string" + "type": ["null", "string"] }, "download_nb": { - "type": "string" + "type": ["null", "string"] }, "unit_price_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "unit_price_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_price_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_price_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_price_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_price_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "purchase_supplier_price": { - "type": "string" + "type": ["null", "string"] }, "original_product_price": { - "type": "string" + "type": ["null", "string"] }, "original_wholesale_price": { - "type": "string" + "type": ["null", "string"] }, "total_refunded_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_refunded_tax_incl": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_histories.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_histories.json index 91e6f65c9344..618bc6421d91 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_histories.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_histories.json @@ -1,21 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_employee": { - "type": "string" + "type": ["null", "string"] }, "id_order_state": { - "type": "string" + "type": ["null", "string"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_invoices.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_invoices.json index 4b41ece21556..2488aa19a4da 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_invoices.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_invoices.json @@ -1,63 +1,63 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "number": { - "type": "string" + "type": ["null", "string"] }, "delivery_number": { - "type": "string" + "type": ["null", "string"] }, "delivery_date": { - "type": "string" + "type": ["null", "string"] }, "total_discount_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_discount_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_paid_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_paid_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_products": { - "type": "string" + "type": ["null", "string"] }, "total_products_wt": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "shipping_tax_computation_method": { - "type": "string" + "type": ["null", "string"] }, "total_wrapping_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_wrapping_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "shop_address": { - "type": "string" + "type": ["null", "string"] }, "note": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_payments.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_payments.json index 1bd81deece57..620712029e36 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_payments.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_payments.json @@ -1,42 +1,42 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "order_reference": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "amount": { - "type": "string" + "type": ["null", "string"] }, "payment_method": { - "type": "string" + "type": ["null", "string"] }, "conversion_rate": { - "type": "string" + "type": ["null", "string"] }, "transaction_id": { - "type": "string" + "type": ["null", "string"] }, "card_number": { - "type": "string" + "type": ["null", "string"] }, "card_brand": { - "type": "string" + "type": ["null", "string"] }, "card_expiration": { - "type": "string" + "type": ["null", "string"] }, "card_holder": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_slip.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_slip.json index c40763e4ee27..db584ab7c9cb 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_slip.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_slip.json @@ -1,53 +1,53 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "conversion_rate": { - "type": "string" + "type": ["null", "string"] }, "total_products_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_products_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "amount": { - "type": "string" + "type": ["null", "string"] }, "shipping_cost": { - "type": "string" + "type": ["null", "string"] }, "shipping_cost_amount": { - "type": "string" + "type": ["null", "string"] }, "partial": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "order_slip_type": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_states.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_states.json index 9cb375f1fa3c..76c58f558314 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_states.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/order_states.json @@ -1,54 +1,54 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "unremovable": { - "type": "string" + "type": ["null", "string"] }, "delivery": { - "type": "string" + "type": ["null", "string"] }, "hidden": { - "type": "string" + "type": ["null", "string"] }, "send_email": { - "type": "string" + "type": ["null", "string"] }, "module_name": { - "type": "string" + "type": ["null", "string"] }, "invoice": { - "type": "string" + "type": ["null", "string"] }, "color": { - "type": "string" + "type": ["null", "string"] }, "logable": { - "type": "string" + "type": ["null", "string"] }, "shipped": { - "type": "string" + "type": ["null", "string"] }, "paid": { - "type": "string" + "type": ["null", "string"] }, "pdf_delivery": { - "type": "string" + "type": ["null", "string"] }, "pdf_invoice": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "template": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/orders.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/orders.json index 4d3e4fca254d..80eec3391fb5 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/orders.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/orders.json @@ -1,146 +1,146 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_address_delivery": { - "type": "string" + "type": ["null", "string"] }, "id_address_invoice": { - "type": "string" + "type": ["null", "string"] }, "id_cart": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "id_lang": { - "type": "string" + "type": ["null", "string"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_carrier": { - "type": "string" + "type": ["null", "string"] }, "current_state": { - "type": "string" + "type": ["null", "string"] }, "module": { - "type": "string" + "type": ["null", "string"] }, "invoice_number": { - "type": "string" + "type": ["null", "string"] }, "invoice_date": { - "type": "string" + "type": ["null", "string"] }, "delivery_number": { - "type": "string" + "type": ["null", "string"] }, "delivery_date": { - "type": "string" + "type": ["null", "string"] }, "valid": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "shipping_number": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "secure_key": { - "type": "string" + "type": ["null", "string"] }, "payment": { - "type": "string" + "type": ["null", "string"] }, "recyclable": { - "type": "string" + "type": ["null", "string"] }, "gift": { - "type": "string" + "type": ["null", "string"] }, "gift_message": { - "type": "string" + "type": ["null", "string"] }, "mobile_theme": { - "type": "string" + "type": ["null", "string"] }, "total_discounts": { - "type": "string" + "type": ["null", "string"] }, "total_discounts_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_discounts_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_paid": { - "type": "string" + "type": ["null", "string"] }, "total_paid_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_paid_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "total_paid_real": { - "type": "string" + "type": ["null", "string"] }, "total_products": { - "type": "string" + "type": ["null", "string"] }, "total_products_wt": { - "type": "string" + "type": ["null", "string"] }, "total_shipping": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_shipping_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "carrier_tax_rate": { - "type": "string" + "type": ["null", "string"] }, "total_wrapping": { - "type": "string" + "type": ["null", "string"] }, "total_wrapping_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "total_wrapping_tax_excl": { - "type": "string" + "type": ["null", "string"] }, "round_mode": { - "type": "string" + "type": ["null", "string"] }, "round_type": { - "type": "string" + "type": ["null", "string"] }, "conversion_rate": { - "type": "string" + "type": ["null", "string"] }, "reference": { - "type": "string" + "type": ["null", "string"] }, "associations": { "type": "object", @@ -151,43 +151,43 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "product_id": { - "type": "string" + "type": ["null", "string"] }, "product_attribute_id": { - "type": "string" + "type": ["null", "string"] }, "product_quantity": { - "type": "string" + "type": ["null", "string"] }, "product_name": { - "type": "string" + "type": ["null", "string"] }, "product_reference": { - "type": "string" + "type": ["null", "string"] }, "product_ean13": { - "type": "string" + "type": ["null", "string"] }, "product_isbn": { - "type": "string" + "type": ["null", "string"] }, "product_upc": { - "type": "string" + "type": ["null", "string"] }, "product_price": { - "type": "string" + "type": ["null", "string"] }, "id_customization": { - "type": "string" + "type": ["null", "string"] }, "unit_price_tax_incl": { - "type": "string" + "type": ["null", "string"] }, "unit_price_tax_excl": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/price_ranges.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/price_ranges.json index 3a2218193d83..8678e7d0993c 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/price_ranges.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/price_ranges.json @@ -1,18 +1,18 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_carrier": { - "type": "string" + "type": ["null", "string"] }, "delimiter1": { - "type": "string" + "type": ["null", "string"] }, "delimiter2": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_customization_fields.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_customization_fields.json index 1274f21ade71..6677da16450e 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_customization_fields.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_customization_fields.json @@ -1,27 +1,27 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "required": { - "type": "string" + "type": ["null", "string"] }, "is_module": { - "type": "string" + "type": ["null", "string"] }, "is_deleted": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_feature_values.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_feature_values.json index d0e1e675b1e3..1a5373434110 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_feature_values.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_feature_values.json @@ -1,18 +1,18 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_feature": { - "type": "string" + "type": ["null", "string"] }, "custom": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_features.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_features.json index 2384936540a6..2e593b3d7cd7 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_features.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_features.json @@ -1,15 +1,15 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "position": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_option_values.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_option_values.json index 98abee145442..614bcf8696dc 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_option_values.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_option_values.json @@ -1,21 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_attribute_group": { - "type": "string" + "type": ["null", "string"] }, "color": { - "type": "string" + "type": ["null", "string"] }, "position": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_options.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_options.json index f99b6adf7d2b..ea23156b61a5 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_options.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_options.json @@ -1,24 +1,24 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "is_color_group": { - "type": "string" + "type": ["null", "string"] }, "group_type": { - "type": "string" + "type": ["null", "string"] }, "position": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "public_name": { - "type": "string" + "type": ["null", "string"] }, "associations": { "type": "object", @@ -29,7 +29,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_suppliers.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_suppliers.json index 578852971575..5835bd26811a 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_suppliers.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/product_suppliers.json @@ -1,27 +1,27 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "id_product_attribute": { - "type": "string" + "type": ["null", "string"] }, "id_supplier": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "product_supplier_reference": { - "type": "string" + "type": ["null", "string"] }, "product_supplier_price_te": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/products.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/products.json index 6db529bb0ef3..31b792ab97ed 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/products.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/products.json @@ -1,218 +1,218 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_manufacturer": { - "type": "string" + "type": ["null", "string"] }, "id_supplier": { - "type": "string" + "type": ["null", "string"] }, "id_category_default": { - "type": "string" + "type": ["null", "string"] }, "new": { "type": ["null", "string"] }, "cache_default_attribute": { - "type": "string" + "type": ["null", "string"] }, "id_default_image": { - "type": "string" + "type": ["null", "string"] }, "id_default_combination": { "type": ["integer", "string"] }, "id_tax_rules_group": { - "type": "string" + "type": ["null", "string"] }, "position_in_category": { - "type": "string" + "type": ["null", "string"] }, "manufacturer_name": { "type": ["string", "boolean"] }, "quantity": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "id_shop_default": { - "type": "string" + "type": ["null", "string"] }, "reference": { - "type": "string" + "type": ["null", "string"] }, "supplier_reference": { - "type": "string" + "type": ["null", "string"] }, "location": { - "type": "string" + "type": ["null", "string"] }, "width": { - "type": "string" + "type": ["null", "string"] }, "height": { - "type": "string" + "type": ["null", "string"] }, "depth": { - "type": "string" + "type": ["null", "string"] }, "weight": { - "type": "string" + "type": ["null", "string"] }, "quantity_discount": { - "type": "string" + "type": ["null", "string"] }, "ean13": { - "type": "string" + "type": ["null", "string"] }, "isbn": { - "type": "string" + "type": ["null", "string"] }, "upc": { - "type": "string" + "type": ["null", "string"] }, "mpn": { - "type": "string" + "type": ["null", "string"] }, "cache_is_pack": { - "type": "string" + "type": ["null", "string"] }, "cache_has_attachments": { - "type": "string" + "type": ["null", "string"] }, "is_virtual": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "additional_delivery_times": { - "type": "string" + "type": ["null", "string"] }, "delivery_in_stock": { - "type": "string" + "type": ["null", "string"] }, "delivery_out_stock": { - "type": "string" + "type": ["null", "string"] }, "on_sale": { - "type": "string" + "type": ["null", "string"] }, "online_only": { - "type": "string" + "type": ["null", "string"] }, "ecotax": { - "type": "string" + "type": ["null", "string"] }, "minimal_quantity": { - "type": "string" + "type": ["null", "string"] }, "low_stock_threshold": { "type": ["null", "string"] }, "low_stock_alert": { - "type": "string" + "type": ["null", "string"] }, "price": { - "type": "string" + "type": ["null", "string"] }, "wholesale_price": { - "type": "string" + "type": ["null", "string"] }, "unity": { - "type": "string" + "type": ["null", "string"] }, "unit_price_ratio": { - "type": "string" + "type": ["null", "string"] }, "additional_shipping_cost": { - "type": "string" + "type": ["null", "string"] }, "customizable": { - "type": "string" + "type": ["null", "string"] }, "text_fields": { - "type": "string" + "type": ["null", "string"] }, "uploadable_files": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "redirect_type": { - "type": "string" + "type": ["null", "string"] }, "id_type_redirected": { - "type": "string" + "type": ["null", "string"] }, "available_for_order": { - "type": "string" + "type": ["null", "string"] }, "available_date": { - "type": "string" + "type": ["null", "string"] }, "show_condition": { - "type": "string" + "type": ["null", "string"] }, "condition": { - "type": "string" + "type": ["null", "string"] }, "show_price": { - "type": "string" + "type": ["null", "string"] }, "indexed": { - "type": "string" + "type": ["null", "string"] }, "visibility": { - "type": "string" + "type": ["null", "string"] }, "advanced_stock_management": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "pack_stock_type": { - "type": "string" + "type": ["null", "string"] }, "meta_description": { - "type": "string" + "type": ["null", "string"] }, "meta_keywords": { - "type": "string" + "type": ["null", "string"] }, "meta_title": { - "type": "string" + "type": ["null", "string"] }, "link_rewrite": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "description_short": { - "type": "string" + "type": ["null", "string"] }, "available_now": { - "type": "string" + "type": ["null", "string"] }, "available_later": { - "type": "string" + "type": ["null", "string"] }, "associations": { "type": "object", @@ -223,7 +223,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } @@ -234,7 +234,7 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] } } } @@ -245,10 +245,10 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "id_product_attribute": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_groups.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_groups.json index 83dee654ac05..48a12527472e 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_groups.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_groups.json @@ -1,27 +1,27 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "share_customer": { - "type": "string" + "type": ["null", "string"] }, "share_order": { - "type": "string" + "type": ["null", "string"] }, "share_stock": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_urls.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_urls.json index abf1bf374c45..95819404b4d9 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_urls.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shop_urls.json @@ -1,30 +1,30 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "main": { - "type": "string" + "type": ["null", "string"] }, "domain": { - "type": "string" + "type": ["null", "string"] }, "domain_ssl": { - "type": "string" + "type": ["null", "string"] }, "physical_uri": { - "type": "string" + "type": ["null", "string"] }, "virtual_uri": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shops.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shops.json index 17ebc2137c99..3f01b8b2c2ee 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shops.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/shops.json @@ -1,27 +1,27 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "id_category": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "theme_name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_price_rules.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_price_rules.json index 8ca90b6402c8..d5b8faac9fb9 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_price_rules.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_price_rules.json @@ -1,45 +1,45 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_country": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "id_group": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "from_quantity": { - "type": "string" + "type": ["null", "string"] }, "price": { - "type": "string" + "type": ["null", "string"] }, "reduction": { - "type": "string" + "type": ["null", "string"] }, "reduction_tax": { - "type": "string" + "type": ["null", "string"] }, "reduction_type": { - "type": "string" + "type": ["null", "string"] }, "from": { - "type": "string" + "type": ["null", "string"] }, "to": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_prices.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_prices.json index a1c9b4846d06..16d3d237f2e2 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_prices.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/specific_prices.json @@ -1,60 +1,60 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_cart": { - "type": "string" + "type": ["null", "string"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "id_product_attribute": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "id_country": { - "type": "string" + "type": ["null", "string"] }, "id_group": { - "type": "string" + "type": ["null", "string"] }, "id_customer": { - "type": "string" + "type": ["null", "string"] }, "id_specific_price_rule": { - "type": "string" + "type": ["null", "string"] }, "price": { - "type": "string" + "type": ["null", "string"] }, "from_quantity": { - "type": "string" + "type": ["null", "string"] }, "reduction": { - "type": "string" + "type": ["null", "string"] }, "reduction_tax": { - "type": "string" + "type": ["null", "string"] }, "reduction_type": { - "type": "string" + "type": ["null", "string"] }, "from": { - "type": "string" + "type": ["null", "string"] }, "to": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/states.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/states.json index ea084dd4bf33..1b87a21f4edc 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/states.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/states.json @@ -1,24 +1,24 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_zone": { - "type": "string" + "type": ["null", "string"] }, "id_country": { - "type": "string" + "type": ["null", "string"] }, "iso_code": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_availables.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_availables.json index 4fb36818e7ec..aa2e254796d7 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_availables.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_availables.json @@ -1,33 +1,33 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "id_product_attribute": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "quantity": { - "type": "string" + "type": ["null", "string"] }, "depends_on_stock": { - "type": "string" + "type": ["null", "string"] }, "out_of_stock": { - "type": "string" + "type": ["null", "string"] }, "location": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movement_reasons.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movement_reasons.json index 06d2d6b33947..1c05f0e17dcd 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movement_reasons.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movement_reasons.json @@ -1,26 +1,26 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "sign": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movements.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movements.json index 990745054665..7a3a85119b7e 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movements.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stock_movements.json @@ -1,42 +1,42 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_product": { - "type": "string" + "type": ["null", "string"] }, "id_product_attribute": { - "type": "string" + "type": ["null", "string"] }, "id_warehouse": { - "type": "string" + "type": ["null", "string"] }, "id_currency": { - "type": "string" + "type": ["null", "string"] }, "management_type": { "type": ["null", "string"] }, "id_employee": { - "type": "string" + "type": ["null", "string"] }, "id_stock": { - "type": "string" + "type": ["null", "string"] }, "id_stock_mvt_reason": { - "type": "string" + "type": ["null", "string"] }, "id_order": { - "type": "string" + "type": ["null", "string"] }, "id_supply_order": { - "type": "string" + "type": ["null", "string"] }, "product_name": { - "type": "boolean" + "type": ["null", "boolean"] }, "ean13": { "type": ["null", "string"] @@ -51,22 +51,22 @@ "type": ["null", "string"] }, "physical_quantity": { - "type": "string" + "type": ["null", "string"] }, "sign": { - "type": "string" + "type": ["null", "string"] }, "last_wa": { - "type": "string" + "type": ["null", "string"] }, "current_wa": { - "type": "string" + "type": ["null", "string"] }, "price_te": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stores.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stores.json index e37a357e7ded..249872b0590b 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stores.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/stores.json @@ -1,62 +1,62 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_country": { - "type": "string" + "type": ["null", "string"] }, "id_state": { - "type": "string" + "type": ["null", "string"] }, "hours": { - "type": "string" + "type": ["null", "string"] }, "postcode": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "latitude": { - "type": "string" + "type": ["null", "string"] }, "longitude": { - "type": "string" + "type": ["null", "string"] }, "phone": { - "type": "string" + "type": ["null", "string"] }, "fax": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "name": { - "type": "string" + "type": ["null", "string"] }, "address1": { - "type": "string" + "type": ["null", "string"] }, "address2": { - "type": "string" + "type": ["null", "string"] }, "note": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/suppliers.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/suppliers.json index 050229c42dcb..9d41f4ba0063 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/suppliers.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/suppliers.json @@ -1,38 +1,38 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "link_rewrite": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "description": { - "type": "string" + "type": ["null", "string"] }, "meta_title": { - "type": "string" + "type": ["null", "string"] }, "meta_description": { - "type": "string" + "type": ["null", "string"] }, "meta_keywords": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tags.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tags.json index bbb5eb128a85..0a98ab1c632b 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tags.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tags.json @@ -1,15 +1,15 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_lang": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rule_groups.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rule_groups.json index 74e15b0b1213..5edf70feec39 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rule_groups.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rule_groups.json @@ -1,25 +1,25 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { - "type": "string", + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rules.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rules.json index 883021d33d2b..44633a7b404a 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rules.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/tax_rules.json @@ -1,33 +1,33 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_tax_rules_group": { - "type": "string" + "type": ["null", "string"] }, "id_state": { - "type": "string" + "type": ["null", "string"] }, "id_country": { - "type": "string" + "type": ["null", "string"] }, "zipcode_from": { - "type": "string" + "type": ["null", "string"] }, "zipcode_to": { - "type": "string" + "type": ["null", "string"] }, "id_tax": { - "type": "string" + "type": ["null", "string"] }, "behavior": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/taxes.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/taxes.json index 7bbb1eeb4c38..a886902e1fda 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/taxes.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/taxes.json @@ -1,21 +1,21 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "rate": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] }, "deleted": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/translated_configurations.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/translated_configurations.json index 2be2f5ca3ccb..15ee3fbc9bb3 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/translated_configurations.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/translated_configurations.json @@ -1,15 +1,15 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "value": { - "type": "string" + "type": ["null", "string"] }, "date_add": { - "type": "string", + "type": ["null", "string"], "format": "date-time" }, "date_upd": { @@ -17,13 +17,13 @@ "format": "date-time" }, "name": { - "type": "string" + "type": ["null", "string"] }, "id_shop_group": { - "type": "string" + "type": ["null", "string"] }, "id_shop": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/weight_ranges.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/weight_ranges.json index 3a2218193d83..8678e7d0993c 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/weight_ranges.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/weight_ranges.json @@ -1,18 +1,18 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "id_carrier": { - "type": "string" + "type": ["null", "string"] }, "delimiter1": { - "type": "string" + "type": ["null", "string"] }, "delimiter2": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/zones.json b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/zones.json index 01b877fb0d71..2f33713b69ed 100644 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/zones.json +++ b/airbyte-integrations/connectors/source-prestashop/source_prestashop/schemas/zones.json @@ -1,15 +1,15 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "active": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-prestashop/source_prestashop/streams.py b/airbyte-integrations/connectors/source-prestashop/source_prestashop/streams.py deleted file mode 100644 index 1c1e34264147..000000000000 --- a/airbyte-integrations/connectors/source-prestashop/source_prestashop/streams.py +++ /dev/null @@ -1,669 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class PrestaShopStream(HttpStream, ABC): - """ - PrestaShop API Reference: https://devdocs.prestashop.com/1.7/basics/introduction/ - """ - - primary_key = "id" - page_size = 50 - - def __init__(self, url: str, **kwargs): - super(PrestaShopStream, self).__init__(**kwargs) - self._url = url - self._current_page = 0 - - @property - def url_base(self) -> str: - return f"{self._url}/api/" - - @property - def data_key(self): - return self.name - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - headers = super(PrestaShopStream, self).request_headers(**kwargs) - return {**headers, "Output-Format": "JSON"} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - if not response_json: - return None - if self._current_page % self.page_size == 0: - return {"limit": f"{self._current_page},{self.page_size}"} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"display": "full", "limit": self.page_size} - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - # pagination can be implemented via limit parameter - # https://devdocs.prestashop.com/1.7/webservice/tutorials/advanced-use/additional-list-parameters/#limit-parameter - # when records exist API returns dict, in another case empty list - if isinstance(response_json, dict): - records = response_json.get(self.data_key, []) - # as API response doesn't contain next_page parameter, we can only check records count to set offset - self._current_page += len(records) - yield from records - - -class IncrementalPrestaShopStream(PrestaShopStream, ABC): - cursor_field = "date_upd" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self._end_date = datetime.now() - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_record_state = latest_record.get(self.cursor_field) - return {self.cursor_field: max(latest_record_state, current_stream_state.get(self.cursor_field, latest_record_state))} - - def request_params(self, stream_state=None, **kwargs): - stream_state = stream_state or {} - params = super().request_params(stream_state=stream_state, **kwargs) - start_date = stream_state.get(self.cursor_field) - # for filtering interval operator is used - # https://devdocs.prestashop.com/1.7/webservice/tutorials/advanced-use/additional-list-parameters/#filter-parameter - if start_date: - params[f"filter[{self.cursor_field}]"] = f"[{start_date},{self._end_date}]" - params["date"] = 1 # needed to filter by dates - # sort by PK as well just in case cursor_fields are equal to not mess up pagination - params["sort"] = f"[{self.cursor_field}_ASC,{self.primary_key}_ASC]" - - return params - - -class Addresses(IncrementalPrestaShopStream): - """ - The Customer, Manufacturer and Customer addresses - https://devdocs.prestashop.com/1.7/webservice/resources/addresses/ - """ - - def path(self, **kwargs) -> str: - return "addresses" - - -class Carriers(PrestaShopStream): - """ - The Carriers that perform deliveries - https://devdocs.prestashop.com/1.7/webservice/resources/carriers/ - """ - - def path(self, **kwargs) -> str: - return "carriers" - - -class CartRules(IncrementalPrestaShopStream): - """ - Cart rules management (discount, promotions, …) - https://devdocs.prestashop.com/1.7/webservice/resources/cart_rules/ - """ - - def path(self, **kwargs) -> str: - return "cart_rules" - - -class Carts(IncrementalPrestaShopStream): - """ - Customer’s carts - https://devdocs.prestashop.com/1.7/webservice/resources/carts/ - """ - - def path(self, **kwargs) -> str: - return "carts" - - -class Categories(IncrementalPrestaShopStream): - """ - The product categories - https://devdocs.prestashop.com/1.7/webservice/resources/categories/ - """ - - def path(self, **kwargs) -> str: - return "categories" - - -class Combinations(PrestaShopStream): - """ - The product combinations - https://devdocs.prestashop.com/1.7/webservice/resources/combinations/ - """ - - def path(self, **kwargs) -> str: - return "combinations" - - -class Configurations(IncrementalPrestaShopStream): - """ - Shop configuration, used to store miscellaneous parameters from the shop - (maintenance, multi shop, email settings, …) - https://devdocs.prestashop.com/1.7/webservice/resources/configurations/ - """ - - def path(self, **kwargs) -> str: - return "configurations" - - -class Contacts(PrestaShopStream): - """ - Shop contacts - https://devdocs.prestashop.com/1.7/webservice/resources/contacts/ - """ - - def path(self, **kwargs) -> str: - return "contacts" - - -class ContentManagementSystem(PrestaShopStream): - """ - Content management system - https://devdocs.prestashop.com/1.7/webservice/resources/content_management_system/ - """ - - def path(self, **kwargs) -> str: - return "content_management_system" - - -class Countries(PrestaShopStream): - """ - The countries available on the shop - https://devdocs.prestashop.com/1.7/webservice/resources/countries/ - """ - - def path(self, **kwargs) -> str: - return "countries" - - -class Currencies(PrestaShopStream): - """ - The currencies installed on the shop - https://devdocs.prestashop.com/1.7/webservice/resources/currencies/ - """ - - def path(self, **kwargs) -> str: - return "currencies" - - -class CustomerMessages(IncrementalPrestaShopStream): - """ - Customer services messages - https://devdocs.prestashop.com/1.7/webservice/resources/customer_messages/ - """ - - def path(self, **kwargs) -> str: - return "customer_messages" - - -class CustomerThreads(IncrementalPrestaShopStream): - """ - Customer services threads - https://devdocs.prestashop.com/1.7/webservice/resources/customer_threads/ - """ - - def path(self, **kwargs) -> str: - return "customer_threads" - - -class Customers(IncrementalPrestaShopStream): - """ - The e-shop’s customers - https://devdocs.prestashop.com/1.7/webservice/resources/customers/ - """ - - def path(self, **kwargs) -> str: - return "customers" - - -class Deliveries(PrestaShopStream): - """ - Product deliveries - https://devdocs.prestashop.com/1.7/webservice/resources/deliveries/ - """ - - def path(self, **kwargs) -> str: - return "deliveries" - - -class Employees(PrestaShopStream): - """ - The Employees - https://devdocs.prestashop.com/1.7/webservice/resources/employees/ - """ - - def path(self, **kwargs) -> str: - return "employees" - - -class Groups(IncrementalPrestaShopStream): - """ - The customer’s groups - https://devdocs.prestashop.com/1.7/webservice/resources/groups/ - """ - - def path(self, **kwargs) -> str: - return "groups" - - -class Guests(PrestaShopStream): - """ - The guests (customers not logged in) - https://devdocs.prestashop.com/1.7/webservice/resources/guests/ - """ - - def path(self, **kwargs) -> str: - return "guests" - - -class ImageTypes(PrestaShopStream): - """ - The image types - https://devdocs.prestashop.com/1.7/webservice/resources/image_types/ - """ - - def path(self, **kwargs) -> str: - return "image_types" - - -class Languages(PrestaShopStream): - """ - Shop languages - https://devdocs.prestashop.com/1.7/webservice/resources/languages/ - """ - - def path(self, **kwargs) -> str: - return "languages" - - -class Manufacturers(IncrementalPrestaShopStream): - """ - The product manufacturers - https://devdocs.prestashop.com/1.7/webservice/resources/manufacturers/ - """ - - def path(self, **kwargs) -> str: - return "manufacturers" - - -class Messages(IncrementalPrestaShopStream): - """ - The customers messages - https://devdocs.prestashop.com/1.7/webservice/resources/messages/ - """ - - cursor_field = "date_add" - - def path(self, **kwargs) -> str: - return "messages" - - -class OrderCarriers(IncrementalPrestaShopStream): - """ - The order carriers - https://devdocs.prestashop.com/1.7/webservice/resources/order_carriers/ - """ - - cursor_field = "date_add" - - def path(self, **kwargs) -> str: - return "order_carriers" - - -class OrderDetails(PrestaShopStream): - """ - The order carriers - https://devdocs.prestashop.com/1.7/webservice/resources/order_details/ - """ - - def path(self, **kwargs) -> str: - return "order_details" - - -class OrderHistories(IncrementalPrestaShopStream): - """ - The Order histories - https://devdocs.prestashop.com/1.7/webservice/resources/order_histories/ - """ - - cursor_field = "date_add" - - def path(self, **kwargs) -> str: - return "order_histories" - - -class OrderInvoices(IncrementalPrestaShopStream): - """ - The Order invoices - https://devdocs.prestashop.com/1.7/webservice/resources/order_invoices/ - """ - - cursor_field = "date_add" - - def path(self, **kwargs) -> str: - return "order_invoices" - - -class OrderPayments(IncrementalPrestaShopStream): - """ - The Order payments - https://devdocs.prestashop.com/1.7/webservice/resources/order_payments/ - """ - - cursor_field = "date_add" - - def path(self, **kwargs) -> str: - return "order_payments" - - -class OrderSlip(IncrementalPrestaShopStream): - """ - The Order slips (used for refund) - https://devdocs.prestashop.com/1.7/webservice/resources/order_slip/ - """ - - data_key = "order_slips" - - def path(self, **kwargs) -> str: - return "order_slip" - - -class OrderStates(PrestaShopStream): - """ - The Order states (Waiting for transfer, Payment accepted, …) - https://devdocs.prestashop.com/1.7/webservice/resources/order_states/ - """ - - def path(self, **kwargs) -> str: - return "order_states" - - -class Orders(IncrementalPrestaShopStream): - """ - The Customers orders - https://devdocs.prestashop.com/1.7/webservice/resources/orders/ - """ - - def path(self, **kwargs) -> str: - return "orders" - - -class PriceRanges(PrestaShopStream): - """ - Price range - https://devdocs.prestashop.com/1.7/webservice/resources/price_ranges/ - """ - - def path(self, **kwargs) -> str: - return "price_ranges" - - -class ProductCustomizationFields(PrestaShopStream): - """ - The Product customization fields - https://devdocs.prestashop.com/1.7/webservice/resources/product_customization_fields/ - """ - - data_key = "customization_fields" - - def path(self, **kwargs) -> str: - return "product_customization_fields" - - -class ProductFeatureValues(PrestaShopStream): - """ - The product feature values (Ceramic, Polyester, … - Removable cover, Short sleeves, …) - https://devdocs.prestashop.com/1.7/webservice/resources/product_feature_values/ - """ - - def path(self, **kwargs) -> str: - return "product_feature_values" - - -class ProductFeatures(PrestaShopStream): - """ - The product features (Composition, Property, …) - https://devdocs.prestashop.com/1.7/webservice/resources/product_features/ - """ - - def path(self, **kwargs) -> str: - return "product_features" - - -class ProductOptionValues(PrestaShopStream): - """ - The product options value (S, M, L, … - White, Camel, …) - https://devdocs.prestashop.com/1.7/webservice/resources/product_option_values/ - """ - - def path(self, **kwargs) -> str: - return "product_option_values" - - -class ProductOptions(PrestaShopStream): - """ - The product options (Size, Color, …) - https://devdocs.prestashop.com/1.7/webservice/resources/product_options/ - """ - - def path(self, **kwargs) -> str: - return "product_options" - - -class ProductSuppliers(PrestaShopStream): - """ - Product Suppliers - https://devdocs.prestashop.com/1.7/webservice/resources/product_suppliers/ - """ - - def path(self, **kwargs) -> str: - return "product_suppliers" - - -class Products(IncrementalPrestaShopStream): - """ - The products - https://devdocs.prestashop.com/1.7/webservice/resources/products/ - """ - - def path(self, **kwargs) -> str: - return "products" - - -class ShopGroups(PrestaShopStream): - """ - Shop groups from multi-shop feature - https://devdocs.prestashop.com/1.7/webservice/resources/shop_groups/ - """ - - def path(self, **kwargs) -> str: - return "shop_groups" - - -class ShopUrls(PrestaShopStream): - """ - Shop urls from multi-shop feature - https://devdocs.prestashop.com/1.7/webservice/resources/shop_urls/ - """ - - def path(self, **kwargs) -> str: - return "shop_urls" - - -class Shops(PrestaShopStream): - """ - Shops from multi-shop feature - https://devdocs.prestashop.com/1.7/webservice/resources/shops/ - """ - - def path(self, **kwargs) -> str: - return "shops" - - -class SpecificPriceRules(PrestaShopStream): - """ - Specific price rules management - https://devdocs.prestashop.com/1.7/webservice/resources/specific_price_rules/ - """ - - def path(self, **kwargs) -> str: - return "specific_price_rules" - - -class SpecificPrices(PrestaShopStream): - """ - Specific price management - https://devdocs.prestashop.com/1.7/webservice/resources/specific_prices/ - """ - - def path(self, **kwargs) -> str: - return "specific_prices" - - -class States(PrestaShopStream): - """ - The available states of countries - https://devdocs.prestashop.com/1.7/webservice/resources/states/ - """ - - def path(self, **kwargs) -> str: - return "states" - - -class StockAvailables(PrestaShopStream): - """ - Available quantities of products - https://devdocs.prestashop.com/1.7/webservice/resources/stock_availables/ - """ - - def path(self, **kwargs) -> str: - return "stock_availables" - - -class StockMovementReasons(IncrementalPrestaShopStream): - """ - The stock movement reason (Increase, Decrease, Custom Order, …) - https://devdocs.prestashop.com/1.7/webservice/resources/stock_movement_reasons/ - """ - - def path(self, **kwargs) -> str: - return "stock_movement_reasons" - - -class StockMovements(IncrementalPrestaShopStream): - """ - Stock movements management - https://devdocs.prestashop.com/1.7/webservice/resources/stock_movements/ - """ - - data_key = "stock_mvts" - cursor_field = "date_add" - - def path(self, **kwargs) -> str: - return "stock_movements" - - -class Stores(IncrementalPrestaShopStream): - """ - The stores - https://devdocs.prestashop.com/1.7/webservice/resources/stores/ - """ - - def path(self, **kwargs) -> str: - return "stores" - - -class Suppliers(IncrementalPrestaShopStream): - """ - The product suppliers - https://devdocs.prestashop.com/1.7/webservice/resources/suppliers/ - """ - - def path(self, **kwargs) -> str: - return "suppliers" - - -class Tags(PrestaShopStream): - """ - The Products tags - https://devdocs.prestashop.com/1.7/webservice/resources/tags/ - """ - - def path(self, **kwargs) -> str: - return "tags" - - -class TaxRuleGroups(IncrementalPrestaShopStream): - """ - Group of Tax rule, along with their name - https://devdocs.prestashop.com/1.7/webservice/resources/tax_rule_groups/ - """ - - def path(self, **kwargs) -> str: - return "tax_rule_groups" - - -class TaxRules(PrestaShopStream): - """ - Tax rules, to associate Tax with a country, zip code, … - https://devdocs.prestashop.com/1.7/webservice/resources/tax_rules/ - """ - - def path(self, **kwargs) -> str: - return "tax_rules" - - -class Taxes(PrestaShopStream): - """ - The tax rate - https://devdocs.prestashop.com/1.7/webservice/resources/taxes/ - """ - - def path(self, **kwargs) -> str: - return "taxes" - - -class TranslatedConfigurations(PrestaShopStream): - """ - Shop configuration which are translated - https://devdocs.prestashop.com/1.7/webservice/resources/translated_configurations/ - """ - - # This API endpoint has cursor field date_upd, but it has empty value - - def path(self, **kwargs) -> str: - return "translated_configurations" - - -class WeightRanges(PrestaShopStream): - """ - Weight ranges for deliveries - https://devdocs.prestashop.com/1.7/webservice/resources/weight_ranges/ - """ - - def path(self, **kwargs) -> str: - return "weight_ranges" - - -class Zones(PrestaShopStream): - """ - The Countries zones - https://devdocs.prestashop.com/1.7/webservice/resources/zones/ - """ - - def path(self, **kwargs) -> str: - return "zones" diff --git a/airbyte-integrations/connectors/source-primetric/metadata.yaml b/airbyte-integrations/connectors/source-primetric/metadata.yaml index 844ddb69c5b3..38b37950abad 100644 --- a/airbyte-integrations/connectors/source-primetric/metadata.yaml +++ b/airbyte-integrations/connectors/source-primetric/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/primetric tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-primetric/requirements.txt b/airbyte-integrations/connectors/source-primetric/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-primetric/requirements.txt +++ b/airbyte-integrations/connectors/source-primetric/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-primetric/setup.py b/airbyte-integrations/connectors/source-primetric/setup.py index 8c22ddd3ed00..00a6f81dc6b3 100644 --- a/airbyte-integrations/connectors/source-primetric/setup.py +++ b/airbyte-integrations/connectors/source-primetric/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-public-apis/metadata.yaml b/airbyte-integrations/connectors/source-public-apis/metadata.yaml index 98793c0d7308..8056a2a39b58 100644 --- a/airbyte-integrations/connectors/source-public-apis/metadata.yaml +++ b/airbyte-integrations/connectors/source-public-apis/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: a4617b39-3c14-44cd-a2eb-6e720f269235 dockerImageTag: 0.1.0 dockerRepository: airbyte/source-public-apis + documentationUrl: https://docs.airbyte.com/integrations/sources/public-apis githubIssueLabel: source-public-apis icon: publicapi.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/public-apis + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-public-apis/requirements.txt b/airbyte-integrations/connectors/source-public-apis/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-public-apis/requirements.txt +++ b/airbyte-integrations/connectors/source-public-apis/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-public-apis/setup.py b/airbyte-integrations/connectors/source-public-apis/setup.py index c18ff92572d5..d4c33672b789 100644 --- a/airbyte-integrations/connectors/source-public-apis/setup.py +++ b/airbyte-integrations/connectors/source-public-apis/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-punk-api/metadata.yaml b/airbyte-integrations/connectors/source-punk-api/metadata.yaml index df1be5e007f6..942b617b200f 100644 --- a/airbyte-integrations/connectors/source-punk-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-punk-api/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-punk-api/requirements.txt b/airbyte-integrations/connectors/source-punk-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-punk-api/requirements.txt +++ b/airbyte-integrations/connectors/source-punk-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-punk-api/setup.py b/airbyte-integrations/connectors/source-punk-api/setup.py index 0d47f7be0057..6e0119b0fd17 100644 --- a/airbyte-integrations/connectors/source-punk-api/setup.py +++ b/airbyte-integrations/connectors/source-punk-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-pypi/metadata.yaml b/airbyte-integrations/connectors/source-pypi/metadata.yaml index 4a99164c157e..90cd84ad6509 100644 --- a/airbyte-integrations/connectors/source-pypi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pypi/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pypi/requirements.txt b/airbyte-integrations/connectors/source-pypi/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-pypi/requirements.txt +++ b/airbyte-integrations/connectors/source-pypi/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-pypi/setup.py b/airbyte-integrations/connectors/source-pypi/setup.py index f8d2b4b035b5..ae9a89a654a5 100644 --- a/airbyte-integrations/connectors/source-pypi/setup.py +++ b/airbyte-integrations/connectors/source-pypi/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-qonto/metadata.yaml b/airbyte-integrations/connectors/source-qonto/metadata.yaml index ac5650be433b..ccf5eb801992 100644 --- a/airbyte-integrations/connectors/source-qonto/metadata.yaml +++ b/airbyte-integrations/connectors/source-qonto/metadata.yaml @@ -14,7 +14,11 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/public-qonto + documentationUrl: https://docs.airbyte.com/integrations/sources/qonto tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qonto/requirements.txt b/airbyte-integrations/connectors/source-qonto/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-qonto/requirements.txt +++ b/airbyte-integrations/connectors/source-qonto/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-qonto/setup.py b/airbyte-integrations/connectors/source-qonto/setup.py index 96cf3e113541..61647d4573b0 100644 --- a/airbyte-integrations/connectors/source-qonto/setup.py +++ b/airbyte-integrations/connectors/source-qonto/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml index 67814fc0cffa..5e8ab3d7a8dc 100644 --- a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml +++ b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: b08e4776-d1de-4e80-ab5c-1e51dad934a2 dockerImageTag: 0.2.0 dockerRepository: airbyte/source-qualaroo + documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo githubIssueLabel: source-qualaroo icon: qualaroo.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qualaroo/requirements.txt b/airbyte-integrations/connectors/source-qualaroo/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-qualaroo/requirements.txt +++ b/airbyte-integrations/connectors/source-qualaroo/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-qualaroo/setup.py b/airbyte-integrations/connectors/source-qualaroo/setup.py index 6c86a85d10ea..b2929697645c 100644 --- a/airbyte-integrations/connectors/source-qualaroo/setup.py +++ b/airbyte-integrations/connectors/source-qualaroo/setup.py @@ -13,7 +13,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-quickbooks/Dockerfile b/airbyte-integrations/connectors/source-quickbooks/Dockerfile index 271ccdc7e009..79811759d651 100644 --- a/airbyte-integrations/connectors/source-quickbooks/Dockerfile +++ b/airbyte-integrations/connectors/source-quickbooks/Dockerfile @@ -34,5 +34,5 @@ COPY source_quickbooks ./source_quickbooks ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.1 +LABEL io.airbyte.version=2.0.4 LABEL io.airbyte.name=airbyte/source-quickbooks diff --git a/airbyte-integrations/connectors/source-quickbooks/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-quickbooks/integration_tests/abnormal_state.json index 035b42974ba8..d69ec95668eb 100644 --- a/airbyte-integrations/connectors/source-quickbooks/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-quickbooks/integration_tests/abnormal_state.json @@ -5,7 +5,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "budgets"} + "stream_descriptor": { "name": "budgets" } } }, { @@ -14,7 +14,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "bills"} + "stream_descriptor": { "name": "bills" } } }, { @@ -23,7 +23,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "classes"} + "stream_descriptor": { "name": "classes" } } }, { @@ -32,7 +32,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "departments"} + "stream_descriptor": { "name": "departments" } } }, { @@ -41,7 +41,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "vendor_credits"} + "stream_descriptor": { "name": "vendor_credits" } } }, { @@ -50,7 +50,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "transfers"} + "stream_descriptor": { "name": "transfers" } } }, { @@ -59,7 +59,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:35:37-08:00" }, - "stream_descriptor": {"name": "accounts"} + "stream_descriptor": { "name": "accounts" } } }, { @@ -68,7 +68,7 @@ "stream_state": { "airbyte_cursor": "2050-02-17T12:51:28-08:00" }, - "stream_descriptor": {"name": "credit_memos"} + "stream_descriptor": { "name": "credit_memos" } } }, { @@ -77,7 +77,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T12:56:17-08:00" }, - "stream_descriptor": {"name": "bill_payments"} + "stream_descriptor": { "name": "bill_payments" } } }, { @@ -86,7 +86,7 @@ "stream_state": { "airbyte_cursor": "2050-02-16T11:40:52-08:00" }, - "stream_descriptor": {"name": "sales_receipts"} + "stream_descriptor": { "name": "sales_receipts" } } }, { @@ -95,7 +95,7 @@ "stream_state": { "airbyte_cursor": "2050-03-15T14:04:25-07:00" }, - "stream_descriptor": {"name": "purchases"} + "stream_descriptor": { "name": "purchases" } } }, { @@ -104,7 +104,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:13:33-08:00" }, - "stream_descriptor": {"name": "payments"} + "stream_descriptor": { "name": "payments" } } }, { @@ -113,7 +113,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:10:36-08:00" }, - "stream_descriptor": {"name": "purchase_orders"} + "stream_descriptor": { "name": "purchase_orders" } } }, { @@ -122,7 +122,7 @@ "stream_state": { "airbyte_cursor": "2050-02-10T14:42:05-08:00" }, - "stream_descriptor": {"name": "payment_methods"} + "stream_descriptor": { "name": "payment_methods" } } }, { @@ -131,7 +131,7 @@ "stream_state": { "airbyte_cursor": "2050-02-15T10:04:24-08:00" }, - "stream_descriptor": {"name": "journal_entries"} + "stream_descriptor": { "name": "journal_entries" } } }, { @@ -140,7 +140,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:16:17-08:00" }, - "stream_descriptor": {"name": "items"} + "stream_descriptor": { "name": "items" } } }, { @@ -149,7 +149,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:16:17-08:00" }, - "stream_descriptor": {"name": "invoices"} + "stream_descriptor": { "name": "invoices" } } }, { @@ -158,7 +158,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:16:17-08:00" }, - "stream_descriptor": {"name": "customers"} + "stream_descriptor": { "name": "customers" } } }, { @@ -167,7 +167,7 @@ "stream_state": { "airbyte_cursor": "2050-02-16T15:35:07-08:00" }, - "stream_descriptor": {"name": "refund_receipts"} + "stream_descriptor": { "name": "refund_receipts" } } }, { @@ -176,7 +176,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:00:56-08:00" }, - "stream_descriptor": {"name": "deposits"} + "stream_descriptor": { "name": "deposits" } } }, { @@ -185,7 +185,7 @@ "stream_state": { "airbyte_cursor": "2050-02-16T11:46:26-08:00" }, - "stream_descriptor": {"name": "employees"} + "stream_descriptor": { "name": "employees" } } }, { @@ -194,7 +194,7 @@ "stream_state": { "airbyte_cursor": "2050-02-18T13:16:17-08:00" }, - "stream_descriptor": {"name": "estimates"} + "stream_descriptor": { "name": "estimates" } } }, { @@ -203,7 +203,7 @@ "stream_state": { "airbyte_cursor": "2050-02-17T12:17:04-08:00" }, - "stream_descriptor": {"name": "tax_agencies"} + "stream_descriptor": { "name": "tax_agencies" } } }, { @@ -212,7 +212,7 @@ "stream_state": { "airbyte_cursor": "2050-02-17T12:17:04-08:00" }, - "stream_descriptor": {"name": "tax_codes"} + "stream_descriptor": { "name": "tax_codes" } } }, { @@ -221,7 +221,7 @@ "stream_state": { "airbyte_cursor": "2050-02-17T12:17:04-08:00" }, - "stream_descriptor": {"name": "tax_rates"} + "stream_descriptor": { "name": "tax_rates" } } }, { @@ -230,7 +230,7 @@ "stream_state": { "airbyte_cursor": "2050-02-15T15:24:26-08:00" }, - "stream_descriptor": {"name": "terms"} + "stream_descriptor": { "name": "terms" } } }, { @@ -239,7 +239,7 @@ "stream_state": { "airbyte_cursor": "2050-02-16T11:55:25-08:00" }, - "stream_descriptor": {"name": "time_activities"} + "stream_descriptor": { "name": "time_activities" } } }, { @@ -248,7 +248,7 @@ "stream_state": { "airbyte_cursor": "2050-03-04T14:29:35-08:00" }, - "stream_descriptor": {"name": "vendors"} + "stream_descriptor": { "name": "vendors" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml index c2cd95cf43ae..b25b55b3e4fe 100644 --- a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: cf9c4355-b171-4477-8f2d-6c5cc5fc8b7e - dockerImageTag: 2.0.1 + dockerImageTag: 2.0.4 dockerRepository: airbyte/source-quickbooks githubIssueLabel: source-quickbooks icon: quickbooks.svg @@ -23,4 +23,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-quickbooks/requirements.txt b/airbyte-integrations/connectors/source-quickbooks/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-quickbooks/requirements.txt +++ b/airbyte-integrations/connectors/source-quickbooks/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-quickbooks/setup.py b/airbyte-integrations/connectors/source-quickbooks/setup.py index 480de3454ed8..025726239f79 100644 --- a/airbyte-integrations/connectors/source-quickbooks/setup.py +++ b/airbyte-integrations/connectors/source-quickbooks/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk>=0.44.0", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py index d91a94389128..d1f3a93fc12e 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py @@ -66,10 +66,10 @@ class CustomDatetimeBasedCursor(DatetimeBasedCursor): To adopt this change to the LowCode CDK, this issue was created - https://github.com/airbytehq/airbyte/issues/25008. """ - def update_cursor(self, stream_slice: StreamSlice, last_record: typing.Optional[Record] = None): - super(CustomDatetimeBasedCursor, self).update_cursor( + def close_slice(self, stream_slice: StreamSlice, most_recent_record: typing.Optional[Record]) -> None: + super(CustomDatetimeBasedCursor, self).close_slice( stream_slice=stream_slice, - last_record=LastRecordDictProxy(last_record, {self.cursor_field.eval(self.config): "MetaData/LastUpdatedTime"}), + last_record=LastRecordDictProxy(most_recent_record, {self.cursor_field.eval(self.config): "MetaData/LastUpdatedTime"}), ) def _format_datetime(self, dt: datetime.datetime): @@ -77,3 +77,16 @@ def _format_datetime(self, dt: datetime.datetime): def parse_date(self, date: str) -> datetime.datetime: return datetime.datetime.strptime(date, self.datetime_format).astimezone(self._timezone) + + def should_be_synced(self, record: Record) -> bool: + """ + As of 2023-06-28, the expectation is that this method will only be used for semi-incremental and data feed and therefore the + implementation is irrelevant for quickbooks + """ + return True + + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: + return super(CustomDatetimeBasedCursor, self).close_slice( + LastRecordDictProxy(first, {self.cursor_field.eval(self.config): "MetaData/LastUpdatedTime"}), + LastRecordDictProxy(second, {self.cursor_field.eval(self.config): "MetaData/LastUpdatedTime"}), + ) diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml index 355540141326..ac70de7bbc6a 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml @@ -21,8 +21,12 @@ definitions: "Accept": "application/json" "User-Agent": "airbyte-connector" authenticator: - type: SingleUseRefreshTokenOAuthAuthenticator + type: OAuthAuthenticator token_refresh_endpoint: "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer" + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: "{{ config['credentials']['client_secret'] }}" + refresh_token: "{{ config['credentials']['refresh_token'] }}" + refresh_token_updater: {} retriever: type: SimpleRetriever record_selector: @@ -50,7 +54,7 @@ definitions: transformations: - type: AddFields fields: - - path: [ "airbyte_cursor" ] + - path: ["airbyte_cursor"] value: "{{ record.MetaData.LastUpdatedTime }}" $parameters: path: "/v3/company/{{ config.credentials.realm_id }}/query" diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/accounts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/accounts.json index d0ac5c695322..752445c28bdb 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/accounts.json @@ -1,155 +1,83 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "CurrentBalanceWithSubAccounts": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AcctNum": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "AccountSubType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Classification": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "FullyQualifiedName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "AccountType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CurrentBalance": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SubAccount": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "ParentRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bill_payments.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bill_payments.json index 562f7c239627..e7d008af6ea3 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bill_payments.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bill_payments.json @@ -1,250 +1,136 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "APAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DepartmentRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CheckPayment": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "BankAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "CurrencyRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PayType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } } }, "VendorRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CreditCardPayment": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "CCAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LinkedTxn": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -253,28 +139,16 @@ } }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json index ef981b39f47d..8498337dfd9b 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json @@ -1,403 +1,220 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "SalesTermRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Balance": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "APAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DueDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "LinkedTxn": { "items": { "properties": { "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "Line": { "items": { "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ItemBasedExpenseLineDetail": { "properties": { "ItemRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Qty": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "UnitPrice": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AccountBasedExpenseLineDetail": { "properties": { "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "AccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "VendorRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DepartmentRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/budgets.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/budgets.json index c7d33f8ca6f7..230838cf0059 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/budgets.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/budgets.json @@ -1,65 +1,35 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "BudgetEntryType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "EndDate": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "StartDate": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BudgetType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BudgetDetail": { "items": { @@ -67,134 +37,74 @@ "ClassRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DepartmentRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "AccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BudgetDate": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/classes.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/classes.json index 69f1a1a98fd0..0c4264fa69b1 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/classes.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/classes.json @@ -1,99 +1,54 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "ParentRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SubClass": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "FullyQualifiedName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json index 18ec8ddabe81..784c98df94c3 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json @@ -1,474 +1,255 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "ClassRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BillEmail": { "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "EmailStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Balance": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "CustomerMemo": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { "items": { "properties": { "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SalesItemLineDetail": { "properties": { "ItemRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "UnitPrice": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Qty": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ApplyTaxAfterDiscount": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SalesTermRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ShipAddr": { "properties": { "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnTaxDetail": { "properties": { "TotalTax": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CustomField": { "items": { "properties": { "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DefinitionId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BillAddr": { "properties": { "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line4": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "RemainingCredit": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HomeTotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/customers.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/customers.json index 4d38670ff789..a5c1598896b7 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/customers.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/customers.json @@ -1,433 +1,232 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "DefaultTaxCodeRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SalesTermRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PaymentMethodRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ResaleNum": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "MiddleName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BillAddr": { "properties": { "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PrimaryEmailAddr": { "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Balance": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CompanyName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "FamilyName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "FullyQualifiedName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrimaryPhone": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BalanceWithJobs": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "BillWithParent": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Level": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "ParentRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PrintOnCheckName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Job": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Mobile": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PreferredDeliveryMethod": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Taxable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "DisplayName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "WebAddr": { "properties": { "URI": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "GivenName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ShipAddr": { "properties": { "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Fax": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/departments.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/departments.json index 1e5468ba2cf7..03a8daead5dd 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/departments.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/departments.json @@ -1,99 +1,54 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "ParentRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "MetaData": { "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "FullyQualifiedName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SubDepartment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/deposits.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/deposits.json index 9439f757594e..e89b73f293bf 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/deposits.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/deposits.json @@ -1,304 +1,166 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "DepartmentRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CashBack": { "properties": { "AccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Memo": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { "items": { "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DepositLineDetail": { "properties": { "PaymentMethodRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "AccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CheckNum": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "LinkedTxn": { "items": { "properties": { "TxnLineId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DepositToAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/employees.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/employees.json index f001c7a3cc1d..799901a52c7f 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/employees.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/employees.json @@ -1,195 +1,105 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "HiredDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrimaryPhone": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BillableTime": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "DisplayName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "GivenName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "FamilyName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "BirthDate": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrintOnCheckName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PrimaryAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": {} }, "PrimaryEmailAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": {} }, "Title": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MiddleName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ReleasedDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "Mobile": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": {} }, "Gender": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BillRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Suffix": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "EmployeeNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Organization": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json index 2539ffa15f66..9d2eb205046b 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/estimates.json @@ -1,548 +1,296 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "BillEmail": { "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BillAddr": { "properties": { "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line4": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "EmailStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DeliveryInfo": { "properties": { "DeliveryType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnTaxDetail": { "properties": { "TotalTax": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxLine": { "items": { "properties": { "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxLineDetail": { "properties": { "PercentBased": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "TaxPercent": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "NetAmountTaxable": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxRateRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "TxnTaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "LinkedTxn": { "items": { "properties": { "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "CustomerMemo": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Line": { "items": { "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SalesItemLineDetail": { "properties": { "ItemRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Qty": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "UnitPrice": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "ApplyTaxAfterDiscount": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CustomField": { "items": { "properties": { "DefinitionId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HomeTotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ShipAddr": { "properties": { "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json index 245398f37d59..9185d986b532 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/invoices.json @@ -1,293 +1,158 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "AllowOnlineACHPayment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BillAddr": { "properties": { "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line4": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DeliveryInfo": { "properties": { "DeliveryType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SalesTermRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "AllowIPNPayment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "AllowOnlineCreditCardPayment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CustomerMemo": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomField": { "items": { "properties": { "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "StringValue": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DefinitionId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "Balance": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "AllowOnlinePayment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "DueDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HomeTotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Line": { "items": { @@ -296,421 +161,226 @@ "items": { "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SalesItemLineDetail": { "properties": { "ClassRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "UnitPrice": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "ServiceDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ItemRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Qty": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DiscountLineDetail": { "properties": { "DiscountAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DiscountPercent": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "PercentBased": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnTaxDetail": { "properties": { "TotalTax": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxLine": { "items": { "properties": { "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TaxLineDetail": { "properties": { "NetAmountTaxable": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxPercent": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxRateRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PercentBased": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "TxnTaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "EmailStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LinkedTxn": { "items": { "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ApplyTaxAfterDiscount": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "BillEmail": { "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ShipAddr": { "properties": { "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json index 91a7d0dcbec8..91b393d430df 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json @@ -1,194 +1,104 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "IncomeAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PurchaseDesc": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ExpenseAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "InvStartDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "FullyQualifiedName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Taxable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TrackQtyOnHand": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "AssetAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "QtyOnHand": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "UnitPrice": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PurchaseCost": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/journal_entries.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/journal_entries.json index c528cfa35aea..82bbe402c26a 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/journal_entries.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/journal_entries.json @@ -1,261 +1,138 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TaxRateRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { "items": { "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "JournalEntryLineDetail": { "properties": { "AccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PostingType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnTaxDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TotalTax": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TxnTaxCodeRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "TaxLine": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxLineDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TaxPercent": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "OverrideDeltaAmount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxInclusiveAmount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PercentBased": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "NetAmountTaxable": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxRateRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -267,56 +144,32 @@ } }, "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Adjustment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payment_methods.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payment_methods.json index 16ce6aed4405..a2608814c1f4 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payment_methods.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payment_methods.json @@ -1,79 +1,43 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } } }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payments.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payments.json index 0cccb9663b1a..21692322e332 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payments.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/payments.json @@ -1,147 +1,78 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "DepositToAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "CurrencyRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "ARAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Line": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LineEx": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "any": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "typeSubstituted": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "scope": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "nil": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "value": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "declaredType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "globalScope": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } } @@ -149,194 +80,107 @@ } }, "LinkedTxn": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] } } } }, "ProcessPayment": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LinkedTxn": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "UnappliedAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } } }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PaymentMethodRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "CustomerRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "PaymentRefNum": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchase_orders.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchase_orders.json index 2a0cb048ef8d..8a5d76409969 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchase_orders.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchase_orders.json @@ -1,572 +1,308 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Memo": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DueDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnTaxDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": {} }, "EmailStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ShipTo": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ClassRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SalesTermRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CurrencyRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "APAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ShipAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "LinkedTxn": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "VendorAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line4": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CustomField": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DefinitionId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "MetaData": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } } }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "POStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "VendorRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DepartmentRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Line": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ItemBasedExpenseLineDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ClassRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "ItemRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "UnitPrice": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxCodeRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Qty": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "CustomerRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchases.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchases.json index db774c4941dc..4e05402c1f2d 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchases.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/purchases.json @@ -1,447 +1,240 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PaymentType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } } }, "RemitToAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "CurrencyRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "AccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "EntityRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Credit": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "ItemBasedExpenseLineDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "ItemRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Qty": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "UnitPrice": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxCodeRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "AccountBasedExpenseLineDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "AccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "CustomerRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TaxCodeRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "PurchaseEx": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "any": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "scope": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "nil": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "globalScope": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "value": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "declaredType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "typeSubstituted": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } } @@ -449,22 +242,13 @@ } }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json index 119a9dca2a13..afefac60eab4 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json @@ -1,412 +1,223 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "BillEmail": { "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PaymentMethodRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { "items": { "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SalesItemLineDetail": { "properties": { "ItemRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Qty": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "UnitPrice": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Balance": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "TxnTaxDetail": { "properties": { "TotalTax": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DepositToAccountRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CustomerMemo": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CustomField": { "items": { "properties": { "DefinitionId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "BillAddr": { "properties": { "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line4": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ApplyTaxAfterDiscount": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HomeTotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json index 0abf80f9260f..291fd278ab75 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json @@ -1,546 +1,294 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Balance": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "CustomerRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "LinkedTxn": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TxnType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "ApplyTaxAfterDiscount": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "PaymentRefNum": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnTaxDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TotalTax": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } } }, "PrintStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } } }, "CustomField": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DefinitionId": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "BillAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line2": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line3": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line4": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "EmailStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "DiscountLineDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "PercentBased": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "DiscountPercent": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DiscountAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SalesItemLineDetail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "ItemRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "Qty": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "UnitPrice": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "TaxCodeRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "CurrencyRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "PaymentMethodRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BillEmail": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "CustomerMemo": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "ShipAddr": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "HomeTotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DepositToAccountRef": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_agencies.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_agencies.json index c455fd330bf5..733ed0d6bc8f 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_agencies.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_agencies.json @@ -1,85 +1,46 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TaxRegistrationNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TaxTrackedOnPurchases": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "DisplayName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TaxTrackedOnSales": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_codes.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_codes.json index 8e6f69974deb..b82b6c0669c6 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_codes.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_codes.json @@ -1,70 +1,37 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "TaxGroup": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "PurchaseTaxRateList": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": {} }, "Hidden": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Taxable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SalesTaxRateList": { "properties": { @@ -72,87 +39,48 @@ "items": { "properties": { "TaxOrder": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "TaxTypeApplicable": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TaxRateRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_rates.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_rates.json index 20a1c81f1302..15ec412bbb08 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_rates.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/tax_rates.json @@ -1,177 +1,99 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "SpecialTaxType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "AgencyRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "RateValue": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DisplayType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "EffectiveTaxRate": { "anyOf": [ { "properties": { "RateValue": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "EndDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "EffectiveDate ": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { "properties": { "RateValue": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "EndDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "EffectiveDate ": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } } ] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/terms.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/terms.json index c4dbb3022e67..d53198aef564 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/terms.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/terms.json @@ -1,115 +1,61 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DiscountPercent": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DayOfMonthDue": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DiscountDayOfMonth": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "DueNextMonthDays": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "DueDays": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DiscountDays": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/time_activities.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/time_activities.json index 0f0d4234d2e8..70769351c802 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/time_activities.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/time_activities.json @@ -1,170 +1,92 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "HourlyRate": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "CustomerRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Taxable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Minutes": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ItemRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TxnDate": { "format": "date", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "EmployeeRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "NameOf": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Hours": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "sparse": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/transfers.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/transfers.json index dadcdaeee312..275f361ce306 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/transfers.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/transfers.json @@ -1,140 +1,77 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "PrivateNote": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MetaData": { "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ToAccountRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CurrencyRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "FromAccountRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendor_credits.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendor_credits.json index 76182505a80e..6d77b07c43a4 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendor_credits.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendor_credits.json @@ -1,292 +1,160 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "MetaData": { "properties": { "LastUpdatedTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "CreateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ExchangeRate": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TotalAmt": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "VendorRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DepartmentRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "APAccountRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "TxnDate": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date" }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line": { "items": { "properties": { "LineNum": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "AccountBasedExpenseLineDetail": { "properties": { "CustomerRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "ClassRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "AccountRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "BillableStatus": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TaxCodeRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Amount": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "DetailType": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, - "type": [ - "null", - "array" - ] + "type": ["null", "array"] }, "CurrencyRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "DocNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendors.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendors.json index 2c2ab1927f86..0a7d672279d5 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendors.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/vendors.json @@ -1,293 +1,158 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "GivenName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrimaryPhone": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "AcctNum": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "MiddleName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "BillAddr": { "properties": { "Lat": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CountrySubDivisionCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "City": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Long": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PostalCode": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Line1": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "PrimaryEmailAddr": { "properties": { "Address": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Balance": { - "type": [ - "null", - "number" - ] + "type": ["null", "number"] }, "TaxIdentifier": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "CompanyName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "FamilyName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "PrintOnCheckName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "WebAddr": { "properties": { "URI": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Mobile": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "CurrencyRef": { "properties": { "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Vendor1099": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "SyncToken": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "DisplayName": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "TermRef": { "properties": { "value": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "MetaData": { "properties": { "CreateTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "LastUpdatedTime": { "format": "date-time", - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Fax": { "properties": { "FreeFormNumber": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, - "type": [ - "null", - "object" - ] + "type": ["null", "object"] }, "Suffix": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "domain": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "Active": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "Title": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "airbyte_cursor": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json index 19b568e63628..ead3ac5eddd6 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/spec.json @@ -4,11 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source QuickBooks Spec", "type": "object", - "required": [ - "credentials", - "start_date", - "sandbox" - ], + "required": ["credentials", "start_date", "sandbox"], "additionalProperties": true, "properties": { "credentials": { @@ -73,14 +69,12 @@ }, "start_date": { "order": 1, - "description": "The default value to use if no bookmark exists for an endpoint (rfc3339 date string). E.g, 2021-03-20T00:00:00+00:00. Any data before this date will not be replicated.", + "description": "The default value to use if no bookmark exists for an endpoint (rfc3339 date string). E.g, 2021-03-20T00:00:00Z. Any data before this date will not be replicated.", "title": "Start Date", "type": "string", "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": [ - "2021-03-20T00:00:00+00:00" - ] + "examples": ["2021-03-20T00:00:00Z"] }, "sandbox": { "order": 2, @@ -91,4 +85,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py b/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py index 9924ed006a5c..3519eabaf89c 100644 --- a/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py +++ b/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py @@ -43,7 +43,7 @@ def test_dict_proxy(): } -def test_custom_datetime_based_cursor__update_cursor(): +def test_custom_datetime_based_cursor__close_slice(): cursor_field_name = "airbyte_cursor" record_cursor_value = "2023-02-10T14:42:05-08:00" @@ -58,10 +58,11 @@ def test_custom_datetime_based_cursor__update_cursor(): parameters={} ) - date_time_based_cursor_component.update_cursor( + slice_end_time = "2023-03-03T00:00:00+00:00" + date_time_based_cursor_component.close_slice( { "start_time": "2023-02-01T00:00:00+00:00", - "end_time": "2023-03-03T00:00:00+00:00" + "end_time": slice_end_time }, { "Id": "1", @@ -71,7 +72,7 @@ def test_custom_datetime_based_cursor__update_cursor(): } } ) - assert date_time_based_cursor_component.get_stream_state() == {cursor_field_name: record_cursor_value} + assert date_time_based_cursor_component.get_stream_state() == {cursor_field_name: slice_end_time} def test_custom_datetime_based_cursor__format_datetime(): diff --git a/airbyte-integrations/connectors/source-railz/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-railz/integration_tests/abnormal_state.json index 8b2c69f9d40a..9484296d8de8 100644 --- a/airbyte-integrations/connectors/source-railz/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-railz/integration_tests/abnormal_state.json @@ -540,4 +540,3 @@ } } ] - diff --git a/airbyte-integrations/connectors/source-railz/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-railz/integration_tests/invalid_config.json index cdc01015b8c4..8c36eb1b4e76 100644 --- a/airbyte-integrations/connectors/source-railz/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-railz/integration_tests/invalid_config.json @@ -3,4 +3,3 @@ "secret_key": "12412512", "start_date": "2022-08-01" } - diff --git a/airbyte-integrations/connectors/source-railz/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-railz/integration_tests/sample_config.json index b32903240618..3df7b385dc44 100644 --- a/airbyte-integrations/connectors/source-railz/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-railz/integration_tests/sample_config.json @@ -3,4 +3,3 @@ "secret_key": "sb_prod_12345", "start_date": "2022-08-01" } - diff --git a/airbyte-integrations/connectors/source-railz/metadata.yaml b/airbyte-integrations/connectors/source-railz/metadata.yaml index 897cab08375e..ea4aa848d729 100644 --- a/airbyte-integrations/connectors/source-railz/metadata.yaml +++ b/airbyte-integrations/connectors/source-railz/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-railz/requirements.txt b/airbyte-integrations/connectors/source-railz/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-railz/requirements.txt +++ b/airbyte-integrations/connectors/source-railz/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-railz/setup.py b/airbyte-integrations/connectors/source-railz/setup.py index dcb46a1aaacc..3b938dc45ff5 100644 --- a/airbyte-integrations/connectors/source-railz/setup.py +++ b/airbyte-integrations/connectors/source-railz/setup.py @@ -12,9 +12,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", "freezegun", ] diff --git a/airbyte-integrations/connectors/source-railz/source_railz/schemas/businesses.json b/airbyte-integrations/connectors/source-railz/source_railz/schemas/businesses.json index 9d9d11b2d112..3fdee707afc7 100644 --- a/airbyte-integrations/connectors/source-railz/source_railz/schemas/businesses.json +++ b/airbyte-integrations/connectors/source-railz/source_railz/schemas/businesses.json @@ -19,7 +19,7 @@ "metadata": { "type": ["null", "array"], "items": { - "type" : "string" + "type": "string" } }, "connections": { diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile b/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile index 94dfec89b0f1..86a602f8dacd 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-rd-station-marketing/Dockerfile @@ -34,5 +34,5 @@ COPY source_rd_station_marketing ./source_rd_station_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-rd-station-marketing diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml b/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml index 27fd3fd51f54..42135ace43a0 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-rd-station-marketing/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: fb141f29-be2a-450b-a4f2-2cd203a00f84 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-rd-station-marketing githubIssueLabel: source-rd-station-marketing icon: rdstation.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/rd-station-marketing tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt b/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt +++ b/airbyte-integrations/connectors/source-rd-station-marketing/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py index b12d2a3db5f9..fc5481e12eb5 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/setup.py +++ b/airbyte-integrations/connectors/source-rd-station-marketing/setup.py @@ -12,7 +12,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json index 998021c78a30..9f797e9a8338 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json +++ b/airbyte-integrations/connectors/source-rd-station-marketing/source_rd_station_marketing/spec.json @@ -54,12 +54,45 @@ } }, "supportsIncremental": true, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["authorization", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": true, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["authorization", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": true, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": true, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["authorization", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["authorization", "client_secret"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 0b8251340fe4..bbc8964f4f60 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index 596b618eae09..c7b80b0e67e0 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: bypass_reason: "updated after login" - name: store/updated_at bypass_reason: "updated after login" - timeout_seconds: 1200 + timeout_seconds: 7200 expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no @@ -35,20 +35,20 @@ acceptance_tests: discovery: tests: - backward_compatibility_tests_config: - disable_for_version: 0.2.0 + disable_for_version: 0.2.10 config_path: secrets/config.json full_refresh: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json - timeout_seconds: 1200 + timeout_seconds: 3200 incremental: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/streams_with_output_records_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 900 + timeout_seconds: 3200 spec: tests: - spec_path: source_recharge/spec.json diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py b/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl index 321e32769096..5c192d0d884c 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl @@ -1,6 +1,6 @@ -{"stream": "addresses", "data": {"address1": "1 9th Ave", "address2": "1", "cart_attributes": null, "cart_note": null, "city": "San Francisco", "company": null, "country": "United States", "country_code": "US", "created_at": "2021-05-12T08:04:06", "customer_id": 64817252, "discount_id": null, "first_name": "Jane", "id": 69105381, "last_name": "Doe", "note_attributes": null, "original_shipping_lines": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_override": null, "updated_at": "2023-01-16T04:59:09", "zip": "94118"}, "emitted_at": 1680895024611} -{"stream": "charges", "data": {"address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country": "United States", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T08:04:07", "currency": "USD", "customer_hash": "23dee52d73734a81", "customer_id": 64817252, "discount_codes": [], "email": "nikolaevaka@yahoo.com", "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "first_name": "Karina", "has_uncommited_changes": false, "id": 386976088, "last_charge_attempt_date": "2022-06-06T16:10:19", "last_name": "Kuznetsova", "line_items": [{"grams": 0, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "price": "24.30", "properties": [], "quantity": 1, "shopify_product_id": "6642695864491", "shopify_variant_id": "39684722131115", "sku": "T1", "subscription_id": 153224593, "tax_lines": [], "title": "Airbit Box Corner Short sleeve t-shirt", "type": "SUBSCRIPTION", "variant_title": "S / Black", "vendor": "Airbyte"}], "note": "", "note_attributes": null, "number_times_tried": 6, "processor_name": "shopify_payments", "requires_shipping": true, "retry_date": "2022-06-12T00:00:00", "scheduled_at": "2022-05-12T00:00:00", "shipments_count": null, "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country": "United States", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "description": "", "price": 4.9, "source": "shopify", "tax_lines": [], "title": "Economy"}], "shopify_order_id": null, "shopify_variant_id_not_found": null, "status": "ERROR", "sub_total": null, "subtotal_price": "24.3", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "0.0", "total_discounts": "0.00", "total_duties": 0.0, "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": null, "total_tax": "0.0", "total_weight": 0, "transaction_id": null, "type": "RECURRING", "updated_at": "2023-01-16T13:08:54"}, "emitted_at": 1680895025756} -{"stream": "customers", "data": {"accepts_marketing": 1, "analytics_data": {"utm_params": []}, "billing_address1": "1 9th Ave", "billing_address2": "1", "billing_city": "San Francisco", "billing_company": null, "billing_country": "United States", "billing_phone": null, "billing_province": "California", "billing_zip": "94118", "created_at": "2021-05-12T08:04:06", "email": "nikolaevaka@yahoo.com", "first_charge_processed_at": "2021-05-12T12:03:59", "first_name": "Karina", "has_card_error_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "id": 64817252, "last_name": "Kuznetsova", "number_active_subscriptions": 0, "number_subscriptions": 1, "phone": null, "processor_type": "shopify_payments", "reason_payment_method_not_valid": null, "shopify_customer_id": "5212085977259", "status": "ACTIVE", "tax_exempt": false, "updated_at": "2023-01-16T13:08:45"}, "emitted_at": 1680895026982} +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1680895024611} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1687184458990} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1687184599794} {"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1680895030371} {"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1680895030371} {"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}}, "emitted_at": 1680895031312} diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index 5bd49ed95411..b4a498ec5a26 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e - dockerImageTag: 0.2.9 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-recharge githubIssueLabel: source-recharge icon: recharge.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/recharge tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-recharge/requirements.txt b/airbyte-integrations/connectors/source-recharge/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-recharge/requirements.txt +++ b/airbyte-integrations/connectors/source-recharge/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-recharge/setup.py b/airbyte-integrations/connectors/source-recharge/setup.py index 256b8ae1a891..bb091a439b28 100644 --- a/airbyte-integrations/connectors/source-recharge/setup.py +++ b/airbyte-integrations/connectors/source-recharge/setup.py @@ -10,6 +10,7 @@ ] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py b/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py index a7aafe9ffd60..d5ab4df2ea0f 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index 204975e60602..c2c9d3e85f8c 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional @@ -12,6 +11,9 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +API_VERSION = "2021-11" +OLD_API_VERSION = "2021-01" + class RechargeStream(HttpStream, ABC): primary_key = "id" @@ -22,7 +24,7 @@ class RechargeStream(HttpStream, ABC): period_in_months = 1 # Slice data request for 1 month raise_on_http_errors = True - # regestring the default schema transformation + # registering the default schema transformation transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) def __init__(self, config, **kwargs): @@ -33,28 +35,32 @@ def __init__(self, config, **kwargs): def data_path(self): return self.name + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"x-recharge-version": API_VERSION} + def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return self.name def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = self.get_stream_data(response.json()) - if len(stream_data) == self.limit: - self.page_num += 1 - return {"page": self.page_num} + cursor = response.json().get("next_cursor") + if cursor: + return {"cursor": cursor} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: params = { "limit": self.limit, - "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), - "updated_at_max": (stream_slice or {}).get("end_date", self._start_date), } if next_page_token: params.update(next_page_token) + else: + params.update({"updated_at_min": (stream_state or {}).get("updated_at", self._start_date)}) return params @@ -174,17 +180,29 @@ class Orders(IncrementalRechargeStream): class Products(RechargeStream): """ Products Stream: https://developer.rechargepayments.com/v1-shopify?python#list-products + Products endpoint has 422 error with 2021-11 API version """ + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"x-recharge-version": OLD_API_VERSION} + class Shop(RechargeStream): """ Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop + Shop endpoint is not available in 2021-11 API version """ primary_key = ["shop", "store"] data_path = None + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"x-recharge-version": OLD_API_VERSION} + class Subscriptions(IncrementalRechargeStream): """ diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json index c3aa406e5cb8..40faaeb50fe6 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json @@ -137,8 +137,7 @@ "format": "date-time" }, "scheduled_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] }, "shipments_count": { "type": ["null", "integer"] @@ -214,7 +213,7 @@ "type": ["null", "string"] }, "shopify_variant_id_not_found": { - "type": ["null", "integer"] + "type": ["null", "string"] }, "status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py index dd552c6f656c..6e1cc27e492a 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py @@ -191,29 +191,30 @@ def generate_records(self, stream_name, count): return {stream_name: result} @pytest.mark.parametrize( - "stream_cls, rec_limit, expected", + "stream_cls, cursor_response, expected", [ - (Collections, 1, {"page": 2}), - (Metafields, 2, {"page": 2}), - (Products, 1, {"page": 2}), - (Shop, 1, {"page": 2}), + (Collections, "some next cursor", {"cursor": "some next cursor"}), + (Metafields, "some next cursor", {"cursor": "some next cursor"}), + (Products, "some next cursor", {"cursor": "some next cursor"}), + (Shop, "some next cursor", {"cursor": "some next cursor"}), ], ) - def test_next_page_token(self, config, stream_cls, rec_limit, requests_mock, expected): + def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): stream = stream_cls(config, authenticator=None) - stream.limit = rec_limit + stream.limit = 2 url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=self.generate_records(stream.name, rec_limit)) + response = {"next_cursor": cursor_response, stream.name: self.generate_records(stream.name, 2)} + requests_mock.get(url, json=response) response = requests.get(url) assert stream.next_page_token(response) == expected @pytest.mark.parametrize( "stream_cls, next_page_token, stream_state, stream_slice, expected", [ - (Collections, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Metafields, {"page": 2}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "page": 2}), - (Products, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Shop, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + (Collections, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Metafields, {"cursor": "12353"}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "cursor": "12353"}), + (Products, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Shop, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z",}), ], ) def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): @@ -287,38 +288,39 @@ def test_cursor_field(self, config, stream_cls, expected): assert result == expected @pytest.mark.parametrize( - "stream_cls, rec_limit, expected", + "stream_cls, cursor_response, expected", [ - (Addresses, 1, {"page": 2}), - (Charges, 2, {"page": 2}), - (Customers, 1, {"page": 2}), - (Discounts, 1, {"page": 2}), - (Onetimes, 1, {"page": 2}), - (Orders, 1, {"page": 2}), - (Subscriptions, 1, {"page": 2}), + (Addresses, "some next cursor", {"cursor": "some next cursor"}), + (Charges, "some next cursor", {"cursor": "some next cursor"}), + (Customers, "some next cursor", {"cursor": "some next cursor"}), + (Discounts, "some next cursor", {"cursor": "some next cursor"}), + (Onetimes, "some next cursor", {"cursor": "some next cursor"}), + (Orders, "some next cursor", {"cursor": "some next cursor"}), + (Subscriptions, "some next cursor", {"cursor": "some next cursor"}), ], ) - def test_next_page_token(self, config, stream_cls, rec_limit, requests_mock, expected): + def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): stream = stream_cls(config, authenticator=None) - stream.limit = rec_limit + stream.limit = 2 url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=self.generate_records(stream.name, rec_limit)) + response = {"next_cursor": cursor_response, stream.name: self.generate_records(stream.name, 2)} + requests_mock.get(url, json=response) response = requests.get(url) assert stream.next_page_token(response) == expected @pytest.mark.parametrize( "stream_cls, next_page_token, stream_state, stream_slice, expected", [ - (Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Charges, {"page": 2}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Onetimes, {"page": 2}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "page": 2, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), - (Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + (Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Charges, {"cursor": "123"}, {"updated_at": "2030-01-01"}, {}, + {"limit": 250, "cursor": "123"}), + (Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + (Onetimes, {"cursor": "123"}, {"updated_at": "2030-01-01"}, {}, + {"limit": 250, "cursor": "123"}), + (Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), (Subscriptions, None, {}, {}, - {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z", "updated_at_max": "2021-08-15T00:00:00Z"}), + {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), ], ) def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): diff --git a/airbyte-integrations/connectors/source-recreation/metadata.yaml b/airbyte-integrations/connectors/source-recreation/metadata.yaml index 64dbdc3ecf93..b217b6b65c04 100644 --- a/airbyte-integrations/connectors/source-recreation/metadata.yaml +++ b/airbyte-integrations/connectors/source-recreation/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-recreation/requirements.txt b/airbyte-integrations/connectors/source-recreation/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-recreation/requirements.txt +++ b/airbyte-integrations/connectors/source-recreation/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-recreation/setup.py b/airbyte-integrations/connectors/source-recreation/setup.py index 620d0949a599..26e4b5657f8a 100644 --- a/airbyte-integrations/connectors/source-recreation/setup.py +++ b/airbyte-integrations/connectors/source-recreation/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-recruitee/metadata.yaml b/airbyte-integrations/connectors/source-recruitee/metadata.yaml index ae5f3c69159e..42407ff998bf 100644 --- a/airbyte-integrations/connectors/source-recruitee/metadata.yaml +++ b/airbyte-integrations/connectors/source-recruitee/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-recruitee/requirements.txt b/airbyte-integrations/connectors/source-recruitee/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-recruitee/requirements.txt +++ b/airbyte-integrations/connectors/source-recruitee/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-recruitee/setup.py b/airbyte-integrations/connectors/source-recruitee/setup.py index 9dd7d2a0ecca..d2b12eff3c68 100644 --- a/airbyte-integrations/connectors/source-recruitee/setup.py +++ b/airbyte-integrations/connectors/source-recruitee/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-recurly/metadata.yaml b/airbyte-integrations/connectors/source-recurly/metadata.yaml index bed1f2b82e51..d10aee87e084 100644 --- a/airbyte-integrations/connectors/source-recurly/metadata.yaml +++ b/airbyte-integrations/connectors/source-recurly/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01 dockerImageTag: 0.4.1 dockerRepository: airbyte/source-recurly + documentationUrl: https://docs.airbyte.com/integrations/sources/recurly githubIssueLabel: source-recurly icon: recurly.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/recurly + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-recurly/requirements.txt b/airbyte-integrations/connectors/source-recurly/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-recurly/requirements.txt +++ b/airbyte-integrations/connectors/source-recurly/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-recurly/setup.py b/airbyte-integrations/connectors/source-recurly/setup.py index d866d9361d1b..1d278d40ec9f 100644 --- a/airbyte-integrations/connectors/source-recurly/setup.py +++ b/airbyte-integrations/connectors/source-recurly/setup.py @@ -8,9 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "recurly==4.10.0", "requests"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index 93906aaf119b..751c8c82da28 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.16 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index baa9c4244af6..d6722aa55388 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -35,3 +35,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-redshift/metadata.yaml b/airbyte-integrations/connectors/source-redshift/metadata.yaml index 22e0f144b942..373c5157645a 100644 --- a/airbyte-integrations/connectors/source-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/source-redshift/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: database connectorType: source definitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 - dockerImageTag: 0.3.16 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-redshift + documentationUrl: https://docs.airbyte.com/integrations/sources/redshift githubIssueLabel: source-redshift icon: redshift.svg - license: MIT + license: ELv2 name: Redshift registries: cloud: @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/redshift + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java index b805587ba2d9..204267aa0304 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java @@ -176,10 +176,10 @@ public AutoCloseableIterator read(final JsonNode config, }); } - private void validateCursorFieldForIncrementalTables( - final Map>> tableNameToTable, - final ConfiguredAirbyteCatalog catalog, - final Database database) + protected void validateCursorFieldForIncrementalTables( + final Map>> tableNameToTable, + final ConfiguredAirbyteCatalog catalog, + final Database database) throws SQLException { final List tablesWithInvalidCursor = new ArrayList<>(); for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { @@ -250,8 +250,7 @@ protected void estimateFullRefreshSyncSize(final Database database, /* no-op */ } - private List>> discoverWithoutSystemTables( - final Database database) + protected List>> discoverWithoutSystemTables(final Database database) throws Exception { final Set systemNameSpaces = getExcludedInternalNameSpaces(); final Set systemViews = getExcludedViews(); @@ -262,12 +261,12 @@ private List>> discoverWithoutSystemTables( Collectors.toList())); } - private List> getFullRefreshIterators( - final Database database, - final ConfiguredAirbyteCatalog catalog, - final Map>> tableNameToTable, - final StateManager stateManager, - final Instant emittedAt) { + protected List> getFullRefreshIterators( + final Database database, + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final StateManager stateManager, + final Instant emittedAt) { return getSelectedIterators( database, catalog, diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java index 977ecc43effc..67fd093f20bd 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java @@ -6,6 +6,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Collections; import java.util.Set; @@ -18,14 +19,17 @@ public class CdcStateManager { private final CdcState initialState; private final Set initialStreamsSynced; - + private final AirbyteStateMessage rawStateMessage; private CdcState currentState; - public CdcStateManager(final CdcState serialized, final Set initialStreamsSynced) { + public CdcStateManager(final CdcState serialized, + final Set initialStreamsSynced, + final AirbyteStateMessage stateMessage) { this.initialState = serialized; this.currentState = serialized; this.initialStreamsSynced = initialStreamsSynced; + this.rawStateMessage = stateMessage; LOGGER.info("Initialized CDC state with: {}", serialized); } @@ -37,6 +41,10 @@ public CdcState getCdcState() { return currentState != null ? Jsons.clone(currentState) : null; } + public AirbyteStateMessage getRawStateMessage() { + return rawStateMessage; + } + public Set getInitialStreamsSynced() { return initialStreamsSynced != null ? Collections.unmodifiableSet(initialStreamsSynced) : null; } diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java index 8f3c8b5b7fa6..2121a16a07ee 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java @@ -52,7 +52,18 @@ public AbstractStateManager(final ConfiguredAirbyteCatalog catalog, final Function> cursorFieldFunction, final Function cursorRecordCountFunction, final Function namespacePairFunction) { - cursorManager = new CursorManager(catalog, streamSupplier, cursorFunction, cursorFieldFunction, cursorRecordCountFunction, namespacePairFunction); + this(catalog, streamSupplier, cursorFunction, cursorFieldFunction, cursorRecordCountFunction, namespacePairFunction, false); + } + + public AbstractStateManager(final ConfiguredAirbyteCatalog catalog, + final Supplier> streamSupplier, + final Function cursorFunction, + final Function> cursorFieldFunction, + final Function cursorRecordCountFunction, + final Function namespacePairFunction, + final boolean onlyIncludeIncrementalStreams) { + cursorManager = new CursorManager(catalog, streamSupplier, cursorFunction, cursorFieldFunction, cursorRecordCountFunction, namespacePairFunction, + onlyIncludeIncrementalStreams); } @Override diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java index f8b32abb74ef..74a24cbdb8ce 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java @@ -9,13 +9,14 @@ import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -59,9 +60,11 @@ public CursorManager(final ConfiguredAirbyteCatalog catalog, final Function cursorFunction, final Function> cursorFieldFunction, final Function cursorRecordCountFunction, - final Function namespacePairFunction) { + final Function namespacePairFunction, + final boolean onlyIncludeIncrementalStreams) { pairToCursorInfo = createCursorInfoMap( - catalog, streamSupplier, cursorFunction, cursorFieldFunction, cursorRecordCountFunction, namespacePairFunction); + catalog, streamSupplier, cursorFunction, cursorFieldFunction, cursorRecordCountFunction, namespacePairFunction, + onlyIncludeIncrementalStreams); } /** @@ -89,15 +92,22 @@ protected Map createCursorInfoMap( final Function cursorFunction, final Function> cursorFieldFunction, final Function cursorRecordCountFunction, - final Function namespacePairFunction) { + final Function namespacePairFunction, + final boolean onlyIncludeIncrementalStreams) { final Set allStreamNames = catalog.getStreams() .stream() + .filter(c -> { + if (onlyIncludeIncrementalStreams) { + return c.getSyncMode() == SyncMode.INCREMENTAL; + } + return true; + }) .map(ConfiguredAirbyteStream::getStream) .map(AirbyteStreamNameNamespacePair::fromAirbyteStream) .collect(Collectors.toSet()); allStreamNames.addAll(streamSupplier.get().stream().map(namespacePairFunction).filter(Objects::nonNull).collect(Collectors.toSet())); - final Map localMap = new HashMap<>(); + final Map localMap = new ConcurrentHashMap<>(); final Map pairToState = streamSupplier.get() .stream() .collect(Collectors.toMap(namespacePairFunction, Function.identity())); diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java index 62e563e447ec..579a3cda805a 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java @@ -57,9 +57,10 @@ public GlobalStateManager(final AirbyteStateMessage airbyteStateMessage, final C CURSOR_FUNCTION, CURSOR_FIELD_FUNCTION, CURSOR_RECORD_COUNT_FUNCTION, - NAME_NAMESPACE_PAIR_FUNCTION); + NAME_NAMESPACE_PAIR_FUNCTION, + true); - this.cdcStateManager = new CdcStateManager(extractCdcState(airbyteStateMessage), extractStreams(airbyteStateMessage)); + this.cdcStateManager = new CdcStateManager(extractCdcState(airbyteStateMessage), extractStreams(airbyteStateMessage), airbyteStateMessage); } @Override diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java index d7831a90ee91..1fd17289f389 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java @@ -78,7 +78,7 @@ public LegacyStateManager(final DbState dbState, final ConfiguredAirbyteCatalog CURSOR_RECORD_COUNT_FUNCTION, NAME_NAMESPACE_PAIR_FUNCTION); - this.cdcStateManager = new CdcStateManager(dbState.getCdcState(), AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog)); + this.cdcStateManager = new CdcStateManager(dbState.getCdcState(), AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog), null); this.isCdc = dbState.getCdc(); if (dbState.getCdc() == null) { this.isCdc = false; @@ -101,7 +101,7 @@ public AirbyteStateMessage toState(final Optional deserializeInitialState(final JsonNode i final AirbyteStateType supportedStateType) { final Optional typedState = StateMessageHelper.getTypedState(initialStateJson, useStreamCapableState); - return typedState.map((state) -> { - switch (state.getStateType()) { - case GLOBAL: - return List.of(StateGeneratorUtils.convertStateMessage(state.getGlobal())); - case STREAM: - return state.getStateMessages() - .stream() - .map(stateMessage -> StateGeneratorUtils.convertStateMessage(stateMessage)).toList(); - case LEGACY: - default: - return List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY) - .withData(state.getLegacyState())); - } - }).orElse(generateEmptyInitialState(supportedStateType)); + return typedState + .map(state -> switch (state.getStateType()) { + case GLOBAL -> List.of(StateGeneratorUtils.convertStateMessage(state.getGlobal())); + case STREAM -> state.getStateMessages() + .stream() + .map(StateGeneratorUtils::convertStateMessage).toList(); + default -> List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY) + .withData(state.getLegacyState())); + }) + .orElse(generateEmptyInitialState(supportedStateType)); } /** diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java index eaa0fc5b2ae3..c1e6c9968552 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java @@ -152,7 +152,8 @@ private CursorManager createCursorManager(final String cursorFiel DbStreamState::getCursor, DbStreamState::getCursorField, CURSOR_RECORD_COUNT_FUNCTION, - s -> nameNamespacePair); + s -> nameNamespacePair, + false); } } diff --git a/airbyte-integrations/connectors/source-reply-io/metadata.yaml b/airbyte-integrations/connectors/source-reply-io/metadata.yaml index 1e688ec14839..9ad7fcafafd6 100644 --- a/airbyte-integrations/connectors/source-reply-io/metadata.yaml +++ b/airbyte-integrations/connectors/source-reply-io/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-reply-io/requirements.txt b/airbyte-integrations/connectors/source-reply-io/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-reply-io/requirements.txt +++ b/airbyte-integrations/connectors/source-reply-io/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-reply-io/setup.py b/airbyte-integrations/connectors/source-reply-io/setup.py index 3c5be0387df6..bc4d841b50fc 100644 --- a/airbyte-integrations/connectors/source-reply-io/setup.py +++ b/airbyte-integrations/connectors/source-reply-io/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-retently/.dockerignore b/airbyte-integrations/connectors/source-retently/.dockerignore index 47c68d9e8182..d01ec2be99ab 100644 --- a/airbyte-integrations/connectors/source-retently/.dockerignore +++ b/airbyte-integrations/connectors/source-retently/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_retently !setup.py diff --git a/airbyte-integrations/connectors/source-retently/.gitignore b/airbyte-integrations/connectors/source-retently/.gitignore deleted file mode 100644 index 76816864d564..000000000000 --- a/airbyte-integrations/connectors/source-retently/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.venv/ -__pycache__ -*.egg-info \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-retently/Dockerfile b/airbyte-integrations/connectors/source-retently/Dockerfile index c93ea6c86a96..b22977158fcc 100644 --- a/airbyte-integrations/connectors/source-retently/Dockerfile +++ b/airbyte-integrations/connectors/source-retently/Dockerfile @@ -1,13 +1,13 @@ -FROM python:3.9-slim as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder WORKDIR /airbyte/integration_code # upgrade pip to the latest version -# RUN apk --no-cache upgrade \ -# && pip install --upgrade pip \ -# && apk --no-cache add tzdata +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base COPY setup.py ./ @@ -25,7 +25,7 @@ COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime RUN echo "Etc/UTC" > /etc/timezone # bash is installed for more convenient debugging. -# RUN apk --no-cache add bash +RUN apk --no-cache add bash # copy payload code only COPY main.py ./ @@ -34,5 +34,5 @@ COPY source_retently ./source_retently ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-retently diff --git a/airbyte-integrations/connectors/source-retently/README.md b/airbyte-integrations/connectors/source-retently/README.md index 43b270e7d38a..f9ff6721bd61 100644 --- a/airbyte-integrations/connectors/source-retently/README.md +++ b/airbyte-integrations/connectors/source-retently/README.md @@ -1,34 +1,10 @@ # Retently Source -This is the repository for the Retently source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/retently). +This is the repository for the Retently configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/retently). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/retently) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_retently/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/retently) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_retently/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source retently test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-retently:dev discover docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-retently:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-retently/__init__.py b/airbyte-integrations/connectors/source-retently/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml b/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml index 433b6b5bc9de..a6f87e59bebf 100644 --- a/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-retently/acceptance-test-config.yml @@ -4,28 +4,26 @@ connector_image: airbyte/source-retently:dev acceptance_tests: spec: tests: - - spec_path: "source_retently/spec.json" + - spec_path: "source_retently/spec.yaml" connection: tests: - config_path: "secrets/config.json" status: "succeed" - - config_path: "secrets/old_config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: "0.1.5" + disable_for_version: "0.1.6" basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: - - name: reports + - name: templates + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100755 --- a/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/__init__.py b/airbyte-integrations/connectors/source-retently/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-retently/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json index dcaeaf54b78e..926f5bfae510 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-retently/integration_tests/configured_catalog.json @@ -4,7 +4,7 @@ "stream": { "name": "campaigns", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -16,7 +16,7 @@ "stream": { "name": "reports", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -28,7 +28,7 @@ "stream": { "name": "customers", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -40,7 +40,7 @@ "stream": { "name": "nps", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -52,7 +52,7 @@ "stream": { "name": "feedback", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -64,7 +64,7 @@ "stream": { "name": "outbox", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -76,7 +76,7 @@ "stream": { "name": "companies", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] @@ -88,7 +88,7 @@ "stream": { "name": "templates", "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object" }, "supported_sync_modes": ["full_refresh"] diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-retently/integration_tests/invalid_config.json index ae8c7b6ef830..cd47fc880322 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-retently/integration_tests/invalid_config.json @@ -1,3 +1,5 @@ { - "api_key": "invalid-key" + "credentials": { + "api_key": "invalid_api_key" + } } diff --git a/airbyte-integrations/connectors/source-retently/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-retently/integration_tests/sample_config.json index 29efee59c2af..edf9b8e4cb69 100644 --- a/airbyte-integrations/connectors/source-retently/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-retently/integration_tests/sample_config.json @@ -1,3 +1,5 @@ { - "api_key": "" + "credentials": { + "api_key": "api_key" + } } diff --git a/airbyte-integrations/connectors/source-retently/metadata.yaml b/airbyte-integrations/connectors/source-retently/metadata.yaml index 8492870726ea..40818fb95bca 100644 --- a/airbyte-integrations/connectors/source-retently/metadata.yaml +++ b/airbyte-integrations/connectors/source-retently/metadata.yaml @@ -1,20 +1,27 @@ data: + allowedHosts: + hosts: + - "*" # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: db04ecd1-42e7-4115-9cec-95812905c626 - dockerImageTag: 0.1.6 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-retently githubIssueLabel: source-retently icon: retently.svg license: MIT name: Retently - registries: - cloud: - enabled: true - oss: - enabled: true releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/retently tags: - - language:python + - language:low-code + ab_internal: + sl: 100 + ql: 100 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-retently/requirements.txt b/airbyte-integrations/connectors/source-retently/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-retently/requirements.txt +++ b/airbyte-integrations/connectors/source-retently/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-retently/sample_files/config.json b/airbyte-integrations/connectors/source-retently/sample_files/config.json deleted file mode 100644 index 29efee59c2af..000000000000 --- a/airbyte-integrations/connectors/source-retently/sample_files/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "api_key": "" -} diff --git a/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json deleted file mode 100644 index d6c36509573e..000000000000 --- a/airbyte-integrations/connectors/source-retently/sample_files/configured_catalog.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "campaigns", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "companies", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "customers", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "feedback", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "outbox", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "reports", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "templates", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object" - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-retently/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-retently/sample_files/invalid_config.json deleted file mode 100644 index ae8c7b6ef830..000000000000 --- a/airbyte-integrations/connectors/source-retently/sample_files/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "api_key": "invalid-key" -} diff --git a/airbyte-integrations/connectors/source-retently/setup.py b/airbyte-integrations/connectors/source-retently/setup.py index c8288dc1a0f1..707ffcee736f 100644 --- a/airbyte-integrations/connectors/source-retently/setup.py +++ b/airbyte-integrations/connectors/source-retently/setup.py @@ -6,24 +6,23 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", - "responses~=0.13.3", ] setup( name="source_retently", - description="Source implementation for Retently", + description="Source implementation for Retently.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-retently/source_retently/__init__.py b/airbyte-integrations/connectors/source-retently/source_retently/__init__.py index ecbf190cf838..5e67e6349bf7 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/__init__.py +++ b/airbyte-integrations/connectors/source-retently/source_retently/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-retently/source_retently/components.py b/airbyte-integrations/connectors/source-retently/source_retently/components.py new file mode 100644 index 000000000000..064ca721a62d --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/components.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, Mapping + +from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator +from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator + + +@dataclass +class AuthenticatorRetently(DeclarativeAuthenticator): + config: Mapping[str, Any] + api_auth: ApiKeyAuthenticator + oauth: DeclarativeOauth2Authenticator + + def __new__(cls, api_auth, oauth, config, *args, **kwargs): + if config["credentials"]["api_key"]: + return api_auth + else: + return oauth diff --git a/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml b/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml new file mode 100644 index 000000000000..8a8a910691da --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/manifest.yaml @@ -0,0 +1,165 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "{{ parameters.path_extractor }}"] + selector_data: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.path_extractor }}"] + + oauth_authenticator: + type: OAuthAuthenticator + token_refresh_endpoint: https://app.retently.com/api/oauth/token + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: "{{ config['credentials']['client_secret'] }}" + refresh_token: "{{ config['credentials']['refresh_token'] }}" + api_authenticator: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "api_key={{ config['credentials']['api_key'] }}" + + requester: + type: HttpRequester + url_base: "https://app.retently.com/api/v2/" + http_method: "GET" + authenticator: + class_name: source_retently.components.AuthenticatorRetently + api_auth: "#/definitions/api_authenticator" + oauth: "#/definitions/oauth_authenticator" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "PageIncrement" + page_size: 20 + start_from_page: 1 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + campaigns_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_data" + paginator: + type: NoPagination + name: "campaigns" + primary_key: "id" + $parameters: + path_extractor: "campaigns" + path: "campaigns" + + companies_stream: + $ref: "#/definitions/base_stream" + name: "companies" + primary_key: "id" + $parameters: + path_extractor: "companies" + path: "companies" + + customers_stream: + $ref: "#/definitions/base_stream" + name: "customers" + primary_key: "id" + $parameters: + path_extractor: "subscribers" + path: "nps/customers" + + feedback_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "PageIncrement" + page_size: 10 + start_from_page: 1 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + name: "feedback" + primary_key: "id" + $parameters: + path_extractor: "responses" + path: "feedback" + + outbox_stream: + $ref: "#/definitions/base_stream" + name: "outbox" + $parameters: + path_extractor: "surveys" + path: "nps/outbox" + + reports_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_data" + paginator: + type: NoPagination + name: "reports" + $parameters: + path_extractor: "data" + path: "reports" + + nps_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_data" + paginator: + type: NoPagination + name: "nps" + $parameters: + path_extractor: "data" + path: "nps/score" + + templates_stream: + $ref: "#/definitions/base_stream" + name: "templates" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_data" + primary_key: "id" + $parameters: + path_extractor: "data" + path: "templates" + +streams: + - "#/definitions/campaigns_stream" + - "#/definitions/companies_stream" + - "#/definitions/customers_stream" + - "#/definitions/feedback_stream" + - "#/definitions/outbox_stream" + - "#/definitions/reports_stream" + - "#/definitions/nps_stream" + - "#/definitions/templates_stream" + +check: + type: CheckStream + stream_names: + - "customers" diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json index 60d56756541d..e53e925f0b44 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/campaigns.json @@ -1,26 +1,28 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": ["string", "null"] + "type": ["null", "string"] }, "isActive": { - "type": ["boolean", "null"] + "type": ["null", "boolean"] }, "templateId": { - "type": ["string", "null"] + "type": ["null", "string"] }, "metric": { - "type": ["string", "null"] + "type": ["null", "string"] }, "type": { - "type": ["string", "null"] + "type": ["null", "string"] }, "channel": { - "type": ["string", "null"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/companies.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/companies.json index 833cbe82a69e..a3b6502f82e6 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/companies.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/companies.json @@ -1,47 +1,50 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "createdDate": { - "type": ["string", "null"], - "format": "date-time" + "type": ["null", "string"], + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ" }, "domain": { - "type": ["string", "null"] + "type": ["null", "string"] }, "companyName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "industryName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "tags": { "type": "array", "items": { - "type": ["string", "null"] + "type": ["null", "string"] } }, "cxMetrics": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "NPS": { - "type": ["number", "null"] + "type": ["null", "number"] }, "CSAT": { - "type": ["number", "null"] + "type": ["null", "number"] }, "CES": { - "type": ["number", "null"] + "type": ["null", "number"] }, "STAR": { - "type": ["number", "null"] + "type": ["null", "number"] } } }, "contactsCount": { - "type": ["number", "null"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/customers.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/customers.json index 2e22c5bf3c26..9a3978edffdb 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/customers.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/customers.json @@ -1,52 +1,53 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "email": { - "type": ["string", "null"] + "type": ["null", "string"] }, "firstName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "lastName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "companyName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "companyId": { - "type": ["string", "null"] + "type": ["null", "string"] }, "tags": { "type": "array", "items": { - "type": ["string", "null"] + "type": ["null", "string"] } }, "createdDate": { - "type": ["string", "null"], - "format": "date-time" + "type": ["null", "string"], + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ" }, "properties": { "type": "array", "items": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "label": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "value": { - "type": ["string", "integer", "number", "boolean"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/feedback.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/feedback.json index bfc6d1fffe35..3fb767bf4f3b 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/feedback.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/feedback.json @@ -3,62 +3,120 @@ "type": "object", "additionalProperties": true, "properties": { - "companyId": { "type": ["null", "string"] }, - "id": { "type": ["null", "string"] }, - "customerId": { "type": ["null", "string"] }, - "email": { "type": ["null", "string"] }, - "firstName": { "type": ["null", "string"] }, - "lastName": { "type": ["null", "string"] }, - "companyName": { "type": ["null", "string"] }, - "jobTitle": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, + "id": { + "type": ["null", "string"] + }, + "customerId": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "companyName": { + "type": ["null", "string"] + }, + "companyId": { + "type": ["null", "string"] + }, + "jobTitle": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, "tags": { - "type": ["null", "array"], - "items": { "type": ["null", "string"] } + "type": "array", + "items": { + "type": "string" + } }, "customProps": { - "type": ["null", "array"], - "items": { "type": ["null", "object"] } - }, - "campaignId": { "type": ["null", "string"] }, - "campaignName": { "type": ["null", "string"] }, - "createdDate": { "type": ["null", "string"] }, - "score": { "type": ["null", "number"] }, - "comment": { "type": ["null", "string"] }, - "checkbox": { "type": ["null", "boolean"] }, + "type": "array", + "items": { + "type": "object" + } + }, + "campaignId": { + "type": ["null", "string"] + }, + "campaignName": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ" + }, + "score": { + "type": ["null", "number"] + }, + "comment": { + "type": ["null", "string"] + }, + "checkbox": { + "type": ["null", "boolean"] + }, "additionalQuestions": { - "type": ["null", "array"], - "items": { "type": ["null", "object"] } + "type": "array", + "items": { + "type": "object" + } + }, + "feedbackTopics": { + "type": "array", + "items": { + "type": "object" + } }, "feedbackTags": { - "type": ["null", "array"], - "items": { "type": ["null", "string"] } + "type": "array", + "items": { + "type": "string" + } }, "feedbackTagsNew": { - "type": ["null", "array"] - }, - "feedbackTopics": { - "type": ["null", "array"], + "type": "array", "items": { - "type": "object", - "properties": { - "name": { "type": ["null", "string"] }, - "sentiment": { "type": ["null", "string"] } - } + "type": "string" } }, "notes": { - "type": ["null", "array"], - "items": { "type": ["null", "object"] } - }, - "status": { "type": ["null", "string"] }, - "assigned": { "type": ["null", "string"] }, - "ratingCategory": { "type": ["null", "string"] }, - "resolved": { "type": ["null", "boolean"] }, - "channel": { "type": ["null", "string"] }, - "metricsType": { "type": ["null", "string"] }, - "isBogus": { "type": ["null", "boolean"] } + "type": "array", + "items": { + "type": "object" + } + }, + "status": { + "type": ["string", "null"] + }, + "assigned": { + "type": ["null", "string"] + }, + "ratingCategory": { + "type": ["null", "string"] + }, + "resolved": { + "type": ["null", "boolean"] + }, + "channel": { + "type": ["null", "string"] + }, + "metricsType": { + "type": ["null", "string"] + }, + "isBogus": { + "type": ["null", "boolean"] + } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/nps.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/nps.json index 0b4fc120669e..ae10c0fd4f13 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/nps.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/nps.json @@ -1,35 +1,37 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "score": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "scoreSum": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "metricsType": { - "type": ["string", "null"] + "type": ["null", "string"] }, "promoters": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "passives": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "detractors": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "promotersCount": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "passivesCount": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "detractorsCount": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "totalResponses": { - "type": ["integer", "null"] + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json index 205ff6b6d304..7d1c766e463c 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/outbox.json @@ -3,104 +3,127 @@ "type": "object", "additionalProperties": true, "properties": { - "attributes": { - "type": "object", - "additionalProperties": true, - "properties": {} - }, "email": { - "type": ["string", "null"] + "type": ["null", "string"] }, "customerId": { - "type": ["string", "null"] + "type": ["null", "string"] }, "firstName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "lastName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "companyName": { - "type": ["string", "null"] + "type": ["null", "string"] }, "companyId": { - "type": ["string", "null"] - }, - "personTags": { - "type": ["array", "null"] + "type": ["null", "string"] }, "sentDate": { - "type": ["string", "null"], - "format": "date-time" + "type": ["null", "string"], + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ" }, "channel": { - "type": ["string", "null"] + "type": ["null", "string"] + }, + "personTags": { + "type": ["array", "null"], + "items": { "type": "string" } }, "campaign": { - "type": ["string", "null"] + "type": ["null", "string"] }, "campaignId": { - "type": ["string", "null"] + "type": ["null", "string"] }, "surveyTemplateId": { - "type": ["string", "null"] + "type": ["null", "string"] }, "subject": { - "type": ["string", "null"] + "type": ["null", "string"] }, "sentBy": { - "type": ["string", "null"] + "type": ["null", "string"] }, "status": { - "type": ["string", "null"] + "type": ["null", "string"] + }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "customerTags": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "customProps": { + "type": ["null", "array"], + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "label": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } }, "detailedStatus": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "isOpened": { - "type": ["boolean", "null"] + "type": ["null", "boolean"] }, "openedDate": { - "type": ["null", "string"], - "format": "date-time" + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ", + "type": ["null", "string"] }, "isResponded": { - "type": ["boolean", "null"] + "type": ["null", "boolean"] }, "respondedDate": { - "type": ["null", "string"], - "format": "date-time" + "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ", + "type": ["null", "string"] }, "hasFeedback": { - "type": ["boolean", "null"] + "type": ["null", "boolean"] }, "isOptedOut": { - "type": ["boolean", "null"] + "type": ["null", "boolean"] }, "isBounced": { - "type": ["boolean", "null"] + "type": ["null", "boolean"] } } }, + "mandrillMessageId": { + "type": ["null", "string"] + }, "additionalRecipients": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "object", - "properties": { - "email": { - "type": ["string", "null"] - }, - "type": { - "type": ["string", "null"] - }, - "mandrillMessageId": { - "type": ["string", "null"] - } - } + "type": ["null", "object"], + "additionalProperties": true + }, + "mandrillMessageId": { + "type": ["null", "string"] } - }, - "mandrillMessageId": { - "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/reports.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/reports.json index 2df244297a37..2c8139e479f5 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/reports.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/reports.json @@ -4,77 +4,81 @@ "additionalProperties": true, "properties": { "campaignId": { - "type": "string" + "type": ["null", "string"] }, - "trend": { - "type": "array", + "questionsStats": { + "type": ["null", "array"], "items": { "type": "object", + "additionalProperties": true + } + }, + "trend": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, "properties": { "day": { - "type": ["string", "null"] + "type": ["null", "string"] }, "promoters": { - "type": ["number", "null"] + "type": ["null", "number"] }, "passives": { - "type": ["number", "null"] + "type": ["null", "number"] }, "detractors": { - "type": ["number", "null"] + "type": ["null", "number"] }, "total": { - "type": ["number", "null"] + "type": ["null", "number"] }, "score": { - "type": ["number", "null"] + "type": ["null", "number"] } } } }, "last": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "promoters": { - "type": ["number", "null"] + "type": ["null", "number"] }, "passives": { - "type": ["number", "null"] + "type": ["null", "number"] }, "detractors": { - "type": ["number", "null"] + "type": ["null", "number"] }, "total": { - "type": ["number", "null"] + "type": ["null", "number"] }, "score": { - "type": ["number", "null"] + "type": ["null", "number"] } } }, - "questionsStats": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, "deliveryStats": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "totalCount": { - "type": ["number", "null"] + "type": ["null", "number"] }, "opened": { - "type": ["number", "null"] + "type": ["null", "number"] }, "responded": { - "type": ["number", "null"] + "type": ["null", "number"] }, "optedOut": { - "type": ["number", "null"] + "type": ["null", "number"] }, "isBounced": { - "type": ["number", "null"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json b/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json index 076593b921cf..a830f55244d0 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json +++ b/airbyte-integrations/connectors/source-retently/source_retently/schemas/templates.json @@ -1,17 +1,19 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": ["string", "null"] + "type": ["null", "string"] }, "name": { - "type": ["string", "null"] + "type": ["null", "string"] }, "channel": { - "type": ["string", "null"] + "type": ["null", "string"] }, "metric": { - "type": ["string", "null"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-retently/source_retently/source.py b/airbyte-integrations/connectors/source-retently/source_retently/source.py index d81fed994766..2443de44572a 100644 --- a/airbyte-integrations/connectors/source-retently/source_retently/source.py +++ b/airbyte-integrations/connectors/source-retently/source_retently/source.py @@ -2,248 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import math -from abc import abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator, TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -BASE_URL = "https://app.retently.com/api/v2/" +WARNING: Do not modify this file. +""" -class SourceRetently(AbstractSource): - @staticmethod - def get_authenticator(config): - credentials = config.get("credentials", {}) - if credentials and "client_id" in credentials: - return Oauth2Authenticator( - token_refresh_endpoint="https://app.retently.com/api/oauth/token", - client_id=credentials["client_id"], - client_secret=credentials["client_secret"], - refresh_token=credentials["refresh_token"], - ) - - api_key = credentials.get("api_key", config.get("api_key")) - if not api_key: - raise Exception("Config validation error: 'api_key' is a required property") - auth_method = f"api_key={api_key}" - return TokenAuthenticator(token="", auth_method=auth_method) - - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - auth = self.get_authenticator(config) - - # NOTE: not all retently instances have companies - stream = Customers(auth) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as e: - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = self.get_authenticator(config) - - return [ - Campaigns(auth), - Companies(auth), - Customers(auth), - Feedback(auth), - Outbox(auth), - Reports(auth), - Nps(auth), - Templates(auth), - ] - - -class RetentlyStream(HttpStream): - primary_key = None - - url_base = BASE_URL - - @property - @abstractmethod - def json_path(self): - pass - - def parse_response( - self, - response: requests.Response, - **kwargs, - ) -> Iterable[Mapping]: - data = response.json().get("data") - stream_data = data.get(self.json_path) if self.json_path else data - yield from stream_data - - @staticmethod - def convert_empty_string_to_null(record: MutableMapping[str, Any], parent_key: str, field_key: str): - """ - Converts empty strings to null in the specified field of the record. - """ - if record.get(parent_key, {}).get(field_key, "") == "": - record[parent_key][field_key] = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - json = response.json().get("data", dict()) - total = json.get("total") - limit = json.get("limit") - page = json.get("page") - if total and limit and page: - pages = math.ceil(total / limit) - if page < pages: - return {"page": page + 1} - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page = next_page_token or {} - return { - # The companies endpoint only supports limit 100 - "limit": 1000 if self.json_path != "companies" else 100, - **next_page, - } - - -class Campaigns(RetentlyStream): - json_path = "campaigns" - - def path(self, **kwargs) -> str: - return "campaigns" - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - data = response.json() - stream_data = data.get(self.json_path) if self.json_path else data - for d in stream_data: - yield d - - # does not support pagination - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - -class Companies(RetentlyStream): - json_path = "companies" - - def path( - self, - **kwargs, - ) -> str: - return "companies" - - -class Customers(RetentlyStream): - json_path = "subscribers" - - def path( - self, - **kwargs, - ) -> str: - return "nps/customers" - - -class Feedback(RetentlyStream): - json_path = "responses" - - def path( - self, - **kwargs, - ) -> str: - return "feedback" - - -class Outbox(RetentlyStream): - json_path = "surveys" - - def path( - self, - **kwargs, - ) -> str: - return "nps/outbox" - - def parse_response( - self, - response: requests.Response, - **kwargs, - ) -> Iterable[Mapping]: - data = response.json().get("data") - stream_data = data.get(self.json_path) if self.json_path else data - for record in stream_data: - self.convert_empty_string_to_null(record, "detailedStatus", "openedDate") - self.convert_empty_string_to_null(record, "detailedStatus", "respondedDate") - yield record - - -class Reports(RetentlyStream): - json_path = None - - def path( - self, - **kwargs, - ) -> str: - return "reports" - - # does not support pagination - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - -class Nps(RetentlyStream): - json_path = None - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "nps/score" - - # does not support pagination - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def parse_response( - self, - response: requests.Response, - **kwargs, - ) -> Iterable[Mapping]: - yield response.json().get("data") - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - return {} - - -class Templates(RetentlyStream): - json_path = "templates" - - def path( - self, - **kwargs, - ) -> str: - return "templates" - - def parse_response( - self, - response: requests.Response, - **kwargs, - ) -> Iterable[Mapping]: - data = response.json() - stream_data = data.get(self.json_path) if self.json_path else data - yield from stream_data +# Declarative Source +class SourceRetently(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-retently/source_retently/spec.json b/airbyte-integrations/connectors/source-retently/source_retently/spec.json deleted file mode 100644 index bec3525fa185..000000000000 --- a/airbyte-integrations/connectors/source-retently/source_retently/spec.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/retently", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Retently Api Spec", - "type": "object", - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authentication Mechanism", - "description": "Choose how to authenticate to Retently", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via Retently (OAuth)", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Retently developer application." - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Retently developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "Retently Refresh Token which can be used to fetch new Bearer Tokens when the current one expires.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "Authenticate with API Token", - "required": ["api_key"], - "additionalProperties": true, - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "order": 0 - }, - "api_key": { - "title": "API Token", - "description": "Retently API Token. See the docs for more information on how to obtain this key.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-retently/source_retently/spec.yaml b/airbyte-integrations/connectors/source-retently/source_retently/spec.yaml new file mode 100644 index 000000000000..92b3bb578366 --- /dev/null +++ b/airbyte-integrations/connectors/source-retently/source_retently/spec.yaml @@ -0,0 +1,96 @@ +--- +documentationUrl: https://docs.airbyte.com/integrations/sources/retently +connectionSpecification: + "$schema": http://json-schema.org/draft-07/schema# + title: Retently Api Spec + type: object + additionalProperties: true + properties: + credentials: + title: Authentication Mechanism + description: Choose how to authenticate to Retently + type: object + oneOf: + - type: object + title: Authenticate via Retently (OAuth) + required: + - client_id + - client_secret + - refresh_token + additionalProperties: true + properties: + auth_type: + type: string + const: Client + order: 0 + client_id: + title: Client ID + type: string + description: The Client ID of your Retently developer application. + client_secret: + title: Client Secret + type: string + description: The Client Secret of your Retently developer application. + airbyte_secret: true + refresh_token: + title: Refresh Token + type: string + description: + Retently Refresh Token which can be used to fetch new Bearer + Tokens when the current one expires. + airbyte_secret: true + - type: object + title: Authenticate with API Token + required: + - api_key + additionalProperties: true + properties: + auth_type: + type: string + const: Token + order: 0 + api_key: + title: API Token + description: + Retently API Token. See the docs + for more information on how to obtain this key. + type: string + airbyte_secret: true +advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - credentials + - auth_type + predicate_value: Client + oauth_config_specification: + complete_oauth_output_specification: + type: object + additionalProperties: true + properties: + refresh_token: + type: string + path_in_connector_config: + - credentials + - refresh_token + complete_oauth_server_input_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + path_in_connector_config: + - credentials + - client_id + client_secret: + type: string + path_in_connector_config: + - credentials + - client_secret diff --git a/airbyte-integrations/connectors/source-retently/unit_tests/__init__.py b/airbyte-integrations/connectors/source-retently/unit_tests/__init__.py deleted file mode 100644 index 9db886e0930f..000000000000 --- a/airbyte-integrations/connectors/source-retently/unit_tests/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# diff --git a/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py b/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py deleted file mode 100644 index 948fcbbb4a47..000000000000 --- a/airbyte-integrations/connectors/source-retently/unit_tests/test_source.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_retently.source import SourceRetently - - -def setup_responses(): - responses.add( - responses.GET, - "https://app.retently.com/api/v2/nps/customers", - json={"data": {"subscribers": [{}]}}, - ) - - -@responses.activate -def test_check_connection(mocker): - setup_responses() - source = SourceRetently() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceRetently() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 8 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py deleted file mode 100644 index 8778a26c8a4a..000000000000 --- a/airbyte-integrations/connectors/source-retently/unit_tests/test_streams.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_retently.source import Campaigns, Companies - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(Companies, "path", "v0/example_endpoint") - mocker.patch.object(Companies, "primary_key", "test_primary_key") - mocker.patch.object(Companies, "__abstractmethods__", set()) - - -def test_request_params_companies(patch_base_class): - stream = Companies() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {'limit': 100} - assert stream.request_params(**inputs) == expected_params - - -def test_request_params_other(patch_base_class): - stream = Campaigns() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {'limit': 1000} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = Companies() - resp = json.loads(json.dumps({"data": {"limit": 20, "total": 10, "page": 1}})) - inputs = {"response": MagicMock(json=MagicMock(return_value=resp))} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - resp = json.loads(json.dumps({"data": {"limit": 20, "total": 30, "page": 1}})) - inputs = {"response": MagicMock(json=MagicMock(return_value=resp))} - expected_token = {"page": 2} - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = Companies() - resp = json.loads(json.dumps({"data": {"limit": 20, "total": 10, "page": 1, "companies": [{"companyName": "foo"}]}})) - inputs = {"response": MagicMock(json=MagicMock(return_value=resp)), "stream_state": {}} - expected_parsed_object = {"companyName": "foo"} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = Companies() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - assert stream.request_headers(**inputs) == {} - - -def test_http_method(patch_base_class): - stream = Companies() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = Companies() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = Companies() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-ringcentral/metadata.yaml b/airbyte-integrations/connectors/source-ringcentral/metadata.yaml index 35ccc1e84eb7..e92f20f242b9 100644 --- a/airbyte-integrations/connectors/source-ringcentral/metadata.yaml +++ b/airbyte-integrations/connectors/source-ringcentral/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-ringcentral/requirements.txt b/airbyte-integrations/connectors/source-ringcentral/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-ringcentral/requirements.txt +++ b/airbyte-integrations/connectors/source-ringcentral/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-ringcentral/setup.py b/airbyte-integrations/connectors/source-ringcentral/setup.py index 16d39413b9e3..a0a34d080b35 100644 --- a/airbyte-integrations/connectors/source-ringcentral/setup.py +++ b/airbyte-integrations/connectors/source-ringcentral/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/manifest.yaml b/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/manifest.yaml index e4b41af35478..51959c696cbe 100644 --- a/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/manifest.yaml +++ b/airbyte-integrations/connectors/source-ringcentral/source_ringcentral/manifest.yaml @@ -134,7 +134,7 @@ definitions: extractorPath: "greetings" $ref: "#/definitions/base_stream_without_pagination" name: "call_record_settings" - + greetings_stream: $parameters: path: "/dictionary/greeting" diff --git a/airbyte-integrations/connectors/source-rki-covid/metadata.yaml b/airbyte-integrations/connectors/source-rki-covid/metadata.yaml index 738e3864e571..fb3acfd90923 100644 --- a/airbyte-integrations/connectors/source-rki-covid/metadata.yaml +++ b/airbyte-integrations/connectors/source-rki-covid/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/rki-covid tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-rki-covid/requirements.txt b/airbyte-integrations/connectors/source-rki-covid/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-rki-covid/requirements.txt +++ b/airbyte-integrations/connectors/source-rki-covid/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-rki-covid/setup.py b/airbyte-integrations/connectors/source-rki-covid/setup.py index b19979c603be..4c8abfd77674 100644 --- a/airbyte-integrations/connectors/source-rki-covid/setup.py +++ b/airbyte-integrations/connectors/source-rki-covid/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "airbyte-cdk"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "airbyte-cdk"] setup( name="source_rki_covid", diff --git a/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml b/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml index ea0b598a0986..b78e01397580 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-rocket-chat/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-rocket-chat/requirements.txt b/airbyte-integrations/connectors/source-rocket-chat/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/requirements.txt +++ b/airbyte-integrations/connectors/source-rocket-chat/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-rocket-chat/setup.py b/airbyte-integrations/connectors/source-rocket-chat/setup.py index 20fd98636627..86e7d41371ab 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/setup.py +++ b/airbyte-integrations/connectors/source-rocket-chat/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-rss/metadata.yaml b/airbyte-integrations/connectors/source-rss/metadata.yaml index afb8733fdabd..50a3f9531e7b 100644 --- a/airbyte-integrations/connectors/source-rss/metadata.yaml +++ b/airbyte-integrations/connectors/source-rss/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/rss tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-rss/requirements.txt b/airbyte-integrations/connectors/source-rss/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-rss/requirements.txt +++ b/airbyte-integrations/connectors/source-rss/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-rss/setup.py b/airbyte-integrations/connectors/source-rss/setup.py index 55b2f153b2ed..7bd87cde621b 100644 --- a/airbyte-integrations/connectors/source-rss/setup.py +++ b/airbyte-integrations/connectors/source-rss/setup.py @@ -8,9 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "feedparser~=6.0.10", "pytz~=2022.6"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index f3afd21f8c38..fe5dfacb3d59 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.0.0 +LABEL io.airbyte.version=3.1.9 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml index d60cb5ed6913..38a38eac75d6 100644 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml @@ -1,114 +1,186 @@ acceptance_tests: basic_read: tests: - - config_path: secrets/config.json - expect_records: - path: integration_tests/expected_records/csv.jsonl - timeout_seconds: 1800 - - config_path: secrets/parquet_config.json - expect_records: - path: integration_tests/expected_records/parquet.jsonl - timeout_seconds: 1800 - - config_path: secrets/parquet_dataset_config.json - expect_records: - path: integration_tests/expected_records/parquet_dataset.jsonl - timeout_seconds: 1800 - - config_path: secrets/avro_config.json - expect_records: - path: integration_tests/expected_records/avro.jsonl - timeout_seconds: 1800 - - config_path: secrets/jsonl_config.json - expect_records: - path: integration_tests/expected_records/jsonl.jsonl - timeout_seconds: 1800 - - config_path: secrets/jsonl_newlines_config.json - expect_records: - path: integration_tests/expected_records/jsonl_newlines.jsonl - timeout_seconds: 1800 + - config_path: secrets/config.json + expect_records: + path: integration_tests/expected_records/csv.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_custom_encoding_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_custom_encoding.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_custom_format_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_custom_format.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_user_schema_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_user_schema.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_no_header_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_no_header.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_skip_rows_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_skip_rows.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_skip_rows_no_header_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_with_nulls_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_with_nulls.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_csv_with_null_bools_config.json + expect_records: + path: integration_tests/expected_records/legacy_csv_with_null_bools.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/parquet_config.json + expect_records: + path: integration_tests/expected_records/parquet.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/parquet_dataset_config.json + expect_records: + path: integration_tests/expected_records/parquet_dataset.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/legacy_parquet_decimal_config.json + expect_records: + path: integration_tests/expected_records/legacy_parquet_decimal.jsonl + timeout_seconds: 1800 + - config_path: secrets/avro_config.json + expect_records: + path: integration_tests/expected_records/avro.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/jsonl_config.json + expect_records: + path: integration_tests/expected_records/jsonl.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/jsonl_newlines_config.json + expect_records: + path: integration_tests/expected_records/jsonl_newlines.jsonl + exact_order: true + timeout_seconds: 1800 connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: secrets/parquet_config.json - status: succeed - - config_path: secrets/avro_config.json - status: succeed - - config_path: secrets/jsonl_config.json - status: succeed - - config_path: secrets/jsonl_newlines_config.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: failed + - config_path: secrets/config.json + status: succeed + - config_path: secrets/legacy_csv_custom_encoding_config.json + status: succeed + - config_path: secrets/legacy_csv_custom_format_config.json + status: succeed + - config_path: secrets/legacy_csv_user_schema_config.json + status: succeed + - config_path: secrets/legacy_csv_no_header_config.json + status: succeed + - config_path: secrets/legacy_csv_skip_rows_config.json + status: succeed + - config_path: secrets/legacy_csv_skip_rows_no_header_config.json + status: succeed + - config_path: secrets/legacy_csv_with_nulls_config.json + status: succeed + - config_path: secrets/legacy_csv_with_null_bools_config.json + status: succeed + - config_path: secrets/parquet_config.json + status: succeed + - config_path: secrets/avro_config.json + status: succeed + - config_path: secrets/jsonl_config.json + status: succeed + - config_path: secrets/jsonl_newlines_config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed discovery: tests: - - config_path: secrets/config.json - - config_path: secrets/parquet_config.json - - config_path: secrets/avro_config.json - - config_path: secrets/jsonl_config.json - - config_path: secrets/jsonl_newlines_config.json + - config_path: secrets/config.json + - config_path: secrets/legacy_csv_custom_encoding_config.json + - config_path: secrets/legacy_csv_custom_format_config.json + - config_path: secrets/legacy_csv_user_schema_config.json + - config_path: secrets/legacy_csv_no_header_config.json + - config_path: secrets/legacy_csv_skip_rows_config.json + - config_path: secrets/legacy_csv_with_nulls_config.json + - config_path: secrets/parquet_config.json + - config_path: secrets/avro_config.json + - config_path: secrets/jsonl_config.json + - config_path: secrets/jsonl_newlines_config.json full_refresh: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalogs/csv.json - timeout_seconds: 1800 - - config_path: secrets/parquet_config.json - configured_catalog_path: integration_tests/configured_catalogs/parquet.json - timeout_seconds: 1800 - - config_path: secrets/avro_config.json - configured_catalog_path: integration_tests/configured_catalogs/avro.json - timeout_seconds: 1800 - - config_path: secrets/jsonl_config.json - configured_catalog_path: integration_tests/configured_catalogs/jsonl.json - timeout_seconds: 1800 - - config_path: secrets/jsonl_newlines_config.json - configured_catalog_path: integration_tests/configured_catalogs/jsonl.json - timeout_seconds: 1800 + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + timeout_seconds: 1800 + - config_path: secrets/parquet_config.json + configured_catalog_path: integration_tests/configured_catalogs/parquet.json + timeout_seconds: 1800 + - config_path: secrets/avro_config.json + configured_catalog_path: integration_tests/configured_catalogs/avro.json + timeout_seconds: 1800 + - config_path: secrets/jsonl_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + timeout_seconds: 1800 + - config_path: secrets/jsonl_newlines_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + timeout_seconds: 1800 incremental: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalogs/csv.json - cursor_paths: - test: - - _ab_source_file_last_modified - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 1800 - - config_path: secrets/parquet_config.json - configured_catalog_path: integration_tests/configured_catalogs/parquet.json - cursor_paths: - test: - - _ab_source_file_last_modified - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 1800 - - config_path: secrets/avro_config.json - configured_catalog_path: integration_tests/configured_catalogs/avro.json - cursor_paths: - test: - - _ab_source_file_last_modified - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 1800 - - config_path: secrets/jsonl_config.json - configured_catalog_path: integration_tests/configured_catalogs/jsonl.json - cursor_paths: - test: - - _ab_source_file_last_modified - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 1800 - - config_path: secrets/jsonl_newlines_config.json - configured_catalog_path: integration_tests/configured_catalogs/jsonl.json - cursor_paths: - test: - - _ab_source_file_last_modified - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 1800 + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + cursor_paths: + test: + - _ab_source_file_last_modified + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/parquet_config.json + configured_catalog_path: integration_tests/configured_catalogs/parquet.json + cursor_paths: + test: + - _ab_source_file_last_modified + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/avro_config.json + configured_catalog_path: integration_tests/configured_catalogs/avro.json + cursor_paths: + test: + - _ab_source_file_last_modified + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/jsonl_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + cursor_paths: + test: + - _ab_source_file_last_modified + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/jsonl_newlines_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + cursor_paths: + test: + - _ab_source_file_last_modified + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 spec: tests: - - spec_path: integration_tests/spec.json + - spec_path: integration_tests/spec.json connector_image: airbyte/source-s3:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/avro.json b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/avro.json index 631648d6329c..e8c3074c368a 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/avro.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/avro.json @@ -3,7 +3,30 @@ { "stream": { "name": "test", - "json_schema": {}, + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["integer", "null"] + }, + "fullname_and_valid": { + "type": ["object", "null"], + "fullname": { + "type": ["string", "null"] + }, + "valid": { + "type": ["boolean", "null"] + } + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["_ab_source_file_last_modified"] diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/csv.json b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/csv.json index 631648d6329c..ddf6ebecec11 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/csv.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/csv.json @@ -3,7 +3,27 @@ { "stream": { "name": "test", - "json_schema": {}, + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["_ab_source_file_last_modified"] diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/jsonl.json b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/jsonl.json index 631648d6329c..df9f183ae10e 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/jsonl.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/jsonl.json @@ -3,7 +3,33 @@ { "stream": { "name": "test", - "json_schema": {}, + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "value": { + "type": ["null", "number"] + }, + "event_date": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["_ab_source_file_last_modified"] diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/parquet.json b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/parquet.json index 631648d6329c..0434f0024405 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/parquet.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/configured_catalogs/parquet.json @@ -3,7 +3,66 @@ { "stream": { "name": "test", - "json_schema": {}, + "json_schema": { + "type": "object", + "properties": { + "Payroll_Number": { + "type": ["null", "number"] + }, + "Last_Name": { + "type": ["null", "string"] + }, + "First_Name": { + "type": ["null", "string"] + }, + "Mid_Init": { + "type": ["null", "string"] + }, + "Agency_Start_Date": { + "type": ["null", "string"] + }, + "Work_Location_Borough": { + "type": ["null", "number"] + }, + "Title_Description": { + "type": ["null", "string"] + }, + "Base_Salary": { + "type": ["null", "number"] + }, + "Regular_Hours": { + "type": ["null", "number"] + }, + "Regular_Gross_Paid": { + "type": ["null", "number"] + }, + "OT_Hours": { + "type": ["null", "number"] + }, + "Total_OT_Paid": { + "type": ["null", "number"] + }, + "Total_Other_Pay": { + "type": ["null", "number"] + }, + "Fiscal_Year": { + "type": ["null", "string"] + }, + "Leave_Status_as_of_June_30": { + "type": ["null", "string"] + }, + "Pay_Basis": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["_ab_source_file_last_modified"] diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl new file mode 100644 index 000000000000..9dcaefdf36a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl new file mode 100644 index 000000000000..d95d1648f7af --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl new file mode 100644 index 000000000000..f71905cdbd4a --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl new file mode 100644 index 000000000000..cb10832287ad --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl new file mode 100644 index 000000000000..a2ebfa4ff896 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl new file mode 100644 index 000000000000..81f1cc5b2d61 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}","_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]","dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array": "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl new file mode 100644 index 000000000000..18644e9dc903 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"id": 1, "name": "null", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "NULL", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl new file mode 100644 index 000000000000..efb0b16d65bd --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl new file mode 100644 index 000000000000..388b10b03112 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl @@ -0,0 +1,2 @@ +{"stream":"test","data":{"id":"row1","value": 12.345,"_ab_source_file_last_modified":"2023-08-06T18:05:00Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637642} +{"stream":"test","data":{"id":"row2","value": 67.89,"_ab_source_file_last_modified":"2023-08-06T18:05:00Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637643} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl index 9bcdb8471ba1..f9d66c2d9918 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl @@ -1,4 +1,15 @@ -{"stream": "test", "data": {"number": 1.0, "name": "foo", "flag": true, "delta": -1.0, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} -{"stream": "test", "data": {"number": 2.0, "name": null, "flag": false, "delta": 2.5, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} -{"stream": "test", "data": {"number": 3.0, "name": "bar", "flag": null, "delta": 0.1, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} -{"stream": "test", "data": {"number": null, "name": "baz", "flag": true, "delta": null, "_ab_source_file_last_modified": "2021-08-30T15:46:17Z", "_ab_source_file_url": "simple_test.parquet"}, "emitted_at": 1630795278000} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_encoded_as_cp1252.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_encoded_as_cp1252.csv new file mode 100644 index 000000000000..a26b937114da --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_encoded_as_cp1252.csv @@ -0,0 +1,9 @@ +id,name,valid +1,PVdhmjb1,False +2,j4DyXTS7,True +3,v0w8fTME,False +4,1q6jD8Np,False +5,77h4aiMP,True +6,Le35Wyic,True +7,xZhh1Kyl,False +8,M2t286iJ,False diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_with_nulls.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_with_nulls.csv new file mode 100644 index 000000000000..609d7f7658b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/csv_with_nulls.csv @@ -0,0 +1,9 @@ +id,name,valid +1,NULL,False +2,j4DyXTS7,True +3,NONE,False +4,1q6jD8Np,False +5,77h4aiMP,True +6,Le35Wyic,True +7,xZhh1Kyl,False +8,M2t286iJ,False diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/custom_format.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/custom_format.csv new file mode 100644 index 000000000000..712c9ca9238c --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/custom_format.csv @@ -0,0 +1,9 @@ +id|name|valid +1|PVdhmj@|b1|False +2|j4DyXTS7|True +3|v0w8fTME|False +4|1q6jD8Np|False +5|77h4aiMP|True +6|Le35Wyic|True +7|xZhh1Kyl|False +8|M2t286iJ|False diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/datatypes.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/datatypes.csv new file mode 100644 index 000000000000..4de8ace1eeb5 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/datatypes.csv @@ -0,0 +1,9 @@ +id,name,valid,valid_string,array,dict +1,PVdhmjb1,False,False,"[""a"", ""b"", ""c""]","{""key"": ""value""}" +2,j4DyXTS7,True,True,"[""a"", ""b""]","{""key"": ""value_with_comma\,""}" +3,v0w8fTME,False,False,"[""a""]","{""key"": ""value""}" +4,1q6jD8Np,False,False,"[]","{}" +5,77h4aiMP,True,True,"[""b"", ""c""]","{}" +6,Le35Wyic,True,True,"[""a"", ""c""]","{}" +7,xZhh1Kyl,False,False,"[""b""]","{}" +8,M2t286iJ,False,False,"[""c""]","{}" diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/no_header.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/no_header.csv new file mode 100644 index 000000000000..f8950c12a32f --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/no_header.csv @@ -0,0 +1,8 @@ +1,PVdhmjb1,False +2,j4DyXTS7,True +3,v0w8fTME,False +4,1q6jD8Np,False +5,77h4aiMP,True +6,Le35Wyic,True +7,xZhh1Kyl,False +8,M2t286iJ,False diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/skip_rows.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/skip_rows.csv new file mode 100644 index 000000000000..d9eb76ad29e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/skip_rows.csv @@ -0,0 +1,11 @@ +skip_this_line +id,name,valid +skip,this,line +1,PVdhmjb1,False +2,j4DyXTS7,True +3,v0w8fTME,False +4,1q6jD8Np,False +5,77h4aiMP,True +6,Le35Wyic,True +7,xZhh1Kyl,False +8,M2t286iJ,False diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/user_schema.csv b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/user_schema.csv new file mode 100644 index 000000000000..4de8ace1eeb5 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/sample_files/csvfile/user_schema.csv @@ -0,0 +1,9 @@ +id,name,valid,valid_string,array,dict +1,PVdhmjb1,False,False,"[""a"", ""b"", ""c""]","{""key"": ""value""}" +2,j4DyXTS7,True,True,"[""a"", ""b""]","{""key"": ""value_with_comma\,""}" +3,v0w8fTME,False,False,"[""a""]","{""key"": ""value""}" +4,1q6jD8Np,False,False,"[]","{}" +5,77h4aiMP,True,True,"[""b"", ""c""]","{}" +6,Le35Wyic,True,True,"[""a"", ""c""]","{}" +7,xZhh1Kyl,False,False,"[""b""]","{}" +8,M2t286iJ,False,False,"[""c""]","{}" diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index eb2b34cc93b1..f0dee45999a1 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -36,6 +36,7 @@ "properties": { "filetype": { "title": "Filetype", + "default": "csv", "const": "csv", "type": "string" }, @@ -122,6 +123,7 @@ "properties": { "filetype": { "title": "Filetype", + "default": "parquet", "const": "parquet", "type": "string" }, @@ -156,6 +158,7 @@ "properties": { "filetype": { "title": "Filetype", + "default": "avro", "const": "avro", "type": "string" } @@ -168,6 +171,7 @@ "properties": { "filetype": { "title": "Filetype", + "default": "jsonl", "const": "jsonl", "type": "string" }, diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/v4_abnormal_state.json b/airbyte-integrations/connectors/source-s3/integration_tests/v4_abnormal_state.json new file mode 100644 index 000000000000..629587fd335e --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/v4_abnormal_state.json @@ -0,0 +1,18 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "simple_test.csv": "2021-07-25T15:33:04.000000Z", + "simple_test_2.csv": "2021-12-20T12:35:05.000000Z", + "redshift_result.csv": "2022-05-26T09:17:47.000000Z", + "redshift_result_3.csv": "2022-05-26T09:55:15.000000Z", + "redshift_result_2.csv": "2022-05-26T09:55:16.000000Z" + }, + "_ab_source_file_last_modified": "2999-01-01T00:00:00.000000Z_redshift_result_2.csv" + }, + "stream_descriptor": { "name": "test" } + } + } +] diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index ecad07ed779f..d01dba0bc09b 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -5,11 +5,11 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 3.0.0 + dockerImageTag: 3.1.9 dockerRepository: airbyte/source-s3 githubIssueLabel: source-s3 icon: s3.svg - license: MIT + license: ELv2 name: S3 registries: cloud: @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/s3 tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-s3/requirements.txt b/airbyte-integrations/connectors/source-s3/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-s3/requirements.txt +++ b/airbyte-integrations/connectors/source-s3/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index 9629076d666c..241605f2aa64 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -6,10 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", - "pyarrow==9.0.0", + "airbyte-cdk>=0.50.2", + "pyarrow==12.0.1", "smart-open[s3]==5.1.0", - "wcmatch==8.2", + "wcmatch==8.4", "dill==0.3.4", "pytz", "fastavro==1.4.11", @@ -17,9 +17,10 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", - "pandas==1.3.1", + "pandas==2.0.3", "psutil", "pytest-order", "netifaces~=0.11.0", diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py index 95e4d829c542..d363f0fa8002 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py @@ -67,6 +67,15 @@ def _validate_field( if field_value in disallow_values: return f"{field_name} can not be {field_value}" + @staticmethod + def _validate_encoding(encoding: str) -> None: + try: + codecs.lookup(encoding) + except LookupError as e: + # UTF8 is the default encoding value, so there is no problem if `encoding` is not set manually + if encoding != "": + raise AirbyteTracedException(str(e), str(e), failure_type=FailureType.config_error) + @classmethod def _validate_options(cls, validator: Callable, options_name: str, format_: Mapping[str, Any]) -> Optional[str]: options = format_.get(options_name, "{}") @@ -98,10 +107,7 @@ def _validate_config(self, config: Mapping[str, Any]): if error_message: raise AirbyteTracedException(error_message, error_message, failure_type=FailureType.config_error) - try: - codecs.lookup(format_.get("encoding")) - except LookupError: - raise AirbyteTracedException(error_message, error_message, failure_type=FailureType.config_error) + self._validate_encoding(format_.get("encoding", "")) def _read_options(self) -> Mapping[str, str]: """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py index 9384886b2aab..7f098fd8a817 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/source.py @@ -90,7 +90,7 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: required to run this integration. """ # make dummy instance of stream_class in order to get 'supports_incremental' property - incremental = self.stream_class(dataset="", provider="", format="", path_pattern="").supports_incremental + incremental = self.stream_class(dataset="", provider={}, format="", path_pattern="").supports_incremental supported_dest_sync_modes = [DestinationSyncMode.overwrite] if incremental: diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py index fa1ab0885d10..e8110eb91191 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py @@ -5,6 +5,7 @@ import json from abc import ABC, abstractmethod +from collections import defaultdict from datetime import datetime, timedelta from functools import cached_property, lru_cache from traceback import format_exc @@ -49,8 +50,10 @@ class FileStream(Stream, ABC): # In version 2.0.1 the datetime format has been changed. Since the state may still store values in the old datetime format, # we need to support both of them for a while deprecated_datetime_format_string = "%Y-%m-%dT%H:%M:%S%z" + # Handle the datetime format used in V4, in the event that we need to roll back + v4_datetime_format_string = "%Y-%m-%dT%H:%M:%S.%fZ" - def __init__(self, dataset: str, provider: dict, format: dict, path_pattern: str, schema: str = None, start_date: str = None): + def __init__(self, dataset: str, provider: dict, format: dict, path_pattern: str, schema: str = None): """ :param dataset: table name for this stream :param provider: provider specific mapping as described in spec.json @@ -63,7 +66,7 @@ def __init__(self, dataset: str, provider: dict, format: dict, path_pattern: str self._provider = provider self._format = format self._user_input_schema: Dict[str, Any] = {} - self.start_date = pendulum.parse(start_date) if start_date else pendulum.from_timestamp(0) + self.start_date = pendulum.parse(provider.get("start_date")) if provider.get("start_date") else pendulum.from_timestamp(0) if schema: self._user_input_schema = self._parse_user_input_schema(schema) LOGGER.info(f"initialised stream with format: {format}") @@ -267,6 +270,7 @@ def _read_from_slice( """ for file_item in stream_slice["files"]: storage_file: StorageFile = file_item["storage_file"] + LOGGER.info(f"Reading from file: {storage_file.file_info}") try: with storage_file.open(file_reader.is_binary) as f: # TODO: make this more efficient than mutating every record one-by-one as they stream @@ -324,6 +328,7 @@ def _get_datetime_from_stream_state(self, stream_state: Mapping[str, Any] = None If there is no state, defaults to 1970-01-01 in order to pick up all files present. The datetime object is localized to UTC to match the timezone of the last_modified attribute of objects in S3. """ + stream_state = self._get_converted_stream_state(stream_state) if stream_state is not None and self.cursor_field in stream_state.keys(): try: state_datetime = datetime.strptime(stream_state[self.cursor_field], self.datetime_format_string) @@ -415,7 +420,6 @@ def stream_slices( yield from super().stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) else: - # logic here is to bundle all files with exact same last modified timestamp together in each slice prev_file_last_mod: datetime = None # init variable to hold previous iterations last modified grouped_files_by_time: List[Dict[str, Any]] = [] @@ -436,3 +440,86 @@ def stream_slices( else: # in case we have no files yield None + + def _is_v4_state_format(self, stream_state: Optional[dict]) -> bool: + """ + Returns True if the stream_state is in the v4 format, otherwise False. + + The stream_state is in the v4 format if the history dictionary is a map + of str to str (instead of str to list) and the cursor value is in the + format `%Y-%m-%dT%H:%M:%S.%fZ` + """ + if not stream_state: + return False + if history := stream_state.get("history"): + item = list(history.items())[0] + if isinstance(item[-1], str): + return True + else: + return False + if cursor := stream_state.get(self.cursor_field): + try: + datetime.strptime(cursor, self.v4_datetime_format_string) + except ValueError: + return False + else: + return True + return False + + def _get_converted_stream_state(self, stream_state: Optional[dict]) -> dict: + """ + Transform the history from the new format to the old. + + This will only be used in the event that we roll back from v4. + + e.g. + { + "stream_name": { + "history": { + "simple_test.csv": "2022-05-26T17:49:11.000000Z", + "simple_test_2.csv": "2022-05-27T01:01:01.000000Z", + "redshift_result.csv": "2022-05-27T04:22:20.000000Z", + ... + }, + "_ab_source_file_last_modified": "2022-05-27T04:22:20.000000Z_redshift_result.csv" + } + } + => + { + "stream_name": { + "history": { + "2022-05-26": ["simple_test.csv.csv"], + "2022-05-27": ["simple_test_2.csv", "redshift_result.csv"], + ... + } + }, + "_ab_source_file_last_modified": "2022-05-26T09:55:16Z" + } + """ + if not self._is_v4_state_format(stream_state): + return stream_state + + converted_history = defaultdict(list) + + for filename, timestamp in stream_state.get("history", {}).items(): + if date_str := self._get_ts_from_millis_ts(timestamp, "%Y-%m-%d"): + converted_history[date_str].append(filename) + + converted_state = {} + if self.cursor_field in stream_state: + timestamp_millis = stream_state[self.cursor_field].split("_")[0] + converted_state[self.cursor_field] = self._get_ts_from_millis_ts(timestamp_millis, self.datetime_format_string) + if "history" in stream_state: + converted_state["history"] = converted_history + + return converted_state + + def _get_ts_from_millis_ts(self, timestamp: Optional[str], output_format: str) -> Optional[str]: + if not timestamp: + return timestamp + try: + timestamp_millis = datetime.strptime(timestamp, self.v4_datetime_format_string) + except ValueError: + self.logger.warning(f"Unable to parse {timestamp} as v4 timestamp.") + return timestamp + return timestamp_millis.strftime(output_format) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/stream.py index 980b1b866a9f..6e5c0baaf7e7 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/stream.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/stream.py @@ -6,9 +6,12 @@ from typing import Any, Iterator, Mapping import pendulum +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException from boto3 import session as boto3session from botocore import UNSIGNED from botocore.config import Config +from botocore.exceptions import ClientError from source_s3.s3_utils import make_s3_client from .s3file import S3File @@ -25,6 +28,7 @@ def filepath_iterator(self, stream_state: Mapping[str, Any] = None) -> Iterator[ """ :yield: url filepath to use in S3File() """ + stream_state = self._get_converted_stream_state(stream_state) prefix = self._provider.get("path_prefix") if prefix is None: prefix = "" @@ -53,14 +57,17 @@ def filepath_iterator(self, stream_state: Mapping[str, Any] = None) -> Iterator[ ) # type: ignore[unreachable] else: kwargs = dict(Bucket=provider["bucket"], Prefix=provider.get("path_prefix", "")) - response = client.list_objects_v2(**kwargs) try: + response = client.list_objects_v2(**kwargs) content = response["Contents"] + except ClientError as e: + message = e.response.get("Error", {}).get("Message", {}) + raise AirbyteTracedException(message, message, failure_type=FailureType.config_error) except KeyError: pass else: for file in content: - if self.is_not_folder(file) and self.filter_by_last_modified_date(file, stream_state): + if self.is_not_folder(file) and self._filter_by_last_modified_date(file, stream_state): yield FileInfo(key=file["Key"], last_modified=file["LastModified"], size=file["Size"]) ctoken = response.get("NextContinuationToken", None) if not ctoken: @@ -70,7 +77,7 @@ def filepath_iterator(self, stream_state: Mapping[str, Any] = None) -> Iterator[ def is_not_folder(file) -> bool: return not file["Key"].endswith("/") - def filter_by_last_modified_date(self, file: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None): + def _filter_by_last_modified_date(self, file: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None): cursor_date = pendulum.parse(stream_state.get(self.cursor_field)) if stream_state else self.start_date file_in_history_and_last_modified_is_earlier_than_cursor_value = ( diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py new file mode 100644 index 000000000000..0869a3773d36 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/__init__.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from .config import Config +from .cursor import Cursor +from .legacy_config_transformer import LegacyConfigTransformer +from .source import SourceS3 +from .stream_reader import SourceS3StreamReader + +__all__ = ["Config", "Cursor", "LegacyConfigTransformer", "SourceS3", "SourceS3StreamReader"] diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py new file mode 100644 index 000000000000..231ff6935f5c --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Optional + +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from pydantic import AnyUrl, Field, ValidationError, root_validator + + +class Config(AbstractFileBasedSpec): + """ + NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes + because it is responsible for converting legacy S3 v3 configs into v4 configs using the File-Based CDK. + """ + + @classmethod + def documentation_url(cls) -> AnyUrl: + return AnyUrl("https://docs.airbyte.com/integrations/sources/s3", scheme="https") + + bucket: str = Field(title="Bucket", description="Name of the S3 bucket where the file(s) exist.", order=0) + + aws_access_key_id: Optional[str] = Field( + title="AWS Access Key ID", + default=None, + description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper " + "permissions. If accessing publicly available data, this field is not necessary.", + airbyte_secret=True, + order=1, + ) + + aws_secret_access_key: Optional[str] = Field( + title="AWS Secret Access Key", + default=None, + description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper " + "permissions. If accessing publicly available data, this field is not necessary.", + airbyte_secret=True, + order=2, + ) + + endpoint: Optional[str] = Field( + "", title="Endpoint", description="Endpoint to an S3 compatible service. Leave empty to use AWS.", order=4 + ) + + @root_validator + def validate_optional_args(cls, values): + aws_access_key_id = values.get("aws_access_key_id") + aws_secret_access_key = values.get("aws_secret_access_key") + if (aws_access_key_id or aws_secret_access_key) and not (aws_access_key_id and aws_secret_access_key): + raise ValidationError( + "`aws_access_key_id` and `aws_secret_access_key` are both required to authenticate with AWS.", model=Config + ) + return values diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py new file mode 100644 index 000000000000..0e6cf7528eee --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/cursor.py @@ -0,0 +1,162 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime, timedelta +from typing import Any, MutableMapping + +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor +from airbyte_cdk.sources.file_based.types import StreamState + +logger = logging.Logger("source-S3") + + +class Cursor(DefaultFileBasedCursor): + _DATE_FORMAT = "%Y-%m-%d" + _LEGACY_DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + _V4_MIGRATION_BUFFER = timedelta(hours=1) + _V3_MIN_SYNC_DATE_FIELD = "v3_min_sync_date" + + def __init__(self, stream_config: FileBasedStreamConfig, **_: Any): + super().__init__(stream_config) + self._running_migration = False + self._v3_migration_start_datetime = None + + def set_initial_state(self, value: StreamState) -> None: + if self._is_legacy_state(value): + self._running_migration = True + value = self._convert_legacy_state(value) + else: + self._running_migration = False + self._v3_migration_start_datetime = ( + datetime.strptime(value.get(Cursor._V3_MIN_SYNC_DATE_FIELD), DefaultFileBasedCursor.DATE_TIME_FORMAT) + if Cursor._V3_MIN_SYNC_DATE_FIELD in value + else None + ) + super().set_initial_state(value) + + def get_state(self) -> StreamState: + state = {"history": self._file_to_datetime_history, self.CURSOR_FIELD: self._get_cursor()} + if self._v3_migration_start_datetime: + return { + **state, + **{ + Cursor._V3_MIN_SYNC_DATE_FIELD: datetime.strftime( + self._v3_migration_start_datetime, DefaultFileBasedCursor.DATE_TIME_FORMAT + ) + }, + } + else: + return state + + def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool: + """ + Never sync files earlier than the v3 migration start date. V3 purged the history from the state, so we assume all files were already synced + Else if the currenty sync is migrating from v3 to v4, sync all files that were modified within one hour of the last sync + Else sync according to the default logic + """ + if self._v3_migration_start_datetime and file.last_modified < self._v3_migration_start_datetime: + return False + elif self._running_migration: + return True + else: + return super()._should_sync_file(file, logger) + + @staticmethod + def _is_legacy_state(value: StreamState) -> bool: + if not value: + return False + try: + # Verify datetime format in history + history = value.get("history", {}).keys() + if history: + item = list(value.get("history", {}).keys())[0] + datetime.strptime(item, Cursor._DATE_FORMAT) + + # verify the format of the last_modified cursor + last_modified_at_cursor = value.get(DefaultFileBasedCursor.CURSOR_FIELD) + if not last_modified_at_cursor: + return False + datetime.strptime(last_modified_at_cursor, Cursor._LEGACY_DATE_TIME_FORMAT) + except ValueError: + return False + return True + + @staticmethod + def _convert_legacy_state(legacy_state: StreamState) -> MutableMapping[str, Any]: + """ + Transform the history from the old state message format to the new. + + e.g. + { + "2022-05-26": ["simple_test.csv.csv", "simple_test_2.csv"], + "2022-05-27": ["simple_test_2.csv", "redshift_result.csv"], + ... + } + => + { + "simple_test.csv": "2022-05-26T00:00:00.000000Z", + "simple_test_2.csv": "2022-05-27T00:00:00.000000Z", + "redshift_result.csv": "2022-05-27T00:00:00.000000Z", + ... + } + """ + converted_history = {} + legacy_cursor = legacy_state[DefaultFileBasedCursor.CURSOR_FIELD] + cursor_datetime = datetime.strptime(legacy_cursor, Cursor._LEGACY_DATE_TIME_FORMAT) + logger.info(f"Converting v3 -> v4 state. v3_cursor={legacy_cursor} v3_history={legacy_state.get('history')}") + + for date_str, filenames in legacy_state.get("history", {}).items(): + datetime_obj = Cursor._get_adjusted_date_timestamp(cursor_datetime, datetime.strptime(date_str, Cursor._DATE_FORMAT)) + + for filename in filenames: + if filename in converted_history: + if datetime_obj > datetime.strptime( + converted_history[filename], + DefaultFileBasedCursor.DATE_TIME_FORMAT, + ): + converted_history[filename] = datetime_obj.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) + else: + # If the file was already synced with a later timestamp, ignore + pass + else: + converted_history[filename] = datetime_obj.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) + + if converted_history: + filename, _ = max(converted_history.items(), key=lambda x: (x[1], x[0])) + cursor = f"{cursor_datetime}_{filename}" + else: + # Having a cursor with empty history is not expected, but we handle it. + logger.warning(f"Cursor found without a history object; this is not expected. cursor_value={legacy_cursor}") + # Note: we convert to the v4 cursor granularity, but since no items are in the history we simply use the + # timestamp as the cursor value instead of the concatenation of timestamp_filename, which is the v4 + # cursor format. + # This is okay because the v4 cursor is kept for posterity but is not actually used in the v4 code. If we + # start to use the cursor we may need to revisit this logic. + cursor = cursor_datetime + converted_history = {} + v3_migration_start_datetime = cursor_datetime - Cursor._V4_MIGRATION_BUFFER + return { + "history": converted_history, + DefaultFileBasedCursor.CURSOR_FIELD: cursor, + Cursor._V3_MIN_SYNC_DATE_FIELD: v3_migration_start_datetime.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT), + } + + @staticmethod + def _get_adjusted_date_timestamp(cursor_datetime: datetime, file_datetime: datetime) -> datetime: + if file_datetime > cursor_datetime: + return file_datetime + else: + # Extract the dates so they can be compared + cursor_date = cursor_datetime.date() + date_obj = file_datetime.date() + + # If same day, update the time to the cursor time + if date_obj == cursor_date: + return file_datetime.replace(hour=cursor_datetime.hour, minute=cursor_datetime.minute, second=cursor_datetime.second) + # If previous, update the time to end of day + else: + return file_datetime.replace(hour=23, minute=59, second=59, microsecond=999999) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py new file mode 100644 index 000000000000..6a0897ae6c77 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py @@ -0,0 +1,162 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from datetime import datetime +from typing import Any, Dict, List, Mapping, Optional, Union + +from source_s3.source import SourceS3Spec +from source_s3.source_files_abstract.formats.avro_spec import AvroFormat +from source_s3.source_files_abstract.formats.csv_spec import CsvFormat +from source_s3.source_files_abstract.formats.jsonl_spec import JsonlFormat +from source_s3.source_files_abstract.formats.parquet_spec import ParquetFormat + +SECONDS_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +MICROS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class LegacyConfigTransformer: + """ + Class that takes in S3 source configs in the legacy format and transforms them into + configs that can be used by the new S3 source built with the file-based CDK. + """ + + @classmethod + def convert(cls, legacy_config: SourceS3Spec) -> Mapping[str, Any]: + transformed_config = { + "bucket": legacy_config.provider.bucket, + "streams": [ + { + "name": legacy_config.dataset, + "file_type": legacy_config.format.filetype, + "globs": cls._create_globs(legacy_config.path_pattern), + "legacy_prefix": legacy_config.provider.path_prefix, + "validation_policy": "Emit Record", + } + ], + } + + if legacy_config.provider.start_date: + transformed_config["start_date"] = cls._transform_seconds_to_micros(legacy_config.provider.start_date) + if legacy_config.provider.aws_access_key_id: + transformed_config["aws_access_key_id"] = legacy_config.provider.aws_access_key_id + if legacy_config.provider.aws_secret_access_key: + transformed_config["aws_secret_access_key"] = legacy_config.provider.aws_secret_access_key + if legacy_config.provider.endpoint: + transformed_config["endpoint"] = legacy_config.provider.endpoint + if legacy_config.user_schema and legacy_config.user_schema != "{}": + transformed_config["streams"][0]["input_schema"] = legacy_config.user_schema + if legacy_config.format: + transformed_config["streams"][0]["format"] = cls._transform_file_format(legacy_config.format) + + return transformed_config + + @classmethod + def _create_globs(cls, path_pattern: str) -> List[str]: + if "|" in path_pattern: + return path_pattern.split("|") + else: + return [path_pattern] + + @classmethod + def _transform_seconds_to_micros(cls, datetime_str: str) -> str: + try: + parsed_datetime = datetime.strptime(datetime_str, SECONDS_FORMAT) + return parsed_datetime.strftime(MICROS_FORMAT) + except ValueError as e: + raise ValueError("Timestamp could not be parsed when transforming legacy connector config") from e + + @classmethod + def _transform_file_format(cls, format_options: Union[CsvFormat, ParquetFormat, AvroFormat, JsonlFormat]) -> Mapping[str, Any]: + if isinstance(format_options, AvroFormat): + return {"filetype": "avro"} + elif isinstance(format_options, CsvFormat): + additional_reader_options = cls.parse_config_options_str("additional_reader_options", format_options.additional_reader_options) + advanced_options = cls.parse_config_options_str("advanced_options", format_options.advanced_options) + + csv_options = { + "filetype": "csv", + "delimiter": format_options.delimiter, + "quote_char": format_options.quote_char, + "double_quote": format_options.double_quote, + # values taken from https://github.com/apache/arrow/blob/43c05c56b37daa93e76b94bc3e6952d56d1ea3f2/cpp/src/arrow/csv/options.cc#L41-L45 + "null_values": additional_reader_options.pop( + "null_values", + [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "N/A", + "NA", + "NULL", + "NaN", + "n/a", + "nan", + "null", + ], + ), + "true_values": additional_reader_options.pop("true_values", ["1", "True", "TRUE", "true"]), + "false_values": additional_reader_options.pop("false_values", ["0", "False", "FALSE", "false"]), + "inference_type": "Primitive Types Only" if format_options.infer_datatypes else "None", + "strings_can_be_null": additional_reader_options.pop("strings_can_be_null", False), + } + + if format_options.escape_char: + csv_options["escape_char"] = format_options.escape_char + if format_options.encoding: + csv_options["encoding"] = format_options.encoding + if skip_rows := advanced_options.pop("skip_rows", None): + csv_options["skip_rows_before_header"] = skip_rows + if skip_rows_after_names := advanced_options.pop("skip_rows_after_names", None): + csv_options["skip_rows_after_header"] = skip_rows_after_names + if autogenerate_column_names := advanced_options.pop("autogenerate_column_names", None): + csv_options["autogenerate_column_names"] = autogenerate_column_names + + cls._filter_legacy_noops(advanced_options) + + if advanced_options or additional_reader_options: + raise ValueError( + "The config options you selected are no longer supported.\n" + f"advanced_options={advanced_options}" + if advanced_options + else "" + f"additional_reader_options={additional_reader_options}" + if additional_reader_options + else "" + ) + + return csv_options + + elif isinstance(format_options, JsonlFormat): + return {"filetype": "jsonl"} + elif isinstance(format_options, ParquetFormat): + return {"filetype": "parquet", "decimal_as_float": True} + else: + # This should never happen because it would fail schema validation + raise ValueError(f"Format filetype {format_options} is not a supported file type") + + @classmethod + def parse_config_options_str(cls, options_field: str, options_value: Optional[str]) -> Dict[str, Any]: + options_str = options_value or "{}" + try: + return json.loads(options_str) + except json.JSONDecodeError as error: + raise ValueError(f"Malformed {options_field} config json: {error}. Please ensure that it is a valid JSON.") + + @staticmethod + def _filter_legacy_noops(advanced_options: Dict[str, Any]): + ignore_all = ("auto_dict_encode", "timestamp_parsers") + ignore_by_value = (("check_utf8", False),) + + for option in ignore_all: + advanced_options.pop(option, None) + + for option, value_to_ignore in ignore_by_value: + if advanced_options.get(option) == value_to_ignore: + advanced_options.pop(option) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py new file mode 100644 index 000000000000..8302673b3ce7 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping + +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from source_s3.source import SourceS3Spec +from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer + + +class SourceS3(FileBasedSource): + def read_config(self, config_path: str) -> Mapping[str, Any]: + """ + Used to override the default read_config so that when the new file-based S3 connector processes a config + in the legacy format, it can be transformed into the new config. This happens in entrypoint before we + validate the config against the new spec. + """ + config = super().read_config(config_path) + if not config.get("streams"): + parsed_legacy_config = SourceS3Spec(**config) + return LegacyConfigTransformer.convert(parsed_legacy_config) + return config diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py new file mode 100644 index 000000000000..14618e84cff9 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py @@ -0,0 +1,159 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from contextlib import contextmanager +from io import IOBase +from typing import Iterable, List, Optional, Set + +import boto3.session +import pytz +import smart_open +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from botocore.client import BaseClient +from botocore.client import Config as ClientConfig +from source_s3.v4.config import Config + + +class SourceS3StreamReader(AbstractFileBasedStreamReader): + def __init__(self): + super().__init__() + self._s3_client = None + + @property + def config(self) -> Config: + return self._config + + @config.setter + def config(self, value: Config): + """ + FileBasedSource reads the config from disk and parses it, and once parsed, the source sets the config on its StreamReader. + + Note: FileBasedSource only requires the keys defined in the abstract config, whereas concrete implementations of StreamReader + will require keys that (for example) allow it to authenticate with the 3rd party. + + Therefore, concrete implementations of AbstractFileBasedStreamReader's config setter should assert that `value` is of the correct + config type for that type of StreamReader. + """ + assert isinstance(value, Config) + self._config = value + + @property + def s3_client(self) -> BaseClient: + if self.config is None: + # We shouldn't hit this; config should always get set before attempting to + # list or read files. + raise ValueError("Source config is missing; cannot create the S3 client.") + if self._s3_client is None: + if self.config.endpoint: + client_kv_args = _get_s3_compatible_client_args(self.config) + self._s3_client = boto3.client("s3", **client_kv_args) + else: + self._s3_client = boto3.client( + "s3", + aws_access_key_id=self.config.aws_access_key_id, + aws_secret_access_key=self.config.aws_secret_access_key, + ) + return self._s3_client + + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Get all files matching the specified glob patterns. + """ + s3 = self.s3_client + prefixes = [prefix] if prefix else self.get_prefixes_from_globs(globs) + seen = set() + total_n_keys = 0 + + try: + if prefixes: + for prefix in prefixes: + for remote_file in self._page(s3, globs, self.config.bucket, prefix, seen, logger): + total_n_keys += 1 + yield remote_file + else: + for remote_file in self._page(s3, globs, self.config.bucket, None, seen, logger): + total_n_keys += 1 + yield remote_file + + logger.info(f"Finished listing objects from S3. Found {total_n_keys} objects total ({len(seen)} unique objects).") + except Exception as exc: + raise ErrorListingFiles( + FileBasedSourceError.ERROR_LISTING_FILES, + source="s3", + bucket=self.config.bucket, + globs=globs, + endpoint=self.config.endpoint, + ) from exc + + @contextmanager + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + try: + params = {"client": self.s3_client} + except Exception as exc: + raise exc + + logger.debug(f"try to open {file.uri}") + try: + result = smart_open.open(f"s3://{self.config.bucket}/{file.uri}", transport_params=params, mode=mode.value, encoding=encoding) + except OSError: + logger.warning( + f"We don't have access to {file.uri}. The file appears to have become unreachable during sync." + f"Check whether key {file.uri} exists in `{self.config.bucket}` bucket and/or has proper ACL permissions" + ) + # see https://docs.python.org/3/library/contextlib.html#contextlib.contextmanager for why we do this + try: + yield result + finally: + result.close() + + @staticmethod + def _is_folder(file) -> bool: + return file["Key"].endswith("/") + + def _page( + self, s3: BaseClient, globs: List[str], bucket: str, prefix: Optional[str], seen: Set[str], logger: logging.Logger + ) -> Iterable[RemoteFile]: + """ + Page through lists of S3 objects. + """ + total_n_keys_for_prefix = 0 + kwargs = {"Bucket": bucket} + while True: + response = s3.list_objects_v2(Prefix=prefix, **kwargs) if prefix else s3.list_objects_v2(**kwargs) + key_count = response.get("KeyCount") + total_n_keys_for_prefix += key_count + logger.info(f"Received {key_count} objects from S3 for prefix '{prefix}'.") + + if "Contents" in response: + for file in response["Contents"]: + if self._is_folder(file): + continue + remote_file = RemoteFile(uri=file["Key"], last_modified=file["LastModified"].astimezone(pytz.utc).replace(tzinfo=None)) + if self.file_matches_globs(remote_file, globs) and remote_file.uri not in seen: + seen.add(remote_file.uri) + yield remote_file + else: + logger.warning(f"Invalid response from S3; missing 'Contents' key. kwargs={kwargs}.") + + if next_token := response.get("NextContinuationToken"): + kwargs["ContinuationToken"] = next_token + else: + logger.info(f"Finished listing objects from S3 for prefix={prefix}. Found {total_n_keys_for_prefix} objects.") + break + + +def _get_s3_compatible_client_args(config: Config) -> dict: + """ + Returns map of args used for creating s3 boto3 client. + """ + client_kv_args = { + "config": ClientConfig(s3={"addressing_style": "auto"}), + "endpoint_url": config.endpoint, + "use_ssl": True, + "verify": True, + } + return client_kv_args diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py index 2bc70bcffdbc..8113904a4098 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py @@ -7,11 +7,14 @@ import random import shutil import string +from contextlib import nullcontext as does_not_raise from pathlib import Path from typing import Any, List, Mapping, Tuple +from unittest.mock import Mock import pendulum import pytest +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from smart_open import open as smart_open from source_s3.source_files_abstract.file_info import FileInfo from source_s3.source_files_abstract.formats.csv_parser import CsvParser @@ -412,3 +415,16 @@ def test_big_file(self) -> None: read_count += 1 assert read_count == expected_count expected_file.close() + + @pytest.mark.parametrize( + "encoding, expectation", + ( + ("UTF8", does_not_raise()), + ("", does_not_raise()), + ("R2D2", pytest.raises(AirbyteTracedException)), + ) + ) + def test_encoding_validation(self, encoding, expectation) -> None: + parser = CsvParser(format=Mock(), master_schema=Mock()) + with expectation: + parser._validate_encoding(encoding) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py index 2eeaae885880..032808b526bb 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py @@ -22,6 +22,7 @@ def mock_big_size_object(): mock = MagicMock() mock.__sizeof__.return_value = 1000000001 + mock.items = lambda: [("2023-08-01", ["file1.txt", "file2.txt"])] return mock @@ -503,10 +504,9 @@ def test_filepath_iterator_date_filter(self, start_date, bucket, path_prefix, li with patch("source_s3.stream.make_s3_client", s3_client_mock): stream_instance = IncrementalFileStreamS3( dataset="dummy", - provider={"bucket": bucket, "path_prefix": path_prefix, **provider}, + provider={"bucket": bucket, "path_prefix": path_prefix, "start_date":start_date, **provider}, format={}, - path_pattern="**/prefix*.png", - start_date=start_date + path_pattern="**/prefix*.png" ) assert len(list(stream_instance.filepath_iterator())) == expected_files_count @@ -593,3 +593,145 @@ def test_migrate_datetime_format(self): path_pattern="**/prefix*.csv" ) assert stream_instance.get_updated_state(current_state, latest_record)["_ab_source_file_last_modified"] == "2022-11-09T11:12:00Z" + + @pytest.mark.parametrize( + "input_state, expected_output", + [ + pytest.param({}, {}, id="empty-input"), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file1.txt", + }, + { + "history": { + "2023-08-01": ["file1.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", + }, + id="single-file-single-timestamp", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T00:00:00.000000Z", + "file2.txt": "2023-08-01T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file2.txt", + }, + { + "history": { + "2023-08-01": ["file1.txt", "file2.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", + }, + id="multiple-files-same-timestamp", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T00:00:00.000000Z", + "file2.txt": "2023-08-02T00:00:00.000000Z", + "file3.txt": "2023-08-02T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-02T00:00:00.000000Z_file3.txt", + }, + { + "history": { + "2023-08-01": ["file1.txt"], + "2023-08-02": ["file2.txt", "file3.txt"], + }, + "_ab_source_file_last_modified": "2023-08-02T00:00:00Z", + }, + id="multiple-files-different-timestamps", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:30:00.000000Z", + "file2.txt": "2023-08-01T15:45:00.000000Z", + "file3.txt": "2023-08-02T02:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-02T00:00:00.000000Z_file3.txt", + }, + { + "history": { + "2023-08-01": ["file1.txt", "file2.txt"], + "2023-08-02": ["file3.txt"], + }, + "_ab_source_file_last_modified": "2023-08-02T00:00:00Z", + }, + id="handling-different-times", + ), + ], + ) + def test_get_converted_stream_state(self, input_state, expected_output): + assert IncrementalFileStreamS3(dataset="dummy", provider={}, format={}, path_pattern="")._get_converted_stream_state(input_state) == expected_output + + @pytest.mark.parametrize( + "stream_state, expected_output", + [ + pytest.param({}, False, id="empty-stream-state"), + pytest.param( + { + "history": { + "2023-08-01": "file1.txt", + "2023-08-02": "file2.txt", + "2023-08-03": "file3.txt", + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z", + }, + True, + id="v4-format-history-and-cursor", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt"], + "2023-08-02": ["file2.txt", "file3.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", + }, + False, + id="v3-format-history-and-cursor", + ), + pytest.param( + { + "history": { + "2023-08-01": "file1.txt", + "2023-08-02": "file2.txt", + }, + }, + True, + id="v4-missing-cursor", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt"], + "2023-08-02": ["file2.txt", "file3.txt"], + }, + }, + False, + id="v3-missing-cursor", + ), + pytest.param( + { + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z", + }, + True, + id="v4-cursor-only", + ), + pytest.param( + { + "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", # Invalid format (missing microseconds) + }, + False, + id="v3-cursor-only", + ), + ], + ) + def test_is_v4_state_format(self, stream_state: Mapping[str, Any], expected_output): + assert IncrementalFileStreamS3(dataset="dummy", provider={}, format={}, path_pattern="")._is_v4_state_format(stream_state) == expected_output diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/__init__.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py new file mode 100644 index 000000000000..76e326fd2a87 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging + +import pytest +from pydantic import ValidationError +from source_s3.v4.config import Config + +logger = logging.Logger("") + + +@pytest.mark.parametrize( + "kwargs,expected_error", + [ + pytest.param({"bucket": "test", "streams": []}, None, id="required-fields"), + pytest.param({"bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key"}, None, id="config-created-with-aws-info"), + pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, None, id="config-created-with-endpoint"), + pytest.param({"bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key", "endpoint": "http://test.com"}, None, id="config-created-with-endpoint-and-aws-info"), + pytest.param({"streams": []}, ValidationError, id="missing-bucket"), + ] +) +def test_config(kwargs, expected_error): + if expected_error: + with pytest.raises(expected_error): + Config(**kwargs) + else: + Config(**kwargs) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py new file mode 100644 index 000000000000..7e612ba89e64 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py @@ -0,0 +1,493 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timezone +from typing import Any, MutableMapping, Optional +from unittest.mock import Mock + +import pytest +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from source_s3.v4.cursor import Cursor + + +def _create_datetime(dt: str) -> datetime: + return datetime.strptime(dt, DefaultFileBasedCursor.DATE_TIME_FORMAT) + + +@pytest.mark.parametrize( + "input_state, expected_state", + [ + pytest.param({}, {"history": {}, "_ab_source_file_last_modified": None}, id="empty-history"), + pytest.param( + {"history": {"2023-08-01": ["file1.txt"]}, "_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, + { + "history": { + "file1.txt": "2023-08-01T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file1.txt", + "v3_min_sync_date": "2023-07-31T23:00:00.000000Z", + }, + id="single-date-single-file", + ), + pytest.param( + {"history": {"2023-08-01": ["file1.txt"]}, "_ab_source_file_last_modified": "2023-08-01T02:03:04Z"}, + { + "history": { + "file1.txt": "2023-08-01T02:03:04.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-01T02:03:04.000000Z_file1.txt", + "v3_min_sync_date": "2023-08-01T01:03:04.000000Z", + }, + id="single-date-not-at-midnight-single-file", + ), + pytest.param( + {"history": {"2023-08-01": ["file1.txt", "file2.txt"]}, "_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, + { + "history": { + "file1.txt": "2023-08-01T00:00:00.000000Z", + "file2.txt": "2023-08-01T00:00:00.000000Z", + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file2.txt", + "v3_min_sync_date": "2023-07-31T23:00:00.000000Z", + }, + id="single-date-multiple-files", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt", "file2.txt"], + "2023-07-31": ["file1.txt", "file3.txt"], + "2023-07-30": ["file3.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", + }, + { + "history": { + "file1.txt": "2023-08-01T00:00:00.000000Z", + "file2.txt": "2023-08-01T00:00:00.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "v3_min_sync_date": "2023-07-31T23:00:00.000000Z", + "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file2.txt", + }, + id="multiple-dates-multiple-files", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt", "file2.txt"], + "2023-07-31": ["file1.txt", "file3.txt"], + "2023-07-30": ["file3.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12Z", + }, + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + "v3_min_sync_date": "2023-08-01T09:11:12.000000Z", + }, + id="multiple-dates-multiple-files-not-at-midnight", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + id="v4-no-migration", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + "v3_min_sync_date": "2023-07-31T23:00:00.000000Z", + }, + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "v3_min_sync_date": "2023-07-31T23:00:00.000000Z", + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + id="v4-migrated-from-v3", + ), + pytest.param( + {"history": {}, "_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, + { + "history": {}, + "_ab_source_file_last_modified": None, + "v3_min_sync_date": "2023-07-31T23:00:00.000000Z", + }, + id="empty-history-with-cursor", + ), + + ], +) +def test_set_initial_state(input_state: MutableMapping[str, Any], expected_state: MutableMapping[str, Any]) -> None: + cursor = _init_cursor_with_state(input_state) + assert cursor.get_state() == expected_state + + +@pytest.mark.parametrize( + "input_state, all_files, expected_files_to_sync, max_history_size", + [ + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", + }, + [RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T00:00:00.000000Z"))], + [RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T00:00:00.000000Z"))], + None, + id="only_one_file_that_was_synced_exactly_at_midnight", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt"], + "2023-08-02": ["file2.txt"], + }, + "_ab_source_file_last_modified": "2023-08-02T00:06:00Z", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T00:00:00.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-02T06:00:00.000000Z")), + ], + [ + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-02T06:00:00.000000Z")), + ], + None, + id="do_not_sync_files_last_updated_on_a_previous_date", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt"], + "2023-08-02": ["file2.txt"], + }, + "_ab_source_file_last_modified": "2023-08-02T00:00:00Z", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T23:00:01.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-02T00:00:00.000000Z")), + ], + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T23:00:01.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-02T00:00:00.000000Z")), + ], + None, + id="sync_files_last_updated_within_one_hour_of_cursor", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt", "file2.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T02:00:00Z", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T01:30:00.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T02:00:00.000000Z")), + ], + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T01:30:00.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T02:00:00.000000Z")), + ], + None, + id="sync_files_last_updated_within_one_hour_of_cursor_on_same_day", + ), + pytest.param( + { + "history": { + "2023-08-01": ["file1.txt", "file2.txt"], + }, + "_ab_source_file_last_modified": "2023-08-01T06:00:00Z", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T01:30:00.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T06:00:00.000000Z")), + ], + [ + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T06:00:00.000000Z")), + ], + None, + id="do_not_sync_files_last_modified_earlier_than_one_hour_before_cursor_on_same_day", + ), + pytest.param( + {}, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T01:30:00.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T06:00:00.000000Z")), + ], + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T01:30:00.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T06:00:00.000000Z")), + ], + None, + id="no_state", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + ], + [], + None, + id="input_state_is_v4_no_new_files", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + RemoteFile(uri="file4.txt", last_modified=_create_datetime("2023-08-02T00:00:00.000000Z")), + ], + [RemoteFile(uri="file4.txt", last_modified=_create_datetime("2023-08-02T00:00:00.000000Z"))], + None, + id="input_state_is_v4_with_new_file_later_than_cursor", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + RemoteFile(uri="file4.txt", last_modified=_create_datetime("2023-08-01T00:00:00.000000Z")), + ], + [RemoteFile(uri="file4.txt", last_modified=_create_datetime("2023-08-01T00:00:00.000000Z"))], + None, + id="input_state_is_v4_with_new_file_earlier_than_cursor", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "v3_min_sync_date": "2023-07-16T00:00:00.000000Z", + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + RemoteFile(uri="file0.txt", last_modified=_create_datetime("2023-07-15T00:00:00.000000Z")), + ], + [], + None, + id="input_state_is_v4_with_a_new_file_earlier_than_migration_start_datetime", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "v3_min_sync_date": "2023-07-01T00:00:00.000000Z", + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + RemoteFile(uri="file0.txt", last_modified=_create_datetime("2023-07-15T00:00:00.000000Z")), + ], + [RemoteFile(uri="file0.txt", last_modified=_create_datetime("2023-07-15T00:00:00.000000Z"))], + None, + id="input_state_is_v4_with_a_new_file_later_than_migration_start_datetime", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "v3_min_sync_date": "2023-07-16T00:00:00.000000Z", + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + RemoteFile(uri="file0.txt", last_modified=_create_datetime("2023-07-15T00:00:00.000000Z")), + ], + [], + 3, + id="input_state_is_v4_history_is_full_but_new_file_is_earlier_than_v3_min_sync_date", + ), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "v3_min_sync_date": "2023-07-01T00:00:00.000000Z", + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + [ + RemoteFile(uri="file1.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file2.txt", last_modified=_create_datetime("2023-08-01T10:11:12.000000Z")), + RemoteFile(uri="file3.txt", last_modified=_create_datetime("2023-07-31T23:59:59.999999Z")), + RemoteFile(uri="file0.txt", last_modified=_create_datetime("2023-07-15T00:00:00.000000Z")), + ], + [], # file0.txt is not synced. It was presumably synced by v4 because its timestamp is later than v3_min_sync_date, + # but it was kicked from the history because it was not in the top 3 most recently modified files. + 3, + id="input_state_is_v4_history_is_full_and_new_file_is_later_than_v3_min_sync_date", + ), + ], +) +def test_list_files_v4_migration(input_state, all_files, expected_files_to_sync, max_history_size): + cursor = _init_cursor_with_state(input_state, max_history_size) + files_to_sync = list(cursor.get_files_to_sync(all_files, Mock())) + assert files_to_sync == expected_files_to_sync + + +@pytest.mark.parametrize( + "input_state, expected", + [ + pytest.param({}, False, id="empty_state_is_not_legacy_state"), + pytest.param( + {"history": {"2023-08-01": ["file1.txt"]}, "_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, + True, + id="legacy_state_with_history_and_last_modified_cursor_is_legacy_state", + ), + pytest.param( + {"history": {"2023-08-01T00:00:00Z": ["file1.txt"]}, "_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, + False, + id="legacy_state_with_invalid_history_date_format_is_not_legacy", + ), + pytest.param( + {"history": {"2023-08-01": ["file1.txt"]}, "_ab_source_file_last_modified": "2023-08-01"}, + False, + id="legacy_state_with_invalid_last_modified_datetime_format_is_not_legacy", + ), + pytest.param( + {"_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, True, id="legacy_state_without_history_is_legacy_state" + ), + pytest.param({"history": {"2023-08-01": ["file1.txt"]}}, False, id="legacy_state_without_last_modified_cursor_is_not_legacy_state"), + pytest.param( + { + "history": { + "file1.txt": "2023-08-01T10:11:12.000000Z", + "file2.txt": "2023-08-01T10:11:12.000000Z", + "file3.txt": "2023-07-31T23:59:59.999999Z", + }, + "_ab_source_file_last_modified": "2023-08-01T10:11:12.000000Z_file2.txt", + }, + False, + id="v4_state_format_is_not_legacy", + ), + ], +) +def test_is_legacy_state(input_state, expected): + is_legacy_state = Cursor._is_legacy_state(input_state) + assert is_legacy_state is expected + + +@pytest.mark.parametrize( + "cursor_datetime, file_datetime, expected_adjusted_datetime", + [ + pytest.param( + datetime(2021, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc), + datetime(2021, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc), + datetime(2021, 1, 1, 0, 0, 0, 0, tzinfo=timezone.utc), + id="cursor_datetime_equals_file_datetime_at_start_of_day", + ), + pytest.param( + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + id="cursor_datetime_equals_file_datetime_not_at_start_of_day", + ), + pytest.param( + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + datetime(2021, 1, 1, 0, 1, 2, tzinfo=timezone.utc), + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + id="cursor_datetime_same_day_but_later", + ), + pytest.param( + datetime(2021, 1, 2, 0, 1, 2, tzinfo=timezone.utc), + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + datetime(2021, 1, 1, 23, 59, 59, 999999, tzinfo=timezone.utc), + id="set_time_to_end_of_day_if_file_date_is_ealier_than_cursor_date", + ), + pytest.param( + datetime(2021, 1, 1, 0, 1, 2, tzinfo=timezone.utc), + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + id="file_datetime_is_unchanged_if_same_day_but_later_than_cursor_datetime", + ), + pytest.param( + datetime(2021, 1, 1, 10, 11, 12, tzinfo=timezone.utc), + datetime(2021, 1, 2, 0, 1, 2, tzinfo=timezone.utc), + datetime(2021, 1, 2, 0, 1, 2, tzinfo=timezone.utc), + id="file_datetime_is_unchanged_if_later_than_cursor_datetime", + ), + ], +) +def test_get_adjusted_date_timestamp(cursor_datetime, file_datetime, expected_adjusted_datetime): + adjusted_datetime = Cursor._get_adjusted_date_timestamp(cursor_datetime, file_datetime) + assert adjusted_datetime == expected_adjusted_datetime + + +def _init_cursor_with_state(input_state, max_history_size: Optional[int] = None) -> Cursor: + cursor = Cursor(stream_config=FileBasedStreamConfig(file_type="csv", name="test", validation_policy="Emit Record")) + cursor.set_initial_state(input_state) + if max_history_size is not None: + cursor.DEFAULT_MAX_HISTORY_SIZE = max_history_size + return cursor diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py new file mode 100644 index 000000000000..dfccfe69b1c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py @@ -0,0 +1,386 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_s3.source import SourceS3Spec +from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer + + +@pytest.mark.parametrize( + "legacy_config, expected_config", + [ + pytest.param( + { + "dataset": "test_data", + "provider": { + "storage": "S3", + "bucket": "test_bucket", + "aws_access_key_id": "some_access_key", + "aws_secret_access_key": "some_secret", + "endpoint": "https://external-s3.com", + "path_prefix": "a_folder/", + "start_date": "2022-01-01T01:02:03Z", + }, + "format": { + "filetype": "avro", + }, + "path_pattern": "**/*.avro", + "schema": '{"col1": "string", "col2": "integer"}', + }, + { + "bucket": "test_bucket", + "aws_access_key_id": "some_access_key", + "aws_secret_access_key": "some_secret", + "endpoint": "https://external-s3.com", + "start_date": "2022-01-01T01:02:03.000000Z", + "streams": [ + { + "name": "test_data", + "file_type": "avro", + "globs": ["**/*.avro"], + "legacy_prefix": "a_folder/", + "validation_policy": "Emit Record", + "input_schema": '{"col1": "string", "col2": "integer"}', + "format": {"filetype": "avro"}, + } + ], + }, + id="test_convert_legacy_config", + ), + pytest.param( + { + "dataset": "test_data", + "provider": { + "storage": "S3", + "bucket": "test_bucket", + }, + "format": { + "filetype": "avro", + }, + "path_pattern": "**/*.avro", + }, + { + "bucket": "test_bucket", + "streams": [ + { + "name": "test_data", + "file_type": "avro", + "globs": ["**/*.avro"], + "legacy_prefix": "", + "validation_policy": "Emit Record", + "format": {"filetype": "avro"}, + } + ], + }, + id="test_convert_no_optional_fields", + ), + pytest.param( + { + "dataset": "test_data", + "provider": { + "storage": "S3", + "bucket": "test_bucket", + "path_prefix": "a_prefix/", + }, + "format": { + "filetype": "avro", + }, + "path_pattern": "*.csv|**/*", + }, + { + "bucket": "test_bucket", + "streams": [ + { + "name": "test_data", + "file_type": "avro", + "globs": ["*.csv", "**/*"], + "validation_policy": "Emit Record", + "legacy_prefix": "a_prefix/", + "format": {"filetype": "avro"}, + } + ] + } + , id="test_convert_with_multiple_path_patterns" + ), + ] +) +def test_convert_legacy_config(legacy_config, expected_config): + parsed_legacy_config = SourceS3Spec(**legacy_config) + actual_config = LegacyConfigTransformer.convert(parsed_legacy_config) + + assert actual_config == expected_config + + +@pytest.mark.parametrize( + "file_type,legacy_format_config,expected_format_config, expected_error", + [ + pytest.param( + "csv", + { + "filetype": "csv", + "delimiter": "&", + "infer_datatypes": False, + "quote_char": "^", + "escape_char": "$", + "encoding": "ansi", + "double_quote": False, + "newlines_in_values": True, + "additional_reader_options": '{"strings_can_be_null": true, "null_values": ["NULL", "NONE"], "true_values": ["yes", "y"], "false_values": ["no", "n"]}', + "advanced_options": '{"skip_rows": 3, "skip_rows_after_names": 5, "autogenerate_column_names": true}', + "blocksize": 20000, + }, + { + "filetype": "csv", + "delimiter": "&", + "quote_char": "^", + "escape_char": "$", + "encoding": "ansi", + "double_quote": False, + "null_values": ["NULL", "NONE"], + "true_values": ["yes", "y"], + "false_values": ["no", "n"], + "inference_type": "None", + "strings_can_be_null": True, + "skip_rows_before_header": 3, + "skip_rows_after_header": 5, + "autogenerate_column_names": True, + }, + None, + id="test_csv_all_legacy_options_set", + ), + pytest.param( + "csv", + { + "filetype": "csv", + "delimiter": "&", + "quote_char": "^", + "double_quote": True, + "newlines_in_values": False, + }, + { + "filetype": "csv", + "delimiter": "&", + "quote_char": "^", + "encoding": "utf8", + "double_quote": True, + "null_values": ["", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null"], + "true_values": ["1", "True", "TRUE", "true"], + "false_values": ["0", "False", "FALSE", "false"], + "inference_type": "Primitive Types Only", + "strings_can_be_null": False, + }, + None, + id="test_csv_only_required_options", + ), + pytest.param( + "csv", + {}, + { + "filetype": "csv", + "delimiter": ",", + "quote_char": '"', + "encoding": "utf8", + "double_quote": True, + "null_values": ["", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null"], + "true_values": ["1", "True", "TRUE", "true"], + "false_values": ["0", "False", "FALSE", "false"], + "inference_type": "Primitive Types Only", + "strings_can_be_null": False, + }, + None, + id="test_csv_empty_format", + ), + pytest.param( + "csv", + { + "additional_reader_options": '{"not_valid": "at all}', + }, + None, + ValueError, + id="test_malformed_additional_reader_options", + ), + pytest.param( + "csv", + { + "additional_reader_options": '{"include_columns": ""}', + }, + None, + ValueError, + id="test_unsupported_additional_reader_options", + ), + pytest.param( + "csv", + { + "advanced_options": '{"not_valid": "at all}', + }, + None, + ValueError, + id="test_malformed_advanced_options", + ), + pytest.param( + "csv", + { + "advanced_options": '{"column_names": ""}', + }, + None, + ValueError, + id="test_unsupported_advanced_options", + ), + pytest.param( + "csv", + { + "advanced_options": '{"check_utf8": false}', + }, + { + "filetype": "csv", + "delimiter": ",", + "quote_char": '"', + "encoding": "utf8", + "double_quote": True, + "null_values": [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "N/A", + "NA", + "NULL", + "NaN", + "n/a", + "nan", + "null", + ], + "true_values": ["1", "True", "TRUE", "true"], + "false_values": ["0", "False", "FALSE", "false"], + "inference_type": "Primitive Types Only", + "strings_can_be_null": False, + }, + None, + id="test_unsupported_advanced_options_by_value_succeeds_if_value_matches_ignored_values", + ), + pytest.param( + "csv", + { + "advanced_options": '{"check_utf8": true}', + }, + None, + ValueError, + id="test_unsupported_advanced_options_by_value_fails_if_value_doesnt_match_ignored_values", + ), + pytest.param( + "csv", + { + "advanced_options": '{"auto_dict_encode": ""}', + }, + { + "filetype": "csv", + "delimiter": ",", + "quote_char": '"', + "encoding": "utf8", + "double_quote": True, + "null_values": [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "N/A", + "NA", + "NULL", + "NaN", + "n/a", + "nan", + "null", + ], + "true_values": ["1", "True", "TRUE", "true"], + "false_values": ["0", "False", "FALSE", "false"], + "inference_type": "Primitive Types Only", + "strings_can_be_null": False, + }, + None, + id="test_ignored_advanced_options", + ), + pytest.param( + "jsonl", + { + "filetype": "jsonl", + "newlines_in_values": True, + "unexpected_field_behavior": "ignore", + "block_size": 0, + }, + {"filetype": "jsonl"}, + None, + id="test_jsonl_format", + ), + pytest.param( + "parquet", + { + "filetype": "parquet", + "columns": ["test"], + "batch_size": 65536, + "buffer_size": 100, + }, + {"filetype": "parquet", "decimal_as_float": True}, + None, + id="test_parquet_format", + ), + pytest.param( + "avro", + { + "filetype": "avro", + }, + {"filetype": "avro"}, + None, + id="test_avro_format", + ), + ], +) +def test_convert_file_format(file_type, legacy_format_config, expected_format_config, expected_error): + legacy_config = { + "dataset": "test_data", + "provider": { + "storage": "S3", + "bucket": "test_bucket", + "aws_access_key_id": "some_access_key", + "aws_secret_access_key": "some_secret", + }, + "format": legacy_format_config, + "path_pattern": f"**/*.{file_type}", + } + + expected_config = { + "bucket": "test_bucket", + "aws_access_key_id": "some_access_key", + "aws_secret_access_key": "some_secret", + "streams": [ + { + "name": "test_data", + "file_type": file_type, + "globs": [f"**/*.{file_type}"], + "legacy_prefix": "", + "validation_policy": "Emit Record", + "format": expected_format_config, + } + ], + } + + parsed_legacy_config = SourceS3Spec(**legacy_config) + + if expected_error: + with pytest.raises(expected_error): + LegacyConfigTransformer.convert(parsed_legacy_config) + else: + actual_config = LegacyConfigTransformer.convert(parsed_legacy_config) + assert actual_config == expected_config diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py new file mode 100644 index 000000000000..9896e298fcf2 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -0,0 +1,235 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import io +import logging +from datetime import datetime +from itertools import product +from typing import Any, Dict, List, Optional, Set +from unittest.mock import patch + +import pytest +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from botocore.stub import Stubber +from pydantic import AnyUrl +from source_s3.v4.config import Config +from source_s3.v4.stream_reader import SourceS3StreamReader + +logger = logging.Logger("") + +endpoint_values = ["http://fake.com", None] +_get_matching_files_cases = [ + pytest.param([], [], False, set(), id="no-files-match-if-no-globs"), + pytest.param( + ["**"], + [ + {"Key": "file1.csv", "LastModified": datetime.now()}, + {"Key": "a/file2.csv", "LastModified": datetime.now()}, + {"Key": "a/b/file3.csv", "LastModified": datetime.now()}, + {"Key": "a/b/c/file4.csv", "LastModified": datetime.now()}, + ], + False, + {"file1.csv", "a/file2.csv", "a/b/file3.csv", "a/b/c/file4.csv"}, + id="all-files-match-single-page", + ), + pytest.param( + ["**"], + [ + {"Key": "file1.csv", "LastModified": datetime.now()}, + {"Key": "a/file2.csv", "LastModified": datetime.now()}, + {"Key": "a/b/file3.csv", "LastModified": datetime.now()}, + {"Key": "a/b/c/file4.csv", "LastModified": datetime.now()}, + ], + True, + {"file1.csv", "a/file2.csv", "a/b/file3.csv", "a/b/c/file4.csv"}, + id="all-files-match-multiple-pages", + ), + pytest.param( + ["**/*.csv"], + [ + {"Key": "file1.csv", "LastModified": datetime.now()}, + {"Key": "a/file2.csv", "LastModified": datetime.now()}, + {"Key": "a/b/file3.jsonl", "LastModified": datetime.now()}, + {"Key": "a/b/c/file4.jsonl", "LastModified": datetime.now()}, + ], + True, + {"file1.csv", "a/file2.csv"}, + id="nonmatching-files-are-filtered", + ), + pytest.param( + ["a/*.csv", "a/*.jsonl"], + [ + {"Key": "file1.csv", "LastModified": datetime.now()}, + {"Key": "file2.jsonl", "LastModified": datetime.now()}, + {"Key": "a/file3.csv", "LastModified": datetime.now()}, + {"Key": "a/file4.jsonl", "LastModified": datetime.now()}, + ], + True, + {"a/file3.csv", "a/file4.jsonl"}, + id="nonmatching-files-are-filtered-multiple-prefixes", + ), + pytest.param( + ["**", "a/*.jsonl"], + [ + {"Key": "file1.csv", "LastModified": datetime.now()}, + {"Key": "file2.jsonl", "LastModified": datetime.now()}, + {"Key": "a/file3.csv", "LastModified": datetime.now()}, + {"Key": "a/file4.jsonl", "LastModified": datetime.now()}, + ], + True, + {"file1.csv", "file2.jsonl", "a/file3.csv", "a/file4.jsonl"}, + id="files-matching-multiple-prefixes-only-listed-once", + ), + pytest.param( + ["**"], + [ + {"Key": "file1.csv", "LastModified": datetime.now()}, + {"Key": "file2.jsonl", "LastModified": datetime.now()}, + {"Key": "file3.csv", "LastModified": datetime.now()}, + {"Key": "file3.csv", "LastModified": datetime.now()}, + ], + True, + {"file1.csv", "file2.jsonl", "file3.csv"}, + id="duplicate-files-only-listed-once", + ), +] + +get_matching_files_cases = [] +for original_case, endpoint_value in product(_get_matching_files_cases, endpoint_values): + params = list(original_case.values) + [endpoint_value] + test_case = pytest.param(*params, id=original_case.id + f"-endpoint-{endpoint_value}") + get_matching_files_cases.append(test_case) + + +@pytest.mark.parametrize( + "globs,mocked_response,multiple_pages,expected_uris,endpoint", + get_matching_files_cases +) +def test_get_matching_files(globs: List[str], mocked_response: List[Dict[str, Any]], multiple_pages: bool, expected_uris: Set[str], endpoint: Optional[str]): + reader = SourceS3StreamReader() + try: + aws_access_key_id = aws_secret_access_key = None if endpoint else "test" + reader.config = Config( + bucket="test", + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + streams=[], + endpoint=endpoint, + ) + except Exception as exc: + raise exc + + stub = set_stub(reader, mocked_response, multiple_pages) + files = list(reader.get_matching_files(globs, None, logger)) + stub.deactivate() + assert set(f.uri for f in files) == expected_uris + + +@patch("boto3.client") +def test_given_multiple_pages_when_get_matching_files_then_pass_continuation_token(boto3_client_mock) -> None: + boto3_client_mock.return_value.list_objects_v2.side_effect = [ + {"Contents": [{"Key": "1", "LastModified": datetime.now()}, {"Key": "2", "LastModified": datetime.now()}], "KeyCount": 2, "NextContinuationToken": "a key"}, + {"Contents": [{"Key": "1", "LastModified": datetime.now()}, {"Key": "2", "LastModified": datetime.now()}], "KeyCount": 2}, + ] + reader = SourceS3StreamReader() + reader.config = Config( + bucket="test", + aws_access_key_id="aws_access_key_id", + aws_secret_access_key="aws_secret_access_key", + streams=[], + endpoint=None, + ) + list(reader.get_matching_files(["**"], None, logger)) + assert boto3_client_mock.return_value.list_objects_v2.call_count == 2 + assert "ContinuationToken" in boto3_client_mock.return_value.list_objects_v2.call_args_list[1].kwargs + + +def test_get_matching_files_exception(): + reader = SourceS3StreamReader() + reader.config = Config(bucket="test", aws_access_key_id="test", aws_secret_access_key="test", streams=[]) + stub = Stubber(reader.s3_client) + stub.add_client_error("list_objects_v2") + stub.activate() + with pytest.raises(ErrorListingFiles) as exc: + list(reader.get_matching_files(["*"], None, logger)) + stub.deactivate() + assert FileBasedSourceError.ERROR_LISTING_FILES.value in exc.value.args[0] + + +def test_get_matching_files_without_config_raises_exception(): + with pytest.raises(ValueError): + next(SourceS3StreamReader().get_matching_files([], None, logger)) + + +def test_open_file_without_config_raises_exception(): + with pytest.raises(ValueError): + with SourceS3StreamReader().open_file(RemoteFile(uri="", last_modified=datetime.now()), FileReadMode.READ, None, logger) as fp: + fp.read() + + +@patch("smart_open.open") +def test_open_file_calls_any_open_with_the_right_encoding(smart_open_mock): + smart_open_mock.return_value = io.BytesIO() + reader = SourceS3StreamReader() + reader.config = Config(bucket="test", aws_access_key_id="test", aws_secret_access_key="test", streams=[]) + try: + reader.config = Config( + bucket="test", + aws_access_key_id="test", + aws_secret_access_key="test", + streams=[], + endpoint=None, + ) + except Exception as exc: + raise exc + + encoding = "utf8" + with reader.open_file(RemoteFile(uri="", last_modified=datetime.now()), FileReadMode.READ, encoding, logger) as fp: + fp.read() + + smart_open_mock.assert_called_once_with('s3://test/', transport_params={"client": reader.s3_client}, mode=FileReadMode.READ.value, encoding=encoding) + + +def test_get_s3_client_without_config_raises_exception(): + with pytest.raises(ValueError): + SourceS3StreamReader().s3_client + + +def test_cannot_set_wrong_config_type(): + stream_reader = SourceS3StreamReader() + + class OtherConfig(AbstractFileBasedSpec): + def documentation_url(cls) -> AnyUrl: + return AnyUrl("https://fake.com", scheme="https") + + other_config = OtherConfig(streams=[]) + with pytest.raises(AssertionError): + stream_reader.config = other_config + + +def set_stub(reader: SourceS3StreamReader, contents: List[Dict[str, Any]], multiple_pages: bool) -> Stubber: + s3_stub = Stubber(reader.s3_client) + split_contents_idx = int(len(contents) / 2) if multiple_pages else -1 + page1, page2 = contents[:split_contents_idx], contents[split_contents_idx:] + resp = { + "KeyCount": len(page1), + "Contents": page1, + } + if page2: + resp["NextContinuationToken"] = "token" + s3_stub.add_response("list_objects_v2", resp) + if page2: + s3_stub.add_response( + "list_objects_v2", + { + "KeyCount": len(page2), + "Contents": page2, + }, + ) + s3_stub.activate() + return s3_stub diff --git a/airbyte-integrations/connectors/source-s3/v4_main.py b/airbyte-integrations/connectors/source-s3/v4_main.py new file mode 100644 index 000000000000..decb39a68c88 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/v4_main.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader + + +def get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + try: + return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +if __name__ == "__main__": + _args = sys.argv[1:] + source = get_source(_args) + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-salesforce-singer/metadata.yaml b/airbyte-integrations/connectors/source-salesforce-singer/metadata.yaml index 873a8e150a30..abcbed19e172 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesforce-singer/metadata.yaml @@ -9,11 +9,15 @@ data: name: Salesforce (Singer) registries: cloud: - enabled: true + enabled: false oss: enabled: false releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce + documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index a32835374a83..8eba23219668 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.14 +LABEL io.airbyte.version=2.1.4 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/metadata.yaml b/airbyte-integrations/connectors/source-salesforce/metadata.yaml index 9eba8f2a7561..ffb43710b442 100644 --- a/airbyte-integrations/connectors/source-salesforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesforce/metadata.yaml @@ -5,15 +5,14 @@ data: connectorSubtype: api connectorType: source definitionId: b117307c-14b6-41aa-9422-947e34922962 - dockerImageTag: 2.0.14 + dockerImageTag: 2.1.4 dockerRepository: airbyte/source-salesforce githubIssueLabel: source-salesforce icon: salesforce.svg - license: MIT + license: ELv2 name: Salesforce registries: cloud: - dockerImageTag: 2.0.9 enabled: true oss: enabled: true @@ -21,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-salesforce/requirements.txt b/airbyte-integrations/connectors/source-salesforce/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-salesforce/requirements.txt +++ b/airbyte-integrations/connectors/source-salesforce/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py index 7e47fce763a0..44c137056bbd 100644 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ b/airbyte-integrations/connectors/source-salesforce/setup.py @@ -5,9 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "vcrpy==4.1.1", "pandas"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.50", "pandas"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "requests_mock", "connector-acceptance-test", "pytest-timeout"] +TEST_REQUIREMENTS = ["freezegun", "pytest~=6.1", "pytest-mock~=3.6", "requests-mock~=1.9.3", "pytest-timeout"] setup( name="source_salesforce", diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index aea3bfb0f876..e88c4db1ff0b 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -8,10 +8,12 @@ import requests # type: ignore[import] from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType from requests import adapters as request_adapters from requests.exceptions import HTTPError, RequestException # type: ignore[import] -from .exceptions import TypeSalesforceException +from .exceptions import AUTHENTICATION_ERROR_MESSAGE_MAPPING, TypeSalesforceException from .rate_limiting import default_backoff_handler from .utils import filter_streams_by_criteria @@ -296,7 +298,7 @@ def _make_request( resp = self.session.post(url, headers=headers, data=body) resp.raise_for_status() except HTTPError as err: - self.logger.warn(f"http error body: {err.response.text}") + self.logger.warning(f"http error body: {err.response.text}") raise return resp @@ -308,9 +310,13 @@ def login(self): "client_secret": self.client_secret, "refresh_token": self.refresh_token, } - - resp = self._make_request("POST", login_url, body=login_body, headers={"Content-Type": "application/x-www-form-urlencoded"}) - + try: + resp = self._make_request("POST", login_url, body=login_body, headers={"Content-Type": "application/x-www-form-urlencoded"}) + except HTTPError as err: + if err.response.status_code == requests.codes.BAD_REQUEST: + if error_message := AUTHENTICATION_ERROR_MESSAGE_MAPPING.get(err.response.json().get("error_description")): + raise AirbyteTracedException(message=error_message, failure_type=FailureType.config_error) + raise auth = resp.json() self.access_token = auth["access_token"] self.instance_url = auth["instance_url"] diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/exceptions.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/exceptions.py index 7e08a8a23ced..441e2ec5a32d 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/exceptions.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/exceptions.py @@ -28,3 +28,8 @@ class TypeSalesforceException(SalesforceException): class TmpFileIOError(Error): def __init__(self, msg: str, err: str = None): self.logger.fatal(f"{msg}. Error: {err}") + + +AUTHENTICATION_ERROR_MESSAGE_MAPPING = { + "expired access/refresh token": "The authentication to SalesForce has expired. Re-authenticate to restore access to SalesForce." +} diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py index 286339fcbe1c..344a6412e024 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/rate_limiting.py @@ -2,11 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import logging import sys import backoff -from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from requests import codes, exceptions # type: ignore[import] @@ -18,7 +17,7 @@ exceptions.HTTPError, ) -logger = AirbyteLogger() +logger = logging.getLogger("airbyte") def default_backoff_handler(max_tries: int, factor: int, **kwargs): diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index 77243ea2b57a..9def53730d79 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -21,6 +21,8 @@ from .api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, Describe, IncrementalRestSalesforceStream, RestSalesforceStream +logger = logging.getLogger("airbyte") + class AirbyteStopSync(AirbyteTracedException): pass @@ -59,13 +61,21 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return True, None @classmethod - def _get_api_type(cls, stream_name, properties): + def _get_api_type(cls, stream_name: str, properties: Mapping[str, Any], force_use_bulk_api: bool) -> str: # Salesforce BULK API currently does not support loading fields with data type base64 and compound data properties_not_supported_by_bulk = { key: value for key, value in properties.items() if value.get("format") == "base64" or "object" in value["type"] } - rest_required = stream_name in UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS or properties_not_supported_by_bulk - if rest_required: + rest_only = stream_name in UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS + if rest_only: + logger.warning(f"BULK API is not supported for stream: {stream_name}") + return "rest" + if force_use_bulk_api and properties_not_supported_by_bulk: + logger.warning( + f"Following properties will be excluded from stream: {stream_name} due to BULK API limitations: {list(properties_not_supported_by_bulk)}" + ) + return "bulk" + if properties_not_supported_by_bulk: return "rest" return "bulk" @@ -77,7 +87,6 @@ def generate_streams( sf_object: Salesforce, ) -> List[Stream]: """ "Generates a list of stream by their names. It can be used for different tests too""" - logger = logging.getLogger() authenticator = TokenAuthenticator(sf_object.access_token) stream_properties = sf_object.generate_schemas(stream_objects) streams = [] @@ -85,7 +94,7 @@ def generate_streams( streams_kwargs = {"sobject_options": sobject_options} selected_properties = stream_properties.get(stream_name, {}).get("properties", {}) - api_type = cls._get_api_type(stream_name, selected_properties) + api_type = cls._get_api_type(stream_name, selected_properties, config.get("force_use_bulk_api", False)) if api_type == "rest": full_refresh, incremental = RestSalesforceStream, IncrementalRestSalesforceStream elif api_type == "bulk": diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml index af3a7655217c..642e180c078b 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml @@ -44,18 +44,25 @@ connectionSpecification: start_date: title: Start Date description: >- - Enter the date in the YYYY-MM-DD format. Airbyte will replicate the data added on and after this date. If this field is blank, Airbyte will replicate the data for last two years. + Enter the date (or date-time) in the YYYY-MM-DD or YYYY-MM-DDTHH:mm:ssZ format. Airbyte will replicate the data updated on and after this date. If this field is blank, Airbyte will replicate the data for last two years. type: string pattern: >- ^([0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?)$ + pattern_descriptor: "YYYY-MM-DD or YYYY-MM-DDTHH:mm:ssZ" examples: - "2021-07-25" - "2021-07-25T00:00:00Z" format: date-time order: 5 + force_use_bulk_api: + title: Force to use BULK API + type: boolean + description: Toggle to use Bulk API (this might cause empty fields for some streams) + default: false + order: 6 streams_criteria: type: array - order: 6 + order: 7 items: type: object required: @@ -81,8 +88,7 @@ connectionSpecification: title: Search value order: 2 title: Filter Salesforce Objects - description: >- - Filter streams relevant to you + description: Add filters to select only required stream based on `SObject` name. Use this field to filter which tables are displayed by this connector. This is useful if your Salesforce account has a large number of tables (>1000), in which case you may find it easier to navigate the UI and speed up the connector's performance if you restrict the tables displayed by this connector. advanced_auth: auth_flow_type: oauth2.0 predicate_key: diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 10c19e9537ed..dc7600453649 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -15,14 +15,16 @@ import pandas as pd import pendulum import requests # type: ignore[import] -from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream, StreamData from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils import AirbyteTracedException from numpy import nan from pendulum import DateTime # type: ignore[attr-defined] from requests import codes, exceptions +from requests.models import PreparedRequest from .api import UNSUPPORTED_FILTERING_STREAMS, Salesforce from .availability_strategy import SalesforceAvailabilityStrategy @@ -280,7 +282,6 @@ def _fetch_next_page_for_chunk( class BulkSalesforceStream(SalesforceStream): - page_size = 15000 DEFAULT_WAIT_TIMEOUT_SECONDS = 86400 # 24-hour bulk job running time MAX_CHECK_INTERVAL_SECONDS = 2.0 MAX_RETRY_NUMBER = 3 @@ -291,8 +292,8 @@ def path(self, next_page_token: Mapping[str, Any] = None, **kwargs: Any) -> str: transformer = TypeTransformer(TransformConfig.CustomSchemaNormalization | TransformConfig.DefaultSchemaNormalization) @default_backoff_handler(max_tries=5, factor=15) - def _send_http_request(self, method: str, url: str, json: dict = None, stream: bool = False): - headers = self.authenticator.get_auth_header() + def _send_http_request(self, method: str, url: str, json: dict = None, headers: dict = None, stream: bool = False): + headers = self.authenticator.get_auth_header() if not headers else headers | self.authenticator.get_auth_header() response = self._session.request(method, url=url, headers=headers, json=json, stream=stream) if response.status_code not in [200, 204]: self.logger.error(f"error body: {response.text}, sobject options: {self.sobject_options}") @@ -347,11 +348,16 @@ def create_stream_job(self, query: str, url: str) -> Optional[str]: f"The stream '{self.name}' is not queryable, " f"sobject options: {self.sobject_options}, error message: '{error_message}'" ) + elif ( + error.response.status_code == codes.BAD_REQUEST + and error_code == "API_ERROR" + and error_message.startswith("Implementation restriction") + ): + message = f"Unable to sync '{self.name}'. To prevent future syncs from failing, ensure the authenticated user has \"View all Data\" permissions." + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error, exception=error) elif error.response.status_code == codes.BAD_REQUEST and error_code == "LIMIT_EXCEEDED": - self.logger.error( - f"Cannot receive data for stream '{self.name}' ," - f"sobject options: {self.sobject_options}, error message: '{error_message}'" - ) + message = "Your API key for Salesforce has reached its limit for the 24-hour period. We will resume replication once the limit has elapsed." + self.logger.error(message) else: raise error else: @@ -368,7 +374,20 @@ def wait_for_job(self, url: str) -> str: # this value was received empirically time.sleep(0.5) while pendulum.now() < expiration_time: - job_info = self._send_http_request("GET", url=url).json() + try: + job_info = self._send_http_request("GET", url=url).json() + except exceptions.HTTPError as error: + error_data = error.response.json()[0] + error_code = error_data.get("errorCode") + error_message = error_data.get("message", "") + if ( + "We can't complete the action because enabled transaction security policies took too long to complete." in error_message + and error_code == "TXN_SECURITY_METERING_ERROR" + ): + message = 'A transient authentication error occurred. To prevent future syncs from failing, assign the "Exempt from Transaction Security" user permission to the authenticated user.' + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error, exception=error) + else: + raise error job_status = job_info["state"] if job_status in ["JobComplete", "Aborted", "Failed"]: if job_status != "JobComplete": @@ -422,7 +441,26 @@ def filter_null_bytes(self, b: bytes): self.logger.warning("Filter 'null' bytes from string, size reduced %d -> %d chars", len(b), len(res)) return res - def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str]: + def get_response_encoding(self, headers) -> str: + """Returns encodings from given HTTP Header Dict. + + :param headers: dictionary to extract encoding from. + :rtype: str + """ + + content_type = headers.get("content-type") + + if not content_type: + return self.encoding + + content_type, params = requests.utils._parse_content_type_header(content_type) + + if "charset" in params: + return params["charset"].strip("'\"") + + return self.encoding + + def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str, dict]: """ Retrieves binary data result from successfully `executed_job`, using chunks, to avoid local memory limitations. @ url: string - the url of the `executed_job` @@ -431,13 +469,16 @@ def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str]: """ # set filepath for binary data from response tmp_file = os.path.realpath(os.path.basename(url)) - with closing(self._send_http_request("GET", f"{url}/results", stream=True)) as response, open(tmp_file, "wb") as data_file: - response_encoding = response.encoding or self.encoding + with closing(self._send_http_request("GET", url, headers={"Accept-Encoding": "gzip"}, stream=True)) as response, open( + tmp_file, "wb" + ) as data_file: + response_headers = response.headers + response_encoding = self.get_response_encoding(response_headers) for chunk in response.iter_content(chunk_size=chunk_size): data_file.write(self.filter_null_bytes(chunk)) # check the file exists if os.path.isfile(tmp_file): - return tmp_file, response_encoding + return tmp_file, response_encoding, response_headers else: raise TmpFileIOError(f"The IO/Error occured while verifying binary data. Stream: {self.name}, file {tmp_file} doesn't exist.") @@ -477,10 +518,17 @@ def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None def next_page_token(self, last_record: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: - if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: - return {"next_token": f"WHERE {self.primary_key} >= '{last_record[self.primary_key]}' "} # type: ignore[index] return None + def get_query_select_fields(self) -> str: + return ", ".join( + { + key: value + for key, value in self.get_json_schema().get("properties", {}).items() + if value.get("format") != "base64" and "object" not in value["type"] + } + ) + def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: @@ -488,13 +536,11 @@ def request_params( Salesforce SOQL Query: https://developer.salesforce.com/docs/atlas.en-us.232.0.api_rest.meta/api_rest/dome_queryall.htm """ - selected_properties = self.get_json_schema().get("properties", {}) - query = f"SELECT {','.join(selected_properties.keys())} FROM {self.name} " + select_fields = self.get_query_select_fields() + query = f"SELECT {select_fields} FROM {self.name}" if next_page_token: query += next_page_token["next_token"] - if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: - query += f"ORDER BY {self.primary_key} ASC LIMIT {self.page_size}" return {"q": query} def read_records( @@ -507,45 +553,38 @@ def read_records( stream_state = stream_state or {} next_page_token = None - while True: - params = self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - path = self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - job_full_url, job_status = self.execute_job(query=params["q"], url=f"{self.url_base}{path}") - if not job_full_url: - if job_status == "Failed": - # As rule as BULK logic returns unhandled error. For instance: - # error message: 'Unexpected exception encountered in query processing. - # Please contact support with the following id: 326566388-63578 (-436445966)'" - # Thus we can try to switch to GET sync request because its response returns obvious error message - standard_instance = self.get_standard_instance() - self.logger.warning("switch to STANDARD(non-BULK) sync. Because the SalesForce BULK job has returned a failed status") - stream_is_available, error = standard_instance.check_availability(self.logger, None) - if not stream_is_available: - self.logger.warning(f"Skipped syncing stream '{standard_instance.name}' because it was unavailable. Error: {error}") - return - yield from standard_instance.read_records( - sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state - ) + params = self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + path = self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + job_full_url, job_status = self.execute_job(query=params["q"], url=f"{self.url_base}{path}") + if not job_full_url: + if job_status == "Failed": + # As rule as BULK logic returns unhandled error. For instance: + # error message: 'Unexpected exception encountered in query processing. + # Please contact support with the following id: 326566388-63578 (-436445966)'" + # Thus we can try to switch to GET sync request because its response returns obvious error message + standard_instance = self.get_standard_instance() + self.logger.warning("switch to STANDARD(non-BULK) sync. Because the SalesForce BULK job has returned a failed status") + stream_is_available, error = standard_instance.check_availability(self.logger, None) + if not stream_is_available: + self.logger.warning(f"Skipped syncing stream '{standard_instance.name}' because it was unavailable. Error: {error}") return - raise SalesforceException(f"Job for {self.name} stream using BULK API was failed.") - - count = 0 - record: Mapping[str, Any] = {} - for record in self.read_with_chunks(*self.download_data(url=job_full_url)): - count += 1 + yield from standard_instance.read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + return + raise SalesforceException(f"Job for {self.name} stream using BULK API was failed.") + salesforce_bulk_api_locator = None + while True: + req = PreparedRequest() + req.prepare_url(f"{job_full_url}/results", {"locator": salesforce_bulk_api_locator}) + tmp_file, response_encoding, response_headers = self.download_data(url=req.url) + for record in self.read_with_chunks(tmp_file, response_encoding): yield record - self.delete_job(url=job_full_url) - if count < self.page_size: - # Salesforce doesn't give a next token or something to know the request was - # the last page. The connectors will sync batches in `page_size` and - # considers that batch is smaller than the `page_size` it must be the last page. - break - - next_page_token = self.next_page_token(record) - if not next_page_token: - # not found a next page data. + if response_headers.get("Sforce-Locator", "null") == "null": break + salesforce_bulk_api_locator = response_headers.get("Sforce-Locator") + self.delete_job(url=job_full_url) def get_standard_instance(self) -> SalesforceStream: """Returns a instance of standard logic(non-BULK) with same settings""" @@ -580,6 +619,7 @@ def transform_empty_string_to_none(instance: Any, schema: Any): class IncrementalRestSalesforceStream(RestSalesforceStream, ABC): state_checkpoint_interval = 500 STREAM_SLICE_STEP = 30 + _slice = None def __init__(self, replication_key: str, start_date: Optional[str], **kwargs): super().__init__(**kwargs) @@ -604,6 +644,7 @@ def stream_slices( while not end == now: start = initial_date.add(days=(slice_number - 1) * self.STREAM_SLICE_STEP) end = min(now, initial_date.add(days=slice_number * self.STREAM_SLICE_STEP)) + self._slice = {"start_date": start.isoformat(timespec="milliseconds"), "end_date": end.isoformat(timespec="milliseconds")} yield {"start_date": start.isoformat(timespec="milliseconds"), "end_date": end.isoformat(timespec="milliseconds")} slice_number = slice_number + 1 @@ -632,17 +673,14 @@ def request_params( select_fields = ",".join(property_chunk.keys()) table_name = self.name where_conditions = [] - order_by_clause = "" if start_date: where_conditions.append(f"{self.cursor_field} >= {start_date}") if end_date: where_conditions.append(f"{self.cursor_field} < {end_date}") - if self.name not in UNSUPPORTED_FILTERING_STREAMS: - order_by_clause = f"ORDER BY {self.cursor_field} ASC" where_clause = f"WHERE {' AND '.join(where_conditions)}" - query = f"SELECT {select_fields} FROM {table_name} {where_clause} {order_by_clause}" + query = f"SELECT {select_fields} FROM {table_name} {where_clause}" return {"q": query} @@ -653,39 +691,33 @@ def cursor_field(self) -> str: def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ Return the latest state by comparing the cursor value in the latest record with the stream's most recent state - object and returning an updated state object. + object and returning an updated state object. Check if latest record is IN stream slice interval => ignore if not """ - latest_benchmark = latest_record[self.cursor_field] + latest_record_value: pendulum.DateTime = pendulum.parse(latest_record[self.cursor_field]) + slice_max_value: pendulum.DateTime = pendulum.parse(self._slice.get("end_date")) + max_possible_value = min(latest_record_value, slice_max_value) if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} + if latest_record_value > slice_max_value: + return {self.cursor_field: max_possible_value.isoformat()} + max_possible_value = max(latest_record_value, pendulum.parse(current_stream_state[self.cursor_field])) + return {self.cursor_field: max_possible_value.isoformat()} class BulkIncrementalSalesforceStream(BulkSalesforceStream, IncrementalRestSalesforceStream): - def next_page_token(self, last_record: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: - return None + state_checkpoint_interval = None def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - start_date = max( - (stream_state or {}).get(self.cursor_field, ""), - (stream_slice or {}).get("start_date", ""), - (next_page_token or {}).get("start_date", ""), - ) + start_date = stream_slice["start_date"] end_date = stream_slice["end_date"] - select_fields = ", ".join(self.get_json_schema().get("properties", {}).keys()) + select_fields = self.get_query_select_fields() table_name = self.name where_conditions = [f"{self.cursor_field} >= {start_date}", f"{self.cursor_field} < {end_date}"] - order_by_clause = "" - - if self.name not in UNSUPPORTED_FILTERING_STREAMS: - order_by_fields = ", ".join([self.cursor_field, self.primary_key] if self.primary_key else [self.cursor_field]) - order_by_clause = f"ORDER BY {order_by_fields} ASC" where_clause = f"WHERE {' AND '.join(where_conditions)}" - query = f"SELECT {select_fields} FROM {table_name} {where_clause} {order_by_clause}" + query = f"SELECT {select_fields} FROM {table_name} {where_clause}" return {"q": query} diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 49f10f73bef4..5e3ec0028b8e 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -10,12 +10,16 @@ from datetime import datetime from unittest.mock import Mock +import freezegun +import pendulum import pytest import requests_mock from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode, Type +from airbyte_cdk.utils import AirbyteTracedException from conftest import encoding_symbols_parameters, generate_stream from requests.exceptions import HTTPError from source_salesforce.api import Salesforce +from source_salesforce.exceptions import AUTHENTICATION_ERROR_MESSAGE_MAPPING from source_salesforce.source import SourceSalesforce from source_salesforce.streams import ( CSV_FIELD_SIZE_LIMIT, @@ -26,6 +30,29 @@ ) +@pytest.mark.parametrize( + "login_status_code, login_json_resp, expected_error_msg, is_config_error", + [ + (400, {"error": "invalid_grant", "error_description": "expired access/refresh token"}, AUTHENTICATION_ERROR_MESSAGE_MAPPING.get("expired access/refresh token"), True), + (400, {"error": "invalid_grant", "error_description": "Authentication failure."}, 'An error occurred: {"error": "invalid_grant", "error_description": "Authentication failure."}', False), + (401, {"error": "Unauthorized", "error_description": "Unautorized"}, 'An error occurred: {"error": "Unauthorized", "error_description": "Unautorized"}', False), + ] +) +def test_login_authentication_error_handler(stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg, is_config_error): + source = SourceSalesforce() + logger = logging.getLogger("airbyte") + requests_mock.register_uri("POST", "https://login.salesforce.com/services/oauth2/token", json=login_json_resp, status_code=login_status_code) + + if is_config_error: + with pytest.raises(AirbyteTracedException) as err: + source.check_connection(logger, stream_config) + assert err.value.message == expected_error_msg + else: + result, msg = source.check_connection(logger, stream_config) + assert result is False + assert msg == expected_error_msg + + def test_bulk_sync_creation_failed(stream_config, stream_api): stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) with requests_mock.Mocker() as m: @@ -61,7 +88,7 @@ def test_bulk_stream_fallback_to_rest(mocker, requests_mock, stream_config, stre assert list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices)) == rest_stream_records -def test_stream_unsupported_by_bulk(stream_config, stream_api, caplog): +def test_stream_unsupported_by_bulk(stream_config, stream_api): """ Stream `AcceptedEventRelation` is not supported by BULK API, so that REST API stream will be used for it. """ @@ -80,30 +107,26 @@ def test_stream_contains_unsupported_properties_by_bulk(stream_config, stream_ap assert not isinstance(stream, BulkSalesforceStream) -@pytest.mark.parametrize("item_number", [0, 15, 2000, 2324, 3000]) -def test_bulk_sync_pagination(item_number, stream_config, stream_api): +def test_bulk_sync_pagination(stream_config, stream_api, requests_mock): stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) - test_ids = [i for i in range(1, item_number)] - pages = [test_ids[i : i + stream.page_size] for i in range(0, len(test_ids), stream.page_size)] - if not pages: - pages = [[]] - with requests_mock.Mocker() as m: - creation_responses = [] + job_id = "fake_job" + requests_mock.register_uri("POST", stream.path(), json={"id": job_id}) + requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) + resp_text = ["Field1,LastModifiedDate,ID"] + [f"test,2021-11-16,{i}" for i in range(5)] + result_uri = requests_mock.register_uri("GET", stream.path() + f"/{job_id}/results", + [{"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "somelocator_1"}}, + {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "somelocator_2"}}, + {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "null"}} + ] + ) + requests_mock.register_uri("DELETE", stream.path() + f"/{job_id}") - for page in range(len(pages)): - job_id = f"fake_job_{page}" - creation_responses.append({"json": {"id": job_id}}) - m.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) - resp = ["Field1,LastModifiedDate,ID"] + [f"test,2021-11-16,{i}" for i in pages[page]] - m.register_uri("GET", stream.path() + f"/{job_id}/results", text="\n".join(resp)) - m.register_uri("DELETE", stream.path() + f"/{job_id}") - m.register_uri("POST", stream.path(), creation_responses) - - stream_slices = next(iter(stream.stream_slices(sync_mode=SyncMode.incremental))) - loaded_ids = [int(record["ID"]) for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices)] - assert not set(test_ids).symmetric_difference(set(loaded_ids)) - post_request_count = len([r for r in m.request_history if r.method == "POST"]) - assert post_request_count == len(pages) + stream_slices = next(iter(stream.stream_slices(sync_mode=SyncMode.incremental))) + loaded_ids = [int(record["ID"]) for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices)] + assert loaded_ids == [0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4] + assert result_uri.call_count == 3 + assert result_uri.request_history[1].query == "locator=somelocator_1" + assert result_uri.request_history[2].query == "locator=somelocator_2" def _prepare_mock(m, stream): @@ -210,61 +233,67 @@ def test_stream_start_datetime_format_should_not_changed(stream_config, stream_a def test_download_data_filter_null_bytes(stream_config, stream_api): - job_full_url: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" + job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) with requests_mock.Mocker() as m: - m.register_uri("GET", f"{job_full_url}/results", content=b"\x00") - res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) + m.register_uri("GET", job_full_url_results, content=b"\x00") + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + res = list(stream.read_with_chunks(tmp_file, response_encoding)) assert res == [] - m.register_uri("GET", f"{job_full_url}/results", content=b'"Id","IsDeleted"\n\x00"0014W000027f6UwQAI","false"\n\x00\x00') - res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) + m.register_uri("GET", job_full_url_results, content=b'"Id","IsDeleted"\n\x00"0014W000027f6UwQAI","false"\n\x00\x00') + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + res = list(stream.read_with_chunks(tmp_file, response_encoding)) assert res == [{"Id": "0014W000027f6UwQAI", "IsDeleted": "false"}] def test_read_with_chunks_should_return_only_object_data_type(stream_config, stream_api): - job_full_url: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" + job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) with requests_mock.Mocker() as m: - m.register_uri("GET", f"{job_full_url}/results", content=b'"IsDeleted","Age"\n"false",24\n') - res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) + m.register_uri("GET", job_full_url_results, content=b'"IsDeleted","Age"\n"false",24\n') + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + res = list(stream.read_with_chunks(tmp_file, response_encoding)) assert res == [{"IsDeleted": "false", "Age": "24"}] def test_read_with_chunks_should_return_a_string_when_a_string_with_only_digits_is_provided(stream_config, stream_api): - job_full_url: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" + job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) with requests_mock.Mocker() as m: - m.register_uri("GET", f"{job_full_url}/results", content=b'"ZipCode"\n"01234"\n') - res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) + m.register_uri("GET", job_full_url_results, content=b'"ZipCode"\n"01234"\n') + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + res = list(stream.read_with_chunks(tmp_file, response_encoding)) assert res == [{"ZipCode": "01234"}] def test_read_with_chunks_should_return_null_value_when_no_data_is_provided(stream_config, stream_api): - job_full_url: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" + job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) with requests_mock.Mocker() as m: - m.register_uri("GET", f"{job_full_url}/results", content=b'"IsDeleted","Age","Name"\n"false",,"Airbyte"\n') - res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url))) + m.register_uri("GET", job_full_url_results, content=b'"IsDeleted","Age","Name"\n"false",,"Airbyte"\n') + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + res = list(stream.read_with_chunks(tmp_file, response_encoding)) assert res == [{"IsDeleted": "false", "Age": None, "Name": "Airbyte"}] @pytest.mark.parametrize( - "chunk_size, content_type, content, expected_result", + "chunk_size, content_type_header, content, expected_result", encoding_symbols_parameters(), ids=[f"charset: {x[1]}, chunk_size: {x[0]}" for x in encoding_symbols_parameters()], ) -def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type, content, expected_result): - job_full_url: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" +def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type_header, content, expected_result): + job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) with requests_mock.Mocker() as m: - m.register_uri("GET", f"{job_full_url}/results", headers={"Content-Type": f"text/html; charset={content_type}"}, content=content) - res = list(stream.read_with_chunks(*stream.download_data(url=job_full_url, chunk_size=chunk_size))) + m.register_uri("GET", job_full_url_results, headers=content_type_header, content=content) + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + res = list(stream.read_with_chunks(tmp_file, response_encoding)) assert res == expected_result @@ -312,6 +341,7 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): While reading `stream_1` if 403 (Rate Limit) is received, it should finish that stream with success and stop the sync process. Next streams should not be executed. """ + stream_config.update({'start_date': '2021-10-01'}) stream_1: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) stream_2: BulkIncrementalSalesforceStream = generate_stream("Asset", stream_config, stream_api) streams = [stream_1, stream_2] @@ -335,7 +365,7 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): m.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) - resp = ["Field1,LastModifiedDate,Id"] + [f"test,2021-11-0{i},{i}" for i in range(1, 7)] # 6 records per page + resp = ["Field1,LastModifiedDate,Id"] + [f"test,2021-10-0{i},{i}" for i in range(1, 7)] # 6 records per page if page == 1: # Read the first page successfully @@ -358,7 +388,7 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): assert len(records) == 6 # stream page size: 6 state_record = [item for item in result if item.type == Type.STATE][0] - assert state_record.state.data["Account"]["LastModifiedDate"] == "2021-11-05" # state checkpoint interval is 5. + assert state_record.state.data["Account"]["LastModifiedDate"] == "2021-10-05T00:00:00+00:00" # state checkpoint interval is 5. def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): @@ -368,6 +398,7 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): While reading `stream_1` if 403 (Rate Limit) is received, it should finish that stream with success and stop the sync process. Next streams should not be executed. """ + stream_config.update({'start_date': '2021-11-01'}) stream_1: IncrementalRestSalesforceStream = generate_stream("KnowledgeArticle", stream_config, stream_api) stream_2: IncrementalRestSalesforceStream = generate_stream("AcceptedEventRelation", stream_config, stream_api) @@ -426,7 +457,7 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): assert len(records) == 5 state_record = [item for item in result if item.type == Type.STATE][0] - assert state_record.state.data["KnowledgeArticle"]["LastModifiedDate"] == "2021-11-17" + assert state_record.state.data["KnowledgeArticle"]["LastModifiedDate"] == "2021-11-17T00:00:00+00:00" def test_pagination_rest(stream_config, stream_api): @@ -474,7 +505,7 @@ def test_pagination_rest(stream_config, stream_api): def test_csv_reader_dialect_unix(): stream: BulkSalesforceStream = BulkSalesforceStream(stream_name=None, sf_api=None, pk=None) - url = "https://fake-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" + url_results = "https://fake-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" data = [ {"Id": "1", "Name": '"first_name" "last_name"'}, @@ -490,8 +521,9 @@ def test_csv_reader_dialect_unix(): text = csvfile.getvalue() with requests_mock.Mocker() as m: - m.register_uri("GET", url + "/results", text=text) - result = [i for i in stream.read_with_chunks(*stream.download_data(url))] + m.register_uri("GET", url_results, text=text) + tmp_file, response_encoding, _ = stream.download_data(url=url_results) + result = [i for i in stream.read_with_chunks(tmp_file, response_encoding)] assert result == data @@ -661,3 +693,116 @@ def test_stream_with_no_records_in_response(stream_config, stream_api_v2_pk_too_ ) records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) assert records == [] + + +@pytest.mark.parametrize( + "status_code,response_json,log_message", + [ + (400, [{"errorCode": "INVALIDENTITY", "message": "Account is not supported by the Bulk API"}], "Account is not supported by the Bulk API"), + (403, [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "API limit reached"}], "API limit reached"), + (400, [{"errorCode": "API_ERROR", "message": "API does not support query"}], "The stream 'Account' is not queryable,"), + (400, [{"errorCode": "LIMIT_EXCEEDED", "message": "Max bulk v2 query jobs (10000) per 24 hrs has been reached (10021)"}], "Your API key for Salesforce has reached its limit for the 24-hour period. We will resume replication once the limit has elapsed.") + ] +) +def test_bulk_stream_error_in_logs_on_create_job(requests_mock, stream_config, stream_api, status_code, response_json, log_message, caplog): + """ + """ + stream = generate_stream("Account", stream_config, stream_api) + url = f"{stream.sf_api.instance_url}/services/data/{stream.sf_api.version}/jobs/query" + requests_mock.register_uri( + "POST", + url, + status_code=status_code, + json=response_json, + ) + query = "Select Id, Subject from Account" + with caplog.at_level(logging.ERROR): + assert stream.create_stream_job(query, url) is None, "this stream should be skipped" + + # check logs + assert log_message in caplog.records[-1].message + + +@pytest.mark.parametrize( + "status_code,response_json,error_message", + [ + (400, [{"errorCode": "TXN_SECURITY_METERING_ERROR", "message": "We can't complete the action because enabled transaction security policies took too long to complete."}], 'A transient authentication error occurred. To prevent future syncs from failing, assign the "Exempt from Transaction Security" user permission to the authenticated user.'), + ] +) +def test_bulk_stream_error_on_wait_for_job(requests_mock, stream_config, stream_api, status_code, response_json, error_message): + + stream = generate_stream("Account", stream_config, stream_api) + url = f"{stream.sf_api.instance_url}/services/data/{stream.sf_api.version}/jobs/query/queryJobId" + requests_mock.register_uri( + "GET", + url, + status_code=status_code, + json=response_json, + ) + with pytest.raises(AirbyteTracedException) as e: + stream.wait_for_job(url=url) + assert e.value.message == error_message + + +@freezegun.freeze_time("2023-01-01") +def test_bulk_stream_slices(stream_config_date_format, stream_api): + stream: BulkIncrementalSalesforceStream = generate_stream("FakeBulkStream", stream_config_date_format, stream_api) + stream_slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) + expected_slices = [] + today = pendulum.today(tz="UTC") + start_date = pendulum.parse(stream.start_date, tz="UTC") + while start_date < today: + expected_slices.append({ + 'start_date': start_date.isoformat(timespec="milliseconds"), + 'end_date': min(today, start_date.add(days=stream.STREAM_SLICE_STEP)).isoformat(timespec="milliseconds") + }) + start_date = start_date.add(days=stream.STREAM_SLICE_STEP) + assert expected_slices == stream_slices + + +@freezegun.freeze_time("2023-04-01") +def test_bulk_stream_request_params_states(stream_config_date_format, stream_api, bulk_catalog, requests_mock): + """Check that request params ignore records cursor and use start date from slice ONLY""" + stream_config_date_format.update({"start_date": "2023-01-01"}) + stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api) + + source = SourceSalesforce() + source.streams = Mock() + source.streams.return_value = [stream] + + job_id_1 = "fake_job_1" + requests_mock.register_uri("GET", stream.path() + f"/{job_id_1}", [{"json": {"state": "JobComplete"}}]) + requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_1}") + requests_mock.register_uri("GET", stream.path() + f"/{job_id_1}/results", text="Field1,LastModifiedDate,ID\ntest,2023-01-15,1") + requests_mock.register_uri("PATCH", stream.path() + f"/{job_id_1}") + + job_id_2 = "fake_job_2" + requests_mock.register_uri("GET", stream.path() + f"/{job_id_2}", [{"json": {"state": "JobComplete"}}]) + requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_2}") + requests_mock.register_uri("GET", stream.path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22") + requests_mock.register_uri("PATCH", stream.path() + f"/{job_id_2}") + + job_id_3 = "fake_job_3" + queries_history = requests_mock.register_uri("POST", stream.path(), [{"json": {"id": job_id_1}}, + {"json": {"id": job_id_2}}, + {"json": {"id": job_id_3}}]) + requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}", [{"json": {"state": "JobComplete"}}]) + requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_3}") + requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,3") + requests_mock.register_uri("PATCH", stream.path() + f"/{job_id_3}") + + logger = logging.getLogger("airbyte") + state = {"Account": {"LastModifiedDate": "2023-01-01T10:10:10.000Z"}} + bulk_catalog.streams.pop(1) + result = [i for i in source.read(logger=logger, config=stream_config_date_format, catalog=bulk_catalog, state=state)] + + actual_state_values = [item.state.data.get("Account").get(stream.cursor_field) for item in result if item.type == Type.STATE] + # assert request params + assert "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" in queries_history.request_history[0].text + assert "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" in queries_history.request_history[1].text + assert "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" in queries_history.request_history[2].text + + # assert states + # if connector meets record with cursor `2023-04-01` out of current slice range 2023-01-31 <> 2023-03-02, we ignore all other values and set state to slice end_date + expected_state_values = ["2023-01-15T00:00:00+00:00", "2023-03-02T10:10:10+00:00", "2023-04-01T00:00:00+00:00"] + assert actual_state_values == expected_state_values diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index 72be013d2d58..b128dc65f1fe 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -130,10 +130,26 @@ def generate_stream(stream_name, stream_config, stream_api): def encoding_symbols_parameters(): - return [(x, "ISO-8859-1", b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) for x in range(1, 11)] + [ + return [(x, {"Content-Type": "text/csv; charset=ISO-8859-1"}, b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) for x in range(1, 11)] + [ ( x, - "utf-8", + {"Content-Type": "text/csv; charset=utf-8"}, + b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', + [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], + ) + for x in range(1, 11) + ] + [ + ( + x, + {"Content-Type": "text/csv"}, + b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', + [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], + ) + for x in range(1, 11) + ] + [ + ( + x, + {}, b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], ) diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py index 197566f7637a..75780d693822 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/test_memory.py @@ -27,16 +27,17 @@ ], ) def test_memory_download_data(stream_config, stream_api, n_records, first_size, first_peak): - job_full_url: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA" + job_full_url_results: str = "https://fase-account.salesforce.com/services/data/v57.0/jobs/query/7504W00000bkgnpQAA/results" stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) content = b'"Id","IsDeleted"' for _ in range(n_records): content += b'"0014W000027f6UwQAI","false"\n' with requests_mock.Mocker() as m: - m.register_uri("GET", f"{job_full_url}/results", content=content) + m.register_uri("GET", job_full_url_results, content=content) tracemalloc.start() - for x in stream.read_with_chunks(*stream.download_data(url=job_full_url)): + tmp_file, response_encoding, _ = stream.download_data(url=job_full_url_results) + for x in stream.read_with_chunks(tmp_file, response_encoding): pass fs, fp = tracemalloc.get_traced_memory() first_size_in_mb, first_peak_in_mb = fs / 1024**2, fp / 1024**2 diff --git a/airbyte-integrations/connectors/source-salesloft/Dockerfile b/airbyte-integrations/connectors/source-salesloft/Dockerfile index 357f6a83814c..b927379a10a7 100644 --- a/airbyte-integrations/connectors/source-salesloft/Dockerfile +++ b/airbyte-integrations/connectors/source-salesloft/Dockerfile @@ -34,5 +34,5 @@ COPY source_salesloft ./source_salesloft ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.2.0 LABEL io.airbyte.name=airbyte/source-salesloft diff --git a/airbyte-integrations/connectors/source-salesloft/acceptance-test-config.yml b/airbyte-integrations/connectors/source-salesloft/acceptance-test-config.yml index 89030aa5c987..fa9271151623 100644 --- a/airbyte-integrations/connectors/source-salesloft/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-salesloft/acceptance-test-config.yml @@ -1,51 +1,60 @@ acceptance_tests: basic_read: tests: - - config_path: secrets/config.json - expect_records: - path: integration_tests/expected_records.jsonl - ignored_fields: - people: - - name: locale_utc_offset - bypass_reason: volatile data - users: - - name: locale_utc_offset - bypass_reason: volatile data - fail_on_extra_columns: false - - config_path: secrets/config_oauth.json - expect_records: - path: integration_tests/expected_records.jsonl - fail_on_extra_columns: false + - config_path: secrets/config.json + expect_records: + path: integration_tests/expected_records.jsonl + ignored_fields: + actions: + - name: updated_at + bypass_reason: auto-updated by provider + people: + - name: locale_utc_offset + bypass_reason: volatile data + users: + - name: locale_utc_offset + bypass_reason: volatile data + fail_on_extra_columns: false + empty_streams: + - name: call_data_records + bypass_reason: "no records" + - config_path: secrets/config_oauth.json + expect_records: + path: integration_tests/expected_records.jsonl + fail_on_extra_columns: false + empty_streams: + - name: call_data_records + bypass_reason: "no records" connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: secrets/config_oauth.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: failed + - config_path: secrets/config.json + status: succeed + - config_path: secrets/config_oauth.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed discovery: tests: - - config_path: secrets/config.json - backward_compatibility_tests_config: - disable_for_version: "0.1.6" - - config_path: secrets/config_oauth.json - backward_compatibility_tests_config: - disable_for_version: "0.1.6" + - config_path: secrets/config.json + backward_compatibility_tests_config: + disable_for_version: "0.1.6" + - config_path: secrets/config_oauth.json + backward_compatibility_tests_config: + disable_for_version: "0.1.6" full_refresh: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json incremental: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/incremental_catalog.json - future_state: - future_state_path: integration_tests/abnormal_state.json + - config_path: secrets/config.json + configured_catalog_path: integration_tests/incremental_catalog.json + future_state: + future_state_path: integration_tests/abnormal_state.json spec: tests: - - spec_path: source_salesloft/spec.json - backward_compatibility_tests_config: - disable_for_version: "0.1.6" + - spec_path: source_salesloft/spec.json + backward_compatibility_tests_config: + disable_for_version: "0.1.6" connector_image: airbyte/source-salesloft:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json index 81a81225bad3..9055a2bf5273 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json @@ -83,11 +83,32 @@ "stream_descriptor": { "name": "team_templates" } } }, - { + { "type": "STREAM", "stream": { "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, "stream_descriptor": { "name": "email_templates" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "call_data_records" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "meetings" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "searches" } + } } ] diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json index 99a5e4c4847d..0c353aa8ae7c 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json @@ -4,19 +4,10 @@ "stream": { "name": "cadences", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -25,19 +16,10 @@ "stream": { "name": "cadence_memberships", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -46,19 +28,10 @@ "stream": { "name": "people", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -67,14 +40,8 @@ "stream": { "name": "users", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -83,19 +50,10 @@ "stream": { "name": "emails", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -104,19 +62,10 @@ "stream": { "name": "calls", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -125,15 +74,8 @@ "stream": { "name": "account_stages", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -142,14 +84,8 @@ "stream": { "name": "account_tiers", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -158,19 +94,10 @@ "stream": { "name": "accounts", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -179,19 +106,10 @@ "stream": { "name": "actions", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -200,15 +118,8 @@ "stream": { "name": "email_templates", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -217,14 +128,8 @@ "stream": { "name": "import", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -233,19 +138,10 @@ "stream": { "name": "notes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -254,14 +150,8 @@ "stream": { "name": "person_stages", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -270,14 +160,8 @@ "stream": { "name": "phone_number_assignments", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -286,14 +170,8 @@ "stream": { "name": "steps", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -302,15 +180,8 @@ "stream": { "name": "team_templates", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -319,14 +190,8 @@ "stream": { "name": "team_template_attachments", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -335,19 +200,10 @@ "stream": { "name": "crm_activities", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -356,14 +212,8 @@ "stream": { "name": "crm_users", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -372,14 +222,8 @@ "stream": { "name": "groups", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -388,19 +232,10 @@ "stream": { "name": "successes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" @@ -409,14 +244,72 @@ "stream": { "name": "email_template_attachments", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_data_records", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_dispositions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_sentiments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "meetings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "searches", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-salesloft/integration_tests/expected_records.jsonl index 2729d7bdf78e..a54421b99d69 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/expected_records.jsonl @@ -1,175 +1,70 @@ -{"stream": "cadences", "data": {"id": 340821, "created_at": "2023-03-07T12:47:07.697191-05:00", "updated_at": "2023-03-07T12:47:07.697191-05:00", "archived_at": null, "latest_active_date": null, "team_cadence": false, "shared": true, "remove_bounces_enabled": false, "remove_replies_enabled": false, "opt_out_link_included": false, "draft": false, "override_contact_restrictions": null, "cadence_framework_id": null, "cadence_function": "outbound", "name": "Cadence 1", "external_identifier": null, "tags": [], "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "bounced_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10147", "id": 10147}, "replied_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "added_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10149", "id": 10149}, "finished_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10151", "id": 10151}, "cadence_priority": null, "groups": [], "counts": {"cadence_people": null, "people_acted_on_count": 0, "target_daily_people": 0, "opportunities_created": 0, "meetings_booked": 0}}, "emitted_at": 1684327170434} -{"stream": "cadences", "data": {"id": 25591, "created_at": "2021-09-16T08:20:08.485246-04:00", "updated_at": "2023-03-07T12:43:17.450246-05:00", "archived_at": null, "latest_active_date": null, "team_cadence": false, "shared": true, "remove_bounces_enabled": false, "remove_replies_enabled": false, "opt_out_link_included": true, "draft": false, "override_contact_restrictions": null, "cadence_framework_id": null, "cadence_function": "event", "name": "New Cadence", "external_identifier": null, "tags": ["opt-out"], "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "bounced_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "replied_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "added_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "finished_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10149", "id": 10149}, "cadence_priority": {"_href": "https://api.salesloft.com/v2/cadence_priorities", "id": 3385}, "groups": [], "counts": {"cadence_people": 11, "people_acted_on_count": 0, "target_daily_people": 0, "opportunities_created": 0, "meetings_booked": 0}}, "emitted_at": 1684327170434} -{"stream": "cadence_memberships", "data": {"id": 71245543, "added_at": "2023-03-07T12:12:53.179454-05:00", "created_at": "2023-03-07T12:12:53.186833-05:00", "updated_at": "2023-03-07T12:12:53.210155-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358221", "id": 103358221}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305169}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550188} -{"stream": "cadence_memberships", "data": {"id": 71245540, "added_at": "2023-03-07T12:12:53.104252-05:00", "created_at": "2023-03-07T12:12:53.111202-05:00", "updated_at": "2023-03-07T12:12:53.140663-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358220", "id": 103358220}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305164}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550189} -{"stream": "cadence_memberships", "data": {"id": 71245537, "added_at": "2023-03-07T12:12:53.034821-05:00", "created_at": "2023-03-07T12:12:53.043064-05:00", "updated_at": "2023-03-07T12:12:53.064853-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358219", "id": 103358219}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305158}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550190} -{"stream": "cadence_memberships", "data": {"id": 71245536, "added_at": "2023-03-07T12:12:52.968153-05:00", "created_at": "2023-03-07T12:12:52.976764-05:00", "updated_at": "2023-03-07T12:12:53.000579-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358218", "id": 103358218}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305154}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550190} -{"stream": "cadence_memberships", "data": {"id": 71245534, "added_at": "2023-03-07T12:12:52.912138-05:00", "created_at": "2023-03-07T12:12:52.918424-05:00", "updated_at": "2023-03-07T12:12:52.935680-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358217", "id": 103358217}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305151}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550190} -{"stream": "cadence_memberships", "data": {"id": 71245530, "added_at": "2023-03-07T12:12:52.849515-05:00", "created_at": "2023-03-07T12:12:52.857286-05:00", "updated_at": "2023-03-07T12:12:52.881880-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358215", "id": 103358215}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305142}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550191} -{"stream": "cadence_memberships", "data": {"id": 71245528, "added_at": "2023-03-07T12:12:52.772223-05:00", "created_at": "2023-03-07T12:12:52.780135-05:00", "updated_at": "2023-03-07T12:12:52.808970-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358214", "id": 103358214}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305138}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550191} -{"stream": "cadence_memberships", "data": {"id": 71245525, "added_at": "2023-03-07T12:12:52.695686-05:00", "created_at": "2023-03-07T12:12:52.702356-05:00", "updated_at": "2023-03-07T12:12:52.721273-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358213", "id": 103358213}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305133}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550191} -{"stream": "cadence_memberships", "data": {"id": 71245522, "added_at": "2023-03-07T12:12:52.625165-05:00", "created_at": "2023-03-07T12:12:52.631608-05:00", "updated_at": "2023-03-07T12:12:52.652336-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358212", "id": 103358212}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305126}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550192} -{"stream": "cadence_memberships", "data": {"id": 71245520, "added_at": "2023-03-07T12:12:52.510254-05:00", "created_at": "2023-03-07T12:12:52.522770-05:00", "updated_at": "2023-03-07T12:12:52.558480-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358211", "id": 103358211}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305124}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550192} -{"stream": "cadence_memberships", "data": {"id": 4482611, "added_at": "2021-09-16T10:22:15.874489-04:00", "created_at": "2021-09-16T10:22:15.886276-04:00", "updated_at": "2021-09-16T10:22:15.920756-04:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 19115635}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1682600550192} -{"stream": "people", "data": {"id": 103358219, "created_at": "2023-03-07T12:12:52.362090-05:00", "updated_at": "2023-03-07T12:49:27.205203-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User8", "last_name": "Sample", "display_name": "User8 Sample", "email_address": "user8.sample.airbyte@outlook.com", "full_email_address": "\"User8 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": "16205829403", "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 8", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600550999} -{"stream": "people", "data": {"id": 103358220, "created_at": "2023-03-07T12:12:52.405367-05:00", "updated_at": "2023-03-07T12:49:12.464381-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User9", "last_name": "Sample", "display_name": "User9 Sample", "email_address": "user9.sample.airbyte@outlook.com", "full_email_address": "\"User9 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": "19125901057", "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 9", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551000} -{"stream": "people", "data": {"id": 103358221, "created_at": "2023-03-07T12:12:52.433805-05:00", "updated_at": "2023-03-07T12:48:51.576983-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User10", "last_name": "Sample", "display_name": "User10 Sample", "email_address": "user10.sample.airbyte@outlook.com", "full_email_address": "\"User10 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": "14246220939", "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 10", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551000} -{"stream": "people", "data": {"id": 103358218, "created_at": "2023-03-07T12:12:52.332996-05:00", "updated_at": "2023-03-07T12:21:29.512819-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User7", "last_name": "Sample", "display_name": "User7 Sample", "email_address": "user7.sample.airbyte@outlook.com", "full_email_address": "\"User7 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 7", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551001} -{"stream": "people", "data": {"id": 103358217, "created_at": "2023-03-07T12:12:52.304206-05:00", "updated_at": "2023-03-07T12:21:29.493399-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User6", "last_name": "Sample", "display_name": "User6 Sample", "email_address": "user6.sample.airbyte@outlook.com", "full_email_address": "\"User6 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 6", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551001} -{"stream": "people", "data": {"id": 103358215, "created_at": "2023-03-07T12:12:52.273260-05:00", "updated_at": "2023-03-07T12:21:29.483727-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User5", "last_name": "Sample", "display_name": "User5 Sample", "email_address": "user5.sample.airbyte@outlook.com", "full_email_address": "\"User5 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 5", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551002} -{"stream": "people", "data": {"id": 103358214, "created_at": "2023-03-07T12:12:52.241485-05:00", "updated_at": "2023-03-07T12:21:29.478054-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User4", "last_name": "Sample", "display_name": "User4 Sample", "email_address": "user4.sample.airbyte@outlook.com", "full_email_address": "\"User4 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 4", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551002} -{"stream": "people", "data": {"id": 103358213, "created_at": "2023-03-07T12:12:52.209832-05:00", "updated_at": "2023-03-07T12:21:29.467289-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User3", "last_name": "Sample", "display_name": "User3 Sample", "email_address": "user3.sample.airbyte@outlook.com", "full_email_address": "\"User3 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 3", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551002} -{"stream": "people", "data": {"id": 103358212, "created_at": "2023-03-07T12:12:52.173789-05:00", "updated_at": "2023-03-07T12:21:29.435970-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User2", "last_name": "Sample", "display_name": "User2 Sample", "email_address": "user2.sample.airbyte@gmail.com", "full_email_address": "\"User2 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 2", "person_company_website": "http://gmail.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092357", "id": 35092357}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551003} -{"stream": "people", "data": {"id": 103358211, "created_at": "2023-03-07T12:12:52.141355-05:00", "updated_at": "2023-03-07T12:21:29.433659-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User1", "last_name": "Sample", "display_name": "User1 Sample", "email_address": "user1.sample@zohomail.eu", "full_email_address": "\"User1 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 1", "person_company_website": "http://zohomail.eu", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092356", "id": 35092356}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551003} -{"stream": "people", "data": {"id": 6509414, "created_at": "2021-09-16T09:54:45.082693-04:00", "updated_at": "2021-09-16T11:03:46.660816-04:00", "last_contacted_at": "2021-09-16T11:02:51.405506-04:00", "last_replied_at": null, "first_name": "Kelly", "last_name": "Irish", "display_name": "Kelly Irish", "email_address": "kellyirish@google.com", "full_email_address": "\"Kelly Irish\" ", "secondary_email_address": null, "personal_email_address": null, "phone": "+1234445556", "phone_extension": null, "home_phone": null, "mobile_phone": "+1234445557", "linkedin_url": null, "title": "seller", "city": null, "state": null, "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": null, "person_company_website": "http://google.com", "person_company_industry": null, "do_not_contact": false, "bouncing": true, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": "email", "job_seniority": "unknown", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 1, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 1, "calls": 1}, "account": {"_href": "https://api.salesloft.com/v2/accounts/2366578", "id": 2366578}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "import": null, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1682600551004} -{"stream": "people", "data": {"id": 6502956, "created_at": "2021-09-16T07:02:46.804979-04:00", "updated_at": "2021-09-16T10:26:17.103697-04:00", "last_contacted_at": "2021-09-16T09:31:12.895350-04:00", "last_replied_at": null, "first_name": "Karina", "last_name": "Kuznietsova", "display_name": "Karina Kuznietsova", "email_address": "karina.kuznietsova@zazmc.com", "full_email_address": "\"Karina Kuznietsova\" ", "secondary_email_address": null, "personal_email_address": null, "phone": "+12345678900", "phone_extension": null, "home_phone": null, "mobile_phone": "+12334567778", "linkedin_url": null, "title": "QA", "city": "San Francisco", "state": "CA", "country": "US", "work_city": "Los Angeles", "work_state": "CA", "work_country": "US", "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": null, "person_company_website": "http://zazmc.com", "person_company_industry": "engineering", "do_not_contact": false, "bouncing": true, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": "email", "job_seniority": "individual_contributor", "custom_fields": {}, "tags": ["airbyte"], "contact_restrictions": [], "success_count": 1, "starred": false, "untouched": false, "counts": {"emails_sent": 3, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 3, "calls": 1}, "account": null, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "import": null, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": null, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": []}, "emitted_at": 1682600551004} -{"stream": "people", "data": {"id": 6503685, "created_at": "2021-09-16T07:24:03.591827-04:00", "updated_at": "2021-09-16T07:24:46.650636-04:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "Tailor", "last_name": "Moon", "display_name": "Tailor Moon", "email_address": "tailormoon@yhoo.com", "full_email_address": "\"Tailor Moon\" ", "secondary_email_address": null, "personal_email_address": null, "phone": "+12345543210", "phone_extension": null, "home_phone": null, "mobile_phone": null, "linkedin_url": null, "title": "Manager", "city": "San Francisco", "state": "CA", "country": "US", "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": null, "person_company_website": "http://yhoo.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": true, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/2364729", "id": 2364729}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": null, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": null, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": []}, "emitted_at": 1682600551005} -{"stream": "users", "data": {"id": 52180, "guid": "58d151f8-9b0b-4422-858a-40ea165edc46", "created_at": "2023-03-07T12:28:31.354490-05:00", "updated_at": "2023-03-07T13:10:23.281437-05:00", "name": "User11 Sample", "first_name": "User11", "last_name": "Sample", "job_role": "Operations", "active": true, "time_zone": "US/Eastern", "locale_utc_offset": -240, "slack_username": "iryna.grankova", "twitter_handle": null, "email": "iryna.grankova@airbyte.io", "email_client_email_address": "iryna.grankova@airbyte.io", "sending_email_address": "iryna.grankova@airbyte.io", "from_address": null, "full_email_address": "\"User11 Sample\" ", "bcc_email_address": null, "work_country": "UA", "seat_package": "prospect", "email_signature": "", "email_signature_type": "html", "email_signature_click_tracking_disabled": false, "team_admin": false, "local_dial_enabled": true, "click_to_call_enabled": false, "email_client_configured": false, "crm_connected": false, "external_feature_flags": {"ma_enabled": true, "ma_live_feed": true, "ma_recent_activities_update": true, "ma_person_page": true, "ma_push_notifications": true, "ma_meetings_push_notifications": true, "ma_websockets": true, "ma_edit_person": true, "ma_dev_qa_tools": false, "ma_cadences": false, "hot_leads": true, "people_crud_allow_create": true, "people_crud_allow_delete": true, "ma_mobile_workflow": true, "linkedin_oauth_flow": true}, "_private_fields": {}, "phone_client": {"id": 53219}, "phone_number_assignment": {"_href": "https://api.salesloft.com/v2/phone_number_assignments/34936", "id": 34936}, "group": null, "team": {"_href": "https://api.salesloft.com/v2/team", "id": 104779}, "role": {"id": "User"}}, "emitted_at": 1682600551673} -{"stream": "users", "data": {"id": 6970, "guid": "54713f8f-6283-453f-84da-e27e28223e8c", "created_at": "2021-09-03T12:04:02.458089-04:00", "updated_at": "2023-03-09T14:50:58.222994-05:00", "name": "Airbyte Team", "first_name": "Airbyte", "last_name": "Team", "job_role": "Account Executive / Account Manager", "active": true, "time_zone": "US/Eastern", "locale_utc_offset": -240, "slack_username": "integration-test", "twitter_handle": null, "email": "integration-test@airbyte.io", "email_client_email_address": "karina.kuznietsova@zazmic.com", "sending_email_address": "karina.kuznietsova@zazmic.com", "from_address": null, "full_email_address": "\"Airbyte Team\" ", "bcc_email_address": null, "work_country": "US", "seat_package": "admin", "email_signature": "", "email_signature_type": "html", "email_signature_click_tracking_disabled": false, "team_admin": true, "local_dial_enabled": true, "click_to_call_enabled": false, "email_client_configured": true, "crm_connected": true, "external_feature_flags": {"ma_enabled": true, "ma_live_feed": true, "ma_recent_activities_update": true, "ma_person_page": true, "ma_push_notifications": true, "ma_meetings_push_notifications": true, "ma_websockets": true, "ma_edit_person": true, "ma_dev_qa_tools": false, "ma_cadences": false, "hot_leads": true, "people_crud_allow_create": true, "people_crud_allow_delete": true, "ma_mobile_workflow": true, "linkedin_oauth_flow": true}, "_private_fields": {}, "phone_client": {"id": 7593}, "phone_number_assignment": {"_href": "https://api.salesloft.com/v2/phone_number_assignments/34935", "id": 34935}, "group": null, "team": {"_href": "https://api.salesloft.com/v2/team", "id": 104779}, "role": {"id": "Admin"}}, "emitted_at": 1682600551674} -{"stream": "emails", "data": {"id": 8202787, "created_at": "2021-09-16T10:02:20.479210-04:00", "updated_at": "2021-09-16T11:03:06.789805-04:00", "recipient_email_address": "kellyirish@google.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T11:02:21.934859-04:00", "sent_at": "2021-09-16T11:02:51.405506-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "mailing": {"id": 3124568}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11367506", "id": 11367506}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1682600552239} -{"stream": "emails", "data": {"id": 8194214, "created_at": "2021-09-16T09:30:13.172463-04:00", "updated_at": "2021-09-16T09:31:23.363226-04:00", "recipient_email_address": "karina.kuznietsova@zazmc.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T09:30:27.343687-04:00", "sent_at": "2021-09-16T09:31:12.895350-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "mailing": {"id": 3122753}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11334017", "id": 11334017}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1682600552240} -{"stream": "emails", "data": {"id": 8190809, "created_at": "2021-09-16T09:04:44.160528-04:00", "updated_at": "2021-09-16T09:05:48.392667-04:00", "recipient_email_address": "karina.kuznietsova@zazmc.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T09:05:06.374217-04:00", "sent_at": "2021-09-16T09:05:39.849349-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "mailing": {"id": 3121331}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11324527", "id": 11324527}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1682600552240} -{"stream": "emails", "data": {"id": 8182634, "created_at": "2021-09-16T07:17:40.583164-04:00", "updated_at": "2021-09-16T07:18:16.263040-04:00", "recipient_email_address": "karina.kuznietsova@zazmc.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T07:17:42.594827-04:00", "sent_at": "2021-09-16T07:18:07.913740-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "mailing": {"id": 3118814}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11305642", "id": 11305642}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1682600552240} -{"stream": "calls", "data": {"id": 2117241, "to": "+1234445556", "duration": null, "sentiment": "Customer", "disposition": "No Answer", "created_at": "2021-09-16T10:04:58.125490-04:00", "updated_at": "2021-09-16T10:04:58.295640-04:00", "recordings": [], "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "action": null, "called_person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11346104", "id": 11346104}, "note": {"_href": "https://api.salesloft.com/v2/notes/1029422", "id": 1029422}, "cadence": null, "step": null}, "emitted_at": 1682600552756} -{"stream": "calls", "data": {"id": 2113993, "to": "+12345678900", "duration": null, "sentiment": null, "disposition": null, "created_at": "2021-09-16T07:17:57.430595-04:00", "updated_at": "2021-09-16T07:17:57.499713-04:00", "recordings": [], "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "action": null, "called_person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11305624", "id": 11305624}, "note": null, "cadence": null, "step": null}, "emitted_at": 1682600552756} -{"stream": "account_stages", "data": {"id": 5006, "name": "Disqualified", "created_at": "2021-09-03T12:03:15.863631-04:00", "updated_at": "2021-09-03T12:03:15.863631-04:00", "order": 3}, "emitted_at": 1682600553230} -{"stream": "account_stages", "data": {"id": 5005, "name": "Completed", "created_at": "2021-09-03T12:03:15.850021-04:00", "updated_at": "2021-09-03T12:03:15.850021-04:00", "order": 2}, "emitted_at": 1682600553231} -{"stream": "account_stages", "data": {"id": 5004, "name": "Working", "created_at": "2021-09-03T12:03:15.835536-04:00", "updated_at": "2021-09-03T12:03:15.835536-04:00", "order": 1}, "emitted_at": 1682600553231} -{"stream": "account_stages", "data": {"id": 5003, "name": "Open", "created_at": "2021-09-03T12:03:15.817610-04:00", "updated_at": "2021-09-03T12:03:15.817610-04:00", "order": 0}, "emitted_at": 1682600553231} -{"stream": "account_tiers", "data": {"id": 3469, "name": "Tier 3", "order": 2, "created_at": "2021-09-03T12:03:16.281412-04:00", "updated_at": "2021-09-03T12:03:16.281412-04:00"}, "emitted_at": 1682600553715} -{"stream": "account_tiers", "data": {"id": 3468, "name": "Tier 2", "order": 1, "created_at": "2021-09-03T12:03:16.270440-04:00", "updated_at": "2021-09-03T12:03:16.270440-04:00"}, "emitted_at": 1682600553716} -{"stream": "account_tiers", "data": {"id": 3467, "name": "Tier 1", "order": 0, "created_at": "2021-09-03T12:03:16.235256-04:00", "updated_at": "2021-09-03T12:03:16.235256-04:00"}, "emitted_at": 1682600553716} -{"stream": "accounts", "data": {"id": 35092356, "created_at": "2023-03-07T12:12:53.353476-05:00", "updated_at": "2023-03-07T12:42:45.816560-05:00", "archived_at": null, "name": "Company 1", "domain": "zohomail.eu", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": [], "counts": {"people": 1}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": null, "account_tier": null}, "emitted_at": 1682600554256} -{"stream": "accounts", "data": {"id": 35092358, "created_at": "2023-03-07T12:12:53.436918-05:00", "updated_at": "2023-03-07T12:42:45.540331-05:00", "archived_at": null, "name": "Company 4", "domain": "outlook.com", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": [], "counts": {"people": 8}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": null, "account_tier": null}, "emitted_at": 1682600554257} -{"stream": "accounts", "data": {"id": 35092357, "created_at": "2023-03-07T12:12:53.393425-05:00", "updated_at": "2023-03-07T12:42:42.384044-05:00", "archived_at": null, "name": "Company 2", "domain": "gmail.com", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": [], "counts": {"people": 1}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": null, "account_tier": null}, "emitted_at": 1682600554258} -{"stream": "accounts", "data": {"id": 2366578, "created_at": "2021-09-16T09:39:28.916301-04:00", "updated_at": "2021-09-16T11:02:51.482085-04:00", "archived_at": null, "name": "Karina Testing", "domain": "karina.io", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": "2021-09-16T11:02:51.405506-04:00", "last_contacted_type": "email", "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": ["salesloft"], "counts": {"people": 1}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "company_stage": null, "account_tier": null}, "emitted_at": 1682600554258} -{"stream": "accounts", "data": {"id": 2364729, "created_at": "2021-09-16T06:57:53.029439-04:00", "updated_at": "2021-09-16T09:38:36.386606-04:00", "archived_at": null, "name": "Airbyte Testing", "domain": "airbyte.io", "conversational_name": "Karina", "description": "Test account", "phone": "14088669100", "website": "https://airbyte.io/", "linkedin_url": null, "twitter_handle": null, "street": "29th Avenue", "city": "San Francisco", "state": "CA", "postal_code": "94121", "country": "US", "locale": "US/Pacific", "industry": "engineering", "company_type": "Main", "founded": "2020", "revenue_range": "5000000", "size": "14", "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": ["airbyte", "salesloft"], "counts": {"people": 1}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": {"_href": "https://api.salesloft.com/v2/account_stages/5004", "id": 5004}, "account_tier": {"_href": "https://api.salesloft.com/v2/account_tiers/3467", "id": 3467}}, "emitted_at": 1682600554258} -{"stream": "actions", "data": {"id": 343305169, "due": true, "created_at": "2023-03-07T12:12:53.201684-05:00", "updated_at": "2023-03-07T12:12:53.201684-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358221", "id": 103358221}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554780} -{"stream": "actions", "data": {"id": 343305164, "due": true, "created_at": "2023-03-07T12:12:53.128204-05:00", "updated_at": "2023-03-07T12:12:53.128204-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358220", "id": 103358220}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554781} -{"stream": "actions", "data": {"id": 343305158, "due": true, "created_at": "2023-03-07T12:12:53.058448-05:00", "updated_at": "2023-03-07T12:12:53.058448-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358219", "id": 103358219}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554782} -{"stream": "actions", "data": {"id": 343305154, "due": true, "created_at": "2023-03-07T12:12:52.993213-05:00", "updated_at": "2023-03-07T12:12:52.993213-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358218", "id": 103358218}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554782} -{"stream": "actions", "data": {"id": 343305151, "due": true, "created_at": "2023-03-07T12:12:52.930479-05:00", "updated_at": "2023-03-07T12:12:52.930479-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358217", "id": 103358217}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554782} -{"stream": "actions", "data": {"id": 343305142, "due": true, "created_at": "2023-03-07T12:12:52.876008-05:00", "updated_at": "2023-03-07T12:12:52.876008-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358215", "id": 103358215}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554782} -{"stream": "actions", "data": {"id": 343305138, "due": true, "created_at": "2023-03-07T12:12:52.799215-05:00", "updated_at": "2023-03-07T12:12:52.799215-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358214", "id": 103358214}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554783} -{"stream": "actions", "data": {"id": 343305133, "due": true, "created_at": "2023-03-07T12:12:52.715735-05:00", "updated_at": "2023-03-07T12:12:52.715735-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358213", "id": 103358213}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554783} -{"stream": "actions", "data": {"id": 343305126, "due": true, "created_at": "2023-03-07T12:12:52.646430-05:00", "updated_at": "2023-03-07T12:12:52.646430-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358212", "id": 103358212}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554783} -{"stream": "actions", "data": {"id": 343305124, "due": true, "created_at": "2023-03-07T12:12:52.543926-05:00", "updated_at": "2023-03-07T12:12:52.543926-05:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358211", "id": 103358211}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554783} -{"stream": "actions", "data": {"id": 19115635, "due": true, "created_at": "2021-09-16T10:22:15.911633-04:00", "updated_at": "2021-09-16T10:22:15.911633-04:00", "type": "email", "status": "in_progress", "due_on": "2021-09-16T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1682600554784} -{"stream": "email_templates", "data": {"id": 6605423, "title": "Team Template 6", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.552863-05:00", "updated_at": "2023-03-07T12:28:31.552863-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605423"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/353f478c-c5dd-4c36-8417-3ad5bb88c927", "id": "353f478c-c5dd-4c36-8417-3ad5bb88c927"}, "groups": []}, "emitted_at": 1682600555791} -{"stream": "email_templates", "data": {"id": 6605422, "title": "Team Template 7", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.526620-05:00", "updated_at": "2023-03-07T12:28:31.526620-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605422"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f", "id": "cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f"}, "groups": []}, "emitted_at": 1682600555792} -{"stream": "email_templates", "data": {"id": 6605421, "title": "Team Template 10", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.524798-05:00", "updated_at": "2023-03-07T12:28:31.524798-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605421"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/75621893-cf7f-41d3-bb45-6d31afc8914c", "id": "75621893-cf7f-41d3-bb45-6d31afc8914c"}, "groups": []}, "emitted_at": 1682600555792} -{"stream": "email_templates", "data": {"id": 6605420, "title": "Team Template 9", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.518429-05:00", "updated_at": "2023-03-07T12:28:31.518429-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605420"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/858de732-204b-4a58-a870-f3b551d2d6f3", "id": "858de732-204b-4a58-a870-f3b551d2d6f3"}, "groups": []}, "emitted_at": 1682600555793} -{"stream": "email_templates", "data": {"id": 6605419, "title": "Team Template 8", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.515791-05:00", "updated_at": "2023-03-07T12:28:31.515791-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605419"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/d90faaf7-903c-4f1d-b3c1-973c7fa9531d", "id": "d90faaf7-903c-4f1d-b3c1-973c7fa9531d"}, "groups": []}, "emitted_at": 1682600555793} -{"stream": "email_templates", "data": {"id": 6605418, "title": "Team Template 1", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.505073-05:00", "updated_at": "2023-03-07T12:28:31.505073-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605418"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2ae537a5-a2fe-48c5-a2f2-df3d110c3af3", "id": "2ae537a5-a2fe-48c5-a2f2-df3d110c3af3"}, "groups": []}, "emitted_at": 1682600555793} -{"stream": "email_templates", "data": {"id": 6605416, "title": "Team Template 4", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.497559-05:00", "updated_at": "2023-03-07T12:28:31.497559-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605416"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c9cb00f1-0c63-4440-80fd-a58b68affe5d", "id": "c9cb00f1-0c63-4440-80fd-a58b68affe5d"}, "groups": []}, "emitted_at": 1682600555793} -{"stream": "email_templates", "data": {"id": 6605415, "title": "Team Template 5", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.495168-05:00", "updated_at": "2023-03-07T12:28:31.495168-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605415"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2e31b292-dcd1-48fd-b4df-4b788090d487", "id": "2e31b292-dcd1-48fd-b4df-4b788090d487"}, "groups": []}, "emitted_at": 1682600555794} -{"stream": "email_templates", "data": {"id": 6605414, "title": "Team Template 3", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.489992-05:00", "updated_at": "2023-03-07T12:28:31.489992-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605414"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c2d18470-2c55-4dd0-a421-c69f993c0487", "id": "c2d18470-2c55-4dd0-a421-c69f993c0487"}, "groups": []}, "emitted_at": 1682600555794} -{"stream": "email_templates", "data": {"id": 6605413, "title": "Team Template 2", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.483018-05:00", "updated_at": "2023-03-07T12:28:31.483018-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605413"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/0d7e1891-893d-47ed-aa4c-9b30dd484a35", "id": "0d7e1891-893d-47ed-aa4c-9b30dd484a35"}, "groups": []}, "emitted_at": 1682600555795} -{"stream": "email_templates", "data": {"id": 6602890, "title": "Test Email Template 9", "subject": "Test Subject", "body": "
    \n
    Tag9
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Tag9 \nTest body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:05:57.501642-05:00", "updated_at": "2023-03-07T12:21:55.154868-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602890"}, "tags": ["Tag9"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555795} -{"stream": "email_templates", "data": {"id": 6604896, "title": "Team Template 10", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:43.204612-05:00", "updated_at": "2023-03-07T12:19:50.648106-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604896"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/75621893-cf7f-41d3-bb45-6d31afc8914c", "id": "75621893-cf7f-41d3-bb45-6d31afc8914c"}, "groups": []}, "emitted_at": 1682600555795} -{"stream": "email_templates", "data": {"id": 6604894, "title": "Team Template 9", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:31.910224-05:00", "updated_at": "2023-03-07T12:19:37.960137-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604894"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/858de732-204b-4a58-a870-f3b551d2d6f3", "id": "858de732-204b-4a58-a870-f3b551d2d6f3"}, "groups": []}, "emitted_at": 1682600555795} -{"stream": "email_templates", "data": {"id": 6604779, "title": "Team Template 8", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:21.638522-05:00", "updated_at": "2023-03-07T12:19:26.970641-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604779"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/d90faaf7-903c-4f1d-b3c1-973c7fa9531d", "id": "d90faaf7-903c-4f1d-b3c1-973c7fa9531d"}, "groups": []}, "emitted_at": 1682600555796} -{"stream": "email_templates", "data": {"id": 6604723, "title": "Team Template 7", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:10.650378-05:00", "updated_at": "2023-03-07T12:19:16.419317-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604723"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f", "id": "cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f"}, "groups": []}, "emitted_at": 1682600555796} -{"stream": "email_templates", "data": {"id": 6604722, "title": "Team Template 6", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:56.781108-05:00", "updated_at": "2023-03-07T12:19:05.569243-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604722"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/353f478c-c5dd-4c36-8417-3ad5bb88c927", "id": "353f478c-c5dd-4c36-8417-3ad5bb88c927"}, "groups": []}, "emitted_at": 1682600555796} -{"stream": "email_templates", "data": {"id": 6604721, "title": "Team Template 5", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:45.877812-05:00", "updated_at": "2023-03-07T12:18:51.721623-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604721"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2e31b292-dcd1-48fd-b4df-4b788090d487", "id": "2e31b292-dcd1-48fd-b4df-4b788090d487"}, "groups": []}, "emitted_at": 1682600555797} -{"stream": "email_templates", "data": {"id": 6604719, "title": "Team Template 4", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:35.440122-05:00", "updated_at": "2023-03-07T12:18:40.690325-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604719"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c9cb00f1-0c63-4440-80fd-a58b68affe5d", "id": "c9cb00f1-0c63-4440-80fd-a58b68affe5d"}, "groups": []}, "emitted_at": 1682600555797} -{"stream": "email_templates", "data": {"id": 6604714, "title": "Team Template 3", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:25.251366-05:00", "updated_at": "2023-03-07T12:18:30.509891-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604714"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c2d18470-2c55-4dd0-a421-c69f993c0487", "id": "c2d18470-2c55-4dd0-a421-c69f993c0487"}, "groups": []}, "emitted_at": 1682600555797} -{"stream": "email_templates", "data": {"id": 6604713, "title": "Team Template 2", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:14.459208-05:00", "updated_at": "2023-03-07T12:18:20.397620-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604713"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/0d7e1891-893d-47ed-aa4c-9b30dd484a35", "id": "0d7e1891-893d-47ed-aa4c-9b30dd484a35"}, "groups": []}, "emitted_at": 1682600555797} -{"stream": "email_templates", "data": {"id": 6604711, "title": "Team Template 1", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:08.212272-05:00", "updated_at": "2023-03-07T12:18:08.212272-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6604711"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2ae537a5-a2fe-48c5-a2f2-df3d110c3af3", "id": "2ae537a5-a2fe-48c5-a2f2-df3d110c3af3"}, "groups": []}, "emitted_at": 1682600555798} -{"stream": "email_templates", "data": {"id": 6602865, "title": "Test Email Template 3", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:04:10.975386-05:00", "updated_at": "2023-03-07T11:10:53.197963-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602865"}, "tags": ["Tag3"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555798} -{"stream": "email_templates", "data": {"id": 6602856, "title": "Test Email Template 1", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:03:47.663912-05:00", "updated_at": "2023-03-07T11:10:35.000495-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602856"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555798} -{"stream": "email_templates", "data": {"id": 6602891, "title": "Test Email Template 10", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:06:09.023206-05:00", "updated_at": "2023-03-07T11:06:15.975470-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602891"}, "tags": ["Tag10", "Tag2"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555799} -{"stream": "email_templates", "data": {"id": 6602889, "title": "Test Email Template 8", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:05:46.915871-05:00", "updated_at": "2023-03-07T11:05:52.679742-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602889"}, "tags": ["Tag8"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555799} -{"stream": "email_templates", "data": {"id": 6602887, "title": "Test Email Template 7", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:05:35.470980-05:00", "updated_at": "2023-03-07T11:05:41.102561-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602887"}, "tags": ["Tag7"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555799} -{"stream": "email_templates", "data": {"id": 6602885, "title": "Test Email Template 6", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:05:22.964779-05:00", "updated_at": "2023-03-07T11:05:29.509996-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602885"}, "tags": ["Tag6"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555799} -{"stream": "email_templates", "data": {"id": 6602883, "title": "Test Email Template 5", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:05:12.722084-05:00", "updated_at": "2023-03-07T11:05:18.659351-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602883"}, "tags": ["Tag5"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555800} -{"stream": "email_templates", "data": {"id": 6602880, "title": "Test Email Template 4", "subject": "Test Subject", "body": "
    \n
    \"\"Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:05:01.756029-05:00", "updated_at": "2023-03-07T11:05:07.264650-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602880"}, "tags": ["Tag4"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555800} -{"stream": "email_templates", "data": {"id": 6602857, "title": "Test Email Template 2", "subject": "Test Subject", "body": "
    \n
    Test body
    \n
     
    \n
    Best Regards,
    \n
    Test signature
    \n
    ", "body_preview": "Test body \n\u00a0 \nBest Regards, \nTest signature", "created_at": "2023-03-07T11:03:57.182749-05:00", "updated_at": "2023-03-07T11:04:04.381615-05:00", "last_used_at": null, "archived_at": null, "shared": true, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": false, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6602857"}, "tags": ["Tag2"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "team_template": null, "groups": []}, "emitted_at": 1682600555800} -{"stream": "import", "data": {"id": 5174826, "created_at": "2023-03-07T12:12:52.105709-05:00", "updated_at": "2023-03-07T12:12:52.105709-05:00", "name": "People_Salesloft.csv", "current_people_count": 10, "imported_people_count": 10}, "emitted_at": 1682600556324} -{"stream": "notes", "data": {"id": 1029422, "content": "missed call", "created_at": "2021-09-16T10:04:58.109554-04:00", "updated_at": "2021-09-16T10:04:58.109554-04:00", "associated_type": "person", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "associated_with": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "call": {"_href": "https://api.salesloft.com/v2/activities/calls/2117241", "id": 2117241}}, "emitted_at": 1682600556828} -{"stream": "notes", "data": {"id": 1028798, "content": "this is a new note", "created_at": "2021-09-16T09:37:38.057127-04:00", "updated_at": "2021-09-16T09:38:08.829201-04:00", "associated_type": "account", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "associated_with": {"_href": "https://api.salesloft.com/v2/accounts/2364729", "id": 2364729}, "call": null}, "emitted_at": 1682600556829} -{"stream": "notes", "data": {"id": 1027747, "content": "Test Note", "created_at": "2021-09-16T07:18:35.368918-04:00", "updated_at": "2021-09-16T07:18:35.368918-04:00", "associated_type": "person", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "associated_with": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "call": null}, "emitted_at": 1682600556829} -{"stream": "person_stages", "data": {"id": 10154, "name": "Replied", "created_at": "2021-09-03T12:03:15.801147-04:00", "updated_at": "2021-09-03T12:03:15.801147-04:00", "order": 7}, "emitted_at": 1682600557293} -{"stream": "person_stages", "data": {"id": 10153, "name": "Completed", "created_at": "2021-09-03T12:03:15.784892-04:00", "updated_at": "2021-09-03T12:03:15.784892-04:00", "order": 6}, "emitted_at": 1682600557294} -{"stream": "person_stages", "data": {"id": 10152, "name": "Do Not Contact", "created_at": "2021-09-03T12:03:15.771896-04:00", "updated_at": "2021-09-03T12:03:15.771896-04:00", "order": 5}, "emitted_at": 1682600557294} -{"stream": "person_stages", "data": {"id": 10151, "name": "Bounced", "created_at": "2021-09-03T12:03:15.757535-04:00", "updated_at": "2021-09-03T12:03:15.757535-04:00", "order": 4}, "emitted_at": 1682600557294} -{"stream": "person_stages", "data": {"id": 10150, "name": "Closed- Not Converted", "created_at": "2021-09-03T12:03:15.735123-04:00", "updated_at": "2021-09-03T12:03:15.735123-04:00", "order": 3}, "emitted_at": 1682600557294} -{"stream": "person_stages", "data": {"id": 10149, "name": "Closed- Converted", "created_at": "2021-09-03T12:03:15.690611-04:00", "updated_at": "2021-09-03T12:03:15.690611-04:00", "order": 2}, "emitted_at": 1682600557294} -{"stream": "person_stages", "data": {"id": 10148, "name": "Working", "created_at": "2021-09-03T12:03:15.669008-04:00", "updated_at": "2021-09-03T12:03:15.669008-04:00", "order": 1}, "emitted_at": 1682600557294} -{"stream": "person_stages", "data": {"id": 10147, "name": "Open", "created_at": "2021-09-03T12:03:15.642239-04:00", "updated_at": "2021-09-03T12:03:15.642239-04:00", "order": 0}, "emitted_at": 1682600557295} -{"stream": "phone_number_assignments", "data": {"id": 34936, "number": "+13807775869", "user": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}}, "emitted_at": 1682600557789} -{"stream": "phone_number_assignments", "data": {"id": 34935, "number": "+13803335311", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600557790} -{"stream": "steps", "data": {"id": 198394, "created_at": "2021-09-16T10:12:17.764130-04:00", "updated_at": "2023-03-07T12:43:13.069452-05:00", "disabled": false, "type": "email", "name": "Sandbox Account", "display_name": "Day 1: Step 1 - Email", "day": 1, "step_number": 1, "multitouch_enabled": false, "details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}}, "emitted_at": 1682600558324} -{"stream": "steps", "data": {"id": 198448, "created_at": "2021-09-16T10:19:08.732399-04:00", "updated_at": "2021-09-16T10:19:08.732399-04:00", "disabled": false, "type": "email", "name": "About company", "display_name": "Day 3: Step 2 - Email", "day": 3, "step_number": 2, "multitouch_enabled": false, "details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198448", "id": 198448}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}}, "emitted_at": 1682600558325} -{"stream": "team_templates", "data": {"id": "75621893-cf7f-41d3-bb45-6d31afc8914c", "title": "Team Template 10", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:43.083542-05:00", "updated_at": "2023-03-07T12:19:50.995059-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:50.579224-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=75621893-cf7f-41d3-bb45-6d31afc8914c"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558969} -{"stream": "team_templates", "data": {"id": "858de732-204b-4a58-a870-f3b551d2d6f3", "title": "Team Template 9", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:31.825973-05:00", "updated_at": "2023-03-07T12:19:38.280673-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:37.878579-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=858de732-204b-4a58-a870-f3b551d2d6f3"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558969} -{"stream": "team_templates", "data": {"id": "d90faaf7-903c-4f1d-b3c1-973c7fa9531d", "title": "Team Template 8", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:21.558099-05:00", "updated_at": "2023-03-07T12:19:27.352314-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:26.901056-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=d90faaf7-903c-4f1d-b3c1-973c7fa9531d"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558969} -{"stream": "team_templates", "data": {"id": "cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f", "title": "Team Template 7", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:10.501230-05:00", "updated_at": "2023-03-07T12:19:16.796639-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:16.334281-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558970} -{"stream": "team_templates", "data": {"id": "353f478c-c5dd-4c36-8417-3ad5bb88c927", "title": "Team Template 6", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:56.710041-05:00", "updated_at": "2023-03-07T12:19:05.988093-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:05.462518-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=353f478c-c5dd-4c36-8417-3ad5bb88c927"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558970} -{"stream": "team_templates", "data": {"id": "2e31b292-dcd1-48fd-b4df-4b788090d487", "title": "Team Template 5", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:45.801760-05:00", "updated_at": "2023-03-07T12:18:52.064748-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:18:51.662511-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=2e31b292-dcd1-48fd-b4df-4b788090d487"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558970} -{"stream": "team_templates", "data": {"id": "c9cb00f1-0c63-4440-80fd-a58b68affe5d", "title": "Team Template 4", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:35.339790-05:00", "updated_at": "2023-03-07T12:18:41.088563-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:18:40.613109-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=c9cb00f1-0c63-4440-80fd-a58b68affe5d"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558970} -{"stream": "team_templates", "data": {"id": "c2d18470-2c55-4dd0-a421-c69f993c0487", "title": "Team Template 3", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:25.166835-05:00", "updated_at": "2023-03-07T12:18:30.886758-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:18:30.432862-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=c2d18470-2c55-4dd0-a421-c69f993c0487"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558971} -{"stream": "team_templates", "data": {"id": "0d7e1891-893d-47ed-aa4c-9b30dd484a35", "title": "Team Template 2", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:14.371691-05:00", "updated_at": "2023-03-07T12:18:20.762914-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:18:20.340839-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=0d7e1891-893d-47ed-aa4c-9b30dd484a35"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558971} -{"stream": "team_templates", "data": {"id": "2ae537a5-a2fe-48c5-a2f2-df3d110c3af3", "title": "Team Template 1", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:18:08.123679-05:00", "updated_at": "2023-03-07T12:18:08.690684-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:18:08.074973-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=2ae537a5-a2fe-48c5-a2f2-df3d110c3af3"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600558971} -{"stream": "team_template_attachments", "data": {"id": 4825, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2ae537a5-a2fe-48c5-a2f2-df3d110c3af3", "id": "2ae537a5-a2fe-48c5-a2f2-df3d110c3af3"}}, "emitted_at": 1682600560611} -{"stream": "team_template_attachments", "data": {"id": 4826, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/0d7e1891-893d-47ed-aa4c-9b30dd484a35", "id": "0d7e1891-893d-47ed-aa4c-9b30dd484a35"}}, "emitted_at": 1682600560612} -{"stream": "team_template_attachments", "data": {"id": 4827, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c2d18470-2c55-4dd0-a421-c69f993c0487", "id": "c2d18470-2c55-4dd0-a421-c69f993c0487"}}, "emitted_at": 1682600560612} -{"stream": "team_template_attachments", "data": {"id": 4828, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c9cb00f1-0c63-4440-80fd-a58b68affe5d", "id": "c9cb00f1-0c63-4440-80fd-a58b68affe5d"}}, "emitted_at": 1682600560613} -{"stream": "team_template_attachments", "data": {"id": 4829, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2e31b292-dcd1-48fd-b4df-4b788090d487", "id": "2e31b292-dcd1-48fd-b4df-4b788090d487"}}, "emitted_at": 1682600560613} -{"stream": "team_template_attachments", "data": {"id": 4830, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/353f478c-c5dd-4c36-8417-3ad5bb88c927", "id": "353f478c-c5dd-4c36-8417-3ad5bb88c927"}}, "emitted_at": 1682600560613} -{"stream": "team_template_attachments", "data": {"id": 4831, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f", "id": "cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f"}}, "emitted_at": 1682600560613} -{"stream": "team_template_attachments", "data": {"id": 4832, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/d90faaf7-903c-4f1d-b3c1-973c7fa9531d", "id": "d90faaf7-903c-4f1d-b3c1-973c7fa9531d"}}, "emitted_at": 1682600560613} -{"stream": "team_template_attachments", "data": {"id": 4833, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/858de732-204b-4a58-a870-f3b551d2d6f3", "id": "858de732-204b-4a58-a870-f3b551d2d6f3"}}, "emitted_at": 1682600560614} -{"stream": "team_template_attachments", "data": {"id": 4834, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/75621893-cf7f-41d3-bb45-6d31afc8914c", "id": "75621893-cf7f-41d3-bb45-6d31afc8914c"}}, "emitted_at": 1682600560614} -{"stream": "crm_activities", "data": {"id": 11367506, "created_at": "2021-09-16T11:02:51.541557-04:00", "updated_at": "2021-09-16T11:03:06.798971-04:00", "subject": "Email: ", "description": "To: \"Kelly Irish\" \n\nSubject: \nBody:\n\n\u00a0 \nhi", "crm_id": null, "activity_type": "email", "error": "Not connected to your CRM", "custom_crm_fields": {"email_message_id": "", "direction": "outbound", "bounced": true}, "person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561128} -{"stream": "crm_activities", "data": {"id": 11346104, "created_at": "2021-09-16T10:04:58.277547-04:00", "updated_at": "2021-09-16T10:04:58.326349-04:00", "subject": "Call: No Answer | Customer", "description": "missed call", "crm_id": null, "activity_type": "phone", "error": "Not connected to your CRM", "custom_crm_fields": {"call_sentiment": "Customer", "call_disposition": "No Answer", "direction": "unknown", "call_to": "+1234445556", "metadata": {}}, "person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561129} -{"stream": "crm_activities", "data": {"id": 11334017, "created_at": "2021-09-16T09:31:13.059842-04:00", "updated_at": "2021-09-16T09:31:23.374677-04:00", "subject": "Email: test 2", "description": "To: \"Karina Kuznietsova\" \n\nSubject: test 2\nBody:\n\n\u00a0 \ntest 2 email", "crm_id": null, "activity_type": "email", "error": "Not connected to your CRM", "custom_crm_fields": {"email_message_id": "", "direction": "outbound", "bounced": true}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561129} -{"stream": "crm_activities", "data": {"id": 11329207, "created_at": "2021-09-16T09:18:09.092324-04:00", "updated_at": "2021-09-16T09:18:09.238934-04:00", "subject": "Meeting Booked: Meeting with Karina Kuznietsova", "description": null, "crm_id": null, "activity_type": "meeting_event", "error": "Not connected to your CRM", "custom_crm_fields": {"meeting_assigned_to": "Karina Kuznietsova", "meeting_name": "Meeting with Karina Kuznietsova", "meeting_duration": 30, "meeting_date": "2021-09-16", "meeting_start_time": "2021-09-16T13:30:00+00:00", "meeting_location": "", "meeting_description": "", "meeting_booked_by": "Karina Kuznietsova", "meeting_type": "Default Personal Meeting", "meeting_guests": [], "meeting_attendees": [], "meeting_all_day": false, "meeting_source": "Booked in SalesLoft", "meeting_no_show": false}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561130} -{"stream": "crm_activities", "data": {"id": 11328442, "created_at": "2021-09-16T09:16:01.712378-04:00", "updated_at": "2021-09-16T09:16:01.769654-04:00", "subject": "Meeting Booked: Meeting with Karina Kuznietsova", "description": null, "crm_id": null, "activity_type": "meeting_event", "error": "Not connected to your CRM", "custom_crm_fields": {"meeting_assigned_to": "Karina Kuznietsova", "meeting_name": "Meeting with Karina Kuznietsova", "meeting_duration": 30, "meeting_date": "2021-09-17", "meeting_start_time": "2021-09-17T07:15:00+00:00", "meeting_location": "", "meeting_description": "", "meeting_booked_by": "Karina Kuznietsova", "meeting_type": "Default Personal Meeting", "meeting_guests": [], "meeting_attendees": [], "meeting_all_day": false, "meeting_source": "Booked in SalesLoft", "meeting_no_show": false}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561130} -{"stream": "crm_activities", "data": {"id": 11324527, "created_at": "2021-09-16T09:05:40.168912-04:00", "updated_at": "2021-09-16T09:05:48.403036-04:00", "subject": "Email: test ", "description": "To: \"Karina Kuznietsova\" \n\nSubject: test \nBody:\n\ntest email \n\u00a0", "crm_id": null, "activity_type": "email", "error": "Not connected to your CRM", "custom_crm_fields": {"email_message_id": "", "direction": "outbound", "bounced": true}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561130} -{"stream": "crm_activities", "data": {"id": 11305693, "created_at": "2021-09-16T07:18:35.435230-04:00", "updated_at": "2021-09-16T07:18:35.484403-04:00", "subject": "Note", "description": "Test Note", "crm_id": null, "activity_type": null, "error": "Not connected to your CRM", "custom_crm_fields": {}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561131} -{"stream": "crm_activities", "data": {"id": 11305642, "created_at": "2021-09-16T07:18:08.100954-04:00", "updated_at": "2021-09-16T07:18:16.273222-04:00", "subject": "Email: test ", "description": "To: \"Karina Kuznietsova\" \n\nSubject: test \nBody:\n\ntest email \n\u00a0", "crm_id": null, "activity_type": "email", "error": "Not connected to your CRM", "custom_crm_fields": {"email_message_id": "", "direction": "outbound", "bounced": true}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561131} -{"stream": "crm_activities", "data": {"id": 11305624, "created_at": "2021-09-16T07:17:57.487992-04:00", "updated_at": "2021-09-16T07:17:57.538224-04:00", "subject": "Call", "description": "", "crm_id": null, "activity_type": "phone", "error": "Not connected to your CRM", "custom_crm_fields": {"direction": "unknown", "call_to": "+12345678900", "metadata": {}}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561131} -{"stream": "crm_users", "data": {"id": 41009, "crm_id": "0055e000005ubSNAAY", "created_at": "2023-03-07T12:32:43.804976-05:00", "updated_at": "2023-03-07T12:32:43.804976-05:00", "first_name": "User11", "last_name": "Sample", "name": "User11 Sample", "user": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}}, "emitted_at": 1682600561623} -{"stream": "crm_users", "data": {"id": 41008, "crm_id": "00505000001npKKAAY", "created_at": "2023-03-07T12:32:22.065502-05:00", "updated_at": "2023-03-07T12:32:22.065502-05:00", "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1682600561624} -{"stream": "groups", "data": {"id": 11820, "name": "Group3", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11820", "id": 11820}, {"_href": "https://api.salesloft.com/v2/groups/11827", "id": 11827}]}, "emitted_at": 1682600562427} -{"stream": "groups", "data": {"id": 11827, "name": "Group10", "parent_id": 11820, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11827", "id": 11827}]}, "emitted_at": 1682600562428} -{"stream": "groups", "data": {"id": 11818, "name": "Group1", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11818", "id": 11818}, {"_href": "https://api.salesloft.com/v2/groups/11821", "id": 11821}, {"_href": "https://api.salesloft.com/v2/groups/11823", "id": 11823}, {"_href": "https://api.salesloft.com/v2/groups/11826", "id": 11826}]}, "emitted_at": 1682600562429} -{"stream": "groups", "data": {"id": 11823, "name": "Group6", "parent_id": 11818, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11823", "id": 11823}, {"_href": "https://api.salesloft.com/v2/groups/11826", "id": 11826}]}, "emitted_at": 1682600562429} -{"stream": "groups", "data": {"id": 11826, "name": "Group9", "parent_id": 11823, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11826", "id": 11826}]}, "emitted_at": 1682600562429} -{"stream": "groups", "data": {"id": 11822, "name": "Group5", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11822", "id": 11822}, {"_href": "https://api.salesloft.com/v2/groups/11825", "id": 11825}]}, "emitted_at": 1682600562429} -{"stream": "groups", "data": {"id": 11825, "name": "Group8", "parent_id": 11822, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11825", "id": 11825}]}, "emitted_at": 1682600562430} -{"stream": "groups", "data": {"id": 11824, "name": "Group7", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11824", "id": 11824}]}, "emitted_at": 1682600562430} -{"stream": "groups", "data": {"id": 11821, "name": "Group4", "parent_id": 11818, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11821", "id": 11821}]}, "emitted_at": 1682600562430} -{"stream": "groups", "data": {"id": 11819, "name": "Group2", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11819", "id": 11819}]}, "emitted_at": 1682600562430} -{"stream": "successes", "data": {"id": 34605, "created_at": "2021-09-16T09:18:30.667051-04:00", "updated_at": "2021-09-16T09:18:30.667051-04:00", "succeeded_at": "2021-09-16T09:18:30.667051-04:00", "success_window_started_at": "2021-09-16T07:20:01.801041-04:00", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "latest_email": null, "latest_call": {"_href": "https://api.salesloft.com/v2/activities/calls/2113993", "id": 2113993}, "latest_action": null, "latest_cadence": null, "latest_step": null, "counts": {"total_emails": 0, "total_calls": 0, "total_other_touches": 0}}, "emitted_at": 1682600562923} -{"stream": "successes", "data": {"id": 34569, "created_at": "2021-09-16T07:20:01.801041-04:00", "updated_at": "2021-09-16T07:20:01.801041-04:00", "succeeded_at": "2021-09-16T07:20:01.801041-04:00", "success_window_started_at": "2021-09-16T07:02:46.804979-04:00", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "latest_email": null, "latest_call": {"_href": "https://api.salesloft.com/v2/activities/calls/2113993", "id": 2113993}, "latest_action": null, "latest_cadence": null, "latest_step": null, "counts": {"total_emails": 0, "total_calls": 1, "total_other_touches": 0}}, "emitted_at": 1682600562924} -{"stream": "email_template_attachments", "data": {"id": 255134, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604711", "id": 6604711}}, "emitted_at": 1682600567092} -{"stream": "email_template_attachments", "data": {"id": 255135, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604713", "id": 6604713}}, "emitted_at": 1682600567093} -{"stream": "email_template_attachments", "data": {"id": 255136, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604714", "id": 6604714}}, "emitted_at": 1682600567093} -{"stream": "email_template_attachments", "data": {"id": 255137, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604719", "id": 6604719}}, "emitted_at": 1682600567093} -{"stream": "email_template_attachments", "data": {"id": 255138, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604721", "id": 6604721}}, "emitted_at": 1682600567093} -{"stream": "email_template_attachments", "data": {"id": 255139, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604722", "id": 6604722}}, "emitted_at": 1682600567093} -{"stream": "email_template_attachments", "data": {"id": 255140, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604723", "id": 6604723}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255141, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604779", "id": 6604779}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255142, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604894", "id": 6604894}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255143, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604896", "id": 6604896}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255144, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602856", "id": 6602856}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255145, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602891", "id": 6602891}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255146, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602857", "id": 6602857}}, "emitted_at": 1682600567094} -{"stream": "email_template_attachments", "data": {"id": 255147, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602865", "id": 6602865}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255148, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602880", "id": 6602880}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255149, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602885", "id": 6602885}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255150, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602889", "id": 6602889}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255151, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602890", "id": 6602890}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255152, "attachment_id": 597204, "name": "Airbyte logo 120x30.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/e27974daab648ebd7f83f7c51e3ce7300dd52968/Airbyte_logo_120x30.png?1678209646", "attachment_file_size": 3361, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "e97f514cc3453d54b8758058723732c7fda2dee3", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6602883", "id": 6602883}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255158, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605413", "id": 6605413}}, "emitted_at": 1682600567095} -{"stream": "email_template_attachments", "data": {"id": 255159, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605414", "id": 6605414}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255160, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605415", "id": 6605415}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255161, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605416", "id": 6605416}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255162, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605419", "id": 6605419}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255163, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605420", "id": 6605420}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255164, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605418", "id": 6605418}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255165, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605421", "id": 6605421}}, "emitted_at": 1682600567096} -{"stream": "email_template_attachments", "data": {"id": 255166, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605422", "id": 6605422}}, "emitted_at": 1682600567097} -{"stream": "email_template_attachments", "data": {"id": 255167, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6605423", "id": 6605423}}, "emitted_at": 1682600567097} +{"stream": "emails", "data": {"id": 8202787, "created_at": "2021-09-16T10:02:20.479210-04:00", "updated_at": "2021-09-16T11:03:06.789805-04:00", "recipient_email_address": "kellyirish@google.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T11:02:21.934859-04:00", "sent_at": "2021-09-16T11:02:51.405506-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "mailing": {"id": 3124568}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11367506", "id": 11367506}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1688663044626} +{"stream": "emails", "data": {"id": 8194214, "created_at": "2021-09-16T09:30:13.172463-04:00", "updated_at": "2021-09-16T09:31:23.363226-04:00", "recipient_email_address": "karina.kuznietsova@zazmc.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T09:30:27.343687-04:00", "sent_at": "2021-09-16T09:31:12.895350-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "mailing": {"id": 3122753}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11334017", "id": 11334017}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1688663044627} +{"stream": "emails", "data": {"id": 8190809, "created_at": "2021-09-16T09:04:44.160528-04:00", "updated_at": "2021-09-16T09:05:48.392667-04:00", "recipient_email_address": "karina.kuznietsova@zazmc.com", "status": "sent", "bounced": true, "send_after": "2021-09-16T09:05:06.374217-04:00", "sent_at": "2021-09-16T09:05:39.849349-04:00", "view_tracking": true, "click_tracking": false, "headers": {}, "personalization": "100.0", "counts": {"clicks": 0, "views": 0, "replies": 0, "unique_devices": 0, "unique_locations": 0, "attachments": 0}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "recipient": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "mailing": {"id": 3121331}, "action": null, "task": null, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11324527", "id": 11324527}, "cadence": null, "step": null, "email_template": null}, "emitted_at": 1688663044627} +{"stream": "searches", "data": {"id": 4002, "user_slug": "karinakuznietsova", "primary_calendar_id": "karina.kuznietsova@zazmic.com", "primary_calendar_name": "Airbyte Team", "email_address": "karina.kuznietsova@zazmic.com", "user_details": {}, "calendar_type": "gmail", "title": null, "description": null, "location": null, "default_meeting_length": 30, "availability_limit_enabled": false, "availability_limit": 0, "schedule_delay": 0, "buffer_time_duration": 0, "schedule_buffer_enabled": false, "times_available": {"Sunday": {"start_time": "09:00", "end_time": "17:00", "enabled": false}, "Monday": {"start_time": "09:00", "end_time": "17:00", "enabled": true}, "Tuesday": {"start_time": "09:00", "end_time": "17:00", "enabled": true}, "Wednesday": {"start_time": "09:00", "end_time": "17:00", "enabled": true}, "Thursday": {"start_time": "09:00", "end_time": "17:00", "enabled": true}, "Friday": {"start_time": "09:00", "end_time": "17:00", "enabled": true}, "Saturday": {"start_time": "09:00", "end_time": "17:00", "enabled": false}}, "allow_booking_on_behalf": true, "allow_booking_overtime": true, "allow_event_overlap": false, "share_event_detail": false, "enable_dynamic_location": false, "created_at": "2021-09-16T11:20:22.900750Z", "updated_at": "2023-03-07T16:01:09.172091Z", "time_zone": "US/Eastern", "primary_calendar_connection_failed": true, "enable_calendar_sync": false, "reschedule_meetings_enabled": true, "user": {"_href": "https://api.salesloft.com/v2/users/54713f8f-6283-453f-84da-e27e28223e8c", "id": "54713f8f-6283-453f-84da-e27e28223e8c"}, "active_meeting_url": {"url": "https://meetings.salesloft.com/airbytedeveloperlicense/karinakuznietsova", "created_at": "2021-09-16T11:20:23.071018Z", "updated_at": "2021-09-16T11:20:27.648100Z"}}, "emitted_at": 1688663058815} +{"stream": "groups", "data": {"id": 11820, "name": "Group3", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11820", "id": 11820}, {"_href": "https://api.salesloft.com/v2/groups/11827", "id": 11827}]}, "emitted_at": 1688663052134} +{"stream": "groups", "data": {"id": 11827, "name": "Group10", "parent_id": 11820, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11827", "id": 11827}]}, "emitted_at": 1688663052135} +{"stream": "groups", "data": {"id": 11818, "name": "Group1", "parent_id": null, "accessible_groups": [{"_href": "https://api.salesloft.com/v2/groups/11818", "id": 11818}, {"_href": "https://api.salesloft.com/v2/groups/11821", "id": 11821}, {"_href": "https://api.salesloft.com/v2/groups/11823", "id": 11823}, {"_href": "https://api.salesloft.com/v2/groups/11826", "id": 11826}]}, "emitted_at": 1688663052136} +{"stream": "successes", "data": {"id": 34605, "created_at": "2021-09-16T09:18:30.667051-04:00", "updated_at": "2021-09-16T09:18:30.667051-04:00", "succeeded_at": "2021-09-16T09:18:30.667051-04:00", "success_window_started_at": "2021-09-16T07:20:01.801041-04:00", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "latest_email": null, "latest_call": {"_href": "https://api.salesloft.com/v2/activities/calls/2113993", "id": 2113993}, "latest_action": null, "latest_cadence": null, "latest_step": null, "counts": {"total_emails": 0, "total_calls": 0, "total_other_touches": 0}}, "emitted_at": 1688663052494} +{"stream": "successes", "data": {"id": 34569, "created_at": "2021-09-16T07:20:01.801041-04:00", "updated_at": "2021-09-16T07:20:01.801041-04:00", "succeeded_at": "2021-09-16T07:20:01.801041-04:00", "success_window_started_at": "2021-09-16T07:02:46.804979-04:00", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "latest_email": null, "latest_call": {"_href": "https://api.salesloft.com/v2/activities/calls/2113993", "id": 2113993}, "latest_action": null, "latest_cadence": null, "latest_step": null, "counts": {"total_emails": 0, "total_calls": 1, "total_other_touches": 0}}, "emitted_at": 1688663052495} +{"stream": "notes", "data": {"id": 1029422, "content": "missed call", "created_at": "2021-09-16T10:04:58.109554-04:00", "updated_at": "2021-09-16T10:04:58.109554-04:00", "associated_type": "person", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "associated_with": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "call": {"_href": "https://api.salesloft.com/v2/activities/calls/2117241", "id": 2117241}}, "emitted_at": 1688663047921} +{"stream": "notes", "data": {"id": 1028798, "content": "this is a new note", "created_at": "2021-09-16T09:37:38.057127-04:00", "updated_at": "2021-09-16T09:38:08.829201-04:00", "associated_type": "account", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "associated_with": {"_href": "https://api.salesloft.com/v2/accounts/2364729", "id": 2364729}, "call": null}, "emitted_at": 1688663047922} +{"stream": "notes", "data": {"id": 1027747, "content": "Test Note", "created_at": "2021-09-16T07:18:35.368918-04:00", "updated_at": "2021-09-16T07:18:35.368918-04:00", "associated_type": "person", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "associated_with": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "call": null}, "emitted_at": 1688663047922} +{"stream": "steps", "data": {"id": 198394, "created_at": "2021-09-16T10:12:17.764130-04:00", "updated_at": "2023-03-07T12:43:13.069452-05:00", "disabled": false, "type": "email", "name": "Sandbox Account", "display_name": "Day 1: Step 1 - Email", "day": 1, "step_number": 1, "multitouch_enabled": false, "details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}}, "emitted_at": 1688663049025} +{"stream": "steps", "data": {"id": 198448, "created_at": "2021-09-16T10:19:08.732399-04:00", "updated_at": "2021-09-16T10:19:08.732399-04:00", "disabled": false, "type": "email", "name": "About company", "display_name": "Day 3: Step 2 - Email", "day": 3, "step_number": 2, "multitouch_enabled": false, "details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198448", "id": 198448}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}}, "emitted_at": 1688663049026} +{"stream": "account_stages", "data": {"id": 5006, "name": "Disqualified", "created_at": "2021-09-03T12:03:15.863631-04:00", "updated_at": "2021-09-03T12:03:15.863631-04:00", "order": 3}, "emitted_at": 1688663045341} +{"stream": "account_stages", "data": {"id": 5005, "name": "Completed", "created_at": "2021-09-03T12:03:15.850021-04:00", "updated_at": "2021-09-03T12:03:15.850021-04:00", "order": 2}, "emitted_at": 1688663045341} +{"stream": "account_stages", "data": {"id": 5004, "name": "Working", "created_at": "2021-09-03T12:03:15.835536-04:00", "updated_at": "2021-09-03T12:03:15.835536-04:00", "order": 1}, "emitted_at": 1688663045341} +{"stream": "account_tiers", "data": {"id": 3469, "name": "Tier 3", "order": 2, "created_at": "2021-09-03T12:03:16.281412-04:00", "updated_at": "2021-09-03T12:03:16.281412-04:00"}, "emitted_at": 1688663045681} +{"stream": "account_tiers", "data": {"id": 3468, "name": "Tier 2", "order": 1, "created_at": "2021-09-03T12:03:16.270440-04:00", "updated_at": "2021-09-03T12:03:16.270440-04:00"}, "emitted_at": 1688663045682} +{"stream": "account_tiers", "data": {"id": 3467, "name": "Tier 1", "order": 0, "created_at": "2021-09-03T12:03:16.235256-04:00", "updated_at": "2021-09-03T12:03:16.235256-04:00"}, "emitted_at": 1688663045682} +{"stream": "people", "data": {"id": 103358219, "created_at": "2023-03-07T12:12:52.362090-05:00", "updated_at": "2023-03-07T12:49:27.205203-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User8", "last_name": "Sample", "display_name": "User8 Sample", "email_address": "user8.sample.airbyte@outlook.com", "full_email_address": "\"User8 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": "16205829403", "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 8", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1688663043752} +{"stream": "people", "data": {"id": 103358220, "created_at": "2023-03-07T12:12:52.405367-05:00", "updated_at": "2023-03-07T12:49:12.464381-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User9", "last_name": "Sample", "display_name": "User9 Sample", "email_address": "user9.sample.airbyte@outlook.com", "full_email_address": "\"User9 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": "19125901057", "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 9", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1688663043753} +{"stream": "people", "data": {"id": 103358221, "created_at": "2023-03-07T12:12:52.433805-05:00", "updated_at": "2023-03-07T12:48:51.576983-05:00", "last_contacted_at": null, "last_replied_at": null, "first_name": "User10", "last_name": "Sample", "display_name": "User10 Sample", "email_address": "user10.sample.airbyte@outlook.com", "full_email_address": "\"User10 Sample\" ", "secondary_email_address": null, "personal_email_address": null, "phone": null, "phone_extension": null, "home_phone": null, "mobile_phone": "14246220939", "linkedin_url": null, "title": "Manager", "city": "San Francisco, CA, US", "state": "CA", "country": null, "work_city": null, "work_state": null, "work_country": null, "crm_url": null, "crm_id": null, "crm_object_type": null, "owner_crm_id": null, "person_company_name": "Company 10", "person_company_website": "http://outlook.com", "person_company_industry": null, "do_not_contact": false, "bouncing": false, "locale": "US/Pacific", "locale_utc_offset": -420, "eu_resident": false, "personal_website": null, "twitter_handle": null, "last_contacted_type": null, "job_seniority": "manager", "custom_fields": {}, "tags": [], "contact_restrictions": [], "success_count": 0, "starred": false, "untouched": false, "counts": {"emails_sent": 0, "emails_viewed": 0, "emails_clicked": 0, "emails_replied_to": 0, "emails_bounced": 0, "calls": 0}, "account": {"_href": "https://api.salesloft.com/v2/accounts/35092358", "id": 35092358}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "import": {"_href": "https://api.salesloft.com/v2/imports/5174826", "id": 5174826}, "person_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "most_recent_cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "last_completed_step_cadence": null, "last_completed_step": null, "cadences": [{"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}]}, "emitted_at": 1688663043754} +{"stream": "team_template_attachments", "data": {"id": 4825, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/2ae537a5-a2fe-48c5-a2f2-df3d110c3af3", "id": "2ae537a5-a2fe-48c5-a2f2-df3d110c3af3"}}, "emitted_at": 1688663051076} +{"stream": "team_template_attachments", "data": {"id": 4826, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/0d7e1891-893d-47ed-aa4c-9b30dd484a35", "id": "0d7e1891-893d-47ed-aa4c-9b30dd484a35"}}, "emitted_at": 1688663051077} +{"stream": "team_template_attachments", "data": {"id": 4827, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/c2d18470-2c55-4dd0-a421-c69f993c0487", "id": "c2d18470-2c55-4dd0-a421-c69f993c0487"}}, "emitted_at": 1688663051077} +{"stream": "phone_number_assignments", "data": {"id": 34936, "number": "+13807775869", "user": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}}, "emitted_at": 1688663048585} +{"stream": "phone_number_assignments", "data": {"id": 34935, "number": "+13803335311", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663048586} +{"stream": "email_templates", "data": {"id": 6605423, "title": "Team Template 6", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.552863-05:00", "updated_at": "2023-03-07T12:28:31.552863-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605423"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/353f478c-c5dd-4c36-8417-3ad5bb88c927", "id": "353f478c-c5dd-4c36-8417-3ad5bb88c927"}, "groups": []}, "emitted_at": 1688663047202} +{"stream": "email_templates", "data": {"id": 6605422, "title": "Team Template 7", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.526620-05:00", "updated_at": "2023-03-07T12:28:31.526620-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605422"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f", "id": "cb3c0c27-95a5-4d06-bc2a-6a1d81859d6f"}, "groups": []}, "emitted_at": 1688663047202} +{"stream": "email_templates", "data": {"id": 6605421, "title": "Team Template 10", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:28:31.524798-05:00", "updated_at": "2023-03-07T12:28:31.524798-05:00", "last_used_at": null, "archived_at": null, "shared": false, "open_tracking_enabled": true, "click_tracking_enabled": false, "cadence_template": null, "_links": {"attachments": "https://api.salesloft.com/v2/email_template_attachments?email_template_id%5B%5D=6605421"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "template_owner": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}, "team_template": {"_href": "https://api.salesloft.com/v2/team_templates/75621893-cf7f-41d3-bb45-6d31afc8914c", "id": "75621893-cf7f-41d3-bb45-6d31afc8914c"}, "groups": []}, "emitted_at": 1688663047202} +{"stream": "import", "data": {"id": 5174826, "created_at": "2023-03-07T12:12:52.105709-05:00", "updated_at": "2023-03-07T12:12:52.105709-05:00", "name": "People_Salesloft.csv", "current_people_count": 10, "imported_people_count": 10}, "emitted_at": 1688663047572} +{"stream": "team_templates", "data": {"id": "75621893-cf7f-41d3-bb45-6d31afc8914c", "title": "Team Template 10", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:43.083542-05:00", "updated_at": "2023-03-07T12:19:50.995059-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:50.579224-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=75621893-cf7f-41d3-bb45-6d31afc8914c"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663049578} +{"stream": "team_templates", "data": {"id": "858de732-204b-4a58-a870-f3b551d2d6f3", "title": "Team Template 9", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:31.825973-05:00", "updated_at": "2023-03-07T12:19:38.280673-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:37.878579-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=858de732-204b-4a58-a870-f3b551d2d6f3"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663049578} +{"stream": "team_templates", "data": {"id": "d90faaf7-903c-4f1d-b3c1-973c7fa9531d", "title": "Team Template 8", "subject": "Test Team Template", "body": "
    \n
    Test Team Template Description
    \n
    ", "body_preview": "Test Team Template Description", "created_at": "2023-03-07T12:19:21.558099-05:00", "updated_at": "2023-03-07T12:19:27.352314-05:00", "last_used_at": null, "archived_at": null, "last_modified_at": "2023-03-07T12:19:26.901056-05:00", "open_tracking_enabled": true, "click_tracking_enabled": false, "_links": {"attachments": "https://api.salesloft.com/v2/team_template_attachments?team_template_id%5B%5D=d90faaf7-903c-4f1d-b3c1-973c7fa9531d"}, "tags": ["Tag1"], "counts": {"sent_emails": 0, "views": 0, "clicks": 0, "replies": 0, "bounces": 0}, "last_modified_user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663049579} +{"stream": "cadence_memberships", "data": {"id": 71245543, "added_at": "2023-03-07T12:12:53.179454-05:00", "created_at": "2023-03-07T12:12:53.186833-05:00", "updated_at": "2023-03-07T12:12:53.210155-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358221", "id": 103358221}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305169}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1688663043136} +{"stream": "cadence_memberships", "data": {"id": 71245540, "added_at": "2023-03-07T12:12:53.104252-05:00", "created_at": "2023-03-07T12:12:53.111202-05:00", "updated_at": "2023-03-07T12:12:53.140663-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358220", "id": 103358220}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305164}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1688663043136} +{"stream": "cadence_memberships", "data": {"id": 71245537, "added_at": "2023-03-07T12:12:53.034821-05:00", "created_at": "2023-03-07T12:12:53.043064-05:00", "updated_at": "2023-03-07T12:12:53.064853-05:00", "person_deleted": false, "currently_on_cadence": true, "current_state": "staged", "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "person": {"_href": "https://api.salesloft.com/v2/people/103358219", "id": 103358219}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "latest_action": {"id": 343305158}, "counts": {"views": 0, "clicks": 0, "replies": 0, "calls": 0, "sent_emails": 0, "bounces": 0}}, "emitted_at": 1688663043136} +{"stream": "person_stages", "data": {"id": 10154, "name": "Replied", "created_at": "2021-09-03T12:03:15.801147-04:00", "updated_at": "2021-09-03T12:03:15.801147-04:00", "order": 7}, "emitted_at": 1688663048246} +{"stream": "person_stages", "data": {"id": 10153, "name": "Completed", "created_at": "2021-09-03T12:03:15.784892-04:00", "updated_at": "2021-09-03T12:03:15.784892-04:00", "order": 6}, "emitted_at": 1688663048246} +{"stream": "person_stages", "data": {"id": 10152, "name": "Do Not Contact", "created_at": "2021-09-03T12:03:15.771896-04:00", "updated_at": "2021-09-03T12:03:15.771896-04:00", "order": 5}, "emitted_at": 1688663048247} +{"stream": "accounts", "data": {"id": 35092356, "created_at": "2023-03-07T12:12:53.353476-05:00", "updated_at": "2023-03-07T12:42:45.816560-05:00", "archived_at": null, "name": "Company 1", "domain": "zohomail.eu", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": [], "counts": {"people": 1}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": null, "account_tier": null}, "emitted_at": 1688663046098} +{"stream": "accounts", "data": {"id": 35092358, "created_at": "2023-03-07T12:12:53.436918-05:00", "updated_at": "2023-03-07T12:42:45.540331-05:00", "archived_at": null, "name": "Company 4", "domain": "outlook.com", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": [], "counts": {"people": 8}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": null, "account_tier": null}, "emitted_at": 1688663046099} +{"stream": "accounts", "data": {"id": 35092357, "created_at": "2023-03-07T12:12:53.393425-05:00", "updated_at": "2023-03-07T12:42:42.384044-05:00", "archived_at": null, "name": "Company 2", "domain": "gmail.com", "conversational_name": null, "description": null, "phone": null, "website": null, "linkedin_url": null, "twitter_handle": null, "street": null, "city": null, "state": null, "postal_code": null, "country": null, "locale": null, "industry": null, "company_type": null, "founded": null, "revenue_range": null, "size": null, "crm_id": null, "crm_url": null, "crm_object_type": "account", "owner_crm_id": null, "last_contacted_at": null, "last_contacted_type": null, "do_not_contact": false, "custom_fields": {}, "user_relationships": [], "tags": [], "counts": {"people": 1}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "last_contacted_by": null, "last_contacted_person": null, "company_stage": null, "account_tier": null}, "emitted_at": 1688663046099} +{"stream": "cadences", "data": {"id": 340821, "created_at": "2023-03-07T12:47:07.697191-05:00", "updated_at": "2023-03-07T12:47:07.697191-05:00", "archived_at": null, "latest_active_date": null, "team_cadence": false, "shared": true, "remove_bounces_enabled": false, "remove_replies_enabled": false, "opt_out_link_included": false, "draft": false, "override_contact_restrictions": null, "cadence_framework_id": null, "cadence_function": "outbound", "name": "Cadence 1", "external_identifier": null, "tags": [], "current_state": "active", "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "bounced_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10147", "id": 10147}, "replied_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "added_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10149", "id": 10149}, "finished_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10151", "id": 10151}, "cadence_priority": null, "groups": [], "counts": {"cadence_people": null, "people_acted_on_count": 0, "target_daily_people": 0, "opportunities_created": 0, "meetings_booked": 0}}, "emitted_at": 1689947169241} +{"stream": "cadences", "data": {"id": 25591, "created_at": "2021-09-16T08:20:08.485246-04:00", "updated_at": "2023-03-07T12:43:17.450246-05:00", "archived_at": null, "latest_active_date": null, "team_cadence": false, "shared": true, "remove_bounces_enabled": false, "remove_replies_enabled": false, "opt_out_link_included": true, "draft": false, "override_contact_restrictions": null, "cadence_framework_id": null, "cadence_function": "event", "name": "New Cadence", "external_identifier": null, "tags": ["opt-out"], "current_state": "active", "creator": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "owner": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "bounced_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "replied_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "added_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10148", "id": 10148}, "finished_stage": {"_href": "https://api.salesloft.com/v2/person_stages/10149", "id": 10149}, "cadence_priority": {"_href": "https://api.salesloft.com/v2/cadence_priorities", "id": 3385}, "groups": [], "counts": {"cadence_people": 11, "people_acted_on_count": 0, "target_daily_people": 0, "opportunities_created": 0, "meetings_booked": 0}}, "emitted_at": 1689947169242} +{"stream": "call_dispositions", "data": {"id": 8740, "created_at": "2021-09-03T12:03:16.028383-04:00", "updated_at": "2021-09-03T12:03:16.028383-04:00", "name": "Busy"}, "emitted_at": 1688663057513} +{"stream": "call_dispositions", "data": {"id": 8741, "created_at": "2021-09-03T12:03:16.052231-04:00", "updated_at": "2021-09-03T12:03:16.052231-04:00", "name": "Connected"}, "emitted_at": 1688663057515} +{"stream": "call_dispositions", "data": {"id": 8745, "created_at": "2021-09-03T12:03:16.107806-04:00", "updated_at": "2021-09-03T12:03:16.107806-04:00", "name": "Left Voicemail"}, "emitted_at": 1688663057515} +{"stream": "crm_users", "data": {"id": 41009, "crm_id": "0055e000005ubSNAAY", "created_at": "2023-03-07T12:32:43.804976-05:00", "updated_at": "2023-03-07T12:32:43.804976-05:00", "first_name": "User11", "last_name": "Sample", "name": "User11 Sample", "user": {"_href": "https://api.salesloft.com/v2/users/52180", "id": 52180}}, "emitted_at": 1688663051796} +{"stream": "crm_users", "data": {"id": 41008, "crm_id": "00505000001npKKAAY", "created_at": "2023-03-07T12:32:22.065502-05:00", "updated_at": "2023-03-07T12:32:22.065502-05:00", "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663051797} +{"stream": "email_template_attachments", "data": {"id": 255134, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604711", "id": 6604711}}, "emitted_at": 1688663056384} +{"stream": "email_template_attachments", "data": {"id": 255135, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604713", "id": 6604713}}, "emitted_at": 1688663056385} +{"stream": "email_template_attachments", "data": {"id": 255136, "attachment_id": 597193, "name": "Airbyte logo 75x75.png", "download_url": "https://sdr-scanii-prod-us3.storage.googleapis.com/attachments/da400c613af3917b3d510b3f9152e0a338d51554/Airbyte_logo_75x75.png?1678209474", "attachment_file_size": 6132, "scanned": true, "attachment_content_type": "image/png", "attachment_fingerprint": "0050641c170ca71780863dc0cdaa63cae7b4101a", "email_template": {"_href": "https://api.salesloft.com/v2/email_templates/6604714", "id": 6604714}}, "emitted_at": 1688663056386} +{"stream": "crm_activities", "data": {"id": 11367506, "created_at": "2021-09-16T11:02:51.541557-04:00", "updated_at": "2021-09-16T11:03:06.798971-04:00", "subject": "Email: ", "description": "To: \"Kelly Irish\" \n\nSubject: \nBody:\n\n\u00a0 \nhi", "crm_id": null, "activity_type": "email", "error": "Not connected to your CRM", "custom_crm_fields": {"email_message_id": "", "direction": "outbound", "bounced": true}, "person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663051444} +{"stream": "crm_activities", "data": {"id": 11346104, "created_at": "2021-09-16T10:04:58.277547-04:00", "updated_at": "2021-09-16T10:04:58.326349-04:00", "subject": "Call: No Answer | Customer", "description": "missed call", "crm_id": null, "activity_type": "phone", "error": "Not connected to your CRM", "custom_crm_fields": {"call_sentiment": "Customer", "call_disposition": "No Answer", "direction": "unknown", "call_to": "+1234445556", "metadata": {}}, "person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663051445} +{"stream": "crm_activities", "data": {"id": 11334017, "created_at": "2021-09-16T09:31:13.059842-04:00", "updated_at": "2021-09-16T09:31:23.374677-04:00", "subject": "Email: test 2", "description": "To: \"Karina Kuznietsova\" \n\nSubject: test 2\nBody:\n\n\u00a0 \ntest 2 email", "crm_id": null, "activity_type": "email", "error": "Not connected to your CRM", "custom_crm_fields": {"email_message_id": "", "direction": "outbound", "bounced": true}, "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}}, "emitted_at": 1688663051446} +{"stream": "call_sentiments", "data": {"id": 15076, "created_at": "2021-09-03T12:03:15.893251-04:00", "updated_at": "2021-09-03T12:03:15.893251-04:00", "name": "Company - Bad Fit"}, "emitted_at": 1688663057904} +{"stream": "call_sentiments", "data": {"id": 15077, "created_at": "2021-09-03T12:03:15.903392-04:00", "updated_at": "2021-09-03T12:03:15.903392-04:00", "name": "Contact - Bad Fit"}, "emitted_at": 1688663057905} +{"stream": "call_sentiments", "data": {"id": 15079, "created_at": "2021-09-03T12:03:15.927346-04:00", "updated_at": "2021-09-03T12:03:15.927346-04:00", "name": "Customer"}, "emitted_at": 1688663057905} +{"stream": "meetings", "data": {"id": 66445, "title": "Meeting with Karina Kuznietsova", "start_time": "2021-09-17T07:15:00.000000Z", "end_time": "2021-09-17T07:45:00.000000Z", "calendar_id": "karina.kuznietsova@zazmic.com", "calendar_type": "gmail", "meeting_type": null, "recipient_name": "Karina Kuznietsova", "recipient_email": "karina.kuznietsova@zazmc.com", "location": "", "description": "", "event_id": "rf641atglm8tt4hkbu91kv1p6c", "account_id": null, "task_id": 123599, "created_at": "2021-09-16T13:15:56.122269Z", "updated_at": "2022-07-27T09:09:28.112086Z", "guests": [], "crm_references": null, "event_source": "internal", "canceled_at": null, "all_day": false, "no_show": false, "crm_custom_fields": null, "strict_attribution": false, "i_cal_uid": "rf641atglm8tt4hkbu91kv1p6c@google.com", "status": "booked", "reschedule_status": null, "attendees": [], "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "cadence": null, "step": null, "owned_by_user": {"_href": "https://api.salesloft.com/v2/users/54713f8f-6283-453f-84da-e27e28223e8c", "id": "54713f8f-6283-453f-84da-e27e28223e8c"}, "booked_by_user": {"_href": "https://api.salesloft.com/v2/users/54713f8f-6283-453f-84da-e27e28223e8c", "id": "54713f8f-6283-453f-84da-e27e28223e8c"}}, "emitted_at": 1689947416440} +{"stream": "meetings", "data": {"id": 66450, "title": "Meeting with Karina Kuznietsova", "start_time": "2021-09-16T13:30:00.000000Z", "end_time": "2021-09-16T14:00:00.000000Z", "calendar_id": "karina.kuznietsova@zazmic.com", "calendar_type": "gmail", "meeting_type": null, "recipient_name": "Karina Kuznietsova", "recipient_email": "karina.kuznietsova@zazmc.com", "location": "", "description": "", "event_id": "9qqo8mlb1llolnp2g1dgrh339c", "account_id": null, "task_id": 123608, "created_at": "2021-09-16T13:18:00.421729Z", "updated_at": "2022-07-27T09:09:28.156604Z", "guests": [], "crm_references": null, "event_source": "internal", "canceled_at": null, "all_day": false, "no_show": false, "crm_custom_fields": null, "strict_attribution": false, "i_cal_uid": "9qqo8mlb1llolnp2g1dgrh339c@google.com", "status": "booked", "reschedule_status": null, "attendees": [], "person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "cadence": null, "step": null, "owned_by_user": {"_href": "https://api.salesloft.com/v2/users/54713f8f-6283-453f-84da-e27e28223e8c", "id": "54713f8f-6283-453f-84da-e27e28223e8c"}, "booked_by_user": {"_href": "https://api.salesloft.com/v2/users/54713f8f-6283-453f-84da-e27e28223e8c", "id": "54713f8f-6283-453f-84da-e27e28223e8c"}}, "emitted_at": 1689947416440} +{"stream": "actions", "data": {"id": 343305169, "due": true, "created_at": "2023-03-07T12:12:53.201684-05:00", "updated_at": "2023-06-17T09:53:56.994997-04:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358221", "id": 103358221}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1688663046502} +{"stream": "actions", "data": {"id": 343305164, "due": true, "created_at": "2023-03-07T12:12:53.128204-05:00", "updated_at": "2023-06-17T09:53:56.950542-04:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358220", "id": 103358220}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1688663046505} +{"stream": "actions", "data": {"id": 343305158, "due": true, "created_at": "2023-03-07T12:12:53.058448-05:00", "updated_at": "2023-06-17T09:53:56.891992-04:00", "type": "email", "status": "in_progress", "due_on": "2023-03-07T00:00:00.000000+00:00", "multitouch_group_id": null, "action_details": {"_href": "https://api.salesloft.com/v2/action_details/email_details/198394", "id": 198394}, "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "person": {"_href": "https://api.salesloft.com/v2/people/103358219", "id": 103358219}, "cadence": {"_href": "https://api.salesloft.com/v2/cadences/25591", "id": 25591}, "step": {"_href": "https://api.salesloft.com/v2/steps/198394", "id": 198394}}, "emitted_at": 1688663046505} +{"stream": "users", "data": {"id": 52180, "guid": "58d151f8-9b0b-4422-858a-40ea165edc46", "created_at": "2023-03-07T12:28:31.354490-05:00", "updated_at": "2023-03-07T13:10:23.281437-05:00", "name": "User11 Sample", "first_name": "User11", "last_name": "Sample", "job_role": "Operations", "active": true, "time_zone": "US/Eastern", "locale_utc_offset": -240, "slack_username": "iryna.grankova", "twitter_handle": null, "email": "iryna.grankova@airbyte.io", "email_client_email_address": "iryna.grankova@airbyte.io", "sending_email_address": "iryna.grankova@airbyte.io", "from_address": null, "full_email_address": "\"User11 Sample\" ", "bcc_email_address": null, "work_country": "UA", "seat_package": "prospect", "email_signature": "", "email_signature_type": "html", "email_signature_click_tracking_disabled": false, "team_admin": false, "local_dial_enabled": true, "click_to_call_enabled": false, "email_client_configured": false, "crm_connected": false, "external_feature_flags": {"ma_enabled": true, "ma_live_feed": true, "ma_recent_activities_update": true, "ma_person_page": true, "ma_push_notifications": true, "ma_meetings_push_notifications": true, "ma_websockets": true, "ma_edit_person": true, "ma_dev_qa_tools": false, "ma_cadences": false, "hot_leads": true, "people_crud_allow_create": true, "people_crud_allow_delete": true, "ma_mobile_workflow": true, "linkedin_oauth_flow": true, "live_feed_from_signal": false}, "_private_fields": {}, "phone_client": {"id": 53219}, "phone_number_assignment": {"_href": "https://api.salesloft.com/v2/phone_number_assignments/34936", "id": 34936}, "group": null, "team": {"_href": "https://api.salesloft.com/v2/team", "id": 104779}, "role": {"id": "User"}}, "emitted_at": 1689947863622} +{"stream": "users", "data": {"id": 6970, "guid": "54713f8f-6283-453f-84da-e27e28223e8c", "created_at": "2021-09-03T12:04:02.458089-04:00", "updated_at": "2023-03-09T14:50:58.222994-05:00", "name": "Airbyte Team", "first_name": "Airbyte", "last_name": "Team", "job_role": "Account Executive / Account Manager", "active": true, "time_zone": "US/Eastern", "locale_utc_offset": -240, "slack_username": "integration-test", "twitter_handle": null, "email": "integration-test@airbyte.io", "email_client_email_address": "karina.kuznietsova@zazmic.com", "sending_email_address": "karina.kuznietsova@zazmic.com", "from_address": null, "full_email_address": "\"Airbyte Team\" ", "bcc_email_address": null, "work_country": "US", "seat_package": "admin", "email_signature": "", "email_signature_type": "html", "email_signature_click_tracking_disabled": false, "team_admin": true, "local_dial_enabled": true, "click_to_call_enabled": false, "email_client_configured": true, "crm_connected": true, "external_feature_flags": {"ma_enabled": true, "ma_live_feed": true, "ma_recent_activities_update": true, "ma_person_page": true, "ma_push_notifications": true, "ma_meetings_push_notifications": true, "ma_websockets": true, "ma_edit_person": true, "ma_dev_qa_tools": false, "ma_cadences": false, "hot_leads": true, "people_crud_allow_create": true, "people_crud_allow_delete": true, "ma_mobile_workflow": true, "linkedin_oauth_flow": true, "live_feed_from_signal": false}, "_private_fields": {}, "phone_client": {"id": 7593}, "phone_number_assignment": {"_href": "https://api.salesloft.com/v2/phone_number_assignments/34935", "id": 34935}, "group": null, "team": {"_href": "https://api.salesloft.com/v2/team", "id": 104779}, "role": {"id": "Admin"}}, "emitted_at": 1689947863624} +{"stream": "calls", "data": {"id": 2117241, "to": "+1234445556", "duration": null, "sentiment": "Customer", "disposition": "No Answer", "created_at": "2021-09-16T10:04:58.125490-04:00", "updated_at": "2021-09-16T10:04:58.295640-04:00", "recordings": [], "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "action": null, "called_person": {"_href": "https://api.salesloft.com/v2/people/6509414", "id": 6509414}, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11346104", "id": 11346104}, "note": {"_href": "https://api.salesloft.com/v2/notes/1029422", "id": 1029422}, "cadence": null, "step": null}, "emitted_at": 1688663045010} +{"stream": "calls", "data": {"id": 2113993, "to": "+12345678900", "duration": null, "sentiment": null, "disposition": null, "created_at": "2021-09-16T07:17:57.430595-04:00", "updated_at": "2021-09-16T07:17:57.499713-04:00", "recordings": [], "user": {"_href": "https://api.salesloft.com/v2/users/6970", "id": 6970}, "action": null, "called_person": {"_href": "https://api.salesloft.com/v2/people/6502956", "id": 6502956}, "crm_activity": {"_href": "https://api.salesloft.com/v2/crm_activities/11305624", "id": 11305624}, "note": null, "cadence": null, "step": null}, "emitted_at": 1688663045010} +{"stream": "custom_fields", "data": {"id": 37515, "name": "airbyte", "field_type": "company", "value_type": "text", "created_at": "2023-07-11T14:07:59.517241-04:00", "updated_at": "2023-07-11T14:07:59.517241-04:00"}, "emitted_at": 1689099854082} diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json index 0998800674d1..40243c5e9248 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json @@ -4,19 +4,10 @@ "stream": { "name": "cadences", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -25,19 +16,10 @@ "stream": { "name": "cadence_memberships", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -46,19 +28,10 @@ "stream": { "name": "people", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -67,19 +40,10 @@ "stream": { "name": "calls", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -88,19 +52,10 @@ "stream": { "name": "actions", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -109,19 +64,10 @@ "stream": { "name": "notes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -130,19 +76,10 @@ "stream": { "name": "crm_activities", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -151,19 +88,10 @@ "stream": { "name": "successes", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -172,19 +100,10 @@ "stream": { "name": "emails", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -193,19 +112,10 @@ "stream": { "name": "accounts", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": [ - "updated_at" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -214,15 +124,8 @@ "stream": { "name": "account_stages", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -231,15 +134,8 @@ "stream": { "name": "email_templates", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" @@ -248,15 +144,32 @@ "stream": { "name": "team_templates", "json_schema": {}, - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_data_records", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "searches", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" diff --git a/airbyte-integrations/connectors/source-salesloft/metadata.yaml b/airbyte-integrations/connectors/source-salesloft/metadata.yaml index c062238989e8..8f713ca61aa4 100644 --- a/airbyte-integrations/connectors/source-salesloft/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesloft/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 41991d12-d4b5-439e-afd0-260a31d4c53f - dockerImageTag: 1.1.0 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-salesloft githubIssueLabel: source-salesloft icon: salesloft.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/salesloft tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-salesloft/requirements.txt b/airbyte-integrations/connectors/source-salesloft/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-salesloft/requirements.txt +++ b/airbyte-integrations/connectors/source-salesloft/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-salesloft/setup.py b/airbyte-integrations/connectors/source-salesloft/setup.py index f8c23e172553..bd9fc0a48179 100644 --- a/airbyte-integrations/connectors/source-salesloft/setup.py +++ b/airbyte-integrations/connectors/source-salesloft/setup.py @@ -11,7 +11,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/__init__.py b/airbyte-integrations/connectors/source-salesloft/source_salesloft/__init__.py index 37679567b7bc..db7ce3df5558 100644 --- a/airbyte-integrations/connectors/source-salesloft/source_salesloft/__init__.py +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json new file mode 100644 index 000000000000..568c45ac8c83 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json @@ -0,0 +1,90 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "to": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "integer"] + }, + "direction": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "call_type": { + "type": ["null", "string"] + }, + "call_uuid": { + "type": ["null", "string"] + }, + "recording": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "recording_status": { + "type": ["null", "string"] + } + } + }, + "call": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "called_person": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json new file mode 100644 index 000000000000..17bd429d1d27 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json new file mode 100644 index 000000000000..17bd429d1d27 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json new file mode 100644 index 000000000000..9d4b6742c00d --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "field_type": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/email_templates.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/email_templates.json index 528418e4baed..3ffb3310cf52 100644 --- a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/email_templates.json +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/email_templates.json @@ -100,25 +100,16 @@ } }, "groups": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { "type": "object", "additionalProperties": true, "properties": { "id": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "_href": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json new file mode 100644 index 000000000000..d95ee8241b72 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json @@ -0,0 +1,192 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "start_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "end_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "calendar_id": { + "type": ["null", "string"] + }, + "calendar_type": { + "type": ["null", "string"] + }, + "meeting_type": { + "type": ["null", "string"] + }, + "recipient_name": { + "type": ["null", "string"] + }, + "recipient_email": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "task_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "guests": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "attendees": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "organizer": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + }, + "status_changed": { + "type": ["null", "boolean"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + }, + "person": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "cadence": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "step": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "booked_by_user": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "crm_references": { + "type": ["null", "object"], + "additionalProperties": true + }, + "event_source": { + "type": ["null", "string"] + }, + "canceled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "all_day": { + "type": ["null", "boolean"] + }, + "no_show": { + "type": ["null", "boolean"] + }, + "crm_custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "strict_attribution": { + "type": ["null", "boolean"] + }, + "i_cal_uid": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "reschedule_status": { + "type": ["null", "string"] + }, + "owned_by_meetings_settings": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "email_address": ["null", "string"] + } + } + }, + "booked_by_meetings_settings": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "email_address": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json new file mode 100644 index 000000000000..3abf87ed7b66 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json @@ -0,0 +1,124 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "user": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "user_slug": { + "type": ["null", "string"] + }, + "primary_calendar_id": { + "type": ["null", "string"] + }, + "primary_calendar_name": { + "type": ["null", "string"] + }, + "email_address": { + "type": ["null", "string"] + }, + "user_details": { + "type": ["null", "object"], + "additionalProperties": true + }, + "calendar_type": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "default_meeting_length": { + "type": ["null", "integer"] + }, + "availability_limit_enabled": { + "type": ["null", "boolean"] + }, + "availability_limit": { + "type": ["null", "integer"] + }, + "schedule_delay": { + "type": ["null", "integer"] + }, + "buffer_time_duration": { + "type": ["null", "integer"] + }, + "schedule_buffer_enabled": { + "type": ["null", "boolean"] + }, + "times_available": { + "type": ["null", "object"], + "additionalProperties": true + }, + "allow_booking_on_behalf": { + "type": ["null", "boolean"] + }, + "allow_booking_overtime": { + "type": ["null", "boolean"] + }, + "allow_event_overlap": { + "type": ["null", "boolean"] + }, + "allow_event_detail": { + "type": ["null", "boolean"] + }, + "enable_dynamic_location": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "time_zone": { + "type": ["null", "string"] + }, + "primary_calendar_connection_failed": { + "type": ["null", "boolean"] + }, + "enable_calendar_sync": { + "type": ["null", "boolean"] + }, + "reschedule_meetings_enabled": { + "type": ["null", "boolean"] + }, + "active_meeting_url": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py b/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py index 51136a94e6db..b784fb98e5e6 100644 --- a/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py @@ -226,6 +226,46 @@ def path(self, **kwargs) -> str: return "groups" +class CustomFields(SalesloftStream): + def path(self, **kwargs) -> str: + return "custom_fields" + + +class CallDataRecords(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "call_data_records" + + +class CallDispositions(SalesloftStream): + def path(self, **kwargs) -> str: + return "call_dispositions" + + +class CallSentiments(SalesloftStream): + def path(self, **kwargs) -> str: + return "call_sentiments" + + +class Meetings(SalesloftStream): + created_at_field = "created_at" + + def path(self, **kwargs) -> str: + return "meetings" + + +class Searches(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "meetings/settings/searches" + + @property + def http_method(self) -> str: + return "POST" + + # Source class SourceSalesloft(AbstractSource): def _create_authenticator(self, config) -> AuthBase: @@ -272,4 +312,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Groups(*args), Successes(*args), EmailTemplateAttachments(*args), + CustomFields(*args), + CallDataRecords(*args), + CallDispositions(*args), + CallSentiments(*args), + Meetings(*args), + Searches(*args), ] diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/spec.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/spec.json index d3f9240f909d..4f6bd0dc237e 100644 --- a/airbyte-integrations/connectors/source-salesloft/source_salesloft/spec.json +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/spec.json @@ -15,7 +15,14 @@ { "title": "Authenticate via OAuth", "type": "object", - "required": ["client_id", "client_secret", "refresh_token", "access_token", "token_expiry_date", "auth_type"], + "required": [ + "client_id", + "client_secret", + "refresh_token", + "access_token", + "token_expiry_date", + "auth_type" + ], "properties": { "auth_type": { "type": "string", diff --git a/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py b/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py index 9664728482da..06efdbbaf2b8 100644 --- a/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py @@ -9,7 +9,7 @@ def test_streams(config): source = SourceSalesloft() streams = source.streams(config) - expected_streams_number = 23 + expected_streams_number = 29 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml b/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml index 06f324bbd146..2fdfa8eeeba4 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml +++ b/airbyte-integrations/connectors/source-sap-fieldglass/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/requirements.txt b/airbyte-integrations/connectors/source-sap-fieldglass/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/requirements.txt +++ b/airbyte-integrations/connectors/source-sap-fieldglass/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/setup.py b/airbyte-integrations/connectors/source-sap-fieldglass/setup.py index 79374dd96e89..0f5e02083b5b 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/setup.py +++ b/airbyte-integrations/connectors/source-sap-fieldglass/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml index 9715513c67c7..65fc8a95d574 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: database connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 @@ -15,6 +17,7 @@ data: license: MIT name: Scaffold Java Jdbc releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/scaffold-java-jdbc tags: diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml index d7fbe4ed450a..a11162b92d68 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 @@ -15,6 +17,7 @@ data: license: MIT name: Scaffold Source Http releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/scaffold-source-http tags: diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/requirements.txt b/airbyte-integrations/connectors/source-scaffold-source-http/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/requirements.txt +++ b/airbyte-integrations/connectors/source-scaffold-source-http/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py index 34660d6cf6d2..4fbb5fc38b4d 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-http/setup.py @@ -10,6 +10,7 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test", diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml index bd20283dc96e..d23f6b533f3a 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml @@ -5,6 +5,8 @@ data: registries: oss: enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 @@ -15,6 +17,7 @@ data: license: MIT name: Scaffold Source Python releaseDate: TODO + supportLevel: community releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/scaffold-source-python tags: diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt b/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt +++ b/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index 3b290beb996f..b302f081011f 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -10,6 +10,8 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.2", "connector-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-search-metrics/metadata.yaml b/airbyte-integrations/connectors/source-search-metrics/metadata.yaml index 10281e2b5ef8..ee0f969317bd 100644 --- a/airbyte-integrations/connectors/source-search-metrics/metadata.yaml +++ b/airbyte-integrations/connectors/source-search-metrics/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/search-metrics tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-search-metrics/requirements.txt b/airbyte-integrations/connectors/source-search-metrics/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-search-metrics/requirements.txt +++ b/airbyte-integrations/connectors/source-search-metrics/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-search-metrics/setup.py b/airbyte-integrations/connectors/source-search-metrics/setup.py index dd8bf2c700b2..fda5333cc2ae 100644 --- a/airbyte-integrations/connectors/source-search-metrics/setup.py +++ b/airbyte-integrations/connectors/source-search-metrics/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-secoda/metadata.yaml b/airbyte-integrations/connectors/source-secoda/metadata.yaml index 408fa362833a..03c428b40206 100644 --- a/airbyte-integrations/connectors/source-secoda/metadata.yaml +++ b/airbyte-integrations/connectors/source-secoda/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-secoda/requirements.txt b/airbyte-integrations/connectors/source-secoda/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-secoda/requirements.txt +++ b/airbyte-integrations/connectors/source-secoda/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-secoda/setup.py b/airbyte-integrations/connectors/source-secoda/setup.py index ad7aeede1bbc..3b603c1d6e21 100644 --- a/airbyte-integrations/connectors/source-secoda/setup.py +++ b/airbyte-integrations/connectors/source-secoda/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-sendgrid/integration_tests/abnormal_state.json index 07fe1a68b5f7..aca4cd802cf8 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/abnormal_state.json @@ -1,45 +1,45 @@ [ { - "type" : "STREAM", - "stream" : { - "stream_state" : { - "created" : "7270247822" + "type": "STREAM", + "stream": { + "stream_state": { + "created": "7270247822" }, - "stream_descriptor" : { - "name" : "global_suppressions" + "stream_descriptor": { + "name": "global_suppressions" } } }, { - "type" : "STREAM", - "stream" : { - "stream_state" : { - "created" : "7270247822" + "type": "STREAM", + "stream": { + "stream_state": { + "created": "7270247822" }, - "stream_descriptor" : { - "name" : "blocks" + "stream_descriptor": { + "name": "blocks" } } }, { - "type" : "STREAM", - "stream" : { - "stream_state" : { - "created" : "7270247822" + "type": "STREAM", + "stream": { + "stream_state": { + "created": "7270247822" }, - "stream_descriptor" : { - "name" : "bounces" + "stream_descriptor": { + "name": "bounces" } } }, { - "type" : "STREAM", - "stream" : { - "stream_state" : { - "created" : "7270247822" + "type": "STREAM", + "stream": { + "stream_state": { + "created": "7270247822" }, - "stream_descriptor" : { - "name" : "invalid_emails" + "stream_descriptor": { + "name": "invalid_emails" } } } diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json index 0488569dd985..2e5c801025c4 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/configured_catalog.json @@ -140,7 +140,7 @@ "sync_mode": "incremental", "cursor_field": ["created"], "destination_sync_mode": "append" - }, + }, { "stream": { "name": "unsubscribe_groups", diff --git a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml index f9aa1e16bf08..7aa5500cc96f 100644 --- a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sendgrid/requirements.txt b/airbyte-integrations/connectors/source-sendgrid/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-sendgrid/requirements.txt +++ b/airbyte-integrations/connectors/source-sendgrid/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index f6aacbfdedd9..64dc92cce236 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -8,8 +8,8 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "backoff", "requests", "pandas"] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", "requests-mock", ] diff --git a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json index c4e159f16363..ba932cf1f2f4 100644 --- a/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json +++ b/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/spec.json @@ -20,10 +20,7 @@ "format": "date-time", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(.\\d+)?Z$", "description": "Start time in ISO8601 format. Any data before this time point will not be replicated.", - "examples": [ - "2020-01-01T01:01:01Z", - "2020-01-01T01:01:01.000001Z" - ], + "examples": ["2020-01-01T01:01:01Z", "2020-01-01T01:01:01.000001Z"], "order": 1 } } diff --git a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml index 8e855617d244..6d5b26d340ba 100644 --- a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 2e88fa20-a2f6-43cc-bba6-98a0a3f244fb dockerImageTag: 0.1.0 dockerRepository: airbyte/source-sendinblue + documentationUrl: https://docs.airbyte.com/integrations/sources/sendinblue githubIssueLabel: source-sendinblue icon: sendinblue.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/sendinblue + supportLevel: community tags: - language:low-code - language:python diff --git a/airbyte-integrations/connectors/source-sendinblue/requirements.txt b/airbyte-integrations/connectors/source-sendinblue/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-sendinblue/requirements.txt +++ b/airbyte-integrations/connectors/source-sendinblue/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sendinblue/setup.py b/airbyte-integrations/connectors/source-sendinblue/setup.py index d416a3913eae..03d9985367a3 100644 --- a/airbyte-integrations/connectors/source-sendinblue/setup.py +++ b/airbyte-integrations/connectors/source-sendinblue/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-senseforce/metadata.yaml b/airbyte-integrations/connectors/source-senseforce/metadata.yaml index 25586469ef38..3c9417880e60 100644 --- a/airbyte-integrations/connectors/source-senseforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-senseforce/metadata.yaml @@ -22,4 +22,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-senseforce/requirements.txt b/airbyte-integrations/connectors/source-senseforce/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-senseforce/requirements.txt +++ b/airbyte-integrations/connectors/source-senseforce/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-senseforce/setup.py b/airbyte-integrations/connectors/source-senseforce/setup.py index cf3a0521a8f1..976590623a83 100644 --- a/airbyte-integrations/connectors/source-senseforce/setup.py +++ b/airbyte-integrations/connectors/source-senseforce/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index 9b4d66a31b28..24ceb52fa4bf 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index bc098588202a..91c39b263e9d 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -5,16 +5,10 @@ acceptance_tests: timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: "events" - bypass_reason: "Data reset after some time" - - name: "issues" - bypass_reason: "Data reset after some time" ignored_fields: project_detail: + - name: access + bypass_reason: "Order access return randomly" - name: features bypass_reason: "Order features return randomly" - name: organization/features @@ -22,6 +16,8 @@ acceptance_tests: - name: plugins/*/features bypass_reason: "Order features return randomly" projects: + - name: access + bypass_reason: "Order access return randomly" - name: features bypass_reason: "Order features return randomly" - name: organization/features diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json index 1faba770179b..943781880339 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/configured_catalog.json @@ -10,12 +10,12 @@ } }, { - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", + "sync_mode": "incremental", + "destination_sync_mode": "append", "stream": { "name": "issues", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] } }, { diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl index 90ec48921b1a..fe181c199a95 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl @@ -1,4 +1,6 @@ -{"stream": "projects", "data": {"id": "6712547", "slug": "demo-integration", "name": "demo-integration", "platform": "javascript-react", "dateCreated": "2022-09-02T15:01:28.946777Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view"], "firstEvent": "2022-09-02T15:36:50.870000Z", "firstTransactionEvent": false, "access": ["team:read", "org:read", "project:admin", "team:write", "team:admin", "alerts:read", "alerts:write", "project:releases", "project:write", "event:admin", "member:read", "org:integrations", "event:write", "event:read", "project:read"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#bf833f", "status": "active", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["performance-n-plus-one-db-queries-visible", "performance-slow-db-query-ingest", "performance-m-n-plus-one-db-queries-ingest", "performance-issues-compressed-assets-detector", "issue-alert-fallback-targeting", "performance-view", "new-spike-protection", "org-subdomains", "performance-consecutive-db-issue", "metrics-extraction", "profiling-ga", "profiling-aggregate-flamegraph", "performance-render-blocking-asset-span-ingest", "india-promotion", "open-ai-suggestion", "event-attachments", "profile-image-decode-main-thread-visible", "profile-image-decode-main-thread-ingest", "device-classification", "performance-consecutive-http-visible", "session-replay-index-subquery", "performance-n-plus-one-api-calls-post-process-group", "profile-image-decode-main-thread-post-process-group", "dashboards-rh-widget", "performance-n-plus-one-db-queries-ingest", "advanced-search", "alert-crash-free-metrics", "performance-issues-m-n-plus-one-db-detector", "issue-alert-incompatible-rules", "performance-file-io-main-thread-post-process-group", "performance-issues-all-events-tab", "customer-domains", "symbol-sources", "performance-consecutive-http-detector", "performance-issues-render-blocking-assets-detector", "onboarding-project-deletion-on-back-click", "mobile-cpu-memory-in-transactions", "performance-n-plus-one-api-calls-visible", "paid-to-free-promotion", "performance-landing-page-stats-period", "shared-issues", "performance-n-plus-one-api-calls-ingest", "profile-json-decode-main-thread-post-process-group", "profile-file-io-main-thread-ingest", "onboarding-remove-multiselect-platform", "minute-resolution-sessions", "performance-transaction-name-only-search-indexed", "session-replay-ui", "performance-slow-db-issue", "profile-json-decode-main-thread-ingest", "ondemand-budgets", "integrations-stacktrace-link", "performance-slow-db-query-post-process-group", "mute-alerts", "metric-alert-chartcuterie", "performance-new-widget-designs", "anr-rate", "performance-file-io-main-thread-detector", "transaction-metrics-extraction", "performance-db-main-thread-visible", "track-button-click-events", "monitors", "device-class-synthesis", "transaction-name-mark-scrubbed-as-sanitized", "performance-render-blocking-asset-span-visible", "issue-alert-preview", "issue-platform", "session-replay-ga", "slack-overage-notifications", "performance-m-n-plus-one-db-queries-post-process-group", "integrations-deployment", "performance-consecutive-http-post-process-group", "performance-m-n-plus-one-db-queries-visible", "performance-db-main-thread-detector", "dynamic-sampling", "issue-alert-test-notifications", "performance-slow-db-query-visible", "business-to-team-promotion", "performance-consecutive-db-queries-ingest", "project-stats", "performance-uncompressed-assets-ingest", "profiling-billing", "performance-onboarding-checklist", "session-replay-network-details", "profiling-span-previews", "issue-states", "promotion-be-adoption-enabled", "performance-consecutive-db-queries-post-process-group", "profiling", "performance-uncompressed-assets-post-process-group", "invite-members-rate-limits", "profile-json-decode-main-thread-visible", "profiling-previews", "performance-metrics-backed-transaction-summary", "profile-file-io-main-thread-post-process-group", "derive-code-mappings", "discover-events-rate-limit", "release-health-drop-sessions", "promotion-mobperf-discount20", "dynamic-sampling-transaction-name-priority", "performance-db-main-thread-post-process-group", "transaction-name-normalize", "sql-format", "performance-consecutive-http-ingest", "streamline-targeting-context", "am2-billing", "onboarding", "issue-list-removal-action", "auto-enable-codecov", "profile-file-io-main-thread-visible", "onboarding-sdk-selection", "performance-render-blocking-asset-span-post-process-group", "performance-file-io-main-thread-visible", "promotion-mobperf-gift50kerr", "performance-span-histogram-view", "session-replay", "performance-n-plus-one-db-queries-post-process-group", "performance-file-io-main-thread-ingest", "open-membership", "performance-n-plus-one-api-calls-detector", "profiling-sampled-format", "performance-uncompressed-assets-visible", "performance-issues-search", "session-replay-recording-scrubbing", "performance-mep-bannerless-ui", "performance-consecutive-db-queries-visible", "source-maps-debug-ids", "performance-db-main-thread-ingest", "mep-rollout-flag"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}}, "emitted_at": 1684220248380} -{"stream": "projects", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["team:read", "org:read", "project:admin", "team:write", "team:admin", "alerts:read", "alerts:write", "project:releases", "project:write", "event:admin", "member:read", "org:integrations", "event:write", "event:read", "project:read"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["performance-n-plus-one-db-queries-visible", "performance-slow-db-query-ingest", "performance-m-n-plus-one-db-queries-ingest", "performance-issues-compressed-assets-detector", "issue-alert-fallback-targeting", "performance-view", "new-spike-protection", "org-subdomains", "performance-consecutive-db-issue", "metrics-extraction", "profiling-ga", "profiling-aggregate-flamegraph", "performance-render-blocking-asset-span-ingest", "india-promotion", "open-ai-suggestion", "event-attachments", "profile-image-decode-main-thread-visible", "profile-image-decode-main-thread-ingest", "device-classification", "performance-consecutive-http-visible", "session-replay-index-subquery", "performance-n-plus-one-api-calls-post-process-group", "profile-image-decode-main-thread-post-process-group", "dashboards-rh-widget", "performance-n-plus-one-db-queries-ingest", "advanced-search", "alert-crash-free-metrics", "performance-issues-m-n-plus-one-db-detector", "issue-alert-incompatible-rules", "performance-file-io-main-thread-post-process-group", "performance-issues-all-events-tab", "customer-domains", "symbol-sources", "performance-consecutive-http-detector", "performance-issues-render-blocking-assets-detector", "onboarding-project-deletion-on-back-click", "mobile-cpu-memory-in-transactions", "performance-n-plus-one-api-calls-visible", "paid-to-free-promotion", "performance-landing-page-stats-period", "shared-issues", "performance-n-plus-one-api-calls-ingest", "profile-json-decode-main-thread-post-process-group", "profile-file-io-main-thread-ingest", "onboarding-remove-multiselect-platform", "minute-resolution-sessions", "performance-transaction-name-only-search-indexed", "session-replay-ui", "performance-slow-db-issue", "profile-json-decode-main-thread-ingest", "ondemand-budgets", "integrations-stacktrace-link", "performance-slow-db-query-post-process-group", "mute-alerts", "metric-alert-chartcuterie", "performance-new-widget-designs", "anr-rate", "performance-file-io-main-thread-detector", "transaction-metrics-extraction", "performance-db-main-thread-visible", "track-button-click-events", "monitors", "device-class-synthesis", "transaction-name-mark-scrubbed-as-sanitized", "performance-render-blocking-asset-span-visible", "issue-alert-preview", "issue-platform", "session-replay-ga", "slack-overage-notifications", "performance-m-n-plus-one-db-queries-post-process-group", "integrations-deployment", "performance-consecutive-http-post-process-group", "performance-m-n-plus-one-db-queries-visible", "performance-db-main-thread-detector", "dynamic-sampling", "issue-alert-test-notifications", "performance-slow-db-query-visible", "business-to-team-promotion", "performance-consecutive-db-queries-ingest", "project-stats", "performance-uncompressed-assets-ingest", "profiling-billing", "performance-onboarding-checklist", "session-replay-network-details", "profiling-span-previews", "issue-states", "promotion-be-adoption-enabled", "performance-consecutive-db-queries-post-process-group", "profiling", "performance-uncompressed-assets-post-process-group", "invite-members-rate-limits", "profile-json-decode-main-thread-visible", "profiling-previews", "performance-metrics-backed-transaction-summary", "profile-file-io-main-thread-post-process-group", "derive-code-mappings", "discover-events-rate-limit", "release-health-drop-sessions", "promotion-mobperf-discount20", "dynamic-sampling-transaction-name-priority", "performance-db-main-thread-post-process-group", "transaction-name-normalize", "sql-format", "performance-consecutive-http-ingest", "streamline-targeting-context", "am2-billing", "onboarding", "issue-list-removal-action", "auto-enable-codecov", "profile-file-io-main-thread-visible", "onboarding-sdk-selection", "performance-render-blocking-asset-span-post-process-group", "performance-file-io-main-thread-visible", "promotion-mobperf-gift50kerr", "performance-span-histogram-view", "session-replay", "performance-n-plus-one-db-queries-post-process-group", "performance-file-io-main-thread-ingest", "open-membership", "performance-n-plus-one-api-calls-detector", "profiling-sampled-format", "performance-uncompressed-assets-visible", "performance-issues-search", "session-replay-recording-scrubbing", "performance-mep-bannerless-ui", "performance-consecutive-db-queries-visible", "source-maps-debug-ids", "performance-db-main-thread-ingest", "mep-rollout-flag"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}}, "emitted_at": 1684220248381} -{"stream": "releases", "data": {"id": 289364918, "version": "checkout-app@3.2", "status": "open", "shortVersion": "checkout-app@3.2", "versionInfo": {"package": "checkout-app", "version": {"raw": "3.2", "major": 3, "minor": 2, "patch": 0, "pre": null, "buildCode": null, "components": 2}, "description": "3.2", "buildHash": null}, "ref": null, "url": null, "dateReleased": null, "dateCreated": "2021-09-02T08:10:12.826000Z", "data": {}, "newGroups": 0, "owner": null, "commitCount": 0, "lastCommit": null, "deployCount": 0, "lastDeploy": null, "authors": [], "projects": [{"id": 5942472, "slug": "airbyte-09", "name": "airbyte-09", "newGroups": 0, "platform": "python", "platforms": ["python"], "hasHealthData": false}], "firstEvent": null, "lastEvent": null, "currentProjectMeta": {}, "userAgent": null}, "emitted_at": 1681113724844} -{"stream": "project_detail", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["event:write", "event:read", "project:admin", "alerts:write", "org:integrations", "alerts:read", "team:admin", "team:write", "project:write", "org:read", "project:read", "member:read", "project:releases", "team:read", "event:admin"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "team": {"id": "1170523", "slug": "airbyte", "name": "Airbyte"}, "teams": [{"id": "1170523", "slug": "airbyte", "name": "Airbyte"}], "latestRelease": {"id": 289364918, "version": "checkout-app@3.2", "status": "open", "shortVersion": "checkout-app@3.2", "versionInfo": {"package": "checkout-app", "version": {"raw": "3.2", "major": 3, "minor": 2, "patch": 0, "pre": null, "buildCode": null, "components": 2}, "description": "3.2", "buildHash": null}, "ref": null, "url": null, "dateReleased": null, "dateCreated": "2021-09-02T08:10:12.826000Z", "data": {}, "newGroups": 0, "owner": null, "commitCount": 0, "lastCommit": null, "deployCount": 0, "lastDeploy": null, "authors": [], "projects": [{"id": 5942472, "slug": "airbyte-09", "name": "airbyte-09", "newGroups": 0, "platform": "python", "platforms": ["python"], "hasHealthData": false}], "firstEvent": null, "lastEvent": null, "currentProjectMeta": {}, "userAgent": null}, "options": {"quotas:spike-protection-disabled": false, "sentry:token": "5006ad000bc111ec95cd8e5fccda0a6a", "sentry:option-epoch": 7, "sentry:csp_ignored_sources_defaults": true, "sentry:csp_ignored_sources": "", "sentry:reprocessing_active": false, "filters:blacklisted_ips": "", "filters:react-hydration-errors": true, "filters:releases": "", "filters:error_messages": "", "feedback:branding": true}, "digestsMinDelay": 300, "digestsMaxDelay": 1800, "subjectPrefix": "", "allowedDomains": ["*"], "resolveAge": 0, "dataScrubber": true, "dataScrubberDefaults": true, "safeFields": [], "storeCrashReports": null, "sensitiveFields": [], "subjectTemplate": "$shortID - $title", "securityToken": "5006ad000bc111ec95cd8e5fccda0a6a", "securityTokenHeader": null, "verifySSL": false, "scrubIPAddresses": false, "scrapeJavaScript": true, "groupingConfig": "newstyle:2019-10-29", "groupingEnhancements": "", "groupingEnhancementsBase": null, "secondaryGroupingExpiry": 0, "secondaryGroupingConfig": null, "groupingAutoUpdate": true, "fingerprintingRules": "", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["performance-consecutive-http-visible", "session-replay-recording-scrubbing", "paid-to-free-promotion", "shared-issues", "auto-enable-codecov", "performance-db-main-thread-post-process-group", "performance-n-plus-one-api-calls-visible", "source-maps-debug-ids", "release-health-drop-sessions", "onboarding-project-deletion-on-back-click", "issue-list-removal-action", "transaction-metrics-extraction", "performance-metrics-backed-transaction-summary", "promotion-mobperf-discount20", "open-membership", "performance-consecutive-db-issue", "performance-file-io-main-thread-ingest", "performance-issues-render-blocking-assets-detector", "profile-file-io-main-thread-post-process-group", "profiling", "anr-rate", "performance-uncompressed-assets-visible", "performance-issues-search", "mobile-cpu-memory-in-transactions", "device-class-synthesis", "performance-consecutive-db-queries-visible", "profile-image-decode-main-thread-visible", "transaction-name-normalize", "discover-events-rate-limit", "open-ai-suggestion", "derive-code-mappings", "performance-n-plus-one-db-queries-post-process-group", "performance-n-plus-one-db-queries-ingest", "performance-n-plus-one-db-queries-visible", "issue-alert-preview", "org-subdomains", "onboarding-remove-multiselect-platform", "dashboards-rh-widget", "performance-slow-db-query-post-process-group", "profile-file-io-main-thread-visible", "performance-render-blocking-asset-span-post-process-group", "onboarding-sdk-selection", "profiling-billing", "performance-db-main-thread-ingest", "session-replay-network-details", "performance-file-io-main-thread-visible", "performance-consecutive-http-detector", "onboarding", "issue-alert-incompatible-rules", "performance-n-plus-one-api-calls-post-process-group", "issue-alert-test-notifications", "profile-image-decode-main-thread-post-process-group", "profile-json-decode-main-thread-post-process-group", "customer-domains", "performance-n-plus-one-api-calls-detector", "metrics-extraction", "am2-billing", "performance-span-histogram-view", "monitors", "profile-image-decode-main-thread-ingest", "performance-view", "profile-json-decode-main-thread-visible", "session-replay-ui", "dynamic-sampling", "performance-uncompressed-assets-ingest", "performance-landing-page-stats-period", "performance-m-n-plus-one-db-queries-ingest", "performance-mep-bannerless-ui", "mep-rollout-flag", "performance-consecutive-http-post-process-group", "performance-slow-db-query-visible", "promotion-mobperf-gift50kerr", "issue-platform", "performance-issues-compressed-assets-detector", "session-replay-index-subquery", "new-spike-protection", "event-attachments", "issue-alert-fallback-targeting", "symbol-sources", "streamline-targeting-context", "performance-m-n-plus-one-db-queries-post-process-group", "metric-alert-chartcuterie", "project-stats", "session-replay", "performance-m-n-plus-one-db-queries-visible", "profile-json-decode-main-thread-ingest", "integrations-stacktrace-link", "track-button-click-events", "advanced-search", "performance-onboarding-checklist", "business-to-team-promotion", "performance-slow-db-issue", "profiling-ga", "minute-resolution-sessions", "performance-new-widget-designs", "performance-render-blocking-asset-span-ingest", "performance-n-plus-one-api-calls-ingest", "performance-consecutive-db-queries-ingest", "sql-format", "performance-issues-m-n-plus-one-db-detector", "performance-uncompressed-assets-post-process-group", "dynamic-sampling-transaction-name-priority", "performance-db-main-thread-detector", "india-promotion", "profile-file-io-main-thread-ingest", "invite-members-rate-limits", "sandbox-kill-switch", "promotion-be-adoption-enabled", "performance-slow-db-query-ingest", "performance-transaction-name-only-search-indexed", "slack-overage-notifications", "alert-crash-free-metrics", "integrations-deployment", "device-classification", "performance-render-blocking-asset-span-visible", "mute-alerts", "ondemand-budgets", "session-replay-ga", "performance-file-io-main-thread-detector", "performance-consecutive-db-queries-post-process-group", "performance-file-io-main-thread-post-process-group", "transaction-name-mark-scrubbed-as-sanitized", "performance-consecutive-http-ingest", "performance-db-main-thread-visible", "performance-issues-all-events-tab"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}, "plugins": [{"id": "asana", "name": "Asana", "slug": "asana", "shortName": "Asana", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nImprove your productivity by creating tasks in Asana directly\nfrom Sentry issues. This integration also allows you to link Sentry\nissues to existing tasks in Asana.\n", "features": ["issue-basic"], "featureDescriptions": [{"description": "Create and link Sentry issue groups directly to an Asana ticket in any of your\n projects, providing a quick way to jump from a Sentry bug to tracked ticket!", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing Asana tickets.", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "bitbucket", "name": "Bitbucket", "slug": "bitbucket", "shortName": "Bitbucket", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": true, "description": "Integrate Bitbucket issues by linking a repository to a project.", "features": ["issue-basic", "commits"], "featureDescriptions": [{"description": "Track commits and releases (learn more\n [here](https://docs.sentry.io/learn/releases/))", "featureGate": "commits"}, {"description": "Create Bitbucket issues from Sentry", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing Bitbucket issues", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "github", "name": "GitHub", "slug": "github", "shortName": "GitHub", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": true, "description": "Integrate GitHub issues by linking a repository to a project.", "features": ["issue-basic", "commits"], "featureDescriptions": [{"description": "Authorize repositories to be added to your Sentry organization to augment\n sentry issues with commit data with [deployment\n tracking](https://docs.sentry.io/learn/releases/).", "featureGate": "commits"}, {"description": "Create and link Sentry issue groups directly to a GitHub issue or pull\n request in any of your repositories, providing a quick way to jump from\n Sentry bug to tracked issue or PR!", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "gitlab", "name": "GitLab", "slug": "gitlab", "shortName": "GitLab", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": true, "description": "Integrate GitLab issues by linking a repository to a project", "features": ["issue-basic", "commits"], "featureDescriptions": [{"description": "Track commits and releases (learn more\n [here](https://docs.sentry.io/learn/releases/))", "featureGate": "commits"}, {"description": "Resolve Sentry issues via GitLab commits and merge requests by\n including `Fixes PROJ-ID` in the message", "featureGate": "commits"}, {"description": "Create GitLab issues from Sentry", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing GitLab issues", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "heroku", "name": "Heroku", "slug": "heroku", "shortName": "Heroku", "type": "release-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "\n

    Add a Sentry release webhook to automatically track new releases.

    \n 
    heroku webhooks:add -i api:release -l notify -u https://sentry.io/api/hooks/release/heroku/5942472/cd92fd5564e9d4a2b82dd03ad2d539a1a2b83ea13e2e205507d2f8d3c513d864/ -a YOUR_APP_NAME
    \n ", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry"}, "isDeprecated": false, "isHidden": false, "description": "Integrate Heroku release tracking.", "features": ["deployment"], "featureDescriptions": [{"description": "Integrate Heroku release tracking.", "featureGate": "deployment"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "jira", "name": "JIRA", "slug": "jira", "shortName": "JIRA", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": true, "description": "Integrate JIRA issues by linking a project.", "features": ["issue-basic"], "featureDescriptions": [{"description": "Create and link Sentry issue groups directly to a Jira ticket in any of your\n projects, providing a quick way to jump from a Sentry bug to tracked ticket!", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "opsgenie", "name": "OpsGenie", "slug": "opsgenie", "shortName": "OpsGenie", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry"}, "isDeprecated": false, "isHidden": false, "description": "\nTrigger alerts in Opsgenie from Sentry.\n\nOpsgenie is a cloud-based service for dev & ops teams, providing reliable\nalerts, on-call schedule management and escalations. OpsGenie integrates with\nmonitoring tools & services, ensures the right people are notified. This\nplugin only supports issue alerts.\n", "features": ["alert-rule", "incident-management"], "featureDescriptions": [{"description": "Manage incidents and outages by sending Sentry notifications to OpsGenie.", "featureGate": "incident-management"}, {"description": "Configure Sentry rules to trigger notifications based on conditions you set.", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "pagerduty", "name": "PagerDuty", "slug": "pagerduty", "shortName": "PagerDuty", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": true, "description": "Send alerts to PagerDuty.", "features": ["alert-rule", "incident-management"], "featureDescriptions": [{"description": "Manage incidents and outages by sending Sentry notifications to PagerDuty.", "featureGate": "incident-management"}, {"description": "Configure rule based PagerDuty alerts to automatically be triggered in a specific\n service - or in multiple services!", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "phabricator", "name": "Phabricator", "slug": "phabricator", "shortName": "Phabricator", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nImprove your productivity by creating tickets in Phabricator directly from Sentry issues.\nThis integration also allows you to link Sentry issues to existing tickets in Phabricator.\n\nPhabricator is a set of tools for developing software. It includes applications for\ncode review, repository hosting, bug tracking, project management, and more.\n", "features": ["issue-basic"], "featureDescriptions": [{"description": "Create and link Sentry issue groups directly to a Phabricator ticket in any of your\n projects, providing a quick way to jump from a Sentry bug to tracked ticket!", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing Phabricator tickets.", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "pivotal", "name": "Pivotal Tracker", "slug": "pivotal", "shortName": "Pivotal Tracker", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nImprove your productivity by creating tickets in Pivotal Tracker directly from Sentry issues.\nThis integration also allows you to link Sentry issues to existing tickets in Pivotal Tracker.\n\nPivotal Tracker is a straightforward project-planning tool that helps software development\nteams form realistic expectations about when work might be completed based on the teams\nongoing performance. Tracker visualizes your projects in the form of stories\nmoving through your workflow, encouraging you to break down projects into manageable\nchunks and have important conversations about deliverables and scope.\n", "features": ["issue-basic"], "featureDescriptions": [{"description": "Create and link Sentry issue groups directly to a Pivotal Tracker ticket in any of your\n projects, providing a quick way to jump from a Sentry bug to tracked ticket!", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing Pivotal Tracker tickets.", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "pushover", "name": "Pushover", "slug": "pushover", "shortName": "Pushover", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nGet notified of Sentry alerts on any device using the Pushover integration.\n\nPushover makes it easy to get real-time notifications on your Android, iPhone, iPad, and Desktop.\n", "features": ["alert-rule", "mobile"], "featureDescriptions": [{"description": "Have Pushover notifications get sent to your mobile device with the Pushover app.", "featureGate": "mobile"}, {"description": "Configure Sentry rules to trigger notifications based on conditions you set.", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "redmine", "name": "Redmine", "slug": "redmine", "shortName": "Redmine", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nCreate issues in Redmine directly from Sentry. This integration also\nallows you to link Sentry issues to existing tickets in Redmine.\n\nRedmine is a flexible project management web application. Written using\nthe Ruby on Rails framework, it is cross-platform and cross-database.\n", "features": ["issue-basic"], "featureDescriptions": [{"description": "Create and link Sentry issue groups directly to an Redmine issue in any of your\n projects, providing a quick way to jump from a Sentry bug to tracked ticket!", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing Redmine issue.", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "sessionstack", "name": "SessionStack", "slug": "sessionstack", "shortName": "SessionStack", "type": "default", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": ["sessionstack"], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "Watch SessionStack recordings in Sentry.", "features": ["session-replay"], "featureDescriptions": [{"description": "Watch the SessionStack session replay of a user in a video widget embedded in the Sentry UI for an issue.", "featureGate": "session-replay"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "slack", "name": "Slack", "slug": "slack", "shortName": "Slack", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": true, "description": "Post notifications to a Slack channel.", "features": ["alert-rule"], "featureDescriptions": [{"description": "Configure rule based Slack notifications to automatically be posted into a\n specific channel. Want any error that's happening more than 100 times a\n minute to be posted in `#critical-errors`? Setup a rule for it!", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "trello", "name": "Trello", "slug": "trello", "shortName": "Trello", "type": "issue-tracking", "canDisable": true, "isTestable": false, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nCreate cards in Trello directly from Sentry. This integration also allows\nyou to link Sentry issues to existing cards in Trello.\n\nTrello is the easy, free, flexible, and visual way to manage your projects\nand organize anything, trusted by millions of people from all over the world.\n", "features": ["issue-basic"], "featureDescriptions": [{"description": "Create and link Sentry issue groups directly to an Trello card in any of your\n projects, providing a quick way to jump from a Sentry bug to tracked ticket!", "featureGate": "issue-basic"}, {"description": "Link Sentry issues to existing Trello cards", "featureGate": "issue-basic"}], "resourceLinks": [{"title": "Trello Setup Instructions", "url": "https://github.com/getsentry/sentry/blob/master/src/sentry_plugins/trello/Trello_Instructions.md"}, {"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "twilio", "name": "Twilio (SMS)", "slug": "twilio", "shortName": "Twilio (SMS)", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nGet notified of Sentry alerts via SMS.\n\nTwilio allows users to send and receive text messages globally with\nthe API that over a million developers depend on.\n", "features": ["alert-rule", "mobile"], "featureDescriptions": [{"description": "Set up SMS notifications to be sent to your mobile device via Twilio.", "featureGate": "mobile"}, {"description": "Configure Sentry rules to trigger notifications based on conditions you set.", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Documentation", "url": "https://github.com/getsentry/sentry/blob/master/src/sentry_plugins/twilio/Twilio_Instructions.md"}, {"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins/twilio"}, {"title": "Twilio", "url": "https://www.twilio.com/"}]}, {"id": "victorops", "name": "VictorOps", "slug": "victorops", "shortName": "VictorOps", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nTrigger alerts in VictorOps from Sentry.\n\nVictorOps is incident response software purpose-built for teams powering the\nevolution of software. With on-call basics, cross-team collaboration, and\nstreamlined visibility, we champion the engineers powering innovation and uptime.\n", "features": ["alert-rule", "incident-management"], "featureDescriptions": [{"description": "Manage incidents and outages by sending Sentry notifications to VictorOps.", "featureGate": "incident-management"}, {"description": "Configure Sentry rules to trigger notifications based on conditions you set.", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry_plugins"}]}, {"id": "webhooks", "name": "WebHooks", "slug": "webhooks", "shortName": "WebHooks", "type": "notification", "canDisable": true, "isTestable": true, "hasConfiguration": true, "metadata": {}, "contexts": [], "status": "unknown", "assets": [], "doc": "", "firstPartyAlternative": null, "deprecationDate": null, "altIsSentryApp": null, "enabled": false, "version": "23.6.0.dev0", "author": {"name": "Sentry Team", "url": "https://github.com/getsentry/sentry"}, "isDeprecated": false, "isHidden": false, "description": "\nTrigger outgoing HTTP POST requests from Sentry.\n\nNote: To configure webhooks over multiple projects, we recommend setting up an\nInternal Integration.\n", "features": ["alert-rule"], "featureDescriptions": [{"description": "Configure rule based outgoing HTTP POST requests from Sentry.", "featureGate": "alert-rule"}], "resourceLinks": [{"title": "Report Issue", "url": "https://github.com/getsentry/sentry/issues"}, {"title": "View Source", "url": "https://github.com/getsentry/sentry/tree/master/src/sentry/plugins/sentry_webhooks"}, {"title": "Internal Integrations", "url": "https://docs.sentry.io/workflow/integrations/integration-platform/#internal-integrations"}]}], "platforms": ["python"], "processingIssues": 0, "defaultEnvironment": null, "relayPiiConfig": null, "builtinSymbolSources": ["ios", "microsoft", "android"], "dynamicSamplingBiases": [{"id": "boostEnvironments", "active": true}, {"id": "boostLatestRelease", "active": true}, {"id": "ignoreHealthChecks", "active": true}, {"id": "boostKeyTransactions", "active": true}, {"id": "boostLowVolumeTransactions", "active": true}, {"id": "boostReplayId", "active": true}], "eventProcessing": {"symbolicationDegraded": false}, "symbolSources": "[]"}, "emitted_at": 1684777365197} +{"stream": "project_detail", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["member:read", "org:read", "team:admin", "event:admin", "org:integrations", "team:read", "event:read", "project:write", "project:releases", "alerts:read", "project:read", "event:write", "team:write", "alerts:write", "project:admin"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "team": {"id": "1170523", "slug": "airbyte", "name": "Airbyte"}, "teams": [{"id": "1170523", "slug": "airbyte", "name": "Airbyte"}], "latestRelease": {"version": "checkout-app@3.2"}, "options": {"quotas:spike-protection-disabled": false, "sentry:token": "5006ad000bc111ec95cd8e5fccda0a6a", "sentry:option-epoch": 7, "sentry:csp_ignored_sources_defaults": true, "sentry:csp_ignored_sources": "", "sentry:reprocessing_active": false, "filters:blacklisted_ips": "", "filters:react-hydration-errors": true, "filters:releases": "", "filters:error_messages": "", "feedback:branding": true}, "digestsMinDelay": 300, "digestsMaxDelay": 1800, "subjectPrefix": "", "allowedDomains": ["*"], "resolveAge": 0, "dataScrubber": true, "dataScrubberDefaults": true, "safeFields": [], "recapServerUrl": null, "storeCrashReports": null, "sensitiveFields": [], "subjectTemplate": "$shortID - $title", "securityToken": "5006ad000bc111ec95cd8e5fccda0a6a", "securityTokenHeader": null, "verifySSL": false, "scrubIPAddresses": false, "scrapeJavaScript": true, "groupingConfig": "newstyle:2023-01-11", "groupingEnhancements": "", "groupingEnhancementsBase": null, "secondaryGroupingExpiry": 0, "secondaryGroupingConfig": null, "groupingAutoUpdate": true, "fingerprintingRules": "", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["anr-rate", "paid-to-free-promotion", "performance-m-n-plus-one-db-queries-visible", "performance-file-io-main-thread-visible", "customer-domains", "performance-landing-page-stats-period", "performance-uncompressed-assets-visible", "profile-image-decode-main-thread-ingest", "integrations-deployment", "performance-consecutive-db-queries-visible", "streamline-targeting-context", "performance-large-http-payload-visible", "assign-to-me", "issue-platform", "performance-render-blocking-asset-span-ingest", "sourcemaps-bundle-indexing", "bundle-plan-checkout", "symbol-sources", "release-health-drop-sessions", "profile-image-decode-main-thread-visible", "performance-db-main-thread-post-process-group", "profile-image-decode-main-thread-post-process-group", "performance-issues-all-events-tab", "performance-consecutive-http-ingest", "performance-n-plus-one-db-queries-post-process-group", "ondemand-budgets", "performance-slow-db-query-ingest", "performance-consecutive-http-post-process-group", "performance-db-main-thread-ingest", "performance-issues-compressed-assets-detector", "crons-timeline-listing-page", "team-project-creation-all", "dynamic-sampling", "promotion-be-adoption-enabled", "onboarding", "performance-onboarding-checklist", "performance-span-histogram-view", "performance-m-n-plus-one-db-queries-ingest", "performance-issues-m-n-plus-one-db-detector", "alert-crash-free-metrics", "performance-large-http-payload-ingest", "invite-members-rate-limits", "integrations-stacktrace-link", "promotion-mobperf-gift50kerr", "performance-render-blocking-asset-span-post-process-group", "transaction-name-normalize", "transaction-metrics-extraction", "business-to-team-promotion", "performance-n-plus-one-api-calls-detector", "dashboards-rh-widget", "transaction-name-mark-scrubbed-as-sanitized", "performance-consecutive-db-queries-ingest", "performance-slow-db-issue", "session-replay", "profile-json-decode-main-thread-visible", "metrics-extraction", "profile-file-io-main-thread-ingest", "performance-m-n-plus-one-db-queries-post-process-group", "performance-uncompressed-assets-post-process-group", "monitors", "onboarding-sdk-selection", "performance-n-plus-one-db-queries-visible", "performance-n-plus-one-api-calls-visible", "performance-n-plus-one-api-calls-post-process-group", "performance-consecutive-db-issue", "performance-mep-bannerless-ui", "performance-consecutive-db-queries-post-process-group", "profile-file-io-main-thread-post-process-group", "mobile-cpu-memory-in-transactions", "performance-slow-db-query-post-process-group", "device-classification", "issue-alert-fallback-targeting", "performance-uncompressed-assets-ingest", "performance-slow-db-query-visible", "profile-file-io-main-thread-visible", "performance-issues-search", "performance-consecutive-http-visible", "promotion-mobperf-discount20", "minute-resolution-sessions", "onboarding-project-deletion-on-back-click", "profiling", "metric-alert-chartcuterie", "mute-metric-alerts", "device-class-synthesis", "performance-new-widget-designs", "getting-started-doc-with-product-selection", "profiling-billing", "profiling-ga", "open-ai-suggestion", "org-subdomains", "performance-view", "session-replay-ui", "performance-n-plus-one-db-queries-ingest", "profile-json-decode-main-thread-post-process-group", "performance-render-blocking-asset-span-visible", "performance-metrics-backed-transaction-summary", "derive-code-mappings", "performance-file-io-main-thread-post-process-group", "performance-db-main-thread-detector", "track-button-click-events", "event-attachments", "performance-file-io-main-thread-ingest", "performance-large-http-payload-post-process-group", "open-membership", "issue-list-better-priority-sort", "shared-issues", "performance-transaction-name-only-search-indexed", "project-stats", "mep-rollout-flag", "ds-sliding-window-org", "am2-billing", "performance-issues-render-blocking-assets-detector", "slack-overage-notifications", "india-promotion", "session-replay-recording-scrubbing", "slack-use-new-lookup", "crons-issue-platform", "performance-file-io-main-thread-detector", "auto-enable-codecov", "discover-events-rate-limit", "performance-db-main-thread-visible", "performance-consecutive-http-detector", "performance-large-http-payload-detector", "profile-json-decode-main-thread-ingest", "advanced-search", "performance-n-plus-one-api-calls-ingest"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}, "plugins": [], "platforms": [], "processingIssues": 0, "defaultEnvironment": null, "relayPiiConfig": null, "builtinSymbolSources": ["ios", "microsoft", "android"], "dynamicSamplingBiases": [{"id": "boostEnvironments", "active": true}, {"id": "boostLatestRelease", "active": true}, {"id": "ignoreHealthChecks", "active": true}, {"id": "boostKeyTransactions", "active": true}, {"id": "boostLowVolumeTransactions", "active": true}, {"id": "boostReplayId", "active": true}, {"id": "recalibrationRule", "active": true}], "eventProcessing": {"symbolicationDegraded": false}, "symbolSources": "[]"}, "emitted_at": 1689246410694} +{"stream":"projects","data":{"id":"6712547","slug":"demo-integration","name":"demo-integration","platform":"javascript-react","dateCreated":"2022-09-02T15:01:28.946777Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view"],"firstEvent":"2022-09-02T15:36:50.870000Z","firstTransactionEvent":false,"access":["org:integrations","team:read","alerts:write","alerts:read","project:read","member:read","project:write","project:admin","event:admin","team:write","event:write","project:releases","team:admin","event:read","org:read"],"hasAccess":true,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf833f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"features":["metrics-extraction","new-spike-protection","promotion-mobperf-discount20","session-replay-recording-scrubbing","performance-m-n-plus-one-db-queries-ingest","performance-consecutive-db-queries-ingest","sql-format","slack-overage-notifications","ondemand-budgets","issue-list-prefetch-issue-on-hover","profiling","performance-render-blocking-asset-span-ingest","getting-started-doc-with-product-selection","performance-consecutive-http-post-process-group","performance-view","integrations-stacktrace-link","dashboards-rh-widget","bundle-plan-checkout","invite-members-rate-limits","advanced-search","performance-db-main-thread-detector","integrations-deployment","profile-json-decode-main-thread-post-process-group","transaction-name-mark-scrubbed-as-sanitized","performance-db-main-thread-ingest","india-promotion","performance-consecutive-db-issue","performance-mep-bannerless-ui","performance-consecutive-http-detector","device-class-synthesis","auto-enable-codecov","performance-n-plus-one-api-calls-post-process-group","performance-render-blocking-asset-span-post-process-group","metric-alert-chartcuterie","session-replay","performance-landing-page-stats-period","transaction-name-normalize","symbol-sources","performance-issues-search","profile-json-decode-main-thread-ingest","sentry-pride-logo-footer","performance-uncompressed-assets-visible","profile-file-io-main-thread-visible","project-stats","performance-n-plus-one-api-calls-visible","performance-file-io-main-thread-ingest","profile-image-decode-main-thread-post-process-group","release-health-drop-sessions","minute-resolution-sessions","onboarding","performance-span-histogram-view","profile-image-decode-main-thread-ingest","profile-json-decode-main-thread-visible","paid-to-free-promotion","performance-file-io-main-thread-post-process-group","source-maps-debug-ids","performance-large-http-payload-post-process-group","performance-m-n-plus-one-db-queries-visible","profile-image-decode-main-thread-visible","ds-sliding-window-org","performance-issues-all-events-tab","performance-consecutive-db-queries-visible","streamline-targeting-context","onboarding-sdk-selection","session-replay-click-search-banner-rollout","performance-onboarding-checklist","discover-events-rate-limit","performance-m-n-plus-one-db-queries-post-process-group","shared-issues","performance-issues-m-n-plus-one-db-detector","profiling-billing","open-ai-suggestion","anr-rate","performance-n-plus-one-db-queries-visible","business-to-team-promotion","monitors","performance-slow-db-issue","team-project-creation-all","session-replay-index-subquery","performance-file-io-main-thread-visible","crons-issue-platform","performance-issues-compressed-assets-detector","performance-db-main-thread-post-process-group","performance-transaction-name-only-search-indexed","alert-crash-free-metrics","performance-slow-db-query-visible","dynamic-sampling-transaction-name-priority","performance-n-plus-one-db-queries-post-process-group","performance-render-blocking-asset-span-visible","onboarding-project-deletion-on-back-click","session-replay-ga","performance-file-io-main-thread-detector","profile-file-io-main-thread-post-process-group","promotion-mobperf-gift50kerr","performance-db-main-thread-visible","performance-n-plus-one-api-calls-ingest","performance-n-plus-one-db-queries-ingest","am2-billing","performance-uncompressed-assets-post-process-group","performance-metrics-backed-transaction-summary","performance-n-plus-one-api-calls-detector","performance-slow-db-query-post-process-group","performance-large-http-payload-ingest","ds-boost-new-projects","customer-domains","mobile-cpu-memory-in-transactions","session-replay-network-details","performance-slow-db-query-ingest","issue-alert-fallback-targeting","performance-consecutive-http-visible","performance-uncompressed-assets-ingest","session-replay-ui","performance-consecutive-db-queries-post-process-group","profiling-ga","open-membership","performance-new-widget-designs","crons-timeline-listing-page","event-attachments","mep-rollout-flag","device-classification","derive-code-mappings","profile-file-io-main-thread-ingest","issue-platform","track-button-click-events","performance-issues-render-blocking-assets-detector","dynamic-sampling","performance-large-http-payload-visible","transaction-metrics-extraction","performance-large-http-payload-detector","performance-consecutive-http-ingest","promotion-be-adoption-enabled","org-subdomains"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1687535328146} +{"stream":"projects","data":{"id":"5942472","slug":"airbyte-09","name":"airbyte-09","platform":"python","dateCreated":"2021-09-02T07:42:22.421223Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","releases"],"firstEvent":null,"firstTransactionEvent":false,"access":["org:integrations","team:read","alerts:write","alerts:read","project:read","member:read","project:write","project:admin","event:admin","team:write","event:write","project:releases","team:admin","event:read","org:read"],"hasAccess":true,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#803fbf","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"features":["metrics-extraction","new-spike-protection","promotion-mobperf-discount20","session-replay-recording-scrubbing","performance-m-n-plus-one-db-queries-ingest","performance-consecutive-db-queries-ingest","sql-format","slack-overage-notifications","ondemand-budgets","issue-list-prefetch-issue-on-hover","profiling","performance-render-blocking-asset-span-ingest","getting-started-doc-with-product-selection","performance-consecutive-http-post-process-group","performance-view","integrations-stacktrace-link","dashboards-rh-widget","bundle-plan-checkout","invite-members-rate-limits","advanced-search","performance-db-main-thread-detector","integrations-deployment","profile-json-decode-main-thread-post-process-group","transaction-name-mark-scrubbed-as-sanitized","performance-db-main-thread-ingest","india-promotion","performance-consecutive-db-issue","performance-mep-bannerless-ui","performance-consecutive-http-detector","device-class-synthesis","auto-enable-codecov","performance-n-plus-one-api-calls-post-process-group","performance-render-blocking-asset-span-post-process-group","metric-alert-chartcuterie","session-replay","performance-landing-page-stats-period","transaction-name-normalize","symbol-sources","performance-issues-search","profile-json-decode-main-thread-ingest","sentry-pride-logo-footer","performance-uncompressed-assets-visible","profile-file-io-main-thread-visible","project-stats","performance-n-plus-one-api-calls-visible","performance-file-io-main-thread-ingest","profile-image-decode-main-thread-post-process-group","release-health-drop-sessions","minute-resolution-sessions","onboarding","performance-span-histogram-view","profile-image-decode-main-thread-ingest","profile-json-decode-main-thread-visible","paid-to-free-promotion","performance-file-io-main-thread-post-process-group","source-maps-debug-ids","performance-large-http-payload-post-process-group","performance-m-n-plus-one-db-queries-visible","profile-image-decode-main-thread-visible","ds-sliding-window-org","performance-issues-all-events-tab","performance-consecutive-db-queries-visible","streamline-targeting-context","onboarding-sdk-selection","session-replay-click-search-banner-rollout","performance-onboarding-checklist","discover-events-rate-limit","performance-m-n-plus-one-db-queries-post-process-group","shared-issues","performance-issues-m-n-plus-one-db-detector","profiling-billing","open-ai-suggestion","anr-rate","performance-n-plus-one-db-queries-visible","business-to-team-promotion","monitors","performance-slow-db-issue","team-project-creation-all","session-replay-index-subquery","performance-file-io-main-thread-visible","crons-issue-platform","performance-issues-compressed-assets-detector","performance-db-main-thread-post-process-group","performance-transaction-name-only-search-indexed","alert-crash-free-metrics","performance-slow-db-query-visible","dynamic-sampling-transaction-name-priority","performance-n-plus-one-db-queries-post-process-group","performance-render-blocking-asset-span-visible","onboarding-project-deletion-on-back-click","session-replay-ga","performance-file-io-main-thread-detector","profile-file-io-main-thread-post-process-group","promotion-mobperf-gift50kerr","performance-db-main-thread-visible","performance-n-plus-one-api-calls-ingest","performance-n-plus-one-db-queries-ingest","am2-billing","performance-uncompressed-assets-post-process-group","performance-metrics-backed-transaction-summary","performance-n-plus-one-api-calls-detector","performance-slow-db-query-post-process-group","performance-large-http-payload-ingest","ds-boost-new-projects","customer-domains","mobile-cpu-memory-in-transactions","session-replay-network-details","performance-slow-db-query-ingest","issue-alert-fallback-targeting","performance-consecutive-http-visible","performance-uncompressed-assets-ingest","session-replay-ui","performance-consecutive-db-queries-post-process-group","profiling-ga","open-membership","performance-new-widget-designs","crons-timeline-listing-page","event-attachments","mep-rollout-flag","device-classification","derive-code-mappings","profile-file-io-main-thread-ingest","issue-platform","track-button-click-events","performance-issues-render-blocking-assets-detector","dynamic-sampling","performance-large-http-payload-visible","transaction-metrics-extraction","performance-large-http-payload-detector","performance-consecutive-http-ingest","promotion-be-adoption-enabled","org-subdomains"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1687535328148} +{"stream": "releases", "data": {"id": 289364918, "version": "checkout-app@3.2", "status": "open", "shortVersion": "checkout-app@3.2", "versionInfo": {"package": "checkout-app", "version": {"raw": "3.2", "major": 3, "minor": 2, "patch": 0, "pre": null, "buildCode": null, "components": 2}, "description": "3.2", "buildHash": null}, "ref": null, "url": null, "dateReleased": null, "dateCreated": "2021-09-02T08:10:12.826000Z", "data": {}, "newGroups": 0, "owner": null, "commitCount": 0, "lastCommit": null, "deployCount": 0, "lastDeploy": null, "authors": [], "projects": [{"id": 5942472, "slug": "airbyte-09", "name": "airbyte-09", "newGroups": 0, "platform": "python", "platforms": [], "hasHealthData": false}], "firstEvent": null, "lastEvent": null, "currentProjectMeta": {}, "userAgent": null}, "emitted_at": 1689246658349} +{"stream": "issues", "data": {"id": "4365423845", "shareId": null, "shortId": "AIRBYTE-09-4", "title": "This is an example Python exception", "culprit": "raven.scripts.runner in main", "permalink": "https://airbyte-09.sentry.io/issues/4365423845/", "logger": null, "level": "error", "status": "unresolved", "statusDetails": {}, "substatus": "ongoing", "isPublic": false, "platform": "python", "project": {"id": "5942472", "name": "airbyte-09", "slug": "airbyte-09", "platform": "python"}, "type": "default", "metadata": {"title": "This is an example Python exception"}, "numComments": 0, "assignedTo": null, "isBookmarked": false, "isSubscribed": false, "subscriptionDetails": null, "hasSeen": true, "annotations": [], "issueType": "error", "issueCategory": "error", "isUnhandled": false, "count": "10", "userCount": 1, "firstSeen": "2023-08-02T23:22:34.982000Z", "lastSeen": "2023-08-02T23:31:20.165000Z"}, "emitted_at": 1691020096265} +{"stream": "events", "data": {"id": "1cce9233aeb04eba8bbdb7bb00f00592", "groupID": "4365423845", "eventID": "1cce9233aeb04eba8bbdb7bb00f00592", "projectID": "5942472", "size": 8151, "entries": [{"data": {"formatted": "This is an example Python exception"}, "type": "message"}, {"data": {"frames": [{"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "build_msg", "rawFunction": null, "symbol": null, "context": [[298, " frames = stack"], [299, ""], [300, " data.update({"], [301, " 'sentry.interfaces.Stacktrace': {"], [302, " 'frames': get_stack_info(frames,"], [303, " transformer=self.transform)"], [304, " },"], [305, " })"], [306, ""], [307, " if 'sentry.interfaces.Stacktrace' in data:"], [308, " if self.include_paths:"]], "lineNo": 303, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'culprit'": null, "'data'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'date'": "datetime.datetime(2013, 8, 13, 3, 8, 24, 880386)", "'event_id'": "'54a322436e1b47b88e239b78998ae742'", "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'frames'": "", "'handler'": "", "'k'": "'sentry.interfaces.Message'", "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'public_key'": null, "'result'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null, "'v'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "capture", "rawFunction": null, "symbol": null, "context": [[454, " if not self.is_enabled():"], [455, " return"], [456, ""], [457, " data = self.build_msg("], [458, " event_type, data, date, time_spent, extra, stack, tags=tags,"], [459, " **kwargs)"], [460, ""], [461, " self.send(**data)"], [462, ""], [463, " return (data.get('event_id'),)"], [464, ""]], "lineNo": 459, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'data'": null, "'date'": null, "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "captureMessage", "rawFunction": null, "symbol": null, "context": [[572, " \"\"\""], [573, " Creates an event from ``message``."], [574, ""], [575, " >>> client.captureMessage('My event just happened!')"], [576, " \"\"\""], [577, " return self.capture('raven.events.Message', message=message, **kwargs)"], [578, ""], [579, " def captureException(self, exc_info=None, **kwargs):"], [580, " \"\"\""], [581, " Creates an event from an exception."], [582, ""]], "lineNo": 577, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'kwargs'": {"'data'": null, "'extra'": {"'go_deeper'": ["[{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}]"], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'level'": 20, "'stack'": true, "'tags'": null}, "'message'": "'This is a test message generated using ``raven test``'", "'self'": ""}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "send_test_message", "rawFunction": null, "symbol": null, "context": [[72, " level=logging.INFO,"], [73, " stack=True,"], [74, " tags=options.get('tags', {}),"], [75, " extra={"], [76, " 'user': get_uid(),"], [77, " 'loadavg': get_loadavg(),"], [78, " },"], [79, " ))"], [80, ""], [81, " if client.state.did_fail():"], [82, " print('error!')"]], "lineNo": 77, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'client'": "", "'data'": null, "'k'": "'secret_key'", "'options'": {"'data'": null, "'tags'": null}}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "main", "rawFunction": null, "symbol": null, "context": [[107, " print(\"Using DSN configuration:\")"], [108, " print(\" \", dsn)"], [109, " print()"], [110, ""], [111, " client = Client(dsn, include_paths=['raven'])"], [112, " send_test_message(client, opts.__dict__)"]], "lineNo": 112, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'args'": ["'test'", "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'"], "'client'": "", "'dsn'": "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'", "'opts'": "", "'parser'": "", "'root'": ""}}], "framesOmitted": null, "registers": null, "hasSystemFrames": false}, "type": "stacktrace"}, {"data": {"apiTarget": null, "method": "GET", "url": "http://example.com/foo", "query": [["foo", "bar"]], "fragment": null, "data": {"hello": "world"}, "headers": [["Content-Type", "application/json"], ["Referer", "http://example.com"], ["User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"]], "cookies": [["foo", "bar"], ["biz", "baz"]], "env": {"ENV": "prod"}, "inferredContentType": "application/json"}, "type": "request"}], "dist": null, "message": "This is an example Python exception", "title": "This is an example Python exception", "location": null, "user": {"id": "1", "email": "sentry@example.com", "username": "sentry", "ip_address": "127.0.0.1", "name": "Sentry", "data": null}, "contexts": {"browser": {"name": "Chrome", "version": "28.0.1500", "type": "browser"}, "client_os": {"name": "Windows", "version": "8", "type": "os"}}, "sdk": null, "context": {"emptyList": [], "emptyMap": {}, "length": 10837790, "results": [1, 2, 3, 4, 5], "session": {"foo": "bar"}, "unauthorized": false, "url": "http://example.org/foo/bar/"}, "packages": {"my.package": "1.0.0"}, "type": "default", "metadata": {"title": "This is an example Python exception"}, "tags": [{"key": "browser", "value": "Chrome 28.0.1500"}, {"key": "browser.name", "value": "Chrome"}, {"key": "client_os", "value": "Windows 8"}, {"key": "client_os.name", "value": "Windows"}, {"key": "environment", "value": "prod"}, {"key": "level", "value": "error"}, {"key": "sample_event", "value": "yes"}, {"key": "server_name", "value": "web01.example.org"}, {"key": "url", "value": "http://example.com/foo"}, {"key": "user", "value": "id:1", "query": "user.id:\"1\""}], "platform": "python", "dateReceived": "2023-08-02T23:31:41.101814Z", "errors": [], "occurrence": null, "_meta": {"entries": {}, "message": null, "user": null, "contexts": null, "sdk": null, "context": null, "packages": null, "tags": {}}, "crashFile": null, "culprit": "raven.scripts.runner in main", "dateCreated": "2023-08-02T23:30:41Z", "fingerprints": ["3a2b45089d0211943e5a6645fb4cea3f"], "groupingConfig": {"id": "newstyle:2023-01-11", "enhancements": "eJybzDRxc15qeXFJZU6qlZGBkbGugaGuoeEEAHJMCAM"}}, "emitted_at": 1691020171602} diff --git a/airbyte-integrations/connectors/source-sentry/metadata.yaml b/airbyte-integrations/connectors/source-sentry/metadata.yaml index bbd7b658d112..1e4b9e046536 100644 --- a/airbyte-integrations/connectors/source-sentry/metadata.yaml +++ b/airbyte-integrations/connectors/source-sentry/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.4 maxSecondsBetweenMessages: 64800 dockerRepository: airbyte/source-sentry githubIssueLabel: source-sentry @@ -22,4 +22,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sentry/requirements.txt b/airbyte-integrations/connectors/source-sentry/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-sentry/requirements.txt +++ b/airbyte-integrations/connectors/source-sentry/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py index 18e79a0e2f48..18d172dcd942 100644 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ b/airbyte-integrations/connectors/source-sentry/setup.py @@ -10,10 +10,10 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "requests_mock~=1.9", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index fb6b39e964d5..2cd4f4d516cf 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -4,7 +4,7 @@ from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional +from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional import pendulum import requests @@ -177,10 +177,33 @@ def request_params( next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"statsPeriod": "", "query": ""}) - + filter_date = self._get_filter_date(stream_state) + params.update(self._build_query_params(filter_date)) return params + def _get_filter_date(self, stream_state: Optional[Mapping[str, Any]]) -> str: + """Retrieve the filter date from the stream state or use the start_date.""" + return stream_state.get(self.cursor_field) or self.start_date if stream_state else self.start_date + + def _build_query_params(self, filter_date: str) -> Dict[str, str]: + """Generate query parameters for the request.""" + filter_date_iso = pendulum.parse(filter_date).to_iso8601_string() + return {"statsPeriod": "", "query": f"lastSeen:>{filter_date_iso}"} + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[MutableMapping]: + json_response = response.json() or [] + + for record in json_response: + cursor_value = self._get_cursor_value(record, stream_state) + self.state = {self.cursor_field: cursor_value} + yield record + + def _get_cursor_value(self, record: Dict[str, Any], stream_state: Mapping[str, Any]) -> pendulum.datetime: + """Compute the maximum cursor value based on the record and stream state.""" + record_time = record[self.cursor_field] + state_time = str(self.get_state_value(stream_state)) + return max(record_time, state_time) + class Projects(SentryIncremental): """ diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 21347f70fb5b..1ac42a5217e2 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -95,7 +95,7 @@ def test_events_request_params(): def test_issues_request_params(): stream = Issues(**INIT_ARGS) - expected = {"cursor": "next-page", "statsPeriod": "", "query": ""} + expected = {"cursor": "next-page", "statsPeriod": "", "query": "lastSeen:>1900-01-01T00:00:00Z"} assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected diff --git a/airbyte-integrations/connectors/source-serpstat/.dockerignore b/airbyte-integrations/connectors/source-serpstat/.dockerignore new file mode 100644 index 000000000000..dbd3fc7ddd07 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_serpstat +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-serpstat/Dockerfile b/airbyte-integrations/connectors/source-serpstat/Dockerfile new file mode 100644 index 000000000000..6e113cf522d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_serpstat ./source_serpstat + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-serpstat diff --git a/airbyte-integrations/connectors/source-serpstat/README.md b/airbyte-integrations/connectors/source-serpstat/README.md new file mode 100644 index 000000000000..c4698cf993b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/README.md @@ -0,0 +1,82 @@ +# Serpstat Source + +This is the repository for the Serpstat configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/serpstat). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-serpstat:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/serpstat) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_serpstat/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source serpstat test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-serpstat:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-serpstat:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-serpstat:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-serpstat:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-serpstat:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-serpstat:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-serpstat:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-serpstat:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-serpstat/__init__.py b/airbyte-integrations/connectors/source-serpstat/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-serpstat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-serpstat/acceptance-test-config.yml new file mode 100644 index 000000000000..da4d7e8dc7ec --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-serpstat:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_serpstat/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-serpstat/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-serpstat/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-serpstat/build.gradle b/airbyte-integrations/connectors/source-serpstat/build.gradle new file mode 100644 index 000000000000..446bea8580d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_serpstat' +} diff --git a/airbyte-integrations/connectors/source-serpstat/icon.svg b/airbyte-integrations/connectors/source-serpstat/icon.svg new file mode 100644 index 000000000000..a0adc252270f --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/icon.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/__init__.py b/airbyte-integrations/connectors/source-serpstat/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-serpstat/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..5bbce393cce7 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "Domain history": { + "date": "1200-12-31" + } +} diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-serpstat/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4527e4cabf2e --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/configured_catalog.json @@ -0,0 +1,81 @@ +{ + "streams": [ + { + "stream": { + "name": "Domain history", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "Domains summary", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["domain"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "Domain keywords", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["keyword"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "Domain keywords by region", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["db_name"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "Domain competitors", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["domain"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "Domain top pages", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object" + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-serpstat/integration_tests/invalid_config.json new file mode 100644 index 000000000000..2a633477032e --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/invalid_config.json @@ -0,0 +1,7 @@ +{ + "api_key": "api_key", + "domain": "serpstat.com", + "region_id": "g_us", + "page_size": 10, + "pages_to_fetch": 1 +} diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-serpstat/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-serpstat/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-serpstat/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-serpstat/main.py b/airbyte-integrations/connectors/source-serpstat/main.py new file mode 100644 index 000000000000..92fb7edc0474 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_serpstat import SourceSerpstat + +if __name__ == "__main__": + source = SourceSerpstat() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-serpstat/metadata.yaml b/airbyte-integrations/connectors/source-serpstat/metadata.yaml new file mode 100644 index 000000000000..e764c9c80059 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/metadata.yaml @@ -0,0 +1,22 @@ +data: + allowedHosts: + hosts: + - api.serpstat.com + registries: + oss: + enabled: true + connectorSubtype: api + connectorType: source + definitionId: 3b2e8fb2-9137-41ff-a1e1-83ecb39e26c8 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-serpstat + githubIssueLabel: source-serpstat + icon: serpstat.svg + license: MIT + name: Serpstat + releaseDate: 2023-08-21 + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/serpstat + tags: + - language:lowcode +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-serpstat/requirements.txt b/airbyte-integrations/connectors/source-serpstat/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-serpstat/setup.py b/airbyte-integrations/connectors/source-serpstat/setup.py new file mode 100644 index 000000000000..42ab32a171b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_serpstat", + description="Source implementation for Serpstat.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-serpstat/source_serpstat/__init__.py b/airbyte-integrations/connectors/source-serpstat/source_serpstat/__init__.py new file mode 100644 index 000000000000..8a15ca81d45f --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/source_serpstat/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceSerpstat + +__all__ = ["SourceSerpstat"] diff --git a/airbyte-integrations/connectors/source-serpstat/source_serpstat/manifest.yaml b/airbyte-integrations/connectors/source-serpstat/source_serpstat/manifest.yaml new file mode 100644 index 000000000000..66cc2151d7ba --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/source_serpstat/manifest.yaml @@ -0,0 +1,551 @@ +version: 0.43.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - Domains summary + +streams: + - type: DeclarativeStream + name: Domain history + primary_key: + - date + schema_loader: + type: InlineSchemaLoader + additionalProperties: true + schema: + $schema: http://json-schema.org/schema# + properties: + ad_keywords: + type: number + ads: + type: number + date: + type: string + domain: + type: string + down_keywords: + type: number + keywords: + type: number + new_keywords: + type: number + out_keywords: + type: number + rised_keywords: + type: number + traff: + type: number + visible: + type: number + visible_static: + type: number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.serpstat.com/v4/ + path: / + http_method: POST + request_parameters: {} + request_headers: + X-request-sender: Airbyte + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + field_name: token + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: RETRY + predicate: "{{response.error.code == 32000}}" + error_message: >- + You are sending more requests per second then available for + your Serpstat plan + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 2 + request_body_json: + id: "{{ now_utc() }}" + method: SerpstatDomainProcedure.getDomainsHistory + params: + se: "{{config['region_id']}}" + page: "{{(next_page_token['next_page_token'] or 0) + 1}}" + size: "{{config['page_size']}}" + sort: + date: desc + domain: "{{config['domain']}}" + during_all_time: true + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: header + field_name: X-page + pagination_strategy: + type: CursorPagination + cursor_value: "{{response.result.summary_info.page}}" + stop_condition: "{{response.result.summary_info.page > config['pages_to_fetch'] - 1}}" + - type: DeclarativeStream + name: Domains summary + primary_key: + - domain + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + ad_keywords: + type: number + ads: + type: number + ads_dynamic: + type: number + domain: + type: string + down_keywords: + type: number + keywords: + type: number + keywords_dynamic: + type: number + new_keywords: + type: number + out_keywords: + type: number + prev_date: + type: string + rised_keywords: + type: number + traff: + type: number + traff_dynamic: + type: number + visible: + type: number + visible_dynamic: + type: number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.serpstat.com/v4/ + path: / + http_method: POST + request_parameters: {} + request_headers: + X-request-sender: Airbyte + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + field_name: token + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - response_filters: + - type: HttpResponseFilter + action: FAIL + predicate: "{{ 'Invalid token' in response.error.message }}" + error_message: Invalid Token + - type: HttpResponseFilter + action: RETRY + predicate: "{{ 'Too Many Requests' in response.error.message }}" + error_message: >- + You are sending more requests per second then available for + your Serpstat plan + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 2 + request_body_json: + id: "{{ now_utc() }}" + method: SerpstatDomainProcedure.getDomainsInfo + params: + se: "{{config['region_id']}}" + domains: "{{config['domains']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - data + paginator: + type: NoPagination + - type: DeclarativeStream + name: Domain keywords + primary_key: + - keyword + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + concurrency: + type: number + cost: + type: number + difficulty: + type: number + domain: + type: string + dynamic: + type: + - "null" + - number + found_results: + type: number + geo_names: + type: array + keyword: + type: string + keyword_length: + type: number + position: + type: number + region_queries_count: + type: number + region_queries_count_wide: + type: number + subdomain: + type: + - "null" + - string + traff: + type: number + types: + items: + type: string + type: array + url: + type: string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.serpstat.com/v4/ + path: / + http_method: POST + request_parameters: {} + request_headers: + X-request-sender: Airbyte + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + field_name: token + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: RETRY + predicate: "{{response.error.code == 32000}}" + error_message: >- + You are sending more requests per second then available for + your Serpstat plan + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 2 + request_body_json: + id: "{{ now_utc() }}" + method: SerpstatDomainProcedure.getDomainKeywords + params: + se: "{{config['region_id']}}" + page: "{{(next_page_token['next_page_token'] or 0) + 1}}" + size: "{{config['page_size']}}" + sort: + "{{config['sort_by']}}": "{{config['sort_value']}}" + domain: "{{config['domain']}}" + filters: + "{{config['filter_by']}}": "{{config['filter_value']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: header + field_name: X-page + pagination_strategy: + type: CursorPagination + cursor_value: "{{response.result.summary_info.page}}" + stop_condition: "{{response.result.summary_info.page > config['pages_to_fetch'] - 1}}" + - type: DeclarativeStream + name: Domain keywords by region + primary_key: + - db_name + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + country_name_en: + type: string + db_name: + type: string + domain: + type: string + keywords_count: + type: number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.serpstat.com/v4/ + path: / + http_method: POST + request_parameters: {} + request_headers: + X-request-sender: Airbyte + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + field_name: token + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: RETRY + predicate: "{{response.error.code == 32000}}" + error_message: >- + You are sending more requests per second then available for + your Serpstat plan + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 2 + request_body_json: + id: "{{ now_utc() }}" + method: SerpstatDomainProcedure.getRegionsCount + params: + sort: "{{config['sort_by']}}" + order: "{{config['sort_value']}}" + domain: "{{config['domain']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - data + paginator: + type: NoPagination + - type: DeclarativeStream + name: Domain competitors + primary_key: + - domain + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + ad_keywords: + type: number + ads: + type: number + ads_dynamic: + type: number + common: + type: number + domain: + type: string + down_keywords: + type: number + intersected: + type: number + keywords: + type: number + keywords_dynamic: + type: number + missing: + type: number + new_keywords: + type: number + new_relevance: + type: number + not_intersected: + type: number + our_relevance: + type: number + out_keywords: + type: number + relevance: + type: number + rised_keywords: + type: number + traff: + type: number + traff_dynamic: + type: number + visible: + type: number + visible_dynamic: + type: number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.serpstat.com/v4/ + path: / + http_method: POST + request_parameters: {} + request_headers: + X-request-sender: Airbyte + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + field_name: token + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: RETRY + predicate: "{{response.error.code == 32000}}" + error_message: >- + You are sending more requests per second then available for + your Serpstat plan + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 2 + request_body_json: + id: "{{ now_utc() }}" + method: SerpstatDomainProcedure.getCompetitors + params: + se: "{{config['region_id']}}" + size: "{{config['page_size']}}" + sort: + "{{config['sort_by']}}": "{{config['sort_value']}}" + domain: "{{config['domain']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - data + paginator: + type: NoPagination + - type: DeclarativeStream + name: Domain top pages + primary_key: [] + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + facebook_shares: + type: number + organic_keywords: + type: number + potencial_traff: + type: number + url: + type: string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.serpstat.com/v4/ + path: / + http_method: POST + request_parameters: {} + request_headers: + X-request-sender: Airbyte + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config['api_key'] }}" + inject_into: + type: RequestOption + field_name: token + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: RETRY + predicate: "{{response.error.code == 32000}}" + error_message: >- + You are sending more requests per second then available for + your Serpstat plan + backoff_strategies: + - type: ExponentialBackoffStrategy + factor: 2 + request_body_json: + id: "{{ now_utc() }}" + method: SerpstatDomainProcedure.getTopUrls + params: + se: "{{config['region_id']}}" + page: "{{(next_page_token['next_page_token'] or 0) + 1}}" + size: "{{config['page_size']}}" + sort: + "{{config['sort_by']}}": "{{config['sort_value']}}" + domain: "{{config['domain']}}" + filters: + "{{config['filter_by']}}": "{{config['filter_value']}}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - result + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: header + field_name: X-page + pagination_strategy: + type: CursorPagination + cursor_value: "{{response.result.summary_info.page}}" + stop_condition: "{{response.result.summary_info.page > config['pages_to_fetch'] - 1}}" + +metadata: + autoImportSchema: + Domain history: true + Domains summary: true + Domain keywords: true + Domain keywords by region: true + Domain competitors: true + Domain top pages: true diff --git a/airbyte-integrations/connectors/source-serpstat/source_serpstat/source.py b/airbyte-integrations/connectors/source-serpstat/source_serpstat/source.py new file mode 100644 index 000000000000..4acdab3e2cec --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/source_serpstat/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceSerpstat(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-serpstat/source_serpstat/spec.yaml b/airbyte-integrations/connectors/source-serpstat/source_serpstat/spec.yaml new file mode 100644 index 000000000000..a75c98aff19b --- /dev/null +++ b/airbyte-integrations/connectors/source-serpstat/source_serpstat/spec.yaml @@ -0,0 +1,87 @@ +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - api_key + properties: + api_key: + type: string + title: API Key + airbyte_secret: true + order: 0 + description: >- + Serpstat API key can be found here: + https://serpstat.com/users/profile/ + domain: + type: string + order: 1 + title: Domain + default: serpstat.com + description: The domain name to get data for (ex. serpstat.com) + page_size: + type: integer + order: 2 + title: Page size + default: 10 + description: >- + The number of data rows per page to be returned. Each data row can + contain multiple data points. The max value is 1000. Reducing the size + of the page will result in fewer API credits spent. + domains: + type: array + order: 3 + title: Domains + description: >- + The list of domains that will be used in streams that support batch + operations + filter_by: + type: string + order: 4 + title: Filter by + description: >- + The field name by which the results should be filtered. Filtering the + results will result in fewer API credits spent. Each stream has + different filtering options. See https://serpstat.com/api/ for more + details. + filter_value: + type: string + order: 5 + title: Filter value + description: >- + The value of the field to filter by. Each stream has different + filtering options. See https://serpstat.com/api/ for more details. + sort_by: + type: string + order: 6 + title: Sort by + description: >- + The field name by which the results should be sorted. Each stream has + different sorting options. See https://serpstat.com/api/ for more + details. + sort_value: + type: string + order: 7 + title: Sort value + description: >- + The value of the field to sort by. Each stream has different sorting + options. See https://serpstat.com/api/ for more details. + pages_to_fetch: + type: integer + order: 8 + title: Pages to fetch + default: 1 + description: >- + The number of pages that should be fetched. All results will be + obtained if left blank. Reducing the number of pages will result in + fewer API credits spent. + region_id: + type: string + order: 9 + title: Region ID + default: g_us + description: >- + The ID of a region to get data from in the form of a two-letter + country code prepended with the g_ prefix. See the list of supported + region IDs here: https://serpstat.com/api/664-request-parameters-v4/. + additionalProperties: true +documentationUrl: https://docs.airbyte.com/integrations/sources/serpstat diff --git a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml index 09f6fffe7b57..9adbfaedfa33 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml +++ b/airbyte-integrations/connectors/source-sftp-bulk/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: file connectorType: source definitionId: 31e3242f-dee7-4cdc-a4b8-8e06c5458517 dockerImageTag: 0.1.2 dockerRepository: airbyte/source-sftp-bulk + documentationUrl: https://docs.airbyte.com/integrations/sources/sftp-bulk githubIssueLabel: source-sftp-bulk icon: sftp.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/sftp-bulk + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/requirements.txt b/airbyte-integrations/connectors/source-sftp-bulk/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/requirements.txt +++ b/airbyte-integrations/connectors/source-sftp-bulk/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sftp-bulk/setup.py b/airbyte-integrations/connectors/source-sftp-bulk/setup.py index d04bb87a1fef..6d9d1990f634 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/setup.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/setup.py @@ -13,7 +13,7 @@ "pandas==1.5.0", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "docker==5.0.3"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "docker==5.0.3"] setup( name="source_sftp_bulk", diff --git a/airbyte-integrations/connectors/source-sftp/metadata.yaml b/airbyte-integrations/connectors/source-sftp/metadata.yaml index 8f84bbb41830..dd9961561c93 100644 --- a/airbyte-integrations/connectors/source-sftp/metadata.yaml +++ b/airbyte-integrations/connectors/source-sftp/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: file connectorType: source definitionId: a827c52e-791c-4135-a245-e233c5255199 dockerImageTag: 0.1.2 dockerRepository: airbyte/source-sftp + documentationUrl: https://docs.airbyte.com/integrations/sources/sftp githubIssueLabel: source-sftp icon: sftp.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/sftp + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 2c41728b28ac..40a304001ec3 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.4 +LABEL io.airbyte.version=0.6.2 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index c457f03c93e5..9c1cea89b117 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -10,19 +10,20 @@ acceptance_tests: status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" + timeout_seconds: 3600 - config_path: "secrets/config_old.json" status: "succeed" - config_path: "integration_tests/invalid_config_old.json" status: "failed" + timeout_seconds: 3600 - config_path: "secrets/config_oauth.json" status: "succeed" - config_path: "integration_tests/invalid_oauth_config.json" status: "failed" + timeout_seconds: 3600 discovery: tests: - config_path: "secrets/config.json" - - config_path: "secrets/config_old.json" - - config_path: "secrets/config_oauth.json" basic_read: tests: - config_path: "secrets/config.json" @@ -34,6 +35,10 @@ acceptance_tests: bypass_reason: The stream holds data up to 1 month then records are removed by Shopify. - name: balance_transactions bypass_reason: The stream requires real purchases to fill in the data. + - name: customer_saved_search + bypass_reason: The stream is not available for our sandbox. + - name: disputes + bypass_reason: The stream requires real purchases to fill in the data. incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index 48398daa6ba7..e18e70cb07f4 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -460,5 +460,41 @@ "name": "metafield_shops" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999, + "customers": { + "updated_at": "2025-02-22T00:37:28-08:00" + } + }, + "stream_descriptor": { + "name": "customer_address" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "customer_saved_search" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 99999999999999 + }, + "stream_descriptor": { + "name": "disputes" + } + } } ] diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json index f804e2bf6242..05f4f41ddf4a 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -12,6 +12,18 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "disputes", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, { "stream": { "name": "metafield_articles", @@ -489,6 +501,42 @@ "sync_mode": "incremental", "cursor_field": ["id"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "customer_saved_search", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "customer_address", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "countries", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl index 6eea46606817..bc5935226ca0 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl @@ -1,73 +1,75 @@ -{"stream": "articles", "data": {"id": 558137508029, "title": "My new Article title", "created_at": "2022-10-07T16:09:02-07:00", "body_html": "

    I like articles

    \n

    Yea, I like posting them through REST.

    ", "blog_id": 80417685693, "author": "John Smith", "user_id": null, "published_at": "2011-03-24T08:45:47-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "summary_html": null, "template_suffix": null, "handle": "my-new-article-title", "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558137508029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360391602} -{"stream": "articles", "data": {"id": 558627979453, "title": "Test Blog Post", "created_at": "2023-04-14T03:19:02-07:00", "body_html": "Test Blog Post 1", "blog_id": 80417685693, "author": "Airbyte Airbyte", "user_id": "74861019325", "published_at": null, "updated_at": "2023-04-14T03:19:18-07:00", "summary_html": "", "template_suffix": "", "handle": "test-blog-post", "tags": "Has Been Tagged", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558627979453", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360391602} -{"stream": "metafield_articles", "data": {"id": 21519818162365, "namespace": "global", "key": "new", "value": "newvalue", "description": null, "owner_id": 558137508029, "created_at": "2022-10-07T16:09:02-07:00", "updated_at": "2022-10-07T16:09:02-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519818162365", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360393628} -{"stream": "metafield_articles", "data": {"id": 22365709992125, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Article Metafield", "description": null, "owner_id": 558137508029, "created_at": "2023-04-14T03:18:26-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365709992125", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360393628} -{"stream": "metafield_articles", "data": {"id": 22365710352573, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Blog Post Metafiled", "description": null, "owner_id": 558627979453, "created_at": "2023-04-14T03:19:18-07:00", "updated_at": "2023-04-14T03:19:18-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710352573", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360394123} -{"stream": "blogs", "data": {"id": 80417685693, "handle": "news", "title": "News", "updated_at": "2023-04-14T03:20:20-07:00", "commentable": "no", "feedburner": null, "feedburner_location": null, "created_at": "2021-06-22T18:00:25-07:00", "template_suffix": null, "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreBlog/80417685693", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360395276} -{"stream": "metafield_blogs", "data": {"id": 21519428255933, "namespace": "some_fields", "key": "sponsor", "value": "Shopify", "description": null, "owner_id": 80417685693, "created_at": "2022-10-07T06:05:23-07:00", "updated_at": "2022-10-07T06:05:23-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519428255933", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360397405} -{"stream": "metafield_blogs", "data": {"id": 22365710745789, "namespace": "custom", "key": "test_blog_metafield", "value": "Test Blog Metafield", "description": null, "owner_id": 80417685693, "created_at": "2023-04-14T03:20:20-07:00", "updated_at": "2023-04-14T03:20:20-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710745789", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360397405} -{"stream": "customers", "data": {"id": 6569096478909, "email": "test@test.com", "accepts_marketing": true, "created_at": "2023-04-13T02:30:04-07:00", "updated_at": "2023-04-24T06:53:48-07:00", "first_name": "New Test", "last_name": "Customer", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": "updated_mon_24.04.2023", "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": "+380639379992", "addresses": [{"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}], "accepts_marketing_updated_at": "2023-04-13T02:30:04-07:00", "marketing_opt_in_level": "single_opt_in", "tax_exemptions": "[]", "email_marketing_consent": {"state": "subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": "2023-04-13T02:30:04-07:00"}, "sms_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null, "consent_collected_from": "SHOPIFY"}, "admin_graphql_api_id": "gid://shopify/Customer/6569096478909", "default_address": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360398525} -{"stream": "metafield_customers", "data": {"id": 22346893361341, "namespace": "custom", "key": "test_definition_list_1", "value": "Teste\n", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893361341", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360400740} -{"stream": "metafield_customers", "data": {"id": 22346893394109, "namespace": "custom", "key": "test_definition", "value": "Taster", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893394109", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360400740} -{"stream": "orders", "data": {"id": 4554821468349, "admin_graphql_api_id": "gid://shopify/Order/4554821468349", "app_id": 580111, "browser_ip": "176.113.167.23", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 25048437719229, "checkout_token": "cf5d16a0a0688905bd551c6dec591506", "client_details": {"accept_language": "en-US,en;q=0.9,uk;q=0.8", "browser_height": 754, "browser_ip": "176.113.167.23", "browser_width": 1519, "session_hash": null, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53"}, "closed_at": "2022-06-15T06:25:43-07:00", "confirmed": true, "contact_email": "integration-test@airbyte.io", "created_at": "2022-06-15T05:16:53-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "integration-test@airbyte.io", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": "fulfilled", "gateway": "bogus", "landing_site": "/wallets/checkouts.json", "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1136", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 136, "order_number": 1136, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/e4f98630ea44a884e33e700203ce2130/authenticate?key=edf087d6ae55a4541bf1375432f6a4b8", "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "presentment_currency": "USD", "processed_at": "2022-06-15T05:16:53-07:00", "processing_method": "direct", "reference": null, "referring_site": "https://airbyte-integration-test.myshopify.com/products/all-black-sneaker-right-foot", "source_identifier": null, "source_name": "web", "source_url": null, "subtotal_price": 57.23, "subtotal_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "tags": "Refund", "tax_lines": [], "taxes_included": true, "test": true, "token": "e4f98630ea44a884e33e700203ce2130", "total_discounts": 1.77, "total_discounts_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}, "total_line_items_price": 59.0, "total_line_items_price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 57.23, "total_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:00:37-07:00", "user_id": null, "billing_address": {"first_name": "Iryna", "address1": "2261 Market Street", "phone": null, "city": "San Francisco", "zip": "94114", "province": "California", "country": "United States", "last_name": "Grankova", "address2": "4381", "company": null, "latitude": 37.7647751, "longitude": -122.4320369, "name": "Iryna Grankova", "country_code": "US", "province_code": "CA"}, "customer": {"id": 5362027233469, "email": "integration-test@airbyte.io", "accepts_marketing": false, "created_at": "2021-07-08T05:41:47-07:00", "updated_at": "2022-06-22T03:50:13-07:00", "first_name": "Airbyte", "last_name": "Team", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-08T05:41:47-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5362027233469", "default_address": {"id": 7492260823229, "customer_id": 5362027233469, "first_name": "Airbyte", "last_name": "Team", "company": null, "address1": "2261 Market Street", "address2": "4381", "city": "San Francisco", "province": "California", "country": "United States", "zip": "94114", "phone": null, "name": "Airbyte Team", "province_code": "CA", "country_code": "US", "country_name": "United States", "default": true}}, "discount_applications": [{"target_type": "line_item", "type": "automatic", "value": "3.0", "value_type": "percentage", "allocation_method": "across", "target_selection": "all", "title": "eeeee"}], "fulfillments": [{"id": 4075788501181, "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "created_at": "2022-06-15T05:16:55-07:00", "location_id": 63590301885, "name": "#1136.1", "order_id": 4554821468349, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2022-06-15T05:16:55-07:00", "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}]}], "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}], "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "payment_terms": null, "refunds": [{"id": 852809646269, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "created_at": "2022-06-15T06:25:43-07:00", "note": null, "order_id": 4554821468349, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5721170968765, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "amount": "57.23", "authorization": null, "created_at": "2022-06-15T06:25:42-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 4554821468349, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "receipt": {"paid_amount": "57.23"}, "source_name": "1830279", "status": "success", "test": true, "user_id": null, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}}], "refund_line_items": [{"id": 363131404477, "line_item_id": 11406125564093, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 57.23, "subtotal_set": {"shop_money": {"amount": "57.23", "currency_code": "USD"}, "presentment_money": {"amount": "57.23", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": "59.00", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}}], "duties": []}], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360401954} -{"stream": "orders", "data": {"id": 4147980107965, "admin_graphql_api_id": "gid://shopify/Order/4147980107965", "app_id": 5505221, "browser_ip": null, "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": null, "checkout_token": null, "closed_at": null, "confirmed": true, "contact_email": "airbyte@airbyte.com", "created_at": "2021-09-19T09:08:23-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": null, "device_id": null, "discount_codes": [], "email": "airbyte@airbyte.com", "estimated_taxes": false, "financial_status": "paid", "fulfillment_status": "fulfilled", "gateway": "", "landing_site": null, "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1121", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 121, "order_number": 1121, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/6adf11e07ccb49b280ea4b9f53d64f12/authenticate?key=4cef2ff10ba4d18f31114df33933f81e", "original_total_duties_set": null, "payment_gateway_names": [], "phone": null, "presentment_currency": "USD", "processed_at": "2021-09-19T09:08:23-07:00", "processing_method": "", "reference": null, "referring_site": null, "source_identifier": null, "source_name": "5505221", "source_url": null, "subtotal_price": 27.0, "subtotal_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "tags": "", "tax_lines": [], "taxes_included": false, "test": false, "token": "6adf11e07ccb49b280ea4b9f53d64f12", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 27.0, "total_line_items_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 27.0, "total_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:03:06-07:00", "user_id": null, "customer": {"id": 5565161144509, "email": "airbyte@airbyte.com", "accepts_marketing": false, "created_at": "2021-09-19T08:31:05-07:00", "updated_at": "2021-09-19T09:08:24-07:00", "first_name": null, "last_name": null, "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-09-19T08:31:05-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5565161144509"}, "discount_applications": [], "fulfillments": [{"id": 3693416710333, "admin_graphql_api_id": "gid://shopify/Fulfillment/3693416710333", "created_at": "2021-09-19T09:08:23-07:00", "location_id": 63590301885, "name": "#1121.1", "order_id": 4147980107965, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": "Amazon Logistics US", "tracking_number": "123456", "tracking_numbers": ["123456"], "tracking_url": "https://track.amazon.com/tracking/123456", "tracking_urls": ["https://track.amazon.com/tracking/123456"], "updated_at": "2022-02-22T00:35:47-08:00", "line_items": [{"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}]}], "line_items": [{"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}], "payment_terms": null, "refunds": [{"id": 845032358077, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "created_at": "2022-03-07T02:09:04-08:00", "note": null, "order_id": 4147980107965, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [], "refund_line_items": [{"id": 352716947645, "line_item_id": 10576771317949, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 27.0, "subtotal_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": "27.00", "price_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}}], "duties": []}], "shipping_address": {"first_name": "John", "address1": "San Francisco", "phone": "", "city": "San Francisco", "zip": "91326", "province": "California", "country": "United States", "last_name": "Doe", "address2": "10", "company": "Umbrella LLC", "latitude": 34.2894584, "longitude": -118.5622893, "name": "John Doe", "country_code": "US", "province_code": "CA"}, "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360401957} -{"stream": "orders", "data": {"id": 3935377129661, "admin_graphql_api_id": "gid://shopify/Order/3935377129661", "app_id": 1354745, "browser_ip": "76.14.176.236", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 21670281707709, "checkout_token": "ea03756d615a5f9e752f3c085e8cf9bd", "client_details": {"accept_language": "en-US,en;q=0.9", "browser_height": null, "browser_ip": "76.14.176.236", "browser_width": null, "session_hash": null, "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"}, "closed_at": null, "confirmed": true, "contact_email": null, "created_at": "2021-07-02T00:51:50-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": null, "device_id": null, "discount_codes": [], "email": "", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": null, "gateway": "bogus", "landing_site": null, "landing_site_ref": null, "location_id": 63590301885, "merchant_of_record_app_id": null, "name": "#1001", "note": null, "note_attributes": [], "number": 1, "order_number": 1001, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/16dd6c6e17f562f1f5eee0fefa00b4cb/authenticate?key=931eb302588779d0ab93839d42bf7166", "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "presentment_currency": "USD", "processed_at": "2021-07-02T00:51:49-07:00", "processing_method": "direct", "reference": null, "referring_site": null, "source_identifier": null, "source_name": "shopify_draft_order", "source_url": null, "subtotal_price": 102.0, "subtotal_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "tags": "teest", "tax_lines": [{"price": 17.0, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "channel_liable": false}], "taxes_included": true, "test": true, "token": "16dd6c6e17f562f1f5eee0fefa00b4cb", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 102.0, "total_line_items_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 102.0, "total_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 63, "updated_at": "2023-04-24T10:59:00-07:00", "user_id": 74861019325, "customer": {"id": 5349364105405, "email": null, "accepts_marketing": false, "created_at": "2021-07-02T00:51:46-07:00", "updated_at": "2021-07-02T00:51:46-07:00", "first_name": "Bogus", "last_name": "Gateway", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": null, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-02T00:51:46-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5349364105405"}, "discount_applications": [], "fulfillments": [], "line_items": [{"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": 102.0, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "payment_terms": null, "refunds": [{"id": 829538369725, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "order_id": 3935377129661, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5189894406333, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333", "amount": "102.00", "authorization": null, "created_at": "2021-09-21T05:31:58-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 3935377129661, "parent_id": 4933790040253, "processed_at": "2021-09-21T05:31:58-07:00", "receipt": {"paid_amount": "102.00"}, "source_name": "1830279", "status": "success", "test": true, "user_id": 74861019325, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}}], "refund_line_items": [{"id": 332807864509, "line_item_id": 10130216452285, "location_id": 63590301885, "quantity": 1, "restock_type": "cancel", "subtotal": 102.0, "subtotal_set": {"shop_money": {"amount": "102.00", "currency_code": "USD"}, "presentment_money": {"amount": "102.00", "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": "17.00", "currency_code": "USD"}, "presentment_money": {"amount": "17.00", "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": "102.00", "price_set": {"shop_money": {"amount": "102.00", "currency_code": "USD"}, "presentment_money": {"amount": "102.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": "17.00", "price_set": {"shop_money": {"amount": "17.00", "currency_code": "USD"}, "presentment_money": {"amount": "17.00", "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}}], "duties": []}], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360401959} -{"stream": "orders", "data": {"id": 5033391718589, "admin_graphql_api_id": "gid://shopify/Order/5033391718589", "app_id": 1354745, "browser_ip": "109.162.18.117", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 27403759190205, "checkout_token": "12e5a00614c174069c9a497253ade884", "client_details": {"accept_language": null, "browser_height": null, "browser_ip": "109.162.18.117", "browser_width": null, "session_hash": null, "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"}, "closed_at": "2023-04-24T11:00:10-07:00", "confirmed": true, "contact_email": null, "created_at": "2023-04-24T11:00:08-07:00", "currency": "USD", "current_subtotal_price": 19.0, "current_subtotal_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 19.0, "current_total_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "current_total_tax": 3.17, "current_total_tax_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "", "estimated_taxes": false, "financial_status": "paid", "fulfillment_status": "fulfilled", "gateway": "manual", "landing_site": null, "landing_site_ref": null, "location_id": 63590301885, "merchant_of_record_app_id": null, "name": "#1145", "note": null, "note_attributes": [], "number": 145, "order_number": 1145, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/db7b4531a7a147c0429617e412537ed5/authenticate?key=a5f50035e7d58d12150c43a437f1d51c", "original_total_duties_set": null, "payment_gateway_names": ["manual"], "phone": null, "presentment_currency": "USD", "processed_at": "2023-04-24T11:00:08-07:00", "processing_method": "manual", "reference": "f8c1317e81a62b14be5e801e80aa2590", "referring_site": null, "source_identifier": "f8c1317e81a62b14be5e801e80aa2590", "source_name": "shopify_draft_order", "source_url": null, "subtotal_price": 19.0, "subtotal_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "tags": "", "tax_lines": [{"price": 3.17, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "channel_liable": false}], "taxes_included": true, "test": false, "token": "db7b4531a7a147c0429617e412537ed5", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 19.0, "total_line_items_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 19.0, "total_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 3.17, "total_tax_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 112, "updated_at": "2023-04-24T11:00:10-07:00", "user_id": 74861019325, "discount_applications": [], "fulfillments": [{"id": 4465911431357, "admin_graphql_api_id": "gid://shopify/Fulfillment/4465911431357", "created_at": "2023-04-24T11:00:09-07:00", "location_id": 63590301885, "name": "#1145.1", "order_id": 5033391718589, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2023-04-24T11:00:09-07:00", "line_items": [{"id": 12247585521853, "admin_graphql_api_id": "gid://shopify/LineItem/12247585521853", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 112, "name": "4 Ounce Soy Candle - Test Variant 2", "price": 19.0, "price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796220989629, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "4 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 41561961824445, "variant_inventory_management": "shopify", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "tax_lines": [{"channel_liable": false, "price": 3.17, "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}]}], "line_items": [{"id": 12247585521853, "admin_graphql_api_id": "gid://shopify/LineItem/12247585521853", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 112, "name": "4 Ounce Soy Candle - Test Variant 2", "price": 19.0, "price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796220989629, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "4 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 41561961824445, "variant_inventory_management": "shopify", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "tax_lines": [{"channel_liable": false, "price": 3.17, "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "payment_terms": null, "refunds": [], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360401962} -{"stream": "metafield_orders", "data": {"id": 22347287855293, "namespace": "my_fields", "key": "purchase_order", "value": "trtrtr", "description": null, "owner_id": 4147980107965, "created_at": "2023-04-13T05:09:08-07:00", "updated_at": "2023-04-13T05:09:08-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347287855293", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360404502} -{"stream": "metafield_orders", "data": {"id": 22365749805245, "namespace": "my_fields", "key": "purchase_order", "value": "Test Draft Order Metafield", "description": null, "owner_id": 3935377129661, "created_at": "2023-04-14T03:52:40-07:00", "updated_at": "2023-04-14T03:52:40-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365749805245", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360404958} -{"stream": "draft_orders", "data": {"id": 929019691197, "note": "updated_mon_24.04.2023", "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2022-02-22T03:23:19-08:00", "updated_at": "2023-04-24T07:18:06-07:00", "tax_exempt": false, "completed_at": null, "name": "#D21", "status": "open", "line_items": [{"id": 58117295538365, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 2, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295538365"}, {"id": 58117295571133, "variant_id": null, "product_id": null, "title": "Test Item", "variant_title": null, "sku": null, "vendor": null, "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 1000, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 0.17}], "applied_discount": null, "name": "Test Item", "properties": [], "custom": true, "price": 1.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295571133"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/12893992cc01fc67935ab014fcf9300f", "applied_discount": null, "order_id": null, "shipping_line": {"title": "Test Shipping Fee", "custom": true, "handle": null, "price": 3.0}, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}, {"rate": 0.2, "title": "PDV", "price": 0.17}], "tags": "", "note_attributes": [], "total_price": "42.00", "subtotal_price": "39.00", "total_tax": "6.50", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/929019691197", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360406601} -{"stream": "draft_orders", "data": {"id": 988639920317, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2023-04-24T11:00:01-07:00", "updated_at": "2023-04-24T11:00:09-07:00", "tax_exempt": false, "completed_at": "2023-04-24T11:00:09-07:00", "name": "#D29", "status": "completed", "line_items": [{"id": 58121808019645, "variant_id": 41561961824445, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Test Variant 2", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Test Variant 2", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58121808019645"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/95271a5eeb083c831f76a98fa3712f89", "applied_discount": null, "order_id": 5033391718589, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "tags": "", "note_attributes": [], "total_price": "19.00", "subtotal_price": "19.00", "total_tax": "3.17", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/988639920317", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360406602} -{"stream": "metafield_draft_orders", "data": {"id": 22532787175613, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 929019691197, "created_at": "2023-04-24T07:18:06-07:00", "updated_at": "2023-04-24T07:18:06-07:00", "owner_resource": "draft_order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22532787175613", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360408812} -{"stream": "products", "data": {"id": 6796220989629, "title": "4 Ounce Soy Candle", "body_html": "updated_mon_24.04.2023", "vendor": "Hartmann Group", "product_type": "Baby", "created_at": "2021-06-22T18:09:47-07:00", "handle": "4-ounce-soy-candle", "updated_at": "2023-04-24T11:05:13-07:00", "published_at": "2021-06-22T18:09:47-07:00", "template_suffix": "", "status": "active", "published_scope": "web", "tags": "developer-tools-generator", "admin_graphql_api_id": "gid://shopify/Product/6796220989629", "variants": [{"id": 40090585923773, "product_id": 6796220989629, "title": "Metal", "price": 19.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Metal", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-13T05:00:55-07:00", "taxable": true, "barcode": null, "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 42185200631997, "inventory_quantity": 15, "old_inventory_quantity": 15, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090585923773"}, {"id": 41561955827901, "product_id": 6796220989629, "title": "Test Variant 1", "price": 19.0, "sku": "", "position": 2, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 1", "option2": null, "option3": null, "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:12:40-08:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653682495677, "inventory_quantity": 2, "old_inventory_quantity": 2, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561955827901"}, {"id": 41561961824445, "product_id": 6796220989629, "title": "Test Variant 2", "price": 19.0, "sku": "", "position": 3, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 2", "option2": null, "option3": null, "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2023-04-24T11:00:10-07:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653688524989, "inventory_quantity": 0, "old_inventory_quantity": 0, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561961824445"}], "options": [{"id": 8720178315453, "product_id": 6796220989629, "name": "Title", "position": 1, "values": ["Metal", "Test Variant 1", "Test Variant 2"]}], "images": [{"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029"}], "image": {"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029"}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360410059} -{"stream": "products_graph_ql", "data": {"id": "gid://shopify/Product/6796220989629", "title": "4 Ounce Soy Candle", "updatedAt": "2023-04-24T18:05:13Z", "createdAt": "2021-06-23T01:09:47Z", "publishedAt": "2021-06-23T01:09:47Z", "status": "ACTIVE", "vendor": "Hartmann Group", "productType": "Baby", "tags": ["developer-tools-generator"], "options": [{"id": "gid://shopify/ProductOption/8720178315453", "name": "Title", "position": 1, "values": ["Metal", "Test Variant 1", "Test Variant 2"]}], "handle": "4-ounce-soy-candle", "description": "updated_mon_24.04.2023", "tracksInventory": true, "totalInventory": 17, "totalVariants": 3, "onlineStoreUrl": null, "onlineStorePreviewUrl": "https://airbyte-integration-test.myshopify.com/products/4-ounce-soy-candle", "descriptionHtml": "updated_mon_24.04.2023", "isGiftCard": false, "legacyResourceId": "6796220989629", "mediaCount": 1, "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360411353} -{"stream": "metafield_products", "data": {"id": 22365706944701, "namespace": "custom", "key": "test_product_metafield", "value": "gid://shopify/Product/6796220989629", "description": null, "owner_id": 6796220989629, "created_at": "2023-04-14T03:15:07-07:00", "updated_at": "2023-04-14T03:15:07-07:00", "owner_resource": "product", "type": "product_reference", "admin_graphql_api_id": "gid://shopify/Metafield/22365706944701", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360412793} -{"stream": "metafield_products", "data": {"id": 22365762486461, "namespace": "custom", "key": "product_metafield_test_2", "value": "Test", "description": null, "owner_id": 6796220989629, "created_at": "2023-04-14T03:59:44-07:00", "updated_at": "2023-04-14T03:59:44-07:00", "owner_resource": "product", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365762486461", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360412793} -{"stream": "product_images", "data": {"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360414451} -{"stream": "metafield_product_images", "data": {"id": 22533588451517, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 29301297316029, "created_at": "2023-04-24T10:32:19-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "owner_resource": "product_image", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22533588451517", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360416087} -{"stream": "product_variants", "data": {"id": 40090585923773, "product_id": 6796220989629, "title": "Metal", "price": 19.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Metal", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-13T05:00:55-07:00", "taxable": true, "barcode": null, "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 42185200631997, "inventory_quantity": 15, "old_inventory_quantity": 15, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090585923773", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360417534} -{"stream": "product_variants", "data": {"id": 41561955827901, "product_id": 6796220989629, "title": "Test Variant 1", "price": 19.0, "sku": "", "position": 2, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 1", "option2": null, "option3": null, "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:12:40-08:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653682495677, "inventory_quantity": 2, "old_inventory_quantity": 2, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561955827901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360417534} -{"stream": "product_variants", "data": {"id": 41561961824445, "product_id": 6796220989629, "title": "Test Variant 2", "price": 19.0, "sku": "", "position": 3, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 2", "option2": null, "option3": null, "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2023-04-24T11:00:10-07:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653688524989, "inventory_quantity": 0, "old_inventory_quantity": 0, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561961824445", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360417535} -{"stream": "metafield_product_variants", "data": {"id": 22365715955901, "namespace": "custom", "key": "test_variant_metafield", "value": "Test Varia", "description": null, "owner_id": 41561961824445, "created_at": "2023-04-14T03:24:03-07:00", "updated_at": "2023-04-14T03:24:03-07:00", "owner_resource": "variant", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365715955901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360419846} -{"stream": "collects", "data": {"id": 29229083197629, "collection_id": 270889287869, "product_id": 6796217811133, "created_at": "2021-06-22T18:09:26-07:00", "updated_at": "2021-06-22T18:09:57-07:00", "position": 1, "sort_value": "0000000001", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360423509} -{"stream": "collects", "data": {"id": 29427031703741, "collection_id": 270889287869, "product_id": 6796220989629, "created_at": "2021-07-19T07:01:36-07:00", "updated_at": "2022-03-06T14:12:21-08:00", "position": 2, "sort_value": "0000000002", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360423510} -{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "products_count": 2, "collection_type": "custom", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360425586} -{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "products_count": 2, "collection_type": "custom", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360426202} -{"stream": "metafield_collections", "data": {"id": 21520343367869, "namespace": "my_fields", "key": "discount", "value": "25%", "description": null, "owner_id": 270889287869, "created_at": "2022-10-08T04:44:51-07:00", "updated_at": "2022-10-08T04:44:51-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21520343367869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360428952} -{"stream": "metafield_collections", "data": {"id": 22365707174077, "namespace": "custom", "key": "test_collection_metafield", "value": "Test Collection Metafield", "description": null, "owner_id": 270889287869, "created_at": "2023-04-14T03:15:30-07:00", "updated_at": "2023-04-14T03:15:30-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365707174077", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360428952} -{"stream": "metafield_collections", "data": {"id": 22365707174077, "namespace": "custom", "key": "test_collection_metafield", "value": "Test Collection Metafield", "description": null, "owner_id": 270889287869, "created_at": "2023-04-14T03:15:30-07:00", "updated_at": "2023-04-14T03:15:30-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365707174077", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360429487} -{"stream": "custom_collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360430585} -{"stream": "smart_collections", "data": {"id": 273278566589, "handle": "test-collection", "title": "Test Collection", "updated_at": "2023-04-24T10:55:09-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-07-19T07:02:54-07:00", "sort_order": "best-selling", "template_suffix": "", "disjunctive": false, "rules": ["{'column': 'type', 'relation': 'equals', 'condition': 'Beauty'}"], "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/273278566589", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360431952} -{"stream": "metafield_smart_collections", "data": {"id": 21525604106429, "namespace": "my_fields", "key": "discount", "value": "50%", "description": null, "owner_id": 273278566589, "created_at": "2022-10-12T13:36:55-07:00", "updated_at": "2022-10-12T13:36:55-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21525604106429", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360434031} -{"stream": "metafield_smart_collections", "data": {"id": 22366265573565, "namespace": "my_fields", "key": "new_key", "value": "51%", "description": null, "owner_id": 273278566589, "created_at": "2023-04-14T05:21:58-07:00", "updated_at": "2023-04-14T05:21:58-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22366265573565", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360434032} -{"stream": "order_refunds", "data": {"id": 829538369725, "order_id": 3935377129661, "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "user_id": 74861019325, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "refund_line_items": [{"id": 332807864509, "quantity": 1, "line_item_id": 10130216452285, "location_id": 63590301885, "restock_type": "cancel", "subtotal": 102.0, "total_tax": 17.0, "subtotal_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "variant_id": 40090604011709, "title": "8 Ounce Soy Candle", "quantity": 1, "sku": "", "variant_title": "Wooden", "vendor": "Bosco Inc", "fulfillment_service": "manual", "product_id": 6796229509309, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "8 Ounce Soy Candle - Wooden", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 63, "price": 102.0, "total_discount": 0.0, "fulfillment_status": null, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "tax_lines": [{"title": "PDV", "price": 17.0, "rate": 0.2, "channel_liable": false, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}}]}}], "transactions": [{"id": 5189894406333, "order_id": 3935377129661, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2021-09-21T05:31:58-07:00", "test": true, "authorization": null, "location_id": null, "user_id": 74861019325, "parent_id": 4933790040253, "processed_at": "2021-09-21T05:31:58-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "102.00"}, "amount": "102.00", "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333"}], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360436453} -{"stream": "order_refunds", "data": {"id": 845032358077, "order_id": 4147980107965, "created_at": "2022-03-07T02:09:04-08:00", "note": null, "user_id": 74861019325, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "refund_line_items": [{"id": 352716947645, "quantity": 1, "line_item_id": 10576771317949, "location_id": 63590301885, "restock_type": "return", "subtotal": 27.0, "total_tax": 0.0, "subtotal_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "variant_id": 40090580615357, "title": "Red & Silver Fishing Lure", "quantity": 1, "sku": "", "variant_title": "Plastic", "vendor": "Harris - Hamill", "fulfillment_service": "manual", "product_id": 6796218302653, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "Red & Silver Fishing Lure - Plastic", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 285, "price": 27.0, "total_discount": 0.0, "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "tax_lines": []}}], "transactions": [], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360437009} -{"stream": "order_refunds", "data": {"id": 852809646269, "order_id": 4554821468349, "created_at": "2022-06-15T06:25:43-07:00", "note": null, "user_id": 74861019325, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "refund_line_items": [{"id": 363131404477, "quantity": 1, "line_item_id": 11406125564093, "location_id": 63590301885, "restock_type": "return", "subtotal": 57.23, "total_tax": 0.0, "subtotal_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "variant_id": 40090597884093, "title": "All Black Sneaker Right Foot", "quantity": 1, "sku": "", "variant_title": "ivory", "vendor": "Becker - Moore", "fulfillment_service": "manual", "product_id": 6796226560189, "requires_shipping": false, "taxable": true, "gift_card": false, "name": "All Black Sneaker Right Foot - ivory", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 0, "price": 59.0, "total_discount": 0.0, "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [{"amount": 1.77, "discount_application_index": 0, "amount_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}}], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "tax_lines": []}}], "transactions": [{"id": 5721170968765, "order_id": 4554821468349, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T06:25:42-07:00", "test": true, "authorization": null, "location_id": null, "user_id": null, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": "57.23", "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765"}], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360437594} -{"stream": "order_risks", "data": {"id": 6446736474301, "order_id": 4147980107965, "checkout_id": null, "source": "External", "score": 1.0, "recommendation": "cancel", "display": true, "cause_cancel": true, "message": "This order came from an anonymous proxy", "merchant_message": "This order came from an anonymous proxy", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360440355} -{"stream": "transactions", "data": {"id": 5721110872253, "order_id": 4554821468349, "kind": "sale", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T05:16:52-07:00", "test": true, "authorization": "53433", "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2022-06-15T05:16:52-07:00", "device_id": null, "error_code": null, "source_name": "580111", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721110872253", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360443429} -{"stream": "transactions", "data": {"id": 5721170968765, "order_id": 4554821468349, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T06:25:42-07:00", "test": true, "authorization": null, "location_id": null, "user_id": null, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360443430} -{"stream": "transactions", "data": {"id": 6302086037693, "order_id": 5033391718589, "kind": "sale", "gateway": "manual", "status": "success", "message": "Marked the manual payment as received", "created_at": "2023-04-24T11:00:08-07:00", "test": false, "authorization": null, "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2023-04-24T11:00:08-07:00", "device_id": null, "error_code": null, "source_name": "checkout_one", "receipt": {}, "amount": 19.0, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/6302086037693", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360444945} -{"stream": "tender_transactions", "data": {"id": 4464009117885, "order_id": 5033391718589, "amount": "19.00", "currency": "USD", "user_id": null, "test": false, "processed_at": "2023-04-24T11:00:08-07:00", "remote_reference": null, "payment_details": null, "payment_method": "other", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360446035} -{"stream": "pages", "data": {"id": 83074252989, "title": "Warranty information", "shop_id": 58033176765, "handle": "warranty-information", "body_html": "updated_mon_24.04.2023", "author": "Shopify API", "created_at": "2021-07-08T05:19:00-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "published_at": "2021-07-08T05:19:00-07:00", "template_suffix": null, "admin_graphql_api_id": "gid://shopify/OnlineStorePage/83074252989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360447063} -{"stream": "metafield_pages", "data": {"id": 22534014828733, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 83074252989, "created_at": "2023-04-24T11:08:41-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534014828733", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360449085} -{"stream": "price_rules", "data": {"id": 945000284349, "value_type": "percentage", "value": "-3.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": true, "usage_limit": 10, "starts_at": "2021-07-07T07:22:04-07:00", "ends_at": null, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "1V8Z165KSH5T", "admin_graphql_api_id": "gid://shopify/PriceRule/945000284349", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360450141} -{"stream": "shop", "data": {"id": 58033176765, "name": "airbyte integration test", "email": "sherif@airbyte.io", "domain": "airbyte-integration-test.myshopify.com", "province": "California", "country": "US", "address1": "350 29th Avenue", "zip": "94121", "city": "San Francisco", "source": null, "phone": "8023494963", "latitude": 37.7827286, "longitude": -122.4889911, "primary_locale": "en", "address2": "", "created_at": "2021-06-22T18:00:23-07:00", "updated_at": "2023-04-30T09:02:52-07:00", "country_code": "US", "country_name": "United States", "currency": "USD", "customer_email": "sherif@airbyte.io", "timezone": "(GMT-08:00) America/Los_Angeles", "iana_timezone": "America/Los_Angeles", "shop_owner": "Airbyte Airbyte", "money_format": "${{amount}}", "money_with_currency_format": "${{amount}} USD", "weight_unit": "kg", "province_code": "CA", "taxes_included": true, "auto_configure_tax_inclusivity": null, "tax_shipping": null, "county_taxes": true, "plan_display_name": "Developer Preview", "plan_name": "partner_test", "has_discounts": true, "has_gift_cards": false, "myshopify_domain": "airbyte-integration-test.myshopify.com", "google_apps_domain": null, "google_apps_login_enabled": null, "money_in_emails_format": "${{amount}}", "money_with_currency_in_emails_format": "${{amount}} USD", "eligible_for_payments": true, "requires_extra_payments_agreement": false, "password_enabled": true, "has_storefront": true, "finances": true, "primary_location_id": 63590301885, "cookie_consent_level": "implicit", "visitor_tracking_consent_preference": "allow_all", "checkout_api_supported": true, "multi_location_enabled": true, "setup_required": false, "pre_launch_enabled": false, "enabled_presentment_currencies": ["USD"], "transactional_sms_disabled": false, "marketing_sms_consent_enabled_at_checkout": false, "shop_url": "airbyte-integration-test"}, "emitted_at": 1683750990098} -{"stream": "metafield_shops", "data": {"id": 22534020104381, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 58033176765, "created_at": "2023-04-24T11:12:38-07:00", "updated_at": "2023-04-24T11:12:38-07:00", "owner_resource": "shop", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534020104381", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360452196} -{"stream": "discount_codes", "data": {"id": 11539415990461, "price_rule_id": 945000284349, "code": "updated_mon_24.04.2023", "usage_count": 0, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360454242} -{"stream": "locations", "data": {"id": 63590301885, "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "zip": "30100", "province": null, "country": "UA", "phone": "", "created_at": "2021-06-22T18:00:29-07:00", "updated_at": "2023-02-25T16:20:00-08:00", "country_code": "UA", "country_name": "Ukraine", "province_code": null, "legacy": false, "active": true, "admin_graphql_api_id": "gid://shopify/Location/63590301885", "localized_country_name": "Ukraine", "localized_province_name": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360455259} -{"stream": "metafield_locations", "data": {"id": 21524407255229, "namespace": "inventory", "key": "warehouse_2", "value": "234", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:27-07:00", "updated_at": "2022-10-12T02:21:27-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407255229", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360457210} -{"stream": "metafield_locations", "data": {"id": 21524407681213, "namespace": "inventory", "key": "warehouse_233", "value": "564", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:35-07:00", "updated_at": "2022-10-12T02:21:35-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407681213", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360457210} -{"stream": "inventory_items", "data": {"id": 42185200631997, "sku": "", "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2022-02-22T00:40:19-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/42185200631997", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360458609} -{"stream": "inventory_items", "data": {"id": 43653682495677, "sku": "", "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:09:20-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/43653682495677", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360458609} -{"stream": "inventory_items", "data": {"id": 43653688524989, "sku": "", "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2022-03-06T14:12:20-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/43653688524989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360458610} -{"stream": "inventory_levels", "data": {"inventory_item_id": 42185228845245, "location_id": 63590301885, "available": 24, "updated_at": "2021-06-22T18:11:25-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185228845245", "shop_url": "airbyte-integration-test", "id": "63590301885|42185228845245"}, "emitted_at": 1682360462664} -{"stream": "inventory_levels", "data": {"inventory_item_id": 42186366255293, "location_id": 63590301885, "available": 6, "updated_at": "2021-07-07T06:05:16-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42186366255293", "shop_url": "airbyte-integration-test", "id": "63590301885|42186366255293"}, "emitted_at": 1682360462664} -{"stream": "inventory_levels", "data": {"inventory_item_id": 43653682495677, "location_id": 63590301885, "available": 2, "updated_at": "2022-03-06T14:12:40-08:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=43653682495677", "shop_url": "airbyte-integration-test", "id": "63590301885|43653682495677"}, "emitted_at": 1682360462664} -{"stream": "inventory_levels", "data": {"inventory_item_id": 43653688524989, "location_id": 63590301885, "available": 0, "updated_at": "2023-04-24T11:00:10-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=43653688524989", "shop_url": "airbyte-integration-test", "id": "63590301885|43653688524989"}, "emitted_at": 1682360462664} -{"stream": "inventory_levels", "data": {"inventory_item_id": 44688621142205, "location_id": 63590301885, "available": 50, "updated_at": "2023-01-06T10:35:26-08:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=44688621142205", "shop_url": "airbyte-integration-test", "id": "63590301885|44688621142205"}, "emitted_at": 1682360462664} -{"stream": "inventory_levels", "data": {"inventory_item_id": 44688621174973, "location_id": 63590301885, "available": 40, "updated_at": "2023-01-06T10:35:27-08:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=44688621174973", "shop_url": "airbyte-integration-test", "id": "63590301885|44688621174973"}, "emitted_at": 1682360462665} -{"stream": "inventory_levels", "data": {"inventory_item_id": 44688621207741, "location_id": 63590301885, "available": 34, "updated_at": "2023-01-06T10:35:27-08:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=44688621207741", "shop_url": "airbyte-integration-test", "id": "63590301885|44688621207741"}, "emitted_at": 1682360462665} -{"stream": "inventory_levels", "data": {"inventory_item_id": 44871665713341, "location_id": 63590301885, "available": 0, "updated_at": "2023-04-14T03:29:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=44871665713341", "shop_url": "airbyte-integration-test", "id": "63590301885|44871665713341"}, "emitted_at": 1682360462665} -{"stream": "fulfillment_orders", "data": {"id": 5558588309693, "shop_id": 58033176765, "order_id": 4554821468349, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": null, "line_items": [{"id": 11564232016061, "shop_id": 58033176765, "fulfillment_order_id": 5558588309693, "quantity": 1, "line_item_id": 11406125564093, "inventory_item_id": 42185212592317, "fulfillable_quantity": 0, "variant_id": 40090597884093}], "fulfill_at": "2022-06-15T05:00:00-07:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "delivery_method": {"id": 119732437181, "method_type": "none", "min_delivery_date_time": null, "max_delivery_date_time": null}, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360465016} -{"stream": "fulfillment_orders", "data": {"id": 5985636450493, "shop_id": 58033176765, "order_id": 5033391718589, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": null, "line_items": [{"id": 12407122067645, "shop_id": 58033176765, "fulfillment_order_id": 5985636450493, "quantity": 1, "line_item_id": 12247585521853, "inventory_item_id": 43653688524989, "fulfillable_quantity": 0, "variant_id": 41561961824445}], "fulfill_at": "2023-04-24T11:00:00-07:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "delivery_method": {"id": 442031046845, "method_type": "shipping", "min_delivery_date_time": null, "max_delivery_date_time": null}, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360466626} -{"stream": "fulfillments", "data": {"id": 4075788501181, "order_id": 4554821468349, "status": "success", "created_at": "2022-06-15T05:16:55-07:00", "service": "manual", "updated_at": "2022-06-15T05:16:55-07:00", "tracking_company": null, "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 11406125564093, "variant_id": 40090597884093, "title": "All Black Sneaker Right Foot", "quantity": 1, "sku": "", "variant_title": "ivory", "vendor": "Becker - Moore", "fulfillment_service": "manual", "product_id": 6796226560189, "requires_shipping": false, "taxable": true, "gift_card": false, "name": "All Black Sneaker Right Foot - ivory", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 0, "price": "59.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [{"amount": "1.77", "discount_application_index": 0, "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}}], "origin_location": {"id": 3007664259261, "country_code": "UA", "province_code": "", "name": "airbyte integration test", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100"}, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "duties": [], "tax_lines": [], "fulfillment_line_item_id": 9633709097149}], "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "receipt": {}, "name": "#1136.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360469004} -{"stream": "fulfillments", "data": {"id": 4465911431357, "order_id": 5033391718589, "status": "success", "created_at": "2023-04-24T11:00:09-07:00", "service": "manual", "updated_at": "2023-04-24T11:00:09-07:00", "tracking_company": null, "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 12247585521853, "variant_id": 41561961824445, "title": "4 Ounce Soy Candle", "quantity": 1, "sku": "", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "fulfillment_service": "manual", "product_id": 6796220989629, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "4 Ounce Soy Candle - Test Variant 2", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 112, "price": "19.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "19.00", "currency_code": "USD"}, "presentment_money": {"amount": "19.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [], "origin_location": {"id": 3000230707389, "country_code": "UA", "province_code": "", "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100"}, "admin_graphql_api_id": "gid://shopify/LineItem/12247585521853", "duties": [], "tax_lines": [{"price": 3.17, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": "3.17", "currency_code": "USD"}, "presentment_money": {"amount": "3.17", "currency_code": "USD"}}, "channel_liable": null}], "fulfillment_line_item_id": 10383179514045}], "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "receipt": {}, "name": "#1145.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/4465911431357", "shop_url": "airbyte-integration-test"}, "emitted_at": 1682360470694} +{"stream": "articles", "data": {"id": 558137508029, "title": "My new Article title", "created_at": "2022-10-07T16:09:02-07:00", "body_html": "

    I like articles

    \n

    Yea, I like posting them through REST.

    ", "blog_id": 80417685693, "author": "John Smith", "user_id": null, "published_at": "2011-03-24T08:45:47-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "summary_html": null, "template_suffix": null, "handle": "my-new-article-title", "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558137508029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315555203} +{"stream": "articles", "data": {"id": 558627979453, "title": "Test Blog Post", "created_at": "2023-04-14T03:19:02-07:00", "body_html": "Test Blog Post 1", "blog_id": 80417685693, "author": "Airbyte Airbyte", "user_id": "74861019325", "published_at": null, "updated_at": "2023-04-14T03:19:18-07:00", "summary_html": "", "template_suffix": "", "handle": "test-blog-post", "tags": "Has Been Tagged", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558627979453", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315555204} +{"stream": "blogs", "data": {"id": 80417685693, "handle": "news", "title": "News", "updated_at": "2023-04-14T03:20:20-07:00", "commentable": "no", "feedburner": null, "feedburner_location": null, "created_at": "2021-06-22T18:00:25-07:00", "template_suffix": null, "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreBlog/80417685693", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315557505} +{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "products_count": 2, "collection_type": "custom", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315559587} +{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "products_count": 2, "collection_type": "custom", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315560044} +{"stream": "collects", "data": {"id": 29229083197629, "collection_id": 270889287869, "product_id": 6796217811133, "created_at": "2021-06-22T18:09:26-07:00", "updated_at": "2021-06-22T18:09:57-07:00", "position": 1, "sort_value": "0000000001", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315561131} +{"stream": "collects", "data": {"id": 29427031703741, "collection_id": 270889287869, "product_id": 6796220989629, "created_at": "2021-07-19T07:01:36-07:00", "updated_at": "2022-03-06T14:12:21-08:00", "position": 2, "sort_value": "0000000002", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315561132} +{"stream": "custom_collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315562170} +{"stream": "customers", "data": {"id": 6569096478909, "email": "test@test.com", "accepts_marketing": true, "created_at": "2023-04-13T02:30:04-07:00", "updated_at": "2023-04-24T06:53:48-07:00", "first_name": "New Test", "last_name": "Customer", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": "updated_mon_24.04.2023", "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": "+380639379992", "addresses": [{"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}], "accepts_marketing_updated_at": "2023-04-13T02:30:04-07:00", "marketing_opt_in_level": "single_opt_in", "tax_exemptions": "[]", "email_marketing_consent": {"state": "subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": "2023-04-13T02:30:04-07:00"}, "sms_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null, "consent_collected_from": "SHOPIFY"}, "admin_graphql_api_id": "gid://shopify/Customer/6569096478909", "default_address": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315563441} +{"stream": "customers", "data": {"id": 6676027932861, "email": "marcos@airbyte.io", "accepts_marketing": false, "created_at": "2023-07-11T13:07:45-07:00", "updated_at": "2023-07-11T13:07:45-07:00", "first_name": "MArcos", "last_name": "Millnitz", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": null, "addresses": [{"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true}], "accepts_marketing_updated_at": "2023-07-11T13:07:45-07:00", "marketing_opt_in_level": null, "tax_exemptions": "[]", "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "admin_graphql_api_id": "gid://shopify/Customer/6676027932861", "default_address": {"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315563443} +{"stream": "discount_codes", "data": {"id": 11539415990461, "price_rule_id": 945000284349, "code": "updated_mon_24.04.2023", "usage_count": 0, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315565794} +{"stream": "draft_orders", "data": {"id": 929019691197, "note": "updated_mon_24.04.2023", "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2022-02-22T03:23:19-08:00", "updated_at": "2023-04-24T07:18:06-07:00", "tax_exempt": false, "completed_at": null, "name": "#D21", "status": "open", "line_items": [{"id": 58117295538365, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 2, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295538365"}, {"id": 58117295571133, "variant_id": null, "product_id": null, "title": "Test Item", "variant_title": null, "sku": null, "vendor": null, "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 1000, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 0.17}], "applied_discount": null, "name": "Test Item", "properties": [], "custom": true, "price": 1.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295571133"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/12893992cc01fc67935ab014fcf9300f", "applied_discount": null, "order_id": null, "shipping_line": {"title": "Test Shipping Fee", "custom": true, "handle": null, "price": 3.0}, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}, {"rate": 0.2, "title": "PDV", "price": 0.17}], "tags": "", "note_attributes": [], "total_price": "42.00", "subtotal_price": "39.00", "total_tax": "6.50", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/929019691197", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315566986} +{"stream": "draft_orders", "data": {"id": 988639920317, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2023-04-24T11:00:01-07:00", "updated_at": "2023-04-24T11:00:09-07:00", "tax_exempt": false, "completed_at": "2023-04-24T11:00:09-07:00", "name": "#D29", "status": "completed", "line_items": [{"id": 58121808019645, "variant_id": 41561961824445, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Test Variant 2", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Test Variant 2", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58121808019645"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/95271a5eeb083c831f76a98fa3712f89", "applied_discount": null, "order_id": 5033391718589, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "tags": "", "note_attributes": [], "total_price": "19.00", "subtotal_price": "19.00", "total_tax": "3.17", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/988639920317", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315566988} +{"stream": "draft_orders", "data": {"id": 997801689277, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2023-07-11T12:57:53-07:00", "updated_at": "2023-07-11T12:57:55-07:00", "tax_exempt": false, "completed_at": null, "name": "#D30", "status": "open", "line_items": [{"id": 58159126905021, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58159126905021"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/a98bc7e113733d6faa36c198cf6c7c1a", "applied_discount": null, "order_id": null, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "tags": "", "note_attributes": [], "total_price": "19.00", "subtotal_price": "19.00", "total_tax": "3.17", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/997801689277", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315566988} +{"stream": "fulfillment_orders", "data": {"id": 5558588309693, "shop_id": 58033176765, "order_id": 4554821468349, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": null, "line_items": [{"id": 11564232016061, "shop_id": 58033176765, "fulfillment_order_id": 5558588309693, "quantity": 1, "line_item_id": 11406125564093, "inventory_item_id": 42185212592317, "fulfillable_quantity": 0, "variant_id": 40090597884093}], "fulfill_at": "2022-06-15T05:00:00-07:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "delivery_method": {"id": 119732437181, "method_type": "none", "min_delivery_date_time": null, "max_delivery_date_time": null}, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315569283} +{"stream": "fulfillment_orders", "data": {"id": 5138290671805, "shop_id": 58033176765, "order_id": 4147980107965, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": {"id": 5183946588349, "address1": "San Francisco", "address2": "10", "city": "San Francisco", "company": "Umbrella LLC", "country": "United States", "email": "airbyte@airbyte.com", "first_name": "John", "last_name": "Doe", "phone": "", "province": "California", "zip": "91326"}, "line_items": [{"id": 10713758531773, "shop_id": 58033176765, "fulfillment_order_id": 5138290671805, "quantity": 1, "line_item_id": 10576771317949, "inventory_item_id": 42185195290813, "fulfillable_quantity": 0, "variant_id": 40090580615357}], "fulfill_at": null, "fulfill_by": null, "international_duties": "{'incoterm': None}", "fulfillment_holds": [], "delivery_method": null, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315569894} +{"stream": "fulfillment_orders", "data": {"id": 4919375659197, "shop_id": 58033176765, "order_id": 3935377129661, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": null, "line_items": [{"id": 10251692081341, "shop_id": 58033176765, "fulfillment_order_id": 4919375659197, "quantity": 1, "line_item_id": 10130216452285, "inventory_item_id": 42185218719933, "fulfillable_quantity": 1, "variant_id": 40090604011709}], "fulfill_at": null, "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "delivery_method": null, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315570459} +{"stream": "fulfillments", "data": {"id": 4075788501181, "order_id": 4554821468349, "status": "success", "created_at": "2022-06-15T05:16:55-07:00", "service": "manual", "updated_at": "2022-06-15T05:16:55-07:00", "tracking_company": null, "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 11406125564093, "variant_id": 40090597884093, "title": "All Black Sneaker Right Foot", "quantity": 1, "sku": "", "variant_title": "ivory", "vendor": "Becker - Moore", "fulfillment_service": "manual", "product_id": 6796226560189, "requires_shipping": false, "taxable": true, "gift_card": false, "name": "All Black Sneaker Right Foot - ivory", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 0, "price": "59.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [{"amount": "1.77", "discount_application_index": 0, "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}}], "origin_location": {"id": 3007664259261, "country_code": "UA", "province_code": "", "name": "airbyte integration test", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100"}, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "duties": [], "tax_lines": [], "fulfillment_line_item_id": 9633709097149}], "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "receipt": {}, "name": "#1136.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315573677} +{"stream": "fulfillments", "data": {"id": 3693416710333, "order_id": 4147980107965, "status": "success", "created_at": "2021-09-19T09:08:23-07:00", "service": "manual", "updated_at": "2022-02-22T00:35:47-08:00", "tracking_company": "Amazon Logistics US", "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 10576771317949, "variant_id": 40090580615357, "title": "Red & Silver Fishing Lure", "quantity": 1, "sku": "", "variant_title": "Plastic", "vendor": "Harris - Hamill", "fulfillment_service": "manual", "product_id": 6796218302653, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "Red & Silver Fishing Lure - Plastic", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 285, "price": "27.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [], "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "duties": [], "tax_lines": [], "fulfillment_line_item_id": 8852381401277}], "tracking_number": "123456", "tracking_numbers": ["123456"], "tracking_url": "https://track.amazon.com/tracking/123456", "tracking_urls": ["https://track.amazon.com/tracking/123456"], "receipt": {}, "name": "#1121.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/3693416710333", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315574294} +{"stream": "fulfillments", "data": {"id": 4465911431357, "order_id": 5033391718589, "status": "success", "created_at": "2023-04-24T11:00:09-07:00", "service": "manual", "updated_at": "2023-04-24T11:00:09-07:00", "tracking_company": null, "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 12247585521853, "variant_id": 41561961824445, "title": "4 Ounce Soy Candle", "quantity": 1, "sku": "", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "fulfillment_service": "manual", "product_id": 6796220989629, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "4 Ounce Soy Candle - Test Variant 2", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 112, "price": "19.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "19.00", "currency_code": "USD"}, "presentment_money": {"amount": "19.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [], "origin_location": {"id": 3000230707389, "country_code": "UA", "province_code": "", "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100"}, "admin_graphql_api_id": "gid://shopify/LineItem/12247585521853", "duties": [], "tax_lines": [{"price": 3.17, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": "3.17", "currency_code": "USD"}, "presentment_money": {"amount": "3.17", "currency_code": "USD"}}, "channel_liable": null}], "fulfillment_line_item_id": 10383179514045}], "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "receipt": {}, "name": "#1145.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/4465911431357", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315575461} +{"stream": "inventory_items", "data": {"id": 42185200631997, "sku": "", "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2022-02-22T00:40:19-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/42185200631997", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315577320} +{"stream": "inventory_items", "data": {"id": 43653682495677, "sku": "", "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:09:20-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/43653682495677", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315577322} +{"stream": "inventory_items", "data": {"id": 43653688524989, "sku": "", "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2022-03-06T14:12:20-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/43653688524989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315577322} +{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194635453, "location_id": 63590301885, "available": 49, "updated_at": "2022-02-27T23:44:30-08:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194635453", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194635453"}, "emitted_at": 1690315579821} +{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194668221, "location_id": 63590301885, "available": 12, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194668221", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194668221"}, "emitted_at": 1690315579822} +{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194700989, "location_id": 63590301885, "available": 3, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194700989", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194700989"}, "emitted_at": 1690315579822} +{"stream": "locations", "data": {"id": 63590301885, "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "zip": "30100", "province": null, "country": "UA", "phone": "", "created_at": "2021-06-22T18:00:29-07:00", "updated_at": "2023-02-25T16:20:00-08:00", "country_code": "UA", "country_name": "Ukraine", "province_code": null, "legacy": false, "active": true, "admin_graphql_api_id": "gid://shopify/Location/63590301885", "localized_country_name": "Ukraine", "localized_province_name": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315582838} +{"stream": "metafield_articles", "data": {"id": 21519818162365, "namespace": "global", "key": "new", "value": "newvalue", "description": null, "owner_id": 558137508029, "created_at": "2022-10-07T16:09:02-07:00", "updated_at": "2022-10-07T16:09:02-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519818162365", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315584977} +{"stream": "metafield_articles", "data": {"id": 22365709992125, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Article Metafield", "description": null, "owner_id": 558137508029, "created_at": "2023-04-14T03:18:26-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365709992125", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315584978} +{"stream": "metafield_articles", "data": {"id": 22365710352573, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Blog Post Metafiled", "description": null, "owner_id": 558627979453, "created_at": "2023-04-14T03:19:18-07:00", "updated_at": "2023-04-14T03:19:18-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710352573", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315585483} +{"stream": "metafield_blogs", "data": {"id": 21519428255933, "namespace": "some_fields", "key": "sponsor", "value": "Shopify", "description": null, "owner_id": 80417685693, "created_at": "2022-10-07T06:05:23-07:00", "updated_at": "2022-10-07T06:05:23-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519428255933", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315587865} +{"stream": "metafield_blogs", "data": {"id": 22365710745789, "namespace": "custom", "key": "test_blog_metafield", "value": "Test Blog Metafield", "description": null, "owner_id": 80417685693, "created_at": "2023-04-14T03:20:20-07:00", "updated_at": "2023-04-14T03:20:20-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710745789", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315587866} +{"stream": "metafield_collections", "data": {"id": 21520343367869, "namespace": "my_fields", "key": "discount", "value": "25%", "description": null, "owner_id": 270889287869, "created_at": "2022-10-08T04:44:51-07:00", "updated_at": "2022-10-08T04:44:51-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21520343367869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315590398} +{"stream": "metafield_collections", "data": {"id": 22365707174077, "namespace": "custom", "key": "test_collection_metafield", "value": "Test Collection Metafield", "description": null, "owner_id": 270889287869, "created_at": "2023-04-14T03:15:30-07:00", "updated_at": "2023-04-14T03:15:30-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365707174077", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315590400} +{"stream": "metafield_collections", "data": {"id": 21520343367869, "namespace": "my_fields", "key": "discount", "value": "25%", "description": null, "owner_id": 270889287869, "created_at": "2022-10-08T04:44:51-07:00", "updated_at": "2022-10-08T04:44:51-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21520343367869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315590879} +{"stream": "metafield_customers", "data": {"id": 22346893361341, "namespace": "custom", "key": "test_definition_list_1", "value": "Teste\n", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893361341", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315593181} +{"stream": "metafield_customers", "data": {"id": 22346893394109, "namespace": "custom", "key": "test_definition", "value": "Taster", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893394109", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315593182} +{"stream": "metafield_draft_orders", "data": {"id": 22532787175613, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 929019691197, "created_at": "2023-04-24T07:18:06-07:00", "updated_at": "2023-04-24T07:18:06-07:00", "owner_resource": "draft_order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22532787175613", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315595976} +{"stream": "metafield_locations", "data": {"id": 21524407255229, "namespace": "inventory", "key": "warehouse_2", "value": "234", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:27-07:00", "updated_at": "2022-10-12T02:21:27-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407255229", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315599010} +{"stream": "metafield_locations", "data": {"id": 21524407681213, "namespace": "inventory", "key": "warehouse_233", "value": "564", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:35-07:00", "updated_at": "2022-10-12T02:21:35-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407681213", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315599012} +{"stream": "metafield_orders", "data": {"id": 22347287855293, "namespace": "my_fields", "key": "purchase_order", "value": "trtrtr", "description": null, "owner_id": 4147980107965, "created_at": "2023-04-13T05:09:08-07:00", "updated_at": "2023-04-13T05:09:08-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347287855293", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315601837} +{"stream": "metafield_orders", "data": {"id": 22365749805245, "namespace": "my_fields", "key": "purchase_order", "value": "Test Draft Order Metafield", "description": null, "owner_id": 3935377129661, "created_at": "2023-04-14T03:52:40-07:00", "updated_at": "2023-04-14T03:52:40-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365749805245", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315602352} +{"stream": "metafield_pages", "data": {"id": 22534014828733, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 83074252989, "created_at": "2023-04-24T11:08:41-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534014828733", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315605006} +{"stream": "metafield_product_images", "data": {"id": 22533588451517, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 29301297316029, "created_at": "2023-04-24T10:32:19-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "owner_resource": "product_image", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22533588451517", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315606428} +{"stream": "metafield_products", "data": {"id": 22365706944701, "namespace": "custom", "key": "test_product_metafield", "value": "gid://shopify/Product/6796220989629", "description": null, "owner_id": 6796220989629, "created_at": "2023-04-14T03:15:07-07:00", "updated_at": "2023-04-14T03:15:07-07:00", "owner_resource": "product", "type": "product_reference", "admin_graphql_api_id": "gid://shopify/Metafield/22365706944701", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315607933} +{"stream": "metafield_products", "data": {"id": 22365762486461, "namespace": "custom", "key": "product_metafield_test_2", "value": "Test", "description": null, "owner_id": 6796220989629, "created_at": "2023-04-14T03:59:44-07:00", "updated_at": "2023-04-14T03:59:44-07:00", "owner_resource": "product", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365762486461", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315607934} +{"stream": "metafield_product_variants", "data": {"id": 22365715955901, "namespace": "custom", "key": "test_variant_metafield", "value": "Test Varia", "description": null, "owner_id": 41561961824445, "created_at": "2023-04-14T03:24:03-07:00", "updated_at": "2023-04-14T03:24:03-07:00", "owner_resource": "variant", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365715955901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315610544} +{"stream": "metafield_shops", "data": {"id": 22534020104381, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 58033176765, "created_at": "2023-04-24T11:12:38-07:00", "updated_at": "2023-04-24T11:12:38-07:00", "owner_resource": "shop", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534020104381", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315611668} +{"stream": "metafield_smart_collections", "data": {"id": 21525604106429, "namespace": "my_fields", "key": "discount", "value": "50%", "description": null, "owner_id": 273278566589, "created_at": "2022-10-12T13:36:55-07:00", "updated_at": "2022-10-12T13:36:55-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21525604106429", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315613924} +{"stream": "metafield_smart_collections", "data": {"id": 22366265573565, "namespace": "my_fields", "key": "new_key", "value": "51%", "description": null, "owner_id": 273278566589, "created_at": "2023-04-14T05:21:58-07:00", "updated_at": "2023-04-14T05:21:58-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22366265573565", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315613926} +{"stream": "order_refunds", "data": {"id": 829538369725, "order_id": 3935377129661, "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "user_id": 74861019325, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "refund_line_items": [{"id": 332807864509, "quantity": 1, "line_item_id": 10130216452285, "location_id": 63590301885, "restock_type": "cancel", "subtotal": 102.0, "total_tax": 17.0, "subtotal_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "variant_id": 40090604011709, "title": "8 Ounce Soy Candle", "quantity": 1, "sku": "", "variant_title": "Wooden", "vendor": "Bosco Inc", "fulfillment_service": "manual", "product_id": 6796229509309, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "8 Ounce Soy Candle - Wooden", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 63, "price": 102.0, "total_discount": 0.0, "fulfillment_status": null, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "tax_lines": [{"title": "PDV", "price": 17.0, "rate": 0.2, "channel_liable": false, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}}]}}], "transactions": [{"id": 5189894406333, "order_id": 3935377129661, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2021-09-21T05:31:58-07:00", "test": true, "authorization": null, "location_id": null, "user_id": 74861019325, "parent_id": 4933790040253, "processed_at": "2021-09-21T05:31:58-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "102.00"}, "amount": "102.00", "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333"}], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315616894} +{"stream": "order_refunds", "data": {"id": 845032358077, "order_id": 4147980107965, "created_at": "2022-03-07T02:09:04-08:00", "note": null, "user_id": 74861019325, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "refund_line_items": [{"id": 352716947645, "quantity": 1, "line_item_id": 10576771317949, "location_id": 63590301885, "restock_type": "return", "subtotal": 27.0, "total_tax": 0.0, "subtotal_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "variant_id": 40090580615357, "title": "Red & Silver Fishing Lure", "quantity": 1, "sku": "", "variant_title": "Plastic", "vendor": "Harris - Hamill", "fulfillment_service": "manual", "product_id": 6796218302653, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "Red & Silver Fishing Lure - Plastic", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 285, "price": 27.0, "total_discount": 0.0, "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "tax_lines": []}}], "transactions": [], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315617504} +{"stream": "order_refunds", "data": {"id": 852809646269, "order_id": 4554821468349, "created_at": "2022-06-15T06:25:43-07:00", "note": null, "user_id": 74861019325, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "refund_line_items": [{"id": 363131404477, "quantity": 1, "line_item_id": 11406125564093, "location_id": 63590301885, "restock_type": "return", "subtotal": 57.23, "total_tax": 0.0, "subtotal_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "variant_id": 40090597884093, "title": "All Black Sneaker Right Foot", "quantity": 1, "sku": "", "variant_title": "ivory", "vendor": "Becker - Moore", "fulfillment_service": "manual", "product_id": 6796226560189, "requires_shipping": false, "taxable": true, "gift_card": false, "name": "All Black Sneaker Right Foot - ivory", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 0, "price": 59.0, "total_discount": 0.0, "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [{"amount": 1.77, "discount_application_index": 0, "amount_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}}], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "tax_lines": []}}], "transactions": [{"id": 5721170968765, "order_id": 4554821468349, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T06:25:42-07:00", "test": true, "authorization": null, "location_id": null, "user_id": null, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": "57.23", "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765"}], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315618226} +{"stream": "order_risks", "data": {"id": 6446736474301, "order_id": 4147980107965, "checkout_id": null, "source": "External", "score": 1.0, "recommendation": "cancel", "display": true, "cause_cancel": true, "message": "This order came from an anonymous proxy", "merchant_message": "This order came from an anonymous proxy", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315621059} +{"stream": "orders", "data": {"id": 4554821468349, "admin_graphql_api_id": "gid://shopify/Order/4554821468349", "app_id": 580111, "browser_ip": "176.113.167.23", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 25048437719229, "checkout_token": "cf5d16a0a0688905bd551c6dec591506", "client_details": {"accept_language": "en-US,en;q=0.9,uk;q=0.8", "browser_height": 754, "browser_ip": "176.113.167.23", "browser_width": 1519, "session_hash": null, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53"}, "closed_at": "2022-06-15T06:25:43-07:00", "confirmed": true, "contact_email": "integration-test@airbyte.io", "created_at": "2022-06-15T05:16:53-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "integration-test@airbyte.io", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": "fulfilled", "gateway": "bogus", "landing_site": "/wallets/checkouts.json", "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1136", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 136, "order_number": 1136, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/e4f98630ea44a884e33e700203ce2130/authenticate?key=edf087d6ae55a4541bf1375432f6a4b8", "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "presentment_currency": "USD", "processed_at": "2022-06-15T05:16:53-07:00", "processing_method": "direct", "reference": null, "referring_site": "https://airbyte-integration-test.myshopify.com/products/all-black-sneaker-right-foot", "source_identifier": null, "source_name": "web", "source_url": null, "subtotal_price": 57.23, "subtotal_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "tags": "Refund", "tax_lines": [], "taxes_included": true, "test": true, "token": "e4f98630ea44a884e33e700203ce2130", "total_discounts": 1.77, "total_discounts_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}, "total_line_items_price": 59.0, "total_line_items_price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 57.23, "total_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:00:37-07:00", "user_id": null, "billing_address": {"first_name": "Iryna", "address1": "2261 Market Street", "phone": null, "city": "San Francisco", "zip": "94114", "province": "California", "country": "United States", "last_name": "Grankova", "address2": "4381", "company": null, "latitude": 37.7647751, "longitude": -122.4320369, "name": "Iryna Grankova", "country_code": "US", "province_code": "CA"}, "customer": {"id": 5362027233469, "email": "integration-test@airbyte.io", "accepts_marketing": false, "created_at": "2021-07-08T05:41:47-07:00", "updated_at": "2022-06-22T03:50:13-07:00", "first_name": "Airbyte", "last_name": "Team", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-08T05:41:47-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5362027233469", "default_address": {"id": 7492260823229, "customer_id": 5362027233469, "first_name": "Airbyte", "last_name": "Team", "company": null, "address1": "2261 Market Street", "address2": "4381", "city": "San Francisco", "province": "California", "country": "United States", "zip": "94114", "phone": null, "name": "Airbyte Team", "province_code": "CA", "country_code": "US", "country_name": "United States", "default": true}}, "discount_applications": [{"target_type": "line_item", "type": "automatic", "value": "3.0", "value_type": "percentage", "allocation_method": "across", "target_selection": "all", "title": "eeeee"}], "fulfillments": [{"id": 4075788501181, "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "created_at": "2022-06-15T05:16:55-07:00", "location_id": 63590301885, "name": "#1136.1", "order_id": 4554821468349, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2022-06-15T05:16:55-07:00", "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}]}], "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}], "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "payment_terms": null, "refunds": [{"id": 852809646269, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "created_at": "2022-06-15T06:25:43-07:00", "note": null, "order_id": 4554821468349, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5721170968765, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "amount": "57.23", "authorization": null, "created_at": "2022-06-15T06:25:42-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 4554821468349, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "receipt": {"paid_amount": "57.23"}, "source_name": "1830279", "status": "success", "test": true, "user_id": null, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}}], "refund_line_items": [{"id": 363131404477, "line_item_id": 11406125564093, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 57.23, "subtotal_set": {"shop_money": {"amount": "57.23", "currency_code": "USD"}, "presentment_money": {"amount": "57.23", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": "59.00", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}}], "duties": []}], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315623322} +{"stream": "orders", "data": {"id": 4147980107965, "admin_graphql_api_id": "gid://shopify/Order/4147980107965", "app_id": 5505221, "browser_ip": null, "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": null, "checkout_token": null, "closed_at": null, "confirmed": true, "contact_email": "airbyte@airbyte.com", "created_at": "2021-09-19T09:08:23-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": null, "device_id": null, "discount_codes": [], "email": "airbyte@airbyte.com", "estimated_taxes": false, "financial_status": "paid", "fulfillment_status": "fulfilled", "gateway": "", "landing_site": null, "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1121", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 121, "order_number": 1121, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/6adf11e07ccb49b280ea4b9f53d64f12/authenticate?key=4cef2ff10ba4d18f31114df33933f81e", "original_total_duties_set": null, "payment_gateway_names": [], "phone": null, "presentment_currency": "USD", "processed_at": "2021-09-19T09:08:23-07:00", "processing_method": "", "reference": null, "referring_site": null, "source_identifier": null, "source_name": "5505221", "source_url": null, "subtotal_price": 27.0, "subtotal_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "tags": "", "tax_lines": [], "taxes_included": false, "test": false, "token": "6adf11e07ccb49b280ea4b9f53d64f12", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 27.0, "total_line_items_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 27.0, "total_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:03:06-07:00", "user_id": null, "customer": {"id": 5565161144509, "email": "airbyte@airbyte.com", "accepts_marketing": false, "created_at": "2021-09-19T08:31:05-07:00", "updated_at": "2021-09-19T09:08:24-07:00", "first_name": null, "last_name": null, "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-09-19T08:31:05-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5565161144509"}, "discount_applications": [], "fulfillments": [{"id": 3693416710333, "admin_graphql_api_id": "gid://shopify/Fulfillment/3693416710333", "created_at": "2021-09-19T09:08:23-07:00", "location_id": 63590301885, "name": "#1121.1", "order_id": 4147980107965, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": "Amazon Logistics US", "tracking_number": "123456", "tracking_numbers": ["123456"], "tracking_url": "https://track.amazon.com/tracking/123456", "tracking_urls": ["https://track.amazon.com/tracking/123456"], "updated_at": "2022-02-22T00:35:47-08:00", "line_items": [{"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}]}], "line_items": [{"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}], "payment_terms": null, "refunds": [{"id": 845032358077, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "created_at": "2022-03-07T02:09:04-08:00", "note": null, "order_id": 4147980107965, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [], "refund_line_items": [{"id": 352716947645, "line_item_id": 10576771317949, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 27.0, "subtotal_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": "27.00", "price_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}}], "duties": []}], "shipping_address": {"first_name": "John", "address1": "San Francisco", "phone": "", "city": "San Francisco", "zip": "91326", "province": "California", "country": "United States", "last_name": "Doe", "address2": "10", "company": "Umbrella LLC", "latitude": 34.2894584, "longitude": -118.5622893, "name": "John Doe", "country_code": "US", "province_code": "CA"}, "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315623326} +{"stream": "orders", "data": {"id": 3935377129661, "admin_graphql_api_id": "gid://shopify/Order/3935377129661", "app_id": 1354745, "browser_ip": "76.14.176.236", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 21670281707709, "checkout_token": "ea03756d615a5f9e752f3c085e8cf9bd", "client_details": {"accept_language": "en-US,en;q=0.9", "browser_height": null, "browser_ip": "76.14.176.236", "browser_width": null, "session_hash": null, "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"}, "closed_at": null, "confirmed": true, "contact_email": null, "created_at": "2021-07-02T00:51:50-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": null, "device_id": null, "discount_codes": [], "email": "", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": null, "gateway": "bogus", "landing_site": null, "landing_site_ref": null, "location_id": 63590301885, "merchant_of_record_app_id": null, "name": "#1001", "note": null, "note_attributes": [], "number": 1, "order_number": 1001, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/16dd6c6e17f562f1f5eee0fefa00b4cb/authenticate?key=931eb302588779d0ab93839d42bf7166", "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "presentment_currency": "USD", "processed_at": "2021-07-02T00:51:49-07:00", "processing_method": "direct", "reference": null, "referring_site": null, "source_identifier": null, "source_name": "shopify_draft_order", "source_url": null, "subtotal_price": 102.0, "subtotal_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "tags": "teest", "tax_lines": [{"price": 17.0, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "channel_liable": false}], "taxes_included": true, "test": true, "token": "16dd6c6e17f562f1f5eee0fefa00b4cb", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 102.0, "total_line_items_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 102.0, "total_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 63, "updated_at": "2023-04-24T10:59:00-07:00", "user_id": 74861019325, "customer": {"id": 5349364105405, "email": null, "accepts_marketing": false, "created_at": "2021-07-02T00:51:46-07:00", "updated_at": "2021-07-02T00:51:46-07:00", "first_name": "Bogus", "last_name": "Gateway", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": null, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-02T00:51:46-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5349364105405"}, "discount_applications": [], "fulfillments": [], "line_items": [{"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": 102.0, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "payment_terms": null, "refunds": [{"id": 829538369725, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "order_id": 3935377129661, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5189894406333, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333", "amount": "102.00", "authorization": null, "created_at": "2021-09-21T05:31:58-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 3935377129661, "parent_id": 4933790040253, "processed_at": "2021-09-21T05:31:58-07:00", "receipt": {"paid_amount": "102.00"}, "source_name": "1830279", "status": "success", "test": true, "user_id": 74861019325, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}}], "refund_line_items": [{"id": 332807864509, "line_item_id": 10130216452285, "location_id": 63590301885, "quantity": 1, "restock_type": "cancel", "subtotal": 102.0, "subtotal_set": {"shop_money": {"amount": "102.00", "currency_code": "USD"}, "presentment_money": {"amount": "102.00", "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": "17.00", "currency_code": "USD"}, "presentment_money": {"amount": "17.00", "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": "102.00", "price_set": {"shop_money": {"amount": "102.00", "currency_code": "USD"}, "presentment_money": {"amount": "102.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": "17.00", "price_set": {"shop_money": {"amount": "17.00", "currency_code": "USD"}, "presentment_money": {"amount": "17.00", "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}}], "duties": []}], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315623329} +{"stream": "pages", "data": {"id": 83074252989, "title": "Warranty information", "shop_id": 58033176765, "handle": "warranty-information", "body_html": "updated_mon_24.04.2023", "author": "Shopify API", "created_at": "2021-07-08T05:19:00-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "published_at": "2021-07-08T05:19:00-07:00", "template_suffix": null, "admin_graphql_api_id": "gid://shopify/OnlineStorePage/83074252989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315624471} +{"stream": "price_rules", "data": {"id": 945000284349, "value_type": "percentage", "value": "-3.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": true, "usage_limit": 10, "starts_at": "2021-07-07T07:22:04-07:00", "ends_at": null, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "1V8Z165KSH5T", "admin_graphql_api_id": "gid://shopify/PriceRule/945000284349", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315625596} +{"stream": "product_images", "data": {"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315627231} +{"stream": "products", "data": {"id": 6796220989629, "title": "4 Ounce Soy Candle", "body_html": "updated_mon_24.04.2023", "vendor": "Hartmann Group", "product_type": "Baby", "created_at": "2021-06-22T18:09:47-07:00", "handle": "4-ounce-soy-candle", "updated_at": "2023-04-24T11:05:13-07:00", "published_at": "2021-06-22T18:09:47-07:00", "template_suffix": "", "status": "active", "published_scope": "web", "tags": "developer-tools-generator", "admin_graphql_api_id": "gid://shopify/Product/6796220989629", "variants": [{"id": 40090585923773, "product_id": 6796220989629, "title": "Metal", "price": 19.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Metal", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-13T05:00:55-07:00", "taxable": true, "barcode": null, "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 42185200631997, "inventory_quantity": 15, "old_inventory_quantity": 15, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090585923773"}, {"id": 41561955827901, "product_id": 6796220989629, "title": "Test Variant 1", "price": 19.0, "sku": "", "position": 2, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 1", "option2": null, "option3": null, "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:12:40-08:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653682495677, "inventory_quantity": 2, "old_inventory_quantity": 2, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561955827901"}, {"id": 41561961824445, "product_id": 6796220989629, "title": "Test Variant 2", "price": 19.0, "sku": "", "position": 3, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 2", "option2": null, "option3": null, "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2023-04-24T11:00:10-07:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653688524989, "inventory_quantity": 0, "old_inventory_quantity": 0, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561961824445"}], "options": [{"id": 8720178315453, "product_id": 6796220989629, "name": "Title", "position": 1, "values": ["Metal", "Test Variant 1", "Test Variant 2"]}], "images": [{"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029"}], "image": {"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029"}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315627645} +{"stream": "products_graph_ql", "data": {"id": "gid://shopify/Product/6796220989629", "title": "4 Ounce Soy Candle", "updatedAt": "2023-04-24T18:05:13Z", "createdAt": "2021-06-23T01:09:47Z", "publishedAt": "2021-06-23T01:09:47Z", "status": "ACTIVE", "vendor": "Hartmann Group", "productType": "Baby", "tags": ["developer-tools-generator"], "options": [{"id": "gid://shopify/ProductOption/8720178315453", "name": "Title", "position": 1, "values": ["Metal", "Test Variant 1", "Test Variant 2"]}], "handle": "4-ounce-soy-candle", "description": "updated_mon_24.04.2023", "tracksInventory": true, "totalInventory": 17, "totalVariants": 3, "onlineStoreUrl": null, "onlineStorePreviewUrl": "https://airbyte-integration-test.myshopify.com/products/4-ounce-soy-candle", "descriptionHtml": "updated_mon_24.04.2023", "isGiftCard": false, "legacyResourceId": "6796220989629", "mediaCount": 1, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315628830} +{"stream": "product_variants", "data": {"id": 40090585923773, "product_id": 6796220989629, "title": "Metal", "price": 19.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Metal", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-13T05:00:55-07:00", "taxable": true, "barcode": null, "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 42185200631997, "inventory_quantity": 15, "old_inventory_quantity": 15, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090585923773", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315630410} +{"stream": "product_variants", "data": {"id": 41561955827901, "product_id": 6796220989629, "title": "Test Variant 1", "price": 19.0, "sku": "", "position": 2, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 1", "option2": null, "option3": null, "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:12:40-08:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653682495677, "inventory_quantity": 2, "old_inventory_quantity": 2, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561955827901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315630418} +{"stream": "product_variants", "data": {"id": 41561961824445, "product_id": 6796220989629, "title": "Test Variant 2", "price": 19.0, "sku": "", "position": 3, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 2", "option2": null, "option3": null, "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2023-04-24T11:00:10-07:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653688524989, "inventory_quantity": 0, "old_inventory_quantity": 0, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561961824445", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315630419} +{"stream": "shop", "data": {"id": 58033176765, "name": "airbyte integration test", "email": "sherif@airbyte.io", "domain": "airbyte-integration-test.myshopify.com", "province": "California", "country": "US", "address1": "350 29th Avenue", "zip": "94121", "city": "San Francisco", "source": null, "phone": "8023494963", "latitude": 37.7827286, "longitude": -122.4889911, "primary_locale": "en", "address2": "", "created_at": "2021-06-22T18:00:23-07:00", "updated_at": "2023-04-30T09:02:52-07:00", "country_code": "US", "country_name": "United States", "currency": "USD", "customer_email": "sherif@airbyte.io", "timezone": "(GMT-08:00) America/Los_Angeles", "iana_timezone": "America/Los_Angeles", "shop_owner": "Airbyte Airbyte", "money_format": "${{amount}}", "money_with_currency_format": "${{amount}} USD", "weight_unit": "kg", "province_code": "CA", "taxes_included": true, "auto_configure_tax_inclusivity": null, "tax_shipping": null, "county_taxes": true, "plan_display_name": "Developer Preview", "plan_name": "partner_test", "has_discounts": true, "has_gift_cards": false, "myshopify_domain": "airbyte-integration-test.myshopify.com", "google_apps_domain": null, "google_apps_login_enabled": null, "money_in_emails_format": "${{amount}}", "money_with_currency_in_emails_format": "${{amount}} USD", "eligible_for_payments": true, "requires_extra_payments_agreement": false, "password_enabled": true, "has_storefront": true, "finances": true, "primary_location_id": 63590301885, "checkout_api_supported": true, "multi_location_enabled": true, "setup_required": false, "pre_launch_enabled": false, "enabled_presentment_currencies": ["USD"], "transactional_sms_disabled": false, "marketing_sms_consent_enabled_at_checkout": false, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315631635} +{"stream": "smart_collections", "data": {"id": 273278566589, "handle": "test-collection", "title": "Test Collection", "updated_at": "2023-04-24T10:55:09-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-07-19T07:02:54-07:00", "sort_order": "best-selling", "template_suffix": "", "disjunctive": false, "rules": ["{'column': 'type', 'relation': 'equals', 'condition': 'Beauty'}"], "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/273278566589", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315632721} +{"stream": "tender_transactions", "data": {"id": 4464009117885, "order_id": 5033391718589, "amount": "19.00", "currency": "USD", "user_id": null, "test": false, "processed_at": "2023-04-24T11:00:08-07:00", "remote_reference": null, "payment_details": null, "payment_method": "other", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315633888} +{"stream": "transactions", "data": {"id": 5721110872253, "order_id": 4554821468349, "kind": "sale", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T05:16:52-07:00", "test": true, "authorization": "53433", "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2022-06-15T05:16:52-07:00", "device_id": null, "error_code": null, "source_name": "580111", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721110872253", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315636101} +{"stream": "transactions", "data": {"id": 5721170968765, "order_id": 4554821468349, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T06:25:42-07:00", "test": true, "authorization": null, "location_id": null, "user_id": null, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315636102} +{"stream": "transactions", "data": {"id": 4933790040253, "order_id": 3935377129661, "kind": "sale", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2021-07-02T00:51:49-07:00", "test": true, "authorization": "53433", "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2021-07-02T00:51:49-07:00", "device_id": null, "error_code": null, "source_name": "shopify_draft_order", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "102.00"}, "amount": 102.0, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/4933790040253", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315637157} +{"stream": "customer_address", "data": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315642565} +{"stream": "customer_address", "data": {"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315643053} +{"stream": "countries", "data": {"id": 417014841533, "name": "Rest of World", "code": "*", "tax_name": "Tax", "tax": 0.0, "provinces": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315644180} +{"stream": "countries", "data": {"id": 417014808765, "name": "Ukraine", "code": "UA", "tax_name": "PDV", "tax": 0.2, "provinces": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315644181} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index 4d523e336a6f..652fa38ae934 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -1,183 +1,183 @@ { "customers": { - "updated_at": "2022-10-11T08:40:51-07:00" + "updated_at": "2022-10-11T08:40:51-07:00" }, "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" + "updated_at": "2022-10-12T11:15:27-07:00" }, "draft_orders": { - "updated_at": "2022-10-08T05:07:29-07:00" + "updated_at": "2022-10-08T05:07:29-07:00" }, "products": { - "updated_at": "2023-03-20T06:08:51-07:00" + "updated_at": "2023-03-20T06:08:51-07:00" }, "products_graph_ql": { - "updatedAt": "2023-03-20T13:08:51Z" + "updatedAt": "2023-03-20T13:08:51Z" }, "abandoned_checkouts": {}, "metafields": { - "updated_at": "2022-05-30T23:42:02-07:00" + "updated_at": "2022-05-30T23:42:02-07:00" }, "collects": { - "id": 29427031703741 + "id": 29427031703741 }, "custom_collections": { - "updated_at": "2023-03-20T06:08:12-07:00" + "updated_at": "2023-03-20T06:08:12-07:00" }, "order_refunds": { - "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" - }, - "created_at": "2022-10-10T06:21:53-07:00" + "orders": { + "updated_at": "2022-10-12T11:15:27-07:00" + }, + "created_at": "2022-10-10T06:21:53-07:00" }, "order_risks": { - "orders": { - "updated_at": "2022-03-07T02:09:04-08:00" - }, - "id": 6446736474301 + "orders": { + "updated_at": "2022-03-07T02:09:04-08:00" + }, + "id": 6446736474301 }, "transactions": { - "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" - }, - "created_at": "2022-10-10T06:21:52-07:00" + "orders": { + "updated_at": "2022-10-12T11:15:27-07:00" + }, + "created_at": "2022-10-10T06:21:52-07:00" }, "tender_transactions": { - "processed_at": "2022-10-10T06:21:52-07:00" + "processed_at": "2022-10-10T06:21:52-07:00" }, "pages": { - "updated_at": "2022-10-08T08:07:00-07:00" + "updated_at": "2022-10-08T08:07:00-07:00" }, "price_rules": { - "updated_at": "2022-10-14T10:19:39-07:00" + "updated_at": "2022-10-14T10:19:39-07:00" }, "discount_codes": { - "price_rules": { - "updated_at": "2022-10-14T10:10:25-07:00" - }, + "price_rules": { "updated_at": "2022-10-14T10:10:25-07:00" + }, + "updated_at": "2022-10-14T10:10:25-07:00" }, "inventory_items": { - "products": { - "updated_at": "2023-03-20T06:08:51-07:00" - }, - "updated_at": "2023-01-06T10:35:26-08:00" + "products": { + "updated_at": "2023-03-20T06:08:51-07:00" + }, + "updated_at": "2023-01-06T10:35:26-08:00" }, "inventory_levels": { - "locations": {}, - "updated_at": "2023-01-06T10:35:27-08:00" + "locations": {}, + "updated_at": "2023-01-06T10:35:27-08:00" }, "fulfillment_orders": { - "orders": { - "updated_at": "2022-10-10T06:05:29-07:00" - }, - "id": 5567724486845 + "orders": { + "updated_at": "2022-10-10T06:05:29-07:00" + }, + "id": 5567724486845 }, "fulfillments": { - "orders": { - "updated_at": "2022-10-10T06:05:29-07:00" - }, - "updated_at": "2022-06-22T03:50:14-07:00" + "orders": { + "updated_at": "2022-10-10T06:05:29-07:00" + }, + "updated_at": "2022-06-22T03:50:14-07:00" }, "balance_transactions": {}, "articles": { - "id": 558137508029 + "id": 558137508029 }, "metafield_articles": { - "articles": { - "id": 558137508029 - }, - "updated_at": "2022-10-07T16:09:02-07:00" + "articles": { + "id": 558137508029 + }, + "updated_at": "2022-10-07T16:09:02-07:00" }, "blogs": { - "id": 80417685693 + "id": 80417685693 }, "metafield_blogs": { - "blogs": { - "id": 80417685693 - }, - "updated_at": "2022-10-07T06:05:23-07:00" + "blogs": { + "id": 80417685693 + }, + "updated_at": "2022-10-07T06:05:23-07:00" }, "metafield_customers": { - "customers": { - "updated_at": "2022-10-11T08:40:51-07:00" - }, + "customers": { "updated_at": "2022-10-11T08:40:51-07:00" + }, + "updated_at": "2022-10-11T08:40:51-07:00" }, "metafield_orders": { - "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" - }, + "orders": { "updated_at": "2022-10-12T11:15:27-07:00" + }, + "updated_at": "2022-10-12T11:15:27-07:00" }, "metafield_draft_orders": { - "draft_orders": { - "updated_at": "2022-10-08T05:07:29-07:00" - }, + "draft_orders": { "updated_at": "2022-10-08T05:07:29-07:00" + }, + "updated_at": "2022-10-08T05:07:29-07:00" }, "metafield_products": { - "products": { - "updated_at": "2023-03-20T06:08:32-07:00" - }, - "updated_at": "2022-10-08T08:45:01-07:00" + "products": { + "updated_at": "2023-03-20T06:08:32-07:00" + }, + "updated_at": "2022-10-08T08:45:01-07:00" }, "product_images": { - "products": { - "updated_at": "2023-03-20T06:08:51-07:00" - }, - "id": 32940458868925 + "products": { + "updated_at": "2023-03-20T06:08:51-07:00" + }, + "id": 32940458868925 }, "metafield_product_images": { - "products": { - "updated_at": "2023-03-20T06:08:51-07:00" - }, - "updated_at": "2022-10-13T04:01:00-07:00" + "products": { + "updated_at": "2023-03-20T06:08:51-07:00" + }, + "updated_at": "2022-10-13T04:01:00-07:00" }, "product_variants": { - "products": { - "updated_at": "2023-03-20T06:08:51-07:00" - }, - "id": 42595864019133 + "products": { + "updated_at": "2023-03-20T06:08:51-07:00" + }, + "id": 42595864019133 }, "metafield_product_variants": { - "products": { - "updated_at": "2023-03-20T06:08:51-07:00" - }, - "updated_at": "2022-10-14T10:16:04-07:00" + "products": { + "updated_at": "2023-03-20T06:08:51-07:00" + }, + "updated_at": "2022-10-14T10:16:04-07:00" }, "collections": { - "updated_at": "2023-03-20T06:08:12-07:00", - "collects": { - "id": 29427031703741 - } + "updated_at": "2023-03-20T06:08:12-07:00", + "collects": { + "id": 29427031703741 + } }, "metafield_collections": { - "updated_at": "2022-10-08T04:44:51-07:00", - "collects": { - "id": 29427031703741 - } + "updated_at": "2022-10-08T04:44:51-07:00", + "collects": { + "id": 29427031703741 + } }, "smart_collections": { - "updated_at": "2023-03-20T06:08:12-07:00" + "updated_at": "2023-03-20T06:08:12-07:00" }, "metafield_smart_collections": { - "smart_collections": { - "updated_at": "2023-03-20T06:08:12-07:00" - }, - "updated_at": "2022-10-12T13:36:55-07:00" + "smart_collections": { + "updated_at": "2023-03-20T06:08:12-07:00" + }, + "updated_at": "2022-10-12T13:36:55-07:00" }, "metafield_pages": { - "pages": { - "updated_at": "2022-10-08T08:07:00-07:00" - }, + "pages": { "updated_at": "2022-10-08T08:07:00-07:00" + }, + "updated_at": "2022-10-08T08:07:00-07:00" }, "metafield_shops": { - "updated_at": "2022-05-30T23:42:02-07:00" + "updated_at": "2022-05-30T23:42:02-07:00" }, "metafield_locations": { - "locations": {}, - "updated_at": "2022-10-12T02:21:35-07:00" + "locations": {}, + "updated_at": "2022-10-12T02:21:35-07:00" } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-shopify/metadata.yaml b/airbyte-integrations/connectors/source-shopify/metadata.yaml index 187c8d14fe2e..c711d5f0c45f 100644 --- a/airbyte-integrations/connectors/source-shopify/metadata.yaml +++ b/airbyte-integrations/connectors/source-shopify/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 - dockerImageTag: 0.3.4 + dockerImageTag: 0.6.2 dockerRepository: airbyte/source-shopify + documentationUrl: https://docs.airbyte.com/integrations/sources/shopify githubIssueLabel: source-shopify icon: shopify.svg - license: MIT + license: ELv2 name: Shopify registries: cloud: @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/shopify + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-shopify/requirements.txt b/airbyte-integrations/connectors/source-shopify/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-shopify/requirements.txt +++ b/airbyte-integrations/connectors/source-shopify/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-shopify/setup.py b/airbyte-integrations/connectors/source-shopify/setup.py index 5d339f8510c3..d1ec88ea5e7b 100644 --- a/airbyte-integrations/connectors/source-shopify/setup.py +++ b/airbyte-integrations/connectors/source-shopify/setup.py @@ -11,7 +11,6 @@ "pytest", "pytest-mock", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py index e3f6021bfedc..30cb4056a659 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py @@ -2,16 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from typing import Any, Dict, Mapping -from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator class NotImplementedAuth(Exception): """Not implemented Auth option error""" - logger = AirbyteLogger() + logger = logging.getLogger("airbyte") def __init__(self, auth_method: str = None): self.message = f"Not implemented Auth method = {auth_method}" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/countries.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/countries.json new file mode 100644 index 000000000000..5c359adba398 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/countries.json @@ -0,0 +1,56 @@ +{ + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "code": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "provinces": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "string"] + }, + "country_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "tax": { + "type": ["null", "number"] + }, + "tax_name": { + "type": ["null", "string"] + }, + "tax_type": { + "type": ["null", "string"] + }, + "tax_percentage": { + "type": ["null", "integer"] + } + } + } + }, + "tax": { + "type": ["null", "number"] + }, + "tax_name": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_address.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_address.json new file mode 100644 index 000000000000..31d24b3c458c --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_address.json @@ -0,0 +1,60 @@ +{ + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "country_name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "last_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + }, + "shop_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_saved_search.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_saved_search.json new file mode 100644 index 000000000000..3bb88300f4c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customer_saved_search.json @@ -0,0 +1,26 @@ +{ + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "query": { + "type": ["null", "string"] + }, + "shop_url": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/disputes.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/disputes.json new file mode 100644 index 000000000000..20e3c4fa89d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/disputes.json @@ -0,0 +1,46 @@ +{ + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "network_reason_code": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "initiated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "evidence_due_by": { + "type": ["null", "string"], + "format": "date-time" + }, + "evidence_sent_on": { + "type": ["null", "string"], + "format": "date-time" + }, + "finalized_on": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json index 4ad7ac51e275..3aceaff8f507 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json @@ -495,7 +495,6 @@ } } } - } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json index 1a9554c48751..9471121a9988 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/product_variants.json @@ -12,8 +12,7 @@ "type": ["null", "string"] }, "price": { - "type": ["null", "number"], - "format": "float" + "type": ["null", "number"] }, "sku": { "type": ["null", "string"] @@ -63,8 +62,7 @@ "type": ["null", "integer"] }, "weight": { - "type": ["null", "number"], - "format": "float" + "type": ["null", "number"] }, "weight_unit": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index ed0c2b2eff50..88f9b2563b2e 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -3,6 +3,7 @@ # +import logging from abc import ABC, abstractmethod from functools import cached_property from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -13,15 +14,16 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from requests.exceptions import RequestException +from requests.exceptions import ConnectionError, InvalidURL, JSONDecodeError, RequestException, SSLError from .auth import ShopifyAuthenticator from .graphql import get_query_products from .transform import DataTypeEnforcer from .utils import SCOPES_MAPPING, ApiTypeEnum from .utils import EagerlyCachedStreamState as stream_state_cache -from .utils import ErrorAccessScopes +from .utils import ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyConnectionError, ShopifyNonRetryableErrors from .utils import ShopifyRateLimiter as limiter +from .utils import ShopifyWrongShopNameError class ShopifyStream(HttpStream, ABC): @@ -34,7 +36,11 @@ class ShopifyStream(HttpStream, ABC): order_field = "updated_at" filter_field = "updated_at_min" + # define default logger + logger = logging.getLogger("airbyte") + raise_on_http_errors = True + max_retries = 5 def __init__(self, config: Dict): super().__init__(authenticator=config["authenticator"]) @@ -77,7 +83,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp records = json_response.get(self.data_field, []) if self.data_field is not None else json_response yield from self.produce_records(records) except RequestException as e: - self.logger.warn(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") + self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") yield {} def produce_records(self, records: Union[Iterable[Mapping[str, Any]], Mapping[str, Any]] = None) -> Iterable[Mapping[str, Any]]: @@ -97,15 +103,11 @@ def produce_records(self, records: Union[Iterable[Mapping[str, Any]], Mapping[st yield self._transformer.transform(record) def should_retry(self, response: requests.Response) -> bool: - error_mapping = { - 404: f"Stream `{self.name}` - not available or missing, skipping...", - 403: f"Stream `{self.name}` - insufficient permissions, skipping...", - # extend the mapping with more handable errors, if needed. - } + known_errors = ShopifyNonRetryableErrors(self.name) status = response.status_code - if status in error_mapping.keys(): + if status in known_errors.keys(): setattr(self, "raise_on_http_errors", False) - self.logger.warn(error_mapping.get(status)) + self.logger.warning(known_errors.get(status)) return False else: return super().should_retry(response) @@ -162,13 +164,13 @@ def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None yield record else: # old entities could have cursor field in place, but set to null - self.logger.warn( + self.logger.warning( f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` cursor value is: {record_value}, record is emitted without state comparison" ) yield record else: # old entities could miss the cursor field - self.logger.warn( + self.logger.warning( f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` missing cursor field: {self.cursor_field}, record is emitted without state comparison" ) yield record @@ -375,6 +377,16 @@ def request_params( return params +class Disputes(IncrementalShopifyStream): + data_field = "disputes" + filter_field = "since_id" + cursor_field = "id" + order_field = "id" + + def path(self, **kwargs) -> str: + return f"shopify_payments/{self.data_field}.json" + + class MetafieldOrders(MetafieldShopifySubstream): parent_stream_class: object = Orders @@ -447,7 +459,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp json_response = response.json()["data"]["products"]["nodes"] yield from self.produce_records(json_response) except RequestException as e: - self.logger.warn(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") + self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") yield {} @@ -752,29 +764,93 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" +class CustomerSavedSearch(IncrementalShopifyStream): + api_version = "2022-01" + cursor_field = "id" + order_field = "id" + data_field = "customer_saved_searches" + filter_field = "since_id" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class CustomerAddress(ShopifySubstream): + parent_stream_class: object = Customers + slice_key = "id" + data_field = "addresses" + cursor_field = "id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + customer_id = stream_slice[self.slice_key] + return f"customers/{customer_id}/{self.data_field}.json" + + +class Countries(ShopifyStream): + data_field = "countries" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class ConnectionCheckTest: + def __init__(self, config: Mapping[str, Any]): + self.config = config + # use `Shop` as a test stream for connection check + self.test_stream = Shop(self.config) + # setting `max_retries` to 0 for the stage of `check connection`, + # because it keeps retrying for wrong shop names, + # but it should stop immediately + self.test_stream.max_retries = 0 + + def describe_error(self, pattern: str, shop_name: str = None, details: Any = None, **kwargs) -> str: + connection_check_errors_map: Mapping[str, Any] = { + "connection_error": f"Connection could not be established using `Shopify Store`: {shop_name}. Make sure it's valid and try again.", + "request_exception": f"Request was not successfull, check your `input configuation` and try again. Details: {details}", + "index_error": f"Failed to access the Shopify store `{shop_name}`. Verify the entered Shopify store or API Key in `input configuration`.", + # add the other patterns and description, if needed... + } + return connection_check_errors_map.get(pattern) + + def test_connection(self) -> tuple[bool, str]: + shop_name = self.config.get("shop") + if not shop_name: + return False, "The `Shopify Store` name is missing. Make sure it's entered and valid." + + try: + response = list(self.test_stream.read_records(sync_mode=None)) + # check for the shop_id is present in the response + shop_id = response[0].get("id") + if shop_id is not None: + return True, None + else: + return False, f"The `shop_id` is invalid: {shop_id}" + except (SSLError, ConnectionError): + return False, self.describe_error("connection_error", shop_name) + except RequestException as req_error: + return False, self.describe_error("request_exception", details=req_error) + except IndexError: + return False, self.describe_error("index_error", shop_name, response) + + class SourceShopify(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: """ Testing connection availability for the connector. """ + config["shop"] = self.get_shop_name(config) config["authenticator"] = ShopifyAuthenticator(config) - try: - response = list(Shop(config).read_records(sync_mode=None)) - # check for the shop_id is present in the response - shop_id = response[0].get("id") - if shop_id is not None: - return True, None - except (requests.exceptions.RequestException, IndexError) as e: - return False, e + return ConnectionCheckTest(config).test_connection() def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ Mapping a input config of the user input configuration as defined in the connector spec. Defining streams to run. """ + config["shop"] = self.get_shop_name(config) config["authenticator"] = ShopifyAuthenticator(config) user_scopes = self.get_user_scopes(config) - always_permitted_streams = ["MetafieldShops", "Shop"] + always_permitted_streams = ["MetafieldShops", "Shop", "Countries"] permitted_streams = [ stream for user_scope in user_scopes @@ -793,6 +869,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: CustomCollections(config), Customers(config), DiscountCodes(config), + Disputes(config), DraftOrders(config), FulfillmentOrders(config), Fulfillments(config), @@ -825,6 +902,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SmartCollections(config), TenderTransactions(config), Transactions(config), + CustomerSavedSearch(config), + CustomerAddress(config), + Countries(config), ] return [stream_instance for stream_instance in stream_instances if self.format_name(stream_instance.name) in permitted_streams] @@ -834,12 +914,27 @@ def get_user_scopes(config): session = requests.Session() url = f"https://{config['shop']}.myshopify.com/admin/oauth/access_scopes.json" headers = config["authenticator"].get_auth_header() - response = session.get(url, headers=headers).json() - access_scopes = response.get("access_scopes") + + try: + response = session.get(url, headers=headers).json() + access_scopes = response.get("access_scopes") + except InvalidURL: + raise ShopifyWrongShopNameError(url) + except JSONDecodeError as json_error: + raise ShopifyBadJsonError(json_error) + except (SSLError, ConnectionError) as con_error: + raise ShopifyConnectionError(con_error) + if access_scopes: return access_scopes else: - raise ErrorAccessScopes(f"Reason: {response}") + raise ShopifyAccessScopesError(response) + + @staticmethod + def get_shop_name(config): + split_pattern = ".myshopify.com" + shop_name = config.get("shop") + return shop_name.split(split_pattern)[0] if split_pattern in shop_name else shop_name @staticmethod def format_name(name): diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json index b6b00a5d1b1d..c73f7884f6e7 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json @@ -10,8 +10,9 @@ "shop": { "type": "string", "title": "Shopify Store", - "description": "The name of your Shopify store found in the URL. For example, if your URL was https://NAME.myshopify.com, then the name would be 'NAME'.", - "pattern": "^(?!https:\/\/)(?!http:\/\/)(?!.*\\.myshopify\\.)(?!.*\\.myshopify).*", + "description": "The name of your Shopify store found in the URL. For example, if your URL was https://NAME.myshopify.com, then the name would be 'NAME' or 'NAME.myshopify.com'.", + "pattern": "^(?!https://)(?!http://).*", + "examples": ["my-store", "my-store.myshopify.com"], "order": 1 }, "credentials": { @@ -96,6 +97,10 @@ "type": "object", "additionalProperties": false, "properties": { + "shop": { + "type": "string", + "path_in_connector_config": ["shop"] + }, "access_token": { "type": "string", "path_in_connector_config": ["credentials", "access_token"] @@ -127,16 +132,6 @@ "path_in_connector_config": ["credentials", "client_secret"] } } - }, - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "shop": { - "type": "string", - "path_in_connector_config": ["shop"] - } - } } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index 88b9a194a157..0ef38c922dcf 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -6,12 +6,12 @@ import enum from functools import wraps from time import sleep -from typing import Dict, List, Optional +from typing import Any, Dict, List, Mapping, Optional import requests SCOPES_MAPPING = { - "read_customers": ["Customers", "MetafieldCustomers"], + "read_customers": ["Customers", "MetafieldCustomers", "CustomerSavedSearch", "CustomerAddress"], "read_orders": [ "Orders", "AbandonedCheckouts", @@ -44,18 +44,57 @@ "read_locations": ["Locations", "MetafieldLocations"], "read_inventory": ["InventoryItems", "InventoryLevels"], "read_merchant_managed_fulfillment_orders": ["FulfillmentOrders"], - "read_shopify_payments_payouts": ["BalanceTransactions"], + "read_shopify_payments_payouts": ["BalanceTransactions", "Disputes"], "read_online_store_pages": ["Articles", "MetafieldArticles", "Blogs", "MetafieldBlogs"], } -class ErrorAccessScopes(Exception): +class ShopifyNonRetryableErrors: + """Holds the errors clasification and messaging scenarios.""" + + def __new__(self, stream: str) -> Mapping[str, Any]: + return { + 401: f"Stream `{stream}`. Failed to access the Shopify store with provided API token. Verify your API token is valid.", + 402: f"Stream `{stream}`. The shop's plan does not have access to this feature. Please upgrade your plan to be able to access this stream.", + 403: f"Stream `{stream}`. Unable to access Shopify endpoint for {stream}. Check that you have the appropriate access scopes to read data from this endpoint.", + 404: f"Stream `{stream}`. Not available or missing.", + 500: f"Stream `{stream}`. Entity might not be available or missing." + # extend the mapping with more handable errors, if needed. + } + + +class ShopifyAccessScopesError(Exception): """Raises the error if authenticated user doesn't have access to verify the grantted scopes.""" help_url = "https://shopify.dev/docs/api/usage/access-scopes#authenticated-access-scopes" + def __init__(self, response): + super().__init__( + f"Reason: Scopes are not available, make sure you're using the correct `Shopify Store` name. Actual response: {response}. More info about: {self.help_url}" + ) + + +class ShopifyBadJsonError(ShopifyAccessScopesError): + """Raises the error when Shopify replies with broken json for `access_scopes` request""" + def __init__(self, message): - super().__init__(f"{message}. More info about: {self.help_url}") + super().__init__(f"Reason: Bad JSON Response from the Shopify server. Details: {message}.") + + +class ShopifyConnectionError(ShopifyAccessScopesError): + """Raises the error when Shopify replies with broken connection error for `access_scopes` request""" + + def __init__(self, details): + super().__init__(f"Invalid `Shopify Store` name used or `host` couldn't be verified by Shopify. Details: {details}") + + +class ShopifyWrongShopNameError(Exception): + """Raises the error when `Shopify Store` name is incorrect or couldn't be verified by the Shopify""" + + def __init__(self, url): + super().__init__( + f"Reason: The `Shopify Store` name is invalid or missing for `input configuration`, make sure it's valid. Details: {url}" + ) class UnrecognisedApiType(Exception): diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index e3f167bb6e93..41ee39eff733 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -176,3 +176,18 @@ def test_get_updated_state(config, last_record, current_state, expected): def test_parse_response_with_bad_json(config, response_with_bad_json): stream = Customers(config) assert list(stream.parse_response(response_with_bad_json)) == [{}] + + +@pytest.mark.parametrize( + "shop, expected", + [ + ("test-store", "test-store"), + ("test-store.myshopify.com", "test-store"), + ], + ids=["old style", "oauth style"] +) +def test_get_shop_name(config, shop, expected): + source = SourceShopify() + config["shop"] = shop + actual = source.get_shop_name(config) + assert actual == expected diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index 66074323060e..a1fea8e4538a 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -5,7 +5,7 @@ import pytest import requests -from source_shopify.source import BalanceTransactions, DiscountCodes, PriceRules, ShopifyStream, SourceShopify +from source_shopify.source import BalanceTransactions, DiscountCodes, FulfillmentOrders, PriceRules, ShopifyStream, SourceShopify def test_get_next_page_token(requests_mock): @@ -47,27 +47,30 @@ def test_privileges_validation(requests_mock, basic_config): "shop", "tender_transactions", "transactions", + "countries", ] assert [stream.name for stream in source.streams(basic_config)] == expected @pytest.mark.parametrize( - "stream, status, json_response, expected_output", + "stream, slice, status, json_response, expected_output", [ - (BalanceTransactions, 404, {"errors": "Not Found"}, False), - (PriceRules, 403, {"errors": "Forbidden"}, False) + (BalanceTransactions, None, 404, {"errors": "Not Found"}, False), + (PriceRules, None, 403, {"errors": "Forbidden"}, False), + (FulfillmentOrders, {"order_id": 123}, 500, {"errors": "Internal Server Error"}, False), ], ids=[ "Stream not found (404)", "No permissions (403)", + "Internal Server Error for slice (500)", ], ) -def test_unavailable_stream(requests_mock, basic_config, stream, status, json_response, expected_output): +def test_unavailable_stream(requests_mock, basic_config, stream, slice, status, json_response, expected_output): config = basic_config config["authenticator"] = None stream = stream(config) - url = stream.url_base + stream.path() + url = stream.url_base + stream.path(stream_slice=slice) requests_mock.get(url=url, json=json_response, status_code=status) response = requests.get(url) assert stream.should_retry(response) is expected_output diff --git a/airbyte-integrations/connectors/source-shortio/Dockerfile b/airbyte-integrations/connectors/source-shortio/Dockerfile index 33aa353896c3..9650d6ff1014 100644 --- a/airbyte-integrations/connectors/source-shortio/Dockerfile +++ b/airbyte-integrations/connectors/source-shortio/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_shortio ./source_shortio + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_shortio ./source_shortio ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-shortio diff --git a/airbyte-integrations/connectors/source-shortio/README.md b/airbyte-integrations/connectors/source-shortio/README.md index 51a9f9902398..3e8a6bdb3870 100644 --- a/airbyte-integrations/connectors/source-shortio/README.md +++ b/airbyte-integrations/connectors/source-shortio/README.md @@ -1,34 +1,10 @@ # Shortio Source -This is the repository for the Shortio source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/shortio). +This is the repository for the Shortio configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/shortio). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/shortio) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shortio/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/shortio) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shortio/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source shortio test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shortio:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shortio:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. @@ -129,7 +80,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Create a Pull Request. 1. Pat yourself on the back for being an awesome contributor. 1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - -## Notes specific to the connector - -1. The links stream output doesn't match exactly what the documentation in the official website say (e.g. an owner object is returned as part of the response but that isn't listed there.) diff --git a/airbyte-integrations/connectors/source-shortio/__init__.py b/airbyte-integrations/connectors/source-shortio/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml index 0e03f5d44cbb..3314e7f6cb11 100644 --- a/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml @@ -1,25 +1,39 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-shortio:dev -tests: +acceptance_tests: spec: - - spec_path: "source_shortio/spec.json" + tests: + - spec_path: "source_shortio/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["clicks"] - # TODO: uncomment when any of incremental streams has records - # incremental: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: clicks + bypass_reason: "Sandbox account cannot seed the stream" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + incremental: + # bypass_reason: "This connector does not implement incremental sync" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py b/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json index b5425313f16e..0771b31e498b 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/abnormal_state.json @@ -1,5 +1,16 @@ -{ - "clicks": { - "dt": "2052-07-17 14:03:43.449925" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2099-07-31T03:43:59.244Z" }, + "stream_descriptor": { "name": "links" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "dt": "2099-09-10T12:44:55.000Z" }, + "stream_descriptor": { "name": "clicks" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py index 82823254d266..d49b55882333 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/acceptance.py @@ -10,5 +10,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json index 025b5475ee2f..483dfc373454 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/configured_catalog.json @@ -1,25 +1,24 @@ { "streams": [ { - "destination_sync_mode": "append", - "sync_mode": "incremental", "stream": { "name": "clicks", "source_defined_cursor": true, "default_cursor_field": ["dt"], "supported_sync_modes": ["incremental"], "json_schema": {} - } + }, + "destination_sync_mode": "append", + "sync_mode": "incremental" }, { - "destination_sync_mode": "append", - "sync_mode": "full_refresh", "stream": { "name": "links", - "source_defined_cursor": true, - "supported_sync_modes": ["full_refresh"], - "json_schema": {} - } + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-shortio/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..1b7a7d79ca97 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/expected_records.jsonl @@ -0,0 +1,4 @@ +{"stream": "links", "data": {"lcpath": "gem9bt", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "https://jk-genesis.com.ua/promotions/kvartiry-rjadom-s-metro-shuljavskaja-ot-980-00-grn?utm_source=lun.ua&utm_medium=referral&utm_content=239563406.1628516963&utm_term=5105", "cloaking": false, "path": "geM9Bt", "idString": "lnk_ZhP_Tw4Ye", "shortURL": "https://1hsf.short.gy/geM9Bt", "secureShortURL": "https://1hsf.short.gy/geM9Bt", "id": "lnk_ZhP_Tw4Ye", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031178} +{"stream": "links", "data": {"lcpath": "4sfi0i", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "https://airbyte.io/connector-development-kit", "cloaking": false, "path": "4SfI0I", "idString": "lnk_ZhP_Tw4Y3", "shortURL": "https://1hsf.short.gy/4SfI0I", "secureShortURL": "https://1hsf.short.gy/4SfI0I", "id": "lnk_ZhP_Tw4Y3", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031183} +{"stream": "links", "data": {"lcpath": "saeipy", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "https://great.com.ua/ua/news/11/znizhki-do-10-u-zhitlovomu-kompleksi-great/?utm_source=lun.ua&utm_medium=referral&utm_content=239563406.1628516963&utm_term=6405", "cloaking": false, "path": "Saeipy", "idString": "lnk_ZhP_Tw4Y9", "shortURL": "https://1hsf.short.gy/Saeipy", "secureShortURL": "https://1hsf.short.gy/Saeipy", "id": "lnk_ZhP_Tw4Y9", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031187} +{"stream": "links", "data": {"lcpath": "48ne6k", "createdAt": "2021-08-09T13:51:46.000Z", "source": "spreadsheets", "DomainId": 235589, "archived": false, "updatedAt": "2021-08-09T13:51:46.000Z", "OwnerId": 231775, "originalURL": "http://www.redstar.ru/2005/03/10_03/1_02.html", "cloaking": false, "path": "48ne6k", "idString": "lnk_ZhP_Tw4Y4", "shortURL": "https://1hsf.short.gy/48ne6k", "secureShortURL": "https://1hsf.short.gy/48ne6k", "id": "lnk_ZhP_Tw4Y4", "User": {"id": 231775, "name": "Jean Lafleur", "email": "integration-test@airbyte.io", "photoURL": null}}, "emitted_at": 1691072031191} diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json index 2017a4d76fa9..68bb7cb0e995 100644 --- a/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { "secret_key": "RANDOMKEY", - "domain_id": "123456", - "start_date": "2021-07-01" + "domain_id": "99999999", + "start_date": "2099-07-01" } diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json new file mode 100644 index 000000000000..83e947a41857 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "secret_key": "KEY", + "domain_id": "123456", + "start_date": "2023-07-30T03:43:59.244Z" +} diff --git a/airbyte-integrations/connectors/source-shortio/integration_tests/state.json b/airbyte-integrations/connectors/source-shortio/integration_tests/sample_state.json similarity index 100% rename from airbyte-integrations/connectors/source-shortio/integration_tests/state.json rename to airbyte-integrations/connectors/source-shortio/integration_tests/sample_state.json diff --git a/airbyte-integrations/connectors/source-shortio/metadata.yaml b/airbyte-integrations/connectors/source-shortio/metadata.yaml index bc9760330719..ad64fec0533c 100644 --- a/airbyte-integrations/connectors/source-shortio/metadata.yaml +++ b/airbyte-integrations/connectors/source-shortio/metadata.yaml @@ -1,20 +1,30 @@ data: + allowedHosts: + hosts: + - https://api.short.io + - https://api-v2.short.cm + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 2fed2292-5586-480c-af92-9944e39fe12d - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-shortio githubIssueLabel: source-shortio - icon: short.svg + icon: shortio.svg license: MIT - name: Short.io - registries: - cloud: - enabled: true - oss: - enabled: true + name: Shortio + releaseDate: 2023-08-02 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/shortio tags: - language:python + - language:lowcode + ab_internal: + sl: 100 + ql: 100 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-shortio/requirements.txt b/airbyte-integrations/connectors/source-shortio/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-shortio/requirements.txt +++ b/airbyte-integrations/connectors/source-shortio/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-shortio/setup.py b/airbyte-integrations/connectors/source-shortio/setup.py index 4c3679af61e2..6c8cbba1b880 100644 --- a/airbyte-integrations/connectors/source-shortio/setup.py +++ b/airbyte-integrations/connectors/source-shortio/setup.py @@ -5,11 +5,14 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.2.5", - "connector-acceptance-test", ] setup( @@ -19,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py b/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py index 73f38b078831..8560123a8d78 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/manifest.yaml b/airbyte-integrations/connectors/source-shortio/source_shortio/manifest.yaml new file mode 100644 index 000000000000..b0f7e60366c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/manifest.yaml @@ -0,0 +1,106 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractor_path }}"] + + v1_api_requester: + type: HttpRequester + url_base: "https://api.short.io/api/" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "Authorization" + api_token: "{{ config['secret_key'] }}" + request_parameters: + domain_id: "{{ config['domain_id'] }}" + + v2_api_requester: + type: HttpRequester + url_base: "https://api-v2.short.cm/statistics/" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "Authorization" + api_token: "{{ config['secret_key'] }}" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['nextPageToken'] }}" + page_token_option: + type: "RequestPath" + field_name: "pageToken" + inject_into: "request_parameter" + + v1_base_stream: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/v1_api_requester" + + v2_base_stream: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/v2_api_requester" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "updatedAt" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + end_time_option: + field_name: "beforeDate" + inject_into: "request_parameter" + start_time_option: + field_name: "afterDate" + inject_into: "request_parameter" + + links_stream: + $ref: "#/definitions/v1_base_stream" + name: "links" + incremental_sync: + $ref: "#/definitions/incremental_base" + primary_key: "id" + $parameters: + extractor_path: "links" + path: "links" + + clicks_stream: + $ref: "#/definitions/v2_base_stream" + name: "clicks" + $parameters: + path: "domain/{{ config['domain_id'] }}/link_clicks" + +streams: + - "#/definitions/links_stream" + - "#/definitions/clicks_stream" + +check: + type: CheckStream + stream_names: + - "links" + - "clicks" diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json index 6f3eed302b60..f1d34afa1540 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/clicks.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "default_cursor_field": ["dt"], + "additionalProperties": true, "properties": { "host": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json index ae204f7aa53f..01723e0eddc4 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/schemas/links.json @@ -1,7 +1,23 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "additionalProperties": true, "properties": { + "lcpath": { + "type": ["null", "string"] + }, + "passwordContact": { + "type": ["null", "string"] + }, + "hasPassword": { + "type": ["null", "boolean"] + }, + "OwnerId": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, "path": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/source.py b/airbyte-integrations/connectors/source-shortio/source_shortio/source.py index 384baccfa7e5..6fe21b6789db 100644 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/source.py +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/source.py @@ -2,228 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import contextlib -import datetime -import json -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -class BasicAuthenticator(TokenAuthenticator): - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"{self._token}"} - - -class Links(HttpStream, ABC): - - url_base = "https://api.short.io/api/" - limit = 150 - primary_key = "idString" - before_id = None - domain_id = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - - links = json.loads(response.text)["links"] - try: - earliest_id_string = sorted(links, key=lambda k: k["createdAt"], reverse=False)[0]["idString"] - if self.before_id != earliest_id_string: - self.before_id = earliest_id_string - return earliest_id_string - else: - return None - except IndexError: - return None - - def request_params( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - return { - "limit": self.limit, - "domain_id": self.domain_id, - "before": next_page_token or None, - } - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - # Get all the links - return "links" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - The short.io API can be inconsistent in its inclusion of UTM parameters. - Here, we check if they've been provided and if they haven't, attempt to extract it from the original url. - """ - utm_response_fields_to_utm_params = { - # Passing secondary UTM Campaign in order to capture either or of the 2 args. - "utmSource": "utm_source", - "utmMedium": "utm_medium", - "utmCampaign": "utm_campaign", - "utmCampaignId": "utm_id", - "utmTerm": "utm_term", - "utmContent": "utm_content", - } - links = json.loads(response.text)["links"] - for item in links: - for resp_field, param in utm_response_fields_to_utm_params.items(): - if resp_field not in item.keys(): - param = f"{param}=" - original_url = item["originalURL"] - param_value = None - with contextlib.suppress(IndexError): - # Extracting parameter value from original URL - # i.e "talent" from http://airbyte.io/?utm_source=talent - param_value = original_url.split(param, 2)[1].split("&", 1)[0] - item[resp_field] = param_value - yield item - - -# Clicks stream -class Clicks(HttpStream, ABC): - """ - This stream attempts to return the list of raw clicks from shortio. - """ - - url_base = "https://api-v2.short.cm/statistics/domain/" - before_dt = datetime.datetime.now().__str__() - domain_id = None - start_date = None - - @property - def http_method(self) -> str: - return "POST" - - @property - def cursor_field(self) -> str: - """ - :return str: The name of the cursor field. - """ - return "dt" - - @property - def primary_key(self) -> Optional[Any]: - return None - - @property - def limit(self) -> int: - return 1000 - - state_checkpoint_interval = limit - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - # Get all the links - return f"{self.domain_id}/last_clicks" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - This function goes through the API responses and ensures that no more requests are left to take place - :return str: min(dt) object from the previous API response. - """ - clicks = json.loads(response.text) - try: - before_dt = sorted(clicks, key=lambda k: k["dt"], reverse=False)[0]["dt"] - return None if self.limit > len(clicks) else before_dt - except IndexError: - return None - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, any]: - """ - Here we keep track of the state between different syncs to ensure that the data fetched is correct. - When the object is created, the datetime is taken and records are fetched until that point. - Due to varying duration possibilities this allows to a reproducable set of results" - """ - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - if current_stream_state is not None and "dt" in current_stream_state: - return {"dt": self.before_dt} - else: - return {"dt": self.start_date} - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - """ - This method passes the arguments necessary to get the clicks from the shortio API. - No parameters have been implemented here at all with the exception of hardcoding human clicks only to come through. - Human clicks are hardcoded to reduce unnecessary clicks from coming through. Some resources from short.io: - https://help.short.io/en/articles/4065954-how-short-io-tracks-clicks - https://help.short.io/en/articles/4890644-what-are-the-redirects - - :return dict: json body for the request - """ - return { - "limit": self.limit, - "include": {"human": True}, - "beforeDate": next_page_token or self.before_dt, - "afterDate": stream_state["dt"] if stream_state and "dt" in stream_state.keys() else self.start_date, - } - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from json.loads(response.text) - - -# Source -class SourceShortio(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - CHeck whether configuration is correct. - - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - - url = "https://api.short.io/api/domains" - api_secret = config["secret_key"] - domain_id = int(config["domain_id"]) - headers = {"Accept": "application/json", "Authorization": api_secret} - - response = requests.request("GET", url, headers=headers) - response.raise_for_status() - for domain in response.json(): - if domain_id == domain["id"]: - return True, None - except Exception as e: - return False, e - - return False, "Domain not found" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - key = config["secret_key"] - auth = BasicAuthenticator(token=key, auth_method=None) - links = Links(authenticator=auth) - links.domain_id = config["domain_id"] - clicks = Clicks(authenticator=auth) - clicks.domain_id = config["domain_id"] - clicks.start_date = config["start_date"] - return [clicks, links] +# Declarative Source +class SourceShortio(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json deleted file mode 100644 index 27e39c4a96ef..000000000000 --- a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "documentationUrl": "https://developers.short.io/reference", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Shortio Spec", - "type": "object", - "required": ["domain_id", "secret_key", "start_date"], - "properties": { - "domain_id": { - "type": "string", - "desciprtion": "Short.io Domain ID", - "title": "Domain ID", - "airbyte_secret": false - }, - "secret_key": { - "type": "string", - "title": "Secret Key", - "description": "Short.io Secret Key", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "airbyte_secret": false - } - } - } -} diff --git a/airbyte-integrations/connectors/source-shortio/source_shortio/spec.yaml b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.yaml new file mode 100644 index 000000000000..09596cd87c36 --- /dev/null +++ b/airbyte-integrations/connectors/source-shortio/source_shortio/spec.yaml @@ -0,0 +1,30 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/shortio/ +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Shortio Spec + type: object + additionalProperties: true + required: + - domain_id + - secret_key + - start_date + properties: + domain_id: + type: string + desciprtion: Short.io Domain ID + title: Domain ID + airbyte_secret: false + secret_key: + type: string + title: Secret Key + description: Short.io Secret Key + airbyte_secret: true + start_date: + type: string + title: Start Date + description: + UTC date and time in the format 2017-01-25T00:00:00Z. Any data + before this date will not be replicated. + examples: + - "2023-07-30T03:43:59.244Z" + airbyte_secret: false diff --git a/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py deleted file mode 100644 index 9961dce6365d..000000000000 --- a/airbyte-integrations/connectors/source-shortio/unit_tests/test_source.py +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import Status -from source_shortio.source import SourceShortio - - -@pytest.fixture -def config(): - return {"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"} - - -def test_source_shortio_client_wrong_credentials(): - source = SourceShortio() - result = source.check(logger=AirbyteLogger, config={"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"}) - assert result.status == Status.FAILED - - -def test_streams(): - source = SourceShortio() - config_mock = {"domain_id": "foo", "secret_key": "bar", "start_date": "2030-01-01"} - streams = source.streams(config_mock) - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-slack/metadata.yaml b/airbyte-integrations/connectors/source-slack/metadata.yaml index 3c822d7ea7df..3195f1807a01 100644 --- a/airbyte-integrations/connectors/source-slack/metadata.yaml +++ b/airbyte-integrations/connectors/source-slack/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/slack tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-slack/requirements.txt b/airbyte-integrations/connectors/source-slack/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-slack/requirements.txt +++ b/airbyte-integrations/connectors/source-slack/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-slack/setup.py b/airbyte-integrations/connectors/source-slack/setup.py index 5948fb91f499..66fc79eeca33 100644 --- a/airbyte-integrations/connectors/source-slack/setup.py +++ b/airbyte-integrations/connectors/source-slack/setup.py @@ -6,6 +6,8 @@ from setuptools import find_packages, setup TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", ] diff --git a/airbyte-integrations/connectors/source-smaily/metadata.yaml b/airbyte-integrations/connectors/source-smaily/metadata.yaml index 919d01a3b769..f2105bbe100b 100644 --- a/airbyte-integrations/connectors/source-smaily/metadata.yaml +++ b/airbyte-integrations/connectors/source-smaily/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-smaily/requirements.txt b/airbyte-integrations/connectors/source-smaily/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-smaily/requirements.txt +++ b/airbyte-integrations/connectors/source-smaily/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-smaily/setup.py b/airbyte-integrations/connectors/source-smaily/setup.py index 1d8901b7a312..aa835b3698f6 100644 --- a/airbyte-integrations/connectors/source-smaily/setup.py +++ b/airbyte-integrations/connectors/source-smaily/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-smartengage/metadata.yaml b/airbyte-integrations/connectors/source-smartengage/metadata.yaml index c59442cde2e0..25485ef405e6 100644 --- a/airbyte-integrations/connectors/source-smartengage/metadata.yaml +++ b/airbyte-integrations/connectors/source-smartengage/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-smartengage/requirements.txt b/airbyte-integrations/connectors/source-smartengage/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-smartengage/requirements.txt +++ b/airbyte-integrations/connectors/source-smartengage/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-smartengage/setup.py b/airbyte-integrations/connectors/source-smartengage/setup.py index f4b8e7f28660..958a3b00df73 100644 --- a/airbyte-integrations/connectors/source-smartengage/setup.py +++ b/airbyte-integrations/connectors/source-smartengage/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-smartsheets/Dockerfile b/airbyte-integrations/connectors/source-smartsheets/Dockerfile index 0259f106f4e0..023118cbedd7 100644 --- a/airbyte-integrations/connectors/source-smartsheets/Dockerfile +++ b/airbyte-integrations/connectors/source-smartsheets/Dockerfile @@ -14,5 +14,5 @@ COPY $CODE_PATH ./$CODE_PATH ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.1.1 LABEL io.airbyte.name=airbyte/source-smartsheets diff --git a/airbyte-integrations/connectors/source-smartsheets/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-smartsheets/integration_tests/configured_catalog.json index dc7ec0af4473..f919a67cd985 100644 --- a/airbyte-integrations/connectors/source-smartsheets/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-smartsheets/integration_tests/configured_catalog.json @@ -1,28 +1,27 @@ { - "streams": [ - { - "stream": { - "name": "aws_s3_sample", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { "type": "string" }, - "first_name": { "type": "string" }, - "last_name": { "type": "string" }, - "gender": { "type": "string" }, - "ip_address": { "type": "string" }, - "primary_email": { "type": "string" }, - "dob": { "type": "string", "format": "date" }, - "modifiedAt": { "type": "string", "format": "date-time" } - } - }, - "supported_sync_modes": ["full_refresh"] + "streams": [ + { + "stream": { + "name": "aws_s3_sample", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": "string" }, + "first_name": { "type": "string" }, + "last_name": { "type": "string" }, + "gender": { "type": "string" }, + "ip_address": { "type": "string" }, + "primary_email": { "type": "string" }, + "dob": { "type": "string", "format": "date" }, + "modifiedAt": { "type": "string", "format": "date-time" } + } }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "overwrite" - } - ] + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "overwrite" + } + ] } - diff --git a/airbyte-integrations/connectors/source-smartsheets/metadata.yaml b/airbyte-integrations/connectors/source-smartsheets/metadata.yaml index 7d2aba198d8d..c88443efa7e2 100644 --- a/airbyte-integrations/connectors/source-smartsheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-smartsheets/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc - dockerImageTag: 1.1.0 + dockerImageTag: 1.1.1 dockerRepository: airbyte/source-smartsheets githubIssueLabel: source-smartsheets icon: smartsheet.svg @@ -14,7 +14,6 @@ data: name: Smartsheets registries: cloud: - dockerImageTag: 1.0.2 # pinned due to https://github.com/airbytehq/alpha-beta-issues/issues/1400 enabled: true oss: enabled: true @@ -22,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/smartsheets tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-smartsheets/requirements.txt b/airbyte-integrations/connectors/source-smartsheets/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-smartsheets/requirements.txt +++ b/airbyte-integrations/connectors/source-smartsheets/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-smartsheets/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-smartsheets/sample_files/configured_catalog.json index e15369ce0735..7fb0a7a9c7c0 100644 --- a/airbyte-integrations/connectors/source-smartsheets/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-smartsheets/sample_files/configured_catalog.json @@ -1,61 +1,57 @@ { - "streams": [ - { - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "stream": { - "name": "Airbyte-connector Test", - "supported_sync_modes": [ - "full_refresh", - "incremental" - ], - "source_defined_cursor": false, - "source_defined_primary_key": [["race"]], - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "race": { - "type": "string" - }, - "height": { - "type": "string" - }, - "weight": { - "type": "string" - }, - "speed": { - "type": "string" - }, - "strength": { - "type": "string" - }, - "dexterity": { - "type": "string" - }, - "constitution": { - "type": "string" - }, - "intelligence": { - "type": "string" - }, - "wisdom": { - "type": "string" - }, - "charisma": { - "type": "string" - }, - "modifiedAt": { - "type": "string", - "format": "date-time" - }, - "row_id": { - "type": "string" - } - } - } + "streams": [ + { + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "stream": { + "name": "Airbyte-connector Test", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "source_defined_primary_key": [["race"]], + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "race": { + "type": "string" + }, + "height": { + "type": "string" + }, + "weight": { + "type": "string" + }, + "speed": { + "type": "string" + }, + "strength": { + "type": "string" + }, + "dexterity": { + "type": "string" + }, + "constitution": { + "type": "string" + }, + "intelligence": { + "type": "string" + }, + "wisdom": { + "type": "string" + }, + "charisma": { + "type": "string" + }, + "modifiedAt": { + "type": "string", + "format": "date-time" + }, + "row_id": { + "type": "string" } + } } - ] + } + } + ] } - diff --git a/airbyte-integrations/connectors/source-smartsheets/setup.py b/airbyte-integrations/connectors/source-smartsheets/setup.py index 358de80a4d93..f30812c4b62f 100644 --- a/airbyte-integrations/connectors/source-smartsheets/setup.py +++ b/airbyte-integrations/connectors/source-smartsheets/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = ["airbyte-cdk", "smartsheet-python-sdk==2.177.1", "urllib3<2.0"] -TEST_REQUIREMENTS = ["pytest", "pytest-mock~=3.6.1"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest", "pytest-mock~=3.6.1"] setup( name="source_smartsheets", diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py index 90d6e0e47828..ac0c398907e0 100644 --- a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py +++ b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/sheet.py @@ -15,7 +15,7 @@ class SmartSheetAPIWrapper: def __init__(self, config: Mapping[str, Any]): self._spreadsheet_id = config["spreadsheet_id"] self._config = config - self._metadata = config["metadata_fields"] + self._metadata = config.get("metadata_fields", []) self.api_client = smartsheet.Smartsheet(self.get_access_token(config)) self.api_client.errors_as_exceptions(True) # each call to `Sheets` makes a new instance, so we save it here to make no more new objects diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/spec.json b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/spec.json index 8821cf98ec41..b2a640d0d777 100644 --- a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/spec.json +++ b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/spec.json @@ -15,7 +15,13 @@ { "type": "object", "title": "OAuth2.0", - "required": ["client_id", "client_secret", "refresh_token", "access_token", "token_expiry_date"], + "required": [ + "client_id", + "client_secret", + "refresh_token", + "access_token", + "token_expiry_date" + ], "properties": { "auth_type": { "type": "string", @@ -86,27 +92,28 @@ "metadata_fields": { "title": "Metadata Fields", "type": "array", - "items":{ + "items": { "title": "Validenums", - "enum":[ - "sheetcreatedAt", - "sheetid", - "sheetmodifiedAt", - "sheetname", - "sheetpermalink", - "sheetversion", - "sheetaccess_level", - "row_id", - "row_access_level", - "row_created_at", - "row_created_by", - "row_expanded", - "row_modified_by", - "row_parent_id", - "row_permalink", - "row_number", - "row_version" - ]}, + "enum": [ + "sheetcreatedAt", + "sheetid", + "sheetmodifiedAt", + "sheetname", + "sheetpermalink", + "sheetversion", + "sheetaccess_level", + "row_id", + "row_access_level", + "row_created_at", + "row_created_by", + "row_expanded", + "row_modified_by", + "row_parent_id", + "row_permalink", + "row_number", + "row_version" + ] + }, "description": "A List of available columns which metadata can be pulled from.", "order": 3 } diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile index c55aae2de494..64dc6bd5a121 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-snapchat-marketing/Dockerfile @@ -25,5 +25,5 @@ COPY source_snapchat_marketing ./source_snapchat_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-snapchat-marketing diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json index b208d1ed715a..1a1e0f1c30fc 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state.json @@ -76,4 +76,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json index 52e63ca3bbe9..8b5afe5cedd4 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/abnormal_state_daily.json @@ -43,4 +43,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl index d5524655c287..b270c11318a9 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-snapchat-marketing/integration_tests/expected_records.jsonl @@ -1,5 +1,4 @@ -{"stream": "adaccounts", "data": {"id": "04214c00-3aa5-4123-b5c8-363c32c40e42", "updated_at": "2021-07-08T13:17:00.876Z", "created_at": "2020-12-15T11:13:04.282Z", "name": "Daxtarity Inc. Self Service", "type": "PARTNER", "status": "ACTIVE", "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "funding_source_ids": ["4d8f0f10-319a-46c0-890a-8ebb0006e34b"], "currency": "USD", "timezone": "America/Los_Angeles", "advertiser_organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "billing_center_id": "0e323e33-9007-4cfe-a551-20aefb4e5291", "billing_type": "REVOLVING", "agency_representing_client": false, "client_paying_invoices": false}, "emitted_at": 1674640109678} -{"stream": "adaccounts", "data": {"id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", "updated_at": "2022-07-03T13:34:15.633Z", "created_at": "2021-07-20T20:31:40.955Z", "name": "Second Integration Account", "type": "PARTNER", "status": "ACTIVE", "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "funding_source_ids": ["4d8f0f10-319a-46c0-890a-8ebb0006e34b"], "currency": "USD", "timezone": "America/Los_Angeles", "advertiser_organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "billing_center_id": "0e323e33-9007-4cfe-a551-20aefb4e5291", "billing_type": "REVOLVING", "agency_representing_client": false, "client_paying_invoices": false, "regulations": {"restricted_delivery_signals": false}}, "emitted_at": 1674640109682} +{"stream": "adaccounts", "data": {"id": "04214c00-3aa5-4123-b5c8-363c32c40e42", "updated_at": "2021-07-08T13:17:00.876Z", "created_at": "2020-12-15T11:13:04.282Z", "name": "Daxtarity Inc. Self Service", "type": "PARTNER", "status": "ACTIVE", "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "funding_source_ids": ["4d8f0f10-319a-46c0-890a-8ebb0006e34b"], "currency": "USD", "timezone": "America/Los_Angeles", "advertiser_organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "billing_center_id": "0e323e33-9007-4cfe-a551-20aefb4e5291", "billing_type": "REVOLVING", "agency_representing_client": false, "client_paying_invoices": false}, "emitted_at": 1686262174296} {"stream": "ads", "data": {"id": "96e5549f-4065-490e-93dd-ffb9f7973b77", "updated_at": "2022-07-01T17:15:14.970Z", "created_at": "2022-07-01T17:02:02.439Z", "name": "Extract yor data anywhere - Ad", "ad_squad_id": "ac6548b1-419e-4137-8320-e4e536766f72", "creative_id": "32224b92-fe08-4fb4-9c13-85f4f794d810", "status": "ACTIVE", "type": "REMOTE_WEBPAGE", "render_type": "STATIC", "review_status": "REJECTED", "review_status_reasons": ["Your ad contains low quality imagery, video, or sound that does not appear to be intentional. Some examples include audio or text that is cut off, imagery that is distorted, blurred, flashing brightly, rotated, or a broken video file. Please edit, resubmit for review, and our team will take another look. Questions? Contact us via our Business Help Center at https://businesshelp.snapchat.com", "Your Brand Name does not reflect the Paying Advertiser. Paying Advertiser examples include your Organization Name, Product Name, Parent Company, Artist Name or Co-Branded Partnerships. Please update your Brand Name to be reflective of the content you are advertising, resubmit for review, and our team will take another look."], "delivery_status": ["INVALID_NOT_APPROVED_REVIEW_STATUS", "INVALID_EFFECTIVE_INVALID"]}, "emitted_at": 1674640110702} {"stream": "ads", "data": {"id": "417d0269-80fb-496a-b5f3-ec0bac665144", "updated_at": "2021-07-22T12:54:00.462Z", "created_at": "2021-07-22T10:40:00.657Z", "name": "Snowflake - Ad", "ad_squad_id": "c67feaff-0ed8-4c05-b6a9-84c6de0e147f", "creative_id": "94aa10ab-97ca-4dc0-900d-4202212d6e2c", "status": "PAUSED", "type": "AD_TO_PLACE", "render_type": "STATIC", "review_status": "REJECTED", "review_status_reasons": ["Your ad encourages Snapchatters to take an action that is not possible with your selected ad type and attachment. Examples include: \n(i) An incorrect prompt to Snapchatters ('Watch' when the attachment is an app download), (ii) No attached content (asking Snapchatters to 'Play Now' with no game attached) or (iii) An action not applicable to the selected ad type ('Swipe Down'). \n\nPlease update your creative, resubmit for review, and our team will take another look."], "delivery_status": ["INVALID_NOT_ACTIVE", "INVALID_NOT_EFFECTIVE_ACTIVE", "INVALID_NOT_APPROVED_REVIEW_STATUS", "INVALID_EFFECTIVE_INVALID"]}, "emitted_at": 1674640110987} {"stream": "ads", "data": {"id": "8831ae74-bd1e-4ea1-9628-76e549041bea", "updated_at": "2022-07-01T17:24:43.416Z", "created_at": "2022-07-01T17:14:30.568Z", "name": "Open-source data integration - Ad", "ad_squad_id": "87f00f80-8ae6-44ac-ab63-2c976d43de64", "creative_id": "88ca89d3-b8a9-47df-984d-27c9e256279b", "status": "ACTIVE", "type": "REMOTE_WEBPAGE", "render_type": "STATIC", "review_status": "APPROVED", "delivery_status": ["INVALID_EFFECTIVE_INVALID"]}, "emitted_at": 1674640110988} @@ -28,7 +27,7 @@ {"stream": "media", "data": {"id": "19aa1d00-c92c-468d-8941-905b0deac1a6", "updated_at": "2021-07-22T10:42:03.830Z", "created_at": "2021-07-22T10:39:50.894Z", "ad_account_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", "type": "VIDEO", "media_status": "READY", "file_name": "file.mp4", "download_link": "https://storage.googleapis.com/creativesuite-prod-media-public/2c8a2bcc-6273-48c1-b8bb-784cf1e33a0b/file.mp4", "duration_in_seconds": 100.2, "video_metadata": {"width_px": 1080, "height_px": 1920, "rotation": null, "integrated_loudness": null, "true_peak": null}, "file_size_in_bytes": 5908636, "is_demo_media": false, "hash": "5mLt+w==", "visibility": "VISIBLE"}, "emitted_at": 1674640113879} {"stream": "media", "data": {"id": "1d78d00f-1189-45c7-b866-a24abbe1cc26", "updated_at": "2021-07-22T10:47:05.780Z", "created_at": "2021-07-22T10:45:25.935Z", "ad_account_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", "type": "VIDEO", "media_status": "READY", "file_name": "file.mp4", "download_link": "https://storage.googleapis.com/creativesuite-prod-media-public/9a0e1076-0023-45b4-8a78-2a7df5b9e5f9/file.mp4", "duration_in_seconds": 100.2, "video_metadata": {"width_px": 1080, "height_px": 1920, "rotation": null, "integrated_loudness": null, "true_peak": null}, "file_size_in_bytes": 5908636, "is_demo_media": false, "hash": "5mLt+w==", "visibility": "VISIBLE"}, "emitted_at": 1674640113879} {"stream": "media", "data": {"id": "56bbbea0-f602-4a81-b8ed-ce8bf5ec91d4", "updated_at": "2022-07-01T17:14:28.577Z", "created_at": "2022-07-01T17:09:02.386Z", "name": "blob.jpeg", "ad_account_id": "e4cd371b-8de8-4011-a8d2-860fe77c09e1", "type": "IMAGE", "media_status": "READY", "file_name": "693ed113-4bb4-4987-89ca-c779d0c9431b.jpeg", "download_link": "https://storage.googleapis.com/ad-manager-creatives-production-europe/56bbbea0-f602-4a81-b8ed-ce8bf5ec91d4/693ed113-4bb4-4987-89ca-c779d0c9431b.jpeg", "image_metadata": {"height_px": 1920, "width_px": 1080, "image_format": "JPEG"}, "file_size_in_bytes": 351893, "is_demo_media": false, "hash": "P0mNqg==", "visibility": "VISIBLE"}, "emitted_at": 1674640113880} -{"stream": "organizations", "data": {"id": "7f064d90-52a1-42db-b25b-7539e663e926", "updated_at": "2021-10-28T06:36:35.170Z", "created_at": "2020-12-15T11:13:03.910Z", "name": "Daxtarity Inc.", "country": "US", "postal_code": "94121", "locality": "San Francisco", "contact_name": "Team Airbyte", "contact_email": "integration-test@airbyte.io", "contact_phone": "+14156236785", "address_line_1": "350 29th avenue", "administrative_district_level_1": "US-CA", "accepted_term_version": "8", "contact_phone_optin": false, "configuration_settings": {"notifications_enabled": true}, "type": "ENTERPRISE", "state": "ACTIVE", "roles": ["admin"], "my_display_name": "Team Airbyte", "my_invited_email": "integration-test@airbyte.io", "my_member_id": "b9b2ab5f-e886-470c-92ae-0725d79a9146", "createdByCaller": true}, "emitted_at": 1674640114384} +{"stream": "organizations", "data": {"id": "7f064d90-52a1-42db-b25b-7539e663e926", "updated_at": "2023-06-07T16:39:22.334Z", "created_at": "2020-12-15T11:13:03.910Z", "name": "Daxtarity Inc.", "country": "US", "postal_code": "94121", "locality": "San Francisco", "contact_name": "Team Airbyte", "contact_email": "integration-test@airbyte.io", "contact_phone": "+14156236785", "address_line_1": "350 29th avenue", "administrative_district_level_1": "US-CA", "accepted_term_version": "8", "contact_phone_optin": true, "configuration_settings": {"notifications_enabled": true}, "type": "ENTERPRISE", "state": "ACTIVE", "roles": ["member", "business_admin", "admin"], "my_display_name": "Team Airbyte", "my_invited_email": "integration-test@airbyte.io", "my_member_id": "b9b2ab5f-e886-470c-92ae-0725d79a9146", "createdByCaller": true}, "emitted_at": 1686262790831} {"stream": "segments", "data": {"id": "4629391772795692", "updated_at": "2021-07-22T21:50:36.818Z", "created_at": "2021-07-22T21:50:36.669Z", "name": "Created from Postman First Account 1", "ad_account_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "description": "Test segment from Postman First Account 1", "status": "ACTIVE", "targetable_status": "READY", "upload_status": "NO_UPLOAD", "source_type": "FIRST_PARTY", "retention_in_days": 180, "approximate_number_users": 0, "visible_to": ["AdAccountEntity_04214c00-3aa5-4123-b5c8-363c32c40e42"]}, "emitted_at": 1674640114897} {"stream": "segments", "data": {"id": "5760336100911495", "updated_at": "2021-07-29T13:01:26.090Z", "created_at": "2021-07-29T13:01:25.961Z", "name": "postman_test_1", "ad_account_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "description": "postman_test_1", "status": "ACTIVE", "targetable_status": "READY", "upload_status": "NO_UPLOAD", "source_type": "FIRST_PARTY", "retention_in_days": 180, "approximate_number_users": 0, "visible_to": ["AdAccountEntity_04214c00-3aa5-4123-b5c8-363c32c40e42"]}, "emitted_at": 1674640114898} {"stream": "segments", "data": {"id": "5790170873686082", "updated_at": "2021-07-22T21:38:32.729Z", "created_at": "2021-07-22T21:38:32.597Z", "name": "Audience_Match_Email_Example", "ad_account_id": "04214c00-3aa5-4123-b5c8-363c32c40e42", "organization_id": "7f064d90-52a1-42db-b25b-7539e663e926", "description": "Custom Email Test Audience", "status": "ACTIVE", "targetable_status": "READY", "upload_status": "COMPLETE", "source_type": "FIRST_PARTY", "retention_in_days": 9999, "approximate_number_users": 0, "visible_to": ["AdAccountEntity_04214c00-3aa5-4123-b5c8-363c32c40e42"]}, "emitted_at": 1674640114898} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml index 9ddd952b3c5f..060d8a709d1e 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-snapchat-marketing/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-snapchat-marketing githubIssueLabel: source-snapchat-marketing icon: snapchat.svg @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/snapchat-marketing tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt b/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt +++ b/airbyte-integrations/connectors/source-snapchat-marketing/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py index 7ad8379d31c7..cc60f43cbe70 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/setup.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "requests_mock"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "requests_mock"] setup( name="source_snapchat_marketing", diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json index f1b5c99d70a9..6d26cdd036b7 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json +++ b/airbyte-integrations/connectors/source-snapchat-marketing/source_snapchat_marketing/spec.json @@ -48,12 +48,42 @@ } } }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": [], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["client_secret"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 03953bbf7c21..4aec528145cd 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -24,5 +24,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.34 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle index 1a75eced1fd5..3b8184d8b668 100644 --- a/airbyte-integrations/connectors/source-snowflake/build.gradle +++ b/airbyte-integrations/connectors/source-snowflake/build.gradle @@ -28,3 +28,4 @@ dependencies { integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' } + diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql index d5aa375c188f..5f0d33fc05f7 100644 --- a/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql @@ -1,45 +1,321 @@ -create schema sat_test_dataset; +CREATE + SCHEMA sat_test_dataset; -- Uncomment the line below if you need to recreate the table. -- DROP TABLE sat_test_dataset.sat_basic_dataset; +CREATE + TABLE + sat_test_dataset.sat_basic_dataset( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 NUMBER, + TEST_COLUMN_10 NUMBER( + 10, + 5 + ), + TEST_COLUMN_11 DOUBLE, + TEST_COLUMN_12 FLOAT, + TEST_COLUMN_14 VARCHAR, + TEST_COLUMN_15 STRING, + TEST_COLUMN_16 TEXT, + TEST_COLUMN_17 CHAR, + TEST_COLUMN_18 BINARY, + TEST_COLUMN_19 BOOLEAN, + TEST_COLUMN_2 DECIMAL, + TEST_COLUMN_20 DATE, + TEST_COLUMN_21 DATETIME, + TEST_COLUMN_23 TIMESTAMP, + TEST_COLUMN_24 TIMESTAMP_LTZ, + TEST_COLUMN_25 TIMESTAMP_NTZ, + TEST_COLUMN_26 TIMESTAMP_TZ, + TEST_COLUMN_27 VARIANT, + TEST_COLUMN_28 ARRAY, + TEST_COLUMN_29 OBJECT, + TEST_COLUMN_3 NUMERIC, + TEST_COLUMN_30 GEOGRAPHY, + TEST_COLUMN_4 BIGINT, + TEST_COLUMN_5 INT, + TEST_COLUMN_6 BIGINT, + TEST_COLUMN_7 SMALLINT, + TEST_COLUMN_8 TINYINT, + TEST_COLUMN_9 BYTEINT + ); -CREATE TABLE sat_test_dataset.sat_basic_dataset -( - ID INTEGER PRIMARY KEY, - TEST_COLUMN_1 NUMBER, - TEST_COLUMN_10 NUMBER(10,5), - TEST_COLUMN_11 DOUBLE, - TEST_COLUMN_12 FLOAT, - TEST_COLUMN_14 VARCHAR, - TEST_COLUMN_15 STRING, - TEST_COLUMN_16 TEXT, - TEST_COLUMN_17 CHAR, - TEST_COLUMN_18 BINARY, - TEST_COLUMN_19 BOOLEAN, - TEST_COLUMN_2 DECIMAL, - TEST_COLUMN_20 DATE, - TEST_COLUMN_21 DATETIME, - TEST_COLUMN_23 TIMESTAMP, - TEST_COLUMN_24 TIMESTAMP_LTZ, - TEST_COLUMN_25 TIMESTAMP_NTZ, - TEST_COLUMN_26 TIMESTAMP_TZ, - TEST_COLUMN_27 VARIANT, - TEST_COLUMN_28 ARRAY, - TEST_COLUMN_29 OBJECT, - TEST_COLUMN_3 NUMERIC, - TEST_COLUMN_30 GEOGRAPHY, - TEST_COLUMN_4 BIGINT, - TEST_COLUMN_5 INT, - TEST_COLUMN_6 BIGINT, - TEST_COLUMN_7 SMALLINT, - TEST_COLUMN_8 TINYINT, - TEST_COLUMN_9 BYTEINT -); - -INSERT INTO sat_test_dataset.sat_basic_dataset select 1, 99999999999999999999999999999999999999, 10.12345, -9007199254740991, 10e-308, 'тест', 'テスト', '-!-', 'a', to_binary('HELP', 'UTF-8'), 'true', 9223372036854775807, '0001-01-01', '0001-01-01 00:00:00', '2018-03-22 12:00:00.123', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 99999999999999999999999999999999999999, 'POINT(-122.35 37.55)', 99999999999999999999999999999999999999, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807; -INSERT INTO sat_test_dataset.sat_basic_dataset select 2, -99999999999999999999999999999999999999, 10.12345, 9007199254740991, 10e+307, '⚡ test ��', 'テスト', '-\x25-', 'ス', to_binary('HELP', 'UTF-8'), 5, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -99999999999999999999999999999999999999, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_basic_dataset select 3, 9223372036854775807, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', '!', to_binary('HELP', 'UTF-8'), 'false', -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 9223372036854775807, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_basic_dataset select 4, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), 0, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_basic_dataset select 5, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('y'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_basic_dataset select 6, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_basic_dataset select 7, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 1, + 99999999999999999999999999999999999999, + 10.12345, + - 9007199254740991, + 10e - 308, + 'тест', + 'テスト', + '-!-', + 'a', + to_binary( + 'HELP', + 'UTF-8' + ), + 'true', + 9223372036854775807, + '0001-01-01', + '0001-01-01 00:00:00', + '2018-03-22 12:00:00.123', + '2018-03-22 12:00:00.123 +05:00', + '2018-03-22 12:00:00.123 +05:00', + '2018-03-22 12:00:00.123 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + 99999999999999999999999999999999999999, + 'POINT(-122.35 37.55)', + 99999999999999999999999999999999999999, + 9223372036854775807, + 9223372036854775807, + 9223372036854775807, + 9223372036854775807, + 9223372036854775807; + +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 2, + - 99999999999999999999999999999999999999, + 10.12345, + 9007199254740991, + 10e + 307, + '⚡ test ��', + 'テスト', + '-\x25-', + 'ス', + to_binary( + 'HELP', + 'UTF-8' + ), + 5, + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 99999999999999999999999999999999999999, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 3, + 9223372036854775807, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + '!', + to_binary( + 'HELP', + 'UTF-8' + ), + 'false', + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + 9223372036854775807, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 4, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + 0, + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 5, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + TO_BOOLEAN('y'), + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 6, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + TO_BOOLEAN('n'), + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_basic_dataset SELECT + 7, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + TO_BOOLEAN('n'), + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql index 35f585d98872..a38e9d0aae7a 100644 --- a/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql @@ -1,46 +1,329 @@ -create schema sat_test_dataset; +CREATE + SCHEMA sat_test_dataset; -- Uncomment the line below if you need to recreate the table. -- DROP TABLE sat_test_dataset.sat_full_dataset; +CREATE + TABLE + sat_test_dataset.sat_full_dataset( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 NUMBER, + TEST_COLUMN_10 NUMBER( + 10, + 5 + ), + TEST_COLUMN_11 DOUBLE, + TEST_COLUMN_12 FLOAT, + TEST_COLUMN_14 VARCHAR, + TEST_COLUMN_15 STRING, + TEST_COLUMN_16 TEXT, + TEST_COLUMN_17 CHAR, + TEST_COLUMN_18 BINARY, + TEST_COLUMN_19 BOOLEAN, + TEST_COLUMN_2 DECIMAL, + TEST_COLUMN_20 DATE, + TEST_COLUMN_21 DATETIME, + TEST_COLUMN_22 TIME, + TEST_COLUMN_23 TIMESTAMP, + TEST_COLUMN_24 TIMESTAMP_LTZ, + TEST_COLUMN_25 TIMESTAMP_NTZ, + TEST_COLUMN_26 TIMESTAMP_TZ, + TEST_COLUMN_27 VARIANT, + TEST_COLUMN_28 ARRAY, + TEST_COLUMN_29 OBJECT, + TEST_COLUMN_3 NUMERIC, + TEST_COLUMN_30 GEOGRAPHY, + TEST_COLUMN_4 BIGINT, + TEST_COLUMN_5 INT, + TEST_COLUMN_6 BIGINT, + TEST_COLUMN_7 SMALLINT, + TEST_COLUMN_8 TINYINT, + TEST_COLUMN_9 BYTEINT + ); -CREATE TABLE sat_test_dataset.sat_full_dataset -( - ID INTEGER PRIMARY KEY, - TEST_COLUMN_1 NUMBER, - TEST_COLUMN_10 NUMBER(10,5), - TEST_COLUMN_11 DOUBLE, - TEST_COLUMN_12 FLOAT, - TEST_COLUMN_14 VARCHAR, - TEST_COLUMN_15 STRING, - TEST_COLUMN_16 TEXT, - TEST_COLUMN_17 CHAR, - TEST_COLUMN_18 BINARY, - TEST_COLUMN_19 BOOLEAN, - TEST_COLUMN_2 DECIMAL, - TEST_COLUMN_20 DATE, - TEST_COLUMN_21 DATETIME, - TEST_COLUMN_22 TIME, - TEST_COLUMN_23 TIMESTAMP, - TEST_COLUMN_24 TIMESTAMP_LTZ, - TEST_COLUMN_25 TIMESTAMP_NTZ, - TEST_COLUMN_26 TIMESTAMP_TZ, - TEST_COLUMN_27 VARIANT, - TEST_COLUMN_28 ARRAY, - TEST_COLUMN_29 OBJECT, - TEST_COLUMN_3 NUMERIC, - TEST_COLUMN_30 GEOGRAPHY, - TEST_COLUMN_4 BIGINT, - TEST_COLUMN_5 INT, - TEST_COLUMN_6 BIGINT, - TEST_COLUMN_7 SMALLINT, - TEST_COLUMN_8 TINYINT, - TEST_COLUMN_9 BYTEINT -); - -INSERT INTO sat_test_dataset.sat_full_dataset select 1, 99999999999999999999999999999999999999, 10.12345, -9007199254740991, 10e-308, 'тест', 'テスト', '-!-', 'a', to_binary('HELP', 'UTF-8'), 'true', 9223372036854775807, '0001-01-01', '0001-01-01 00:00:00', '00:00:00', '2018-03-22 12:00:00.123', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 99999999999999999999999999999999999999, 'POINT(-122.35 37.55)', 99999999999999999999999999999999999999, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807; -INSERT INTO sat_test_dataset.sat_full_dataset select 2, -99999999999999999999999999999999999999, 10.12345, 9007199254740991, 10e+307, '⚡ test ��', 'テスト', '-\x25-', 'ス', to_binary('HELP', 'UTF-8'), 5, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59', '1:59 PM', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -99999999999999999999999999999999999999, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_full_dataset select 3, 9223372036854775807, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', '!', to_binary('HELP', 'UTF-8'), 'false', -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 9223372036854775807, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_full_dataset select 4, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), 0, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_full_dataset select 5, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('y'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_full_dataset select 6, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; -INSERT INTO sat_test_dataset.sat_full_dataset select 7, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 1, + 99999999999999999999999999999999999999, + 10.12345, + - 9007199254740991, + 10e - 308, + 'тест', + 'テスト', + '-!-', + 'a', + to_binary( + 'HELP', + 'UTF-8' + ), + 'true', + 9223372036854775807, + '0001-01-01', + '0001-01-01 00:00:00', + '00:00:00', + '2018-03-22 12:00:00.123', + '2018-03-22 12:00:00.123 +05:00', + '2018-03-22 12:00:00.123 +05:00', + '2018-03-22 12:00:00.123 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + 99999999999999999999999999999999999999, + 'POINT(-122.35 37.55)', + 99999999999999999999999999999999999999, + 9223372036854775807, + 9223372036854775807, + 9223372036854775807, + 9223372036854775807, + 9223372036854775807; + +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 2, + - 99999999999999999999999999999999999999, + 10.12345, + 9007199254740991, + 10e + 307, + '⚡ test ��', + 'テスト', + '-\x25-', + 'ス', + to_binary( + 'HELP', + 'UTF-8' + ), + 5, + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59', + '1:59 PM', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 99999999999999999999999999999999999999, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 3, + 9223372036854775807, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + '!', + to_binary( + 'HELP', + 'UTF-8' + ), + 'false', + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + 9223372036854775807, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 4, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + 0, + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 5, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + TO_BOOLEAN('y'), + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 6, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + TO_BOOLEAN('n'), + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; + +INSERT + INTO + sat_test_dataset.sat_full_dataset SELECT + 7, + - 9223372036854775808, + 10.12345, + 9007199254740991, + 10e + 307, + '!"#$%&''()*+,-./:;<=>?\@[\]^_\`{|}~', + 'テスト', + '-\x25-', + 'ї', + to_binary( + 'HELP', + 'UTF-8' + ), + TO_BOOLEAN('n'), + - 9223372036854775808, + '9999-12-31', + '9999-12-31 23:59:59.123456', + '23:59:59.123456', + '2018-03-22 12:00:00.123456', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + '2018-03-22 12:00:00.123456 +05:00', + parse_json(' { "key1": "value1", "key2": "value2" } '), + array_construct( + 1, + 2, + 3 + ), + parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), + - 9223372036854775808, + 'LINESTRING(-124.20 42.00, -120.01 41.99)', + - 99999999999999999999999999999999999999, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808, + - 9223372036854775808; diff --git a/airbyte-integrations/connectors/source-snowflake/metadata.yaml b/airbyte-integrations/connectors/source-snowflake/metadata.yaml index dfff0fb24c5c..deb9bc02d5a1 100644 --- a/airbyte-integrations/connectors/source-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/source-snowflake/metadata.yaml @@ -1,15 +1,19 @@ data: + ab_internal: + ql: 200 + sl: 100 allowedHosts: hosts: - ${host} connectorSubtype: database connectorType: source definitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 - dockerImageTag: 0.1.34 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-snowflake + documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake githubIssueLabel: source-snowflake icon: snowflake.svg - license: MIT + license: ELv2 name: Snowflake registries: cloud: @@ -17,7 +21,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake + supportLevel: community tags: - language:java - language:python diff --git a/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml b/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml index c6913a27a231..1d1cf8e9b488 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-sonar-cloud/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt b/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt +++ b/airbyte-integrations/connectors/source-sonar-cloud/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sonar-cloud/setup.py b/airbyte-integrations/connectors/source-sonar-cloud/setup.py index b0819508b947..e99684d61a65 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/setup.py +++ b/airbyte-integrations/connectors/source-sonar-cloud/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml index c74f034dab3a..41f21b162e59 100644 --- a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-spacex-api/requirements.txt b/airbyte-integrations/connectors/source-spacex-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-spacex-api/requirements.txt +++ b/airbyte-integrations/connectors/source-spacex-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-spacex-api/setup.py b/airbyte-integrations/connectors/source-spacex-api/setup.py index db91e8b969e6..2d202b30ec9a 100644 --- a/airbyte-integrations/connectors/source-spacex-api/setup.py +++ b/airbyte-integrations/connectors/source-spacex-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-square/Dockerfile b/airbyte-integrations/connectors/source-square/Dockerfile index 313a7ff11d6b..c18311a47f48 100644 --- a/airbyte-integrations/connectors/source-square/Dockerfile +++ b/airbyte-integrations/connectors/source-square/Dockerfile @@ -34,5 +34,5 @@ COPY source_square ./source_square ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.0 +LABEL io.airbyte.version=1.1.2 LABEL io.airbyte.name=airbyte/source-square diff --git a/airbyte-integrations/connectors/source-square/metadata.yaml b/airbyte-integrations/connectors/source-square/metadata.yaml index bff953d208cb..23c1c36c7654 100644 --- a/airbyte-integrations/connectors/source-square/metadata.yaml +++ b/airbyte-integrations/connectors/source-square/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 - dockerImageTag: 1.1.0 + dockerImageTag: 1.1.2 dockerRepository: airbyte/source-square githubIssueLabel: source-square icon: square.svg @@ -22,4 +22,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-square/requirements.txt b/airbyte-integrations/connectors/source-square/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-square/requirements.txt +++ b/airbyte-integrations/connectors/source-square/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-square/setup.py b/airbyte-integrations/connectors/source-square/setup.py index 65a0fd5d30c6..051433b5675c 100644 --- a/airbyte-integrations/connectors/source-square/setup.py +++ b/airbyte-integrations/connectors/source-square/setup.py @@ -6,14 +6,14 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk>=0.44.2", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "freezegun", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-square/source_square/components.py b/airbyte-integrations/connectors/source-square/source_square/components.py index 0801d5849f5a..5da5ec2111dd 100644 --- a/airbyte-integrations/connectors/source-square/source_square/components.py +++ b/airbyte-integrations/connectors/source-square/source_square/components.py @@ -62,7 +62,7 @@ def get_request_body_json( } return json_payload - def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, **kwargs) -> Iterable[StreamSlice]: + def stream_slices(self) -> Iterable[StreamSlice]: locations_records = self.parent_stream.read_records(sync_mode=SyncMode.full_refresh) location_ids = [location[self.parent_key] for location in locations_records] @@ -78,10 +78,10 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, * for location in separated_locations: stream_slice = {"location_ids": location} cursor_field = self.cursor_field.eval(self.config) - if cursor_field and cursor_field in stream_state: + if self._cursor: # The Square API throws an error if when a datetime is greater than the current time current_datetime = datetime.now(timezone.utc) - cursor_datetime = self.parse_date(stream_state[cursor_field]) + cursor_datetime = self.parse_date(self._cursor) slice_datetime = ( current_datetime.strftime(self.datetime_format) if cursor_datetime > current_datetime diff --git a/airbyte-integrations/connectors/source-square/source_square/spec.yaml b/airbyte-integrations/connectors/source-square/source_square/spec.yaml index bd2ddf0cdd52..afd0e28e7a82 100644 --- a/airbyte-integrations/connectors/source-square/source_square/spec.yaml +++ b/airbyte-integrations/connectors/source-square/source_square/spec.yaml @@ -67,7 +67,7 @@ connectionSpecification: title: Start Date default: "2021-01-01" pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ - order: 2, + order: 2 format: date include_deleted_objects: type: boolean @@ -112,4 +112,4 @@ advanced_auth: type: string path_in_connector_config: - credentials - - client_secret \ No newline at end of file + - client_secret diff --git a/airbyte-integrations/connectors/source-square/unit_tests/test_component.py b/airbyte-integrations/connectors/source-square/unit_tests/test_component.py index d4653ad31cab..d991026313d5 100644 --- a/airbyte-integrations/connectors/source-square/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-square/unit_tests/test_component.py @@ -10,7 +10,6 @@ import pendulum import pytest import requests_mock -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator from airbyte_cdk.sources.declarative.datetime import MinMaxDatetime from source_square.components import SquareSubstreamIncrementalSync @@ -109,16 +108,15 @@ def test_substream_incremental_sync(state, last_record, expected, expected_strea parent_stream=parent_stream, ) - actual_stream_slice = next(slicer.stream_slices(SyncMode, state)) if records else {} + slicer.set_initial_state(state) + actual_stream_slice = next(slicer.stream_slices()) if records else {} # Covers the test case for abnormal state that is greater than the current time if "updated_at" in state and state["updated_at"] > datetime.now().strftime(DATETIME_FORMAT): assert actual_stream_slice["updated_at"] != state["updated_at"] - slicer.update_cursor(stream_slice=actual_stream_slice, last_record=last_record) - assert slicer.get_stream_state()["updated_at"] < state["updated_at"] else: assert actual_stream_slice == expected_stream_slice - slicer.update_cursor(stream_slice=actual_stream_slice, last_record=last_record) + slicer.close_slice(actual_stream_slice, last_record) assert slicer.get_stream_state() == expected @@ -152,7 +150,7 @@ def test_sub_slicer_request_body(last_record, records, expected_data): parent_key="id", parent_stream=parent_stream, ) - stream_slice = next(slicer.stream_slices(SyncMode, {})) if records else {} + stream_slice = next(slicer.stream_slices()) if records else {} expected_request_body = { "location_ids": expected_data.get("location_ids"), "query": { diff --git a/airbyte-integrations/connectors/source-statuspage/metadata.yaml b/airbyte-integrations/connectors/source-statuspage/metadata.yaml index 46f6adfe5bb4..85a2cf0283ab 100644 --- a/airbyte-integrations/connectors/source-statuspage/metadata.yaml +++ b/airbyte-integrations/connectors/source-statuspage/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-statuspage/requirements.txt b/airbyte-integrations/connectors/source-statuspage/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-statuspage/requirements.txt +++ b/airbyte-integrations/connectors/source-statuspage/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-statuspage/setup.py b/airbyte-integrations/connectors/source-statuspage/setup.py index a91ba9036ea5..772fedc5e034 100644 --- a/airbyte-integrations/connectors/source-statuspage/setup.py +++ b/airbyte-integrations/connectors/source-statuspage/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py b/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py index c9ec489513c2..f03a5373120a 100644 --- a/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py +++ b/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py @@ -22,13 +22,14 @@ import argparse # helps parse commandline arguments +import datetime import json -import sys import os +import sys +from datetime import date, timedelta + import requests -import datetime -from datetime import date -from datetime import timedelta + def read(config, catalog): # Assert required configuration was provided @@ -52,7 +53,7 @@ def read(config, catalog): sys.exit(1) # If we've made it this far, all the configuration is good and we can pull the last 7 days of market data - response = _call_api(ticker=config["stock_ticker"], token = config["api_key"]) + response = _call_api(ticker=config["stock_ticker"], token=config["api_key"]) if response.status_code != 200: # In a real scenario we'd handle this error better :) log_error("Failure occurred when calling Polygon.io API") @@ -62,7 +63,11 @@ def read(config, catalog): # We want to output them one by one as AirbyteMessages results = response.json()["results"] for result in results: - data = {"date": date.fromtimestamp(result["t"]/1000).isoformat(), "stock_ticker": config["stock_ticker"], "price": result["c"]} + data = { + "date": date.fromtimestamp(result["t"] / 1000).isoformat(), + "stock_ticker": config["stock_ticker"], + "price": result["c"], + } record = {"stream": "stock_prices", "data": data, "emitted_at": int(datetime.datetime.now().timestamp()) * 1000} output_message = {"type": "RECORD", "record": record} print(json.dumps(output_message)) @@ -115,23 +120,15 @@ def log_error(error_message): def discover(): catalog = { - "streams": [{ - "name": "stock_prices", - "supported_sync_modes": ["full_refresh"], - "json_schema": { - "properties": { - "date": { - "type": "string" - }, - "price": { - "type": "number" - }, - "stock_ticker": { - "type": "string" - } - } + "streams": [ + { + "name": "stock_prices", + "supported_sync_modes": ["full_refresh"], + "json_schema": { + "properties": {"date": {"type": "string"}, "price": {"type": "number"}, "stock_ticker": {"type": "string"}} + }, } - }] + ] } airbyte_message = {"type": "CATALOG", "catalog": catalog} print(json.dumps(airbyte_message)) @@ -179,9 +176,7 @@ def run(args): read_parser.add_argument("--state", type=str, required=False, help="path to the json-encoded state file") required_read_parser = read_parser.add_argument_group("required named arguments") required_read_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") - required_read_parser.add_argument( - "--catalog", type=str, required=True, help="path to the catalog used to determine which data to read" - ) + required_read_parser.add_argument("--catalog", type=str, required=True, help="path to the catalog used to determine which data to read") parsed_args = main_parser.parse_args(args) command = parsed_args.command diff --git a/airbyte-integrations/connectors/source-strava/metadata.yaml b/airbyte-integrations/connectors/source-strava/metadata.yaml index ea9bb41a10be..99c0c5bd2ef0 100644 --- a/airbyte-integrations/connectors/source-strava/metadata.yaml +++ b/airbyte-integrations/connectors/source-strava/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/strava tags: - language:python + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-strava/requirements.txt b/airbyte-integrations/connectors/source-strava/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-strava/requirements.txt +++ b/airbyte-integrations/connectors/source-strava/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-strava/setup.py b/airbyte-integrations/connectors/source-strava/setup.py index 16b43e8de640..1afea1f455f7 100644 --- a/airbyte-integrations/connectors/source-strava/setup.py +++ b/airbyte-integrations/connectors/source-strava/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index e968523d3f8e..7038660a56b2 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -13,6 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=3.6.0 +LABEL io.airbyte.version=3.17.4 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index e083bade27b3..1a76cc309b6f 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -14,7 +14,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: "3.3.0" + disable_for_version: "3.17.0" # invoices schema fix basic_read: tests: - config_path: "secrets/config.json" @@ -24,8 +24,12 @@ acceptance_tests: bypass_reason: "This stream can't be seeded in our sandbox account" - name: "application_fees_refunds" bypass_reason: "this stream can't be seeded in our sandbox account" + - name: "authorizations" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "bank_accounts" bypass_reason: "this stream can't be seeded in our sandbox account" + - name: "cards" + bypass_reason: "this stream can't be seeded in our sandbox account" - name: "early_fraud_warnings" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "external_account_bank_accounts" @@ -36,14 +40,21 @@ acceptance_tests: bypass_reason: "This stream can't be seeded in our sandbox account" - name: "checkout_sessions_line_items" bypass_reason: "This stream can't be seeded in our sandbox account" + - name: "payment_methods" + bypass_reason: "this stream can't be seeded in our sandbox account" + - name: "persons" + bypass_reason: "this stream can't be seeded in our sandbox account" + - name: "reviews" + bypass_reason: "this stream can't be seeded in our sandbox account" - name: "transfers" bypass_reason: "This stream can't be seeded in our sandbox account" + - name: "transactions" + bypass_reason: "This stream can't be seeded in our sandbox account" expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no exact_order: no extra_records: yes - fail_on_extra_columns: false ignored_fields: invoices: - name: invoice_pdf @@ -66,6 +77,15 @@ acceptance_tests: bypass_reason: "URL changes upon each request for privacy/security" - name: charges/data/*/payment_method_details bypass_reason: "Randomly added network_token field to the record" + credit_notes: + - name: pdf + bypass_reason: "URL changes upon each request for privacy/security" + files: + - name: links/data + bypass_reason: "Order of links/data elements changes on every request" + usage_records: + - name: id + bypass_reason: "id field is randomly generated" incremental: tests: - config_path: "secrets/config.json" @@ -82,3 +102,6 @@ acceptance_tests: bypass_reason: "URL changes upon each request for privacy/security" - name: hosted_invoice_url bypass_reason: "URL changes upon each request for privacy/security" + usage_records: + - name: id + bypass_reason: "id field is randomly generated" diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/__init__.py b/airbyte-integrations/connectors/source-stripe/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json index ea4e7f990baf..fa0c3e0dcf85 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json @@ -13,6 +13,27 @@ "stream_descriptor": { "name": "application_fees_refunds" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "date": 10000000000 }, + "stream_descriptor": { "name": "authorizations" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "cards" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "cardholders" } + } + }, { "type": "STREAM", "stream": { @@ -48,6 +69,13 @@ "stream_descriptor": { "name": "plans" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "persons" } + } + }, { "type": "STREAM", "stream": { @@ -69,6 +97,13 @@ "stream_descriptor": { "name": "transfers" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "shipping_rates" } + } + }, { "type": "STREAM", "stream": { @@ -97,6 +132,13 @@ "stream_descriptor": { "name": "payouts" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "prices" } + } + }, { "type": "STREAM", "stream": { @@ -146,11 +188,53 @@ "stream_descriptor": { "name": "checkout_sessions_line_items" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "reviews" } + } + }, { "type": "STREAM", "stream": { "stream_state": { "created": 10000000000 }, "stream_descriptor": { "name": "setup_intents" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "top_ups" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "files" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "file_links" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "setup_attempts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created": 10000000000 }, + "stream_descriptor": { "name": "transactions" } + } } ] diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json index 3bd0c03a75c0..31f8336f0ba2 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json @@ -36,7 +36,7 @@ }, { "stream": { - "name": "balance_transactions", + "name": "authorizations", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -50,18 +50,21 @@ }, { "stream": { - "name": "bank_accounts", + "name": "balance_transactions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "external_account_bank_accounts", + "name": "bank_accounts", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] @@ -72,7 +75,7 @@ }, { "stream": { - "name": "charges", + "name": "cards", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -86,35 +89,32 @@ }, { "stream": { - "name": "checkout_sessions", + "name": "cardholders", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["expires_at"], + "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "cursor_field": ["expires_at"], + "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "checkout_sessions_line_items", + "name": "external_account_bank_accounts", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["checkout_session_expires_at"], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "cursor_field": ["checkout_session_expires_at"], "primary_key": [["id"]] }, { "stream": { - "name": "coupons", + "name": "invoices", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -128,18 +128,21 @@ }, { "stream": { - "name": "customer_balance_transactions", + "name": "payment_intents", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "customers", + "name": "payouts", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -153,7 +156,7 @@ }, { "stream": { - "name": "disputes", + "name": "plans", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -167,7 +170,7 @@ }, { "stream": { - "name": "events", + "name": "prices", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -181,55 +184,63 @@ }, { "stream": { - "name": "early_fraud_warnings", + "name": "products", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"], + "primary_key": [["id"]] }, { "stream": { - "name": "invoice_items", + "name": "charges", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["date"], + "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "cursor_field": ["date"], + "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "invoice_line_items", + "name": "checkout_sessions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["expires_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", + "cursor_field": ["expires_at"], "primary_key": [["id"]] }, { "stream": { - "name": "invoices", + "name": "checkout_sessions_line_items", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["checkout_session_expires_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "cursor_field": ["created"], + "cursor_field": ["checkout_session_expires_at"], "primary_key": [["id"]] }, { "stream": { - "name": "payment_intents", + "name": "coupons", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -243,7 +254,7 @@ }, { "stream": { - "name": "payouts", + "name": "shipping_rates", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -257,35 +268,29 @@ }, { "stream": { - "name": "plans", + "name": "subscription_items", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "products", + "name": "customer_balance_transactions", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "promotion_codes", + "name": "files", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -299,7 +304,7 @@ }, { "stream": { - "name": "refunds", + "name": "persons", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -313,7 +318,7 @@ }, { "stream": { - "name": "setup_intents", + "name": "products", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -327,18 +332,21 @@ }, { "stream": { - "name": "subscription_items", + "name": "file_links", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "cursor_field": ["created"], "primary_key": [["id"]] }, { "stream": { - "name": "subscriptions", + "name": "top_ups", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -352,7 +360,7 @@ }, { "stream": { - "name": "subscription_schedule", + "name": "setup_attempts", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -366,7 +374,27 @@ }, { "stream": { - "name": "transfers", + "name": "usage_records", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "transfer_reversals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, + { + "stream": { + "name": "transactions", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -377,6 +405,15 @@ "destination_sync_mode": "overwrite", "cursor_field": ["created"], "primary_key": [["id"]] + }, + { + "stream": { + "name": "credit_notes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json index 1b208fe4bc92..29700a83523b 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json @@ -14,6 +14,20 @@ "cursor_field": ["created"], "primary_key": [["id"]] }, + { + "stream": { + "name": "shipping_rates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"], + "primary_key": [["id"]] + }, { "stream": { "name": "application_fees", diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index 25f58d2d858c..8ca6426f4d31 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -1,45 +1,58 @@ -{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1683202549890} -{"stream": "accounts", "data": {"id": "acct_1HRPLyCpK2Z3jTFF", "object": "account", "capabilities": {"acss_debit_payments": "inactive", "afterpay_clearpay_payments": "inactive", "bancontact_payments": "inactive", "card_payments": "inactive", "eps_payments": "inactive", "giropay_payments": "inactive", "ideal_payments": "inactive", "p24_payments": "inactive", "sepa_debit_payments": "inactive", "sofort_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "default_currency": "usd", "details_submitted": false, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "America/Los_Angeles"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1683202549892} -{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1683202550254} -{"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1683202551421} -{"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1683202551422} -{"stream": "balance_transactions", "data": {"id": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "object": "balance_transaction", "amount": 4200, "available_on": 1640649600, "created": 1640119035, "currency": "usd", "description": "edgao test", "exchange_rate": null, "fee": 152, "fee_details": [{"amount": 152, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 4048, "reporting_category": "charge", "source": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "status": "available", "type": "charge"}, "emitted_at": 1683202551423} -{"stream": "charges", "data": {"id": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "object": "charge", "amount": 5300, "amount_captured": 5300, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640120473, "currency": "usd", "customer": null, "description": null, "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 48, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "payment_method": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKPnDzqIGMgbc92m3bso6LBarH3soPvzWT4bk1mTnqAcxvE4_81jWX2ZPBmBsViqGVaqFVpUSHQSK4dJt", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9FSOEcXtiJtvvh0zxb7clc/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_3WszbFGtWT8vmMjqnNztOwhU", "created": 1640120473, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1683202553349} -{"stream": "charges", "data": {"id": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "object": "charge", "amount": 4200, "amount_captured": 4200, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 63, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9F5DEcXtiJtvvh16scJMp6", "payment_method": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKPnDzqIGMgbuNjOzTHA6LBYU7lOVhARXIWtoHjfl46GCnQO8Nx3y6FK-CMAtIOnryK0haVSYsI9emTXW", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F5DEcXtiJtvvh1w2MaTpj/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_QyH8xuqSyiZh8oxzzIszqQ92", "created": 1640119035, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1683202553353} -{"stream": "charges", "data": {"id": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "object": "charge", "amount": 4200, "amount_captured": 0, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": null, "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": false, "created": 1640119009, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": "card_declined", "failure_message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "not_sent_to_network", "reason": "test_mode_live_card", "risk_level": "normal", "risk_score": 6, "seller_message": "This charge request was in test mode, but did not use a Stripe test card number. For the list of these numbers, see stripe.com/docs/testing", "type": "invalid"}, "paid": false, "payment_intent": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "payment_method": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": null}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "installments": null, "last4": "8097", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": null, "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F4mEcXtiJtvvh1kUzxjwN/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": null, "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119009, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "failed", "transfer_data": null, "transfer_group": null}, "emitted_at": 1683202553356} -{"stream": "coupons", "data": {"id": "Coupon000001", "object": "coupon", "amount_off": 500, "created": 1675345584, "currency": "usd", "duration": "once", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "Test Coupon 1", "percent_off": null, "redeem_by": null, "times_redeemed": 1, "valid": true}, "emitted_at": 1683202555524} -{"stream": "coupons", "data": {"id": "4SUEGKZg", "object": "coupon", "amount_off": 200, "created": 1674209030, "currency": "usd", "duration": "repeating", "duration_in_months": 3, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0406\u0435\u043a\u0448\u0437\u0443", "percent_off": null, "redeem_by": null, "times_redeemed": 0, "valid": true}, "emitted_at": 1683202555525} -{"stream": "coupons", "data": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "emitted_at": 1683202555526} -{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MX2zPEcXtiJtvvhr4L2D3Q1", "object": "customer_balance_transaction", "amount": -50000.0, "created": 1675345091, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": null, "ending_balance": 0.0, "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "type": "applied_to_invoice"}, "emitted_at": 1683202557980} -{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MWIPLEcXtiJtvvhLnQYjVCj", "object": "customer_balance_transaction", "amount": 50000.0, "created": 1675166031, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": "Test credit balance", "ending_balance": 50000.0, "invoice": null, "livemode": false, "metadata": {}, "type": "adjustment"}, "emitted_at": 1683202557981} -{"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null}, "emitted_at": 1683202558255} -{"stream": "customers", "data": {"id": "cus_Kou8knsO3qQOwU", "object": "customer", "address": null, "balance": 0, "created": 1640123795, "currency": "usd", "default_currency": "usd", "default_source": "src_1MSID8EcXtiJtvvhxIT9lXRy", "delinquent": true, "description": null, "discount": null, "email": "edward.gao+stripe-test-customer-1@airbyte.io", "invoice_prefix": "CA35DF83", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "edgao-test-customer-1", "next_invoice_sequence": 2, "phone": null, "preferred_locales": [], "shipping": null, "tax_exempt": "none", "test_clock": null}, "emitted_at": 1683202558257} -{"stream": "customers", "data": {"id": "cus_NGoTFiJFVbSsvZ", "object": "customer", "address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "balance": 0, "created": 1675160053, "currency": "usd", "default_currency": "usd", "default_source": "src_1MWGs8EcXtiJtvvh4nYdQvEr", "delinquent": false, "description": "Test Customer 2 description", "discount": null, "email": "user1.sample@zohomail.eu", "invoice_prefix": "C09C1837", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test Customer 2", "next_invoice_sequence": 3, "phone": null, "preferred_locales": ["en-US"], "shipping": {"address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "name": "Test Customer 2", "phone": ""}, "tax_exempt": "none", "test_clock": null}, "emitted_at": 1683202558259} -{"stream": "disputes", "data": {"id": "dp_1MSI78EcXtiJtvvhxC77m2kh", "object": "dispute", "amount": 700, "balance_transaction": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "balance_transactions": [{"id": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "object": "balance_transaction", "amount": -700, "available_on": 1674518400, "created": 1674211590, "currency": "usd", "description": "Chargeback withdrawal for ch_3MSI77EcXtiJtvvh1GzoukUC", "exchange_rate": null, "fee": 1500, "fee_details": [{"amount": 1500, "application": null, "currency": "usd", "description": "Dispute fee", "type": "stripe_fee"}], "net": -2200, "reporting_category": "dispute", "source": "dp_1MSI78EcXtiJtvvhxC77m2kh", "status": "available", "type": "adjustment"}], "charge": "ch_3MSI77EcXtiJtvvh1GzoukUC", "created": 1674211590, "currency": "usd", "evidence": {"access_activity_log": null, "billing_address": "12345", "cancellation_policy": null, "cancellation_policy_disclosure": null, "cancellation_rebuttal": null, "customer_communication": null, "customer_email_address": null, "customer_name": null, "customer_purchase_ip": null, "customer_signature": null, "duplicate_charge_documentation": null, "duplicate_charge_explanation": null, "duplicate_charge_id": null, "product_description": null, "receipt": null, "refund_policy": null, "refund_policy_disclosure": null, "refund_refusal_explanation": null, "service_date": null, "service_documentation": null, "shipping_address": null, "shipping_carrier": null, "shipping_date": null, "shipping_documentation": null, "shipping_tracking_number": null, "uncategorized_file": null, "uncategorized_text": null}, "evidence_details": {"due_by": 1675036799.0, "has_evidence": false, "past_due": false, "submission_count": 0}, "is_charge_refundable": false, "livemode": false, "metadata": {}, "payment_intent": "pi_3MSI77EcXtiJtvvh1glmQd8s", "reason": "fraudulent", "status": "lost"}, "emitted_at": 1683202559823} -{"stream": "events", "data": {"id": "evt_1N3fEnEcXtiJtvvhd8k5QuXP", "object": "event", "api_version": "2020-08-27", "created": 1683118613, "data": {"object": {"id": "in_1MyD24EcXtiJtvvhlLv0vbgK", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 600, "amount_paid": 0, "amount_remaining": 600, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 4, "attempted": true, "auto_advance": true, "automatic_tax": {"enabled": false, "status": null}, "billing_reason": "subscription_create", "charge": null, "collection_method": "charge_automatically", "created": 1681818552, "currency": "usd", "custom_fields": null, "customer": "cus_Kou8knsO3qQOwU", "customer_address": null, "customer_email": "edward.gao+stripe-test-customer-1@airbyte.io", "customer_name": "edgao-test-customer-1", "customer_phone": null, "customer_shipping": null, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": "Thanks for your business!", "discount": null, "discounts": [], "due_date": null, "ending_balance": 0, "footer": "Test Invoice", "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OamdPZ3V3TmJFRG5NdVpGSnVVWXBoSVI2Yk9LSzNNLDczNjU5NDEz0200guq6K8DB?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OamdPZ3V3TmJFRG5NdVpGSnVVWXBoSVI2Yk9LSzNNLDczNjU5NDEz0200guq6K8DB/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1MyD24EcXtiJtvvhqc0Wwp8d", "object": "line_item", "amount": 600, "amount_excluding_tax": 600, "currency": "usd", "description": "1 \u00d7 tu (at $6.00 / month)", "discount_amounts": [], "discountable": true, "discounts": [], "livemode": false, "metadata": {}, "period": {"end": 1684410495, "start": 1681818495}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "subscription_item": "si_NjgONMC8J5TcRf", "tax_amounts": [], "tax_rates": [], "type": "subscription", "unit_amount_excluding_tax": "600"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1MyD24EcXtiJtvvhlLv0vbgK/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "CA35DF83-0001", "on_behalf_of": null, "paid": false, "paid_out_of_band": false, "payment_intent": "pi_3MyE3KEcXtiJtvvh0AaLHfBm", "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1681818495, "period_start": 1681818495, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 0, "quote": null, "receipt_number": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "open", "status_transitions": {"finalized_at": 1681822474, "marked_uncollectible_at": null, "paid_at": null, "voided_at": null}, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "subtotal": 600, "subtotal_excluding_tax": 600, "tax": null, "test_clock": null, "total": 600, "total_discount_amounts": [], "total_excluding_tax": 600, "total_tax_amounts": [], "transfer_data": null, "webhooks_delivered_at": 1681818552}}, "livemode": false, "pending_webhooks": 0, "request": {"id": null, "idempotency_key": null}, "type": "invoice.payment_failed"}, "emitted_at": 1685612075099} -{"stream": "invoice_items", "data": {"id": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "object": "invoiceitem", "amount": 8400, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "date": 1640123817, "description": "a box of parsnips", "discountable": true, "discounts": [], "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "livemode": false, "metadata": {}, "period": {"end": 1640123817.0, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "quantity": 1, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 8400, "unit_amount_decimal": "8400"}, "emitted_at": 1683202562589} -{"stream": "invoice_items", "data": {"id": "ii_1MX384EcXtiJtvvhguyn3iYb", "object": "invoiceitem", "amount": 6000, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345628, "description": "Test Product 1", "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice": "in_1MX37hEcXtiJtvvhRSl1KbQm", "livemode": false, "metadata": {}, "period": {"end": 1675345628.0, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "quantity": 3, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 2000, "unit_amount_decimal": "2000"}, "emitted_at": 1683202562845} -{"stream": "invoice_items", "data": {"id": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "object": "invoiceitem", "amount": 25200, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345045, "description": "edgao-test-product", "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "period": {"end": 1675345045.0, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "quantity": 2, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 12600, "unit_amount_decimal": "12600"}, "emitted_at": 1683202562846} -{"stream": "invoice_line_items", "data": {"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817.0, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400", "invoice_id": "in_1K9GK0EcXtiJtvvhSo2LvGqT"}, "emitted_at": 1683202564784} -{"stream": "invoice_line_items", "data": {"id": "il_1MyD24EcXtiJtvvhqc0Wwp8d", "object": "line_item", "amount": 600, "amount_excluding_tax": 600, "currency": "usd", "description": "1 \u00d7 tu (at $6.00 / month)", "discount_amounts": [], "discountable": true, "discounts": [], "livemode": false, "metadata": {}, "period": {"end": 1684410495.0, "start": 1681818495}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "subscription_item": "si_NjgONMC8J5TcRf", "tax_amounts": [], "tax_rates": [], "type": "subscription", "unit_amount_excluding_tax": "600", "invoice_id": "in_1MyD24EcXtiJtvvhlLv0vbgK"}, "emitted_at": 1683202565421} -{"stream": "invoices", "data": {"id": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 8400, "amount_paid": 0, "amount_remaining": 8400, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 0, "attempted": false, "auto_advance": false, "automatic_tax": {"enabled": false, "status": null}, "billing_reason": "manual", "charge": null, "collection_method": "send_invoice", "created": 1640123796, "currency": "usd", "custom_fields": null, "customer": "cus_Kou8knsO3qQOwU", "customer_address": null, "customer_email": "edward.gao+stripe-test-customer-1@airbyte.io", "customer_name": "edgao-test-customer-1", "customer_phone": null, "customer_shipping": null, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "discounts": [], "due_date": null, "ending_balance": null, "footer": null, "from_invoice": null, "hosted_invoice_url": null, "invoice_pdf": null, "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1K9GK0EcXtiJtvvhSo2LvGqT/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": null, "on_behalf_of": null, "paid": false, "paid_out_of_band": false, "payment_intent": null, "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1640123795.0, "period_start": 1640123795.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 0, "quote": null, "receipt_number": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "draft", "status_transitions": {"finalized_at": null, "marked_uncollectible_at": null, "paid_at": null, "voided_at": null}, "subscription": null, "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax": null, "test_clock": null, "total": 8400, "total_discount_amounts": [], "total_excluding_tax": 8400, "total_tax_amounts": [], "transfer_data": null, "webhooks_delivered_at": 1640123796.0}, "emitted_at": 1683202566590} -{"stream": "invoices", "data": {"id": "in_1MyD24EcXtiJtvvhlLv0vbgK", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 600, "amount_paid": 0, "amount_remaining": 600, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 4, "attempted": true, "auto_advance": false, "automatic_tax": {"enabled": false, "status": null}, "billing_reason": "subscription_create", "charge": null, "collection_method": "charge_automatically", "created": 1681818552, "currency": "usd", "custom_fields": null, "customer": "cus_Kou8knsO3qQOwU", "customer_address": null, "customer_email": "edward.gao+stripe-test-customer-1@airbyte.io", "customer_name": "edgao-test-customer-1", "customer_phone": null, "customer_shipping": null, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": "Thanks for your business!", "discount": null, "discounts": [], "due_date": null, "ending_balance": 0, "footer": "Test Invoice", "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OamdPZ3V3TmJFRG5NdVpGSnVVWXBoSVI2Yk9LSzNNLDczNzQzMzY20200vMNvN7ml?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OamdPZ3V3TmJFRG5NdVpGSnVVWXBoSVI2Yk9LSzNNLDczNzQzMzY30200gzT4bV4t/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1MyD24EcXtiJtvvhqc0Wwp8d", "object": "line_item", "amount": 600, "amount_excluding_tax": 600, "currency": "usd", "description": "1 \u00d7 tu (at $6.00 / month)", "discount_amounts": [], "discountable": true, "discounts": [], "livemode": false, "metadata": {}, "period": {"end": 1684410495, "start": 1681818495}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "subscription_item": "si_NjgONMC8J5TcRf", "tax_amounts": [], "tax_rates": [], "type": "subscription", "unit_amount_excluding_tax": "600"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1MyD24EcXtiJtvvhlLv0vbgK/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "CA35DF83-0001", "on_behalf_of": null, "paid": false, "paid_out_of_band": false, "payment_intent": "pi_3MyE3KEcXtiJtvvh0AaLHfBm", "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1681818495.0, "period_start": 1681818495.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 0, "quote": null, "receipt_number": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "open", "status_transitions": {"finalized_at": 1681822474, "marked_uncollectible_at": null, "paid_at": null, "voided_at": null}, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "subtotal": 600, "subtotal_excluding_tax": 600, "tax": null, "test_clock": null, "total": 600, "total_discount_amounts": [], "total_excluding_tax": 600, "total_tax_amounts": [], "transfer_data": null, "webhooks_delivered_at": 1681818552.0}, "emitted_at": 1683202567256} -{"stream": "payment_intents", "data": {"id": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "object": "payment_intent", "amount": 5300, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 5300, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "charges": {"object": "list", "data": [{"id": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "object": "charge", "amount": 5300, "amount_captured": 5300, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640120473, "currency": "usd", "customer": null, "description": null, "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 48, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "payment_method": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKIjEzqIGMgYH80xGLO46LBZGbKnbaIwAOJb-c03oF8tZ4ndm_hhPSv_L1XIIKvul1tKVEbTTJYvtz5S8", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/charges/ch_3K9FSOEcXtiJtvvh0zxb7clc/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_3WszbFGtWT8vmMjqnNztOwhU", "created": 1640120473, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}], "has_more": false, "total_count": 1, "url": "/v1/charges?payment_intent=pi_3K9FSOEcXtiJtvvh0AEIFllC"}, "client_secret": "pi_3K9FSOEcXtiJtvvh0AEIFllC_secret_uPUtIaSltgtW0qK7mLD0uF2Mr", "confirmation_method": "automatic", "created": 1640120472, "currency": "usd", "customer": null, "description": null, "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1683202568662} -{"stream": "payment_intents", "data": {"id": "pi_3K9F5DEcXtiJtvvh16scJMp6", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 4200, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "charges": {"object": "list", "data": [{"id": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "object": "charge", "amount": 4200, "amount_captured": 4200, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 63, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9F5DEcXtiJtvvh16scJMp6", "payment_method": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKIjEzqIGMgYOb5wz8ww6LBZ4H_fDRs-kuwMhGhNwK8rBjDpm9S88452M2dYAWNetsJgt3TA1dBF4o53K", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/charges/ch_3K9F5DEcXtiJtvvh1w2MaTpj/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_QyH8xuqSyiZh8oxzzIszqQ92", "created": 1640119035, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}], "has_more": false, "total_count": 1, "url": "/v1/charges?payment_intent=pi_3K9F5DEcXtiJtvvh16scJMp6"}, "client_secret": "pi_3K9F5DEcXtiJtvvh16scJMp6_secret_YwhzCTpXtfcKYeklXnPnysRRi", "confirmation_method": "automatic", "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1683202568664} -{"stream": "payment_intents", "data": {"id": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 0, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "charges": {"object": "list", "data": [{"id": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "object": "charge", "amount": 4200, "amount_captured": 0, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": null, "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": false, "created": 1640119009, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": "card_declined", "failure_message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "not_sent_to_network", "reason": "test_mode_live_card", "risk_level": "normal", "risk_score": 6, "seller_message": "This charge request was in test mode, but did not use a Stripe test card number. For the list of these numbers, see stripe.com/docs/testing", "type": "invalid"}, "paid": false, "payment_intent": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "payment_method": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": null}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "installments": null, "last4": "8097", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": null, "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/charges/ch_3K9F4mEcXtiJtvvh1kUzxjwN/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": null, "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119009, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "failed", "transfer_data": null, "transfer_group": null}], "has_more": false, "total_count": 1, "url": "/v1/charges?payment_intent=pi_3K9F4mEcXtiJtvvh18NKhEuo"}, "client_secret": "pi_3K9F4mEcXtiJtvvh18NKhEuo_secret_pfUt7CTkPjVdJacycm0bMpdLt", "confirmation_method": "automatic", "created": 1640119008, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": {"charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "code": "card_declined", "decline_code": "test_mode_live_card", "doc_url": "https://stripe.com/docs/error-codes/card-declined", "message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "unchecked", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119003, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "type": "card_error"}, "latest_charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "requires_payment_method", "transfer_data": null, "transfer_group": null}, "emitted_at": 1683202568666} -{"stream": "payouts", "data": {"id": "po_1KVQhfEcXtiJtvvhZlUkl08U", "object": "payout", "amount": 9164, "arrival_date": 1645488000, "automatic": true, "balance_transaction": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "destination": "ba_1KUL7UEcXtiJtvvhAEUlStmv", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "completed", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": null, "status": "paid", "type": "bank_account"}, "emitted_at": 1683202570861} -{"stream": "payouts", "data": {"id": "po_1MXKoPEcXtiJtvvhwmqjvKoO", "object": "payout", "amount": 665880, "arrival_date": 1675382400, "automatic": false, "balance_transaction": "txn_1MXKoQEcXtiJtvvh65SHFZS6", "created": 1675413601, "currency": "usd", "description": "", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1683202571130} -{"stream": "payouts", "data": {"id": "po_1MWHzjEcXtiJtvvhIdUQHhLq", "object": "payout", "amount": 154900, "arrival_date": 1675123200, "automatic": false, "balance_transaction": "txn_1MWHzjEcXtiJtvvhtydemd2Y", "created": 1675164443, "currency": "usd", "description": "Test", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": "issuing", "source_type": "issuing", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1683202571130} -{"stream": "plans", "data": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "emitted_at": 1683202572463} -{"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1675345058, "url": null}, "emitted_at": 1683202573619} -{"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1675345505, "url": null}, "emitted_at": 1683202573867} -{"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1674209524, "url": null}, "emitted_at": 1683202573868} -{"stream": "promotion_codes", "data": {"id": "promo_1MVtmyEcXtiJtvvhkV5jPFPU", "object": "promotion_code", "active": true, "code": "g20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071396, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": false, "minimum_amount": null, "minimum_amount_currency": null}, "times_redeemed": 0}, "emitted_at": 1683202575237} -{"stream": "promotion_codes", "data": {"id": "promo_1MVtmkEcXtiJtvvht0RA3MKg", "object": "promotion_code", "active": true, "code": "FRIENDS20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071382, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": true, "minimum_amount": 10000, "minimum_amount_currency": "usd"}, "times_redeemed": 0}, "emitted_at": 1683202575239} -{"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1683202576655} -{"stream": "subscription_items", "data": {"id": "si_NjgONMC8J5TcRf", "object": "subscription_item", "billing_thresholds": null, "created": 1681818552, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "tax_rates": []}, "emitted_at": 1683202579169} -{"stream": "subscriptions", "data": {"id": "sub_1MyD24EcXtiJtvvhI14S8cOw", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": 1681818495.0, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": 1683118613.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "payment_failed"}, "collection_method": "charge_automatically", "created": 1681818495, "currency": "usd", "current_period_end": 1684410495.0, "current_period_start": 1681818495, "customer": "cus_Kou8knsO3qQOwU", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": 1683118613.0, "items": {"object": "list", "data": [{"id": "si_NjgONMC8J5TcRf", "object": "subscription_item", "billing_thresholds": null, "created": 1681818552, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1MyD24EcXtiJtvvhI14S8cOw"}, "latest_invoice": "in_1MyD24EcXtiJtvvhlLv0vbgK", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1MvgI8EcXtiJtvvh7sfEXZHU", "start_date": 1681818495, "status": "canceled", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null}, "emitted_at": 1683202580698} -{"stream": "subscription_schedule", "data": {"id": "sub_sched_1MvgI8EcXtiJtvvh7sfEXZHU", "object": "subscription_schedule", "application": null, "canceled_at": "1683118613", "completed_at": null, "created": 1681216040, "current_phase": null, "customer": "cus_Kou8knsO3qQOwU", "default_settings": {"application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": "automatic", "billing_thresholds": null, "collection_method": "charge_automatically", "default_payment_method": null, "default_source": null, "description": null, "invoice_settings": null, "on_behalf_of": null, "transfer_data": null}, "end_behavior": "release", "livemode": false, "metadata": {}, "phases": [{"add_invoice_items": [], "application_fee_percent": null, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": null, "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": null, "end_date": 1713440895, "invoice_settings": null, "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1681818495, "transfer_data": null, "trial_end": null}], "released_at": null, "released_subscription": null, "renewal_interval": null, "status": "canceled", "subscription": "sub_1MyD24EcXtiJtvvhI14S8cOw", "test_clock": null}, "emitted_at": 1683202582143} -{"stream": "setup_intents", "data": {"id": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIjEcXtiJtvvhPw5znVKY_secret_LUebPsqMz6AF4ivxIg4LMaAT0OdZF5L", "created": 1649752937, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1683202583460} -{"stream": "setup_intents", "data": {"id": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIcEcXtiJtvvh61qlCaDf_secret_LUebcbyw8V1e8Pxk3aAjzDXMOXdFMCe", "created": 1649752930, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1683202583461} -{"stream": "setup_intents", "data": {"id": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIVEcXtiJtvvhWiIbMkpH_secret_LUebIUsiFnm75EzDUzf2RLhJ9WQ92Dp", "created": 1649752923, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1683202583462} +{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1689684208922} +{"stream": "accounts", "data": {"id": "acct_1HRPLyCpK2Z3jTFF", "object": "account", "capabilities": {"acss_debit_payments": "inactive", "afterpay_clearpay_payments": "inactive", "bancontact_payments": "inactive", "card_payments": "inactive", "eps_payments": "inactive", "giropay_payments": "inactive", "ideal_payments": "inactive", "p24_payments": "inactive", "sepa_debit_payments": "inactive", "sofort_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "default_currency": "usd", "details_submitted": false, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "America/Los_Angeles"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1689684208922} +{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1689684209297} +{"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1689684215036} +{"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1689684215036} +{"stream": "balance_transactions", "data": {"id": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "object": "balance_transaction", "amount": 4200, "available_on": 1640649600, "created": 1640119035, "currency": "usd", "description": "edgao test", "exchange_rate": null, "fee": 152, "fee_details": [{"amount": 152, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 4048, "reporting_category": "charge", "source": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "status": "available", "type": "charge"}, "emitted_at": 1689684215037} +{"stream": "cardholders", "data": {"id": "ich_1KUKBeEcXtiJtvvhCEFgko6h", "object": "issuing.cardholder", "billing": {"address": {"city": "San Francisco", "country": "US", "line1": "1234 Main Street", "line2": null, "postal_code": "94111", "state": "CA"}}, "company": null, "created": 1645143542, "email": "jenny.rosen@example.com", "individual": null, "livemode": false, "metadata": {}, "name": "Jenny Rosen", "phone_number": "+18888675309", "preferred_locales": [], "requirements": {"disabled_reason": null, "past_due": []}, "spending_controls": {"allowed_categories": [], "blocked_categories": [], "spending_limits": [], "spending_limits_currency": null}, "status": "active", "type": "individual"}, "emitted_at": 1689684219593} +{"stream": "charges", "data": {"id": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "object": "charge", "amount": 5300, "amount_captured": 5300, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640120473, "currency": "usd", "customer": null, "description": null, "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 48, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "payment_method": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1509-9197", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ6C36UGMgbHNxWFIZs6LBZJopnCEqXAr1eGbMC_s5Z-CjimuY7h3BPBSgA3kwq3H409GQNJ-c_Rq1jL", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9FSOEcXtiJtvvh0zxb7clc/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_3WszbFGtWT8vmMjqnNztOwhU", "created": 1640120473, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689764126216} +{"stream": "charges", "data": {"id": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "object": "charge", "amount": 4200, "amount_captured": 4200, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 63, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9F5DEcXtiJtvvh16scJMp6", "payment_method": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1549-5630", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ6C36UGMgap0zETEvk6LBZ9WiYPxmb7YdOMYue__fsqDfzaiB9KIoJo1the8s2BD-W_hVVO6OkQI4Mu", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F5DEcXtiJtvvh1w2MaTpj/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_QyH8xuqSyiZh8oxzzIszqQ92", "created": 1640119035, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689764126217} +{"stream": "charges", "data": {"id": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "object": "charge", "amount": 4200, "amount_captured": 0, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": null, "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": false, "created": 1640119009, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": "card_declined", "failure_message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "not_sent_to_network", "reason": "test_mode_live_card", "risk_level": "normal", "risk_score": 6, "seller_message": "This charge request was in test mode, but did not use a Stripe test card number. For the list of these numbers, see stripe.com/docs/testing", "type": "invalid"}, "paid": false, "payment_intent": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "payment_method": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": null}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "installments": null, "last4": "8097", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": null, "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F4mEcXtiJtvvh1kUzxjwN/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": null, "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119009, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "failed", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689764126219} +{"stream": "coupons", "data": {"id": "Coupon000001", "object": "coupon", "amount_off": 500, "created": 1675345584, "currency": "usd", "duration": "once", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "Test Coupon 1", "percent_off": null, "redeem_by": null, "times_redeemed": 1, "valid": true}, "emitted_at": 1689684226504} +{"stream": "coupons", "data": {"id": "4SUEGKZg", "object": "coupon", "amount_off": 200, "created": 1674209030, "currency": "usd", "duration": "repeating", "duration_in_months": 3, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0406\u0435\u043a\u0448\u0437\u0443", "percent_off": null, "redeem_by": null, "times_redeemed": 0, "valid": true}, "emitted_at": 1689684226504} +{"stream": "coupons", "data": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "emitted_at": 1689684226505} +{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MX2zPEcXtiJtvvhr4L2D3Q1", "object": "customer_balance_transaction", "amount": -50000.0, "created": 1675345091, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": null, "ending_balance": 0.0, "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "type": "applied_to_invoice"}, "emitted_at": 1689684227663} +{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MWIPLEcXtiJtvvhLnQYjVCj", "object": "customer_balance_transaction", "amount": 50000.0, "created": 1675166031, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": "Test credit balance", "ending_balance": 50000.0, "invoice": null, "livemode": false, "metadata": {}, "type": "adjustment"}, "emitted_at": 1689684227664} +{"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null}, "emitted_at": 1689691086995} +{"stream": "disputes", "data": {"id": "dp_1MSI78EcXtiJtvvhxC77m2kh", "object": "dispute", "amount": 700, "balance_transaction": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "balance_transactions": [{"id": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "object": "balance_transaction", "amount": -700, "available_on": 1674518400, "created": 1674211590, "currency": "usd", "description": "Chargeback withdrawal for ch_3MSI77EcXtiJtvvh1GzoukUC", "exchange_rate": null, "fee": 1500, "fee_details": [{"amount": 1500, "application": null, "currency": "usd", "description": "Dispute fee", "type": "stripe_fee"}], "net": -2200, "reporting_category": "dispute", "source": "dp_1MSI78EcXtiJtvvhxC77m2kh", "status": "available", "type": "adjustment"}], "charge": "ch_3MSI77EcXtiJtvvh1GzoukUC", "created": 1674211590, "currency": "usd", "evidence": {"access_activity_log": null, "billing_address": "12345", "cancellation_policy": null, "cancellation_policy_disclosure": null, "cancellation_rebuttal": null, "customer_communication": null, "customer_email_address": null, "customer_name": null, "customer_purchase_ip": null, "customer_signature": null, "duplicate_charge_documentation": null, "duplicate_charge_explanation": null, "duplicate_charge_id": null, "product_description": null, "receipt": null, "refund_policy": null, "refund_policy_disclosure": null, "refund_refusal_explanation": null, "service_date": null, "service_documentation": null, "shipping_address": null, "shipping_carrier": null, "shipping_date": null, "shipping_documentation": null, "shipping_tracking_number": null, "uncategorized_file": null, "uncategorized_text": null}, "evidence_details": {"due_by": 1675036799.0, "has_evidence": false, "past_due": false, "submission_count": 0}, "is_charge_refundable": false, "livemode": false, "metadata": {}, "payment_intent": "pi_3MSI77EcXtiJtvvh1glmQd8s", "reason": "fraudulent", "status": "lost"}, "emitted_at": 1689691088952} +{"stream": "events", "data": {"id": "evt_1NSrYfEcXtiJtvvhoo4M4yDP", "object": "event", "api_version": "2020-08-27", "created": 1689124173, "data": {"object": {"object": "balance", "available": [{"amount": 513474, "currency": "usd", "source_types": {"card": 513474}}], "connect_reserved": [{"amount": 0, "currency": "usd"}], "issuing": {"available": [{"amount": 150000, "currency": "usd"}]}, "livemode": false, "pending": [{"amount": 0, "currency": "usd", "source_types": {"card": 0}}]}}, "livemode": false, "pending_webhooks": 0, "request": {"id": null, "idempotency_key": null}, "type": "balance.available"}, "emitted_at": 1689691091429} +{"stream": "invoice_items", "data": {"id": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "object": "invoiceitem", "amount": 8400, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "date": 1640123817, "description": "a box of parsnips", "discountable": true, "discounts": [], "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "quantity": 1, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 8400, "unit_amount_decimal": "8400"}, "emitted_at": 1689691092541} +{"stream": "invoice_items", "data": {"id": "ii_1MX384EcXtiJtvvhguyn3iYb", "object": "invoiceitem", "amount": 6000, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345628, "description": "Test Product 1", "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice": "in_1MX37hEcXtiJtvvhRSl1KbQm", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "quantity": 3, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 2000, "unit_amount_decimal": "2000"}, "emitted_at": 1689691092833} +{"stream": "invoice_items", "data": {"id": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "object": "invoiceitem", "amount": 25200, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345045, "description": "edgao-test-product", "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "quantity": 2, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 12600, "unit_amount_decimal": "12600"}, "emitted_at": 1689691092834} +{"stream": "invoice_line_items", "data": {"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400", "invoice_id": "in_1K9GK0EcXtiJtvvhSo2LvGqT"}, "emitted_at": 1689691095089} +{"stream": "invoices", "data": {"id": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 0, "amount_paid": 0, "amount_remaining": 0, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 0, "attempted": true, "auto_advance": false, "automatic_tax": {"enabled": false, "status": null}, "billing_reason": "manual", "charge": null, "collection_method": "send_invoice", "created": 1640123796, "currency": "usd", "custom_fields": null, "customer": "cus_Kou8knsO3qQOwU", "customer_address": null, "customer_email": "edward.gao+stripe-test-customer-1@airbyte.io", "customer_name": "edgao-test-customer-1", "customer_phone": null, "customer_shipping": null, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "discounts": [], "due_date": 1688750070.0, "effective_at": 1686158070, "ending_balance": 0, "footer": null, "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDgwODE4MzQ00200vG3gv95N?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDgwODE4MzQ00200vG3gv95N/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1K9GK0EcXtiJtvvhSo2LvGqT/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "CA35DF83-0001", "on_behalf_of": null, "paid": true, "paid_out_of_band": false, "payment_intent": null, "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1640123795.0, "period_start": 1640123795.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 8400, "quote": null, "receipt_number": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "paid", "status_transitions": {"finalized_at": 1686158070, "marked_uncollectible_at": null, "paid_at": 1686158100, "voided_at": null}, "subscription": null, "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax": null, "test_clock": null, "total": 8400, "total_discount_amounts": [], "total_excluding_tax": 8400, "total_tax_amounts": [], "transfer_data": null, "webhooks_delivered_at": 1640123796.0}, "emitted_at": 1690277544798} +{"stream": "payment_intents", "data": {"id": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "object": "payment_intent", "amount": 5300, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 5300, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9FSOEcXtiJtvvh0AEIFllC_secret_uPUtIaSltgtW0qK7mLD0uF2Mr", "confirmation_method": "automatic", "created": 1640120472, "currency": "usd", "customer": null, "description": null, "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689691099579} +{"stream": "payment_intents", "data": {"id": "pi_3K9F5DEcXtiJtvvh16scJMp6", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 4200, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F5DEcXtiJtvvh16scJMp6_secret_YwhzCTpXtfcKYeklXnPnysRRi", "confirmation_method": "automatic", "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689691099579} +{"stream": "payment_intents", "data": {"id": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 0, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F4mEcXtiJtvvh18NKhEuo_secret_pfUt7CTkPjVdJacycm0bMpdLt", "confirmation_method": "automatic", "created": 1640119008, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": {"charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "code": "card_declined", "decline_code": "test_mode_live_card", "doc_url": "https://stripe.com/docs/error-codes/card-declined", "message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "unchecked", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119003, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "type": "card_error"}, "latest_charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "requires_payment_method", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689691099580} +{"stream": "payouts", "data": {"id": "po_1KVQhfEcXtiJtvvhZlUkl08U", "object": "payout", "amount": 9164, "arrival_date": 1645488000, "automatic": true, "balance_transaction": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "destination": "ba_1KUL7UEcXtiJtvvhAEUlStmv", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "completed", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": null, "status": "paid", "type": "bank_account"}, "emitted_at": 1689691101795} +{"stream": "payouts", "data": {"id": "po_1MXKoPEcXtiJtvvhwmqjvKoO", "object": "payout", "amount": 665880, "arrival_date": 1675382400, "automatic": false, "balance_transaction": "txn_1MXKoQEcXtiJtvvh65SHFZS6", "created": 1675413601, "currency": "usd", "description": "", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1689691102064} +{"stream": "payouts", "data": {"id": "po_1MWHzjEcXtiJtvvhIdUQHhLq", "object": "payout", "amount": 154900, "arrival_date": 1675123200, "automatic": false, "balance_transaction": "txn_1MWHzjEcXtiJtvvhtydemd2Y", "created": 1675164443, "currency": "usd", "description": "Test", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": "issuing", "source_type": "issuing", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1689691102066} +{"stream": "plans", "data": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "emitted_at": 1689691103696} +{"stream": "prices", "data": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600.0, "unit_amount_decimal": "12600"}, "emitted_at": 1690480900454} +{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700.0, "unit_amount_decimal": "1700"}, "emitted_at": 1690480900634} +{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000.0, "unit_amount_decimal": "2000"}, "emitted_at": 1690480900634} +{"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1675345058, "url": null}, "emitted_at": 1689684235151} +{"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1675345505, "url": null}, "emitted_at": 1689684235408} +{"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1674209524, "url": null}, "emitted_at": 1689684235411} +{"stream": "promotion_codes", "data": {"id": "promo_1MVtmyEcXtiJtvvhkV5jPFPU", "object": "promotion_code", "active": true, "code": "g20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071396, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": false, "minimum_amount": null, "minimum_amount_currency": null}, "times_redeemed": 0}, "emitted_at": 1689691105334} +{"stream": "promotion_codes", "data": {"id": "promo_1MVtmkEcXtiJtvvht0RA3MKg", "object": "promotion_code", "active": true, "code": "FRIENDS20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071382, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": true, "minimum_amount": 10000, "minimum_amount_currency": "usd"}, "times_redeemed": 0}, "emitted_at": 1689691105335} +{"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1689691106971} +{"stream": "subscription_items", "data": {"id": "si_O2toUlN7ELjLcM", "object": "subscription_item", "billing_thresholds": null, "created": 1686250591, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1NGo0YEcXtiJtvvh9rKhuT2H", "tax_rates": []}, "emitted_at": 1689691109493} +{"stream": "subscriptions", "data": {"id": "sub_1NGo0YEcXtiJtvvh9rKhuT2H", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": 1686250590.0, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": null, "cancellation_details": {"comment": null, "feedback": null, "reason": null}, "collection_method": "charge_automatically", "created": 1686250590, "currency": "usd", "current_period_end": 1691520990.0, "current_period_start": 1688842590, "customer": "cus_O2topVBsfeTMXg", "days_until_due": null, "default_payment_method": "pm_1NGo0XEcXtiJtvvhZosRvz8G", "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": null, "items": {"object": "list", "data": [{"id": "si_O2toUlN7ELjLcM", "object": "subscription_item", "billing_thresholds": null, "created": 1686250591, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1NGo0YEcXtiJtvvh9rKhuT2H", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1NGo0YEcXtiJtvvh9rKhuT2H"}, "latest_invoice": "in_1NRgM1EcXtiJtvvhQT20qPcf", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": "off"}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": null, "start_date": 1686250590, "status": "active", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null}, "emitted_at": 1689691110777} +{"stream": "subscription_schedule", "data": {"id": "sub_sched_1NGRenEcXtiJtvvhmM4eGaJN", "object": "subscription_schedule", "application": null, "canceled_at": null, "completed_at": null, "created": 1686164673, "current_phase": {"end_date": 1717787073, "start_date": 1686164673}, "customer": "cus_NGoTFiJFVbSsvZ", "default_settings": {"application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": "automatic", "billing_thresholds": null, "collection_method": "charge_automatically", "default_payment_method": null, "default_source": null, "description": null, "invoice_settings": null, "on_behalf_of": null, "transfer_data": null}, "end_behavior": "cancel", "livemode": false, "metadata": {}, "phases": [{"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": null, "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": null, "end_date": 1717787073, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1686164673, "transfer_data": null, "trial_end": null}, {"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": null, "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": null, "end_date": 1749323073, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1717787073, "transfer_data": null, "trial_end": null}], "released_at": null, "released_subscription": null, "renewal_interval": null, "status": "active", "subscription": "sub_1NGReoEcXtiJtvvhq6goDeqt", "test_clock": null}, "emitted_at": 1689691112514} +{"stream": "setup_intents", "data": {"id": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIjEcXtiJtvvhPw5znVKY_secret_LUebPsqMz6AF4ivxIg4LMaAT0OdZF5L", "created": 1649752937, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689691114375} +{"stream": "setup_intents", "data": {"id": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIcEcXtiJtvvh61qlCaDf_secret_LUebcbyw8V1e8Pxk3aAjzDXMOXdFMCe", "created": 1649752930, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689691114377} +{"stream": "setup_intents", "data": {"id": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIVEcXtiJtvvhWiIbMkpH_secret_LUebIUsiFnm75EzDUzf2RLhJ9WQ92Dp", "created": 1649752923, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689691114378} +{"stream": "shipping_rates", "data": {"id": "shr_1NXgplEcXtiJtvvhA1ntV782", "object": "shipping_rate", "active": true, "created": 1690274589, "delivery_estimate": "{'maximum': {'unit': 'business_day', 'value': 14}, 'minimum': {'unit': 'business_day', 'value': 10}}", "display_name": "Test Ground Shipping", "fixed_amount": {"amount": 999, "currency": "usd"}, "livemode": false, "metadata": {}, "tax_behavior": "inclusive", "tax_code": "txcd_92010001", "type": "fixed_amount"}, "emitted_at": 1690274706704} +{"stream": "credit_notes", "data": {"id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", "object": "credit_note", "amount": 8400, "amount_shipping": 0, "created": 1686158100, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "customer_balance_transaction": null, "discount_amount": "0", "discount_amounts": [], "effective_at": 1686158100, "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "lines": {"object": "list", "data": [{"id": "cnli_1NGPwmEcXtiJtvvhcL7yEIBJ", "object": "credit_note_line_item", "amount": 8400, "amount_excluding_tax": 8400, "description": "a box of parsnips", "discount_amount": 0, "discount_amounts": [], "invoice_line_item": "il_1K9GKLEcXtiJtvvhhHaYMebN", "livemode": false, "quantity": 1, "tax_amounts": [], "tax_rates": [], "type": "invoice_line_item", "unit_amount": 8400, "unit_amount_decimal": 8400.0, "unit_amount_excluding_tax": 8400.0}], "has_more": false, "url": "/v1/credit_notes/cn_1NGPwmEcXtiJtvvhNXwHpgJF/lines"}, "livemode": false, "memo": null, "metadata": {}, "number": "CA35DF83-0001-CN-01", "out_of_band_amount": null, "pdf": "https://pay.stripe.com/credit_notes/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9PMlV3dFlJelh4NHM1R0VIWnhMR3RjWUtlejFlRWtILDgwODIyOTk50200TyP0Z9BQ/pdf?s=ap", "reason": null, "refund": null, "shipping_cost": null, "status": "issued", "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax_amounts": [], "total": 8400, "total_excluding_tax": 8400, "type": "pre_payment", "voided_at": null}, "emitted_at": 1690282199229} +{"stream": "top_ups", "data": {"id": "tu_1MXKmvEcXtiJtvvhAbDiH3sm", "object": "topup", "amount": 100000, "balance_transaction": null, "created": 1675413509, "currency": "usd", "description": "Test test", "destination_balance": null, "expected_availability_date": 1675413509, "failure_code": "R03", "failure_message": "no_account", "livemode": false, "metadata": {}, "source": {"id": "src_1MXKmvEcXtiJtvvhV0fY3ZBF", "object": "source", "ach_debit": {"bank_name": "STRIPE TEST BANK", "country": "US", "fingerprint": "KUgiD3MWWaMMufNe", "last4": "1116", "routing_number": "110000000", "type": "individual"}, "amount": null, "client_secret": "src_client_secret_qghd0H1WkqAuSDuJBrYBSfD6", "code_verification": {"attempts_remaining": 3, "status": "succeeded"}, "created": 1675413509, "currency": "usd", "flow": "code_verification", "livemode": false, "metadata": {}, "owner": {"address": {"city": null, "country": "US", "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": "Jenny Rosen", "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "chargeable", "type": "ach_debit", "usage": "reusable"}, "statement_descriptor": "Test", "status": "failed", "transfer_group": null}, "emitted_at": 1689684238843} +{"stream": "files", "data": {"id": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "object": "file", "created": 1637224506, "expires_at": null, "filename": "1200x1200 logo.png", "links": {"object": "list", "data": [{"id": "link_1Jx65KEcXtiJtvvhtzDKT46T", "object": "file_link", "created": 1637224510, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfS0VrR1lkejVjaTFlNmpuMUNHYmFkYmVT00UG0pCDcI"}, {"id": "link_1Jx65HEcXtiJtvvhJxpyHQyb", "object": "file_link", "created": 1637224507, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdTlISjJ4UzM3QUlVa3M4cDdGRm9HWEUz00YHXUch3u"}], "has_more": false, "url": "/v1/file_links?file=file_1Jx65GEcXtiJtvvhxZSXTW0X"}, "purpose": "business_logo", "size": 188116, "title": null, "type": "png", "url": "https://files.stripe.com/v1/files/file_1Jx65GEcXtiJtvvhxZSXTW0X/contents"}, "emitted_at": 1689684229233} +{"stream": "file_links", "data": {"id": "link_1KnfIiEcXtiJtvvhCNceSyei", "object": "file_link", "created": 1649752936, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfY1FvanBFTmt0dUdrRWJXTHBpUlVYVUtu007305bsv3"}, "emitted_at": 1689684236617} +{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "object": "setup_attempt", "application": null, "created": 1649752937, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689684241319} +{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "object": "setup_attempt", "application": null, "created": 1649752931, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689684242319} +{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "object": "setup_attempt", "application": null, "created": 1649752923, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689684243299} +{"stream": "usage_records", "data": {"id": "sis_1NVDGQEcXtiJtvvhMFxjewz0", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2toUlN7ELjLcM", "total_usage": 1}, "emitted_at": 1689684267054} +{"stream": "usage_records", "data": {"id": "sis_1NVDGREcXtiJtvvhoIIYCISg", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2WiEt5xsnTylQ", "total_usage": 1}, "emitted_at": 1689684267303} +{"stream": "usage_records", "data": {"id": "sis_1NVDGREcXtiJtvvhlkyGWg6j", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2WdoFASN6MHzF", "total_usage": 1}, "emitted_at": 1689684267550} +{"stream": "usage_records", "data": {"id": "sis_1NVDGREcXtiJtvvhx53SMf1G", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2WZCuG0IxqQHl", "total_usage": 1}, "emitted_at": 1689684267794} +{"stream": "transfer_reversals", "data": {"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}, "emitted_at": 1689684270142} diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index ec8dca953924..e46be4d24e8f 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -5,11 +5,11 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 3.6.0 + dockerImageTag: 3.17.4 dockerRepository: airbyte/source-stripe githubIssueLabel: source-stripe icon: stripe.svg - license: MIT + license: ELv2 name: Stripe registries: cloud: @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/stripe tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-stripe/requirements.txt b/airbyte-integrations/connectors/source-stripe/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-stripe/requirements.txt +++ b/airbyte-integrations/connectors/source-stripe/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-stripe/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-stripe/sample_files/configured_catalog.json index 991ffce7065f..7e23659decc7 100644 --- a/airbyte-integrations/connectors/source-stripe/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/sample_files/configured_catalog.json @@ -3931,6 +3931,73 @@ "destination_sync_mode": "overwrite", "cursor_field": ["created"] }, + { + "stream": { + "name": "shipping_rates", + "json_schema": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "integer"] + }, + "delivery_estimate": {}, + "display_name": { + "type": ["null", "string"] + }, + "fixed_amount": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "tax_behavior": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["created"], + "primary_key": [["id"]] + }, { "stream": { "name": "transfers", diff --git a/airbyte-integrations/connectors/source-stripe/sample_files/state.json b/airbyte-integrations/connectors/source-stripe/sample_files/state.json index a7d63da80fd8..a3d34d59ba76 100644 --- a/airbyte-integrations/connectors/source-stripe/sample_files/state.json +++ b/airbyte-integrations/connectors/source-stripe/sample_files/state.json @@ -7,6 +7,7 @@ "invoices": { "created": 1617490175 }, "invoice_items": { "date": 1594946981 }, "transfers": { "created": 1610995824 }, + "shipping_rates": { "created": 1599686873 }, "subscriptions": { "created": 1599686873 }, "subscription_schedule": { "created": 1678454575 }, "balance_transactions": { "created": 1617067556 }, diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index 759389223cce..2ec69eca75f0 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -8,10 +8,10 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "stripe==2.56.0", "pendulum==2.1.2"] TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock", "requests_mock~=1.8", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/__init__.py b/airbyte-integrations/connectors/source-stripe/source_stripe/__init__.py index 1b13a124e63e..06387f3072c5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/__init__.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from .source import SourceStripe __all__ = ["SourceStripe"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py index 41189ed0952c..3d8caf860ee5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py @@ -8,6 +8,35 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from requests import HTTPError + +STRIPE_ERROR_CODES = { + "more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate", + "account_invalid": "The card, or account the card is connected to, is invalid. You need to contact your card issuer " + "to check that the card is working correctly.", + "oauth_not_supported": "Please use a different authentication method.", +} + + +class StripeAvailabilityStrategy(HttpAvailabilityStrategy): + def handle_http_error( + self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError + ) -> Tuple[bool, Optional[str]]: + status_code = error.response.status_code + if status_code not in [400, 403]: + raise error + parsed_error = error.response.json() + error_code = parsed_error.get("error", {}).get("code") + error_message = STRIPE_ERROR_CODES.get(error_code, parsed_error.get("error", {}).get("message")) + if not error_message: + raise error + doc_ref = self._visit_docs_message(logger, source) + reason = f"The endpoint {error.response.url} returned {status_code}: {error.response.reason}. {error_message}. {doc_ref} " + response_error_message = stream.parse_response_error_message(error.response) + if response_error_message: + reason += response_error_message + return False, reason class StripeSubStreamAvailabilityStrategy(HttpAvailabilityStrategy): diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json index 1b7769a88a73..b1a68dbcb95d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/accounts.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "additionalProperties": true, "type": "object", "properties": { @@ -16,27 +16,7 @@ "type": ["null", "string"] }, "support_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "support_email": { "type": ["null", "string"] @@ -132,28 +112,7 @@ "additionalProperties": true, "properties": { "address": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "address_kana": { "type": ["null", "object"], @@ -338,28 +297,7 @@ "type": ["null", "string"] }, "address": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "address_kana": { "type": ["null", "object"], @@ -549,28 +487,7 @@ "type": ["null", "string"] }, "registered_address": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "relationship": { "type": ["null", "object"], @@ -822,6 +739,89 @@ "type": { "enum": ["custom", "express", "standard"], "type": ["null", "string"] + }, + "future_requirements": { + "type": ["null", "object"], + "properties": { + "alternatives": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "alternative_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "original_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + }, + "current_deadline": { + "type": ["null", "integer"] + }, + "currently_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "disabled_reason": { + "type": ["null", "string"] + }, + "errors": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "requirement": { + "type": ["null", "string"] + } + } + } + }, + "eventually_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "past_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "pending_verification": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "controller": { + "type": ["null", "object"], + "properties": { + "is_controller": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json new file mode 100644 index 000000000000..861a99722d0a --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json @@ -0,0 +1,264 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + }, + "approved": { + "type": ["null", "boolean"] + }, + "authorization_method": { + "type": ["null", "string"] + }, + "balance_transactions": { + "items": { + "type": ["null", "object"], + "$ref": "balance_transactions.json" + }, + "type": ["null", "array"] + }, + "card": { + "$ref": "card.json" + }, + "cardholder": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "merchant_amount": { + "type": ["null", "integer"] + }, + "merchant_currency": { + "type": ["null", "string"] + }, + "merchant_data": { + "type": ["null", "object"], + "properties": { + "category": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "network_id": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "metadata": { + "type": ["null", "object"], + "additionalProperties": true + }, + "object": { + "type": ["null", "string"] + }, + "pending_request": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + }, + "currency": { + "type": ["null", "string"] + }, + "is_amount_controllable": { + "type": ["null", "boolean"] + }, + "merchant_amount": { + "type": ["null", "integer"] + }, + "merchant_currency": { + "type": ["null", "string"] + } + } + }, + "request_history": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + }, + "approved": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "merchant_amount": { + "type": ["null", "integer"] + }, + "merchant_currency": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + } + } + } + }, + "status": { + "type": ["null", "string"] + }, + "transactions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + }, + "authorization": { + "type": ["null", "string"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "string"] + }, + "cardholder": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "dispute": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "merchant_amount": { + "type": ["null", "integer"] + }, + "merchant_currency": { + "type": ["null", "string"] + }, + "merchant_data": { + "type": ["null", "object"], + "properties": { + "category": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "network_id": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "metadata": { + "type": ["null", "object"], + "additionalProperties": true + }, + "object": { + "type": ["null", "string"] + }, + "purchase_details": { + "$ref": "issuing_transaction_purchase_details.json" + } + } + } + }, + "verification_data": { + "type": ["null", "object"], + "properties": { + "address_line1_check": { + "type": ["null", "string"] + }, + "address_postal_code_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "expiry_check": { + "type": ["null", "string"] + } + } + }, + "wallet": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/balance_transactions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/balance_transactions.json index 432a3a37ce62..b70c584a0f7a 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/balance_transactions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/balance_transactions.json @@ -1,71 +1,5 @@ { - "type": ["null", "object"], - "properties": { - "fee": { - "type": ["null", "integer"] - }, - "currency": { - "type": ["null", "string"] - }, - "source": { - "type": ["null", "string"] - }, - "fee_details": { - "type": ["null", "array"], - "items": { - "properties": { - "application": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "integer"] - }, - "currency": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - } - }, - "available_on": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "net": { - "type": ["null", "integer"] - }, - "exchange_rate": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "sourced_transfers": { - "items": {}, - "type": ["null", "array"] - }, - "id": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "integer"] - }, - "amount": { - "type": ["null", "integer"] - } - } + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "$ref": "balance_transactions.json" } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/cardholders.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/cardholders.json new file mode 100644 index 000000000000..239c9ef8986b --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/cardholders.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "$ref": "cardholder.json" +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/cards.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/cards.json new file mode 100644 index 000000000000..5571c6fbc590 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/cards.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "$ref": "card.json" +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json index ca2736b1a8b6..0a6bb196269d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json @@ -155,27 +155,7 @@ "type": ["null", "string"] }, "address": { - "type": ["null", "object"], - "properties": { - "line2": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "verified_email": { "type": ["null", "string"] @@ -677,27 +657,7 @@ "type": ["null", "object"], "properties": { "billing_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "email": { "type": ["null", "string"] @@ -706,27 +666,7 @@ "type": ["null", "string"] }, "shipping_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" } } }, @@ -741,27 +681,7 @@ "type": ["null", "object"], "properties": { "billing_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" }, "email": { "type": ["null", "string"] @@ -770,27 +690,7 @@ "type": ["null", "string"] }, "shipping_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } + "$ref": "address.json" } } } @@ -1058,6 +958,84 @@ }, "description": { "type": ["null", "string"] + }, + "statement_descriptor_suffix": { + "type": ["null", "string"] + }, + "calculated_statement_descriptor": { + "type": ["null", "string"] + }, + "receipt_url": { + "type": ["null", "string"] + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "billing_details": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "failure_balance_transaction": { + "type": ["null", "string"] + }, + "amount_captured": { + "type": ["null", "integer"] + }, + "application_fee_amount": { + "type": ["null", "integer"] + }, + "amount_updates": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "payment_method": { + "type": ["null", "string"] + }, + "disputed": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json index 18e90e928a95..0d5ee3731c4c 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json @@ -131,15 +131,7 @@ "type": ["null", "object"], "properties": { "address": { - "type": ["null", "object"], - "properties": { - "city": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] }, - "line1": { "type": ["null", "string"] }, - "line2": { "type": ["null", "string"] }, - "postal_code": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] } - } + "$ref": "address.json" }, "name": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json new file mode 100644 index 000000000000..b5f962fc3101 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json @@ -0,0 +1,346 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "amount_shipping": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "customer_balance_transaction": { + "type": ["null", "string"] + }, + "discount_amount": { + "type": ["null", "string"] + }, + "discount_amounts": { + "type": ["null", "array"] + }, + "invoice": { + "type": ["null", "string"] + }, + "lines": { + "type": ["null", "object"], + "properties": { + "object": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "amount_excluding_tax": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "discount_amount": { + "type": ["null", "integer"] + }, + "discount_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "discount": { + "type": ["null", "string"] + } + } + } + }, + "invoice_line_item": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "tax_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "tax_rate": { + "type": ["null", "string"] + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } + }, + "tax_rates": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "country": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "display_name": { + "type": ["null", "string"] + }, + "effective_percentage": { + "type": ["null", "number"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "jurisdiction": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"] + }, + "percentage": { + "type": ["null", "number"] + }, + "state": { + "type": ["null", "string"] + }, + "tax_type": { + "type": ["null", "string"] + } + } + } + }, + "type": { + "type": ["null", "string"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "unit_amount_decimal": { + "type": ["null", "number"] + }, + "unit_amount_excluding_tax": { + "type": ["null", "number"] + } + } + } + } + }, + "has_more": { + "type": ["null", "boolean"] + }, + "url": { + "type": ["null", "string"] + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "memo": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "number": { + "type": ["null", "string"] + }, + "out_of_band_amount": { + "type": ["null", "integer"] + }, + "pdf": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "refund": { + "type": ["null", "string"] + }, + "shipping_cost": { + "type": ["null", "object"], + "properties": { + "amount_subtotal": { + "type": ["null", "integer"] + }, + "amount_tax": { + "type": ["null", "integer"] + }, + "amount_total": { + "type": ["null", "integer"] + }, + "shipping_rate": { + "type": ["null", "string"] + }, + "taxes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "rate": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "country": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "display_name": { + "type": ["null", "string"] + }, + "effective_percentage": { + "type": ["null", "number"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "jurisdiction": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "boolean"] + }, + "percentage": { + "type": ["null", "number"] + }, + "state": { + "type": ["null", "string"] + }, + "tax_type": { + "type": ["null", "string"] + } + } + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "status": { + "type": ["null", "string"] + }, + "subtotal": { + "type": ["null", "integer"] + }, + "subtotal_excluding_tax": { + "type": ["null", "integer"] + }, + "tax_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "tax_rate": { + "type": ["null", "string"] + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } + }, + "total": { + "type": ["null", "integer"] + }, + "total_excluding_tax": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "voided_at": { + "type": ["null", "integer"] + }, + "effective_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/customers.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/customers.json index 576d68592e02..10fea6bb44a3 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/customers.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/customers.json @@ -1,920 +1,5 @@ { - "type": ["null", "object"], - "properties": { - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "preferred_locales": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "invoice_settings": { - "type": ["null", "object"], - "properties": { - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "default_payment_method": { - "type": ["null", "string"] - }, - "footer": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "tax_exempt": { - "type": ["null", "string"] - }, - "next_invoice_sequence": { - "type": ["null", "integer"] - }, - "balance": { - "type": ["null", "integer"] - }, - "phone": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "shipping": { - "type": ["null", "object"], - "properties": { - "address": { - "type": ["null", "object"], - "properties": { - "line2": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - } - } - }, - "sources": { - "anyOf": [ - { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "type": { - "type": ["null", "string"] - }, - "address_zip": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "card": { - "type": ["null", "object"], - "properties": { - "fingerprint": { - "type": ["null", "string"] - }, - "last4": { - "type": ["null", "string"] - }, - "dynamic_last4": { - "type": ["null", "string"] - }, - "address_line1_check": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "tokenization_method": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "three_d_secure": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "brand": { - "type": ["null", "string"] - }, - "cvc_check": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "address_zip_check": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - } - }, - "statement_descriptor": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "address_country": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "dynamic_last4": { - "type": ["null", "string"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "last4": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "brand": { - "type": ["null", "string"] - }, - "address_line2": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "integer"] - }, - "cvc_check": { - "type": ["null", "string"] - }, - "usage": { - "type": ["null", "string"] - }, - "address_line1": { - "type": ["null", "string"] - }, - "owner": { - "type": ["null", "object"], - "properties": { - "verified_address": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "object"], - "properties": { - "line2": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - } - } - }, - "verified_email": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "verified_name": { - "type": ["null", "string"] - }, - "verified_phone": { - "type": ["null", "string"] - } - } - }, - "tokenization_method": { - "type": ["null", "string"] - }, - "client_secret": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "address_city": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "address_line1_check": { - "type": ["null", "string"] - }, - "receiver": { - "type": ["null", "object"], - "properties": { - "refund_attributes_method": { - "type": ["null", "string"] - }, - "amount_returned": { - "type": ["null", "integer"] - }, - "amount_received": { - "type": ["null", "integer"] - }, - "refund_attributes_status": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "amount_charged": { - "type": ["null", "integer"] - } - } - }, - "flow": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "ach_credit_transfer": { - "type": ["null", "object"], - "properties": { - "bank_name": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "routing_number": { - "type": ["null", "string"] - }, - "swift_code": { - "type": ["null", "string"] - }, - "refund_account_holder_type": { - "type": ["null", "string"] - }, - "refund_account_holder_name": { - "type": ["null", "string"] - }, - "refund_account_number": { - "type": ["null", "string"] - }, - "refund_routing_number": { - "type": ["null", "string"] - }, - "account_number": { - "type": ["null", "string"] - } - } - }, - "customer": { - "type": ["null", "string"] - }, - "address_zip_check": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "integer"] - }, - "address_state": { - "type": ["null", "string"] - }, - "alipay": { - "type": ["null", "object"], - "properties": {} - }, - "bancontact": { - "type": ["null", "object"], - "properties": {} - }, - "eps": { - "type": ["null", "object"], - "properties": {} - }, - "ideal": { - "type": ["null", "object"], - "properties": {} - }, - "multibanco": { - "type": ["null", "object"], - "properties": {} - }, - "redirect": { - "type": ["null", "object"], - "properties": { - "failure_reason": { - "type": ["null", "string"] - }, - "return_url": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - } - } - } - }, - { - "type": ["null", "object"], - "properties": { - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "type": { - "type": ["null", "string"] - }, - "address_zip": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "card": { - "type": ["null", "object"], - "properties": { - "fingerprint": { - "type": ["null", "string"] - }, - "last4": { - "type": ["null", "string"] - }, - "dynamic_last4": { - "type": ["null", "string"] - }, - "address_line1_check": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "tokenization_method": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "three_d_secure": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "brand": { - "type": ["null", "string"] - }, - "cvc_check": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "address_zip_check": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - } - }, - "statement_descriptor": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "address_country": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "dynamic_last4": { - "type": ["null", "string"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "last4": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "brand": { - "type": ["null", "string"] - }, - "address_line2": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "integer"] - }, - "cvc_check": { - "type": ["null", "string"] - }, - "usage": { - "type": ["null", "string"] - }, - "address_line1": { - "type": ["null", "string"] - }, - "owner": { - "type": ["null", "object"], - "properties": { - "verified_address": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "object"], - "properties": { - "line2": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - } - } - }, - "verified_email": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "verified_name": { - "type": ["null", "string"] - }, - "verified_phone": { - "type": ["null", "string"] - } - } - }, - "tokenization_method": { - "type": ["null", "string"] - }, - "client_secret": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "address_city": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "address_line1_check": { - "type": ["null", "string"] - }, - "receiver": { - "type": ["null", "object"], - "properties": { - "refund_attributes_method": { - "type": ["null", "string"] - }, - "amount_returned": { - "type": ["null", "integer"] - }, - "amount_received": { - "type": ["null", "integer"] - }, - "refund_attributes_status": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "amount_charged": { - "type": ["null", "integer"] - } - } - }, - "flow": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "ach_credit_transfer": { - "type": ["null", "object"], - "properties": { - "bank_name": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "routing_number": { - "type": ["null", "string"] - }, - "swift_code": { - "type": ["null", "string"] - }, - "refund_account_holder_type": { - "type": ["null", "string"] - }, - "refund_account_holder_name": { - "type": ["null", "string"] - }, - "refund_account_number": { - "type": ["null", "string"] - }, - "refund_routing_number": { - "type": ["null", "string"] - }, - "account_number": { - "type": ["null", "string"] - } - } - }, - "customer": { - "type": ["null", "string"] - }, - "address_zip_check": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "integer"] - }, - "address_state": { - "type": ["null", "string"] - }, - "alipay": { - "type": ["null", "object"], - "properties": {} - }, - "bancontact": { - "type": ["null", "object"], - "properties": {} - }, - "eps": { - "type": ["null", "object"], - "properties": {} - }, - "ideal": { - "type": ["null", "object"], - "properties": {} - }, - "multibanco": { - "type": ["null", "object"], - "properties": {} - }, - "redirect": { - "type": ["null", "object"], - "properties": { - "failure_reason": { - "type": ["null", "string"] - }, - "return_url": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - } - } - } - ] - }, - "delinquent": { - "type": ["null", "boolean"] - }, - "description": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "default_source": { - "type": ["null", "string"] - }, - "cards": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "object": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "dynamic_last4": { - "type": ["null", "string"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "last4": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "brand": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "string"] - }, - "cvc_check": { - "type": ["null", "string"] - }, - "address_line2": { - "type": ["null", "string"] - }, - "address_line1": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "address_zip": { - "type": ["null", "string"] - }, - "address_city": { - "type": ["null", "string"] - }, - "address_country": { - "type": ["null", "string"] - }, - "address_line1_check": { - "type": ["null", "string"] - }, - "tokenization_method": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address_state": { - "type": ["null", "string"] - }, - "address_zip_check": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - } - } - }, - "email": { - "type": ["null", "string"] - }, - "default_card": { - "type": ["null", "string"] - }, - "subscriptions": { - "type": ["null", "object"], - "properties": { - "object": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "array"] - }, - "has_more": { - "type": ["null", "boolean"] - }, - "total_count": { - "type": ["null", "number"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "discount": { - "type": ["null", "object"], - "properties": { - "end": { - "type": ["null", "integer"] - }, - "coupon": { - "type": ["null", "object"], - "properties": { - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "valid": { - "type": ["null", "boolean"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "amount_off": { - "type": ["null", "integer"] - }, - "redeem_by": { - "type": ["null", "integer"] - }, - "duration_in_months": { - "type": ["null", "integer"] - }, - "percent_off_precise": { - "type": ["null", "number"] - }, - "max_redemptions": { - "type": ["null", "integer"] - }, - "currency": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "times_redeemed": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "percent_off": { - "type": ["null", "number"] - }, - "created": { - "type": ["null", "integer"] - } - } - }, - "customer": { - "type": ["null", "string"] - }, - "start": { - "type": ["null", "integer"] - }, - "object": { - "type": ["null", "string"] - }, - "subscription": { - "type": ["null", "string"] - } - } - }, - "account_balance": { - "type": ["null", "integer"] - }, - "currency": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "invoice_prefix": { - "type": ["null", "string"] - }, - "tax_info_verification": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "integer"] - }, - "tax_info": { - "type": ["null", "string"] - } - } + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "$ref": "customer.json" } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json index 4c0f997ed4a3..b627b72dd7e0 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json @@ -148,6 +148,12 @@ }, "status": { "type": ["null", "string"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "balance_transaction": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/file_links.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/file_links.json new file mode 100644 index 000000000000..7884f7bad3d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/file_links.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "expires_at": { + "type": ["null", "integer"] + }, + "file": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "url": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "expired": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/files.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/files.json new file mode 100644 index 000000000000..b13f3edfb0f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/files.json @@ -0,0 +1,82 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "purpose": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "expires_at": { + "type": ["null", "integer"] + }, + "filename": { + "type": ["null", "string"] + }, + "links": { + "type": ["null", "object"], + "properties": { + "object": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "expired": { + "type": ["null", "integer"] + }, + "expires_at": { + "type": ["null", "integer"] + }, + "file": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"] + }, + "url": { + "type": ["null", "string"] + } + } + } + }, + "has_more": { "type": ["null", "boolean"] }, + "url": { "type": ["null", "string"] } + } + }, + "size": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json index 0f301c0e28d8..0c2f07854952 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json @@ -105,7 +105,7 @@ "type": ["null", "object"], "properties": { "end": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "start": { "type": ["null", "integer"] @@ -150,6 +150,24 @@ }, "subscription_item": { "type": ["null", "string"] + }, + "price": { + "$ref": "price.json" + }, + "test_clock": { + "type": ["null", "string"] + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "tax_rates": { + "$ref": "tax_rates.json" + }, + "unit_amount_decimal": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json index e679e6a7f2d6..3ade45cbe340 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json @@ -48,7 +48,7 @@ "type": ["null", "integer"] }, "end": { - "type": ["null", "number"] + "type": ["null", "integer"] } } }, @@ -149,6 +149,80 @@ }, "currency": { "type": ["null", "string"] + }, + "amount_excluding_tax": { + "type": ["null", "integer"] + }, + "unit_amount_excluding_tax": { + "type": ["null", "string"] + }, + "proration_details": { + "type": ["null", "object"], + "properties": { + "credited_items": { + "type": ["null", "object"], + "properties": { + "invoice": { + "type": ["null", "string"] + }, + "invoice_line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + }, + "price": { + "$ref": "price.json" + }, + "discount_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "discount": { + "type": ["null", "string"] + } + } + } + }, + "discounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "tax_rates": { + "$ref": "tax_rates.json" + }, + "tax_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "tax_rate": { + "type": ["null", "string"] + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json index 8c30c5f37a15..8189cefc7df7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json @@ -255,6 +255,337 @@ "type": ["null", "integer"] } } + }, + "post_payment_credit_notes_amount": { + "type": ["null", "integer"] + }, + "paid_out_of_band": { + "type": ["null", "boolean"] + }, + "total_discount_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "discount": { + "type": ["null", "string"] + } + } + } + }, + "customer_name": { + "type": ["null", "string"] + }, + "shipping_cost": { + "type": ["null", "object"], + "properties": { + "amount_subtotal": { + "type": ["null", "integer"] + }, + "amount_tax": { + "type": ["null", "integer"] + }, + "amount_total": { + "type": ["null", "integer"] + }, + "shipping_rate": { + "type": ["null", "string"] + }, + "taxes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "application_fee_amount": { + "type": ["null", "integer"] + }, + "customer_shipping": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "application": { + "type": ["null", "string"] + }, + "amount_shipping": { + "type": ["null", "integer"] + }, + "from_invoice": { + "type": ["null", "object"], + "properties": { + "actions": { + "type": ["null", "string"] + }, + "invoice": { + "type": ["null", "string"] + } + } + }, + "customer_tax_exempt": { + "type": ["null", "string"] + }, + "total_tax_amounts": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "tax_rate": { + "type": ["null", "string"] + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } + }, + "footer": { + "type": ["null", "string"] + }, + "test_clock": { + "type": ["null", "string"] + }, + "automatic_tax": { + "type": ["null", "object"], + "properties": { + "enabled": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + } + } + }, + "payment_settings": { + "type": ["null", "object"], + "properties": { + "default_mandate": { + "type": ["null", "string"] + }, + "payment_method_options": { + "type": ["null", "object"] + }, + "payment_method_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "default_source": { + "type": ["null", "string"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "default_payment_method": { + "type": ["null", "string"] + }, + "shipping_details": { + "type": ["null", "object"], + "properties": { + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "collection_method": { + "type": ["null", "string"] + }, + "effective_at": { + "type": ["null", "integer"] + }, + "default_tax_rates": { + "type": ["null", "array"], + "items": { + "$ref": "tax_rates.json" + } + }, + "total_excluding_tax": { + "type": ["null", "integer"] + }, + "subtotal_excluding_tax": { + "type": ["null", "integer"] + }, + "last_finalization_error": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "doc_url": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "param": { + "type": ["null", "string"] + }, + "payment_method_type": { + "type": ["null", "string"] + } + } + }, + "latest_revision": { + "type": ["null", "string"] + }, + "rendering_options": { + "type": ["null", "object"], + "properties": { + "amount_tax_display": { + "type": ["null", "string"] + } + } + }, + "quote": { + "type": ["null", "string"] + }, + "pre_payment_credit_notes_amount": { + "type": ["null", "integer"] + }, + "customer_phone": { + "type": ["null", "string"] + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "account_tax_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "customer_email": { + "type": ["null", "string"] + }, + "customer_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "account_name": { + "type": ["null", "string"] + }, + "account_country": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_intents.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_intents.json index 8f0b97caf851..c847ddf78692 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_intents.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_intents.json @@ -1,958 +1,5 @@ { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "integer"] - }, - "amount_capturable": { - "type": ["null", "integer"] - }, - "amount_received": { - "type": ["null", "integer"] - }, - "application": { - "type": ["null", "string"] - }, - "application_fee_amount": { - "type": ["null", "integer"] - }, - "canceled_at": { - "type": ["null", "integer"] - }, - "cancellation_reason": { - "type": ["null", "string"] - }, - "capture_method": { - "type": ["null", "string"], - "enum": ["automatic", "manual"] - }, - "charges": { - "type": ["null", "object"], - "properties": { - "object": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "array"] - }, - "has_more": { - "type": ["null", "boolean"] - }, - "total_count": { - "type": ["null", "integer"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "client_secret": { - "type": ["null", "string"] - }, - "confirmation_method": { - "type": ["null", "string"], - "enum": ["automatic", "manual"] - }, - "created": { - "type": ["null", "integer"] - }, - "currency": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "invoice": { - "type": ["null", "string"] - }, - "last_payment_error": { - "type": ["null", "object"], - "properties": { - "charge": { - "type": ["null", "string"] - }, - "code": { - "type": ["null", "string"] - }, - "decline_code": { - "type": ["null", "string"] - }, - "doc_url": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - }, - "param": { - "type": ["null", "string"] - }, - "payment_method": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "acss_debit": { - "type": ["null", "object"], - "properties": { - "bank_name": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "institution_number": { - "type": ["null", "string"] - }, - "last4": { - "type": ["null", "string"] - }, - "transit_number": { - "type": ["null", "string"] - } - } - }, - "afterpay_clearpay": { - "type": ["null", "string"] - }, - "alipay": { - "type": ["null", "string"] - }, - "au_becs_debit": { - "type": ["null", "object"], - "properties": { - "bsb_number": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "last4": { - "type": ["null", "string"] - } - } - }, - "bacs_debit": { - "type": ["null", "object"], - "properties": { - "fingerprint": { - "type": ["null", "string"] - }, - "last4": { - "type": ["null", "string"] - }, - "sort_code": { - "type": ["null", "string"] - } - } - }, - "bancontact": { - "type": ["null", "string"] - }, - "billing_details": { - "type": ["null", "object"], - "properties": { - "address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "email": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - } - } - }, - "boleto": { - "type": ["null", "object"], - "properties": { - "tax_id": { - "type": ["null", "string"] - } - } - }, - "card": { - "type": ["null", "object"], - "properties": { - "brand": { - "type": ["null", "string"] - }, - "checks": { - "type": ["null", "object"], - "properties": { - "address_line1_check": { - "type": ["null", "string"] - }, - "address_postal_code_check": { - "type": ["null", "string"] - }, - "cvc_check": { - "type": ["null", "string"] - } - } - }, - "country": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"] - }, - "generated_from": { - "type": ["null", "object"], - "properties": { - "charge": { - "type": ["null", "string"] - }, - "payment_method_details": { - "type": ["null", "object"], - "properties": { - "card_present": { - "type": ["null", "object"], - "properties": { - "brand": { - "type": ["null", "string"] - }, - "cardholder_name": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "emv_auth_data": { - "type": ["null", "string"] - }, - "exp_month": { - "type": ["null", "integer"] - }, - "exp_year": { - "type": ["null", "integer"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "funding": { - "type": ["null", "string"], - "enum": ["credit", "debit", "prepaid", "unknown"] - }, - "generated_card": { - "type": ["null", "string"] - }, - "lsat4": { - "type": ["null", "string"] - }, - "network": { - "type": ["null", "string"], - "enum": [ - "contact_emv", - "contactless_emv", - "magnetic_stripe_track2", - "magnetic_stripe_fallback", - "contactless_magstripe_mode" - ] - }, - "read_method": { - "type": ["null", "string"] - }, - "receipt": { - "type": ["null", "object"], - "properties": { - "account_type": { - "type": ["null", "string"], - "enum": [ - "credit", - "checking", - "prepaid", - "unknown" - ] - }, - "application_cryptogram": { - "type": ["null", "string"] - }, - "application_preferred_name": { - "type": ["null", "string"] - }, - "authorization_code": { - "type": ["null", "string"] - }, - "authorization_response_code": { - "type": ["null", "string"] - }, - "cardholder_verification_method": { - "type": ["null", "string"] - }, - "dedicated_file_name": { - "type": ["null", "string"] - }, - "terminal_verification_results": { - "type": ["null", "string"] - }, - "transaction_status_information": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": ["null", "string"] - } - } - }, - "setup_attempt": { - "type": ["null", "string"] - } - } - }, - "last4": { - "type": ["null", "string"] - }, - "networks": { - "type": ["null", "object"], - "properties": { - "available": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "preferred": { - "type": ["null", "string"] - } - } - }, - "three_d_secure_usage": { - "type": ["null", "object"], - "properties": { - "supported": { - "type": ["null", "boolean"] - } - } - }, - "wallet": { - "type": ["null", "object"], - "properties": { - "amex_express_checkout": { - "type": ["null", "string"] - }, - "apple_pay": { - "type": ["null", "string"] - }, - "dynamic_last4": { - "type": ["null", "string"] - }, - "google_pay": { - "type": ["null", "string"] - }, - "masterpass": { - "type": ["null", "object"], - "properties": { - "billing_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "email": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "shipping_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - } - } - }, - "samsung_pay": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "visa_checkout": { - "type": ["null", "object"], - "properties": { - "billing_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "email": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "shipping_address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - } - } - } - } - } - } - }, - "card_present": { - "type": ["null", "object"], - "properties": {} - }, - "created": { - "type": ["null", "integer"] - }, - "customer": { - "type": ["null", "string"] - }, - "eps": { - "type": ["null", "object"], - "properties": { - "bank": { - "type": ["null", "string"], - "enum": [ - "arzte_und_apotheker_bank", - "austrian_anadi_bank_ag", - "bank_austria", - "bankhaus_carl_spangler", - "bankhaus_schelhammer_und_schattera_ag", - "bawag_psk_ag", - "bks_bank_ag", - "brull_kallmus_bank_ag", - "btv_vier_lander_bank", - "capital_bank_grawe_gruppe_ag", - "dolomitenbank", - "easybank_ag", - "erste_bank_und_sparkassen", - "hypo_alpeadriabank_international_ag", - "hypo_noe_lb_fur_niederosterreich_u_wien", - "hypo_oberosterreich_salzburg_steiermark", - "hypo_tirol_bank_ag", - "hypo_vorarlberg_bank_ag", - "hypo_bank_burgenland_aktiengesellschaft", - "marchfelder_bank", - "oberbank_ag", - "raiffeisen_bankengruppe_osterreich", - "schoellerbank_ag", - "sparda_bank_wien", - "volksbank_gruppe", - "volkskreditbank_ag", - "vr_bank_braunau" - ] - } - } - }, - "fpx": { - "type": ["null", "object"], - "properties": { - "bank": { - "type": ["null", "string"], - "enum": [ - "affin_bank", - "alliance_bank", - "ambank", - "bank_islam", - "bank_muamalat", - "bank_rakyat", - "bsn", - "cimb", - "hong_leong_bank", - "hsbc", - "kfh", - "maybank2u", - "ocbc", - "public_bank", - "rhb", - "standard_chartered", - "uob", - "deutsche_bank", - "maybank2e", - "pb_enterprise" - ] - } - } - }, - "giropay": { - "type": ["null", "object"], - "properties": {} - }, - "grabpay": { - "type": ["null", "object"], - "properties": {} - }, - "ideal": { - "type": ["null", "object"], - "properties": { - "bank": { - "type": ["null", "string"], - "enum": [ - "abn_amro", - "asn_bank", - "bunq", - "handelsbanken", - "ing", - "knab", - "moneyou", - "rabobank", - "regiobank", - "revolut", - "sns_bank", - "triodos_bank", - "van_lanschot" - ] - } - } - }, - "interac_present": { - "type": ["null", "object"], - "properties": {} - }, - "livemode": { - "type": ["null", "boolean"] - }, - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "oxxo": { - "type": ["null", "object"], - "properties": {} - }, - "p24": { - "type": ["null", "object"], - "properties": { - "bank": { - "type": ["null", "string"] - } - } - }, - "sepa_debit": { - "type": ["null", "object"], - "properties": { - "bank_code": { - "type": ["null", "string"] - }, - "branch_code": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "fingerprint": { - "type": ["null", "string"] - }, - "generated_from": { - "type": ["null", "object"], - "properties": { - "charge": { - "type": ["null", "string"] - }, - "setup_attempt": { - "type": ["null", "string"] - } - } - }, - "last4": { - "type": ["null", "string"] - } - } - }, - "sofort": { - "type": ["null", "object"], - "properties": { - "country": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": ["null", "string"], - "enum": [ - "acss_debit", - "afterpay_clearpay", - "alipay", - "au_becs_debit", - "bacs_debit", - "bancontact", - "boleto", - "card", - "card_present", - "eps", - "fpx", - "giropay", - "grabpay", - "ideal", - "interac_present", - "oxxo", - "p24", - "sepa_debit", - "sofort", - "wechat_pay" - ] - }, - "wechat_pay": { - "type": ["null", "object"], - "properties": {} - } - } - }, - "payment_method_type": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"], - "enum": [ - "api_error", - "card_error", - "idempotency_error", - "invalid_request_error" - ] - } - } - }, - "livemode": { - "type": ["null", "boolean"] - }, - "metadata": { - "type": ["null", "object"], - "properties": {} - }, - "next_action": { - "type": ["null", "object"], - "properties": { - "alipay_handle_redirect": { - "type": ["null", "object"], - "properties": { - "native_data": { - "type": ["null", "string"] - }, - "native_url": { - "type": ["null", "string"] - }, - "return_url": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "boleto_display_details": { - "type": ["null", "object"], - "properties": { - "expires_at": { - "type": ["null", "integer"] - }, - "hosted_voucher_url": { - "type": ["null", "string"] - }, - "number": { - "type": ["null", "string"] - }, - "pdf": { - "type": ["null", "string"] - } - } - }, - "oxxo_display_details": { - "type": ["null", "object"], - "properties": { - "expires_after": { - "type": ["null", "integer"] - }, - "hosted_voucher_url": { - "type": ["null", "string"] - }, - "number": { - "type": ["null", "string"] - } - } - }, - "redirect_to_url": { - "type": ["null", "object"], - "properties": { - "return_url": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": ["null", "string"] - }, - "use_stripe_sdk": { - "type": ["null", "object"], - "properties": {} - }, - "verify_with_microdeposits": { - "type": ["null", "object"], - "properties": { - "arrival_date": { - "type": ["null", "integer"] - }, - "hosted_verification_url": { - "type": ["null", "string"] - } - } - }, - "wechat_pay_display_qr_code": { - "type": ["null", "object"], - "properties": { - "data": { - "type": ["null", "string"] - }, - "image_data_url": { - "type": ["null", "string"] - } - } - }, - "wechat_pay_redirect_to_android_app": { - "type": ["null", "object"], - "properties": { - "app_id": { - "type": ["null", "string"] - }, - "nonce_str": { - "type": ["null", "string"] - }, - "package": { - "type": ["null", "string"] - }, - "partner_id": { - "type": ["null", "string"] - }, - "prepay_id": { - "type": ["null", "string"] - }, - "sign": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - } - } - }, - "wechat_pay_redirect_to_ios_app": { - "type": ["null", "object"], - "properties": { - "native_url": { - "type": ["null", "string"] - } - } - } - } - }, - "on_behalf_of": { - "type": ["null", "string"] - }, - "payment_method": { - "type": ["null", "string"] - }, - "payment_method_options": { - "type": ["null", "object"], - "properties": {} - }, - "payment_method_types": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "receipt_email": { - "type": ["null", "string"] - }, - "review": { - "type": ["null", "string"] - }, - "setup_future_usage": { - "type": ["null", "string"] - }, - "shipping": { - "type": ["null", "object"], - "properties": { - "address": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "line1": { - "type": ["null", "string"] - }, - "line2": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "carrier": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "tracking_number": { - "type": ["null", "string"] - } - } - }, - "source": { - "type": ["null", "string"] - }, - "statement_description": { - "type": ["null", "string"] - }, - "statement_descriptor_suffix": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "transfer_data": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "integer"] - }, - "destination": { - "type": ["null", "string"] - } - } - }, - "transfer_group": { - "type": ["null", "string"] - } - } + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "$ref": "payment_intent.json" } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_methods.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_methods.json new file mode 100644 index 000000000000..e6ece32c7f63 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payment_methods.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "$ref": "payment_method.json" +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json index 49806969dc1b..a057a81ec4fb 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json @@ -123,6 +123,18 @@ }, "source_transaction": { "type": ["null", "string"] + }, + "original_payout": { + "type": ["null", "string"] + }, + "reconciliation_status": { + "type": ["null", "string"] + }, + "source_balance": { + "type": ["null", "string"] + }, + "reversed_by": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json new file mode 100644 index 000000000000..a409b35c1551 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json @@ -0,0 +1,422 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Persons Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address_kana": { + "type": ["null", "string"] + }, + "address_kanji": { + "type": ["null", "string"] + }, + "first_name_kana": { + "type": ["null", "string"] + }, + "gender": { + "type": ["null", "string"] + }, + "full_name_aliases": { + "type": ["null", "string"] + }, + "id_number_secondary_provided": { + "type": ["null", "string"] + }, + "first_name_kanji": { + "type": ["null", "string"] + }, + "nationality": { + "type": ["null", "string"] + }, + "political_exposure": { + "type": ["null", "string"] + }, + "registered_address": { + "type": ["null", "string"] + }, + "account": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "created": { + "type": ["null", "integer"] + }, + "dob": { + "type": ["null", "object"], + "properties": { + "day": { + "type": ["null", "integer"] + }, + "month": { + "type": ["null", "integer"] + }, + "year": { + "type": ["null", "integer"] + } + } + }, + "first_name": { + "type": ["null", "string"] + }, + "future_requirements": { + "type": ["null", "object"], + "properties": { + "alternatives": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + }, + "alternative_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "original_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + }, + "currently_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "errors": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "eventually_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "past_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "pending_verification": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "id_number_provided": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "id_number_provided": { + "type": ["null", "boolean"] + } + } + }, + "relationship": { + "type": ["null", "object"], + "properties": { + "director": { + "type": ["null", "boolean"] + }, + "executive": { + "type": ["null", "boolean"] + }, + "owner": { + "type": ["null", "boolean"] + }, + "percent_ownership": { + "type": ["null", "string"] + }, + "representative": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + } + } + }, + "requirements": { + "type": ["null", "object"], + "properties": { + "alternatives": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + }, + "alternative_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "original_fields_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + }, + "currently_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "errors": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "eventually_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "past_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + }, + "pending_verification": { + "type": ["null", "array"], + "items": { + "type": ["null", "string", "object"], + "properties": { + "flat_amount": { + "type": ["null", "integer"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "up_to": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "ssn_last_4_provided": { + "type": ["null", "boolean"] + }, + "verification": { + "type": ["null", "object"], + "properties": { + "additional_document": { + "type": ["null", "object"], + "properties": { + "back": { + "type": ["null", "string"] + }, + "details": { + "type": ["null", "string"] + }, + "details_code": { + "type": ["null", "string"] + }, + "front": { + "type": ["null", "string"] + } + } + }, + "details": { + "type": ["null", "string"] + }, + "details_code": { + "type": ["null", "string"] + }, + "document": { + "type": ["null", "object"], + "properties": { + "back": { + "type": ["null", "string"] + }, + "details": { + "type": ["null", "string"] + }, + "details_code": { + "type": ["null", "string"] + }, + "front": { + "type": ["null", "string"] + } + } + }, + "status": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json index 3af88a962ca0..2b46c1435c85 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json @@ -87,6 +87,9 @@ "metadata": { "type": ["null", "object"], "properties": {} + }, + "amount_decimal": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json new file mode 100644 index 000000000000..699186837509 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json @@ -0,0 +1,87 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Prices Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "custom_unit_amount": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "lookup_key": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "nickname": { + "type": ["null", "string"] + } + } + }, + "nickname": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "recurring": { + "type": ["null", "object"], + "properties": { + "aggregate_usage": { + "type": ["null", "string"] + }, + "interval": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "number"] + }, + "trial_period_days": { + "type": ["null", "string"] + }, + "usage_type": { + "type": ["null", "string"] + } + } + }, + "tax_behavior": { + "type": ["null", "string"] + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "transform_quantity": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "unit_amount": { + "type": ["null", "number"] + }, + "unit_amount_decimal": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json index 42558f33e9b4..b7416f78a356 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json @@ -81,6 +81,12 @@ }, "url": { "type": ["null", "string"] + }, + "default_price": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json index 030254e5a0ab..c602c4398f13 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json @@ -54,6 +54,9 @@ "minimum_amount": { "type": ["null", "integer"] }, "minimum_amount_currency": { "type": ["null", "string"] } } + }, + "times_redeemed": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json new file mode 100644 index 000000000000..aef12a767d8a --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "billing_zip": { + "type": ["null", "string"] + }, + "charge": { + "type": ["null", "string"] + }, + "closed_reason": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "ip_address": { + "type": ["null", "string"] + }, + "ip_address_location": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "region": { + "type": ["null", "number"] + } + } + }, + "livemode": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "open": { + "type": ["null", "boolean"] + }, + "opened_reason": { + "type": ["null", "string"] + }, + "payment_intent": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "session": { + "type": ["null", "object"], + "properties": { + "browser": { + "type": ["null", "string"] + }, + "device": { + "type": ["null", "string"] + }, + "platform": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json new file mode 100644 index 000000000000..5b6d7122987d --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json @@ -0,0 +1,222 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "application": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "customer": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "object": { + "type": ["null", "string"] + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "payment_method": { + "type": ["null", "string"] + }, + "payment_method_details": { + "type": ["null", "object"], + "properties": { + "au_becs_debit": { + "type": ["null", "object"], + "additional_properties": true, + "properties": {} + }, + "bacs_debit": { + "type": ["null", "object"], + "additional_properties": true, + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "generated_sepa_debit": { + "type": ["null", "string"] + }, + "generated_sepa_debit_mandate": { + "type": ["null", "string"] + }, + "iban_last4": { + "type": ["null", "string"] + }, + "preferred_language": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "card": { + "type": ["null", "object"], + "properties": { + "three_d_secure": { + "type": ["null", "object"], + "properties": { + "authentication_flow": { + "type": ["null", "string"] + }, + "result": { + "type": ["null", "string"] + }, + "result_reason": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "string"] + } + } + } + } + }, + "card_present": { + "type": ["null", "object"], + "properties": { + "generated_card": { + "type": ["null", "string"] + } + } + }, + "ideal": { + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "generated_sepa_debit": { + "type": ["null", "string"] + }, + "generated_sepa_debit_mandate": { + "type": ["null", "string"] + }, + "iban_last4": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "sepa_debit": { + "type": ["null", "object"], + "additional_properties": true, + "properties": {} + }, + "sofort": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "bank_name": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + }, + "generated_sepa_debit": { + "type": ["null", "string"] + }, + "generated_sepa_debit_mandate": { + "type": ["null", "string"] + }, + "iban_last4": { + "type": ["null", "string"] + }, + "preferred_language": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } + }, + "setup_error": { + "type": ["null", "object"], + "properties": { + "charge": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "decline_code": { + "type": ["null", "string"] + }, + "doc_url": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "param": { + "type": ["null", "string"] + }, + "payment_intent": { + "$ref": "payment_intent.json" + }, + "payment_method": { + "$ref": "payment_method.json" + }, + "payment_method_type": { + "type": ["null", "string"] + }, + "setup_intent": { + "$ref": "setup_intent.json" + }, + "source": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + } + } + }, + "setup_intent": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "flow_directions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_intents.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_intents.json index 86cd01effedd..89fa1fa32c20 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_intents.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_intents.json @@ -1,79 +1,5 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": ["null", "object"], + "$schema": "https://json-schema.org/draft-04/schema#", "additionalProperties": true, - "properties": { - "id": { - "type": ["string"] - }, - "object": { - "type": ["string"], - "enum": ["setup_intent"] - }, - "application": { - "type": ["null", "string"] - }, - "cancellation_reason": { - "type": ["null", "string"] - }, - "created": { - "type": ["integer"] - }, - "customer": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "flow_directions": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "last_setup_error": { - "type": ["null", "string"] - }, - "latest_attempt": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "mandate": { - "type": ["null", "string"] - }, - "metadata": { - "type": ["null", "object"] - }, - "next_action": { - "type": ["null", "string"] - }, - "on_behalf_of": { - "type": ["null", "string"] - }, - "payment_method": { - "type": ["null", "string"] - }, - "payment_method_options": { - "type": ["null", "object"], - "additionalProperties": true - }, - "payment_method_types": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "single_use_mandate": { - "type": ["null", "string"] - }, - "status": { - "type": ["string"] - }, - "usage": { - "type": ["string"], - "enum": ["on_session", "off_session"] - } - } + "$ref": "setup_intent.json" } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/address.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/address.json new file mode 100644 index 000000000000..79395aa9d274 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/address.json @@ -0,0 +1,23 @@ +{ + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "line1": { + "type": ["null", "string"] + }, + "line2": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json new file mode 100644 index 000000000000..f544551605e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/balance_transactions.json @@ -0,0 +1,74 @@ +{ + "type": ["null", "object"], + "properties": { + "fee": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "fee_details": { + "type": ["null", "array"], + "items": { + "properties": { + "application": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "available_on": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "net": { + "type": ["null", "integer"] + }, + "exchange_rate": { + "type": ["null", "number"] + }, + "type": { + "type": ["null", "string"] + }, + "sourced_transfers": { + "items": {}, + "type": ["null", "array"] + }, + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "integer"] + }, + "reporting_category": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json new file mode 100644 index 000000000000..532e23a24519 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json @@ -0,0 +1,101 @@ +{ + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "cancellation_reason": { + "type": ["null", "string"] + }, + "cardholder": { + "$ref": "cardholder.json" + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "cvc": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "additionalProperties": true + }, + "number": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "replaced_by": { + "type": ["null", "object"], + "additionalProperties": true + }, + "replacement_for": { + "type": ["null", "object"], + "additionalProperties": true + }, + "replacement_reason": { + "type": ["null", "string"] + }, + "shipping": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "address": { + "$ref": "address.json" + }, + "carrier": { + "type": ["null", "string"] + }, + "eta": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "service": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "tracking_number": { + "type": ["null", "string"] + }, + "tracking_url": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "spending_controls": { + "$ref": "spending_controls.json" + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json new file mode 100644 index 000000000000..8c48e4e47803 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json @@ -0,0 +1,115 @@ +{ + "type": ["null", "object"], + "properties": { + "billing": { + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + } + } + }, + "company": { + "type": ["null", "object"], + "properties": { + "tax_id_provided": { + "type": "boolean" + } + } + }, + "created": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "individual": { + "type": ["null", "object"], + "properties": { + "dob": { + "type": ["null", "object"], + "properties": { + "day": { + "type": ["null", "integer"] + }, + "month": { + "type": ["null", "integer"] + }, + "year": { + "type": ["null", "integer"] + } + } + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "verification": { + "type": ["null", "object"], + "properties": { + "document": { + "type": ["null", "object"], + "properties": { + "back": { + "type": ["null", "string"] + }, + "front": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"] + }, + "name": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "phone_number": { + "type": ["null", "string"] + }, + "requirements": { + "type": ["null", "object"], + "properties": { + "disabled_reason": { + "type": ["null", "string"] + }, + "past_due": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, + "spending_controls": { + "$ref": "spending_controls.json" + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "preferred_locales": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json new file mode 100644 index 000000000000..9d7850a5d562 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json @@ -0,0 +1,843 @@ +{ + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "preferred_locales": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "invoice_settings": { + "type": ["null", "object"], + "properties": { + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "default_payment_method": { + "type": ["null", "string"] + }, + "footer": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "string"] + }, + "next_invoice_sequence": { + "type": ["null", "integer"] + }, + "balance": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "string"] + }, + "address": { + "$ref": "address.json" + }, + "shipping": { + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "sources": { + "anyOf": [ + { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "$ref": "address.json" + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + }, + { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "type": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "three_d_secure": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "brand": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "usage": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "owner": { + "type": ["null", "object"], + "properties": { + "verified_address": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "address": { + "$ref": "address.json" + }, + "verified_email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "verified_name": { + "type": ["null", "string"] + }, + "verified_phone": { + "type": ["null", "string"] + } + } + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "receiver": { + "type": ["null", "object"], + "properties": { + "refund_attributes_method": { + "type": ["null", "string"] + }, + "amount_returned": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "refund_attributes_status": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "amount_charged": { + "type": ["null", "integer"] + } + } + }, + "flow": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "ach_credit_transfer": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "routing_number": { + "type": ["null", "string"] + }, + "swift_code": { + "type": ["null", "string"] + }, + "refund_account_holder_type": { + "type": ["null", "string"] + }, + "refund_account_holder_name": { + "type": ["null", "string"] + }, + "refund_account_number": { + "type": ["null", "string"] + }, + "refund_routing_number": { + "type": ["null", "string"] + }, + "account_number": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "address_state": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "object"], + "properties": {} + }, + "bancontact": { + "type": ["null", "object"], + "properties": {} + }, + "eps": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": {} + }, + "multibanco": { + "type": ["null", "object"], + "properties": {} + }, + "redirect": { + "type": ["null", "object"], + "properties": { + "failure_reason": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + ] + }, + "delinquent": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "default_source": { + "type": ["null", "string"] + }, + "cards": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "object": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "last4": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "brand": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "address_line1": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + }, + "address_city": { + "type": ["null", "string"] + }, + "address_country": { + "type": ["null", "string"] + }, + "address_line1_check": { + "type": ["null", "string"] + }, + "tokenization_method": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address_state": { + "type": ["null", "string"] + }, + "address_zip_check": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "email": { + "type": ["null", "string"] + }, + "default_card": { + "type": ["null", "string"] + }, + "subscriptions": { + "type": ["null", "object"], + "properties": { + "object": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "array"] + }, + "has_more": { + "type": ["null", "boolean"] + }, + "total_count": { + "type": ["null", "number"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "discount": { + "type": ["null", "object"], + "properties": { + "end": { + "type": ["null", "integer"] + }, + "coupon": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "valid": { + "type": ["null", "boolean"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "amount_off": { + "type": ["null", "integer"] + }, + "redeem_by": { + "type": ["null", "integer"] + }, + "duration_in_months": { + "type": ["null", "integer"] + }, + "percent_off_precise": { + "type": ["null", "number"] + }, + "max_redemptions": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "times_redeemed": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "percent_off": { + "type": ["null", "number"] + }, + "created": { + "type": ["null", "integer"] + } + } + }, + "customer": { + "type": ["null", "string"] + }, + "start": { + "type": ["null", "integer"] + }, + "object": { + "type": ["null", "string"] + }, + "subscription": { + "type": ["null", "string"] + } + } + }, + "account_balance": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "invoice_prefix": { + "type": ["null", "string"] + }, + "tax_info_verification": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "tax_info": { + "type": ["null", "string"] + }, + "test_clock": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/issuing_transaction_purchase_details.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/issuing_transaction_purchase_details.json new file mode 100644 index 000000000000..76c35dfa9590 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/issuing_transaction_purchase_details.json @@ -0,0 +1,99 @@ +{ + "type": ["null", "object"], + "properties": { + "flight": { + "type": ["null", "object"], + "properties": { + "departure_at": { + "type": ["null", "integer"] + }, + "passenger_name": { + "type": ["null", "string"] + }, + "refundable": { + "type": ["null", "boolean"] + }, + "segments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "arrival_airport_code": { + "type": ["null", "string"] + }, + "carrier": { + "type": ["null", "string"] + }, + "departure_airport_code": { + "type": ["null", "string"] + }, + "flight_number": { + "type": ["null", "string"] + }, + "service_class": { + "type": ["null", "string"] + }, + "stopover_allowed": { + "type": ["null", "boolean"] + } + } + } + } + }, + "travel_agency": { + "type": ["null", "string"] + } + } + }, + "fuel": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "unit": { + "type": ["null", "string"] + }, + "unit_cost_decimal": { + "type": ["null", "string"] + }, + "volume_decimal": { + "type": ["null", "string"] + } + } + }, + "lodging": { + "type": ["null", "object"], + "properties": { + "check_in_at": { + "type": ["null", "integer"] + }, + "nights": { + "type": ["null", "integer"] + } + } + }, + "receipt": { + "items": { + "type": ["null", "object"], + "properties": { + "description": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "number"] + }, + "total": { + "type": ["null", "integer"] + }, + "unit_cost": { + "type": ["null", "integer"] + } + } + }, + "type": ["null", "array"] + }, + "reference": { + "type": ["null", "string"] + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json new file mode 100644 index 000000000000..ba4e006c1ee8 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json @@ -0,0 +1,892 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "amount_capturable": { + "type": ["null", "integer"] + }, + "amount_received": { + "type": ["null", "integer"] + }, + "application": { + "type": ["null", "string"] + }, + "application_fee_amount": { + "type": ["null", "integer"] + }, + "canceled_at": { + "type": ["null", "integer"] + }, + "cancellation_reason": { + "type": ["null", "string"] + }, + "capture_method": { + "type": ["null", "string"], + "enum": ["automatic", "manual"] + }, + "charges": { + "type": ["null", "object"], + "properties": { + "object": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "array"] + }, + "has_more": { + "type": ["null", "boolean"] + }, + "total_count": { + "type": ["null", "integer"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "client_secret": { + "type": ["null", "string"] + }, + "confirmation_method": { + "type": ["null", "string"], + "enum": ["automatic", "manual"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "invoice": { + "type": ["null", "string"] + }, + "last_payment_error": { + "type": ["null", "object"], + "properties": { + "charge": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "decline_code": { + "type": ["null", "string"] + }, + "doc_url": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "param": { + "type": ["null", "string"] + }, + "payment_method": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "acss_debit": { + "type": ["null", "object"], + "properties": { + "bank_name": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "institution_number": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "transit_number": { + "type": ["null", "string"] + } + } + }, + "afterpay_clearpay": { + "type": ["null", "string"] + }, + "alipay": { + "type": ["null", "string"] + }, + "au_becs_debit": { + "type": ["null", "object"], + "properties": { + "bsb_number": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + } + } + }, + "bacs_debit": { + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "sort_code": { + "type": ["null", "string"] + } + } + }, + "bancontact": { + "type": ["null", "string"] + }, + "billing_details": { + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "boleto": { + "type": ["null", "object"], + "properties": { + "tax_id": { + "type": ["null", "string"] + } + } + }, + "card": { + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "checks": { + "type": ["null", "object"], + "properties": { + "address_line1_check": { + "type": ["null", "string"] + }, + "address_postal_code_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + } + } + }, + "country": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "generated_from": { + "type": ["null", "object"], + "properties": { + "charge": { + "type": ["null", "string"] + }, + "payment_method_details": { + "type": ["null", "object"], + "properties": { + "card_present": { + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "cardholder_name": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "emv_auth_data": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"], + "enum": ["credit", "debit", "prepaid", "unknown"] + }, + "generated_card": { + "type": ["null", "string"] + }, + "lsat4": { + "type": ["null", "string"] + }, + "network": { + "type": ["null", "string"], + "enum": [ + "contact_emv", + "contactless_emv", + "magnetic_stripe_track2", + "magnetic_stripe_fallback", + "contactless_magstripe_mode" + ] + }, + "read_method": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "account_type": { + "type": ["null", "string"], + "enum": [ + "credit", + "checking", + "prepaid", + "unknown" + ] + }, + "application_cryptogram": { + "type": ["null", "string"] + }, + "application_preferred_name": { + "type": ["null", "string"] + }, + "authorization_code": { + "type": ["null", "string"] + }, + "authorization_response_code": { + "type": ["null", "string"] + }, + "cardholder_verification_method": { + "type": ["null", "string"] + }, + "dedicated_file_name": { + "type": ["null", "string"] + }, + "terminal_verification_results": { + "type": ["null", "string"] + }, + "transaction_status_information": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } + }, + "setup_attempt": { + "type": ["null", "string"] + } + } + }, + "last4": { + "type": ["null", "string"] + }, + "networks": { + "type": ["null", "object"], + "properties": { + "available": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "preferred": { + "type": ["null", "string"] + } + } + }, + "three_d_secure_usage": { + "type": ["null", "object"], + "properties": { + "supported": { + "type": ["null", "boolean"] + } + } + }, + "wallet": { + "type": ["null", "object"], + "properties": { + "amex_express_checkout": { + "type": ["null", "string"] + }, + "apple_pay": { + "type": ["null", "string"] + }, + "dynamic_last4": { + "type": ["null", "string"] + }, + "google_pay": { + "type": ["null", "string"] + }, + "masterpass": { + "type": ["null", "object"], + "properties": { + "billing_address": { + "$ref": "address.json" + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "shipping_address": { + "$ref": "address.json" + } + } + }, + "samsung_pay": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "visa_checkout": { + "type": ["null", "object"], + "properties": { + "billing_address": { + "$ref": "address.json" + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "shipping_address": { + "$ref": "address.json" + } + } + } + } + } + } + }, + "card_present": { + "type": ["null", "object"], + "properties": {} + }, + "created": { + "type": ["null", "integer"] + }, + "customer": { + "type": ["null", "string"] + }, + "eps": { + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"], + "enum": [ + "arzte_und_apotheker_bank", + "austrian_anadi_bank_ag", + "bank_austria", + "bankhaus_carl_spangler", + "bankhaus_schelhammer_und_schattera_ag", + "bawag_psk_ag", + "bks_bank_ag", + "brull_kallmus_bank_ag", + "btv_vier_lander_bank", + "capital_bank_grawe_gruppe_ag", + "dolomitenbank", + "easybank_ag", + "erste_bank_und_sparkassen", + "hypo_alpeadriabank_international_ag", + "hypo_noe_lb_fur_niederosterreich_u_wien", + "hypo_oberosterreich_salzburg_steiermark", + "hypo_tirol_bank_ag", + "hypo_vorarlberg_bank_ag", + "hypo_bank_burgenland_aktiengesellschaft", + "marchfelder_bank", + "oberbank_ag", + "raiffeisen_bankengruppe_osterreich", + "schoellerbank_ag", + "sparda_bank_wien", + "volksbank_gruppe", + "volkskreditbank_ag", + "vr_bank_braunau" + ] + } + } + }, + "fpx": { + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"], + "enum": [ + "affin_bank", + "alliance_bank", + "ambank", + "bank_islam", + "bank_muamalat", + "bank_rakyat", + "bsn", + "cimb", + "hong_leong_bank", + "hsbc", + "kfh", + "maybank2u", + "ocbc", + "public_bank", + "rhb", + "standard_chartered", + "uob", + "deutsche_bank", + "maybank2e", + "pb_enterprise" + ] + } + } + }, + "giropay": { + "type": ["null", "object"], + "properties": {} + }, + "grabpay": { + "type": ["null", "object"], + "properties": {} + }, + "ideal": { + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"], + "enum": [ + "abn_amro", + "asn_bank", + "bunq", + "handelsbanken", + "ing", + "knab", + "moneyou", + "rabobank", + "regiobank", + "revolut", + "sns_bank", + "triodos_bank", + "van_lanschot" + ] + } + } + }, + "interac_present": { + "type": ["null", "object"], + "properties": {} + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "oxxo": { + "type": ["null", "object"], + "properties": {} + }, + "p24": { + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "sepa_debit": { + "type": ["null", "object"], + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "branch_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "generated_from": { + "type": ["null", "object"], + "properties": { + "charge": { + "type": ["null", "string"] + }, + "setup_attempt": { + "type": ["null", "string"] + } + } + }, + "last4": { + "type": ["null", "string"] + } + } + }, + "sofort": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"], + "enum": [ + "acss_debit", + "afterpay_clearpay", + "alipay", + "au_becs_debit", + "bacs_debit", + "bancontact", + "boleto", + "card", + "card_present", + "eps", + "fpx", + "giropay", + "grabpay", + "ideal", + "interac_present", + "oxxo", + "p24", + "sepa_debit", + "sofort", + "wechat_pay" + ] + }, + "wechat_pay": { + "type": ["null", "object"], + "properties": {} + } + } + }, + "payment_method_type": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"], + "enum": [ + "api_error", + "card_error", + "idempotency_error", + "invalid_request_error" + ] + } + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "next_action": { + "type": ["null", "object"], + "properties": { + "alipay_handle_redirect": { + "type": ["null", "object"], + "properties": { + "native_data": { + "type": ["null", "string"] + }, + "native_url": { + "type": ["null", "string"] + }, + "return_url": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "boleto_display_details": { + "type": ["null", "object"], + "properties": { + "expires_at": { + "type": ["null", "integer"] + }, + "hosted_voucher_url": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + }, + "pdf": { + "type": ["null", "string"] + } + } + }, + "oxxo_display_details": { + "type": ["null", "object"], + "properties": { + "expires_after": { + "type": ["null", "integer"] + }, + "hosted_voucher_url": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + } + } + }, + "redirect_to_url": { + "type": ["null", "object"], + "properties": { + "return_url": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + }, + "use_stripe_sdk": { + "type": ["null", "object"], + "properties": {} + }, + "verify_with_microdeposits": { + "type": ["null", "object"], + "properties": { + "arrival_date": { + "type": ["null", "integer"] + }, + "hosted_verification_url": { + "type": ["null", "string"] + } + } + }, + "wechat_pay_display_qr_code": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "string"] + }, + "image_data_url": { + "type": ["null", "string"] + } + } + }, + "wechat_pay_redirect_to_android_app": { + "type": ["null", "object"], + "properties": { + "app_id": { + "type": ["null", "string"] + }, + "nonce_str": { + "type": ["null", "string"] + }, + "package": { + "type": ["null", "string"] + }, + "partner_id": { + "type": ["null", "string"] + }, + "prepay_id": { + "type": ["null", "string"] + }, + "sign": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"] + } + } + }, + "wechat_pay_redirect_to_ios_app": { + "type": ["null", "object"], + "properties": { + "native_url": { + "type": ["null", "string"] + } + } + } + } + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "payment_method": { + "type": ["null", "string"] + }, + "payment_method_options": { + "type": ["null", "object"], + "properties": {} + }, + "payment_method_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "receipt_email": { + "type": ["null", "string"] + }, + "review": { + "type": ["null", "string"] + }, + "setup_future_usage": { + "type": ["null", "string"] + }, + "shipping": { + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + }, + "carrier": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "tracking_number": { + "type": ["null", "string"] + } + } + }, + "source": { + "type": ["null", "string"] + }, + "statement_description": { + "type": ["null", "string"] + }, + "statement_descriptor_suffix": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "transfer_group": { + "type": ["null", "string"] + }, + "latest_charge": { + "type": ["null", "string"] + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "tip": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + } + } + } + } + }, + "processing": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "customer_notification": { + "type": ["null", "object"], + "properties": { + "approval_requested": { + "type": ["null", "boolean"] + }, + "completes_at": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "automatic_payment_methods": { + "type": ["null", "object"], + "properties": { + "allow_redirects": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json new file mode 100644 index 000000000000..e181f7ea0e9b --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json @@ -0,0 +1,261 @@ +{ + "type": ["object", "null"], + "properties": { + "afterpay_clearpay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "alipay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "au_becs_debit": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bsb_number": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + } + } + }, + "bacs_debit": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "sort_code": { + "type": ["null", "string"] + } + } + }, + "bancontact": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "billing_details": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "card": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "checks": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "address_line1_check": { + "type": ["null", "string"] + }, + "address_postal_code_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + } + } + }, + "country": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "generated_from": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "last4": { + "type": ["null", "string"] + }, + "networks": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "available": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "preferred": { + "type": ["null", "string"] + } + } + }, + "three_d_secure_usage": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "supported": { + "type": ["null", "boolean"] + } + } + }, + "wallet": { + "additionalProperties": true, + "type": ["null", "object"] + } + } + }, + "card_present": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "created": { + "type": ["null", "integer"] + }, + "customer": { + "$ref": "customer.json" + }, + "eps": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "fpx": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "giropay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "grabpay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "id": { + "type": ["null", "string"] + }, + "ideal": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + } + } + }, + "interac_present": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "object": { + "type": ["null", "string"] + }, + "oxxo": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "p24": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "sepa_debit": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "branch_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "generated_from": { + "type": ["null", "object"], + "properties": { + "charge": { + "type": ["null", "string"] + }, + "setup_attempt": { + "type": ["null", "string"] + } + } + }, + "last4": { + "type": ["null", "string"] + } + } + }, + "sofort": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "country": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json new file mode 100644 index 000000000000..7c2a6344423b --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/price.json @@ -0,0 +1,104 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "billing_scheme": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "currency_options": { + "type": ["null", "string"] + }, + "custom_unit_amount": { + "type": ["null", "object"], + "properties": { + "maximum": { + "type": ["null", "integer"] + }, + "minimum": { + "type": ["null", "integer"] + }, + "preset": { + "type": ["null", "integer"] + } + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "lookup_key": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "nickname": { + "type": ["null", "string"] + }, + "product": { + "type": ["null", "string"] + }, + "recurring": { + "type": ["null", "object"], + "properties": { + "aggregate_usage": { + "type": ["null", "string"] + }, + "interval": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + }, + "usage_type": { + "type": ["null", "string"] + } + } + }, + "tax_behavior": { + "type": ["null", "string"] + }, + "tiers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "tiers_mode": { + "type": ["null", "string"] + }, + "transform_quantity": { + "type": ["null", "object"], + "properties": { + "divide_by": { + "type": ["null", "integer"] + }, + "round": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + }, + "unit_amount": { + "type": ["null", "integer"] + }, + "unit_amount_decimal": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json new file mode 100644 index 000000000000..878fae7e2f0f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json @@ -0,0 +1,91 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["string"] + }, + "object": { + "type": ["string"], + "enum": ["setup_intent"] + }, + "application": { + "type": ["null", "string"] + }, + "cancellation_reason": { + "type": ["null", "string"] + }, + "created": { + "type": ["integer"] + }, + "customer": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "flow_directions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "last_setup_error": { + "type": ["null", "string"] + }, + "latest_attempt": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "mandate": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "next_action": { + "type": ["null", "string"] + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "payment_method": { + "type": ["null", "string"] + }, + "payment_method_options": { + "type": ["null", "object"], + "additionalProperties": true + }, + "payment_method_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "single_use_mandate": { + "type": ["null", "string"] + }, + "status": { + "type": ["string"] + }, + "usage": { + "type": ["string"], + "enum": ["on_session", "off_session"] + }, + "client_secret": { + "type": ["null", "string"] + }, + "automatic_payment_methods": { + "type": ["null", "object"], + "properties": { + "allow_redirects": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/spending_controls.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/spending_controls.json new file mode 100644 index 000000000000..0059ab093b4e --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/spending_controls.json @@ -0,0 +1,26 @@ +{ + "type": ["null", "object"], + "properties": { + "allowed_categories": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "blocked_categories": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "spending_limits": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "spending_limits_currency": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json new file mode 100644 index 000000000000..ee5b078c5d73 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json @@ -0,0 +1,53 @@ +{ + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "country": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "display_name": { + "type": ["null", "string"] + }, + "effective_percentage": { + "type": ["null", "number"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "jurisdiction": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"] + }, + "percentage": { + "type": ["null", "number"] + }, + "state": { + "type": ["null", "string"] + }, + "tax_type": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shipping_rates.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shipping_rates.json new file mode 100644 index 000000000000..fecd2f4f75b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shipping_rates.json @@ -0,0 +1,60 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Shipping Rates Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "created": { + "type": ["null", "integer"] + }, + "delivery_estimate": { + "type": ["null", "string"] + }, + "display_name": { + "type": ["null", "string"] + }, + "fixed_amount": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "tax_behavior": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json index 04ca778c8da8..c6d36caab561 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json @@ -153,6 +153,20 @@ }, "trial_end": { "type": ["null", "number"] + }, + "billing_thresholds": { + "type": ["null", "object"], + "properties": { + "usage_gte": { + "type": ["null", "integer"] + } + } + }, + "tax_rates": { + "$ref": "tax_rates.json" + }, + "price": { + "$ref": "price.json" } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json index 6afdd775bd21..e8729bfefab4 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json @@ -258,6 +258,9 @@ }, "test_clock": { "type": ["null", "string"] + }, + "renewal_interval": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json index 5d7e5a3713ff..405647bd85cc 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json @@ -247,6 +247,201 @@ }, "object": { "type": ["null", "string"] + }, + "pending_setup_intent": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "transfer_data": { + "type": ["null", "object"], + "properties": { + "amount_percent": { + "type": ["null", "number"] + }, + "destination": { + "type": ["null", "string"] + } + } + }, + "application": { + "type": ["null", "string"] + }, + "test_clock": { + "type": ["null", "string"] + }, + "automatic_tax": { + "type": ["null", "object"], + "properties": { + "enabled": { + "type": ["null", "boolean"] + } + } + }, + "payment_settings": { + "type": ["null", "object"], + "properties": { + "payment_method_options": { + "type": ["null", "object"] + }, + "payment_method_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "save_default_payment_method": { + "type": ["null", "string"] + } + } + }, + "next_pending_invoice_item_invoice": { + "type": ["null", "integer"] + }, + "default_source": { + "type": ["null", "string"] + }, + "default_payment_method": { + "type": ["null", "string"] + }, + "collection_method": { + "type": ["null", "string"] + }, + "pending_invoice_item_interval": { + "type": ["null", "object"], + "properties": { + "interval": { + "type": ["null", "string"] + }, + "interval_count": { + "type": ["null", "integer"] + } + } + }, + "default_tax_rates": { + "type": ["null", "array"], + "items": { + "$ref": "tax_rates.json" + } + }, + "pause_collection": { + "type": ["null", "object"], + "properties": { + "behavior": { + "type": ["null", "string"] + }, + "resumes_at": { + "type": ["null", "integer"] + } + } + }, + "cancellation_details": { + "type": ["null", "object"], + "properties": { + "comment": { + "type": ["null", "string"] + }, + "feedback": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + } + } + }, + "latest_invoice": { + "type": ["null", "string"] + }, + "pending_update": { + "type": ["null", "object"], + "properties": { + "billing_cycle_anchor": { + "type": ["null", "integer"] + }, + "expires_at": { + "type": ["null", "integer"] + }, + "subscription_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "billing_thresholds": { + "type": ["null", "object"], + "properties": { + "usage_gte": { + "type": ["null", "integer"] + } + } + }, + "created": { + "type": ["null", "integer"] + }, + "metadata": { + "type": ["null", "object"] + }, + "price": { + "$ref": "price.json" + }, + "quantity": { + "type": ["null", "integer"] + }, + "subscription": { + "type": ["null", "string"] + }, + "tax_rates": { + "$ref": "tax_rates.json" + } + } + } + }, + "trial_end": { + "type": ["null", "integer"] + }, + "trial_from_plan": { + "type": ["null", "boolean"] + } + } + }, + "description": { + "type": ["null", "string"] + }, + "schedule": { + "type": ["null", "string"] + }, + "trial_settings": { + "type": ["null", "object"], + "properties": { + "end_behavior": { + "type": ["null", "object"], + "properties": { + "missing_payment_method": { + "type": ["null", "string"] + } + } + } + } + }, + "on_behalf_of": { + "type": ["null", "string"] + }, + "billing_thresholds": { + "type": ["null", "object"], + "properties": { + "amount_gte": { + "type": ["null", "integer"] + }, + "reset_billing_cycle_anchor": { + "type": ["null", "boolean"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json new file mode 100644 index 000000000000..ec3424a67763 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json @@ -0,0 +1,60 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"], + "properties": {} + }, + "status": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "destination_balance": { + "type": ["null", "string"] + }, + "expected_availability_date": { + "type": ["null", "integer"] + }, + "failure_code": { + "type": ["null", "string"] + }, + "failure_message": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "source": { + "type": ["null", "object"], + "properties": {} + }, + "statement_descriptor": { + "type": ["null", "string"] + }, + "transfer_group": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json new file mode 100644 index 000000000000..fed1ef8855e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + }, + "authorization": { + "type": ["null", "string"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "string"] + }, + "cardholder": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "dispute": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "merchant_amount": { + "type": ["null", "integer"] + }, + "merchant_currency": { + "type": ["null", "string"] + }, + "merchant_data": { + "type": ["null", "object"], + "properties": { + "category": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "network_id": { + "type": ["null", "string"] + }, + "postal_code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "metadata": { + "type": ["null", "object"], + "additionalProperties": true + }, + "object": { + "type": ["null", "string"] + }, + "purchase_details": { + "$ref": "issuing_transaction_purchase_details.json" + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfer_reversals.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfer_reversals.json new file mode 100644 index 000000000000..e3bd65e85820 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfer_reversals.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "balance_transaction": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "destination_payment_refund": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "metadata": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": {} + }, + "object": { + "type": ["null", "string"] + }, + "source_refund": { + "type": ["null", "string"] + }, + "transfer": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/usage_records.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/usage_records.json new file mode 100644 index 000000000000..d5578fd5fb7d --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/usage_records.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "invoice": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "object": { + "type": ["null", "string"] + }, + "period": { + "type": ["null", "object"], + "properties": { + "start": { + "type": ["null", "integer"] + }, + "end": { + "type": ["null", "integer"] + } + } + }, + "subscription_item": { + "type": ["null", "string"] + }, + "total_usage": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index 8ce7fcc4c0f2..a2f2b3303a12 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -15,12 +15,16 @@ Accounts, ApplicationFees, ApplicationFeesRefunds, + Authorizations, BalanceTransactions, BankAccounts, + Cardholders, + Cards, Charges, CheckoutSessions, CheckoutSessionsLineItems, Coupons, + CreditNotes, CustomerBalanceTransactions, Customers, Disputes, @@ -28,20 +32,32 @@ Events, ExternalAccountBankAccounts, ExternalAccountCards, + FileLinks, + Files, InvoiceItems, InvoiceLineItems, Invoices, PaymentIntents, + PaymentMethods, Payouts, + Persons, Plans, + Prices, Products, PromotionCodes, Refunds, + Reviews, + SetupAttempts, SetupIntents, + ShippingRates, SubscriptionItems, Subscriptions, SubscriptionSchedule, + TopUps, + Transactions, + TransferReversals, Transfers, + UsageRecords, ) @@ -68,31 +84,47 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Accounts(**args), ApplicationFees(**incremental_args), ApplicationFeesRefunds(**args), + Authorizations(**incremental_args), BalanceTransactions(**incremental_args), BankAccounts(**args), + Cardholders(**incremental_args), + Cards(**incremental_args), Charges(**incremental_args), CheckoutSessions(**args), CheckoutSessionsLineItems(**args), Coupons(**incremental_args), + CreditNotes(**args), CustomerBalanceTransactions(**args), Customers(**incremental_args), Disputes(**incremental_args), - Events(**incremental_args), EarlyFraudWarnings(**args), + Events(**incremental_args), + ExternalAccountBankAccounts(**args), + ExternalAccountCards(**args), + FileLinks(**incremental_args), + Files(**incremental_args), InvoiceItems(**incremental_args), InvoiceLineItems(**args), Invoices(**incremental_args), PaymentIntents(**incremental_args), + PaymentMethods(**args), Payouts(**incremental_args), + Persons(**incremental_args), Plans(**incremental_args), + Prices(**incremental_args), Products(**incremental_args), PromotionCodes(**incremental_args), Refunds(**incremental_args), + Reviews(**incremental_args), + SetupAttempts(**incremental_args), + SetupIntents(**incremental_args), + ShippingRates(**incremental_args), SubscriptionItems(**args), Subscriptions(**incremental_args), SubscriptionSchedule(**incremental_args), + TopUps(**incremental_args), + Transactions(**incremental_args), + TransferReversals(**args), Transfers(**incremental_args), - ExternalAccountBankAccounts(**args), - ExternalAccountCards(**args), - SetupIntents(**incremental_args), + UsageRecords(**args), ] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 530980cb5966..224a33a84ee1 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -11,16 +11,11 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from source_stripe.availability_strategy import StripeSubStreamAvailabilityStrategy +from source_stripe.availability_strategy import StripeAvailabilityStrategy, StripeSubStreamAvailabilityStrategy -STRIPE_ERROR_CODES: List = [ - # stream requires additional permissions - "more_permissions_required", - # account_id doesn't have the access to the stream - "account_invalid", -] +STRIPE_API_VERSION = "2022-11-15" class StripeStream(HttpStream, ABC): @@ -29,6 +24,10 @@ class StripeStream(HttpStream, ABC): DEFAULT_SLICE_RANGE = 365 transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeAvailabilityStrategy() + def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_SLICE_RANGE, **kwargs): super().__init__(**kwargs) self.account_id = account_id @@ -37,7 +36,7 @@ def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() - if bool(decoded_response.get("has_more", "False")) and decoded_response.get("data", []): + if "has_more" in decoded_response and decoded_response["has_more"] and decoded_response.get("data", []): last_object_id = decoded_response["data"][-1]["id"] return {"starting_after": last_object_id} @@ -49,10 +48,6 @@ def request_params( ) -> MutableMapping[str, Any]: # Stripe default pagination is 10, max is 100 params = {"limit": 100} - for key in ("created[gte]", "created[lte]"): - if key in stream_slice: - params[key] = stream_slice[key] - # Handle pagination by inserting the next page's token in the request parameters if next_page_token: params.update(next_page_token) @@ -60,15 +55,29 @@ def request_params( return params def request_headers(self, **kwargs) -> Mapping[str, Any]: + headers = {"Stripe-Version": STRIPE_API_VERSION} if self.account_id: - return {"Stripe-Account": self.account_id} - - return {} + headers["Stripe-Account"] = self.account_id + return headers def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() yield from response_json.get("data", []) # Stripe puts records in a container array "data" + +class BasePaginationStripeStream(StripeStream, ABC): + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + for key in ("created[gte]", "created[lte]"): + if key in stream_slice: + params[key] = stream_slice[key] + return params + def chunk_dates(self, start_date_ts: int) -> Iterable[Tuple[int, int]]: now = pendulum.now().int_timestamp step = int(pendulum.duration(days=self.slice_range).total_seconds()) @@ -94,22 +103,10 @@ def read_records( if stream_slice is None: return [] - try: - yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - except requests.exceptions.HTTPError as e: - status_code = e.response.status_code - parsed_error = e.response.json() - error_code = parsed_error.get("error", {}).get("code") - error_message = parsed_error.get("message") - # if the API Key doesn't have required permissions to particular stream, this stream will be skipped - if status_code == 403 and error_code in STRIPE_ERROR_CODES: - self.logger.warn(f"Stream {self.name} is skipped, due to {error_code}. Full message: {error_message}") - pass - else: - self.logger.error(f"Syncing stream {self.name} is failed, due to {error_code}. Full message: {error_message}") - - -class IncrementalStripeStream(StripeStream, ABC): + yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + + +class IncrementalStripeStream(BasePaginationStripeStream, ABC): # Stripe returns most recently created objects first, so we don't want to persist state until the entire stream has been read state_checkpoint_interval = math.inf @@ -156,6 +153,17 @@ def get_start_timestamp(self, stream_state) -> int: return start_point +class Authorizations(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/issuing/authorizations/list + """ + + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "issuing/authorizations" + + class Customers(IncrementalStripeStream): """ API docs: https://stripe.com/docs/api/customers/list @@ -180,6 +188,17 @@ def path(self, **kwargs) -> str: return "balance_transactions" +class Cardholders(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/issuing/cardholders/list + """ + + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "issuing/cardholders" + + class Charges(IncrementalStripeStream): """ API docs: https://stripe.com/docs/api/charges/list @@ -190,8 +209,18 @@ class Charges(IncrementalStripeStream): def path(self, **kwargs) -> str: return "charges" + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params["expand[]"] = ["data.refunds"] + return params + -class CustomerBalanceTransactions(StripeStream): +class CustomerBalanceTransactions(BasePaginationStripeStream): """ API docs: https://stripe.com/docs/api/customer_balance_transactions/list """ @@ -243,17 +272,6 @@ class EarlyFraudWarnings(StripeStream): API docs: https://stripe.com/docs/api/radar/early_fraud_warnings/list """ - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = {} - - if next_page_token: - params.update(next_page_token) - def path(self, **kwargs): return "radar/early_fraud_warnings" @@ -269,7 +287,7 @@ def path(self, **kwargs): return "events" -class StripeSubStream(StripeStream, ABC): +class StripeSubStream(BasePaginationStripeStream, ABC): """ Research shows that records related to SubStream can be extracted from Parent streams which already contain 1st page of needed items. Thus, it significantly decreases a number of requests needed to get @@ -476,6 +494,17 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): return params +class Prices(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/prices/list + """ + + cursor_field = "created" + + def path(self, **kwargs): + return "prices" + + class Products(IncrementalStripeStream): """ API docs: https://stripe.com/docs/api/products/list @@ -487,11 +516,34 @@ def path(self, **kwargs): return "products" +class ShippingRates(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/shipping_rates/list + """ + + cursor_field = "created" + + def path(self, **kwargs): + return "shipping_rates" + + +class Reviews(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/radar/reviews/list + """ + + cursor_field = "created" + + def path(self, **kwargs): + return "reviews" + + class Subscriptions(IncrementalStripeStream): """ API docs: https://stripe.com/docs/api/subscriptions/list """ + use_cache = True cursor_field = "created" status = "all" @@ -510,6 +562,8 @@ class SubscriptionItems(StripeSubStream): API docs: https://stripe.com/docs/api/subscription_items/list """ + use_cache = True + name = "subscription_items" parent: StripeStream = Subscriptions @@ -541,6 +595,7 @@ class Transfers(IncrementalStripeStream): API docs: https://stripe.com/docs/api/transfers/list """ + use_cache = True cursor_field = "created" def path(self, **kwargs): @@ -569,6 +624,15 @@ def path(self, **kwargs): return "payment_intents" +class PaymentMethods(StripeStream): + """ + API docs: https://stripe.com/docs/api/payment_methods/list + """ + + def path(self, **kwargs): + return "payment_methods" + + class BankAccounts(StripeSubStream): """ API docs: https://stripe.com/docs/api/customer_bank_accounts/list @@ -697,7 +761,7 @@ def path(self, **kwargs): return "promotion_codes" -class ExternalAccount(StripeStream, ABC): +class ExternalAccount(BasePaginationStripeStream, ABC): """ Bank Accounts and Cards are separate streams because they have different schemas """ @@ -744,7 +808,7 @@ def path(self, **kwargs): return "setup_intents" -class Accounts(StripeStream): +class Accounts(BasePaginationStripeStream): """ Docs: https://stripe.com/docs/api/accounts/list Even the endpoint allow to filter based on created the data usually don't have this field. @@ -752,3 +816,185 @@ class Accounts(StripeStream): def path(self, **kwargs): return "accounts" + + +class Persons(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/persons/list + """ + + name = "persons" + cursor_field = "created" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): + return f"accounts/{stream_slice['id']}/persons" + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream = Accounts(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) + slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) + for _slice in slices: + for account in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): + # we use `get` here because some attributes may not be returned by some API versions + yield account + + +class CreditNotes(StripeStream): + """ + API docs: https://stripe.com/docs/api/credit_notes/list + """ + + name = "credit_notes" + + def path(self, **kwargs) -> str: + return "credit_notes" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return next_page_token or {} + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + yield from [{}] + + +class Cards(IncrementalStripeStream): + """ + Docs: https://stripe.com/docs/api/issuing/cards/list + """ + + cursor_field = "created" + + def path(self, **kwargs): + return "issuing/cards" + + +class TopUps(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/topups/list + """ + + name = "top_ups" + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "topups" + + +class Files(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/files/list + """ + + name = "files" + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "files" + + +class FileLinks(IncrementalStripeStream): + """ + API docs: https://stripe.com/docs/api/file_links/list + """ + + name = "file_links" + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "file_links" + + +class SetupAttempts(IncrementalStripeStream, HttpSubStream): + """ + Docs: https://stripe.com/docs/api/setup_attempts/list + """ + + cursor_field = "created" + + def __init__(self, **kwargs): + parent = SetupIntents(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path(self, **kwargs) -> str: + return "setup_attempts" + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + incremental_slices = list( + IncrementalStripeStream.stream_slices(self, sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) + ) + if incremental_slices: + parent_records = HttpSubStream.stream_slices(self, sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) + yield from (slice | rec for rec in parent_records for slice in incremental_slices) + else: + yield None + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + setup_intent_id = stream_slice.get("parent", {}).get("id") + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params.update(setup_intent=setup_intent_id) + return params + + +class UsageRecords(StripeStream, HttpSubStream): + """ + Docs: https://stripe.com/docs/api/usage_records/subscription_item_summary_list + """ + + primary_key = None + + def __init__(self, **kwargs): + parent = SubscriptionItems(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + subscription_item_id = stream_slice.get("parent", {}).get("id") + return f"subscription_items/{subscription_item_id}/usage_record_summaries" + + +class TransferReversals(StripeStream, HttpSubStream): + """ + Docs: https://stripe.com/docs/api/transfer_reversals/list + """ + + def __init__(self, **kwargs): + parent = Transfers(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + transfer_id = stream_slice.get("parent", {}).get("id") + return f"transfers/{transfer_id}/reversals" + + +class Transactions(IncrementalStripeStream): + """ + Docs: https://stripe.com/docs/api/issuing/transactions/list + """ + + cursor_field = "created" + + def path(self, **kwargs) -> str: + return "issuing/transactions" diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py new file mode 100644 index 000000000000..f72068c051d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +@pytest.fixture(autouse=True) +def disable_cache(mocker): + mocker.patch( + "source_stripe.streams.Customers.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_stripe.streams.Transfers.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_stripe.streams.Subscriptions.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + mocker.patch( + "source_stripe.streams.SubscriptionItems.use_cache", + new_callable=mocker.PropertyMock, + return_value=False + ) + + +@pytest.fixture(name="config") +def config_fixture(): + config = {"client_secret": "sk_test(live)_", + "account_id": "", "start_date": "2020-05-01T00:00:00Z"} + return config + + +@pytest.fixture(name="stream_args") +def stream_args_fixture(): + authenticator = TokenAuthenticator("sk_test(live)_") + args = { + "authenticator": authenticator, + "account_id": "", + "start_date": 1588315041, + "slice_range": 365, + } + return args diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 0eca0175d775..f5ea6c77b31c 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -42,30 +42,22 @@ def test_source_streams(): with open("sample_files/config.json") as f: config = json.load(f) streams = SourceStripe().streams(config=config) - assert len(streams) == 30 - - -@pytest.fixture(name="config") -def config_fixture(): - config = {"client_secret": "sk_test(live)_", - "account_id": "", "start_date": "2020-05-01T00:00:00Z"} - return config - - -@pytest.fixture(name="logger_mock") -def logger_mock_fixture(): - return patch("source_tiktok_marketing.source.logger") + assert len(streams) == 46 @patch.object(source_stripe.source, "stripe") -def test_source_check_connection_ok(mocked_client, config, logger_mock): - assert SourceStripe().check_connection( - logger_mock, config=config) == (True, None) +def test_source_check_connection_ok(mocked_client, config): + assert SourceStripe().check_connection(None, config=config) == (True, None) @patch.object(source_stripe.source, "stripe") -def test_source_check_connection_failure(mocked_client, config, logger_mock): +def test_source_check_connection_failure(mocked_client, config): exception = Exception("Test") mocked_client.Account.retrieve = Mock(side_effect=exception) - assert SourceStripe().check_connection( - logger_mock, config=config) == (False, exception) + assert SourceStripe().check_connection(None, config=config) == (False, exception) + + +@patch.object(source_stripe.source, "stripe") +def test_streams_are_unique(mocked_client, config): + streams = [s.name for s in SourceStripe().streams(config)] + assert sorted(streams) == sorted(set(streams)) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index db1f4bf0b27c..1fd42de179e9 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -2,9 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + import pendulum import pytest from airbyte_cdk.models import SyncMode +from source_stripe.availability_strategy import STRIPE_ERROR_CODES from source_stripe.streams import ( ApplicationFees, ApplicationFeesRefunds, @@ -27,11 +30,14 @@ Invoices, PaymentIntents, Payouts, + Persons, Plans, + Prices, Products, PromotionCodes, Refunds, SetupIntents, + ShippingRates, SubscriptionItems, Subscriptions, SubscriptionSchedule, @@ -149,14 +155,8 @@ def test_sub_stream(requests_mock): ] -@pytest.fixture(name="config") -def config_fixture(): - config = {"authenticator": "authenticator", "account_id": "", "start_date": 1596466368} - return config - - @pytest.mark.parametrize( - "stream, kwargs, expected", + "stream_cls, kwargs, expected", [ (ApplicationFees, {}, "application_fees"), (ApplicationFeesRefunds, {"stream_slice": {"refund_id": "fr"}}, "application_fees/fr/refunds"), @@ -172,7 +172,9 @@ def config_fixture(): (InvoiceLineItems, {"stream_slice": {"invoice_id": "I1"}}, "invoices/I1/lines"), (InvoiceItems, {}, "invoiceitems"), (Payouts, {}, "payouts"), + (Persons, {"stream_slice": {"id": "A1"}}, "accounts/A1/persons"), (Plans, {}, "plans"), + (Prices, {}, "prices"), (Products, {}, "products"), (Subscriptions, {}, "subscriptions"), (SubscriptionItems, {}, "subscription_items"), @@ -186,15 +188,19 @@ def config_fixture(): (PromotionCodes, {}, "promotion_codes"), (ExternalAccount, {}, "accounts//external_accounts"), (SetupIntents, {}, "setup_intents"), + (ShippingRates, {}, "shipping_rates"), ], ) -def test_path( - stream, +def test_path_and_headers( + stream_cls, kwargs, expected, - config, + stream_args, ): - assert stream(**config).path(**kwargs) == expected + stream = stream_cls(**stream_args) + assert stream.path(**kwargs) == expected + headers = stream.request_headers(**kwargs) + assert headers["Stripe-Version"] == "2022-11-15" @pytest.mark.parametrize( @@ -232,6 +238,44 @@ def test_request_params( stream, kwargs, expected, - config, + stream_args, ): - assert stream(**config).request_params(**kwargs) == expected + assert stream(**stream_args).request_params(**kwargs) == expected + + +@pytest.mark.parametrize( + "stream_cls", + ( + ApplicationFees, + Customers, + BalanceTransactions, + Charges, + Coupons, + Disputes, + Events, + Invoices, + InvoiceItems, + Payouts, + Plans, + Prices, + Products, + Subscriptions, + SubscriptionSchedule, + Transfers, + Refunds, + PaymentIntents, + CheckoutSessions, + PromotionCodes, + ExternalAccount, + SetupIntents, + ShippingRates + ) +) +def test_403_error_handling(stream_args, stream_cls, requests_mock): + stream = stream_cls(**stream_args) + logger = logging.getLogger("airbyte") + for error_code in STRIPE_ERROR_CODES: + requests_mock.get(f"{stream.url_base}{stream.path()}", status_code=403, json={"error": {"code": f"{error_code}"}}) + available, message = stream.check_availability(logger) + assert not available + assert STRIPE_ERROR_CODES[error_code] in message diff --git a/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml b/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml index ea5b10770297..3a127ebc7732 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml +++ b/airbyte-integrations/connectors/source-survey-sparrow/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-survey-sparrow/requirements.txt b/airbyte-integrations/connectors/source-survey-sparrow/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/requirements.txt +++ b/airbyte-integrations/connectors/source-survey-sparrow/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-survey-sparrow/setup.py b/airbyte-integrations/connectors/source-survey-sparrow/setup.py index d52615593592..da1ca261a46e 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/setup.py +++ b/airbyte-integrations/connectors/source-survey-sparrow/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-surveycto/Dockerfile b/airbyte-integrations/connectors/source-surveycto/Dockerfile index 3ea0fac5aad8..a98c8726003a 100644 --- a/airbyte-integrations/connectors/source-surveycto/Dockerfile +++ b/airbyte-integrations/connectors/source-surveycto/Dockerfile @@ -37,6 +37,6 @@ COPY source_surveycto ./source_surveycto ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-surveycto diff --git a/airbyte-integrations/connectors/source-surveycto/metadata.yaml b/airbyte-integrations/connectors/source-surveycto/metadata.yaml index 5e0563ff88ef..d8bc3a5a52f9 100644 --- a/airbyte-integrations/connectors/source-surveycto/metadata.yaml +++ b/airbyte-integrations/connectors/source-surveycto/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: dd4632f4-15e0-4649-9b71-41719fb1fdee - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-surveycto githubIssueLabel: source-surveycto icon: surveycto.svg @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/surveycto tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-surveycto/requirements.txt b/airbyte-integrations/connectors/source-surveycto/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-surveycto/requirements.txt +++ b/airbyte-integrations/connectors/source-surveycto/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-surveycto/setup.py b/airbyte-integrations/connectors/source-surveycto/setup.py index 86059a2bfcfd..be5f78fbdd14 100644 --- a/airbyte-integrations/connectors/source-surveycto/setup.py +++ b/airbyte-integrations/connectors/source-surveycto/setup.py @@ -8,9 +8,9 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "bigquery_schema_generator~=1.5", "gbqschema_converter~=1.2.0"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py index e83dcbfb5c70..816b0dad755e 100644 --- a/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/source.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -109,8 +110,22 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: # Source class SourceSurveycto(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - return True, None + def check_connection(self, logger, config) -> Tuple[bool, Any]: + + form_ids = config["form_id"] + + try: + for form_id in form_ids: + schema = Helpers.call_survey_cto(config, form_id) + filter_data = Helpers.get_filter_data(schema) + schema_res = Helpers.get_json_schema(filter_data) + stream = SurveyctoStream(config=config, form_id=form_id, schema=schema_res) + next(stream.read_records(sync_mode=SyncMode.full_refresh)) + + return True, None + + except Exception as error: + return False, f"Unable to connect - {(error)}" def generate_streams(self, config: str) -> List[Stream]: forms = config.get("form_id", []) diff --git a/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml b/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml index a86d4f46d808..f9ab7aed5d6d 100644 --- a/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml +++ b/airbyte-integrations/connectors/source-surveycto/source_surveycto/spec.yaml @@ -27,7 +27,7 @@ connectionSpecification: order: 2 form_id: type: array - title: Form's Id + title: Form Id's description: Unique identifier for one of your forms order: 3 start_date: diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py index 343043787611..0e3396d52a68 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py @@ -5,32 +5,53 @@ from unittest.mock import MagicMock, patch import pytest -from airbyte_cdk.models import ConnectorSpecification -from source_surveycto.helpers import Helpers -from source_surveycto.source import SourceSurveycto +from source_surveycto.source import SourceSurveycto, SurveyctoStream @pytest.fixture(name='config') def config_fixture(): - return {'server_name': 'server_name', 'form_id': 'form_id', 'start_date': 'Jan 09, 2022 00:00:00 AM', 'password': 'password', 'username': 'username'} + return { + 'server_name': 'server_name', + 'form_id': ['form_id_1', 'form_id_2'], + 'start_date': 'Jan 09, 2022 00:00:00 AM', + 'password': 'password', + 'username': 'username' + } -def test_spec(): - source = SourceSurveycto() +@pytest.fixture(name='source') +def source_fixture(): + return SourceSurveycto() + + +@pytest.fixture(name='mock_survey_cto') +def mock_survey_cto_fixture(): + with patch('source_surveycto.source.Helpers.call_survey_cto', return_value="value") as mock_call_survey_cto, \ + patch('source_surveycto.source.Helpers.get_filter_data', return_value="value") as mock_filter_data, \ + patch('source_surveycto.source.Helpers.get_json_schema', return_value="value") as mock_json_schema: + yield mock_call_survey_cto, mock_filter_data, mock_json_schema + + +def test_check_connection_valid(mock_survey_cto, source, config): logger_mock = MagicMock() - spec = source.spec(logger_mock) - assert source.check_connection(spec, ConnectorSpecification) + records = iter(["record1", "record2"]) + with patch.object(SurveyctoStream, 'read_records', return_value=records): + assert source.check_connection(logger_mock, config) == (True, None) -@patch("requests.get") -def test_check_connection(config): - source = SourceSurveycto() + +def test_check_connection_failure(mock_survey_cto, source, config): logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == (True, None) + expected_outcome = 'Unable to connect - 400 Client Error: 400 for url: https://server_name.surveycto.com/api/v2/forms/data/wide/json/form_id_1?date=Jan+09%2C+2022+00%3A00%3A00+AM' + assert source.check_connection(logger_mock, config) == (False, expected_outcome) + + +def test_generate_streams(mock_survey_cto, source, config): + streams = source.generate_streams(config) + assert len(streams) == 2 -def test_streams(config): - source = SourceSurveycto() - Helpers.call_survey_cto = MagicMock() +@patch('source_surveycto.source.SourceSurveycto.generate_streams', return_value=['stream_1', 'stream2']) +def test_streams(mock_generate_streams, source, config): streams = source.streams(config) - assert len(streams) == 7 + assert len(streams) == 2 diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml index f9080a16f431..cc2061c4bc2e 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" expect_records: - path: "integration_tests/expected_records.txt" + path: "integration_tests/expected_records.jsonl" fail_on_extra_columns: false incremental: tests: diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..4a7c296bd341 --- /dev/null +++ b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.jsonl @@ -0,0 +1,453 @@ +{"stream": "surveys", "data": {"title": "Market Research - Product Testing Template", "nickname": "", "language": "en", "folder_id": "0", "category": "market_research", "question_count": 13, "page_count": 1, "response_count": 13, "date_created": "2021-05-07T06:18:00", "date_modified": "2021-06-08T18:09:00", "id": "306079584", "buttons_text": {"next_button": "Next >>", "prev_button": "<< Prev", "done_button": "Done", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/306079584", "analyze_url": "https://www.surveymonkey.com/analyze/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "summary_url": "https://www.surveymonkey.com/summary/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=8T1PwDGoJHE1lbkxjUnaGitKu8jxWzyoclw9fNsShflPlk6MYIzwJ2NgjlBw_2B7iV"}, "emitted_at": 1681752753912} +{"stream": "surveys", "data": {"title": "yswa8kobijei1mkwaqxgy", "nickname": "7b4p9vssf810mslcd0eqpcg9s7p0h", "language": "it", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T10:59:00", "id": "307785429", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785429", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=YRpP3kxXMi2aJkgYoeyZrvuErii13mQ5DRN67Vm4WJ5avIMZ6YvzI_2Bc3FpERJDqx"}, "emitted_at": 1681752754883} +{"stream": "surveys", "data": {"title": "wsfqk1di34d", "nickname": "3pax9qjasev22ofir5dm4x45s", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:00:00", "id": "307785444", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785444", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=8fhTDITxRdeFQ_2BLWzVWxsJSU0nfgGgIbcvuAJ6C8LCdfivLn4FPj6xZP2o8_2FINMc"}, "emitted_at": 1681752755938} +{"stream": "surveys", "data": {"title": "vpoha5euc66vp", "nickname": "etv0tds1e45", "language": "en", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:19:00", "date_modified": "2021-06-10T11:01:00", "id": "307785394", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785394", "analyze_url": "https://www.surveymonkey.com/analyze/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "summary_url": "https://www.surveymonkey.com/summary/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=RPtFlMc_2B10dLjP_2BMbJ9eseMZkg_2FE5of5WUErPhwmC57Ij1xz0JOW7uC8i49BGEl8"}, "emitted_at": 1681752757038} +{"stream": "surveys", "data": {"title": "s2d9px7cdril0v7789ab4f", "nickname": "wnhin1ctnss8ebdgjef", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:02:00", "id": "307785402", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785402", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=RId_2BH4A8CUUX6zNvnFnfsb3wKFGuv8kLhz_2BApiG6Mbvu_2BLypJpz_2BM9EfoUuRXBcL"}, "emitted_at": 1681752757741} +{"stream": "surveys", "data": {"title": "muxx41av9mp", "nickname": "hfs5uo9cw1ce3j7rn7n8ncu88myc", "language": "de", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:02:00", "id": "307785408", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785408", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=aPPwgAP8stOxnud8WXi8VHKMrUtKIqvd7JhVnp1f0Ucqjb7cbMATzGizMgcLO_2BzA"}, "emitted_at": 1681752758782} +{"stream": "surveys", "data": {"title": "2iokp4jvp9ru5", "nickname": "j2a0kxhq8lmawfqjkg0hx", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 21, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:03:00", "id": "307784834", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784834", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=5ovW9LrIPsdAlqAloFxtr_2BhNVquSJyqeGOsrEnkZn56chkdLSKQISgfvIUejUonU"}, "emitted_at": 1681752759856} +{"stream": "surveys", "data": {"title": "9cnwcmdn39ox", "nickname": "vih7eeixclb", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:03:00", "id": "307785448", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785448", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=CJF1TcWP7MthjVWkHhiX8ggWVFe484BBhGYkoi2XqDCTB9FcR1nlBSJ_2FeL47hNDV"}, "emitted_at": 1681752760573} +{"stream": "surveys", "data": {"title": "i2bm4lqt5hxv614n4jcl0guxt5ehgf", "nickname": "ti241ke4qo1i8iyqgpo0u6b2", "language": "de", "folder_id": "0", "category": "", "question_count": 2, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:04:00", "id": "307784863", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784863", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=kSWrVa29zWxvAB20ibAMkgPUDRBw_2B_2BTV0eX3oRPIEDNdxc2vxtOGJkQUGITIEart"}, "emitted_at": 1681752761722} +{"stream": "surveys", "data": {"title": "j057iyqgxlotswo070", "nickname": "wxuyqq4cgmfo69ik778r", "language": "ru", "folder_id": "0", "category": "", "question_count": 6, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:05:00", "id": "307784846", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784846", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=c5YOr2YH8sSXNlr7K0Tsuhs54aXml2seWFuXS8MIqk7n5MinBfQ7OjzW_2BtjWV1Cv"}, "emitted_at": 1681752762326} +{"stream": "surveys", "data": {"title": "u7r02s47jr", "nickname": "ye7fubxhua91ce0fxm", "language": "ru", "folder_id": "0", "category": "", "question_count": 3, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:05:00", "id": "307784856", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784856", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=hW7YBNPL4euOVIMWVdvchE0xWtfXGoQrT7wUyFAtDel65HbgmAwoV7JJRkkkFmhn"}, "emitted_at": 1681752763451} +{"stream": "surveys", "data": {"title": "igpfp2yfsw90df6nxbsb49v", "nickname": "h23gl22ulmfsyt4q7xt", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:19:00", "date_modified": "2021-06-10T11:06:00", "id": "307785388", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785388", "analyze_url": "https://www.surveymonkey.com/analyze/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "summary_url": "https://www.surveymonkey.com/summary/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=khUJQv9z4_2FXXzGUox57WEUPwppIr8YqRqVru77WpakX1HW8hHMmGXZiDGslFZym6"}, "emitted_at": 1681752764067} +{"stream": "surveys", "data": {"title": "b9jo5h23l7pa", "nickname": "qhs5vg2qi0o4arsjiwy2ay00n82n", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:07:00", "id": "307785415", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785415", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=YVdtL_2BP5oiGTrfksyofvENkBr7v87Xfh8hbcJr8rbqgesWvwJjz5N1F7pCSRcDoy"}, "emitted_at": 1681752765140} +{"stream": "surveys", "data": {"title": "jjj", "nickname": "", "language": "en", "folder_id": "0", "category": "", "question_count": 0, "page_count": 1, "response_count": 0, "date_created": "2023-01-17T09:17:00", "date_modified": "2023-01-17T09:17:00", "id": "510388524", "buttons_text": {"next_button": "Next", "prev_button": "Prev", "done_button": "Done", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "10292568", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/510388524", "analyze_url": "https://www.surveymonkey.com/analyze/VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "summary_url": "https://www.surveymonkey.com/summary/VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=TPlNncbuCs17cvxwjuS74VC03_2FOcqpP_2F03m2gerTSI_2FQvLWoY2yn_2FWxLDmxYOp5L"}, "emitted_at": 1681752766116} +{"stream": "survey_responses", "data": {"id": "12706126725", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "124.123.178.184", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hNu3QJYf07WiUPOwxCYcARURFGB3ruOrs9slcTHVhmhgDhoNJ0k7w3jCvo0nLM40", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706126725", "total_time": 62, "date_modified": "2021-06-01T17:40:54+00:00", "date_created": "2021-06-01T17:39:51+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706126725", "pages": [{"id": "165250506", "questions": [{"id": "652286715", "answers": [{"choice_id": "4285525064"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525089"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525074"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525058", "row_id": "4285525061", "choice_metadata": {"weight": "0"}}]}]}]}, "emitted_at": 1690184765464} +{"stream": "survey_responses", "data": {"id": "12706152767", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "37.229.17.15", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YIZz5DiXEDES47ARxTbRPzAA9ZOwCjcN_2FDSFTYGWgCVPQCo_2B3EeLirGlON5_2BjrX5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706152767", "total_time": 55, "date_modified": "2021-06-01T17:50:03+00:00", "date_created": "2021-06-01T17:49:08+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706152767", "pages": [{"id": "165250506", "questions": [{"id": "652286726", "answers": [{"tag_data": [], "text": "fuck this"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525067"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525087"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525072"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525091"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525077"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525052", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "waste of time"}]}]}]}, "emitted_at": 1690184765465} +{"stream": "survey_responses", "data": {"id": "12706159691", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "157.48.231.67", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=P4z1eeLex6p2OQEYYXRJKBxPyHk6ljkOskXPds2olEToYrU_2FwTZWAyllEtgJRyQL", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706159691", "total_time": 104, "date_modified": "2021-06-01T17:52:27+00:00", "date_created": "2021-06-01T17:50:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706159691", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "78.63", "y": "13.19"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "41.94", "y": "50.16"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525086"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525055", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}]}]}, "emitted_at": 1690184765465} +{"stream": "survey_responses", "data": {"id": "12706182356", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "76.14.176.236", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=oRBufZrmuNVZW5ou_2B4ZICuFqW6p3uYPmpjTb5IJ5Zf_2BH4FPoHKfnz_2BSC_2FR_2FpxWNq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706182356", "total_time": 36, "date_modified": "2021-06-01T18:00:12+00:00", "date_created": "2021-06-01T17:59:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706182356", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "58.66", "y": "54.49"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "56.64", "y": "71.09"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525063"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525072"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525082"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525092"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525077"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525060", "row_id": "4285525061", "choice_metadata": {"weight": "100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "gekkiadsuigasdf;oij sefhello \ud83c\udf50\ud83c\udf50\ud83c\udf50\ud83c\udf50\ud83c\udf50"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "good"}]}]}]}, "emitted_at": 1690184765465} +{"stream": "survey_responses", "data": {"id": "12706201784", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.37.158.6", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MD0lXMX2bJm93XLiSKWuW2p52_2BwlWpayf88naadbuO5wITz0TijA3kwSis907xu1", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706201784", "total_time": 183, "date_modified": "2021-06-01T18:06:11+00:00", "date_created": "2021-06-01T18:03:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706201784", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "87.90", "y": "67.29"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "7.66", "y": "92.21"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "Colour"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525064"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525055", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Nothing"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "I don't know"}]}]}]}, "emitted_at": 1690184765466} +{"stream": "survey_responses", "data": {"id": "12706203862", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.205.239.133", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=sNkTdEY_2FEibixIxcUhotq6hQ3muFVlLkg0cE531VDB5Ya2U21pwazZRwwSXFqqtK", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706203862", "total_time": 114, "date_modified": "2021-06-01T18:06:49+00:00", "date_created": "2021-06-01T18:04:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706203862", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "40.32", "y": "93.35"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "37.10", "y": "66.67"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "because pets should not be tied,they should have their own freedom to move"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525080"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525055", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}]}]}, "emitted_at": 1690184765466} +{"stream": "survey_responses", "data": {"id": "12706264166", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "27.6.69.132", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=OboVhBA1nCZ4ejfYCmJ6WDu5SxhIUnnNn2dCz_2BTqxksFnjOSpy88MtS4B5Wbpk_2BW", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706264166", "total_time": 309, "date_modified": "2021-06-01T18:27:03+00:00", "date_created": "2021-06-01T18:21:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706264166", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "43.59", "y": "28.51"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "41.31", "y": "91.08"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "\ud83e\udd14"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525091"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525058", "row_id": "4285525061", "choice_metadata": {"weight": "0"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "\ud83e\uddb4"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "\ud83d\udcaa"}]}]}]}, "emitted_at": 1690184765466} +{"stream": "survey_responses", "data": {"id": "12706274940", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.205.116.166", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=D1IcNXq6BKOBmkySTfHcUfHuH3_2Fa0aniMPQEG23UrQ16iSsyx8ye2hPRQt3C61Jd", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706274940", "total_time": 105, "date_modified": "2021-06-01T18:30:56+00:00", "date_created": "2021-06-01T18:29:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706274940", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "7.12", "y": "89.86"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "87.90", "y": "9.02"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "Didn't like the product"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525067"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525087"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525072"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525092"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525077"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525050", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Nothing"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Better don't try to work on it"}]}]}]}, "emitted_at": 1690184765467} +{"stream": "survey_responses", "data": {"id": "12706353147", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "176.37.67.33", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=qwY5AKZSqBd7DfoZDGr4x_2FJr28RhtoeaQ7_2F6VBS1G3yK_2FPH86sPCcFs1zACVlbMO", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706353147", "total_time": 162, "date_modified": "2021-06-01T18:58:44+00:00", "date_created": "2021-06-01T18:56:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706353147", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "78.00", "y": "6.84"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "77.62", "y": "8.50"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "I like logo."}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525056", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Logo."}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Nothing."}]}]}]}, "emitted_at": 1690184765467} +{"stream": "survey_responses", "data": {"id": "12707255568", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "157.48.145.117", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3XYG_2F55lVJgeQ0_2FWG3xyRxOF5gCKFR0p1HPkdv1iiMZ1h5MIYxNR12enFBgK9TCS", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12707255568", "total_time": 263, "date_modified": "2021-06-02T01:13:33+00:00", "date_created": "2021-06-02T01:09:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12707255568", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "38.17", "y": "40.98"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "42.90", "y": "93.75"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "I love puppies but hate heart symbol"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525063"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525080"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525089"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525075"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525056", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "It's for animals"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Approach to customer"}]}]}]}, "emitted_at": 1690184765467} +{"stream": "survey_responses", "data": {"id": "12707566461", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "106.195.73.137", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FskCSEofrdcabC7MVfRNtzeiZ4C4kiwx_2FpBdpRfIsd5SgVGi4N9znXMS9exRXf27", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12707566461", "total_time": 233, "date_modified": "2021-06-02T04:06:48+00:00", "date_created": "2021-06-02T04:02:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12707566461", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "32.67", "y": "32.51"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "89.00", "y": "55.34"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525069"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525080"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525075"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525058", "row_id": "4285525061", "choice_metadata": {"weight": "0"}}]}]}]}, "emitted_at": 1690184765468} +{"stream": "survey_responses", "data": {"id": "12709748835", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.196.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=x5BdjfYWF_2B2dRs9qPRcHJ_2BqMN8Dpfn_2FfhKhtD7G8W4_2BX9ECDWh9wAXjYd4mxXk_2F_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12709748835", "total_time": 127, "date_modified": "2021-06-02T18:14:19+00:00", "date_created": "2021-06-02T18:12:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12709748835", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "41.85", "y": "55.96"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "31.35", "y": "35.64"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "because"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525063"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525069"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525089"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525074"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525060", "row_id": "4285525061", "choice_metadata": {"weight": "100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "u"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "f"}]}]}]}, "emitted_at": 1690184765468} +{"stream": "survey_responses", "data": {"id": "12706107193", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.37.150.53", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=7vMzRy38ln3J_2FJiU8YD1uve9yI6cAQIQ_2FuVxRipS_2FyB57w9vo9xpgShOuFWVcoI2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706107193", "total_time": 607550, "date_modified": "2021-06-08T18:17:17+00:00", "date_created": "2021-06-01T17:31:26+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706107193", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "77.56", "y": "6.01"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "15.71", "y": "8.23"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "It seems in that way"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525075"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525050", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Nothing much"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Will include Tagline"}]}]}]}, "emitted_at": 1690184765469} +{"stream": "survey_responses", "data": {"id": "12731040927", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YORvkBiLvNm2647vGYxs1GGGoUjDrz_2FRfIWw1i07UtykH_2BBJHDTB3ujkOPfyAxqP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731040927", "total_time": 31, "date_modified": "2021-06-10T08:46:53+00:00", "date_created": "2021-06-10T08:46:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731040927", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175881"}]}]}]}, "emitted_at": 1690184766569} +{"stream": "survey_responses", "data": {"id": "12731055204", "recipient_id": "", "collection_mode": "default", "response_status": "partial", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": ["168831413", "168831415", "168831437"], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tcbSSptg67E5MmPiY_2BCTC0GEk5rcm_2FHHcASKwxBGLOX_2BBByesO_2Fh848B_2FqaaVF8d", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731055204", "total_time": 15, "date_modified": "2021-06-10T08:54:22+00:00", "date_created": "2021-06-10T08:54:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731055204", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": []}]}, "emitted_at": 1690184766571} +{"stream": "survey_responses", "data": {"id": "12731069666", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tHWbA6E0Q6UEylVBreS0XaGKh3GcjDQu_2FBytp_2F_2FcCSkYTgRKkVt3jyyhUFoh6T_2Bs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731069666", "total_time": 33, "date_modified": "2021-06-10T09:02:19+00:00", "date_created": "2021-06-10T09:01:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731069666", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175733"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175752"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1690184766572} +{"stream": "survey_responses", "data": {"id": "12731085951", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=c_2B3Xk1rn3Lhz9GqaPNSRmjd03YiGx88BQu_2BtAvHiMmp5a0BR68kWLCOfALzBwKH4", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731085951", "total_time": 31, "date_modified": "2021-06-10T09:10:05+00:00", "date_created": "2021-06-10T09:09:34+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731085951", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175564"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175745"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1690184766574} +{"stream": "survey_responses", "data": {"id": "12731102076", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=s_2BQLUH6Nb049vqmxYdKTQlIV_2FCXzxmRR5F_2B_2Fe9FaqXRh3H_2FZAFF51mqyI3e8s666", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731102076", "total_time": 32, "date_modified": "2021-06-10T09:17:44+00:00", "date_created": "2021-06-10T09:17:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731102076", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175561"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175739"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175745"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1690184766576} +{"stream": "survey_responses", "data": {"id": "12731118899", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hRALwRgDjAOTNk8Hpt0wWWn9X3xurqWW9vynXjPkvIvI4Xofu_2FJSgEVwtWK23vLd", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731118899", "total_time": 31, "date_modified": "2021-06-10T09:25:25+00:00", "date_created": "2021-06-10T09:24:53+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731118899", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175734"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175745"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1690184766577} +{"stream": "survey_responses", "data": {"id": "12731135865", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=OEPuy4AY_2B47LLaYrOMe90r_2BJdHiKxv12FpoQr0N94GGA0TFN7l7tk5QH14HD_2FaBy", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731135865", "total_time": 31, "date_modified": "2021-06-10T09:33:08+00:00", "date_created": "2021-06-10T09:32:37+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731135865", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175748"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1690184766578} +{"stream": "survey_responses", "data": {"id": "12731153599", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KPgqZwkF7GLrzwtQWhwpZBCuPbq_2F_2BuXfuUXpCBXX1y_2BwKBvvXi7s1ob9AMVJymCk", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731153599", "total_time": 31, "date_modified": "2021-06-10T09:40:49+00:00", "date_created": "2021-06-10T09:40:18+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731153599", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175561"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175751"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1690184766580} +{"stream": "survey_responses", "data": {"id": "12731170943", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=e6QtiM7XZfbHB3Ln1ifDr6Ct8PD0j6Nikh_2BTvBikLVfpeCSzS5WYkg2D_2BDVygAee", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731170943", "total_time": 31, "date_modified": "2021-06-10T09:48:36+00:00", "date_created": "2021-06-10T09:48:04+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731170943", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175733"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175748"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175879"}]}]}]}, "emitted_at": 1690184766581} +{"stream": "survey_responses", "data": {"id": "12731188992", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VATG0VEAVLLh8CTsnDoMZ_2FR3ida2C_2FIqVwt3FHGuc0GRu_2BA6Qa9E4Ewd3iEHt5YQ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731188992", "total_time": 35, "date_modified": "2021-06-10T09:56:51+00:00", "date_created": "2021-06-10T09:56:15+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731188992", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175738"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175881"}]}]}]}, "emitted_at": 1690184766583} +{"stream": "survey_responses", "data": {"id": "12731208790", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KH_2Bhsc8oIpTdaX60RIwFQ1SDyudOf2_2B61W7cPCZGGXEKsY5_2FN_2BA_2B_2FXh0DeOOdA7F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731208790", "total_time": 32, "date_modified": "2021-06-10T10:05:01+00:00", "date_created": "2021-06-10T10:04:28+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731208790", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175752"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1690184766584} +{"stream": "survey_responses", "data": {"id": "12731228560", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vbsxUKDxxVbnJW7Gng6Y5VblTyjW_2F29grieRYQImUaIVF77GuhY3KDlqPsNfIlx6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731228560", "total_time": 31, "date_modified": "2021-06-10T10:12:51+00:00", "date_created": "2021-06-10T10:12:19+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731228560", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175561"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175882"}]}]}]}, "emitted_at": 1690184766585} +{"stream": "survey_responses", "data": {"id": "12731247619", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=8lyrgf9sVRbhXAHse5glKYEJoYMvqlR2TNQIhI8Ycw716W_2FHlQeX6Ru4SKObInXR", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731247619", "total_time": 31, "date_modified": "2021-06-10T10:20:41+00:00", "date_created": "2021-06-10T10:20:09+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731247619", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175734"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175747"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175881"}]}]}]}, "emitted_at": 1690184766587} +{"stream": "survey_responses", "data": {"id": "12731266056", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tkqgGP2ApNTakGiA0JrfYbKgIZElTj6_2FPaJ_2FDj0QNjpJDKCexid0Z_2Bq8vZoam1vK", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731266056", "total_time": 31, "date_modified": "2021-06-10T10:28:20+00:00", "date_created": "2021-06-10T10:27:49+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731266056", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175738"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175747"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175882"}]}]}]}, "emitted_at": 1690184766588} +{"stream": "survey_responses", "data": {"id": "12731286200", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=nYsBr3IsP19O_2BhTeV_2BWGnAdXc_2FFsnqMT6maJ0BS3QD9FqjkWLrNYzzEqKsT7c191", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731286200", "total_time": 35, "date_modified": "2021-06-10T10:36:27+00:00", "date_created": "2021-06-10T10:35:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731286200", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175367"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175564"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175738"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175752"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1690184766589} +{"stream": "survey_responses", "data": {"id": "12731305366", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=iIxE7kshwG8vR6T6c2D0swVsNTfDLqcBVkfrnf_2FGZBTuoHjMm9ksd3LbOIXuF6lp", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731305366", "total_time": 34, "date_modified": "2021-06-10T10:44:10+00:00", "date_created": "2021-06-10T10:43:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731305366", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175879"}]}]}]}, "emitted_at": 1690184766591} +{"stream": "survey_responses", "data": {"id": "12731325134", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=cUq2_2BSagT6pk152_2BsGjjMAAQ2Qq0R0cxleIxEdHEEVKwV5oPCvktmmnTV82pa5S_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731325134", "total_time": 31, "date_modified": "2021-06-10T10:52:02+00:00", "date_created": "2021-06-10T10:51:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731325134", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175739"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175751"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1690184766592} +{"stream": "survey_responses", "data": {"id": "12731344038", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=F_2BuWfRunQKdf1g0xuu4mTzXMCAlXia21_2BnF47p_2FbjU4QII1hLaE3Xo6tWdfbzKqr", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731344038", "total_time": 31, "date_modified": "2021-06-10T10:59:42+00:00", "date_created": "2021-06-10T10:59:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731344038", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175733"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175739"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175750"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1690184766593} +{"stream": "survey_responses", "data": {"id": "12731042086", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=SvCebsqkF1mO3kCZX7XsmQ4ABz0LFCf_2FyW5N2JOLuGh5ixjzbj2i04SarOUiUgPa", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731042086", "total_time": 31, "date_modified": "2021-06-10T08:47:30+00:00", "date_created": "2021-06-10T08:46:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731042086", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176960"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177005"}]}]}]}, "emitted_at": 1690184767383} +{"stream": "survey_responses", "data": {"id": "12731056238", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=r7v44AE_2Bx_2Fam5dgCBnlUkUkL0aSWyzhJkIPWmmYa5VnqMtd4X2DBzf9U9erpnCvI", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731056238", "total_time": 31, "date_modified": "2021-06-10T08:55:13+00:00", "date_created": "2021-06-10T08:54:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731056238", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176738"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176962"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176979"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177005"}]}]}]}, "emitted_at": 1690184767383} +{"stream": "survey_responses", "data": {"id": "12731070937", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=_2FuS4ZECIFCnV4LLPEe6gryzUPrf_2FMB8MB51Cv5cU6IQGsWb_2FRIb0BOaRXRodLTAq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731070937", "total_time": 32, "date_modified": "2021-06-10T09:02:57+00:00", "date_created": "2021-06-10T09:02:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731070937", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176767"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176975"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176978"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176990"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177007"}]}]}]}, "emitted_at": 1690184767383} +{"stream": "survey_responses", "data": {"id": "12731087215", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=eSF0qBZZ3aR1SHBtkTZz83mgEBvJuBfou_2BJGBoFPbxv5bluXZuIMKDaGn5x5xba5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731087215", "total_time": 31, "date_modified": "2021-06-10T09:10:42+00:00", "date_created": "2021-06-10T09:10:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731087215", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176736"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176918"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176962"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176972"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176983"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177007"}]}]}]}, "emitted_at": 1690184767384} +{"stream": "survey_responses", "data": {"id": "12731103402", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QOikOvuCCRFkDsJoRozKO9_2BNPZmp7mliotl5xt4QnStPgBKgrvz6GZ7vdHXf3eMG", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731103402", "total_time": 31, "date_modified": "2021-06-10T09:18:22+00:00", "date_created": "2021-06-10T09:17:50+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731103402", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176736"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176962"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176974"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176978"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177002"}]}]}]}, "emitted_at": 1690184767384} +{"stream": "survey_responses", "data": {"id": "12731120214", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=sV4Oq2KXvMuxWQpflbd5L_2FyoB6ImaDnm9BFwJGk9W_2BwE7YWlvW6MsxaqaeWWY_2Fzh", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731120214", "total_time": 31, "date_modified": "2021-06-10T09:26:01+00:00", "date_created": "2021-06-10T09:25:29+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731120214", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176734"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176960"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176968"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176976"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177009"}]}]}]}, "emitted_at": 1690184767384} +{"stream": "survey_responses", "data": {"id": "12731137499", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Hy3W8rkVoiYFKw_2BdJM6cChYRJWAXaGTQTol4ykCh_2FxYDhPmBpC753rbLshVSZjNE", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731137499", "total_time": 31, "date_modified": "2021-06-10T09:33:45+00:00", "date_created": "2021-06-10T09:33:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731137499", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176738"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176766"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176982"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177008"}]}]}]}, "emitted_at": 1690184767385} +{"stream": "survey_responses", "data": {"id": "12731154912", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=BLjiuDSDI39GvYDURd_2FmEfCd9jRrZ5nLwAgWG65zJj2dA1DskdMkMBHVZjzLHkk6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731154912", "total_time": 31, "date_modified": "2021-06-10T09:41:24+00:00", "date_created": "2021-06-10T09:40:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731154912", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176738"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176963"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177006"}]}]}]}, "emitted_at": 1690184767385} +{"stream": "survey_responses", "data": {"id": "12731172230", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=jiMm_2FJVQj8fPN6i2HlASqxeTd4rx_2FcPoCLMAznqiCWgY_2B98x39SfA1kKHTo8CgEC", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731172230", "total_time": 31, "date_modified": "2021-06-10T09:49:12+00:00", "date_created": "2021-06-10T09:48:40+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731172230", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176737"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176767"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176963"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176972"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176990"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177006"}]}]}]}, "emitted_at": 1690184767385} +{"stream": "survey_responses", "data": {"id": "12731190528", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VrEu2qgFZYPclcWCzyV5DO0WUUCjOIMFC77k1XLOofwOqYD5vHAej03viembeuF8", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731190528", "total_time": 35, "date_modified": "2021-06-10T09:57:32+00:00", "date_created": "2021-06-10T09:56:57+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731190528", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176725"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176959"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176972"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176978"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177004"}]}]}]}, "emitted_at": 1690184767386} +{"stream": "survey_responses", "data": {"id": "12731210366", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=xOrc7mbcYE0NFiqzEpenJPpJJ1OHiCt_2FUDSohMb9nmqttp0v1il2MNjb_2BRGE177T", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731210366", "total_time": 33, "date_modified": "2021-06-10T10:05:40+00:00", "date_created": "2021-06-10T10:05:06+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731210366", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176722"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176971"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177003"}]}]}]}, "emitted_at": 1690184767386} +{"stream": "survey_responses", "data": {"id": "12731230116", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=xmmEsbDmCA6Nvi6PKgkHAZR5hqRDTYPELB_2BIyMCUzF63brYH0R2ZrSJb9f_2BXtWRa", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731230116", "total_time": 32, "date_modified": "2021-06-10T10:13:28+00:00", "date_created": "2021-06-10T10:12:55+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731230116", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176959"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176973"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177007"}]}]}]}, "emitted_at": 1690184767386} +{"stream": "survey_responses", "data": {"id": "12731249077", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=1cQksQ2IA1zuU8mpR9JawSnSqr2zaPV1juVQ7ZEPrEqxr231kL_2F2eIW48UokyJBm", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731249077", "total_time": 31, "date_modified": "2021-06-10T10:21:17+00:00", "date_created": "2021-06-10T10:20:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731249077", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176963"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176968"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176982"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176989"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177008"}]}]}]}, "emitted_at": 1690184767387} +{"stream": "survey_responses", "data": {"id": "12731267503", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zOrptpJoo7QbXn1DoS0HIo4GcoAK8cIZ83MYidVo_2FuK_2FDvnnrsXK2SDzhyjssMdI", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731267503", "total_time": 31, "date_modified": "2021-06-10T10:28:56+00:00", "date_created": "2021-06-10T10:28:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731267503", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176736"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176983"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177003"}]}]}]}, "emitted_at": 1690184767387} +{"stream": "survey_responses", "data": {"id": "12731287789", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zXARfqNZgptvNWp9PboSVlElugTboBiAj_2FSG_2BkbD8e8OUJGsJ8FMRpMFap0qzcYY", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731287789", "total_time": 31, "date_modified": "2021-06-10T10:37:03+00:00", "date_created": "2021-06-10T10:36:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731287789", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176726"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176918"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176965"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176971"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176981"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176989"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177004"}]}]}]}, "emitted_at": 1690184767387} +{"stream": "survey_responses", "data": {"id": "12731307187", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=d07mxUJIp3BjkeFrp8gTQvz67vJrBUQWNX2tzrHPvkrZ0piZOPTQDKl_2BrNNjOvJO", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731307187", "total_time": 35, "date_modified": "2021-06-10T10:44:49+00:00", "date_created": "2021-06-10T10:44:14+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731307187", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176734"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176964"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176975"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176982"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177004"}]}]}]}, "emitted_at": 1690184767388} +{"stream": "survey_responses", "data": {"id": "12731326595", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=BCy4vPYrkiObJMS9B6GjZD2jwLTDr1luSZYtqGxH8zxGvmteSPixhqGNTTdp_2BRwb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731326595", "total_time": 32, "date_modified": "2021-06-10T10:52:38+00:00", "date_created": "2021-06-10T10:52:05+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731326595", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176722"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176767"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176918"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176959"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176974"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176981"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176990"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177003"}]}]}]}, "emitted_at": 1690184767388} +{"stream": "survey_responses", "data": {"id": "12731345509", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KmG9rTOuf2GcFWGtNhaBIfC5S80jehYrziix2nmuPzJw9_2FzLpaENGb_2F8j4NkPfTJ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731345509", "total_time": 31, "date_modified": "2021-06-10T11:00:18+00:00", "date_created": "2021-06-10T10:59:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731345509", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176725"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176734"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176766"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176964"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176968"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176983"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176991"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177005"}]}]}]}, "emitted_at": 1690184767388} +{"stream": "survey_responses", "data": {"id": "12731043150", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=LKTTnPMSZ1aoM4WL0f1Ja7_2FdIZyTQ0DexNwU_2FFgEeZeT6B9T2m2HJuQjxPHn2OpZ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731043150", "total_time": 36, "date_modified": "2021-06-10T08:48:12+00:00", "date_created": "2021-06-10T08:47:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731043150", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173238"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173269"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173482"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173525"}]}]}]}, "emitted_at": 1690184768429} +{"stream": "survey_responses", "data": {"id": "12731057303", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Tj5vPzxUmWLsaqGk6A3N4tXy8F1lrXDh3cgduiFC1tcjk3fKqd5Jc_2FYl9kqbrtXl", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731057303", "total_time": 35, "date_modified": "2021-06-10T08:55:53+00:00", "date_created": "2021-06-10T08:55:17+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731057303", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173478"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1690184768430} +{"stream": "survey_responses", "data": {"id": "12731072147", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=bAFsDUmEPpCdobkl_2BIl_2F6rdICCatPVmCxnojDaOR9ZJwGzYj7h29WMvC195fRe31", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731072147", "total_time": 35, "date_modified": "2021-06-10T09:03:38+00:00", "date_created": "2021-06-10T09:03:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731072147", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173240"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173257"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173268"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173452"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173476"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173483"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1690184768430} +{"stream": "survey_responses", "data": {"id": "12731088506", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MrquTcpA82ItgrSQWG_2BswTK4ClFm9rTuG5GWb85hiLDvaK2dr6F7vrYvOlwqkkC6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731088506", "total_time": 35, "date_modified": "2021-06-10T09:11:22+00:00", "date_created": "2021-06-10T09:10:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731088506", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173236"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173269"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173458"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173484"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173525"}]}]}]}, "emitted_at": 1690184768431} +{"stream": "survey_responses", "data": {"id": "12731104696", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=RO8uULIefm7fQ9VJXKzxtKBfRj4g0rYOE0LRuSA8OT2GVPEFDFQFmSOII7UlshDk", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731104696", "total_time": 36, "date_modified": "2021-06-10T09:19:03+00:00", "date_created": "2021-06-10T09:18:26+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731104696", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173240"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173474"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1690184768431} +{"stream": "survey_responses", "data": {"id": "12731121617", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Z_2BP_2BwLxa1Cmlcetqz6B1oaPvJSUU_2F5d018eOau5jIs8q4IlOyw7MT9YtxrlZnZHx", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731121617", "total_time": 37, "date_modified": "2021-06-10T09:26:44+00:00", "date_created": "2021-06-10T09:26:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731121617", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173228"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173268"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173458"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1690184768431} +{"stream": "survey_responses", "data": {"id": "12731139029", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=R0ZtH7VCYs_2FMuOE8hZBLqJdLMcu2RDqvrV04C34ivpIzGYhRzaVoAG0LDF83Ln_2B2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731139029", "total_time": 37, "date_modified": "2021-06-10T09:34:27+00:00", "date_created": "2021-06-10T09:33:50+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731139029", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173269"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173415"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173459"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173482"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1690184768432} +{"stream": "survey_responses", "data": {"id": "12731156353", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=t_2BeAHBUtkTES9TALR_2F4aDfYgtWlbqJv08b_2B6EYbplYKefvn8GrYTkoGugKOevrCn", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731156353", "total_time": 35, "date_modified": "2021-06-10T09:42:06+00:00", "date_created": "2021-06-10T09:41:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731156353", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173455"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173474"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1690184768432} +{"stream": "survey_responses", "data": {"id": "12731173574", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=mVe08d8mokL6AWRdTdpFz7jaz6dJhmAV4ZlfpEH6B8OQwQXlC2RxAxy2Z_2BtSrLXG", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731173574", "total_time": 36, "date_modified": "2021-06-10T09:49:53+00:00", "date_created": "2021-06-10T09:49:16+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731173574", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173241"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173458"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1690184768432} +{"stream": "survey_responses", "data": {"id": "12731192021", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=XFNknH0sj9IIdSvzYR75kcMldl4TuTz7kma3E7SGri2Rq_2FvcCRLj4ug5HOCnqMQz", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731192021", "total_time": 39, "date_modified": "2021-06-10T09:58:16+00:00", "date_created": "2021-06-10T09:57:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731192021", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173456"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1690184768433} +{"stream": "survey_responses", "data": {"id": "12731211903", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=V3KUOw0NUFH27PiVeUNuTKv_2BkMDveRdHYnln_2FOZcYY0I7L0VIMlksx3CdNizHX_2BP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731211903", "total_time": 36, "date_modified": "2021-06-10T10:06:21+00:00", "date_created": "2021-06-10T10:05:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731211903", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173456"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173484"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1690184768433} +{"stream": "survey_responses", "data": {"id": "12731231636", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=xv8ZLGu_2FUg9VAJ8hvx10oji6MvB1PbiAyA98qWpExvZvSaPKJ7cIFH8nZktkDKH0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731231636", "total_time": 36, "date_modified": "2021-06-10T10:14:08+00:00", "date_created": "2021-06-10T10:13:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731231636", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173226"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173454"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173527"}]}]}]}, "emitted_at": 1690184768433} +{"stream": "survey_responses", "data": {"id": "12731250495", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=juR_2BEnc_2BsJLlCc6Sg7p2Ql_2B4icSx_2BsZwo_2FU1RVgYxqI2OzngMKnAUlrdJUS39ewn", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731250495", "total_time": 36, "date_modified": "2021-06-10T10:21:58+00:00", "date_created": "2021-06-10T10:21:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731250495", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173257"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173268"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173415"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173459"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173474"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1690184768434} +{"stream": "survey_responses", "data": {"id": "12731268932", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vXLZxpfaNUwy1JRrQLJCNL0kcVa5U5M5FtCofSN4MCcTJt9om9nRi2D4C4xJeXUg", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731268932", "total_time": 35, "date_modified": "2021-06-10T10:29:36+00:00", "date_created": "2021-06-10T10:29:00+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731268932", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173228"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173242"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173257"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173454"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173483"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173524"}]}]}]}, "emitted_at": 1690184768434} +{"stream": "survey_responses", "data": {"id": "12731289280", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=M4Lzmv89J2ZSnoBKgwxsZqDP3em9icPdsBglzN7NQB_2BM5AQ1bWpjH53fB0IsVZi6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731289280", "total_time": 36, "date_modified": "2021-06-10T10:37:44+00:00", "date_created": "2021-06-10T10:37:08+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731289280", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173242"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173454"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173485"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1690184768434} +{"stream": "survey_responses", "data": {"id": "12731308800", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=GQAEVhmAlwb5r_2FWfXNC_2Fqzqv5ULovU_2BGovk8oDp5nCvL2982DEXHbEhjSg4A3FdA", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731308800", "total_time": 39, "date_modified": "2021-06-10T10:45:33+00:00", "date_created": "2021-06-10T10:44:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731308800", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173459"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173473"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173482"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1690184768435} +{"stream": "survey_responses", "data": {"id": "12731328082", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MwpXysDo7Lhe8X4oi_2BfwDcfgG4XMlnjXbj783SvdsQcVCLn0Y0mvT87cyq6wLtri", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731328082", "total_time": 36, "date_modified": "2021-06-10T10:53:17+00:00", "date_created": "2021-06-10T10:52:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731328082", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173241"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173456"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173524"}]}]}]}, "emitted_at": 1690184768435} +{"stream": "survey_responses", "data": {"id": "12731346960", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=i9GzbJPTq_2BCfXqXCYc91kWSkkbD40J_2FGL8hESaU68w7nuZXOyFZ7xNpNth1mmf72", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731346960", "total_time": 35, "date_modified": "2021-06-10T11:00:58+00:00", "date_created": "2021-06-10T11:00:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731346960", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173241"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173476"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173483"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173527"}]}]}]}, "emitted_at": 1690184768435} +{"stream": "survey_responses", "data": {"id": "12731044345", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=RiSWqWbbXVsX_2BRIyudfJwY_2BODcC0aHC0L77gSNMTF5T_2BsPWUy2vlIWpXf01Hqy8k", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731044345", "total_time": 34, "date_modified": "2021-06-10T08:48:50+00:00", "date_created": "2021-06-10T08:48:16+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731044345", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173538"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173549"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173732"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1690184769552} +{"stream": "survey_responses", "data": {"id": "12731058644", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=8fjqeuyiG6EvuMWbYy64yqPXBnCSWaEpW8DInIUYXRbFyqerWPCENf5izNrPGoNw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731058644", "total_time": 33, "date_modified": "2021-06-10T08:56:31+00:00", "date_created": "2021-06-10T08:55:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731058644", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173662"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173678"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174063"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174183"}]}]}]}, "emitted_at": 1690184769552} +{"stream": "survey_responses", "data": {"id": "12731073567", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ILVcHPPYp1qBeboZrnUw9zU8OfPDI3pBaSHE_2FPA1mcsZYgDgyEvD2DNJEXG50BwI", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731073567", "total_time": 34, "date_modified": "2021-06-10T09:04:16+00:00", "date_created": "2021-06-10T09:03:42+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731073567", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173552"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173663"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173711"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173734"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174066"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174183"}]}]}]}, "emitted_at": 1690184769553} +{"stream": "survey_responses", "data": {"id": "12731089919", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=WJVGjucUzW8fBD4StuC_2FFZoPB0pCtugYpTzgDrCOMnJyyzUp5Q2v55jJ8xqcAaJv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731089919", "total_time": 33, "date_modified": "2021-06-10T09:12:00+00:00", "date_created": "2021-06-10T09:11:26+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731089919", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173536"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173551"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173658"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173714"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173728"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174060"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174184"}]}]}]}, "emitted_at": 1690184769553} +{"stream": "survey_responses", "data": {"id": "12731106311", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=pYr2BTIwVP0O3EzGC27aOOaCA0hnWvJ3FzJOiB_2Fhgw0UUSDh2QpQTcjrfXltZ8dv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731106311", "total_time": 33, "date_modified": "2021-06-10T09:19:41+00:00", "date_created": "2021-06-10T09:19:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731106311", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173549"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173663"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173680"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174061"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1690184769553} +{"stream": "survey_responses", "data": {"id": "12731123001", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=iMOq3PmDu_2Fpfu6Lqltp5VPoRV2azTczcfDSAY7AjMIzECAtQhUwU_2FNMCCwq_2BSvsN", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731123001", "total_time": 34, "date_modified": "2021-06-10T09:27:23+00:00", "date_created": "2021-06-10T09:26:49+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731123001", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173658"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1690184769554} +{"stream": "survey_responses", "data": {"id": "12731140625", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QTesi81CB_2BwJ62wJ8op06oDEuscTPvq1ke99azBrWELfhmlgdyrrpw0NVegVV_2BE7", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731140625", "total_time": 33, "date_modified": "2021-06-10T09:35:05+00:00", "date_created": "2021-06-10T09:34:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731140625", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173538"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173678"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174062"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174185"}]}]}]}, "emitted_at": 1690184769554} +{"stream": "survey_responses", "data": {"id": "12731157855", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Y_2BseyWnN45hqrdC63g5th2srbJCqZZuaaHgJaVUVwfuBPwNI2IdB2Dc1yEhLsRcy", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731157855", "total_time": 33, "date_modified": "2021-06-10T09:42:46+00:00", "date_created": "2021-06-10T09:42:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731157855", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173536"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174060"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174183"}]}]}]}, "emitted_at": 1690184769554} +{"stream": "survey_responses", "data": {"id": "12731175182", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=wvlpuaxYEnle4EG2hvBzqRUelWi_2BwXRQmZSU2ru959wJ7Ly3p7I9sg1wjhKu7Bm_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731175182", "total_time": 27, "date_modified": "2021-06-10T09:50:26+00:00", "date_created": "2021-06-10T09:49:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731175182", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": []}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173731"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174061"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1690184769554} +{"stream": "survey_responses", "data": {"id": "12731193598", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=mqmf_2BkbhXVJbx4WQXC0_2F72pU_2FFf6FkOVA8_2F7REJy4j9HAEPKg_2BCPmL8F2wfc6SKi", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731193598", "total_time": 37, "date_modified": "2021-06-10T09:58:58+00:00", "date_created": "2021-06-10T09:58:20+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731193598", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173539"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173657"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173714"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173733"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174065"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174185"}]}]}]}, "emitted_at": 1690184769555} +{"stream": "survey_responses", "data": {"id": "12731213708", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FmbWXWfmfbsDK8MEuKSsHPviZSUYgchniK99dQeGSFpf_2BWnh6cTWlg0o5YRIRtn7", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731213708", "total_time": 33, "date_modified": "2021-06-10T10:06:59+00:00", "date_created": "2021-06-10T10:06:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731213708", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173663"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173683"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174061"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1690184769555} +{"stream": "survey_responses", "data": {"id": "12731233283", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VRo6oIX2X5OOWhsAVRYKuhrXr5zszr6duv29Vq3RdwAJMtFNcQqAJk71Bi4Oq3Bo", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731233283", "total_time": 33, "date_modified": "2021-06-10T10:14:47+00:00", "date_created": "2021-06-10T10:14:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731233283", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173554"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173656"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173679"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173715"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174063"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174185"}]}]}]}, "emitted_at": 1690184769555} +{"stream": "survey_responses", "data": {"id": "12731252105", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=TQiDe8sSx4_2FuWzT1Lr7RKGOBO2iwOmTuPPOzcvcHL45tRjbdZSW9UHv6_2B2nLtY6n", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731252105", "total_time": 34, "date_modified": "2021-06-10T10:22:37+00:00", "date_created": "2021-06-10T10:22:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731252105", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173552"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173659"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173731"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174062"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1690184769556} +{"stream": "survey_responses", "data": {"id": "12731270690", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ipONp_2F6FrM7HwOh1_2BpJXXqL65aZR6EN_2BmCdmmlM4Etuv_2BqfU5P8wsYtpW2_2BvsbR2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731270690", "total_time": 33, "date_modified": "2021-06-10T10:30:16+00:00", "date_created": "2021-06-10T10:29:42+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731270690", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173536"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173656"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173715"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173734"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174064"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1690184769556} +{"stream": "survey_responses", "data": {"id": "12731290962", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=eWOUaV5xBiUO9JblKm5rYeLfk2_2FPRiygYknRKYIPlBC3Vl9805OBGP8f2kjnm0r2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731290962", "total_time": 33, "date_modified": "2021-06-10T10:38:22+00:00", "date_created": "2021-06-10T10:37:48+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731290962", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173538"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173658"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173678"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173715"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174066"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1690184769556} +{"stream": "survey_responses", "data": {"id": "12731310541", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=wFNF4x4oPfyHaT5uvGstRuK1DUDfrHJyxBXB95bRgrB_2FdJojFuoGIhxA0BIH0XSO", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731310541", "total_time": 37, "date_modified": "2021-06-10T10:46:14+00:00", "date_created": "2021-06-10T10:45:37+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731310541", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173551"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173732"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174184"}]}]}]}, "emitted_at": 1690184769557} +{"stream": "survey_responses", "data": {"id": "12731329746", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vGkpaT5QedSZDkH1LkdPEMwKHIlyZ2mKjKyt8tgek5uzfoU30oxZnac34WvjYm1h", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731329746", "total_time": 33, "date_modified": "2021-06-10T10:53:55+00:00", "date_created": "2021-06-10T10:53:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731329746", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173551"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173711"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173732"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1690184769557} +{"stream": "survey_responses", "data": {"id": "12731348717", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vuWK6q7Hz9dNybRoH2rP8Ux0qNc_2B0E3lMUmtDlPUbslKbHc7FranQhTJxg_2BIo3Pw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731348717", "total_time": 26, "date_modified": "2021-06-10T11:01:30+00:00", "date_created": "2021-06-10T11:01:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731348717", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": []}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173714"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1690184769557} +{"stream": "survey_responses", "data": {"id": "12731045521", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=0SWEpDfTb_2FAVFz7Ddryxk5koNgVExk8Oke4TJ8tz6QD5Eqc7uoqHQWRcLUi7yOAb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731045521", "total_time": 32, "date_modified": "2021-06-10T08:49:26+00:00", "date_created": "2021-06-10T08:48:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731045521", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174496"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770269} +{"stream": "survey_responses", "data": {"id": "12731059832", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vhiMNZdaQtV6_2FPkJM_2BP8UtinkM87qYXz4VjmfwvWnjK1NFrLnN1P_2FH3w9GU7JUxX", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731059832", "total_time": 32, "date_modified": "2021-06-10T08:57:08+00:00", "date_created": "2021-06-10T08:56:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731059832", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174644"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770270} +{"stream": "survey_responses", "data": {"id": "12731074829", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ZUZi7r7io9fCwXP7l4K5oirdqpAhNEdbesFS73p617lSOzxuK5oKGDsqiK8zdA8E", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731074829", "total_time": 31, "date_modified": "2021-06-10T09:04:52+00:00", "date_created": "2021-06-10T09:04:20+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731074829", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174622"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174645"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770270} +{"stream": "survey_responses", "data": {"id": "12731091270", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=eL4vDkGrDZfi5BiJC001E_2FoL_2Bo1mAtZxB97VuOw0Ue7x2Y_2Fvj9cWXWEFAG1_2For6z", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731091270", "total_time": 33, "date_modified": "2021-06-10T09:12:38+00:00", "date_created": "2021-06-10T09:12:05+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731091270", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174493"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770270} +{"stream": "survey_responses", "data": {"id": "12731107632", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VzELV_2Bc1GPJfJsXsRIabrQVAi9_2BSDOS6F_2B68zR_2BP4u98MGtxvy_2BeSOjCh5s1HKa5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731107632", "total_time": 32, "date_modified": "2021-06-10T09:20:17+00:00", "date_created": "2021-06-10T09:19:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731107632", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174254"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174496"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770270} +{"stream": "survey_responses", "data": {"id": "12731124423", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QIT1tkBlcTVTBkkHm7dpbWllNNXyBriN0RSobCplWUm5LoJpA_2FC93B5N8fFbc0Zs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731124423", "total_time": 31, "date_modified": "2021-06-10T09:28:00+00:00", "date_created": "2021-06-10T09:27:28+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731124423", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174484"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174620"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174645"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770271} +{"stream": "survey_responses", "data": {"id": "12731142107", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=URkLmYAmILJkZgOzrwRncRyLAGrxL_2FGHdeqmjSqrSYjS_2F7cjqy4e3nBWqRsOOv0r", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731142107", "total_time": 31, "date_modified": "2021-06-10T09:35:42+00:00", "date_created": "2021-06-10T09:35:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731142107", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174601"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174643"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770271} +{"stream": "survey_responses", "data": {"id": "12731159230", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=O0O262_2ByzCPZXGbz3NLSBdAJfB9rNAyLSz4YX4ubxd_2BIrVIo2jJ_2Bzk8KQNxCcjXb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731159230", "total_time": 31, "date_modified": "2021-06-10T09:43:20+00:00", "date_created": "2021-06-10T09:42:49+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731159230", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174643"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770271} +{"stream": "survey_responses", "data": {"id": "12731176347", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=LVM3eoF2xyKmckO7bkvBEw0i_2FoajMZMA_2Fc_2FVcI95ReDdVUuQQ_2Babxm1nILbPUISC", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731176347", "total_time": 33, "date_modified": "2021-06-10T09:51:04+00:00", "date_created": "2021-06-10T09:50:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731176347", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174486"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174676"}]}]}]}, "emitted_at": 1690184770272} +{"stream": "survey_responses", "data": {"id": "12731195152", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=SDlGK5Xxc7lpBehXWM_2B47eY24yPtwGB4NBBehIidUXtChXAXOgjA_2F8CxAHW97nUA", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731195152", "total_time": 35, "date_modified": "2021-06-10T09:59:39+00:00", "date_created": "2021-06-10T09:59:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731195152", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174647"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770272} +{"stream": "survey_responses", "data": {"id": "12731215248", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ft3pzaYpicC1h6Ah26Wj0wRkCmTK1Yu7BTxxTQ0po_2FC00oJaK3YHuX9XV5WB5T60", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731215248", "total_time": 32, "date_modified": "2021-06-10T10:07:34+00:00", "date_created": "2021-06-10T10:07:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731215248", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174493"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174676"}]}]}]}, "emitted_at": 1690184770273} +{"stream": "survey_responses", "data": {"id": "12731234853", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=2W_2B7kOeuvhHSN7s0oUQZQwt3iySUHWLv5qq0xw8gojd0RWsFcAMM6NILAk_2FZJbdX", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731234853", "total_time": 31, "date_modified": "2021-06-10T10:15:23+00:00", "date_created": "2021-06-10T10:14:51+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731234853", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174601"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174622"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174650"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770273} +{"stream": "survey_responses", "data": {"id": "12731253651", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=0yectjqXWcCHTMhyv3pd4HqlpH_2FjslmIWhw59V07Em7ASx_2FsHGk5ZFvSKThLEb3f", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731253651", "total_time": 31, "date_modified": "2021-06-10T10:23:12+00:00", "date_created": "2021-06-10T10:22:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731253651", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770273} +{"stream": "survey_responses", "data": {"id": "12731272195", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=owJREFrbM6_2Fu6fqPiWbW3rJ15U9q9DLJ_2BlAZMq6Gi_2BGKjQ2AGiWVNE5RBM_2FKLyMs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731272195", "total_time": 31, "date_modified": "2021-06-10T10:30:52+00:00", "date_created": "2021-06-10T10:30:20+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731272195", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174495"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770273} +{"stream": "survey_responses", "data": {"id": "12731292534", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zAYnDyr9Dxd9Em4YpxVpKH7_2BtJbsMFiI0bH89dH2LhhokCrBrNjKYFI0nacFEIpq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731292534", "total_time": 32, "date_modified": "2021-06-10T10:38:57+00:00", "date_created": "2021-06-10T10:38:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731292534", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174254"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174494"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770274} +{"stream": "survey_responses", "data": {"id": "12731312208", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=uSGoF1sumsyf_2B3iBhadh_2F73k_2BM0dxEwFfAlWl70AwraHwQHcfj64cL0FUt9StbRx", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731312208", "total_time": 34, "date_modified": "2021-06-10T10:46:52+00:00", "date_created": "2021-06-10T10:46:18+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731312208", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174486"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174495"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174643"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1690184770274} +{"stream": "survey_responses", "data": {"id": "12731331308", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vQGF4JMphBOwv8kJxOmtjdRG81lTGr7CW73feMGxDEMLSwybystpUFV76KKxYQR6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731331308", "total_time": 31, "date_modified": "2021-06-10T10:54:29+00:00", "date_created": "2021-06-10T10:53:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731331308", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174486"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174620"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174644"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174676"}]}]}]}, "emitted_at": 1690184770274} +{"stream": "survey_responses", "data": {"id": "12731350053", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=uQGI1ouCt3TwYg_2F0OjcapOOokxhfAvoP3vsYcJntZXz4yNaoZRXNQePI1o27JEfS", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731350053", "total_time": 32, "date_modified": "2021-06-10T11:02:07+00:00", "date_created": "2021-06-10T11:01:34+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731350053", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174496"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1690184770275} +{"stream": "survey_responses", "data": {"id": "12730895819", "recipient_id": "", "collection_mode": "default", "response_status": "partial", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": ["168830049", "168830050", "168830060"], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=2awnrk_2BH5OxFP_2BQgeiHHjR9yR3bHHngtP06WJ08L7vUw_2Bi4HP_2BoIM_2Fzdi4XZBg4c", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12730895819", "total_time": 3851, "date_modified": "2021-06-10T08:30:24+00:00", "date_created": "2021-06-10T07:26:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12730895819", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455130", "answers": [{"choice_id": "4385137345"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137473"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137488"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137493"}]}]}, {"id": "168830060", "questions": []}]}, "emitted_at": 1690184771496} +{"stream": "survey_responses", "data": {"id": "12731026318", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=9zPtrl88F3NYIsYeyGVBOAH8xcbXCyfUnb5wJmafeNnqKu0OuTBMd3AMA70CA0Dd", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731026318", "total_time": 42, "date_modified": "2021-06-10T08:38:55+00:00", "date_created": "2021-06-10T08:38:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731026318", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137324"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137342"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137468"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137486"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137702"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137743"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137776"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137797"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137810"}]}]}]}, "emitted_at": 1690184771496} +{"stream": "survey_responses", "data": {"id": "12731034119", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=_2F1iWQXdsu69adcyLuCe0CWLGDC2dm6Mi6UQBAgHHW5tqLGpm6S8VgnyogtlqQcF4", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731034119", "total_time": 38, "date_modified": "2021-06-10T08:43:18+00:00", "date_created": "2021-06-10T08:42:40+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731034119", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137328"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137342"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137470"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137488"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137742"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137782"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137793"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137813"}]}]}]}, "emitted_at": 1690184771496} +{"stream": "survey_responses", "data": {"id": "12731048348", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=teZL209m07VmwPLhipUVAeikuCFifwYD7Xbn856h7nliTsJok_2BJ8cUsANlBG1x09", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731048348", "total_time": 37, "date_modified": "2021-06-10T08:51:05+00:00", "date_created": "2021-06-10T08:50:27+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731048348", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137324"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137345"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137472"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137488"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137741"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137780"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137793"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137812"}]}]}]}, "emitted_at": 1690184771497} +{"stream": "survey_responses", "data": {"id": "12731062826", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=iBwqqk7dTblJHtZ5UWdoNXNfZvamq_2FWqVX9lPmgcaEk_2FJSEZbpEjvTDEp1kntA4I", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731062826", "total_time": 37, "date_modified": "2021-06-10T08:58:47+00:00", "date_created": "2021-06-10T08:58:09+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731062826", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137325"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137341"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137473"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137485"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137741"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137782"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137794"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137813"}]}]}]}, "emitted_at": 1690184771497} +{"stream": "survey_responses", "data": {"id": "12731078267", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=EWvr594gc6fTNOWCnzJcr1awvjtYrfcHOS1dCZlEmJOVdekL3kH9fN015tpi7AVW", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731078267", "total_time": 37, "date_modified": "2021-06-10T09:06:31+00:00", "date_created": "2021-06-10T09:05:53+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731078267", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137329"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137342"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137468"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137495"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137741"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137775"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137796"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771497} +{"stream": "survey_responses", "data": {"id": "12731094627", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=UEDBFTIm3XGLxP7HF9d4l806UiymFhRau6MyuUYkfb1y7zOR96xOkRxERkrOJ9ca", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731094627", "total_time": 38, "date_modified": "2021-06-10T09:14:16+00:00", "date_created": "2021-06-10T09:13:38+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731094627", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137329"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137345"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137470"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137486"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137495"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137743"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137780"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137792"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137810"}]}]}]}, "emitted_at": 1690184771498} +{"stream": "survey_responses", "data": {"id": "12731111090", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=X3JQr1B6y2spk9th1IPF0YF8VQxsZUEffTzik0RdvFtFl2SnrnJQRca_2BDoV2r4mq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731111090", "total_time": 38, "date_modified": "2021-06-10T09:21:54+00:00", "date_created": "2021-06-10T09:21:16+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731111090", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137326"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137341"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137474"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137493"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137741"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137775"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137793"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771498} +{"stream": "survey_responses", "data": {"id": "12731127763", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ETzE9VL934Fk2aBcNUUcKaSh2V3EZHIzNm7GmAoHdznlQgZLuN90AHX13K3n6CVD", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731127763", "total_time": 37, "date_modified": "2021-06-10T09:29:36+00:00", "date_created": "2021-06-10T09:28:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731127763", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137325"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137341"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137472"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137488"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137496"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137702"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137740"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137775"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137797"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137813"}]}]}]}, "emitted_at": 1690184771498} +{"stream": "survey_responses", "data": {"id": "12731145509", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VdEi6nthWHwzn1S6g9Ejj36ATaNJTgGRTvPg9OSgXQX3v7zuJ7b8kUW4JpPgoH8N", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731145509", "total_time": 37, "date_modified": "2021-06-10T09:37:18+00:00", "date_created": "2021-06-10T09:36:40+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731145509", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137324"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137344"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137473"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137486"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137492"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137743"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137782"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137795"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137810"}]}]}]}, "emitted_at": 1690184771499} +{"stream": "survey_responses", "data": {"id": "12731162853", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=SpVNwOtvMvqHvq3zHMXNhDHSmc7HA4m3XTWJpvcSJxvt1b1rvgGaOej3oC_2BolYPy", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731162853", "total_time": 38, "date_modified": "2021-06-10T09:45:01+00:00", "date_created": "2021-06-10T09:44:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731162853", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137328"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137340"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137469"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137492"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137743"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137777"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137795"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771499} +{"stream": "survey_responses", "data": {"id": "12731180052", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=JiwKhmlzAA4BE2G7wLdiXM8p7qGKqy7FVolioM_2BFHLNo4MIvbtO7U2T2MtIhjHQT", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731180052", "total_time": 40, "date_modified": "2021-06-10T09:52:49+00:00", "date_created": "2021-06-10T09:52:09+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731180052", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137324"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137342"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137468"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137702"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137740"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137775"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137794"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137812"}]}]}]}, "emitted_at": 1690184771499} +{"stream": "survey_responses", "data": {"id": "12731198914", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=4lczf0MNm2lOpWvmpCXz8qaQFfYxrkqDyo3Na6JqZ3SoyW9vfVtRkWihGPcm3TJ9", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731198914", "total_time": 37, "date_modified": "2021-06-10T10:01:22+00:00", "date_created": "2021-06-10T10:00:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731198914", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137329"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137345"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137472"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137485"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137741"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137776"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137795"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771500} +{"stream": "survey_responses", "data": {"id": "12731219287", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=BuKPZoR_2B7yhJkjbyRrpCv9Xsryak851Sl2u8K17_2FsYnSAZUFHdu4fJB0umnx5Y9_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731219287", "total_time": 37, "date_modified": "2021-06-10T10:09:13+00:00", "date_created": "2021-06-10T10:08:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731219287", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137327"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137341"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137470"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137488"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137494"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137740"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137776"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137792"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771500} +{"stream": "survey_responses", "data": {"id": "12731238577", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=gYK41YcmDya_2Bvsx_2BqjLe992H7r6ORc6kyUL3YhAqx78syU1tTY9dv_2Fn92J5xM033", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731238577", "total_time": 37, "date_modified": "2021-06-10T10:16:59+00:00", "date_created": "2021-06-10T10:16:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731238577", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137327"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137340"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137472"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137485"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137496"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137702"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137741"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137778"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137793"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137810"}]}]}]}, "emitted_at": 1690184771500} +{"stream": "survey_responses", "data": {"id": "12731257342", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=gXcd4_2B0crZ3JHyflBVCFpzaABCmJO9PKQzLnjMuvOHPsOS0sIISJZjVscMHtusHw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731257342", "total_time": 37, "date_modified": "2021-06-10T10:24:47+00:00", "date_created": "2021-06-10T10:24:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731257342", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137324"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137340"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137471"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137486"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137495"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137740"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137782"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137793"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137813"}]}]}]}, "emitted_at": 1690184771501} +{"stream": "survey_responses", "data": {"id": "12731276322", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=fdG3OylITjbhJlFfCAIGHmTQ6YpdyOcILnp8DliR5bm9qPSRMG6AkkWm3Gu2gJoP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731276322", "total_time": 41, "date_modified": "2021-06-10T10:32:36+00:00", "date_created": "2021-06-10T10:31:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731276322", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137328"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137340"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137468"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137495"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137702"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137742"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137776"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137793"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137812"}]}]}]}, "emitted_at": 1690184771501} +{"stream": "survey_responses", "data": {"id": "12731296299", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=PijLaybIFW4zp3WXn6Qn_2Bes5snwJCUJdhdklhMDUCFNxV8kYJTspV5CKmi9JxcxR", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731296299", "total_time": 28, "date_modified": "2021-06-10T10:40:25+00:00", "date_created": "2021-06-10T10:39:56+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731296299", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": []}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137742"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137778"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137795"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137812"}]}]}]}, "emitted_at": 1690184771501} +{"stream": "survey_responses", "data": {"id": "12731316167", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=DdWwrUlBvSWJMjuKWSDCTJGmzJlZPpJ9HEf_2F_2BW1pzPCQ9WlJ7Su0kQcly_2B_2F6O8il", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731316167", "total_time": 37, "date_modified": "2021-06-10T10:48:31+00:00", "date_created": "2021-06-10T10:47:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731316167", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137327"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137342"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137469"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137492"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137700"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137740"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137776"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137797"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771501} +{"stream": "survey_responses", "data": {"id": "12731335082", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=JT5nsI3dl0SB9aaAvObos67DXY6qy_2FDXI1TTpgeEv1m3sqvcxy3MhVQQlApbsDQN", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731335082", "total_time": 37, "date_modified": "2021-06-10T10:56:07+00:00", "date_created": "2021-06-10T10:55:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731335082", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137325"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137340"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137473"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137487"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137493"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137742"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137782"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137794"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137810"}]}]}]}, "emitted_at": 1690184771502} +{"stream": "survey_responses", "data": {"id": "12731354013", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829319", "survey_id": "307784834", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VD2H3n1aqESl1LrF5U_2B7SlCYCH1VXiN11NQ44aSykUk8VdT8zluzTaiFP1WRilVh", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D?respondent_id=12731354013", "total_time": 37, "date_modified": "2021-06-10T11:03:44+00:00", "date_created": "2021-06-10T11:03:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784834/responses/12731354013", "pages": [{"id": "168830049", "questions": []}, {"id": "168830050", "questions": [{"id": "667455128", "answers": [{"choice_id": "4385137325"}]}, {"id": "667455130", "answers": [{"choice_id": "4385137344"}]}, {"id": "667455161", "answers": [{"choice_id": "4385137473"}]}, {"id": "667455172", "answers": [{"choice_id": "4385137488"}]}, {"id": "667455179", "answers": [{"choice_id": "4385137493"}]}]}, {"id": "168830060", "questions": [{"id": "667455202", "answers": [{"choice_id": "4385137701"}]}, {"id": "667455205", "answers": [{"choice_id": "4385137740"}]}, {"id": "667455210", "answers": [{"choice_id": "4385137781"}]}, {"id": "667455212", "answers": [{"choice_id": "4385137792"}]}, {"id": "667455215", "answers": [{"choice_id": "4385137811"}]}]}]}, "emitted_at": 1690184771502} +{"stream": "survey_responses", "data": {"id": "12731046667", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3m5bJqYOPzi9hTHWeQaMVChOOH_2FutIpC1GCzcEhDLsPuEi5IsDvCVew50TpjMJ7_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731046667", "total_time": 36, "date_modified": "2021-06-10T08:50:07+00:00", "date_created": "2021-06-10T08:49:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731046667", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177067"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177098"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177138"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177155"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177202"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177213"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177216"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177393"}]}]}]}, "emitted_at": 1690184772385} +{"stream": "survey_responses", "data": {"id": "12731061053", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=RliJfymKsHA5ERiuKeRola_2B_2F2kldmGpI58KjGRQTMy_2F7NOeFnJ8VaeyjrQYqefuv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731061053", "total_time": 36, "date_modified": "2021-06-10T08:57:50+00:00", "date_created": "2021-06-10T08:57:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731061053", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177068"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177098"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177139"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177158"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177171"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177202"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177217"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177395"}]}]}]}, "emitted_at": 1690184772386} +{"stream": "survey_responses", "data": {"id": "12731076167", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Fbm6tEId2O2bNT8H0BJ8ZuinueLnpLoMwgotsY_2BRD_2FzbvCnhEaxZHye5e5bkXF_2B0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731076167", "total_time": 36, "date_modified": "2021-06-10T09:05:33+00:00", "date_created": "2021-06-10T09:04:57+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731076167", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177067"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177097"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177142"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177153"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177204"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177210"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177220"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177392"}]}]}]}, "emitted_at": 1690184772386} +{"stream": "survey_responses", "data": {"id": "12731092606", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FSRkUyOqh9XslZbpVGw5556THT2fSckee67DmK8vg8QM5E_2BIPpatzKJNVF6E4pI2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731092606", "total_time": 36, "date_modified": "2021-06-10T09:13:19+00:00", "date_created": "2021-06-10T09:12:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731092606", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177066"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177098"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177138"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177153"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177173"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177202"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177213"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177215"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177399"}]}]}]}, "emitted_at": 1690184772386} +{"stream": "survey_responses", "data": {"id": "12731109051", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hFBXkisB3CTNYQ4UTLe_2FgTWjKwcR0l_2Fk9xXG2L2eiiv2m1A4IK6anbX8e98lxI_2BJ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731109051", "total_time": 36, "date_modified": "2021-06-10T09:20:57+00:00", "date_created": "2021-06-10T09:20:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731109051", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177068"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177098"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177139"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177156"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177171"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177204"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177220"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177383"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177398"}]}]}]}, "emitted_at": 1690184772387} +{"stream": "survey_responses", "data": {"id": "12731125773", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=IitR8qwcQJhjvJZGpnRnQ51uSFMITWI_2Fj71B7P5SgZeC4E3yXEi1_2B_2FMm5wo_2BDV8h", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731125773", "total_time": 37, "date_modified": "2021-06-10T09:28:41+00:00", "date_created": "2021-06-10T09:28:04+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731125773", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177066"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177099"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177142"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177158"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177171"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177203"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177220"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177392"}]}]}]}, "emitted_at": 1690184772387} +{"stream": "survey_responses", "data": {"id": "12731143496", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=afwrrGApCHDKe6n5FTBrtOaHsGGtP_2BhesVrwRj_2FP25EjYCQxzsAC6aAkUchVScSv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731143496", "total_time": 36, "date_modified": "2021-06-10T09:36:22+00:00", "date_created": "2021-06-10T09:35:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731143496", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177066"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177097"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177138"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177154"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177174"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177201"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177213"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177216"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177392"}]}]}]}, "emitted_at": 1690184772387} +{"stream": "survey_responses", "data": {"id": "12731160592", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=gi5j6iOFt63bX7t4yH9tggXGZlHkMPjNnRuEfb_2BfIFrcSwpamxUO_2B9Y0WXX9oSHD", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731160592", "total_time": 35, "date_modified": "2021-06-10T09:44:01+00:00", "date_created": "2021-06-10T09:43:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731160592", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177065"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177098"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177142"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177156"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177174"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177203"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177210"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177219"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177398"}]}]}]}, "emitted_at": 1690184772388} +{"stream": "survey_responses", "data": {"id": "12731177842", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Mdft9tg4e78_2Fa7lJAhODrqVOjXTcfF0kM1rlL9FH_2B4OFTc8svhyUTN194QTJecZV", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731177842", "total_time": 38, "date_modified": "2021-06-10T09:51:48+00:00", "date_created": "2021-06-10T09:51:09+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731177842", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177068"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177097"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177141"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177155"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177174"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177204"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177220"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177398"}]}]}]}, "emitted_at": 1690184772388} +{"stream": "survey_responses", "data": {"id": "12731196644", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=j0mjP6Z7LrcWGOboY6jmQkq3eLodG5VEJ1xjmK_2FVYi4g_2BLXzw2TXVIZDXlMQfSce", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731196644", "total_time": 38, "date_modified": "2021-06-10T10:00:23+00:00", "date_created": "2021-06-10T09:59:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731196644", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177067"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177096"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177141"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177153"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177174"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177202"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177218"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177393"}]}]}]}, "emitted_at": 1690184772388} +{"stream": "survey_responses", "data": {"id": "12731216845", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=42BF5q8j9sIJ7FZW3oHucGLBgLbhnDLuy54e3nC1dr_2BV65av2cB1XpRMxXE8qLrk", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731216845", "total_time": 35, "date_modified": "2021-06-10T10:08:14+00:00", "date_created": "2021-06-10T10:07:38+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731216845", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177066"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177099"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177139"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177158"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177202"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177215"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177383"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177392"}]}]}]}, "emitted_at": 1690184772389} +{"stream": "survey_responses", "data": {"id": "12731236327", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=NqwagvKjeHPRtNo1Xjgzhlf20oC0V9pmZhKn9M_2FI7d0WQpLl5SvMJmpPx_2FMR8WOA", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731236327", "total_time": 36, "date_modified": "2021-06-10T10:16:03+00:00", "date_created": "2021-06-10T10:15:27+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731236327", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177066"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177099"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177138"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177153"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177171"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177204"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177212"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177214"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177394"}]}]}]}, "emitted_at": 1690184772389} +{"stream": "survey_responses", "data": {"id": "12731255115", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=68xsdhMhxZnM6kEBzq6YOW2Pcy65CdGFnjt47M9pu1JOngmURrmW_2BioZOUdc5iwt", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731255115", "total_time": 35, "date_modified": "2021-06-10T10:23:53+00:00", "date_created": "2021-06-10T10:23:17+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731255115", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177067"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177095"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177141"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177158"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177202"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177211"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177218"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177395"}]}]}]}, "emitted_at": 1690184772389} +{"stream": "survey_responses", "data": {"id": "12731273731", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=5V96fOaPSetDK7aFZp_2BQMlsfP5WVMuaklhLOVJzxjkLao_2BG1CUfWUAOvXCoIqzu7", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731273731", "total_time": 35, "date_modified": "2021-06-10T10:31:32+00:00", "date_created": "2021-06-10T10:30:56+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731273731", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177065"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177096"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177140"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177155"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177171"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177203"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177210"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177215"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177398"}]}]}]}, "emitted_at": 1690184772389} +{"stream": "survey_responses", "data": {"id": "12731294056", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=yqW6gwVkcj7CYuN1hQxBNE2_2BGnnSO_2FMoG_2FJOr_2Bl9LOFwGfvlISTxAL6DegDXn5gw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731294056", "total_time": 36, "date_modified": "2021-06-10T10:39:38+00:00", "date_created": "2021-06-10T10:39:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731294056", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177067"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177096"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177142"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177154"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177204"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177213"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177214"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177383"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177393"}]}]}]}, "emitted_at": 1690184772390} +{"stream": "survey_responses", "data": {"id": "12731313837", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=D85HuJHtbiPW1vl_2FIvU2ocvUVCdJggW_2F0WR5WpMSLCGZaGIg8UdLj54drv_2FcmRMz", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731313837", "total_time": 37, "date_modified": "2021-06-10T10:47:34+00:00", "date_created": "2021-06-10T10:46:57+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731313837", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177067"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177097"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177139"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177156"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177200"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177212"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177220"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177395"}]}]}]}, "emitted_at": 1690184772390} +{"stream": "survey_responses", "data": {"id": "12731332767", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hztkSX5ztC9RUhcoXXjOEZsT0_2BjH5f6tNA9Fh0afPXTD4285Qb8YReTOBLbelKig", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731332767", "total_time": 35, "date_modified": "2021-06-10T10:55:09+00:00", "date_created": "2021-06-10T10:54:33+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731332767", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177068"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177096"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177139"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177158"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177204"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177210"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177217"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177382"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177398"}]}]}]}, "emitted_at": 1690184772390} +{"stream": "survey_responses", "data": {"id": "12731351604", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829931", "survey_id": "307785448", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=4s6X1KZyt2TAwDeH6zLlLbhI1hJkumblOrBysCkO3gPTDp7tdY9BOD_2FvWkRVgEAz", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D?respondent_id=12731351604", "total_time": 36, "date_modified": "2021-06-10T11:02:48+00:00", "date_created": "2021-06-10T11:02:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785448/responses/12731351604", "pages": [{"id": "168831470", "questions": []}, {"id": "168831471", "questions": [{"id": "667462113", "answers": [{"choice_id": "4385177068"}]}, {"id": "667462114", "answers": [{"choice_id": "4385177098"}]}, {"id": "667462122", "answers": [{"choice_id": "4385177138"}]}, {"id": "667462126", "answers": [{"choice_id": "4385177154"}]}, {"id": "667462130", "answers": [{"choice_id": "4385177172"}]}]}, {"id": "168831478", "questions": [{"id": "667462135", "answers": [{"choice_id": "4385177200"}]}, {"id": "667462136", "answers": [{"choice_id": "4385177213"}]}, {"id": "667462138", "answers": [{"choice_id": "4385177218"}]}, {"id": "667462170", "answers": [{"choice_id": "4385177381"}]}, {"id": "667462172", "answers": [{"choice_id": "4385177397"}]}]}]}, "emitted_at": 1690184772391} +{"stream": "survey_responses", "data": {"id": "12731027651", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=fTFc6V6Lifr7HoJ_2Bl_2Bc74bpg_2B9JRwagJN_2FN3k2RwdJ8CrOldvM08uLZMXEGk5uR_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731027651", "total_time": 29, "date_modified": "2021-06-10T08:39:30+00:00", "date_created": "2021-06-10T08:39:01+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731027651", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138588"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138599"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138621"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138707"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139016"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139062"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139232"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139246"}]}]}]}, "emitted_at": 1690184773257} +{"stream": "survey_responses", "data": {"id": "12731035521", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=P3yPukgvQbdQ0TVJQrLvSnRv3NllOsosNLSuGp34tGmMRN3SsXsVqXxRkHoUAZzR", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731035521", "total_time": 27, "date_modified": "2021-06-10T08:43:50+00:00", "date_created": "2021-06-10T08:43:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731035521", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138593"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138600"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138623"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138700"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139018"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139231"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773257} +{"stream": "survey_responses", "data": {"id": "12731049611", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=1ZxbynfKwvppvatsw2rpbv64I6YRz_2BRUyTcv8dLIbh63fwCm_2Fl9TFtbLXUoePm1q", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731049611", "total_time": 27, "date_modified": "2021-06-10T08:51:37+00:00", "date_created": "2021-06-10T08:51:09+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731049611", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138588"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138598"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138623"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138701"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138782"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139017"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139232"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139245"}]}]}]}, "emitted_at": 1690184773258} +{"stream": "survey_responses", "data": {"id": "12731064110", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Vxei42yDMEVFSnyt9M3fpCOreC5nzmSKyEY9iwPt1QtQFzPyFiaGMuw2XQKlF4Tu", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731064110", "total_time": 27, "date_modified": "2021-06-10T08:59:20+00:00", "date_created": "2021-06-10T08:58:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731064110", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138591"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138600"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138620"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138706"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138782"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139013"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139112"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139231"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139246"}]}]}]}, "emitted_at": 1690184773258} +{"stream": "survey_responses", "data": {"id": "12731079723", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=pucu83BOi19Z2n_2Bq7Cmuq6GrV4auydnhRuXlDsCZ5DwqqAhRkuX0GL8i_2FXCRgVnb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731079723", "total_time": 27, "date_modified": "2021-06-10T09:07:03+00:00", "date_created": "2021-06-10T09:06:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731079723", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138589"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138599"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138622"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138707"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139016"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139063"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139111"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139235"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773258} +{"stream": "survey_responses", "data": {"id": "12731096120", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=cmwD6t6spTjvPh8zt8wpzOowMPktBxJBhl8ROXhWK7NsNRfaf9VJ51wtPZfMsbPJ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731096120", "total_time": 27, "date_modified": "2021-06-10T09:14:48+00:00", "date_created": "2021-06-10T09:14:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731096120", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138593"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138597"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138623"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138706"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139016"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139063"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139235"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139245"}]}]}]}, "emitted_at": 1690184773259} +{"stream": "survey_responses", "data": {"id": "12731112612", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tYfRsLpnvLXiDU1aSXTH5nIXnauCRjmyXNWWEYT8RsVkytyEn7W9JT8TX_2FQ_2FZXmm", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731112612", "total_time": 27, "date_modified": "2021-06-10T09:22:26+00:00", "date_created": "2021-06-10T09:21:59+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731112612", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138593"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138595"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138625"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138700"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139014"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139063"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139110"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139230"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139245"}]}]}]}, "emitted_at": 1690184773259} +{"stream": "survey_responses", "data": {"id": "12731129209", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Smvc_2Fhq34btK33i0b1sGxYeWInTkfyhJ_2BYMIBz_2B0I9Z2ZIyvlPNu3h_2B65f84DuA0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731129209", "total_time": 27, "date_modified": "2021-06-10T09:30:08+00:00", "date_created": "2021-06-10T09:29:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731129209", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138588"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138597"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138622"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138704"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139015"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139111"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139234"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773259} +{"stream": "survey_responses", "data": {"id": "12731147035", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=U9JFMHqI683O1sqxW8mLF_2FgU_2F7fYGjSSm6LScE6pu2LDPRKAAVQXvZThLKDp2GXJ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731147035", "total_time": 27, "date_modified": "2021-06-10T09:37:50+00:00", "date_created": "2021-06-10T09:37:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731147035", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138589"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138595"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138625"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138703"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138781"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139017"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139110"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139236"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139245"}]}]}]}, "emitted_at": 1690184773260} +{"stream": "survey_responses", "data": {"id": "12731164403", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=0jJiVT47Abo_2BkHRNm2_2FqakaCDR5LNaVs9r5z_2BNCgUb1UPEiUm95gFyEF3xtdkYv3", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731164403", "total_time": 27, "date_modified": "2021-06-10T09:45:34+00:00", "date_created": "2021-06-10T09:45:06+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731164403", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138593"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138600"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138626"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138701"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139018"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139235"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139245"}]}]}]}, "emitted_at": 1690184773260} +{"stream": "survey_responses", "data": {"id": "12731181688", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=UQZSADafI1w57hYP_2FS2NflQKw4HeCD0bqS8NFrI480xqhT_2BmBKvsbhwZESEP6bWH", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731181688", "total_time": 33, "date_modified": "2021-06-10T09:53:29+00:00", "date_created": "2021-06-10T09:52:55+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731181688", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138592"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138598"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138622"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138707"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139013"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139062"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139112"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139230"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139246"}]}]}]}, "emitted_at": 1690184773261} +{"stream": "survey_responses", "data": {"id": "12731200697", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Z_2FtAEFehOtxFbKh3vENz8bUgBufRs3dp1FPH_2BaixcDbpWdxRX3s0ydAEfVu_2F_2FV9k", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731200697", "total_time": 27, "date_modified": "2021-06-10T10:01:55+00:00", "date_created": "2021-06-10T10:01:27+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731200697", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138589"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138598"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138622"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138706"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139013"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139063"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139112"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139233"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139246"}]}]}]}, "emitted_at": 1690184773261} +{"stream": "survey_responses", "data": {"id": "12731221002", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YYEQ9YPblwXypUXP9UsD2QOQI_2B52u8EQSph1_2FmToJDM9mtFZbqeiOLVv8CbSN_2BdL", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731221002", "total_time": 27, "date_modified": "2021-06-10T10:09:45+00:00", "date_created": "2021-06-10T10:09:18+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731221002", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138592"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138600"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138620"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138702"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139017"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139062"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139233"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139246"}]}]}]}, "emitted_at": 1690184773261} +{"stream": "survey_responses", "data": {"id": "12731240798", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=4ZdZmIl45awCiUkNXcRb9vC0pYIZMfHtCv3apho2x9McRs1lDMv5yNnqz4z3VnSP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731240798", "total_time": 27, "date_modified": "2021-06-10T10:17:43+00:00", "date_created": "2021-06-10T10:17:16+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731240798", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138590"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138600"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138620"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138703"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139018"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139062"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139230"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773262} +{"stream": "survey_responses", "data": {"id": "12731258961", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=emSG54_2BrHmGYvPyc21uSnPZeh8nU2iPahv_2B21isNydKP_2Bl4t3C_2B821nvNucSSxD_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731258961", "total_time": 27, "date_modified": "2021-06-10T10:25:19+00:00", "date_created": "2021-06-10T10:24:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731258961", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138588"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138596"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138624"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138701"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139016"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139062"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139235"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139245"}]}]}]}, "emitted_at": 1690184773262} +{"stream": "survey_responses", "data": {"id": "12731278204", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=IqIG55XKRRDg9SlyFaGnS3M3bEcCu3Kc0adBsNxgJoXgsm_2FSsAZO5dQiHmAg137N", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731278204", "total_time": 29, "date_modified": "2021-06-10T10:33:09+00:00", "date_created": "2021-06-10T10:32:40+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731278204", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138589"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138596"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138621"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138705"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139016"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139063"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139113"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139232"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773263} +{"stream": "survey_responses", "data": {"id": "12731297693", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=4YsTEO_2Ft1AqmjnchtBZcCuBm0IyfVnOc8GnFYBmeTI3DsacLLtX6HINSLOXrSTvD", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731297693", "total_time": 27, "date_modified": "2021-06-10T10:40:58+00:00", "date_created": "2021-06-10T10:40:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731297693", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138588"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138598"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138626"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138704"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138780"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139015"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139063"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139111"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139230"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773263} +{"stream": "survey_responses", "data": {"id": "12731317951", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=0MbtsVOF4vEh5Xcf2H_2B4YhWcgeBtjnt_2BlJxPWFG88HIYuNIOdDcQkUMEv2VCZHUT", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731317951", "total_time": 27, "date_modified": "2021-06-10T10:49:06+00:00", "date_created": "2021-06-10T10:48:39+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731317951", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138589"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138596"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138624"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138702"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138779"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139018"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139109"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139236"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773263} +{"stream": "survey_responses", "data": {"id": "12731336744", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=jRZyZbcPITg018MLWZvCHAcG0IsxU_2BF2d4PQO0uZu8xhWEPv58hIqjKkEBTk1g1x", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731336744", "total_time": 27, "date_modified": "2021-06-10T10:56:40+00:00", "date_created": "2021-06-10T10:56:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731336744", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138591"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138596"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138623"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138702"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138781"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139016"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139113"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139234"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139246"}]}]}]}, "emitted_at": 1690184773264} +{"stream": "survey_responses", "data": {"id": "12731356183", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843442", "survey_id": "307784863", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=j2Vv3hWWCKKX8TGm3kD6g4UQCdPiw0tyfJlXlm3Yr4aHlHW_2FZ_2FCSgfXWR8mLWGn0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D?respondent_id=12731356183", "total_time": 28, "date_modified": "2021-06-10T11:04:28+00:00", "date_created": "2021-06-10T11:04:00+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784863/responses/12731356183", "pages": [{"id": "168830093", "questions": []}, {"id": "168830094", "questions": [{"id": "667455348", "answers": [{"choice_id": "4385138593"}]}, {"id": "667455351", "answers": [{"choice_id": "4385138600"}]}, {"id": "667455358", "answers": [{"choice_id": "4385138621"}]}, {"id": "667455370", "answers": [{"choice_id": "4385138702"}]}, {"id": "667455395", "answers": [{"choice_id": "4385138782"}]}]}, {"id": "168830108", "questions": [{"id": "667455427", "answers": [{"choice_id": "4385139018"}]}, {"id": "667455439", "answers": [{"choice_id": "4385139064"}]}, {"id": "667455443", "answers": [{"choice_id": "4385139110"}]}, {"id": "667455463", "answers": [{"choice_id": "4385139233"}]}, {"id": "667455466", "answers": [{"choice_id": "4385139244"}]}]}]}, "emitted_at": 1690184773264} +{"stream": "survey_responses", "data": {"id": "12731028687", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=dUjh6A_2FaiaLmSiByVq0EnrrvhqLUItqI0TokJUP8jl9SbUi5cd6hNui1hNqGv56Z", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731028687", "total_time": 33, "date_modified": "2021-06-10T08:40:08+00:00", "date_created": "2021-06-10T08:39:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731028687", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137907"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137947"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138061"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138109"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138133"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138157"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138210"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138240"}]}]}]}, "emitted_at": 1690184774468} +{"stream": "survey_responses", "data": {"id": "12731036487", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=5ir6yV1yx1xcxnDqd5CbKT7hxkDCbqeG_2BHctWsXeZb015fsixx9pWXx0NgTuAnPh", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731036487", "total_time": 31, "date_modified": "2021-06-10T08:44:28+00:00", "date_created": "2021-06-10T08:43:56+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731036487", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137911"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137935"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137946"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138064"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138110"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138126"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138210"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138243"}]}]}]}, "emitted_at": 1690184774468} +{"stream": "survey_responses", "data": {"id": "12731050647", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=5b_2BmMZNzhREunXwy6FMGNiacjaGpQeXb3F9yW3dcaVweZLGQY8t0kgV3XBTUneNG", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731050647", "total_time": 31, "date_modified": "2021-06-10T08:52:13+00:00", "date_created": "2021-06-10T08:51:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731050647", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137907"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137933"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137949"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138063"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138108"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138126"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138216"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138243"}]}]}]}, "emitted_at": 1690184774469} +{"stream": "survey_responses", "data": {"id": "12731065101", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ivYcxChrgUjC39ad0me9CZw6BmVAkTQjhWy6e7bR1QV_2BZ7rOJIplHeJnM7IdV0ev", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731065101", "total_time": 31, "date_modified": "2021-06-10T08:59:55+00:00", "date_created": "2021-06-10T08:59:24+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731065101", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137908"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137925"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137933"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137949"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138061"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138113"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138131"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138155"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138216"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138243"}]}]}]}, "emitted_at": 1690184774469} +{"stream": "survey_responses", "data": {"id": "12731080861", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3uxM3bC1ju6y0ZbYbgP_2BvgFNEF0MBf07Z1sR5msqxiFnqdE8BnFPnrip62Dyzq_2FJ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731080861", "total_time": 31, "date_modified": "2021-06-10T09:07:39+00:00", "date_created": "2021-06-10T09:07:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731080861", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137906"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137924"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137931"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137944"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138061"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138111"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138129"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138209"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138243"}]}]}]}, "emitted_at": 1690184774469} +{"stream": "survey_responses", "data": {"id": "12731097254", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zCaGCrPMhDj216XESYkQCRoFWd5JT94Z_2F_2Bns224Ht4yvHiBMzA0R869PAJ_2FrE_2F_2BS", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731097254", "total_time": 31, "date_modified": "2021-06-10T09:15:24+00:00", "date_created": "2021-06-10T09:14:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731097254", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137904"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137936"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137950"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138110"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138131"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138155"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138216"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138239"}]}]}]}, "emitted_at": 1690184774469} +{"stream": "survey_responses", "data": {"id": "12731113718", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VoxvgtLVPESmmstl7tMZG_2BuMDM0Aa9E5fCEa86a_2BuXZa6pilvyY520S9G1tWgrRY", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731113718", "total_time": 31, "date_modified": "2021-06-10T09:23:02+00:00", "date_created": "2021-06-10T09:22:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731113718", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137907"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137925"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137935"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137949"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138063"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138110"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138130"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138210"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138241"}]}]}]}, "emitted_at": 1690184774470} +{"stream": "survey_responses", "data": {"id": "12731130369", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=z7UD89cfARngMPT8rraMXX6AgkCQ7ffFPZ0s_2BXsx7CQathuufUT_2FM9g_2B2Y8fjZzA", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731130369", "total_time": 31, "date_modified": "2021-06-10T09:30:44+00:00", "date_created": "2021-06-10T09:30:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731130369", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137905"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137927"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137934"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137947"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138064"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138111"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138130"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138155"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138213"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138239"}]}]}]}, "emitted_at": 1690184774470} +{"stream": "survey_responses", "data": {"id": "12731148248", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ju0hkPiuicpj80ml6Hl_2F22nPizQlWQLGiYwxc_2FDGxTXzgomNhfw66UPrCE0D636O", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731148248", "total_time": 33, "date_modified": "2021-06-10T09:38:28+00:00", "date_created": "2021-06-10T09:37:55+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731148248", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137907"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137927"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137936"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137949"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138112"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138130"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138157"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138214"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138242"}]}]}]}, "emitted_at": 1690184774470} +{"stream": "survey_responses", "data": {"id": "12731165533", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=CJNhekEYhzbzSqz1R0VHxVjTmJ866MvLYw4aVPGTNOnsy33BOg1S1_2F_2Faesq41w0h", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731165533", "total_time": 32, "date_modified": "2021-06-10T09:46:10+00:00", "date_created": "2021-06-10T09:45:38+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731165533", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137909"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137927"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137945"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138110"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138128"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138213"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138240"}]}]}]}, "emitted_at": 1690184774471} +{"stream": "survey_responses", "data": {"id": "12731183192", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=bEWAYVF9toetHEI2_2BOwwwWoYM7U4BuBikNpbjp7LRt5pr8geoSPYOEqTu3ySeLJ0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731183192", "total_time": 35, "date_modified": "2021-06-10T09:54:12+00:00", "date_created": "2021-06-10T09:53:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731183192", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137911"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137924"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137935"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137949"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138063"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138111"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138127"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138216"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138241"}]}]}]}, "emitted_at": 1690184774471} +{"stream": "survey_responses", "data": {"id": "12731202203", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=cVfxg1YUnkHWG5xh95O_2FZH6JrXJKyH7dK1jz4klmqpr6kkJW4Nyg6pAF9RU6K8Yo", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731202203", "total_time": 33, "date_modified": "2021-06-10T10:02:33+00:00", "date_created": "2021-06-10T10:02:00+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731202203", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137908"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137925"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137947"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138064"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138109"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138133"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138214"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138243"}]}]}]}, "emitted_at": 1690184774471} +{"stream": "survey_responses", "data": {"id": "12731222304", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YIMZ6ZxwVSC8E_2FINoIUPvqiJVxX2_2B7h72mIiWrB82xJdDZqY_2BGZdCN_2FCfsiwQr_2Bv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731222304", "total_time": 31, "date_modified": "2021-06-10T10:10:21+00:00", "date_created": "2021-06-10T10:09:50+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731222304", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137909"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137925"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137937"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137948"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138108"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138131"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138157"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138209"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138243"}]}]}]}, "emitted_at": 1690184774472} +{"stream": "survey_responses", "data": {"id": "12731242020", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Qg5RZVMhJ6CDLIoNt_2Fdza6f_2FQJjxJXII1xxBSj3_2Fq6BnYaKChCZI73piSP5nhUtB", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731242020", "total_time": 31, "date_modified": "2021-06-10T10:18:19+00:00", "date_created": "2021-06-10T10:17:48+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731242020", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137904"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137924"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137948"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138109"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138127"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138155"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138211"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138239"}]}]}]}, "emitted_at": 1690184774472} +{"stream": "survey_responses", "data": {"id": "12731260237", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=mm7m2qJMFmTa3oybMk65fiRc3YyuaqIC4txcJF43fNUph71wneRBif0eIJvUBTcs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731260237", "total_time": 31, "date_modified": "2021-06-10T10:25:55+00:00", "date_created": "2021-06-10T10:25:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731260237", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137906"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137948"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138112"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138126"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138157"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138211"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138241"}]}]}]}, "emitted_at": 1690184774472} +{"stream": "survey_responses", "data": {"id": "12731279578", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FEjSAqhsmwQRvI0nIbXznSQrW_2Fs9Zok9WSmUIEpPh_2Ftrvxkif6aWXIcsv7S3npp4", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731279578", "total_time": 34, "date_modified": "2021-06-10T10:33:48+00:00", "date_created": "2021-06-10T10:33:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731279578", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137911"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137934"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137951"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138113"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138130"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138156"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138212"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138241"}]}]}]}, "emitted_at": 1690184774473} +{"stream": "survey_responses", "data": {"id": "12731299025", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3IJV7_2FsHEFUvR3AFurcV71Uh5BITVe91sCc8NyxnqEaGN7SJCiiuPB87bmvoaGw1", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731299025", "total_time": 34, "date_modified": "2021-06-10T10:41:36+00:00", "date_created": "2021-06-10T10:41:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731299025", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137905"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137927"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137936"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137950"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138062"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138109"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138130"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138157"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138215"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138239"}]}]}]}, "emitted_at": 1690184774473} +{"stream": "survey_responses", "data": {"id": "12731319249", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=GlfLviGx8LUycI0kX0T8cF_2FFpkVAIQj5obyoRevuZv4czryRkMD2PZvND9Sb17Up", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731319249", "total_time": 33, "date_modified": "2021-06-10T10:49:44+00:00", "date_created": "2021-06-10T10:49:11+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731319249", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137909"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137924"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137931"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137944"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138064"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138109"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138133"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138157"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138210"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138239"}]}]}]}, "emitted_at": 1690184774473} +{"stream": "survey_responses", "data": {"id": "12731338028", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=H_2BVgWdKB0xr0jY94jPfEAo0VmKxhUUF4pufmYQx4fDmquorp1yOG5FlQBPgiHLrx", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731338028", "total_time": 31, "date_modified": "2021-06-10T10:57:15+00:00", "date_created": "2021-06-10T10:56:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731338028", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137911"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137927"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137950"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138060"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138111"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138131"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138155"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138215"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138241"}]}]}]}, "emitted_at": 1690184774474} +{"stream": "survey_responses", "data": {"id": "12731357558", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405829776", "survey_id": "307784846", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tfLWR7lVR8G3JC7jFc4srAqfLvwuFEvSW_2Ffbm1CZxKHZ6J8nad7_2FnXzcLHhgXchk", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D?respondent_id=12731357558", "total_time": 31, "date_modified": "2021-06-10T11:05:04+00:00", "date_created": "2021-06-10T11:04:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784846/responses/12731357558", "pages": [{"id": "168830067", "questions": []}, {"id": "168830068", "questions": [{"id": "667455236", "answers": [{"choice_id": "4385137904"}]}, {"id": "667455240", "answers": [{"choice_id": "4385137926"}]}, {"id": "667455243", "answers": [{"choice_id": "4385137932"}]}, {"id": "667455245", "answers": [{"choice_id": "4385137950"}]}, {"id": "667455263", "answers": [{"choice_id": "4385138060"}]}]}, {"id": "168830074", "questions": [{"id": "667455268", "answers": [{"choice_id": "4385138108"}]}, {"id": "667455272", "answers": [{"choice_id": "4385138126"}]}, {"id": "667455276", "answers": [{"choice_id": "4385138155"}]}, {"id": "667455290", "answers": [{"choice_id": "4385138214"}]}, {"id": "667455293", "answers": [{"choice_id": "4385138240"}]}]}]}, "emitted_at": 1690184774474} +{"stream": "survey_responses", "data": {"id": "12731029969", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=APV_2FZ7VTm09gZj5b0f0NUIlksURZJjTLWSe4fg4_2FRKA8y93l4UjzYAPkgAla3AIo", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731029969", "total_time": 32, "date_modified": "2021-06-10T08:40:52+00:00", "date_created": "2021-06-10T08:40:19+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731029969", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138286"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138291"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138316"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138427"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138437"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138442"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138468"}]}]}]}, "emitted_at": 1690184775696} +{"stream": "survey_responses", "data": {"id": "12731037636", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=iToNu2Xqp1PZ2VPT32fencmUbPumpf00lYTMhCdGbbG2LXZSZ277leTsfXYDwFtv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731037636", "total_time": 32, "date_modified": "2021-06-10T08:45:07+00:00", "date_created": "2021-06-10T08:44:34+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731037636", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138286"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138288"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138315"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138389"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138413"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138424"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138432"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138446"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138501"}]}]}]}, "emitted_at": 1690184775697} +{"stream": "survey_responses", "data": {"id": "12731051876", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QRhskGxFdS_2BJr5Lu32xTKbFyIMZUHZjCRh3x_2Bw039aa_2BRtw9tyy6pHa1Lf_2FC57ic", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731051876", "total_time": 32, "date_modified": "2021-06-10T08:52:52+00:00", "date_created": "2021-06-10T08:52:19+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731051876", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138287"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138294"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138318"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138416"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138422"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138439"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138446"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138500"}]}]}]}, "emitted_at": 1690184775697} +{"stream": "survey_responses", "data": {"id": "12731066155", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=McbGtSUJW9mHgwplanqgtYWhK7utHcK1iZyh5FsyiMXjBFayK7UQxIRtfgwtM_2Bca", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731066155", "total_time": 31, "date_modified": "2021-06-10T09:00:33+00:00", "date_created": "2021-06-10T09:00:01+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731066155", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138283"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138293"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138317"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138423"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138438"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138442"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138502"}]}]}]}, "emitted_at": 1690184775697} +{"stream": "survey_responses", "data": {"id": "12731082183", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=IG3sd3Y0qV_2FQGladjE04s9jje9y8SwVJC2bX5QbuekXXduCLZZ2TedtTVTZU5dGc", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731082183", "total_time": 32, "date_modified": "2021-06-10T09:08:18+00:00", "date_created": "2021-06-10T09:07:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731082183", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138278"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138284"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138288"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138318"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138416"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138423"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138438"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138446"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138499"}]}]}]}, "emitted_at": 1690184775698} +{"stream": "survey_responses", "data": {"id": "12731098436", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MV4ZWqb4qCvpVsm9l8DE5bPr9uyUbMzuYGBxSnRcZ1REiHW2ugohyYyDG7vYEDPl", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731098436", "total_time": 31, "date_modified": "2021-06-10T09:16:00+00:00", "date_created": "2021-06-10T09:15:28+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731098436", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138284"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138291"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138320"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138389"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138421"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138439"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138443"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138501"}]}]}]}, "emitted_at": 1690184775698} +{"stream": "survey_responses", "data": {"id": "12731115074", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=BHzC6lqHgVLzZ1lCVMXIDPuMoH8yVGnrQsGh6WbIIUI3nRO_2F6cMb_2BuTaApRu6hdt", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731115074", "total_time": 31, "date_modified": "2021-06-10T09:23:39+00:00", "date_created": "2021-06-10T09:23:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731115074", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138287"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138290"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138318"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138387"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138416"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138422"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138432"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138441"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138499"}]}]}]}, "emitted_at": 1690184775698} +{"stream": "survey_responses", "data": {"id": "12731131693", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=WYN2n8yADqGUJHwNtkRzDB8UgTsiinkFRQKNlRO2tOl_2FgYSeg5AwgcAfpVBqPc9Z", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731131693", "total_time": 31, "date_modified": "2021-06-10T09:31:22+00:00", "date_created": "2021-06-10T09:30:50+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731131693", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138278"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138287"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138292"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138317"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138387"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138423"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138438"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138444"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138503"}]}]}]}, "emitted_at": 1690184775699} +{"stream": "survey_responses", "data": {"id": "12731149596", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=I4wWuuErezAO1McHSX4e_2FC_2BNPLPDMiVmc0o7Q21l2qpRsReL4GXorsYqqLman2mF", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731149596", "total_time": 31, "date_modified": "2021-06-10T09:39:04+00:00", "date_created": "2021-06-10T09:38:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731149596", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138286"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138288"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138319"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138388"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138415"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138422"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138434"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138444"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138498"}]}]}]}, "emitted_at": 1690184775699} +{"stream": "survey_responses", "data": {"id": "12731167001", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=oF11Z5EpZ0lcqLyzezPfZ7YyRd8_2FNkQHaVMTUl3_2B_2FvwfgV5z5JscZwGxgOXTCvE_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731167001", "total_time": 32, "date_modified": "2021-06-10T09:46:50+00:00", "date_created": "2021-06-10T09:46:17+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731167001", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138282"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138295"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138320"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138426"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138434"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138441"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138502"}]}]}]}, "emitted_at": 1690184775699} +{"stream": "survey_responses", "data": {"id": "12731184662", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=2m4cTOSjC_2FTrSbvTeTRqC4CdH_2F10I0y0VrEPTPWCsU9zVgHoB7Jj9EFyjh7Eso8A", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731184662", "total_time": 35, "date_modified": "2021-06-10T09:54:53+00:00", "date_created": "2021-06-10T09:54:17+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731184662", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138284"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138292"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138315"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138412"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138421"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138438"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138441"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138498"}]}]}]}, "emitted_at": 1690184775700} +{"stream": "survey_responses", "data": {"id": "12731204167", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MDNWDmNrUDODNjQjATix_2BWBUiXJqVC8POKFh8fvebZujKwgJ_2B6XlCVEKCced6L16", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731204167", "total_time": 32, "date_modified": "2021-06-10T10:03:16+00:00", "date_created": "2021-06-10T10:02:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731204167", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138287"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138292"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138315"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138388"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138423"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138432"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138447"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138499"}]}]}]}, "emitted_at": 1690184775700} +{"stream": "survey_responses", "data": {"id": "12731223860", "recipient_id": "", "collection_mode": "default", "response_status": "partial", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": ["168830082", "168830083", "168830087"], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=cwttWQdofPxpfAkwREhinTUX2Ac_2BMviHT6jlan_2BWYlBpX3h0gEfYptMi_2F2sF_2FIut", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731223860", "total_time": 16, "date_modified": "2021-06-10T10:10:45+00:00", "date_created": "2021-06-10T10:10:28+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731223860", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138282"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138292"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138317"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138388"}]}]}, {"id": "168830087", "questions": []}]}, "emitted_at": 1690184775700} +{"stream": "survey_responses", "data": {"id": "12731243435", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=kl6SOoj4lel7HllGtpDwHTPILQJujXr_2Bpkcndfg_2FPQpyHzOR6XWv1saPjDTiOeH4", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731243435", "total_time": 31, "date_modified": "2021-06-10T10:18:55+00:00", "date_created": "2021-06-10T10:18:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731243435", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138283"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138288"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138317"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138389"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138414"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138426"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138434"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138441"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138468"}]}]}]}, "emitted_at": 1690184775701} +{"stream": "survey_responses", "data": {"id": "12731261675", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=5CaWLgB4g6FYPekVOqKYg2mpmDFcRgFD_2FvNGi7Xqozpk2f_2BA5WGRd_2F8q5jr1Jwan", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731261675", "total_time": 31, "date_modified": "2021-06-10T10:26:32+00:00", "date_created": "2021-06-10T10:26:01+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731261675", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138282"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138290"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138315"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138389"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138416"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138423"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138434"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138445"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138500"}]}]}]}, "emitted_at": 1690184775701} +{"stream": "survey_responses", "data": {"id": "12731281414", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=q5Ky3UvnlTpc15S78mm8JQSFeuSBuv8iuZCnl9MZYl_2BB5nkG8L9uJmUl1JNNtwyv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731281414", "total_time": 34, "date_modified": "2021-06-10T10:34:33+00:00", "date_created": "2021-06-10T10:33:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731281414", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138279"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138282"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138293"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138316"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138388"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138412"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138427"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138439"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138445"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138468"}]}]}]}, "emitted_at": 1690184775701} +{"stream": "survey_responses", "data": {"id": "12731300699", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ATEXrllobiYc9DU6PcEpkS_2F8617dJW0dLGMQupo4QwGEGwqFQJ7m6Bks7oWF7Rae", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731300699", "total_time": 34, "date_modified": "2021-06-10T10:42:18+00:00", "date_created": "2021-06-10T10:41:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731300699", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138282"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138295"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138315"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138416"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138427"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138433"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138442"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138503"}]}]}]}, "emitted_at": 1690184775702} +{"stream": "survey_responses", "data": {"id": "12731320713", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=319qghptwfcFVSVtQrSzrw_2BQzR8BlHJbviMDirTnupaGrVddWmTDjhG7R3Tkvb5J", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731320713", "total_time": 31, "date_modified": "2021-06-10T10:50:19+00:00", "date_created": "2021-06-10T10:49:48+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731320713", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138283"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138293"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138317"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138386"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138415"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138426"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138433"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138444"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138499"}]}]}]}, "emitted_at": 1690184775702} +{"stream": "survey_responses", "data": {"id": "12731339540", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=unN8NiMSyoHjiCeMXzhPxCruuBwPKAXbf1kk1TZxnnc04vi5qaipiErEFBUKU3Vw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731339540", "total_time": 32, "date_modified": "2021-06-10T10:57:54+00:00", "date_created": "2021-06-10T10:57:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731339540", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138287"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138291"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138318"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138387"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138412"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138426"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138432"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138444"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138498"}]}]}]}, "emitted_at": 1690184775702} +{"stream": "survey_responses", "data": {"id": "12731359116", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843460", "survey_id": "307784856", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=t_2BHFU_2BkwbG0L8cOxkJuYSUIt0ArFixObCD15GpXjKrp4kEQv20jOpUIgPXB30R0f", "analyze_url": "https://www.surveymonkey.com/analyze/browse/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D?respondent_id=12731359116", "total_time": 32, "date_modified": "2021-06-10T11:05:43+00:00", "date_created": "2021-06-10T11:05:11+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307784856/responses/12731359116", "pages": [{"id": "168830082", "questions": []}, {"id": "168830083", "questions": [{"id": "667455297", "answers": [{"choice_id": "4385138277"}]}, {"id": "667455299", "answers": [{"choice_id": "4385138284"}]}, {"id": "667455301", "answers": [{"choice_id": "4385138290"}]}, {"id": "667455314", "answers": [{"choice_id": "4385138319"}]}, {"id": "667455318", "answers": [{"choice_id": "4385138387"}]}]}, {"id": "168830087", "questions": [{"id": "667455323", "answers": [{"choice_id": "4385138412"}]}, {"id": "667455325", "answers": [{"choice_id": "4385138426"}]}, {"id": "667455328", "answers": [{"choice_id": "4385138434"}]}, {"id": "667455329", "answers": [{"choice_id": "4385138441"}]}, {"id": "667455332", "answers": [{"choice_id": "4385138501"}]}]}]}, "emitted_at": 1690184775703} +{"stream": "survey_responses", "data": {"id": "12731031048", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=oj7KuTDJShqSiV97LkSV6uvYXz_2BBmc1ocrvgJB_2F6EIN4dNLEz7QkUKli1frVW9z_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731031048", "total_time": 32, "date_modified": "2021-06-10T08:41:28+00:00", "date_created": "2021-06-10T08:40:56+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731031048", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172937"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172950"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172991"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173023"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173041"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173088"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173103"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173115"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173170"}]}]}]}, "emitted_at": 1690184776892} +{"stream": "survey_responses", "data": {"id": "12731038731", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=NvnexeVe1TL2cbU10b8utpSHSxTK1HTwbsncQeCDCuIJKnwvRo58aul3yetztS1_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731038731", "total_time": 31, "date_modified": "2021-06-10T08:45:43+00:00", "date_created": "2021-06-10T08:45:11+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731038731", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172935"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172953"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172990"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173021"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173041"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173089"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173104"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173116"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173170"}]}]}]}, "emitted_at": 1690184776892} +{"stream": "survey_responses", "data": {"id": "12731053047", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=TWL1aXuGzW_2Bld3kwRCla449GewPFQF3JFwcpsO1LndH1s3tFfKtesgelUuLTCGR6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731053047", "total_time": 31, "date_modified": "2021-06-10T08:53:29+00:00", "date_created": "2021-06-10T08:52:57+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731053047", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172937"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172948"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172991"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173020"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173045"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173088"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173105"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173114"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173168"}]}]}]}, "emitted_at": 1690184776893} +{"stream": "survey_responses", "data": {"id": "12731067385", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3aXaERIhrxNROtqwHMzJ7BQrVYzI9ua7E_2FxCsZfjUOxvEf88Av09k8zGLpOFLBw_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731067385", "total_time": 31, "date_modified": "2021-06-10T09:01:09+00:00", "date_created": "2021-06-10T09:00:37+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731067385", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172934"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172953"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172995"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173042"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173091"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173104"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173114"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173168"}]}]}]}, "emitted_at": 1690184776893} +{"stream": "survey_responses", "data": {"id": "12731083422", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=kmKEpAG_2FngBuSmfP0e49vf6Q2YDpzhvzQyHTBrwvk0ZeV8Na8b5Evuz7N4wEaQuN", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731083422", "total_time": 32, "date_modified": "2021-06-10T09:08:54+00:00", "date_created": "2021-06-10T09:08:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731083422", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172932"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172953"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172990"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173042"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173089"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173105"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173113"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173169"}]}]}]}, "emitted_at": 1690184776893} +{"stream": "survey_responses", "data": {"id": "12731099693", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YgZuSogQMKDrPTmOoLetMqFtiUi_2BJHKxcATXloIoBnmzHLDvYmNSSoajIC4_2BX2qc", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731099693", "total_time": 31, "date_modified": "2021-06-10T09:16:36+00:00", "date_created": "2021-06-10T09:16:04+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731099693", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172933"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172951"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172993"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173023"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173043"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173076"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173090"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173105"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173113"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173168"}]}]}]}, "emitted_at": 1690184776894} +{"stream": "survey_responses", "data": {"id": "12731116372", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=w67tX9wQ238KZ8cKM0cnTSfg3hzhdt35TpCZfm6_2B7K6B_2F0VZpeEefqMTVZQXnuVe", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731116372", "total_time": 31, "date_modified": "2021-06-10T09:24:15+00:00", "date_created": "2021-06-10T09:23:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731116372", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172932"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172948"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172990"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173046"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173089"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173103"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173117"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173167"}]}]}]}, "emitted_at": 1690184776894} +{"stream": "survey_responses", "data": {"id": "12731133155", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KkixO6u45W2ICclAKrxKYNgJxrynUdDZxArXycLA_2FpZUBsG3FcMXHvqQmk3DsHGL", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731133155", "total_time": 31, "date_modified": "2021-06-10T09:31:58+00:00", "date_created": "2021-06-10T09:31:27+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731133155", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172933"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172948"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172990"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173044"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173090"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173105"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173118"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173168"}]}]}]}, "emitted_at": 1690184776894} +{"stream": "survey_responses", "data": {"id": "12731151051", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Ep0IYx019QcuhuuRrvANkBTw0Ti8WirkyKLpMo62qXxWHwkYGtgh9c143qYGmywG", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731151051", "total_time": 31, "date_modified": "2021-06-10T09:39:40+00:00", "date_created": "2021-06-10T09:39:08+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731151051", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172937"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172949"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172993"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173020"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173046"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173076"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173091"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173105"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173118"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173169"}]}]}]}, "emitted_at": 1690184776895} +{"stream": "survey_responses", "data": {"id": "12731168310", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=HS7SOSb9j2AarAA_2BdLKuhIUhWoas6e_2BtUPh9aNHRDTHOTwcnvASYUjIFJk2Aha4I", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731168310", "total_time": 31, "date_modified": "2021-06-10T09:47:26+00:00", "date_created": "2021-06-10T09:46:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731168310", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172933"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172955"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172995"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173020"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173047"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173089"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173106"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173113"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173168"}]}]}]}, "emitted_at": 1690184776895} +{"stream": "survey_responses", "data": {"id": "12731186075", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Cc0IypCRFbsr7v94sQfKFHhIf_2FH89oCDUARL5wWkYsguNXp0zyA6DxHEdm_2BNMhJ_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731186075", "total_time": 35, "date_modified": "2021-06-10T09:55:32+00:00", "date_created": "2021-06-10T09:54:57+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731186075", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172932"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172954"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172992"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173041"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173088"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173104"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173118"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173169"}]}]}]}, "emitted_at": 1690184776895} +{"stream": "survey_responses", "data": {"id": "12731205704", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=nuK5QZZCJUv9YoeVqOpq22Tt3VOETEzxSzV_2Fk4xjbEXXiwUUe6gMiDzLVDs1IWo_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731205704", "total_time": 31, "date_modified": "2021-06-10T10:03:50+00:00", "date_created": "2021-06-10T10:03:19+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731205704", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172933"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172948"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172995"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173023"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173045"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173090"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173104"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173115"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173168"}]}]}]}, "emitted_at": 1690184776896} +{"stream": "survey_responses", "data": {"id": "12731225522", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=E977wLXc6H_2BfVEbR4rzSs1q6JCgk1O_2BeQ5Sr2f_2BEeSGpDaPDGuslUALFqPRPlZDW", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731225522", "total_time": 31, "date_modified": "2021-06-10T10:11:38+00:00", "date_created": "2021-06-10T10:11:06+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731225522", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172935"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172951"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172995"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173044"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173076"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173091"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173103"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173118"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173167"}]}]}]}, "emitted_at": 1690184776896} +{"stream": "survey_responses", "data": {"id": "12731244870", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3wY0IYE_2F0V4hMJO6Hd2YuVpnFpa2qt5soGs4KM6L3wgFb3k2BHMHLGtDGfFRCSfQ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731244870", "total_time": 31, "date_modified": "2021-06-10T10:19:30+00:00", "date_created": "2021-06-10T10:18:59+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731244870", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172932"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172953"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172993"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173020"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173040"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173091"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173102"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173116"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173170"}]}]}]}, "emitted_at": 1690184776896} +{"stream": "survey_responses", "data": {"id": "12731263131", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=StRkIgYeE5JwNLawW20wxeS_2FED8yP1J_2FHGw3CFhxGAXfN5lodZFoZ7n2CQYuFI4m", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731263131", "total_time": 31, "date_modified": "2021-06-10T10:27:08+00:00", "date_created": "2021-06-10T10:26:37+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731263131", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172933"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172951"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172995"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173021"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173046"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173089"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173106"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173118"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173169"}]}]}]}, "emitted_at": 1690184776897} +{"stream": "survey_responses", "data": {"id": "12731283031", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=J4cSx3UMmpoujky2cMr76AMl3Fslun9t7M7fr8ahVpWNGLwaW_2Bu8X3jQJkNttUcC", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731283031", "total_time": 34, "date_modified": "2021-06-10T10:35:10+00:00", "date_created": "2021-06-10T10:34:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731283031", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172936"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172948"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172994"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173021"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173041"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173090"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173103"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173118"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173169"}]}]}]}, "emitted_at": 1690184776897} +{"stream": "survey_responses", "data": {"id": "12731302340", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ILLNvXOj_2BrxmX_2By2ZRAmTpFslgCTd_2Feoso7P36bgr0_2FDMlf4kKul9iF6Ox4uTGnl", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731302340", "total_time": 34, "date_modified": "2021-06-10T10:42:56+00:00", "date_created": "2021-06-10T10:42:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731302340", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172937"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172951"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172994"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173042"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173091"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173105"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173114"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173167"}]}]}]}, "emitted_at": 1690184776897} +{"stream": "survey_responses", "data": {"id": "12731322211", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=4UUhK9SRY_2FM3fA_2FqkLCL5mS8aBtccVCgcBQMTDfpX84us_2FZDOkgCVuCexjzjooNm", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731322211", "total_time": 31, "date_modified": "2021-06-10T10:50:54+00:00", "date_created": "2021-06-10T10:50:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731322211", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172935"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172950"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172995"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173023"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173047"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173090"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173102"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173115"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173170"}]}]}]}, "emitted_at": 1690184776898} +{"stream": "survey_responses", "data": {"id": "12731341043", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=K51RB0BiJ39RT0TzevECB3iQ54AUAqtSYC5AGG_2Fb3JiEOIV74uOcQhGpP_2B9Xzumv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731341043", "total_time": 32, "date_modified": "2021-06-10T10:58:30+00:00", "date_created": "2021-06-10T10:57:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731341043", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172932"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172949"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172991"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173020"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173042"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173078"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173088"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173106"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173114"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173170"}]}]}]}, "emitted_at": 1690184776898} +{"stream": "survey_responses", "data": {"id": "12731360608", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843624", "survey_id": "307785388", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=joQxJxoDIf61JZdELXuIeomTgZquDM7T76ksdKdRUHFrgw1Him4UO_2BbCMcv0tXrC", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D?respondent_id=12731360608", "total_time": 31, "date_modified": "2021-06-10T11:06:19+00:00", "date_created": "2021-06-10T11:05:47+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785388/responses/12731360608", "pages": [{"id": "168831335", "questions": []}, {"id": "168831336", "questions": [{"id": "667461429", "answers": [{"choice_id": "4385172934"}]}, {"id": "667461433", "answers": [{"choice_id": "4385172951"}]}, {"id": "667461439", "answers": [{"choice_id": "4385172991"}]}, {"id": "667461441", "answers": [{"choice_id": "4385173022"}]}, {"id": "667461444", "answers": [{"choice_id": "4385173040"}]}]}, {"id": "168831340", "questions": [{"id": "667461449", "answers": [{"choice_id": "4385173077"}]}, {"id": "667461452", "answers": [{"choice_id": "4385173089"}]}, {"id": "667461454", "answers": [{"choice_id": "4385173103"}]}, {"id": "667461456", "answers": [{"choice_id": "4385173113"}]}, {"id": "667461462", "answers": [{"choice_id": "4385173167"}]}]}]}, "emitted_at": 1690184776898} +{"stream": "survey_responses", "data": {"id": "12731032160", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=k5_2BoYr8L1JivcJuf1rH2GX_2BHMQI9F9YIVUvhVjaOj1ZaTr1AN4RrYerLAgE0dex5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731032160", "total_time": 30, "date_modified": "2021-06-10T08:42:04+00:00", "date_created": "2021-06-10T08:41:34+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731032160", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174976"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175058"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175072"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175099"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175117"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175148"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175199"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175281"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777745} +{"stream": "survey_responses", "data": {"id": "12731039882", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ueh4txTCmZOSY56GbXU_2BI53xrA9e5AOQ2EOFNrDgskH2E4ZDeJ4HcDXDLxDPUl8L", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731039882", "total_time": 29, "date_modified": "2021-06-10T08:46:16+00:00", "date_created": "2021-06-10T08:45:47+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731039882", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174972"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175072"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175100"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175116"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175148"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175200"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175283"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777746} +{"stream": "survey_responses", "data": {"id": "12731054183", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=6Utx9fT_2BqzHMlSdMksExZ0ph_2BvNfvb4CBc5S5YuvlA0kAV4HfmiM38IyZMLnHuoK", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731054183", "total_time": 29, "date_modified": "2021-06-10T08:54:02+00:00", "date_created": "2021-06-10T08:53:33+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731054183", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174971"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175055"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175070"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175102"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175120"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175152"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175200"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175284"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777746} +{"stream": "survey_responses", "data": {"id": "12731068596", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Y_2BpA02_2Br_2B8t0cCK8Ua99SpTBk8YAUIPP5rIlV7TWE2bRQsxJ7Wy8nlQiPeSSH1Aj", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731068596", "total_time": 29, "date_modified": "2021-06-10T09:01:43+00:00", "date_created": "2021-06-10T09:01:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731068596", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174974"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175058"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175070"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175102"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175117"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175147"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175200"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175279"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777746} +{"stream": "survey_responses", "data": {"id": "12731084672", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=oDcOq8OB_2BpjrJzvV_2B78DTMP7bGXGg_2FFfY5m4CGX3bYG_2BHn3WUTPsqZIYhsLwynBU", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731084672", "total_time": 29, "date_modified": "2021-06-10T09:09:28+00:00", "date_created": "2021-06-10T09:08:59+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731084672", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174970"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175097"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175115"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175146"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175200"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175284"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175294"}]}]}]}, "emitted_at": 1690184777747} +{"stream": "survey_responses", "data": {"id": "12731100913", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=5kE4DYJKeO6DrzEs62N47RQ2rioxaIjqDc5G3RxsiNkZeZZ2L6PL09YQRzsKPf5P", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731100913", "total_time": 29, "date_modified": "2021-06-10T09:17:08+00:00", "date_created": "2021-06-10T09:16:39+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731100913", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174701"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174975"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175058"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175102"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175121"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175148"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175202"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175279"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777747} +{"stream": "survey_responses", "data": {"id": "12731117664", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MK3EF7tVkhtzcIOL0XN0yqP2HY1_2Fqf1WCffD2MxDMDr4xww37ssvSgs9tAUezwLH", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731117664", "total_time": 29, "date_modified": "2021-06-10T09:24:48+00:00", "date_created": "2021-06-10T09:24:19+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731117664", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174977"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175103"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175119"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175147"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175204"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175279"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175294"}]}]}]}, "emitted_at": 1690184777748} +{"stream": "survey_responses", "data": {"id": "12731134542", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vaKWu1zSFOcQlKfr2_2FhbI4n7QTTLkC4cwx4WG8fdzf9IVtI3MzE7e_2FRnH89alHIc", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731134542", "total_time": 29, "date_modified": "2021-06-10T09:32:32+00:00", "date_created": "2021-06-10T09:32:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731134542", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174701"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174973"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175070"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175099"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175118"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175149"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175203"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175284"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777748} +{"stream": "survey_responses", "data": {"id": "12731152328", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=IVs8i7sYR_2FPHAlgRFsF_2FqtmR8VKhb4HLvOemx_2F_2Bx31hPXV_2FqSHT1WSMqyx9fGWRv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731152328", "total_time": 29, "date_modified": "2021-06-10T09:40:14+00:00", "date_created": "2021-06-10T09:39:45+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731152328", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174970"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175056"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175103"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175116"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175152"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175204"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175282"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777748} +{"stream": "survey_responses", "data": {"id": "12731169686", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=I6lNoZHdDu_2FqdBBrqWGXmuNL5Ezwzz6bS8n4LrSLKUFZ5_2FTaf8k7MnS_2FxWjh5UYN", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731169686", "total_time": 29, "date_modified": "2021-06-10T09:48:00+00:00", "date_created": "2021-06-10T09:47:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731169686", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174701"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174973"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175056"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175070"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175102"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175116"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175152"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175200"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175282"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777749} +{"stream": "survey_responses", "data": {"id": "12731187603", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FiK9D4oAC52vzwBrNjbYa9wHiLT6nJc_2BvhbFVDpRqaS4N9_2FeTfbrKcUlu1iDPmLr", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731187603", "total_time": 32, "date_modified": "2021-06-10T09:56:10+00:00", "date_created": "2021-06-10T09:55:38+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731187603", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174701"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174970"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175058"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175070"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175101"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175116"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175150"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175201"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175281"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777749} +{"stream": "survey_responses", "data": {"id": "12731207265", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=DxJNIKLDsZB0jSdF3mQJziPzHSc1QhKVEAwTlaWxpmlfbYqie5vDH6pcFcXka8HP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731207265", "total_time": 29, "date_modified": "2021-06-10T10:04:24+00:00", "date_created": "2021-06-10T10:03:55+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731207265", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174976"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175055"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175071"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175102"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175121"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175150"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175203"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175280"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777749} +{"stream": "survey_responses", "data": {"id": "12731227055", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hifnhc2BvbuUJ2eC5tqTz1sgyzN6BVSqhHrOZMc67UAWrZ86iRSDesnY2EqLW3sf", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731227055", "total_time": 30, "date_modified": "2021-06-10T10:12:13+00:00", "date_created": "2021-06-10T10:11:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731227055", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174974"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175096"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175121"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175148"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175202"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175279"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777749} +{"stream": "survey_responses", "data": {"id": "12731246288", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=9TFIXzJKIgVd5dHU3k_2BM3i00wcrMbvDj3vBf1YYe_2FDh5Hv2kyiiZ13WyXCNq5Og9", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731246288", "total_time": 29, "date_modified": "2021-06-10T10:20:04+00:00", "date_created": "2021-06-10T10:19:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731246288", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174974"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175098"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175117"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175151"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175198"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175284"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175294"}]}]}]}, "emitted_at": 1690184777750} +{"stream": "survey_responses", "data": {"id": "12731264698", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=s1IbylETE10QgXtryui0Bdz_2Bj5IU5LzqBMHDbnhunQHBP5CedlG7l56df1BhmnL2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731264698", "total_time": 29, "date_modified": "2021-06-10T10:27:44+00:00", "date_created": "2021-06-10T10:27:15+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731264698", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174977"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175058"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175096"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175119"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175146"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175199"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175281"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175294"}]}]}]}, "emitted_at": 1690184777750} +{"stream": "survey_responses", "data": {"id": "12731284644", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=gRmE1HF_2BBq7nrEqx2yWW8g0zVXXWYXVJ2u7ohGYqL6fyVTm6aN1WqlcfWrYGOvpl", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731284644", "total_time": 32, "date_modified": "2021-06-10T10:35:47+00:00", "date_created": "2021-06-10T10:35:15+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731284644", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174972"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175055"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175098"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175116"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175146"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175202"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175283"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777750} +{"stream": "survey_responses", "data": {"id": "12731303856", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=R15XZO_2FAOou8IYPELNzZWNvHyYHz5f2yg5XY9DyTPB_2F1F9zoalgsQcLiDzAzYgfp", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731303856", "total_time": 31, "date_modified": "2021-06-10T10:43:31+00:00", "date_created": "2021-06-10T10:43:00+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731303856", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174972"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175056"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175072"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175101"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175115"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175148"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175198"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175284"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175296"}]}]}]}, "emitted_at": 1690184777751} +{"stream": "survey_responses", "data": {"id": "12731323720", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=EiMwydLUjYosaJKqAS7XvTZYSRk1ONKVDTvKy287UbU9nHh4OUTunOiBk4lTOMIX", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731323720", "total_time": 29, "date_modified": "2021-06-10T10:51:27+00:00", "date_created": "2021-06-10T10:50:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731323720", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174700"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174977"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175057"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175071"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175103"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175119"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175152"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175201"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175282"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777751} +{"stream": "survey_responses", "data": {"id": "12731342570", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=26PW15rB0RZChb0Db5YdoDywvKkwI95rERJAB_2BeZi_2F8EbRztchqEvwih9zygXayj", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731342570", "total_time": 30, "date_modified": "2021-06-10T10:59:05+00:00", "date_created": "2021-06-10T10:58:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731342570", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174701"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174971"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175056"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175073"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175099"}]}]}, {"id": "168831402", "questions": [{"id": "667461801", "answers": [{"choice_id": "4385175116"}]}, {"id": "667461805", "answers": [{"choice_id": "4385175146"}]}, {"id": "667461811", "answers": [{"choice_id": "4385175202"}]}, {"id": "667461833", "answers": [{"choice_id": "4385175283"}]}, {"id": "667461834", "answers": [{"choice_id": "4385175295"}]}]}]}, "emitted_at": 1690184777751} +{"stream": "survey_responses", "data": {"id": "12731362003", "recipient_id": "", "collection_mode": "default", "response_status": "partial", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "logic_path": {}, "metadata": {"contact": {}}, "page_path": ["168831392", "168831393", "168831402"], "collector_id": "405843634", "survey_id": "307785415", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=LFmWWSa5B0YPzIswFuEpoIxYrVHHhorNICipbTUhp6Z3p5apUZsA3ENZGZbaPFH0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D?respondent_id=12731362003", "total_time": 19, "date_modified": "2021-06-10T11:06:42+00:00", "date_created": "2021-06-10T11:06:23+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785415/responses/12731362003", "pages": [{"id": "168831392", "questions": []}, {"id": "168831393", "questions": [{"id": "667461690", "answers": [{"choice_id": "4385174702"}]}, {"id": "667461777", "answers": [{"choice_id": "4385174976"}]}, {"id": "667461791", "answers": [{"choice_id": "4385175058"}]}, {"id": "667461794", "answers": [{"choice_id": "4385175071"}]}, {"id": "667461797", "answers": [{"choice_id": "4385175099"}]}]}, {"id": "168831402", "questions": []}]}, "emitted_at": 1690184777752} +{"stream":"survey_pages","data":{"title":"sy4ara","description":"Сурвейманки жлобы","position":1,"question_count":13,"id":"165250506","href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506"},"emitted_at":1674149681286} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831413","href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831413"},"emitted_at":1674149682414} +{"stream":"survey_pages","data":{"title":"xsgqdhdakh7x","description":"wlju6xsgkxyig0s1","position":2,"question_count":5,"id":"168831415","href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415"},"emitted_at":1674149682415} +{"stream":"survey_pages","data":{"title":"ajsn8v0tvicgt7u063","description":"dcwmhxdx6p8buu","position":3,"question_count":5,"id":"168831437","href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437"},"emitted_at":1674149682415} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831459","href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831459"},"emitted_at":1674149683744} +{"stream":"survey_pages","data":{"title":"ijw0pw2tlfb0vd3","description":"k8tycaedxbl4","position":2,"question_count":5,"id":"168831461","href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461"},"emitted_at":1674149683744} +{"stream":"survey_pages","data":{"title":"krd3l3bj7vaym6pc4","description":"oy458fugj0k","position":3,"question_count":5,"id":"168831467","href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467"},"emitted_at":1674149683745} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831344","href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831344"},"emitted_at":1674149684444} +{"stream":"survey_pages","data":{"title":"q4fuvltqc6","description":"7kfibw7aer8mr937a3ko","position":2,"question_count":5,"id":"168831345","href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345"},"emitted_at":1674149684445} +{"stream":"survey_pages","data":{"title":"p5sdpb0pus6","description":"o9gbkpmfik2x","position":3,"question_count":5,"id":"168831352","href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352"},"emitted_at":1674149684445} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831357","href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831357"},"emitted_at":1674149685687} +{"stream":"survey_pages","data":{"title":"133nvr2cx99r","description":"shwtfx0edv","position":2,"question_count":5,"id":"168831358","href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358"},"emitted_at":1674149685687} +{"stream":"survey_pages","data":{"title":"otgwn5b4wicdemu1q","description":"s3ndgrck3qr4898qwtgh","position":3,"question_count":5,"id":"168831365","href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365"},"emitted_at":1674149685687} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831381","href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831381"},"emitted_at":1674149686909} +{"stream":"survey_pages","data":{"title":"ooj54g8q2thh","description":"8hrryr258se","position":2,"question_count":5,"id":"168831382","href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382"},"emitted_at":1674149686909} +{"stream":"survey_pages","data":{"title":"mva5ojqgmx6wnv62as","description":"wq6q460p143mi0","position":3,"question_count":5,"id":"168831388","href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388"},"emitted_at":1674149686909} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830049","href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830049"},"emitted_at":1674149687592} +{"stream":"survey_pages","data":{"title":"v3q97ckq2438fqkppcyn","description":"oforikk3wu4gin","position":2,"question_count":5,"id":"168830050","href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050"},"emitted_at":1674149687593} +{"stream":"survey_pages","data":{"title":"t57ybjyll8fwgu39w","description":"ttlkuqp07ua6kpsh","position":3,"question_count":5,"id":"168830060","href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060"},"emitted_at":1674149687594} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831470","href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831470"},"emitted_at":1674149688355} +{"stream":"survey_pages","data":{"title":"q3wpp1ufpi5r058o","description":"ay91aymge2vuacmrl9co","position":2,"question_count":5,"id":"168831471","href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471"},"emitted_at":1674149688356} +{"stream":"survey_pages","data":{"title":"m345ab1u6tjjo4nn3d","description":"ec22umvdkhxd0ne575","position":3,"question_count":5,"id":"168831478","href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478"},"emitted_at":1674149688357} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830093","href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830093"},"emitted_at":1674149689582} +{"stream":"survey_pages","data":{"title":"7t9dgejtlw5wsofbt","description":"mfqrejibgc831bp31","position":2,"question_count":5,"id":"168830094","href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094"},"emitted_at":1674149689583} +{"stream":"survey_pages","data":{"title":"9kri0ao4fh8e3i0j2hms","description":"dh7qg9jc1k65x","position":3,"question_count":5,"id":"168830108","href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108"},"emitted_at":1674149689583} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830067","href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830067"},"emitted_at":1674149690505} +{"stream":"survey_pages","data":{"title":"9vpm4e5ecjis5hge0p","description":"r6e38n2fp33skkjrl","position":2,"question_count":5,"id":"168830068","href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068"},"emitted_at":1674149690506} +{"stream":"survey_pages","data":{"title":"nvv6kl2njpt5b1l2p","description":"rd2j09sxv4ssu976g","position":3,"question_count":5,"id":"168830074","href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074"},"emitted_at":1674149690506} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830082","href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830082"},"emitted_at":1674149691681} +{"stream":"survey_pages","data":{"title":"v1yxmq6n1ix","description":"aeoyc3hiak9vui1hevm","position":2,"question_count":5,"id":"168830083","href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083"},"emitted_at":1674149691682} +{"stream":"survey_pages","data":{"title":"g84sqoltkc2jen8iaj0","description":"ss2439kly1u4j1k1","position":3,"question_count":5,"id":"168830087","href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087"},"emitted_at":1674149691682} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831335","href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831335"},"emitted_at":1674149692346} +{"stream":"survey_pages","data":{"title":"k91l1laduo8","description":"4tmb1eke23bi1l2ev","position":2,"question_count":5,"id":"168831336","href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336"},"emitted_at":1674149692347} +{"stream":"survey_pages","data":{"title":"gisj5ms868kxxv","description":"4g1iiqg0sa15pbk","position":3,"question_count":5,"id":"168831340","href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340"},"emitted_at":1674149692348} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831392","href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831392"},"emitted_at":1674149693064} +{"stream":"survey_pages","data":{"title":"p71uerk2uh7k5","description":"92cb9d98j15jmfo","position":2,"question_count":5,"id":"168831393","href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393"},"emitted_at":1674149693064} +{"stream":"survey_pages","data":{"title":"bqd6mn6bdgv5u1rnstkx","description":"e0jrpexyx6t","position":3,"question_count":5,"id":"168831402","href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402"},"emitted_at":1674149693065} +{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"36710109","href":"https://api.surveymonkey.com/v3/surveys/510388524/pages/36710109"},"emitted_at":1674149694191} +{"stream":"survey_questions","data":{"id":"652286724","position":1,"visible":true,"family":"click_map","subtype":"single","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"Click on the area you like best about this product.","image":{"url":"https://surveymonkey-assets.s3.amazonaws.com/survey/306079584/20535460-8f99-41b0-ac96-b9f4f2aecb96.png"}}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286724","answers":{"rows":[{"position":1,"visible":true,"text":"Click 1","id":"4285525098"}]},"page_id":"165250506"},"emitted_at":1674149694470} +{"stream":"survey_questions","data":{"id":"652286725","position":2,"visible":true,"family":"click_map","subtype":"single","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"Click on the area you like least about this product.","image":{"url":"https://surveymonkey-assets.s3.amazonaws.com/survey/306079584/79215d25-9dbc-4870-91cd-3a36778aae52.png"}}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286725","answers":{"rows":[{"position":1,"visible":true,"text":"Click 1","id":"4285525102"}]},"page_id":"165250506"},"emitted_at":1674149694470} +{"stream":"survey_questions","data":{"id":"652286726","position":3,"visible":true,"family":"open_ended","subtype":"essay","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"Why did you make that selection?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286726","page_id":"165250506"},"emitted_at":1674149694471} +{"stream":"survey_questions","data":{"id":"652286715","position":4,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"What is your first reaction to the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286715","answers":{"choices":[{"position":1,"visible":true,"text":"Very positive","quiz_options":{"score":0},"id":"4285525063"},{"position":2,"visible":true,"text":"Somewhat positive","quiz_options":{"score":0},"id":"4285525064"},{"position":3,"visible":true,"text":"Neutral","quiz_options":{"score":0},"id":"4285525065"},{"position":4,"visible":true,"text":"Somewhat negative","quiz_options":{"score":0},"id":"4285525066"},{"position":5,"visible":true,"text":"Very negative","quiz_options":{"score":0},"id":"4285525067"}]},"page_id":"165250506"},"emitted_at":1674149694471} +{"stream":"survey_questions","data":{"id":"652286721","position":5,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How would you rate the quality of the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286721","answers":{"choices":[{"position":1,"visible":true,"text":"Very high quality","quiz_options":{"score":0},"id":"4285525083"},{"position":2,"visible":true,"text":"High quality","quiz_options":{"score":0},"id":"4285525084"},{"position":3,"visible":true,"text":"Neither high nor low quality","quiz_options":{"score":0},"id":"4285525085"},{"position":4,"visible":true,"text":"Low quality","quiz_options":{"score":0},"id":"4285525086"},{"position":5,"visible":true,"text":"Very low quality","quiz_options":{"score":0},"id":"4285525087"}]},"page_id":"165250506"},"emitted_at":1674149694472} +{"stream":"survey_questions","data":{"id":"652286716","position":6,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How innovative is the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286716","answers":{"choices":[{"position":1,"visible":true,"text":"Extremely innovative","quiz_options":{"score":0},"id":"4285525068"},{"position":2,"visible":true,"text":"Very innovative","quiz_options":{"score":0},"id":"4285525069"},{"position":3,"visible":true,"text":"Somewhat innovative","quiz_options":{"score":0},"id":"4285525070"},{"position":4,"visible":true,"text":"Not so innovative","quiz_options":{"score":0},"id":"4285525071"},{"position":5,"visible":true,"text":"Not at all innovative","quiz_options":{"score":0},"id":"4285525072"}]},"page_id":"165250506"},"emitted_at":1674149694473} +{"stream":"survey_questions","data":{"id":"652286718","position":7,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"When you think about the product, do you think of it as something you need or don’t need?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286718","answers":{"choices":[{"position":1,"visible":true,"text":"Definitely need","quiz_options":{"score":0},"id":"4285525078"},{"position":2,"visible":true,"text":"Probably need","quiz_options":{"score":0},"id":"4285525079"},{"position":3,"visible":true,"text":"Neutral","quiz_options":{"score":0},"id":"4285525080"},{"position":4,"visible":true,"text":"Probably don’t need","quiz_options":{"score":0},"id":"4285525081"},{"position":5,"visible":true,"text":"Definitely don’t need","quiz_options":{"score":0},"id":"4285525082"}]},"page_id":"165250506"},"emitted_at":1674149694474} +{"stream":"survey_questions","data":{"id":"652286722","position":8,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How would you rate the value for money of the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286722","answers":{"choices":[{"position":1,"visible":true,"text":"Excellent","quiz_options":{"score":0},"id":"4285525088"},{"position":2,"visible":true,"text":"Above average","quiz_options":{"score":0},"id":"4285525089"},{"position":3,"visible":true,"text":"Average","quiz_options":{"score":0},"id":"4285525090"},{"position":4,"visible":true,"text":"Below average","quiz_options":{"score":0},"id":"4285525091"},{"position":5,"visible":true,"text":"Poor","quiz_options":{"score":0},"id":"4285525092"}]},"page_id":"165250506"},"emitted_at":1674149694474} +{"stream":"survey_questions","data":{"id":"652286717","position":9,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"If the product were available today, how likely would you be to buy the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286717","answers":{"choices":[{"position":1,"visible":true,"text":"Extremely likely","quiz_options":{"score":0},"id":"4285525073"},{"position":2,"visible":true,"text":"Very likely","quiz_options":{"score":0},"id":"4285525074"},{"position":3,"visible":true,"text":"Somewhat likely","quiz_options":{"score":0},"id":"4285525075"},{"position":4,"visible":true,"text":"Not so likely","quiz_options":{"score":0},"id":"4285525076"},{"position":5,"visible":true,"text":"Not at all likely","quiz_options":{"score":0},"id":"4285525077"}]},"page_id":"165250506"},"emitted_at":1674149694475} +{"stream":"survey_questions","data":{"id":"652286723","position":10,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How likely are you to replace your current product with the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286723","answers":{"choices":[{"position":1,"visible":true,"text":"Extremely likely ","quiz_options":{"score":0},"id":"4285525093"},{"position":2,"visible":true,"text":"Very likely ","quiz_options":{"score":0},"id":"4285525094"},{"position":3,"visible":true,"text":"Somewhat likely","quiz_options":{"score":0},"id":"4285525095"},{"position":4,"visible":true,"text":"Not so likely","quiz_options":{"score":0},"id":"4285525096"},{"position":5,"visible":true,"text":"Not at all likely","quiz_options":{"score":0},"id":"4285525097"}]},"page_id":"165250506"},"emitted_at":1674149694475} +{"stream":"survey_questions","data":{"id":"652286714","position":11,"visible":true,"family":"matrix","subtype":"rating","layout":{"bottom_spacing":0,"col_width":80,"col_width_format":"percent","left_spacing":0,"num_chars":null,"num_lines":null,"position":"new_row","right_spacing":0,"top_spacing":0,"width":100,"width_format":"percent"},"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How likely is it that you would recommend our new product to a friend or colleague?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286714","answers":{"rows":[{"position":1,"visible":true,"text":"","id":"4285525061"}],"choices":[{"position":1,"visible":true,"text":"Not at all likely - 0","id":"4285525050","is_na":false,"weight":-100,"description":"Not at all likely"},{"position":2,"visible":true,"text":"1","id":"4285525051","is_na":false,"weight":-100,"description":""},{"position":3,"visible":true,"text":"2","id":"4285525052","is_na":false,"weight":-100,"description":""},{"position":4,"visible":true,"text":"3","id":"4285525053","is_na":false,"weight":-100,"description":""},{"position":5,"visible":true,"text":"4","id":"4285525054","is_na":false,"weight":-100,"description":""},{"position":6,"visible":true,"text":"5","id":"4285525055","is_na":false,"weight":-100,"description":""},{"position":7,"visible":true,"text":"6","id":"4285525056","is_na":false,"weight":-100,"description":""},{"position":8,"visible":true,"text":"7","id":"4285525057","is_na":false,"weight":0,"description":""},{"position":9,"visible":true,"text":"8","id":"4285525058","is_na":false,"weight":0,"description":""},{"position":10,"visible":true,"text":"9","id":"4285525059","is_na":false,"weight":100,"description":""},{"position":11,"visible":true,"text":"Extremely likely - 10","id":"4285525060","is_na":false,"weight":100,"description":"Extremely likely"}]},"page_id":"165250506"},"emitted_at":1674149694476} +{"stream":"survey_questions","data":{"id":"652286719","position":12,"visible":true,"family":"open_ended","subtype":"essay","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"In your own words, what are the things that you like most about this new product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286719","page_id":"165250506"},"emitted_at":1674149694476} +{"stream":"survey_questions","data":{"id":"652286720","position":13,"visible":true,"family":"open_ended","subtype":"essay","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"In your own words, what are the things that you would most like to improve in this new product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286720","page_id":"165250506"},"emitted_at":1674149694477} +{"stream":"survey_questions","data":{"id":"667461858","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vprw9pgg3d xiygcvd0suru k7rews838g6qc ndsukv2 7sa31urnvoskixw c52sg4uoete874i"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461858","answers":{"choices":[{"position":1,"visible":true,"text":"m5kpo9621yynjey kdt5w6pkkit yqyocxqf yw1p3uh2e5b 7gtmvs4 6em5ugqat x6pmhcfrvq4pit t67pif54aj jbgure","quiz_options":{"score":0},"id":"4385175366"},{"position":2,"visible":true,"text":"usdfft kvi9yqh1m38w3m 6uxryyvhrk1 nfxlt gnhjy826e rqks3jjuyj9hd 3y8755o","quiz_options":{"score":0},"id":"4385175367"},{"position":3,"visible":true,"text":"m6xv3yca7 up9u0qwx23h2skj 0cjlw19k5emypgm awi5tg l9atp kv4jrd73y9","quiz_options":{"score":0},"id":"4385175368"},{"position":4,"visible":true,"text":"todhc7 krw2v8qa rt2iu19vhxyw1dp x6oav54yak4vj yu4le2fc7 fksvl ejbr7x2u69 k9n9n7g3f","quiz_options":{"score":0},"id":"4385175369"}]},"page_id":"168831415"},"emitted_at":1674149694578} +{"stream":"survey_questions","data":{"id":"667461861","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"rl3uabslq46p mnwh0fle3xfs ejupx8e26q55va svfm11o"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461861","answers":{"choices":[{"position":1,"visible":true,"text":"m6g15xqsuwpbh3 x0116lkpod 5vkgg7duiq23sp ot884xd67v6fv2q 1u2mpgo ttj3ehahbljf1j6 pwj46w1d","quiz_options":{"score":0},"id":"4385175380"},{"position":2,"visible":true,"text":"cuff7 mbn2k1hxd6n6 jg9kffdkccjh bpodqpt2wtxu 7x38qxmvg42ap qpv0cddfumvix s0vv161iytceelx","quiz_options":{"score":0},"id":"4385175381"},{"position":3,"visible":true,"text":"jm5q6yu4rn pl8wwv23lnxs ou5r8m3np4fis6 6wlatg yeh3kafns0 h8u0o8f yhqni064ev6","quiz_options":{"score":0},"id":"4385175382"}]},"page_id":"168831415"},"emitted_at":1674149694579} +{"stream":"survey_questions","data":{"id":"667461876","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"hn4xnf ox0joj4inoy6ja jh02428n qeqxm9nopevjca sccwladi v63ks6mdqf0h0 4pug94eya 5et1g3t4 exbryyy6hv9mvd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461876","answers":{"choices":[{"position":1,"visible":true,"text":"w6nvr mkrj8q1g 740smg3nda m3afibg8 224jb59fon975t w9t8ma","quiz_options":{"score":0},"id":"4385175447"},{"position":2,"visible":true,"text":"cerw942pk xv1wg4gk4l7jq q3grdgasaol 75ghj ppo6ivm3r hxodiktx9rxs","quiz_options":{"score":0},"id":"4385175448"},{"position":3,"visible":true,"text":"5os82a1jwgygye 61dhsf6v sgy0ui7ib78ws7f j3pymv","quiz_options":{"score":0},"id":"4385175449"}]},"page_id":"168831415"},"emitted_at":1674149694580} +{"stream":"survey_questions","data":{"id":"667461897","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5a32s3dhl a967a54aoj3cr ttu0uk4hu4h1 r360wdohq1x9xu7 qvqpg1eb6qg3 01ogn ucbhive fpwstwi6hre0ynb xu3c3txpma7eoh3"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461897","answers":{"choices":[{"position":1,"visible":true,"text":"6gngt 5w10n7fl47r07 68t8t79 66pqqth5urrw 1ve2kn385x0u9s 99vcfs08at","quiz_options":{"score":0},"id":"4385175533"},{"position":2,"visible":true,"text":"pnwrx dwj1dx dpdan1wpqs 9lhgks36 1w8a2utjbxas31t rlc1u51mdpjr 90tcj6i8ibicvxt q1ahtd2x doujpba kjjjdi0","quiz_options":{"score":0},"id":"4385175534"},{"position":3,"visible":true,"text":"wu74ewyb4grv fqb8h3yoldsn 0nxv5844yn0lpx jct7na y9sp3u ueq7vk83ix7g7sx f5sl73r2r29e84","quiz_options":{"score":0},"id":"4385175535"}]},"page_id":"168831415"},"emitted_at":1674149694581} +{"stream":"survey_questions","data":{"id":"667461902","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"sdek6wejcdn 82r223sfhy6xkm5 65gns2m7phi 0fx8dx5bp psvndjnn5b 5kki467 a8faadeid0gl13 x2t3e 03xco2 cf39nv9mdq3vj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461902","answers":{"choices":[{"position":1,"visible":true,"text":"nfiq0 d7gwft9 8bhinfrsv6r6 k3vylofokamx3 9hrik k2ageb5amyj89a toli0yrsq bqbrcp","quiz_options":{"score":0},"id":"4385175559"},{"position":2,"visible":true,"text":"8c5qdiklqb8tb 5uhc6w1a1o9 er8h4w0mf779o 14nsksqs65j10","quiz_options":{"score":0},"id":"4385175561"},{"position":3,"visible":true,"text":"5wpggsufojm 4decq179m5 0brpk1la0kyno e8ctqi fxa4j0uo9atp","quiz_options":{"score":0},"id":"4385175563"},{"position":4,"visible":true,"text":"5it4y4q 49im15osxk2j0 j8twpv8j nei1egowtm9a lyrigqwu0eby tpg5o7kuvgn34l spdu5icxlc 2f4qf qb55g 8si14ri4bdw1v72","quiz_options":{"score":0},"id":"4385175564"}]},"page_id":"168831415"},"emitted_at":1674149694582} +{"stream":"survey_questions","data":{"id":"667461933","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"4vnmxluvh7a l8s5r0i5ck pa7pckqf tnah7fan76p38n8 m1i5r ea9ib0t823amcn k98290k23wh7qn"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461933","answers":{"choices":[{"position":1,"visible":true,"text":"aldwijo3p9mh3 v26dql0wbrctg0 98sqogqfmi2 558gsvip","quiz_options":{"score":0},"id":"4385175732"},{"position":2,"visible":true,"text":"c5fqe o43qm3oq9i 23qgoxg85 cg5hr5wj79469n 328j7ji1kugb 60jlwtbu4eoh63 alyho5","quiz_options":{"score":0},"id":"4385175733"},{"position":3,"visible":true,"text":"882jtclxvq15yk dgive oqhqdf a8a2rgxse4b8fw 87oeyfbk7 wcevsx 4mv1lyp6lyxysrm 3m8yayarq7wm 9bpmul9 el2j1j4yw","quiz_options":{"score":0},"id":"4385175734"},{"position":4,"visible":true,"text":"3so1buxa 88ypp61nq6bhi0 lwa2pfg6tg lra8e1r5bn4k umstgwdck9 wslq681gvn3g2f a20nr1eovr3 feo0p5bqbgrcvun np851e23cojfv1 uqlwg","quiz_options":{"score":0},"id":"4385175735"}]},"page_id":"168831437"},"emitted_at":1674149694582} +{"stream":"survey_questions","data":{"id":"667461934","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"k6ba3wdl 6c0nvj5es h0dttl3krobdcd va6klkdyf 79e2a lhmov"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461934","answers":{"choices":[{"position":1,"visible":true,"text":"h8dywnj ithq4el 98e20manw xxyp0f9oike55 t56arby dhmmxh4pehs","quiz_options":{"score":0},"id":"4385175736"},{"position":2,"visible":true,"text":"14p6263c 7khx2y rpjy6432cy7kkr 0po0vol3uk1cuf 1oejmy viircj9b nyw76yqpf","quiz_options":{"score":0},"id":"4385175737"},{"position":3,"visible":true,"text":"vtdf2x 2kme3 vhpqi5s82k7v4 1mjr5r jp0ox03i6t d5ef5228du3ck 536btd7etv","quiz_options":{"score":0},"id":"4385175738"},{"position":4,"visible":true,"text":"yt2e6nk6 v2wl4k047h1n5 civs8 tjjr7lkeay9i3","quiz_options":{"score":0},"id":"4385175739"}]},"page_id":"168831437"},"emitted_at":1674149694583} +{"stream":"survey_questions","data":{"id":"667461936","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"u1jbhrxyrg30ra9 qs6b5m237lag 7tlvv 05okuge2imipht bg813 hixvf9bd9n10gk lwxdjnajbu9gsra uyh3fjd40bs"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461936","answers":{"choices":[{"position":1,"visible":true,"text":"yl8yqqhtr9rxi 049x312 oogub1figyg4e tixix79g85hxi2l bin0fp5g goq3kwu2eaase x9mihu0erdhl9v4 4yausp34y2rgyx iwyfuo4b7yme8lr","quiz_options":{"score":0},"id":"4385175740"},{"position":2,"visible":true,"text":"gfqbxk8 3tosymacon00av uoxydvdy28 iuce5bjdvpg gwxdpkudg24ouk u2ns1u x10hujmiiy9l62i soln75a14nm8q0","quiz_options":{"score":0},"id":"4385175741"},{"position":3,"visible":true,"text":"j299iy64y804 k709a5kk9 yjsifyt4ksu b4fnp4a","quiz_options":{"score":0},"id":"4385175742"}]},"page_id":"168831437"},"emitted_at":1674149694584} +{"stream":"survey_questions","data":{"id":"667461937","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"06qgl j7f89g3 5ok0ilvhvpa ojcs6ixbovvf u6xympul0 o00vc 6bjg7jmvy9s 543ndxd jmugw njqhxa3phj4jd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461937","answers":{"choices":[{"position":1,"visible":true,"text":"5v1hangb7u0bj8 44xo3y4o inytm95s16m dvn5e2leb6sf7 hamaw8l538eph 6xuyp5030i538c 0ym6m831 duar0bo76 kl36ykf5w9ugh","quiz_options":{"score":0},"id":"4385175745"},{"position":2,"visible":true,"text":"3iiaxf5dgi tr6crqs i51ocgg ka1ri0ffd8xeh cv12807q5pq 7fmq8lxy7 5wuas7gcn4kln tv6mf3pqq4jm x3yfr sm7idq2nvoonk","quiz_options":{"score":0},"id":"4385175746"},{"position":3,"visible":true,"text":"mj5cmssx3htoni ld622ppaiqr uw63p1q4up3 lfxioj94gd3jky0 2tj631 5ql6116xsyp r5hwhy2","quiz_options":{"score":0},"id":"4385175747"},{"position":4,"visible":true,"text":"llb9oe7fk8v3w4q xpd66rgbyp9m 0opo19df7n6 scr70cg86pn o1qtr7rclqxjl kmpnf79790 6wmig6mjwflh2vq 1oe033jyd","quiz_options":{"score":0},"id":"4385175748"},{"position":5,"visible":true,"text":"pg851 mju7py cdp1jqcaeg66 3gv05ohwromt u0uot","quiz_options":{"score":0},"id":"4385175749"},{"position":6,"visible":true,"text":"mu0hn799v 4ch8i7j5o5tf8s 6rn9y0ft67mchc u5ds14s9 aj1qg26dwf41suc i1mdhdk","quiz_options":{"score":0},"id":"4385175750"},{"position":7,"visible":true,"text":"6w4xjejtc9j 50n12 u1cdbulvkqykvci x79sdq9y hmbem37x7 s7pwufdjnmn xo8qy81a3fjmv","quiz_options":{"score":0},"id":"4385175751"},{"position":8,"visible":true,"text":"k582dst vgjvse b2h3mxi dteo4p9lrtx m54ug","quiz_options":{"score":0},"id":"4385175752"}]},"page_id":"168831437"},"emitted_at":1674149694585} +{"stream":"survey_questions","data":{"id":"667461986","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"gjahy0fl7 fs6b3q6e7 rsvbmotgt9p7 w9l2l7 hvj8fhrqwc vays8 4yh7qch hjj5lx0 co6a5rqd 5pt79or0a7evc"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461986","answers":{"choices":[{"position":1,"visible":true,"text":"b7fbynqu hwhvtbqwbmxy 9bjgxs1 1e05mo toj685p5v w34e97j","quiz_options":{"score":0},"id":"4385175878"},{"position":2,"visible":true,"text":"n5e5yl9c6 yw12588olh swl7nwm1dl9n2l 9v7n8wursm6739 p67woq9u27w7p y3ge5y1iji819g7 uklmy7q8 7ocv68por","quiz_options":{"score":0},"id":"4385175879"},{"position":3,"visible":true,"text":"h08nyyi 1393hst fcdij6j yepfw2","quiz_options":{"score":0},"id":"4385175880"},{"position":4,"visible":true,"text":"rdme8pwwjl07 4ju4xn47ofvbj i31u4ty4f4 wteatx2 gc3nqgji pu9h7","quiz_options":{"score":0},"id":"4385175881"},{"position":5,"visible":true,"text":"jxkod8gx x8tcsxxle4f0lv4 vcmjicpk7v i19dxl3 3lvmmdkx","quiz_options":{"score":0},"id":"4385175882"}]},"page_id":"168831437"},"emitted_at":1674149694587} +{"stream":"survey_questions","data":{"id":"667462078","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vrwr5e5qwxqu4 mn9jcdpf 0e5u3k ansge 2hwkipig u0wn3acc2sct xnv4y8 3irjcv i0cgva8762tfosw"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462078","answers":{"choices":[{"position":1,"visible":true,"text":"55yvst1hihf4br ialhkcxm73fjln 1lelxuw93g0 nukml0","quiz_options":{"score":0},"id":"4385176721"},{"position":2,"visible":true,"text":"okdweedyn93 mua59r2l4haj orem637bgltyb 7jc2yral4o8qov j15ry fkkgjxdea rjd297yot eq57kefir4if3c 2l098 ech2y5fv","quiz_options":{"score":0},"id":"4385176722"},{"position":3,"visible":true,"text":"b3sgu7kkbjowg ux7qqyw41yb umfqpq6 dfgu4awr uy2i4j626 cb17jp5xal6","quiz_options":{"score":0},"id":"4385176723"},{"position":4,"visible":true,"text":"5igq4pw5ul la3i72sh30 uk24o0qi jh51hl9s3a43s 9tgq0ip8k1nev ar6it adgfobu491 f8qke95 o2f9u2ubb49t28c obyoj9cfsl","quiz_options":{"score":0},"id":"4385176724"},{"position":5,"visible":true,"text":"r73ge7qkd8mjfu d7kdhfmco d0pcoyqqyjrph4g 06xs83492x 5ajmrgy1 x4ev3aroh9q86r gwiu17g02i6h75c np3r62er4x5n 73hbkk43af 7f5b333hk7tkc","quiz_options":{"score":0},"id":"4385176725"},{"position":6,"visible":true,"text":"vo6u3 9gpy2s57xh u8dcib 4f8gbbng3wq0h36","quiz_options":{"score":0},"id":"4385176726"}]},"page_id":"168831461"},"emitted_at":1674149694691} +{"stream":"survey_questions","data":{"id":"667462079","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6d3w2gqt8056 h5vl4u3kcuvlqc lk2ip62d kpkowwj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462079","answers":{"choices":[{"position":1,"visible":true,"text":"agvpu24j1 g51hr01jbjsk vw6gq cftg9oeklijpda qbibhtf35pl4gtq wu48wsd5 c6jifqlyt4e","quiz_options":{"score":0},"id":"4385176727"},{"position":2,"visible":true,"text":"4kbb46n 0vmm0c4we qwaiv1f731l y8iaiu3bkcb6 loqrsy","quiz_options":{"score":0},"id":"4385176728"},{"position":3,"visible":true,"text":"rwm2cgejb qc4g9 7y68obgewd fou0um xh7dkb89o bfosq3 v9bdp7s9450","quiz_options":{"score":0},"id":"4385176729"}]},"page_id":"168831461"},"emitted_at":1674149694692} +{"stream":"survey_questions","data":{"id":"667462082","position":3,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"dvpu842k yh6g475bbwk75 qtkq7f5yd01igo ixldsnn uqbpr ngarg6pf f6ueafaq 4ch8dhr nqsm5uydajns9"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462082","answers":{"choices":[{"position":1,"visible":true,"text":"hfrp92k576cl 4yjdkhqj0vr9 qx16a oc86p8hhonp5xs fn4lo1wmewe5n7 sue7kinfb6 8qpyh auccq7xlw1b","quiz_options":{"score":0},"id":"4385176734"},{"position":2,"visible":true,"text":"lqon4mp qhj98iee7o 37f50nvkrs3va 18vxkbch636kh yn8ih 8jxqpq03v 6j5lurdu7c17 knmjb3 0868hyuu7","quiz_options":{"score":0},"id":"4385176735"},{"position":3,"visible":true,"text":"obj26 pb0pwmt gxvk8isp04c42 77ocbs1 y7jyxsbl vbbtsy d1t8el31vu7x6d r44iyhr2y810o0o","quiz_options":{"score":0},"id":"4385176736"},{"position":4,"visible":true,"text":"yebv8qvq dd5qeecs45k 2un02jrywfqf1 u96tcry54cxiyu mnbgr2mqe a43wm7 1rnggj b99hudv2g0kgor","quiz_options":{"score":0},"id":"4385176737"},{"position":5,"visible":true,"text":"b6v1txm65a40 h1sah8 84enie0e 57clim 2eew4","quiz_options":{"score":0},"id":"4385176738"},{"position":6,"visible":true,"text":"d88o04mls0oddci d61gj k4acpqp r3ugg2t1e55s ldb3ll5gjkn ruerq4w4w95j649 c3bea007si4t 4jp1ctllptvfao2 m78584fllehmi b9fpb","quiz_options":{"score":0},"id":"4385176739"}]},"page_id":"168831461"},"emitted_at":1674149694692} +{"stream":"survey_questions","data":{"id":"667462084","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"n6y1nnlv45movkl vkjge u5wp4unnsf cp6y14e"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462084","answers":{"choices":[{"position":1,"visible":true,"text":"2ak2bbt6l 958nphvdetkv4 ycoevsx u49by0 eohbcorqyd2fk","quiz_options":{"score":0},"id":"4385176763"},{"position":2,"visible":true,"text":"6lnrx0h 8rtivmuwxpy8wv ebygucb3xu1m42 iyep7ijy4gu0mvl s1oco7cxc0","quiz_options":{"score":0},"id":"4385176764"},{"position":3,"visible":true,"text":"m5ysyahej9ffbl hiob9pqg7je5r oqyup1bmda i142y odbg2sgkibp5a5n","quiz_options":{"score":0},"id":"4385176765"},{"position":4,"visible":true,"text":"r56bqr7ts07qj7l 0odv2rxnmffpw ctmr30tp61ibr l64xab0cs nmqjd 21rw3b cy2xks0me55b05","quiz_options":{"score":0},"id":"4385176766"},{"position":5,"visible":true,"text":"wjnn6f 4at64hj rif45s1bu9asypf vmjw8wv55isa 87va6t705w","quiz_options":{"score":0},"id":"4385176767"}]},"page_id":"168831461"},"emitted_at":1674149694693} +{"stream":"survey_questions","data":{"id":"667462086","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ikhsonq7 fadyfis1w 1qmo7d2wgcq 5nht9h9j esydnd1m a5ehikxk6ypubk dtanss721p6"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462086","answers":{"choices":[{"position":1,"visible":true,"text":"3p8dvrh6lr0 iejlsiah ch3n24c7d9tt 7w9rvdkxpqe8xh fo9jq7k8sa1 er31i44d633 xurs2ie khm8mmp5d8gc9 egn43 86jj22","quiz_options":{"score":0},"id":"4385176918"},{"position":2,"visible":true,"text":"r5qhjy06onu7loq n6htekeur 2xm1jh99 ibsp4oat8878fy 940xq9 n4tmuhn e5901s317nbq pevqvednus0ph8g","quiz_options":{"score":0},"id":"4385176919"},{"position":3,"visible":true,"text":"at4b6f9tud4tvpl uf1d5p jgci9m0u17qkj 8tmdkc9o lb7b63gth6 6ds89i5 uu91pd2ybc wwk60i ntwtf5enlg5h8o","quiz_options":{"score":0},"id":"4385176920"},{"position":4,"visible":true,"text":"ovrn2np gg0nxt88wji6fp ckphrf1l3 0um296qkhvgh","quiz_options":{"score":0},"id":"4385176921"}]},"page_id":"168831461"},"emitted_at":1674149694694} +{"stream":"survey_questions","data":{"id":"667462094","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"uhxc9mpgrh9c9aa 05iqvfqi y8rip1qcmmxh b8jj5k cavri2y5 1sqge 7ywuxm3awoh ddul9pvje6dcr kggd2y4"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462094","answers":{"choices":[{"position":1,"visible":true,"text":"t30umxtof3b6e 9o88p i9hcrmm57pq42 16pnnhw idraibx70sby1i","quiz_options":{"score":0},"id":"4385176959"},{"position":2,"visible":true,"text":"xnxw9wa4fo 157vk4i0d 1m9fluu gmmy39m41 icu2b548yd9p3o 3yj0tyamvyjci1 1c19qkbuqb031r","quiz_options":{"score":0},"id":"4385176960"},{"position":3,"visible":true,"text":"ehjwxib3hha9 bjj11etv shhdikfh1iy0u nioigschrcpqi2 v7mgatm l98c4no7k9 bf2j11cmi1dgbt2 f8etga4g kg2w2yngt62","quiz_options":{"score":0},"id":"4385176961"},{"position":4,"visible":true,"text":"uyqt6g2o61rc wcwn8q7ohbhegr wtwkdefljgy uli9v6","quiz_options":{"score":0},"id":"4385176962"},{"position":5,"visible":true,"text":"7pg26slappuq1 aw3vsgc4317be eqe9a8v6s5whg0 dvrw9imoe o9wl91vi282 4h7f26","quiz_options":{"score":0},"id":"4385176963"},{"position":6,"visible":true,"text":"wjipnxgtewp8r ddtad8 yqo4rj5x657nge ykw2qghp3e r3tqrvk1m7l ir60pwi5g5 h7exqf4","quiz_options":{"score":0},"id":"4385176964"},{"position":7,"visible":true,"text":"aafwgbh80 ip6mgadg 8ls8v4ss2fk4y s19ti1jvfnaey g0v4qdn ifk0ve j2trlbjolhuc3 op07p93t1xsh0 kqiw7s ktofe9muryqxe","quiz_options":{"score":0},"id":"4385176965"}]},"page_id":"168831467"},"emitted_at":1674149694694} +{"stream":"survey_questions","data":{"id":"667462096","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"hh8r3c6evhaj2ug 34w4tpf8 b41jtyj tn7vh1hwl 7rrs5 p5rct6v9pg7k 7nw3etg9"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462096","answers":{"choices":[{"position":1,"visible":true,"text":"4jlthohsbk gq1uy9yt 83htjom d5udup0g0jfsido a6nxtc2vkdn 7tfgt9 6e6wsq","quiz_options":{"score":0},"id":"4385176968"},{"position":2,"visible":true,"text":"nt9b0e1tk eq9ckgs2u2fiu probh n7929oe950jcs7c uvdxlffx66obu 6oschegtcq q2a7d hwp71gp7 s3b03f5p3g24h","quiz_options":{"score":0},"id":"4385176969"},{"position":3,"visible":true,"text":"u1p9r on01v021jli y4n4d2gpcbvk fe3y53pjwje6ms atckrpa2r44f 5lkaim1 coo8gs4mer wfbusogrereep 3r2i1qlkh6097d1","quiz_options":{"score":0},"id":"4385176970"},{"position":4,"visible":true,"text":"i4liv68323s xc5i56ba5pa osxulquad221n xbmov jnhv1ogkdw9","quiz_options":{"score":0},"id":"4385176971"},{"position":5,"visible":true,"text":"tlinp 6b4ig tcw2f9no5xm7stv qv6l4foeesgh","quiz_options":{"score":0},"id":"4385176972"},{"position":6,"visible":true,"text":"jfxlbnonc tee9khi75t0ah wr83dgnnsc0l pvyf5t266eq1ev","quiz_options":{"score":0},"id":"4385176973"},{"position":7,"visible":true,"text":"rv7aly5o19 y3hvj5byk uojji58u9thv w8dnv stba3pan 5yho9m1f3o097n7","quiz_options":{"score":0},"id":"4385176974"},{"position":8,"visible":true,"text":"0kmmyvxkq0ixg wmm2bnydk3xg nf2e5e3fn4e 0jd59 tu9bib d6i9t6 3ikku8yd42gnt 0uusc9kip2w","quiz_options":{"score":0},"id":"4385176975"}]},"page_id":"168831467"},"emitted_at":1674149694695} +{"stream":"survey_questions","data":{"id":"667462099","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"952l4noismh4 jwlr6yhgbpo 9waador5oikm xg4aqk0w5o08j 7bg3d02bw22df 84xdy6eq34snqhk h4ngqp6rpih4 9jevsisqvxcv"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462099","answers":{"choices":[{"position":1,"visible":true,"text":"lknhasa3eu03sf p1uflsed0h3sr q1b0gmga v8swv5rbv","quiz_options":{"score":0},"id":"4385176976"},{"position":2,"visible":true,"text":"u6hdwwqb4cc84s8 8jo8vmqx7 klsxteic7f 9m3mg59ov s7npsi1 89i5dq8chb 919k2r7u0t","quiz_options":{"score":0},"id":"4385176977"},{"position":3,"visible":true,"text":"xjpll0bpay536 2ex024o9rd 7boywqww rktvkxinf9fmoh 7d9cb06es933e jm2hsgge2cjy5jj 1ecnm8shkp a3flgyb gcf1622ijpsj ydnun9wos44","quiz_options":{"score":0},"id":"4385176978"},{"position":4,"visible":true,"text":"n7q0ogrerpn8dl qmip86lbe sl1xjutnlap7 3bov7 5fihf81ek49s 9p0f4 r1rptp","quiz_options":{"score":0},"id":"4385176979"},{"position":5,"visible":true,"text":"kwdbb5 hi349arw9 r2f0df eee097e k5035hpewmi6 ky0f9 kh7oio2u0bilxue 5l0moimxob7fid oeno82n5rkplm6i","quiz_options":{"score":0},"id":"4385176980"},{"position":6,"visible":true,"text":"finhomjk 48ciiw7x1g2f c50x4 9kenej45r","quiz_options":{"score":0},"id":"4385176981"},{"position":7,"visible":true,"text":"e7ravpe7reoe5il 9kgwh794cl9x0w snp3wnpq1ryga p4vpk03q7","quiz_options":{"score":0},"id":"4385176982"},{"position":8,"visible":true,"text":"ao3rdw8o5r0xvy x07pd7op60cce ylfrtqpvu81e3u 3f7jtv448v mfoeywlt","quiz_options":{"score":0},"id":"4385176983"}]},"page_id":"168831467"},"emitted_at":1674149694695} +{"stream":"survey_questions","data":{"id":"667462100","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"onm04qqjiu e5hh4bm1nrm 4lql830cednt4y m4vd6n kk1rvup ghucvtc uw8xrjg6 u6giu1qxo oywqk5fc4elxs"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462100","answers":{"choices":[{"position":1,"visible":true,"text":"n66vr4n4vn w57cw3jovcl9 9xhjvp8 mkuivep2954e71r","quiz_options":{"score":0},"id":"4385176986"},{"position":2,"visible":true,"text":"dncqrcm vkp1wf2u0d9j 9o2d4ag7rwt827i wo23d8v14j1e3p 8wty8ixwwhnc6 huw4we49ag40 fruxk5p9080lxhb hjvf468vja8rcxr","quiz_options":{"score":0},"id":"4385176987"},{"position":3,"visible":true,"text":"54u3kd450tpm i9jjwlr8r3xbduw 3k1ic1tg4 4tijdweafs ypprmy7wcpxeg yvvc4l19sd5p gmleu45","quiz_options":{"score":0},"id":"4385176988"},{"position":4,"visible":true,"text":"a1hrtqesu1ph 88spqcx6hyo7 dt64okx798gal cy3tbwljajmrr","quiz_options":{"score":0},"id":"4385176989"},{"position":5,"visible":true,"text":"ehg03 uyx121pwsyo 7rhvgcdfy2st8p 7ahaiboegtn5kd iu8bkj3imm3eo8d k2s2gg4g 2ys9hfx 8vpui26opm6n vsf5rfq6tqi2 jwxoe4suo4","quiz_options":{"score":0},"id":"4385176990"},{"position":6,"visible":true,"text":"c7lxx90luvc6y ogvbopnx 12v4swfg lofg3gcj dq1r6s8ptp5qho h04pqe67tbcnno xhyh0rp62kqkb dvtuwyu u89ppkdl876m","quiz_options":{"score":0},"id":"4385176991"}]},"page_id":"168831467"},"emitted_at":1674149694696} +{"stream":"survey_questions","data":{"id":"667462102","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"bwurewbkcd12 acvusgq 3u00c224nnv iyfj31 i9qtkl9tolx2fjr"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462102","answers":{"choices":[{"position":1,"visible":true,"text":"m6ihyhdpxt1 0xf9391 y8wvcnlw tfvxn6m 0bswb5w5p4","quiz_options":{"score":0},"id":"4385177002"},{"position":2,"visible":true,"text":"2kdjxc93o enx72tbxki oj43409eg90kt1 5fbnj d8rvo5c7a7vrgf 3c66h1 guo3c x6uv8kl qy0ttl 8r8dgqope","quiz_options":{"score":0},"id":"4385177003"},{"position":3,"visible":true,"text":"h3292 dtp4cjf11njvbhh huvgv1 onie86 tvdr09jg2lvp","quiz_options":{"score":0},"id":"4385177004"},{"position":4,"visible":true,"text":"16ua2fvav 75q7b2odhv fbw0xrnfn 2t3ivdrphr agrq241x d447hwdy rxa3g9gwe23","quiz_options":{"score":0},"id":"4385177005"},{"position":5,"visible":true,"text":"1dl0nhl0bx 7v1hc ooiv3gogf nxecupy","quiz_options":{"score":0},"id":"4385177006"},{"position":6,"visible":true,"text":"97geexnx4ussry4 s2x8sduenrfgwq y4s8f5v e6nrx0st23b f1xq1ax2xwl6yl c1to792d3i l0aq9 92s0aix8l31n5 qnnn38brgah","quiz_options":{"score":0},"id":"4385177007"},{"position":7,"visible":true,"text":"cg3w94v47ecgrik hviug 5xub14 808gw4 g0j3vn 5y58dv7vn8r5 xg72mp91 2l4ubwdo1wrw0t6","quiz_options":{"score":0},"id":"4385177008"},{"position":8,"visible":true,"text":"t92n3lmvuxaf5o5 29jh4afc h2wvqr9xf1q4w 27xf5d9ij1on qvbuiu7 8oghx0dt3nr5 g1mw6f9y6 muu0hhili","quiz_options":{"score":0},"id":"4385177009"}]},"page_id":"168831467"},"emitted_at":1674149694697} +{"stream":"survey_questions","data":{"id":"667461468","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"fn1i25upg io6aniuf yrrrp8vt07fo19 48d5ol09146s8m ldy0ebojp819g"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461468","answers":{"choices":[{"position":1,"visible":true,"text":"t56ghqeix s9huggwiub4 7tqxa1m0h7kj5 yk8r2 0f65tvl3fnpo 0gebiihpxcovnht","quiz_options":{"score":0},"id":"4385173226"},{"position":2,"visible":true,"text":"5o75tse0c okpssn319qiklp 5uyby90fx7n slfgxco07i ejp0kn xmr7ghykkxbypc","quiz_options":{"score":0},"id":"4385173227"},{"position":3,"visible":true,"text":"qe6tkjiccorb lxdmlb30 jmh6j4d6p vces42b344ry gbgmssamx0tvcs 785dl brkbvpl3ctq4nj2 yjjy0hh9b4sr yq2v7lhwtje33x2 7pbqmqgul4gqg","quiz_options":{"score":0},"id":"4385173228"},{"position":4,"visible":true,"text":"lyn712e8uyrrar 0qvy855feo 5l3egg 6t5uin58qrj1hj","quiz_options":{"score":0},"id":"4385173229"}]},"page_id":"168831345"},"emitted_at":1674149694802} +{"stream":"survey_questions","data":{"id":"667461471","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"wcrsas 0v14j5i3e8eb2an ymydb45sqwsn g8j35s649o3rq g582wdp eoraxaf8dc7gf qtdw5a27iucmesj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461471","answers":{"choices":[{"position":1,"visible":true,"text":"nd18cy2gv u2bvpyki k0f38w8 qtlqut6ttsdyk83 pptqq84jmec6bfd a1aks3iy0i oykgg4gv0 busn0o8cc","quiz_options":{"score":0},"id":"4385173236"},{"position":2,"visible":true,"text":"mbd0ll7jes0qm 02w87s60 jslfxkj7yeh1 ol4yj 405yk1b kddsvxnk4hsw7 v1t7q9h8k26rt","quiz_options":{"score":0},"id":"4385173237"},{"position":3,"visible":true,"text":"jl49hsj6tjdf aq1njo56no2 f2gw9 eyqqekovg0c7ov kwwlc9b77 i4yu6aa7uxu","quiz_options":{"score":0},"id":"4385173238"},{"position":4,"visible":true,"text":"5v78pcotkwvvlr 7gvcf3xsd1s2 1y6oc8df 3n3f0 d3cg8ihr3ox9qdc rnyw7g 0lv7p6bka ia0p1tqjcew","quiz_options":{"score":0},"id":"4385173239"},{"position":5,"visible":true,"text":"xtha0tjj1 1j0vri su2xvilf9734 te51yhl03q558lc","quiz_options":{"score":0},"id":"4385173240"},{"position":6,"visible":true,"text":"w4st1 gljqfop06 2bu4fs7b fiia7kj22xl0i4v dl88vjnwo3pf","quiz_options":{"score":0},"id":"4385173241"},{"position":7,"visible":true,"text":"ivfna3y9ru jhxob vuncvdavfek1dsd 80ha2m2 8raod4lyb40 3r895onujni4onh xm9h0g6di g8e735xi ax62ju3eihf9a84","quiz_options":{"score":0},"id":"4385173242"}]},"page_id":"168831345"},"emitted_at":1674149694803} +{"stream":"survey_questions","data":{"id":"667461473","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"xc4wjnko0i6 g7p33u41n 1tjsntksw58hn4 ewweiwcgt 58rs9ek5qi1bgvx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461473","answers":{"choices":[{"position":1,"visible":true,"text":"t09fsskd08sl88r 36sh46a2jva sd91wxy 5dncy1vabsf yfkliw o2spc5k ud2oktdabb7bv l76he8ba3y6 1veap0sd vh2aeoqb5","quiz_options":{"score":0},"id":"4385173256"},{"position":2,"visible":true,"text":"fvdf828g92n4cr4 jt9qg8529uihyge n69seyn8haq ido6m3","quiz_options":{"score":0},"id":"4385173257"},{"position":3,"visible":true,"text":"4n2rpc6iv82cene b7pvp 9fakf0 qmfrbf3","quiz_options":{"score":0},"id":"4385173258"}]},"page_id":"168831345"},"emitted_at":1674149694804} +{"stream":"survey_questions","data":{"id":"667461476","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"wojs6fb lri4g 9cd8kjrhy9rd3 j9s5e32o8i l82xpfkqgwhj jma6htdue e1kk3u071rcef 9ivvnisv asaqww"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461476","answers":{"choices":[{"position":1,"visible":true,"text":"j5lja4pu f0iq7pnkrk 9pra3hkah6abp 92wxt7ox6c dcukwklc k46geqe 2xk4g405p39t62 516tsgu xes5tkhjts","quiz_options":{"score":0},"id":"4385173268"},{"position":2,"visible":true,"text":"1l04q3mtxn75 t7ghyls2hqug j2hu42 uyoecoks6edsqns 57ya7vtyoyt u6k0gtnmf 1mm6wtektvp808 fkk89 qqprq k07oheboie1tfhs","quiz_options":{"score":0},"id":"4385173269"},{"position":3,"visible":true,"text":"9kvc37 gqyklsggws9 db070xge8y7fjr qmdym3ft1jp 5rh1txs","quiz_options":{"score":0},"id":"4385173270"},{"position":4,"visible":true,"text":"y9obh2bu8yk7e94 0ftyrjtcxhicd4 iobxi70wog4vf uokiplls1e ah0g8v mpsslhisijk lmomrov 07cfwh auqwaujyytig","quiz_options":{"score":0},"id":"4385173271"}]},"page_id":"168831345"},"emitted_at":1674149694805} +{"stream":"survey_questions","data":{"id":"667461498","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0tljucg0y3kh l2fh1q1i umeldoo665wgb 92c0k ldeevedeust7pg icrfpt7m1cq l6sfwhhk3vtql qpue297bcaeb7 yrn4gauy007n8q"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461498","answers":{"choices":[{"position":1,"visible":true,"text":"8lcirsh0a t4ydqb06w0pqv 54k3qmnrjdyf j3ebgxwih2m4wa calo6wrbt 88n96xatpw3 hxaq56r brlwuuymmm jc5psliso3joc aoa41qwdhm0inny","quiz_options":{"score":0},"id":"4385173412"},{"position":2,"visible":true,"text":"bxc3u7guumu w525yaov3om5 3n0ld61 bhlpnw e252afl6giiia 56ybyswiyivgqs s1svdd30gf9kqks 522nxgw3obgj","quiz_options":{"score":0},"id":"4385173413"},{"position":3,"visible":true,"text":"dc3t2djjjqkvmfi cu7osl pxtlaevr117irr ji3qcr","quiz_options":{"score":0},"id":"4385173414"},{"position":4,"visible":true,"text":"qqr9ksi gob32wx p01fp4yqobput39 fryunx r020wh2ctxg 7hxy9g8m9m nuqel75h vwr1mjecaopo1hi o9fc3o","quiz_options":{"score":0},"id":"4385173415"},{"position":5,"visible":true,"text":"cf6162qvdywm crcubxjk5m5ju 7vhy5q5kj4j2q 5b55n9h w9v6c9n r7m6xx","quiz_options":{"score":0},"id":"4385173416"}]},"page_id":"168831345"},"emitted_at":1674149694806} +{"stream":"survey_questions","data":{"id":"667461513","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"18i8lhabavlr 32wsccj5f rx8pvk4unsgm7r s5x7es2uepdb 96xgtjo7voy08d7 n4e9p fayts khd1nb8pow rpjp6"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461513","answers":{"choices":[{"position":1,"visible":true,"text":"b0wj1y3wel57 ktqj3mdnpi dhr0k r3tasjvw 5mx1wclhjj","quiz_options":{"score":0},"id":"4385173452"},{"position":2,"visible":true,"text":"qott2n8w37qaai 1qn102yos4 imupcls4nm th4ob4","quiz_options":{"score":0},"id":"4385173453"},{"position":3,"visible":true,"text":"9uksb 862xwr8j1 6vxdyc4 goi32thkhq93 helvbkt266nnrw xvit2acd67olt pvvkx","quiz_options":{"score":0},"id":"4385173454"},{"position":4,"visible":true,"text":"ees3ivm7fphpd 9jj95pohh4 8sc972k8tyxk ge9j787 nxjkeee670jiec o36kcvbluxrp03 mb4e0et0qwqjxyc d13juonhq","quiz_options":{"score":0},"id":"4385173455"},{"position":5,"visible":true,"text":"294dtkjuyn ttbf6ikcy7s 84hmb t17hw 82aljhi8fwnf65 usuwl5d7ytrca x52ehyic447miq isobapv 3gkr1 2wphdit0p","quiz_options":{"score":0},"id":"4385173456"},{"position":6,"visible":true,"text":"nc1kush2rbxkqrr rhbdvjy xn4au2o0o94 xjq5ll","quiz_options":{"score":0},"id":"4385173457"},{"position":7,"visible":true,"text":"mh1igjt m7gteqlf8 q87g5gj n1g1w7s80","quiz_options":{"score":0},"id":"4385173458"},{"position":8,"visible":true,"text":"sfnq7n g13n21dx5cy aky2s t6h2y8j 5rbdufdcu7i ji8jgrcanxhxv piyrbdr72031gh6 7nna7 gn4gfqo3o 9tbatn8r","quiz_options":{"score":0},"id":"4385173459"}]},"page_id":"168831352"},"emitted_at":1674149694807} +{"stream":"survey_questions","data":{"id":"667461516","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"7gnta43 5s7jfw8axcw3tkf vkg8bmasyo6u 91wwhy c1pae9eu342 3tg0jl70wuoc9 yyxo4v8se myu4ls9xoaiqsep"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461516","answers":{"choices":[{"position":1,"visible":true,"text":"qfjo9n 5a7yv16lp71 ow2bbt3n2ke 0jb1xfpxf51s1u f4k77hkwmu wprku4 55311lxm0wcpw0 c53g4v24","quiz_options":{"score":0},"id":"4385173473"},{"position":2,"visible":true,"text":"j4n9jfdyr4cgs gfc0c a8trns3c4m 5mem4dona y097q wxpm5 q4gqos2ukx ufjxck1lmi1nn1","quiz_options":{"score":0},"id":"4385173474"},{"position":3,"visible":true,"text":"4kb04jje8c8d m12jddaa1iljt fr4qqhj 579w6 6u57xdn gaduvgl0u3 887ffq2vq9b58 q4r02mfsjdx6 3v8yskph8uyu srpxeiuxm0ber3","quiz_options":{"score":0},"id":"4385173475"},{"position":4,"visible":true,"text":"biom7l20wjpvl la1rwpf51ia1ex 23gosg8xvae87 ev2iaj4bo c2581d1re2 93rl9 tmj3467ajulb","quiz_options":{"score":0},"id":"4385173476"},{"position":5,"visible":true,"text":"om88wcjqp6f 5fn54j nmdg9 1sriyr pt6p1h4cm vkdmgxanhh 96847e1mgp l8b2t jl8oxa ifs1c","quiz_options":{"score":0},"id":"4385173477"},{"position":6,"visible":true,"text":"l8bo1yes158 w294970vmw33c yp6x582 6sa834 j86xhb qfm2f2 tj7gt7g xkgtymut","quiz_options":{"score":0},"id":"4385173478"}]},"page_id":"168831352"},"emitted_at":1674149694808} +{"stream":"survey_questions","data":{"id":"667461517","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"uy1ifum6sjcimr he9dly7g it4v8exuhulo ffdod9aof90 9cdfum2phl11nin 5habw5hvuj9pqio ce6ftayekj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461517","answers":{"choices":[{"position":1,"visible":true,"text":"gd5221owwi1 tjya80m2al3k62t xq0sa9fp leyp52b5ahghpaa","quiz_options":{"score":0},"id":"4385173481"},{"position":2,"visible":true,"text":"lk1rk9 94xju3kfoc1w5 8dybd90 9qdq98 rxu5edq9gs99 kks85ifc03qye","quiz_options":{"score":0},"id":"4385173482"},{"position":3,"visible":true,"text":"ykpin7yh wv14vdjfwf0l 295s1en8cwa2 s6vafhu1fops 8k2vvkt9pftfc2 d1jxuoudjb 614hx251hr6fouy x3w49ql1 7b50ohc76","quiz_options":{"score":0},"id":"4385173483"},{"position":4,"visible":true,"text":"dewxv4aajhaw yij71ndhch8 76bjhg71csdc4w 5gn1l cmu7yo vdn7mtuy 6oplwmikjyvv1bi l3kw7j","quiz_options":{"score":0},"id":"4385173484"},{"position":5,"visible":true,"text":"kuoin09q0 lm3vqjykpb ji7lrp5ylkt58 qb0mhh86kcgm crjjoqq gvfejauprgj mlsjegwm 7psv8r4tc4365j8 98t96g2ifjm","quiz_options":{"score":0},"id":"4385173485"},{"position":6,"visible":true,"text":"5c8y8 f7droof ha5vfclhyivm6kl bfylm txvwxasx1j9l qpyifm4y qwhcsobdo 3ui2qts 97h9i1v5jv5g7","quiz_options":{"score":0},"id":"4385173486"}]},"page_id":"168831352"},"emitted_at":1674149694809} +{"stream":"survey_questions","data":{"id":"667461521","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"pgb7ltsb96ed dep0echkqixfc l4rcb230c wl14ohr2hre1o fn94g0r87nde"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461521","answers":{"choices":[{"position":1,"visible":true,"text":"55yon1b4lyavl 6v6trvxwrc8 s1hwd2 d4hrxv9tbga9e0","quiz_options":{"score":0},"id":"4385173492"},{"position":2,"visible":true,"text":"x8ak4mn2 601o7cb6qnhxox2 k8q23 24nb8q6tx diuko kre6p82em vrf8yxch2 53klbp1xtber s29v8xwbiqm4s8 trwpy6qd1464q","quiz_options":{"score":0},"id":"4385173493"},{"position":3,"visible":true,"text":"s1pxp wsngl9et r2o40 18ltd3 r1b8qpyqskyx a94qup","quiz_options":{"score":0},"id":"4385173494"}]},"page_id":"168831352"},"emitted_at":1674149694810} +{"stream":"survey_questions","data":{"id":"667461526","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"q859nk 369qqyw7scs77jo 9ji0c9vow2uv yjucgiioytvwm x0x95w 222dyntjt8pue 1ndsg06k"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461526","answers":{"choices":[{"position":1,"visible":true,"text":"tnea3634 p2lnkyut0h yllu3g pufn062o 2etbix7jtgiqx4","quiz_options":{"score":0},"id":"4385173522"},{"position":2,"visible":true,"text":"0qf4fyco1ea itos8brey3wgkp xje8mlffe4m8wlv a9b44 jgmtf","quiz_options":{"score":0},"id":"4385173523"},{"position":3,"visible":true,"text":"hc4a1wgbqqryji 4x7enmw1nam8o1o yn3nxtni y7bwpr wm1oww66f5ox","quiz_options":{"score":0},"id":"4385173524"},{"position":4,"visible":true,"text":"0giuwu7l3v 5ufp8 u3mupcfr0 r8977cguqc2 hrfeq0ug7wo0 fjmkook g8up66tqb7shgg","quiz_options":{"score":0},"id":"4385173525"},{"position":5,"visible":true,"text":"xp5qsik ipf6q be83j63io 5kw68q bwb1mxf gt4l3besb ikfckl9f f3lgigcoib t8wa3hjgn00 csln5ikv2a","quiz_options":{"score":0},"id":"4385173526"},{"position":6,"visible":true,"text":"ai0mef32co1ohl qfwbhctboxq89ye u5v2kyq ebo7e 2u7q9yvlgd1oc 8uxx84l65n7lf0","quiz_options":{"score":0},"id":"4385173527"}]},"page_id":"168831352"},"emitted_at":1674149694811} +{"stream":"survey_questions","data":{"id":"667461529","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5c2dy1fcqn80 athbhqoadncx3n v1lt9m7799 6s154slj6ga"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461529","answers":{"choices":[{"position":1,"visible":true,"text":"rqa77mh3qxuc 6c5uywhchx1bgd f6or0t6s ds7hiuwb3bu9 u9x2d8hs 52pocgqntfeejo4 oaryhk woq1xb7akj","quiz_options":{"score":0},"id":"4385173536"},{"position":2,"visible":true,"text":"hxex6h1upkfw81 pad50wjl 8fo1fiqvtqfe 3xg4xms66g6l luckdtn1 udk8r284o iwpvh21 xotxpvww6p29 4gmqucj4 my8vynnaemow89q","quiz_options":{"score":0},"id":"4385173537"},{"position":3,"visible":true,"text":"jyb1qcl8o2gtte jlwtdhkxh65a9e q6xfymirg7g 24hkqyd0x0 9w4wb4dostk 6i8s8mh5 rxmq5y8ti r99xgye0urvl bdq2598nnha o6gaqqdvg5q","quiz_options":{"score":0},"id":"4385173538"},{"position":4,"visible":true,"text":"qtgjiim109 xoslnipbfe v28m271knt yquk8m2j96w6 xff8kw1 1jhr5l14","quiz_options":{"score":0},"id":"4385173539"},{"position":5,"visible":true,"text":"me39vca xqytbsfb 8uiad g1elq77xy1 tv88etyc gj1g8580e1lbmg4 bjnj4es","quiz_options":{"score":0},"id":"4385173540"}]},"page_id":"168831358"},"emitted_at":1674149694919} +{"stream":"survey_questions","data":{"id":"667461530","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"v2whrhb82ev kvvqlgl ek50nx1ar9 a7h4fo9xuaynt2"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461530","answers":{"choices":[{"position":1,"visible":true,"text":"e8nxccvdcdm 6wfcehdl1 o7g5ce64yvs efob6e 9bj7ny6aipb 4dc2wsqwtr6u24 xsxa081k8 4fyis3oyl9t42 lmu0hhgj4ok24m6 fg0p0lt","quiz_options":{"score":0},"id":"4385173549"},{"position":2,"visible":true,"text":"a5cw9gn7fy95q 8ui6bhr1j0djvtl sr2o8a3 kojha7w 1itxl7twgq avmrl2q5jbwds9 gva3tqhpgk8y5 v4aylod k5ci4cngn6h44jv","quiz_options":{"score":0},"id":"4385173550"},{"position":3,"visible":true,"text":"s3s7w5clw y2uvtas6qhvbk qc1vwrlyi2bk goqfgfe1 1ayx9l73nb78dq a0ykwprnx8gxm1 xn695vlym9rt4mk","quiz_options":{"score":0},"id":"4385173551"},{"position":4,"visible":true,"text":"c4qwlt8a dgnsqhqut5cy iq1htk kitbw 5mwuqxv92w70s c8hxelnesk 75craldvy4w","quiz_options":{"score":0},"id":"4385173552"},{"position":5,"visible":true,"text":"qp2pgo2f63 wmlcuryftjwedj yq0xo a1n5ayjf jikmqnf4mdbw 02r05wsu14a146b","quiz_options":{"score":0},"id":"4385173553"},{"position":6,"visible":true,"text":"ootl4 kp8ae6ggbjabg hgusbwsla83 rnl7dnuqo","quiz_options":{"score":0},"id":"4385173554"}]},"page_id":"168831358"},"emitted_at":1674149694920} +{"stream":"survey_questions","data":{"id":"667461549","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"sn8gp1r0uc1og wobjh 6x0e6t4wug m6466nxyawm k81p20 883oi8u wgeeiqdcmxwa1rt mc0fnojb"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461549","answers":{"choices":[{"position":1,"visible":true,"text":"wy826 0uog090ox923 2m3arldq 9v6wr7xwvxfk lbxye5v0svm7 93ji5 vkfvxgr7iy 1r09yxnkpjm23 0bbktk3rfg0bscf","quiz_options":{"score":0},"id":"4385173656"},{"position":2,"visible":true,"text":"krqy1l9f dj0wppa p93gk rvffc v6cfo0 o1areevq8 jbhm4t gcc224tu 3thelsk2","quiz_options":{"score":0},"id":"4385173657"},{"position":3,"visible":true,"text":"f1cpwqowl4ju9 43sdy2 qdqyqw17lxw2v enee166iohhy kq5khb6s r3pucoufnsh hrhcpt q4ukten6scjuvg ldvapiy bg61mb0pf8ofd5i","quiz_options":{"score":0},"id":"4385173658"},{"position":4,"visible":true,"text":"eq2hnewfaec e9jxrip xrrkl1wh3rbcx7 23t2bp5535o2e qwjbhf5051mi iqgykvmdvsbqr 8s3wwn8tqc513c 4q3a2gr6uhdvlc 7dr6m","quiz_options":{"score":0},"id":"4385173659"},{"position":5,"visible":true,"text":"jth67q sto4nhh23s op593ly6lw4p1o gxwnyfr4m99nbvc dplgcarhd m24hy6i1d n28tg7mytetk","quiz_options":{"score":0},"id":"4385173660"},{"position":6,"visible":true,"text":"rlo3t 7ppem2 bx29gt ucqxmy0 jm67u2dynhq iqfpiyufxif 8lm6ydc9u16rj y7ombxa0meqr2","quiz_options":{"score":0},"id":"4385173661"},{"position":7,"visible":true,"text":"vijhvdm7x6h6 4swh9tom vcr06ugp1hl6 50al5 3ndajrt9ilers 2gnm8il0h ry9dtv jqpu16b0ukt qsb5rqrnmglh arno84j1kroy3","quiz_options":{"score":0},"id":"4385173662"},{"position":8,"visible":true,"text":"3ujkb4krosaln 4lk1ckhounns7qy ap07a65ruvnjed vygxbabompj8dn op8annyrush8v p6t0nd2qi w4pp18y4x lljwuc ubknhjf31m6l1tu","quiz_options":{"score":0},"id":"4385173663"}]},"page_id":"168831358"},"emitted_at":1674149694920} +{"stream":"survey_questions","data":{"id":"667461551","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"jf7fw46e14 vd9x8vs6f0av npyn7 ymxbaq24mqtvj bci1xcln9ch6302 eeghuyauh mjkw9hkx0oooduf 0ui0l93x 6m5tbpl yoplf8cqmbl"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461551","answers":{"choices":[{"position":1,"visible":true,"text":"rwk1x rtfy9upgpmj xkawlfe0vo8 3ounnnx3e9rl2d4 e8pqns3nwx pr5st9qy o6yi4klar9onlp innrgi2ua3f","quiz_options":{"score":0},"id":"4385173675"},{"position":2,"visible":true,"text":"pcupu8t3iodmw34 67g78ktu6rmr6nq 8nph4ohmv 9xtmtr2a7p9 6rb6h6","quiz_options":{"score":0},"id":"4385173676"},{"position":3,"visible":true,"text":"m6xlaj5xutmgw l08mgyjovq 6vk9sibk r9am0kbr 9qmm4d18mxnx u1sw4to 3a8ek ekp479980 hbuxaj2bf sbio7yw7","quiz_options":{"score":0},"id":"4385173677"}]},"page_id":"168831358"},"emitted_at":1674149694921} +{"stream":"survey_questions","data":{"id":"667461553","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"uue1b3 s4rj4 dawj4ao j5xhorntprj5p9s obfe7 y47vcjc s2gp2fh6 oysg9o5"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461553","answers":{"choices":[{"position":1,"visible":true,"text":"bf1i44 xxi5e2h7orevv2 d737wgpt4pu3 pmbeixrmvw1 ya3ww2x gpvmeo04nh5 sx2a7ey cv9ldq57qkus","quiz_options":{"score":0},"id":"4385173678"},{"position":2,"visible":true,"text":"ocmcxbjsqw o1vr3t 8fhpimedvui rn1hqm5m559bcq kw3138dxm77 vsdor8m v55p3f tn721 l4w4a6j3c","quiz_options":{"score":0},"id":"4385173679"},{"position":3,"visible":true,"text":"sme3nthgos 7c5bv0n80ymsa7 x02op26 jjum7w0s7lt2ll6","quiz_options":{"score":0},"id":"4385173680"},{"position":4,"visible":true,"text":"3p2mj4nlrsgu4d 8rygmmsqqxm2pl6 yshbvwq l5mcc5a kdpqvuul","quiz_options":{"score":0},"id":"4385173681"},{"position":5,"visible":true,"text":"o7ehnkg 134kywmgi wm8jcrpnl45if j86ffs6jfgcp 2e25qvayvwj18pp 3rduk lqjqyar2 xhq7yxb8i9ef6u","quiz_options":{"score":0},"id":"4385173682"},{"position":6,"visible":true,"text":"axh7312qg827nk s7tl78iwcr h1b6pv7hn6 o51sbag","quiz_options":{"score":0},"id":"4385173683"},{"position":7,"visible":true,"text":"onu50gtox9de40 v638eyavt886 mu6y85 6a54r1n9","quiz_options":{"score":0},"id":"4385173684"},{"position":8,"visible":true,"text":"0on3vhb3 94ujem74f4 0git5 fr73t6hds rfc4iyd4gwc","quiz_options":{"score":0},"id":"4385173685"}]},"page_id":"168831358"},"emitted_at":1674149694921} +{"stream":"survey_questions","data":{"id":"667461555","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"8nym349ifndc4 1ergxvcs rg8nxji7wngvfo ha1fx7qgh0p4m 3tkgf"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461555","answers":{"choices":[{"position":1,"visible":true,"text":"r7wm8q65rf0ed wj7kk61hjpqrt nkjjva6busq fuhp9098sc6 8ro2p6ww","quiz_options":{"score":0},"id":"4385173711"},{"position":2,"visible":true,"text":"wjxlbkjgaps2 sep73b5ia8yj5 a9wr0gq 9o2wqf7du1v0tl 81u2apxnt ww3588h","quiz_options":{"score":0},"id":"4385173712"},{"position":3,"visible":true,"text":"8r7tkvpljdu 8a667c88iw vtasj a5vacvo6pbxwsw 8in6kb nv2yeug1x 3lv9de7rw8rtq","quiz_options":{"score":0},"id":"4385173713"},{"position":4,"visible":true,"text":"obyau9nl8c9 jj7tk8n3wv1gb qmiuhuql 1m01qx26c213r6 aj6u9iw459er mxax2id3o8iv riondp2tea3 ay77ba gpl30307 440clb21coauy","quiz_options":{"score":0},"id":"4385173714"},{"position":5,"visible":true,"text":"0wm0f2w wykr4 9lcofhvrxdiwlp0 u8yf3 p1bwwa","quiz_options":{"score":0},"id":"4385173715"}]},"page_id":"168831365"},"emitted_at":1674149694922} +{"stream":"survey_questions","data":{"id":"667461558","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0wjijbi1kwvxt h19om9 kjvx4 jqeprmna15i bwqfaihmlk0"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461558","answers":{"choices":[{"position":1,"visible":true,"text":"q7y7hkgfgi1plk s1nwdd 0d7dt14f q604qs 47cwawfbo m1mmdsyi3fsp3l r0qma8q f5buoh","quiz_options":{"score":0},"id":"4385173727"},{"position":2,"visible":true,"text":"sn5hskkfn0ykw if30ikcbmfjsxj g6u5fay eauotfitd2 9cg6qkon 9t0l88ops7633","quiz_options":{"score":0},"id":"4385173728"},{"position":3,"visible":true,"text":"c678ymx 92bxd icea1i4p cjdl771k u37cuw rnjvrihdi89s2x a1vef","quiz_options":{"score":0},"id":"4385173729"},{"position":4,"visible":true,"text":"uqlmm5yk5 1v12y59qw hv82bo91a0leef 111gmwav8pnwe np5mfx6nsq6 cxgilxkmtvmm2kw 8mb5wiet5q wdbyo993 uh7kcfhy59 sebg3lik","quiz_options":{"score":0},"id":"4385173730"},{"position":5,"visible":true,"text":"uq2uqj20gchd ly2k4d4goq1 ehe66mwx ei0a0d4ggv0al9a 744s2h u254g40o5m 0i9o7dqjdkrtys","quiz_options":{"score":0},"id":"4385173731"},{"position":6,"visible":true,"text":"nyy288shht 3eibqfcm69se94 de2u01w0k5o 342ouq sb68ogkspj61j3p blbhcqok 8vdd2u7b2 su21c5iwou qgi1dj","quiz_options":{"score":0},"id":"4385173732"},{"position":7,"visible":true,"text":"fkidasstba rfn3erxiv282 vy0fqx 3acgtlyka1 mykyngnc 8u2kws4tp 35wst9aglv2 chbms5wm974 ajaygp8vjkhq","quiz_options":{"score":0},"id":"4385173733"},{"position":8,"visible":true,"text":"lmp71nc3bt 6eic0k8kacvx 17ititl v9u25fh1x78 vgfpglqvf6j ajv15gaccdsa elo40oe8ht75c","quiz_options":{"score":0},"id":"4385173734"}]},"page_id":"168831365"},"emitted_at":1674149694922} +{"stream":"survey_questions","data":{"id":"667461561","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"su1ullpk4 t553g unva8lk6wfw7 ipvkkika"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461561","answers":{"choices":[{"position":1,"visible":true,"text":"j463gx5a 1n4fa1q94 x4yw9ack6xoh u5l2fmvwlvktyw7 kpgulj5y49q d03pdq 0bl58ibk39hpe g6m66xw y8ajm5qk2uyu 02pbmuywb7h9m","quiz_options":{"score":0},"id":"4385173755"},{"position":2,"visible":true,"text":"4w8kyesurm3674 est71dh9qi23fw jbowojvmon40p jeggkq5soym 4dgqd70 6fx65rd2f94b5t lnsm1pjg0bypfv jt7d9jj","quiz_options":{"score":0},"id":"4385173756"},{"position":3,"visible":true,"text":"atun8p07f006myd wotd669048pp4 j2wa6v97pbj d7uvpqvrv7omvxw p8ef7giw88ft04 rgn3uwqgx08 rhe5yu0hg16b tdaflfh 2jpl4e","quiz_options":{"score":0},"id":"4385173757"}]},"page_id":"168831365"},"emitted_at":1674149694923} +{"stream":"survey_questions","data":{"id":"667461580","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"in0q9d6e580p28o 65hh3n3rbth4000 jowj89wgvpem9u lc918vs7s pt2r8mt n5t61mrm1nw i64uqjqjxw jhpgayjd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461580","answers":{"choices":[{"position":1,"visible":true,"text":"d1k3kg doxgp648opvyvwx o6xwbacsy2nvo oglkc2p5fegn l7cfksxmkd3ekk es7rvcb8t8v243","quiz_options":{"score":0},"id":"4385174059"},{"position":2,"visible":true,"text":"tc374t uo4de jkgh56fi 7ttvi102i5fbjw 78i6rml o5gekwgh5r rtxw6mrnqi9","quiz_options":{"score":0},"id":"4385174060"},{"position":3,"visible":true,"text":"mfwrrhume d5c5kcy7n32d ovt2xo d3cm1 shv19 r339qbp","quiz_options":{"score":0},"id":"4385174061"},{"position":4,"visible":true,"text":"l4pnl52o9gc 6q4ufrpc7q bw16kqt 6bagaclfl21f9 x92gg7slb2h gs7qxwmj6mg o81ntc46r8q3 3tkbuhrp0","quiz_options":{"score":0},"id":"4385174062"},{"position":5,"visible":true,"text":"9nbdwy6f06wn g2tdvcsqsg 74jmcd ulao4 i4f5y r3yn39fp vsxqmxocm59s10d 9l98r8lp7554 eek3weuw","quiz_options":{"score":0},"id":"4385174063"},{"position":6,"visible":true,"text":"l7am7i8hxiy u3ylucwt8lvms bl5ftdk19mxheie gt5x6 4eiq6 7an0oa731ay2p a9gyy1qsjse xif4ton3b hu5v6cg kuasc8ce9ihbjxi","quiz_options":{"score":0},"id":"4385174064"},{"position":7,"visible":true,"text":"iw5t0k3fr7tv454 lmsbkhfkfx6 vq5ds3yhq20b2 jom26vaad fu7w8 f0t9nanj2","quiz_options":{"score":0},"id":"4385174065"},{"position":8,"visible":true,"text":"18u6s9r5 na05j55lqd i6its t96py9n9q h03ueyjkfewy rhiv9a8nxh 3adtql0ksc9l2s","quiz_options":{"score":0},"id":"4385174066"}]},"page_id":"168831365"},"emitted_at":1674149694923} +{"stream":"survey_questions","data":{"id":"667461598","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"fg2ok1k5x73j xi2i2leb9xip0jl nbjab ut48risbmg0 cmxjg1y14a b350n2yl90sq r2fpg 9u3fc44vd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461598","answers":{"choices":[{"position":1,"visible":true,"text":"6ia2cfy435ajc4 qwja8 186ntjx 8jshqf5 i32esbjh08 yhf12enewt5 8shld76x18d18 qw01jndxbuhj50 4mtgy6fbn7nfbs","quiz_options":{"score":0},"id":"4385174182"},{"position":2,"visible":true,"text":"e5phn4tqfq 5xee0uu5x v4lw52 hkhk8lpakvb 304yi j23bhqyat9h5c j52nrstj5cqo3q5","quiz_options":{"score":0},"id":"4385174183"},{"position":3,"visible":true,"text":"swapu0ru 89dqkvum0e2 ybf7xmg d0atxxfgpb3ag ylnjkwaufdj8 rj6dedrfte xkc6hqtimwg","quiz_options":{"score":0},"id":"4385174184"},{"position":4,"visible":true,"text":"n4l9n4wq5yas blbdmjmr nkhx7l gh95jmffy0p69j 8kg39ic3a05l41 dmjd7","quiz_options":{"score":0},"id":"4385174185"},{"position":5,"visible":true,"text":"du734l u98n1d5ovgxe 1i94woho8w2k u16an6dw ha0sc8v11aeep 1p9pawo5f6v 9s6ysd nv3v3","quiz_options":{"score":0},"id":"4385174186"}]},"page_id":"168831365"},"emitted_at":1674149694924} +{"stream":"survey_questions","data":{"id":"667461606","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"f9jgr myr928nokvyw5jg fqoan6w aqdnfas0gapwd 9hobof x7d7vy7xd9il kh6cd dteve1vnaoq6o uccii826ldkj1c"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461606","answers":{"choices":[{"position":1,"visible":true,"text":"1ak71apsl1lb2 dtgtym8ftk 52syq wtnfautgqxev t2sp9icu3465pll i4rhctmtx rh5yifue1 k91tlt","quiz_options":{"score":0},"id":"4385174252"},{"position":2,"visible":true,"text":"3t2uxb my2rtujx8njmbtm a2cyspuk yvfub6ofm8a9 819o7h8o eac0xea","quiz_options":{"score":0},"id":"4385174253"},{"position":3,"visible":true,"text":"4t3bj1jwi5bo ofhe7 cg6nbcys7lu v2qv7q11u48 7oowwv12fndqe","quiz_options":{"score":0},"id":"4385174254"},{"position":4,"visible":true,"text":"7etbg4thd4k k6d0nlv 02cobf bby0p2i","quiz_options":{"score":0},"id":"4385174255"}]},"page_id":"168831382"},"emitted_at":1674149695048} +{"stream":"survey_questions","data":{"id":"667461628","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"fx8h961tahe na1oufba ybjyqi qm6girwad8b1xq"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461628","answers":{"choices":[{"position":1,"visible":true,"text":"lakrew7c6qd wsih7a2nkq0no4 94gtxvs36 87wp1kbadg1l 1hkmpi6 2g1juofi5a fmmygrjehmcuygi 0bs6ka 2pj0gvmjpd6b","quiz_options":{"score":0},"id":"4385174360"},{"position":2,"visible":true,"text":"jjsfl l1sxtcih8fyeeof 6m9lk94qqmuaegl ki7wf11a9i93 xypqh9bjq1av7d1","quiz_options":{"score":0},"id":"4385174361"},{"position":3,"visible":true,"text":"eb4np f3xpqwjcv ypljmiu0y337 lu913m i9uet4t 6dvp7afcy3uqh 1xiu25 1qqn7cyv","quiz_options":{"score":0},"id":"4385174362"}]},"page_id":"168831382"},"emitted_at":1674149695048} +{"stream":"survey_questions","data":{"id":"667461630","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"576vlo18en3 1bxycqddfhn6v rsbd7a1e4bbyf ugieqgvywq4 g3qltn1jy8 0ebdn 78lxt9iwu1jx rbdjre1cujgtsq"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461630","answers":{"choices":[{"position":1,"visible":true,"text":"5k5p528j2o b9frywomc0 aukxrjdqt wiuvud9asln2f3w 6a4mdsa22mygo bnwh1sv 5l5s9ae rbo5hecsrd w0x40 6a86lloan5","quiz_options":{"score":0},"id":"4385174370"},{"position":2,"visible":true,"text":"o8jtnh4keupcl6 ff77c s862cc89 avdo6n25pw","quiz_options":{"score":0},"id":"4385174371"},{"position":3,"visible":true,"text":"vq7kbm97 p8svkwom2lo dg91j0 i570c 7gve09y 9xn2svsiu 22oh8ub7crysocm","quiz_options":{"score":0},"id":"4385174372"}]},"page_id":"168831382"},"emitted_at":1674149695049} +{"stream":"survey_questions","data":{"id":"667461651","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"799t8 uw9o03 89r6ndohi8r8bvs gv887i 14be70qd88w qgux0yh3"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461651","answers":{"choices":[{"position":1,"visible":true,"text":"dtvq3u jyp1h165k86n y0rkc4ich02nw qshv8da ppf7j9x raeq9k abqfxgd","quiz_options":{"score":0},"id":"4385174481"},{"position":2,"visible":true,"text":"i2muubbwbuf 3ffi5361bv wdoq2xkw2hsgpkj t45p9rr5mhi 8xwetg9 gc9u9p4hpmkkev do1ik5y 1439x s9k4vbcj4uaay8","quiz_options":{"score":0},"id":"4385174482"},{"position":3,"visible":true,"text":"0bwq2juhggn68 d2ioj rs8ybqs07gxkcri pgqiiha ir4cb94rbh71q u5mssukw nsyithmc5ismqc fyjxgqjx48fsdp dcua4s9spewq63","quiz_options":{"score":0},"id":"4385174483"},{"position":4,"visible":true,"text":"t3298cfyqx7toco rvkbw4338qk9 k5vd0u42if tkkkkm hfxufgj1i3f7vm8 69yd8hxcl5 swv8ipasty 4g6qsfyb0hdpa mx6s5tmxs","quiz_options":{"score":0},"id":"4385174484"},{"position":5,"visible":true,"text":"o7itx tcbcw8snyw 6jmor7biu7 x59bcw2hwr t94tgt axflfvftpmd0h 46td0yk 67ydkpti6t m3v2de7","quiz_options":{"score":0},"id":"4385174485"},{"position":6,"visible":true,"text":"u7s76bgo3nsbob i7yi958x br6wqw77f3mv b4f5yubeq q7ubaeju1t x5t0cx1bc25jhy acvms2 pggajefpr6 c7prvy9 6ain0uox3g3rd95","quiz_options":{"score":0},"id":"4385174486"}]},"page_id":"168831382"},"emitted_at":1674149695050} +{"stream":"survey_questions","data":{"id":"667461652","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ske92t16t19co1u 3qou5xj75ksuy 1rfxe q5tbfl9xa6dx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461652","answers":{"choices":[{"position":1,"visible":true,"text":"gn0e03jdsij ops28 wxpn34 o9q6o9otb uxw5rwr5xh g855ayu37wsg","quiz_options":{"score":0},"id":"4385174493"},{"position":2,"visible":true,"text":"6yft8d7xv vwn2h vrlpr8r7ppbnfr vtvgbje916jf5 kv7g2ax 292bgj62fdk","quiz_options":{"score":0},"id":"4385174494"},{"position":3,"visible":true,"text":"iicugnyhyj 0t742iccrlgnc olaw2v e67g2xd1etgcq","quiz_options":{"score":0},"id":"4385174495"},{"position":4,"visible":true,"text":"55ganyc qk6vvo 1gfswk294 t4qxy0nirv3a y1wr6vmvd","quiz_options":{"score":0},"id":"4385174496"},{"position":5,"visible":true,"text":"5hjxv2qf8jr svkm4b90cs6n7u 44ewbk0getyk g6hhmaqvlcdj9g7 if6lqdknx rr5abg5ylw7ho2f rabuvef2t0k5x 0a7x5oqof9xhi8 o8tv6ke3ev7r","quiz_options":{"score":0},"id":"4385174497"},{"position":6,"visible":true,"text":"sy4uf cxu4rsabrlmaw kdov7 hx6ta8d4c0x290","quiz_options":{"score":0},"id":"4385174498"}]},"page_id":"168831382"},"emitted_at":1674149695051} +{"stream":"survey_questions","data":{"id":"667461666","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ssubl 48hjj9vj31v omr18dpic pp65f94 j35xxoh u04pidx8dp7i bo1ae4ptsntu3 o6v99q"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461666","answers":{"choices":[{"position":1,"visible":true,"text":"04s0qv msoghfkw0fr4hmq kdv92tuo2 l8htirpt 4h2nxjcla5rjor n1wy2w5","quiz_options":{"score":0},"id":"4385174598"},{"position":2,"visible":true,"text":"64o6m jya4p7twvvs ls75jv9lvf 8gr7y35au5hcqfq","quiz_options":{"score":0},"id":"4385174599"},{"position":3,"visible":true,"text":"x6ijo4tj5qrljkh dk5dmknmhn 3rm63 kdym4 bxjsru9kvh1 g2vp966a8nkh 6dhh6k99a88gt 9b7emois0ldfr swbnqor4k66","quiz_options":{"score":0},"id":"4385174600"},{"position":4,"visible":true,"text":"4uhchfo7n2wmrtx e211emk0v53w a7ckw5lg40n qrx8pw0r5xrph q8ndfdm5g08tex8","quiz_options":{"score":0},"id":"4385174601"}]},"page_id":"168831388"},"emitted_at":1674149695052} +{"stream":"survey_questions","data":{"id":"667461670","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"lbje0ppud4so8xk ax7vnbef0l3fv7j alcnsi48jehkf8c 9fghb3uop 8qfnul9 w8d6lma9 ejcm4j9x604p t7b7dk"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461670","answers":{"choices":[{"position":1,"visible":true,"text":"i6exa76ck98 9pkrhl37j88s 8k6dvdhxg5axw3q 0dtn8xt6 ji1gm0qu 9pie3iotb","quiz_options":{"score":0},"id":"4385174620"},{"position":2,"visible":true,"text":"knjsfkea66j bp5sxiba qa1yvg m3vul 96hgbjtin7ux7","quiz_options":{"score":0},"id":"4385174621"},{"position":3,"visible":true,"text":"jf90cc rc80q71mvp1 8f3qlcc4pls wvixh60l7b","quiz_options":{"score":0},"id":"4385174622"},{"position":4,"visible":true,"text":"u8275ses4k sq0bax8 vbpfs7qwh yhbborapj3ai t6eo1n6o9rf uitrqawsi p2oe4x5ie","quiz_options":{"score":0},"id":"4385174623"}]},"page_id":"168831388"},"emitted_at":1674149695053} +{"stream":"survey_questions","data":{"id":"667461674","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"7xuudm hd85d2fbd49 hxo4gmsnhaxn53y r3oo0piiij5 hvifmm5 mfep196v1yi q9o6w9gksyedtgs o3y01cyw4ca91"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461674","answers":{"choices":[{"position":1,"visible":true,"text":"7h3hoeucihjd41o 4la76h8i8 8dgl2v5eub 962jah6wws8 qdtm85x9t3y96w5 mkq9yvj 81nsju0 koqkfhpe7pw","quiz_options":{"score":0},"id":"4385174634"},{"position":2,"visible":true,"text":"vpmf8jyppptscjq 3ujxkee7dye ggqm9tg9svjgx ashilo1ffin","quiz_options":{"score":0},"id":"4385174635"},{"position":3,"visible":true,"text":"nselknrgymf vq2vd1efu 3a9b7whs2k7bj5 nujxdbcbg8qcjp 2va7wdag r0k00wo6 bd5u5 btk0p","quiz_options":{"score":0},"id":"4385174636"}]},"page_id":"168831388"},"emitted_at":1674149695053} +{"stream":"survey_questions","data":{"id":"667461676","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0g353xw0tmg9uj yjpp4gp6d kkvtb90xqw7se5s 1rtjhd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461676","answers":{"choices":[{"position":1,"visible":true,"text":"qi3ml trfm3r0 hij01mx6lt6 olgi265lrg7a45","quiz_options":{"score":0},"id":"4385174643"},{"position":2,"visible":true,"text":"sip0lix7kpf g1vjmce6cj u3qgbnitdp2w3 gxgg5cc 53rn81g6mr 008agselt2cfwmo","quiz_options":{"score":0},"id":"4385174644"},{"position":3,"visible":true,"text":"29qp4n5j92gre bwkmqp821xab2ri kyqdft g46vvp19jlvq1k 2sgm90q2b6 1ewvm 8491gfakn3p esqiug2cxmg6ka gb9cdrtlc9qs","quiz_options":{"score":0},"id":"4385174645"},{"position":4,"visible":true,"text":"mcijdsetfwvx 9b6tmxv phlw18ap37 20s5wv","quiz_options":{"score":0},"id":"4385174646"},{"position":5,"visible":true,"text":"3g748xwkq6o 2ut8g nb63dyj stbl5wm 2dkk61h6 29bjrer dxet33 v6f36uqes9qu vgqoxu1nmi5lm gtuqaph","quiz_options":{"score":0},"id":"4385174647"},{"position":6,"visible":true,"text":"j5p31 t1ltv5rpt 85q5tlwq2sfsv3l bh4a2e rev7lbj7 2qwf83ylrc3n fiufpgf h3fp2fd0y5o uw3phwkrek ihy0faqy9s1rsp","quiz_options":{"score":0},"id":"4385174648"},{"position":7,"visible":true,"text":"prd59 gqs7t8tmls 633ujsd7d9ni u1fkx5a6mx318w k71rc82lydu43 orkxf utbtip9c1ky72","quiz_options":{"score":0},"id":"4385174649"},{"position":8,"visible":true,"text":"dc5yl63yj6lv jqeok71lte 4leao9ms1xu3oh 0fdb4m50ip t31tl6q57jjpmh v2osljafdca1x6","quiz_options":{"score":0},"id":"4385174650"}]},"page_id":"168831388"},"emitted_at":1674149695054} +{"stream":"survey_questions","data":{"id":"667461686","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"987ob 7kiuo4teh1 8s7deqmvpopsp txo4fxjj5e 8eqdafh1k7yjhwf 889ll 0s23juj5vyyq99g 7390ddsh565 lpni4hn1bi0"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461686","answers":{"choices":[{"position":1,"visible":true,"text":"1vdqk14 sjvrskliu ejn42p38f ougj5qtxi6k5q 6d2ccy9073i9","quiz_options":{"score":0},"id":"4385174676"},{"position":2,"visible":true,"text":"b0yc27 c3v3i nhneas7k4y dqth374wqfs9hqf eg6g53l03g 8fg7266kcffpx7 6i9d7rffvncy s0f2on","quiz_options":{"score":0},"id":"4385174677"},{"position":3,"visible":true,"text":"k2kv88vnieg5pk fxdpcorngy bmidk g2lb58 qkbeew82lmprw","quiz_options":{"score":0},"id":"4385174678"}]},"page_id":"168831388"},"emitted_at":1674149695055} +{"stream":"survey_questions","data":{"id":"667455128","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"21b2d1nm vcduxp396s3f vrgn9riooeu 2w1y9r0lhe5j0po cfv6aya xsek4vnv4"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455128","answers":{"choices":[{"position":1,"visible":true,"text":"1cqki1ijt cnkrfpewyh hloep tun23aq fm2hlturu wp1hn1005","quiz_options":{"score":0},"id":"4385137324"},{"position":2,"visible":true,"text":"8q54jh807g67 0chem4m32w 6g6cg4kfbpyp48 wgbo0l1 3ivc27 h1g0jagebgvj36d loo7agxubx imfmbchanx4w47p o4rfv9","quiz_options":{"score":0},"id":"4385137325"},{"position":3,"visible":true,"text":"axjh0ev2g5g0s01 d1f1ekg41aq 1r7fcho4 adaclkcgj0ra1h5 obp9ot rak96hmu bx3804jjlr5vg wruubenes3g4p h41e18dufrs40kb 9t34k1uad6g2a0y","quiz_options":{"score":0},"id":"4385137326"},{"position":4,"visible":true,"text":"0rvnqa934hd356 yx0p7kab01kubvj yuy2tik0o6t 3854yf6up2mgfc kqn54h6qvfoj jecuuxm0d9s u5d8y0hmv5bcww eumwsigjumvc543","quiz_options":{"score":0},"id":"4385137327"},{"position":5,"visible":true,"text":"sutsvkyg7 x3pr1f8 gnn8yg 32ls0it6m8c5c3 5xhh23hb0bck c5kbcphdgm 8n1839 8oe5f4wdsxv oyijcnki143a","quiz_options":{"score":0},"id":"4385137328"},{"position":6,"visible":true,"text":"a14e7gtqqbrm ei6h8ynhoi92vs dq0oj6m rulpoxmdtew b1cnc7v 8pxkgtbq6vu0o","quiz_options":{"score":0},"id":"4385137329"}]},"page_id":"168830050"},"emitted_at":1674149695180} +{"stream":"survey_questions","data":{"id":"667455130","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"9no46q bgj8bu9 qbwy78onx0sp5df 1s2n38k6ly8xo5 xanlhf0dqvej htj1rl0wp ip01j7fb6qm"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455130","answers":{"choices":[{"position":1,"visible":true,"text":"7dpysnial1 w484xmm33rpralf fsb2j19 r599iaw30wgp gk8tit5i681m 14lsb5gmddkr qrncp46 vp0lw srtca1i587dq","quiz_options":{"score":0},"id":"4385137340"},{"position":2,"visible":true,"text":"0yflf5ua3 5l5tu93l 45fu03hrvt kii5twmbs6n uqppc ca18nfpevvny 8od85fpdw gc9isqks5ou7","quiz_options":{"score":0},"id":"4385137341"},{"position":3,"visible":true,"text":"uuv580b3 ejgygxa3nxuod qx2s8cx85l1vu c96g3513rcjdmuv 161c5f mrclx05fg2o 9r8lk172","quiz_options":{"score":0},"id":"4385137342"},{"position":4,"visible":true,"text":"8jhq2vl1hj6 dvnvp6su wshwoa9 unbnfs wjkm9w4w5w glupwsmbjeb3 26q9svrti 21b4potlf4d","quiz_options":{"score":0},"id":"4385137343"},{"position":5,"visible":true,"text":"y1xu41qdgnk hw9yooajs6n xhaqwoc8 hxi6ioaqcnf3 le1ein6yt","quiz_options":{"score":0},"id":"4385137344"},{"position":6,"visible":true,"text":"y7vgshrq086w 18x5gdxg0 ibct5mdxqm78py aq9lw3kx kq3rfwnnod q4a8qk7q 2mytdfvtgxe q7w14xsgcfsw4","quiz_options":{"score":0},"id":"4385137345"}]},"page_id":"168830050"},"emitted_at":1674149695181} +{"stream":"survey_questions","data":{"id":"667455161","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"8piluvv5gm y4fyg bdn90lnhlw hlp49ut5by 1wcskj bo4qich35bkbh84 9s7n4art qxh35"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455161","answers":{"choices":[{"position":1,"visible":true,"text":"7v9pb77j2u5h8lq fjdckl44c6tdi oxtly8metbj35c7 2coi267okn 772jt3x4 8pdrkhod14i eeyu3ox bkl79rtci65 0jt9nxkpamu5","quiz_options":{"score":0},"id":"4385137468"},{"position":2,"visible":true,"text":"w13utk5t7u98 ltxgdnhqgh3 tv729vqrf2g8 a3kf9ym ddp3yd ryqfnhyyshm","quiz_options":{"score":0},"id":"4385137469"},{"position":3,"visible":true,"text":"4o86igx uap6ssaxn rej7oi49j4nj g1afe aj0rahjii gdy2ygm yrq0r33ljaqav","quiz_options":{"score":0},"id":"4385137470"},{"position":4,"visible":true,"text":"ywmratfmy ko10n3cmegap9f vor9gwdvwevpi2 7oid9t72lp yc37r5pssxrm d9fdyo6kj9g5","quiz_options":{"score":0},"id":"4385137471"},{"position":5,"visible":true,"text":"l1ei3wav57twf yn5mmi2almiidx7 gdkyhn 3npig9sip","quiz_options":{"score":0},"id":"4385137472"},{"position":6,"visible":true,"text":"rr84y ht7saym46 rqxpn8 qix3c","quiz_options":{"score":0},"id":"4385137473"},{"position":7,"visible":true,"text":"dtaeau8 wv5ay39bjtph6 uxvtv8 bfa13j2 t1cxi9sgxs9u wh5it","quiz_options":{"score":0},"id":"4385137474"}]},"page_id":"168830050"},"emitted_at":1674149695182} +{"stream":"survey_questions","data":{"id":"667455172","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"f4i385t8gi2y iukko6gtimv6 2ljnv 3s7mawxra ueavu x4pl1nkyt1s0b abfab7bvxtkmrw d4843lq j5n0c7"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455172","answers":{"choices":[{"position":1,"visible":true,"text":"5dkq8pnhf1gag8 u73xqg mej8lwdqo qs1tuihyc heabchr403srhq0 51c5ipel70a csfc3dwuxn9mt9m m9j6t37f xe547smil yy8ji7kgqx","quiz_options":{"score":0},"id":"4385137485"},{"position":2,"visible":true,"text":"q02a6wo56w 6fc33b k7d7a9n7grsgt n7j84b6552eyy58 xapgm1 xxxxklio 1m6q53gxh23a sl0kn09v kdw23flh1k9ss 1kfm8tao","quiz_options":{"score":0},"id":"4385137486"},{"position":3,"visible":true,"text":"d3482iajo1udye ian1i2sop8y5 6j6u9vi h2flbi3mhh 8bo3d4b7wnwto x7dlap9vcos","quiz_options":{"score":0},"id":"4385137487"},{"position":4,"visible":true,"text":"ytjtpqs9tcdnb 5pxact7wjjdtx filtao5oyv w02h42un1hxtd5h dbagrj511v8 d2k7wxlw","quiz_options":{"score":0},"id":"4385137488"}]},"page_id":"168830050"},"emitted_at":1674149695184} +{"stream":"survey_questions","data":{"id":"667455179","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"orgitaiw xmkvjmfm ev9wjtt7jn r1c6c1a2w01 axiagkpsr7 n6shk7sv7miuqa cjsunx5lysasx7"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455179","answers":{"choices":[{"position":1,"visible":true,"text":"v2fci0n3 q922d8hp kc1mka6c9 9hkt1vtlsv nx48ppjd kaisn r586rs 5awcm 1y8b5ew","quiz_options":{"score":0},"id":"4385137492"},{"position":2,"visible":true,"text":"fv4rhyxgmweyo1r g2yoh7nm88 ubgo22v 8e2ukol","quiz_options":{"score":0},"id":"4385137493"},{"position":3,"visible":true,"text":"6b1kwthm24uw054 42yns43d9d2 9aelxhek6m9cer 1scie9 ob3erxdvkvv 5bccusj 9pwd586","quiz_options":{"score":0},"id":"4385137494"},{"position":4,"visible":true,"text":"0x7t0 engoyj qcv0e ta9rgyjcob sqj1y247 2f6nqh0s4e9qtbb o3q661emk54yvlq am4wkqctn26fblr 4yd280s9dpbyq","quiz_options":{"score":0},"id":"4385137495"},{"position":5,"visible":true,"text":"1mm0gylkc8tpmj vpmvo0eme0 9kriqmlb dm300brkw7certk 096rh8ts1ll 4j1pr","quiz_options":{"score":0},"id":"4385137496"}]},"page_id":"168830050"},"emitted_at":1674149695185} +{"stream":"survey_questions","data":{"id":"667455202","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"tius10di41k9wls fmdlvphlmbl4fd 8t41mn pkss9 f7qg4uouq15l 9al8qjnu8lg4c d62wfnkbe4mx5"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455202","answers":{"choices":[{"position":1,"visible":true,"text":"pxw0w9ui9m4oc inicqftdpfe dyp4ai vn1pmsp6 6cm2gr9b 0ruklaf8 xhb0a8q","quiz_options":{"score":0},"id":"4385137700"},{"position":2,"visible":true,"text":"p28lapcj xmelobuak9wnfe k0ilacxb575 54eam 96ng7","quiz_options":{"score":0},"id":"4385137701"},{"position":3,"visible":true,"text":"gw913n4emtltob sqr3rxe9q alm4u96n5dbp 79stl7bky bseq17ndb ibhcv2mf06av","quiz_options":{"score":0},"id":"4385137702"}]},"page_id":"168830060"},"emitted_at":1674149695186} +{"stream":"survey_questions","data":{"id":"667455205","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"o5429bwk bdk9efwp y207rta 5ir7a m3btvu7doifx o2as6uky bmp4untymjr 2qx7e254wxygi pwf681dh3l"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455205","answers":{"choices":[{"position":1,"visible":true,"text":"ihqtbuwrc7tkk t8rxgekiec q546dffi6yo7ato ip17xm5fe","quiz_options":{"score":0},"id":"4385137740"},{"position":2,"visible":true,"text":"ithl3 gl2tfo1dl uuiy7ls5 7wkeslse","quiz_options":{"score":0},"id":"4385137741"},{"position":3,"visible":true,"text":"ul101be 6w0o3urllk5o4rc m6ttbhcts3nrpq cd93g3j6","quiz_options":{"score":0},"id":"4385137742"},{"position":4,"visible":true,"text":"keye6y0 vxjehf4oga975i hh0hwvp20y0 hmvp1i de9i0gf 309gv73vahw5tv6 b7th25myd dl56yk9tjsnwbkg nlwrih","quiz_options":{"score":0},"id":"4385137743"}]},"page_id":"168830060"},"emitted_at":1674149695187} +{"stream":"survey_questions","data":{"id":"667455210","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ccus8ci s30ote6pf5enhkv 3xw226wx0r4r soqp6i56rx7p"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455210","answers":{"choices":[{"position":1,"visible":true,"text":"ror4u04bm535i fajv10 ji7v3kn9 nu7pljg4qd vtkx1s","quiz_options":{"score":0},"id":"4385137775"},{"position":2,"visible":true,"text":"5xjy8qckk 6w20pov 9ductfhj r469e wem8w rc0jks9hwv","quiz_options":{"score":0},"id":"4385137776"},{"position":3,"visible":true,"text":"difesvxw9al ygl163e5w9dv6 x2glb8m5g6fxm eiibhejl","quiz_options":{"score":0},"id":"4385137777"},{"position":4,"visible":true,"text":"tnfsnewa16k155 klbxdolp it7hi 3bwcsq4kxfs6ag y4rjmghck8fb1 kbr64tikluwhtsh m6uwesc5861 tdfj8p qrf7oe2ydofs vy2xx8qsmcvubj","quiz_options":{"score":0},"id":"4385137778"},{"position":5,"visible":true,"text":"w4u167 jcxljoenbixyu2 xmxki pnigr00vdimxddq","quiz_options":{"score":0},"id":"4385137779"},{"position":6,"visible":true,"text":"s4ksxp4wgiv24 jd0h5q9cdqbaf2 at9rhpltsnm9e 9vvcvfubdxoda 3dr2s4l bcvpvk5qq","quiz_options":{"score":0},"id":"4385137780"},{"position":7,"visible":true,"text":"p0nngm2a15gos i48053r8tp2si 5c356fdw 7lwo3016oo2u ysdomx7ts utp4qqa6","quiz_options":{"score":0},"id":"4385137781"},{"position":8,"visible":true,"text":"hjopydw3jidvpdk opbjs2sr 86c6g94l 1045p9imdm h0ewun d0ki3w4t li53wdcc9 v0m8nd0q8rim c9kuoh 986j08boiew4s","quiz_options":{"score":0},"id":"4385137782"}]},"page_id":"168830060"},"emitted_at":1674149695188} +{"stream":"survey_questions","data":{"id":"667455212","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ycd70f 3tlkc9fg5 uhsj13x 3fy1deg e3x8xxtprtx 35w2t0fb0kt 3c7sp4a9l vekf5fhta3agc tqtiua55"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455212","answers":{"choices":[{"position":1,"visible":true,"text":"3nd2k 12iqy qcy1q78egxti b88f20djp60v 6orn1h44 xb2fsut8p63fd spd9rybvtlrx6 p79lf27l","quiz_options":{"score":0},"id":"4385137792"},{"position":2,"visible":true,"text":"w6va3wm5af u5earys3j7d2w0 aigku 8mhkw6l24gftb lj78ittnu2lhxct","quiz_options":{"score":0},"id":"4385137793"},{"position":3,"visible":true,"text":"1vjpoerlc8kf lnqycklqdoe yt06t24brl aoqrq","quiz_options":{"score":0},"id":"4385137794"},{"position":4,"visible":true,"text":"syf9d5k3axfjhqx f1un5ergu68m7h hf5by9f1 0qf94 docxia9h qcda6u ja83rjw7gdm","quiz_options":{"score":0},"id":"4385137795"},{"position":5,"visible":true,"text":"al4dbwow9is7xut pif50a9 434fsto nh16xfiu34c0eld beshoy","quiz_options":{"score":0},"id":"4385137796"},{"position":6,"visible":true,"text":"q9ufu06kh4d 4oogck630yox 7wsoh 0l2dsb3 8noi1cwam8ukth","quiz_options":{"score":0},"id":"4385137797"},{"position":7,"visible":true,"text":"3e5ijt o0m3chiw3pitxr8 4hciaiuh9c gpc5q4olp3cib 0bhsd3payjog 562gi7o4647qe 39j8aa4ptw jbydrqc7ujb iqlxnqn4uea","quiz_options":{"score":0},"id":"4385137798"}]},"page_id":"168830060"},"emitted_at":1674149695189} +{"stream":"survey_questions","data":{"id":"667455215","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0qm15jfh tv1ui2w edoha95n wbhjp4 rui3rtv6xf4n5 v920f2a1hrd 5d7gj7guq r7ljcjdk2f8t jv5iocm9mv7pg cmow0h6"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455215","answers":{"choices":[{"position":1,"visible":true,"text":"sewil7695q3 nu4xd9w cr1qqpg4h9qlr ktchj5oeb4 1ns6cdqbjex5f w2y8iflb52 kpkid 1f8unrp e17xh5 qn47cr","quiz_options":{"score":0},"id":"4385137810"},{"position":2,"visible":true,"text":"q21t3nuf 71sfp kag1g9kes bx74gjdj1 uvk0chaofja60","quiz_options":{"score":0},"id":"4385137811"},{"position":3,"visible":true,"text":"nkbskl7xaxqbh76 pgggq5trhj3t isut5qlmwmxnbw apxr23h4v0l","quiz_options":{"score":0},"id":"4385137812"},{"position":4,"visible":true,"text":"jotjdok64gv ya4g5j b0w379 riavnfi10mu3bm 739xph torva74 9dcgi9ns8qlnho","quiz_options":{"score":0},"id":"4385137813"}]},"page_id":"168830060"},"emitted_at":1674149695190} +{"stream":"survey_questions","data":{"id":"667462113","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"xvodyrmsj8o5 vqjqrurdq h74609w5o8cj kpy303cf 5qyfp1flg hpfvhtg412qu bwhmup tfxwjcltbmp"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462113","answers":{"choices":[{"position":1,"visible":true,"text":"x5gvnnvh 39thim64p k3naeh9 ebx5et8ci8vnjwq mwfb0867jts36 lq8ghnv8c23c86i","quiz_options":{"score":0},"id":"4385177065"},{"position":2,"visible":true,"text":"5atgqxr5w uk3ussdg7 29qlw s20qfx0w1u1d cdx47v m76g66t1j x9wuj hyv5095ipd9ly","quiz_options":{"score":0},"id":"4385177066"},{"position":3,"visible":true,"text":"yqvo5j rojr27j6ww 5k0ra1y96j6 vbd70ncr0 wdoxqqhkv a283r7g tshidt7i0jw 70gxqa3 d1wh0y 8q2x4yu5u3tcga","quiz_options":{"score":0},"id":"4385177067"},{"position":4,"visible":true,"text":"53838lqaxys gt190mgek1r2r llxh86fi38xyyb puwhs54wnxa8m5r cb8w1f312hts1 80gh5hrp0o 9a6siov 5i3l99eiefhoq","quiz_options":{"score":0},"id":"4385177068"}]},"page_id":"168831471"},"emitted_at":1674149695299} +{"stream":"survey_questions","data":{"id":"667462114","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"r5gme6gxar 540ixc g0a32bjvkgg4lh 9n8hpb qor7i9od6r max0ae1vu08"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462114","answers":{"choices":[{"position":1,"visible":true,"text":"tk3pmts41 aw7w4y8e5 65iplksmu5wa0o 9ve8f0vgh eiy8hki8e3 5vbwg fa3nnnvu3uq c43makf6i","quiz_options":{"score":0},"id":"4385177095"},{"position":2,"visible":true,"text":"3kvwm3v j0x7vrrpnxmtcub 7h9g43wg71ppkn 71nswd5eq ocqpss07r05dej4 7ln0vdgw0a","quiz_options":{"score":0},"id":"4385177096"},{"position":3,"visible":true,"text":"07td1maptf9wde nrlxmvshy dgpasbrawvpndo id9yjt2tsi mkm0ri7epkw5d poj8pv8m4lu0hc 2xh0i62g5j wxokec50ps52h8 d6uqtrf td70pitsu","quiz_options":{"score":0},"id":"4385177097"},{"position":4,"visible":true,"text":"ccw6cg wd5s85rpk dqlppdkh wpqg1t9vhdq8c","quiz_options":{"score":0},"id":"4385177098"},{"position":5,"visible":true,"text":"8b85ql72lufp 3r2k118 vy5uh6mnntsq5 x49hvqp19g 3oc9laa75hjxwn7 a77dw8 aiskfi350fyh5w 2h7ra ry4mj i2el2","quiz_options":{"score":0},"id":"4385177099"}]},"page_id":"168831471"},"emitted_at":1674149695300} +{"stream":"survey_questions","data":{"id":"667462122","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0wpcptufvllj 4yjhb78a bxn0uwh wxberovodxawinb 414fl1hatnpl 6tdxbt dpieidpjnqba9r"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462122","answers":{"choices":[{"position":1,"visible":true,"text":"c7nxl 3qrbcjg tddcypx5j63ne1 ebrs5911s ilsx5w8mvfs s24oodbw hbah99rw65wt7i","quiz_options":{"score":0},"id":"4385177138"},{"position":2,"visible":true,"text":"1lx4btr3s0pqq63 y93x5lqe xmyta126vo 07q0rnh 4f2bxvoc441 wcnvx ytjeo8g5a wjqrg 134n2q fgobl","quiz_options":{"score":0},"id":"4385177139"},{"position":3,"visible":true,"text":"t9hwm2nyw se2sl72a t70iepatjw0rsh kq6ta 6mhce jgfjh52 w3ivi m0sxg2i4 pup4tsn7a18","quiz_options":{"score":0},"id":"4385177140"},{"position":4,"visible":true,"text":"j6w78imo yhv9sgxs64ii ep2ckum1ge3 3d8gcbsuw j0b95oqfwn","quiz_options":{"score":0},"id":"4385177141"},{"position":5,"visible":true,"text":"5gn4d25al 29tdmvxhe1 svxcxt7qsq pbnlphkoa xvotqve1o79","quiz_options":{"score":0},"id":"4385177142"},{"position":6,"visible":true,"text":"sexvxfmqb jp6xina8q 3gsyu2jfgvpag d2wcp0k5ukuk igg5ecqj rtkd5j 485nmxjd tjodfsyhu5","quiz_options":{"score":0},"id":"4385177143"}]},"page_id":"168831471"},"emitted_at":1674149695301} +{"stream":"survey_questions","data":{"id":"667462126","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5r90gmyyy 1eaacbfag0g4dtj 4rlwnt3r jxg5ruhn tkkml hhlu8r79jma"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462126","answers":{"choices":[{"position":1,"visible":true,"text":"vln5gpqjq tglcfbuxu7b dob43c qnisv2 td55l7hc91pd00q 8gig93a5h 7hrhg6drxsxy bj0oldpxa33by1 4q9mx98wbi p0wb03eabrt87q","quiz_options":{"score":0},"id":"4385177153"},{"position":2,"visible":true,"text":"fj0a0nfwmfjrong q38px84nyk rvqmj 2sv0x6n9a hm88jqn87kvws","quiz_options":{"score":0},"id":"4385177154"},{"position":3,"visible":true,"text":"ms6ilfpbca 8ota11f 1qvg0ellfxis pfyvkkuujjjg qih57geu 366h9cnbrnny9 4sx4q0hbgu","quiz_options":{"score":0},"id":"4385177155"},{"position":4,"visible":true,"text":"si9tpsg54u 13dds2djq4kh kra03y5p43 j8rej5mv d0tkvig6","quiz_options":{"score":0},"id":"4385177156"},{"position":5,"visible":true,"text":"k5ijfpctrn flm2q7kn5t4jf xkjec55tbfsgsn7 r61gy346sxt","quiz_options":{"score":0},"id":"4385177157"},{"position":6,"visible":true,"text":"v9u400dqc fii651ftc 29rn2dffioa0 1m22amm42b qhs0wpth5 mva3eqo09hvo7x 5od22d7a 7m4606e271a6yk","quiz_options":{"score":0},"id":"4385177158"}]},"page_id":"168831471"},"emitted_at":1674149695302} +{"stream":"survey_questions","data":{"id":"667462130","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"a8ibp46e09n3ae uwv4r3pbpla uxqx8 8nlfqe2ekyj6b1u ju2o3ts 29q5blc7t5m225x rgcjb uek905hycm3 8nkpkppcdm4s vfcx6c63bfppfk"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462130","answers":{"choices":[{"position":1,"visible":true,"text":"xkuphh 9sloeexwdfla qyx58xq9u ne6p3x","quiz_options":{"score":0},"id":"4385177171"},{"position":2,"visible":true,"text":"83uko6624m otef3go 1ba21xa3 6k24mmw t82m7my9ipep","quiz_options":{"score":0},"id":"4385177172"},{"position":3,"visible":true,"text":"ke2gqx614f p7y9pi89 asll5jaju89e1g vgrj3wbe x815bgx 67md1tfhviy9v","quiz_options":{"score":0},"id":"4385177173"},{"position":4,"visible":true,"text":"w5raj381jcgffq 9o1b73 0jnat47bs8bq9e 41ffc wcmivj410jtu19 njxb3v7 gurti qsjowjls niwyt2clulpdpa fe0skx96v529ui","quiz_options":{"score":0},"id":"4385177174"}]},"page_id":"168831471"},"emitted_at":1674149695302} +{"stream":"survey_questions","data":{"id":"667462135","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ictucjn9av1 hqjkdsl56q dw9ouny 3wbjmkmik8m pxirnbm7f jq31e0w572q61j le5ke3if 8wkwmox1ow fsu8b5 ci0ot"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462135","answers":{"choices":[{"position":1,"visible":true,"text":"k4q8hnwb0uticgy kx7rcy sp8nvx036 u552gi 1icsxiqmdmects 0c1d69aee soiye","quiz_options":{"score":0},"id":"4385177200"},{"position":2,"visible":true,"text":"bolb3i7f7 5d7y5 n7il93or tb2599 wlr1hh88d5bjpr vu3v5uudajbgoa s7kmwnhqpph","quiz_options":{"score":0},"id":"4385177201"},{"position":3,"visible":true,"text":"ve8olc60hjdia ake87 4iv5po m484jgkaut 5e285a0hgsrirjm qtnofl05rdtx","quiz_options":{"score":0},"id":"4385177202"},{"position":4,"visible":true,"text":"1pix5bfrx 9pncw4rp5g xf1sm2qr c474c1s67jgcrw trxj2k huiccx1kxt1 l13mo1hij12","quiz_options":{"score":0},"id":"4385177203"},{"position":5,"visible":true,"text":"kf0u04jlsopl 1b4xnt0g 3ehjhwg d9465xfy24uisef 0xv77xfgm7x","quiz_options":{"score":0},"id":"4385177204"}]},"page_id":"168831478"},"emitted_at":1674149695303} +{"stream":"survey_questions","data":{"id":"667462136","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"c4f5ohh3sr2w xoto95lot 7lykuivbhs078b h524lu75 mvss35agi"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462136","answers":{"choices":[{"position":1,"visible":true,"text":"1blqraty 6uddh9u imhcfci dho72p k7feqgqqqr3jm7a 8y7drc4m8m8f1l0","quiz_options":{"score":0},"id":"4385177210"},{"position":2,"visible":true,"text":"cnxmsuqasj3g gm6e1da2102igu c3w07jugiuro afj54 kpoom7 n11yf","quiz_options":{"score":0},"id":"4385177211"},{"position":3,"visible":true,"text":"82nkl1jumcnc6y3 raogw1xp84nswty h77x7 f26dju09o2 ajgbdjdjne6i","quiz_options":{"score":0},"id":"4385177212"},{"position":4,"visible":true,"text":"dukxs7vo kj6j0x kjsmang 6g8vgh t9ymo1h2hfln6 iyuklw ou0uq6","quiz_options":{"score":0},"id":"4385177213"}]},"page_id":"168831478"},"emitted_at":1674149695304} +{"stream":"survey_questions","data":{"id":"667462138","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"jvr1s akb7hxxk2ptt1q jhgxvvgirs2d9 rae88p7 b54u3m1g1 vflgmpv75w5q fauxphpa8bxdutx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462138","answers":{"choices":[{"position":1,"visible":true,"text":"jnkfpiwx3ye b6osvfmkkn eek8p036ayn rt6k5em681q78c 8j9brp8e2qsa 3wu71v85le4x65g","quiz_options":{"score":0},"id":"4385177214"},{"position":2,"visible":true,"text":"oriu3pl31qd r6u7isqu2p1 s6lrtm ybawl9g3oafvxx9 sw16llmo3dpc2xx v0r4lrj serixtbxt sy64a2vcyjf f8mhp","quiz_options":{"score":0},"id":"4385177215"},{"position":3,"visible":true,"text":"v0j4ixrd gxjah4rsf03mhtt selnh 311w2 80liajkb9v2h8 m7aggx9 kmjjabnan kk3g4y8jiuv n15toj tt60bhmd","quiz_options":{"score":0},"id":"4385177216"},{"position":4,"visible":true,"text":"vmvi1nlvkd1vy 13gv0qjxtri sty84 df77l wwh82kck3x0 7jer7beg 8x6vcrnh9qiaf3 nhl5ref5v5bud dol9qivxsg8owna xsw7e0s1","quiz_options":{"score":0},"id":"4385177217"},{"position":5,"visible":true,"text":"a2fdw9jbcytl8ok 0ir460hfwm8 rqecrtla vyi4l3tpv eyhkdh47w2uixp","quiz_options":{"score":0},"id":"4385177218"},{"position":6,"visible":true,"text":"yvusg0ns7jw9tt nadlph4 0ur0o5dhan 9jlq9878999lbf","quiz_options":{"score":0},"id":"4385177219"},{"position":7,"visible":true,"text":"48eqbvdq1ubpvyk 5f4pwr438ik81xj x7ioyubgl90808k 12tr703dg0s 9xpyni000c9sbr","quiz_options":{"score":0},"id":"4385177220"}]},"page_id":"168831478"},"emitted_at":1674149695305} +{"stream":"survey_questions","data":{"id":"667462170","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"csowuuey o93r9336 qf9fym8wbog1q l48c9"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462170","answers":{"choices":[{"position":1,"visible":true,"text":"0k0agd84ev s54c44f90gaj pol5dlpg9t fa5trrngfu cvrd4cmtmq","quiz_options":{"score":0},"id":"4385177381"},{"position":2,"visible":true,"text":"9egly8imf1 9k1iapox y80fk4 58nrajt2 swkm1na","quiz_options":{"score":0},"id":"4385177382"},{"position":3,"visible":true,"text":"se2mcirb6lt5ty qqt53dn jpg4b3wk0c7 91onwco8d7ll0 e7y4mwxa ojs5nky 6u7am","quiz_options":{"score":0},"id":"4385177383"}]},"page_id":"168831478"},"emitted_at":1674149695305} +{"stream":"survey_questions","data":{"id":"667462172","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ypxnp3k6 708jao1b5ok671 cv8x263j0f nk1a0a5cm 7y4r2bclg2rgkd7 alax412w9y76"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462172","answers":{"choices":[{"position":1,"visible":true,"text":"dil7s3hpsuf ms76n abih45plj 9bw5noujg c759tcfvo7t3e","quiz_options":{"score":0},"id":"4385177392"},{"position":2,"visible":true,"text":"okltv66m4qav 88ijt vcq7k5m xcpwi9e4v7 ujqbg24ap g715rctpafkm0g","quiz_options":{"score":0},"id":"4385177393"},{"position":3,"visible":true,"text":"vev8vhb11 dc5mn3colgaea8g jflk5j niy1qt8onj788","quiz_options":{"score":0},"id":"4385177394"},{"position":4,"visible":true,"text":"hq1wbebc2gleq tj2vf867 n1aqc afpekt91re 89guq9pn9rrww 0bgvg74","quiz_options":{"score":0},"id":"4385177395"},{"position":5,"visible":true,"text":"bxkr18qipct gp9wcqfk n69e6ai66xmy01w xlh7s17hlsbcyt o16ecq6vf42q crmh6p ykbam9mxc37ah2 ii40872r6mws 2xobrm","quiz_options":{"score":0},"id":"4385177396"},{"position":6,"visible":true,"text":"x18wk 3k4tt8ae kkrj0xjbf krc0uyrj2u 0qq68","quiz_options":{"score":0},"id":"4385177397"},{"position":7,"visible":true,"text":"fkx0pjep58wx jpxs8a7vfnf bq2hlx3vhqsn qv4d1evms x64h8k3e tqaeb3mc a1yiqm7qjgi8 cnaw9173k6js msisp4d7bf12 qan9ffq3sv","quiz_options":{"score":0},"id":"4385177398"},{"position":8,"visible":true,"text":"tdjr97it3qvsl 8pfqjvhuo91 mp8kdeymnwv9f 8apy9eshudwhq veo63l6q9np w18s0102tvjru 6lckytkggn6 ncdlx68","quiz_options":{"score":0},"id":"4385177399"}]},"page_id":"168831478"},"emitted_at":1674149695306} +{"stream":"survey_questions","data":{"id":"667455348","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"se3rt7c 9l8d9rig3sk jk791y4nv9qp2 nof9xr8 6irr0mr2uv73g4 rmv1ko9gx2 qievxp819lc o9jbr n1vrq"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455348","answers":{"choices":[{"position":1,"visible":true,"text":"6sudul87 6605b94 w6ubg6a7p0v ufh3d5da nyic6pot11a64v9 awpjgnn22pdgup aidahk8krd 0vhentor51fp ujasxxx1srf","quiz_options":{"score":0},"id":"4385138588"},{"position":2,"visible":true,"text":"08yvwmw 441si6svo1bkxm o85tbsa3k wlkn3","quiz_options":{"score":0},"id":"4385138589"},{"position":3,"visible":true,"text":"uvmasd gagqr6qlswnr af0hp3nfg2vucd vbtkx8vssxlvg6k 5g3rhca48m2 y8gwlm2","quiz_options":{"score":0},"id":"4385138590"},{"position":4,"visible":true,"text":"mrjr3x0 57yyh77a3cmlcmj mmwked8tvqnt5 bwe0hh5dqp1sn3 pia1tu u7ujusnwtvt be24cc8s5olebg 5n65mmu5nki2b","quiz_options":{"score":0},"id":"4385138591"},{"position":5,"visible":true,"text":"k3wcw3bjn hq24e21 f5vvmq0kipi 1smhqty4 p8x2xxy0obkxv qsa4cuvmy81r7s 651b5hlw 2ej3xrm289eqd2 ugnf354v90h 92mk44ghps6cm","quiz_options":{"score":0},"id":"4385138592"},{"position":6,"visible":true,"text":"k8yajsxquuu 94uubd5wo9 piwa122aeek 3fhrq5gvwh14jho 6n8k5a54q7nf rnoqwd82y1 tkhxop blr0p2l7utyydd 7eto449ipd","quiz_options":{"score":0},"id":"4385138593"}]},"page_id":"168830094"},"emitted_at":1674149695413} +{"stream":"survey_questions","data":{"id":"667455351","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"dni8kmob8f47ly 92hb1pg6lvuklf f3hvi6 yjiq3bvv yys743quapbmwm 9xclo63 ydxt59b89 iwks1mdba9iuje vmaq243f75y4m 8cqcf54w"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455351","answers":{"choices":[{"position":1,"visible":true,"text":"9e7yu 2366vw4sa3 rhhgm0 yg8p9f9","quiz_options":{"score":0},"id":"4385138595"},{"position":2,"visible":true,"text":"1ldfxfr k0hnmi dcjdiaquo0xyac remaba5g8 9pslek2fmlvxf7 vn4gl8yjs10up4d b4i21994 78q3d","quiz_options":{"score":0},"id":"4385138596"},{"position":3,"visible":true,"text":"aq8d89iqk6 ytv9mg39tq5 lb5gtx1kdm5a 7scilf5j73g580i tr22c460a1vu dvpernw oy2j9qqnhhbpia9 ufy7twqsl2ovj","quiz_options":{"score":0},"id":"4385138597"},{"position":4,"visible":true,"text":"dlt857wic34l 5y0sijl8 1jmsvy9r e7psvexn8rj0nw ioo4ka04w vtk0ihmg27ac bm60oah9 chljr8vrnt8adx1","quiz_options":{"score":0},"id":"4385138598"},{"position":5,"visible":true,"text":"mk4nh6bnddk1ph 63551586l v0d3vhn w9s7xmbbm3 cbddi6vgmddb se5grc1xr2ycuu bmk1yr7kq3e","quiz_options":{"score":0},"id":"4385138599"},{"position":6,"visible":true,"text":"ob4dd4 r36mdre p96uaag57ld7vo yeqwbn7w2tgi4sj wcgvpd0 4jq0nh42ev8 23i0ye5x 4vuki","quiz_options":{"score":0},"id":"4385138600"}]},"page_id":"168830094"},"emitted_at":1674149695414} +{"stream":"survey_questions","data":{"id":"667455358","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"79kmj4p2d 3sbolhwfrs ioe6b93hwr70jt7 679dpby y4mxy 1pr869ii"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455358","answers":{"choices":[{"position":1,"visible":true,"text":"mqjitf64d0 k83dvyn 227nrf 7a5bulp 0jvp9hmiew umwdpysrqbgp he3hkvoxi8d t0ekmaqml bniybp7","quiz_options":{"score":0},"id":"4385138620"},{"position":2,"visible":true,"text":"ocaqjvupc or8s5on 2txbvlu6e20 8uxgb ll6ktjtk gnwy7knkeeev y2t9cah46lu7 rkmxywsr 8qgmy6gm3fd84 eko2r","quiz_options":{"score":0},"id":"4385138621"},{"position":3,"visible":true,"text":"3kjxifk f8vld xx9lg2ejtf3ccwx 1rfll3376hrm xwaevy7i2krc 93h2fggv1nak vvo0hqr0n 27qcal0ao","quiz_options":{"score":0},"id":"4385138622"},{"position":4,"visible":true,"text":"qqsv99lam uidx4csd1npsc nccxm6ueer 2ihx3c5ysd","quiz_options":{"score":0},"id":"4385138623"},{"position":5,"visible":true,"text":"3ooy243l5h usutwf3do32na0o t5vo8vggvjl pp44k52k3dw swg9bvhtl v4mn5goadw4 9g9jefyf6qhsijc 2j1bmvuu83brk","quiz_options":{"score":0},"id":"4385138624"},{"position":6,"visible":true,"text":"fnme6flca1yfyo kj1q1j tsfq15e2iam8 tyny80 fjb3nf p0mtkwgteciioq p5b79nk1a44d0re lwpwx8wu0e2 at5p0nbaqbp 67tqasnk0lb9dr","quiz_options":{"score":0},"id":"4385138625"},{"position":7,"visible":true,"text":"8n8el8 it06oap6nv fapkjmk yxemrfihr61i heif6anfb o9nel86ws 2t1vpytr","quiz_options":{"score":0},"id":"4385138626"}]},"page_id":"168830094"},"emitted_at":1674149695414} +{"stream":"survey_questions","data":{"id":"667455370","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"lqroep8etllsp 3o8w6vilbyqlwkm b5velmro8 b4ylvv9kkt"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455370","answers":{"choices":[{"position":1,"visible":true,"text":"4bdsyi00551 hsh1woqfmclyma sslfhk y11rdswosp4l 2my277i74fedi lool7","quiz_options":{"score":0},"id":"4385138700"},{"position":2,"visible":true,"text":"qdn040 3tljs j7juh9xjucx 31royj9xs6q lg56gf8c81s","quiz_options":{"score":0},"id":"4385138701"},{"position":3,"visible":true,"text":"ytndri0sy3noxx p2d3euo 4d2jqiygj8jwasr 38lf63x39sxb irlep2 cgi9pr 4pcvga t64nykcc4gn tv9xfejs41 er8a8pw","quiz_options":{"score":0},"id":"4385138702"},{"position":4,"visible":true,"text":"n2hd3mf5 7gg2j7cn55e77 1j1ijenv62 ntd44byw6 g86vqe62ogytk qcodrayuwmht5tn ix3qc42ybhqd 76ry3 goorkieate s2dofir","quiz_options":{"score":0},"id":"4385138703"},{"position":5,"visible":true,"text":"ct0y2yaibh 5hhaeb2u5jmncca ibf7eql6h w3k37s","quiz_options":{"score":0},"id":"4385138704"},{"position":6,"visible":true,"text":"w9647 j2wq2eisb85rlu js474wesi1j 1d7eiqq 3o7ubmmvsbl4","quiz_options":{"score":0},"id":"4385138705"},{"position":7,"visible":true,"text":"deheetxt3hlox 8e4hd76i 517ltrj0v dgi0r19ud5srqj 1qdru4f sl8p3 7gxolfxkhlc5x cm0seo7wroouww","quiz_options":{"score":0},"id":"4385138706"},{"position":8,"visible":true,"text":"4xv8h753xoo8n1 9atxgj8 5eaohgaugpu3in ybq32s67 atg5l7u4aotebk1 0asuiahw4 dyakwm kpqe9vu81dkfvcf","quiz_options":{"score":0},"id":"4385138707"}]},"page_id":"168830094"},"emitted_at":1674149695415} +{"stream":"survey_questions","data":{"id":"667455395","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0qnmuchico4fa8q dfvkni 15mwj2a8 1c8qh1 y3dml65 yfodif0 uik9a vja72b"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455395","answers":{"choices":[{"position":1,"visible":true,"text":"k9cwqyrhvt6t2y lijf97ywdxctyr nahqrqwo o73wy4r13r","quiz_options":{"score":0},"id":"4385138779"},{"position":2,"visible":true,"text":"0dtp7umdw2tn 3qerkatyyrcndup s70sgadrf0pjwna 13cc30nv 3fku5vgj5","quiz_options":{"score":0},"id":"4385138780"},{"position":3,"visible":true,"text":"8vwwlvlx3 rl3x4l r3itwqo 2uiml2j417 p0x4d9pxyhs bdyo06b oyuo333qq","quiz_options":{"score":0},"id":"4385138781"},{"position":4,"visible":true,"text":"ua5rn0o lgn9qrvh a3xi82nkmd9s2d sdhqh3q8m yu21i9gn3 u4wyck8efnu 47ubnif60vxre wfs92q1c84 qyyup","quiz_options":{"score":0},"id":"4385138782"}]},"page_id":"168830094"},"emitted_at":1674149695415} +{"stream":"survey_questions","data":{"id":"667455427","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"bja345d 7gmh5 u335j3ifd bmn2iwulckt qbbgde6 4l86ghgt3bnplod fsq6qpqp ogqqnp01j gycd6318in4xl s7sxc7ric"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455427","answers":{"choices":[{"position":1,"visible":true,"text":"h1f77hud lvvbp8566vit8 eij3l1f6m86rt9 658f6 auhky2x 1qb9805ptcl4yvs","quiz_options":{"score":0},"id":"4385139013"},{"position":2,"visible":true,"text":"82ow11qv0ewplb iipebne7de pccthtsby8y 87tn87egsum9nd 41xopta 65fhf560c 326vjb4pp4mtdvx 4eqmo8viakl pmuv626kfxaqr 9laiyqibgse9","quiz_options":{"score":0},"id":"4385139014"},{"position":3,"visible":true,"text":"8prnel y3ygc5kxmjbxfc hdd0bos b9w3a1gfm1sm6 8wwao s6j2lw5ennapne 6psoe bxyxi pp141xm uatdplo5f60et","quiz_options":{"score":0},"id":"4385139015"},{"position":4,"visible":true,"text":"yejw4cxbnjeup 9war0kp9wl ngag1wd y6quyhtcv lhai6 6fo022bo2 0y9e1wuj sm9c4jui n9udx","quiz_options":{"score":0},"id":"4385139016"},{"position":5,"visible":true,"text":"o9i4r2ej 825mg 5rifxtuu83ox y3nhp","quiz_options":{"score":0},"id":"4385139017"},{"position":6,"visible":true,"text":"3vukbvawqxox yxo374n8xcpw s6ai05vb64 gld3jyi97 auhaq08 sl5tt43hgv n549d7mf0n7cr2q 44x9v5o31yor0 oel767u6o1evo5 knipqna","quiz_options":{"score":0},"id":"4385139018"}]},"page_id":"168830108"},"emitted_at":1674149695416} +{"stream":"survey_questions","data":{"id":"667455439","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"yvl318t7lj 7wdmc8mttupa 2g7ogtjr976g lm77fy59yrhs4 tb928lr2 kiug5rc d4hq6y1"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455439","answers":{"choices":[{"position":1,"visible":true,"text":"du6tsy5k rhhm4v540es3mo oqkt8crv5wvgvj s00kbxja5h 6kmpu","quiz_options":{"score":0},"id":"4385139062"},{"position":2,"visible":true,"text":"b52dpf uy6exuyx l6ugq4ki qkl8h9l 8ptnm 2jwismc2by8 ls428u","quiz_options":{"score":0},"id":"4385139063"},{"position":3,"visible":true,"text":"86bpwgdk1q0 4iigbk1xjrm hncx3xkk5lj e75h213rkrpjg cku6p9no3qv rn1dvjp5hmtbfar 694ly6v9m ue5ad4q xomxi5c69o6pqm f75mwyy5nd","quiz_options":{"score":0},"id":"4385139064"}]},"page_id":"168830108"},"emitted_at":1674149695416} +{"stream":"survey_questions","data":{"id":"667455443","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6myggi xgx2lptp oxfg8bgavhkxd h8586wxt2vv7 tba2bd 07altrm 2vmmtgfir gchrhk6kdw"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455443","answers":{"choices":[{"position":1,"visible":true,"text":"5acyluby8j u7jbf p5x8foahy k5uk8ubi0 ov4ouretpl lxrf3 ufct467j99f2hfl 7dlgr8eo","quiz_options":{"score":0},"id":"4385139109"},{"position":2,"visible":true,"text":"vpcmt6klp47b0 q3o6j96 3mm9gebofu1n2 jq7dq","quiz_options":{"score":0},"id":"4385139110"},{"position":3,"visible":true,"text":"r1wmg0i0ae892 acmd77ws2k 0pk531 pb5k6xev ury0cf","quiz_options":{"score":0},"id":"4385139111"},{"position":4,"visible":true,"text":"0hemigqnwym5f0j vn57ess p5vjtn nylhr7","quiz_options":{"score":0},"id":"4385139112"},{"position":5,"visible":true,"text":"gl7otpnp 0b43hto2wr0o tnavjwnce9lc5d 7j6hs ca3qbj70t b1nc7q4 7j0696hum k6ytijiprdmdvd","quiz_options":{"score":0},"id":"4385139113"}]},"page_id":"168830108"},"emitted_at":1674149695417} +{"stream":"survey_questions","data":{"id":"667455463","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"pjksyv7o2s qtssl1k83 r6ypxm 30krmk4j0e bp8y6m0or"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455463","answers":{"choices":[{"position":1,"visible":true,"text":"olpxel7n8ktrjx l0x3kfm fg4f96cpv60w2 kpp8waqc9j1y6 2bjrk4f v33tofsgnbg s6gcg386 kcubx22oev1ju i3n5l7veb","quiz_options":{"score":0},"id":"4385139230"},{"position":2,"visible":true,"text":"jkdmotc7gfb43b2 4cfkvtut 6w3ys rn3v7u17ccsx 0gyynlvql xgsrdfrtffvr","quiz_options":{"score":0},"id":"4385139231"},{"position":3,"visible":true,"text":"7ngjrf1wlkv1 wox4shhjp4 yivmg3epgwl5d 11tfmviaf7pv59s 14c6b1vudh brl9d3b799gibr fknowr6c fi6k91n34pm270","quiz_options":{"score":0},"id":"4385139232"},{"position":4,"visible":true,"text":"o8r5r32univvk5 dhx2tc7hbbgl63n bxhmmoi6vk 1ynu42 8goapcp gh1gr8dab p1iik6y eud8jxcg jilsond0434xp77 g1s9ydwgrd6y5tb","quiz_options":{"score":0},"id":"4385139233"},{"position":5,"visible":true,"text":"p9xp9x7yepqx wg9w4q8 9f8n1maik5weupa 9s5nnuan5i co88w9s89g2pfq lxrdlybm6 fl702pf2x7 xsp9xl8yrrg m8xre7g s61if4ojul0qlm0","quiz_options":{"score":0},"id":"4385139234"},{"position":6,"visible":true,"text":"5k001wyjow 9rfq00sg6vcytu yxddli0wkif3wk1 vxk0kcy6l80jfc a06swtlqbdg 2lk8kytqeeqs 81w3s8","quiz_options":{"score":0},"id":"4385139235"},{"position":7,"visible":true,"text":"ia4knoy0u1vku io2wydrh90d6 52j9qii2gcgc9fk tx7409a0o59ffk0","quiz_options":{"score":0},"id":"4385139236"}]},"page_id":"168830108"},"emitted_at":1674149695417} +{"stream":"survey_questions","data":{"id":"667455466","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"17ks2i1p 1roi7n8j p21kn jkrvnckxot 3nkruc3e ugsatu9qx7wsw qsttqci3i03e4"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455466","answers":{"choices":[{"position":1,"visible":true,"text":"n1ic8gxnyrf1 5qacyxsh ixbp7b33rq4kigy 2rdbf2a4 rfms8di taxqa3oc","quiz_options":{"score":0},"id":"4385139244"},{"position":2,"visible":true,"text":"i4kx3k5s1t 3qw1r6n two72xjft yvu2k2 lywg4w3xhffbwrv pu20atqyr5g 34tsp4wk nwadhm3qirol8y o5x91 755w2suug91yywd","quiz_options":{"score":0},"id":"4385139245"},{"position":3,"visible":true,"text":"2xl9i1c4 11w3fsb78 bfoj2 4rmfw06mn2lxpjo wfijsno52y90 wxuik4 70c6ioht","quiz_options":{"score":0},"id":"4385139246"}]},"page_id":"168830108"},"emitted_at":1674149695418} +{"stream":"survey_questions","data":{"id":"667455236","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ibdmqewlt gbp5yhhp c8i5s75wwg 0xosr1devp 2ijbswdblw0bjko k6httw7 kfr0tk"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455236","answers":{"choices":[{"position":1,"visible":true,"text":"3nj2ikx od392yg7 akufy84 6gw9q axmbaw59fn nfc8xe410jg 6allsd7jg7u 3wyrciq0uje8rn0","quiz_options":{"score":0},"id":"4385137904"},{"position":2,"visible":true,"text":"cb0o3tqf5 ijxi31vsg1 i0ga1cu10vmcgry gfmk6","quiz_options":{"score":0},"id":"4385137905"},{"position":3,"visible":true,"text":"i4frvbso f2k3tyww70ehqkp 5qy73 21lpebsil0fwspd kbx0uypchk wqdb8j0a0 8dr1nvs 592s2coha4r0v","quiz_options":{"score":0},"id":"4385137906"},{"position":4,"visible":true,"text":"lmg2vsbrw180cm otp0wq1e2 530jnetga2dp88 e2uploqajl","quiz_options":{"score":0},"id":"4385137907"},{"position":5,"visible":true,"text":"p0gqg3u03 11u6b6 xdb1lqr bxjqwc48sis 5dd68pfpw lxtumpinwgj vwdyg0uw4u wlvjnya7n7fr","quiz_options":{"score":0},"id":"4385137908"},{"position":6,"visible":true,"text":"74mma6 8c3o4 sjp4tg uagi32nfox489p 33m8i9q2t g1ecllv3xjnumg w3i1p26o789 3g8l35xh9m33","quiz_options":{"score":0},"id":"4385137909"},{"position":7,"visible":true,"text":"217g3m8t fdh1g7f6sng3r nt3u7d6n8j1d5 k8hwfegvg4i3xhs 6wung qmv6ilblwpg4t4c r04g9x2 ra26624 hl1gnkjlcu3pn","quiz_options":{"score":0},"id":"4385137910"},{"position":8,"visible":true,"text":"hc8vkvc9 yd3dll3x 2mo5ol4vkqru4k ubqgf odk7ghioir7gc lm0je2 4q3r0 ks32ix ra4rjxv2d9","quiz_options":{"score":0},"id":"4385137911"}]},"page_id":"168830068"},"emitted_at":1674149695519} +{"stream":"survey_questions","data":{"id":"667455240","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6vdliabstjs pjq6v4ea sux2ll21xwm7h qt6k4qflxuslj ig5xnx aqyngvnivqhi63 k7trhm7s9n yc1uiij"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455240","answers":{"choices":[{"position":1,"visible":true,"text":"n59jsjeq v4yp5kweu77pjd 6dsaj62 f51922ixg bfy1tdx0ajmwq sq2g2ca6t1ag t91oetm","quiz_options":{"score":0},"id":"4385137924"},{"position":2,"visible":true,"text":"miou6 s74ictupiv15j a1ojxwpxsc 24wjut22cino3li 3nef4p8a1onycu s2iku aluhi 78a20fa","quiz_options":{"score":0},"id":"4385137925"},{"position":3,"visible":true,"text":"96wygu4eqjryyr vft0o 1omksgrj4e4u hov2cmxl xryqlb9qe5s 69bf1gxp prv2hpeebouh","quiz_options":{"score":0},"id":"4385137926"},{"position":4,"visible":true,"text":"80jv70rswm80ng s3yupsiahmmfxf 0qerhyspf y1shqteym hu7hk9dkpo 58idxrntiwiqc h36nn","quiz_options":{"score":0},"id":"4385137927"}]},"page_id":"168830068"},"emitted_at":1674149695520} +{"stream":"survey_questions","data":{"id":"667455243","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"1lmh6obuf8d67h 9f7639oxptfr pgb8r ud5359op7klll"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455243","answers":{"choices":[{"position":1,"visible":true,"text":"ht5dtccv20hfamc 6m2dk7 edktxyp 9bpptvmpf 76dv4sin8ps33ac","quiz_options":{"score":0},"id":"4385137931"},{"position":2,"visible":true,"text":"m1e1sw2b xk8ujpclrw rnv8g5qtu04 g1hy75 887fxgre","quiz_options":{"score":0},"id":"4385137932"},{"position":3,"visible":true,"text":"8iqn4b9 ico6j1l7h b44vd6d3383m0 rkm2tqhsi00qy 5hso1f919pq 5yyhjb5h7 dhhsrml5g6kiefx apd2weqls w4hg7 p1rwc2o7pko","quiz_options":{"score":0},"id":"4385137933"},{"position":4,"visible":true,"text":"erh5b0d 47uxrflw 23r21lf24iwf bb8yqiqs79 3y8eb7le7y2ocb8 juni8","quiz_options":{"score":0},"id":"4385137934"},{"position":5,"visible":true,"text":"x32489wko4ns f0ob7be3j pmc9ui3s6qp0 08kfm8yaqcanw 8aot5prgvkqyseo q4vp4n656gj57g xygwva 96gw2r2npb2","quiz_options":{"score":0},"id":"4385137935"},{"position":6,"visible":true,"text":"sjsfbmxa97 xq0084k5hm3 hivfs05sfir 40dj15utx bo9mx1yu0","quiz_options":{"score":0},"id":"4385137936"},{"position":7,"visible":true,"text":"jrb1fu2j x90bw8mlv85gpl0 xfo84sk0jy6 l3392k bucd4nmlc7yj jdj3x2clsir95 uw9dhluee e9ai5v8pm5 4eufmbvvi","quiz_options":{"score":0},"id":"4385137937"}]},"page_id":"168830068"},"emitted_at":1674149695520} +{"stream":"survey_questions","data":{"id":"667455245","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"h3en3q3evla 6eye52o9ijuo 078omcsgacycjn9 a76ryw2wl 5ejj2a399 r62c2jrxj2x7y"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455245","answers":{"choices":[{"position":1,"visible":true,"text":"90t457 4q172 f4kmgq3 vc09o 7hqxmsg9jx6 7fenorkuyq ov1kus876aiv0p rrmrq5qlittb6y","quiz_options":{"score":0},"id":"4385137944"},{"position":2,"visible":true,"text":"9aii09 2r9oaqs23 j2x9qy94cnxfgn jrlxnqn9n","quiz_options":{"score":0},"id":"4385137945"},{"position":3,"visible":true,"text":"r8jgwe j87njrw3yo1fla bbngq3 g26et","quiz_options":{"score":0},"id":"4385137946"},{"position":4,"visible":true,"text":"91n95cu 5fp11un vngsubusfe4 vgbho","quiz_options":{"score":0},"id":"4385137947"},{"position":5,"visible":true,"text":"9qw7f8sdfws1 6wfueox bghljj3 4yal6iqt73m6han l921hy 6yr04p7c","quiz_options":{"score":0},"id":"4385137948"},{"position":6,"visible":true,"text":"trrepiynd5phbcr 35t494bg wm02s1clg9wkxl1 eemhx faxm4gd3aqepewc rp34jr2ho5gb5q ol65t043p1 66bg3yos kh4xbmdvqyvm ibrokgs","quiz_options":{"score":0},"id":"4385137949"},{"position":7,"visible":true,"text":"ni9kx4ob31 mbn9d2n3t4j6lal xel50s53bw6eydo b0s9p p512cv6lrh 2jq1h yqmu3hg70qxw99e k1xjbd vv9lbb4kvlt1jg p86c1lob16h","quiz_options":{"score":0},"id":"4385137950"},{"position":8,"visible":true,"text":"3tx1f04qbxgku4k modppm wc68tgts7o me6s4w7ikqolcg4","quiz_options":{"score":0},"id":"4385137951"}]},"page_id":"168830068"},"emitted_at":1674149695521} +{"stream":"survey_questions","data":{"id":"667455263","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"gx0d844wwsnss b89h0hrw ihlud3p23xbv3 mtv2g99i bda7267b5 pxxsxit4ey lcji2t"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455263","answers":{"choices":[{"position":1,"visible":true,"text":"e0dhk875qtxip 22ci0nbrenwe tnj1517py jfe77 wnmap goid8 s5r92q9nx9gbxf oy5bkxrwsyqx 7oemt6oc3wcw2 5iwm8","quiz_options":{"score":0},"id":"4385138060"},{"position":2,"visible":true,"text":"nnkw2nj3g1wb 6os5x3rph9 oejmdr7e u9vf9uaxis17w7p vee9xcim164h jkcq9v7e01i njv8rrgfo3hfumd ha6djf y3jckk4un mwvekphui4","quiz_options":{"score":0},"id":"4385138061"},{"position":3,"visible":true,"text":"0b1s6goa 31eeolk05pyxfvy 5fd1rlv8g8g 9iachco 3gqpac6rjo hmq9tr5huh4mxm","quiz_options":{"score":0},"id":"4385138062"},{"position":4,"visible":true,"text":"kkpmmshmp2owso tm5dsf0bt4rm474 w1wwr4 p6ltlf pxm8o1um1cv 8lh8goe9rqqo58x 9j7ej5b 4chhf50nlimyy","quiz_options":{"score":0},"id":"4385138063"},{"position":5,"visible":true,"text":"hygcvxvhqh1um 80ak8e wd9b3vkvg hkyetchu2pdm1 tn8hu6fropre lqx4jilgig1929w","quiz_options":{"score":0},"id":"4385138064"}]},"page_id":"168830068"},"emitted_at":1674149695521} +{"stream":"survey_questions","data":{"id":"667455268","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"of74qo8c 333x5lwc2aral be0joqsxrs hvgeb7ltbcu dtyfa2y98wnn2 jbv7c vjee2juk267 o2fd4pyua"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455268","answers":{"choices":[{"position":1,"visible":true,"text":"jyb7orcvpwo c9lcx 9aq0cld2fwngg5 ple2j5y3xn2px itcvwnp0rpn2 ykoj6qw las5p s01fya1co65 q6qrkss","quiz_options":{"score":0},"id":"4385138108"},{"position":2,"visible":true,"text":"vd9eniq 7lpv8hbl bq0pjeiplw0 f0hjp7xyv2pk8 82d5r7ciqjbl ngrw1mugyp 0qb5g4vnxxjjpf","quiz_options":{"score":0},"id":"4385138109"},{"position":3,"visible":true,"text":"s84erg ueidus67wjw w849glheqbx5p0m w2am53 erjlbf9pu7","quiz_options":{"score":0},"id":"4385138110"},{"position":4,"visible":true,"text":"n9yd3gp4otjuvma 16axdetwq epitspj3f8hq bgqks3 1s7wh28 qn8cff mfk5el7vr2w26pg 7bkx6xqe 1d9eryxaimnk u6sr9v3id0t","quiz_options":{"score":0},"id":"4385138111"},{"position":5,"visible":true,"text":"l3bswpvl 9obip6i6bvg ytwh8i mmn1n0p9xe9kiu rjgto5mi9ce5dm6","quiz_options":{"score":0},"id":"4385138112"},{"position":6,"visible":true,"text":"a7j5u 2td37rxj7 1lwcfapxi2cw6g 32egv55cc52fcv ay1jvha 1169qapnsa0avix w406ev2kwt1k8n y7xqqn i8uoafghx8 gelu50jf149xtg8","quiz_options":{"score":0},"id":"4385138113"}]},"page_id":"168830074"},"emitted_at":1674149695522} +{"stream":"survey_questions","data":{"id":"667455272","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"svumjkj5ei6 t1jkf ttf93l bilmloj40l q6er242eh bojtdo6 sf5jtk 66a4anc 3j5cbh3k xdfpwwtdf3hpb3"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455272","answers":{"choices":[{"position":1,"visible":true,"text":"i6ece2o dl6k3s11vd0 shf3m get86","quiz_options":{"score":0},"id":"4385138126"},{"position":2,"visible":true,"text":"fufxxy5ueb 5u74cl6a lol396aaape2 dwgaeqcphflg ogf55axqis54 1a338abonkbvc5 f0hj7rjhu 15gsns71xund4xp","quiz_options":{"score":0},"id":"4385138127"},{"position":3,"visible":true,"text":"di9p4lr30wahw 2rg14l uqqnx1roqy5k65r 2we720uhdva oyb5f74cil odhas8h1n4u3rj dqrhl943a","quiz_options":{"score":0},"id":"4385138128"},{"position":4,"visible":true,"text":"b2j0ivu58yysns 4roq2 mqukye4mmik92 wpr49dqhc6y g6ivs5m7n9 iotdbwjay566 06vv0nn17yfqb59 peuvkh9jjd jly7qt4151 roq63i3ld","quiz_options":{"score":0},"id":"4385138129"},{"position":5,"visible":true,"text":"1rceqano97 sd5m1s3hsskyv7 bwmei412e ikfkbu xhdo2mx6 aqg5dpo10 sfjb38 vj05jf71y","quiz_options":{"score":0},"id":"4385138130"},{"position":6,"visible":true,"text":"7nlryn01exq u14xxgcx8mngy 4pc2y3 48lxatpljatuox vbt7cvv2cipj7 ise8v3j9wnomr ms330kk mgjaw7kgowjfq6","quiz_options":{"score":0},"id":"4385138131"},{"position":7,"visible":true,"text":"y0sx70a14 527jr the5p62a hga0cg5nio53 hv2s0l 6e2lq473mrdqgo","quiz_options":{"score":0},"id":"4385138132"},{"position":8,"visible":true,"text":"rxtxctm7 veua1a ds34biwojid 5cjj1qvd3 v8ksghp4g 6ist9e a5xrfr8r2 ae9rb2xw lguj1iafi","quiz_options":{"score":0},"id":"4385138133"}]},"page_id":"168830074"},"emitted_at":1674149695522} +{"stream":"survey_questions","data":{"id":"667455276","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6ms3lgjw25pq b20i2lk pkqw6iy j77i7ux32y8 e323sjcc kh8fxxm724seg"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455276","answers":{"choices":[{"position":1,"visible":true,"text":"nqewt2eyb57 a96u7u 4sr85tfu1 acbvejxvxdui ur1p5sb 4a2r81h0hnw3m 51q3v7s4kllox","quiz_options":{"score":0},"id":"4385138155"},{"position":2,"visible":true,"text":"maso4 ovhyv0ixo hu4gwpq8ky21j9 qo2qc3kuc8n lvou1 a0wbyyp bqr9bre7 csdtfe ya1ltf6kynj93y","quiz_options":{"score":0},"id":"4385138156"},{"position":3,"visible":true,"text":"bjaogae0yct1yk slo3wuygf6 wh0fyn7lym fof1mvu a23qhlj 0rkb9sms 14urdydlb5vht dbw8uh0n3rwdj2i haqry8lhmmpcnmy gw3bvde3lsyue3","quiz_options":{"score":0},"id":"4385138157"}]},"page_id":"168830074"},"emitted_at":1674149695523} +{"stream":"survey_questions","data":{"id":"667455290","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"8rej58igix 5ev85 tiximo qcv1v11x0ixckwa uhte1umn0p v1h2mr0lm6 duw2034nurju"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455290","answers":{"choices":[{"position":1,"visible":true,"text":"xx157diwdtvnm 27fuvnqueywcf2d 7fgvt3b 7qte6 xb8oaf6bdynl","quiz_options":{"score":0},"id":"4385138209"},{"position":2,"visible":true,"text":"7uixmrb45 5217t8r2se vv8oeea62 4mjg3vqh og2vgrc5v251 xe8bb89i iefjxgj 5flbdfdb77 amgpmbo6gav6i","quiz_options":{"score":0},"id":"4385138210"},{"position":3,"visible":true,"text":"1sn25lv32td8 516w18 pdmovblb0hbd50 42vmf7","quiz_options":{"score":0},"id":"4385138211"},{"position":4,"visible":true,"text":"rnlfk p5x50v6jxbdfnkc lfjnc39nl8o ok6oyhwope sr35gd1kr5r8fg5 em4qqf7wj tt8linqt 8e8c6d9 s2geie6vw15ny nbkq71k87eu","quiz_options":{"score":0},"id":"4385138212"},{"position":5,"visible":true,"text":"g0otgfss8 l5ag3n97qp b3hbjigxocewjyi dflt05hus4w ddpumu2h7dx6ff pby1r9n a8d6xu9db rwapp","quiz_options":{"score":0},"id":"4385138213"},{"position":6,"visible":true,"text":"1w7np c3t0x knc262g yrfo4f1r4f4reh i8dhd9l1v77 72eamvjcaggrn9 fb7v3v2g2","quiz_options":{"score":0},"id":"4385138214"},{"position":7,"visible":true,"text":"n5h0km6i 58fjgrqq s17q2dwxiha 9wl01dstrdvo upbsfwpyyryn 5ducq pm1vbp w4no5od9pwqf b3e18e3i","quiz_options":{"score":0},"id":"4385138215"},{"position":8,"visible":true,"text":"o9ln4neod6l3v 5p7w4sosqt95e qk9mir6c48fbqj xlglulme cbafrf2g69p6nj htqcbq8v7u24lyc","quiz_options":{"score":0},"id":"4385138216"}]},"page_id":"168830074"},"emitted_at":1674149695523} +{"stream":"survey_questions","data":{"id":"667455293","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"virtcpk896m68m bbdg4mss12ps3 qo4gk153jqp9 iotdms cxo7l2t9 gtofci5cg0er"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455293","answers":{"choices":[{"position":1,"visible":true,"text":"1vflj0ggk1o9l nbqw3aj69wprwix i955uorj 69jyg2ci6m2lj sb3sor4o2 jmnjbw3i f6weyfas9eq1 yanlggtlu8823f s7fpa7","quiz_options":{"score":0},"id":"4385138239"},{"position":2,"visible":true,"text":"mgeif2h8oilulvj 4q5x7owpgk sxoc48wbr7u3 rmt30uk4m7w7q hpqj8faxcl3qrc5 qfuhdde a1itv i3292al93b","quiz_options":{"score":0},"id":"4385138240"},{"position":3,"visible":true,"text":"3an10n o05hsocsiq0quj5 9ay5x f6p79mjl jjcw6ym9dpfwrj sygnf5 bh3mo 1x146ti6 x4u0e3pxa1ko","quiz_options":{"score":0},"id":"4385138241"},{"position":4,"visible":true,"text":"va9yadd6 xsc7t62edxbwl d7d7n7ecqsealn u9ognb1ox nmaht pwy1d 5mdngtxn4ol1tel dxlmr67c00e hw11e9xn7h","quiz_options":{"score":0},"id":"4385138242"},{"position":5,"visible":true,"text":"b1qfaxqnj91j8mj edwip5b22pdd tuh6g5uodx2 sn4e9lv7xsuul jxmu0iubodnpw 7rqts1liyv27j","quiz_options":{"score":0},"id":"4385138243"}]},"page_id":"168830074"},"emitted_at":1674149695524} +{"stream":"survey_questions","data":{"id":"667455297","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"lq8cc4kb4wb hagmv535tvyfw4 q505o lc9pke7la"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455297","answers":{"choices":[{"position":1,"visible":true,"text":"gnig56iqkr f2p71ed2w9fr gnpyvvl1 9mc6qpbow5tam mxta8 t9blqkndj9c 1gisvkrl2 0p43lebad 6v56y0e392el ahfwj9tq6lbhm7","quiz_options":{"score":0},"id":"4385138277"},{"position":2,"visible":true,"text":"77wwt btv0ntp gym5s54 lr3ji8wtg4dd qoy58mimj d2yjili","quiz_options":{"score":0},"id":"4385138278"},{"position":3,"visible":true,"text":"9hphq 5qj6yrbg2na mnmodl22e8cg siook2te8gpl","quiz_options":{"score":0},"id":"4385138279"}]},"page_id":"168830083"},"emitted_at":1674149695661} +{"stream":"survey_questions","data":{"id":"667455299","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0cphg3eq5u jtapt0bso07ghd 0bgtvsb 5pd5xfhq5t1fgf w0jm2nstiu 93om3crky6skr q49leuh249 3q2tkvncda03g 40orw7354cy p8eku"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455299","answers":{"choices":[{"position":1,"visible":true,"text":"mrhrmoj8c c8fd51r 0lbuywjnvnmijj 0tmyxfrg fggvqmm3bvivav 8b3elaxwyb d2wp2i hr0jllmfkd7 prv23lvvws27gx 0e6okfhy5sn","quiz_options":{"score":0},"id":"4385138282"},{"position":2,"visible":true,"text":"mv5uq8u7dup2r lfo2ih2jkc5cp r6jydm0x6w 1p1c3s67p57 7eulrlih 1v7p2vig8 99esod2scbs pg87n9lp9mg476 6hmjxdey","quiz_options":{"score":0},"id":"4385138283"},{"position":3,"visible":true,"text":"ahxj3 imh6reai78juny6 bop9te8ej8q6l gwxwkjup43o6tr1 nbvorbchco4ptow pwomv9iyd9t jkrjgggo3s 6ipaxevsfrxrmtw uq3n0cmg k1odeemd29l","quiz_options":{"score":0},"id":"4385138284"},{"position":4,"visible":true,"text":"t4pu3i ixvxd q10uqer3 gkqtljjmflbts","quiz_options":{"score":0},"id":"4385138285"},{"position":5,"visible":true,"text":"xwwgqr 7hlt9dq 1tloksa kehvt","quiz_options":{"score":0},"id":"4385138286"},{"position":6,"visible":true,"text":"2vinv5qis ipjdbl cuwxgei6t8g is2ihbn xs3q9m3rl","quiz_options":{"score":0},"id":"4385138287"}]},"page_id":"168830083"},"emitted_at":1674149695661} +{"stream":"survey_questions","data":{"id":"667455301","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5h0aahwgk8 eoh0xct ytmtsdr15y fawlco duk30p0qejro"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455301","answers":{"choices":[{"position":1,"visible":true,"text":"1ib8lq0i q896a iwg01totbr1c8 nic9ye883le 9pjfihiuxbp5 6wu9jitrk1 k3kvcggtbgboo9 nenddst 1qgr2y ilmy2v1ddb9","quiz_options":{"score":0},"id":"4385138288"},{"position":2,"visible":true,"text":"05nxifhb7u4 v0tu1 i8u8dd63ekyj h3xah1h0s7k9vn m95qi vmfn7l5i7iu3es hrywpnsl6rp","quiz_options":{"score":0},"id":"4385138289"},{"position":3,"visible":true,"text":"1fgt8f3w 2i0cyt47w3sl6o k8x6l8i3stl2cc 2h13e9tt1cwaa9 e5l78fvc x2y42gkhqhkc7r","quiz_options":{"score":0},"id":"4385138290"},{"position":4,"visible":true,"text":"ye6nyey79pu2 596slmq qgawtw10v06mtad m113i7i8sd1l x9o2f","quiz_options":{"score":0},"id":"4385138291"},{"position":5,"visible":true,"text":"2mr26jkd368pv wfxs2sxlxag3o3 o15fq 5u1n5tdvs7j0","quiz_options":{"score":0},"id":"4385138292"},{"position":6,"visible":true,"text":"ffdpeyyy nvtjrvxqnqmr f2jwutj 5uw3e0w4n2h dmah35mk v979ctn 2s683h24","quiz_options":{"score":0},"id":"4385138293"},{"position":7,"visible":true,"text":"245tmey w2ltcq50f sktfit h9ymojx j3xrggyo 51d1y","quiz_options":{"score":0},"id":"4385138294"},{"position":8,"visible":true,"text":"lv4qkg6meoylx rmlf7cdb1aht r316f1u kfcwrh5 cm1m3 s5x3eqj3t v1h721uqo3k5km7 9n1oqah9","quiz_options":{"score":0},"id":"4385138295"}]},"page_id":"168830083"},"emitted_at":1674149695662} +{"stream":"survey_questions","data":{"id":"667455314","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ux15jxtnsx3 vofmkp85 a5kvupg5km6vq0 t5g3uf3q7hn i661htxcb s7x7r26 orjn3oisiik"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455314","answers":{"choices":[{"position":1,"visible":true,"text":"9fr03iy 9xmpt6xgqwpc k6t7k9gp1ht1 6h620md9wd70hh bs3kw i9htv yil47l xn823m6ih","quiz_options":{"score":0},"id":"4385138315"},{"position":2,"visible":true,"text":"it0ef6tr6h9nt t2jwq18hq3w59p4 oyx7e1bj86bcm 02aaa4kvmf8ru77 ssmts45dadkf gl1spgihc4 acylm 0vmvvgxv6yvey0 8xi073ec2m5","quiz_options":{"score":0},"id":"4385138316"},{"position":3,"visible":true,"text":"8m4mr5q 7wkbq4t8vplwih 0cvqrnnt qt9mjry1n xyqbuaepupf3 ed52xu5ak bd1vxipoo5ad s7pxs874 a9imnp7nm","quiz_options":{"score":0},"id":"4385138317"},{"position":4,"visible":true,"text":"2clmq75t 252dtb2ce3i fld27xux vrip5ox3ds8qnb lvp972rcrcjc ruvk2sclvimuvx 1ud7hrsbm567","quiz_options":{"score":0},"id":"4385138318"},{"position":5,"visible":true,"text":"0l86fyh8uo 090ymhll1pq 9lkwl89vq f1pcky3lidacy9 3ecmm11niu fu8tfp","quiz_options":{"score":0},"id":"4385138319"},{"position":6,"visible":true,"text":"h37f95j7qxup0fe tbx1l3bgii ol4bri0itarcwk doh3p2p0pi jq9guw3h382 08fje7vyonhmfe5 s2ioi7c4v6ci","quiz_options":{"score":0},"id":"4385138320"}]},"page_id":"168830083"},"emitted_at":1674149695662} +{"stream":"survey_questions","data":{"id":"667455318","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"1gm3ed2s89 p2xfqhaj4r p7fwo6 12imab59cds2p aqmmilr2dvmwvky nljfs ts3g4cw6au9jii snbv40hbjcu3"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455318","answers":{"choices":[{"position":1,"visible":true,"text":"dfyvo2qlanm8s ikbv3wa7030 rmjt80 o7cym2 1r6qae7c70v30 fko724jo7me82s1 bsuhjbov2ttwr q3w1wpn5twsv5e slrnk3sx m3u87rixhv6nmc","quiz_options":{"score":0},"id":"4385138386"},{"position":2,"visible":true,"text":"f47smf7vov8sp1 24r5d2b6q4s duka34dqpn6si 4r2wn 92ekkv2p794l8h l8n6cdc","quiz_options":{"score":0},"id":"4385138387"},{"position":3,"visible":true,"text":"okl5ki7v1r5 4oqdy4 x7ny0qmas 0ddqlr1 ja5wspe 2ieqa m3ucowjq1krai","quiz_options":{"score":0},"id":"4385138388"},{"position":4,"visible":true,"text":"gt5y6 vpjp0e5p6 vqhwb2dytiuihsv ru25v6bm mcihbuved71h2 quy2rej9e8eb97","quiz_options":{"score":0},"id":"4385138389"}]},"page_id":"168830083"},"emitted_at":1674149695662} +{"stream":"survey_questions","data":{"id":"667455323","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"wv0en qigyrej bappxu8q j3ihl9p6ki"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455323","answers":{"choices":[{"position":1,"visible":true,"text":"xv1me9s6hd aihplmw6 plhv5b k52pvh68j so1ggnlishy5m 6qe1fhw75gt k1jxdmddlhj35en","quiz_options":{"score":0},"id":"4385138412"},{"position":2,"visible":true,"text":"7ecrthfm97aysv 8u9a9sn1f9kj8 90owtm 0h8qgota7j3qpn vnfs9vleja36","quiz_options":{"score":0},"id":"4385138413"},{"position":3,"visible":true,"text":"v519ikhgw0fl s8x0shqsi ssr005 8xm1b7fal622l","quiz_options":{"score":0},"id":"4385138414"},{"position":4,"visible":true,"text":"gbpim9ar0dfgi94 nj0mq3ejst csj5e 763j6d5eo gf4fvw0 s2ea20n33yo iqd5r5l9 3t0okvw2oyh","quiz_options":{"score":0},"id":"4385138415"},{"position":5,"visible":true,"text":"jyxp0s7xfc7td5y vom52gda3dxr ko6256dtc5nv5f 7s8nej n32hyka ywsoxywn","quiz_options":{"score":0},"id":"4385138416"}]},"page_id":"168830087"},"emitted_at":1674149695662} +{"stream":"survey_questions","data":{"id":"667455325","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"jdkmr4b5t3o9qd6 0u4huyxr8whu cxqri4e2i1a 88yyx cq3xbymudltf y6hmsrn4socbj"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455325","answers":{"choices":[{"position":1,"visible":true,"text":"5aacsmqn 9p8l1tvy9i 0f92rvnxn9 to5xiaghqmiw 6xik80 jxvy64ut9 bjk3w6ywb","quiz_options":{"score":0},"id":"4385138421"},{"position":2,"visible":true,"text":"bc85jh39p 7nskr87a3x ny6d4 jlbavo9t8h6j6hu dh5ne","quiz_options":{"score":0},"id":"4385138422"},{"position":3,"visible":true,"text":"3vykabcspbl7qt ajqad2gu3v0jbg yeo3sobedrdfe0 1wcrn35 5l7jwq3 f5bc8bx","quiz_options":{"score":0},"id":"4385138423"},{"position":4,"visible":true,"text":"sm6lx7btphddbw 0882qf4o omh4u2i446c9p4q 5hqyq27jlse1e7 ns3xkqg8gcx8pc sjd0skhv 9ydxkadh8e814j7 0mk8m5tm9d38e 23cvhf22g lxb5c7c3p0oe1","quiz_options":{"score":0},"id":"4385138424"},{"position":5,"visible":true,"text":"2x7f7xmgur5a 6mn40jjs2dde 3th2mj8cn dv2pbu6s 7n8hw1f ptapt6nxgddk7a 4cj77u6m3mm idvm31 mx9ygnq3i 4lw1gmm4cwaig","quiz_options":{"score":0},"id":"4385138425"},{"position":6,"visible":true,"text":"3q51n sofjjqlu2y 6088c4c ncdkdt8exikoiir ew86v6gkob94v 7jsgkctqkhm1","quiz_options":{"score":0},"id":"4385138426"},{"position":7,"visible":true,"text":"7lffwkal h4d2j5 dqjf3y5 jwopu 0xf2vqmb6an igo5ri3px747b 0l5s9df7w7s","quiz_options":{"score":0},"id":"4385138427"}]},"page_id":"168830087"},"emitted_at":1674149695662} +{"stream":"survey_questions","data":{"id":"667455328","position":3,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ieffti0yi qcxktrskkug9ij ebvs68ni79 1vrpnp mobkmem70 7uc86c7sx"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455328","answers":{"choices":[{"position":1,"visible":true,"text":"7vvg5h tc4vy 5c5194t1n6eu wutfyil","quiz_options":{"score":0},"id":"4385138432"},{"position":2,"visible":true,"text":"17dh5pnpe3bc 6gmy6r15tq7 fuwlrcnnt61wotb mj08g80e2pri","quiz_options":{"score":0},"id":"4385138433"},{"position":3,"visible":true,"text":"414pty e5a4vs1 4p6122ihy qnjg9a xuowh226f18 hc449ct tdwm0wu29u","quiz_options":{"score":0},"id":"4385138434"},{"position":4,"visible":true,"text":"24k06ig dhy3huyx0plis 0n5vomlwuo38j 0nft5aw obgn4qcoq0l44b l5dtviydcom58cn 699vqm6 06i0mr52i0 u1mvn mm1bqovxpvtwkok","quiz_options":{"score":0},"id":"4385138435"},{"position":5,"visible":true,"text":"3fr4ttmtr3mhvr d7xcfp6tx48mne3 ttcyvypnom ik9eqkf o7q5x4veph 31y5w8u036c56rh 9yar58y9t5d","quiz_options":{"score":0},"id":"4385138436"},{"position":6,"visible":true,"text":"jrghjb5g3h6t dlpx7hve7lijy 77ergx421ad dgekp5dauuod5t 3mn6a 7m9wvlhvgeua 5orruhepinotb hd948u958 23p4f fmprms","quiz_options":{"score":0},"id":"4385138437"},{"position":7,"visible":true,"text":"dvjf23fp1f8slys 6120e2kbl8p1 f2ildddc i9ocnxo dk1c5jm5bx3 1mmcj3qmntljpbt 889694rivh72g07 25yrmna iwjlytheaogoxq fanj2","quiz_options":{"score":0},"id":"4385138438"},{"position":8,"visible":true,"text":"f2tlh hih5om u7aqefshc47ph sxxt22yg7hi","quiz_options":{"score":0},"id":"4385138439"}]},"page_id":"168830087"},"emitted_at":1674149695663} +{"stream":"survey_questions","data":{"id":"667455329","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"96mfgjfkj4g 1roun x4g5tcrq d52byhs855 cjc897qm8l udgjrsby"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455329","answers":{"choices":[{"position":1,"visible":true,"text":"2ny2fhgpm6 aturc40fuggi de2bnbhjnc a8jmpcw54h3nu pij7dkc3 sbrbi0rbi40ox2 wskx4gyt7aoc65 coi4cfe4y p44mj88ikx 93sodjvxi7ny30","quiz_options":{"score":0},"id":"4385138440"},{"position":2,"visible":true,"text":"t1f53t6jo 5s7kaoe ma28g93gfjjsm nwfjlfr2 h4k3jav 4xvidc3fv 8enehee7txlhvp ca6vkoxrb465pi xmi35um8q54r6 4rvruqr46m","quiz_options":{"score":0},"id":"4385138441"},{"position":3,"visible":true,"text":"i7iw5 ap5gjyafbm2l adp3lcc1 52hc8 j6ldt5","quiz_options":{"score":0},"id":"4385138442"},{"position":4,"visible":true,"text":"468nunilylfthe gwy8lhtgga9re4f 5xacgti673jfgs 3eei9s6qwg4avy ksqn6cwpvl585 wtyw59jhy7kck m462cr65qglmq ev7c0b5","quiz_options":{"score":0},"id":"4385138443"},{"position":5,"visible":true,"text":"yxrha98q xgjtmqc6x6tbsq l5co87ln2j3044 49lpv4 l8rfhvt2 rp4v9ofww2bekc7 ops08osul9","quiz_options":{"score":0},"id":"4385138444"},{"position":6,"visible":true,"text":"dbah1h80x 07f9vbs n89jmtwm0t2 47sd0ilc umky7iesp5j1ye f825fm7sn5fteb","quiz_options":{"score":0},"id":"4385138445"},{"position":7,"visible":true,"text":"9ybbt8x1xk o12xtb esgrab5p169kpou jyx54 456l76rs9f 3pcmlfoju rfyofv71 lb7gr6gi2ab0 gmexy","quiz_options":{"score":0},"id":"4385138446"},{"position":8,"visible":true,"text":"k3s5bwg5b2q 0ikv741vhxu3x4w efpp0p21i1s 44ca0fl4bklmn","quiz_options":{"score":0},"id":"4385138447"}]},"page_id":"168830087"},"emitted_at":1674149695663} +{"stream":"survey_questions","data":{"id":"667455332","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"v80ku mx1co9qwm34tat e9i3mdjcvixs grggkttt lhn014lmqj86 achjddrt9o5fx 9v07aged4niq0g ye1woaclolxuaq w5e7jclooee"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455332","answers":{"choices":[{"position":1,"visible":true,"text":"u1e4v4rx27412v v9mjw7oaf 0t873cte 57y89l xdqtturimm5 b4stpodx65s8u 2mk2es7jwrpn 7enb3sp29","quiz_options":{"score":0},"id":"4385138468"},{"position":2,"visible":true,"text":"w0mid5oagg qx0e1bvil6w5v6 0cadbm51x7hbg rpxgn9yni","quiz_options":{"score":0},"id":"4385138498"},{"position":3,"visible":true,"text":"7rex013 m75tu1nu9orrc2 k4du9rcsy2n5l0 cfbiki1u6cp2f qe81rjnguphrum j4019 tbl21q37","quiz_options":{"score":0},"id":"4385138499"},{"position":4,"visible":true,"text":"sat97b1hk6dx9k uq924nht7pr8cb 7nr3h2hclmiqg txrkxr29wtrc217 bxmed4ll1b23561 vvsu7 x293il lrl3e","quiz_options":{"score":0},"id":"4385138500"},{"position":5,"visible":true,"text":"h3yh19ckoclpq 0hb213i nj1mfmvbj9 p4ibgetarc6h6u 8kahs","quiz_options":{"score":0},"id":"4385138501"},{"position":6,"visible":true,"text":"xyvg34bae2 7u2n4l87h aec3h1sy5aw62r 60yajbqvxifw65 c7q9ty4pdby2d vyjp2 n7tavs0550g46 07p64c9pp8oo","quiz_options":{"score":0},"id":"4385138502"},{"position":7,"visible":true,"text":"kn8f11mlx sucpq9a 79n0u6vi1tgt b0dom486a929h ocblyvsm6 ti4tnjv533","quiz_options":{"score":0},"id":"4385138503"}]},"page_id":"168830087"},"emitted_at":1674149695663} +{"stream":"survey_questions","data":{"id":"667461429","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vq7ho4rb qt0fpw3 fvn1b2y21n fpkcw9v73dqvfq0"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461429","answers":{"choices":[{"position":1,"visible":true,"text":"p7fu5tjqvd qgxdych hv1xcryuq7jlia 1qjuijk40 2v6u2e kwqcepbnx 2hjta3wqc je2wbyr6337d5 o0fq46u48nsa as5jyqahxo","quiz_options":{"score":0},"id":"4385172931"},{"position":2,"visible":true,"text":"tubba qc6l28jc4m1xi 5lf38au3ry9d o9eh4us9ki ul832 jippjf4 bcsoug6 9w7mnssanfsknl f422osb58c","quiz_options":{"score":0},"id":"4385172932"},{"position":3,"visible":true,"text":"juswx5 78b5n495 duu5kiikm feknbdqqtxg memsnyrcmao1lh","quiz_options":{"score":0},"id":"4385172933"},{"position":4,"visible":true,"text":"xpu48c 3cdbygss10 ujxmb ore7vx7o0x 9qmlb9fig6p3u w6c8oqr5dhp1l g3ihi9p1x kf2lvtbxo 4guw65","quiz_options":{"score":0},"id":"4385172934"},{"position":5,"visible":true,"text":"i9508b8n ld7powh72 nfvmljfhgn3n 643ydxghpbak7v ehe18sjo56yx m1bpaoj4epr 5sv5aw6 7m0bt","quiz_options":{"score":0},"id":"4385172935"},{"position":6,"visible":true,"text":"vwtr9m5fu3ot ltiqhsx3fi uuylf62qec mmn5fxqj","quiz_options":{"score":0},"id":"4385172936"},{"position":7,"visible":true,"text":"ww4qdi5pqg 3qvmj8g0yvx rrr5fx06d7 t6giac3k8 t3d6exqx175ft 10k251y 47v1vnu938 kkqqcsl50d7i","quiz_options":{"score":0},"id":"4385172937"}]},"page_id":"168831336"},"emitted_at":1674149695747} +{"stream":"survey_questions","data":{"id":"667461433","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"3bdmcu bth1w4ifhtcbm slcl3p6ynqhjtyp n5nea bdq5iuvq77m0t"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461433","answers":{"choices":[{"position":1,"visible":true,"text":"26j27d 5s3pbf08free77b ttq5exq7 n2gm99qkada 3rosqp hn74g juh2ww6","quiz_options":{"score":0},"id":"4385172948"},{"position":2,"visible":true,"text":"phmu9ghnjc vkh0ury2y6et tqj1lwjv 37m5bx1itv a5x88phj3g5 832qd1l rpbrunm7v49 kpmqo a8eqht49077t","quiz_options":{"score":0},"id":"4385172949"},{"position":3,"visible":true,"text":"jbsjkqa2f2xb ks824l5wnkd b0p0elet784r6rw ay5sqsv2vdron lr8mx7r6pc im42wmwt ltyhtay7p8u","quiz_options":{"score":0},"id":"4385172950"},{"position":4,"visible":true,"text":"ikkvwye7tah 7shp28 p7k7je0as5u sdc3f1rvin staohk6a44k nqjsxn3 97bog4jfxn qvorj30xpuh1gip","quiz_options":{"score":0},"id":"4385172951"},{"position":5,"visible":true,"text":"p83chdtsw6s88u 0hqgb8h871mw p0ej9 djy316qsa7iv6pe","quiz_options":{"score":0},"id":"4385172952"},{"position":6,"visible":true,"text":"kgfa1 tyxdxoe3gc xclen0vw9oa2 6bu9o8b6awx hcg9pgsi1av9v 7dicbt6wsee70f glwmxxtcdr 4qi7m9p8tfkxur3 ir5jr31364","quiz_options":{"score":0},"id":"4385172953"},{"position":7,"visible":true,"text":"u6xyfnav qw5qefsi 6ttsauh 3jwvmju8sdjk87 bql4ra2ww 8nxxobw4o58 omi87y6ur8l1f2g 62gaxuq","quiz_options":{"score":0},"id":"4385172954"},{"position":8,"visible":true,"text":"xdjcisgicnaix2 fj8bpbqaqgntr fb2n0o73 mgsb8xg5x3nfg9 9t18omvng4p6e 06wepywm4wku 82pemp2 l3nsu2ib2erbva tj475a","quiz_options":{"score":0},"id":"4385172955"}]},"page_id":"168831336"},"emitted_at":1674149695747} +{"stream":"survey_questions","data":{"id":"667461439","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vrhftdyyte qir7jajr68td obg64x tu1rcy2h pqnxtdrwxk00a c173brbv 6qfxck7huyx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461439","answers":{"choices":[{"position":1,"visible":true,"text":"q6705n7lidhb 0gsqn8 ocnyaudmo ulp9r0rsoheh4f cjwcqblbh o28tilm13w384","quiz_options":{"score":0},"id":"4385172990"},{"position":2,"visible":true,"text":"vyrx1jpjcl07 etxm7itb161 51rf5csw6e5tsh 3ux7rxq2 vct6fys4r7","quiz_options":{"score":0},"id":"4385172991"},{"position":3,"visible":true,"text":"pp1lc12uhii 2qmm3xnfsp i9912c8ac5k74i hxew5625hxtm3 4o658 jenbgi9o89 t4ppqc2qvhlui iu92ym nqjkka1i7","quiz_options":{"score":0},"id":"4385172992"},{"position":4,"visible":true,"text":"6mx9j44l0oa yv2wb1letc4p d3u87l59 vp7a65ykcfjyt tpe3k92l y2flusuc3tc t220oi sekyd","quiz_options":{"score":0},"id":"4385172993"},{"position":5,"visible":true,"text":"4fbdo eli65un2emx e6oyl3a41ugoxb2 saxpsxn6fv l8s2mk0e57d60","quiz_options":{"score":0},"id":"4385172994"},{"position":6,"visible":true,"text":"r5ly6h 7eu4pmnx7tv04g rdtjvlsup3gdn h4qoreg 3ct3fudxbuuw2 b975v8 ilsfmaa22","quiz_options":{"score":0},"id":"4385172995"}]},"page_id":"168831336"},"emitted_at":1674149695747} +{"stream":"survey_questions","data":{"id":"667461441","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5p8n613hcmidh w4gyoyf stviro3om xey832w t9w23kj3j4rp5j h2w1swynkq1n9e vqrk2q5eb76p9yn pbj778q19u1hqy r9uvl76qqhfg"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461441","answers":{"choices":[{"position":1,"visible":true,"text":"tmdpekdn75l9b 3nuus dqbdxtj od2mltbx tmw5kcvpiw 6n41343132bdmc kb3i9er4qg","quiz_options":{"score":0},"id":"4385173020"},{"position":2,"visible":true,"text":"6a8efsocxmc eeohurhhyduo31w c8na1dub0ycx q8314ir twgrq 2ukcrksjt30s","quiz_options":{"score":0},"id":"4385173021"},{"position":3,"visible":true,"text":"ou9m3xl2n9wvn m0wews5 il8o8so pygnm380cd66 7nhjkpk9lu65n 4e3ifrwcb8wr 46bo0ani86m du57mphcnvf1in 6gf58fwm2c50tp","quiz_options":{"score":0},"id":"4385173022"},{"position":4,"visible":true,"text":"nahvw2sd3 hn5trfgqbuso cq82jp7 k8ev0 s8a0a23m0p g2jewuy0wdadgsa hca0mm5q 8agnm fxnf8vrgdybkg04","quiz_options":{"score":0},"id":"4385173023"}]},"page_id":"168831336"},"emitted_at":1674149695748} +{"stream":"survey_questions","data":{"id":"667461444","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"khys853d7u tgmr69e7mu7 0h6sn7 cja88 nj2os3 wnand9tdmohxca rowe88asxsja1dn rvs12kt5m0wqd2 41pwt1vnhst008o"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461444","answers":{"choices":[{"position":1,"visible":true,"text":"3k875epq o7u7f5 kihtesav7 qwxlu7j5dj59i 8ghsap3n","quiz_options":{"score":0},"id":"4385173040"},{"position":2,"visible":true,"text":"cemvy 140ek4 d1fumxxlqqnyt 99pb868kg2b hqur2f44k uwr5dyif8g6","quiz_options":{"score":0},"id":"4385173041"},{"position":3,"visible":true,"text":"iykvyahtufq91l bvs9t6mmoyw ld2hep1somcl7 2rs7wk3q 5ncge9mfj1ac r30355p0g4wt c1bx5 n5wgmcud6h","quiz_options":{"score":0},"id":"4385173042"},{"position":4,"visible":true,"text":"a4xcahm1euf l3h1jx abcc0r500rlqyhv 9w01sfmci3j ur9vp3sxfioh r0dksavcmduhk2 kw7nbl 5hys2r8vebx4e 94rajuno uwm3ajywh3vlqbf","quiz_options":{"score":0},"id":"4385173043"},{"position":5,"visible":true,"text":"0whm5dw5kuk44 efrr6i iu9592 a968tp4ff0uaf q0f6mtpcp2x82ae 97m1gvsnfthibp agt6dm3 ip1e3y9","quiz_options":{"score":0},"id":"4385173044"},{"position":6,"visible":true,"text":"hwbtcf7 60gji9qoeovhlf 9b3ijm8 xiw5w09k 460v2o27hacdts 30eq74bg9m qpfv2jd9f3ur rlacn2rf273ck7 2welkh7188h","quiz_options":{"score":0},"id":"4385173045"},{"position":7,"visible":true,"text":"cbqj2nlsnyy tmjyvoija wirje1e bxsqxqe g12fuxmtgfq 7fc74o yrdsjey 1xmv6u077j7l 8hkf7","quiz_options":{"score":0},"id":"4385173046"},{"position":8,"visible":true,"text":"uqlus j3nd9u8g faqa6ghioy60h9 a95spokyj58 ndgctguy75jr ei16b1p7jabc2 hfkfpruds","quiz_options":{"score":0},"id":"4385173047"}]},"page_id":"168831336"},"emitted_at":1674149695748} +{"stream":"survey_questions","data":{"id":"667461449","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"cwipmppqgy2vy 3mn05n4 gedv0twor lyawl9v528587w wfmifrr9hk pykvx52nu1ds5p hh28niyix"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461449","answers":{"choices":[{"position":1,"visible":true,"text":"3gjvb8l80nc8j 90ebdlnd3hg spis2ipbim9l fjt4xli4a 0bqmti2wp0juwcl cdwl64u2yfghjdf 190uek5lw","quiz_options":{"score":0},"id":"4385173076"},{"position":2,"visible":true,"text":"xjdswmkwm24 rrmxo7otpear8 ogepd7v8br0y5m vi2vyip49ux uvtnq2sem 5gwcppvvvc bw56us8o 1huql09g6pa1ylb","quiz_options":{"score":0},"id":"4385173077"},{"position":3,"visible":true,"text":"gyq16tgah s669s7 j5isg9 l63lf q61pyy oywk58jfmnvy 6lhnukrce1px bwt066s8k5248o sylwt em0x3","quiz_options":{"score":0},"id":"4385173078"}]},"page_id":"168831340"},"emitted_at":1674149695748} +{"stream":"survey_questions","data":{"id":"667461452","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"aku2037bp sxtf1busk2uj 7scwpc00giyau urduwqdsyr3 xk3y39yjywei1y uw7y5s3ky fetwdqr6n qp00a8rofpy 4rwl41yk78jd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461452","answers":{"choices":[{"position":1,"visible":true,"text":"86hx9ifui6c4o 02xkn5ghldfw35k faipmm g7634wrx1jfvy jewjfdq kkyyxx00brj wtu0xrhfhvu5lv8 573fv0vtm0n","quiz_options":{"score":0},"id":"4385173088"},{"position":2,"visible":true,"text":"nfeyvnb0kat1yb0 3lk3rse66t8jp w90vixgpu32cyir q03jxd8vch9xn7t 3a0swht9ykadh8 fvoa9t8dri fm0twcdhxm 4ot9xpxe9s08b ho7d1","quiz_options":{"score":0},"id":"4385173089"},{"position":3,"visible":true,"text":"k5pyc4y 9uul1o 7lvigo 7xsdo49jx2yc d07jr1w4","quiz_options":{"score":0},"id":"4385173090"},{"position":4,"visible":true,"text":"3e49s5 gxqwbv w6c2e tjaf7rpgtksjpe ivr0he574 ft8qso pqq4l5hbchy 6mhw0ksgrh xbj4l7a2g","quiz_options":{"score":0},"id":"4385173091"}]},"page_id":"168831340"},"emitted_at":1674149695748} +{"stream":"survey_questions","data":{"id":"667461454","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"4i1qajt 6sy8nl3hjougqbu okftpe4sw0bpr6 gv1pbyk8km 9ihd103uu2n3lx c6spvurq080unx uswpaexarx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461454","answers":{"choices":[{"position":1,"visible":true,"text":"vg0vpi1 100y8 03nnh0 947rcth5ofg x4xfc38q81i6e7 bgf20442q i2jh2 lolt43y3ri2 vl0t6 2rkshapnf","quiz_options":{"score":0},"id":"4385173102"},{"position":2,"visible":true,"text":"o7qfwgqys eyumel 7spc5 42qxefd","quiz_options":{"score":0},"id":"4385173103"},{"position":3,"visible":true,"text":"usb43wqju6w csearclgaedae 2oo3m2a79oo5si granw6hf 9g9ms37peouy gqaxry74 x45yqa0xtkcu g0in9 lyp7xe6 few8vkh7yf","quiz_options":{"score":0},"id":"4385173104"},{"position":4,"visible":true,"text":"spfrf5sfebrhj8 gwodgm3o1 m5cydcxbtk2 pgrj6h mome4a us97ellx2peg3s ilidjy8juu0","quiz_options":{"score":0},"id":"4385173105"},{"position":5,"visible":true,"text":"w579aalpr5gaj1 1h2ud 53d80pebt4ep 0l2gw8fk7fa","quiz_options":{"score":0},"id":"4385173106"}]},"page_id":"168831340"},"emitted_at":1674149695748} +{"stream":"survey_questions","data":{"id":"667461456","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"is9uyf6ka iel16fh 1f0xg 7mx16glcei fygrn g7l10em5fbeybb 1vgps0k3 q4fls qryybq07jy f3luik1nx09b7so"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461456","answers":{"choices":[{"position":1,"visible":true,"text":"0fg2b3v4 jmdxep5of1y q2ag243 cfytydw942toqsl 2ixd8iynbs eg22ia","quiz_options":{"score":0},"id":"4385173113"},{"position":2,"visible":true,"text":"0am3pervuj biqdk6g5yw i5xvk20h1n6jv l0v9m5cxn 2vyqvcp8rlthxfv k0v6o48p3v8 551pws1020t98f","quiz_options":{"score":0},"id":"4385173114"},{"position":3,"visible":true,"text":"cjtogxiie0arj ltkgiwrpoa4x1v foqfr1gk406a rale1dx 2p4gjy2g","quiz_options":{"score":0},"id":"4385173115"},{"position":4,"visible":true,"text":"h31va0f12qx3vg 51m9g4bo9 r4ofiqr j1yym2ma8m13kch 1e4jxhdyol2ny nf3fh0h2e qw8euwei1lyhemq 3j4un2sjdoj7 q92100o573tbom 5phvt3n22","quiz_options":{"score":0},"id":"4385173116"},{"position":5,"visible":true,"text":"h8hktik e03ehrspn7 rnjqq4431q2 3bs1nevr8j0 audw020tl kalstea2","quiz_options":{"score":0},"id":"4385173117"},{"position":6,"visible":true,"text":"9w090 e6835j0fvfg89 kk6swkb5g oo0f4ho99x437 qf7b8y1aaa4 i0r03","quiz_options":{"score":0},"id":"4385173118"}]},"page_id":"168831340"},"emitted_at":1674149695749} +{"stream":"survey_questions","data":{"id":"667461462","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"r1kf2oglhju ou2ldq63qu2g1 ufs1pkw pgr9p75sh9k7 vnu477cixnk1kx nxnfn0ti3x3u9wh ew6ho e18cmaeovxhx h1fac8j8 8ni6ay"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461462","answers":{"choices":[{"position":1,"visible":true,"text":"xytk5fnqv6odvms sw94ohnav5npnm lg9uf4 iwuqgfd0 5ee0wd5 lfha63ve x55yip4 gxgdkff3sckdn mq1khupscjgqj 8mtp54i5c3rjonv","quiz_options":{"score":0},"id":"4385173167"},{"position":2,"visible":true,"text":"2ammt8omj0l c0i1q 1uvaf203rh1 8wj2w7pp qlqayl9e8ldc y7iivv cv189py2di xuihxup7b2 rh8owrr595st","quiz_options":{"score":0},"id":"4385173168"},{"position":3,"visible":true,"text":"o9cvkj8x k0txiswxc5 ogf66jujgcrwdb l7n0c0rodcx 2gduko0wwimb21 afw8mi","quiz_options":{"score":0},"id":"4385173169"},{"position":4,"visible":true,"text":"mcb62sefmuo plnbygilddeqg u64kkkjvoms4b5q jw4tashu6c7ve12 8di4g100598 ad1bet nnqd7jmg","quiz_options":{"score":0},"id":"4385173170"}]},"page_id":"168831340"},"emitted_at":1674149695749} +{"stream":"survey_questions","data":{"id":"667461690","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"53o3ibly at73qjs4e4 y9dug7jxfmpmr 8esacb5"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461690","answers":{"choices":[{"position":1,"visible":true,"text":"lg2mcft4e64 ywiatkmeo ci3rr4l2v0 ot6un49a 4b28sq4g8qv7tj 4ihpko73bp0k6lf swaeo3o4mg2jf5g rnh225wj520w1ps p9emk1wg64vwl","quiz_options":{"score":0},"id":"4385174700"},{"position":2,"visible":true,"text":"ywg8bovna adsahna5kd1jg vdism1 w045ovutkx9 oubne2u vd0x7lh3 y3npa4kfb5","quiz_options":{"score":0},"id":"4385174701"},{"position":3,"visible":true,"text":"xsy4kv tqp8vty29815 de8nt5ab2fyr m6jilru2ek l7fktx3j5mbj l33ip83t4p29 exfygne a1btj95m1r","quiz_options":{"score":0},"id":"4385174702"}]},"page_id":"168831393"},"emitted_at":1674149695838} +{"stream":"survey_questions","data":{"id":"667461777","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"kjqdk eo7hfnu or7bmd1iwqxxp sguqta4f8141iy"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461777","answers":{"choices":[{"position":1,"visible":true,"text":"11bp1ll11nu0 ool67 tkbke01j3mtq 22f4r54u073p h6kt4puolum4","quiz_options":{"score":0},"id":"4385174970"},{"position":2,"visible":true,"text":"8q53omsxw8 08yyjvj3ns9j yu7yap87 d2tgjv55j5d5o3y dbd69m94qav1wma 8upqf7cliu hb26pytfkwyt rfo2ac4","quiz_options":{"score":0},"id":"4385174971"},{"position":3,"visible":true,"text":"6d7qmnw obxwg4elaab6 2sby04sor66 1wuoh26aftxu7","quiz_options":{"score":0},"id":"4385174972"},{"position":4,"visible":true,"text":"n0xwexbwtviyj1a midgl2jpfdy a72ut27ta 8i9fmkwg0q mbtxhkn b2ut8mtsslkt609 tgmnd7ovnqlbr","quiz_options":{"score":0},"id":"4385174973"},{"position":5,"visible":true,"text":"qjfs0pmb iecatmqyxtk w1s0fs9vcbayf5 rwsneyp0wx6lsyq pq99n hrx1mk4saug gv06qshlabe 0s2t4 h11ee2xna0m8r","quiz_options":{"score":0},"id":"4385174974"},{"position":6,"visible":true,"text":"11uf3he wbstw etbysmu4 c84vqddvx","quiz_options":{"score":0},"id":"4385174975"},{"position":7,"visible":true,"text":"rnfx7m ndifoe7ihy q98pov78016t 8smlnm lb3xicjp9 0r30sie97y12ve7","quiz_options":{"score":0},"id":"4385174976"},{"position":8,"visible":true,"text":"jc8s2ra5qxytxbu u6tj7jgep95 vbva1b4uslioa omku9","quiz_options":{"score":0},"id":"4385174977"}]},"page_id":"168831393"},"emitted_at":1674149695839} +{"stream":"survey_questions","data":{"id":"667461791","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0qw6a5lnf426 2sh3g9f8wu xmgflj 41pjy"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461791","answers":{"choices":[{"position":1,"visible":true,"text":"7kxk7bkhfdx86sh 3rnrsj70ud 048jbf4qx4 p96o8 sn7xi oh02tfput4 6js84u99m5t","quiz_options":{"score":0},"id":"4385175055"},{"position":2,"visible":true,"text":"x259osu33y 8qadkcxpsnk4o20 m4wo3183nwxhgye q4mpg srpfibk96sf t3h2cx58eji x7l0sdipnjece8 7tgwfdfmh9hgdwi w99mkib2","quiz_options":{"score":0},"id":"4385175056"},{"position":3,"visible":true,"text":"lil1tboe p80wa8yed7w8 cll24c2lls6cc0 gpbv7rnap psk1et","quiz_options":{"score":0},"id":"4385175057"},{"position":4,"visible":true,"text":"wodtghhkt 2ae1c8q5s1ha 8lppd7ko84al j95eq1imtu7 6x8qknrhn0 l7h53","quiz_options":{"score":0},"id":"4385175058"}]},"page_id":"168831393"},"emitted_at":1674149695840} +{"stream":"survey_questions","data":{"id":"667461794","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"q3ay58w3 2rfjgu4 0cf9uh1 pu4fo16w 6c2wkn 1oo7d8"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461794","answers":{"choices":[{"position":1,"visible":true,"text":"1orbs vtqu62x9bp t75k10e89krhn bdnsfy6ng34g 8yv9p1c92jlbt0s","quiz_options":{"score":0},"id":"4385175070"},{"position":2,"visible":true,"text":"5j8dww2lxevx4a wv3ppbb vnccslwrjjdc n5pjsmw m7b4533y8tcbbus","quiz_options":{"score":0},"id":"4385175071"},{"position":3,"visible":true,"text":"fnjqkqy2 44brrpru jllsj9cdggwt4 behkog76y5ua 7ftpd8c8qhblii","quiz_options":{"score":0},"id":"4385175072"},{"position":4,"visible":true,"text":"srjre1h3w9 qojsh5w2 sq7wva6tkl9 raxp5mldrp","quiz_options":{"score":0},"id":"4385175073"}]},"page_id":"168831393"},"emitted_at":1674149695841} +{"stream":"survey_questions","data":{"id":"667461797","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"bvrdxa swsrjt sjox8u6767lv5 wgcomvtnoi0yg namiomuh6cou61u nl2v5bfu15i7 sqpu07jp489uc"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461797","answers":{"choices":[{"position":1,"visible":true,"text":"y97bshsv ite5mgk76p89o yrtt28bmm4jo9 ftc2tnjg","quiz_options":{"score":0},"id":"4385175096"},{"position":2,"visible":true,"text":"r970efm0 5p96h9iy1 o7ft83xrqgsrh8 owk30 buqg6ksd297lw9 lh6ygen9s2rac2b k5d3lbr7m37p","quiz_options":{"score":0},"id":"4385175097"},{"position":3,"visible":true,"text":"ktg10 vp7khp0ucx vuo5qrcor po9nbn6cdpdu56a rt8eiu0umg0dkx j2k8vgtr6","quiz_options":{"score":0},"id":"4385175098"},{"position":4,"visible":true,"text":"iubh35s1gvpm4gj svwbyf7npunm3 0thmsjmt2qb5im0 undxh7b frxykv55emi padtjsk69 qa0jrnwrfoj qqjg6ifvlx0abdb","quiz_options":{"score":0},"id":"4385175099"},{"position":5,"visible":true,"text":"w64hwv9edeaf55 l0gkthucpqj 80wgqsffl 0m45xm56a25psm 8opb8b0gw2w6 n8xex","quiz_options":{"score":0},"id":"4385175100"},{"position":6,"visible":true,"text":"ju3rt297a t028c0b35635 l0kj9vj seuar76 89587qhw46295","quiz_options":{"score":0},"id":"4385175101"},{"position":7,"visible":true,"text":"c4de01u4eil p1p2vy 0gqjglc mc2r97p07 d8d90 j15xktb2idx91 tecpeak3 4anh9o5w7h0runq yr0nd0q9392229","quiz_options":{"score":0},"id":"4385175102"},{"position":8,"visible":true,"text":"yc5erasa3ovk4d ed9adudq8e1s 7wrf8k w9ohrhltg3kv1 wgrnemp 7dqxmy5e bxnsro2sl","quiz_options":{"score":0},"id":"4385175103"}]},"page_id":"168831393"},"emitted_at":1674149695841} +{"stream":"survey_questions","data":{"id":"667461801","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"iu425c2v4yqs04 43g37 wg8awi s2pjwsm vjhybbs wry73cuukw85l2"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461801","answers":{"choices":[{"position":1,"visible":true,"text":"xjortc6k0sxjydf rdusho82tsr3l 3b3gch ogabx6895eb3 e7bj5pq poft6c4g1","quiz_options":{"score":0},"id":"4385175115"},{"position":2,"visible":true,"text":"bscm9v7d9nv0 e5x94dt0402ge i7mwtey74y4 er7bwam13 6xcjpw pre922tv ihmvbih 9piadim1lterm","quiz_options":{"score":0},"id":"4385175116"},{"position":3,"visible":true,"text":"ywtecquds5ctgu sjcgsa3hm d087wy 6yjqp0jgm 1ywj8v3wuuq wmlmq essefj rbgrjtv6smxcmag","quiz_options":{"score":0},"id":"4385175117"},{"position":4,"visible":true,"text":"gc8d58x66m ftpowgvwodht9h fj47r927vh826 qrkgkb bcvxni fo6g9wdlxgvnq","quiz_options":{"score":0},"id":"4385175118"},{"position":5,"visible":true,"text":"fxsrgxts qih9ukhxafmmiv4 h2ujh1va9jf b6ho30","quiz_options":{"score":0},"id":"4385175119"},{"position":6,"visible":true,"text":"81binesi6f 7urb7 ylotwabgvbt 03ke1u5h 3ehye3g olw0f83a1h667t 71ujnoyf p49ce","quiz_options":{"score":0},"id":"4385175120"},{"position":7,"visible":true,"text":"ht1rd9ymh 2tftisj80s74mop b1eavw d6vgqwrj","quiz_options":{"score":0},"id":"4385175121"}]},"page_id":"168831402"},"emitted_at":1674149695842} +{"stream":"survey_questions","data":{"id":"667461805","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"mxlksuvmoras9o 94fj2 dieyg92v384lfv8 f9rwin4 cdmg95wcnt2xa ybcmni7yd1x 1yl4j4q7j"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461805","answers":{"choices":[{"position":1,"visible":true,"text":"mevjvlslfppe ex251ss vkrs7swus g72vplm9svejkdw 7onhrwlh bouaam3k7cnn yadoqmnhn4swehq u8lhv3fdh58o3 wrbc0y197d","quiz_options":{"score":0},"id":"4385175146"},{"position":2,"visible":true,"text":"uglir jabjq1 poswkedidqpmj ta5ma4ep9xxr ghu3n2a4u7f3orh 9oud3lwe0f vwip14snnv6gtb 5fw29neis71ogsm dpq7m 9an28j1styhc","quiz_options":{"score":0},"id":"4385175147"},{"position":3,"visible":true,"text":"p3ndk7 nxpv9grg77 ek2kndt51g 2v10497 bdr0a3466ao","quiz_options":{"score":0},"id":"4385175148"},{"position":4,"visible":true,"text":"7w76l 9k393odbjg7cht7 mio9w4tcv 6wvef4vm orgg1n 20d8lh8x9osqcv dv50mjj w3g96tt0m3rf9 24uun3grfy2u 4vns2lt","quiz_options":{"score":0},"id":"4385175149"},{"position":5,"visible":true,"text":"4tcuvnn1wxy cqpr795s sfyecjwup fn76iwks5hko rk6wvgyblb3gqe4 rl5ulee1w rq66d","quiz_options":{"score":0},"id":"4385175150"},{"position":6,"visible":true,"text":"cmrjgc4 dwotyvr4o n9jid3i79xoql klkrt23lklso4p hh6d57t5 9xk3o9me 8bkpgry1yu009y","quiz_options":{"score":0},"id":"4385175151"},{"position":7,"visible":true,"text":"43ghcfhsl 74xoo rn7rmgjhd3cq u2x2ir6n449kqxp 8isq7wb tccg39oy1b 9mw0eu1ho0 a4x77foba5y ywgyosh9ue ynh9u8odsos5q2","quiz_options":{"score":0},"id":"4385175152"}]},"page_id":"168831402"},"emitted_at":1674149695842} +{"stream":"survey_questions","data":{"id":"667461811","position":3,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"i4mol250lne3 bhrh2dvt9b qss461 lkb1u chpwmgcnuoeec un2l5"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461811","answers":{"choices":[{"position":1,"visible":true,"text":"s4xpl2l93 57k4asd04 gyddhg dn53f8bd 8wtgobxts3 ms7nan ns5wv6q2vy6 nnaudmbyu80llen 8be4urorunk","quiz_options":{"score":0},"id":"4385175197"},{"position":2,"visible":true,"text":"ec2l4hr 5lwp46ij8 3tqigw elleyat98j2jjd if8hiia3 vad578","quiz_options":{"score":0},"id":"4385175198"},{"position":3,"visible":true,"text":"9t9nsl0tjlcjxr k1chdb iislvtl gpcnyi82o5ebu 46ayfj 5r3b3w92l6 vaqskragdor","quiz_options":{"score":0},"id":"4385175199"},{"position":4,"visible":true,"text":"8ad404t4 86hyrfxr7 xef8em2 g7u8fc2 rsslpdptcgrsh9n n5pb1u9b","quiz_options":{"score":0},"id":"4385175200"},{"position":5,"visible":true,"text":"bijp3kiqfs quasi89mov1y hj9ku 9w6iuh 81sng4yu32tyh d4q9kbxuoqd2xaq","quiz_options":{"score":0},"id":"4385175201"},{"position":6,"visible":true,"text":"73xiyg2gc q1l6a28s 991jaxujf56sqi rhxrnjum ges25br tb2x1wamrh3jac1 t3s8ocme8q9d8 c505btw99r hwljwx","quiz_options":{"score":0},"id":"4385175202"},{"position":7,"visible":true,"text":"ri75nf 5yy3nq 8m5e68j4mh8m sf1v3 60nijf1oeq9 bwp7bfx9u11a474 w66gfkiayng55q 6h0gp80h","quiz_options":{"score":0},"id":"4385175203"},{"position":8,"visible":true,"text":"nbqtnbbuiue5fr a9s8yrpjm7x0p qid4y913k 8ueagmuy2 5kvul122lseh5h5","quiz_options":{"score":0},"id":"4385175204"}]},"page_id":"168831402"},"emitted_at":1674149695843} +{"stream":"survey_questions","data":{"id":"667461833","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"s3pjyrvc bj99egp0o99 f4ddk7sed bdc2yh24yf tyii1jye nmvwhj18oqxna6b lku2vt8hrnx4 j327a"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461833","answers":{"choices":[{"position":1,"visible":true,"text":"hqpox2w6wuwyd2 fm6kvjiq6ns5k jv1eutgqn1jj if8dj81e 57l25ev1tal9j","quiz_options":{"score":0},"id":"4385175279"},{"position":2,"visible":true,"text":"vr2p6mvbedcpkak 2c91smhshw9ee mwdy43um3334e7i u4o5frorc3py srt09vtrol825 i9s8n2koaoc6fu","quiz_options":{"score":0},"id":"4385175280"},{"position":3,"visible":true,"text":"1icyn0f tifktyc2uwd k8ehexjojth9a2f 0n7sh5p4i6kswe","quiz_options":{"score":0},"id":"4385175281"},{"position":4,"visible":true,"text":"6ju066 4chnhs0be43dy2 xdkxk37j1i0qy1 43b22jang8 na1yapnjj 7tvgbeu v1dw7as","quiz_options":{"score":0},"id":"4385175282"},{"position":5,"visible":true,"text":"vnslaachd7t07f0 db6whw u6ahc71ajst 2cn114ialhcvex kpwm1qo1y g82xup","quiz_options":{"score":0},"id":"4385175283"},{"position":6,"visible":true,"text":"aniu1f d47vbpsl mm26jpf7 g2io86ycj6yk","quiz_options":{"score":0},"id":"4385175284"}]},"page_id":"168831402"},"emitted_at":1674149695844} +{"stream":"survey_questions","data":{"id":"667461834","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ce8esfvsy7xcwqu gemf05b3s5ap5 76oc1 srngx7qca"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461834","answers":{"choices":[{"position":1,"visible":true,"text":"jggh1bnginkodsv 4jhtwffnlgybux 1na25qx xr5jtwfp vvip26cqr st09ps653caiyj 1icxwhc1hut6","quiz_options":{"score":0},"id":"4385175294"},{"position":2,"visible":true,"text":"gdxye rstmylwe4l w2lkwbdf87e735u rdxn1vxbg3aw kwkn1gfsu s3oa2wx7 6vegglr1ihckyxa","quiz_options":{"score":0},"id":"4385175295"},{"position":3,"visible":true,"text":"qsghk1r8e3p ciuick1mgdwbyc k8wbxctpmtu2v xau05rusflq k3as06r35dl9 38xpts","quiz_options":{"score":0},"id":"4385175296"}]},"page_id":"168831402"},"emitted_at":1674149695844} +{"stream":"collectors","data":{"status":"open","id":"405437100","survey_id":"306079584","type":"weblink","name":"Web Link 1","thank_you_message":"Thank you for submitting your responses!","thank_you_page":{"is_enabled":false,"message":"Thank you for submitting your responses!"},"disqualification_type":"message","disqualification_message":"Thank you for completing our survey!","disqualification_url":"https://www.surveymonkey.com","closed_page_message":"This survey is currently closed. Please contact the author of this survey for further assistance.","redirect_type":"url","redirect_url":"https://www.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-02T18:14:00+00:00","date_created":"2021-06-01T17:30:00+00:00","response_count":13,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://www.surveymonkey.com/r/BQB2V52","href":"https://api.surveymonkey.com/v3/collectors/405437100"},"emitted_at":1687778040457} +{"stream":"collectors","data":{"status":"open","id":"405843657","survey_id":"307785429","type":"weblink","name":"Web Link 1","thank_you_message":"Grazie per avere inviato le risposte.","thank_you_page":{"is_enabled":false,"message":"Grazie per avere inviato le risposte."},"disqualification_type":"message","disqualification_message":"Grazie per aver partecipato al nostro sondaggio!","disqualification_url":"https://it.surveymonkey.com","closed_page_message":"Il sondaggio è stato chiuso. Per ulteriori informazioni, contatta l’autore del sondaggio.","redirect_type":"url","redirect_url":"https://it.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T10:59:00+00:00","date_created":"2021-06-10T06:41:00+00:00","response_count":18,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://it.surveymonkey.com/r/GYXQMYF","href":"https://api.surveymonkey.com/v3/collectors/405843657"},"emitted_at":1687778042387} +{"stream":"collectors","data":{"status":"open","id":"405843665","survey_id":"307785444","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:00:00+00:00","date_created":"2021-06-10T06:42:00+00:00","response_count":18,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/GYNFJLH","href":"https://api.surveymonkey.com/v3/collectors/405843665"},"emitted_at":1687778044085} +{"stream":"collectors","data":{"status":"open","id":"405843672","survey_id":"307785394","type":"weblink","name":"Web Link 1","thank_you_message":"Thank you for submitting your responses!","thank_you_page":{"is_enabled":false,"message":"Thank you for submitting your responses!"},"disqualification_type":"message","disqualification_message":"Thank you for completing our survey!","disqualification_url":"https://www.surveymonkey.com","closed_page_message":"This survey is currently closed. Please contact the author of this survey for further assistance.","redirect_type":"url","redirect_url":"https://www.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:01:00+00:00","date_created":"2021-06-10T06:42:00+00:00","response_count":18,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://www.surveymonkey.com/r/GYFBNFM","href":"https://api.surveymonkey.com/v3/collectors/405843672"},"emitted_at":1687778045615} +{"stream":"collectors","data":{"status":"open","id":"405843682","survey_id":"307785402","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:02:00+00:00","date_created":"2021-06-10T06:42:00+00:00","response_count":18,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/GYGQJ5R","href":"https://api.surveymonkey.com/v3/collectors/405843682"},"emitted_at":1687778047089} +{"stream":"collectors","data":{"status":"open","id":"405843688","survey_id":"307785408","type":"weblink","name":"Web Link 1","thank_you_message":"Vielen Dank für die Teilnahme an der Umfrage!","thank_you_page":{"is_enabled":false,"message":"Vielen Dank für die Teilnahme an der Umfrage!"},"disqualification_type":"message","disqualification_message":"Vielen Dank, dass Sie die Umfrage abgeschlossen haben!","disqualification_url":"https://de.surveymonkey.com","closed_page_message":"Diese Umfrage ist derzeit geschlossen. Wenden Sie sich an den Autor dieser Umfrage, um weitere Hilfe zu erhalten.","redirect_type":"url","redirect_url":"https://de.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:02:00+00:00","date_created":"2021-06-10T06:43:00+00:00","response_count":18,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://de.surveymonkey.com/r/GYBJDCP","href":"https://api.surveymonkey.com/v3/collectors/405843688"},"emitted_at":1687778048609} +{"stream":"collectors","data":{"status":"open","id":"405829319","survey_id":"307784834","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:03:00+00:00","date_created":"2021-06-09T21:08:00+00:00","response_count":21,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/NCH93N6","href":"https://api.surveymonkey.com/v3/collectors/405829319"},"emitted_at":1687778050140} +{"stream":"collectors","data":{"status":"open","id":"405829931","survey_id":"307785448","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:03:00+00:00","date_created":"2021-06-09T21:21:00+00:00","response_count":18,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/NS2ZPQC","href":"https://api.surveymonkey.com/v3/collectors/405829931"},"emitted_at":1687778051829} +{"stream":"collectors","data":{"status":"open","id":"405843442","survey_id":"307784863","type":"weblink","name":"Web Link 1","thank_you_message":"Vielen Dank für die Teilnahme an der Umfrage!","thank_you_page":{"is_enabled":false,"message":"Vielen Dank für die Teilnahme an der Umfrage!"},"disqualification_type":"message","disqualification_message":"Vielen Dank, dass Sie die Umfrage abgeschlossen haben!","disqualification_url":"https://de.surveymonkey.com","closed_page_message":"Diese Umfrage ist derzeit geschlossen. Wenden Sie sich an den Autor dieser Umfrage, um weitere Hilfe zu erhalten.","redirect_type":"url","redirect_url":"https://de.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:04:00+00:00","date_created":"2021-06-10T06:32:00+00:00","response_count":20,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://de.surveymonkey.com/r/GDQLK9D","href":"https://api.surveymonkey.com/v3/collectors/405843442"},"emitted_at":1687778053347} +{"stream":"collectors","data":{"status":"open","id":"405829776","survey_id":"307784846","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:05:00+00:00","date_created":"2021-06-09T21:18:00+00:00","response_count":20,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/NWW9NKW","href":"https://api.surveymonkey.com/v3/collectors/405829776"},"emitted_at":1687778054580} +{"stream":"collectors","data":{"status":"open","id":"405843460","survey_id":"307784856","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:05:00+00:00","date_created":"2021-06-10T06:32:00+00:00","response_count":20,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/GDSZR22","href":"https://api.surveymonkey.com/v3/collectors/405843460"},"emitted_at":1687778056142} +{"stream":"collectors","data":{"status":"open","id":"405843624","survey_id":"307785388","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:06:00+00:00","date_created":"2021-06-10T06:40:00+00:00","response_count":20,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/GYWV533","href":"https://api.surveymonkey.com/v3/collectors/405843624"},"emitted_at":1687778057704} +{"stream":"collectors","data":{"status":"open","id":"405843634","survey_id":"307785415","type":"weblink","name":"Web Link 1","thank_you_message":"Спасибо за ответы!","thank_you_page":{"is_enabled":false,"message":"Спасибо за ответы!"},"disqualification_type":"message","disqualification_message":"Спасибо за участие в нашем опросе!","disqualification_url":"https://ru.surveymonkey.com","closed_page_message":"В настоящее время этот опрос закрыт. За дальнейшей информацией обращайтесь к автору опроса.","redirect_type":"url","redirect_url":"https://ru.surveymonkey.com","display_survey_results":false,"edit_response_type":"until_complete","anonymous_type":"not_anonymous","allow_multiple_responses":false,"date_modified":"2021-06-10T11:07:00+00:00","date_created":"2021-06-10T06:40:00+00:00","response_count":20,"password_enabled":false,"response_limit":null,"respondent_authentication":false,"sender_email":null,"close_date":null,"url":"https://ru.surveymonkey.com/r/GYS33KN","href":"https://api.surveymonkey.com/v3/collectors/405843634"},"emitted_at":1687778058997} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405437100", "href": "https://api.surveymonkey.com/v3/collectors/405437100", "type": "weblink", "survey_id": "306079584"}, "emitted_at": 1681918860499} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843657", "href": "https://api.surveymonkey.com/v3/collectors/405843657", "type": "weblink", "survey_id": "307785429"}, "emitted_at": 1681918861357} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843665", "href": "https://api.surveymonkey.com/v3/collectors/405843665", "type": "weblink", "survey_id": "307785444"}, "emitted_at": 1681918862236} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843672", "href": "https://api.surveymonkey.com/v3/collectors/405843672", "type": "weblink", "survey_id": "307785394"}, "emitted_at": 1681918863235} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843682", "href": "https://api.surveymonkey.com/v3/collectors/405843682", "type": "weblink", "survey_id": "307785402"}, "emitted_at": 1681918864098} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843688", "href": "https://api.surveymonkey.com/v3/collectors/405843688", "type": "weblink", "survey_id": "307785408"}, "emitted_at": 1681918865029} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405829319", "href": "https://api.surveymonkey.com/v3/collectors/405829319", "type": "weblink", "survey_id": "307784834"}, "emitted_at": 1681918865930} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405829931", "href": "https://api.surveymonkey.com/v3/collectors/405829931", "type": "weblink", "survey_id": "307785448"}, "emitted_at": 1681918866981} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843442", "href": "https://api.surveymonkey.com/v3/collectors/405843442", "type": "weblink", "survey_id": "307784863"}, "emitted_at": 1681918868050} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405829776", "href": "https://api.surveymonkey.com/v3/collectors/405829776", "type": "weblink", "survey_id": "307784846"}, "emitted_at": 1681918868953} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843460", "href": "https://api.surveymonkey.com/v3/collectors/405843460", "type": "weblink", "survey_id": "307784856"}, "emitted_at": 1681918870019} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843624", "href": "https://api.surveymonkey.com/v3/collectors/405843624", "type": "weblink", "survey_id": "307785388"}, "emitted_at": 1681918870844} +{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843634", "href": "https://api.surveymonkey.com/v3/collectors/405843634", "type": "weblink", "survey_id": "307785415"}, "emitted_at": 1681918871852} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.txt deleted file mode 100644 index 135a42f6a50b..000000000000 --- a/airbyte-integrations/connectors/source-surveymonkey/integration_tests/expected_records.txt +++ /dev/null @@ -1,313 +0,0 @@ -{"stream": "surveys", "data": {"title": "Market Research - Product Testing Template", "nickname": "", "language": "en", "folder_id": "0", "category": "market_research", "question_count": 13, "page_count": 1, "response_count": 13, "date_created": "2021-05-07T06:18:00", "date_modified": "2021-06-08T18:09:00", "id": "306079584", "buttons_text": {"next_button": "Next >>", "prev_button": "<< Prev", "done_button": "Done", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/306079584", "analyze_url": "https://www.surveymonkey.com/analyze/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "summary_url": "https://www.surveymonkey.com/summary/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=8T1PwDGoJHE1lbkxjUnaGitKu8jxWzyoclw9fNsShflPlk6MYIzwJ2NgjlBw_2B7iV"}, "emitted_at": 1681752753912} -{"stream": "surveys", "data": {"title": "yswa8kobijei1mkwaqxgy", "nickname": "7b4p9vssf810mslcd0eqpcg9s7p0h", "language": "it", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T10:59:00", "id": "307785429", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785429", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=YRpP3kxXMi2aJkgYoeyZrvuErii13mQ5DRN67Vm4WJ5avIMZ6YvzI_2Bc3FpERJDqx"}, "emitted_at": 1681752754883} -{"stream": "surveys", "data": {"title": "wsfqk1di34d", "nickname": "3pax9qjasev22ofir5dm4x45s", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:00:00", "id": "307785444", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785444", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=8fhTDITxRdeFQ_2BLWzVWxsJSU0nfgGgIbcvuAJ6C8LCdfivLn4FPj6xZP2o8_2FINMc"}, "emitted_at": 1681752755938} -{"stream": "surveys", "data": {"title": "vpoha5euc66vp", "nickname": "etv0tds1e45", "language": "en", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:19:00", "date_modified": "2021-06-10T11:01:00", "id": "307785394", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785394", "analyze_url": "https://www.surveymonkey.com/analyze/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "summary_url": "https://www.surveymonkey.com/summary/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=RPtFlMc_2B10dLjP_2BMbJ9eseMZkg_2FE5of5WUErPhwmC57Ij1xz0JOW7uC8i49BGEl8"}, "emitted_at": 1681752757038} -{"stream": "surveys", "data": {"title": "s2d9px7cdril0v7789ab4f", "nickname": "wnhin1ctnss8ebdgjef", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:02:00", "id": "307785402", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785402", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=RId_2BH4A8CUUX6zNvnFnfsb3wKFGuv8kLhz_2BApiG6Mbvu_2BLypJpz_2BM9EfoUuRXBcL"}, "emitted_at": 1681752757741} -{"stream": "surveys", "data": {"title": "muxx41av9mp", "nickname": "hfs5uo9cw1ce3j7rn7n8ncu88myc", "language": "de", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:02:00", "id": "307785408", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785408", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=aPPwgAP8stOxnud8WXi8VHKMrUtKIqvd7JhVnp1f0Ucqjb7cbMATzGizMgcLO_2BzA"}, "emitted_at": 1681752758782} -{"stream": "surveys", "data": {"title": "2iokp4jvp9ru5", "nickname": "j2a0kxhq8lmawfqjkg0hx", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 21, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:03:00", "id": "307784834", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784834", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XGwf91XTSWLx4XNlQvyUVl4Y_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=5ovW9LrIPsdAlqAloFxtr_2BhNVquSJyqeGOsrEnkZn56chkdLSKQISgfvIUejUonU"}, "emitted_at": 1681752759856} -{"stream": "surveys", "data": {"title": "9cnwcmdn39ox", "nickname": "vih7eeixclb", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 18, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:03:00", "id": "307785448", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785448", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxi9bVow7JzoPHlGgDld6S4o_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=CJF1TcWP7MthjVWkHhiX8ggWVFe484BBhGYkoi2XqDCTB9FcR1nlBSJ_2FeL47hNDV"}, "emitted_at": 1681752760573} -{"stream": "surveys", "data": {"title": "i2bm4lqt5hxv614n4jcl0guxt5ehgf", "nickname": "ti241ke4qo1i8iyqgpo0u6b2", "language": "de", "folder_id": "0", "category": "", "question_count": 2, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:04:00", "id": "307784863", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784863", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XGwAs0GsZ1qsocisThgUFPk0_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=kSWrVa29zWxvAB20ibAMkgPUDRBw_2B_2BTV0eX3oRPIEDNdxc2vxtOGJkQUGITIEart"}, "emitted_at": 1681752761722} -{"stream": "surveys", "data": {"title": "j057iyqgxlotswo070", "nickname": "wxuyqq4cgmfo69ik778r", "language": "ru", "folder_id": "0", "category": "", "question_count": 6, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:05:00", "id": "307784846", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784846", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XG0D0cevwMQIvVNwWzg_2Bmm4o_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=c5YOr2YH8sSXNlr7K0Tsuhs54aXml2seWFuXS8MIqk7n5MinBfQ7OjzW_2BtjWV1Cv"}, "emitted_at": 1681752762326} -{"stream": "surveys", "data": {"title": "u7r02s47jr", "nickname": "ye7fubxhua91ce0fxm", "language": "ru", "folder_id": "0", "category": "", "question_count": 3, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:07:00", "date_modified": "2021-06-10T11:05:00", "id": "307784856", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307784856", "analyze_url": "https://www.surveymonkey.com/analyze/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "summary_url": "https://www.surveymonkey.com/summary/moGgts_2Bl1LYlJ1mbVw6XG6cfdBSOknoT0fzVp7iTUO4_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=hW7YBNPL4euOVIMWVdvchE0xWtfXGoQrT7wUyFAtDel65HbgmAwoV7JJRkkkFmhn"}, "emitted_at": 1681752763451} -{"stream": "surveys", "data": {"title": "igpfp2yfsw90df6nxbsb49v", "nickname": "h23gl22ulmfsyt4q7xt", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:19:00", "date_modified": "2021-06-10T11:06:00", "id": "307785388", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785388", "analyze_url": "https://www.surveymonkey.com/analyze/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "summary_url": "https://www.surveymonkey.com/summary/5QHdVgvFd_2Bn4fvmj_2F1aNtwM9q4oP_2B3VqXy_2BeJTiumoQ_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=khUJQv9z4_2FXXzGUox57WEUPwppIr8YqRqVru77WpakX1HW8hHMmGXZiDGslFZym6"}, "emitted_at": 1681752764067} -{"stream": "surveys", "data": {"title": "b9jo5h23l7pa", "nickname": "qhs5vg2qi0o4arsjiwy2ay00n82n", "language": "ru", "folder_id": "0", "category": "", "question_count": 10, "page_count": 3, "response_count": 20, "date_created": "2021-06-09T21:20:00", "date_modified": "2021-06-10T11:07:00", "id": "307785415", "buttons_text": {"next_button": "Nex >>>>>", "prev_button": "Nix <<<<<", "done_button": "Nax_Don_Gon!", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "4510354", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/307785415", "analyze_url": "https://www.surveymonkey.com/analyze/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "summary_url": "https://www.surveymonkey.com/summary/BPAkhAawaMN8C17tmmNFxjZ0KOiJJ3FCQU4krShVQhg_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=YVdtL_2BP5oiGTrfksyofvENkBr7v87Xfh8hbcJr8rbqgesWvwJjz5N1F7pCSRcDoy"}, "emitted_at": 1681752765140} -{"stream": "surveys", "data": {"title": "jjj", "nickname": "", "language": "en", "folder_id": "0", "category": "", "question_count": 0, "page_count": 1, "response_count": 0, "date_created": "2023-01-17T09:17:00", "date_modified": "2023-01-17T09:17:00", "id": "510388524", "buttons_text": {"next_button": "Next", "prev_button": "Prev", "done_button": "Done", "exit_button": ""}, "is_owner": true, "footer": true, "theme_id": "10292568", "custom_variables": {}, "href": "https://api.surveymonkey.com/v3/surveys/510388524", "analyze_url": "https://www.surveymonkey.com/analyze/VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "edit_url": "https://www.surveymonkey.com/create/?sm=VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "collect_url": "https://www.surveymonkey.com/collect/list?sm=VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "summary_url": "https://www.surveymonkey.com/summary/VXMmVNBbmOp9KTSvXdhjIr3FHnqleAX32lr9MLBIOE0_3D", "preview": "https://www.surveymonkey.com/r/Preview/?sm=TPlNncbuCs17cvxwjuS74VC03_2FOcqpP_2F03m2gerTSI_2FQvLWoY2yn_2FWxLDmxYOp5L"}, "emitted_at": 1681752766116} -{"stream": "survey_responses", "data": {"id": "12706126725", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "124.123.178.184", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hNu3QJYf07WiUPOwxCYcARURFGB3ruOrs9slcTHVhmhgDhoNJ0k7w3jCvo0nLM40", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706126725", "total_time": 62, "date_modified": "2021-06-01T17:40:54+00:00", "date_created": "2021-06-01T17:39:51+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706126725", "pages": [{"id": "165250506", "questions": [{"id": "652286715", "answers": [{"choice_id": "4285525064"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525089"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525074"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525058", "row_id": "4285525061", "choice_metadata": {"weight": "0"}}]}]}]}, "emitted_at": 1681238097211} -{"stream": "survey_responses", "data": {"id": "12706152767", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "37.229.17.15", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YIZz5DiXEDES47ARxTbRPzAA9ZOwCjcN_2FDSFTYGWgCVPQCo_2B3EeLirGlON5_2BjrX5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706152767", "total_time": 55, "date_modified": "2021-06-01T17:50:03+00:00", "date_created": "2021-06-01T17:49:08+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706152767", "pages": [{"id": "165250506", "questions": [{"id": "652286726", "answers": [{"tag_data": [], "text": "fuck this"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525067"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525087"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525072"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525091"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525077"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525052", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "waste of time"}]}]}]}, "emitted_at": 1681238097211} -{"stream": "survey_responses", "data": {"id": "12706159691", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "157.48.231.67", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=P4z1eeLex6p2OQEYYXRJKBxPyHk6ljkOskXPds2olEToYrU_2FwTZWAyllEtgJRyQL", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706159691", "total_time": 104, "date_modified": "2021-06-01T17:52:27+00:00", "date_created": "2021-06-01T17:50:43+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706159691", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "78.63", "y": "13.19"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "41.94", "y": "50.16"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525086"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525055", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}]}]}, "emitted_at": 1681238097211} -{"stream": "survey_responses", "data": {"id": "12706182356", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "76.14.176.236", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=oRBufZrmuNVZW5ou_2B4ZICuFqW6p3uYPmpjTb5IJ5Zf_2BH4FPoHKfnz_2BSC_2FR_2FpxWNq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706182356", "total_time": 36, "date_modified": "2021-06-01T18:00:12+00:00", "date_created": "2021-06-01T17:59:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706182356", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "58.66", "y": "54.49"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "56.64", "y": "71.09"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525063"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525072"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525082"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525092"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525077"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525060", "row_id": "4285525061", "choice_metadata": {"weight": "100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "gekkiadsuigasdf;oij sefhello \ud83c\udf50\ud83c\udf50\ud83c\udf50\ud83c\udf50\ud83c\udf50"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "good"}]}]}]}, "emitted_at": 1681238097212} -{"stream": "survey_responses", "data": {"id": "12706201784", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.37.158.6", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MD0lXMX2bJm93XLiSKWuW2p52_2BwlWpayf88naadbuO5wITz0TijA3kwSis907xu1", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706201784", "total_time": 183, "date_modified": "2021-06-01T18:06:11+00:00", "date_created": "2021-06-01T18:03:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706201784", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "87.90", "y": "67.29"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "7.66", "y": "92.21"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "Colour"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525064"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525055", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Nothing"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "I don't know"}]}]}]}, "emitted_at": 1681238097212} -{"stream": "survey_responses", "data": {"id": "12706203862", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.205.239.133", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=sNkTdEY_2FEibixIxcUhotq6hQ3muFVlLkg0cE531VDB5Ya2U21pwazZRwwSXFqqtK", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706203862", "total_time": 114, "date_modified": "2021-06-01T18:06:49+00:00", "date_created": "2021-06-01T18:04:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706203862", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "40.32", "y": "93.35"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "37.10", "y": "66.67"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "because pets should not be tied,they should have their own freedom to move"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525080"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525055", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}]}]}, "emitted_at": 1681238097213} -{"stream": "survey_responses", "data": {"id": "12706264166", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "27.6.69.132", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=OboVhBA1nCZ4ejfYCmJ6WDu5SxhIUnnNn2dCz_2BTqxksFnjOSpy88MtS4B5Wbpk_2BW", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706264166", "total_time": 309, "date_modified": "2021-06-01T18:27:03+00:00", "date_created": "2021-06-01T18:21:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706264166", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "43.59", "y": "28.51"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "41.31", "y": "91.08"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "\ud83e\udd14"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525091"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525058", "row_id": "4285525061", "choice_metadata": {"weight": "0"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "\ud83e\uddb4"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "\ud83d\udcaa"}]}]}]}, "emitted_at": 1681238097213} -{"stream": "survey_responses", "data": {"id": "12706274940", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.205.116.166", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=D1IcNXq6BKOBmkySTfHcUfHuH3_2Fa0aniMPQEG23UrQ16iSsyx8ye2hPRQt3C61Jd", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706274940", "total_time": 105, "date_modified": "2021-06-01T18:30:56+00:00", "date_created": "2021-06-01T18:29:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706274940", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "7.12", "y": "89.86"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "87.90", "y": "9.02"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "Didn't like the product"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525067"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525087"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525072"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525092"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525077"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525097"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525050", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Nothing"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Better don't try to work on it"}]}]}]}, "emitted_at": 1681238097214} -{"stream": "survey_responses", "data": {"id": "12706353147", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "176.37.67.33", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=qwY5AKZSqBd7DfoZDGr4x_2FJr28RhtoeaQ7_2F6VBS1G3yK_2FPH86sPCcFs1zACVlbMO", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706353147", "total_time": 162, "date_modified": "2021-06-01T18:58:44+00:00", "date_created": "2021-06-01T18:56:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706353147", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "78.00", "y": "6.84"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "77.62", "y": "8.50"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "I like logo."}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525076"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525056", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Logo."}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Nothing."}]}]}]}, "emitted_at": 1681238097215} -{"stream": "survey_responses", "data": {"id": "12707255568", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "157.48.145.117", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=3XYG_2F55lVJgeQ0_2FWG3xyRxOF5gCKFR0p1HPkdv1iiMZ1h5MIYxNR12enFBgK9TCS", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12707255568", "total_time": 263, "date_modified": "2021-06-02T01:13:33+00:00", "date_created": "2021-06-02T01:09:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12707255568", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "38.17", "y": "40.98"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "42.90", "y": "93.75"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "I love puppies but hate heart symbol"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525063"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525070"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525080"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525089"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525075"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525056", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "It's for animals"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Approach to customer"}]}]}]}, "emitted_at": 1681238097216} -{"stream": "survey_responses", "data": {"id": "12707566461", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "106.195.73.137", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FskCSEofrdcabC7MVfRNtzeiZ4C4kiwx_2FpBdpRfIsd5SgVGi4N9znXMS9exRXf27", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12707566461", "total_time": 233, "date_modified": "2021-06-02T04:06:48+00:00", "date_created": "2021-06-02T04:02:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12707566461", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "32.67", "y": "32.51"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "89.00", "y": "55.34"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525069"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525080"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525075"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525058", "row_id": "4285525061", "choice_metadata": {"weight": "0"}}]}]}]}, "emitted_at": 1681238097216} -{"stream": "survey_responses", "data": {"id": "12709748835", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.196.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=x5BdjfYWF_2B2dRs9qPRcHJ_2BqMN8Dpfn_2FfhKhtD7G8W4_2BX9ECDWh9wAXjYd4mxXk_2F_2B", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12709748835", "total_time": 127, "date_modified": "2021-06-02T18:14:19+00:00", "date_created": "2021-06-02T18:12:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12709748835", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "41.85", "y": "55.96"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "31.35", "y": "35.64"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "because"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525063"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525084"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525069"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525079"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525089"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525074"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525095"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525060", "row_id": "4285525061", "choice_metadata": {"weight": "100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "u"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "f"}]}]}]}, "emitted_at": 1681238097217} -{"stream": "survey_responses", "data": {"id": "12706107193", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "49.37.150.53", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405437100", "survey_id": "306079584", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=7vMzRy38ln3J_2FJiU8YD1uve9yI6cAQIQ_2FuVxRipS_2FyB57w9vo9xpgShOuFWVcoI2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5jPPwKLnlqevaUQom_2BgYAWJWlrKNA2ZFTOYrMBrqW2c_3D?respondent_id=12706107193", "total_time": 607550, "date_modified": "2021-06-08T18:17:17+00:00", "date_created": "2021-06-01T17:31:26+00:00", "href": "https://api.surveymonkey.com/v3/surveys/306079584/responses/12706107193", "pages": [{"id": "165250506", "questions": [{"id": "652286724", "answers": [{"row_id": "4285525098", "x": "77.56", "y": "6.01"}]}, {"id": "652286725", "answers": [{"row_id": "4285525102", "x": "15.71", "y": "8.23"}]}, {"id": "652286726", "answers": [{"tag_data": [], "text": "It seems in that way"}]}, {"id": "652286715", "answers": [{"choice_id": "4285525065"}]}, {"id": "652286721", "answers": [{"choice_id": "4285525085"}]}, {"id": "652286716", "answers": [{"choice_id": "4285525071"}]}, {"id": "652286718", "answers": [{"choice_id": "4285525081"}]}, {"id": "652286722", "answers": [{"choice_id": "4285525090"}]}, {"id": "652286717", "answers": [{"choice_id": "4285525075"}]}, {"id": "652286723", "answers": [{"choice_id": "4285525096"}]}, {"id": "652286714", "answers": [{"choice_id": "4285525050", "row_id": "4285525061", "choice_metadata": {"weight": "-100"}}]}, {"id": "652286719", "answers": [{"tag_data": [], "text": "Nothing much"}]}, {"id": "652286720", "answers": [{"tag_data": [], "text": "Will include Tagline"}]}]}]}, "emitted_at": 1681238097217} -{"stream": "survey_responses", "data": {"id": "12731040927", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=YORvkBiLvNm2647vGYxs1GGGoUjDrz_2FRfIWw1i07UtykH_2BBJHDTB3ujkOPfyAxqP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731040927", "total_time": 31, "date_modified": "2021-06-10T08:46:53+00:00", "date_created": "2021-06-10T08:46:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731040927", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175881"}]}]}]}, "emitted_at": 1681238098508} -{"stream": "survey_responses", "data": {"id": "12731055204", "recipient_id": "", "collection_mode": "default", "response_status": "partial", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": ["168831413", "168831415", "168831437"], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tcbSSptg67E5MmPiY_2BCTC0GEk5rcm_2FHHcASKwxBGLOX_2BBByesO_2Fh848B_2FqaaVF8d", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731055204", "total_time": 15, "date_modified": "2021-06-10T08:54:22+00:00", "date_created": "2021-06-10T08:54:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731055204", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": []}]}, "emitted_at": 1681238098508} -{"stream": "survey_responses", "data": {"id": "12731069666", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tHWbA6E0Q6UEylVBreS0XaGKh3GcjDQu_2FBytp_2F_2FcCSkYTgRKkVt3jyyhUFoh6T_2Bs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731069666", "total_time": 33, "date_modified": "2021-06-10T09:02:19+00:00", "date_created": "2021-06-10T09:01:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731069666", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175733"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175752"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1681238098509} -{"stream": "survey_responses", "data": {"id": "12731085951", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=c_2B3Xk1rn3Lhz9GqaPNSRmjd03YiGx88BQu_2BtAvHiMmp5a0BR68kWLCOfALzBwKH4", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731085951", "total_time": 31, "date_modified": "2021-06-10T09:10:05+00:00", "date_created": "2021-06-10T09:09:34+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731085951", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175564"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175745"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1681238098509} -{"stream": "survey_responses", "data": {"id": "12731102076", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=s_2BQLUH6Nb049vqmxYdKTQlIV_2FCXzxmRR5F_2B_2Fe9FaqXRh3H_2FZAFF51mqyI3e8s666", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731102076", "total_time": 32, "date_modified": "2021-06-10T09:17:44+00:00", "date_created": "2021-06-10T09:17:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731102076", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175561"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175739"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175745"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1681238098510} -{"stream": "survey_responses", "data": {"id": "12731118899", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=hRALwRgDjAOTNk8Hpt0wWWn9X3xurqWW9vynXjPkvIvI4Xofu_2FJSgEVwtWK23vLd", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731118899", "total_time": 31, "date_modified": "2021-06-10T09:25:25+00:00", "date_created": "2021-06-10T09:24:53+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731118899", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175734"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175745"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1681238098510} -{"stream": "survey_responses", "data": {"id": "12731135865", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=OEPuy4AY_2B47LLaYrOMe90r_2BJdHiKxv12FpoQr0N94GGA0TFN7l7tk5QH14HD_2FaBy", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731135865", "total_time": 31, "date_modified": "2021-06-10T09:33:08+00:00", "date_created": "2021-06-10T09:32:37+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731135865", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175748"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1681238098511} -{"stream": "survey_responses", "data": {"id": "12731153599", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KPgqZwkF7GLrzwtQWhwpZBCuPbq_2F_2BuXfuUXpCBXX1y_2BwKBvvXi7s1ob9AMVJymCk", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731153599", "total_time": 31, "date_modified": "2021-06-10T09:40:49+00:00", "date_created": "2021-06-10T09:40:18+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731153599", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175561"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175751"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1681238098511} -{"stream": "survey_responses", "data": {"id": "12731170943", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=e6QtiM7XZfbHB3Ln1ifDr6Ct8PD0j6Nikh_2BTvBikLVfpeCSzS5WYkg2D_2BDVygAee", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731170943", "total_time": 31, "date_modified": "2021-06-10T09:48:36+00:00", "date_created": "2021-06-10T09:48:04+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731170943", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175733"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175748"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175879"}]}]}]}, "emitted_at": 1681238098513} -{"stream": "survey_responses", "data": {"id": "12731188992", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VATG0VEAVLLh8CTsnDoMZ_2FR3ida2C_2FIqVwt3FHGuc0GRu_2BA6Qa9E4Ewd3iEHt5YQ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731188992", "total_time": 35, "date_modified": "2021-06-10T09:56:51+00:00", "date_created": "2021-06-10T09:56:15+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731188992", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175738"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175881"}]}]}]}, "emitted_at": 1681238098513} -{"stream": "survey_responses", "data": {"id": "12731208790", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KH_2Bhsc8oIpTdaX60RIwFQ1SDyudOf2_2B61W7cPCZGGXEKsY5_2FN_2BA_2B_2FXh0DeOOdA7F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731208790", "total_time": 32, "date_modified": "2021-06-10T10:05:01+00:00", "date_created": "2021-06-10T10:04:28+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731208790", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175752"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1681238098514} -{"stream": "survey_responses", "data": {"id": "12731228560", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vbsxUKDxxVbnJW7Gng6Y5VblTyjW_2F29grieRYQImUaIVF77GuhY3KDlqPsNfIlx6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731228560", "total_time": 31, "date_modified": "2021-06-10T10:12:51+00:00", "date_created": "2021-06-10T10:12:19+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731228560", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175561"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175737"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175882"}]}]}]}, "emitted_at": 1681238098514} -{"stream": "survey_responses", "data": {"id": "12731247619", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=8lyrgf9sVRbhXAHse5glKYEJoYMvqlR2TNQIhI8Ycw716W_2FHlQeX6Ru4SKObInXR", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731247619", "total_time": 31, "date_modified": "2021-06-10T10:20:41+00:00", "date_created": "2021-06-10T10:20:09+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731247619", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175734"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175747"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175881"}]}]}]}, "emitted_at": 1681238098515} -{"stream": "survey_responses", "data": {"id": "12731266056", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=tkqgGP2ApNTakGiA0JrfYbKgIZElTj6_2FPaJ_2FDj0QNjpJDKCexid0Z_2Bq8vZoam1vK", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731266056", "total_time": 31, "date_modified": "2021-06-10T10:28:20+00:00", "date_created": "2021-06-10T10:27:49+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731266056", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175449"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175738"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175747"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175882"}]}]}]}, "emitted_at": 1681238098515} -{"stream": "survey_responses", "data": {"id": "12731286200", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=nYsBr3IsP19O_2BhTeV_2BWGnAdXc_2FFsnqMT6maJ0BS3QD9FqjkWLrNYzzEqKsT7c191", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731286200", "total_time": 35, "date_modified": "2021-06-10T10:36:27+00:00", "date_created": "2021-06-10T10:35:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731286200", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175367"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175564"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175738"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175752"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1681238098516} -{"stream": "survey_responses", "data": {"id": "12731305366", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=iIxE7kshwG8vR6T6c2D0swVsNTfDLqcBVkfrnf_2FGZBTuoHjMm9ksd3LbOIXuF6lp", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731305366", "total_time": 34, "date_modified": "2021-06-10T10:44:10+00:00", "date_created": "2021-06-10T10:43:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731305366", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175366"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175380"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175533"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175732"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175736"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175741"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175749"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175879"}]}]}]}, "emitted_at": 1681238098516} -{"stream": "survey_responses", "data": {"id": "12731325134", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=cUq2_2BSagT6pk152_2BsGjjMAAQ2Qq0R0cxleIxEdHEEVKwV5oPCvktmmnTV82pa5S_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731325134", "total_time": 31, "date_modified": "2021-06-10T10:52:02+00:00", "date_created": "2021-06-10T10:51:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731325134", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175369"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175382"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175447"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175535"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175559"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175735"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175739"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175742"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175751"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175878"}]}]}]}, "emitted_at": 1681238098517} -{"stream": "survey_responses", "data": {"id": "12731344038", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843657", "survey_id": "307785429", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=F_2BuWfRunQKdf1g0xuu4mTzXMCAlXia21_2BnF47p_2FbjU4QII1hLaE3Xo6tWdfbzKqr", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxoENnL6uEj_2FlAo5YSBwashU_3D?respondent_id=12731344038", "total_time": 31, "date_modified": "2021-06-10T10:59:42+00:00", "date_created": "2021-06-10T10:59:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785429/responses/12731344038", "pages": [{"id": "168831413", "questions": []}, {"id": "168831415", "questions": [{"id": "667461858", "answers": [{"choice_id": "4385175368"}]}, {"id": "667461861", "answers": [{"choice_id": "4385175381"}]}, {"id": "667461876", "answers": [{"choice_id": "4385175448"}]}, {"id": "667461897", "answers": [{"choice_id": "4385175534"}]}, {"id": "667461902", "answers": [{"choice_id": "4385175563"}]}]}, {"id": "168831437", "questions": [{"id": "667461933", "answers": [{"choice_id": "4385175733"}]}, {"id": "667461934", "answers": [{"choice_id": "4385175739"}]}, {"id": "667461936", "answers": [{"choice_id": "4385175740"}]}, {"id": "667461937", "answers": [{"choice_id": "4385175750"}]}, {"id": "667461986", "answers": [{"choice_id": "4385175880"}]}]}]}, "emitted_at": 1681238098517} -{"stream": "survey_responses", "data": {"id": "12731042086", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=SvCebsqkF1mO3kCZX7XsmQ4ABz0LFCf_2FyW5N2JOLuGh5ixjzbj2i04SarOUiUgPa", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731042086", "total_time": 31, "date_modified": "2021-06-10T08:47:30+00:00", "date_created": "2021-06-10T08:46:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731042086", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176960"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177005"}]}]}]}, "emitted_at": 1681238099800} -{"stream": "survey_responses", "data": {"id": "12731056238", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=r7v44AE_2Bx_2Fam5dgCBnlUkUkL0aSWyzhJkIPWmmYa5VnqMtd4X2DBzf9U9erpnCvI", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731056238", "total_time": 31, "date_modified": "2021-06-10T08:55:13+00:00", "date_created": "2021-06-10T08:54:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731056238", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176738"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176962"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176979"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177005"}]}]}]}, "emitted_at": 1681238099800} -{"stream": "survey_responses", "data": {"id": "12731070937", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=_2FuS4ZECIFCnV4LLPEe6gryzUPrf_2FMB8MB51Cv5cU6IQGsWb_2FRIb0BOaRXRodLTAq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731070937", "total_time": 32, "date_modified": "2021-06-10T09:02:57+00:00", "date_created": "2021-06-10T09:02:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731070937", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176767"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176975"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176978"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176990"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177007"}]}]}]}, "emitted_at": 1681238099801} -{"stream": "survey_responses", "data": {"id": "12731087215", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=eSF0qBZZ3aR1SHBtkTZz83mgEBvJuBfou_2BJGBoFPbxv5bluXZuIMKDaGn5x5xba5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731087215", "total_time": 31, "date_modified": "2021-06-10T09:10:42+00:00", "date_created": "2021-06-10T09:10:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731087215", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176736"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176918"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176962"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176972"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176983"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177007"}]}]}]}, "emitted_at": 1681238099801} -{"stream": "survey_responses", "data": {"id": "12731103402", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QOikOvuCCRFkDsJoRozKO9_2BNPZmp7mliotl5xt4QnStPgBKgrvz6GZ7vdHXf3eMG", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731103402", "total_time": 31, "date_modified": "2021-06-10T09:18:22+00:00", "date_created": "2021-06-10T09:17:50+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731103402", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176736"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176962"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176974"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176978"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177002"}]}]}]}, "emitted_at": 1681238099802} -{"stream": "survey_responses", "data": {"id": "12731120214", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=sV4Oq2KXvMuxWQpflbd5L_2FyoB6ImaDnm9BFwJGk9W_2BwE7YWlvW6MsxaqaeWWY_2Fzh", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731120214", "total_time": 31, "date_modified": "2021-06-10T09:26:01+00:00", "date_created": "2021-06-10T09:25:29+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731120214", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176734"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176960"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176968"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176976"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177009"}]}]}]}, "emitted_at": 1681238099802} -{"stream": "survey_responses", "data": {"id": "12731137499", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Hy3W8rkVoiYFKw_2BdJM6cChYRJWAXaGTQTol4ykCh_2FxYDhPmBpC753rbLshVSZjNE", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731137499", "total_time": 31, "date_modified": "2021-06-10T09:33:45+00:00", "date_created": "2021-06-10T09:33:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731137499", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176738"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176766"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176982"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177008"}]}]}]}, "emitted_at": 1681238099803} -{"stream": "survey_responses", "data": {"id": "12731154912", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=BLjiuDSDI39GvYDURd_2FmEfCd9jRrZ5nLwAgWG65zJj2dA1DskdMkMBHVZjzLHkk6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731154912", "total_time": 31, "date_modified": "2021-06-10T09:41:24+00:00", "date_created": "2021-06-10T09:40:52+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731154912", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176738"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176963"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177006"}]}]}]}, "emitted_at": 1681238099803} -{"stream": "survey_responses", "data": {"id": "12731172230", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=jiMm_2FJVQj8fPN6i2HlASqxeTd4rx_2FcPoCLMAznqiCWgY_2B98x39SfA1kKHTo8CgEC", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731172230", "total_time": 31, "date_modified": "2021-06-10T09:49:12+00:00", "date_created": "2021-06-10T09:48:40+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731172230", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176723"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176737"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176767"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176963"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176972"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176990"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177006"}]}]}]}, "emitted_at": 1681238099804} -{"stream": "survey_responses", "data": {"id": "12731190528", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VrEu2qgFZYPclcWCzyV5DO0WUUCjOIMFC77k1XLOofwOqYD5vHAej03viembeuF8", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731190528", "total_time": 35, "date_modified": "2021-06-10T09:57:32+00:00", "date_created": "2021-06-10T09:56:57+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731190528", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176725"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176959"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176972"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176978"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176987"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177004"}]}]}]}, "emitted_at": 1681238099804} -{"stream": "survey_responses", "data": {"id": "12731210366", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=xOrc7mbcYE0NFiqzEpenJPpJJ1OHiCt_2FUDSohMb9nmqttp0v1il2MNjb_2BRGE177T", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731210366", "total_time": 33, "date_modified": "2021-06-10T10:05:40+00:00", "date_created": "2021-06-10T10:05:06+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731210366", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176722"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176971"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177003"}]}]}]}, "emitted_at": 1681238099805} -{"stream": "survey_responses", "data": {"id": "12731230116", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=xmmEsbDmCA6Nvi6PKgkHAZR5hqRDTYPELB_2BIyMCUzF63brYH0R2ZrSJb9f_2BXtWRa", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731230116", "total_time": 32, "date_modified": "2021-06-10T10:13:28+00:00", "date_created": "2021-06-10T10:12:55+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731230116", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176920"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176959"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176973"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176977"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176988"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177007"}]}]}]}, "emitted_at": 1681238099805} -{"stream": "survey_responses", "data": {"id": "12731249077", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=1cQksQ2IA1zuU8mpR9JawSnSqr2zaPV1juVQ7ZEPrEqxr231kL_2F2eIW48UokyJBm", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731249077", "total_time": 31, "date_modified": "2021-06-10T10:21:17+00:00", "date_created": "2021-06-10T10:20:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731249077", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176764"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176963"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176968"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176982"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176989"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177008"}]}]}]}, "emitted_at": 1681238099806} -{"stream": "survey_responses", "data": {"id": "12731267503", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zOrptpJoo7QbXn1DoS0HIo4GcoAK8cIZ83MYidVo_2FuK_2FDvnnrsXK2SDzhyjssMdI", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731267503", "total_time": 31, "date_modified": "2021-06-10T10:28:56+00:00", "date_created": "2021-06-10T10:28:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731267503", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176721"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176736"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176961"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176970"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176983"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177003"}]}]}]}, "emitted_at": 1681238099806} -{"stream": "survey_responses", "data": {"id": "12731287789", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zXARfqNZgptvNWp9PboSVlElugTboBiAj_2FSG_2BkbD8e8OUJGsJ8FMRpMFap0qzcYY", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731287789", "total_time": 31, "date_modified": "2021-06-10T10:37:03+00:00", "date_created": "2021-06-10T10:36:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731287789", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176726"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176735"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176763"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176918"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176965"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176971"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176981"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176989"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177004"}]}]}]}, "emitted_at": 1681238099807} -{"stream": "survey_responses", "data": {"id": "12731307187", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=d07mxUJIp3BjkeFrp8gTQvz67vJrBUQWNX2tzrHPvkrZ0piZOPTQDKl_2BrNNjOvJO", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731307187", "total_time": 35, "date_modified": "2021-06-10T10:44:49+00:00", "date_created": "2021-06-10T10:44:14+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731307187", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176724"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176727"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176734"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176765"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176921"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176964"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176975"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176982"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176986"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177004"}]}]}]}, "emitted_at": 1681238099807} -{"stream": "survey_responses", "data": {"id": "12731326595", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=BCy4vPYrkiObJMS9B6GjZD2jwLTDr1luSZYtqGxH8zxGvmteSPixhqGNTTdp_2BRwb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731326595", "total_time": 32, "date_modified": "2021-06-10T10:52:38+00:00", "date_created": "2021-06-10T10:52:05+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731326595", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176722"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176728"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176739"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176767"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176918"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176959"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176974"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176981"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176990"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177003"}]}]}]}, "emitted_at": 1681238099808} -{"stream": "survey_responses", "data": {"id": "12731345509", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843665", "survey_id": "307785444", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=KmG9rTOuf2GcFWGtNhaBIfC5S80jehYrziix2nmuPzJw9_2FzLpaENGb_2F8j4NkPfTJ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxrQYdYnKpzayiBZVWsmE3jE_3D?respondent_id=12731345509", "total_time": 31, "date_modified": "2021-06-10T11:00:18+00:00", "date_created": "2021-06-10T10:59:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785444/responses/12731345509", "pages": [{"id": "168831459", "questions": []}, {"id": "168831461", "questions": [{"id": "667462078", "answers": [{"choice_id": "4385176725"}]}, {"id": "667462079", "answers": [{"choice_id": "4385176729"}]}, {"id": "667462082", "answers": [{"choice_id": "4385176734"}]}, {"id": "667462084", "answers": [{"choice_id": "4385176766"}]}, {"id": "667462086", "answers": [{"choice_id": "4385176919"}]}]}, {"id": "168831467", "questions": [{"id": "667462094", "answers": [{"choice_id": "4385176964"}]}, {"id": "667462096", "answers": [{"choice_id": "4385176968"}]}, {"id": "667462099", "answers": [{"choice_id": "4385176983"}]}, {"id": "667462100", "answers": [{"choice_id": "4385176991"}]}, {"id": "667462102", "answers": [{"choice_id": "4385177005"}]}]}]}, "emitted_at": 1681238099808} -{"stream": "survey_responses", "data": {"id": "12731043150", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=LKTTnPMSZ1aoM4WL0f1Ja7_2FdIZyTQ0DexNwU_2FFgEeZeT6B9T2m2HJuQjxPHn2OpZ", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731043150", "total_time": 36, "date_modified": "2021-06-10T08:48:12+00:00", "date_created": "2021-06-10T08:47:35+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731043150", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173238"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173269"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173482"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173525"}]}]}]}, "emitted_at": 1681238101175} -{"stream": "survey_responses", "data": {"id": "12731057303", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Tj5vPzxUmWLsaqGk6A3N4tXy8F1lrXDh3cgduiFC1tcjk3fKqd5Jc_2FYl9kqbrtXl", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731057303", "total_time": 35, "date_modified": "2021-06-10T08:55:53+00:00", "date_created": "2021-06-10T08:55:17+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731057303", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173478"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1681238101176} -{"stream": "survey_responses", "data": {"id": "12731072147", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=bAFsDUmEPpCdobkl_2BIl_2F6rdICCatPVmCxnojDaOR9ZJwGzYj7h29WMvC195fRe31", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731072147", "total_time": 35, "date_modified": "2021-06-10T09:03:38+00:00", "date_created": "2021-06-10T09:03:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731072147", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173240"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173257"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173268"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173452"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173476"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173483"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1681238101176} -{"stream": "survey_responses", "data": {"id": "12731088506", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MrquTcpA82ItgrSQWG_2BswTK4ClFm9rTuG5GWb85hiLDvaK2dr6F7vrYvOlwqkkC6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731088506", "total_time": 35, "date_modified": "2021-06-10T09:11:22+00:00", "date_created": "2021-06-10T09:10:46+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731088506", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173236"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173269"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173458"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173484"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173525"}]}]}]}, "emitted_at": 1681238101177} -{"stream": "survey_responses", "data": {"id": "12731104696", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=RO8uULIefm7fQ9VJXKzxtKBfRj4g0rYOE0LRuSA8OT2GVPEFDFQFmSOII7UlshDk", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731104696", "total_time": 36, "date_modified": "2021-06-10T09:19:03+00:00", "date_created": "2021-06-10T09:18:26+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731104696", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173240"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173474"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1681238101177} -{"stream": "survey_responses", "data": {"id": "12731121617", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Z_2BP_2BwLxa1Cmlcetqz6B1oaPvJSUU_2F5d018eOau5jIs8q4IlOyw7MT9YtxrlZnZHx", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731121617", "total_time": 37, "date_modified": "2021-06-10T09:26:44+00:00", "date_created": "2021-06-10T09:26:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731121617", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173228"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173268"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173458"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1681238101178} -{"stream": "survey_responses", "data": {"id": "12731139029", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=R0ZtH7VCYs_2FMuOE8hZBLqJdLMcu2RDqvrV04C34ivpIzGYhRzaVoAG0LDF83Ln_2B2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731139029", "total_time": 37, "date_modified": "2021-06-10T09:34:27+00:00", "date_created": "2021-06-10T09:33:50+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731139029", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173269"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173415"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173459"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173482"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1681238101178} -{"stream": "survey_responses", "data": {"id": "12731156353", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=t_2BeAHBUtkTES9TALR_2F4aDfYgtWlbqJv08b_2B6EYbplYKefvn8GrYTkoGugKOevrCn", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731156353", "total_time": 35, "date_modified": "2021-06-10T09:42:06+00:00", "date_created": "2021-06-10T09:41:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731156353", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173455"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173474"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1681238101178} -{"stream": "survey_responses", "data": {"id": "12731173574", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=mVe08d8mokL6AWRdTdpFz7jaz6dJhmAV4ZlfpEH6B8OQwQXlC2RxAxy2Z_2BtSrLXG", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731173574", "total_time": 36, "date_modified": "2021-06-10T09:49:53+00:00", "date_created": "2021-06-10T09:49:16+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731173574", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173241"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173414"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173458"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1681238101179} -{"stream": "survey_responses", "data": {"id": "12731192021", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=XFNknH0sj9IIdSvzYR75kcMldl4TuTz7kma3E7SGri2Rq_2FvcCRLj4ug5HOCnqMQz", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731192021", "total_time": 39, "date_modified": "2021-06-10T09:58:16+00:00", "date_created": "2021-06-10T09:57:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731192021", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173456"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173481"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173523"}]}]}]}, "emitted_at": 1681238101179} -{"stream": "survey_responses", "data": {"id": "12731211903", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=V3KUOw0NUFH27PiVeUNuTKv_2BkMDveRdHYnln_2FOZcYY0I7L0VIMlksx3CdNizHX_2BP", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731211903", "total_time": 36, "date_modified": "2021-06-10T10:06:21+00:00", "date_created": "2021-06-10T10:05:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731211903", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173456"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173484"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1681238101180} -{"stream": "survey_responses", "data": {"id": "12731231636", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=xv8ZLGu_2FUg9VAJ8hvx10oji6MvB1PbiAyA98qWpExvZvSaPKJ7cIFH8nZktkDKH0", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731231636", "total_time": 36, "date_modified": "2021-06-10T10:14:08+00:00", "date_created": "2021-06-10T10:13:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731231636", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173226"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173454"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173527"}]}]}]}, "emitted_at": 1681238101180} -{"stream": "survey_responses", "data": {"id": "12731250495", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=juR_2BEnc_2BsJLlCc6Sg7p2Ql_2B4icSx_2BsZwo_2FU1RVgYxqI2OzngMKnAUlrdJUS39ewn", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731250495", "total_time": 36, "date_modified": "2021-06-10T10:21:58+00:00", "date_created": "2021-06-10T10:21:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731250495", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173237"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173257"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173268"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173415"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173459"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173474"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1681238101181} -{"stream": "survey_responses", "data": {"id": "12731268932", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vXLZxpfaNUwy1JRrQLJCNL0kcVa5U5M5FtCofSN4MCcTJt9om9nRi2D4C4xJeXUg", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731268932", "total_time": 35, "date_modified": "2021-06-10T10:29:36+00:00", "date_created": "2021-06-10T10:29:00+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731268932", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173228"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173242"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173257"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173454"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173475"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173483"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173524"}]}]}]}, "emitted_at": 1681238101181} -{"stream": "survey_responses", "data": {"id": "12731289280", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=M4Lzmv89J2ZSnoBKgwxsZqDP3em9icPdsBglzN7NQB_2BM5AQ1bWpjH53fB0IsVZi6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731289280", "total_time": 36, "date_modified": "2021-06-10T10:37:44+00:00", "date_created": "2021-06-10T10:37:08+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731289280", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173242"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173258"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173416"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173454"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173485"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1681238101182} -{"stream": "survey_responses", "data": {"id": "12731308800", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=GQAEVhmAlwb5r_2FWfXNC_2Fqzqv5ULovU_2BGovk8oDp5nCvL2982DEXHbEhjSg4A3FdA", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731308800", "total_time": 39, "date_modified": "2021-06-10T10:45:33+00:00", "date_created": "2021-06-10T10:44:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731308800", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173239"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173459"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173473"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173482"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173494"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173526"}]}]}]}, "emitted_at": 1681238101182} -{"stream": "survey_responses", "data": {"id": "12731328082", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=MwpXysDo7Lhe8X4oi_2BfwDcfgG4XMlnjXbj783SvdsQcVCLn0Y0mvT87cyq6wLtri", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731328082", "total_time": 36, "date_modified": "2021-06-10T10:53:17+00:00", "date_created": "2021-06-10T10:52:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731328082", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173227"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173241"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173271"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173413"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173456"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173477"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173486"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173492"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173524"}]}]}]}, "emitted_at": 1681238101183} -{"stream": "survey_responses", "data": {"id": "12731346960", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843672", "survey_id": "307785394", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=i9GzbJPTq_2BCfXqXCYc91kWSkkbD40J_2FGL8hESaU68w7nuZXOyFZ7xNpNth1mmf72", "analyze_url": "https://www.surveymonkey.com/analyze/browse/5QHdVgvFd_2Bn4fvmj_2F1aNt3d6AR0J1aeeTU4KsCULsx8_3D?respondent_id=12731346960", "total_time": 35, "date_modified": "2021-06-10T11:00:58+00:00", "date_created": "2021-06-10T11:00:22+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785394/responses/12731346960", "pages": [{"id": "168831344", "questions": []}, {"id": "168831345", "questions": [{"id": "667461468", "answers": [{"choice_id": "4385173229"}]}, {"id": "667461471", "answers": [{"choice_id": "4385173241"}]}, {"id": "667461473", "answers": [{"choice_id": "4385173256"}]}, {"id": "667461476", "answers": [{"choice_id": "4385173270"}]}, {"id": "667461498", "answers": [{"choice_id": "4385173412"}]}]}, {"id": "168831352", "questions": [{"id": "667461513", "answers": [{"choice_id": "4385173457"}]}, {"id": "667461516", "answers": [{"choice_id": "4385173476"}]}, {"id": "667461517", "answers": [{"choice_id": "4385173483"}]}, {"id": "667461521", "answers": [{"choice_id": "4385173493"}]}, {"id": "667461526", "answers": [{"choice_id": "4385173527"}]}]}]}, "emitted_at": 1681238101183} -{"stream": "survey_responses", "data": {"id": "12731044345", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=RiSWqWbbXVsX_2BRIyudfJwY_2BODcC0aHC0L77gSNMTF5T_2BsPWUy2vlIWpXf01Hqy8k", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731044345", "total_time": 34, "date_modified": "2021-06-10T08:48:50+00:00", "date_created": "2021-06-10T08:48:16+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731044345", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173538"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173549"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173732"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1681238102523} -{"stream": "survey_responses", "data": {"id": "12731058644", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=8fjqeuyiG6EvuMWbYy64yqPXBnCSWaEpW8DInIUYXRbFyqerWPCENf5izNrPGoNw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731058644", "total_time": 33, "date_modified": "2021-06-10T08:56:31+00:00", "date_created": "2021-06-10T08:55:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731058644", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173662"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173678"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174063"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174183"}]}]}]}, "emitted_at": 1681238102523} -{"stream": "survey_responses", "data": {"id": "12731073567", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ILVcHPPYp1qBeboZrnUw9zU8OfPDI3pBaSHE_2FPA1mcsZYgDgyEvD2DNJEXG50BwI", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731073567", "total_time": 34, "date_modified": "2021-06-10T09:04:16+00:00", "date_created": "2021-06-10T09:03:42+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731073567", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173552"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173663"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173711"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173734"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174066"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174183"}]}]}]}, "emitted_at": 1681238102524} -{"stream": "survey_responses", "data": {"id": "12731089919", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=WJVGjucUzW8fBD4StuC_2FFZoPB0pCtugYpTzgDrCOMnJyyzUp5Q2v55jJ8xqcAaJv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731089919", "total_time": 33, "date_modified": "2021-06-10T09:12:00+00:00", "date_created": "2021-06-10T09:11:26+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731089919", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173536"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173551"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173658"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173714"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173728"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174060"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174184"}]}]}]}, "emitted_at": 1681238102524} -{"stream": "survey_responses", "data": {"id": "12731106311", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=pYr2BTIwVP0O3EzGC27aOOaCA0hnWvJ3FzJOiB_2Fhgw0UUSDh2QpQTcjrfXltZ8dv", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731106311", "total_time": 33, "date_modified": "2021-06-10T09:19:41+00:00", "date_created": "2021-06-10T09:19:07+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731106311", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173549"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173663"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173680"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174061"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1681238102525} -{"stream": "survey_responses", "data": {"id": "12731123001", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=iMOq3PmDu_2Fpfu6Lqltp5VPoRV2azTczcfDSAY7AjMIzECAtQhUwU_2FNMCCwq_2BSvsN", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731123001", "total_time": 34, "date_modified": "2021-06-10T09:27:23+00:00", "date_created": "2021-06-10T09:26:49+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731123001", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173658"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1681238102525} -{"stream": "survey_responses", "data": {"id": "12731140625", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QTesi81CB_2BwJ62wJ8op06oDEuscTPvq1ke99azBrWELfhmlgdyrrpw0NVegVV_2BE7", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731140625", "total_time": 33, "date_modified": "2021-06-10T09:35:05+00:00", "date_created": "2021-06-10T09:34:32+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731140625", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173538"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173678"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174062"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174185"}]}]}]}, "emitted_at": 1681238102526} -{"stream": "survey_responses", "data": {"id": "12731157855", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=Y_2BseyWnN45hqrdC63g5th2srbJCqZZuaaHgJaVUVwfuBPwNI2IdB2Dc1yEhLsRcy", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731157855", "total_time": 33, "date_modified": "2021-06-10T09:42:46+00:00", "date_created": "2021-06-10T09:42:12+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731157855", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173536"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174060"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174183"}]}]}]}, "emitted_at": 1681238102526} -{"stream": "survey_responses", "data": {"id": "12731175182", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=wvlpuaxYEnle4EG2hvBzqRUelWi_2BwXRQmZSU2ru959wJ7Ly3p7I9sg1wjhKu7Bm_2F", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731175182", "total_time": 27, "date_modified": "2021-06-10T09:50:26+00:00", "date_created": "2021-06-10T09:49:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731175182", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": []}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173713"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173731"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174061"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1681238102527} -{"stream": "survey_responses", "data": {"id": "12731193598", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=mqmf_2BkbhXVJbx4WQXC0_2F72pU_2FFf6FkOVA8_2F7REJy4j9HAEPKg_2BCPmL8F2wfc6SKi", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731193598", "total_time": 37, "date_modified": "2021-06-10T09:58:58+00:00", "date_created": "2021-06-10T09:58:20+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731193598", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173539"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173657"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173714"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173733"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174065"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174185"}]}]}]}, "emitted_at": 1681238102527} -{"stream": "survey_responses", "data": {"id": "12731213708", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=FmbWXWfmfbsDK8MEuKSsHPviZSUYgchniK99dQeGSFpf_2BWnh6cTWlg0o5YRIRtn7", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731213708", "total_time": 33, "date_modified": "2021-06-10T10:06:59+00:00", "date_created": "2021-06-10T10:06:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731213708", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173663"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173683"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174061"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1681238102527} -{"stream": "survey_responses", "data": {"id": "12731233283", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VRo6oIX2X5OOWhsAVRYKuhrXr5zszr6duv29Vq3RdwAJMtFNcQqAJk71Bi4Oq3Bo", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731233283", "total_time": 33, "date_modified": "2021-06-10T10:14:47+00:00", "date_created": "2021-06-10T10:14:13+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731233283", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173554"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173656"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173676"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173679"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173715"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173727"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174063"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174185"}]}]}]}, "emitted_at": 1681238102528} -{"stream": "survey_responses", "data": {"id": "12731252105", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=TQiDe8sSx4_2FuWzT1Lr7RKGOBO2iwOmTuPPOzcvcHL45tRjbdZSW9UHv6_2B2nLtY6n", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731252105", "total_time": 34, "date_modified": "2021-06-10T10:22:37+00:00", "date_created": "2021-06-10T10:22:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731252105", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173537"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173552"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173659"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173731"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174062"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1681238102529} -{"stream": "survey_responses", "data": {"id": "12731270690", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ipONp_2F6FrM7HwOh1_2BpJXXqL65aZR6EN_2BmCdmmlM4Etuv_2BqfU5P8wsYtpW2_2BvsbR2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731270690", "total_time": 33, "date_modified": "2021-06-10T10:30:16+00:00", "date_created": "2021-06-10T10:29:42+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731270690", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173536"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173550"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173656"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173681"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173715"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173734"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174064"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1681238102529} -{"stream": "survey_responses", "data": {"id": "12731290962", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=eWOUaV5xBiUO9JblKm5rYeLfk2_2FPRiygYknRKYIPlBC3Vl9805OBGP8f2kjnm0r2", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731290962", "total_time": 33, "date_modified": "2021-06-10T10:38:22+00:00", "date_created": "2021-06-10T10:37:48+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731290962", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173538"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173553"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173658"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173675"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173678"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173715"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173757"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174066"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1681238102530} -{"stream": "survey_responses", "data": {"id": "12731310541", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=wFNF4x4oPfyHaT5uvGstRuK1DUDfrHJyxBXB95bRgrB_2FdJojFuoGIhxA0BIH0XSO", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731310541", "total_time": 37, "date_modified": "2021-06-10T10:46:14+00:00", "date_created": "2021-06-10T10:45:37+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731310541", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173551"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173712"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173732"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173756"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174184"}]}]}]}, "emitted_at": 1681238102530} -{"stream": "survey_responses", "data": {"id": "12731329746", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vGkpaT5QedSZDkH1LkdPEMwKHIlyZ2mKjKyt8tgek5uzfoU30oxZnac34WvjYm1h", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731329746", "total_time": 33, "date_modified": "2021-06-10T10:53:55+00:00", "date_created": "2021-06-10T10:53:21+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731329746", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": [{"id": "667461529", "answers": [{"choice_id": "4385173540"}]}, {"id": "667461530", "answers": [{"choice_id": "4385173551"}]}, {"id": "667461549", "answers": [{"choice_id": "4385173661"}]}, {"id": "667461551", "answers": [{"choice_id": "4385173677"}]}, {"id": "667461553", "answers": [{"choice_id": "4385173682"}]}]}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173711"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173732"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174182"}]}]}]}, "emitted_at": 1681238102531} -{"stream": "survey_responses", "data": {"id": "12731348717", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843682", "survey_id": "307785402", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vuWK6q7Hz9dNybRoH2rP8Ux0qNc_2B0E3lMUmtDlPUbslKbHc7FranQhTJxg_2BIo3Pw", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxuFqiaqWbXDPfS4KhzQnJ4c_3D?respondent_id=12731348717", "total_time": 26, "date_modified": "2021-06-10T11:01:30+00:00", "date_created": "2021-06-10T11:01:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785402/responses/12731348717", "pages": [{"id": "168831357", "questions": []}, {"id": "168831358", "questions": []}, {"id": "168831365", "questions": [{"id": "667461555", "answers": [{"choice_id": "4385173714"}]}, {"id": "667461558", "answers": [{"choice_id": "4385173729"}]}, {"id": "667461561", "answers": [{"choice_id": "4385173755"}]}, {"id": "667461580", "answers": [{"choice_id": "4385174059"}]}, {"id": "667461598", "answers": [{"choice_id": "4385174186"}]}]}]}, "emitted_at": 1681238102531} -{"stream": "survey_responses", "data": {"id": "12731045521", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=0SWEpDfTb_2FAVFz7Ddryxk5koNgVExk8Oke4TJ8tz6QD5Eqc7uoqHQWRcLUi7yOAb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731045521", "total_time": 32, "date_modified": "2021-06-10T08:49:26+00:00", "date_created": "2021-06-10T08:48:54+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731045521", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174496"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1681238104077} -{"stream": "survey_responses", "data": {"id": "12731059832", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vhiMNZdaQtV6_2FPkJM_2BP8UtinkM87qYXz4VjmfwvWnjK1NFrLnN1P_2FH3w9GU7JUxX", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731059832", "total_time": 32, "date_modified": "2021-06-10T08:57:08+00:00", "date_created": "2021-06-10T08:56:36+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731059832", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174644"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104078} -{"stream": "survey_responses", "data": {"id": "12731074829", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ZUZi7r7io9fCwXP7l4K5oirdqpAhNEdbesFS73p617lSOzxuK5oKGDsqiK8zdA8E", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731074829", "total_time": 31, "date_modified": "2021-06-10T09:04:52+00:00", "date_created": "2021-06-10T09:04:20+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731074829", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174622"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174645"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1681238104078} -{"stream": "survey_responses", "data": {"id": "12731091270", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=eL4vDkGrDZfi5BiJC001E_2FoL_2Bo1mAtZxB97VuOw0Ue7x2Y_2Fvj9cWXWEFAG1_2For6z", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731091270", "total_time": 33, "date_modified": "2021-06-10T09:12:38+00:00", "date_created": "2021-06-10T09:12:05+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731091270", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174493"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1681238104079} -{"stream": "survey_responses", "data": {"id": "12731107632", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=VzELV_2Bc1GPJfJsXsRIabrQVAi9_2BSDOS6F_2B68zR_2BP4u98MGtxvy_2BeSOjCh5s1HKa5", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731107632", "total_time": 32, "date_modified": "2021-06-10T09:20:17+00:00", "date_created": "2021-06-10T09:19:44+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731107632", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174254"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174496"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1681238104079} -{"stream": "survey_responses", "data": {"id": "12731124423", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=QIT1tkBlcTVTBkkHm7dpbWllNNXyBriN0RSobCplWUm5LoJpA_2FC93B5N8fFbc0Zs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731124423", "total_time": 31, "date_modified": "2021-06-10T09:28:00+00:00", "date_created": "2021-06-10T09:27:28+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731124423", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174484"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174620"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174645"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104080} -{"stream": "survey_responses", "data": {"id": "12731142107", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=URkLmYAmILJkZgOzrwRncRyLAGrxL_2FGHdeqmjSqrSYjS_2F7cjqy4e3nBWqRsOOv0r", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731142107", "total_time": 31, "date_modified": "2021-06-10T09:35:42+00:00", "date_created": "2021-06-10T09:35:10+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731142107", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174601"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174643"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104080} -{"stream": "survey_responses", "data": {"id": "12731159230", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=O0O262_2ByzCPZXGbz3NLSBdAJfB9rNAyLSz4YX4ubxd_2BIrVIo2jJ_2Bzk8KQNxCcjXb", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731159230", "total_time": 31, "date_modified": "2021-06-10T09:43:20+00:00", "date_created": "2021-06-10T09:42:49+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731159230", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174643"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104080} -{"stream": "survey_responses", "data": {"id": "12731176347", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=LVM3eoF2xyKmckO7bkvBEw0i_2FoajMZMA_2Fc_2FVcI95ReDdVUuQQ_2Babxm1nILbPUISC", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731176347", "total_time": 33, "date_modified": "2021-06-10T09:51:04+00:00", "date_created": "2021-06-10T09:50:30+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731176347", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174486"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174676"}]}]}]}, "emitted_at": 1681238104081} -{"stream": "survey_responses", "data": {"id": "12731195152", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=SDlGK5Xxc7lpBehXWM_2B47eY24yPtwGB4NBBehIidUXtChXAXOgjA_2F8CxAHW97nUA", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731195152", "total_time": 35, "date_modified": "2021-06-10T09:59:39+00:00", "date_created": "2021-06-10T09:59:03+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731195152", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174647"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104081} -{"stream": "survey_responses", "data": {"id": "12731215248", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=ft3pzaYpicC1h6Ah26Wj0wRkCmTK1Yu7BTxxTQ0po_2FC00oJaK3YHuX9XV5WB5T60", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731215248", "total_time": 32, "date_modified": "2021-06-10T10:07:34+00:00", "date_created": "2021-06-10T10:07:02+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731215248", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174360"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174493"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174676"}]}]}]}, "emitted_at": 1681238104082} -{"stream": "survey_responses", "data": {"id": "12731234853", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=2W_2B7kOeuvhHSN7s0oUQZQwt3iySUHWLv5qq0xw8gojd0RWsFcAMM6NILAk_2FZJbdX", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731234853", "total_time": 31, "date_modified": "2021-06-10T10:15:23+00:00", "date_created": "2021-06-10T10:14:51+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731234853", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174372"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174485"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174601"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174622"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174650"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1681238104082} -{"stream": "survey_responses", "data": {"id": "12731253651", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=0yectjqXWcCHTMhyv3pd4HqlpH_2FjslmIWhw59V07Em7ASx_2FsHGk5ZFvSKThLEb3f", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731253651", "total_time": 31, "date_modified": "2021-06-10T10:23:12+00:00", "date_created": "2021-06-10T10:22:41+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731253651", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174252"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174497"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104082} -{"stream": "survey_responses", "data": {"id": "12731272195", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=owJREFrbM6_2Fu6fqPiWbW3rJ15U9q9DLJ_2BlAZMq6Gi_2BGKjQ2AGiWVNE5RBM_2FKLyMs", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731272195", "total_time": 31, "date_modified": "2021-06-10T10:30:52+00:00", "date_created": "2021-06-10T10:30:20+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731272195", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174482"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174495"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174600"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174646"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104083} -{"stream": "survey_responses", "data": {"id": "12731292534", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=zAYnDyr9Dxd9Em4YpxVpKH7_2BtJbsMFiI0bH89dH2LhhokCrBrNjKYFI0nacFEIpq", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731292534", "total_time": 32, "date_modified": "2021-06-10T10:38:57+00:00", "date_created": "2021-06-10T10:38:25+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731292534", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174254"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174483"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174494"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174599"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174623"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174634"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174649"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174678"}]}]}]}, "emitted_at": 1681238104083} -{"stream": "survey_responses", "data": {"id": "12731312208", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=uSGoF1sumsyf_2B3iBhadh_2F73k_2BM0dxEwFfAlWl70AwraHwQHcfj64cL0FUt9StbRx", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731312208", "total_time": 34, "date_modified": "2021-06-10T10:46:52+00:00", "date_created": "2021-06-10T10:46:18+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731312208", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174255"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174361"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174371"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174486"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174495"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174621"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174635"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174643"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174677"}]}]}]}, "emitted_at": 1681238104084} -{"stream": "survey_responses", "data": {"id": "12731331308", "recipient_id": "", "collection_mode": "default", "response_status": "completed", "custom_value": "", "first_name": "", "last_name": "", "email_address": "", "ip_address": "91.242.192.88", "language": null, "logic_path": {}, "metadata": {"contact": {}}, "page_path": [], "collector_id": "405843688", "survey_id": "307785408", "custom_variables": {}, "edit_url": "https://www.surveymonkey.com/r/?sm=vQGF4JMphBOwv8kJxOmtjdRG81lTGr7CW73feMGxDEMLSwybystpUFV76KKxYQR6", "analyze_url": "https://www.surveymonkey.com/analyze/browse/BPAkhAawaMN8C17tmmNFxvvr7YEM4YZ5j7dMDUtBHEw_3D?respondent_id=12731331308", "total_time": 31, "date_modified": "2021-06-10T10:54:29+00:00", "date_created": "2021-06-10T10:53:58+00:00", "href": "https://api.surveymonkey.com/v3/surveys/307785408/responses/12731331308", "pages": [{"id": "168831381", "questions": []}, {"id": "168831382", "questions": [{"id": "667461606", "answers": [{"choice_id": "4385174253"}]}, {"id": "667461628", "answers": [{"choice_id": "4385174362"}]}, {"id": "667461630", "answers": [{"choice_id": "4385174370"}]}, {"id": "667461651", "answers": [{"choice_id": "4385174486"}]}, {"id": "667461652", "answers": [{"choice_id": "4385174498"}]}]}, {"id": "168831388", "questions": [{"id": "667461666", "answers": [{"choice_id": "4385174598"}]}, {"id": "667461670", "answers": [{"choice_id": "4385174620"}]}, {"id": "667461674", "answers": [{"choice_id": "4385174636"}]}, {"id": "667461676", "answers": [{"choice_id": "4385174644"}]}, {"id": "667461686", "answers": [{"choice_id": "4385174676"}]}]}]}, "emitted_at": 1681238104084} -{"stream":"survey_pages","data":{"title":"sy4ara","description":"Сурвейманки жлобы","position":1,"question_count":13,"id":"165250506","href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506"},"emitted_at":1674149681286} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831413","href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831413"},"emitted_at":1674149682414} -{"stream":"survey_pages","data":{"title":"xsgqdhdakh7x","description":"wlju6xsgkxyig0s1","position":2,"question_count":5,"id":"168831415","href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415"},"emitted_at":1674149682415} -{"stream":"survey_pages","data":{"title":"ajsn8v0tvicgt7u063","description":"dcwmhxdx6p8buu","position":3,"question_count":5,"id":"168831437","href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437"},"emitted_at":1674149682415} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831459","href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831459"},"emitted_at":1674149683744} -{"stream":"survey_pages","data":{"title":"ijw0pw2tlfb0vd3","description":"k8tycaedxbl4","position":2,"question_count":5,"id":"168831461","href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461"},"emitted_at":1674149683744} -{"stream":"survey_pages","data":{"title":"krd3l3bj7vaym6pc4","description":"oy458fugj0k","position":3,"question_count":5,"id":"168831467","href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467"},"emitted_at":1674149683745} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831344","href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831344"},"emitted_at":1674149684444} -{"stream":"survey_pages","data":{"title":"q4fuvltqc6","description":"7kfibw7aer8mr937a3ko","position":2,"question_count":5,"id":"168831345","href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345"},"emitted_at":1674149684445} -{"stream":"survey_pages","data":{"title":"p5sdpb0pus6","description":"o9gbkpmfik2x","position":3,"question_count":5,"id":"168831352","href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352"},"emitted_at":1674149684445} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831357","href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831357"},"emitted_at":1674149685687} -{"stream":"survey_pages","data":{"title":"133nvr2cx99r","description":"shwtfx0edv","position":2,"question_count":5,"id":"168831358","href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358"},"emitted_at":1674149685687} -{"stream":"survey_pages","data":{"title":"otgwn5b4wicdemu1q","description":"s3ndgrck3qr4898qwtgh","position":3,"question_count":5,"id":"168831365","href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365"},"emitted_at":1674149685687} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831381","href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831381"},"emitted_at":1674149686909} -{"stream":"survey_pages","data":{"title":"ooj54g8q2thh","description":"8hrryr258se","position":2,"question_count":5,"id":"168831382","href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382"},"emitted_at":1674149686909} -{"stream":"survey_pages","data":{"title":"mva5ojqgmx6wnv62as","description":"wq6q460p143mi0","position":3,"question_count":5,"id":"168831388","href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388"},"emitted_at":1674149686909} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830049","href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830049"},"emitted_at":1674149687592} -{"stream":"survey_pages","data":{"title":"v3q97ckq2438fqkppcyn","description":"oforikk3wu4gin","position":2,"question_count":5,"id":"168830050","href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050"},"emitted_at":1674149687593} -{"stream":"survey_pages","data":{"title":"t57ybjyll8fwgu39w","description":"ttlkuqp07ua6kpsh","position":3,"question_count":5,"id":"168830060","href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060"},"emitted_at":1674149687594} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831470","href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831470"},"emitted_at":1674149688355} -{"stream":"survey_pages","data":{"title":"q3wpp1ufpi5r058o","description":"ay91aymge2vuacmrl9co","position":2,"question_count":5,"id":"168831471","href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471"},"emitted_at":1674149688356} -{"stream":"survey_pages","data":{"title":"m345ab1u6tjjo4nn3d","description":"ec22umvdkhxd0ne575","position":3,"question_count":5,"id":"168831478","href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478"},"emitted_at":1674149688357} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830093","href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830093"},"emitted_at":1674149689582} -{"stream":"survey_pages","data":{"title":"7t9dgejtlw5wsofbt","description":"mfqrejibgc831bp31","position":2,"question_count":5,"id":"168830094","href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094"},"emitted_at":1674149689583} -{"stream":"survey_pages","data":{"title":"9kri0ao4fh8e3i0j2hms","description":"dh7qg9jc1k65x","position":3,"question_count":5,"id":"168830108","href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108"},"emitted_at":1674149689583} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830067","href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830067"},"emitted_at":1674149690505} -{"stream":"survey_pages","data":{"title":"9vpm4e5ecjis5hge0p","description":"r6e38n2fp33skkjrl","position":2,"question_count":5,"id":"168830068","href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068"},"emitted_at":1674149690506} -{"stream":"survey_pages","data":{"title":"nvv6kl2njpt5b1l2p","description":"rd2j09sxv4ssu976g","position":3,"question_count":5,"id":"168830074","href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074"},"emitted_at":1674149690506} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168830082","href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830082"},"emitted_at":1674149691681} -{"stream":"survey_pages","data":{"title":"v1yxmq6n1ix","description":"aeoyc3hiak9vui1hevm","position":2,"question_count":5,"id":"168830083","href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083"},"emitted_at":1674149691682} -{"stream":"survey_pages","data":{"title":"g84sqoltkc2jen8iaj0","description":"ss2439kly1u4j1k1","position":3,"question_count":5,"id":"168830087","href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087"},"emitted_at":1674149691682} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831335","href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831335"},"emitted_at":1674149692346} -{"stream":"survey_pages","data":{"title":"k91l1laduo8","description":"4tmb1eke23bi1l2ev","position":2,"question_count":5,"id":"168831336","href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336"},"emitted_at":1674149692347} -{"stream":"survey_pages","data":{"title":"gisj5ms868kxxv","description":"4g1iiqg0sa15pbk","position":3,"question_count":5,"id":"168831340","href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340"},"emitted_at":1674149692348} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"168831392","href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831392"},"emitted_at":1674149693064} -{"stream":"survey_pages","data":{"title":"p71uerk2uh7k5","description":"92cb9d98j15jmfo","position":2,"question_count":5,"id":"168831393","href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393"},"emitted_at":1674149693064} -{"stream":"survey_pages","data":{"title":"bqd6mn6bdgv5u1rnstkx","description":"e0jrpexyx6t","position":3,"question_count":5,"id":"168831402","href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402"},"emitted_at":1674149693065} -{"stream":"survey_pages","data":{"title":"","description":"","position":1,"question_count":0,"id":"36710109","href":"https://api.surveymonkey.com/v3/surveys/510388524/pages/36710109"},"emitted_at":1674149694191} -{"stream":"survey_questions","data":{"id":"652286724","position":1,"visible":true,"family":"click_map","subtype":"single","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"Click on the area you like best about this product.","image":{"url":"https://surveymonkey-assets.s3.amazonaws.com/survey/306079584/20535460-8f99-41b0-ac96-b9f4f2aecb96.png"}}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286724","answers":{"rows":[{"position":1,"visible":true,"text":"Click 1","id":"4285525098"}]},"page_id":"165250506"},"emitted_at":1674149694470} -{"stream":"survey_questions","data":{"id":"652286725","position":2,"visible":true,"family":"click_map","subtype":"single","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"Click on the area you like least about this product.","image":{"url":"https://surveymonkey-assets.s3.amazonaws.com/survey/306079584/79215d25-9dbc-4870-91cd-3a36778aae52.png"}}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286725","answers":{"rows":[{"position":1,"visible":true,"text":"Click 1","id":"4285525102"}]},"page_id":"165250506"},"emitted_at":1674149694470} -{"stream":"survey_questions","data":{"id":"652286726","position":3,"visible":true,"family":"open_ended","subtype":"essay","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"Why did you make that selection?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286726","page_id":"165250506"},"emitted_at":1674149694471} -{"stream":"survey_questions","data":{"id":"652286715","position":4,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"What is your first reaction to the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286715","answers":{"choices":[{"position":1,"visible":true,"text":"Very positive","quiz_options":{"score":0},"id":"4285525063"},{"position":2,"visible":true,"text":"Somewhat positive","quiz_options":{"score":0},"id":"4285525064"},{"position":3,"visible":true,"text":"Neutral","quiz_options":{"score":0},"id":"4285525065"},{"position":4,"visible":true,"text":"Somewhat negative","quiz_options":{"score":0},"id":"4285525066"},{"position":5,"visible":true,"text":"Very negative","quiz_options":{"score":0},"id":"4285525067"}]},"page_id":"165250506"},"emitted_at":1674149694471} -{"stream":"survey_questions","data":{"id":"652286721","position":5,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How would you rate the quality of the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286721","answers":{"choices":[{"position":1,"visible":true,"text":"Very high quality","quiz_options":{"score":0},"id":"4285525083"},{"position":2,"visible":true,"text":"High quality","quiz_options":{"score":0},"id":"4285525084"},{"position":3,"visible":true,"text":"Neither high nor low quality","quiz_options":{"score":0},"id":"4285525085"},{"position":4,"visible":true,"text":"Low quality","quiz_options":{"score":0},"id":"4285525086"},{"position":5,"visible":true,"text":"Very low quality","quiz_options":{"score":0},"id":"4285525087"}]},"page_id":"165250506"},"emitted_at":1674149694472} -{"stream":"survey_questions","data":{"id":"652286716","position":6,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How innovative is the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286716","answers":{"choices":[{"position":1,"visible":true,"text":"Extremely innovative","quiz_options":{"score":0},"id":"4285525068"},{"position":2,"visible":true,"text":"Very innovative","quiz_options":{"score":0},"id":"4285525069"},{"position":3,"visible":true,"text":"Somewhat innovative","quiz_options":{"score":0},"id":"4285525070"},{"position":4,"visible":true,"text":"Not so innovative","quiz_options":{"score":0},"id":"4285525071"},{"position":5,"visible":true,"text":"Not at all innovative","quiz_options":{"score":0},"id":"4285525072"}]},"page_id":"165250506"},"emitted_at":1674149694473} -{"stream":"survey_questions","data":{"id":"652286718","position":7,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"When you think about the product, do you think of it as something you need or don’t need?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286718","answers":{"choices":[{"position":1,"visible":true,"text":"Definitely need","quiz_options":{"score":0},"id":"4285525078"},{"position":2,"visible":true,"text":"Probably need","quiz_options":{"score":0},"id":"4285525079"},{"position":3,"visible":true,"text":"Neutral","quiz_options":{"score":0},"id":"4285525080"},{"position":4,"visible":true,"text":"Probably don’t need","quiz_options":{"score":0},"id":"4285525081"},{"position":5,"visible":true,"text":"Definitely don’t need","quiz_options":{"score":0},"id":"4285525082"}]},"page_id":"165250506"},"emitted_at":1674149694474} -{"stream":"survey_questions","data":{"id":"652286722","position":8,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How would you rate the value for money of the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286722","answers":{"choices":[{"position":1,"visible":true,"text":"Excellent","quiz_options":{"score":0},"id":"4285525088"},{"position":2,"visible":true,"text":"Above average","quiz_options":{"score":0},"id":"4285525089"},{"position":3,"visible":true,"text":"Average","quiz_options":{"score":0},"id":"4285525090"},{"position":4,"visible":true,"text":"Below average","quiz_options":{"score":0},"id":"4285525091"},{"position":5,"visible":true,"text":"Poor","quiz_options":{"score":0},"id":"4285525092"}]},"page_id":"165250506"},"emitted_at":1674149694474} -{"stream":"survey_questions","data":{"id":"652286717","position":9,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"If the product were available today, how likely would you be to buy the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286717","answers":{"choices":[{"position":1,"visible":true,"text":"Extremely likely","quiz_options":{"score":0},"id":"4285525073"},{"position":2,"visible":true,"text":"Very likely","quiz_options":{"score":0},"id":"4285525074"},{"position":3,"visible":true,"text":"Somewhat likely","quiz_options":{"score":0},"id":"4285525075"},{"position":4,"visible":true,"text":"Not so likely","quiz_options":{"score":0},"id":"4285525076"},{"position":5,"visible":true,"text":"Not at all likely","quiz_options":{"score":0},"id":"4285525077"}]},"page_id":"165250506"},"emitted_at":1674149694475} -{"stream":"survey_questions","data":{"id":"652286723","position":10,"visible":true,"family":"single_choice","subtype":"vertical_two_col","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How likely are you to replace your current product with the product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286723","answers":{"choices":[{"position":1,"visible":true,"text":"Extremely likely ","quiz_options":{"score":0},"id":"4285525093"},{"position":2,"visible":true,"text":"Very likely ","quiz_options":{"score":0},"id":"4285525094"},{"position":3,"visible":true,"text":"Somewhat likely","quiz_options":{"score":0},"id":"4285525095"},{"position":4,"visible":true,"text":"Not so likely","quiz_options":{"score":0},"id":"4285525096"},{"position":5,"visible":true,"text":"Not at all likely","quiz_options":{"score":0},"id":"4285525097"}]},"page_id":"165250506"},"emitted_at":1674149694475} -{"stream":"survey_questions","data":{"id":"652286714","position":11,"visible":true,"family":"matrix","subtype":"rating","layout":{"bottom_spacing":0,"col_width":80,"col_width_format":"percent","left_spacing":0,"num_chars":null,"num_lines":null,"position":"new_row","right_spacing":0,"top_spacing":0,"width":100,"width_format":"percent"},"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"How likely is it that you would recommend our new product to a friend or colleague?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286714","answers":{"rows":[{"position":1,"visible":true,"text":"","id":"4285525061"}],"choices":[{"position":1,"visible":true,"text":"Not at all likely - 0","id":"4285525050","is_na":false,"weight":-100,"description":"Not at all likely"},{"position":2,"visible":true,"text":"1","id":"4285525051","is_na":false,"weight":-100,"description":""},{"position":3,"visible":true,"text":"2","id":"4285525052","is_na":false,"weight":-100,"description":""},{"position":4,"visible":true,"text":"3","id":"4285525053","is_na":false,"weight":-100,"description":""},{"position":5,"visible":true,"text":"4","id":"4285525054","is_na":false,"weight":-100,"description":""},{"position":6,"visible":true,"text":"5","id":"4285525055","is_na":false,"weight":-100,"description":""},{"position":7,"visible":true,"text":"6","id":"4285525056","is_na":false,"weight":-100,"description":""},{"position":8,"visible":true,"text":"7","id":"4285525057","is_na":false,"weight":0,"description":""},{"position":9,"visible":true,"text":"8","id":"4285525058","is_na":false,"weight":0,"description":""},{"position":10,"visible":true,"text":"9","id":"4285525059","is_na":false,"weight":100,"description":""},{"position":11,"visible":true,"text":"Extremely likely - 10","id":"4285525060","is_na":false,"weight":100,"description":"Extremely likely"}]},"page_id":"165250506"},"emitted_at":1674149694476} -{"stream":"survey_questions","data":{"id":"652286719","position":12,"visible":true,"family":"open_ended","subtype":"essay","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"In your own words, what are the things that you like most about this new product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286719","page_id":"165250506"},"emitted_at":1674149694476} -{"stream":"survey_questions","data":{"id":"652286720","position":13,"visible":true,"family":"open_ended","subtype":"essay","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"In your own words, what are the things that you would most like to improve in this new product?"}],"href":"https://api.surveymonkey.com/v3/surveys/306079584/pages/165250506/questions/652286720","page_id":"165250506"},"emitted_at":1674149694477} -{"stream":"survey_questions","data":{"id":"667461858","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vprw9pgg3d xiygcvd0suru k7rews838g6qc ndsukv2 7sa31urnvoskixw c52sg4uoete874i"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461858","answers":{"choices":[{"position":1,"visible":true,"text":"m5kpo9621yynjey kdt5w6pkkit yqyocxqf yw1p3uh2e5b 7gtmvs4 6em5ugqat x6pmhcfrvq4pit t67pif54aj jbgure","quiz_options":{"score":0},"id":"4385175366"},{"position":2,"visible":true,"text":"usdfft kvi9yqh1m38w3m 6uxryyvhrk1 nfxlt gnhjy826e rqks3jjuyj9hd 3y8755o","quiz_options":{"score":0},"id":"4385175367"},{"position":3,"visible":true,"text":"m6xv3yca7 up9u0qwx23h2skj 0cjlw19k5emypgm awi5tg l9atp kv4jrd73y9","quiz_options":{"score":0},"id":"4385175368"},{"position":4,"visible":true,"text":"todhc7 krw2v8qa rt2iu19vhxyw1dp x6oav54yak4vj yu4le2fc7 fksvl ejbr7x2u69 k9n9n7g3f","quiz_options":{"score":0},"id":"4385175369"}]},"page_id":"168831415"},"emitted_at":1674149694578} -{"stream":"survey_questions","data":{"id":"667461861","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"rl3uabslq46p mnwh0fle3xfs ejupx8e26q55va svfm11o"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461861","answers":{"choices":[{"position":1,"visible":true,"text":"m6g15xqsuwpbh3 x0116lkpod 5vkgg7duiq23sp ot884xd67v6fv2q 1u2mpgo ttj3ehahbljf1j6 pwj46w1d","quiz_options":{"score":0},"id":"4385175380"},{"position":2,"visible":true,"text":"cuff7 mbn2k1hxd6n6 jg9kffdkccjh bpodqpt2wtxu 7x38qxmvg42ap qpv0cddfumvix s0vv161iytceelx","quiz_options":{"score":0},"id":"4385175381"},{"position":3,"visible":true,"text":"jm5q6yu4rn pl8wwv23lnxs ou5r8m3np4fis6 6wlatg yeh3kafns0 h8u0o8f yhqni064ev6","quiz_options":{"score":0},"id":"4385175382"}]},"page_id":"168831415"},"emitted_at":1674149694579} -{"stream":"survey_questions","data":{"id":"667461876","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"hn4xnf ox0joj4inoy6ja jh02428n qeqxm9nopevjca sccwladi v63ks6mdqf0h0 4pug94eya 5et1g3t4 exbryyy6hv9mvd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461876","answers":{"choices":[{"position":1,"visible":true,"text":"w6nvr mkrj8q1g 740smg3nda m3afibg8 224jb59fon975t w9t8ma","quiz_options":{"score":0},"id":"4385175447"},{"position":2,"visible":true,"text":"cerw942pk xv1wg4gk4l7jq q3grdgasaol 75ghj ppo6ivm3r hxodiktx9rxs","quiz_options":{"score":0},"id":"4385175448"},{"position":3,"visible":true,"text":"5os82a1jwgygye 61dhsf6v sgy0ui7ib78ws7f j3pymv","quiz_options":{"score":0},"id":"4385175449"}]},"page_id":"168831415"},"emitted_at":1674149694580} -{"stream":"survey_questions","data":{"id":"667461897","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5a32s3dhl a967a54aoj3cr ttu0uk4hu4h1 r360wdohq1x9xu7 qvqpg1eb6qg3 01ogn ucbhive fpwstwi6hre0ynb xu3c3txpma7eoh3"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461897","answers":{"choices":[{"position":1,"visible":true,"text":"6gngt 5w10n7fl47r07 68t8t79 66pqqth5urrw 1ve2kn385x0u9s 99vcfs08at","quiz_options":{"score":0},"id":"4385175533"},{"position":2,"visible":true,"text":"pnwrx dwj1dx dpdan1wpqs 9lhgks36 1w8a2utjbxas31t rlc1u51mdpjr 90tcj6i8ibicvxt q1ahtd2x doujpba kjjjdi0","quiz_options":{"score":0},"id":"4385175534"},{"position":3,"visible":true,"text":"wu74ewyb4grv fqb8h3yoldsn 0nxv5844yn0lpx jct7na y9sp3u ueq7vk83ix7g7sx f5sl73r2r29e84","quiz_options":{"score":0},"id":"4385175535"}]},"page_id":"168831415"},"emitted_at":1674149694581} -{"stream":"survey_questions","data":{"id":"667461902","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"sdek6wejcdn 82r223sfhy6xkm5 65gns2m7phi 0fx8dx5bp psvndjnn5b 5kki467 a8faadeid0gl13 x2t3e 03xco2 cf39nv9mdq3vj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831415/questions/667461902","answers":{"choices":[{"position":1,"visible":true,"text":"nfiq0 d7gwft9 8bhinfrsv6r6 k3vylofokamx3 9hrik k2ageb5amyj89a toli0yrsq bqbrcp","quiz_options":{"score":0},"id":"4385175559"},{"position":2,"visible":true,"text":"8c5qdiklqb8tb 5uhc6w1a1o9 er8h4w0mf779o 14nsksqs65j10","quiz_options":{"score":0},"id":"4385175561"},{"position":3,"visible":true,"text":"5wpggsufojm 4decq179m5 0brpk1la0kyno e8ctqi fxa4j0uo9atp","quiz_options":{"score":0},"id":"4385175563"},{"position":4,"visible":true,"text":"5it4y4q 49im15osxk2j0 j8twpv8j nei1egowtm9a lyrigqwu0eby tpg5o7kuvgn34l spdu5icxlc 2f4qf qb55g 8si14ri4bdw1v72","quiz_options":{"score":0},"id":"4385175564"}]},"page_id":"168831415"},"emitted_at":1674149694582} -{"stream":"survey_questions","data":{"id":"667461933","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"4vnmxluvh7a l8s5r0i5ck pa7pckqf tnah7fan76p38n8 m1i5r ea9ib0t823amcn k98290k23wh7qn"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461933","answers":{"choices":[{"position":1,"visible":true,"text":"aldwijo3p9mh3 v26dql0wbrctg0 98sqogqfmi2 558gsvip","quiz_options":{"score":0},"id":"4385175732"},{"position":2,"visible":true,"text":"c5fqe o43qm3oq9i 23qgoxg85 cg5hr5wj79469n 328j7ji1kugb 60jlwtbu4eoh63 alyho5","quiz_options":{"score":0},"id":"4385175733"},{"position":3,"visible":true,"text":"882jtclxvq15yk dgive oqhqdf a8a2rgxse4b8fw 87oeyfbk7 wcevsx 4mv1lyp6lyxysrm 3m8yayarq7wm 9bpmul9 el2j1j4yw","quiz_options":{"score":0},"id":"4385175734"},{"position":4,"visible":true,"text":"3so1buxa 88ypp61nq6bhi0 lwa2pfg6tg lra8e1r5bn4k umstgwdck9 wslq681gvn3g2f a20nr1eovr3 feo0p5bqbgrcvun np851e23cojfv1 uqlwg","quiz_options":{"score":0},"id":"4385175735"}]},"page_id":"168831437"},"emitted_at":1674149694582} -{"stream":"survey_questions","data":{"id":"667461934","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"k6ba3wdl 6c0nvj5es h0dttl3krobdcd va6klkdyf 79e2a lhmov"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461934","answers":{"choices":[{"position":1,"visible":true,"text":"h8dywnj ithq4el 98e20manw xxyp0f9oike55 t56arby dhmmxh4pehs","quiz_options":{"score":0},"id":"4385175736"},{"position":2,"visible":true,"text":"14p6263c 7khx2y rpjy6432cy7kkr 0po0vol3uk1cuf 1oejmy viircj9b nyw76yqpf","quiz_options":{"score":0},"id":"4385175737"},{"position":3,"visible":true,"text":"vtdf2x 2kme3 vhpqi5s82k7v4 1mjr5r jp0ox03i6t d5ef5228du3ck 536btd7etv","quiz_options":{"score":0},"id":"4385175738"},{"position":4,"visible":true,"text":"yt2e6nk6 v2wl4k047h1n5 civs8 tjjr7lkeay9i3","quiz_options":{"score":0},"id":"4385175739"}]},"page_id":"168831437"},"emitted_at":1674149694583} -{"stream":"survey_questions","data":{"id":"667461936","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"u1jbhrxyrg30ra9 qs6b5m237lag 7tlvv 05okuge2imipht bg813 hixvf9bd9n10gk lwxdjnajbu9gsra uyh3fjd40bs"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461936","answers":{"choices":[{"position":1,"visible":true,"text":"yl8yqqhtr9rxi 049x312 oogub1figyg4e tixix79g85hxi2l bin0fp5g goq3kwu2eaase x9mihu0erdhl9v4 4yausp34y2rgyx iwyfuo4b7yme8lr","quiz_options":{"score":0},"id":"4385175740"},{"position":2,"visible":true,"text":"gfqbxk8 3tosymacon00av uoxydvdy28 iuce5bjdvpg gwxdpkudg24ouk u2ns1u x10hujmiiy9l62i soln75a14nm8q0","quiz_options":{"score":0},"id":"4385175741"},{"position":3,"visible":true,"text":"j299iy64y804 k709a5kk9 yjsifyt4ksu b4fnp4a","quiz_options":{"score":0},"id":"4385175742"}]},"page_id":"168831437"},"emitted_at":1674149694584} -{"stream":"survey_questions","data":{"id":"667461937","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"06qgl j7f89g3 5ok0ilvhvpa ojcs6ixbovvf u6xympul0 o00vc 6bjg7jmvy9s 543ndxd jmugw njqhxa3phj4jd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461937","answers":{"choices":[{"position":1,"visible":true,"text":"5v1hangb7u0bj8 44xo3y4o inytm95s16m dvn5e2leb6sf7 hamaw8l538eph 6xuyp5030i538c 0ym6m831 duar0bo76 kl36ykf5w9ugh","quiz_options":{"score":0},"id":"4385175745"},{"position":2,"visible":true,"text":"3iiaxf5dgi tr6crqs i51ocgg ka1ri0ffd8xeh cv12807q5pq 7fmq8lxy7 5wuas7gcn4kln tv6mf3pqq4jm x3yfr sm7idq2nvoonk","quiz_options":{"score":0},"id":"4385175746"},{"position":3,"visible":true,"text":"mj5cmssx3htoni ld622ppaiqr uw63p1q4up3 lfxioj94gd3jky0 2tj631 5ql6116xsyp r5hwhy2","quiz_options":{"score":0},"id":"4385175747"},{"position":4,"visible":true,"text":"llb9oe7fk8v3w4q xpd66rgbyp9m 0opo19df7n6 scr70cg86pn o1qtr7rclqxjl kmpnf79790 6wmig6mjwflh2vq 1oe033jyd","quiz_options":{"score":0},"id":"4385175748"},{"position":5,"visible":true,"text":"pg851 mju7py cdp1jqcaeg66 3gv05ohwromt u0uot","quiz_options":{"score":0},"id":"4385175749"},{"position":6,"visible":true,"text":"mu0hn799v 4ch8i7j5o5tf8s 6rn9y0ft67mchc u5ds14s9 aj1qg26dwf41suc i1mdhdk","quiz_options":{"score":0},"id":"4385175750"},{"position":7,"visible":true,"text":"6w4xjejtc9j 50n12 u1cdbulvkqykvci x79sdq9y hmbem37x7 s7pwufdjnmn xo8qy81a3fjmv","quiz_options":{"score":0},"id":"4385175751"},{"position":8,"visible":true,"text":"k582dst vgjvse b2h3mxi dteo4p9lrtx m54ug","quiz_options":{"score":0},"id":"4385175752"}]},"page_id":"168831437"},"emitted_at":1674149694585} -{"stream":"survey_questions","data":{"id":"667461986","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"gjahy0fl7 fs6b3q6e7 rsvbmotgt9p7 w9l2l7 hvj8fhrqwc vays8 4yh7qch hjj5lx0 co6a5rqd 5pt79or0a7evc"}],"href":"https://api.surveymonkey.com/v3/surveys/307785429/pages/168831437/questions/667461986","answers":{"choices":[{"position":1,"visible":true,"text":"b7fbynqu hwhvtbqwbmxy 9bjgxs1 1e05mo toj685p5v w34e97j","quiz_options":{"score":0},"id":"4385175878"},{"position":2,"visible":true,"text":"n5e5yl9c6 yw12588olh swl7nwm1dl9n2l 9v7n8wursm6739 p67woq9u27w7p y3ge5y1iji819g7 uklmy7q8 7ocv68por","quiz_options":{"score":0},"id":"4385175879"},{"position":3,"visible":true,"text":"h08nyyi 1393hst fcdij6j yepfw2","quiz_options":{"score":0},"id":"4385175880"},{"position":4,"visible":true,"text":"rdme8pwwjl07 4ju4xn47ofvbj i31u4ty4f4 wteatx2 gc3nqgji pu9h7","quiz_options":{"score":0},"id":"4385175881"},{"position":5,"visible":true,"text":"jxkod8gx x8tcsxxle4f0lv4 vcmjicpk7v i19dxl3 3lvmmdkx","quiz_options":{"score":0},"id":"4385175882"}]},"page_id":"168831437"},"emitted_at":1674149694587} -{"stream":"survey_questions","data":{"id":"667462078","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vrwr5e5qwxqu4 mn9jcdpf 0e5u3k ansge 2hwkipig u0wn3acc2sct xnv4y8 3irjcv i0cgva8762tfosw"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462078","answers":{"choices":[{"position":1,"visible":true,"text":"55yvst1hihf4br ialhkcxm73fjln 1lelxuw93g0 nukml0","quiz_options":{"score":0},"id":"4385176721"},{"position":2,"visible":true,"text":"okdweedyn93 mua59r2l4haj orem637bgltyb 7jc2yral4o8qov j15ry fkkgjxdea rjd297yot eq57kefir4if3c 2l098 ech2y5fv","quiz_options":{"score":0},"id":"4385176722"},{"position":3,"visible":true,"text":"b3sgu7kkbjowg ux7qqyw41yb umfqpq6 dfgu4awr uy2i4j626 cb17jp5xal6","quiz_options":{"score":0},"id":"4385176723"},{"position":4,"visible":true,"text":"5igq4pw5ul la3i72sh30 uk24o0qi jh51hl9s3a43s 9tgq0ip8k1nev ar6it adgfobu491 f8qke95 o2f9u2ubb49t28c obyoj9cfsl","quiz_options":{"score":0},"id":"4385176724"},{"position":5,"visible":true,"text":"r73ge7qkd8mjfu d7kdhfmco d0pcoyqqyjrph4g 06xs83492x 5ajmrgy1 x4ev3aroh9q86r gwiu17g02i6h75c np3r62er4x5n 73hbkk43af 7f5b333hk7tkc","quiz_options":{"score":0},"id":"4385176725"},{"position":6,"visible":true,"text":"vo6u3 9gpy2s57xh u8dcib 4f8gbbng3wq0h36","quiz_options":{"score":0},"id":"4385176726"}]},"page_id":"168831461"},"emitted_at":1674149694691} -{"stream":"survey_questions","data":{"id":"667462079","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6d3w2gqt8056 h5vl4u3kcuvlqc lk2ip62d kpkowwj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462079","answers":{"choices":[{"position":1,"visible":true,"text":"agvpu24j1 g51hr01jbjsk vw6gq cftg9oeklijpda qbibhtf35pl4gtq wu48wsd5 c6jifqlyt4e","quiz_options":{"score":0},"id":"4385176727"},{"position":2,"visible":true,"text":"4kbb46n 0vmm0c4we qwaiv1f731l y8iaiu3bkcb6 loqrsy","quiz_options":{"score":0},"id":"4385176728"},{"position":3,"visible":true,"text":"rwm2cgejb qc4g9 7y68obgewd fou0um xh7dkb89o bfosq3 v9bdp7s9450","quiz_options":{"score":0},"id":"4385176729"}]},"page_id":"168831461"},"emitted_at":1674149694692} -{"stream":"survey_questions","data":{"id":"667462082","position":3,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"dvpu842k yh6g475bbwk75 qtkq7f5yd01igo ixldsnn uqbpr ngarg6pf f6ueafaq 4ch8dhr nqsm5uydajns9"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462082","answers":{"choices":[{"position":1,"visible":true,"text":"hfrp92k576cl 4yjdkhqj0vr9 qx16a oc86p8hhonp5xs fn4lo1wmewe5n7 sue7kinfb6 8qpyh auccq7xlw1b","quiz_options":{"score":0},"id":"4385176734"},{"position":2,"visible":true,"text":"lqon4mp qhj98iee7o 37f50nvkrs3va 18vxkbch636kh yn8ih 8jxqpq03v 6j5lurdu7c17 knmjb3 0868hyuu7","quiz_options":{"score":0},"id":"4385176735"},{"position":3,"visible":true,"text":"obj26 pb0pwmt gxvk8isp04c42 77ocbs1 y7jyxsbl vbbtsy d1t8el31vu7x6d r44iyhr2y810o0o","quiz_options":{"score":0},"id":"4385176736"},{"position":4,"visible":true,"text":"yebv8qvq dd5qeecs45k 2un02jrywfqf1 u96tcry54cxiyu mnbgr2mqe a43wm7 1rnggj b99hudv2g0kgor","quiz_options":{"score":0},"id":"4385176737"},{"position":5,"visible":true,"text":"b6v1txm65a40 h1sah8 84enie0e 57clim 2eew4","quiz_options":{"score":0},"id":"4385176738"},{"position":6,"visible":true,"text":"d88o04mls0oddci d61gj k4acpqp r3ugg2t1e55s ldb3ll5gjkn ruerq4w4w95j649 c3bea007si4t 4jp1ctllptvfao2 m78584fllehmi b9fpb","quiz_options":{"score":0},"id":"4385176739"}]},"page_id":"168831461"},"emitted_at":1674149694692} -{"stream":"survey_questions","data":{"id":"667462084","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"n6y1nnlv45movkl vkjge u5wp4unnsf cp6y14e"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462084","answers":{"choices":[{"position":1,"visible":true,"text":"2ak2bbt6l 958nphvdetkv4 ycoevsx u49by0 eohbcorqyd2fk","quiz_options":{"score":0},"id":"4385176763"},{"position":2,"visible":true,"text":"6lnrx0h 8rtivmuwxpy8wv ebygucb3xu1m42 iyep7ijy4gu0mvl s1oco7cxc0","quiz_options":{"score":0},"id":"4385176764"},{"position":3,"visible":true,"text":"m5ysyahej9ffbl hiob9pqg7je5r oqyup1bmda i142y odbg2sgkibp5a5n","quiz_options":{"score":0},"id":"4385176765"},{"position":4,"visible":true,"text":"r56bqr7ts07qj7l 0odv2rxnmffpw ctmr30tp61ibr l64xab0cs nmqjd 21rw3b cy2xks0me55b05","quiz_options":{"score":0},"id":"4385176766"},{"position":5,"visible":true,"text":"wjnn6f 4at64hj rif45s1bu9asypf vmjw8wv55isa 87va6t705w","quiz_options":{"score":0},"id":"4385176767"}]},"page_id":"168831461"},"emitted_at":1674149694693} -{"stream":"survey_questions","data":{"id":"667462086","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ikhsonq7 fadyfis1w 1qmo7d2wgcq 5nht9h9j esydnd1m a5ehikxk6ypubk dtanss721p6"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831461/questions/667462086","answers":{"choices":[{"position":1,"visible":true,"text":"3p8dvrh6lr0 iejlsiah ch3n24c7d9tt 7w9rvdkxpqe8xh fo9jq7k8sa1 er31i44d633 xurs2ie khm8mmp5d8gc9 egn43 86jj22","quiz_options":{"score":0},"id":"4385176918"},{"position":2,"visible":true,"text":"r5qhjy06onu7loq n6htekeur 2xm1jh99 ibsp4oat8878fy 940xq9 n4tmuhn e5901s317nbq pevqvednus0ph8g","quiz_options":{"score":0},"id":"4385176919"},{"position":3,"visible":true,"text":"at4b6f9tud4tvpl uf1d5p jgci9m0u17qkj 8tmdkc9o lb7b63gth6 6ds89i5 uu91pd2ybc wwk60i ntwtf5enlg5h8o","quiz_options":{"score":0},"id":"4385176920"},{"position":4,"visible":true,"text":"ovrn2np gg0nxt88wji6fp ckphrf1l3 0um296qkhvgh","quiz_options":{"score":0},"id":"4385176921"}]},"page_id":"168831461"},"emitted_at":1674149694694} -{"stream":"survey_questions","data":{"id":"667462094","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"uhxc9mpgrh9c9aa 05iqvfqi y8rip1qcmmxh b8jj5k cavri2y5 1sqge 7ywuxm3awoh ddul9pvje6dcr kggd2y4"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462094","answers":{"choices":[{"position":1,"visible":true,"text":"t30umxtof3b6e 9o88p i9hcrmm57pq42 16pnnhw idraibx70sby1i","quiz_options":{"score":0},"id":"4385176959"},{"position":2,"visible":true,"text":"xnxw9wa4fo 157vk4i0d 1m9fluu gmmy39m41 icu2b548yd9p3o 3yj0tyamvyjci1 1c19qkbuqb031r","quiz_options":{"score":0},"id":"4385176960"},{"position":3,"visible":true,"text":"ehjwxib3hha9 bjj11etv shhdikfh1iy0u nioigschrcpqi2 v7mgatm l98c4no7k9 bf2j11cmi1dgbt2 f8etga4g kg2w2yngt62","quiz_options":{"score":0},"id":"4385176961"},{"position":4,"visible":true,"text":"uyqt6g2o61rc wcwn8q7ohbhegr wtwkdefljgy uli9v6","quiz_options":{"score":0},"id":"4385176962"},{"position":5,"visible":true,"text":"7pg26slappuq1 aw3vsgc4317be eqe9a8v6s5whg0 dvrw9imoe o9wl91vi282 4h7f26","quiz_options":{"score":0},"id":"4385176963"},{"position":6,"visible":true,"text":"wjipnxgtewp8r ddtad8 yqo4rj5x657nge ykw2qghp3e r3tqrvk1m7l ir60pwi5g5 h7exqf4","quiz_options":{"score":0},"id":"4385176964"},{"position":7,"visible":true,"text":"aafwgbh80 ip6mgadg 8ls8v4ss2fk4y s19ti1jvfnaey g0v4qdn ifk0ve j2trlbjolhuc3 op07p93t1xsh0 kqiw7s ktofe9muryqxe","quiz_options":{"score":0},"id":"4385176965"}]},"page_id":"168831467"},"emitted_at":1674149694694} -{"stream":"survey_questions","data":{"id":"667462096","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"hh8r3c6evhaj2ug 34w4tpf8 b41jtyj tn7vh1hwl 7rrs5 p5rct6v9pg7k 7nw3etg9"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462096","answers":{"choices":[{"position":1,"visible":true,"text":"4jlthohsbk gq1uy9yt 83htjom d5udup0g0jfsido a6nxtc2vkdn 7tfgt9 6e6wsq","quiz_options":{"score":0},"id":"4385176968"},{"position":2,"visible":true,"text":"nt9b0e1tk eq9ckgs2u2fiu probh n7929oe950jcs7c uvdxlffx66obu 6oschegtcq q2a7d hwp71gp7 s3b03f5p3g24h","quiz_options":{"score":0},"id":"4385176969"},{"position":3,"visible":true,"text":"u1p9r on01v021jli y4n4d2gpcbvk fe3y53pjwje6ms atckrpa2r44f 5lkaim1 coo8gs4mer wfbusogrereep 3r2i1qlkh6097d1","quiz_options":{"score":0},"id":"4385176970"},{"position":4,"visible":true,"text":"i4liv68323s xc5i56ba5pa osxulquad221n xbmov jnhv1ogkdw9","quiz_options":{"score":0},"id":"4385176971"},{"position":5,"visible":true,"text":"tlinp 6b4ig tcw2f9no5xm7stv qv6l4foeesgh","quiz_options":{"score":0},"id":"4385176972"},{"position":6,"visible":true,"text":"jfxlbnonc tee9khi75t0ah wr83dgnnsc0l pvyf5t266eq1ev","quiz_options":{"score":0},"id":"4385176973"},{"position":7,"visible":true,"text":"rv7aly5o19 y3hvj5byk uojji58u9thv w8dnv stba3pan 5yho9m1f3o097n7","quiz_options":{"score":0},"id":"4385176974"},{"position":8,"visible":true,"text":"0kmmyvxkq0ixg wmm2bnydk3xg nf2e5e3fn4e 0jd59 tu9bib d6i9t6 3ikku8yd42gnt 0uusc9kip2w","quiz_options":{"score":0},"id":"4385176975"}]},"page_id":"168831467"},"emitted_at":1674149694695} -{"stream":"survey_questions","data":{"id":"667462099","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"952l4noismh4 jwlr6yhgbpo 9waador5oikm xg4aqk0w5o08j 7bg3d02bw22df 84xdy6eq34snqhk h4ngqp6rpih4 9jevsisqvxcv"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462099","answers":{"choices":[{"position":1,"visible":true,"text":"lknhasa3eu03sf p1uflsed0h3sr q1b0gmga v8swv5rbv","quiz_options":{"score":0},"id":"4385176976"},{"position":2,"visible":true,"text":"u6hdwwqb4cc84s8 8jo8vmqx7 klsxteic7f 9m3mg59ov s7npsi1 89i5dq8chb 919k2r7u0t","quiz_options":{"score":0},"id":"4385176977"},{"position":3,"visible":true,"text":"xjpll0bpay536 2ex024o9rd 7boywqww rktvkxinf9fmoh 7d9cb06es933e jm2hsgge2cjy5jj 1ecnm8shkp a3flgyb gcf1622ijpsj ydnun9wos44","quiz_options":{"score":0},"id":"4385176978"},{"position":4,"visible":true,"text":"n7q0ogrerpn8dl qmip86lbe sl1xjutnlap7 3bov7 5fihf81ek49s 9p0f4 r1rptp","quiz_options":{"score":0},"id":"4385176979"},{"position":5,"visible":true,"text":"kwdbb5 hi349arw9 r2f0df eee097e k5035hpewmi6 ky0f9 kh7oio2u0bilxue 5l0moimxob7fid oeno82n5rkplm6i","quiz_options":{"score":0},"id":"4385176980"},{"position":6,"visible":true,"text":"finhomjk 48ciiw7x1g2f c50x4 9kenej45r","quiz_options":{"score":0},"id":"4385176981"},{"position":7,"visible":true,"text":"e7ravpe7reoe5il 9kgwh794cl9x0w snp3wnpq1ryga p4vpk03q7","quiz_options":{"score":0},"id":"4385176982"},{"position":8,"visible":true,"text":"ao3rdw8o5r0xvy x07pd7op60cce ylfrtqpvu81e3u 3f7jtv448v mfoeywlt","quiz_options":{"score":0},"id":"4385176983"}]},"page_id":"168831467"},"emitted_at":1674149694695} -{"stream":"survey_questions","data":{"id":"667462100","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"onm04qqjiu e5hh4bm1nrm 4lql830cednt4y m4vd6n kk1rvup ghucvtc uw8xrjg6 u6giu1qxo oywqk5fc4elxs"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462100","answers":{"choices":[{"position":1,"visible":true,"text":"n66vr4n4vn w57cw3jovcl9 9xhjvp8 mkuivep2954e71r","quiz_options":{"score":0},"id":"4385176986"},{"position":2,"visible":true,"text":"dncqrcm vkp1wf2u0d9j 9o2d4ag7rwt827i wo23d8v14j1e3p 8wty8ixwwhnc6 huw4we49ag40 fruxk5p9080lxhb hjvf468vja8rcxr","quiz_options":{"score":0},"id":"4385176987"},{"position":3,"visible":true,"text":"54u3kd450tpm i9jjwlr8r3xbduw 3k1ic1tg4 4tijdweafs ypprmy7wcpxeg yvvc4l19sd5p gmleu45","quiz_options":{"score":0},"id":"4385176988"},{"position":4,"visible":true,"text":"a1hrtqesu1ph 88spqcx6hyo7 dt64okx798gal cy3tbwljajmrr","quiz_options":{"score":0},"id":"4385176989"},{"position":5,"visible":true,"text":"ehg03 uyx121pwsyo 7rhvgcdfy2st8p 7ahaiboegtn5kd iu8bkj3imm3eo8d k2s2gg4g 2ys9hfx 8vpui26opm6n vsf5rfq6tqi2 jwxoe4suo4","quiz_options":{"score":0},"id":"4385176990"},{"position":6,"visible":true,"text":"c7lxx90luvc6y ogvbopnx 12v4swfg lofg3gcj dq1r6s8ptp5qho h04pqe67tbcnno xhyh0rp62kqkb dvtuwyu u89ppkdl876m","quiz_options":{"score":0},"id":"4385176991"}]},"page_id":"168831467"},"emitted_at":1674149694696} -{"stream":"survey_questions","data":{"id":"667462102","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"bwurewbkcd12 acvusgq 3u00c224nnv iyfj31 i9qtkl9tolx2fjr"}],"href":"https://api.surveymonkey.com/v3/surveys/307785444/pages/168831467/questions/667462102","answers":{"choices":[{"position":1,"visible":true,"text":"m6ihyhdpxt1 0xf9391 y8wvcnlw tfvxn6m 0bswb5w5p4","quiz_options":{"score":0},"id":"4385177002"},{"position":2,"visible":true,"text":"2kdjxc93o enx72tbxki oj43409eg90kt1 5fbnj d8rvo5c7a7vrgf 3c66h1 guo3c x6uv8kl qy0ttl 8r8dgqope","quiz_options":{"score":0},"id":"4385177003"},{"position":3,"visible":true,"text":"h3292 dtp4cjf11njvbhh huvgv1 onie86 tvdr09jg2lvp","quiz_options":{"score":0},"id":"4385177004"},{"position":4,"visible":true,"text":"16ua2fvav 75q7b2odhv fbw0xrnfn 2t3ivdrphr agrq241x d447hwdy rxa3g9gwe23","quiz_options":{"score":0},"id":"4385177005"},{"position":5,"visible":true,"text":"1dl0nhl0bx 7v1hc ooiv3gogf nxecupy","quiz_options":{"score":0},"id":"4385177006"},{"position":6,"visible":true,"text":"97geexnx4ussry4 s2x8sduenrfgwq y4s8f5v e6nrx0st23b f1xq1ax2xwl6yl c1to792d3i l0aq9 92s0aix8l31n5 qnnn38brgah","quiz_options":{"score":0},"id":"4385177007"},{"position":7,"visible":true,"text":"cg3w94v47ecgrik hviug 5xub14 808gw4 g0j3vn 5y58dv7vn8r5 xg72mp91 2l4ubwdo1wrw0t6","quiz_options":{"score":0},"id":"4385177008"},{"position":8,"visible":true,"text":"t92n3lmvuxaf5o5 29jh4afc h2wvqr9xf1q4w 27xf5d9ij1on qvbuiu7 8oghx0dt3nr5 g1mw6f9y6 muu0hhili","quiz_options":{"score":0},"id":"4385177009"}]},"page_id":"168831467"},"emitted_at":1674149694697} -{"stream":"survey_questions","data":{"id":"667461468","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"fn1i25upg io6aniuf yrrrp8vt07fo19 48d5ol09146s8m ldy0ebojp819g"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461468","answers":{"choices":[{"position":1,"visible":true,"text":"t56ghqeix s9huggwiub4 7tqxa1m0h7kj5 yk8r2 0f65tvl3fnpo 0gebiihpxcovnht","quiz_options":{"score":0},"id":"4385173226"},{"position":2,"visible":true,"text":"5o75tse0c okpssn319qiklp 5uyby90fx7n slfgxco07i ejp0kn xmr7ghykkxbypc","quiz_options":{"score":0},"id":"4385173227"},{"position":3,"visible":true,"text":"qe6tkjiccorb lxdmlb30 jmh6j4d6p vces42b344ry gbgmssamx0tvcs 785dl brkbvpl3ctq4nj2 yjjy0hh9b4sr yq2v7lhwtje33x2 7pbqmqgul4gqg","quiz_options":{"score":0},"id":"4385173228"},{"position":4,"visible":true,"text":"lyn712e8uyrrar 0qvy855feo 5l3egg 6t5uin58qrj1hj","quiz_options":{"score":0},"id":"4385173229"}]},"page_id":"168831345"},"emitted_at":1674149694802} -{"stream":"survey_questions","data":{"id":"667461471","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"wcrsas 0v14j5i3e8eb2an ymydb45sqwsn g8j35s649o3rq g582wdp eoraxaf8dc7gf qtdw5a27iucmesj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461471","answers":{"choices":[{"position":1,"visible":true,"text":"nd18cy2gv u2bvpyki k0f38w8 qtlqut6ttsdyk83 pptqq84jmec6bfd a1aks3iy0i oykgg4gv0 busn0o8cc","quiz_options":{"score":0},"id":"4385173236"},{"position":2,"visible":true,"text":"mbd0ll7jes0qm 02w87s60 jslfxkj7yeh1 ol4yj 405yk1b kddsvxnk4hsw7 v1t7q9h8k26rt","quiz_options":{"score":0},"id":"4385173237"},{"position":3,"visible":true,"text":"jl49hsj6tjdf aq1njo56no2 f2gw9 eyqqekovg0c7ov kwwlc9b77 i4yu6aa7uxu","quiz_options":{"score":0},"id":"4385173238"},{"position":4,"visible":true,"text":"5v78pcotkwvvlr 7gvcf3xsd1s2 1y6oc8df 3n3f0 d3cg8ihr3ox9qdc rnyw7g 0lv7p6bka ia0p1tqjcew","quiz_options":{"score":0},"id":"4385173239"},{"position":5,"visible":true,"text":"xtha0tjj1 1j0vri su2xvilf9734 te51yhl03q558lc","quiz_options":{"score":0},"id":"4385173240"},{"position":6,"visible":true,"text":"w4st1 gljqfop06 2bu4fs7b fiia7kj22xl0i4v dl88vjnwo3pf","quiz_options":{"score":0},"id":"4385173241"},{"position":7,"visible":true,"text":"ivfna3y9ru jhxob vuncvdavfek1dsd 80ha2m2 8raod4lyb40 3r895onujni4onh xm9h0g6di g8e735xi ax62ju3eihf9a84","quiz_options":{"score":0},"id":"4385173242"}]},"page_id":"168831345"},"emitted_at":1674149694803} -{"stream":"survey_questions","data":{"id":"667461473","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"xc4wjnko0i6 g7p33u41n 1tjsntksw58hn4 ewweiwcgt 58rs9ek5qi1bgvx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461473","answers":{"choices":[{"position":1,"visible":true,"text":"t09fsskd08sl88r 36sh46a2jva sd91wxy 5dncy1vabsf yfkliw o2spc5k ud2oktdabb7bv l76he8ba3y6 1veap0sd vh2aeoqb5","quiz_options":{"score":0},"id":"4385173256"},{"position":2,"visible":true,"text":"fvdf828g92n4cr4 jt9qg8529uihyge n69seyn8haq ido6m3","quiz_options":{"score":0},"id":"4385173257"},{"position":3,"visible":true,"text":"4n2rpc6iv82cene b7pvp 9fakf0 qmfrbf3","quiz_options":{"score":0},"id":"4385173258"}]},"page_id":"168831345"},"emitted_at":1674149694804} -{"stream":"survey_questions","data":{"id":"667461476","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"wojs6fb lri4g 9cd8kjrhy9rd3 j9s5e32o8i l82xpfkqgwhj jma6htdue e1kk3u071rcef 9ivvnisv asaqww"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461476","answers":{"choices":[{"position":1,"visible":true,"text":"j5lja4pu f0iq7pnkrk 9pra3hkah6abp 92wxt7ox6c dcukwklc k46geqe 2xk4g405p39t62 516tsgu xes5tkhjts","quiz_options":{"score":0},"id":"4385173268"},{"position":2,"visible":true,"text":"1l04q3mtxn75 t7ghyls2hqug j2hu42 uyoecoks6edsqns 57ya7vtyoyt u6k0gtnmf 1mm6wtektvp808 fkk89 qqprq k07oheboie1tfhs","quiz_options":{"score":0},"id":"4385173269"},{"position":3,"visible":true,"text":"9kvc37 gqyklsggws9 db070xge8y7fjr qmdym3ft1jp 5rh1txs","quiz_options":{"score":0},"id":"4385173270"},{"position":4,"visible":true,"text":"y9obh2bu8yk7e94 0ftyrjtcxhicd4 iobxi70wog4vf uokiplls1e ah0g8v mpsslhisijk lmomrov 07cfwh auqwaujyytig","quiz_options":{"score":0},"id":"4385173271"}]},"page_id":"168831345"},"emitted_at":1674149694805} -{"stream":"survey_questions","data":{"id":"667461498","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0tljucg0y3kh l2fh1q1i umeldoo665wgb 92c0k ldeevedeust7pg icrfpt7m1cq l6sfwhhk3vtql qpue297bcaeb7 yrn4gauy007n8q"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831345/questions/667461498","answers":{"choices":[{"position":1,"visible":true,"text":"8lcirsh0a t4ydqb06w0pqv 54k3qmnrjdyf j3ebgxwih2m4wa calo6wrbt 88n96xatpw3 hxaq56r brlwuuymmm jc5psliso3joc aoa41qwdhm0inny","quiz_options":{"score":0},"id":"4385173412"},{"position":2,"visible":true,"text":"bxc3u7guumu w525yaov3om5 3n0ld61 bhlpnw e252afl6giiia 56ybyswiyivgqs s1svdd30gf9kqks 522nxgw3obgj","quiz_options":{"score":0},"id":"4385173413"},{"position":3,"visible":true,"text":"dc3t2djjjqkvmfi cu7osl pxtlaevr117irr ji3qcr","quiz_options":{"score":0},"id":"4385173414"},{"position":4,"visible":true,"text":"qqr9ksi gob32wx p01fp4yqobput39 fryunx r020wh2ctxg 7hxy9g8m9m nuqel75h vwr1mjecaopo1hi o9fc3o","quiz_options":{"score":0},"id":"4385173415"},{"position":5,"visible":true,"text":"cf6162qvdywm crcubxjk5m5ju 7vhy5q5kj4j2q 5b55n9h w9v6c9n r7m6xx","quiz_options":{"score":0},"id":"4385173416"}]},"page_id":"168831345"},"emitted_at":1674149694806} -{"stream":"survey_questions","data":{"id":"667461513","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"18i8lhabavlr 32wsccj5f rx8pvk4unsgm7r s5x7es2uepdb 96xgtjo7voy08d7 n4e9p fayts khd1nb8pow rpjp6"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461513","answers":{"choices":[{"position":1,"visible":true,"text":"b0wj1y3wel57 ktqj3mdnpi dhr0k r3tasjvw 5mx1wclhjj","quiz_options":{"score":0},"id":"4385173452"},{"position":2,"visible":true,"text":"qott2n8w37qaai 1qn102yos4 imupcls4nm th4ob4","quiz_options":{"score":0},"id":"4385173453"},{"position":3,"visible":true,"text":"9uksb 862xwr8j1 6vxdyc4 goi32thkhq93 helvbkt266nnrw xvit2acd67olt pvvkx","quiz_options":{"score":0},"id":"4385173454"},{"position":4,"visible":true,"text":"ees3ivm7fphpd 9jj95pohh4 8sc972k8tyxk ge9j787 nxjkeee670jiec o36kcvbluxrp03 mb4e0et0qwqjxyc d13juonhq","quiz_options":{"score":0},"id":"4385173455"},{"position":5,"visible":true,"text":"294dtkjuyn ttbf6ikcy7s 84hmb t17hw 82aljhi8fwnf65 usuwl5d7ytrca x52ehyic447miq isobapv 3gkr1 2wphdit0p","quiz_options":{"score":0},"id":"4385173456"},{"position":6,"visible":true,"text":"nc1kush2rbxkqrr rhbdvjy xn4au2o0o94 xjq5ll","quiz_options":{"score":0},"id":"4385173457"},{"position":7,"visible":true,"text":"mh1igjt m7gteqlf8 q87g5gj n1g1w7s80","quiz_options":{"score":0},"id":"4385173458"},{"position":8,"visible":true,"text":"sfnq7n g13n21dx5cy aky2s t6h2y8j 5rbdufdcu7i ji8jgrcanxhxv piyrbdr72031gh6 7nna7 gn4gfqo3o 9tbatn8r","quiz_options":{"score":0},"id":"4385173459"}]},"page_id":"168831352"},"emitted_at":1674149694807} -{"stream":"survey_questions","data":{"id":"667461516","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"7gnta43 5s7jfw8axcw3tkf vkg8bmasyo6u 91wwhy c1pae9eu342 3tg0jl70wuoc9 yyxo4v8se myu4ls9xoaiqsep"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461516","answers":{"choices":[{"position":1,"visible":true,"text":"qfjo9n 5a7yv16lp71 ow2bbt3n2ke 0jb1xfpxf51s1u f4k77hkwmu wprku4 55311lxm0wcpw0 c53g4v24","quiz_options":{"score":0},"id":"4385173473"},{"position":2,"visible":true,"text":"j4n9jfdyr4cgs gfc0c a8trns3c4m 5mem4dona y097q wxpm5 q4gqos2ukx ufjxck1lmi1nn1","quiz_options":{"score":0},"id":"4385173474"},{"position":3,"visible":true,"text":"4kb04jje8c8d m12jddaa1iljt fr4qqhj 579w6 6u57xdn gaduvgl0u3 887ffq2vq9b58 q4r02mfsjdx6 3v8yskph8uyu srpxeiuxm0ber3","quiz_options":{"score":0},"id":"4385173475"},{"position":4,"visible":true,"text":"biom7l20wjpvl la1rwpf51ia1ex 23gosg8xvae87 ev2iaj4bo c2581d1re2 93rl9 tmj3467ajulb","quiz_options":{"score":0},"id":"4385173476"},{"position":5,"visible":true,"text":"om88wcjqp6f 5fn54j nmdg9 1sriyr pt6p1h4cm vkdmgxanhh 96847e1mgp l8b2t jl8oxa ifs1c","quiz_options":{"score":0},"id":"4385173477"},{"position":6,"visible":true,"text":"l8bo1yes158 w294970vmw33c yp6x582 6sa834 j86xhb qfm2f2 tj7gt7g xkgtymut","quiz_options":{"score":0},"id":"4385173478"}]},"page_id":"168831352"},"emitted_at":1674149694808} -{"stream":"survey_questions","data":{"id":"667461517","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"uy1ifum6sjcimr he9dly7g it4v8exuhulo ffdod9aof90 9cdfum2phl11nin 5habw5hvuj9pqio ce6ftayekj"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461517","answers":{"choices":[{"position":1,"visible":true,"text":"gd5221owwi1 tjya80m2al3k62t xq0sa9fp leyp52b5ahghpaa","quiz_options":{"score":0},"id":"4385173481"},{"position":2,"visible":true,"text":"lk1rk9 94xju3kfoc1w5 8dybd90 9qdq98 rxu5edq9gs99 kks85ifc03qye","quiz_options":{"score":0},"id":"4385173482"},{"position":3,"visible":true,"text":"ykpin7yh wv14vdjfwf0l 295s1en8cwa2 s6vafhu1fops 8k2vvkt9pftfc2 d1jxuoudjb 614hx251hr6fouy x3w49ql1 7b50ohc76","quiz_options":{"score":0},"id":"4385173483"},{"position":4,"visible":true,"text":"dewxv4aajhaw yij71ndhch8 76bjhg71csdc4w 5gn1l cmu7yo vdn7mtuy 6oplwmikjyvv1bi l3kw7j","quiz_options":{"score":0},"id":"4385173484"},{"position":5,"visible":true,"text":"kuoin09q0 lm3vqjykpb ji7lrp5ylkt58 qb0mhh86kcgm crjjoqq gvfejauprgj mlsjegwm 7psv8r4tc4365j8 98t96g2ifjm","quiz_options":{"score":0},"id":"4385173485"},{"position":6,"visible":true,"text":"5c8y8 f7droof ha5vfclhyivm6kl bfylm txvwxasx1j9l qpyifm4y qwhcsobdo 3ui2qts 97h9i1v5jv5g7","quiz_options":{"score":0},"id":"4385173486"}]},"page_id":"168831352"},"emitted_at":1674149694809} -{"stream":"survey_questions","data":{"id":"667461521","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"pgb7ltsb96ed dep0echkqixfc l4rcb230c wl14ohr2hre1o fn94g0r87nde"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461521","answers":{"choices":[{"position":1,"visible":true,"text":"55yon1b4lyavl 6v6trvxwrc8 s1hwd2 d4hrxv9tbga9e0","quiz_options":{"score":0},"id":"4385173492"},{"position":2,"visible":true,"text":"x8ak4mn2 601o7cb6qnhxox2 k8q23 24nb8q6tx diuko kre6p82em vrf8yxch2 53klbp1xtber s29v8xwbiqm4s8 trwpy6qd1464q","quiz_options":{"score":0},"id":"4385173493"},{"position":3,"visible":true,"text":"s1pxp wsngl9et r2o40 18ltd3 r1b8qpyqskyx a94qup","quiz_options":{"score":0},"id":"4385173494"}]},"page_id":"168831352"},"emitted_at":1674149694810} -{"stream":"survey_questions","data":{"id":"667461526","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"q859nk 369qqyw7scs77jo 9ji0c9vow2uv yjucgiioytvwm x0x95w 222dyntjt8pue 1ndsg06k"}],"href":"https://api.surveymonkey.com/v3/surveys/307785394/pages/168831352/questions/667461526","answers":{"choices":[{"position":1,"visible":true,"text":"tnea3634 p2lnkyut0h yllu3g pufn062o 2etbix7jtgiqx4","quiz_options":{"score":0},"id":"4385173522"},{"position":2,"visible":true,"text":"0qf4fyco1ea itos8brey3wgkp xje8mlffe4m8wlv a9b44 jgmtf","quiz_options":{"score":0},"id":"4385173523"},{"position":3,"visible":true,"text":"hc4a1wgbqqryji 4x7enmw1nam8o1o yn3nxtni y7bwpr wm1oww66f5ox","quiz_options":{"score":0},"id":"4385173524"},{"position":4,"visible":true,"text":"0giuwu7l3v 5ufp8 u3mupcfr0 r8977cguqc2 hrfeq0ug7wo0 fjmkook g8up66tqb7shgg","quiz_options":{"score":0},"id":"4385173525"},{"position":5,"visible":true,"text":"xp5qsik ipf6q be83j63io 5kw68q bwb1mxf gt4l3besb ikfckl9f f3lgigcoib t8wa3hjgn00 csln5ikv2a","quiz_options":{"score":0},"id":"4385173526"},{"position":6,"visible":true,"text":"ai0mef32co1ohl qfwbhctboxq89ye u5v2kyq ebo7e 2u7q9yvlgd1oc 8uxx84l65n7lf0","quiz_options":{"score":0},"id":"4385173527"}]},"page_id":"168831352"},"emitted_at":1674149694811} -{"stream":"survey_questions","data":{"id":"667461529","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5c2dy1fcqn80 athbhqoadncx3n v1lt9m7799 6s154slj6ga"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461529","answers":{"choices":[{"position":1,"visible":true,"text":"rqa77mh3qxuc 6c5uywhchx1bgd f6or0t6s ds7hiuwb3bu9 u9x2d8hs 52pocgqntfeejo4 oaryhk woq1xb7akj","quiz_options":{"score":0},"id":"4385173536"},{"position":2,"visible":true,"text":"hxex6h1upkfw81 pad50wjl 8fo1fiqvtqfe 3xg4xms66g6l luckdtn1 udk8r284o iwpvh21 xotxpvww6p29 4gmqucj4 my8vynnaemow89q","quiz_options":{"score":0},"id":"4385173537"},{"position":3,"visible":true,"text":"jyb1qcl8o2gtte jlwtdhkxh65a9e q6xfymirg7g 24hkqyd0x0 9w4wb4dostk 6i8s8mh5 rxmq5y8ti r99xgye0urvl bdq2598nnha o6gaqqdvg5q","quiz_options":{"score":0},"id":"4385173538"},{"position":4,"visible":true,"text":"qtgjiim109 xoslnipbfe v28m271knt yquk8m2j96w6 xff8kw1 1jhr5l14","quiz_options":{"score":0},"id":"4385173539"},{"position":5,"visible":true,"text":"me39vca xqytbsfb 8uiad g1elq77xy1 tv88etyc gj1g8580e1lbmg4 bjnj4es","quiz_options":{"score":0},"id":"4385173540"}]},"page_id":"168831358"},"emitted_at":1674149694919} -{"stream":"survey_questions","data":{"id":"667461530","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"v2whrhb82ev kvvqlgl ek50nx1ar9 a7h4fo9xuaynt2"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461530","answers":{"choices":[{"position":1,"visible":true,"text":"e8nxccvdcdm 6wfcehdl1 o7g5ce64yvs efob6e 9bj7ny6aipb 4dc2wsqwtr6u24 xsxa081k8 4fyis3oyl9t42 lmu0hhgj4ok24m6 fg0p0lt","quiz_options":{"score":0},"id":"4385173549"},{"position":2,"visible":true,"text":"a5cw9gn7fy95q 8ui6bhr1j0djvtl sr2o8a3 kojha7w 1itxl7twgq avmrl2q5jbwds9 gva3tqhpgk8y5 v4aylod k5ci4cngn6h44jv","quiz_options":{"score":0},"id":"4385173550"},{"position":3,"visible":true,"text":"s3s7w5clw y2uvtas6qhvbk qc1vwrlyi2bk goqfgfe1 1ayx9l73nb78dq a0ykwprnx8gxm1 xn695vlym9rt4mk","quiz_options":{"score":0},"id":"4385173551"},{"position":4,"visible":true,"text":"c4qwlt8a dgnsqhqut5cy iq1htk kitbw 5mwuqxv92w70s c8hxelnesk 75craldvy4w","quiz_options":{"score":0},"id":"4385173552"},{"position":5,"visible":true,"text":"qp2pgo2f63 wmlcuryftjwedj yq0xo a1n5ayjf jikmqnf4mdbw 02r05wsu14a146b","quiz_options":{"score":0},"id":"4385173553"},{"position":6,"visible":true,"text":"ootl4 kp8ae6ggbjabg hgusbwsla83 rnl7dnuqo","quiz_options":{"score":0},"id":"4385173554"}]},"page_id":"168831358"},"emitted_at":1674149694920} -{"stream":"survey_questions","data":{"id":"667461549","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"sn8gp1r0uc1og wobjh 6x0e6t4wug m6466nxyawm k81p20 883oi8u wgeeiqdcmxwa1rt mc0fnojb"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461549","answers":{"choices":[{"position":1,"visible":true,"text":"wy826 0uog090ox923 2m3arldq 9v6wr7xwvxfk lbxye5v0svm7 93ji5 vkfvxgr7iy 1r09yxnkpjm23 0bbktk3rfg0bscf","quiz_options":{"score":0},"id":"4385173656"},{"position":2,"visible":true,"text":"krqy1l9f dj0wppa p93gk rvffc v6cfo0 o1areevq8 jbhm4t gcc224tu 3thelsk2","quiz_options":{"score":0},"id":"4385173657"},{"position":3,"visible":true,"text":"f1cpwqowl4ju9 43sdy2 qdqyqw17lxw2v enee166iohhy kq5khb6s r3pucoufnsh hrhcpt q4ukten6scjuvg ldvapiy bg61mb0pf8ofd5i","quiz_options":{"score":0},"id":"4385173658"},{"position":4,"visible":true,"text":"eq2hnewfaec e9jxrip xrrkl1wh3rbcx7 23t2bp5535o2e qwjbhf5051mi iqgykvmdvsbqr 8s3wwn8tqc513c 4q3a2gr6uhdvlc 7dr6m","quiz_options":{"score":0},"id":"4385173659"},{"position":5,"visible":true,"text":"jth67q sto4nhh23s op593ly6lw4p1o gxwnyfr4m99nbvc dplgcarhd m24hy6i1d n28tg7mytetk","quiz_options":{"score":0},"id":"4385173660"},{"position":6,"visible":true,"text":"rlo3t 7ppem2 bx29gt ucqxmy0 jm67u2dynhq iqfpiyufxif 8lm6ydc9u16rj y7ombxa0meqr2","quiz_options":{"score":0},"id":"4385173661"},{"position":7,"visible":true,"text":"vijhvdm7x6h6 4swh9tom vcr06ugp1hl6 50al5 3ndajrt9ilers 2gnm8il0h ry9dtv jqpu16b0ukt qsb5rqrnmglh arno84j1kroy3","quiz_options":{"score":0},"id":"4385173662"},{"position":8,"visible":true,"text":"3ujkb4krosaln 4lk1ckhounns7qy ap07a65ruvnjed vygxbabompj8dn op8annyrush8v p6t0nd2qi w4pp18y4x lljwuc ubknhjf31m6l1tu","quiz_options":{"score":0},"id":"4385173663"}]},"page_id":"168831358"},"emitted_at":1674149694920} -{"stream":"survey_questions","data":{"id":"667461551","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"jf7fw46e14 vd9x8vs6f0av npyn7 ymxbaq24mqtvj bci1xcln9ch6302 eeghuyauh mjkw9hkx0oooduf 0ui0l93x 6m5tbpl yoplf8cqmbl"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461551","answers":{"choices":[{"position":1,"visible":true,"text":"rwk1x rtfy9upgpmj xkawlfe0vo8 3ounnnx3e9rl2d4 e8pqns3nwx pr5st9qy o6yi4klar9onlp innrgi2ua3f","quiz_options":{"score":0},"id":"4385173675"},{"position":2,"visible":true,"text":"pcupu8t3iodmw34 67g78ktu6rmr6nq 8nph4ohmv 9xtmtr2a7p9 6rb6h6","quiz_options":{"score":0},"id":"4385173676"},{"position":3,"visible":true,"text":"m6xlaj5xutmgw l08mgyjovq 6vk9sibk r9am0kbr 9qmm4d18mxnx u1sw4to 3a8ek ekp479980 hbuxaj2bf sbio7yw7","quiz_options":{"score":0},"id":"4385173677"}]},"page_id":"168831358"},"emitted_at":1674149694921} -{"stream":"survey_questions","data":{"id":"667461553","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"uue1b3 s4rj4 dawj4ao j5xhorntprj5p9s obfe7 y47vcjc s2gp2fh6 oysg9o5"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831358/questions/667461553","answers":{"choices":[{"position":1,"visible":true,"text":"bf1i44 xxi5e2h7orevv2 d737wgpt4pu3 pmbeixrmvw1 ya3ww2x gpvmeo04nh5 sx2a7ey cv9ldq57qkus","quiz_options":{"score":0},"id":"4385173678"},{"position":2,"visible":true,"text":"ocmcxbjsqw o1vr3t 8fhpimedvui rn1hqm5m559bcq kw3138dxm77 vsdor8m v55p3f tn721 l4w4a6j3c","quiz_options":{"score":0},"id":"4385173679"},{"position":3,"visible":true,"text":"sme3nthgos 7c5bv0n80ymsa7 x02op26 jjum7w0s7lt2ll6","quiz_options":{"score":0},"id":"4385173680"},{"position":4,"visible":true,"text":"3p2mj4nlrsgu4d 8rygmmsqqxm2pl6 yshbvwq l5mcc5a kdpqvuul","quiz_options":{"score":0},"id":"4385173681"},{"position":5,"visible":true,"text":"o7ehnkg 134kywmgi wm8jcrpnl45if j86ffs6jfgcp 2e25qvayvwj18pp 3rduk lqjqyar2 xhq7yxb8i9ef6u","quiz_options":{"score":0},"id":"4385173682"},{"position":6,"visible":true,"text":"axh7312qg827nk s7tl78iwcr h1b6pv7hn6 o51sbag","quiz_options":{"score":0},"id":"4385173683"},{"position":7,"visible":true,"text":"onu50gtox9de40 v638eyavt886 mu6y85 6a54r1n9","quiz_options":{"score":0},"id":"4385173684"},{"position":8,"visible":true,"text":"0on3vhb3 94ujem74f4 0git5 fr73t6hds rfc4iyd4gwc","quiz_options":{"score":0},"id":"4385173685"}]},"page_id":"168831358"},"emitted_at":1674149694921} -{"stream":"survey_questions","data":{"id":"667461555","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"8nym349ifndc4 1ergxvcs rg8nxji7wngvfo ha1fx7qgh0p4m 3tkgf"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461555","answers":{"choices":[{"position":1,"visible":true,"text":"r7wm8q65rf0ed wj7kk61hjpqrt nkjjva6busq fuhp9098sc6 8ro2p6ww","quiz_options":{"score":0},"id":"4385173711"},{"position":2,"visible":true,"text":"wjxlbkjgaps2 sep73b5ia8yj5 a9wr0gq 9o2wqf7du1v0tl 81u2apxnt ww3588h","quiz_options":{"score":0},"id":"4385173712"},{"position":3,"visible":true,"text":"8r7tkvpljdu 8a667c88iw vtasj a5vacvo6pbxwsw 8in6kb nv2yeug1x 3lv9de7rw8rtq","quiz_options":{"score":0},"id":"4385173713"},{"position":4,"visible":true,"text":"obyau9nl8c9 jj7tk8n3wv1gb qmiuhuql 1m01qx26c213r6 aj6u9iw459er mxax2id3o8iv riondp2tea3 ay77ba gpl30307 440clb21coauy","quiz_options":{"score":0},"id":"4385173714"},{"position":5,"visible":true,"text":"0wm0f2w wykr4 9lcofhvrxdiwlp0 u8yf3 p1bwwa","quiz_options":{"score":0},"id":"4385173715"}]},"page_id":"168831365"},"emitted_at":1674149694922} -{"stream":"survey_questions","data":{"id":"667461558","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0wjijbi1kwvxt h19om9 kjvx4 jqeprmna15i bwqfaihmlk0"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461558","answers":{"choices":[{"position":1,"visible":true,"text":"q7y7hkgfgi1plk s1nwdd 0d7dt14f q604qs 47cwawfbo m1mmdsyi3fsp3l r0qma8q f5buoh","quiz_options":{"score":0},"id":"4385173727"},{"position":2,"visible":true,"text":"sn5hskkfn0ykw if30ikcbmfjsxj g6u5fay eauotfitd2 9cg6qkon 9t0l88ops7633","quiz_options":{"score":0},"id":"4385173728"},{"position":3,"visible":true,"text":"c678ymx 92bxd icea1i4p cjdl771k u37cuw rnjvrihdi89s2x a1vef","quiz_options":{"score":0},"id":"4385173729"},{"position":4,"visible":true,"text":"uqlmm5yk5 1v12y59qw hv82bo91a0leef 111gmwav8pnwe np5mfx6nsq6 cxgilxkmtvmm2kw 8mb5wiet5q wdbyo993 uh7kcfhy59 sebg3lik","quiz_options":{"score":0},"id":"4385173730"},{"position":5,"visible":true,"text":"uq2uqj20gchd ly2k4d4goq1 ehe66mwx ei0a0d4ggv0al9a 744s2h u254g40o5m 0i9o7dqjdkrtys","quiz_options":{"score":0},"id":"4385173731"},{"position":6,"visible":true,"text":"nyy288shht 3eibqfcm69se94 de2u01w0k5o 342ouq sb68ogkspj61j3p blbhcqok 8vdd2u7b2 su21c5iwou qgi1dj","quiz_options":{"score":0},"id":"4385173732"},{"position":7,"visible":true,"text":"fkidasstba rfn3erxiv282 vy0fqx 3acgtlyka1 mykyngnc 8u2kws4tp 35wst9aglv2 chbms5wm974 ajaygp8vjkhq","quiz_options":{"score":0},"id":"4385173733"},{"position":8,"visible":true,"text":"lmp71nc3bt 6eic0k8kacvx 17ititl v9u25fh1x78 vgfpglqvf6j ajv15gaccdsa elo40oe8ht75c","quiz_options":{"score":0},"id":"4385173734"}]},"page_id":"168831365"},"emitted_at":1674149694922} -{"stream":"survey_questions","data":{"id":"667461561","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"su1ullpk4 t553g unva8lk6wfw7 ipvkkika"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461561","answers":{"choices":[{"position":1,"visible":true,"text":"j463gx5a 1n4fa1q94 x4yw9ack6xoh u5l2fmvwlvktyw7 kpgulj5y49q d03pdq 0bl58ibk39hpe g6m66xw y8ajm5qk2uyu 02pbmuywb7h9m","quiz_options":{"score":0},"id":"4385173755"},{"position":2,"visible":true,"text":"4w8kyesurm3674 est71dh9qi23fw jbowojvmon40p jeggkq5soym 4dgqd70 6fx65rd2f94b5t lnsm1pjg0bypfv jt7d9jj","quiz_options":{"score":0},"id":"4385173756"},{"position":3,"visible":true,"text":"atun8p07f006myd wotd669048pp4 j2wa6v97pbj d7uvpqvrv7omvxw p8ef7giw88ft04 rgn3uwqgx08 rhe5yu0hg16b tdaflfh 2jpl4e","quiz_options":{"score":0},"id":"4385173757"}]},"page_id":"168831365"},"emitted_at":1674149694923} -{"stream":"survey_questions","data":{"id":"667461580","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"in0q9d6e580p28o 65hh3n3rbth4000 jowj89wgvpem9u lc918vs7s pt2r8mt n5t61mrm1nw i64uqjqjxw jhpgayjd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461580","answers":{"choices":[{"position":1,"visible":true,"text":"d1k3kg doxgp648opvyvwx o6xwbacsy2nvo oglkc2p5fegn l7cfksxmkd3ekk es7rvcb8t8v243","quiz_options":{"score":0},"id":"4385174059"},{"position":2,"visible":true,"text":"tc374t uo4de jkgh56fi 7ttvi102i5fbjw 78i6rml o5gekwgh5r rtxw6mrnqi9","quiz_options":{"score":0},"id":"4385174060"},{"position":3,"visible":true,"text":"mfwrrhume d5c5kcy7n32d ovt2xo d3cm1 shv19 r339qbp","quiz_options":{"score":0},"id":"4385174061"},{"position":4,"visible":true,"text":"l4pnl52o9gc 6q4ufrpc7q bw16kqt 6bagaclfl21f9 x92gg7slb2h gs7qxwmj6mg o81ntc46r8q3 3tkbuhrp0","quiz_options":{"score":0},"id":"4385174062"},{"position":5,"visible":true,"text":"9nbdwy6f06wn g2tdvcsqsg 74jmcd ulao4 i4f5y r3yn39fp vsxqmxocm59s10d 9l98r8lp7554 eek3weuw","quiz_options":{"score":0},"id":"4385174063"},{"position":6,"visible":true,"text":"l7am7i8hxiy u3ylucwt8lvms bl5ftdk19mxheie gt5x6 4eiq6 7an0oa731ay2p a9gyy1qsjse xif4ton3b hu5v6cg kuasc8ce9ihbjxi","quiz_options":{"score":0},"id":"4385174064"},{"position":7,"visible":true,"text":"iw5t0k3fr7tv454 lmsbkhfkfx6 vq5ds3yhq20b2 jom26vaad fu7w8 f0t9nanj2","quiz_options":{"score":0},"id":"4385174065"},{"position":8,"visible":true,"text":"18u6s9r5 na05j55lqd i6its t96py9n9q h03ueyjkfewy rhiv9a8nxh 3adtql0ksc9l2s","quiz_options":{"score":0},"id":"4385174066"}]},"page_id":"168831365"},"emitted_at":1674149694923} -{"stream":"survey_questions","data":{"id":"667461598","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"fg2ok1k5x73j xi2i2leb9xip0jl nbjab ut48risbmg0 cmxjg1y14a b350n2yl90sq r2fpg 9u3fc44vd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785402/pages/168831365/questions/667461598","answers":{"choices":[{"position":1,"visible":true,"text":"6ia2cfy435ajc4 qwja8 186ntjx 8jshqf5 i32esbjh08 yhf12enewt5 8shld76x18d18 qw01jndxbuhj50 4mtgy6fbn7nfbs","quiz_options":{"score":0},"id":"4385174182"},{"position":2,"visible":true,"text":"e5phn4tqfq 5xee0uu5x v4lw52 hkhk8lpakvb 304yi j23bhqyat9h5c j52nrstj5cqo3q5","quiz_options":{"score":0},"id":"4385174183"},{"position":3,"visible":true,"text":"swapu0ru 89dqkvum0e2 ybf7xmg d0atxxfgpb3ag ylnjkwaufdj8 rj6dedrfte xkc6hqtimwg","quiz_options":{"score":0},"id":"4385174184"},{"position":4,"visible":true,"text":"n4l9n4wq5yas blbdmjmr nkhx7l gh95jmffy0p69j 8kg39ic3a05l41 dmjd7","quiz_options":{"score":0},"id":"4385174185"},{"position":5,"visible":true,"text":"du734l u98n1d5ovgxe 1i94woho8w2k u16an6dw ha0sc8v11aeep 1p9pawo5f6v 9s6ysd nv3v3","quiz_options":{"score":0},"id":"4385174186"}]},"page_id":"168831365"},"emitted_at":1674149694924} -{"stream":"survey_questions","data":{"id":"667461606","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"f9jgr myr928nokvyw5jg fqoan6w aqdnfas0gapwd 9hobof x7d7vy7xd9il kh6cd dteve1vnaoq6o uccii826ldkj1c"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461606","answers":{"choices":[{"position":1,"visible":true,"text":"1ak71apsl1lb2 dtgtym8ftk 52syq wtnfautgqxev t2sp9icu3465pll i4rhctmtx rh5yifue1 k91tlt","quiz_options":{"score":0},"id":"4385174252"},{"position":2,"visible":true,"text":"3t2uxb my2rtujx8njmbtm a2cyspuk yvfub6ofm8a9 819o7h8o eac0xea","quiz_options":{"score":0},"id":"4385174253"},{"position":3,"visible":true,"text":"4t3bj1jwi5bo ofhe7 cg6nbcys7lu v2qv7q11u48 7oowwv12fndqe","quiz_options":{"score":0},"id":"4385174254"},{"position":4,"visible":true,"text":"7etbg4thd4k k6d0nlv 02cobf bby0p2i","quiz_options":{"score":0},"id":"4385174255"}]},"page_id":"168831382"},"emitted_at":1674149695048} -{"stream":"survey_questions","data":{"id":"667461628","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"fx8h961tahe na1oufba ybjyqi qm6girwad8b1xq"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461628","answers":{"choices":[{"position":1,"visible":true,"text":"lakrew7c6qd wsih7a2nkq0no4 94gtxvs36 87wp1kbadg1l 1hkmpi6 2g1juofi5a fmmygrjehmcuygi 0bs6ka 2pj0gvmjpd6b","quiz_options":{"score":0},"id":"4385174360"},{"position":2,"visible":true,"text":"jjsfl l1sxtcih8fyeeof 6m9lk94qqmuaegl ki7wf11a9i93 xypqh9bjq1av7d1","quiz_options":{"score":0},"id":"4385174361"},{"position":3,"visible":true,"text":"eb4np f3xpqwjcv ypljmiu0y337 lu913m i9uet4t 6dvp7afcy3uqh 1xiu25 1qqn7cyv","quiz_options":{"score":0},"id":"4385174362"}]},"page_id":"168831382"},"emitted_at":1674149695048} -{"stream":"survey_questions","data":{"id":"667461630","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"576vlo18en3 1bxycqddfhn6v rsbd7a1e4bbyf ugieqgvywq4 g3qltn1jy8 0ebdn 78lxt9iwu1jx rbdjre1cujgtsq"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461630","answers":{"choices":[{"position":1,"visible":true,"text":"5k5p528j2o b9frywomc0 aukxrjdqt wiuvud9asln2f3w 6a4mdsa22mygo bnwh1sv 5l5s9ae rbo5hecsrd w0x40 6a86lloan5","quiz_options":{"score":0},"id":"4385174370"},{"position":2,"visible":true,"text":"o8jtnh4keupcl6 ff77c s862cc89 avdo6n25pw","quiz_options":{"score":0},"id":"4385174371"},{"position":3,"visible":true,"text":"vq7kbm97 p8svkwom2lo dg91j0 i570c 7gve09y 9xn2svsiu 22oh8ub7crysocm","quiz_options":{"score":0},"id":"4385174372"}]},"page_id":"168831382"},"emitted_at":1674149695049} -{"stream":"survey_questions","data":{"id":"667461651","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"799t8 uw9o03 89r6ndohi8r8bvs gv887i 14be70qd88w qgux0yh3"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461651","answers":{"choices":[{"position":1,"visible":true,"text":"dtvq3u jyp1h165k86n y0rkc4ich02nw qshv8da ppf7j9x raeq9k abqfxgd","quiz_options":{"score":0},"id":"4385174481"},{"position":2,"visible":true,"text":"i2muubbwbuf 3ffi5361bv wdoq2xkw2hsgpkj t45p9rr5mhi 8xwetg9 gc9u9p4hpmkkev do1ik5y 1439x s9k4vbcj4uaay8","quiz_options":{"score":0},"id":"4385174482"},{"position":3,"visible":true,"text":"0bwq2juhggn68 d2ioj rs8ybqs07gxkcri pgqiiha ir4cb94rbh71q u5mssukw nsyithmc5ismqc fyjxgqjx48fsdp dcua4s9spewq63","quiz_options":{"score":0},"id":"4385174483"},{"position":4,"visible":true,"text":"t3298cfyqx7toco rvkbw4338qk9 k5vd0u42if tkkkkm hfxufgj1i3f7vm8 69yd8hxcl5 swv8ipasty 4g6qsfyb0hdpa mx6s5tmxs","quiz_options":{"score":0},"id":"4385174484"},{"position":5,"visible":true,"text":"o7itx tcbcw8snyw 6jmor7biu7 x59bcw2hwr t94tgt axflfvftpmd0h 46td0yk 67ydkpti6t m3v2de7","quiz_options":{"score":0},"id":"4385174485"},{"position":6,"visible":true,"text":"u7s76bgo3nsbob i7yi958x br6wqw77f3mv b4f5yubeq q7ubaeju1t x5t0cx1bc25jhy acvms2 pggajefpr6 c7prvy9 6ain0uox3g3rd95","quiz_options":{"score":0},"id":"4385174486"}]},"page_id":"168831382"},"emitted_at":1674149695050} -{"stream":"survey_questions","data":{"id":"667461652","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ske92t16t19co1u 3qou5xj75ksuy 1rfxe q5tbfl9xa6dx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831382/questions/667461652","answers":{"choices":[{"position":1,"visible":true,"text":"gn0e03jdsij ops28 wxpn34 o9q6o9otb uxw5rwr5xh g855ayu37wsg","quiz_options":{"score":0},"id":"4385174493"},{"position":2,"visible":true,"text":"6yft8d7xv vwn2h vrlpr8r7ppbnfr vtvgbje916jf5 kv7g2ax 292bgj62fdk","quiz_options":{"score":0},"id":"4385174494"},{"position":3,"visible":true,"text":"iicugnyhyj 0t742iccrlgnc olaw2v e67g2xd1etgcq","quiz_options":{"score":0},"id":"4385174495"},{"position":4,"visible":true,"text":"55ganyc qk6vvo 1gfswk294 t4qxy0nirv3a y1wr6vmvd","quiz_options":{"score":0},"id":"4385174496"},{"position":5,"visible":true,"text":"5hjxv2qf8jr svkm4b90cs6n7u 44ewbk0getyk g6hhmaqvlcdj9g7 if6lqdknx rr5abg5ylw7ho2f rabuvef2t0k5x 0a7x5oqof9xhi8 o8tv6ke3ev7r","quiz_options":{"score":0},"id":"4385174497"},{"position":6,"visible":true,"text":"sy4uf cxu4rsabrlmaw kdov7 hx6ta8d4c0x290","quiz_options":{"score":0},"id":"4385174498"}]},"page_id":"168831382"},"emitted_at":1674149695051} -{"stream":"survey_questions","data":{"id":"667461666","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ssubl 48hjj9vj31v omr18dpic pp65f94 j35xxoh u04pidx8dp7i bo1ae4ptsntu3 o6v99q"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461666","answers":{"choices":[{"position":1,"visible":true,"text":"04s0qv msoghfkw0fr4hmq kdv92tuo2 l8htirpt 4h2nxjcla5rjor n1wy2w5","quiz_options":{"score":0},"id":"4385174598"},{"position":2,"visible":true,"text":"64o6m jya4p7twvvs ls75jv9lvf 8gr7y35au5hcqfq","quiz_options":{"score":0},"id":"4385174599"},{"position":3,"visible":true,"text":"x6ijo4tj5qrljkh dk5dmknmhn 3rm63 kdym4 bxjsru9kvh1 g2vp966a8nkh 6dhh6k99a88gt 9b7emois0ldfr swbnqor4k66","quiz_options":{"score":0},"id":"4385174600"},{"position":4,"visible":true,"text":"4uhchfo7n2wmrtx e211emk0v53w a7ckw5lg40n qrx8pw0r5xrph q8ndfdm5g08tex8","quiz_options":{"score":0},"id":"4385174601"}]},"page_id":"168831388"},"emitted_at":1674149695052} -{"stream":"survey_questions","data":{"id":"667461670","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"lbje0ppud4so8xk ax7vnbef0l3fv7j alcnsi48jehkf8c 9fghb3uop 8qfnul9 w8d6lma9 ejcm4j9x604p t7b7dk"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461670","answers":{"choices":[{"position":1,"visible":true,"text":"i6exa76ck98 9pkrhl37j88s 8k6dvdhxg5axw3q 0dtn8xt6 ji1gm0qu 9pie3iotb","quiz_options":{"score":0},"id":"4385174620"},{"position":2,"visible":true,"text":"knjsfkea66j bp5sxiba qa1yvg m3vul 96hgbjtin7ux7","quiz_options":{"score":0},"id":"4385174621"},{"position":3,"visible":true,"text":"jf90cc rc80q71mvp1 8f3qlcc4pls wvixh60l7b","quiz_options":{"score":0},"id":"4385174622"},{"position":4,"visible":true,"text":"u8275ses4k sq0bax8 vbpfs7qwh yhbborapj3ai t6eo1n6o9rf uitrqawsi p2oe4x5ie","quiz_options":{"score":0},"id":"4385174623"}]},"page_id":"168831388"},"emitted_at":1674149695053} -{"stream":"survey_questions","data":{"id":"667461674","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"7xuudm hd85d2fbd49 hxo4gmsnhaxn53y r3oo0piiij5 hvifmm5 mfep196v1yi q9o6w9gksyedtgs o3y01cyw4ca91"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461674","answers":{"choices":[{"position":1,"visible":true,"text":"7h3hoeucihjd41o 4la76h8i8 8dgl2v5eub 962jah6wws8 qdtm85x9t3y96w5 mkq9yvj 81nsju0 koqkfhpe7pw","quiz_options":{"score":0},"id":"4385174634"},{"position":2,"visible":true,"text":"vpmf8jyppptscjq 3ujxkee7dye ggqm9tg9svjgx ashilo1ffin","quiz_options":{"score":0},"id":"4385174635"},{"position":3,"visible":true,"text":"nselknrgymf vq2vd1efu 3a9b7whs2k7bj5 nujxdbcbg8qcjp 2va7wdag r0k00wo6 bd5u5 btk0p","quiz_options":{"score":0},"id":"4385174636"}]},"page_id":"168831388"},"emitted_at":1674149695053} -{"stream":"survey_questions","data":{"id":"667461676","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0g353xw0tmg9uj yjpp4gp6d kkvtb90xqw7se5s 1rtjhd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461676","answers":{"choices":[{"position":1,"visible":true,"text":"qi3ml trfm3r0 hij01mx6lt6 olgi265lrg7a45","quiz_options":{"score":0},"id":"4385174643"},{"position":2,"visible":true,"text":"sip0lix7kpf g1vjmce6cj u3qgbnitdp2w3 gxgg5cc 53rn81g6mr 008agselt2cfwmo","quiz_options":{"score":0},"id":"4385174644"},{"position":3,"visible":true,"text":"29qp4n5j92gre bwkmqp821xab2ri kyqdft g46vvp19jlvq1k 2sgm90q2b6 1ewvm 8491gfakn3p esqiug2cxmg6ka gb9cdrtlc9qs","quiz_options":{"score":0},"id":"4385174645"},{"position":4,"visible":true,"text":"mcijdsetfwvx 9b6tmxv phlw18ap37 20s5wv","quiz_options":{"score":0},"id":"4385174646"},{"position":5,"visible":true,"text":"3g748xwkq6o 2ut8g nb63dyj stbl5wm 2dkk61h6 29bjrer dxet33 v6f36uqes9qu vgqoxu1nmi5lm gtuqaph","quiz_options":{"score":0},"id":"4385174647"},{"position":6,"visible":true,"text":"j5p31 t1ltv5rpt 85q5tlwq2sfsv3l bh4a2e rev7lbj7 2qwf83ylrc3n fiufpgf h3fp2fd0y5o uw3phwkrek ihy0faqy9s1rsp","quiz_options":{"score":0},"id":"4385174648"},{"position":7,"visible":true,"text":"prd59 gqs7t8tmls 633ujsd7d9ni u1fkx5a6mx318w k71rc82lydu43 orkxf utbtip9c1ky72","quiz_options":{"score":0},"id":"4385174649"},{"position":8,"visible":true,"text":"dc5yl63yj6lv jqeok71lte 4leao9ms1xu3oh 0fdb4m50ip t31tl6q57jjpmh v2osljafdca1x6","quiz_options":{"score":0},"id":"4385174650"}]},"page_id":"168831388"},"emitted_at":1674149695054} -{"stream":"survey_questions","data":{"id":"667461686","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"987ob 7kiuo4teh1 8s7deqmvpopsp txo4fxjj5e 8eqdafh1k7yjhwf 889ll 0s23juj5vyyq99g 7390ddsh565 lpni4hn1bi0"}],"href":"https://api.surveymonkey.com/v3/surveys/307785408/pages/168831388/questions/667461686","answers":{"choices":[{"position":1,"visible":true,"text":"1vdqk14 sjvrskliu ejn42p38f ougj5qtxi6k5q 6d2ccy9073i9","quiz_options":{"score":0},"id":"4385174676"},{"position":2,"visible":true,"text":"b0yc27 c3v3i nhneas7k4y dqth374wqfs9hqf eg6g53l03g 8fg7266kcffpx7 6i9d7rffvncy s0f2on","quiz_options":{"score":0},"id":"4385174677"},{"position":3,"visible":true,"text":"k2kv88vnieg5pk fxdpcorngy bmidk g2lb58 qkbeew82lmprw","quiz_options":{"score":0},"id":"4385174678"}]},"page_id":"168831388"},"emitted_at":1674149695055} -{"stream":"survey_questions","data":{"id":"667455128","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"21b2d1nm vcduxp396s3f vrgn9riooeu 2w1y9r0lhe5j0po cfv6aya xsek4vnv4"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455128","answers":{"choices":[{"position":1,"visible":true,"text":"1cqki1ijt cnkrfpewyh hloep tun23aq fm2hlturu wp1hn1005","quiz_options":{"score":0},"id":"4385137324"},{"position":2,"visible":true,"text":"8q54jh807g67 0chem4m32w 6g6cg4kfbpyp48 wgbo0l1 3ivc27 h1g0jagebgvj36d loo7agxubx imfmbchanx4w47p o4rfv9","quiz_options":{"score":0},"id":"4385137325"},{"position":3,"visible":true,"text":"axjh0ev2g5g0s01 d1f1ekg41aq 1r7fcho4 adaclkcgj0ra1h5 obp9ot rak96hmu bx3804jjlr5vg wruubenes3g4p h41e18dufrs40kb 9t34k1uad6g2a0y","quiz_options":{"score":0},"id":"4385137326"},{"position":4,"visible":true,"text":"0rvnqa934hd356 yx0p7kab01kubvj yuy2tik0o6t 3854yf6up2mgfc kqn54h6qvfoj jecuuxm0d9s u5d8y0hmv5bcww eumwsigjumvc543","quiz_options":{"score":0},"id":"4385137327"},{"position":5,"visible":true,"text":"sutsvkyg7 x3pr1f8 gnn8yg 32ls0it6m8c5c3 5xhh23hb0bck c5kbcphdgm 8n1839 8oe5f4wdsxv oyijcnki143a","quiz_options":{"score":0},"id":"4385137328"},{"position":6,"visible":true,"text":"a14e7gtqqbrm ei6h8ynhoi92vs dq0oj6m rulpoxmdtew b1cnc7v 8pxkgtbq6vu0o","quiz_options":{"score":0},"id":"4385137329"}]},"page_id":"168830050"},"emitted_at":1674149695180} -{"stream":"survey_questions","data":{"id":"667455130","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"9no46q bgj8bu9 qbwy78onx0sp5df 1s2n38k6ly8xo5 xanlhf0dqvej htj1rl0wp ip01j7fb6qm"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455130","answers":{"choices":[{"position":1,"visible":true,"text":"7dpysnial1 w484xmm33rpralf fsb2j19 r599iaw30wgp gk8tit5i681m 14lsb5gmddkr qrncp46 vp0lw srtca1i587dq","quiz_options":{"score":0},"id":"4385137340"},{"position":2,"visible":true,"text":"0yflf5ua3 5l5tu93l 45fu03hrvt kii5twmbs6n uqppc ca18nfpevvny 8od85fpdw gc9isqks5ou7","quiz_options":{"score":0},"id":"4385137341"},{"position":3,"visible":true,"text":"uuv580b3 ejgygxa3nxuod qx2s8cx85l1vu c96g3513rcjdmuv 161c5f mrclx05fg2o 9r8lk172","quiz_options":{"score":0},"id":"4385137342"},{"position":4,"visible":true,"text":"8jhq2vl1hj6 dvnvp6su wshwoa9 unbnfs wjkm9w4w5w glupwsmbjeb3 26q9svrti 21b4potlf4d","quiz_options":{"score":0},"id":"4385137343"},{"position":5,"visible":true,"text":"y1xu41qdgnk hw9yooajs6n xhaqwoc8 hxi6ioaqcnf3 le1ein6yt","quiz_options":{"score":0},"id":"4385137344"},{"position":6,"visible":true,"text":"y7vgshrq086w 18x5gdxg0 ibct5mdxqm78py aq9lw3kx kq3rfwnnod q4a8qk7q 2mytdfvtgxe q7w14xsgcfsw4","quiz_options":{"score":0},"id":"4385137345"}]},"page_id":"168830050"},"emitted_at":1674149695181} -{"stream":"survey_questions","data":{"id":"667455161","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"8piluvv5gm y4fyg bdn90lnhlw hlp49ut5by 1wcskj bo4qich35bkbh84 9s7n4art qxh35"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455161","answers":{"choices":[{"position":1,"visible":true,"text":"7v9pb77j2u5h8lq fjdckl44c6tdi oxtly8metbj35c7 2coi267okn 772jt3x4 8pdrkhod14i eeyu3ox bkl79rtci65 0jt9nxkpamu5","quiz_options":{"score":0},"id":"4385137468"},{"position":2,"visible":true,"text":"w13utk5t7u98 ltxgdnhqgh3 tv729vqrf2g8 a3kf9ym ddp3yd ryqfnhyyshm","quiz_options":{"score":0},"id":"4385137469"},{"position":3,"visible":true,"text":"4o86igx uap6ssaxn rej7oi49j4nj g1afe aj0rahjii gdy2ygm yrq0r33ljaqav","quiz_options":{"score":0},"id":"4385137470"},{"position":4,"visible":true,"text":"ywmratfmy ko10n3cmegap9f vor9gwdvwevpi2 7oid9t72lp yc37r5pssxrm d9fdyo6kj9g5","quiz_options":{"score":0},"id":"4385137471"},{"position":5,"visible":true,"text":"l1ei3wav57twf yn5mmi2almiidx7 gdkyhn 3npig9sip","quiz_options":{"score":0},"id":"4385137472"},{"position":6,"visible":true,"text":"rr84y ht7saym46 rqxpn8 qix3c","quiz_options":{"score":0},"id":"4385137473"},{"position":7,"visible":true,"text":"dtaeau8 wv5ay39bjtph6 uxvtv8 bfa13j2 t1cxi9sgxs9u wh5it","quiz_options":{"score":0},"id":"4385137474"}]},"page_id":"168830050"},"emitted_at":1674149695182} -{"stream":"survey_questions","data":{"id":"667455172","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"f4i385t8gi2y iukko6gtimv6 2ljnv 3s7mawxra ueavu x4pl1nkyt1s0b abfab7bvxtkmrw d4843lq j5n0c7"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455172","answers":{"choices":[{"position":1,"visible":true,"text":"5dkq8pnhf1gag8 u73xqg mej8lwdqo qs1tuihyc heabchr403srhq0 51c5ipel70a csfc3dwuxn9mt9m m9j6t37f xe547smil yy8ji7kgqx","quiz_options":{"score":0},"id":"4385137485"},{"position":2,"visible":true,"text":"q02a6wo56w 6fc33b k7d7a9n7grsgt n7j84b6552eyy58 xapgm1 xxxxklio 1m6q53gxh23a sl0kn09v kdw23flh1k9ss 1kfm8tao","quiz_options":{"score":0},"id":"4385137486"},{"position":3,"visible":true,"text":"d3482iajo1udye ian1i2sop8y5 6j6u9vi h2flbi3mhh 8bo3d4b7wnwto x7dlap9vcos","quiz_options":{"score":0},"id":"4385137487"},{"position":4,"visible":true,"text":"ytjtpqs9tcdnb 5pxact7wjjdtx filtao5oyv w02h42un1hxtd5h dbagrj511v8 d2k7wxlw","quiz_options":{"score":0},"id":"4385137488"}]},"page_id":"168830050"},"emitted_at":1674149695184} -{"stream":"survey_questions","data":{"id":"667455179","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"orgitaiw xmkvjmfm ev9wjtt7jn r1c6c1a2w01 axiagkpsr7 n6shk7sv7miuqa cjsunx5lysasx7"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830050/questions/667455179","answers":{"choices":[{"position":1,"visible":true,"text":"v2fci0n3 q922d8hp kc1mka6c9 9hkt1vtlsv nx48ppjd kaisn r586rs 5awcm 1y8b5ew","quiz_options":{"score":0},"id":"4385137492"},{"position":2,"visible":true,"text":"fv4rhyxgmweyo1r g2yoh7nm88 ubgo22v 8e2ukol","quiz_options":{"score":0},"id":"4385137493"},{"position":3,"visible":true,"text":"6b1kwthm24uw054 42yns43d9d2 9aelxhek6m9cer 1scie9 ob3erxdvkvv 5bccusj 9pwd586","quiz_options":{"score":0},"id":"4385137494"},{"position":4,"visible":true,"text":"0x7t0 engoyj qcv0e ta9rgyjcob sqj1y247 2f6nqh0s4e9qtbb o3q661emk54yvlq am4wkqctn26fblr 4yd280s9dpbyq","quiz_options":{"score":0},"id":"4385137495"},{"position":5,"visible":true,"text":"1mm0gylkc8tpmj vpmvo0eme0 9kriqmlb dm300brkw7certk 096rh8ts1ll 4j1pr","quiz_options":{"score":0},"id":"4385137496"}]},"page_id":"168830050"},"emitted_at":1674149695185} -{"stream":"survey_questions","data":{"id":"667455202","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"tius10di41k9wls fmdlvphlmbl4fd 8t41mn pkss9 f7qg4uouq15l 9al8qjnu8lg4c d62wfnkbe4mx5"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455202","answers":{"choices":[{"position":1,"visible":true,"text":"pxw0w9ui9m4oc inicqftdpfe dyp4ai vn1pmsp6 6cm2gr9b 0ruklaf8 xhb0a8q","quiz_options":{"score":0},"id":"4385137700"},{"position":2,"visible":true,"text":"p28lapcj xmelobuak9wnfe k0ilacxb575 54eam 96ng7","quiz_options":{"score":0},"id":"4385137701"},{"position":3,"visible":true,"text":"gw913n4emtltob sqr3rxe9q alm4u96n5dbp 79stl7bky bseq17ndb ibhcv2mf06av","quiz_options":{"score":0},"id":"4385137702"}]},"page_id":"168830060"},"emitted_at":1674149695186} -{"stream":"survey_questions","data":{"id":"667455205","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"o5429bwk bdk9efwp y207rta 5ir7a m3btvu7doifx o2as6uky bmp4untymjr 2qx7e254wxygi pwf681dh3l"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455205","answers":{"choices":[{"position":1,"visible":true,"text":"ihqtbuwrc7tkk t8rxgekiec q546dffi6yo7ato ip17xm5fe","quiz_options":{"score":0},"id":"4385137740"},{"position":2,"visible":true,"text":"ithl3 gl2tfo1dl uuiy7ls5 7wkeslse","quiz_options":{"score":0},"id":"4385137741"},{"position":3,"visible":true,"text":"ul101be 6w0o3urllk5o4rc m6ttbhcts3nrpq cd93g3j6","quiz_options":{"score":0},"id":"4385137742"},{"position":4,"visible":true,"text":"keye6y0 vxjehf4oga975i hh0hwvp20y0 hmvp1i de9i0gf 309gv73vahw5tv6 b7th25myd dl56yk9tjsnwbkg nlwrih","quiz_options":{"score":0},"id":"4385137743"}]},"page_id":"168830060"},"emitted_at":1674149695187} -{"stream":"survey_questions","data":{"id":"667455210","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ccus8ci s30ote6pf5enhkv 3xw226wx0r4r soqp6i56rx7p"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455210","answers":{"choices":[{"position":1,"visible":true,"text":"ror4u04bm535i fajv10 ji7v3kn9 nu7pljg4qd vtkx1s","quiz_options":{"score":0},"id":"4385137775"},{"position":2,"visible":true,"text":"5xjy8qckk 6w20pov 9ductfhj r469e wem8w rc0jks9hwv","quiz_options":{"score":0},"id":"4385137776"},{"position":3,"visible":true,"text":"difesvxw9al ygl163e5w9dv6 x2glb8m5g6fxm eiibhejl","quiz_options":{"score":0},"id":"4385137777"},{"position":4,"visible":true,"text":"tnfsnewa16k155 klbxdolp it7hi 3bwcsq4kxfs6ag y4rjmghck8fb1 kbr64tikluwhtsh m6uwesc5861 tdfj8p qrf7oe2ydofs vy2xx8qsmcvubj","quiz_options":{"score":0},"id":"4385137778"},{"position":5,"visible":true,"text":"w4u167 jcxljoenbixyu2 xmxki pnigr00vdimxddq","quiz_options":{"score":0},"id":"4385137779"},{"position":6,"visible":true,"text":"s4ksxp4wgiv24 jd0h5q9cdqbaf2 at9rhpltsnm9e 9vvcvfubdxoda 3dr2s4l bcvpvk5qq","quiz_options":{"score":0},"id":"4385137780"},{"position":7,"visible":true,"text":"p0nngm2a15gos i48053r8tp2si 5c356fdw 7lwo3016oo2u ysdomx7ts utp4qqa6","quiz_options":{"score":0},"id":"4385137781"},{"position":8,"visible":true,"text":"hjopydw3jidvpdk opbjs2sr 86c6g94l 1045p9imdm h0ewun d0ki3w4t li53wdcc9 v0m8nd0q8rim c9kuoh 986j08boiew4s","quiz_options":{"score":0},"id":"4385137782"}]},"page_id":"168830060"},"emitted_at":1674149695188} -{"stream":"survey_questions","data":{"id":"667455212","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ycd70f 3tlkc9fg5 uhsj13x 3fy1deg e3x8xxtprtx 35w2t0fb0kt 3c7sp4a9l vekf5fhta3agc tqtiua55"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455212","answers":{"choices":[{"position":1,"visible":true,"text":"3nd2k 12iqy qcy1q78egxti b88f20djp60v 6orn1h44 xb2fsut8p63fd spd9rybvtlrx6 p79lf27l","quiz_options":{"score":0},"id":"4385137792"},{"position":2,"visible":true,"text":"w6va3wm5af u5earys3j7d2w0 aigku 8mhkw6l24gftb lj78ittnu2lhxct","quiz_options":{"score":0},"id":"4385137793"},{"position":3,"visible":true,"text":"1vjpoerlc8kf lnqycklqdoe yt06t24brl aoqrq","quiz_options":{"score":0},"id":"4385137794"},{"position":4,"visible":true,"text":"syf9d5k3axfjhqx f1un5ergu68m7h hf5by9f1 0qf94 docxia9h qcda6u ja83rjw7gdm","quiz_options":{"score":0},"id":"4385137795"},{"position":5,"visible":true,"text":"al4dbwow9is7xut pif50a9 434fsto nh16xfiu34c0eld beshoy","quiz_options":{"score":0},"id":"4385137796"},{"position":6,"visible":true,"text":"q9ufu06kh4d 4oogck630yox 7wsoh 0l2dsb3 8noi1cwam8ukth","quiz_options":{"score":0},"id":"4385137797"},{"position":7,"visible":true,"text":"3e5ijt o0m3chiw3pitxr8 4hciaiuh9c gpc5q4olp3cib 0bhsd3payjog 562gi7o4647qe 39j8aa4ptw jbydrqc7ujb iqlxnqn4uea","quiz_options":{"score":0},"id":"4385137798"}]},"page_id":"168830060"},"emitted_at":1674149695189} -{"stream":"survey_questions","data":{"id":"667455215","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0qm15jfh tv1ui2w edoha95n wbhjp4 rui3rtv6xf4n5 v920f2a1hrd 5d7gj7guq r7ljcjdk2f8t jv5iocm9mv7pg cmow0h6"}],"href":"https://api.surveymonkey.com/v3/surveys/307784834/pages/168830060/questions/667455215","answers":{"choices":[{"position":1,"visible":true,"text":"sewil7695q3 nu4xd9w cr1qqpg4h9qlr ktchj5oeb4 1ns6cdqbjex5f w2y8iflb52 kpkid 1f8unrp e17xh5 qn47cr","quiz_options":{"score":0},"id":"4385137810"},{"position":2,"visible":true,"text":"q21t3nuf 71sfp kag1g9kes bx74gjdj1 uvk0chaofja60","quiz_options":{"score":0},"id":"4385137811"},{"position":3,"visible":true,"text":"nkbskl7xaxqbh76 pgggq5trhj3t isut5qlmwmxnbw apxr23h4v0l","quiz_options":{"score":0},"id":"4385137812"},{"position":4,"visible":true,"text":"jotjdok64gv ya4g5j b0w379 riavnfi10mu3bm 739xph torva74 9dcgi9ns8qlnho","quiz_options":{"score":0},"id":"4385137813"}]},"page_id":"168830060"},"emitted_at":1674149695190} -{"stream":"survey_questions","data":{"id":"667462113","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"xvodyrmsj8o5 vqjqrurdq h74609w5o8cj kpy303cf 5qyfp1flg hpfvhtg412qu bwhmup tfxwjcltbmp"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462113","answers":{"choices":[{"position":1,"visible":true,"text":"x5gvnnvh 39thim64p k3naeh9 ebx5et8ci8vnjwq mwfb0867jts36 lq8ghnv8c23c86i","quiz_options":{"score":0},"id":"4385177065"},{"position":2,"visible":true,"text":"5atgqxr5w uk3ussdg7 29qlw s20qfx0w1u1d cdx47v m76g66t1j x9wuj hyv5095ipd9ly","quiz_options":{"score":0},"id":"4385177066"},{"position":3,"visible":true,"text":"yqvo5j rojr27j6ww 5k0ra1y96j6 vbd70ncr0 wdoxqqhkv a283r7g tshidt7i0jw 70gxqa3 d1wh0y 8q2x4yu5u3tcga","quiz_options":{"score":0},"id":"4385177067"},{"position":4,"visible":true,"text":"53838lqaxys gt190mgek1r2r llxh86fi38xyyb puwhs54wnxa8m5r cb8w1f312hts1 80gh5hrp0o 9a6siov 5i3l99eiefhoq","quiz_options":{"score":0},"id":"4385177068"}]},"page_id":"168831471"},"emitted_at":1674149695299} -{"stream":"survey_questions","data":{"id":"667462114","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"r5gme6gxar 540ixc g0a32bjvkgg4lh 9n8hpb qor7i9od6r max0ae1vu08"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462114","answers":{"choices":[{"position":1,"visible":true,"text":"tk3pmts41 aw7w4y8e5 65iplksmu5wa0o 9ve8f0vgh eiy8hki8e3 5vbwg fa3nnnvu3uq c43makf6i","quiz_options":{"score":0},"id":"4385177095"},{"position":2,"visible":true,"text":"3kvwm3v j0x7vrrpnxmtcub 7h9g43wg71ppkn 71nswd5eq ocqpss07r05dej4 7ln0vdgw0a","quiz_options":{"score":0},"id":"4385177096"},{"position":3,"visible":true,"text":"07td1maptf9wde nrlxmvshy dgpasbrawvpndo id9yjt2tsi mkm0ri7epkw5d poj8pv8m4lu0hc 2xh0i62g5j wxokec50ps52h8 d6uqtrf td70pitsu","quiz_options":{"score":0},"id":"4385177097"},{"position":4,"visible":true,"text":"ccw6cg wd5s85rpk dqlppdkh wpqg1t9vhdq8c","quiz_options":{"score":0},"id":"4385177098"},{"position":5,"visible":true,"text":"8b85ql72lufp 3r2k118 vy5uh6mnntsq5 x49hvqp19g 3oc9laa75hjxwn7 a77dw8 aiskfi350fyh5w 2h7ra ry4mj i2el2","quiz_options":{"score":0},"id":"4385177099"}]},"page_id":"168831471"},"emitted_at":1674149695300} -{"stream":"survey_questions","data":{"id":"667462122","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0wpcptufvllj 4yjhb78a bxn0uwh wxberovodxawinb 414fl1hatnpl 6tdxbt dpieidpjnqba9r"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462122","answers":{"choices":[{"position":1,"visible":true,"text":"c7nxl 3qrbcjg tddcypx5j63ne1 ebrs5911s ilsx5w8mvfs s24oodbw hbah99rw65wt7i","quiz_options":{"score":0},"id":"4385177138"},{"position":2,"visible":true,"text":"1lx4btr3s0pqq63 y93x5lqe xmyta126vo 07q0rnh 4f2bxvoc441 wcnvx ytjeo8g5a wjqrg 134n2q fgobl","quiz_options":{"score":0},"id":"4385177139"},{"position":3,"visible":true,"text":"t9hwm2nyw se2sl72a t70iepatjw0rsh kq6ta 6mhce jgfjh52 w3ivi m0sxg2i4 pup4tsn7a18","quiz_options":{"score":0},"id":"4385177140"},{"position":4,"visible":true,"text":"j6w78imo yhv9sgxs64ii ep2ckum1ge3 3d8gcbsuw j0b95oqfwn","quiz_options":{"score":0},"id":"4385177141"},{"position":5,"visible":true,"text":"5gn4d25al 29tdmvxhe1 svxcxt7qsq pbnlphkoa xvotqve1o79","quiz_options":{"score":0},"id":"4385177142"},{"position":6,"visible":true,"text":"sexvxfmqb jp6xina8q 3gsyu2jfgvpag d2wcp0k5ukuk igg5ecqj rtkd5j 485nmxjd tjodfsyhu5","quiz_options":{"score":0},"id":"4385177143"}]},"page_id":"168831471"},"emitted_at":1674149695301} -{"stream":"survey_questions","data":{"id":"667462126","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5r90gmyyy 1eaacbfag0g4dtj 4rlwnt3r jxg5ruhn tkkml hhlu8r79jma"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462126","answers":{"choices":[{"position":1,"visible":true,"text":"vln5gpqjq tglcfbuxu7b dob43c qnisv2 td55l7hc91pd00q 8gig93a5h 7hrhg6drxsxy bj0oldpxa33by1 4q9mx98wbi p0wb03eabrt87q","quiz_options":{"score":0},"id":"4385177153"},{"position":2,"visible":true,"text":"fj0a0nfwmfjrong q38px84nyk rvqmj 2sv0x6n9a hm88jqn87kvws","quiz_options":{"score":0},"id":"4385177154"},{"position":3,"visible":true,"text":"ms6ilfpbca 8ota11f 1qvg0ellfxis pfyvkkuujjjg qih57geu 366h9cnbrnny9 4sx4q0hbgu","quiz_options":{"score":0},"id":"4385177155"},{"position":4,"visible":true,"text":"si9tpsg54u 13dds2djq4kh kra03y5p43 j8rej5mv d0tkvig6","quiz_options":{"score":0},"id":"4385177156"},{"position":5,"visible":true,"text":"k5ijfpctrn flm2q7kn5t4jf xkjec55tbfsgsn7 r61gy346sxt","quiz_options":{"score":0},"id":"4385177157"},{"position":6,"visible":true,"text":"v9u400dqc fii651ftc 29rn2dffioa0 1m22amm42b qhs0wpth5 mva3eqo09hvo7x 5od22d7a 7m4606e271a6yk","quiz_options":{"score":0},"id":"4385177158"}]},"page_id":"168831471"},"emitted_at":1674149695302} -{"stream":"survey_questions","data":{"id":"667462130","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"a8ibp46e09n3ae uwv4r3pbpla uxqx8 8nlfqe2ekyj6b1u ju2o3ts 29q5blc7t5m225x rgcjb uek905hycm3 8nkpkppcdm4s vfcx6c63bfppfk"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831471/questions/667462130","answers":{"choices":[{"position":1,"visible":true,"text":"xkuphh 9sloeexwdfla qyx58xq9u ne6p3x","quiz_options":{"score":0},"id":"4385177171"},{"position":2,"visible":true,"text":"83uko6624m otef3go 1ba21xa3 6k24mmw t82m7my9ipep","quiz_options":{"score":0},"id":"4385177172"},{"position":3,"visible":true,"text":"ke2gqx614f p7y9pi89 asll5jaju89e1g vgrj3wbe x815bgx 67md1tfhviy9v","quiz_options":{"score":0},"id":"4385177173"},{"position":4,"visible":true,"text":"w5raj381jcgffq 9o1b73 0jnat47bs8bq9e 41ffc wcmivj410jtu19 njxb3v7 gurti qsjowjls niwyt2clulpdpa fe0skx96v529ui","quiz_options":{"score":0},"id":"4385177174"}]},"page_id":"168831471"},"emitted_at":1674149695302} -{"stream":"survey_questions","data":{"id":"667462135","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ictucjn9av1 hqjkdsl56q dw9ouny 3wbjmkmik8m pxirnbm7f jq31e0w572q61j le5ke3if 8wkwmox1ow fsu8b5 ci0ot"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462135","answers":{"choices":[{"position":1,"visible":true,"text":"k4q8hnwb0uticgy kx7rcy sp8nvx036 u552gi 1icsxiqmdmects 0c1d69aee soiye","quiz_options":{"score":0},"id":"4385177200"},{"position":2,"visible":true,"text":"bolb3i7f7 5d7y5 n7il93or tb2599 wlr1hh88d5bjpr vu3v5uudajbgoa s7kmwnhqpph","quiz_options":{"score":0},"id":"4385177201"},{"position":3,"visible":true,"text":"ve8olc60hjdia ake87 4iv5po m484jgkaut 5e285a0hgsrirjm qtnofl05rdtx","quiz_options":{"score":0},"id":"4385177202"},{"position":4,"visible":true,"text":"1pix5bfrx 9pncw4rp5g xf1sm2qr c474c1s67jgcrw trxj2k huiccx1kxt1 l13mo1hij12","quiz_options":{"score":0},"id":"4385177203"},{"position":5,"visible":true,"text":"kf0u04jlsopl 1b4xnt0g 3ehjhwg d9465xfy24uisef 0xv77xfgm7x","quiz_options":{"score":0},"id":"4385177204"}]},"page_id":"168831478"},"emitted_at":1674149695303} -{"stream":"survey_questions","data":{"id":"667462136","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"c4f5ohh3sr2w xoto95lot 7lykuivbhs078b h524lu75 mvss35agi"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462136","answers":{"choices":[{"position":1,"visible":true,"text":"1blqraty 6uddh9u imhcfci dho72p k7feqgqqqr3jm7a 8y7drc4m8m8f1l0","quiz_options":{"score":0},"id":"4385177210"},{"position":2,"visible":true,"text":"cnxmsuqasj3g gm6e1da2102igu c3w07jugiuro afj54 kpoom7 n11yf","quiz_options":{"score":0},"id":"4385177211"},{"position":3,"visible":true,"text":"82nkl1jumcnc6y3 raogw1xp84nswty h77x7 f26dju09o2 ajgbdjdjne6i","quiz_options":{"score":0},"id":"4385177212"},{"position":4,"visible":true,"text":"dukxs7vo kj6j0x kjsmang 6g8vgh t9ymo1h2hfln6 iyuklw ou0uq6","quiz_options":{"score":0},"id":"4385177213"}]},"page_id":"168831478"},"emitted_at":1674149695304} -{"stream":"survey_questions","data":{"id":"667462138","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"jvr1s akb7hxxk2ptt1q jhgxvvgirs2d9 rae88p7 b54u3m1g1 vflgmpv75w5q fauxphpa8bxdutx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462138","answers":{"choices":[{"position":1,"visible":true,"text":"jnkfpiwx3ye b6osvfmkkn eek8p036ayn rt6k5em681q78c 8j9brp8e2qsa 3wu71v85le4x65g","quiz_options":{"score":0},"id":"4385177214"},{"position":2,"visible":true,"text":"oriu3pl31qd r6u7isqu2p1 s6lrtm ybawl9g3oafvxx9 sw16llmo3dpc2xx v0r4lrj serixtbxt sy64a2vcyjf f8mhp","quiz_options":{"score":0},"id":"4385177215"},{"position":3,"visible":true,"text":"v0j4ixrd gxjah4rsf03mhtt selnh 311w2 80liajkb9v2h8 m7aggx9 kmjjabnan kk3g4y8jiuv n15toj tt60bhmd","quiz_options":{"score":0},"id":"4385177216"},{"position":4,"visible":true,"text":"vmvi1nlvkd1vy 13gv0qjxtri sty84 df77l wwh82kck3x0 7jer7beg 8x6vcrnh9qiaf3 nhl5ref5v5bud dol9qivxsg8owna xsw7e0s1","quiz_options":{"score":0},"id":"4385177217"},{"position":5,"visible":true,"text":"a2fdw9jbcytl8ok 0ir460hfwm8 rqecrtla vyi4l3tpv eyhkdh47w2uixp","quiz_options":{"score":0},"id":"4385177218"},{"position":6,"visible":true,"text":"yvusg0ns7jw9tt nadlph4 0ur0o5dhan 9jlq9878999lbf","quiz_options":{"score":0},"id":"4385177219"},{"position":7,"visible":true,"text":"48eqbvdq1ubpvyk 5f4pwr438ik81xj x7ioyubgl90808k 12tr703dg0s 9xpyni000c9sbr","quiz_options":{"score":0},"id":"4385177220"}]},"page_id":"168831478"},"emitted_at":1674149695305} -{"stream":"survey_questions","data":{"id":"667462170","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"csowuuey o93r9336 qf9fym8wbog1q l48c9"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462170","answers":{"choices":[{"position":1,"visible":true,"text":"0k0agd84ev s54c44f90gaj pol5dlpg9t fa5trrngfu cvrd4cmtmq","quiz_options":{"score":0},"id":"4385177381"},{"position":2,"visible":true,"text":"9egly8imf1 9k1iapox y80fk4 58nrajt2 swkm1na","quiz_options":{"score":0},"id":"4385177382"},{"position":3,"visible":true,"text":"se2mcirb6lt5ty qqt53dn jpg4b3wk0c7 91onwco8d7ll0 e7y4mwxa ojs5nky 6u7am","quiz_options":{"score":0},"id":"4385177383"}]},"page_id":"168831478"},"emitted_at":1674149695305} -{"stream":"survey_questions","data":{"id":"667462172","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ypxnp3k6 708jao1b5ok671 cv8x263j0f nk1a0a5cm 7y4r2bclg2rgkd7 alax412w9y76"}],"href":"https://api.surveymonkey.com/v3/surveys/307785448/pages/168831478/questions/667462172","answers":{"choices":[{"position":1,"visible":true,"text":"dil7s3hpsuf ms76n abih45plj 9bw5noujg c759tcfvo7t3e","quiz_options":{"score":0},"id":"4385177392"},{"position":2,"visible":true,"text":"okltv66m4qav 88ijt vcq7k5m xcpwi9e4v7 ujqbg24ap g715rctpafkm0g","quiz_options":{"score":0},"id":"4385177393"},{"position":3,"visible":true,"text":"vev8vhb11 dc5mn3colgaea8g jflk5j niy1qt8onj788","quiz_options":{"score":0},"id":"4385177394"},{"position":4,"visible":true,"text":"hq1wbebc2gleq tj2vf867 n1aqc afpekt91re 89guq9pn9rrww 0bgvg74","quiz_options":{"score":0},"id":"4385177395"},{"position":5,"visible":true,"text":"bxkr18qipct gp9wcqfk n69e6ai66xmy01w xlh7s17hlsbcyt o16ecq6vf42q crmh6p ykbam9mxc37ah2 ii40872r6mws 2xobrm","quiz_options":{"score":0},"id":"4385177396"},{"position":6,"visible":true,"text":"x18wk 3k4tt8ae kkrj0xjbf krc0uyrj2u 0qq68","quiz_options":{"score":0},"id":"4385177397"},{"position":7,"visible":true,"text":"fkx0pjep58wx jpxs8a7vfnf bq2hlx3vhqsn qv4d1evms x64h8k3e tqaeb3mc a1yiqm7qjgi8 cnaw9173k6js msisp4d7bf12 qan9ffq3sv","quiz_options":{"score":0},"id":"4385177398"},{"position":8,"visible":true,"text":"tdjr97it3qvsl 8pfqjvhuo91 mp8kdeymnwv9f 8apy9eshudwhq veo63l6q9np w18s0102tvjru 6lckytkggn6 ncdlx68","quiz_options":{"score":0},"id":"4385177399"}]},"page_id":"168831478"},"emitted_at":1674149695306} -{"stream":"survey_questions","data":{"id":"667455348","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"se3rt7c 9l8d9rig3sk jk791y4nv9qp2 nof9xr8 6irr0mr2uv73g4 rmv1ko9gx2 qievxp819lc o9jbr n1vrq"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455348","answers":{"choices":[{"position":1,"visible":true,"text":"6sudul87 6605b94 w6ubg6a7p0v ufh3d5da nyic6pot11a64v9 awpjgnn22pdgup aidahk8krd 0vhentor51fp ujasxxx1srf","quiz_options":{"score":0},"id":"4385138588"},{"position":2,"visible":true,"text":"08yvwmw 441si6svo1bkxm o85tbsa3k wlkn3","quiz_options":{"score":0},"id":"4385138589"},{"position":3,"visible":true,"text":"uvmasd gagqr6qlswnr af0hp3nfg2vucd vbtkx8vssxlvg6k 5g3rhca48m2 y8gwlm2","quiz_options":{"score":0},"id":"4385138590"},{"position":4,"visible":true,"text":"mrjr3x0 57yyh77a3cmlcmj mmwked8tvqnt5 bwe0hh5dqp1sn3 pia1tu u7ujusnwtvt be24cc8s5olebg 5n65mmu5nki2b","quiz_options":{"score":0},"id":"4385138591"},{"position":5,"visible":true,"text":"k3wcw3bjn hq24e21 f5vvmq0kipi 1smhqty4 p8x2xxy0obkxv qsa4cuvmy81r7s 651b5hlw 2ej3xrm289eqd2 ugnf354v90h 92mk44ghps6cm","quiz_options":{"score":0},"id":"4385138592"},{"position":6,"visible":true,"text":"k8yajsxquuu 94uubd5wo9 piwa122aeek 3fhrq5gvwh14jho 6n8k5a54q7nf rnoqwd82y1 tkhxop blr0p2l7utyydd 7eto449ipd","quiz_options":{"score":0},"id":"4385138593"}]},"page_id":"168830094"},"emitted_at":1674149695413} -{"stream":"survey_questions","data":{"id":"667455351","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"dni8kmob8f47ly 92hb1pg6lvuklf f3hvi6 yjiq3bvv yys743quapbmwm 9xclo63 ydxt59b89 iwks1mdba9iuje vmaq243f75y4m 8cqcf54w"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455351","answers":{"choices":[{"position":1,"visible":true,"text":"9e7yu 2366vw4sa3 rhhgm0 yg8p9f9","quiz_options":{"score":0},"id":"4385138595"},{"position":2,"visible":true,"text":"1ldfxfr k0hnmi dcjdiaquo0xyac remaba5g8 9pslek2fmlvxf7 vn4gl8yjs10up4d b4i21994 78q3d","quiz_options":{"score":0},"id":"4385138596"},{"position":3,"visible":true,"text":"aq8d89iqk6 ytv9mg39tq5 lb5gtx1kdm5a 7scilf5j73g580i tr22c460a1vu dvpernw oy2j9qqnhhbpia9 ufy7twqsl2ovj","quiz_options":{"score":0},"id":"4385138597"},{"position":4,"visible":true,"text":"dlt857wic34l 5y0sijl8 1jmsvy9r e7psvexn8rj0nw ioo4ka04w vtk0ihmg27ac bm60oah9 chljr8vrnt8adx1","quiz_options":{"score":0},"id":"4385138598"},{"position":5,"visible":true,"text":"mk4nh6bnddk1ph 63551586l v0d3vhn w9s7xmbbm3 cbddi6vgmddb se5grc1xr2ycuu bmk1yr7kq3e","quiz_options":{"score":0},"id":"4385138599"},{"position":6,"visible":true,"text":"ob4dd4 r36mdre p96uaag57ld7vo yeqwbn7w2tgi4sj wcgvpd0 4jq0nh42ev8 23i0ye5x 4vuki","quiz_options":{"score":0},"id":"4385138600"}]},"page_id":"168830094"},"emitted_at":1674149695414} -{"stream":"survey_questions","data":{"id":"667455358","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"79kmj4p2d 3sbolhwfrs ioe6b93hwr70jt7 679dpby y4mxy 1pr869ii"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455358","answers":{"choices":[{"position":1,"visible":true,"text":"mqjitf64d0 k83dvyn 227nrf 7a5bulp 0jvp9hmiew umwdpysrqbgp he3hkvoxi8d t0ekmaqml bniybp7","quiz_options":{"score":0},"id":"4385138620"},{"position":2,"visible":true,"text":"ocaqjvupc or8s5on 2txbvlu6e20 8uxgb ll6ktjtk gnwy7knkeeev y2t9cah46lu7 rkmxywsr 8qgmy6gm3fd84 eko2r","quiz_options":{"score":0},"id":"4385138621"},{"position":3,"visible":true,"text":"3kjxifk f8vld xx9lg2ejtf3ccwx 1rfll3376hrm xwaevy7i2krc 93h2fggv1nak vvo0hqr0n 27qcal0ao","quiz_options":{"score":0},"id":"4385138622"},{"position":4,"visible":true,"text":"qqsv99lam uidx4csd1npsc nccxm6ueer 2ihx3c5ysd","quiz_options":{"score":0},"id":"4385138623"},{"position":5,"visible":true,"text":"3ooy243l5h usutwf3do32na0o t5vo8vggvjl pp44k52k3dw swg9bvhtl v4mn5goadw4 9g9jefyf6qhsijc 2j1bmvuu83brk","quiz_options":{"score":0},"id":"4385138624"},{"position":6,"visible":true,"text":"fnme6flca1yfyo kj1q1j tsfq15e2iam8 tyny80 fjb3nf p0mtkwgteciioq p5b79nk1a44d0re lwpwx8wu0e2 at5p0nbaqbp 67tqasnk0lb9dr","quiz_options":{"score":0},"id":"4385138625"},{"position":7,"visible":true,"text":"8n8el8 it06oap6nv fapkjmk yxemrfihr61i heif6anfb o9nel86ws 2t1vpytr","quiz_options":{"score":0},"id":"4385138626"}]},"page_id":"168830094"},"emitted_at":1674149695414} -{"stream":"survey_questions","data":{"id":"667455370","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"lqroep8etllsp 3o8w6vilbyqlwkm b5velmro8 b4ylvv9kkt"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455370","answers":{"choices":[{"position":1,"visible":true,"text":"4bdsyi00551 hsh1woqfmclyma sslfhk y11rdswosp4l 2my277i74fedi lool7","quiz_options":{"score":0},"id":"4385138700"},{"position":2,"visible":true,"text":"qdn040 3tljs j7juh9xjucx 31royj9xs6q lg56gf8c81s","quiz_options":{"score":0},"id":"4385138701"},{"position":3,"visible":true,"text":"ytndri0sy3noxx p2d3euo 4d2jqiygj8jwasr 38lf63x39sxb irlep2 cgi9pr 4pcvga t64nykcc4gn tv9xfejs41 er8a8pw","quiz_options":{"score":0},"id":"4385138702"},{"position":4,"visible":true,"text":"n2hd3mf5 7gg2j7cn55e77 1j1ijenv62 ntd44byw6 g86vqe62ogytk qcodrayuwmht5tn ix3qc42ybhqd 76ry3 goorkieate s2dofir","quiz_options":{"score":0},"id":"4385138703"},{"position":5,"visible":true,"text":"ct0y2yaibh 5hhaeb2u5jmncca ibf7eql6h w3k37s","quiz_options":{"score":0},"id":"4385138704"},{"position":6,"visible":true,"text":"w9647 j2wq2eisb85rlu js474wesi1j 1d7eiqq 3o7ubmmvsbl4","quiz_options":{"score":0},"id":"4385138705"},{"position":7,"visible":true,"text":"deheetxt3hlox 8e4hd76i 517ltrj0v dgi0r19ud5srqj 1qdru4f sl8p3 7gxolfxkhlc5x cm0seo7wroouww","quiz_options":{"score":0},"id":"4385138706"},{"position":8,"visible":true,"text":"4xv8h753xoo8n1 9atxgj8 5eaohgaugpu3in ybq32s67 atg5l7u4aotebk1 0asuiahw4 dyakwm kpqe9vu81dkfvcf","quiz_options":{"score":0},"id":"4385138707"}]},"page_id":"168830094"},"emitted_at":1674149695415} -{"stream":"survey_questions","data":{"id":"667455395","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0qnmuchico4fa8q dfvkni 15mwj2a8 1c8qh1 y3dml65 yfodif0 uik9a vja72b"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830094/questions/667455395","answers":{"choices":[{"position":1,"visible":true,"text":"k9cwqyrhvt6t2y lijf97ywdxctyr nahqrqwo o73wy4r13r","quiz_options":{"score":0},"id":"4385138779"},{"position":2,"visible":true,"text":"0dtp7umdw2tn 3qerkatyyrcndup s70sgadrf0pjwna 13cc30nv 3fku5vgj5","quiz_options":{"score":0},"id":"4385138780"},{"position":3,"visible":true,"text":"8vwwlvlx3 rl3x4l r3itwqo 2uiml2j417 p0x4d9pxyhs bdyo06b oyuo333qq","quiz_options":{"score":0},"id":"4385138781"},{"position":4,"visible":true,"text":"ua5rn0o lgn9qrvh a3xi82nkmd9s2d sdhqh3q8m yu21i9gn3 u4wyck8efnu 47ubnif60vxre wfs92q1c84 qyyup","quiz_options":{"score":0},"id":"4385138782"}]},"page_id":"168830094"},"emitted_at":1674149695415} -{"stream":"survey_questions","data":{"id":"667455427","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"bja345d 7gmh5 u335j3ifd bmn2iwulckt qbbgde6 4l86ghgt3bnplod fsq6qpqp ogqqnp01j gycd6318in4xl s7sxc7ric"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455427","answers":{"choices":[{"position":1,"visible":true,"text":"h1f77hud lvvbp8566vit8 eij3l1f6m86rt9 658f6 auhky2x 1qb9805ptcl4yvs","quiz_options":{"score":0},"id":"4385139013"},{"position":2,"visible":true,"text":"82ow11qv0ewplb iipebne7de pccthtsby8y 87tn87egsum9nd 41xopta 65fhf560c 326vjb4pp4mtdvx 4eqmo8viakl pmuv626kfxaqr 9laiyqibgse9","quiz_options":{"score":0},"id":"4385139014"},{"position":3,"visible":true,"text":"8prnel y3ygc5kxmjbxfc hdd0bos b9w3a1gfm1sm6 8wwao s6j2lw5ennapne 6psoe bxyxi pp141xm uatdplo5f60et","quiz_options":{"score":0},"id":"4385139015"},{"position":4,"visible":true,"text":"yejw4cxbnjeup 9war0kp9wl ngag1wd y6quyhtcv lhai6 6fo022bo2 0y9e1wuj sm9c4jui n9udx","quiz_options":{"score":0},"id":"4385139016"},{"position":5,"visible":true,"text":"o9i4r2ej 825mg 5rifxtuu83ox y3nhp","quiz_options":{"score":0},"id":"4385139017"},{"position":6,"visible":true,"text":"3vukbvawqxox yxo374n8xcpw s6ai05vb64 gld3jyi97 auhaq08 sl5tt43hgv n549d7mf0n7cr2q 44x9v5o31yor0 oel767u6o1evo5 knipqna","quiz_options":{"score":0},"id":"4385139018"}]},"page_id":"168830108"},"emitted_at":1674149695416} -{"stream":"survey_questions","data":{"id":"667455439","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"yvl318t7lj 7wdmc8mttupa 2g7ogtjr976g lm77fy59yrhs4 tb928lr2 kiug5rc d4hq6y1"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455439","answers":{"choices":[{"position":1,"visible":true,"text":"du6tsy5k rhhm4v540es3mo oqkt8crv5wvgvj s00kbxja5h 6kmpu","quiz_options":{"score":0},"id":"4385139062"},{"position":2,"visible":true,"text":"b52dpf uy6exuyx l6ugq4ki qkl8h9l 8ptnm 2jwismc2by8 ls428u","quiz_options":{"score":0},"id":"4385139063"},{"position":3,"visible":true,"text":"86bpwgdk1q0 4iigbk1xjrm hncx3xkk5lj e75h213rkrpjg cku6p9no3qv rn1dvjp5hmtbfar 694ly6v9m ue5ad4q xomxi5c69o6pqm f75mwyy5nd","quiz_options":{"score":0},"id":"4385139064"}]},"page_id":"168830108"},"emitted_at":1674149695416} -{"stream":"survey_questions","data":{"id":"667455443","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6myggi xgx2lptp oxfg8bgavhkxd h8586wxt2vv7 tba2bd 07altrm 2vmmtgfir gchrhk6kdw"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455443","answers":{"choices":[{"position":1,"visible":true,"text":"5acyluby8j u7jbf p5x8foahy k5uk8ubi0 ov4ouretpl lxrf3 ufct467j99f2hfl 7dlgr8eo","quiz_options":{"score":0},"id":"4385139109"},{"position":2,"visible":true,"text":"vpcmt6klp47b0 q3o6j96 3mm9gebofu1n2 jq7dq","quiz_options":{"score":0},"id":"4385139110"},{"position":3,"visible":true,"text":"r1wmg0i0ae892 acmd77ws2k 0pk531 pb5k6xev ury0cf","quiz_options":{"score":0},"id":"4385139111"},{"position":4,"visible":true,"text":"0hemigqnwym5f0j vn57ess p5vjtn nylhr7","quiz_options":{"score":0},"id":"4385139112"},{"position":5,"visible":true,"text":"gl7otpnp 0b43hto2wr0o tnavjwnce9lc5d 7j6hs ca3qbj70t b1nc7q4 7j0696hum k6ytijiprdmdvd","quiz_options":{"score":0},"id":"4385139113"}]},"page_id":"168830108"},"emitted_at":1674149695417} -{"stream":"survey_questions","data":{"id":"667455463","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"pjksyv7o2s qtssl1k83 r6ypxm 30krmk4j0e bp8y6m0or"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455463","answers":{"choices":[{"position":1,"visible":true,"text":"olpxel7n8ktrjx l0x3kfm fg4f96cpv60w2 kpp8waqc9j1y6 2bjrk4f v33tofsgnbg s6gcg386 kcubx22oev1ju i3n5l7veb","quiz_options":{"score":0},"id":"4385139230"},{"position":2,"visible":true,"text":"jkdmotc7gfb43b2 4cfkvtut 6w3ys rn3v7u17ccsx 0gyynlvql xgsrdfrtffvr","quiz_options":{"score":0},"id":"4385139231"},{"position":3,"visible":true,"text":"7ngjrf1wlkv1 wox4shhjp4 yivmg3epgwl5d 11tfmviaf7pv59s 14c6b1vudh brl9d3b799gibr fknowr6c fi6k91n34pm270","quiz_options":{"score":0},"id":"4385139232"},{"position":4,"visible":true,"text":"o8r5r32univvk5 dhx2tc7hbbgl63n bxhmmoi6vk 1ynu42 8goapcp gh1gr8dab p1iik6y eud8jxcg jilsond0434xp77 g1s9ydwgrd6y5tb","quiz_options":{"score":0},"id":"4385139233"},{"position":5,"visible":true,"text":"p9xp9x7yepqx wg9w4q8 9f8n1maik5weupa 9s5nnuan5i co88w9s89g2pfq lxrdlybm6 fl702pf2x7 xsp9xl8yrrg m8xre7g s61if4ojul0qlm0","quiz_options":{"score":0},"id":"4385139234"},{"position":6,"visible":true,"text":"5k001wyjow 9rfq00sg6vcytu yxddli0wkif3wk1 vxk0kcy6l80jfc a06swtlqbdg 2lk8kytqeeqs 81w3s8","quiz_options":{"score":0},"id":"4385139235"},{"position":7,"visible":true,"text":"ia4knoy0u1vku io2wydrh90d6 52j9qii2gcgc9fk tx7409a0o59ffk0","quiz_options":{"score":0},"id":"4385139236"}]},"page_id":"168830108"},"emitted_at":1674149695417} -{"stream":"survey_questions","data":{"id":"667455466","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"17ks2i1p 1roi7n8j p21kn jkrvnckxot 3nkruc3e ugsatu9qx7wsw qsttqci3i03e4"}],"href":"https://api.surveymonkey.com/v3/surveys/307784863/pages/168830108/questions/667455466","answers":{"choices":[{"position":1,"visible":true,"text":"n1ic8gxnyrf1 5qacyxsh ixbp7b33rq4kigy 2rdbf2a4 rfms8di taxqa3oc","quiz_options":{"score":0},"id":"4385139244"},{"position":2,"visible":true,"text":"i4kx3k5s1t 3qw1r6n two72xjft yvu2k2 lywg4w3xhffbwrv pu20atqyr5g 34tsp4wk nwadhm3qirol8y o5x91 755w2suug91yywd","quiz_options":{"score":0},"id":"4385139245"},{"position":3,"visible":true,"text":"2xl9i1c4 11w3fsb78 bfoj2 4rmfw06mn2lxpjo wfijsno52y90 wxuik4 70c6ioht","quiz_options":{"score":0},"id":"4385139246"}]},"page_id":"168830108"},"emitted_at":1674149695418} -{"stream":"survey_questions","data":{"id":"667455236","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ibdmqewlt gbp5yhhp c8i5s75wwg 0xosr1devp 2ijbswdblw0bjko k6httw7 kfr0tk"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455236","answers":{"choices":[{"position":1,"visible":true,"text":"3nj2ikx od392yg7 akufy84 6gw9q axmbaw59fn nfc8xe410jg 6allsd7jg7u 3wyrciq0uje8rn0","quiz_options":{"score":0},"id":"4385137904"},{"position":2,"visible":true,"text":"cb0o3tqf5 ijxi31vsg1 i0ga1cu10vmcgry gfmk6","quiz_options":{"score":0},"id":"4385137905"},{"position":3,"visible":true,"text":"i4frvbso f2k3tyww70ehqkp 5qy73 21lpebsil0fwspd kbx0uypchk wqdb8j0a0 8dr1nvs 592s2coha4r0v","quiz_options":{"score":0},"id":"4385137906"},{"position":4,"visible":true,"text":"lmg2vsbrw180cm otp0wq1e2 530jnetga2dp88 e2uploqajl","quiz_options":{"score":0},"id":"4385137907"},{"position":5,"visible":true,"text":"p0gqg3u03 11u6b6 xdb1lqr bxjqwc48sis 5dd68pfpw lxtumpinwgj vwdyg0uw4u wlvjnya7n7fr","quiz_options":{"score":0},"id":"4385137908"},{"position":6,"visible":true,"text":"74mma6 8c3o4 sjp4tg uagi32nfox489p 33m8i9q2t g1ecllv3xjnumg w3i1p26o789 3g8l35xh9m33","quiz_options":{"score":0},"id":"4385137909"},{"position":7,"visible":true,"text":"217g3m8t fdh1g7f6sng3r nt3u7d6n8j1d5 k8hwfegvg4i3xhs 6wung qmv6ilblwpg4t4c r04g9x2 ra26624 hl1gnkjlcu3pn","quiz_options":{"score":0},"id":"4385137910"},{"position":8,"visible":true,"text":"hc8vkvc9 yd3dll3x 2mo5ol4vkqru4k ubqgf odk7ghioir7gc lm0je2 4q3r0 ks32ix ra4rjxv2d9","quiz_options":{"score":0},"id":"4385137911"}]},"page_id":"168830068"},"emitted_at":1674149695519} -{"stream":"survey_questions","data":{"id":"667455240","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6vdliabstjs pjq6v4ea sux2ll21xwm7h qt6k4qflxuslj ig5xnx aqyngvnivqhi63 k7trhm7s9n yc1uiij"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455240","answers":{"choices":[{"position":1,"visible":true,"text":"n59jsjeq v4yp5kweu77pjd 6dsaj62 f51922ixg bfy1tdx0ajmwq sq2g2ca6t1ag t91oetm","quiz_options":{"score":0},"id":"4385137924"},{"position":2,"visible":true,"text":"miou6 s74ictupiv15j a1ojxwpxsc 24wjut22cino3li 3nef4p8a1onycu s2iku aluhi 78a20fa","quiz_options":{"score":0},"id":"4385137925"},{"position":3,"visible":true,"text":"96wygu4eqjryyr vft0o 1omksgrj4e4u hov2cmxl xryqlb9qe5s 69bf1gxp prv2hpeebouh","quiz_options":{"score":0},"id":"4385137926"},{"position":4,"visible":true,"text":"80jv70rswm80ng s3yupsiahmmfxf 0qerhyspf y1shqteym hu7hk9dkpo 58idxrntiwiqc h36nn","quiz_options":{"score":0},"id":"4385137927"}]},"page_id":"168830068"},"emitted_at":1674149695520} -{"stream":"survey_questions","data":{"id":"667455243","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"1lmh6obuf8d67h 9f7639oxptfr pgb8r ud5359op7klll"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455243","answers":{"choices":[{"position":1,"visible":true,"text":"ht5dtccv20hfamc 6m2dk7 edktxyp 9bpptvmpf 76dv4sin8ps33ac","quiz_options":{"score":0},"id":"4385137931"},{"position":2,"visible":true,"text":"m1e1sw2b xk8ujpclrw rnv8g5qtu04 g1hy75 887fxgre","quiz_options":{"score":0},"id":"4385137932"},{"position":3,"visible":true,"text":"8iqn4b9 ico6j1l7h b44vd6d3383m0 rkm2tqhsi00qy 5hso1f919pq 5yyhjb5h7 dhhsrml5g6kiefx apd2weqls w4hg7 p1rwc2o7pko","quiz_options":{"score":0},"id":"4385137933"},{"position":4,"visible":true,"text":"erh5b0d 47uxrflw 23r21lf24iwf bb8yqiqs79 3y8eb7le7y2ocb8 juni8","quiz_options":{"score":0},"id":"4385137934"},{"position":5,"visible":true,"text":"x32489wko4ns f0ob7be3j pmc9ui3s6qp0 08kfm8yaqcanw 8aot5prgvkqyseo q4vp4n656gj57g xygwva 96gw2r2npb2","quiz_options":{"score":0},"id":"4385137935"},{"position":6,"visible":true,"text":"sjsfbmxa97 xq0084k5hm3 hivfs05sfir 40dj15utx bo9mx1yu0","quiz_options":{"score":0},"id":"4385137936"},{"position":7,"visible":true,"text":"jrb1fu2j x90bw8mlv85gpl0 xfo84sk0jy6 l3392k bucd4nmlc7yj jdj3x2clsir95 uw9dhluee e9ai5v8pm5 4eufmbvvi","quiz_options":{"score":0},"id":"4385137937"}]},"page_id":"168830068"},"emitted_at":1674149695520} -{"stream":"survey_questions","data":{"id":"667455245","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"h3en3q3evla 6eye52o9ijuo 078omcsgacycjn9 a76ryw2wl 5ejj2a399 r62c2jrxj2x7y"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455245","answers":{"choices":[{"position":1,"visible":true,"text":"90t457 4q172 f4kmgq3 vc09o 7hqxmsg9jx6 7fenorkuyq ov1kus876aiv0p rrmrq5qlittb6y","quiz_options":{"score":0},"id":"4385137944"},{"position":2,"visible":true,"text":"9aii09 2r9oaqs23 j2x9qy94cnxfgn jrlxnqn9n","quiz_options":{"score":0},"id":"4385137945"},{"position":3,"visible":true,"text":"r8jgwe j87njrw3yo1fla bbngq3 g26et","quiz_options":{"score":0},"id":"4385137946"},{"position":4,"visible":true,"text":"91n95cu 5fp11un vngsubusfe4 vgbho","quiz_options":{"score":0},"id":"4385137947"},{"position":5,"visible":true,"text":"9qw7f8sdfws1 6wfueox bghljj3 4yal6iqt73m6han l921hy 6yr04p7c","quiz_options":{"score":0},"id":"4385137948"},{"position":6,"visible":true,"text":"trrepiynd5phbcr 35t494bg wm02s1clg9wkxl1 eemhx faxm4gd3aqepewc rp34jr2ho5gb5q ol65t043p1 66bg3yos kh4xbmdvqyvm ibrokgs","quiz_options":{"score":0},"id":"4385137949"},{"position":7,"visible":true,"text":"ni9kx4ob31 mbn9d2n3t4j6lal xel50s53bw6eydo b0s9p p512cv6lrh 2jq1h yqmu3hg70qxw99e k1xjbd vv9lbb4kvlt1jg p86c1lob16h","quiz_options":{"score":0},"id":"4385137950"},{"position":8,"visible":true,"text":"3tx1f04qbxgku4k modppm wc68tgts7o me6s4w7ikqolcg4","quiz_options":{"score":0},"id":"4385137951"}]},"page_id":"168830068"},"emitted_at":1674149695521} -{"stream":"survey_questions","data":{"id":"667455263","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"gx0d844wwsnss b89h0hrw ihlud3p23xbv3 mtv2g99i bda7267b5 pxxsxit4ey lcji2t"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830068/questions/667455263","answers":{"choices":[{"position":1,"visible":true,"text":"e0dhk875qtxip 22ci0nbrenwe tnj1517py jfe77 wnmap goid8 s5r92q9nx9gbxf oy5bkxrwsyqx 7oemt6oc3wcw2 5iwm8","quiz_options":{"score":0},"id":"4385138060"},{"position":2,"visible":true,"text":"nnkw2nj3g1wb 6os5x3rph9 oejmdr7e u9vf9uaxis17w7p vee9xcim164h jkcq9v7e01i njv8rrgfo3hfumd ha6djf y3jckk4un mwvekphui4","quiz_options":{"score":0},"id":"4385138061"},{"position":3,"visible":true,"text":"0b1s6goa 31eeolk05pyxfvy 5fd1rlv8g8g 9iachco 3gqpac6rjo hmq9tr5huh4mxm","quiz_options":{"score":0},"id":"4385138062"},{"position":4,"visible":true,"text":"kkpmmshmp2owso tm5dsf0bt4rm474 w1wwr4 p6ltlf pxm8o1um1cv 8lh8goe9rqqo58x 9j7ej5b 4chhf50nlimyy","quiz_options":{"score":0},"id":"4385138063"},{"position":5,"visible":true,"text":"hygcvxvhqh1um 80ak8e wd9b3vkvg hkyetchu2pdm1 tn8hu6fropre lqx4jilgig1929w","quiz_options":{"score":0},"id":"4385138064"}]},"page_id":"168830068"},"emitted_at":1674149695521} -{"stream":"survey_questions","data":{"id":"667455268","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"of74qo8c 333x5lwc2aral be0joqsxrs hvgeb7ltbcu dtyfa2y98wnn2 jbv7c vjee2juk267 o2fd4pyua"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455268","answers":{"choices":[{"position":1,"visible":true,"text":"jyb7orcvpwo c9lcx 9aq0cld2fwngg5 ple2j5y3xn2px itcvwnp0rpn2 ykoj6qw las5p s01fya1co65 q6qrkss","quiz_options":{"score":0},"id":"4385138108"},{"position":2,"visible":true,"text":"vd9eniq 7lpv8hbl bq0pjeiplw0 f0hjp7xyv2pk8 82d5r7ciqjbl ngrw1mugyp 0qb5g4vnxxjjpf","quiz_options":{"score":0},"id":"4385138109"},{"position":3,"visible":true,"text":"s84erg ueidus67wjw w849glheqbx5p0m w2am53 erjlbf9pu7","quiz_options":{"score":0},"id":"4385138110"},{"position":4,"visible":true,"text":"n9yd3gp4otjuvma 16axdetwq epitspj3f8hq bgqks3 1s7wh28 qn8cff mfk5el7vr2w26pg 7bkx6xqe 1d9eryxaimnk u6sr9v3id0t","quiz_options":{"score":0},"id":"4385138111"},{"position":5,"visible":true,"text":"l3bswpvl 9obip6i6bvg ytwh8i mmn1n0p9xe9kiu rjgto5mi9ce5dm6","quiz_options":{"score":0},"id":"4385138112"},{"position":6,"visible":true,"text":"a7j5u 2td37rxj7 1lwcfapxi2cw6g 32egv55cc52fcv ay1jvha 1169qapnsa0avix w406ev2kwt1k8n y7xqqn i8uoafghx8 gelu50jf149xtg8","quiz_options":{"score":0},"id":"4385138113"}]},"page_id":"168830074"},"emitted_at":1674149695522} -{"stream":"survey_questions","data":{"id":"667455272","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"svumjkj5ei6 t1jkf ttf93l bilmloj40l q6er242eh bojtdo6 sf5jtk 66a4anc 3j5cbh3k xdfpwwtdf3hpb3"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455272","answers":{"choices":[{"position":1,"visible":true,"text":"i6ece2o dl6k3s11vd0 shf3m get86","quiz_options":{"score":0},"id":"4385138126"},{"position":2,"visible":true,"text":"fufxxy5ueb 5u74cl6a lol396aaape2 dwgaeqcphflg ogf55axqis54 1a338abonkbvc5 f0hj7rjhu 15gsns71xund4xp","quiz_options":{"score":0},"id":"4385138127"},{"position":3,"visible":true,"text":"di9p4lr30wahw 2rg14l uqqnx1roqy5k65r 2we720uhdva oyb5f74cil odhas8h1n4u3rj dqrhl943a","quiz_options":{"score":0},"id":"4385138128"},{"position":4,"visible":true,"text":"b2j0ivu58yysns 4roq2 mqukye4mmik92 wpr49dqhc6y g6ivs5m7n9 iotdbwjay566 06vv0nn17yfqb59 peuvkh9jjd jly7qt4151 roq63i3ld","quiz_options":{"score":0},"id":"4385138129"},{"position":5,"visible":true,"text":"1rceqano97 sd5m1s3hsskyv7 bwmei412e ikfkbu xhdo2mx6 aqg5dpo10 sfjb38 vj05jf71y","quiz_options":{"score":0},"id":"4385138130"},{"position":6,"visible":true,"text":"7nlryn01exq u14xxgcx8mngy 4pc2y3 48lxatpljatuox vbt7cvv2cipj7 ise8v3j9wnomr ms330kk mgjaw7kgowjfq6","quiz_options":{"score":0},"id":"4385138131"},{"position":7,"visible":true,"text":"y0sx70a14 527jr the5p62a hga0cg5nio53 hv2s0l 6e2lq473mrdqgo","quiz_options":{"score":0},"id":"4385138132"},{"position":8,"visible":true,"text":"rxtxctm7 veua1a ds34biwojid 5cjj1qvd3 v8ksghp4g 6ist9e a5xrfr8r2 ae9rb2xw lguj1iafi","quiz_options":{"score":0},"id":"4385138133"}]},"page_id":"168830074"},"emitted_at":1674149695522} -{"stream":"survey_questions","data":{"id":"667455276","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"6ms3lgjw25pq b20i2lk pkqw6iy j77i7ux32y8 e323sjcc kh8fxxm724seg"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455276","answers":{"choices":[{"position":1,"visible":true,"text":"nqewt2eyb57 a96u7u 4sr85tfu1 acbvejxvxdui ur1p5sb 4a2r81h0hnw3m 51q3v7s4kllox","quiz_options":{"score":0},"id":"4385138155"},{"position":2,"visible":true,"text":"maso4 ovhyv0ixo hu4gwpq8ky21j9 qo2qc3kuc8n lvou1 a0wbyyp bqr9bre7 csdtfe ya1ltf6kynj93y","quiz_options":{"score":0},"id":"4385138156"},{"position":3,"visible":true,"text":"bjaogae0yct1yk slo3wuygf6 wh0fyn7lym fof1mvu a23qhlj 0rkb9sms 14urdydlb5vht dbw8uh0n3rwdj2i haqry8lhmmpcnmy gw3bvde3lsyue3","quiz_options":{"score":0},"id":"4385138157"}]},"page_id":"168830074"},"emitted_at":1674149695523} -{"stream":"survey_questions","data":{"id":"667455290","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"8rej58igix 5ev85 tiximo qcv1v11x0ixckwa uhte1umn0p v1h2mr0lm6 duw2034nurju"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455290","answers":{"choices":[{"position":1,"visible":true,"text":"xx157diwdtvnm 27fuvnqueywcf2d 7fgvt3b 7qte6 xb8oaf6bdynl","quiz_options":{"score":0},"id":"4385138209"},{"position":2,"visible":true,"text":"7uixmrb45 5217t8r2se vv8oeea62 4mjg3vqh og2vgrc5v251 xe8bb89i iefjxgj 5flbdfdb77 amgpmbo6gav6i","quiz_options":{"score":0},"id":"4385138210"},{"position":3,"visible":true,"text":"1sn25lv32td8 516w18 pdmovblb0hbd50 42vmf7","quiz_options":{"score":0},"id":"4385138211"},{"position":4,"visible":true,"text":"rnlfk p5x50v6jxbdfnkc lfjnc39nl8o ok6oyhwope sr35gd1kr5r8fg5 em4qqf7wj tt8linqt 8e8c6d9 s2geie6vw15ny nbkq71k87eu","quiz_options":{"score":0},"id":"4385138212"},{"position":5,"visible":true,"text":"g0otgfss8 l5ag3n97qp b3hbjigxocewjyi dflt05hus4w ddpumu2h7dx6ff pby1r9n a8d6xu9db rwapp","quiz_options":{"score":0},"id":"4385138213"},{"position":6,"visible":true,"text":"1w7np c3t0x knc262g yrfo4f1r4f4reh i8dhd9l1v77 72eamvjcaggrn9 fb7v3v2g2","quiz_options":{"score":0},"id":"4385138214"},{"position":7,"visible":true,"text":"n5h0km6i 58fjgrqq s17q2dwxiha 9wl01dstrdvo upbsfwpyyryn 5ducq pm1vbp w4no5od9pwqf b3e18e3i","quiz_options":{"score":0},"id":"4385138215"},{"position":8,"visible":true,"text":"o9ln4neod6l3v 5p7w4sosqt95e qk9mir6c48fbqj xlglulme cbafrf2g69p6nj htqcbq8v7u24lyc","quiz_options":{"score":0},"id":"4385138216"}]},"page_id":"168830074"},"emitted_at":1674149695523} -{"stream":"survey_questions","data":{"id":"667455293","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"virtcpk896m68m bbdg4mss12ps3 qo4gk153jqp9 iotdms cxo7l2t9 gtofci5cg0er"}],"href":"https://api.surveymonkey.com/v3/surveys/307784846/pages/168830074/questions/667455293","answers":{"choices":[{"position":1,"visible":true,"text":"1vflj0ggk1o9l nbqw3aj69wprwix i955uorj 69jyg2ci6m2lj sb3sor4o2 jmnjbw3i f6weyfas9eq1 yanlggtlu8823f s7fpa7","quiz_options":{"score":0},"id":"4385138239"},{"position":2,"visible":true,"text":"mgeif2h8oilulvj 4q5x7owpgk sxoc48wbr7u3 rmt30uk4m7w7q hpqj8faxcl3qrc5 qfuhdde a1itv i3292al93b","quiz_options":{"score":0},"id":"4385138240"},{"position":3,"visible":true,"text":"3an10n o05hsocsiq0quj5 9ay5x f6p79mjl jjcw6ym9dpfwrj sygnf5 bh3mo 1x146ti6 x4u0e3pxa1ko","quiz_options":{"score":0},"id":"4385138241"},{"position":4,"visible":true,"text":"va9yadd6 xsc7t62edxbwl d7d7n7ecqsealn u9ognb1ox nmaht pwy1d 5mdngtxn4ol1tel dxlmr67c00e hw11e9xn7h","quiz_options":{"score":0},"id":"4385138242"},{"position":5,"visible":true,"text":"b1qfaxqnj91j8mj edwip5b22pdd tuh6g5uodx2 sn4e9lv7xsuul jxmu0iubodnpw 7rqts1liyv27j","quiz_options":{"score":0},"id":"4385138243"}]},"page_id":"168830074"},"emitted_at":1674149695524} -{"stream":"survey_questions","data":{"id":"667455297","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"lq8cc4kb4wb hagmv535tvyfw4 q505o lc9pke7la"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455297","answers":{"choices":[{"position":1,"visible":true,"text":"gnig56iqkr f2p71ed2w9fr gnpyvvl1 9mc6qpbow5tam mxta8 t9blqkndj9c 1gisvkrl2 0p43lebad 6v56y0e392el ahfwj9tq6lbhm7","quiz_options":{"score":0},"id":"4385138277"},{"position":2,"visible":true,"text":"77wwt btv0ntp gym5s54 lr3ji8wtg4dd qoy58mimj d2yjili","quiz_options":{"score":0},"id":"4385138278"},{"position":3,"visible":true,"text":"9hphq 5qj6yrbg2na mnmodl22e8cg siook2te8gpl","quiz_options":{"score":0},"id":"4385138279"}]},"page_id":"168830083"},"emitted_at":1674149695661} -{"stream":"survey_questions","data":{"id":"667455299","position":2,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0cphg3eq5u jtapt0bso07ghd 0bgtvsb 5pd5xfhq5t1fgf w0jm2nstiu 93om3crky6skr q49leuh249 3q2tkvncda03g 40orw7354cy p8eku"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455299","answers":{"choices":[{"position":1,"visible":true,"text":"mrhrmoj8c c8fd51r 0lbuywjnvnmijj 0tmyxfrg fggvqmm3bvivav 8b3elaxwyb d2wp2i hr0jllmfkd7 prv23lvvws27gx 0e6okfhy5sn","quiz_options":{"score":0},"id":"4385138282"},{"position":2,"visible":true,"text":"mv5uq8u7dup2r lfo2ih2jkc5cp r6jydm0x6w 1p1c3s67p57 7eulrlih 1v7p2vig8 99esod2scbs pg87n9lp9mg476 6hmjxdey","quiz_options":{"score":0},"id":"4385138283"},{"position":3,"visible":true,"text":"ahxj3 imh6reai78juny6 bop9te8ej8q6l gwxwkjup43o6tr1 nbvorbchco4ptow pwomv9iyd9t jkrjgggo3s 6ipaxevsfrxrmtw uq3n0cmg k1odeemd29l","quiz_options":{"score":0},"id":"4385138284"},{"position":4,"visible":true,"text":"t4pu3i ixvxd q10uqer3 gkqtljjmflbts","quiz_options":{"score":0},"id":"4385138285"},{"position":5,"visible":true,"text":"xwwgqr 7hlt9dq 1tloksa kehvt","quiz_options":{"score":0},"id":"4385138286"},{"position":6,"visible":true,"text":"2vinv5qis ipjdbl cuwxgei6t8g is2ihbn xs3q9m3rl","quiz_options":{"score":0},"id":"4385138287"}]},"page_id":"168830083"},"emitted_at":1674149695661} -{"stream":"survey_questions","data":{"id":"667455301","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5h0aahwgk8 eoh0xct ytmtsdr15y fawlco duk30p0qejro"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455301","answers":{"choices":[{"position":1,"visible":true,"text":"1ib8lq0i q896a iwg01totbr1c8 nic9ye883le 9pjfihiuxbp5 6wu9jitrk1 k3kvcggtbgboo9 nenddst 1qgr2y ilmy2v1ddb9","quiz_options":{"score":0},"id":"4385138288"},{"position":2,"visible":true,"text":"05nxifhb7u4 v0tu1 i8u8dd63ekyj h3xah1h0s7k9vn m95qi vmfn7l5i7iu3es hrywpnsl6rp","quiz_options":{"score":0},"id":"4385138289"},{"position":3,"visible":true,"text":"1fgt8f3w 2i0cyt47w3sl6o k8x6l8i3stl2cc 2h13e9tt1cwaa9 e5l78fvc x2y42gkhqhkc7r","quiz_options":{"score":0},"id":"4385138290"},{"position":4,"visible":true,"text":"ye6nyey79pu2 596slmq qgawtw10v06mtad m113i7i8sd1l x9o2f","quiz_options":{"score":0},"id":"4385138291"},{"position":5,"visible":true,"text":"2mr26jkd368pv wfxs2sxlxag3o3 o15fq 5u1n5tdvs7j0","quiz_options":{"score":0},"id":"4385138292"},{"position":6,"visible":true,"text":"ffdpeyyy nvtjrvxqnqmr f2jwutj 5uw3e0w4n2h dmah35mk v979ctn 2s683h24","quiz_options":{"score":0},"id":"4385138293"},{"position":7,"visible":true,"text":"245tmey w2ltcq50f sktfit h9ymojx j3xrggyo 51d1y","quiz_options":{"score":0},"id":"4385138294"},{"position":8,"visible":true,"text":"lv4qkg6meoylx rmlf7cdb1aht r316f1u kfcwrh5 cm1m3 s5x3eqj3t v1h721uqo3k5km7 9n1oqah9","quiz_options":{"score":0},"id":"4385138295"}]},"page_id":"168830083"},"emitted_at":1674149695662} -{"stream":"survey_questions","data":{"id":"667455314","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ux15jxtnsx3 vofmkp85 a5kvupg5km6vq0 t5g3uf3q7hn i661htxcb s7x7r26 orjn3oisiik"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455314","answers":{"choices":[{"position":1,"visible":true,"text":"9fr03iy 9xmpt6xgqwpc k6t7k9gp1ht1 6h620md9wd70hh bs3kw i9htv yil47l xn823m6ih","quiz_options":{"score":0},"id":"4385138315"},{"position":2,"visible":true,"text":"it0ef6tr6h9nt t2jwq18hq3w59p4 oyx7e1bj86bcm 02aaa4kvmf8ru77 ssmts45dadkf gl1spgihc4 acylm 0vmvvgxv6yvey0 8xi073ec2m5","quiz_options":{"score":0},"id":"4385138316"},{"position":3,"visible":true,"text":"8m4mr5q 7wkbq4t8vplwih 0cvqrnnt qt9mjry1n xyqbuaepupf3 ed52xu5ak bd1vxipoo5ad s7pxs874 a9imnp7nm","quiz_options":{"score":0},"id":"4385138317"},{"position":4,"visible":true,"text":"2clmq75t 252dtb2ce3i fld27xux vrip5ox3ds8qnb lvp972rcrcjc ruvk2sclvimuvx 1ud7hrsbm567","quiz_options":{"score":0},"id":"4385138318"},{"position":5,"visible":true,"text":"0l86fyh8uo 090ymhll1pq 9lkwl89vq f1pcky3lidacy9 3ecmm11niu fu8tfp","quiz_options":{"score":0},"id":"4385138319"},{"position":6,"visible":true,"text":"h37f95j7qxup0fe tbx1l3bgii ol4bri0itarcwk doh3p2p0pi jq9guw3h382 08fje7vyonhmfe5 s2ioi7c4v6ci","quiz_options":{"score":0},"id":"4385138320"}]},"page_id":"168830083"},"emitted_at":1674149695662} -{"stream":"survey_questions","data":{"id":"667455318","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"1gm3ed2s89 p2xfqhaj4r p7fwo6 12imab59cds2p aqmmilr2dvmwvky nljfs ts3g4cw6au9jii snbv40hbjcu3"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830083/questions/667455318","answers":{"choices":[{"position":1,"visible":true,"text":"dfyvo2qlanm8s ikbv3wa7030 rmjt80 o7cym2 1r6qae7c70v30 fko724jo7me82s1 bsuhjbov2ttwr q3w1wpn5twsv5e slrnk3sx m3u87rixhv6nmc","quiz_options":{"score":0},"id":"4385138386"},{"position":2,"visible":true,"text":"f47smf7vov8sp1 24r5d2b6q4s duka34dqpn6si 4r2wn 92ekkv2p794l8h l8n6cdc","quiz_options":{"score":0},"id":"4385138387"},{"position":3,"visible":true,"text":"okl5ki7v1r5 4oqdy4 x7ny0qmas 0ddqlr1 ja5wspe 2ieqa m3ucowjq1krai","quiz_options":{"score":0},"id":"4385138388"},{"position":4,"visible":true,"text":"gt5y6 vpjp0e5p6 vqhwb2dytiuihsv ru25v6bm mcihbuved71h2 quy2rej9e8eb97","quiz_options":{"score":0},"id":"4385138389"}]},"page_id":"168830083"},"emitted_at":1674149695662} -{"stream":"survey_questions","data":{"id":"667455323","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"wv0en qigyrej bappxu8q j3ihl9p6ki"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455323","answers":{"choices":[{"position":1,"visible":true,"text":"xv1me9s6hd aihplmw6 plhv5b k52pvh68j so1ggnlishy5m 6qe1fhw75gt k1jxdmddlhj35en","quiz_options":{"score":0},"id":"4385138412"},{"position":2,"visible":true,"text":"7ecrthfm97aysv 8u9a9sn1f9kj8 90owtm 0h8qgota7j3qpn vnfs9vleja36","quiz_options":{"score":0},"id":"4385138413"},{"position":3,"visible":true,"text":"v519ikhgw0fl s8x0shqsi ssr005 8xm1b7fal622l","quiz_options":{"score":0},"id":"4385138414"},{"position":4,"visible":true,"text":"gbpim9ar0dfgi94 nj0mq3ejst csj5e 763j6d5eo gf4fvw0 s2ea20n33yo iqd5r5l9 3t0okvw2oyh","quiz_options":{"score":0},"id":"4385138415"},{"position":5,"visible":true,"text":"jyxp0s7xfc7td5y vom52gda3dxr ko6256dtc5nv5f 7s8nej n32hyka ywsoxywn","quiz_options":{"score":0},"id":"4385138416"}]},"page_id":"168830087"},"emitted_at":1674149695662} -{"stream":"survey_questions","data":{"id":"667455325","position":2,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"jdkmr4b5t3o9qd6 0u4huyxr8whu cxqri4e2i1a 88yyx cq3xbymudltf y6hmsrn4socbj"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455325","answers":{"choices":[{"position":1,"visible":true,"text":"5aacsmqn 9p8l1tvy9i 0f92rvnxn9 to5xiaghqmiw 6xik80 jxvy64ut9 bjk3w6ywb","quiz_options":{"score":0},"id":"4385138421"},{"position":2,"visible":true,"text":"bc85jh39p 7nskr87a3x ny6d4 jlbavo9t8h6j6hu dh5ne","quiz_options":{"score":0},"id":"4385138422"},{"position":3,"visible":true,"text":"3vykabcspbl7qt ajqad2gu3v0jbg yeo3sobedrdfe0 1wcrn35 5l7jwq3 f5bc8bx","quiz_options":{"score":0},"id":"4385138423"},{"position":4,"visible":true,"text":"sm6lx7btphddbw 0882qf4o omh4u2i446c9p4q 5hqyq27jlse1e7 ns3xkqg8gcx8pc sjd0skhv 9ydxkadh8e814j7 0mk8m5tm9d38e 23cvhf22g lxb5c7c3p0oe1","quiz_options":{"score":0},"id":"4385138424"},{"position":5,"visible":true,"text":"2x7f7xmgur5a 6mn40jjs2dde 3th2mj8cn dv2pbu6s 7n8hw1f ptapt6nxgddk7a 4cj77u6m3mm idvm31 mx9ygnq3i 4lw1gmm4cwaig","quiz_options":{"score":0},"id":"4385138425"},{"position":6,"visible":true,"text":"3q51n sofjjqlu2y 6088c4c ncdkdt8exikoiir ew86v6gkob94v 7jsgkctqkhm1","quiz_options":{"score":0},"id":"4385138426"},{"position":7,"visible":true,"text":"7lffwkal h4d2j5 dqjf3y5 jwopu 0xf2vqmb6an igo5ri3px747b 0l5s9df7w7s","quiz_options":{"score":0},"id":"4385138427"}]},"page_id":"168830087"},"emitted_at":1674149695662} -{"stream":"survey_questions","data":{"id":"667455328","position":3,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ieffti0yi qcxktrskkug9ij ebvs68ni79 1vrpnp mobkmem70 7uc86c7sx"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455328","answers":{"choices":[{"position":1,"visible":true,"text":"7vvg5h tc4vy 5c5194t1n6eu wutfyil","quiz_options":{"score":0},"id":"4385138432"},{"position":2,"visible":true,"text":"17dh5pnpe3bc 6gmy6r15tq7 fuwlrcnnt61wotb mj08g80e2pri","quiz_options":{"score":0},"id":"4385138433"},{"position":3,"visible":true,"text":"414pty e5a4vs1 4p6122ihy qnjg9a xuowh226f18 hc449ct tdwm0wu29u","quiz_options":{"score":0},"id":"4385138434"},{"position":4,"visible":true,"text":"24k06ig dhy3huyx0plis 0n5vomlwuo38j 0nft5aw obgn4qcoq0l44b l5dtviydcom58cn 699vqm6 06i0mr52i0 u1mvn mm1bqovxpvtwkok","quiz_options":{"score":0},"id":"4385138435"},{"position":5,"visible":true,"text":"3fr4ttmtr3mhvr d7xcfp6tx48mne3 ttcyvypnom ik9eqkf o7q5x4veph 31y5w8u036c56rh 9yar58y9t5d","quiz_options":{"score":0},"id":"4385138436"},{"position":6,"visible":true,"text":"jrghjb5g3h6t dlpx7hve7lijy 77ergx421ad dgekp5dauuod5t 3mn6a 7m9wvlhvgeua 5orruhepinotb hd948u958 23p4f fmprms","quiz_options":{"score":0},"id":"4385138437"},{"position":7,"visible":true,"text":"dvjf23fp1f8slys 6120e2kbl8p1 f2ildddc i9ocnxo dk1c5jm5bx3 1mmcj3qmntljpbt 889694rivh72g07 25yrmna iwjlytheaogoxq fanj2","quiz_options":{"score":0},"id":"4385138438"},{"position":8,"visible":true,"text":"f2tlh hih5om u7aqefshc47ph sxxt22yg7hi","quiz_options":{"score":0},"id":"4385138439"}]},"page_id":"168830087"},"emitted_at":1674149695663} -{"stream":"survey_questions","data":{"id":"667455329","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"96mfgjfkj4g 1roun x4g5tcrq d52byhs855 cjc897qm8l udgjrsby"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455329","answers":{"choices":[{"position":1,"visible":true,"text":"2ny2fhgpm6 aturc40fuggi de2bnbhjnc a8jmpcw54h3nu pij7dkc3 sbrbi0rbi40ox2 wskx4gyt7aoc65 coi4cfe4y p44mj88ikx 93sodjvxi7ny30","quiz_options":{"score":0},"id":"4385138440"},{"position":2,"visible":true,"text":"t1f53t6jo 5s7kaoe ma28g93gfjjsm nwfjlfr2 h4k3jav 4xvidc3fv 8enehee7txlhvp ca6vkoxrb465pi xmi35um8q54r6 4rvruqr46m","quiz_options":{"score":0},"id":"4385138441"},{"position":3,"visible":true,"text":"i7iw5 ap5gjyafbm2l adp3lcc1 52hc8 j6ldt5","quiz_options":{"score":0},"id":"4385138442"},{"position":4,"visible":true,"text":"468nunilylfthe gwy8lhtgga9re4f 5xacgti673jfgs 3eei9s6qwg4avy ksqn6cwpvl585 wtyw59jhy7kck m462cr65qglmq ev7c0b5","quiz_options":{"score":0},"id":"4385138443"},{"position":5,"visible":true,"text":"yxrha98q xgjtmqc6x6tbsq l5co87ln2j3044 49lpv4 l8rfhvt2 rp4v9ofww2bekc7 ops08osul9","quiz_options":{"score":0},"id":"4385138444"},{"position":6,"visible":true,"text":"dbah1h80x 07f9vbs n89jmtwm0t2 47sd0ilc umky7iesp5j1ye f825fm7sn5fteb","quiz_options":{"score":0},"id":"4385138445"},{"position":7,"visible":true,"text":"9ybbt8x1xk o12xtb esgrab5p169kpou jyx54 456l76rs9f 3pcmlfoju rfyofv71 lb7gr6gi2ab0 gmexy","quiz_options":{"score":0},"id":"4385138446"},{"position":8,"visible":true,"text":"k3s5bwg5b2q 0ikv741vhxu3x4w efpp0p21i1s 44ca0fl4bklmn","quiz_options":{"score":0},"id":"4385138447"}]},"page_id":"168830087"},"emitted_at":1674149695663} -{"stream":"survey_questions","data":{"id":"667455332","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"v80ku mx1co9qwm34tat e9i3mdjcvixs grggkttt lhn014lmqj86 achjddrt9o5fx 9v07aged4niq0g ye1woaclolxuaq w5e7jclooee"}],"href":"https://api.surveymonkey.com/v3/surveys/307784856/pages/168830087/questions/667455332","answers":{"choices":[{"position":1,"visible":true,"text":"u1e4v4rx27412v v9mjw7oaf 0t873cte 57y89l xdqtturimm5 b4stpodx65s8u 2mk2es7jwrpn 7enb3sp29","quiz_options":{"score":0},"id":"4385138468"},{"position":2,"visible":true,"text":"w0mid5oagg qx0e1bvil6w5v6 0cadbm51x7hbg rpxgn9yni","quiz_options":{"score":0},"id":"4385138498"},{"position":3,"visible":true,"text":"7rex013 m75tu1nu9orrc2 k4du9rcsy2n5l0 cfbiki1u6cp2f qe81rjnguphrum j4019 tbl21q37","quiz_options":{"score":0},"id":"4385138499"},{"position":4,"visible":true,"text":"sat97b1hk6dx9k uq924nht7pr8cb 7nr3h2hclmiqg txrkxr29wtrc217 bxmed4ll1b23561 vvsu7 x293il lrl3e","quiz_options":{"score":0},"id":"4385138500"},{"position":5,"visible":true,"text":"h3yh19ckoclpq 0hb213i nj1mfmvbj9 p4ibgetarc6h6u 8kahs","quiz_options":{"score":0},"id":"4385138501"},{"position":6,"visible":true,"text":"xyvg34bae2 7u2n4l87h aec3h1sy5aw62r 60yajbqvxifw65 c7q9ty4pdby2d vyjp2 n7tavs0550g46 07p64c9pp8oo","quiz_options":{"score":0},"id":"4385138502"},{"position":7,"visible":true,"text":"kn8f11mlx sucpq9a 79n0u6vi1tgt b0dom486a929h ocblyvsm6 ti4tnjv533","quiz_options":{"score":0},"id":"4385138503"}]},"page_id":"168830087"},"emitted_at":1674149695663} -{"stream":"survey_questions","data":{"id":"667461429","position":1,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vq7ho4rb qt0fpw3 fvn1b2y21n fpkcw9v73dqvfq0"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461429","answers":{"choices":[{"position":1,"visible":true,"text":"p7fu5tjqvd qgxdych hv1xcryuq7jlia 1qjuijk40 2v6u2e kwqcepbnx 2hjta3wqc je2wbyr6337d5 o0fq46u48nsa as5jyqahxo","quiz_options":{"score":0},"id":"4385172931"},{"position":2,"visible":true,"text":"tubba qc6l28jc4m1xi 5lf38au3ry9d o9eh4us9ki ul832 jippjf4 bcsoug6 9w7mnssanfsknl f422osb58c","quiz_options":{"score":0},"id":"4385172932"},{"position":3,"visible":true,"text":"juswx5 78b5n495 duu5kiikm feknbdqqtxg memsnyrcmao1lh","quiz_options":{"score":0},"id":"4385172933"},{"position":4,"visible":true,"text":"xpu48c 3cdbygss10 ujxmb ore7vx7o0x 9qmlb9fig6p3u w6c8oqr5dhp1l g3ihi9p1x kf2lvtbxo 4guw65","quiz_options":{"score":0},"id":"4385172934"},{"position":5,"visible":true,"text":"i9508b8n ld7powh72 nfvmljfhgn3n 643ydxghpbak7v ehe18sjo56yx m1bpaoj4epr 5sv5aw6 7m0bt","quiz_options":{"score":0},"id":"4385172935"},{"position":6,"visible":true,"text":"vwtr9m5fu3ot ltiqhsx3fi uuylf62qec mmn5fxqj","quiz_options":{"score":0},"id":"4385172936"},{"position":7,"visible":true,"text":"ww4qdi5pqg 3qvmj8g0yvx rrr5fx06d7 t6giac3k8 t3d6exqx175ft 10k251y 47v1vnu938 kkqqcsl50d7i","quiz_options":{"score":0},"id":"4385172937"}]},"page_id":"168831336"},"emitted_at":1674149695747} -{"stream":"survey_questions","data":{"id":"667461433","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"3bdmcu bth1w4ifhtcbm slcl3p6ynqhjtyp n5nea bdq5iuvq77m0t"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461433","answers":{"choices":[{"position":1,"visible":true,"text":"26j27d 5s3pbf08free77b ttq5exq7 n2gm99qkada 3rosqp hn74g juh2ww6","quiz_options":{"score":0},"id":"4385172948"},{"position":2,"visible":true,"text":"phmu9ghnjc vkh0ury2y6et tqj1lwjv 37m5bx1itv a5x88phj3g5 832qd1l rpbrunm7v49 kpmqo a8eqht49077t","quiz_options":{"score":0},"id":"4385172949"},{"position":3,"visible":true,"text":"jbsjkqa2f2xb ks824l5wnkd b0p0elet784r6rw ay5sqsv2vdron lr8mx7r6pc im42wmwt ltyhtay7p8u","quiz_options":{"score":0},"id":"4385172950"},{"position":4,"visible":true,"text":"ikkvwye7tah 7shp28 p7k7je0as5u sdc3f1rvin staohk6a44k nqjsxn3 97bog4jfxn qvorj30xpuh1gip","quiz_options":{"score":0},"id":"4385172951"},{"position":5,"visible":true,"text":"p83chdtsw6s88u 0hqgb8h871mw p0ej9 djy316qsa7iv6pe","quiz_options":{"score":0},"id":"4385172952"},{"position":6,"visible":true,"text":"kgfa1 tyxdxoe3gc xclen0vw9oa2 6bu9o8b6awx hcg9pgsi1av9v 7dicbt6wsee70f glwmxxtcdr 4qi7m9p8tfkxur3 ir5jr31364","quiz_options":{"score":0},"id":"4385172953"},{"position":7,"visible":true,"text":"u6xyfnav qw5qefsi 6ttsauh 3jwvmju8sdjk87 bql4ra2ww 8nxxobw4o58 omi87y6ur8l1f2g 62gaxuq","quiz_options":{"score":0},"id":"4385172954"},{"position":8,"visible":true,"text":"xdjcisgicnaix2 fj8bpbqaqgntr fb2n0o73 mgsb8xg5x3nfg9 9t18omvng4p6e 06wepywm4wku 82pemp2 l3nsu2ib2erbva tj475a","quiz_options":{"score":0},"id":"4385172955"}]},"page_id":"168831336"},"emitted_at":1674149695747} -{"stream":"survey_questions","data":{"id":"667461439","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"vrhftdyyte qir7jajr68td obg64x tu1rcy2h pqnxtdrwxk00a c173brbv 6qfxck7huyx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461439","answers":{"choices":[{"position":1,"visible":true,"text":"q6705n7lidhb 0gsqn8 ocnyaudmo ulp9r0rsoheh4f cjwcqblbh o28tilm13w384","quiz_options":{"score":0},"id":"4385172990"},{"position":2,"visible":true,"text":"vyrx1jpjcl07 etxm7itb161 51rf5csw6e5tsh 3ux7rxq2 vct6fys4r7","quiz_options":{"score":0},"id":"4385172991"},{"position":3,"visible":true,"text":"pp1lc12uhii 2qmm3xnfsp i9912c8ac5k74i hxew5625hxtm3 4o658 jenbgi9o89 t4ppqc2qvhlui iu92ym nqjkka1i7","quiz_options":{"score":0},"id":"4385172992"},{"position":4,"visible":true,"text":"6mx9j44l0oa yv2wb1letc4p d3u87l59 vp7a65ykcfjyt tpe3k92l y2flusuc3tc t220oi sekyd","quiz_options":{"score":0},"id":"4385172993"},{"position":5,"visible":true,"text":"4fbdo eli65un2emx e6oyl3a41ugoxb2 saxpsxn6fv l8s2mk0e57d60","quiz_options":{"score":0},"id":"4385172994"},{"position":6,"visible":true,"text":"r5ly6h 7eu4pmnx7tv04g rdtjvlsup3gdn h4qoreg 3ct3fudxbuuw2 b975v8 ilsfmaa22","quiz_options":{"score":0},"id":"4385172995"}]},"page_id":"168831336"},"emitted_at":1674149695747} -{"stream":"survey_questions","data":{"id":"667461441","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"5p8n613hcmidh w4gyoyf stviro3om xey832w t9w23kj3j4rp5j h2w1swynkq1n9e vqrk2q5eb76p9yn pbj778q19u1hqy r9uvl76qqhfg"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461441","answers":{"choices":[{"position":1,"visible":true,"text":"tmdpekdn75l9b 3nuus dqbdxtj od2mltbx tmw5kcvpiw 6n41343132bdmc kb3i9er4qg","quiz_options":{"score":0},"id":"4385173020"},{"position":2,"visible":true,"text":"6a8efsocxmc eeohurhhyduo31w c8na1dub0ycx q8314ir twgrq 2ukcrksjt30s","quiz_options":{"score":0},"id":"4385173021"},{"position":3,"visible":true,"text":"ou9m3xl2n9wvn m0wews5 il8o8so pygnm380cd66 7nhjkpk9lu65n 4e3ifrwcb8wr 46bo0ani86m du57mphcnvf1in 6gf58fwm2c50tp","quiz_options":{"score":0},"id":"4385173022"},{"position":4,"visible":true,"text":"nahvw2sd3 hn5trfgqbuso cq82jp7 k8ev0 s8a0a23m0p g2jewuy0wdadgsa hca0mm5q 8agnm fxnf8vrgdybkg04","quiz_options":{"score":0},"id":"4385173023"}]},"page_id":"168831336"},"emitted_at":1674149695748} -{"stream":"survey_questions","data":{"id":"667461444","position":5,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"khys853d7u tgmr69e7mu7 0h6sn7 cja88 nj2os3 wnand9tdmohxca rowe88asxsja1dn rvs12kt5m0wqd2 41pwt1vnhst008o"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831336/questions/667461444","answers":{"choices":[{"position":1,"visible":true,"text":"3k875epq o7u7f5 kihtesav7 qwxlu7j5dj59i 8ghsap3n","quiz_options":{"score":0},"id":"4385173040"},{"position":2,"visible":true,"text":"cemvy 140ek4 d1fumxxlqqnyt 99pb868kg2b hqur2f44k uwr5dyif8g6","quiz_options":{"score":0},"id":"4385173041"},{"position":3,"visible":true,"text":"iykvyahtufq91l bvs9t6mmoyw ld2hep1somcl7 2rs7wk3q 5ncge9mfj1ac r30355p0g4wt c1bx5 n5wgmcud6h","quiz_options":{"score":0},"id":"4385173042"},{"position":4,"visible":true,"text":"a4xcahm1euf l3h1jx abcc0r500rlqyhv 9w01sfmci3j ur9vp3sxfioh r0dksavcmduhk2 kw7nbl 5hys2r8vebx4e 94rajuno uwm3ajywh3vlqbf","quiz_options":{"score":0},"id":"4385173043"},{"position":5,"visible":true,"text":"0whm5dw5kuk44 efrr6i iu9592 a968tp4ff0uaf q0f6mtpcp2x82ae 97m1gvsnfthibp agt6dm3 ip1e3y9","quiz_options":{"score":0},"id":"4385173044"},{"position":6,"visible":true,"text":"hwbtcf7 60gji9qoeovhlf 9b3ijm8 xiw5w09k 460v2o27hacdts 30eq74bg9m qpfv2jd9f3ur rlacn2rf273ck7 2welkh7188h","quiz_options":{"score":0},"id":"4385173045"},{"position":7,"visible":true,"text":"cbqj2nlsnyy tmjyvoija wirje1e bxsqxqe g12fuxmtgfq 7fc74o yrdsjey 1xmv6u077j7l 8hkf7","quiz_options":{"score":0},"id":"4385173046"},{"position":8,"visible":true,"text":"uqlus j3nd9u8g faqa6ghioy60h9 a95spokyj58 ndgctguy75jr ei16b1p7jabc2 hfkfpruds","quiz_options":{"score":0},"id":"4385173047"}]},"page_id":"168831336"},"emitted_at":1674149695748} -{"stream":"survey_questions","data":{"id":"667461449","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"cwipmppqgy2vy 3mn05n4 gedv0twor lyawl9v528587w wfmifrr9hk pykvx52nu1ds5p hh28niyix"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461449","answers":{"choices":[{"position":1,"visible":true,"text":"3gjvb8l80nc8j 90ebdlnd3hg spis2ipbim9l fjt4xli4a 0bqmti2wp0juwcl cdwl64u2yfghjdf 190uek5lw","quiz_options":{"score":0},"id":"4385173076"},{"position":2,"visible":true,"text":"xjdswmkwm24 rrmxo7otpear8 ogepd7v8br0y5m vi2vyip49ux uvtnq2sem 5gwcppvvvc bw56us8o 1huql09g6pa1ylb","quiz_options":{"score":0},"id":"4385173077"},{"position":3,"visible":true,"text":"gyq16tgah s669s7 j5isg9 l63lf q61pyy oywk58jfmnvy 6lhnukrce1px bwt066s8k5248o sylwt em0x3","quiz_options":{"score":0},"id":"4385173078"}]},"page_id":"168831340"},"emitted_at":1674149695748} -{"stream":"survey_questions","data":{"id":"667461452","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"aku2037bp sxtf1busk2uj 7scwpc00giyau urduwqdsyr3 xk3y39yjywei1y uw7y5s3ky fetwdqr6n qp00a8rofpy 4rwl41yk78jd"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461452","answers":{"choices":[{"position":1,"visible":true,"text":"86hx9ifui6c4o 02xkn5ghldfw35k faipmm g7634wrx1jfvy jewjfdq kkyyxx00brj wtu0xrhfhvu5lv8 573fv0vtm0n","quiz_options":{"score":0},"id":"4385173088"},{"position":2,"visible":true,"text":"nfeyvnb0kat1yb0 3lk3rse66t8jp w90vixgpu32cyir q03jxd8vch9xn7t 3a0swht9ykadh8 fvoa9t8dri fm0twcdhxm 4ot9xpxe9s08b ho7d1","quiz_options":{"score":0},"id":"4385173089"},{"position":3,"visible":true,"text":"k5pyc4y 9uul1o 7lvigo 7xsdo49jx2yc d07jr1w4","quiz_options":{"score":0},"id":"4385173090"},{"position":4,"visible":true,"text":"3e49s5 gxqwbv w6c2e tjaf7rpgtksjpe ivr0he574 ft8qso pqq4l5hbchy 6mhw0ksgrh xbj4l7a2g","quiz_options":{"score":0},"id":"4385173091"}]},"page_id":"168831340"},"emitted_at":1674149695748} -{"stream":"survey_questions","data":{"id":"667461454","position":3,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"4i1qajt 6sy8nl3hjougqbu okftpe4sw0bpr6 gv1pbyk8km 9ihd103uu2n3lx c6spvurq080unx uswpaexarx"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461454","answers":{"choices":[{"position":1,"visible":true,"text":"vg0vpi1 100y8 03nnh0 947rcth5ofg x4xfc38q81i6e7 bgf20442q i2jh2 lolt43y3ri2 vl0t6 2rkshapnf","quiz_options":{"score":0},"id":"4385173102"},{"position":2,"visible":true,"text":"o7qfwgqys eyumel 7spc5 42qxefd","quiz_options":{"score":0},"id":"4385173103"},{"position":3,"visible":true,"text":"usb43wqju6w csearclgaedae 2oo3m2a79oo5si granw6hf 9g9ms37peouy gqaxry74 x45yqa0xtkcu g0in9 lyp7xe6 few8vkh7yf","quiz_options":{"score":0},"id":"4385173104"},{"position":4,"visible":true,"text":"spfrf5sfebrhj8 gwodgm3o1 m5cydcxbtk2 pgrj6h mome4a us97ellx2peg3s ilidjy8juu0","quiz_options":{"score":0},"id":"4385173105"},{"position":5,"visible":true,"text":"w579aalpr5gaj1 1h2ud 53d80pebt4ep 0l2gw8fk7fa","quiz_options":{"score":0},"id":"4385173106"}]},"page_id":"168831340"},"emitted_at":1674149695748} -{"stream":"survey_questions","data":{"id":"667461456","position":4,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"is9uyf6ka iel16fh 1f0xg 7mx16glcei fygrn g7l10em5fbeybb 1vgps0k3 q4fls qryybq07jy f3luik1nx09b7so"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461456","answers":{"choices":[{"position":1,"visible":true,"text":"0fg2b3v4 jmdxep5of1y q2ag243 cfytydw942toqsl 2ixd8iynbs eg22ia","quiz_options":{"score":0},"id":"4385173113"},{"position":2,"visible":true,"text":"0am3pervuj biqdk6g5yw i5xvk20h1n6jv l0v9m5cxn 2vyqvcp8rlthxfv k0v6o48p3v8 551pws1020t98f","quiz_options":{"score":0},"id":"4385173114"},{"position":3,"visible":true,"text":"cjtogxiie0arj ltkgiwrpoa4x1v foqfr1gk406a rale1dx 2p4gjy2g","quiz_options":{"score":0},"id":"4385173115"},{"position":4,"visible":true,"text":"h31va0f12qx3vg 51m9g4bo9 r4ofiqr j1yym2ma8m13kch 1e4jxhdyol2ny nf3fh0h2e qw8euwei1lyhemq 3j4un2sjdoj7 q92100o573tbom 5phvt3n22","quiz_options":{"score":0},"id":"4385173116"},{"position":5,"visible":true,"text":"h8hktik e03ehrspn7 rnjqq4431q2 3bs1nevr8j0 audw020tl kalstea2","quiz_options":{"score":0},"id":"4385173117"},{"position":6,"visible":true,"text":"9w090 e6835j0fvfg89 kk6swkb5g oo0f4ho99x437 qf7b8y1aaa4 i0r03","quiz_options":{"score":0},"id":"4385173118"}]},"page_id":"168831340"},"emitted_at":1674149695749} -{"stream":"survey_questions","data":{"id":"667461462","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"r1kf2oglhju ou2ldq63qu2g1 ufs1pkw pgr9p75sh9k7 vnu477cixnk1kx nxnfn0ti3x3u9wh ew6ho e18cmaeovxhx h1fac8j8 8ni6ay"}],"href":"https://api.surveymonkey.com/v3/surveys/307785388/pages/168831340/questions/667461462","answers":{"choices":[{"position":1,"visible":true,"text":"xytk5fnqv6odvms sw94ohnav5npnm lg9uf4 iwuqgfd0 5ee0wd5 lfha63ve x55yip4 gxgdkff3sckdn mq1khupscjgqj 8mtp54i5c3rjonv","quiz_options":{"score":0},"id":"4385173167"},{"position":2,"visible":true,"text":"2ammt8omj0l c0i1q 1uvaf203rh1 8wj2w7pp qlqayl9e8ldc y7iivv cv189py2di xuihxup7b2 rh8owrr595st","quiz_options":{"score":0},"id":"4385173168"},{"position":3,"visible":true,"text":"o9cvkj8x k0txiswxc5 ogf66jujgcrwdb l7n0c0rodcx 2gduko0wwimb21 afw8mi","quiz_options":{"score":0},"id":"4385173169"},{"position":4,"visible":true,"text":"mcb62sefmuo plnbygilddeqg u64kkkjvoms4b5q jw4tashu6c7ve12 8di4g100598 ad1bet nnqd7jmg","quiz_options":{"score":0},"id":"4385173170"}]},"page_id":"168831340"},"emitted_at":1674149695749} -{"stream":"survey_questions","data":{"id":"667461690","position":1,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"53o3ibly at73qjs4e4 y9dug7jxfmpmr 8esacb5"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461690","answers":{"choices":[{"position":1,"visible":true,"text":"lg2mcft4e64 ywiatkmeo ci3rr4l2v0 ot6un49a 4b28sq4g8qv7tj 4ihpko73bp0k6lf swaeo3o4mg2jf5g rnh225wj520w1ps p9emk1wg64vwl","quiz_options":{"score":0},"id":"4385174700"},{"position":2,"visible":true,"text":"ywg8bovna adsahna5kd1jg vdism1 w045ovutkx9 oubne2u vd0x7lh3 y3npa4kfb5","quiz_options":{"score":0},"id":"4385174701"},{"position":3,"visible":true,"text":"xsy4kv tqp8vty29815 de8nt5ab2fyr m6jilru2ek l7fktx3j5mbj l33ip83t4p29 exfygne a1btj95m1r","quiz_options":{"score":0},"id":"4385174702"}]},"page_id":"168831393"},"emitted_at":1674149695838} -{"stream":"survey_questions","data":{"id":"667461777","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"kjqdk eo7hfnu or7bmd1iwqxxp sguqta4f8141iy"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461777","answers":{"choices":[{"position":1,"visible":true,"text":"11bp1ll11nu0 ool67 tkbke01j3mtq 22f4r54u073p h6kt4puolum4","quiz_options":{"score":0},"id":"4385174970"},{"position":2,"visible":true,"text":"8q53omsxw8 08yyjvj3ns9j yu7yap87 d2tgjv55j5d5o3y dbd69m94qav1wma 8upqf7cliu hb26pytfkwyt rfo2ac4","quiz_options":{"score":0},"id":"4385174971"},{"position":3,"visible":true,"text":"6d7qmnw obxwg4elaab6 2sby04sor66 1wuoh26aftxu7","quiz_options":{"score":0},"id":"4385174972"},{"position":4,"visible":true,"text":"n0xwexbwtviyj1a midgl2jpfdy a72ut27ta 8i9fmkwg0q mbtxhkn b2ut8mtsslkt609 tgmnd7ovnqlbr","quiz_options":{"score":0},"id":"4385174973"},{"position":5,"visible":true,"text":"qjfs0pmb iecatmqyxtk w1s0fs9vcbayf5 rwsneyp0wx6lsyq pq99n hrx1mk4saug gv06qshlabe 0s2t4 h11ee2xna0m8r","quiz_options":{"score":0},"id":"4385174974"},{"position":6,"visible":true,"text":"11uf3he wbstw etbysmu4 c84vqddvx","quiz_options":{"score":0},"id":"4385174975"},{"position":7,"visible":true,"text":"rnfx7m ndifoe7ihy q98pov78016t 8smlnm lb3xicjp9 0r30sie97y12ve7","quiz_options":{"score":0},"id":"4385174976"},{"position":8,"visible":true,"text":"jc8s2ra5qxytxbu u6tj7jgep95 vbva1b4uslioa omku9","quiz_options":{"score":0},"id":"4385174977"}]},"page_id":"168831393"},"emitted_at":1674149695839} -{"stream":"survey_questions","data":{"id":"667461791","position":3,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"0qw6a5lnf426 2sh3g9f8wu xmgflj 41pjy"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461791","answers":{"choices":[{"position":1,"visible":true,"text":"7kxk7bkhfdx86sh 3rnrsj70ud 048jbf4qx4 p96o8 sn7xi oh02tfput4 6js84u99m5t","quiz_options":{"score":0},"id":"4385175055"},{"position":2,"visible":true,"text":"x259osu33y 8qadkcxpsnk4o20 m4wo3183nwxhgye q4mpg srpfibk96sf t3h2cx58eji x7l0sdipnjece8 7tgwfdfmh9hgdwi w99mkib2","quiz_options":{"score":0},"id":"4385175056"},{"position":3,"visible":true,"text":"lil1tboe p80wa8yed7w8 cll24c2lls6cc0 gpbv7rnap psk1et","quiz_options":{"score":0},"id":"4385175057"},{"position":4,"visible":true,"text":"wodtghhkt 2ae1c8q5s1ha 8lppd7ko84al j95eq1imtu7 6x8qknrhn0 l7h53","quiz_options":{"score":0},"id":"4385175058"}]},"page_id":"168831393"},"emitted_at":1674149695840} -{"stream":"survey_questions","data":{"id":"667461794","position":4,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"q3ay58w3 2rfjgu4 0cf9uh1 pu4fo16w 6c2wkn 1oo7d8"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461794","answers":{"choices":[{"position":1,"visible":true,"text":"1orbs vtqu62x9bp t75k10e89krhn bdnsfy6ng34g 8yv9p1c92jlbt0s","quiz_options":{"score":0},"id":"4385175070"},{"position":2,"visible":true,"text":"5j8dww2lxevx4a wv3ppbb vnccslwrjjdc n5pjsmw m7b4533y8tcbbus","quiz_options":{"score":0},"id":"4385175071"},{"position":3,"visible":true,"text":"fnjqkqy2 44brrpru jllsj9cdggwt4 behkog76y5ua 7ftpd8c8qhblii","quiz_options":{"score":0},"id":"4385175072"},{"position":4,"visible":true,"text":"srjre1h3w9 qojsh5w2 sq7wva6tkl9 raxp5mldrp","quiz_options":{"score":0},"id":"4385175073"}]},"page_id":"168831393"},"emitted_at":1674149695841} -{"stream":"survey_questions","data":{"id":"667461797","position":5,"visible":true,"family":"single_choice","subtype":"horiz","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"bvrdxa swsrjt sjox8u6767lv5 wgcomvtnoi0yg namiomuh6cou61u nl2v5bfu15i7 sqpu07jp489uc"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831393/questions/667461797","answers":{"choices":[{"position":1,"visible":true,"text":"y97bshsv ite5mgk76p89o yrtt28bmm4jo9 ftc2tnjg","quiz_options":{"score":0},"id":"4385175096"},{"position":2,"visible":true,"text":"r970efm0 5p96h9iy1 o7ft83xrqgsrh8 owk30 buqg6ksd297lw9 lh6ygen9s2rac2b k5d3lbr7m37p","quiz_options":{"score":0},"id":"4385175097"},{"position":3,"visible":true,"text":"ktg10 vp7khp0ucx vuo5qrcor po9nbn6cdpdu56a rt8eiu0umg0dkx j2k8vgtr6","quiz_options":{"score":0},"id":"4385175098"},{"position":4,"visible":true,"text":"iubh35s1gvpm4gj svwbyf7npunm3 0thmsjmt2qb5im0 undxh7b frxykv55emi padtjsk69 qa0jrnwrfoj qqjg6ifvlx0abdb","quiz_options":{"score":0},"id":"4385175099"},{"position":5,"visible":true,"text":"w64hwv9edeaf55 l0gkthucpqj 80wgqsffl 0m45xm56a25psm 8opb8b0gw2w6 n8xex","quiz_options":{"score":0},"id":"4385175100"},{"position":6,"visible":true,"text":"ju3rt297a t028c0b35635 l0kj9vj seuar76 89587qhw46295","quiz_options":{"score":0},"id":"4385175101"},{"position":7,"visible":true,"text":"c4de01u4eil p1p2vy 0gqjglc mc2r97p07 d8d90 j15xktb2idx91 tecpeak3 4anh9o5w7h0runq yr0nd0q9392229","quiz_options":{"score":0},"id":"4385175102"},{"position":8,"visible":true,"text":"yc5erasa3ovk4d ed9adudq8e1s 7wrf8k w9ohrhltg3kv1 wgrnemp 7dqxmy5e bxnsro2sl","quiz_options":{"score":0},"id":"4385175103"}]},"page_id":"168831393"},"emitted_at":1674149695841} -{"stream":"survey_questions","data":{"id":"667461801","position":1,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"iu425c2v4yqs04 43g37 wg8awi s2pjwsm vjhybbs wry73cuukw85l2"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461801","answers":{"choices":[{"position":1,"visible":true,"text":"xjortc6k0sxjydf rdusho82tsr3l 3b3gch ogabx6895eb3 e7bj5pq poft6c4g1","quiz_options":{"score":0},"id":"4385175115"},{"position":2,"visible":true,"text":"bscm9v7d9nv0 e5x94dt0402ge i7mwtey74y4 er7bwam13 6xcjpw pre922tv ihmvbih 9piadim1lterm","quiz_options":{"score":0},"id":"4385175116"},{"position":3,"visible":true,"text":"ywtecquds5ctgu sjcgsa3hm d087wy 6yjqp0jgm 1ywj8v3wuuq wmlmq essefj rbgrjtv6smxcmag","quiz_options":{"score":0},"id":"4385175117"},{"position":4,"visible":true,"text":"gc8d58x66m ftpowgvwodht9h fj47r927vh826 qrkgkb bcvxni fo6g9wdlxgvnq","quiz_options":{"score":0},"id":"4385175118"},{"position":5,"visible":true,"text":"fxsrgxts qih9ukhxafmmiv4 h2ujh1va9jf b6ho30","quiz_options":{"score":0},"id":"4385175119"},{"position":6,"visible":true,"text":"81binesi6f 7urb7 ylotwabgvbt 03ke1u5h 3ehye3g olw0f83a1h667t 71ujnoyf p49ce","quiz_options":{"score":0},"id":"4385175120"},{"position":7,"visible":true,"text":"ht1rd9ymh 2tftisj80s74mop b1eavw d6vgqwrj","quiz_options":{"score":0},"id":"4385175121"}]},"page_id":"168831402"},"emitted_at":1674149695842} -{"stream":"survey_questions","data":{"id":"667461805","position":2,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"mxlksuvmoras9o 94fj2 dieyg92v384lfv8 f9rwin4 cdmg95wcnt2xa ybcmni7yd1x 1yl4j4q7j"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461805","answers":{"choices":[{"position":1,"visible":true,"text":"mevjvlslfppe ex251ss vkrs7swus g72vplm9svejkdw 7onhrwlh bouaam3k7cnn yadoqmnhn4swehq u8lhv3fdh58o3 wrbc0y197d","quiz_options":{"score":0},"id":"4385175146"},{"position":2,"visible":true,"text":"uglir jabjq1 poswkedidqpmj ta5ma4ep9xxr ghu3n2a4u7f3orh 9oud3lwe0f vwip14snnv6gtb 5fw29neis71ogsm dpq7m 9an28j1styhc","quiz_options":{"score":0},"id":"4385175147"},{"position":3,"visible":true,"text":"p3ndk7 nxpv9grg77 ek2kndt51g 2v10497 bdr0a3466ao","quiz_options":{"score":0},"id":"4385175148"},{"position":4,"visible":true,"text":"7w76l 9k393odbjg7cht7 mio9w4tcv 6wvef4vm orgg1n 20d8lh8x9osqcv dv50mjj w3g96tt0m3rf9 24uun3grfy2u 4vns2lt","quiz_options":{"score":0},"id":"4385175149"},{"position":5,"visible":true,"text":"4tcuvnn1wxy cqpr795s sfyecjwup fn76iwks5hko rk6wvgyblb3gqe4 rl5ulee1w rq66d","quiz_options":{"score":0},"id":"4385175150"},{"position":6,"visible":true,"text":"cmrjgc4 dwotyvr4o n9jid3i79xoql klkrt23lklso4p hh6d57t5 9xk3o9me 8bkpgry1yu009y","quiz_options":{"score":0},"id":"4385175151"},{"position":7,"visible":true,"text":"43ghcfhsl 74xoo rn7rmgjhd3cq u2x2ir6n449kqxp 8isq7wb tccg39oy1b 9mw0eu1ho0 a4x77foba5y ywgyosh9ue ynh9u8odsos5q2","quiz_options":{"score":0},"id":"4385175152"}]},"page_id":"168831402"},"emitted_at":1674149695842} -{"stream":"survey_questions","data":{"id":"667461811","position":3,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"i4mol250lne3 bhrh2dvt9b qss461 lkb1u chpwmgcnuoeec un2l5"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461811","answers":{"choices":[{"position":1,"visible":true,"text":"s4xpl2l93 57k4asd04 gyddhg dn53f8bd 8wtgobxts3 ms7nan ns5wv6q2vy6 nnaudmbyu80llen 8be4urorunk","quiz_options":{"score":0},"id":"4385175197"},{"position":2,"visible":true,"text":"ec2l4hr 5lwp46ij8 3tqigw elleyat98j2jjd if8hiia3 vad578","quiz_options":{"score":0},"id":"4385175198"},{"position":3,"visible":true,"text":"9t9nsl0tjlcjxr k1chdb iislvtl gpcnyi82o5ebu 46ayfj 5r3b3w92l6 vaqskragdor","quiz_options":{"score":0},"id":"4385175199"},{"position":4,"visible":true,"text":"8ad404t4 86hyrfxr7 xef8em2 g7u8fc2 rsslpdptcgrsh9n n5pb1u9b","quiz_options":{"score":0},"id":"4385175200"},{"position":5,"visible":true,"text":"bijp3kiqfs quasi89mov1y hj9ku 9w6iuh 81sng4yu32tyh d4q9kbxuoqd2xaq","quiz_options":{"score":0},"id":"4385175201"},{"position":6,"visible":true,"text":"73xiyg2gc q1l6a28s 991jaxujf56sqi rhxrnjum ges25br tb2x1wamrh3jac1 t3s8ocme8q9d8 c505btw99r hwljwx","quiz_options":{"score":0},"id":"4385175202"},{"position":7,"visible":true,"text":"ri75nf 5yy3nq 8m5e68j4mh8m sf1v3 60nijf1oeq9 bwp7bfx9u11a474 w66gfkiayng55q 6h0gp80h","quiz_options":{"score":0},"id":"4385175203"},{"position":8,"visible":true,"text":"nbqtnbbuiue5fr a9s8yrpjm7x0p qid4y913k 8ueagmuy2 5kvul122lseh5h5","quiz_options":{"score":0},"id":"4385175204"}]},"page_id":"168831402"},"emitted_at":1674149695843} -{"stream":"survey_questions","data":{"id":"667461833","position":4,"visible":true,"family":"single_choice","subtype":"menu","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"s3pjyrvc bj99egp0o99 f4ddk7sed bdc2yh24yf tyii1jye nmvwhj18oqxna6b lku2vt8hrnx4 j327a"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461833","answers":{"choices":[{"position":1,"visible":true,"text":"hqpox2w6wuwyd2 fm6kvjiq6ns5k jv1eutgqn1jj if8dj81e 57l25ev1tal9j","quiz_options":{"score":0},"id":"4385175279"},{"position":2,"visible":true,"text":"vr2p6mvbedcpkak 2c91smhshw9ee mwdy43um3334e7i u4o5frorc3py srt09vtrol825 i9s8n2koaoc6fu","quiz_options":{"score":0},"id":"4385175280"},{"position":3,"visible":true,"text":"1icyn0f tifktyc2uwd k8ehexjojth9a2f 0n7sh5p4i6kswe","quiz_options":{"score":0},"id":"4385175281"},{"position":4,"visible":true,"text":"6ju066 4chnhs0be43dy2 xdkxk37j1i0qy1 43b22jang8 na1yapnjj 7tvgbeu v1dw7as","quiz_options":{"score":0},"id":"4385175282"},{"position":5,"visible":true,"text":"vnslaachd7t07f0 db6whw u6ahc71ajst 2cn114ialhcvex kpwm1qo1y g82xup","quiz_options":{"score":0},"id":"4385175283"},{"position":6,"visible":true,"text":"aniu1f d47vbpsl mm26jpf7 g2io86ycj6yk","quiz_options":{"score":0},"id":"4385175284"}]},"page_id":"168831402"},"emitted_at":1674149695844} -{"stream":"survey_questions","data":{"id":"667461834","position":5,"visible":true,"family":"single_choice","subtype":"vertical","layout":null,"sorting":null,"required":null,"validation":null,"forced_ranking":false,"headings":[{"heading":"ce8esfvsy7xcwqu gemf05b3s5ap5 76oc1 srngx7qca"}],"href":"https://api.surveymonkey.com/v3/surveys/307785415/pages/168831402/questions/667461834","answers":{"choices":[{"position":1,"visible":true,"text":"jggh1bnginkodsv 4jhtwffnlgybux 1na25qx xr5jtwfp vvip26cqr st09ps653caiyj 1icxwhc1hut6","quiz_options":{"score":0},"id":"4385175294"},{"position":2,"visible":true,"text":"gdxye rstmylwe4l w2lkwbdf87e735u rdxn1vxbg3aw kwkn1gfsu s3oa2wx7 6vegglr1ihckyxa","quiz_options":{"score":0},"id":"4385175295"},{"position":3,"visible":true,"text":"qsghk1r8e3p ciuick1mgdwbyc k8wbxctpmtu2v xau05rusflq k3as06r35dl9 38xpts","quiz_options":{"score":0},"id":"4385175296"}]},"page_id":"168831402"},"emitted_at":1674149695844} -{"stream": "collectors", "data": {"status": "open", "id": "405437100", "survey_id": "306079584", "type": "weblink", "name": "Web Link 1", "thank_you_message": "Thank you for completing our survey!", "thank_you_page": {"is_enabled": false, "message": "Thank you for completing our survey!"}, "disqualification_type": "message", "disqualification_message": "Thank you for completing our survey!", "disqualification_url": "https://www.surveymonkey.com", "closed_page_message": "This survey is currently closed. Please contact the author of this survey for further assistance.", "redirect_type": "url", "redirect_url": "https://www.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-02T18:14:00+00:00", "date_created": "2021-06-01T17:30:00+00:00", "response_count": 13, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://www.surveymonkey.com/r/BQB2V52", "href": "https://api.surveymonkey.com/v3/collectors/405437100"}, "emitted_at": 1681915512508} -{"stream": "collectors", "data": {"status": "open", "id": "405843657", "survey_id": "307785429", "type": "weblink", "name": "Web Link 1", "thank_you_message": "Grazie per aver partecipato al nostro sondaggio!", "thank_you_page": {"is_enabled": false, "message": "Grazie per aver partecipato al nostro sondaggio!"}, "disqualification_type": "message", "disqualification_message": "Grazie per aver partecipato al nostro sondaggio!", "disqualification_url": "https://it.surveymonkey.com", "closed_page_message": "Il sondaggio \u00e8 stato chiuso. Per ulteriori informazioni, contatta l\u2019autore del sondaggio.", "redirect_type": "url", "redirect_url": "https://it.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T10:59:00+00:00", "date_created": "2021-06-10T06:41:00+00:00", "response_count": 18, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://it.surveymonkey.com/r/GYXQMYF", "href": "https://api.surveymonkey.com/v3/collectors/405843657"}, "emitted_at": 1681915514460} -{"stream": "collectors", "data": {"status": "open", "id": "405843665", "survey_id": "307785444", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:00:00+00:00", "date_created": "2021-06-10T06:42:00+00:00", "response_count": 18, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/GYNFJLH", "href": "https://api.surveymonkey.com/v3/collectors/405843665"}, "emitted_at": 1681915516165} -{"stream": "collectors", "data": {"status": "open", "id": "405843672", "survey_id": "307785394", "type": "weblink", "name": "Web Link 1", "thank_you_message": "Thank you for completing our survey!", "thank_you_page": {"is_enabled": false, "message": "Thank you for completing our survey!"}, "disqualification_type": "message", "disqualification_message": "Thank you for completing our survey!", "disqualification_url": "https://www.surveymonkey.com", "closed_page_message": "This survey is currently closed. Please contact the author of this survey for further assistance.", "redirect_type": "url", "redirect_url": "https://www.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:01:00+00:00", "date_created": "2021-06-10T06:42:00+00:00", "response_count": 18, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://www.surveymonkey.com/r/GYFBNFM", "href": "https://api.surveymonkey.com/v3/collectors/405843672"}, "emitted_at": 1681915517864} -{"stream": "collectors", "data": {"status": "open", "id": "405843682", "survey_id": "307785402", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:02:00+00:00", "date_created": "2021-06-10T06:42:00+00:00", "response_count": 18, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/GYGQJ5R", "href": "https://api.surveymonkey.com/v3/collectors/405843682"}, "emitted_at": 1681915519653} -{"stream": "collectors", "data": {"status": "open", "id": "405843688", "survey_id": "307785408", "type": "weblink", "name": "Web Link 1", "thank_you_message": "Vielen Dank, dass Sie die Umfrage abgeschlossen haben!", "thank_you_page": {"is_enabled": false, "message": "Vielen Dank, dass Sie die Umfrage abgeschlossen haben!"}, "disqualification_type": "message", "disqualification_message": "Vielen Dank, dass Sie die Umfrage abgeschlossen haben!", "disqualification_url": "https://de.surveymonkey.com", "closed_page_message": "Diese Umfrage ist derzeit geschlossen. Wenden Sie sich an den Autor dieser Umfrage, um weitere Hilfe zu erhalten.", "redirect_type": "url", "redirect_url": "https://de.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:02:00+00:00", "date_created": "2021-06-10T06:43:00+00:00", "response_count": 18, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://de.surveymonkey.com/r/GYBJDCP", "href": "https://api.surveymonkey.com/v3/collectors/405843688"}, "emitted_at": 1681915521088} -{"stream": "collectors", "data": {"status": "open", "id": "405829319", "survey_id": "307784834", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:03:00+00:00", "date_created": "2021-06-09T21:08:00+00:00", "response_count": 21, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/NCH93N6", "href": "https://api.surveymonkey.com/v3/collectors/405829319"}, "emitted_at": 1681915522481} -{"stream": "collectors", "data": {"status": "open", "id": "405829931", "survey_id": "307785448", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:03:00+00:00", "date_created": "2021-06-09T21:21:00+00:00", "response_count": 18, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/NS2ZPQC", "href": "https://api.surveymonkey.com/v3/collectors/405829931"}, "emitted_at": 1681915524428} -{"stream": "collectors", "data": {"status": "open", "id": "405843442", "survey_id": "307784863", "type": "weblink", "name": "Web Link 1", "thank_you_message": "Vielen Dank, dass Sie die Umfrage abgeschlossen haben!", "thank_you_page": {"is_enabled": false, "message": "Vielen Dank, dass Sie die Umfrage abgeschlossen haben!"}, "disqualification_type": "message", "disqualification_message": "Vielen Dank, dass Sie die Umfrage abgeschlossen haben!", "disqualification_url": "https://de.surveymonkey.com", "closed_page_message": "Diese Umfrage ist derzeit geschlossen. Wenden Sie sich an den Autor dieser Umfrage, um weitere Hilfe zu erhalten.", "redirect_type": "url", "redirect_url": "https://de.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:04:00+00:00", "date_created": "2021-06-10T06:32:00+00:00", "response_count": 20, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://de.surveymonkey.com/r/GDQLK9D", "href": "https://api.surveymonkey.com/v3/collectors/405843442"}, "emitted_at": 1681915526346} -{"stream": "collectors", "data": {"status": "open", "id": "405829776", "survey_id": "307784846", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:05:00+00:00", "date_created": "2021-06-09T21:18:00+00:00", "response_count": 20, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/NWW9NKW", "href": "https://api.surveymonkey.com/v3/collectors/405829776"}, "emitted_at": 1681915528130} -{"stream": "collectors", "data": {"status": "open", "id": "405843460", "survey_id": "307784856", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:05:00+00:00", "date_created": "2021-06-10T06:32:00+00:00", "response_count": 20, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/GDSZR22", "href": "https://api.surveymonkey.com/v3/collectors/405843460"}, "emitted_at": 1681915529815} -{"stream": "collectors", "data": {"status": "open", "id": "405843624", "survey_id": "307785388", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:06:00+00:00", "date_created": "2021-06-10T06:40:00+00:00", "response_count": 20, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/GYWV533", "href": "https://api.surveymonkey.com/v3/collectors/405843624"}, "emitted_at": 1681915531700} -{"stream": "collectors", "data": {"status": "open", "id": "405843634", "survey_id": "307785415", "type": "weblink", "name": "Web Link 1", "thank_you_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "thank_you_page": {"is_enabled": false, "message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!"}, "disqualification_type": "message", "disqualification_message": "\u0421\u043f\u0430\u0441\u0438\u0431\u043e \u0437\u0430 \u0443\u0447\u0430\u0441\u0442\u0438\u0435 \u0432 \u043d\u0430\u0448\u0435\u043c \u043e\u043f\u0440\u043e\u0441\u0435!", "disqualification_url": "https://ru.surveymonkey.com", "closed_page_message": "\u0412 \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0432\u0440\u0435\u043c\u044f \u044d\u0442\u043e\u0442 \u043e\u043f\u0440\u043e\u0441 \u0437\u0430\u043a\u0440\u044b\u0442. \u0417\u0430 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u0439 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u0435\u0439 \u043e\u0431\u0440\u0430\u0449\u0430\u0439\u0442\u0435\u0441\u044c \u043a \u0430\u0432\u0442\u043e\u0440\u0443 \u043e\u043f\u0440\u043e\u0441\u0430.", "redirect_type": "url", "redirect_url": "https://ru.surveymonkey.com", "display_survey_results": false, "edit_response_type": "until_complete", "anonymous_type": "not_anonymous", "allow_multiple_responses": false, "date_modified": "2021-06-10T11:07:00+00:00", "date_created": "2021-06-10T06:40:00+00:00", "response_count": 20, "password_enabled": false, "response_limit": null, "respondent_authentication": false, "sender_email": null, "close_date": null, "url": "https://ru.surveymonkey.com/r/GYS33KN", "href": "https://api.surveymonkey.com/v3/collectors/405843634"}, "emitted_at": 1681915533510} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405437100", "href": "https://api.surveymonkey.com/v3/collectors/405437100", "type": "weblink", "survey_id": "306079584"}, "emitted_at": 1681918860499} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843657", "href": "https://api.surveymonkey.com/v3/collectors/405843657", "type": "weblink", "survey_id": "307785429"}, "emitted_at": 1681918861357} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843665", "href": "https://api.surveymonkey.com/v3/collectors/405843665", "type": "weblink", "survey_id": "307785444"}, "emitted_at": 1681918862236} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843672", "href": "https://api.surveymonkey.com/v3/collectors/405843672", "type": "weblink", "survey_id": "307785394"}, "emitted_at": 1681918863235} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843682", "href": "https://api.surveymonkey.com/v3/collectors/405843682", "type": "weblink", "survey_id": "307785402"}, "emitted_at": 1681918864098} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843688", "href": "https://api.surveymonkey.com/v3/collectors/405843688", "type": "weblink", "survey_id": "307785408"}, "emitted_at": 1681918865029} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405829319", "href": "https://api.surveymonkey.com/v3/collectors/405829319", "type": "weblink", "survey_id": "307784834"}, "emitted_at": 1681918865930} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405829931", "href": "https://api.surveymonkey.com/v3/collectors/405829931", "type": "weblink", "survey_id": "307785448"}, "emitted_at": 1681918866981} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843442", "href": "https://api.surveymonkey.com/v3/collectors/405843442", "type": "weblink", "survey_id": "307784863"}, "emitted_at": 1681918868050} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405829776", "href": "https://api.surveymonkey.com/v3/collectors/405829776", "type": "weblink", "survey_id": "307784846"}, "emitted_at": 1681918868953} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843460", "href": "https://api.surveymonkey.com/v3/collectors/405843460", "type": "weblink", "survey_id": "307784856"}, "emitted_at": 1681918870019} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843624", "href": "https://api.surveymonkey.com/v3/collectors/405843624", "type": "weblink", "survey_id": "307785388"}, "emitted_at": 1681918870844} -{"stream": "survey_collectors", "data": {"name": "Web Link 1", "id": "405843634", "href": "https://api.surveymonkey.com/v3/collectors/405843634", "type": "weblink", "survey_id": "307785415"}, "emitted_at": 1681918871852} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml b/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml index 9c5054626e8c..1c5c6fc423a9 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml +++ b/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/surveymonkey tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-surveymonkey/requirements.txt b/airbyte-integrations/connectors/source-surveymonkey/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/requirements.txt +++ b/airbyte-integrations/connectors/source-surveymonkey/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-surveymonkey/setup.py b/airbyte-integrations/connectors/source-surveymonkey/setup.py index 50e81f4d6047..f70f3e894857 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/setup.py +++ b/airbyte-integrations/connectors/source-surveymonkey/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "vcrpy==4.1.1", "urllib3<2.0"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "requests_mock"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "requests_mock"] setup( name="source_surveymonkey", diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml b/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml index 7a55e865084d..c9c5cbe7dcf0 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml +++ b/airbyte-integrations/connectors/source-talkdesk-explore/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/talkdesk-explore tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/requirements.txt b/airbyte-integrations/connectors/source-talkdesk-explore/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/requirements.txt +++ b/airbyte-integrations/connectors/source-talkdesk-explore/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py index 67019b6374bd..2694e175a333 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/setup.py +++ b/airbyte-integrations/connectors/source-talkdesk-explore/setup.py @@ -10,8 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-tempo/metadata.yaml b/airbyte-integrations/connectors/source-tempo/metadata.yaml index ac804c5692d5..a3bc6a870eba 100644 --- a/airbyte-integrations/connectors/source-tempo/metadata.yaml +++ b/airbyte-integrations/connectors/source-tempo/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tempo/requirements.txt b/airbyte-integrations/connectors/source-tempo/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-tempo/requirements.txt +++ b/airbyte-integrations/connectors/source-tempo/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-tempo/setup.py b/airbyte-integrations/connectors/source-tempo/setup.py index e51f66bd0411..16051a8dfa6b 100644 --- a/airbyte-integrations/connectors/source-tempo/setup.py +++ b/airbyte-integrations/connectors/source-tempo/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-teradata/metadata.yaml b/airbyte-integrations/connectors/source-teradata/metadata.yaml index 94b6f23cd69b..fc063a148c33 100644 --- a/airbyte-integrations/connectors/source-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/source-teradata/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-teradata/src/main/resources/spec.json b/airbyte-integrations/connectors/source-teradata/src/main/resources/spec.json index 25603e4e7330..82eaaeda13cf 100644 --- a/airbyte-integrations/connectors/source-teradata/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-teradata/src/main/resources/spec.json @@ -1,187 +1,164 @@ { - "documentationUrl" : "https://docs.airbyte.com/integrations/sources/teradata", - "connectionSpecification" : { - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "Teradata Source Spec", - "type" : "object", - "required" : [ - "host", - "database", - "username" - ], - "properties" : { - "host" : { - "title" : "Host", - "description" : "Hostname of the database.", - "type" : "string", - "order" : 0 + "documentationUrl": "https://docs.airbyte.com/integrations/sources/teradata", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Teradata Source Spec", + "type": "object", + "required": ["host", "database", "username"], + "properties": { + "host": { + "title": "Host", + "description": "Hostname of the database.", + "type": "string", + "order": 0 }, - "port" : { - "title" : "Port", - "description" : "Port of the database.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 3306, - "examples" : [ - "3306" - ], - "order" : 1 + "port": { + "title": "Port", + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 }, - "database" : { - "title" : "Database", - "description" : "Name of the database.", - "type" : "string", - "order" : 2 + "database": { + "title": "Database", + "description": "Name of the database.", + "type": "string", + "order": 2 }, - "username" : { - "title" : "Username", - "description" : "Username to use to access the database.", - "type" : "string", - "order" : 3 + "username": { + "title": "Username", + "description": "Username to use to access the database.", + "type": "string", + "order": 3 }, - "password" : { - "title" : "Password", - "description" : "Password associated with the username.", - "type" : "string", - "airbyte_secret" : true, - "order" : 4 + "password": { + "title": "Password", + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 }, - "jdbc_url_params" : { - "title" : "JDBC URL params", - "description" : "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", - "type" : "string", - "order" : 5 + "jdbc_url_params": { + "title": "JDBC URL params", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "type": "string", + "order": 5 }, - "replication_method" : { - "title" : "Replication method", - "description" : "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "type" : "string", - "order" : 6, - "default" : "STANDARD", - "enum" : [ - "STANDARD", - "CDC" - ] + "replication_method": { + "title": "Replication method", + "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "type": "string", + "order": 6, + "default": "STANDARD", + "enum": ["STANDARD", "CDC"] }, - "ssl" : { - "title" : "SSL Connection", - "description" : "Encrypt data using SSL. When activating SSL, please select one of the connection modes.", - "type" : "boolean", - "default" : false, - "order" : 7 + "ssl": { + "title": "SSL Connection", + "description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes.", + "type": "boolean", + "default": false, + "order": 7 }, - "ssl_mode" : { - "title" : "SSL Modes", - "description" : "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the destination database\n prefer - Chose this mode to allow unencrypted connection only if the destination database does not support encryption\n require - Chose this mode to always require encryption. If the destination database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the destination database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the destination database server\n See more information - in the docs.", - "type" : "object", - "order" : 8, - "oneOf" : [ + "ssl_mode": { + "title": "SSL Modes", + "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the destination database\n prefer - Chose this mode to allow unencrypted connection only if the destination database does not support encryption\n require - Chose this mode to always require encryption. If the destination database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the destination database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the destination database server\n See more information - in the docs.", + "type": "object", + "order": 8, + "oneOf": [ { - "title" : "disable", - "additionalProperties" : true, - "description" : "Disable SSL.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "disable", - "order" : 0 + "title": "disable", + "additionalProperties": true, + "description": "Disable SSL.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "disable", + "order": 0 } } }, { - "title" : "allow", - "additionalProperties" : true, - "description" : "Allow SSL mode.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "allow", - "order" : 0 + "title": "allow", + "additionalProperties": true, + "description": "Allow SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "allow", + "order": 0 } } }, { - "title" : "prefer", - "additionalProperties" : true, - "description" : "Prefer SSL mode.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "prefer", - "order" : 0 + "title": "prefer", + "additionalProperties": true, + "description": "Prefer SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "prefer", + "order": 0 } } }, { - "title" : "require", - "additionalProperties" : true, - "description" : "Require SSL mode.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "require", - "order" : 0 + "title": "require", + "additionalProperties": true, + "description": "Require SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "require", + "order": 0 } } }, { - "title" : "verify-ca", - "additionalProperties" : true, - "description" : "Verify-ca SSL mode.", - "required" : [ - "mode", - "ssl_ca_certificate" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "verify-ca", - "order" : 0 + "title": "verify-ca", + "additionalProperties": true, + "description": "Verify-ca SSL mode.", + "required": ["mode", "ssl_ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify-ca", + "order": 0 }, - "ssl_ca_certificate" : { - "type" : "string", - "title" : "CA certificate", - "description" : "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n See more information - in the docs.", - "airbyte_secret" : true, - "multiline" : true, - "order" : 1 + "ssl_ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n See more information - in the docs.", + "airbyte_secret": true, + "multiline": true, + "order": 1 } } }, { - "title" : "verify-full", - "additionalProperties" : true, - "description" : "Verify-full SSL mode.", - "required" : [ - "mode", - "ssl_ca_certificate" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "verify-full", - "order" : 0 + "title": "verify-full", + "additionalProperties": true, + "description": "Verify-full SSL mode.", + "required": ["mode", "ssl_ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify-full", + "order": 0 }, - "ssl_ca_certificate" : { - "type" : "string", - "title" : "CA certificate", - "description" : "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n See more information - in the docs.", - "airbyte_secret" : true, - "multiline" : true, - "order" : 1 + "ssl_ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n See more information - in the docs.", + "airbyte_secret": true, + "multiline": true, + "order": 1 } } } diff --git a/airbyte-integrations/connectors/source-teradata/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-teradata/src/test-integration/resources/expected_spec.json index 774f80d71932..fa89553125c6 100644 --- a/airbyte-integrations/connectors/source-teradata/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-teradata/src/test-integration/resources/expected_spec.json @@ -1,187 +1,164 @@ { - "documentationUrl" : "https://docs.airbyte.com/integrations/sources/teradata", - "connectionSpecification" : { - "$schema" : "http://json-schema.org/draft-07/schema#", - "title" : "Teradata Source Spec", - "type" : "object", - "required" : [ - "host", - "database", - "username" - ], - "properties" : { - "host" : { - "title" : "Host", - "description" : "Hostname of the database.", - "type" : "string", - "order" : 0 + "documentationUrl": "https://docs.airbyte.com/integrations/sources/teradata", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Teradata Source Spec", + "type": "object", + "required": ["host", "database", "username"], + "properties": { + "host": { + "title": "Host", + "description": "Hostname of the database.", + "type": "string", + "order": 0 }, - "port" : { - "title" : "Port", - "description" : "Port of the database.", - "type" : "integer", - "minimum" : 0, - "maximum" : 65536, - "default" : 3306, - "examples" : [ - "3306" - ], - "order" : 1 + "port": { + "title": "Port", + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 }, - "database" : { - "title" : "Database", - "description" : "Name of the database.", - "type" : "string", - "order" : 2 + "database": { + "title": "Database", + "description": "Name of the database.", + "type": "string", + "order": 2 }, - "username" : { - "title" : "Username", - "description" : "Username to use to access the database.", - "type" : "string", - "order" : 3 + "username": { + "title": "Username", + "description": "Username to use to access the database.", + "type": "string", + "order": 3 }, - "password" : { - "title" : "Password", - "description" : "Password associated with the username.", - "type" : "string", - "airbyte_secret" : true, - "order" : 4 + "password": { + "title": "Password", + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 }, - "jdbc_url_params" : { - "title" : "JDBC URL params", - "description" : "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", - "type" : "string", - "order" : 5 + "jdbc_url_params": { + "title": "JDBC URL params", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "type": "string", + "order": 5 }, - "replication_method" : { - "title" : "Replication method", - "description" : "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "type" : "string", - "order" : 6, - "default" : "STANDARD", - "enum" : [ - "STANDARD", - "CDC" - ] + "replication_method": { + "title": "Replication method", + "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "type": "string", + "order": 6, + "default": "STANDARD", + "enum": ["STANDARD", "CDC"] }, - "ssl" : { - "title" : "SSL Connection", - "description" : "Encrypt data using SSL. When activating SSL, please select one of the connection modes.", - "type" : "boolean", - "default" : false, - "order" : 7 + "ssl": { + "title": "SSL Connection", + "description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes.", + "type": "boolean", + "default": false, + "order": 7 }, - "ssl_mode" : { - "title" : "SSL Modes", - "description" : "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the destination database\n prefer - Chose this mode to allow unencrypted connection only if the destination database does not support encryption\n require - Chose this mode to always require encryption. If the destination database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the destination database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the destination database server\n See more information - in the docs.", - "type" : "object", - "order" : 8, - "oneOf" : [ + "ssl_mode": { + "title": "SSL Modes", + "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the destination database\n prefer - Chose this mode to allow unencrypted connection only if the destination database does not support encryption\n require - Chose this mode to always require encryption. If the destination database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the destination database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the destination database server\n See more information - in the docs.", + "type": "object", + "order": 8, + "oneOf": [ { - "title" : "disable", - "additionalProperties" : true, - "description" : "Disable SSL.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "disable", - "order" : 0 + "title": "disable", + "additionalProperties": true, + "description": "Disable SSL.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "disable", + "order": 0 } } }, { - "title" : "allow", - "additionalProperties" : true, - "description" : "Allow SSL mode.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "allow", - "order" : 0 + "title": "allow", + "additionalProperties": true, + "description": "Allow SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "allow", + "order": 0 } } }, { - "title" : "prefer", - "additionalProperties" : true, - "description" : "Prefer SSL mode.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "prefer", - "order" : 0 + "title": "prefer", + "additionalProperties": true, + "description": "Prefer SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "prefer", + "order": 0 } } }, { - "title" : "require", - "additionalProperties" : true, - "description" : "Require SSL mode.", - "required" : [ - "mode" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "require", - "order" : 0 + "title": "require", + "additionalProperties": true, + "description": "Require SSL mode.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "require", + "order": 0 } } }, { - "title" : "verify-ca", - "additionalProperties" : true, - "description" : "Verify-ca SSL mode.", - "required" : [ - "mode", - "ssl_ca_certificate" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "verify-ca", - "order" : 0 + "title": "verify-ca", + "additionalProperties": true, + "description": "Verify-ca SSL mode.", + "required": ["mode", "ssl_ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify-ca", + "order": 0 }, - "ssl_ca_certificate" : { - "type" : "string", - "title" : "CA certificate", - "description" : "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n See more information - in the docs.", - "airbyte_secret" : true, - "multiline" : true, - "order" : 1 + "ssl_ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-ca.\n See more information - in the docs.", + "airbyte_secret": true, + "multiline": true, + "order": 1 } } }, { - "title" : "verify-full", - "additionalProperties" : true, - "description" : "Verify-full SSL mode.", - "required" : [ - "mode", - "ssl_ca_certificate" - ], - "properties" : { - "mode" : { - "type" : "string", - "const" : "verify-full", - "order" : 0 + "title": "verify-full", + "additionalProperties": true, + "description": "Verify-full SSL mode.", + "required": ["mode", "ssl_ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify-full", + "order": 0 }, - "ssl_ca_certificate" : { - "type" : "string", - "title" : "CA certificate", - "description" : "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n See more information - in the docs.", - "airbyte_secret" : true, - "multiline" : true, - "order" : 1 + "ssl_ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "Specifies the file name of a PEM file that contains Certificate Authority (CA) certificates for use with SSLMODE=verify-full.\n See more information - in the docs.", + "airbyte_secret": true, + "multiline": true, + "order": 1 } } } @@ -189,5 +166,5 @@ } } }, - "supported_destination_sync_modes" : [] + "supported_destination_sync_modes": [] } diff --git a/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml b/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml index 22227c9db433..f074b8f5153b 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-the-guardian-api/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-the-guardian-api/requirements.txt b/airbyte-integrations/connectors/source-the-guardian-api/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/requirements.txt +++ b/airbyte-integrations/connectors/source-the-guardian-api/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-the-guardian-api/setup.py b/airbyte-integrations/connectors/source-the-guardian-api/setup.py index 9fdca8e789d0..1fb11d947926 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/setup.py +++ b/airbyte-integrations/connectors/source-the-guardian-api/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index ddfe4ccea227..26069fa25844 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -25,5 +25,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index b4f9f28d3ee5..6f41920bc5ca 100644 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -35,3 +35,4 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } + diff --git a/airbyte-integrations/connectors/source-tidb/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-tidb/integration_tests/seed/basic.sql index 6566b7384411..88d111d3ed06 100644 --- a/airbyte-integrations/connectors/source-tidb/integration_tests/seed/basic.sql +++ b/airbyte-integrations/connectors/source-tidb/integration_tests/seed/basic.sql @@ -1,4 +1,51 @@ -CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200)); -INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash'); -CREATE TABLE starships(id INTEGER, name VARCHAR(200)); -INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato'); +CREATE + TABLE + id_and_name( + id INTEGER, + name VARCHAR(200) + ); + +INSERT + INTO + id_and_name( + id, + name + ) + VALUES( + 1, + 'picard' + ), + ( + 2, + 'crusher' + ), + ( + 3, + 'vash' + ); + +CREATE + TABLE + starships( + id INTEGER, + name VARCHAR(200) + ); + +INSERT + INTO + starships( + id, + name + ) + VALUES( + 1, + 'enterprise-d' + ), + ( + 2, + 'defiant' + ), + ( + 3, + 'yamato' + ); diff --git a/airbyte-integrations/connectors/source-tidb/metadata.yaml b/airbyte-integrations/connectors/source-tidb/metadata.yaml index b6f4ec5288da..47a33d680ae8 100644 --- a/airbyte-integrations/connectors/source-tidb/metadata.yaml +++ b/airbyte-integrations/connectors/source-tidb/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: database connectorType: source definitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-tidb githubIssueLabel: source-tidb icon: tidb.svg @@ -22,4 +22,8 @@ data: tags: - language:java - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile index 10d07304d5c7..4b62a8e5fe52 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile @@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=3.2.1 +LABEL io.airbyte.version=3.4.1 LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json index f9422134089a..90acbeb472d2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json @@ -116,7 +116,12 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["stat_time_day"], - "source_defined_primary_key": [["ad_id"], ["stat_time_day"], ["gender"], ["age"]] + "source_defined_primary_key": [ + ["ad_id"], + ["stat_time_day"], + ["gender"], + ["age"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", @@ -130,7 +135,12 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["stat_time_day"], - "source_defined_primary_key": [["adgroup_id"], ["stat_time_day"], ["gender"], ["age"]] + "source_defined_primary_key": [ + ["adgroup_id"], + ["stat_time_day"], + ["gender"], + ["age"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", @@ -144,7 +154,11 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["stat_time_day"], - "source_defined_primary_key": [["campaign_id"], ["stat_time_day"], ["country_code"]] + "source_defined_primary_key": [ + ["campaign_id"], + ["stat_time_day"], + ["country_code"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", @@ -158,7 +172,12 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["stat_time_day"], - "source_defined_primary_key": [["advertiser_id"], ["stat_time_day"], ["gender"], ["age"]] + "source_defined_primary_key": [ + ["advertiser_id"], + ["stat_time_day"], + ["gender"], + ["age"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl index 59538f3b9b1c..dc6fbcbf35a0 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl @@ -1,50 +1,50 @@ -{"stream": "advertisers", "data": {"role": "ROLE_ADVERTISER", "timezone": "Etc/GMT+8", "license_url": null, "promotion_area": "0", "balance": 10, "telephone_number": "+14156****85", "name": "Airbyte0830", "create_time": 1630335591, "advertiser_id": 7002238017842757633, "country": "US", "promotion_center_province": null, "email": "i***************@**********", "promotion_center_city": null, "brand": null, "rejection_reason": "", "cellphone_number": "+13477****53", "language": "en", "status": "STATUS_ENABLE", "advertiser_account_type": "AUCTION", "display_timezone": "America/Los_Angeles", "address": "350 29th avenue, San Francisco", "company": "Airbyte", "currency": "USD", "industry": "291905", "license_no": "", "contacter": "Ai***te", "description": "https://", "license_city": null, "license_province": null}, "emitted_at": 1685013713817} -{"stream": "ads", "data": {"landing_page_url": "https://airbyte.com", "playable_url": "", "app_name": "", "impression_tracking_url": null, "brand_safety_vast_url": null, "is_aco": false, "ad_format": "SINGLE_VIDEO", "image_ids": ["v0201/7f371ff6f0764f8b8ef4f37d7b980d50"], "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "adgroup_name": "AdGroupVadim", "display_name": "airbyte", "landing_page_urls": null, "deeplink": "", "is_new_structure": true, "creative_type": null, "identity_type": "CUSTOMIZED_USER", "ad_id": 1728545390695442, "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "campaign_name": "CampaignVadimTraffic", "deeplink_type": "NORMAL", "click_tracking_url": null, "identity_id": "7080121820963422209", "modify_time": "2022-03-28 21:34:26", "ad_text": "Open-source\ndata integration for modern data teams", "card_id": null, "create_time": "2022-03-28 12:09:09", "operation_status": "ENABLE", "vast_moat_enabled": false, "optimization_event": null, "page_id": null, "music_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "ad_texts": null, "call_to_action_id": "7080120957230238722", "creative_authorized": false, "brand_safety_postbid_partner": "UNSET", "viewability_postbid_partner": "UNSET", "viewability_vast_url": null, "adgroup_id": 1728545385226289, "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", "advertiser_id": 7002238017842757633, "campaign_id": 1728545382536225, "fallback_type": "UNSET"}, "emitted_at": 1685013714777} -{"stream": "ads", "data": {"landing_page_url": "https://airbyte.io", "playable_url": "", "app_name": "", "impression_tracking_url": null, "brand_safety_vast_url": null, "is_aco": false, "ad_format": "SINGLE_VIDEO", "image_ids": ["v0201/8f77082a1f3c40c586f8282356490c58"], "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "adgroup_name": "Ad Group20211020010107", "display_name": "Airbyte", "call_to_action": "LEARN_MORE", "landing_page_urls": null, "deeplink": "", "is_new_structure": true, "creative_type": null, "identity_type": "UNSET", "ad_id": 1714125051115569, "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "campaign_name": "Website Traffic20211020010104", "deeplink_type": "NORMAL", "click_tracking_url": null, "identity_id": "", "modify_time": "2021-10-21 17:50:11", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "card_id": null, "create_time": "2021-10-20 08:04:06", "operation_status": "ENABLE", "vast_moat_enabled": false, "optimization_event": null, "page_id": null, "music_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "ad_texts": null, "call_to_action_id": null, "creative_authorized": true, "brand_safety_postbid_partner": "UNSET", "viewability_postbid_partner": "UNSET", "viewability_vast_url": null, "adgroup_id": 1714125049901106, "video_id": "v10033g50000c5nsqcbc77ubdn136b70", "advertiser_id": 7002238017842757633, "campaign_id": 1714125042508817, "fallback_type": "UNSET"}, "emitted_at": 1685013714779} -{"stream": "ads", "data": {"landing_page_url": "https://airbyte.io", "playable_url": "", "app_name": "", "impression_tracking_url": null, "brand_safety_vast_url": null, "is_aco": false, "ad_format": "SINGLE_IMAGE", "image_ids": ["ad-site-i18n-sg/202110195d0db51e12a222ee4334a396"], "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202110205d0d4ce78859d72d409eb82d", "adgroup_name": "Ad Group20211020005346", "display_name": "Airbyte", "call_to_action": "LEARN_MORE", "landing_page_urls": null, "deeplink": "", "is_new_structure": true, "creative_type": null, "identity_type": "UNSET", "ad_id": 1714124564763650, "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "campaign_name": "Website Traffic20211020005342", "deeplink_type": "NORMAL", "click_tracking_url": null, "identity_id": "", "modify_time": "2021-10-20 08:05:12", "ad_text": "Airbyte - open source data portability platform - from nywhere to anywhere!", "card_id": null, "create_time": "2021-10-20 07:56:39", "operation_status": "DISABLE", "vast_moat_enabled": false, "optimization_event": null, "page_id": null, "music_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d4ce78859d72d409eb82d", "ad_name": "1200x1200 logo_1634667070143.png_2021-10-20 10:54:51", "ad_texts": null, "call_to_action_id": null, "creative_authorized": false, "brand_safety_postbid_partner": "UNSET", "viewability_postbid_partner": "UNSET", "viewability_vast_url": null, "adgroup_id": 1714124588896305, "video_id": null, "advertiser_id": 7002238017842757633, "campaign_id": 1714124576938033, "fallback_type": "UNSET"}, "emitted_at": 1685013714780} -{"stream": "ad_groups", "data": {"budget": 20, "campaign_name": "CampaignVadimTraffic", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "schedule_type": "SCHEDULE_FROM_NOW", "auto_targeting_enabled": false, "placements": null, "adgroup_name": "AdGroupVadim", "frequency": null, "excluded_audience_ids": [], "included_custom_actions": [], "inventory_filter_enabled": false, "budget_mode": "BUDGET_MODE_DAY", "adgroup_app_profile_page_state": null, "adgroup_id": 1728545385226289, "bid_price": 0, "pixel_id": null, "scheduled_budget": 0, "schedule_end_time": "2032-03-25 13:02:23", "app_type": null, "conversion_bid_price": 0, "conversion_window": null, "search_result_enabled": false, "billing_event": "CPC", "interest_category_ids": [15], "app_download_url": null, "bid_display_mode": "CPMV", "interest_keyword_ids": [], "device_price_ranges": [], "secondary_optimization_event": null, "advertiser_id": 7002238017842757633, "optimization_event": null, "feed_type": null, "statistic_type": null, "share_disabled": false, "app_id": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "is_new_structure": true, "creative_material_mode": "CUSTOM", "optimization_goal": "CLICK", "next_day_retention": null, "video_download_disabled": false, "rf_estimated_frequency": null, "campaign_id": 1728545382536225, "is_hfss": false, "location_ids": [6252001], "promotion_type": "WEBSITE", "purchased_impression": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "excluded_custom_actions": [], "operating_systems": [], "languages": [], "ios14_quota_type": "UNOCCUPIED", "create_time": "2022-03-28 12:09:07", "deep_bid_type": null, "gender": "GENDER_UNLIMITED", "skip_learning_phase": 0, "bid_type": "BID_TYPE_NO_BID", "rf_purchased_type": null, "device_model_ids": [], "pacing": "PACING_MODE_SMOOTH", "keywords": null, "age_groups": ["AGE_25_34", "AGE_35_44"], "modify_time": "2022-03-31 08:13:30", "purchased_reach": null, "network_types": [], "audience_ids": [], "operation_status": "ENABLE", "brand_safety_partner": null, "category_id": 0, "schedule_start_time": "2022-03-28 13:02:23", "actions": [], "delivery_mode": null, "brand_safety_type": "NO_BRAND_SAFETY", "comment_disabled": false, "schedule_infos": null, "deep_cpa_bid": 0, "rf_estimated_cpr": null, "frequency_schedule": null}, "emitted_at": 1685013715593} -{"stream": "ad_groups", "data": {"budget": 20, "campaign_name": "Website Traffic20211020010104", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "schedule_type": "SCHEDULE_START_END", "auto_targeting_enabled": false, "placements": null, "adgroup_name": "Ad Group20211020010107", "frequency": null, "excluded_audience_ids": [], "included_custom_actions": [], "inventory_filter_enabled": false, "budget_mode": "BUDGET_MODE_DAY", "adgroup_app_profile_page_state": null, "adgroup_id": 1714125049901106, "bid_price": 0, "pixel_id": null, "scheduled_budget": 0, "schedule_end_time": "2021-10-31 09:01:07", "app_type": null, "conversion_bid_price": 0, "conversion_window": null, "search_result_enabled": false, "billing_event": "CPC", "interest_category_ids": [], "app_download_url": null, "bid_display_mode": "CPMV", "interest_keyword_ids": [], "device_price_ranges": [], "secondary_optimization_event": null, "advertiser_id": 7002238017842757633, "optimization_event": null, "feed_type": null, "statistic_type": null, "share_disabled": false, "app_id": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "is_new_structure": true, "creative_material_mode": "CUSTOM", "optimization_goal": "CLICK", "next_day_retention": null, "video_download_disabled": false, "rf_estimated_frequency": null, "campaign_id": 1714125042508817, "is_hfss": false, "location_ids": [6252001], "promotion_type": "WEBSITE", "purchased_impression": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "excluded_custom_actions": [], "operating_systems": [], "languages": ["en"], "ios14_quota_type": "UNOCCUPIED", "create_time": "2021-10-20 08:04:05", "deep_bid_type": null, "gender": "GENDER_UNLIMITED", "skip_learning_phase": 0, "bid_type": "BID_TYPE_NO_BID", "rf_purchased_type": null, "device_model_ids": [], "pacing": "PACING_MODE_SMOOTH", "keywords": null, "age_groups": ["AGE_25_34", "AGE_35_44", "AGE_45_54"], "modify_time": "2022-03-24 12:06:54", "purchased_reach": null, "network_types": [], "audience_ids": [], "operation_status": "ENABLE", "brand_safety_partner": null, "category_id": 0, "schedule_start_time": "2021-10-20 09:01:07", "actions": [], "delivery_mode": null, "brand_safety_type": "NO_BRAND_SAFETY", "comment_disabled": false, "schedule_infos": null, "deep_cpa_bid": 0, "rf_estimated_cpr": null, "frequency_schedule": null}, "emitted_at": 1685013715595} -{"stream": "ad_groups", "data": {"budget": 20, "campaign_name": "Website Traffic20211020005342", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "schedule_type": "SCHEDULE_START_END", "auto_targeting_enabled": false, "placements": null, "adgroup_name": "Ad Group20211020005346", "frequency": null, "excluded_audience_ids": [], "included_custom_actions": [], "inventory_filter_enabled": false, "budget_mode": "BUDGET_MODE_DAY", "adgroup_app_profile_page_state": null, "adgroup_id": 1714124588896305, "bid_price": 0, "pixel_id": null, "scheduled_budget": 0, "schedule_end_time": "2021-10-31 08:53:46", "app_type": null, "conversion_bid_price": 0, "conversion_window": null, "search_result_enabled": false, "billing_event": "CPC", "interest_category_ids": [], "app_download_url": null, "bid_display_mode": "CPMV", "interest_keyword_ids": [], "device_price_ranges": [], "secondary_optimization_event": null, "advertiser_id": 7002238017842757633, "optimization_event": null, "feed_type": null, "statistic_type": null, "share_disabled": false, "app_id": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "is_new_structure": true, "creative_material_mode": "CUSTOM", "optimization_goal": "CLICK", "next_day_retention": null, "video_download_disabled": false, "rf_estimated_frequency": null, "campaign_id": 1714124576938033, "is_hfss": false, "location_ids": [6252001], "promotion_type": "WEBSITE", "purchased_impression": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "excluded_custom_actions": [], "operating_systems": [], "languages": [], "ios14_quota_type": "UNOCCUPIED", "create_time": "2021-10-20 07:56:39", "deep_bid_type": null, "gender": "GENDER_UNLIMITED", "skip_learning_phase": 0, "bid_type": "BID_TYPE_NO_BID", "rf_purchased_type": null, "device_model_ids": [], "pacing": "PACING_MODE_SMOOTH", "keywords": null, "age_groups": null, "modify_time": "2021-10-20 08:08:14", "purchased_reach": null, "network_types": [], "audience_ids": [], "operation_status": "DISABLE", "brand_safety_partner": null, "category_id": 0, "schedule_start_time": "2021-10-20 08:53:46", "actions": [], "delivery_mode": null, "brand_safety_type": "NO_BRAND_SAFETY", "comment_disabled": false, "schedule_infos": null, "deep_cpa_bid": 0, "rf_estimated_cpr": null, "frequency_schedule": null}, "emitted_at": 1685013715596} -{"stream": "campaigns", "data": {"create_time": "2022-03-28 12:09:05", "modify_time": "2022-03-30 21:23:52", "campaign_type": "REGULAR_CAMPAIGN", "budget": 0, "campaign_id": 1728545382536225, "advertiser_id": 7002238017842757633, "campaign_name": "CampaignVadimTraffic", "roas_bid": 0, "is_new_structure": true, "budget_mode": "BUDGET_MODE_INFINITE", "is_smart_performance_campaign": false, "objective": "LANDING_PAGE", "deep_bid_type": null, "operation_status": "DISABLE", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE"}, "emitted_at": 1685013716180} -{"stream": "campaigns", "data": {"create_time": "2021-10-20 08:04:04", "modify_time": "2022-03-24 12:08:29", "campaign_type": "REGULAR_CAMPAIGN", "budget": 0, "campaign_id": 1714125042508817, "advertiser_id": 7002238017842757633, "campaign_name": "Website Traffic20211020010104", "roas_bid": 0, "is_new_structure": true, "budget_mode": "BUDGET_MODE_INFINITE", "is_smart_performance_campaign": false, "objective": "LANDING_PAGE", "deep_bid_type": null, "operation_status": "DISABLE", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE"}, "emitted_at": 1685013716180} -{"stream": "campaigns", "data": {"create_time": "2021-10-20 07:56:38", "modify_time": "2021-10-20 08:01:18", "campaign_type": "REGULAR_CAMPAIGN", "budget": 0, "campaign_id": 1714124576938033, "advertiser_id": 7002238017842757633, "campaign_name": "Website Traffic20211020005342", "roas_bid": 0, "is_new_structure": true, "budget_mode": "BUDGET_MODE_INFINITE", "is_smart_performance_campaign": false, "objective": "LANDING_PAGE", "deep_bid_type": null, "operation_status": "DISABLE", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE"}, "emitted_at": 1685013716181} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7001035076276387841, "advertiser_name": "Airbyte0827"}, "emitted_at": 1685013716765} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7001040009704833026, "advertiser_name": "Airbyte08270"}, "emitted_at": 1685013716766} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7002238017842757633, "advertiser_name": "Airbyte0830"}, "emitted_at": 1685013716766} -{"stream": "ads_reports_daily", "data": {"metrics": {"adgroup_id": 1714125049901106, "reach": "4806", "placement_type": "Automatic Placement", "tt_app_name": "0", "average_video_play": "1.52", "real_time_conversion_rate": "0.00", "video_views_p25": "513", "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "comments": "0", "cost_per_secondary_goal_result": null, "real_time_app_install": "0", "promotion_type": "Website", "spend": "20.000", "follows": "0", "real_time_result_rate": "1.18", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "dpa_target_audience_type": null, "video_play_actions": "5173", "real_time_result": "69", "conversion_rate": "0.00", "cost_per_1000_reached": "4.161", "result_rate": "1.18", "ctr": "1.18", "real_time_app_install_cost": "0.000", "cpm": "3.430", "frequency": "1.21", "profile_visits": "0", "cpc": "0.290", "app_install": "0", "adgroup_name": "Ad Group20211020010107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_cost_per_result": "0.2899", "video_views_p50": "214", "average_video_play_per_user": "1.64", "campaign_id": 1714125042508817, "mobile_app_id": "0", "impressions": "5830", "video_views_p75": "140", "video_watched_6s": "180", "clicks_on_music_disc": "0", "result": "69", "cost_per_result": "0.2899", "cost_per_conversion": "0.000", "likes": "36", "video_views_p100": "92", "video_watched_2s": "686", "secondary_goal_result": null, "shares": "0", "conversion": "0", "clicks": "69", "secondary_goal_result_rate": null, "real_time_conversion": "0"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1685014267382} -{"stream": "ads_reports_daily", "data": {"metrics": {"adgroup_id": 1714125049901106, "reach": "3134", "placement_type": "Automatic Placement", "tt_app_name": "0", "average_video_play": "1.45", "real_time_conversion_rate": "0.00", "video_views_p25": "295", "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "comments": "1", "cost_per_secondary_goal_result": null, "real_time_app_install": "0", "promotion_type": "Website", "spend": "20.000", "follows": "0", "real_time_result_rate": "1.41", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "dpa_target_audience_type": null, "video_play_actions": "3344", "real_time_result": "53", "conversion_rate": "0.00", "cost_per_1000_reached": "6.382", "result_rate": "1.41", "ctr": "1.41", "real_time_app_install_cost": "0.000", "cpm": "5.310", "frequency": "1.20", "profile_visits": "0", "cpc": "0.380", "app_install": "0", "adgroup_name": "Ad Group20211020010107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_cost_per_result": "0.3774", "video_views_p50": "130", "average_video_play_per_user": "1.55", "campaign_id": 1714125042508817, "mobile_app_id": "0", "impressions": "3765", "video_views_p75": "74", "video_watched_6s": "106", "clicks_on_music_disc": "0", "result": "53", "cost_per_result": "0.3774", "cost_per_conversion": "0.000", "likes": "36", "video_views_p100": "52", "video_watched_2s": "408", "secondary_goal_result": null, "shares": "0", "conversion": "0", "clicks": "53", "secondary_goal_result_rate": null, "real_time_conversion": "0"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1685014267385} -{"stream": "ads_reports_daily", "data": {"metrics": {"adgroup_id": 1714125049901106, "reach": "3119", "placement_type": "Automatic Placement", "tt_app_name": "0", "average_video_play": "1.50", "real_time_conversion_rate": "0.00", "video_views_p25": "297", "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "comments": "1", "cost_per_secondary_goal_result": null, "real_time_app_install": "0", "promotion_type": "Website", "spend": "20.000", "follows": "0", "real_time_result_rate": "1.23", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "dpa_target_audience_type": null, "video_play_actions": "3344", "real_time_result": "46", "conversion_rate": "0.00", "cost_per_1000_reached": "6.412", "result_rate": "1.23", "ctr": "1.23", "real_time_app_install_cost": "0.000", "cpm": "5.330", "frequency": "1.20", "profile_visits": "0", "cpc": "0.430", "app_install": "0", "adgroup_name": "Ad Group20211020010107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_cost_per_result": "0.4348", "video_views_p50": "142", "average_video_play_per_user": "1.61", "campaign_id": 1714125042508817, "mobile_app_id": "0", "impressions": "3750", "video_views_p75": "90", "video_watched_6s": "112", "clicks_on_music_disc": "0", "result": "46", "cost_per_result": "0.4348", "cost_per_conversion": "0.000", "likes": "25", "video_views_p100": "71", "video_watched_2s": "413", "secondary_goal_result": null, "shares": "0", "conversion": "0", "clicks": "46", "secondary_goal_result_rate": null, "real_time_conversion": "0"}, "dimensions": {"stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1685014267388} -{"stream": "ads_reports_lifetime", "data": {"metrics": {"frequency": "1.20", "average_video_play_per_user": "1.39", "comments": "0", "result": "145", "cost_per_secondary_goal_result": null, "tt_app_name": "0", "impressions": "15689", "video_watched_6s": "402", "real_time_app_install_cost": "0.000", "spend": "60.000", "video_views_p50": "907", "tt_app_id": 0, "real_time_result_rate": "0.92", "real_time_conversion": "0", "app_install": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "promotion_type": "Website", "adgroup_id": 1728545385226289, "result_rate": "0.92", "campaign_id": 1728545382536225, "secondary_goal_result": null, "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "cpc": "0.410", "profile_visits": "0", "video_views_p75": "522", "video_watched_2s": "1364", "video_views_p25": "3339", "reach": "13052", "real_time_app_install": "0", "clicks_on_music_disc": "0", "campaign_name": "CampaignVadimTraffic", "cost_per_conversion": "0.000", "cost_per_result": "0.4138", "video_views_p100": "402", "mobile_app_id": "0", "cost_per_1000_reached": "4.597", "cpm": "3.820", "dpa_target_audience_type": null, "real_time_cost_per_conversion": "0.000", "secondary_goal_result_rate": null, "conversion": "0", "average_video_play": "1.26", "real_time_cost_per_result": "0.4138", "real_time_result": "145", "ad_text": "Open-source\ndata integration for modern data teams", "adgroup_name": "AdGroupVadim", "follows": "0", "placement_type": "Automatic Placement", "video_play_actions": "14333", "shares": "0", "likes": "11", "ctr": "0.92", "clicks": "145"}, "dimensions": {"ad_id": 1728545390695442}, "ad_id": 1728545390695442}, "emitted_at": 1685014288730} -{"stream": "ads_reports_lifetime", "data": {"metrics": {"frequency": "1.37", "average_video_play_per_user": "1.80", "comments": "2", "result": "540", "cost_per_secondary_goal_result": null, "tt_app_name": "0", "impressions": "46116", "video_watched_6s": "1295", "real_time_app_install_cost": "0.000", "spend": "200.000", "video_views_p50": "1588", "tt_app_id": 0, "real_time_result_rate": "1.17", "real_time_conversion": "0", "app_install": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "promotion_type": "Website", "adgroup_id": 1714125049901106, "result_rate": "1.17", "campaign_id": 1714125042508817, "secondary_goal_result": null, "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "cpc": "0.370", "profile_visits": "0", "video_views_p75": "998", "video_watched_2s": "5100", "video_views_p25": "3674", "reach": "33556", "real_time_app_install": "0", "clicks_on_music_disc": "0", "campaign_name": "Website Traffic20211020010104", "cost_per_conversion": "0.000", "cost_per_result": "0.3704", "video_views_p100": "723", "mobile_app_id": "0", "cost_per_1000_reached": "5.960", "cpm": "4.340", "dpa_target_audience_type": null, "real_time_cost_per_conversion": "0.000", "secondary_goal_result_rate": null, "conversion": "0", "average_video_play": "1.48", "real_time_cost_per_result": "0.3704", "real_time_result": "540", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "adgroup_name": "Ad Group20211020010107", "follows": "0", "placement_type": "Automatic Placement", "video_play_actions": "40753", "shares": "0", "likes": "263", "ctr": "1.17", "clicks": "540"}, "dimensions": {"ad_id": 1714125051115569}, "ad_id": 1714125051115569}, "emitted_at": 1685014288733} -{"stream": "ads_reports_lifetime", "data": {"metrics": {"frequency": "0.00", "average_video_play_per_user": "0.00", "comments": "0", "result": "0", "cost_per_secondary_goal_result": null, "tt_app_name": "0", "impressions": "0", "video_watched_6s": "0", "real_time_app_install_cost": "0.000", "spend": "0.000", "video_views_p50": "0", "tt_app_id": 0, "real_time_result_rate": "0.00", "real_time_conversion": "0", "app_install": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "promotion_type": "Website", "adgroup_id": 1714124588896305, "result_rate": "0.00", "campaign_id": 1714124576938033, "secondary_goal_result": null, "ad_name": "1200x1200 logo_1634667070143.png_2021-10-20 10:54:51", "cpc": "0.000", "profile_visits": "0", "video_views_p75": "0", "video_watched_2s": "0", "video_views_p25": "0", "reach": "0", "real_time_app_install": "0", "clicks_on_music_disc": "0", "campaign_name": "Website Traffic20211020005342", "cost_per_conversion": "0.000", "cost_per_result": "0.0000", "video_views_p100": "0", "mobile_app_id": "0", "cost_per_1000_reached": "0.000", "cpm": "0.000", "dpa_target_audience_type": null, "real_time_cost_per_conversion": "0.000", "secondary_goal_result_rate": null, "conversion": "0", "average_video_play": "0.00", "real_time_cost_per_result": "0.0000", "real_time_result": "0", "ad_text": "Airbyte - open source data portability platform - from nywhere to anywhere!", "adgroup_name": "Ad Group20211020005346", "follows": "0", "placement_type": "Automatic Placement", "video_play_actions": "0", "shares": "0", "likes": "0", "ctr": "0.00", "clicks": "0"}, "dimensions": {"ad_id": 1714124564763650}, "ad_id": 1714124564763650}, "emitted_at": 1685014288737} -{"stream": "ad_groups_reports_daily", "data": {"metrics": {"real_time_cost_per_result": "0.3774", "video_watched_2s": "408", "video_views_p100": "52", "dpa_target_audience_type": null, "clicks": "53", "average_video_play": "1.45", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_play_actions": "3344", "average_video_play_per_user": "1.55", "promotion_type": "Website", "result": "53", "impressions": "3765", "profile_visits": "0", "adgroup_name": "Ad Group20211020010107", "conversion_rate": "0.00", "cost_per_result": "0.3774", "shares": "0", "secondary_goal_result": null, "video_views_p50": "130", "real_time_conversion": "0", "mobile_app_id": "0", "comments": "1", "secondary_goal_result_rate": null, "campaign_name": "Website Traffic20211020010104", "real_time_app_install": "0", "placement_type": "Automatic Placement", "cpc": "0.380", "video_views_p25": "295", "tt_app_name": "0", "real_time_result_rate": "1.41", "frequency": "1.20", "campaign_id": 1714125042508817, "cpm": "5.310", "result_rate": "1.41", "ctr": "1.41", "video_watched_6s": "106", "real_time_cost_per_conversion": "0.000", "spend": "20.000", "app_install": "0", "reach": "3134", "real_time_result": "53", "conversion": "0", "real_time_conversion_rate": "0.00", "cost_per_secondary_goal_result": null, "follows": "0", "video_views_p75": "74", "clicks_on_music_disc": "0", "likes": "36", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "6.382"}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1685014836070} -{"stream": "ad_groups_reports_daily", "data": {"metrics": {"real_time_cost_per_result": "0.2899", "video_watched_2s": "686", "video_views_p100": "92", "dpa_target_audience_type": null, "clicks": "69", "average_video_play": "1.52", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_play_actions": "5173", "average_video_play_per_user": "1.64", "promotion_type": "Website", "result": "69", "impressions": "5830", "profile_visits": "0", "adgroup_name": "Ad Group20211020010107", "conversion_rate": "0.00", "cost_per_result": "0.2899", "shares": "0", "secondary_goal_result": null, "video_views_p50": "214", "real_time_conversion": "0", "mobile_app_id": "0", "comments": "0", "secondary_goal_result_rate": null, "campaign_name": "Website Traffic20211020010104", "real_time_app_install": "0", "placement_type": "Automatic Placement", "cpc": "0.290", "video_views_p25": "513", "tt_app_name": "0", "real_time_result_rate": "1.18", "frequency": "1.21", "campaign_id": 1714125042508817, "cpm": "3.430", "result_rate": "1.18", "ctr": "1.18", "video_watched_6s": "180", "real_time_cost_per_conversion": "0.000", "spend": "20.000", "app_install": "0", "reach": "4806", "real_time_result": "69", "conversion": "0", "real_time_conversion_rate": "0.00", "cost_per_secondary_goal_result": null, "follows": "0", "video_views_p75": "140", "clicks_on_music_disc": "0", "likes": "36", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "4.161"}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-25 00:00:00"}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1685014836073} -{"stream": "ad_groups_reports_daily", "data": {"metrics": {"real_time_cost_per_result": "0.5000", "video_watched_2s": "436", "video_views_p100": "66", "dpa_target_audience_type": null, "clicks": "40", "average_video_play": "1.41", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_play_actions": "3852", "average_video_play_per_user": "1.50", "promotion_type": "Website", "result": "40", "impressions": "4394", "profile_visits": "0", "adgroup_name": "Ad Group20211020010107", "conversion_rate": "0.00", "cost_per_result": "0.5000", "shares": "0", "secondary_goal_result": null, "video_views_p50": "130", "real_time_conversion": "0", "mobile_app_id": "0", "comments": "0", "secondary_goal_result_rate": null, "campaign_name": "Website Traffic20211020010104", "real_time_app_install": "0", "placement_type": "Automatic Placement", "cpc": "0.500", "video_views_p25": "306", "tt_app_name": "0", "real_time_result_rate": "0.91", "frequency": "1.21", "campaign_id": 1714125042508817, "cpm": "4.550", "result_rate": "0.91", "ctr": "0.91", "video_watched_6s": "104", "real_time_cost_per_conversion": "0.000", "spend": "20.000", "app_install": "0", "reach": "3621", "real_time_result": "40", "conversion": "0", "real_time_conversion_rate": "0.00", "cost_per_secondary_goal_result": null, "follows": "0", "video_views_p75": "85", "clicks_on_music_disc": "0", "likes": "13", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "5.523"}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-29 00:00:00"}, "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1685014836076} -{"stream": "ad_groups_reports_lifetime", "data": {"dimensions": {"adgroup_id": 1728545385226289}, "metrics": {"video_views_p50": "907", "secondary_goal_result_rate": null, "average_video_play": "1.26", "campaign_id": 1728545382536225, "tt_app_name": "0", "real_time_cost_per_result": "0.4138", "clicks_on_music_disc": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "video_views_p25": "3339", "cost_per_1000_reached": "4.597", "profile_visits": "0", "shares": "0", "frequency": "1.20", "likes": "11", "real_time_cost_per_conversion": "0.000", "cost_per_conversion": "0.000", "comments": "0", "real_time_result_rate": "0.92", "adgroup_name": "AdGroupVadim", "video_watched_2s": "1364", "tt_app_id": 0, "video_views_p75": "522", "promotion_type": "Website", "result": "145", "real_time_result": "145", "dpa_target_audience_type": null, "spend": "60.000", "cpm": "3.820", "real_time_app_install": "0", "impressions": "15689", "follows": "0", "reach": "13052", "video_watched_6s": "402", "cpc": "0.410", "placement_type": "Automatic Placement", "average_video_play_per_user": "1.39", "cost_per_result": "0.4138", "conversion_rate": "0.00", "clicks": "145", "video_play_actions": "14333", "video_views_p100": "402", "conversion": "0", "cost_per_secondary_goal_result": null, "secondary_goal_result": null, "real_time_app_install_cost": "0.000", "ctr": "0.92", "real_time_conversion": "0", "result_rate": "0.92", "app_install": "0", "campaign_name": "CampaignVadimTraffic"}, "adgroup_id": 1728545385226289}, "emitted_at": 1685014858398} -{"stream": "ad_groups_reports_lifetime", "data": {"dimensions": {"adgroup_id": 1714125049901106}, "metrics": {"video_views_p50": "1588", "secondary_goal_result_rate": null, "average_video_play": "1.48", "campaign_id": 1714125042508817, "tt_app_name": "0", "real_time_cost_per_result": "0.3704", "clicks_on_music_disc": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "video_views_p25": "3674", "cost_per_1000_reached": "5.960", "profile_visits": "0", "shares": "0", "frequency": "1.37", "likes": "263", "real_time_cost_per_conversion": "0.000", "cost_per_conversion": "0.000", "comments": "2", "real_time_result_rate": "1.17", "adgroup_name": "Ad Group20211020010107", "video_watched_2s": "5100", "tt_app_id": 0, "video_views_p75": "998", "promotion_type": "Website", "result": "540", "real_time_result": "540", "dpa_target_audience_type": null, "spend": "200.000", "cpm": "4.340", "real_time_app_install": "0", "impressions": "46116", "follows": "0", "reach": "33556", "video_watched_6s": "1295", "cpc": "0.370", "placement_type": "Automatic Placement", "average_video_play_per_user": "1.80", "cost_per_result": "0.3704", "conversion_rate": "0.00", "clicks": "540", "video_play_actions": "40753", "video_views_p100": "723", "conversion": "0", "cost_per_secondary_goal_result": null, "secondary_goal_result": null, "real_time_app_install_cost": "0.000", "ctr": "1.17", "real_time_conversion": "0", "result_rate": "1.17", "app_install": "0", "campaign_name": "Website Traffic20211020010104"}, "adgroup_id": 1714125049901106}, "emitted_at": 1685014858401} -{"stream": "ad_groups_reports_lifetime", "data": {"dimensions": {"adgroup_id": 1714124588896305}, "metrics": {"video_views_p50": "0", "secondary_goal_result_rate": null, "average_video_play": "0.00", "campaign_id": 1714124576938033, "tt_app_name": "0", "real_time_cost_per_result": "0.0000", "clicks_on_music_disc": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "video_views_p25": "0", "cost_per_1000_reached": "0.000", "profile_visits": "0", "shares": "0", "frequency": "0.00", "likes": "0", "real_time_cost_per_conversion": "0.000", "cost_per_conversion": "0.000", "comments": "0", "real_time_result_rate": "0.00", "adgroup_name": "Ad Group20211020005346", "video_watched_2s": "0", "tt_app_id": 0, "video_views_p75": "0", "promotion_type": "Website", "result": "0", "real_time_result": "0", "dpa_target_audience_type": null, "spend": "0.000", "cpm": "0.000", "real_time_app_install": "0", "impressions": "0", "follows": "0", "reach": "0", "video_watched_6s": "0", "cpc": "0.000", "placement_type": "Automatic Placement", "average_video_play_per_user": "0.00", "cost_per_result": "0.0000", "conversion_rate": "0.00", "clicks": "0", "video_play_actions": "0", "video_views_p100": "0", "conversion": "0", "cost_per_secondary_goal_result": null, "secondary_goal_result": null, "real_time_app_install_cost": "0.000", "ctr": "0.00", "real_time_conversion": "0", "result_rate": "0.00", "app_install": "0", "campaign_name": "Website Traffic20211020005342"}, "adgroup_id": 1714124588896305}, "emitted_at": 1685014858403} -{"stream": "campaigns_reports_daily", "data": {"dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-27 00:00:00"}, "metrics": {"video_views_p50": "164", "clicks_on_music_disc": "0", "likes": "18", "frequency": "1.23", "video_watched_2s": "493", "average_video_play": "1.48", "real_time_app_install_cost": "0.000", "app_install": "0", "real_time_app_install": "0", "cpm": "4.260", "video_views_p25": "355", "cpc": "0.390", "video_play_actions": "4179", "video_views_p75": "108", "shares": "0", "video_watched_6s": "132", "cost_per_1000_reached": "5.233", "clicks": "51", "spend": "20.000", "ctr": "1.09", "follows": "0", "profile_visits": "0", "reach": "3822", "average_video_play_per_user": "1.61", "video_views_p100": "76", "comments": "0", "campaign_name": "Website Traffic20211020010104", "impressions": "4696"}, "stat_time_day": "2021-10-27 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1685015395835} -{"stream": "campaigns_reports_daily", "data": {"dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-22 00:00:00"}, "metrics": {"video_views_p50": "112", "clicks_on_music_disc": "0", "likes": "17", "frequency": "1.21", "video_watched_2s": "390", "average_video_play": "1.46", "real_time_app_install_cost": "0.000", "app_install": "0", "real_time_app_install": "0", "cpm": "5.680", "video_views_p25": "277", "cpc": "0.480", "video_play_actions": "3118", "video_views_p75": "74", "shares": "0", "video_watched_6s": "92", "cost_per_1000_reached": "6.878", "clicks": "42", "spend": "20.000", "ctr": "1.19", "follows": "0", "profile_visits": "0", "reach": "2908", "average_video_play_per_user": "1.57", "video_views_p100": "59", "comments": "0", "campaign_name": "Website Traffic20211020010104", "impressions": "3520"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1685015395839} -{"stream": "campaigns_reports_daily", "data": {"dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-28 00:00:00"}, "metrics": {"video_views_p50": "144", "clicks_on_music_disc": "0", "likes": "18", "frequency": "1.22", "video_watched_2s": "471", "average_video_play": "1.42", "real_time_app_install_cost": "0.000", "app_install": "0", "real_time_app_install": "0", "cpm": "4.180", "video_views_p25": "328", "cpc": "0.420", "video_play_actions": "4253", "video_views_p75": "100", "shares": "0", "video_watched_6s": "120", "cost_per_1000_reached": "5.079", "clicks": "48", "spend": "20.000", "ctr": "1.00", "follows": "0", "profile_visits": "0", "reach": "3938", "average_video_play_per_user": "1.54", "video_views_p100": "70", "comments": "0", "campaign_name": "Website Traffic20211020010104", "impressions": "4787"}, "stat_time_day": "2021-10-28 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1685015395842} -{"stream": "campaigns_reports_lifetime", "data": {"dimensions": {"campaign_id": 1728545382536225}, "metrics": {"video_watched_6s": "402", "shares": "0", "clicks_on_music_disc": "0", "video_play_actions": "14333", "profile_visits": "0", "video_views_p50": "907", "frequency": "1.20", "app_install": "0", "impressions": "15689", "follows": "0", "reach": "13052", "spend": "60.000", "clicks": "145", "cost_per_1000_reached": "4.597", "likes": "11", "video_views_p100": "402", "ctr": "0.92", "video_watched_2s": "1364", "average_video_play": "1.26", "cpm": "3.820", "campaign_name": "CampaignVadimTraffic", "cpc": "0.410", "video_views_p75": "522", "real_time_app_install": "0", "real_time_app_install_cost": "0.000", "comments": "0", "video_views_p25": "3339", "average_video_play_per_user": "1.39"}, "campaign_id": 1728545382536225}, "emitted_at": 1685015413278} -{"stream": "campaigns_reports_lifetime", "data": {"dimensions": {"campaign_id": 1714125042508817}, "metrics": {"video_watched_6s": "1295", "shares": "0", "clicks_on_music_disc": "0", "video_play_actions": "40753", "profile_visits": "0", "video_views_p50": "1588", "frequency": "1.37", "app_install": "0", "impressions": "46116", "follows": "0", "reach": "33556", "spend": "200.000", "clicks": "540", "cost_per_1000_reached": "5.960", "likes": "263", "video_views_p100": "723", "ctr": "1.17", "video_watched_2s": "5100", "average_video_play": "1.48", "cpm": "4.340", "campaign_name": "Website Traffic20211020010104", "cpc": "0.370", "video_views_p75": "998", "real_time_app_install": "0", "real_time_app_install_cost": "0.000", "comments": "2", "video_views_p25": "3674", "average_video_play_per_user": "1.80"}, "campaign_id": 1714125042508817}, "emitted_at": 1685015413281} -{"stream": "campaigns_reports_lifetime", "data": {"dimensions": {"campaign_id": 1714124576938033}, "metrics": {"video_watched_6s": "0", "shares": "0", "clicks_on_music_disc": "0", "video_play_actions": "0", "profile_visits": "0", "video_views_p50": "0", "frequency": "0.00", "app_install": "0", "impressions": "0", "follows": "0", "reach": "0", "spend": "0.000", "clicks": "0", "cost_per_1000_reached": "0.000", "likes": "0", "video_views_p100": "0", "ctr": "0.00", "video_watched_2s": "0", "average_video_play": "0.00", "cpm": "0.000", "campaign_name": "Website Traffic20211020005342", "cpc": "0.000", "video_views_p75": "0", "real_time_app_install": "0", "real_time_app_install_cost": "0.000", "comments": "0", "video_views_p25": "0", "average_video_play_per_user": "0.00"}, "campaign_id": 1714124576938033}, "emitted_at": 1685015413284} -{"stream": "advertisers_reports_daily", "data": {"metrics": {"video_views_p100": "70", "cpm": "4.180", "impressions": "4787", "follows": "0", "profile_visits": "0", "video_watched_6s": "120", "clicks": "48", "shares": "0", "voucher_spend": "0.000", "video_views_p75": "100", "average_video_play": "1.42", "likes": "18", "spend": "20.000", "cash_spend": "20.000", "comments": "0", "cpc": "0.420", "average_video_play_per_user": "1.54", "frequency": "1.22", "ctr": "1.00", "clicks_on_music_disc": "0", "video_views_p50": "144", "video_play_actions": "4253", "app_install": "0", "video_watched_2s": "471", "real_time_app_install_cost": "0.000", "reach": "3938", "cost_per_1000_reached": "5.079", "real_time_app_install": "0", "video_views_p25": "328"}, "dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-28 00:00:00"}, "stat_time_day": "2021-10-28 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1685015970942} -{"stream": "advertisers_reports_daily", "data": {"metrics": {"video_views_p100": "65", "cpm": "4.910", "impressions": "4077", "follows": "0", "profile_visits": "0", "video_watched_6s": "124", "clicks": "67", "shares": "0", "voucher_spend": "0.000", "video_views_p75": "95", "average_video_play": "1.53", "likes": "19", "spend": "20.000", "cash_spend": "20.000", "comments": "0", "cpc": "0.300", "average_video_play_per_user": "1.65", "frequency": "1.23", "ctr": "1.64", "clicks_on_music_disc": "0", "video_views_p50": "146", "video_play_actions": "3590", "app_install": "0", "video_watched_2s": "463", "real_time_app_install_cost": "0.000", "reach": "3322", "cost_per_1000_reached": "6.020", "real_time_app_install": "0", "video_views_p25": "338"}, "dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-23 00:00:00"}, "stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1685015970946} -{"stream": "advertisers_reports_daily", "data": {"metrics": {"video_views_p100": "71", "cpm": "5.330", "impressions": "3750", "follows": "0", "profile_visits": "0", "video_watched_6s": "112", "clicks": "46", "shares": "0", "voucher_spend": "0.000", "video_views_p75": "90", "average_video_play": "1.50", "likes": "25", "spend": "20.000", "cash_spend": "20.000", "comments": "1", "cpc": "0.430", "average_video_play_per_user": "1.61", "frequency": "1.20", "ctr": "1.23", "clicks_on_music_disc": "0", "video_views_p50": "142", "video_play_actions": "3344", "app_install": "0", "video_watched_2s": "413", "real_time_app_install_cost": "0.000", "reach": "3119", "cost_per_1000_reached": "6.412", "real_time_app_install": "0", "video_views_p25": "297"}, "dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-26 00:00:00"}, "stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1685015970949} -{"stream": "advertisers_reports_lifetime", "data": {"metrics": {"cpm": "4.200", "spend": "280.000", "real_time_app_install_cost": "0.000", "video_views_p50": "2665", "comments": "2", "clicks_on_music_disc": "0", "ctr": "1.12", "video_views_p100": "1205", "video_views_p25": "7364", "cost_per_1000_reached": "5.554", "clicks": "750", "video_views_p75": "1636", "shares": "0", "app_install": "0", "impressions": "66691", "video_watched_2s": "6941", "average_video_play": "1.43", "reach": "50418", "real_time_app_install": "0", "profile_visits": "0", "frequency": "1.32", "video_watched_6s": "1838", "likes": "328", "follows": "0", "video_play_actions": "59390", "cpc": "0.370", "average_video_play_per_user": "1.68"}, "dimensions": {"advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633}, "emitted_at": 1685015990512} -{"stream": "ads_audience_reports_daily", "data": {"metrics": {"campaign_name": "Website Traffic20211020010104", "mobile_app_id": "0", "clicks": "12", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_cost_per_conversion": "0.000", "adgroup_name": "Ad Group20211020010107", "promotion_type": "Website", "cost_per_result": "0.3075", "tt_app_name": "0", "cost_per_conversion": "0.000", "result": "12", "cpm": "3.020", "real_time_conversion_rate": "0.00", "cpc": "0.310", "spend": "3.690", "real_time_cost_per_result": "0.3075", "tt_app_id": "0", "impressions": "1222", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "dpa_target_audience_type": null, "real_time_result_rate": "0.98", "conversion_rate": "0.00", "placement_type": "Automatic Placement", "adgroup_id": 1714125049901106, "conversion": "0", "ctr": "0.98", "real_time_result": "12", "result_rate": "0.98", "real_time_conversion": "0", "campaign_id": 1714125042508817}, "dimensions": {"gender": "MALE", "ad_id": 1714125051115569, "stat_time_day": "2021-10-25 00:00:00", "age": "AGE_25_34"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_25_34"}, "emitted_at": 1685015994001} -{"stream": "ads_audience_reports_daily", "data": {"metrics": {"campaign_name": "Website Traffic20211020010104", "mobile_app_id": "0", "clicks": "5", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_cost_per_conversion": "0.000", "adgroup_name": "Ad Group20211020010107", "promotion_type": "Website", "cost_per_result": "0.3540", "tt_app_name": "0", "cost_per_conversion": "0.000", "result": "5", "cpm": "5.380", "real_time_conversion_rate": "0.00", "cpc": "0.350", "spend": "1.770", "real_time_cost_per_result": "0.3540", "tt_app_id": "0", "impressions": "329", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "dpa_target_audience_type": null, "real_time_result_rate": "1.52", "conversion_rate": "0.00", "placement_type": "Automatic Placement", "adgroup_id": 1714125049901106, "conversion": "0", "ctr": "1.52", "real_time_result": "5", "result_rate": "1.52", "real_time_conversion": "0", "campaign_id": 1714125042508817}, "dimensions": {"gender": "MALE", "ad_id": 1714125051115569, "stat_time_day": "2021-10-21 00:00:00", "age": "AGE_45_54"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1685015994006} -{"stream": "ads_audience_reports_daily", "data": {"metrics": {"campaign_name": "Website Traffic20211019110444", "mobile_app_id": "0", "clicks": "0", "ad_text": "Open Source ETL", "real_time_cost_per_conversion": "0.000", "adgroup_name": "Ad Group20211019111040", "promotion_type": "Website", "cost_per_result": "0.0000", "tt_app_name": "0", "cost_per_conversion": "0.000", "result": "0", "cpm": "0.000", "real_time_conversion_rate": "0.00", "cpc": "0.000", "spend": "0.000", "real_time_cost_per_result": "0.0000", "tt_app_id": "0", "impressions": "56", "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "dpa_target_audience_type": null, "real_time_result_rate": "0.00", "conversion_rate": "0.00", "placement_type": "Automatic Placement", "adgroup_id": 1714073022392322, "conversion": "0", "ctr": "0.00", "real_time_result": "0", "result_rate": "0.00", "real_time_conversion": "0", "campaign_id": 1714073078669329}, "dimensions": {"gender": "MALE", "ad_id": 1714073085256738, "stat_time_day": "2021-10-19 00:00:00", "age": "AGE_25_34"}, "stat_time_day": "2021-10-19 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_25_34"}, "emitted_at": 1685015994012} -{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-27 00:00:00", "gender": "MALE", "age": "AGE_45_54"}, "metrics": {"cpc": "0.430", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "promotion_type": "Website", "result_rate": "1.34", "cost_per_result": "0.4280", "campaign_name": "Website Traffic20211020010104", "tt_app_name": "0", "real_time_conversion_rate": "0.00", "cpm": "5.740", "result": "5", "impressions": "373", "cost_per_conversion": "0.000", "campaign_id": 1714125042508817, "real_time_conversion": "0", "real_time_result_rate": "1.34", "real_time_result": "5", "mobile_app_id": "0", "adgroup_name": "Ad Group20211020010107", "real_time_cost_per_result": "0.4280", "spend": "2.140", "conversion": "0", "clicks": "5", "real_time_cost_per_conversion": "0.000", "ctr": "1.34", "dpa_target_audience_type": null, "tt_app_id": "0"}, "stat_time_day": "2021-10-27 00:00:00", "adgroup_id": 1714125049901106, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1685016083073} -{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-22 00:00:00", "gender": "FEMALE", "age": "AGE_35_44"}, "metrics": {"cpc": "0.480", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "promotion_type": "Website", "result_rate": "1.41", "cost_per_result": "0.4789", "campaign_name": "Website Traffic20211020010104", "tt_app_name": "0", "real_time_conversion_rate": "0.00", "cpm": "6.760", "result": "9", "impressions": "638", "cost_per_conversion": "0.000", "campaign_id": 1714125042508817, "real_time_conversion": "0", "real_time_result_rate": "1.41", "real_time_result": "9", "mobile_app_id": "0", "adgroup_name": "Ad Group20211020010107", "real_time_cost_per_result": "0.4789", "spend": "4.310", "conversion": "0", "clicks": "9", "real_time_cost_per_conversion": "0.000", "ctr": "1.41", "dpa_target_audience_type": null, "tt_app_id": "0"}, "stat_time_day": "2021-10-22 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1685016083076} -{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"adgroup_id": 1714073022392322, "stat_time_day": "2021-10-19 00:00:00", "gender": "FEMALE", "age": "AGE_35_44"}, "metrics": {"cpc": "0.000", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "promotion_type": "Website", "result_rate": "0.00", "cost_per_result": "0.0000", "campaign_name": "Website Traffic20211019110444", "tt_app_name": "0", "real_time_conversion_rate": "0.00", "cpm": "0.000", "result": "0", "impressions": "41", "cost_per_conversion": "0.000", "campaign_id": 1714073078669329, "real_time_conversion": "0", "real_time_result_rate": "0.00", "real_time_result": "0", "mobile_app_id": "0", "adgroup_name": "Ad Group20211019111040", "real_time_cost_per_result": "0.0000", "spend": "0.000", "conversion": "0", "clicks": "0", "real_time_cost_per_conversion": "0.000", "ctr": "0.00", "dpa_target_audience_type": null, "tt_app_id": "0"}, "stat_time_day": "2021-10-19 00:00:00", "adgroup_id": 1714073022392322, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1685016083078} -{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "metrics": {"cpm": "4.100", "cpc": "0.310", "clicks": "65", "spend": "20.000", "ctr": "1.33", "campaign_name": "Website Traffic20211019110444", "impressions": "4874"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1685016192929} -{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "metrics": {"cpm": "0.000", "cpc": "0.000", "clicks": "0", "spend": "0.000", "ctr": "0.00", "campaign_name": "Website Traffic20211019110444", "impressions": "12"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1685016192931} -{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "metrics": {"cpm": "0.000", "cpc": "0.000", "clicks": "0", "spend": "0.000", "ctr": "0.00", "campaign_name": "Website Traffic20211019110444", "impressions": "0"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1685016192934} -{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-19 00:00:00", "gender": "MALE", "age": "AGE_13_17"}, "metrics": {"cpc": "0.320", "cpm": "3.930", "impressions": "1814", "spend": "7.130", "clicks": "22", "ctr": "1.21"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1685016247542} -{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-20 00:00:00", "gender": "MALE", "age": "AGE_13_17"}, "metrics": {"cpc": "0.000", "cpm": "0.000", "impressions": "4", "spend": "0.000", "clicks": "0", "ctr": "0.00"}, "stat_time_day": "2021-10-20 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1685016247545} -{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-19 00:00:00", "gender": "FEMALE", "age": "AGE_13_17"}, "metrics": {"cpc": "0.290", "cpm": "3.880", "impressions": "2146", "spend": "8.320", "clicks": "29", "ctr": "1.35"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1685016247547} -{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.580", "impressions": "6897", "clicks": "87", "spend": "31.560", "cpc": "0.360", "ctr": "1.26"}, "dimensions": {"gender": "MALE", "age": "AGE_35_44", "advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1685016272174} -{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.280", "impressions": "3450", "clicks": "39", "spend": "14.770", "cpc": "0.380", "ctr": "1.13"}, "dimensions": {"gender": "MALE", "age": "AGE_45_54", "advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1685016272177} -{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "3.920", "impressions": "1818", "clicks": "22", "spend": "7.130", "cpc": "0.320", "ctr": "1.21"}, "dimensions": {"gender": "MALE", "age": "AGE_13_17", "advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1685016272180} +{"stream": "advertisers", "data": {"description": "https://", "contacter": "Ai***te", "license_city": null, "timezone": "Etc/GMT+8", "promotion_center_province": null, "address": "350 29th avenue, San Francisco", "country": "US", "brand": null, "status": "STATUS_ENABLE", "role": "ROLE_ADVERTISER", "rejection_reason": "", "email": "i***************@**********", "license_province": null, "industry": "291905", "license_no": "", "name": "Airbyte0830", "create_time": 1630335591, "promotion_area": "0", "advertiser_account_type": "AUCTION", "cellphone_number": "+13477****53", "company": "Airbyte", "advertiser_id": 7002238017842757633, "promotion_center_city": null, "telephone_number": "+14156****85", "display_timezone": "America/Los_Angeles", "license_url": null, "currency": "USD", "language": "en", "balance": 10}, "emitted_at": 1691143342127} +{"stream": "ads", "data": {"app_name": "", "fallback_type": "UNSET", "advertiser_id": 7002238017842757633, "landing_page_urls": null, "brand_safety_postbid_partner": "UNSET", "campaign_id": 1728545382536225, "adgroup_id": 1728545385226289, "campaign_name": "CampaignVadimTraffic", "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "ad_texts": null, "create_time": "2022-03-28 12:09:09", "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "playable_url": "", "click_tracking_url": null, "is_aco": false, "impression_tracking_url": null, "call_to_action_id": "7080120957230238722", "modify_time": "2022-03-28 21:34:26", "image_ids": ["v0201/7f371ff6f0764f8b8ef4f37d7b980d50"], "operation_status": "ENABLE", "viewability_postbid_partner": "UNSET", "identity_type": "CUSTOMIZED_USER", "creative_type": null, "deeplink": "", "adgroup_name": "AdGroupVadim", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "display_name": "airbyte", "viewability_vast_url": null, "brand_safety_vast_url": null, "identity_id": "7080121820963422209", "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "deeplink_type": "NORMAL", "creative_authorized": false, "optimization_event": null, "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", "ad_format": "SINGLE_VIDEO", "ad_id": 1728545390695442, "ad_text": "Open-source\ndata integration for modern data teams", "card_id": null, "landing_page_url": "https://airbyte.com", "vast_moat_enabled": false, "music_id": null, "is_new_structure": true}, "emitted_at": 1691143343208} +{"stream": "ads", "data": {"app_name": "", "fallback_type": "UNSET", "advertiser_id": 7002238017842757633, "landing_page_urls": null, "brand_safety_postbid_partner": "UNSET", "campaign_id": 1714125042508817, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104", "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "ad_texts": null, "create_time": "2021-10-20 08:04:06", "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "playable_url": "", "click_tracking_url": null, "is_aco": false, "impression_tracking_url": null, "call_to_action_id": null, "modify_time": "2021-10-21 17:50:11", "image_ids": ["v0201/8f77082a1f3c40c586f8282356490c58"], "call_to_action": "LEARN_MORE", "operation_status": "ENABLE", "viewability_postbid_partner": "UNSET", "identity_type": "UNSET", "creative_type": null, "deeplink": "", "adgroup_name": "Ad Group20211020010107", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "display_name": "Airbyte", "viewability_vast_url": null, "brand_safety_vast_url": null, "identity_id": "", "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "deeplink_type": "NORMAL", "creative_authorized": true, "optimization_event": null, "video_id": "v10033g50000c5nsqcbc77ubdn136b70", "ad_format": "SINGLE_VIDEO", "ad_id": 1714125051115569, "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "card_id": null, "landing_page_url": "https://airbyte.io", "vast_moat_enabled": false, "music_id": null, "is_new_structure": true}, "emitted_at": 1691143343209} +{"stream": "ads", "data": {"app_name": "", "fallback_type": "UNSET", "advertiser_id": 7002238017842757633, "landing_page_urls": null, "brand_safety_postbid_partner": "UNSET", "campaign_id": 1714124576938033, "adgroup_id": 1714124588896305, "campaign_name": "Website Traffic20211020005342", "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d4ce78859d72d409eb82d", "ad_texts": null, "create_time": "2021-10-20 07:56:39", "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202110205d0d4ce78859d72d409eb82d", "playable_url": "", "click_tracking_url": null, "is_aco": false, "impression_tracking_url": null, "call_to_action_id": null, "modify_time": "2021-10-20 08:05:12", "image_ids": ["ad-site-i18n-sg/202110195d0db51e12a222ee4334a396"], "call_to_action": "LEARN_MORE", "operation_status": "DISABLE", "viewability_postbid_partner": "UNSET", "identity_type": "UNSET", "creative_type": null, "deeplink": "", "adgroup_name": "Ad Group20211020005346", "ad_name": "1200x1200 logo_1634667070143.png_2021-10-20 10:54:51", "display_name": "Airbyte", "viewability_vast_url": null, "brand_safety_vast_url": null, "identity_id": "", "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "deeplink_type": "NORMAL", "creative_authorized": false, "optimization_event": null, "video_id": null, "ad_format": "SINGLE_IMAGE", "ad_id": 1714124564763650, "ad_text": "Airbyte - open source data portability platform - from nywhere to anywhere!", "card_id": null, "landing_page_url": "https://airbyte.io", "vast_moat_enabled": false, "music_id": null, "is_new_structure": true}, "emitted_at": 1691143343210} +{"stream": "ad_groups", "data": {"secondary_optimization_event": null, "operating_systems": [], "interest_keyword_ids": [], "campaign_name": "CampaignVadimTraffic", "pixel_id": null, "device_price_ranges": [], "excluded_custom_actions": [], "skip_learning_phase": 0, "delivery_mode": null, "pacing": "PACING_MODE_SMOOTH", "languages": [], "age_groups": ["AGE_25_34", "AGE_35_44"], "adgroup_app_profile_page_state": null, "search_result_enabled": false, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "video_download_disabled": false, "billing_event": "CPC", "app_download_url": null, "conversion_window": null, "optimization_goal": "CLICK", "adgroup_id": 1728545385226289, "excluded_audience_ids": [], "auto_targeting_enabled": false, "gender": "GENDER_UNLIMITED", "is_new_structure": true, "brand_safety_type": "NO_BRAND_SAFETY", "adgroup_name": "AdGroupVadim", "budget": 20, "schedule_end_time": "2032-03-25 13:02:23", "statistic_type": null, "schedule_start_time": "2022-03-28 13:02:23", "schedule_type": "SCHEDULE_FROM_NOW", "category_exclusion_ids": [], "operation_status": "ENABLE", "rf_estimated_cpr": null, "network_types": [], "deep_cpa_bid": 0, "frequency_schedule": null, "bid_display_mode": "CPMV", "inventory_filter_enabled": false, "next_day_retention": null, "rf_estimated_frequency": null, "share_disabled": false, "bid_price": 0, "keywords": null, "promotion_type": "WEBSITE", "comment_disabled": false, "budget_mode": "BUDGET_MODE_DAY", "creative_material_mode": "CUSTOM", "category_id": 0, "deep_bid_type": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "optimization_event": null, "location_ids": [6252001], "actions": [], "app_type": null, "brand_safety_partner": null, "bid_type": "BID_TYPE_NO_BID", "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "campaign_id": 1728545382536225, "modify_time": "2022-03-31 08:13:30", "rf_purchased_type": null, "audience_ids": [], "advertiser_id": 7002238017842757633, "conversion_bid_price": 0, "app_id": null, "is_hfss": false, "feed_type": null, "device_model_ids": [], "schedule_infos": null, "included_custom_actions": [], "ios14_quota_type": "UNOCCUPIED", "scheduled_budget": 0, "frequency": null, "is_smart_performance_campaign": false, "create_time": "2022-03-28 12:09:07", "interest_category_ids": [15], "purchased_impression": null, "purchased_reach": null}, "emitted_at": 1691143344341} +{"stream": "ad_groups", "data": {"secondary_optimization_event": null, "operating_systems": [], "interest_keyword_ids": [], "campaign_name": "Website Traffic20211020010104", "pixel_id": null, "device_price_ranges": [], "excluded_custom_actions": [], "skip_learning_phase": 0, "delivery_mode": null, "pacing": "PACING_MODE_SMOOTH", "languages": ["en"], "age_groups": ["AGE_25_34", "AGE_35_44", "AGE_45_54"], "adgroup_app_profile_page_state": null, "search_result_enabled": false, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "video_download_disabled": false, "billing_event": "CPC", "app_download_url": null, "conversion_window": null, "optimization_goal": "CLICK", "adgroup_id": 1714125049901106, "excluded_audience_ids": [], "auto_targeting_enabled": false, "gender": "GENDER_UNLIMITED", "is_new_structure": true, "brand_safety_type": "NO_BRAND_SAFETY", "adgroup_name": "Ad Group20211020010107", "budget": 20, "schedule_end_time": "2021-10-31 09:01:07", "statistic_type": null, "schedule_start_time": "2021-10-20 09:01:07", "schedule_type": "SCHEDULE_START_END", "category_exclusion_ids": [], "operation_status": "ENABLE", "rf_estimated_cpr": null, "network_types": [], "deep_cpa_bid": 0, "frequency_schedule": null, "bid_display_mode": "CPMV", "inventory_filter_enabled": false, "next_day_retention": null, "rf_estimated_frequency": null, "share_disabled": false, "bid_price": 0, "keywords": null, "promotion_type": "WEBSITE", "comment_disabled": false, "budget_mode": "BUDGET_MODE_DAY", "creative_material_mode": "CUSTOM", "category_id": 0, "deep_bid_type": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "optimization_event": null, "location_ids": [6252001], "actions": [], "app_type": null, "brand_safety_partner": null, "bid_type": "BID_TYPE_NO_BID", "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "campaign_id": 1714125042508817, "modify_time": "2022-03-24 12:06:54", "rf_purchased_type": null, "audience_ids": [], "advertiser_id": 7002238017842757633, "conversion_bid_price": 0, "app_id": null, "is_hfss": false, "feed_type": null, "device_model_ids": [], "schedule_infos": null, "included_custom_actions": [], "ios14_quota_type": "UNOCCUPIED", "scheduled_budget": 0, "frequency": null, "is_smart_performance_campaign": false, "create_time": "2021-10-20 08:04:05", "interest_category_ids": [], "purchased_impression": null, "purchased_reach": null}, "emitted_at": 1691143344343} +{"stream": "ad_groups", "data": {"secondary_optimization_event": null, "operating_systems": [], "interest_keyword_ids": [], "campaign_name": "Website Traffic20211020005342", "pixel_id": null, "device_price_ranges": [], "excluded_custom_actions": [], "skip_learning_phase": 0, "delivery_mode": null, "pacing": "PACING_MODE_SMOOTH", "languages": [], "age_groups": null, "adgroup_app_profile_page_state": null, "search_result_enabled": false, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "video_download_disabled": false, "billing_event": "CPC", "app_download_url": null, "conversion_window": null, "optimization_goal": "CLICK", "adgroup_id": 1714124588896305, "excluded_audience_ids": [], "auto_targeting_enabled": false, "gender": "GENDER_UNLIMITED", "is_new_structure": true, "brand_safety_type": "NO_BRAND_SAFETY", "adgroup_name": "Ad Group20211020005346", "budget": 20, "schedule_end_time": "2021-10-31 08:53:46", "statistic_type": null, "schedule_start_time": "2021-10-20 08:53:46", "schedule_type": "SCHEDULE_START_END", "category_exclusion_ids": [], "operation_status": "DISABLE", "rf_estimated_cpr": null, "network_types": [], "deep_cpa_bid": 0, "frequency_schedule": null, "bid_display_mode": "CPMV", "inventory_filter_enabled": false, "next_day_retention": null, "rf_estimated_frequency": null, "share_disabled": false, "bid_price": 0, "keywords": null, "promotion_type": "WEBSITE", "comment_disabled": false, "budget_mode": "BUDGET_MODE_DAY", "creative_material_mode": "CUSTOM", "category_id": 0, "deep_bid_type": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "optimization_event": null, "location_ids": [6252001], "actions": [], "app_type": null, "brand_safety_partner": null, "bid_type": "BID_TYPE_NO_BID", "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_HELO", "PLACEMENT_PANGLE"], "campaign_id": 1714124576938033, "modify_time": "2021-10-20 08:08:14", "rf_purchased_type": null, "audience_ids": [], "advertiser_id": 7002238017842757633, "conversion_bid_price": 0, "app_id": null, "is_hfss": false, "feed_type": null, "device_model_ids": [], "schedule_infos": null, "included_custom_actions": [], "ios14_quota_type": "UNOCCUPIED", "scheduled_budget": 0, "frequency": null, "is_smart_performance_campaign": false, "create_time": "2021-10-20 07:56:39", "interest_category_ids": [], "purchased_impression": null, "purchased_reach": null}, "emitted_at": 1691143344345} +{"stream": "campaigns", "data": {"budget": 0, "campaign_type": "REGULAR_CAMPAIGN", "is_smart_performance_campaign": false, "budget_mode": "BUDGET_MODE_INFINITE", "campaign_name": "CampaignVadimTraffic", "deep_bid_type": null, "objective": "LANDING_PAGE", "operation_status": "DISABLE", "objective_type": "TRAFFIC", "modify_time": "2022-03-30 21:23:52", "create_time": "2022-03-28 12:09:05", "roas_bid": 0, "advertiser_id": 7002238017842757633, "secondary_status": "CAMPAIGN_STATUS_DISABLE", "campaign_id": 1728545382536225, "is_new_structure": true}, "emitted_at": 1691143345193} +{"stream": "campaigns", "data": {"budget": 0, "campaign_type": "REGULAR_CAMPAIGN", "is_smart_performance_campaign": false, "budget_mode": "BUDGET_MODE_INFINITE", "campaign_name": "Website Traffic20211020010104", "deep_bid_type": null, "objective": "LANDING_PAGE", "operation_status": "DISABLE", "objective_type": "TRAFFIC", "modify_time": "2022-03-24 12:08:29", "create_time": "2021-10-20 08:04:04", "roas_bid": 0, "advertiser_id": 7002238017842757633, "secondary_status": "CAMPAIGN_STATUS_DISABLE", "campaign_id": 1714125042508817, "is_new_structure": true}, "emitted_at": 1691143345193} +{"stream": "campaigns", "data": {"budget": 0, "campaign_type": "REGULAR_CAMPAIGN", "is_smart_performance_campaign": false, "budget_mode": "BUDGET_MODE_INFINITE", "campaign_name": "Website Traffic20211020005342", "deep_bid_type": null, "objective": "LANDING_PAGE", "operation_status": "DISABLE", "objective_type": "TRAFFIC", "modify_time": "2021-10-20 08:01:18", "create_time": "2021-10-20 07:56:38", "roas_bid": 0, "advertiser_id": 7002238017842757633, "secondary_status": "CAMPAIGN_STATUS_DISABLE", "campaign_id": 1714124576938033, "is_new_structure": true}, "emitted_at": 1691143345194} +{"stream": "advertiser_ids", "data": {"advertiser_id": 7001035076276387841, "advertiser_name": "Airbyte0827"}, "emitted_at": 1691143345771} +{"stream": "advertiser_ids", "data": {"advertiser_id": 7001040009704833026, "advertiser_name": "Airbyte08270"}, "emitted_at": 1691143345772} +{"stream": "advertiser_ids", "data": {"advertiser_id": 7002238017842757633, "advertiser_name": "Airbyte0830"}, "emitted_at": 1691143345772} +{"stream": "ads_reports_daily", "data": {"metrics": {"mobile_app_id": "0", "result": "69", "follows": "0", "cost_per_secondary_goal_result": null, "ctr": "1.18", "promotion_type": "Website", "clicks_on_music_disc": "0", "real_time_conversion": "0", "real_time_result": "69", "total_complete_payment_rate": "0.000", "frequency": "1.21", "placement_type": "Automatic Placement", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "shares": "0", "video_views_p75": "140", "spend": "20.000", "cost_per_1000_reached": "4.161", "likes": "36", "video_watched_2s": "686", "video_views_p50": "214", "complete_payment": "0", "cpc": "0.290", "vta_purchase": "0", "real_time_result_rate": "1.18", "real_time_conversion_rate": "0.00", "comments": "0", "conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play": "1.52", "onsite_shopping": "0", "campaign_id": 1714125042508817, "cta_purchase": "0", "video_views_p25": "513", "video_watched_6s": "180", "conversion_rate": "0.00", "cost_per_conversion": "0.000", "cpm": "3.430", "secondary_goal_result_rate": null, "impressions": "5830", "video_views_p100": "92", "total_purchase_value": "0.000", "vta_conversion": "0", "reach": "4806", "adgroup_id": 1714125049901106, "profile_visits": "0", "dpa_target_audience_type": null, "clicks": "69", "real_time_app_install_cost": "0.000", "real_time_cost_per_result": "0.2899", "value_per_complete_payment": "0.000", "secondary_goal_result": null, "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "average_video_play_per_user": "1.64", "real_time_app_install": "0", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "video_play_actions": "5173", "cost_per_result": "0.2899", "app_install": "0", "real_time_cost_per_conversion": "0.000", "total_onsite_shopping_value": "0.000", "cta_conversion": "0", "total_pageview": "0", "result_rate": "1.18"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1691143872903} +{"stream": "ads_reports_daily", "data": {"metrics": {"mobile_app_id": "0", "result": "53", "follows": "0", "cost_per_secondary_goal_result": null, "ctr": "1.41", "promotion_type": "Website", "clicks_on_music_disc": "0", "real_time_conversion": "0", "real_time_result": "53", "total_complete_payment_rate": "0.000", "frequency": "1.20", "placement_type": "Automatic Placement", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "shares": "0", "video_views_p75": "74", "spend": "20.000", "cost_per_1000_reached": "6.382", "likes": "36", "video_watched_2s": "408", "video_views_p50": "130", "complete_payment": "0", "cpc": "0.380", "vta_purchase": "0", "real_time_result_rate": "1.41", "real_time_conversion_rate": "0.00", "comments": "1", "conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play": "1.45", "onsite_shopping": "0", "campaign_id": 1714125042508817, "cta_purchase": "0", "video_views_p25": "295", "video_watched_6s": "106", "conversion_rate": "0.00", "cost_per_conversion": "0.000", "cpm": "5.310", "secondary_goal_result_rate": null, "impressions": "3765", "video_views_p100": "52", "total_purchase_value": "0.000", "vta_conversion": "0", "reach": "3134", "adgroup_id": 1714125049901106, "profile_visits": "0", "dpa_target_audience_type": null, "clicks": "53", "real_time_app_install_cost": "0.000", "real_time_cost_per_result": "0.3774", "value_per_complete_payment": "0.000", "secondary_goal_result": null, "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "average_video_play_per_user": "1.55", "real_time_app_install": "0", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "video_play_actions": "3344", "cost_per_result": "0.3774", "app_install": "0", "real_time_cost_per_conversion": "0.000", "total_onsite_shopping_value": "0.000", "cta_conversion": "0", "total_pageview": "0", "result_rate": "1.41"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1691143872907} +{"stream": "ads_reports_daily", "data": {"metrics": {"mobile_app_id": "0", "result": "46", "follows": "0", "cost_per_secondary_goal_result": null, "ctr": "1.23", "promotion_type": "Website", "clicks_on_music_disc": "0", "real_time_conversion": "0", "real_time_result": "46", "total_complete_payment_rate": "0.000", "frequency": "1.20", "placement_type": "Automatic Placement", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "shares": "0", "video_views_p75": "90", "spend": "20.000", "cost_per_1000_reached": "6.412", "likes": "25", "video_watched_2s": "413", "video_views_p50": "142", "complete_payment": "0", "cpc": "0.430", "vta_purchase": "0", "real_time_result_rate": "1.23", "real_time_conversion_rate": "0.00", "comments": "1", "conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play": "1.50", "onsite_shopping": "0", "campaign_id": 1714125042508817, "cta_purchase": "0", "video_views_p25": "297", "video_watched_6s": "112", "conversion_rate": "0.00", "cost_per_conversion": "0.000", "cpm": "5.330", "secondary_goal_result_rate": null, "impressions": "3750", "video_views_p100": "71", "total_purchase_value": "0.000", "vta_conversion": "0", "reach": "3119", "adgroup_id": 1714125049901106, "profile_visits": "0", "dpa_target_audience_type": null, "clicks": "46", "real_time_app_install_cost": "0.000", "real_time_cost_per_result": "0.4348", "value_per_complete_payment": "0.000", "secondary_goal_result": null, "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "average_video_play_per_user": "1.61", "real_time_app_install": "0", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "video_play_actions": "3344", "cost_per_result": "0.4348", "app_install": "0", "real_time_cost_per_conversion": "0.000", "total_onsite_shopping_value": "0.000", "cta_conversion": "0", "total_pageview": "0", "result_rate": "1.23"}, "dimensions": {"stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1691143872911} +{"stream": "ads_reports_lifetime", "data": {"dimensions": {"ad_id": 1728545390695442}, "metrics": {"app_install": "0", "average_video_play": "1.26", "complete_payment": "0", "video_watched_2s": "1364", "clicks": "145", "clicks_on_music_disc": "0", "total_onsite_shopping_value": "0.000", "real_time_app_install_cost": "0.000", "video_views_p100": "402", "result_rate": "0.92", "result": "145", "value_per_complete_payment": "0.000", "secondary_goal_result_rate": null, "ctr": "0.92", "total_pageview": "0", "cpc": "0.410", "campaign_name": "CampaignVadimTraffic", "conversion": "0", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "real_time_cost_per_result": "0.4138", "video_watched_6s": "402", "cost_per_conversion": "0.000", "follows": "0", "comments": "0", "impressions": "15689", "cta_purchase": "0", "dpa_target_audience_type": null, "total_complete_payment_rate": "0.000", "promotion_type": "Website", "video_play_actions": "14333", "cta_conversion": "0", "real_time_result": "145", "tt_app_name": "0", "mobile_app_id": "0", "spend": "60.000", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "real_time_conversion_rate": "0.00", "cpm": "3.820", "shares": "0", "frequency": "1.20", "reach": "13052", "adgroup_id": 1728545385226289, "video_views_p50": "907", "likes": "11", "vta_purchase": "0", "onsite_shopping": "0", "conversion_rate": "0.00", "real_time_app_install": "0", "total_purchase_value": "0.000", "average_video_play_per_user": "1.39", "cost_per_secondary_goal_result": null, "adgroup_name": "AdGroupVadim", "campaign_id": 1728545382536225, "real_time_result_rate": "0.92", "cost_per_1000_reached": "4.597", "video_views_p75": "522", "ad_text": "Open-source\ndata integration for modern data teams", "secondary_goal_result": null, "profile_visits": "0", "vta_conversion": "0", "placement_type": "Automatic Placement", "video_views_p25": "3339", "real_time_conversion": "0", "cost_per_result": "0.4138"}, "ad_id": 1728545390695442}, "emitted_at": 1691143894042} +{"stream": "ads_reports_lifetime", "data": {"dimensions": {"ad_id": 1714125051115569}, "metrics": {"app_install": "0", "average_video_play": "1.48", "complete_payment": "0", "video_watched_2s": "5100", "clicks": "540", "clicks_on_music_disc": "0", "total_onsite_shopping_value": "0.000", "real_time_app_install_cost": "0.000", "video_views_p100": "723", "result_rate": "1.17", "result": "540", "value_per_complete_payment": "0.000", "secondary_goal_result_rate": null, "ctr": "1.17", "total_pageview": "0", "cpc": "0.370", "campaign_name": "Website Traffic20211020010104", "conversion": "0", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "real_time_cost_per_result": "0.3704", "video_watched_6s": "1295", "cost_per_conversion": "0.000", "follows": "0", "comments": "2", "impressions": "46116", "cta_purchase": "0", "dpa_target_audience_type": null, "total_complete_payment_rate": "0.000", "promotion_type": "Website", "video_play_actions": "40753", "cta_conversion": "0", "real_time_result": "540", "tt_app_name": "0", "mobile_app_id": "0", "spend": "200.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "real_time_conversion_rate": "0.00", "cpm": "4.340", "shares": "0", "frequency": "1.37", "reach": "33556", "adgroup_id": 1714125049901106, "video_views_p50": "1588", "likes": "263", "vta_purchase": "0", "onsite_shopping": "0", "conversion_rate": "0.00", "real_time_app_install": "0", "total_purchase_value": "0.000", "average_video_play_per_user": "1.80", "cost_per_secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "campaign_id": 1714125042508817, "real_time_result_rate": "1.17", "cost_per_1000_reached": "5.960", "video_views_p75": "998", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "secondary_goal_result": null, "profile_visits": "0", "vta_conversion": "0", "placement_type": "Automatic Placement", "video_views_p25": "3674", "real_time_conversion": "0", "cost_per_result": "0.3704"}, "ad_id": 1714125051115569}, "emitted_at": 1691143894045} +{"stream": "ads_reports_lifetime", "data": {"dimensions": {"ad_id": 1714124564763650}, "metrics": {"app_install": "0", "average_video_play": "0.00", "complete_payment": "0", "video_watched_2s": "0", "clicks": "0", "clicks_on_music_disc": "0", "total_onsite_shopping_value": "0.000", "real_time_app_install_cost": "0.000", "video_views_p100": "0", "result_rate": "0.00", "result": "0", "value_per_complete_payment": "0.000", "secondary_goal_result_rate": null, "ctr": "0.00", "total_pageview": "0", "cpc": "0.000", "campaign_name": "Website Traffic20211020005342", "conversion": "0", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "real_time_cost_per_result": "0.0000", "video_watched_6s": "0", "cost_per_conversion": "0.000", "follows": "0", "comments": "0", "impressions": "0", "cta_purchase": "0", "dpa_target_audience_type": null, "total_complete_payment_rate": "0.000", "promotion_type": "Website", "video_play_actions": "0", "cta_conversion": "0", "real_time_result": "0", "tt_app_name": "0", "mobile_app_id": "0", "spend": "0.000", "ad_name": "1200x1200 logo_1634667070143.png_2021-10-20 10:54:51", "real_time_conversion_rate": "0.00", "cpm": "0.000", "shares": "0", "frequency": "0.00", "reach": "0", "adgroup_id": 1714124588896305, "video_views_p50": "0", "likes": "0", "vta_purchase": "0", "onsite_shopping": "0", "conversion_rate": "0.00", "real_time_app_install": "0", "total_purchase_value": "0.000", "average_video_play_per_user": "0.00", "cost_per_secondary_goal_result": null, "adgroup_name": "Ad Group20211020005346", "campaign_id": 1714124576938033, "real_time_result_rate": "0.00", "cost_per_1000_reached": "0.000", "video_views_p75": "0", "ad_text": "Airbyte - open source data portability platform - from nywhere to anywhere!", "secondary_goal_result": null, "profile_visits": "0", "vta_conversion": "0", "placement_type": "Automatic Placement", "video_views_p25": "0", "real_time_conversion": "0", "cost_per_result": "0.0000"}, "ad_id": 1714124564763650}, "emitted_at": 1691143894049} +{"stream": "ad_groups_reports_daily", "data": {"metrics": {"conversion": "0", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.000", "comments": "1", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_result": "0.3774", "cost_per_1000_reached": "6.382", "cpc": "0.380", "average_video_play_per_user": "1.55", "likes": "36", "real_time_conversion_rate": "0.00", "dpa_target_audience_type": null, "mobile_app_id": "0", "video_views_p25": "295", "video_watched_2s": "408", "cpm": "5.310", "app_install": "0", "frequency": "1.20", "adgroup_name": "Ad Group20211020010107", "clicks_on_music_disc": "0", "real_time_app_install_cost": "0.000", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_views_p75": "74", "real_time_app_install": "0", "shares": "0", "secondary_goal_result": null, "promotion_type": "Website", "clicks": "53", "cost_per_result": "0.3774", "result_rate": "1.41", "video_views_p50": "130", "video_play_actions": "3344", "placement_type": "Automatic Placement", "real_time_result_rate": "1.41", "cost_per_secondary_goal_result": null, "real_time_result": "53", "reach": "3134", "video_watched_6s": "106", "average_video_play": "1.45", "tt_app_name": "0", "impressions": "3765", "follows": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "ctr": "1.41", "secondary_goal_result_rate": null, "profile_visits": "0", "result": "53", "video_views_p100": "52", "spend": "20.000"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1691144407230} +{"stream": "ad_groups_reports_daily", "data": {"metrics": {"conversion": "0", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.000", "comments": "0", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_result": "0.2899", "cost_per_1000_reached": "4.161", "cpc": "0.290", "average_video_play_per_user": "1.64", "likes": "36", "real_time_conversion_rate": "0.00", "dpa_target_audience_type": null, "mobile_app_id": "0", "video_views_p25": "513", "video_watched_2s": "686", "cpm": "3.430", "app_install": "0", "frequency": "1.21", "adgroup_name": "Ad Group20211020010107", "clicks_on_music_disc": "0", "real_time_app_install_cost": "0.000", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_views_p75": "140", "real_time_app_install": "0", "shares": "0", "secondary_goal_result": null, "promotion_type": "Website", "clicks": "69", "cost_per_result": "0.2899", "result_rate": "1.18", "video_views_p50": "214", "video_play_actions": "5173", "placement_type": "Automatic Placement", "real_time_result_rate": "1.18", "cost_per_secondary_goal_result": null, "real_time_result": "69", "reach": "4806", "video_watched_6s": "180", "average_video_play": "1.52", "tt_app_name": "0", "impressions": "5830", "follows": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "ctr": "1.18", "secondary_goal_result_rate": null, "profile_visits": "0", "result": "69", "video_views_p100": "92", "spend": "20.000"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1691144407234} +{"stream": "ad_groups_reports_daily", "data": {"metrics": {"conversion": "0", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.000", "comments": "0", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_result": "0.5000", "cost_per_1000_reached": "5.523", "cpc": "0.500", "average_video_play_per_user": "1.50", "likes": "13", "real_time_conversion_rate": "0.00", "dpa_target_audience_type": null, "mobile_app_id": "0", "video_views_p25": "306", "video_watched_2s": "436", "cpm": "4.550", "app_install": "0", "frequency": "1.21", "adgroup_name": "Ad Group20211020010107", "clicks_on_music_disc": "0", "real_time_app_install_cost": "0.000", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_views_p75": "85", "real_time_app_install": "0", "shares": "0", "secondary_goal_result": null, "promotion_type": "Website", "clicks": "40", "cost_per_result": "0.5000", "result_rate": "0.91", "video_views_p50": "130", "video_play_actions": "3852", "placement_type": "Automatic Placement", "real_time_result_rate": "0.91", "cost_per_secondary_goal_result": null, "real_time_result": "40", "reach": "3621", "video_watched_6s": "104", "average_video_play": "1.41", "tt_app_name": "0", "impressions": "4394", "follows": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "ctr": "0.91", "secondary_goal_result_rate": null, "profile_visits": "0", "result": "40", "video_views_p100": "66", "spend": "20.000"}, "dimensions": {"stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1691144407237} +{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"secondary_goal_result_rate": null, "follows": "0", "dpa_target_audience_type": null, "campaign_name": "CampaignVadimTraffic", "video_views_p75": "522", "video_views_p25": "3339", "real_time_app_install_cost": "0.000", "cost_per_conversion": "0.000", "placement_type": "Automatic Placement", "tt_app_name": "0", "spend": "60.000", "clicks_on_music_disc": "0", "cost_per_secondary_goal_result": null, "result": "145", "app_install": "0", "video_views_p100": "402", "shares": "0", "secondary_goal_result": null, "profile_visits": "0", "mobile_app_id": "0", "average_video_play_per_user": "1.39", "average_video_play": "1.26", "conversion_rate": "0.00", "real_time_result": "145", "frequency": "1.20", "real_time_conversion_rate": "0.00", "cpm": "3.820", "adgroup_name": "AdGroupVadim", "cpc": "0.410", "reach": "13052", "campaign_id": 1728545382536225, "video_watched_6s": "402", "cost_per_result": "0.4138", "result_rate": "0.92", "video_watched_2s": "1364", "real_time_conversion": "0", "real_time_cost_per_result": "0.4138", "real_time_result_rate": "0.92", "real_time_app_install": "0", "tt_app_id": 0, "promotion_type": "Website", "video_play_actions": "14333", "cost_per_1000_reached": "4.597", "impressions": "15689", "video_views_p50": "907", "comments": "0", "likes": "11", "conversion": "0", "real_time_cost_per_conversion": "0.000", "clicks": "145", "ctr": "0.92"}, "dimensions": {"adgroup_id": 1728545385226289}, "adgroup_id": 1728545385226289}, "emitted_at": 1691144426151} +{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"secondary_goal_result_rate": null, "follows": "0", "dpa_target_audience_type": null, "campaign_name": "Website Traffic20211020010104", "video_views_p75": "998", "video_views_p25": "3674", "real_time_app_install_cost": "0.000", "cost_per_conversion": "0.000", "placement_type": "Automatic Placement", "tt_app_name": "0", "spend": "200.000", "clicks_on_music_disc": "0", "cost_per_secondary_goal_result": null, "result": "540", "app_install": "0", "video_views_p100": "723", "shares": "0", "secondary_goal_result": null, "profile_visits": "0", "mobile_app_id": "0", "average_video_play_per_user": "1.80", "average_video_play": "1.48", "conversion_rate": "0.00", "real_time_result": "540", "frequency": "1.37", "real_time_conversion_rate": "0.00", "cpm": "4.340", "adgroup_name": "Ad Group20211020010107", "cpc": "0.370", "reach": "33556", "campaign_id": 1714125042508817, "video_watched_6s": "1295", "cost_per_result": "0.3704", "result_rate": "1.17", "video_watched_2s": "5100", "real_time_conversion": "0", "real_time_cost_per_result": "0.3704", "real_time_result_rate": "1.17", "real_time_app_install": "0", "tt_app_id": 0, "promotion_type": "Website", "video_play_actions": "40753", "cost_per_1000_reached": "5.960", "impressions": "46116", "video_views_p50": "1588", "comments": "2", "likes": "263", "conversion": "0", "real_time_cost_per_conversion": "0.000", "clicks": "540", "ctr": "1.17"}, "dimensions": {"adgroup_id": 1714125049901106}, "adgroup_id": 1714125049901106}, "emitted_at": 1691144426155} +{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"secondary_goal_result_rate": null, "follows": "0", "dpa_target_audience_type": null, "campaign_name": "Website Traffic20211020005342", "video_views_p75": "0", "video_views_p25": "0", "real_time_app_install_cost": "0.000", "cost_per_conversion": "0.000", "placement_type": "Automatic Placement", "tt_app_name": "0", "spend": "0.000", "clicks_on_music_disc": "0", "cost_per_secondary_goal_result": null, "result": "0", "app_install": "0", "video_views_p100": "0", "shares": "0", "secondary_goal_result": null, "profile_visits": "0", "mobile_app_id": "0", "average_video_play_per_user": "0.00", "average_video_play": "0.00", "conversion_rate": "0.00", "real_time_result": "0", "frequency": "0.00", "real_time_conversion_rate": "0.00", "cpm": "0.000", "adgroup_name": "Ad Group20211020005346", "cpc": "0.000", "reach": "0", "campaign_id": 1714124576938033, "video_watched_6s": "0", "cost_per_result": "0.0000", "result_rate": "0.00", "video_watched_2s": "0", "real_time_conversion": "0", "real_time_cost_per_result": "0.0000", "real_time_result_rate": "0.00", "real_time_app_install": "0", "tt_app_id": 0, "promotion_type": "Website", "video_play_actions": "0", "cost_per_1000_reached": "0.000", "impressions": "0", "video_views_p50": "0", "comments": "0", "likes": "0", "conversion": "0", "real_time_cost_per_conversion": "0.000", "clicks": "0", "ctr": "0.00"}, "dimensions": {"adgroup_id": 1714124588896305}, "adgroup_id": 1714124588896305}, "emitted_at": 1691144426159} +{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": "493", "campaign_name": "Website Traffic20211020010104", "video_views_p50": "164", "average_video_play_per_user": "1.61", "cpc": "0.390", "impressions": "4696", "follows": "0", "video_views_p100": "76", "real_time_app_install": "0", "ctr": "1.09", "clicks_on_music_disc": "0", "comments": "0", "frequency": "1.23", "reach": "3822", "average_video_play": "1.48", "shares": "0", "profile_visits": "0", "video_play_actions": "4179", "video_views_p25": "355", "video_views_p75": "108", "app_install": "0", "clicks": "51", "spend": "20.000", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "5.233", "video_watched_6s": "132", "likes": "18", "cpm": "4.260"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-27 00:00:00"}, "stat_time_day": "2021-10-27 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1691144967333} +{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": "390", "campaign_name": "Website Traffic20211020010104", "video_views_p50": "112", "average_video_play_per_user": "1.57", "cpc": "0.480", "impressions": "3520", "follows": "0", "video_views_p100": "59", "real_time_app_install": "0", "ctr": "1.19", "clicks_on_music_disc": "0", "comments": "0", "frequency": "1.21", "reach": "2908", "average_video_play": "1.46", "shares": "0", "profile_visits": "0", "video_play_actions": "3118", "video_views_p25": "277", "video_views_p75": "74", "app_install": "0", "clicks": "42", "spend": "20.000", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "6.878", "video_watched_6s": "92", "likes": "17", "cpm": "5.680"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-22 00:00:00"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1691144967337} +{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": "471", "campaign_name": "Website Traffic20211020010104", "video_views_p50": "144", "average_video_play_per_user": "1.54", "cpc": "0.420", "impressions": "4787", "follows": "0", "video_views_p100": "70", "real_time_app_install": "0", "ctr": "1.00", "clicks_on_music_disc": "0", "comments": "0", "frequency": "1.22", "reach": "3938", "average_video_play": "1.42", "shares": "0", "profile_visits": "0", "video_play_actions": "4253", "video_views_p25": "328", "video_views_p75": "100", "app_install": "0", "clicks": "48", "spend": "20.000", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "5.079", "video_watched_6s": "120", "likes": "18", "cpm": "4.180"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-28 00:00:00"}, "stat_time_day": "2021-10-28 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1691144967339} +{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"cpm": "3.820", "campaign_name": "CampaignVadimTraffic", "video_views_p25": "3339", "impressions": "15689", "frequency": "1.20", "cpc": "0.410", "follows": "0", "video_play_actions": "14333", "profile_visits": "0", "real_time_app_install": "0", "ctr": "0.92", "cost_per_1000_reached": "4.597", "average_video_play_per_user": "1.39", "likes": "11", "comments": "0", "app_install": "0", "average_video_play": "1.26", "real_time_app_install_cost": "0.000", "video_watched_2s": "1364", "clicks": "145", "video_views_p50": "907", "spend": "60.000", "video_watched_6s": "402", "video_views_p75": "522", "video_views_p100": "402", "shares": "0", "clicks_on_music_disc": "0", "reach": "13052"}, "dimensions": {"campaign_id": 1728545382536225}, "campaign_id": 1728545382536225}, "emitted_at": 1691144987206} +{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"cpm": "4.340", "campaign_name": "Website Traffic20211020010104", "video_views_p25": "3674", "impressions": "46116", "frequency": "1.37", "cpc": "0.370", "follows": "0", "video_play_actions": "40753", "profile_visits": "0", "real_time_app_install": "0", "ctr": "1.17", "cost_per_1000_reached": "5.960", "average_video_play_per_user": "1.80", "likes": "263", "comments": "2", "app_install": "0", "average_video_play": "1.48", "real_time_app_install_cost": "0.000", "video_watched_2s": "5100", "clicks": "540", "video_views_p50": "1588", "spend": "200.000", "video_watched_6s": "1295", "video_views_p75": "998", "video_views_p100": "723", "shares": "0", "clicks_on_music_disc": "0", "reach": "33556"}, "dimensions": {"campaign_id": 1714125042508817}, "campaign_id": 1714125042508817}, "emitted_at": 1691144987209} +{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"cpm": "0.000", "campaign_name": "Website Traffic20211020005342", "video_views_p25": "0", "impressions": "0", "frequency": "0.00", "cpc": "0.000", "follows": "0", "video_play_actions": "0", "profile_visits": "0", "real_time_app_install": "0", "ctr": "0.00", "cost_per_1000_reached": "0.000", "average_video_play_per_user": "0.00", "likes": "0", "comments": "0", "app_install": "0", "average_video_play": "0.00", "real_time_app_install_cost": "0.000", "video_watched_2s": "0", "clicks": "0", "video_views_p50": "0", "spend": "0.000", "video_watched_6s": "0", "video_views_p75": "0", "video_views_p100": "0", "shares": "0", "clicks_on_music_disc": "0", "reach": "0"}, "dimensions": {"campaign_id": 1714124576938033}, "campaign_id": 1714124576938033}, "emitted_at": 1691144987213} +{"stream": "advertisers_reports_daily", "data": {"dimensions": {"stat_time_day": "2021-10-28 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"cash_spend": "20.000", "video_views_p75": "100", "spend": "20.000", "app_install": "0", "average_video_play": "1.42", "average_video_play_per_user": "1.54", "frequency": "1.22", "clicks": "48", "video_views_p100": "70", "comments": "0", "real_time_app_install": "0", "likes": "18", "video_watched_6s": "120", "video_views_p25": "328", "ctr": "1.00", "follows": "0", "shares": "0", "cost_per_1000_reached": "5.079", "video_watched_2s": "471", "reach": "3938", "real_time_app_install_cost": "0.000", "cpc": "0.420", "cpm": "4.180", "voucher_spend": "0.000", "video_play_actions": "4253", "profile_visits": "0", "impressions": "4787", "clicks_on_music_disc": "0", "video_views_p50": "144"}, "stat_time_day": "2021-10-28 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1691145506565} +{"stream": "advertisers_reports_daily", "data": {"dimensions": {"stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"cash_spend": "20.000", "video_views_p75": "95", "spend": "20.000", "app_install": "0", "average_video_play": "1.53", "average_video_play_per_user": "1.65", "frequency": "1.23", "clicks": "67", "video_views_p100": "65", "comments": "0", "real_time_app_install": "0", "likes": "19", "video_watched_6s": "124", "video_views_p25": "338", "ctr": "1.64", "follows": "0", "shares": "0", "cost_per_1000_reached": "6.020", "video_watched_2s": "463", "reach": "3322", "real_time_app_install_cost": "0.000", "cpc": "0.300", "cpm": "4.910", "voucher_spend": "0.000", "video_play_actions": "3590", "profile_visits": "0", "impressions": "4077", "clicks_on_music_disc": "0", "video_views_p50": "146"}, "stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1691145506568} +{"stream": "advertisers_reports_daily", "data": {"dimensions": {"stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"cash_spend": "20.000", "video_views_p75": "90", "spend": "20.000", "app_install": "0", "average_video_play": "1.50", "average_video_play_per_user": "1.61", "frequency": "1.20", "clicks": "46", "video_views_p100": "71", "comments": "1", "real_time_app_install": "0", "likes": "25", "video_watched_6s": "112", "video_views_p25": "297", "ctr": "1.23", "follows": "0", "shares": "0", "cost_per_1000_reached": "6.412", "video_watched_2s": "413", "reach": "3119", "real_time_app_install_cost": "0.000", "cpc": "0.430", "cpm": "5.330", "voucher_spend": "0.000", "video_play_actions": "3344", "profile_visits": "0", "impressions": "3750", "clicks_on_music_disc": "0", "video_views_p50": "142"}, "stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1691145506571} +{"stream": "advertisers_reports_lifetime", "data": {"metrics": {"follows": "0", "video_views_p50": "2665", "reach": "50418", "shares": "0", "clicks_on_music_disc": "0", "likes": "328", "video_play_actions": "59390", "spend": "280.000", "video_views_p75": "1636", "app_install": "0", "video_watched_6s": "1838", "video_views_p25": "7364", "video_views_p100": "1205", "ctr": "1.12", "clicks": "750", "cpc": "0.370", "profile_visits": "0", "video_watched_2s": "6941", "comments": "2", "real_time_app_install_cost": "0.000", "cpm": "4.200", "impressions": "66691", "average_video_play": "1.43", "cost_per_1000_reached": "5.554", "real_time_app_install": "0", "average_video_play_per_user": "1.68", "frequency": "1.32"}, "dimensions": {"advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633}, "emitted_at": 1691145526253} +{"stream": "ads_audience_reports_daily", "data": {"metrics": {"spend": "3.690", "ctr": "0.98", "cost_per_conversion": "0.000", "result": "12", "adgroup_name": "Ad Group20211020010107", "cpm": "3.020", "result_rate": "0.98", "real_time_conversion": "0", "placement_type": "Automatic Placement", "impressions": "1222", "real_time_cost_per_result": "0.3075", "real_time_result_rate": "0.98", "promotion_type": "Website", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "campaign_id": 1714125042508817, "dpa_target_audience_type": null, "real_time_result": "12", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "real_time_cost_per_conversion": "0.000", "tt_app_id": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "conversion": "0", "cpc": "0.310", "campaign_name": "Website Traffic20211020010104", "cost_per_result": "0.3075", "tt_app_name": "0", "clicks": "12", "adgroup_id": 1714125049901106}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_25_34"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_25_34"}, "emitted_at": 1691145528615} +{"stream": "ads_audience_reports_daily", "data": {"metrics": {"spend": "1.770", "ctr": "1.52", "cost_per_conversion": "0.000", "result": "5", "adgroup_name": "Ad Group20211020010107", "cpm": "5.380", "result_rate": "1.52", "real_time_conversion": "0", "placement_type": "Automatic Placement", "impressions": "329", "real_time_cost_per_result": "0.3540", "real_time_result_rate": "1.52", "promotion_type": "Website", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "campaign_id": 1714125042508817, "dpa_target_audience_type": null, "real_time_result": "5", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "real_time_cost_per_conversion": "0.000", "tt_app_id": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "conversion": "0", "cpc": "0.350", "campaign_name": "Website Traffic20211020010104", "cost_per_result": "0.3540", "tt_app_name": "0", "clicks": "5", "adgroup_id": 1714125049901106}, "dimensions": {"stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1691145528618} +{"stream": "ads_audience_reports_daily", "data": {"metrics": {"spend": "0.000", "ctr": "0.00", "cost_per_conversion": "0.000", "result": "0", "adgroup_name": "Ad Group20211019111040", "cpm": "0.000", "result_rate": "0.00", "real_time_conversion": "0", "placement_type": "Automatic Placement", "impressions": "56", "real_time_cost_per_result": "0.0000", "real_time_result_rate": "0.00", "promotion_type": "Website", "ad_text": "Open Source ETL", "campaign_id": 1714073078669329, "dpa_target_audience_type": null, "real_time_result": "0", "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "real_time_cost_per_conversion": "0.000", "tt_app_id": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "conversion": "0", "cpc": "0.000", "campaign_name": "Website Traffic20211019110444", "cost_per_result": "0.0000", "tt_app_name": "0", "clicks": "0", "adgroup_id": 1714073022392322}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_25_34"}, "stat_time_day": "2021-10-19 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_25_34"}, "emitted_at": 1691145528621} +{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"age": "AGE_45_54", "gender": "MALE", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-27 00:00:00"}, "metrics": {"clicks": "5", "tt_app_name": "0", "campaign_id": 1714125042508817, "result_rate": "1.34", "dpa_target_audience_type": null, "impressions": "373", "mobile_app_id": "0", "conversion_rate": "0.00", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_conversion": "0.000", "ctr": "1.34", "real_time_conversion": "0", "real_time_cost_per_result": "0.4280", "cpm": "5.740", "cost_per_conversion": "0.000", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "real_time_result_rate": "1.34", "placement_type": "Automatic Placement", "promotion_type": "Website", "result": "5", "real_time_result": "5", "cpc": "0.430", "cost_per_result": "0.4280", "conversion": "0", "spend": "2.140", "adgroup_name": "Ad Group20211020010107"}, "stat_time_day": "2021-10-27 00:00:00", "adgroup_id": 1714125049901106, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1691145591995} +{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"age": "AGE_35_44", "gender": "FEMALE", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-22 00:00:00"}, "metrics": {"clicks": "9", "tt_app_name": "0", "campaign_id": 1714125042508817, "result_rate": "1.41", "dpa_target_audience_type": null, "impressions": "638", "mobile_app_id": "0", "conversion_rate": "0.00", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_conversion": "0.000", "ctr": "1.41", "real_time_conversion": "0", "real_time_cost_per_result": "0.4789", "cpm": "6.760", "cost_per_conversion": "0.000", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "real_time_result_rate": "1.41", "placement_type": "Automatic Placement", "promotion_type": "Website", "result": "9", "real_time_result": "9", "cpc": "0.480", "cost_per_result": "0.4789", "conversion": "0", "spend": "4.310", "adgroup_name": "Ad Group20211020010107"}, "stat_time_day": "2021-10-22 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1691145591998} +{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"age": "AGE_35_44", "gender": "FEMALE", "adgroup_id": 1714073022392322, "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "0", "tt_app_name": "0", "campaign_id": 1714073078669329, "result_rate": "0.00", "dpa_target_audience_type": null, "impressions": "41", "mobile_app_id": "0", "conversion_rate": "0.00", "campaign_name": "Website Traffic20211019110444", "real_time_cost_per_conversion": "0.000", "ctr": "0.00", "real_time_conversion": "0", "real_time_cost_per_result": "0.0000", "cpm": "0.000", "cost_per_conversion": "0.000", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "real_time_result_rate": "0.00", "placement_type": "Automatic Placement", "promotion_type": "Website", "result": "0", "real_time_result": "0", "cpc": "0.000", "cost_per_result": "0.0000", "conversion": "0", "spend": "0.000", "adgroup_name": "Ad Group20211019111040"}, "stat_time_day": "2021-10-19 00:00:00", "adgroup_id": 1714073022392322, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1691145592001} +{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"campaign_id": 1714073078669329, "country_code": "US", "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "65", "impressions": "4874", "campaign_name": "Website Traffic20211019110444", "ctr": "1.33", "cpm": "4.100", "cpc": "0.310", "spend": "20.000"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1691145665950} +{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"campaign_id": 1714073078669329, "country_code": "US", "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"clicks": "0", "impressions": "12", "campaign_name": "Website Traffic20211019110444", "ctr": "0.00", "cpm": "0.000", "cpc": "0.000", "spend": "0.000"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1691145665953} +{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"campaign_id": 1714073078669329, "country_code": "US", "stat_time_day": "2021-10-22 00:00:00"}, "metrics": {"clicks": "0", "impressions": "0", "campaign_name": "Website Traffic20211019110444", "ctr": "0.00", "cpm": "0.000", "cpc": "0.000", "spend": "0.000"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1691145665956} +{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "22", "impressions": "1814", "ctr": "1.21", "cpm": "3.930", "cpc": "0.320", "spend": "7.130"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1691145699763} +{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "MALE", "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"clicks": "0", "impressions": "4", "ctr": "0.00", "cpm": "0.000", "cpc": "0.000", "spend": "0.000"}, "stat_time_day": "2021-10-20 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1691145699766} +{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "FEMALE", "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "29", "impressions": "2146", "ctr": "1.35", "cpm": "3.880", "cpc": "0.290", "spend": "8.320"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1691145699769} +{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.580", "impressions": "6897", "cpc": "0.360", "ctr": "1.26", "clicks": "87", "spend": "31.560"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_35_44", "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1691145715370} +{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.280", "impressions": "3450", "cpc": "0.380", "ctr": "1.13", "clicks": "39", "spend": "14.770"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_45_54", "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1691145715374} +{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "3.920", "impressions": "1818", "cpc": "0.320", "ctr": "1.21", "clicks": "22", "spend": "7.130"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1691145715376} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl index cc229dc0512c..1e6c7a91d969 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl @@ -1,8 +1,8 @@ -{"stream":"ads_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.52","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.2899","video_views_p50":"214","comments":"0","video_watched_6s":"180","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.18","result":"69","real_time_app_install":"0","likes":"36","video_views_p100":"92","impressions":"5830","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"140","cpc":"0.290","average_video_play_per_user":"1.64","cost_per_1000_reached":"4.161","promotion_type":"Website","video_views_p25":"513","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.21","follows":"0","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!","clicks":"69","ctr":"1.18","conversion":"0","adgroup_id":1714125049901106,"ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","video_play_actions":"5173","cpm":"3.430","reach":"4806","video_watched_2s":"686","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.2899","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"69","real_time_result_rate":"1.18","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"stat_time_day":"2021-10-25 00:00:00","ad_id":1714125051115569},"stat_time_day":"2021-10-25 00:00:00","ad_id":1714125051115569},"emitted_at":1680792290408} -{"stream":"ads_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.45","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.3774","video_views_p50":"130","comments":"1","video_watched_6s":"106","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.41","result":"53","real_time_app_install":"0","likes":"36","video_views_p100":"52","impressions":"3765","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"74","cpc":"0.380","average_video_play_per_user":"1.55","cost_per_1000_reached":"6.382","promotion_type":"Website","video_views_p25":"295","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.20","follows":"0","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!","clicks":"53","ctr":"1.41","conversion":"0","adgroup_id":1714125049901106,"ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","video_play_actions":"3344","cpm":"5.310","reach":"3134","video_watched_2s":"408","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.3774","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"53","real_time_result_rate":"1.41","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"stat_time_day":"2021-10-20 00:00:00","ad_id":1714125051115569},"stat_time_day":"2021-10-20 00:00:00","ad_id":1714125051115569},"emitted_at":1680792290412} -{"stream":"ads_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.68","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.0000","video_views_p50":"0","comments":"0","video_watched_6s":"0","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"0.00","result":"0","real_time_app_install":"0","likes":"2","video_views_p100":"0","impressions":"12","profile_visits":"0","adgroup_name":"Ad Group20211019111040","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"0","cpc":"0.000","average_video_play_per_user":"1.85","cost_per_1000_reached":"0.000","promotion_type":"Website","video_views_p25":"2","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.20","follows":"0","ad_text":"Open Source ETL","clicks":"0","ctr":"0.00","conversion":"0","adgroup_id":1714073022392322,"ad_name":"Optimized Version 1_202110192111_2021-10-19 21:11:39","video_play_actions":"11","cpm":"0.000","reach":"10","video_watched_2s":"3","campaign_name":"Website Traffic20211019110444","spend":"0.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.0000","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"0","real_time_result_rate":"0.00","campaign_id":1714073078669329,"tt_app_name":"0"},"dimensions":{"stat_time_day":"2021-10-20 00:00:00","ad_id":1714073085256738},"stat_time_day":"2021-10-20 00:00:00","ad_id":1714073085256738},"emitted_at":1680792290415} -{"stream":"ads_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.53","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.2985","video_views_p50":"146","comments":"0","video_watched_6s":"124","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.64","result":"67","real_time_app_install":"0","likes":"19","video_views_p100":"65","impressions":"4077","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"95","cpc":"0.300","average_video_play_per_user":"1.65","cost_per_1000_reached":"6.020","promotion_type":"Website","video_views_p25":"338","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.23","follows":"0","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!","clicks":"67","ctr":"1.64","conversion":"0","adgroup_id":1714125049901106,"ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","video_play_actions":"3590","cpm":"4.910","reach":"3322","video_watched_2s":"463","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.2985","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"67","real_time_result_rate":"1.64","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"stat_time_day":"2021-10-23 00:00:00","ad_id":1714125051115569},"stat_time_day":"2021-10-23 00:00:00","ad_id":1714125051115569},"emitted_at":1680792290418} -{"stream":"ads_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.50","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.4348","video_views_p50":"142","comments":"1","video_watched_6s":"112","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.23","result":"46","real_time_app_install":"0","likes":"25","video_views_p100":"71","impressions":"3750","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"90","cpc":"0.430","average_video_play_per_user":"1.61","cost_per_1000_reached":"6.412","promotion_type":"Website","video_views_p25":"297","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.20","follows":"0","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!","clicks":"46","ctr":"1.23","conversion":"0","adgroup_id":1714125049901106,"ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","video_play_actions":"3344","cpm":"5.330","reach":"3119","video_watched_2s":"413","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.4348","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"46","real_time_result_rate":"1.23","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"stat_time_day":"2021-10-26 00:00:00","ad_id":1714125051115569},"stat_time_day":"2021-10-26 00:00:00","ad_id":1714125051115569},"emitted_at":1680792290421} +{"stream": "ads_reports", "data": {"metrics": {"cpm": "3.430", "real_time_result": "69", "vta_conversion": "0", "real_time_conversion": "0", "comments": "0", "impressions": "5830", "campaign_id": 1714125042508817, "real_time_result_rate": "1.18", "campaign_name": "Website Traffic20211020010104", "cpc": "0.290", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "140", "video_views_p100": "92", "clicks": "69", "cost_per_result": "0.2899", "real_time_cost_per_result": "0.2899", "reach": "4806", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.21", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.18", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.64", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "36", "secondary_goal_result": null, "ctr": "1.18", "video_watched_2s": "686", "video_views_p50": "214", "video_watched_6s": "180", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.52", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "513", "result": "69", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "4.161", "video_play_actions": "5173"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-25 00:00:00"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563238} +{"stream": "ads_reports", "data": {"metrics": {"cpm": "5.310", "real_time_result": "53", "vta_conversion": "0", "real_time_conversion": "0", "comments": "1", "impressions": "3765", "campaign_id": 1714125042508817, "real_time_result_rate": "1.41", "campaign_name": "Website Traffic20211020010104", "cpc": "0.380", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "74", "video_views_p100": "52", "clicks": "53", "cost_per_result": "0.3774", "real_time_cost_per_result": "0.3774", "reach": "3134", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.20", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.41", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.55", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "36", "secondary_goal_result": null, "ctr": "1.41", "video_watched_2s": "408", "video_views_p50": "130", "video_watched_6s": "106", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.45", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "295", "result": "53", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.382", "video_play_actions": "3344"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563242} +{"stream": "ads_reports", "data": {"metrics": {"cpm": "4.910", "real_time_result": "67", "vta_conversion": "0", "real_time_conversion": "0", "comments": "0", "impressions": "4077", "campaign_id": 1714125042508817, "real_time_result_rate": "1.64", "campaign_name": "Website Traffic20211020010104", "cpc": "0.300", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "95", "video_views_p100": "65", "clicks": "67", "cost_per_result": "0.2985", "real_time_cost_per_result": "0.2985", "reach": "3322", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.23", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.64", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.65", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "19", "secondary_goal_result": null, "ctr": "1.64", "video_watched_2s": "463", "video_views_p50": "146", "video_watched_6s": "124", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.53", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "338", "result": "67", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.020", "video_play_actions": "3590"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-23 00:00:00"}, "stat_time_day": "2021-10-23 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563247} +{"stream": "ads_reports", "data": {"metrics": {"cpm": "5.330", "real_time_result": "46", "vta_conversion": "0", "real_time_conversion": "0", "comments": "1", "impressions": "3750", "campaign_id": 1714125042508817, "real_time_result_rate": "1.23", "campaign_name": "Website Traffic20211020010104", "cpc": "0.430", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "90", "video_views_p100": "71", "clicks": "46", "cost_per_result": "0.4348", "real_time_cost_per_result": "0.4348", "reach": "3119", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.20", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.23", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.61", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "25", "secondary_goal_result": null, "ctr": "1.23", "video_watched_2s": "413", "video_views_p50": "142", "video_watched_6s": "112", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.50", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "297", "result": "46", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.412", "video_play_actions": "3344"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-26 00:00:00"}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563249} +{"stream": "ads_reports", "data": {"metrics": {"cpm": "5.030", "real_time_result": "45", "vta_conversion": "0", "real_time_conversion": "0", "comments": "0", "impressions": "3977", "campaign_id": 1714125042508817, "real_time_result_rate": "1.13", "campaign_name": "Website Traffic20211020010104", "cpc": "0.440", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "77", "video_views_p100": "54", "clicks": "45", "cost_per_result": "0.4444", "real_time_cost_per_result": "0.4444", "reach": "3227", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.23", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.13", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.57", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "25", "secondary_goal_result": null, "ctr": "1.13", "video_watched_2s": "422", "video_views_p50": "128", "video_watched_6s": "107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.46", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "301", "result": "45", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.198", "video_play_actions": "3460"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-21 00:00:00"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563252} {"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.45","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.3774","video_views_p50":"130","comments":"1","video_watched_6s":"106","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.41","result":"53","real_time_app_install":"0","likes":"36","video_views_p100":"52","impressions":"3765","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"74","cpc":"0.380","average_video_play_per_user":"1.55","cost_per_1000_reached":"6.382","promotion_type":"Website","video_views_p25":"295","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.20","follows":"0","clicks":"53","ctr":"1.41","conversion":"0","video_play_actions":"3344","cpm":"5.310","reach":"3134","video_watched_2s":"408","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.3774","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"53","real_time_result_rate":"1.41","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-20 00:00:00"},"stat_time_day":"2021-10-20 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355223} {"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.52","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.2899","video_views_p50":"214","comments":"0","video_watched_6s":"180","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.18","result":"69","real_time_app_install":"0","likes":"36","video_views_p100":"92","impressions":"5830","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"140","cpc":"0.290","average_video_play_per_user":"1.64","cost_per_1000_reached":"4.161","promotion_type":"Website","video_views_p25":"513","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.21","follows":"0","clicks":"69","ctr":"1.18","conversion":"0","video_play_actions":"5173","cpm":"3.430","reach":"4806","video_watched_2s":"686","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.2899","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"69","real_time_result_rate":"1.18","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-25 00:00:00"},"stat_time_day":"2021-10-25 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355227} {"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.48","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.3922","video_views_p50":"164","comments":"0","video_watched_6s":"132","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.09","result":"51","real_time_app_install":"0","likes":"18","video_views_p100":"76","impressions":"4696","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"108","cpc":"0.390","average_video_play_per_user":"1.61","cost_per_1000_reached":"5.233","promotion_type":"Website","video_views_p25":"355","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.23","follows":"0","clicks":"51","ctr":"1.09","conversion":"0","video_play_actions":"4179","cpm":"4.260","reach":"3822","video_watched_2s":"493","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.3922","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"51","real_time_result_rate":"1.09","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-27 00:00:00"},"stat_time_day":"2021-10-27 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355230} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml index 5c9fa7fd006a..3ab49d44427f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 - dockerImageTag: 3.2.1 + dockerImageTag: 3.4.1 dockerRepository: airbyte/source-tiktok-marketing githubIssueLabel: source-tiktok-marketing icon: tiktok.svg @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt b/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt +++ b/airbyte-integrations/connectors/source-tiktok-marketing/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py index 28f943b256a9..3a5d282355a8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/setup.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "requests-mock==1.9.3", "timeout-decorator==0.5.0"] +TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock==1.9.3", "timeout-decorator==0.5.0"] setup( name="source_tiktok_marketing", diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json index b32cac273b3f..d958b38cf2d2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json @@ -293,6 +293,9 @@ "is_new_structure": { "type": "boolean" }, + "is_smart_performance_campaign": { + "type": ["null", "boolean"] + }, "catalog_id": { "type": ["null", "integer"] }, @@ -428,6 +431,12 @@ }, "delivery_mode": { "type": ["null", "string"] + }, + "category_exclusion_ids": { + "type": ["null", "array"], + "items": { + "type": "string" + } } } } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json index 51656689e407..6acd7fe268e4 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json @@ -140,6 +140,39 @@ }, "frequency": { "type": ["null", "string"] + }, + "total_purchase_value": { + "type": ["null", "string"] + }, + "total_onsite_shopping_value": { + "type": ["null", "string"] + }, + "onsite_shopping": { + "type": ["null", "string"] + }, + "vta_purchase": { + "type": ["null", "string"] + }, + "cta_purchase": { + "type": ["null", "string"] + }, + "cta_conversion": { + "type": ["null", "string"] + }, + "vta_conversion": { + "type": ["null", "string"] + }, + "total_pageview": { + "type": ["null", "string"] + }, + "complete_payment": { + "type": ["null", "string"] + }, + "value_per_complete_payment": { + "type": ["null", "string"] + }, + "total_complete_payment_rate": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 21407a8dc058..61d8aff8c89c 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -95,6 +95,7 @@ def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]: "access_token": access_token, "is_sandbox": is_sandbox, "attribution_window": config.get("attribution_window"), + "include_deleted": config.get("include_deleted"), } if advertiser_id: diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json index cf5cc8f31855..5751d01fdedb 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.json @@ -99,6 +99,13 @@ "default": 3, "order": 3, "type": "integer" + }, + "include_deleted": { + "title": "Include Deleted Data in Reports", + "description": "Set to active if you want to include deleted data in reports.", + "default": false, + "order": 4, + "type": "boolean" } } }, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index 849ef6ee89ac..3e9724eee853 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -51,6 +51,17 @@ "real_time_app_install", "real_time_app_install_cost", "app_install", + "total_purchase_value", + "total_onsite_shopping_value", + "onsite_shopping", + "vta_purchase", + "cta_purchase", + "vta_conversion", + "cta_conversion", + "total_pageview", + "complete_payment", + "value_per_complete_payment", + "total_complete_payment_rate", ] T = TypeVar("T") @@ -502,6 +513,7 @@ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: def __init__(self, **kwargs): report_granularity = kwargs.pop("report_granularity", None) self.attribution_window = kwargs.get("attribution_window") or 0 + self.include_deleted = kwargs.get("include_deleted", False) super().__init__(**kwargs) # Important: @@ -510,6 +522,16 @@ def __init__(self, **kwargs): if report_granularity: self.report_granularity = report_granularity + @property + def filters(self) -> List[MutableMapping[str, Any]]: + if self.include_deleted: + return [ + {"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, + {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}, + ] + return [] + @property @abstractmethod def report_level(self) -> ReportLevel: @@ -656,7 +678,24 @@ def _get_metrics(self): ) if self.report_level == ReportLevel.AD: - result.extend(["adgroup_id", "ad_name", "ad_text"]) + result.extend( + [ + "adgroup_id", + "ad_name", + "ad_text", + "total_purchase_value", + "total_onsite_shopping_value", + "onsite_shopping", + "vta_purchase", + "vta_conversion", + "cta_purchase", + "cta_conversion", + "total_pageview", + "complete_payment", + "value_per_complete_payment", + "total_complete_payment_rate", + ] + ) return result @@ -696,6 +735,8 @@ def request_params( params["start_date"] = stream_slice["start_date"] params["end_date"] = stream_slice["end_date"] + if self.filters: + params["filters"] = json.dumps(self.filters) return params def get_json_schema(self) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py index 2cdb296f4e7e..6d98cd6c569e 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py @@ -33,6 +33,7 @@ "end_date": END_DATE, "app_id": 1234, "advertiser_id": 0, + "include_deleted": True, } CONFIG_SANDBOX = { "access_token": "access_token", @@ -68,13 +69,15 @@ def advertiser_ids_fixture(): ], ) def test_get_time_interval(pendulum_now_mock, granularity, intervals_len): - intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity) + intervals = BasicReports._get_time_interval( + start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity) assert len(list(intervals)) == intervals_len @patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25")) def test_get_time_interval_past(pendulum_now_mock_past): - intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY) + intervals = BasicReports._get_time_interval( + start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY) assert len(list(intervals)) == 1 @@ -109,25 +112,32 @@ def test_stream_slices_basic_sandbox(advertiser_ids, config_name, slices_expecte ( Daily, [ - {"advertiser_id": 1, "end_date": "2020-01-30", "start_date": "2020-01-01"}, - {"advertiser_id": 1, "end_date": "2020-02-29", "start_date": "2020-01-31"}, - {"advertiser_id": 1, "end_date": "2020-03-01", "start_date": "2020-03-01"}, - {"advertiser_id": 2, "end_date": "2020-01-30", "start_date": "2020-01-01"}, - {"advertiser_id": 2, "end_date": "2020-02-29", "start_date": "2020-01-31"}, - {"advertiser_id": 2, "end_date": "2020-03-01", "start_date": "2020-03-01"}, + {"advertiser_id": 1, "end_date": "2020-01-30", + "start_date": "2020-01-01"}, + {"advertiser_id": 1, "end_date": "2020-02-29", + "start_date": "2020-01-31"}, + {"advertiser_id": 1, "end_date": "2020-03-01", + "start_date": "2020-03-01"}, + {"advertiser_id": 2, "end_date": "2020-01-30", + "start_date": "2020-01-01"}, + {"advertiser_id": 2, "end_date": "2020-02-29", + "start_date": "2020-01-31"}, + {"advertiser_id": 2, "end_date": "2020-03-01", + "start_date": "2020-03-01"}, ], ), ], ) def test_stream_slices_report(advertiser_ids, granularity, slices_expected, pendulum_now_mock): - slices = get_report_stream(AdsReports, granularity)(**CONFIG).stream_slices() + slices = get_report_stream(AdsReports, granularity)( + **CONFIG).stream_slices() assert list(slices) == slices_expected @pytest.mark.parametrize( "stream, metrics_number", [ - (AdsReports, 54), + (AdsReports, 65), (AdGroupsReports, 51), (AdvertisersReports, 29), (CampaignsReports, 28), @@ -143,7 +153,7 @@ def test_basic_reports_get_metrics_day(stream, metrics_number): @pytest.mark.parametrize( "stream, metrics_number", [ - (AdsReports, 54), + (AdsReports, 65), (AdGroupsReports, 51), (AdvertisersReports, 27), (CampaignsReports, 28), @@ -166,7 +176,8 @@ def test_basic_reports_get_metrics_lifetime(stream, metrics_number): ], ) def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expected): - dimensions = get_report_stream(stream, Lifetime)(**CONFIG)._get_reporting_dimensions() + dimensions = get_report_stream(stream, Lifetime)( + **CONFIG)._get_reporting_dimensions() assert dimensions == dimensions_expected @@ -177,11 +188,13 @@ def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expe (AdGroupsReports, ["adgroup_id", "stat_time_day"]), (AdvertisersReports, ["advertiser_id", "stat_time_day"]), (CampaignsReports, ["campaign_id", "stat_time_day"]), - (AdvertisersAudienceReports, ["advertiser_id", "stat_time_day", "gender", "age"]), + (AdvertisersAudienceReports, [ + "advertiser_id", "stat_time_day", "gender", "age"]), ], ) def test_basic_reports_get_reporting_dimensions_day(stream, dimensions_expected): - dimensions = get_report_stream(stream, Daily)(**CONFIG)._get_reporting_dimensions() + dimensions = get_report_stream(stream, Daily)( + **CONFIG)._get_reporting_dimensions() assert dimensions == dimensions_expected @@ -200,14 +213,17 @@ def test_basic_reports_cursor_field(granularity, cursor_field_expected): def test_request_params(): - stream_slice = {"advertiser_id": 1, "start_date": "2020", "end_date": "2021"} - params = get_report_stream(AdvertisersAudienceReports, Daily)(**CONFIG).request_params(stream_slice=stream_slice) + stream_slice = {"advertiser_id": 1, + "start_date": "2020", "end_date": "2021"} + params = get_report_stream(AdvertisersAudienceReports, Daily)( + **CONFIG).request_params(stream_slice=stream_slice) assert params == { "advertiser_id": 1, "data_level": "AUCTION_ADVERTISER", "dimensions": '["advertiser_id", "stat_time_day", "gender", "age"]', "end_date": "2021", "metrics": '["spend", "cpc", "cpm", "impressions", "clicks", "ctr"]', + "filters": '[{"filter_value": ["STATUS_ALL"], "field_name": "ad_status", "filter_type": "IN"}, {"filter_value": ["STATUS_ALL"], "field_name": "campaign_status", "filter_type": "IN"}, {"filter_value": ["STATUS_ALL"], "field_name": "adgroup_status", "filter_type": "IN"}]', "page_size": 1000, "report_type": "AUDIENCE", "service_type": "AUCTION", @@ -226,11 +242,14 @@ def test_get_updated_state(): # state should be empty while stream is reading records ads.max_cursor_date = "2020-01-08 00:00:00" is_finished.return_value = False - state1 = ads.get_updated_state(current_stream_state=state, latest_record={}) + state1 = ads.get_updated_state( + current_stream_state=state, latest_record={}) assert state1 == {"modify_time": ""} # state should be updated only when all records have been read (is_finished = True) is_finished.return_value = True - state2 = ads.get_updated_state(current_stream_state=state, latest_record={}) - state2_modify_time = state2["modify_time"] # state2_modify_time is JsonUpdatedState object + state2 = ads.get_updated_state( + current_stream_state=state, latest_record={}) + # state2_modify_time is JsonUpdatedState object + state2_modify_time = state2["modify_time"] assert state2_modify_time.dict() == "2020-01-08 00:00:00" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index 80c1c2124d33..073b318e006f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -81,7 +81,7 @@ def unixtime2str(unix_time: int) -> str: def test_random_items(prepared_prod_args): stream = Ads(**prepared_prod_args) advertiser_count = 100 - test_advertiser_ids = set([random_integer() for _ in range(advertiser_count)]) + test_advertiser_ids = set([str(random_integer()) for _ in range(advertiser_count)]) advertiser_count = len(test_advertiser_ids) page_size = 100 with requests_mock.Mocker() as m: diff --git a/airbyte-integrations/connectors/source-timely/metadata.yaml b/airbyte-integrations/connectors/source-timely/metadata.yaml index 68a17938d022..f0783a97a8b5 100644 --- a/airbyte-integrations/connectors/source-timely/metadata.yaml +++ b/airbyte-integrations/connectors/source-timely/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/timely tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-timely/requirements.txt b/airbyte-integrations/connectors/source-timely/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-timely/requirements.txt +++ b/airbyte-integrations/connectors/source-timely/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-timely/setup.py b/airbyte-integrations/connectors/source-timely/setup.py index fbab92dcc63c..f4579a7b9c50 100644 --- a/airbyte-integrations/connectors/source-timely/setup.py +++ b/airbyte-integrations/connectors/source-timely/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-tmdb/metadata.yaml b/airbyte-integrations/connectors/source-tmdb/metadata.yaml index 4f5aab1c8afc..7979d306d363 100644 --- a/airbyte-integrations/connectors/source-tmdb/metadata.yaml +++ b/airbyte-integrations/connectors/source-tmdb/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tmdb/requirements.txt b/airbyte-integrations/connectors/source-tmdb/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-tmdb/requirements.txt +++ b/airbyte-integrations/connectors/source-tmdb/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-tmdb/setup.py b/airbyte-integrations/connectors/source-tmdb/setup.py index 17694b4d93c8..35310f227836 100644 --- a/airbyte-integrations/connectors/source-tmdb/setup.py +++ b/airbyte-integrations/connectors/source-tmdb/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json index 8dcb02a810ec..09f16c3ccf2a 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json @@ -1,25 +1,27 @@ { "tasks": { - "id" : false, - "project_id" : 10, - "section_id" : -10, - "content" : 10, - "description" : true, - "is_completed" : "not so true", - "labels" : [true, false], - "parent_id" : -50, - "order" : true, - "priority" : "10", - "due" : true, - "url" : 50, - "comment_count" : "10", - "created_at" : {"when" : true}, + "id": false, + "project_id": 10, + "section_id": -10, + "content": 10, + "description": true, + "is_completed": "not so true", + "labels": [true, false], + "parent_id": -50, + "order": true, + "priority": "10", + "due": true, + "url": 50, + "comment_count": "10", + "created_at": { "when": true }, "creator_id": -1, - "assignee_id" : 20, - "assigner_id" : 50 + "assignee_id": 20, + "assigner_id": 50 }, - "projects" : { - "id": {"number" : "2203306141"}, + "projects": { + "id": { + "number": "2203306141" + }, "name": 50, "comment_count": false, "order": "1", @@ -30,6 +32,6 @@ "is_inbox_project": [true], "is_team_inbox": "false", "view_style": ["list"], - "url": ["https://todoist.com/showProject?id=2203306141" -} + "url": ["https://todoist.com/showProject?id=2203306141"] + } } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json index 916619770ff2..275f28ac14f5 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json @@ -10,14 +10,8 @@ "name": "tasks", "namespace": null, "source_defined_cursor": null, - "source_defined_primary_key": [ - [ - "id" - ] - ], - "supported_sync_modes": [ - "full_refresh" - ] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh" }, @@ -31,16 +25,10 @@ "name": "projects", "namespace": null, "source_defined_cursor": null, - "source_defined_primary_key": [ - [ - "id" - ] - ], - "supported_sync_modes": [ - "full_refresh" - ] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-todoist/metadata.yaml b/airbyte-integrations/connectors/source-todoist/metadata.yaml index 6a6a0907f775..f32560daec66 100644 --- a/airbyte-integrations/connectors/source-todoist/metadata.yaml +++ b/airbyte-integrations/connectors/source-todoist/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/todoist tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-todoist/requirements.txt b/airbyte-integrations/connectors/source-todoist/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-todoist/requirements.txt +++ b/airbyte-integrations/connectors/source-todoist/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index 1ef88776b537..cf672aeb4515 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -10,6 +10,7 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8", diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json index 1d93cef6281a..f43f7f3da3f5 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/projects.json @@ -1,43 +1,43 @@ { - "$schema": "http://json-schema.org/schema#", - "additionalProperties": true, - "type": "object", - "properties": { - "color": { - "type": ["null", "string"] - }, - "comment_count": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "string"] - }, - "is_favorite": { - "type": ["null", "boolean"] - }, - "is_inbox_project": { - "type": ["null", "boolean"] - }, - "is_shared": { - "type": ["null", "boolean"] - }, - "is_team_inbox": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "order": { - "type": ["null", "integer"] - }, - "parent_id": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "view_style": { - "type": ["null", "string"] - } + "$schema": "http://json-schema.org/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "color": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "is_favorite": { + "type": ["null", "boolean"] + }, + "is_inbox_project": { + "type": ["null", "boolean"] + }, + "is_shared": { + "type": ["null", "boolean"] + }, + "is_team_inbox": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "order": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "view_style": { + "type": ["null", "string"] } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json index 9502db6b4277..cf22f8da8a4b 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json @@ -1,94 +1,82 @@ { - "$schema": "http://json-schema.org/schema#", - "additionalProperties": true, - "type": "object", - "properties": { - "assignee_id": { - "type": [ - "null", - "string" - ] - }, - "assigner_id": { - "type": [ - "null", - "string" - ] - }, - "comment_count": { - "type": ["null", "integer"] - }, - "content": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"] - }, - "creator_id": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "due": { - "anyOf": [ - { - "type": ["null", "object"] - }, - { - "properties": { - "date": { - "type": ["null", "string"] - }, - "is_recurring": { - "type": ["null", "boolean"] - }, - "lang": { - "type": ["null", "string"] - }, - "string": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - } - ] - }, - "id": { - "type": ["null", "string"] - }, - "is_completed": { - "type": ["null", "boolean"] - }, - "labels": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] + "$schema": "http://json-schema.org/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "assignee_id": { + "type": ["null", "string"] + }, + "assigner_id": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "due": { + "anyOf": [ + { + "type": ["null", "object"] + }, + { + "properties": { + "date": { + "type": ["null", "string"] + }, + "is_recurring": { + "type": ["null", "boolean"] + }, + "lang": { + "type": ["null", "string"] + }, + "string": { + "type": ["null", "string"] } - }, - "order": { - "type": ["null", "integer"] - }, - "parent_id": { - "type": [ - "null", - "string" - ] - }, - "priority": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "string"] - }, - "section_id": { - "type": [ - "null", - "string" - ] - }, - "url": { - "type": ["null", "string"] + }, + "type": ["null", "object"] } + ] + }, + "id": { + "type": ["null", "string"] + }, + "is_completed": { + "type": ["null", "boolean"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "order": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "string"] + }, + "section_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-toggl/metadata.yaml b/airbyte-integrations/connectors/source-toggl/metadata.yaml index 3f62850f1bb8..cac927f82274 100644 --- a/airbyte-integrations/connectors/source-toggl/metadata.yaml +++ b/airbyte-integrations/connectors/source-toggl/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-toggl/requirements.txt b/airbyte-integrations/connectors/source-toggl/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-toggl/requirements.txt +++ b/airbyte-integrations/connectors/source-toggl/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-toggl/setup.py b/airbyte-integrations/connectors/source-toggl/setup.py index abba78cf301e..bb481cee5328 100644 --- a/airbyte-integrations/connectors/source-toggl/setup.py +++ b/airbyte-integrations/connectors/source-toggl/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-tplcentral/metadata.yaml b/airbyte-integrations/connectors/source-tplcentral/metadata.yaml index 97635e5ad46c..e47c7d06d92e 100644 --- a/airbyte-integrations/connectors/source-tplcentral/metadata.yaml +++ b/airbyte-integrations/connectors/source-tplcentral/metadata.yaml @@ -16,4 +16,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/tplcentral tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tplcentral/requirements.txt b/airbyte-integrations/connectors/source-tplcentral/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-tplcentral/requirements.txt +++ b/airbyte-integrations/connectors/source-tplcentral/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-tplcentral/setup.py b/airbyte-integrations/connectors/source-tplcentral/setup.py index 35f879d6eb9d..602d12056cb3 100644 --- a/airbyte-integrations/connectors/source-tplcentral/setup.py +++ b/airbyte-integrations/connectors/source-tplcentral/setup.py @@ -14,7 +14,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock~=1.9.3", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-trello/Dockerfile b/airbyte-integrations/connectors/source-trello/Dockerfile index 55d69c3b4bc0..ab1cf6ecde2d 100644 --- a/airbyte-integrations/connectors/source-trello/Dockerfile +++ b/airbyte-integrations/connectors/source-trello/Dockerfile @@ -29,5 +29,5 @@ COPY source_trello ./source_trello ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.3.4 LABEL io.airbyte.name=airbyte/source-trello diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl index 40b317ed5c44..6951286fd9f5 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl @@ -1,11 +1,11 @@ {"stream": "actions", "data": {"id": "6413643417610bdd29d36c47", "idMemberCreator": "610be3762899a26d04256dae", "data": {"card": {"name": "test_1 (fixed)", "id": "611aa1eae05ae943ada67f8a", "idShort": 3, "shortLink": "FhvCm05w"}, "old": {"name": "test_1"}, "board": {"id": "611aa0ef37acd675af67dc9b", "name": "test_board", "shortLink": "Lj11WRiJ"}, "list": {"id": "611aa0ef37acd675af67dc9c", "name": "To Do"}}, "appCreator": null, "type": "updateCard", "date": "2023-03-16T18:47:16.404Z", "limits": null, "memberCreator": {"id": "610be3762899a26d04256dae", "activityBlocked": false, "avatarHash": "eaed38443ff1d12aa4ce2c5d03faafd1", "avatarUrl": "https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/eaed38443ff1d12aa4ce2c5d03faafd1", "fullName": "integration test", "idMemberReferrer": null, "initials": "IT", "nonPublic": {}, "nonPublicAvailable": true, "username": "integrationtest19"}}, "emitted_at": 1683406404994} {"stream": "actions", "data": {"id": "6127a34a824394333c4b089e", "idMemberCreator": "610be3762899a26d04256dae", "data": {"old": {"closed": false}, "card": {"closed": true, "id": "6127a3488a027b342502ba23", "name": "Demo_test", "idShort": 7, "shortLink": "bbhWatIg"}, "board": {"id": "611aa0ef37acd675af67dc9b", "name": "test_board", "shortLink": "Lj11WRiJ"}, "list": {"id": "611aa0ef37acd675af67dc9d", "name": "Doing"}}, "appCreator": {"id": "56ff07afbc287582d36e19ac", "name": "Butler", "icon": {"url": "https://app.butlerfortrello.com/assets/icon-butler-bot.png"}}, "type": "updateCard", "date": "2021-08-26T14:20:58.523Z", "limits": null, "memberCreator": {"id": "610be3762899a26d04256dae", "activityBlocked": false, "avatarHash": "eaed38443ff1d12aa4ce2c5d03faafd1", "avatarUrl": "https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/eaed38443ff1d12aa4ce2c5d03faafd1", "fullName": "integration test", "idMemberReferrer": null, "initials": "IT", "nonPublic": {}, "nonPublicAvailable": true, "username": "integrationtest19"}}, "emitted_at": 1683406404996} {"stream": "actions", "data": {"id": "6127a34ac4dc0b1d29f7667f", "idMemberCreator": "610be3762899a26d04256dae", "data": {"old": {"pos": 131071}, "card": {"pos": 32767.5, "id": "6127a3488a027b342502ba23", "name": "Demo_test", "idShort": 7, "shortLink": "bbhWatIg"}, "board": {"id": "611aa0ef37acd675af67dc9b", "name": "test_board", "shortLink": "Lj11WRiJ"}, "list": {"id": "611aa0ef37acd675af67dc9d", "name": "Doing"}}, "appCreator": {"id": "56ff07afbc287582d36e19ac", "name": "Butler", "icon": {"url": "https://app.butlerfortrello.com/assets/icon-butler-bot.png"}}, "type": "updateCard", "date": "2021-08-26T14:20:58.193Z", "limits": null, "memberCreator": {"id": "610be3762899a26d04256dae", "activityBlocked": false, "avatarHash": "eaed38443ff1d12aa4ce2c5d03faafd1", "avatarUrl": "https://trello-members.s3.amazonaws.com/610be3762899a26d04256dae/eaed38443ff1d12aa4ce2c5d03faafd1", "fullName": "integration test", "idMemberReferrer": null, "initials": "IT", "nonPublic": {}, "nonPublicAvailable": true, "username": "integrationtest19"}}, "emitted_at": 1683406404996} -{"stream": "boards", "data": {"id": "611aa0ef37acd675af67dc9b", "nodeId": "ari:cloud:trello::board/workspace/610be50a0537086c571a5684/611aa0ef37acd675af67dc9b", "name": "test_board", "desc": "", "descData": null, "closed": false, "dateClosed": null, "idOrganization": "610be50a0537086c571a5684", "idEnterprise": null, "limits": {"attachments": {"perBoard": {"status": "ok", "disableAt": 36000, "warnAt": 28800}, "perCard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "boards": {"totalMembersPerBoard": {"status": "ok", "disableAt": 1600, "warnAt": 1280}, "totalAccessRequestsPerBoard": {"status": "ok", "disableAt": 4000, "warnAt": 3200}}, "cards": {"openPerBoard": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "openPerList": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "totalPerBoard": {"status": "ok", "disableAt": 2000000, "warnAt": 1600000}, "totalPerList": {"status": "ok", "disableAt": 1000000, "warnAt": 800000}}, "checklists": {"perBoard": {"status": "ok", "disableAt": 1800000, "warnAt": 1440000}, "perCard": {"status": "ok", "disableAt": 500, "warnAt": 400}}, "checkItems": {"perChecklist": {"status": "ok", "disableAt": 200, "warnAt": 160}}, "customFields": {"perBoard": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "customFieldOptions": {"perField": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "labels": {"perBoard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "lists": {"openPerBoard": {"status": "ok", "disableAt": 500, "warnAt": 400}, "totalPerBoard": {"status": "ok", "disableAt": 3000, "warnAt": 2400}}, "stickers": {"perCard": {"status": "ok", "disableAt": 70, "warnAt": 56}}, "reactions": {"perAction": {"status": "ok", "disableAt": 900, "warnAt": 720}, "uniquePerAction": {"status": "ok", "disableAt": 17, "warnAt": 14}}}, "pinned": false, "starred": false, "url": "https://trello.com/b/Lj11WRiJ/testboard", "prefs": {"permissionLevel": "org", "hideVotes": false, "voting": "disabled", "comments": "members", "invitations": "members", "selfJoin": true, "cardCovers": true, "isTemplate": false, "cardAging": "regular", "calendarFeedEnabled": false, "hiddenPluginBoardButtons": [], "switcherViews": [{"_id": "63dee894b0ed3558c379bcf1", "viewType": "Board", "enabled": true, "typeName": "SwitcherViews", "id": "63dee894b0ed3558c379bcf1"}, {"_id": "63dee894b0ed3558c379bcf2", "viewType": "Table", "enabled": true, "typeName": "SwitcherViews", "id": "63dee894b0ed3558c379bcf2"}, {"_id": "63dee894b0ed3558c379bcf3", "viewType": "Calendar", "enabled": false, "typeName": "SwitcherViews", "id": "63dee894b0ed3558c379bcf3"}, {"_id": "63dee894b0ed3558c379bcf4", "viewType": "Dashboard", "enabled": false, "typeName": "SwitcherViews", "id": "63dee894b0ed3558c379bcf4"}, {"_id": "63dee894b0ed3558c379bcf5", "viewType": "Timeline", "enabled": false, "typeName": "SwitcherViews", "id": "63dee894b0ed3558c379bcf5"}, {"_id": "63dee894b0ed3558c379bcf6", "viewType": "Map", "enabled": false, "typeName": "SwitcherViews", "id": "63dee894b0ed3558c379bcf6"}], "background": "611a97ed741bfb5782acf105", "backgroundColor": null, "backgroundImage": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730", "backgroundImageScaled": [{"width": 67, "height": 100, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/67x100/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 128, "height": 192, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/128x192/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 320, "height": 480, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/320x480/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 640, "height": 960, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/640x960/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 683, "height": 1024, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/683x1024/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 854, "height": 1280, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/854x1280/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1067, "height": 1600, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1067x1600/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1281, "height": 1920, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1281x1920/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1366, "height": 2048, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730"}], "backgroundTile": false, "backgroundBrightness": "light", "backgroundBottomColor": "#7c8280", "backgroundTopColor": "#aeb0ae", "canBePublic": true, "canBeEnterprise": true, "canBeOrg": true, "canBePrivate": true, "canInvite": true}, "shortLink": "Lj11WRiJ", "subscribed": false, "labelNames": {"green": "", "yellow": "", "orange": "", "red": "", "purple": "", "blue": "", "sky": "", "lime": "", "pink": "", "black": "", "green_dark": "", "yellow_dark": "", "orange_dark": "", "red_dark": "", "purple_dark": "", "blue_dark": "", "sky_dark": "", "lime_dark": "", "pink_dark": "", "black_dark": "", "green_light": "", "yellow_light": "", "orange_light": "", "red_light": "", "purple_light": "", "blue_light": "", "sky_light": "", "lime_light": "", "pink_light": "", "black_light": ""}, "powerUps": [], "dateLastActivity": "2023-03-16T18:47:16.386Z", "dateLastView": "2023-03-16T20:10:44.953Z", "shortUrl": "https://trello.com/b/Lj11WRiJ", "idTags": [], "datePluginDisable": null, "creationMethod": "automatic", "ixUpdate": "90", "templateGallery": null, "enterpriseOwned": false, "idBoardSource": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "idMemberCreator": "610be3762899a26d04256dae", "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "611aa0ef37acd675af67dc9f"}]}, "emitted_at": 1683406406577} -{"stream": "boards", "data": {"id": "611aa586ef5f2c8e1deec8b6", "nodeId": "ari:cloud:trello::board/workspace/610be50a0537086c571a5684/611aa586ef5f2c8e1deec8b6", "name": "test_board_new", "desc": "", "descData": null, "closed": false, "dateClosed": null, "idOrganization": "610be50a0537086c571a5684", "idEnterprise": null, "limits": {"attachments": {"perBoard": {"status": "ok", "disableAt": 36000, "warnAt": 28800}, "perCard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "boards": {"totalMembersPerBoard": {"status": "ok", "disableAt": 1600, "warnAt": 1280}, "totalAccessRequestsPerBoard": {"status": "ok", "disableAt": 4000, "warnAt": 3200}}, "cards": {"openPerBoard": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "openPerList": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "totalPerBoard": {"status": "ok", "disableAt": 2000000, "warnAt": 1600000}, "totalPerList": {"status": "ok", "disableAt": 1000000, "warnAt": 800000}}, "checklists": {"perBoard": {"status": "ok", "disableAt": 1800000, "warnAt": 1440000}, "perCard": {"status": "ok", "disableAt": 500, "warnAt": 400}}, "checkItems": {"perChecklist": {"status": "ok", "disableAt": 200, "warnAt": 160}}, "customFields": {"perBoard": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "customFieldOptions": {"perField": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "labels": {"perBoard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "lists": {"openPerBoard": {"status": "ok", "disableAt": 500, "warnAt": 400}, "totalPerBoard": {"status": "ok", "disableAt": 3000, "warnAt": 2400}}, "stickers": {"perCard": {"status": "ok", "disableAt": 70, "warnAt": 56}}, "reactions": {"perAction": {"status": "ok", "disableAt": 900, "warnAt": 720}, "uniquePerAction": {"status": "ok", "disableAt": 17, "warnAt": 14}}}, "pinned": false, "starred": false, "url": "https://trello.com/b/qpYBekjt/testboardnew", "prefs": {"permissionLevel": "org", "hideVotes": false, "voting": "disabled", "comments": "members", "invitations": "members", "selfJoin": true, "cardCovers": true, "isTemplate": false, "cardAging": "regular", "calendarFeedEnabled": false, "hiddenPluginBoardButtons": [], "switcherViews": [{"_id": "63dee887b0ed3558c3793a3a", "viewType": "Board", "enabled": true, "typeName": "SwitcherViews", "id": "63dee887b0ed3558c3793a3a"}, {"_id": "63dee887b0ed3558c3793a3b", "viewType": "Table", "enabled": true, "typeName": "SwitcherViews", "id": "63dee887b0ed3558c3793a3b"}, {"_id": "63dee887b0ed3558c3793a3c", "viewType": "Calendar", "enabled": false, "typeName": "SwitcherViews", "id": "63dee887b0ed3558c3793a3c"}, {"_id": "63dee887b0ed3558c3793a3d", "viewType": "Dashboard", "enabled": false, "typeName": "SwitcherViews", "id": "63dee887b0ed3558c3793a3d"}, {"_id": "63dee887b0ed3558c3793a3e", "viewType": "Timeline", "enabled": false, "typeName": "SwitcherViews", "id": "63dee887b0ed3558c3793a3e"}, {"_id": "63dee887b0ed3558c3793a3f", "viewType": "Map", "enabled": false, "typeName": "SwitcherViews", "id": "63dee887b0ed3558c3793a3f"}], "background": "611a97ed741bfb5782acf105", "backgroundColor": null, "backgroundImage": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730", "backgroundImageScaled": [{"width": 67, "height": 100, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/67x100/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 128, "height": 192, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/128x192/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 320, "height": 480, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/320x480/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 640, "height": 960, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/640x960/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 683, "height": 1024, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/683x1024/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 854, "height": 1280, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/854x1280/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1067, "height": 1600, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1067x1600/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1281, "height": 1920, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1281x1920/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1366, "height": 2048, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730"}], "backgroundTile": false, "backgroundBrightness": "light", "backgroundBottomColor": "#7c8280", "backgroundTopColor": "#aeb0ae", "canBePublic": true, "canBeEnterprise": true, "canBeOrg": true, "canBePrivate": true, "canInvite": true}, "shortLink": "qpYBekjt", "subscribed": false, "labelNames": {"green": "", "yellow": "", "orange": "", "red": "", "purple": "", "blue": "", "sky": "", "lime": "", "pink": "", "black": "", "green_dark": "", "yellow_dark": "", "orange_dark": "", "red_dark": "", "purple_dark": "", "blue_dark": "", "sky_dark": "", "lime_dark": "", "pink_dark": "", "black_dark": "", "green_light": "", "yellow_light": "", "orange_light": "", "red_light": "", "purple_light": "", "blue_light": "", "sky_light": "", "lime_light": "", "pink_light": "", "black_light": ""}, "powerUps": [], "dateLastActivity": "2023-03-16T20:10:54.595Z", "dateLastView": "2023-03-17T11:01:50.283Z", "shortUrl": "https://trello.com/b/qpYBekjt", "idTags": [], "datePluginDisable": null, "creationMethod": "automatic", "ixUpdate": "52", "templateGallery": null, "enterpriseOwned": false, "idBoardSource": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "idMemberCreator": "610be3762899a26d04256dae", "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "611aa586ef5f2c8e1deec8ba"}]}, "emitted_at": 1683406406579} -{"stream": "cards", "data": {"id": "6127a3488a027b342502ba23", "badges": {"attachmentsByType": {"trello": {"board": 0, "card": 0}}, "location": false, "votes": 0, "viewingMemberVoted": false, "subscribed": false, "fogbugz": "", "checkItems": 0, "checkItemsChecked": 0, "checkItemsEarliestDue": null, "comments": 0, "attachments": 0, "description": false, "due": null, "dueComplete": false, "start": null}, "checkItemStates": null, "closed": true, "dueComplete": false, "dateLastActivity": "2021-08-26T14:20:58.508Z", "desc": "", "descData": null, "due": null, "dueReminder": null, "email": null, "idBoard": "611aa0ef37acd675af67dc9b", "idChecklists": [], "idList": "611aa0ef37acd675af67dc9d", "idMembers": [], "idMembersVoted": [], "idShort": 7, "idAttachmentCover": null, "labels": [], "idLabels": [], "manualCoverAttachment": false, "name": "Demo_test", "pos": 32767.5, "shortLink": "bbhWatIg", "shortUrl": "https://trello.com/c/bbhWatIg", "start": null, "subscribed": false, "url": "https://trello.com/c/bbhWatIg/7-demotest", "cover": {"idAttachment": null, "color": null, "idUploadedBackground": null, "size": "normal", "brightness": "dark", "idPlugin": null}, "isTemplate": false, "cardRole": null, "pluginData": [], "customFieldItems": [], "members": []}, "emitted_at": 1683406407691} -{"stream": "cards", "data": {"id": "6127a338b1d0335b7d609c28", "badges": {"attachmentsByType": {"trello": {"board": 0, "card": 0}}, "location": false, "votes": 0, "viewingMemberVoted": false, "subscribed": false, "fogbugz": "", "checkItems": 0, "checkItemsChecked": 0, "checkItemsEarliestDue": null, "comments": 0, "attachments": 0, "description": false, "due": null, "dueComplete": false, "start": null}, "checkItemStates": null, "closed": true, "dueComplete": false, "dateLastActivity": "2021-08-26T14:20:42.690Z", "desc": "", "descData": null, "due": null, "dueReminder": null, "email": null, "idBoard": "611aa0ef37acd675af67dc9b", "idChecklists": [], "idList": "611aa0ef37acd675af67dc9c", "idMembers": [], "idMembersVoted": [], "idShort": 6, "idAttachmentCover": null, "labels": [], "idLabels": [], "manualCoverAttachment": false, "name": "demo_test", "pos": 16383.75, "shortLink": "NbePPuOM", "shortUrl": "https://trello.com/c/NbePPuOM", "start": null, "subscribed": false, "url": "https://trello.com/c/NbePPuOM/6-demotest", "cover": {"idAttachment": null, "color": null, "idUploadedBackground": null, "size": "normal", "brightness": "dark", "idPlugin": null}, "isTemplate": false, "cardRole": null, "pluginData": [], "customFieldItems": [], "members": []}, "emitted_at": 1683406407692} -{"stream": "cards", "data": {"id": "612628f089a80219c8caa562", "badges": {"attachmentsByType": {"trello": {"board": 0, "card": 0}}, "location": false, "votes": 0, "viewingMemberVoted": false, "subscribed": false, "fogbugz": "", "checkItems": 0, "checkItemsChecked": 0, "checkItemsEarliestDue": null, "comments": 0, "attachments": 0, "description": false, "due": null, "dueComplete": false, "start": null}, "checkItemStates": null, "closed": true, "dueComplete": false, "dateLastActivity": "2021-08-25T11:26:41.767Z", "desc": "", "descData": null, "due": null, "dueReminder": null, "email": null, "idBoard": "611aa0ef37acd675af67dc9b", "idChecklists": [], "idList": "611aa0ef37acd675af67dc9c", "idMembers": [], "idMembersVoted": [], "idShort": 5, "idAttachmentCover": null, "labels": [], "idLabels": [], "manualCoverAttachment": false, "name": "new_test", "pos": 16383.75, "shortLink": "npcCoHA2", "shortUrl": "https://trello.com/c/npcCoHA2", "start": null, "subscribed": false, "url": "https://trello.com/c/npcCoHA2/5-newtest", "cover": {"idAttachment": null, "color": null, "idUploadedBackground": null, "size": "normal", "brightness": "dark", "idPlugin": null}, "isTemplate": false, "cardRole": null, "pluginData": [], "customFieldItems": [], "members": []}, "emitted_at": 1683406407693} +{"stream": "boards", "data": {"id": "611aa0ef37acd675af67dc9b", "nodeId": "ari:cloud:trello::board/workspace/610be50a0537086c571a5684/611aa0ef37acd675af67dc9b", "name": "test_board", "desc": "", "descData": null, "closed": false, "dateClosed": null, "idOrganization": "610be50a0537086c571a5684", "idEnterprise": null, "limits": {"attachments": {"perBoard": {"status": "ok", "disableAt": 36000, "warnAt": 28800}, "perCard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "boards": {"totalMembersPerBoard": {"status": "ok", "disableAt": 1600, "warnAt": 1280}, "totalAccessRequestsPerBoard": {"status": "ok", "disableAt": 4000, "warnAt": 3200}}, "cards": {"openPerBoard": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "openPerList": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "totalPerBoard": {"status": "ok", "disableAt": 2000000, "warnAt": 1600000}, "totalPerList": {"status": "ok", "disableAt": 1000000, "warnAt": 800000}}, "checklists": {"perBoard": {"status": "ok", "disableAt": 1800000, "warnAt": 1440000}, "perCard": {"status": "ok", "disableAt": 500, "warnAt": 400}}, "checkItems": {"perChecklist": {"status": "ok", "disableAt": 200, "warnAt": 160}}, "customFields": {"perBoard": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "customFieldOptions": {"perField": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "labels": {"perBoard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "lists": {"openPerBoard": {"status": "ok", "disableAt": 500, "warnAt": 400}, "totalPerBoard": {"status": "ok", "disableAt": 3000, "warnAt": 2400}}, "stickers": {"perCard": {"status": "ok", "disableAt": 70, "warnAt": 56}}, "reactions": {"perAction": {"status": "ok", "disableAt": 900, "warnAt": 720}, "uniquePerAction": {"status": "ok", "disableAt": 17, "warnAt": 14}}}, "pinned": false, "starred": false, "url": "https://trello.com/b/Lj11WRiJ/testboard", "prefs": {"permissionLevel": "org", "hideVotes": false, "voting": "disabled", "comments": "members", "invitations": "members", "selfJoin": true, "cardCovers": true, "isTemplate": false, "cardAging": "regular", "calendarFeedEnabled": false, "hiddenPluginBoardButtons": [], "switcherViews": [{"viewType": "Board", "enabled": true}, {"viewType": "Table", "enabled": true}, {"viewType": "Calendar", "enabled": false}, {"viewType": "Dashboard", "enabled": false}, {"viewType": "Timeline", "enabled": false}, {"viewType": "Map", "enabled": false}], "background": "611a97ed741bfb5782acf105", "backgroundColor": null, "backgroundImage": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730", "backgroundImageScaled": [{"width": 67, "height": 100, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/67x100/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 128, "height": 192, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/128x192/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 320, "height": 480, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/320x480/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 640, "height": 960, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/640x960/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 683, "height": 1024, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/683x1024/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 854, "height": 1280, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/854x1280/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1067, "height": 1600, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1067x1600/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1281, "height": 1920, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1281x1920/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1366, "height": 2048, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730"}], "backgroundTile": false, "backgroundBrightness": "light", "backgroundBottomColor": "#7c8280", "backgroundTopColor": "#aeb0ae", "canBePublic": true, "canBeEnterprise": true, "canBeOrg": true, "canBePrivate": true, "canInvite": true}, "shortLink": "Lj11WRiJ", "subscribed": false, "labelNames": {"green": "", "yellow": "", "orange": "", "red": "", "purple": "", "blue": "", "sky": "", "lime": "", "pink": "", "black": "", "green_dark": "", "yellow_dark": "", "orange_dark": "", "red_dark": "", "purple_dark": "", "blue_dark": "", "sky_dark": "", "lime_dark": "", "pink_dark": "", "black_dark": "", "green_light": "", "yellow_light": "", "orange_light": "", "red_light": "", "purple_light": "", "blue_light": "", "sky_light": "", "lime_light": "", "pink_light": "", "black_light": ""}, "powerUps": [], "dateLastActivity": "2023-03-16T18:47:16.386Z", "dateLastView": "2023-03-16T20:10:44.953Z", "shortUrl": "https://trello.com/b/Lj11WRiJ", "idTags": [], "datePluginDisable": null, "creationMethod": "automatic", "ixUpdate": "90", "templateGallery": null, "enterpriseOwned": false, "idBoardSource": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "idMemberCreator": "610be3762899a26d04256dae", "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "611aa0ef37acd675af67dc9f"}]}, "emitted_at": 1689923514346} +{"stream": "boards", "data": {"id": "611aa586ef5f2c8e1deec8b6", "nodeId": "ari:cloud:trello::board/workspace/610be50a0537086c571a5684/611aa586ef5f2c8e1deec8b6", "name": "test_board_new", "desc": "", "descData": null, "closed": false, "dateClosed": null, "idOrganization": "610be50a0537086c571a5684", "idEnterprise": null, "limits": {"attachments": {"perBoard": {"status": "ok", "disableAt": 36000, "warnAt": 28800}, "perCard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "boards": {"totalMembersPerBoard": {"status": "ok", "disableAt": 1600, "warnAt": 1280}, "totalAccessRequestsPerBoard": {"status": "ok", "disableAt": 4000, "warnAt": 3200}}, "cards": {"openPerBoard": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "openPerList": {"status": "ok", "disableAt": 5000, "warnAt": 4000}, "totalPerBoard": {"status": "ok", "disableAt": 2000000, "warnAt": 1600000}, "totalPerList": {"status": "ok", "disableAt": 1000000, "warnAt": 800000}}, "checklists": {"perBoard": {"status": "ok", "disableAt": 1800000, "warnAt": 1440000}, "perCard": {"status": "ok", "disableAt": 500, "warnAt": 400}}, "checkItems": {"perChecklist": {"status": "ok", "disableAt": 200, "warnAt": 160}}, "customFields": {"perBoard": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "customFieldOptions": {"perField": {"status": "ok", "disableAt": 50, "warnAt": 40}}, "labels": {"perBoard": {"status": "ok", "disableAt": 1000, "warnAt": 800}}, "lists": {"openPerBoard": {"status": "ok", "disableAt": 500, "warnAt": 400}, "totalPerBoard": {"status": "ok", "disableAt": 3000, "warnAt": 2400}}, "stickers": {"perCard": {"status": "ok", "disableAt": 70, "warnAt": 56}}, "reactions": {"perAction": {"status": "ok", "disableAt": 900, "warnAt": 720}, "uniquePerAction": {"status": "ok", "disableAt": 17, "warnAt": 14}}}, "pinned": false, "starred": false, "url": "https://trello.com/b/qpYBekjt/testboardnew", "prefs": {"permissionLevel": "org", "hideVotes": false, "voting": "disabled", "comments": "members", "invitations": "members", "selfJoin": true, "cardCovers": true, "isTemplate": false, "cardAging": "regular", "calendarFeedEnabled": false, "hiddenPluginBoardButtons": [], "switcherViews": [{"viewType": "Board", "enabled": true}, {"viewType": "Table", "enabled": true}, {"viewType": "Calendar", "enabled": false}, {"viewType": "Dashboard", "enabled": false}, {"viewType": "Timeline", "enabled": false}, {"viewType": "Map", "enabled": false}], "background": "611a97ed741bfb5782acf105", "backgroundColor": null, "backgroundImage": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730", "backgroundImageScaled": [{"width": 67, "height": 100, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/67x100/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 128, "height": 192, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/128x192/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 320, "height": 480, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/320x480/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 640, "height": 960, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/640x960/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 683, "height": 1024, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/683x1024/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 854, "height": 1280, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/854x1280/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1067, "height": 1600, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1067x1600/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1281, "height": 1920, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/1281x1920/60a452e3b6247993c45a17686603549a/photo-1629006307992-dc1891987730.jpg"}, {"width": 1366, "height": 2048, "url": "https://trello-backgrounds.s3.amazonaws.com/SharedBackground/original/a2614b98135f746b4b1d38a1eef0240c/photo-1629006307992-dc1891987730"}], "backgroundTile": false, "backgroundBrightness": "light", "backgroundBottomColor": "#7c8280", "backgroundTopColor": "#aeb0ae", "canBePublic": true, "canBeEnterprise": true, "canBeOrg": true, "canBePrivate": true, "canInvite": true}, "shortLink": "qpYBekjt", "subscribed": false, "labelNames": {"green": "", "yellow": "", "orange": "", "red": "", "purple": "", "blue": "", "sky": "", "lime": "", "pink": "", "black": "", "green_dark": "", "yellow_dark": "", "orange_dark": "", "red_dark": "", "purple_dark": "", "blue_dark": "", "sky_dark": "", "lime_dark": "", "pink_dark": "", "black_dark": "", "green_light": "", "yellow_light": "", "orange_light": "", "red_light": "", "purple_light": "", "blue_light": "", "sky_light": "", "lime_light": "", "pink_light": "", "black_light": ""}, "powerUps": [], "dateLastActivity": "2023-03-16T20:10:54.595Z", "dateLastView": "2023-03-17T11:01:50.283Z", "shortUrl": "https://trello.com/b/qpYBekjt", "idTags": [], "datePluginDisable": null, "creationMethod": "automatic", "ixUpdate": "52", "templateGallery": null, "enterpriseOwned": false, "idBoardSource": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "idMemberCreator": "610be3762899a26d04256dae", "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "611aa586ef5f2c8e1deec8ba"}]}, "emitted_at": 1689923514348} +{"stream":"cards","data":{"id":"6127a3488a027b342502ba23","badges":{"attachmentsByType":{"trello":{"board":0,"card":0}},"location":false,"votes":0,"viewingMemberVoted":false,"subscribed":false,"fogbugz":"","checkItems":0,"checkItemsChecked":0,"checkItemsEarliestDue":null,"comments":0,"attachments":0,"description":false,"due":null,"dueComplete":false,"start":null},"checkItemStates":[],"closed":true,"dueComplete":false,"dateLastActivity":"2021-08-26T14:20:58.508Z","desc":"","descData":null,"due":null,"dueReminder":null,"email":null,"idBoard":"611aa0ef37acd675af67dc9b","idChecklists":[],"idList":"611aa0ef37acd675af67dc9d","idMembers":[],"idMembersVoted":[],"idShort":7,"idAttachmentCover":null,"labels":[],"idLabels":[],"manualCoverAttachment":false,"name":"Demo_test","pos":32767.5,"shortLink":"bbhWatIg","shortUrl":"https://trello.com/c/bbhWatIg","start":null,"subscribed":false,"url":"https://trello.com/c/bbhWatIg/7-demotest","cover":{"idAttachment":null,"color":null,"idUploadedBackground":null,"size":"normal","brightness":"dark","idPlugin":null},"isTemplate":false,"cardRole":null,"pluginData":[],"customFieldItems":[],"members":[]},"emitted_at":1687178464945} +{"stream":"cards","data":{"id":"6127a338b1d0335b7d609c28","badges":{"attachmentsByType":{"trello":{"board":0,"card":0}},"location":false,"votes":0,"viewingMemberVoted":false,"subscribed":false,"fogbugz":"","checkItems":0,"checkItemsChecked":0,"checkItemsEarliestDue":null,"comments":0,"attachments":0,"description":false,"due":null,"dueComplete":false,"start":null},"checkItemStates":[],"closed":true,"dueComplete":false,"dateLastActivity":"2021-08-26T14:20:42.690Z","desc":"","descData":null,"due":null,"dueReminder":null,"email":null,"idBoard":"611aa0ef37acd675af67dc9b","idChecklists":[],"idList":"611aa0ef37acd675af67dc9c","idMembers":[],"idMembersVoted":[],"idShort":6,"idAttachmentCover":null,"labels":[],"idLabels":[],"manualCoverAttachment":false,"name":"demo_test","pos":16383.75,"shortLink":"NbePPuOM","shortUrl":"https://trello.com/c/NbePPuOM","start":null,"subscribed":false,"url":"https://trello.com/c/NbePPuOM/6-demotest","cover":{"idAttachment":null,"color":null,"idUploadedBackground":null,"size":"normal","brightness":"dark","idPlugin":null},"isTemplate":false,"cardRole":null,"pluginData":[],"customFieldItems":[],"members":[]},"emitted_at":1687178464948} +{"stream":"cards","data":{"id":"612628f089a80219c8caa562","badges":{"attachmentsByType":{"trello":{"board":0,"card":0}},"location":false,"votes":0,"viewingMemberVoted":false,"subscribed":false,"fogbugz":"","checkItems":0,"checkItemsChecked":0,"checkItemsEarliestDue":null,"comments":0,"attachments":0,"description":false,"due":null,"dueComplete":false,"start":null},"checkItemStates":[],"closed":true,"dueComplete":false,"dateLastActivity":"2021-08-25T11:26:41.767Z","desc":"","descData":null,"due":null,"dueReminder":null,"email":null,"idBoard":"611aa0ef37acd675af67dc9b","idChecklists":[],"idList":"611aa0ef37acd675af67dc9c","idMembers":[],"idMembersVoted":[],"idShort":5,"idAttachmentCover":null,"labels":[],"idLabels":[],"manualCoverAttachment":false,"name":"new_test","pos":16383.75,"shortLink":"npcCoHA2","shortUrl":"https://trello.com/c/npcCoHA2","start":null,"subscribed":false,"url":"https://trello.com/c/npcCoHA2/5-newtest","cover":{"idAttachment":null,"color":null,"idUploadedBackground":null,"size":"normal","brightness":"dark","idPlugin":null},"isTemplate":false,"cardRole":null,"pluginData":[],"customFieldItems":[],"members":[]},"emitted_at":1687178464949} {"stream": "checklists", "data": {"id": "611ac881dccde179ffc692a6", "name": "Checklist", "idBoard": "611aa0ef37acd675af67dc9b", "idCard": "611aa19024689d7d2ca8b025", "pos": 16384, "limits": {"checkItems": {"perChecklist": {"status": "ok", "disableAt": 200, "warnAt": 160}}}, "checkItems": [{"id": "611ac8846914570b0c18c69d", "name": "test", "nameData": null, "pos": 17372, "state": "incomplete", "due": null, "dueReminder": null, "idMember": null, "idChecklist": "611ac881dccde179ffc692a6"}, {"id": "611ac8862fa38e7189b0a998", "name": "test", "nameData": null, "pos": 33793, "state": "complete", "due": null, "dueReminder": null, "idMember": null, "idChecklist": "611ac881dccde179ffc692a6"}], "creationMethod": null}, "emitted_at": 1683406409615} {"stream": "checklists", "data": {"id": "611ab11ff5162b8b5b82e717", "name": "Checklist", "idBoard": "611aa586ef5f2c8e1deec8b6", "idCard": "611aa58c6e865e4c1c76727e", "pos": 16384, "limits": {"checkItems": {"perChecklist": {"status": "ok", "disableAt": 200, "warnAt": 160}}}, "checkItems": [{"id": "611ab123c3617b6c6d97272d", "name": "test", "nameData": null, "pos": 16823, "state": "incomplete", "due": null, "dueReminder": null, "idMember": null, "idChecklist": "611ab11ff5162b8b5b82e717"}, {"id": "611ab126bbd321152136bcc9", "name": "test_1", "nameData": null, "pos": 33688, "state": "complete", "due": null, "dueReminder": null, "idMember": null, "idChecklist": "611ab11ff5162b8b5b82e717"}], "creationMethod": null}, "emitted_at": 1683406409806} {"stream": "lists", "data": {"id": "611aa0ef37acd675af67dc9c", "name": "To Do", "closed": false, "idBoard": "611aa0ef37acd675af67dc9b", "pos": 16384, "subscribed": false, "softLimit": null, "status": null}, "emitted_at": 1683406411202} @@ -13,4 +13,4 @@ {"stream": "lists", "data": {"id": "611aa0ef37acd675af67dc9e", "name": "Done", "closed": false, "idBoard": "611aa0ef37acd675af67dc9b", "pos": 49152, "subscribed": false, "softLimit": null, "status": null}, "emitted_at": 1683406411203} {"stream": "users", "data": {"id": "610be3762899a26d04256dae", "fullName": "integration test", "username": "integrationtest19"}, "emitted_at": 1683406412874} {"stream": "users", "data": {"id": "610be3762899a26d04256dae", "fullName": "integration test", "username": "integrationtest19"}, "emitted_at": 1683406413051} -{"stream": "organizations", "data": {"id": "610be50a0537086c571a5684", "creationMethod": null, "name": "airbyteworkspace", "credits": [], "displayName": "Airbyte Workspace", "desc": "", "descData": {"emoji": {}}, "domainName": "airbyte.io", "idBoards": ["611aa0ef37acd675af67dc9b", "611aa586ef5f2c8e1deec8b6"], "idEnterprise": null, "idMemberCreator": "610be3762899a26d04256dae", "invited": false, "invitations": [], "limits": {"orgs": {"totalMembersPerOrg": {"status": "ok", "disableAt": 4000, "warnAt": 3200}, "freeBoardsPerOrg": {"status": "ok", "disableAt": 10, "warnAt": 3}}}, "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "610be50a0537086c571a5685"}], "membersCount": 1, "prefs": {"permissionLevel": "private", "orgInviteRestrict": [], "boardInviteRestrict": "any", "externalMembersDisabled": false, "associatedDomain": null, "googleAppsVersion": 1, "boardVisibilityRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "boardDeleteRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "attachmentRestrictions": null, "newLicenseInviteRestrict": null}, "powerUps": [], "products": [], "billableMemberCount": 1, "billableCollaboratorCount": 0, "url": "https://trello.com/w/airbyteworkspace", "website": null, "logoHash": null, "logoUrl": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "promotions": [], "enterpriseJoinRequest": {}, "standardVariation": null, "availableLicenseCount": null, "maximumLicenseCount": null, "ixUpdate": "6", "teamType": null}, "emitted_at": 1683406413871} +{"stream": "organizations", "data": {"id": "610be50a0537086c571a5684", "creationMethod": null, "name": "airbyteworkspace", "credits": [], "displayName": "Airbyte Workspace", "desc": "", "descData": {"emoji": {}}, "domainName": "airbyte.io", "idBoards": ["611aa0ef37acd675af67dc9b", "611aa586ef5f2c8e1deec8b6"], "idEnterprise": null, "idMemberCreator": "610be3762899a26d04256dae", "invited": false, "invitations": [], "limits": {"orgs": {"totalMembersPerOrg": {"status": "ok", "disableAt": 4000, "warnAt": 3200}, "freeBoardsPerOrg": {"status": "ok", "disableAt": 10, "warnAt": 3}}}, "membersCount": 1, "prefs": {"permissionLevel": "private", "orgInviteRestrict": [], "boardInviteRestrict": "any", "externalMembersDisabled": false, "associatedDomain": null, "googleAppsVersion": 1, "boardVisibilityRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "boardDeleteRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "attachmentRestrictions": null, "seatAutomation": null, "seatAutomationActiveDays": null, "seatAutomationInactiveDays": null, "newLicenseInviteRestrict": null, "newLicenseInviteRestrictUrl": null, "atlassianIntelligenceEnabled": false}, "powerUps": [], "products": [], "billableMemberCount": 1, "billableCollaboratorCount": 0, "url": "https://trello.com/w/airbyteworkspace", "website": null, "logoHash": null, "logoUrl": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "promotions": [], "enterpriseJoinRequest": {}, "standardVariation": null, "availableLicenseCount": null, "maximumLicenseCount": null, "ixUpdate": "6", "teamType": null, "dateLastActivity": "2023-03-16T20:10:54.595Z", "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "610be50a0537086c571a5685"}]}, "emitted_at": 1690381821566} diff --git a/airbyte-integrations/connectors/source-trello/metadata.yaml b/airbyte-integrations/connectors/source-trello/metadata.yaml index f77b1b8ab5b7..8d72b0bc1636 100644 --- a/airbyte-integrations/connectors/source-trello/metadata.yaml +++ b/airbyte-integrations/connectors/source-trello/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 8da67652-004c-11ec-9a03-0242ac130003 - dockerImageTag: 0.3.2 + dockerImageTag: 0.3.4 dockerRepository: airbyte/source-trello githubIssueLabel: source-trello icon: trello.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/trello tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-trello/requirements.txt b/airbyte-integrations/connectors/source-trello/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-trello/requirements.txt +++ b/airbyte-integrations/connectors/source-trello/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-trello/setup.py b/airbyte-integrations/connectors/source-trello/setup.py index 34c123dff159..4209a8aabf1b 100644 --- a/airbyte-integrations/connectors/source-trello/setup.py +++ b/airbyte-integrations/connectors/source-trello/setup.py @@ -11,7 +11,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json b/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json index c88c17c9c09e..ee290179a228 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-trello/source_trello/schemas/organizations.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -196,6 +196,9 @@ }, "newLicenseInviteRestrict": { "type": ["null", "string"] + }, + "newLicenseInviteRestrictUrl": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-trello/source_trello/spec.json b/airbyte-integrations/connectors/source-trello/source_trello/spec.json index 632e2a78d0f4..5576ac580c4f 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/spec.json +++ b/airbyte-integrations/connectors/source-trello/source_trello/spec.json @@ -70,20 +70,6 @@ "type": "string" } } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } } } } diff --git a/airbyte-integrations/connectors/source-trustpilot/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-trustpilot/integration_tests/invalid_config.json index dd916f2dd87b..18ba88b6b60f 100644 --- a/airbyte-integrations/connectors/source-trustpilot/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-trustpilot/integration_tests/invalid_config.json @@ -1,7 +1,7 @@ { "credentials": { "auth_type": "oauth2.0", - "client_id": "aoVLPAXHFwryzNepiAY3yR7urlx9hhGr", + "client_id": "aoVLPAXHFwryzNepiAY3yR7urlx9hhGr", "client_secret": "EDLVnp9t11Oa375D", "access_token": "MB4o9G3MpM5yrdIKo0OcYsOjeaBT", "token_expiry_date": "2023-03-15T00:11:00Z", diff --git a/airbyte-integrations/connectors/source-trustpilot/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-trustpilot/integration_tests/sample_config.json index a911e26242e9..75908bd4f8a9 100644 --- a/airbyte-integrations/connectors/source-trustpilot/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-trustpilot/integration_tests/sample_config.json @@ -1,14 +1,12 @@ { "credentials": { "auth_type": "oauth2.0", - "client_id": "aoVLPAXHFwryzNepiAY3yR7urlx9hhGr", + "client_id": "aoVLPAXHFwryzNepiAY3yR7urlx9hhGr", "client_secret": "EDLVnp9t11Oa375D", "access_token": "MB4o9G3MpM5yrdIKo0OcYsOjeaBT", "token_expiry_date": "2023-03-15T00:11:00Z", "refresh_token": "ttfqulk6gPg2TOPL4wS23m6ZBR7wEsqG" }, - "business_units": [ - "my_domain.com" - ], + "business_units": ["my_domain.com"], "start_date": "2023-03-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-trustpilot/metadata.yaml b/airbyte-integrations/connectors/source-trustpilot/metadata.yaml index 773e465b0da9..96e957b54a11 100644 --- a/airbyte-integrations/connectors/source-trustpilot/metadata.yaml +++ b/airbyte-integrations/connectors/source-trustpilot/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/trustpilot tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-trustpilot/requirements.txt b/airbyte-integrations/connectors/source-trustpilot/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-trustpilot/requirements.txt +++ b/airbyte-integrations/connectors/source-trustpilot/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-trustpilot/setup.py b/airbyte-integrations/connectors/source-trustpilot/setup.py index a0927bb4e2ba..4b8001807ec1 100644 --- a/airbyte-integrations/connectors/source-trustpilot/setup.py +++ b/airbyte-integrations/connectors/source-trustpilot/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/schemas/private_reviews.json b/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/schemas/private_reviews.json index 3fd2f129396d..34f857b7f57a 100644 --- a/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/schemas/private_reviews.json +++ b/airbyte-integrations/connectors/source-trustpilot/source_trustpilot/schemas/private_reviews.json @@ -123,53 +123,53 @@ "type": "object", "properties": { "isEligible": { - "type": "boolean" + "type": "boolean" }, "requests": { - "type": "array", - "items": { - "type": "object", - "properties": { - "status": { - "type": ["string", "null"] - }, - "consumerResponse": { - "type": ["object","null"], - "properties": { - "referenceId": { - "type": ["string", "null"] - }, - "name": { - "type": ["string", "null"] - }, - "submittedAt": { - "type": ["string", "null"], - "format": "date-time" - }, - "phoneNumber": { - "type": ["string", "null"] - }, - "address": { - "type": ["string", "null"] - }, - "message": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - } + "type": "array", + "items": { + "type": "object", + "properties": { + "status": { + "type": ["string", "null"] + }, + "consumerResponse": { + "type": ["object", "null"], + "properties": { + "referenceId": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "submittedAt": { + "type": ["string", "null"], + "format": "date-time" + }, + "phoneNumber": { + "type": ["string", "null"] + }, + "address": { + "type": ["string", "null"] + }, + "message": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] } - }, - "businessUserMessage": { - "type": ["string", "null"] - }, - "id": { - "type": ["string", "null"] - }, - "created": { - "type": ["string", "null"], - "format": "date-time" } + }, + "businessUserMessage": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "created": { + "type": ["string", "null"], + "format": "date-time" + } } } } @@ -219,7 +219,7 @@ "type": ["boolean", "null"] }, "invitation": { - "type": ["object","null"], + "type": ["object", "null"], "properties": { "businessUnitId": { "type": ["string", "null"] @@ -251,4 +251,4 @@ "type": ["string", "null"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml b/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml index 697ee5f79fa7..0da8f2802ab0 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt b/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py index 2880c2daa2d2..77c97057564e 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml b/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml index 66a7ad2b9f56..69d1bbb070b4 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/requirements.txt b/airbyte-integrations/connectors/source-twilio-taskrouter/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/requirements.txt +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py b/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py index 1e3c44ed9019..be86feef816f 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile index bc5063c02f26..b8e2f02135cb 100644 --- a/airbyte-integrations/connectors/source-twilio/Dockerfile +++ b/airbyte-integrations/connectors/source-twilio/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.7.0 +LABEL io.airbyte.version=0.10.0 LABEL io.airbyte.name=airbyte/source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml index 78fe978d9096..9c1849dc0b9a 100644 --- a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml @@ -46,6 +46,10 @@ acceptance_tests: bypass_reason: "very volatile data" - name: "usage_records" bypass_reason: "very volatile data" + - name: step + bypass_reason: "No data for this stream currently" + - name: executions + bypass_reason: "No data for this stream currently" timeout_seconds: 600 fail_on_extra_columns: false incremental: diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/constant_records_catalog.json b/airbyte-integrations/connectors/source-twilio/integration_tests/constant_records_catalog.json index 04795e8d144a..b683b79f692c 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/constant_records_catalog.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/constant_records_catalog.json @@ -227,6 +227,42 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "step", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "user_conversations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "verify_services", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl index f36b1c30e145..330566a9ea1a 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl @@ -1,66 +1,57 @@ -{"stream": "addresses", "data": {"sid": "ADa29b1ee20cf61d213f7d7f1a3298309a", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": null, "customer_name": "test-customer_name_1", "street": "test-street_1", "street_secondary": null, "city": "test-city_1", "region": "test-region_1", "postal_code": "test-postal_code_1", "iso_country": "US", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses/ADa29b1ee20cf61d213f7d7f1a3298309a.json", "date_created": "2020-11-25T09:41:48Z", "date_updated": "2020-11-25T09:41:48Z", "emergency_enabled": false, "validated": false, "verified": false}, "emitted_at": 1682602854052} -{"stream": "addresses", "data": {"sid": "ADc5e31ae6ae46befadd5c3f053c5a7153", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "Test Addr 2", "customer_name": "Test Customer 2", "street": "Test Street 2", "street_secondary": null, "city": "Test sity 2", "region": "CA", "postal_code": "94121", "iso_country": "US", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses/ADc5e31ae6ae46befadd5c3f053c5a7153.json", "date_created": "2023-02-14T15:46:16Z", "date_updated": "2023-02-14T15:46:16Z", "emergency_enabled": false, "validated": false, "verified": false}, "emitted_at": 1682602854054} -{"stream": "addresses", "data": {"sid": "ADeca271710de05eb798c9c6df10215e8b", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "Adress1", "customer_name": "Contact", "street": "2261 Market Street #4381", "street_secondary": "00805 Walton Road, 4381", "city": "San Francisco", "region": "CA", "postal_code": "94114", "iso_country": "US", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses/ADeca271710de05eb798c9c6df10215e8b.json", "date_created": "2022-11-22T15:24:00Z", "date_updated": "2022-11-22T15:27:27Z", "emergency_enabled": false, "validated": false, "verified": false}, "emitted_at": 1682602854056} -{"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/APd6232730849b51fb86fa20a8081fa27c.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "APd6232730849b51fb86fa20a8081fa27c", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1682602855500} -{"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/APe7ed98d5222e25db0938c1efc5c661b2.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "APe7ed98d5222e25db0938c1efc5c661b2", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1682602855502} -{"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP731b039bbb9103a1ae2f0afbe85949d4.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP731b039bbb9103a1ae2f0afbe85949d4", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1682602855503} -{"stream": "applications", "data": {"sms_status_callback": null, "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2020-11-25T09:47:31Z", "sms_fallback_method": "POST", "friendly_name": "Test friendly name", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP1c10c50172412d3a65dfd7395d11640f.json", "sms_fallback_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP1c10c50172412d3a65dfd7395d11640f", "date_created": "2020-11-25T09:47:31Z", "sms_url": null, "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": null}, "emitted_at": 1682602855505} -{"stream": "available_phone_number_countries", "data": {"country_code": "BA", "country": "Bosnia and Herzegovina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BA.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BA/Local.json"}}, "emitted_at": 1682602857571} -{"stream": "available_phone_number_countries", "data": {"country_code": "BJ", "country": "Benin", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BJ.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BJ/Mobile.json"}}, "emitted_at": 1682602857571} -{"stream": "available_phone_number_countries", "data": {"country_code": "BB", "country": "Barbados", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BB.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/BB/Local.json"}}, "emitted_at": 1682602857572} -{"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1682602857572} -{"stream": "available_phone_number_countries", "data": {"country_code": "AR", "country": "Argentina", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/Local.json", "toll_free": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/AvailablePhoneNumbers/AR/TollFree.json"}}, "emitted_at": 1682602857573} -{"stream": "calls", "data": {"date_updated": "2023-03-15T11:46:33Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": "", "duration": 0, "from": "+12056561170", "to": "+14156236785", "annotation": null, "answered_by": null, "sid": "CA630cc72be341dc7bee221d88a9ca237c", "queue_time": 0, "price": null, "api_version": "2010-04-01", "status": "busy", "direction": "outbound-api", "start_time": "2023-03-15T11:46:11Z", "date_created": "2023-03-15T11:46:11Z", "from_formatted": "(205) 656-1170", "group_sid": null, "trunk_sid": "", "forwarded_from": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-03-15T11:46:33Z", "to_formatted": "(415) 623-6785", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA630cc72be341dc7bee221d88a9ca237c/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1682602860153} -{"stream": "calls", "data": {"date_updated": "2023-03-15T11:46:34Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": "", "duration": 0, "from": "+12056561170", "to": "+14156236785", "annotation": null, "answered_by": null, "sid": "CA7a5d05a684c40d50187701f0b906cfd4", "queue_time": 0, "price": null, "api_version": "2010-04-01", "status": "busy", "direction": "outbound-api", "start_time": "2023-03-15T11:46:18Z", "date_created": "2023-03-15T11:46:18Z", "from_formatted": "(205) 656-1170", "group_sid": null, "trunk_sid": "", "forwarded_from": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-03-15T11:46:34Z", "to_formatted": "(415) 623-6785", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7a5d05a684c40d50187701f0b906cfd4/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1682602860155} -{"stream": "calls", "data": {"date_updated": "2023-03-15T11:46:35Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": "", "duration": 0, "from": "+12056561170", "to": "+14156236785", "annotation": null, "answered_by": null, "sid": "CAadd59be8099005b4c1433ca9fa2ccfe6", "queue_time": 0, "price": null, "api_version": "2010-04-01", "status": "busy", "direction": "outbound-api", "start_time": "2023-03-15T11:46:19Z", "date_created": "2023-03-15T11:46:19Z", "from_formatted": "(205) 656-1170", "group_sid": null, "trunk_sid": "", "forwarded_from": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-03-15T11:46:35Z", "to_formatted": "(415) 623-6785", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAadd59be8099005b4c1433ca9fa2ccfe6/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1682602860158} -{"stream": "calls", "data": {"date_updated": "2023-03-15T11:46:39Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": "", "duration": 0, "from": "+12056561170", "to": "+14156236785", "annotation": null, "answered_by": null, "sid": "CA7b08148666a4d91a2cbfa335b92ddf5c", "queue_time": 6000, "price": null, "api_version": "2010-04-01", "status": "busy", "direction": "outbound-api", "start_time": "2023-03-15T11:46:13Z", "date_created": "2023-03-15T11:46:13Z", "from_formatted": "(205) 656-1170", "group_sid": null, "trunk_sid": "", "forwarded_from": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-03-15T11:46:39Z", "to_formatted": "(415) 623-6785", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA7b08148666a4d91a2cbfa335b92ddf5c/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1682602860160} -{"stream": "calls", "data": {"date_updated": "2023-04-03T20:04:23Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 32, "from": "+19125503157", "to": "+19125901057", "annotation": null, "answered_by": null, "sid": "CAd6d69b48d5dd857fc0c8640234e3f6fc", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2023-04-03T20:03:51Z", "date_created": "2023-04-03T20:03:51Z", "from_formatted": "(912) 550-3157", "group_sid": null, "trunk_sid": "", "forwarded_from": "+19125901057", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-04-03T20:04:23Z", "to_formatted": "(912) 590-1057", "phone_number_sid": "PN99400a65bf5a4305d5420060842d4d2c", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAd6d69b48d5dd857fc0c8640234e3f6fc/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1682602860163} -{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-03-15T11:38:29Z", "region": "us1", "friendly_name": "EH5793263d703ad674bbcdeb31ac80e359", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF321885a5b10a0703fd571166e1265141.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA79c69bbd0c31771eab5f6a7faa15331c", "sid": "CF321885a5b10a0703fd571166e1265141", "date_created": "2023-03-15T11:38:12Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF321885a5b10a0703fd571166e1265141/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF321885a5b10a0703fd571166e1265141/Recordings.json"}}, "emitted_at": 1682602861807} -{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-03-15T11:38:48Z", "region": "us1", "friendly_name": "EH85d42ab639efdcbdbe8da21f798d00af", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF2b53c04f42bc7879b1c17e7715172d88.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA602fbb208d79d96b051e29b2554084cd", "sid": "CF2b53c04f42bc7879b1c17e7715172d88", "date_created": "2023-03-15T11:38:31Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF2b53c04f42bc7879b1c17e7715172d88/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF2b53c04f42bc7879b1c17e7715172d88/Recordings.json"}}, "emitted_at": 1682602861808} -{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-03-15T11:46:39Z", "region": "us1", "friendly_name": "EH85d42ab639efdcbdbe8da21f798d00af", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF00e87d952b927f55f6abb3f9d9a1b91c.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA7b08148666a4d91a2cbfa335b92ddf5c", "sid": "CF00e87d952b927f55f6abb3f9d9a1b91c", "date_created": "2023-03-15T11:46:08Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF00e87d952b927f55f6abb3f9d9a1b91c/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF00e87d952b927f55f6abb3f9d9a1b91c/Recordings.json"}}, "emitted_at": 1682602861809} -{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-04-03T20:04:23Z", "region": "us1", "friendly_name": "Conference2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFc9ece7e3f80f85e05a61c69faee62a2a.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CAd6d69b48d5dd857fc0c8640234e3f6fc", "sid": "CFc9ece7e3f80f85e05a61c69faee62a2a", "date_created": "2023-04-03T20:03:52Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFc9ece7e3f80f85e05a61c69faee62a2a/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFc9ece7e3f80f85e05a61c69faee62a2a/Recordings.json"}}, "emitted_at": 1682602861810} -{"stream": "conversations", "data": {"unique_name": null, "date_updated": "2023-03-21T13:39:44Z", "friendly_name": "Friendly Conversation", "timers": {}, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "state": "active", "date_created": "2023-03-21T13:39:44Z", "messaging_service_sid": "MGfdf707ca9a7e03496ad79dc64e5e543e", "sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "attributes": "{}", "bindings": null, "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": {"participants": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants", "messages": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages", "webhooks": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Webhooks"}}, "emitted_at": 1682602862574} -{"stream": "conversation_messages", "data": {"body": "Ahoy there", "index": 0, "author": "smee", "date_updated": "2023-04-01T12:37:19Z", "media": null, "participant_sid": null, "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "delivery": null, "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages/IMd28bbec7d60f4c9b84595170871c6f28", "date_created": "2023-04-01T12:37:19Z", "content_sid": null, "sid": "IMd28bbec7d60f4c9b84595170871c6f28", "attributes": "{}", "links": {"delivery_receipts": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages/IMd28bbec7d60f4c9b84595170871c6f28/Receipts"}}, "emitted_at": 1682602863760} -{"stream": "conversation_participants", "data": {"last_read_message_index": null, "date_updated": "2023-04-13T11:52:17Z", "last_read_timestamp": null, "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB0a984a4238f14b828cf277becf880bd4", "date_created": "2023-04-13T11:52:17Z", "role_sid": "RLca3ff6cb9bc9404caf14e43b63fed446", "sid": "MB0a984a4238f14b828cf277becf880bd4", "attributes": "{}", "identity": "Integration Test 2", "messaging_binding": null}, "emitted_at": 1682602864970} -{"stream": "conversation_participants", "data": {"last_read_message_index": null, "date_updated": "2023-04-13T11:52:02Z", "last_read_timestamp": null, "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB41bb002a3a5e412fa7f2459dcfb4925f", "date_created": "2023-04-13T11:52:02Z", "role_sid": "RLca3ff6cb9bc9404caf14e43b63fed446", "sid": "MB41bb002a3a5e412fa7f2459dcfb4925f", "attributes": "{}", "identity": "Integration Test", "messaging_binding": null}, "emitted_at": 1682602864972} -{"stream": "flows", "data": {"status": "published", "date_updated": "2022-09-23T14:31:33Z", "friendly_name": "conference_test", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://studio.twilio.com/v1/Flows/FW7ad717a690629a6da33bd3c8b9cf7d97", "version": 15, "sid": "FW7ad717a690629a6da33bd3c8b9cf7d97", "date_created": "2022-09-23T14:28:11Z", "links": {"engagements": "https://studio.twilio.com/v1/Flows/FW7ad717a690629a6da33bd3c8b9cf7d97/Engagements", "executions": "https://studio.twilio.com/v1/Flows/FW7ad717a690629a6da33bd3c8b9cf7d97/Executions"}}, "emitted_at": 1682602865755} -{"stream": "flows", "data": {"status": "published", "date_updated": "2021-06-23T23:04:40Z", "friendly_name": "SMS To slack", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "version": 18, "sid": "FWbd726b7110b21294a9f27a47f4ab0080", "date_created": "2021-02-01T07:22:57Z", "links": {"engagements": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Engagements", "executions": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Executions"}}, "emitted_at": 1682602865756} -{"stream": "executions", "data": {"status": "ended", "date_updated": "2023-05-26T15:34:49Z", "contact_channel_address": "+380636306253", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Executions/FN8a53edd7a12a4b6ea4670db32f69b0d4", "context": {}, "sid": "FN8a53edd7a12a4b6ea4670db32f69b0d4", "date_created": "2023-05-26T15:34:49Z", "contact_sid": "FC6e50c6ac811cb5ea82c72b5ded9766f9", "flow_sid": "FWbd726b7110b21294a9f27a47f4ab0080", "links": {"steps": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Executions/FN8a53edd7a12a4b6ea4670db32f69b0d4/Steps", "execution_context": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Executions/FN8a53edd7a12a4b6ea4670db32f69b0d4/Context"}}, "emitted_at": 1685115592196} -{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T08:00:58Z", "voice_url": "https://handler.twilio.com/twiml/EH7af811843f38093d724a5c2e80b3eabe", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true }, "api_version": "2010-04-01", "sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+12056561170", "emergency_address_sid": null, "beta": false, "address_sid": "AD9cc2cc40dafe63c70e17ad3b8bfe9ffa", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "2FA Number - PLEASE DO NOT TOUCH. Use another number for anythin", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNe40bd7f3ac343b32fd51275d2d5b3dcc.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2020-12-11T04:28:40Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNe40bd7f3ac343b32fd51275d2d5b3dcc/AssignedAddOns.json"} }, "emitted_at": 1682602868613} -{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T08:01:20Z", "voice_url": "https://handler.twilio.com/twiml/EH3c0946e5d905d6563a71ef432575a1ff", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true }, "api_version": "2010-04-01", "sid": "PN8c084924cc64659889aaa98af937de56", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+12232174137", "emergency_address_sid": null, "beta": false, "address_sid": "ADa29b1ee20cf61d213f7d7f1a3298309a", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 7", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN8c084924cc64659889aaa98af937de56.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-16T14:29:03Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN8c084924cc64659889aaa98af937de56/AssignedAddOns.json"} }, "emitted_at": 1682602868615} -{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T08:01:44Z", "voice_url": "https://handler.twilio.com/twiml/EHcdb15ded7c5343ca4e52d85d4d94ebad", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true }, "api_version": "2010-04-01", "sid": "PN63c288b22a08ce3339371b4e6e10877e", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+19704017747", "emergency_address_sid": null, "beta": false, "address_sid": "ADc5e31ae6ae46befadd5c3f053c5a7153", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 4", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN63c288b22a08ce3339371b4e6e10877e.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-16T10:07:11Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN63c288b22a08ce3339371b4e6e10877e/AssignedAddOns.json"} }, "emitted_at": 1682602868617} -{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-23T13:05:32Z", "voice_url": "https://demo.twilio.com/welcome/voice/", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true }, "api_version": "2010-04-01", "sid": "PN24bccae6bf6f7aeb9ab1d4aad7e443af", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+14066376761", "emergency_address_sid": null, "beta": false, "address_sid": null, "sms_url": "https://demo.twilio.com/welcome/sms/reply", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Dependent Phone Number 1", "uri": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/IncomingPhoneNumbers/PN24bccae6bf6f7aeb9ab1d4aad7e443af.json", "sms_fallback_url": "", "account_sid": "AC4cac489c46197c9ebc91c840120a4dee", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-03-23T13:04:22Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/AC4cac489c46197c9ebc91c840120a4dee/IncomingPhoneNumbers/PN24bccae6bf6f7aeb9ab1d4aad7e443af/AssignedAddOns.json"} }, "emitted_at": 1682602868917} -{"stream": "message_media", "data": {"sid": "ME84dfe1b31474032acc79cdb6e793db7c", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM70ec51fd8ba9408302cdd16b98a47c81", "content_type": "image/png", "date_created": "2023-02-14T14:03:37Z", "date_updated": "2023-02-14T14:03:37Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81/Media/ME84dfe1b31474032acc79cdb6e793db7c.json"}, "emitted_at": 1682602936584} -{"stream": "message_media", "data": {"sid": "MEe1c564d5ca2b25f1f1ffaa87031625af", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM863b367b7d0725532b80a161c9dab4e5", "content_type": "image/png", "date_created": "2023-02-14T14:03:42Z", "date_updated": "2023-02-14T14:03:42Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM863b367b7d0725532b80a161c9dab4e5/Media/MEe1c564d5ca2b25f1f1ffaa87031625af.json"}, "emitted_at": 1682602936760} -{"stream": "message_media", "data": {"sid": "MEd3dfbf9f6f4b59bbb86a19a11e435371", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM5a93ae7d20c07ceae87cd2649485ba72", "content_type": "image/png", "date_created": "2023-02-14T14:03:47Z", "date_updated": "2023-02-14T14:03:47Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM5a93ae7d20c07ceae87cd2649485ba72/Media/MEd3dfbf9f6f4b59bbb86a19a11e435371.json"}, "emitted_at": 1682602936931} -{"stream": "message_media", "data": {"sid": "MEe3d25f69ded8cfdb9ea42b4ebeea1094", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM89c1a0785d96b5faa30d65aa644c70b4", "content_type": "image/png", "date_created": "2023-02-14T14:03:52Z", "date_updated": "2023-02-14T14:03:52Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM89c1a0785d96b5faa30d65aa644c70b4/Media/MEe3d25f69ded8cfdb9ea42b4ebeea1094.json"}, "emitted_at": 1682602937099} -{"stream": "messages", "data": {"body": "Test 7!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T16:00:41Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMfba455dd05ec9a6b9db40622fce7a0ea.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:12Z", "status": "sent", "sid": "MMfba455dd05ec9a6b9db40622fce7a0ea", "date_sent": "2023-02-14T14:03:15Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMfba455dd05ec9a6b9db40622fce7a0ea/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMfba455dd05ec9a6b9db40622fce7a0ea/Feedback.json"}}, "emitted_at": 1682602966549} -{"stream": "messages", "data": {"body": "Hi there, Test 6!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T16:03:03Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMdd3ef6d9c3158e5d8e60ea156f8a8d40.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:26Z", "status": "sent", "sid": "MMdd3ef6d9c3158e5d8e60ea156f8a8d40", "date_sent": "2023-02-14T14:03:26Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMdd3ef6d9c3158e5d8e60ea156f8a8d40/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMdd3ef6d9c3158e5d8e60ea156f8a8d40/Feedback.json"}}, "emitted_at": 1682602966551} -{"stream": "messages", "data": {"body": "Hi there, Test 5!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T16:08:15Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMf112c5528853756d2a01de58caa8b820.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:32Z", "status": "sent", "sid": "MMf112c5528853756d2a01de58caa8b820", "date_sent": "2023-02-14T14:03:33Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMf112c5528853756d2a01de58caa8b820/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMf112c5528853756d2a01de58caa8b820/Feedback.json"}}, "emitted_at": 1682602966553} -{"stream": "messages", "data": {"body": "Hi there, Test 4!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T16:03:17Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:37Z", "status": "sent", "sid": "MM70ec51fd8ba9408302cdd16b98a47c81", "date_sent": "2023-02-14T14:03:38Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81/Feedback.json"}}, "emitted_at": 1682602966554} -{"stream": "messages", "data": {"body": "Hi there, Test 3!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T15:53:24Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM863b367b7d0725532b80a161c9dab4e5.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:42Z", "status": "sent", "sid": "MM863b367b7d0725532b80a161c9dab4e5", "date_sent": "2023-02-14T14:03:43Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM863b367b7d0725532b80a161c9dab4e5/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM863b367b7d0725532b80a161c9dab4e5/Feedback.json"}}, "emitted_at": 1682602966556} -{"stream": "messages", "data": {"body": "Hi there, Test 2!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T15:54:04Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM5a93ae7d20c07ceae87cd2649485ba72.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:47Z", "status": "sent", "sid": "MM5a93ae7d20c07ceae87cd2649485ba72", "date_sent": "2023-02-14T14:03:48Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM5a93ae7d20c07ceae87cd2649485ba72/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM5a93ae7d20c07ceae87cd2649485ba72/Feedback.json"}}, "emitted_at": 1682602966557} -{"stream": "messages", "data": {"body": "Hi there, Test 1!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T16:12:21Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM89c1a0785d96b5faa30d65aa644c70b4.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:52Z", "status": "sent", "sid": "MM89c1a0785d96b5faa30d65aa644c70b4", "date_sent": "2023-02-14T14:03:53Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM89c1a0785d96b5faa30d65aa644c70b4/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM89c1a0785d96b5faa30d65aa644c70b4/Feedback.json"}}, "emitted_at": 1682602966559} -{"stream": "outgoing_caller_ids", "data": {"phone_number": "+14153597503", "date_updated": "2020-11-17T04:17:37Z", "friendly_name": "(415) 359-7503", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN16ba111c0df5756cfe37044ed0ee3136.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN16ba111c0df5756cfe37044ed0ee3136", "date_created": "2020-11-17T04:17:37Z"}, "emitted_at": 1682602968873} -{"stream": "outgoing_caller_ids", "data": {"phone_number": "+18023494963", "date_updated": "2020-12-11T04:28:02Z", "friendly_name": "(802) 349-4963", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN726d635f970c30193cd12e7b994510a1.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN726d635f970c30193cd12e7b994510a1", "date_created": "2020-12-11T04:28:02Z"}, "emitted_at": 1682602968876} -{"stream": "outgoing_caller_ids", "data": {"phone_number": "+14156236785", "date_updated": "2023-02-15T15:33:09Z", "friendly_name": "Slack sms channel", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PNbb9c658169cfd057a46cdce9dc00afa3.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PNbb9c658169cfd057a46cdce9dc00afa3", "date_created": "2023-02-14T12:11:53Z"}, "emitted_at": 1682602968877} -{"stream": "queues", "data": {"date_updated": "2020-11-25T10:01:02Z", "current_size": 0, "friendly_name": "friendly_name_5", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QUbda7dcdeafaf6509b45c4a43e4c4519d.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QUbda7dcdeafaf6509b45c4a43e4c4519d", "date_created": "2020-11-25T10:01:02Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QUbda7dcdeafaf6509b45c4a43e4c4519d/Members.json"}}, "emitted_at": 1682602970288} -{"stream": "queues", "data": {"date_updated": "2020-11-25T10:01:01Z", "current_size": 0, "friendly_name": "friendly_name_4", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU9d308605319c35298f9833888d13c1fb.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QU9d308605319c35298f9833888d13c1fb", "date_created": "2020-11-25T10:01:01Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU9d308605319c35298f9833888d13c1fb/Members.json"}}, "emitted_at": 1682602970289} -{"stream": "queues", "data": {"date_updated": "2020-11-25T10:00:59Z", "current_size": 0, "friendly_name": "friendly_name_3", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU7a9ca432cb8ed145439bf74c27a3b587.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QU7a9ca432cb8ed145439bf74c27a3b587", "date_created": "2020-11-25T10:00:59Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU7a9ca432cb8ed145439bf74c27a3b587/Members.json"}}, "emitted_at": 1682602970291} -{"stream": "queues", "data": {"date_updated": "2020-11-25T10:00:57Z", "current_size": 0, "friendly_name": "friendly_name_2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU345bae62186d3e58ba1338d4b8a60456.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QU345bae62186d3e58ba1338d4b8a60456", "date_created": "2020-11-25T10:00:57Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU345bae62186d3e58ba1338d4b8a60456/Members.json"}}, "emitted_at": 1682602970292} -{"stream": "queues", "data": {"date_updated": "2020-11-25T10:00:55Z", "current_size": 0, "friendly_name": "friendly_name_1", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU5f9b1d245de682b4c3830689bfc8a484.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QU5f9b1d245de682b4c3830689bfc8a484", "date_created": "2020-11-25T10:00:55Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU5f9b1d245de682b4c3830689bfc8a484/Members.json"}}, "emitted_at": 1682602970293} -{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CAa001cfc4dc6dd62fca57820260e8b73c", "conference_sid": null, "date_created": "2023-02-15T09:49:49Z", "date_updated": "2023-02-15T09:49:49Z", "start_time": "2023-02-15T09:49:48Z", "duration": 1, "sid": "RE8d5413a5ede0900adc05c494135faa55", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE8d5413a5ede0900adc05c494135faa55.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE8d5413a5ede0900adc05c494135faa55/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE8d5413a5ede0900adc05c494135faa55/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE8d5413a5ede0900adc05c494135faa55"}, "emitted_at": 1682602973557} -{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CAa001cfc4dc6dd62fca57820260e8b73c", "conference_sid": null, "date_created": "2023-02-15T09:49:50Z", "date_updated": "2023-02-15T09:49:51Z", "start_time": "2023-02-15T09:49:50Z", "duration": 1, "sid": "RE58e437f7e690f806c4869e4fb3f273a8", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE58e437f7e690f806c4869e4fb3f273a8.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE58e437f7e690f806c4869e4fb3f273a8/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE58e437f7e690f806c4869e4fb3f273a8/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE58e437f7e690f806c4869e4fb3f273a8"}, "emitted_at": 1682602973558} -{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CAa001cfc4dc6dd62fca57820260e8b73c", "conference_sid": null, "date_created": "2023-02-15T09:49:52Z", "date_updated": "2023-02-15T09:49:52Z", "start_time": "2023-02-15T09:49:51Z", "duration": 1, "sid": "RE68b33779206ef216be16550aa31ad011", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE68b33779206ef216be16550aa31ad011.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE68b33779206ef216be16550aa31ad011/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE68b33779206ef216be16550aa31ad011/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE68b33779206ef216be16550aa31ad011"}, "emitted_at": 1682602973560} -{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CAa001cfc4dc6dd62fca57820260e8b73c", "conference_sid": null, "date_created": "2023-02-15T09:49:53Z", "date_updated": "2023-02-15T09:49:53Z", "start_time": "2023-02-15T09:49:53Z", "duration": 1, "sid": "RE3a29c0a0b50a3b594be12ee2b2e5e45d", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE3a29c0a0b50a3b594be12ee2b2e5e45d.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE3a29c0a0b50a3b594be12ee2b2e5e45d/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE3a29c0a0b50a3b594be12ee2b2e5e45d/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE3a29c0a0b50a3b594be12ee2b2e5e45d"}, "emitted_at": 1682602973562} -{"stream": "transcriptions", "data": {"sid": "TR9e1c09eb90898441a1f4d0356bac9516", "date_created": "2021-06-24T10:35:26Z", "date_updated": "2021-06-24T10:35:27Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "failed", "type": "fast", "recording_sid": "RE35c31ba04fb700777a050c4ac9e3c141", "duration": 1, "transcription_text": null, "api_version": "2008-08-01", "price": null, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR9e1c09eb90898441a1f4d0356bac9516.json"}, "emitted_at": 1684146239439} -{"stream": "transcriptions", "data": {"sid": "TR68d28cd2551c2c01922b79cb41ee5d51", "date_created": "2021-06-24T10:35:24Z", "date_updated": "2021-06-24T10:35:25Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "failed", "type": "fast", "recording_sid": "RE37d45d62a6b3f418dcb4c925d4e967cc", "duration": 1, "transcription_text": null, "api_version": "2008-08-01", "price": null, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR68d28cd2551c2c01922b79cb41ee5d51.json"}, "emitted_at": 1684146239439} -{"stream": "transcriptions", "data": {"sid": "TR4eebd9a480634dab945972fc7f76b1cd", "date_created": "2021-06-17T08:34:34Z", "date_updated": "2021-06-17T08:53:53Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "completed", "type": "fast", "recording_sid": "RE1c6686eb72437bb252384e1b84c7ed7b", "duration": 3, "transcription_text": "Best here.", "api_version": "2008-08-01", "price": -0.05, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR4eebd9a480634dab945972fc7f76b1cd.json"}, "emitted_at": 1684146239440} -{"stream": "transcriptions", "data": {"sid": "TR5de772ec5e698c5d36d04ab2d9a641c2", "date_created": "2021-06-17T08:34:30Z", "date_updated": "2021-06-17T08:34:31Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "failed", "type": "fast", "recording_sid": "RE26ad99320c5bacd3cb97b7ec8029c9a9", "duration": 2, "transcription_text": null, "api_version": "2008-08-01", "price": null, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR5de772ec5e698c5d36d04ab2d9a641c2.json"}, "emitted_at": 1684146239441} -{"stream": "usage_triggers", "data": {"sid": "UT36985e3a860748f984c3b9e77a991388", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-02-14T14:34:30Z", "date_updated": "2023-02-14T14:34:30Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT36985e3a860748f984c3b9e77a991388.json", "api_version": "2010-04-01"}, "emitted_at": 1682602977674} -{"stream": "usage_triggers", "data": {"sid": "UTed691d2a56a64c45baa9810c16f5931a", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-02-14T14:34:32Z", "date_updated": "2023-02-14T14:34:32Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UTed691d2a56a64c45baa9810c16f5931a.json", "api_version": "2010-04-01"}, "emitted_at": 1682602977675} -{"stream": "usage_triggers", "data": {"sid": "UT73ded683c9884266befb032487fac75e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-02-14T14:34:33Z", "date_updated": "2023-02-14T14:34:33Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT73ded683c9884266befb032487fac75e.json", "api_version": "2010-04-01"}, "emitted_at": 1682602977676} -{"stream": "usage_triggers", "data": {"sid": "UT5b9448b2d4e943539f1bba5f052ef01b", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-02-14T14:34:34Z", "date_updated": "2023-02-14T14:34:34Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT5b9448b2d4e943539f1bba5f052ef01b.json", "api_version": "2010-04-01"}, "emitted_at": 1682602977678} -{"stream": "trunks", "data": {"auth_type": "", "transfer_mode": "disable-all", "secure": false, "auth_type_set": [], "date_updated": "2023-05-10T17:29:44Z", "friendly_name": "integration-test-trunk", "domain_name": null, "disaster_recovery_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "recording": {"trim": "do-not-trim", "mode": "do-not-record"}, "transfer_caller_id": "from-transferee", "disaster_recovery_method": null, "url": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237", "sid": "TKdd4b0b21323f45ad4ce9164761d18237", "date_created": "2023-05-10T17:27:17Z", "cnam_lookup_enabled": false, "links": {"phone_numbers": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/PhoneNumbers", "ip_access_control_lists": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/IpAccessControlLists", "origination_urls": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/OriginationUrls", "credential_lists": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/CredentialLists"}}, "emitted_at": 1684432326862} -{"stream": "roles", "data": {"date_updated": "2023-03-21T13:35:15Z", "friendly_name": "service user", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Roles/RL1c0ab592f9724a10992c2ea29709f6cd", "sid": "RL1c0ab592f9724a10992c2ea29709f6cd", "date_created": "2023-03-21T13:35:15Z", "service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "type": "deployment", "permissions": ["createChannel", "joinChannel", "editOwnUserInfo"]}, "emitted_at": 1684513502733} -{"stream": "services", "data": {"typing_indicator_timeout": 5.0, "date_updated": "2023-03-21T13:35:15Z", "post_webhook_url": null, "read_status_enabled": true, "consumption_report_interval": 10.0, "pre_webhook_retry_count": 0.0, "default_service_role_sid": "RL1c0ab592f9724a10992c2ea29709f6cd", "media": {"compatibility_message": "Media messages are not supported by your client", "size_limit_mb": 150.0}, "default_channel_creator_role_sid": "RL3efa7fddc245451cbb76cde110621614", "reachability_enabled": false, "webhook_filters": null, "post_webhook_retry_count": 0.0, "sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "pre_webhook_url": null, "notifications": {"removed_from_channel": {"enabled": false}, "log_enabled": false, "added_to_channel": {"enabled": false}, "new_message": {"enabled": false}, "invited_to_channel": {"enabled": false}}, "webhook_method": null, "limits": {"user_channels": 1000.0, "channel_members": 1000.0}, "url": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f", "friendly_name": "Default Conversations Service", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-03-21T13:35:15Z", "default_channel_role_sid": "RLca3ff6cb9bc9404caf14e43b63fed446", "links": {"channels": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Channels", "bindings": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Bindings", "users": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Users", "roles": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Roles"}}, "emitted_at": 1684513526771} +{"stream": "addresses", "data": {"sid": "AD0164001bc0f84d9bc29e17378fe47c20", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "Test Dep Nu 1", "customer_name": "Test Customer 1", "street": "test-street_2", "street_secondary": null, "city": "test-city_2", "region": "test-region_2", "postal_code": "test-postal_code_2", "iso_country": "US", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses/AD0164001bc0f84d9bc29e17378fe47c20.json", "date_created": "2020-11-25T01:41:48Z", "date_updated": "2023-02-14T07:36:54Z", "emergency_enabled": false, "validated": false, "verified": false}, "emitted_at": 1691419680244} +{"stream": "addresses", "data": {"sid": "AD07820b628d536f40af85140c67e108f0", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "Test Addr 4", "customer_name": "Test Customer 4", "street": "Test Street 4", "street_secondary": null, "city": "Test sity 4", "region": "CA", "postal_code": "94121", "iso_country": "US", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses/AD07820b628d536f40af85140c67e108f0.json", "date_created": "2023-02-14T07:47:37Z", "date_updated": "2023-02-14T07:47:37Z", "emergency_enabled": false, "validated": false, "verified": false}, "emitted_at": 1691419680245} +{"stream": "addresses", "data": {"sid": "AD0e69bf9110f766787a88f99b507c9eeb", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "friendly_name": "Test Addr 3", "customer_name": "Test Customer 3", "street": "Test Street 3", "street_secondary": null, "city": "Test sity 3", "region": "CA", "postal_code": "94121", "iso_country": "US", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Addresses/AD0e69bf9110f766787a88f99b507c9eeb.json", "date_created": "2023-02-14T07:47:23Z", "date_updated": "2023-02-14T07:47:23Z", "emergency_enabled": false, "validated": false, "verified": false}, "emitted_at": 1691419680246} +{"stream": "applications", "data": {"sms_status_callback": "http://twilio.com/sms-status-callback", "voice_caller_id_lookup": false, "voice_fallback_url": null, "date_updated": "2023-02-16T15:19:41Z", "sms_fallback_method": "POST", "friendly_name": "Test SMS 1", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP9370b66dc53499e2459d82d75d21c6f8.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "GET", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP9370b66dc53499e2459d82d75d21c6f8", "date_created": "2020-11-25T09:47:31Z", "sms_url": "http://twilio.com/sms-fallback", "status_callback": null, "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": "http://twilio.com/sms-status-callback"}, "emitted_at": 1691419683126} +{"stream": "applications", "data": {"sms_status_callback": "", "voice_caller_id_lookup": false, "voice_fallback_url": "", "date_updated": "2023-02-14T14:43:08Z", "sms_fallback_method": "POST", "friendly_name": "Test Queue 10", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/APce869eb64ab8f4c849cedf2cb4e39726.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "POST", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "APce869eb64ab8f4c849cedf2cb4e39726", "date_created": "2023-02-14T14:43:08Z", "sms_url": "http://demo.twilio.com/docs/voice.xml", "status_callback": "", "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": ""}, "emitted_at": 1691419683128} +{"stream": "applications", "data": {"sms_status_callback": "", "voice_caller_id_lookup": false, "voice_fallback_url": "", "date_updated": "2023-02-14T14:42:53Z", "sms_fallback_method": "POST", "friendly_name": "Test Queue 9", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Applications/AP3df1f6928cffe2cc8fe6cc03905f675b.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "voice_method": "POST", "voice_url": "http://demo.twilio.com/docs/voice.xml", "sms_method": "POST", "public_application_connect_enabled": false, "status_callback_method": "POST", "sid": "AP3df1f6928cffe2cc8fe6cc03905f675b", "date_created": "2023-02-14T14:42:53Z", "sms_url": "http://demo.twilio.com/docs/voice.xml", "status_callback": "", "voice_fallback_method": "POST", "api_version": "2010-04-01", "message_status_callback": ""}, "emitted_at": 1691419683130} +{"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1691419684730} +{"stream": "available_phone_number_countries", "data": {"country_code": "BE", "country": "Belgium", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE/TollFree.json"}}, "emitted_at": 1691419684732} +{"stream": "available_phone_number_countries", "data": {"country_code": "SE", "country": "Sweden", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE/Mobile.json"}}, "emitted_at": 1691419684732} +{"stream": "calls", "data": {"date_updated": "2022-07-13T19:44:06Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 62, "from": "+13392299964", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA0bc0fdb5783917659002fe37ba581e11", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-07-13T19:43:04Z", "date_created": "2022-07-13T19:43:04Z", "from_formatted": "(339) 229-9964", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-07-13T19:44:06Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1691419833550} +{"stream": "calls", "data": {"date_updated": "2022-07-19T20:11:13Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 12, "from": "+16613444179", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA22893fba469912ccdf127472f69b301d", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-07-19T20:11:01Z", "date_created": "2022-07-19T20:11:01Z", "from_formatted": "(661) 344-4179", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-07-19T20:11:13Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1691419833553} +{"stream": "calls", "data": {"date_updated": "2022-07-20T18:24:12Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 11, "from": "+16613444179", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAcf07b73a4f6c8e50942ca59b7d0468e9", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-07-20T18:24:01Z", "date_created": "2022-07-20T18:24:01Z", "from_formatted": "(661) 344-4179", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-07-20T18:24:12Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1691419833554} +{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2022-09-23T14:44:41Z", "region": "us1", "friendly_name": "test_conference", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA8858f240bdccfb3393def1682c2dbdf0", "sid": "CFca0fa08200f55a6d60779d18b644a675", "date_created": "2022-09-23T14:44:11Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675/Recordings.json"}}, "emitted_at": 1691419855153} +{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-02-15T14:49:37Z", "region": "us1", "friendly_name": "Conference2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF15e8707d15e02c1af88809b159ff8b42.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA04ae9210566d36c425bae2087736f6ac", "sid": "CF15e8707d15e02c1af88809b159ff8b42", "date_created": "2023-02-15T14:49:21Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF15e8707d15e02c1af88809b159ff8b42/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF15e8707d15e02c1af88809b159ff8b42/Recordings.json"}}, "emitted_at": 1691419855509} +{"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-02-16T09:57:39Z", "region": "us1", "friendly_name": "Conference2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF33199d5a9a0b202b3bd9558438a052d8.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CAf8464ca5eda3ab7cc3e2d86cdb3c720f", "sid": "CF33199d5a9a0b202b3bd9558438a052d8", "date_created": "2023-02-16T09:57:11Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF33199d5a9a0b202b3bd9558438a052d8/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF33199d5a9a0b202b3bd9558438a052d8/Recordings.json"}}, "emitted_at": 1691419855510} +{"stream": "conversations", "data": {"unique_name": null, "date_updated": "2023-03-21T13:39:44Z", "friendly_name": "Friendly Conversation", "timers": {}, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "state": "active", "date_created": "2023-03-21T13:39:44Z", "messaging_service_sid": "MGfdf707ca9a7e03496ad79dc64e5e543e", "sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "attributes": "{}", "bindings": null, "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": {"participants": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants", "messages": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages", "webhooks": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Webhooks"}}, "emitted_at": 1691419856305} +{"stream": "conversation_messages", "data": {"body": "Ahoy there", "index": 0, "author": "smee", "date_updated": "2023-04-01T12:37:19Z", "media": null, "participant_sid": null, "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "delivery": null, "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages/IMd28bbec7d60f4c9b84595170871c6f28", "date_created": "2023-04-01T12:37:19Z", "content_sid": null, "sid": "IMd28bbec7d60f4c9b84595170871c6f28", "attributes": "{}", "links": {"delivery_receipts": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages/IMd28bbec7d60f4c9b84595170871c6f28/Receipts", "channel_metadata": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Messages/IMd28bbec7d60f4c9b84595170871c6f28/ChannelMetadata"}}, "emitted_at": 1691419857506} +{"stream": "conversation_participants", "data": {"last_read_message_index": null, "date_updated": "2023-04-13T11:52:17Z", "last_read_timestamp": null, "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB0a984a4238f14b828cf277becf880bd4", "date_created": "2023-04-13T11:52:17Z", "role_sid": "RLca3ff6cb9bc9404caf14e43b63fed446", "sid": "MB0a984a4238f14b828cf277becf880bd4", "attributes": "{}", "identity": "Integration Test 2", "messaging_binding": null}, "emitted_at": 1691419858698} +{"stream": "conversation_participants", "data": {"last_read_message_index": null, "date_updated": "2023-04-13T11:52:02Z", "last_read_timestamp": null, "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB41bb002a3a5e412fa7f2459dcfb4925f", "date_created": "2023-04-13T11:52:02Z", "role_sid": "RLca3ff6cb9bc9404caf14e43b63fed446", "sid": "MB41bb002a3a5e412fa7f2459dcfb4925f", "attributes": "{}", "identity": "Integration Test", "messaging_binding": null}, "emitted_at": 1691419858700} +{"stream": "flows", "data": {"status": "published", "date_updated": "2022-09-23T14:31:33Z", "friendly_name": "conference_test", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://studio.twilio.com/v1/Flows/FW7ad717a690629a6da33bd3c8b9cf7d97", "version": 15, "sid": "FW7ad717a690629a6da33bd3c8b9cf7d97", "date_created": "2022-09-23T14:28:11Z", "links": {"engagements": "https://studio.twilio.com/v1/Flows/FW7ad717a690629a6da33bd3c8b9cf7d97/Engagements", "executions": "https://studio.twilio.com/v1/Flows/FW7ad717a690629a6da33bd3c8b9cf7d97/Executions"}}, "emitted_at": 1691419865022} +{"stream": "flows", "data": {"status": "draft", "date_updated": "2023-06-16T16:46:27Z", "friendly_name": "SMS To slack", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "version": 66, "sid": "FWbd726b7110b21294a9f27a47f4ab0080", "date_created": "2021-02-01T07:22:57Z", "links": {"engagements": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Engagements", "executions": "https://studio.twilio.com/v1/Flows/FWbd726b7110b21294a9f27a47f4ab0080/Executions"}}, "emitted_at": 1691419865023} +{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T07:57:03Z", "voice_url": "https://handler.twilio.com/twiml/EH5793263d703ad674bbcdeb31ac80e359", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true}, "api_version": "2010-04-01", "sid": "PNf2eb05a16e73094f891b01076b830a6a", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+16508997708", "emergency_address_sid": null, "beta": false, "address_sid": "AD07820b628d536f40af85140c67e108f0", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 8", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNf2eb05a16e73094f891b01076b830a6a.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-16T14:31:29Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNf2eb05a16e73094f891b01076b830a6a/AssignedAddOns.json"}}, "emitted_at": 1691419867845} +{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T07:58:14Z", "voice_url": "https://handler.twilio.com/twiml/EHb6471af720e8b66baa14e7226227893b", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true}, "api_version": "2010-04-01", "sid": "PNd74715bab1be123cc9004f03b85bb067", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+14246220939", "emergency_address_sid": null, "beta": false, "address_sid": "AD0164001bc0f84d9bc29e17378fe47c20", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 9", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNd74715bab1be123cc9004f03b85bb067.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-16T14:34:00Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNd74715bab1be123cc9004f03b85bb067/AssignedAddOns.json"}}, "emitted_at": 1691419867848} +{"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T07:58:40Z", "voice_url": "https://handler.twilio.com/twiml/EHb77bc7c1f889b6c9fe5202d0463edfc4", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true}, "api_version": "2010-04-01", "sid": "PN99400a65bf5a4305d5420060842d4d2c", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+19125901057", "emergency_address_sid": null, "beta": false, "address_sid": "AD0e69bf9110f766787a88f99b507c9eeb", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN99400a65bf5a4305d5420060842d4d2c.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-15T09:31:24Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN99400a65bf5a4305d5420060842d4d2c/AssignedAddOns.json"}}, "emitted_at": 1691419867849} +{"stream": "message_media", "data": {"sid": "MEa79c9e10f96b3c6018caa5e9a62567cc", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM9170a757ee3976ecb8ebeb4e9580f9c0", "content_type": "image/png", "date_created": "2022-09-23T20:17:19Z", "date_updated": "2022-09-23T20:17:20Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Media/MEa79c9e10f96b3c6018caa5e9a62567cc.json"}, "emitted_at": 1691419887396} +{"stream": "message_media", "data": {"sid": "ME34324546a4398b36fc96fd36500038c3", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM56662e159d1a5d1f1c6e2d43202b7940", "content_type": "image/png", "date_created": "2023-02-14T14:02:28Z", "date_updated": "2023-02-14T14:02:28Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM56662e159d1a5d1f1c6e2d43202b7940/Media/ME34324546a4398b36fc96fd36500038c3.json"}, "emitted_at": 1691419915272} +{"stream": "message_media", "data": {"sid": "ME45c86c927aa3eb6749bac07b9bc6f418", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM5e9241ae9a444f8061b28e3de05fe818", "content_type": "image/png", "date_created": "2023-02-14T14:02:59Z", "date_updated": "2023-02-14T14:02:59Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM5e9241ae9a444f8061b28e3de05fe818/Media/ME45c86c927aa3eb6749bac07b9bc6f418.json"}, "emitted_at": 1691419915437} +{"stream": "messages", "data": {"body": "Hello there!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2022-09-23T20:17:20Z", "price": -0.02, "error_message": "Unknown error", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+19142793086", "date_created": "2022-09-23T20:17:19Z", "status": "undelivered", "sid": "MM9170a757ee3976ecb8ebeb4e9580f9c0", "date_sent": "2022-09-23T20:17:20Z", "messaging_service_sid": null, "error_code": "30008", "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Feedback.json"}}, "emitted_at": 1691419956845} +{"stream": "messages", "data": {"body": "Test", "num_segments": 1, "direction": "inbound", "from": "+12025502908", "date_updated": "2022-12-16T18:58:29Z", "price": -0.0079, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SMf73a453514f0a1d8bd4d0121713a8be9.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 0, "to": "+12056561170", "date_created": "2022-12-16T18:58:28Z", "status": "received", "sid": "SMf73a453514f0a1d8bd4d0121713a8be9", "date_sent": "2022-12-16T18:58:29Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SMf73a453514f0a1d8bd4d0121713a8be9/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SMf73a453514f0a1d8bd4d0121713a8be9/Feedback.json"}}, "emitted_at": 1691419957399} +{"stream": "messages", "data": {"body": "Airbyte", "num_segments": 1, "direction": "inbound", "from": "+12025502908", "date_updated": "2022-12-16T18:58:47Z", "price": -0.0079, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SM0ae34204de318609bd2801af5396442d.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 0, "to": "+12056561170", "date_created": "2022-12-16T18:58:47Z", "status": "received", "sid": "SM0ae34204de318609bd2801af5396442d", "date_sent": "2022-12-16T18:58:47Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SM0ae34204de318609bd2801af5396442d/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SM0ae34204de318609bd2801af5396442d/Feedback.json"}}, "emitted_at": 1691419957401} +{"stream": "outgoing_caller_ids", "data": {"phone_number": "+14153597503", "date_updated": "2020-11-17T04:17:37Z", "friendly_name": "(415) 359-7503", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN16ba111c0df5756cfe37044ed0ee3136.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN16ba111c0df5756cfe37044ed0ee3136", "date_created": "2020-11-17T04:17:37Z"}, "emitted_at": 1691419960444} +{"stream": "outgoing_caller_ids", "data": {"phone_number": "+18023494963", "date_updated": "2020-12-11T04:28:02Z", "friendly_name": "(802) 349-4963", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN726d635f970c30193cd12e7b994510a1.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN726d635f970c30193cd12e7b994510a1", "date_created": "2020-12-11T04:28:02Z"}, "emitted_at": 1691419960446} +{"stream": "outgoing_caller_ids", "data": {"phone_number": "+14156236785", "date_updated": "2023-02-15T15:33:09Z", "friendly_name": "Slack sms channel", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PNbb9c658169cfd057a46cdce9dc00afa3.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PNbb9c658169cfd057a46cdce9dc00afa3", "date_created": "2023-02-14T12:11:53Z"}, "emitted_at": 1691419960447} +{"stream": "queues", "data": {"date_updated": "2023-02-14T14:59:25Z", "current_size": 0, "friendly_name": "Test Queue 10", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QUe3deeb42ef5f50ba5384340a6fed3843.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QUe3deeb42ef5f50ba5384340a6fed3843", "date_created": "2023-02-14T14:59:25Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QUe3deeb42ef5f50ba5384340a6fed3843/Members.json"}}, "emitted_at": 1691419961877} +{"stream": "queues", "data": {"date_updated": "2023-02-14T14:59:20Z", "current_size": 0, "friendly_name": "Test Queue 9", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU3635d67fab98b364cdde4477ffe3db3b.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QU3635d67fab98b364cdde4477ffe3db3b", "date_created": "2023-02-14T14:59:20Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QU3635d67fab98b364cdde4477ffe3db3b/Members.json"}}, "emitted_at": 1691419961879} +{"stream": "queues", "data": {"date_updated": "2023-02-14T14:59:16Z", "current_size": 0, "friendly_name": "Test Queue 8", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QUddcfe8becf4830ddf0d88d884ad6cd1d.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "average_wait_time": 0, "sid": "QUddcfe8becf4830ddf0d88d884ad6cd1d", "date_created": "2023-02-14T14:59:16Z", "max_size": 100, "subresource_uris": {"members": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Queues/QUddcfe8becf4830ddf0d88d884ad6cd1d/Members.json"}}, "emitted_at": 1691419961880} +{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CA78611ecf5e7f101b1a59be31b8f520f7", "conference_sid": null, "date_created": "2022-03-13T23:56:27Z", "date_updated": "2022-03-13T23:56:28Z", "start_time": "2022-03-13T23:56:26Z", "duration": 2, "sid": "RE6be6c79bca501a7d5284c5ebcd87ec22", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE6be6c79bca501a7d5284c5ebcd87ec22.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE6be6c79bca501a7d5284c5ebcd87ec22/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE6be6c79bca501a7d5284c5ebcd87ec22/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/RE6be6c79bca501a7d5284c5ebcd87ec22"}, "emitted_at": 1691419964228} +{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CA78611ecf5e7f101b1a59be31b8f520f7", "conference_sid": null, "date_created": "2022-03-13T23:56:29Z", "date_updated": "2022-03-13T23:56:30Z", "start_time": "2022-03-13T23:56:29Z", "duration": 1, "sid": "REb6d63081540fd7ec9835f267fa722ff4", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REb6d63081540fd7ec9835f267fa722ff4.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REb6d63081540fd7ec9835f267fa722ff4/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REb6d63081540fd7ec9835f267fa722ff4/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REb6d63081540fd7ec9835f267fa722ff4"}, "emitted_at": 1691419964230} +{"stream": "recordings", "data": {"account_sid": "ACdade166c12e160e9ed0a6088226718fb", "api_version": "2010-04-01", "call_sid": "CA78611ecf5e7f101b1a59be31b8f520f7", "conference_sid": null, "date_created": "2022-03-13T23:56:31Z", "date_updated": "2022-03-13T23:56:31Z", "start_time": "2022-03-13T23:56:31Z", "duration": 1, "sid": "REa944f91cad14528766b3dfb3152fbb89", "price": -0.0025, "price_unit": "USD", "status": "completed", "channels": 1, "source": "RecordVerb", "error_code": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REa944f91cad14528766b3dfb3152fbb89.json", "encryption_details": null, "subresource_uris": {"add_on_results": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REa944f91cad14528766b3dfb3152fbb89/AddOnResults.json", "transcriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REa944f91cad14528766b3dfb3152fbb89/Transcriptions.json"}, "media_url": "https://api.twilio.com/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Recordings/REa944f91cad14528766b3dfb3152fbb89"}, "emitted_at": 1691419964231} +{"stream": "roles", "data": {"date_updated": "2023-03-21T13:35:15Z", "friendly_name": "service user", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Roles/RL1c0ab592f9724a10992c2ea29709f6cd", "sid": "RL1c0ab592f9724a10992c2ea29709f6cd", "date_created": "2023-03-21T13:35:15Z", "service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "type": "deployment", "permissions": ["createChannel", "joinChannel", "editOwnUserInfo"]}, "emitted_at": 1691419966289} +{"stream": "roles", "data": {"date_updated": "2023-03-21T13:35:15Z", "friendly_name": "channel admin", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Roles/RL3efa7fddc245451cbb76cde110621614", "sid": "RL3efa7fddc245451cbb76cde110621614", "date_created": "2023-03-21T13:35:15Z", "service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "type": "channel", "permissions": ["destroyChannel", "inviteMember", "removeMember", "editChannelName", "editChannelAttributes", "addMember", "addNonChatParticipant", "sendMessage", "sendMediaMessage", "leaveChannel", "editAnyMessage", "editAnyMessageAttributes", "editAnyMemberAttributes", "deleteAnyMessage", "editNotificationLevel"]}, "emitted_at": 1691419966290} +{"stream": "roles", "data": {"date_updated": "2023-03-21T13:35:15Z", "friendly_name": "service admin", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Roles/RL75d8c6ea62dd4eb8a4a7764f9ee4d048", "sid": "RL75d8c6ea62dd4eb8a4a7764f9ee4d048", "date_created": "2023-03-21T13:35:15Z", "service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "type": "deployment", "permissions": ["createChannel", "joinChannel", "destroyChannel", "inviteMember", "removeMember", "editChannelName", "editChannelAttributes", "addMember", "editAnyMessage", "editAnyMessageAttributes", "editAnyMemberAttributes", "deleteAnyMessage", "editAnyUserInfo"]}, "emitted_at": 1691419966291} +{"stream": "services", "data": {"typing_indicator_timeout": 5.0, "date_updated": "2023-03-21T13:35:15Z", "post_webhook_url": null, "read_status_enabled": true, "consumption_report_interval": 10.0, "pre_webhook_retry_count": 0.0, "default_service_role_sid": "RL1c0ab592f9724a10992c2ea29709f6cd", "media": {"compatibility_message": "Media messages are not supported by your client", "size_limit_mb": 150.0}, "default_channel_creator_role_sid": "RL3efa7fddc245451cbb76cde110621614", "reachability_enabled": false, "webhook_filters": null, "post_webhook_retry_count": 0.0, "sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "pre_webhook_url": null, "notifications": {"removed_from_channel": {"enabled": false}, "log_enabled": false, "added_to_channel": {"enabled": false}, "new_message": {"enabled": false}, "invited_to_channel": {"enabled": false}}, "webhook_method": null, "limits": {"user_channels": 1000.0, "channel_members": 1000.0}, "url": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f", "friendly_name": "Default Conversations Service", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-03-21T13:35:15Z", "default_channel_role_sid": "RLca3ff6cb9bc9404caf14e43b63fed446", "links": {"channels": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Channels", "bindings": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Bindings", "users": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Users", "roles": "https://chat.twilio.com/v2/Services/IS5fcc074f7ead44c99a0a24a374a7e19f/Roles"}}, "emitted_at": 1691419966869} +{"stream": "transcriptions", "data": {"sid": "TR040fc55e657ba82c848e297903a03c51", "date_created": "2023-01-19T23:30:49Z", "date_updated": "2023-01-20T02:38:20Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "completed", "type": "fast", "recording_sid": "RE13348c019d7efa727d0edfe33a4329b1", "duration": 3, "transcription_text": "Ms clara and I'm with medicare benefits.", "api_version": "2010-04-01", "price": -0.05, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR040fc55e657ba82c848e297903a03c51.json"}, "emitted_at": 1691419969942} +{"stream": "transcriptions", "data": {"sid": "TR0266f60acd1c7ffccd469f1e4cfcfde4", "date_created": "2023-01-19T23:30:46Z", "date_updated": "2023-01-20T02:38:13Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "completed", "type": "fast", "recording_sid": "REadc78cf3ce609e38649a93a42577d203", "duration": 3, "transcription_text": "With medicare benefits are.", "api_version": "2010-04-01", "price": -0.05, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR0266f60acd1c7ffccd469f1e4cfcfde4.json"}, "emitted_at": 1691419969944} +{"stream": "transcriptions", "data": {"sid": "TR0ebc4ead073f5bd002cb45ffd18f2183", "date_created": "2022-11-21T18:57:40Z", "date_updated": "2022-11-21T18:57:40Z", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "status": "failed", "type": "fast", "recording_sid": "REbc137c3284c9929b324d92d71790736c", "duration": 1, "transcription_text": null, "api_version": "2010-04-01", "price": null, "price_unit": "USD", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Transcriptions/TR0ebc4ead073f5bd002cb45ffd18f2183.json"}, "emitted_at": 1691419969945} +{"stream": "trunks", "data": {"auth_type": "", "transfer_mode": "disable-all", "secure": false, "auth_type_set": [], "date_updated": "2023-05-10T17:29:44Z", "friendly_name": "integration-test-trunk", "domain_name": null, "disaster_recovery_url": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "recording": {"trim": "do-not-trim", "mode": "do-not-record"}, "transfer_caller_id": "from-transferee", "disaster_recovery_method": null, "url": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237", "sid": "TKdd4b0b21323f45ad4ce9164761d18237", "date_created": "2023-05-10T17:27:17Z", "cnam_lookup_enabled": false, "links": {"phone_numbers": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/PhoneNumbers", "ip_access_control_lists": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/IpAccessControlLists", "origination_urls": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/OriginationUrls", "credential_lists": "https://trunking.twilio.com/v1/Trunks/TKdd4b0b21323f45ad4ce9164761d18237/CredentialLists"}}, "emitted_at": 1691419970823} +{"stream": "usage_triggers", "data": {"sid": "UT33bd2bf238d94863a609133da897d676", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:19Z", "date_updated": "2020-11-25T10:02:19Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 1000.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT33bd2bf238d94863a609133da897d676.json", "api_version": "2010-04-01"}, "emitted_at": 1691420016931} +{"stream": "usage_triggers", "data": {"sid": "UT3c3c157dcaf347829d5a0f75e97b572e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:34Z", "date_updated": "2020-11-25T10:02:34Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 999.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT3c3c157dcaf347829d5a0f75e97b572e.json", "api_version": "2010-04-01"}, "emitted_at": 1691420016933} +{"stream": "usage_triggers", "data": {"sid": "UT7170996eff504647ac9f215222ee296f", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2020-11-25T10:02:41Z", "date_updated": "2020-11-25T10:02:41Z", "date_fired": null, "friendly_name": null, "usage_category": "sms", "trigger_by": "usage", "recurring": "", "trigger_value": 943.0, "current_value": 147.0, "callback_url": "http://www.example.com/", "callback_method": "POST", "usage_record_uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Records.json?Category=sms", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Usage/Triggers/UT7170996eff504647ac9f215222ee296f.json", "api_version": "2010-04-01"}, "emitted_at": 1691420016934} +{"stream": "users", "data": {"is_notifiable": null, "date_updated": "2023-04-13T11:52:17Z", "is_online": null, "friendly_name": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Users/US4373c40fffca48dcab7498989c484a0d", "date_created": "2023-04-13T11:52:17Z", "role_sid": "RL1c0ab592f9724a10992c2ea29709f6cd", "sid": "US4373c40fffca48dcab7498989c484a0d", "attributes": "{}", "identity": "Integration Test 2", "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": {"user_conversations": "https://conversations.twilio.com/v1/Users/US4373c40fffca48dcab7498989c484a0d/Conversations"}}, "emitted_at": 1691420017691} +{"stream": "users", "data": {"is_notifiable": null, "date_updated": "2023-04-13T11:52:02Z", "is_online": null, "friendly_name": null, "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "url": "https://conversations.twilio.com/v1/Users/US9d8279d5e8954fd1b9804c853be5baa3", "date_created": "2023-04-13T11:52:02Z", "role_sid": "RL1c0ab592f9724a10992c2ea29709f6cd", "sid": "US9d8279d5e8954fd1b9804c853be5baa3", "attributes": "{}", "identity": "Integration Test", "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": {"user_conversations": "https://conversations.twilio.com/v1/Users/US9d8279d5e8954fd1b9804c853be5baa3/Conversations"}}, "emitted_at": 1691420017693} +{"stream": "user_conversations", "data": {"notification_level": "default", "unique_name": null, "user_sid": "US4373c40fffca48dcab7498989c484a0d", "friendly_name": "Friendly Conversation", "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "unread_messages_count": null, "created_by": "system", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "last_read_message_index": null, "date_created": "2023-03-21T13:39:44Z", "timers": {}, "url": "https://conversations.twilio.com/v1/Users/US4373c40fffca48dcab7498989c484a0d/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "date_updated": "2023-03-21T13:39:44Z", "attributes": "{}", "participant_sid": "MB0a984a4238f14b828cf277becf880bd4", "conversation_state": "active", "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": {"conversation": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "participant": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB0a984a4238f14b828cf277becf880bd4"}}, "emitted_at": 1691420018889} +{"stream": "user_conversations", "data": {"notification_level": "default", "unique_name": null, "user_sid": "US9d8279d5e8954fd1b9804c853be5baa3", "friendly_name": "Friendly Conversation", "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "unread_messages_count": null, "created_by": "system", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "last_read_message_index": null, "date_created": "2023-03-21T13:39:44Z", "timers": {}, "url": "https://conversations.twilio.com/v1/Users/US9d8279d5e8954fd1b9804c853be5baa3/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "date_updated": "2023-03-21T13:39:44Z", "attributes": "{}", "participant_sid": "MB41bb002a3a5e412fa7f2459dcfb4925f", "conversation_state": "active", "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": {"conversation": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "participant": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB41bb002a3a5e412fa7f2459dcfb4925f"}}, "emitted_at": 1691420019069} +{"stream": "verify_services", "data": {"default_template_sid": null, "tts_name": null, "psd2_enabled": false, "do_not_share_warning_enabled": false, "mailer_sid": null, "friendly_name": "MyServiceName", "url": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_updated": "2022-12-01T04:35:52Z", "totp": {"time_step": 30.0, "skew": 1.0, "code_length": 6.0, "issuer": "MyServiceName"}, "code_length": 6.0, "custom_code_enabled": false, "sid": "VAf7a6bc96c5a594a56b3ccacfaf4c2e6e", "push": {"apn_credential_sid": null, "include_date": false, "fcm_credential_sid": null}, "date_created": "2022-12-01T04:35:52Z", "dtmf_input_required": true, "skip_sms_to_landlines": false, "lookup_enabled": false, "links": {"verification_checks": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/VerificationCheck", "rate_limits": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/RateLimits", "entities": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/Entities", "access_tokens": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/AccessTokens", "verifications": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/Verifications", "webhooks": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/Webhooks", "messaging_configurations": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/MessagingConfigurations"}}, "emitted_at": 1691420020246} +{"stream": "verify_services", "data": {"default_template_sid": null, "tts_name": null, "psd2_enabled": false, "do_not_share_warning_enabled": false, "mailer_sid": null, "friendly_name": "MyServiceName", "url": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_updated": "2022-12-02T18:08:14Z", "totp": {"time_step": 30.0, "skew": 1.0, "code_length": 6.0, "issuer": "MyServiceName"}, "code_length": 6.0, "custom_code_enabled": false, "sid": "VA5d72879e05e83a236433c203b4d8ecf4", "push": {"apn_credential_sid": null, "include_date": false, "fcm_credential_sid": null}, "date_created": "2022-12-02T18:08:14Z", "dtmf_input_required": true, "skip_sms_to_landlines": false, "lookup_enabled": false, "links": {"verification_checks": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/VerificationCheck", "rate_limits": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/RateLimits", "entities": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/Entities", "access_tokens": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/AccessTokens", "verifications": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/Verifications", "webhooks": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/Webhooks", "messaging_configurations": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/MessagingConfigurations"}}, "emitted_at": 1691420020247} +{"stream": "verify_services", "data": {"default_template_sid": null, "tts_name": null, "psd2_enabled": false, "do_not_share_warning_enabled": false, "mailer_sid": null, "friendly_name": "MyServiceName", "url": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_updated": "2022-12-01T04:38:44Z", "totp": {"time_step": 30.0, "skew": 1.0, "code_length": 6.0, "issuer": "MyServiceName"}, "code_length": 6.0, "custom_code_enabled": false, "sid": "VA3c8282c56e66bd5b015a9196c09be5be", "push": {"apn_credential_sid": null, "include_date": false, "fcm_credential_sid": null}, "date_created": "2022-12-01T04:38:44Z", "dtmf_input_required": true, "skip_sms_to_landlines": false, "lookup_enabled": false, "links": {"verification_checks": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/VerificationCheck", "rate_limits": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/RateLimits", "entities": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/Entities", "access_tokens": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/AccessTokens", "verifications": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/Verifications", "webhooks": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/Webhooks", "messaging_configurations": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/MessagingConfigurations"}}, "emitted_at": 1691420020248} diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index 5535f500e9c1..34410a3acd58 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 - dockerImageTag: 0.7.0 + dockerImageTag: 0.10.0 dockerRepository: airbyte/source-twilio githubIssueLabel: source-twilio icon: twilio.svg @@ -23,4 +23,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/twilio tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-twilio/requirements.txt b/airbyte-integrations/connectors/source-twilio/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-twilio/requirements.txt +++ b/airbyte-integrations/connectors/source-twilio/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-twilio/setup.py b/airbyte-integrations/connectors/source-twilio/setup.py index f19be385e1a5..66a2cb8a7ee0 100644 --- a/airbyte-integrations/connectors/source-twilio/setup.py +++ b/airbyte-integrations/connectors/source-twilio/setup.py @@ -11,7 +11,7 @@ "requests", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "requests_mock", "freezegun"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock", "freezegun"] setup( name="source_twilio", diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json index 3631be2fb9ff..478a3d47b38f 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json @@ -89,6 +89,9 @@ "properties": { "delivery_receipts": { "type": ["null", "string"] + }, + "channel_metadata": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json new file mode 100644 index 000000000000..14a7affc830f --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Step Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "parent_step_sid": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "date_updated": { + "type": ["null", "string"] + }, + "transitioned_to": { + "type": ["null", "string"] + }, + "account_sid": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "context": { + "type": ["null", "string"] + }, + "sid": { + "type": ["null", "string"] + }, + "transitioned_from": { + "type": ["null", "string"] + }, + "date_created": { + "type": ["null", "string"] + }, + "execution_sid": { + "type": ["null", "string"] + }, + "flow_sid": { + "type": ["null", "string"] + }, + "links": { + "type": ["null", "object"], + "properties": { + "step_context": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json new file mode 100644 index 000000000000..6c69bb037d5f --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json @@ -0,0 +1,75 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "User Conversation Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "notification_level": { + "type": ["null", "string"] + }, + "unique_name": { + "type": ["null", "string"] + }, + "user_sid": { + "type": ["null", "string"] + }, + "friendly_name": { + "type": ["null", "string"] + }, + "conversation_sid": { + "type": ["null", "string"] + }, + "unread_messages_count": { + "type": ["null", "integer"] + }, + "created_by": { + "type": ["null", "string"] + }, + "account_sid": { + "type": ["null", "string"] + }, + "last_read_message_index": { + "type": ["null", "integer"] + }, + "date_created": { + "type": ["null", "string"] + }, + "timers": { + "type": ["null", "object"], + "properties": { + "chat_service_sid": { + "type": ["null", "string"] + } + } + }, + "url": { + "type": ["null", "string"] + }, + "date_updated": { + "type": ["null", "string"] + }, + "attributes": { + "type": ["null", "string"] + }, + "participant_sid": { + "type": ["null", "string"] + }, + "conversation_state": { + "type": ["null", "string"] + }, + "chat_service_sid": { + "type": ["null", "string"] + }, + "links": { + "type": ["null", "object"], + "properties": { + "conversation": { + "type": ["null", "string"] + }, + "participant": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json new file mode 100644 index 000000000000..d0942ed00aea --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Users Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "is_notifiable": { + "type": ["null", "string"] + }, + "date_updated": { + "type": ["null", "string"] + }, + "is_online": { + "type": ["null", "string"] + }, + "friendly_name": { + "type": ["null", "string"] + }, + "account_sid": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "date_created": { + "type": ["null", "string"] + }, + "role_sid": { + "type": ["null", "string"] + }, + "sid": { + "type": ["null", "string"] + }, + "attributes": { + "type": ["null", "string"] + }, + "identity": { + "type": ["null", "string"] + }, + "chat_service_sid": { + "type": ["null", "string"] + }, + "links": { + "type": ["null", "object"], + "properties": { + "user_conversations": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json new file mode 100644 index 000000000000..63acbe0ad91f --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Verify - Services Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "default_template_sid": { + "type": ["null", "string"] + }, + "tts_name": { + "type": ["null", "string"] + }, + "psd2_enabled": { + "type": ["null", "boolean"] + }, + "do_not_share_warning_enabled": { + "type": ["null", "boolean"] + }, + "mailer_sid": { + "type": ["null", "string"] + }, + "friendly_name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "account_sid": { + "type": ["null", "string"] + }, + "date_updated": { + "type": ["null", "string"] + }, + "totp": { + "type": ["null", "object"], + "properties": { + "time_step": { + "type": ["null", "number"] + }, + "skew": { + "type": ["null", "number"] + }, + "code_length": { + "type": ["null", "number"] + }, + "issuer": { + "type": ["null", "string"] + } + } + }, + "code_length": { + "type": ["null", "number"] + }, + "custom_code_enabled": { + "type": ["null", "boolean"] + }, + "sid": { + "type": ["null", "string"] + }, + "push": { + "type": ["null", "object"], + "properties": { + "apn_credential_sid": { + "type": ["null", "string"] + }, + "include_date": { + "type": ["null", "boolean"] + }, + "fcm_credential_sid": { + "type": ["null", "string"] + } + } + }, + "date_created": { + "type": ["null", "string"] + }, + "dtmf_input_required": { + "type": ["null", "boolean"] + }, + "skip_sms_to_landlines": { + "type": ["null", "boolean"] + }, + "lookup_enabled": { + "type": ["null", "boolean"] + }, + "links": { + "type": ["null", "object"], + "properties": { + "verification_checks": { + "type": ["null", "string"] + }, + "rate_limits": { + "type": ["null", "string"] + }, + "entities": { + "type": ["null", "string"] + }, + "access_tokens": { + "type": ["null", "string"] + }, + "verifications": { + "type": ["null", "string"] + }, + "webhooks": { + "type": ["null", "string"] + }, + "messaging_configurations": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index 64512e3f89bc..6b1e852d8ce3 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -38,10 +38,14 @@ Recordings, Roles, Services, + Step, Transcriptions, Trunks, UsageRecords, UsageTriggers, + UserConversations, + Users, + VerifyServices, ) RETENTION_WINDOW_LIMIT = 400 @@ -115,9 +119,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Recordings(**incremental_stream_kwargs), Roles(**full_refresh_stream_kwargs), Services(**full_refresh_stream_kwargs), + Step(**full_refresh_stream_kwargs), Transcriptions(**full_refresh_stream_kwargs), Trunks(**full_refresh_stream_kwargs), UsageRecords(**incremental_stream_kwargs), UsageTriggers(**full_refresh_stream_kwargs), + Users(**full_refresh_stream_kwargs), + UserConversations(**full_refresh_stream_kwargs), + VerifyServices(**full_refresh_stream_kwargs), ] return streams diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 8d9d5f4cc213..0d1c3b44b6b6 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -27,6 +27,7 @@ TWILIO_STUDIO_API_BASE = "https://studio.twilio.com/v1/" TWILIO_CONVERSATIONS_URL_BASE = "https://conversations.twilio.com/v1/" TWILIO_TRUNKING_URL_BASE = "https://trunking.twilio.com/v1/" +TWILIO_VERIFY_BASE_V2 = "https://verify.twilio.com/v2/" class TwilioStream(HttpStream, ABC): @@ -431,6 +432,23 @@ def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[st return {"flow_sid": record["sid"]} +class Step(TwilioNestedStream): + """ + https://www.twilio.com/docs/studio/rest-api/v2/step#read-a-list-of-step-resources + """ + + parent_stream = Executions + url_base = TWILIO_STUDIO_API_BASE + uri_from_subresource = False + data_field = "steps" + + def path(self, stream_slice: Mapping[str, Any], **kwargs): + return f"Flows/{stream_slice['flow_sid']}/Executions/{stream_slice['execution_sid']}/Steps" + + def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + return {"flow_sid": record["flow_sid"], "execution_sid": record["sid"]} + + class OutgoingCallerIds(TwilioNestedStream): """https://www.twilio.com/docs/voice/api/outgoing-caller-ids#outgoingcallerids-list-resource""" @@ -457,6 +475,20 @@ def path(self, **kwargs): return "Services" +class VerifyServices(TwilioStream): + """ + https://www.twilio.com/docs/chat/rest/service-resource#read-multiple-service-resources + """ + + # Unlike other endpoints, this one won't accept requests where pageSize >100 + page_size = 100 + data_field = "services" + url_base = TWILIO_VERIFY_BASE_V2 + + def path(self, **kwargs): + return "Services" + + class Roles(TwilioNestedStream): """ https://www.twilio.com/docs/chat/rest/role-resource#read-multiple-role-resources @@ -611,3 +643,28 @@ def path(self, stream_slice: Mapping[str, Any], **kwargs): def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: return {"conversation_sid": record["sid"]} + + +class Users(TwilioStream): + """https://www.twilio.com/docs/conversations/api/user-resource""" + + url_base = TWILIO_CONVERSATIONS_URL_BASE + + def path(self, **kwargs): + return self.name.title() + + +class UserConversations(TwilioNestedStream): + """https://www.twilio.com/docs/conversations/api/user-conversation-resource#list-all-of-a-users-conversations""" + + parent_stream = Users + url_base = TWILIO_CONVERSATIONS_URL_BASE + uri_from_subresource = False + data_field = "conversations" + primary_key = ["account_sid"] + + def path(self, stream_slice: Mapping[str, Any], **kwargs): + return f"Users/{stream_slice['user_sid']}/Conversations" + + def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: + return {"user_sid": record["sid"]} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py index 346bb5dbf45c..ad932927703f 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py @@ -29,9 +29,13 @@ OutgoingCallerIds, Queues, Recordings, + Step, Transcriptions, UsageRecords, UsageTriggers, + UserConversations, + Users, + VerifyServices, ) @@ -95,11 +99,15 @@ def test_check_connection_handles_exceptions(mocker, config, exception, expected (OutgoingCallerIds), (Queues), (Recordings), + (Step), (Transcriptions), (UsageRecords), (UsageTriggers), (Conversations), - (ConversationParticipants) + (ConversationParticipants), + (Users), + (UserConversations), + (VerifyServices), ], ) def test_streams(stream_cls, config): diff --git a/airbyte-integrations/connectors/source-twitter/README.md b/airbyte-integrations/connectors/source-twitter/README.md index 6ce7dfd8f94a..59078caa9682 100644 --- a/airbyte-integrations/connectors/source-twitter/README.md +++ b/airbyte-integrations/connectors/source-twitter/README.md @@ -50,7 +50,9 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat #### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` To run your integration tests with docker ### Using gradle to run tests diff --git a/airbyte-integrations/connectors/source-twitter/metadata.yaml b/airbyte-integrations/connectors/source-twitter/metadata.yaml index 8c02ecb98883..11ff23358334 100644 --- a/airbyte-integrations/connectors/source-twitter/metadata.yaml +++ b/airbyte-integrations/connectors/source-twitter/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-twitter/requirements.txt b/airbyte-integrations/connectors/source-twitter/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-twitter/requirements.txt +++ b/airbyte-integrations/connectors/source-twitter/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-twitter/setup.py b/airbyte-integrations/connectors/source-twitter/setup.py index b2895bc25171..bcc921fcba46 100644 --- a/airbyte-integrations/connectors/source-twitter/setup.py +++ b/airbyte-integrations/connectors/source-twitter/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml b/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml index 171edeca50d2..e2b450284998 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml +++ b/airbyte-integrations/connectors/source-tyntec-sms/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tyntec-sms/requirements.txt b/airbyte-integrations/connectors/source-tyntec-sms/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/requirements.txt +++ b/airbyte-integrations/connectors/source-tyntec-sms/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-tyntec-sms/setup.py b/airbyte-integrations/connectors/source-tyntec-sms/setup.py index 2af069cccb8e..1a15c6223694 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/setup.py +++ b/airbyte-integrations/connectors/source-tyntec-sms/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-typeform/Dockerfile b/airbyte-integrations/connectors/source-typeform/Dockerfile index 1acb6942318c..82ab07c099d5 100644 --- a/airbyte-integrations/connectors/source-typeform/Dockerfile +++ b/airbyte-integrations/connectors/source-typeform/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-typeform diff --git a/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml b/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml index 57452a7e9a1c..c8f7582adcad 100644 --- a/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml @@ -1,27 +1,33 @@ -connector_image: airbyte/source-typeform:dev +connector_image: airbyte/source-typeform:1.0.0 +test_strictness_level: "high" acceptance_tests: spec: tests: - spec_path: "source_typeform/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.0" connection: tests: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.0" basic_read: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: - name: webhooks bypass_reason: "no data" expect_records: path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + fail_on_extra_columns: true incremental: tests: - config_path: "secrets/incremental_config.json" diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/__init__.py b/airbyte-integrations/connectors/source-typeform/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json index 6cbda9426eeb..3b04806c8ce9 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json @@ -1,16 +1,22 @@ -{ - "responses": { - "SdMKQYkv": { - "submitted_at": 9999999999 - }, - "XtrcGoGJ": { - "submitted_at": 9999999999 - }, - "kRt99jlK": { - "submitted_at": 9999999999 - }, - "VWO7mLtl": { - "submitted_at": 9999999999 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "SdMKQYkv": { + "submitted_at": 9999999999 + }, + "XtrcGoGJ": { + "submitted_at": 9999999999 + }, + "kRt99jlK": { + "submitted_at": 9999999999 + }, + "VWO7mLtl": { + "submitted_at": 9999999999 + } + }, + "stream_descriptor": { "name": "responses" } } } -} +] diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl index a93cde833349..bf37dab0f628 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl @@ -1,21 +1,18 @@ -{"stream": "forms", "data": {"id": "VWO7mLtl", "type": "quiz", "title": "Connector Extensibility meetup", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "qvDqCNAHuIC8", "ref": "01GHC6KQ5Y0M8VN6XHVAG75J0G", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "default_redirect", "button_text": "Create a typeform"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "ZdzF0rrvsVdB", "title": "What times work for you to visit San Francisco to work with the team?", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "nLpt4rvNjFB3", "ref": "01GHC6KQ5Y155J0F550BGYYS1A", "label": "Dec 12-16"}, {"id": "4xpK9sqA06eL", "ref": "01GHC6KQ5YBATX0CFENVVB5BYG", "label": "Dec 19-23"}, {"id": "jQHb3mqslOsZ", "ref": "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "label": "Jan 9-14"}, {"id": "wS5FKMUnMgqR", "ref": "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "label": "Jan 16-20"}, {"id": "uvmLX80Loava", "ref": "8fffd3a8-1e96-421d-a605-a7029bd55e97", "label": "Jan 22-26"}, {"id": "7ubtgCrW2meb", "ref": "17403cc9-74cd-49d1-856a-be6662b3b497", "label": "Jan30 - Feb3"}, {"id": "51q0g4fTFtYc", "ref": "3a1295b4-97b9-4986-9c37-f1af1d72501d", "label": "Feb 6 - 11"}, {"id": "vi3iwtpETqlb", "ref": "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "label": "Feb 13-17"}, {"id": "iI0hDpta14Kk", "ref": "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee", "label": "Feb 19-24"}]}, "validations": {"required": false}, "type": "multiple_choice", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}], "created_at": "2022-11-08T18:04:03+00:00", "last_updated_at": "2022-11-08T21:10:54+00:00", "published_at": "2022-11-08T21:10:54+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/VWO7mLtl"}}, "emitted_at": 1675773700149} -{"stream": "forms", "data": {"id": "SdMKQYkv", "type": "quiz", "title": "Event Registration (copy)", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/JPnxbU"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": true}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "welcome_screens": [{"id": "3rc53L8DmKJ5", "ref": "70d54ea2e68f27ae", "title": "The annual FormConf is almost here.\n\nWant to come?", "properties": {"show_button": true, "button_text": "Count me in"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/UD82iitWn5XY"}}], "fields": [{"id": "63WvXUnvSCa9", "title": "Great, can we get your full name?", "ref": "ef34b985c51e4131", "properties": {"description": "We'll print this on your event pass."}, "validations": {"required": true}, "type": "short_text"}, {"id": "kwpFrd2lI3ok", "title": "And what's your email address?", "ref": "0c3cabd70157cf16", "properties": {"description": "We'll only use it to send you a confirmation."}, "validations": {"required": true}, "type": "email"}, {"id": "1Ua3d1mzhJwj", "title": "Are you planning on staying for the afterparty?", "ref": "7207397713e2b5e3", "properties": {"description": "We have a surprise guest lined up...", "randomize": false, "allow_multiple_selection": false, "allow_other_choice": false, "supersized": false, "show_labels": true, "choices": [{"id": "z5hxxjpJl07L", "ref": "bfcc3fbf608583f7", "label": "Yes", "attachment": {"type": "image", "href": "https://images.typeform.com/images/xDriVAzzHfVq"}}, {"id": "37iaitPaS03r", "ref": "5bf390ce5210d38b", "label": "No", "attachment": {"type": "image", "href": "https://images.typeform.com/images/Rn4AmMgzPrYg"}}]}, "validations": {"required": false}, "type": "picture_choice"}, {"id": "MmrPLXSaCF5B", "title": "And do you have any food allergies we should know about?", "ref": "9aaaeeebe70858c4", "properties": {}, "validations": {"required": false}, "type": "short_text"}, {"id": "gurSOcuvNnvb", "title": "Any questions about the event?", "ref": "18842abd9aa9ded4", "properties": {"description": "Write them here and we'll get back to you via email."}, "validations": {"required": false}, "type": "long_text"}, {"id": "fCaCvjCJ57cO", "title": "And finally, would you mind telling us how you heard about the FormConf?", "ref": "261d0775b1f029cb", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": true, "vertical_alignment": false, "choices": [{"id": "IhbdgAo2LHXJ", "ref": "51c76f5fa66c6725", "label": "Social media"}, {"id": "h6Ss8i8gQdOw", "ref": "87408191d53179ee", "label": "Google"}, {"id": "45jv6vFn2nrz", "ref": "8bc14882d0521d6a", "label": "Local advertising"}, {"id": "Iahtxl1jeQwh", "ref": "fce1c86f-fb00-4c33-8085-6e4a4f12ea35", "label": "From a friend"}]}, "validations": {"required": false}, "type": "multiple_choice"}, {"id": "i8rReP3KV7c0", "title": "That's everything. We'll send you an email confirmation with some details a few minutes after you submit this form.\n\nWe hope you're excited as we are :)", "ref": "c6d179ae9c4794e0", "properties": {"button_text": "See you there!", "hide_marks": true}, "type": "statement"}], "created_at": "2021-06-26T14:39:53+00:00", "last_updated_at": "2021-06-27T15:15:56+00:00", "published_at": "2021-06-27T15:15:56+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/SdMKQYkv"}}, "emitted_at": 1675773700338} -{"stream": "forms", "data": {"id": "kRt99jlK", "type": "quiz", "title": "Political Poll [DEMO 2] (copy)", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/wvWlco"}, "settings": {"language": "en", "progress_bar": "percentage", "meta": {"allow_indexing": true}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "welcome_screens": [{"id": "4jguOatzh4QX", "ref": "e8ee8d1500fec6d9", "title": "*National Voting Intentions*", "properties": {"show_button": true, "button_text": "Take Poll", "description": "If you're ok with it, we'd like to know about how you might vote in a general election"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/ty98jF8FppfC"}}], "fields": [{"id": "qthblBc7InVU", "title": "Let's get right to the point:\nIf there was a general election tomorrow, which party would you vote for?", "ref": "8267768033031e53", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "3XbsOhLiGFkv", "ref": "fc3c3f5dbe75a01e", "label": "Center-right party"}, {"id": "FesJYxqJ0SNX", "ref": "61f5e8b36fcdbd91", "label": "Center-left party"}, {"id": "prJzwOH23zsc", "ref": "495e234f7c65d582", "label": "Green party"}, {"id": "6IXNu85c5dOl", "ref": "566c959b6ef92437", "label": "Don't know"}]}, "validations": {"required": true}, "type": "multiple_choice"}, {"id": "rB7FJUThFlu4", "title": "OK, how do you feel about the general direction of our country at the moment?", "ref": "b13b02912db6f287", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": false, "supersized": true, "show_labels": true, "choices": [{"id": "Jt8FTorS35Sb", "ref": "b99bb45c5b8c25be", "label": "Going in the wrong direction", "attachment": {"type": "image", "href": "https://images.typeform.com/images/evDEZYspmvjm"}}, {"id": "KUJ0sF2mqG5A", "ref": "2e07534517a152a2", "label": "Going in the right direction", "attachment": {"type": "image", "href": "https://images.typeform.com/images/DgjPyuz9Aphy"}}, {"id": "Dlb3UhA4keI4", "ref": "50be578304fe3e92", "label": "At a standstill", "attachment": {"type": "image", "href": "https://images.typeform.com/images/ZqbqJ6h4zGmM"}}, {"id": "21TAgDkTV2zE", "ref": "d9993c86773807a8", "label": "Unsure", "attachment": {"type": "image", "href": "https://images.typeform.com/images/mNFNeMbxPQMt"}}]}, "validations": {"required": true}, "type": "picture_choice"}, {"id": "vV7ISYSgZ94I", "title": "Thanks, and how do you feel about your own situation this year?", "ref": "f1939629f760be75", "properties": {"start_at_one": true, "steps": 5, "labels": {"left": "Much worse", "center": "About the same", "right": "Much better"}}, "validations": {"required": true}, "type": "opinion_scale"}, {"id": "Mrq4qNeRInni", "title": "Thanks again. Just a couple more questions to go. \nWhich of these issues is most important to you?", "ref": "c52566d91c5052e2", "properties": {"randomize": true, "allow_multiple_selection": false, "allow_other_choice": false, "supersized": false, "show_labels": true, "choices": [{"id": "pTMw2DVbpMyD", "ref": "d9374e70c6cb2f9c", "label": "Taxes & Economy", "attachment": {"type": "image", "href": "https://images.typeform.com/images/6eSzJ9khSfvS"}}, {"id": "dymEoJ3SUDZp", "ref": "d3278566f523cef0", "label": "Labor & Business", "attachment": {"type": "image", "href": "https://images.typeform.com/images/HWfXuXCR3Ls8"}}, {"id": "mOq857nilN8V", "ref": "39ecf093bc5e645b", "label": "Infrastructures", "attachment": {"type": "image", "href": "https://images.typeform.com/images/rW2P45guvd63"}}, {"id": "rnul7dwtsWbg", "ref": "aa1c16d517ffbea0", "label": "Health", "attachment": {"type": "image", "href": "https://images.typeform.com/images/nVsmUESsAzCs"}}, {"id": "Ii1K3mlYioOm", "ref": "4b9a4b0defbf04e5", "label": "Environment", "attachment": {"type": "image", "href": "https://images.typeform.com/images/7ZwHmRi3ZYeg"}}, {"id": "TGZeXHyDgTrQ", "ref": "536d99289a1f80d1", "label": "Education", "attachment": {"type": "image", "href": "https://images.typeform.com/images/Z8qCFjGRD78P"}}, {"id": "1Asa5xFxfuCi", "ref": "68b9b4d37fabf862", "label": "Family & Equality", "attachment": {"type": "image", "href": "https://images.typeform.com/images/YC4Fx6ud6bKq"}}, {"id": "szBe0vqnkCUK", "ref": "3f1692f85c3cd73b", "label": "Military & Defense", "attachment": {"type": "image", "href": "https://images.typeform.com/images/YA746sDt87Xf"}}]}, "validations": {"required": true}, "type": "picture_choice"}, {"id": "aDJXqNTyDXxD", "title": "To finish up, would you mind telling us how you think the current government doing on these issues?", "ref": "f684d8bc4fbca5d3", "properties": {"description": "From 1, doing badly, to 5, doing great...", "button_text": "Continue", "show_button": false, "fields": [{"id": "x9myjwStSn9a", "title": "Economy", "ref": "55c2e5c15f7dccec", "properties": {"shape": "star", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "zaP8jDAArI5x", "title": "National Debt", "ref": "f853e99096a32208", "properties": {"shape": "up", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "VFmcjbHlFTzg", "title": "Employment", "ref": "6f0fd734177ecf27", "properties": {"shape": "user", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "DgGh4ZkRBAyH", "title": "Healthcare", "ref": "b4171ed292dc3cee", "properties": {"shape": "heart", "steps": 5}, "validations": {"required": true}, "type": "rating"}, {"id": "yC3UrwN1LKT8", "title": "Education", "ref": "c8dd7d63c26777d9", "properties": {"shape": "pencil", "steps": 5}, "validations": {"required": true}, "type": "rating"}]}, "type": "group"}], "created_at": "2021-06-26T14:39:14+00:00", "last_updated_at": "2021-07-01T10:03:13+00:00", "published_at": "2021-07-01T10:03:13+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/kRt99jlK"}}, "emitted_at": 1675773700751} -{"stream": "forms", "data": {"id": "XtrcGoGJ", "type": "quiz", "title": "Basic Form", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "Xg85PhXqk4HR", "ref": "01F8N53B82QB6T2VM7ASP16198", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "reload", "button_text": "reload"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "8VK4KwNd0DgB", "title": "Hello, what's your name?", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG", "properties": {}, "validations": {"required": false}, "type": "short_text", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}, {"id": "HbNDNK4LLOXB", "title": "Fill your email here:", "ref": "b7980774-f17f-43bc-920b-586e03398f03", "properties": {}, "validations": {"required": false}, "type": "email"}, {"id": "5l7cx4NRa7aX", "title": "enter site", "ref": "e98312ff-df57-4de2-82ae-3617e6dd32ab", "properties": {}, "validations": {"required": false}, "type": "website"}, {"id": "6lGZzhNfrqwB", "title": "Multi-Select question.", "ref": "43153da3-fbbc-443e-b66f-1752770c0e0a", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": false, "vertical_alignment": true, "choices": [{"id": "3HfyxDo5JoXf", "ref": "f83999f6-c869-47cc-af2f-f22b628a0fdb", "label": "choice 3"}, {"id": "03VP9UxCwCLT", "ref": "27b8dfcb-ef16-4ad7-b2be-734ec24c34ca", "label": "choice 4"}, {"id": "ELm7HbFr0OOq", "ref": "ce51ab49-2cce-490d-b831-309337c79fa0", "label": "choice2"}, {"id": "acwDGU8NeO2A", "ref": "74ef0411-0c8a-4c09-a6f3-7a62b0745f68", "label": "choice1"}]}, "validations": {"required": false}, "type": "multiple_choice"}, {"id": "X6dq0mumvtKq", "title": "Nice to meet you, {{field:01F8N53B7KPZ2A1DWGZTTK9SKG}}, how is your day going?", "ref": "01F8N53B8293QHVDDHT84RZR6K", "properties": {"randomize": false, "allow_multiple_selection": false, "allow_other_choice": false, "vertical_alignment": true, "choices": [{"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}, {"id": "7jNEfjJ2cDAl", "ref": "01F8N53B82RE3YZK7RR50KNRQ0", "label": "Not so well..."}]}, "validations": {"required": false}, "type": "multiple_choice"}], "created_at": "2021-06-20T16:47:13+00:00", "last_updated_at": "2022-06-20T10:19:58+00:00", "published_at": "2021-06-20T16:47:26+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ"}}, "emitted_at": 1675773700952} -{"stream":"responses","data":{"landing_id":"fr2wm964fnyxpdx9a8tfr2wmlph34hqi","token":"fr2wm964fnyxpdx9a8tfr2wmlph34hqi","response_id":"fr2wm964fnyxpdx9a8tfr2wmlph34hqi","landed_at":"2022-11-08T21:59:53Z","submitted_at":"2022-11-08T22:00:24Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"8a0111039f","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","4xpK9sqA06eL","jQHb3mqslOsZ","wS5FKMUnMgqR","uvmLX80Loava","7ubtgCrW2meb","iI0hDpta14Kk"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","01GHC6KQ5YBATX0CFENVVB5BYG","1c392fa3-e693-49fe-b334-3a5cddc1db6f","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97","17403cc9-74cd-49d1-856a-be6662b3b497","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Dec 12-16","Dec 19-23","Jan 9-14","Jan 16-20","Jan 22-26","Jan30 - Feb3","Feb 19-24"]}}]},"emitted_at":1673035160703} -{"stream":"responses","data":{"landing_id":"0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r","token":"0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r","response_id":"0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r","landed_at":"2022-11-08T22:08:39Z","submitted_at":"2022-11-08T22:10:04Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"d4b74277d2","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","wS5FKMUnMgqR","jQHb3mqslOsZ","51q0g4fTFtYc","vi3iwtpETqlb","iI0hDpta14Kk"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","1c392fa3-e693-49fe-b334-3a5cddc1db6f","3a1295b4-97b9-4986-9c37-f1af1d72501d","54edf52a-c9c7-4bc4-a5a6-bd86115f5adb","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Dec 12-16","Jan 16-20","Jan 9-14","Feb 6 - 11","Feb 13-17","Feb 19-24"]}}]},"emitted_at":1673035160703} -{"stream":"responses","data":{"landing_id":"ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq","token":"ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq","response_id":"ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq","landed_at":"2022-11-09T06:16:08Z","submitted_at":"2022-11-09T06:16:10Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"2be9dd4bab","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","wS5FKMUnMgqR","uvmLX80Loava","7ubtgCrW2meb","51q0g4fTFtYc","vi3iwtpETqlb","iI0hDpta14Kk"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97","17403cc9-74cd-49d1-856a-be6662b3b497","3a1295b4-97b9-4986-9c37-f1af1d72501d","54edf52a-c9c7-4bc4-a5a6-bd86115f5adb","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Dec 12-16","Jan 16-20","Jan 22-26","Jan30 - Feb3","Feb 6 - 11","Feb 13-17","Feb 19-24"]}}]},"emitted_at":1673035160855} -{"stream":"responses","data":{"landing_id":"e7hli3wynwfkiwaebwe7h2aeeso4xrum","token":"e7hli3wynwfkiwaebwe7h2aeeso4xrum","response_id":"e7hli3wynwfkiwaebwe7h2aeeso4xrum","landed_at":"2022-11-09T08:09:11Z","submitted_at":"2022-11-09T08:33:38Z","metadata":{"user_agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Mobile/15E148 Safari/604.1","platform":"mobile","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl?typeform-source=www.linkedin.com","network_id":"fec8bf87e1","browser":"touch"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["nLpt4rvNjFB3","wS5FKMUnMgqR","uvmLX80Loava"],"refs":["01GHC6KQ5Y155J0F550BGYYS1A","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97"],"labels":["Dec 12-16","Jan 16-20","Jan 22-26"]}}]},"emitted_at":1673035160855} -{"stream":"responses","data":{"landing_id":"r4epuzzxlonggr4epp07wenb3a58sm6h","token":"r4epuzzxlonggr4epp07wenb3a58sm6h","response_id":"r4epuzzxlonggr4epp07wenb3a58sm6h","landed_at":"2022-11-09T14:33:46Z","submitted_at":"2022-11-09T14:35:10Z","metadata":{"user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"7e338d2504","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["51q0g4fTFtYc","vi3iwtpETqlb","iI0hDpta14Kk"],"refs":["3a1295b4-97b9-4986-9c37-f1af1d72501d","54edf52a-c9c7-4bc4-a5a6-bd86115f5adb","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Feb 6 - 11","Feb 13-17","Feb 19-24"]}}]},"emitted_at":1673035161002} -{"stream":"responses","data":{"landing_id":"ic7ydv73zomudp1p9ic7yp9spye7h72b","token":"ic7ydv73zomudp1p9ic7yp9spye7h72b","response_id":"ic7ydv73zomudp1p9ic7yp9spye7h72b","landed_at":"2022-11-15T02:31:04Z","submitted_at":"2022-11-15T02:34:53Z","metadata":{"user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"8284380108","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["jQHb3mqslOsZ","wS5FKMUnMgqR","uvmLX80Loava","7ubtgCrW2meb","51q0g4fTFtYc","iI0hDpta14Kk"],"refs":["1c392fa3-e693-49fe-b334-3a5cddc1db6f","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97","17403cc9-74cd-49d1-856a-be6662b3b497","3a1295b4-97b9-4986-9c37-f1af1d72501d","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Jan 9-14","Jan 16-20","Jan 22-26","Jan30 - Feb3","Feb 6 - 11","Feb 19-24"]}}]},"emitted_at":1673035161002} -{"stream":"responses","data":{"landing_id":"trdyqvm2wmf9b0dhoostrdtugo8fdcoa","token":"trdyqvm2wmf9b0dhoostrdtugo8fdcoa","response_id":"trdyqvm2wmf9b0dhoostrdtugo8fdcoa","landed_at":"2022-11-15T02:36:40Z","submitted_at":"2022-11-15T02:39:52Z","metadata":{"user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/VWO7mLtl","network_id":"8284380108","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"ZdzF0rrvsVdB","ref":"01GHC6KQ5Y6S9ZQH5CHKZPT1RM","type":"multiple_choice"},"type":"choices","choices":{"ids":["jQHb3mqslOsZ","wS5FKMUnMgqR","uvmLX80Loava","7ubtgCrW2meb","51q0g4fTFtYc","iI0hDpta14Kk"],"refs":["1c392fa3-e693-49fe-b334-3a5cddc1db6f","2ac396a3-1b8e-4e56-b36d-d1f27c1b834d","8fffd3a8-1e96-421d-a605-a7029bd55e97","17403cc9-74cd-49d1-856a-be6662b3b497","3a1295b4-97b9-4986-9c37-f1af1d72501d","e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"],"labels":["Jan 9-14","Jan 16-20","Jan 22-26","Jan30 - Feb3","Feb 6 - 11","Feb 19-24"]}}]},"emitted_at":1673035161150} -{"stream":"responses","data":{"landing_id":"w9yrjygpz00o1vop20h68tlw9yriwyqr","token":"w9yrjygpz00o1vop20h68tlw9yriwyqr","response_id":"w9yrjygpz00o1vop20h68tlw9yriwyqr","landed_at":"2021-06-27T15:16:09Z","submitted_at":"2021-06-27T15:18:12Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/SdMKQYkv","network_id":"abd4cbf203","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"63WvXUnvSCa9","ref":"ef34b985c51e4131","type":"short_text"},"type":"text","text":"Mr X"},{"field":{"id":"kwpFrd2lI3ok","ref":"0c3cabd70157cf16","type":"email"},"type":"email","email":"mrx@airbyte.com"},{"field":{"id":"1Ua3d1mzhJwj","ref":"7207397713e2b5e3","type":"picture_choice"},"type":"choice","choice":{"id":"z5hxxjpJl07L","ref":"bfcc3fbf608583f7","label":"Yes"}},{"field":{"id":"MmrPLXSaCF5B","ref":"9aaaeeebe70858c4","type":"short_text"},"type":"text","text":"water"},{"field":{"id":"gurSOcuvNnvb","ref":"18842abd9aa9ded4","type":"long_text"},"type":"text","text":"do you know who I am ?"}]},"emitted_at":1673035161294} -{"stream":"responses","data":{"landing_id":"x9ege1s9u758nla0bx9ege1bg63u7daz","token":"x9ege1s9u758nla0bx9ege1bg63u7daz","response_id":"x9ege1s9u758nla0bx9ege1bg63u7daz","landed_at":"2021-07-01T10:03:21Z","submitted_at":"2021-07-01T10:04:01Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/kRt99jlK","network_id":"32d3e45763","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"qthblBc7InVU","ref":"8267768033031e53","type":"multiple_choice"},"type":"choice","choice":{"id":"3XbsOhLiGFkv","ref":"fc3c3f5dbe75a01e","label":"Center-right party"}},{"field":{"id":"rB7FJUThFlu4","ref":"b13b02912db6f287","type":"picture_choice"},"type":"choice","choice":{"id":"Jt8FTorS35Sb","ref":"b99bb45c5b8c25be","label":"Going in the wrong direction"}},{"field":{"id":"vV7ISYSgZ94I","ref":"f1939629f760be75","type":"opinion_scale"},"type":"number","number":1},{"field":{"id":"Mrq4qNeRInni","ref":"c52566d91c5052e2","type":"picture_choice"},"type":"choice","choice":{"id":"szBe0vqnkCUK","ref":"3f1692f85c3cd73b","label":"Military & Defense"}},{"field":{"id":"x9myjwStSn9a","ref":"55c2e5c15f7dccec","type":"rating"},"type":"number","number":1},{"field":{"id":"zaP8jDAArI5x","ref":"f853e99096a32208","type":"rating"},"type":"number","number":1},{"field":{"id":"VFmcjbHlFTzg","ref":"6f0fd734177ecf27","type":"rating"},"type":"number","number":1},{"field":{"id":"DgGh4ZkRBAyH","ref":"b4171ed292dc3cee","type":"rating"},"type":"number","number":1},{"field":{"id":"yC3UrwN1LKT8","ref":"c8dd7d63c26777d9","type":"rating"},"type":"number","number":5}]},"emitted_at":1673035161618} -{"stream":"responses","data":{"landing_id":"ohgpotzjg8w852pohgpo1ub0gq76tks2","token":"ohgpotzjg8w852pohgpo1ub0gq76tks2","response_id":"ohgpotzjg8w852pohgpo1ub0gq76tks2","landed_at":"2021-06-20T16:49:16Z","submitted_at":"2021-06-20T16:49:20Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/XtrcGoGJ","network_id":"1866502915","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"8VK4KwNd0DgB","ref":"01F8N53B7KPZ2A1DWGZTTK9SKG","type":"short_text"},"type":"text","text":"11"},{"field":{"id":"X6dq0mumvtKq","ref":"01F8N53B8293QHVDDHT84RZR6K","type":"multiple_choice"},"type":"choice","choice":{"id":"FWQrVLFdHroI","ref":"01F8N53B82JXPXZ1B53BMJY0X2","label":"Terrific!"}}]},"emitted_at":1673035161756} -{"stream":"responses","data":{"landing_id":"74zhlkhbmspze5nllpl143674zhlkh81","token":"74zhlkhbmspze5nllpl143674zhlkh81","response_id":"74zhlkhbmspze5nllpl143674zhlkh81","landed_at":"2021-06-27T15:32:07Z","submitted_at":"2021-06-27T15:32:14Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/XtrcGoGJ","network_id":"abd4cbf203","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"8VK4KwNd0DgB","ref":"01F8N53B7KPZ2A1DWGZTTK9SKG","type":"short_text"},"type":"text","text":"222"},{"field":{"id":"X6dq0mumvtKq","ref":"01F8N53B8293QHVDDHT84RZR6K","type":"multiple_choice"},"type":"choice","choice":{"id":"7jNEfjJ2cDAl","ref":"01F8N53B82RE3YZK7RR50KNRQ0","label":"Not so well..."}}]},"emitted_at":1673035161757} -{"stream":"responses","data":{"landing_id":"9f7s89wh4wagc4qbr1c9f7s89tr0cu6g","token":"9f7s89wh4wagc4qbr1c9f7s89tr0cu6g","response_id":"9f7s89wh4wagc4qbr1c9f7s89tr0cu6g","landed_at":"2021-06-27T15:32:33Z","submitted_at":"2021-06-27T15:32:39Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/XtrcGoGJ","network_id":"abd4cbf203","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"8VK4KwNd0DgB","ref":"01F8N53B7KPZ2A1DWGZTTK9SKG","type":"short_text"},"type":"text","text":"There is a new library called furl. I find this library to be most pythonic for doing url algebra. To install:"},{"field":{"id":"X6dq0mumvtKq","ref":"01F8N53B8293QHVDDHT84RZR6K","type":"multiple_choice"},"type":"choice","choice":{"id":"FWQrVLFdHroI","ref":"01F8N53B82JXPXZ1B53BMJY0X2","label":"Terrific!"}}]},"emitted_at":1673035161918} -{"stream":"responses","data":{"landing_id":"zn4d1osa0ou5gitzn4k3e3am0c0q7vgd","token":"zn4d1osa0ou5gitzn4k3e3am0c0q7vgd","response_id":"zn4d1osa0ou5gitzn4k3e3am0c0q7vgd","landed_at":"2021-07-01T14:06:57Z","submitted_at":"2021-07-01T14:07:03Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/XtrcGoGJ","network_id":"32d3e45763","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"8VK4KwNd0DgB","ref":"01F8N53B7KPZ2A1DWGZTTK9SKG","type":"short_text"},"type":"text","text":"124125125"},{"field":{"id":"X6dq0mumvtKq","ref":"01F8N53B8293QHVDDHT84RZR6K","type":"multiple_choice"},"type":"choice","choice":{"id":"7jNEfjJ2cDAl","ref":"01F8N53B82RE3YZK7RR50KNRQ0","label":"Not so well..."}}]},"emitted_at":1673035161918} -{"stream":"responses","data":{"landing_id":"s3ah741anof3uot3340qs3ah7q62w544","token":"s3ah741anof3uot3340qs3ah7q62w544","response_id":"s3ah741anof3uot3340qs3ah7q62w544","landed_at":"2021-09-04T14:35:19Z","submitted_at":"2021-09-04T14:35:30Z","metadata":{"user_agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36","platform":"other","referer":"https://xe03v5buli4.typeform.com/to/XtrcGoGJ","network_id":"8c4966ac74","browser":"default"},"hidden":{},"calculated":{"score":0},"answers":[{"field":{"id":"8VK4KwNd0DgB","ref":"01F8N53B7KPZ2A1DWGZTTK9SKG","type":"short_text"},"type":"text","text":"test123"},{"field":{"id":"6lGZzhNfrqwB","ref":"43153da3-fbbc-443e-b66f-1752770c0e0a","type":"multiple_choice"},"type":"choices","choices":{"ids":["03VP9UxCwCLT","ELm7HbFr0OOq","acwDGU8NeO2A","3HfyxDo5JoXf"],"refs":["27b8dfcb-ef16-4ad7-b2be-734ec24c34ca","ce51ab49-2cce-490d-b831-309337c79fa0","74ef0411-0c8a-4c09-a6f3-7a62b0745f68","f83999f6-c869-47cc-af2f-f22b628a0fdb"],"labels":["choice 4","choice2","choice1","choice 3"]}},{"field":{"id":"X6dq0mumvtKq","ref":"01F8N53B8293QHVDDHT84RZR6K","type":"multiple_choice"},"type":"choice","choice":{"id":"FWQrVLFdHroI","ref":"01F8N53B82JXPXZ1B53BMJY0X2","label":"Terrific!"}}]},"emitted_at":1673035162064} +{"stream": "forms", "data": {"id": "VWO7mLtl", "type": "quiz", "title": "Connector Extensibility meetup", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "use_lead_qualification": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "qvDqCNAHuIC8", "ref": "01GHC6KQ5Y0M8VN6XHVAG75J0G", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "default_redirect", "button_text": "Create a typeform"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "ZdzF0rrvsVdB", "title": "What times work for you to visit San Francisco to work with the team?", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "nLpt4rvNjFB3", "ref": "01GHC6KQ5Y155J0F550BGYYS1A", "label": "Dec 12-16"}, {"id": "4xpK9sqA06eL", "ref": "01GHC6KQ5YBATX0CFENVVB5BYG", "label": "Dec 19-23"}, {"id": "jQHb3mqslOsZ", "ref": "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "label": "Jan 9-14"}, {"id": "wS5FKMUnMgqR", "ref": "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "label": "Jan 16-20"}, {"id": "uvmLX80Loava", "ref": "8fffd3a8-1e96-421d-a605-a7029bd55e97", "label": "Jan 22-26"}, {"id": "7ubtgCrW2meb", "ref": "17403cc9-74cd-49d1-856a-be6662b3b497", "label": "Jan30 - Feb3"}, {"id": "51q0g4fTFtYc", "ref": "3a1295b4-97b9-4986-9c37-f1af1d72501d", "label": "Feb 6 - 11"}, {"id": "vi3iwtpETqlb", "ref": "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "label": "Feb 13-17"}, {"id": "iI0hDpta14Kk", "ref": "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee", "label": "Feb 19-24"}]}, "validations": {"required": false}, "type": "multiple_choice", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}], "created_at": "2022-11-08T18:04:03+00:00", "last_updated_at": "2022-11-08T21:10:54+00:00", "published_at": "2022-11-08T21:10:54+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "responses": "https://api.typeform.com/forms/VWO7mLtl/responses"}}, "emitted_at": 1686590629013} +{"stream": "responses", "data": {"landing_id": "fr2wm964fnyxpdx9a8tfr2wmlph34hqi", "token": "fr2wm964fnyxpdx9a8tfr2wmlph34hqi", "response_id": "fr2wm964fnyxpdx9a8tfr2wmlph34hqi", "landed_at": "2022-11-08T21:59:53Z", "submitted_at": "2022-11-08T22:00:24Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8a0111039f", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "4xpK9sqA06eL", "jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "iI0hDpta14Kk"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "01GHC6KQ5YBATX0CFENVVB5BYG", "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Dec 12-16", "Dec 19-23", "Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222458} +{"stream": "responses", "data": {"landing_id": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "token": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "response_id": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "landed_at": "2022-11-08T22:08:39Z", "submitted_at": "2022-11-08T22:10:04Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "d4b74277d2", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "wS5FKMUnMgqR", "jQHb3mqslOsZ", "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Dec 12-16", "Jan 16-20", "Jan 9-14", "Feb 6 - 11", "Feb 13-17", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222461} +{"stream": "responses", "data": {"landing_id": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "token": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "response_id": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "landed_at": "2022-11-09T06:16:08Z", "submitted_at": "2022-11-09T06:16:10Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "2be9dd4bab", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Dec 12-16", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 13-17", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222826} +{"stream": "responses", "data": {"landing_id": "e7hli3wynwfkiwaebwe7h2aeeso4xrum", "token": "e7hli3wynwfkiwaebwe7h2aeeso4xrum", "response_id": "e7hli3wynwfkiwaebwe7h2aeeso4xrum", "landed_at": "2022-11-09T08:09:11Z", "submitted_at": "2022-11-09T08:33:38Z", "metadata": {"user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Mobile/15E148 Safari/604.1", "platform": "mobile", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl?typeform-source=www.linkedin.com", "network_id": "fec8bf87e1", "browser": "touch"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "wS5FKMUnMgqR", "uvmLX80Loava"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97"], "labels": ["Dec 12-16", "Jan 16-20", "Jan 22-26"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222827} +{"stream": "responses", "data": {"landing_id": "r4epuzzxlonggr4epp07wenb3a58sm6h", "token": "r4epuzzxlonggr4epp07wenb3a58sm6h", "response_id": "r4epuzzxlonggr4epp07wenb3a58sm6h", "landed_at": "2022-11-09T14:33:46Z", "submitted_at": "2022-11-09T14:35:10Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "7e338d2504", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk"], "refs": ["3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Feb 6 - 11", "Feb 13-17", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522223051} +{"stream": "responses", "data": {"landing_id": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "token": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "response_id": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "landed_at": "2022-11-15T02:31:04Z", "submitted_at": "2022-11-15T02:34:53Z", "metadata": {"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8284380108", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "iI0hDpta14Kk"], "refs": ["1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522223053} +{"stream": "responses", "data": {"landing_id": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "token": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "response_id": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "landed_at": "2022-11-15T02:36:40Z", "submitted_at": "2022-11-15T02:39:52Z", "metadata": {"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8284380108", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "iI0hDpta14Kk"], "refs": ["1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522223249} +{"stream": "responses", "data": {"landing_id": "w9yrjygpz00o1vop20h68tlw9yriwyqr", "token": "w9yrjygpz00o1vop20h68tlw9yriwyqr", "response_id": "w9yrjygpz00o1vop20h68tlw9yriwyqr", "landed_at": "2021-06-27T15:16:09Z", "submitted_at": "2021-06-27T15:18:12Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/SdMKQYkv", "network_id": "abd4cbf203", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "63WvXUnvSCa9", "type": "short_text", "ref": "ef34b985c51e4131"}, "type": "text", "text": "Mr X"}, {"field": {"id": "kwpFrd2lI3ok", "type": "email", "ref": "0c3cabd70157cf16"}, "type": "email", "email": "mrx@airbyte.com"}, {"field": {"id": "1Ua3d1mzhJwj", "type": "picture_choice", "ref": "7207397713e2b5e3"}, "type": "choice", "choice": {"id": "z5hxxjpJl07L", "ref": "bfcc3fbf608583f7", "label": "Yes"}}, {"field": {"id": "MmrPLXSaCF5B", "type": "short_text", "ref": "9aaaeeebe70858c4"}, "type": "text", "text": "water"}, {"field": {"id": "gurSOcuvNnvb", "type": "long_text", "ref": "18842abd9aa9ded4"}, "type": "text", "text": "do you know who I am ?"}], "form_id": "SdMKQYkv"}, "emitted_at": 1687522223916} +{"stream": "responses", "data": {"landing_id": "x9ege1s9u758nla0bx9ege1bg63u7daz", "token": "x9ege1s9u758nla0bx9ege1bg63u7daz", "response_id": "x9ege1s9u758nla0bx9ege1bg63u7daz", "landed_at": "2021-07-01T10:03:21Z", "submitted_at": "2021-07-01T10:04:01Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/kRt99jlK", "network_id": "32d3e45763", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "qthblBc7InVU", "type": "multiple_choice", "ref": "8267768033031e53"}, "type": "choice", "choice": {"id": "3XbsOhLiGFkv", "ref": "fc3c3f5dbe75a01e", "label": "Center-right party"}}, {"field": {"id": "rB7FJUThFlu4", "type": "picture_choice", "ref": "b13b02912db6f287"}, "type": "choice", "choice": {"id": "Jt8FTorS35Sb", "ref": "b99bb45c5b8c25be", "label": "Going in the wrong direction"}}, {"field": {"id": "vV7ISYSgZ94I", "type": "opinion_scale", "ref": "f1939629f760be75"}, "type": "number", "number": 1}, {"field": {"id": "Mrq4qNeRInni", "type": "picture_choice", "ref": "c52566d91c5052e2"}, "type": "choice", "choice": {"id": "szBe0vqnkCUK", "ref": "3f1692f85c3cd73b", "label": "Military & Defense"}}, {"field": {"id": "x9myjwStSn9a", "type": "rating", "ref": "55c2e5c15f7dccec"}, "type": "number", "number": 1}, {"field": {"id": "zaP8jDAArI5x", "type": "rating", "ref": "f853e99096a32208"}, "type": "number", "number": 1}, {"field": {"id": "VFmcjbHlFTzg", "type": "rating", "ref": "6f0fd734177ecf27"}, "type": "number", "number": 1}, {"field": {"id": "DgGh4ZkRBAyH", "type": "rating", "ref": "b4171ed292dc3cee"}, "type": "number", "number": 1}, {"field": {"id": "yC3UrwN1LKT8", "type": "rating", "ref": "c8dd7d63c26777d9"}, "type": "number", "number": 5}], "form_id": "kRt99jlK"}, "emitted_at": 1687522224466} +{"stream": "responses", "data": {"landing_id": "ohgpotzjg8w852pohgpo1ub0gq76tks2", "token": "ohgpotzjg8w852pohgpo1ub0gq76tks2", "response_id": "ohgpotzjg8w852pohgpo1ub0gq76tks2", "landed_at": "2021-06-20T16:49:16Z", "submitted_at": "2021-06-20T16:49:20Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "1866502915", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "11"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225029} +{"stream": "responses", "data": {"landing_id": "74zhlkhbmspze5nllpl143674zhlkh81", "token": "74zhlkhbmspze5nllpl143674zhlkh81", "response_id": "74zhlkhbmspze5nllpl143674zhlkh81", "landed_at": "2021-06-27T15:32:07Z", "submitted_at": "2021-06-27T15:32:14Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "abd4cbf203", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "222"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "7jNEfjJ2cDAl", "ref": "01F8N53B82RE3YZK7RR50KNRQ0", "label": "Not so well..."}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225029} +{"stream": "responses", "data": {"landing_id": "9f7s89wh4wagc4qbr1c9f7s89tr0cu6g", "token": "9f7s89wh4wagc4qbr1c9f7s89tr0cu6g", "response_id": "9f7s89wh4wagc4qbr1c9f7s89tr0cu6g", "landed_at": "2021-06-27T15:32:33Z", "submitted_at": "2021-06-27T15:32:39Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "abd4cbf203", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "There is a new library called furl. I find this library to be most pythonic for doing url algebra. To install:"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225225} +{"stream": "responses", "data": {"landing_id": "zn4d1osa0ou5gitzn4k3e3am0c0q7vgd", "token": "zn4d1osa0ou5gitzn4k3e3am0c0q7vgd", "response_id": "zn4d1osa0ou5gitzn4k3e3am0c0q7vgd", "landed_at": "2021-07-01T14:06:57Z", "submitted_at": "2021-07-01T14:07:03Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "32d3e45763", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "124125125"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "7jNEfjJ2cDAl", "ref": "01F8N53B82RE3YZK7RR50KNRQ0", "label": "Not so well..."}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225226} +{"stream": "responses", "data": {"landing_id": "s3ah741anof3uot3340qs3ah7q62w544", "token": "s3ah741anof3uot3340qs3ah7q62w544", "response_id": "s3ah741anof3uot3340qs3ah7q62w544", "landed_at": "2021-09-04T14:35:19Z", "submitted_at": "2021-09-04T14:35:30Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "8c4966ac74", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "test123"}, {"field": {"id": "6lGZzhNfrqwB", "type": "multiple_choice", "ref": "43153da3-fbbc-443e-b66f-1752770c0e0a"}, "type": "choices", "choices": {"ids": ["03VP9UxCwCLT", "ELm7HbFr0OOq", "acwDGU8NeO2A", "3HfyxDo5JoXf"], "refs": ["27b8dfcb-ef16-4ad7-b2be-734ec24c34ca", "ce51ab49-2cce-490d-b831-309337c79fa0", "74ef0411-0c8a-4c09-a6f3-7a62b0745f68", "f83999f6-c869-47cc-af2f-f22b628a0fdb"], "labels": ["choice 4", "choice2", "choice1", "choice 3"]}}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225440} {"stream":"workspaces","data":{"default":false,"forms":{"count":4,"href":"https://api.typeform.com/forms?workspace_id=sDaAqs"},"id":"sDaAqs","name":"My workspace","account_id":"01F8CZR731ZFGBGBEKHMFD5J6Y","self":{"href":"https://api.typeform.com/workspaces/sDaAqs"},"shared":false},"emitted_at":1673035162976} {"stream":"images","data":{"id":"JD76sXLuakwc","src":"https://images.typeform.com/images/JD76sXLuakwc","file_name":"1200x1200 logo.png","width":1200,"height":1200,"media_type":"image/png","has_alpha":true,"avg_color":"d1cbfe"},"emitted_at":1673035163945} {"stream":"images","data":{"id":"D7r8BDHAa5ac","src":"https://images.typeform.com/images/D7r8BDHAa5ac","file_name":"1200x1200 logo.png","width":1200,"height":1200,"media_type":"image/png","has_alpha":true,"avg_color":"d1cbfe"},"emitted_at":1673035163946} diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-typeform/integration_tests/invalid_config_oauth.json new file mode 100644 index 000000000000..8cee0b0c3b77 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/invalid_config_oauth.json @@ -0,0 +1,10 @@ +{ + "credentials": { + "access_token": "fake-token", + "refresh_token": "fake-refresh-token", + "client_id": "fake_client_id", + "client_secret": "fake_client_secret", + "token_expiry_date": "2021-06-27T15:32:38Z" + }, + "start_date": "2021-06-27T15:32:38Z" +} diff --git a/airbyte-integrations/connectors/source-typeform/metadata.yaml b/airbyte-integrations/connectors/source-typeform/metadata.yaml index 9f211e41deda..8c8904a2e6a7 100644 --- a/airbyte-integrations/connectors/source-typeform/metadata.yaml +++ b/airbyte-integrations/connectors/source-typeform/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7eff203-90bf-43e5-a240-19ea3056c474 - dockerImageTag: 0.1.12 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-typeform githubIssueLabel: source-typeform icon: typeform.svg @@ -14,10 +14,15 @@ data: registries: cloud: enabled: true + dockerImageTag: 1.0.0 oss: enabled: true - releaseStage: beta + releaseStage: generally_available documentationUrl: https://docs.airbyte.com/integrations/sources/typeform tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-typeform/requirements.txt b/airbyte-integrations/connectors/source-typeform/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-typeform/requirements.txt +++ b/airbyte-integrations/connectors/source-typeform/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-typeform/setup.py b/airbyte-integrations/connectors/source-typeform/setup.py index 05ed7daeb78a..720f7ea64da1 100644 --- a/airbyte-integrations/connectors/source-typeform/setup.py +++ b/airbyte-integrations/connectors/source-typeform/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "connector-acceptance-test", "pytest-mock~=3.6", "requests_mock~=1.8"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] setup( name="source_typeform", diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py b/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py index e6773c778de9..4398c5361697 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# """ MIT License diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/forms.json b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/forms.json index 3e8697331b7a..74fabf0520e6 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/forms.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/forms.json @@ -7,6 +7,18 @@ "type": { "type": ["null", "string"] }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, "title": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/images.json b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/images.json index 919ebb768cc1..c63362ff716f 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/images.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/images.json @@ -9,6 +9,21 @@ }, "src": { "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + }, + "media_type": { + "type": ["null", "string"] + }, + "avg_color": { + "type": ["null", "string"] + }, + "has_alpha": { + "type": ["null", "boolean"] } }, "$schema": "http://json-schema.org/draft-07/schema#" diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json index 88c145066497..6097f2e5d155 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json @@ -7,9 +7,18 @@ "landed_at": { "type": ["null", "string"] }, + "landing_id": { + "type": ["null", "string"] + }, "submitted_at": { "type": ["null", "string"] }, + "token": { + "type": ["null", "string"] + }, + "form_id": { + "type": ["null", "string"] + }, "metadata": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/themes.json b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/themes.json index 6fa549bb3f73..8bfacfea8eee 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/themes.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/themes.json @@ -55,6 +55,9 @@ "name": { "type": ["null", "string"] }, + "rounded_corners": { + "type": ["null", "string"] + }, "screens": { "type": ["null", "object"], "properties": { @@ -68,6 +71,14 @@ }, "visibility": { "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" } }, "$schema": "http://json-schema.org/draft-07/schema#" diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/workspaces.json b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/workspaces.json index d0fad009fda9..4497275e891a 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/workspaces.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/workspaces.json @@ -1,6 +1,12 @@ { "type": "object", "properties": { + "account_id": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + }, "forms": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py index 23e36f656c8b..17c48ae7b27a 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py @@ -16,7 +16,9 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator from pendulum.datetime import DateTime +from requests.auth import AuthBase class TypeformStream(HttpStream, ABC): @@ -28,7 +30,10 @@ class TypeformStream(HttpStream, ABC): def __init__(self, **kwargs: Mapping[str, Any]): super().__init__(authenticator=kwargs["authenticator"]) self.config: Mapping[str, Any] = kwargs - self.start_date: DateTime = pendulum.from_format(kwargs["start_date"], self.date_format) + # if start_date is not provided during setup, use date from a year ago instead + self.start_date: DateTime = pendulum.today().subtract(years=1) + if kwargs.get("start_date"): + self.start_date: DateTime = pendulum.from_format(kwargs["start_date"], self.date_format) # changes page limit, this param is using for development and debugging if kwargs.get("page_size"): @@ -192,6 +197,12 @@ def request_params( return params + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + responses = response.json()["items"] + for response in responses: + response["form_id"] = stream_slice["form_id"] + return responses + class Webhooks(TrimFormsMixin, TypeformStream): """ @@ -245,13 +256,20 @@ def path(self, **kwargs) -> str: class SourceTypeform(AbstractSource): + def get_auth(self, config: MutableMapping) -> AuthBase: + credentials = config.get("credentials") + if credentials and credentials.get("access_token"): + return TokenAuthenticator(token=credentials["access_token"]) + return SingleUseRefreshTokenOauth2Authenticator(config, token_refresh_endpoint="https://api.typeform.com/oauth/token") + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: try: form_ids = config.get("form_ids", []).copy() + auth = self.get_auth(config) # verify if form inputted by user is valid try: url = urlparse.urljoin(TypeformStream.url_base, "me") - auth_headers = {"Authorization": f"Bearer {config['token']}"} + auth_headers = auth.get_auth_header() session = requests.get(url, headers=auth_headers) session.raise_for_status() except Exception as e: @@ -260,7 +278,6 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> for form in form_ids: try: url = urlparse.urljoin(TypeformStream.url_base, f"forms/{form}") - auth_headers = {"Authorization": f"Bearer {config['token']}"} response = requests.get(url, headers=auth_headers) response.raise_for_status() except Exception as e: @@ -276,7 +293,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = TokenAuthenticator(token=config["token"]) + auth = self.get_auth(config) return [ Forms(authenticator=auth, **config), Responses(authenticator=auth, **config), diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json b/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json index ebd8e7a279fb..ea424a86dc97 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json @@ -2,25 +2,84 @@ "documentationUrl": "https://docs.airbyte.com/integrations/sources/typeform", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Typeform Spec", + "title": "Source Typeform Spec", "type": "object", - "required": ["token", "start_date"], + "required": ["credentials"], "additionalProperties": true, "properties": { - "token": { - "type": "string", - "description": "The API Token for a Typeform account.", - "title": "API Token", - "airbyte_secret": true, - "order": 0 + "credentials": { + "title": "Authorization Method", + "type": "object", + "order": 0, + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": [ + "client_id", + "client_secret", + "refresh_token", + "access_token", + "token_expiry_date" + ], + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0" + }, + "client_id": { + "type": "string", + "description": "The Client ID of the Typeform developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "description": "The Client Secret the Typeform developer application.", + "airbyte_secret": true + }, + "access_token": { + "type": "string", + "description": "Access Token for making authenticated requests.", + "airbyte_secret": true + }, + "token_expiry_date": { + "type": "string", + "description": "The date-time when the access token should be refreshed.", + "format": "date-time" + }, + "refresh_token": { + "type": "string", + "description": "The key to refresh the expired access_token.", + "airbyte_secret": true + } + } + }, + { + "title": "Private Token", + "type": "object", + "required": ["access_token"], + "properties": { + "auth_type": { + "type": "string", + "const": "access_token" + }, + "access_token": { + "type": "string", + "title": "Private Token", + "description": "Log into your Typeform account and then generate a personal Access Token.", + "airbyte_secret": true + } + } + } + ] }, "start_date": { "type": "string", - "description": "UTC date and time in the format: YYYY-MM-DDTHH:mm:ss[Z]. Any data before this date will not be replicated.", "title": "Start Date", - "examples": ["2020-01-01T00:00:00Z"], + "description": "The date from which you'd like to replicate data for Typeform API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "examples": ["2021-03-01T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 1, + "order": 2, "format": "date-time" }, "form_ids": { @@ -31,7 +90,56 @@ "type": "string" }, "uniqueItems": true, - "order": 2 + "order": 3 + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + }, + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + }, + "token_expiry_date": { + "type": "string", + "format": "date-time", + "path_in_connector_config": ["credentials", "token_expiry_date"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py b/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py index 670b3c9d4a6e..ef5db854a4a9 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py @@ -7,12 +7,12 @@ @pytest.fixture def config(): - return {"start_date": "2020-01-01T00:00:00Z", "token": "7607999ef26581e81726777b7b79f20e70e75602", "form_ids": ["u6nXL7", "k9xNV4"]} + return {"start_date": "2020-01-01T00:00:00Z", "credentials": {"access_token": "7607999ef26581e81726777b7b79f20e70e75602"}, "form_ids": ["u6nXL7", "k9xNV4"]} @pytest.fixture def config_without_forms(): - return {"start_date": "2020-01-01T00:00:00Z", "token": "7607999ef26581e81726777b7b79f20e70e75602"} + return {"start_date": "2020-01-01T00:00:00Z", "credentials":{"access_token": "7607999ef26581e81726777b7b79f20e70e75602"}} @pytest.fixture diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py index b5fd2ec9a515..50e4ce49a0e7 100644 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py @@ -49,7 +49,8 @@ def test_check_connection_empty(): def test_check_connection_incomplete(config): - config.pop("token") + credentials = config["credentials"] + credentials.pop("access_token") ok, error = SourceTypeform().check_connection(logger, config) diff --git a/airbyte-integrations/connectors/source-unleash/metadata.yaml b/airbyte-integrations/connectors/source-unleash/metadata.yaml index ff3c4db0dc1a..84e08f649241 100644 --- a/airbyte-integrations/connectors/source-unleash/metadata.yaml +++ b/airbyte-integrations/connectors/source-unleash/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/unleash tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-unleash/requirements.txt b/airbyte-integrations/connectors/source-unleash/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-unleash/requirements.txt +++ b/airbyte-integrations/connectors/source-unleash/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-unleash/setup.py b/airbyte-integrations/connectors/source-unleash/setup.py index a64ced4cf045..8c00746f3fd4 100644 --- a/airbyte-integrations/connectors/source-unleash/setup.py +++ b/airbyte-integrations/connectors/source-unleash/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-unleash/source_unleash/spec.yaml b/airbyte-integrations/connectors/source-unleash/source_unleash/spec.yaml index b254b90a4d29..fb3738ce437c 100644 --- a/airbyte-integrations/connectors/source-unleash/source_unleash/spec.yaml +++ b/airbyte-integrations/connectors/source-unleash/source_unleash/spec.yaml @@ -29,11 +29,11 @@ connectionSpecification: title: Project Name type: string description: >- - Use this if you want to filter the API call for only one given project (can be used in addition to the "Feature Name Prefix" field). - See here + Use this if you want to filter the API call for only one given project (can be used in addition to the "Feature Name Prefix" field). + See here nameprefix: title: Feature Name Prefix type: string description: >- - Use this if you want to filter the API call for only one given project (can be used in addition to the "Feature Name Prefix" field). - See here + Use this if you want to filter the API call for only one given project (can be used in addition to the "Feature Name Prefix" field). + See here diff --git a/airbyte-integrations/connectors/source-unleash/source_unleash/unleash.yaml b/airbyte-integrations/connectors/source-unleash/source_unleash/unleash.yaml index 32f0711073f2..1c2a9b172e3c 100644 --- a/airbyte-integrations/connectors/source-unleash/source_unleash/unleash.yaml +++ b/airbyte-integrations/connectors/source-unleash/source_unleash/unleash.yaml @@ -40,11 +40,11 @@ definitions: # API Docs: https://docs.getunleash.io/reference/api/legacy/unleash/client/features features_stream: - $ref: "#/definitions/base_stream" + $ref: "#/definitions/base_stream" retriever: - $ref: "#/definitions/retriever" + $ref: "#/definitions/retriever" record_selector: - $ref: "#/definitions/selector_features" + $ref: "#/definitions/selector_features" $parameters: name: "features" primary_key: "name" diff --git a/airbyte-integrations/connectors/source-us-census/metadata.yaml b/airbyte-integrations/connectors/source-us-census/metadata.yaml index 3034da3858bf..47c5e9e77e8b 100644 --- a/airbyte-integrations/connectors/source-us-census/metadata.yaml +++ b/airbyte-integrations/connectors/source-us-census/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/us-census tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-us-census/requirements.txt b/airbyte-integrations/connectors/source-us-census/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-us-census/requirements.txt +++ b/airbyte-integrations/connectors/source-us-census/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-us-census/setup.py b/airbyte-integrations/connectors/source-us-census/setup.py index a130afc5e117..6b00267e73c8 100644 --- a/airbyte-integrations/connectors/source-us-census/setup.py +++ b/airbyte-integrations/connectors/source-us-census/setup.py @@ -10,9 +10,10 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", "pytest~=6.1", "responses~=0.13", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-vantage/metadata.yaml b/airbyte-integrations/connectors/source-vantage/metadata.yaml index 09feed11eca2..968cef66b541 100644 --- a/airbyte-integrations/connectors/source-vantage/metadata.yaml +++ b/airbyte-integrations/connectors/source-vantage/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-vantage/requirements.txt b/airbyte-integrations/connectors/source-vantage/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-vantage/requirements.txt +++ b/airbyte-integrations/connectors/source-vantage/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-vantage/setup.py b/airbyte-integrations/connectors/source-vantage/setup.py index 2ece4a4120e2..11fe68724009 100644 --- a/airbyte-integrations/connectors/source-vantage/setup.py +++ b/airbyte-integrations/connectors/source-vantage/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-visma-economic/metadata.yaml b/airbyte-integrations/connectors/source-visma-economic/metadata.yaml index 6f6bbe8814ba..07e591456923 100644 --- a/airbyte-integrations/connectors/source-visma-economic/metadata.yaml +++ b/airbyte-integrations/connectors/source-visma-economic/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/visma-economic tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-visma-economic/requirements.txt b/airbyte-integrations/connectors/source-visma-economic/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-visma-economic/requirements.txt +++ b/airbyte-integrations/connectors/source-visma-economic/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-visma-economic/setup.py b/airbyte-integrations/connectors/source-visma-economic/setup.py index 9e25020f1bdf..4f72cf569184 100644 --- a/airbyte-integrations/connectors/source-visma-economic/setup.py +++ b/airbyte-integrations/connectors/source-visma-economic/setup.py @@ -9,7 +9,7 @@ "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "connector-acceptance-test", "responses~=0.13.3"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.13.3"] setup( name="source_visma_economic", diff --git a/airbyte-integrations/connectors/source-vitally/metadata.yaml b/airbyte-integrations/connectors/source-vitally/metadata.yaml index 4f62f7002a2d..d233f0fcd5c2 100644 --- a/airbyte-integrations/connectors/source-vitally/metadata.yaml +++ b/airbyte-integrations/connectors/source-vitally/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-vitally/requirements.txt b/airbyte-integrations/connectors/source-vitally/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-vitally/requirements.txt +++ b/airbyte-integrations/connectors/source-vitally/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-vitally/setup.py b/airbyte-integrations/connectors/source-vitally/setup.py index 40bde7a03043..f7091d2171d5 100644 --- a/airbyte-integrations/connectors/source-vitally/setup.py +++ b/airbyte-integrations/connectors/source-vitally/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-waiteraid/metadata.yaml b/airbyte-integrations/connectors/source-waiteraid/metadata.yaml index 859ffffff3f8..bc77862fa906 100644 --- a/airbyte-integrations/connectors/source-waiteraid/metadata.yaml +++ b/airbyte-integrations/connectors/source-waiteraid/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-waiteraid/requirements.txt b/airbyte-integrations/connectors/source-waiteraid/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-waiteraid/requirements.txt +++ b/airbyte-integrations/connectors/source-waiteraid/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-waiteraid/setup.py b/airbyte-integrations/connectors/source-waiteraid/setup.py index 084bcbd878b6..fe5d754e3df5 100644 --- a/airbyte-integrations/connectors/source-waiteraid/setup.py +++ b/airbyte-integrations/connectors/source-waiteraid/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-weatherstack/metadata.yaml b/airbyte-integrations/connectors/source-weatherstack/metadata.yaml index 1156a1ab550c..2317e207aacf 100644 --- a/airbyte-integrations/connectors/source-weatherstack/metadata.yaml +++ b/airbyte-integrations/connectors/source-weatherstack/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/weatherstack tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-weatherstack/requirements.txt b/airbyte-integrations/connectors/source-weatherstack/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-weatherstack/requirements.txt +++ b/airbyte-integrations/connectors/source-weatherstack/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-weatherstack/setup.py b/airbyte-integrations/connectors/source-weatherstack/setup.py index 7b1423a452f4..98887751b488 100644 --- a/airbyte-integrations/connectors/source-weatherstack/setup.py +++ b/airbyte-integrations/connectors/source-weatherstack/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-webflow/metadata.yaml b/airbyte-integrations/connectors/source-webflow/metadata.yaml index 3870faf201cf..2f37cd559298 100644 --- a/airbyte-integrations/connectors/source-webflow/metadata.yaml +++ b/airbyte-integrations/connectors/source-webflow/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/webflow tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-webflow/requirements.txt b/airbyte-integrations/connectors/source-webflow/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-webflow/requirements.txt +++ b/airbyte-integrations/connectors/source-webflow/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-webflow/setup.py b/airbyte-integrations/connectors/source-webflow/setup.py index 80c466796440..8c550837d124 100644 --- a/airbyte-integrations/connectors/source-webflow/setup.py +++ b/airbyte-integrations/connectors/source-webflow/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml b/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml index c84972d7c79d..145617426b9b 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml +++ b/airbyte-integrations/connectors/source-whisky-hunter/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt b/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt +++ b/airbyte-integrations/connectors/source-whisky-hunter/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-whisky-hunter/setup.py b/airbyte-integrations/connectors/source-whisky-hunter/setup.py index 98e0f4f79600..6ea55fe522d9 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/setup.py +++ b/airbyte-integrations/connectors/source-whisky-hunter/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml b/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml index 03b8a5af6732..a553cff05092 100644 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/requirements.txt b/airbyte-integrations/connectors/source-wikipedia-pageviews/requirements.txt index cc57334ef619..d6e1198b1ab1 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/requirements.txt +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py b/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py index 40f708cbe986..9716b7daa4ee 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml index 48a08c12226f..7acd33c6dcdf 100644 --- a/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml @@ -24,9 +24,6 @@ acceptance_tests: timeout_seconds: 3600 expect_records: path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: no fail_on_extra_columns: false incremental: tests: diff --git a/airbyte-integrations/connectors/source-woocommerce/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-woocommerce/integration_tests/configured_catalog.json index 4a715b7ff302..928e62004c4d 100644 --- a/airbyte-integrations/connectors/source-woocommerce/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-woocommerce/integration_tests/configured_catalog.json @@ -98,7 +98,8 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, { + }, + { "stream": { "name": "product_variations", "json_schema": {}, @@ -106,7 +107,8 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, { + }, + { "stream": { "name": "products", "json_schema": {}, diff --git a/airbyte-integrations/connectors/source-woocommerce/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-woocommerce/integration_tests/expected_records.jsonl index ef75432bf7b2..41c734966d3d 100644 --- a/airbyte-integrations/connectors/source-woocommerce/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-woocommerce/integration_tests/expected_records.jsonl @@ -1,67 +1,44 @@ -{"stream":"coupons","data":{"id":44,"code":"test10off","amount":"10.00","status":"publish","date_created":"2022-11-30T20:17:38","date_created_gmt":"2022-11-30T20:17:38","date_modified":"2022-11-30T20:17:38","date_modified_gmt":"2022-11-30T20:17:38","discount_type":"percent","description":"10% OFF","date_expires":null,"date_expires_gmt":null,"usage_count":1,"individual_use":false,"product_ids":[],"excluded_product_ids":[],"usage_limit":null,"usage_limit_per_user":null,"limit_usage_to_x_items":null,"free_shipping":false,"product_categories":[],"excluded_product_categories":[],"exclude_sale_items":false,"minimum_amount":"0.00","maximum_amount":"0.00","email_restrictions":[],"used_by":["210684523"],"meta_data":[],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/coupons/44"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/coupons"}]}},"emitted_at":1671056260289} -{"stream":"customers","data":{"id":210684524,"date_created":"2022-11-30T20:40:42","date_created_gmt":"2022-11-30T20:40:42","date_modified":"2022-11-30T20:40:43","date_modified_gmt":"2022-11-30T20:40:43","email":"john.doe@example.com","first_name":"John","last_name":"Doe","role":"customer","username":"john.doe","billing":{"first_name":"John","last_name":"Doe","company":"","address_1":"969 Market","address_2":"","city":"San Francisco","postcode":"94103","country":"US","state":"CA","email":"john.doe@example.com","phone":"(555) 555-5555"},"shipping":{"first_name":"John","last_name":"Doe","company":"","address_1":"969 Market","address_2":"","city":"San Francisco","postcode":"94103","country":"US","state":"CA","phone":""},"is_paying_customer":false,"avatar_url":"https://secure.gravatar.com/avatar/8eb1b522f60d11fa897de1dc6351b7e8?s=96&d=identicon&r=g","meta_data":[{"id":65,"key":"_wcs_subscription_ids_cache","value":[]}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/customers/210684524"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/customers"}]}},"emitted_at":1671056261444} -{"stream":"order_notes","data":{"id":3,"author":"airbytetesting","date_created":"2022-11-30T20:18:29","date_created_gmt":"2022-11-30T20:18:29","note":"Added line items: test_product_1 (test_prod_1)","customer_note":false,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45/notes/3"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45/notes"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45"}]}},"emitted_at":1671056263076} -{"stream":"orders","data":{"id":45,"parent_id":0,"status":"pending","currency":"USD","version":"7.1.0","prices_include_tax":false,"date_created":"2022-11-30T20:17:50","date_modified":"2022-11-30T20:19:01","discount_total":"0.00","discount_tax":"0.00","shipping_total":"0.00","shipping_tax":"0.00","cart_tax":"0.00","total":"90.00","total_tax":"0.00","customer_id":210684523,"order_key":"wc_order_QxkrRLu9BwBQE","billing":{"first_name":"","last_name":"","company":"","address_1":"","address_2":"","city":"","state":"","postcode":"","country":"","email":"integration-test@airbyte.io","phone":""},"shipping":{"first_name":"","last_name":"","company":"","address_1":"","address_2":"","city":"","state":"","postcode":"","country":"","phone":""},"payment_method":"","payment_method_title":"","transaction_id":"","customer_ip_address":"","customer_user_agent":"","created_via":"admin","customer_note":"","date_completed":null,"date_paid":null,"cart_hash":"","number":"45","meta_data":[{"id":193,"key":"coupon_amount","value":""}],"line_items":[{"id":2,"name":"test_product_1","product_id":40,"variation_id":0,"quantity":1,"tax_class":"","subtotal":"90.00","subtotal_tax":"0.00","total":"90.00","total_tax":"0.00","taxes":[],"meta_data":[],"sku":"test_prod_1","price":90,"image":{"id":"","src":""},"parent_name":null}],"tax_lines":[],"shipping_lines":[],"fee_lines":[],"coupon_lines":[{"id":1,"code":"test10off","discount":"0","discount_tax":"0","meta_data":[{"id":3,"key":"coupon_data","value":{"id":44,"code":"test10off","amount":"10","status":"publish","date_created":{"date":"2022-11-30 20:17:38.000000","timezone_type":3,"timezone":"UTC"},"date_modified":{"date":"2022-11-30 20:17:38.000000","timezone_type":3,"timezone":"UTC"},"date_expires":null,"discount_type":"percent","description":"10% OFF","usage_count":0,"individual_use":false,"product_ids":[],"excluded_product_ids":[],"usage_limit":0,"usage_limit_per_user":0,"limit_usage_to_x_items":null,"free_shipping":false,"product_categories":[],"excluded_product_categories":[],"exclude_sale_items":false,"minimum_amount":"","maximum_amount":"","email_restrictions":[],"virtual":false,"meta_data":[]},"display_key":"coupon_data","display_value":{"id":44,"code":"test10off","amount":"10","status":"publish","date_created":{"date":"2022-11-30 20:17:38.000000","timezone_type":3,"timezone":"UTC"},"date_modified":{"date":"2022-11-30 20:17:38.000000","timezone_type":3,"timezone":"UTC"},"date_expires":null,"discount_type":"percent","description":"10% OFF","usage_count":0,"individual_use":false,"product_ids":[],"excluded_product_ids":[],"usage_limit":0,"usage_limit_per_user":0,"limit_usage_to_x_items":null,"free_shipping":false,"product_categories":[],"excluded_product_categories":[],"exclude_sale_items":false,"minimum_amount":"","maximum_amount":"","email_restrictions":[],"virtual":false,"meta_data":[]}}]}],"refunds":[{"id":60,"reason":"bonus","total":"-10.00"}],"payment_url":"https://airbyte.store/checkout/order-pay/45/?pay_for_order=true&key=wc_order_QxkrRLu9BwBQE","is_editable":true,"needs_payment":true,"needs_processing":true,"date_created_gmt":"2022-11-30T20:17:50","date_modified_gmt":"2022-11-30T20:19:01","date_completed_gmt":null,"date_paid_gmt":null,"currency_symbol":"$","_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/orders"}],"customer":[{"href":"https://airbyte.store/wp-json/wc/v3/customers/210684523"}]}},"emitted_at":1671056264573} -{"stream": "payment_gateways", "data": {"id": "woocommerce_payments", "title": "Credit card / debit card", "description": "Enter your card details", "order": -1, "enabled": false, "method_title": "WooCommerce Payments", "method_description": "WooCommerce Payments gives your store flexibility to accept credit cards, debit cards, and Apple Pay. Enable popular local payment methods and other digital wallets like Google Pay to give customers even more choice.

    \n\t\t\tBy using WooCommerce Payments you agree to be bound by our Terms of Service and acknowledge that you have read our Privacy Policy", "method_supports": ["products", "refunds", "multiple_subscriptions", "subscription_cancellation", "subscription_payment_method_change_admin", "subscription_payment_method_change_customer", "subscription_payment_method_change", "subscription_reactivation", "subscription_suspension", "subscriptions", "gateway_scheduled_payments", "tokenization", "add_payment_method"], "settings": {"account_statement_descriptor": {"id": "account_statement_descriptor", "label": "Customer bank statement", "description": "Edit the way your store name appears on your customers\u2019 bank statements (read more about requirements here).", "type": "account_statement_descriptor", "value": "", "default": "", "tip": "Edit the way your store name appears on your customers\u2019 bank statements (read more about requirements here).", "placeholder": ""}, "manual_capture": {"id": "manual_capture", "label": "Issue an authorization on checkout, and capture later.", "description": "Charge must be captured within 7 days of authorization, otherwise the authorization and order will be canceled.", "type": "checkbox", "value": "no", "default": "no", "tip": "Charge must be captured within 7 days of authorization, otherwise the authorization and order will be canceled.", "placeholder": ""}, "saved_cards": {"id": "saved_cards", "label": "Enable payment via saved cards", "description": "If enabled, users will be able to pay with a saved card during checkout. Card details are saved on our platform, not on your store.", "type": "checkbox", "value": "yes", "default": "yes", "tip": "If enabled, users will be able to pay with a saved card during checkout. Card details are saved on our platform, not on your store.", "placeholder": ""}, "test_mode": {"id": "test_mode", "label": "Enable test mode", "description": "Simulate transactions using test card numbers.", "type": "checkbox", "value": "no", "default": "no", "tip": "Simulate transactions using test card numbers.", "placeholder": ""}, "enable_logging": {"id": "enable_logging", "label": "When enabled debug notes will be added to the log.", "description": "", "type": "checkbox", "value": "no", "default": "no", "tip": "", "placeholder": ""}, "payment_request_details": {"id": "payment_request_details", "label": "Payment request buttons", "description": "", "type": "title", "value": "", "default": "", "tip": "", "placeholder": ""}, "payment_request": {"id": "payment_request", "label": "Enable payment request buttons (Apple Pay, Google Pay, and more).
    By using Apple Pay, you agree to Stripe and Apple's Terms of Service.", "description": "If enabled, users will be able to pay using Apple Pay, Google Pay or the Payment Request API if supported by the browser.", "type": "checkbox", "value": "yes", "default": "no", "tip": "If enabled, users will be able to pay using Apple Pay, Google Pay or the Payment Request API if supported by the browser.", "placeholder": ""}, "payment_request_button_type": {"id": "payment_request_button_type", "label": "Button type", "description": "Select the button type you would like to show.", "type": "select", "value": "buy", "default": "buy", "tip": "Select the button type you would like to show.", "placeholder": "", "options": {"default": "Only icon", "buy": "Buy", "donate": "Donate", "book": "Book"}}, "payment_request_button_theme": {"id": "payment_request_button_theme", "label": "Button theme", "description": "Select the button theme you would like to show.", "type": "select", "value": "dark", "default": "dark", "tip": "Select the button theme you would like to show.", "placeholder": "", "options": {"dark": "Dark", "light": "Light", "light-outline": "Light-Outline"}}, "payment_request_button_height": {"id": "payment_request_button_height", "label": "Button height", "description": "Enter the height you would like the button to be in pixels. Width will always be 100%.", "type": "text", "value": "44", "default": "44", "tip": "Enter the height you would like the button to be in pixels. Width will always be 100%.", "placeholder": ""}, "payment_request_button_label": {"id": "payment_request_button_label", "label": "Custom button label", "description": "Enter the custom text you would like the button to have.", "type": "text", "value": "Buy now", "default": "Buy now", "tip": "Enter the custom text you would like the button to have.", "placeholder": ""}, "payment_request_button_locations": {"id": "payment_request_button_locations", "label": "Button locations", "description": "Select where you would like to display the button.", "type": "multiselect", "value": ["product", "cart", "checkout"], "default": ["product", "cart", "checkout"], "tip": "Select where you would like to display the button.", "placeholder": "", "options": {"product": "Product", "cart": "Cart", "checkout": "Checkout"}}, "upe_enabled_payment_method_ids": {"id": "upe_enabled_payment_method_ids", "label": "Payments accepted on checkout", "description": "", "type": "multiselect", "value": ["card"], "default": ["card"], "tip": "", "placeholder": ""}, "payment_request_button_size": {"id": "payment_request_button_size", "label": "Size of the button displayed for Express Checkouts", "description": "Select the size of the button.", "type": "select", "value": "default", "default": "default", "tip": "Select the size of the button.", "placeholder": "", "options": {"default": "Default", "medium": "Medium", "large": "Large"}}}, "needs_setup": true, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=woocommerce_payments", "connection_url": "", "setup_help_text": "Next we\u2019ll ask you to share a few details about your business to create your account.", "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/woocommerce_payments"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1675782304665} -{"stream": "payment_gateways", "data": {"id": "bacs", "title": "Direct bank transfer", "description": "Make your payment directly into our bank account. Please use your Order ID as the payment reference. Your order will not be shipped until the funds have cleared in our account.", "order": "", "enabled": false, "method_title": "Direct bank transfer", "method_description": "Take payments in person via BACS. More commonly known as direct bank/wire transfer.", "method_supports": ["products"], "settings": {"title": {"id": "title", "label": "Title", "description": "This controls the title which the user sees during checkout.", "type": "safe_text", "value": "Direct bank transfer", "default": "Direct bank transfer", "tip": "This controls the title which the user sees during checkout.", "placeholder": ""}, "instructions": {"id": "instructions", "label": "Instructions", "description": "Instructions that will be added to the thank you page and emails.", "type": "textarea", "value": "", "default": "", "tip": "Instructions that will be added to the thank you page and emails.", "placeholder": ""}, "accounts": {"id": "accounts", "value": []}}, "needs_setup": false, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=bacs", "connection_url": null, "setup_help_text": null, "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/bacs"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1675782304666} -{"stream": "payment_gateways", "data": {"id": "cheque", "title": "Check payments", "description": "Pay for this order by check.", "order": "", "enabled": false, "method_title": "Check payments", "method_description": "Take payments in person via checks. This offline gateway can also be useful to test purchases.", "method_supports": ["products"], "settings": {"title": {"id": "title", "label": "Title", "description": "This controls the title which the user sees during checkout.", "type": "safe_text", "value": "Check payments", "default": "Check payments", "tip": "This controls the title which the user sees during checkout.", "placeholder": ""}, "instructions": {"id": "instructions", "label": "Instructions", "description": "Instructions that will be added to the thank you page and emails.", "type": "textarea", "value": "Make your check payable to...", "default": "", "tip": "Instructions that will be added to the thank you page and emails.", "placeholder": ""}}, "needs_setup": false, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=cheque", "connection_url": null, "setup_help_text": null, "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/cheque"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1675782304667} -{"stream": "payment_gateways", "data": {"id": "cod", "title": "Cash on delivery", "description": "Pay with cash upon delivery.", "order": "", "enabled": false, "method_title": "Cash on delivery", "method_description": "Have your customers pay with cash (or by other means) upon delivery.", "method_supports": ["products"], "settings": {"title": {"id": "title", "label": "Title", "description": "Payment method description that the customer will see on your checkout.", "type": "safe_text", "value": "Cash on delivery", "default": "Cash on delivery", "tip": "Payment method description that the customer will see on your checkout.", "placeholder": ""}, "instructions": {"id": "instructions", "label": "Instructions", "description": "Instructions that will be added to the thank you page.", "type": "textarea", "value": "Pay with cash upon delivery.", "default": "Pay with cash upon delivery.", "tip": "Instructions that will be added to the thank you page.", "placeholder": ""}, "enable_for_methods": {"id": "enable_for_methods", "label": "Enable for shipping methods", "description": "If COD is only available for certain methods, set it up here. Leave blank to enable for all methods.", "type": "multiselect", "value": "", "default": "", "tip": "If COD is only available for certain methods, set it up here. Leave blank to enable for all methods.", "placeholder": "", "options": {"Flat rate": {"flat_rate": "Any "Flat rate" method"}, "Free shipping": {"free_shipping": "Any "Free shipping" method"}, "Local pickup": {"local_pickup": "Any "Local pickup" method", "local_pickup:1": "California ZIP 90210 – Local pickup (#1)"}}}, "enable_for_virtual": {"id": "enable_for_virtual", "label": "Accept COD if the order is virtual", "description": "", "type": "checkbox", "value": "yes", "default": "yes", "tip": "", "placeholder": ""}}, "needs_setup": false, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=cod", "connection_url": null, "setup_help_text": null, "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/cod"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1675782304669} -{"stream":"product_attribute_terms","data":{"id":1385,"name":"7","slug":"7","description":"size 7","menu_order":0,"count":1,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms/1385"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms"}]}},"emitted_at":1671056267133} -{"stream":"product_attribute_terms","data":{"id":1386,"name":"8","slug":"8","description":"size 8","menu_order":0,"count":1,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms/1386"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms"}]}},"emitted_at":1671056267134} -{"stream":"product_attribute_terms","data":{"id":1387,"name":"6","slug":"6","description":"size 6","menu_order":0,"count":1,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms/1387"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms"}]}},"emitted_at":1671056267134} -{"stream":"product_attributes","data":{"id":1,"name":"size","slug":"pa_size","type":"select","order_by":"menu_order","has_archives":false,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes/1"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/attributes"}]}},"emitted_at":1671056267733} -{"stream":"product_categories","data":{"id":1374,"name":"Uncategorized","slug":"uncategorized","parent":0,"description":"","display":"default","image":null,"menu_order":0,"count":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/categories/1374"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/categories"}]}},"emitted_at":1671056268340} -{"stream":"product_categories","data":{"id":1378,"name":"sale_products","slug":"sale_products","parent":0,"description":"","display":"default","image":null,"menu_order":0,"count":2,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/categories/1378"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/categories"}]}},"emitted_at":1671056268341} -{"stream":"product_categories","data":{"id":1381,"name":"purchasable_products","slug":"purchasable_products","parent":0,"description":"Purchasable","display":"products","image":null,"menu_order":0,"count":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/categories/1381"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/categories"}]}},"emitted_at":1671056268342} -{"stream":"product_reviews","data":{"id":1,"date_created":"2022-11-30T20:05:50","date_created_gmt":"2022-11-30T20:05:50","product_id":40,"product_name":"test_product_1","product_permalink":"https://airbyte.store/product/test_product_1/","status":"approved","reviewer":"airbytetesting","reviewer_email":"integration-test@airbyte.io","review":"

    test_review

    \n","rating":5,"verified":false,"reviewer_avatar_urls":{"24":"https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=24&d=identicon&r=g","48":"https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=48&d=identicon&r=g","96":"https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=96&d=identicon&r=g"},"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/reviews/1"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/reviews"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/40"}],"reviewer":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wp/v2/users/210684523"}]}},"emitted_at":1671056269385} -{"stream":"product_reviews","data":{"id":2,"date_created":"2022-11-30T20:06:01","date_created_gmt":"2022-11-30T20:06:01","product_id":40,"product_name":"test_product_1","product_permalink":"https://airbyte.store/product/test_product_1/","status":"approved","reviewer":"airbytetesting","reviewer_email":"integration-test@airbyte.io","review":"

    test_review 2

    \n","rating":4,"verified":false,"reviewer_avatar_urls":{"24":"https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=24&d=identicon&r=g","48":"https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=48&d=identicon&r=g","96":"https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=96&d=identicon&r=g"},"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/reviews/2"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/reviews"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/40"}],"reviewer":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wp/v2/users/210684523"}]}},"emitted_at":1671056269387} -{"stream":"product_shipping_classes","data":{"id":1388,"name":"test_ship_class","slug":"test_ship_class","description":"","count":2,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/shipping_classes/1388"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/shipping_classes"}]}},"emitted_at":1671056270524} -{"stream":"product_tags","data":{"id":1379,"name":"sample_tag","slug":"sample_tag","description":"","count":2,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags/1379"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags"}]}},"emitted_at":1671056271109} -{"stream":"product_tags","data":{"id":1380,"name":"saleable","slug":"saleable","description":"","count":2,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags/1380"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags"}]}},"emitted_at":1671056271118} -{"stream":"product_tags","data":{"id":1382,"name":"clothes","slug":"clothes","description":"Clothes","count":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags/1382"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags"}]}},"emitted_at":1671056271120} -{"stream":"product_tags","data":{"id":1383,"name":"sports","slug":"sports","description":"","count":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags/1383"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags"}]}},"emitted_at":1671056271127} -{"stream":"product_tags","data":{"id":1384,"name":"food","slug":"food","description":"food","count":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags/1384"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/tags"}]}},"emitted_at":1671056271128} -{"stream":"product_variations","data":{"id":51,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=white&attribute_pa_size=6","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"white"},{"id":1,"name":"size","option":"6"}],"menu_order":1,"meta_data":[{"id":314,"key":"_created_via","value":"ajax-unknown"},{"id":333,"key":"_subscription_period","value":"month"},{"id":334,"key":"_subscription_period_interval","value":"1"},{"id":335,"key":"_subscription_length","value":"0"},{"id":336,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/51"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274266} -{"stream":"product_variations","data":{"id":52,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=white&attribute_pa_size=7","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"white"},{"id":1,"name":"size","option":"7"}],"menu_order":2,"meta_data":[{"id":337,"key":"_created_via","value":"ajax-unknown"},{"id":356,"key":"_subscription_period","value":"month"},{"id":357,"key":"_subscription_period_interval","value":"1"},{"id":358,"key":"_subscription_length","value":"0"},{"id":359,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/52"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274270} -{"stream":"product_variations","data":{"id":53,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=white&attribute_pa_size=8","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"white"},{"id":1,"name":"size","option":"8"}],"menu_order":3,"meta_data":[{"id":360,"key":"_created_via","value":"ajax-unknown"},{"id":379,"key":"_subscription_period","value":"month"},{"id":380,"key":"_subscription_period_interval","value":"1"},{"id":381,"key":"_subscription_length","value":"0"},{"id":382,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/53"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274272} -{"stream":"product_variations","data":{"id":54,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=red&attribute_pa_size=6","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"red"},{"id":1,"name":"size","option":"6"}],"menu_order":4,"meta_data":[{"id":383,"key":"_created_via","value":"ajax-unknown"},{"id":402,"key":"_subscription_period","value":"month"},{"id":403,"key":"_subscription_period_interval","value":"1"},{"id":404,"key":"_subscription_length","value":"0"},{"id":405,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/54"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274274} -{"stream":"product_variations","data":{"id":55,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=red&attribute_pa_size=7","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"red"},{"id":1,"name":"size","option":"7"}],"menu_order":5,"meta_data":[{"id":406,"key":"_created_via","value":"ajax-unknown"},{"id":425,"key":"_subscription_period","value":"month"},{"id":426,"key":"_subscription_period_interval","value":"1"},{"id":427,"key":"_subscription_length","value":"0"},{"id":428,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/55"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274276} -{"stream":"product_variations","data":{"id":56,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=red&attribute_pa_size=8","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"red"},{"id":1,"name":"size","option":"8"}],"menu_order":6,"meta_data":[{"id":429,"key":"_created_via","value":"ajax-unknown"},{"id":448,"key":"_subscription_period","value":"month"},{"id":449,"key":"_subscription_period_interval","value":"1"},{"id":450,"key":"_subscription_length","value":"0"},{"id":451,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/56"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274278} -{"stream":"product_variations","data":{"id":57,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=black&attribute_pa_size=6","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"black"},{"id":1,"name":"size","option":"6"}],"menu_order":7,"meta_data":[{"id":452,"key":"_created_via","value":"ajax-unknown"},{"id":471,"key":"_subscription_period","value":"month"},{"id":472,"key":"_subscription_period_interval","value":"1"},{"id":473,"key":"_subscription_length","value":"0"},{"id":474,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/57"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274279} -{"stream":"product_variations","data":{"id":58,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=black&attribute_pa_size=7","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"black"},{"id":1,"name":"size","option":"7"}],"menu_order":8,"meta_data":[{"id":475,"key":"_created_via","value":"ajax-unknown"},{"id":494,"key":"_subscription_period","value":"month"},{"id":495,"key":"_subscription_period_interval","value":"1"},{"id":496,"key":"_subscription_length","value":"0"},{"id":497,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/58"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274281} -{"stream":"product_variations","data":{"id":59,"date_created":"2022-11-30T20:56:17","date_created_gmt":"2022-11-30T20:56:17","date_modified":"2022-11-30T20:56:17","date_modified_gmt":"2022-11-30T20:56:17","description":"","permalink":"https://airbyte.store/product/test_product_2/?attribute_color=black&attribute_pa_size=8","sku":"test_prod_1-1","price":"","regular_price":"","sale_price":"","date_on_sale_from":null,"date_on_sale_from_gmt":null,"date_on_sale_to":null,"date_on_sale_to_gmt":null,"on_sale":false,"status":"publish","purchasable":false,"virtual":false,"downloadable":false,"downloads":[],"download_limit":-1,"download_expiry":-1,"tax_status":"taxable","tax_class":"","manage_stock":"parent","stock_quantity":33,"stock_status":"instock","backorders":"no","backorders_allowed":false,"backordered":false,"low_stock_amount":null,"weight":"10","dimensions":{"length":"10","width":"10","height":"10"},"shipping_class":"test_ship_class","shipping_class_id":1388,"image":null,"attributes":[{"id":0,"name":"color","option":"black"},{"id":1,"name":"size","option":"8"}],"menu_order":9,"meta_data":[{"id":498,"key":"_created_via","value":"ajax-unknown"},{"id":517,"key":"_subscription_period","value":"month"},{"id":518,"key":"_subscription_period_interval","value":"1"},{"id":519,"key":"_subscription_length","value":"0"},{"id":520,"key":"_subscription_trial_period","value":"month"}],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations/59"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46/variations"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/products/46"}]}},"emitted_at":1671056274283} -{"stream": "products", "data": {"id": 26, "name": "Connector source Facebook Marketing", "slug": "album", "permalink": "https://airbyte.store/product/album/", "date_created": "2022-11-23T08:21:10", "date_created_gmt": "2022-11-23T08:21:10", "date_modified": "2022-11-23T08:27:43", "date_modified_gmt": "2022-11-23T08:27:43", "type": "simple", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "

    Album reviewed and chosen by Woo.

    \n", "short_description": "", "sku": "", "price": "0.01", "regular_price": "0.02", "sale_price": "0.01", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": true, "purchasable": true, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": false, "stock_quantity": null, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "", "dimensions": {"length": "", "width": "", "height": ""}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "", "shipping_class_id": 0, "reviews_allowed": true, "average_rating": "0.00", "rating_count": 0, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "", "categories": [], "tags": [], "images": [{"id": 27, "date_created": "2022-11-23T08:26:11", "date_created_gmt": "2022-11-23T08:26:11", "date_modified": "2022-11-23T08:26:33", "date_modified_gmt": "2022-11-23T08:26:33", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1312%2C542&ssl=1", "name": "Facebook Marketing source", "alt": ""}], "attributes": [], "default_attributes": [], "variations": [], "grouped_products": [], "menu_order": 0, "price_html": "$0.02 $0.01", "related_ids": [], "meta_data": [{"id": 62, "key": "_created_via", "value": "unknown"}, {"id": 63, "key": "_wpas_done_all", "value": "1"}, {"id": 80, "key": "_last_editor_used_jetpack", "value": "classic-editor"}], "stock_status": "instock", "has_options": false, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 28, "url": "https://airbyte.store/product/connector-source-google-ads/", "url_meta": {"origin": 26, "position": 0}, "title": "Connector source Google Ads", "author": "airbytetesting", "date": "November 23, 2022", "format": false, "excerpt": "", "rel": "", "context": "Similar post", "block_context": {"text": "Similar post", "link": ""}, "img": {"alt_text": "", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1200%2C483&ssl=1&resize=350%2C200", "width": 350, "height": 200}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/26"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1685713399179} -{"stream": "products", "data": {"id": 28, "name": "Connector source Google Ads", "slug": "connector-source-google-ads", "permalink": "https://airbyte.store/product/connector-source-google-ads/", "date_created": "2022-11-23T08:30:31", "date_created_gmt": "2022-11-23T08:30:31", "date_modified": "2022-11-23T08:30:31", "date_modified_gmt": "2022-11-23T08:30:31", "type": "simple", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "", "short_description": "", "sku": "", "price": "0.01", "regular_price": "0.02", "sale_price": "0.01", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": true, "purchasable": true, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": false, "stock_quantity": null, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "", "dimensions": {"length": "", "width": "", "height": ""}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "", "shipping_class_id": 0, "reviews_allowed": true, "average_rating": "0.00", "rating_count": 0, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "", "categories": [], "tags": [], "images": [{"id": 29, "date_created": "2022-11-23T08:29:16", "date_created_gmt": "2022-11-23T08:29:16", "date_modified": "2022-11-23T08:29:35", "date_modified_gmt": "2022-11-23T08:29:35", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1317%2C530&ssl=1", "name": "Google Ads source", "alt": ""}], "attributes": [], "default_attributes": [], "variations": [], "grouped_products": [], "menu_order": 0, "price_html": "$0.02 $0.01", "related_ids": [], "meta_data": [{"id": 89, "key": "_created_via", "value": "post-new"}, {"id": 90, "key": "_last_editor_used_jetpack", "value": "classic-editor"}, {"id": 115, "key": "_wpas_done_all", "value": "1"}], "stock_status": "instock", "has_options": false, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 26, "url": "https://airbyte.store/product/album/", "url_meta": {"origin": 28, "position": 0}, "title": "Connector source Facebook Marketing", "author": "airbytetesting", "date": "November 23, 2022", "format": false, "excerpt": "Album reviewed and chosen by Woo.", "rel": "", "context": "Similar post", "block_context": {"text": "Similar post", "link": ""}, "img": {"alt_text": "", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1200%2C496&ssl=1&resize=350%2C200", "width": 350, "height": 200}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/28"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1685713399184} -{"stream": "products", "data": {"id": 40, "name": "test_product_1", "slug": "test_product_1", "permalink": "https://airbyte.store/product/test_product_1/", "date_created": "2022-11-30T19:59:56", "date_created_gmt": "2022-11-30T19:59:56", "date_modified": "2022-11-30T20:44:57", "date_modified_gmt": "2022-11-30T20:44:57", "type": "simple", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "

    test_product description

    \n", "short_description": "", "sku": "test_prod_1", "price": "90", "regular_price": "100", "sale_price": "90", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": true, "purchasable": true, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": true, "stock_quantity": 33, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "10", "dimensions": {"length": "10", "width": "10", "height": "10"}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "test_ship_class", "shipping_class_id": 1388, "reviews_allowed": true, "average_rating": "4.50", "rating_count": 2, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "

    vendor_info

    \n", "categories": [{"id": 1378, "name": "sale_products", "slug": "sale_products"}], "tags": [{"id": 1380, "name": "saleable", "slug": "saleable"}, {"id": 1379, "name": "sample_tag", "slug": "sample_tag"}], "images": [], "attributes": [{"id": 0, "name": "color", "position": 0, "visible": true, "variation": false, "options": ["white", "red", "black"]}], "default_attributes": [], "variations": [], "grouped_products": [], "menu_order": 0, "price_html": "$100.00 $90.00", "related_ids": [46], "meta_data": [{"id": 130, "key": "_created_via", "value": "post-new"}, {"id": 131, "key": "_last_editor_used_jetpack", "value": "classic-editor"}, {"id": 160, "key": "_wpas_done_all", "value": "1"}], "stock_status": "instock", "has_options": false, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 46, "url": "https://airbyte.store/product/test_product_2/", "url_meta": {"origin": 40, "position": 0}, "title": "test_product_2", "author": "airbytetesting", "date": "November 30, 2022", "format": false, "excerpt": "test_product\u00a0 2 description", "rel": "", "context": "Similar post", "block_context": {"text": "Similar post", "link": ""}, "img": {"alt_text": "", "src": "", "width": 0, "height": 0}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/40"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1685713399188} -{"stream": "products", "data": {"id": 46, "name": "test_product_2", "slug": "test_product_2", "permalink": "https://airbyte.store/product/test_product_2/", "date_created": "2022-11-30T20:54:22", "date_created_gmt": "2022-11-30T20:54:22", "date_modified": "2022-11-30T20:56:30", "date_modified_gmt": "2022-11-30T20:56:30", "type": "variable", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "

    test_product\u00a0 2 description

    \n", "short_description": "", "sku": "test_prod_1-1", "price": "", "regular_price": "", "sale_price": "", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": false, "purchasable": false, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": true, "stock_quantity": 33, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "10", "dimensions": {"length": "10", "width": "10", "height": "10"}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "test_ship_class", "shipping_class_id": 1388, "reviews_allowed": true, "average_rating": "0.00", "rating_count": 0, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "

    vendor_info

    \n", "categories": [{"id": 1378, "name": "sale_products", "slug": "sale_products"}], "tags": [{"id": 1380, "name": "saleable", "slug": "saleable"}, {"id": 1379, "name": "sample_tag", "slug": "sample_tag"}], "images": [], "attributes": [{"id": 0, "name": "color", "position": 0, "visible": true, "variation": true, "options": ["white", "red", "black"]}, {"id": 1, "name": "size", "position": 1, "visible": true, "variation": true, "options": ["6", "7", "8"]}], "default_attributes": [], "variations": [51, 52, 53, 54, 55, 56, 57, 58, 59], "grouped_products": [], "menu_order": 0, "price_html": "", "related_ids": [40], "meta_data": [{"id": 198, "key": "_created_via", "value": "unknown"}, {"id": 224, "key": "_created_via", "value": "post-new"}, {"id": 225, "key": "_last_editor_used_jetpack", "value": "classic-editor"}, {"id": 226, "key": "_wpas_done_all", "value": "1"}], "stock_status": "instock", "has_options": true, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 40, "url": "https://airbyte.store/product/test_product_1/", "url_meta": {"origin": 46, "position": 0}, "title": "test_product_1", "author": "airbytetesting", "date": "November 30, 2022", "format": false, "excerpt": "test_product description", "rel": "", "context": "With 2 comments", "block_context": {"text": "With 2 comments", "link": "https://airbyte.store/product/test_product_1/#comments"}, "img": {"alt_text": "", "src": "", "width": 0, "height": 0}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1685713399193} -{"stream":"refunds","data":{"id":60,"date_created":"2022-11-30T21:07:07","date_created_gmt":"2022-11-30T21:07:07","amount":"10.00","reason":"bonus","refunded_by":210684523,"refunded_payment":false,"meta_data":[],"line_items":[{"id":3,"name":"test_product_1","product_id":40,"variation_id":0,"quantity":0,"tax_class":"","subtotal":"-10.00","subtotal_tax":"0.00","total":"-10.00","total_tax":"0.00","taxes":[],"meta_data":[{"id":22,"key":"_refunded_item_id","value":"2","display_key":"_refunded_item_id","display_value":"2"}],"sku":"test_prod_1","price":0,"image":{"id":"","src":""},"parent_name":null}],"shipping_lines":[],"tax_lines":[],"fee_lines":[],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45/refunds/60"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45/refunds"}],"up":[{"href":"https://airbyte.store/wp-json/wc/v3/orders/45"}]}},"emitted_at":1671056278499} -{"stream":"shipping_methods","data":{"id":"flat_rate","title":"Flat rate","description":"Lets you charge a fixed rate for shipping.","_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping_methods/flat_rate"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping_methods"}]}},"emitted_at":1671056279633} -{"stream":"shipping_methods","data":{"id":"free_shipping","title":"Free shipping","description":"Free shipping is a special method which can be triggered with coupons and minimum spends.","_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping_methods/free_shipping"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping_methods"}]}},"emitted_at":1671056279637} -{"stream":"shipping_methods","data":{"id":"local_pickup","title":"Local pickup","description":"Allow customers to pick up orders themselves. By default, when using local pickup store base taxes will apply regardless of customer address.","_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping_methods/local_pickup"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping_methods"}]}},"emitted_at":1671056279639} -{"stream":"shipping_zone_locations","data":{"code":"US:CA","type":"state","_links":{"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1/locations"}],"describes":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}]}},"emitted_at":1671056281366} -{"stream":"shipping_zone_locations","data":{"code":"90210","type":"postcode","_links":{"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1/locations"}],"describes":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}]}},"emitted_at":1671056281368} -{"stream":"shipping_zone_methods","data":{"id":1,"instance_id":1,"title":"Local pickup","order":1,"enabled":true,"method_id":"local_pickup","method_title":"Local pickup","method_description":"

    Allow customers to pick up orders themselves. By default, when using local pickup store base taxes will apply regardless of customer address.

    \n","settings":{"title":{"id":"title","label":"Title","description":"This controls the title which the user sees during checkout.","type":"text","value":"Local pickup","default":"Local pickup","tip":"This controls the title which the user sees during checkout.","placeholder":""},"tax_status":{"id":"tax_status","label":"Tax status","description":"","type":"select","value":"taxable","default":"taxable","tip":"","placeholder":"","options":{"taxable":"Taxable","none":"None"}},"cost":{"id":"cost","label":"Cost","description":"Optional cost for local pickup.","type":"text","value":"","default":"","tip":"Optional cost for local pickup.","placeholder":""}},"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1/methods/1"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1/methods"}],"describes":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}]}},"emitted_at":1671056283076} -{"stream":"shipping_zones","data":{"id":0,"name":"Locations not covered by your other zones","order":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/0"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones"}],"describedby":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/0/locations"}]}},"emitted_at":1671056283676} -{"stream":"shipping_zones","data":{"id":1,"name":"California ZIP 90210","order":0,"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones"}],"describedby":[{"href":"https://airbyte.store/wp-json/wc/v3/shipping/zones/1/locations"}]}},"emitted_at":1671056283678} -{"stream":"system_status_tools","data":{"id":"clear_transients","name":"WooCommerce transients","action":"Clear transients","description":"This tool will clear the product/shop transients cache.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_transients"}]}},"emitted_at":1671056284378} -{"stream":"system_status_tools","data":{"id":"clear_expired_transients","name":"Expired transients","action":"Clear transients","description":"This tool will clear ALL expired transients from WordPress.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_expired_transients"}]}},"emitted_at":1671056284379} -{"stream":"system_status_tools","data":{"id":"delete_orphaned_variations","name":"Orphaned variations","action":"Delete orphaned variations","description":"This tool will delete all variations which have no parent.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/delete_orphaned_variations"}]}},"emitted_at":1671056284380} -{"stream":"system_status_tools","data":{"id":"clear_expired_download_permissions","name":"Used-up download permissions","action":"Clean up download permissions","description":"This tool will delete expired download permissions and permissions with 0 remaining downloads.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_expired_download_permissions"}]}},"emitted_at":1671056284381} -{"stream":"system_status_tools","data":{"id":"regenerate_product_lookup_tables","name":"Product lookup tables","action":"Regenerate","description":"This tool will regenerate product lookup table data. This process may take a while.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/regenerate_product_lookup_tables"}]}},"emitted_at":1671056284382} -{"stream":"system_status_tools","data":{"id":"recount_terms","name":"Term counts","action":"Recount terms","description":"This tool will recount product terms - useful when changing your settings in a way which hides products from the catalog.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/recount_terms"}]}},"emitted_at":1671056284383} -{"stream":"system_status_tools","data":{"id":"reset_roles","name":"Capabilities","action":"Reset capabilities","description":"This tool will reset the admin, customer and shop_manager roles to default. Use this if your users cannot access all of the WooCommerce admin pages.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/reset_roles"}]}},"emitted_at":1671056284384} -{"stream":"system_status_tools","data":{"id":"clear_sessions","name":"Clear customer sessions","action":"Clear","description":"Note: This tool will delete all customer session data from the database, including current carts and saved carts in the database.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_sessions"}]}},"emitted_at":1671056284385} -{"stream":"system_status_tools","data":{"id":"clear_template_cache","name":"Clear template cache","action":"Clear","description":"Note: This tool will empty the template cache.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_template_cache"}]}},"emitted_at":1671056284386} -{"stream":"system_status_tools","data":{"id":"install_pages","name":"Create default WooCommerce pages","action":"Create pages","description":"Note: This tool will install all the missing WooCommerce pages. Pages already defined and set up will not be replaced.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/install_pages"}]}},"emitted_at":1671056284387} -{"stream":"system_status_tools","data":{"id":"delete_taxes","name":"Delete WooCommerce tax rates","action":"Delete tax rates","description":"Note: This option will delete ALL of your tax rates, use with caution. This action cannot be reversed.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/delete_taxes"}]}},"emitted_at":1671056284388} -{"stream":"system_status_tools","data":{"id":"db_update_routine","name":"Update database","action":"Update database","description":"Note: This tool will update your WooCommerce database to the latest version. Please ensure you make sufficient backups before proceeding.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/db_update_routine"}]}},"emitted_at":1671056284389} -{"stream":"system_status_tools","data":{"id":"verify_db_tables","name":"Verify base database tables","action":"Verify database","description":"Verify if all base database tables are present.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/verify_db_tables"}]}},"emitted_at":1671056284390} -{"stream":"system_status_tools","data":{"id":"regenerate_product_attributes_lookup_table","name":"Regenerate the product attributes lookup table","action":"Regenerate","description":"This tool will regenerate the product attributes lookup table data from existing product(s) data. This process may take a while.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/regenerate_product_attributes_lookup_table"}]}},"emitted_at":1671056284391} -{"stream":"system_status_tools","data":{"id":"clear_woocommerce_analytics_cache","name":"Clear analytics cache","action":"Clear","description":"This tool will reset the cached values used in WooCommerce Analytics. If numbers still look off, try Reimporting Historical Data.","_links":{"item":[{"embeddable":true,"href":"https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_woocommerce_analytics_cache"}]}},"emitted_at":1671056284393} -{"stream":"tax_classes","data":{"slug":"standard","name":"Standard rate","_links":{"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/classes"}]}},"emitted_at":1671056284988} -{"stream":"tax_classes","data":{"slug":"reduced-rate","name":"Reduced rate","_links":{"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/classes"}]}},"emitted_at":1671056284988} -{"stream":"tax_classes","data":{"slug":"zero-rate","name":"Zero rate","_links":{"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/classes"}]}},"emitted_at":1671056284989} -{"stream":"tax_rates","data":{"id":1,"country":"CA","state":"BC","postcode":"","city":"VANCOUVER","rate":"10.0000","name":"canada_tax","priority":1,"compound":false,"shipping":true,"order":0,"class":"standard","postcodes":[],"cities":["VANCOUVER"],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/1"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes"}]}},"emitted_at":1671056285575} -{"stream":"tax_rates","data":{"id":2,"country":"MX","state":"","postcode":"","city":"","rate":"5.0000","name":"mexico","priority":1,"compound":false,"shipping":true,"order":0,"class":"reduced-rate","postcodes":[],"cities":[],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/2"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes"}]}},"emitted_at":1671056285575} -{"stream":"tax_rates","data":{"id":3,"country":"CU","state":"","postcode":"","city":"","rate":"0.0000","name":"cuba_tax","priority":1,"compound":false,"shipping":true,"order":0,"class":"zero-rate","postcodes":[],"cities":[],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/3"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes"}]}},"emitted_at":1671056285576} -{"stream":"tax_rates","data":{"id":4,"country":"US","state":"CA","postcode":"94133","city":"SAN FRANCISCO","rate":"8.6250","name":"US-CA-SAN FRANCISCO COUNTY-SAN FRANCISCO Tax","priority":1,"compound":false,"shipping":false,"order":0,"class":"standard","postcodes":["94133"],"cities":["SAN FRANCISCO"],"_links":{"self":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes/4"}],"collection":[{"href":"https://airbyte.store/wp-json/wc/v3/taxes"}]}},"emitted_at":1671056285576} +{"stream": "customers", "data": {"id": 210684524, "date_created": "2022-11-30T20:40:42", "date_created_gmt": "2022-11-30T20:40:42", "date_modified": "2022-11-30T20:40:43", "date_modified_gmt": "2022-11-30T20:40:43", "email": "john.doe@example.com", "first_name": "John", "last_name": "Doe", "role": "customer", "username": "john.doe", "billing": {"first_name": "John", "last_name": "Doe", "company": "", "address_1": "969 Market", "address_2": "", "city": "San Francisco", "postcode": "94103", "country": "US", "state": "CA", "email": "john.doe@example.com", "phone": "(555) 555-5555"}, "shipping": {"first_name": "John", "last_name": "Doe", "company": "", "address_1": "969 Market", "address_2": "", "city": "San Francisco", "postcode": "94103", "country": "US", "state": "CA", "phone": ""}, "is_paying_customer": false, "avatar_url": "https://secure.gravatar.com/avatar/8eb1b522f60d11fa897de1dc6351b7e8?s=96&d=identicon&r=g", "meta_data": [{"id": 65, "key": "_wcs_subscription_ids_cache", "value": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/customers/210684524"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/customers"}]}}, "emitted_at": 1691420985506} +{"stream": "coupons", "data": {"id": 44, "code": "test10off", "amount": "10.00", "status": "publish", "date_created": "2022-11-30T20:17:38", "date_created_gmt": "2022-11-30T20:17:38", "date_modified": "2022-11-30T20:17:38", "date_modified_gmt": "2022-11-30T20:17:38", "discount_type": "percent", "description": "10% OFF", "date_expires": null, "date_expires_gmt": null, "usage_count": 1, "individual_use": false, "product_ids": [], "excluded_product_ids": [], "usage_limit": null, "usage_limit_per_user": null, "limit_usage_to_x_items": null, "free_shipping": false, "product_categories": [], "excluded_product_categories": [], "exclude_sale_items": false, "minimum_amount": "0.00", "maximum_amount": "0.00", "email_restrictions": [], "used_by": ["210684523"], "meta_data": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/coupons/44"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/coupons"}]}}, "emitted_at": 1691420986086} +{"stream": "orders", "data": {"id": 45, "parent_id": 0, "status": "pending", "currency": "USD", "version": "7.1.0", "prices_include_tax": false, "date_created": "2022-11-30T20:17:50", "date_modified": "2022-11-30T20:19:01", "discount_total": "0.00", "discount_tax": "0.00", "shipping_total": "0.00", "shipping_tax": "0.00", "cart_tax": "0.00", "total": "90.00", "total_tax": "0.00", "customer_id": 210684523, "order_key": "wc_order_QxkrRLu9BwBQE", "billing": {"first_name": "", "last_name": "", "company": "", "address_1": "", "address_2": "", "city": "", "state": "", "postcode": "", "country": "", "email": "integration-test@airbyte.io", "phone": ""}, "shipping": {"first_name": "", "last_name": "", "company": "", "address_1": "", "address_2": "", "city": "", "state": "", "postcode": "", "country": "", "phone": ""}, "payment_method": "", "payment_method_title": "", "transaction_id": "", "customer_ip_address": "", "customer_user_agent": "", "created_via": "admin", "customer_note": "", "date_completed": null, "date_paid": null, "cart_hash": "", "number": "45", "meta_data": [{"id": 193, "key": "coupon_amount", "value": ""}], "line_items": [{"id": 2, "name": "test_product_1", "product_id": 40, "variation_id": 0, "quantity": 1, "tax_class": "", "subtotal": "90.00", "subtotal_tax": "0.00", "total": "90.00", "total_tax": "0.00", "taxes": [], "meta_data": [], "sku": "test_prod_1", "price": 90, "image": {"id": "", "src": ""}, "parent_name": null}], "tax_lines": [], "shipping_lines": [], "fee_lines": [], "coupon_lines": [{"id": 1, "code": "test10off", "discount": "0", "discount_tax": "0", "meta_data": [{"id": 3, "key": "coupon_data", "value": {"id": 44, "code": "test10off", "amount": "10", "status": "publish", "date_created": {"date": "2022-11-30 20:17:38.000000", "timezone_type": 3, "timezone": "UTC"}, "date_modified": {"date": "2022-11-30 20:17:38.000000", "timezone_type": 3, "timezone": "UTC"}, "date_expires": null, "discount_type": "percent", "description": "10% OFF", "usage_count": 0, "individual_use": false, "product_ids": [], "excluded_product_ids": [], "usage_limit": 0, "usage_limit_per_user": 0, "limit_usage_to_x_items": null, "free_shipping": false, "product_categories": [], "excluded_product_categories": [], "exclude_sale_items": false, "minimum_amount": "", "maximum_amount": "", "email_restrictions": [], "virtual": false, "meta_data": []}, "display_key": "coupon_data", "display_value": {"id": 44, "code": "test10off", "amount": "10", "status": "publish", "date_created": {"date": "2022-11-30 20:17:38.000000", "timezone_type": 3, "timezone": "UTC"}, "date_modified": {"date": "2022-11-30 20:17:38.000000", "timezone_type": 3, "timezone": "UTC"}, "date_expires": null, "discount_type": "percent", "description": "10% OFF", "usage_count": 0, "individual_use": false, "product_ids": [], "excluded_product_ids": [], "usage_limit": 0, "usage_limit_per_user": 0, "limit_usage_to_x_items": null, "free_shipping": false, "product_categories": [], "excluded_product_categories": [], "exclude_sale_items": false, "minimum_amount": "", "maximum_amount": "", "email_restrictions": [], "virtual": false, "meta_data": []}}]}], "refunds": [{"id": 60, "reason": "bonus", "total": "-10.00"}], "payment_url": "https://airbyte.store/checkout/order-pay/45/?pay_for_order=true&key=wc_order_QxkrRLu9BwBQE", "is_editable": true, "needs_payment": true, "needs_processing": true, "date_created_gmt": "2022-11-30T20:17:50", "date_modified_gmt": "2022-11-30T20:19:01", "date_completed_gmt": null, "date_paid_gmt": null, "currency_symbol": "$", "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/orders"}], "customer": [{"href": "https://airbyte.store/wp-json/wc/v3/customers/210684523"}]}}, "emitted_at": 1691420991514} +{"stream": "order_notes", "data": {"id": 3, "author": "airbytetesting", "date_created": "2022-11-30T20:18:29", "date_created_gmt": "2022-11-30T20:18:29", "note": "Added line items: test_product_1 (test_prod_1)", "customer_note": false, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45/notes/3"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45/notes"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45"}]}}, "emitted_at": 1691420997658} +{"stream": "payment_gateways", "data": {"id": "woocommerce_payments", "title": "Credit card / debit card", "description": "Enter your card details", "order": -1, "enabled": false, "method_title": "WooPayments", "method_description": "WooPayments gives your store flexibility to accept credit cards, debit cards, and Apple Pay. Enable popular local payment methods and other digital wallets like Google Pay to give customers even more choice.

    \n\t\t\tBy using WooPayments you agree to be bound by our Terms of Service and acknowledge that you have read our Privacy Policy", "method_supports": ["products", "refunds", "multiple_subscriptions", "subscription_cancellation", "subscription_payment_method_change_admin", "subscription_payment_method_change_customer", "subscription_payment_method_change", "subscription_reactivation", "subscription_suspension", "subscriptions", "gateway_scheduled_payments", "tokenization", "add_payment_method"], "settings": {"account_statement_descriptor": {"id": "account_statement_descriptor", "label": "Customer bank statement", "description": "Edit the way your store name appears on your customers\u2019 bank statements (read more about requirements here).", "type": "account_statement_descriptor", "value": "", "default": "", "tip": "Edit the way your store name appears on your customers\u2019 bank statements (read more about requirements here).", "placeholder": ""}, "manual_capture": {"id": "manual_capture", "label": "Issue an authorization on checkout, and capture later.", "description": "Charge must be captured within 7 days of authorization, otherwise the authorization and order will be canceled.", "type": "checkbox", "value": "no", "default": "no", "tip": "Charge must be captured within 7 days of authorization, otherwise the authorization and order will be canceled.", "placeholder": ""}, "saved_cards": {"id": "saved_cards", "label": "Enable payment via saved cards", "description": "If enabled, users will be able to pay with a saved card during checkout. Card details are saved on our platform, not on your store.", "type": "checkbox", "value": "yes", "default": "yes", "tip": "If enabled, users will be able to pay with a saved card during checkout. Card details are saved on our platform, not on your store.", "placeholder": ""}, "test_mode": {"id": "test_mode", "label": "Enable test mode", "description": "Simulate transactions using test card numbers.", "type": "checkbox", "value": "no", "default": "no", "tip": "Simulate transactions using test card numbers.", "placeholder": ""}, "enable_logging": {"id": "enable_logging", "label": "When enabled debug notes will be added to the log.", "description": "", "type": "checkbox", "value": "no", "default": "no", "tip": "", "placeholder": ""}, "payment_request_details": {"id": "payment_request_details", "label": "Payment request buttons", "description": "", "type": "title", "value": "", "default": "", "tip": "", "placeholder": ""}, "payment_request": {"id": "payment_request", "label": "Enable payment request buttons (Apple Pay, Google Pay, and more).
    By using Apple Pay, you agree to Stripe and Apple's Terms of Service.", "description": "If enabled, users will be able to pay using Apple Pay, Google Pay or the Payment Request API if supported by the browser.", "type": "checkbox", "value": "yes", "default": "no", "tip": "If enabled, users will be able to pay using Apple Pay, Google Pay or the Payment Request API if supported by the browser.", "placeholder": ""}, "payment_request_button_type": {"id": "payment_request_button_type", "label": "Button type", "description": "Select the button type you would like to show.", "type": "select", "value": "buy", "default": "buy", "tip": "Select the button type you would like to show.", "placeholder": "", "options": {"default": "Only icon", "buy": "Buy", "donate": "Donate", "book": "Book"}}, "payment_request_button_theme": {"id": "payment_request_button_theme", "label": "Button theme", "description": "Select the button theme you would like to show.", "type": "select", "value": "dark", "default": "dark", "tip": "Select the button theme you would like to show.", "placeholder": "", "options": {"dark": "Dark", "light": "Light", "light-outline": "Light-Outline"}}, "payment_request_button_height": {"id": "payment_request_button_height", "label": "Button height", "description": "Enter the height you would like the button to be in pixels. Width will always be 100%.", "type": "text", "value": "44", "default": "44", "tip": "Enter the height you would like the button to be in pixels. Width will always be 100%.", "placeholder": ""}, "payment_request_button_label": {"id": "payment_request_button_label", "label": "Custom button label", "description": "Enter the custom text you would like the button to have.", "type": "text", "value": "Buy now", "default": "Buy now", "tip": "Enter the custom text you would like the button to have.", "placeholder": ""}, "payment_request_button_locations": {"id": "payment_request_button_locations", "label": "Button locations", "description": "Select where you would like to display the button.", "type": "multiselect", "value": ["product", "cart", "checkout"], "default": ["product", "cart", "checkout"], "tip": "Select where you would like to display the button.", "placeholder": "", "options": {"product": "Product", "cart": "Cart", "checkout": "Checkout"}}, "upe_enabled_payment_method_ids": {"id": "upe_enabled_payment_method_ids", "label": "Payments accepted on checkout", "description": "", "type": "multiselect", "value": ["card"], "default": ["card"], "tip": "", "placeholder": ""}, "payment_request_button_size": {"id": "payment_request_button_size", "label": "Size of the button displayed for Express Checkouts", "description": "Select the size of the button.", "type": "select", "value": "default", "default": "default", "tip": "Select the size of the button.", "placeholder": "", "options": {"default": "Default", "medium": "Medium", "large": "Large"}}}, "needs_setup": true, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=woocommerce_payments", "connection_url": "", "setup_help_text": "Next we\u2019ll ask you to share a few details about your business to create your account.", "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/woocommerce_payments"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1691421003301} +{"stream": "payment_gateways", "data": {"id": "bacs", "title": "Direct bank transfer", "description": "Make your payment directly into our bank account. Please use your Order ID as the payment reference. Your order will not be shipped until the funds have cleared in our account.", "order": "", "enabled": false, "method_title": "Direct bank transfer", "method_description": "Take payments in person via BACS. More commonly known as direct bank/wire transfer.", "method_supports": ["products"], "settings": {"title": {"id": "title", "label": "Title", "description": "This controls the title which the user sees during checkout.", "type": "safe_text", "value": "Direct bank transfer", "default": "Direct bank transfer", "tip": "This controls the title which the user sees during checkout.", "placeholder": ""}, "instructions": {"id": "instructions", "label": "Instructions", "description": "Instructions that will be added to the thank you page and emails.", "type": "textarea", "value": "", "default": "", "tip": "Instructions that will be added to the thank you page and emails.", "placeholder": ""}, "accounts": {"id": "accounts", "value": []}}, "needs_setup": false, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=bacs", "connection_url": null, "setup_help_text": null, "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/bacs"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1691421003305} +{"stream": "payment_gateways", "data": {"id": "cheque", "title": "Check payments", "description": "Pay for this order by check.", "order": "", "enabled": false, "method_title": "Check payments", "method_description": "Take payments in person via checks. This offline gateway can also be useful to test purchases.", "method_supports": ["products"], "settings": {"title": {"id": "title", "label": "Title", "description": "This controls the title which the user sees during checkout.", "type": "safe_text", "value": "Check payments", "default": "Check payments", "tip": "This controls the title which the user sees during checkout.", "placeholder": ""}, "instructions": {"id": "instructions", "label": "Instructions", "description": "Instructions that will be added to the thank you page and emails.", "type": "textarea", "value": "Make your check payable to...", "default": "", "tip": "Instructions that will be added to the thank you page and emails.", "placeholder": ""}}, "needs_setup": false, "post_install_scripts": [], "settings_url": "https://airbyte.store/wp-admin/admin.php?page=wc-settings&tab=checkout§ion=cheque", "connection_url": null, "setup_help_text": null, "required_settings_keys": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways/cheque"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/payment_gateways"}]}}, "emitted_at": 1691421003307} +{"stream": "products", "data": {"id": 26, "name": "Connector source Facebook Marketing", "slug": "album", "permalink": "https://airbyte.store/product/album/", "date_created": "2022-11-23T08:21:10", "date_created_gmt": "2022-11-23T08:21:10", "date_modified": "2022-11-23T08:27:43", "date_modified_gmt": "2022-11-23T08:27:43", "type": "simple", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "

    Album reviewed and chosen by Woo.

    \n", "short_description": "", "sku": "", "price": "0.01", "regular_price": "0.02", "sale_price": "0.01", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": true, "purchasable": true, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": false, "stock_quantity": null, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "", "dimensions": {"length": "", "width": "", "height": ""}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "", "shipping_class_id": 0, "reviews_allowed": true, "average_rating": "0.00", "rating_count": 0, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "", "categories": [], "tags": [], "images": [{"id": 27, "date_created": "2022-11-23T08:26:11", "date_created_gmt": "2022-11-23T08:26:11", "date_modified": "2022-11-23T08:26:33", "date_modified_gmt": "2022-11-23T08:26:33", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1312%2C542&ssl=1", "name": "Facebook Marketing source", "alt": ""}], "attributes": [], "default_attributes": [], "variations": [], "grouped_products": [], "menu_order": 0, "price_html": "$0.02 $0.01", "related_ids": [], "meta_data": [{"id": 62, "key": "_created_via", "value": "unknown"}, {"id": 63, "key": "_wpas_done_all", "value": "1"}, {"id": 80, "key": "_last_editor_used_jetpack", "value": "classic-editor"}], "stock_status": "instock", "has_options": false, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 28, "url": "https://airbyte.store/product/connector-source-google-ads/", "url_meta": {"origin": 26, "position": 0}, "title": "Connector source Google Ads", "author": "airbytetesting", "date": "November 23, 2022", "format": false, "excerpt": "", "rel": "", "context": "Similar post", "block_context": {"text": "Similar post", "link": ""}, "img": {"alt_text": "", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1200%2C483&ssl=1&resize=350%2C200", "width": 350, "height": 200, "srcset": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1200%2C483&ssl=1&resize=350%2C200 1x, https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1200%2C483&ssl=1&resize=525%2C300 1.5x, https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1200%2C483&ssl=1&resize=700%2C400 2x, https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1200%2C483&ssl=1&resize=1050%2C600 3x"}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/26"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1691421004948} +{"stream": "products", "data": {"id": 28, "name": "Connector source Google Ads", "slug": "connector-source-google-ads", "permalink": "https://airbyte.store/product/connector-source-google-ads/", "date_created": "2022-11-23T08:30:31", "date_created_gmt": "2022-11-23T08:30:31", "date_modified": "2022-11-23T08:30:31", "date_modified_gmt": "2022-11-23T08:30:31", "type": "simple", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "", "short_description": "", "sku": "", "price": "0.01", "regular_price": "0.02", "sale_price": "0.01", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": true, "purchasable": true, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": false, "stock_quantity": null, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "", "dimensions": {"length": "", "width": "", "height": ""}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "", "shipping_class_id": 0, "reviews_allowed": true, "average_rating": "0.00", "rating_count": 0, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "", "categories": [], "tags": [], "images": [{"id": 29, "date_created": "2022-11-23T08:29:16", "date_created_gmt": "2022-11-23T08:29:16", "date_modified": "2022-11-23T08:29:35", "date_modified_gmt": "2022-11-23T08:29:35", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/GA.png?fit=1317%2C530&ssl=1", "name": "Google Ads source", "alt": ""}], "attributes": [], "default_attributes": [], "variations": [], "grouped_products": [], "menu_order": 0, "price_html": "$0.02 $0.01", "related_ids": [], "meta_data": [{"id": 89, "key": "_created_via", "value": "post-new"}, {"id": 90, "key": "_last_editor_used_jetpack", "value": "classic-editor"}, {"id": 115, "key": "_wpas_done_all", "value": "1"}], "stock_status": "instock", "has_options": false, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 26, "url": "https://airbyte.store/product/album/", "url_meta": {"origin": 28, "position": 0}, "title": "Connector source Facebook Marketing", "author": "airbytetesting", "date": "November 23, 2022", "format": false, "excerpt": "Album reviewed and chosen by Woo.", "rel": "", "context": "Similar post", "block_context": {"text": "Similar post", "link": ""}, "img": {"alt_text": "", "src": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1200%2C496&ssl=1&resize=350%2C200", "width": 350, "height": 200, "srcset": "https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1200%2C496&ssl=1&resize=350%2C200 1x, https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1200%2C496&ssl=1&resize=525%2C300 1.5x, https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1200%2C496&ssl=1&resize=700%2C400 2x, https://i0.wp.com/airbyte.store/wp-content/uploads/2022/11/FB.png?fit=1200%2C496&ssl=1&resize=1050%2C600 3x"}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/28"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1691421004957} +{"stream": "products", "data": {"id": 40, "name": "test_product_1", "slug": "test_product_1", "permalink": "https://airbyte.store/product/test_product_1/", "date_created": "2022-11-30T19:59:56", "date_created_gmt": "2022-11-30T19:59:56", "date_modified": "2022-11-30T20:44:57", "date_modified_gmt": "2022-11-30T20:44:57", "type": "simple", "status": "publish", "featured": false, "catalog_visibility": "visible", "description": "

    test_product description

    \n", "short_description": "", "sku": "test_prod_1", "price": "90", "regular_price": "100", "sale_price": "90", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": true, "purchasable": true, "total_sales": 0, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "external_url": "", "button_text": "", "tax_status": "taxable", "tax_class": "", "manage_stock": true, "stock_quantity": 33, "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "sold_individually": false, "weight": "10", "dimensions": {"length": "10", "width": "10", "height": "10"}, "shipping_required": true, "shipping_taxable": true, "shipping_class": "test_ship_class", "shipping_class_id": 1388, "reviews_allowed": true, "average_rating": "4.50", "rating_count": 2, "upsell_ids": [], "cross_sell_ids": [], "parent_id": 0, "purchase_note": "

    vendor_info

    \n", "categories": [{"id": 1378, "name": "sale_products", "slug": "sale_products"}], "tags": [{"id": 1380, "name": "saleable", "slug": "saleable"}, {"id": 1379, "name": "sample_tag", "slug": "sample_tag"}], "images": [], "attributes": [{"id": 0, "name": "color", "position": 0, "visible": true, "variation": false, "options": ["white", "red", "black"]}], "default_attributes": [], "variations": [], "grouped_products": [], "menu_order": 0, "price_html": "$100.00 $90.00", "related_ids": [46], "meta_data": [{"id": 130, "key": "_created_via", "value": "post-new"}, {"id": 131, "key": "_last_editor_used_jetpack", "value": "classic-editor"}, {"id": 160, "key": "_wpas_done_all", "value": "1"}], "stock_status": "instock", "has_options": false, "jetpack_publicize_connections": [], "jetpack_likes_enabled": true, "jetpack_sharing_enabled": true, "jetpack-related-posts": [{"id": 46, "url": "https://airbyte.store/product/test_product_2/", "url_meta": {"origin": 40, "position": 0}, "title": "test_product_2", "author": "airbytetesting", "date": "November 30, 2022", "format": false, "excerpt": "test_product\u00a0 2 description", "rel": "", "context": "Similar post", "block_context": {"text": "Similar post", "link": ""}, "img": {"alt_text": "", "src": "", "width": 0, "height": 0}, "classes": []}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/40"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products"}]}}, "emitted_at": 1691421004966} +{"stream": "product_attributes", "data": {"id": 1, "name": "size", "slug": "pa_size", "type": "select", "order_by": "menu_order", "has_archives": false, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes"}]}}, "emitted_at": 1691421010528} +{"stream": "product_attribute_terms", "data": {"id": 1385, "name": "7", "slug": "7", "description": "size 7", "menu_order": 0, "count": 1, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms/1385"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms"}]}}, "emitted_at": 1691421011709} +{"stream": "product_attribute_terms", "data": {"id": 1386, "name": "8", "slug": "8", "description": "size 8", "menu_order": 0, "count": 1, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms/1386"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms"}]}}, "emitted_at": 1691421011711} +{"stream": "product_attribute_terms", "data": {"id": 1387, "name": "6", "slug": "6", "description": "size 6", "menu_order": 0, "count": 1, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms/1387"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/attributes/1/terms"}]}}, "emitted_at": 1691421011713} +{"stream": "product_categories", "data": {"id": 1374, "name": "Uncategorized", "slug": "uncategorized", "parent": 0, "description": "", "display": "default", "image": null, "menu_order": 0, "count": 0, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/categories/1374"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/categories"}]}}, "emitted_at": 1691421012349} +{"stream": "product_categories", "data": {"id": 1378, "name": "sale_products", "slug": "sale_products", "parent": 0, "description": "", "display": "default", "image": null, "menu_order": 0, "count": 2, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/categories/1378"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/categories"}]}}, "emitted_at": 1691421012353} +{"stream": "product_categories", "data": {"id": 1381, "name": "purchasable_products", "slug": "purchasable_products", "parent": 0, "description": "Purchasable", "display": "products", "image": null, "menu_order": 0, "count": 0, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/categories/1381"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/categories"}]}}, "emitted_at": 1691421012356} +{"stream": "product_reviews", "data": {"id": 1, "date_created": "2022-11-30T20:05:50", "date_created_gmt": "2022-11-30T20:05:50", "product_id": 40, "product_name": "test_product_1", "product_permalink": "https://airbyte.store/product/test_product_1/", "status": "approved", "reviewer": "airbytetesting", "reviewer_email": "integration-test@airbyte.io", "review": "

    test_review

    \n", "rating": 5, "verified": false, "reviewer_avatar_urls": {"24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=24&d=identicon&r=g", "48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=48&d=identicon&r=g", "96": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=96&d=identicon&r=g"}, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/reviews/1"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/reviews"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/products/40"}], "reviewer": [{"embeddable": true, "href": "https://airbyte.store/wp-json/wp/v2/users/210684523"}]}}, "emitted_at": 1691421012989} +{"stream": "product_reviews", "data": {"id": 2, "date_created": "2022-11-30T20:06:01", "date_created_gmt": "2022-11-30T20:06:01", "product_id": 40, "product_name": "test_product_1", "product_permalink": "https://airbyte.store/product/test_product_1/", "status": "approved", "reviewer": "airbytetesting", "reviewer_email": "integration-test@airbyte.io", "review": "

    test_review 2

    \n", "rating": 4, "verified": false, "reviewer_avatar_urls": {"24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=24&d=identicon&r=g", "48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=48&d=identicon&r=g", "96": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=96&d=identicon&r=g"}, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/reviews/2"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/reviews"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/products/40"}], "reviewer": [{"embeddable": true, "href": "https://airbyte.store/wp-json/wp/v2/users/210684523"}]}}, "emitted_at": 1691421012992} +{"stream": "product_shipping_classes", "data": {"id": 1388, "name": "test_ship_class", "slug": "test_ship_class", "description": "", "count": 2, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/shipping_classes/1388"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/shipping_classes"}]}}, "emitted_at": 1691421018578} +{"stream": "product_tags", "data": {"id": 1379, "name": "sample_tag", "slug": "sample_tag", "description": "", "count": 2, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/tags/1379"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/tags"}]}}, "emitted_at": 1691421019168} +{"stream": "product_tags", "data": {"id": 1380, "name": "saleable", "slug": "saleable", "description": "", "count": 2, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/tags/1380"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/tags"}]}}, "emitted_at": 1691421019171} +{"stream": "product_tags", "data": {"id": 1382, "name": "clothes", "slug": "clothes", "description": "Clothes", "count": 0, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/tags/1382"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/tags"}]}}, "emitted_at": 1691421019172} +{"stream": "product_variations", "data": {"id": 51, "date_created": "2022-11-30T20:56:17", "date_created_gmt": "2022-11-30T20:56:17", "date_modified": "2022-11-30T20:56:17", "date_modified_gmt": "2022-11-30T20:56:17", "description": "", "permalink": "https://airbyte.store/product/test_product_2/?attribute_color=white&attribute_pa_size=6", "sku": "test_prod_1-1", "price": "", "regular_price": "", "sale_price": "", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": false, "status": "publish", "purchasable": false, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "tax_status": "taxable", "tax_class": "", "manage_stock": "parent", "stock_quantity": 33, "stock_status": "instock", "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "weight": "10", "dimensions": {"length": "10", "width": "10", "height": "10"}, "shipping_class": "test_ship_class", "shipping_class_id": 1388, "image": null, "attributes": [{"id": 0, "name": "color", "option": "white"}, {"id": 1, "name": "size", "option": "6"}], "menu_order": 1, "meta_data": [{"id": 314, "key": "_created_via", "value": "ajax-unknown"}, {"id": 333, "key": "_subscription_period", "value": "month"}, {"id": 334, "key": "_subscription_period_interval", "value": "1"}, {"id": 335, "key": "_subscription_length", "value": "0"}, {"id": 336, "key": "_subscription_trial_period", "value": "month"}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46/variations/51"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46/variations"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46"}]}}, "emitted_at": 1691421021897} +{"stream": "product_variations", "data": {"id": 52, "date_created": "2022-11-30T20:56:17", "date_created_gmt": "2022-11-30T20:56:17", "date_modified": "2022-11-30T20:56:17", "date_modified_gmt": "2022-11-30T20:56:17", "description": "", "permalink": "https://airbyte.store/product/test_product_2/?attribute_color=white&attribute_pa_size=7", "sku": "test_prod_1-1", "price": "", "regular_price": "", "sale_price": "", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": false, "status": "publish", "purchasable": false, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "tax_status": "taxable", "tax_class": "", "manage_stock": "parent", "stock_quantity": 33, "stock_status": "instock", "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "weight": "10", "dimensions": {"length": "10", "width": "10", "height": "10"}, "shipping_class": "test_ship_class", "shipping_class_id": 1388, "image": null, "attributes": [{"id": 0, "name": "color", "option": "white"}, {"id": 1, "name": "size", "option": "7"}], "menu_order": 2, "meta_data": [{"id": 337, "key": "_created_via", "value": "ajax-unknown"}, {"id": 356, "key": "_subscription_period", "value": "month"}, {"id": 357, "key": "_subscription_period_interval", "value": "1"}, {"id": 358, "key": "_subscription_length", "value": "0"}, {"id": 359, "key": "_subscription_trial_period", "value": "month"}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46/variations/52"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46/variations"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46"}]}}, "emitted_at": 1691421021903} +{"stream": "product_variations", "data": {"id": 53, "date_created": "2022-11-30T20:56:17", "date_created_gmt": "2022-11-30T20:56:17", "date_modified": "2022-11-30T20:56:17", "date_modified_gmt": "2022-11-30T20:56:17", "description": "", "permalink": "https://airbyte.store/product/test_product_2/?attribute_color=white&attribute_pa_size=8", "sku": "test_prod_1-1", "price": "", "regular_price": "", "sale_price": "", "date_on_sale_from": null, "date_on_sale_from_gmt": null, "date_on_sale_to": null, "date_on_sale_to_gmt": null, "on_sale": false, "status": "publish", "purchasable": false, "virtual": false, "downloadable": false, "downloads": [], "download_limit": -1, "download_expiry": -1, "tax_status": "taxable", "tax_class": "", "manage_stock": "parent", "stock_quantity": 33, "stock_status": "instock", "backorders": "no", "backorders_allowed": false, "backordered": false, "low_stock_amount": null, "weight": "10", "dimensions": {"length": "10", "width": "10", "height": "10"}, "shipping_class": "test_ship_class", "shipping_class_id": 1388, "image": null, "attributes": [{"id": 0, "name": "color", "option": "white"}, {"id": 1, "name": "size", "option": "8"}], "menu_order": 3, "meta_data": [{"id": 360, "key": "_created_via", "value": "ajax-unknown"}, {"id": 379, "key": "_subscription_period", "value": "month"}, {"id": 380, "key": "_subscription_period_interval", "value": "1"}, {"id": 381, "key": "_subscription_length", "value": "0"}, {"id": 382, "key": "_subscription_trial_period", "value": "month"}], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46/variations/53"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46/variations"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/products/46"}]}}, "emitted_at": 1691421021908} +{"stream": "refunds", "data": {"id": 60, "date_created": "2022-11-30T21:07:07", "date_created_gmt": "2022-11-30T21:07:07", "amount": "10.00", "reason": "bonus", "refunded_by": 210684523, "refunded_payment": false, "meta_data": [], "line_items": [{"id": 3, "name": "test_product_1", "product_id": 40, "variation_id": 0, "quantity": 0, "tax_class": "", "subtotal": "-10.00", "subtotal_tax": "0.00", "total": "-10.00", "total_tax": "0.00", "taxes": [], "meta_data": [{"id": 22, "key": "_refunded_item_id", "value": "2", "display_key": "_refunded_item_id", "display_value": "2"}], "sku": "test_prod_1", "price": 0, "image": {"id": "", "src": ""}, "parent_name": null}], "shipping_lines": [], "tax_lines": [], "fee_lines": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45/refunds/60"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45/refunds"}], "up": [{"href": "https://airbyte.store/wp-json/wc/v3/orders/45"}]}}, "emitted_at": 1691421027570} +{"stream": "shipping_methods", "data": {"id": "flat_rate", "title": "Flat rate", "description": "Lets you charge a fixed rate for shipping.", "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping_methods/flat_rate"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping_methods"}]}}, "emitted_at": 1691421032876} +{"stream": "shipping_methods", "data": {"id": "free_shipping", "title": "Free shipping", "description": "Free shipping is a special method which can be triggered with coupons and minimum spends.", "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping_methods/free_shipping"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping_methods"}]}}, "emitted_at": 1691421032878} +{"stream": "shipping_methods", "data": {"id": "local_pickup", "title": "Local pickup", "description": "Allow customers to pick up orders themselves. By default, when using local pickup store base taxes will apply regardless of customer address.", "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping_methods/local_pickup"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping_methods"}]}}, "emitted_at": 1691421032880} +{"stream": "shipping_zone_locations", "data": {"code": "US:CA", "type": "state", "_links": {"collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1/locations"}], "describes": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}]}}, "emitted_at": 1691421034719} +{"stream": "shipping_zone_locations", "data": {"code": "90210", "type": "postcode", "_links": {"collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1/locations"}], "describes": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}]}}, "emitted_at": 1691421034723} +{"stream": "shipping_zone_methods", "data": {"id": 1, "instance_id": 1, "title": "Local pickup", "order": 1, "enabled": true, "method_id": "local_pickup", "method_title": "Local pickup", "method_description": "

    Allow customers to pick up orders themselves. By default, when using local pickup store base taxes will apply regardless of customer address.

    \n", "settings": {"title": {"id": "title", "label": "Title", "description": "This controls the title which the user sees during checkout.", "type": "text", "value": "Local pickup", "default": "Local pickup", "tip": "This controls the title which the user sees during checkout.", "placeholder": ""}, "tax_status": {"id": "tax_status", "label": "Tax status", "description": "", "type": "select", "value": "taxable", "default": "taxable", "tip": "", "placeholder": "", "options": {"taxable": "Taxable", "none": "None"}}, "cost": {"id": "cost", "label": "Cost", "description": "Optional cost for local pickup.", "type": "text", "value": "", "default": "", "tip": "Optional cost for local pickup.", "placeholder": ""}}, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1/methods/1"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1/methods"}], "describes": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}]}}, "emitted_at": 1691421036680} +{"stream": "shipping_zones", "data": {"id": 0, "name": "Locations not covered by your other zones", "order": 0, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/0"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones"}], "describedby": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/0/locations"}]}}, "emitted_at": 1691421037296} +{"stream": "shipping_zones", "data": {"id": 1, "name": "California ZIP 90210", "order": 0, "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones"}], "describedby": [{"href": "https://airbyte.store/wp-json/wc/v3/shipping/zones/1/locations"}]}}, "emitted_at": 1691421037299} +{"stream": "system_status_tools", "data": {"id": "clear_transients", "name": "WooCommerce transients", "action": "Clear transients", "description": "This tool will clear the product/shop transients cache.", "_links": {"item": [{"embeddable": true, "href": "https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_transients"}]}}, "emitted_at": 1691421037883} +{"stream": "system_status_tools", "data": {"id": "clear_expired_transients", "name": "Expired transients", "action": "Clear transients", "description": "This tool will clear ALL expired transients from WordPress.", "_links": {"item": [{"embeddable": true, "href": "https://airbyte.store/wp-json/wc/v3/system_status/tools/clear_expired_transients"}]}}, "emitted_at": 1691421037886} +{"stream": "system_status_tools", "data": {"id": "delete_orphaned_variations", "name": "Orphaned variations", "action": "Delete orphaned variations", "description": "This tool will delete all variations which have no parent.", "_links": {"item": [{"embeddable": true, "href": "https://airbyte.store/wp-json/wc/v3/system_status/tools/delete_orphaned_variations"}]}}, "emitted_at": 1691421037888} +{"stream": "tax_classes", "data": {"slug": "standard", "name": "Standard rate", "_links": {"collection": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes/classes"}]}}, "emitted_at": 1691421038625} +{"stream": "tax_classes", "data": {"slug": "reduced-rate", "name": "Reduced rate", "_links": {"collection": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes/classes"}]}}, "emitted_at": 1691421038628} +{"stream": "tax_classes", "data": {"slug": "zero-rate", "name": "Zero rate", "_links": {"collection": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes/classes"}]}}, "emitted_at": 1691421038630} +{"stream": "tax_rates", "data": {"id": 1, "country": "CA", "state": "BC", "postcode": "", "city": "VANCOUVER", "rate": "10.0000", "name": "canada_tax", "priority": 1, "compound": false, "shipping": true, "order": 0, "class": "standard", "postcodes": [], "cities": ["VANCOUVER"], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes/1"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes"}]}}, "emitted_at": 1691421039264} +{"stream": "tax_rates", "data": {"id": 2, "country": "MX", "state": "", "postcode": "", "city": "", "rate": "5.0000", "name": "mexico", "priority": 1, "compound": false, "shipping": true, "order": 0, "class": "reduced-rate", "postcodes": [], "cities": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes/2"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes"}]}}, "emitted_at": 1691421039268} +{"stream": "tax_rates", "data": {"id": 3, "country": "CU", "state": "", "postcode": "", "city": "", "rate": "0.0000", "name": "cuba_tax", "priority": 1, "compound": false, "shipping": true, "order": 0, "class": "zero-rate", "postcodes": [], "cities": [], "_links": {"self": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes/3"}], "collection": [{"href": "https://airbyte.store/wp-json/wc/v3/taxes"}]}}, "emitted_at": 1691421039270} diff --git a/airbyte-integrations/connectors/source-woocommerce/metadata.yaml b/airbyte-integrations/connectors/source-woocommerce/metadata.yaml index 684491333c46..1415b8c68efc 100644 --- a/airbyte-integrations/connectors/source-woocommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-woocommerce/metadata.yaml @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-woocommerce/requirements.txt b/airbyte-integrations/connectors/source-woocommerce/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-woocommerce/requirements.txt +++ b/airbyte-integrations/connectors/source-woocommerce/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-woocommerce/setup.py b/airbyte-integrations/connectors/source-woocommerce/setup.py index 284c9d10f0d0..40945a630f15 100644 --- a/airbyte-integrations/connectors/source-woocommerce/setup.py +++ b/airbyte-integrations/connectors/source-woocommerce/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/schemas/products.json b/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/schemas/products.json index 880595bdb4c1..0e96601c0bf1 100644 --- a/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/schemas/products.json +++ b/airbyte-integrations/connectors/source-woocommerce/source_woocommerce/schemas/products.json @@ -228,7 +228,7 @@ } }, "title": { - "type": ["null", "string"] + "type": ["null", "string"] }, "author": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-workable/metadata.yaml b/airbyte-integrations/connectors/source-workable/metadata.yaml index 6869323430c5..6862708b8a23 100644 --- a/airbyte-integrations/connectors/source-workable/metadata.yaml +++ b/airbyte-integrations/connectors/source-workable/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-workable/requirements.txt b/airbyte-integrations/connectors/source-workable/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-workable/requirements.txt +++ b/airbyte-integrations/connectors/source-workable/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-workable/setup.py b/airbyte-integrations/connectors/source-workable/setup.py index 9dcceb71ecc4..67f123878709 100644 --- a/airbyte-integrations/connectors/source-workable/setup.py +++ b/airbyte-integrations/connectors/source-workable/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-workramp/metadata.yaml b/airbyte-integrations/connectors/source-workramp/metadata.yaml index b366c4aa43fc..0fff3108cfc5 100644 --- a/airbyte-integrations/connectors/source-workramp/metadata.yaml +++ b/airbyte-integrations/connectors/source-workramp/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-workramp/requirements.txt b/airbyte-integrations/connectors/source-workramp/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-workramp/requirements.txt +++ b/airbyte-integrations/connectors/source-workramp/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-workramp/setup.py b/airbyte-integrations/connectors/source-workramp/setup.py index 97ec3beccd74..a2b4fc952a19 100644 --- a/airbyte-integrations/connectors/source-workramp/setup.py +++ b/airbyte-integrations/connectors/source-workramp/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-wrike/metadata.yaml b/airbyte-integrations/connectors/source-wrike/metadata.yaml index c3046bae1fa2..eae619a4402a 100644 --- a/airbyte-integrations/connectors/source-wrike/metadata.yaml +++ b/airbyte-integrations/connectors/source-wrike/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/wrike tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-wrike/requirements.txt b/airbyte-integrations/connectors/source-wrike/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-wrike/requirements.txt +++ b/airbyte-integrations/connectors/source-wrike/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-wrike/setup.py b/airbyte-integrations/connectors/source-wrike/setup.py index 2d8f1d7e8bad..d41b776a27b3 100644 --- a/airbyte-integrations/connectors/source-wrike/setup.py +++ b/airbyte-integrations/connectors/source-wrike/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-xero/Dockerfile b/airbyte-integrations/connectors/source-xero/Dockerfile index 81b3ed59f61e..a97eba3cca01 100644 --- a/airbyte-integrations/connectors/source-xero/Dockerfile +++ b/airbyte-integrations/connectors/source-xero/Dockerfile @@ -34,5 +34,5 @@ COPY source_xero ./source_xero ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/source-xero diff --git a/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl index 1b50c1d4f18b..08f96ab1c914 100644 --- a/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl @@ -1,36 +1,31 @@ -{"stream": "bank_transactions", "data": {"BankTransactionID": "4848c602-aeba-4e01-a533-8eae3e090633", "BankAccount": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account"}, "Type": "SPEND", "Reference": "", "IsReconciled": false, "HasAttachments": false, "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "Status": "AUTHORISED", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 3.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 2.0, "LineItemID": "340a70b3-d4ff-4bad-85c8-d88331fc4b20", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 3.0, "TotalTax": 0.0, "Total": 3.0, "UpdatedDateUTC": "2021-08-31T12:25:54+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106848643} -{"stream": "bank_transactions", "data": {"BankTransactionID": "550c811d-66d3-4b72-9334-4555d22c85b5", "BankAccount": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account"}, "Type": "SPEND", "Reference": "PHTR", "IsReconciled": false, "HasAttachments": false, "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "Status": "AUTHORISED", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 10.0, "TaxType": "GSTONIMPORTS", "TaxAmount": 0.0, "LineAmount": 10.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "6ffc731f-b203-4078-ae50-893f5e24d808", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 10.0, "TotalTax": 0.0, "Total": 10.0, "UpdatedDateUTC": "2021-08-31T12:31:27+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106848644} -{"stream": "bank_transactions", "data": {"BankTransactionID": "9a704749-8084-4eed-9554-4edccaa1b6ce", "BankAccount": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account"}, "Type": "SPEND", "Reference": "YTR", "IsReconciled": false, "HasAttachments": false, "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "Status": "AUTHORISED", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 3.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 2.0, "LineItemID": "67f578d5-522d-4c0a-9e54-c68df01c9093", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}, {"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 12.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 2.0, "LineItemID": "5f40636c-c56b-48d7-bf2d-dc47cd3c24d7", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 15.0, "TotalTax": 0.0, "Total": 15.0, "UpdatedDateUTC": "2021-08-31T12:32:29+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106848645} -{"stream": "contacts", "data": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "ContactStatus": "ACTIVE", "Name": "Petra Shop", "FirstName": "Mickle", "LastName": "Born", "EmailAddress": "mickleborn@yahoo.com", "SkypeUserName": "skype.mickleborn@petrashop.com", "BankAccountDetails": "1112223334", "TaxNumber": "987654321", "AccountsReceivableTaxType": "GSTONIMPORTS", "AccountsPayableTaxType": "OUTPUT", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}, {"AddressType": "POBOX", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}], "Phones": [{"PhoneType": "DDI", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}, {"PhoneType": "DEFAULT", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}, {"PhoneType": "MOBILE", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:29:18+00:00", "ContactGroups": [{"ContactGroupID": "051f5017-8e97-4c8d-a5d6-1f3e83c79d14", "Name": "California", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}], "IsSupplier": true, "IsCustomer": false, "DefaultCurrency": "USD", "Discount": 10.0, "Website": "http://petrashop.com", "PurchasesDefaultAccountCode": "6040", "SalesDefaultLineAmountType": "INCLUSIVE", "PurchasesDefaultLineAmountType": "NONE", "SalesDefaultAccountCode": "4000", "BatchPayments": {"BankAccountNumber": "1112223334", "BankAccountName": "Citibank", "Details": "", "Code": "", "Reference": ""}, "PaymentTerms": {"Bills": {"Day": 0, "Type": "DAYSAFTERBILLDATE"}, "Sales": {"Day": 0, "Type": "DAYSAFTERBILLMONTH"}}, "ContactPersons": [], "HasAttachments": false, "HasValidationErrors": false}, "emitted_at": 1683106849228} -{"stream": "contacts", "data": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "ContactStatus": "ACTIVE", "Name": "Milly", "FirstName": "Ashly", "LastName": "Simon", "EmailAddress": "asimon@gmail.com", "SkypeUserName": "skype.asimon@milly.com", "BankAccountDetails": "123456789", "TaxNumber": "123456789", "AccountsReceivableTaxType": "GSTONIMPORTS", "AccountsPayableTaxType": "GSTONIMPORTS", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "United States", "AttentionTo": ""}], "Phones": [{"PhoneType": "DDI", "PhoneNumber": "6179804", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "DEFAULT", "PhoneNumber": "6179800", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "6179802", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "6179803", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:30:20+00:00", "ContactGroups": [{"ContactGroupID": "7ba28f5a-62c4-4555-bb2f-36e005ee1a2b", "Name": "Georgia", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}], "IsSupplier": false, "IsCustomer": true, "DefaultCurrency": "USD", "Discount": 5.0, "Website": "http://milly.com/us", "PurchasesDefaultAccountCode": "6000", "SalesDefaultLineAmountType": "INCLUSIVE", "PurchasesDefaultLineAmountType": "INCLUSIVE", "SalesDefaultAccountCode": "4300", "BatchPayments": {"BankAccountNumber": "123456789", "BankAccountName": "Bank of America", "Details": "123", "Code": "", "Reference": ""}, "ContactPersons": [], "HasAttachments": false, "HasValidationErrors": false}, "emitted_at": 1683106849229} -{"stream": "contacts", "data": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "ContactStatus": "ACTIVE", "Name": "Paragorn", "FirstName": "Robert", "LastName": "McNamara", "EmailAddress": "robert.paragorn@gmail.com", "BankAccountDetails": "444111111", "TaxNumber": "123456780", "AccountsReceivableTaxType": "OUTPUT", "AccountsPayableTaxType": "INPUT", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1200 S Main St", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Walnut Creek", "Region": "CA", "PostalCode": "94596", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "1200 S Main St", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Walnut Creek", "Region": "CA", "PostalCode": "94596", "Country": "USA"}], "Phones": [{"PhoneType": "DDI", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}, {"PhoneType": "DEFAULT", "PhoneNumber": "9490009", "PhoneAreaCode": "925", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "9490009", "PhoneAreaCode": "925", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}], "UpdatedDateUTC": "2021-08-31T12:25:53+00:00", "ContactGroups": [{"ContactGroupID": "051f5017-8e97-4c8d-a5d6-1f3e83c79d14", "Name": "California", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}], "IsSupplier": true, "IsCustomer": true, "DefaultCurrency": "USD", "Discount": 0.0, "Website": "http://paragorntrading.com", "PurchasesDefaultAccountCode": "4715", "SalesDefaultAccountCode": "4300", "BatchPayments": {"BankAccountNumber": "444111111", "BankAccountName": "American Express", "Details": "", "Code": "", "Reference": ""}, "Balances": {"AccountsReceivable": {"Outstanding": -12.0, "Overdue": 0.0}, "AccountsPayable": {"Outstanding": 0.0, "Overdue": 0.0}}, "ContactPersons": [], "HasAttachments": false, "HasValidationErrors": false}, "emitted_at": 1683106849230} -{"stream": "credit_notes", "data": {"CreditNoteID": "c38b210c-699c-43c6-a91d-22792093f85a", "CreditNoteNumber": "CN-0003", "Payments": [], "ID": "c38b210c-699c-43c6-a91d-22792093f85a", "HasErrors": false, "CurrencyRate": 1.0, "Type": "ACCRECCREDIT", "Reference": "QSDF", "RemainingCredit": 12.0, "Allocations": [], "HasAttachments": false, "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "AUTHORISED", "LineAmountTypes": "Exclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "OUTPUT", "TaxAmount": 0.0, "LineAmount": 12.0, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 1.0, "LineItemID": "b581ad7d-50fa-4e47-a9e9-9594a56ac27f"}], "SubTotal": 12.0, "TotalTax": 0.0, "Total": 12.0, "UpdatedDateUTC": "2021-08-31T12:34:40+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106849824} -{"stream": "credit_notes", "data": {"CreditNoteID": "8892719f-019e-42b2-bef8-a6f595fde1eb", "CreditNoteNumber": "CN-0004", "Payments": [], "ID": "8892719f-019e-42b2-bef8-a6f595fde1eb", "HasErrors": false, "CurrencyRate": 1.0, "Type": "ACCRECCREDIT", "Reference": "QWSD", "RemainingCredit": 300.0, "Allocations": [], "HasAttachments": false, "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10002", "Description": "Cap 'Airbyte'", "UnitAmount": 15.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 180.0, "AccountCode": "4000", "Item": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Name": "Cap 'Airbyte'", "Code": "10002"}, "Tracking": [], "Quantity": 12.0, "LineItemID": "f575191e-41ee-486b-b89c-fc7fa23b5bec"}, {"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 120.0, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 10.0, "LineItemID": "2208b4bf-5dc8-46d2-9bf6-3efcaad8ff00"}, {"UnitAmount": 0.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 0.0, "AccountCode": "4300", "Tracking": [], "LineItemID": "089b68a3-8f8d-47a6-aa86-073ad74fb31b"}], "SubTotal": 300.0, "TotalTax": 0.0, "Total": 300.0, "UpdatedDateUTC": "2021-08-31T12:36:07+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106849826} -{"stream": "invoices", "data": {"Type": "ACCREC", "InvoiceID": "76fe4d26-6cca-4b04-b93c-31f18681c727", "InvoiceNumber": "INV-0002", "Reference": "RPT-DD", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 120.0, "AmountPaid": 0.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": true, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-07T00:00:00", "DueDate": "2021-08-07T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Exclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "OUTPUT", "TaxAmount": 0.0, "LineAmount": 120.0, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 10.0, "DiscountRate": 0.0, "LineItemID": "4bcdc96d-852d-4069-88c2-f44d84b7096e"}], "SubTotal": 120.0, "TotalTax": 0.0, "Total": 120.0, "UpdatedDateUTC": "2021-08-31T11:30:43+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106850525} -{"stream": "invoices", "data": {"Type": "ACCPAY", "InvoiceID": "c1779067-71c7-4fd9-bb62-11239ecc99df", "InvoiceNumber": "RPT-DD", "Reference": "", "Payments": [{"PaymentID": "516a589d-c97f-4918-9ebd-9cc0473b252c", "Date": "2021-08-31T00:00:00+00:00", "Amount": 84.0, "Reference": "RPT-DD", "CurrencyRate": 1.0, "HasAccount": false, "HasValidationErrors": false}], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 0.0, "AmountPaid": 84.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": false, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "Name": "Petra Shop", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-31T00:00:00", "DueDate": "2021-08-31T00:00:00+00:00", "Status": "PAID", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10002", "Description": "Cap 'Airbyte'", "UnitAmount": 18.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 54.0, "AccountCode": "123451234", "Item": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Name": "Cap 'Airbyte'", "Code": "10002"}, "Tracking": [], "Quantity": 3.0, "LineItemID": "d507f2b7-8daa-49f6-9498-7c47fc50a559"}, {"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 10.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 30.0, "AccountCode": "123451234", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 3.0, "LineItemID": "d6712837-d0dd-4930-b160-222743722766"}], "SubTotal": 84.0, "TotalTax": 0.0, "Total": 84.0, "UpdatedDateUTC": "2021-08-31T11:39:27+00:00", "CurrencyCode": "USD", "FullyPaidOnDate": "2021-08-31T00:00:00+00:00"}, "emitted_at": 1683106850526} -{"stream": "invoices", "data": {"Type": "ACCPAY", "InvoiceID": "dfe48ca9-b4c8-4441-93c9-30b5818eaa6d", "InvoiceNumber": "RPT-DD", "Reference": "", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 1.5, "AmountPaid": 0.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": false, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-07T00:00:00", "DueDate": "2021-08-07T00:00:00+00:00", "Status": "DRAFT", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 1.5, "AccountCode": "123451234", "Item": {"ItemID": "e6f28cdb-2fa3-4b4d-af19-c77497211cec", "Name": "Pen 'Airbyte'", "Code": "10005"}, "Tracking": [], "Quantity": 1.0, "LineItemID": "4fe37d11-b6a9-42c4-80ad-c6cd21286014"}], "SubTotal": 1.5, "TotalTax": 0.0, "Total": 1.5, "UpdatedDateUTC": "2021-08-31T11:58:21+00:00", "CurrencyCode": "USD"}, "emitted_at": 1683106850527} -{"stream": "invoices", "data": {"Type": "ACCREC", "InvoiceID": "4289a27d-27a5-4569-bbdc-48e0cf14f366", "InvoiceNumber": "INV-0001", "Reference": "RPT-DD", "Payments": [{"PaymentID": "498cc360-e1ce-4056-83a5-ea29f8a7a595", "Date": "2021-08-31T00:00:00+00:00", "Amount": 34.15, "Reference": "RPT-DD", "CurrencyRate": 1.0, "HasAccount": false, "HasValidationErrors": false}], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 0.0, "AmountPaid": 34.15, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": true, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-07T00:00:00", "DueDate": "2021-08-07T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "PAID", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 11.4, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 1.0, "DiscountRate": 5.0, "LineItemID": "72af4707-a92d-4468-909e-90c61f30cd6b"}, {"ItemCode": "10002", "Description": "Cap 'Airbyte'", "UnitAmount": 15.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 14.25, "AccountCode": "4000", "Item": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Name": "Cap 'Airbyte'", "Code": "10002"}, "Tracking": [], "Quantity": 1.0, "DiscountRate": 5.0, "LineItemID": "e967a601-e865-40ba-9a8b-b560e0b83899"}, {"ItemCode": "1003", "Description": "Cup 'Airbyte'", "UnitAmount": 8.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 8.5, "AccountCode": "4000", "Item": {"ItemID": "0b218105-49f9-4d89-8aaa-4de437422b22", "Name": "Cup 'Airbyte'", "Code": "1003"}, "Tracking": [], "Quantity": 1.0, "DiscountRate": 0.0, "LineItemID": "690f5e11-c297-458d-8414-e399ada23c79"}], "SubTotal": 34.15, "TotalTax": 0.0, "Total": 34.15, "UpdatedDateUTC": "2021-08-31T12:04:21+00:00", "CurrencyCode": "USD", "FullyPaidOnDate": "2021-08-31T00:00:00+00:00"}, "emitted_at": 1683106850527} -{"stream": "purchase_orders", "data": {"PurchaseOrderID": "5e634e70-837c-41c6-a924-c5f43370f613", "PurchaseOrderNumber": "PO-0001", "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DeliveryDateString": "2021-08-10T00:00:00", "DeliveryDate": "2021-08-10T00:00:00+00:00", "DeliveryAddress": "350 29th Avenue\nSan Francisco\nCalifornia\n94121\nUnited States", "AttentionTo": "Airbyte Testing", "Telephone": " +1 925-949-5463", "DeliveryInstructions": "FedEx", "HasErrors": false, "IsDiscounted": false, "Reference": "RPT-DD", "Type": "PURCHASEORDER", "CurrencyRate": 1.0, "CurrencyCode": "USD", "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "ContactStatus": "ACTIVE", "Name": "Petra Shop", "FirstName": "Mickle", "LastName": "Born", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}, {"AddressType": "POBOX", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}], "Phones": [{"PhoneType": "DEFAULT", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:29:18+00:00", "ContactGroups": [], "DefaultCurrency": "USD", "ContactPersons": [], "HasValidationErrors": false}, "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 6.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "2435b710-f070-444c-ad74-1ab284bde68f"}], "SubTotal": 6.0, "TotalTax": 0.0, "Total": 6.0, "UpdatedDateUTC": "2021-08-31T11:52:32+00:00", "HasAttachments": false}, "emitted_at": 1683106851245} -{"stream": "purchase_orders", "data": {"PurchaseOrderID": "23b4ba7d-c171-46d6-90b8-8985ddedf116", "PurchaseOrderNumber": "PO-0002", "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DeliveryAddress": "350 29th Avenue\nSan Francisco\nCalifornia\n94121\nUnited States", "AttentionTo": "Airbyte Testing", "Telephone": "", "DeliveryInstructions": "", "HasErrors": false, "IsDiscounted": false, "Reference": "", "Type": "PURCHASEORDER", "CurrencyRate": 1.0, "CurrencyCode": "USD", "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "ContactStatus": "ACTIVE", "Name": "Milly", "FirstName": "Ashly", "LastName": "Simon", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "United States", "AttentionTo": ""}], "Phones": [{"PhoneType": "DEFAULT", "PhoneNumber": "6179800", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "DDI", "PhoneNumber": "6179804", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "6179802", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "6179803", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:30:20+00:00", "ContactGroups": [], "DefaultCurrency": "USD", "ContactPersons": [], "HasValidationErrors": false}, "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 6.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "646a2f78-2175-483c-868b-95c2d8b1445d"}, {"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 1.5, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "10138b09-066c-4405-98bb-b6a67c41391a"}], "SubTotal": 7.5, "TotalTax": 0.0, "Total": 7.5, "UpdatedDateUTC": "2021-08-31T11:54:19+00:00", "HasAttachments": false}, "emitted_at": 1683106851246} -{"stream": "accounts", "data": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account", "Status": "ACTIVE", "Type": "BANK", "TaxType": "NONE", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountNumber": "1234567890", "BankAccountType": "BANK", "CurrencyCode": "USD", "ReportingCode": "ASS.CUR.CAS.CAS", "ReportingCodeName": "Cash on hand", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T12:09:02+00:00", "AddToWatchlist": false}, "emitted_at": 1683106851846} -{"stream": "accounts", "data": {"AccountID": "7dc83cc5-4a75-40aa-abe6-806b934bbce0", "Code": "1200", "Name": "Accounts Receivable", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "NONE", "Description": "Outstanding invoices the company has issued out to the client but has not yet received in cash at balance date.", "Class": "ASSET", "SystemAccount": "DEBTORS", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS.CUR.REC.ACR", "HasAttachments": false, "UpdatedDateUTC": "2021-08-27T14:03:33+00:00", "AddToWatchlist": false}, "emitted_at": 1683106851847} -{"stream": "accounts", "data": {"AccountID": "5043da8e-9646-4ffa-a235-707ea08e23f0", "Code": "1231231230", "Name": "Current Asset account", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "NONE", "Description": "Current Asset account", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS", "ReportingCodeName": "Assets", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T10:41:57+00:00", "AddToWatchlist": true}, "emitted_at": 1683106851847} -{"stream": "items", "data": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Code": "10001", "Description": "T-shirt 'Airbyte'", "PurchaseDescription": "T-shirt 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:18:51+00:00", "PurchaseDetails": {"UnitPrice": 10.0, "COGSAccountCode": "5100", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 12.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "T-shirt 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 40.0, "QuantityOnHand": 4.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1683106852494} -{"stream": "items", "data": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Code": "10002", "Description": "Cap 'Airbyte'", "PurchaseDescription": "Cap 'Airbyte'", "UpdatedDateUTC": "2021-08-31T10:51:03+00:00", "PurchaseDetails": {"UnitPrice": 18.0, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 15.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Cap 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 36.0, "QuantityOnHand": 2.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1683106852495} -{"stream": "items", "data": {"ItemID": "27212cb0-4d3e-4655-8d90-f2895452cd23", "Code": "10004", "Description": "Notebook", "PurchaseDescription": "Notebook 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:51:30+00:00", "PurchaseDetails": {"UnitPrice": 6.0, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 7.5, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Notebook 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 12.0, "QuantityOnHand": 2.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1683106852495} -{"stream": "items", "data": {"ItemID": "e6f28cdb-2fa3-4b4d-af19-c77497211cec", "Code": "10005", "Description": "Pen 'Airbyte'", "PurchaseDescription": "Pen 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:54:11+00:00", "PurchaseDetails": {"UnitPrice": 1.5, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 1.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Pen 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 6.0, "QuantityOnHand": 4.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1683106852496} -{"stream": "items", "data": {"ItemID": "0b218105-49f9-4d89-8aaa-4de437422b22", "Code": "1003", "Description": "Cup 'Airbyte'", "UpdatedDateUTC": "2021-08-31T10:52:13+00:00", "PurchaseDetails": {}, "SalesDetails": {"UnitPrice": 8.5, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Cup 'Airbyte'", "IsTrackedAsInventory": false, "IsSold": true, "IsPurchased": false}, "emitted_at": 1683106852496} -{"stream": "payments", "data": {"PaymentID": "516a589d-c97f-4918-9ebd-9cc0473b252c", "Date": "2021-08-31T00:00:00+00:00", "BankAmount": 84.0, "Amount": 84.0, "Reference": "RPT-DD", "CurrencyRate": 1.0, "PaymentType": "ACCPAYPAYMENT", "Status": "AUTHORISED", "UpdatedDateUTC": "2021-08-31T11:39:27+00:00", "HasAccount": true, "IsReconciled": false, "Account": {"AccountID": "b93d9b74-636f-4221-8b07-6b3e04f7e553", "Code": "2500"}, "Invoice": {"Type": "ACCPAY", "InvoiceID": "c1779067-71c7-4fd9-bb62-11239ecc99df", "InvoiceNumber": "RPT-DD", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "IsDiscounted": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "Name": "Petra Shop", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "LineItems": [], "CurrencyCode": "USD"}, "HasValidationErrors": false}, "emitted_at": 1684221140987} -{"stream": "payments", "data": {"PaymentID": "498cc360-e1ce-4056-83a5-ea29f8a7a595", "Date": "2021-08-31T00:00:00+00:00", "BankAmount": 34.15, "Amount": 34.15, "Reference": "RPT-DD", "CurrencyRate": 1.0, "PaymentType": "ACCRECPAYMENT", "Status": "AUTHORISED", "UpdatedDateUTC": "2021-08-31T12:04:21+00:00", "HasAccount": true, "IsReconciled": false, "Account": {"AccountID": "b93d9b74-636f-4221-8b07-6b3e04f7e553", "Code": "2500"}, "Invoice": {"Type": "ACCREC", "InvoiceID": "4289a27d-27a5-4569-bbdc-48e0cf14f366", "InvoiceNumber": "INV-0001", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "IsDiscounted": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "LineItems": [], "CurrencyCode": "USD"}, "HasValidationErrors": false}, "emitted_at": 1684221140989} -{"stream": "users", "data": {"GlobalUserID": "2dee575d-3d00-4cbd-8922-64da694c8d85", "UserID": "2e127993-414a-4165-895c-c70c3498c0c8", "EmailAddress": "integration-test@airbyte.io", "FirstName": "Airbyte", "LastName": "Testing", "UpdatedDateUTC": "2021-08-27T14:02:19+00:00", "IsSubscriber": true, "OrganisationRole": "STANDARD"}, "emitted_at": 1683106853603} -{"stream": "branding_themes", "data": {"BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Name": "Standard", "LogoUrl": "", "Type": "INVOICE", "SortOrder": 0, "CreatedDateUTC": "2021-08-31T10:52:32+00:00"}, "emitted_at": 1683106854087} -{"stream": "contact_groups", "data": {"ContactGroupID": "051f5017-8e97-4c8d-a5d6-1f3e83c79d14", "Name": "California", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}, "emitted_at": 1683106854612} -{"stream": "contact_groups", "data": {"ContactGroupID": "7ba28f5a-62c4-4555-bb2f-36e005ee1a2b", "Name": "Georgia", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}, "emitted_at": 1683106854613} -{"stream": "contact_groups", "data": {"ContactGroupID": "bba22118-594c-4bf3-831b-987078449779", "Name": "New York", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}, "emitted_at": 1683106854613} -{"stream": "currencies", "data": {"Code": "USD", "Description": "United States Dollar"}, "emitted_at": 1683106855101} -{"stream": "organisations", "data": {"APIKey": "LZ2BMV53EQJAKSNODYPCG3JZF6WMOO", "Name": "Airbyte Testing", "LegalName": "Airbyte", "PaysTax": true, "Version": "US", "OrganisationType": "TRUST", "BaseCurrency": "USD", "CountryCode": "US", "IsDemoCompany": false, "OrganisationStatus": "ACTIVE", "RegistrationNumber": "10000100001", "EmployerIdentificationNumber": "123456789", "FinancialYearEndDay": 31, "FinancialYearEndMonth": 12, "DefaultSalesTax": "Tax Exclusive", "DefaultPurchasesTax": "No Tax", "CreatedDateUTC": "2021-08-27T14:03:33+00:00", "OrganisationEntityType": "TRUST", "Timezone": "EASTERNSTANDARDTIME", "ShortCode": "!Tg3zw", "OrganisationID": "a680e319-119a-4547-8d32-321bdc79efb4", "Edition": "BUSINESS", "Class": "STARTER", "LineOfBusiness": "Technology Integration", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Avenue", "City": "San Francisco", "Region": "California", "PostalCode": "94121", "Country": "United States", "AttentionTo": "Airbyte Testing"}, {"AddressType": "POBOX", "AddressLine1": "350 29th Avenue", "City": "San Francisco", "Region": "California", "PostalCode": "94121", "Country": "United States", "AttentionTo": "Airbyte Testing"}], "Phones": [], "ExternalLinks": [], "PaymentTerms": {}}, "emitted_at": 1683106855673} -{"stream": "tax_rates", "data": {"Name": "Auto Look Up", "TaxType": "AVALARA", "ReportTaxType": "AVALARA", "CanApplyToAssets": true, "CanApplyToEquity": true, "CanApplyToExpenses": true, "CanApplyToLiabilities": true, "CanApplyToRevenue": true, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "Auto Look Up", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1683106856298} -{"stream": "tax_rates", "data": {"Name": "Sales Tax on Imports", "TaxType": "GSTONIMPORTS", "ReportTaxType": "GSTONIMPORTS", "CanApplyToAssets": false, "CanApplyToEquity": false, "CanApplyToExpenses": false, "CanApplyToLiabilities": true, "CanApplyToRevenue": false, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "TAX", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1683106856298} -{"stream": "tax_rates", "data": {"Name": "Tax Exempt", "TaxType": "NONE", "ReportTaxType": "NONE", "CanApplyToAssets": true, "CanApplyToEquity": true, "CanApplyToExpenses": true, "CanApplyToLiabilities": true, "CanApplyToRevenue": true, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "No Tax", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1683106856299} -{"stream": "tax_rates", "data": {"Name": "Tax on Purchases", "TaxType": "INPUT", "ReportTaxType": "INPUT", "CanApplyToAssets": true, "CanApplyToEquity": true, "CanApplyToExpenses": true, "CanApplyToLiabilities": true, "CanApplyToRevenue": true, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "Purchases Tax", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1683106856299} -{"stream": "tax_rates", "data": {"Name": "Tax on Sales", "TaxType": "OUTPUT", "ReportTaxType": "OUTPUT", "CanApplyToAssets": true, "CanApplyToEquity": true, "CanApplyToExpenses": true, "CanApplyToLiabilities": true, "CanApplyToRevenue": true, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "Sales Tax", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1683106856299} +{"stream": "bank_transactions", "data": {"BankTransactionID": "4848c602-aeba-4e01-a533-8eae3e090633", "BankAccount": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account"}, "Type": "SPEND", "Reference": "", "IsReconciled": false, "HasAttachments": false, "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "Status": "AUTHORISED", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 3.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 2.0, "LineItemID": "340a70b3-d4ff-4bad-85c8-d88331fc4b20", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 3.0, "TotalTax": 0.0, "Total": 3.0, "UpdatedDateUTC": "2021-08-31T12:25:54+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572844029} +{"stream": "bank_transactions", "data": {"BankTransactionID": "550c811d-66d3-4b72-9334-4555d22c85b5", "BankAccount": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account"}, "Type": "SPEND", "Reference": "PHTR", "IsReconciled": false, "HasAttachments": false, "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "Status": "AUTHORISED", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 10.0, "TaxType": "GSTONIMPORTS", "TaxAmount": 0.0, "LineAmount": 10.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "6ffc731f-b203-4078-ae50-893f5e24d808", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 10.0, "TotalTax": 0.0, "Total": 10.0, "UpdatedDateUTC": "2021-08-31T12:31:27+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572844030} +{"stream": "bank_transactions", "data": {"BankTransactionID": "9a704749-8084-4eed-9554-4edccaa1b6ce", "BankAccount": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account"}, "Type": "SPEND", "Reference": "YTR", "IsReconciled": false, "HasAttachments": false, "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "Status": "AUTHORISED", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 3.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 2.0, "LineItemID": "67f578d5-522d-4c0a-9e54-c68df01c9093", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}, {"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 12.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 2.0, "LineItemID": "5f40636c-c56b-48d7-bf2d-dc47cd3c24d7", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 15.0, "TotalTax": 0.0, "Total": 15.0, "UpdatedDateUTC": "2021-08-31T12:32:29+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572844030} +{"stream": "contacts", "data": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "ContactStatus": "ACTIVE", "Name": "Petra Shop", "FirstName": "Mickle", "LastName": "Born", "EmailAddress": "mickleborn@yahoo.com", "BankAccountDetails": "1112223334", "TaxNumber": "987654321", "AccountsReceivableTaxType": "GSTONIMPORTS", "AccountsPayableTaxType": "OUTPUT", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}, {"AddressType": "POBOX", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}], "Phones": [{"PhoneType": "DDI", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}, {"PhoneType": "DEFAULT", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}, {"PhoneType": "MOBILE", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:29:18+00:00", "ContactGroups": [{"ContactGroupID": "051f5017-8e97-4c8d-a5d6-1f3e83c79d14", "Name": "California", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}], "IsSupplier": true, "IsCustomer": false, "DefaultCurrency": "USD", "Discount": 10.0, "Website": "http://petrashop.com", "PurchasesDefaultAccountCode": "6040", "SalesDefaultLineAmountType": "INCLUSIVE", "PurchasesDefaultLineAmountType": "NONE", "SalesDefaultAccountCode": "4000", "BatchPayments": {"BankAccountNumber": "1112223334", "BankAccountName": "Citibank", "Details": "", "Code": "", "Reference": ""}, "PaymentTerms": {"Bills": {"Day": 0, "Type": "DAYSAFTERBILLDATE"}, "Sales": {"Day": 0, "Type": "DAYSAFTERBILLMONTH"}}, "ContactPersons": [], "HasAttachments": false, "HasValidationErrors": false}, "emitted_at": 1691572844729} +{"stream": "contacts", "data": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "ContactStatus": "ACTIVE", "Name": "Milly", "FirstName": "Ashly", "LastName": "Simon", "EmailAddress": "asimon@gmail.com", "BankAccountDetails": "123456789", "TaxNumber": "123456789", "AccountsReceivableTaxType": "GSTONIMPORTS", "AccountsPayableTaxType": "GSTONIMPORTS", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "United States", "AttentionTo": ""}], "Phones": [{"PhoneType": "DDI", "PhoneNumber": "6179804", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "DEFAULT", "PhoneNumber": "6179800", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "6179802", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "6179803", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:30:20+00:00", "ContactGroups": [{"ContactGroupID": "7ba28f5a-62c4-4555-bb2f-36e005ee1a2b", "Name": "Georgia", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}], "IsSupplier": false, "IsCustomer": true, "DefaultCurrency": "USD", "Discount": 5.0, "Website": "http://milly.com/us", "PurchasesDefaultAccountCode": "6000", "SalesDefaultLineAmountType": "INCLUSIVE", "PurchasesDefaultLineAmountType": "INCLUSIVE", "SalesDefaultAccountCode": "4300", "BatchPayments": {"BankAccountNumber": "123456789", "BankAccountName": "Bank of America", "Details": "123", "Code": "", "Reference": ""}, "ContactPersons": [], "HasAttachments": false, "HasValidationErrors": false}, "emitted_at": 1691572844730} +{"stream": "contacts", "data": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "ContactStatus": "ACTIVE", "Name": "Paragorn", "FirstName": "Robert", "LastName": "McNamara", "EmailAddress": "robert.paragorn@gmail.com", "BankAccountDetails": "444111111", "TaxNumber": "123456780", "AccountsReceivableTaxType": "OUTPUT", "AccountsPayableTaxType": "INPUT", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1200 S Main St", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Walnut Creek", "Region": "CA", "PostalCode": "94596", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "1200 S Main St", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Walnut Creek", "Region": "CA", "PostalCode": "94596", "Country": "USA"}], "Phones": [{"PhoneType": "DDI", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}, {"PhoneType": "DEFAULT", "PhoneNumber": "9490009", "PhoneAreaCode": "925", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "9490009", "PhoneAreaCode": "925", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "", "PhoneAreaCode": "", "PhoneCountryCode": ""}], "UpdatedDateUTC": "2021-08-31T12:25:53+00:00", "ContactGroups": [{"ContactGroupID": "051f5017-8e97-4c8d-a5d6-1f3e83c79d14", "Name": "California", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}], "IsSupplier": true, "IsCustomer": true, "DefaultCurrency": "USD", "Discount": 0.0, "Website": "http://paragorntrading.com", "PurchasesDefaultAccountCode": "4715", "SalesDefaultAccountCode": "4300", "BatchPayments": {"BankAccountNumber": "444111111", "BankAccountName": "American Express", "Details": "", "Code": "", "Reference": ""}, "Balances": {"AccountsReceivable": {"Outstanding": -12.0, "Overdue": 0.0}, "AccountsPayable": {"Outstanding": 0.0, "Overdue": 0.0}}, "ContactPersons": [], "HasAttachments": false, "HasValidationErrors": false}, "emitted_at": 1691572844730} +{"stream": "credit_notes", "data": {"CreditNoteID": "c38b210c-699c-43c6-a91d-22792093f85a", "CreditNoteNumber": "CN-0003", "Payments": [], "ID": "c38b210c-699c-43c6-a91d-22792093f85a", "HasErrors": false, "CurrencyRate": 1.0, "Type": "ACCRECCREDIT", "Reference": "QSDF", "RemainingCredit": 12.0, "Allocations": [], "HasAttachments": false, "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "AUTHORISED", "LineAmountTypes": "Exclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "OUTPUT", "TaxAmount": 0.0, "LineAmount": 12.0, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 1.0, "LineItemID": "b581ad7d-50fa-4e47-a9e9-9594a56ac27f", "AccountID": "6306b763-7e88-40d2-83a4-caf976c02696"}], "SubTotal": 12.0, "TotalTax": 0.0, "Total": 12.0, "UpdatedDateUTC": "2021-08-31T12:34:40+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572845461} +{"stream": "credit_notes", "data": {"CreditNoteID": "8892719f-019e-42b2-bef8-a6f595fde1eb", "CreditNoteNumber": "CN-0004", "Payments": [], "ID": "8892719f-019e-42b2-bef8-a6f595fde1eb", "HasErrors": false, "CurrencyRate": 1.0, "Type": "ACCRECCREDIT", "Reference": "QWSD", "RemainingCredit": 300.0, "Allocations": [], "HasAttachments": false, "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10002", "Description": "Cap 'Airbyte'", "UnitAmount": 15.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 180.0, "AccountCode": "4000", "Item": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Name": "Cap 'Airbyte'", "Code": "10002"}, "Tracking": [], "Quantity": 12.0, "LineItemID": "f575191e-41ee-486b-b89c-fc7fa23b5bec", "AccountID": "6306b763-7e88-40d2-83a4-caf976c02696"}, {"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 120.0, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 10.0, "LineItemID": "2208b4bf-5dc8-46d2-9bf6-3efcaad8ff00", "AccountID": "6306b763-7e88-40d2-83a4-caf976c02696"}, {"UnitAmount": 0.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 0.0, "AccountCode": "4300", "Tracking": [], "LineItemID": "089b68a3-8f8d-47a6-aa86-073ad74fb31b", "AccountID": "ca893dbb-0f63-4bc4-abaa-58478cdb4124"}], "SubTotal": 300.0, "TotalTax": 0.0, "Total": 300.0, "UpdatedDateUTC": "2021-08-31T12:36:07+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572845462} +{"stream": "invoices", "data": {"Type": "ACCREC", "InvoiceID": "76fe4d26-6cca-4b04-b93c-31f18681c727", "InvoiceNumber": "INV-0002", "Reference": "RPT-DD", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 120.0, "AmountPaid": 0.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": true, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-07T00:00:00", "DueDate": "2021-08-07T00:00:00+00:00", "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Exclusive", "LineItems": [{"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 12.0, "TaxType": "OUTPUT", "TaxAmount": 0.0, "LineAmount": 120.0, "AccountCode": "4000", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 10.0, "DiscountRate": 0.0, "LineItemID": "4bcdc96d-852d-4069-88c2-f44d84b7096e", "AccountID": "6306b763-7e88-40d2-83a4-caf976c02696"}], "SubTotal": 120.0, "TotalTax": 0.0, "Total": 120.0, "UpdatedDateUTC": "2021-08-31T11:30:43+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572846164} +{"stream": "invoices", "data": {"Type": "ACCPAY", "InvoiceID": "c1779067-71c7-4fd9-bb62-11239ecc99df", "InvoiceNumber": "RPT-DD", "Reference": "", "Payments": [{"PaymentID": "516a589d-c97f-4918-9ebd-9cc0473b252c", "Date": "2021-08-31T00:00:00+00:00", "Amount": 84.0, "Reference": "RPT-DD", "CurrencyRate": 1.0, "HasAccount": false, "HasValidationErrors": false}], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 0.0, "AmountPaid": 84.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": false, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "Name": "Petra Shop", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-31T00:00:00", "DueDate": "2021-08-31T00:00:00+00:00", "Status": "PAID", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10002", "Description": "Cap 'Airbyte'", "UnitAmount": 18.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 54.0, "AccountCode": "123451234", "Item": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Name": "Cap 'Airbyte'", "Code": "10002"}, "Tracking": [], "Quantity": 3.0, "LineItemID": "d507f2b7-8daa-49f6-9498-7c47fc50a559", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}, {"ItemCode": "10001", "Description": "T-shirt 'Airbyte'", "UnitAmount": 10.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 30.0, "AccountCode": "123451234", "Item": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Name": "T-shirt 'Airbyte'", "Code": "10001"}, "Tracking": [], "Quantity": 3.0, "LineItemID": "d6712837-d0dd-4930-b160-222743722766", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 84.0, "TotalTax": 0.0, "Total": 84.0, "UpdatedDateUTC": "2021-08-31T11:39:27+00:00", "CurrencyCode": "USD", "FullyPaidOnDate": "2021-08-31T00:00:00+00:00"}, "emitted_at": 1691572846165} +{"stream": "invoices", "data": {"Type": "ACCPAY", "InvoiceID": "dfe48ca9-b4c8-4441-93c9-30b5818eaa6d", "InvoiceNumber": "RPT-DD", "Reference": "", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 1.5, "AmountPaid": 0.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": false, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-07T00:00:00", "DueDate": "2021-08-07T00:00:00+00:00", "Status": "DRAFT", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 1.5, "AccountCode": "123451234", "Item": {"ItemID": "e6f28cdb-2fa3-4b4d-af19-c77497211cec", "Name": "Pen 'Airbyte'", "Code": "10005"}, "Tracking": [], "Quantity": 1.0, "LineItemID": "4fe37d11-b6a9-42c4-80ad-c6cd21286014", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 1.5, "TotalTax": 0.0, "Total": 1.5, "UpdatedDateUTC": "2021-08-31T11:58:21+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572846166} +{"stream": "purchase_orders", "data": {"PurchaseOrderID": "5e634e70-837c-41c6-a924-c5f43370f613", "PurchaseOrderNumber": "PO-0001", "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DeliveryDateString": "2021-08-10T00:00:00", "DeliveryDate": "2021-08-10T00:00:00+00:00", "DeliveryAddress": "350 29th Avenue\nSan Francisco\nCalifornia\n94121\nUnited States", "AttentionTo": "Airbyte Testing", "Telephone": " +1 925-949-5463", "DeliveryInstructions": "FedEx", "HasErrors": false, "IsDiscounted": false, "Reference": "RPT-DD", "Type": "PURCHASEORDER", "CurrencyRate": 1.0, "CurrencyCode": "USD", "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "ContactStatus": "ACTIVE", "Name": "Petra Shop", "FirstName": "Mickle", "LastName": "Born", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}, {"AddressType": "POBOX", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}], "Phones": [{"PhoneType": "DEFAULT", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:29:18+00:00", "ContactGroups": [], "DefaultCurrency": "USD", "ContactPersons": [], "HasValidationErrors": false}, "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 6.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "2435b710-f070-444c-ad74-1ab284bde68f"}], "SubTotal": 6.0, "TotalTax": 0.0, "Total": 6.0, "UpdatedDateUTC": "2021-08-31T11:52:32+00:00", "HasAttachments": false}, "emitted_at": 1691572848638} +{"stream": "purchase_orders", "data": {"PurchaseOrderID": "23b4ba7d-c171-46d6-90b8-8985ddedf116", "PurchaseOrderNumber": "PO-0002", "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DeliveryAddress": "350 29th Avenue\nSan Francisco\nCalifornia\n94121\nUnited States", "AttentionTo": "Airbyte Testing", "Telephone": "", "DeliveryInstructions": "", "HasErrors": false, "IsDiscounted": false, "Reference": "", "Type": "PURCHASEORDER", "CurrencyRate": 1.0, "CurrencyCode": "USD", "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "ContactStatus": "ACTIVE", "Name": "Milly", "FirstName": "Ashly", "LastName": "Simon", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "United States", "AttentionTo": ""}], "Phones": [{"PhoneType": "DEFAULT", "PhoneNumber": "6179800", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "DDI", "PhoneNumber": "6179804", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "6179802", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "6179803", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:30:20+00:00", "ContactGroups": [], "DefaultCurrency": "USD", "ContactPersons": [], "HasValidationErrors": false}, "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 6.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "646a2f78-2175-483c-868b-95c2d8b1445d"}, {"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 1.5, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "10138b09-066c-4405-98bb-b6a67c41391a"}], "SubTotal": 7.5, "TotalTax": 0.0, "Total": 7.5, "UpdatedDateUTC": "2021-08-31T11:54:19+00:00", "HasAttachments": false}, "emitted_at": 1691572848639} +{"stream": "accounts", "data": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account", "Status": "ACTIVE", "Type": "BANK", "TaxType": "NONE", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountNumber": "1234567890", "BankAccountType": "BANK", "CurrencyCode": "USD", "ReportingCode": "ASS.CUR.CAS.CAS", "ReportingCodeName": "Cash on hand", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T12:09:02+00:00", "AddToWatchlist": false}, "emitted_at": 1691572849400} +{"stream": "accounts", "data": {"AccountID": "7dc83cc5-4a75-40aa-abe6-806b934bbce0", "Code": "1200", "Name": "Accounts Receivable", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "NONE", "Description": "Outstanding invoices the company has issued out to the client but has not yet received in cash at balance date.", "Class": "ASSET", "SystemAccount": "DEBTORS", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS.CUR.REC.ACR", "HasAttachments": false, "UpdatedDateUTC": "2021-08-27T14:03:33+00:00", "AddToWatchlist": false}, "emitted_at": 1691572849401} +{"stream": "accounts", "data": {"AccountID": "5043da8e-9646-4ffa-a235-707ea08e23f0", "Code": "1231231230", "Name": "Current Asset account", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "NONE", "Description": "Current Asset account", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS", "ReportingCodeName": "Assets", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T10:41:57+00:00", "AddToWatchlist": true}, "emitted_at": 1691572849401} +{"stream": "items", "data": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Code": "10001", "Description": "T-shirt 'Airbyte'", "PurchaseDescription": "T-shirt 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:18:51+00:00", "PurchaseDetails": {"UnitPrice": 10.0, "COGSAccountCode": "5100", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 12.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "T-shirt 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 40.0, "QuantityOnHand": 4.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1691572851425} +{"stream": "items", "data": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Code": "10002", "Description": "Cap 'Airbyte'", "PurchaseDescription": "Cap 'Airbyte'", "UpdatedDateUTC": "2021-08-31T10:51:03+00:00", "PurchaseDetails": {"UnitPrice": 18.0, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 15.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Cap 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 36.0, "QuantityOnHand": 2.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1691572851426} +{"stream": "items", "data": {"ItemID": "27212cb0-4d3e-4655-8d90-f2895452cd23", "Code": "10004", "Description": "Notebook", "PurchaseDescription": "Notebook 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:51:30+00:00", "PurchaseDetails": {"UnitPrice": 6.0, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 7.5, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Notebook 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 12.0, "QuantityOnHand": 2.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1691572851427} +{"stream": "payments", "data": {"PaymentID": "516a589d-c97f-4918-9ebd-9cc0473b252c", "Date": "2021-08-31T00:00:00+00:00", "BankAmount": 84.0, "Amount": 84.0, "Reference": "RPT-DD", "CurrencyRate": 1.0, "PaymentType": "ACCPAYPAYMENT", "Status": "AUTHORISED", "UpdatedDateUTC": "2021-08-31T11:39:27+00:00", "HasAccount": true, "IsReconciled": false, "Account": {"AccountID": "b93d9b74-636f-4221-8b07-6b3e04f7e553", "Code": "2500"}, "Invoice": {"Type": "ACCPAY", "InvoiceID": "c1779067-71c7-4fd9-bb62-11239ecc99df", "InvoiceNumber": "RPT-DD", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "IsDiscounted": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "Name": "Petra Shop", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "LineItems": [], "CurrencyCode": "USD"}, "HasValidationErrors": false}, "emitted_at": 1691572852043} +{"stream": "payments", "data": {"PaymentID": "498cc360-e1ce-4056-83a5-ea29f8a7a595", "Date": "2021-08-31T00:00:00+00:00", "BankAmount": 34.15, "Amount": 34.15, "Reference": "RPT-DD", "CurrencyRate": 1.0, "PaymentType": "ACCRECPAYMENT", "Status": "AUTHORISED", "UpdatedDateUTC": "2021-08-31T12:04:21+00:00", "HasAccount": true, "IsReconciled": false, "Account": {"AccountID": "b93d9b74-636f-4221-8b07-6b3e04f7e553", "Code": "2500"}, "Invoice": {"Type": "ACCREC", "InvoiceID": "4289a27d-27a5-4569-bbdc-48e0cf14f366", "InvoiceNumber": "INV-0001", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "IsDiscounted": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "Name": "Milly", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "LineItems": [], "CurrencyCode": "USD"}, "HasValidationErrors": false}, "emitted_at": 1691572852044} +{"stream": "users", "data": {"GlobalUserID": "2dee575d-3d00-4cbd-8922-64da694c8d85", "UserID": "2e127993-414a-4165-895c-c70c3498c0c8", "EmailAddress": "integration-test@airbyte.io", "FirstName": "Airbyte", "LastName": "Testing", "UpdatedDateUTC": "2021-08-27T14:02:19+00:00", "IsSubscriber": true, "OrganisationRole": "STANDARD"}, "emitted_at": 1691572852657} +{"stream": "branding_themes", "data": {"BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Name": "Standard", "LogoUrl": "", "Type": "INVOICE", "SortOrder": 0, "CreatedDateUTC": "2021-08-31T10:52:32+00:00"}, "emitted_at": 1691572853325} +{"stream": "contact_groups", "data": {"ContactGroupID": "051f5017-8e97-4c8d-a5d6-1f3e83c79d14", "Name": "California", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}, "emitted_at": 1691572853921} +{"stream": "contact_groups", "data": {"ContactGroupID": "7ba28f5a-62c4-4555-bb2f-36e005ee1a2b", "Name": "Georgia", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}, "emitted_at": 1691572853922} +{"stream": "contact_groups", "data": {"ContactGroupID": "bba22118-594c-4bf3-831b-987078449779", "Name": "New York", "Status": "ACTIVE", "Contacts": [], "HasValidationErrors": false}, "emitted_at": 1691572853923} +{"stream": "currencies", "data": {"Code": "USD", "Description": "United States Dollar"}, "emitted_at": 1691572854494} +{"stream": "organisations", "data": {"APIKey": "LZ2BMV53EQJAKSNODYPCG3JZF6WMOO", "Name": "Airbyte Testing", "LegalName": "Airbyte", "PaysTax": true, "Version": "US", "OrganisationType": "TRUST", "BaseCurrency": "USD", "CountryCode": "US", "IsDemoCompany": false, "OrganisationStatus": "ACTIVE", "RegistrationNumber": "10000100001", "EmployerIdentificationNumber": "123456789", "FinancialYearEndDay": 31, "FinancialYearEndMonth": 12, "DefaultSalesTax": "Tax Exclusive", "DefaultPurchasesTax": "No Tax", "CreatedDateUTC": "2021-08-27T14:03:33+00:00", "OrganisationEntityType": "TRUST", "Timezone": "EASTERNSTANDARDTIME", "ShortCode": "!Tg3zw", "OrganisationID": "a680e319-119a-4547-8d32-321bdc79efb4", "Edition": "BUSINESS", "Class": "STARTER", "LineOfBusiness": "Technology Integration", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Avenue", "City": "San Francisco", "Region": "California", "PostalCode": "94121", "Country": "United States", "AttentionTo": "Airbyte Testing"}, {"AddressType": "POBOX", "AddressLine1": "350 29th Avenue", "City": "San Francisco", "Region": "California", "PostalCode": "94121", "Country": "United States", "AttentionTo": "Airbyte Testing"}], "Phones": [], "ExternalLinks": [], "PaymentTerms": {}}, "emitted_at": 1691572855310} +{"stream": "tax_rates", "data": {"Name": "Auto Look Up", "TaxType": "AVALARA", "ReportTaxType": "AVALARA", "CanApplyToAssets": true, "CanApplyToEquity": true, "CanApplyToExpenses": true, "CanApplyToLiabilities": true, "CanApplyToRevenue": true, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "Auto Look Up", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1691572856619} +{"stream": "tax_rates", "data": {"Name": "Sales Tax on Imports", "TaxType": "GSTONIMPORTS", "ReportTaxType": "GSTONIMPORTS", "CanApplyToAssets": false, "CanApplyToEquity": false, "CanApplyToExpenses": false, "CanApplyToLiabilities": true, "CanApplyToRevenue": false, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "TAX", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1691572856620} +{"stream": "tax_rates", "data": {"Name": "Tax Exempt", "TaxType": "NONE", "ReportTaxType": "NONE", "CanApplyToAssets": true, "CanApplyToEquity": true, "CanApplyToExpenses": true, "CanApplyToLiabilities": true, "CanApplyToRevenue": true, "DisplayTaxRate": 0.0, "EffectiveRate": 0.0, "Status": "ACTIVE", "TaxComponents": [{"Name": "No Tax", "Rate": 0.0, "IsCompound": false, "IsNonRecoverable": false}]}, "emitted_at": 1691572856620} diff --git a/airbyte-integrations/connectors/source-xero/metadata.yaml b/airbyte-integrations/connectors/source-xero/metadata.yaml index e5b5e6b85d03..f70a0bf641f2 100644 --- a/airbyte-integrations/connectors/source-xero/metadata.yaml +++ b/airbyte-integrations/connectors/source-xero/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6fd1e833-dd6e-45ec-a727-ab917c5be892 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.3 dockerRepository: airbyte/source-xero githubIssueLabel: source-xero icon: xero.svg @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/xero tags: - language:python + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-xero/requirements.txt b/airbyte-integrations/connectors/source-xero/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-xero/requirements.txt +++ b/airbyte-integrations/connectors/source-xero/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-xero/setup.py b/airbyte-integrations/connectors/source-xero/setup.py index bbdadb993c1c..b93015ee5795 100644 --- a/airbyte-integrations/connectors/source-xero/setup.py +++ b/airbyte-integrations/connectors/source-xero/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.40", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index b29cfaa27cbf..1e1ccf3f7727 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -83,8 +83,8 @@ def get_authenticator(config: Mapping[str, Any]) -> Mapping[str, Any]: return XeroSingleUseRefreshTokenOauth2Authenticator( connector_config=config, token_refresh_endpoint="https://identity.xero.com/connect/token", - client_id_config_path=["authentication", "client_id"], - client_secret_config_path=["authentication", "client_secret"], + client_id=config["authentication"]["client_id"], + client_secret=config["authentication"]["client_secret"], access_token_config_path=["authentication", "access_token"], refresh_token_config_path=["authentication", "refresh_token"], token_expiry_date_config_path=["authentication", "token_expiry_date"], diff --git a/airbyte-integrations/connectors/source-xkcd/metadata.yaml b/airbyte-integrations/connectors/source-xkcd/metadata.yaml index 82f30bd52fde..6fb89af41897 100644 --- a/airbyte-integrations/connectors/source-xkcd/metadata.yaml +++ b/airbyte-integrations/connectors/source-xkcd/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/xkcd tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-xkcd/requirements.txt b/airbyte-integrations/connectors/source-xkcd/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-xkcd/requirements.txt +++ b/airbyte-integrations/connectors/source-xkcd/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-xkcd/setup.py b/airbyte-integrations/connectors/source-xkcd/setup.py index 34223ef3ca68..7fe43ebd17d6 100644 --- a/airbyte-integrations/connectors/source-xkcd/setup.py +++ b/airbyte-integrations/connectors/source-xkcd/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/Dockerfile b/airbyte-integrations/connectors/source-yahoo-finance-price/Dockerfile index 62249e997e9e..1e8d54c94f3a 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/Dockerfile +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/Dockerfile @@ -34,5 +34,5 @@ COPY source_yahoo_finance_price ./source_yahoo_finance_price ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-yahoo-finance-price diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-config.yml b/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-config.yml index a5c66824d764..eb26bc8e2f06 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-config.yml @@ -1,30 +1,22 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-yahoo-finance-price:dev -tests: - spec: - - spec_path: "source_yahoo_finance_price/spec.json" +acceptance_tests: connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh old mode 100644 new mode 100755 index 5797d20fe9a7..b6d65deeccb4 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/icon.svg b/airbyte-integrations/connectors/source-yahoo-finance-price/icon.svg new file mode 100644 index 000000000000..902356742b6a --- /dev/null +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/icon.svg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/__init__.py b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/catalog.json b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/catalog.json index 6799946a6851..76c42d22ff64 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/catalog.json @@ -1,39 +1,23 @@ { "streams": [ { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" + "stream": { + "name": "price", + "json_schema": { + "type": "object", + "properties": { + "ticker": { + "type": "array", + "items": { + "type": "string" + } + } } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/sample_config.json index eb215748c345..0cee7abf1aea 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/integration_tests/sample_config.json @@ -1,5 +1,5 @@ { - "tickers": "MMM, AOS, ABT, ABBV, ACN, ATVI, AYI, ADBE, AAP, AMD, AES, AET, AMG, AFL, A, APD, AKAM, ALK, ALB, ARE, ALGN, ALLE, AGN, ADS, LNT, ALL, GOOGL, GOOG, MO, AMZN, AEE, AAL, AEP, AXP, AIG, AMT, AWK, AMP, ABC, AME, AMGN, APH, APC, ADI, ANDV, ANSS, ANTM, AON, APA, AIV, AAPL, AMAT, APTV, ADM, ARNC, AJG, AIZ, T, ADSK, ADP, AZO, AVB, AVY, BLL, BAC, BAX, BBT, BDX, BRK.B, BBY, BIIB, BLK, HRB, BA, BKNG, BWA, BXP, BSX, BHF, BMY, AVGO, BF.B, CHRW, CA, COG, CDNS, CPB, COF, CAH, KMX, CCL, CAT, CBOE, CBRE, CBS, CELG, CNC, CNP, CTL, CERN, CF, SCHW, CHTR, CVX, CMG, CB, CHD, CI, XEC, CINF, CTAS, CSCO, C, CFG, CTXS, CME, CMS, KO, CTSH, CL, CMCSA, CMA, CAG, CXO, COP, ED, STZ, GLW, COST, COTY, CCI, CSRA, CSX, CMI, CVS, DHI, DHR, DRI", + "tickers": "USD", "interval": "5m", "range": "1d" } diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml b/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml new file mode 100644 index 000000000000..83b5ea098897 --- /dev/null +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/metadata.yaml @@ -0,0 +1,24 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: 09a517d3-803f-448d-97bf-0b1ee64b90ef + dockerImageTag: 0.2.0 + dockerRepository: airbyte/source-yahoo-finance-price + githubIssueLabel: source-yahoo-finance-price + icon: yahoo-finance-price.svg + license: MIT + name: Yahoo Finance Price + registries: + cloud: + enabled: false + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/yahoo-finance-price + tags: + - language:low-code + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/requirements.txt b/airbyte-integrations/connectors/source-yahoo-finance-price/requirements.txt index cc57334ef619..5b8864c417d3 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/requirements.txt +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/requirements.txt @@ -1,2 +1,2 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/connector-acceptance-test --e . diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py b/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py index ad0329bced99..862dc4e0083d 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/setup.py @@ -10,19 +10,19 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", + "requests-mock~=1.9.3", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( name="source_yahoo_finance_price", - description="Source implementation for Yahoo Finance Price API.", - author="Luca Crema", - author_email="luca.crema.98@gmail.com", + description="Source implementation for Yahoo Finance Price.", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/manifest.yaml b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/manifest.yaml new file mode 100644 index 000000000000..55b89403a6d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/manifest.yaml @@ -0,0 +1,260 @@ +version: 0.50.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - price +streams: + - type: DeclarativeStream + name: price + primary_key: [] + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + chart: + properties: + result: + items: + properties: + indicators: + properties: + quote: + items: + properties: + close: + items: + type: + - "null" + - number + type: array + high: + items: + type: + - "null" + - number + type: array + low: + items: + type: + - "null" + - number + type: array + open: + items: + type: + - "null" + - number + type: array + volume: + items: + type: + - "null" + - number + type: array + type: object + type: array + type: object + meta: + properties: + chartPreviousClose: + type: number + currency: + type: + - "null" + - string + currentTradingPeriod: + properties: + post: + properties: + end: + type: number + gmtoffset: + type: number + start: + type: number + timezone: + type: string + type: object + pre: + properties: + end: + type: number + gmtoffset: + type: number + start: + type: number + timezone: + type: string + type: object + regular: + properties: + end: + type: number + gmtoffset: + type: number + start: + type: number + timezone: + type: string + type: object + type: object + dataGranularity: + type: string + exchangeName: + type: string + exchangeTimezoneName: + type: string + firstTradeDate: + type: + - "null" + - number + gmtoffset: + type: number + instrumentType: + type: string + previousClose: + type: number + priceHint: + type: number + range: + type: string + regularMarketPrice: + type: number + regularMarketTime: + type: number + scale: + type: number + symbol: + type: string + timezone: + type: string + tradingPeriods: + items: + items: + properties: + end: + type: number + gmtoffset: + type: number + start: + type: number + timezone: + type: string + type: object + type: array + type: array + validRanges: + items: + type: string + type: array + type: object + timestamp: + items: + type: number + type: array + type: object + type: array + type: object + required: + - chart + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://query1.finance.yahoo.com + path: >- + {% if (next_page_token['next_page_token'] or 0) < + config['tickers'].split(',')|length %}/v8/finance/chart/{{ + config['tickers'].split(',')[next_page_token['next_page_token'] or + 0].strip() }}{% endif %} + http_method: GET + request_parameters: + range: "{{ config['range'] }}" + symbol: >- + {% if (next_page_token['next_page_token'] or 0) < + config['tickers'].split(',')|length %}{{ + config['tickers'].split(',')[next_page_token['next_page_token'] or + 0].strip() }}{% else %} finish {% endif %} + interval: "{{ config['interval'] }}" + request_headers: + Accept: application/json + User-Agent: Mozilla/5.0 (X11; Linux x86_64) + authenticator: + type: NoAuth + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + action: SUCCESS + http_codes: + - 403 + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: DefaultPaginator + pagination_strategy: + type: PageIncrement + start_from_page: 0 +spec: + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - tickers + properties: + tickers: + type: string + order: 0 + title: Tickers + description: >- + Comma-separated identifiers for the stocks to be queried. Whitespaces + are allowed. + interval: + type: string + order: 1 + title: Interval + description: The interval of between prices queried. + enum: + - 1m + - 5m + - 15m + - 30m + - 90m + - 1h + - 1d + - 5d + - 1wk + - 1mo + - 3mo + range: + type: string + order: 2 + title: Range + description: The range of prices to be queried. + enum: + - 1d + - 5d + - 7d + - 1mo + - 3mo + - 6mo + - 1y + - 2y + - 5y + - ytd + - max + additionalProperties: true + documentation_url: https://example.org + type: Spec +metadata: + autoImportSchema: + price: false diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/schemas/price.json b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/schemas/price.json deleted file mode 100644 index af4b1d88f8a9..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/schemas/price.json +++ /dev/null @@ -1,218 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "chart": { - "type": "object", - "properties": { - "result": { - "type": "array", - "items": { - "type": "object", - "properties": { - "meta": { - "type": "object", - "properties": { - "currency": { - "type": "string" - }, - "symbol": { - "type": "string" - }, - "exchangeName": { - "type": "string" - }, - "instrumentType": { - "type": "string" - }, - "firstTradeDate": { - "type": "integer" - }, - "regularMarketTime": { - "type": "integer" - }, - "gmtoffset": { - "type": "integer" - }, - "timezone": { - "type": "string" - }, - "exchangeTimezoneName": { - "type": "string" - }, - "regularMarketPrice": { - "type": "number" - }, - "chartPreviousClose": { - "type": "number" - }, - "previousClose": { - "type": "number" - }, - "scale": { - "type": "integer" - }, - "priceHint": { - "type": "integer" - }, - "currentTradingPeriod": { - "type": "object", - "properties": { - "pre": { - "type": "object", - "properties": { - "timezone": { - "type": "string" - }, - "end": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "gmtoffset": { - "type": "integer" - } - }, - "required": ["timezone", "end", "start", "gmtoffset"] - }, - "regular": { - "type": "object", - "properties": { - "timezone": { - "type": "string" - }, - "end": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "gmtoffset": { - "type": "integer" - } - }, - "required": ["timezone", "end", "start", "gmtoffset"] - }, - "post": { - "type": "object", - "properties": { - "timezone": { - "type": "string" - }, - "end": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "gmtoffset": { - "type": "integer" - } - }, - "required": ["timezone", "end", "start", "gmtoffset"] - } - }, - "required": ["pre", "regular", "post"] - }, - "tradingPeriods": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "timezone": { - "type": "string" - }, - "end": { - "type": "integer" - }, - "start": { - "type": "integer" - }, - "gmtoffset": { - "type": "integer" - } - }, - "required": ["timezone", "end", "start", "gmtoffset"] - } - } - }, - "dataGranularity": { - "type": "string" - }, - "range": { - "type": "string" - }, - "validRanges": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "timestamp": { - "type": "array", - "items": { - "type": "integer" - } - }, - "indicators": { - "type": "object", - "properties": { - "quote": { - "type": "array", - "items": { - "type": "object", - "properties": { - "volume": { - "type": "array", - "items": { - "type": ["integer", "null"] - } - }, - "high": { - "type": "array", - "items": { - "type": ["number", "null"] - } - }, - "close": { - "type": "array", - "items": { - "type": ["number", "null"] - } - }, - "low": { - "type": "array", - "items": { - "type": ["number", "null"] - } - }, - "open": { - "type": "array", - "items": { - "type": ["number", "null"] - } - } - }, - "required": ["volume", "high", "close", "low", "open"] - } - } - }, - "required": ["quote"] - } - }, - "required": ["meta", "timestamp", "indicators"] - } - }, - "error": { - "type": "null" - } - } - } - }, - "required": ["chart"] -} diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/source.py b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/source.py index 099565d963e1..794946e31c95 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/source.py +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/source.py @@ -2,109 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +WARNING: Do not modify this file. +""" -# Full refresh stream -class Price(HttpStream, ABC): - """ - Queries the Yahoo Finance API for the price history of the given stock. - The length of the price history is determined by the interval parameter: - - 1m to 90m: up to 7 days of data - - 1h to 3mo: up to 730 days of data - - Based on the documentation found at https://stackoverflow.com/questions/44030983/yahoo-finance-url-not-working. - """ - - url_base = "https://query1.finance.yahoo.com/" - primary_key = None - - def __init__(self, tickers: str, interval: str, range: str, **kwargs): - super().__init__(**kwargs) - self.tickers = tickers - self.next_index = 0 - self.interval = interval - self.range = range - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - next_index = next_page_token or 0 # At the first request next_page_token is None - return f"v8/finance/chart/{self.tickers[next_index]}" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - We re-use pagination functionality to get one ticker history at a time. - Updates the next_index counter to the next ticker in the list. - """ - self.next_index += 1 - if self.next_index >= len(self.tickers): - return None - return self.next_index - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - next_index = next_page_token or 0 # At the first request next_page_token is None - return { - "symbol": self.tickers[next_index], - "interval": self.interval, - "range": self.range, - } - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - base_headers = super().request_headers(**kwargs) - headers = {"Accept": "application/json", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"} # Required to avoid 403 response - return {**base_headers, **headers} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code != 200: - return [] - yield from [response.json()] - - -# Source -class SourceYahooFinancePrice(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - # Check that the tickers are valid - tickers = list(map(str.strip, config["tickers"].split(","))) - if len(tickers) == 0: - return False, "No valid tickers provided" - for ticker in tickers: - # Check that yahoo finance has the ticker - response = requests.get( - url=f"https://query1.finance.yahoo.com/v6/finance/autocomplete?query={ticker}&lang=en", - headers={"Accept": "application/json", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"}, # Required to avoid 403 response - ) - if response.status_code != 200: - return False, f"Ticker {ticker} not found" - response_json = response.json() - if "ResultSet" not in response_json: - return False, f"Invalid check response format for ticker {ticker}" - if "Result" not in response_json["ResultSet"]: - return False, f"Invalid check response format for ticker {ticker}" - if len(response_json["ResultSet"]["Result"]) == 0: - return False, f"Ticker {ticker} not found" - - # Check that the range parameter is configured according to the interval parameter - # If the range DOES NOT end in "d" we cannot use minute intervals (end in "m") - if "interval" in config and "range" in config: - if config["interval"][-1] == "m" and config["range"][-1] != "d": - return False, "Range parameter must end in 'd' for minute intervals" - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = { - # Split tickers by comma and strip whitespaces - "tickers": list(map(str.strip, config["tickers"].split(","))), - "interval": config.get("interval", "7d"), - "range": config.get("range", "1m"), - } - return [Price(**args)] +# Declarative Source +class SourceYahooFinancePrice(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/spec.json b/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/spec.json deleted file mode 100644 index f1c981efb918..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/source_yahoo_finance_price/spec.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Yahoo Finance Spec", - "type": "object", - "required": ["tickers"], - "additionalProperties": false, - "properties": { - "tickers": { - "type": "string", - "order": 0, - "description": "Comma-separated identifiers for the stocks to be queried. Whitespaces are allowed.", - "examples": ["AAPL, GOOGL, GEO.MI"] - }, - "interval": { - "title": "Interval", - "order": 1, - "description": "The interval of between prices queried.", - "type": "string", - "enum": [ - "1m", - "5m", - "15m", - "30m", - "90m", - "1h", - "1d", - "5d", - "1wk", - "1mo", - "3mo" - ] - }, - "range": { - "title": "Range", - "order": 2, - "description": "The range of prices to be queried.", - "type": "string", - "enum": [ - "1d", - "5d", - "7d", - "1mo", - "3mo", - "6mo", - "1y", - "2y", - "5y", - "ytd", - "max" - ] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/__init__.py b/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/test_source.py b/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/test_source.py deleted file mode 100644 index 330abbd3f790..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/test_source.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_yahoo_finance_price.source import SourceYahooFinancePrice - - -def test_check_connection(mocker): - source = SourceYahooFinancePrice() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceYahooFinancePrice() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/test_streams.py deleted file mode 100644 index 12348dc5f048..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/unit_tests/test_streams.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_yahoo_finance_price.source import Price - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(Price, "path", "v0/example_endpoint") - mocker.patch.object(Price, "primary_key", "test_primary_key") - mocker.patch.object(Price, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = Price() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = Price() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = Price() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = Price() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = Price() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = Price() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = Price() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml b/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml index e1c969e1b889..c20762bdfbd5 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml @@ -19,10 +19,13 @@ tests: extra_fields: no exact_order: no extra_records: yes + timeout_seconds: 3600 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" \ No newline at end of file + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 3600 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/abnormal_state.json index 3383f9cde281..94faaa3b4316 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/abnormal_state.json @@ -2,14 +2,14 @@ { "type": "STREAM", "stream": { - "stream_state": { "dateTime": "2100-01-01T00:00:00+00:00"}, + "stream_state": { "dateTime": "2100-01-01T00:00:00+00:00" }, "stream_descriptor": { "name": "sessions" } } }, { "type": "STREAM", "stream": { - "stream_state": { "dateTime": "2100-01-01T00:00:00+00:00"}, + "stream_state": { "dateTime": "2100-01-01T00:00:00+00:00" }, "stream_descriptor": { "name": "views" } } } diff --git a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/sample_state.json index 3f16647731d0..3df69de3b423 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-yandex-metrica/integration_tests/sample_state.json @@ -2,14 +2,14 @@ { "type": "STREAM", "stream": { - "stream_state": { "dateTime": "2022-11-03T03:22:50+00:00"}, + "stream_state": { "dateTime": "2022-11-03T03:22:50+00:00" }, "stream_descriptor": { "name": "sessions" } } }, { "type": "STREAM", "stream": { - "stream_state": { "dateTime": "2022-11-03T03:22:50+00:00"}, + "stream_state": { "dateTime": "2022-11-03T03:22:50+00:00" }, "stream_descriptor": { "name": "views" } } } diff --git a/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml b/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml index fc252c4e31fa..7fa1a9be1deb 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml +++ b/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/yandex-metrica tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-yandex-metrica/requirements.txt b/airbyte-integrations/connectors/source-yandex-metrica/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/requirements.txt +++ b/airbyte-integrations/connectors/source-yandex-metrica/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-yandex-metrica/setup.py b/airbyte-integrations/connectors/source-yandex-metrica/setup.py index c083d6dc8dd3..ce5d5b361010 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/setup.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = ["freezegun", "pytest~=6.1", "pytest-mock", "requests_mock", "connector-acceptance-test"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "freezegun", "pytest~=6.1", "pytest-mock", "requests_mock"] setup( name="source_yandex_metrica", diff --git a/airbyte-integrations/connectors/source-yotpo/metadata.yaml b/airbyte-integrations/connectors/source-yotpo/metadata.yaml index 1d821924cbea..56487a95fd70 100644 --- a/airbyte-integrations/connectors/source-yotpo/metadata.yaml +++ b/airbyte-integrations/connectors/source-yotpo/metadata.yaml @@ -18,4 +18,8 @@ data: tags: - language:low-code - language:python -metadataSpecVersion: '1.0' + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-yotpo/requirements.txt b/airbyte-integrations/connectors/source-yotpo/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-yotpo/requirements.txt +++ b/airbyte-integrations/connectors/source-yotpo/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-yotpo/setup.py b/airbyte-integrations/connectors/source-yotpo/setup.py index 4714ade53c35..ed41819fb6d9 100644 --- a/airbyte-integrations/connectors/source-yotpo/setup.py +++ b/airbyte-integrations/connectors/source-yotpo/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-younium/metadata.yaml b/airbyte-integrations/connectors/source-younium/metadata.yaml index 6c20e0ac262e..5eeb118c878d 100644 --- a/airbyte-integrations/connectors/source-younium/metadata.yaml +++ b/airbyte-integrations/connectors/source-younium/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/younium tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-younium/requirements.txt b/airbyte-integrations/connectors/source-younium/requirements.txt index 91de78ac4144..ecf975e2fa63 100644 --- a/airbyte-integrations/connectors/source-younium/requirements.txt +++ b/airbyte-integrations/connectors/source-younium/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/setup.py b/airbyte-integrations/connectors/source-younium/setup.py index 64c602d11bc8..b2a5f3f95a39 100644 --- a/airbyte-integrations/connectors/source-younium/setup.py +++ b/airbyte-integrations/connectors/source-younium/setup.py @@ -10,10 +10,10 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.22.0", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml b/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml index 584f0cdf225c..0562428d329f 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml +++ b/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/youtube-analytics tags: - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-youtube-analytics/requirements.txt b/airbyte-integrations/connectors/source-youtube-analytics/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/requirements.txt +++ b/airbyte-integrations/connectors/source-youtube-analytics/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-youtube-analytics/setup.py b/airbyte-integrations/connectors/source-youtube-analytics/setup.py index 6edf86deac15..795c2044a1ca 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/setup.py +++ b/airbyte-integrations/connectors/source-youtube-analytics/setup.py @@ -13,7 +13,6 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock==1.9.3", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml b/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml index 0e20c0f4696a..d5f9864b86ab 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/metadata.yaml @@ -14,8 +14,12 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-supported-storage + documentationUrl: https://docs.airbyte.com/integrations/sources/zapier-supported-storage tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/requirements.txt b/airbyte-integrations/connectors/source-zapier-supported-storage/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/requirements.txt +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py b/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py index a8f3533de56f..1ffccc828857 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt index f30a97e9c7dd..33c52b2289bf 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt @@ -2,8 +2,8 @@ {"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-01-17T13:20:50Z", "status": "invisible", "duration": 789.733983, "id": "360786799676|2022-01-17T13:20:50Z"}, "emitted_at": 1672828433249} {"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-06-30T17:16:55Z", "status": "invisible", "duration": 61.089883, "id": "360786799676|2022-06-30T17:16:55Z"}, "emitted_at": 1672828433249} {"stream": "agent_timeline", "data": {"agent_id": 360786799676, "engagement_count": 0, "start_time": "2022-10-28T12:43:05Z", "status": "invisible", "duration": 370.793077, "id": "360786799676|2022-10-28T12:43:05Z"}, "emitted_at": 1672828433249} -{"stream": "agents", "data": {"enabled_departments": [2148322881, 2150687801, 2150687841], "last_name": "", "enabled": true, "departments": [2148322881, 2150687801, 2150687841], "role_id": 360002848976, "skills": [1300601], "create_date": "2021-04-23T14:33:11Z", "display_name": "Fake User number - 1", "first_name": "Fake User number - 1", "email": "fake.user-1@email.com", "id": 361084605116, "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1681819085184} -{"stream": "agents", "data": {"first_name": "Fake Agent number - 1", "create_date": "2021-04-23T14:34:20Z", "enabled_departments": [2148322881], "last_name": "", "role_id": 360002848976, "id": 361089721035, "departments": [2148322881, 2148322921], "display_name": "Fake Agent number - 1", "enabled": true, "skills": [1296081, 1300641], "email": "fake.agent-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1672828433502} +{"stream": "agents", "data": {"role_id": 360002848976, "departments": [7282640316815, 7282630247567, 7282624630287, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5059452990735, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5059403825935, 5060108375311, 5059473809295, 5059436284943, 360003074836], "enabled_departments": [7282640316815, 7282630247567, 7282624630287], "last_name": "", "create_date": "2021-04-23T14:33:11Z", "first_name": "Fake User number - 1", "enabled": true, "skills": [1300601, 8565161], "id": 361084605116, "display_name": "Fake User number - 1", "email": "fake.user-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1688547518353} +{"stream": "agents", "data": {"role_id": 360002848976, "departments": [7282630247567, 7282657193103, 5059439464079, 5060105343503, 5060005480847, 5060049125391, 5060061403535, 5060061508879, 5060049288719, 5060049443215, 5060066676751, 5060066798607, 5060071902479, 5060093166863, 5060100872591, 5060101239823, 5060072765583, 5060101350159, 5060077702799, 5060088742799, 5060103345935, 5060078913935, 5060103664783, 5060079026575, 5060055796111, 5060090959759, 5059473603087, 5060108375311, 5059473809295, 5059436114575, 5059404003599, 360003074836], "enabled_departments": [7282630247567], "last_name": "", "create_date": "2021-04-23T14:34:20Z", "first_name": "Fake Agent number - 1", "enabled": true, "skills": [1296081, 1300641], "id": 361089721035, "display_name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "last_login": null, "login_count": 0, "roles": {"administrator": false, "owner": false}}, "emitted_at": 1688547518353} {"stream": "bans", "data": {"type": "visitor", "id": 75411361, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 47225177", "visitor_id": "10414779.13ojzHu7ISdt0SM"}, "emitted_at": 1672828433831} {"stream": "bans", "data": {"type": "visitor", "id": 75411401, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 62959049", "visitor_id": "10414779.13ojzHu7at4VKcG"}, "emitted_at": 1672828433831} {"stream": "bans", "data": {"created_at": "2021-04-27T15:52:32Z", "visitor_id": "10414779.13ojzHu7at4VKcG", "id": 75411401, "reason": "Spammer", "visitor_name": "Visitor 62959049", "type": "visitor"}, "emitted_at": 1672828434000} @@ -12,9 +12,9 @@ {"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "6.42465", "name": "Fake user - chat 117", "email": "fake_user_chat_117@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsbJTqVJsF", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} {"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "8.89712", "name": "Fake user - chat 118", "email": "fake_user_chat_118@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsgcJUJbVN", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} {"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "9.61246", "name": "Fake user - chat 119", "email": "fake_user_chat_119@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WslzhLr3zm", "zendesk_ticket_id": null}, "emitted_at": 1672828434385} -{"stream": "departments", "data": {"description": "A sample department", "members": [360786799676], "settings": {}, "enabled": true, "id": 2148316401, "name": "Department 1"}, "emitted_at": 1672828434639} -{"stream": "departments", "data": {"description": "A sample department 2", "members": [361089721035, 361084605116], "settings": {}, "enabled": true, "id": 2148322881, "name": "Department 2"}, "emitted_at": 1672828434640} -{"stream": "departments", "data": {"description": "A sample department 3", "members": [361089721035], "settings": {}, "enabled": false, "id": 2148322921, "name": "Department 3"}, "emitted_at": 1672828434640} +{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282640316815}, "members": [361084605116], "name": "Airbyte Department 1", "enabled": true, "description": "A sample department", "id": 7282640316815}, "emitted_at": 1688547521914} +{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282618889231}, "members": [360786799676], "name": "Department 1", "enabled": true, "description": "A sample department", "id": 7282618889231}, "emitted_at": 1688547521914} +{"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282630247567}, "members": [361089721035, 361084605116], "name": "Department 2", "enabled": true, "description": "A sample department 2", "id": 7282630247567}, "emitted_at": 1688547521914} {"stream": "goals", "data": {"description": "A new goal", "id": 513481, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal 3", "enabled": true, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434873} {"stream": "goals", "data": {"description": "A new goal - 1", "id": 529641, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal one", "enabled": false, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434874} {"stream": "goals", "data": {"description": "A new goal - 2", "id": 529681, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal two", "enabled": false, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434874} @@ -28,7 +28,7 @@ {"stream": "skills", "data": {"id": 1300601, "name": "english", "enabled": true, "description": "English language", "members": [361084605116]}, "emitted_at": 1672828435627} {"stream": "skills", "data": {"id": 1300641, "name": "france", "enabled": true, "description": "France language", "members": [361089721035]}, "emitted_at": 1672828435628} {"stream": "skills", "data": {"id": 1296081, "name": "mandarin", "enabled": true, "description": "Chinese language", "members": [361089721035]}, "emitted_at": 1672828435628} -{"stream": "triggers", "data": {"definition": {"actions": [["sendMessageToVisitor", "Customer Service", "Hi, are you interested in [insert product name]? We're offering a one-time 20% discount. Chat with me to find out more."]], "version": 1, "event": "chat_requested", "condition": ["and", ["icontains", "@visitor_page_url", "[product name]"], ["stillOnPage", 30], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]]}, "enabled": false, "id": 66052801, "name": "Product Discounts", "description": "Offer your returning customers a discount on one of your products or services. This Trigger will need to be customized based on the page."}, "emitted_at": 1672828435882} -{"stream": "triggers", "data": {"definition": {"actions": [["addTag", "Away_request"], ["sendMessageToVisitor", "Customer Service", "Hi, sorry we are away at the moment. Please leave your email address and we will get back to you as soon as possible."]], "version": 1, "event": "chat_requested", "condition": ["and", ["eq", "@account_status", "away"], ["not", ["firedBefore"]]]}, "enabled": false, "id": 66052841, "name": "Request Contact Details", "description": "When your account is set to away, ask customer's requesting a chat to leave their email address."}, "emitted_at": 1672828435882} -{"stream": "triggers", "data": {"definition": {"actions": [["addTag", "5times"]], "version": 1, "event": "page_enter", "condition": ["and", ["gte", "@visitor_previous_visits", 5]]}, "enabled": false, "id": 66052881, "name": "Tag Repeat Visitors", "description": "Add a tag to a visitor that has visited your site 5 or more times. This helps you identify potential customers who are very interested in your brand."}, "emitted_at": 1672828435883} -{"stream": "routing_settings", "data": {"routing_mode": "broadcast", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": false, "max_wait_time": 30}, "reassignment": {"enabled": false, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}}, "emitted_at": 1672828436146} +{"stream": "triggers", "data": {"name": "Product Discounts", "enabled": true, "description": "Offer your returning customers a discount on one of your products or services. This Trigger will need to be customized based on the page.", "id": 66052801, "definition": {"event": "chat_requested", "condition": ["and", ["icontains", "@visitor_page_url", "[product name]"], ["stillOnPage", 30], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Customer Service", "Hi, are you interested in [insert product name]? We're offering a one-time 20% discount. Chat with me to find out more."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} +{"stream": "triggers", "data": {"name": "Request Contact Details", "enabled": true, "description": "When your account is set to away, ask customer's requesting a chat to leave their email address.", "id": 66052841, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@account_status", "away"], ["not", ["firedBefore"]]], "actions": [["addTag", "Away_request"], ["sendMessageToVisitor", "Customer Service", "Hi, sorry we are away at the moment. Please leave your email address and we will get back to you as soon as possible."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} +{"stream": "triggers", "data": {"name": "Tag Repeat Visitors", "enabled": true, "description": "Add a tag to a visitor that has visited your site 5 or more times. This helps you identify potential customers who are very interested in your brand.", "id": 66052881, "definition": {"event": "page_enter", "condition": ["and", ["gte", "@visitor_previous_visits", 5]], "actions": [["addTag", "5times"]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} +{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}}, "emitted_at": 1688547526146} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 11b3d82a32ee..5d9c006e0853 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -20,4 +20,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt b/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt +++ b/airbyte-integrations/connectors/source-zendesk-chat/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zendesk-chat/setup.py b/airbyte-integrations/connectors/source-zendesk-chat/setup.py index 98f9d779a4f4..8e1732196deb 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-chat/setup.py @@ -7,7 +7,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "requests_mock"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock"] setup( name="source_zendesk_chat", diff --git a/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml index 6fe67444947b..5920a672a86f 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt b/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt +++ b/airbyte-integrations/connectors/source-zendesk-sell/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py index 3031bb9adfea..4f3427a1a107 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile b/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile index d8c346198423..ce27f0f1141e 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/Dockerfile @@ -1,16 +1,28 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base +FROM base as builder -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base WORKDIR /airbyte/integration_code -COPY source_zendesk_sunshine ./source_zendesk_sunshine -COPY main.py ./ COPY setup.py ./ -RUN pip install . +RUN pip install --prefix=/install . + +FROM base +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +WORKDIR /airbyte/integration_code +COPY main.py ./ +COPY source_zendesk_sunshine ./source_zendesk_sunshine + ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-zendesk-sunshine diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/README.md b/airbyte-integrations/connectors/source-zendesk-sunshine/README.md index 04b9cba3ab92..452bbe11f615 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/README.md +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/README.md @@ -1,34 +1,10 @@ # Zendesk Sunshine Source -This is the repository for the Zendesk Sunshine source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-sunshine). +This is the repository for the Zendesk Sunshine configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zendesk-sunshine). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -38,22 +14,14 @@ To build using Gradle, from the Airbyte repository root, run: ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-sunshine) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sunshine/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-sunshine) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sunshine/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-sunshine test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build @@ -78,32 +46,15 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sunshine:dev d docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-sunshine:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests All commands should be run from airbyte project root. diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py b/airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml index 66c733314699..c418358b0f89 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-config.yml @@ -1,35 +1,34 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-zendesk-sunshine:dev -tests: - spec: - - spec_path: "source_zendesk_sunshine/spec.json" +test_strictness_level: low +acceptance_tests: connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "secrets/config_api_token.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/invalid_config_api_token.json" - status: "failed" - - config_path: "integration_tests/invalid_config_oauth.json" - status: "failed" + tests: + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "secrets/config_api_token.json" + status: "succeed" + - config_path: "integration_tests/invalid_config_api_token.json" + status: "failed" + - config_path: "integration_tests/invalid_config_oauth.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config_oauth.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.1" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_api_token.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" -# incremental: # complex state ( {parent_id: {cur_field: value}} still not supported ) -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config_api_token.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + # incremental: # complex state ( {parent_id: {cur_field: value}} still not supported ) + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh index 5797d20fe9a7..b6d65deeccb4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh @@ -1,2 +1,3 @@ #!/usr/bin/env sh + source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml index da9a26dfa363..003390c81a7a 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml @@ -1,20 +1,28 @@ data: + allowedHosts: + hosts: + - ${subdomain}.zendesk.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 325e0640-e7b3-4e24-b823-3361008f603f - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-zendesk-sunshine githubIssueLabel: source-zendesk-sunshine icon: zendesk-sunshine.svg license: MIT name: Zendesk Sunshine - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2021-07-08 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sunshine tags: - - language:python + - language:low-code + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/requirements.txt b/airbyte-integrations/connectors/source-zendesk-sunshine/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/requirements.txt +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py index d09cfec2ebce..1c47ce8ab0a9 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/setup.py @@ -10,8 +10,9 @@ ] TEST_REQUIREMENTS = [ - "pytest~=6.1", - "connector-acceptance-test", + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", + "pytest~=6.2", ] setup( @@ -21,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py index f1f84df11521..d6f54ad6589f 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/__init__.py @@ -1,26 +1,7 @@ -""" -MIT License +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourceZendeskSunshine diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py new file mode 100644 index 000000000000..23a10b7587fb --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/components.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, Mapping + +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator + + +@dataclass +class AuthenticatorZendeskSunshine(DeclarativeAuthenticator): + config: Mapping[str, Any] + basic_auth: BasicHttpAuthenticator + oauth2: BearerAuthenticator + + def __new__(cls, basic_auth, oauth2, config, *args, **kwargs): + if config["credentials"]["auth_method"] == "api_token": + return basic_auth + else: + return oauth2 diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml new file mode 100644 index 000000000000..9e0053abbf68 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml @@ -0,0 +1,312 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - limits + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + type: RequestOption + field_name: per_page + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("links", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("links", {}).get("next", {}) }}' + basic_authenticator: + type: BasicHttpAuthenticator + password: "{{ config['credentials']['api_token'] }}" + username: "{{ config['credentials']['email'] }}/token" + oauth2_authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + requester: + type: HttpRequester + url_base: https://{{ config['subdomain'] }}.zendesk.com/api/sunshine/ + http_method: GET + request_headers: + Content-Type: application/json + authenticator: + class_name: source_zendesk_sunshine.components.AuthenticatorZendeskSunshine + basic_auth: "#/definitions/basic_authenticator" + oauth2: "#/definitions/oauth2_authenticator" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitTimeFromHeader + header: Retry-After + request_body_json: {} + base_stream: + type: DeclarativeStream + primary_key: key + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: "{{ parameters.path }}" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + limits_stream: + $ref: "#/definitions/base_stream" + name: limits + $parameters: + path: limits + relationship_types_stream: + $ref: "#/definitions/base_stream" + name: relationship_types + $parameters: + path: relationships/types + object_types_stream: + $ref: "#/definitions/base_stream" + name: object_types + $parameters: + path: objects/types + object_records_stream: + type: DeclarativeStream + name: object_records + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: objects/query + http_method: POST + request_body_json: + query: + _type: + $eq: "{{ stream_partition.type }}" + sort_by: _updated_at asc + _updated_at: + start: "{{ stream_interval.start_time.strftime('%Y-%m-%d %H:%M:%s.%f')[:-3] }}" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: key + partition_field: type + stream: + $ref: "#/definitions/object_types_stream" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + object_type_policies_stream: + type: DeclarativeStream + name: object_type_policies + primary_key: [] + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: objects/types/{{ stream_partition.type }}/permissions + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: key + partition_field: type + stream: + $ref: "#/definitions/object_types_stream" + transformations: + - type: AddFields + fields: + - path: + - object_type + value: "{{ stream_partition.type }}" + relationship_records_stream: + type: DeclarativeStream + name: relationship_records + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: relationships/records + request_parameters: + type: "{{ stream_partition.type }}" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: key + partition_field: type + stream: + $ref: "#/definitions/relationship_types_stream" + +streams: + - "#/definitions/limits_stream" + - "#/definitions/object_types_stream" + - "#/definitions/object_records_stream" + - "#/definitions/object_type_policies_stream" + - "#/definitions/relationship_types_stream" + - "#/definitions/relationship_records_stream" + +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/zendesk_sunshine + type: Spec + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - start_date + - subdomain + properties: + subdomain: + type: string + order: 0 + title: Subdomain + description: The subdomain for your Zendesk Account. + start_date: + type: string + title: Start date + format: date-time + description: The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + examples: + - "2021-01-01T00:00:00Z" + order: 1 + credentials: + title: Authorization Method + type: object + oneOf: + - type: object + title: OAuth2.0 + required: + - auth_method + - client_id + - client_secret + - access_token + properties: + auth_method: + type: string + const: oauth2.0 + enum: + - oauth2.0 + default: oauth2.0 + order: 0 + client_id: + type: string + title: Client ID + description: The Client ID of your OAuth application. + airbyte_secret: true + client_secret: + type: string + title: Client Secret + description: The Client Secret of your OAuth application. + airbyte_secret: true + access_token: + type: string + title: Access Token + description: Long-term access Token for making authenticated requests. + airbyte_secret: true + - type: object + title: API Token + required: + - auth_method + - api_token + - email + properties: + auth_method: + type: string + const: api_token + enum: + - api_token + default: api_token + order: 1 + api_token: + type: string + title: API Token + description: + API Token. See the docs + for information on how to generate this key. + airbyte_secret: true + email: + type: string + title: Email + description: The user email for your Zendesk account + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - credentials + - auth_method + predicate_value: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + additionalProperties: false + properties: + access_token: + type: string + path_in_connector_config: + - credentials + - access_token + complete_oauth_server_input_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + path_in_connector_config: + - credentials + - client_id + client_secret: + type: string + path_in_connector_config: + - credentials + - client_secret + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + subdomain: + type: string + path_in_connector_config: + - subdomain diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/jobs.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/jobs.json index 9325c5687d8d..bb94de97c423 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/jobs.json @@ -1,20 +1,22 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "job_status": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] }, "completed_at": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/limits.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/limits.json index 14c7b0e353b5..c4583d508122 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/limits.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/limits.json @@ -1,14 +1,16 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "key": { - "type": "string" + "type": ["null", "string"] }, "limit": { - "type": "integer" + "type": ["null", "integer"] }, "count": { - "type": "integer" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_records.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_records.json index ab297c1759de..e7a5270a6821 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_records.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_records.json @@ -1,23 +1,26 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "type": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "external_id": { "type": ["string", "null"] }, "attributes": { - "type": "object" + "type": ["null", "object"], + "additionalProperties": true }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_type_policies.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_type_policies.json index bc5a007ed826..a662de564d17 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_type_policies.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_type_policies.json @@ -1,60 +1,66 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "object_type": { - "type": "string" + "type": ["null", "string"] }, "rbac": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "admin": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "create": { - "type": "boolean" + "type": ["null", "boolean"] }, "read": { - "type": "boolean" + "type": ["null", "boolean"] }, "update": { - "type": "boolean" + "type": ["null", "boolean"] }, "delete": { - "type": "boolean" + "type": ["null", "boolean"] } } }, "agent": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "create": { - "type": "boolean" + "type": ["null", "boolean"] }, "read": { - "type": "boolean" + "type": ["null", "boolean"] }, "update": { - "type": "boolean" + "type": ["null", "boolean"] }, "delete": { - "type": "boolean" + "type": ["null", "boolean"] } } }, "end_user": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "create": { - "type": "boolean" + "type": ["null", "boolean"] }, "read": { - "type": "boolean" + "type": ["null", "boolean"] }, "update": { - "type": "boolean" + "type": ["null", "boolean"] }, "delete": { - "type": "boolean" + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_types.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_types.json index 5414d5ba1550..4a331703b94f 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_types.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/object_types.json @@ -1,31 +1,32 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "key": { - "type": "string" + "type": ["null", "string"] }, "schema": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "properties": { - "type": "object" + "type": ["null", "object"], + "additionalProperties": true }, "required": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "string" + "type": ["null", "string"] } - }, - "additionalProperties": { - "type": "boolean" } } }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_records.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_records.json index 621db91cdcff..b834e5cda0cc 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_records.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_records.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_types.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_types.json index 6c5ae84b3a89..08c8027fcdc9 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_types.json +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/schemas/relationship_types.json @@ -1,20 +1,22 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "key": { - "type": "string" + "type": ["null", "string"] }, "source": { - "type": "string" + "type": ["null", "string"] }, "target": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py index c848aa744881..aac70fbfffcb 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/source.py @@ -2,73 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import base64 -from typing import Any, List, Mapping, Tuple, Union +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -from .streams import Limits, ObjectRecords, ObjectTypePolicies, ObjectTypes, RelationshipRecords, RelationshipTypes - -class Base64HttpAuthenticator(TokenAuthenticator): - def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): - auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") - b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) - - -class ZendeskSunshineAuthenticator: - """Provides the authentication capabilities for both old and new methods.""" - - @staticmethod - def get_auth(config: Mapping[str, Any]) -> Union[Base64HttpAuthenticator, TokenAuthenticator]: - credentials = config.get("credentials", {}) - token = config.get("api_token") or credentials.get("api_token") - email = config.get("email") or credentials.get("email") - if email and token: - return Base64HttpAuthenticator(auth=(f"{email}/token", token)) - return TokenAuthenticator(token=credentials["access_token"]) - - -class SourceZendeskSunshine(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - try: - pendulum.parse(config["start_date"], strict=True) - authenticator = ZendeskSunshineAuthenticator.get_auth(config) - stream = Limits(authenticator=authenticator, subdomain=config["subdomain"], start_date=pendulum.parse(config["start_date"])) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as e: - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - CustomObjectEvents stream is an early access stream. (looks like it is a new feature) - It requires activation in site ui + manual activation from Zendesk via call. - I requested the call, but since they did not approve it, - this endpoint will return 403 Forbidden. Thats why it is disabled here. - - Jobs stream is also commented out. Reason: It is dynamic. - It can have the data, but this data have time to live. - After this time is passed we have no data. It will require permanent population, to pass - the test criteria `stream should contain at least 1 record) - """ - authenticator = ZendeskSunshineAuthenticator.get_auth(config) - args = {"authenticator": authenticator, "subdomain": config["subdomain"], "start_date": config["start_date"]} - return [ - ObjectTypes(**args), - ObjectRecords(**args), - RelationshipTypes(**args), - RelationshipRecords(**args), - # CustomObjectEvents(**args), - ObjectTypePolicies(**args), - # Jobs(**args), - Limits(**args), - ] +# Declarative Source +class SourceZendeskSunshine(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json deleted file mode 100644 index 29e900a4b1a5..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/spec.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/zendesk_sunshine", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zendesk Sunshine Spec", - "type": "object", - "required": ["start_date", "subdomain"], - "additionalProperties": true, - "properties": { - "subdomain": { - "title": "Subdomain", - "type": "string", - "description": "The subdomain for your Zendesk Account." - }, - "start_date": { - "title": "Start Date", - "type": "string", - "description": "The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2021-01-01T00:00:00Z"] - }, - "credentials": { - "title": "Authorization Method", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "auth_method", - "client_id", - "client_secret", - "access_token" - ], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "enum": ["oauth2.0"], - "default": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Long-term access Token for making authenticated requests.", - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API Token", - "required": ["auth_method", "api_token", "email"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_token", - "enum": ["api_token"], - "default": "api_token", - "order": 1 - }, - "api_token": { - "type": "string", - "title": "API Token", - "description": "API Token. See the docs for information on how to generate this key.", - "airbyte_secret": true - }, - "email": { - "type": "string", - "title": "Email", - "description": "The user email for your Zendesk account" - } - } - } - ] - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_method"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - }, - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "subdomain": { - "type": "string", - "path_in_connector_config": ["subdomain"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py deleted file mode 100644 index b68ee15d8c58..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/streams.py +++ /dev/null @@ -1,212 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import urllib.parse -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -class SunshineStream(HttpStream, ABC): - primary_key = "id" - data_field = "data" - page_size = 100 - - def __init__(self, subdomain: str, start_date: pendulum.datetime, **kwargs): - self._start_date = start_date - self.subdomain = subdomain - super().__init__(**kwargs) - - @property - def url_base(self) -> str: - return f"https://{self.subdomain}.zendesk.com/api/sunshine/" - - def backoff_time(self, response: requests.Response) -> Optional[float]: - delay_time = response.headers.get("Retry-After") - if delay_time: - return float(delay_time) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - resp_json = response.json() - if resp_json.get("links") and resp_json.get("links").get("next"): - next_query_string = urllib.parse.urlsplit(resp_json.get("links").get("next")).query - params = dict(urllib.parse.parse_qsl(next_query_string)) - return params - return {} - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Content-Type": "application/json"} - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - """ - The response data field is mostly a list of objects. Sometimes we can have object in data field. - (example `ObjectTypePolicies`). In this case this method should be overridden. - """ - response_json = response.json() - yield from response_json.get(self.data_field, []) - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = {"per_page": self.page_size} - if next_page_token: - params.update(next_page_token) - return params - - -class IncrementalSunshineStream(SunshineStream, ABC): - state_checkpoint_interval = 1000 - cursor_field = "updated_at" # most common - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_state = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(self.cursor_field) or latest_state - # dates are ISO-formatted, no need to parse - return {self.cursor_field: max(latest_state, current_state)} - - -class ObjectTypes(SunshineStream): - primary_key = "key" - - def path(self, **kwargs) -> str: - return "objects/types" - - -class ObjectRecords(IncrementalSunshineStream): - """ - The get method supports only the full-refresh way to get the information fron this source. - This source has date fields in all the endpoints, but we cannot query this field during GET requests. - To support Incremental for this stream I had to use `query` endpoint instead of `objects/records` - - this allows me to use date filters. This is the only way to have incremental support. - """ - - http_method = "POST" - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: - type_ = stream_slice["type"] - state_value = stream_state.get(type_, {}).get(self.cursor_field) - start_date = state_value or self._start_date - formatted_start_date = pendulum.parse(start_date).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - query = { - "query": {"_type": {"$eq": type_}}, - "_updated_at": { - "start": formatted_start_date, - }, - "sort_by": "_updated_at asc", - } - return query - - def path(self, **kwargs) -> str: - return "objects/query" - - def stream_slices(self, **kwargs): - parent_stream = ObjectTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date) - for obj_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"type": obj_type["key"]} - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - type_ = latest_record.get("type") - latest_cursor_value = latest_record.get(self.cursor_field) - current_stream_state = current_stream_state or {} - current_state = current_stream_state.get(type_) if current_stream_state else None - if current_state: - current_state = current_state.get(self.cursor_field) - current_state_value = current_state or latest_cursor_value - max_value = max(current_state_value, latest_cursor_value) - new_value = {self.cursor_field: max_value} - - current_stream_state[type_] = new_value - return current_stream_state - - -class RelationshipTypes(SunshineStream): - primary_key = "key" - - def path(self, **kwargs) -> str: - return "relationships/types" - - -class RelationshipRecords(SunshineStream): - def path(self, **kwargs) -> str: - return "relationships/records" - - def stream_slices(self, **kwargs): - parent_stream = RelationshipTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date) - for rel_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"type": rel_type["key"]} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - type_ = stream_slice["type"] - params["type"] = type_ - return params - - -class CustomObjectEvents(SunshineStream): - """ - This stream is early access stream. (look like a new feature) - It requires activation in site ui + manual activation from Zendesk via call. - I requested the call, but since they did not approve it, - this endpoint will return 403 Forbidden - """ - - def path(self, **kwargs) -> str: - return "objects/events" - - -class ObjectTypePolicies(SunshineStream): - primary_key = None - - def stream_slices(self, **kwargs): - parent_stream = ObjectTypes(authenticator=self.authenticator, subdomain=self.subdomain, start_date=self._start_date) - for obj_type in parent_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"type": obj_type["key"]} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - obj_type = stream_slice["type"] - return f"objects/types/{obj_type}/permissions" - - def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping]: - response_json = response.json() - data = response_json.get(self.data_field, {}) - # the response does not contain info about parent itself - only rules. Need to add this. - data["object_type"] = stream_slice["type"] - yield data - - -class Jobs(SunshineStream): - """ - This stream is dynamic. The data can exist today, but may be absent tomorrow. - Since we need to have some data in the stream this stream is disabled. - """ - - def path(self, **kwargs) -> str: - return "jobs" - - -class Limits(SunshineStream): - primary_key = "key" - - def path(self, **kwargs) -> str: - return "limits" diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile index 568932774988..35c4c32298d4 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile @@ -25,5 +25,5 @@ COPY source_zendesk_support ./source_zendesk_support ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.1 -LABEL io.airbyte.name=airbyte/source-zendesk-support +LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.name=airbyte/source-zendesk-support \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-support/README.md b/airbyte-integrations/connectors/source-zendesk-support/README.md index 93eeea0bafa7..9d259a28a0e2 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/README.md +++ b/airbyte-integrations/connectors/source-zendesk-support/README.md @@ -129,4 +129,4 @@ You've checked out the repo, implemented a million dollar feature, and you're re 1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). 1. Create a Pull Request. 1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml index ba77323f1501..02fbf382affb 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml @@ -17,10 +17,15 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.10.6" - config_path: "secrets/config_oauth.json" + backward_compatibility_tests_config: + disable_for_version: "0.10.6" basic_read: tests: - config_path: "secrets/config.json" + timeout_seconds: 2400 expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no @@ -28,9 +33,12 @@ acceptance_tests: extra_records: yes fail_on_extra_columns: false empty_streams: - # This stream is only available for enterprise accounts https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/ - - name: "audit_logs" - bypass_reason: "no records" + - name: "post_comments" + bypass_reason: "not available in current subscription plan" + - name: "post_votes" + bypass_reason: "not available in current subscription plan" + - name: "post_comment_votes" + bypass_reason: "not available in current subscription plan" incremental: tests: - config_path: "secrets/config.json" @@ -39,7 +47,9 @@ acceptance_tests: future_state_path: "integration_tests/abnormal_state.json" cursor_paths: ticket_comments: ["created_at"] + threshold_days: 100 full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 2400 diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json index 20d8b3c729e1..1f2f9af814dd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json @@ -30,7 +30,7 @@ { "type": "STREAM", "stream": { - "stream_state": { "updated_at": "2222-07-20T10:05:18Z" }, + "stream_state": { "generated_timestamp": 8001820514 }, "stream_descriptor": { "name": "tickets" } } }, @@ -96,5 +96,33 @@ "stream_state": { "created_at": "2222-07-19T22:21:26Z" }, "stream_descriptor": { "name": "audit_logs" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-12-11T19:34:06Z" }, + "stream_descriptor": { "name": "posts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-12-11T19:34:06Z" }, + "stream_descriptor": { "name": "topics" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "organization_memberships" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "ticket_skips" } + } } ] diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json index 3762b4850acb..0354f87101be 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json @@ -164,6 +164,18 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, + { + "stream": { + "name": "ticket_skips", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, { "stream": { "name": "ticket_metric_events", @@ -199,6 +211,96 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "topics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "post_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "post_votes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "post_comment_votes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "organization_memberships", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_attributes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "attribute_definitions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index 982e66615476..5931bfa47ac0 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -1,45 +1,62 @@ -{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360007820916.json", "id": 360007820916, "user_id": 360786799676, "group_id": 360003074836, "default": true, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z"}, "emitted_at": 1682939863906} -{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011727976.json", "id": 360011727976, "user_id": 361084605116, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:33:11Z", "updated_at": "2021-04-23T14:33:11Z"}, "emitted_at": 1682939863906} -{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011812655.json", "id": 360011812655, "user_id": 361089721035, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:34:20Z", "updated_at": "2021-04-23T14:34:20Z"}, "emitted_at": 1682939863907} -{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/5059439464079.json", "id": 5059439464079, "is_public": true, "name": "Group 1", "description": "", "default": false, "deleted": false, "created_at": "2022-06-29T12:29:26Z", "updated_at": "2022-06-29T12:29:26Z"}, "emitted_at": 1682939866081} -{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/5059474192015.json", "id": 5059474192015, "is_public": true, "name": "Group 10", "description": "", "default": false, "deleted": false, "created_at": "2022-06-29T12:30:58Z", "updated_at": "2022-06-29T12:30:58Z"}, "emitted_at": 1682939866082} -{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/5060105343503.json", "id": 5060105343503, "is_public": true, "name": "Group 100", "description": "", "default": false, "deleted": false, "created_at": "2022-06-29T16:22:26Z", "updated_at": "2022-06-29T16:22:26Z"}, "emitted_at": 1682939866082} -{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363556.json", "id": 360011363556, "title": "Customer not responding", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "status", "value": "pending"}, {"field": "comment_value", "value": "Hello {{ticket.requester.name}}. Our agent {{current_user.name}} has tried to contact you about this request but we haven't heard back from you yet. Please let us know if we can be of further assistance. Thanks. "}], "restriction": null, "raw_title": "Customer not responding"}, "emitted_at": 1682939868108} -{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363536.json", "id": 360011363536, "title": "Downgrade and inform", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "priority", "value": "low"}, {"field": "comment_value", "value": "We're currently experiencing unusually high traffic. We'll get back to you as soon as possible."}], "restriction": null, "raw_title": "Downgrade and inform"}, "emitted_at": 1682939868109} -{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360033549136.json", "id": 360033549136, "name": "Airbyte", "shared_tickets": true, "shared_comments": true, "external_id": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2023-04-13T14:51:21Z", "domain_names": ["cloud.airbyte.com"], "details": "test", "notes": "test", "group_id": 6770788212111, "tags": ["test"], "organization_fields": {}}, "emitted_at": 1682939870308} -{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360045373216.json", "id": 360045373216, "name": "ressssssss", "shared_tickets": false, "shared_comments": false, "external_id": null, "created_at": "2021-07-15T18:29:14Z", "updated_at": "2021-07-15T18:29:14Z", "domain_names": [], "details": "", "notes": "", "group_id": null, "tags": [], "organization_fields": {}}, "emitted_at": 1682939870310} -{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/4992997209743.json", "id": 4992997209743, "assignee_id": null, "group_id": null, "requester_id": 4992781783439, "ticket_id": 121, "score": "offered", "created_at": "2022-06-17T16:01:42Z", "updated_at": "2022-06-17T16:01:42Z", "comment": null}, "emitted_at": 1682939872220} -{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/4993646311567.json", "id": 4993646311567, "assignee_id": null, "group_id": null, "requester_id": 4993467856015, "ticket_id": 122, "score": "offered", "created_at": "2022-06-17T21:01:41Z", "updated_at": "2022-06-17T21:01:41Z", "comment": null}, "emitted_at": 1682939872221} -{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5138125924367.json", "id": 5138125924367, "assignee_id": null, "group_id": null, "requester_id": 5137812260495, "ticket_id": 123, "score": "offered", "created_at": "2022-07-13T16:02:03Z", "updated_at": "2022-07-13T16:02:03Z", "comment": null}, "emitted_at": 1682939872221} -{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001110696.json", "id": 360001110696, "title": "test police", "description": "for tests", "position": 1, "filter": {"all": [{"field": "assignee_id", "operator": "is", "value": 361089721035}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 61, "business_hours": false}], "created_at": "2021-07-16T11:05:31Z", "updated_at": "2021-07-16T11:05:31Z"}, "emitted_at": 1682939873261} -{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001113715.json", "id": 360001113715, "title": "test police 2", "description": "test police 2", "position": 2, "filter": {"all": [{"field": "organization_id", "operator": "is", "value": 360033549136}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 121, "business_hours": false}], "created_at": "2021-07-16T11:06:01Z", "updated_at": "2021-07-16T11:06:01Z"}, "emitted_at": 1682939873262} -{"stream": "tags", "data": {"name": "test", "count": 2}, "emitted_at": 1682939874989} -{"stream": "tags", "data": {"name": "tag1204", "count": 1}, "emitted_at": 1682939874990} -{"stream": "ticket_audits", "data": {"id": 6764092975119, "ticket_id": 145, "created_at": "2023-04-12T14:04:56Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.34", "ip_address": "85.209.47.207", "location": "Kyiv, 30, Ukraine", "latitude": 50.458, "longitude": 30.5303}, "custom": {}}, "events": [{"id": 6764092975247, "type": "Change", "value": "low", "field_name": "priority", "previous_value": null}, {"id": 6764092975375, "type": "Change", "value": ["tag1204", "test"], "field_name": "tags", "previous_value": "[]"}, {"id": 6764092975503, "type": "Change", "value": "question", "field_name": "type", "previous_value": null}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1682939882143} -{"stream": "ticket_audits", "data": {"id": 5909514817039, "ticket_id": 25, "created_at": "2022-11-22T17:02:04Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 5909514817167, "type": "Notification", "subject": "Request #{{ticket.id}}: How would you rate the support you received?", "body": "Hello {{ticket.requester.name}},\n\nWe'd love to hear what you think of our customer service. Please take a moment to answer one simple question by clicking either link below:\n\n{{satisfaction.rating_section}}\n\nHere's a reminder of what this request was about:\n\n{{ticket.comments_formatted}}\n", "recipients": [360786799676]}, {"id": 5909514817295, "type": "Change", "value": "offered", "field_name": "satisfaction_score", "previous_value": "unoffered"}], "via": {"channel": "rule", "source": {"to": {}, "from": {"deleted": false, "title": "Request customer satisfaction rating (system automation)", "id": 360021281435}, "rel": "automation"}}}, "emitted_at": 1682939882145} -{"stream": "ticket_audits", "data": {"id": 5909508365711, "ticket_id": 141, "created_at": "2022-11-22T17:02:04Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 5909502407183, "type": "Notification", "subject": "Request #{{ticket.id}}: How would you rate the support you received?", "body": "Hello {{ticket.requester.name}},\n\nWe'd love to hear what you think of our customer service. Please take a moment to answer one simple question by clicking either link below:\n\n{{satisfaction.rating_section}}\n\nHere's a reminder of what this request was about:\n\n{{ticket.comments_formatted}}\n", "recipients": [360786799676]}], "via": {"channel": "rule", "source": {"to": {}, "from": {"deleted": false, "title": "Request customer satisfaction rating (system automation)", "id": 360021281435}, "rel": "automation"}}}, "emitted_at": 1682939882146} -{"stream": "ticket_comments", "data": {"id": 5162146653071, "via": {"channel": "web", "source": {"from": {}, "to": {"name": "Team Airbyte", "address": "integration-test@airbyte.io"}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": " 163748", "html_body": "
     163748
    ", "plain_body": " 163748", "public": true, "attachments": [], "audit_id": 5162146652943, "created_at": "2022-07-18T09:58:23Z", "event_type": "Comment", "ticket_id": 124, "timestamp": 1658138303}, "emitted_at": 1682939885008} -{"stream": "ticket_comments", "data": {"id": 5162208963983, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "238473846", "html_body": "
    238473846
    ", "plain_body": "238473846", "public": false, "attachments": [], "audit_id": 5162208963855, "created_at": "2022-07-18T10:16:53Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139413}, "emitted_at": 1682939885009} -{"stream": "ticket_comments", "data": {"id": 5162223308559, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "Airbyte", "html_body": "
    Airbyte
    ", "plain_body": "Airbyte", "public": false, "attachments": [], "audit_id": 5162223308431, "created_at": "2022-07-18T10:25:21Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139921}, "emitted_at": 1682939885009} -{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833076.json", "id": 360002833076, "type": "subject", "title": "Subject", "raw_title": "Subject", "description": "", "raw_description": "", "position": 1, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Subject", "raw_title_in_portal": "Subject", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1682939886459} -{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json", "id": 360002833096, "type": "description", "title": "Description", "raw_title": "Description", "description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "raw_description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "position": 2, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Description", "raw_title_in_portal": "Description", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1682939886461} -{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1682939886462} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/5527084754447.json", "id": 5527084754447, "ticket_id": 145, "created_at": "2022-09-19T14:53:49Z", "updated_at": "2022-09-19T14:53:49Z", "group_stations": 0, "assignee_stations": 0, "reopens": 0, "replies": 0, "assignee_updated_at": null, "requester_updated_at": "2022-09-19T14:53:49Z", "status_updated_at": "2022-09-19T14:53:49Z", "initially_assigned_at": null, "assigned_at": null, "solved_at": null, "latest_comment_added_at": "2022-09-19T14:53:49Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}}, "emitted_at": 1682939888430} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/5527080505487.json", "id": 5527080505487, "ticket_id": 144, "created_at": "2022-09-19T14:43:42Z", "updated_at": "2022-09-19T14:43:42Z", "group_stations": 0, "assignee_stations": 0, "reopens": 0, "replies": 0, "assignee_updated_at": null, "requester_updated_at": "2022-09-19T14:43:42Z", "status_updated_at": "2022-09-19T14:43:42Z", "initially_assigned_at": null, "assigned_at": null, "solved_at": null, "latest_comment_added_at": "2022-09-19T14:43:42Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}}, "emitted_at": 1682939888431} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/5455947694223.json", "id": 5455947694223, "ticket_id": 143, "created_at": "2022-09-06T19:51:07Z", "updated_at": "2022-09-06T19:51:07Z", "group_stations": 0, "assignee_stations": 0, "reopens": 0, "replies": 0, "assignee_updated_at": null, "requester_updated_at": "2022-09-06T19:51:07Z", "status_updated_at": "2022-09-06T19:51:07Z", "initially_assigned_at": null, "assigned_at": null, "solved_at": null, "latest_comment_added_at": "2022-09-06T19:51:07Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}}, "emitted_at": 1682939888431} -{"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1682939890408} -{"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1682939890408} -{"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1682939890409} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655481702}, "emitted_at": 1682939892117} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/122.json", "id": 122, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (912) 420-0314", "phone": "+19124200314", "name": "Caller +1 (912) 420-0314"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T19:52:39Z", "updated_at": "2022-06-17T21:01:41Z", "type": null, "subject": "Voicemail from: Caller +1 (912) 420-0314", "raw_subject": "Voicemail from: Caller +1 (912) 420-0314", "description": "Call from: +1 (912) 420-0314\\nTime of call: June 17, 2022 at 7:52:02 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4993467856015, "submitter_id": 4993467856015, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655499701}, "emitted_at": 1682939892120} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/123.json", "id": 123, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (607) 210-9549", "phone": "+16072109549", "name": "Caller +1 (607) 210-9549"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-07-13T14:34:05Z", "updated_at": "2022-07-13T16:02:03Z", "type": null, "subject": "Voicemail from: Caller +1 (607) 210-9549", "raw_subject": "Voicemail from: Caller +1 (607) 210-9549", "description": "Call from: +1 (607) 210-9549\\nTime of call: July 13, 2022 at 2:33:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 5137812260495, "submitter_id": 5137812260495, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1657728123}, "emitted_at": 1682939892122} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1658140562}, "emitted_at": 1682939892123} -{"stream": "users", "data": {"id": 4992781783439, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json", "name": "Caller +1 (689) 689-8023", "email": null, "created_at": "2022-06-17T14:49:19Z", "updated_at": "2022-06-17T14:49:19Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16896898023", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1682939896920} -{"stream": "users", "data": {"id": 4993467856015, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json", "name": "Caller +1 (912) 420-0314", "email": null, "created_at": "2022-06-17T19:52:38Z", "updated_at": "2022-06-17T19:52:38Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+19124200314", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1682939896922} -{"stream": "users", "data": {"id": 361084605116, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/361084605116.json", "name": "Fake User number - 1", "email": "fake.user-1@email.com", "created_at": "2021-04-23T14:33:11Z", "updated_at": "2022-06-29T16:29:29Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": null, "shared_phone_number": null, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "agent", "verified": false, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": null, "signature": null, "details": null, "notes": null, "role_type": 3, "custom_role_id": 360000210576, "moderator": false, "ticket_restriction": "groups", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": 360003074836, "report_csv": false, "user_fields": {}}, "emitted_at": 1682939896922} -{"stream": "users", "data": {"id": 361089721035, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/361089721035.json", "name": "Fake Agent number - 1", "email": "fake.agent-1@email.com", "created_at": "2021-04-23T14:34:20Z", "updated_at": "2022-06-29T16:29:29Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": null, "shared_phone_number": null, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "agent", "verified": false, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": null, "signature": null, "details": null, "notes": null, "role_type": 0, "custom_role_id": 360000210616, "moderator": false, "ticket_restriction": null, "only_private_comments": false, "restricted_agent": false, "suspended": false, "default_group_id": 360003074836, "report_csv": false, "user_fields": {}}, "emitted_at": 1682939896923} -{"stream": "brands", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/brands/360000358316.json", "id": 360000358316, "name": "Airbyte", "brand_url": "https://d3v-airbyte.zendesk.com", "subdomain": "d3v-airbyte", "host_mapping": null, "has_help_center": false, "help_center_state": "disabled", "active": true, "default": true, "is_deleted": false, "logo": null, "ticket_form_ids": [360000084116], "signature_template": "{{agent.signature}}", "created_at": "2020-12-11T18:34:04Z", "updated_at": "2020-12-11T18:34:09Z"}, "emitted_at": 1682939898371} -{"stream": "custom_roles", "data": {"id": 360000210636, "name": "Advisor", "description": "Can automate ticket workflows, manage channels and make private comments on tickets", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": false, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "none", "ticket_deletion": false, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "full", "report_access": "none", "ticket_editing": true, "ticket_merge": false, "user_view_access": "full", "view_access": "full", "voice_dashboard_access": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_automations": true, "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_skills": true, "manage_slas": true, "manage_triggers": true, "manage_roles": "none"}, "team_member_count": 0}, "emitted_at": 1682939900972} -{"stream": "custom_roles", "data": {"id": 360000210596, "name": "Staff", "description": "Can edit tickets within their groups", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": false, "manage_dynamic_content": false, "manage_extensions_and_channels": false, "manage_facebook": false, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "public", "ticket_deletion": false, "ticket_tag_editing": false, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "manage-personal", "report_access": "readonly", "ticket_editing": true, "ticket_merge": false, "user_view_access": "manage-personal", "view_access": "manage-personal", "voice_dashboard_access": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_automations": false, "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_skills": false, "manage_slas": false, "manage_triggers": false, "manage_roles": "none"}, "team_member_count": 0}, "emitted_at": 1682939900973} -{"stream": "schedules", "data": {"id": 4567312249615, "name": "Test Schedule", "time_zone": "New Caledonia", "created_at": "2022-03-25T10:23:34Z", "updated_at": "2022-03-25T10:23:34Z", "intervals": [{"start_time": 1980, "end_time": 2460}, {"start_time": 3420, "end_time": 3900}, {"start_time": 4860, "end_time": 5340}, {"start_time": 6300, "end_time": 6780}, {"start_time": 7740, "end_time": 8220}]}, "emitted_at": 1682939902170} -{"stream": "ticket_forms", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_forms/360000084116.json", "name": "Default Ticket Form", "display_name": "Default Ticket Form", "id": 360000084116, "raw_name": "Default Ticket Form", "raw_display_name": "Default Ticket Form", "end_user_visible": true, "position": 1, "ticket_field_ids": [360002833076, 360002833096, 360002833116, 360002833136, 360002833156, 360002833176, 360002833196], "active": true, "default": true, "created_at": "2020-12-11T18:34:37Z", "updated_at": "2020-12-11T18:34:37Z", "in_all_brands": true, "restricted_brand_ids": [], "end_user_conditions": [], "agent_conditions": []}, "emitted_at": 1682939903018} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7502393054223.json", "id": 7502393054223, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "109.86.166.58", "created_at": "2023-07-24T10:56:28Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150345} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7465455408271.json", "id": 7465455408271, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "109.86.166.58", "created_at": "2023-07-21T08:03:28Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150346} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7453133196303.json", "id": 7453133196303, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "136.24.229.166", "created_at": "2023-07-19T19:09:32Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150346} +{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360007820916.json", "id": 360007820916, "user_id": 360786799676, "group_id": 360003074836, "default": true, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z"}, "emitted_at": 1690888151470} +{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011727976.json", "id": 360011727976, "user_id": 361084605116, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:33:11Z", "updated_at": "2021-04-23T14:33:11Z"}, "emitted_at": 1690888151471} +{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011812655.json", "id": 360011812655, "user_id": 361089721035, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:34:20Z", "updated_at": "2021-04-23T14:34:20Z"}, "emitted_at": 1690888151471} +{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282640316815.json", "id": 7282640316815, "is_public": true, "name": "Airbyte Department 1", "description": "A sample department", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:12Z", "updated_at": "2023-06-26T10:09:12Z"}, "emitted_at": 1690888152597} +{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282618889231.json", "id": 7282618889231, "is_public": true, "name": "Department 1", "description": "A sample department", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:14Z", "updated_at": "2023-06-26T10:09:14Z"}, "emitted_at": 1690888152598} +{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282630247567.json", "id": 7282630247567, "is_public": true, "name": "Department 2", "description": "A sample department 2", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:14Z", "updated_at": "2023-06-26T10:09:14Z"}, "emitted_at": 1690888152598} +{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363556.json", "id": 360011363556, "title": "Customer not responding", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "status", "value": "pending"}, {"field": "comment_value", "value": "Hello {{ticket.requester.name}}. Our agent {{current_user.name}} has tried to contact you about this request but we haven't heard back from you yet. Please let us know if we can be of further assistance. Thanks. "}], "restriction": null, "raw_title": "Customer not responding"}, "emitted_at": 1690888153534} +{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363536.json", "id": 360011363536, "title": "Downgrade and inform", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "priority", "value": "low"}, {"field": "comment_value", "value": "We're currently experiencing unusually high traffic. We'll get back to you as soon as possible."}], "restriction": null, "raw_title": "Downgrade and inform"}, "emitted_at": 1690888153535} +{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360033549136.json", "id": 360033549136, "name": "Airbyte", "shared_tickets": true, "shared_comments": true, "external_id": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2023-04-13T14:51:21Z", "domain_names": ["cloud.airbyte.com"], "details": "test", "notes": "test", "group_id": 6770788212111, "tags": ["test"], "organization_fields": {"test_check_box_field_1": false, "test_drop_down_field_1": null, "test_number_field_1": null}}, "emitted_at": 1690888154541} +{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360045373216.json", "id": 360045373216, "name": "ressssssss", "shared_tickets": false, "shared_comments": false, "external_id": null, "created_at": "2021-07-15T18:29:14Z", "updated_at": "2021-07-15T18:29:14Z", "domain_names": [], "details": "", "notes": "", "group_id": null, "tags": [], "organization_fields": {"test_check_box_field_1": false, "test_drop_down_field_1": null, "test_number_field_1": null}}, "emitted_at": 1690888154543} +{"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/360057705196.json", "id": 360057705196, "user_id": 360786799676, "organization_id": 360033549136, "default": true, "created_at": "2020-12-11T18:34:05Z", "organization_name": "Airbyte", "updated_at": "2020-12-11T18:34:05Z", "view_tickets": true}, "emitted_at": 1690888156003} +{"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/7282880134671.json", "id": 7282880134671, "user_id": 7282634891791, "organization_id": 360033549136, "default": true, "created_at": "2023-06-26T11:03:38Z", "organization_name": "Airbyte", "updated_at": "2023-06-26T11:03:38Z", "view_tickets": true}, "emitted_at": 1690888156004} +{"stream": "posts", "data": {"id": 7253351904271, "title": "How do I get around the community?", "details": "

    You can use search to find answers. You can also browse topics and posts using views and filters. See Getting around the community.

    ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253351904271-How-do-I-get-around-the-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271-How-do-I-get-around-the-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1692700567985} +{"stream": "posts", "data": {"id": 7253375870607, "title": "Which topics should I add to my community?", "details": "

    That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.

    A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.

    \n\n

    To create your own topics, see Adding community discussion topics.

    ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1692700567986} +{"stream": "posts", "data": {"id": 7253375879055, "title": "I'd like a way for users to submit feature requests", "details": "

    You can add a topic like this one in your community. End users can add feature requests and describe their use cases. Other users can comment on the requests and vote for them. Product managers can review feature requests and provide feedback.

    ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253394974479, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1692700567986} +{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/7235633102607.json", "id": 7235633102607, "assignee_id": null, "group_id": null, "requester_id": 361089721035, "ticket_id": 146, "score": "offered", "created_at": "2023-06-19T18:01:40Z", "updated_at": "2023-06-19T18:01:40Z", "comment": null}, "emitted_at": 1690888165601} +{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5909514818319.json", "id": 5909514818319, "assignee_id": null, "group_id": null, "requester_id": 360786799676, "ticket_id": 25, "score": "offered", "created_at": "2022-11-22T17:02:04Z", "updated_at": "2022-11-22T17:02:04Z", "comment": null}, "emitted_at": 1690888165602} +{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5527212710799.json", "id": 5527212710799, "assignee_id": null, "group_id": null, "requester_id": 5527080499599, "ticket_id": 144, "score": "offered", "created_at": "2022-09-19T16:01:43Z", "updated_at": "2022-09-19T16:01:43Z", "comment": null}, "emitted_at": 1690888165602} +{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001110696.json", "id": 360001110696, "title": "test police", "description": "for tests", "position": 1, "filter": {"all": [{"field": "assignee_id", "operator": "is", "value": 361089721035}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 61, "business_hours": false}], "created_at": "2021-07-16T11:05:31Z", "updated_at": "2021-07-16T11:05:31Z"}, "emitted_at": 1690888166730} +{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001113715.json", "id": 360001113715, "title": "test police 2", "description": "test police 2", "position": 2, "filter": {"all": [{"field": "organization_id", "operator": "is", "value": 360033549136}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 121, "business_hours": false}], "created_at": "2021-07-16T11:06:01Z", "updated_at": "2021-07-16T11:06:01Z"}, "emitted_at": 1690888166731} +{"stream": "tags", "data": {"name": "test", "count": 6}, "emitted_at": 1690888168471} +{"stream": "tags", "data": {"name": "tag2", "count": 3}, "emitted_at": 1690888168472} +{"stream": "tags", "data": {"name": "tag1", "count": 2}, "emitted_at": 1690888168472} +{"stream": "ticket_audits", "data": {"id": 7429253845903, "ticket_id": 152, "created_at": "2023-07-16T12:01:39Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 7429253846031, "type": "Change", "value": "closed", "field_name": "status", "previous_value": "solved"}], "via": {"channel": "rule", "source": {"to": {}, "from": {"deleted": false, "title": "Close ticket 4 days after status is set to solved", "id": 6241378811151}, "rel": "automation"}}}, "emitted_at": 1690888174095} +{"stream": "ticket_audits", "data": {"id": 7283194465039, "ticket_id": 141, "created_at": "2023-06-26T12:15:34Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58", "ip_address": "85.209.47.207", "location": "Kyiv, 30, Ukraine", "latitude": 50.458, "longitude": 30.5303}, "custom": {}}, "events": [{"id": 7283194465167, "type": "Change", "value": "7282634891791", "field_name": "assignee_id", "previous_value": null}, {"id": 7283194465295, "type": "Change", "value": "360006394556", "field_name": "group_id", "previous_value": null}, {"id": 7283194465423, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of assignment", "id": 360011363256, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Assignment: {{ticket.title}}", "body": "You have been assigned to this ticket (#{{ticket.id}}).\n\n{{ticket.comments_formatted}}", "recipients": [7282634891791]}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1690888174096} +{"stream": "ticket_audits", "data": {"id": 7283163099535, "ticket_id": 153, "created_at": "2023-06-26T12:13:42Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58", "ip_address": "85.209.47.207", "location": "Kyiv, 30, Ukraine", "latitude": 50.458, "longitude": 30.5303}, "custom": {}}, "events": [{"id": 7283163099663, "type": "Change", "value": "7282634891791", "field_name": "assignee_id", "previous_value": "360786799676"}, {"id": 7283163099791, "type": "Change", "value": "360006394556", "field_name": "group_id", "previous_value": "6770788212111"}, {"id": 7283163099919, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of assignment", "id": 360011363256, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Assignment: {{ticket.title}}", "body": "You have been assigned to this ticket (#{{ticket.id}}).\n\n{{ticket.comments_formatted}}", "recipients": [7282634891791]}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1690888174097} +{"stream": "ticket_comments", "data": {"id": 5162146653071, "via": {"channel": "web", "source": {"from": {}, "to": {"name": "Team Airbyte", "address": "integration-test@airbyte.io"}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": " 163748", "html_body": "
     163748
    ", "plain_body": " 163748", "public": true, "attachments": [], "audit_id": 5162146652943, "created_at": "2022-07-18T09:58:23Z", "event_type": "Comment", "ticket_id": 124, "timestamp": 1658138303}, "emitted_at": 1690888176621} +{"stream": "ticket_comments", "data": {"id": 5162208963983, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "238473846", "html_body": "
    238473846
    ", "plain_body": "238473846", "public": false, "attachments": [], "audit_id": 5162208963855, "created_at": "2022-07-18T10:16:53Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139413}, "emitted_at": 1690888176622} +{"stream": "ticket_comments", "data": {"id": 5162223308559, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "Airbyte", "html_body": "
    Airbyte
    ", "plain_body": "Airbyte", "public": false, "attachments": [], "audit_id": 5162223308431, "created_at": "2022-07-18T10:25:21Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139921}, "emitted_at": 1690888176622} +{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833076.json", "id": 360002833076, "type": "subject", "title": "Subject", "raw_title": "Subject", "description": "", "raw_description": "", "position": 1, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Subject", "raw_title_in_portal": "Subject", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1690888178196} +{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json", "id": 360002833096, "type": "description", "title": "Description", "raw_title": "Description", "description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "raw_description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "position": 2, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Description", "raw_title_in_portal": "Description", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1690888178197} +{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1690888178198} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1690888179326} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1690888179326} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282901696015.json", "id": 7282901696015, "ticket_id": 151, "created_at": "2023-06-26T11:09:33Z", "updated_at": "2023-06-26T12:03:38Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T12:03:37Z", "requester_updated_at": "2023-06-26T11:09:33Z", "status_updated_at": "2023-06-26T11:09:33Z", "initially_assigned_at": "2023-06-26T11:09:33Z", "assigned_at": "2023-06-26T11:09:33Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T12:03:37Z", "reply_time_in_minutes": {"calendar": 54, "business": 0}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:09:33Z"}, "emitted_at": 1690888179327} +{"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1690888180347} +{"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1690888180348} +{"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1690888180348} +{"stream": "ticket_skips", "data": {"id": 7290033348623, "ticket_id": 121, "user_id": 360786799676, "reason": "I have no idea.", "created_at": "2023-06-27T08:24:02Z", "updated_at": "2023-06-27T08:24:02Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1690888182191} +{"stream": "ticket_skips", "data": {"id": 7290088475023, "ticket_id": 125, "user_id": 360786799676, "reason": "Another test skip.", "created_at": "2023-06-27T08:30:01Z", "updated_at": "2023-06-27T08:30:01Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "custom_status_id": 4044376, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1690888182192} +{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655481702}, "emitted_at": 1690888183377} +{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/122.json", "id": 122, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (912) 420-0314", "phone": "+19124200314", "name": "Caller +1 (912) 420-0314"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T19:52:39Z", "updated_at": "2022-06-17T21:01:41Z", "type": null, "subject": "Voicemail from: Caller +1 (912) 420-0314", "raw_subject": "Voicemail from: Caller +1 (912) 420-0314", "description": "Call from: +1 (912) 420-0314\\nTime of call: June 17, 2022 at 7:52:02 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4993467856015, "submitter_id": 4993467856015, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655499701}, "emitted_at": 1690888183379} +{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/123.json", "id": 123, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (607) 210-9549", "phone": "+16072109549", "name": "Caller +1 (607) 210-9549"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-07-13T14:34:05Z", "updated_at": "2022-07-13T16:02:03Z", "type": null, "subject": "Voicemail from: Caller +1 (607) 210-9549", "raw_subject": "Voicemail from: Caller +1 (607) 210-9549", "description": "Call from: +1 (607) 210-9549\\nTime of call: July 13, 2022 at 2:33:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 5137812260495, "submitter_id": 5137812260495, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1657728123}, "emitted_at": 1690888183380} +{"stream": "users", "data": {"id": 4992781783439, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json", "name": "Caller +1 (689) 689-8023", "email": null, "created_at": "2022-06-17T14:49:19Z", "updated_at": "2022-06-17T14:49:19Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16896898023", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1690888188031} +{"stream": "users", "data": {"id": 4993467856015, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json", "name": "Caller +1 (912) 420-0314", "email": null, "created_at": "2022-06-17T19:52:38Z", "updated_at": "2022-06-17T19:52:38Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+19124200314", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1690888188032} +{"stream": "users", "data": {"id": 5137812260495, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/5137812260495.json", "name": "Caller +1 (607) 210-9549", "email": null, "created_at": "2022-07-13T14:34:04Z", "updated_at": "2022-07-13T14:34:04Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16072109549", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1690888188033} +{"stream": "brands", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/brands/360000358316.json", "id": 360000358316, "name": "Airbyte", "brand_url": "https://d3v-airbyte.zendesk.com", "subdomain": "d3v-airbyte", "host_mapping": null, "has_help_center": true, "help_center_state": "enabled", "active": true, "default": true, "is_deleted": false, "logo": null, "ticket_form_ids": [360000084116], "signature_template": "{{agent.signature}}", "created_at": "2020-12-11T18:34:04Z", "updated_at": "2020-12-11T18:34:09Z"}, "emitted_at": 1690888190028} +{"stream": "custom_roles", "data": {"id": 360000210636, "name": "Advisor", "description": "Can automate ticket workflows, manage channels and make private comments on tickets", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": false, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "none", "ticket_deletion": false, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "full", "report_access": "none", "ticket_editing": true, "ticket_merge": false, "user_view_access": "full", "view_access": "full", "voice_dashboard_access": false, "manage_automations": true, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": true, "manage_team_members": "readonly"}, "team_member_count": 1}, "emitted_at": 1692702313283} +{"stream": "custom_roles", "data": {"id": 360000210596, "name": "Staff", "description": "Can edit tickets within their groups", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": false, "manage_dynamic_content": false, "manage_extensions_and_channels": false, "manage_facebook": false, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "public", "ticket_deletion": false, "ticket_tag_editing": false, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "manage-personal", "report_access": "readonly", "ticket_editing": true, "ticket_merge": false, "user_view_access": "manage-personal", "view_access": "manage-personal", "voice_dashboard_access": false, "manage_automations": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": false, "manage_slas": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": false, "manage_team_members": "readonly"}, "team_member_count": 1}, "emitted_at": 1692702313283} +{"stream": "custom_roles", "data": {"id": 360000210616, "name": "Team lead", "description": "Can manage all tickets and forums", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2023-06-26T11:06:24Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": true, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "all", "ticket_comment_access": "public", "ticket_deletion": true, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": true, "voice_access": true, "group_access": true, "organization_editing": true, "organization_notes_editing": true, "assign_tickets_to_any_group": false, "end_user_profile_access": "full", "explore_access": "edit", "forum_access": "full", "macro_access": "full", "report_access": "full", "ticket_editing": true, "ticket_merge": true, "user_view_access": "full", "view_access": "playonly", "voice_dashboard_access": true, "manage_automations": true, "manage_contextual_workspaces": true, "manage_organization_fields": true, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": true, "manage_ticket_forms": true, "manage_user_fields": true, "ticket_redaction": true, "manage_roles": "all-except-self", "manage_groups": true, "manage_group_memberships": true, "manage_organizations": true, "manage_suspended_tickets": true, "manage_triggers": true, "manage_team_members": "readonly"}, "team_member_count": 2}, "emitted_at": 1692702313284} +{"stream": "schedules", "data": {"id": 4567312249615, "name": "Test Schedule", "time_zone": "New Caledonia", "created_at": "2022-03-25T10:23:34Z", "updated_at": "2022-03-25T10:23:34Z", "intervals": [{"start_time": 1980, "end_time": 2460}, {"start_time": 3420, "end_time": 3900}, {"start_time": 4860, "end_time": 5340}, {"start_time": 6300, "end_time": 6780}, {"start_time": 7740, "end_time": 8220}]}, "emitted_at": 1690888192224} +{"stream": "ticket_forms", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_forms/360000084116.json", "name": "Default Ticket Form", "display_name": "Default Ticket Form", "id": 360000084116, "raw_name": "Default Ticket Form", "raw_display_name": "Default Ticket Form", "end_user_visible": true, "position": 1, "ticket_field_ids": [360002833076, 360002833096, 360002833116, 360002833136, 360002833156, 360002833176, 360002833196], "active": true, "default": true, "created_at": "2020-12-11T18:34:37Z", "updated_at": "2020-12-11T18:34:37Z", "in_all_brands": true, "restricted_brand_ids": [], "end_user_conditions": [], "agent_conditions": []}, "emitted_at": 1690888193249} +{"stream": "account_attributes", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/routing/attributes/ac43b460-0ebd-11ee-85a3-4750db6aa722.json", "id": "ac43b460-0ebd-11ee-85a3-4750db6aa722", "name": "Language", "created_at": "2023-06-19T16:23:49Z", "updated_at": "2023-06-19T16:23:49Z"}, "emitted_at": 1690888194272} +{"stream": "account_attributes", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/routing/attributes/c15cdb76-0ebd-11ee-a37f-f315f48c0150.json", "id": "c15cdb76-0ebd-11ee-a37f-f315f48c0150", "name": "Quality", "created_at": "2023-06-19T16:24:25Z", "updated_at": "2023-06-19T16:24:25Z"}, "emitted_at": 1690888194273} +{"stream": "attribute_definitions", "data": {"title": "Number of incidents", "subject": "number_of_incidents", "type": "text", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "less_than", "title": "Less than", "terminal": false}, {"value": "greater_than", "title": "Greater than", "terminal": false}, {"value": "is", "title": "Is", "terminal": false}, {"value": "less_than_equal", "title": "Less than or equal to", "terminal": false}, {"value": "greater_than_equal", "title": "Greater than or equal to", "terminal": false}], "condition": "all"}, "emitted_at": 1690888195504} +{"stream": "attribute_definitions", "data": {"title": "Brand", "subject": "brand_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000358316", "title": "Airbyte", "enabled": true}], "condition": "all"}, "emitted_at": 1690888195504} +{"stream": "attribute_definitions", "data": {"title": "Form", "subject": "ticket_form_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000084116", "title": "Default Ticket Form", "enabled": true}], "condition": "all"}, "emitted_at": 1690888195505} +{"stream":"topics","data":{"id":7253394974479,"url":"https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253394974479.json","html_url":"https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253394974479-Feature-Requests","name":"Feature Requests","description":null,"position":0,"follower_count":1,"community_id":7253391140495,"created_at":"2023-06-22T00:32:21Z","updated_at":"2023-06-22T00:32:21Z","manageable_by":"managers","user_segment_id":null},"emitted_at":1687861697934} +{"stream":"topics","data":{"id":7253351897871,"url":"https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253351897871.json","html_url":"https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253351897871-General-Discussion","name":"General Discussion","description":null,"position":0,"follower_count":1,"community_id":7253391140495,"created_at":"2023-06-22T00:32:20Z","updated_at":"2023-06-22T00:32:20Z","manageable_by":"managers","user_segment_id":null},"emitted_at":1687861697934} diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json index 86364ee37603..42aa125638b1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json @@ -156,6 +156,18 @@ "sync_mode": "incremental", "destination_sync_mode": "append" }, + { + "stream": { + "name": "topics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, { "stream": { "name": "users", @@ -167,6 +179,42 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "organization_memberships", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ticket_skips", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml index eb52b8a92069..9a082be9b570 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml @@ -7,11 +7,11 @@ data: connectorType: source maxSecondsBetweenMessages: 10800 definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 - dockerImageTag: 0.3.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-zendesk-support githubIssueLabel: source-zendesk-support icon: zendesk-support.svg - license: MIT + license: ELv2 name: Zendesk Support registries: cloud: @@ -22,4 +22,13 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support tags: - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified + releases: + breakingChanges: + 1.0.0: + message: "`cursor_field` for `Tickets` stream is changed to `generated_timestamp`" + upgradeDeadline: "2023-07-19" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-support/requirements.txt b/airbyte-integrations/connectors/source-zendesk-support/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/requirements.txt +++ b/airbyte-integrations/connectors/source-zendesk-support/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zendesk-support/setup.py b/airbyte-integrations/connectors/source-zendesk-support/setup.py index 132d2f3ed272..58dc32bc30d0 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-support/setup.py @@ -5,9 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "pytz", "requests-futures~=1.0.0", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "pytz"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "connector-acceptance-test", "requests-mock==1.9.3"] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "requests-mock==1.9.3"] setup( version="0.1.0", diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/__init__.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/__init__.py index b9df6c98610b..455b0492fccf 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/__init__.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/__init__.py @@ -1,3 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# """ MIT License diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/account_attributes.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/account_attributes.json new file mode 100644 index 000000000000..2f46c1d6ff65 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/account_attributes.json @@ -0,0 +1,22 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json new file mode 100644 index 000000000000..11aae8663482 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json @@ -0,0 +1,60 @@ +{ + "type": ["null", "object"], + "properties": { + "title": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "group": { + "type": ["null", "string"] + }, + "nullable": { + "type": ["null", "boolean"] + }, + "repeatable": { + "type": ["null", "boolean"] + }, + "operators": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "terminal": { + "type": ["null", "boolean"] + } + } + } + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "value": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + } + } + } + }, + "condition": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json index df6a9b40fb64..dbc7ebd9cdf2 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json @@ -75,6 +75,9 @@ "manage_user_fields": { "type": ["null", "boolean"] }, + "manage_team_members": { + "type": ["null", "string"] + }, "moderate_forums": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/organization_memberships.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/organization_memberships.json new file mode 100644 index 000000000000..21139855401e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/organization_memberships.json @@ -0,0 +1,34 @@ +{ + "properties": { + "default": { + "type": ["null", "boolean"] + }, + "url": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "organization_id": { + "type": ["null", "integer"] + }, + "organization_name": { + "type": ["null", "string"] + }, + "view_tickets": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/post_comments.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/post_comments.json new file mode 100644 index 000000000000..e5495f26de78 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/post_comments.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Post Comments", + "type": ["null", "object"], + "properties": { + "author_id": { + "type": ["null", "integer"] + }, + "body": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "html_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "non_author_editor_id": { + "type": ["null", "integer"] + }, + "non_author_updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "official": { + "type": ["null", "boolean"] + }, + "post_id": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "vote_count": { + "type": ["null", "integer"] + }, + "vote_sum": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json new file mode 100644 index 000000000000..cf3bf282738f --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Posts Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "author_id": { + "type": ["null", "number"] + }, + "content_tag_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "number"] + } + }, + "featured": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json new file mode 100644 index 000000000000..95534c91f74e --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json @@ -0,0 +1,191 @@ +{ + "properties": { + "organization_id": { + "type": ["null", "integer"] + }, + "requester_id": { + "type": ["null", "integer"] + }, + "problem_id": { + "type": ["null", "integer"] + }, + "is_public": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "follower_ids": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "submitter_id": { + "type": ["null", "integer"] + }, + "generated_timestamp": { + "type": ["null", "integer"] + }, + "brand_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "recipient": { + "type": ["null", "string"] + }, + "collaborator_ids": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "tags": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "has_incidents": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "raw_subject": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "custom_fields": { + "items": { + "properties": { + "id": { + "type": ["null", "integer"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "url": { + "type": ["null", "string"] + }, + "allow_channelback": { + "type": ["null", "boolean"] + }, + "allow_attachments": { + "type": ["null", "boolean"] + }, + "due_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "followup_ids": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "priority": { + "type": ["null", "string"] + }, + "assignee_id": { + "type": ["null", "integer"] + }, + "subject": { + "type": ["null", "string"] + }, + "external_id": { + "type": ["null", "string"] + }, + "via": { + "$ref": "via_channel.json" + }, + "ticket_form_id": { + "type": ["null", "integer"] + }, + "satisfaction_rating": { + "type": ["null", "object", "string"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "assignee_id": { + "type": ["null", "integer"] + }, + "group_id": { + "type": ["null", "integer"] + }, + "reason_id": { + "type": ["null", "integer"] + }, + "requester_id": { + "type": ["null", "integer"] + }, + "ticket_id": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "string"] + }, + "reason": { + "type": ["null", "string"] + }, + "comment": { + "type": ["null", "string"] + } + } + }, + "sharing_agreement_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "email_cc_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "forum_topic_id": { + "type": ["null", "integer"] + }, + "custom_status_id": { + "type": ["null", "integer"] + }, + "from_messaging_channel": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json index a139c863d2b9..893519aa0201 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json @@ -145,6 +145,10 @@ "assignee_updated_at": { "type": ["null", "string"], "format": "date-time" + }, + "custom_status_updated_at": { + "type": ["null", "string"], + "format": "date-time" } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_skips.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_skips.json new file mode 100644 index 000000000000..39d4f8843b56 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_skips.json @@ -0,0 +1,28 @@ +{ + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + }, + "ticket_id": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "user_id": { + "type": ["null", "integer"] + }, + "ticket": { + "$ref": "tickets.json" + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json index 95534c91f74e..bc7c5fe3de7a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/tickets.json @@ -1,191 +1 @@ -{ - "properties": { - "organization_id": { - "type": ["null", "integer"] - }, - "requester_id": { - "type": ["null", "integer"] - }, - "problem_id": { - "type": ["null", "integer"] - }, - "is_public": { - "type": ["null", "boolean"] - }, - "description": { - "type": ["null", "string"] - }, - "follower_ids": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "submitter_id": { - "type": ["null", "integer"] - }, - "generated_timestamp": { - "type": ["null", "integer"] - }, - "brand_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "recipient": { - "type": ["null", "string"] - }, - "collaborator_ids": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "tags": { - "items": { - "type": ["null", "string"] - }, - "type": ["null", "array"] - }, - "has_incidents": { - "type": ["null", "boolean"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "raw_subject": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "custom_fields": { - "items": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "url": { - "type": ["null", "string"] - }, - "allow_channelback": { - "type": ["null", "boolean"] - }, - "allow_attachments": { - "type": ["null", "boolean"] - }, - "due_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "followup_ids": { - "items": { - "type": ["null", "integer"] - }, - "type": ["null", "array"] - }, - "priority": { - "type": ["null", "string"] - }, - "assignee_id": { - "type": ["null", "integer"] - }, - "subject": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "via": { - "$ref": "via_channel.json" - }, - "ticket_form_id": { - "type": ["null", "integer"] - }, - "satisfaction_rating": { - "type": ["null", "object", "string"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "assignee_id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "reason_id": { - "type": ["null", "integer"] - }, - "requester_id": { - "type": ["null", "integer"] - }, - "ticket_id": { - "type": ["null", "integer"] - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "url": { - "type": ["null", "string"] - }, - "score": { - "type": ["null", "string"] - }, - "reason": { - "type": ["null", "string"] - }, - "comment": { - "type": ["null", "string"] - } - } - }, - "sharing_agreement_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "email_cc_ids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "forum_topic_id": { - "type": ["null", "integer"] - }, - "custom_status_id": { - "type": ["null", "integer"] - }, - "from_messaging_channel": { - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"] -} +{ "$ref": "tickets.json" } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json new file mode 100644 index 000000000000..43f8439b48ab --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Topics Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "html_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "description": { + "type": ["null", "string"] + }, + "manageable_by": { + "type": ["null", "string"] + }, + "follower_count": { + "type": ["null", "number"] + }, + "position": { + "type": ["null", "number"] + }, + "user_segment_id": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/votes.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/votes.json new file mode 100644 index 000000000000..84f70c88d0c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/votes.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Votes", + "type": ["null", "object"], + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "item_id": { + "type": ["null", "integer"] + }, + "item_type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "value": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py index b6aea876345c..2aafd9029aa6 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py @@ -14,13 +14,20 @@ from source_zendesk_support.streams import DATETIME_FORMAT, SourceZendeskException from .streams import ( + AccountAttributes, + AttributeDefinitions, AuditLogs, Brands, CustomRoles, GroupMemberships, Groups, Macros, + OrganizationMemberships, Organizations, + PostComments, + PostCommentVotes, + Posts, + PostVotes, SatisfactionRatings, Schedules, SlaPolicies, @@ -32,6 +39,8 @@ TicketMetricEvents, TicketMetrics, Tickets, + TicketSkips, + Topics, Users, UserSettingsStream, ) @@ -55,7 +64,7 @@ class SourceZendeskSupport(AbstractSource): """ @classmethod - def get_authenticator(cls, config: Mapping[str, Any]) -> BasicApiTokenAuthenticator: + def get_authenticator(cls, config: Mapping[str, Any]) -> [TokenAuthenticator, BasicApiTokenAuthenticator]: # old authentication flow support auth_old = config.get("auth_method") @@ -117,6 +126,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Groups(**args), Macros(**args), Organizations(**args), + OrganizationMemberships(**args), + Posts(**args), + PostComments(**args), + PostCommentVotes(**args), + PostVotes(**args), SatisfactionRatings(**args), SlaPolicies(**args), Tags(**args), @@ -125,20 +139,25 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: TicketFields(**args), TicketMetrics(**args), TicketMetricEvents(**args), + TicketSkips(**args), Tickets(**args), + Topics(**args), Users(**args), Brands(**args), CustomRoles(**args), Schedules(**args), ] ticket_forms_stream = TicketForms(**args) - # TicketForms stream is only available for Enterprise Plan users but Zendesk API does not provide - # a public API to get user's subscription plan. That's why we try to read at least one record and expose this stream - # in case of success or skip it otherwise + account_attributes = AccountAttributes(**args) + attribute_definitions = AttributeDefinitions(**args) + # TicketForms, AccountAttributes and AttributeDefinitions streams are only available for Enterprise Plan users, + # but Zendesk API does not provide a public API to get user's subscription plan. + # That's why we try to read at least one record from one of these streams and expose all of them in case of success + # or skip them otherwise try: for stream_slice in ticket_forms_stream.stream_slices(sync_mode=SyncMode.full_refresh): for _ in ticket_forms_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): - streams.append(ticket_forms_stream) + streams.extend([ticket_forms_stream, account_attributes, attribute_definitions]) break except Exception as e: logger.warning(f"An exception occurred while trying to access TicketForms stream: {str(e)}. Skipping this stream.") diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json index ef4d118d9f11..8ad90bb867de 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json @@ -10,7 +10,7 @@ "start_date": { "type": "string", "title": "Start Date", - "description": "The date from which you'd like to replicate data for Zendesk Support API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "description": "The UTC date and time from which you'd like to replicate data, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", "examples": ["2020-10-15T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "format": "date-time" @@ -18,12 +18,12 @@ "subdomain": { "type": "string", "title": "Subdomain", - "description": "This is your Zendesk subdomain that can be found in your account URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN is the value of your subdomain." + "description": "This is your unique Zendesk subdomain that can be found in your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/, MY_SUBDOMAIN is the value of your subdomain." }, "credentials": { "title": "Authentication", "type": "object", - "description": "Zendesk service provides two authentication methods. Choose between: `OAuth2.0` or `API token`.", + "description": "Zendesk allows two authentication methods. We recommend using `OAuth2.0` for Airbyte Cloud users and `API token` for Airbyte Open Source users.", "oneOf": [ { "title": "OAuth2.0", @@ -39,7 +39,19 @@ "access_token": { "type": "string", "title": "Access Token", - "description": "The value of the API token generated. See the docs for more information.", + "description": "The OAuth access token. See the Zendesk docs for more information on generating this token.", + "airbyte_secret": true + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The OAuth client's ID. See this guide for more information.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The OAuth client secret. See this guide for more information.", "airbyte_secret": true } } @@ -63,7 +75,7 @@ "api_token": { "title": "API Token", "type": "string", - "description": "The value of the API token generated. See the docs for more information.", + "description": "The value of the API token generated. See our full documentation for more information on generating this token.", "airbyte_secret": true } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index c23e974905d6..6dacf68b0148 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -3,17 +3,10 @@ # import calendar -import functools import logging import re -import time from abc import ABC -from collections import deque -from concurrent.futures import Future, ProcessPoolExecutor -from datetime import datetime, timedelta -from functools import partial -from math import ceil -from pickle import PickleError, dumps +from datetime import datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import parse_qsl, urljoin, urlparse @@ -22,14 +15,11 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from airbyte_cdk.sources.streams.http.rate_limiting import TRANSIENT_EXCEPTIONS +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from requests.auth import AuthBase -from requests_futures.sessions import PICKLE_ERROR, FuturesSession from source_zendesk_support.ZendeskSupportAvailabilityStrategy import ZendeskSupportAvailabilityStrategy DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" @@ -38,34 +28,9 @@ logger = logging.getLogger("airbyte") -# For some streams, multiple http requests are running at the same time for performance reasons. -# However, it may result in hitting the rate limit, therefore subsequent requests have to be made after a pause. -# The idea is to sustain a pause once and continue making multiple requests at a time. -# A single `retry_at` variable is introduced here, which prevents us from duplicate sleeping in the main thread -# before each request is made as it used to be in prior versions. -# It acts like a global counter - increased each time a 429 status is met -# only if it is greater than the current value. On the other hand, no request may be made before this moment. -# Because the requests are made in parallel, time.sleep will be called in parallel as well. -# This is possible because it is a point in time, not timedelta. -retry_at: Optional[datetime] = None - - -def sleep_before_executing(sleep_time: float): - def wrapper(function): - @functools.wraps(function) - def inner(*args, **kwargs): - logger.info(f"Sleeping {sleep_time} seconds before next request") - time.sleep(int(sleep_time)) - result = function(*args, **kwargs) - return result, datetime.utcnow() - - return inner - - return wrapper - def to_int(s): - "https://github.com/airbytehq/airbyte/issues/13673" + """https://github.com/airbytehq/airbyte/issues/13673""" if isinstance(s, str): res = re.findall(r"[-+]?\d+", s) if res: @@ -77,41 +42,7 @@ class SourceZendeskException(Exception): """default exception of custom SourceZendesk logic""" -class SourceZendeskSupportFuturesSession(FuturesSession): - """ - Check the docs at https://github.com/ross/requests-futures - Used to async execute a set of requests. - """ - - def send_future(self, request: requests.PreparedRequest, **kwargs) -> Future: - """ - Use instead of default `Session.send()` method. - `Session.send()` should not be overridden as it used by `requests-futures` lib. - """ - - if self.session: - func = self.session.send - else: - sleep_time = 0 - now = datetime.utcnow() - if retry_at and retry_at > now: - sleep_time = (retry_at - datetime.utcnow()).seconds - # avoid calling super to not break pickled method - func = partial(requests.Session.send, self) - func = sleep_before_executing(sleep_time)(func) - - if isinstance(self.executor, ProcessPoolExecutor): - self.logger.warning("ProcessPoolExecutor is used to perform IO related tasks for unknown reason!") - # verify function can be pickled - try: - dumps(func) - except (TypeError, PickleError): - raise RuntimeError(PICKLE_ERROR) - - return self.executor.submit(func, request, **kwargs) - - -class BaseSourceZendeskSupportStream(HttpStream, ABC): +class BaseZendeskSupportStream(HttpStream, ABC): raise_on_http_errors = True def __init__(self, subdomain: str, start_date: str, ignore_pagination: bool = False, **kwargs): @@ -170,16 +101,6 @@ def str2unixtime(str_dt: str) -> Optional[int]: dt = datetime.strptime(str_dt, DATETIME_FORMAT) return calendar.timegm(dt.utctimetuple()) - @staticmethod - def _parse_next_page_number(response: requests.Response) -> Optional[int]: - """Parses a response and tries to find next page number""" - try: - next_page = response.json().get("next_page") - except requests.exceptions.JSONDecodeError: - next_page = None - - return dict(parse_qsl(urlparse(next_page).query)).get("page") if next_page else None - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: """try to select relevant data only""" @@ -198,18 +119,20 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record def should_retry(self, response: requests.Response) -> bool: - if response.status_code == 403: + status_code = response.status_code + if status_code == 403 or status_code == 404: try: error = response.json().get("error") except requests.exceptions.JSONDecodeError: - error = {"title": "Forbidden", "message": "Received empty JSON response"} + reason = response.reason + error = {"title": f"{reason}", "message": "Received empty JSON response"} self.logger.error(f"Skipping stream {self.name}: Check permissions, error message: {error}.") setattr(self, "raise_on_http_errors", False) return False return super().should_retry(response) -class SourceZendeskSupportStream(BaseSourceZendeskSupportStream): +class SourceZendeskSupportStream(BaseZendeskSupportStream): """Basic Zendesk class""" primary_key = "id" @@ -218,17 +141,9 @@ class SourceZendeskSupportStream(BaseSourceZendeskSupportStream): cursor_field = "updated_at" response_list_name: str = None - future_requests: deque = None transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, **kwargs): - super().__init__(**kwargs) - - self._session = SourceZendeskSupportFuturesSession() - self._session.auth = authenticator - self.future_requests = deque() - @property def url_base(self) -> str: return f"https://{self._subdomain}.zendesk.com/api/v2/" @@ -264,54 +179,16 @@ def get_api_records_count(self, stream_slice: Mapping[str, Any] = None, stream_s if start_date: params["start_time"] = self.str2datetime(start_date) - response = self._session.request("get", count_url).result() + response = self._session.request("get", count_url) records_count = response.json().get("count", {}).get("value", 0) return records_count - def generate_future_requests( + def request_params( self, - sync_mode: SyncMode, - cursor_field: List[str] = None, + stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ): - records_count = self.get_api_records_count(stream_slice=stream_slice, stream_state=stream_state) - self.logger.info(f"Records count is {records_count}") - page_count = ceil(records_count / self.page_size) - for page_number in range(1, page_count + 1): - params = self.request_params(stream_state=stream_state, stream_slice=stream_slice) - params["page"] = page_number - request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice) - - request = self._create_prepared_request( - path=self.path(stream_state=stream_state, stream_slice=stream_slice), - headers=dict(request_headers, **self.authenticator.get_auth_header()), - params=params, - json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice), - data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice), - ) - - request_kwargs = self.request_kwargs(stream_state=stream_state, stream_slice=stream_slice) - self.future_requests.append( - { - "future": self._send_request(request, request_kwargs), - "request": request, - "request_kwargs": request_kwargs, - "retries": 0, - } - ) - self.logger.info(f"Generated {len(self.future_requests)} future requests") - - def _send(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> Future: - response: Future = self._session.send_future(request, **request_kwargs) - return response - - def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> Future: - return self._send(request, request_kwargs) - - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = {} stream_state = stream_state or {} @@ -330,64 +207,8 @@ def request_params( return params - def _retry( - self, - request: requests.PreparedRequest, - retries: int, - original_exception: Exception = None, - response: requests.Response = None, - finished_at: Optional[datetime] = None, - **request_kwargs, - ): - if retries == self.max_retries: - if original_exception: - raise original_exception - raise DefaultBackoffException(request=request, response=response) - sleep_time = self.backoff_time(response) - if response is not None and finished_at and sleep_time: - current_retry_at = finished_at + timedelta(seconds=sleep_time) - global retry_at - if not retry_at or (retry_at < current_retry_at): - retry_at = current_retry_at - self.logger.info(f"Adding a request to be retried in {sleep_time} seconds") - self.future_requests.append( - { - "future": self._send_request(request, request_kwargs), - "request": request, - "request_kwargs": request_kwargs, - "retries": retries + 1, - } - ) - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - self.generate_future_requests(sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) - - while len(self.future_requests) > 0: - self.logger.info("Starting another while loop iteration") - item = self.future_requests.popleft() - request, retries, future, kwargs = item["request"], item["retries"], item["future"], item["request_kwargs"] - try: - response, finished_at = future.result() - except TRANSIENT_EXCEPTIONS as exc: - self.logger.info("Will retry the request because of a transient exception") - self._retry(request=request, retries=retries, original_exception=exc, **kwargs) - continue - if self.should_retry(response): - self.logger.info("Will retry the request for other reason") - self._retry(request=request, retries=retries, response=response, finished_at=finished_at, **kwargs) - continue - self.logger.info("Request successful, will parse the response now") - yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice) - - -class SourceZendeskSupportFullRefreshStream(BaseSourceZendeskSupportStream): +class FullRefreshZendeskSupportStream(BaseZendeskSupportStream): """ Endpoints don't provide the updated_at/created_at fields Thus we can't implement an incremental logic for them @@ -407,24 +228,23 @@ def path(self, **kwargs): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: if self._ignore_pagination: return None - next_page = self._parse_next_page_number(response) - if not next_page: - self._finished = True - return None - return next_page - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(next_page_token=next_page_token, **kwargs) - params.update( - { - "page": next_page_token or 1, - "per_page": self.page_size, - } - ) + + meta = response.json().get("meta", {}) if response.content else {} + return {"page[after]": meta.get("after_cursor")} if meta.get("has_more") else None + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = {"page[size]": self.page_size} + if next_page_token: + params.update(next_page_token) return params -class SourceZendeskSupportCursorPaginationStream(SourceZendeskSupportFullRefreshStream): +class IncrementalZendeskSupportStream(FullRefreshZendeskSupportStream): """ Endpoints provide a cursor pagination and sorting mechanism """ @@ -439,6 +259,41 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late new_value = str((latest_record or {}).get(self.cursor_field, "")) return {self.cursor_field: max(new_value, old_value)} + def check_stream_state(self, stream_state: Mapping[str, Any] = None) -> int: + """ + Returns the state value, if exists. Otherwise, returns user defined `Start Date`. + """ + state = stream_state.get(self.cursor_field) or self._start_date if stream_state else self._start_date + return calendar.timegm(pendulum.parse(state).utctimetuple()) + + +class CursorPaginationZendeskSupportStream(IncrementalZendeskSupportStream): + """Zendesk Support Cursor Pagination, see https://developer.zendesk.com/api-reference/introduction/pagination/#using-cursor-pagination""" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + if self._ignore_pagination: + return None + + meta = response.json().get("meta", {}) + return {"page[after]": meta.get("after_cursor")} if meta.get("has_more") else None + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = { + "start_time": self.check_stream_state(stream_state), + "page[size]": self.page_size, + } + if next_page_token: + params.pop("start_time", None) + params.update(next_page_token) + return params + + +class TimeBasedPaginationZendeskSupportStream(IncrementalZendeskSupportStream): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: if self._ignore_pagination: return None @@ -447,15 +302,11 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, self.prev_start_time = start_time return {self.cursor_field: int(start_time)} - def check_stream_state(self, stream_state: Mapping[str, Any] = None): - """ - Returns the state value, if exists. Otherwise, returns user defined `Start Date`. - """ - state = stream_state.get(self.cursor_field) or self._start_date if stream_state else self._start_date - return calendar.timegm(pendulum.parse(state).utctimetuple()) - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: next_page_token = next_page_token or {} parsed_state = self.check_stream_state(stream_state) @@ -466,7 +317,7 @@ def request_params( return params -class SourceZendeskIncrementalExportStream(SourceZendeskSupportCursorPaginationStream): +class SourceZendeskIncrementalExportStream(IncrementalZendeskSupportStream): """Incremental Export from Tickets stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-export-time-based @@ -479,7 +330,7 @@ class SourceZendeskIncrementalExportStream(SourceZendeskSupportCursorPaginationS sideload_param: str = None @staticmethod - def check_start_time_param(requested_start_time: int, value: int = 1): + def check_start_time_param(requested_start_time: int, value: int = 1) -> int: """ Requesting tickets in the future is not allowed, hits 400 - bad request. We get current UNIX timestamp minus `value` from now(), default = 1 (minute). @@ -496,17 +347,25 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, """ Returns next_page_token based on `end_of_stream` parameter inside of response """ - next_page_token = super().next_page_token(response) - return None if response.json().get(END_OF_STREAM_KEY, False) else next_page_token + if self._ignore_pagination: + return None + response_json = response.json() + return None if response_json.get(END_OF_STREAM_KEY, True) else {"cursor": response_json.get("after_cursor")} def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, next_page_token, **kwargs) + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) # check "start_time" is not in the future params["start_time"] = self.check_start_time_param(params["start_time"]) if self.sideload_param: params["include"] = self.sideload_param + if next_page_token: + params.pop("start_time", None) + params.update(next_page_token) return params def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: @@ -530,6 +389,42 @@ class SourceZendeskSupportTicketEventsExportStream(SourceZendeskIncrementalExpor list_entities_from_event: List[str] = None event_type: str = None + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + return f"incremental/{self.response_list_name}.json" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Returns next_page_token based on `end_of_stream` parameter inside of response + """ + response_json = response.json() + return None if response_json.get(END_OF_STREAM_KEY, True) else {"start_time": response_json.get("end_time")} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + next_page_token = next_page_token or {} + parsed_state = self.check_stream_state(stream_state) + if self.cursor_field: + params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} + else: + params = {"start_time": calendar.timegm(pendulum.parse(self._start_date).utctimetuple())} + # check "start_time" is not in the future + params["start_time"] = self.check_start_time_param(params["start_time"]) + if self.sideload_param: + params["include"] = self.sideload_param + if next_page_token: + params.update(next_page_token) + return params + @property def update_event_from_record(self) -> bool: """Returns True/False based on list_entities_from_event property""" @@ -545,10 +440,14 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield event -class AuditLogs(SourceZendeskSupportCursorPaginationStream): +class OrganizationMemberships(CursorPaginationZendeskSupportStream): + """OrganizationMemberships stream: https://developer.zendesk.com/api-reference/ticketing/organizations/organization_memberships/""" + + +class AuditLogs(CursorPaginationZendeskSupportStream): """AuditLogs stream: https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/#list-audit-logs""" - # can request a maximum of 1,00 results + # can request a maximum of 100 results page_size = 100 # audit_logs doesn't have the 'updated_by' field cursor_field = "created_at" @@ -559,24 +458,87 @@ class Users(SourceZendeskIncrementalExportStream): response_list_name: str = "users" + def path(self, **kwargs) -> str: + return "incremental/users/cursor.json" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + next_page_token = next_page_token or {} + parsed_state = self.check_stream_state(stream_state) + if self.cursor_field: + params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} + else: + params = {"start_time": calendar.timegm(pendulum.parse(self._start_date).utctimetuple())} + # check "start_time" is not in the future + params["start_time"] = self.check_start_time_param(params["start_time"]) + if self.sideload_param: + params["include"] = self.sideload_param + if next_page_token: + params.update(next_page_token) + return params + class Organizations(SourceZendeskSupportStream): """Organizations stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/""" +class Posts(CursorPaginationZendeskSupportStream): + """Posts stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/posts/#list-posts""" + + use_cache = True + + cursor_field = "updated_at" + + def path(self, **kwargs): + return "community/posts" + + class Tickets(SourceZendeskIncrementalExportStream): """Tickets stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-export-time-based""" response_list_name: str = "tickets" transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - @staticmethod - def check_start_time_param(requested_start_time: int, value: int = 1): + cursor_field = "generated_timestamp" + + def path(self, **kwargs) -> str: + return "incremental/tickets/cursor.json" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + parsed_state = self.check_stream_state(stream_state) + params = {"start_time": self.check_start_time_param(parsed_state)} + if self.sideload_param: + params["include"] = self.sideload_param + if next_page_token: + params.update(next_page_token) + return params + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + old_value = (current_stream_state or {}).get(self.cursor_field, pendulum.parse(self._start_date).int_timestamp) + new_value = (latest_record or {}).get(self.cursor_field, pendulum.parse(self._start_date).int_timestamp) + return {self.cursor_field: max(new_value, old_value)} + + def check_stream_state(self, stream_state: Mapping[str, Any] = None) -> int: + """ + Returns the state value, if exists. Otherwise, returns user defined `Start Date`. + """ + return stream_state.get(self.cursor_field) if stream_state else pendulum.parse(self._start_date).int_timestamp + + def check_start_time_param(self, requested_start_time: int, value: int = 1) -> int: """ The stream returns 400 Bad Request StartTimeTooRecent when requesting tasks 1 second before now. Figured out during experiments that the most recent time needed for request to be successful is 3 seconds before now. """ - return SourceZendeskIncrementalExportStream.check_start_time_param(requested_start_time, value=3) + return super().check_start_time_param(requested_start_time, value=3) class TicketComments(SourceZendeskSupportTicketEventsExportStream): @@ -600,45 +562,33 @@ class Groups(SourceZendeskSupportStream): """Groups stream: https://developer.zendesk.com/api-reference/ticketing/groups/groups/""" -class GroupMemberships(SourceZendeskSupportCursorPaginationStream): +class GroupMemberships(CursorPaginationZendeskSupportStream): """GroupMemberships stream: https://developer.zendesk.com/api-reference/ticketing/groups/group_memberships/""" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - if self._ignore_pagination: - return None - next_page = self._parse_next_page_number(response) - return next_page if next_page else None - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = {"page": 1, "per_page": self.page_size, "sort_by": "asc"} - start_time = self.str2unixtime((stream_state or {}).get(self.cursor_field)) - params["start_time"] = start_time if start_time else self.str2unixtime(self._start_date) - if next_page_token: - params["page"] = next_page_token + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params.update({"sort_by": "asc"}) return params -class SatisfactionRatings(SourceZendeskSupportCursorPaginationStream): +class SatisfactionRatings(CursorPaginationZendeskSupportStream): """ SatisfactionRatings stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/satisfaction_ratings/ """ - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - if self._ignore_pagination: - return None - next_page = self._parse_next_page_number(response) - return next_page if next_page else None - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = {"page": 1, "per_page": self.page_size, "sort_by": "asc"} - start_time = self.str2unixtime((stream_state or {}).get(self.cursor_field)) - params["start_time"] = start_time if start_time else self.str2unixtime(self._start_date) - if next_page_token: - params["page"] = next_page_token + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params.update({"sort_by": "asc"}) return params @@ -646,40 +596,24 @@ class TicketFields(SourceZendeskSupportStream): """TicketFields stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_fields/""" -class TicketForms(SourceZendeskSupportCursorPaginationStream): +class TicketForms(TimeBasedPaginationZendeskSupportStream): """TicketForms stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_forms""" -class TicketMetrics(SourceZendeskSupportCursorPaginationStream): +class TicketMetrics(CursorPaginationZendeskSupportStream): """TicketMetric stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metrics/""" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - https://developer.zendesk.com/documentation/api-basics/pagination/paginating-through-lists-using-cursor-pagination/#when-to-stop-paginating - """ - meta = response.json().get("meta", {}) - return meta.get("after_cursor") if meta.get("has_more", False) else None - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - """ - To make the Cursor Pagination to return `after_cursor` we should follow these instructions: - https://developer.zendesk.com/documentation/api-basics/pagination/paginating-through-lists-using-cursor-pagination/#enabling-cursor-pagination - """ - params = { - "start_time": self.check_stream_state(stream_state), - "page[size]": self.page_size, - } - if next_page_token: - # when cursor pagination is used, we can pass only `after` and `page size` params, - # other params should be omitted. - params.pop("start_time", None) - params["page[after]"] = next_page_token - return params +class TicketSkips(CursorPaginationZendeskSupportStream): + """TicketSkips stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_skips/""" + + response_list_name = "skips" + + def path(self, **kwargs): + return "skips.json" -class TicketMetricEvents(SourceZendeskSupportCursorPaginationStream): +class TicketMetricEvents(CursorPaginationZendeskSupportStream): """ TicketMetricEvents stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/ """ @@ -689,12 +623,26 @@ class TicketMetricEvents(SourceZendeskSupportCursorPaginationStream): def path(self, **kwargs): return "incremental/ticket_metric_events" + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = { + "start_time": self.check_stream_state(stream_state), + "page[size]": self.page_size, + } + if next_page_token: # need keep start_time for this stream + params.update(next_page_token) + return params + class Macros(SourceZendeskSupportStream): """Macros stream: https://developer.zendesk.com/api-reference/ticketing/business-rules/macros/""" -class TicketAudits(SourceZendeskSupportCursorPaginationStream): +class TicketAudits(IncrementalZendeskSupportStream): """TicketAudits stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_audits/""" # can request a maximum of 1,000 results @@ -708,49 +656,124 @@ class TicketAudits(SourceZendeskSupportCursorPaginationStream): transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) # This endpoint uses a variant of cursor pagination with some differences from cursor pagination used in other endpoints. - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: params = {"sort_by": self.cursor_field, "sort_order": "desc", "limit": self.page_size} - if next_page_token: - params["cursor"] = next_page_token + params.pop("start_time", None) + params.update(next_page_token) return params def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: if self._ignore_pagination: return None - return response.json().get("before_cursor") + response_json = response.json() + return {"cursor": response.json().get("before_cursor")} if response_json.get("before_cursor") else None -class Tags(SourceZendeskSupportFullRefreshStream): +class Tags(FullRefreshZendeskSupportStream): """Tags stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/tags/""" # doesn't have the 'id' field primary_key = "name" -class SlaPolicies(SourceZendeskSupportFullRefreshStream): +class Topics(CursorPaginationZendeskSupportStream): + """ + Topics stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/topics/#list-topics + """ + + cursor_field = "updated_at" + + def path(self, **kwargs): + return "community/topics" + + +class SlaPolicies(FullRefreshZendeskSupportStream): """SlaPolicies stream: https://developer.zendesk.com/api-reference/ticketing/business-rules/sla_policies/""" def path(self, *args, **kwargs) -> str: return "slas/policies.json" + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + return {} -class Brands(SourceZendeskSupportFullRefreshStream): + +class Brands(FullRefreshZendeskSupportStream): """Brands stream: https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands""" -class CustomRoles(SourceZendeskSupportFullRefreshStream): +class CustomRoles(FullRefreshZendeskSupportStream): """CustomRoles stream: https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles""" + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + return {} + -class Schedules(SourceZendeskSupportFullRefreshStream): +class Schedules(FullRefreshZendeskSupportStream): """Schedules stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules""" def path(self, *args, **kwargs) -> str: return "business_hours/schedules.json" -class UserSettingsStream(SourceZendeskSupportFullRefreshStream): +class AccountAttributes(FullRefreshZendeskSupportStream): + """Account attributes stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-account-attributes""" + + response_list_name = "attributes" + + def path(self, *args, **kwargs) -> str: + return "routing/attributes" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + return {} + + +class AttributeDefinitions(FullRefreshZendeskSupportStream): + """Attribute definitions stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-routing-attribute-definitions""" + + primary_key = None + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + for definition in response.json()["definitions"]["conditions_all"]: + definition["condition"] = "all" + yield definition + for definition in response.json()["definitions"]["conditions_any"]: + definition["confition"] = "any" + yield definition + + def path(self, *args, **kwargs) -> str: + return "routing/attributes/definitions" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + return {} + + +class UserSettingsStream(FullRefreshZendeskSupportStream): """Stream for checking of a request token and permissions""" def path(self, *args, **kwargs) -> str: @@ -770,3 +793,69 @@ def get_settings(self) -> Mapping[str, Any]: for resp in self.read_records(SyncMode.full_refresh): return resp raise SourceZendeskException("not found settings") + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + return {} + + +class PostComments(FullRefreshZendeskSupportStream, HttpSubStream): + response_list_name = "comments" + + def __init__(self, **kwargs): + parent = Posts(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + post_id = stream_slice.get("parent").get("id") + return f"community/posts/{post_id}/comments" + + +class AbstractVotes(FullRefreshZendeskSupportStream, ABC): + response_list_name = "votes" + + def get_json_schema(self) -> Mapping[str, Any]: + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("votes") + + +class PostVotes(AbstractVotes, HttpSubStream): + def __init__(self, **kwargs): + parent = Posts(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + post_id = stream_slice.get("parent").get("id") + return f"community/posts/{post_id}/votes" + + +class PostCommentVotes(AbstractVotes, HttpSubStream): + def __init__(self, **kwargs): + parent = PostComments(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + post_id = stream_slice.get("parent").get("post_id") + comment_id = stream_slice.get("parent").get("id") + return f"community/posts/{post_id}/comments/{comment_id}/votes" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_futures.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_futures.py deleted file mode 100644 index 90c8e6efc627..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_futures.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import datetime -import json -from datetime import timedelta -from urllib.parse import urljoin - -import pendulum -import pytest -import requests -import requests_mock -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from requests.exceptions import ConnectionError -from source_zendesk_support.source import BasicApiTokenAuthenticator -from source_zendesk_support.streams import Macros, Organizations - -STREAM_ARGS: dict = { - "subdomain": "fake-subdomain", - "start_date": "2021-01-27T00:00:00Z", - "authenticator": BasicApiTokenAuthenticator("test@airbyte.io", "api_token"), -} - - -@pytest.fixture() -def time_sleep_mock(mocker): - time_mock = mocker.patch("time.sleep", lambda x: None) - yield time_mock - - -@pytest.mark.parametrize( - "records_count,page_size,expected_futures_deque_len", - [ - (1000, 100, 10), - (1000, 10, 100), - (0, 100, 0), - (1, 100, 1), - (101, 100, 2), - ], -) -def test_proper_number_of_future_requests_generated(records_count, page_size, expected_futures_deque_len, time_sleep_mock): - stream = Macros(**STREAM_ARGS) - stream.page_size = page_size - - with requests_mock.Mocker() as m: - count_url = urljoin(stream.url_base, f"{stream.path()}/count.json") - m.get(count_url, text=json.dumps({"count": {"value": records_count}})) - records_url = urljoin(stream.url_base, stream.path()) - m.get(records_url) - stream.generate_future_requests(sync_mode=SyncMode.full_refresh, cursor_field=stream.cursor_field) - assert len(stream.future_requests) == expected_futures_deque_len - - -@pytest.mark.parametrize( - "records_count,page_size,expected_futures_deque_len", - [ - (10, 10, 10), - (10, 100, 10), - (10, 10, 0), - ], -) -def test_parse_future_records(records_count, page_size, expected_futures_deque_len, time_sleep_mock): - stream = Macros(**STREAM_ARGS) - stream.page_size = page_size - expected_records = [ - {f"key{i}": f"val{i}", stream.cursor_field: (pendulum.parse("2020-01-01") + timedelta(days=i)).isoformat()} - for i in range(records_count) - ] - - with requests_mock.Mocker() as m: - count_url = urljoin(stream.url_base, f"{stream.path()}/count.json") - m.get( - count_url, - text=json.dumps({"count": {"value": records_count}}), - ) - - records_url = urljoin(stream.url_base, stream.path()) - m.get(records_url, text=json.dumps({stream.name: expected_records})) - - stream.generate_future_requests(sync_mode=SyncMode.full_refresh, cursor_field=stream.cursor_field) - if not stream.future_requests and not expected_futures_deque_len: - assert len(stream.future_requests) == 0 and not expected_records - else: - response, _ = stream.future_requests[0]["future"].result() - records = list(stream.parse_response(response, stream_state=None, stream_slice=None)) - assert records == expected_records - - -@pytest.mark.parametrize( - "records_count, page_size, expected_futures_deque_len, expected_exception", - [ - (1000, 10, 100, DefaultBackoffException), - (0, 100, 0, DefaultBackoffException), - (150, 100, 2, ConnectionError), - (1, 100, 1, None), - (101, 101, 2, None), - ], -) -def test_read_records(mocker, records_count, page_size, expected_futures_deque_len, expected_exception, time_sleep_mock): - stream = Macros(**STREAM_ARGS) - stream.page_size = page_size - should_retry = bool(expected_exception) - expected_records_count = min(page_size, records_count) if should_retry else records_count - - def record_gen(start=0, end=page_size): - for i in range(start, end): - yield {f"key{i}": f"val{i}", stream.cursor_field: (pendulum.parse("2020-01-01") + timedelta(days=i)).isoformat()} - - with requests_mock.Mocker() as m: - count_url = urljoin(stream.url_base, f"{stream.path()}/count.json") - m.get(count_url, text=json.dumps({"count": {"value": records_count}})) - - records_url = urljoin(stream.url_base, stream.path()) - responses = [ - { - "status_code": 429 if should_retry else 200, - "headers": {"X-Rate-Limit": "700"}, - "text": "{}" - if should_retry - else json.dumps({"macros": list(record_gen(page * page_size, min(records_count, (page + 1) * page_size)))}), - } - for page in range(expected_futures_deque_len) - ] - m.get(records_url, responses) - - if expected_exception is ConnectionError: - mocker.patch.object(requests.Session, "send", side_effect=ConnectionError()) - if should_retry and expected_futures_deque_len: - with pytest.raises(expected_exception): - list(stream.read_records(sync_mode=SyncMode.full_refresh)) - else: - assert list(stream.read_records(sync_mode=SyncMode.full_refresh)) == list(record_gen(end=expected_records_count)) - - -def test_sleep_time(): - page_size = 100 - x_rate_limit = 10 - records_count = 350 - pages = 4 - - start = datetime.datetime.now() - stream = Organizations(**STREAM_ARGS) - stream.page_size = page_size - - def record_gen(start=0, end=100): - for i in range(start, end): - yield {f"key{i}": f"val{i}", stream.cursor_field: (pendulum.parse("2020-01-01") + timedelta(days=i)).isoformat()} - - with requests_mock.Mocker() as m: - count_url = urljoin(stream.url_base, f"{stream.path()}/count.json") - m.get(count_url, text=json.dumps({"count": {"value": records_count}})) - - records_url = urljoin(stream.url_base, stream.path()) - responses = [ - { - "status_code": 429, - "headers": {"X-Rate-Limit": str(x_rate_limit)}, - "text": "{}" - } - for _ in range(pages) - ] + [ - { - "status_code": 200, - "headers": {}, - "text": json.dumps({"organizations": list(record_gen(page * page_size, min(records_count, (page + 1) * page_size)))}) - } - for page in range(pages) - ] - m.get(records_url, responses) - records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - assert len(records) == records_count - end = datetime.datetime.now() - sleep_time = int(60 / x_rate_limit) - assert sleep_time - 1 <= (end - start).seconds <= sleep_time + 1 diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 3b2e81d105c5..8037215adb23 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -20,14 +20,19 @@ DATETIME_FORMAT, END_OF_STREAM_KEY, LAST_END_TIME_KEY, + AccountAttributes, + AttributeDefinitions, AuditLogs, - BaseSourceZendeskSupportStream, + BaseZendeskSupportStream, Brands, CustomRoles, GroupMemberships, Groups, Macros, + OrganizationMemberships, Organizations, + PostCommentVotes, + Posts, SatisfactionRatings, Schedules, SlaPolicies, @@ -40,6 +45,8 @@ TicketMetricEvents, TicketMetrics, Tickets, + TicketSkips, + Topics, Users, UserSettingsStream, ) @@ -134,20 +141,28 @@ def test_check(response, start_date, check_passed): @pytest.mark.parametrize( - "ticket_forms_response, status_code, expected_n_streams, expected_warnings", + "ticket_forms_response, status_code, expected_n_streams, expected_warnings, reason", [ - ({"ticket_forms": [{"id": 1, "updated_at": "2021-07-08T00:05:45Z"}]}, 200, 19, []), + ('{"ticket_forms": [{"id": 1, "updated_at": "2021-07-08T00:05:45Z"}]}', 200, 28, [], None), ( - {"error": "Not sufficient permissions"}, - 403, - 18, - ["Skipping stream ticket_forms: Check permissions, error message: Not sufficient permissions."], + '{"error": "Not sufficient permissions"}', + 403, + 25, + ["Skipping stream ticket_forms: Check permissions, error message: Not sufficient permissions."], + None + ), + ( + '', + 404, + 25, + ["Skipping stream ticket_forms: Check permissions, error message: {'title': 'Not Found', 'message': 'Received empty JSON response'}."], + 'Not Found' ), ], - ids=["forms_accessible", "forms_inaccessible"], + ids=["forms_accessible", "forms_inaccessible", "forms_not_exists"], ) -def test_full_access_streams(caplog, requests_mock, ticket_forms_response, status_code, expected_n_streams, expected_warnings): - requests_mock.get("/api/v2/ticket_forms", status_code=status_code, json=ticket_forms_response) +def test_full_access_streams(caplog, requests_mock, ticket_forms_response, status_code, expected_n_streams, expected_warnings, reason): + requests_mock.get("/api/v2/ticket_forms", status_code=status_code, text=ticket_forms_response, reason=reason) result = SourceZendeskSupport().streams(config=TEST_CONFIG) assert len(result) == expected_n_streams logged_warnings = iter([record for record in caplog.records if record.levelname == "ERROR"]) @@ -163,19 +178,19 @@ def time_sleep_mock(mocker): def test_str2datetime(): expected = datetime.strptime(DATETIME_STR, DATETIME_FORMAT) - output = BaseSourceZendeskSupportStream.str2datetime(DATETIME_STR) + output = BaseZendeskSupportStream.str2datetime(DATETIME_STR) assert output == expected def test_datetime2str(): expected = datetime.strftime(DATETIME_FROM_STR.replace(tzinfo=pytz.UTC), DATETIME_FORMAT) - output = BaseSourceZendeskSupportStream.datetime2str(DATETIME_FROM_STR) + output = BaseZendeskSupportStream.datetime2str(DATETIME_FROM_STR) assert output == expected def test_str2unixtime(): expected = calendar.timegm(DATETIME_FROM_STR.utctimetuple()) - output = BaseSourceZendeskSupportStream.str2unixtime(DATETIME_STR) + output = BaseZendeskSupportStream.str2unixtime(DATETIME_STR) assert output == expected @@ -186,61 +201,20 @@ def test_check_start_time_param(): assert output == expected -def test_parse_next_page_number(requests_mock): - expected = dict(parse_qsl(urlparse(TICKET_EVENTS_STREAM_RESPONSE.get("next_page")).query)).get("page") - requests_mock.get(STREAM_URL, json=TICKET_EVENTS_STREAM_RESPONSE) - test_response = requests.get(STREAM_URL) - output = BaseSourceZendeskSupportStream._parse_next_page_number(test_response) - assert output == expected - - -def test_parse_next_page_number_from_empty_json(requests_mock): - requests_mock.get(STREAM_URL, text="", status_code=403) - test_response = requests.get(STREAM_URL) - output = BaseSourceZendeskSupportStream._parse_next_page_number(test_response) - assert output is None - - -def test_next_page_token(requests_mock): - # mocking the logic of next_page_token - if TICKET_EVENTS_STREAM_RESPONSE.get(END_OF_STREAM_KEY) is False: - expected = {"created_at": 1122334455} - else: - expected = None - requests_mock.get(STREAM_URL, json=TICKET_EVENTS_STREAM_RESPONSE) - test_response = requests.get(STREAM_URL) - output = TicketComments(**STREAM_ARGS).next_page_token(test_response) - assert expected == output - - @pytest.mark.parametrize( "stream_state, expected", [ # valid state, expect the value of the state - ({"updated_at": "2022-04-01"}, 1648771200), - # invalid state, expect the start_date from STREAM_ARGS - ({"updated_at": ""}, 1622505600), - ({"updated_at": None}, 1622505600), - ({"missing_cursor": "2022-04-01"}, 1622505600), + ({"generated_timestamp": 1648771200}, 1648771200), + (None, 1622505600), ], - ids=["state present", "empty string in state", "state is None", "cursor is not in the state object"], + ids=["state present", "state is None"], ) def test_check_stream_state(stream_state, expected): result = Tickets(**STREAM_ARGS).check_stream_state(stream_state) assert result == expected -def test_request_params(requests_mock): - expected = {"start_time": calendar.timegm(pendulum.parse(STREAM_ARGS.get("start_date")).utctimetuple()), "include": "comment_events"} - stream_state = None - requests_mock.get(STREAM_URL, json=TICKET_EVENTS_STREAM_RESPONSE) - test_response = requests.get(STREAM_URL) - next_page_token = TicketComments(**STREAM_ARGS).next_page_token(test_response) - output = TicketComments(**STREAM_ARGS).request_params(stream_state, None) - assert next_page_token is not None - assert expected == output - - def test_parse_response_from_empty_json(requests_mock): requests_mock.get(STREAM_URL, text="", status_code=403) test_response = requests.get(STREAM_URL) @@ -268,6 +242,8 @@ class TestAllStreams: (Groups), (Macros), (Organizations), + (Posts), + (OrganizationMemberships), (SatisfactionRatings), (SlaPolicies), (Tags), @@ -276,12 +252,16 @@ class TestAllStreams: (TicketFields), (TicketForms), (TicketMetrics), + (TicketSkips), (TicketMetricEvents), (Tickets), + (Topics), (Users), (Brands), (CustomRoles), (Schedules), + (AccountAttributes), + (AttributeDefinitions), ], ids=[ "AuditLogs", @@ -289,6 +269,8 @@ class TestAllStreams: "Groups", "Macros", "Organizations", + "Posts", + "OrganizationMemberships", "SatisfactionRatings", "SlaPolicies", "Tags", @@ -297,12 +279,16 @@ class TestAllStreams: "TicketFields", "TicketForms", "TicketMetrics", + "TicketSkips", "TicketMetricEvents", "Tickets", + "Topics", "Users", "Brands", "CustomRoles", "Schedules", + "AccountAttributes", + "AttributeDefinitions", ], ) def test_streams(self, expected_stream_cls): @@ -321,6 +307,8 @@ def test_streams(self, expected_stream_cls): (Groups, "groups"), (Macros, "macros"), (Organizations, "organizations"), + (Posts, "community/posts"), + (OrganizationMemberships, "organization_memberships"), (SatisfactionRatings, "satisfaction_ratings"), (SlaPolicies, "slas/policies.json"), (Tags, "tags"), @@ -329,12 +317,16 @@ def test_streams(self, expected_stream_cls): (TicketFields, "ticket_fields"), (TicketForms, "ticket_forms"), (TicketMetrics, "ticket_metrics"), + (TicketSkips, "skips.json"), (TicketMetricEvents, "incremental/ticket_metric_events"), - (Tickets, "incremental/tickets.json"), - (Users, "incremental/users.json"), + (Tickets, "incremental/tickets/cursor.json"), + (Users, "incremental/users/cursor.json"), + (Topics, "community/topics"), (Brands, "brands"), (CustomRoles, "custom_roles"), (Schedules, "business_hours/schedules.json"), + (AccountAttributes, "routing/attributes"), + (AttributeDefinitions, "routing/attributes/definitions"), ], ids=[ "AuditLogs", @@ -342,6 +334,8 @@ def test_streams(self, expected_stream_cls): "Groups", "Macros", "Organizations", + "Posts", + "OrganizationMemberships", "SatisfactionRatings", "SlaPolicies", "Tags", @@ -350,12 +344,16 @@ def test_streams(self, expected_stream_cls): "TicketFields", "TicketForms", "TicketMetrics", + "TicketSkips", "TicketMetricEvents", "Tickets", + "Topics", "Users", "Brands", "CustomRoles", "Schedules", + "AccountAttributes", + "AttributeDefinitions", ], ) def test_path(self, stream_cls, expected): @@ -370,18 +368,22 @@ class TestSourceZendeskSupportStream: [ (Macros), (Organizations), + (Posts), (Groups), (SatisfactionRatings), (TicketFields), (TicketMetrics), + (Topics) ], ids=[ "Macros", "Organizations", + "Posts", "Groups", "SatisfactionRatings", "TicketFields", "TicketMetrics", + "Topics" ], ) def test_parse_response(self, requests_mock, stream_cls): @@ -398,18 +400,22 @@ def test_parse_response(self, requests_mock, stream_cls): [ (Macros), (Organizations), + (Posts), (Groups), (SatisfactionRatings), (TicketFields), (TicketMetrics), + (Topics) ], ids=[ "Macros", "Organizations", + "Posts", "Groups", "SatisfactionRatings", "TicketFields", "TicketMetrics", + "Topics" ], ) def test_url_base(self, stream_cls): @@ -421,24 +427,28 @@ def test_url_base(self, stream_cls): "stream_cls, current_state, last_record, expected", [ (Macros, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), + (Posts, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), ( - Organizations, - {"updated_at": "2022-03-17T16:03:07Z"}, - {"updated_at": "2023-03-17T16:03:07Z"}, - {"updated_at": "2023-03-17T16:03:07Z"}, + Organizations, + {"updated_at": "2022-03-17T16:03:07Z"}, + {"updated_at": "2023-03-17T16:03:07Z"}, + {"updated_at": "2023-03-17T16:03:07Z"}, ), (Groups, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), (SatisfactionRatings, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), (TicketFields, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), (TicketMetrics, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), + (Topics, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), ], ids=[ "Macros", + "Posts", "Organizations", "Groups", "SatisfactionRatings", "TicketFields", "TicketMetrics", + "Topics" ], ) def test_get_updated_state(self, stream_cls, current_state, last_record, expected): @@ -450,20 +460,24 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte "stream_cls, expected", [ (Macros, None), + (Posts, None), (Organizations, None), (Groups, None), (TicketFields, None), ], ids=[ "Macros", + "Posts", "Organizations", "Groups", "TicketFields", ], ) - def test_next_page_token(self, stream_cls, expected): + def test_next_page_token(self, stream_cls, expected, mocker): stream = stream_cls(**STREAM_ARGS) - result = stream.next_page_token() + posts_response = mocker.Mock() + posts_response.json.return_value = {"next_page": None} + result = stream.next_page_token(response=posts_response) assert expected == result @pytest.mark.parametrize( @@ -483,7 +497,7 @@ def test_next_page_token(self, stream_cls, expected): ) def test_request_params(self, stream_cls, expected): stream = stream_cls(**STREAM_ARGS) - result = stream.request_params() + result = stream.request_params(stream_state={}) assert expected == result @@ -497,6 +511,8 @@ class TestSourceZendeskSupportFullRefreshStream: (CustomRoles), (Schedules), (UserSettingsStream), + (AccountAttributes), + (AttributeDefinitions) ], ids=[ "Tags", @@ -505,6 +521,8 @@ class TestSourceZendeskSupportFullRefreshStream: "CustomRoles", "Schedules", "UserSettingsStream", + "AccountAttributes", + "AttributeDefinitions", ], ) def test_url_base(self, stream_cls): @@ -521,6 +539,8 @@ def test_url_base(self, stream_cls): (CustomRoles), (Schedules), (UserSettingsStream), + (AccountAttributes), + (AttributeDefinitions), ], ids=[ "Tags", @@ -529,6 +549,8 @@ def test_url_base(self, stream_cls): "CustomRoles", "Schedules", "UserSettingsStream", + "AccountAttributes", + "AttributeDefinitions", ], ) def test_next_page_token(self, requests_mock, stream_cls): @@ -540,14 +562,16 @@ def test_next_page_token(self, requests_mock, stream_cls): assert output is None @pytest.mark.parametrize( - "stream_cls", + "stream_cls, expected_params", [ - (Tags), - (SlaPolicies), - (Brands), - (CustomRoles), - (Schedules), - (UserSettingsStream), + (Tags, {"page[size]": 100}), + (SlaPolicies, {}), + (Brands, {"page[size]": 100}), + (CustomRoles, {}), + (Schedules, {"page[size]": 100}), + (UserSettingsStream, {}), + (AccountAttributes, {}), + (AttributeDefinitions, {}), ], ids=[ "Tags", @@ -556,13 +580,14 @@ def test_next_page_token(self, requests_mock, stream_cls): "CustomRoles", "Schedules", "UserSettingsStream", + "AccountAttributes", + "AttributeDefinitions", ], ) - def test_request_params(self, stream_cls): - expected = {"page": 1, "per_page": 100} + def test_request_params(self, stream_cls, expected_params): stream = stream_cls(**STREAM_ARGS) result = stream.request_params(next_page_token=None, stream_state=None) - assert expected == result + assert expected_params == result class TestSourceZendeskSupportCursorPaginationStream: @@ -573,12 +598,16 @@ class TestSourceZendeskSupportCursorPaginationStream: (TicketForms, {}, {"updated_at": "2023-03-17T16:03:07Z"}, {"updated_at": "2023-03-17T16:03:07Z"}), (TicketMetricEvents, {}, {"time": "2024-03-17T16:03:07Z"}, {"time": "2024-03-17T16:03:07Z"}), (TicketAudits, {}, {"created_at": "2025-03-17T16:03:07Z"}, {"created_at": "2025-03-17T16:03:07Z"}), + (OrganizationMemberships, {}, {"updated_at": "2025-03-17T16:03:07Z"}, {"updated_at": "2025-03-17T16:03:07Z"}), + (TicketSkips, {}, {"updated_at": "2025-03-17T16:03:07Z"}, {"updated_at": "2025-03-17T16:03:07Z"}), ], ids=[ "GroupMemberships", "TicketForms", "TicketMetricEvents", "TicketAudits", + "OrganizationMemberships", + "TicketSkips", ], ) def test_get_updated_state(self, stream_cls, current_state, last_record, expected): @@ -591,20 +620,50 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte [ (GroupMemberships, {}, None), (TicketForms, {}, None), - (TicketMetricEvents, {}, None), + (TicketMetricEvents, { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + }, + }, + {"page[after]": ""}), (TicketAudits, {}, None), ( - TicketMetrics, - { - "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, - "links": { - "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", - "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + TicketMetrics, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + }, }, - }, - "", + {"page[after]": ""}, ), (SatisfactionRatings, {}, None), + ( + OrganizationMemberships, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + }, + }, + {"page[after]": ""}, + ), + ( + TicketSkips, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + }, + }, + {"page[after]": ""}, + ), + ], ids=[ "GroupMemberships", @@ -613,11 +672,12 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte "TicketAudits", "TicketMetrics", "SatisfactionRatings", + "OrganizationMemberships", + "TicketSkips", ], ) def test_next_page_token(self, requests_mock, stream_cls, response, expected): stream = stream_cls(**STREAM_ARGS) - # stream_name = snake_case(stream.__class__.__name__) requests_mock.get(STREAM_URL, json=response) test_response = requests.get(STREAM_URL) output = stream.next_page_token(test_response) @@ -630,12 +690,16 @@ def test_next_page_token(self, requests_mock, stream_cls, response, expected): (TicketForms, 1622505600), (TicketMetricEvents, 1622505600), (TicketAudits, 1622505600), + (OrganizationMemberships, 1622505600), + (TicketSkips, 1622505600), ], ids=[ "GroupMemberships", "TicketForms", "TicketMetricEvents", "TicketAudits", + "OrganizationMemberships", + "TicketSkips" ], ) def test_check_stream_state(self, stream_cls, expected): @@ -646,12 +710,14 @@ def test_check_stream_state(self, stream_cls, expected): @pytest.mark.parametrize( "stream_cls, expected", [ - (GroupMemberships, {"page": 1, "per_page": 100, "sort_by": "asc", "start_time": 1622505600}), + (GroupMemberships, {'page[size]': 100, 'sort_by': 'asc', 'start_time': 1622505600}), (TicketForms, {"start_time": 1622505600}), - (TicketMetricEvents, {"start_time": 1622505600}), + (TicketMetricEvents, {'page[size]': 100, 'start_time': 1622505600}), (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 1000}), - (SatisfactionRatings, {"page": 1, "per_page": 100, "sort_by": "asc", "start_time": 1622505600}), + (SatisfactionRatings, {'page[size]': 100, 'sort_by': 'asc', 'start_time': 1622505600}), (TicketMetrics, {"page[size]": 100, "start_time": 1622505600}), + (OrganizationMemberships, {"page[size]": 100, "start_time": 1622505600}), + (TicketSkips, {"page[size]": 100, "start_time": 1622505600}), ], ids=[ "GroupMemberships", @@ -660,6 +726,8 @@ def test_check_stream_state(self, stream_cls, expected): "TicketAudits", "SatisfactionRatings", "TicketMetrics", + "OrganizationMemberships", + "TicketSkips", ], ) def test_request_params(self, stream_cls, expected): @@ -686,22 +754,6 @@ def test_check_start_time_param(self, stream_cls): result = stream.check_start_time_param(expected) assert result == expected - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Users, "incremental/users.json"), - (Tickets, "incremental/tickets.json"), - ], - ids=[ - "Users", - "Tickets", - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(**STREAM_ARGS) - result = stream.path() - assert result == expected - @pytest.mark.parametrize( "stream_cls", [ @@ -863,7 +915,7 @@ def test_event_type(self, stream_cls, expected): def test_read_tickets_stream(requests_mock): requests_mock.get( - "https://subdomain.zendesk.com/api/v2/incremental/tickets.json", + "https://subdomain.zendesk.com/api/v2/incremental/tickets/cursor.json", json={ "tickets": [ {"custom_fields": []}, @@ -876,7 +928,8 @@ def test_read_tickets_stream(requests_mock): {"id": 360023712840, "value": False}, ] }, - ] + ], + "end_of_stream": True }, ) @@ -894,3 +947,80 @@ def test_read_tickets_stream(requests_mock): ] }, ] + + +def test_read_post_comment_votes_stream(requests_mock): + post_response = { + "posts": [ + {"id": 7253375870607, "title": "Test_post", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T00:00:00Z"} + ] + } + requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts", json=post_response) + + post_comments_response = { + "comments": [ + {"author_id": 89567, "body": "Test_comment for Test_post", "id": 35467, "post_id": 7253375870607} + ] + } + requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts/7253375870607/comments", json=post_comments_response) + + votes = [{"id": 35467, "user_id": 888887, "value": -1}] + requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts/7253375870607/comments/35467/votes", + json={"votes": votes}) + stream = PostCommentVotes(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + records = read_full_refresh(stream) + assert records == votes + + +def test_read_ticket_metric_events_request_params(requests_mock): + first_page_response = { + "ticket_metric_events": [ + {"id": 1, "ticket_id": 123, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2020-01-01T01:00:00Z"}, + {"id": 2, "ticket_id": 123, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2020-01-01T01:00:00Z"}, + {"id": 3, "ticket_id": 123, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2020-01-01T01:00:00Z"}, + {"id": 4, "ticket_id": 123, "metric": "requester_wait_time", "instance_id": 1, "type": "activate", "time": "2020-01-01T01:00:00Z"} + ], + "meta": { + "has_more": True, + "after_cursor": "", + "before_cursor": "" + }, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800", + "next": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bafter%5D=&page%5Bsize%5D=100&start_time=1577836800" + }, + "end_of_stream": False + } + + second_page_response = { + "ticket_metric_events": [ + {"id": 5163373143183, "ticket_id": 130, "metric": "reply_time", "instance_id": 1, "type": "fulfill", "time": "2022-07-18T16:39:48Z"}, + {"id": 5163373143311, "ticket_id": 130, "metric": "requester_wait_time", "instance_id": 0, "type": "measure", "time": "2022-07-18T16:39:48Z"} + ], + "meta": { + "has_more": False, + "after_cursor": "", + "before_cursor": "" + }, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800", + "next": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800" + }, + "end_of_stream": True + } + request_history = requests_mock.get("https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events", [ + {"json": first_page_response}, {"json": second_page_response}]) + stream = TicketMetricEvents(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + read_full_refresh(stream) + assert request_history.call_count == 2 + assert request_history.last_request.qs == {"page[after]": [""], "page[size]": ["100"], "start_time": ["1577836800"]} + + +@pytest.mark.parametrize("status_code",[(403),(404)]) +def test_read_tickets_comment(requests_mock, status_code): + request_history = requests_mock.get( + "https://subdomain.zendesk.com/api/v2/incremental/ticket_events.json", status_code=status_code, json={"error": "wrong permissions"} + ) + stream = TicketComments(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + read_full_refresh(stream) + assert request_history.call_count == 1 diff --git a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile index 2292ad539d4f..eb5a50eb68fa 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/source-zendesk-talk diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl index 4e1934c30313..056bf233dc49 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl @@ -3,7 +3,7 @@ {"stream": "addresses", "data": {"id": 360000049296, "name": "Fake Zendesk 999", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "ADad90fc569206d6cbe5f5a6f160c01516"}, "emitted_at": 1674159463525} {"stream": "addresses", "data": {"id": 360000047915, "name": "Fake Zendesk 998", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "ADa89d87601b4f38b45ca172ba36bc4c36"}, "emitted_at": 1674159463525} {"stream": "addresses", "data": {"id": 360000049276, "name": "Fake Zendesk 997", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "AD1b03f6250ae793c562f9290a47404e5b"}, "emitted_at": 1674159463525} -{"stream": "agents_activity", "data": {"name": "Team Airbyte", "agent_id": 360786799676, "via": "client", "avatar_url": "https://d3v-airbyte.zendesk.com/images/2016/default-avatar-80.png", "forwarding_number": null, "average_talk_time": 0, "calls_accepted": 0, "calls_denied": 0, "calls_missed": 0, "online_time": 0, "available_time": 0, "total_call_duration": 0, "total_talk_time": 0, "total_wrap_up_time": 0, "away_time": 0, "call_status": null, "agent_state": "offline", "transfers_only_time": 0, "average_wrap_up_time": 0, "accepted_transfers": 0, "started_transfers": 0, "calls_put_on_hold": 0, "average_hold_time": 0, "total_hold_time": 0, "started_third_party_conferences": 0, "accepted_third_party_conferences": 0}, "emitted_at": 1674159464163} +{"stream": "agents_activity", "data": {"name": "Team Airbyte", "agent_id": 360786799676, "via": "client", "avatar_url": "https://d3v-airbyte.zendesk.com/system/photos/7282824912911/Airbyte_logo_220x220_thumb.png", "forwarding_number": null, "average_talk_time": 0, "calls_accepted": 0, "calls_denied": 0, "calls_missed": 0, "online_time": 0, "available_time": 0, "total_call_duration": 0, "total_talk_time": 0, "total_wrap_up_time": 0, "away_time": 0, "call_status": null, "agent_state": "offline", "transfers_only_time": 0, "average_wrap_up_time": 0, "accepted_transfers": 0, "started_transfers": 0, "calls_put_on_hold": 0, "average_hold_time": 0, "total_hold_time": 0, "started_third_party_conferences": 0, "accepted_third_party_conferences": 0}, "emitted_at": 1688470692771} {"stream": "calls", "data": {"id": 360088814475, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:15Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "failed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 18, "exceeded_queue_wait_time": null, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": null, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": null, "ivr_routed_to": null, "callback": null, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["silence"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472100} {"stream": "calls", "data": {"id": 360120314196, "created_at": "2021-10-20T15:16:31Z", "updated_at": "2021-10-20T15:56:54Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 8, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472101} {"stream": "calls", "data": {"id": 360121169675, "created_at": "2021-10-20T15:16:42Z", "updated_at": "2021-10-20T15:57:03Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 7, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472101} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml index e86a4d70bc46..98b742f21721 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: api connectorType: source definitionId: c8630570-086d-4a40-99ae-ea5b18673071 - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.9 dockerRepository: airbyte/source-zendesk-talk githubIssueLabel: source-zendesk-talk icon: zendesk-talk.svg @@ -21,4 +21,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-talk tags: - language:python + ab_internal: + sl: 200 + ql: 400 + supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt b/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt index 9ce85523c234..7b9114ed5867 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt +++ b/airbyte-integrations/connectors/source-zendesk-talk/requirements.txt @@ -1,3 +1,2 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zendesk-talk/setup.py b/airbyte-integrations/connectors/source-zendesk-talk/setup.py index 928bd57d3e2d..e0e910f6461b 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/setup.py @@ -8,6 +8,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8", diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json index 865948b49771..b205a1f064c2 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json @@ -57,6 +57,18 @@ "title": "Access Token", "description": "The value of the API token generated. See the docs for more information.", "airbyte_secret": true + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "Client ID", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "Client Secret", + "airbyte_secret": true } } } diff --git a/airbyte-integrations/connectors/source-zenefits/metadata.yaml b/airbyte-integrations/connectors/source-zenefits/metadata.yaml index 472672e91fee..9c96ce0d6f3d 100644 --- a/airbyte-integrations/connectors/source-zenefits/metadata.yaml +++ b/airbyte-integrations/connectors/source-zenefits/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/zenefits tags: - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zenefits/requirements.txt b/airbyte-integrations/connectors/source-zenefits/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zenefits/requirements.txt +++ b/airbyte-integrations/connectors/source-zenefits/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zenefits/setup.py b/airbyte-integrations/connectors/source-zenefits/setup.py index 23b7ca3be65e..cdd39928c1ce 100644 --- a/airbyte-integrations/connectors/source-zenefits/setup.py +++ b/airbyte-integrations/connectors/source-zenefits/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zenloop/Dockerfile b/airbyte-integrations/connectors/source-zenloop/Dockerfile index 00d3f3ecc3a8..d88a8850387a 100644 --- a/airbyte-integrations/connectors/source-zenloop/Dockerfile +++ b/airbyte-integrations/connectors/source-zenloop/Dockerfile @@ -34,5 +34,5 @@ COPY source_zenloop ./source_zenloop ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-zenloop diff --git a/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml index de6f12368cf2..e5b5f839bec9 100644 --- a/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml @@ -24,6 +24,9 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + "answers": [ "states", 0, "cursor", "inserted_at" ] + "answers_survey_group": [ "states", 0, "cursor", "inserted_at" ] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zenloop/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json index ad6fd553b49a..9861c74223c7 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/abnormal_state.json @@ -2,15 +2,43 @@ { "type": "STREAM", "stream": { - "stream_state": { "inserted_at": "2099-08-18T08:35:49.540Z" }, - "stream_descriptor": { "name": "answers" } + "stream_state": { + "states": [ + { + "partition": { + "id": "WlRBek9ESTFNREl0TmpJMk9DMDBOR0V4TFRoaE16UXRZV1UyWW1SbU56WTNPVGRs", + "parent_slice": {} + }, + "cursor": { + "inserted_at": "2099-08-18T08:35:49.540Z" + } + } + ] + }, + "stream_descriptor": { + "name": "answers" + } } }, { "type": "STREAM", "stream": { - "stream_state": { "inserted_at": "2099-08-18T08:35:49.540Z" }, - "stream_descriptor": { "name": "answers_survey_group" } + "stream_state": { + "states": [ + { + "partition": { + "id": "TnpKaE1UVmhObUV0WkdFME15MDBZMkUyTFRsalpXRXROamt5TkRVd05EZzVOelEy", + "parent_slice": {} + }, + "cursor": { + "inserted_at": "2099-08-18T08:35:49.540Z" + } + } + ] + }, + "stream_descriptor": { + "name": "answers_survey_group" + } } } ] diff --git a/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json index 9cebc455f98f..b9d5fa07108e 100644 --- a/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-zenloop/integration_tests/sample_state.json @@ -2,15 +2,43 @@ { "type": "STREAM", "stream": { - "stream_state": { "inserted_at": "2021-08-18T08:35:49.540Z" }, - "stream_descriptor": { "name": "answers" } + "stream_state": { + "states": [ + { + "partition": { + "id": "WlRBek9ESTFNREl0TmpJMk9DMDBOR0V4TFRoaE16UXRZV1UyWW1SbU56WTNPVGRs", + "parent_slice": {} + }, + "cursor": { + "inserted_at": "2021-08-18T08:35:49.540Z" + } + } + ] + }, + "stream_descriptor": { + "name": "answers" + } } }, { "type": "STREAM", "stream": { - "stream_state": { "inserted_at": "2021-08-18T08:35:49.540Z" }, - "stream_descriptor": { "name": "answers_survey_group" } + "stream_state": { + "states": [ + { + "partition": { + "id": "TnpKaE1UVmhObUV0WkdFME15MDBZMkUyTFRsalpXRXROamt5TkRVd05EZzVOelEy", + "parent_slice": {} + }, + "cursor": { + "inserted_at": "2021-08-18T08:35:49.540Z" + } + } + ] + }, + "stream_descriptor": { + "name": "answers_survey_group" + } } } ] diff --git a/airbyte-integrations/connectors/source-zenloop/metadata.yaml b/airbyte-integrations/connectors/source-zenloop/metadata.yaml index e225463b8135..7c364bba5b83 100644 --- a/airbyte-integrations/connectors/source-zenloop/metadata.yaml +++ b/airbyte-integrations/connectors/source-zenloop/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: f1e4c7f6-db5c-4035-981f-d35ab4998794 - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.10 dockerRepository: airbyte/source-zenloop githubIssueLabel: source-zenloop icon: zenloop.svg @@ -21,4 +21,8 @@ data: tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 300 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zenloop/requirements.txt b/airbyte-integrations/connectors/source-zenloop/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zenloop/requirements.txt +++ b/airbyte-integrations/connectors/source-zenloop/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zenloop/setup.py b/airbyte-integrations/connectors/source-zenloop/setup.py index 6935d723bde5..be53c38ee558 100644 --- a/airbyte-integrations/connectors/source-zenloop/setup.py +++ b/airbyte-integrations/connectors/source-zenloop/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk>=0.44.1", ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", "responses~=0.13.3", ] diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/__init__.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/__init__.py index 222f86afc829..ab9590ca6b4a 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/__init__.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py b/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py index e4b981bdbc38..909e0bfc7054 100644 --- a/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py +++ b/airbyte-integrations/connectors/source-zenloop/source_zenloop/components.py @@ -6,9 +6,8 @@ from dataclasses import dataclass from typing import Iterable -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.partition_routers import SubstreamPartitionRouter -from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, StreamSlice @dataclass @@ -16,7 +15,7 @@ class ZenloopPartitionRouter(SubstreamPartitionRouter): config: Config - def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]: + def stream_slices(self) -> Iterable[StreamSlice]: """ config_parent_field : parent field name in config @@ -29,7 +28,7 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Itera custom_stream_state_value = self.config.get(parent_field) if not custom_stream_state_value: - yield from super().stream_slices(sync_mode, stream_state) + yield from super().stream_slices() else: for parent_stream_config in self.parent_stream_configs: stream_state_field = parent_stream_config.partition_field.eval(self.config) diff --git a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/abnormal_state.json index 94e0187a5dc6..9078244cd4f7 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-zoho-crm/integration_tests/abnormal_state.json @@ -3,140 +3,148 @@ "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_leads_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_notes_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_contacts_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_accounts_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_activities_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_tasks_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_calls_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_deals_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_events_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_vendors_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_products_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_quotes_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { - "stream_descriptor": { "name": "incremental_sales__orders_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_descriptor": { + "name": "incremental_sales__orders_zoho_crm_stream" + }, + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { - "stream_descriptor": { "name": "incremental_purchase__orders_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_descriptor": { + "name": "incremental_purchase__orders_zoho_crm_stream" + }, + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { - "stream_descriptor": { "name": "incremental_price__books_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_descriptor": { + "name": "incremental_price__books_zoho_crm_stream" + }, + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_solutions_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_cases_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_invoices_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { - "stream_descriptor": { "name": "incremental_attachments_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_descriptor": { + "name": "incremental_attachments_zoho_crm_stream" + }, + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } }, { "type": "STREAM", "stream": { "stream_descriptor": { "name": "incremental_campaigns_zoho_crm_stream" }, - "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00"} + "stream_state": { "Modified_Time": "2220-03-07T11:30:00+00:00" } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml b/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml index 272aebbfcbd2..915c75b1b4e3 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml +++ b/airbyte-integrations/connectors/source-zoho-crm/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 4942d392-c7b5-4271-91f9-3b4f4e51eb3e dockerImageTag: 0.1.2 dockerRepository: airbyte/source-zoho-crm + documentationUrl: https://docs.airbyte.com/integrations/sources/zoho-crm githubIssueLabel: source-zoho-crm icon: zohocrm.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/zoho-crm + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zoho-crm/requirements.txt b/airbyte-integrations/connectors/source-zoho-crm/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/requirements.txt +++ b/airbyte-integrations/connectors/source-zoho-crm/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zoho-crm/setup.py b/airbyte-integrations/connectors/source-zoho-crm/setup.py index 932664b2453d..4627c0115e63 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/setup.py +++ b/airbyte-integrations/connectors/source-zoho-crm/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zoom/Dockerfile b/airbyte-integrations/connectors/source-zoom/Dockerfile index d8781d6a788b..2fcce7c308da 100644 --- a/airbyte-integrations/connectors/source-zoom/Dockerfile +++ b/airbyte-integrations/connectors/source-zoom/Dockerfile @@ -36,5 +36,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-zoom diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json index 6a603fda8000..72a8ba046730 100644 --- a/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/invalid_config.json @@ -1,3 +1,5 @@ { - "jwt_token": "dummy" + "client_id": "client_id", + "client_secret": "client_secret", + "authorization_endpoint": "https://zoom.us/oauth/token" } diff --git a/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json index f875ad8416c6..fa709018b12f 100644 --- a/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-zoom/integration_tests/sample_config.json @@ -1,3 +1,6 @@ { - "jwt_token": "abcd" + "account_id": "account_id", + "client_id": "client_id", + "client_secret": "client_secret", + "authorization_endpoint": "https://zoom.us/oauth/token" } diff --git a/airbyte-integrations/connectors/source-zoom/metadata.yaml b/airbyte-integrations/connectors/source-zoom/metadata.yaml index 831d157fe7d1..9d5eb4f875ba 100644 --- a/airbyte-integrations/connectors/source-zoom/metadata.yaml +++ b/airbyte-integrations/connectors/source-zoom/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: cbfd9856-1322-44fb-bcf1-0b39b7a8e92e - dockerImageTag: 0.1.1 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-zoom githubIssueLabel: source-zoom icon: zoom.svg @@ -14,8 +14,12 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.io/integrations/sources/zoom + documentationUrl: https://docs.airbyte.com/integrations/sources/zoom tags: - language:low-code - language:python + ab_internal: + sl: 100 + ql: 200 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zoom/requirements.txt b/airbyte-integrations/connectors/source-zoom/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zoom/requirements.txt +++ b/airbyte-integrations/connectors/source-zoom/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zoom/setup.py b/airbyte-integrations/connectors/source-zoom/setup.py index fd070f71454a..edc76fde557e 100644 --- a/airbyte-integrations/connectors/source-zoom/setup.py +++ b/airbyte-integrations/connectors/source-zoom/setup.py @@ -10,9 +10,9 @@ ] TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/components.py b/airbyte-integrations/connectors/source-zoom/source_zoom/components.py new file mode 100644 index 000000000000..e2f9a8af12f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/components.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import base64 +import time +from dataclasses import dataclass +from http import HTTPStatus +from typing import Any, Mapping, Union + +import requests +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.interpolation import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config +from requests import HTTPError + +# https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response +# The Bearer token generated by server-to-server token will expire in one hour +BEARER_TOKEN_EXPIRES_IN = 3590 + + +class SingletonMeta(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + """ + Possible changes to the value of the `__init__` argument do not affect + the returned instance. + """ + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +@dataclass +class ServerToServerOauthAuthenticator(NoAuth): + config: Config + account_id: Union[InterpolatedString, str] + client_id: Union[InterpolatedString, str] + client_secret: Union[InterpolatedString, str] + authorization_endpoint: Union[InterpolatedString, str] + + _instance = None + _generate_token_time = 0 + _access_token = None + _grant_type = "account_credentials" + + def __post_init__(self, parameters: Mapping[str, Any]): + self._account_id = InterpolatedString.create(self.account_id, parameters=parameters).eval(self.config) + self._client_id = InterpolatedString.create(self.client_id, parameters=parameters).eval(self.config) + self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters).eval(self.config) + self._authorization_endpoint = InterpolatedString.create(self.authorization_endpoint, parameters=parameters).eval(self.config) + + def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: + """Attach the page access token to params to authenticate on the HTTP request""" + if self._access_token is None or ((time.time() - self._generate_token_time) > BEARER_TOKEN_EXPIRES_IN): + self._generate_token_time = time.time() + self._access_token = self.generate_access_token() + headers = {"Authorization": f"Bearer {self._access_token}", "Content-type": "application/json"} + request.headers.update(headers) + + return request + + @property + def auth_header(self) -> dict[str, str]: + return {"Authorization": f"Bearer {self.token}", "Content-type": "application/json"} + + @property + def token(self) -> str: + return self._access_token + + def generate_access_token(self) -> str: + self._generate_token_time = time.time() + try: + token = base64.b64encode(f"{self._client_id}:{self._client_secret}".encode("ascii")).decode("utf-8") + headers = {"Authorization": f"Basic {token}", "Content-type": "application/json"} + rest = requests.post( + url=f"{self._authorization_endpoint}?grant_type={self._grant_type}&account_id={self._account_id}", headers=headers + ) + if rest.status_code != HTTPStatus.OK: + raise HTTPError(rest.text) + return rest.json().get("access_token") + except Exception as e: + raise Exception(f"Error while generating access token: {e}") from e diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/manifest.yaml b/airbyte-integrations/connectors/source-zoom/source_zoom/manifest.yaml index 92c993f37ef9..21ec25e9ff0b 100644 --- a/airbyte-integrations/connectors/source-zoom/source_zoom/manifest.yaml +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/manifest.yaml @@ -1,12 +1,17 @@ version: "0.29.0" definitions: + # Server to Server Oauth Authenticator requester: - url_base: "https://api.zoom.us/v2/" + url_base: "https://api.zoom.us/v2" http_method: "GET" authenticator: - type: BearerAuthenticator - api_token: "{{ config['jwt_token'] }}" + class_name: source_zoom.components.ServerToServerOauthAuthenticator + client_id: "{{ config['client_id'] }}" + account_id: "{{ config['account_id'] }}" + client_secret: "{{ config['client_secret'] }}" + authorization_endpoint: "{{ config['authorization_endpoint'] }}" + grant_type: "account_credentials" zoom_paginator: type: DefaultPaginator diff --git a/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml b/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml index a8170e08c3b7..d91cbe0bd6c9 100644 --- a/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml +++ b/airbyte-integrations/connectors/source-zoom/source_zoom/spec.yaml @@ -4,10 +4,26 @@ connectionSpecification: title: Zoom Spec type: object required: - - jwt_token + - account_id + - client_id + - client_secret + - authorization_endpoint additionalProperties: true properties: - jwt_token: + account_id: type: string - description: JWT Token + order: 0 + description: 'The account ID for your Zoom account. You can find this in the Zoom Marketplace under the "Manage" tab for your app.' + client_id: + type: string + order: 1 + description: 'The client ID for your Zoom app. You can find this in the Zoom Marketplace under the "Manage" tab for your app.' + client_secret: + type: string + order: 2 + description: 'The client secret for your Zoom app. You can find this in the Zoom Marketplace under the "Manage" tab for your app.' airbyte_secret: true + authorization_endpoint: + type: string + order: 3 + default: "https://zoom.us/oauth/token" diff --git a/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py b/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py new file mode 100755 index 000000000000..7706d1eaa079 --- /dev/null +++ b/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import base64 +import unittest +from http import HTTPStatus + +import requests +import requests_mock +from source_zoom.components import ServerToServerOauthAuthenticator + + +class TestOAuthClient(unittest.TestCase): + def test_generate_access_token(self): + except_access_token = "rc-test-token" + except_token_response = {"access_token": except_access_token} + + config = { + "account_id": "rc-asdfghjkl", + "client_id": "rc-123456789", + "client_secret": "rc-test-secret", + "authorization_endpoint": "https://example.zoom.com/oauth/token", + "grant_type": "account_credentials" + } + parameters = config + client = ServerToServerOauthAuthenticator(config=config, + account_id=config["account_id"], + client_id=config["client_id"], + client_secret=config["client_secret"], + grant_type=config["grant_type"], + authorization_endpoint=config["authorization_endpoint"], + parameters=parameters) + + # Encode the client credentials in base64 + token = base64.b64encode(f'{config.get("client_id")}:{config.get("client_secret")}'.encode('ascii')).decode('utf-8') + + # Define the headers that should be sent in the request + headers = {'Authorization': f'Basic {token}', + 'Content-type': 'application/json'} + + # Define the URL containing the grant_type and account_id as query parameters + url = f'{config.get("authorization_endpoint")}?grant_type={config.get("grant_type")}&account_id={config.get("account_id")}' + + with requests_mock.Mocker() as m: + # Mock the requests.post call with the expected URL, headers and token response + m.post(url, json=except_token_response, request_headers=headers, status_code=HTTPStatus.OK) + + # Call the generate_access_token function and assert it returns the expected access token + self.assertEqual(client.generate_access_token(), except_access_token) + + # Test case when the endpoint has some error, like a timeout + with requests_mock.Mocker() as m: + m.post(url, exc=requests.exceptions.RequestException) + with self.assertRaises(Exception) as cm: + client.generate_access_token() + self.assertIn("Error while generating access token", str(cm.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/airbyte-integrations/connectors/source-zuora/metadata.yaml b/airbyte-integrations/connectors/source-zuora/metadata.yaml index cd23252b0f6a..4298aba664ae 100644 --- a/airbyte-integrations/connectors/source-zuora/metadata.yaml +++ b/airbyte-integrations/connectors/source-zuora/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: api connectorType: source definitionId: 3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5 dockerImageTag: 0.1.3 dockerRepository: airbyte/source-zuora + documentationUrl: https://docs.airbyte.com/integrations/sources/zuora githubIssueLabel: source-zuora icon: zuora.svg license: MIT @@ -14,7 +18,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/zuora + supportLevel: community tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zuora/requirements.txt b/airbyte-integrations/connectors/source-zuora/requirements.txt index cc57334ef619..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-zuora/requirements.txt +++ b/airbyte-integrations/connectors/source-zuora/requirements.txt @@ -1,2 +1 @@ --e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-zuora/setup.py b/airbyte-integrations/connectors/source-zuora/setup.py index 00bc550d91d2..6fb6e922ce98 100644 --- a/airbyte-integrations/connectors/source-zuora/setup.py +++ b/airbyte-integrations/connectors/source-zuora/setup.py @@ -11,7 +11,8 @@ ] TEST_REQUIREMENTS = [ - "connector-acceptance-test", + "requests-mock~=1.9.3", + "pytest-mock~=3.6.1", ] setup( diff --git a/airbyte-integrations/connectors/tasks.py b/airbyte-integrations/connectors/tasks.py index 3323183fedcc..5463f1f16809 100644 --- a/airbyte-integrations/connectors/tasks.py +++ b/airbyte-integrations/connectors/tasks.py @@ -1,6 +1,7 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import os import shutil import tempfile diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml b/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml index 778f70484891..6c0b76c003d1 100644 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml +++ b/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/customer-io tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-harness-source/metadata.yaml b/airbyte-integrations/connectors/third-party/farosai/airbyte-harness-source/metadata.yaml index 9b13f4c13296..0228cb4a3afc 100644 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-harness-source/metadata.yaml +++ b/airbyte-integrations/connectors/third-party/farosai/airbyte-harness-source/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/harness tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-jenkins-source/metadata.yaml b/airbyte-integrations/connectors/third-party/farosai/airbyte-jenkins-source/metadata.yaml index d825d0c3e09c..124a2a1737e0 100644 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-jenkins-source/metadata.yaml +++ b/airbyte-integrations/connectors/third-party/farosai/airbyte-jenkins-source/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/jenkins tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-pagerduty-source/metadata.yaml b/airbyte-integrations/connectors/third-party/farosai/airbyte-pagerduty-source/metadata.yaml index 729b30b713a9..4ff72efd5cc1 100644 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-pagerduty-source/metadata.yaml +++ b/airbyte-integrations/connectors/third-party/farosai/airbyte-pagerduty-source/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pagerduty tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-victorops-source/metadata.yaml b/airbyte-integrations/connectors/third-party/farosai/airbyte-victorops-source/metadata.yaml index 09219a297e32..5fed1a8ee0c5 100644 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-victorops-source/metadata.yaml +++ b/airbyte-integrations/connectors/third-party/farosai/airbyte-victorops-source/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/victorops tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/third-party/ghcr/streamr-airbyte-connector/metadata.yaml b/airbyte-integrations/connectors/third-party/ghcr/streamr-airbyte-connector/metadata.yaml index a4ff5224b957..557420199661 100644 --- a/airbyte-integrations/connectors/third-party/ghcr/streamr-airbyte-connector/metadata.yaml +++ b/airbyte-integrations/connectors/third-party/ghcr/streamr-airbyte-connector/metadata.yaml @@ -17,4 +17,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/destinations/streamr tags: - language:unknown + ab_internal: + sl: 100 + ql: 100 + supportLevel: community metadataSpecVersion: "1.0" diff --git a/build.gradle b/build.gradle index 14981691db53..3a94a394ec5e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,6 @@ import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.github.spotbugs.snom.SpotBugsTask +import ru.vyarus.gradle.plugin.python.task.PythonTask // The buildscript block defines dependencies in order for .gradle file evaluation. // This is separate from application dependencies. @@ -28,7 +29,8 @@ buildscript { plugins { id 'base' id 'pmd' - id 'com.diffplug.spotless' version '6.12.0' + id 'com.diffplug.spotless' version '6.20.0' + id 'com.github.node-gradle.node' version '3.5.1' id 'com.github.hierynomus.license' version '0.16.1' id 'com.github.spotbugs' version '5.0.13' // The distribution plugin has been added to address the an issue with the copyGeneratedTar @@ -37,6 +39,7 @@ plugins { id 'distribution' id 'version-catalog' id 'maven-publish' + id 'ru.vyarus.use-python' } apply from: "$rootDir/publish-repositories.gradle" @@ -132,6 +135,7 @@ def createSpotlessTarget = { pattern -> 'airbyte-webapp', // The webapp module uses its own auto-formatter, so spotless is not necessary here 'airbyte-webapp-e2e-tests', // This module also uses its own auto-formatter 'airbyte-connector-builder-server/connector_builder/generated', // autogenerated code doesn't need to be formatted + 'airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', // These are deliberately invalid and unformattable. ] if (System.getenv().containsKey("SUB_BUILD")) { @@ -141,6 +145,15 @@ def createSpotlessTarget = { pattern -> return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" }) } +node { + download = true + version = '18.16.1' + npmVersion = '9.5.1' + // when setting both these directories, npm and node will be in separate directories + workDir = file("${buildDir}/nodejs") + npmWorkDir = file("${buildDir}/npm") +} + spotless { java { target createSpotlessTarget('**/*.java') @@ -165,8 +178,15 @@ spotless { target createSpotlessTarget(['**/*.yaml', '**/*.json']) prettier() + .npmExecutable("${tasks.named('npmSetup').get().npmDir.get()}/bin/npm") // get the npm executable path from gradle-node-plugin + .nodeExecutable("${tasks.named('nodeSetup').get().nodeDir.get()}/bin/node") // get the node executable path from gradle-node-plugin } } + +tasks.named('spotlessStyling').configure { + it.dependsOn('nodeSetup', 'npmSetup') +} + check.dependsOn 'spotlessApply' @SuppressWarnings('GroovyAssignabilityCheck') @@ -214,6 +234,63 @@ allprojects { version = rootProject.ext.version } +def getCDKTargetVersion() { + def props = new Properties() + file("airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties").withInputStream { props.load(it) } + return props.getProperty('version') +} +def getLatestFileModifiedTimeFromFiles(files) { + if (files.isEmpty()) { + return null + } + return files.findAll { it.isFile() } + .collect { it.lastModified() } + .max() +} +def checkCDKJarExists(requiredSnapshotVersion) { + if (requiredSnapshotVersion == null) { + // Connector does not require CDK snapshot. + return + } + final boolean checkFileChanges = true + final cdkTargetVersion = getCDKTargetVersion() + if (requiredSnapshotVersion != cdkTargetVersion) { + if (!cdkTargetVersion.contains("-SNAPSHOT")) { + throw new GradleException( + "CDK JAR version is not publishing snapshot but connector requires version ${requiredSnapshotVersion}.\n" + + "Please check that the version in the CDK properties file matches the connector build.gradle." + ) + } + throw new GradleException( + "CDK JAR version ${cdkTargetVersion} does not match connector's required version ${requiredSnapshotVersion}.\n" + + "Please check that the version in the CDK properties file matches the connector build.gradle." + ) + } + + def cdkJar = file("${System.properties['user.home']}/.m2/repository/io/airbyte/airbyte-cdk/${cdkTargetVersion}/airbyte-cdk-${cdkTargetVersion}.jar") + if (!cdkJar.exists()) { + println("WARNING: CDK JAR does not exist at ${cdkJar.path}.\nPlease run './gradlew :airbyte-cdk:java:airbyte-cdk:build'.") + } + if (checkFileChanges) { + def latestJavaFileTimestamp = getLatestFileModifiedTimeFromFiles(file("${rootDir}/airbyte-cdk/java/airbyte-cdk/src").listFiles().findAll { it.isFile() }) + if (cdkJar.lastModified() < latestJavaFileTimestamp) { + throw new GradleException("CDK JAR is out of date. Please run './gradlew :airbyte-cdk:java:airbyte-cdk:build'.") + } + } +} +def getCDKSnapshotRequirement(dependenciesList) { + def cdkSnapshotRequirement = dependenciesList.find { + it.requested instanceof ModuleComponentSelector && + it.requested.group == 'io.airbyte' && + it.requested.module == 'airbyte-cdk' && + it.requested.version.endsWith('-SNAPSHOT') + } + if (cdkSnapshotRequirement == null) { + return null + } else { + return cdkSnapshotRequirement.requested.version + } +} // Java projects common configurations subprojects { subproj -> @@ -289,6 +366,23 @@ subprojects { subproj -> sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 + if (subproj.name.startsWith("source-") || subproj.name.startsWith("destination-")) { + // This is a Java connector project. + + // Evaluate CDK project before evaluating the connector. + evaluationDependsOn(':airbyte-cdk:java:airbyte-cdk') + + if (!gradle.startParameter.taskNames.any { it.contains(':airbyte-cdk:') } && + gradle.startParameter.taskNames.any { it.contains(':source-') || it.contains(':destination-') }) { + // We are building a connector. Warn if the CDK JAR is missing or out of date. + final String cdkRelativePath = 'airbyte-cdk/java/airbyte-cdk' + afterEvaluate { + def cdkVersionNeeded = getCDKSnapshotRequirement(configurations.compileClasspath.incoming.resolutionResult.allDependencies) + checkCDKJarExists(cdkVersionNeeded) + } + } + } + repositories { mavenCentral() maven { @@ -297,6 +391,7 @@ subprojects { subproj -> maven { url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' } + mavenLocal() } pmd { @@ -462,16 +557,6 @@ subprojects { subproj -> check.dependsOn 'jacocoTestCoverageVerification' } -task('generate') { - dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } - dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } -} - -task('format') { - dependsOn generate - dependsOn spotlessApply - dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) } -} // add licenses for python projects. subprojects { @@ -507,6 +592,78 @@ subprojects { licenseTask.dependsOn generateManifestFilesTask } } +} + +python { + envPath = '.venv' + minPythonVersion = '3.9' + scope = 'VIRTUALENV' + installVirtualenv = true + pip 'pip:21.3.1' + // https://github.com/csachs/pyproject-flake8/issues/13 + pip 'flake8:4.0.1' + // flake8 doesn't support pyproject.toml files + // and thus there is the wrapper "pyproject-flake8" for this + pip 'pyproject-flake8:0.0.1a2' + pip 'black:22.3.0' + pip 'mypy:1.4.1' + pip 'isort:5.6.4' + pip 'coverage[toml]:6.3.1' +} + +task('generate') { + dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } + dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } +} + +license { + header rootProject.file("LICENSE_SHORT") +} + +task licensePythonGlobalFormat(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { + header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) + source = fileTree(dir: rootProject.rootDir) + .include("**/*.py") + .exclude("**/.venv/**") + .exclude("**/build/**") + .exclude("**/node_modules/**") + .exclude("**/airbyte_api_client/**") + .exclude("**/__init__.py") + .exclude("airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py") + .exclude("airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py") + .exclude("resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py") + .exclude("tools/git_hooks/tests/test_spec_linter.py") + .exclude("tools/schema_generator/schema_generator/infer_schemas.py") + strictCheck = true + dependsOn generate +} + +task isortGlobalFormat(type: PythonTask) { + module = "isort" + command = "--settings-file=${rootProject.file('pyproject.toml').absolutePath} ./" + dependsOn generate +} + +task blackGlobalFormat(type: PythonTask) { + module = "black" + // the line length should match .isort.cfg + command = "--config ${rootProject.file('pyproject.toml').absolutePath} ./" + dependsOn generate +} + +task('format') { + dependsOn generate + dependsOn spotlessApply + dependsOn licensePythonGlobalFormat + dependsOn isortGlobalFormat + dependsOn blackGlobalFormat + tasks.findByName('spotlessApply').mustRunAfter 'generate' + tasks.findByName('licensePythonGlobalFormat').mustRunAfter 'generate' + tasks.findByName('isortGlobalFormat').mustRunAfter 'licensePythonGlobalFormat' + tasks.findByName('blackGlobalFormat').mustRunAfter 'isortGlobalFormat' +} + +subprojects { task listAllDependencies(type: DependencyReportTask) {} } diff --git a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle index a6938bc96791..e650889c417c 100644 --- a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle +++ b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle @@ -53,6 +53,11 @@ class AirbyteIntegrationTestJavaPlugin implements Plugin { // This is needed to make the destination-snowflake tests succeed - https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 jvmArgs = ["--add-opens=java.base/java.nio=ALL-UNNAMED"] + + systemProperties = [ + // Allow tests to set @Execution(ExecutionMode.CONCURRENT) + 'junit.jupiter.execution.parallel.enabled': 'true' + ] } // make sure we create the integrationTest task once in case a standard source test was already initialized diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 2a348cf5ef8b..b7884f661ee2 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -90,7 +90,7 @@ class AirbytePythonPlugin implements Plugin { // and thus there is the wrapper "pyproject-flake8" for this pip 'pyproject-flake8:0.0.1a2' pip 'black:22.3.0' - pip 'mypy:0.930' + pip 'mypy:1.4.1' pip 'isort:5.6.4' pip 'pytest:6.2.5' pip 'coverage[toml]:6.3.1' diff --git a/deps.toml b/deps.toml index 42ee78ea9ce4..4a33456a7e48 100644 --- a/deps.toml +++ b/deps.toml @@ -121,6 +121,7 @@ temporal-testing = { module = "io.temporal:temporal-testing", version.ref = "tem debezium-api = { module = "io.debezium:debezium-api", version.ref = "debezium"} debezium-embedded = { module = "io.debezium:debezium-embedded", version.ref = "debezium"} debezium-sqlserver = { module = "io.debezium:debezium-connector-sqlserver", version.ref = "debezium"} +debezium-mongodb = { module = "io.debezium:debezium-connector-mongodb", version.ref = "debezium" } debezium-mysql = { module = "io.debezium:debezium-connector-mysql", version.ref = "debezium"} debezium-postgres = { module = "io.debezium:debezium-connector-postgres", version.ref = "debezium"} @@ -165,4 +166,4 @@ micronaut-test = ["micronaut-test-core", "micronaut-test-junit5", "h2-database"] micronaut-test-annotation-processor = ["micronaut-inject-java"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] temporal = ["temporal-sdk", "temporal-serviceclient"] -debezium-bundle = ["debezium-api", "debezium-embedded", "debezium-sqlserver", "debezium-mysql", "debezium-postgres"] +debezium-bundle = ["debezium-api", "debezium-embedded", "debezium-sqlserver", "debezium-mysql", "debezium-postgres", "debezium-mongodb"] diff --git a/docs/.gitbook/assets/airbyte_kestra_1.gif b/docs/.gitbook/assets/airbyte_kestra_1.gif new file mode 100644 index 000000000000..b991c2b39c7e Binary files /dev/null and b/docs/.gitbook/assets/airbyte_kestra_1.gif differ diff --git a/docs/.gitbook/assets/airbyte_kestra_2.gif b/docs/.gitbook/assets/airbyte_kestra_2.gif new file mode 100644 index 000000000000..287abb179776 Binary files /dev/null and b/docs/.gitbook/assets/airbyte_kestra_2.gif differ diff --git a/docs/.gitbook/assets/orchestrator-lifecycle.png b/docs/.gitbook/assets/orchestrator-lifecycle.png new file mode 100644 index 000000000000..7db52fea872a Binary files /dev/null and b/docs/.gitbook/assets/orchestrator-lifecycle.png differ diff --git a/docs/.gitbook/assets/sentry-flow-v1.png b/docs/.gitbook/assets/sentry-flow-v1.png deleted file mode 100644 index 6293059a0a15..000000000000 Binary files a/docs/.gitbook/assets/sentry-flow-v1.png and /dev/null differ diff --git a/docs/airbyte-enterprise.md b/docs/airbyte-enterprise.md new file mode 100644 index 000000000000..df04379d8445 --- /dev/null +++ b/docs/airbyte-enterprise.md @@ -0,0 +1,62 @@ +# Airbyte Enterprise + +Airbyte Enterprise is a self-hosted version of Airbyte with additional features for enterprise customers. Airbyte Enterprise is currently in early development. Interested in Airbyte Enterprise for your organization? [Learn more](https://airbyte.com/solutions/airbyte-enterprise). + +## Airbyte Enterprise License Key + +A valid license key is required for Airbyte Enterprise. Talk to your Sales Representative to receive your license key before installing Airbyte Enterprise. + +## Single Sign-On (SSO) + +Airbyte Enterprise supports Single Sign-On, allowing an organization to manage user acces to their Airbyte Enterprise instance through the configuration of an Identity Provider (IdP). + +Airbyte Enterprise currently supports SSO via OIDC with [Okta](https://www.okta.com/) as an IdP. + +### Setting up Okta for SSO + +You will need to create a new Okta OIDC App Integration for your Airbyte instance. Documentation on how to do this in Okta can be found [here](https://help.okta.com/en-us/Content/Topics/Apps/Apps_App_Integration_Wizard_OIDC.htm). + +You should create an app integration with **OIDC - OpenID Connect** as the sign-in method and **Web Application** as the application type: + +![Screenshot of Okta app integration creation modal](./assets/docs/okta-create-new-app-integration.png) + +#### App integration name + +Please choose a URL-friendly app integraiton name without spaces or special characters, such as `my-airbyte-app`: + +![Screenshot of Okta app integration name](./assets/docs/okta-app-integration-name.png) + +Spaces or special characters in this field could result in invalid redirect URIs. + +#### Redirect URIs + +In the **Login** section, set the following fields, substituting `` and `` for your own values: + +Sign-in redirect URIs: + +``` +/auth/realms/airbyte/broker//endpoint +``` + +Sign-out redirect URIs + +``` +/auth/realms/airbyte/broker//endpoint/logout_response +``` + +![Okta app integration name screenshot](./assets/docs/okta-login-redirect-uris.png) + +_Example values_ + +`` should point to where your Airbyte instance will be available, including the http/https protocol. + +#### Deploying Airbyte Enterprise with Okta + +Your Okta app is now set up and you're ready to deploy Airbyte with SSO! Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: + +- Okta domain ([how to find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) +- App integration name +- Client ID +- Client Secret + +Visit [Airbyte Enterprise deployment](/deploying-airbyte/on-kubernetes-via-helm#alpha-airbyte-pro-deployment) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. diff --git a/docs/api-documentation.md b/docs/api-documentation.md index b1786dbc12c1..448a63b2e952 100644 --- a/docs/api-documentation.md +++ b/docs/api-documentation.md @@ -5,7 +5,7 @@ Airbyte has two sets of APIs which are intended for different uses. The table be | | **Airbyte API** | **Configuration API** | |------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | **Description** | Airbyte API is a reliable, easy-to-use interface for programmatically controlling the Airbyte platform. With full support from the Airbyte team. | The Config API is an internal Airbyte API that is designed for communications between different Airbyte components. | -| **Use Cases** | Enables users to control Airbyte programmatically and use with Orchestration tools (ex: Airflow)

    Exists for Airbyte users to write applications against.

    Enables [Powered by Airbyte](https://airbyte.com/embed-airbyte-connectors-with-api) | Enables OSS users to configure their own Self-Hosted Airbyte deployment (internal state, etc)


    Enables Airbyte Engineering team to configure Airbyte Cloud (internal state) | -| **Available for** | Cloud users. | OSS users | -| **Status** | Available to all Cloud users. Learn more in our [launch post](https://airbyte.com/blog/airbyte-api).

    Full support from the Airbyte team. | Airbyte does NOT have active commitments to support this API long-term. OSS users can utilize the Config API, but at their own risk.

    This API is utilized internally by the Airbyte Engineering team and may be modified in the future if the need arises.

    Modifications by the Airbyte Engineering team could create breaking changes and OSS users would need to update their code to catch up to any backwards incompatible changes in the API. | +| **Use Cases** | Enables users to control Airbyte programmatically and use with Orchestration tools (ex: Airflow)

    Exists for Airbyte users to write applications against.

    Enables [Powered by Airbyte](https://airbyte.com/embed-airbyte-connectors-with-api) | Enables Airbyte Engineering team to configure Airbyte | +| **Intended users** | Airbyte OSS, Cloud & Self-Hosted Enterprise | Airbyte Engineering Team | +| **Status** | Available to all Airbyte users (OSS, Cloud, Self-Hosted Enterprise). Learn more on our [blog](https://airbyte.com/blog/airbyte-api).

    Full support from the Airbyte team. | Airbyte does NOT have active commitments to support this API long-term. Users utilize the Config API, at their own risk.

    This API is utilized internally by the Airbyte Engineering team and may be modified in the future if the need arises.

    Modifications by the Airbyte Engineering team could create breaking changes and OSS users would need to update their code to catch up to any backwards incompatible changes in the API. | | **Documentation** | [Available here](https://api.airbyte.com) | [Available here](https://airbyte-public-api-docs.s3.us-east-2.amazonaws.com/rapidoc-api-docs.html) diff --git a/docs/archive/changelog/connectors.md b/docs/archive/changelog/connectors.md index 3391650de8f6..a1f8b8126e07 100644 --- a/docs/archive/changelog/connectors.md +++ b/docs/archive/changelog/connectors.md @@ -15,7 +15,7 @@ Check out our [connector roadmap](https://github.com/airbytehq/airbyte/projects/ New sources: - [**Chartmogul**](https://docs.airbyte.com/integrations/sources/chartmogul) -- [**Baton**](https://docs.airbyte.com/integrations/sources/baton) +- [**Hellobaton**](https://docs.airbyte.com/integrations/sources/hellobaton) - [**Flexport**](https://docs.airbyte.com/integrations/sources/flexport) - [**PersistIq**](https://docs.airbyte.com/integrations/sources/persistiq) diff --git a/docs/archive/examples/zoom-activity-dashboard.md b/docs/archive/examples/zoom-activity-dashboard.md index 8dc8b2c98ad1..a141f2da418a 100644 --- a/docs/archive/examples/zoom-activity-dashboard.md +++ b/docs/archive/examples/zoom-activity-dashboard.md @@ -32,7 +32,7 @@ In order to replicate Zoom data, we will need to use [Airbyte’s Zoom connector `docker-compose up` -You can find more details about this in the [Getting Started FAQ](https://discuss.airbyte.io/c/faq/15) on our Discourse Forum. +You can find more details about this in the [Getting Started FAQ](https://discuss.airbyte.io/c/faq/15) on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). This will start up Airbyte on `localhost:8000`; open that address in your browser to access the Airbyte dashboard. diff --git a/docs/archive/faq/README.md b/docs/archive/faq/README.md index daec5e69ead1..1f6a217b74c7 100644 --- a/docs/archive/faq/README.md +++ b/docs/archive/faq/README.md @@ -1,5 +1,5 @@ # FAQ -Our FAQ is now a section on our Discourse forum. Check it out [here](https://discuss.airbyte.io/c/faq/15)! +Our FAQ is now a section on our Airbyte Forum. Check it out [here](https://github.com/airbytehq/airbyte/discussions)! If you don't see your question answered, feel free to open up a new topic for it. \ No newline at end of file diff --git a/docs/assets/docs/okta-app-integration-name.png b/docs/assets/docs/okta-app-integration-name.png new file mode 100644 index 000000000000..87a9b89d77e4 Binary files /dev/null and b/docs/assets/docs/okta-app-integration-name.png differ diff --git a/docs/assets/docs/okta-create-new-app-integration.png b/docs/assets/docs/okta-create-new-app-integration.png new file mode 100644 index 000000000000..bff936656aad Binary files /dev/null and b/docs/assets/docs/okta-create-new-app-integration.png differ diff --git a/docs/assets/docs/okta-login-redirect-uris.png b/docs/assets/docs/okta-login-redirect-uris.png new file mode 100644 index 000000000000..40463b1acfd4 Binary files /dev/null and b/docs/assets/docs/okta-login-redirect-uris.png differ diff --git a/docs/cli-documentation.md b/docs/cli-documentation.md index 1df96e2b6f27..17b13dac71bd 100644 --- a/docs/cli-documentation.md +++ b/docs/cli-documentation.md @@ -1,9 +1,12 @@ # CLI documentation -## Disclaimer +:::caution +The Octavia CLI is an alpha, unofficial CLI that won't be maintained. +::: -The project is in **alpha** version. -Readers can refer to our [opened GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) to check the ongoing work on this project. +:::tip Recommendation +We recommend all users leverage the official [Airbyte Terraform Provider](https://reference.airbyte.com/reference/using-the-terraform-provider), instead of this CLI. +::: ## What is `octavia` CLI? @@ -27,7 +30,7 @@ These are non-exhaustive use cases `octavia` can be convenient for: - Integrating the Airbyte configuration deployment in a dev ops tooling stack: Helm, Ansible etc. - Streamlining the deployment of Airbyte configurations to multiple Airbyte instance. -Feel free to share your use cases with the community in [#octavia-cli](https://airbytehq.slack.com/archives/C02RRUG9CP5) or on [Discourse](https://discuss.airbyte.io/). +Readers can refer to our [opened GitHub issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Foctavia-cli) to check the ongoing work on this project. ## Table of content @@ -706,20 +709,3 @@ The telemetry sends data about: - The current Airbyte workspace id if the user has not set the _anonymous data collection_ on their Airbyte instance. You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment variable to `False` or using the `--disable-telemetry` flag. - -## Changelog - -| Version | Date | Description | PR | -| ------- | ---------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------- | -| 0.41.0 | 2022-10-13 | Use Basic Authentication for making API requests | [#17982](https://github.com/airbytehq/airbyte/pull/17982) | -| 0.40.32 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | -| 0.39.33 | 2022-07-05 | Add `octavia import all` command | [#14374](https://github.com/airbytehq/airbyte/pull/14374) | -| 0.39.32 | 2022-06-30 | Create import command to import and manage existing Airbyte resource from octavia-cli | [#14137](https://github.com/airbytehq/airbyte/pull/14137) | -| 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | -| 0.39.19 | 2022-06-16 | Allow connection management on multiple workspaces | [#13070](https://github.com/airbytehq/airbyte/pull/12727) | -| 0.39.19 | 2022-06-15 | Allow users to set custom HTTP headers | [#12893](https://github.com/airbytehq/airbyte/pull/12893) | -| 0.39.14 | 2022-05-12 | Enable normalization on connection | [#12727](https://github.com/airbytehq/airbyte/pull/12727) | -| 0.37.0 | 2022-05-05 | Use snake case in connection fields | [#12133](https://github.com/airbytehq/airbyte/pull/12133) | -| 0.35.68 | 2022-04-15 | Improve telemetry | [#12072](https://github.com/airbytehq/airbyte/issues/11896) | -| 0.35.68 | 2022-04-12 | Add telemetry | [#11896](https://github.com/airbytehq/airbyte/issues/11896) | -| 0.35.61 | 2022-04-07 | Alpha release | [EPIC](https://github.com/airbytehq/airbyte/issues/10704) | diff --git a/docs/cloud/core-concepts.md b/docs/cloud/core-concepts.md index 1439c769e013..9383c6ffd036 100644 --- a/docs/cloud/core-concepts.md +++ b/docs/cloud/core-concepts.md @@ -1,12 +1,12 @@ # Core Concepts -Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, what format the data is written to in the destination, and if the data is stored in raw tables format or basic normalized (or JSON) format. +Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, what format the data is written to in the destination, and if the data is stored in raw tables format or basic normalized (or JSON) format. This page describes the concepts you need to know to use Airbyte. -## Source +## Source -A source is an API, file, database, or data warehouse that you want to ingest data from. +A source is an API, file, database, or data warehouse that you want to ingest data from. ## Destination @@ -18,7 +18,7 @@ An Airbyte component which pulls data from a source or pushes data to a destinat ## Connection -A connection is an automated data pipeline that replicates data from a source to a destination. +A connection is an automated data pipeline that replicates data from a source to a destination. Setting up a connection involves configuring the following parameters: @@ -38,7 +38,7 @@ Setting up a connection involves configuring the following parameters: Destination Namespace and stream names - Where should the replicated data be written? + Where should the replicated data be written? @@ -63,28 +63,28 @@ Setting up a connection involves configuring the following parameters: ## Stream -A stream is a group of related records. +A stream is a group of related records. Examples of streams: -* A table in a relational database -* A resource or API endpoint for a REST API -* The records from a directory containing many files in a filesystem +- A table in a relational database +- A resource or API endpoint for a REST API +- The records from a directory containing many files in a filesystem ## Field -A field is an attribute of a record in a stream. +A field is an attribute of a record in a stream. -Examples of fields: +Examples of fields: -* A column in the table in a relational database -* A field in an API response +- A column in the table in a relational database +- A field in an API response ## Namespace Namespace is a group of streams in a source or destination. Common use cases for namespaces are enforcing permissions, segregating test and production data, and general data organization. -A schema in a relational database system is an example of a namespace. +A schema in a relational database system is an example of a namespace. In a source, the namespace is the location from where the data is replicated to the destination. @@ -121,32 +121,32 @@ In a destination, the namespace is the location where the replicated data is sto A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes to account for various use cases. -* **Full Refresh | Overwrite:** Sync all records from the source and replace data in destination by overwriting it. -* **Full Refresh | Append:** Sync all records from the source and add them to the destination without deleting any data. -* **Incremental Sync | Append:** Sync new records from the source and add them to the destination without deleting any data. -* **Incremental Sync | Deduped History:** Sync new records from the source and add them to the destination. Also provides a de-duplicated view mirroring the state of the stream in the source. +- **Full Refresh | Overwrite:** Sync all records from the source and replace data in destination by overwriting it. +- **Full Refresh | Append:** Sync all records from the source and add them to the destination without deleting any data. +- **Incremental Sync | Append:** Sync new records from the source and add them to the destination without deleting any data. +- **Incremental Sync | Append + Deduped:** Sync new records from the source and add them to the destination. Also provides a de-duplicated view mirroring the state of the stream in the source. ## Normalization Normalization is the process of structuring data from the source into a format appropriate for consumption in the destination. For example, when writing data from a nested, dynamically typed source like a JSON API to a relational destination like Postgres, normalization is the process which un-nests JSON from the source into a relational table format which uses the appropriate column types in the destination. -Note that normalization is only relevant for the following relational database & warehouse destinations: +Note that normalization is only relevant for the following relational database & warehouse destinations: -* BigQuery -* Snowflake -* Redshift -* Postgres -* Oracle -* MySQL -* MSSQL +- BigQuery +- Snowflake +- Redshift +- Postgres +- Oracle +- MySQL +- MSSQL -Other destinations do not support normalization as described in this section, though they may normalize data in a format that makes sense for them. For example, the S3 destination connector offers the option of writing JSON files in S3, but also offers the option of writing statically typed files such as Parquet or Avro. +Other destinations do not support normalization as described in this section, though they may normalize data in a format that makes sense for them. For example, the S3 destination connector offers the option of writing JSON files in S3, but also offers the option of writing statically typed files such as Parquet or Avro. After a sync is complete, Airbyte normalizes the data. When setting up a connection, you can choose one of the following normalization options: -* Raw data (no normalization): Airbyte places the JSON blob version of your data in a table called `_airbyte_raw_` -* Basic Normalization: Airbyte converts the raw JSON blob version of your data to the format of your destination. *Note: Not all destinations support normalization.* -* [dbt Cloud integration](https://docs.airbyte.com/cloud/managing-airbyte-cloud/dbt-cloud-integration): Airbyte's dbt Cloud integration allows you to use dbt Cloud for transforming and cleaning your data during the normalization process. +- Raw data (no normalization): Airbyte places the JSON blob version of your data in a table called `_airbyte_raw_` +- Basic Normalization: Airbyte converts the raw JSON blob version of your data to the format of your destination. _Note: Not all destinations support normalization._ +- [dbt Cloud integration](https://docs.airbyte.com/cloud/managing-airbyte-cloud/dbt-cloud-integration): Airbyte's dbt Cloud integration allows you to use dbt Cloud for transforming and cleaning your data during the normalization process. :::note @@ -156,7 +156,7 @@ Normalizing data may cause an increase in your destination's compute cost. This ## Workspace -A workspace is a grouping of sources, destinations, connections, and other configurations. It lets you collaborate with team members and share resources across your team under a shared billing account. +A workspace is a grouping of sources, destinations, connections, and other configurations. It lets you collaborate with team members and share resources across your team under a shared billing account. When you [sign up](http://cloud.airbyte.com/signup) for Airbyte Cloud, we automatically create your first workspace where you are the only user with access. You can set up your sources and destinations to start syncing data and invite other users to join your workspace. diff --git a/docs/cloud/getting-started-with-airbyte-cloud.md b/docs/cloud/getting-started-with-airbyte-cloud.md index 2453c0bcf6e0..1071020a92c9 100644 --- a/docs/cloud/getting-started-with-airbyte-cloud.md +++ b/docs/cloud/getting-started-with-airbyte-cloud.md @@ -64,26 +64,27 @@ A connection is an automated data pipeline that replicates data from a source to Setting up a connection involves configuring the following parameters: -| Parameter | Description | -|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Replication frequency | How often should the data sync? | -| [Data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-the-data-residency-for-a-connection) | Where should the data be processed? | -| Destination Namespace and stream names | Where should the replicated data be written? | -| Catalog selection | Which streams and fields should be replicated from the source to the destination? | -| Sync mode | How should the streams be replicated (read and written)? | -| Optional transformations | How should Airbyte protocol messages (raw JSON blob) data be converted into other data representations? | +| Parameter | Description | +| ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Replication frequency | How often should the data sync? | +| [Data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-the-data-residency-for-a-connection) | Where should the data be processed? | +| Destination Namespace and stream names | Where should the replicated data be written? | +| Catalog selection | Which streams and fields should be replicated from the source to the destination? | +| Sync mode | How should the streams be replicated (read and written)? | +| Optional transformations | How should Airbyte protocol messages (raw JSON blob) data be converted into other data representations? | For more information, see [Connections and Sync Modes](../understanding-airbyte/connections/README.md) and [Namespaces](../understanding-airbyte/namespaces.md) If you need to use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html): -1. In the **Replication Frequency** dropdown, click **Cron**. + +1. In the **Replication Frequency** dropdown, click **Cron**. 2. Enter a cron expression and choose a time zone to create a sync schedule. :::note -* Only one sync per connection can run at a time. -* If cron schedules a sync to run before the last one finishes, the scheduled sync will start after the last sync completes. -* Cloud does not allow schedules that sync more than once per hour. +- Only one sync per connection can run at a time. +- If cron schedules a sync to run before the last one finishes, the scheduled sync will start after the last sync completes. +- Cloud does not allow schedules that sync more than once per hour. ::: @@ -171,12 +172,12 @@ To better understand the destination namespace configurations, see [Destination - Select **Overwrite** to erase the old data and replace it completely - Select **Append** to capture changes to your table **Note:** This creates duplicate records - - Select **Deduped + history** to mirror your source while keeping records unique + - Select **Append + Deduped** to mirror your source while keeping records unique **Note:** Some sync modes may not yet be available for your source or destination 4. **Cursor field**: Used in **Incremental** sync mode to determine which records to sync. Airbyte pre-selects the cursor field for you (example: updated date). If you have multiple cursor fields, select the one you want. - 5. **Primary key**: Used in **Deduped + history** sync mode to determine the unique identifier. + 5. **Primary key**: Used in **Append + Deduped** sync mode to determine the unique identifier. 6. **Destination**: - **Namespace:** The database schema of your destination tables. - **Stream name:** The final table name in destination. @@ -193,38 +194,28 @@ Verify the sync by checking the logs: 3. Check the data at your destination. If you added a Destination Stream Prefix while setting up the connection, make sure to search for the stream name with the prefix. ## Allowlist IP addresses + Depending on your [data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-your-default-data-residency) location, you may need to allowlist the following IP addresses to enable access to Airbyte: ### United States and Airbyte Default -#### GCP region: us-west3 -* 34.106.109.131 -* 34.106.196.165 -* 34.106.60.246 -* 34.106.229.69 -* 34.106.127.139 -* 34.106.218.58 -* 34.106.115.240 -* 34.106.225.141 - -### European Union -:::note +#### GCP region: us-west3 -Some workflows still run in the US, even when the data residency is in the EU. If you use the EU as a data residency, you must allowlist the following IP addresses from both GCP us-west3 and AWS eu-west-3. +[comment]: # "IMPORTANT: if changing the list of IP addresses below, you must also update the connector.airbyteCloudIpAddresses LaunchDarkly flag to show the new list so that the correct list is shown in the Airbyte Cloud UI, then reach out to the frontend team and ask them to update the default value in the useAirbyteCloudIps hook!" -::: +- 34.106.109.131 +- 34.106.196.165 +- 34.106.60.246 +- 34.106.229.69 +- 34.106.127.139 +- 34.106.218.58 +- 34.106.115.240 +- 34.106.225.141 -#### GCP region: us-west3 -* 34.106.109.131 -* 34.106.196.165 -* 34.106.60.246 -* 34.106.229.69 -* 34.106.127.139 -* 34.106.218.58 -* 34.106.115.240 -* 34.106.225.141 +### European Union #### AWS region: eu-west-3 -* 13.37.4.46 -* 13.37.142.60 -* 35.181.124.238 + +- 13.37.4.46 +- 13.37.142.60 +- 35.181.124.238 diff --git a/docs/cloud/managing-airbyte-cloud.md b/docs/cloud/managing-airbyte-cloud.md index 1dfc12f28c5e..372cfeb5e81c 100644 --- a/docs/cloud/managing-airbyte-cloud.md +++ b/docs/cloud/managing-airbyte-cloud.md @@ -103,6 +103,7 @@ For individual connections, you can choose a data residency that is different fr :::note While the data is processed in a data plane in the chosen residency, the cursor and primary key data is stored in the US control plane. If you have data that cannot be stored in the US, do not use it as a cursor or primary key. +All your account information such as the name and email addresses of the Airbyte users are stored in the US as well. ::: diff --git a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md index 896d318fc923..1e6e8e0eb3e1 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md +++ b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md @@ -2,7 +2,8 @@ This page provides guidance on how to manage notifications for Airbyte Cloud, allowing you to stay up-to-date on the activities in your workspace. -## Set up Webhook notifications + +## Configure Notification Settings To set up Webhook notifications: @@ -10,18 +11,26 @@ To set up Webhook notifications: 2. Click **Notifications**. -3. [Create an Incoming Webhook for Slack](https://api.slack.com/messaging/webhooks). +3. Have a webhook URL ready if you plan to use webhook notifications. Using a Slack webook is recommended. [Create an Incoming Webhook for Slack](https://api.slack.com/messaging/webhooks). + +4. Toggle the type of events you are interested to receive notifications for. + 1. To enable webhook notifications, the webhook URL is required. For your convenience, we provide a 'test' function to send a test message to your webhook URL so you can make sure it's working as expected. + +5. Click **Save changes**. -4. Navigate back to the Airbyte Cloud dashboard > Settings > Notifications and enter the Webhook URL. +## Notification Event Types -5. Toggle the **When sync fails** and **When sync succeeds** buttons as required. +1. Failed syncs and successful syncs: When a connection sync has finished, you can choose to be notified whether the sync has failed, succeeded or both. Note that if sync runs frequently or if there are many syncs in the workspace these types of events can be noisy. +1. Automated Connection Updates: when Airbyte detects that the connection's source schema has changed and can be updated automatically, Airbyte will update your connection and send you a notification message. +1. Connection Updates Requiring Action: When Airbyte detects some updates that requires your action to run syncs. Since this will affect your sync from running as scheduled, you cannot disable this type of email notification. +1. Sync Disabled and Sync Disabled Warning: If a sync has been failing for multiple days or many times consecutively, Airbyte will disable the connection to prevent it from running further. Airbyte will send a Sync Disabled Warning notification when we detect the trend, and once the failure counts hits a threshold Airbyte will send a Sync Disabled notification and will actually disable the connection. Again, because the sync will not continue to run as you have configured, the Sync Disabled notification cannot be disabled. -6. Click **Save changes**. + ## Enable schema update notifications To get notified when your source schema changes: -1. Make sure you have [Webhook notifications](#set-up-webhook-notifications) set up. +1. Make sure you have `Automatic Connection Updates` and `Connection Updates Requiring Action` turned on for your desired notification channels; If these are off, even if you turned on schema update notifications in a connection's settings, Airbyte will *NOT* send out any notifications related to these types of events. 2. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection you want to receive notifications for. diff --git a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md index c1614793502e..0d494e3b0526 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md +++ b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md @@ -1,12 +1,26 @@ # Manage schema changes -Once every 24 hours, Airbyte checks for changes in your source schema and allows you to review the changes and fix breaking changes. This process helps ensure accurate and efficient data syncs, minimizing errors and saving you time and effort in managing your data pipelines. +You can specify for each connection how Airbyte should handle any change of schema in the source. This process helps ensure accurate and efficient data syncs, minimizing errors and saving you time and effort in managing your data pipelines. -:::note +Airbyte checks for any changes in your source schema before syncing, at most once every 24 hours. -Schema changes are flagged in your connection but are not propagated to your destination. - -::: +Based on your configured settings for "Detect and propagate schema changes", Airbyte can automatically sync those changes or ignore them: +* **Propagate all changes** automatically propagates stream changes (additions or deletions) or column changes (additions or deletions) detected in the source +* **Propagate column changes only** automatically propagates column changes detected in the source +* **Ignore** any schema change, in which case the schema you’ve set up will not change even if the source schema changes until you approve the changes manually +* **Pause connection** disables the connection from syncing further once a change is detected + +When a new column is detected and propagated, values for that column will be filled in for the updated rows. If you are missing values for rows not updated, a backfill can be done by completing a full refresh. + +When a column is deleted, the values for that column will stop updating for the updated rows and be filled with Null values. + +When a new stream is detected and propagated, the first sync will fill all data in as if it is a historical sync. When a stream is deleted from the source, the stream will stop updating, and we leave any existing data in the destination. The rest of the enabled streams will continue syncing. + +In all cases, if a breaking change is detected, the connection will be paused for manual review to prevent future syncs from failing. Breaking schema changes occur when: +* An existing primary key is removed from the source +* An existing cursor is removed from the source + +See "Fix breaking schema changes" to understand how to resolve these types of changes. ## Review non-breaking schema changes @@ -29,11 +43,9 @@ To review non-breaking schema changes: ## Fix breaking schema changes -:::note - -Breaking changes can only occur in the **Cursor** or **Primary key** fields. - -::: +Breaking schema changes occur when: +* An existing primary key is removed from the source +* An existing cursor is removed from the source To review and fix breaking schema changes: 1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with breaking changes (indicated by a **red exclamation mark** icon). @@ -68,4 +80,7 @@ In addition to Airbyte Cloud’s automatic schema change detection, you can manu 3. In the **Activate the streams you want to sync** table, click **Refresh source schema** to fetch the schema of your data source. - 2. If there are changes to the schema, you can review them in the **Refreshed source schema** dialog. + 4. If there are changes to the schema, you can review them in the **Refreshed source schema** dialog. + +## Manage Schema Change Notifications +[Refer to our notification documentation](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications#enable-schema-update-notifications) to understand how to stay updated on any schema updates to your connections. \ No newline at end of file diff --git a/docs/cloud/managing-airbyte-cloud/review-sync-summary.md b/docs/cloud/managing-airbyte-cloud/review-sync-summary.md index eebab7fc47b0..7d6da7b286a0 100644 --- a/docs/cloud/managing-airbyte-cloud/review-sync-summary.md +++ b/docs/cloud/managing-airbyte-cloud/review-sync-summary.md @@ -10,7 +10,7 @@ To review the sync summary: :::note - Airbyte will try to sync your data three times. After a third failure, it will stop attempting to sync. + In the event of a failure, Airbyte will make several attempts to sync your data before giving up. The latest rules can be read about [here](../../understanding-airbyte/jobs.md#retry-rules). ::: @@ -21,12 +21,12 @@ To review the sync summary: | Data | Description | |--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| | x GB (also measured in KB, MB) | Amount of data moved during the sync. If basic normalization is on, the amount of data would not change since normalization occurs in the destination. | -| x emitted records | Number of records read from the source during the sync. | -| x committed records | Number of records the destination confirmed it received. | +| x extracted records | Number of records read from the source during the sync. | +| x loaded records | Number of records the destination confirmed it received. | | xh xm xs | Total time (hours, minutes, seconds) for the sync and basic normalization, if enabled, to complete. | :::note -In a successful sync, the number of emitted records and committed records should be the same. +In a successful sync, the number of extracted records and loaded records should be the same. ::: diff --git a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md index 61bc6941ce0a..470a50cc64d2 100644 --- a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md +++ b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md @@ -14,4 +14,4 @@ Understanding the following limitations will help you more effectively manage Ai * Shortest sync schedule: Every 60 min * Schedule accuracy: +/- 30 min -*Limits on workspaces, sources, destinations and connections do not apply to customers of [Powered by Airbyte](https://airbyte.com/embed-airbyte-connectors-with-api). To learn more [contact us](https://airbyte.com/talk-to-sales)! +*Limits on workspaces, sources, destinations and connections do not apply to customers of [Powered by Airbyte](https://airbyte.com/solutions/powered-by-airbyte). To learn more [contact us](https://airbyte.com/talk-to-sales)! diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index 5ca20f73c1b6..9a011cb3f4e3 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -143,12 +143,7 @@ Once you've finished iterating on the changes to a connector as specified in its 1. Bump the version in the `Dockerfile` of the connector \(`LABEL io.airbyte.version=X.X.X`\). 2. Bump the docker image version in the [metadata.yaml](connector-metadata-file.md) of the connector. 3. Submit a PR containing the changes you made. -4. One of Airbyte maintainers will review the change in the new version. Triggering tests can be done by leaving a comment on the PR with the following format \(the PR must be from the Airbyte repo, not a fork\): - ```text - # to run integration tests for the connector - # Example: /test connector=connectors/source-hubspot - /test connector=(connectors|bases)/ - ``` +4. One of Airbyte maintainers will review the change in the new version and make sure the tests are passing. 5. You our an Airbyte maintainer can merge the PR once it is approved and all the required CI checks are passing you. 6. Once the PR is merged the new connector version will be published to DockerHub and the connector should now be available for everyone who uses it. Thank you! diff --git a/docs/connector-development/cdk-python/README.md b/docs/connector-development/cdk-python/README.md index b6e97a65e167..5e2c63af32e2 100644 --- a/docs/connector-development/cdk-python/README.md +++ b/docs/connector-development/cdk-python/README.md @@ -18,7 +18,7 @@ The CDK provides an improved developer experience by providing basic implementat This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding. -If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to [the Connector Development section of our Discourse forum](https://discuss.airbyte.io/c/connector-development/16) to inquire further! +If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to [the Connector Development section of our Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) to inquire further! ## Getting Started diff --git a/docs/connector-development/connector-builder-ui/authentication.md b/docs/connector-development/connector-builder-ui/authentication.md index e4dce29c9939..0a6cb8b67763 100644 --- a/docs/connector-development/connector-builder-ui/authentication.md +++ b/docs/connector-development/connector-builder-ui/authentication.md @@ -17,6 +17,7 @@ Check the documentation of the API you want to integrate for the used authentica * [Bearer Token](#bearer-token) * [API Key](#api-key) * [OAuth](#oauth) +* [Session Token](#session-token) Select the matching authentication method for your API and check the sections below for more information about individual methods. @@ -63,9 +64,16 @@ curl -X GET \ ### API Key -The API key authentication method is similar to the Bearer authentication but allows to configure as which HTTP header the API key is sent as part of the request. The http header name is part of the connector definition while the API key itself can be set via "Testing values" in the connector builder as well as when configuring this connector as a Source. +The API key authentication method is similar to the Bearer authentication but allows to configure where to inject the API key (header, request param or request body), as well as under which field name. The used injection mechanism and the field name is part of the connector definition while the API key itself can be set via "Testing values" in the connector builder as well as when configuring this connector as a Source. -This form of authentication is often called "(custom) header authentication". It only supports setting the token to an HTTP header, for other cases, see the ["Other authentication methods" section](#access-token-as-query-or-body-parameter) +The following table helps with which mechanism to use for which API: + +| Description | Injection mechanism | +|----------|----------| +| (HTTP) header | `header` | +| Query parameter / query string / request parameter / URL parameter | `request_parameter` | +| Form encoded request body / form data | `body_data` | +| JSON encoded request body | `body_json` | #### Example @@ -78,6 +86,8 @@ curl -X GET \ https://rest.coinapi.io/v1/ ``` +In this case the injection mechanism is `header` and the field name is `X-CoinAPI-Key`. + ### OAuth The OAuth authentication method implements authentication using an OAuth2.0 flow with a [refresh token grant type](https://oauth.net/2/grant-types/refresh-token/) and [client credentiuals grant type](https://oauth.net/2/grant-types/client-credentials/). @@ -93,7 +103,7 @@ Depending on how the refresh endpoint is implemented exactly, additional configu * **Token expire property date format** - if not specified, the expiry property is interpreted as the number of seconds the access token will be valid * **Access token property name** - the name of the property in the response that contains the access token to do requests. If not specified, it's set to `access_token` -If the API uses a short-lived refresh token that expires after a short amount of time and needs to be refreshed as well or if other grant types like PKCE are required, it's not possible to use the connector builder with OAuth authentication - check out the [compatibility guide](/connector-development/connector-builder-ui/connector-builder-compatibility#oauth) for more information. +If the API uses other grant types like PKCE are required, it's not possible to use the connector builder with OAuth authentication - check out the [compatibility guide](/connector-development/connector-builder-ui/connector-builder-compatibility#oauth) for more information. Keep in mind that the OAuth authentication method does not implement a single-click authentication experience for the end user configuring the connector - it will still be necessary to obtain client id, client secret and refresh token from the API and manually enter them into the configuration form. @@ -128,22 +138,56 @@ curl -X GET \ https://connect.squareup.com/v2/ ``` -### Other authentication methods - -If your API is not using one of the natively supported authentication methods, it's still possible to build an Airbyte connector as described below. - -#### Access token as query or body parameter - -Some APIs require to include the access token in different parts of the request (for example as a request parameter). For example, the [Breezometer API](https://docs.breezometer.com/api-documentation/introduction/#authentication) is using this kind of authentication. In these cases it's also possible to configure authentication manually: -* Add a user input as secret field on the "User inputs" page (e.g. named `api_key`) -* On the stream page, add a new "Request parameter" -* As key, configure the name of the query parameter the API requires (e.g. named `key`) -* As value, configure a [placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables) for the created user input (e.g. `{{ config['api_key'] }}`) - - - -The same approach can be used to add the token to the request body. - -#### Custom authentication methods +#### Update refresh token from authentication response + +In a lot of cases, OAuth refresh tokens are long-lived and can be used to create access tokens for every sync. In some cases however, a refresh token becomes invalid after it has been used to create an access token. In these situations, a new refresh token is returned along with the access token. One example of this behavior is the [Smartsheets API](https://smartsheet.redoc.ly/#section/OAuth-Walkthrough/Get-or-Refresh-an-Access-Token). In these cases, it's necessary to update the refresh token in the configuration every time an access token is generated, so the next sync will still succeed. + +This can be done using the "Overwrite config with refresh token response" setting. If enabled, the authenticator expects a new refresh token to be returned from the token refresh endpoint. By default, the property `refresh_token` is used to extract the new refresh token, but this can be configured using the "Refresh token property name" setting. The connector then updates its own configuration with the new refresh token and uses it the next time an access token needs to be generated. If this option is used, it's necessary to specify an initial access token along with its expiry date in the "Testing values" menu. + +### Session Token +Some APIs require callers to first fetch a unique token from one endpoint, then make the rest of their calls to all other endpoints using that token to authenticate themselves. These tokens usually have an expiration time, after which a new token needs to be re-fetched to continue making requests. This flow can be achieved through using the Session Token Authenticator. + +If requests are authenticated using the Session Token authentication method, the API documentation page will likely contain one of the following keywords: +- "Session Token" +- "Session ID" +- "Auth Token" +- "Access Token" +- "Temporary Token" + +#### Configuration +The configuration of a Session Token authenticator is a bit more involved than other authenticators, as you need to configure both how to make requests to the session token retrieval endpoint (which requires its own authentication method), as well as how the token is extracted from that response and used for the data requests. + +We will walk through each part of the configuration below. Throughout this, we will refer to the [Metabase API](https://www.metabase.com/learn/administration/metabase-api#authenticate-your-requests-with-a-session-token) as an example of an API that uses session token authentication. +- `Session Token Retrieval` - this is a group of fields which configures how the session token is fetched from the session token endpoint in your API. Once the session token is retrieved, your connector will reuse that token until it expires, at which point it will retrieve a new session token using this configuration. + - `URL` - the full URL of the session token endpoint + - For Metabase, this would be `https://.metabaseapp.com/api/session`. + - `HTTP Method` - the HTTP method that should be used when retrieving the session token endpoint, either `GET` or `POST` + - Metabase requires `POST` for its `/api/session` requests. + - `Authentication Method` - configures the method of authentication to use **for the session token retrieval request only** + - Note that this is separate from the parent Session Token Authenticator. It contains the same options as the parent Authenticator Method dropdown, except for OAuth (which is unlikely to be used for obtaining session tokens) and Session Token (as it does not make sense to nest). + - For Metabase, the `/api/session` endpoint takes in a `username` and `password` in the request body. Since this is a non-standard authentication method, we must set this inner `Authentication Method` to `No Auth`, and instead configure the `Request Body` to pass these credentials (discussed below). + - `Query Parameters` - used to attach query parameters to the session token retrieval request + - Metabase does not require any query parameters in the `/api/session` request, so this is left unset. + - `Request Headers` - used to attach headers to the sesssion token retrieval request + - Metabase does not require any headers in the `/api/session` request, so this is left unset. + - `Request Body` - used to attach a request body to the session token retrieval request + - As mentioned above, Metabase requires the username and password to be sent in the request body, so we can select `JSON (key-value pairs)` here and set the username and password fields (using User Inputs for the values to make the connector reusable), so this would end up looking like: + - Key: `username`, Value: `{{ config['username'] }}` + - Key: `password`, Value: `{{ config['password'] }}` + - `Error Handler` - used to handle errors encountered when retrieving the session token + - See the [Error Handling](/connector-development/connector-builder-ui/error-handling) page for more info about configuring this component. +- `Session Token Path` - an array of values to form a path into the session token retrieval response which points to the session token value + - For Metabase, the `/api/session` response looks like `{"id":""}`, so the value here would simply be `id`. +- `Expiration Duration` - an [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) indicating how long the session token has until it expires + - Once this duration is reached, your connector will automatically fetch a new session token, and continue making data requests with that new one. + - If this is left unset, the session token will be refreshed before every single data request. This is **not recommended** if it can be avoided, as this will cause the connector to run much slower, as it will need to make an extra token request for every data request. + - Note: this **does _not_ support dynamic expiration durations of session tokens**. If your token expiration duration is dynamic, you should set the `Expiration Duration` field to the expected minimum duration to avoid problems during syncing. + - For Metabase, the token retrieved from the `/api/session` endpoint expires after 14 days by default, so this value can be set to `P2W` or `P14D`. +- `Data Request Authentication` - configures how the session token is used to authenticate the data requests made to the API + - Choose `API Key` if your session token needs to be injected into a query parameter or header of the data requests. + - Metabase takes in the session token through a specific header, so this would be set to `API Key`, Inject Session Token into outgoing HTTP Request would be set to `Header`, and Header Name would be set to `X-Metabase-Session`. + - Choose `Bearer` if your session token needs to be sent as a standard Bearer token. + +### Custom authentication methods Some APIs require complex custom authentication schemes involving signing requests or doing multiple requests to authenticate. In these cases, it's required to use the [low-code CDK](/connector-development/config-based/low-code-cdk-overview) or [Python CDK](/connector-development/cdk-python/). diff --git a/docs/connector-development/connector-builder-ui/connector-builder-compatibility.md b/docs/connector-development/connector-builder-ui/connector-builder-compatibility.md index c9fdaf5de3f8..73df9137d71b 100644 --- a/docs/connector-development/connector-builder-ui/connector-builder-compatibility.md +++ b/docs/connector-development/connector-builder-ui/connector-builder-compatibility.md @@ -1,6 +1,7 @@ # Compatibility Guide Answer the following questions to determine whether the Connector Builder is the right tool to build the connector you need: - [ ] [Is it an HTTP API returning a collection of records synchronously?](#is-the-integration-an-http-api-returning-a-collection-of-records-synchronously) +- [ ] [Are data endpoints fixed?](#are-data-endpoints-fixed) - [ ] [Is the API using one of the following authentication mechanism?](#what-type-of-authentication-is-required) - [Basic HTTP](#basic-http) - [API key injected in request header or query parameter](#api-key) @@ -79,6 +80,14 @@ Examples: If the integration is not an HTTP API returning the records synchronously, use the Python CDK. +## Are data endpoints fixed? + +The connector builder requires the data endpoints to be fixed. This means the data endpoints representing separate streams are not dynamically generated based on the data or user configuration, but specified as part of the API documentation. + +For example, the [Congress API](https://api.congress.gov/#/) specifies the data endpoints as part of the documentation, while the [Salesforce API](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_discoveryresource.htm) features a dynamic and configurable list of resources that can't be known in advance. + +If an integration has a dynamic list of data endpoints representing separate streams, use the Python CDK. + ## What type of authentication is required? Look up the authentication mechanism in the API documentation, and identify which type it is. @@ -104,28 +113,16 @@ Are requests authenticated using an OAuth2.0 flow with a refresh token grant typ Examples: [Square](https://developer.squareup.com/docs/oauth-api/overview), [Woocommerce](https://woocommerce.github.io/woocommerce-rest-api-docs/#introduction) -#### Is the OAuth refresh token long-lived? -Using [Gitlab](https://docs.gitlab.com/ee/api/oauth2.html) as an example, you can tell it uses an ephemeral refresh token because the authorization request returns a new refresh token in addition to the access token. This indicates a new refresh token should be used next time. +If the refresh request requires custom query parameters or request headers, use the Python CDK.
    +If the refresh request requires a [grant type](https://oauth.net/2/grant-types/) that is not "Refresh Token" or "Client Credentials", such as an Authorization Code, or a PKCE, use the Python CDK.
    +If the authentication mechanism is OAuth flow 2.0 with refresh token or client credentials and does not require custom query params, it is compatible with the Connector Builder. -Example response: -``` -{ - "access_token": "de6780bc506a0446309bd9362820ba8aed28aa506c71eedbe1c5c4f9dd350e54", - "token_type": "bearer", - "expires_in": 7200, - "refresh_token": "8257e65c97202ed1726cf9571600918f3bffb2544b26e00a61df9897668c33a1", - "created_at": 1607635748 -} -``` +### Session Token +Are data requests authenticated using a temporary session token that is obtained through a separate request? -Example: -- Yes: [Gitlab](https://docs.gitlab.com/ee/api/oauth2.html) -- No: [Square](https://developer.squareup.com/docs/oauth-api/overview), [Woocommerce](https://woocommerce.github.io/woocommerce-rest-api-docs/#introduction) +Examples: [Metabase](https://www.metabase.com/learn/administration/metabase-api#authenticate-your-requests-with-a-session-token), [Splunk](https://dev.splunk.com/observability/reference/api/sessiontokens/latest) -If the OAuth flow requires a single-use refresh token, use the Python CDK. -If the refresh request requires custom query parameters or request headers, use the Python CDK. -If the refresh request requires a [grant type](https://oauth.net/2/grant-types/) that is not "Refresh Token", such as an Authorization Code, or a PKCE, use the Python CDK. -If the authentication mechanism is OAuth flow 2.0 with refresh token and does not require refreshing the refresh token or custom query params, it is compatible with the Connector Builder. +If the authentication mechanism is a session token obtained through calling a separate endpoint, and which expires after some amount of time and needs to be re-obtained, it is compatible with the Connector Builder. ### Other AWS endpoints are examples of APIs requiring a non-standard authentication mechanism. You can tell from [the documentation](https://docs.aws.amazon.com/pdfs/awscloudtrail/latest/APIReference/awscloudtrail-api.pdf#Welcome) that requests need to be signed with a hash. diff --git a/docs/connector-development/connector-builder-ui/incremental-sync.md b/docs/connector-development/connector-builder-ui/incremental-sync.md index 8eb7c621eec2..c6f780fe12cc 100644 --- a/docs/connector-development/connector-builder-ui/incremental-sync.md +++ b/docs/connector-development/connector-builder-ui/incremental-sync.md @@ -7,24 +7,26 @@ This is especially important if there are a large number of records to sync and/ Incremental syncs are usually implemented using a cursor value (like a timestamp) that delineates which data was pulled and which data is new. A very common cursor value is an `updated_at` timestamp. This cursor means that records whose `updated_at` value is less than or equal than that cursor value have been synced already, and that the next sync should only export records whose `updated_at` value is greater than the cursor value. To use incremental syncs, the API endpoint needs to fullfil the following requirements: -* Records contain a top-level date/time field that defines when this record was last updated (the "cursor field") - * If the record's cursor field is nested, you can use an "Add Field" transformation to copy it to the top-level, and a Remove Field to remove it from the object. This will effectively move the field to the top-level of the record -* It's possible to filter/request records by the cursor field -* The records are sorted in ascending order based on their cursor field -The knowledge of a cursor value also allows the Airbyte system to automatically keep a history of changes to records in the destination. To learn more about how different modes of incremental syncs, check out the [Incremental Sync - Append](/understanding-airbyte/connections/incremental-append/) and [Incremental Sync - Deduped History](/understanding-airbyte/connections/incremental-deduped-history) pages. +- Records contain a top-level date/time field that defines when this record was last updated (the "cursor field") + - If the record's cursor field is nested, you can use an "Add Field" transformation to copy it to the top-level, and a Remove Field to remove it from the object. This will effectively move the field to the top-level of the record +- It's possible to filter/request records by the cursor field +- The records are sorted in ascending order based on their cursor field + +The knowledge of a cursor value also allows the Airbyte system to automatically keep a history of changes to records in the destination. To learn more about how different modes of incremental syncs, check out the [Incremental Sync - Append](/understanding-airbyte/connections/incremental-append/) and [Incremental Sync - Append + Deduped](/understanding-airbyte/connections/incremental-append-deduped) pages. ## Configuration To configure incremental syncs for a stream in the connector builder, you have to specify how the records will represent the **"last changed" / "updated at" timestamp**, the **initial time range** to fetch records for and **how to request records from a certain time range**. In the builder UI, these things are specified like this: -* The "Cursor field" is the property in the record that defines the date and time when the record got changed. It's used to decide which records are synced already and which records are "new" -* The "Datetime format" specifies the [format](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes) the cursor field is using to specify date and time, -* The "Cursor granularity" is the smallest time unit supported by the API to specify the time range to request records for expressed as [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) -* The "Start datetime" is the initial start date of the time range to fetch records for. When doing incremental syncs, the second sync will overwrite this date with the last record that got synced so far. -* The "End datetime" is the end date of the time range to fetch records for. In most cases it's set to the current date and time when the sync is started to sync all changes that happened so far. -* The "Inject start/end time into outgoing HTTP request" defines how to request records that got changed in the time range to sync. In most cases the start and end time is added as a request parameter or body parameter + +- The "Cursor field" is the property in the record that defines the date and time when the record got changed. It's used to decide which records are synced already and which records are "new" +- The "Datetime format" specifies the format the cursor field is using to specify date and time. Check out the [YAML reference](/connector-development/config-based/understanding-the-yaml-file/reference#/definitions/DatetimeBasedCursor) for a full list of supported formats. +- "API time filtering capabilities" specifies if the API allows filtering by start and end datetime or whether it's a "feed" of data going from newest to oldest records. See the "Incremental sync without time filtering" section below for details. +- The "Start datetime" is the initial start date of the time range to fetch records for. When doing incremental syncs, the second sync will overwrite this date with the last record that got synced so far. +- The "End datetime" is the end date of the time range to fetch records for. In most cases it's set to the current date and time when the sync is started to sync all changes that happened so far. +- The "Inject start/end time into outgoing HTTP request" defines how to request records that got changed in the time range to sync. In most cases the start and end time is added as a query parameter or body parameter ## Example @@ -33,6 +35,7 @@ The [API of The Guardian](https://open-platform.theguardian.com/documentation/se The `/search` endpoint has a `from-date` and a `to-date` query parameter which can be used to only request data for a certain time range. Content records have the following form: + ``` { "id": "world/2022/oct/21/russia-ukraine-war-latest-what-we-know-on-day-240-of-the-invasion", @@ -46,27 +49,30 @@ Content records have the following form: ``` As this fulfills the requirements for incremental syncs, we can configure the "Incremental sync" section in the following way: -* "Cursor field" is set to `webPublicationDate` -* "Datetime format" is set to `%Y-%m-%dT%H:%M:%SZ` -* "Cursor granularity is set to `PT1S` as this API can handle date/time values on the second level -* "Start datetime" is set to "user input" to allow the user of the connector configuring a Source to specify the time to start syncing -* "End datetime" is set to "now" to fetch all articles up to the current date -* "Inject start time into outgoing HTTP request" is set to `request_parameter` with "Field" set to `from-date` -* "Inject end time into outgoing HTTP request" is set to `request_parameter` with "Field" set to `to-date` + +- "Cursor field" is set to `webPublicationDate` +- "Datetime format" is set to `%Y-%m-%dT%H:%M:%SZ` +- "Start datetime" is set to "user input" to allow the user of the connector configuring a Source to specify the time to start syncing +- "End datetime" is set to "now" to fetch all articles up to the current date +- "Inject start time into outgoing HTTP request" is set to `request_parameter` with "Field" set to `from-date` +- "Inject end time into outgoing HTTP request" is set to `request_parameter` with "Field" set to `to-date` This API orders records by default from new to old, which is not optimal for a reliable sync as the last encountered cursor value will be the most recent date even if some older records did not get synced (for example if a sync fails halfway through). It's better to start with the oldest records and work your way up to make sure that all older records are synced already once a certain date is encountered on a record. In this case the API can be configured to behave like this by setting an additional parameter: -* At the bottom of the stream configuration page, add a new "Request parameter" -* Set the key to `order-by` -* Set the value to `oldest` + +- Add a new "Query Parameter" near the top of the page +- Set the key to `order-by` +- Set the value to `oldest` Setting the start date in the "Testing values" to a date in the past like **2023-04-09T00:00:00Z** results in the following request: + 
     curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-04-09T00:00:00Z&to-date={`now`}'
    The last encountered date will be saved as part of the connection - when the next sync is running, it picks up from the last record. Let's assume the last ecountered article looked like this: + 
     {`{
   "id": "business/live/2023/apr/15/uk-bosses-more-optimistic-energy-prices-fall-ai-spending-boom-economics-business-live",
@@ -78,6 +84,7 @@ The last encountered date will be saved as part of the connection - when the nex
    Then when a sync is triggered for the same connection the next day, the following request is made: + 
     curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-04-15T07:30:58Z&to-date={``}'
    @@ -88,15 +95,28 @@ The `from-date` is set to the cutoff date of articles synced already and the `to In some cases, it's helpful to reference the start and end date of the interval that's currently synced, for example if it needs to be injected into the URL path of the current stream. In these cases it can be referenced using the `{{ stream_interval.start_date }}` and `{{ stream_interval.end_date }}` [placeholders](/connector-development/config-based/understanding-the-yaml-file/reference#variables). Check out [the tutorial](./tutorial.mdx#adding-incremental-reads) for such a case. ::: +## Incremental sync without time filtering + +Some APIs do not allow filtering records by a date field, but instead only provide a paginated "feed" of data that is ordered from newest to oldest. In these cases, the "API time filtering capabilities" option needs to be set to "No filter". As they can't be applied in this situation, the "Inject start time into outgoing HTTP request" and "Inject end time into outgoing HTTP request" options as well as the "Split up interval" option are disabled automatically. + +The `/new` endpoint of the [Reddit API](https://www.reddit.com/dev/api/#GET_new) is such an API. By configuring pagination and setting time filtering capabilities to the "No filter" option, the connector will automatically request the next page of records until the cutoff datetime is encountered. This is done by comparing the cursor value of the records with the either the configured start date or the latest cursor value that was encountered in a previous sync - if the cursor value is less than or equal to that cutoff date, the sync is finished. The latest cursor value is saved as part of the connection and used as the cutoff date for the next sync. + +:::warning +The "No filter" option can only be used if the data is sorted from newest to oldest across pages. If the data is sorted differently, the connector will stop syncing records too late or too early. In these cases it's better to disable incremental syncs and sync the full set of records on a regular schedule. +::: + ## Advanced settings -The description above is sufficient for a lot of APIs. However there are some more subtle configurations which sometimes become relevant. +The description above is sufficient for a lot of APIs. However there are some more subtle configurations which sometimes become relevant. + +### Split up interval + +When incremental syncs are enabled and "Split up interval" is set, the connector is not fetching all records since the cutoff date at once - instead it's splitting up the time range between the cutoff date and the desired end date into intervals based on the "Step" configuration expressed as [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations). -### Step +The "cursor granularity" also needs to be set to an ISO 8601 duration - it represents the smallest possible time unit the API supports to filter records by. It's used to ensure the start of a interval does not overlap with the end of the previous one. -When incremental syncs are enabled and "Step" is set, the connector is not fetching all records since the cutoff date at once - instead it's splitting up the time range between the cutoff date and the desired end date into intervals based on the "Step" configuration expressed as [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations). +For example if the "Step" is set to 10 days (`P10D`) and the "Cursor granularity" set to second (`PT1S`) for the Guardian articles stream described above and a longer time range, then the following requests will be performed: -For example if the "Step" is set to 10 days (`P10D`) for the Guardian articles stream described above and a longer time range, then the following requests will be performed: 
     curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-01-01T00:00:00Z&to-date=2023-01-10T00:00:00Z'{`\n`}
 curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-01-10T00:00:00Z&to-date=2023-01-20T00:00:00Z'{`\n`}
@@ -106,9 +126,10 @@ curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-
 
 After an interval is processed, the cursor value of the last record will be saved as part of the connection as the new cutoff date.
 
-This value is optional and if left unset, the connector will not split up the time range at all but will instead just request all records for the entire target time range. This configuration works for all connectors, but there are two reasons to change it:
-* **To protect a connection against intermittent failures** - if the "Step" size is a day, the cutoff date is saved after all records associated with a day are proccessed. If a sync fails halfway through because the API, the Airbyte system, the destination or the network between these components has a failure, then at most one day worth of data needs to be resynced. However, a smaller step size might cause more requests to the API and more load on the system. It depends on the expected amount of data and load characteristics of an API what step size is optimal, but for a lot of applications the default of one month is a good starting point.
-* **The API requires the connector to fetch data in pre-specified chunks** - for example the [Exchange Rates API](https://exchangeratesapi.io/documentation/) makes the date to fetch data for part of the URL path and only allows to fetch data for a single day at a time
+If left unset, the connector will not split up the time range at all but will instead just request all records for the entire target time range. This configuration works for all connectors, but there are two reasons to change it:
+
+- **To protect a connection against intermittent failures** - if the "Step" size is a day, the cutoff date is saved after all records associated with a day are proccessed. If a sync fails halfway through because the API, the Airbyte system, the destination or the network between these components has a failure, then at most one day worth of data needs to be resynced. However, a smaller step size might cause more requests to the API and more load on the system. It depends on the expected amount of data and load characteristics of an API what step size is optimal, but for a lot of applications the default of one month is a good starting point.
+- **The API requires the connector to fetch data in pre-specified chunks** - for example the [Exchange Rates API](https://exchangeratesapi.io/documentation/) makes the date to fetch data for part of the URL path and only allows to fetch data for a single day at a time
 
 ### Lookback window
 
@@ -117,10 +138,12 @@ The "Lookback window" specifies a duration that is subtracted from the last cuto
 Some APIs update records over time but do not allow to filter or search by modification date, only by creation date. For example the API of The Guardian might change the title of an article after it got published, but the `webPublicationDate` still shows the original date the article got published initially.
 
 In these cases, there are two options:
-* **Do not use incremental sync** and always sync the full set of records to always have a consistent state, losing the advantages of reduced load and [automatic history keeping in the destination](/understanding-airbyte/connections/incremental-deduped-history)
-* **Configure the "Lookback window"** to not only sync exclusively new records, but resync some portion of records before the cutoff date to catch changes that were made to existing records, trading off data consistency and the amount of synced records. In the case of the API of The Guardian, news articles tend to only be updated for a few days after the initial release date, so this strategy should be able to catch most updates without having to resync all articles.
+
+- **Do not use incremental sync** and always sync the full set of records to always have a consistent state, losing the advantages of reduced load and [automatic history keeping in the destination](/understanding-airbyte/connections/incremental-append-deduped)
+- **Configure the "Lookback window"** to not only sync exclusively new records, but resync some portion of records before the cutoff date to catch changes that were made to existing records, trading off data consistency and the amount of synced records. In the case of the API of The Guardian, news articles tend to only be updated for a few days after the initial release date, so this strategy should be able to catch most updates without having to resync all articles.
 
 Reiterating the example from above with a "Lookback window" of 2 days configured, let's assume the last encountered article looked like this:
+
 
     {`{
   "id": "business/live/2023/apr/15/uk-bosses-more-optimistic-energy-prices-fall-ai-spending-boom-economics-business-live",
@@ -132,6 +155,7 @@ Reiterating the example from above with a "Lookback window" of 2 days configured
 
    
 
 Then when a sync is triggered for the same connection the next day, the following request is made:
+
 
     curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-04-13T07:30:58Z&to-date={``}'
 
    
@@ -139,12 +163,13 @@ curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-
 ## Custom parameter injection
 
 Using the "Inject start time / end time into outgoing HTTP request" option in the incremental sync form works for most cases, but sometimes the API has special requirements that can't be handled this way:
-* The API requires adding a prefix or a suffix to the actual value
-* Multiple values need to be put together in a single parameter
-* The value needs to be injected into the URL path
-* Some conditional logic needs to be applied
+
+- The API requires adding a prefix or a suffix to the actual value
+- Multiple values need to be put together in a single parameter
+- The value needs to be injected into the URL path
+- Some conditional logic needs to be applied
 
 To handle these cases, disable injection in the incremental sync form and use the generic parameter section at the bottom of the stream configuration form to freely configure query parameters, headers and properties of the JSON body, by using jinja expressions and [available variables](/connector-development/config-based/understanding-the-yaml-file/reference/#/variables). You can also use these variables as part of the URL path.
 
 For example the [Sendgrid API](https://docs.sendgrid.com/api-reference/e-mail-activity/filter-all-messages) requires setting both start and end time in a `query` parameter.
-For this case, you can use the `stream_interval` variable to configure a request parameter with "key" `query` and "value" `last_event_time BETWEEN TIMESTAMP "{{stream_interval.start_time}}" AND TIMESTAMP "{{stream_interval.end_time}}"` to filter down to the right window in time.
\ No newline at end of file
+For this case, you can use the `stream_interval` variable to configure a query parameter with "key" `query` and "value" `last_event_time BETWEEN TIMESTAMP "{{stream_interval.start_time}}" AND TIMESTAMP "{{stream_interval.end_time}}"` to filter down to the right window in time.
diff --git a/docs/connector-development/connector-builder-ui/pagination.md b/docs/connector-development/connector-builder-ui/pagination.md
index 9d19a970c60e..f10328be6811 100644
--- a/docs/connector-development/connector-builder-ui/pagination.md
+++ b/docs/connector-development/connector-builder-ui/pagination.md
@@ -91,7 +91,7 @@ GET https://api.example.com/products?limit=3&offset=4
      // less than 2 records returned -> stop
 ```
 
-The Connector Builder currently supports injecting these values into the request parameters (i.e. query parameters), headers, or body.
+The Connector Builder currently supports injecting these values into the query parameters (i.e. request parameters), headers, or body.
 
 #### Examples
 
@@ -189,7 +189,7 @@ GET https://api.example.com/products?page_size=3&page=3
      // less than 2 records returned -> stop
 ```
 
-The Connector Builder currently supports injecting these values into the request parameters (i.e. query parameters), headers, or body.
+The Connector Builder currently supports injecting these values into the query parameters (i.e. request parameters), headers, or body.
 
 #### Examples
 
@@ -264,7 +264,7 @@ This API also has a boolean `has_more` property included in the response to indi
 
 The following APIs implement cursor pagination in various ways:
 
-- [Twitter API](https://developer.twitter.com/en/docs/twitter-api/pagination) - includes `next_token` IDs in its responses which are passed in as request parameters to subsequent requests
+- [Twitter API](https://developer.twitter.com/en/docs/twitter-api/pagination) - includes `next_token` IDs in its responses which are passed in as query parameters to subsequent requests
 - [GitHub API](https://docs.github.com/en/rest/guides/using-pagination-in-the-rest-api?apiVersion=2022-11-28) - includes full-URL `link`s to subsequent pages of results
 - [FourSquare API](https://location.foursquare.com/developer/reference/pagination) - includes full-URL `link`s to subsequent pages of results
 
@@ -278,5 +278,5 @@ Using the "Inject page size / limit / offset into outgoing HTTP request" option
 
 To handle these cases, disable injection in the pagination form and use the generic parameter section at the bottom of the stream configuration form to freely configure query parameters, headers and properties of the JSON body, by using jinja expressions and [available variables](/connector-development/config-based/understanding-the-yaml-file/reference/#/variables). You can also use these variables as part of the URL path.
 
-For example the [Prestashop API](https://devdocs.prestashop-project.org/8/webservice/cheat-sheet/#list-options) requires to set offset and limit separated by a comma into a single request parameter (`?limit=,`)
-For this case, you can use the `next_page_token` variable to configure a request parameter with key `limit` and value `{{ next_page_token['next_page_token'] or '0' }},50` to inject the offset from the pagination strategy and a hardcoded limit of 50 into the same parameter.
\ No newline at end of file
+For example the [Prestashop API](https://devdocs.prestashop-project.org/8/webservice/cheat-sheet/#list-options) requires to set offset and limit separated by a comma into a single query parameter (`?limit=,`)
+For this case, you can use the `next_page_token` variable to configure a query parameter with key `limit` and value `{{ next_page_token['next_page_token'] or '0' }},50` to inject the offset from the pagination strategy and a hardcoded limit of 50 into the same parameter.
\ No newline at end of file
diff --git a/docs/connector-development/connector-builder-ui/partitioning.md b/docs/connector-development/connector-builder-ui/partitioning.md
index df4ed933a01b..3ac5afcb862a 100644
--- a/docs/connector-development/connector-builder-ui/partitioning.md
+++ b/docs/connector-development/connector-builder-ui/partitioning.md
@@ -45,7 +45,7 @@ To enable user-configurable static partitions for the [Woocommerce API](https://
 * Name it `Order IDs`, set type to `array` and click create
 * Set "Current partition value identifier" to `order`
 * "Inject partition value into outgoing HTTP request" is disabled, because the order id needs to be injected into the path
-* In the general section of the stream configuration, the "Path URL" is set to `/orders/{{ stream_partition.order }}/notes`
+* In the general section of the stream configuration, the "URL Path" is set to `/orders/{{ stream_partition.order }}/notes`
 
 
 
@@ -70,7 +70,7 @@ The following fields have to be configured to use the substream partition router
 To enable dynamic partition routing for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, first an "orders" stream needs to be configured for the `/orders` endpoint to fetch a list of orders. Once this is done, the partition router for responses has be configured like this:
 * "Parent key" is set to `id`
 * "Current partition value identifier" is set to `order`
-* In the general section of the stream configuration, the "Path URL" is set to `/orders/{{ stream_partition.order }}/notes`
+* In the general section of the stream configuration, the "URL Path" is set to `/orders/{{ stream_partition.order }}/notes`
 
 
 
diff --git a/docs/connector-development/connector-builder-ui/record-processing.mdx b/docs/connector-development/connector-builder-ui/record-processing.mdx
index f4a50ec9b4a4..d5ac0dbb88de 100644
--- a/docs/connector-development/connector-builder-ui/record-processing.mdx
+++ b/docs/connector-development/connector-builder-ui/record-processing.mdx
@@ -1,27 +1,39 @@
-import Diff from './assets/record-processing-schema-diff.png';
+import Diff from "./assets/record-processing-schema-diff.png";
 
 # Record processing
 
 Connectors built with the connector builder always make HTTP requests, receive the responses and emit records. Besides making the right requests, it's important to properly hand over the records to the system:
-* Extract the records (record selection)
-* Do optional post-processing (transformations)
-* Provide record meta data to the system to inform downstream processes (primary key and declared schema)
+
+- Extract the records (record selection)
+- Do optional post-processing (transformations)
+- Provide record meta data to the system to inform downstream processes (primary key and declared schema)
 
 ## Record selection
 
-
+
 
 When doing HTTP requests, the connector expects the records to be part of the response JSON body. The "Record selector" field of the stream needs to be set to the property of the response object that holds the records.
 
 Very often, the response body contains an array of records along with some suplementary information (for example meta data for pagination).
 
 For example the ["Most popular" NY Times API](https://developer.nytimes.com/docs/most-popular-product/1/overview) returns the following response body:
+
 
    -{`{
+  {`{
     "status": "OK",
     "copyright": "Copyright (c) 2023 The New York Times Company.  All Rights Reserved.",
     "num_results": 20,
-    `}{`"results": [`}{`
+    `}
+  {`"results": [`}
+  {`
       {
         "uri": "nyt://article/c15e5227-ed68-54d9-9e5b-acf5a451ec37",
         "url": "https://www.nytimes.com/2023/04/16/us/science-of-reading-literacy-parents.html",
@@ -31,7 +43,9 @@ For example the ["Most popular" NY Times API](https://developer.nytimes.com/docs
         // ...
       },
       // ..
-    `}{`]`}{`,
+    `}
+  {`]`}
+  {`,
     // ...
 }`}
 
    
@@ -41,11 +55,14 @@ For example the ["Most popular" NY Times API](https://developer.nytimes.com/docs
 ### Nested objects
 
 In some cases the array of actual records is nested multiple levels deep in the response, like for the ["Archive" NY Times API](https://developer.nytimes.com/docs/archive-product/1/overview):
+
 
    -{`{
+  {`{
     "copyright": "Copyright (c) 2020 The New York Times Company. All Rights Reserved.",
     "response": {
-      `}{`"docs": [`}{`
+      `}
+  {`"docs": [`}
+  {`
         {
           "abstract": "From the Treaty of Versailles to Prohibition, the events of that year shaped America, and the world, for a century to come. ",
           "web_url": "https://www.nytimes.com/2018/12/31/opinion/1919-america.html",
@@ -53,9 +70,11 @@ In some cases the array of actual records is nested multiple levels deep in the
           // ...
         },
         // ...
-      `}{`]`}{`
+      `}
+  {`]`}
+  {`
     }
-}`}    
+}`}
 
    
 
 **Setting the record selector needs to be set to "`response`,`docs`"** selects the nested array.
@@ -63,8 +82,10 @@ In some cases the array of actual records is nested multiple levels deep in the
 ### Root array
 
 In some cases, the response body itself is an array of records, like in the [CoinAPI API](https://docs.coinapi.io/market-data/rest-api/quotes):
+
 
    -{`[`}{`
+  {`[`}
+  {`
   {
     "symbol_id": "BITSTAMP_SPOT_BTC_USD",
     "time_exchange": "2013-09-28T22:40:50.0000000Z",
@@ -78,7 +99,8 @@ In some cases, the response body itself is an array of records, like in the [Coi
    // ..
   }
   // ...
-`}{`]`}
+`}
+  {`]`}
 
    
 
 In this case, **the record selector can be omitted** and the whole response becomes the list of records.
@@ -86,18 +108,23 @@ In this case, **the record selector can be omitted** and the whole response beco
 ### Single object
 
 Sometimes, there is only one record returned per request from the API. In this case, the record selector can also point to an object instead of an array which will be handled as the only record, like in the case of the [Exchange Rates API](https://exchangeratesapi.io/documentation/#historicalrates):
+
 
    -{`{
+  {`{
     "success": true,
     "historical": true,
     "date": "2013-12-24",
     "timestamp": 1387929599,
     "base": "GBP",
-    `}{`"rates": {`}{`
+    `}
+  {`"rates": {`}
+  {`
         "USD": 1.636492,
         "EUR": 1.196476,
         "CAD": 1.739516
-    `}{`}`}{`
+    `}
+  {`}`}
+  {`
 }`}
 
    
 
@@ -142,17 +169,27 @@ In this case a record selector with a placeholder `*` selects all children at th
 ## Transformations
 
 It is recommended to not change records during the extraction process the connector is performing, but instead load them into the downstream warehouse unchanged and perform necessary transformations there in order to stay flexible in what data is required. However there are some reasons that require the modifying the fields of records before they are sent to the warehouse:
-* Remove personally identifiable information (PII) to ensure compliance with local legislation
-* Pseudonymise sensitive fields
-* Remove large fields that don't contain interesting information and significantly increase load on the system
+
+- Remove personally identifiable information (PII) to ensure compliance with local legislation
+- Pseudonymise sensitive fields
+- Remove large fields that don't contain interesting information and significantly increase load on the system
 
 The "transformations" feature can be used for these purposes.
 
 ### Removing fields
 
-
+
 
 To remove a field from a record, add a new transformation in the "Transformations" section of type "remove" and enter the field path. For example in case of the [EmailOctopus API](https://emailoctopus.com/api-documentation/campaigns/get-all), the campaigns records also include the html content of the mailing which takes up a lot of space:
+
 ```
 {
     "data": [
@@ -177,6 +214,7 @@ To remove a field from a record, add a new transformation in the "Transformation
 ```
 
 Setting the "Path" of the remove-transformation to `content` removes these fields from the records:
+
 ```
 {
     "id": "00000000-0000-0000-0000-000000000000",
@@ -212,6 +250,7 @@ Imagine that regardless of which level a properties appears, it should be remove
 ```
 
 The `*` character can also be used as a placeholder to filter for all fields that start with a certain prefix - the "Path" `s*` will remove all fields from the top level that start with the character s:
+
 ```
 {
     "id": "00000000-0000-0000-0000-000000000000",
@@ -222,12 +261,20 @@ The `*` character can also be used as a placeholder to filter for all fields tha
 }
 ```
 
-
 ### Adding fields
 
-
+
 
 Adding fields can be used to apply a hashing function to an existing field to pseudonymize it. To do this, add a new transformation in the "Transformations" section of type "add" and enter the field path and the new value. For example in case of the [EmailOctopus API](https://emailoctopus.com/api-documentation/campaigns/get-all), the campaigns records include the name of the sender:
+
 ```
 {
     "data": [
@@ -248,6 +295,7 @@ Adding fields can be used to apply a hashing function to an existing field to ps
 ```
 
 To apply a hash function to it, set the "Path" to "`from`, `name`" to select the name property nested in the from object and set the value to `{{ record['from']['name'] | hash('md5') }}`. This hashes the name in the record:
+
 ```
 {
     "id": "00000000-0000-0000-0000-000000000000",
@@ -273,11 +321,12 @@ Besides bringing the records in the right shape, it's important to communicate s
 
 ### Primary key
 
-The "Primary key" field specifies how to uniquely identify a record. This is important for downstream de-duplication of records (e.g. by the [incremental sync - deduped history sync mode](/understanding-airbyte/connections/incremental-deduped-history)).
+The "Primary key" field specifies how to uniquely identify a record. This is important for downstream de-duplication of records (e.g. by the [incremental sync - Append + Deduped sync mode](/understanding-airbyte/connections/incremental-append-deduped)).
 
 In a lot of cases, like for the EmailOctopus example from above, there is a dedicated id field that can be used for this purpose. It's important that the value of the id field is guaranteed to only occur once for a single record.
 
 In some cases there is no such field but a combination of multiple fields is guaranteed to be unique, for example the shipping zone locations of the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#shipping-zone-locations) do not have an id, but each combination of the `code` and `type` fields is guaranteed to be unique:
+
 ```
 [
   {
@@ -298,13 +347,16 @@ In this case, the "Primary key" can be set to "`code`, `type`" to allow automati
 Similar to the "Primary key", the "Declared schema" defines how the records will be shaped via a [JSON Schema definition](https://json-schema.org/). It defines which fields and nested fields occur in the records, whether they are always available or sometimes missing and which types they are.
 
 This information is used by the Airbyte system for different purposes:
-* **Column selection** when configuring a connection - in Airbyte cloud, the declared schema allows the user to pick which columns/fields are passed to the destination to dynamically reduce the amount of synced data
-* **Recreating the data structure with right columns** in destination - this allows a warehouse destination to create a SQL table which the columns matching the fields of records
-* **Detecting schema changes** - if the schema of a stream changes for an existing connection, this situation can be handled gracefully by Airbyte instead of causing errors in the destination
 
-When doing test reads, the connector builder analyzes the test records and shows the derived schema in the "Detected schema" tab. This schema can be copied over as the declared schema of the stream with a single click.
+- **Column selection** when configuring a connection - in Airbyte cloud, the declared schema allows the user to pick which columns/fields are passed to the destination to dynamically reduce the amount of synced data
+- **Recreating the data structure with right columns** in destination - this allows a warehouse destination to create a SQL table which the columns matching the fields of records
+- **Detecting schema changes** - if the schema of a stream changes for an existing connection, this situation can be handled gracefully by Airbyte instead of causing errors in the destination
+
+When doing test reads, the connector builder analyzes the test records and shows the derived schema in the "Detected schema" tab. By default, new streams are configured to automatically import the detected schema into the declared schema on every test read.
+This behavior can be toggled off by disabling the `Automatically import declared schema` switch, in which case the declared schema can be manually edited in the UI and it will no longer be automatically updated when triggering test reads.
 
 For example the following test records:
+
 ```
 [
   {
@@ -325,6 +377,7 @@ For example the following test records:
 ```
 
 result in the following schema:
+
 ```
 {
   "$schema": "http://json-schema.org/schema#",
@@ -351,11 +404,16 @@ result in the following schema:
 
 More strict is always better, but the detected schema is a good default to rely on. See the documentation about [supported data types](https://docs.airbyte.com/understanding-airbyte/supported-data-types/) for JSON schema structures that will be picked up by the Airbyte system.
 
-In case the declared schema deviates from the detected schema, the "Detected schema" tab in the testing panel highlights the differences. It's important to note that differences are not necessarily a problem that needs to be fixed - in some cases the currently loaded set of records in the testing panel doesn't feature all possible cases so the detected schema is too strict. However, if the declared schema is incompatible with the detected schema based on the test records, it's very likely there will be errors when running syncs.
+If `Automatically import detected schema` is disabled, and the declared schema deviates from the detected schema, the "Detected schema" tab in the testing panel highlights the differences. It's important to note that differences are not necessarily a problem that needs to be fixed - in some cases the currently loaded set of records in the testing panel doesn't feature all possible cases so the detected schema is too strict. However, if the declared schema is incompatible with the detected schema based on the test records, it's very likely there will be errors when running syncs.
 
-Detected schema with highlighted differences
+Detected schema with highlighted differences
 
 In the case of the example above, there are two differences between detected and declared schema. The first difference for the `name` field is not problematic:
+
 ```
      "name": {
 -      "type": [
@@ -369,6 +427,7 @@ In the case of the example above, there are two differences between detected and
 The declared schema allows the `null` value for the name while the detected schema only encountered strings. If it's possible the `name` is set to null, the detected schema is configured correctly.
 
 The second difference will likely cause problems:
+
 ```
      "subject": {
 -      "type": "number"
@@ -377,7 +436,8 @@ The second difference will likely cause problems:
 ```
 
 The `subject` field was detected as `string`, but is configured to be a `number` in the declared schema. As the API returned string subjects during testing, it's likely this will also happen during syncs which would render the declared schema inaccurate. Depending on the situation this can be fixed in multiple ways:
-* If the API changed and subject is always a string now, the declared schema should be updated to reflect this: `"subject": { "type": "string" }`
-* If the API is sometimes returning subject as number of string depending on the record, the declared schema should be updated to allow both data types: `"subject": { "type": ["string","number"] }`
+
+- If the API changed and subject is always a string now, the declared schema should be updated to reflect this: `"subject": { "type": "string" }`
+- If the API is sometimes returning subject as number of string depending on the record, the declared schema should be updated to allow both data types: `"subject": { "type": ["string","number"] }`
 
 A common situation is that certain record fields do not have any any values for the test read data, so they are set to `null`. In the detected schema, these field are of type `"null"` which is most likely not correct for all cases. In these situations, the declared schema should be manually corrected.
diff --git a/docs/connector-development/connector-builder-ui/tutorial.mdx b/docs/connector-development/connector-builder-ui/tutorial.mdx
index 16e22dc5dc31..3be949c1c23f 100644
--- a/docs/connector-development/connector-builder-ui/tutorial.mdx
+++ b/docs/connector-development/connector-builder-ui/tutorial.mdx
@@ -104,17 +104,7 @@ In a real sync, this record will be passed on to a destination like a warehouse.
 
 The request/response tabs are helpful during development to see which requests and responses your connector will send and receive from the API.
 
-### Declaring the record schema
-
-
    
-
-Each stream of a connector needs to declare how emitted records will look like (which properties do they have, what data types will be used, ...). During a sync, this information will be passed on to the destination to configure it correctly - for example a SQL database destination can use it to properly set up the destination table, assigning the right type to each column.
-
-By default, the stream schema is set to a simple object with unspecified properties. However, the connector builder can infer the schema based on the test read you just issued. To use the infered schema, switch to the "Detected schema" tab and click the "Import schema" button.
-
-The warning icon disappears indicating that the declared schema of your stream matches the test data.
-
-You can find more information about schema declaration on the [record processing concept page](./record-processing).
+The detected schema tab indicates the schema that was detected by analyzing the returned records; this detected schema is automatically set as the declared schema for this stream, which you can see by visiting the Declared schema tab in the center stream configuration view.
 
 ## Step 3 - Advanced configuration
 
@@ -122,10 +112,10 @@ You can find more information about schema declaration on the [record processing
 
 
    
 
-The exchange rate API supports configuring a different base currency via request parameter - let's make this part of the user inputs that can be controlled by the user of the connector when configuring a source, similar to the API key.
+The exchange rate API supports configuring a different base currency via query parameter - let's make this part of the user inputs that can be controlled by the user of the connector when configuring a source, similar to the API key.
 
 To do so, follow these steps:
-* Scroll down to the "Request parameters" section and add a new request parameter
+* Scroll down to the "Query Parameters" section and add a new query parameter
 * Set the key to `base`
 * For the value, click the user icon in the input and select "New user input"
 * Set the name to "Base"
@@ -153,7 +143,7 @@ The record should update to use USD as the base currency:
 
 ### Adding incremental reads
 
-
    
+
    
 
 We now have a working implementation of a connector reading the latest exchange rates for a given currency.
 In this section, we'll update the source to read historical data instead of only reading the latest exchange rates.
@@ -161,16 +151,17 @@ In this section, we'll update the source to read historical data instead of only
 According to the API documentation, we can read the exchange rate for a specific date range by querying the `"/exchangerates_data/{date}"` endpoint instead of `"/exchangerates_data/latest"`.
 
 To configure your connector to request every day individually, follow these steps:
-* On top of the form, change the "Path URL" input to `/exchangerates_data/{{ stream_interval.start_time }}` to [inject](/connector-development/config-based/understanding-the-yaml-file/reference#variables) the date to fetch data for into the path of the request
+* On top of the form, change the "URL Path" input to `/exchangerates_data/{{ stream_interval.start_time }}` to [inject](/connector-development/config-based/understanding-the-yaml-file/reference#variables) the date to fetch data for into the path of the request
 * Enable "Incremental sync" for the Rates stream
-* Set the "Cursor field" to `date` - this is the property in our records to check what date got synced last
-* Set the "Datetime format" to `%Y-%m-%d` to match the format of the date in the record returned from the API
-* Set the "Cursor granularity" to `P1D` to tell the connector the API only supports daily increments
+* Set the "Cursor Field" to `date` - this is the property in our records to check what date got synced last
+* Set the "Cursor Field Datetime Format" to `%Y-%m-%d` to match the format of the date in the record returned from the API
 * Leave start time to "User input" so the end user can set the desired start time for syncing data
 * Leave end time to "Now" to always sync exchange rates up to the current date
-* In a lot of cases the start and end date are injected into the request body or request parameters. However in the case of the exchange rate API it needs to be added to the path of the request, so disable the "Inject start/end time into outgoing HTTP request" options
-* Open the "Advanced" section and set "Step" to `P1D` to configure the connector to do one separate request per day by partitioning the dataset into daily intervals
-* Set a start date (like `2023-03-03`) in the "Testing values" menu
+* In a lot of cases the start and end date are injected into the request body or query parameters. However in the case of the exchange rate API it needs to be added to the path of the request, so disable the "Inject start/end time into outgoing HTTP request" options
+* Open the "Advanced" section and enable "Split up interval" so that the connector will partition the dataset into chunks
+* Set "Step" to `P1D` to configure the connector to do one separate request per day
+* Set the "Cursor granularity" to `P1D` to tell the connector the API only supports daily increments
+* Set a start date (like `2023-06-11`) in the "Testing values" menu
 * Hit the "Test" button to trigger a new test read
 
 Now, you should see a dropdown above the records view that lets you step through the daily exchange rates along with the requests performed to fetch this data. Note that in the connector builder at most 5 partitions are requested to speed up testing. During a proper sync the full time range between your configured start date and the current day will be executed.
@@ -179,21 +170,6 @@ When used in a connection, the connector will make sure exchange rates for the s
 
 You can find more information about incremental syncs on the [incremental sync concept page](./incremental-sync).
 
-### Adding transformations
-
-
    
-
-Note that a warning icon should show next to the "Detected schema" tab - using the per-date endpoint instead of the latest endpoint slightly changed the shape of the records by adding a `historical` property. As we don't need this property in our destination, we can remove it using a transformation.
-
-To do so, follow these steps:
-* Enable the "Transformations" section
-* Set the "Path" to `historical`
-* Trigger a new test read
-
-The `historical` property in the records tab and the schema warning should disappear.
-
-You can find more information about incremental syncs on the [incremental sync concept page](./incremental-sync).
-
 ## Step 4 - Publishing and syncing
 
 
    
@@ -215,7 +191,6 @@ Congratulations! You just completed the following steps:
 * Configured a production-ready connector to extract currency exchange data from an HTTP-based API:
   * Configurable API key, start date and base currency
   * Incremental sync to keep the number of requests small
-  * Schema declaration to enable normalization in the destination
 * Tested whether the connector works correctly in the builder
 * Made the working connector available to configure sources in the workspace
 * Set up a connection using the published connector and synced data from the Exchange Rates API
@@ -226,8 +201,8 @@ This tutorial didn't go into depth about all features that can be used in the co
 * [Authentication](/connector-development/connector-builder-ui/authentication/)
 * [Record processing](/connector-development/connector-builder-ui/record-processing/)
 * [Pagination](/connector-development/connector-builder-ui/pagination/)
-* [Error handling](/connector-development/connector-builder-ui/error-handling/)
 * [Incremental sync](/connector-development/connector-builder-ui/incremental-sync/)
 * [Partitioning](/connector-development/connector-builder-ui/partitioning/)
+* [Error handling](/connector-development/connector-builder-ui/error-handling/)
 
 Not every possible API can be consumed by connectors configured in the connector builder. The [compatibility guide](/connector-development/connector-builder-ui/connector-builder-compatibility#oauth) can help determining whether another technology should be used to integrate an API with the Airbyte platform.
diff --git a/docs/connector-development/connector-metadata-file.md b/docs/connector-development/connector-metadata-file.md
index 7a274de5dba0..bef8204a61a8 100644
--- a/docs/connector-development/connector-metadata-file.md
+++ b/docs/connector-development/connector-metadata-file.md
@@ -33,8 +33,8 @@ data:
       enabled: true
     oss:
       enabled: true
-  releaseStage: generally_available
-  supportUrl: https://docs.airbyte.com/integrations/sources/postgres
+  supportLevel: certified
+  documentationUrl: https://docs.airbyte.com/integrations/sources/postgres
 metadataSpecVersion: "1.0"
 ```
 
@@ -85,4 +85,42 @@ In the example above, the connector has three tags. Tags are used for two primar
 
 2. **Keywords for Searching**: Tags that begin with keyword: are used to make the connector more discoverable by adding searchable terms related to it. In the example above, the tags keyword:database and keyword:SQL can be used to find this connector when searching for `database` or `SQL`.
 
-These are just examples of how tags can be used. As a free-form field, the tags list can be customized as required for each connector. This flexibility allows tags to be a powerful tool for managing and discovering connectors.
\ No newline at end of file
+These are just examples of how tags can be used. As a free-form field, the tags list can be customized as required for each connector. This flexibility allows tags to be a powerful tool for managing and discovering connectors.
+
+## The `icon` Field
+This denotes the name of the icon file for the connector. At this time the icon file is located in the `platform-internal` repository. So please ensure that the icon file is present in the `platform-internal` repository at [oss/airbyte-config/init/src/main/resources/icons](https://github.com/airbytehq/airbyte-platform-internal/tree/master/oss/airbyte-config/init/src/main/resources/icons) before adding the icon name to the `metadata.yaml` file.
+
+### Future Plans
+_⚠️ This property is in the process of being refactored to be a file in the connector folder_
+
+You may notice a `icon.svg` file in the connectors folder.
+
+This is because we are transitioning away from icons being stored in the `platform-internal` repository. Instead, we will be storing them in the connector folder itself. This will allow us to have a single source of truth for all connector-related information.
+
+This transition is currently in progress. Once it is complete, the `icon` field in the `metadata.yaml` file will be removed, and the `icon.svg` file will be used instead.
+
+## The `releases` Section
+The `releases` section contains extra information about certain types of releases. The current types of releases are:
+* `breakingChanges`
+
+### `breakingChanges`
+
+The `breakingChanges` section of `releases` contains a dictionary of version numbers (usually major versions, i.e. `1.0.0`) and information about
+their associated breaking changes. Each entry must contain the following parameters:
+* `message`: A description of the breaking change, written in a user-friendly format. This message should briefly describe
+  * What the breaking change is
+  * How it affects the user (or which users it will affect)
+  * What the user should do to fix the issue
+* `upgradeDeadline`: (`YYYY-MM-DD`) The date by which the user should upgrade to the new version.
+
+Note that the `message` should be brief no matter how involved the fix is - the user will be redirected to the migration documentation for the
+full upgrade/migration instructions.
+
+Here is an example:
+```yaml
+releases:
+  breakingChanges:
+    1.0.0:
+      message: "This version changes the connector’s authentication by removing ApiKey authentication, which is now deprecated by the [upstream source](upsteam-docs-url.com). Users currently using ApiKey auth will need to reauthenticate with OAuth after upgrading to continue syncing."
+      upgradeDeadline: "2023-12-31"  # The date that the upstream API stops support for ApiKey authentication
+```
\ No newline at end of file
diff --git a/docs/connector-development/connector-specification-reference.md b/docs/connector-development/connector-specification-reference.md
index 56fcf0ce49bc..6bdb79b2aef4 100644
--- a/docs/connector-development/connector-specification-reference.md
+++ b/docs/connector-development/connector-specification-reference.md
@@ -57,7 +57,9 @@ Additionally, `order` values cannot be duplicated within the same object or grou
 
 By default, all optional fields will be collapsed into an `Optional fields` section which can be expanded or collapsed by the user. This helps streamline the UI for setting up a connector by initially focusing attention on the required fields only. For existing connectors, if their configuration contains a non-empty and non-default value for a collapsed optional field, then that section will be automatically opened when the connector is opened in the UI.
 
-These `Optional fields` sections are placed at the bottom of a field group, meaning that all required fields in the same group will be placed above it. To interleave optional fields with required fields, set `always_show: true` on the optional field along with an `order`, which will cause the field to no longer be collapsed in an `Optional fields` section and be ordered as normal. **Note:** `always_show` is only allowed on optional fields.
+These `Optional fields` sections are placed at the bottom of a field group, meaning that all required fields in the same group will be placed above it. To interleave optional fields with required fields, set `always_show: true` on the optional field along with an `order`, which will cause the field to no longer be collapsed in an `Optional fields` section and be ordered as normal. 
+
+**Note:** `always_show` also causes fields that are normally hidden by an OAuth button to still be shwon.
 
 Within a collapsed `Optional fields` section, the optional fields' `order` defines their position in the section; those without an `order` will be placed after fields with an `order`, and will themselves be ordered alphabetically by field name.
 
diff --git a/docs/connector-development/debugging-docker.md b/docs/connector-development/debugging-docker.md
index bacedbcc0aeb..3f707fc0d3d8 100644
--- a/docs/connector-development/debugging-docker.md
+++ b/docs/connector-development/debugging-docker.md
@@ -84,7 +84,7 @@ In the `docker-compose.debug.yaml` file you should see an entry for the `worker`
 Similar to the previous debugging example, we want to pass an environment variable to the `docker compose` command. This time we're setting the 
 `DEBUG_CONTAINER_IMAGE` environment variable to the name of the container we're targeting. For our example that is `destination-postgres` so run the command:
 ```bash
-DEBUG_CONTAINER_IMAGE="destination-postgres" VERSION="dev" docker compose -f docker-compose.yaml -f docker-compose.debug.yaml up
+DEBUG_CONTAINER_IMAGE="destination-postgres:5005" VERSION="dev" docker compose -f docker-compose.yaml -f docker-compose.debug.yaml up
 ```
 The `worker` container now has an environment variable `DEBUG_CONTAINER_IMAGE` with a value of `destination-postgres` which when it compares when it is
 spawning containers. If the container name matches the environment variable, it will set the `JAVA_TOOL_OPTIONS` environment variable in the container to
diff --git a/docs/connector-development/testing-connectors/README.md b/docs/connector-development/testing-connectors/README.md
index 4d212ba610eb..4c0abe0e51fc 100644
--- a/docs/connector-development/testing-connectors/README.md
+++ b/docs/connector-development/testing-connectors/README.md
@@ -1,62 +1,44 @@
 # Testing Connectors
 
-## Running Integration tests
-
-The GitHub `master` and branch builds will build the core Airbyte infrastructure \(scheduler, ui, etc\) as well as the images for all connectors. Integration tests \(tests that run a connector's image against an external resource\) can be run one of three ways.
-
-### 1. Local iteration
-
-First, you can run the image locally. Connectors should have instructions in the connector's README on how to create or pull credentials necessary for the test. Also, during local development, there is usually a `main` entrypoint for Java integrations or `main_dev.py` for Python integrations that let you run your connector without containerization, which is fastest for iteration.
-
-### 2. Code Static Checkers
-
-#### Python Code
-Using the following tools:
-1. flake8
-2. black
-3. isort
-4. mypy
-
-Airbyte CI/CD workflows use them during "test" commands obligatorily. 
-All their settings are aggregated into the single file `pyproject.toml` into Airbyte project root.
-Locally all these tools can be launched by the following gradle command:
-```
- ./gradlew --no-daemon  :airbyte-integrations:connectors::airbytePythonFormat
-```
-For instance:
+## Our testing pyramid
+Multiple tests suites compose the Airbyte connector testing pyramid:
+Connector specific tests declared in the connector code directory:
+* Unit tests
+* Integration tests
+
+Tests common to all connectors:
+* [QA checks](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/connector_ops/ci_connector_ops/qa_checks.py#L1)
+* [Connector Acceptance tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/)
+
+## Running tests
+Unit and integration tests can be run directly from the connector code.
+
+Using `pytest` for Python connectors:
+```bash
+python -m pytest unit_tests/
+python -m pytest integration_tests/
 ```
-./gradlew --no-daemon  :airbyte-integrations:connectors:source-s3:airbytePythonFormat
-./gradlew --no-daemon  :airbyte-integrations:connectors:source-salesforce:airbytePythonFormat
-```
-### 3. Requesting GitHub PR Integration Test Runs
-
-:::caution
-
-This option is not available to PRs from forks, so it is effectively limited to Airbyte employees.
-
-:::
 
-If you don't want to handle secrets, you're making a relatively minor change, or you want to ensure the connector's integration test will run remotely, you should request builds on GitHub. You can request an integration test run by creating a comment with a slash command.
+Using `gradle` for Java connectors:
 
-Here are some example commands:
-
-1. `/test connector=all` - Runs integration tests for all connectors in a single GitHub workflow. Some of our integration tests interact with rate-limited resources, so please use this judiciously.
-2. `/test connector=source-sendgrid` - Runs integration tests for a single connector on the latest PR commit.
-3. `/test connector=connectors/source-sendgrid` - Runs integration tests for a single connector on the latest PR commit.
-4. `/test connector=source-sendgrid ref=master` - Runs integration tests for a single connector on a different branch.
-5. `/test connector=source-sendgrid ref=d5c53102` - Runs integration tests for a single connector on a specific commit.
-6. `/test connector=source-sendgrid local_cdk=1` - Runs integration tests for a single connector on the latest PR commit, against any CDK changes on that commit.
-
-A command dispatcher GitHub workflow will launch on comment submission. This dispatcher will add an :eyes: reaction to the comment when it starts processing. If there is an error dispatching your request, an error will be appended to your comment. If it launches the test run successfully, a :rocket: reaction will appear on your comment.
+```bash
+./gradlew :airbyte-integrations:connectors:source-postgres:test
+./gradlew :airbyte-integrations:connectors:source-postgres:integrationTestJava
+```
 
-Once the integration test workflow launches, it will append a link to the workflow at the end of the comment. A success or failure response will also be added upon workflow completion.
+Please note that according to the test implementation you might have to provide connector configurations as a `config.json` file in a `.secrets` folder in the connector code directory.
 
-Integration tests can also be manually requested by clicking "[Run workflow](https://github.com/airbytehq/airbyte/actions?query=workflow%3Aintegration-test)" and specifying the connector and GitHub ref.
 
+If you want to run the global test suite, exactly like what is run in CI, you should install [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) and use the following command:
 
-### 4. Automatically Run From `master`
+```bash
+airbyte-ci connectors --name= test
+```
 
-Commits to `master` attempt to launch integration tests. Two workflows launch for each commit: one is a launcher for integration tests, the other is the core build \(the same as the default for PR and branch builds\).
+This will run all the tests for the connector, including the QA checks and the Connector Acceptance tests.
+Connector Acceptance tests require connector configuration to be provided as a `config.json` file in a `.secrets` folder in the connector code directory.
 
-Since some of our connectors use rate-limited external resources, we don't want to overload from multiple commits to master. If a certain threshold of `master` integration tests are running, the integration test launcher passes but does not launch any tests. This can manually be re-run if necessary. The `master` build also runs every few hours automatically, and will launch the integration tests at that time.
 
+## Tests on pull requests
+Our CI infrastructure runs the connector tests with [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md). Connectors tests are automatically and remotely triggered on your branch according to the changes made in your branch.
+**Passing tests are required to merge a connector pull request.**
\ No newline at end of file
diff --git a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md
index a33b9cd95537..2d9196626f3a 100644
--- a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md
+++ b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md
@@ -36,27 +36,38 @@ Build your connector image if needed.
 docker build .
 ```
 
-Run one of the two scripts in the root of the connector:
+And test via one of the two following Options
 
-- `python -m pytest -p integration_tests.acceptance` - to run tests inside virtual environment
+### (Prefered) Option 1: Run against the production acceptance test image
 
-  - On test completion, a log will be outputted to the terminal verifying:
+From the root of your connector run:
+```bash
+./acceptance-test-docker.sh
+```
+
+This will run you local connector image against the same test suite that Airbyte uses in production
 
-    - The connector the tests were ran for
-    - The git hash of the code used
-    - Whether the tests passed or failed
+### Option 2: Run against the Airbyte CI test suite
+```bash
+pipx install airbyte-ci/connectors/pipelines/
+airbyte-ci connectors --name= --use-remote-secrets=false test
+```
 
-    This is useful to provide in your PR as evidence of the acceptance tests passing locally.
+### (Debugging) Option 3: Run against the acceptance tests on your branch
 
-- `./acceptance-test-docker.sh` - to run tests from a docker container
+This will run the acceptance test suite directly with pytest. Allowing you to set breakpoints and debug your connector locally.
 
-If the test fails you will see detail about the test and where to find its inputs and outputs to reproduce it. You can also debug failed tests by adding `—pdb —last-failed`:
+The only pre-requisite is that you have [Poetry](https://python-poetry.org/docs/#installation) installed.
 
-```text
-python -m pytest -p integration_tests.acceptance --pdb --last-failed
+Afterwards you do the following from the root of the `airbyte` repo:
+```bash
+cd airbyte-integrations/bases/connector-acceptance-test/
+poetry install
+poetry run pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/ --pdb
 ```
 
 See other useful pytest options [here](https://docs.pytest.org/en/stable/usage.html)
+See a more comprehensive guide in our README [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/connector-acceptance-test/README.md)
 
 ## Dynamically managing inputs & resources used in standard tests
 
@@ -370,6 +381,36 @@ acceptance_tests:
 We cache discovered catalogs by default for performance and reuse the same discovered catalog through all tests.
 You can disable this behavior by setting `cached_discovered_catalog: False` at the root of the configuration.
 
+## Breaking Changes and Backwards Compatibility
+
+Breaking changes are modifications that make previous versions of the connector incompatible, requiring a major version bump. Here are the various types of changes that we consider breaking:
+
+1. **Changes to Stream Schema**
+    - **Removing a Field**: If a field is removed from the stream's schema, it's a breaking change. Clients expecting the field may fail when it's absent.
+    - **Changing Field Type**: If the data type of a field is changed, it could break clients expecting the original type. For instance, changing a field from string to integer would be a breaking change.
+    - **Renaming a Field**: If a field is renamed, it can break existing clients that expect the field by its original name.
+
+2. **Changes to Stream Behaviour**
+    - **Changing the Cursor**: Changing the cursor field for incremental streams can cause data discrepancies or synchronization issues. Therefore, it's considered a breaking change.
+    - **Renaming a Stream**: If a stream is renamed, it could cause failures for clients expecting the stream with its original name. Hence, this is a breaking change.
+    - **Changing Sync Mechanism**: If a stream's sync mechanism changes, such as switching from full refresh sync to incremental sync (or vice versa), it's a breaking change. Existing workflows may fail or behave unexpectedly due to this change.
+
+3. **Changes to Configuration Options**
+    - **Removing or Renaming Options**: If configuration options are removed or renamed, it could break clients using those options, hence, is considered a breaking change.
+    - **Changing Default Values or Behaviours**: Altering default values or behaviours of configuration options can break existing clients that rely on previous defaults.
+
+4. **Changes to Authentication Mechanism**
+    - Any change to the connector's authentication mechanism that isn't backwards compatible is a breaking change. For example, switching from API key authentication to OAuth without supporting both is a breaking change.
+
+5. **Changes to Error Handling**
+    - Altering the way errors are handled can be a breaking change. For example, if a certain type of error was previously ignored and now causes the connector to fail, it could break user's existing workflows.
+
+6. **Changes That Require User Intervention**
+    - If a change requires user intervention, such as manually updating settings or reconfiguring workflows, it would be considered a breaking change.
+
+Please note that this is an exhaustive but not an exclusive list. Other changes could be considered breaking if they disrupt the functionality of the connector or alter user expectations in a significant way.
+
+
 ## Additional Checks
 
 While not necessarily related to Connector Acceptance Testing, Airbyte employs a number of additional checks which run on connector Pull Requests which check the following items:
diff --git a/docs/connector-development/tutorials/building-a-python-destination.md b/docs/connector-development/tutorials/building-a-python-destination.md
index 4e744e4835e0..73c9325a9e8a 100644
--- a/docs/connector-development/tutorials/building-a-python-destination.md
+++ b/docs/connector-development/tutorials/building-a-python-destination.md
@@ -1,5 +1,11 @@
 # Building a Python Destination
 
+:::warning
+Airbyte has a Python Destination CDK that allows you to create quick and simple destinations for particular use cases. 
+Currently, the project is not accepting new Python Destinations in the official catalog.
+However, you can still build the connector and use it for your projects locally or import it into Airbyte Cloud.
+:::
+
 ## Summary
 
 This article provides a checklist for how to create a Python destination. Each step in the checklist has a link to a more detailed explanation below.
diff --git a/docs/connector-development/tutorials/building-a-python-source.md b/docs/connector-development/tutorials/building-a-python-source.md
index ae15a8b986a8..dc86631782be 100644
--- a/docs/connector-development/tutorials/building-a-python-source.md
+++ b/docs/connector-development/tutorials/building-a-python-source.md
@@ -230,7 +230,17 @@ Run integration tests using `python -m pytest -s integration_tests`.
 The template fills in most of the information for the readme for you. Unless there is a special case, the only piece of information you need to add is how one can get the credentials required to run the source. e.g. Where one can find the relevant API key, etc.
 
 ### Step 11: Add the connector to the API/UI
-# TODO ben link to metadata doc and create generator
+There are multiple ways to use the connector you have built.
+
+If you are self hosting Airbyte (OSS) you are able to use the Custom Connector feature. This feature allows you to run any Docker container that implements the Airbye protocol. You can read more about it [here](https://docs.airbyte.com/integrations/custom-connectors/).
+
+If you are using Airbyte Cloud (or OSS), you can submit a PR to add your connector to the Airbyte repository. Once the PR is merged, the connector will be available to all Airbyte Cloud users. You can read more about it [here](https://docs.airbyte.com/contributing-to-airbyte/submit-new-connector).
+
+Note that when submitting an Airbyte connector, you will need to ensure that
+1. The connector passes the standard test suite. See [Set up Standard Tests](#step-8-set-up-standard-tests).
+2. The metadata.yaml file (created by our generator) is filed out and valid. See [Connector Metadata File](https://docs.airbyte.com/connector-development/connector-metadata-file).
+3. You have created appropriate documentation for the connector. See [Add docs](#step-12-add-docs).
+
 
 ### Step 12: Add docs
 
diff --git a/docs/contributing-to-airbyte/README.md b/docs/contributing-to-airbyte/README.md
index dd6fd5d46940..b8dda6f8b28c 100644
--- a/docs/contributing-to-airbyte/README.md
+++ b/docs/contributing-to-airbyte/README.md
@@ -4,193 +4,74 @@ description: 'We love contributions to Airbyte, big or small.'
 
 # Contributing to Airbyte
 
-Thank you for your interest in contributing! We love community contributions. Contribution guidelines are listed below. If you're unsure about how to start contributing or have any questions even after reading them, feel free to ask us on [Slack](https://slack.airbyte.io) in the \#help-contributions.
-
-However, for those who want a bit more guidance on the best way to contribute to Airbyte, read on. This document will cover what we're looking for. By addressing the points below, the chances that we can quickly merge or address your contributions will increase.
-
-## Code of conduct
-
-Please follow our [Code of conduct](code-of-conduct.md) in the context of any contributions made to Airbyte.
-
-## Airbyte specification
-
-Before you can start contributing, you need to understand [Airbyte's data protocol specification](../understanding-airbyte/airbyte-protocol.md).
-
-## First-time contributors, welcome!
-
+Thank you for your interest in contributing! We love community contributions. 
+Read on to learn how to contribute to Airbyte.
 We appreciate first time contributors and we are happy to assist you in getting started. In case of questions, just reach out to us via [email](mailto:hey@airbyte.io) or [Slack](https://slack.airbyte.io)!
 
-Here is a list of easy [good first issues](https://github.com/airbytehq/airbyte/labels/good%20first%20issue) to do.
-
-## Contributing to the codebase
-
-We gladly welcome all improvements to the codebase.
-
-### Steps to contributing code
-
-#### 1. Open an issue, or find a similar one.
-Before jumping into the code please first:
-1. Verify if an existing [connector](https://github.com/airbytehq/airbyte/issues) or [platform](https://github.com/airbytehq/airbyte-platform/issues) GitHub issue matches your contribution project.
-2. If you don't find an existing issue, create a new [connector](https://github.com/airbytehq/airbyte/issues/new/choose) or [platform](https://github.com/airbytehq/airbyte-platform/issues/new/choose) issue to explain what you want to achieve.
-3. Assign the issue to yourself and add a comment to tell that you want to work on this.
-
-This will enable our team to make sure your contribution does not overlap with existing works and will comply with the design orientation we are currently heading the product toward.
-If you do not receive an update on the issue from our team, please ping us on [Slack](https://slack.airbyte.io)!
+Before getting started, please review Airbyte's Code of Conduct. Everyone interacting in Slack, codebases, mailing lists, events, or other Airbyte activities is expected to follow [Code of Conduct](../project-overview/code-of-conduct.md).
 
-#### 2. Code your contribution
-1. To contribute to a connector, fork the [Connector repository](https://github.com/airbytehq/airbyte). To contribute to the Airbyte platform, fork our [Platform repository](https://github.com/airbytehq/airbyte-platform).
-2. If contributing a new connector, check out our [new connectors guide](#new-connectors).
-3. Open a branch for your work.
-4. Code, and please write **tests**.
-5. Ensure all tests pass. For connectors, this includes acceptance tests as well.
-6. For connectors, make sure to increment the connector's version according to our [Semantic Versioning](#semantic-versioning-for-connectors) guidelines.
+## Code Contributions
 
-#### 3. Open a pull request
-1. Rebase master with your branch before submitting a pull request.
-2. Open the pull request.
-3. Wait for a review from a community maintainer or our team.
+Most of the issues that are open for contributions are tagged with `good first issue` or `help-welcome`. 
+A great place to start looking will be our GitHub projects for:
 
-#### 4. Review process
-When we review, we look at:
-* ‌Does the PR solve the issue?
-* Is the proposed solution reasonable?
-* Is it tested? \(unit tests or integration tests\)
-* Is it introducing security risks?
-‌Once your PR passes, we will merge it 🎉.
+[**Community Connector Issues Project**](https://github.com/orgs/airbytehq/projects/50)
 
-### New connectors
+Due to project priorities, we may not be able to accept all contributions at this time. 
+We are prioritizing the following contributions: 
+* Bug fixes, features, and enhancements to existing API source connectors
+* New connector sources built with the Low-Code CDK and Connector Builder, as these connectors are easier to maintain.
+* Bug fixes, features, and enhancements to the following database sources: MongoDB, Postgres, MySQL, MSSQL
+* Bug fixes to the following destinations: BigQuery, Snowflake, Redshift, S3, and Postgres
+* Helm Charts features, bug fixes, and other platform bug fixes
 
-It's easy to add your own connector to Airbyte! **Since Airbyte connectors are encapsulated within Docker containers, you can use any language you like.** Here are some links on how to add sources and destinations. We haven't built the documentation for all languages yet, so don't hesitate to reach out to us if you'd like help developing connectors in other languages.
 
-For sources, simply head over to our [Python CDK](../connector-development/cdk-python/).
-
-:::info
-The CDK currently does not support creating destinations, but it will very soon.
+:::warning
+Contributions outside of these will be evaluated on a case-by-case basis by our engineering team.
 :::
 
-* See [Building new connectors](../connector-development/) to get started.
-* Since we frequently build connectors in Python, on top of Singer or in Java, we've created generator libraries to get you started quickly: [Build Python Source Connectors](../connector-development/tutorials/building-a-python-source.md) and [Build Java Destination Connectors](../connector-development/tutorials/building-a-java-destination.md)
-* Integration tests \(tests that run a connector's image against an external resource\) can be run one of three ways, as detailed [here](../connector-development/testing-connectors/connector-acceptance-tests-reference.md)
-
-**Please note that, at no point in time, we will ask you to maintain your connector.** The goal is that the Airbyte team and the community helps maintain the connector.
-
-### Semantic versioning for connectors
-
-Changes to connector behavior should always be accompanied by a version bump and a changelog entry. We use [semantic versioning](https://semver.org/) to version changes to connectors. Since connectors are a bit different from APIs, we have our own take on semantic versioning, focusing on maintaining the best user experience of using a connector.
-
-- Major: a version in which a change is made which requires manual intervention (update to config or configured catalog) for an existing connection to continue to succeed, or one in which data that was previously being synced will no longer be synced
-- Minor: a version that introduces user-facing functionality in a backwards compatible manner
-- Patch: a version that introduces backwards compatible bug fixes or performance improvements
-
-#### Examples
-
-Here are some examples of code changes and their respective version changes:
-
-| Change                                                                                        | Impact                                                                                                           | Version Change |
-|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|----------------|
-| Adding a required parameter to a connector's `spec`                                           | Users will have to add the new parameter to their `config`                                                       | Major          |
-| Changing a format of a parameter in a connector's `spec` from a single parameter to a `oneOf` | Users will have to edit their `config` to define their old parameter value in the `oneOf` format                 | Major          |
-| Removing a stream from a connector's `catalog`                                                | Data that was being synced will no longer be synced                                                              | Major          |
-| Renaming a stream in a connector's `catalog`                                                  | Users will have to update the name of the stream in their `catalog`                                              | Major          |
-| Removing a column from a stream in a connector's `catalog`                                    | Users will have to remove that column from their `catalog`, data that was being synced will no longer be synced  | Major          |
-| Renaming a column from a stream in a connector's `catalog`                                    | Users will have to update the name of the column in their `catalog`                                              | Major          |
-| Changing the datatype for a column of a stream in a connector's `catalog`                     | Users will have to update that data type in their `catalog`, data that was being synced will have changed format | Major          |
-| Adding a non-required parameter to a connector's `spec`                                       | Users will have the option to use the required parameter in the future                                           | Minor          |
-| Adding a stream in a connector's `catalog`                                                    | Additional data will be synced                                                                                   | Minor          |
-| Adding a column to a stream's schema in a connector's `catalog`                               | Additional data will be synced                                                                                   | Minor          |
-| Updating the format of the connector's `STATE`                                                | Incremental streams will automatically run a full refresh only for the next sync                                 | Patch          |
-| Optimizing a connector's performance                                                          | Syncs will be faster                                                                                             | Patch          |
-| Fixing a bug in a connector                                                                   | Some syncs that would have failed will now succeed                                                               | Patch          |
-
-Trying to contribute, and don't see the change you want to make in this list? Call it out in your PR and your reviewer will help you pick the correct type of version change. Feel free to contribute the results back to this list!
-
-
-### Airbyte CI workflows
-* [Testing by SonarQube](sonar-qube-workflow.md)
+The usual workflow of code contribution is:
+1. Fork the Airbyte repository
+2. Clone the repository locally
+3. Make changes and commit them
+4. Push your local branch to your fork
+5. Submit a Pull Request so that we can review your changes
+6. [Link an existing Issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) without `needs triage` label to your Pull Request (PR without this will be closed)
+7. Write a commit message
+8. An Airbyte maintainer will trigger the CI tests for you and review the code
+9. Update the comments and review
+10. Merge the contribution
 
-## Breaking Changes to Connectors
+Pull Request reviews are done on a regular basis. 
 
-Often times, changes to connectors can be made without impacting the user experience.  However, there are some changes that will require users to take action before they can continue to sync data.  These changes are considered **Breaking Changes** and require a
-
-1. A **Major Version** increase 
-2. An Airbyte Engineer to follow the  [Connector Breaking Change Release Playbook](https://docs.google.com/document/u/0/d/1VYQggHbL_PN0dDDu7rCyzBLGRtX-R3cpwXaY8QxEgzw/edit) before merging.
-
-
-### Types of Breaking Change(s):
-A breaking change is any change that will require users to take action before they can continue to sync data. The following are examples of breaking changes:
-
-- **Spec Change** - The configuration required by users of this connector have been changed and syncs will fail until users reconfigure or re-authenticate.  This change is not possible via a Config Migration 
-- **Schema Change** - The type of a property previously present within a record has changed
-- **Stream or Property Removal** - Data that was previously being synced is no longer going to be synced.
-- **Destination Format / Normalization Change** - The way the destination writes the final data or how normalization cleans that data is changing in a way that requires a full-refresh.**
-- **State Changes** - The format of the source’s state has changed, and the full dataset will need to be re-synced
-
-
-### Checklist for Contributors
-
-First, ask yourself does your change correspond to any of the breaking changes above?
-
-If so then follow this checklist below
-
-- [ ] Apply the label breaking-change to your PR
-- [ ] Apply a Major Version bump to your PR. See [Semantic Versioning for Connectors](#semantic-versioning-for-connectors) for more information.
-- [ ] Prepend your PR title with the 🚨🚨 emoji.
-- [ ] Add a section to the PR description titled Breaking Change that describes why this is a breaking change, and if possible how you can migrate users and/or rollback
-- [ ] Assign an Airbyte Engineer through the `airbytehq/connector-operations` group and have them start the [Connector Breaking Change Playbook](https://docs.google.com/document/d/1VYQggHbL_PN0dDDu7rCyzBLGRtX-R3cpwXaY8QxEgzw/edit#)
+:::info
+Please make sure you respond to our feedback/questions and sign our CLA.
 
-## Contributing to documentation
+Pull Requests without updates will be closed due inactivity.
+:::
 
-Our goal is to keep our docs comprehensive and updated. If you would like to help us in doing so, we are grateful for any kind of contribution:
+Guidelines to common code contributions:
+- [Submit code change to existing Source Connector](change-cdk-connector.md)
+- [Submit a New Connector](submit-new-connector.md)
 
-* Report missing content
-* Fix errors in existing docs
-* Help us in adding to the docs
+## Documentation
 
-The contributing guide for docs can be found [here](contribute-documentation.md).
+We welcome Pull Requests that enhance the grammar, structure, or fix typos in our documentation.
 
-## Contributing community content
+- Check the [guidelines](writing-docs.md) to submit documentation changes
 
-We welcome contributions as new tutorials / showcases / articles, or to any of the existing guides on our [tutorials page](https://airbyte.com/tutorials):
+## Community Content
 
-* Fix errors in existing tutorials
-* Add new tutorials \(please reach out to us if you have ideas to avoid duplicate work\)
-* Request tutorials
+We welcome contributions as new tutorials / showcases / articles, or to any of the existing guides on our tutorials page.
 
 We have a repo dedicated to community content. Everything is documented [there](https://github.com/airbytehq/community-content/).
 
 Feel free to submit a pull request in this repo, if you have something to add even if it's not related to anything mentioned above.
 
-## Other ways to contribute
-
-### Upvoting issues, feature and connector requests
-
-You are welcome to add your own reactions to the existing issues. We will take them in consideration in our prioritization efforts, especially for connectors.
-
-❤️ means that this task is CRITICAL to you.
-👍 means it is important to you.
-
-### Requesting new features
-
-To request new features, please create an issue on this project.
-
-If you would like to suggest a new feature, we ask that you please use our issue template. It contains a few essential questions that help us understand the problem you are looking to solve and how you think your recommendation will address it. We also tag incoming issues from this template with the "**community\_new**" label. This lets our teams quickly see what has been raised and better address the community recommendations.
-
-To see what has already been proposed by the community, you can look [here](https://github.com/airbytehq/airbyte/labels/community).
-
-Watch out for duplicates! If you are creating a new platform issue, please check [open](https://github.com/airbytehq/airbyte-platform/issues), or [recently closed](https://github.com/airbytehq/airbyte-platform/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20).
-
-### Requesting new connectors
-
-This is very similar to requesting new features. The template will change a bit and all connector requests will be tagged with the “**community**” and “**area/connectors**” labels.
-
-To see what has already been proposed by the community, you can look [here](https://github.com/airbytehq/airbyte/labels/area%2Fconnectors). Again, watch out for duplicates!
-
-### Reporting bugs
-
-**‌**Bug reports help us make Airbyte better for everyone. We provide a preconfigured template for bugs to make it very clear what information we need.
+## Engage with the Community
 
-‌Please search within our [already reported bugs](https://github.com/airbytehq/airbyte/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fbug) before raising a new one to make sure you're not raising a duplicate.
+Another crucial way to contribute is by reporting bugs and helping other users in the community.
 
-### Reporting security issues
+You're welcome to enter the Community Slack and help other users or report bugs in Github.
 
-Please do not create a public GitHub issue. If you've found a security issue, please email us directly at [security@airbyte.io](mailto:security@airbyte.io) instead of raising an issue.
+- How to report a bug [guideline](issues-and-requests.md)
diff --git a/docs/contributing-to-airbyte/change-cdk-connector.md b/docs/contributing-to-airbyte/change-cdk-connector.md
new file mode 100644
index 000000000000..f4becce2492f
--- /dev/null
+++ b/docs/contributing-to-airbyte/change-cdk-connector.md
@@ -0,0 +1,72 @@
+# Changes to CDK or Low-Code Connector
+
+## Contribution Process
+
+### Open an issue, or find a similar one.
+Before jumping into the code please first:
+1. Check if the improvement you want to make or bug you want to fix is already captured in an [existing issue](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Fconnectors+-label%3Aneeds-triage+label%3Acommunity) 
+2. If you don't find an existing issue, either
+  - [Report a Connector Bug](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fbug%2Carea%2Fconnectors%2Cneeds-triage&projects=&template=1-issue-connector.yaml), or
+  - [Request a New Connector Feature](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement%2Cneeds-triage&projects=&template=6-feature-request.yaml)
+
+This will enable our team to make sure your contribution does not overlap with existing works and will comply with the design orientation we are currently heading the product toward. If you do not receive an update on the issue from our team, please ping us on [Slack](https://slack.airbyte.io)!
+
+:::info
+Make sure you're working on an issue had been already triaged to not have your contribution declined.
+:::
+
+### Code your contribution
+1. To contribute to a connector, fork the [Connector repository](https://github.com/airbytehq/airbyte). 
+2. Open a branch for your work
+3. Code the change
+4. Write a unit test for each custom function you added or changed
+5. Ensure all tests, including connector acceptance tests, pass
+6. Update the `metadata.yaml` and `Dockerfile` version following the [guidelines](./resources/pull-requests-handbook.md#semantic-versioning-for-connectors)
+7. Update the changelog entry in documentation in `docs/integrations/.md`
+
+A comment will automatically be added to your PR with a checklist containing the necessary steps to complete your contribution and get it merged.
+
+:::info
+There is a README file inside each connector folder containing instructions to run that connector's tests locally.
+:::
+
+:::warning
+Pay attention to breaking changes to connectors. You can read more [here](#breaking-changes-to-connectors).
+:::
+
+
+### Open a pull request
+1. Rebase master with your branch before submitting a pull request.
+2. Open the pull request.
+3. Follow the [title convention](./resources/pull-requests-handbook.md#pull-request-title-convention) for Pull Requests
+4. Link to an existing Issue
+5. Update the [description](./resources/pull-requests-handbook.md#descriptions)
+6. Wait for a review from a community maintainer or our team.
+
+### Review process
+When we review, we look at:
+* ‌Does the PR solve the issue?
+* Is the proposed solution reasonable?
+* Is it tested? \(unit tests or integration tests\)
+* Is it introducing security risks?
+* Is it introducing a breaking change?
+‌Once your PR passes, we will merge it 🎉.
+
+
+## Breaking Changes to Connectors
+
+Often times, changes to connectors can be made without impacting the user experience.  However, there are some changes that will require users to take action before they can continue to sync data.  These changes are considered **Breaking Changes** and require a
+
+1. A **Major Version** increase 
+2. A [`breakingChanges` entry](https://docs.airbyte.com/connector-development/connector-metadata-file/) in the `releases` section of the `metadata.yaml` file
+3. A migration guide which details steps that users should take to resolve the change
+4. An Airbyte Engineer to follow the  [Connector Breaking Change Release Playbook](https://docs.google.com/document/u/0/d/1VYQggHbL_PN0dDDu7rCyzBLGRtX-R3cpwXaY8QxEgzw/edit) before merging.
+
+### Types of Breaking Changes
+A breaking change is any change that will require users to take action before they can continue to sync data. The following are examples of breaking changes:
+
+- **Spec Change** - The configuration required by users of this connector have been changed and syncs will fail until users reconfigure or re-authenticate.  This change is not possible via a Config Migration 
+- **Schema Change** - The type of a property previously present within a record has changed
+- **Stream or Property Removal** - Data that was previously being synced is no longer going to be synced.
+- **Destination Format / Normalization Change** - The way the destination writes the final data or how normalization cleans that data is changing in a way that requires a full-refresh.
+- **State Changes** - The format of the source’s state has changed, and the full dataset will need to be re-synced
\ No newline at end of file
diff --git a/docs/contributing-to-airbyte/contribute-documentation.md b/docs/contributing-to-airbyte/contribute-documentation.md
deleted file mode 100644
index 1ec084ba21c2..000000000000
--- a/docs/contributing-to-airbyte/contribute-documentation.md
+++ /dev/null
@@ -1,151 +0,0 @@
-# Updating Documentation
-
-We welcome contributions to the Airbyte documentation! 
-
-Our docs are written in [Markdown](https://guides.github.com/features/mastering-markdown/) following the [Google developer documentation style guide](https://developers.google.com/style/highlights) and the files are stored in our [Github repository](https://github.com/airbytehq/airbyte/tree/master/docs). The docs are published at [docs.airbyte.com](https://docs.airbyte.com/) using [Docusaurus](https://docusaurus.io/) and [GitHub Pages](https://pages.github.com/). 
-
-## Finding good first issues
-
-The Docs team maintains a list of [#good-first-issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Fdocumentation+label%3A%22good+first+issue%22) for new contributors. 
-
-- If you're new to technical writing, start with the smaller issues (fixing typos, broken links, spelling and grammar, and so on). You can [edit the files directly on GitHub](#editing-directly-on-github).
-- If you're an experienced technical writer or a developer interested in technical writing, comment on an issue that interests you to discuss it with the Docs team. Once we decide on the approach and the tasks involved, [edit the files and open a Pull Request](#editing-on-your-local-machine) for the Docs team to review.
-
-## Contributing to Airbyte docs
-
-Before contributing to Airbyte docs, read the Airbyte Community [Code of Conduct](code-of-conduct.md)
-
-:::tip
-If you're new to GitHub and Markdown, complete [the First Contributions tutorial](https://github.com/firstcontributions/first-contributions) and learn [Markdown basics](https://guides.github.com/features/mastering-markdown/) before contributing to Airbyte documentation. 
-:::
-
-You can contribute to Airbyte docs in two ways:
-
-### Editing directly on GitHub
-
-To make minor changes (example: fixing typos) or edit a single file, you can edit the file directly on GitHub:
-
-1. Click **Edit this page** at the bottom of any published document on [docs.airbyte.com](https://docs.airbyte.com/). You'll be taken to the GitHub editor. 
-2. [Edit the file directly on GitHub and open a Pull Request](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files).
-
-### Editing on your local machine
-
-To make complex changes or edit multiple files, edit the files on your local machine:
-
-1. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the Airbyte [repository](https://github.com/airbytehq/airbyte).
-2. Clone the fork on your local machine:
-
-   ```bash
-   git clone git@github.com:{YOUR_USERNAME}/airbyte.git
-   cd airbyte
-   ```
-
-   Or
-
-   ```bash
-   git clone https://github.com/{YOUR_USERNAME}/airbyte.git
-   cd airbyte
-   ```
-
-   While cloning on Windows, you might encounter errors about long filenames. Refer to the instructions [here](../deploying-airbyte/local-deployment.md#handling-long-filename-error) to correct it.
-
-3. Test changes locally:
-
-   To install the docs locally, run the following commands in your terminal:
-
-   ```bash
-   cd docusaurus
-   yarn install
-   ```
-
-   To see changes as you make them, run:
-
-   ```bash
-   yarn start
-   ```
-
-   Then navigate to [http://localhost:3000/](http://localhost:3000/). Whenever you make and save changes, you will see them reflected in the server. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal.  
-
-   You can also build the docs locally and see the resulting changes. This is useful if you introduce changes that need to be run at build-time (e.g. adding a docs plug-in). To do so, run:
-
-   ```bash
-   yarn build
-   yarn serve
-   ```
-  
-   Then navigate to [http://localhost:3000/](http://localhost:3000/) to see your changes. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal.  
-
-
-4. [Follow the GitHub workflow](https://docs.github.com/en/get-started/quickstart/contributing-to-projects/) to edit the files and create a pull request.
-
-    :::note
-    Before we accept any contributions, you'll need to sign the Contributor License Agreement (CLA). By signing a CLA, we can ensure that the community is free and confident in its ability to use your contributions. You will be prompted to sign the CLA while opening a pull request.
-    :::
-
-5. Assign `airbytehq/docs` as a Reviewer for your pull request. 
-
-## Additional guidelines
-
-- If you're updating a connector doc, follow the [Connector documentation template](https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw)
-- If you're adding a new file, update the [sidebars.js file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js)
-- If you're adding a README to a code module, make sure the README has the following components:
-    - A brief description of the module
-    - Development pre-requisites (like which language or binaries are required for development)
-    - How to install dependencies
-    - How to build and run the code locally & via Docker
-    - Any other information needed for local iteration
-
-## Advanced tasks 
-
-### Adding a redirect
-
-To add a redirect, open the [`docusaurus.config.js`](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) file and locate the following commented section:
-
-```js
-//                        {
-//                         from: '/some-lame-path',
-//                         to: '/a-much-cooler-uri',
-//                        },
-```
-
-Copy this section, replace the values, and [test the changes locally](#editing-on-your-local-machine) by going to the path you created a redirect for and verify that the address changes to the new one.
-
-:::note 
-Your path **needs** a leading slash `/` to work
-:::
-
-### Deploying and reverting the documentation site
-
-:::note
-Only the Airbyte team and maintainers have permissions to deploy the documentation site.
-:::
-
-#### Automated documentation site deployment
-
-When `docs/` folder gets changed in `master` branch of the repository, [`Deploy docs.airbyte.com` Github workflow](https://github.com/airbytehq/airbyte/actions/workflows/deploy-docs-site.yml) steps in, builds and deploys the documentation site.  This process is automatic, takes five to ten minutes, and needs no human intervention.
-
-#### Manual documentation site deployment
-
-:::note
-Manual deployment is reserved for emergency cases.  Please, bear in mind that automatic deployment is triggered by changes to `docs/` folder, so it needs to be disabled to avoid interference with manual deployment.
-:::
-
-You'll need a GitHub SSH key to deploy the documentation site using the [deployment tool](https://github.com/airbytehq/airbyte/blob/master/tools/bin/deploy_docusaurus). 
-
-To deploy the documentation site, run:
-
-```bash
-cd airbyte
-# or cd airbyte-cloud  
-git checkout master
-git pull
-./tools/bin/deploy_docusaurus
-```
-
-To revert/rollback doc changes, run:
-
-```
-cd airbyte
-git checkout 
-./tools/bin/deploy_docusaurus
-```
diff --git a/docs/contributing-to-airbyte/issues-and-pull-requests.md b/docs/contributing-to-airbyte/issues-and-pull-requests.md
deleted file mode 100644
index 22f953828909..000000000000
--- a/docs/contributing-to-airbyte/issues-and-pull-requests.md
+++ /dev/null
@@ -1,51 +0,0 @@
-# Issues & Pull Requests
-
-## Titles
-
-**Describe outputs, not implementation**: An issue or PR title should describe the desired end result, not the implementation. The exception is child issues/subissues of an epic. **Be specific about the domain**. Airbyte operates a monorepo, so being specific about what is being changed in the PR or issue title is important.
-
-Some examples: _subpar issue title_: `Remove airbyteCdk.dependsOn("unrelatedPackage")`. This describes a solution not a problem.
-
-_good issue title_: `Building the Airbyte Python CDK should not build unrelated packages`. Describes desired end state and the intent is understandable without reading the full issue.
-
-_subpar PR title_: `Update tests`. Which tests? What was the update?
-
-_good PR title_: `Source MySQL: update acceptance tests to connect to SSL-enabled database`. Specific about the domain and change that was made.
-
-### Pull Request Title Convention
-
-When creating a pull request follow the naming conventions depending on the change being made.
-In general the pull request title starts with an emoji with the connector you're doing the changes, eg (✨ Source E-Commerce: add new stream `Users`).
-Airbyte uses this pattern to automatically assign team reviews and build the product release notes.
-
-| Pull Request Type | Emoji | Examples |
-| ----------------- | ----- | ---------|
-| New Connector (Source or Destination)  | 🎉 | 🎉 New Destination: Database                           |
-| Add a feature to an existing connector | ✨ | ✨ Source E-Commerce: add new stream `Users`           |
-| Fix a bug                              | 🐛 | 🐛 Source E-Commerce: fix start date parameter in spec |
-| Documentation (updates or new entries) | 📝 | 📝 Fix Database connector changelog                    |
-| It's a breaking change                 | 🚨 | 🚨🚨🐛 Source Kafka: fix a complex bug                  |
-
-For more information about [breaking changes](README.md#breaking-changes-to-connectors). A maintainer will help and instruct about possible breaking changes.
-
-Any refactors, cleanups, etc.. that are not visible improvements to the user should not have emojis.
-
-If you're code change is doing more than one change type at once we strongly recommend to break into multiple pull requests. It helps us to review and merge your contribution.
-
-## Descriptions
-
-**Context**: Provide enough information \(or a link to enough information\) in the description so team members with no context can understand what the issue or PR is trying to accomplish. This usually means you should include two things:
-
-1. Some background information motivating the problem
-2. A description of the problem itself
-3. Good places to start reading and file changes that can be skipped
-
-   Some examples:
-
-_insufficient context_: `Create an OpenAPI to JSON schema generator`. Unclear what the value or problem being solved here is.
-
-_good context_:
-
-```text
-When creating or updating connectors, we spend a lot of time manually transcribing JSON Schema files based on OpenAPI docs. This is ncessary because OpenAPI and JSON schema are very similar but not perfectly compatible. This process is automatable. Therefore we should create a program which converts from OpenAPI to JSONSchema format.
-```
\ No newline at end of file
diff --git a/docs/contributing-to-airbyte/issues-and-requests.md b/docs/contributing-to-airbyte/issues-and-requests.md
new file mode 100644
index 000000000000..3ad347bdb68c
--- /dev/null
+++ b/docs/contributing-to-airbyte/issues-and-requests.md
@@ -0,0 +1,23 @@
+# Issues and Requests
+
+## Report a Bug
+Bug reports help us make Airbyte better for everyone. We provide a preconfigured template for bugs to make it very clear what information we need.
+
+‌Please search within our [already reported bugs](https://github.com/airbytehq/airbyte/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fbug) before raising a new one to make sure you're not raising a duplicate.
+
+
+## Request new Features or Connector
+
+Requesting new features or connectors is an essential way to contribute. Your input helps us understand your needs and priorities, enabling us to enhance the functionality and versatility of Airbyte.
+
+## Reporting Security Issues
+
+Please do not create a public GitHub issue. If you've found a security issue, please email us directly at [security@airbyte.io](mailto:security@airbyte.io) instead of raising an issue.
+
+## Upvoting Issues, Features or Connector Requests
+
+You are welcome to add your own reactions to the existing issues. We will take them in consideration in our prioritization efforts, especially for connectors.
+
+❤️ means that this task is CRITICAL to you.
+
+👍 means it is important to you.
diff --git a/docs/contributing-to-airbyte/code-style.md b/docs/contributing-to-airbyte/resources/code-style.md
similarity index 96%
rename from docs/contributing-to-airbyte/code-style.md
rename to docs/contributing-to-airbyte/resources/code-style.md
index 9de6a447ac04..40a14c163200 100644
--- a/docs/contributing-to-airbyte/code-style.md
+++ b/docs/contributing-to-airbyte/resources/code-style.md
@@ -31,7 +31,7 @@ Install it in IntelliJ:
    2. Turn on the auto add final. Go into IntelliJ Preferences
       1. Plugins - install Save Actions if not already installed
          1. If you're running Intellij 2023.1 or higher, the official version may not work. Try manually installing [this fork](https://github.com/fishermans/intellij-plugin-save-actions/releases/tag/v2.6.0) (see [Github issue](https://github.com/dubreuia/intellij-plugin-save-actions/issues/427)).
-      2. Go to Save Actions in the preferences [left navigation column](../assets/docs/save_actions_settings.png) (NOT Tools > Actions on Save -- that is a different tool)
+      2. Go to Save Actions in the preferences [left navigation column](../../assets/docs/save_actions_settings.png) (NOT Tools > Actions on Save -- that is a different tool)
          1. `Activate save actions on save` > check the box
          2. `Active save actions on shortcut` > check the box
          3. `Activate save actions on batch` > check the box
diff --git a/docs/contributing-to-airbyte/developing-locally.md b/docs/contributing-to-airbyte/resources/developing-locally.md
similarity index 100%
rename from docs/contributing-to-airbyte/developing-locally.md
rename to docs/contributing-to-airbyte/resources/developing-locally.md
diff --git a/docs/contributing-to-airbyte/developing-on-docker.md b/docs/contributing-to-airbyte/resources/developing-on-docker.md
similarity index 100%
rename from docs/contributing-to-airbyte/developing-on-docker.md
rename to docs/contributing-to-airbyte/resources/developing-on-docker.md
diff --git a/docs/contributing-to-airbyte/gradle.md b/docs/contributing-to-airbyte/resources/gradle.md
similarity index 100%
rename from docs/contributing-to-airbyte/gradle.md
rename to docs/contributing-to-airbyte/resources/gradle.md
diff --git a/docs/contributing-to-airbyte/resources/pull-requests-handbook.md b/docs/contributing-to-airbyte/resources/pull-requests-handbook.md
new file mode 100644
index 000000000000..b4517652e67e
--- /dev/null
+++ b/docs/contributing-to-airbyte/resources/pull-requests-handbook.md
@@ -0,0 +1,69 @@
+# Pull Request Handbook
+
+### Pull Request Title Convention
+
+When creating a pull request follow the naming conventions depending on the change being made.
+In general the pull request title starts with an emoji with the connector you're doing the changes, eg (✨ Source E-Commerce: add new stream `Users`).
+Airbyte uses this pattern to automatically assign team reviews and build the product release notes.
+
+| Pull Request Type | Emoji | Examples |
+| ----------------- | ----- | ---------|
+| New Connector (Source or Destination)  | 🎉 | 🎉 New Destination: Database                           |
+| Add a feature to an existing connector | ✨ | ✨ Source E-Commerce: add new stream `Users`           |
+| Fix a bug                              | 🐛 | 🐛 Source E-Commerce: fix start date parameter in spec |
+| Documentation (updates or new entries) | 📝 | 📝 Fix Database connector changelog                    |
+| It's a breaking change                 | 🚨 | 🚨🚨🐛 Source Kafka: fix a complex bug                  |
+
+For more information about [breaking changes](#breaking-changes-to-connectors). A maintainer will help and instruct about possible breaking changes.
+
+Any refactors, cleanups, etc.. that are not visible improvements to the user should not have emojis.
+
+If you're code change is doing more than one change type at once we strongly recommend to break into multiple pull requests. It helps us to review and merge your contribution.
+
+## Descriptions
+
+**Context**: Provide enough information \(or a link to enough information\) in the description so team members with no context can understand what the issue or PR is trying to accomplish. This usually means you should include two things:
+
+1. Some background information motivating the problem
+2. A description of the problem itself
+3. Good places to start reading and file changes that can be skipped
+
+   Some examples:
+
+_insufficient context_: `Create an OpenAPI to JSON schema generator`. Unclear what the value or problem being solved here is.
+
+_good context_:
+
+```text
+When creating or updating connectors, we spend a lot of time manually transcribing JSON Schema files based on OpenAPI docs. This is ncessary because OpenAPI and JSON schema are very similar but not perfectly compatible. This process is automatable. Therefore we should create a program which converts from OpenAPI to JSONSchema format.
+```
+
+## Semantic Versioning for Connectors
+
+Changes to connector behavior should always be accompanied by a version bump and a changelog entry. We use [semantic versioning](https://semver.org/) to version changes to connectors. Since connectors are a bit different from APIs, we have our own take on semantic versioning, focusing on maintaining the best user experience of using a connector.
+
+- Major: a version in which a change is made which requires manual intervention (update to config or configured catalog) for an existing connection to continue to succeed, or one in which data that was previously being synced will no longer be synced
+- Minor: a version that introduces user-facing functionality in a backwards compatible manner
+- Patch: a version that introduces backwards compatible bug fixes or performance improvements
+
+### Examples
+
+Here are some examples of code changes and their respective version changes:
+
+| Change                                                                                        | Impact                                                                                                           | Version Change |
+|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------|----------------|
+| Adding a required parameter to a connector's `spec`                                           | Users will have to add the new parameter to their `config`                                                       | Major          |
+| Changing a format of a parameter in a connector's `spec` from a single parameter to a `oneOf` | Users will have to edit their `config` to define their old parameter value in the `oneOf` format                 | Major          |
+| Removing a stream from a connector's `catalog`                                                | Data that was being synced will no longer be synced                                                              | Major          |
+| Renaming a stream in a connector's `catalog`                                                  | Users will have to update the name of the stream in their `catalog`                                              | Major          |
+| Removing a column from a stream in a connector's `catalog`                                    | Users will have to remove that column from their `catalog`, data that was being synced will no longer be synced  | Major          |
+| Renaming a column from a stream in a connector's `catalog`                                    | Users will have to update the name of the column in their `catalog`                                              | Major          |
+| Changing the datatype for a column of a stream in a connector's `catalog`                     | Users will have to update that data type in their `catalog`, data that was being synced will have changed format | Major          |
+| Adding a non-required parameter to a connector's `spec`                                       | Users will have the option to use the required parameter in the future                                           | Minor          |
+| Adding a stream in a connector's `catalog`                                                    | Additional data will be synced                                                                                   | Minor          |
+| Adding a column to a stream's schema in a connector's `catalog`                               | Additional data will be synced                                                                                   | Minor          |
+| Updating the format of the connector's `STATE`                                                | Incremental streams will automatically run a full refresh only for the next sync                                 | Patch          |
+| Optimizing a connector's performance                                                          | Syncs will be faster                                                                                             | Patch          |
+| Fixing a bug in a connector                                                                   | Some syncs that would have failed will now succeed                                                               | Patch          |
+
+Trying to contribute, and don't see the change you want to make in this list? Call it out in your PR and your reviewer will help you pick the correct type of version change. Feel free to contribute the results back to this list!
diff --git a/docs/contributing-to-airbyte/python-gradle-setup.md b/docs/contributing-to-airbyte/resources/python-gradle-setup.md
similarity index 99%
rename from docs/contributing-to-airbyte/python-gradle-setup.md
rename to docs/contributing-to-airbyte/resources/python-gradle-setup.md
index 0d5403755d10..37740a3765bb 100644
--- a/docs/contributing-to-airbyte/python-gradle-setup.md
+++ b/docs/contributing-to-airbyte/resources/python-gradle-setup.md
@@ -100,7 +100,6 @@ By default, the find function in IntelliJ is not scoped and will include all fil
 4. Add the following filter to the `Exclude files` option: `connectors/**/.venv`
 5. Press OK to confirm your options.
 
-![](../.gitbook/assets/monorepo-exclude-files.png)
 
 #### Manual Workaround
 
diff --git a/docs/contributing-to-airbyte/sonar-qube-workflow.md b/docs/contributing-to-airbyte/sonar-qube-workflow.md
deleted file mode 100644
index 5314655118ba..000000000000
--- a/docs/contributing-to-airbyte/sonar-qube-workflow.md
+++ /dev/null
@@ -1,34 +0,0 @@
-# SonarQube workflow
-
-## Goals
- The Airbyte monorepo receives contributions from a lot of developers, and there is no way around human errors while merging PRs.
-Likely every language has different tools for testing and validation of source files. And while it's best practice to lint and validate code before pushing to git branches, it doesn't always happen.
-But it is optional, and as rule as we detect possible problems after launch test/publish commands only. Therefore, using of automated CI code validation can  provided the following benefits:
-* Problem/vulnerability reports available when the PR was created. And developers would fix bugs and remove smells before code reviews.
-* Reviewers would be sure all standard checks were made and code changes satisfy the requirements.
-* Set of tools and their options can be changed anytime globally.
-* Progress of code changes are saved in SonarQube and this information helps to analyse quality of the product  integrally and also its separate parts.
-
-
-## UML diagram 
-![image](https://user-images.githubusercontent.com/11213273/149561440-0aceaa30-8f82-4e5b-9ee5-77bdcfd87695.png)
-
-
-## Used tools
-### Python
-* [flake8](https://flake8.pycqa.org/en/stable/)
-* [mypy](https://mypy.readthedocs.io/en/stable/)
-* [isort](https://pycqa.github.io/isort/)
-* [black](https://black.readthedocs.io/en/stable/)
-* [coverage](https://coverage.readthedocs.io/en/6.2/)
-
-All Python tools use the common [pyproject.toml](https://github.com/airbytehq/airbyte/blob/master/pyproject.toml) file.
-
-### Common tools
-* [SonarQube Scanner](https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/)
-
-## Access to SonarQube
-The Airbyte project uses a custom SonarQube instance. Access to it is explained [here](https://github.com/airbytehq/airbyte-cloud/wiki/IAP-tunnel-to-the-SonarQube-instance).
-
-## SonarQube settings
-The SonarQube server uses default settings. All customisations are implemented into the Github WorkFlows. More details are [here](https://github.com/airbytehq/airbyte/tree/master/.github/actions/ci-tests-runner/action.yml)
\ No newline at end of file
diff --git a/docs/contributing-to-airbyte/submit-new-connector.md b/docs/contributing-to-airbyte/submit-new-connector.md
new file mode 100644
index 000000000000..82cbb25fbf66
--- /dev/null
+++ b/docs/contributing-to-airbyte/submit-new-connector.md
@@ -0,0 +1,38 @@
+# Submit a New Connector
+
+:::info
+Due to project priorities, we may not be able to accept all contributions at this time. 
+
+:::
+
+#### Find an Issue or Create it!
+Before jumping into the code please first:
+1. Verify if there is an existing [Issue](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Fconnectors+-label%3Aneeds-triage+label%3Acommunity) 
+2. If you don't find an existing issue, [Request a New Connector](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=area%2Fconnectors%2Cnew-connector&projects=&template=5-feature-new-connector.yaml)
+
+This will enable our team to make sure your contribution does not overlap with existing works and will comply with the design orientation we are currently heading the product toward. If you do not receive an update on the issue from our team, please ping us on [Slack](https://slack.airbyte.io)!
+
+
+#### Code your contribution
+1. To contribute to a connector, fork the [Connector repository](https://github.com/airbytehq/airbyte). 
+2. Open a branch for your work
+3. Code the change
+4. Ensure all tests pass. For connectors, this includes acceptance tests as well.
+5. Update documentation in `docs/integrations/.md` 
+
+
+#### Open a pull request
+1. Rebase master with your branch before submitting a pull request.
+2. Open the pull request.
+3. Follow the [title convention](./resources/pull-requests-handbook.md#pull-request-title-convention) for Pull Requests
+4. Link to an existing Issue
+5. Update the [description](./resources/pull-requests-handbook.md#descriptions)
+6. Wait for a review from a community maintainer or our team.
+
+#### 4. Review process
+When we review, we look at:
+* ‌Does the PR add all existing streams, pagination and incremental syncs?
+* Is the proposed solution reasonable?
+* Is it tested? \(unit tests or integation tests\)
+‌Once your PR passes, we will merge it 🎉.
+
diff --git a/docs/contributing-to-airbyte/templates/README.md b/docs/contributing-to-airbyte/templates/README.md
deleted file mode 100644
index 60457c1d765b..000000000000
--- a/docs/contributing-to-airbyte/templates/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
----
-description: All the documentation templates should live here.
----
-
-# Templates
-
diff --git a/docs/contributing-to-airbyte/templates/integration-documentation-template.md b/docs/contributing-to-airbyte/templates/integration-documentation-template.md
deleted file mode 100644
index f1eb4337f55a..000000000000
--- a/docs/contributing-to-airbyte/templates/integration-documentation-template.md
+++ /dev/null
@@ -1,57 +0,0 @@
----
-description: >-
-  This is the template that should be used when adding documentation for a new
-  connector.
----
-
-# Connector Doc Template
-
-## Sync overview
-
-### Output schema
-
-Is the output schema fixed \(e.g: for an API like Stripe\)? If so, point to the connector's schema \(e.g: link to Stripe’s documentation\) or describe the schema here directly \(e.g: include a diagram or paragraphs describing the schema\).
-
-Describe how the connector's schema is mapped to Airbyte concepts. An example description might be: “MagicDB tables become Airbyte Streams and MagicDB columns become Airbyte Fields. In addition, an extracted\_at column is appended to each row being read.”
-
-### Data type mapping
-
-This section should contain a table mapping each of the connector's data types to Airbyte types. At the moment, Airbyte uses the same types used by [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html). `string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number` are the most commonly used data types.
-
-| Integration Type | Airbyte Type | Notes |
-| :--- | :--- | :--- |
-
-
-### Features
-
-This section should contain a table with the following format:
-
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync |  |  |
-| Incremental Sync |  |  |
-| Replicate Incremental Deletes |  |  |
-| For databases, WAL/Logical replication |  |  |
-| SSL connection |  |  |
-| SSH Tunnel Support |  |  |
-| \(Any other source-specific features\) |  |  |
-
-### Performance considerations
-
-Could this connector hurt the user's database/API/etc... or put too much strain on it in certain circumstances? For example, if there are a lot of tables or rows in a table? What is the breaking point \(e.g: 100mm> records\)? What can the user do to prevent this? \(e.g: use a read-only replica, or schedule frequent syncs, etc..\)
-
-## Getting started
-
-### Requirements
-
-* What versions of this connector does this implementation support? \(e.g: `postgres v3.14 and above`\) 
-* What configurations, if any, are required on the connector? \(e.g: `buffer_size > 1024`\)
-* Network accessibility requirements
-* Credentials/authentication requirements? \(e.g: A  DB user with read permissions on certain tables\) 
-
-### Setup guide
-
-For each of the above high-level requirements as appropriate, add or point to a follow-along guide. See existing source or destination guides for an example.
-
-For each major cloud provider we support, also add a follow-along guide for setting up Airbyte to connect to that destination. See the Postgres destination guide for an example of what this should look like.
-
diff --git a/docs/contributing-to-airbyte/writing-docs.md b/docs/contributing-to-airbyte/writing-docs.md
new file mode 100644
index 000000000000..b3a927b36dea
--- /dev/null
+++ b/docs/contributing-to-airbyte/writing-docs.md
@@ -0,0 +1,151 @@
+# Updating Documentation
+
+We welcome contributions to the Airbyte documentation! 
+
+Our docs are written in [Markdown](https://guides.github.com/features/mastering-markdown/) following the [Google developer documentation style guide](https://developers.google.com/style/highlights) and the files are stored in our [Github repository](https://github.com/airbytehq/airbyte/tree/master/docs). The docs are published at [docs.airbyte.com](https://docs.airbyte.com/) using [Docusaurus](https://docusaurus.io/) and [GitHub Pages](https://pages.github.com/). 
+
+## Finding good first issues
+
+The Docs team maintains a list of [#good-first-issues](https://github.com/airbytehq/airbyte/issues?q=is%3Aopen+is%3Aissue+label%3Aarea%2Fdocumentation+label%3A%22good+first+issue%22) for new contributors. 
+
+- If you're new to technical writing, start with the smaller issues (fixing typos, broken links, spelling and grammar, and so on). You can [edit the files directly on GitHub](#editing-directly-on-github).
+- If you're an experienced technical writer or a developer interested in technical writing, comment on an issue that interests you to discuss it with the Docs team. Once we decide on the approach and the tasks involved, [edit the files and open a Pull Request](#editing-on-your-local-machine) for the Docs team to review.
+
+## Contributing to Airbyte docs
+
+Before contributing to Airbyte docs, read the Airbyte Community
+
+:::tip
+If you're new to GitHub and Markdown, complete [the First Contributions tutorial](https://github.com/firstcontributions/first-contributions) and learn [Markdown basics](https://guides.github.com/features/mastering-markdown/) before contributing to Airbyte documentation. 
+:::
+
+You can contribute to Airbyte docs in two ways:
+
+### Editing directly on GitHub
+
+To make minor changes (example: fixing typos) or edit a single file, you can edit the file directly on GitHub:
+
+1. Click **Edit this page** at the bottom of any published document on [docs.airbyte.com](https://docs.airbyte.com/). You'll be taken to the GitHub editor. 
+2. [Edit the file directly on GitHub and open a Pull Request](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files).
+
+### Editing on your local machine
+
+To make complex changes or edit multiple files, edit the files on your local machine:
+
+1. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the Airbyte [repository](https://github.com/airbytehq/airbyte).
+2. Clone the fork on your local machine:
+
+   ```bash
+   git clone git@github.com:{YOUR_USERNAME}/airbyte.git
+   cd airbyte
+   ```
+
+   Or
+
+   ```bash
+   git clone https://github.com/{YOUR_USERNAME}/airbyte.git
+   cd airbyte
+   ```
+
+   While cloning on Windows, you might encounter errors about long filenames. Refer to the instructions [here](../deploying-airbyte/local-deployment.md#handling-long-filename-error) to correct it.
+
+3. Test changes locally:
+
+   To install the docs locally, run the following commands in your terminal:
+
+   ```bash
+   cd docusaurus
+   yarn install
+   ```
+
+   To see changes as you make them, run:
+
+   ```bash
+   yarn start
+   ```
+
+   Then navigate to [http://localhost:3000/](http://localhost:3000/). Whenever you make and save changes, you will see them reflected in the server. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal.  
+
+   You can also build the docs locally and see the resulting changes. This is useful if you introduce changes that need to be run at build-time (e.g. adding a docs plug-in). To do so, run:
+
+   ```bash
+   yarn build
+   yarn serve
+   ```
+  
+   Then navigate to [http://localhost:3000/](http://localhost:3000/) to see your changes. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal.  
+
+
+4. [Follow the GitHub workflow](https://docs.github.com/en/get-started/quickstart/contributing-to-projects/) to edit the files and create a pull request.
+
+    :::note
+    Before we accept any contributions, you'll need to sign the Contributor License Agreement (CLA). By signing a CLA, we can ensure that the community is free and confident in its ability to use your contributions. You will be prompted to sign the CLA while opening a pull request.
+    :::
+
+5. Assign `airbytehq/docs` as a Reviewer for your pull request. 
+
+## Additional guidelines
+
+- If you're updating a connector doc, follow the [Connector documentation template](https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw)
+- If you're adding a new file, update the [sidebars.js file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js)
+- If you're adding a README to a code module, make sure the README has the following components:
+    - A brief description of the module
+    - Development pre-requisites (like which language or binaries are required for development)
+    - How to install dependencies
+    - How to build and run the code locally & via Docker
+    - Any other information needed for local iteration
+
+## Advanced tasks 
+
+### Adding a redirect
+
+To add a redirect, open the [`docusaurus.config.js`](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) file and locate the following commented section:
+
+```js
+//                        {
+//                         from: '/some-lame-path',
+//                         to: '/a-much-cooler-uri',
+//                        },
+```
+
+Copy this section, replace the values, and [test the changes locally](#editing-on-your-local-machine) by going to the path you created a redirect for and verify that the address changes to the new one.
+
+:::note 
+Your path **needs** a leading slash `/` to work
+:::
+
+### Deploying and reverting the documentation site
+
+:::note
+Only the Airbyte team and maintainers have permissions to deploy the documentation site.
+:::
+
+#### Automated documentation site deployment
+
+When `docs/` folder gets changed in `master` branch of the repository, [`Deploy docs.airbyte.com` Github workflow](https://github.com/airbytehq/airbyte/actions/workflows/deploy-docs-site.yml) steps in, builds and deploys the documentation site.  This process is automatic, takes five to ten minutes, and needs no human intervention.
+
+#### Manual documentation site deployment
+
+:::note
+Manual deployment is reserved for emergency cases.  Please, bear in mind that automatic deployment is triggered by changes to `docs/` folder, so it needs to be disabled to avoid interference with manual deployment.
+:::
+
+You'll need a GitHub SSH key to deploy the documentation site using the [deployment tool](https://github.com/airbytehq/airbyte/blob/master/tools/bin/deploy_docusaurus). 
+
+To deploy the documentation site, run:
+
+```bash
+cd airbyte
+# or cd airbyte-cloud  
+git checkout master
+git pull
+./tools/bin/deploy_docusaurus
+```
+
+To revert/rollback doc changes, run:
+
+```
+cd airbyte
+git checkout 
+./tools/bin/deploy_docusaurus
+```
diff --git a/docs/deploying-airbyte/README.md b/docs/deploying-airbyte/README.md
index 072401786401..2f8a6e290a36 100644
--- a/docs/deploying-airbyte/README.md
+++ b/docs/deploying-airbyte/README.md
@@ -8,8 +8,7 @@
 - [On Azure VM Cloud Shell](on-azure-vm-cloud-shell.md)
 - [On Digital Ocean Droplet](on-digitalocean-droplet.md)
 - [On GCP.md](on-gcp-compute-engine.md)
-- [On Kubernetes](on-kubernetes.md)
-    - [Using Helm](on-kubernetes-via-helm.md)
+- [On Kubernetes](on-kubernetes-via-helm.md)
 - [On OCI VM](on-oci-vm.md)
 - [On Restack](on-restack.md)
 - [On Plural](on-plural.md)
diff --git a/docs/deploying-airbyte/on-kubernetes-via-helm.md b/docs/deploying-airbyte/on-kubernetes-via-helm.md
index 2add966a933b..1efea587491b 100644
--- a/docs/deploying-airbyte/on-kubernetes-via-helm.md
+++ b/docs/deploying-airbyte/on-kubernetes-via-helm.md
@@ -1,4 +1,4 @@
-# Deploy Airbyte on Kubernetes using Helm (Beta)
+# Deploy Airbyte on Kubernetes using Helm
 
 ## Overview
 
@@ -16,10 +16,10 @@ Alternatively, you can deploy Airbyte on [Restack](https://www.restack.io) to pr
 
 For local testing we recommend following one of the following setup guides:
 
-* [Docker Desktop \(Mac\)](https://docs.docker.com/desktop/kubernetes)
-* [Minikube](https://minikube.sigs.k8s.io/docs/start)
-  * NOTE: Start Minikube with at least 4gb RAM with `minikube start --memory=4000`
-* [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
+- [Docker Desktop \(Mac\)](https://docs.docker.com/desktop/kubernetes)
+- [Minikube](https://minikube.sigs.k8s.io/docs/start)
+  - NOTE: Start Minikube with at least 4gb RAM with `minikube start --memory=4000`
+- [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
 
 For testing on GKE you can [create a cluster with the command line or the Cloud Console UI](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster).
 
@@ -40,7 +40,7 @@ For GKE:
 1. Configure `gcloud` with `gcloud auth login`.
 2. On the Google Cloud Console, the cluster page will have a `Connect` button, which will give a command to run locally that looks like
 
-    `gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME`.
+   `gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME`.
 
 3. Use `kubectl config get-contexts` to show the contexts available.
 4. Run `kubectl config use-context $GKE_CONTEXT` to access the cluster from `kubectl`.
@@ -58,8 +58,8 @@ For EKS:
 To install helm simply run:
 
 For MacOS:
-  
-  `brew install helm`
+
+`brew install helm`
 
 For Linux:
 
@@ -82,14 +82,14 @@ After this you can browse all charts uploaded to repository by running `helm sea
 It'll produce the output below:
 
 ```text
-NAME                            CHART VERSION   APP VERSION     DESCRIPTION                             
-airbyte-oss/airbyte             0.30.23         0.39.37-alpha   Helm chart to deploy airbyte            
-airbyte-oss/airbyte-bootloader  0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-bootloader 
+NAME                            CHART VERSION   APP VERSION     DESCRIPTION
+airbyte-oss/airbyte             0.30.23         0.39.37-alpha   Helm chart to deploy airbyte
+airbyte-oss/airbyte-bootloader  0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-bootloader
 airbyte-oss/pod-sweeper         0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-pod-sweeper
-airbyte-oss/server              0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-server     
-airbyte-oss/temporal            0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-temporal   
-airbyte-oss/webapp              0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-webapp     
-airbyte-oss/worker              0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-worker  
+airbyte-oss/server              0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-server
+airbyte-oss/temporal            0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-temporal
+airbyte-oss/webapp              0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-webapp
+airbyte-oss/worker              0.30.23         0.39.37-alpha   Helm chart to deploy airbyte-worker
 ```
 
 ## Deploy Airbyte
@@ -98,7 +98,8 @@ airbyte-oss/worker              0.30.23         0.39.37-alpha   Helm chart to de
 
 If you don't intend to customise your deployment, you can deploy airbyte as is with default values.
 
-In order to do so, run the command: 
+In order to do so, run the command:
+
 ```
 helm install %release_name% airbyte/airbyte
 ```
@@ -115,6 +116,40 @@ After specifying your own configuration, run the following command:
 helm install --values path/to/values.yaml %release_name% airbyte/airbyte
 ```
 
+### (Alpha) Airbyte Enterprise deployment
+
+[Airbyte Enterprise](/airbyte-enterprise) is in early alpha stages, so this section will likely evolve. That said, if you have an Airbyte Enterprise license key and wish to install Airbyte Enterprise via helm, follow these steps:
+
+1. Checkout the latest revision of the [airbyte-platform repository](https://github.com/airbytehq/airbyte-platform)
+
+2. Add your Airbyte Enterprise license key and [auth configuration details](/airbyte-enterprise#single-sign-on-sso) to a file called `airbyte.yml` in the root directory of `airbyte-platform`. You can copy `airbyte.sample.yml` to use as a template:
+
+```text
+cp airbyte.sample.yml airbyte.yml
+```
+
+Then, open up `airbyte.yml` in your text editor to fill in the indicated fields.
+
+:::caution
+
+For now, auth configurations aren't easy to modify once initially installed, so please double check them to make sure they're accurate before proceeding! This will be improved in the near future.
+
+:::
+
+3. Make sure your helm repository is up to date:
+
+```text
+helm repo update
+```
+
+4. Install Airbyte Enterprise on helm using the following command:
+
+```text
+RELEASE_NAME=./tools/bin/install_airbyte_pro_on_helm.sh
+```
+
+If unspecified, the default release name is `airbyte-pro`. You can change this by editing the `install_airbyte_pro_on_helm.sh` script.
+
 ## Migrate from old charts to new ones
 
 Starting from `0.39.37-alpha` we've revisited helm charts structure and separated all components of airbyte into their own independent charts, thus by allowing our developers to test single component without deploying airbyte as a whole and by upgrading single component at a time.
@@ -128,12 +163,15 @@ Since the latest release of bitnami/minio chart, they've changed the way of sett
 Going forward in new version you need to specify the following values in values yaml for user/password instead old one
 
 Before:
+
 ```text
 minio:
   rootUser: airbyte-user
   rootPassword: airbyte-password-123
 ```
+
 After:
+
 ```text
 minio:
   auth:
@@ -144,9 +182,9 @@ minio:
 
 Before upgrading the chart update values.yaml as stated above and then run:
 
-* Get the old rootPassword by running `export ROOT_PASSWORD=$(kubectl get secret --namespace "default" %release_name%-minio -o jsonpath="{.data.root-password}" | base64 -d)`
-* Perform upgrade of chart by running `helm upgrade %release_name% airbyte/airbyte --set auth.rootPassword=$ROOT_PASSWORD`
-  * If you get an error about setting the auth.rootPassword, then you forgot to update the `values.yaml` file
+- Get the old rootPassword by running `export ROOT_PASSWORD=$(kubectl get secret --namespace "default" %release_name%-minio -o jsonpath="{.data.root-password}" | base64 -d)`
+- Perform upgrade of chart by running `helm upgrade %release_name% airbyte/airbyte --set auth.rootPassword=$ROOT_PASSWORD`
+  - If you get an error about setting the auth.rootPassword, then you forgot to update the `values.yaml` file
 
 ### Custom logging and jobs configuration
 
@@ -162,7 +200,8 @@ global:
         %your_jobs_options_here%
 ```
 
-After updating `values.yaml` simply upgrade your chart by running command: 
+After updating `values.yaml` simply upgrade your chart by running command:
+
 ```shell
 helm upgrade -f path/to/values.yaml %release_name% airbyte/airbyte
 ```
@@ -179,7 +218,8 @@ If you're using external DB secrets, then provide them in `values.yaml` under gl
     port: "5432"
 ```
 
-And upgrade the chart by running: 
+And upgrade the chart by running:
+
 ```shell
 helm upgrade -f path/to/values.yaml %release_name% airbyte/airbyte
 ```
diff --git a/docs/deploying-airbyte/on-kubernetes.md b/docs/deploying-airbyte/on-kubernetes.md
deleted file mode 100644
index a0ec981275ee..000000000000
--- a/docs/deploying-airbyte/on-kubernetes.md
+++ /dev/null
@@ -1,277 +0,0 @@
-# (Deprecated) Deploy Airbyte on Kubernetes using Kustomize
-
-:::caution
-This deployment method uses Kustomize and is only supported up to [Airbyte version `0.40.32`](https://github.com/airbytehq/airbyte/releases/tag/v0.40.32). For existing deployments, check out [commit `21a7e102183e20d2d4998ea70c2a8fe4eac8921b`](https://github.com/airbytehq/airbyte/commit/21a7e102183e20d2d4998ea70c2a8fe4eac8921b) to continue deploying using Kustomize. For new deployments, [deploy Airbyte on Kubernetes via Helm](https://docs.airbyte.com/deploying-airbyte/on-kubernetes-via-helm).
-:::
-
-This page guides you through deploying Airbyte Open Source on Kubernetes. 
-
-
-## Requirements
-
-To test locally, you can use one of the following:
-
-* [Docker Desktop](https://docs.docker.com/desktop/) with [Kubernetes](https://docs.docker.com/desktop/kubernetes/#enable-kubernetes) enabled
-* [Minikube](https://docs.docker.com/desktop/kubernetes/#enable-kubernetes) with at least 4GB RAM
-* [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/)
-
-
-To test on Google Kubernetes Engine(GKE), create a standard zonal cluster.
-
-To test on  Amazon Elastic Kubernetes Service (Amazon EKS), install eksctl and create a cluster.
-
-:::info 
-Airbyte deployment is tested on GKE and EKS with version v1.19 and above. If you run into problems, reach out on the `#airbyte-help` channel in our Slack or create an issue on GitHub.
-:::
-
-## Install and configure `kubectl `
-
-Install `kubectl` and run the following command to configure it and connect to your cluster:
-
-```bash
-kubectl use-context 
-```
-
-To configure `kubectl` in `GKE`:
-
-1. Initialize the `gcloud` cli.
-2. To view cluster details, go to the `cluster` page in the Google Cloud Console and click `connect`. Run the following command to test cluster details: 
-`gcloud container clusters get-credentials  --zone  --project `.
-3. To view contexts, run: `kubectl config get-contexts`.
-4. To access the cluster from `kubectl` run : `kubectl config use-context `.
-
-To configure `kubectl` in  `EKS`:
-
-1. [Configure AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) to connect to your project.
-2. Install [`eksctl`](https://eksctl.io/introduction/).
-3. To Make contexts available to `kubectl`, run `eksctl utils write-kubeconfig --cluster=`  
-4. To view available contexts,  run `kubectl config get-contexts`.
-5. To access the cluster, run `kubectl config use-context `.
-
-## Configure Logs
-
-### Default configuration
-
-Airbyte comes with a self-contained Kubernetes deployment and uses a stand-alone `Minio` deployment in both the `dev` and `stable` versions. Logs are published to the `Minio` deployment by default.
-
-To send the logs to the local `Minio` deployment, make sure the specified credentials have both read and write permissions.
-
-### Custom configuration
-
-Airbyte supports logging to the `Minio` layer, `S3` bucket, and `GCS` bucket.
-
-### Customize the `Minio` log location
-
-To write to a custom location, update the following `.env` variable in the `kube/overlays/stable` directory (you will find this directory at the location you launched Airbyte)
-
-``` bash
-S3_LOG_BUCKET=
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-S3_MINIO_ENDPOINT=
-S3_LOG_BUCKET_REGION=
-```
-Set the` S3_PATH_STYLE_ACCESS variable to `true`.
-Let the `S3_LOG_BUCKET_REGION` variable remain empty.
-
-### Configure the Custom `S3` Log Location​
-
-For the `S3` log location, create an S3 bucket with your AWS credentials.
-
-To write to a custom location, update the following `.env` variable in the `kube/overlays/stable` directory (you can find this directory at the location you launched Airbyte)
-
-``` bash
-S3_LOG_BUCKET=
-S3_LOG_BUCKET_REGION=
-# Set this to empty.
-S3_MINIO_ENDPOINT=
-# Set this to empty.
-S3_PATH_STYLE_ACCESS=
-```
-Replace the following variable in `.secrets` file in the `kube/overlays/stable` directory:
-
-```bash
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-```
-
-### Configure the Custom GCS Log Location​
-
-Create a GCS bucket and  GCP credentials if you haven’t already. Make sure your GCS log bucket has read/write permission.
-
-To configure the custom log location:
-
-Base encode the GCP JSON secret with the following command:
-
-```bash
-# The output of this command will be a Base64 string.
-$ cat gcp.json | base64
-```
-To populate the `gcs-log-creds` secrets with the Base64-encoded credential, take the encoded GCP JSON secret from the previous step and add it to `secret-gcs-log-creds.yaml` file as the value for `gcp.json` key. 
-
-```bash
-apiVersion: v1
-kind: Secret
-metadata:
- name: gcs-log-creds
- namespace: default
-data:
- gcp.json: 
-```
-
-In the `kube/overlays/stable` directory, update the  `GCS_LOG_BUCKET` with your GCS log bucket credentials:
-
-```bash
-GCS_LOG_BUCKET=
-```
-
-Modify `GOOGLE_APPLICATION_CREDENTIALS` to the path to `gcp.json` in the `.secrets` file at `kube/overlays/stable` directory.
-
-```bash
-# The path the GCS creds are written to. Unless you know what you are doing, use the below default value.
-
-GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcs-log-creds/gcp.json
-```
-
-
-## Launch Airbyte
-
-The following commands will help you launch Airbyte:
-
-```bash
-git clone https://github.com/airbytehq/airbyte.git
-cd airbyte
-kubectl apply -k kube/overlays/stable
-```
-
-To check the pod status, run `kubectl get pods | grep airbyte`.
-
-If you are on Windows, run `kubectl get pods` to the list of pods.
-
-Run `kubectl port-forward svc/airbyte-webapp-svc 8000:80`  to allow access to the UI/API.
-Navigate to http://localhost:8000 in your browser to verify the deployment.
-
-## Deploy Airbyte on Kubernetes in production
-
-### Set resource limits
-
-* Core container pods 
-
-  * To provide different resource requirements for core pods, set resource limits in the  `kube/overlays/stable-with-resource-limits/set-resource-limits.yaml` file.
-
-  * To launch Airbyte with new resource limits, 	use the `kubectl apply -k kube/overlays/stable-with-resource-limits command.
-
-* Connector pods
-	
-  * By default, connector pods launch without resource limits. To add resource limit, configure the `Docker resource limits` section of the `.env` file in the `kube/overlays` directory.
-
-* Volume sizes
-
-  * To specify different volume sizes for the persistent volume backing Airbyte, modify `kube/resources/volume-*`  files.
-
-
-### Increase job parallelism
-
-The ability to run parallel jobs like getting specs, checking connections, discovering schemas and performing syncs is limited by a few factors. `Airbyte-worker-pods` picks and executes the job. Increasing the number of workers will allow more jobs to be processed.
-
-To create more worker pods, increase the number of replicas for the `airbyte-worker` deployment. Refer to examples of increasing worker pods in a Kustomization patch in `airbyte/kube/overlays/dev-integration-test/kustomization.yaml` and `airbyte/kube/overlays/dev-integration-test/parallelize-worker.yaml` 
- 
-To limit the exposed ports in `.env`  file, set the value to `TEMPORAL_WORKER_PORTS`. You can run jobs parallely at each exposed port.
-If you do not have enough ports to communicate, the jobs might not complete or halt until ports become available.
-
-You can set a limit for the maximum parallel jobs that run on the pod. Set the value to `MAX_SPEC_WORKERS`, `MAX_CHECK_WORKERS`, `MAX_DISCOVER_WORKERS`, and `MAX_SYNC_WORKERS` variables in the worker pod deployment and not in `.env` file. You can use these values to create separate worker deployments for each type of worker with different resource allocations.
-
-
-### Cloud Logging
-
-Airbyte writes logs to two different directories: The `App-logging` directory and the `job-logging` directory. App logs, server logs, and scheduler logs are written to the `app-logging` directory. Job logs are written to the `job-logging` directory. Both directories live at the top level. For example, the app logging directory may live at `s3://log-bucket/app-logging`. We recommend having a dedicated logging bucket and not using it for other purposes.
-
-Airbyte publishes logs every minute, so it’s normal to have minute-long log delays. Cloud Storages do not support append operations. Each publisher creates its own log files, which means you will have hundreds of files in your log bucket.
-
-Each log file is uncompressed and named `{yyyyMMddHH24mmss}_{podname}_{UUID}`.
-To view logs, navigate to the relevant folder and download the file for the time period you want.
-
-### Use external databases
-
-You can configure a custom database instead of a simple `postgres` container in Kubernetes. This separate instance (AWS RDS or Google Cloud SQL) should be easier and safer to maintain than Postgres on your cluster.
-
-## Customize Airbytes Manifests
-
-We use Kustomize to allow configuration for different environments. Our shared resources are in the `kube/resources` directory. We recommend defining overlays for each environment and creating your own overlay to customize your deployments. The overlay can live in your own version control system.
-An example of `kustomization.yaml`  file:
-
-```bash
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-bases: https://github.com/airbytehq/airbyte.git/kube/overlays/stable?ref=master
-```
-
-### View Raw Manifests
-
-To view manifests for a specific overlay that Kustomize applies to your Kubernetes cluster, run `kubectl kustomize kube/overlays/stable`. 
-
-
-### Helm Charts
-
-For detailed information about Helm Charts, refer to the charts [readme](https://github.com/airbytehq/airbyte/tree/master/charts/airbyte) file.
-
-
-## Operator Guide
-
-### View API server logs
-
-You can view real-time logs in `kubectl logs deployments/airbyte-server` directory and download them from the Admin Tab.
-
-### Connector Container Logs​
-
-All logs can be accessed by viewing the scheduler logs. As for connector container logs, use Airbyte UI or Airbyte API to isolate them for a specific job attempt and for easier understanding. Connector pods launched by Airbyte will not relay logs directly to Kubernetes logging. You must access these logs through Airbyte.
-
-
-### Resize Volumes
-
-To resize a volume, change the `.spec.resources.requests.storage` value. After re-applying, extend the mount(if that operation is supported for your mount type). For a production deployment, track the usage of volumes to ensure they don't run out of space.
-
-### Copy Files in Volumes
-
-To copy files, use the [`cp` command in kubectl](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#cp).
-
-### List Files
-
-To list files, run:
-
-`kubectl exec -it airbyte-server-6b5747df5c-bj4fx ls /tmp/workspace/8`
-
-### Read Files
-
-To read files, run:
-
-`kubectl exec -it airbyte-server-6b5747df5c-bj4fx cat /tmp/workspace/8/0/logs.log`
-
-### Persistent storage on Google Kubernetes Engine(GKE) regional cluster
-
-Running Airbyte on a GKE regional cluster requires enabling persistent regional storage. Start with [enabling CSE driver](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/gce-pd-csi-driver#enabling_the_on_an_existing_cluster) on GKE and add `storageClassName: standard-rwo` to the [volume-configs.yamll](https://github.com/airbytehq/airbyte/blob/86ee2ad05bccb4aca91df2fb07c412efde5ba71c/kube/resources/volume-configs.yaml).
-
-Sample `volume-configs.yaml` file:
-
-```bash
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: airbyte-volume-configs
-  labels:
-    airbyte: volume-configs
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 500Mi
-  storageClassName: standard-rwo
-```
-
-## Troubleshooting
-If you encounter any issues, reach out to our community on [Slack](https://slack.airbyte.com/).
-
-
-
diff --git a/docs/integrations/destinations/amazon-sqs.md b/docs/integrations/destinations/amazon-sqs.md
index 04143f055ccc..6178d690e0f0 100644
--- a/docs/integrations/destinations/amazon-sqs.md
+++ b/docs/integrations/destinations/amazon-sqs.md
@@ -18,7 +18,7 @@ Amazon SQS messages can only contain JSON, XML or text, and this connector suppo
 | :--- | :--- | :--- |
 | Full Refresh Sync | No |  |
 | Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No |  |
+| Incremental - Append + Deduped | No |  |
 | Namespaces | No |  |
 
 ## Getting started
@@ -33,7 +33,7 @@ Amazon SQS messages can only contain JSON, XML or text, and this connector suppo
 
 If the target SQS Queue is not public, you will need the following permissions on the Queue:
 
-* `sqs:SendMessage` 
+* `sqs:SendMessage`
 
 ### Properties
 
@@ -54,9 +54,9 @@ Required properties are 'Queue URL' and 'AWS Region' as noted in **bold** below.
 * Message Body Key (STRING)
   * Rather than sending the entire Record as the Message Body, use this property to reference a Key in the Record to use as the message body. The value of this property should be the Key name in the input Record. The key must be at the top level of the Record, nested Keys are not supported.
 * Message Group Id (STRING)
-  * When using a FIFO queue, this property is **required**. 
+  * When using a FIFO queue, this property is **required**.
   * See the [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) for more detail.
-  
+
 ### Setup guide
 
 * [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/)
diff --git a/docs/integrations/destinations/azure-blob-storage.md b/docs/integrations/destinations/azure-blob-storage.md
index 4677d3df8390..d93f132cfb99 100644
--- a/docs/integrations/destinations/azure-blob-storage.md
+++ b/docs/integrations/destinations/azure-blob-storage.md
@@ -6,53 +6,50 @@ This destination writes data to Azure Blob Storage.
 
 The Airbyte Azure Blob Storage destination allows you to sync data to Azure Blob Storage. Each stream is written to its own blob under the container.
 
-:::info
-Cloud storage may incur egress costs. Egress refers to data that is transferred out of the cloud storage system, such as when you download files or access them from a different location. For more information, see the [Azure Blob Storage pricing guide](https://azure.microsoft.com/en-us/pricing/details/storage/blobs/).
-:::
-
 ## Prerequisites
+
 - For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your AzureBlobStorage connector to version `0.1.6` or newer
 
 ## Sync Mode
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured blob. |
-| Incremental - Append Sync | ✅ | The append mode would only work for "Append blobs" blobs as per Azure limitations, more details [https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction\#blobs](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction#blobs) |
-| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. |
+| Feature                        | Support | Notes                                                                                                                                                                                                                                                                           |
+| :----------------------------- | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured blob.                                                                                                                                                                                                   |
+| Incremental - Append Sync      |   ✅    | The append mode would only work for "Append blobs" blobs as per Azure limitations, more details [https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction\#blobs](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction#blobs) |
+| Incremental - Append + Deduped |   ❌    | destination.                                                                                                                                                                                                                                                                    |
 
 ## Configuration
 
-| Parameter                                    | Type    | Notes                                                                                                                                                                     |
-|:---------------------------------------------|:-------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Parameter                                    |  Type   | Notes                                                                                                                                                                     |
+| :------------------------------------------- | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | Endpoint Domain Name                         | string  | This is Azure Blob Storage endpoint domain name. Leave default value \(or leave it empty if run container from command line\) to use Microsoft native one.                |
 | Azure blob storage container \(Bucket\) Name | string  | A name of the Azure blob storage container. If not exists - will be created automatically. If leave empty, then will be created automatically airbytecontainer+timestamp. |
 | Azure Blob Storage account name              | string  | The account's name of the Azure Blob Storage.                                                                                                                             |
 | The Azure blob storage account key           | string  | Azure blob storage account key. Example: `abcdefghijklmnopqrstuvwxyz/0123456789+ABCDEFGHIJKLMNOPQRSTUVWXYZ/0123456789%++sampleKey==`.                                     |
 | Azure Blob Storage output buffer size        | integer | Azure Blob Storage output buffer size, in megabytes. Example: 5                                                                                                           |
-| Azure Blob Storage spill size                | integer | Azure Blob Storage spill size, in megabytes. Example: 500. After exceeding threshold connector will create new blob with incremented sequence number 'prefix_name'_seq+1  |
+| Azure Blob Storage spill size                | integer | Azure Blob Storage spill size, in megabytes. Example: 500. After exceeding threshold connector will create new blob with incremented sequence number 'prefix_name'\_seq+1 |
 | Format                                       | object  | Format specific configuration. See below for details.                                                                                                                     |
 
 ⚠️ Please note that under "Full Refresh Sync" mode, data in the configured blob will be wiped out before each sync. We recommend you to provision a dedicated Azure Blob Storage Container resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️
 
 ## Output Schema
 
-Each stream will be outputted to its dedicated Blob according to the configuration. The complete datastore of each stream includes all the output files under that Blob. You can think of the Blob as equivalent of a Table in the database world. 
+Each stream will be outputted to its dedicated Blob according to the configuration. The complete datastore of each stream includes all the output files under that Blob. You can think of the Blob as equivalent of a Table in the database world.
 If stream replication exceeds configured threshold data will continue to be replicated in a new blob file for better read performance
 
-* Under Full Refresh Sync mode, old output files will be purged before new files are created.
-* Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
+- Under Full Refresh Sync mode, old output files will be purged before new files are created.
+- Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
 
 ### CSV
 
 Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, an emission timestamp, and the data blob. With the CSV output, it is possible to normalize \(flatten\) the data blob to multiple columns.
 
-| Column | Condition | Description |
-| :--- | :--- | :--- |
-| `_airbyte_ab_id` | Always exists | A uuid assigned by Airbyte to each processed record. |
-| `_airbyte_emitted_at` | Always exists. | A timestamp representing when the event was pulled from the data source. |
-| `_airbyte_data` | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |  |
-| root level fields | When root level normalization \(flattening\) is selected, the root level fields are expanded. |  |
+| Column                | Condition                                                                                         | Description                                                              |
+| :-------------------- | :------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- |
+| `_airbyte_ab_id`      | Always exists                                                                                     | A uuid assigned by Airbyte to each processed record.                     |
+| `_airbyte_emitted_at` | Always exists.                                                                                    | A timestamp representing when the event was pulled from the data source. |
+| `_airbyte_data`       | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |                                                                          |
+| root level fields     | When root level normalization \(flattening\) is selected, the root level fields are expanded.     |                                                                          |
 
 For example, given the following json object from a source:
 
@@ -68,15 +65,15 @@ For example, given the following json object from a source:
 
 With no normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `_airbyte_data` |
-| :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `_airbyte_data`                                                |
+| :------------------------------------- | :-------------------- | :------------------------------------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
 
 With root level normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `user_id` | `name` |
-| :--- | :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | 123 | `{ "first": "John", "last": "Doe" }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `user_id` | `name`                               |
+| :------------------------------------- | :-------------------- | :-------- | :----------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | 123       | `{ "first": "John", "last": "Doe" }` |
 
 ### JSON Lines \(JSONL\)
 
@@ -95,20 +92,20 @@ For example, given the following two json objects from a source:
 ```javascript
 [
   {
-    "user_id": 123,
-    "name": {
-      "first": "John",
-      "last": "Doe"
-    }
+    user_id: 123,
+    name: {
+      first: "John",
+      last: "Doe",
+    },
   },
   {
-    "user_id": 456,
-    "name": {
-      "first": "Jane",
-      "last": "Roe"
-    }
-  }
-]
+    user_id: 456,
+    name: {
+      first: "Jane",
+      last: "Roe",
+    },
+  },
+];
 ```
 
 They will be like this in the output file:
@@ -127,25 +124,25 @@ They will be like this in the output file:
 
 ### Setup guide
 
-* Fill up AzureBlobStorage info
-  * **Endpoint Domain Name**
-    * Leave default value \(or leave it empty if run container from command line\) to use Microsoft native one or use your own.
-  * **Azure blob storage container**
-    * If not exists - will be created automatically. If leave empty, then will be created automatically airbytecontainer+timestamp..
-  * **Azure Blob Storage account name**
-    * See [this](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create?tabs=azure-portal) on how to create an account.
-  * **The Azure blob storage account key**
-    * Corresponding key to the above user.
-  * **Format**
-    * Data format that will be use for a migrated data representation in blob.
-* Make sure your user has access to Azure from the machine running Airbyte.
-  * This depends on your networking setup.
-  * The easiest way to verify if Airbyte is able to connect to your Azure blob storage container is via the check connection tool in the UI.
+- Fill up AzureBlobStorage info
+  - **Endpoint Domain Name**
+    - Leave default value \(or leave it empty if run container from command line\) to use Microsoft native one or use your own.
+  - **Azure blob storage container**
+    - If not exists - will be created automatically. If leave empty, then will be created automatically airbytecontainer+timestamp..
+  - **Azure Blob Storage account name**
+    - See [this](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create?tabs=azure-portal) on how to create an account.
+  - **The Azure blob storage account key**
+    - Corresponding key to the above user.
+  - **Format**
+    - Data format that will be use for a migrated data representation in blob.
+- Make sure your user has access to Azure from the machine running Airbyte.
+  - This depends on your networking setup.
+  - The easiest way to verify if Airbyte is able to connect to your Azure blob storage container is via the check connection tool in the UI.
 
 ## CHANGELOG
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                         |
-|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | 0.2.0   | 2023-01-18 | [\#15318](https://github.com/airbytehq/airbyte/pull/21467) | Support spilling of objects exceeding configured size threshold                                                                                                 |
 | 0.1.6   | 2022-08-08 | [\#15318](https://github.com/airbytehq/airbyte/pull/15318) | Support per-stream state                                                                                                                                        |
 | 0.1.5   | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors                                                                                                          |
diff --git a/docs/integrations/destinations/bigquery-denormalized.md b/docs/integrations/destinations/bigquery-denormalized.md
index 76651c2645bf..a69a2c5ac896 100644
--- a/docs/integrations/destinations/bigquery-denormalized.md
+++ b/docs/integrations/destinations/bigquery-denormalized.md
@@ -8,7 +8,11 @@ See [destinations/bigquery](https://docs.airbyte.com/integrations/destinations/b
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                  |
 | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------- |
-| 1.4.1   | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count                                                                              | 
+| 1.5.3   | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild                                                                                     |
+| 1.5.2   | 2023-07-05 | [\#27936](https://github.com/airbytehq/airbyte/pull/27936) | Internal code change                                                                                                     |
+| 1.5.1   | 2023-06-30 | [\#27891](https://github.com/airbytehq/airbyte/pull/27891) | Revert bugged update                                                                                                     |
+| 1.5.0   | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                                     |
+| 1.4.1   | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count                                                                              |
 | 1.4.0   | 2023-04-28 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Fix: all integer schemas should be converted to Avro longs                                                               |
 | 1.3.3   | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup                                                                                                    |
 | 1.3.0   | 2023-04-19 | [\#25287](https://github.com/airbytehq/airbyte/pull/25287) | Add parameter to configure the number of file buffers when GCS is used as the loading method                             |
diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md
index 3c01bd9a683c..e6d94ce03776 100644
--- a/docs/integrations/destinations/bigquery.md
+++ b/docs/integrations/destinations/bigquery.md
@@ -73,7 +73,7 @@ The BigQuery destination connector supports the following [sync modes](https://d
 
 - Full Refresh Sync
 - Incremental - Append Sync
-- Incremental - Deduped History
+- Incremental - Append + Deduped
 
 ## Output schema
 
@@ -133,74 +133,102 @@ Now that you have set up the BigQuery destination connector, check out the follo
 
 ### bigquery
 
-| Version | Date       | Pull Request                                               | Subject                                                                                                                  |
-|:--------|:-----------|:-----------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
-| 1.4.4   | 2023-05-25 | [\#26585](https://github.com/airbytehq/airbyte/pull/26585) | Small tweak in logs for clarity                                                                                          |
-| 1.4.3   | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count                                                                              |
-| 1.4.2   | 2023-05-10 | [\#25925](https://github.com/airbytehq/airbyte/pull/25925) | Testing update. Normalization tests are now done in the destination container.                                           |
-| 1.4.1   | 2023-05-11 | [\#25993](https://github.com/airbytehq/airbyte/pull/25993) | Internal library update                                                                                                  |
-| 1.4.0   | 2023-04-29 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Internal library update. Bumping version to stay in sync with BigQuery-denormalized.                                     |
-| 1.3.4   | 2023-04-28 | [\#25588](https://github.com/airbytehq/airbyte/pull/25588) | Internal scaffolding change for future development                                                                       |
-| 1.3.3   | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup                                                                                                    |
-| 1.3.1   | 2023-04-20 | [\#25097](https://github.com/airbytehq/airbyte/pull/25097) | Internal scaffolding change for future development                                                                       |
-| 1.3.0   | 2023-04-19 | [\#25287](https://github.com/airbytehq/airbyte/pull/25287) | Add parameter to configure the number of file buffers when GCS is used as the loading method                             |
-| 1.2.20  | 2023-04-12 | [\#25122](https://github.com/airbytehq/airbyte/pull/25122) | Add additional data centers                                                                                              |
-| 1.2.19  | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases                                                                                       |
-| 1.2.18  | 2023-03-23 | [\#24447](https://github.com/airbytehq/airbyte/pull/24447) | Set the Service Account Key JSON field to always_show: true so that it isn't collapsed into an optional fields section   |
-| 1.2.17  | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays                                                               |
-| 1.2.16  | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush                                                                                  |
-| 1.2.15  | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long                                                                                    |
-| 1.2.14  | 2023-02-08 | [\#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error                                                                                         |
-| 1.2.13  | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging                                                                 |
-| 1.2.12  | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions                                                                          |
-| 1.2.11  | 2023-01-18 | [\#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue                                                         |
-| 1.2.9   | 2022-12-14 | [\#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check                                                           |
-| 1.2.8   | 2022-11-22 | [\#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage                                                             |
-| 1.2.7   | 2022-11-11 | [\#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location                                                                  |
-| 1.2.6   | 2022-11-10 | [\#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors                                                                    |
-| 1.2.5   | 2022-10-19 | [\#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs                                                                                                       |
-| 1.2.4   | 2022-09-26 | [\#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header                                                                                                    |
-| 1.2.3   | 2022-09-22 | [\#17054](https://github.com/airbytehq/airbyte/pull/17054) | Respect stream namespace                                                                                                 |
-| 1.2.1   | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | (bugged, do not use) Wrap logs in AirbyteLogMessage                                                                      |
-| 1.2.0   | 2022-09-09 | [\#14023](https://github.com/airbytehq/airbyte/pull/14023) | (bugged, do not use) Cover arrays only if they are nested                                                                |
-| 1.1.16  | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |
-| 1.1.15  | 2022-08-22 | [\#15787](https://github.com/airbytehq/airbyte/pull/15787) | Throw exception if job failed                                                                                            |
-| 1.1.14  | 2022-08-03 | [\#14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials                                                                                 |
-| 1.1.13  | 2022-08-02 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings                                                                                                |
-| 1.1.12  | 2022-08-02 | [\#15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode                                                                                                |
-| 1.1.11  | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging                                              |
-| 1.1.10  | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors                                                                   |
-| 1.1.9   | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager                                  |
-| 1.1.8   | 2022-06-07 | [\#13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys.                                               |
-| 1.1.7   | 2022-06-07 | [\#13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification.                                                                                      |
-| 1.1.6   | 2022-05-15 | [\#12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud.                                                    |
-| 1.1.5   | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error.                                                        |
-| 1.1.4   | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging.                          |
-| 1.1.3   | 2022-05-02 | [\#12528](https://github.com/airbytehq/airbyte/pull/12528) | Update Dataset location field description                                                                                |
-| 1.1.2   | 2022-04-29 | [\#12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field                                                                                     |
-| 1.1.1   | 2022-04-15 | [\#12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding                                                                            |
-| 1.1.0   | 2022-04-06 | [\#11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption.                                                          |
-| 1.0.2   | 2022-03-30 | [\#11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec                                                                                                             |
-| 1.0.1   | 2022-03-24 | [\#11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance                                                                                                |
-| 1.0.0   | 2022-03-18 | [\#11238](https://github.com/airbytehq/airbyte/pull/11238) | Updated spec and documentation                                                                                           |
-| 0.6.12  | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters                                                                                    |
-| 0.6.11  | 2022-03-03 | [\#10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM                                                                          |
-| 0.6.8   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                             |
-| 0.6.6   | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959)   | Fix null pointer exception from buffered stream consumer.                                                                |
-| 0.6.6   | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745)   | Integrate with Sentry.                                                                                                   |
-| 0.6.5   | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573)   | BigQuery Destination : update description for some input fields                                                          |
-| 0.6.4   | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id                                                                                |
-| 0.6.3   | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415)   | BigQuery Destination : Fix GCS processing of Facebook data                                                               |
-| 0.6.2   | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121)   | Fixed check method for GCS mode to verify if all roles assigned to user                                                  |
-| 0.6.1   | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039)   | Added part_size configuration to UI for GCS staging                                                                      |
-| 0.6.0   | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files                              |
-| 0.5.1   | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations                                                                                                 |
-| 0.5.0   | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables                                                                                      |
-| 0.4.1   | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers                                                                                    |
-| 0.4.0   | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option                                                                                       |
-| 0.3.12  | 2021-08-03 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add optional arg to make a possibility to change the BigQuery client's chunk\buffer size                                 |
-| 0.3.11  | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125)   | Enable `additionalPropertities` in spec.json                                                                             |
-| 0.3.10  | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId                                                        |
-| 0.3.9   | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026)   | Add sanitized json fields in raw tables to handle quotes in column names                                                 |
-| 0.3.6   | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional.                                                                            |
-| 0.3.4   | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option                                                                                              |
+| Version | Date       | Pull Request                                               | Subject                                                                                                                                                         |
+|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.10.2  | 2023-08-24 | [\#29805](https://github.com/airbytehq/airbyte/pull/29805) | Destinations v2: Don't soft reset in migration                                                                                                                  |
+| 1.10.1  | 2023-08-23 | [\#29774](https://github.com/airbytehq/airbyte/pull/29774) | Destinations v2: Don't soft reset overwrite syncs                                                                                                               |
+| 1.10.0  | 2023-08-21 | [\#29636](https://github.com/airbytehq/airbyte/pull/29636) | Destinations v2: Several Critical Bug Fixes (cursorless dedup, improved floating-point handling, improved special characters handling; improved error handling) |
+| 1.9.1   | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Under the hood: Add dependency on Java CDK v0.0.1.                                                                                                              |
+| 1.9.0   | 2023-08-17 | [\#29560](https://github.com/airbytehq/airbyte/pull/29560) | Destinations v2: throw an error on disallowed column name prefixes                                                                                              |
+| 1.8.1   | 2023-08-17 | [\#29522](https://github.com/airbytehq/airbyte/pull/29522) | Migration BugFix - ensure raw dataset created                                                                                                                   |
+| 1.8.0   | 2023-08-17 | [\#29498](https://github.com/airbytehq/airbyte/pull/29498) | Fix checkpointing logic in GCS staging mode                                                                                                                     |
+| 1.7.8   | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Migration BugFix - ensure migration happens before table creation for GCS staging.                                                                              |
+| 1.7.7   | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns                                                                                                        |
+| 1.7.6   | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Add v1 -> v2 migration Logic                                                                                                                   |
+| 1.7.5   | 2023-08-04 | [\#29106](https://github.com/airbytehq/airbyte/pull/29106) | Destinations v2: handle unusual CDC deletion edge case                                                                                                          |
+| 1.7.4   | 2023-08-04 | [\#29089](https://github.com/airbytehq/airbyte/pull/29089) | Destinations v2: improve special character handling in column names                                                                                             |
+| 1.7.3   | 2023-08-03 | [\#28890](https://github.com/airbytehq/airbyte/pull/28890) | Internal code updates; improved testing                                                                                                                         |
+| 1.7.2   | 2023-08-02 | [\#28976](https://github.com/airbytehq/airbyte/pull/28976) | Fix composite PK handling in v1 mode                                                                                                                            |
+| 1.7.1   | 2023-08-02 | [\#28959](https://github.com/airbytehq/airbyte/pull/28959) | Destinations v2: Fix CDC syncs in non-dedup mode                                                                                                                |
+| 1.7.0   | 2023-08-01 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Open up early access program opt-in                                                                                                            |
+| 1.6.0   | 2023-07-26 | [\#28723](https://github.com/airbytehq/airbyte/pull/28723) | Destinations v2: Change raw table dataset and naming convention                                                                                                 |
+| 1.5.8   | 2023-07-25 | [\#28721](https://github.com/airbytehq/airbyte/pull/28721) | Destinations v2: Handle cursor change across syncs                                                                                                              |
+| 1.5.7   | 2023-07-24 | [\#28625](https://github.com/airbytehq/airbyte/pull/28625) | Destinations v2: Limit Clustering Columns to 4                                                                                                                  |
+| 1.5.6   | 2023-07-21 | [\#28580](https://github.com/airbytehq/airbyte/pull/28580) | Destinations v2: Create dataset in user-specified location                                                                                                      |
+| 1.5.5   | 2023-07-20 | [\#28490](https://github.com/airbytehq/airbyte/pull/28490) | Destinations v2: Fix schema change detection in OVERWRITE mode when existing table is empty; other code refactoring                                             |
+| 1.5.4   | 2023-07-17 | [\#28382](https://github.com/airbytehq/airbyte/pull/28382) | Destinations v2: Schema Change Detection                                                                                                                        |
+| 1.5.3   | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild                                                                                                                            |
+| 1.5.2   | 2023-07-05 | [\#27936](https://github.com/airbytehq/airbyte/pull/27936) | Internal scaffolding change for future development                                                                                                              |
+| 1.5.1   | 2023-06-30 | [\#27891](https://github.com/airbytehq/airbyte/pull/27891) | Revert bugged update                                                                                                                                            |
+| 1.5.0   | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                                                                            |
+| 1.4.6   | 2023-06-28 | [\#27268](https://github.com/airbytehq/airbyte/pull/27268) | Internal scaffolding change for future development                                                                                                              |
+| 1.4.5   | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size                                                                                                                                               |
+| 1.4.4   | 2023-05-25 | [\#26585](https://github.com/airbytehq/airbyte/pull/26585) | Small tweak in logs for clarity                                                                                                                                 |
+| 1.4.3   | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count                                                                                                                     |
+| 1.4.2   | 2023-05-10 | [\#25925](https://github.com/airbytehq/airbyte/pull/25925) | Testing update. Normalization tests are now done in the destination container.                                                                                  |
+| 1.4.1   | 2023-05-11 | [\#25993](https://github.com/airbytehq/airbyte/pull/25993) | Internal library update                                                                                                                                         |
+| 1.4.0   | 2023-04-29 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Internal library update. Bumping version to stay in sync with BigQuery-denormalized.                                                                            |
+| 1.3.4   | 2023-04-28 | [\#25588](https://github.com/airbytehq/airbyte/pull/25588) | Internal scaffolding change for future development                                                                                                              |
+| 1.3.3   | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup                                                                                                                                           |
+| 1.3.1   | 2023-04-20 | [\#25097](https://github.com/airbytehq/airbyte/pull/25097) | Internal scaffolding change for future development                                                                                                              |
+| 1.3.0   | 2023-04-19 | [\#25287](https://github.com/airbytehq/airbyte/pull/25287) | Add parameter to configure the number of file buffers when GCS is used as the loading method                                                                    |
+| 1.2.20  | 2023-04-12 | [\#25122](https://github.com/airbytehq/airbyte/pull/25122) | Add additional data centers                                                                                                                                     |
+| 1.2.19  | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases                                                                                                                              |
+| 1.2.18  | 2023-03-23 | [\#24447](https://github.com/airbytehq/airbyte/pull/24447) | Set the Service Account Key JSON field to always_show: true so that it isn't collapsed into an optional fields section                                          |
+| 1.2.17  | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays                                                                                                      |
+| 1.2.16  | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush                                                                                                                         |
+| 1.2.15  | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long                                                                                                                           |
+| 1.2.14  | 2023-02-08 | [\#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error                                                                                                                                |
+| 1.2.13  | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging                                                                                                        |
+| 1.2.12  | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions                                                                                                                 |
+| 1.2.11  | 2023-01-18 | [\#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue                                                                                                |
+| 1.2.9   | 2022-12-14 | [\#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check                                                                                                  |
+| 1.2.8   | 2022-11-22 | [\#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage                                                                                                    |
+| 1.2.7   | 2022-11-11 | [\#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location                                                                                                         |
+| 1.2.6   | 2022-11-10 | [\#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors                                                                                                           |
+| 1.2.5   | 2022-10-19 | [\#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs                                                                                                                                              |
+| 1.2.4   | 2022-09-26 | [\#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header                                                                                                                                           |
+| 1.2.3   | 2022-09-22 | [\#17054](https://github.com/airbytehq/airbyte/pull/17054) | Respect stream namespace                                                                                                                                        |
+| 1.2.1   | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | (bugged, do not use) Wrap logs in AirbyteLogMessage                                                                                                             |
+| 1.2.0   | 2022-09-09 | [\#14023](https://github.com/airbytehq/airbyte/pull/14023) | (bugged, do not use) Cover arrays only if they are nested                                                                                                       |
+| 1.1.16  | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields)                                        |
+| 1.1.15  | 2022-08-22 | [\#15787](https://github.com/airbytehq/airbyte/pull/15787) | Throw exception if job failed                                                                                                                                   |
+| 1.1.14  | 2022-08-03 | [\#14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials                                                                                                                        |
+| 1.1.13  | 2022-08-02 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings                                                                                                                                       |
+| 1.1.12  | 2022-08-02 | [\#15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode                                                                                                                                       |
+| 1.1.11  | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging                                                                                     |
+| 1.1.10  | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors                                                                                                          |
+| 1.1.9   | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager                                                                         |
+| 1.1.8   | 2022-06-07 | [\#13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys.                                                                                      |
+| 1.1.7   | 2022-06-07 | [\#13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification.                                                                                                                             |
+| 1.1.6   | 2022-05-15 | [\#12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud.                                                                                           |
+| 1.1.5   | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error.                                                                                               |
+| 1.1.4   | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging.                                                                 |
+| 1.1.3   | 2022-05-02 | [\#12528](https://github.com/airbytehq/airbyte/pull/12528) | Update Dataset location field description                                                                                                                       |
+| 1.1.2   | 2022-04-29 | [\#12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field                                                                                                                            |
+| 1.1.1   | 2022-04-15 | [\#12068](https://github.com/airbytehq/airbyte/pull/12068) | Fixed bug with GCS bucket conditional binding                                                                                                                   |
+| 1.1.0   | 2022-04-06 | [\#11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption.                                                                                                 |
+| 1.0.2   | 2022-03-30 | [\#11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec                                                                                                                                                    |
+| 1.0.1   | 2022-03-24 | [\#11350](https://github.com/airbytehq/airbyte/pull/11350) | Improve check performance                                                                                                                                       |
+| 1.0.0   | 2022-03-18 | [\#11238](https://github.com/airbytehq/airbyte/pull/11238) | Updated spec and documentation                                                                                                                                  |
+| 0.6.12  | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters                                                                                                                           |
+| 0.6.11  | 2022-03-03 | [\#10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM                                                                                                                 |
+| 0.6.8   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                                                                    |
+| 0.6.6   | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959)   | Fix null pointer exception from buffered stream consumer.                                                                                                       |
+| 0.6.6   | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745)   | Integrate with Sentry.                                                                                                                                          |
+| 0.6.5   | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573)   | BigQuery Destination : update description for some input fields                                                                                                 |
+| 0.6.4   | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | Support dataset-id prefixed by project-id                                                                                                                       |
+| 0.6.3   | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415)   | BigQuery Destination : Fix GCS processing of Facebook data                                                                                                      |
+| 0.6.2   | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121)   | Fixed check method for GCS mode to verify if all roles assigned to user                                                                                         |
+| 0.6.1   | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039)   | Added part_size configuration to UI for GCS staging                                                                                                             |
+| 0.6.0   | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/issues/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files                                                                     |
+| 0.5.1   | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations                                                                                                                                        |
+| 0.5.0   | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables                                                                                                                             |
+| 0.4.1   | 2021-10-04 | [\#6733](https://github.com/airbytehq/airbyte/issues/6733) | Support dataset starting with numbers                                                                                                                           |
+| 0.4.0   | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added GCS Staging uploading option                                                                                                                              |
+| 0.3.12  | 2021-08-03 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add optional arg to make a possibility to change the BigQuery client's chunk\buffer size                                                                        |
+| 0.3.11  | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125)   | Enable `additionalPropertities` in spec.json                                                                                                                    |
+| 0.3.10  | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId                                                                                               |
+| 0.3.9   | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026)   | Add sanitized json fields in raw tables to handle quotes in column names                                                                                        |
+| 0.3.6   | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional.                                                                                                                   |
+| 0.3.4   | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option                                                                                                                                     |
diff --git a/docs/integrations/destinations/cassandra.md b/docs/integrations/destinations/cassandra.md
index 8ec03e961189..0b78fc28b2bd 100644
--- a/docs/integrations/destinations/cassandra.md
+++ b/docs/integrations/destinations/cassandra.md
@@ -19,12 +19,12 @@ contain the following columns.
 
 ### Features
 
-| Feature                       | Support | Notes                                                                                        |
-| :---------------------------- | :-----: | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             |   ✅    | Warning: this mode deletes all previously synced data in the configured DynamoDB table.      |
-| Incremental - Append Sync     |   ✅    |                                                                                              |
-| Incremental - Deduped History |   ❌    | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    |   ✅    | Namespace will be used as part of the table name.                                            |
+| Feature                        | Support | Notes                                                                                   |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured DynamoDB table. |
+| Incremental - Append Sync      |   ✅    |                                                                                         |
+| Incremental - Append + Deduped |   ❌    |                                                                                         |
+| Namespaces                     |   ✅    | Namespace will be used as part of the table name.                                       |
 
 ### Performance considerations
 
diff --git a/docs/integrations/destinations/chargify.md b/docs/integrations/destinations/chargify.md
index 0bab856d6b30..68fb1c2fdaa3 100644
--- a/docs/integrations/destinations/chargify.md
+++ b/docs/integrations/destinations/chargify.md
@@ -17,12 +17,12 @@ Each replicated stream from Airbyte will output data into a corresponding event
 
 #### Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | No |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | No                   |       |
 
 ## Getting started
 
@@ -32,8 +32,8 @@ To use the Chargify destination, you'll first need to create a [Chargify account
 
 Once you have a Chargify account, you can use the following credentials to set up the connector
 
-* A Project ID associated with the site
-* A Master API key associated with the site
+- A Project ID associated with the site
+- A Master API key associated with the site
 
 You can reach out to [support@chargify.com](mailto:support@chargify.com) to request your Project ID and Master API key for the Airbyte destination connector.
 
@@ -63,9 +63,9 @@ The `Infer Timestamp` field lets you specify if you want the connector to infer
 
 Now, you should have all the parameters needed to configure Chargify destination.
 
-* **Project ID**
-* **Master API Key**
-* **Infer Timestamp**
+- **Project ID**
+- **Master API Key**
+- **Infer Timestamp**
 
 Connect your first source and then head to the Chargify application. You can seamlessly run [custom analysis](https://www.chargify.com/business-intelligence/) on your data and build [multi-attribute, usage-based pricing models](http://chargify.com/events-based-billing/).
 
@@ -73,8 +73,8 @@ If you have any questions or want to get started, [please reach out to a billing
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.2 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
-| 0.2.0 | 2021-09-10 | [\#5973](https://github.com/airbytehq/airbyte/pull/5973) | Fix timestamp inference for complex schemas |
-| 0.1.0 | 2021-08-18 | [\#5339](https://github.com/airbytehq/airbyte/pull/5339) | Chargify Destination Release! |
+| Version | Date       | Pull Request                                             | Subject                                      |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------- |
+| 0.2.2   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
+| 0.2.0   | 2021-09-10 | [\#5973](https://github.com/airbytehq/airbyte/pull/5973) | Fix timestamp inference for complex schemas  |
+| 0.1.0   | 2021-08-18 | [\#5339](https://github.com/airbytehq/airbyte/pull/5339) | Chargify Destination Release!                |
diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md
index 4db25cdc5d7e..75da81407f48 100644
--- a/docs/integrations/destinations/clickhouse.md
+++ b/docs/integrations/destinations/clickhouse.md
@@ -1,22 +1,21 @@
-
 # ClickHouse
 
 ## Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes |
-|:------------------------------|:---------------------|:------|
-| Full Refresh Sync             | Yes                  |       |
-| Incremental - Append Sync     | Yes                  |       |
-| Incremental - Deduped History | Yes                  |       |
-| Namespaces                    | Yes                  |       |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | Yes                  |       |
+| Namespaces                     | Yes                  |       |
 
 #### Output Schema
 
 Each stream will be output into its own table in ClickHouse. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in ClickHouse is `String`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in ClickHouse is `DateTime64`.
-* `_airbyte_data`: a json blob representing with the event data. The column type in ClickHouse is `String`.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in ClickHouse is `String`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in ClickHouse is `DateTime64`.
+- `_airbyte_data`: a json blob representing with the event data. The column type in ClickHouse is `String`.
 
 ## Getting Started \(Airbyte Cloud\)
 
@@ -28,7 +27,7 @@ Airbyte Cloud only supports connecting to your ClickHouse instance with SSL or T
 
 To use the ClickHouse destination, you'll need:
 
-* A ClickHouse server version 21.8.10.19 or above
+- A ClickHouse server version 21.8.10.19 or above
 
 #### Configure Network Access
 
@@ -38,8 +37,8 @@ Make sure your ClickHouse database can be accessed by Airbyte. If your database
 
 You need a ClickHouse user with the following permissions:
 
-* can create tables and write rows.
-* can create databases e.g:
+- can create tables and write rows.
+- can create databases e.g:
 
 You can create such a user by running:
 
@@ -57,40 +56,42 @@ You will need to choose an existing database or create a new database that will
 
 You should now have all the requirements needed to configure ClickHouse as a destination in the UI. You'll need the following information to configure the ClickHouse destination:
 
-* **Host**
-* **Port**
-* **Username**
-* **Password**
-* **Database**
-* **Jdbc_url_params**
+- **Host**
+- **Port**
+- **Username**
+- **Password**
+- **Database**
+- **Jdbc_url_params**
 
 ## Naming Conventions
 
 From [ClickHouse SQL Identifiers syntax](https://clickhouse.com/docs/en/sql-reference/syntax/):
 
-* SQL identifiers and key words must begin with a letter \(a-z, but also letters with diacritical marks and non-Latin letters\) or an underscore \(\_\).
-* Subsequent characters in an identifier or key word can be letters, underscores, digits \(0-9\).
-* Identifiers can be quoted or non-quoted. The latter is preferred.
-* If you want to use identifiers the same as keywords or you want to use other symbols in identifiers, quote it using double quotes or backticks, for example, "id", `id`.
-* If you want to write portable applications you are advised to always quote a particular name or never quote it.
+- SQL identifiers and key words must begin with a letter \(a-z, but also letters with diacritical marks and non-Latin letters\) or an underscore \(\_\).
+- Subsequent characters in an identifier or key word can be letters, underscores, digits \(0-9\).
+- Identifiers can be quoted or non-quoted. The latter is preferred.
+- If you want to use identifiers the same as keywords or you want to use other symbols in identifiers, quote it using double quotes or backticks, for example, "id", `id`.
+- If you want to write portable applications you are advised to always quote a particular name or never quote it.
 
 Therefore, Airbyte ClickHouse destination will create tables and schemas using the Unquoted identifiers when possible or fallback to Quoted Identifiers if the names are containing special characters.
 
 ## Changelog
 
-| Version | Date       | Pull Request                                               | Subject                                                             |
-|:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------------------------------------|
-| 0.2.3   | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                               |
-| 0.2.2   | 2023-02-21 | [\#21509](https://github.com/airbytehq/airbyte/pull/21509) | Compatibility update with security patch for strict encrypt version |
-| 0.2.1   | 2022-12-06 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update dbt version to 1.3.1                                         |
-| 0.2.0   | 2022-09-27 | [\#16970](https://github.com/airbytehq/airbyte/pull/16970) | Remove TCP port from spec parameters                                |
-| 0.1.12  | 2022-09-08 | [\#16444](https://github.com/airbytehq/airbyte/pull/16444) | Added custom jdbc params field                                      |
-| 0.1.10  | 2022-07-05 | [\#13639](https://github.com/airbytehq/airbyte/pull/13639) | Change JDBC ClickHouse version into 0.3.2-patch9                    |
-| 0.1.8   | 2022-07-05 | [\#13516](https://github.com/airbytehq/airbyte/pull/13516) | Added JDBC default parameter socket timeout                         |
-| 0.1.7   | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors              |
-| 0.1.6   | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                              |
-| 0.1.5   | 2022-04-06 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0                                  |
-| 0.1.4   | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                   |
-| 0.1.3   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                        |
-| 0.1.1   | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982)   | Set isSchemaRequired to false                                       |
-| 0.1.0   | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620)   | Add ClickHouse destination                                          |
+| Version | Date       | Pull Request                                               | Subject                                                                                       |
+| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------- |
+| 0.2.5   | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size                                                                             |
+| 0.2.4   | 2023-06-05 | [\#27036](https://github.com/airbytehq/airbyte/pull/27036) | Internal code change for future development (install normalization packages inside connector) |
+| 0.2.3   | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                                                         |
+| 0.2.2   | 2023-02-21 | [\#21509](https://github.com/airbytehq/airbyte/pull/21509) | Compatibility update with security patch for strict encrypt version                           |
+| 0.2.1   | 2022-12-06 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update dbt version to 1.3.1                                                                   |
+| 0.2.0   | 2022-09-27 | [\#16970](https://github.com/airbytehq/airbyte/pull/16970) | Remove TCP port from spec parameters                                                          |
+| 0.1.12  | 2022-09-08 | [\#16444](https://github.com/airbytehq/airbyte/pull/16444) | Added custom jdbc params field                                                                |
+| 0.1.10  | 2022-07-05 | [\#13639](https://github.com/airbytehq/airbyte/pull/13639) | Change JDBC ClickHouse version into 0.3.2-patch9                                              |
+| 0.1.8   | 2022-07-05 | [\#13516](https://github.com/airbytehq/airbyte/pull/13516) | Added JDBC default parameter socket timeout                                                   |
+| 0.1.7   | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors                                        |
+| 0.1.6   | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                        |
+| 0.1.5   | 2022-04-06 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0                                                            |
+| 0.1.4   | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                             |
+| 0.1.3   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                  |
+| 0.1.1   | 2021-12-21 | [\#8982](https://github.com/airbytehq/airbyte/pull/8982)   | Set isSchemaRequired to false                                                                 |
+| 0.1.0   | 2021-11-04 | [\#7620](https://github.com/airbytehq/airbyte/pull/7620)   | Add ClickHouse destination                                                                    |
diff --git a/docs/integrations/destinations/csv.md b/docs/integrations/destinations/csv.md
index 77ffadec8e70..4cc00f440c79 100644
--- a/docs/integrations/destinations/csv.md
+++ b/docs/integrations/destinations/csv.md
@@ -22,18 +22,18 @@ Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a M
 
 Each stream will be output into its own file. Each file will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
-* `_airbyte_data`: a json blob representing with the event data.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_data`: a json blob representing with the event data.
 
 #### Features
 
-| Feature | Supported |  |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | No |  |
+| Feature                        | Supported |     |
+| :----------------------------- | :-------- | :-- |
+| Full Refresh Sync              | Yes       |     |
+| Incremental - Append Sync      | Yes       |     |
+| Incremental - Append + Deduped | No        |     |
+| Namespaces                     | No        |     |
 
 #### Performance considerations
 
@@ -49,13 +49,13 @@ The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local`
 
 ### Example:
 
-* If `destination_path` is set to `/local/cars/models`
-* the local mount is using the `/tmp/airbyte_local` default
-* then all data will be written to `/tmp/airbyte_local/cars/models` directory.
+- If `destination_path` is set to `/local/cars/models`
+- the local mount is using the `/tmp/airbyte_local` default
+- then all data will be written to `/tmp/airbyte_local/cars/models` directory.
 
 ## Access Replicated Data Files
 
-If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
+If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
 
 1. Access the scheduler container using `docker exec -it airbyte-server bash`
 2. Navigate to the default local mount using `cd /tmp/airbyte_local`
@@ -74,7 +74,7 @@ Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                         |
-|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------ |
 | 1.0.0   | 2022-12-20 | [17998](https://github.com/airbytehq/airbyte/pull/17998) | Breaking changes: non backwards compatible. Adds delimiter dropdown.            |
 | 0.2.10  | 2022-06-20 | [13932](https://github.com/airbytehq/airbyte/pull/13932) | Merging published connector changes                                             |
 | 0.2.9   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add ExitOnOutOfMemoryError to java connectors and bump versions                 |
@@ -95,4 +95,3 @@ Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have
 | 0.1.2   | 2020-11-18 | [998](https://github.com/airbytehq/airbyte/pull/998)     | Adding incremental to the data model                                            |
 | 0.1.1   | 2020-11-10 | [895](https://github.com/airbytehq/airbyte/pull/895)     | bump versions: all destinations and source exchange rate                        |
 | 0.1.0   | 2020-10-21 | [676](https://github.com/airbytehq/airbyte/pull/676)     | Integrations Reorganization: Connectors                                         |
-
diff --git a/docs/integrations/destinations/cumulio.md b/docs/integrations/destinations/cumulio.md
index 4b73c2facce9..6cae834e0cd0 100644
--- a/docs/integrations/destinations/cumulio.md
+++ b/docs/integrations/destinations/cumulio.md
@@ -1,14 +1,10 @@
-
-
 # Cumul.io
 
-
 ## General
 
-The Airbyte Cumul.io destination connector allows you to stream data into Cumul.io from [any Airbyte Source](https://airbyte.io/connectors?connector-type=Sources). 
-
-Cumul.io is an **[Embedded analytics SaaS solution](https://cumul.io/product/embedded-analytics)** that enables other SaaS companies to grow with an **engaging customer analytics experience**, seamlessly embedded in their product. Cumul.io's intuitive, low-code interface empowers business users with insight-driven actions in record time **without straining engineering resources from the core product**. 
+The Airbyte Cumul.io destination connector allows you to stream data into Cumul.io from [any Airbyte Source](https://airbyte.io/connectors?connector-type=Sources).
 
+Cumul.io is an **[Embedded analytics SaaS solution](https://cumul.io/product/embedded-analytics)** that enables other SaaS companies to grow with an **engaging customer analytics experience**, seamlessly embedded in their product. Cumul.io's intuitive, low-code interface empowers business users with insight-driven actions in record time **without straining engineering resources from the core product**.
 
 ## Getting started
 
@@ -16,83 +12,86 @@ In order to use the Cumul.io destination, you'll first need to **create a [Cumul
 After logging in to Cumul.io, you can **generate an API key and token** in your [Profile -> API Tokens](https://app.cumul.io/start/profile/integration).
 To set up the destination connector in Airbyte, you'll need to provide the following Cumul.io properties:
 
-* "**Cumul.io API Host URL**": the API host URL for the **Cumul.io environment** where your **Cumul.io account resides** (i.e. `https://api.cumul.io` for EU multi-tenant users, `https://api.us.cumul.io/` for US multi-tenant users, or a VPC-specific address). This property depends on the environment in which your Cumul.io account was created (e.g. if you have signed up via https://app.us.cumul.io/signup, the API host URL would be `https://api.us.cumul.io/`).
-* "**Cumul.io API key**": a Cumul.io API key (see above how to generate an API key-token pair)
-* "**Cumul.io API token**": the corresponding Cumul.io API token (see above how to generate an API key-token pair)
+- "**Cumul.io API Host URL**": the API host URL for the **Cumul.io environment** where your **Cumul.io account resides** (i.e. `https://api.cumul.io` for EU multi-tenant users, `https://api.us.cumul.io/` for US multi-tenant users, or a VPC-specific address). This property depends on the environment in which your Cumul.io account was created (e.g. if you have signed up via https://app.us.cumul.io/signup, the API host URL would be `https://api.us.cumul.io/`).
+- "**Cumul.io API key**": a Cumul.io API key (see above how to generate an API key-token pair)
+- "**Cumul.io API token**": the corresponding Cumul.io API token (see above how to generate an API key-token pair)
 
-As soon as you've connected a source and the **first stream synchronization** has **succeeded**, the desired **Dataset(s)** will be **available in Cumul.io to build dashboards on** (Cumul.io's ["Getting started" Academy course](https://academy.cumul.io/course/a0bf5530-edfb-441e-901b-e1fcb95dfac7) might be interesting to get familiar with its platform). 
+As soon as you've connected a source and the **first stream synchronization** has **succeeded**, the desired **Dataset(s)** will be **available in Cumul.io to build dashboards on** (Cumul.io's ["Getting started" Academy course](https://academy.cumul.io/course/a0bf5530-edfb-441e-901b-e1fcb95dfac7) might be interesting to get familiar with its platform).
 Depending on the **synchronization mode** set up, the **next synchronizations** will either **replace/append data in/to these datasets**!
 
-*If you have any questions or want to get started with Cumul.io, don't hesitate to reach out via [our contact page](https://cumul.io/contact).*
-
+_If you have any questions or want to get started with Cumul.io, don't hesitate to reach out via [our contact page](https://cumul.io/contact)._
 
 ## Connector overview
 
 ### Sync modes support
-|  [Sync modes](https://docs.airbyte.com/understanding-airbyte/connections/#sync-modes) | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append/) | Yes | / |
-| [Full Refresh - Replace](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) | Yes | / |
-| [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append/) | Yes | / |
-| [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) | No | Cumul.io's data warehouse does not support dbt (yet). |
+
+| [Sync modes](https://docs.airbyte.com/understanding-airbyte/connections/#sync-modes)                                     | Supported?\(Yes/No\) | Notes                                                 |
+| :----------------------------------------------------------------------------------------------------------------------- | :------------------- | :---------------------------------------------------- |
+| [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append/)                 | Yes                  | /                                                     |
+| [Full Refresh - Replace](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)             | Yes                  | /                                                     |
+| [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append/)              | Yes                  | /                                                     |
+| [Incremental - Append + Deduped ](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) | No                   | Cumul.io's data warehouse does not support dbt (yet). |
 
 ### Airbyte Features support
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| [Namespaces](https://docs.airbyte.com/understanding-airbyte/namespaces/) | Yes | (***Highly recommended***) A **concatenation of the namespace and stream name** will be used as a unique identifier for the related Cumul.io dataset (using [Tags](https://academy.cumul.io/article/mam7lkdt)) and ensures next synchronizations can target the same dataset. Use this property to **ensure identically named destination streams** from different connections **do not coincide**!|
-| [Reset data](https://docs.airbyte.com/operator-guides/reset) | Yes | **Existing data** in a dataset is **not deleted** upon resetting a stream in Airbyte, however the next synchronization batch will replace all existing data. This ensures that the dataset is never empty (e.g. upon disabling the synchronization), which would otherwise result in "No data" upon querying it.|
+
+| Feature                                                                  | Supported?\(Yes/No\) | Notes                                                                                                                                                                                                                                                                                                                                                                                               |
+| :----------------------------------------------------------------------- | :------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Namespaces](https://docs.airbyte.com/understanding-airbyte/namespaces/) | Yes                  | (**_Highly recommended_**) A **concatenation of the namespace and stream name** will be used as a unique identifier for the related Cumul.io dataset (using [Tags](https://academy.cumul.io/article/mam7lkdt)) and ensures next synchronizations can target the same dataset. Use this property to **ensure identically named destination streams** from different connections **do not coincide**! |
+| [Reset data](https://docs.airbyte.com/operator-guides/reset)             | Yes                  | **Existing data** in a dataset is **not deleted** upon resetting a stream in Airbyte, however the next synchronization batch will replace all existing data. This ensures that the dataset is never empty (e.g. upon disabling the synchronization), which would otherwise result in "No data" upon querying it.                                                                                    |
 
 ### Airbyte data types support
-| [Airbyte data types](https://docs.airbyte.com/understanding-airbyte/supported-data-types#the-types) | Remarks |
-| :--- | :--- |
-| Array & Object | To support a limited amount of insights, this connector  will **stringify data values with type `Array` or `Object`** ([recommended by Airbyte](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#unsupported-types)) as Cumul.io does not support storing nor querying such data types. For analytical purposes, it's always recommended to **unpack these values in different rows or columns** (depending on the use-case) before pushing the data to Cumul.io!|
-| Time with(out) timezone | While these values **will be stored as-is** in Cumul.io, they should be interpreted as `hierarchy`* (i.e. text/string, see [Cumul.io's data types Academy article](https://academy.cumul.io/article/p68253bn)). Alternatively, you could either **provide a (default) date and timezone** for these values, or **unpack them in different columns** (e.g. `hour`, `minute`, `second` columns), before pushing the data to Cumul.io.|
-| Timestamp without timezone | Cumul.io **does not support storing dates without timestamps**, these timestamps will be **interpreted as UTC date values**.|
-| Number & Integer data types with NaN, Infinity, -Infinity values | While these values **will be stored as-is** in Cumul.io, they will not support numeric aggregations such as sum, avg, etc. (*using such aggregations on these values likely causes unexpected behavior*). Ideally, such values are **converted into meaningful values** (e.g. no value, 0, a specific value, etc.) before pushing the data to Cumul.io. |
-| Boolean | Boolean values **will be stringified** ([recommended by Airbyte](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#unsupported-types)) and result in a hierarchy column type (i.e. text/string, see [Cumul.io's data types Academy article](https://academy.cumul.io/article/p68253bn)). You could use Cumul.io's hierarchy translation (see [this Academy article](https://academy.cumul.io/article/dqgn0316)) to assign translations to `true` and `false` that are meaningful to the business user in the column's context. |
-| All other data types | Should be supported and correctly interpreted by Cumul.io's Data API service*. |
 
+| [Airbyte data types](https://docs.airbyte.com/understanding-airbyte/supported-data-types#the-types) | Remarks                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
+| :-------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Array & Object                                                                                      | To support a limited amount of insights, this connector will **stringify data values with type `Array` or `Object`** ([recommended by Airbyte](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#unsupported-types)) as Cumul.io does not support storing nor querying such data types. For analytical purposes, it's always recommended to **unpack these values in different rows or columns** (depending on the use-case) before pushing the data to Cumul.io!                                                              |
+| Time with(out) timezone                                                                             | While these values **will be stored as-is** in Cumul.io, they should be interpreted as `hierarchy`\* (i.e. text/string, see [Cumul.io's data types Academy article](https://academy.cumul.io/article/p68253bn)). Alternatively, you could either **provide a (default) date and timezone** for these values, or **unpack them in different columns** (e.g. `hour`, `minute`, `second` columns), before pushing the data to Cumul.io.                                                                                                              |
+| Timestamp without timezone                                                                          | Cumul.io **does not support storing dates without timestamps**, these timestamps will be **interpreted as UTC date values**.                                                                                                                                                                                                                                                                                                                                                                                                                      |
+| Number & Integer data types with NaN, Infinity, -Infinity values                                    | While these values **will be stored as-is** in Cumul.io, they will not support numeric aggregations such as sum, avg, etc. (_using such aggregations on these values likely causes unexpected behavior_). Ideally, such values are **converted into meaningful values** (e.g. no value, 0, a specific value, etc.) before pushing the data to Cumul.io.                                                                                                                                                                                           |
+| Boolean                                                                                             | Boolean values **will be stringified** ([recommended by Airbyte](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#unsupported-types)) and result in a hierarchy column type (i.e. text/string, see [Cumul.io's data types Academy article](https://academy.cumul.io/article/p68253bn)). You could use Cumul.io's hierarchy translation (see [this Academy article](https://academy.cumul.io/article/dqgn0316)) to assign translations to `true` and `false` that are meaningful to the business user in the column's context. |
+| All other data types                                                                                | Should be supported and correctly interpreted by Cumul.io's Data API service\*.                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 
-**Note: It might be that Cumul.io's automatic typing could initially interpret this type of data wrongly due to its format (see `Possible future improvements` below), you could then alter the column type in the Cumul.io UI to try changing it manually.*
+\*_Note: It might be that Cumul.io's automatic typing could initially interpret this type of data wrongly due to its format (see `Possible future improvements` below), you could then alter the column type in the Cumul.io UI to try changing it manually._
 
 ### Output schema in Cumul.io
+
 Each replicated stream from Airbyte will output data into a corresponding dataset in Cumul.io. Each dataset will **initially** have an **`Airbyte - ` English name** which can be **further adapted in Cumul.io's UI**, or even [via API](https://developer.cumul.io/#dashboard_update). If the request of pushing a batch of data fails, the connector will gracefully retry pushing the batch up to three times, with a backoff interval of 5 minutes, 10 minutes, and 20 minutes, respectively.
 
 The connector will **associate one or more of the following tags to each dataset**:
-* `[AIRBYTE - DO NOT DELETE] - `: this tag will be **used to retrieve the dataset ID and its current columns** from Cumul.io, and will be associated with the dataset after the first batch of data is written to a new dataset.
-* `[AIRBYTE - DO NOT DELETE] - REPLACE DATA`: this tag will be **associated to a dataset** when it should be "resetted" (i.e. the **existing data should be replaced**, see `Feature` -> `Reset data` above). The first batch of data of the next synchronization will replace all existing data if this tag is present on a dataset.
 
-As noted in the tag name, it is important to **never remove such tags from the dataset(s) nor manually set them** on other datasets. Doing so might break existing or new synchronizations!
+- `[AIRBYTE - DO NOT DELETE] - `: this tag will be **used to retrieve the dataset ID and its current columns** from Cumul.io, and will be associated with the dataset after the first batch of data is written to a new dataset.
+- `[AIRBYTE - DO NOT DELETE] - REPLACE DATA`: this tag will be **associated to a dataset** when it should be "resetted" (i.e. the **existing data should be replaced**, see `Feature` -> `Reset data` above). The first batch of data of the next synchronization will replace all existing data if this tag is present on a dataset.
 
+As noted in the tag name, it is important to **never remove such tags from the dataset(s) nor manually set them** on other datasets. Doing so might break existing or new synchronizations!
 
 ## Data recommendations
 
-### Data structure 
+### Data structure
+
 To ensure the most performant queries, we recommend to **denormalize your data as much as possible beforehand** (this ensures that the least amount of joins are required to achieve your desired insights). Denormalized datasets also ensure that they can be easily consumed by less technical users, who often do not understand relations between tables! Instead of denormalizing your datasets to specific insights, it is recommended to **set up one or more dimensional data models** that support all kinds of slicing and dicing within a dashboard: this ensures a **flexible & scalable setup** which is **easy-to-understand and performant-to-query**!
 
 This Cumul.io blog post goes into more detail on why customer-facing analytics requires a simple data model: https://blog.cumul.io/2022/12/07/why-a-dimensional-data-model-for-embedded-analytics/.
 
 ### Pushing data
-Cumul.io uses an **OLAP database** to **ensure the most performant concurrent "Read" queries** on large amounts of data. OLAP databases, such as Cumul.io's database, are however often less suitable for a lot of "Write" queries with small amounts of data.
 
-To ensure the best performance when writing data, we **recommend synchronizing larger amounts of data less frequently** rather than *smaller amounts of data more frequently*!
+Cumul.io uses an **OLAP database** to **ensure the most performant concurrent "Read" queries** on large amounts of data. OLAP databases, such as Cumul.io's database, are however often less suitable for a lot of "Write" queries with small amounts of data.
 
+To ensure the best performance when writing data, we **recommend synchronizing larger amounts of data less frequently** rather than _smaller amounts of data more frequently_!
 
 ## Possible future improvements
-* In case of many concurrent synchronizations, the following issues might arise at one point (not evaluated yet):
-  * The combination of all write buffers' data could cause memory overload, in that case it might be interesting to alter the flush rate by changing the `flush_interval` variable in `destination_cumulio/writer.py` (currently set to 10 000, which is the maximum amount of data points that can be sent via Cumul.io's Data API service in a single request, see note [here](https://developer.cumul.io/#data_create)). We do recommend keeping the `flush_interval` value **as high as possible** to ensure the least amount of total overhead on all batches pushed!
-  * Having more than 200 concurrent Airbyte connections flushing the data simultaneously, and using the same Cumul.io API key and token for each connection, might run into [Cumul.io's API Rate limit](https://developer.cumul.io/#core_api_ratelimiting). As this will rarely occur due to Cumul.io's burstable rate limit, we recommend using separate API key and tokens for identical destination connectors in case you would expect such concurrency. Note that synchronizing multiple streams in a single connection will happen sequentially and thus not run into the rate limit.
-* The current connector will not take into account the Airbyte source data types, instead Cumul.io's API will automatically detect column types based on a random data sample. If Cumul.io's detected data type is not as desired, it's possible to alter the column's type via Cumul.io's UI to manually change the column type (e.g. if a `VARCHAR` column  would only contain numeric values, it could initially be interpreted as a `numeric` column in Cumul.io but can at any point be changed to `hierarchy` if more appropriate). 
-	* As a  future improvement, it is possible to:
-		1. Create a new dataset - [Create Dataset API Documentation](https://developer.cumul.io/#dataset_create)
-		2. Create the appropriate tag (`[AIRBYTE - DO NOT DELETE] - `) and associate it with the newly created dataset (in `destination_cumulio/client.py`, a method `_validate_tag_dataset_id_association(stream_name, dataset_id)` is defined which could be used for this step)
-		3. Create each column with the correct Cumul.io type - [Create Column API Documentation](https://developer.cumul.io/#column_create)
-		4. Associate each column with the dataset - [Associate Dataset Column API Documentation](https://developer.cumul.io/#column_assoc_dataset)
-		5. From there on out, you can replace/append data for this dataset based on the tag (already implemented).
 
+- In case of many concurrent synchronizations, the following issues might arise at one point (not evaluated yet):
+  - The combination of all write buffers' data could cause memory overload, in that case it might be interesting to alter the flush rate by changing the `flush_interval` variable in `destination_cumulio/writer.py` (currently set to 10 000, which is the maximum amount of data points that can be sent via Cumul.io's Data API service in a single request, see note [here](https://developer.cumul.io/#data_create)). We do recommend keeping the `flush_interval` value **as high as possible** to ensure the least amount of total overhead on all batches pushed!
+  - Having more than 200 concurrent Airbyte connections flushing the data simultaneously, and using the same Cumul.io API key and token for each connection, might run into [Cumul.io's API Rate limit](https://developer.cumul.io/#core_api_ratelimiting). As this will rarely occur due to Cumul.io's burstable rate limit, we recommend using separate API key and tokens for identical destination connectors in case you would expect such concurrency. Note that synchronizing multiple streams in a single connection will happen sequentially and thus not run into the rate limit.
+- The current connector will not take into account the Airbyte source data types, instead Cumul.io's API will automatically detect column types based on a random data sample. If Cumul.io's detected data type is not as desired, it's possible to alter the column's type via Cumul.io's UI to manually change the column type (e.g. if a `VARCHAR` column would only contain numeric values, it could initially be interpreted as a `numeric` column in Cumul.io but can at any point be changed to `hierarchy` if more appropriate).
+  - As a future improvement, it is possible to:
+    1. Create a new dataset - [Create Dataset API Documentation](https://developer.cumul.io/#dataset_create)
+    2. Create the appropriate tag (`[AIRBYTE - DO NOT DELETE] - `) and associate it with the newly created dataset (in `destination_cumulio/client.py`, a method `_validate_tag_dataset_id_association(stream_name, dataset_id)` is defined which could be used for this step)
+    3. Create each column with the correct Cumul.io type - [Create Column API Documentation](https://developer.cumul.io/#column_create)
+    4. Associate each column with the dataset - [Associate Dataset Column API Documentation](https://developer.cumul.io/#column_assoc_dataset)
+    5. From there on out, you can replace/append data for this dataset based on the tag (already implemented).
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.0 | 2023-02-16 |  | Initial release of Cumul.io's Destination connector |
+| Version | Date       | Pull Request | Subject                                             |
+| :------ | :--------- | :----------- | :-------------------------------------------------- |
+| 0.1.0   | 2023-02-16 |              | Initial release of Cumul.io's Destination connector |
diff --git a/docs/integrations/destinations/databricks.md b/docs/integrations/destinations/databricks.md
index fd2c7c666d16..c8a6a0516d49 100644
--- a/docs/integrations/destinations/databricks.md
+++ b/docs/integrations/destinations/databricks.md
@@ -11,20 +11,25 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 ## Getting started
 
 ## Databricks AWS Setup
+
 ### 1. Create a Databricks Workspace
+
 - Follow Databricks guide [Create a workspace using the account console](https://docs.databricks.com/administration-guide/workspace/create-workspace.html#create-a-workspace-using-the-account-console).
-> **_IMPORTANT:_** Don't forget to create a [cross-account IAM role](https://docs.databricks.com/administration-guide/cloud-configurations/aws/iam-role.html#create-a-cross-account-iam-role) for workspaces
+  > **_IMPORTANT:_** Don't forget to create a [cross-account IAM role](https://docs.databricks.com/administration-guide/cloud-configurations/aws/iam-role.html#create-a-cross-account-iam-role) for workspaces
 
 > **_TIP:_** Alternatively use Databricks quickstart for new workspace
 > ![](../../.gitbook/assets/destination/databricks/databricks_workspace_quciksetup.png)
 
 ### 2. Create a metastore and attach it to workspace
+
 > **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier.
 
 #### Setup storage bucket and IAM role in AWS
- Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws) to setup AWS bucket with necessary permissions.
+
+Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws) to setup AWS bucket with necessary permissions.
 
 #### Create metastore
+
 - Login into Databricks [account console](https://accounts.cloud.databricks.com/login) with admin permissions.
 - Go to Data tab and hit Create metastore button:
 
@@ -32,14 +37,17 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 
 - Provide all necessary data and click Create:
 
-    ![](../../.gitbook/assets/destination/databricks/databrikcs_metastore_fields.png)
+  ![](../../.gitbook/assets/destination/databricks/databrikcs_metastore_fields.png)
+
   - `Name`
   - `Region` The metastore should be in same region as the workspace.
   - `S3 bucket path` created at [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step.
   - `IAM role ARN` created at [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. Example: `arn:aws:iam:::role/`
+
 - Select the workspaces in `Assign to workspaces` tab and click Assign.
 
 ### 3. Create Databricks SQL Warehouse
+
 > **_TIP:_** If you use Databricks cluster skip this step
 
 - Open the workspace tab and click on created workspace console:
@@ -55,6 +63,7 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 - After SQL warehouse was created we can it's Connection details to con
 
 ### 4. Databricks SQL Warehouse connection details
+
 > **_TIP:_** If you use Databricks cluster skip this step
 
 - Open workspace console.
@@ -69,6 +78,7 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 > **_IMPORTANT:_** `Server hostname`, `Port`, `HTTP path` are used for Airbyte connection
 
 ### 5. Create Databricks Cluster
+
 > **_TIP:_** If you use Databricks SQL Warehouse skip this step
 
 - Open the workspace tab and click on created workspace console:
@@ -79,11 +89,12 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 
   ![](../../.gitbook/assets/destination/databricks/databrick_new_cluster.png)
 
-    - Switch to Data science & Engineering
-    - Click New button
-    - Choose Cluster
+  - Switch to Data science & Engineering
+  - Click New button
+  - Choose Cluster
 
 ### 6. Databricks Cluster connection details
+
 > **_TIP:_** If you use Databricks SQL Warehouse skip this step
 
 - Open workspace console.
@@ -94,9 +105,11 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 - Open Advanced options under Configuration, choose JDBC/ODBC tab:
 
   ![](../../.gitbook/assets/destination/databricks/databricks_cluster_connection_details2.png)
-> **_IMPORTANT:_** `Server hostname`, `Port`, `HTTP path` are used for Airbyte connection
+
+  > **_IMPORTANT:_** `Server hostname`, `Port`, `HTTP path` are used for Airbyte connection
 
 ### 7. Create Databricks Token
+
 - Open workspace console.
 - Open User Settings, go to Access tokens tab and click Generate new token:
 
@@ -109,6 +122,7 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 > **_TIP:_** `Lifetime` can be set to `0`
 
 ### 8. Adding External Locations (Optional)
+
 > **_TIP:_** Skip this step if no external data source is used.
 
 - Open workspace console.
@@ -123,18 +137,22 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing
 > **_TIP:_** The new `Storage credential` can be added in the `Storage Credentials` tab or use same as for Metastore.
 
 ## Airbyte Setup
+
 ### Databricks fields
-- `Agree to the Databricks JDBC Driver Terms & Conditions`  - [Databricks JDBC ODBC driver license](https://www.databricks.com/legal/jdbc-odbc-driver-license).
-- `Server Hostname` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details)  or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps.
-- `HTTP Path` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details)  or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps.
-- `Port` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details)  or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps.
+
+- `Agree to the Databricks JDBC Driver Terms & Conditions` - [Databricks JDBC ODBC driver license](https://www.databricks.com/legal/jdbc-odbc-driver-license).
+- `Server Hostname` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps.
+- `HTTP Path` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps.
+- `Port` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps.
 - `Access Token` - can be taken from [7. Create Databricks Token](#7-create-databricks-token) step.
 
 ### Data Source
+
 You could choose a data source type
- - Managed tables
- - Amazon S3 (External storage)
- - Azure Blob Storage (External storage)
+
+- Managed tables
+- Amazon S3 (External storage)
+- Azure Blob Storage (External storage)
 
 #### Managed tables data source type
 
@@ -143,13 +161,15 @@ Please check Databricks documentation about [What is managed tables](https://doc
 > **_TIP:_** There is no addition setup should be done for this type.
 
 #### Amazon S3 data source type (External storage)
+
 > **_IMPORTANT:_** Make sure the `External Locations` has been added to the workspace. Check [Adding External Locations](#8-adding-external-locations-optional) step.
 
 Provide your Amazon S3 data:
+
 - `S3 Bucket Name` - The bucket name
 - `S3 Bucket Path` - Subdirectory under the above bucket to sync the data into
 - `S3 Bucket Region` - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.
-> **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier.
+  > **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier.
 - `S3 Access Key ID` - Corresponding key to the above key id
 - `S3 Secret Access Key` -
   - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
@@ -157,44 +177,47 @@ Provide your Amazon S3 data:
 - `S3 Filename pattern` - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't be recognized
 
 #### Azure Blob Storage data source type (External storage)
+
 > **_IMPORTANT:_** The work in progress.
 
 ## Sync Mode
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. |
-| Incremental - Append Sync | ✅ |  |
-| Incremental - Deduped History | ❌ |  |
-| Namespaces | ✅ |  |
+| Feature                        | Support | Notes                                                                                |
+| :----------------------------- | :-----: | :----------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path. |
+| Incremental - Append Sync      |   ✅    |                                                                                      |
+| Incremental - Append + Deduped |   ❌    |                                                                                      |
+| Namespaces                     |   ✅    |                                                                                      |
 
 ## Configuration
 
-| Category            | Parameter               |  Type   | Notes                                                                                                                                                                                                                                                                                                                                                       |
-|:--------------------|:------------------------|:-------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Databricks          | Server Hostname         | string  | Required. Example: `abc-12345678-wxyz.cloud.databricks.com`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). Please note that this is the server for the Databricks Cluster. It is different from the SQL Endpoint Cluster.                                             |
-|                     | HTTP Path               | string  | Required. Example: `sql/protocolvx/o/1234567489/0000-1111111-abcd90`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url).                                                                                                                                                   |
-|                     | Port                    | string  | Optional. Default to "443". See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url).                                                                                                                                                                                             |
-|                     | Personal Access Token   | string  | Required. Example: `dapi0123456789abcdefghij0123456789AB`. See [documentation](https://docs.databricks.com/sql/user/security/personal-access-tokens.html).                                                                                                                                                                                                  |
-| General             | Databricks catalog      | string  | Optional. The name of the catalog. If not specified otherwise, the "hive_metastore" will be used.                                                                                                                                                                                                                                                           |
-|                     | Database schema         | string  | Optional. The default schema tables are written. If not specified otherwise, the "default" will be used.                                                                                                                                                                                                                                                    |
-|                     | Purge Staging Data      | boolean | The connector creates staging files and tables on S3 or Azure. By default, they will be purged when the data sync is complete. Set it to `false` for debugging purposes.                                                                                                                                                                                    |
-| Data Source - S3    | Bucket Name             | string  | Name of the bucket to sync data into.                                                                                                                                                                                                                                                                                                                       |
-|                     | Bucket Path             | string  | Subdirectory under the above bucket to sync the data into.                                                                                                                                                                                                                                                                                                  |
-|                     | Region                  | string  | See [documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.                                                                                                                                                                                             |
-|                     | Access Key ID           | string  | AWS/Minio credential.                                                                                                                                                                                                                                                                                                                                       |
-|                     | Secret Access Key       | string  | AWS/Minio credential.                                                                                                                                                                                                                                                                                                                                       |
-|                     | S3 Filename pattern     | string  | The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. |
-| Data Source - Azure | Account Name            | string  | Name of the account to sync data into.                                                                                                                                                                                                                                                                                                                      |
-|                     | Container Name          | string  | Container under the above account to sync the data into.                                                                                                                                                                                                                                                                                                    |
-|                     | SAS token               | string  | Shared-access signature token for the above account.                                                                                                                                                                                                                                                                                                        |
-|                     | Endpoint domain name    | string  | Usually blob.core.windows.net.                                                                                                                                                                                                                                                                                                                              |
+| Category            | Parameter             |  Type   | Notes                                                                                                                                                                                                                                                                                                                                                       |
+| :------------------ | :-------------------- | :-----: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Databricks          | Server Hostname       | string  | Required. Example: `abc-12345678-wxyz.cloud.databricks.com`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). Please note that this is the server for the Databricks Cluster. It is different from the SQL Endpoint Cluster.                                             |
+|                     | HTTP Path             | string  | Required. Example: `sql/protocolvx/o/1234567489/0000-1111111-abcd90`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url).                                                                                                                                                   |
+|                     | Port                  | string  | Optional. Default to "443". See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url).                                                                                                                                                                                             |
+|                     | Personal Access Token | string  | Required. Example: `dapi0123456789abcdefghij0123456789AB`. See [documentation](https://docs.databricks.com/sql/user/security/personal-access-tokens.html).                                                                                                                                                                                                  |
+| General             | Databricks catalog    | string  | Optional. The name of the catalog. If not specified otherwise, the "hive_metastore" will be used.                                                                                                                                                                                                                                                           |
+|                     | Database schema       | string  | Optional. The default schema tables are written. If not specified otherwise, the "default" will be used.                                                                                                                                                                                                                                                    |
+|                     | Schema evolution      | boolean | Optional. The connector enables automatic schema evolution in the destination tables.                                                                                                                                                                                                                                                                       |
+|                     | Purge Staging Data    | boolean | The connector creates staging files and tables on S3 or Azure. By default, they will be purged when the data sync is complete. Set it to `false` for debugging purposes.                                                                                                                                                                                    |
+| Data Source - S3    | Bucket Name           | string  | Name of the bucket to sync data into.                                                                                                                                                                                                                                                                                                                       |
+|                     | Bucket Path           | string  | Subdirectory under the above bucket to sync the data into.                                                                                                                                                                                                                                                                                                  |
+|                     | Region                | string  | See [documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.                                                                                                                                                                                             |
+|                     | Access Key ID         | string  | AWS/Minio credential.                                                                                                                                                                                                                                                                                                                                       |
+|                     | Secret Access Key     | string  | AWS/Minio credential.                                                                                                                                                                                                                                                                                                                                       |
+|                     | S3 Filename pattern   | string  | The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. |
+| Data Source - Azure | Account Name          | string  | Name of the account to sync data into.                                                                                                                                                                                                                                                                                                                      |
+|                     | Container Name        | string  | Container under the above account to sync the data into.                                                                                                                                                                                                                                                                                                    |
+|                     | SAS token             | string  | Shared-access signature token for the above account.                                                                                                                                                                                                                                                                                                        |
+|                     | Endpoint domain name  | string  | Usually blob.core.windows.net.                                                                                                                                                                                                                                                                                                                              |
 
 ⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you provision a dedicated S3 or Azure resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️
 
 ## Staging Files (Delta Format)
 
 ### S3
+
 Data streams are first written as staging delta-table ([Parquet](https://parquet.apache.org/) + [Transaction Log](https://databricks.com/blog/2019/08/21/diving-into-delta-lake-unpacking-the-transaction-log.html)) files on S3, and then loaded into Databricks delta-tables. All the staging files will be deleted after the sync is done. For debugging purposes, here is the full path for a staging file:
 
 ```text
@@ -214,6 +237,7 @@ s3://testing_bucket/data_output_path/98c450be-5b1c-422d-b8b5-6ca9903727d9/users/
 ```
 
 ### Azure
+
 Similarly, streams are first written to a staging location, but the Azure option uses CSV format. A staging table is created from the CSV files.
 
 ## Unmanaged Spark SQL Table
@@ -240,43 +264,46 @@ In Azure, the full path of each data stream is:
 ```text
 abfss://@.dfs.core.windows.net//
 ```
+
 Please keep these data directories on S3/Azure. Otherwise, the corresponding tables will have no data in Databricks.
 
 ## Output Schema
 
 Each table will have the following columns:
 
-| Column | Type | Notes |
-| :--- | :---: | :--- |
-| `_airbyte_ab_id` | string | UUID. |
-| `_airbyte_emitted_at` | timestamp | Data emission timestamp. |
-| Data fields from the source stream | various | All fields in the staging files will be expanded in the table. |
+| Column                             |   Type    | Notes                                                          |
+| :--------------------------------- | :-------: | :------------------------------------------------------------- |
+| `_airbyte_ab_id`                   |  string   | UUID.                                                          |
+| `_airbyte_emitted_at`              | timestamp | Data emission timestamp.                                       |
+| Data fields from the source stream |  various  | All fields in the staging files will be expanded in the table. |
 
 Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the Json object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the Json to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion).
 
 ## Related tutorial
+
 Suppose you are interested in learning more about the Databricks connector or details on how the Delta Lake tables are created. You may want to consult the tutorial on [How to Load Data into Delta Lake on Databricks Lakehouse](https://airbyte.com/tutorials/load-data-into-delta-lake-on-databricks-lakehouse).
 
 ## CHANGELOG
 
-| Version | Date       | Pull Request                                                                                                                                                             | Subject                                                                                                                  |
-|:--------|:-----------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------|
-| 1.0.2   | 2023-04-20 | [\#25366](https://github.com/airbytehq/airbyte/pull/25366)                                                                                                               | Fix default catalog to be `hive_metastore`                                                                               |
-| 1.0.1   | 2023-03-30 | [\#24657](https://github.com/airbytehq/airbyte/pull/24657)                                                                                                               | Fix support for external tables on S3                                                                                    |
-| 1.0.0   | 2023-03-21 | [\#23965](https://github.com/airbytehq/airbyte/pull/23965)                                                                                                               | Added: Managed table storage type, Databricks Catalog field                                                              |
-| 0.3.1   | 2022-10-15 | [\#18032](https://github.com/airbytehq/airbyte/pull/18032)                                                                                                               | Add `SSL=1` to the JDBC URL to ensure SSL connection.                                                                    |
-| 0.3.0   | 2022-10-14 | [\#15329](https://github.com/airbytehq/airbyte/pull/15329)                                                                                                               | Add support for Azure storage.                                                                                           |
-|         | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243)                                                                                                               | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |
-| 0.2.6   | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801)                                                                                                               | Fix multiply log bindings                                                                                                |
-| 0.2.5   | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494)                                                                                                               | Make S3 output filename configurable.                                                                                    |
-| 0.2.4   | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618)                                                                                                               | Removed additionalProperties: false from JDBC destination connectors                                                     |
-| 0.2.3   | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852)                                                                                                               | Updated stacktrace format for any trace message errors                                                                   |
-| 0.2.2   | 2022-06-13 | [\#13722](https://github.com/airbytehq/airbyte/pull/13722)                                                                                                               | Rename to "Databricks Lakehouse".                                                                                        |
-| 0.2.1   | 2022-06-08 | [\#13630](https://github.com/airbytehq/airbyte/pull/13630)                                                                                                               | Rename to "Databricks Delta Lake" and add field orders in the spec.                                                      |
-| 0.2.0   | 2022-05-15 | [\#12861](https://github.com/airbytehq/airbyte/pull/12861)                                                                                                               | Use new public Databricks JDBC driver, and open source the connector.                                                    |
-| 0.1.5   | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578)                                                                                                               | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging.                          |
-| 0.1.4   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256)                                                                                                               | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                             |
-| 0.1.3   | 2022-01-06 | [\#7622](https://github.com/airbytehq/airbyte/pull/7622) [\#9153](https://github.com/airbytehq/airbyte/issues/9153)                                                      | Upgrade Spark JDBC driver to `2.6.21` to patch Log4j vulnerability; update connector fields title/description.           |
-| 0.1.2   | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288)                                                                                                               | Support Json `additionalProperties`.                                                                                     |
-| 0.1.1   | 2021-10-05 | [\#6792](https://github.com/airbytehq/airbyte/pull/6792)                                                                                                                 | Require users to accept Databricks JDBC Driver [Terms & Conditions](https://databricks.com/jdbc-odbc-driver-license).    |
-| 0.1.0   | 2021-09-14 | [\#5998](https://github.com/airbytehq/airbyte/pull/5998)                                                                                                                 | Initial private release.                                                                                                 |
+| Version | Date       | Pull Request                                                                                                        | Subject                                                                                                                  |
+| :------ | :--------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------- |
+| 1.1.0   | 2023-06-02 | [\#26942](https://github.com/airbytehq/airbyte/pull/26942)                                                          | Support schema evolution                                                                                                 |
+| 1.0.2   | 2023-04-20 | [\#25366](https://github.com/airbytehq/airbyte/pull/25366)                                                          | Fix default catalog to be `hive_metastore`                                                                               |
+| 1.0.1   | 2023-03-30 | [\#24657](https://github.com/airbytehq/airbyte/pull/24657)                                                          | Fix support for external tables on S3                                                                                    |
+| 1.0.0   | 2023-03-21 | [\#23965](https://github.com/airbytehq/airbyte/pull/23965)                                                          | Added: Managed table storage type, Databricks Catalog field                                                              |
+| 0.3.1   | 2022-10-15 | [\#18032](https://github.com/airbytehq/airbyte/pull/18032)                                                          | Add `SSL=1` to the JDBC URL to ensure SSL connection.                                                                    |
+| 0.3.0   | 2022-10-14 | [\#15329](https://github.com/airbytehq/airbyte/pull/15329)                                                          | Add support for Azure storage.                                                                                           |
+|         | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243)                                                          | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |
+| 0.2.6   | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801)                                                          | Fix multiply log bindings                                                                                                |
+| 0.2.5   | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494)                                                          | Make S3 output filename configurable.                                                                                    |
+| 0.2.4   | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618)                                                          | Removed additionalProperties: false from JDBC destination connectors                                                     |
+| 0.2.3   | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852)                                                          | Updated stacktrace format for any trace message errors                                                                   |
+| 0.2.2   | 2022-06-13 | [\#13722](https://github.com/airbytehq/airbyte/pull/13722)                                                          | Rename to "Databricks Lakehouse".                                                                                        |
+| 0.2.1   | 2022-06-08 | [\#13630](https://github.com/airbytehq/airbyte/pull/13630)                                                          | Rename to "Databricks Delta Lake" and add field orders in the spec.                                                      |
+| 0.2.0   | 2022-05-15 | [\#12861](https://github.com/airbytehq/airbyte/pull/12861)                                                          | Use new public Databricks JDBC driver, and open source the connector.                                                    |
+| 0.1.5   | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578)                                                          | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging.                          |
+| 0.1.4   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256)                                                          | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                             |
+| 0.1.3   | 2022-01-06 | [\#7622](https://github.com/airbytehq/airbyte/pull/7622) [\#9153](https://github.com/airbytehq/airbyte/issues/9153) | Upgrade Spark JDBC driver to `2.6.21` to patch Log4j vulnerability; update connector fields title/description.           |
+| 0.1.2   | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288)                                                          | Support Json `additionalProperties`.                                                                                     |
+| 0.1.1   | 2021-10-05 | [\#6792](https://github.com/airbytehq/airbyte/pull/6792)                                                            | Require users to accept Databricks JDBC Driver [Terms & Conditions](https://databricks.com/jdbc-odbc-driver-license).    |
+| 0.1.0   | 2021-09-14 | [\#5998](https://github.com/airbytehq/airbyte/pull/5998)                                                            | Initial private release.                                                                                                 |
diff --git a/docs/integrations/destinations/doris.md b/docs/integrations/destinations/doris.md
index 0f0554e88b9e..91b0e1176f0a 100644
--- a/docs/integrations/destinations/doris.md
+++ b/docs/integrations/destinations/doris.md
@@ -20,7 +20,7 @@ This section should contain a table with the following format:
 | :------------------------------------- | :----------------- | :----------------------- |
 | Full Refresh Sync                      | Yes                |                          |
 | Incremental - Append Sync              | Yes                |                          |
-| Incremental - Deduped History          | No                 | it will soon be realized |
+| Incremental - Append + Deduped         | No                 | it will soon be realized |
 | For databases, WAL/Logical replication | Yes                |                          |
 
 ### Performance considerations
diff --git a/docs/integrations/destinations/duckdb.md b/docs/integrations/destinations/duckdb.md
index 345576e31cf1..352975308ec4 100644
--- a/docs/integrations/destinations/duckdb.md
+++ b/docs/integrations/destinations/duckdb.md
@@ -1,5 +1,4 @@
-
-# DuckDB 
+# DuckDB
 
 :::danger
 
@@ -21,18 +20,18 @@ If you set [Normalization](https://docs.airbyte.com/understanding-airbyte/basic-
 
 Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
-* `_airbyte_data`: a json blob representing with the event data.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_data`: a json blob representing with the event data.
 
 #### Features
 
-| Feature | Supported |  |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No |  |
-| Namespaces | No |  |
+| Feature                        | Supported |     |
+| :----------------------------- | :-------- | :-- |
+| Full Refresh Sync              | Yes       |     |
+| Incremental - Append Sync      | Yes       |     |
+| Incremental - Append + Deduped | No        |     |
+| Namespaces                     | No        |     |
 
 #### Performance consideration
 
@@ -52,16 +51,15 @@ Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a M
 
 :::
 
-
 ### Example:
 
-* If `destination_path` is set to `/local/destination.duckdb`
-* the local mount is using the `/tmp/airbyte_local` default
-* then all data will be written to `/tmp/airbyte_local/destination.duckdb`.
+- If `destination_path` is set to `/local/destination.duckdb`
+- the local mount is using the `/tmp/airbyte_local` default
+- then all data will be written to `/tmp/airbyte_local/destination.duckdb`.
 
 ## Access Replicated Data Files
 
-If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
+If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
 
 1. Access the scheduler container using `docker exec -it airbyte-server bash`
 2. Navigate to the default local mount using `cd /tmp/airbyte_local`
@@ -78,7 +76,6 @@ Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.0 | 2022-10-14 | [17494](https://github.com/airbytehq/airbyte/pull/17494) | New DuckDB destination |
-
+| Version | Date       | Pull Request                                             | Subject                |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------- |
+| 0.1.0   | 2022-10-14 | [17494](https://github.com/airbytehq/airbyte/pull/17494) | New DuckDB destination |
diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md
index a0fbccc87b0b..08000797ba53 100644
--- a/docs/integrations/destinations/dynamodb.md
+++ b/docs/integrations/destinations/dynamodb.md
@@ -5,6 +5,7 @@ This destination writes data to AWS DynamoDB.
 The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each stream is written to its own table under the DynamoDB.
 
 ## Prerequisites
+
 - For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your DynamoDB connector to version `0.1.5` or newer
 
 ## Sync overview
@@ -13,19 +14,19 @@ The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each s
 
 Each stream will be output into its own DynamoDB table. Each table will a collections of `json` objects containing 4 fields:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
-* `_airbyte_data`: a json blob representing with the extracted data.
-* `sync_time`: a timestamp representing when the sync up task be triggered.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_data`: a json blob representing with the extracted data.
+- `sync_time`: a timestamp representing when the sync up task be triggered.
 
 ### Features
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured DynamoDB table. |
-| Incremental - Append Sync | ✅ |  |
-| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | ✅ | Namespace will be used as part of the table name. |
+| Feature                        | Support | Notes                                                                                   |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured DynamoDB table. |
+| Incremental - Append Sync      |   ✅    |                                                                                         |
+| Incremental - Append + Deduped |   ❌    |                                                                                         |
+| Namespaces                     |   ✅    | Namespace will be used as part of the table name.                                       |
 
 ### Performance considerations
 
@@ -40,33 +41,32 @@ This connector by default uses 10 capacity units for both Read and Write in Dyna
 
 ### Setup guide
 
-* Fill up DynamoDB info
-  * **DynamoDB Endpoint**
-    * Leave empty if using AWS DynamoDB, fill in endpoint URL if using customized endpoint.
-  * **DynamoDB Table Name**
-    * The name prefix of the DynamoDB table to store the extracted data. The table name is \\_\\_\.
-  * **DynamoDB Region**
-    * The region of the DynamoDB.
-  * **Access Key Id**
-    * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
-    * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_dynamodb_specific-table.html) to the DynamoDB table.
-  * **Secret Access Key**
-    * Corresponding key to the above key id.
-* Make sure your DynamoDB tables are accessible from the machine running Airbyte.
-  * This depends on your networking setup.
-  * You can check AWS DynamoDB documentation with a tutorial on how to properly configure your DynamoDB's access [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/access-control-overview.html).
-  * The easiest way to verify if Airbyte is able to connect to your DynamoDB tables is via the check connection tool in the UI.
+- Fill up DynamoDB info
+  - **DynamoDB Endpoint**
+    - Leave empty if using AWS DynamoDB, fill in endpoint URL if using customized endpoint.
+  - **DynamoDB Table Name**
+    - The name prefix of the DynamoDB table to store the extracted data. The table name is \\_\\_\.
+  - **DynamoDB Region**
+    - The region of the DynamoDB.
+  - **Access Key Id**
+    - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
+    - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_dynamodb_specific-table.html) to the DynamoDB table.
+  - **Secret Access Key**
+    - Corresponding key to the above key id.
+- Make sure your DynamoDB tables are accessible from the machine running Airbyte.
+  - This depends on your networking setup.
+  - You can check AWS DynamoDB documentation with a tutorial on how to properly configure your DynamoDB's access [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/access-control-overview.html).
+  - The easiest way to verify if Airbyte is able to connect to your DynamoDB tables is via the check connection tool in the UI.
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.7 | 2022-11-03 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Added strict-encrypt cloud runner |
-| 0.1.6 | 2022-11-01 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Enforce to use ssl connection |
-| 0.1.5 | 2022-08-05 | [\#15350](https://github.com/airbytehq/airbyte/pull/15350) | Added per-stream handling |
-| 0.1.4 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors |
-| 0.1.3 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
-| 0.1.2 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
-| 0.1.1 | 2022-12-05 | [\#9314](https://github.com/airbytehq/airbyte/pull/9314) | Rename dynamo_db_table_name to dynamo_db_table_name_prefix. |
-| 0.1.0 | 2021-08-20 | [\#5561](https://github.com/airbytehq/airbyte/pull/5561) | Initial release. |
-
+| Version | Date       | Pull Request                                               | Subject                                                     |
+| :------ | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------- |
+| 0.1.7   | 2022-11-03 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Added strict-encrypt cloud runner                           |
+| 0.1.6   | 2022-11-01 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Enforce to use ssl connection                               |
+| 0.1.5   | 2022-08-05 | [\#15350](https://github.com/airbytehq/airbyte/pull/15350) | Added per-stream handling                                   |
+| 0.1.4   | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors      |
+| 0.1.3   | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820)   | Improved 'check' operation performance                      |
+| 0.1.2   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256)   | Add `-XX:+ExitOnOutOfMemoryError` JVM option                |
+| 0.1.1   | 2022-12-05 | [\#9314](https://github.com/airbytehq/airbyte/pull/9314)   | Rename dynamo_db_table_name to dynamo_db_table_name_prefix. |
+| 0.1.0   | 2021-08-20 | [\#5561](https://github.com/airbytehq/airbyte/pull/5561)   | Initial release.                                            |
diff --git a/docs/integrations/destinations/exasol.md b/docs/integrations/destinations/exasol.md
index 9c030a8a0bc2..179ff93359cb 100644
--- a/docs/integrations/destinations/exasol.md
+++ b/docs/integrations/destinations/exasol.md
@@ -16,15 +16,15 @@ Each Airbyte Stream becomes an Exasol table and each Airbyte Field becomes an Ex
 
 The Exasol destination supports the following features:
 
-| Feature                       | Supported? (Yes/No) | Notes |
-| :---------------------------- | :------------------ | :---- |
-| Full Refresh Sync             | Yes                 |       |
-| Incremental - Append Sync     | Yes                 |       |
-| Incremental - Deduped History | No                  |       |
-| Normalization                 | No                  |       |
-| Namespaces                    | Yes                 |       |
-| SSL connection                | Yes                 | TLS   |
-| SSH Tunnel Support            | No                  |       |
+| Feature                        | Supported? (Yes/No) | Notes |
+| :----------------------------- | :------------------ | :---- |
+| Full Refresh Sync              | Yes                 |       |
+| Incremental - Append Sync      | Yes                 |       |
+| Incremental - Append + Deduped | No                  |       |
+| Normalization                  | No                  |       |
+| Namespaces                     | Yes                 |       |
+| SSL connection                 | Yes                 | TLS   |
+| SSH Tunnel Support             | No                  |       |
 
 ### Limitations
 
diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md
index 2fe6dea591d4..df8405a3448d 100644
--- a/docs/integrations/destinations/gcs.md
+++ b/docs/integrations/destinations/gcs.md
@@ -6,32 +6,28 @@ This destination writes data to GCS bucket.
 
 The Airbyte GCS destination allows you to sync data to cloud storage buckets. Each stream is written to its own directory under the bucket.
 
-:::info
-Cloud storage may incur egress costs. Egress refers to data that is transferred out of the cloud storage system, such as when you download files or access them from a different location. For more information, see the [Google Cloud Storage pricing guide](https://cloud.google.com/storage/pricing).
-:::
-
 ### Sync overview
 
 #### Features
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. |
-| Incremental - Append Sync | ✅ |  |
-| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. |
+| Feature                        | Support | Notes                                                                                                                                                                                               |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path.                                                                                                                |
+| Incremental - Append Sync      |   ✅    | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/understanding-airbyte/connections/incremental-append#inclusive-cursors) |
+| Incremental - Append + Deduped |   ❌    |                                                                                                                                                                                                     |
+| Namespaces                     |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.                                                                                                                         |
 
 ## Configuration
 
-| Parameter | Type | Notes |
-| :--- | :---: | :--- |
-| GCS Bucket Name | string | Name of the bucket to sync data into. |
-| GCS Bucket Path | string | Subdirectory under the above bucket to sync the data into. |
-| GCS Region | string | See [here](https://cloud.google.com/storage/docs/locations) for all region codes. |
-| HMAC Key Access ID | string | HMAC key access ID . The access ID for the GCS bucket. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long. See [HMAC key](https://cloud.google.com/storage/docs/authentication/hmackeys) for details. |
-| HMAC Key Secret | string | The corresponding secret for the access ID. It is a 40-character base-64 encoded string. |
-| Format | object | Format specific configuration. See below [for details](https://docs.airbyte.com/integrations/destinations/gcs#output-schema). |
-| Part Size | integer | Arg to configure a block size. Max allowed blocks by GCS = 10,000, i.e. max stream size = blockSize \* 10,000 blocks. |
+| Parameter          |  Type   | Notes                                                                                                                                                                                                                                                                       |
+| :----------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| GCS Bucket Name    | string  | Name of the bucket to sync data into.                                                                                                                                                                                                                                       |
+| GCS Bucket Path    | string  | Subdirectory under the above bucket to sync the data into.                                                                                                                                                                                                                  |
+| GCS Region         | string  | See [here](https://cloud.google.com/storage/docs/locations) for all region codes.                                                                                                                                                                                           |
+| HMAC Key Access ID | string  | HMAC key access ID . The access ID for the GCS bucket. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long. See [HMAC key](https://cloud.google.com/storage/docs/authentication/hmackeys) for details. |
+| HMAC Key Secret    | string  | The corresponding secret for the access ID. It is a 40-character base-64 encoded string.                                                                                                                                                                                    |
+| Format             | object  | Format specific configuration. See below [for details](https://docs.airbyte.com/integrations/destinations/gcs#output-schema).                                                                                                                                               |
+| Part Size          | integer | Arg to configure a block size. Max allowed blocks by GCS = 10,000, i.e. max stream size = blockSize \* 10,000 blocks.                                                                                                                                                       |
 
 Currently, only the [HMAC key](https://cloud.google.com/storage/docs/authentication/hmackeys) is supported. More credential types will be added in the future, please [submit an issue](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement%2C+needs-triage&template=feature-request.md&title=) with your request.
 
@@ -70,8 +66,8 @@ A data sync may create multiple files as the output files can be partitioned by
 
 Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world.
 
-* Under Full Refresh Sync mode, old output files will be purged before new files are created.
-* Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
+- Under Full Refresh Sync mode, old output files will be purged before new files are created.
+- Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
 
 ### Avro
 
@@ -81,28 +77,28 @@ Each stream will be outputted to its dedicated directory according to the config
 
 Here is the available compression codecs:
 
-* No compression
-* `deflate`
-  * Compression level
-    * Range `[0, 9]`. Default to 0.
-    * Level 0: no compression & fastest.
-    * Level 9: best compression & slowest.
-* `bzip2`
-* `xz`
-  * Compression level
-    * Range `[0, 9]`. Default to 6.
-    * Level 0-3 are fast with medium compression.
-    * Level 4-6 are fairly slow with high compression.
-    * Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively.
-* `zstandard`
-  * Compression level
-    * Range `[-5, 22]`. Default to 3.
-    * Negative levels are 'fast' modes akin to `lz4` or `snappy`.
-    * Levels above 9 are generally for archival purposes.
-    * Levels above 18 use a lot of memory.
-  * Include checksum
-    * If set to `true`, a checksum will be included in each data block.
-* `snappy`
+- No compression
+- `deflate`
+  - Compression level
+    - Range `[0, 9]`. Default to 0.
+    - Level 0: no compression & fastest.
+    - Level 9: best compression & slowest.
+- `bzip2`
+- `xz`
+  - Compression level
+    - Range `[0, 9]`. Default to 6.
+    - Level 0-3 are fast with medium compression.
+    - Level 4-6 are fairly slow with high compression.
+    - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively.
+- `zstandard`
+  - Compression level
+    - Range `[-5, 22]`. Default to 3.
+    - Negative levels are 'fast' modes akin to `lz4` or `snappy`.
+    - Levels above 9 are generally for archival purposes.
+    - Levels above 18 use a lot of memory.
+  - Include checksum
+    - If set to `true`, a checksum will be included in each data block.
+- `snappy`
 
 #### Data schema
 
@@ -112,12 +108,12 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A
 
 Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, an emission timestamp, and the data blob. With the CSV output, it is possible to normalize \(flatten\) the data blob to multiple columns.
 
-| Column | Condition | Description |
-| :--- | :--- | :--- |
-| `_airbyte_ab_id` | Always exists | A uuid assigned by Airbyte to each processed record. |
-| `_airbyte_emitted_at` | Always exists. | A timestamp representing when the event was pulled from the data source. |
-| `_airbyte_data` | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |  |
-| root level fields | When root level normalization \(flattening\) is selected, the root level fields are expanded. |  |
+| Column                | Condition                                                                                         | Description                                                              |
+| :-------------------- | :------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- |
+| `_airbyte_ab_id`      | Always exists                                                                                     | A uuid assigned by Airbyte to each processed record.                     |
+| `_airbyte_emitted_at` | Always exists.                                                                                    | A timestamp representing when the event was pulled from the data source. |
+| `_airbyte_data`       | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |                                                                          |
+| root level fields     | When root level normalization \(flattening\) is selected, the root level fields are expanded.     |                                                                          |
 
 For example, given the following json object from a source:
 
@@ -133,15 +129,15 @@ For example, given the following json object from a source:
 
 With no normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `_airbyte_data` |
-| :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `_airbyte_data`                                                |
+| :------------------------------------- | :-------------------- | :------------------------------------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
 
 With root level normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `user_id` | `name` |
-| :--- | :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | 123 | `{ "first": "John", "last": "Doe" }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `user_id` | `name`                               |
+| :------------------------------------- | :-------------------- | :-------- | :----------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | 123       | `{ "first": "John", "last": "Doe" }` |
 
 Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.csv.gz`).
 
@@ -193,14 +189,14 @@ Output files can be compressed. The default option is GZIP compression. If compr
 
 The following configuration is available to configure the Parquet output:
 
-| Parameter | Type | Default | Description |
-| :--- | :---: | :---: | :--- |
-| `compression_codec` | enum | `UNCOMPRESSED` | **Compression algorithm**. Available candidates are: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `BROTLI`, `LZ4`, and `ZSTD`. |
-| `block_size_mb` | integer | 128 \(MB\) | **Block size \(row group size\)** in MB. This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. |
-| `max_padding_size_mb` | integer | 8 \(MB\) | **Max padding size** in MB. This is the maximum size allowed as padding to align row groups. This is also the minimum size of a row group. |
-| `page_size_kb` | integer | 1024 \(KB\) | **Page size** in KB. The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. |
-| `dictionary_page_size_kb` | integer | 1024 \(KB\) | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. |
-| `dictionary_encoding` | boolean | `true` | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on. |
+| Parameter                 |  Type   |    Default     | Description                                                                                                                                                                                                                       |
+| :------------------------ | :-----: | :------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `compression_codec`       |  enum   | `UNCOMPRESSED` | **Compression algorithm**. Available candidates are: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `BROTLI`, `LZ4`, and `ZSTD`.                                                                                                        |
+| `block_size_mb`           | integer |   128 \(MB\)   | **Block size \(row group size\)** in MB. This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. |
+| `max_padding_size_mb`     | integer |    8 \(MB\)    | **Max padding size** in MB. This is the maximum size allowed as padding to align row groups. This is also the minimum size of a row group.                                                                                        |
+| `page_size_kb`            | integer |  1024 \(KB\)   | **Page size** in KB. The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. |
+| `dictionary_page_size_kb` | integer |  1024 \(KB\)   | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary.                                     |
+| `dictionary_encoding`     | boolean |     `true`     | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on.                                                                                                                                        |
 
 These parameters are related to the `ParquetOutputFormat`. See the [Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) for more details. Also see [Parquet documentation](https://parquet.apache.org/docs/file-format/configurations) for their recommended configurations \(512 - 1024 MB block size, 8 KB page size\).
 
@@ -217,13 +213,13 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A
 
 ### Setup guide
 
-* Fill up GCS info
-  * **GCS Bucket Name**
-    * See [this](https://cloud.google.com/storage/docs/creating-buckets) for instructions on how to create a GCS bucket. The bucket cannot have a retention policy. Set Protection Tools to none or Object versioning.
-  * **GCS Bucket Region**
-  * **HMAC Key Access ID**
-    * See [this](https://cloud.google.com/storage/docs/authentication/managing-hmackeys) on how to generate an access key. For more information on hmac keys please reference the [GCP docs](https://cloud.google.com/storage/docs/authentication/hmackeys)
-    * We recommend creating an Airbyte-specific user or service account. This user or account will require the following permissions for the bucket:
+- Fill up GCS info
+  - **GCS Bucket Name**
+    - See [this](https://cloud.google.com/storage/docs/creating-buckets) for instructions on how to create a GCS bucket. The bucket cannot have a retention policy. Set Protection Tools to none or Object versioning.
+  - **GCS Bucket Region**
+  - **HMAC Key Access ID**
+    - See [this](https://cloud.google.com/storage/docs/authentication/managing-hmackeys) on how to generate an access key. For more information on hmac keys please reference the [GCP docs](https://cloud.google.com/storage/docs/authentication/hmackeys)
+    - We recommend creating an Airbyte-specific user or service account. This user or account will require the following permissions for the bucket:
       ```
       storage.multipartUploads.abort
       storage.multipartUploads.create
@@ -233,43 +229,48 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A
       storage.objects.list
       ```
       You can set those by going to the permissions tab in the GCS bucket and adding the appropriate the email address of the service account or user and adding the aforementioned permissions.
-  * **Secret Access Key**
-    * Corresponding key to the above access ID.
-* Make sure your GCS bucket is accessible from the machine running Airbyte. This depends on your networking setup. The easiest way to verify if Airbyte is able to connect to your GCS bucket is via the check connection tool in the UI.
+  - **Secret Access Key**
+    - Corresponding key to the above access ID.
+- Make sure your GCS bucket is accessible from the machine running Airbyte. This depends on your networking setup. The easiest way to verify if Airbyte is able to connect to your GCS bucket is via the check connection tool in the UI.
 
 ## CHANGELOG
 
-| Version | Date | Pull Request                                                | Subject |
-|:--------| :--- |:------------------------------------------------------------| :--- |
-| 0.3.0   | 2023-04-28 | [#25570](https://github.com/airbytehq/airbyte/pull/25570)   | Fix: all integer schemas should be converted to Avro longs                                                                                           |
-| 0.2.17  | 2023-04-27 | [#25346](https://github.com/airbytehq/airbyte/pull/25346)   | Internal code cleanup                                                                                                                                |
-| 0.2.16  | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788)   | S3-Parquet: added handler to process null values in arrays                                                                                           |
-| 0.2.15  | 2023-03-10 | [#23466](https://github.com/airbytehq/airbyte/pull/23466)   | Changed S3 Avro type from Int to Long                                                                                                                |
-| 0.2.14  | 2023-11-23 | [\#21682](https://github.com/airbytehq/airbyte/pull/21682)  | Add support for buckets with Customer-Managed Encryption Key                                                               |
-| 0.2.13  | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087)   | Wrap Authentication Errors as Config Exceptions                                                                          |
-| 0.2.12  | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901)  | Fix logging to GCS |
-| 0.2.11  | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243)  | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) |
-| 0.2.10  | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801)  | Fix multiple log bindings |
-| 0.2.9   | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114)  | Remove "additionalProperties": false from specs for connectors with staging  |
-| 0.2.8   | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753)  | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager  |
-| 0.2.7   | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483)  | Added support for int, long, float data types to Avro/Parquet formats. |
-| 0.2.6   | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820)    | Improved 'check' operation performance |
-| 0.2.5   | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578)  | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. |
-| 0.2.4   | 2022-04-22 | [\#12167](https://github.com/airbytehq/airbyte/pull/12167)  | Add gzip compression option for CSV and JSONL formats. |
-| 0.2.3   | 2022-04-22 | [\#11795](https://github.com/airbytehq/airbyte/pull/11795)  | Fix the connection check to verify the provided bucket path. |
-| 0.2.2   | 2022-04-05 | [\#11728](https://github.com/airbytehq/airbyte/pull/11728)  | Properly clean-up bucket when running OVERWRITE sync mode |
-| 0.2.1   | 2022-04-05 | [\#11499](https://github.com/airbytehq/airbyte/pull/11499)  | Updated spec and documentation. |
-| 0.2.0   | 2022-04-04 | [\#11686](https://github.com/airbytehq/airbyte/pull/11686)  | Use serialized buffering strategy to reduce memory consumption; compress CSV and JSONL formats. |
-| 0.1.22  | 2022-02-12 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256)  | Add JVM flag to exist on OOME. |
-| 0.1.21  | 2022-02-12 | [\#10299](https://github.com/airbytehq/airbyte/pull/10299)  | Fix connection check to require only the necessary permissions. |
-| 0.1.20  | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367)    | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string. |
-| 0.1.19  | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121)    | Fixed check method for GCS mode to verify if all roles assigned to user |
-| 0.1.18  | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809)    | Update connector fields title/description |
-| 0.1.17  | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574)    | Added namespace to Avro and Parquet record types |
-| 0.1.16  | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974)    | Release a new version to ensure there is no excessive logging. |
-| 0.1.15  | 2021-12-03 | [\#8386](https://github.com/airbytehq/airbyte/pull/8386)    | Add new GCP regions |
-| 0.1.14  | 2021-12-01 | [\#7732](https://github.com/airbytehq/airbyte/pull/7732)    | Support timestamp in Avro and Parquet |
-| 0.1.13  | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288)  | Support Json `additionalProperties`. |
-| 0.1.2   | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720)  | Added configurable block size for stream. Each stream is limited to 10,000 by GCS |
-| 0.1.1   | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296)  | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) |
-| 0.1.0   | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784)    | Initial release.                                                                                                           |
+| Version | Date       | Pull Request                                               | Subject                                                                                                                    |
+| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- |
+| 0.4.4   | 2023-07-14 | [#28345](https://github.com/airbytehq/airbyte/pull/28345)  | Increment patch to trigger a rebuild                                                                                       |
+| 0.4.3   | 2023-07-05 | [#27936](https://github.com/airbytehq/airbyte/pull/27936)  | Internal code update                                                                                                       |
+| 0.4.2   | 2023-06-30 | [#27891](https://github.com/airbytehq/airbyte/pull/27891)  | Internal code update                                                                                                       |
+| 0.4.1   | 2023-06-28 | [#27268](https://github.com/airbytehq/airbyte/pull/27268)  | Internal code update                                                                                                       |
+| 0.4.0   | 2023-06-26 | [#27725](https://github.com/airbytehq/airbyte/pull/27725)  | License Update: Elv2                                                                                                       |
+| 0.3.0   | 2023-04-28 | [#25570](https://github.com/airbytehq/airbyte/pull/25570)  | Fix: all integer schemas should be converted to Avro longs                                                                 |
+| 0.2.17  | 2023-04-27 | [#25346](https://github.com/airbytehq/airbyte/pull/25346)  | Internal code cleanup                                                                                                      |
+| 0.2.16  | 2023-03-17 | [#23788](https://github.com/airbytehq/airbyte/pull/23788)  | S3-Parquet: added handler to process null values in arrays                                                                 |
+| 0.2.15  | 2023-03-10 | [#23466](https://github.com/airbytehq/airbyte/pull/23466)  | Changed S3 Avro type from Int to Long                                                                                      |
+| 0.2.14  | 2023-11-23 | [\#21682](https://github.com/airbytehq/airbyte/pull/21682) | Add support for buckets with Customer-Managed Encryption Key                                                               |
+| 0.2.13  | 2023-01-18 | [#21087](https://github.com/airbytehq/airbyte/pull/21087)  | Wrap Authentication Errors as Config Exceptions                                                                            |
+| 0.2.12  | 2022-10-18 | [\#17901](https://github.com/airbytehq/airbyte/pull/17901) | Fix logging to GCS                                                                                                         |
+| 0.2.11  | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields)   |
+| 0.2.10  | 2022-08-05 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings                                                                                                  |
+| 0.2.9   | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging                                                |
+| 0.2.8   | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager                                    |
+| 0.2.7   | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats.                                                     |
+| 0.2.6   | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820)   | Improved 'check' operation performance                                                                                     |
+| 0.2.5   | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging.                            |
+| 0.2.4   | 2022-04-22 | [\#12167](https://github.com/airbytehq/airbyte/pull/12167) | Add gzip compression option for CSV and JSONL formats.                                                                     |
+| 0.2.3   | 2022-04-22 | [\#11795](https://github.com/airbytehq/airbyte/pull/11795) | Fix the connection check to verify the provided bucket path.                                                               |
+| 0.2.2   | 2022-04-05 | [\#11728](https://github.com/airbytehq/airbyte/pull/11728) | Properly clean-up bucket when running OVERWRITE sync mode                                                                  |
+| 0.2.1   | 2022-04-05 | [\#11499](https://github.com/airbytehq/airbyte/pull/11499) | Updated spec and documentation.                                                                                            |
+| 0.2.0   | 2022-04-04 | [\#11686](https://github.com/airbytehq/airbyte/pull/11686) | Use serialized buffering strategy to reduce memory consumption; compress CSV and JSONL formats.                            |
+| 0.1.22  | 2022-02-12 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add JVM flag to exist on OOME.                                                                                             |
+| 0.1.21  | 2022-02-12 | [\#10299](https://github.com/airbytehq/airbyte/pull/10299) | Fix connection check to require only the necessary permissions.                                                            |
+| 0.1.20  | 2022-01-11 | [\#9367](https://github.com/airbytehq/airbyte/pull/9367)   | Avro & Parquet: support array field with unknown item type; default any improperly typed field to string.                  |
+| 0.1.19  | 2022-01-10 | [\#9121](https://github.com/airbytehq/airbyte/pull/9121)   | Fixed check method for GCS mode to verify if all roles assigned to user                                                    |
+| 0.1.18  | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809)   | Update connector fields title/description                                                                                  |
+| 0.1.17  | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574)   | Added namespace to Avro and Parquet record types                                                                           |
+| 0.1.16  | 2021-12-20 | [\#8974](https://github.com/airbytehq/airbyte/pull/8974)   | Release a new version to ensure there is no excessive logging.                                                             |
+| 0.1.15  | 2021-12-03 | [\#8386](https://github.com/airbytehq/airbyte/pull/8386)   | Add new GCP regions                                                                                                        |
+| 0.1.14  | 2021-12-01 | [\#7732](https://github.com/airbytehq/airbyte/pull/7732)   | Support timestamp in Avro and Parquet                                                                                      |
+| 0.1.13  | 2021-11-03 | [\#7288](https://github.com/airbytehq/airbyte/issues/7288) | Support Json `additionalProperties`.                                                                                       |
+| 0.1.2   | 2021-09-12 | [\#5720](https://github.com/airbytehq/airbyte/issues/5720) | Added configurable block size for stream. Each stream is limited to 10,000 by GCS                                          |
+| 0.1.1   | 2021-08-26 | [\#5296](https://github.com/airbytehq/airbyte/issues/5296) | Added storing gcsCsvFileLocation property for CSV format. This is used by destination-bigquery \(GCS Staging upload type\) |
+| 0.1.0   | 2021-07-16 | [\#4329](https://github.com/airbytehq/airbyte/pull/4784)   | Initial release.                                                                                                           |
diff --git a/docs/integrations/destinations/google-sheets.md b/docs/integrations/destinations/google-sheets.md
index f1824be795a1..14c148a957e7 100644
--- a/docs/integrations/destinations/google-sheets.md
+++ b/docs/integrations/destinations/google-sheets.md
@@ -128,8 +128,11 @@ You cannot create more than 200 worksheets within single spreadsheet.
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                             |
-| ------- | ---------- | -------------------------------------------------------- | ----------------------------------- |
-| 0.1.2   | 2022-10-31 | [18729](https://github.com/airbytehq/airbyte/pull/18729) | Fix empty headers list              |
-| 0.1.1   | 2022-06-15 | [14751](https://github.com/airbytehq/airbyte/pull/14751) | Yield state only when records saved |
-| 0.1.0   | 2022-04-26 | [12135](https://github.com/airbytehq/airbyte/pull/12135) | Initial Release                     |
+| Version | Date       | Pull Request                                             | Subject                                        |
+|---------|------------|----------------------------------------------------------|------------------------------------------------|
+| 0.2.2   | 2023-07-06 | [28035](https://github.com/airbytehq/airbyte/pull/28035) | Migrate from authSpecification to advancedAuth |
+| 0.2.1   | 2023-06-26 | [27782](https://github.com/airbytehq/airbyte/pull/27782) | Only allow HTTPS urls                          |
+| 0.2.0   | 2023-06-26 | [27780](https://github.com/airbytehq/airbyte/pull/27780) | License Update: Elv2                           |
+| 0.1.2   | 2022-10-31 | [18729](https://github.com/airbytehq/airbyte/pull/18729) | Fix empty headers list                         |
+| 0.1.1   | 2022-06-15 | [14751](https://github.com/airbytehq/airbyte/pull/14751) | Yield state only when records saved            |
+| 0.1.0   | 2022-04-26 | [12135](https://github.com/airbytehq/airbyte/pull/12135) | Initial Release                                |
diff --git a/docs/integrations/destinations/iceberg.md b/docs/integrations/destinations/iceberg.md
index 75062620b371..6b48df61743d 100644
--- a/docs/integrations/destinations/iceberg.md
+++ b/docs/integrations/destinations/iceberg.md
@@ -50,11 +50,17 @@ specify the target size of compacted Iceberg data file.
     rename. For `HadoopCatalog`, this connector use **Storage Config** (S3 or HDFS) to manage Iceberg tables.
   - [JdbcCatalog](https://iceberg.apache.org/docs/latest/jdbc/) uses a table in a relational database to manage
     Iceberg tables through JDBC. So far, this connector supports **PostgreSQL** only.
+  - [RESTCatalog](https://iceberg.apache.org/docs/latest/spark-configuration/#catalog-configuration) connects to a REST
+    server, which manages Iceberg tables.
 - **Storage medium** means where Iceberg data files storages in. So far, this connector supports **S3/S3N/S3N**
-  object-storage only.
+  object-storage. When using the RESTCatalog, it is possible to have storage be managed by the server.
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject        |
-| :------ | :--------- | :------------------------------------------------------- | :------------- |
+|:--------|:-----------| :------------------------------------------------------- | :------------- |
+| 0.1.4   | 2023-07-20 | [28506](https://github.com/airbytehq/airbyte/pull/28506) | Support server-managed storage config |
+| 0.1.3   | 2023-07-12 | [28158](https://github.com/airbytehq/airbyte/pull/28158) | Bump Iceberg library to 1.3.0 and add REST catalog support |
+| 0.1.2   | 2023-07-14 | [28345](https://github.com/airbytehq/airbyte/pull/28345) | Trigger rebuild of image |
+| 0.1.1   | 2023-02-27 | [23201](https://github.com/airbytehq/airbyte/pull/23301) | Bump Iceberg library to 1.1.0 |
 | 0.1.0   | 2022-11-01 | [18836](https://github.com/airbytehq/airbyte/pull/18836) | Initial Commit |
diff --git a/docs/integrations/destinations/kafka.md b/docs/integrations/destinations/kafka.md
index 1d9ac1d8bde2..6415dc1cf9b4 100644
--- a/docs/integrations/destinations/kafka.md
+++ b/docs/integrations/destinations/kafka.md
@@ -25,12 +25,12 @@ Each record will contain in its key the uuid assigned by Airbyte, and in the val
 
 #### Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes                                                                                        |
-| :---------------------------- | :------------------- | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             | No                   |                                                                                              |
-| Incremental - Append Sync     | Yes                  |                                                                                              |
-| Incremental - Deduped History | No                   | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    | Yes                  |                                                                                              |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | No                   |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ## Getting started
 
diff --git a/docs/integrations/destinations/keen.md b/docs/integrations/destinations/keen.md
index 37c33f75ca80..dc8db2b25877 100644
--- a/docs/integrations/destinations/keen.md
+++ b/docs/integrations/destinations/keen.md
@@ -21,12 +21,12 @@ Each replicated stream from Airbyte will output data into a corresponding event
 
 #### Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes                                                                                        |
-| :---------------------------- | :------------------- | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             | Yes                  |                                                                                              |
-| Incremental - Append Sync     | Yes                  |                                                                                              |
-| Incremental - Deduped History | No                   | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    | No                   |                                                                                              |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | No                   |       |
 
 ## Getting started
 
diff --git a/docs/integrations/destinations/kinesis.md b/docs/integrations/destinations/kinesis.md
index 21f288f573a9..be39fa228a52 100644
--- a/docs/integrations/destinations/kinesis.md
+++ b/docs/integrations/destinations/kinesis.md
@@ -17,12 +17,12 @@ This connector maps an incoming data from a namespace and stream to a unique Kin
 
 ### Features
 
-| Feature                       | Support | Notes                                                                             |
-| :---------------------------- | :-----: | :-------------------------------------------------------------------------------- |
-| Full Refresh Sync             |   ❌    |                                                                                   |
-| Incremental - Append Sync     |   ✅    | Incoming messages are streamed/appended to a Kinesis stream as they are received. |
-| Incremental - Deduped History |   ❌    |                                                                                   |
-| Namespaces                    |   ✅    | Namespaces will be used to determine the Kinesis stream name.                     |
+| Feature                        | Support | Notes                                                                             |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ❌    |                                                                                   |
+| Incremental - Append Sync      |   ✅    | Incoming messages are streamed/appended to a Kinesis stream as they are received. |
+| Incremental - Append + Deduped |   ❌    |                                                                                   |
+| Namespaces                     |   ✅    | Namespaces will be used to determine the Kinesis stream name.                     |
 
 ### Performance considerations
 
diff --git a/docs/integrations/destinations/langchain.md b/docs/integrations/destinations/langchain.md
new file mode 100644
index 000000000000..8f3c9a8625d5
--- /dev/null
+++ b/docs/integrations/destinations/langchain.md
@@ -0,0 +1,149 @@
+# Vector Database (powered by LangChain)
+
+
+## Overview
+
+This destination prepares data to be used by [Langchain](https://langchain.com/) to retrieve relevant context for question answering use cases.
+
+There are three parts to this:
+* Processing - split up individual records in chunks so they will fit the context window and decide which fields to use as context and which are supplementary metadata.
+* Embedding - convert the text into a vector representation using a pre-trained model (currently only OpenAI `text-embedding-ada-002` is supported)
+* Indexing - store the vectors in a vector database for similarity search
+
+### Processing
+
+Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. The meta fields are stored as-is along with the embedded text chunks. Please note that meta data fields can only be used for filtering and not for retrieval and have to be of type string, number, boolean (all other values are ignored). Depending on the chosen vector store, additional limitations might apply.
+
+When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.
+
+The chunk length is measured in tokens produced by the `tiktoken` library. The maximum is 8191 tokens, which is the maximum length supported by the `text-embedding-ada-002` model.
+
+The stream name gets added as a metadata field `_airbyte_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_natural_id` metadata field.
+
+### Embedding
+
+THe OpenAI embedding API is used to calculate embeddings - see [OpenAI API](https://beta.openai.com/docs/api-reference/text-embedding) for details. To do so, an OpenAI API key is required.
+
+This integration will be constrained by the [speed of the OpenAI embedding API](https://platform.openai.com/docs/guides/rate-limits/overview).
+
+For testing purposes, it's also possible to use the [Fake embeddings](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) integration. It will generate random embeddings and is suitable to test a data pipeline without incurring embedding costs.
+
+### Indexing
+
+#### Pinecone vector store
+
+For production use, use the pinecone vector store. Use the Pinecone web UI or API to create a project and an index before running the destination. All streams will be indexed into the same index, the `_airbyte_stream` metadata field is used to distinguish between streams. Overall, the size of the metadata fields is limited to 30KB per document. Both OpenAI and Fake embeddings are produced with 1536 vector dimensions, make sure to configure the index accordingly.
+
+To initialize a langchain QA chain based on the indexed data, use the following code (set the open API key and pinecone key and environment as `OPENAI_API_KEY`, `PINECONE_KEY` and `PINECONE_ENV` env variables):
+
+```python
+from langchain import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+from langchain.vectorstores import Pinecone
+from langchain.embeddings import OpenAIEmbeddings
+import pinecone
+import os
+
+embeddings = OpenAIEmbeddings()
+pinecone.init(api_key=os.environ["PINECONE_KEY"], environment=os.environ["PINECONE_ENV"])
+index = pinecone.Index("")
+vector_store = Pinecone(index, embeddings.embed_query, "text")
+
+qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever())
+```
+
+:::caution
+
+For Pinecone pods of type starter, only up to 10,000 chunks can be indexed. For production use, please use a higher tier.
+
+:::
+
+#### Chroma vector store
+
+The [Chroma vector store](https://trychroma.com) is running the Chroma embedding database as persistent client and stores the vectors in a local file.
+
+The `destination_path` has to start with `/local`. Any directory nesting within local will be mapped onto the local mount.
+
+By default, the `LOCAL_ROOT` env variable in the `.env` file is set `/tmp/airbyte_local`.
+
+The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local` is substituted by `/tmp/airbyte_local` by default.
+
+To initialize a langchain QA chain based on the indexed data, use the following code (set the openai API key as `OPENAI_API_KEY` env variable):
+
+```python
+from langchain import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+from langchain.vectorstores import Chroma
+from langchain.embeddings import OpenAIEmbeddings
+
+embeddings = OpenAIEmbeddings()
+vector_store = Chroma(embedding_function=embeddings, persist_directory="/tmp/airbyte_local/")
+
+qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever())
+```
+
+:::caution
+
+Chroma is meant to be used on a local workstation and won't work on Kubernetes.
+
+Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a MacOS, as /tmp has a symlink that points to /private. It will not work otherwise). You allow it with "File sharing" in `Settings -> Resources -> File sharing -> add the one or two above folder` and hit the "Apply & restart" button.
+
+:::
+
+
+#### DocArrayHnswSearch vector store
+
+For local testing, the [DocArrayHnswSearch](https://python.langchain.com/docs/modules/data_connection/vectorstores/integrations/docarray_hnsw) is recommended - it stores the vectors in a local file with a sqlite database for metadata. It is not suitable for production use, but it is the easiest to set up for testing and development purposes.
+
+The `destination_path` has to start with `/local`. Any directory nesting within local will be mapped onto the local mount.
+
+By default, the `LOCAL_ROOT` env variable in the `.env` file is set `/tmp/airbyte_local`.
+
+The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local` is substituted by `/tmp/airbyte_local` by default.
+
+DocArrayHnswSearch does not support incremental sync, so the destination will always do a full refresh sync.
+
+To initialize a langchain QA chain based on the indexed data, use the following code (set the openai API key as `OPENAI_API_KEY` env variable):
+
+```python
+from langchain import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.llms import OpenAI
+from langchain.vectorstores import DocArrayHnswSearch
+from langchain.embeddings import OpenAIEmbeddings
+
+embeddings = OpenAIEmbeddings()
+vector_store = DocArrayHnswSearch.from_params(embeddings, "/tmp/airbyte_local/", 1536)
+
+qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever())
+```
+
+:::danger
+
+This destination will delete all existing files in the configured directory on each. Make sure to not use a directory that contains other files.
+
+:::
+
+:::caution
+
+DocArrayHnswSearch is meant to be used on a local workstation and won't work on Kubernetes.
+
+Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a MacOS, as /tmp has a symlink that points to /private. It will not work otherwise). You allow it with "File sharing" in `Settings -> Resources -> File sharing -> add the one or two above folder` and hit the "Apply & restart" button.
+
+:::
+
+
+## CHANGELOG
+
+| Version | Date       | Pull Request                                                  | Subject                                                                                                                                              |
+|:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.0.8   | 2023-08-21 | [#29515](https://github.com/airbytehq/airbyte/pull/29515)     | Clean up generated schema spec  |
+| 0.0.7   | 2023-08-18 | [#29513](https://github.com/airbytehq/airbyte/pull/29513)     | Fix for starter pods  |
+| 0.0.6   | 2023-08-02 | [#28977](https://github.com/airbytehq/airbyte/pull/28977)     | Validate pinecone index dimensions during check  |
+| 0.0.5   | 2023-07-25 | [#28605](https://github.com/airbytehq/airbyte/pull/28605)     | Add Chroma support  |
+| 0.0.4   | 2023-07-21 | [#28556](https://github.com/airbytehq/airbyte/pull/28556)     | Correctly dedupe records with composite and nested primary keys  |
+| 0.0.3   | 2023-07-20 | [#28509](https://github.com/airbytehq/airbyte/pull/28509)     | Change the base image to python:3.9-slim to fix build  |
+| 0.0.2   | 2023-07-18 | [#26184](https://github.com/airbytehq/airbyte/pull/28398)     | Adjust python dependencies and release on cloud  |
+| 0.0.1   | 2023-07-12 | [#26184](https://github.com/airbytehq/airbyte/pull/26184)     | Initial release  |
diff --git a/docs/integrations/destinations/local-json.md b/docs/integrations/destinations/local-json.md
index 4e8081bd3ec1..11870a8d5177 100644
--- a/docs/integrations/destinations/local-json.md
+++ b/docs/integrations/destinations/local-json.md
@@ -16,18 +16,18 @@ This destination writes data to a directory on the _local_ filesystem on the hos
 
 Each stream will be output into its own file. Each file will a collections of `json` objects containing 3 fields:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
-* `_airbyte_data`: a json blob representing with the extracted data.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_data`: a json blob representing with the extracted data.
 
 #### Features
 
-| Feature | Supported |  |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | No |  |
+| Feature                        | Supported |     |
+| :----------------------------- | :-------- | :-- |
+| Full Refresh Sync              | Yes       |     |
+| Incremental - Append Sync      | Yes       |     |
+| Incremental - Append + Deduped | No        |     |
+| Namespaces                     | No        |     |
 
 #### Performance considerations
 
@@ -47,16 +47,15 @@ Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a M
 
 :::
 
-
 ### Example:
 
-* If `destination_path` is set to `/local/cars/models`
-* the local mount is using the `/tmp/airbyte_local` default
-* then all data will be written to `/tmp/airbyte_local/cars/models` directory.
+- If `destination_path` is set to `/local/cars/models`
+- the local mount is using the `/tmp/airbyte_local` default
+- then all data will be written to `/tmp/airbyte_local/cars/models` directory.
 
 ## Access Replicated Data Files
 
-If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
+If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
 
 1. Access the scheduler container using `docker exec -it airbyte-server bash`
 2. Navigate to the default local mount using `cd /tmp/airbyte_local`
@@ -74,6 +73,6 @@ Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.11 | 2022-02-14 | [14641](https://github.com/airbytehq/airbyte/pull/14641) | Include lifecycle management |
+| Version | Date       | Pull Request                                             | Subject                      |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------- |
+| 0.2.11  | 2022-02-14 | [14641](https://github.com/airbytehq/airbyte/pull/14641) | Include lifecycle management |
diff --git a/docs/integrations/destinations/meilisearch.md b/docs/integrations/destinations/meilisearch.md
index c8da4eba7652..d7f40201b775 100644
--- a/docs/integrations/destinations/meilisearch.md
+++ b/docs/integrations/destinations/meilisearch.md
@@ -12,18 +12,18 @@ Each stream will be output into its own index in MeiliSearch. Each table will be
 
 #### Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | No |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | No                   |       |
 
 ## Getting started
 
 ### Requirements
 
-To use the MeiliSearch destination, you'll need an existing MeiliSearch instance. You can learn about how to create one in the [MeiliSearch docs](https://docs.meilisearch.com/reference/features/installation.html#download-and-launch).
+To use the MeiliSearch destination, you'll need an existing MeiliSearch instance. You can learn about how to create one in the [MeiliSearch docs](https://www.meilisearch.com/docs/learn/getting_started/installation).
 
 ### Setup guide
 
@@ -31,9 +31,9 @@ The setup only requires two fields. First is the `host` which is the address at
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 1.0.0 | 2022-10-26 | [18036](https://github.com/airbytehq/airbyte/pull/18036) |  Migrate MeiliSearch to Python CDK |
-| 0.2.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
-| 0.2.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
-| 0.2.11 | 2021-12-28 | [9156](https://github.com/airbytehq/airbyte/pull/9156) | Update connector fields title/description |
+| Version | Date       | Pull Request                                             | Subject                                                |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------- |
+| 1.0.0   | 2022-10-26 | [18036](https://github.com/airbytehq/airbyte/pull/18036) | Migrate MeiliSearch to Python CDK                      |
+| 0.2.13  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
+| 0.2.12  | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option           |
+| 0.2.11  | 2021-12-28 | [9156](https://github.com/airbytehq/airbyte/pull/9156)   | Update connector fields title/description              |
diff --git a/docs/integrations/destinations/mongodb.md b/docs/integrations/destinations/mongodb.md
index ea44a2f1f5f7..51bd94cb8c46 100644
--- a/docs/integrations/destinations/mongodb.md
+++ b/docs/integrations/destinations/mongodb.md
@@ -2,24 +2,24 @@
 
 ## Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ## Prerequisites
-- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your MongoDB connector to version `0.1.6` or newer
 
+- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your MongoDB connector to version `0.1.6` or newer
 
 ## Output Schema for `destination-mongodb`
 
 Each stream will be output into its own collection in MongoDB. Each collection will contain 3 fields:
 
-* `_id`: an identifier assigned to each document that is processed. The filed type in MongoDB is `String`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The field type in MongoDB is `Timestamp`.
-* `_airbyte_data`: a json blob representing with the event data. The field type in MongoDB is `Object`.
+- `_id`: an identifier assigned to each document that is processed. The filed type in MongoDB is `String`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The field type in MongoDB is `Timestamp`.
+- `_airbyte_data`: a json blob representing with the event data. The field type in MongoDB is `Object`.
 
 ## Getting Started \(Airbyte Cloud\)
 
@@ -31,7 +31,7 @@ Airbyte Cloud only supports connecting to your MongoDB instance with TLS encrypt
 
 To use the MongoDB destination, you'll need:
 
-* A MongoDB server
+- A MongoDB server
 
 #### **Permissions**
 
@@ -45,18 +45,18 @@ You will need to choose an existing database or create a new database that will
 
 You should now have all the requirements needed to configure MongoDB as a destination in the UI. You'll need the following information to configure the MongoDB destination:
 
-* **Standalone MongoDb instance**
-  * Host: URL of the database
-  * Port: Port to use for connecting to the database
-  * TLS: indicates whether to create encrypted connection
-* **Replica Set**
-  * Server addresses: the members of a replica set
-  * Replica Set: A replica set name
-* **MongoDb Atlas Cluster**
-  * Cluster URL: URL of a cluster to connect to
-* **Database**
-* **Username**
-* **Password**
+- **Standalone MongoDb instance**
+  - Host: URL of the database
+  - Port: Port to use for connecting to the database
+  - TLS: indicates whether to create encrypted connection
+- **Replica Set**
+  - Server addresses: the members of a replica set
+  - Replica Set: A replica set name
+- **MongoDb Atlas Cluster**
+  - Cluster URL: URL of a cluster to connect to
+- **Database**
+- **Username**
+- **Password**
 
 For more information regarding configuration parameters, please see [MongoDb Documentation](https://docs.mongodb.com/drivers/java/sync/v4.3/fundamentals/connection/).
 
@@ -72,8 +72,8 @@ Using this feature requires additional configuration, when creating the source.
 
 1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`.
 2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`.
-    1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
-    2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
+   1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
+   2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
 3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address.
 4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default.
 5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the TiDB username.
@@ -108,15 +108,16 @@ Database names cannot be empty and must have fewer than 64 characters.
 
 Collection names should begin with an underscore or a letter character, and cannot:
 
-* contain the $.
-* be an empty string \(e.g. ""\).
-* contain the null character.
-* begin with the system. prefix. \(Reserved for internal use.\)
+- contain the $.
+- be an empty string \(e.g. ""\).
+- contain the null character.
+- begin with the system. prefix. \(Reserved for internal use.\)
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- |
+| 0.2.0   | 2023-06-27 | [27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                       |
 | 0.1.9   | 2022-11-08 | [18892](https://github.com/airbytehq/airbyte/pull/18892) | Adds check for TLS flag                                    |
 | 0.1.8   | 2022-10-26 | [18280](https://github.com/airbytehq/airbyte/pull/18280) | Adds SSH tunneling                                         |
 | 0.1.7   | 2022-09-02 | [16025](https://github.com/airbytehq/airbyte/pull/16025) | Remove additionalProperties:false from spec                |
@@ -126,4 +127,3 @@ Collection names should begin with an underscore or a letter character, and cann
 | 0.1.3   | 2021-12-30 | [8809](https://github.com/airbytehq/airbyte/pull/8809)   | Update connector fields title/description                  |
 | 0.1.2   | 2021-10-18 | [6945](https://github.com/airbytehq/airbyte/pull/6945)   | Create a secure-only MongoDb destination                   |
 | 0.1.1   | 2021-09-29 | [6536](https://github.com/airbytehq/airbyte/pull/6536)   | Destination MongoDb: added support via TLS/SSL             |
-
diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md
index d7f012097dbf..0c33e9ed9348 100644
--- a/docs/integrations/destinations/mqtt.md
+++ b/docs/integrations/destinations/mqtt.md
@@ -5,6 +5,7 @@
 The Airbyte MQTT destination allows you to sync data to any MQTT system compliance with version 3.1.X. Each stream is written to the corresponding MQTT topic.
 
 ## Prerequisites
+
 - For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your MQTT connector to the latest version
 
 ### Sync overview
@@ -17,19 +18,19 @@ This connector writes data with JSON format (in bytes).
 
 Each record will contain in its payload these 4 fields:
 
-* `_airbyte_ab_id`: an uuid assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`:  a timestamp representing when the event was pulled from the data source.
-* `_airbyte_data`: a json blob representing with the event data.
-* `_airbyte_stream`: the name of each record's stream.
+- `_airbyte_ab_id`: an uuid assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_data`: a json blob representing with the event data.
+- `_airbyte_stream`: the name of each record's stream.
 
 #### Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | No |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | No                   |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ## Getting started
 
@@ -37,7 +38,7 @@ Each record will contain in its payload these 4 fields:
 
 To use the MQTT destination, you'll need:
 
-* A MQTT broker implementing MQTT protocol version 3.1.X.
+- A MQTT broker implementing MQTT protocol version 3.1.X.
 
 ### Setup guide
 
@@ -61,30 +62,29 @@ To define the output topics dynamically, you can leverage the `{namespace}` and
 
 You should now have all the requirements needed to configure MQTT as a destination in the UI. You can configure the following parameters on the MQTT destination \(though many of these are optional or have default values\):
 
-* **MQTT broker host**
-* **MQTT broker port**
-* **Use TLS**
-* **Username**
-* **Password**
-* **Topic pattern**
-* **Test topic**
-* **Client ID**
-* **Sync publisher**
-* **Connect timeout**
-* **Automatic reconnect**
-* **Clean session**
-* **Message retained**
-* **Message QoS**
+- **MQTT broker host**
+- **MQTT broker port**
+- **Use TLS**
+- **Username**
+- **Password**
+- **Topic pattern**
+- **Test topic**
+- **Client ID**
+- **Sync publisher**
+- **Connect timeout**
+- **Automatic reconnect**
+- **Clean session**
+- **Message retained**
+- **Message QoS**
 
 More info about this can be found in the [OASIS MQTT standard site](http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/mqtt-v3.1.1.html).
 
 _NOTE_: MQTT version 5 is not supported yet.
 
-
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.3 | 2022-09-02 | [16263](https://github.com/airbytehq/airbyte/pull/16263) | Marked password field in spec as airbyte_secret |
-| 0.1.2 | 2022-07-12 | [14648](https://github.com/airbytehq/airbyte/pull/14648) | Include lifecycle management |
-| 0.1.1 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests |
+| Version | Date       | Pull Request                                             | Subject                                         |
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- |
+| 0.1.3   | 2022-09-02 | [16263](https://github.com/airbytehq/airbyte/pull/16263) | Marked password field in spec as airbyte_secret |
+| 0.1.2   | 2022-07-12 | [14648](https://github.com/airbytehq/airbyte/pull/14648) | Include lifecycle management                    |
+| 0.1.1   | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests                             |
diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md
index 785d5b909b76..c48261be1a0b 100644
--- a/docs/integrations/destinations/mssql.md
+++ b/docs/integrations/destinations/mssql.md
@@ -2,36 +2,36 @@
 
 ## Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes                                                                                        |
-|:------------------------------|:---------------------|:---------------------------------------------------------------------------------------------|
-| Full Refresh Sync             | Yes                  |                                                                                              |
-| Incremental - Append Sync     | Yes                  |                                                                                              |
-| Incremental - Deduped History | No                   | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    | Yes                  |                                                                                              |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | Yes                  |       |
+| Namespaces                     | Yes                  |       |
 
 ## Output Schema
 
 Each stream will be output into its own table in SQL Server. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in SQL Server is `VARCHAR(64)`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in SQL Server is `DATETIMEOFFSET(7)`.
-* `_airbyte_data`: a JSON blob representing with the event data. The column type in SQL Server is `NVARCHAR(MAX)`.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in SQL Server is `VARCHAR(64)`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in SQL Server is `DATETIMEOFFSET(7)`.
+- `_airbyte_data`: a JSON blob representing with the event data. The column type in SQL Server is `NVARCHAR(MAX)`.
 
 #### Microsoft SQL Server specifics or why NVARCHAR type is used here:
 
-* NVARCHAR is Unicode - 2 bytes per character, therefore max. of 1 billion characters; will handle East Asian, Arabic, Hebrew, Cyrillic etc. characters just fine.
-* VARCHAR is non-Unicode - 1 byte per character, max. capacity is 2 billion characters, but limited to the character set you're SQL Server is using, basically - no support for those languages mentioned before
+- NVARCHAR is Unicode - 2 bytes per character, therefore max. of 1 billion characters; will handle East Asian, Arabic, Hebrew, Cyrillic etc. characters just fine.
+- VARCHAR is non-Unicode - 1 byte per character, max. capacity is 2 billion characters, but limited to the character set you're SQL Server is using, basically - no support for those languages mentioned before
 
 ## Getting Started \(Airbyte Cloud\)
 
 Airbyte Cloud only supports connecting to your MSSQL instance with TLS encryption. Other than that, you can proceed with the open-source instructions below.
 
-| Feature                       | Supported?\(Yes/No\) | Notes |
-|:------------------------------|:---------------------|:------|
-| Full Refresh Sync             | Yes                  |       |
-| Incremental - Append Sync     | Yes                  |       |
-| Incremental - Deduped History | Yes                  |       |
-| Namespaces                    | Yes                  |       |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | Yes                  |       |
+| Namespaces                     | Yes                  |       |
 
 ## Getting Started \(Airbyte Open-Source\)
 
@@ -47,7 +47,7 @@ To sync **with** normalization you'll need to use MS SQL Server of the following
 
 ### Setup guide
 
-* MS SQL Server: `Azure SQL Database`, `Azure Synapse Analytics`, `Azure SQL Managed Instance`, `SQL Server 2019`, `SQL Server 2017`, `SQL Server 2016`, `SQL Server 2014`, `SQL Server 2012`, or `PDW 2008R2 AU34`.
+- MS SQL Server: `Azure SQL Database`, `Azure Synapse Analytics`, `Azure SQL Managed Instance`, `SQL Server 2019`, `SQL Server 2017`, `SQL Server 2016`, `SQL Server 2014`, `SQL Server 2012`, or `PDW 2008R2 AU34`.
 
 #### Network Access
 
@@ -56,7 +56,7 @@ Make sure your SQL Server database can be accessed by Airbyte. If your database
 #### **Permissions**
 
 You need a user configured in SQL Server that can create tables and write rows. We highly recommend creating an Airbyte-specific user for this purpose.
-In order to allow for normalization, please grant ALTER permissions for the user configured. 
+In order to allow for normalization, please grant ALTER permissions for the user configured.
 
 #### Target Database
 
@@ -70,19 +70,19 @@ Airbyte supports a SSL-encrypted connection to the database. If you want to use
 
 You should now have all the requirements needed to configure SQL Server as a destination in the UI. You'll need the following information to configure the MSSQL destination:
 
-* **Host**
-* **Port**
-* **Username**
-* **Password**
-* **Schema**
-* **Database**
-  * This database needs to exist within the schema provided.
-* **SSL Method**:
-  * The SSL configuration supports three modes: Unencrypted, Encrypted \(trust server certificate\), and Encrypted \(verify certificate\).
-    * **Unencrypted**: Do not use SSL encryption on the database connection
-    * **Encrypted \(trust server certificate\)**: Use SSL encryption without verifying the server's certificate.  This is useful for self-signed certificates in testing scenarios, but should not be used in production.
-    * **Encrypted \(verify certificate\)**: Use the server's SSL certificate, after standard certificate verification.
-  * **Host Name In Certificate** \(optional\): When using certificate verification, this property can be set to specify an expected name for added security.  If this value is present, and the server's certificate's host name does not match it, certificate verification will fail.
+- **Host**
+- **Port**
+- **Username**
+- **Password**
+- **Schema**
+- **Database**
+  - This database needs to exist within the schema provided.
+- **SSL Method**:
+  - The SSL configuration supports three modes: Unencrypted, Encrypted \(trust server certificate\), and Encrypted \(verify certificate\).
+    - **Unencrypted**: Do not use SSL encryption on the database connection
+    - **Encrypted \(trust server certificate\)**: Use SSL encryption without verifying the server's certificate. This is useful for self-signed certificates in testing scenarios, but should not be used in production.
+    - **Encrypted \(verify certificate\)**: Use the server's SSL certificate, after standard certificate verification.
+  - **Host Name In Certificate** \(optional\): When using certificate verification, this property can be set to specify an expected name for added security. If this value is present, and the server's certificate's host name does not match it, certificate verification will fail.
 
 ## Connection via SSH Tunnel
 
@@ -115,7 +115,10 @@ Using this feature requires additional configuration, when creating the source.
 ## Changelog
 
 | Version | Date       | Pull Request                                               | Subject                                                                                             |
-|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- |
+| 0.2.0   | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                |
+| 0.1.25  | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size                                                                                   |
+| 0.1.24  | 2023-06-05 | [\#27034](https://github.com/airbytehq/airbyte/pull/27034) | Internal code change for future development (install normalization packages inside connector)       |
 | 0.1.23  | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                                                               |
 | 0.1.22  | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889                                                             |
 | 0.1.20  | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors                                |
@@ -137,21 +140,3 @@ Using this feature requires additional configuration, when creating the source.
 | 0.1.3   | 2021-05-28 | [\#3728](https://github.com/airbytehq/airbyte/pull/3973)   | Change dockerfile entrypoint                                                                        |
 | 0.1.2   | 2021-05-13 | [\#3367](https://github.com/airbytehq/airbyte/pull/3671)   | Fix handle symbols unicode                                                                          |
 | 0.1.1   | 2021-05-11 | [\#3566](https://github.com/airbytehq/airbyte/pull/3195)   | MS SQL Server Destination Release!                                                                  |
-
-### Changelog (Strict Encrypt)
-
-| Version | Date       | Pull Request                                               | Subject                                                                                             |
-|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
-| 0.1.23  | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                                                               |
-| 0.1.22  | 2022-10-21 | [\#18275](https://github.com/airbytehq/airbyte/pull/18275) | Upgrade commons-text for CVE 2022-42889                                                             |
-| 0.1.21  | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage                                                                      |
-| 0.1.20  | 2022-07-14 | [\#15260](https://github.com/airbytehq/airbyte/pull/15260) | Align version of strict encrypt connector with regular connector                                    |
-| 0.1.10  | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors                                |
-| 0.1.9   | 2022-06-17 | [\#13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors                                              |
-| 0.1.8   | 2022-05-25 | [\#13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support.                                            |
-| 0.1.6   | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                              |
-| 0.1.5   | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                   |
-| 0.1.4   | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                        |
-| 0.1.3   | 2021-12-28 | [\#9158](https://github.com/airbytehq/airbyte/pull/9158)   | Update connector fields title/description                                                           |
-| 0.1.2   | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371)   | Fixed incorrect handling "\n" in ssh key                                                            |
-| 0.1.1   | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719)   | Improve handling of wide rows by buffering records based on their byte size rather than their count |
diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md
index e4565888c739..58826d631142 100644
--- a/docs/integrations/destinations/mysql.md
+++ b/docs/integrations/destinations/mysql.md
@@ -1,27 +1,27 @@
 # MySQL
 
 There are two flavors of connectors for this destination:
+
 1. destination-mysql connector. Supports both SSL and non SSL connections.
 2. destination-mysql-strict-encrypt connector. Pretty same as connector above, but supports SSL connections only.
 
-
 ## Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No |  |
-| Namespaces | Yes |  |
-| SSH Tunnel Connection | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | Yes                  |       |
+| Namespaces                     | Yes                  |       |
+| SSH Tunnel Connection          | Yes                  |       |
 
 #### Output Schema
 
 Each stream will be output into its own table in MySQL. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in MySQL is `VARCHAR(256)`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in MySQL is `TIMESTAMP(6)`.
-* `_airbyte_data`: a json blob representing with the event data. The column type in MySQL is `JSON`.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in MySQL is `VARCHAR(256)`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in MySQL is `TIMESTAMP(6)`.
+- `_airbyte_data`: a json blob representing with the event data. The column type in MySQL is `JSON`.
 
 ## Getting Started \(Airbyte Cloud\)
 
@@ -33,8 +33,8 @@ Airbyte Cloud only supports connecting to your MySQL instance with TLS encryptio
 
 To use the MySQL destination, you'll need:
 
-* To sync data to MySQL **with** normalization MySQL database 8.0.0 or above
-* To sync data to MySQL **without** normalization you'll need MySQL 5.0 or above.
+- To sync data to MySQL **with** normalization MySQL database 8.0.0 or above
+- To sync data to MySQL **without** normalization you'll need MySQL 5.0 or above.
 
 #### Troubleshooting
 
@@ -55,25 +55,25 @@ MySQL doesn't differentiate between a database and schema. A database is essenti
 
 ### Setup the MySQL destination in Airbyte
 
-Before setting up MySQL destination in Airbyte, you need to set the [local\_infile](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_local_infile) system variable to true. You can do this by running the query `SET GLOBAL local_infile = true` with a user with [SYSTEM\_VARIABLES\_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_system-variables-admin) permission. This is required cause Airbyte uses `LOAD DATA LOCAL INFILE` to load data into table.
+Before setting up MySQL destination in Airbyte, you need to set the [local_infile](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_local_infile) system variable to true. You can do this by running the query `SET GLOBAL local_infile = true` with a user with [SYSTEM_VARIABLES_ADMIN](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_system-variables-admin) permission. This is required cause Airbyte uses `LOAD DATA LOCAL INFILE` to load data into table.
 
 You should now have all the requirements needed to configure MySQL as a destination in the UI. You'll need the following information to configure the MySQL destination:
 
-* **Host**
-* **Port**
-* **Username**
-* **Password**
-* **Database**
-* **jdbc_url_params** (Optional)
+- **Host**
+- **Port**
+- **Username**
+- **Password**
+- **Database**
+- **jdbc_url_params** (Optional)
 
 ### Default JDBC URL Parameters
 
 The following JDBC URL parameters are set by Airbyte and cannot be overridden by the `jdbc_url_params` field:
 
-* `useSSL=true` (unless `ssl` is set to false)
-* `requireSSL=true` (unless `ssl` is set to false)
-* `verifyServerCertificate=false` (unless `ssl` is set to false)
-* `zeroDateTimeBehavior=convertToNull`
+- `useSSL=true` (unless `ssl` is set to false)
+- `requireSSL=true` (unless `ssl` is set to false)
+- `verifyServerCertificate=false` (unless `ssl` is set to false)
+- `zeroDateTimeBehavior=convertToNull`
 
 ## Known Limitations
 
@@ -98,11 +98,12 @@ Using this feature requires additional configuration, when creating the destinat
 
 1. Configure all fields for the destination as you normally would, except `SSH Tunnel Method`.
 2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`.
+
    1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
    2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
 
    :::warning
-    Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **preferred** as the **SSL Mode**; otherwise, the connection will fail.
+   Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **preferred** as the **SSL Mode**; otherwise, the connection will fail.
    :::
 
 3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address.
@@ -113,16 +114,17 @@ Using this feature requires additional configuration, when creating the destinat
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject                                                                                             |
-|:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------|
-| 0.1.21  | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage  |
-| 0.1.20  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
-| 0.1.19  | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
+| Version | Date       | Pull Request                                             | Subject                                                                                             |
+| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- |
+| 0.2.0   | 2023-06-27 | [27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                |
+| 0.1.21  | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage                                                                      |
+| 0.1.20  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors                                              |
+| 0.1.19  | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                              |
 | 0.1.18  | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                   |
 | 0.1.17  | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters                                            |
 | 0.1.16  | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                        |
-| 0.1.15  | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key                                                            |
-| 0.1.14  | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count |
+| 0.1.15  | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371)   | Fixed incorrect handling "\n" in ssh key                                                            |
+| 0.1.14  | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719)  | Improve handling of wide rows by buffering records based on their byte size rather than their count |
 | 0.1.13  | 2021-09-28 | [\#6506](https://github.com/airbytehq/airbyte/pull/6506) | Added support for MySQL destination via TLS/SSL                                                     |
 | 0.1.12  | 2021-09-24 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317) | Added option to connect to DB via SSH                                                               |
 | 0.1.11  | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json                                                        |
@@ -135,15 +137,3 @@ Using this feature requires additional configuration, when creating the destinat
 | 0.1.2   | 2021-07-03 | [\#3327](https://github.com/airbytehq/airbyte/pull/3327) | Fixed LSEP unicode characters.                                                                      |
 | 0.1.1   | 2021-07-03 | [\#3289](https://github.com/airbytehq/airbyte/pull/3289) | Added support for outputting messages.                                                              |
 | 0.1.0   | 2021-05-06 | [\#3242](https://github.com/airbytehq/airbyte/pull/3242) | Added MySQL destination.                                                                            |
-
-## CHANGELOG destination-mysql-strict-encrypt
-
-| Version | Date       | Pull Request                                             | Subject                                                                                             |
-|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
-| 0.1.20  | 2022-08-03 | [15258](https://github.com/airbytehq/airbyte/pull/15258) | Align versions of strict encrypt and regular connectors                                             |
-| 0.1.5   | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                              |
-| 0.1.4   | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                   |
-| 0.1.3   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                        |
-| 0.1.2   | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key                                                            |
-| 0.1.1   | 2021-11-08 | [#7719](https://github.com/airbytehq/airbyte/pull/7719)  | Improve handling of wide rows by buffering records based on their byte size rather than their count |
-| 0.1.0   | 06.10.2021 | [\#6763](https://github.com/airbytehq/airbyte/pull/6763) | Added destination-mysql-strict-encrypt that supports SSL connections only.                          |
\ No newline at end of file
diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md
index 21486973b269..2b26a69cbf6c 100644
--- a/docs/integrations/destinations/oracle.md
+++ b/docs/integrations/destinations/oracle.md
@@ -2,23 +2,23 @@
 
 ## Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | Yes |  |
-| Namespaces | Yes |  |
-| Basic Normalization | Yes | Doesn't support for nested json yet |
-| SSH Tunnel Connection | Yes |  |
-| Encryption | Yes | Support Native Network Encryption (NNE) as well as TLS using SSL cert |
+| Feature                        | Supported?\(Yes/No\) | Notes                                                                 |
+| :----------------------------- | :------------------- | :-------------------------------------------------------------------- |
+| Full Refresh Sync              | Yes                  |                                                                       |
+| Incremental - Append Sync      | Yes                  |                                                                       |
+| Incremental - Append + Deduped | Yes                  |                                                                       |
+| Namespaces                     | Yes                  |                                                                       |
+| Basic Normalization            | Yes                  | Doesn't support for nested json yet                                   |
+| SSH Tunnel Connection          | Yes                  |                                                                       |
+| Encryption                     | Yes                  | Support Native Network Encryption (NNE) as well as TLS using SSL cert |
 
 ## Output Schema
 
 By default, each stream will be output into its own table in Oracle. Each table will contain 3 columns:
 
-* `_AIRBYTE_AB_ID`: a uuid assigned by Airbyte to each event that is processed. The column type in Oracle is `VARCHAR(64)`.
-* `_AIRBYTE_EMITTED_AT`: a timestamp representing when the event was pulled from the data source. The column type in Oracle is `TIMESTAMP WITH TIME ZONE`.
-* `_AIRBYTE_DATA`: a json blob representing with the event data. The column type in Oracles is `NCLOB`.
+- `_AIRBYTE_AB_ID`: a uuid assigned by Airbyte to each event that is processed. The column type in Oracle is `VARCHAR(64)`.
+- `_AIRBYTE_EMITTED_AT`: a timestamp representing when the event was pulled from the data source. The column type in Oracle is `TIMESTAMP WITH TIME ZONE`.
+- `_AIRBYTE_DATA`: a json blob representing with the event data. The column type in Oracles is `NCLOB`.
 
 Enabling normalization will also create normalized, strongly typed tables.
 
@@ -32,8 +32,8 @@ The Oracle connector is currently in Alpha on Airbyte Cloud. Only TLS encrypted
 
 To use the Oracle destination, you'll need:
 
-* An Oracle server version 18 or above
-* It's possible to use Oracle 12+ but you need to configure the table name length to 120 chars.
+- An Oracle server version 18 or above
+- It's possible to use Oracle 12+ but you need to configure the table name length to 120 chars.
 
 #### Network Access
 
@@ -43,12 +43,12 @@ Make sure your Oracle database can be accessed by Airbyte. If your database is w
 
 As Airbyte namespaces allows us to store data into different schemas, we have different scenarios and list of required permissions:
 
-| Login user | Destination user | Required permissions | Comment |
-| :--- | :--- | :--- | :--- |
-| DBA User | Any user | - |  |
-| Regular user | Same user as login | Create, drop and write table, create session |  |
-| Regular user | Any existing user | Create, drop and write ANY table, create session | Grants can be provided on a system level by DBA or by target user directly |
-| Regular user | Not existing user | Create, drop and write ANY table, create user, create session | Grants should be provided on a system level by DBA |
+| Login user   | Destination user   | Required permissions                                          | Comment                                                                    |
+| :----------- | :----------------- | :------------------------------------------------------------ | :------------------------------------------------------------------------- |
+| DBA User     | Any user           | -                                                             |                                                                            |
+| Regular user | Same user as login | Create, drop and write table, create session                  |                                                                            |
+| Regular user | Any existing user  | Create, drop and write ANY table, create session              | Grants can be provided on a system level by DBA or by target user directly |
+| Regular user | Not existing user  | Create, drop and write ANY table, create user, create session | Grants should be provided on a system level by DBA                         |
 
 We highly recommend creating an Airbyte-specific user for this purpose.
 
@@ -56,12 +56,12 @@ We highly recommend creating an Airbyte-specific user for this purpose.
 
 You should now have all the requirements needed to configure Oracle as a destination in the UI. You'll need the following information to configure the Oracle destination:
 
-* **Host**
-* **Port**
-* **Username**
-* **Password**
-* **Database**
-* **Connection via SSH Tunnel**
+- **Host**
+- **Port**
+- **Username**
+- **Password**
+- **Database**
+- **Connection via SSH Tunnel**
 
 Airbyte has the ability to connect to a Oracle instance via an SSH Tunnel. The reason you might want to do this because it is not possible \(or against security policy\) to connect to the database directly \(e.g. it does not have a public IP address\).
 
@@ -71,8 +71,8 @@ Using this feature requires additional configuration, when creating the source.
 
 1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`.
 2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`.
-    1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
-    2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
+   1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
+   2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
 3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address.
 4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default.
 5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the Oracle username.
@@ -84,40 +84,29 @@ Using this feature requires additional configuration, when creating the source.
 Airbyte has the ability to connect to the Oracle source with 3 network connectivity options:
 
 1. `Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form.
-2. `Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. In this case, the *SQLNET.ENCRYPTION_CLIENT*
-   option will always be set as *REQUIRED* by default: The client or server will only accept encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs.
+2. `Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. In this case, the _SQLNET.ENCRYPTION_CLIENT_
+   option will always be set as _REQUIRED_ by default: The client or server will only accept encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs.
 3. `TLS Encrypted` (verify certificate) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification. To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field
 
 ## Changelog
 
-| Version     | Date | Pull Request                                             | Subject                                                                                             |
-|:------------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
-| 0.1.19      | 2022-07-26 | [\#10719](https://github.com/airbytehq/airbyte/pull/)    | Destination Oracle: added custom JDBC parameters support.                                           |
-| 0.1.18      | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors |
-| unpublished | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
-| 0.1.16      | 2022-04-06 | [11514](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0                                                                  |
-| 0.1.15      | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support                                            |
-| 0.1.14      | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option                                          |
-| 0.1.13      | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description                                                           |
-| 0.1.12      | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count |
-| 0.1.10      | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893) | 🎉 Destination Oracle: implemented connection encryption                                            |
-| 0.1.9       | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611) | 🐛 Destination Oracle: maxStringLength should be 128                                                |
-| 0.1.8       | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370) | Add SSH Support for Oracle Destination                                                              |
-| 0.1.7       | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746) | Use default column name for raw tables                                                              |
-| 0.1.6       | 2021-08-23 | [\#5542](https://github.com/airbytehq/airbyte/pull/5542) | Remove support for Oracle 11g to allow normalization                                                |
-| 0.1.5       | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307) | 🐛 Destination Oracle: Fix destination check for users without dba role                             |
-| 0.1.4       | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json                                                        |
-| 0.1.3       | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer                                                           |
-| 0.1.2       | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | Require `sid` instead of `database` in connector specification                                      |
-
-### Changelog (Strict Encrypt)
-
-| Version | Date       | Pull Request                                            | Subject                                                                                                                     |
-|:--------|:-----------|:--------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------|
-| 0.1.19   | 2022-07-26 | [\#10719](https://github.com/airbytehq/airbyte/pull/) | Destination Oracle: added custom JDBC parameters support.                                        |
-| 0.1.7   | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors |
-| 0.1.5   | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
-| 0.1.4   | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support                                                                    |
-| 0.1.3   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                  |
-| 0.1.2   | 2021-01-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177) | Update connector fields title/description                                                                                   |
-| 0.1.1   | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count                         |
+| Version     | Date       | Pull Request                                               | Subject                                                                                             |
+| :---------- | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- |
+| 0.2.0       | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                |
+| 0.1.19      | 2022-07-26 | [\#10719](https://github.com/airbytehq/airbyte/pull/)      | Destination Oracle: added custom JDBC parameters support.                                           |
+| 0.1.18      | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors                                |
+| unpublished | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820)   | Improved 'check' operation performance                                                              |
+| 0.1.16      | 2022-04-06 | [11514](https://github.com/airbytehq/airbyte/pull/11514)   | Bump mina-sshd from 2.7.0 to 2.8.0                                                                  |
+| 0.1.15      | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421)   | Refactor JDBC parameters handling and remove DBT support                                            |
+| 0.1.14      | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256)   | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option                                          |
+| 0.1.13      | 2021-12-29 | [\#9177](https://github.com/airbytehq/airbyte/pull/9177)   | Update connector fields title/description                                                           |
+| 0.1.12      | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719)   | Improve handling of wide rows by buffering records based on their byte size rather than their count |
+| 0.1.10      | 2021-10-08 | [\#6893](https://github.com/airbytehq/airbyte/pull/6893)   | 🎉 Destination Oracle: implemented connection encryption                                            |
+| 0.1.9       | 2021-10-06 | [\#6611](https://github.com/airbytehq/airbyte/pull/6611)   | 🐛 Destination Oracle: maxStringLength should be 128                                                |
+| 0.1.8       | 2021-09-28 | [\#6370](https://github.com/airbytehq/airbyte/pull/6370)   | Add SSH Support for Oracle Destination                                                              |
+| 0.1.7       | 2021-08-30 | [\#5746](https://github.com/airbytehq/airbyte/pull/5746)   | Use default column name for raw tables                                                              |
+| 0.1.6       | 2021-08-23 | [\#5542](https://github.com/airbytehq/airbyte/pull/5542)   | Remove support for Oracle 11g to allow normalization                                                |
+| 0.1.5       | 2021-08-10 | [\#5307](https://github.com/airbytehq/airbyte/pull/5307)   | 🐛 Destination Oracle: Fix destination check for users without dba role                             |
+| 0.1.4       | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125)   | Enable `additionalPropertities` in spec.json                                                        |
+| 0.1.3       | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555)   | Partial Success in BufferedStreamConsumer                                                           |
+| 0.1.2       | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874)   | Require `sid` instead of `database` in connector specification                                      |
diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md
index 49e83d273587..a05718c145e2 100644
--- a/docs/integrations/destinations/postgres.md
+++ b/docs/integrations/destinations/postgres.md
@@ -139,12 +139,12 @@ characters.
 The Postgres destination connector supports the
 following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-| Feature                       | Supported?\(Yes/No\) | Notes |
-| :---------------------------- | :------------------- | :---- |
-| Full Refresh Sync             | Yes                  |       |
-| Incremental - Append Sync     | Yes                  |       |
-| Incremental - Deduped History | Yes                  |       |
-| Namespaces                    | Yes                  |       |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | Yes                  |       |
+| Namespaces                     | Yes                  |       |
 
 ## Schema map
 
@@ -170,6 +170,7 @@ Now that you have set up the Postgres destination connector, check out the follo
 
 | Version | Date       | Pull Request                                               | Subject                                                                                             |
 | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- |
+| 0.4.0   | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                |
 | 0.3.27  | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                                                               |
 | 0.3.26  | 2022-09-27 | [\#17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres destination                                      |
 | 0.3.24  | 2022-09-08 | [\#16046](https://github.com/airbytehq/airbyte/pull/16046) | Fix missing database name URL Encoding                                                              |
diff --git a/docs/integrations/destinations/pubsub.md b/docs/integrations/destinations/pubsub.md
index 2888af138c0f..a3aef1932a7e 100644
--- a/docs/integrations/destinations/pubsub.md
+++ b/docs/integrations/destinations/pubsub.md
@@ -11,6 +11,7 @@ description: >-
 The Airbyte Google PubSub destination allows you to send/stream data into PubSub. Pub/Sub is an asynchronous messaging service provided by Google Cloud Provider.
 
 ## Prerequisites
+
 - For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your PubSub connector to version `0.1.6` or newer
 
 ### Sync overview
@@ -19,23 +20,23 @@ The Airbyte Google PubSub destination allows you to send/stream data into PubSub
 
 Each stream will be output a PubSubMessage with attributes. The message attributes will be
 
-* `_stream`: the name of stream where the data is coming from
-* `_namespace`: namespace if available from the stream
+- `_stream`: the name of stream where the data is coming from
+- `_namespace`: namespace if available from the stream
 
 The data will be a serialized JSON, containing the following fields
 
-* `_airbyte_ab_id`: a uuid string assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`: a long timestamp\(ms\) representing when the event was pulled from the data source.
-* `_airbyte_data`: a json string representing source data.
+- `_airbyte_ab_id`: a uuid string assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a long timestamp\(ms\) representing when the event was pulled from the data source.
+- `_airbyte_data`: a json string representing source data.
 
 #### Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ## Getting started
 
@@ -43,10 +44,10 @@ The data will be a serialized JSON, containing the following fields
 
 To use the PubSub destination, you'll need:
 
-* A Google Cloud Project with PubSub enabled
-* A PubSub Topic to which Airbyte can stream/sync your data
-* A Google Cloud Service Account with the `Pub/Sub Editor` role in your GCP project
-* A Service Account Key to authenticate into your Service Account
+- A Google Cloud Project with PubSub enabled
+- A PubSub Topic to which Airbyte can stream/sync your data
+- A Google Cloud Service Account with the `Pub/Sub Editor` role in your GCP project
+- A Service Account Key to authenticate into your Service Account
 
 See the setup guide for more information about how to create the required resources.
 
@@ -80,23 +81,22 @@ Follow the [Creating and Managing Service Account Keys](https://cloud.google.com
 
 ### Setup the PubSub destination in Airbyte
 
-You should now have all the requirements needed to configure BigQuery as a destination in the UI. You'll need the following information to configure the BigQuery destination:
+You should now have all the requirements needed to configure PubSub as a destination in the UI. You'll need the following information to configure the PubSub destination:
 
-* **Project ID**: GCP project id
-* **Topic ID**: name of pubsub topic under the project
-* **Service Account Key**: the contents of your Service Account Key JSON file
+- **Project ID**: GCP project id
+- **Topic ID**: name of pubsub topic under the project
+- **Service Account Key**: the contents of your Service Account Key JSON file
 
 Once you've configured PubSub as a destination, delete the Service Account Key from your computer.
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.0 | August 16, 2022 | [15705](https://github.com/airbytehq/airbyte/pull/15705) | Add configuration for Batching and Ordering |
-| 0.1.5 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
-| 0.1.4 | February 21, 2022 | [\#9819](https://github.com/airbytehq/airbyte/pull/9819) | Upgrade version of google-cloud-pubsub |
-| 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option |
-| 0.1.2 | December 29, 2021 | [\#9183](https://github.com/airbytehq/airbyte/pull/9183) | Update connector fields title/description |
-| 0.1.1 | August 13, 2021 | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |
-| 0.1.0 | June 24, 2021 | [\#4339](https://github.com/airbytehq/airbyte/pull/4339) | Initial release |
-
+| Version | Date              | Pull Request                                             | Subject                                                    |
+| :------ | :---------------- | :------------------------------------------------------- | :--------------------------------------------------------- |
+| 0.2.0   | August 16, 2022   | [15705](https://github.com/airbytehq/airbyte/pull/15705) | Add configuration for Batching and Ordering                |
+| 0.1.5   | 2022-06-17        | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors     |
+| 0.1.4   | February 21, 2022 | [\#9819](https://github.com/airbytehq/airbyte/pull/9819) | Upgrade version of google-cloud-pubsub                     |
+| 0.1.3   | 2022-02-14        | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option |
+| 0.1.2   | December 29, 2021 | [\#9183](https://github.com/airbytehq/airbyte/pull/9183) | Update connector fields title/description                  |
+| 0.1.1   | August 13, 2021   | [\#4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator                                |
+| 0.1.0   | June 24, 2021     | [\#4339](https://github.com/airbytehq/airbyte/pull/4339) | Initial release                                            |
diff --git a/docs/integrations/destinations/pulsar.md b/docs/integrations/destinations/pulsar.md
index 9a44887da19f..67d0bf40c006 100644
--- a/docs/integrations/destinations/pulsar.md
+++ b/docs/integrations/destinations/pulsar.md
@@ -25,12 +25,12 @@ Each record will contain in its key the uuid assigned by Airbyte, and in the val
 
 #### Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes                                                                                        |
-| :---------------------------- | :------------------- | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             | No                   |                                                                                              |
-| Incremental - Append Sync     | Yes                  |                                                                                              |
-| Incremental - Deduped History | No                   | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    | Yes                  |                                                                                              |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | No                   |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ## Getting started
 
diff --git a/docs/integrations/destinations/r2.md b/docs/integrations/destinations/r2.md
index b18a2475a2c1..6ed9c67deebc 100644
--- a/docs/integrations/destinations/r2.md
+++ b/docs/integrations/destinations/r2.md
@@ -3,12 +3,14 @@
 This page guides you through the process of setting up the R2 destination connector.
 
 ## Prerequisites
+
 List of required fields:
-* **Account ID**
-* **Access Key ID**
-* **Secret Access Key**
-* **R2 Bucket Name**
-* **R2 Bucket Path**
+
+- **Account ID**
+- **Access Key ID**
+- **Secret Access Key**
+- **R2 Bucket Name**
+- **R2 Bucket Path**
 
 1. Allow connections from Airbyte server to your Cloudflare R2 bucket
 
@@ -30,22 +32,21 @@ to create an S3 bucket, or you can create bucket via R2 module of [dashboard](ht
 2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**.
 3. On the destination setup page, select **R2** from the Destination type dropdown and enter a name for this connector.
 4. Configure fields:
-    * **Account Id**
-        * See [this](https://developers.cloudflare.com/r2/get-started/#4-bind-your-bucket-to-a-worker) to copy your Account ID.
-    * **Access Key Id**
-        * See [this](https://developers.cloudflare.com/r2/platform/s3-compatibility/tokens) on how to generate an access key.
-    * **Secret Access Key**
-        * Corresponding key to the above key id.
-    * **R2 Bucket Name**
-        * See [this](https://developers.cloudflare.com/r2/get-started/#3-create-your-bucket) to create an R2 bucket or you can create bucket via R2 module of [dashboard](https://dash.cloudflare.com).
-    * **R2 Bucket Path**
-        * Subdirectory under the above bucket to sync the data into.
-    * **R2 Path Format**
-        *  Additional string format on how to store data under R2 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}
-           _${EPOCH}_`.
-    * **R2 Filename pattern**
-        * The pattern allows you to set the file-name format for the R2 staging file(s), next placeholders combinations are currently supported: 
-          {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
+   - **Account Id**
+     - See [this](https://developers.cloudflare.com/r2/get-started/#4-bind-your-bucket-to-a-worker) to copy your Account ID.
+   - **Access Key Id**
+     - See [this](https://developers.cloudflare.com/r2/platform/s3-compatibility/tokens) on how to generate an access key.
+   - **Secret Access Key**
+     - Corresponding key to the above key id.
+   - **R2 Bucket Name**
+     - See [this](https://developers.cloudflare.com/r2/get-started/#3-create-your-bucket) to create an R2 bucket or you can create bucket via R2 module of [dashboard](https://dash.cloudflare.com).
+   - **R2 Bucket Path**
+     - Subdirectory under the above bucket to sync the data into.
+   - **R2 Path Format** - Additional string format on how to store data under R2 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}
+_${EPOCH}_`.
+   - **R2 Filename pattern**
+     - The pattern allows you to set the file-name format for the R2 staging file(s), next placeholders combinations are currently supported:
+       {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
 5. Click `Set up destination`.
 
 **For Airbyte OSS:**
@@ -54,25 +55,25 @@ to create an S3 bucket, or you can create bucket via R2 module of [dashboard](ht
 2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**.
 3. On the destination setup page, select **R2** from the Destination type dropdown and enter a name for this connector.
 4. Configure fields:
-    * **Account Id**
-        * See [this](https://developers.cloudflare.com/r2/get-started/#4-bind-your-bucket-to-a-worker) to copy your Account ID.
-    * **Access Key Id**
-        * See [this](https://developers.cloudflare.com/r2/platform/s3-compatibility/tokens) on how to generate an access key.
-    * **Secret Access Key**
-        * Corresponding key to the above key id.
-    * Make sure your R2 bucket is accessible from the machine running Airbyte.
-       * This depends on your networking setup.
-       * The easiest way to verify if Airbyte is able to connect to your R2 bucket is via the check connection tool in the UI.
-    * **R2 Bucket Name**
-        * See [this](https://developers.cloudflare.com/r2/get-started/#3-create-your-bucket) to create an R2 bucket or you can create bucket via R2 module of [dashboard](https://dash.cloudflare.com).
-    * **R2 Bucket Path**
-        * Subdirectory under the above bucket to sync the data into.
-    * **R2 Path Format**
-        *  Additional string format on how to store data under R2 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}
-           _${EPOCH}_`.
-    * **R2 Filename pattern**
-        * The pattern allows you to set the file-name format for the R2 staging file(s), next placeholders combinations are currently supported:
-          {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
+
+   - **Account Id**
+     - See [this](https://developers.cloudflare.com/r2/get-started/#4-bind-your-bucket-to-a-worker) to copy your Account ID.
+   - **Access Key Id**
+     - See [this](https://developers.cloudflare.com/r2/platform/s3-compatibility/tokens) on how to generate an access key.
+   - **Secret Access Key**
+     - Corresponding key to the above key id.
+   - Make sure your R2 bucket is accessible from the machine running Airbyte.
+     - This depends on your networking setup.
+     - The easiest way to verify if Airbyte is able to connect to your R2 bucket is via the check connection tool in the UI.
+   - **R2 Bucket Name**
+     - See [this](https://developers.cloudflare.com/r2/get-started/#3-create-your-bucket) to create an R2 bucket or you can create bucket via R2 module of [dashboard](https://dash.cloudflare.com).
+   - **R2 Bucket Path**
+     - Subdirectory under the above bucket to sync the data into.
+   - **R2 Path Format** - Additional string format on how to store data under R2 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}
+_${EPOCH}_`.
+   - **R2 Filename pattern**
+     - The pattern allows you to set the file-name format for the R2 staging file(s), next placeholders combinations are currently supported:
+       {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
 
 5. Click `Set up destination`.
 
@@ -104,6 +105,7 @@ The rationales behind this naming pattern are:
 3. The upload time composes of a date part and millis part so that it is both readable and unique.
 
 But it is possible to further customize by using the available variables to format the bucket path:
+
 - `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace fields.
 - `${STREAM_NAME}`: Name of the stream
 - `${YEAR}`: Year in which the sync was writing the output data in.
@@ -117,6 +119,7 @@ But it is possible to further customize by using the available variables to form
 - `${UUID}`: random uuid string
 
 Note:
+
 - Multiple `/` characters in the R2 path are collapsed into a single `/` character.
 - If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise.
 
@@ -125,12 +128,12 @@ A data sync may create multiple files as the output files can be partitioned by
 
 ## Supported sync modes
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. |
-| Incremental - Append Sync | ✅ |  |
-| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. |
+| Feature                        | Support | Notes                                                                                |
+| :----------------------------- | :-----: | :----------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path. |
+| Incremental - Append Sync      |   ✅    |                                                                                      |
+| Incremental - Append + Deduped |   ❌    |                                                                                      |
+| Namespaces                     |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.          |
 
 The Airbyte R2 destination allows you to sync data to Cloudflare R2. Each stream is written to its own directory under the bucket.
 ⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated R2 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️
@@ -139,8 +142,8 @@ The Airbyte R2 destination allows you to sync data to Cloudflare R2. Each stream
 
 Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world.
 
-* Under Full Refresh Sync mode, old output files will be purged before new files are created.
-* Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
+- Under Full Refresh Sync mode, old output files will be purged before new files are created.
+- Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
 
 ### Avro
 
@@ -150,28 +153,28 @@ Each stream will be outputted to its dedicated directory according to the config
 
 Here is the available compression codecs:
 
-* No compression
-* `deflate`
-    * Compression level
-        * Range `[0, 9]`. Default to 0.
-        * Level 0: no compression & fastest.
-        * Level 9: best compression & slowest.
-* `bzip2`
-* `xz`
-    * Compression level
-        * Range `[0, 9]`. Default to 6.
-        * Level 0-3 are fast with medium compression.
-        * Level 4-6 are fairly slow with high compression.
-        * Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively.
-* `zstandard`
-    * Compression level
-        * Range `[-5, 22]`. Default to 3.
-        * Negative levels are 'fast' modes akin to `lz4` or `snappy`.
-        * Levels above 9 are generally for archival purposes.
-        * Levels above 18 use a lot of memory.
-    * Include checksum
-        * If set to `true`, a checksum will be included in each data block.
-* `snappy`
+- No compression
+- `deflate`
+  - Compression level
+    - Range `[0, 9]`. Default to 0.
+    - Level 0: no compression & fastest.
+    - Level 9: best compression & slowest.
+- `bzip2`
+- `xz`
+  - Compression level
+    - Range `[0, 9]`. Default to 6.
+    - Level 0-3 are fast with medium compression.
+    - Level 4-6 are fairly slow with high compression.
+    - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively.
+- `zstandard`
+  - Compression level
+    - Range `[-5, 22]`. Default to 3.
+    - Negative levels are 'fast' modes akin to `lz4` or `snappy`.
+    - Levels above 9 are generally for archival purposes.
+    - Levels above 18 use a lot of memory.
+  - Include checksum
+    - If set to `true`, a checksum will be included in each data block.
+- `snappy`
 
 #### Data schema
 
@@ -181,12 +184,12 @@ Under the hood, an Airbyte data stream in JSON schema is first converted to an A
 
 Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, an emission timestamp, and the data blob. With the CSV output, it is possible to normalize \(flatten\) the data blob to multiple columns.
 
-| Column | Condition | Description |
-| :--- | :--- | :--- |
-| `_airbyte_ab_id` | Always exists | A uuid assigned by Airbyte to each processed record. |
-| `_airbyte_emitted_at` | Always exists. | A timestamp representing when the event was pulled from the data source. |
-| `_airbyte_data` | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |  |
-| root level fields | When root level normalization \(flattening\) is selected, the root level fields are expanded. |  |
+| Column                | Condition                                                                                         | Description                                                              |
+| :-------------------- | :------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- |
+| `_airbyte_ab_id`      | Always exists                                                                                     | A uuid assigned by Airbyte to each processed record.                     |
+| `_airbyte_emitted_at` | Always exists.                                                                                    | A timestamp representing when the event was pulled from the data source. |
+| `_airbyte_data`       | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |                                                                          |
+| root level fields     | When root level normalization \(flattening\) is selected, the root level fields are expanded.     |                                                                          |
 
 For example, given the following json object from a source:
 
@@ -202,15 +205,15 @@ For example, given the following json object from a source:
 
 With no normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `_airbyte_data` |
-| :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `_airbyte_data`                                                |
+| :------------------------------------- | :-------------------- | :------------------------------------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
 
 With root level normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `user_id` | `name` |
-| :--- | :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | 123 | `{ "first": "John", "last": "Doe" }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `user_id` | `name`                               |
+| :------------------------------------- | :-------------------- | :-------- | :----------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | 123       | `{ "first": "John", "last": "Doe" }` |
 
 Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.csv.gz`).
 
@@ -262,14 +265,14 @@ Output files can be compressed. The default option is GZIP compression. If compr
 
 The following configuration is available to configure the Parquet output:
 
-| Parameter | Type | Default | Description |
-| :--- | :---: | :---: | :--- |
-| `compression_codec` | enum | `UNCOMPRESSED` | **Compression algorithm**. Available candidates are: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `BROTLI`, `LZ4`, and `ZSTD`. |
-| `block_size_mb` | integer | 128 \(MB\) | **Block size \(row group size\)** in MB. This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. |
-| `max_padding_size_mb` | integer | 8 \(MB\) | **Max padding size** in MB. This is the maximum size allowed as padding to align row groups. This is also the minimum size of a row group. |
-| `page_size_kb` | integer | 1024 \(KB\) | **Page size** in KB. The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. |
-| `dictionary_page_size_kb` | integer | 1024 \(KB\) | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. |
-| `dictionary_encoding` | boolean | `true` | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on. |
+| Parameter                 |  Type   |    Default     | Description                                                                                                                                                                                                                       |
+| :------------------------ | :-----: | :------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `compression_codec`       |  enum   | `UNCOMPRESSED` | **Compression algorithm**. Available candidates are: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `BROTLI`, `LZ4`, and `ZSTD`.                                                                                                        |
+| `block_size_mb`           | integer |   128 \(MB\)   | **Block size \(row group size\)** in MB. This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. |
+| `max_padding_size_mb`     | integer |    8 \(MB\)    | **Max padding size** in MB. This is the maximum size allowed as padding to align row groups. This is also the minimum size of a row group.                                                                                        |
+| `page_size_kb`            | integer |  1024 \(KB\)   | **Page size** in KB. The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. |
+| `dictionary_page_size_kb` | integer |  1024 \(KB\)   | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary.                                     |
+| `dictionary_encoding`     | boolean |     `true`     | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on.                                                                                                                                        |
 
 These parameters are related to the `ParquetOutputFormat`. See the [Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) for more details. Also see [Parquet documentation](https://parquet.apache.org/docs/file-format/configurations/) for their recommended configurations \(512 - 1024 MB block size, 8 KB page size\).
 
@@ -279,6 +282,6 @@ Under the hood, an Airbyte data stream in JSON schema is first converted to an A
 
 ## CHANGELOG
 
-| Version | Date       | Pull Request                                               | Subject |
-|:--------|:-----------|:-----------------------------------------------------------|:--------|
+| Version | Date       | Pull Request                                               | Subject          |
+| :------ | :--------- | :--------------------------------------------------------- | :--------------- |
 | 0.1.0   | 2022-09-25 | [\#15296](https://github.com/airbytehq/airbyte/pull/15296) | Initial release. |
diff --git a/docs/integrations/destinations/rabbitmq.md b/docs/integrations/destinations/rabbitmq.md
index d448fd4788d9..b1cb1a730236 100644
--- a/docs/integrations/destinations/rabbitmq.md
+++ b/docs/integrations/destinations/rabbitmq.md
@@ -10,22 +10,22 @@ The RabbitMQ destination allows you to send/stream data to a RabbitMQ routing ke
 
 Each stream will be output a RabbitMQ message with properties. The message properties will be
 
-* `content_type`: set as `application/json`
-* `headers`: message headers, which include:
-  * `stream`: the name of stream where the data is coming from
-  * `namespace`: namespace if available from the stream
-  * `emitted_at`: timestamp the `AirbyteRecord` was emitted at.
+- `content_type`: set as `application/json`
+- `headers`: message headers, which include:
+  - `stream`: the name of stream where the data is coming from
+  - `namespace`: namespace if available from the stream
+  - `emitted_at`: timestamp the `AirbyteRecord` was emitted at.
 
 The `AirbyteRecord` data will be serialized as JSON and set as the RabbitMQ message body.
 
 #### Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ## Getting started
 
@@ -33,16 +33,15 @@ The `AirbyteRecord` data will be serialized as JSON and set as the RabbitMQ mess
 
 To use the RabbitMQ destination, you'll need:
 
-* A RabbitMQ host and credentials (username/password) to publish messages, if required.
-* A RabbitMQ routing key.
-* RabbitMQ exchange is optional. If specified, a binding between exchange and routing key is required.
-* RabbitMQ port is optional (it defaults to 5672).
-* RabbitMQ virtual host is also optional.
+- A RabbitMQ host and credentials (username/password) to publish messages, if required.
+- A RabbitMQ routing key.
+- RabbitMQ exchange is optional. If specified, a binding between exchange and routing key is required.
+- RabbitMQ port is optional (it defaults to 5672).
+- RabbitMQ virtual host is also optional.
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.1 | 2022-09-09 | [16528](https://github.com/airbytehq/airbyte/pull/16528) | Marked password field in spec as airbyte_secret |
-| 0.1.0 | October 29, 2021 | [\#7560](https://github.com/airbytehq/airbyte/pull/7560) | Initial release |
-
+| Version | Date             | Pull Request                                             | Subject                                         |
+| :------ | :--------------- | :------------------------------------------------------- | :---------------------------------------------- |
+| 0.1.1   | 2022-09-09       | [16528](https://github.com/airbytehq/airbyte/pull/16528) | Marked password field in spec as airbyte_secret |
+| 0.1.0   | October 29, 2021 | [\#7560](https://github.com/airbytehq/airbyte/pull/7560) | Initial release                                 |
diff --git a/docs/integrations/destinations/redis.md b/docs/integrations/destinations/redis.md
index e46a449a8f54..fd07573388e4 100644
--- a/docs/integrations/destinations/redis.md
+++ b/docs/integrations/destinations/redis.md
@@ -12,46 +12,44 @@ For the **_hash_** implementation as a Redis data type the keys and the hashes a
 **_key_**:
 
     namespace:stream:id
-    
-**_hash_**:
 
-* `_airbyte_ab_id`: Sequential id for a given key generated by using the INCR Redis command.
-* `_airbyte_emitted_at`: a timestamp representing when the event was received from the data source.
-* `_airbyte_data`: a json text/object representing the data that was received from the data source.
+**_hash_**:
 
+- `_airbyte_ab_id`: Sequential id for a given key generated by using the INCR Redis command.
+- `_airbyte_emitted_at`: a timestamp representing when the event was received from the data source.
+- `_airbyte_data`: a json text/object representing the data that was received from the data source.
 
 ### Features
 
-| Feature                       | Support| Notes                                                                          |
-|:------------------------------| :-----:|:-------------------------------------------------------------------------------|
-| Full Refresh Sync             | ✅     | Existing keys in the Redis cache are deleted and replaced with the new keys.   |
-| Incremental - Append Sync     | ✅     | New keys are inserted in the same keyspace without touching the existing keys. |
-| Incremental - Deduped History | ❌     |                                                                                |
-| Namespaces                    | ✅     | Namespaces will be used to determine the correct Redis key.                    |
-| SSH Tunnel Connection         | ✅     |                                                                                |
-| SSL connection                | ✅     |                                                                                |
-
+| Feature                        | Support | Notes                                                                          |
+| :----------------------------- | :-----: | :----------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Existing keys in the Redis cache are deleted and replaced with the new keys.   |
+| Incremental - Append Sync      |   ✅    | New keys are inserted in the same keyspace without touching the existing keys. |
+| Incremental - Append + Deduped |   ❌    |                                                                                |
+| Namespaces                     |   ✅    | Namespaces will be used to determine the correct Redis key.                    |
+| SSH Tunnel Connection          |   ✅    |                                                                                |
+| SSL connection                 |   ✅    |                                                                                |
 
 ### Performance considerations
 
-As long as you have the necessary memory capacity for your cache, Redis should be able to handle even millions of records without any issues since the data is stored in-memory with the option to 
+As long as you have the necessary memory capacity for your cache, Redis should be able to handle even millions of records without any issues since the data is stored in-memory with the option to
 save snapshots periodically on disk.
 
 ## Getting started
 
 ### Requirements
 
-* The connector is fully compatible with redis 2.8.x, 3.x.x and above
-* Configuration
-    * **_host_**: Hostname or address of the Redis server where to connect.
-    * **_port_**: Port of the Redis server where to connect.
-    * **_username_**: Username for authenticating with the Redis server.
-    * **_password_**: Password for authenticating with the Redis server.
-    * **_cache_type_**: Redis cache/data type to use when storing the incoming messages. i.e hash,set,list,stream,etc.
-* SSL toggle the switch to connect using SSL
-*  For SSL Modes, select:
-    - **disable** to disable encrypted communication between Airbyte and the source
-    - **verify-full** to always require encryption and verify the identity of the source
+- The connector is fully compatible with redis 2.8.x, 3.x.x and above
+- Configuration
+  - **_host_**: Hostname or address of the Redis server where to connect.
+  - **_port_**: Port of the Redis server where to connect.
+  - **_username_**: Username for authenticating with the Redis server.
+  - **_password_**: Password for authenticating with the Redis server.
+  - **_cache_type_**: Redis cache/data type to use when storing the incoming messages. i.e hash,set,list,stream,etc.
+- SSL toggle the switch to connect using SSL
+- For SSL Modes, select:
+  - **disable** to disable encrypted communication between Airbyte and the source
+  - **verify-full** to always require encryption and verify the identity of the source
 
 ### Setup guide
 
@@ -59,7 +57,7 @@ save snapshots periodically on disk.
 
 ## Changelog
 
-| Version | Date       | Pull Request                                               | Subject          |
-|:--------|:-----------|:-----------------------------------------------------------|:-----------------|
-| 0.1.4   | 2022-10-25 | [\#18358](https://github.com/airbytehq/airbyte/pull/18358) | TLS support      |
-| 0.1.3   | 2022-10-18 | [\#17951](https://github.com/airbytehq/airbyte/pull/17951) | Add SSH support  |
+| Version | Date       | Pull Request                                               | Subject         |
+| :------ | :--------- | :--------------------------------------------------------- | :-------------- |
+| 0.1.4   | 2022-10-25 | [\#18358](https://github.com/airbytehq/airbyte/pull/18358) | TLS support     |
+| 0.1.3   | 2022-10-18 | [\#17951](https://github.com/airbytehq/airbyte/pull/17951) | Add SSH support |
diff --git a/docs/integrations/destinations/redpanda.md b/docs/integrations/destinations/redpanda.md
index e1f4496a2e6e..410b17d453e8 100644
--- a/docs/integrations/destinations/redpanda.md
+++ b/docs/integrations/destinations/redpanda.md
@@ -27,12 +27,12 @@ Each record will contain in its key the uuid assigned by Airbyte, and in the val
 
 This section should contain a table with the following format:
 
-| Feature                       | Supported?\(Yes/No\) | Notes                                                                                        |
-| :---------------------------- | :------------------- | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             | No                   |                                                                                              |
-| Incremental - Append Sync     | Yes                  |                                                                                              |
-| Incremental - Deduped History | No                   | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    | Yes                  |                                                                                              |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | No                   |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ### Performance considerations
 
diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md
index 19f3a56f2823..4dfedfd02f34 100644
--- a/docs/integrations/destinations/redshift.md
+++ b/docs/integrations/destinations/redshift.md
@@ -11,14 +11,15 @@ This Redshift destination connector has two replication strategies:
 1. INSERT: Replicates data via SQL INSERT queries. This is built on top of the destination-jdbc code base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Mulesoft [here](https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42) as described in Redshift documentation [here](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-install.html). **Not recommended for production workloads as this does not scale well**.
 
 For INSERT strategy:
-* **Host**
-* **Port**
-* **Username**
-* **Password**
-* **Schema**
-* **Database**
-    * This database needs to exist within the cluster provided.
-* **JDBC URL Params** (optional)
+
+- **Host**
+- **Port**
+- **Username**
+- **Password**
+- **Schema**
+- **Database**
+  - This database needs to exist within the cluster provided.
+- **JDBC URL Params** (optional)
 
 2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is the recommended loading approach described by Redshift [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_loading-data-best-practices.html). Requires an S3 bucket and credentials.
 
@@ -26,25 +27,26 @@ Airbyte automatically picks an approach depending on the given configuration - i
 
 For COPY strategy:
 
-* **S3 Bucket Name**
-    * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket.
-* **S3 Bucket Region**
-    * Place the S3 bucket and the Redshift cluster in the same region to save on networking costs.
-* **Access Key Id**
-    * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
-    * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket.
-* **Secret Access Key**
-    * Corresponding key to the above key id.
-* **Part Size**
-    * Affects the size limit of an individual Redshift table. Optional. Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care.
-* **S3 Filename pattern**
-    * The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
+- **S3 Bucket Name**
+  - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket.
+- **S3 Bucket Region**
+  - Place the S3 bucket and the Redshift cluster in the same region to save on networking costs.
+- **Access Key Id**
+  - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
+  - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket.
+- **Secret Access Key**
+  - Corresponding key to the above key id.
+- **Part Size**
+  - Affects the size limit of an individual Redshift table. Optional. Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care.
+- **S3 Filename pattern**
+  - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
 
 Optional parameters:
-* **Bucket Path**
-    * The directory within the S3 bucket to place the staging data. For example, if you set this to `yourFavoriteSubdirectory`, we will place the staging data inside `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory.
-* **Purge Staging Data**
-    * Whether to delete the staging files from S3 after completing the sync. Specifically, the connector will create CSV files named `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`.
+
+- **Bucket Path**
+  - The directory within the S3 bucket to place the staging data. For example, if you set this to `yourFavoriteSubdirectory`, we will place the staging data inside `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory.
+- **Purge Staging Data**
+  - Whether to delete the staging files from S3 after completing the sync. Specifically, the connector will create CSV files named `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`.
 
 NOTE: S3 staging does not use the SSH Tunnel option, if configured. SSH Tunnel supports the SQL connection only. S3 is secured through public HTTPS access only.
 
@@ -57,12 +59,14 @@ NOTE: S3 staging does not use the SSH Tunnel option, if configured. SSH Tunnel s
 4. (Optional) [Allow](https://aws.amazon.com/premiumsupport/knowledge-center/cannot-connect-redshift-cluster/) connections from Airbyte to your Redshift cluster \(if they exist in separate VPCs\)
 5. (Optional) [Create](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) a staging S3 bucket \(for the COPY strategy\).
 6. Create a user with at least create table permissions for the schema. If the schema does not exist you need to add permissions for that, too. Something like this:
+
 ```
 GRANT CREATE ON DATABASE database_name TO airflow_user; -- add create schema permission
 GRANT usage, create on schema my_schema TO airflow_user; -- add create table permission
 ```
 
 ### Optional Use of SSH Bastion Host
+
 This connector supports the use of a Bastion host as a gateway to a private Redshift cluster via SSH Tunneling.
 Setup of the host is beyond the scope of this document but several tutorials are available online to fascilitate this task.
 Enter the bastion host, port and credentials in the destination configuration.
@@ -85,13 +89,13 @@ Enter the bastion host, port and credentials in the destination configuration.
 4. Fill in all the required fields to use the INSERT or COPY strategy
 5. Click `Set up destination`.
 
-
 ## Supported sync modes
 
 The Redshift destination connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode):
+
 - Full Refresh
 - Incremental - Append Sync
-- Incremental - Deduped History
+- Incremental - Append + Deduped
 
 ## Performance considerations
 
@@ -106,10 +110,10 @@ From [Redshift Names & Identifiers](https://docs.aws.amazon.com/redshift/latest/
 
 #### Standard Identifiers
 
-* Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte character two to four bytes long.
-* Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar signs, or UTF-8 multibyte characters two to four bytes long.
-* Be between 1 and 127 bytes in length, not including quotation marks for delimited identifiers.
-* Contain no quotation marks and no spaces.
+- Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte character two to four bytes long.
+- Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar signs, or UTF-8 multibyte characters two to four bytes long.
+- Be between 1 and 127 bytes in length, not including quotation marks for delimited identifiers.
+- Contain no quotation marks and no spaces.
 
 #### Delimited Identifiers
 
@@ -130,28 +134,36 @@ All Redshift connections are encrypted using SSL
 
 Each stream will be output into its own raw table in Redshift. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Redshift is `VARCHAR`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Redshift is `TIMESTAMP WITH TIME ZONE`.
-* `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is `SUPER`.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Redshift is `VARCHAR`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Redshift is `TIMESTAMP WITH TIME ZONE`.
+- `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is `SUPER`.
 
 ## Data type mapping
 
-| Redshift Type | Airbyte Type | Notes |
-| :--- | :--- | :--- |
-| `boolean` | `boolean` | |
-| `int` | `integer` | |
-| `float` | `number` | |
-| `varchar` | `string` | |
-| `date/varchar` | `date` | |
-| `time/varchar` | `time` | |
-| `timestamptz/varchar` | `timestamp_with_timezone` | |
-| `varchar` | `array` | |
-| `varchar` | `object` | |
+| Redshift Type         | Airbyte Type              | Notes |
+| :-------------------- | :------------------------ | :---- |
+| `boolean`             | `boolean`                 |       |
+| `int`                 | `integer`                 |       |
+| `float`               | `number`                  |       |
+| `varchar`             | `string`                  |       |
+| `date/varchar`        | `date`                    |       |
+| `time/varchar`        | `time`                    |       |
+| `timestamptz/varchar` | `timestamp_with_timezone` |       |
+| `varchar`             | `array`                   |       |
+| `varchar`             | `object`                  |       |
 
 ## Changelog
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                                                                          |
 |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.6.5   | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async)                                                                                                                                        |
+| 0.6.4   | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging                                                                                                                                                                                     |
+| 0.6.3   | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring                                                                                                                                                                                        |
+| 0.6.2   | 2023-07-24 | [\#28618](https://github.com/airbytehq/airbyte/pull/28618) | Add hooks in preparation for destinations v2 implementation                                                                                                                                                      |
+| 0.6.1   | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild                                                                                                                                                                             |
+| 0.6.0   | 2023-06-27 | [\#27993](https://github.com/airbytehq/airbyte/pull/27993) | destination-redshift will fail syncs if records or properties are too large, rather than silently skipping records and succeeding                                                                                |
+| 0.5.0   | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                                                                                                                             |
+| 0.4.9   | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size                                                                                                                                                                                                |
 | 0.4.8   | 2023-05-17 | [\#26165](https://github.com/airbytehq/airbyte/pull/26165) | Internal code change for future development (install normalization packages inside connector)                                                                                                                    |
 | 0.4.7   | 2023-05-01 | [\#25698](https://github.com/airbytehq/airbyte/pull/25698) | Remove old VARCHAR to SUPER migration Java functionality                                                                                                                                                         |
 | 0.4.6   | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup                                                                                                                                                                                            |
@@ -165,7 +177,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c
 | 0.3.55  | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging                                                                                                                                                         |
 | 0.3.54  | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions                                                                                                                                                                  |
 | 0.3.53  | 2023-01-03 | [\#17273](https://github.com/airbytehq/airbyte/pull/17273) | Flatten JSON arrays to fix maximum size check for SUPER field                                                                                                                                                    |
-| 0.3.52  | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead)                                                                                      |
+| 0.3.52  | 2022-12-30 | [\#20879](https://github.com/airbytehq/airbyte/pull/20879) | Added configurable parameter for number of file buffers (⛔ this version has a bug and will not work; use `0.3.56` instead)                                                                                       |
 | 0.3.51  | 2022-10-26 | [\#18434](https://github.com/airbytehq/airbyte/pull/18434) | Fix empty S3 bucket path handling                                                                                                                                                                                |
 | 0.3.50  | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage                                                                                                                                                                                   |
 | 0.3.49  | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields)                                                                                         |
@@ -176,7 +188,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c
 | 0.3.44  | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging                                                                                                                                      |
 | 0.3.43  | 2022-06-24 | [\#13690](https://github.com/airbytehq/airbyte/pull/13690) | Improved discovery for NOT SUPER column                                                                                                                                                                          |
 | 0.3.42  | 2022-06-21 | [\#14013](https://github.com/airbytehq/airbyte/pull/14013) | Add an option to use encryption with staging in Redshift Destination                                                                                                                                             |
-| 0.3.40  | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager                                                                                                                          |
+| 0.3.40  | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART\*SIZE_MB fields from connectors based on StreamTransferManager                                                                                                                         |
 | 0.3.39  | 2022-06-02 | [\#13415](https://github.com/airbytehq/airbyte/pull/13415) | Add dropdown to select Uploading Method. 
     **PLEASE NOTICE**: After this update your **uploading method** will be set to **Standard**, you will need to reconfigure the method to use **S3 Staging** again. |
 | 0.3.37  | 2022-05-23 | [\#13090](https://github.com/airbytehq/airbyte/pull/13090) | Removed redshiftDataTmpTableMode. Some refactoring.                                                                                                                                                              |
 | 0.3.36  | 2022-05-23 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                                                                                                                                           |
@@ -184,8 +196,8 @@ Each stream will be output into its own raw table in Redshift. Each table will c
 | 0.3.34  | 2022-05-16 | [\#12869](https://github.com/airbytehq/airbyte/pull/12869) | Fixed NPE in S3 staging check                                                                                                                                                                                    |
 | 0.3.33  | 2022-05-04 | [\#12601](https://github.com/airbytehq/airbyte/pull/12601) | Apply buffering strategy for S3 staging                                                                                                                                                                          |
 | 0.3.32  | 2022-04-20 | [\#12085](https://github.com/airbytehq/airbyte/pull/12085) | Fixed bug with switching between INSERT and COPY config                                                                                                                                                          |
-| 0.3.31  | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064) | Added option to support SUPER datatype in _airbyte_raw_** table                                                                                                                                                  |
-| 0.3.29  | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name                                                                                                                                                                             |                                                                   |
+| 0.3.31  | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064) | Added option to support SUPER datatype in \_airbyte_raw\*\*\* table                                                                                                                                              |
+| 0.3.29  | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name                                                                                                                                                                             |     |
 | 0.3.28  | 2022-03-18 | [\#11254](https://github.com/airbytehq/airbyte/pull/11254) | Fixed missing records during S3 staging                                                                                                                                                                          |
 | 0.3.27  | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                                                                                                                                |
 | 0.3.25  | 2022-02-14 | [\#9920](https://github.com/airbytehq/airbyte/pull/9920)   | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file.                                                              |
diff --git a/docs/integrations/destinations/rockset.md b/docs/integrations/destinations/rockset.md
index 4e2597bcf0d3..0ab1709a68b6 100644
--- a/docs/integrations/destinations/rockset.md
+++ b/docs/integrations/destinations/rockset.md
@@ -6,12 +6,12 @@
 
 ## Features
 
-| Feature                       | Support |
-| :---------------------------- | :-----: |
-| Full Refresh Sync             |   ✅    |
-| Incremental - Append Sync     |   ✅    |
-| Incremental - Deduped History |   ❌    |
-| Namespaces                    |   ❌    |
+| Feature                        | Support |
+| :----------------------------- | :-----: |
+| Full Refresh Sync              |   ✅    |
+| Incremental - Append Sync      |   ✅    |
+| Incremental - Append + Deduped |   ❌    |
+| Namespaces                     |   ❌    |
 
 ## Troubleshooting
 
diff --git a/docs/integrations/destinations/s3-glue.md b/docs/integrations/destinations/s3-glue.md
index 92ab54fb9fe0..5e66cf7d6e70 100644
--- a/docs/integrations/destinations/s3-glue.md
+++ b/docs/integrations/destinations/s3-glue.md
@@ -175,12 +175,12 @@ A data sync may create multiple files as the output files can be partitioned by
 
 ## Supported sync modes
 
-| Feature                       | Support | Notes                                                                                        |
-| :---------------------------- | :-----: | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path.         |
-| Incremental - Append Sync     |   ✅    |                                                                                              |
-| Incremental - Deduped History |   ❌    | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.                  |
+| Feature                        | Support | Notes                                                                                                                                                                                               |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path.                                                                                                                |
+| Incremental - Append Sync      |   ✅    | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/understanding-airbyte/connections/incremental-append#inclusive-cursors) |
+| Incremental - Append + Deduped |   ❌    |                                                                                                                                                                                                     |
+| Namespaces                     |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.                                                                                                                         |
 
 The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket.
 ⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️
@@ -244,12 +244,12 @@ Output files can be compressed. The default option is GZIP compression. If compr
 ## CHANGELOG
 
 | Version | Date       | Pull Request                                             | Subject                                                                                 |
-|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------- |
 | 0.1.7   | 2023-05-01 | [25724](https://github.com/airbytehq/airbyte/pull/25724) | Fix decimal type creation syntax to avoid overflow                                      |
 | 0.1.6   | 2023-04-13 | [25178](https://github.com/airbytehq/airbyte/pull/25178) | Fix decimal precision and scale to allow for a wider range of numeric values            |
 | 0.1.5   | 2023-04-11 | [25048](https://github.com/airbytehq/airbyte/pull/25048) | Fix config schema to support new JSONL flattening configuration interface               |
 | 0.1.4   | 2023-03-10 | [23950](https://github.com/airbytehq/airbyte/pull/23950) | Fix schema syntax error for struct fields and handle missing `items` in array fields    |
-| 0.1.3   | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for _ab_emitted_at column in table definition                             |
+| 0.1.3   | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for \_ab_emitted_at column in table definition                            |
 | 0.1.2   | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition |
 | 0.1.1   | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema                                                      |
 | 0.1.0   | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit                                                                          |
diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md
index 5a3c768d64b6..d7c5291fca0c 100644
--- a/docs/integrations/destinations/s3.md
+++ b/docs/integrations/destinations/s3.md
@@ -2,18 +2,15 @@
 
 This page guides you through the process of setting up the S3 destination connector.
 
-:::info
-Cloud storage may incur egress costs. Egress refers to data that is transferred out of the cloud storage system, such as when you download files or access them from a different location. For more information, see the [Amazon S3 pricing guide](https://aws.amazon.com/s3/pricing/).
-:::
-
 ## Prerequisites
 
 List of required fields:
-* **Access Key ID**
-* **Secret Access Key**
-* **S3 Bucket Name**
-* **S3 Bucket Path**
-* **S3 Bucket Region**
+
+- **Access Key ID**
+- **Secret Access Key**
+- **S3 Bucket Name**
+- **S3 Bucket Path**
+- **S3 Bucket Region**
 
 1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\).
 2. An S3 bucket with credentials or an instance profile with read/write permissions configured for the host (ec2, eks).
@@ -23,7 +20,6 @@ List of required fields:
 
 ### Step 1: Set up S3
 
-
 [Sign in](https://console.aws.amazon.com/iam/) to your AWS account.
 Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide).
 
@@ -34,90 +30,93 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S
 ### Step 2: Set up the S3 destination connector in Airbyte
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
 2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**.
 3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector.
 4. Configure fields:
-    * **Access Key Id**
-        * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
-        * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket.
-    * **Secret Access Key**
-        * Corresponding key to the above key id.
-    * **S3 Bucket Name**
-        * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket.
-    * **S3 Bucket Path**
-        * Subdirectory under the above bucket to sync the data into.
-    * **S3 Bucket Region**
-        * See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.
-    * **S3 Path Format**
-        *  Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`.
-    * **S3 Endpoint**
-        * Leave empty if using AWS S3, fill in S3 URL if using Minio S3.
-    * **S3 Filename pattern**
-      * The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't be recognized.
+   - **Access Key Id**
+     - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
+     - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket.
+   - **Secret Access Key**
+     - Corresponding key to the above key id.
+   - **S3 Bucket Name**
+     - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket.
+   - **S3 Bucket Path**
+     - Subdirectory under the above bucket to sync the data into.
+   - **S3 Bucket Region**
+     - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.
+   - **S3 Path Format**
+     - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`.
+   - **S3 Endpoint**
+     - Leave empty if using AWS S3, fill in S3 URL if using Minio S3.
+   - **S3 Filename pattern**
+     - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't be recognized.
 5. Click `Set up destination`.
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Go to local Airbyte page.
 2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**.
 3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector.
 4. Configure fields:
-    * **Access Key Id**
-        * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
-        * See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile.
-        * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket.
-        * If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile.
-    * **Secret Access Key**
-        * Corresponding key to the above key id.
-    * Make sure your S3 bucket is accessible from the machine running Airbyte.
-        * This depends on your networking setup.
-        * You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html).
-        * If you use instance profile authentication, make sure the role has permission to read/write on the bucket.
-        * The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI.
-    * **S3 Bucket Name**
-        * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket.
-    * **S3 Bucket Path**
-        * Subdirectory under the above bucket to sync the data into.
-    * **S3 Bucket Region**
-        * See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.
-    * **S3 Path Format**
-        * Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`.
-    * **S3 Endpoint**
-        * Leave empty if using AWS S3, fill in S3 URL if using Minio S3.
-   * **S3 Filename pattern**
-        * The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
-
+   _ **Access Key Id**
+   _ See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key.
+   _ See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile.
+   _ We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket.
+   _ If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile.
+   _ **Secret Access Key**
+   _ Corresponding key to the above key id.
+   _ Make sure your S3 bucket is accessible from the machine running Airbyte.
+   _ This depends on your networking setup.
+   _ You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html).
+   _ If you use instance profile authentication, make sure the role has permission to read/write on the bucket.
+   _ The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI.
+   _ **S3 Bucket Name**
+   _ See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket.
+   _ **S3 Bucket Path**
+   _ Subdirectory under the above bucket to sync the data into.
+   _ **S3 Bucket Region**
+   _ See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes.
+   _ **S3 Path Format**
+   _ Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`.
+   _ **S3 Endpoint**
+   _ Leave empty if using AWS S3, fill in S3 URL if using Minio S3.
+
+   - **S3 Filename pattern** \* The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized.
+   
 
 5. Click `Set up destination`.
 
 In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies to use:
+
 ```json
 {
   "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-             "Action": [
-                "s3:PutObject",
-                "s3:GetObject",
-                "s3:DeleteObject",
-                "s3:PutObjectAcl",
-                "s3:ListBucket",
-                "s3:ListBucketMultipartUploads",
-                "s3:AbortMultipartUpload",
-                "s3:GetBucketLocation"
-             ],
-            "Resource": [
-                "arn:aws:s3:::YOUR_BUCKET_NAME/*",
-                "arn:aws:s3:::YOUR_BUCKET_NAME"
-            ]
-        }
-    ]
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:PutObject",
+        "s3:GetObject",
+        "s3:DeleteObject",
+        "s3:PutObjectAcl",
+        "s3:ListBucket",
+        "s3:ListBucketMultipartUploads",
+        "s3:AbortMultipartUpload",
+        "s3:GetBucketLocation"
+      ],
+      "Resource": [
+        "arn:aws:s3:::YOUR_BUCKET_NAME/*",
+        "arn:aws:s3:::YOUR_BUCKET_NAME"
+      ]
+    }
+  ]
 }
 ```
 
@@ -149,6 +148,7 @@ The rationales behind this naming pattern are:
 3. The upload time composes of a date part and millis part so that it is both readable and unique.
 
 But it is possible to further customize by using the available variables to format the bucket path:
+
 - `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace fields.
 - `${STREAM_NAME}`: Name of the stream
 - `${YEAR}`: Year in which the sync was writing the output data in.
@@ -162,6 +162,7 @@ But it is possible to further customize by using the available variables to form
 - `${UUID}`: random uuid string
 
 Note:
+
 - Multiple `/` characters in the S3 path are collapsed into a single `/` character.
 - If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise.
 
@@ -170,22 +171,23 @@ A data sync may create multiple files as the output files can be partitioned by
 
 ## Supported sync modes
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. |
-| Incremental - Append Sync | ✅ |  |
-| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. |
+| Feature                        | Support | Notes                                                                                                                                                                                               |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path.                                                                                                                |
+| Incremental - Append Sync      |   ✅    | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/understanding-airbyte/connections/incremental-append#inclusive-cursors) |
+| Incremental - Append + Deduped |   ❌    |                                                                                                                                                                                                     |
+| Namespaces                     |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.                                                                                                                         |
 
 The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket.
+
 ⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️
 
 ## Supported Output schema
 
 Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world.
 
-* Under Full Refresh Sync mode, old output files will be purged before new files are created.
-* Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
+- Under Full Refresh Sync mode, old output files will be purged before new files are created.
+- Under Incremental - Append Sync mode, new output files will be added that only contain the new data.
 
 ### Avro
 
@@ -195,28 +197,28 @@ Each stream will be outputted to its dedicated directory according to the config
 
 Here is the available compression codecs:
 
-* No compression
-* `deflate`
-  * Compression level
-    * Range `[0, 9]`. Default to 0.
-    * Level 0: no compression & fastest.
-    * Level 9: best compression & slowest.
-* `bzip2`
-* `xz`
-  * Compression level
-    * Range `[0, 9]`. Default to 6.
-    * Level 0-3 are fast with medium compression.
-    * Level 4-6 are fairly slow with high compression.
-    * Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively.
-* `zstandard`
-  * Compression level
-    * Range `[-5, 22]`. Default to 3.
-    * Negative levels are 'fast' modes akin to `lz4` or `snappy`.
-    * Levels above 9 are generally for archival purposes.
-    * Levels above 18 use a lot of memory.
-  * Include checksum
-    * If set to `true`, a checksum will be included in each data block.
-* `snappy`
+- No compression
+- `deflate`
+  - Compression level
+    - Range `[0, 9]`. Default to 0.
+    - Level 0: no compression & fastest.
+    - Level 9: best compression & slowest.
+- `bzip2`
+- `xz`
+  - Compression level
+    - Range `[0, 9]`. Default to 6.
+    - Level 0-3 are fast with medium compression.
+    - Level 4-6 are fairly slow with high compression.
+    - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively.
+- `zstandard`
+  - Compression level
+    - Range `[-5, 22]`. Default to 3.
+    - Negative levels are 'fast' modes akin to `lz4` or `snappy`.
+    - Levels above 9 are generally for archival purposes.
+    - Levels above 18 use a lot of memory.
+  - Include checksum
+    - If set to `true`, a checksum will be included in each data block.
+- `snappy`
 
 #### Data schema
 
@@ -226,12 +228,12 @@ Under the hood, an Airbyte data stream in JSON schema is first converted to an A
 
 Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, an emission timestamp, and the data blob. With the CSV output, it is possible to normalize \(flatten\) the data blob to multiple columns.
 
-| Column | Condition | Description |
-| :--- | :--- | :--- |
-| `_airbyte_ab_id` | Always exists | A uuid assigned by Airbyte to each processed record. |
-| `_airbyte_emitted_at` | Always exists. | A timestamp representing when the event was pulled from the data source. |
-| `_airbyte_data` | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |  |
-| root level fields | When root level normalization \(flattening\) is selected, the root level fields are expanded. |  |
+| Column                | Condition                                                                                         | Description                                                              |
+| :-------------------- | :------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- |
+| `_airbyte_ab_id`      | Always exists                                                                                     | A uuid assigned by Airbyte to each processed record.                     |
+| `_airbyte_emitted_at` | Always exists.                                                                                    | A timestamp representing when the event was pulled from the data source. |
+| `_airbyte_data`       | When no normalization \(flattening\) is needed, all data reside under this column as a json blob. |                                                                          |
+| root level fields     | When root level normalization \(flattening\) is selected, the root level fields are expanded.     |                                                                          |
 
 For example, given the following json object from a source:
 
@@ -247,15 +249,15 @@ For example, given the following json object from a source:
 
 With no normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `_airbyte_data` |
-| :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `_airbyte_data`                                                |
+| :------------------------------------- | :-------------------- | :------------------------------------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | `{ "user_id": 123, name: { "first": "John", "last": "Doe" } }` |
 
 With root level normalization, the output CSV is:
 
-| `_airbyte_ab_id` | `_airbyte_emitted_at` | `user_id` | `name` |
-| :--- | :--- | :--- | :--- |
-| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | 123 | `{ "first": "John", "last": "Doe" }` |
+| `_airbyte_ab_id`                       | `_airbyte_emitted_at` | `user_id` | `name`                               |
+| :------------------------------------- | :-------------------- | :-------- | :----------------------------------- |
+| `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000         | 123       | `{ "first": "John", "last": "Doe" }` |
 
 Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.csv.gz`).
 
@@ -307,14 +309,14 @@ Output files can be compressed. The default option is GZIP compression. If compr
 
 The following configuration is available to configure the Parquet output:
 
-| Parameter | Type | Default | Description |
-| :--- | :---: | :---: | :--- |
-| `compression_codec` | enum | `UNCOMPRESSED` | **Compression algorithm**. Available candidates are: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `BROTLI`, `LZ4`, and `ZSTD`. |
-| `block_size_mb` | integer | 128 \(MB\) | **Block size \(row group size\)** in MB. This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. |
-| `max_padding_size_mb` | integer | 8 \(MB\) | **Max padding size** in MB. This is the maximum size allowed as padding to align row groups. This is also the minimum size of a row group. |
-| `page_size_kb` | integer | 1024 \(KB\) | **Page size** in KB. The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. |
-| `dictionary_page_size_kb` | integer | 1024 \(KB\) | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. |
-| `dictionary_encoding` | boolean | `true` | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on. |
+| Parameter                 |  Type   |    Default     | Description                                                                                                                                                                                                                       |
+| :------------------------ | :-----: | :------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `compression_codec`       |  enum   | `UNCOMPRESSED` | **Compression algorithm**. Available candidates are: `UNCOMPRESSED`, `SNAPPY`, `GZIP`, `LZO`, `BROTLI`, `LZ4`, and `ZSTD`.                                                                                                        |
+| `block_size_mb`           | integer |   128 \(MB\)   | **Block size \(row group size\)** in MB. This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. |
+| `max_padding_size_mb`     | integer |    8 \(MB\)    | **Max padding size** in MB. This is the maximum size allowed as padding to align row groups. This is also the minimum size of a row group.                                                                                        |
+| `page_size_kb`            | integer |  1024 \(KB\)   | **Page size** in KB. The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. |
+| `dictionary_page_size_kb` | integer |  1024 \(KB\)   | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary.                                     |
+| `dictionary_encoding`     | boolean |     `true`     | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on.                                                                                                                                        |
 
 These parameters are related to the `ParquetOutputFormat`. See the [Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) for more details. Also see [Parquet documentation](https://parquet.apache.org/docs/file-format/configurations/) for their recommended configurations \(512 - 1024 MB block size, 8 KB page size\).
 
@@ -323,26 +325,30 @@ These parameters are related to the `ParquetOutputFormat`. See the [Java doc](ht
 Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion).
 
 In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use:
+
 ```json
 {
   "Version": "2012-10-17",
-    "Statement": [
-        {
-            "Effect": "Allow",
-            "Action": "s3:*",
-            "Resource": [
-                "arn:aws:s3:::YOUR_BUCKET_NAME/*",
-                "arn:aws:s3:::YOUR_BUCKET_NAME"
-            ]
-            }
-    ]
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::YOUR_BUCKET_NAME/*",
+        "arn:aws:s3:::YOUR_BUCKET_NAME"
+      ]
+    }
+  ]
 }
 ```
 
 ## CHANGELOG
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                              |
-|:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.5.1   | 2023-06-26 | [#27786](https://github.com/airbytehq/airbyte/pull/27786)  | Fix build                                                                                                                                            |
+| 0.5.0   | 2023-06-26 | [#27725](https://github.com/airbytehq/airbyte/pull/27725)  | License Update: Elv2                                                                                                                                 |
+| 0.4.2   | 2023-06-21 | [#27555](https://github.com/airbytehq/airbyte/pull/27555)  | Reduce image size                                                                                                                                    |
 | 0.4.1   | 2023-05-18 | [#26284](https://github.com/airbytehq/airbyte/pull/26284)  | Fix: reenable LZO compression for Parquet output                                                                                                     |
 | 0.4.0   | 2023-04-28 | [#25570](https://github.com/airbytehq/airbyte/pull/25570)  | Fix: all integer schemas should be converted to Avro longs                                                                                           |
 | 0.3.25  | 2023-04-27 | [#25346](https://github.com/airbytehq/airbyte/pull/25346)  | Internal code cleanup                                                                                                                                |
diff --git a/docs/integrations/destinations/scylla.md b/docs/integrations/destinations/scylla.md
index dfd9a621b702..7af55bda5341 100644
--- a/docs/integrations/destinations/scylla.md
+++ b/docs/integrations/destinations/scylla.md
@@ -1,9 +1,11 @@
 # Scylla
 
 ## Prerequisites
+
 - For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your Scylla connector to version `0.1.3` or newer
 
 ## Sync overview
+
 ### Output schema
 
 The incoming airbyte data is structured in keyspaces and tables and is partitioned and replicated across different nodes
@@ -11,18 +13,18 @@ in the cluster. This connector maps an incoming `stream` to a Scylla `table` and
 Fields in the airbyte message become different columns in the Scylla tables. Each table will contain the following
 columns.
 
-* `_airbyte_ab_id`: A random uuid generated to be used as a partition key.
-* `_airbyte_emitted_at`: a timestamp representing when the event was received from the data source.
-* `_airbyte_data`: a json text representing the extracted data.
+- `_airbyte_ab_id`: A random uuid generated to be used as a partition key.
+- `_airbyte_emitted_at`: a timestamp representing when the event was received from the data source.
+- `_airbyte_data`: a json text representing the extracted data.
 
 ### Features
 
-| Feature | Support | Notes |
-| :--- | :---: | :--- |
-| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured DynamoDB table. |
-| Incremental - Append Sync | ✅ |  |
-| Incremental - Deduped History | ❌ | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | ✅ | Namespace will be used as part of the table name. |
+| Feature                        | Support | Notes                                                                                   |
+| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ✅    | Warning: this mode deletes all previously synced data in the configured DynamoDB table. |
+| Incremental - Append Sync      |   ✅    |                                                                                         |
+| Incremental - Append + Deduped |   ❌    |                                                                                         |
+| Namespaces                     |   ✅    | Namespace will be used as part of the table name.                                       |
 
 ### Performance considerations
 
@@ -34,18 +36,19 @@ and handle any amount of data from the connector.
 
 ### Requirements
 
-* Driver compatibility: NA
-* Configuration
-    * Keyspace [default keyspace to use when writing data]
-    * Username [authentication username]
-    * Password [authentication password]
-    * Address [cluster address]
-    * Port [default: 9042]
-    * Replication [optional] [default: 1]
+- Driver compatibility: NA
+- Configuration
+  - Keyspace [default keyspace to use when writing data]
+  - Username [authentication username]
+  - Password [authentication password]
+  - Address [cluster address]
+  - Port [default: 9042]
+  - Replication [optional] [default: 1]
 
 ### Setup guide
+
 ## Changelog
 
-| Version | Date       | Pull Request | Subject                                                                                             |
-|:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------|
-| 0.1.3  | 2022-08-10 | [153999](https://github.com/airbytehq/airbyte/pull/15399) | handling per-stream state                                                   |
+| Version | Date       | Pull Request                                              | Subject                   |
+| :------ | :--------- | :-------------------------------------------------------- | :------------------------ |
+| 0.1.3   | 2022-08-10 | [153999](https://github.com/airbytehq/airbyte/pull/15399) | handling per-stream state |
diff --git a/docs/integrations/destinations/selectdb.md b/docs/integrations/destinations/selectdb.md
index 626d7e15ecd2..032fdae475a7 100644
--- a/docs/integrations/destinations/selectdb.md
+++ b/docs/integrations/destinations/selectdb.md
@@ -12,19 +12,16 @@ Each stream will be output into its own table in SelectDB. Each table will conta
 - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Doris is `BIGINT`.
 - `_airbyte_data`: a json blob representing with the event data. The column type in SelectDB is `String`.
 
-
 ### Features
 
 This section should contain a table with the following format:
 
-| Feature                                | Supported?(Yes/No) | Notes                    |
-| :------------------------------------- | :----------------- | :----------------------- |
-| Full Refresh Sync                      | Yes                |                          |
-| Incremental - Append Sync              | Yes                |                          |
-| Incremental - Deduped History          | No                 | it will soon be realized |
-| For databases, WAL/Logical replication | Yes                |                          |
-
-
+| Feature                                | Supported?(Yes/No) | Notes |
+| :------------------------------------- | :----------------- | :---- |
+| Full Refresh Sync                      | Yes                |       |
+| Incremental - Append Sync              | Yes                |       |
+| Incremental - Append + Deduped         | No                 |       |
+| For databases, WAL/Logical replication | Yes                |       |
 
 ### Performance considerations
 
@@ -37,7 +34,7 @@ Importing multiple tables will generate multiple transactions, which should be s
 
 To use the SelectDB destination, you'll need:
 
-- A SelectDB server and your 
+- A SelectDB server and your
 - Make sure your SelectDB http port and mysql query port can be accessed by Airbyte.
 - Make sure your SelectDB host can be accessed by Airbyte. if use a public network to access SelectDB, please ensure that your airbyte public network IP is in the ip whitelist of your SelectDB.
 - Make sure your SelectDB user with read/write permissions on certain tables.
@@ -58,7 +55,6 @@ You need to prepare database that will be used to store synced data from Airbyte
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.0 | 2023-04-03 | [\#20881](https://github.com/airbytehq/airbyte/pull/20881) | Initial release SelectDB Destination |
-
+| Version | Date       | Pull Request                                               | Subject                              |
+| :------ | :--------- | :--------------------------------------------------------- | :----------------------------------- |
+| 0.1.0   | 2023-04-03 | [\#20881](https://github.com/airbytehq/airbyte/pull/20881) | Initial release SelectDB Destination |
diff --git a/docs/integrations/destinations/snowflake-migrations.md b/docs/integrations/destinations/snowflake-migrations.md
new file mode 100644
index 000000000000..3076e9e9f77d
--- /dev/null
+++ b/docs/integrations/destinations/snowflake-migrations.md
@@ -0,0 +1,4 @@
+# Snowflake Migration Guide
+
+## Upgrading to 2.0.0
+Snowflake no longer supports GCS/S3. Please migrate to the Internal Staging option. This is recommended by Snowflake and is cheaper and faster.
diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md
index 7f04c68ad1a6..5a80360e48ae 100644
--- a/docs/integrations/destinations/snowflake.md
+++ b/docs/integrations/destinations/snowflake.md
@@ -35,8 +35,8 @@ To set up the Snowflake destination connector, you first need to create Airbyte-
 
 You can use the following script in a new [Snowflake worksheet](https://docs.snowflake.com/en/user-guide/ui-worksheet.html) to create the entities:
 
-1. [Log into your Snowflake account](https://www.snowflake.com/login/).
-2. Edit the following script to change the password to a more secure password and to change the names of other resources if you so desire.
+1.  [Log into your Snowflake account](https://www.snowflake.com/login/).
+2.  Edit the following script to change the password to a more secure password and to change the names of other resources if you so desire.
 
     **Note:** Make sure you follow the [Snowflake identifier requirements](https://docs.snowflake.com/en/sql-reference/identifiers-syntax.html) while renaming the resources.
 
@@ -109,8 +109,7 @@ You can use the following script in a new [Snowflake worksheet](https://docs.sno
 
         commit;
 
-
-3. Run the script using the [Worksheet page](https://docs.snowflake.com/en/user-guide/ui-worksheet.html) or [Snowsight](https://docs.snowflake.com/en/user-guide/ui-snowsight-gs.html). Make sure to select the **All Queries** checkbox.
+3.  Run the script using the [Worksheet page](https://docs.snowflake.com/en/user-guide/ui-worksheet.html) or [Snowsight](https://docs.snowflake.com/en/user-guide/ui-snowsight-gs.html). Make sure to select the **All Queries** checkbox.
 
 ### Step 2: Set up a data loading method
 
@@ -122,7 +121,6 @@ Make sure the database and schema have the `USAGE` privilege.
 
 To use an Amazon S3 bucket, [create a new Amazon S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) with read/write access for Airbyte to stage data to Snowflake.
 
-
 #### Using a Google Cloud Storage bucket
 
 To use a Google Cloud Storage bucket:
@@ -130,34 +128,36 @@ To use a Google Cloud Storage bucket:
 1. Navigate to the Google Cloud Console and [create a new bucket](https://cloud.google.com/storage/docs/creating-buckets) with read/write access for Airbyte to stage data to Snowflake.
 2. [Generate a JSON key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) for your service account.
 3. Edit the following script to replace `AIRBYTE_ROLE` with the role you used for Airbyte's Snowflake configuration and `YOURBUCKETNAME` with your bucket name.
-    ```text
-    create storage INTEGRATION gcs_airbyte_integration
-      TYPE = EXTERNAL_STAGE
-      STORAGE_PROVIDER = GCS
-      ENABLED = TRUE
-      STORAGE_ALLOWED_LOCATIONS = ('gcs://YOURBUCKETNAME');
 
-    create stage gcs_airbyte_stage
-      url = 'gcs://YOURBUCKETNAME'
-      storage_integration = gcs_airbyte_integration;
+   ```text
+   create storage INTEGRATION gcs_airbyte_integration
+     TYPE = EXTERNAL_STAGE
+     STORAGE_PROVIDER = GCS
+     ENABLED = TRUE
+     STORAGE_ALLOWED_LOCATIONS = ('gcs://YOURBUCKETNAME');
 
-    GRANT USAGE ON integration gcs_airbyte_integration TO ROLE AIRBYTE_ROLE;
-    GRANT USAGE ON stage gcs_airbyte_stage TO ROLE AIRBYTE_ROLE;
+   create stage gcs_airbyte_stage
+     url = 'gcs://YOURBUCKETNAME'
+     storage_integration = gcs_airbyte_integration;
 
-    DESC STORAGE INTEGRATION gcs_airbyte_integration;
-    ```
-    The final query should show a `STORAGE_GCP_SERVICE_ACCOUNT` property with an email as the property value. Add read/write permissions to your bucket with that email.
+   GRANT USAGE ON integration gcs_airbyte_integration TO ROLE AIRBYTE_ROLE;
+   GRANT USAGE ON stage gcs_airbyte_stage TO ROLE AIRBYTE_ROLE;
 
-4. Navigate to the Snowflake UI and run the script as a [Snowflake account admin](https://docs.snowflake.com/en/user-guide/security-access-control-considerations.html) using the [Worksheet page](https://docs.snowflake.com/en/user-guide/ui-worksheet.html) or [Snowsight](https://docs.snowflake.com/en/user-guide/ui-snowsight-gs.html).
+   DESC STORAGE INTEGRATION gcs_airbyte_integration;
+   ```
 
+   The final query should show a `STORAGE_GCP_SERVICE_ACCOUNT` property with an email as the property value. Add read/write permissions to your bucket with that email.
+
+4. Navigate to the Snowflake UI and run the script as a [Snowflake account admin](https://docs.snowflake.com/en/user-guide/security-access-control-considerations.html) using the [Worksheet page](https://docs.snowflake.com/en/user-guide/ui-worksheet.html) or [Snowsight](https://docs.snowflake.com/en/user-guide/ui-snowsight-gs.html).
 
 ### Step 3: Set up Snowflake as a destination in Airbyte
 
 Navigate to the Airbyte UI to set up Snowflake as a destination. You can authenticate using username/password or OAuth 2.0:
 
 ### Login and Password
+
 | Field                                                                                                 | Description                                                                                                                                                                                       |
-|-------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html)                        | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` |
 | [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles)          | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                           |
 | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE`                                                                                                   |
@@ -167,21 +167,21 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent
 | Password                                                                                              | The password associated with the username.                                                                                                                                                        |
 | [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional)           | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`   |
 
-
 ### OAuth 2.0
-| Field                                                                                                  | Description                                                                                                                                                                                        |
-|:-------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html)                         | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com`  |
-| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles)           | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                            |
-| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses)  | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE`                                                                                                    |
-| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)    | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE`                                                                                                      |
-| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)      | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name.                                                           |
-| Username                                                                                               | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER`                                                                                                |
-| OAuth2                                                                                                 | The Login name and password to obtain auth token.                                                                                                                                                  |
-| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional)            | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`    |
 
+| Field                                                                                                 | Description                                                                                                                                                                                       |
+| :---------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html)                        | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` |
+| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles)          | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                           |
+| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE`                                                                                                   |
+| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)   | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE`                                                                                                     |
+| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)     | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name.                                                          |
+| Username                                                                                              | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER`                                                                                               |
+| OAuth2                                                                                                | The Login name and password to obtain auth token.                                                                                                                                                 |
+| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional)           | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`   |
 
 ### Key pair authentication
+
     In order to configure key pair authentication you will need a private/public key pair.
     If you do not have the key pair yet, you can generate one using openssl command line tool
     Use this command in order to generate an unencrypted private key file:
@@ -204,16 +204,14 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent
 
     and replace  with your user name and  with your public key.
 
-
-
 To use AWS S3 as the cloud storage, enter the information for the S3 bucket you created in Step 2:
 
 | Field                          | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
-|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | S3 Bucket Name                 | The name of the staging S3 bucket (Example: `airbyte.staging`). Airbyte will write files to this bucket and read them via statements on Snowflake.                                                                                                                                                                                                                                                                                                                                                                                                      |
 | S3 Bucket Region               | The S3 staging bucket region used.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
-| S3 Key Id *                    | The Access Key ID granting access to the S3 staging bucket. Airbyte requires Read and Write permissions for the bucket.                                                                                                                                                                                                                                                                                                                                                                                                                                 |
-| S3 Access Key *                | The corresponding secret to the S3 Key ID.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
+| S3 Key Id \*                   | The Access Key ID granting access to the S3 staging bucket. Airbyte requires Read and Write permissions for the bucket.                                                                                                                                                                                                                                                                                                                                                                                                                                 |
+| S3 Access Key \*               | The corresponding secret to the S3 Key ID.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              |
 | Stream Part Size (Optional)    | Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 5MB by default, resulting in a default limit of 100GB tables. 
    Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care. (e.g. 5)                                                                     |
 | Purge Staging Files and Tables | Determines whether to delete the staging files from S3 after completing the sync. Specifically, the connector will create CSV files named `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after sync; if you want to keep them for other purposes, set `purge_staging_data` to false.                                                                                                                                                  |
 | Encryption                     | Whether files on S3 are encrypted. You probably don't need to enable this, but it can provide an additional layer of security if you are sharing your data storage with other applications. If you do use encryption, you must choose between ephemeral keys (Airbyte will automatically generate a new key for each sync, and nobody but Airbyte and Snowflake will be able to read the data on S3) or providing your own key (if you have the "Purge staging files and tables" option disabled, and you want to be able to decrypt the data yourself) |
@@ -222,7 +220,7 @@ To use AWS S3 as the cloud storage, enter the information for the S3 bucket you
 To use a Google Cloud Storage bucket, enter the information for the bucket you created in Step 2:
 
 | Field                          | Description                                                                                                                                                                                                                                                                                                                                                                                          |
-|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | GCP Project ID                 | The name of the GCP project ID for your credentials. (Example: `my-project`)                                                                                                                                                                                                                                                                                                                         |
 | GCP Bucket Name                | The name of the staging bucket. Airbyte will write files to this bucket and read them via statements on Snowflake. (Example: `airbyte-staging`)                                                                                                                                                                                                                                                      |
 | Google Application Credentials | The contents of the JSON key file that has read/write permissions to the staging GCS bucket. You will separately need to grant bucket access to your Snowflake GCP service account. See the [Google Cloud docs](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) for more information on how to generate a JSON key for your service account. |
@@ -231,15 +229,14 @@ To use a Google Cloud Storage bucket, enter the information for the bucket you c
 
 Airbyte outputs each stream into its own table with the following columns in Snowflake:
 
-| Airbyte field       | Description                                                    | Column type              |
-|---------------------|----------------------------------------------------------------|--------------------------|
-| _airbyte_ab_id      | A UUID assigned to each processed event                        | VARCHAR                  |
-| _airbyte_emitted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE |
-| _airbyte_data       | A JSON blob with the event data.                               | VARIANT                  |
+| Airbyte field        | Description                                                    | Column type              |
+| -------------------- | -------------------------------------------------------------- | ------------------------ |
+| \_airbyte_ab_id      | A UUID assigned to each processed event                        | VARCHAR                  |
+| \_airbyte_emitted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE |
+| \_airbyte_data       | A JSON blob with the event data.                               | VARIANT                  |
 
 **Note:** By default, Airbyte creates permanent tables. If you prefer transient tables, create a dedicated transient database for Airbyte. For more information, refer to[ Working with Temporary and Transient Tables](https://docs.snowflake.com/en/user-guide/tables-temp-transient.html)
 
-
 ## Supported sync modes
 
 The Snowflake destination supports the following sync modes:
@@ -247,7 +244,7 @@ The Snowflake destination supports the following sync modes:
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Snowflake tutorials
 
@@ -261,6 +258,7 @@ Now that you have set up the Snowflake destination connector, check out the foll
 ## Troubleshooting
 
 ### 'Current role does not have permissions on the target schema'
+
 If you receive an error stating `Current role does not have permissions on the target schema` make sure that the
 Snowflake destination `SCHEMA` is one that the role you've provided has permissions on. When creating a connection,
 it may allow you to select `Mirror source structure` for the `Destination namespace`, which if you have followed
@@ -271,83 +269,105 @@ Otherwise, make sure to grant the role the required permissions in the desired n
 
 ## Changelog
 
-| Version         | Date       | Pull Request                                               | Subject                                                                                                                                             |
-|:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------|
-| 1.0.5           | 2023-05-31 | [\#25782](https://github.com/airbytehq/airbyte/pull/25782) | Internal scaffolding for future development                                                                                                         |
-| 1.0.4           | 2023-05-19 | [\#26323](https://github.com/airbytehq/airbyte/pull/26323) | Prevent infinite retry loop under specific circumstances                                                                                            |
-| 1.0.3           | 2023-05-15 | [\#26081](https://github.com/airbytehq/airbyte/pull/26081) | Reverts splits bases                                                                                                                                |
-| 1.0.2           | 2023-05-05 | [\#25649](https://github.com/airbytehq/airbyte/pull/25649) | Splits bases (reverted)                                                                                                                             |
-| 1.0.1           | 2023-04-29 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Internal library update                                                                                                                             |
-| 1.0.0           | 2023-05-02 | [\#25739](https://github.com/airbytehq/airbyte/pull/25739) | Removed Azure Blob Storage as a loading method                                                                                                      |
-| 0.4.63          | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Added FlushBufferFunction interface                                                                                                                 |
-| 0.4.61          | 2023-03-30 | [\#24736](https://github.com/airbytehq/airbyte/pull/24736) | Improve behavior when throttled by AWS API                                                                                                          |
-| 0.4.60          | 2023-03-30 | [\#24698](https://github.com/airbytehq/airbyte/pull/24698) | Add option in spec to allow increasing the stream buffer size to 50                                                                                 |
-| 0.4.59          | 2023-03-23 | [\#23904](https://github.com/airbytehq/airbyte/pull/24405) | Fail faster in certain error cases                                                                                                                  |
-| 0.4.58          | 2023-03-27 | [\#24615](https://github.com/airbytehq/airbyte/pull/24615) | Fixed host validation by pattern on UI                                                                                                              |
-| 0.4.56 (broken) | 2023-03-22 | [\#23904](https://github.com/airbytehq/airbyte/pull/23904) | Added host validation by pattern on UI                                                                                                              |
-| 0.4.54          | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays                                                                                          |
-| 0.4.53          | 2023-03-15 | [\#24058](https://github.com/airbytehq/airbyte/pull/24058) | added write attempt to internal staging Check method                                                                                                |
-| 0.4.52          | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush                                                                                                             |
-| 0.4.51          | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long                                                                                                               |
-| 0.4.49          | 2023-02-27 | [\#23360](https://github.com/airbytehq/airbyte/pull/23360) | Added logging for flushing and writing data to destination storage                                                                                  |
-| 0.4.48          | 2023-02-23 | [\#22877](https://github.com/airbytehq/airbyte/pull/22877) | Add handler for IP not in whitelist error and more handlers for insufficient permission error                                                       |
-| 0.4.47          | 2023-01-30 | [\#21912](https://github.com/airbytehq/airbyte/pull/21912) | Catch "Create" Table and Stage Known Permissions and rethrow as ConfigExceptions                                                                    |
-| 0.4.46          | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging                                                                                            |
-| 0.4.45          | 2023-01-25 | [\#21087](https://github.com/airbytehq/airbyte/pull/21764) | Catch Known Permissions and rethrow as ConfigExceptions                                                                                             |
-| 0.4.44          | 2023-01-20 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions                                                                                                     |
-| 0.4.43          | 2023-01-20 | [\#21450](https://github.com/airbytehq/airbyte/pull/21450) | Updated Check methods to handle more possible s3 and gcs stagings issues                                                                            |
-| 0.4.42          | 2023-01-12 | [\#21342](https://github.com/airbytehq/airbyte/pull/21342) | Better handling for conflicting destination streams                                                                                                 |
-| 0.4.41          | 2022-12-16 | [\#20566](https://github.com/airbytehq/airbyte/pull/20566) | Improve spec to adhere to standards                                                                                                                 |
-| 0.4.40          | 2022-11-11 | [\#19302](https://github.com/airbytehq/airbyte/pull/19302) | Set jdbc application env variable depends on env - airbyte_oss or airbyte_cloud                                                                     |
-| 0.4.39          | 2022-11-09 | [\#18970](https://github.com/airbytehq/airbyte/pull/18970) | Updated "check" connection method to handle more errors                                                                                             |
-| 0.4.38          | 2022-09-26 | [\#17115](https://github.com/airbytehq/airbyte/pull/17115) | Added connection string identifier                                                                                                                  |
-| 0.4.37          | 2022-09-21 | [\#16839](https://github.com/airbytehq/airbyte/pull/16839) | Update JDBC driver for Snowflake to 3.13.19                                                                                                         |
-| 0.4.36          | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage                                                                                                                      |
-| 0.4.35          | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields).                           |
-| 0.4.34          | 2022-07-23 | [\#14388](https://github.com/airbytehq/airbyte/pull/14388) | Add support for key pair authentication                                                                                                             |
-| 0.4.33          | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable.                                                                                                               |
-| 0.4.32          | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors                                                                                |
-| 0.4.31          | 2022-07-07 | [\#13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field description                                                                                                             |
-| 0.4.30          | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging                                                                         |
-| 0.4.29          | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager                                                             |
-| 0.4.28          | 2022-05-18 | [\#12952](https://github.com/airbytehq/airbyte/pull/12952) | Apply buffering strategy on GCS staging                                                                                                             |
-| 0.4.27          | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                                                                              |
-| 0.4.26          | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessages on error.                                                                                  |
-| 0.4.25          | 2022-05-03 | [\#12452](https://github.com/airbytehq/airbyte/pull/12452) | Add support for encrypted staging on S3; fix the purge_staging_files option                                                                         |
-| 0.4.24          | 2022-03-24 | [\#11093](https://github.com/airbytehq/airbyte/pull/11093) | Added OAuth support (Compatible with Airbyte Version 0.35.60+)                                                                                      |
-| 0.4.22          | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters                                                                                                               |
-| 0.4.21          | 2022-03-18 | [\#11071](https://github.com/airbytehq/airbyte/pull/11071) | Switch to compressed on-disk buffering before staging to s3/internal stage                                                                          |
-| 0.4.20          | 2022-03-14 | [\#10341](https://github.com/airbytehq/airbyte/pull/10341) | Add Azure blob staging support                                                                                                                      |
-| 0.4.19          | 2022-03-11 | [\#10699](https://github.com/airbytehq/airbyte/pull/10699) | Added unit tests                                                                                                                                    |
-| 0.4.17          | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                                                                   |
-| 0.4.16          | 2022-02-25 | [\#10627](https://github.com/airbytehq/airbyte/pull/10627) | Add try catch to make sure all handlers are closed                                                                                                  |
-| 0.4.15          | 2022-02-22 | [\#10459](https://github.com/airbytehq/airbyte/pull/10459) | Add FailureTrackingAirbyteMessageConsumer                                                                                                           |
-| 0.4.14          | 2022-02-17 | [\#10394](https://github.com/airbytehq/airbyte/pull/10394) | Reduce memory footprint.                                                                                                                            |
-| 0.4.13          | 2022-02-16 | [\#10212](https://github.com/airbytehq/airbyte/pull/10212) | Execute COPY command in parallel for S3 and GCS staging                                                                                             |
-| 0.4.12          | 2022-02-15 | [\#10342](https://github.com/airbytehq/airbyte/pull/10342) | Use connection pool, and fix connection leak.                                                                                                       |
-| 0.4.11          | 2022-02-14 | [\#9920](https://github.com/airbytehq/airbyte/pull/9920)   | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. |
-| 0.4.10          | 2022-02-14 | [\#10297](https://github.com/airbytehq/airbyte/pull/10297) | Halve the record buffer size to reduce memory consumption.                                                                                          |
-| 0.4.9           | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `ExitOnOutOfMemoryError` JVM flag.                                                                                                              |
-| 0.4.8           | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959)   | Fix null pointer exception from buffered stream consumer.                                                                                           |
-| 0.4.7           | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745)   | Integrate with Sentry.                                                                                                                              |
-| 0.4.6           | 2022-01-28 | [\#9623](https://github.com/airbytehq/airbyte/pull/9623)   | Add jdbc_url_params support for optional JDBC parameters                                                                                            |
-| 0.4.5           | 2021-12-29 | [\#9184](https://github.com/airbytehq/airbyte/pull/9184)   | Update connector fields title/description                                                                                                           |
-| 0.4.4           | 2022-01-24 | [\#9743](https://github.com/airbytehq/airbyte/pull/9743)   | Fixed bug with dashes in schema name                                                                                                                |
-| 0.4.3           | 2022-01-20 | [\#9531](https://github.com/airbytehq/airbyte/pull/9531)   | Start using new S3StreamCopier and expose the purgeStagingData option                                                                               |
-| 0.4.2           | 2022-01-10 | [\#9141](https://github.com/airbytehq/airbyte/pull/9141)   | Fixed duplicate rows on retries                                                                                                                     |
-| 0.4.1           | 2021-01-06 | [\#9311](https://github.com/airbytehq/airbyte/pull/9311)   | Update сreating schema during check                                                                                                                 |
-| 0.4.0           | 2021-12-27 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063)   | Updated normalization to produce permanent tables                                                                                                   |
-| 0.3.24          | 2021-12-23 | [\#8869](https://github.com/airbytehq/airbyte/pull/8869)   | Changed staging approach to Byte-Buffered                                                                                                           |
-| 0.3.23          | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039)   | Added part_size configuration in UI for S3 loading method                                                                                           |
-| 0.3.22          | 2021-12-21 | [\#9006](https://github.com/airbytehq/airbyte/pull/9006)   | Updated jdbc schema naming to follow Snowflake Naming Conventions                                                                                   |
-| 0.3.21          | 2021-12-15 | [\#8781](https://github.com/airbytehq/airbyte/pull/8781)   | Updated check method to verify permissions to create/drop stage for internal staging; compatibility fix for Java 17                                 |
-| 0.3.20          | 2021-12-10 | [\#8562](https://github.com/airbytehq/airbyte/pull/8562)   | Moving classes around for better dependency management; compatibility fix for Java 17                                                               |
-| 0.3.19          | 2021-12-06 | [\#8528](https://github.com/airbytehq/airbyte/pull/8528)   | Set Internal Staging as default choice                                                                                                              |
-| 0.3.18          | 2021-11-26 | [\#8253](https://github.com/airbytehq/airbyte/pull/8253)   | Snowflake Internal Staging Support                                                                                                                  |
-| 0.3.17          | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719)   | Improve handling of wide rows by buffering records based on their byte size rather than their count                                                 |
-| 0.3.15          | 2021-10-11 | [\#6949](https://github.com/airbytehq/airbyte/pull/6949)   | Each stream was split into files of 10,000 records each for copying using S3 or GCS                                                                 |
-| 0.3.14          | 2021-09-08 | [\#5924](https://github.com/airbytehq/airbyte/pull/5924)   | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table                                                             |
-| 0.3.13          | 2021-09-01 | [\#5784](https://github.com/airbytehq/airbyte/pull/5784)   | Updated query timeout from 30 minutes to 3 hours                                                                                                    |
-| 0.3.12          | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125)   | Enable `additionalPropertities` in spec.json                                                                                                        |
-| 0.3.11          | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555)   | Partial Success in BufferedStreamConsumer                                                                                                           |
-| 0.3.10          | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713)   | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake                                                                |
+| Version         | Date       | Pull Request                                               | Subject                                                                                                                                                         |
+|:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.1.3           | 2023-08-25 | [\#29881](https://github.com/airbytehq/airbyte/pull/29881) | Destinations v2: Only run T+D once at end of sync, to prevent data loss under async conditions                                                                  |
+| 2.1.2           | 2023-08-24 | [\#29805](https://github.com/airbytehq/airbyte/pull/29805) | Destinations v2: Don't soft reset in migration                                                                                                                  |
+| 2.1.1           | 2023-08-23 | [\#29774](https://github.com/airbytehq/airbyte/pull/29774) | Destinations v2: Don't soft reset overwrite syncs                                                                                                               |
+| 2.1.0           | 2023-08-21 | [\#29636](https://github.com/airbytehq/airbyte/pull/29636) | Destinations v2: Several Critical Bug Fixes (cursorless dedup, improved floating-point handling, improved special characters handling; improved error handling) |
+| 2.0.0           | 2023-08-09 | [\#28894](https://github.com/airbytehq/airbyte/pull/29236) | Remove support for Snowflake GCS/S3 loading method in favor of Snowflake Internal staging                                                                       |
+| 1.3.3           | 2023-08-15 | [\#29461](https://github.com/airbytehq/airbyte/pull/29461) | Changing a static constant reference                                                                                                                            |
+| 1.3.2           | 2023-08-11 | [\#29381](https://github.com/airbytehq/airbyte/pull/29381) | Destinations v2: Add support for streams with no columns                                                                                                        |
+| 1.3.1           | 2023-08-04 | [\#28894](https://github.com/airbytehq/airbyte/pull/28894) | Destinations v2: Update SqlGenerator                                                                                                                            |
+| 1.3.0           | 2023-08-07 | [\#29174](https://github.com/airbytehq/airbyte/pull/29174) | Destinations v2: early access release                                                                                                                           |
+| 1.2.10          | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring                                                                                                                                       |
+| 1.2.9           | 2023-08-04 | [\#28677](https://github.com/airbytehq/airbyte/pull/28677) | Destinations v2: internal code changes to prepare for early access release                                                                                      |
+| 1.2.8           | 2023-08-03 | [\#29047](https://github.com/airbytehq/airbyte/pull/29047) | Avoid logging record if the format is invalid                                                                                                                   |
+| 1.2.7           | 2023-08-02 | [\#28976](https://github.com/airbytehq/airbyte/pull/28976) | Fix composite PK handling in v1 mode                                                                                                                            |
+| 1.2.6           | 2023-08-01 | [\#28618](https://github.com/airbytehq/airbyte/pull/28618) | Reduce logging noise                                                                                                                                            |
+| 1.2.5           | 2023-07-24 | [\#28618](https://github.com/airbytehq/airbyte/pull/28618) | Add hooks in preparation for destinations v2 implementation                                                                                                     |
+| 1.2.4           | 2023-07-21 | [\#28584](https://github.com/airbytehq/airbyte/pull/28584) | Install dependencies in preparation for destinations v2 work                                                                                                    |
+| 1.2.3           | 2023-07-21 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Pull in async framework minor bug fix for race condition on state emission                                                                                      |
+| 1.2.2           | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild                                                                                                                            |
+| 1.2.1           | 2023-07-14 | [\#28315](https://github.com/airbytehq/airbyte/pull/28315) | Pull in async framework minor bug fix to avoid Snowflake hanging on close                                                                                       |
+| 1.2.0           | 2023-07-5  | [\#27935](https://github.com/airbytehq/airbyte/pull/27935) | Enable Faster Snowflake Syncs with Asynchronous writes                                                                                                          |
+| 1.1.0           | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2                                                                                                                                            |
+| 1.0.6           | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size                                                                                                                                               |
+| 1.0.5           | 2023-05-31 | [\#25782](https://github.com/airbytehq/airbyte/pull/25782) | Internal scaffolding for future development                                                                                                                     |
+| 1.0.4           | 2023-05-19 | [\#26323](https://github.com/airbytehq/airbyte/pull/26323) | Prevent infinite retry loop under specific circumstances                                                                                                        |
+| 1.0.3           | 2023-05-15 | [\#26081](https://github.com/airbytehq/airbyte/pull/26081) | Reverts splits bases                                                                                                                                            |
+| 1.0.2           | 2023-05-05 | [\#25649](https://github.com/airbytehq/airbyte/pull/25649) | Splits bases (reverted)                                                                                                                                         |
+| 1.0.1           | 2023-04-29 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Internal library update                                                                                                                                         |
+| 1.0.0           | 2023-05-02 | [\#25739](https://github.com/airbytehq/airbyte/pull/25739) | Removed Azure Blob Storage as a loading method                                                                                                                  |
+| 0.4.63          | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Added FlushBufferFunction interface                                                                                                                             |
+| 0.4.61          | 2023-03-30 | [\#24736](https://github.com/airbytehq/airbyte/pull/24736) | Improve behavior when throttled by AWS API                                                                                                                      |
+| 0.4.60          | 2023-03-30 | [\#24698](https://github.com/airbytehq/airbyte/pull/24698) | Add option in spec to allow increasing the stream buffer size to 50                                                                                             |
+| 0.4.59          | 2023-03-23 | [\#23904](https://github.com/airbytehq/airbyte/pull/24405) | Fail faster in certain error cases                                                                                                                              |
+| 0.4.58          | 2023-03-27 | [\#24615](https://github.com/airbytehq/airbyte/pull/24615) | Fixed host validation by pattern on UI                                                                                                                          |
+| 0.4.56 (broken) | 2023-03-22 | [\#23904](https://github.com/airbytehq/airbyte/pull/23904) | Added host validation by pattern on UI                                                                                                                          |
+| 0.4.54          | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays                                                                                                      |
+| 0.4.53          | 2023-03-15 | [\#24058](https://github.com/airbytehq/airbyte/pull/24058) | added write attempt to internal staging Check method                                                                                                            |
+| 0.4.52          | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush                                                                                                                         |
+| 0.4.51          | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long                                                                                                                           |
+| 0.4.49          | 2023-02-27 | [\#23360](https://github.com/airbytehq/airbyte/pull/23360) | Added logging for flushing and writing data to destination storage                                                                                              |
+| 0.4.48          | 2023-02-23 | [\#22877](https://github.com/airbytehq/airbyte/pull/22877) | Add handler for IP not in whitelist error and more handlers for insufficient permission error                                                                   |
+| 0.4.47          | 2023-01-30 | [\#21912](https://github.com/airbytehq/airbyte/pull/21912) | Catch "Create" Table and Stage Known Permissions and rethrow as ConfigExceptions                                                                                |
+| 0.4.46          | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging                                                                                                        |
+| 0.4.45          | 2023-01-25 | [\#21087](https://github.com/airbytehq/airbyte/pull/21764) | Catch Known Permissions and rethrow as ConfigExceptions                                                                                                         |
+| 0.4.44          | 2023-01-20 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions                                                                                                                 |
+| 0.4.43          | 2023-01-20 | [\#21450](https://github.com/airbytehq/airbyte/pull/21450) | Updated Check methods to handle more possible s3 and gcs stagings issues                                                                                        |
+| 0.4.42          | 2023-01-12 | [\#21342](https://github.com/airbytehq/airbyte/pull/21342) | Better handling for conflicting destination streams                                                                                                             |
+| 0.4.41          | 2022-12-16 | [\#20566](https://github.com/airbytehq/airbyte/pull/20566) | Improve spec to adhere to standards                                                                                                                             |
+| 0.4.40          | 2022-11-11 | [\#19302](https://github.com/airbytehq/airbyte/pull/19302) | Set jdbc application env variable depends on env - airbyte_oss or airbyte_cloud                                                                                 |
+| 0.4.39          | 2022-11-09 | [\#18970](https://github.com/airbytehq/airbyte/pull/18970) | Updated "check" connection method to handle more errors                                                                                                         |
+| 0.4.38          | 2022-09-26 | [\#17115](https://github.com/airbytehq/airbyte/pull/17115) | Added connection string identifier                                                                                                                              |
+| 0.4.37          | 2022-09-21 | [\#16839](https://github.com/airbytehq/airbyte/pull/16839) | Update JDBC driver for Snowflake to 3.13.19                                                                                                                     |
+| 0.4.36          | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage                                                                                                                                  |
+| 0.4.35          | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields).                                       |
+| 0.4.34          | 2022-07-23 | [\#14388](https://github.com/airbytehq/airbyte/pull/14388) | Add support for key pair authentication                                                                                                                         |
+| 0.4.33          | 2022-07-15 | [\#14494](https://github.com/airbytehq/airbyte/pull/14494) | Make S3 output filename configurable.                                                                                                                           |
+| 0.4.32          | 2022-07-14 | [\#14618](https://github.com/airbytehq/airbyte/pull/14618) | Removed additionalProperties: false from JDBC destination connectors                                                                                            |
+| 0.4.31          | 2022-07-07 | [\#13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field description                                                                                                                         |
+| 0.4.30          | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging                                                                                     |
+| 0.4.29          | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager                                                                         |
+| 0.4.28          | 2022-05-18 | [\#12952](https://github.com/airbytehq/airbyte/pull/12952) | Apply buffering strategy on GCS staging                                                                                                                         |
+| 0.4.27          | 2022-05-17 | [\#12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance                                                                                                                          |
+| 0.4.26          | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessages on error.                                                                                              |
+| 0.4.25          | 2022-05-03 | [\#12452](https://github.com/airbytehq/airbyte/pull/12452) | Add support for encrypted staging on S3; fix the purge_staging_files option                                                                                     |
+| 0.4.24          | 2022-03-24 | [\#11093](https://github.com/airbytehq/airbyte/pull/11093) | Added OAuth support (Compatible with Airbyte Version 0.35.60+)                                                                                                  |
+| 0.4.22          | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters                                                                                                                           |
+| 0.4.21          | 2022-03-18 | [\#11071](https://github.com/airbytehq/airbyte/pull/11071) | Switch to compressed on-disk buffering before staging to s3/internal stage                                                                                      |
+| 0.4.20          | 2022-03-14 | [\#10341](https://github.com/airbytehq/airbyte/pull/10341) | Add Azure blob staging support                                                                                                                                  |
+| 0.4.19          | 2022-03-11 | [\#10699](https://github.com/airbytehq/airbyte/pull/10699) | Added unit tests                                                                                                                                                |
+| 0.4.17          | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling                                                                                                                               |
+| 0.4.16          | 2022-02-25 | [\#10627](https://github.com/airbytehq/airbyte/pull/10627) | Add try catch to make sure all handlers are closed                                                                                                              |
+| 0.4.15          | 2022-02-22 | [\#10459](https://github.com/airbytehq/airbyte/pull/10459) | Add FailureTrackingAirbyteMessageConsumer                                                                                                                       |
+| 0.4.14          | 2022-02-17 | [\#10394](https://github.com/airbytehq/airbyte/pull/10394) | Reduce memory footprint.                                                                                                                                        |
+| 0.4.13          | 2022-02-16 | [\#10212](https://github.com/airbytehq/airbyte/pull/10212) | Execute COPY command in parallel for S3 and GCS staging                                                                                                         |
+| 0.4.12          | 2022-02-15 | [\#10342](https://github.com/airbytehq/airbyte/pull/10342) | Use connection pool, and fix connection leak.                                                                                                                   |
+| 0.4.11          | 2022-02-14 | [\#9920](https://github.com/airbytehq/airbyte/pull/9920)   | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file.             |
+| 0.4.10          | 2022-02-14 | [\#10297](https://github.com/airbytehq/airbyte/pull/10297) | Halve the record buffer size to reduce memory consumption.                                                                                                      |
+| 0.4.9           | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `ExitOnOutOfMemoryError` JVM flag.                                                                                                                          |
+| 0.4.8           | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959)   | Fix null pointer exception from buffered stream consumer.                                                                                                       |
+| 0.4.7           | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745)   | Integrate with Sentry.                                                                                                                                          |
+| 0.4.6           | 2022-01-28 | [\#9623](https://github.com/airbytehq/airbyte/pull/9623)   | Add jdbc_url_params support for optional JDBC parameters                                                                                                        |
+| 0.4.5           | 2021-12-29 | [\#9184](https://github.com/airbytehq/airbyte/pull/9184)   | Update connector fields title/description                                                                                                                       |
+| 0.4.4           | 2022-01-24 | [\#9743](https://github.com/airbytehq/airbyte/pull/9743)   | Fixed bug with dashes in schema name                                                                                                                            |
+| 0.4.3           | 2022-01-20 | [\#9531](https://github.com/airbytehq/airbyte/pull/9531)   | Start using new S3StreamCopier and expose the purgeStagingData option                                                                                           |
+| 0.4.2           | 2022-01-10 | [\#9141](https://github.com/airbytehq/airbyte/pull/9141)   | Fixed duplicate rows on retries                                                                                                                                 |
+| 0.4.1           | 2021-01-06 | [\#9311](https://github.com/airbytehq/airbyte/pull/9311)   | Update сreating schema during check                                                                                                                             |
+| 0.4.0           | 2021-12-27 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063)   | Updated normalization to produce permanent tables                                                                                                               |
+| 0.3.24          | 2021-12-23 | [\#8869](https://github.com/airbytehq/airbyte/pull/8869)   | Changed staging approach to Byte-Buffered                                                                                                                       |
+| 0.3.23          | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039)   | Added part_size configuration in UI for S3 loading method                                                                                                       |
+| 0.3.22          | 2021-12-21 | [\#9006](https://github.com/airbytehq/airbyte/pull/9006)   | Updated jdbc schema naming to follow Snowflake Naming Conventions                                                                                               |
+| 0.3.21          | 2021-12-15 | [\#8781](https://github.com/airbytehq/airbyte/pull/8781)   | Updated check method to verify permissions to create/drop stage for internal staging; compatibility fix for Java 17                                             |
+| 0.3.20          | 2021-12-10 | [\#8562](https://github.com/airbytehq/airbyte/pull/8562)   | Moving classes around for better dependency management; compatibility fix for Java 17                                                                           |
+| 0.3.19          | 2021-12-06 | [\#8528](https://github.com/airbytehq/airbyte/pull/8528)   | Set Internal Staging as default choice                                                                                                                          |
+| 0.3.18          | 2021-11-26 | [\#8253](https://github.com/airbytehq/airbyte/pull/8253)   | Snowflake Internal Staging Support                                                                                                                              |
+| 0.3.17          | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719)   | Improve handling of wide rows by buffering records based on their byte size rather than their count                                                             |
+| 0.3.15          | 2021-10-11 | [\#6949](https://github.com/airbytehq/airbyte/pull/6949)   | Each stream was split into files of 10,000 records each for copying using S3 or GCS                                                                             |
+| 0.3.14          | 2021-09-08 | [\#5924](https://github.com/airbytehq/airbyte/pull/5924)   | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table                                                                         |
+| 0.3.13          | 2021-09-01 | [\#5784](https://github.com/airbytehq/airbyte/pull/5784)   | Updated query timeout from 30 minutes to 3 hours                                                                                                                |
+| 0.3.12          | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125)   | Enable `additionalPropertities` in spec.json                                                                                                                    |
+| 0.3.11          | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555)   | Partial Success in BufferedStreamConsumer                                                                                                                       |
+| 0.3.10          | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713)   | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake                                                                            |
diff --git a/docs/integrations/destinations/sqlite.md b/docs/integrations/destinations/sqlite.md
index 7c35543f5cc2..eb266b61eee8 100644
--- a/docs/integrations/destinations/sqlite.md
+++ b/docs/integrations/destinations/sqlite.md
@@ -22,18 +22,18 @@ Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a M
 
 Each stream will be output into its own table `_airbyte_raw_{stream_name}`. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
-* `_airbyte_data`: a json blob representing with the event data.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_data`: a json blob representing with the event data.
 
 #### Features
 
-| Feature | Supported |  |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces | No |  |
+| Feature                        | Supported |     |
+| :----------------------------- | :-------- | :-- |
+| Full Refresh Sync              | Yes       |     |
+| Incremental - Append Sync      | Yes       |     |
+| Incremental - Append + Deduped | No        |     |
+| Namespaces                     | No        |     |
 
 #### Performance considerations
 
@@ -49,13 +49,13 @@ The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local`
 
 ### Example:
 
-* If `destination_path` is set to `/local/sqlite.db`
-* the local mount is using the `/tmp/airbyte_local` default
-* then all data will be written to `/tmp/airbyte_local/sqlite.db`.
+- If `destination_path` is set to `/local/sqlite.db`
+- the local mount is using the `/tmp/airbyte_local` default
+- then all data will be written to `/tmp/airbyte_local/sqlite.db`.
 
 ## Access Replicated Data Files
 
-If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte\_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
+If your Airbyte instance is running on the same computer that you are navigating with, you can open your browser and enter [file:///tmp/airbyte_local](file:///tmp/airbyte_local) to look at the replicated data locally. If the first approach fails or if your Airbyte instance is running on a remote server, follow the following steps to access the replicated files:
 
 1. Access the scheduler container using `docker exec -it airbyte-server bash`
 2. Navigate to the default local mount using `cd /tmp/airbyte_local`
@@ -72,6 +72,6 @@ Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.0 | 2022-07-25 | [15018](https://github.com/airbytehq/airbyte/pull/15018) | New SQLite destination |
+| Version | Date       | Pull Request                                             | Subject                |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------- |
+| 0.1.0   | 2022-07-25 | [15018](https://github.com/airbytehq/airbyte/pull/15018) | New SQLite destination |
diff --git a/docs/integrations/destinations/starburst-galaxy.md b/docs/integrations/destinations/starburst-galaxy.md
index 710cbb2f0883..2fdfc77cd0b5 100644
--- a/docs/integrations/destinations/starburst-galaxy.md
+++ b/docs/integrations/destinations/starburst-galaxy.md
@@ -2,47 +2,47 @@
 
 ## Overview
 
-The Starburst Galaxy destination syncs data to Starburst Galaxy [great lake catalogs](https://docs.starburst.io/starburst-galaxy/sql/great-lakes.html)  
+The Starburst Galaxy destination syncs data to Starburst Galaxy [great lake catalogs](https://docs.starburst.io/starburst-galaxy/sql/great-lakes.html)
 in [Apache Iceberg](https://iceberg.apache.org/) table format. Each stream is written to its own Iceberg table.
 
 ## Features
 
-| Feature         | Supported | Notes                                                                               |
-|:----------------|:---------:|:------------------------------------------------------------------------------------|
-| Overwrite Sync  |     ✅     | **Warning**: this mode deletes all previously synced data in the destination table. |
-| Append Sync     |     ✅     |                                                                                     |
-| Deduped History |     ❌     |                                                                                     |
-| Namespaces      |     ✅     |                                                                                     |
-| SSL             |     ✅     | SSL is enabled.                                                                     |    
+| Feature          | Supported | Notes                                                                               |
+| :--------------- | :-------: | :---------------------------------------------------------------------------------- |
+| Overwrite Sync   |    ✅     | **Warning**: this mode deletes all previously synced data in the destination table. |
+| Append Sync      |    ✅     |                                                                                     |
+| Append + Deduped |    ❌     |                                                                                     |
+| Namespaces       |    ✅     |                                                                                     |
+| SSL              |    ✅     | SSL is enabled.                                                                     |
 
 ## Data storage
 
-Starburst Galaxy supports various [object storages](https://docs.starburst.io/starburst-galaxy/catalogs/index.html#object-storage); 
+Starburst Galaxy supports various [object storages](https://docs.starburst.io/starburst-galaxy/catalogs/index.html#object-storage);
 however, only Amazon S3 is supported by this connector.
 
 ## Configuration
 
-| Category                         | Parameter                     |  Type   | Notes                                                                                                                                                                                                                                                                                            |
-|:---------------------------------|:------------------------------|:-------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Starburst Galaxy                 | `Hostname`                    | string  | Required. Located in the **Connection info** section of the [view clusters](https://docs.starburst.io/starburst-galaxy/clusters/index.html#manage-clusters) pane in Starburst Galaxy.                                                                                                            |
-|                                  | `Port`                        | string  | Optional. Located in the **Connection info** section of the [view clusters](https://docs.starburst.io/starburst-galaxy/clusters/index.html#manage-clusters) pane in Starburst Galaxy. Defaults to `443`.                                                                                         |
-|                                  | `User`                        | string  | Required. Galaxy user found in the **Connection info** section of the [view clusters](https://docs.starburst.io/starburst-galaxy/clusters/index.html#manage-clusters) pane in Starburst Galaxy.                                                                                                  |
-|                                  | `Password`                    | string  | Required. Password for the specified Galaxy user.                                                                                                                                                                                                                                                |
-|                                  | `Amazon S3 catalog`           | string  | Required. Name of the [Amazon S3 catalog](https://docs.starburst.io/starburst-galaxy/catalogs/s3.html) created in the Galaxy domain.                                                                                                                                                             |
-|                                  | `Amazon S3 catalog schema`    | string  | Optional. The default Starburst Galaxy Amazon S3 catalog schema where tables are written to if the source does not specify a namespace. Each data stream is written to a table in this schema. Defaults to `public`.                                                                             |
-| Staging Object Store - Amazon S3 | `Bucket name`                 | string  | Required. Name of the bucket where the staging data is stored.                                                                                                                                                                                                                                   |
-|                                  | `Bucket path`                 | string  | Required. Sets the subdirectory of the specified S3 bucket used for storing staging data.                                                                                                                                                                                                        |
-|                                  | `Bucket region`               | string  | Required. Sets the region of the specified S3 bucket.                                                                                                                                                                                                                                            |
-|                                  | `Access key`                  | string  | Required. AWS/Minio credential.                                                                                                                                                                                                                                                                  |
-|                                  | `Secret key`                  | string  | Required. AWS/Minio credential.                                                                                                                                                                                                                                                                  |
-| General                          | `Purge staging Iceberg table` | boolean | Optional. Indicates that staging Iceberg table is purged after a data sync is complete. Enabled by default. Disable it for debugging purposes only.                                                                                                                                              |
+| Category                         | Parameter                     |  Type   | Notes                                                                                                                                                                                                                |
+| :------------------------------- | :---------------------------- | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Starburst Galaxy                 | `Hostname`                    | string  | Required. Located in the **Connection info** section of the [view clusters](https://docs.starburst.io/starburst-galaxy/clusters/index.html#manage-clusters) pane in Starburst Galaxy.                                |
+|                                  | `Port`                        | string  | Optional. Located in the **Connection info** section of the [view clusters](https://docs.starburst.io/starburst-galaxy/clusters/index.html#manage-clusters) pane in Starburst Galaxy. Defaults to `443`.             |
+|                                  | `User`                        | string  | Required. Galaxy user found in the **Connection info** section of the [view clusters](https://docs.starburst.io/starburst-galaxy/clusters/index.html#manage-clusters) pane in Starburst Galaxy.                      |
+|                                  | `Password`                    | string  | Required. Password for the specified Galaxy user.                                                                                                                                                                    |
+|                                  | `Amazon S3 catalog`           | string  | Required. Name of the [Amazon S3 catalog](https://docs.starburst.io/starburst-galaxy/catalogs/s3.html) created in the Galaxy domain.                                                                                 |
+|                                  | `Amazon S3 catalog schema`    | string  | Optional. The default Starburst Galaxy Amazon S3 catalog schema where tables are written to if the source does not specify a namespace. Each data stream is written to a table in this schema. Defaults to `public`. |
+| Staging Object Store - Amazon S3 | `Bucket name`                 | string  | Required. Name of the bucket where the staging data is stored.                                                                                                                                                       |
+|                                  | `Bucket path`                 | string  | Required. Sets the subdirectory of the specified S3 bucket used for storing staging data.                                                                                                                            |
+|                                  | `Bucket region`               | string  | Required. Sets the region of the specified S3 bucket.                                                                                                                                                                |
+|                                  | `Access key`                  | string  | Required. AWS/Minio credential.                                                                                                                                                                                      |
+|                                  | `Secret key`                  | string  | Required. AWS/Minio credential.                                                                                                                                                                                      |
+| General                          | `Purge staging Iceberg table` | boolean | Optional. Indicates that staging Iceberg table is purged after a data sync is complete. Enabled by default. Disable it for debugging purposes only.                                                                  |
 
 ## Staging files
 
 ### S3
 
-Data streams are written to a temporary Iceberg table, and then loaded into Amazon S3 Starburst Galaxy catalog in the Iceberg table format. 
-Staging table is deleted after a sync is complete if the `Purge staging Iceberg table` is enabled. 
+Data streams are written to a temporary Iceberg table, and then loaded into Amazon S3 Starburst Galaxy catalog in the Iceberg table format.
+Staging table is deleted after a sync is complete if the `Purge staging Iceberg table` is enabled.
 The following is an example of a full path for a staging file:
 
 ```text
@@ -54,9 +54,9 @@ For example:
 ```text
 s3://galaxy_bucket/data_output_path/test_schema/_airbyte_tmp_qey_user
      ↑              ↑                     ↑                ↑
-     |              |                     |                temporary Iceberg table holding data 
+     |              |                     |                temporary Iceberg table holding data
      |              |                     source namespace or provided schema name
-     |              |                
+     |              |
      |              bucket path
      bucket name
 ```
@@ -69,21 +69,21 @@ Streams are synced in the Starburst Galaxy Amazon S3 catalog with Iceberg table
 
 Each table in the output schema has the following columns:
 
-| Column                                                        |         Type          | Description                                                                                          |
-|:--------------------------------------------------------------|:---------------------:|:-----------------------------------------------------------------------------------------------------|
-| `_airbyte_ab_id`                                              |        varchar        | UUID.                                                                                                |
-| `_airbyte_emitted_at`                                         |     timestamp(6)      | Data emission timestamp.                                                                             |
-| Data fields from the source stream                            |        various        | All the fields from the source stream will be populated as an individual column in the target table. |
-| `_airbyte_additional_properties`                              | map(varchar, varchar) | Additional properties.                                                                               |
+| Column                             |         Type          | Description                                                                                          |
+| :--------------------------------- | :-------------------: | :--------------------------------------------------------------------------------------------------- |
+| `_airbyte_ab_id`                   |        varchar        | UUID.                                                                                                |
+| `_airbyte_emitted_at`              |     timestamp(6)      | Data emission timestamp.                                                                             |
+| Data fields from the source stream |        various        | All the fields from the source stream will be populated as an individual column in the target table. |
+| `_airbyte_additional_properties`   | map(varchar, varchar) | Additional properties.                                                                               |
 
-The Airbyte data stream's JSON schema is converted to an Avro schema. The JSON object is then converted to an Avro record; 
-the Avro record is written to a staging Iceberg table. As the data stream can be generated from any data source, 
-the JSON-to-Avro conversion process has arbitrary rules and limitations. 
+The Airbyte data stream's JSON schema is converted to an Avro schema. The JSON object is then converted to an Avro record;
+the Avro record is written to a staging Iceberg table. As the data stream can be generated from any data source,
+the JSON-to-Avro conversion process has arbitrary rules and limitations.
 Learn more about [how source data is converted to Avro](https://docs.airbyte.io/understanding-airbyte/json-avro-conversion).
 
 ### Datatype support
 
-Learn more about [Starburst Galaxy Iceberg type mapping](https://docs.starburst.io/latest/connector/iceberg.html#iceberg-to-trino-type-mapping). 
+Learn more about [Starburst Galaxy Iceberg type mapping](https://docs.starburst.io/latest/connector/iceberg.html#iceberg-to-trino-type-mapping).
 
 ## Getting started
 
@@ -97,5 +97,5 @@ Learn more about [Starburst Galaxy Iceberg type mapping](https://docs.starburst.
 ## Changelog
 
 | Version | Date       | Pull Request                                               | Subject                 |
-|:--------|:-----------|:-----------------------------------------------------------|:------------------------|
+| :------ | :--------- | :--------------------------------------------------------- | :---------------------- |
 | 0.0.1   | 2023-03-28 | [\#24620](https://github.com/airbytehq/airbyte/pull/24620) | Initial public release. |
diff --git a/docs/integrations/destinations/streamr.md b/docs/integrations/destinations/streamr.md
index 40f6a0c76a5c..97fbe15765df 100644
--- a/docs/integrations/destinations/streamr.md
+++ b/docs/integrations/destinations/streamr.md
@@ -2,12 +2,12 @@
 
 ## Features
 
-| Feature                       | Support | Notes                                                                                        |
-| :---------------------------- | :-----: | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             |   ❌    | Warning: this mode deletes all previously synced data in the configured bucket path.         |
-| Incremental - Append Sync     |   ✅    |                                                                                              |
-| Incremental - Deduped History |   ❌    | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.                  |
+| Feature                        | Support | Notes                                                                                |
+| :----------------------------- | :-----: | :----------------------------------------------------------------------------------- |
+| Full Refresh Sync              |   ❌    | Warning: this mode deletes all previously synced data in the configured bucket path. |
+| Incremental - Append Sync      |   ✅    |                                                                                      |
+| Incremental - Append + Deduped |   ❌    |                                                                                      |
+| Namespaces                     |   ❌    | Setting a specific bucket path is equivalent to having separate namespaces.          |
 
 The Streamr destination allows you to sync data to Streamr - The decentralized
 real‑time data network.
diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md
index 6ea038fce3ea..213f4d0692da 100644
--- a/docs/integrations/destinations/teradata.md
+++ b/docs/integrations/destinations/teradata.md
@@ -6,17 +6,17 @@ This page guides you through the process of setting up the Teradata destination
 
 To use the Teradata destination connector, you'll need:
 
-* Access to a Teradata Vantage instance
+- Access to a Teradata Vantage instance
 
-    **Note:** If you need a new instance of Vantage, you can install a free version called Vantage Express in the cloud on [Google Cloud](https://quickstarts.teradata.com/vantage.express.gcp.html), [Azure](https://quickstarts.teradata.com/run-vantage-express-on-microsoft-azure.html), and [AWS](https://quickstarts.teradata.com/run-vantage-express-on-aws.html). You can also run Vantage Express on your local machine using [VMware](https://quickstarts.teradata.com/getting.started.vmware.html), [VirtualBox](https://quickstarts.teradata.com/getting.started.vbox.html), or [UTM](https://quickstarts.teradata.com/getting.started.utm.html).
+  **Note:** If you need a new instance of Vantage, you can install a free version called Vantage Express in the cloud on [Google Cloud](https://quickstarts.teradata.com/vantage.express.gcp.html), [Azure](https://quickstarts.teradata.com/run-vantage-express-on-microsoft-azure.html), and [AWS](https://quickstarts.teradata.com/run-vantage-express-on-aws.html). You can also run Vantage Express on your local machine using [VMware](https://quickstarts.teradata.com/getting.started.vmware.html), [VirtualBox](https://quickstarts.teradata.com/getting.started.vbox.html), or [UTM](https://quickstarts.teradata.com/getting.started.utm.html).
 
 You'll need the following information to configure the Teradata destination:
 
-* **Host** - The host name of the Teradata Vantage instance.
-* **Username**
-* **Password**
-* **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in the search-path. These schemas will be used to resolve unqualified object names used in statements executed over this connection.
-* **JDBC URL Params** (optional)
+- **Host** - The host name of the Teradata Vantage instance.
+- **Username**
+- **Password**
+- **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in the search-path. These schemas will be used to resolve unqualified object names used in statements executed over this connection.
+- **JDBC URL Params** (optional)
 
 [Refer to this guide for more details](https://downloads.teradata.com/doc/connectivity/jdbc/reference/current/jdbcug_chapter_2.html#BGBHDDGB)
 
@@ -26,23 +26,21 @@ You'll need the following information to configure the Teradata destination:
 
 Each stream will be output into its own table in Teradata. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`.
-* `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`.
-
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Teradata is `VARCHAR(256)`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Teradata is `TIMESTAMP(6)`.
+- `_airbyte_data`: a json blob representing with the event data. The column type in Teradata is `JSON`.
 
 ### Features
 
 The Teradata destination connector supports the
 following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No |  |
-| Namespaces | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | Yes                  |       |
 
 ### Performance considerations
 
@@ -52,8 +50,8 @@ following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-s
 
 You need a Teradata user with the following permissions:
 
-* can create tables and write permission.
-* can create schemas e.g:
+- can create tables and write permission.
+- can create schemas e.g:
 
 You can create such a user by running:
 
@@ -64,6 +62,7 @@ GRANT ALL on dbc to airbyte_user;
 ```
 
 You can also use a pre-existing user but we highly recommend creating a dedicated user for Airbyte.
+
 ### Setup guide
 
 #### Set up the Teradata Destination connector
@@ -83,7 +82,8 @@ You can also use a pre-existing user but we highly recommend creating a dedicate
 
 ## CHANGELOG
 
-| Version | Date       | Pull Request                                                  | Subject                          |
-|:--------|:-----------|:--------------------------------------------------------------|:---------------------------------|
-| 0.1.0   | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428			   | New Destination Teradata Vantage |
-| 0.1.1   | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760			   | Added SSL support                |
\ No newline at end of file
+| Version | Date       | Pull Request                                    | Subject                          |
+| :------ | :--------- | :---------------------------------------------- | :------------------------------- |
+| 0.1.2   | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor          |
+| 0.1.1   | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support                |
+| 0.1.0   | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage |
diff --git a/docs/integrations/destinations/tidb.md b/docs/integrations/destinations/tidb.md
index ff7dc37a57ce..c7f646c4bedd 100644
--- a/docs/integrations/destinations/tidb.md
+++ b/docs/integrations/destinations/tidb.md
@@ -6,21 +6,21 @@ This page guides you through the process of setting up the TiDB destination conn
 
 ## Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes |
-|:------------------------------|:---------------------|:------|
-| Full Refresh Sync             | Yes                  |       |
-| Incremental - Append Sync     | Yes                  |       |
-| Incremental - Deduped History | Yes                  |       |
-| Namespaces                    | Yes                  |       |
-| SSH Tunnel Connection         | Yes                  |       |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | Yes                  |       |
+| Namespaces                     | Yes                  |       |
+| SSH Tunnel Connection          | Yes                  |       |
 
 #### Output Schema
 
 Each stream will be output into its own table in TiDB. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in TiDB is `VARCHAR(256)`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in TiDB is `TIMESTAMP(6)`.
-* `_airbyte_data`: a json blob representing with the event data. The column type in TiDB is `JSON`.
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in TiDB is `VARCHAR(256)`.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in TiDB is `TIMESTAMP(6)`.
+- `_airbyte_data`: a json blob representing with the event data. The column type in TiDB is `JSON`.
 
 ## Getting Started
 
@@ -28,7 +28,7 @@ Each stream will be output into its own table in TiDB. Each table will contain 3
 
 To use the TiDB destination, you'll need:
 
-* To sync data to TiDB **with normalization** you should have a TiDB database v5.4.0 or above.
+- To sync data to TiDB **with normalization** you should have a TiDB database v5.4.0 or above.
 
 #### Network Access
 
@@ -58,20 +58,20 @@ TiDB doesn't differentiate between a database and schema. A database is essentia
 
 Config the following information in the TiDB destination:
 
-* **Host**
-* **Port**
-* **Username**
-* **Password**
-* **Database**
-* **jdbc_url_params** (Optional)
+- **Host**
+- **Port**
+- **Username**
+- **Password**
+- **Database**
+- **jdbc_url_params** (Optional)
 
 **Note:** When connecting to TiDB Cloud with TLS enabled, you need to specify TLS protocol, such as `enabledTLSProtocols=TLSv1.2` or `enabledTLSProtocols=TLSv1.3` in the JDBC parameters.
 
 ### Default JDBC URL Parameters
 
-* `useSSL=false` (unless `ssl` is set to true)
-* `requireSSL=false` (unless `ssl` is set to true)
-* `verifyServerCertificate=false` (unless `ssl` is set to true)
+- `useSSL=false` (unless `ssl` is set to true)
+- `requireSSL=false` (unless `ssl` is set to true)
+- `verifyServerCertificate=false` (unless `ssl` is set to true)
 
 ## Known Limitations
 
@@ -87,8 +87,8 @@ Using this feature requires additional configuration, when creating the destinat
 
 1. Configure all fields for the destination as you normally would, except `SSH Tunnel Method`.
 2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`.
-    1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
-    2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
+   1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\).
+   2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel.
 3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address.
 4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default.
 5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the TiDB username.
@@ -97,8 +97,10 @@ Using this feature requires additional configuration, when creating the destinat
 
 ## CHANGELOG
 
-| Version | Date       | Pull Request                                               | Subject                               |
-|:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------|
-| 0.1.2   | 2023-05-23 | [\#19109](https://github.com/airbytehq/airbyte/pull/19109) | Enabled Append Dedub mode             |
-| 0.1.1   | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing |
-| 0.1.0   | 2022-08-12 | [\#15592](https://github.com/airbytehq/airbyte/pull/15592) | Added TiDB destination.               |
+| Version | Date       | Pull Request                                               | Subject                                                                                       |
+| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------- |
+| 0.1.4   | 2023-06-21 | [\#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size                                                                             |
+| 0.1.3   | 2023-06-05 | [\#27025](https://github.com/airbytehq/airbyte/pull/27025) | Internal code change for future development (install normalization packages inside connector) |
+| 0.1.2   | 2023-05-23 | [\#19109](https://github.com/airbytehq/airbyte/pull/19109) | Enabled Append Dedub mode                                                                     |
+| 0.1.1   | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing                                                         |
+| 0.1.0   | 2022-08-12 | [\#15592](https://github.com/airbytehq/airbyte/pull/15592) | Added TiDB destination.                                                                       |
diff --git a/docs/integrations/destinations/timeplus.md b/docs/integrations/destinations/timeplus.md
new file mode 100644
index 000000000000..dcf43cc48225
--- /dev/null
+++ b/docs/integrations/destinations/timeplus.md
@@ -0,0 +1,37 @@
+# Timeplus
+
+This page guides you through the process of setting up the [Timeplus](https://timeplus.com) destination connector.
+
+## Features
+
+| Feature | Supported?\(Yes/No\) | Notes |
+| :--- | :--- | :--- |
+| Overwrite | Yes |  |
+| Incremental - Append Sync | Yes |  |
+
+
+#### Output Schema
+
+Each stream will be output into its own stream in Timeplus, with corresponding schema/columns.
+## Getting Started (Airbyte Cloud)
+Coming soon...
+
+## Getting Started (Airbyte Open-Source)
+You can follow the [Quickstart with Timeplus Ingestion API](https://docs.timeplus.com/quickstart-ingest-api) to createa a workspace and API key.
+
+### Setup the Timeplus Destination in Airbyte
+
+You should now have all the requirements needed to configure Timeplus as a destination in the UI. You'll need the following information to configure the Timeplus destination:
+
+* **Endpoint** example https://us.timeplus.cloud/randomId123
+* **API key**
+
+## Compatibility
+
+
+## Changelog
+
+| Version | Date       | Pull Request                                             | Subject                                         |
+|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------|
+| 0.1.0   | 2023-06-14 | [21226](https://github.com/airbytehq/airbyte/pull/21226) | Destination Timeplus                            |
+
diff --git a/docs/integrations/destinations/typesense.md b/docs/integrations/destinations/typesense.md
index 3bf9aac71c75..f9dfff351c60 100644
--- a/docs/integrations/destinations/typesense.md
+++ b/docs/integrations/destinations/typesense.md
@@ -16,12 +16,12 @@ Each stream will be output into its own collection in Typesense. If an id column
 
 #### Features
 
-| Feature                       | Supported?\(Yes/No\) | Notes                                                                                        |
-| :---------------------------- | :------------------- | :------------------------------------------------------------------------------------------- |
-| Full Refresh Sync             | Yes                  |                                                                                              |
-| Incremental - Append Sync     | Yes                  |                                                                                              |
-| Incremental - Deduped History | No                   | As this connector does not support dbt, we don't support this sync mode on this destination. |
-| Namespaces                    | No                   |                                                                                              |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | No                   |       |
 
 ## Getting started
 
@@ -35,6 +35,7 @@ The setup only requires two fields. First is the `host` which is the address at
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                   |
-| :-------| :--------- | :------------------------------------------------------- | :-------------------------|
-| 0.1.0   | 2022-10-28 | [18349](https://github.com/airbytehq/airbyte/pull/18349) | New Typesense destination |
+| Version | Date       | Pull Request                                             | Subject                       |
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------- |
+| 0.1.0   | 2022-10-28 | [18349](https://github.com/airbytehq/airbyte/pull/18349) | New Typesense destination     |
+| 0.1.1   | 2023-22-17 | [29555](https://github.com/airbytehq/airbyte/pull/29555) | Increasing connection timeout |
diff --git a/docs/integrations/destinations/vertica.md b/docs/integrations/destinations/vertica.md
index 6d5ba806dd25..cd829121f50a 100644
--- a/docs/integrations/destinations/vertica.md
+++ b/docs/integrations/destinations/vertica.md
@@ -6,21 +6,21 @@ This page guides you through the process of setting up the vertica destination c
 
 To use the Vertica destination, you'll need:
 
-* A V
-ertica server version 11.0 or above
+- A V
+  ertica server version 11.0 or above
 
 Airbyte Cloud only supports connecting to your Vertica instances with SSL or TLS encryption. TLS is
 used by default. Other than that, you can proceed with the open-source instructions below.
 
 You'll need the following information to configure the Vertica destination:
 
-* **Host** - The host name of the server.
-* **Port** - The port number the server is listening on. Defaults to the VSQL™ standard port number (5433).
-* **Username**
-* **Password**
-* **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in the search-path. These schemas will be used to resolve unqualified object names used in statements executed over this connection.
-* **Database** - The database name. The default is to connect to a database with the same name as the user name.
-* **JDBC URL Params** (optional)
+- **Host** - The host name of the server.
+- **Port** - The port number the server is listening on. Defaults to the VSQL™ standard port number (5433).
+- **Username**
+- **Password**
+- **Default Schema Name** - Specify the schema (or several schemas separated by commas) to be set in the search-path. These schemas will be used to resolve unqualified object names used in statements executed over this connection.
+- **Database** - The database name. The default is to connect to a database with the same name as the user name.
+- **JDBC URL Params** (optional)
 
 [Refer to this guide for more details](https://www.vertica.com/docs/12.0.4/HTML/Content/Authoring/ConnectingToVertica/ClientJDBC/JDBCConnectionProperties.htm)
 
@@ -35,8 +35,8 @@ may need to allow access from the IP you're using to expose Airbyte.
 
 You need a Vertica user with the following permissions:
 
-* can create tables and write rows.
-* can create schemas e.g:
+- can create tables and write rows.
+- can create schemas e.g:
 
 You can create such a user by running:
 
@@ -58,9 +58,9 @@ synced data from Airbyte.
 
 From [Vertica SQL Identifiers syntax](https://www.vertica.com/docs/12.0.x/HTML/Content/Authoring/ConnectingToVertica/ClientJDBC/ExecutingQueriesThroughJDBC.htm?tocpath=Connecting%20to%20Vertica%7CClient%20Libraries%7CProgramming%20JDBC%20Client%20Applications%7C_____4):
 
-* SQL identifiers and key words must begin with a letter \(a-z, but also letters with diacritical
+- SQL identifiers and key words must begin with a letter \(a-z, but also letters with diacritical
   marks and non-Latin letters\) or an underscore \(\_\).
-* Subsequent characters in an identifier or key word can be letters, underscores, digits \(0-9\), or
+- Subsequent characters in an identifier or key word can be letters, underscores, digits \(0-9\), or
   dollar signs \($\).
 
   Note that dollar signs are not allowed in identifiers according to the SQL standard,
@@ -68,16 +68,16 @@ From [Vertica SQL Identifiers syntax](https://www.vertica.com/docs/12.0.x/HTML/C
   that contains digits or starts or ends with an underscore, so identifiers of this form are safe
   against possible conflict with future extensions of the standard.
 
-* The system uses no more than NAMEDATALEN-1 bytes of an identifier; longer names can be written in
+- The system uses no more than NAMEDATALEN-1 bytes of an identifier; longer names can be written in
   commands, but they will be truncated. By default, NAMEDATALEN is 64 so the maximum identifier
   length is 63 bytes
-* Quoted identifiers can contain any character, except the character with code zero. \(To include a
+- Quoted identifiers can contain any character, except the character with code zero. \(To include a
   double quote, write two double quotes.\) This allows constructing table or column names that would
   otherwise not be possible, such as ones containing spaces or ampersands. The length limitation
   still applies.
-* Quoting an identifier also makes it case-sensitive, whereas unquoted names are always folded to
+- Quoting an identifier also makes it case-sensitive, whereas unquoted names are always folded to
   lower case.
-* In order to make your applications portable and less error-prone, use consistent quoting with each name (either always quote it or never quote it).
+- In order to make your applications portable and less error-prone, use consistent quoting with each name (either always quote it or never quote it).
 
 Note, that Airbyte Vertica destination will create tables and schemas using the Unquoted
 identifiers when possible or fallback to Quoted Identifiers if the names are containing special
@@ -92,19 +92,19 @@ characters.
 4. Enter a name for your source.
 5. For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your Vertica database.
 6. List the **Default Schemas**.
-    :::note
-    The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing.
-    :::
+   :::note
+   The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing.
+   :::
 7. For **User** and **Password**, enter the username and password you created in [Step 1](#step-1-optional-create-a-dedicated-read-only-user).
-9. For Airbyte Open Source, toggle the switch to connect using SSL. For Airbyte Cloud uses SSL by default.
-10. For SSL Modes, select:
-    - **disable** to disable encrypted communication between Airbyte and the source
-    - **allow** to enable encrypted communication only when required by the source
-    - **prefer** to allow unencrypted communication only when the source doesn't support encryption
-    - **require** to always require encryption. Note: The connection will fail if the source doesn't support encryption.
-    - **verify-ca** to always require encryption and verify that the source has a valid SSL certificate
-    - **verify-full** to always require encryption and verify the identity of the source
-11. To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://www.vertica.com/docs/12.0.x/HTML/Content/Authoring/ConnectingToVertica/ClientJDBC/JDBCConnectionProperties.htm) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field.
+8. For Airbyte Open Source, toggle the switch to connect using SSL. For Airbyte Cloud uses SSL by default.
+9. For SSL Modes, select:
+   - **disable** to disable encrypted communication between Airbyte and the source
+   - **allow** to enable encrypted communication only when required by the source
+   - **prefer** to allow unencrypted communication only when the source doesn't support encryption
+   - **require** to always require encryption. Note: The connection will fail if the source doesn't support encryption.
+   - **verify-ca** to always require encryption and verify that the source has a valid SSL certificate
+   - **verify-full** to always require encryption and verify the identity of the source
+10. To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://www.vertica.com/docs/12.0.x/HTML/Content/Authoring/ConnectingToVertica/ClientJDBC/JDBCConnectionProperties.htm) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field.
 
     Example: key1=value1&key2=value2&key3=value3
 
@@ -118,14 +118,17 @@ characters.
     :::warning
     This is an advanced configuration option. Users are advised to use it with caution.
     :::
+
 11. For SSH Tunnel Method, select:
+
     - **No Tunnel** for a direct connection to the database
     - **SSH Key Authentication** to use an RSA Private as your secret for establishing the SSH tunnel
     - **Password Authentication** to use a password as your secret for establishing the SSH tunnel
-    
+
     :::warning
     Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **disable**, **allow**, or **prefer** as the **SSL Mode**; otherwise, the connection will fail.
     :::
+
 12. Click **Set up destination**.
 
 ## Supported sync modes
@@ -133,12 +136,12 @@ characters.
 The Vertica destination connector supports the
 following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-| Feature                       | Supported?\(Yes/No\) | Notes |
-|:------------------------------|:---------------------|:------|
-| Full Refresh Sync             | Yes                  |       |
-| Incremental - Append Sync     | No                  |       |
-| Incremental - Deduped History | No                  |       |
-| Namespaces                    | No                  |       |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | Yes                  |       |
+| Incremental - Append Sync      | No                   |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | No                   |       |
 
 ## Schema map
 
@@ -146,15 +149,15 @@ following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-s
 
 Each stream will be mapped to a separate table in Vertica. Each table will contain 3 columns:
 
-* `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in
+- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in
   Vertica is `VARCHAR`.
-* `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
+- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source.
   The column type in Vertica is `TIMESTAMP WITH TIME ZONE`.
-* `_airbyte_data`: a json blob representing with the event data. The column type in Vertica
+- `_airbyte_data`: a json blob representing with the event data. The column type in Vertica
   is `JSONB`.
 
 ## Changelog
 
-| Version | Date       | Pull Request | Subject                                      |
-|:--------|:-----------| :--- |:---------------------------------------------|
+| Version | Date       | Pull Request                                               | Subject                 |
+| :------ | :--------- | :--------------------------------------------------------- | :---------------------- |
 | 0.1.0   | 2023-05-29 | [\#25682](https://github.com/airbytehq/airbyte/pull/25682) | Add Vertica Destination |
diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md
index 25812fd2b92b..3b54c45a0d6e 100644
--- a/docs/integrations/destinations/weaviate.md
+++ b/docs/integrations/destinations/weaviate.md
@@ -2,13 +2,13 @@
 
 ## Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | No |  |
-| Incremental - Append Sync | Yes |  |
-| Incremental - Deduped History | No |  |
-| Namespaces | No |  |
-| Provide vector | Yes |  |
+| Feature                        | Supported?\(Yes/No\) | Notes |
+| :----------------------------- | :------------------- | :---- |
+| Full Refresh Sync              | No                   |       |
+| Incremental - Append Sync      | Yes                  |       |
+| Incremental - Append + Deduped | No                   |       |
+| Namespaces                     | No                   |       |
+| Provide vector                 | Yes                  |       |
 
 #### Output Schema
 
@@ -18,15 +18,17 @@ in the Weaviate class.
 **Uploading Vectors:** Use the vectors configuration if you want to upload
 vectors from a source database into Weaviate. You can do this by specifying
 the stream name and vector field name in the following format:
+
 ```
 ., .
 ```
+
 For example, if you have a table named `my_table` and the vector is stored using the column `vector` then
 you should use the following `vectors`configuration: `my_table.vector`.
 
 Dynamic Schema: Weaviate will automatically create a schema for the stream if no class was defined unless
 you have disabled the Dynamic Schema feature in Weaviate. You can also create the class in Weaviate in advance
-if you need more control over the schema in Weaviate. 
+if you need more control over the schema in Weaviate.
 
 IDs: If your source table has an int based id stored as field name `id` then the
 ID will automatically be converted to a UUID. Weaviate only supports the ID to be a UUID.
@@ -48,7 +50,7 @@ password.
 
 To use the Weaviate destination, you'll need:
 
-* A Weaviate cluster version 21.8.10.19 or above
+- A Weaviate cluster version 21.8.10.19 or above
 
 #### Configure Network Access
 
@@ -58,24 +60,21 @@ Make sure your Weaviate database can be accessed by Airbyte. If your database is
 
 You need a Weaviate user or use a Weaviate instance that's accessible to all
 
-
 ### Setup the Weaviate Destination in Airbyte
 
 You should now have all the requirements needed to configure Weaviate as a destination in the UI. You'll need the following information to configure the Weaviate destination:
 
-* **URL** for example http://localhost:8080 or https://my-wcs.semi.network
-* **Username** (Optional)
-* **Password** (Optional)
-* **Batch Size** (Optional, defaults to 100)
-* **Vectors** a comma separated list of `` to specify the field
-* **ID Schema** a comma separated list of `` to specify the field
+- **URL** for example http://localhost:8080 or https://my-wcs.semi.network
+- **Username** (Optional)
+- **Password** (Optional)
+- **Batch Size** (Optional, defaults to 100)
+- **Vectors** a comma separated list of `` to specify the field
+- **ID Schema** a comma separated list of `` to specify the field
   name that contains the ID of a record
 
-
 ## Changelog
 
-| Version | Date       | Pull Request | Subject                                      |
-|:--------|:-----------| :--- |:---------------------------------------------|
+| Version | Date       | Pull Request                                               | Subject                                                                                                                          |
+| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- |
 | 0.1.1   | 2022-02-08 | [\#22527](https://github.com/airbytehq/airbyte/pull/22527) | Multiple bug fixes: Support String based IDs, arrays of uknown type and additionalProperties of type object and array of objects |
-| 0.1.0   | 2022-12-06 | [\#20094](https://github.com/airbytehq/airbyte/pull/20094) | Add Weaviate destination |
-
+| 0.1.0   | 2022-12-06 | [\#20094](https://github.com/airbytehq/airbyte/pull/20094) | Add Weaviate destination                                                                                                         |
diff --git a/docs/integrations/destinations/xata.md b/docs/integrations/destinations/xata.md
new file mode 100644
index 000000000000..148cf17a77e0
--- /dev/null
+++ b/docs/integrations/destinations/xata.md
@@ -0,0 +1,36 @@
+# Xata
+
+Airbyte destination connector for Xata.
+
+## Introduction
+
+Currently only `append` is supported.
+
+Conventions:
+
+- The `stream` name will define the name of the table in Xata.
+- The `message` data will be mapped one by one to the table schema.
+
+For example, a stream name `nyc_taxi_fares_2022` will attempt to write to a table with the same name. 
+If the message has the following shape:
+```
+{
+    "name": "Yellow Cab, co",
+    "date": "2022-05-15",
+    "driver": "Joe Doe"
+}
+```
+the table must have the same columns, mapping the names and [data types](https://xata.io/docs/concepts/data-model), one-by-one.
+
+## Getting Started
+
+In order to connect, you need:
+* API Key: go to your [account settings](https://app.xata.io/settings) to generate a key.
+* Database URL: navigate to the configuration tab in your workspace and copy the `Workspace API base URL`.
+
+## CHANGELOG
+
+| Version | Date       | Pull Request                                                  | Subject                 |
+|:--------|:-----------|:--------------------------------------------------------------|:------------------------|
+| 0.1.1   | 2023-06-21 | [#27542](https://github.com/airbytehq/airbyte/pull/27542)     | Mark api_key as Airbyte Secret |
+| 0.1.0   | 2023-06-14 | [#24192](https://github.com/airbytehq/airbyte/pull/24192)     | New Destination Connector Xata |
diff --git a/docs/integrations/sources/aha.md b/docs/integrations/sources/aha.md
index 7535cf0ab311..b5ec1d410aef 100644
--- a/docs/integrations/sources/aha.md
+++ b/docs/integrations/sources/aha.md
@@ -37,6 +37,7 @@ Rate Limiting information is updated [here](https://www.aha.io/api#rate-limiting
 
 | Version | Date       | Pull Request                                             | Subject                                         |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------|
+| 0.3.1   | 2023-06-05 | [27002](https://github.com/airbytehq/airbyte/pull/27002) | Flag spec `api_key` field as `airbyte-secret` |
 | 0.3.0   | 2023-05-30 | [22642](https://github.com/airbytehq/airbyte/pull/22642) | Add `idea_comments`, `idea_endorsements`, and `idea_categories` streams |
 | 0.2.0   | 2023-05-26 | [26666](https://github.com/airbytehq/airbyte/pull/26666) | Fix integration test and schemas                |
 | 0.1.0   | 2022-11-02 | [18883](https://github.com/airbytehq/airbyte/pull/18893) | 🎉 New Source: Aha                              |
diff --git a/docs/integrations/sources/aircall.md b/docs/integrations/sources/aircall.md
index e3750fabfc7a..01685351dbb0 100644
--- a/docs/integrations/sources/aircall.md
+++ b/docs/integrations/sources/aircall.md
@@ -53,7 +53,7 @@ The Aircall source connector supports the following [sync modes](https://docs.ai
 - contacts
 - numbers
 - tags
-- user_availablity
+- user_availability
 - users
 - teams
 - webhooks
@@ -68,6 +68,7 @@ Aircall [API reference](https://api.aircall.io/v1) has v1 at present. The connec
 
 ## Changelog
 
-| Version | Date       | Pull Request                                           | Subject        |
-| :------ | :--------- | :----------------------------------------------------- | :------------- |
-| 0.1.0   | 2023-04-19 | [Init](https://github.com/airbytehq/airbyte/pull/)| Initial commit |
\ No newline at end of file
+| Version | Date       | Pull Request                                                                   | Subject        |
+|:--------|:-----------|:-------------------------------------------------------------------------------| :------------- |
+| 0.1.0   | 2023-04-19 | [Init](https://github.com/airbytehq/airbyte/pull/)                             | Initial commit |
+| 0.2.0   | 2023-06-20 | [Correcting availablity typo](https://github.com/airbytehq/airbyte/pull/27433) | Correcting availablity typo |
\ No newline at end of file
diff --git a/docs/integrations/sources/airtable.inapp.md b/docs/integrations/sources/airtable.inapp.md
new file mode 100644
index 000000000000..200b8f65ec25
--- /dev/null
+++ b/docs/integrations/sources/airtable.inapp.md
@@ -0,0 +1,22 @@
+:::info
+Currently, this source connector works with `Standard` subscription plan only. `Enterprise` level accounts are not supported yet.
+:::
+
+## Prerequisites
+
+* An active Airtable account
+
+## Setup guide
+1. Name your source.
+2. You can use OAuth or a Personal Access Token to authenticate your Airtable account. We recommend using OAuth for Airbyte Cloud.
+   - To authenticate using OAuth, select **OAuth2.0** from the Authentication dropdown click **Authenticate your Airtable account** to sign in with Airtable, select required workspaces you want to sync and authorize your account.
+   - To authenticate using a [Personal Access Token](https://airtable.com/developers/web/guides/personal-access-tokens), select **Personal Access Token** from the Authentication dropdown and enter the Access Token for your Airtable account. The following scopes are required:
+     - `data.records:read`
+     - `data.recordComments:read`
+     - `schema.bases:read`
+
+:::info
+When using OAuth, you may see a `400` or `401` error causing a failed sync. You can re-authenticate your Airtable connector to solve the issue temporarily. We are working on a permanent fix that you can follow [here](https://github.com/airbytehq/airbyte/issues/25278).
+:::
+
+3. Click **Set up source**.
diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md
index b931713ebc5f..1768d08fc432 100644
--- a/docs/integrations/sources/alloydb.md
+++ b/docs/integrations/sources/alloydb.md
@@ -20,7 +20,6 @@ If your goal is to maintain a snapshot of your table in the destination but the
 
 If your dataset is small and you just want a snapshot of your table in the destination, consider using [Full Refresh replication](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite) for your table instead of CDC.
 
-
 ### Step 1: (Optional) Create a dedicated read-only user
 
 We recommend creating a dedicated read-only user for better permission control and auditing. Alternatively, you can use an existing AlloyDB user in your database.
@@ -71,6 +70,7 @@ The workaround for partial table syncing is to create a view on the specific col
 ```
 CREATE VIEW  as SELECT  FROM ;
 ```
+
 ```
 GRANT SELECT ON TABLE  IN SCHEMA  to ;
 ```
@@ -123,16 +123,17 @@ When using an SSH tunnel, you are configuring Airbyte to connect to an intermedi
 To connect to a AlloyDB instance via an SSH tunnel:
 
 1. While [setting up](#setup-guide) the AlloyDB source connector, from the SSH tunnel dropdown, select:
-    - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel
-    - Password Authentication to use a password as your secret for establishing the SSH Tunnel
+   - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel
+   - Password Authentication to use a password as your secret for establishing the SSH Tunnel
 2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to.
 3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22.
 4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the AlloyDB username.
 5. For authentication:
-    - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [RSA Private Key](#generating-an-rsa-private-key​) that you are using to create the SSH connection.
-    - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the AlloyDB password.
+   - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [RSA Private Key](#generating-an-rsa-private-key​) that you are using to create the SSH connection.
+   - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the AlloyDB password.
 
 #### Generating an RSA Private Key​
+
 The connector expects an RSA key in PEM format. To generate this key, run:
 
 ```
@@ -154,8 +155,8 @@ Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-re
 - The records produced by `DELETE` statements only contain primary keys. All other data fields are unset.
 - Log-based replication only works for master instances of AlloyDB.
 - Using logical replication increases disk space used on the database server. The additional data is stored until it is consumed.
-    - Set frequent syncs for CDC to ensure that the data doesn't fill up your disk space.
-    - If you stop syncing a CDC-configured AlloyDB instance with Airbyte, delete the replication slot. Otherwise, it may fill up your disk space.
+  - Set frequent syncs for CDC to ensure that the data doesn't fill up your disk space.
+  - If you stop syncing a CDC-configured AlloyDB instance with Airbyte, delete the replication slot. Otherwise, it may fill up your disk space.
 
 ### Setting up CDC for AlloyDB
 
@@ -232,7 +233,7 @@ The AlloyDB source connector supports the following [sync modes](https://docs.ai
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported cursors
 
@@ -253,105 +254,110 @@ The AlloyDB source connector supports the following [sync modes](https://docs.ai
 - `BINARY/BLOB`
 
 ## Data type mapping
+
 The AlloyDb is a fully managed PostgreSQL-compatible database service.
 
 According to Postgres [documentation](https://www.postgresql.org/docs/14/datatype.html), Postgres data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test!
 
-| Postgres Type                         | Resulting Type | Notes                                                                                                                                                                                           |
-|:--------------------------------------|:---------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `bigint`                              | number         |                                                                                                                                                                                                 |
-| `bigserial`, `serial8`                | number         |                                                                                                                                                                                                 |
-| `bit`                                 | string         | Fixed-length bit string (e.g. "0100").                                                                                                                                                          |
-| `bit varying`, `varbit`               | string         | Variable-length bit string (e.g. "0100").                                                                                                                                                       |
-| `boolean`, `bool`                     | boolean        |                                                                                                                                                                                                 |
-| `box`                                 | string         |                                                                                                                                                                                                 |
-| `bytea`                               | string         | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a").                                                                                                      |
-| `character`, `char`                   | string         |                                                                                                                                                                                                 |
-| `character varying`, `varchar`        | string         |                                                                                                                                                                                                 |
-| `cidr`                                | string         |                                                                                                                                                                                                 |
-| `circle`                              | string         |                                                                                                                                                                                                 |
-| `date`                                | string         | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590)                                            |
-| `double precision`, `float`, `float8` | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).                                                 |
-| `hstore`                              | string         |                                                                                                                                                                                                 |
-| `inet`                                | string         |                                                                                                                                                                                                 |
-| `integer`, `int`, `int4`              | number         |                                                                                                                                                                                                 |
-| `interval`                            | string         |                                                                                                                                                                                                 |
-| `json`                                | string         |                                                                                                                                                                                                 |
-| `jsonb`                               | string         |                                                                                                                                                                                                 |
-| `line`                                | string         |                                                                                                                                                                                                 |
-| `lseg`                                | string         |                                                                                                                                                                                                 |
-| `macaddr`                             | string         |                                                                                                                                                                                                 |
-| `macaddr8`                            | string         |                                                                                                                                                                                                 |
-| `money`                               | number         |                                                                                                                                                                                                 |
-| `numeric`, `decimal`                  | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).                                                 |
-| `path`                                | string         |                                                                                                                                                                                                 |
-| `pg_lsn`                              | string         |                                                                                                                                                                                                 |
-| `point`                               | string         |                                                                                                                                                                                                 |
-| `polygon`                             | string         |                                                                                                                                                                                                 |
-| `real`, `float4`                      | number         |                                                                                                                                                                                                 |
-| `smallint`, `int2`                    | number         |                                                                                                                                                                                                 |
-| `smallserial`, `serial2`              | number         |                                                                                                                                                                                                 |
-| `serial`, `serial4`                   | number         |                                                                                                                                                                                                 |
-| `text`                                | string         |                                                                                                                                                                                                 |
-| `time`                                | string         | Parsed as a time string without a time-zone in the ISO-8601 calendar system.                                                                                                                    |
-| `timetz`                              | string         | Parsed as a time string with time-zone in the ISO-8601 calendar system.                                                                                                                         |
-| `timestamp`                           | string         | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system.                                                                                                               |
-| `timestamptz`                         | string         | Parsed as a date-time string with time-zone in the ISO-8601 calendar system.                                                                                                                    |
-| `tsquery`                             | string         |                                                                                                                                                                                                 |
-| `tsvector`                            | string         |                                                                                                                                                                                                 |
-| `uuid`                                | string         |                                                                                                                                                                                                 |
-| `xml`                                 | string         |                                                                                                                                                                                                 |
-| `enum`                                | string         |                                                                                                                                                                                                 |
-| `tsrange`                             | string         |                                                                                                                                                                                                 |
-| `array`                               | array          | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]".                                                                                                                                               |
-| composite type                        | string         |                                                                                                                                                                                                 |
+| Postgres Type                         | Resulting Type | Notes                                                                                                                                                |
+| :------------------------------------ | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `bigint`                              | number         |                                                                                                                                                      |
+| `bigserial`, `serial8`                | number         |                                                                                                                                                      |
+| `bit`                                 | string         | Fixed-length bit string (e.g. "0100").                                                                                                               |
+| `bit varying`, `varbit`               | string         | Variable-length bit string (e.g. "0100").                                                                                                            |
+| `boolean`, `bool`                     | boolean        |                                                                                                                                                      |
+| `box`                                 | string         |                                                                                                                                                      |
+| `bytea`                               | string         | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a").                                                           |
+| `character`, `char`                   | string         |                                                                                                                                                      |
+| `character varying`, `varchar`        | string         |                                                                                                                                                      |
+| `cidr`                                | string         |                                                                                                                                                      |
+| `circle`                              | string         |                                                                                                                                                      |
+| `date`                                | string         | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590) |
+| `double precision`, `float`, `float8` | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).      |
+| `hstore`                              | string         |                                                                                                                                                      |
+| `inet`                                | string         |                                                                                                                                                      |
+| `integer`, `int`, `int4`              | number         |                                                                                                                                                      |
+| `interval`                            | string         |                                                                                                                                                      |
+| `json`                                | string         |                                                                                                                                                      |
+| `jsonb`                               | string         |                                                                                                                                                      |
+| `line`                                | string         |                                                                                                                                                      |
+| `lseg`                                | string         |                                                                                                                                                      |
+| `macaddr`                             | string         |                                                                                                                                                      |
+| `macaddr8`                            | string         |                                                                                                                                                      |
+| `money`                               | number         |                                                                                                                                                      |
+| `numeric`, `decimal`                  | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).      |
+| `path`                                | string         |                                                                                                                                                      |
+| `pg_lsn`                              | string         |                                                                                                                                                      |
+| `point`                               | string         |                                                                                                                                                      |
+| `polygon`                             | string         |                                                                                                                                                      |
+| `real`, `float4`                      | number         |                                                                                                                                                      |
+| `smallint`, `int2`                    | number         |                                                                                                                                                      |
+| `smallserial`, `serial2`              | number         |                                                                                                                                                      |
+| `serial`, `serial4`                   | number         |                                                                                                                                                      |
+| `text`                                | string         |                                                                                                                                                      |
+| `time`                                | string         | Parsed as a time string without a time-zone in the ISO-8601 calendar system.                                                                         |
+| `timetz`                              | string         | Parsed as a time string with time-zone in the ISO-8601 calendar system.                                                                              |
+| `timestamp`                           | string         | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system.                                                                    |
+| `timestamptz`                         | string         | Parsed as a date-time string with time-zone in the ISO-8601 calendar system.                                                                         |
+| `tsquery`                             | string         |                                                                                                                                                      |
+| `tsvector`                            | string         |                                                                                                                                                      |
+| `uuid`                                | string         |                                                                                                                                                      |
+| `xml`                                 | string         |                                                                                                                                                      |
+| `enum`                                | string         |                                                                                                                                                      |
+| `tsrange`                             | string         |                                                                                                                                                      |
+| `array`                               | array          | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]".                                                                                                    |
+| composite type                        | string         |                                                                                                                                                      |
 
 ## Limitations
 
 - The AlloyDB source connector currently does not handle schemas larger than 4MB.
 - The AlloyDB source connector does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details.
 - The following two schema evolution actions are currently supported:
-    - Adding/removing tables without resetting the entire connection at the destination
-      Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes.
-    - Resetting a single table within the connection without resetting the rest of the destination tables in that connection
+  - Adding/removing tables without resetting the entire connection at the destination
+    Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes.
+  - Resetting a single table within the connection without resetting the rest of the destination tables in that connection
 - Changing a column data type or removing a column might break connections.
 
-
 ## Changelog
 
-| Version | Date       | Pull Request                                               | Subject                                                                                                                                   |
-|:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------|
-| 2.0.28  | 2023-04-26 | [25401](https://github.com/airbytehq/airbyte/pull/25401)   | CDC : Upgrade Debezium to version 2.2.0                                                                                                   |
-| 2.0.23  | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582)   | CDC : Enable frequent state emission during incremental syncs + refactor for performance improvement                                      |
-| 2.0.22  | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220)   | Logging changes : Log additional metadata & clean up noisy logs                                                                           |
-| 2.0.21  | 2023-04-12 | [25131](https://github.com/airbytehq/airbyte/pull/25131)   | Make Client Certificate and Client Key always show                                                                                        |
-| 2.0.19  | 2023-04-11 | [24656](https://github.com/airbytehq/airbyte/pull/24656)   | CDC minor refactor                                                                                                                        |
-| 2.0.17  | 2023-04-05 | [24622](https://github.com/airbytehq/airbyte/pull/24622)   | Allow streams not in CDC publication to be synced in Full-refresh mode                                                                    |
-| 2.0.15  | 2023-04-04 | [24833](https://github.com/airbytehq/airbyte/pull/24833)   | Disallow the "disable" SSL Modes; fix Debezium retry policy configuration                                                                 |
-| 2.0.13  | 2023-03-28 | [24166](https://github.com/airbytehq/airbyte/pull/24166)   | Fix InterruptedException bug during Debezium shutdown                                                                                     |
-| 2.0.11  | 2023-03-27 | [24529](https://github.com/airbytehq/airbyte/pull/24373)   | Preparing the connector for CDC checkpointing                                                                                             |
-| 2.0.10  | 2023-03-24 | [24529](https://github.com/airbytehq/airbyte/pull/24529)   | Set SSL Mode to required on strict-encrypt variant                                                                                        |
-| 2.0.9   | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760)   | Removed redundant date-time datatypes formatting                                                                                          |
-| 2.0.6   | 2023-03-21 | [24271](https://github.com/airbytehq/airbyte/pull/24271)   | Fix NPE in CDC mode                                                                                                                       |
-| 2.0.3   | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275)   | Fix error with CDC checkpointing                                                                                                          |
-| 2.0.2   | 2023-03-13 | [23112](https://github.com/airbytehq/airbyte/pull/21727)   | Add state checkpointing for CDC sync.                                                                                                     |
-| 2.0.1   | 2023-03-08 | [23596](https://github.com/airbytehq/airbyte/pull/23596)   | For network isolation, source connector accepts a list of hosts it is allowed to connect                                                  |
-| 2.0.0   | 2023-03-06 | [23112](https://github.com/airbytehq/airbyte/pull/23112)   | Upgrade Debezium version to 2.1.2                                                                                                         |
-| 1.0.51  | 2023-03-02 | [23642](https://github.com/airbytehq/airbyte/pull/23642)   | Revert : Support JSONB datatype for Standard sync mode                                                                                    |
-| 1.0.49  | 2023-02-27 | [21695](https://github.com/airbytehq/airbyte/pull/21695)   | Support JSONB datatype for Standard sync mode                                                                                             |
-| 1.0.48  | 2023-02-24 | [23383](https://github.com/airbytehq/airbyte/pull/23383)   | Fixed bug with non readable double-quoted values within a database name or column name                                                    |
-| 1.0.47  | 2023-02-22 | [22221](https://github.com/airbytehq/airbyte/pull/23138)   | Fix previous versions which doesn't verify privileges correctly, preventing CDC syncs to run.                                             |
-| 1.0.46  | 2023-02-21 | [23105](https://github.com/airbytehq/airbyte/pull/23105)   | Include log levels and location information (class, method and line number) with source connector logs published to Airbyte Platform.     |
-| 1.0.45  | 2023-02-09 | [22221](https://github.com/airbytehq/airbyte/pull/22371)   | Ensures that user has required privileges for CDC syncs.                                                                                  |
-|         | 2023-02-15 | [23028](https://github.com/airbytehq/airbyte/pull/23028)   |                                                                                                                                           |
-| 1.0.44  | 2023-02-06 | [22221](https://github.com/airbytehq/airbyte/pull/22221)   | Exclude new set of system tables when using `pg_stat_statements` extension.                                                               |
-| 1.0.43  | 2023-02-06 | [21634](https://github.com/airbytehq/airbyte/pull/21634)   | Improve Standard sync performance by caching objects.                                                                                     |
-| 1.0.36  | 2023-01-24 | [21825](https://github.com/airbytehq/airbyte/pull/21825)   | Put back the original change that will cause an incremental sync to error if table contains a NULL value in cursor column.                |
-| 1.0.35  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)   | Consolidate date/time values mapping for JDBC sources                                                                                     |
-| 1.0.34  | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378)   | Improve descriptions                                                                                                                      |
-| 1.0.17  | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538)   | Encode database name                                                                                                                      |
-| 1.0.16  | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256)   | Disable allow and prefer ssl modes in CDC mode                                                                                            |
-|         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238)   | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
-| 1.0.15  | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782)   | Align with Postgres source v.1.0.15                                                                                                       |
-| 1.0.0   | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776)   | Align with strict-encrypt version                                                                                                         |
-| 0.1.0   | 2022-09-05 | [16323](https://github.com/airbytehq/airbyte/pull/16323)   | Initial commit. Based on source-postgres v.1.0.7                                                                                          |
+| Version | Date       | Pull Request                                             | Subject                                                                                                                                   |
+|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.1.5   | 2023-08-22 | [29534](https://github.com/airbytehq/airbyte/pull/29534) | Support "options" JDBC URL parameter                                                                                                      |
+| 3.1.3   | 2023-08-03 | [28708](https://github.com/airbytehq/airbyte/pull/28708) | Enable checkpointing snapshots in CDC connections                                                                                         |
+| 3.1.2   | 2023-08-01 | [28954](https://github.com/airbytehq/airbyte/pull/28954) | Fix an issue that prevented use of tables with names containing uppercase letters                                                         |
+| 3.1.1   | 2023-07-31 | [28892](https://github.com/airbytehq/airbyte/pull/28892) | Fix an issue that prevented use of cursor columns with names containing uppercase letters                                                 |
+| 3.1.0   | 2023-07-25 | [28339](https://github.com/airbytehq/airbyte/pull/28339) | Checkpointing initial load for incremental syncs: enabled for xmin and cursor based only.                                                 |
+| 2.0.28  | 2023-04-26 | [25401](https://github.com/airbytehq/airbyte/pull/25401) | CDC : Upgrade Debezium to version 2.2.0                                                                                                   |
+| 2.0.23  | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582) | CDC : Enable frequent state emission during incremental syncs + refactor for performance improvement                                      |
+| 2.0.22  | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220) | Logging changes : Log additional metadata & clean up noisy logs                                                                           |
+| 2.0.21  | 2023-04-12 | [25131](https://github.com/airbytehq/airbyte/pull/25131) | Make Client Certificate and Client Key always show                                                                                        |
+| 2.0.19  | 2023-04-11 | [24656](https://github.com/airbytehq/airbyte/pull/24656) | CDC minor refactor                                                                                                                        |
+| 2.0.17  | 2023-04-05 | [24622](https://github.com/airbytehq/airbyte/pull/24622) | Allow streams not in CDC publication to be synced in Full-refresh mode                                                                    |
+| 2.0.15  | 2023-04-04 | [24833](https://github.com/airbytehq/airbyte/pull/24833) | Disallow the "disable" SSL Modes; fix Debezium retry policy configuration                                                                 |
+| 2.0.13  | 2023-03-28 | [24166](https://github.com/airbytehq/airbyte/pull/24166) | Fix InterruptedException bug during Debezium shutdown                                                                                     |
+| 2.0.11  | 2023-03-27 | [24529](https://github.com/airbytehq/airbyte/pull/24373) | Preparing the connector for CDC checkpointing                                                                                             |
+| 2.0.10  | 2023-03-24 | [24529](https://github.com/airbytehq/airbyte/pull/24529) | Set SSL Mode to required on strict-encrypt variant                                                                                        |
+| 2.0.9   | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting                                                                                          |
+| 2.0.6   | 2023-03-21 | [24271](https://github.com/airbytehq/airbyte/pull/24271) | Fix NPE in CDC mode                                                                                                                       |
+| 2.0.3   | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275) | Fix error with CDC checkpointing                                                                                                          |
+| 2.0.2   | 2023-03-13 | [23112](https://github.com/airbytehq/airbyte/pull/21727) | Add state checkpointing for CDC sync.                                                                                                     |
+| 2.0.1   | 2023-03-08 | [23596](https://github.com/airbytehq/airbyte/pull/23596) | For network isolation, source connector accepts a list of hosts it is allowed to connect                                                  |
+| 2.0.0   | 2023-03-06 | [23112](https://github.com/airbytehq/airbyte/pull/23112) | Upgrade Debezium version to 2.1.2                                                                                                         |
+| 1.0.51  | 2023-03-02 | [23642](https://github.com/airbytehq/airbyte/pull/23642) | Revert : Support JSONB datatype for Standard sync mode                                                                                    |
+| 1.0.49  | 2023-02-27 | [21695](https://github.com/airbytehq/airbyte/pull/21695) | Support JSONB datatype for Standard sync mode                                                                                             |
+| 1.0.48  | 2023-02-24 | [23383](https://github.com/airbytehq/airbyte/pull/23383) | Fixed bug with non readable double-quoted values within a database name or column name                                                    |
+| 1.0.47  | 2023-02-22 | [22221](https://github.com/airbytehq/airbyte/pull/23138) | Fix previous versions which doesn't verify privileges correctly, preventing CDC syncs to run.                                             |
+| 1.0.46  | 2023-02-21 | [23105](https://github.com/airbytehq/airbyte/pull/23105) | Include log levels and location information (class, method and line number) with source connector logs published to Airbyte Platform.     |
+| 1.0.45  | 2023-02-09 | [22221](https://github.com/airbytehq/airbyte/pull/22371) | Ensures that user has required privileges for CDC syncs.                                                                                  |
+|         | 2023-02-15 | [23028](https://github.com/airbytehq/airbyte/pull/23028) |                                                                                                                                           |
+| 1.0.44  | 2023-02-06 | [22221](https://github.com/airbytehq/airbyte/pull/22221) | Exclude new set of system tables when using `pg_stat_statements` extension.                                                               |
+| 1.0.43  | 2023-02-06 | [21634](https://github.com/airbytehq/airbyte/pull/21634) | Improve Standard sync performance by caching objects.                                                                                     |
+| 1.0.36  | 2023-01-24 | [21825](https://github.com/airbytehq/airbyte/pull/21825) | Put back the original change that will cause an incremental sync to error if table contains a NULL value in cursor column.                |
+| 1.0.35  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources                                                                                     |
+| 1.0.34  | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions                                                                                                                      |
+| 1.0.17  | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name                                                                                                                      |
+| 1.0.16  | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode                                                                                            |
+|         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
+| 1.0.15  | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Align with Postgres source v.1.0.15                                                                                                       |
+| 1.0.0   | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776) | Align with strict-encrypt version                                                                                                         |
+| 0.1.0   | 2022-09-05 | [16323](https://github.com/airbytehq/airbyte/pull/16323) | Initial commit. Based on source-postgres v.1.0.7                                                                                          |
diff --git a/docs/integrations/sources/amazon-ads-migrations.md b/docs/integrations/sources/amazon-ads-migrations.md
new file mode 100644
index 000000000000..6b95c39af7b9
--- /dev/null
+++ b/docs/integrations/sources/amazon-ads-migrations.md
@@ -0,0 +1,6 @@
+# Amazon Ads Migration Guide
+
+## Upgrading to 3.0.0
+
+A major update of attribution report stream schemas.
+For a smooth migration, a data reset and a schema refresh are needed.
\ No newline at end of file
diff --git a/docs/integrations/sources/amazon-ads.inapp.md b/docs/integrations/sources/amazon-ads.inapp.md
new file mode 100644
index 000000000000..d501c7be4642
--- /dev/null
+++ b/docs/integrations/sources/amazon-ads.inapp.md
@@ -0,0 +1,20 @@
+## Prerequisites
+
+- An [Amazon user](https://www.amazon.com) with access to an [Amazon Ads account](https://advertising.amazon.com)
+
+## Setup Guide
+
+1. Click `Authenticate your Amazon Ads account`. Log in and authorize access to the Amazon account.
+2. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [Amazon Ads documentation](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details.
+3. (Optional) **Start Date** can be used to generate reports starting from the specified start date in the format YYYY-MM-DD. The date should not be more than 60 days in the past. If not specified, today's date is used. The date is treated in the timezone of the processed profile.
+4. (Optional) **Profile ID(s)** you want to fetch data for. A profile is an advertiser's account in a specific marketplace. See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. If not specified, data from all Profiles will be synced.
+5. (Optional) **State Filter** Filter for Display, Product, and Brand Campaign streams with a state of enabled, paused, or archived. If not specified, all streams regardless of state will be synced.
+6. (Optional) **Look Back Window** The amount of days to go back in time to get the updated data from Amazon Ads. After the first sync, data from this date will be synced. 
+7. (Optional) **Report Record Types** Optional configuration which accepts an array of string of record types. Leave blank for default behaviour to pull all report types. Use this config option only if you want to pull specific report type(s). See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/reporting/v2/report-types) for more details
+9. Click `Set up source`.
+
+### Report Timezones
+
+All the reports are generated relative to the target profile' timezone.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Amazon Ads](https://docs.airbyte.com/integrations/sources/amazon-ads).
diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md
index 17db91e9d935..0955fe1e7906 100644
--- a/docs/integrations/sources/amazon-ads.md
+++ b/docs/integrations/sources/amazon-ads.md
@@ -52,6 +52,7 @@ The Amazon Ads source connector supports the following [sync modes](https://docs
 This source is capable of syncing the following streams:
 
 * [Profiles](https://advertising.amazon.com/API/docs/en-us/reference/2/profiles#/Profiles)
+* [Portfolios](https://advertising.amazon.com/API/docs/en-us/reference/2/portfolios#/Portfolios%20extended)
 * [Sponsored Brands Campaigns](https://advertising.amazon.com/API/docs/en-us/sponsored-brands/3-0/openapi#/Campaigns)
 * [Sponsored Brands Ad groups](https://advertising.amazon.com/API/docs/en-us/sponsored-brands/3-0/openapi#/Ad%20groups)
 * [Sponsored Brands Keywords](https://advertising.amazon.com/API/docs/en-us/sponsored-brands/3-0/openapi#/Keywords)
@@ -59,10 +60,14 @@ This source is capable of syncing the following streams:
 * [Sponsored Display Ad groups](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Ad%20groups)
 * [Sponsored Display Product Ads](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Product%20ads)
 * [Sponsored Display Targetings](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Targeting)
+* [Sponsored Display Budget Rules](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi/prod#/BudgetRules/GetSDBudgetRulesForAdvertiser)
 * [Sponsored Products Campaigns](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Campaigns)
 * [Sponsored Products Ad groups](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Ad%20groups)
+* [Sponsored Products Ad Group Bid Recommendations](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Bid%20recommendations/getAdGroupBidRecommendations)
+* [Sponsored Products Ad Group Suggested Keywords](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Suggested%20keywords)
 * [Sponsored Products Keywords](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Keywords)
 * [Sponsored Products Negative keywords](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Negative%20keywords)
+* [Sponsored Products Campaign Negative keywords](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Negative%20keywords)
 * [Sponsored Products Ads](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Product%20ads)
 * [Sponsored Products Targetings](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Product%20targeting)
 * [Brands Reports](https://advertising.amazon.com/API/docs/en-us/reference/sponsored-brands/2/reports)
@@ -75,6 +80,10 @@ This source is capable of syncing the following streams:
 
 All the reports are generated relative to the target profile' timezone.
 
+Campaign reports may sometimes have no data or not presenting in records. This can occur when there are no clicks or views associated with the campaigns on the requested day - [details](https://advertising.amazon.com/API/docs/en-us/guides/reporting/v2/faq#why-is-my-report-empty).
+
+Report data synchronization only cover the last 60 days - [details](https://advertising.amazon.com/API/docs/en-us/reference/1/reports#parameters).
+
 ## Performance considerations
 
 Information about expected report generation waiting time you may find [here](https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes).
@@ -94,6 +103,13 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 3.1.0   | 2023-08-08 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add `T00030` tactic support for `sponsored_display_report_stream`                                               |
+| 3.0.0   | 2023-07-24 | [27868](https://github.com/airbytehq/airbyte/pull/27868) | Fix attribution report stream schemas                                                                           |
+| 2.3.1   | 2023-07-11 | [28155](https://github.com/airbytehq/airbyte/pull/28155) | Bugfix: validation error when record values are missing                                                         |
+| 2.3.0   | 2023-07-06 | [28002](https://github.com/airbytehq/airbyte/pull/28002) | Add sponsored_product_ad_group_suggested_keywords, sponsored_product_ad_group_bid_recommendations streams       |
+| 2.2.0   | 2023-07-05 | [27607](https://github.com/airbytehq/airbyte/pull/27607) | Add stream for sponsored brands v3 purchased product reports                                                    |
+| 2.1.0   | 2023-06-19 | [25412](https://github.com/airbytehq/airbyte/pull/25412) | Add sponsored_product_campaign_negative_keywords, sponsored_display_budget_rules streams                        |
+| 2.0.0   | 2023-05-31 | [25874](https://github.com/airbytehq/airbyte/pull/25874) | Type `portfolioId` as integer                                                                                   |
 | 1.1.0   | 2023-04-22 | [25412](https://github.com/airbytehq/airbyte/pull/25412) | Add missing reporting metrics                                                                                   |
 | 1.0.6   | 2023-05-09 | [25913](https://github.com/airbytehq/airbyte/pull/25913) | Small schema fixes                                                                                              |
 | 1.0.5   | 2023-05-08 | [25885](https://github.com/airbytehq/airbyte/pull/25885) | Improve error handling for attribution_report(s) streams                                                        |
@@ -115,7 +131,7 @@ Information about expected report generation waiting time you may find [here](ht
 | 0.1.19  | 2022-08-31 | [16191](https://github.com/airbytehq/airbyte/pull/16191) | Improved connector's input configuration validation                                                             |
 | 0.1.18  | 2022-08-25 | [15951](https://github.com/airbytehq/airbyte/pull/15951) | Skip API error "Tactic T00020 is not supported for report API in marketplace A1C3SOZRARQ6R3."                   |
 | 0.1.17  | 2022-08-24 | [15921](https://github.com/airbytehq/airbyte/pull/15921) | Skip API error "Report date is too far in the past."                                                            |
-| 0.1.16  | 2022-08-23 | [15822](https://github.com/airbytehq/airbyte/pull/15822) | Set default value for 'region' if needed                                                                        |
+| 0.1.16  | 2022-08-23 | [15822](https://github.com/airbytehq/airbyte/pull/15822) | Set default value for `region` if needed                                                                        |
 | 0.1.15  | 2022-08-20 | [15816](https://github.com/airbytehq/airbyte/pull/15816) | Update STATE of incremental sync if no records                                                                  |
 | 0.1.14  | 2022-08-15 | [15637](https://github.com/airbytehq/airbyte/pull/15637) | Generate slices by lazy evaluation                                                                              |
 | 0.1.12  | 2022-08-09 | [15469](https://github.com/airbytehq/airbyte/pull/15469) | Define primary_key for all report streams                                                                       |
@@ -125,7 +141,7 @@ Information about expected report generation waiting time you may find [here](ht
 | 0.1.8   | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy                                                                                 |
 | 0.1.7   | 2022-04-27 | [11730](https://github.com/airbytehq/airbyte/pull/11730) | Update fields in source-connectors specifications                                                               |
 | 0.1.6   | 2022-04-20 | [11659](https://github.com/airbytehq/airbyte/pull/11659) | Add adId to products report                                                                                     |
-| 0.1.5   | 2022-04-08 | [11430](https://github.com/airbytehq/airbyte/pull/11430) | Added support OAuth2.0                                                                                          |
+| 0.1.5   | 2022-04-08 | [11430](https://github.com/airbytehq/airbyte/pull/11430) | Add support OAuth2.0                                                                                            |
 | 0.1.4   | 2022-02-21 | [10513](https://github.com/airbytehq/airbyte/pull/10513) | Increasing REPORT_WAIT_TIMEOUT for supporting report generation which takes longer time                         |
 | 0.1.3   | 2021-12-28 | [8388](https://github.com/airbytehq/airbyte/pull/8388)   | Add retry if recoverable error  occured for reporting stream processing                                         |
 | 0.1.2   | 2021-10-01 | [6367](https://github.com/airbytehq/airbyte/pull/6461)   | Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback |
diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md
index 637c30032d57..bdef35431f89 100644
--- a/docs/integrations/sources/amazon-seller-partner.md
+++ b/docs/integrations/sources/amazon-seller-partner.md
@@ -4,22 +4,22 @@ This page guides you through the process of setting up the Amazon Seller Partner
 
 ## Prerequisites
 
-- app_id
-- lwa_app_id
-- lwa_client_secret
-- refresh_token
-- aws_access_key
-- aws_secret_key
-- role_arn
-- aws_environment
-- region
-- replication_start_date
+- AWS Environment
+- AWS Region
+- AWS Access Key
+- AWS Secret Key
+- Role ARN
+- LWA Client ID (LWA App ID)**
+- LWA Client Secret**
+- Refresh token**
+- Replication Start Date
+
+**not required for Airbyte Cloud
 
 ## Step 1: Set up Amazon Seller Partner
 
 1. [Register](https://developer-docs.amazon.com/sp-api/docs/registering-your-application) Amazon Seller Partner application.
     - The application must be published as Amazon does not allow external parties such as Airbyte to access draft applications.
-    - If using the connector on Airbyte Cloud, the Redirect URL must be set to `https://cloud.airbyte.com/auth_flow`
 2. [Create](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) IAM user.
 
 ## Step 2: Set up the source connector in Airbyte
@@ -29,8 +29,8 @@ This page guides you through the process of setting up the Amazon Seller Partner
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 
 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. 
 3. On the source setup page, select **Amazon Seller Partner** from the Source type dropdown and enter a name for this connector.
-4. Fill in your `app_id` and click `Authenticate your account`.
-5. Log in and Authorize to the Amazon Seller Partner account.
+4. Click `Authenticate your account`.
+5. Log in and Authorize to your Amazon Seller Partner account.
 6. Paste all other data to required fields using your IAM user.
 7. Click `Set up source`.
 
@@ -62,6 +62,7 @@ This source is capable of syncing the following tables and their data:
 - [FBA Replacements Reports](https://sellercentral.amazon.com/help/hub/reference/200453300)
 - [FBA Storage Fees Report](https://sellercentral.amazon.com/help/hub/reference/G202086720)
 - [Restock Inventory Reports](https://sellercentral.amazon.com/help/hub/reference/202105670)
+- [Flat File Actionable Order Data Shipping](https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_flat_file_actionable_order_data_shipping)
 - [Flat File Open Listings Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
 - [Flat File Orders Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
 - [Flat File Orders Reports By Last Update](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\)
@@ -70,6 +71,7 @@ This source is capable of syncing the following tables and their data:
 - [Vendor Direct Fulfillment Shipping](https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference)
 - [Vendor Inventory Health Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
 - [Orders](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) \(incremental\)
+- [Orders Items](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems) \(incremental\)
 - [Seller Feedback Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\)
 - [Brand Analytics Alternate Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
 - [Brand Analytics Item Comparison Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
@@ -102,6 +104,7 @@ This source is capable of syncing the following tables and their data:
 - [FBA Manage Inventory Health Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
 - [Inventory Ledger Report - Summary View](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
 - [FBA Reimbursements Report](https://sellercentral.amazon.com/help/hub/reference/G200732720)
+- [Order Data Shipping Report](https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping)
 
 ## Report options
 
@@ -124,44 +127,49 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic
 
 ## Changelog
 
-| Version  | Date       | Pull Request                                               | Subject                                                                                                                                                                             |
-|:---------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `1.2.0`  | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration                      |
-| `1.1.0`  | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream                                    |
-| `1.0.1`  | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace                                                                                                                                                             |
-| `1.0.0`  | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts |
-| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor                                                                                                                        |
-| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional                                                                                                                        |
-| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams                                                                                                                                                |
-| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data                                                                                                                                            |
-| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT                                                                                                                         |
-| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types                                                                                                                                                    |
-| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types                                                                                                                   |
-| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report                                                                                                                                                        |
-| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report                                                                                                                                                         |
-| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports                                                                                                                         |
-| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream                                                                                                                                        |
-| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams                                                                                                                                                               |
-| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config                                                                                                                                                  |
-| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support                                                                                                                                                                |
-| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report                                                                                                                                                 |
-| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams                                                                                                                                 |
-| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role arn                                                                                                                                           |
-| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789)   | Add stream FbaReplacementsReports                                                                                                                                                   |
-| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621)   | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report                                                                                                                     |
-| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581)   | Change createdSince parameter to dataStartTime                                                                                                                                      |
-| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312)   | Add all remaining brand analytics report streams                                                                                                                                    |
-| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115)   | Fix reading only 100 orders                                                                                                                                                         |
-| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236)   | Fix NoAuth deprecation warning                                                                                                                                                      |
-| `0.2.9`  | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212)   | Normalize GET_SELLER_FEEDBACK_DATA header field names                                                                                                                               |
-| `0.2.8`  | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810)   | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format                                                                                                                               |
-| `0.2.7`  | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002)   | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter                                                                                                                          |
-| `0.2.6`  | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179)   | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report                                                                                                                                  |
-| `0.2.5`  | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425)   | Update title, description fields in spec                                                                                                                                            |
-| `0.2.4`  | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021)   | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability                                                                                                              |
-| `0.2.3`  | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828)   | Remove datetime format from all streams                                                                                                                                             |
-| `0.2.2`  | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752)   | Change `check_connection` function to use stream Orders                                                                                                                             |
-| `0.2.1`  | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248)   | `Added extra stream support. Updated reports streams logics`                                                                                                                        |
-| `0.2.0`  | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863)   | `Rebuild source with airbyte-cdk`                                                                                                                                                   |
-| `0.1.3`  | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288)   | `Bugfix failing connection check`                                                                                                                                                   |
-| `0.1.2`  | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108)   | `Fixed: Sync fails with timeout when create report is CANCELLED`                                                                                                                    |
+| Version  | Date       | Pull Request                                                  | Subject                                                                                                                                                                             |
+|:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `1.5.1`  | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255)    | role_arn is optional on UI but not really on the backend blocking connector set up using oauth                                                                                      |
+| `1.5.0`  | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054)    | Add new stream `OrderItems`                                                                                                                                                         |
+| `1.4.1`  | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050)    | Fix - non vendor accounts connector create/check issue                                                                                                                              |
+| `1.4.0`  | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110)    | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams                                                                                     |
+| `1.3.0`  | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110)    | Removed `app_id` from `InputConfiguration`, refactored `spec`                                                                                                                       |
+| `1.2.0`  | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503)    | Enabled stream attribute customization from Source configuration                                                                                                                    |
+| `1.1.0`  | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605)    | Add FBA Reimbursement Report stream                                                                                                                                                 |
+| `1.0.1`  | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098)    | Add Belgium Marketplace                                                                                                                                                             |
+| `1.0.0`  | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980)    | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts |
+| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606)    | Implement reportOptions for all missing reports and refactor                                                                                                                        |
+| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300)    | Make AWS Access Key, AWS Secret Access and Role ARN optional                                                                                                                        |
+| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430)    | Implement slicing for report streams                                                                                                                                                |
+| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896)    | Validate connections without orders data                                                                                                                                            |
+| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581)    | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT                                                                                                                         |
+| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283)    | Added multiple (22) report types                                                                                                                                                    |
+| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629)    | Report API version to 2021-06-30, added multiple (5) report types                                                                                                                   |
+| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063)    | Add Restock Inventory Report                                                                                                                                                        |
+| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625)    | Add FBA Storage Fees Report                                                                                                                                                         |
+| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604)    | Add new streams: Fullfiments returns and Settlement reports                                                                                                                         |
+| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633)    | Fix - handle start date for financial stream                                                                                                                                        |
+| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364)    | Add financial streams                                                                                                                                                               |
+| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059)    | Add replication end date to config                                                                                                                                                  |
+| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119)    | Add OAuth2.0 support                                                                                                                                                                |
+| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663)    | Add GET_XML_BROWSE_TREE_DATA report                                                                                                                                                 |
+| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946)    | Add throttling exception managing in Orders streams                                                                                                                                 |
+| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523)    | allow to use IAM user arn or IAM role                                                                                                                                               |
+| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789)      | Add stream FbaReplacementsReports                                                                                                                                                   |
+| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621)      | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report                                                                                                                     |
+| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581)      | Change createdSince parameter to dataStartTime                                                                                                                                      |
+| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312)      | Add all remaining brand analytics report streams                                                                                                                                    |
+| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115)      | Fix reading only 100 orders                                                                                                                                                         |
+| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236)      | Fix NoAuth deprecation warning                                                                                                                                                      |
+| `0.2.9`  | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212)      | Normalize GET_SELLER_FEEDBACK_DATA header field names                                                                                                                               |
+| `0.2.8`  | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810)      | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format                                                                                                                               |
+| `0.2.7`  | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002)      | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter                                                                                                                          |
+| `0.2.6`  | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179)      | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report                                                                                                                                  |
+| `0.2.5`  | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425)      | Update title, description fields in spec                                                                                                                                            |
+| `0.2.4`  | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021)      | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability                                                                                                              |
+| `0.2.3`  | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828)      | Remove datetime format from all streams                                                                                                                                             |
+| `0.2.2`  | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752)      | Change `check_connection` function to use stream Orders                                                                                                                             |
+| `0.2.1`  | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248)      | Added `extra stream` support. Updated `reports streams` logics                                                                                                                      |
+| `0.2.0`  | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863)      | Rebuild source with `airbyte-cdk`                                                                                                                                                   |
+| `0.1.3`  | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288)      | Bugfix failing `connection check`                                                                                                                                                   |
+| `0.1.2`  | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108)      | Fixed: Sync fails with timeout when create report is CANCELLED`                                                                                                                     |
diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md
index d06102a16ac3..c729b8ba854d 100644
--- a/docs/integrations/sources/amplitude.md
+++ b/docs/integrations/sources/amplitude.md
@@ -27,7 +27,7 @@ The Amplitude source connector supports the following streams:
 * [Events](https://developers.amplitude.com/docs/export-api#export-api---export-your-projects-event-data) \(Incremental sync\)
 
 If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose)
-
+
 ## Supported sync modes
 
 The Amplitude source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
@@ -73,3 +73,4 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio
 | 0.1.2   | 2021-09-21 | [6353](https://github.com/airbytehq/airbyte/pull/6353)   | Correct output schemas on cohorts, events, active\_users, and average\_session\_lengths streams           |
 | 0.1.1   | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Add AIRBYTE\_ENTRYPOINT for kubernetes support                                                            |
 | 0.1.0   | 2021-06-08 | [3664](https://github.com/airbytehq/airbyte/pull/3664)   | New Source: Amplitude                                                                                     |
+
\ No newline at end of file
diff --git a/docs/integrations/sources/apify-dataset.md b/docs/integrations/sources/apify-dataset.md
index 1390e4e36339..6793c861387b 100644
--- a/docs/integrations/sources/apify-dataset.md
+++ b/docs/integrations/sources/apify-dataset.md
@@ -20,7 +20,7 @@ When your Apify job \(aka [actor run](https://docs.apify.com/actors/running)\) f
 
 ### Output schema
 
-Since the dataset items do not have strongly typed schema, they are synced as objects, without any assumption on their content.
+Since the dataset items do not have strongly typed schema, they are synced as objects stored in the `data` field, without any assumption on their content.
 
 ### Features
 
@@ -43,6 +43,7 @@ The Apify dataset connector uses [Apify Python Client](https://docs.apify.com/ap
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.2.0 | 2022-06-20 | [28290](https://github.com/airbytehq/airbyte/pull/28290) | Make connector work with platform changes not syncing empty stream schemas. |
 | 0.1.11 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. |
 | 0.1.9 | 2022-04-05 | [PR\#11712](https://github.com/airbytehq/airbyte/pull/11712) | No changes from 0.1.4. Used connector to test publish workflow changes. |
 | 0.1.4 | 2021-12-23 | [PR\#8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications |
diff --git a/docs/integrations/sources/appfollow-migrations.md b/docs/integrations/sources/appfollow-migrations.md
new file mode 100644
index 000000000000..69485b8ad800
--- /dev/null
+++ b/docs/integrations/sources/appfollow-migrations.md
@@ -0,0 +1,5 @@
+# Appfollow Migration Guide
+
+## Upgrading to 1.0.0
+
+Remove connector parameters to ingest all possible apps and add new streams.
\ No newline at end of file
diff --git a/docs/integrations/sources/appfollow.md b/docs/integrations/sources/appfollow.md
index 08f321c8b941..7d7fdb9e2332 100644
--- a/docs/integrations/sources/appfollow.md
+++ b/docs/integrations/sources/appfollow.md
@@ -37,4 +37,5 @@ The Appfollow connector ideally should gracefully handle Appfollow API limitatio
 
 | Version | Date       | Pull Request                                             | Subject               |
 | :------ | :--------- | :------------------------------------------------------- | :-------------------- |
+| 1.0.0   | 2023-08-05 | [29128](https://github.com/airbytehq/airbyte/pull/29128) | Migrate to low-code and add new streams  |
 | 0.1.1   | 2022-08-11 | [14418](https://github.com/airbytehq/airbyte/pull/14418) | New Source: Appfollow |
diff --git a/docs/integrations/sources/apple-search-ads.md b/docs/integrations/sources/apple-search-ads.md
index f1c6e9ea9580..ffba4f33a596 100644
--- a/docs/integrations/sources/apple-search-ads.md
+++ b/docs/integrations/sources/apple-search-ads.md
@@ -1,14 +1,18 @@
 # Apple Search Ads
+
 This page contains the setup guide and reference information for the Apple Search Ads source connector.
 
 ## Setup guide
+
 ### Step 1: Set up Apple Search Ads
+
 1. With an administrator account, [create an API user role](https://developer.apple.com/documentation/apple_search_ads/implementing_oauth_for_the_apple_search_ads_api) from the Apple Search Ads UI.
 2. Then [implement OAuth for your API user](https://developer.apple.com/documentation/apple_search_ads/implementing_oauth_for_the_apple_search_ads_api) in order to the required Client Secret and Client Id.
 
-
 ### Step 2: Set up the source connector in Airbyte
+
 #### For Airbyte Open Source
+
 1. Log in to your Airbyte Open Source account.
 2. Click **Sources** and then click **+ New source**.
 3. On the Set up the source page, select **Apple Search Ads** from the **Source type** dropdown.
@@ -19,32 +23,39 @@ This page contains the setup guide and reference information for the Apple Searc
 8. Click **Set up source**.
 
 ## Supported sync modes
+
 The Apple Search Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
+
 The Apple Ads source connector supports the following streams. For more information, see the [Apple Search Ads API](https://developer.apple.com/documentation/apple_search_ads).
 
 ### Base streams
+
 - [campaigns](https://developer.apple.com/documentation/apple_search_ads/get_all_campaigns)
 - [adgroups](https://developer.apple.com/documentation/apple_search_ads/get_all_ad_groups)
 - [keywords](https://developer.apple.com/documentation/apple_search_ads/get_all_targeting_keywords_in_an_ad_group)
 
 ### Report Streams
+
 - [campaigns_report_daily](https://developer.apple.com/documentation/apple_search_ads/get_campaign-level_reports)
 - [adgroups_report_daily](https://developer.apple.com/documentation/apple_search_ads/get__ad_group-level_reports)
 - [keywords_report_daily](https://developer.apple.com/documentation/apple_search_ads/get_keyword-level_reports)
 
 ### Report aggregation
+
 The Apple Search Ads currently offers [aggregation](https://developer.apple.com/documentation/apple_search_ads/reportingrequest) at hourly, daily, weekly, or monthly level.
 
 However, at this moment and as indicated in the stream names, the connector only offers data with daily aggregation.
 
-
 ## Changelog
-| Version | Date       | Pull Request                                            | Subject                                                                              |
-| :------ |:-----------|:--------------------------------------------------------|:-------------------------------------------------------------------------------------|
+
+| Version | Date       | Pull Request                                             | Subject                                                                              |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------- |
+| 0.1.1   | 2023-07-11 | [28153](https://github.com/airbytehq/airbyte/pull/28153) | Fix manifest duplicate key (no change in behavior for the syncs)                     |
 | 0.1.0   | 2022-11-17 | [19557](https://github.com/airbytehq/airbyte/pull/19557) | Initial release with campaigns, adgroups & keywords streams (base and daily reports) |
diff --git a/docs/integrations/sources/asana.inapp.md b/docs/integrations/sources/asana.inapp.md
new file mode 100644
index 000000000000..887635811141
--- /dev/null
+++ b/docs/integrations/sources/asana.inapp.md
@@ -0,0 +1,13 @@
+## Prerequisites
+
+* OAuth access or a Personal Access Token
+
+## Setup guide
+1. Enter a name for your source
+2. Authenticate using OAuth (recommended) or enter your `personal_access_token`. Please follow these [steps](https://developers.asana.com/docs/personal-access-token) to obtain a Personal Access Token for your account.
+3. Click **Set up source**
+
+### Syncing Multiple Projects
+If you have access to multiple projects, Airbyte will sync data related to all projects you have access to. The ability to filter to specific projects is not available at this time.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Asana](https://docs.airbyte.com/integrations/sources/asana).
\ No newline at end of file
diff --git a/docs/integrations/sources/auth0.md b/docs/integrations/sources/auth0.md
index d3f43b6c1101..e3e63e390f29 100644
--- a/docs/integrations/sources/auth0.md
+++ b/docs/integrations/sources/auth0.md
@@ -41,7 +41,12 @@ The Auth0 source connector supports the following [sync modes](https://docs.airb
 
 ## Supported Streams
 
+- [Clients](https://auth0.com/docs/api/management/v2#!/Clients/get_clients)
+- [Organizations](https://auth0.com/docs/api/management/v2#!/Organizations/get_organizations)
+- [OrganizationMembers](https://auth0.com/docs/api/management/v2#!/Organizations/get_members)
+- [OrganizationMemberRoles](https://auth0.com/docs/api/management/v2#!/Organizations/get_organization_member_roles)
 - [Users](https://auth0.com/docs/api/management/v2#!/Users/get_users)
+- [Clients](https://auth0.com/docs/api/management/v2/clients/get-clients)
 
 ## Performance considerations
 
@@ -51,6 +56,8 @@ The connector is restricted by Auth0 [rate limits](https://auth0.com/docs/troubl
 
 | Version | Date       | Pull Request                                             | Subject                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
-| 0.2.0  | 2023-05-23 | 26445 | Add Clients stream |
-| 0.1.0  | 2022-10-21 | TBD | Add Auth0 and Users stream |
+|  0.4.0  | 2023-08-03 | [28972](https://github.com/airbytehq/airbyte/pull/28972) | Migrate to Low-Code CDK                                                        |
+|  0.3.0  | 2023-06-20 | [29001](https://github.com/airbytehq/airbyte/pull/29001) | Add Organizations, OrganizationMembers, OrganizationMemberRoles streams        |
+|  0.2.0  | 2023-05-23 | [26445](https://github.com/airbytehq/airbyte/pull/26445) | Add Clients stream                                                             |
+|  0.1.0  | 2022-10-21 | [18338](https://github.com/airbytehq/airbyte/pull/18338) | Add Auth0 and Users stream                                                     |
 
diff --git a/docs/integrations/sources/babelforce.md b/docs/integrations/sources/babelforce.md
index 49b0daaa67f5..749fbf11059a 100644
--- a/docs/integrations/sources/babelforce.md
+++ b/docs/integrations/sources/babelforce.md
@@ -47,4 +47,5 @@ Generate a API access key ID and token using the [Babelforce documentation](http
 
 | Version | Date       | Pull Request                                             | Subject                     |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------|
+ 0.2.0    | 2023-08-24 | [29314](https://github.com/airbytehq/airbyte/pull/29314) | Migrate to Low Code         |
  0.1.0    | 2022-05-09 | [12700](https://github.com/airbytehq/airbyte/pull/12700) | Introduce Babelforce source |
diff --git a/docs/integrations/sources/bamboo-hr.inapp.md b/docs/integrations/sources/bamboo-hr.inapp.md
new file mode 100644
index 000000000000..f7d75a171f76
--- /dev/null
+++ b/docs/integrations/sources/bamboo-hr.inapp.md
@@ -0,0 +1,14 @@
+## Prerequisites
+
+* BambooHR [API Key](https://documentation.bamboohr.com/docs#authentication)
+
+
+## Setup guide
+1. Name your BambooHR connector
+2. Enter your `api_key`. To generate an API key, log in and click your name in the upper right-hand corner of any page to get to the user context menu. If you have sufficient administrator permissions, there will be an "API Keys" option in that menu to go to the page.
+3. Enter your `subdomain`. If you access BambooHR at https://mycompany.bamboohr.com, then the subdomain is "mycompany"
+4. (Optional) Enter any `Custom Report Fields` as a comma-separated list of fields to include in your custom reports. Example: `firstName,lastName`. If none are listed, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned.
+5. Toggle `Custom Reports Include Default Fields`. If true, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. If false, then the values defined in `Custom Report Fields` will be returned. 
+6. Click **Set up source**
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [BambooHR](https://docs.airbyte.com/integrations/sources/bamboo-hr).
diff --git a/docs/integrations/sources/baton.md b/docs/integrations/sources/baton.md
deleted file mode 100644
index 950ac1c0060b..000000000000
--- a/docs/integrations/sources/baton.md
+++ /dev/null
@@ -1,56 +0,0 @@
-# Baton
-
-## Sync overview
-
-This source can sync data from the [baton API](https://app.hellobaton.com/api/redoc/). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records.
-
-## This Source Supports the Following Streams
-
-- activity
-- companies
-- milestones
-- phases
-- project_attachments
-- projects
-- task_attachemnts
-- tasks
-- templates
-- time_entries
-- users
-
-Baton adds new streams fairly regularly please submit an issue or PR if this project doesn't support required streams for your use case.
-
-### Data type mapping
-
-| Integration Type | Airbyte Type | Notes |
-| :--------------- | :----------- | :---- |
-| `string`         | `string`     |       |
-| `integer`        | `integer`    |       |
-| `number`         | `number`     |       |
-| `array`          | `array`      |       |
-| `object`         | `object`     |       |
-
-### Features
-
-| Feature           | Supported?\(Yes/No\) | Notes |
-| :---------------- | :------------------- | :---- |
-| Full Refresh Sync | Yes                  |       |
-| Incremental Sync  | No                   |       |
-| Namespaces        | No                   |       |
-
-### Performance considerations
-
-The connector is rate limited at 1000 requests per minute per api key. If you find yourself receiving errors contact your customer success manager and request a rate limit increase.
-
-## Getting started
-
-### Requirements
-
-- Baton account
-- Baton api key
-
-## Changelog
-
-| Version | Date       | Pull Request                                           | Subject                   |
-| :------ | :--------- | :----------------------------------------------------- | :------------------------ |
-| 0.1.0   | 2022-01-14 | [8461](https://github.com/airbytehq/airbyte/pull/8461) | 🎉 New Source: Hellobaton |
diff --git a/docs/integrations/sources/bigquery.md b/docs/integrations/sources/bigquery.md
index eefecdef7b3f..c4b3d042b519 100644
--- a/docs/integrations/sources/bigquery.md
+++ b/docs/integrations/sources/bigquery.md
@@ -18,31 +18,31 @@ The BigQuery source does not alter the schema present in your database. Dependin
 
 The BigQuery data types mapping:
 
-| BigQuery Type | Resulting Type | Notes |
-| :--- | :--- | :--- |
-| `BOOL` | Boolean |  |
-| `INT64` | Number |  |
-| `FLOAT64` | Number |  |
-| `NUMERIC` | Number |  |
-| `BIGNUMERIC` | Number |  |
-| `STRING` | String |  |
-| `BYTES` | String |  |
-| `DATE` | String | In ISO8601 format |
-| `DATETIME` | String | In ISO8601 format |
-| `TIMESTAMP` | String | In ISO8601 format |
-| `TIME` | String |  |
-| `ARRAY` | Array |  |
-| `STRUCT` | Object |  |
-| `GEOGRAPHY` | String |  |
+| BigQuery Type | Resulting Type | Notes             |
+| :------------ | :------------- | :---------------- |
+| `BOOL`        | Boolean        |                   |
+| `INT64`       | Number         |                   |
+| `FLOAT64`     | Number         |                   |
+| `NUMERIC`     | Number         |                   |
+| `BIGNUMERIC`  | Number         |                   |
+| `STRING`      | String         |                   |
+| `BYTES`       | String         |                   |
+| `DATE`        | String         | In ISO8601 format |
+| `DATETIME`    | String         | In ISO8601 format |
+| `TIMESTAMP`   | String         | In ISO8601 format |
+| `TIME`        | String         |                   |
+| `ARRAY`       | Array          |                   |
+| `STRUCT`      | Object         |                   |
+| `GEOGRAPHY`   | String         |                   |
 
 ### Features
 
-| Feature | Supported | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental Sync | Yes |  |
-| Change Data Capture | No |  |
-| SSL Support | Yes |  |
+| Feature             | Supported | Notes |
+| :------------------ | :-------- | :---- |
+| Full Refresh Sync   | Yes       |       |
+| Incremental Sync    | Yes       |       |
+| Change Data Capture | No        |       |
+| SSL Support         | Yes       |       |
 
 ## Getting started
 
@@ -50,9 +50,9 @@ The BigQuery data types mapping:
 
 To use the BigQuery source, you'll need:
 
-* A Google Cloud Project with BigQuery enabled
-* A Google Cloud Service Account with the "BigQuery User" and "BigQuery Data Editor" roles in your GCP project
-* A Service Account Key to authenticate into your Service Account
+- A Google Cloud Project with BigQuery enabled
+- A Google Cloud Service Account with the "BigQuery User" and "BigQuery Data Editor" roles in your GCP project
+- A Service Account Key to authenticate into your Service Account
 
 See the setup guide for more information about how to create the required resources.
 
@@ -76,9 +76,9 @@ Follow the [Creating and Managing Service Account Keys](https://cloud.google.com
 
 You should now have all the requirements needed to configure BigQuery as a source in the UI. You'll need the following information to configure the BigQuery source:
 
-* **Project ID**
-* **Default Dataset ID \[Optional\]**: the schema name if only one schema is interested. Dramatically boost source discover operation.
-* **Credentials JSON**: the contents of your Service Account Key JSON file
+- **Project ID**
+- **Default Dataset ID \[Optional\]**: the schema name if only one schema is interested. Dramatically boost source discover operation.
+- **Credentials JSON**: the contents of your Service Account Key JSON file
 
 Once you've configured BigQuery as a source, delete the Service Account Key from your computer.
 
@@ -86,20 +86,20 @@ Once you've configured BigQuery as a source, delete the Service Account Key from
 
 ### source-bigquery
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.3 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
-| 0.2.2 | 2022-09-22 | [16902](https://github.com/airbytehq/airbyte/pull/16902) | Source BigQuery: added user agent header  |
-| 0.2.1 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage  |
-| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. |
-| 0.1.9 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
-| 0.1.8 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
-| 0.1.7 | 2022-04-11 | [11484](https://github.com/airbytehq/airbyte/pull/11484) | BigQuery connector escape column names            |
-| 0.1.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
-| 0.1.5 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434)   | Update fields in source-connectors specifications |
-| 0.1.4 | 2021-09-30 | [\#6524](https://github.com/airbytehq/airbyte/pull/6524) | Allow `dataset_id` null in spec |
-| 0.1.3 | 2021-09-16 | [\#6051](https://github.com/airbytehq/airbyte/pull/6051) | Handle NPE `dataset_id` is not provided |
-| 0.1.2 | 2021-09-16 | [\#6135](https://github.com/airbytehq/airbyte/pull/6135) | 🐛 BigQuery source: Fix nested structs |
-| 0.1.1 | 2021-07-28 | [\#4981](https://github.com/airbytehq/airbyte/pull/4981) | 🐛 BigQuery source: Fix nested arrays |
-| 0.1.0 | 2021-07-22 | [\#4457](https://github.com/airbytehq/airbyte/pull/4457) | 🎉 New Source: Big Query. |
-
+| Version | Date       | Pull Request                                             | Subject                                                                                                                                   |
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.3.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2                                                                                                                      |
+| 0.2.3   | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
+| 0.2.2   | 2022-09-22 | [16902](https://github.com/airbytehq/airbyte/pull/16902) | Source BigQuery: added user agent header                                                                                                  |
+| 0.2.1   | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage                                                                                                            |
+| 0.2.0   | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields.                                                                                      |
+| 0.1.9   | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors                                                                            |
+| 0.1.8   | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors                                                                                    |
+| 0.1.7   | 2022-04-11 | [11484](https://github.com/airbytehq/airbyte/pull/11484) | BigQuery connector escape column names                                                                                                    |
+| 0.1.6   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                                              |
+| 0.1.5   | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434)   | Update fields in source-connectors specifications                                                                                         |
+| 0.1.4   | 2021-09-30 | [\#6524](https://github.com/airbytehq/airbyte/pull/6524) | Allow `dataset_id` null in spec                                                                                                           |
+| 0.1.3   | 2021-09-16 | [\#6051](https://github.com/airbytehq/airbyte/pull/6051) | Handle NPE `dataset_id` is not provided                                                                                                   |
+| 0.1.2   | 2021-09-16 | [\#6135](https://github.com/airbytehq/airbyte/pull/6135) | 🐛 BigQuery source: Fix nested structs                                                                                                    |
+| 0.1.1   | 2021-07-28 | [\#4981](https://github.com/airbytehq/airbyte/pull/4981) | 🐛 BigQuery source: Fix nested arrays                                                                                                     |
+| 0.1.0   | 2021-07-22 | [\#4457](https://github.com/airbytehq/airbyte/pull/4457) | 🎉 New Source: Big Query.                                                                                                                 |
diff --git a/docs/integrations/sources/bing-ads.inapp.md b/docs/integrations/sources/bing-ads.inapp.md
new file mode 100644
index 000000000000..225e16d1352a
--- /dev/null
+++ b/docs/integrations/sources/bing-ads.inapp.md
@@ -0,0 +1,33 @@
+## Prerequisites
+* [Microsoft developer token](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) of an authorized Bing Ads OAuth application
+
+To use the Bing Ads API, you must have a developer token and valid user credentials. You will need to register an OAuth app to get a refresh token.
+1. [Register your application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in the Azure portal.
+2. [Request user consent](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get the authorization code.
+3. Use the authorization code to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13).
+
+:::note
+
+The refresh token expires every 90 days. Repeat the authorization process to get a new refresh token. The full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token).
+Please be sure to authenticate with the email (personal or work) that you used to sign in to the Bing ads/Microsoft ads platform.
+:::
+
+4. Request your [Microsoft developer token](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) in the Microsoft Advertising Developer Portal account tab. 
+
+
+## Setup guide
+1. Enter a name for your source.
+2. Enter the developer token
+3. For **Tenant ID**, enter the custom tenant or use the common tenant. If your OAuth app has a custom tenant and you cannot use Microsoft’s recommended common tenant, use the custom tenant in the Tenant ID field when you set up the connector. 
+
+:::info
+The custom tenant is used in the authentication URL, for example: `https://login.microsoftonline.com//oauth2/v2.0/authorize`
+
+:::
+
+4. For **Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
+5. For **Lookback window** (also known as attribution or conversion window), enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value.
+6. Click **Authenticate your Bing Ads account** and authorize your account.
+8. Click **Set up source**.  
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Bing Ads](https://docs.airbyte.com/integrations/sources/bing-ads).
diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md
index 4054957f369a..20910a85be35 100644
--- a/docs/integrations/sources/bing-ads.md
+++ b/docs/integrations/sources/bing-ads.md
@@ -1,8 +1,11 @@
 # Bing Ads
+
 This page contains the setup guide and reference information for the Bing Ads source connector.
 
 ## Setup guide
+
 ### Step 1: Set up Bing Ads
+
 1. [Register your application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in the Azure portal.
 2. [Request user consent](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get the authorization code.
 3. Use the authorization code to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13).
@@ -25,7 +28,9 @@ The tenant is used in the authentication URL, for example: `https://login.micros
 ### Step 2: Set up the source connector in Airbyte
 
 
+
 **For Airbyte Cloud:**
+
 1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
 2. Click **Sources** and then click **+ New source**.
 3. On the Set up the source page, select **Bing Ads** from the **Source type** dropdown.
@@ -36,11 +41,13 @@ The tenant is used in the authentication URL, for example: `https://login.micros
 8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value.
 9. Click **Authenticate your Bing Ads account**.
 10. Log in and authorize the Bing Ads account.
-11. Click **Set up source**.  
+11. Click **Set up source**.
 
 
 
+
 **For Airbyte Open Source:**
+
 1. Log in to your Airbyte Open Source account.
 2. Click **Sources** and then click **+ New source**.
 3. On the Set up the source page, select **Bing Ads** from the **Source type** dropdown.
@@ -53,22 +60,27 @@ The tenant is used in the authentication URL, for example: `https://login.micros
 
 
 ## Supported sync modes
+
 The Bing Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
+
 The Bing Ads source connector supports the following streams. For more information, see the [Bing Ads API](https://docs.microsoft.com/en-us/advertising/guides/?view=bingads-13).
 
 ### Basic streams
+
 - [accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13)
 - [ad_groups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13)
 - [ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13)
 - [campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13)
 
 ### Report Streams
+
 - [account_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13)
 - [account_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13)
 - [account_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13)
@@ -81,6 +93,10 @@ The Bing Ads source connector supports the following streams. For more informati
 - [ad_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13)
 - [ad_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13)
 - [ad_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13)
+- [geographic_performance_report_hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13)
+- [geographic_performance_report_daily](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13)
+- [geographic_performance_report_weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13)
+- [geographic_performance_report_monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13)
 - [budget_summary_report](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13)
 - [campaign_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13)
 - [campaign_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13)
@@ -92,6 +108,7 @@ The Bing Ads source connector supports the following streams. For more informati
 - [keyword_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13)
 
 ### Report aggregation
+
 All reports synced by this connector can be [aggregated](https://docs.microsoft.com/en-us/advertising/reporting-service/reportaggregation?view=bingads-13) using hourly, daily, weekly, or monthly time windows.
 
 For example, if you select a report with daily aggregation, the report will contain a row for each day for the duration of the report. Each row will indicate the number of impressions recorded on that day.
@@ -99,32 +116,36 @@ For example, if you select a report with daily aggregation, the report will cont
 A report's aggregation window is indicated in its name. For example, `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window.
 
 ## Performance considerations
+
 The Bing Ads API limits the number of requests for all Microsoft Advertising clients. You can find detailed info [here](https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling).
 
 ## Changelog
-| Version | Date       | Pull Request                                                                                                                     | Subject                                                                                                                                       |
-|:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------|
-| 0.1.23  | 2023-05-11 | [25996](https://github.com/airbytehq/airbyte/pull/25996)                                                                         | Implement a retry logic if SSL certificate validation fails.   |
-| 0.1.22  | 2023-05-08 | [24223](https://github.com/airbytehq/airbyte/pull/24223)                                                                         | Add CampaignLabels report column in campaign performance report   |
-| 0.1.21  | 2023-04-28 | [25668](https://github.com/airbytehq/airbyte/pull/25668)                                                                         | Add undeclared fields to accounts, campaigns,  campaign_performance_report, keyword_performance_report and account_performance_report streams |
-| 0.1.20  | 2023-03-09 | [23663](https://github.com/airbytehq/airbyte/pull/23663)                                                                         | Add lookback window for performance reports in incremental mode                                                                               |
-| 0.1.19  | 2023-03-08 | [23868](https://github.com/airbytehq/airbyte/pull/23868)                                                                         | Add dimensional-type columns for reports.                                                                                                     |
-| 0.1.18  | 2023-01-30 | [22073](https://github.com/airbytehq/airbyte/pull/22073)                                                                         | Fix null values in the `Keyword` column of `keyword_performance_report` streams                                                               |
-| 0.1.17  | 2022-12-10 | [20005](https://github.com/airbytehq/airbyte/pull/20005)                                                                         | Add `Keyword` to `keyword_performance_report` stream                                                                                          |
-| 0.1.16  | 2022-10-12 | [17873](https://github.com/airbytehq/airbyte/pull/17873)                                                                         | Fix: added missing campaign types in (Audience, Shopping and DynamicSearchAds) in campaigns stream                                            |
-| 0.1.15  | 2022-10-03 | [17505](https://github.com/airbytehq/airbyte/pull/17505)                                                                         | Fix: limit cache size for ServiceClient instances                                                                                             |
-| 0.1.14  | 2022-09-29 | [17403](https://github.com/airbytehq/airbyte/pull/17403)                                                                         | Fix: limit cache size for ReportingServiceManager instances                                                                                   |
-| 0.1.13  | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386)                                                                         | Migrate to per-stream states.                                                                                                                 |
-| 0.1.12  | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335)                                                                         | Added backoff for socket.timeout                                                                                                              |
-| 0.1.11  | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable                                                                                                           |
-| 0.1.10  | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602)                                                                         | Fixed bug caused Hourly Reports to crash due to invalid fields set                                                                            |
-| 0.1.9   | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862)                                                                         | Added missing columns                                                                                                                         |
-| 0.1.8   | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801)                                                                         | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration                    |
-| 0.1.7   | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937)                                                                         | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration                                                         |
-| 0.1.6   | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500)                                                                         | Improve input configuration copy                                                                                                              |
-| 0.1.5   | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652)                                                                         | Rebump attempt after DockerHub failure at registring the 0.1.4                                                                                |
-| 0.1.4   | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311)                                                                         | Added optional Redirect URI & Tenant ID to spec                                                                                               |
-| 0.1.3   | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510)                                                                           | Fixed broken dependency that blocked connector's operations                                                                                   |
-| 0.1.2   | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429)                                                                           | Update titles and descriptions                                                                                                                |
-| 0.1.1   | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750)                                                                           | Added reporting streams\)                                                                                                                     |
-| 0.1.0   | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911)                                                                           | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\)                                                                 |
+
+| Version | Date       | Pull Request                                                                                                                     | Subject                                                                                                                                      |
+| :------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.2.0  | 2023-08-17 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Add Geographic Performance Report  |
+| 0.1.24  | 2023-06-22 | [27619](https://github.com/airbytehq/airbyte/pull/27619)                                                                         | Retry request after facing temporary name resolution error.                                                                                  |
+| 0.1.23  | 2023-05-11 | [25996](https://github.com/airbytehq/airbyte/pull/25996)                                                                         | Implement a retry logic if SSL certificate validation fails.                                                                                 |
+| 0.1.22  | 2023-05-08 | [24223](https://github.com/airbytehq/airbyte/pull/24223)                                                                         | Add CampaignLabels report column in campaign performance report                                                                              |
+| 0.1.21  | 2023-04-28 | [25668](https://github.com/airbytehq/airbyte/pull/25668)                                                                         | Add undeclared fields to accounts, campaigns, campaign_performance_report, keyword_performance_report and account_performance_report streams |
+| 0.1.20  | 2023-03-09 | [23663](https://github.com/airbytehq/airbyte/pull/23663)                                                                         | Add lookback window for performance reports in incremental mode                                                                              |
+| 0.1.19  | 2023-03-08 | [23868](https://github.com/airbytehq/airbyte/pull/23868)                                                                         | Add dimensional-type columns for reports.                                                                                                    |
+| 0.1.18  | 2023-01-30 | [22073](https://github.com/airbytehq/airbyte/pull/22073)                                                                         | Fix null values in the `Keyword` column of `keyword_performance_report` streams                                                              |
+| 0.1.17  | 2022-12-10 | [20005](https://github.com/airbytehq/airbyte/pull/20005)                                                                         | Add `Keyword` to `keyword_performance_report` stream                                                                                         |
+| 0.1.16  | 2022-10-12 | [17873](https://github.com/airbytehq/airbyte/pull/17873)                                                                         | Fix: added missing campaign types in (Audience, Shopping and DynamicSearchAds) in campaigns stream                                           |
+| 0.1.15  | 2022-10-03 | [17505](https://github.com/airbytehq/airbyte/pull/17505)                                                                         | Fix: limit cache size for ServiceClient instances                                                                                            |
+| 0.1.14  | 2022-09-29 | [17403](https://github.com/airbytehq/airbyte/pull/17403)                                                                         | Fix: limit cache size for ReportingServiceManager instances                                                                                  |
+| 0.1.13  | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386)                                                                         | Migrate to per-stream states.                                                                                                                |
+| 0.1.12  | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335)                                                                         | Added backoff for socket.timeout                                                                                                             |
+| 0.1.11  | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable                                                                                                          |
+| 0.1.10  | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602)                                                                         | Fixed bug caused Hourly Reports to crash due to invalid fields set                                                                           |
+| 0.1.9   | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862)                                                                         | Added missing columns                                                                                                                        |
+| 0.1.8   | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801)                                                                         | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration                   |
+| 0.1.7   | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937)                                                                         | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration                                                        |
+| 0.1.6   | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500)                                                                         | Improve input configuration copy                                                                                                             |
+| 0.1.5   | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652)                                                                         | Rebump attempt after DockerHub failure at registring the 0.1.4                                                                               |
+| 0.1.4   | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311)                                                                         | Added optional Redirect URI & Tenant ID to spec                                                                                              |
+| 0.1.3   | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510)                                                                           | Fixed broken dependency that blocked connector's operations                                                                                  |
+| 0.1.2   | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429)                                                                           | Update titles and descriptions                                                                                                               |
+| 0.1.1   | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750)                                                                           | Added reporting streams\)                                                                                                                    |
+| 0.1.0   | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911)                                                                           | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\)                                                                |
diff --git a/docs/integrations/sources/braintree.md b/docs/integrations/sources/braintree.md
index 2411a2ab2aec..fb4663ce73b2 100644
--- a/docs/integrations/sources/braintree.md
+++ b/docs/integrations/sources/braintree.md
@@ -71,6 +71,7 @@ The Braintree connector should not run into Braintree API limitations under norm
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.2.0 | 2023-07-17 | [29200](https://github.com/airbytehq/airbyte/pull/29200) | Migrate connector to low-code framework |
 | 0.1.5 | 2023-05-24 | [26340](https://github.com/airbytehq/airbyte/pull/26340) | Fix error in `check_connection` in integration tests |
 | 0.1.4 | 2023-03-13 | [23548](https://github.com/airbytehq/airbyte/pull/23548) | Update braintree python library version to 4.18.1 |
 | 0.1.3 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications |
diff --git a/docs/integrations/sources/captain-data.md b/docs/integrations/sources/captain-data.md
new file mode 100644
index 000000000000..dc7fff642f97
--- /dev/null
+++ b/docs/integrations/sources/captain-data.md
@@ -0,0 +1,65 @@
+# Captain Data
+
+This page contains the setup guide and reference information for the [Captain Data](https://docs.captaindata.co/#intro) source connector.
+
+## Prerequisites
+
+Api key and project UID are mandate for this connector to work, It could be generated from the dashboard settings (ref - https://app.captaindata.co/settings).
+
+## Setup guide
+
+### Step 1: Set up Captain Data connection
+
+- Available params
+  - api_key: The api_key
+  - project_uid: The project UID
+
+## Step 2: Set up the Captain Data connector in Airbyte
+
+### For Airbyte Cloud:
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**.
+3. On the Set up the source page, enter the name for the Captain Data connector and select **Captain Data** from the Source type dropdown.
+4. Enter your `api_key` and `project_uid`.
+5. Click **Set up source**.
+
+### For Airbyte OSS:
+
+1. Navigate to the Airbyte Open Source dashboard.
+2. Set the name for your source.
+3. Enter your `api_key` and `project_uid`.
+4. Click **Set up source**.
+
+## Supported sync modes
+
+The Captain Data source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+
+| Feature                       | Supported? |
+| :---------------------------- | :--------- |
+| Full Refresh Sync             | Yes        |
+| Incremental Sync              | No         |
+| Replicate Incremental Deletes | No         |
+| SSL connection                | Yes        |
+| Namespaces                    | No         |
+
+## Supported Streams
+
+- workspace
+- workflows
+- jobs
+- job_results
+
+## API method example
+
+GET https://api.captaindata.co/v3/
+
+## Performance considerations
+
+Captain Data [API reference](https://docs.captaindata.co/#intro) has v3 at present. The connector as default uses v3.
+
+## Changelog
+
+| Version | Date       | Pull Request                                            | Subject        |
+| :------ | :--------- | :------------------------------------------------------ | :------------- |
+| 0.1.0   | 2023-04-15 | [Init](https://github.com/airbytehq/airbyte/pull/25230) | Initial commit |
diff --git a/docs/integrations/sources/chargebee.md b/docs/integrations/sources/chargebee.md
index d9f8f15d2688..81a64c2a707c 100644
--- a/docs/integrations/sources/chargebee.md
+++ b/docs/integrations/sources/chargebee.md
@@ -74,6 +74,7 @@ The Chargebee connector should not run into [Chargebee API](https://apidocs.char
 
 | Version | Date       | Pull Request                                             | Subject                                                                                             |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------|
+| 0.2.4   | 2023-08-01 | [28905](https://github.com/airbytehq/airbyte/pull/28905) | Updated the connector to use latest CDK version                                                               |
 | 0.2.3   | 2023-03-22 | [24370](https://github.com/airbytehq/airbyte/pull/24370) | Ignore 404 errors for `Contact` stream                                                              |
 | 0.2.2   | 2023-02-17 | [21688](https://github.com/airbytehq/airbyte/pull/21688) | Migrate to CDK beta 0.29; fix schemas                                                               |
 | 0.2.1   | 2023-02-17 | [23207](https://github.com/airbytehq/airbyte/pull/23207) | Edited stream schemas to get rid of unnecessary `enum`                                              |
diff --git a/docs/integrations/sources/chargify.md b/docs/integrations/sources/chargify.md
index e78fed1a4a95..f533658f4f7e 100644
--- a/docs/integrations/sources/chargify.md
+++ b/docs/integrations/sources/chargify.md
@@ -8,20 +8,20 @@ The Chargify source supports Full Refresh syncs for Customers and Subscriptions
 
 Several output streams are available from this source:
 
-* [Customers](https://developers.chargify.com/docs/api-docs/b3A6MTQxMDgyNzY-list-or-find-customers)
-* [Subscriptions](https://developers.chargify.com/docs/api-docs/b3A6MTQxMDgzODk-list-subscriptions)
+- [Customers](https://developers.chargify.com/docs/api-docs/b3A6MTQxMDgyNzY-list-or-find-customers)
+- [Subscriptions](https://developers.chargify.com/docs/api-docs/b3A6MTQxMDgzODk-list-subscriptions)
 
 If there are more streams you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose)
 
 ### Features
 
-| Feature | Supported? |
-| :--- | :--- |
-| Full Refresh Sync | Yes |
-| Incremental Sync | No |
-| Replicate Incremental Deletes | No |
-| SSL connection | Yes |
-| Namespaces | No |
+| Feature                       | Supported? |
+| :---------------------------- | :--------- |
+| Full Refresh Sync             | Yes        |
+| Incremental Sync              | No         |
+| Replicate Incremental Deletes | No         |
+| SSL connection                | Yes        |
+| Namespaces                    | No         |
 
 ### Performance considerations
 
@@ -31,8 +31,8 @@ The Chargify connector should not run into Chargify API limitations under normal
 
 ### Requirements
 
-* Chargify API Key
-* Chargify domain
+- Chargify API Key
+- Chargify domain
 
 ### Setup guide
 
@@ -40,7 +40,8 @@ Please follow the [Chargify documentation for generating an API key](https://dev
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.0 | 2022-03-16 | [10853](https://github.com/airbytehq/airbyte/pull/10853) | Initial release |
-
+| Version | Date       | Pull Request                                             | Subject                        |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------- |
+| 0.3.0   | 2023-08-10 | [29130](https://github.com/airbytehq/airbyte/pull/29130) | Migrate Python CDK to Low Code |
+| 0.2.0   | 2023-08-08 | [29218](https://github.com/airbytehq/airbyte/pull/29218) | Fix schema                     |
+| 0.1.0   | 2022-03-16 | [10853](https://github.com/airbytehq/airbyte/pull/10853) | Initial release                |
diff --git a/docs/integrations/sources/chartmogul.md b/docs/integrations/sources/chartmogul.md
index 70a52d3dec3e..8d7726109a28 100644
--- a/docs/integrations/sources/chartmogul.md
+++ b/docs/integrations/sources/chartmogul.md
@@ -1,34 +1,46 @@
 # Chartmogul
-This page contains the setup guide and reference information for the Chartmogul source connector.
+This page contains the setup guide and reference information for the [Chartmogul](https://chartmogul.com/) source connector.
 
 ## Prerequisites
-* API key
-* Start date
-* Interval
+- A Chartmogul API Key.
+- A desired start date from which to begin replicating data.
+- A desired interval period for the `CustomerCount` stream. The available options are **day**, **week**, **month**, and **quarter**.
 
 ## Setup guide
-### Step 1: Set up Chartmogul
-
-1. To get access to the Chartmogul API you need to create an API key, please follow the instructions in this [documentation](https://help.chartmogul.com/hc/en-us/articles/4407796325906-Creating-and-Managing-API-keys#creating-an-api-key).
+### Step 1: Set up a Chartmogul API key
+1. Log in to your Chartmogul account.
+2. In the left navbar, select **Profile** > **View Profile**.
+3. Select **NEW API KEY**.
+4. In the **Name** field, enter a unique name for the key.
+5. If you are a Staff, Admin, or Owner, set the **Access Level** to either **Read-only** or **Read & Write** using the dropdown menu. We recommend **Read-only**.
+6. Click **ADD** to create the key.
+7. Click the **Reveal** icon to see the key, and the **Copy** icon to copy it to your clipboard.
+
+For further reading on Chartmogul API Key creation and maintenance, please refer to the official 
+[Chartmogul documentation](https://help.chartmogul.com/hc/en-us/articles/4407796325906-Creating-and-Managing-API-keys#creating-an-api-key).
 
 ### Step 2: Set up the Chartmogul connector in Airbyte
-**For Airbyte Cloud:**
-
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
-3. On the source setup page, select **Chartmogul** from the Source type dropdown and enter a name for this connector.
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to the Airbyte Open Source dashboard.
+2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **Chartmogul** from the list of available sources.
+3. Enter a **Source name** of your choosing.
 4. Enter the **API key** that you obtained.
-5. Enter **Start date** - UTC date and time in the format 2017-01-25T00:00:00Z. The data added on and after this date will be replicated.
-6. Enter the **Interval** - day, week, month, quarter for `CustomerCount` stream.
+5. Enter a **Start date**. If you are configuring this connector programmatically, please format the date as such: `yyyy-mm-ddThh:mm:ssZ`. For example, an input of `2017-01-25T06:30:00Z` will signify a start date of 6:30 AM UTC on January 25th, 2017. When feasible, any data before this date will not be replicated.
+
+:::note
+The **Start date** will only apply to the `Activities` stream. The `Customers` endpoint does not provide a way to filter by the creation or update dates.
+:::
+
+6. From the **Interval** dropdown menu, select an interval period for the `CustomerCount` stream. 
+7. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
-The Chartmogul source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+The Chartmogul source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
 * [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite)
 * [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 
-## Supported Streams
+## Supported streams
 
 This connector outputs the following full refresh streams:
 
@@ -36,11 +48,7 @@ This connector outputs the following full refresh streams:
 * [CustomerCount](https://dev.chartmogul.com/reference/retrieve-customer-count)
 * [Customers](https://dev.chartmogul.com/reference/list-customers)
 
-### Notes
-
-The **Start date** will only apply to the `Activities` stream. The `Customers` endpoint does not provide a way to filter by the creation or update dates.
-
-### Performance considerations
+## Performance considerations
 
 The Chartmogul connector should not run into Chartmogul API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully.
 
diff --git a/docs/integrations/sources/clockify.md b/docs/integrations/sources/clockify.md
index 5e5c386f5763..ec3b68c6ee93 100644
--- a/docs/integrations/sources/clockify.md
+++ b/docs/integrations/sources/clockify.md
@@ -4,6 +4,8 @@ The Airbyte Source for [Clockify](https://clockify.me)
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                 |
-| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------- |
-| 0.1.0   | 2022-10-26 | [17767](https://github.com/airbytehq/airbyte/pull/17767) | 🎉 New Connector: Clockify [python cdk] |
+| Version | Date       | Pull Request                                             | Subject                                                   |
+| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- |
+| 0.2.1   | 2023-08-01 | [27881](https://github.com/airbytehq/airbyte/pull/27881) | 🐛 Source Clockify: Source Clockify: Fix pagination logic |
+| 0.2.0   | 2023-08-01 | [27689](https://github.com/airbytehq/airbyte/pull/27689) | ✨ Source Clockify: Add Optional API Url parameter        |
+| 0.1.0   | 2022-10-26 | [17767](https://github.com/airbytehq/airbyte/pull/17767) | 🎉 New Connector: Clockify [python cdk]                   |
diff --git a/docs/integrations/sources/close-com.md b/docs/integrations/sources/close-com.md
index 21587b28dd21..9cb94dcc2de1 100644
--- a/docs/integrations/sources/close-com.md
+++ b/docs/integrations/sources/close-com.md
@@ -1,29 +1,44 @@
 # Close.com
 
+This page contains the setup guide and reference information for the [Close.com](https://www.close.com/) source connector.
+
 ## Prerequisites
 
-* Close.com Account
 * Close.com API Key
 
-Visit the [Close.com API Keys page](https://app.close.com/settings/api/) in the Close.com dashboard to access the secret key for your account. Secret key will be prefixed with `api_`.
-See [this guide](https://help.close.com/docs/api-keys) if you need to create a new one.
-
 We recommend creating a restricted key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. For ease of use, we recommend using read permissions for all resources and configuring which resource to replicate in the Airbyte UI.
 
 ## Setup guide
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**.
-3. On the Set up the source page, enter the name for the Close.com connector and select **Close.com** from the Source type dropdown.
-4. Fill in the API Key and Start date fields and click **Set up source**.
+### Step 1: Set up your Close.com API Key
+1. [Log in to your Close.com](https://www.close.com) account.
+2. At the bottom of the left navbar, select **Settings**.
+3. In the left menu, select **Developer**.
+4. At the top of the page, click **+ New API Key**.
+
+:::caution
+For security purposes, the API Key will only be displayed once upon creation. Be sure to copy and store the key in a secure location.
+:::
+
+For further reading on creating and maintaining Close.com API keys, refer to the 
+[official documentation](https://help.close.com/docs/api-keys-oauth).
+
+### Step 2: Set up the Close.com connector in Airbyte
+
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to the Airbyte Open Source dashboard.
+2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **Close.com** from the list of available sources.
+3. Enter a **Source name** of your choosing.
+4. In the **API Key** field, enter your Close.com **API Key**
+5. *Optional* - In the **Replication Start Date** field, you may enter a starting date cutoff for the data you want to replicate. The format for this date should be as such: `YYYY-MM-DD`. Leaving this field blank will replicate all data. 
+6. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
-The Close.com source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run.
+The Close.com source supports both **Full Refresh** and **Incremental** syncs. You can choose if this connector will copy only the new/updated data, or all rows in the tables and columns you set up for replication. These settings will take effect every time a sync is run.
 
-## Supported Streams
+## Supported streams
 
-This Source is capable of syncing the following core Streams:
+This source is capable of syncing the following core streams:
 
 * [Leads](https://developer.close.com/#leads) \(Incremental\)
 * [Created Activities](https://developer.close.com/#activities-list-or-filter-all-created-activities) \(Incremental\)
@@ -73,9 +88,9 @@ This Source is capable of syncing the following core Streams:
 
 ### Notes
 
-Leads, Events Incremental streams use `date_updated` field as a cursor. All other Incremental streams use `date_created` field for the same purpose.
+Leads and Events Incremental streams use the `date_updated` field as a cursor. All other Incremental streams use the `date_created` field for the same purpose.
 
-`SendAs` stream requires payment.
+The `SendAs` stream requires payment.
 
 ### Data type mapping
 
@@ -83,12 +98,16 @@ The [Close.com API](https://developer.close.com/) uses the same [JSONSchema](htt
 
 ### Performance considerations
 
-The Close.com Connector has rate limit. There are 60 RPS for Organizations. You can find detailed info [here](https://developer.close.com/#ratelimits).
+The Close.com connector is subject to rate limits. For more information on this topic,
+[click here](https://developer.close.com/topics/rate-limits/).
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------|
+| 0.4.2   | 2023-08-08 | [29206](https://github.com/airbytehq/airbyte/pull/29206) | Fixed the issue with `DatePicker` format for `start date`                                            |
+| 0.4.1   | 2023-07-04 | [27950](https://github.com/airbytehq/airbyte/pull/27950) | Add human readable titles to API Key and Start Date fields                                             |
+| 0.4.0   | 2023-06-27 | [27776](https://github.com/airbytehq/airbyte/pull/27776) | Update the `Email Followup Tasks` stream schema                                                        |
 | 0.3.0   | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Update the `Email sequences` stream schema                                                             |
 | 0.2.2   | 2023-05-05 | [25868](https://github.com/airbytehq/airbyte/pull/25868) | Added `CDK TypeTransformer` to gurantee JSON Schema types, added missing properties for `roles` stream |
 | 0.2.1   | 2023-02-15 | [23074](https://github.com/airbytehq/airbyte/pull/23074) | Specified date formatting in specification                                                             |
diff --git a/docs/integrations/sources/coda.md b/docs/integrations/sources/coda.md
index c39e90b19cc0..26d854f62132 100755
--- a/docs/integrations/sources/coda.md
+++ b/docs/integrations/sources/coda.md
@@ -49,6 +49,7 @@ The Coda source connector supports the following [sync modes](https://docs.airby
 - [Tables](https://coda.io/developers/apis/v1#tag/Tables/operation/listTables)
 - [Formulas](https://coda.io/developers/apis/v1#tag/Formulas/operation/listFormulas)
 - [Controls](https://coda.io/developers/apis/v1#tag/Controls/operation/listControls)
+- [Rows](https://coda.io/developers/apis/v1#tag/Rows/operation/listRows)
 
 ## Data type map
 
@@ -61,6 +62,9 @@ The Coda source connector supports the following [sync modes](https://docs.airby
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                          |
-| :------ | :--------- | :------------------------------------------------------- | :------------------------------- |
+| Version | Date       | Pull Request                                             | Subject                           |
+| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- |
+| 1.2.0   | 2023-08-13 | [29288](https://github.com/airbytehq/airbyte/pull/29288) | Migrate python cdk to low-code    |
+| 1.1.0   | 2023-07-10 | [27797](https://github.com/airbytehq/airbyte/pull/27797) | Add `rows` stream                |
+| 1.0.0   | 2023-07-10 | [28093](https://github.com/airbytehq/airbyte/pull/28093) | Update `docs` and `pages` schemas |
 | 0.1.0   | 2022-11-17 | [18675](https://github.com/airbytehq/airbyte/pull/18675) | 🎉 New source: Coda [python cdk] |
diff --git a/docs/integrations/sources/commercetools.md b/docs/integrations/sources/commercetools.md
index 66755961a829..a65bf4b76168 100644
--- a/docs/integrations/sources/commercetools.md
+++ b/docs/integrations/sources/commercetools.md
@@ -50,4 +50,6 @@ Commercetools has some [rate limit restrictions](https://docs.commercetools.com/
 
 | Version | Date       | Pull Request | Subject |
 | :------ | :--------  | :-----       | :------ |
+| 0.2.0  | 2023-08-24 | [29384](https://github.com/airbytehq/airbyte/pull/29384) | Migrate to low code |
+| 0.1.1  | 2023-08-23 | [5957](https://github.com/airbytehq/airbyte/pull/5957) | Fix schemas |
 | 0.1.0  | 2021-08-19 | [5957](https://github.com/airbytehq/airbyte/pull/5957) | Initial Release. Source Commercetools |
diff --git a/docs/integrations/sources/confluence.md b/docs/integrations/sources/confluence.md
index 38b82ce800ca..76fe7adf7b6e 100644
--- a/docs/integrations/sources/confluence.md
+++ b/docs/integrations/sources/confluence.md
@@ -1,56 +1,67 @@
 # Confluence
 
-This page contains the setup guide and reference information for the Confluence source connector.
+This page contains the setup guide and reference information for the
+[Confluence](https://www.atlassian.com/software/confluence) source connector.
 
 ## Prerequisites
 
-* [API Token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)
-* Your Confluence domain name
-* Your Confluence login email
+- Atlassian API Token
+- Your Confluence domain name
+- Your Confluence login email
 
 ## Setup guide
-### Step 1: Set up Confluence connector
-1. Log into your [Airbyte Cloud](https://cloud.airbyte.io/workspaces) or Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**. 
-3. On the Set up the source page, select **Confluence** from the Source type dropdown.
-4. Enter a name for your source.
-5. For **API Token** follow the Jira confluence for generating an  [API Token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)
-6. For **Domain name** enter your Confluence domain name.
-7. For **Email** enter your Confluence login email.
+
+### Step 1: Create an API Token
+
+For detailed instructions on creating an Atlassian API Token, please refer to the
+[official documentation](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/).
+
+### Step 2: Set up the Confluence connector in Airbyte
+
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to the Airbyte Open Source dashboard.
+2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **Confluence** from the list of available sources.
+3. Enter a **Source name** of your choosing.
+4. In the **API Token** field, enter your Atlassian API Token.
+5. In the **Domain name** field, enter your Confluence domain name.
+6. In the **Email** field, enter your Confluence login email.
+7. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
 | Feature                   | Supported? |
-|:--------------------------|:-----------|
+| :------------------------ | :--------- |
 | Full Refresh Sync         | Yes        |
 | Incremental - Append Sync | No         |
 | Incremental - Dedupe Sync | No         |
 | SSL connection            | No         |
 | Namespaces                | No         |
 
-## Supported Streams
+## Supported streams
 
-* [Pages](https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get)
-* [Blog Posts](https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get)
-* [Space](https://developer.atlassian.com/cloud/confluence/rest/api-group-space/#api-wiki-rest-api-space-get)
-* [Group](https://developer.atlassian.com/cloud/confluence/rest/api-group-group/#api-wiki-rest-api-group-get)
-* [Audit](https://developer.atlassian.com/cloud/confluence/rest/api-group-audit/#api-wiki-rest-api-audit-get)
+- [Audit](https://developer.atlassian.com/cloud/confluence/rest/api-group-audit/#api-wiki-rest-api-audit-get)
+- [Blog Posts](https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get)
+- [Group](https://developer.atlassian.com/cloud/confluence/rest/api-group-group/#api-wiki-rest-api-group-get)
+- [Pages](https://developer.atlassian.com/cloud/confluence/rest/api-group-content/#api-wiki-rest-api-content-get)
+- [Space](https://developer.atlassian.com/cloud/confluence/rest/api-group-space/#api-wiki-rest-api-space-get)
 
 :::note
-Stream Audit requires Standard or Premium plan.
+The `audit` stream requires a Standard or Premium plan.
 :::
-## Data type map
-The [Confluence API](https://developer.atlassian.com/cloud/confluence/rest/intro/#about) uses the same [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions happen as part of this source.
 
-### Performance considerations
+## Data type mapping
+
+The [Confluence Cloud REST API](https://developer.atlassian.com/cloud/confluence/rest/v1/intro/#about) uses the same [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions happen as part of this source.
+
+## Performance considerations
 
 The Confluence connector should not run into Confluence API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully.
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                        |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------- |
+| 0.2.0   | 2023-08-14 | [29125](https://github.com/airbytehq/airbyte/pull/29125) | Migrate Confluence Source Connector to Low Code                |
 | 0.1.3   | 2023-03-13 | [23988](https://github.com/airbytehq/airbyte/pull/23988) | Add view and storage to pages body, add check for stream Audit |
 | 0.1.2   | 2023-03-06 | [23775](https://github.com/airbytehq/airbyte/pull/23775) | Set additionalProperties: true, update docs and spec           |
 | 0.1.1   | 2022-01-31 | [9831](https://github.com/airbytehq/airbyte/pull/9831)   | Fix: Spec was not pushed to cache                              |
-| 0.1.0   | 2021-11-05 | [7241](https://github.com/airbytehq/airbyte/pull/7241)   | 🎉 New Source: Confluence                                      |
\ No newline at end of file
+| 0.1.0   | 2021-11-05 | [7241](https://github.com/airbytehq/airbyte/pull/7241)   | 🎉 New Source: Confluence                                      |
diff --git a/docs/integrations/sources/convex.md b/docs/integrations/sources/convex.md
index c1a4796c7784..c0dd127574aa 100644
--- a/docs/integrations/sources/convex.md
+++ b/docs/integrations/sources/convex.md
@@ -72,5 +72,6 @@ In the Data tab, you should see the tables and a sample of the data that will be
 
 | Version | Date       | Pull Request                                             | Subject               |
 | :------ | :--------- | :------------------------------------------------------- | :-------------------- |
+| 0.2.0 | 2023-06-21 | [27226](https://github.com/airbytehq/airbyte/pull/27226) | 🐛 Convex source fix skipped records |
 | 0.1.1 | 2023-03-06 | [23797](https://github.com/airbytehq/airbyte/pull/23797) | 🐛 Convex source connector error messages |
-| 0.1.0   | 2022-10-24 | [18403](https://github.com/airbytehq/airbyte/pull/18403) | 🎉 New Source: Convex |
+| 0.1.0 | 2022-10-24 | [18403](https://github.com/airbytehq/airbyte/pull/18403) | 🎉 New Source: Convex |
diff --git a/docs/integrations/sources/copper.md b/docs/integrations/sources/copper.md
index f87eba5d6f1a..4b5827c5b117 100644
--- a/docs/integrations/sources/copper.md
+++ b/docs/integrations/sources/copper.md
@@ -40,5 +40,6 @@ The Copper source connector supports the following [sync modes](https://docs.air
 
 | Version | Date       | Pull Request                                              | Subject                            |
 | :------ | :--------- | :-------------------------------------------------------- | :--------------------------------- |
+| 0.2.1   | 2023-08-16 | [24824](https://github.com/airbytehq/airbyte/pull/24824) | Fix schemas and tests  |
 | 0.2.0   | 2023-04-17 | [24824](https://github.com/airbytehq/airbyte/pull/24824) | Add `opportunities` stream |
 | 0.1.0   | 2022-11-17 | [18848](https://github.com/airbytehq/airbyte/pull/18848) | 🎉 New Source: Copper [python cdk] |
diff --git a/docs/integrations/sources/datadog.md b/docs/integrations/sources/datadog.md
index 60f77bdaed66..c2799e5cf1e0 100644
--- a/docs/integrations/sources/datadog.md
+++ b/docs/integrations/sources/datadog.md
@@ -21,7 +21,11 @@ An API key is required as well as an API application key. See the [Datadog API a
 7. Enter your `limit` - Number of records to collect per request.
 8. Enter your `start_date` - Optional. Start date to filter records when collecting data from Logs and AuditLogs stream.
 9. Enter your `end_date` - Optional. End date to filter records when collecting data from Logs and AuditLogs stream.
-10. Click **Set up source**.
+10. Enter your `queries` - Optional. Multiple queries resulting in multiple streams.
+    1. Enter the `name`- Optional. Query Name.
+    2. Select the `data_source` from dropdown - Optional. Supported data sources - metrics, cloud_cost, logs, rum.
+    3. Enter the `query`- Optional. A classic query string. Example - _"kubernetes_state.node.count{*}"_, _"@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)"_
+11. Click **Set up source**.
 
 ### For Airbyte OSS:
 
@@ -33,6 +37,10 @@ An API key is required as well as an API application key. See the [Datadog API a
 7. Enter your `limit` - Number of records to collect per request.
 8. Enter your `start_date` - Optional. Start date to filter records when collecting data from Logs and AuditLogs stream.
 9. Enter your `end_date` - Optional. End date to filter records when collecting data from Logs and AuditLogs stream.
+10. Enter your `queries` - Optional. Multiple queries resulting in multiple streams.
+    1. Enter the `name`- Required. Query Name.
+    2. Select the `data_source` - Required. Supported data sources - metrics, cloud_cost, logs, rum.
+    3. Enter the `query`- Required. A classic query string. Example - _"kubernetes_state.node.count{*}"_, _"@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)"_
 10. Click **Set up source**.
 
 ## Supported sync modes
@@ -57,10 +65,14 @@ The Datadog source connector supports the following [sync modes](https://docs.ai
 * [Metrics](https://docs.datadoghq.com/api/latest/metrics/#get-a-list-of-metrics)
 * [SyntheticTests](https://docs.datadoghq.com/api/latest/synthetics/#get-the-list-of-all-tests)
 * [Users](https://docs.datadoghq.com/api/latest/users/#list-all-users)
+* [Series](https://docs.datadoghq.com/api/latest/metrics/?code-lang=curl#query-timeseries-data-across-multiple-products)
 
 ## Changelog
 
-| Version | Date       | Pull Request                                              | Subject             |
-|:--------|:-----------|:----------------------------------------------------------|:--------------------|
-| 0.1.1   | 2023-04-27 | [25562](https://github.com/airbytehq/airbyte/pull/25562)  | Update testing dependencies|
-| 0.1.0   | 2022-10-18 | [18150](https://github.com/airbytehq/airbyte/pull/18150)  | New Source: Datadog |
+| Version | Date       | Pull Request                                              | Subject                                                                      |
+|:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------|
+| 0.2.2   | 2023-07-10 | [28089](https://github.com/airbytehq/airbyte/pull/28089)  | Additional stream and query details in response                              |
+| 0.2.1   | 2023-06-28 | [26534](https://github.com/airbytehq/airbyte/pull/26534)  | Support multiple query streams and pulling data from different datadog sites |
+| 0.2.0   | 2023-06-28 | [27784](https://github.com/airbytehq/airbyte/pull/27784)  | Add necessary fields to schemas                                              |
+| 0.1.1   | 2023-04-27 | [25562](https://github.com/airbytehq/airbyte/pull/25562)  | Update testing dependencies                                                  |
+| 0.1.0   | 2022-10-18 | [18150](https://github.com/airbytehq/airbyte/pull/18150)  | New Source: Datadog                                                          |
diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md
index 33460c6a113f..ff3f706265e6 100644
--- a/docs/integrations/sources/db2.md
+++ b/docs/integrations/sources/db2.md
@@ -59,25 +59,26 @@ You can also enter your own password for the keystore, but if you don't, the pas
 ## Changelog
 
 | Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.19 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting   |
-| 0.1.18 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to |
-| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources                          |
-|        | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
-| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params |
-| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
-| 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
-| 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected |
-| 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
-| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
-| 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |
-| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats |
-| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds |
-| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |****
-| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table |
-| 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables |
-| 0.1.4 | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. |
-| 0.1.3 | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation |
-| 0.1.2 | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support |
-| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |
-| 0.1.0 | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 |
+|:--------| :--- | :--- | :--- |
+| 0.1.20  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                                                  |
+| 0.1.19  | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting   |
+| 0.1.18  | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to |
+| 0.1.17  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources                          |
+|         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
+| 0.1.16  | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params |
+| 0.1.15  | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
+| 0.1.14  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
+| 0.1.13  | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected |
+| 0.1.12  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
+| 0.1.11  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
+| 0.1.10  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |
+| 0.1.9   | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats |
+| 0.1.8   | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds |
+| 0.1.7   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |****
+| 0.1.6   | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table |
+| 0.1.5   | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables |
+| 0.1.4   | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. |
+| 0.1.3   | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation |
+| 0.1.2   | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support |
+| 0.1.1   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator |
+| 0.1.0   | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 |
diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md
index f5208e1f2feb..5491a650b605 100644
--- a/docs/integrations/sources/delighted.md
+++ b/docs/integrations/sources/delighted.md
@@ -1,44 +1,58 @@
 # Delighted
 
-This page contains the setup guide and reference information for the Delighted source connector.
+This page contains the setup guide and reference information for the [Delighted](https://delighted.com/) source connector.
 
 ## Prerequisites
 
-To set up the Delighted source connector, you'll need the [Delighted API key](https://app.delighted.com/docs/api#authentication).
+- A Delighted API Key.
+- A desired start date and time. Only data added on and after this point will be replicated.
 
-## Set up the Delighted connector in Airbyte
+## Setup guide
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, enter the name for the Delighted connector and select **Delighted** from the Source type dropdown.
-4. For **Since**, enter the date in a Unix Timestamp format. The data added on and after this date will be replicated.
-5. For **API Key**, enter your [Delighted `API Key`](https://delighted.com/account/api).
-6. Click **Set up source**.
+### Step 1: Obtain a Delighted API Key
+
+To set up the Delighted source connector, you'll need a Delighted API key. For detailed instructions, please refer to the
+[official Delighted documentation](https://app.delighted.com/docs/api).
+
+### Step 2: Set up the Delighted connector in Airbyte
+
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to the Airbyte Open Source dashboard.
+2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **Delighted** from the list of available sources.
+3. Enter a **Source name** of your choosing.
+4. Enter your **Delighted API Key**.
+5. In the **Replication Start Date** field, enter the desired UTC date and time. Only the data added on and after this date will be replicated.
+
+:::note
+If you are configuring this connector programmatically, please format your date as such: `yyyy-mm-ddThh:mm:ssZ`. For example, an input of `2022-05-30T14:50:00Z` signifies a start date of May 30th, 2022 at 2:50 PM UTC. For help converting UTC to your local time,
+[use a UTC Time Zone Converter](https://dateful.com/convert/utc).
+:::
+
+6. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
-The Delighted source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+The Delighted source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
-## Supported Streams
+## Supported streams
 
-This Source is capable of syncing the following core Streams:
+This source is capable of syncing the following core streams:
 
-* [Survey Responses](https://app.delighted.com/docs/api/listing-survey-responses)
-* [People](https://app.delighted.com/docs/api/listing-people)
-* [Bounced People](https://app.delighted.com/docs/api/listing-bounced-people)
-* [Unsubscribed People](https://app.delighted.com/docs/api/listing-unsubscribed-people)
+- [Bounced People](https://app.delighted.com/docs/api/listing-bounced-people)
+- [People](https://app.delighted.com/docs/api/listing-people)
+- [Survey Responses](https://app.delighted.com/docs/api/listing-survey-responses)
+- [Unsubscribed People](https://app.delighted.com/docs/api/listing-unsubscribed-people)
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                              |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------|
-| 0.2.2   | 2023-03-09 | [23909](https://github.com/airbytehq/airbyte/pull/23909) | Updated the input config pattern to accept both `RFC3339` and `datetime string` formats in UI    |
-| 0.2.1   | 2023-02-14 | [23009](https://github.com/airbytehq/airbyte/pull/23009) |Specified date formatting in specification                                                                |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------- |
+| 0.2.2   | 2023-03-09 | [23909](https://github.com/airbytehq/airbyte/pull/23909) | Updated the input config pattern to accept both `RFC3339` and `datetime string` formats in UI        |
+| 0.2.1   | 2023-02-14 | [23009](https://github.com/airbytehq/airbyte/pull/23009) | Specified date formatting in specification                                                           |
 | 0.2.0   | 2022-11-22 | [19822](https://github.com/airbytehq/airbyte/pull/19822) | Migrate to Low code + certify to Beta                                                                |
 | 0.1.4   | 2022-06-10 | [13439](https://github.com/airbytehq/airbyte/pull/13439) | Change since parameter input to iso date                                                             |
 | 0.1.3   | 2022-01-31 | [9550](https://github.com/airbytehq/airbyte/pull/9550)   | Output only records in which cursor field is greater than the value in state for incremental streams |
diff --git a/docs/integrations/sources/dixa.md b/docs/integrations/sources/dixa.md
index c1a1b1c961bf..ced877c4a6b3 100644
--- a/docs/integrations/sources/dixa.md
+++ b/docs/integrations/sources/dixa.md
@@ -51,6 +51,7 @@ When using the connector, keep in mind that increasing the `batch_size` paramete
 
 | Version | Date       | Pull Request                                             | Subject                                                               |
 | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------- |
+| 0.2.0   | 2023-06-08 | [25103](https://github.com/airbytehq/airbyte/pull/25103) | Add fields to `conversation_export` stream                            |
 | 0.1.3   | 2022-07-07 | [14437](https://github.com/airbytehq/airbyte/pull/14437) | 🎉 Source Dixa: bump version 0.1.3                                    |
 | 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                       |
 | 0.1.1   | 2021-08-12 | [5367](https://github.com/airbytehq/airbyte/pull/5367)   | Migrated to CI Sandbox, refactorred code structure for future support |
diff --git a/docs/integrations/sources/dockerhub.md b/docs/integrations/sources/dockerhub.md
index e87706cf73b6..ae2b1f24913c 100644
--- a/docs/integrations/sources/dockerhub.md
+++ b/docs/integrations/sources/dockerhub.md
@@ -36,5 +36,7 @@ This connector has been tested for the Airbyte organization, which has 266 repos
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.2.0 | 2023-08-24 | [29320](https://github.com/airbytehq/airbyte/pull/29320) | Migrate to Low Code |
+| 0.1.1 | 2023-08-16 | [13007](https://github.com/airbytehq/airbyte/pull/13007) | Fix schema and tests |
 | 0.1.0 | 2022-05-20 | [13007](https://github.com/airbytehq/airbyte/pull/13007) | New source |
 
diff --git a/docs/integrations/sources/drift.md b/docs/integrations/sources/drift.md
index 4456ec21e9e4..02461174047e 100644
--- a/docs/integrations/sources/drift.md
+++ b/docs/integrations/sources/drift.md
@@ -49,9 +49,11 @@ The Drift connector should not run into Drift API limitations under normal usage
 
 ## CHANGELOG
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.2.6 | 2023-03-07 | [23810](https://github.com/airbytehq/airbyte/pull/23810) | Prepare for cloud |
-| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Updated titles and descriptions |
-| 0.2.3 | 2021-10-25 | [7337](https://github.com/airbytehq/airbyte/pull/7337) | Added support of `OAuth 2.0` authorisation option |
-| `0.2.3` | 2021-10-27 | [7247](https://github.com/airbytehq/airbyte/pull/7247) | Migrate to the CDK |
+| Version | Date       | Pull Request                                             | Subject                                                             |
+|:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------|
+| 0.3.0   | 2023-08-05 | [29121](https://github.com/airbytehq/airbyte/pull/29121) | Migrate Python CDK to Low Code CDK                                  |
+| 0.2.7   | 2023-06-09 | [27202](https://github.com/airbytehq/airbyte/pull/27202) | Remove authSpecification in favour of advancedAuth in specification |
+| 0.2.6   | 2023-03-07 | [23810](https://github.com/airbytehq/airbyte/pull/23810) | Prepare for cloud                                                   |
+| 0.2.5   | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Updated titles and descriptions                                     |
+| 0.2.3   | 2021-10-25 | [7337](https://github.com/airbytehq/airbyte/pull/7337)   | Added support of `OAuth 2.0` authorisation option                   |
+| 0.2.3   | 2021-10-27 | [7247](https://github.com/airbytehq/airbyte/pull/7247)   | Migrate to the CDK                                                  |
diff --git a/docs/integrations/sources/exchange-rates.inapp.md b/docs/integrations/sources/exchange-rates.inapp.md
new file mode 100644
index 000000000000..a883ff6a7054
--- /dev/null
+++ b/docs/integrations/sources/exchange-rates.inapp.md
@@ -0,0 +1,25 @@
+## Prerequisites
+
+- API Access Key
+
+In order to get a free `API Access Key` please go to [this](https://manage.exchangeratesapi.io/signup/free) page and enter the required information. After registration and login, you will see your `API Access Key`. You can also locate it [here](https://manage.exchangeratesapi.io/dashboard).
+
+If you have a `free` subscription plan, you will have two limitations to the plan: 
+
+1. Limit of 1,000 API calls per month
+2. You won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is EUR.
+
+## Setup guide
+1. Enter a **Name** for your source.
+2. Enter your **API key** as the `access_key` from the prerequisites.
+3. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. 
+4. (Optional) Enter a **base** currency. For those on the free plan, `EUR` is the only option available. If none are specified, `EUR` will be used.
+5. Click **Set up source**.
+
+### Exchange Rates data output
+- The sync will include one stream: `exchange_rates`
+- Each record in the stream contains many fields:
+  - The date of the record
+  - One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Exchange Rates](https://docs.airbyte.com/integrations/sources/exchange-rates/).
diff --git a/docs/integrations/sources/exchange-rates.md b/docs/integrations/sources/exchange-rates.md
index 71e2e53d3229..3d5f8d623131 100644
--- a/docs/integrations/sources/exchange-rates.md
+++ b/docs/integrations/sources/exchange-rates.md
@@ -46,6 +46,8 @@ If you have `free` subscription plan \(you may check it [here](https://manage.ex
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                             |
 | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------ |
+| 1.3.0   | 2023-08-25 | [29299](https://github.com/airbytehq/airbyte/pull/29299) | Migrate to low-code |
+| 1.2.9   | 2023-08-15 | [23000](https://github.com/airbytehq/airbyte/pull/23000) | Fix schema and tests  |
 | 1.2.8   | 2023-02-14 | [23000](https://github.com/airbytehq/airbyte/pull/23000) | Specified date formatting in specification                                                                          |
 | 1.2.7   | 2022-10-31 | [18726](https://github.com/airbytehq/airbyte/pull/18726) | Fix handling error during check connection                                                                          |
 | 1.2.6   | 2022-08-23 | [15884](https://github.com/airbytehq/airbyte/pull/15884) | Migrated to new API Layer endpoint                                                                                  |
diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md
index 031971b8cd11..2f8055158b75 100644
--- a/docs/integrations/sources/facebook-marketing.md
+++ b/docs/integrations/sources/facebook-marketing.md
@@ -4,109 +4,124 @@ This page guides you through the process of setting up the Facebook Marketing so
 
 ## Prerequisites
 
-* A [Facebook Ad Account ID](https://www.facebook.com/business/help/1492627900875762)
+- A [Facebook Ad Account ID](https://www.facebook.com/business/help/1492627900875762)
 
-* (For Open Source) A [Facebook App](https://developers.facebook.com/apps/) with the Marketing API enabled
+- (For Airbyte Open Source) A [Facebook app](https://developers.facebook.com/apps/) with the Marketing API enabled
 
 
 ## Setup guide
 
-
-### For Airbyte Cloud:
+
+
+### (For Airbyte Open Source) Generate an access token and request a rate limit increase:
+
+To set up Facebook Marketing as a source in Airbyte Open Source, you will first need to create a Facebook app and generate a Marketing API access token. You will then need to request a rate limit increase from Facebook. The following steps will guide you through this process:
+
+1. Navigate to [Meta for Developers](https://developers.facebook.com/apps/) and follow the steps provided in the [Facebook documentation](https://developers.facebook.com/docs/development/create-an-app/) to create a Facebook app. Set the app type to **Business** when prompted.
+2. From your App’s dashboard, [set up the Marketing API](https://developers.facebook.com/docs/marketing-apis/get-started).
+3. Generate a Marketing API access token: From your App’s Dashboard, click **Marketing API** --> **Tools**. Select all the available token permissions (`ads_management`, `ads_read`, `read_insights`, `business_management`) and click **Get token**. Copy the generated token for later use.
+4. Request a rate limit increase: Facebook [heavily throttles](https://developers.facebook.com/docs/marketing-api/overview/authorization#limits) API tokens generated from Facebook apps with the default Standard Access tier, making it infeasible to use the token for syncs with Airbyte. You'll need to request an upgrade to Advanced Access for your app on the following permissions:
+
+   - Ads Management Standard Access
+   - ads_read
+   - Ads_management
+
+   See the Facebook [documentation on Authorization](https://developers.facebook.com/docs/marketing-api/overview/authorization/#access-levels) to request Advanced Access to the relevant permissions.
+
+
 
-**To set up Facebook Marketing as a source in Airbyte Cloud:**
+### Set up Facebook Marketing as a source in Airbyte:
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account, or navigate to your Airbyte Open Source dashboard.
 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
-3. On the Set up the source page, select **Facebook Marketing** from the **Source type** dropdown.
-4. For Name, enter a name for your Facebook Marketing connector.
-
-**Facebook Marketing Source Settings:**
-
-1. Click **Authenticate your account** to authorize your [Meta for Developers](https://developers.facebook.com/) account. Airbyte will authenticate the account you are already logged in to. Make sure you are logged into the right account.
-2. Account ID: [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762): The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. Open your Meta Ads Manager. The Ad account ID number is in the account dropdown menu or in your browser's address bar. See the [docs](https://www.facebook.com/business/help/1492627900875762) for more information. 
-3. For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-
-    :::warning
-    Insight tables are only able to pull data from 37 months. If you are syncing insight tables and your start date is older than 37 months, your sync will fail.
-    :::
-
-4. For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The date until which you'd like to replicate data for all incremental streams. All data generated between the start date and this end date will be replicated. Not setting this option will result in always syncing the latest data. 
-5. For **Access Token**, if you don't use OAuth. [Generate Access Token:](https://docs.airbyte.com/integrations/sources/facebook-marketing). The value of the generated access token. From your App’s Dashboard, click on "Marketing API" then "Tools". Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". See the [docs](https://docs.airbyte.com/integrations/sources/facebook-marketing) for more information. 
-6. For Account ID, enter your [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762): The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. Open your Meta Ads Manager. The Ad account ID number is in the account dropdown menu or in your browser's address bar. See the [docs](https://www.facebook.com/business/help/1492627900875762) for more information.
-7. (Optional) Toggle the **Include Deleted** button to include data from deleted Campaigns, Ads, and AdSets.
-
-    :::info
-    The Facebook Marketing API does not have a concept of deleting records in the same way that a database does. While you can archive or delete an ad campaign, the API maintains a record of the campaign. Toggling the **Include Deleted** button lets you replicate records for campaigns or ads even if they were archived or deleted from the Facebook platform.
-    :::
-
-8. (Optional) Toggle the **Fetch Thumbnail Images** button to fetch the `thumbnail_url` and store the result in `thumbnail_data_url` for each [Ad Creative](https://developers.facebook.com/docs/marketing-api/creative/).
-9. (Optional) In the Custom Insights section. A list which contains ad statistics entries, each entry must have a name and can contain fields, breakdowns or action_breakdowns. Click on "add" to fill this field.
-    To retrieve specific fields from Facebook Ads Insights combined with other breakdowns, you can choose which fields and breakdowns to sync.
-    We recommend following the Facebook Marketing [documentation](https://developers.facebook.com/docs/marketing-api/insights/breakdowns) to understand the breakdown limitations. Some fields can not be requested and many others only work when combined with specific fields. For example, the breakdown `app_id` is only supported with the `total_postbacks` field.
-
-    To configure Custom Insights:
-
-       1. For **Name**, enter a name for the insight. This will be used as the Airbyte stream name 
-       2. For **Level**, enter the level of the fields you want to pull from the Facebook Marketing API. By default, 'ad'. You can specify also account, campaign or adset. 
-       3. For **Fields**, enter a list of the fields you want to pull from the Facebook Marketing API.
-       4. For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and before this date will be replicated. If this field is blank, Airbyte will replicate the latest data.
-       5. For **Breakdowns**, enter a list of the breakdowns you want to configure.
-       6. For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-       7. For **Action Breakdown**, enter a list of the action breakdowns you want to configure.
-       8. For **Custom Insights Lookback Window**, fill in the appropriate value. See [more](#facebook-marketing-attribution-reporting) on this parameter.
-       9. Click **Done**.
-      10.  For **Page Size of Requests**, fill in the page size in case pagination kicks in. Feel free to ignore it, the default value should work in most cases. 
-      12.  For **Insights Lookback Window**, fill in the appropriate value. Facebook freezes insight data 28 days after it was generated, which means that all data from the past 28 days may have changed since we last emitted it, so you can retrieve refreshed insights from the past by setting this parameter. If you set a custom lookback window value in Facebook account, please provide the same value here. See [more](#facebook-marketing-attribution-reporting) on this parameter. 
-      13.  Click **Set up source**.
-
-    :::warning
-    Additional streams for Facebook Marketing are dynamically created based on the specified Custom Insights. For an existing Facebook Marketing source, when you are updating or removing Custom Insights, you should also ensure that any connections syncing to these streams are either disabled or have had their source schema refreshed.
-    :::
-
-
+3. Find and select **Facebook Marketing** from the list of available sources.
+4. For **Source name**, enter a name for your Facebook Marketing connector.
+5. To authenticate the connection:
 
-
-**For Airbyte Open Source:**
+   
+   **For Airbyte Cloud**: Click **Authenticate your account** to authorize your Facebook account. Make sure you are logged into the right account, as Airbyte will authenticate the account you are currently logged in to.
+   
+   
+   **For Airbyte Open Source**: In the **Access Token** field, enter the access token you generated with your Facebook app.
+   
 
-To set up Facebook Marketing as a source in Airbyte Open Source:
+#### Facebook Marketing Source Settings:
 
-1. Navigate to [Meta for Developers](https://developers.facebook.com/apps/) and [create an app](https://developers.facebook.com/docs/development/create-an-app/) with the app type Business.
-2. From your App’s dashboard, [setup the Marketing API](https://developers.facebook.com/docs/marketing-apis/get-started).
-3. Generate a Marketing API access token: From your App’s Dashboard, click **Marketing API** --> **Tools**. Select all the available token permissions (`ads_management`, `ads_read`, `read_insights`, `business_management`) and click **Get token**. Copy the generated token for later use.
-4. Request a rate increase limit: Facebook [heavily throttles](https://developers.facebook.com/docs/marketing-api/overview/authorization#limits) API tokens generated from Facebook Apps with the "Standard Access" tier (the default tier for new apps), making it infeasible to use the token for syncs with Airbyte. You'll need to request an upgrade to Advanced Access for your app on the following permissions:
+1. For **Account ID**, enter the [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information.
+2. For **Start Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate all data.
 
-    * Ads Management Standard Access
-    * ads_read
-    * Ads_management
+   :::warning
+   Insight tables are only able to pull data from the last 37 months. If you are syncing insight tables and your start date is older than 37 months, your sync will fail.
+   :::
 
-    See the Facebook [documentation on Authorization](https://developers.facebook.com/docs/marketing-api/overview/authorization/#access-levels) to request Advanced Access to the relevant permissions.
-5. Navigate to the Airbyte Open Source Dashboard. Add the access token when prompted to do so and follow the same instructions as for [setting up the Facebook Connector on Airbyte Cloud](#for-airbyte-cloud).
-
+3. (Optional) For **End Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. This is the date until which you'd like to replicate data for all Incremental streams. All data generated between the start date and this end date will be replicated. Not setting this option will result in always syncing the latest data.
+4. (Optional) Toggle the **Include Deleted Campaigns, Ads, and AdSets** button to include data from deleted Campaigns, Ads, and AdSets.
 
-## Supported sync modes
+   :::info
+   The Facebook Marketing API does not have a concept of deleting records in the same way that a database does. While you can archive or delete an ad campaign, the API maintains a record of the campaign. Toggling the **Include Deleted** button lets you replicate records for campaigns or ads even if they were archived or deleted from the Facebook platform.
+   :::
 
-The Facebook Marketing source connector supports the following sync modes:
+5. (Optional) Toggle the **Fetch Thumbnail Images** button to fetch the `thumbnail_url` and store the result in `thumbnail_data_url` for each [Ad Creative](https://developers.facebook.com/docs/marketing-api/creative/).
+6. (Optional) In the **Custom Insights** section, you may provide a list of ad statistics entries. Each entry should have a unique name and can contain fields, breakdowns or action_breakdowns. Fields refer to the different data points you can collect from an ad, while breakdowns and action_breakdowns let you segment this data for more detailed insights. Click on **Add** to create a new entry in this list.
+
+   :::note
+   To retrieve specific fields from Facebook Ads Insights combined with other breakdowns, you can choose which fields and breakdowns to sync. However, please note that not all fields can be requested, and many are only functional when combined with specific other fields. For example, the breakdown `app_id` is only supported with the `total_postbacks` field. For more information on the breakdown limitations, refer to the [Facebook documentation](https://developers.facebook.com/docs/marketing-api/insights/breakdowns).
+   :::
+
+   To configure Custom Insights:
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) (except for the AdCreatives and AdAccount tables)
-* [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) (except for the AdCreatives and AdAccount tables)
+    1. For **Name**, enter a name for the insight. This will be used as the Airbyte stream name.
+    2. (Optional) For **Level**, enter the level of granularity for the data you want to pull from the Facebook Marketing API (`account`, `ad`, `adset`, `campaign`). Set to `ad` by default.
+    3. (Optional) For **Fields**, use the dropdown list to select the fields you want to pull from the Facebook Marketing API.
+    4. (Optional) For **Breakdowns**, use the dropdown list to select the breakdowns you want to configure.
+    5. (Optional) For **Action Breakdowns**, use the dropdown list to select the action breakdowns you want to configure.
+    6. (Optional) For **Action Report Time**, enter the action report time you want to configure. This value determines the timing used to report action statistics. For example, if a user sees an ad on Jan 1st but converts on Jan 2nd, this value will determine how the action is reported.
 
-## Supported Streams
+        - `impression`: Actions are attributed to the time the ad was viewed (Jan 1st).
+        - `conversion`: Actions are attributed to the time the action was taken (Jan 2nd).
+        - `mixed`: Click-through actions are attributed to the time the ad was viewed (Jan 1st), and view-through actions are attributed to the time the action was taken (Jan 2nd).
 
-* [Activities](https://developers.facebook.com/docs/marketing-api/reference/ad-activity)
-* [AdAccount](https://developers.facebook.com/docs/marketing-api/business-asset-management/guides/ad-accounts)
-* [AdCreatives](https://developers.facebook.com/docs/marketing-api/reference/ad-creative#fields)
-* [AdSets](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign#fields)
-* [Ads](https://developers.facebook.com/docs/marketing-api/reference/adgroup#fields)
-* [AdInsights](https://developers.facebook.com/docs/marketing-api/reference/adgroup/insights/)
-* [Campaigns](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group#fields)
-* [CustomConversions](https://developers.facebook.com/docs/marketing-api/reference/custom-conversion)
-* [Images](https://developers.facebook.com/docs/marketing-api/reference/ad-image)
-* [Videos](https://developers.facebook.com/docs/marketing-api/reference/video)
+    7. (Optional) For **Time Increment**, you may provide a value in days by which to aggregate statistics. The sync will be chunked into intervals of this size. For example, if you set this value to 7, the sync will be chunked into 7-day intervals. The default value is 1 day.
+    8. (Optional) For **Start Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate all data.
+    9. (Optional) For **End Date**, enter the date in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and before this date will be replicated. If this field is left blank, Airbyte will replicate the latest data.
+    10. (Optional) For **Custom Insights Lookback Window**, you may set a window in days to revisit data during syncing to capture updated conversion data from the API. Facebook allows for attribution windows of up to 28 days, during which time a conversion can be attributed to an ad. If you have set a custom attribution window in your Facebook account, please set the same value here. Otherwise, you may leave it at the default value of 28. For more information on action attributions, please refer to [the Meta Help Center](https://www.facebook.com/business/help/458681590974355?id=768381033531365).
 
-Airbyte also supports the following Prebuilt Facebook AdInsights Reports:
+   :::warning
+   Additional data streams for your Facebook Marketing connector are dynamically generated according to the Custom Insights you specify. If you have an existing Facebook Marketing source and you decide to update or remove some of your Custom Insights, you must also adjust the connections that sync to these streams. Specifically, you should either disable these connections or refresh the source schema associated with them to reflect the changes.
+   :::
+
+7. (Optional) For **Page Size of Requests**, you can specify the number of records per page for paginated responses. Most users do not need to set this field unless specific issues arise or there are unique use cases that require tuning the connector's settings. The default value is set to retrieve 100 records per page.
+8. (Optional) For **Insights Window Lookback**, you may set a window in days to revisit data during syncing to capture updated conversion data from the API. Facebook allows for attribution windows of up to 28 days, during which time a conversion can be attributed to an ad. If you have set a custom attribution window in your Facebook account, please set the same value here. Otherwise, you may leave it at the default value of 28. For more information on action attributions, please refer to [the Meta Help Center](https://www.facebook.com/business/help/458681590974355?id=768381033531365).
+9. (Optional) You can set a **Maximum size of Batched Requests** for the connector. This is the maximum number of records that will be sent in a single request to the Facebook Marketing API. Most users do not need to configure this field, unless specific issues arise of there are unique use cases that require tuning the connector's settings. The maximum number of requests per batch allowed by the API is 50. More information on this topic can be found in the [Facebook documentation](https://developers.facebook.com/docs/graph-api/batch-requests).
+10. Click **Set up source** and wait for the tests to complete.
+
+## Supported sync modes
+
+The Facebook Marketing source connector supports the following sync modes:
+
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) (except for the AdCreatives and AdAccount tables)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) (except for the AdCreatives and AdAccount tables)
+
+## Supported streams
+
+- [Activities](https://developers.facebook.com/docs/marketing-api/reference/ad-activity)
+- [AdAccount](https://developers.facebook.com/docs/marketing-api/business-asset-management/guides/ad-accounts)
+- [AdCreatives](https://developers.facebook.com/docs/marketing-api/reference/ad-creative#fields)
+- [AdSets](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign#fields)
+- [Ads](https://developers.facebook.com/docs/marketing-api/reference/adgroup#fields)
+- [AdInsights](https://developers.facebook.com/docs/marketing-api/reference/adgroup/insights/)
+- [Campaigns](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group#fields)
+- [CustomConversions](https://developers.facebook.com/docs/marketing-api/reference/custom-conversion)
+- [CustomAudiences](https://developers.facebook.com/docs/marketing-api/reference/custom-audience)
+:::caution CustomAudiences
+The `rule` field may not be synced for all records because it caused the error message `Please reduce the amount of data...`.
+:::
+- [Images](https://developers.facebook.com/docs/marketing-api/reference/ad-image)
+- [Videos](https://developers.facebook.com/docs/marketing-api/reference/video)
+
+Airbyte also supports the following Prebuilt Facebook Ad Insights Reports:
 
 | Stream                                            |                           Breakdowns                           |                    Action Breakdowns                    |
 |:--------------------------------------------------|:--------------------------------------------------------------:|:-------------------------------------------------------:|
@@ -130,32 +145,30 @@ Airbyte also supports the following Prebuilt Facebook AdInsights Reports:
 | Ad Insights Platform And Device                   | `publisher_platform`, `platform_position`, `impression_device` |                      `action_type`                      |
 | Ad Insights Region                                |                            `region`                            | `action_type`, `action_target_id`, `action_destination` |
 
+You can segment the Ad Insights table into parts based on the following information. Each part will be synced as a separate table if normalization is enabled:
 
-
-You can segment the AdInsights table into parts based on the following information. Each part will be synced as a separate table if normalization is enabled:
-
-* Country
-* DMA (Designated Market Area)
-* Gender & Age
-* Platform & Device
-* Region
+- Country
+- DMA (Designated Market Area)
+- Gender & Age
+- Platform & Device
+- Region
 
 For more information, see the [Facebook Insights API documentation.](https://developers.facebook.com/docs/marketing-api/reference/adgroup/insights/)
 
-:::note
-
-Pay attention, that not all fields (e.g. conversions, conversion_values) will be returned for AdInsights, see [docs](https://developers.facebook.com/docs/marketing-api/reference/ads-action-stats/).
-To get all fields You should use custom insights with **breakdowns**.
+
 
 ## Facebook Marketing Attribution Reporting
-Please be informed that the connector uses the `lookback_window` parameter to perform the repetitive read of the last `` days in the Incremental sync mode. This means some data will be synced twice (or possibly more often) despite the cursor value being up-to-date. You can change this date window by modifying the `lookback_window` parameter when setting up the source. The smaller the value - the fewer duplicates you will have. The greater the value - the more precise results you will get. More details on what the attribution window is and what purpose it serves can be found in this [Facebook Article](https://www.facebook.com/business/help/458681590974355?id=768381033531365).
+
+The Facebook Marketing connector uses the `lookback_window` parameter to repeatedly read data from the last `` days during an Incremental sync. This means some data will be synced twice (or possibly more often) despite the cursor value being up to date, in order to capture updated ads conversion data from Facebook. You can change this date window by adjusting the `lookback_window` parameter when setting up the source, up to a maximum of 28 days. Smaller values will result in fewer duplicates, while larger values provide more accurate results. For a deeper understanding of the purpose and role of the attribution window, refer to this [Meta article](https://www.facebook.com/business/help/458681590974355?id=768381033531365).
 
 ## Data type mapping
 
 | Integration Type | Airbyte Type |
-|:----------------:|:------------:|
+| :--------------: | :----------: |
 |      string      |    string    |
 |      number      |    number    |
 |      array       |    array     |
@@ -165,6 +178,20 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.1.7   | 2023-08-21 | [29674](https://github.com/airbytehq/airbyte/pull/29674) | Exclude `rule` from stream `CustomAudiences`                                                                                                                                                                                                                                                      |
+| 1.1.6   | 2023-08-18 | [29642](https://github.com/airbytehq/airbyte/pull/29642) | Stop batch requests if only 1 left in a batch                                                                                                                                                                                                                                                     |
+| 1.1.5   | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size                                                                                                                                                                                                                                                                   |
+| 1.1.4   | 2023-08-08 | [29412](https://github.com/airbytehq/airbyte/pull/29412) | Add new custom_audience stream                                                                                                                                                                                                                                                                    |
+| 1.1.3   | 2023-08-08 | [29208](https://github.com/airbytehq/airbyte/pull/29208) | Add account type validation during check                                                                                                                                                                                                                                                          |
+| 1.1.2   | 2023-08-03 | [29042](https://github.com/airbytehq/airbyte/pull/29042) | Fix broken `advancedAuth` references for `spec`                                                                                                                                                                                                                                                   |
+| 1.1.1   | 2023-07-26 | [27996](https://github.com/airbytehq/airbyte/pull/27996) | Remove reference to authSpecification                                                                                                                                                                                                                                                             |
+| 1.1.0   | 2023-07-11 | [26345](https://github.com/airbytehq/airbyte/pull/26345) | Add new `action_report_time` attribute to `AdInsights` class                                                                                                                                                                                                                                      |
+| 1.0.1   | 2023-07-07 | [27979](https://github.com/airbytehq/airbyte/pull/27979) | Added the ability to restore the reduced request record limit after the successful retry, and handle the `unknown error`  (code 99) with the retry strategy                                                                                                                                       |
+| 1.0.0   | 2023-07-05 | [27563](https://github.com/airbytehq/airbyte/pull/27563) | Migrate to FB SDK version 17                                                                                                                                                                                                                                                                      |
+| 0.5.0   | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2                                                                                                                                                                                                                                                                              |
+| 0.4.3   | 2023-05-12 | [27483](https://github.com/airbytehq/airbyte/pull/27483) | Reduce replication start date by one more day                                                                                                                                                                                                                                                     |
+| 0.4.2   | 2023-06-09 | [27201](https://github.com/airbytehq/airbyte/pull/27201) | Add `complete_oauth_server_output_specification` to spec                                                                                                                                                                                                                                          |
+| 0.4.1   | 2023-06-02 | [26941](https://github.com/airbytehq/airbyte/pull/26941) | Remove `authSpecification` from spec.json, use `advanced_auth` instead                                                                                                                                                                                                                            |
 | 0.4.0   | 2023-05-29 | [26720](https://github.com/airbytehq/airbyte/pull/26720) | Add Prebuilt Ad Insights reports                                                                                                                                                                                                                                                                  |
 | 0.3.7   | 2023-05-12 | [26000](https://github.com/airbytehq/airbyte/pull/26000) | Handle config errors                                                                                                                                                                                                                                                                              |
 | 0.3.6   | 2023-04-27 | [22999](https://github.com/airbytehq/airbyte/pull/22999) | Specified date formatting in specification                                                                                                                                                                                                                                                        |
@@ -182,13 +209,13 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 | 0.2.81  | 2023-01-05 | [21057](https://github.com/airbytehq/airbyte/pull/21057) | Remove unsupported fields from request                                                                                                                                                                                                                                                            |
 | 0.2.80  | 2022-12-21 | [20736](https://github.com/airbytehq/airbyte/pull/20736) | Fix update next cursor                                                                                                                                                                                                                                                                            |
 | 0.2.79  | 2022-12-07 | [20402](https://github.com/airbytehq/airbyte/pull/20402) | Exclude Not supported fields from request                                                                                                                                                                                                                                                         |
-| 0.2.78  | 2022-12-07 | [20165](https://github.com/airbytehq/airbyte/pull/20165) | fix fields permission error                                                                                                                                                                                                                                                                       |
-| 0.2.77  | 2022-12-06 | [20131](https://github.com/airbytehq/airbyte/pull/20131) | update next cursor value at read start                                                                                                                                                                                                                                                            |
+| 0.2.78  | 2022-12-07 | [20165](https://github.com/airbytehq/airbyte/pull/20165) | Fix fields permission error                                                                                                                                                                                                                                                                       |
+| 0.2.77  | 2022-12-06 | [20131](https://github.com/airbytehq/airbyte/pull/20131) | Update next cursor value at read start                                                                                                                                                                                                                                                            |
 | 0.2.76  | 2022-12-03 | [20043](https://github.com/airbytehq/airbyte/pull/20043) | Allows `action_breakdowns` to be an empty list - bugfix for #20016                                                                                                                                                                                                                                |
 | 0.2.75  | 2022-12-03 | [20016](https://github.com/airbytehq/airbyte/pull/20016) | Allows `action_breakdowns` to be an empty list                                                                                                                                                                                                                                                    |
 | 0.2.74  | 2022-11-25 | [19803](https://github.com/airbytehq/airbyte/pull/19803) | New default for `action_breakdowns`, improve "check" command speed                                                                                                                                                                                                                                |
 | 0.2.73  | 2022-11-21 | [19645](https://github.com/airbytehq/airbyte/pull/19645) | Check "breakdowns" combinations                                                                                                                                                                                                                                                                   |
-| 0.2.72  | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | handle FacebookBadObjectError for empty results on async jobs                                                                                                                                                                                                                                     |
+| 0.2.72  | 2022-11-04 | [18971](https://github.com/airbytehq/airbyte/pull/18971) | Handle FacebookBadObjectError for empty results on async jobs                                                                                                                                                                                                                                     |
 | 0.2.71  | 2022-10-31 | [18734](https://github.com/airbytehq/airbyte/pull/18734) | Reduce request record limit on retry                                                                                                                                                                                                                                                              |
 | 0.2.70  | 2022-10-26 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Upgrade FB SDK to v15.0                                                                                                                                                                                                                                                                           |
 | 0.2.69  | 2022-10-17 | [18045](https://github.com/airbytehq/airbyte/pull/18045) | Remove "pixel" field from the Custom Conversions stream schema                                                                                                                                                                                                                                    |
@@ -205,7 +232,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 | 0.2.58  | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue                                                                                                                                                                                                    |
 | 0.2.57  | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0                                                                                                                                                                                                                                                             |
 | 0.2.56  | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation                                                                                                                                                                                                                                                 |
-| 0.2.55  | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad\_account stream                                                                                                                                                           |
+| 0.2.55  | 2022-07-18 | [14786](https://github.com/airbytehq/airbyte/pull/14786) | Check if the authorized user has the "MANAGE" task permission when getting the `funding_source_details` field in the ad_account stream                                                                                                                                                            |
 | 0.2.54  | 2022-06-29 | [14267](https://github.com/airbytehq/airbyte/pull/14267) | Make MAX_BATCH_SIZE available in config                                                                                                                                                                                                                                                           |
 | 0.2.53  | 2022-06-16 | [13623](https://github.com/airbytehq/airbyte/pull/13623) | Add fields `bid_amount` `bid_strategy` `bid_constraints` to `ads_set` stream                                                                                                                                                                                                                      |
 | 0.2.52  | 2022-06-14 | [13749](https://github.com/airbytehq/airbyte/pull/13749) | Fix the `not syncing any data` issue                                                                                                                                                                                                                                                              |
@@ -240,7 +267,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 | 0.2.23  | 2021-11-08 | [7734](https://github.com/airbytehq/airbyte/pull/7734)   | Resolve $ref field for discover schema                                                                                                                                                                                                                                                            |
 | 0.2.22  | 2021-11-05 | [7605](https://github.com/airbytehq/airbyte/pull/7605)   | Add job retry logics to AdsInsights stream                                                                                                                                                                                                                                                        |
 | 0.2.21  | 2021-10-05 | [4864](https://github.com/airbytehq/airbyte/pull/4864)   | Update insights streams with custom entries for fields, breakdowns and action_breakdowns                                                                                                                                                                                                          |
-| 0.2.20  | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719)   | Update version of facebook\_business package to 12.0                                                                                                                                                                                                                                              |
+| 0.2.20  | 2021-10-04 | [6719](https://github.com/airbytehq/airbyte/pull/6719)   | Update version of facebook_business package to 12.0                                                                                                                                                                                                                                               |
 | 0.2.19  | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438)   | Annotate Oauth2 flow initialization parameters in connector specification                                                                                                                                                                                                                         |
 | 0.2.18  | 2021-09-28 | [6499](https://github.com/airbytehq/airbyte/pull/6499)   | Fix field values converting fail                                                                                                                                                                                                                                                                  |
 | 0.2.17  | 2021-09-14 | [4978](https://github.com/airbytehq/airbyte/pull/4978)   | Convert values' types according to schema types                                                                                                                                                                                                                                                   |
@@ -248,7 +275,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 | 0.2.15  | 2021-09-14 | [5958](https://github.com/airbytehq/airbyte/pull/5958)   | Fix url parsing and add report that exposes conversions                                                                                                                                                                                                                                           |
 | 0.2.14  | 2021-07-19 | [4820](https://github.com/airbytehq/airbyte/pull/4820)   | Improve the rate limit management                                                                                                                                                                                                                                                                 |
 | 0.2.12  | 2021-06-20 | [3743](https://github.com/airbytehq/airbyte/pull/3743)   | Refactor connector to use CDK: - Improve error handling. - Improve async job performance \(insights\). - Add new configuration parameter `insights_days_per_job`. - Rename stream `adsets` to `ad_sets`. - Refactor schema logic for insights, allowing to configure any possible insight stream. |
-| 0.2.10  | 2021-06-16 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Update version of facebook\_business to 11.0                                                                                                                                                                                                                                                      |
+| 0.2.10  | 2021-06-16 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Update version of facebook_business to 11.0                                                                                                                                                                                                                                                       |
 | 0.2.9   | 2021-06-10 | [3996](https://github.com/airbytehq/airbyte/pull/3996)   | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support                                                                                                                                                                                                                                                   |
 | 0.2.8   | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Add 80000 as a rate-limiting error code                                                                                                                                                                                                                                                           |
 | 0.2.7   | 2021-06-03 | [3646](https://github.com/airbytehq/airbyte/pull/3646)   | Add missing fields to AdInsights streams                                                                                                                                                                                                                                                          |
diff --git a/docs/integrations/sources/facebook-pages.md b/docs/integrations/sources/facebook-pages.md
index a6a56b8b1dac..4f60c987df4d 100644
--- a/docs/integrations/sources/facebook-pages.md
+++ b/docs/integrations/sources/facebook-pages.md
@@ -11,15 +11,16 @@ The Facebook Pages souce connector is currently only compatible with v15 of the
 :::
 
 ## Setup guide
+
 ### Step 1: Set up Facebook Pages
 
 1. Create Facebook Developer Account. Follow [instruction](https://developers.facebook.com/async/registration/) to create one.
 2. Create [Facebook App](https://developers.facebook.com/apps/). Choose "Company" as the purpose of the app. Fill out the remaining fields to create your app, then follow along the "Connect a User Page" section.
 3. Connect a User [Page](https://developers.facebook.com/tools/explorer/). Choose your app at `Meta App` field. Choose your Page at `User or Page` field. Add next permission:
-   * pages\_read\_engagement
-   * pages\_read\_user\_content 
-   * pages\_show\_list
-   * read\_insights
+   - pages_read_engagement
+   - pages_read_user_content
+   - pages_show_list
+   - read_insights
 4. Click Generate Access Token and follow instructions.
 
 After all the steps, it should look something like this
@@ -40,8 +41,9 @@ After all the steps, it should look something like this
 5. Fill in Page ID (if you have a page URL such as `https://www.facebook.com/Test-1111111111`, the ID would be`Test-1111111111`)
 
 ### For Airbyte OSS:
+
 1. Navigate to the Airbyte Open Source dashboard.
-2. Set the name for your source. 
+2. Set the name for your source.
 3. On the Set up the source page, enter the name for the Facebook Pages connector and select **Facebook Pages** from the Source type dropdown.
 4. Fill in Page Access Token with Long-Lived Page Token
 5. Fill in Page ID (if you have a page URL such as `https://www.facebook.com/Test-1111111111`, the ID would be`Test-1111111111`)
@@ -49,40 +51,39 @@ After all the steps, it should look something like this
 ## Supported sync modes
 
 The Facebook Pages source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 
 ## Supported Streams
 
-* [Page](https://developers.facebook.com/docs/graph-api/reference/v15.0/page/#overview)
-* [Post](https://developers.facebook.com/docs/graph-api/reference/v15.0/page/feed)
-* [Page Insights](https://developers.facebook.com/docs/graph-api/reference/v15.0/page/insights)
-* [Post Insights](https://developers.facebook.com/docs/graph-api/reference/v15.0/insights)
+- [Page](https://developers.facebook.com/docs/graph-api/reference/v15.0/page/#overview)
+- [Post](https://developers.facebook.com/docs/graph-api/reference/v15.0/page/feed)
+- [Page Insights](https://developers.facebook.com/docs/graph-api/reference/v15.0/page/insights)
+- [Post Insights](https://developers.facebook.com/docs/graph-api/reference/v15.0/insights)
 
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `number`         | `number`     |       |
 | `array`          | `array`      |       |
 | `object`         | `object`     |       |
 
-
-
 ## Performance considerations
 
 Facebook heavily throttles API tokens generated from Facebook Apps by default, making it infeasible to use such a token for syncs with Airbyte. To be able to use this connector without your syncs taking days due to rate limiting follow the instructions in the Setup Guide below to access better rate limits.
 
 See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting) for more information on requesting a quota upgrade.
 
-
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                   |
-|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------|
-| 0.2.4   | 2023-04-13 | [25143](https://github.com/airbytehq/airbyte/pull/25143) | Update insight metrics request params                                                 |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------ |
+| 0.3.0   | 2023-06-26 | [27728](https://github.com/airbytehq/airbyte/pull/27728) | License Update: Elv2                                                      |
+| 0.2.5   | 2023-04-13 | [26939](https://github.com/airbytehq/airbyte/pull/26939) | Add advancedAuth to the connector spec                                    |
+| 0.2.4   | 2023-04-13 | [25143](https://github.com/airbytehq/airbyte/pull/25143) | Update insight metrics request params                                     |
 | 0.2.3   | 2023-02-23 | [23395](https://github.com/airbytehq/airbyte/pull/23395) | Parse datetime to rfc3339                                                 |
 | 0.2.2   | 2023-02-10 | [22804](https://github.com/airbytehq/airbyte/pull/22804) | Retry 500 errors                                                          |
 | 0.2.1   | 2022-12-29 | [20925](https://github.com/airbytehq/airbyte/pull/20925) | Fix tests; modify expected records                                        |
diff --git a/docs/integrations/sources/faker-migrations.md b/docs/integrations/sources/faker-migrations.md
new file mode 100644
index 000000000000..46dc3247f93f
--- /dev/null
+++ b/docs/integrations/sources/faker-migrations.md
@@ -0,0 +1,8 @@
+# Sample Data (Faker) Migration Guide
+
+## Upgrading to 5.0.0
+Some columns are narrowing from `number` to `integer`. You may need to force normalization to rebuild your destination tables by manually dropping the SCD and final tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset.
+
+## Upgrading to 4.0.0
+
+Nothing to do here - this was a test breaking change.
diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md
index 77d4d759eab7..8e2feb4f99b1 100644
--- a/docs/integrations/sources/faker.md
+++ b/docs/integrations/sources/faker.md
@@ -10,27 +10,42 @@ This source will generate an "e-commerce-like" dataset with users, products, and
 
 ```sql
 CREATE TABLE "public"."users" (
-    "id" float8,
-    "age" int8,
-    "name" text,
-    "email" text,
-    "title" text,
+    "address" jsonb,
+    "occupation" text,
     "gender" text,
-    "height" text,
+    "academic_degree" text,
     "weight" int8,
+    "created_at" timestamptz,
     "language" text,
     "telephone" text,
-    "blood_type" text,
-    "created_at" timestamptz,
-    "occupation" text,
+    "title" text,
     "updated_at" timestamptz,
     "nationality" text,
-    "academic_degree" text,
+    "blood_type" text,
+    "name" text,
+    "id" float8,
+    "age" int8,
+    "email" text,
+    "height" text,
     -- "_airbyte_ab_id" varchar,
     -- "_airbyte_emitted_at" timestamptz,
     -- "_airbyte_normalized_at" timestamptz,
-    -- "_airbyte_dev_users_hashid" text,
-    -- "_airbyte_unique_key" text
+    -- "_airbyte_users_hashid" text
+);
+
+CREATE TABLE "public"."users_address" (
+    "_airbyte_users_hashid" text,
+    "country_code" text,
+    "province" text,
+    "city" text,
+    "street_number" text,
+    "state" text,
+    "postal_code" text,
+    "street_name" text,
+    -- "_airbyte_ab_id" varchar,
+    -- "_airbyte_emitted_at" timestamptz,
+    -- "_airbyte_normalized_at" timestamptz,
+    -- "_airbyte_address_hashid" text
 );
 
 CREATE TABLE "public"."products" (
@@ -44,21 +59,19 @@ CREATE TABLE "public"."products" (
     -- "_airbyte_emitted_at" timestamptz,
     -- "_airbyte_normalized_at" timestamptz,
     -- "_airbyte_dev_products_hashid" text,
-    -- "_airbyte_unique_key" text
 );
 
 CREATE TABLE "public"."purchases" (
     "id" float8,
     "user_id" float8,
     "product_id" float8,
-    "returned_at" timestamptz,
     "purchased_at" timestamptz,
     "added_to_cart_at" timestamptz,
+    "returned_at" timestamptz,
     -- "_airbyte_ab_id" varchar,
     -- "_airbyte_emitted_at" timestamptz,
     -- "_airbyte_normalized_at" timestamptz,
     -- "_airbyte_dev_purchases_hashid" text,
-    -- "_airbyte_unique_key" text
 );
 
 ```
@@ -82,11 +95,16 @@ None!
 ## Changelog
 
 | Version | Date       | Pull Request                                                                                                          | Subject                                                                                                         |
-| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- |
-| 2.1.0   | 2022-05-08 | [25903](https://github.com/airbytehq/airbyte/pull/25903)                                                              | Add user.address (object)                                                                                       |
-| 2.0.3   | 2022-02-20 | [23259](https://github.com/airbytehq/airbyte/pull/23259)                                                              | bump to test publication                                                                                        |
-| 2.0.2   | 2022-02-20 | [23259](https://github.com/airbytehq/airbyte/pull/23259)                                                              | bump to test publication                                                                                        |
-| 2.0.1   | 2022-01-30 | [22117](https://github.com/airbytehq/airbyte/pull/22117)                                                              | `source-faker` goes beta                                                                                        |
+|:--------|:-----------|:----------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 5.0.0   | 2023-08-08 | [29213](https://github.com/airbytehq/airbyte/pull/29213)                                                              | Change all `*id` fields and `products.year` to be integer                                                       |
+| 4.0.0   | 2023-07-19 | [28485](https://github.com/airbytehq/airbyte/pull/28485)                                                              | Bump to test publication                                                                                        |
+| 3.0.2   | 2023-07-07 | [27807](https://github.com/airbytehq/airbyte/pull/28060)                                                              | Bump to test publication                                                                                        |
+| 3.0.1   | 2023-06-28 | [27807](https://github.com/airbytehq/airbyte/pull/27807)                                                              | Fix bug with purchase stream updated_at                                                                         |
+| 3.0.0   | 2023-06-23 | [27684](https://github.com/airbytehq/airbyte/pull/27684)                                                              | Stream cursor is now `updated_at` & remove `records_per_sync` option                                            |
+| 2.1.0   | 2023-05-08 | [25903](https://github.com/airbytehq/airbyte/pull/25903)                                                              | Add user.address (object)                                                                                       |
+| 2.0.3   | 2023-02-20 | [23259](https://github.com/airbytehq/airbyte/pull/23259)                                                              | bump to test publication                                                                                        |
+| 2.0.2   | 2023-02-20 | [23259](https://github.com/airbytehq/airbyte/pull/23259)                                                              | bump to test publication                                                                                        |
+| 2.0.1   | 2023-01-30 | [22117](https://github.com/airbytehq/airbyte/pull/22117)                                                              | `source-faker` goes beta                                                                                        |
 | 2.0.0   | 2022-12-14 | [20492](https://github.com/airbytehq/airbyte/pull/20492) and [20741](https://github.com/airbytehq/airbyte/pull/20741) | Decouple stream states for better parallelism                                                                   |
 | 1.0.0   | 2022-11-28 | [19490](https://github.com/airbytehq/airbyte/pull/19490)                                                              | Faker uses the CDK; rename streams to be lower-case (breaking), add determinism to random purchases, and rename |
 | 0.2.1   | 2022-10-14 | [19197](https://github.com/airbytehq/airbyte/pull/19197)                                                              | Emit `AirbyteEstimateTraceMessage`                                                                              |
diff --git a/docs/integrations/sources/fastbill.md b/docs/integrations/sources/fastbill.md
index 05cf2c4793bd..df8178af469d 100644
--- a/docs/integrations/sources/fastbill.md
+++ b/docs/integrations/sources/fastbill.md
@@ -61,4 +61,5 @@ The Fastbill source connector supports the following [sync modes](https://docs.a
 
 | Version | Date        | Pull Request                                             | Subject                                           |
 |:--------|:------------|:---------------------------------------------------------|:--------------------------------------------------|
-| 0.1.0   | 2022-10-TBA | [18522](https://github.com/airbytehq/airbyte/pull/18593)   | New Source: Fastbill                                |
+| 0.2.0   | 2023-08-13 | [29390](https://github.com/airbytehq/airbyte/pull/29390)   | Migrated to Low Code CDK                                    |
+| 0.1.0   | 2022-11-08 | [18522](https://github.com/airbytehq/airbyte/pull/18593)   | New Source: Fastbill                                |
diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md
index cefbe549b08c..99ee0add09bd 100644
--- a/docs/integrations/sources/file.md
+++ b/docs/integrations/sources/file.md
@@ -4,72 +4,97 @@ This page contains the setup guide and reference information for the Files sourc
 
 ## Prerequisites
 
-- URL to access the file
-- Format
-- Reader options
-- Storage Providers
+- A file hosted on AWS S3, GCS, HTTPS, or an SFTP server
 
 ## Setup guide
 
 
 
-**For Airbyte Cloud:**
-
-Setup through Airbyte Cloud will be exactly the same as the open-source setup, except for the fact that local files are disabled.
+**For Airbyte Cloud users:** Please note that locally stored files cannot be used as a source in Airbyte Cloud.
 
 
 
+### Step 1: Set up the connector in Airbyte
+
+1. From the Airbyte UI, click the **Sources** tab, then click **+ New source** and select **Files (CSV, JSON, Excel, Feather, Parquet)** from the list of available sources.
+2. Enter a **Source name** of your choosing.
+3. For **Dataset Name**, enter the _name_ of the final table to replicate this file into (should include letters, numbers, dashes and underscores only).
+4. For **File Format**, select the _format_ of the file to replicate from the dropdown menu (Warning: some formats may be experimental. Please refer to [the table of supported formats](#file-formats)).
+
+### Step 2: Select the provider and set provider-specific configurations:
+
+1. For **Storage Provider**, use the dropdown menu to select the _Storage Provider_ or _Location_ of the file(s) which should be replicated, then configure the provider-specific fields as needed:
+
+#### HTTPS: Public Web [Default]
+- `User-Agent` (Optional)
+
+Set this to active if you want to add the [User-Agent header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) to requests (inactive by default).
+
+#### GCS: Google Cloud Storage
+- `Service Account JSON` (Required for **private** buckets) 
+
+To access **private** buckets stored on Google Cloud, this connector requires a service account JSON credentials file with the appropriate permissions. A detailed breakdown of this topic can be found at the [Google Cloud service accounts page](https://cloud.google.com/iam/docs/service-accounts). Please generate the "credentials.json" file and copy its content to this field, ensuring it is in JSON format. **If you are accessing publicly available data**, this field is not required.
+
+#### S3: Amazon Web Services
+- `AWS Access Key ID` (Required for **private** buckets)
+- `AWS Secret Access Key` (Required for **private** buckets)
+
+To access **private** buckets stored on AWS S3, this connector requires valid credentials with the necessary permissions. To access these keys, refer to the 
+[AWS IAM documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html).
+More information on setting permissions in AWS can be found 
+[here](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html). **If you are accessing publicly available data**, these fields are not required.
+
+#### AzBlob: Azure Blob Storage
+- `Storage Account` (Required)
+
+This is the globally unique name of the storage account that the desired blob sits within. See the [Azure documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview) for more details.
+
+**If you are accessing private storage**, you must also provide _one_ of the following security credentials with the necessary permissions:
+
+- `SAS Token`: [Find more information here](https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview).
+- `Shared Key`: [Find more information here](https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key).
+  
+#### SSH: Secure Shell / SCP: Secure Copy Protocol / SFTP: Secure File Transfer Protocol
+- `Host` (Required)
+  
+Enter the _hostname_ or _IP address_ of the remote server where the file trasfer will take place.
+- `User` (Required)
+  
+Enter the _username_ associated with your account on the remote server.
+- `Password` (Optional) 
+  
+**If required by the remote server**, enter the _password_ associated with your user account. Otherwise, leave this field blank.
+- `Port` (Optional) 
+
+Specify the _port number_ to use for the connection. The default port is usually 22. However, if your remote server uses a non-standard port, you can enter the appropriate port number here.
+
 
+#### Local Filesystem (Airbyte Open Source only)
+- `Storage`
+
+:::caution 
+Currently, the local storage URL for reading must start with the local mount "/local/".
+:::
 
-**For Airbyte Open Source:**
+Please note that if you are replicating data from a locally stored file on Windows OS, you will need to open the `.env` file in your local Airbyte root folder and change the values for: 
+- `LOCAL_ROOT` 
+- `LOCAL_DOCKER_MOUNT`
+- `HACK_LOCAL_ROOT_PARENT` 
 
-1. Once the File Source is selected, you should define both the storage provider along its URL and format of the file.
-2. Depending on the provider choice and privacy of the data, you will have to configure more options.
+Please set these to an existing absolute path on your machine. Colons in the path need to be replaced with a double forward slash, `//`. `LOCAL_ROOT` & `LOCAL_DOCKER_MOUNT` should be set to the same value, and `HACK_LOCAL_ROOT_PARENT` should be set to their parent directory.
 
 
-### Fields description
-
-- For `Dataset Name` use the _name_ of the final table to replicate this file into (should include letters, numbers dash and underscores only).
-- For `File Format` use the _format_ of the file which should be replicated (Warning: some formats may be experimental, please refer to the docs).
-- For `Reader Options` use a _string in JSON_ format. It depends on the chosen file format to provide additional options and tune its behavior. For example, `{}` for empty options, `{"sep": " "}` for set up separator to one space ' '.
-- For `URL` use the _URL_ path to access the file which should be replicated.
-- For `Storage Provider` use the _storage Provider_ or _Location_ of the file(s) which should be replicated.
-  - [Default] _Public Web_
-    - `User-Agent` set to active if you want to add User-Agent to requests
-  - _GCS: Google Cloud Storage_
-    - `Service Account JSON` In order to access private Buckets stored on Google Cloud, this connector would need a service account json credentials with the proper permissions as described here. Please generate the credentials.json file and copy/paste its content to this field (expecting JSON formats). If accessing publicly available data, this field is not necessary.
-  - _S3: Amazon Web Services_
-    - `AWS Access Key ID` In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary.
-    - `AWS Secret Access Key`In order to access private Buckets stored on AWS S3, this connector would need credentials with the proper permissions. If accessing publicly available data, this field is not necessary.
-  - _AzBlob: Azure Blob Storage_
-    - `Storage Account` The globally unique name of the storage account that the desired blob sits within. See here for more details.
-    - `SAS Token` To access Azure Blob Storage, this connector would need credentials with the proper permissions. One option is a SAS (Shared Access Signature) token. If accessing publicly available data, this field is not necessary.
-    - `Shared Key` To access Azure Blob Storage, this connector would need credentials with the proper permissions. One option is a storage account shared key (aka account key or access key). If accessing publicly available data, this field is not necessary.
-  - _SSH: Secure Shell_
-    - `User` use _username_.
-    - `Password` use _password_.
-    - `Host` use a _host_.
-    - `Port` use a _port_ for your host.
-  - _SCP: Secure copy protocol_
-    - `User` use _username_.
-    - `Password` use _password_.
-    - `Host` use a _host_.
-    - `Port` use a _port_ for your host.
-  - _SFTP: Secure File Transfer Protocol_
-    - `User` use _username_.
-    - `Password` use _password_.
-    - `Host` use a _host_.
-    - `Port` use a _port_ for your host.
-  - _Local Filesystem (limited)_
-    - `Storage` WARNING: Note that the local storage URL available for reading must start with the local mount "/local/" at the moment until we implement more advanced docker mounting options.
-
-#### Provider Specific Information
-
-- In case of Google Drive, it is necesary to use the Download URL, the format for that is `https://drive.google.com/uc?export=download&id=[DRIVE_FILE_ID]` where `[DRIVE_FILE_ID]` is the string found in the Share URL here `https://drive.google.com/file/d/[DRIVE_FILE_ID]/view?usp=sharing`
-- In case of GCS, it is necessary to provide the content of the service account keyfile to access private buckets. See settings of [BigQuery Destination](../destinations/bigquery.md)
-- In case of AWS S3, the pair of `aws_access_key_id` and `aws_secret_access_key` is necessary to access private S3 buckets.
-- In case of AzBlob, it is necessary to provide the `storage_account` in which the blob you want to access resides. Either `sas_token` [(info)](https://docs.microsoft.com/en-us/azure/storage/blobs/sas-service-create?tabs=dotnet) or `shared_key` [(info)](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal) is necessary to access private blobs.
-- In case of a locally stored file on a Windows OS, it's necessary to change the values for `LOCAL_ROOT`, `LOCAL_DOCKER_MOUNT` and `HACK_LOCAL_ROOT_PARENT` in the `.env` file to an existing absolute path on your machine (colons in the path need to be replaced with a double forward slash, //). `LOCAL_ROOT` & `LOCAL_DOCKER_MOUNT` should be the same value, and `HACK_LOCAL_ROOT_PARENT` should be the parent directory of the other two.
+### Step 3: Complete the connector setup
+1. For **URL**, enter the _URL path_ of the file to be replicated.
+
+:::note
+When connecting to a file located in **Google Drive**, please note that you need to utilize the Download URL format: `https://drive.google.com/uc?export=download&id=[DRIVE_FILE_ID]`. `[DRIVE_FILE_ID]` should be replaced with the unique string found in the Share URL specific to Google Drive. You can find the Share URL by visiting `https://drive.google.com/file/d/[DRIVE_FILE_ID]/view?usp=sharing`.
+
+When connecting to a file using **Azure Blob Storage**, please note that we account for the base URL. Therefore, you should only need to include the path to your specific file (eg `container/file.csv`).
+:::
+
+2. For **Reader Options** (Optional), you may choose to enter a _string_ in JSON format. Depending on the file format of your source, this will provide additional options and tune the Reader's behavior. Please refer to the [next section](#reader-options) for a breakdown of the possible inputs. This field may be left blank if you do not wish to configure custom Reader options.
+3. Click **Set up source** and wait for the tests to complete.
 
 ### Reader Options
 
@@ -94,9 +119,9 @@ For example, you can use the `{"orient" : "records"}` to change how orientation
 
 If you need to read Excel Binary Workbook, please specify `excel_binary` format in `File Format` select.
 
-    :::warning
-    This connector does not support syncing unstructured data files such as raw text, audio, or videos.
-    :::
+:::caution
+This connector does not support syncing unstructured data files such as raw text, audio, or videos.
+:::
 
 ## Supported sync modes
 
@@ -108,9 +133,9 @@ If you need to read Excel Binary Workbook, please specify `excel_binary` format
 | Replicate Folders (multiple Files)       | No         |
 | Replicate Glob Patterns (multiple Files) | No         |
 
-    :::info
-    This source produces a single table for the target file as it replicates only one file at a time for the moment. Note that you should provide the `dataset_name` which dictates how the table will be identified in the destination (since `URL` can be made of complex characters).
-    :::
+:::note
+This source produces a single table for the target file as it replicates only one file at a time for the moment. Note that you should provide the `dataset_name` which dictates how the table will be identified in the destination (since `URL` can be made of complex characters).
+:::
 
 ## File / Stream Compression
 
@@ -139,7 +164,7 @@ If you need to read Excel Binary Workbook, please specify `excel_binary` format
 | Format                | Supported? |
 | --------------------- | ---------- |
 | CSV                   | Yes        |
-| JSON                  | Yes        |
+| JSON/JSONL            | Yes        |
 | HTML                  | No         |
 | XML                   | No         |
 | Excel                 | Yes        |
@@ -189,57 +214,59 @@ In order to read large files from a remote location, this connector uses the [sm
 
 ## Changelog
 
-| Version   | Date         | Pull Request                                               | Subject                                                                                                   |
-|:----------|:-------------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------|
-| 0.3.9     | 2023-05-18   | [26275](https://github.com/airbytehq/airbyte/pull/26275)   | add ParserError handling                                                                                  |
-| 0.3.8     | 2023-05-17   | [26210](https://github.com/airbytehq/airbyte/pull/26210)   | Bugfix for https://github.com/airbytehq/airbyte/pull/26115                                                |
-| 0.3.7     | 2023-05-16   | [26131](https://github.com/airbytehq/airbyte/pull/26131)   | Re-release source-file to be in sync with source-file-secure                                              |
-| 0.3.6     | 2023-05-16   | [26115](https://github.com/airbytehq/airbyte/pull/26115)   | Add retry on SSHException('Error reading SSH protocol banner')                                            |
-| 0.3.5     | 2023-05-16   | [26117](https://github.com/airbytehq/airbyte/pull/26117)   | Check if reader options is a valid JSON object                                                            |
-| 0.3.4     | 2023-05-10   | [25965](https://github.com/airbytehq/airbyte/pull/25965)   | fix Pandas date-time parsing to airbyte type                                                              |
-| 0.3.3     | 2023-05-04   | [25819](https://github.com/airbytehq/airbyte/pull/25819)   | GCP service_account_json is a secret                                                                      |
-| 0.3.2     | 2023-05-01   | [25641](https://github.com/airbytehq/airbyte/pull/25641)   | Handle network errors                                                                                     |
-| 0.3.1     | 2023-04-27   | [25575](https://github.com/airbytehq/airbyte/pull/25575)   | Fix OOM; read Excel files in chunks using `openpyxl`                                                      |
-| 0.3.0     | 2023-04-24   | [25445](https://github.com/airbytehq/airbyte/pull/25445)   | Add datatime format parsing support for csv files                                                         |
-| 0.2.38    | 2023-04-12   | [23759](https://github.com/airbytehq/airbyte/pull/23759)   | Fix column data types for numerical values                                                                |
-| 0.2.37    | 2023-04-06   | [24525](https://github.com/airbytehq/airbyte/pull/24525)   | Fix examples in spec                                                                                      |
-| 0.2.36    | 2023-03-27   | [24588](https://github.com/airbytehq/airbyte/pull/24588)   | Remove traceback from user messages.                                                                      |
-| 0.2.35    | 2023-03-03   | [24278](https://github.com/airbytehq/airbyte/pull/24278)   | Read only file header when checking connectivity; read only a single chunk when discovering the schema.   |
-| 0.2.34    | 2023-03-03   | [23723](https://github.com/airbytehq/airbyte/pull/23723)   | Update description in spec, make user-friendly error messages and docs.                                   |
-| 0.2.33    | 2023-01-04   | [21012](https://github.com/airbytehq/airbyte/pull/21012)   | Fix special characters bug                                                                                |
-| 0.2.32    | 2022-12-21   | [20740](https://github.com/airbytehq/airbyte/pull/20740)   | Source File: increase SSH timeout to 60s                                                                  |
-| 0.2.31    | 2022-11-17   | [19567](https://github.com/airbytehq/airbyte/pull/19567)   | Source File: bump 0.2.31                                                                                  |
-| 0.2.30    | 2022-11-10   | [19222](https://github.com/airbytehq/airbyte/pull/19222)   | Use AirbyteConnectionStatus for "check" command                                                           |
-| 0.2.29    | 2022-11-08   | [18587](https://github.com/airbytehq/airbyte/pull/18587)   | Fix pandas read_csv header none issue.                                                                    |
-| 0.2.28    | 2022-10-27   | [18428](https://github.com/airbytehq/airbyte/pull/18428)   | Add retry logic for `Connection reset error - 104`                                                        |
-| 0.2.27    | 2022-10-26   | [18481](https://github.com/airbytehq/airbyte/pull/18481)   | Fix check for wrong format                                                                                |
-| 0.2.26    | 2022-10-18   | [18116](https://github.com/airbytehq/airbyte/pull/18116)   | Transform Dropbox shared link                                                                             |
-| 0.2.25    | 2022-10-14   | [17994](https://github.com/airbytehq/airbyte/pull/17994)   | Handle `UnicodeDecodeError` during discover step.                                                         |
-| 0.2.24    | 2022-10-03   | [17504](https://github.com/airbytehq/airbyte/pull/17504)   | Validate data for `HTTPS` while `check_connection`                                                        |
-| 0.2.23    | 2022-09-28   | [17304](https://github.com/airbytehq/airbyte/pull/17304)   | Migrate to per-stream state.                                                                              |
-| 0.2.22    | 2022-09-15   | [16772](https://github.com/airbytehq/airbyte/pull/16772)   | Fix schema generation for JSON files containing arrays                                                    |
-| 0.2.21    | 2022-08-26   | [15568](https://github.com/airbytehq/airbyte/pull/15568)   | Specify `pyxlsb` library for Excel Binary Workbook files                                                  |
-| 0.2.20    | 2022-08-23   | [15870](https://github.com/airbytehq/airbyte/pull/15870)   | Fix CSV schema discovery                                                                                  |
-| 0.2.19    | 2022-08-19   | [15768](https://github.com/airbytehq/airbyte/pull/15768)   | Convert 'nan' to 'null'                                                                                   |
-| 0.2.18    | 2022-08-16   | [15698](https://github.com/airbytehq/airbyte/pull/15698)   | Cache binary stream to file for discover                                                                  |
-| 0.2.17    | 2022-08-11   | [15501](https://github.com/airbytehq/airbyte/pull/15501)   | Cache binary stream to file                                                                               |
-| 0.2.16    | 2022-08-10   | [15293](https://github.com/airbytehq/airbyte/pull/15293)   | Add support for encoding reader option                                                                    |
-| 0.2.15    | 2022-08-05   | [15269](https://github.com/airbytehq/airbyte/pull/15269)   | Bump `smart-open` version to 6.0.0                                                                        |
-| 0.2.12    | 2022-07-12   | [14535](https://github.com/airbytehq/airbyte/pull/14535)   | Fix invalid schema generation for JSON files                                                              |
-| 0.2.11    | 2022-07-12   | [9974](https://github.com/airbytehq/airbyte/pull/14588)    | Add support to YAML format                                                                                |
-| 0.2.9     | 2022-02-01   | [9974](https://github.com/airbytehq/airbyte/pull/9974)     | Update airbyte-cdk 0.1.47                                                                                 |
-| 0.2.8     | 2021-12-06   | [8524](https://github.com/airbytehq/airbyte/pull/8524)     | Update connector fields title/description                                                                 |
-| 0.2.7     | 2021-10-28   | [7387](https://github.com/airbytehq/airbyte/pull/7387)     | Migrate source to CDK structure, add SAT testing.                                                         |
-| 0.2.6     | 2021-08-26   | [5613](https://github.com/airbytehq/airbyte/pull/5613)     | Add support to xlsb format                                                                                |
-| 0.2.5     | 2021-07-26   | [4953](https://github.com/airbytehq/airbyte/pull/4953)     | Allow non-default port for SFTP type                                                                      |
-| 0.2.4     | 2021-06-09   | [3973](https://github.com/airbytehq/airbyte/pull/3973)     | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                                             |
-| 0.2.3     | 2021-06-01   | [3771](https://github.com/airbytehq/airbyte/pull/3771)     | Add Azure Storage Blob Files option                                                                       |
-| 0.2.2     | 2021-04-16   | [2883](https://github.com/airbytehq/airbyte/pull/2883)     | Fix CSV discovery memory consumption                                                                      |
-| 0.2.1     | 2021-04-03   | [2726](https://github.com/airbytehq/airbyte/pull/2726)     | Fix base connector versioning                                                                             |
-| 0.2.0     | 2021-03-09   | [2238](https://github.com/airbytehq/airbyte/pull/2238)     | Protocol allows future/unknown properties                                                                 |
-| 0.1.10    | 2021-02-18   | [2118](https://github.com/airbytehq/airbyte/pull/2118)     | Support JSONL format                                                                                      |
-| 0.1.9     | 2021-02-02   | [1768](https://github.com/airbytehq/airbyte/pull/1768)     | Add test cases for all formats                                                                            |
-| 0.1.8     | 2021-01-27   | [1738](https://github.com/airbytehq/airbyte/pull/1738)     | Adopt connector best practices                                                                            |
-| 0.1.7     | 2020-12-16   | [1331](https://github.com/airbytehq/airbyte/pull/1331)     | Refactor Python base connector                                                                            |
-| 0.1.6     | 2020-12-08   | [1249](https://github.com/airbytehq/airbyte/pull/1249)     | Handle NaN values                                                                                         |
-| 0.1.5     | 2020-11-30   | [1046](https://github.com/airbytehq/airbyte/pull/1046)     | Add connectors using an index YAML file                                                                   |
+| Version | Date       | Pull Request                                               | Subject                                                                                                 |
+|:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------------------------------------------------------------------------|
+| 0.3.11  | 2023-06-08 | [27157](https://github.com/airbytehq/airbyte/pull/27157)   | Force smart open log level to ERROR                                                                     |
+| 0.3.10  | 2023-06-07 | [27107](https://github.com/airbytehq/airbyte/pull/27107)   | Make source-file testable in our new airbyte-ci pipelines                                               |
+| 0.3.9   | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275)   | Add ParserError handling                                                                                |
+| 0.3.8   | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210)   | Bugfix for https://github.com/airbytehq/airbyte/pull/26115                                              |
+| 0.3.7   | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131)   | Re-release source-file to be in sync with source-file-secure                                            |
+| 0.3.6   | 2023-05-16 | [26115](https://github.com/airbytehq/airbyte/pull/26115)   | Add retry on SSHException('Error reading SSH protocol banner')                                          |
+| 0.3.5   | 2023-05-16 | [26117](https://github.com/airbytehq/airbyte/pull/26117)   | Check if reader options is a valid JSON object                                                          |
+| 0.3.4   | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965)   | fix Pandas date-time parsing to airbyte type                                                            |
+| 0.3.3   | 2023-05-04 | [25819](https://github.com/airbytehq/airbyte/pull/25819)   | GCP service_account_json is a secret                                                                    |
+| 0.3.2   | 2023-05-01 | [25641](https://github.com/airbytehq/airbyte/pull/25641)   | Handle network errors                                                                                   |
+| 0.3.1   | 2023-04-27 | [25575](https://github.com/airbytehq/airbyte/pull/25575)   | Fix OOM; read Excel files in chunks using `openpyxl`                                                    |
+| 0.3.0   | 2023-04-24 | [25445](https://github.com/airbytehq/airbyte/pull/25445)   | Add datatime format parsing support for csv files                                                       |
+| 0.2.38  | 2023-04-12 | [23759](https://github.com/airbytehq/airbyte/pull/23759)   | Fix column data types for numerical values                                                              |
+| 0.2.37  | 2023-04-06 | [24525](https://github.com/airbytehq/airbyte/pull/24525)   | Fix examples in spec                                                                                    |
+| 0.2.36  | 2023-03-27 | [24588](https://github.com/airbytehq/airbyte/pull/24588)   | Remove traceback from user messages.                                                                    |
+| 0.2.35  | 2023-03-03 | [24278](https://github.com/airbytehq/airbyte/pull/24278)   | Read only file header when checking connectivity; read only a single chunk when discovering the schema. |
+| 0.2.34  | 2023-03-03 | [23723](https://github.com/airbytehq/airbyte/pull/23723)   | Update description in spec, make user-friendly error messages and docs.                                 |
+| 0.2.33  | 2023-01-04 | [21012](https://github.com/airbytehq/airbyte/pull/21012)   | Fix special characters bug                                                                              |
+| 0.2.32  | 2022-12-21 | [20740](https://github.com/airbytehq/airbyte/pull/20740)   | Source File: increase SSH timeout to 60s                                                                |
+| 0.2.31  | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567)   | Source File: bump 0.2.31                                                                                |
+| 0.2.30  | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222)   | Use AirbyteConnectionStatus for "check" command                                                         |
+| 0.2.29  | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587)   | Fix pandas read_csv header none issue.                                                                  |
+| 0.2.28  | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428)   | Add retry logic for `Connection reset error - 104`                                                      |
+| 0.2.27  | 2022-10-26 | [18481](https://github.com/airbytehq/airbyte/pull/18481)   | Fix check for wrong format                                                                              |
+| 0.2.26  | 2022-10-18 | [18116](https://github.com/airbytehq/airbyte/pull/18116)   | Transform Dropbox shared link                                                                           |
+| 0.2.25  | 2022-10-14 | [17994](https://github.com/airbytehq/airbyte/pull/17994)   | Handle `UnicodeDecodeError` during discover step.                                                       |
+| 0.2.24  | 2022-10-03 | [17504](https://github.com/airbytehq/airbyte/pull/17504)   | Validate data for `HTTPS` while `check_connection`                                                      |
+| 0.2.23  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304)   | Migrate to per-stream state.                                                                            |
+| 0.2.22  | 2022-09-15 | [16772](https://github.com/airbytehq/airbyte/pull/16772)   | Fix schema generation for JSON files containing arrays                                                  |
+| 0.2.21  | 2022-08-26 | [15568](https://github.com/airbytehq/airbyte/pull/15568)   | Specify `pyxlsb` library for Excel Binary Workbook files                                                |
+| 0.2.20  | 2022-08-23 | [15870](https://github.com/airbytehq/airbyte/pull/15870)   | Fix CSV schema discovery                                                                                |
+| 0.2.19  | 2022-08-19 | [15768](https://github.com/airbytehq/airbyte/pull/15768)   | Convert 'nan' to 'null'                                                                                 |
+| 0.2.18  | 2022-08-16 | [15698](https://github.com/airbytehq/airbyte/pull/15698)   | Cache binary stream to file for discover                                                                |
+| 0.2.17  | 2022-08-11 | [15501](https://github.com/airbytehq/airbyte/pull/15501)   | Cache binary stream to file                                                                             |
+| 0.2.16  | 2022-08-10 | [15293](https://github.com/airbytehq/airbyte/pull/15293)   | Add support for encoding reader option                                                                  |
+| 0.2.15  | 2022-08-05 | [15269](https://github.com/airbytehq/airbyte/pull/15269)   | Bump `smart-open` version to 6.0.0                                                                      |
+| 0.2.12  | 2022-07-12 | [14535](https://github.com/airbytehq/airbyte/pull/14535)   | Fix invalid schema generation for JSON files                                                            |
+| 0.2.11  | 2022-07-12 | [9974](https://github.com/airbytehq/airbyte/pull/14588)    | Add support to YAML format                                                                              |
+| 0.2.9   | 2022-02-01 | [9974](https://github.com/airbytehq/airbyte/pull/9974)     | Update airbyte-cdk 0.1.47                                                                               |
+| 0.2.8   | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524)     | Update connector fields title/description                                                               |
+| 0.2.7   | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387)     | Migrate source to CDK structure, add SAT testing.                                                       |
+| 0.2.6   | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613)     | Add support to xlsb format                                                                              |
+| 0.2.5   | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953)     | Allow non-default port for SFTP type                                                                    |
+| 0.2.4   | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973)     | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                                           |
+| 0.2.3   | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771)     | Add Azure Storage Blob Files option                                                                     |
+| 0.2.2   | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883)     | Fix CSV discovery memory consumption                                                                    |
+| 0.2.1   | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726)     | Fix base connector versioning                                                                           |
+| 0.2.0   | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238)     | Protocol allows future/unknown properties                                                               |
+| 0.1.10  | 2021-02-18 | [2118](https://github.com/airbytehq/airbyte/pull/2118)     | Support JSONL format                                                                                    |
+| 0.1.9   | 2021-02-02 | [1768](https://github.com/airbytehq/airbyte/pull/1768)     | Add test cases for all formats                                                                          |
+| 0.1.8   | 2021-01-27 | [1738](https://github.com/airbytehq/airbyte/pull/1738)     | Adopt connector best practices                                                                          |
+| 0.1.7   | 2020-12-16 | [1331](https://github.com/airbytehq/airbyte/pull/1331)     | Refactor Python base connector                                                                          |
+| 0.1.6   | 2020-12-08 | [1249](https://github.com/airbytehq/airbyte/pull/1249)     | Handle NaN values                                                                                       |
+| 0.1.5   | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046)     | Add connectors using an index YAML file                                                                 |
diff --git a/docs/integrations/sources/firebolt-migrations.md b/docs/integrations/sources/firebolt-migrations.md
new file mode 100644
index 000000000000..094e6edfb188
--- /dev/null
+++ b/docs/integrations/sources/firebolt-migrations.md
@@ -0,0 +1,5 @@
+# Firebolt Migration Guide
+
+## Upgrading to 1.0.0
+
+Add new data type column.
diff --git a/docs/integrations/sources/firebolt.md b/docs/integrations/sources/firebolt.md
index fe991988aef0..c59271237667 100644
--- a/docs/integrations/sources/firebolt.md
+++ b/docs/integrations/sources/firebolt.md
@@ -51,6 +51,7 @@ You can now use the Airbyte Firebolt source.
 
 | Version | Date       | Pull Request                                             | Subject                                           |
 | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ |
+| 1.0.0   | 2023-07-20 | [21842](https://github.com/airbytehq/airbyte/pull/21842) | PGDate, TimestampTZ, TimestampNTZ and Boolean column support |
 | 0.2.1   | 2022-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix DATETIME conversion to Airbyte date-time type |
 | 0.2.0   | 2022-09-09 | [16583](https://github.com/airbytehq/airbyte/pull/16583) | Reading from views                                |
 | 0.1.0   | 2022-04-28 | [13874](https://github.com/airbytehq/airbyte/pull/13874) | Create Firebolt source                            |
diff --git a/docs/integrations/sources/flexport.md b/docs/integrations/sources/flexport.md
index 032a4644d365..20cb5f41a8d5 100644
--- a/docs/integrations/sources/flexport.md
+++ b/docs/integrations/sources/flexport.md
@@ -46,5 +46,6 @@ Authentication uses a pre-created API token which can be [created in the UI](htt
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.2.0 | 2023-08-23 | [29151](https://github.com/airbytehq/airbyte/pull/29151) | Migrate to low-code |
 | 0.1.1 | 2022-07-26 | [15033](https://github.com/airbytehq/airbyte/pull/15033) | Source Flexport: Update schemas |
 | 0.1.0 | 2021-12-14 | [8777](https://github.com/airbytehq/airbyte/pull/8777) | New Source: Flexport |
diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md
index 96de2159ff2f..126173fefbbe 100644
--- a/docs/integrations/sources/freshdesk.md
+++ b/docs/integrations/sources/freshdesk.md
@@ -18,45 +18,45 @@ To set up the Freshdesk source connector, you'll need the Freshdesk [domain URL]
 8. For **Requests per minute**, enter the number of requests per minute that this source allowed to use. The Freshdesk rate limit is 50 requests per minute per app per account.
 9. Click **Set up source**.
 
-## Supported sync modes 
+## Supported sync modes
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
 Several output streams are available from this source:
 
-* [Agents](https://developers.freshdesk.com/api/#agents)
-* [Business Hours](https://developers.freshdesk.com/api/#business-hours)
-* [Canned Responses](https://developers.freshdesk.com/api/#canned-responses)
-* [Canned Response Folders](https://developers.freshdesk.com/api/#list_all_canned_response_folders)
-* [Companies](https://developers.freshdesk.com/api/#companies)
-* [Contacts](https://developers.freshdesk.com/api/#contacts) \(Native Incremental Sync\)
-* [Conversations](https://developers.freshdesk.com/api/#conversations)
-* [Discussion Categories](https://developers.freshdesk.com/api/#category_attributes)
-* [Discussion Comments](https://developers.freshdesk.com/api/#comment_attributes)
-* [Discussion Forums](https://developers.freshdesk.com/api/#forum_attributes)
-* [Discussion Topics](https://developers.freshdesk.com/api/#topic_attributes)
-* [Email Configs](https://developers.freshdesk.com/api/#email-configs)
-* [Email Mailboxes](https://developers.freshdesk.com/api/#email-mailboxes)
-* [Groups](https://developers.freshdesk.com/api/#groups)
-* [Products](https://developers.freshdesk.com/api/#products)
-* [Roles](https://developers.freshdesk.com/api/#roles)
-* [Satisfaction Ratings](https://developers.freshdesk.com/api/#satisfaction-ratings)
-* [Scenario Automations](https://developers.freshdesk.com/api/#scenario-automations)
-* [Settings](https://developers.freshdesk.com/api/#settings)
-* [Skills](https://developers.freshdesk.com/api/#skills)
-* [SLA Policies](https://developers.freshdesk.com/api/#sla-policies)
-* [Solution Articles](https://developers.freshdesk.com/api/#solution_article_attributes)
-* [Solution Categories](https://developers.freshdesk.com/api/#solution_category_attributes)
-* [Solution Folders](https://developers.freshdesk.com/api/#solution_folder_attributes)
-* [Surveys](https://developers.freshdesk.com/api/#surveys)
-* [Tickets](https://developers.freshdesk.com/api/#tickets) \(Native Incremental Sync\)
-* [Ticket Fields](https://developers.freshdesk.com/api/#ticket-fields)
-* [Time Entries](https://developers.freshdesk.com/api/#time-entries)
+- [Agents](https://developers.freshdesk.com/api/#agents)
+- [Business Hours](https://developers.freshdesk.com/api/#business-hours)
+- [Canned Responses](https://developers.freshdesk.com/api/#canned-responses)
+- [Canned Response Folders](https://developers.freshdesk.com/api/#list_all_canned_response_folders)
+- [Companies](https://developers.freshdesk.com/api/#companies)
+- [Contacts](https://developers.freshdesk.com/api/#contacts) \(Native Incremental Sync\)
+- [Conversations](https://developers.freshdesk.com/api/#conversations)
+- [Discussion Categories](https://developers.freshdesk.com/api/#category_attributes)
+- [Discussion Comments](https://developers.freshdesk.com/api/#comment_attributes)
+- [Discussion Forums](https://developers.freshdesk.com/api/#forum_attributes)
+- [Discussion Topics](https://developers.freshdesk.com/api/#topic_attributes)
+- [Email Configs](https://developers.freshdesk.com/api/#email-configs)
+- [Email Mailboxes](https://developers.freshdesk.com/api/#email-mailboxes)
+- [Groups](https://developers.freshdesk.com/api/#groups)
+- [Products](https://developers.freshdesk.com/api/#products)
+- [Roles](https://developers.freshdesk.com/api/#roles)
+- [Satisfaction Ratings](https://developers.freshdesk.com/api/#satisfaction-ratings)
+- [Scenario Automations](https://developers.freshdesk.com/api/#scenario-automations)
+- [Settings](https://developers.freshdesk.com/api/#settings)
+- [Skills](https://developers.freshdesk.com/api/#skills)
+- [SLA Policies](https://developers.freshdesk.com/api/#sla-policies)
+- [Solution Articles](https://developers.freshdesk.com/api/#solution_article_attributes)
+- [Solution Categories](https://developers.freshdesk.com/api/#solution_category_attributes)
+- [Solution Folders](https://developers.freshdesk.com/api/#solution_folder_attributes)
+- [Surveys](https://developers.freshdesk.com/api/#surveys)
+- [Tickets](https://developers.freshdesk.com/api/#tickets) \(Native Incremental Sync\)
+- [Ticket Fields](https://developers.freshdesk.com/api/#ticket-fields)
+- [Time Entries](https://developers.freshdesk.com/api/#time-entries)
 
 ## Performance considerations
 
@@ -64,12 +64,13 @@ The Freshdesk connector should not run into Freshdesk API limitations under norm
 
 If you don't use the start date Freshdesk will retrieve only the last 30 days. More information [here](https://developers.freshdesk.com/api/#list_all_tickets).
 
-
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                               |
-|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------|
-| 3.0.2   | 2023-02-06 | [21970](https://github.com/airbytehq/airbyte/pull/21970) | Enable availability strategy for all streams                                                       |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------ |
+| 3.0.4   | 2023-06-24 | [27680](https://github.com/airbytehq/airbyte/pull/27680) | Fix formatting                                                                        |
+| 3.0.3   | 2023-06-02 | [26978](https://github.com/airbytehq/airbyte/pull/26978) | Skip the stream if subscription level had changed during sync                         |
+| 3.0.2   | 2023-02-06 | [21970](https://github.com/airbytehq/airbyte/pull/21970) | Enable availability strategy for all streams                                          |
 | 3.0.0   | 2023-01-31 | [22164](https://github.com/airbytehq/airbyte/pull/22164) | Rename nested `business_hours` table to `working_hours`                               |
 | 2.0.1   | 2023-01-27 | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None`                           |
 | 2.0.0   | 2022-12-20 | [20416](https://github.com/airbytehq/airbyte/pull/20416) | Fix `SlaPolicies` stream schema                                                       |
diff --git a/docs/integrations/sources/freshservice.md b/docs/integrations/sources/freshservice.md
index 7dbf88854194..6481eb888c3f 100644
--- a/docs/integrations/sources/freshservice.md
+++ b/docs/integrations/sources/freshservice.md
@@ -54,6 +54,7 @@ Please read [How to find your API key](https://api.freshservice.com/#authenticat
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 1.2.0 | 2023-08-06 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Migrated to Low-Code CDK |
 | 1.1.0 | 2023-05-09 | [25929](https://github.com/airbytehq/airbyte/pull/25929) | Add stream for customer satisfaction survey responses endpoint  |
 | 1.0.0 | 2023-05-02 | [25743](https://github.com/airbytehq/airbyte/pull/25743) | Correct data types in tickets, agents and requesters schemas to match Freshservice API |
 | 0.1.1 | 2021-12-28 | [9143](https://github.com/airbytehq/airbyte/pull/9143) | Update titles and descriptions |
diff --git a/docs/integrations/sources/gainsight-px.md b/docs/integrations/sources/gainsight-px.md
new file mode 100644
index 000000000000..5ad6d9d4c27b
--- /dev/null
+++ b/docs/integrations/sources/gainsight-px.md
@@ -0,0 +1,74 @@
+# Gainsight-API
+
+This page contains the setup guide and reference information for the [Gainsight-PX-API](https://gainsightpx.docs.apiary.io/) source connector from [Gainsight-PX](https://support.gainsight.com/PX/API_for_Developers)
+
+## Prerequisites
+
+Api key is mandate for this connector to work, It could be generated from the dashboard settings (ref - https://app.aptrinsic.com/settings/api-keys). 
+
+## Setup guide
+
+### Step 1: Set up Gainsight-API connection
+
+- Generate an API key (Example: 12345)
+- Params (If specific info is needed)
+- Available params
+    - api_key: The aptrinsic api_key
+
+## Step 2: Set up the Gainsight-APIs connector in Airbyte
+
+### For Airbyte Cloud:
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**.
+3. On the Set up the source page, enter the name for the Gainsight-API connector and select **Gainsight-API** from the Source type dropdown.
+4. Enter your `api_key`.
+5. Enter the params configuration if needed. Supported params are: query, orientation, size, color, locale, collection_id \ 
+video_id, photo_id
+6. Click **Set up source**.
+
+### For Airbyte OSS:
+
+1. Navigate to the Airbyte Open Source dashboard.
+2. Set the name for your source.
+3. Enter your `api_key`.
+4. Enter the params configuration if needed. Supported params are: query, orientation, size, color, locale, collection_id \ 
+video_id, photo_id
+5. Click **Set up source**.
+
+## Supported sync modes
+
+The Gainsight-API source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+
+| Feature                       | Supported? |
+| :---------------------------- | :--------- |
+| Full Refresh Sync             | Yes        |
+| Incremental Sync              | No         |
+| Replicate Incremental Deletes | No         |
+| SSL connection                | Yes        |
+| Namespaces                    | No         |
+
+## Supported Streams
+
+- accounts
+- admin_attributes
+- articles
+- feature
+- kcbot
+- segments
+- user_attributes
+- users
+
+## API method example
+
+GET https://api.aptrinsic.com/v1/accounts
+
+## Performance considerations
+
+Gainsight-PX-API's [API reference](https://gainsightpx.docs.apiary.io/) has v1 at present. The connector as default uses v1.
+
+## Changelog
+
+| Version | Date       | Pull Request                                           | Subject        |
+| :------ | :--------- | :----------------------------------------------------- | :------------- |
+| 0.1.0   | 2023-05-10 | [Init](https://github.com/airbytehq/airbyte/pull/26998)| Initial PR     |
diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md
index 6d81fa95b408..8b309e911197 100644
--- a/docs/integrations/sources/gcs.md
+++ b/docs/integrations/sources/gcs.md
@@ -8,9 +8,9 @@ Cloud storage may incur egress costs. Egress refers to data that is transferred
 
 ## Prerequisites
 
-* JSON credentials for the service account that has access to GCS. For more details check [instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts)
-* GCS bucket
-* Path to file(s)
+- JSON credentials for the service account that has access to GCS. For more details check [instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts)
+- GCS bucket
+- Path to file(s)
 
 ## Set up Source
 
@@ -29,11 +29,13 @@ Use the service account ID from above, grant read access to your target bucket.
 
 ### Set up the source in Airbyte UI
 
-* Paste the service account JSON key to `service_account`
-* Enter your GCS bucket name to `gcs_bucket`
-* Enter path to your file(s) to `gcs_path`
+- Paste the service account JSON key to `service_account`
+- Enter your GCS bucket name to `gcs_bucket`
+- Enter path to your file(s) to `gcs_path`
 
 ## Changelog
-| Version | Date       | Pull Request                                             | Subject                     |
-| :------ | :--------- | :------------------------------------------------------- | :-------------------------- |
-| 0.1.0   | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS             |
+
+| Version | Date       | Pull Request                                             | Subject              |
+| :------ | :--------- | :------------------------------------------------------- | :------------------- |
+| 0.2.0   | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 |
+| 0.1.0   | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS      |
diff --git a/docs/integrations/sources/genesys.md b/docs/integrations/sources/genesys.md
index e756d2dd2688..36b66946e13c 100644
--- a/docs/integrations/sources/genesys.md
+++ b/docs/integrations/sources/genesys.md
@@ -24,4 +24,5 @@ You can follow the documentation on [API credentials](https://developer.genesys.
 ## Changelog
 | Version | Date       | Pull Request                                             | Subject                     |
 | :------ | :--------- | :------------------------------------------------------- | :-------------------------- |
+| 0.1.1   | 2023-04-27 | [25598](https://github.com/airbytehq/airbyte/pull/25598) | Use region specific API server |
 | 0.1.0   | 2022-10-06 | [17559](https://github.com/airbytehq/airbyte/pull/17559) | The Genesys Source is created |
diff --git a/docs/integrations/sources/github.inapp.md b/docs/integrations/sources/github.inapp.md
new file mode 100644
index 000000000000..5bd87e059bf4
--- /dev/null
+++ b/docs/integrations/sources/github.inapp.md
@@ -0,0 +1,47 @@
+## Prerequisites
+
+- Access to a Github repository
+
+## Setup guide
+
+1. Name your source.
+2. Click `Authenticate your GitHub account` or use a [Personal Access Token](https://github.com/settings/tokens) for Authentication. For Personal Access Tokens, refer to the list of required [permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes).
+3. **Start date** Enter the date you'd like to replicate data from. 
+
+These streams will only sync records generated on or after the **Start Date**: 
+
+`comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`.
+
+The **Start Date** does not apply to the streams below and all data will be synced for these streams:
+
+`assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`,  `tags`,  `teams`, `users`
+
+4. **GitHub Repositories** - Enter a space-delimited list of GitHub organizations or repositories.
+
+Example of a single repository:
+```
+airbytehq/airbyte
+```
+Example of multiple repositories:
+```
+airbytehq/airbyte airbytehq/another-repo
+```
+Example of an organization to receive data from all of its repositories: 
+```
+airbytehq/*
+```
+Repositories which have a misspelled name, do not exist, or have the wrong name format will return an error.
+
+5. (Optional) **Branch** - Enter a space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`).
+6. (Optional) **Max requests per hour** - The GitHub API allows for a maximum of 5000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota.
+
+### Incremental Sync Methods
+Incremental sync is offered for most streams, with some differences in sync behavior.
+
+1. `comments`, `commits`, `issues` and `review comments` only syncs new records. Only new records will be synced.
+
+2. `workflow_runs` and `worflow_jobs` syncs new records and any records run in the [last 30 days](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs)
+
+3. All other incremental streams sync all historical records and output any updated or new records.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [GitHub](https://docs.airbyte.com/integrations/sources/github/).
diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md
index 5e42ebeb5605..aca33b1c377d 100644
--- a/docs/integrations/sources/github.md
+++ b/docs/integrations/sources/github.md
@@ -163,6 +163,9 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.0.4   | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes  |
+| 1.0.3   | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references  |
+| 1.0.2   | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144)                                                          | Add `archived_at` property to `Organizations` schema parameter                                                                                                                               |
 | 1.0.1   | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838)                                                          | Deprecate "page size" input parameter                                                                                                                               |
 | 1.0.0   | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778)                                                          | Improve repo(s) name validation on UI                                                                                                                               |
 | 0.5.0   | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793)                                                          | Implement client-side throttling of requests                                                                                                                        |
diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md
index 174b2f1c91ff..caf45350163f 100644
--- a/docs/integrations/sources/gitlab.md
+++ b/docs/integrations/sources/gitlab.md
@@ -10,6 +10,7 @@ This page contains the setup guide and reference information for the Gitlab Sour
 - GitLab Projects (Optional)
 
 
+
 **For Airbyte Cloud:**
 
 - Personal Access Token (see [personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html))
@@ -17,6 +18,7 @@ This page contains the setup guide and reference information for the Gitlab Sour
 
 
 
+
 **For Airbyte Open Source:**
 
 - Personal Access Token (see [personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html))
@@ -29,12 +31,15 @@ This page contains the setup guide and reference information for the Gitlab Sour
 Create a [GitLab Account](https://gitlab.com) or set up a local instance of GitLab.
 
 
+
 **Airbyte Open Source additional setup steps**
 
 Log into [GitLab](https://gitlab.com) and then generate a [personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html). Your token should have the `read_api` scope, that Grants read access to the API, including all groups and projects, the container registry, and the package registry.
+
 
 
 
+
 ### Step 2: Set up the GitLab connector in Airbyte
 
 **For Airbyte Cloud:**
@@ -55,6 +60,7 @@ Log into [GitLab](https://gitlab.com) and then generate a [personal access token
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Authenticate with **Personal Access Token**.
@@ -64,34 +70,34 @@ Log into [GitLab](https://gitlab.com) and then generate a [personal access token
 
 The Gitlab Source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
 This connector outputs the following streams:
 
-* [Branches](https://docs.gitlab.com/ee/api/branches.html)
-* [Commits](https://docs.gitlab.com/ee/api/commits.html) \(Incremental\)
-* [Issues](https://docs.gitlab.com/ee/api/issues.html) \(Incremental\)
-* [Group Issue Boards](https://docs.gitlab.com/ee/api/group_boards.html)
-* [Pipelines](https://docs.gitlab.com/ee/api/pipelines.html) \(Incremental\)
-* [Jobs](https://docs.gitlab.com/ee/api/jobs.html)
-* [Projects](https://docs.gitlab.com/ee/api/projects.html)
-* [Project Milestones](https://docs.gitlab.com/ee/api/milestones.html)
-* [Project Merge Requests](https://docs.gitlab.com/ee/api/merge_requests.html) \(Incremental\)
-* [Users](https://docs.gitlab.com/ee/api/users.html)
-* [Groups](https://docs.gitlab.com/ee/api/groups.html)
-* [Group Milestones](https://docs.gitlab.com/ee/api/group_milestones.html)
-* [Group and Project members](https://docs.gitlab.com/ee/api/members.html)
-* [Tags](https://docs.gitlab.com/ee/api/tags.html)
-* [Releases](https://docs.gitlab.com/ee/api/releases/index.html)
-* [Group Labels](https://docs.gitlab.com/ee/api/group_labels.html)
-* [Project Labels](https://docs.gitlab.com/ee/api/labels.html)
-* [Epics](https://docs.gitlab.com/ee/api/epics.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\)
-* [Epic Issues](https://docs.gitlab.com/ee/api/epic_issues.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\)
+- [Branches](https://docs.gitlab.com/ee/api/branches.html)
+- [Commits](https://docs.gitlab.com/ee/api/commits.html) \(Incremental\)
+- [Issues](https://docs.gitlab.com/ee/api/issues.html) \(Incremental\)
+- [Group Issue Boards](https://docs.gitlab.com/ee/api/group_boards.html)
+- [Pipelines](https://docs.gitlab.com/ee/api/pipelines.html) \(Incremental\)
+- [Jobs](https://docs.gitlab.com/ee/api/jobs.html)
+- [Projects](https://docs.gitlab.com/ee/api/projects.html)
+- [Project Milestones](https://docs.gitlab.com/ee/api/milestones.html)
+- [Project Merge Requests](https://docs.gitlab.com/ee/api/merge_requests.html) \(Incremental\)
+- [Users](https://docs.gitlab.com/ee/api/users.html)
+- [Groups](https://docs.gitlab.com/ee/api/groups.html)
+- [Group Milestones](https://docs.gitlab.com/ee/api/group_milestones.html)
+- [Group and Project members](https://docs.gitlab.com/ee/api/members.html)
+- [Tags](https://docs.gitlab.com/ee/api/tags.html)
+- [Releases](https://docs.gitlab.com/ee/api/releases/index.html)
+- [Group Labels](https://docs.gitlab.com/ee/api/group_labels.html)
+- [Project Labels](https://docs.gitlab.com/ee/api/labels.html)
+- [Epics](https://docs.gitlab.com/ee/api/epics.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\)
+- [Epic Issues](https://docs.gitlab.com/ee/api/epic_issues.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\)
 
 ## Additional information
 
@@ -104,7 +110,15 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------- |
+| 1.6.0   | 2023-06-30 | [27869](https://github.com/airbytehq/airbyte/pull/27869) | Add `shared_runners_setting` field to groups                                               |
+| 1.5.1   | 2023-06-24 | [27679](https://github.com/airbytehq/airbyte/pull/27679) | Fix formatting                                                                             |
+| 1.5.0   | 2023-06-15 | [27392](https://github.com/airbytehq/airbyte/pull/27392) | Make API URL an optional parameter in spec.                                                |
+| 1.4.2   | 2023-06-15 | [27346](https://github.com/airbytehq/airbyte/pull/27346) | Partially revert changes made in version 1.0.4, disallow http calls in cloud.              |
+| 1.4.1   | 2023-06-13 | [27351](https://github.com/airbytehq/airbyte/pull/27351) | Fix OAuth token expiry date.                                                               |
+| 1.4.0   | 2023-06-12 | [27234](https://github.com/airbytehq/airbyte/pull/27234) | Skip stream slices on 403/404 errors, do not fail syncs.                                   |
+| 1.3.1   | 2023-06-08 | [27147](https://github.com/airbytehq/airbyte/pull/27147) | Improve connectivity check for connections with no projects/groups                         |
+| 1.3.0   | 2023-06-08 | [27150](https://github.com/airbytehq/airbyte/pull/27150) | Update stream schemas                                                                      |
 | 1.2.1   | 2023-06-02 | [26947](https://github.com/airbytehq/airbyte/pull/26947) | New field `name` added to `Pipelines` and `PipelinesExtended` stream schema                |
 | 1.2.0   | 2023-05-17 | [22293](https://github.com/airbytehq/airbyte/pull/22293) | Preserve data in records with flattened keys                                               |
 | 1.1.1   | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled |
diff --git a/docs/integrations/sources/glassfrog.md b/docs/integrations/sources/glassfrog.md
index b6d307810185..b12151215226 100644
--- a/docs/integrations/sources/glassfrog.md
+++ b/docs/integrations/sources/glassfrog.md
@@ -48,5 +48,6 @@ This Source is capable of syncing the following Streams:
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
-| 0.1.0 | 2022-06-16 | [13868g](https://github.com/airbytehq/airbyte/pull/13868) | Add Native Glassfrog Source Connector |
+| 0.1.1 | 2023-08-15 | [13868](https://github.com/airbytehq/airbyte/pull/13868) | Fix schema and tests |
+| 0.1.0 | 2022-06-16 | [13868](https://github.com/airbytehq/airbyte/pull/13868) | Add Native Glassfrog Source Connector |
 
diff --git a/docs/integrations/sources/google-ads.inapp.md b/docs/integrations/sources/google-ads.inapp.md
new file mode 100644
index 000000000000..ff2b18e3edc9
--- /dev/null
+++ b/docs/integrations/sources/google-ads.inapp.md
@@ -0,0 +1,72 @@
+## Prerequisites
+
+- A [Google Ads Account](https://support.google.com/google-ads/answer/6366720) [linked](https://support.google.com/google-ads/answer/7459601) to a Google Ads Manager account
+
+- (For Airbyte Open Source):
+  - A Developer Token
+  - OAuth credentials to authenticate your Google account
+
+
+## Setup guide
+
+
+
+To set up the Google Ads source connector with Airbyte Open Source, you will first need to obtain a developer token, as well as credentials for OAuth authentication. For more information on the steps involved, please refer to our [full documentation](https://docs.airbyte.com/integrations/sources/google-ads#setup-guide).
+
+
+
+
+### For Airbyte Cloud:
+
+1. Enter a **Source name** of your choosing.
+2. Click **Sign in with Google** to authenticate your Google Ads account. In the pop-up, select the appropriate Google account and click **Continue** to proceed.
+3. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344).
+4. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated.
+5. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language).
+6. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account.
+7. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en).  This field defaults to 14 days.
+8. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward.
+9. Click **Set up source** and wait for the tests to complete.
+
+
+
+
+### For Airbyte Open Source:
+
+1. Enter a **Source name** of your choosing.
+2. Enter the **Developer Token** you obtained from Google.
+3. To authenticate your Google account, enter your Google application's **Client ID**, **Client Secret**, **Refresh Token**, and optionally, the **Access Token**.
+4. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344).
+5. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated.
+6. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language).
+7. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account.
+8. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days.
+9. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward.
+10. Click **Set up source** and wait for the tests to complete.
+
+
+
+## Custom Query: Understanding Google Ads Query Language
+Additional streams for Google Ads can be dynamically created using custom queries. 
+
+The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v13/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source.
+
+Example GAQL Custom Query:
+```
+SELECT 
+    campaign.name, 
+    metrics.conversions, 
+    metrics.conversions_by_conversion_date 
+FROM ad_group
+```
+Note the segments.date is automatically added to the output, and does not need to be specified in the custom query. All custom reports will by synced by day.
+
+Each custom query in the input configuration must work for all the customer account IDs. Otherwise, the customer ID will be skipped for every query that fails the validation test. For example, if your query contains metrics fields in the select clause, it will not be executed against manager accounts.
+
+Follow Google's guidance on [Selectability between segments and metrics](https://developers.google.com/google-ads/api/docs/reporting/segmentation#selectability_between_segments_and_metrics) when editing custom queries or default stream schemas (which will also be turned into GAQL queries by the connector). Fields like `segments.keyword.info.text`, `segments.keyword.info.match_type`, `segments.keyword.ad_group_criterion` in the `SELECT` clause tell the query to only get the rows of data that have keywords and remove any row that is not associated with a keyword. This is often unobvious and undesired behavior and can lead to missing data records. If you need this field in the stream, add a new stream instead of editing the existing ones.
+
+:::info
+For an existing Google Ads source, when you are updating or removing Custom GAQL Queries, you should also subsequently refresh your source schema to pull in any changes.
+:::
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the [full documentation for Google Ads](https://docs.airbyte.com/integrations/sources/google-ads/).
diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md
index 1949da023521..13b35b9277d9 100644
--- a/docs/integrations/sources/google-ads.md
+++ b/docs/integrations/sources/google-ads.md
@@ -4,70 +4,89 @@ This page contains the setup guide and reference information for the Google Ads
 
 ## Prerequisites
 
-- A [Google Ads Account](https://support.google.com/google-ads/answer/6366720) [linked](https://support.google.com/google-ads/answer/7459601) to a [Google Ads Manager account](https://ads.google.com/home/tools/manager-accounts/)
+- A [Google Ads Account](https://support.google.com/google-ads/answer/6366720) [linked](https://support.google.com/google-ads/answer/7459601) to a Google Ads Manager account
 
-- (For Airbyte Open Source) [A developer token](#step-1-for-airbyte-oss-apply-for-a-developer-token)
-
+- (For Airbyte Open Source):
+  - A Developer Token
+  - OAuth credentials to authenticate your Google account
+  
 
 ## Setup guide
+
 
+
 ### Step 1: (For Airbyte Open Source) Apply for a developer token
 
+To set up the Google Ads source connector with Airbyte Open Source, you will need to obtain a developer token. This token allows you to access your data from the Google Ads API. Please note that Google is selective about which software and use cases are issued this token. The Airbyte team has worked with the Google Ads team to allowlist Airbyte and ensure you can get a developer token (see [issue 1981](https://github.com/airbytehq/airbyte/issues/1981) for more information on this topic).
+
+1. To proceed with obtaining a developer token, you will first need to create a [Google Ads Manager account](https://ads.google.com/home/tools/manager-accounts/). Standard Google Ads accounts cannot generate a developer token.
+
+2. To apply for the developer token, please follow [Google's instructions](https://developers.google.com/google-ads/api/docs/first-call/dev-token).
+
+3. When you apply for the token, make sure to include the following:
+   - Why you need the token (example: Want to run some internal analytics)
+   - That you will be using the Airbyte Open Source project
+   - That you have full access to the code base (because we're open source)
+   - That you have full access to the server running the code (because you're self-hosting Airbyte)
+
 :::note
-You'll need to create a [Google Ads Manager account](https://ads.google.com/home/tools/manager-accounts/) since Google Ads accounts cannot generate a developer token.
+You will _not_ be able to access your data via the Google Ads API until this token is approved. You cannot use a test developer token; it has to be at least a basic developer token. The approval process typically takes around 24 hours.
 :::
 
-To set up the Google Ads source connector with Airbyte Open Source, you'll need a developer token. This token allows you to access your data from the Google Ads API. However, Google is selective about which software and use cases can get a developer token. The Airbyte team has worked with the Google Ads team to allowlist Airbyte and make sure you can get a developer token (see [issue 1981](https://github.com/airbytehq/airbyte/issues/1981) for more information).
+### Step 2: (For Airbyte Open Source) Obtain your OAuth credentials
 
-Follow [Google's instructions](https://developers.google.com/google-ads/api/docs/first-call/dev-token) to apply for the token. Note that you will _not_ be able to access your data via the Google Ads API until this token is approved. You cannot use a test developer token; it has to be at least a basic developer token. It usually takes Google 24 hours to respond to these applications.
+If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate your Google Ads account:
 
-When you apply for a token, make sure to mention:
+- Client ID
+- Client Secret
+- Refresh Token
 
-- Why you need the token (example: Want to run some internal analytics)
-- That you will be using the Airbyte Open Source project
-- That you have full access to the code base (because we're open source)
-- That you have full access to the server running the code (because you're self-hosting Airbyte)
+Please refer to [Google's documentation](https://developers.google.com/identity/protocols/oauth2) for detailed instructions on how to obtain these credentials.
+
+### Step 3: Set up the Google Ads connector in Airbyte
 
-### Step 2: Set up the Google Ads connector in Airbyte
 
 
 
-**For Airbyte Cloud:**
+#### For Airbyte Cloud:
 
 To set up Google Ads as a source in Airbyte Cloud:
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Google Ads** from the Source type dropdown.
-4. Enter a **Name** for your source.
-5. Click **Sign in with Google** to authenticate your Google Ads account.
-6. Enter a comma-separated list of the [Customer ID(s)](https://support.google.com/google-ads/answer/1704344) for your account.
-7. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-8. (Optional) Enter a custom [GAQL](#custom-query-understanding-google-ads-query-language) query.
-9. (Optional) If the access to your account is through a [Google Ads Manager account](https://ads.google.com/home/tools/manager-accounts/), enter the [**Login Customer ID for Managed Accounts**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Google Ads Manager account.
-10. (Optional) Enter a [**Conversion Window**](https://support.google.com/google-ads/answer/3123169?hl=en).
-11. (Optional) Enter the **End Date** in YYYY-MM-DD format. The data added after this date will not be replicated.
-12. Click **Set up source**.
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Google Ads** from the list of available sources.
+4. Enter a **Source name** of your choosing.
+5. Click **Sign in with Google** to authenticate your Google Ads account. In the pop-up, select the appropriate Google account and click **Continue** to proceed.
+6. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344).
+7. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated.
+8. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language).
+9. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account.
+10. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days.
+11. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward.
+12. Click **Set up source** and wait for the tests to complete.
 
 
 
-**For Airbyte Open Source:**
+
+#### For Airbyte Open Source:
 
 To set up Google Ads as a source in Airbyte Open Source:
 
-1. Log into your Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Google Ads** from the Source type dropdown.
-4. Enter a **Name** for your source.
-5. Enter the [**Developer Token**](#step-1-for-airbyte-oss-apply-for-a-developer-token).
-6. To authenticate your Google account via OAuth, enter your Google application's [**Client ID**, **Client Secret**, **Refresh Token**, and optionally, the **Access Token**](https://developers.google.com/google-ads/api/docs/first-call/overview).
-7. Enter a comma-separated list of the [Customer ID(s)](https://support.google.com/google-ads/answer/1704344) for your account.
-8. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-9. (Optional) Enter a custom [GAQL](#custom-query-understanding-google-ads-query-language) query.
-10. (Optional) If the access to your account is through a [Google Ads Manager account](https://ads.google.com/home/tools/manager-accounts/), enter the [**Login Customer ID for Managed Accounts**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Google Ads Manager account.
-11. (Optional) Enter a [**Conversion Window**](https://support.google.com/google-ads/answer/3123169?hl=en).
-12. (Optional) Enter the **End Date** in YYYY-MM-DD format. The data added after this date will not be replicated.
-13. Click **Set up source**.
+1. Log in to your Airbyte Open Source account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Google Ads** from the list of available sources.
+4. Enter a **Source name** of your choosing.
+5. Enter the **Developer Token** you obtained from Google.
+6. To authenticate your Google account, enter your Google application's **Client ID**, **Client Secret**, **Refresh Token**, and optionally, the **Access Token**.
+7. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344).
+8. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated.
+9. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language).
+10. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account.
+11. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, see the section on [Conversion Windows](#note-on-conversion-windows) below, or refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days.
+12. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward.
+13. Click **Set up source** and wait for the tests to complete.
+
+
 
 ## Supported sync modes
 
@@ -76,12 +95,7 @@ The Google Ads source connector supports the following [sync modes](https://docs
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
-
-**Important note**:
-
-    Usage of Conversion Window may lead to duplicates in Incremental Sync,
-    because connector is forced to read data in the given range (Last Sync - Conversion window)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
@@ -96,18 +110,25 @@ The Google Ads source connector can sync the following tables. It can also sync
 - [ad_group_labels](https://developers.google.com/google-ads/api/fields/v11/ad_group_label)
 - [campaign_labels](https://developers.google.com/google-ads/api/fields/v11/campaign_label)
 - [click_view](https://developers.google.com/google-ads/api/reference/rpc/v11/ClickView)
-- [keyword](https://developers.google.com/google-ads/api/fields/v11/keyword_view)
 - [geographic](https://developers.google.com/google-ads/api/fields/v11/geographic_view)
+- [keyword](https://developers.google.com/google-ads/api/fields/v11/keyword_view)
 
 Note that `ad_groups`, `ad_group_ads`, and `campaigns` contain a `labels` field, which should be joined against their respective `*_labels` streams if you want to view the actual labels. For example, the `ad_groups` stream contains an `ad_group.labels` field, which you would join against the `ad_group_labels` stream's `label.resource_name` field.
 
+
 ### Report Tables
 
-- [campaigns](https://developers.google.com/google-ads/api/fields/v11/campaign)
 - [account_performance_report](https://developers.google.com/google-ads/api/docs/migration/mapping#account_performance)
+- [ad_groups](https://developers.google.com/google-ads/api/fields/v14/ad_group)
 - [ad_group_ad_report](https://developers.google.com/google-ads/api/docs/migration/mapping#ad_performance)
+- [ad_group_criterions](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion)
+- [ad_group_criterion_labels](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion_label)
+- [campaigns](https://developers.google.com/google-ads/api/fields/v11/campaign)
+- [campaign_budget](https://developers.google.com/google-ads/api/fields/v13/campaign_budget)
+- [customer_labels](https://developers.google.com/google-ads/api/fields/v14/customer_label)
 - [display_keyword_report](https://developers.google.com/google-ads/api/docs/migration/mapping#display_keyword_performance)
 - [display_topics_report](https://developers.google.com/google-ads/api/docs/migration/mapping#display_topics_performance)
+- [labels](https://developers.google.com/google-ads/api/fields/v14/label)
 - [shopping_performance_report](https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance)
 - [user_location_report](https://developers.google.com/google-ads/api/fields/v11/user_location_view)
 
@@ -115,22 +136,30 @@ Note that `ad_groups`, `ad_group_ads`, and `campaigns` contain a `labels` field,
 Due to Google Ads API constraints, the `click_view` stream retrieves data one day at a time and can only retrieve data newer than 90 days ago. Also, [metrics](https://developers.google.com/google-ads/api/fields/v11/metrics) cannot be requested for a Google Ads Manager account. Report streams are only available when pulling data from a non-manager account.
 :::
 
+:::warning
+Google Ads doesn't support `PERFORMACE_MAX` campaigns on `ad_group` or `ad` stream level, only on `campaign` level. 
+If you have this type of campaign Google will remove them from the results for the `ads` reports.
+More [info](https://github.com/airbytehq/airbyte/issues/11062) and [Google Discussions](https://groups.google.com/g/adwords-api/c/_mxbgNckaLQ).
+:::
+
 For incremental streams, data is synced up to the previous day using your Google Ads account time zone since Google Ads can filter data only by [date](https://developers.google.com/google-ads/api/fields/v11/ad_group_ad#segments.date) without time. Also, some reports cannot load data real-time due to Google Ads [limitations](https://support.google.com/google-ads/answer/2544985?hl=en).
-
 
 ## Custom Query: Understanding Google Ads Query Language
-Additional streams for Google Ads can be dynamically created using custom queries. 
+
+Additional streams for Google Ads can be dynamically created using custom queries.
 
 The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v13/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source.
 
 Example GAQL Custom Query:
+
 ```
-SELECT 
-    campaign.name, 
-    metrics.conversions, 
-    metrics.conversions_by_conversion_date 
+SELECT
+    campaign.name,
+    metrics.conversions,
+    metrics.conversions_by_conversion_date
 FROM ad_group
 ```
+
 Note the segments.date is automatically added to the output, and does not need to be specified in the custom query. All custom reports will by synced by day.
 
 Each custom query in the input configuration must work for all the customer account IDs. Otherwise, the customer ID will be skipped for every query that fails the validation test. For example, if your query contains metrics fields in the select clause, it will not be executed against manager accounts.
@@ -141,7 +170,17 @@ Follow Google's guidance on [Selectability between segments and metrics](https:/
 For an existing Google Ads source, when you are updating or removing Custom GAQL Queries, you should also subsequently refresh your source schema to pull in any changes.
 :::
 
-
+## Note on Conversion Windows
+
+In digital advertising, a 'conversion' typically refers to a user undertaking a desired action after viewing or interacting with an ad. This could be anything from clicking through to the advertiser's website, signing up for a newsletter, making a purchase, and so on. The conversion window is the period of time after a user sees or clicks on an ad during which their actions can still be credited to that ad.
+
+For example, imagine an online shoe store runs an ad and sets a conversion window of 30 days. If you click on that ad today, any purchases you make on the shoe store's site within the next 30 days will be considered conversions resulting from that ad.
+The length of the conversion window can vary depending on the goals of the advertiser and the nature of the product or service. Some businesses might set a shorter conversion window if they're promoting a limited-time offer, while others might set a longer window if they're advertising a product that consumers typically take a while to think about before buying.
+
+In essence, the conversion window is a tool for measuring the effectiveness of an advertising campaign. By tracking the actions users take after viewing or interacting with an ad, businesses can gain insight into how well their ads are working and adjust their strategies accordingly.
+
+In the case of configuring the Google Ads source connector, each time a sync is run the connector will retrieve all conversions that were active within the specified conversion window. For example, if you set a conversion window of 30 days, each time a sync is run, the connector will pull all conversions that were active within the past 30 days. Due to this mechanism, it may seem like the same campaigns, ad groups, or ads have different conversion numbers. However, in reality, each data record accurately reflects the number of conversions for that particular resource at the time of extracting the data from the Google Ads API.
+
 ## Performance considerations
 
 This source is constrained by the [Google Ads API limits](https://developers.google.com/google-ads/api/docs/best-practices/quotas)
@@ -151,15 +190,30 @@ Due to a limitation in the Google Ads API which does not allow getting performan
 ## Changelog
 
 | Version  | Date       | Pull Request                                             | Subject                                                                                                                              |
-|:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
+| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------- |
+| `0.7.4`  | 2023-07-28 | [28832](https://github.com/airbytehq/airbyte/pull/28832) | Update field descriptions                                                                                                            |
+| `0.7.3`  | 2023-07-24 | [28510](https://github.com/airbytehq/airbyte/pull/28510) | Set dates with client's timezone                                                                                                     |
+| `0.7.2`  | 2023-07-20 | [28535](https://github.com/airbytehq/airbyte/pull/28535) | UI improvement: Make the query field in custom reports a multi-line string field                                                     |
+| `0.7.1`  | 2023-07-17 | [28365](https://github.com/airbytehq/airbyte/pull/28365) | 0.3.1 and 0.3.2 follow up: make today the end date, not yesterday                                                                    |
+| `0.7.0`  | 2023-07-12 | [28246](https://github.com/airbytehq/airbyte/pull/28246) | Add new streams: labels, criterions, biddig strategies                                                                               |
+| `0.6.1`  | 2023-07-12 | [28230](https://github.com/airbytehq/airbyte/pull/28230) | Reduce amount of logs produced by the connector while working with big amount of data                                                |
+| `0.6.0`  | 2023-07-10 | [28078](https://github.com/airbytehq/airbyte/pull/28078) | Add new stream `Campaign Budget`                                                                                                     |
+| `0.5.0`  | 2023-07-07 | [28042](https://github.com/airbytehq/airbyte/pull/28042) | Add metrics & segment to `Campaigns` stream                                                                                          |
+| `0.4.3`  | 2023-07-05 | [27959](https://github.com/airbytehq/airbyte/pull/27959) | Add `audience` and `user_interest` streams                                                                                           |
+| `0.3.3`  | 2023-07-03 | [27913](https://github.com/airbytehq/airbyte/pull/27913) | Improve Google Ads exception handling (wrong customer ID)                                                                            |
+| `0.3.2`  | 2023-06-29 | [27835](https://github.com/airbytehq/airbyte/pull/27835) | Fix bug introduced in 0.3.1: update query template                                                                                   |
+| `0.3.1`  | 2023-06-26 | [27711](https://github.com/airbytehq/airbyte/pull/27711) | Refactor date slicing; make start date inclusive                                                                                     |
+| `0.3.0`  | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2                                                                                                                 |
+| `0.2.24` | 2023-06-06 | [27608](https://github.com/airbytehq/airbyte/pull/27608) | Improve Google Ads exception handling                                                                                                |
+| `0.2.23` | 2023-06-06 | [26905](https://github.com/airbytehq/airbyte/pull/26905) | Replace deprecated `authSpecification` in the connector specification with `advancedAuth`                                            |
 | `0.2.22` | 2023-06-02 | [26948](https://github.com/airbytehq/airbyte/pull/26948) | Refactor error messages; add `pattern_descriptor` for fields in spec                                                                 |
-| `0.2.21` | 2023-05-30 | [25314](https://github.com/airbytehq/airbyte/pull/25314) | add full refresh custom table `asset_group_listing_group_filter`                                                                     |
+| `0.2.21` | 2023-05-30 | [25314](https://github.com/airbytehq/airbyte/pull/25314) | Add full refresh custom table `asset_group_listing_group_filter`                                                                     |
 | `0.2.20` | 2023-05-30 | [25624](https://github.com/airbytehq/airbyte/pull/25624) | Add `asset` Resource to full refresh custom tables (GAQL Queries)                                                                    |
 | `0.2.19` | 2023-05-15 | [26209](https://github.com/airbytehq/airbyte/pull/26209) | Handle Token Refresh errors as `config_error`                                                                                        |
 | `0.2.18` | 2023-05-15 | [25947](https://github.com/airbytehq/airbyte/pull/25947) | Improve GAQL parser error message if multiple resources provided                                                                     |
 | `0.2.17` | 2023-05-11 | [25987](https://github.com/airbytehq/airbyte/pull/25987) | Categorized Config Errors Accurately                                                                                                 |
 | `0.2.16` | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types                                                                                                     |
-| `0.2.14` | 2023-03-21 | [24945](https://github.com/airbytehq/airbyte/pull/24945) | for custom google query fixed schema type for "data_type: ENUM" and "is_repeated: true" to array of strings                          |
+| `0.2.14` | 2023-03-21 | [24945](https://github.com/airbytehq/airbyte/pull/24945) | For custom google query fixed schema type for "data_type: ENUM" and "is_repeated: true" to array of strings                          |
 | `0.2.13` | 2023-03-21 | [24338](https://github.com/airbytehq/airbyte/pull/24338) | Migrate to v13                                                                                                                       |
 | `0.2.12` | 2023-03-17 | [22985](https://github.com/airbytehq/airbyte/pull/22985) | Specified date formatting in specification                                                                                           |
 | `0.2.11` | 2023-03-13 | [23999](https://github.com/airbytehq/airbyte/pull/23999) | Fix incremental sync for Campaigns stream                                                                                            |
@@ -216,9 +270,3 @@ Due to a limitation in the Google Ads API which does not allow getting performan
 | `0.1.3`  | 2021-07-23 | [4788](https://github.com/airbytehq/airbyte/pull/4788)   | Support main streams, fix bug with exception `DATE_RANGE_TOO_NARROW` for incremental streams                                         |
 | `0.1.2`  | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539)   | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support                                                                                      |
 | `0.1.1`  | 2021-06-23 | [4288](https://github.com/airbytehq/airbyte/pull/4288)   | Fix `Bugfix: Correctly declare required parameters`                                                                                  |
-
-
-
-
-For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Ads](https://docs.airbyte.com/integrations/sources/google-ads/).
-
diff --git a/docs/integrations/sources/google-analytics-data-api.md b/docs/integrations/sources/google-analytics-data-api.md
index 7debb0665e8e..857918a2bd6b 100644
--- a/docs/integrations/sources/google-analytics-data-api.md
+++ b/docs/integrations/sources/google-analytics-data-api.md
@@ -2,107 +2,212 @@
 
 This page contains the setup guide and reference information for the Google Analytics 4 source connector.
 
+Google Analytics 4 (GA4) is the latest version of Google Analytics, introduced in 2020. It offers a new data model that emphasizes events and user properties, rather than pageviews and sessions. This updated model allows for more flexibility and customization in reporting, and provides more accurate measurement of user behavior across various devices and platforms.
+
 :::note
+The [Google Analytics Universal Analytics (UA) connector](https://docs.airbyte.com/integrations/sources/google-analytics-v4) utilizes the older version of Google Analytics, which was the standard for tracking website and app user behavior before the introduction of GA4. Please note that the UA connector is being deprecated in favor of this one. As of July 1, 2023, standard Universal Analytics properties no longer process hits. For further reading on the transition from UA to GA4, refer to [Google's official support page](https://support.google.com/analytics/answer/11583528).
+:::
+
+## Prerequisites
+
+- A Google Analytics account with access to the GA4 property you want to sync
+
+## Setup guide
 
-[Google Analytics Universal Analytics (UA) connector](https://docs.airbyte.com/integrations/sources/google-analytics-v4), uses the older version of Google Analytics, which has been the standard for tracking website and app user behavior since 2012. 
+### For Airbyte Cloud
 
-Google Analytics 4 (GA4) connector is the latest version of Google Analytics, which was introduced in 2020. It offers a new data model that emphasizes events and user properties, rather than pageviews and sessions. This new model allows for more flexible and customizable reporting, as well as more accurate measurement of user behavior across devices and platforms.
+
+For **Airbyte Cloud** users, we highly recommend using OAuth for authentication, as this significantly simplifies the setup process by allowing you to authenticate your Google Analytics account directly in the Airbyte UI. Please follow the steps below to set up the connector using this method.
 
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Google Analytics 4 (GA4)** from the list of available sources.
+4. In the **Source name** field, enter a name to help you identify this source.
+5. Select **Authenticate via Google (Oauth)** from the dropdown menu and click **Authenticate your Google Analytics 4 (GA4) account**. This will open a pop-up window where you can log in to your Google account and grant Airbyte access to your Google Analytics account.
+6. Enter the **Property ID** whose events are tracked. This ID should be a numeric value, such as `123456789`. If you are unsure where to find this value, refer to [Google's documentation](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id).
+:::note
+If the Property Settings shows a "Tracking Id" such as "UA-123...-1", this denotes that the property is a Universal Analytics property, and the Analytics data for that property cannot be reported on using this connector. You can create a new Google Analytics 4 property by following [these instructions](https://support.google.com/analytics/answer/9744165?hl=en).
 :::
 
-## Prerequisites
+7. In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports.
+8. (Optional) In the **Custom Reports** field, you may optionally provide a JSON array describing any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports.
+9. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field.
 
-* JSON credentials for the service account that has access to Google Analytics. For more details check [instructions](https://support.google.com/analytics/answer/1009702)
-* OAuth 2.0 credentials for the service account that has access to Google Analytics
-* Property ID
+:::caution
 
-## Step 1: Set up Source
+It's important to consider how dimensions like `month` or `yearMonth` are specified. These dimensions organize the data according to your preferences.
+However, keep in mind that the data presentation is also influenced by the chosen date range for the report. In cases where a very specific date range is selected, such as a single day (**Data Request Interval (Days)** set to one day), duplicated data entries for each day might appear.
+To mitigate this, we recommend adjusting the **Data Request Interval (Days)** value to 364. By doing so, you can obtain more precise results and prevent the occurrence of duplicated data. 
 
-### Create a Service Account
+:::
 
-First, you need to select existing or create a new project in the Google Developers Console:
+10. Click **Set up source** and wait for the tests to complete.
 
-1. Sign in to the Google Account you are using for Google Analytics as an admin.
-2. Go to the [Service Accounts](https://console.developers.google.com/iam-admin/serviceaccounts) page.
-3. Click `Create service account`.
-4. Create a JSON key file for the service user. The contents of this file will be provided as the `credentials_json` in the UI when authorizing GA after you grant permissions \(see below\).
+
+
+
 
-### Add service account to the Google Analytics account
+### For Airbyte Open Source
 
-Use the service account email address to [add a user](https://support.google.com/analytics/answer/1009702) to the Google analytics view you want to access via the API. You will need to grant [Viewer permissions](https://support.google.com/analytics/answer/2884495).
+For **Airbyte Open Source** users, the recommended way to set up the Google Analytics 4 connector is to create a Service Account and set up a JSON key file for authentication. Please follow the steps below to set up the connector using this method.
 
-### Enable the APIs
+#### Create a Service Account for authentication
 
-1. Go to the [Google Analytics Reporting API dashboard](https://console.developers.google.com/apis/api/analyticsreporting.googleapis.com/overview) in the project for your service user. Enable the API for your account. You can set quotas and check usage.
-2. Go to the [Google Analytics API dashboard](https://console.developers.google.com/apis/api/analytics.googleapis.com/overview) in the project for your service user. Enable the API for your account.
+1. Sign in to the Google Account you are using for Google Analytics as an admin.
+2. Go to the [Service Accounts](https://console.developers.google.com/iam-admin/serviceaccounts) page in the Google Developers console.
+3. Select the project you want to use (or create a new one) and click **Continue**.
+4. Click **+ Create Service Account** at the top of the page.
+5. Enter a name for the service account, and optionally, a description. Click **Create and Continue**.
+6. Choose the role for the service account. We recommend the **Viewer** role (Read & Analyze permissions). Click **Continue**.
+7. Select your new service account from the list, and open the **Keys** tab. Click **Add Key** > **Create New Key**.
+8. Select **JSON** as the Key type. This will generate and download the JSON key file that you'll use for authentication. Click **Continue**.
 
-### Step 2: Set up the Google Analytics connector in Airbyte
+#### Enable the Google Analytics APIs
 
-**For Airbyte Cloud:**
+Before you can use the service account to access Google Analytics data, you need to enable the required APIs:
 
-1. [Login to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
-3. On the source setup page, select **Google Analytics 4 (GA4)** from the Source type dropdown and enter a name for this connector.
-4. Click `Authenticate your account` by selecting Oauth or Service Account for Authentication.
-5. Log in and Authorize the Google Analytics account.
-6. Enter the [**Property ID**](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) whose events are tracked.
-7. Enter the **Start Date** from which to replicate report data in the format YYYY-MM-DD. (Not applied to custom Cohort reports).
-8. Enter the **Custom Reports (Optional)** a JSON array describing the custom reports you want to sync from Google Analytics.
-9. Enter the **Data request time increment in days (Optional)**. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. (Not applied to custom Cohort reports).
+1. Go to the [Google Analytics Reporting API dashboard](https://console.developers.google.com/apis/api/analyticsreporting.googleapis.com/overview). Make sure you have selected the associated project for your service account, and enable the API. You can also set quotas and check usage.
+2. Go to the [Google Analytics API dashboard](https://console.developers.google.com/apis/api/analytics.googleapis.com/overview). Make sure you have selected the associated project for your service account, and enable the API.
 
-**For Airbyte Open Source:**
+#### Set up the Google Analytics connector in Airbyte
 
 1. Navigate to the Airbyte Open Source dashboard.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
-3. On the source setup page, select **Google Analytics 4 (GA4)** from the Source type dropdown and enter a name for this connector.
-4. Select Service Account for Authentication in dropdown list and enter **Service Account JSON Key** from Step 1.
-5. Enter the [**Property ID**](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) whose events are tracked.
-6. Enter the **Start Date** from which to replicate report data in the format YYYY-MM-DD. (Not applied to custom Cohort reports).
-7. Enter the **Custom Reports (Optional)** a JSON array describing the custom reports you want to sync from Google Analytics.
-8. Enter the **Data request time increment in days (Optional)**. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. (Not applied to custom Cohort reports).
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Google Analytics 4 (GA4)** from the list of available sources.
+4. Select **Service Account Key Authenication** dropdown list and enter **Service Account JSON Key** from Step 1.
+5. Enter the **Property ID** whose events are tracked. This ID should be a numeric value, such as `123456789`. If you are unsure where to find this value, refer to [Google's documentation](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id).
+:::note
+If the Property Settings shows a "Tracking Id" such as "UA-123...-1", this denotes that the property is a Universal Analytics property, and the Analytics data for that property cannot be reported on in the Data API. You can create a new Google Analytics 4 property by following [these instructions](https://support.google.com/analytics/answer/9744165?hl=en).
+:::
 
+6. In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports.
+7. (Optional) In the **Custom Reports** field, you may optionally provide a JSON array describing any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports.
+8. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field.
+
+:::caution
+
+It's important to consider how dimensions like `month` or `yearMonth` are specified. These dimensions organize the data according to your preferences.
+However, keep in mind that the data presentation is also influenced by the chosen date range for the report. In cases where a very specific date range is selected, such as a single day (**Data Request Interval (Days)** set to one day), duplicated data entries for each day might appear.
+To mitigate this, we recommend adjusting the **Data Request Interval (Days)** value to 364. By doing so, you can obtain more precise results and prevent the occurrence of duplicated data. 
+
+:::
+
+9. Click **Set up source** and wait for the tests to complete.
+
 
 ## Supported sync modes
 
 The Google Analytics source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
-## Supported Streams
+## Supported streams
 
 This connector outputs the following incremental streams:
 
-* Preconfigured streams:
-  * [daily_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [devices](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [four_weekly_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [locations](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [pages](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [traffic_sources](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [website_overview](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-  * [weekly_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
-* [Custom stream\(s\)](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+- Preconfigured streams:
+  - [daily_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [devices](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [four_weekly_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [locations](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [pages](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [traffic_sources](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [website_overview](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+  - [weekly_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
+- [Custom stream\(s\)](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport)
 
 ## Connector-specific features
 
-:::note
-
-  * Custom reports should be provided in format `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...], "cohortSpec": "", "pivots": ""}]`
-  * Both `pivots` and `cohortSpec` are optional. Detailed description of the `cohortSpec` and the `pivots` objects you can find [here](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/CohortSpec) and [here](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot).
-  * To enable Incremental sync for Custom reports, you need to include the `date` dimension (except for custom Cohort reports).
-:::
+### Custom Reports
+
+Custom reports in Google Analytics allow for flexibility in querying specific data tailored to your needs. You can define the following components:
+
+- **Name**: The name of the custom report.
+- **Dimensions**: An array of categories for data, such as city, user type, etc.
+- **Metrics**: An array of quantitative measurements, such as active users, page views, etc.
+- **CohortSpec**: (Optional) An object containing specific cohort analysis settings, such as cohort size and date range. More information on this object can be found in [the GA4 documentation](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/CohortSpec).
+- **Pivots**: (Optional) An array of pivot tables for data, such as page views by city, etc. More information on pivots can be found in [the GA4 documentation](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot).
+
+A full list of dimensions and metrics supported in the API can be found [here](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema). To ensure your dimensions and metrics are compatible for your GA4 property, you can use the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/).
+
+Custom reports should be constructed as an array of JSON objects in the following format:
+
+```json
+[
+  {
+    "name": "", 
+    "dimensions": ["", ...], 
+    "metrics": ["", ...], 
+    "cohortSpec": {/* cohortSpec object */},
+    "pivots": [{/* pivot object */}, ...]
+  }
+]
+```
+
+The following is an example of a basic User Engagement report to track sessions and bounce rate, segmented by city:
+
+```json
+[
+  {
+    "name": "User Engagement Report",
+    "dimensions": ["city"],
+    "metrics": ["sessions", "bounceRate"]
+  }
+]
+```
+
+By specifying a cohort with a 7-day range and pivoting on the city dimension, the report can be further tailored to offer a detailed view of engagement trends within the top 50 cities for the specified date range.
+
+```json
+[
+  {
+    "name": "User Engagement Report",
+    "dimensions": ["city"],
+    "metrics": ["sessions", "bounceRate"],
+    "cohortSpec": {
+      "cohorts": [
+        {
+          "name": "Last 7 Days",
+          "dateRange": {
+            "startDate": "2023-07-27",
+            "endDate": "2023-08-03"
+          }
+        }
+      ],
+      "cohortReportSettings": {
+        "accumulate": true
+      }
+    },
+    "pivots": [
+      {
+        "fieldNames": ["city"],
+        "limit": 50,
+        "metricAggregations": ["TOTAL"]
+      }
+    ]
+  }
+]
+```
+
+### Data Sampling and Data Request Intervals
+
+Data sampling in Google Analytics 4 refers to the process of estimating analytics data when the amount of data in an account exceeds Google's predefined compute thresholds. To mitigate the chances of data sampling being applied to the results, the **Data Request Interval** field allows users to specify the interval used when requesting data from the Google Analytics API.
+
+By setting the interval to 1 day, users can reduce the data processed per request, minimizing the likelihood of data sampling and ensuring more accurate results. While larger time intervals (up to 364 days) can speed up the sync, we recommend choosing a smaller value to prioritize data accuracy unless there is a specific need for faster synchronization at the expense of some potential inaccuracies. Please note that this field does _not_ apply to custom Cohort reports.
+
+Refer to the [Google Analytics documentation](https://support.google.com/analytics/topic/13384306?sjid=2450288706152247916-NA) for more information on data sampling.
 
 ## Performance Considerations
 
-[Google Analytics Data API Quotas docs](https://developers.google.com/analytics/devguides/reporting/data/v1/quotas).
+The Google Analytics connector is subject to Google Analytics Data API quotas. Please refer to [Google's documentation](https://developers.google.com/analytics/devguides/reporting/data/v1/quotas) for specific breakdowns on these quotas.
 
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `number`         | `number`     |       |
 | `array`          | `array`      |       |
@@ -112,6 +217,14 @@ This connector outputs the following incremental streams:
 
 | Version | Date       | Pull Request                                             | Subject                                                                       |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------|
+| 1.1.3   | 2023-08-04 | [29103](https://github.com/airbytehq/airbyte/pull/29103) | Update input field descriptions                   |
+| 1.1.2   | 2023-07-03 | [27909](https://github.com/airbytehq/airbyte/pull/27909) | Limit the page size of custom report streams                                  |
+| 1.1.1   | 2023-06-26 | [27718](https://github.com/airbytehq/airbyte/pull/27718) | Limit the page size when calling `check()`                                    |
+| 1.1.0   | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2                                                          |
+| 1.0.0   | 2023-06-22 | [26283](https://github.com/airbytehq/airbyte/pull/26283) | Added primary_key and lookback window                                         |
+| 0.2.7   | 2023-06-21 | [27531](https://github.com/airbytehq/airbyte/pull/27531) | Fix formatting                                                                |
+| 0.2.6   | 2023-06-09 | [27207](https://github.com/airbytehq/airbyte/pull/27207) | Improve api rate limit messages                                               |
+| 0.2.5   | 2023-06-08 | [27175](https://github.com/airbytehq/airbyte/pull/27175) | Improve Error Messages                                                        |
 | 0.2.4   | 2023-06-01 | [26887](https://github.com/airbytehq/airbyte/pull/26887) | Remove `authSpecification` from connector spec in favour of `advancedAuth`    |
 | 0.2.3   | 2023-05-16 | [26126](https://github.com/airbytehq/airbyte/pull/26126) | Fix pagination                                                                |
 | 0.2.2   | 2023-05-12 | [25987](https://github.com/airbytehq/airbyte/pull/25987) | Categorized Config Errors Accurately                                          |
diff --git a/docs/integrations/sources/google-analytics-v4.inapp.md b/docs/integrations/sources/google-analytics-v4.inapp.md
index cda9df66bce4..305fd218af5a 100644
--- a/docs/integrations/sources/google-analytics-v4.inapp.md
+++ b/docs/integrations/sources/google-analytics-v4.inapp.md
@@ -1,3 +1,15 @@
+:::caution
+
+**The Google Analytics (Universal Analytics) connector will be deprecated soon.**
+
+Google is phasing out Universal Analytics in favor of Google Analytics 4 (GA4). In consequence, we are deprecating the Google Analytics (Universal Analytics) connector and recommend that you migrate to the [Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) as soon as possible to ensure your syncs are not affected.
+
+Due to this deprecation, we will not be accepting new contributions for this source.
+
+For more information, see ["Universal Analytics is going away"](https://support.google.com/analytics/answer/11583528).
+
+:::
+
 ## Prerequisite
 
 * Administrator access to a Google Analytics 4 (GA4) property
@@ -12,11 +24,78 @@
 5. (Optional) Airbyte generates 8 default reports. To add more reports, you need to add **Custom Reports** as a JSON array describing the custom reports you want to sync from Google Analytics. See below for more information.
 6. (Optional) Enter the **Data request time increment in days**. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364.
 
-## (Optional) Custom reports
+## (Optional) Custom Reports
+Custom Reports allow for flexibility in the reporting dimensions and metrics to meet your specific use case. Use the [GA4 Query Explorer](https://ga-dev-tools.google/ga4/query-explorer/) to help build your report. To ensure your dimensions and metrics are compatible, you can also refer to the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/).
+
+A custom report is formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]`
+
+Example of a custom report:
+```
+[{
+  "name" : "page_views_and_users",    
+  "dimensions" :[ 
+    "ga:date",
+    "ga:pagePath",      
+    "ga:sessionDefaultChannelGrouping"
+  ],
+  "metrics" :[
+    "ga:screenPageViews",
+    "ga:totalUsers"
+  ]
+}]
+```
+Multiple custom reports should be entered with a comma separator. Each custom report is created as it's own stream. 
+Example of multiple custom reports:
+```
+[
+  {
+    "name" : "page_views_and_users",    
+    "dimensions" :[ 
+      "ga:date",
+      "ga:pagePath"
+    ],
+    "metrics" :[
+      "ga:screenPageViews",
+      "ga:totalUsers"
+    ]
+  },
+  {
+    "name" : "sessions_by_region",    
+    "dimensions" :[ 
+      "ga:date",
+      "ga:region"
+    ],
+    "metrics" :[
+      "ga:totalUsers",
+      "ga:sessions"
+    ]
+  }
+]
+```
+
+Custom reports can also include segments and filters to pull a subset of your data. The report should be formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...], "segments":  [""}]`
+
+* When using segments, make sure you also add the `ga:segment` dimension.
 
-* Custom reports in format `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]`
-* Custom report format when using segments and / or filters `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...], "segments":  [""}]`
-* When using segments, make sure you add the `ga:segment` dimension.
-* Custom reports: [Dimensions and metrics explorer](https://ga-dev-tools.web.app/dimensions-metrics-explorer/)
+Example of a custom report with segments and/or filters:
+```
+[{ "name" : "page_views_and_users",    
+	"dimensions" :[ 
+		"ga:date",
+		"ga:pagePath",      
+      "ga:segment"
+   ],
+   "metrics" :[
+      "ga:sessions",
+      "ga:totalUsers"
+   ],
+   "segments" :[
+      "ga:sessionSource!=(direct)"
+   ],
+   "filter" :[
+      "ga:sessionSource!=(direct);ga:sessionSource!=(not set)"
+   ]
+}]
+```
 
 For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Analytics 4 (GA4)](https://docs.airbyte.com/integrations/sources/google-analytics-v4).
diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md
index 43274b070635..2b32a9d7d7a6 100644
--- a/docs/integrations/sources/google-analytics-v4.md
+++ b/docs/integrations/sources/google-analytics-v4.md
@@ -4,9 +4,21 @@ This page contains the setup guide and reference information for the Google Anal
 
 This connector supports Universal Analytics properties through the [Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4).
 
+:::caution
+
+**The Google Analytics (Universal Analytics) connector will be deprecated soon.**
+
+Google is phasing out Universal Analytics in favor of Google Analytics 4 (GA4). In consequence, we are deprecating the Google Analytics (Universal Analytics) connector and recommend that you migrate to the [Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) as soon as possible to ensure your syncs are not affected.
+
+Due to this deprecation, we will not be accepting new contributions for this source.
+
+For more information, see ["Universal Analytics is going away"](https://support.google.com/analytics/answer/11583528).
+
+:::
+
 :::note
 
-Google Analytics Universal Analytics (UA) connector, uses the older version of Google Analytics, which has been the standard for tracking website and app user behavior since 2012. 
+Google Analytics Universal Analytics (UA) connector, uses the older version of Google Analytics, which has been the standard for tracking website and app user behavior since 2012.
 
 [Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) is the latest version of Google Analytics, which was introduced in 2020. It offers a new data model that emphasizes events and user properties, rather than pageviews and sessions. This new model allows for more flexible and customizable reporting, as well as more accurate measurement of user behavior across devices and platforms.
 
@@ -19,6 +31,7 @@ A Google Cloud account with [Viewer permissions](https://support.google.com/anal
 ## Setup guide
 
 
+
 **For Airbyte Cloud:**
 
 To set up Google Analytics as a source in Airbyte Cloud:
@@ -28,14 +41,15 @@ To set up Google Analytics as a source in Airbyte Cloud:
 3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown.
 4. For Name, enter a name for the Google Analytics connector.
 5. Authenticate your Google account via OAuth or Service Account Key Authentication.
-    - **(Recommended)** To authenticate your Google account via OAuth, click **Sign in with Google** and complete the authentication workflow.
-    - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Make sure the Service Account has the Project Viewer permission.
+   - To authenticate your Google account via OAuth, click **Sign in with Google** and complete the authentication workflow.
+   - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Make sure the Service Account has the Project Viewer permission.
 6. Enter the **Replication Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
 7. Enter the [**View ID**](https://ga-dev-tools.appspot.com/account-explorer/) for the Google Analytics View you want to fetch data from.
 8. Leave **Data request time increment in days (Optional)** blank or set to 1. For faster syncs, set this value to more than 1 but that might result in the Google Analytics API returning [sampled data](#sampled-data-in-reports), potentially causing inaccuracies in the returned results. The maximum allowed value is 364.
 
 
 
+
 **For Airbyte Open Source:**
 
 To set up Google Analytics as a source in Airbyte Open Source:
@@ -44,8 +58,8 @@ To set up Google Analytics as a source in Airbyte Open Source:
 2. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown.
 3. Enter a name for the Google Analytics connector.
 4. Authenticate your Google account via OAuth or Service Account Key Authentication:
-    - To authenticate your Google account via OAuth, enter your Google application's [client ID, client secret, and refresh token](https://developers.google.com/identity/protocols/oauth2).
-    - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Use the service account email address to [add a user](https://support.google.com/analytics/answer/1009702) to the Google analytics view you want to access via the API and grant [Read and Analyze permissions](https://support.google.com/analytics/answer/2884495).
+   - To authenticate your Google account via OAuth, enter your Google application's [client ID, client secret, and refresh token](https://developers.google.com/identity/protocols/oauth2).
+   - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Use the service account email address to [add a user](https://support.google.com/analytics/answer/1009702) to the Google analytics view you want to access via the API and grant [Read and Analyze permissions](https://support.google.com/analytics/answer/2884495).
 5. Enter the **Replication Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
 6. Enter the [**View ID**](https://ga-dev-tools.appspot.com/account-explorer/) for the Google Analytics View you want to fetch data from.
 7. Optionally, enter a JSON object as a string in the **Custom Reports** field. For details, refer to [Requesting custom reports](#requesting-custom-reports)
@@ -59,7 +73,7 @@ The Google Analytics source connector supports the following [sync modes](https:
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 :::caution
 
@@ -67,7 +81,6 @@ You need to add the service account email address on the account level, not the
 
 :::
 
-
 ## Supported streams
 
 The Google Analytics (Universal Analytics) source connector can sync the following tables:
@@ -92,16 +105,16 @@ Reach out to us on Slack or [create an issue](https://github.com/airbytehq/airby
 
 [Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4/limits-quotas)
 
-* Number of requests per day per project: 50,000
-* Number of requests per view (profile) per day: 10,000 (cannot be increased)
-* Number of requests per 100 seconds per project: 2,000
-* Number of requests per 100 seconds per user per project: 100 (can be increased in Google API Console to 1,000).
+- Number of requests per day per project: 50,000
+- Number of requests per view (profile) per day: 10,000 (cannot be increased)
+- Number of requests per 100 seconds per project: 2,000
+- Number of requests per 100 seconds per user per project: 100 (can be increased in Google API Console to 1,000).
 
 The Google Analytics connector should not run into the "requests per 100 seconds" limitation under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully and try increasing the `window_in_days` value.
 
 ## Sampled data in reports
 
-If you are not on the Google Analytics 360 tier, the Google Analytics API may return sampled data if the amount of data in your Google Analytics account exceeds Google's [pre-determined compute thresholds](https://support.google.com/analytics/answer/2637192?hl=en&ref_topic=2601030&visit_id=637868645346124317-2833523666&rd=1#thresholds&zippy=%2Cin-this-article). This means the data returned in the report is an estimate which may have some inaccuracy. This [Google page](https://support.google.com/analytics/answer/2637192) provides a comprehensive overview of how Google applies sampling to your data.  
+If you are not on the Google Analytics 360 tier, the Google Analytics API may return sampled data if the amount of data in your Google Analytics account exceeds Google's [pre-determined compute thresholds](https://support.google.com/analytics/answer/2637192?hl=en&ref_topic=2601030&visit_id=637868645346124317-2833523666&rd=1#thresholds&zippy=%2Cin-this-article). This means the data returned in the report is an estimate which may have some inaccuracy. This [Google page](https://support.google.com/analytics/answer/2637192) provides a comprehensive overview of how Google applies sampling to your data.
 
 In order to minimize the chances of sampling being applied to your data, Airbyte makes data requests to Google in one day increments (the smallest allowed date increment). This reduces the amount of data the Google API processes per request, thus minimizing the chances of sampling being applied. The downside of requesting data in one day increments is that it increases the time it takes to export your Google Analytics data. If sampling is not a concern, you can override this behavior by setting the optional `window_in_day` parameter to specify the number of days to look back and avoid sampling.
 When sampling occurs, a warning is logged to the sync log.
@@ -132,54 +145,58 @@ Here is an example input "Custom Reports" field:
 
 To create a list of dimensions, you can use default Google Analytics dimensions (listed below) or custom dimensions if you have some defined. Each report can contain no more than 7 dimensions, and they must all be unique. The default Google Analytics dimensions are:
 
-* `ga:browser`
-* `ga:city`
-* `ga:continent`
-* `ga:country`
-* `ga:date`
-* `ga:deviceCategory`
-* `ga:hostname`
-* `ga:medium`
-* `ga:metro`
-* `ga:operatingSystem`
-* `ga:pagePath`
-* `ga:region`
-* `ga:socialNetwork`
-* `ga:source`
-* `ga:subContinent`
-
-To create a list of metrics, use a default Google Analytics metric (values from the list below) or custom metrics if you have defined them.  
+- `ga:browser`
+- `ga:city`
+- `ga:continent`
+- `ga:country`
+- `ga:date`
+- `ga:deviceCategory`
+- `ga:hostname`
+- `ga:medium`
+- `ga:metro`
+- `ga:operatingSystem`
+- `ga:pagePath`
+- `ga:region`
+- `ga:socialNetwork`
+- `ga:source`
+- `ga:subContinent`
+
+To create a list of metrics, use a default Google Analytics metric (values from the list below) or custom metrics if you have defined them.
 A custom report can contain no more than 10 unique metrics. The default available Google Analytics metrics are:
 
-* `ga:14dayUsers`
-* `ga:1dayUsers`
-* `ga:28dayUsers`
-* `ga:30dayUsers`
-* `ga:7dayUsers`
-* `ga:avgSessionDuration`
-* `ga:avgTimeOnPage`
-* `ga:bounceRate`
-* `ga:entranceRate`
-* `ga:entrances`
-* `ga:exitRate`
-* `ga:exits`
-* `ga:newUsers`
-* `ga:pageviews`
-* `ga:pageviewsPerSession`
-* `ga:sessions`
-* `ga:sessionsPerUser`
-* `ga:uniquePageviews`
-* `ga:users`
+- `ga:14dayUsers`
+- `ga:1dayUsers`
+- `ga:28dayUsers`
+- `ga:30dayUsers`
+- `ga:7dayUsers`
+- `ga:avgSessionDuration`
+- `ga:avgTimeOnPage`
+- `ga:bounceRate`
+- `ga:entranceRate`
+- `ga:entrances`
+- `ga:exitRate`
+- `ga:exits`
+- `ga:newUsers`
+- `ga:pageviews`
+- `ga:pageviewsPerSession`
+- `ga:sessions`
+- `ga:sessionsPerUser`
+- `ga:uniquePageviews`
+- `ga:users`
 
 Incremental sync is supported only if you add `ga:date` dimension to your custom report.
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                      |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------|
-| 0.1.34  | 2023-01-27 | [22006](https://github.com/airbytehq/airbyte/pull/22006) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                     |
-| 0.1.33  | 2022-12-23 | [20858](https://github.com/airbytehq/airbyte/pull/20858) | Fix check connection                                                                         |  
-| 0.1.32  | 2022-11-04 | [18965](https://github.com/airbytehq/airbyte/pull/18965) | Fix for `discovery` stage, when `custom_reports` are provided with single stream as `dict`   |  
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------- |
+| 0.2.1   | 2023-07-11 | [28149](https://github.com/airbytehq/airbyte/pull/28149) | Specify date format to support datepicker in UI                                              |
+| 0.2.0   | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2                                                                         |
+| 0.1.36  | 2023-04-13 | [22223](https://github.com/airbytehq/airbyte/pull/22223) | Fix custom report with Segments dimensions                                                   |
+| 0.1.35  | 2023-05-31 | [26885](https://github.com/airbytehq/airbyte/pull/26885) | Remove `authSpecification` from spec in favour of `advancedAuth`                             |
+| 0.1.34  | 2023-01-27 | [22006](https://github.com/airbytehq/airbyte/pull/22006) | Set `AvailabilityStrategy` for streams explicitly to `None`                                  |
+| 0.1.33  | 2022-12-23 | [20858](https://github.com/airbytehq/airbyte/pull/20858) | Fix check connection                                                                         |
+| 0.1.32  | 2022-11-04 | [18965](https://github.com/airbytehq/airbyte/pull/18965) | Fix for `discovery` stage, when `custom_reports` are provided with single stream as `dict`   |
 | 0.1.31  | 2022-10-30 | [18670](https://github.com/airbytehq/airbyte/pull/18670) | Add `Custom Reports` schema validation on `check connection`                                 |
 | 0.1.30  | 2022-10-13 | [17943](https://github.com/airbytehq/airbyte/pull/17943) | Fix pagination                                                                               |
 | 0.1.29  | 2022-10-12 | [17905](https://github.com/airbytehq/airbyte/pull/17905) | Handle exceeded daily quota gracefully                                                       |
diff --git a/docs/integrations/sources/google-directory.md b/docs/integrations/sources/google-directory.md
index bded9621fbf6..b0e570f7544f 100644
--- a/docs/integrations/sources/google-directory.md
+++ b/docs/integrations/sources/google-directory.md
@@ -68,6 +68,7 @@ You should now be ready to use the Google Directory connector in Airbyte.
 
 | Version | Date       | Pull Request                                             | Subject                                                      |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------|
+| 0.2.1   | 2023-05-30 | [27236](https://github.com/airbytehq/airbyte/pull/27236) | Autoformat code                                              |
 | 0.2.0   | 2023-05-30 | [26775](https://github.com/airbytehq/airbyte/pull/26775) | Remove `authSpecification` from spec; update stream schemas. |
 | 0.1.9   | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524)   | Update connector fields title/description                    |
 | 0.1.8   | 2021-11-02 | [7409](https://github.com/airbytehq/airbyte/pull/7409)   | Support oauth (update publish)                               |
diff --git a/docs/integrations/sources/google-search-console.inapp.md b/docs/integrations/sources/google-search-console.inapp.md
index 3ee6ecbdc103..de2bc9bc4b35 100644
--- a/docs/integrations/sources/google-search-console.inapp.md
+++ b/docs/integrations/sources/google-search-console.inapp.md
@@ -1,13 +1,28 @@
 ## Prerequisite
 
-* Credentials to a Google Service Account (or Google Service Account with delegated Domain Wide Authority) or Google User Account
-​
+- A verified property in Google Search Console
+
+- Google Search Console API enabled for your project (**Airbyte Open Source** only)
+
+
 ## Setup guide
-​
-Click Authenticate your account to sign in with Google and authorize your account.
-1. Enter the site URL.
-2. Enter the Start Date in YYYY-MM-DD
-3. Click Sign in with Google to authenticate your account
-4. (Optional) Fill in the custom reports in format `{"name": "", "dimensions": ["", ...]}`
-
-For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Search Console](https://docs.airbyte.com/integrations/sources/google-search-console/).
\ No newline at end of file
+
+1. For **Source name**, enter a name to help you identify this source.
+2. For **Website URL Property**, enter the specific website property in Google Seach Console with data you want to replicate.
+3. For **Start Date**, use the provided datepicker or enter a date in the format `YYYY-MM-DD`. Any data created on or after this date will be replicated.
+4. To authenticate the connection:
+
+   
+   - **For Airbyte Cloud**: Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. More information on authentication methods can be found in our [full Google Search Console documentation](https://docs.airbyte.io/integrations/sources/google-search-console#setup-guide).
+   
+   
+   - (Recommended) To authenticate with a service account, select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}`.
+   - To authenticate with OAuth, select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. More information on authentication methods for Airbyte Open Source can be found in our [full Google Search Console documentation](https://docs.airbyte.io/integrations/sources/google-search-console#setup-guide).
+   
+
+5. (Optional) For **End Date**, you may optionally provide a date in the format `YYYY-MM-DD`. Any data created between the defined Start Date and End Date will be replicated. Leaving this field blank will replicate all data created on or after the Start Date to the present.
+6. (Optional) For **Custom Reports**, you may optionally provide an array of JSON objects representing any custom reports you wish to query the API with. Refer to the [Custom reports](https://docs.airbyte.com/integrations/sources/google-search-console#custom-reports) section in our full documentation for more information on formulating these reports.
+7. (Optional) For **Data Freshness**, you may choose whether to include "fresh" data that has not been finalized by Google, and may be subject to change. Please note that if you are using Incremental sync mode, we highly recommend leaving this option to its default value of `final`. Refer to the [Data Freshness](https://docs.airbyte.com/integrations/sources/google-search-console#data-freshness) section in our full documentation for more information on this parameter.
+8. Click **Set up source** and wait for the tests to complete.
+
+For detailed information on supported sync modes, supported streams, and performance considerations, refer to the full documentation for [Google Search Console](https://docs.airbyte.com/integrations/sources/google-search-console/).
diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md
index 844bb51260da..4aeff27233ee 100644
--- a/docs/integrations/sources/google-search-console.md
+++ b/docs/integrations/sources/google-search-console.md
@@ -1,163 +1,218 @@
 # Google Search Console
 
-This page contains the setup guide and reference information for the google search console source connector.
-
+This page contains the setup guide and reference information for the Google Search Console source connector.
 
 ## Prerequisites
 
-* A verified property in Google Search Console
-* Enable Google Search Console API for GCP project at [GCP console](https://console.cloud.google.com/apis/library/searchconsole.googleapis.com)
-* Credentials to a Google Service Account \(or Google Service Account with delegated Domain Wide Authority\) or Google User Account
-* Enable Google Search Console API
-
+- A verified property in Google Search Console
+
+- Google Search Console API enabled for your project (**Airbyte Open Source** only)
+
 
 ## Setup guide
-### Step 1: Set up google search console
-
-#### How to create the client credentials for Google Search Console, to use with Airbyte?
 
-You can either:
+### Step 1: Set up Google Search Console authentication
 
-* Use the existing `Service Account` for your Google Project with granted Admin Permissions
-* Use your personal Google User Account with oauth. If you choose this option, your account must have permissions to view the Google Search Console project you choose.
-* Create the new `Service Account` credentials for your Google Project, and grant Admin Permissions to it
-* Follow the `Delegating domain-wide authority` process to obtain the necessary permissions to your google account from the administrator of Workspace
+To authenticate the Google Search Console connector, you will need to use one of the following methods:
 
-### Creating a Google service account
+#### I: OAuth (Recommended for Airbyte Cloud)
 
-A service account's credentials include a generated email address that is unique and at least one public/private key pair. If domain-wide delegation is enabled, then a client ID is also part of the service account's credentials.
+
+You can authenticate using your Google Account with OAuth if you are the owner of the Google Search Console property or have view permissions. Follow [Google's instructions](https://support.google.com/webmasters/answer/7687615?sjid=11103698321670173176-NA) to ensure that your account has the necessary permissions (**Owner** or **Full User**) to view the Google Search Console property. This option is recommended for **Airbyte Cloud** users, as it significantly simplifies the setup process and allows you to authenticate the connection [directly from the Airbyte UI](#step-2-set-up-the-google-search-console-connector-in-airbyte).
+
 
-1. Open the [Service accounts page](https://console.developers.google.com/iam-admin/serviceaccounts)
-2. If prompted, select an existing project, or create a new project.
-3. Click `+ Create service account`.
-4. Under Service account details, type a `name`, `ID`, and `description` for the service account, then click `Create`.
-   * Optional: Under `Service account permissions`, select the `IAM roles` to grant to the service account, then click `Continue`.
-   * Optional: Under `Grant users access to this service account`, add the `users` or `groups` that are allowed to use and manage the service account.
-5. Go to [API Console/Credentials](https://console.cloud.google.com/apis/credentials), check the `Service Accounts` section, click on the Email address of service account you just created.
-6. Open `Details` tab and find `Show domain-wide delegation`, checkmark the `Enable Google Workspace Domain-wide Delegation`.
-7. On `Keys` tab click `+ Add key`, then click `Create new key`.
+
+To authenticate with OAuth in **Airbyte Open Source**, you will need to create an authentication app and obtain the following credentials and tokens:
 
-Your new public/private key pair should be now generated and downloaded to your machine as `.json` you can find it in the `Downloads` folder or somewhere else if you use another default destination for downloaded files. This file serves as the only copy of the private key. You are responsible for storing it securely. If you lose this key pair, you will need to generate a new one!
+- Client ID
+- Client Secret
+- Refresh Token
+- Access Token
 
-### Using the existing Service Account
+More information on the steps to create an OAuth app to access Google APIs and obtain these credentials can be found [in Google's documentation](https://developers.google.com/identity/protocols/oauth2).
 
-1. Go to [API Console/Credentials](https://console.cloud.google.com/apis/credentials), check the `Service Accounts` section, click on the Email address of service account you just created.
-2. Click on `Details` tab and find `Show domain-wide delegation`, checkmark the `Enable Google Workspace Domain-wide Delegation`.
-3. On `Keys` tab click `+ Add key`, then click `Create new key`.
+#### II: Google service account with JSON key file (Recommended for Airbyte Open Source)
 
-Your new public/private key pair should be now generated and downloaded to your machine as `.json` you can find it in the `Downloads` folder or somewhere else if you use another default destination for downloaded files. This file serves as the only copy of the private key. You are responsible for storing it securely. If you lose this key pair, you will need to generate a new one!
+You can authenticate the connection using a JSON key file associated with a Google service account. This option is recommended for **Airbyte Open Source** users. Follow the steps below to create a service account and generate the JSON key file:
 
-### Note
+1. Open the [Service Accounts page](https://console.developers.google.com/iam-admin/serviceaccounts).
+2. Select an existing project, or create a new project.
+3. At the top of the page, click **+ Create service account**.
+4. Enter a name and description for the service account, then click **Create and Continue**.
+5. Under **Service account permissions**, select the roles to grant to the service account, then click **Continue**. We recommend the **Viewer** role.
+   - Optional: Under **Grant users access to this service account**, you may specify the users or groups that are allowed to use and manage the service account.
+6. Go to the [API Console/Credentials](https://console.cloud.google.com/apis/credentials) and click on the email address of the service account you just created.
+7. In the **Keys** tab, click **+ Add key**, then click **Create new key**.
+8. Select **JSON** as the Key type. This will generate and download the JSON key file that you'll use for authentication. Click **Continue**.
 
-You can return to the [API Console/Credentials](https://console.cloud.google.com/apis/credentials) at any time to view the email address, public key fingerprints, and other information, or to generate additional public/private key pairs. For more details about service account credentials in the API Console, see [Service accounts](https://cloud.google.com/iam/docs/understanding-service-accounts) in the API Console help file.
+:::caution
+This file serves as the only copy of your JSON service key, and you will not be able to re-download it. Be sure to store it in a secure location.
+:::
 
-### Create a Service Account with delegated domain-wide authority
+:::note
+You can return to the [API Console/Credentials](https://console.cloud.google.com/apis/credentials) at any time to manage your service account or generate additional JSON keys. For more details about service account credentials, see [Google's IAM documentation](https://cloud.google.com/iam/docs/understanding-service-accounts).
+:::
 
-Follow the Google Documentation for performing [Delegating domain-wide authority](https://developers.google.com/identity/protocols/oauth2/service-account#delegatingauthority) to create a Service account with delegated domain-wide authority. This account must be created by an administrator of your Google Workspace. Please make sure to grant the following `OAuth scopes` to the service user:
+#### Note on delegating domain-wide authority to the service account
 
-* `https://www.googleapis.com/auth/webmasters.readonly`
+Domain-wide delegation is a powerful feature that allows service accounts to access users' data across an organization's Google Workspace environment through 'impersonation'. This authority is necessary in certain use cases, such as when a service account needs broad access across multiple users and services within a domain.
 
-At the end of this process, you should have JSON credentials to this Google Service Account.
+:::note
+Only the super admin of your Google Workspace domain can enable domain-wide delegation of authority to a service account.
+:::
 
-## Step 2: Set up the google search console connector in Airbyte
+To enable delegated domain-wide authority, follow the steps listed in the [Google documentation](https://developers.google.com/identity/protocols/oauth2/service-account#delegatingauthority). Please make sure to grant the following OAuth scopes to the service account:
 
-
-**For Airbyte Cloud:**
-
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**.
-3. On the Set up the source page, enter the name for the google search console connector and select **google search console** from the Source type dropdown.
-4. Click Authenticate your account to sign in with Google and authorize your account.
-5. Fill in the `site_urls` field.
-6. Fill in the `start date` field.
-7. Fill in the `custom reports` (optionally) in format `{"name": "", "dimensions": ["", ...]}`
-8. Fill in the `data_state` (optionally) in case you want to sync fresher data use `all' value, otherwise use 'final'.
-8. You should be ready to sync data.
-
+- `https://www.googleapis.com/auth/webmasters.readonly`
 
-
-**For Airbyte Open Source:**
-
-1. Fill in the `service_account_info` and `email` fields for authentication.
-2. Fill in the `site_urls` field.
-3. Fill in the `start date` field.
-4. Fill in the `custom reports` (optionally) in format `{"name": "", "dimensions": ["", ...]}`
-5. Fill in the `data_state` (optionally) in case you want to sync fresher data use `all' value, otherwise use 'final'. 
-6. You should be ready to sync data.
+For more information on this topic, please refer to [this Google article](https://support.google.com/a/answer/162106?hl=en).
 
 
+### Step 2: Set up the Google Search Console connector in Airbyte
+
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Google Search Console** from the list of available sources.
+4. For **Source name**, enter a name to help you identify this source.
+5. For **Website URL Property**, enter the specific website property in Google Seach Console with data you want to replicate.
+6. For **Start Date**, use the provided datepicker or enter a date in the format `YYYY-MM-DD`. Any data created on or after this date will be replicated.
+7. To authenticate the connection:
+
+   
+   - **For Airbyte Cloud**: Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account.
+   
+   
+   - **For Airbyte Open Source**:
+      - (Recommended) Select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}`
+      - Select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**.
+   
+
+8. (Optional) For **End Date**, you may optionally provide a date in the format `YYYY-MM-DD`. Any data created between the defined Start Date and End Date will be replicated. Leaving this field blank will replicate all data created on or after the Start Date to the present.
+9. (Optional) For **Custom Reports**, you may optionally provide an array of JSON objects representing any custom reports you wish to query the API with. Refer to the [Custom reports](#custom-reports) section below for more information on formulating these reports.
+10. (Optional) For **Data Freshness**, you may choose whether to include "fresh" data that has not been finalized by Google, and may be subject to change. Please note that if you are using Incremental sync mode, we highly recommend leaving this option to its default value of `final`. Refer to the [Data Freshness](#data-freshness) section below for more information on this parameter.
+11. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
-The Google Search Console Source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-
+The Google Search Console Source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 :::note
-   The granularity for the cursor is 1 day, so Incremental Sync in Append mode may result in duplicating the data.
+The granularity for the cursor is 1 day, so Incremental Sync in Append mode may result in duplicating the data.
 :::
 
-:::note
-    Parameter `data_state='all'` should not be used with Incremental Sync mode as it may cause data loss.
-:::
+## Supported streams
+
+- [Sites](https://developers.google.com/webmaster-tools/search-console-api-original/v3/sites/get)
+- [Sitemaps](https://developers.google.com/webmaster-tools/search-console-api-original/v3/sitemaps/list)
+- [Full Analytics report](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query) \(this stream has a long sync time because it is very detailed, use with care\)
+- [Analytics report by country](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics report by date](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics report by device](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics report by page](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics report by query](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics keyword report](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics keyword report by page](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics page report](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics site report by page](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- [Analytics site report by site](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
+- Analytics report by custom dimensions
+
+## Connector-specific configurations
+
+### Custom reports
+
+Custom reports allow you to query the API with a custom set of dimensions to group results by. Results are grouped in the order that you supply these dimensions. Each custom report should be constructed as a JSON object in the following format:
+
+```json
+{
+   "name": "",
+   "dimensions": ["", "", ...]
+   }
+```
+
+The available dimensions are:
 
-## Supported Streams
+- `country`
+- `date`
+- `device`
+- `page`
+- `query`
+- `searchAppearance`
 
-* [Sites](https://developers.google.com/webmaster-tools/search-console-api-original/v3/sites/get)
-* [Sitemaps](https://developers.google.com/webmaster-tools/search-console-api-original/v3/sitemaps/list)
-* [Full Analytics report](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query) \(this stream has a long sync time because it is very detailed, use with care\)
-* [Analytics report by country](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
-* [Analytics report by date](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
-* [Analytics report by device](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
-* [Analytics report by page](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
-* [Analytics report by query](https://developers.google.com/webmaster-tools/search-console-api-original/v3/searchanalytics/query)
-* Analytics report by custom dimensions
+For example, to query the API for a report that groups results by country, then by date, you could enter the following custom report:
 
+```json
+[
+   {
+      "name": "country_date",
+      "dimensions": ["country", "date"]
+   }
+]
+```
+
+You can use the [Google APIS Explorer](https://developers.google.com/webmaster-tools/v1/searchanalytics/query) to build and test the reports you want to use.
+
+### Data Freshness
+
+The **Data Freshness** parameter deals with the "freshness", or finality of the data that is being queried.
+
+- `final`: The query will include only finalized, stable data. This is data that has been processed, verified, and is unlikely to change. When you select this option, you are querying for the definitive statistics and information that Google has analyzed and confirmed.
+- `all`: The query will return both finalized data and what Google terms "fresh" data. Fresh data includes more recent data that hasn't gone through the full processing and verification that finalized data has. This option can give you more up-to-the-minute insights, but it may be subject to change as Google continues to process and analyze it.
+
+:::caution
+When using Incremental Sync mode, we recommend leaving this parameter to its default state of `final`, as the `all` option may cause discrepancies between the data in your destination table and the finalized data in Google Search Console.
+:::
 
 ## Performance considerations
 
 This connector attempts to back off gracefully when it hits Reports API's rate limits. To find more information about limits, see [Usage Limits](https://developers.google.com/webmaster-tools/limits) documentation.
 
-
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `number`         | `number`     |       |
 | `array`          | `array`      |       |
 | `object`         | `object`     |       |
 
-
 ## Changelog
 
-| Version  | Date       | Pull Request                                                                                                  | Subject                                                                  |
-|:---------|:-----------|:--------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------|
-| `1.0.1`  | 2023-05-30 | [26746](https://github.com/airbytehq/airbyte/pull/26746)                                                      | Remove `authSpecification` from connector spec in favour of advancedAuth |
-| `1.0.0`  | 2023-05-24 | [26452](https://github.com/airbytehq/airbyte/pull/26452)                                                      | Add data_state parameter to specification                                |
-| `0.1.22` | 2023-03-20 | [22295](https://github.com/airbytehq/airbyte/pull/22295)                                                      | Update specification examples                                            |
-| `0.1.21` | 2023-02-14 | [22984](https://github.com/airbytehq/airbyte/pull/22984)                                                      | Specified date formatting in specification                               |
-| `0.1.20` | 2023-02-02 | [22334](https://github.com/airbytehq/airbyte/pull/22334)                                                      | Turn on default HttpAvailabilityStrategy                                 |
-| `0.1.19` | 2023-01-27 | [22007](https://github.com/airbytehq/airbyte/pull/22007)                                                      | Set `AvailabilityStrategy` for streams explicitly to `None`              |
-| `0.1.18` | 2022-10-27 | [18568](https://github.com/airbytehq/airbyte/pull/18568)                                                      | Improved config validation: custom_reports.dimension                     |
-| `0.1.17` | 2022-10-08 | [17751](https://github.com/airbytehq/airbyte/pull/17751)                                                      | Improved config validation: start_date, end_date, site_urls              |
-| `0.1.16` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304)                                                      | Migrate to per-stream state.                                             |
-| `0.1.15` | 2022-09-16 | [16819](https://github.com/airbytehq/airbyte/pull/16819)                                                      | Check available site urls to avoid 403 error on sync                     |
-| `0.1.14` | 2022-09-08 | [16433](https://github.com/airbytehq/airbyte/pull/16433)                                                      | Add custom analytics stream.                                             |
-| `0.1.13` | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924)                                                      | Remove `additionalProperties` field from specs                           |
-| `0.1.12` | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482)                                                      | Update input configuration copy                                          |
-| `0.1.11` | 2022-01-05 | [9186](https://github.com/airbytehq/airbyte/pull/9186) [9194](https://github.com/airbytehq/airbyte/pull/9194) | Fix incremental sync: keep all urls in state object                      |
-| `0.1.10` | 2021-12-23 | [9073](https://github.com/airbytehq/airbyte/pull/9073)                                                        | Add slicing by date range                                                |
-| `0.1.9`  | 2021-12-22 | [9047](https://github.com/airbytehq/airbyte/pull/9047)                                                        | Add 'order' to spec.json props                                           |
-| `0.1.8`  | 2021-12-21 | [8248](https://github.com/airbytehq/airbyte/pull/8248)                                                        | Enable Sentry for performance and errors tracking                        |
-| `0.1.7`  | 2021-11-26 | [7431](https://github.com/airbytehq/airbyte/pull/7431)                                                        | Add default `end_date` param value                                       |
-| `0.1.6`  | 2021-09-27 | [6460](https://github.com/airbytehq/airbyte/pull/6460)                                                        | Update OAuth Spec File                                                   |
-| `0.1.4`  | 2021-09-23 | [6394](https://github.com/airbytehq/airbyte/pull/6394)                                                        | Update Doc link Spec File                                                |
-| `0.1.3`  | 2021-09-23 | [6405](https://github.com/airbytehq/airbyte/pull/6405)                                                        | Correct Spec File                                                        |
-| `0.1.2`  | 2021-09-17 | [6222](https://github.com/airbytehq/airbyte/pull/6222)                                                        | Correct Spec File                                                        |
-| `0.1.1`  | 2021-09-22 | [6315](https://github.com/airbytehq/airbyte/pull/6315)                                                        | Verify access to all sites when performing connection check              |
-| `0.1.0`  | 2021-09-03 | [5350](https://github.com/airbytehq/airbyte/pull/5350)                                                        | Initial Release                                                          |
+| Version  | Date       | Pull Request                                                                                                  | Subject                                                                                                                        |
+| :------- | :--------- | :------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------- |
+| `1.3.1`  | 2023-08-24 | [29329](https://github.com/airbytehq/airbyte/pull/29329)                                                      | Update tooltip descriptions                     |
+| `1.3.0`  | 2023-08-24 | [29750](https://github.com/airbytehq/airbyte/pull/29750)                                                      | Add new `Keyword-Site-Report-By-Site` stream |
+| `1.2.2`  | 2023-08-23 | [29741](https://github.com/airbytehq/airbyte/pull/29741)                                                      | Handle `HTTP-401`, `HTTP-403` errors |
+| `1.2.1`  | 2023-07-04 | [27952](https://github.com/airbytehq/airbyte/pull/27952)                                                      | Removed deprecated `searchType`, added `discover`(Discover results) and `googleNews`(Results from news.google.com, etc.) types |
+| `1.2.0`  | 2023-06-29 | [27831](https://github.com/airbytehq/airbyte/pull/27831)                                                      | Add new streams                                                                                                                |
+| `1.1.0`  | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738)                                                      | License Update: Elv2                                                                                                           |
+| `1.0.2`  | 2023-06-13 | [27307](https://github.com/airbytehq/airbyte/pull/27307)                                                      | Fix `data_state` config typo                                                                                                   |
+| `1.0.1`  | 2023-05-30 | [26746](https://github.com/airbytehq/airbyte/pull/26746)                                                      | Remove `authSpecification` from connector spec in favour of advancedAuth                                                       |
+| `1.0.0`  | 2023-05-24 | [26452](https://github.com/airbytehq/airbyte/pull/26452)                                                      | Add data_state parameter to specification                                                                                      |
+| `0.1.22` | 2023-03-20 | [22295](https://github.com/airbytehq/airbyte/pull/22295)                                                      | Update specification examples                                                                                                  |
+| `0.1.21` | 2023-02-14 | [22984](https://github.com/airbytehq/airbyte/pull/22984)                                                      | Specified date formatting in specification                                                                                     |
+| `0.1.20` | 2023-02-02 | [22334](https://github.com/airbytehq/airbyte/pull/22334)                                                      | Turn on default HttpAvailabilityStrategy                                                                                       |
+| `0.1.19` | 2023-01-27 | [22007](https://github.com/airbytehq/airbyte/pull/22007)                                                      | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                    |
+| `0.1.18` | 2022-10-27 | [18568](https://github.com/airbytehq/airbyte/pull/18568)                                                      | Improved config validation: custom_reports.dimension                                                                           |
+| `0.1.17` | 2022-10-08 | [17751](https://github.com/airbytehq/airbyte/pull/17751)                                                      | Improved config validation: start_date, end_date, site_urls                                                                    |
+| `0.1.16` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304)                                                      | Migrate to per-stream state.                                                                                                   |
+| `0.1.15` | 2022-09-16 | [16819](https://github.com/airbytehq/airbyte/pull/16819)                                                      | Check available site urls to avoid 403 error on sync                                                                           |
+| `0.1.14` | 2022-09-08 | [16433](https://github.com/airbytehq/airbyte/pull/16433)                                                      | Add custom analytics stream.                                                                                                   |
+| `0.1.13` | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924)                                                      | Remove `additionalProperties` field from specs                                                                                 |
+| `0.1.12` | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482)                                                      | Update input configuration copy                                                                                                |
+| `0.1.11` | 2022-01-05 | [9186](https://github.com/airbytehq/airbyte/pull/9186) [9194](https://github.com/airbytehq/airbyte/pull/9194) | Fix incremental sync: keep all urls in state object                                                                            |
+| `0.1.10` | 2021-12-23 | [9073](https://github.com/airbytehq/airbyte/pull/9073)                                                        | Add slicing by date range                                                                                                      |
+| `0.1.9`  | 2021-12-22 | [9047](https://github.com/airbytehq/airbyte/pull/9047)                                                        | Add 'order' to spec.json props                                                                                                 |
+| `0.1.8`  | 2021-12-21 | [8248](https://github.com/airbytehq/airbyte/pull/8248)                                                        | Enable Sentry for performance and errors tracking                                                                              |
+| `0.1.7`  | 2021-11-26 | [7431](https://github.com/airbytehq/airbyte/pull/7431)                                                        | Add default `end_date` param value                                                                                             |
+| `0.1.6`  | 2021-09-27 | [6460](https://github.com/airbytehq/airbyte/pull/6460)                                                        | Update OAuth Spec File                                                                                                         |
+| `0.1.4`  | 2021-09-23 | [6394](https://github.com/airbytehq/airbyte/pull/6394)                                                        | Update Doc link Spec File                                                                                                      |
+| `0.1.3`  | 2021-09-23 | [6405](https://github.com/airbytehq/airbyte/pull/6405)                                                        | Correct Spec File                                                                                                              |
+| `0.1.2`  | 2021-09-17 | [6222](https://github.com/airbytehq/airbyte/pull/6222)                                                        | Correct Spec File                                                                                                              |
+| `0.1.1`  | 2021-09-22 | [6315](https://github.com/airbytehq/airbyte/pull/6315)                                                        | Verify access to all sites when performing connection check                                                                    |
+| `0.1.0`  | 2021-09-03 | [5350](https://github.com/airbytehq/airbyte/pull/5350)                                                        | Initial Release                                                                                                                |
diff --git a/docs/integrations/sources/google-sheets.inapp.md b/docs/integrations/sources/google-sheets.inapp.md
new file mode 100644
index 000000000000..3e8374aef9d2
--- /dev/null
+++ b/docs/integrations/sources/google-sheets.inapp.md
@@ -0,0 +1,45 @@
+## Prerequisites
+- Spreadsheet Link - The link to the Google spreadsheet you want to sync.
+- A Google Workspace user with access to the spreadsheet
+
+:::info
+The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance.
+:::
+
+## Setup guide
+
+1. For **Source name**, enter a name to help you identify this source.
+2. Select your authentication method:
+
+
+
+#### For Airbyte Cloud
+
+- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow.
+
+
+
+
+#### For Airbyte Open Source
+
+- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format:
+
+    ```js
+    { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... }
+    ```
+
+- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token.
+
+For detailed instructions on how to generate a service account key or OAuth credentials, refer to the [full documentation](https://docs.airbyte.io/integrations/sources/google-sheets#setup-guide).
+
+
+
+3. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**.
+4. (Optional) You may enable the option to **Convert Column Names to SQL-Compliant Format**. Enabling this option will allow the connector to convert column names to a standardized, SQL-friendly format. For example, a column name of `Café Earnings 2022` will be converted to `cafe_earnings_2022`. We recommend enabling this option if your target destination is SQL-based (ie Postgres, MySQL). Set to false by default.
+5. Click **Set up source** and wait for the tests to complete.
+
+### Output schema
+
+- Airbyte only supports replicating [Grid](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetType) sheets.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Sheets](https://docs.airbyte.com/integrations/sources/google-sheets/).
diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md
index a605b282c8f0..b40cbf6a2317 100644
--- a/docs/integrations/sources/google-sheets.md
+++ b/docs/integrations/sources/google-sheets.md
@@ -1,112 +1,176 @@
 # Google Sheets
 
-This page guides you through the process of setting up the Google Sheets source connector.
+This page contains the setup guide and reference information for the Google Sheets source connector.
 
 :::info
-The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance.
+The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. Each sheet (tab) within a spreadsheet can be replicated. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. No other files in your Google Drive are accessed.
 :::
 
+### Prerequisites
+- Spreadsheet Link - The link to the Google spreadsheet you want to sync.
+
+- **For Airbyte Cloud** A Google Workspace user with access to the spreadsheet  
+
+
+-  **For Airbyte Open Source:** 
+  - A GCP project
+  - Enable the Google Sheets API in your GCP project
+  - Service Account Key with access to the Spreadsheet you want to replicate
+
+
 ## Setup guide
 
+The Google Sheets source connector supports authentication via either OAuth or Service Account Key Authentication.
+
 
-**For Airbyte Cloud:**
+For **Airbyte Cloud** users, we highly recommend using OAuth, as it significantly simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-google-sheets-source-connector-in-airbyte).
+
+
+
+
+
+For **Airbyte Open Source** users, we recommend using Service Account Key Authentication. Follow the steps below to create a service account, generate a key, and enable the Google Sheets API.
+
+:::note
+If you prefer to use OAuth for authentication with **Airbyte Open Source**, you can follow [Google's OAuth instructions](https://developers.google.com/identity/protocols/oauth2) to create an authentication app. Be sure to set the scopes to `https://www.googleapis.com/auth/spreadsheets.readonly`. You will need to obtain your client ID, client secret, and refresh token for the connector setup.
+:::
+
+### Set up the service account key (Airbyte Open Source)
+
+#### Create a service account
+
+1. Open the [Service Accounts page](https://console.cloud.google.com/projectselector2/iam-admin/serviceaccounts) in your Google Cloud console.
+2. Select an existing project, or create a new project.
+3. At the top of the page, click **+ Create service account**.
+4. Enter a name and description for the service account, then click **Create and Continue**.
+5. Under **Service account permissions**, select the roles to grant to the service account, then click **Continue**. We recommend the **Viewer** role.
+
+#### Generate a key
+
+1. Go to the [API Console/Credentials](https://console.cloud.google.com/apis/credentials) page and click on the email address of the service account you just created.
+2. In the **Keys** tab, click **+ Add key**, then click **Create new key**.
+3. Select **JSON** as the Key type. This will generate and download the JSON key file that you'll use for authentication. Click **Continue**.
+
+#### Enable the Google Sheets API
+
+1. Go to the [API Console/Library](https://console.cloud.google.com/apis/library) page.
+2. Make sure you have selected the correct project from the top.
+3. Find and select the **Google Sheets API**.
+4. Click **ENABLE**.
+
+If your spreadsheet is viewable by anyone with its link, no further action is needed. If not, [give your Service account access to your spreadsheet](https://youtu.be/GyomEw5a2NQ%22).
+
+
+
+### Set up the Google Sheets source connector in Airbyte
 
 To set up Google Sheets as a source in Airbyte Cloud:
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
-3. On the Set up the source page, select **Google Sheets** from the **Source type** dropdown.
-4. Enter a name for the Google Sheets connector.
-5. Authenticate your Google account via OAuth or Service Account Key Authentication.
-    - **(Recommended)** To authenticate your Google account via OAuth, click **Sign in with Google** and complete the authentication workflow.
-    - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Make sure the Service Account has the Project Viewer permission. If your spreadsheet is viewable by anyone with its link, no further action is needed. If not, [give your Service account access to your spreadsheet](https://youtu.be/GyomEw5a2NQ%22).
-6. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**.
-7. For **Row Batch Size**, define the number of records you want the Google API to fetch at a time. The default value is 200.
-
+3. Find and select **Google Sheets** from the list of available sources.
+4. For **Source name**, enter a name to help you identify this source.
+5. Select your authentication method:
 
+
+
+#### For Airbyte Cloud
+
+- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow.
+
+
 
-**For Airbyte Open Source:**
 
-To set up Google Sheets as a source in Airbyte Open Source:
+#### For Airbyte Open Source
 
-1. [Enable the Google Cloud Platform APIs for your personal or organization account](https://support.google.com/googleapi/answer/6158841?hl=en).
+- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format:
 
-    :::info
-    The connector only finds the spreadsheet you want to replicate; it does not access any of your other files in Google Drive.
-    :::
+    ```js
+    { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... }
+    ```
+
+- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token.
 
-2. Go to the Airbyte UI and in the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
-3. On the Set up the source page, select **Google Sheets** from the Source type dropdown.
-4. Enter a name for the Google Sheets connector.
-5. Authenticate your Google account via OAuth or Service Account Key Authentication:
-    - To authenticate your Google account via OAuth, enter your Google application's [client ID, client secret, and refresh token](https://developers.google.com/identity/protocols/oauth2).
-    - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Make sure the Service Account has the Project Viewer permission. If your spreadsheet is viewable by anyone with its link, no further action is needed. If not, [give your Service account access to your spreadsheet](https://youtu.be/GyomEw5a2NQ%22).
-6. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**.
 
-   
+
+6. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**.
+7. (Optional) You may enable the option to **Convert Column Names to SQL-Compliant Format**. Enabling this option will allow the connector to convert column names to a standardized, SQL-friendly format. For example, a column name of `Café Earnings 2022` will be converted to `cafe_earnings_2022`. We recommend enabling this option if your target destination is SQL-based (ie Postgres, MySQL). Set to false by default.
+8. Click **Set up source** and wait for the tests to complete.
 
 ### Output schema
 
 Each sheet in the selected spreadsheet is synced as a separate stream. Each selected column in the sheet is synced as a string field.
 
-**Note: Sheet names and column headers must contain only alphanumeric characters or `_`, as specified in the** [**Airbyte Protocol**](../../understanding-airbyte/airbyte-protocol.md). For example, if your sheet or column header is named `the data`, rename it to `the_data`. This restriction does not apply to non-header cell values.
-
 Airbyte only supports replicating [Grid](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetType) sheets.
 
 ## Supported sync modes
 
 The Google Sheets source connector supports the following sync modes:
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 
-## Data type mapping
+## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-| :--------------- | :----------- | :---- |
+|:-----------------|:-------------|:------|
 | any type         | `string`     |       |
 
-
 ## Performance consideration
 
-The [Google API rate limit](https://developers.google.com/sheets/api/limits) is 100 requests per 100 seconds per user and 500 requests per 100 seconds per project. Airbyte batches requests to the API in order to efficiently pull data and respects these rate limits. We recommended not using the same service user for more than 3 instances of the Google Sheets source connector to ensure high transfer speeds.
+The [Google API rate limits](https://developers.google.com/sheets/api/limits) are:
+
+- 300 read requests per minute per project
+- 60 requests per minute per user per project
+
+Airbyte batches requests to the API in order to efficiently pull data and respect these rate limits. We recommend not using the same user or service account for more than 3 instances of the Google Sheets source connector to ensure high transfer speeds.
 
+## Troubleshooting
+- If your sheet is completely empty(no header rows) or deleted, Airbyte will not delete the table in the destination. If this happens, the sync logs will contain a message saying the sheet has been skipped when syncing the full spreadsheet.
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                       |
-|---------|------------|----------------------------------------------------------|-------------------------------------------------------------------------------|
-| 0.2.39  | 2023-05-31 | [26833](https://github.com/airbytehq/airbyte/pull/26833) | Remove authSpecification in favour of advancedAuth in specification           |
-| 0.2.38  | 2023-05-16 | [26097](https://github.com/airbytehq/airbyte/pull/26097) | Refactor config error                                                         |
-| 0.2.37  | 2023-02-21 | [23292](https://github.com/airbytehq/airbyte/pull/23292) | Skip non grid sheets.                                                         |
-| 0.2.36  | 2023-02-21 | [23272](https://github.com/airbytehq/airbyte/pull/23272) | Handle empty sheets gracefully.                                               |
-| 0.2.35  | 2023-02-23 | [23057](https://github.com/airbytehq/airbyte/pull/23057) | Slugify column names                                                          |
-| 0.2.34  | 2023-02-15 | [23071](https://github.com/airbytehq/airbyte/pull/23071) | Change min spreadsheet id size to 20 symbols                                  |
-| 0.2.33  | 2023-02-13 | [23278](https://github.com/airbytehq/airbyte/pull/23278) | Handle authentication errors                                                  |
-| 0.2.32  | 2023-02-13 | [22884](https://github.com/airbytehq/airbyte/pull/22884) | Do not consume http spreadsheets.                                             |
-| 0.2.31  | 2022-10-09 | [19574](https://github.com/airbytehq/airbyte/pull/19574) | Revert 'Add row_id to rows and use as primary key'                            |
-| 0.2.30  | 2022-10-09 | [19215](https://github.com/airbytehq/airbyte/pull/19215) | Add row_id to rows and use as primary key                                     |
-| 0.2.21  | 2022-10-04 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream                                          |
-| 0.2.20  | 2022-10-10 | [17766](https://github.com/airbytehq/airbyte/pull/17766) | Fix null pointer exception when parsing the spreadsheet id.                   |
-| 0.2.19  | 2022-09-29 | [17410](https://github.com/airbytehq/airbyte/pull/17410) | Use latest CDK.                                                               |
-| 0.2.18  | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states.                                                 |
-| 0.2.17  | 2022-08-03 | [15107](https://github.com/airbytehq/airbyte/pull/15107) | Expose Row Batch Size in Connector Specification                              |
-| 0.2.16  | 2022-07-07 | [13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field description                                       |
-| 0.2.15  | 2022-06-02 | [13446](https://github.com/airbytehq/airbyte/pull/13446) | Retry requests resulting in a server error                                    |
-| 0.2.13  | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions  |
-| 0.2.12  | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml`                                         |
-| 0.2.11  | 2022-04-13 | [11977](https://github.com/airbytehq/airbyte/pull/11977) | Replace leftover print statement with airbyte logger                          |
-| 0.2.10  | 2022-03-25 | [11404](https://github.com/airbytehq/airbyte/pull/11404) | Allow using Spreadsheet Link/URL instead of Spreadsheet ID                    |
-| 0.2.9   | 2022-01-25 | [9208](https://github.com/airbytehq/airbyte/pull/9208)   | Update title and descriptions                                                 |
-| 0.2.7   | 2021-09-27 | [8470](https://github.com/airbytehq/airbyte/pull/8470)   | Migrate to the CDK                                                            |
-| 0.2.6   | 2021-09-27 | [6354](https://github.com/airbytehq/airbyte/pull/6354)   | Support connecting via Oauth webflow                                          |
-| 0.2.5   | 2021-09-12 | [5972](https://github.com/airbytehq/airbyte/pull/5972)   | Fix full_refresh test by adding supported_sync_modes to Stream initialization |
-| 0.2.4   | 2021-08-05 | [5233](https://github.com/airbytehq/airbyte/pull/5233)   | Fix error during listing sheets with diagram only                             |
-| 0.2.3   | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                 |
-| 0.2.2   | 2021-04-20 | [2994](https://github.com/airbytehq/airbyte/pull/2994)   | Formatting spec                                                               |
-| 0.2.1   | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726)   | Fix base connector versioning                                                 |
-| 0.2.0   | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238)   | Protocol allows future/unknown properties                                     |
-| 0.1.7   | 2021-01-21 | [1762](https://github.com/airbytehq/airbyte/pull/1762)   | Fix issue large spreadsheet                                                   |
-| 0.1.6   | 2021-01-27 | [1668](https://github.com/airbytehq/airbyte/pull/1668)   | Adopt connector best practices                                                |
-| 0.1.5   | 2020-12-30 | [1438](https://github.com/airbytehq/airbyte/pull/1438)   | Implement backoff                                                             |
-| 0.1.4   | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046)   | Add connectors using an index YAML file                                       |
+| Version | Date       | Pull Request                                             | Subject                                                                           |
+|---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------|
+| 0.3.7   | 2023-08-25 | [29826](https://github.com/airbytehq/airbyte/pull/29826) | Remove row batch size from spec, add auto increase this value when rate limits    |
+| 0.3.6   | 2023-08-16 | [29491](https://github.com/airbytehq/airbyte/pull/29491) | Update to latest CDK                                                              |
+| 0.3.5   | 2023-08-16 | [29427](https://github.com/airbytehq/airbyte/pull/29427) | Add stop reading in case of 429 error                                             |
+| 0.3.4   | 2023-05-15 | [29453](https://github.com/airbytehq/airbyte/pull/29453) | Update spec descriptions                                                          |
+| 0.3.3   | 2023-08-10 | [29327](https://github.com/airbytehq/airbyte/pull/29327) | Add user-friendly error message for 404 and 403 error while discover              |
+| 0.3.2   | 2023-08-09 | [29246](https://github.com/airbytehq/airbyte/pull/29246) | Add checking while reading to skip modified sheets                                |
+| 0.3.1   | 2023-07-06 | [28033](https://github.com/airbytehq/airbyte/pull/28033) | Fixed several reported vulnerabilities (25 total), CVE-2022-37434, CVE-2022-42898 |
+| 0.3.0   | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2                                                              |
+| 0.2.39  | 2023-05-31 | [26833](https://github.com/airbytehq/airbyte/pull/26833) | Remove authSpecification in favour of advancedAuth in specification               |
+| 0.2.38  | 2023-05-16 | [26097](https://github.com/airbytehq/airbyte/pull/26097) | Refactor config error                                                             |
+| 0.2.37  | 2023-02-21 | [23292](https://github.com/airbytehq/airbyte/pull/23292) | Skip non grid sheets.                                                             |
+| 0.2.36  | 2023-02-21 | [23272](https://github.com/airbytehq/airbyte/pull/23272) | Handle empty sheets gracefully.                                                   |
+| 0.2.35  | 2023-02-23 | [23057](https://github.com/airbytehq/airbyte/pull/23057) | Slugify column names                                                              |
+| 0.2.34  | 2023-02-15 | [23071](https://github.com/airbytehq/airbyte/pull/23071) | Change min spreadsheet id size to 20 symbols                                      |
+| 0.2.33  | 2023-02-13 | [23278](https://github.com/airbytehq/airbyte/pull/23278) | Handle authentication errors                                                      |
+| 0.2.32  | 2023-02-13 | [22884](https://github.com/airbytehq/airbyte/pull/22884) | Do not consume http spreadsheets.                                                 |
+| 0.2.31  | 2022-10-09 | [19574](https://github.com/airbytehq/airbyte/pull/19574) | Revert 'Add row_id to rows and use as primary key'                                |
+| 0.2.30  | 2022-10-09 | [19215](https://github.com/airbytehq/airbyte/pull/19215) | Add row_id to rows and use as primary key                                         |
+| 0.2.21  | 2022-10-04 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Clean instantiation of AirbyteStream                                              |
+| 0.2.20  | 2022-10-10 | [17766](https://github.com/airbytehq/airbyte/pull/17766) | Fix null pointer exception when parsing the spreadsheet id.                       |
+| 0.2.19  | 2022-09-29 | [17410](https://github.com/airbytehq/airbyte/pull/17410) | Use latest CDK.                                                                   |
+| 0.2.18  | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states.                                                     |
+| 0.2.17  | 2022-08-03 | [15107](https://github.com/airbytehq/airbyte/pull/15107) | Expose Row Batch Size in Connector Specification                                  |
+| 0.2.16  | 2022-07-07 | [13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field description                                           |
+| 0.2.15  | 2022-06-02 | [13446](https://github.com/airbytehq/airbyte/pull/13446) | Retry requests resulting in a server error                                        |
+| 0.2.13  | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions      |
+| 0.2.12  | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml`                                             |
+| 0.2.11  | 2022-04-13 | [11977](https://github.com/airbytehq/airbyte/pull/11977) | Replace leftover print statement with airbyte logger                              |
+| 0.2.10  | 2022-03-25 | [11404](https://github.com/airbytehq/airbyte/pull/11404) | Allow using Spreadsheet Link/URL instead of Spreadsheet ID                        |
+| 0.2.9   | 2022-01-25 | [9208](https://github.com/airbytehq/airbyte/pull/9208)   | Update title and descriptions                                                     |
+| 0.2.7   | 2021-09-27 | [8470](https://github.com/airbytehq/airbyte/pull/8470)   | Migrate to the CDK                                                                |
+| 0.2.6   | 2021-09-27 | [6354](https://github.com/airbytehq/airbyte/pull/6354)   | Support connecting via Oauth webflow                                              |
+| 0.2.5   | 2021-09-12 | [5972](https://github.com/airbytehq/airbyte/pull/5972)   | Fix full_refresh test by adding supported_sync_modes to Stream initialization     |
+| 0.2.4   | 2021-08-05 | [5233](https://github.com/airbytehq/airbyte/pull/5233)   | Fix error during listing sheets with diagram only                                 |
+| 0.2.3   | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                     |
+| 0.2.2   | 2021-04-20 | [2994](https://github.com/airbytehq/airbyte/pull/2994)   | Formatting spec                                                                   |
+| 0.2.1   | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726)   | Fix base connector versioning                                                     |
+| 0.2.0   | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238)   | Protocol allows future/unknown properties                                         |
+| 0.1.7   | 2021-01-21 | [1762](https://github.com/airbytehq/airbyte/pull/1762)   | Fix issue large spreadsheet                                                       |
+| 0.1.6   | 2021-01-27 | [1668](https://github.com/airbytehq/airbyte/pull/1668)   | Adopt connector best practices                                                    |
+| 0.1.5   | 2020-12-30 | [1438](https://github.com/airbytehq/airbyte/pull/1438)   | Implement backoff                                                                 |
+| 0.1.4   | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046)   | Add connectors using an index YAML file                                           |
diff --git a/docs/integrations/sources/greenhouse.md b/docs/integrations/sources/greenhouse.md
index ff8ccce0ce67..50ef4dc63de0 100644
--- a/docs/integrations/sources/greenhouse.md
+++ b/docs/integrations/sources/greenhouse.md
@@ -12,49 +12,49 @@ To set up the Greenhouse source connector, you'll need the [Harvest API key](htt
 2. Click **Sources** and then click **+ New source**.
 3. On the Set up the source page, select **Greenhouse** from the Source type dropdown.
 4. Enter the name for the Greenhouse connector.
-4. Enter your [**Harvest API Key**](https://developers.greenhouse.io/harvest.html#authentication) that you obtained from Greenhouse.
-5. Click **Set up source**.
+5. Enter your [**Harvest API Key**](https://developers.greenhouse.io/harvest.html#authentication) that you obtained from Greenhouse.
+6. Click **Set up source**.
 
 ## Supported sync modes
 
 The Greenhouse source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Activity Feed](https://developers.greenhouse.io/harvest.html#get-retrieve-activity-feed)
-* [Applications](https://developers.greenhouse.io/harvest.html#get-list-applications)
-* [Applications Interviews](https://developers.greenhouse.io/harvest.html#get-list-scheduled-interviews-for-application)
-* [Approvals](https://developers.greenhouse.io/harvest.html#get-list-approvals-for-job)
-* [Candidates](https://developers.greenhouse.io/harvest.html#get-list-candidates)
-* [Close Reasons](https://developers.greenhouse.io/harvest.html#get-list-close-reasons)
-* [Custom Fields](https://developers.greenhouse.io/harvest.html#get-list-custom-fields)
-* [Degrees](https://developers.greenhouse.io/harvest.html#get-list-degrees)
-* [Departments](https://developers.greenhouse.io/harvest.html#get-list-departments)
-* [Disciplines](https://developers.greenhouse.io/harvest.html#get-list-approvals-for-job)
-* [EEOC](https://developers.greenhouse.io/harvest.html#get-list-eeoc)
-* [Email Templates](https://developers.greenhouse.io/harvest.html#get-list-email-templates)
-* [Interviews](https://developers.greenhouse.io/harvest.html#get-list-scheduled-interviews)
-* [Job Posts](https://developers.greenhouse.io/harvest.html#get-list-job-posts)
-* [Job Stages](https://developers.greenhouse.io/harvest.html#get-list-job-stages)
-* [Jobs](https://developers.greenhouse.io/harvest.html#get-list-jobs)
-* [Job Openings](https://developers.greenhouse.io/harvest.html#get-list-job-openings)
-* [Jobs Stages](https://developers.greenhouse.io/harvest.html#get-list-job-stages-for-job)
-* [Offers](https://developers.greenhouse.io/harvest.html#get-list-offers)
-* [Offices](https://developers.greenhouse.io/harvest.html#get-list-offices)
-* [Prospect Pools](https://developers.greenhouse.io/harvest.html#get-list-prospect-pools)
-* [Rejection Reasons](https://developers.greenhouse.io/harvest.html#get-list-rejection-reasons)
-* [Schools](https://developers.greenhouse.io/harvest.html#get-list-schools)
-* [Scorecards](https://developers.greenhouse.io/harvest.html#get-list-scorecards)
-* [Sources](https://developers.greenhouse.io/harvest.html#get-list-sources)
-* [Tags](https://developers.greenhouse.io/harvest.html#get-list-candidate-tags)
-* [Users](https://developers.greenhouse.io/harvest.html#get-list-users)
-* [User Permissions](https://developers.greenhouse.io/harvest.html#get-list-job-permissions)
-* [User Roles](https://developers.greenhouse.io/harvest.html#the-user-role-object)
+- [Activity Feed](https://developers.greenhouse.io/harvest.html#get-retrieve-activity-feed)
+- [Applications](https://developers.greenhouse.io/harvest.html#get-list-applications)
+- [Applications Interviews](https://developers.greenhouse.io/harvest.html#get-list-scheduled-interviews-for-application)
+- [Approvals](https://developers.greenhouse.io/harvest.html#get-list-approvals-for-job)
+- [Candidates](https://developers.greenhouse.io/harvest.html#get-list-candidates)
+- [Close Reasons](https://developers.greenhouse.io/harvest.html#get-list-close-reasons)
+- [Custom Fields](https://developers.greenhouse.io/harvest.html#get-list-custom-fields)
+- [Degrees](https://developers.greenhouse.io/harvest.html#get-list-degrees)
+- [Departments](https://developers.greenhouse.io/harvest.html#get-list-departments)
+- [Disciplines](https://developers.greenhouse.io/harvest.html#get-list-approvals-for-job)
+- [EEOC](https://developers.greenhouse.io/harvest.html#get-list-eeoc)
+- [Email Templates](https://developers.greenhouse.io/harvest.html#get-list-email-templates)
+- [Interviews](https://developers.greenhouse.io/harvest.html#get-list-scheduled-interviews)
+- [Job Posts](https://developers.greenhouse.io/harvest.html#get-list-job-posts)
+- [Job Stages](https://developers.greenhouse.io/harvest.html#get-list-job-stages)
+- [Jobs](https://developers.greenhouse.io/harvest.html#get-list-jobs)
+- [Job Openings](https://developers.greenhouse.io/harvest.html#get-list-job-openings)
+- [Jobs Stages](https://developers.greenhouse.io/harvest.html#get-list-job-stages-for-job)
+- [Offers](https://developers.greenhouse.io/harvest.html#get-list-offers)
+- [Offices](https://developers.greenhouse.io/harvest.html#get-list-offices)
+- [Prospect Pools](https://developers.greenhouse.io/harvest.html#get-list-prospect-pools)
+- [Rejection Reasons](https://developers.greenhouse.io/harvest.html#get-list-rejection-reasons)
+- [Schools](https://developers.greenhouse.io/harvest.html#get-list-schools)
+- [Scorecards](https://developers.greenhouse.io/harvest.html#get-list-scorecards)
+- [Sources](https://developers.greenhouse.io/harvest.html#get-list-sources)
+- [Tags](https://developers.greenhouse.io/harvest.html#get-list-candidate-tags)
+- [Users](https://developers.greenhouse.io/harvest.html#get-list-users)
+- [User Permissions](https://developers.greenhouse.io/harvest.html#get-list-job-permissions)
+- [User Roles](https://developers.greenhouse.io/harvest.html#the-user-role-object)
 
 ## Performance considerations
 
@@ -62,16 +62,18 @@ The Greenhouse connector should not run into Greenhouse API limitations under no
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                        |
-|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
+| Version | Date       | Pull Request                                             | Subject                                                                                                                                                              |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.4.2   | 2023-08-02 | [28969](https://github.com/airbytehq/airbyte/pull/28969) | Update CDK version                                                                                                                                                   |
+| 0.4.1   | 2023-06-28 | [27773](https://github.com/airbytehq/airbyte/pull/27773) | Update following state breaking changes                                                                                                                              |
 | 0.4.0   | 2023-04-26 | [25332](https://github.com/airbytehq/airbyte/pull/25332) | Add new streams: `ActivityFeed`, `Approvals`, `Disciplines`, `Eeoc`, `EmailTemplates`, `Offices`, `ProspectPools`, `Schools`, `Tags`, `UserPermissions`, `UserRoles` |
-| 0.3.1   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                                        |
-| 0.3.0   | 2022-10-19 | [18154](https://github.com/airbytehq/airbyte/pull/18154) | Extend `Users` stream schema                                                   |
-| 0.2.11  | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK                               |
-| 0.2.10  | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs                                         |
-| 0.2.9   | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime                     |
-| 0.2.8   | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework                                    |
-| 0.2.7   | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users    |
-| 0.2.6   | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607)   | Implement demographics streams support. Update SAT for demographics streams    |
-| 0.2.5   | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377)   | Refactor the connector to use CDK. Implement additional stream support         |
-| 0.2.4   | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238)   | Add identification of accessible streams for API keys with limited permissions |
+| 0.3.1   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                                                                                                                              |
+| 0.3.0   | 2022-10-19 | [18154](https://github.com/airbytehq/airbyte/pull/18154) | Extend `Users` stream schema                                                                                                                                         |
+| 0.2.11  | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK                                                                                                                     |
+| 0.2.10  | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs                                                                                                                               |
+| 0.2.9   | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime                                                                                                           |
+| 0.2.8   | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework                                                                                                                          |
+| 0.2.7   | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users                                                                                          |
+| 0.2.6   | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607)   | Implement demographics streams support. Update SAT for demographics streams                                                                                          |
+| 0.2.5   | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377)   | Refactor the connector to use CDK. Implement additional stream support                                                                                               |
+| 0.2.4   | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238)   | Add identification of accessible streams for API keys with limited permissions                                                                                       |
diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md
index d0bb23ecf711..e53ff9afbe96 100644
--- a/docs/integrations/sources/harvest.md
+++ b/docs/integrations/sources/harvest.md
@@ -9,6 +9,7 @@ To set up the Harvest source connector, you'll need the [Harvest Account ID and
 ## Setup guide
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces).
@@ -22,6 +23,7 @@ To set up the Harvest source connector, you'll need the [Harvest Account ID and
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Navigate to the Airbyte Open Source dashboard.
@@ -38,38 +40,38 @@ To set up the Harvest source connector, you'll need the [Harvest Account ID and
 
 The Harvest source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Client Contacts](https://help.getharvest.com/api-v2/clients-api/clients/contacts/) \(Incremental\)
-* [Clients](https://help.getharvest.com/api-v2/clients-api/clients/clients/) \(Incremental\)
-* [Company](https://help.getharvest.com/api-v2/company-api/company/company/)
-* [Invoice Messages](https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-messages/) \(Incremental\)
-* [Invoice Payments](https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-payments/) \(Incremental\)
-* [Invoices](https://help.getharvest.com/api-v2/invoices-api/invoices/invoices/) \(Incremental\)
-* [Invoice Item Categories](https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-item-categories/) \(Incremental\)
-* [Estimate Messages](https://help.getharvest.com/api-v2/estimates-api/estimates/estimate-messages/) \(Incremental\)
-* [Estimates](https://help.getharvest.com/api-v2/estimates-api/estimates/estimates/) \(Incremental\)
-* [Estimate Item Categories](https://help.getharvest.com/api-v2/estimates-api/estimates/estimate-item-categories/) \(Incremental\)
-* [Expenses](https://help.getharvest.com/api-v2/expenses-api/expenses/expenses/) \(Incremental\)
-* [Expense Categories](https://help.getharvest.com/api-v2/expenses-api/expenses/expense-categories/) \(Incremental\)
-* [Tasks](https://help.getharvest.com/api-v2/tasks-api/tasks/tasks/) \(Incremental\)
-* [Time Entries](https://help.getharvest.com/api-v2/timesheets-api/timesheets/time-entries/) \(Incremental\)
-* [Project User Assignments](https://help.getharvest.com/api-v2/projects-api/projects/user-assignments/) \(Incremental\)
-* [Project Task Assignments](https://help.getharvest.com/api-v2/projects-api/projects/task-assignments/) \(Incremental\)
-* [Projects](https://help.getharvest.com/api-v2/projects-api/projects/projects/) \(Incremental\)
-* [Roles](https://help.getharvest.com/api-v2/roles-api/roles/roles/) \(Incremental\)
-* [User Billable Rates](https://help.getharvest.com/api-v2/users-api/users/billable-rates/)
-* [User Cost Rates](https://help.getharvest.com/api-v2/users-api/users/cost-rates/)
-* [User Project Assignments](https://help.getharvest.com/api-v2/users-api/users/project-assignments/) \(Incremental\)
-* [Expense Reports](https://help.getharvest.com/api-v2/reports-api/reports/expense-reports/)
-* [Uninvoiced Report](https://help.getharvest.com/api-v2/reports-api/reports/uninvoiced-report/)
-* [Time Reports](https://help.getharvest.com/api-v2/reports-api/reports/time-reports/)
-* [Project Budget Report](https://help.getharvest.com/api-v2/reports-api/reports/project-budget-report/)
+- [Client Contacts](https://help.getharvest.com/api-v2/clients-api/clients/contacts/) \(Incremental\)
+- [Clients](https://help.getharvest.com/api-v2/clients-api/clients/clients/) \(Incremental\)
+- [Company](https://help.getharvest.com/api-v2/company-api/company/company/)
+- [Invoice Messages](https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-messages/) \(Incremental\)
+- [Invoice Payments](https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-payments/) \(Incremental\)
+- [Invoices](https://help.getharvest.com/api-v2/invoices-api/invoices/invoices/) \(Incremental\)
+- [Invoice Item Categories](https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-item-categories/) \(Incremental\)
+- [Estimate Messages](https://help.getharvest.com/api-v2/estimates-api/estimates/estimate-messages/) \(Incremental\)
+- [Estimates](https://help.getharvest.com/api-v2/estimates-api/estimates/estimates/) \(Incremental\)
+- [Estimate Item Categories](https://help.getharvest.com/api-v2/estimates-api/estimates/estimate-item-categories/) \(Incremental\)
+- [Expenses](https://help.getharvest.com/api-v2/expenses-api/expenses/expenses/) \(Incremental\)
+- [Expense Categories](https://help.getharvest.com/api-v2/expenses-api/expenses/expense-categories/) \(Incremental\)
+- [Tasks](https://help.getharvest.com/api-v2/tasks-api/tasks/tasks/) \(Incremental\)
+- [Time Entries](https://help.getharvest.com/api-v2/timesheets-api/timesheets/time-entries/) \(Incremental\)
+- [Project User Assignments](https://help.getharvest.com/api-v2/projects-api/projects/user-assignments/) \(Incremental\)
+- [Project Task Assignments](https://help.getharvest.com/api-v2/projects-api/projects/task-assignments/) \(Incremental\)
+- [Projects](https://help.getharvest.com/api-v2/projects-api/projects/projects/) \(Incremental\)
+- [Roles](https://help.getharvest.com/api-v2/roles-api/roles/roles/) \(Incremental\)
+- [User Billable Rates](https://help.getharvest.com/api-v2/users-api/users/billable-rates/)
+- [User Cost Rates](https://help.getharvest.com/api-v2/users-api/users/cost-rates/)
+- [User Project Assignments](https://help.getharvest.com/api-v2/users-api/users/project-assignments/) \(Incremental\)
+- [Expense Reports](https://help.getharvest.com/api-v2/reports-api/reports/expense-reports/)
+- [Uninvoiced Report](https://help.getharvest.com/api-v2/reports-api/reports/uninvoiced-report/)
+- [Time Reports](https://help.getharvest.com/api-v2/reports-api/reports/time-reports/)
+- [Project Budget Report](https://help.getharvest.com/api-v2/reports-api/reports/project-budget-report/)
 
 ## Performance considerations
 
@@ -78,7 +80,7 @@ The connector is restricted by the [Harvest rate limits](https://help.getharvest
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                            |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------- |
 | 0.1.18  | 2023-05-29 | [26714](https://github.com/airbytehq/airbyte/pull/26714) | Remove `authSpecification` from spec in favour of `advancedAuth`                   |
 | 0.1.17  | 2023-03-03 | [22983](https://github.com/airbytehq/airbyte/pull/22983) | Specified date formatting in specification                                         |
 | 0.1.16  | 2023-02-07 | [22417](https://github.com/airbytehq/airbyte/pull/22417) | Turn on default HttpAvailabilityStrategy                                           |
diff --git a/docs/integrations/sources/hellobaton.md b/docs/integrations/sources/hellobaton.md
new file mode 100644
index 000000000000..3b6d38a1ba65
--- /dev/null
+++ b/docs/integrations/sources/hellobaton.md
@@ -0,0 +1,57 @@
+# Hellobaton
+
+## Sync overview
+
+This source can sync data from the [hellobaton API](https://app.hellobaton.com/api/redoc/). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records.
+
+## This Source Supports the Following Streams
+
+- activity
+- companies
+- milestones
+- phases
+- project_attachments
+- projects
+- task_attachemnts
+- tasks
+- templates
+- time_entries
+- users
+
+Hellobaton adds new streams fairly regularly please submit an issue or PR if this project doesn't support required streams for your use case.
+
+### Data type mapping
+
+| Integration Type | Airbyte Type | Notes |
+| :--------------- | :----------- | :---- |
+| `string`         | `string`     |       |
+| `integer`        | `integer`    |       |
+| `number`         | `number`     |       |
+| `array`          | `array`      |       |
+| `object`         | `object`     |       |
+
+### Features
+
+| Feature           | Supported?\(Yes/No\) | Notes |
+| :---------------- | :------------------- | :---- |
+| Full Refresh Sync | Yes                  |       |
+| Incremental Sync  | No                   |       |
+| Namespaces        | No                   |       |
+
+### Performance considerations
+
+The connector is rate limited at 1000 requests per minute per api key. If you find yourself receiving errors contact your customer success manager and request a rate limit increase.
+
+## Getting started
+
+### Requirements
+
+- Hellobaton account
+- Hellobaton api key
+
+## Changelog
+
+| Version | Date       | Pull Request                                           | Subject                   |
+| :------ | :--------- | :----------------------------------------------------- | :------------------------ |
+| 0.2.0   | 2023-08-19 | [29490](https://github.com/airbytehq/airbyte/pull/29490) | Migrate CDK from Python to Low Code |
+| 0.1.0   | 2022-01-14 | [8461](https://github.com/airbytehq/airbyte/pull/8461) | 🎉 New Source: Hellobaton |
diff --git a/docs/integrations/sources/hubspot.inapp.md b/docs/integrations/sources/hubspot.inapp.md
index 2241c2100cab..fdafd906607e 100644
--- a/docs/integrations/sources/hubspot.inapp.md
+++ b/docs/integrations/sources/hubspot.inapp.md
@@ -1,13 +1,57 @@
-## Prerequisite
+## Prerequisites
 
-* Access to your HubSpot account
+- HubSpot Account
+- **For Airbyte Open Source**: Private App with Access Token
+
+### Authentication method
+You can use OAuth or a Private App to authenticate your HubSpot account.  For Airbyte Cloud users, we highly recommend you use OAuth rather than Private App authentication, as it significantly simplifies the setup process.
+
+For more information on which authentication method to choose and the required setup steps, see our full
+[Hubspot documentation](https://docs.airbyte.com/integrations/sources/hubspot/).
+
+### Scopes Required (for Private App and Open Source OAuth)
+Unless you are authenticating via OAuth on **Airbyte Cloud**, you must manually configure scopes to ensure Airbyte can sync all available data. To see a breakdown of the specific scopes each stream uses, see our full [Hubspot documentation](https://docs.airbyte.com/integrations/sources/hubspot/).
+
+* `content`
+* `forms`
+* `tickets`
+* `automation`
+* `e-commerce`
+* `sales-email-read`
+* `crm.objects.companies.read`
+* `crm.schemas.companies.read`
+* `crm.objects.lists.read`
+* `crm.objects.contacts.read`
+* `crm.objects.deals.read`
+* `crm.schemas.deals.read`
+* `crm.objects.goals.read`
+* `crm.objects.owners.read`
+* `crm.objects.custom.read`
 
 ## Setup guide
 
-1. For **Start date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-2. You can use OAuth or an API key to authenticate your HubSpot account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source.
-   * To authenticate using OAuth for Airbyte Cloud, ensure you have set the appropriate scopes for HubSpot and then click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account.
-   * To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter the [API key](https://knowledge.hubspot.com/integrations/how-do-i-get-my-hubspot-api-key) for your HubSpot account. Check the [performance considerations](https://docs.airbyte.com/integrations/sources/hubspot/#performance-considerations) before using an API key.
-3. Click Set up source.
+1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **HubSpot** from the list of available sources.
+3. Enter a **Source name** of your choosing.
+4. From the **Authentication** dropdown, select your chosen authentication method:
+
+
+#### For Airbyte Cloud users:
+- To authenticate using OAuth, select **OAuth** and click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account.
+- To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account.
+
+
+
+#### For Airbyte Open Source users:
+- To authenticate using OAuth, select **OAuth** and enter your Client ID, Client Secret, and Refresh Token.
+- To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account.
+
+
+5. For **Start date**, use the provided datepicker or enter the date programmatically in the following format:
+`yyyy-mm-ddThh:mm:ssZ`. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
+6. Click **Set up source** and wait for the tests to complete.
+
+## Supported Objects
+Airbyte supports syncing standard and custom CRM objects. Custom CRM objects will appear as streams available for sync, alongside the standard objects.
 
-For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Hubspot](https://docs.airbyte.com/integrations/sources/hubspot/).
\ No newline at end of file
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Hubspot](https://docs.airbyte.com/integrations/sources/hubspot/).
diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
index 25ec73eb9258..ebf4ce938044 100644
--- a/docs/integrations/sources/hubspot.md
+++ b/docs/integrations/sources/hubspot.md
@@ -1,17 +1,47 @@
 # HubSpot
 
-This page guides you through setting up the HubSpot source connector.
+This page contains the setup guide and reference information for the [HubSpot](https://www.hubspot.com/) source connector.
 
-## Prerequisite
+## Prerequisites
 
-You can use OAuth or a Private App to authenticate your HubSpot account. 
+- HubSpot Account
+- **For Airbyte Open Source**: Private App with Access Token
 
-In Airbyte Cloud, we highly recommend you use OAuth and not Private App authentication as it significantly simplifies the setup process. 
+## Setup guide
 
-If you are using either OAuth in Airbyte OSS or Private App authentication, you need to configure the appropriate [scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for the following streams:
+**For Airbyte Cloud** users, we highly recommend you use OAuth rather than Private App authentication, as it significantly simplifies the setup process.
+
+**For Airbyte Open Source** users we recommend Private App authentication.
+
+More information on HubSpot authentication methods can be found
+[here](https://developers.hubspot.com/docs/api/intro-to-auth).
+
+### Step 1: Set up the authentication method
+
+#### Private App setup (Recommended for Airbyte Open Source)
+
+If you are authenticating via a Private App, you will need to use your Access Token to set up the connector. Please refer to the
+[official HubSpot documentation](https://developers.hubspot.com/docs/api/private-apps) for a detailed guide.
+
+
+
+#### OAuth setup for Airbyte Open Source (Not recommended)
+
+If you are using Oauth to authenticate on Airbyte Open Source, please refer to [Hubspot's detailed walkthrough](https://developers.hubspot.com/docs/api/working-with-oauth). To set up the connector, you will need to acquire your:
+
+- Client ID
+- Client Secret
+- Refresh Token
+
+
+
+### Step 2: Configure the scopes for your streams
+
+Next, you need to configure the appropriate scopes for the following streams. Please refer to
+[Hubspot's page on scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for instructions.
 
 | Stream                      | Required Scope                                                                                               |
-|:----------------------------|:-------------------------------------------------------------------------------------------------------------|
+| :-------------------------- | :----------------------------------------------------------------------------------------------------------- |
 | `campaigns`                 | `content`                                                                                                    |
 | `companies`                 | `crm.objects.companies.read`, `crm.schemas.companies.read`                                                   |
 | `contact_lists`             | `crm.objects.lists.read`                                                                                     |
@@ -27,7 +57,7 @@ If you are using either OAuth in Airbyte OSS or Private App authentication, you
 | `engagements_emails`        | `sales-email-read`                                                                                           |
 | `forms`                     | `forms`                                                                                                      |
 | `form_submissions`          | `forms`                                                                                                      |
-| `goals`                     | `crm.objects.goals.read`                                                                              |
+| `goals`                     | `crm.objects.goals.read`                                                                                     |
 | `line_items`                | `e-commerce`                                                                                                 |
 | `owners`                    | `crm.objects.owners.read`                                                                                    |
 | `products`                  | `e-commerce`                                                                                                 |
@@ -36,80 +66,98 @@ If you are using either OAuth in Airbyte OSS or Private App authentication, you
 | `tickets`                   | `tickets`                                                                                                    |
 | `workflows`                 | `automation`                                                                                                 |
 
+### Step 3: Set up the HubSpot source connector in Airbyte
+
+1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **HubSpot** from the list of available sources.
+3. Enter a **Source name** of your choosing.
+4. From the **Authentication** dropdown, select your chosen authentication method:
+
+
 
-## Set up the HubSpot source connector
+#### For Airbyte Cloud users:
 
-1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**. 
-3. On the Set up the source page, select **HubSpot** from the Source type dropdown.
-4. Enter a name for your source.
-5. For **Start date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-6. You can use OAuth or an Private Apps to authenticate your HubSpot account. We recommend using OAuth for Airbyte Cloud and an Private Apps for Airbyte Open Source.
-    - To authenticate using OAuth for Airbyte Cloud, ensure you have [set the appropriate scopes for HubSpot](#prerequisite) and then click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account. 
-    - To authenticate using a Private App, select **Private App** from the Authentication dropdown and enter the Access Token for your HubSpot account which you can obtain by following the instructions provided by Hubspot [here](https://developers.hubspot.com/docs/api/private-apps).    
-7. Click **Set up source**.
+- **Recommended:** To authenticate using OAuth, select **OAuth** and click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account.
+- **Not Recommended:**To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account.
+
+
+
+
+#### For Airbyte Open Source users:
+
+- **Recommended:** To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account.
+- **Not Recommended:**To authenticate using OAuth, select **OAuth** and enter your Client ID, Client Secret, and Refresh Token.
+
+
+
+5. For **Start date**, use the provided datepicker or enter the date programmatically in the following format:
+   `yyyy-mm-ddThh:mm:ssZ`. The data added on and after this date will be replicated.
+6. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
 The HubSpot source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
- - Full Refresh
- - Incremental
-
-## Supported Streams
+- Full Refresh
+- Incremental
 
 :::note
 There are two types of incremental sync:
+
 1. Incremental (standard server-side, where API returns only the data updated or generated since the last sync)
 2. Client-Side Incremental (API returns all available data and connector filters out only new records)
 :::
 
+## Supported streams
+
 The HubSpot source connector supports the following streams:
 
-* [Campaigns](https://developers.hubspot.com/docs/methods/email/get_campaign_data) \(Client-Side Incremental\)
-* [Companies](https://developers.hubspot.com/docs/api/crm/companies) \(Incremental\)
-* [Contact Lists](http://developers.hubspot.com/docs/methods/lists/get_lists) \(Incremental\)
-* [Contacts](https://developers.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\)
-* [Contacts List Memberships](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts)
-* [Deal Pipelines](https://developers.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type) \(Client-Side Incremental\)
-* [Deals](https://developers.hubspot.com/docs/api/crm/deals) \(including Contact associations\) \(Incremental\)
-  * Records that have been deleted (archived) and stored in HubSpot's recycle bin will only be kept for 90 days, see [response from HubSpot Team](https://community.hubspot.com/t5/APIs-Integrations/Archived-deals-deleted-or-different/m-p/714157)
-* [Deals Archived](https://developers.hubspot.com/docs/api/crm/deals) \(including Contact associations\) \(Incremental\)
-* [Email Events](https://developers.hubspot.com/docs/methods/email/get_events) \(Incremental\)
-* [Email Subscriptions](https://developers.hubspot.com/docs/methods/email/get_subscriptions)
-* [Engagements](https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements) \(Incremental\)
-* [Engagements Calls](https://developers.hubspot.com/docs/api/crm/calls) \(Incremental\)
-* [Engagements Emails](https://developers.hubspot.com/docs/api/crm/email) \(Incremental\)
-* [Engagements Meetings](https://developers.hubspot.com/docs/api/crm/meetings) \(Incremental\)
-* [Engagements Notes](https://developers.hubspot.com/docs/api/crm/notes) \(Incremental\)
-* [Engagements Tasks](https://developers.hubspot.com/docs/api/crm/tasks) \(Incremental\)
-* [Forms](https://developers.hubspot.com/docs/api/marketing/forms) \(Client-Side Incremental\)
-* [Form Submissions](https://legacydocs.hubspot.com/docs/methods/forms/get-submissions-for-a-form) \(Client-Side Incremental\)
-* [Goals](https://developers.hubspot.com/docs/api/crm/goals) \(Incremental\)
-* [Line Items](https://developers.hubspot.com/docs/api/crm/line-items) \(Incremental\)
-* [Marketing Emails](https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-email-statistics)
-* [Owners](https://developers.hubspot.com/docs/methods/owners/get_owners) \(Client-Side Incremental\)
-* [Products](https://developers.hubspot.com/docs/api/crm/products) \(Incremental\)
-* [Property History](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\)
-* [Subscription Changes](https://developers.hubspot.com/docs/methods/email/get_subscriptions_timeline) \(Incremental\)
-* [Tickets](https://developers.hubspot.com/docs/api/crm/tickets) \(Incremental\)
-* [Ticket Pipelines](https://developers.hubspot.com/docs/api/crm/pipelines) \(Client-Side Incremental\)
-* [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows) \(Client-Side Incremental\)
+- [Campaigns](https://developers.hubspot.com/docs/methods/email/get_campaign_data) \(Client-Side Incremental\)
+- [Companies](https://developers.hubspot.com/docs/api/crm/companies) \(Incremental\)
+- [Contact Lists](http://developers.hubspot.com/docs/methods/lists/get_lists) \(Incremental\)
+- [Contacts](https://developers.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\)
+- [Contacts List Memberships](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts)
+- [Deal Pipelines](https://developers.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type) \(Client-Side Incremental\)
+- [Deals](https://developers.hubspot.com/docs/api/crm/deals) \(including Contact associations\) \(Incremental\)
+  - Records that have been deleted (archived) and stored in HubSpot's recycle bin will only be kept for 90 days, see [response from HubSpot Team](https://community.hubspot.com/t5/APIs-Integrations/Archived-deals-deleted-or-different/m-p/714157)
+- [Deals Archived](https://developers.hubspot.com/docs/api/crm/deals) \(including Contact associations\) \(Incremental\)
+- [Email Events](https://developers.hubspot.com/docs/methods/email/get_events) \(Incremental\)
+- [Email Subscriptions](https://developers.hubspot.com/docs/methods/email/get_subscriptions)
+- [Engagements](https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements) \(Incremental\)
+- [Engagements Calls](https://developers.hubspot.com/docs/api/crm/calls) \(Incremental\)
+- [Engagements Emails](https://developers.hubspot.com/docs/api/crm/email) \(Incremental\)
+- [Engagements Meetings](https://developers.hubspot.com/docs/api/crm/meetings) \(Incremental\)
+- [Engagements Notes](https://developers.hubspot.com/docs/api/crm/notes) \(Incremental\)
+- [Engagements Tasks](https://developers.hubspot.com/docs/api/crm/tasks) \(Incremental\)
+- [Forms](https://developers.hubspot.com/docs/api/marketing/forms) \(Client-Side Incremental\)
+- [Form Submissions](https://legacydocs.hubspot.com/docs/methods/forms/get-submissions-for-a-form) \(Client-Side Incremental\)
+- [Goals](https://developers.hubspot.com/docs/api/crm/goals) \(Incremental\)
+- [Line Items](https://developers.hubspot.com/docs/api/crm/line-items) \(Incremental\)
+- [Marketing Emails](https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-email-statistics)
+- [Owners](https://developers.hubspot.com/docs/methods/owners/get_owners) \(Client-Side Incremental\)
+- [Products](https://developers.hubspot.com/docs/api/crm/products) \(Incremental\)
+- [Property History](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\)
+- [Subscription Changes](https://developers.hubspot.com/docs/methods/email/get_subscriptions_timeline) \(Incremental\)
+- [Tickets](https://developers.hubspot.com/docs/api/crm/tickets) \(Incremental\)
+- [Ticket Pipelines](https://developers.hubspot.com/docs/api/crm/pipelines) \(Client-Side Incremental\)
+- [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows) \(Client-Side Incremental\)
 
 ### Custom CRM Objects
-Custom CRM Objects will appear as streams available for sync, alongside the standard objects listed above. 
 
-If you setup your connections before April 15th, 2023 (on Cloud) or before 0.8.0 (OSS) then you'll need to do some additional work to sync custom CRM objects.
+Custom CRM Objects will appear as streams available for sync, alongside the standard objects listed above.
+
+If you set up your connections before April 15th, 2023 (on Cloud) or before 0.8.0 (OSS) then you'll need to do some additional work to sync custom CRM objects.
 
-First you need to give the connector some additional permissions: 
-* **If you are using OAuth on Cloud** go to the Hubspot source settings page in the Airbyte UI and reauthenticate via Oauth to allow Airbyte the permissions to access custom objects. 
-* **If you are using OAuth on OSS or Private App auth (on OSS or Cloud)**: you'll need to go into the Hubspot UI where you created your private app or oauth application and add the `crm.objects.custom.read` scope to your app's scopes. See Hubspot's instructions here. 
+First you need to give the connector some additional permissions:
 
-Then, go to the replication settings of your connection and click “refresh source schema” to pull in those new streams for syncing. 
+- **If you are using OAuth on Cloud** go to the Hubspot source settings page in the Airbyte UI and reauthenticate via Oauth to allow Airbyte the permissions to access custom objects.
+- **If you are using OAuth on OSS or Private App auth (on OSS or Cloud)**: you'll need to go into the Hubspot UI where you created your Private App or OAuth application and add the `crm.objects.custom.read` scope to your app's scopes. See HubSpot's instructions [here](https://developers.hubspot.com/docs/api/working-with-oauth#scopes).
 
-### A note on the `engagements` stream
+Then, go to the replication settings of your connection and click **refresh source schema** to pull in those new streams for syncing.
 
-Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`. Depending on the type of engagement, different properties is set for that object in the `engagements_metadata` table in the destination:
+### Notes on the `engagements` stream
+
+1. Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`. Depending on the type of engagement, different properties are set for that object in the `engagements_metadata` table in the destination:
 
 - A `call` engagement has a corresponding `engagements_metadata` object with non-null values in the `toNumber`, `fromNumber`, `status`, `externalId`, `durationMilliseconds`, `externalAccountId`, `recordingUrl`, `body`, and `disposition` columns.
 - An `email` engagement has a corresponding `engagements_metadata` object with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`.
@@ -117,11 +165,20 @@ Objects in the `engagements` stream can have one of the following types: `note`,
 - A `note` engagement has a corresponding `engagements_metadata` object with non-null values in the `body` column.
 - A `task` engagement has a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns.
 
+2. The `engagements` stream uses two different APIs based on the length of time since the last sync and the number of records which Airbyte hasn't yet synced.
+
+- **EngagementsRecent** if the following two criteria are met:
+  - The last sync was performed within the last 30 days
+  - Fewer than 10,000 records are being synced
+- **EngagementsAll** if either of these criteria are not met.
+
+Because of this, the `engagements` stream can be slow to sync if it hasn't synced within the last 30 days and/or is generating large volumes of new data. We therefore recommend scheduling frequent syncs.
+
 ## Performance considerations
 
 The connector is restricted by normal HubSpot [rate limitations](https://legacydocs.hubspot.com/apps/api_guidelines).
 
-Some streams, such as `workflows` need to be enabled before they can be read using a connector authenticated using an `API Key`. If reading a stream that is not enabled, a log message returned to the output and the sync operation only sync the other streams available.
+Some streams, such as `workflows`, need to be enabled before they can be read using a connector authenticated using an `API Key`. If reading a stream that is not enabled, a log message returned to the output and the sync operation only sync the other streams available.
 
 Example of the output message when trying to read `workflows` stream with missing permissions for the `API Key`:
 
@@ -144,103 +201,117 @@ Now that you have set up the Hubspot source connector, check out the following H
 [Build a single customer view with open-source tools](https://airbyte.com/tutorials/single-customer-view)
 
 ## Changelog
-| Version | Date       | Pull Request                                             | Subject                                                                                                                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 0.8.4   | 2023-05-17 | [25667](https://github.com/airbytehq/airbyte/pull/26082) | Fixed bug with wrong parsing of boolean encoded like "false" parsed as True
-| 0.8.3   | 2023-05-31 | [26831](https://github.com/airbytehq/airbyte/pull/26831) | Remove authSpecification from connector specification in favour of advancedAuth                                                                            |
-| 0.8.2   | 2023-05-16 | [26418](https://github.com/airbytehq/airbyte/pull/26418) | Added custom availability strategy which catches permission errors from parent streams                                                                     |
-| 0.8.1   | 2023-05-29 | [26719](https://github.com/airbytehq/airbyte/pull/26719) | Handle issue when `state` value is literally `"" (empty str)`                                                                                              |
-| 0.8.0   | 2023-04-10 | [16032](https://github.com/airbytehq/airbyte/pull/16032) | Add new stream `Custom Object`                                                                                                                             |
-| 0.7.0   | 2023-04-10 | [24450](https://github.com/airbytehq/airbyte/pull/24450) | Add new stream `Goals`                                                                                                                                     |
-| 0.6.2   | 2023-04-28 | [25667](https://github.com/airbytehq/airbyte/pull/25667) | Fixed bug with `Invalid Date` like `2000-00-00T00:00:00Z` while settip up the connector                                                                    |
-| 0.6.1   | 2023-04-10 | [21423](https://github.com/airbytehq/airbyte/pull/21423) | Update scope for `DealPipelines` stream to only `crm.objects.contacts.read`                                                                                |
-| 0.6.0   | 2023-04-07 | [24980](https://github.com/airbytehq/airbyte/pull/24980) | Add new stream `DealsArchived`                                                                                                                             |
-| 0.5.2   | 2023-04-07 | [24915](https://github.com/airbytehq/airbyte/pull/24915) | Fix field key parsing (replace whitespace with uderscore)                                                                                                  |
-| 0.5.1   | 2023-04-05 | [22982](https://github.com/airbytehq/airbyte/pull/22982) | Specified date formatting in specification                                                                                                                 |
-| 0.5.0   | 2023-03-30 | [24711](https://github.com/airbytehq/airbyte/pull/24711) | Add incremental sync support for `campaigns`, `deal_pipelines`, `ticket_pipelines`, `forms`, `form_submissions`, `form_submissions`, `workflows`, `owners` |
-| 0.4.0   | 2023-03-31 | [22910](https://github.com/airbytehq/airbyte/pull/22910) | Add `email_subscriptions` stream                                                                                                                           |
-| 0.3.4   | 2023-03-28 | [24641](https://github.com/airbytehq/airbyte/pull/24641) | Convert to int only numeric values                                                                                                                         |
-| 0.3.3   | 2023-03-27 | [24591](https://github.com/airbytehq/airbyte/pull/24591) | Fix pagination for `marketing emails` stream                                                                                                               |
-| 0.3.2   | 2023-02-07 | [22479](https://github.com/airbytehq/airbyte/pull/22479) | Turn on default HttpAvailabilityStrategy                                                                                                                   |
-| 0.3.1   | 2023-01-27 | [22009](https://github.com/airbytehq/airbyte/pull/22009) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                                                |
-| 0.3.0   | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available                                                                                  |
-| 0.2.2   | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation                                                                                                                         |
-| 0.2.1   | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state.                                                                                                                               |
-| 0.2.0   | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API.                                                                                   |
-| 0.1.83  | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated.                                                                                    |
-| 0.1.82  | 2022-08-18 | [15110](https://github.com/airbytehq/airbyte/pull/15110) | Check if it has a state on search streams before first sync                                                                                                |
-| 0.1.81  | 2022-08-05 | [15354](https://github.com/airbytehq/airbyte/pull/15354) | Fix `Deals` stream schema                                                                                                                                  |
-| 0.1.80  | 2022-08-01 | [15156](https://github.com/airbytehq/airbyte/pull/15156) | Fix 401 error while retrieving associations using OAuth                                                                                                    |
-| 0.1.79  | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues                                                                                                                    |
-| 0.1.78  | 2022-07-28 | [15099](https://github.com/airbytehq/airbyte/pull/15099) | Fix to fetch associations when using incremental mode                                                                                                      |
-| 0.1.77  | 2022-07-26 | [15035](https://github.com/airbytehq/airbyte/pull/15035) | Make PropertyHistory stream read historic data not limited to 30 days                                                                                      |
-| 0.1.76  | 2022-07-25 | [14999](https://github.com/airbytehq/airbyte/pull/14999) | Partially revert changes made in v0.1.75                                                                                                                   |
-| 0.1.75  | 2022-07-18 | [14744](https://github.com/airbytehq/airbyte/pull/14744) | Remove override of private CDK method                                                                                                                      |
-| 0.1.74  | 2022-07-25 | [14412](https://github.com/airbytehq/airbyte/pull/14412) | Add private app authentication                                                                                                                             |
-| 0.1.73  | 2022-07-13 | [14666](https://github.com/airbytehq/airbyte/pull/14666) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream                                                                 |
-| 0.1.72  | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging                                                                                                                                     |
-| 0.1.71  | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code                                                                                                    |
-| 0.1.70  | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue                                                                                                                  |
-| 0.1.69  | 2022-06-10 | [13691](https://github.com/airbytehq/airbyte/pull/13691) | Fix the `URI Too Long` issue                                                                                                                               |
-| 0.1.68  | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records                                                                                                  |
-| 0.1.67  | 2022-06-07 | [13566](https://github.com/airbytehq/airbyte/pull/13566) | Report which scopes are missing to the user                                                                                                                |
-| 0.1.66  | 2022-06-05 | [13475](https://github.com/airbytehq/airbyte/pull/13475) | Scope `crm.objects.feedback_submissions.read` added for `feedback_submissions` stream                                                                      |
-| 0.1.65  | 2022-06-03 | [13455](https://github.com/airbytehq/airbyte/pull/13455) | Discover only returns streams for which required scopes were granted                                                                                       |
-| 0.1.64  | 2022-06-03 | [13218](https://github.com/airbytehq/airbyte/pull/13218) | Transform `contact_lists` data to comply with schema                                                                                                       |
-| 0.1.63  | 2022-06-02 | [13320](https://github.com/airbytehq/airbyte/pull/13320) | Fix connector incremental state handling                                                                                                                   |
-| 0.1.62  | 2022-06-01 | [13383](https://github.com/airbytehq/airbyte/pull/13383) | Add `line items` to `deals` stream                                                                                                                         |
-| 0.1.61  | 2022-05-25 | [13381](https://github.com/airbytehq/airbyte/pull/13381) | Requests scopes as optional instead of required                                                                                                            |
-| 0.1.60  | 2022-05-25 | [13159](https://github.com/airbytehq/airbyte/pull/13159) | Use RFC3339 datetime                                                                                                                                       |
-| 0.1.59  | 2022-05-10 | [12711](https://github.com/airbytehq/airbyte/pull/12711) | Ensure oauth2.0 token has all needed scopes in "check" command                                                                                             |
-| 0.1.58  | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy                                                                                                                            |
-| 0.1.57  | 2022-05-04 | [12198](https://github.com/airbytehq/airbyte/pull/12198) | Add deals associations for quotes                                                                                                                          |
-| 0.1.56  | 2022-05-02 | [12515](https://github.com/airbytehq/airbyte/pull/12515) | Extra logs for troubleshooting 403 errors                                                                                                                  |
-| 0.1.55  | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream                                                                                                                  |
-| 0.1.54  | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s                                                                                                                              |
-| 0.1.53  | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Change spec json to yaml format                                                                                                                            |
-| 0.1.52  | 2022-03-25 | [11423](https://github.com/airbytehq/airbyte/pull/11423) | Add tickets associations to engagements streams                                                                                                            |
-| 0.1.51  | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue                                                                                                                      |
-| 0.1.50  | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination                                                                                                                          |
-| 0.1.49  | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration                                                                                                                    |
-| 0.1.48  | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs                                                                                                                                |
-| 0.1.47  | 2022-03-15 | [11121](https://github.com/airbytehq/airbyte/pull/11121) | Add partition keys where appropriate                                                                                                                       |
-| 0.1.46  | 2022-03-14 | [10700](https://github.com/airbytehq/airbyte/pull/10700) | Handle 10k+ records reading in Hubspot streams                                                                                                             |
-| 0.1.45  | 2022-03-04 | [10707](https://github.com/airbytehq/airbyte/pull/10707) | Remove stage history from deals stream to increase efficiency                                                                                              |
-| 0.1.44  | 2022-02-24 | [9027](https://github.com/airbytehq/airbyte/pull/9027)   | Add associations companies to deals, ticket and contact stream                                                                                             |
-| 0.1.43  | 2022-02-24 | [10576](https://github.com/airbytehq/airbyte/pull/10576) | Cast timestamp to date/datetime                                                                                                                            |
-| 0.1.42  | 2022-02-22 | [10492](https://github.com/airbytehq/airbyte/pull/10492) | Add `date-time` format to datetime fields                                                                                                                  |
-| 0.1.41  | 2022-02-21 | [10177](https://github.com/airbytehq/airbyte/pull/10177) | Migrate to CDK                                                                                                                                             |
-| 0.1.40  | 2022-02-10 | [10142](https://github.com/airbytehq/airbyte/pull/10142) | Add associations to ticket stream                                                                                                                          |
-| 0.1.39  | 2022-02-10 | [10055](https://github.com/airbytehq/airbyte/pull/10055) | Bug fix: reading not initialized stream                                                                                                                    |
-| 0.1.38  | 2022-02-03 | [9786](https://github.com/airbytehq/airbyte/pull/9786)   | Add new streams for engagements(calls, emails, meetings, notes and tasks)                                                                                  |
-| 0.1.37  | 2022-01-27 | [9555](https://github.com/airbytehq/airbyte/pull/9555)   | Getting form_submission for all forms                                                                                                                      |
-| 0.1.36  | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784)   | Add Property History Stream                                                                                                                                |
-| 0.1.35  | 2021-12-24 | [9081](https://github.com/airbytehq/airbyte/pull/9081)   | Add Feedback Submissions stream and update Ticket Pipelines stream                                                                                         |
-| 0.1.34  | 2022-01-20 | [9641](https://github.com/airbytehq/airbyte/pull/9641)   | Add more fields for `email_events` stream                                                                                                                  |
-| 0.1.33  | 2022-01-14 | [8887](https://github.com/airbytehq/airbyte/pull/8887)   | More efficient support for incremental updates on Companies, Contact, Deals and Engagement streams                                                         |
-| 0.1.32  | 2022-01-13 | [8011](https://github.com/airbytehq/airbyte/pull/8011)   | Add new stream form_submissions                                                                                                                            |
-| 0.1.31  | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385)   | Remove auto-generated `properties` from `Engagements` stream                                                                                               |
-| 0.1.30  | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129)   | Created Contacts list memberships streams                                                                                                                  |
-| 0.1.29  | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699)   | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams                  |
-| 0.1.28  | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Update fields and descriptions                                                                                                                             |
-| 0.1.27  | 2021-12-09 | [8658](https://github.com/airbytehq/airbyte/pull/8658)   | Fixed config backward compatibility issue by allowing additional properties in the spec                                                                    |
-| 0.1.26  | 2021-11-30 | [8329](https://github.com/airbytehq/airbyte/pull/8329)   | Removed 'skip_dynamic_fields' config param                                                                                                                 |
-| 0.1.25  | 2021-11-23 | [8216](https://github.com/airbytehq/airbyte/pull/8216)   | Add skip dynamic fields for testing only                                                                                                                   |
-| 0.1.24  | 2021-11-09 | [7683](https://github.com/airbytehq/airbyte/pull/7683)   | Fix name issue 'Hubspot' -> 'HubSpot'                                                                                                                      |
-| 0.1.23  | 2021-11-08 | [7730](https://github.com/airbytehq/airbyte/pull/7730)   | Fix OAuth flow schema                                                                                                                                      |
-| 0.1.22  | 2021-11-03 | [7562](https://github.com/airbytehq/airbyte/pull/7562)   | Migrate Hubspot source to CDK structure                                                                                                                    |
-| 0.1.21  | 2021-10-27 | [7405](https://github.com/airbytehq/airbyte/pull/7405)   | Change of package `import` from `urllib` to `urllib.parse`                                                                                                 |
-| 0.1.20  | 2021-10-26 | [7393](https://github.com/airbytehq/airbyte/pull/7393)   | Hotfix for `split_properties` function, add the length of separator symbol `,`(`%2C` in HTTP format) to the checking of the summary URL length             |
-| 0.1.19  | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954)   | Fix issue with getting `414` HTTP error for streams                                                                                                        |
-| 0.1.18  | 2021-10-18 | [5840](https://github.com/airbytehq/airbyte/pull/5840)   | Add new marketing emails (with statistics) stream                                                                                                          |
-| 0.1.17  | 2021-10-14 | [6995](https://github.com/airbytehq/airbyte/pull/6995)   | Update `discover` method: disable `quotes` stream when using OAuth config                                                                                  |
-| 0.1.16  | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465)   | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator                                                                 |
-| 0.1.15  | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374)   | Use correct schema for `owners` stream                                                                                                                     |
-| 0.1.14  | 2021-09-08 | [5693](https://github.com/airbytehq/airbyte/pull/5693)   | Include deal\_to\_contact association when pulling deal stream and include contact ID in contact stream                                                    |
-| 0.1.13  | 2021-09-08 | [5834](https://github.com/airbytehq/airbyte/pull/5834)   | Fixed array fields without items property in schema                                                                                                        |
-| 0.1.12  | 2021-09-02 | [5798](https://github.com/airbytehq/airbyte/pull/5798)   | Treat empty string values as None for field with format to fix normalization errors                                                                        |
-| 0.1.11  | 2021-08-26 | [5685](https://github.com/airbytehq/airbyte/pull/5685)   | Remove all date-time format from schemas                                                                                                                   |
-| 0.1.10  | 2021-08-17 | [5463](https://github.com/airbytehq/airbyte/pull/5463)   | Fix fail on reading stream using `API Key` without required permissions                                                                                    |
-| 0.1.9   | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334)   | Fix empty strings inside float datatype                                                                                                                    |
-| 0.1.8   | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250)   | Fix issue with printing exceptions                                                                                                                         |
-| 0.1.7   | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913)   | Update fields schema                                                                                                                                       |
+
+| Version | Date       | Pull Request                                             | Subject                                                                                                                                                                            |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| 1.4.1   | 2023-08-22 | [29715](https://github.com/airbytehq/airbyte/pull/29715) | Fix python package configuration  stream                                                                                                                                               |
+| 1.4.0   | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream                                                                                                                                               |
+| 1.3.3   | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string                                                                                                                                               |
+| 1.3.2   | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` |
+| 1.3.1   | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync                                |
+| 1.3.0   | 2023-08-01 | [28909](https://github.com/airbytehq/airbyte/pull/28909) | Add handling of source connection errors                                                                                                                                               |
+| 1.2.0   | 2023-07-27 | [27091](https://github.com/airbytehq/airbyte/pull/27091) | Add new stream `ContactsMergedAudit`                                                                                                                                               |
+| 1.1.2   | 2023-07-27 | [28558](https://github.com/airbytehq/airbyte/pull/28558) | Improve error messages during connector setup                                                                                                                                      |
+| 1.1.1   | 2023-07-25 | [28705](https://github.com/airbytehq/airbyte/pull/28705) | Fix retry handler for token expired error                                                                                                                                          |
+| 1.1.0   | 2023-07-18 | [28349](https://github.com/airbytehq/airbyte/pull/28349) | Add unexpected fields in schemas of streams `email_events`, `email_subscriptions`, `engagements`, `campaigns`                                                                      |
+| 1.0.1   | 2023-06-23 | [27658](https://github.com/airbytehq/airbyte/pull/27658) | Use fully qualified name to retrieve custom objects                                                                                                                                |
+| 1.0.0   | 2023-06-08 | [27161](https://github.com/airbytehq/airbyte/pull/27161) | Fixed increment sync for engagements stream, 'Recent' API is used for recent syncs of last recent 30 days and less than 10k records, otherwise full sync if performed by 'All' API |
+| 0.9.0   | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2                                                                                                                                                               |
+| 0.8.4   | 2023-05-17 | [25667](https://github.com/airbytehq/airbyte/pull/26082) | Fixed bug with wrong parsing of boolean encoded like "false" parsed as True                                                                                                        |
+| 0.8.3   | 2023-05-31 | [26831](https://github.com/airbytehq/airbyte/pull/26831) | Remove authSpecification from connector specification in favour of advancedAuth                                                                                                    |
+| 0.8.2   | 2023-05-16 | [26418](https://github.com/airbytehq/airbyte/pull/26418) | Added custom availability strategy which catches permission errors from parent streams                                                                                             |
+| 0.8.1   | 2023-05-29 | [26719](https://github.com/airbytehq/airbyte/pull/26719) | Handle issue when `state` value is literally `"" (empty str)`                                                                                                                      |
+| 0.8.0   | 2023-04-10 | [16032](https://github.com/airbytehq/airbyte/pull/16032) | Add new stream `Custom Object`                                                                                                                                                     |
+| 0.7.0   | 2023-04-10 | [24450](https://github.com/airbytehq/airbyte/pull/24450) | Add new stream `Goals`                                                                                                                                                             |
+| 0.6.2   | 2023-04-28 | [25667](https://github.com/airbytehq/airbyte/pull/25667) | Fixed bug with `Invalid Date` like `2000-00-00T00:00:00Z` while settip up the connector                                                                                            |
+| 0.6.1   | 2023-04-10 | [21423](https://github.com/airbytehq/airbyte/pull/21423) | Update scope for `DealPipelines` stream to only `crm.objects.contacts.read`                                                                                                        |
+| 0.6.0   | 2023-04-07 | [24980](https://github.com/airbytehq/airbyte/pull/24980) | Add new stream `DealsArchived`                                                                                                                                                     |
+| 0.5.2   | 2023-04-07 | [24915](https://github.com/airbytehq/airbyte/pull/24915) | Fix field key parsing (replace whitespace with uderscore)                                                                                                                          |
+| 0.5.1   | 2023-04-05 | [22982](https://github.com/airbytehq/airbyte/pull/22982) | Specified date formatting in specification                                                                                                                                         |
+| 0.5.0   | 2023-03-30 | [24711](https://github.com/airbytehq/airbyte/pull/24711) | Add incremental sync support for `campaigns`, `deal_pipelines`, `ticket_pipelines`, `forms`, `form_submissions`, `form_submissions`, `workflows`, `owners`                         |
+| 0.4.0   | 2023-03-31 | [22910](https://github.com/airbytehq/airbyte/pull/22910) | Add `email_subscriptions` stream                                                                                                                                                   |
+| 0.3.4   | 2023-03-28 | [24641](https://github.com/airbytehq/airbyte/pull/24641) | Convert to int only numeric values                                                                                                                                                 |
+| 0.3.3   | 2023-03-27 | [24591](https://github.com/airbytehq/airbyte/pull/24591) | Fix pagination for `marketing emails` stream                                                                                                                                       |
+| 0.3.2   | 2023-02-07 | [22479](https://github.com/airbytehq/airbyte/pull/22479) | Turn on default HttpAvailabilityStrategy                                                                                                                                           |
+| 0.3.1   | 2023-01-27 | [22009](https://github.com/airbytehq/airbyte/pull/22009) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                                                                        |
+| 0.3.0   | 2022-10-27 | [18546](https://github.com/airbytehq/airbyte/pull/18546) | Sunsetting API Key authentication. `Quotes` stream is no longer available                                                                                                          |
+| 0.2.2   | 2022-10-03 | [16914](https://github.com/airbytehq/airbyte/pull/16914) | Fix 403 forbidden error validation                                                                                                                                                 |
+| 0.2.1   | 2022-09-26 | [17120](https://github.com/airbytehq/airbyte/pull/17120) | Migrate to per-stream state.                                                                                                                                                       |
+| 0.2.0   | 2022-09-13 | [16632](https://github.com/airbytehq/airbyte/pull/16632) | Remove Feedback Submissions stream as the one using unstable (beta) API.                                                                                                           |
+| 0.1.83  | 2022-09-01 | [16214](https://github.com/airbytehq/airbyte/pull/16214) | Update Tickets, fix missing properties and change how state is updated.                                                                                                            |
+| 0.1.82  | 2022-08-18 | [15110](https://github.com/airbytehq/airbyte/pull/15110) | Check if it has a state on search streams before first sync                                                                                                                        |
+| 0.1.81  | 2022-08-05 | [15354](https://github.com/airbytehq/airbyte/pull/15354) | Fix `Deals` stream schema                                                                                                                                                          |
+| 0.1.80  | 2022-08-01 | [15156](https://github.com/airbytehq/airbyte/pull/15156) | Fix 401 error while retrieving associations using OAuth                                                                                                                            |
+| 0.1.79  | 2022-07-28 | [15144](https://github.com/airbytehq/airbyte/pull/15144) | Revert v0.1.78 due to permission issues                                                                                                                                            |
+| 0.1.78  | 2022-07-28 | [15099](https://github.com/airbytehq/airbyte/pull/15099) | Fix to fetch associations when using incremental mode                                                                                                                              |
+| 0.1.77  | 2022-07-26 | [15035](https://github.com/airbytehq/airbyte/pull/15035) | Make PropertyHistory stream read historic data not limited to 30 days                                                                                                              |
+| 0.1.76  | 2022-07-25 | [14999](https://github.com/airbytehq/airbyte/pull/14999) | Partially revert changes made in v0.1.75                                                                                                                                           |
+| 0.1.75  | 2022-07-18 | [14744](https://github.com/airbytehq/airbyte/pull/14744) | Remove override of private CDK method                                                                                                                                              |
+| 0.1.74  | 2022-07-25 | [14412](https://github.com/airbytehq/airbyte/pull/14412) | Add private app authentication                                                                                                                                                     |
+| 0.1.73  | 2022-07-13 | [14666](https://github.com/airbytehq/airbyte/pull/14666) | Decrease number of http requests made, disable Incremental mode for PropertyHistory stream                                                                                         |
+| 0.1.72  | 2022-06-24 | [14054](https://github.com/airbytehq/airbyte/pull/14054) | Extended error logging                                                                                                                                                             |
+| 0.1.71  | 2022-06-24 | [14102](https://github.com/airbytehq/airbyte/pull/14102) | Removed legacy `AirbyteSentry` dependency from the code                                                                                                                            |
+| 0.1.70  | 2022-06-16 | [13837](https://github.com/airbytehq/airbyte/pull/13837) | Fix the missing data in CRM streams issue                                                                                                                                          |
+| 0.1.69  | 2022-06-10 | [13691](https://github.com/airbytehq/airbyte/pull/13691) | Fix the `URI Too Long` issue                                                                                                                                                       |
+| 0.1.68  | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records                                                                                                                          |
+| 0.1.67  | 2022-06-07 | [13566](https://github.com/airbytehq/airbyte/pull/13566) | Report which scopes are missing to the user                                                                                                                                        |
+| 0.1.66  | 2022-06-05 | [13475](https://github.com/airbytehq/airbyte/pull/13475) | Scope `crm.objects.feedback_submissions.read` added for `feedback_submissions` stream                                                                                              |
+| 0.1.65  | 2022-06-03 | [13455](https://github.com/airbytehq/airbyte/pull/13455) | Discover only returns streams for which required scopes were granted                                                                                                               |
+| 0.1.64  | 2022-06-03 | [13218](https://github.com/airbytehq/airbyte/pull/13218) | Transform `contact_lists` data to comply with schema                                                                                                                               |
+| 0.1.63  | 2022-06-02 | [13320](https://github.com/airbytehq/airbyte/pull/13320) | Fix connector incremental state handling                                                                                                                                           |
+| 0.1.62  | 2022-06-01 | [13383](https://github.com/airbytehq/airbyte/pull/13383) | Add `line items` to `deals` stream                                                                                                                                                 |
+| 0.1.61  | 2022-05-25 | [13381](https://github.com/airbytehq/airbyte/pull/13381) | Requests scopes as optional instead of required                                                                                                                                    |
+| 0.1.60  | 2022-05-25 | [13159](https://github.com/airbytehq/airbyte/pull/13159) | Use RFC3339 datetime                                                                                                                                                               |
+| 0.1.59  | 2022-05-10 | [12711](https://github.com/airbytehq/airbyte/pull/12711) | Ensure oauth2.0 token has all needed scopes in "check" command                                                                                                                     |
+| 0.1.58  | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy                                                                                                                                                    |
+| 0.1.57  | 2022-05-04 | [12198](https://github.com/airbytehq/airbyte/pull/12198) | Add deals associations for quotes                                                                                                                                                  |
+| 0.1.56  | 2022-05-02 | [12515](https://github.com/airbytehq/airbyte/pull/12515) | Extra logs for troubleshooting 403 errors                                                                                                                                          |
+| 0.1.55  | 2022-04-28 | [12424](https://github.com/airbytehq/airbyte/pull/12424) | Correct schema for ticket_pipeline stream                                                                                                                                          |
+| 0.1.54  | 2022-04-28 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Mock time slep in unit test s                                                                                                                                                      |
+| 0.1.53  | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Change spec json to yaml format                                                                                                                                                    |
+| 0.1.52  | 2022-03-25 | [11423](https://github.com/airbytehq/airbyte/pull/11423) | Add tickets associations to engagements streams                                                                                                                                    |
+| 0.1.51  | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue                                                                                                                                              |
+| 0.1.50  | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination                                                                                                                                                  |
+| 0.1.49  | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration                                                                                                                                            |
+| 0.1.48  | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs                                                                                                                                                        |
+| 0.1.47  | 2022-03-15 | [11121](https://github.com/airbytehq/airbyte/pull/11121) | Add partition keys where appropriate                                                                                                                                               |
+| 0.1.46  | 2022-03-14 | [10700](https://github.com/airbytehq/airbyte/pull/10700) | Handle 10k+ records reading in Hubspot streams                                                                                                                                     |
+| 0.1.45  | 2022-03-04 | [10707](https://github.com/airbytehq/airbyte/pull/10707) | Remove stage history from deals stream to increase efficiency                                                                                                                      |
+| 0.1.44  | 2022-02-24 | [9027](https://github.com/airbytehq/airbyte/pull/9027)   | Add associations companies to deals, ticket and contact stream                                                                                                                     |
+| 0.1.43  | 2022-02-24 | [10576](https://github.com/airbytehq/airbyte/pull/10576) | Cast timestamp to date/datetime                                                                                                                                                    |
+| 0.1.42  | 2022-02-22 | [10492](https://github.com/airbytehq/airbyte/pull/10492) | Add `date-time` format to datetime fields                                                                                                                                          |
+| 0.1.41  | 2022-02-21 | [10177](https://github.com/airbytehq/airbyte/pull/10177) | Migrate to CDK                                                                                                                                                                     |
+| 0.1.40  | 2022-02-10 | [10142](https://github.com/airbytehq/airbyte/pull/10142) | Add associations to ticket stream                                                                                                                                                  |
+| 0.1.39  | 2022-02-10 | [10055](https://github.com/airbytehq/airbyte/pull/10055) | Bug fix: reading not initialized stream                                                                                                                                            |
+| 0.1.38  | 2022-02-03 | [9786](https://github.com/airbytehq/airbyte/pull/9786)   | Add new streams for engagements(calls, emails, meetings, notes and tasks)                                                                                                          |
+| 0.1.37  | 2022-01-27 | [9555](https://github.com/airbytehq/airbyte/pull/9555)   | Getting form_submission for all forms                                                                                                                                              |
+| 0.1.36  | 2022-01-22 | [7784](https://github.com/airbytehq/airbyte/pull/7784)   | Add Property History Stream                                                                                                                                                        |
+| 0.1.35  | 2021-12-24 | [9081](https://github.com/airbytehq/airbyte/pull/9081)   | Add Feedback Submissions stream and update Ticket Pipelines stream                                                                                                                 |
+| 0.1.34  | 2022-01-20 | [9641](https://github.com/airbytehq/airbyte/pull/9641)   | Add more fields for `email_events` stream                                                                                                                                          |
+| 0.1.33  | 2022-01-14 | [8887](https://github.com/airbytehq/airbyte/pull/8887)   | More efficient support for incremental updates on Companies, Contact, Deals and Engagement streams                                                                                 |
+| 0.1.32  | 2022-01-13 | [8011](https://github.com/airbytehq/airbyte/pull/8011)   | Add new stream form_submissions                                                                                                                                                    |
+| 0.1.31  | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385)   | Remove auto-generated `properties` from `Engagements` stream                                                                                                                       |
+| 0.1.30  | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129)   | Created Contacts list memberships streams                                                                                                                                          |
+| 0.1.29  | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699)   | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams                                          |
+| 0.1.28  | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Update fields and descriptions                                                                                                                                                     |
+| 0.1.27  | 2021-12-09 | [8658](https://github.com/airbytehq/airbyte/pull/8658)   | Fixed config backward compatibility issue by allowing additional properties in the spec                                                                                            |
+| 0.1.26  | 2021-11-30 | [8329](https://github.com/airbytehq/airbyte/pull/8329)   | Removed 'skip_dynamic_fields' config param                                                                                                                                         |
+| 0.1.25  | 2021-11-23 | [8216](https://github.com/airbytehq/airbyte/pull/8216)   | Add skip dynamic fields for testing only                                                                                                                                           |
+| 0.1.24  | 2021-11-09 | [7683](https://github.com/airbytehq/airbyte/pull/7683)   | Fix name issue 'Hubspot' -> 'HubSpot'                                                                                                                                              |
+| 0.1.23  | 2021-11-08 | [7730](https://github.com/airbytehq/airbyte/pull/7730)   | Fix OAuth flow schema                                                                                                                                                              |
+| 0.1.22  | 2021-11-03 | [7562](https://github.com/airbytehq/airbyte/pull/7562)   | Migrate Hubspot source to CDK structure                                                                                                                                            |
+| 0.1.21  | 2021-10-27 | [7405](https://github.com/airbytehq/airbyte/pull/7405)   | Change of package `import` from `urllib` to `urllib.parse`                                                                                                                         |
+| 0.1.20  | 2021-10-26 | [7393](https://github.com/airbytehq/airbyte/pull/7393)   | Hotfix for `split_properties` function, add the length of separator symbol `,`(`%2C` in HTTP format) to the checking of the summary URL length                                     |
+| 0.1.19  | 2021-10-26 | [6954](https://github.com/airbytehq/airbyte/pull/6954)   | Fix issue with getting `414` HTTP error for streams                                                                                                                                |
+| 0.1.18  | 2021-10-18 | [5840](https://github.com/airbytehq/airbyte/pull/5840)   | Add new marketing emails (with statistics) stream                                                                                                                                  |
+| 0.1.17  | 2021-10-14 | [6995](https://github.com/airbytehq/airbyte/pull/6995)   | Update `discover` method: disable `quotes` stream when using OAuth config                                                                                                          |
+| 0.1.16  | 2021-09-27 | [6465](https://github.com/airbytehq/airbyte/pull/6465)   | Implement OAuth support. Use CDK authenticator instead of connector specific authenticator                                                                                         |
+| 0.1.15  | 2021-09-23 | [6374](https://github.com/airbytehq/airbyte/pull/6374)   | Use correct schema for `owners` stream                                                                                                                                             |
+| 0.1.14  | 2021-09-08 | [5693](https://github.com/airbytehq/airbyte/pull/5693)   | Include deal_to_contact association when pulling deal stream and include contact ID in contact stream                                                                              |
+| 0.1.13  | 2021-09-08 | [5834](https://github.com/airbytehq/airbyte/pull/5834)   | Fixed array fields without items property in schema                                                                                                                                |
+| 0.1.12  | 2021-09-02 | [5798](https://github.com/airbytehq/airbyte/pull/5798)   | Treat empty string values as None for field with format to fix normalization errors                                                                                                |
+| 0.1.11  | 2021-08-26 | [5685](https://github.com/airbytehq/airbyte/pull/5685)   | Remove all date-time format from schemas                                                                                                                                           |
+| 0.1.10  | 2021-08-17 | [5463](https://github.com/airbytehq/airbyte/pull/5463)   | Fix fail on reading stream using `API Key` without required permissions                                                                                                            |
+| 0.1.9   | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334)   | Fix empty strings inside float datatype                                                                                                                                            |
+| 0.1.8   | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250)   | Fix issue with printing exceptions                                                                                                                                                 |
+| 0.1.7   | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913)   | Update fields schema                                                                                                                                                               |
diff --git a/docs/integrations/sources/instagram.inapp.md b/docs/integrations/sources/instagram.inapp.md
index e1562f503502..6be4f2da558e 100644
--- a/docs/integrations/sources/instagram.inapp.md
+++ b/docs/integrations/sources/instagram.inapp.md
@@ -1,7 +1,11 @@
 ## Prerequisite
 
 * [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page
- 
+
+:::info
+The Instagram connector syncs data related to Users, Media, and Stories and their insights from the [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/). For performance data related to Instagram Ads, use the Facebook Marketing source.
+:::
+
 ## Setup guide
 
 1. Click Authenticate your Instagram account.
@@ -9,4 +13,5 @@
 3. (Optional) Select a start date date. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data.
 4. Click Set up source.
 ​
+
 For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Instagram](https://docs.airbyte.com/integrations/sources/instagram).
\ No newline at end of file
diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md
index 9822b891bfdd..cfe5d8b7c9a6 100644
--- a/docs/integrations/sources/instagram.md
+++ b/docs/integrations/sources/instagram.md
@@ -4,16 +4,18 @@ This page contains the setup guide and reference information for the Instagram s
 
 ## Prerequisites
 
-* [Meta for Developers account](https://developers.facebook.com)
-* [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page
-* [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/) to your Facebook app
-* [Facebook OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference)
-* [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte)
+- [Meta for Developers account](https://developers.facebook.com)
+- [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page
+- [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/) to your Facebook app
+- [Facebook OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference)
+- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte)
 
 ## Setup Guide
+
 ### Set up the Instagram connector in Airbyte
 
 
+
 **For Airbyte Cloud:**
 
 1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -27,6 +29,7 @@ This page contains the setup guide and reference information for the Instagram s
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Log in to your Airbyte Open Source account.
@@ -36,15 +39,17 @@ This page contains the setup guide and reference information for the Instagram s
 5. Click **Authenticate your Instagram account**.
 6. Log in and authorize the Instagram account.
 7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-9. Click **Set up source**.
+8. Click **Set up source**.
 
 
 ## Supported sync modes
+
 The Instagram source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 :::note
 
@@ -53,47 +58,50 @@ Incremental sync modes are only available for the [User Insights](https://develo
 :::
 
 ## Supported Streams
+
 The Instagram source connector supports the following streams. For more information, see the [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/) and [Instagram Insights API documentation](https://developers.facebook.com/docs/instagram-api/guides/insights/).
 
-* [User](https://developers.facebook.com/docs/instagram-api/reference/ig-user)
-  * [User Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights)
-* [Media](https://developers.facebook.com/docs/instagram-api/reference/ig-user/media)
-  * [Media Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights)
-* [Stories](https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories/)
-  * [Story Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights)
+- [User](https://developers.facebook.com/docs/instagram-api/reference/ig-user)
+  - [User Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights)
+- [Media](https://developers.facebook.com/docs/instagram-api/reference/ig-user/media)
+  - [Media Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights)
+- [Stories](https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories/)
+  - [Story Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights)
 
 ### Rate Limiting and Performance Considerations
 
 Instagram limits the number of requests that can be made at a time, but the Instagram connector gracefully handles rate limiting. See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information.
 
-
 ## Data type map
+
 AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyte.com/understanding-airbyte/supported-data-types/) system. This means that all sources must produce schemas and records within these types and all destinations must handle records that conform to this type system.
 
 | Integration Type | Airbyte Type |
-|:-----------------|:-------------|
+| :--------------- | :----------- |
 | `string`         | `string`     |
 | `number`         | `number`     |
 | `array`          | `array`      |
 | `object`         | `object`     |
 
-
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                                                         |
-|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| 1.0.8   | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for insights                                                                            |
-| 1.0.7   | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove authSpecification from connector specification in favour of advancedAuth                                 |
-| 1.0.6   | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion                                                |
-| 1.0.5   | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case                                               |
-| 1.0.4   | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user                         |
-| 1.0.3   | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream                                                       |
-| 1.0.2   | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow                                                                                               |
-| 1.0.1   | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values                                                                            |
-| 1.0.0   | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state                                                     |
-| 0.1.11  | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type                                                               |
-| 0.1.10  | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81)                                                                   |
-| 0.1.9   | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438)   | Annotate Oauth2 flow initialization parameters in connector specification                                       |
-| 0.1.8   | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354)   | added check for empty state and fixed tests.                                                                    |
-| 0.1.7   | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805)   | Add support for previous format of STATE.                                                                       |
-| 0.1.6   | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210)   | Refactor connector to use CDK: - improve error handling. - fix sync fail with HTTP status 400. - integrate SAT. |
+| Version | Date       | Pull Request                                             | Subject                                                                                                                   |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ |
+| 1.0.11  | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes                                                                                      |
+| 1.0.10  | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references                                                                                  |
+| 1.0.9   | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` |
+| 1.0.8   | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights`                                                                                    |
+| 1.0.7   | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth`                                       |
+| 1.0.6   | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion                                                          |
+| 1.0.5   | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case                                                         |
+| 1.0.4   | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user                                   |
+| 1.0.3   | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream                                                                 |
+| 1.0.2   | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow                                                                                                         |
+| 1.0.1   | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values                                                                                      |
+| 1.0.0   | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state                                                               |
+| 0.1.11  | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type                                                                         |
+| 0.1.10  | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81)                                                                             |
+| 0.1.9   | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438)   | Annotate Oauth2 flow initialization parameters in connector specification                                                 |
+| 0.1.8   | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354)   | Added check for empty state and fixed tests                                                                               |
+| 0.1.7   | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805)   | Add support for previous `STATE` format                                                                                   |
+| 0.1.6   | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210)   | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT              |
diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md
index d55f5d1106ee..82bbdc6e68cf 100644
--- a/docs/integrations/sources/intercom.md
+++ b/docs/integrations/sources/intercom.md
@@ -1,87 +1,113 @@
 # Intercom
 
-This page guides you through the process of setting up the Intercom source connector.
+This page contains the setup guide and reference information for the Intercom source connector.
 
-## Set up the Intercom connector 
+## Prerequisites
 
-1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**. 
-3. On the Set up the source page, select **Intercom** from the Source type dropdown.
-4. Enter a name for your source.
-5. For **Start date**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-6. For Airbyte Cloud, click **Authenticate your Intercom account** to sign in with Intercom and authorize your account. 
-   For Airbyte Open Source, enter your [Access Token](https://developers.intercom.com/building-apps/docs/authentication-types#section-how-to-get-your-access-token) to authenticate your account.
-7. Click **Set up source**.
+- Access to an Intercom account with the data you want to replicate
+
+## Setup guide
+
+
+
+### Obtain an Intercom access token (Airbyte Open Source)
+
+To authenticate the connector in **Airbyte Open Source**, you will need to obtain an access token. You can follow the setup steps below to create an Intercom app and generate the token. For more information on Intercom's authentication flow, refer to the [official documentation](https://developers.intercom.com/building-apps/docs/authentication-types).
+
+1. Log in to your Intercom account and navigate to the [Developer Hub](https://developers.intercom.com/).
+2. Click **Your apps** in the top-right corner, then click **New app**.
+3. Choose an **App name**, select your Workspace from the dropdown, and click **Create app**.
+4. To set the appropriate permissions, from the **Authentication** tab, click **Edit** in the top right corner and check the permissions you want to grant to the app. We recommend only granting **read** permissions (not **write**). Click **Save** when you are finished.
+5. Under the **Access token** header, you will be prompted to regenerate your access token. Follow the instructions to do so, and copy the new token.
+
+
+
+### Set up the Intercom connector in Airbyte
+
+1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Intercom** from the list of available sources.
+4. Enter a **Source name** to help you identify this source.
+5. To authenticate:
+
+
+- For **Airbyte Cloud**, click **Authenticate your Intercom account**. When the pop-up appears, select the appropriate workspace from the dropdown and click **Authorize access**.
+
+
+- For **Airbyte Open Source**, enter your access token to authenticate your account.
+
+
+6. For **Start date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated.
+7. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
 The Intercom source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
- - Full Refresh
- - Incremental
+- Full Refresh
+- Incremental
 
-## Supported Streams
+## Supported streams
 
 The Intercom source connector supports the following streams:
 
-* [Admins](https://developers.intercom.com/intercom-api-reference/reference#list-admins) \(Full table\)
-* [Companies](https://developers.intercom.com/intercom-api-reference/reference#list-companies) \(Incremental\)
-  * [Company Segments](https://developers.intercom.com/intercom-api-reference/reference#list-attached-segments-1) \(Incremental\)
-* [Conversations](https://developers.intercom.com/intercom-api-reference/reference#list-conversations) \(Incremental\)
-  * [Conversation Parts](https://developers.intercom.com/intercom-api-reference/reference#get-a-single-conversation) \(Incremental\)
-* [Data Attributes](https://developers.intercom.com/intercom-api-reference/reference#data-attributes) \(Full table\)
-  * [Customer Attributes](https://developers.intercom.com/intercom-api-reference/reference#list-customer-data-attributes) \(Full table\)
-  * [Company Attributes](https://developers.intercom.com/intercom-api-reference/reference#list-company-data-attributes) \(Full table\)
-* [Contacts](https://developers.intercom.com/intercom-api-reference/reference#list-contacts) \(Incremental\)
-* [Segments](https://developers.intercom.com/intercom-api-reference/reference#list-segments) \(Incremental\)
-* [Tags](https://developers.intercom.com/intercom-api-reference/reference#list-tags-for-an-app) \(Full table\)
-* [Teams](https://developers.intercom.com/intercom-api-reference/reference#list-teams) \(Full table\)
-
+- [Admins](https://developers.intercom.com/intercom-api-reference/reference/listadmins) \(Full table\)
+- [Companies](https://developers.intercom.com/intercom-api-reference/reference/listallcompanies) \(Incremental\)
+  - [Company Segments](https://developers.intercom.com/intercom-api-reference/reference/listattachedsegmentsforcompanies) \(Incremental\)
+- [Conversations](https://developers.intercom.com/intercom-api-reference/reference/listconversations) \(Incremental\)
+  - [Conversation Parts](https://developers.intercom.com/intercom-api-reference/reference/retrieveconversation) \(Incremental\)
+- [Data Attributes](https://developers.intercom.com/intercom-api-reference/reference/lisdataattributes) \(Full table\)
+  - [Customer Attributes](https://developers.intercom.com/intercom-api-reference/reference/lisdataattributes) \(Full table\)
+  - [Company Attributes](https://developers.intercom.com/intercom-api-reference/reference/lisdataattributes) \(Full table\)
+- [Contacts](https://developers.intercom.com/intercom-api-reference/reference/listcontacts) \(Incremental\)
+- [Segments](https://developers.intercom.com/intercom-api-reference/reference/listsegments) \(Incremental\)
+- [Tags](https://developers.intercom.com/intercom-api-reference/reference/listtags) \(Full table\)
+- [Teams](https://developers.intercom.com/intercom-api-reference/reference/listteams) \(Full table\)
 
 ## Performance considerations
 
-The connector is restricted by normal Intercom [requests limitation](https://developers.intercom.com/intercom-api-reference/reference#rate-limiting).
+The connector is restricted by normal Intercom [request limitations](https://developers.intercom.com/intercom-api-reference/reference/rate-limiting).
 
 The Intercom connector should not run into Intercom API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully.
 
-
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                                       |
-|:--------| :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------- |
-| 0.2.1   | 2023-05-25 | [26571](https://github.com/airbytehq/airbyte/pull/26571) | Remove authSpecification from spec.json in favour of advancedAuth                             |
-| 0.2.0   | 2023-04-05 | [23013](https://github.com/airbytehq/airbyte/pull/23013) | Migrated to Low-code (YAML Frramework)                                                        |
-| 0.1.33  | 2023-03-20 | [22980](https://github.com/airbytehq/airbyte/pull/22980) | Specified date formatting in specification                                                    |
-| 0.1.32  | 2023-02-27 | [22095](https://github.com/airbytehq/airbyte/pull/22095) | Extended `Contacts` schema adding `opted_out_subscription_types` property                     |
-| 0.1.31  | 2023-02-17 | [23152](https://github.com/airbytehq/airbyte/pull/23152) | Add `TypeTransformer` to stream `companies`                                                   |
-| 0.1.30  | 2023-01-27 | [22010](https://github.com/airbytehq/airbyte/pull/22010) | Set `AvailabilityStrategy` for streams explicitly to `None`                                   |
-| 0.1.29  | 2022-10-31 | [18681](https://github.com/airbytehq/airbyte/pull/18681) | Define correct version for airbyte-cdk~=0.2                                                   |
-| 0.1.28  | 2022-10-20 | [18216](https://github.com/airbytehq/airbyte/pull/18216) | Use airbyte-cdk~=0.2.0 with SQLite caching                                                    |
-| 0.1.27  | 2022-08-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states.                                                                 |
-| 0.1.26  | 2022-08-18 | [16540](https://github.com/airbytehq/airbyte/pull/16540) | Fix JSON schema                                                                               |
-| 0.1.25  | 2022-08-18 | [15681](https://github.com/airbytehq/airbyte/pull/15681) | Update Intercom API to v 2.5                                                                  |
-| 0.1.24  | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas                                              |
-| 0.1.23  | 2022-07-19 | [14830](https://github.com/airbytehq/airbyte/pull/14830) | Added `checkpoint_interval` for Incremental streams                                           |
-| 0.1.22  | 2022-07-09 | [14554](https://github.com/airbytehq/airbyte/pull/14554) | Fixed `conversation_parts` stream schema definition                                           |
-| 0.1.21  | 2022-07-05 | [14403](https://github.com/airbytehq/airbyte/pull/14403) | Refactored  `Conversations`, `Conversation Parts`, `Company Segments` to increase performance |
-| 0.1.20  | 2022-06-24 | [14099](https://github.com/airbytehq/airbyte/pull/14099) | Extended `Contacts` stream schema with `sms_consent`,`unsubscribe_from_sms` properties        |
-| 0.1.19  | 2022-05-25 | [13204](https://github.com/airbytehq/airbyte/pull/13204) | Fixed `conversation_parts` stream schema definition                                           |
-| 0.1.18  | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy                                                               |
-| 0.1.17  | 2022-04-29 | [12374](https://github.com/airbytehq/airbyte/pull/12374) | Fixed filtering of conversation_parts                                                         |
-| 0.1.16  | 2022-03-23 | [11206](https://github.com/airbytehq/airbyte/pull/11206) | Added conversation_id field to conversation_part records                                      |
-| 0.1.15  | 2022-03-22 | [11176](https://github.com/airbytehq/airbyte/pull/11176) | Correct `check_connection` URL                                                                |
-| 0.1.14  | 2022-03-16 | [11208](https://github.com/airbytehq/airbyte/pull/11208) | Improve 'conversations' incremental sync speed                                                |
-| 0.1.13  | 2022-01-14 | [9513](https://github.com/airbytehq/airbyte/pull/9513)   | Added handling of scroll param when it expired                                                |
-| 0.1.12  | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Updated fields and descriptions                                                               |
-| 0.1.11  | 2021-12-13 | [8685](https://github.com/airbytehq/airbyte/pull/8685)   | Remove time.sleep for rate limit                                                              |
-| 0.1.10  | 2021-12-10 | [8637](https://github.com/airbytehq/airbyte/pull/8637)   | Fix 'conversations' order and sorting. Correction of the companies stream                     |
-| 0.1.9   | 2021-12-03 | [8395](https://github.com/airbytehq/airbyte/pull/8395)   | Fix backoff of 'companies' stream                                                             |
-| 0.1.8   | 2021-11-09 | [7060](https://github.com/airbytehq/airbyte/pull/7060)   | Added oauth support                                                                           |
-| 0.1.7   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                                               |
-| 0.1.6   | 2021-10-07 | [6879](https://github.com/airbytehq/airbyte/pull/6879)   | Corrected pagination for contacts                                                             |
-| 0.1.5   | 2021-09-28 | [6082](https://github.com/airbytehq/airbyte/pull/6082)   | Corrected android\_last\_seen\_at field data type in schemas                                  |
-| 0.1.4   | 2021-09-20 | [6087](https://github.com/airbytehq/airbyte/pull/6087)   | Corrected updated\_at field data type in schemas                                              |
-| 0.1.3   | 2021-09-08 | [5908](https://github.com/airbytehq/airbyte/pull/5908)   | Corrected timestamp and arrays in schemas                                                     |
-| 0.1.2   | 2021-08-19 | [5531](https://github.com/airbytehq/airbyte/pull/5531)   | Corrected pagination                                                                          |
-| 0.1.1   | 2021-07-31 | [5123](https://github.com/airbytehq/airbyte/pull/5123)   | Corrected rate limit                                                                          |
-| 0.1.0   | 2021-07-19 | [4676](https://github.com/airbytehq/airbyte/pull/4676)   | Release Intercom CDK Connector                                                                |
+| Version | Date       | Pull Request                                             | Subject                                                                                                  |
+|:--------| :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------- |
+| 0.3.0   | 2023-05-25 | [29598](https://github.com/airbytehq/airbyte/pull/29598) | Update custom components to make them compatible with latest cdk version, simplify logic, update schemas |
+| 0.2.1   | 2023-05-25 | [26571](https://github.com/airbytehq/airbyte/pull/26571) | Remove authSpecification from spec.json in favour of advancedAuth                                        |
+| 0.2.0   | 2023-04-05 | [23013](https://github.com/airbytehq/airbyte/pull/23013) | Migrated to Low-code (YAML Frramework)                                                                   |
+| 0.1.33  | 2023-03-20 | [22980](https://github.com/airbytehq/airbyte/pull/22980) | Specified date formatting in specification                                                               |
+| 0.1.32  | 2023-02-27 | [22095](https://github.com/airbytehq/airbyte/pull/22095) | Extended `Contacts` schema adding `opted_out_subscription_types` property                                |
+| 0.1.31  | 2023-02-17 | [23152](https://github.com/airbytehq/airbyte/pull/23152) | Add `TypeTransformer` to stream `companies`                                                              |
+| 0.1.30  | 2023-01-27 | [22010](https://github.com/airbytehq/airbyte/pull/22010) | Set `AvailabilityStrategy` for streams explicitly to `None`                                              |
+| 0.1.29  | 2022-10-31 | [18681](https://github.com/airbytehq/airbyte/pull/18681) | Define correct version for airbyte-cdk~=0.2                                                              |
+| 0.1.28  | 2022-10-20 | [18216](https://github.com/airbytehq/airbyte/pull/18216) | Use airbyte-cdk~=0.2.0 with SQLite caching                                                               |
+| 0.1.27  | 2022-08-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states.                                                                            |
+| 0.1.26  | 2022-08-18 | [16540](https://github.com/airbytehq/airbyte/pull/16540) | Fix JSON schema                                                                                          |
+| 0.1.25  | 2022-08-18 | [15681](https://github.com/airbytehq/airbyte/pull/15681) | Update Intercom API to v 2.5                                                                             |
+| 0.1.24  | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas                                                         |
+| 0.1.23  | 2022-07-19 | [14830](https://github.com/airbytehq/airbyte/pull/14830) | Added `checkpoint_interval` for Incremental streams                                                      |
+| 0.1.22  | 2022-07-09 | [14554](https://github.com/airbytehq/airbyte/pull/14554) | Fixed `conversation_parts` stream schema definition                                                      |
+| 0.1.21  | 2022-07-05 | [14403](https://github.com/airbytehq/airbyte/pull/14403) | Refactored  `Conversations`, `Conversation Parts`, `Company Segments` to increase performance            |
+| 0.1.20  | 2022-06-24 | [14099](https://github.com/airbytehq/airbyte/pull/14099) | Extended `Contacts` stream schema with `sms_consent`,`unsubscribe_from_sms` properties                   |
+| 0.1.19  | 2022-05-25 | [13204](https://github.com/airbytehq/airbyte/pull/13204) | Fixed `conversation_parts` stream schema definition                                                      |
+| 0.1.18  | 2022-05-04 | [12482](https://github.com/airbytehq/airbyte/pull/12482) | Update input configuration copy                                                                          |
+| 0.1.17  | 2022-04-29 | [12374](https://github.com/airbytehq/airbyte/pull/12374) | Fixed filtering of conversation_parts                                                                    |
+| 0.1.16  | 2022-03-23 | [11206](https://github.com/airbytehq/airbyte/pull/11206) | Added conversation_id field to conversation_part records                                                 |
+| 0.1.15  | 2022-03-22 | [11176](https://github.com/airbytehq/airbyte/pull/11176) | Correct `check_connection` URL                                                                           |
+| 0.1.14  | 2022-03-16 | [11208](https://github.com/airbytehq/airbyte/pull/11208) | Improve 'conversations' incremental sync speed                                                           |
+| 0.1.13  | 2022-01-14 | [9513](https://github.com/airbytehq/airbyte/pull/9513)   | Added handling of scroll param when it expired                                                           |
+| 0.1.12  | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Updated fields and descriptions                                                                          |
+| 0.1.11  | 2021-12-13 | [8685](https://github.com/airbytehq/airbyte/pull/8685)   | Remove time.sleep for rate limit                                                                         |
+| 0.1.10  | 2021-12-10 | [8637](https://github.com/airbytehq/airbyte/pull/8637)   | Fix 'conversations' order and sorting. Correction of the companies stream                                |
+| 0.1.9   | 2021-12-03 | [8395](https://github.com/airbytehq/airbyte/pull/8395)   | Fix backoff of 'companies' stream                                                                        |
+| 0.1.8   | 2021-11-09 | [7060](https://github.com/airbytehq/airbyte/pull/7060)   | Added oauth support                                                                                      |
+| 0.1.7   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                                                          |
+| 0.1.6   | 2021-10-07 | [6879](https://github.com/airbytehq/airbyte/pull/6879)   | Corrected pagination for contacts                                                                        |
+| 0.1.5   | 2021-09-28 | [6082](https://github.com/airbytehq/airbyte/pull/6082)   | Corrected android\_last\_seen\_at field data type in schemas                                             |
+| 0.1.4   | 2021-09-20 | [6087](https://github.com/airbytehq/airbyte/pull/6087)   | Corrected updated\_at field data type in schemas                                                         |
+| 0.1.3   | 2021-09-08 | [5908](https://github.com/airbytehq/airbyte/pull/5908)   | Corrected timestamp and arrays in schemas                                                                |
+| 0.1.2   | 2021-08-19 | [5531](https://github.com/airbytehq/airbyte/pull/5531)   | Corrected pagination                                                                                     |
+| 0.1.1   | 2021-07-31 | [5123](https://github.com/airbytehq/airbyte/pull/5123)   | Corrected rate limit                                                                                     |
+| 0.1.0   | 2021-07-19 | [4676](https://github.com/airbytehq/airbyte/pull/4676)   | Release Intercom CDK Connector                                                                           |
diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md
index 3900607d4441..ec0bb73ea0fb 100644
--- a/docs/integrations/sources/iterable.md
+++ b/docs/integrations/sources/iterable.md
@@ -20,57 +20,57 @@ To set up the Iterable source connector, you'll need the Iterable [`Server-side`
 
 The Iterable source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Campaigns](https://api.iterable.com/api/docs#campaigns_campaigns)
-* [Campaign Metrics](https://api.iterable.com/api/docs#campaigns_metrics)
-* [Channels](https://api.iterable.com/api/docs#channels_channels)
-* [Email Bounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Click](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Complaint](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Open](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Send](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Send Skip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Subscribe](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Email Unsubscribe](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Events](https://api.iterable.com/api/docs#events_User_events)
-* [Lists](https://api.iterable.com/api/docs#lists_getLists)
-* [List Users](https://api.iterable.com/api/docs#lists_getLists_0)
-* [Message Types](https://api.iterable.com/api/docs#messageTypes_messageTypes)
-* [Metadata](https://api.iterable.com/api/docs#metadata_list_tables)
-* [Templates](https://api.iterable.com/api/docs#templates_getTemplates) \(Incremental\)
-* [Users](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [PushSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [PushSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [PushOpen](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [PushUninstall](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [PushBounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [WebPushSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [WebPushClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [WebPushSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppOpen](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppClose](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppDelete](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppDelivery](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InAppSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InboxSession](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [InboxMessageImpression](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [SmsSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [SmsBounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [SmsClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [SmsReceived](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [SmsSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [SmsUsageInfo](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [Purchase](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [CustomEvent](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
-* [HostedUnsubscribeClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Campaigns](https://api.iterable.com/api/docs#campaigns_campaigns)
+- [Campaign Metrics](https://api.iterable.com/api/docs#campaigns_metrics)
+- [Channels](https://api.iterable.com/api/docs#channels_channels)
+- [Email Bounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Click](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Complaint](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Open](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Send](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Send Skip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Subscribe](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Email Unsubscribe](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Events](https://api.iterable.com/api/docs#events_User_events)
+- [Lists](https://api.iterable.com/api/docs#lists_getLists)
+- [List Users](https://api.iterable.com/api/docs#lists_getLists_0)
+- [Message Types](https://api.iterable.com/api/docs#messageTypes_messageTypes)
+- [Metadata](https://api.iterable.com/api/docs#metadata_list_tables)
+- [Templates](https://api.iterable.com/api/docs#templates_getTemplates) \(Incremental\)
+- [Users](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [PushSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [PushSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [PushOpen](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [PushUninstall](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [PushBounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [WebPushSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [WebPushClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [WebPushSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppOpen](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppClose](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppDelete](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppDelivery](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InAppSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InboxSession](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [InboxMessageImpression](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [SmsSend](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [SmsBounce](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [SmsClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [SmsReceived](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [SmsSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [SmsUsageInfo](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [Purchase](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [CustomEvent](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
+- [HostedUnsubscribeClick](https://api.iterable.com/api/docs#export_exportDataJson) \(Incremental\)
 
 ## Additional notes
 
@@ -78,28 +78,29 @@ The Iterable source connector supports the following [sync modes](https://docs.a
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------|
-| 0.1.28  | 2023-05-12 | [26014](https://github.com/airbytehq/airbyte/pull/26014) | Improve 500 handling for Events stream                                     |
-| 0.1.27  | 2023-04-06 | [24962](https://github.com/airbytehq/airbyte/pull/24962) | `UserList` stream when meet `500 - Generic Error` will skip a broken slice and keep going with the next one |
-| 0.1.26  | 2023-03-10 | [23938](https://github.com/airbytehq/airbyte/pull/23938) | Improve retry for `500 - Generic Error` |
+| Version | Date       | Pull Request                                             | Subject                                                                                                      |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------- |
+| 0.1.30  | 2023-07-19 | [28457](https://github.com/airbytehq/airbyte/pull/28457) | Fixed TypeError for StreamSlice in debug mode                                                                |
+| 0.1.29  | 2023-05-24 | [26459](https://github.com/airbytehq/airbyte/pull/26459) | Added requests reading timeout 300 seconds                                                                   |
+| 0.1.28  | 2023-05-12 | [26014](https://github.com/airbytehq/airbyte/pull/26014) | Improve 500 handling for Events stream                                                                       |
+| 0.1.27  | 2023-04-06 | [24962](https://github.com/airbytehq/airbyte/pull/24962) | `UserList` stream when meet `500 - Generic Error` will skip a broken slice and keep going with the next one  |
+| 0.1.26  | 2023-03-10 | [23938](https://github.com/airbytehq/airbyte/pull/23938) | Improve retry for `500 - Generic Error`                                                                      |
 | 0.1.25  | 2023-03-07 | [23821](https://github.com/airbytehq/airbyte/pull/23821) | Added retry for `500 - Generic Error`, increased max attempts number to `6` to handle `ChunkedEncodingError` |
-| 0.1.24  | 2023-02-14 | [22979](https://github.com/airbytehq/airbyte/pull/22979) | Specified date formatting in specification  |
-| 0.1.23  | 2023-01-27 | [22011](https://github.com/airbytehq/airbyte/pull/22011) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                     |
-| 0.1.22  | 2022-11-30 | [19913](https://github.com/airbytehq/airbyte/pull/19913) | Replace pendulum.parse -> dateutil.parser.parse to avoid memory leak       |
-| 0.1.21  | 2022-10-27 | [18537](https://github.com/airbytehq/airbyte/pull/18537) | Improve streams discovery                                                  |
-| 0.1.20  | 2022-10-21 | [18292](https://github.com/airbytehq/airbyte/pull/18292) | Better processing of 401 and 429 errors                                    |
-| 0.1.19  | 2022-10-05 | [17602](https://github.com/airbytehq/airbyte/pull/17602) | Add check for stream permissions                                           |
-| 0.1.18  | 2022-10-04 | [17573](https://github.com/airbytehq/airbyte/pull/17573) | Limit time range for SATs                                                  |
-| 0.1.17  | 2022-09-02 | [16067](https://github.com/airbytehq/airbyte/pull/16067) | added new events streams                                                   |
-| 0.1.16  | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header                                               |
-| 0.1.15  | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524)   | Update connector fields title/description                                  |
-| 0.1.14  | 2021-12-01 | [8380](https://github.com/airbytehq/airbyte/pull/8380)   | Update `Events` stream to use `export/userEvents` endpoint                 |
-| 0.1.13  | 2021-11-22 | [8091](https://github.com/airbytehq/airbyte/pull/8091)   | Adjust slice ranges for email streams                                      |
-| 0.1.12  | 2021-11-09 | [7780](https://github.com/airbytehq/airbyte/pull/7780)   | Split EmailSend stream into slices to fix premature connection close error |
-| 0.1.11  | 2021-11-03 | [7619](https://github.com/airbytehq/airbyte/pull/7619)   | Bugfix type error while incrementally loading the `Templates` stream       |
-| 0.1.10  | 2021-11-03 | [7591](https://github.com/airbytehq/airbyte/pull/7591)   | Optimize export streams memory consumption for large requests              |
-| 0.1.9   | 2021-10-06 | [5915](https://github.com/airbytehq/airbyte/pull/5915)   | Enable campaign_metrics stream                                             |
-| 0.1.8   | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915)   | Add new streams: campaign_metrics, events                                  |
-| 0.1.7   | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242)   | Updated schema for: campaigns, lists, templates, metadata                  |
-
+| 0.1.24  | 2023-02-14 | [22979](https://github.com/airbytehq/airbyte/pull/22979) | Specified date formatting in specification                                                                   |
+| 0.1.23  | 2023-01-27 | [22011](https://github.com/airbytehq/airbyte/pull/22011) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                  |
+| 0.1.22  | 2022-11-30 | [19913](https://github.com/airbytehq/airbyte/pull/19913) | Replace pendulum.parse -> dateutil.parser.parse to avoid memory leak                                         |
+| 0.1.21  | 2022-10-27 | [18537](https://github.com/airbytehq/airbyte/pull/18537) | Improve streams discovery                                                                                    |
+| 0.1.20  | 2022-10-21 | [18292](https://github.com/airbytehq/airbyte/pull/18292) | Better processing of 401 and 429 errors                                                                      |
+| 0.1.19  | 2022-10-05 | [17602](https://github.com/airbytehq/airbyte/pull/17602) | Add check for stream permissions                                                                             |
+| 0.1.18  | 2022-10-04 | [17573](https://github.com/airbytehq/airbyte/pull/17573) | Limit time range for SATs                                                                                    |
+| 0.1.17  | 2022-09-02 | [16067](https://github.com/airbytehq/airbyte/pull/16067) | added new events streams                                                                                     |
+| 0.1.16  | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header                                                                                 |
+| 0.1.15  | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524)   | Update connector fields title/description                                                                    |
+| 0.1.14  | 2021-12-01 | [8380](https://github.com/airbytehq/airbyte/pull/8380)   | Update `Events` stream to use `export/userEvents` endpoint                                                   |
+| 0.1.13  | 2021-11-22 | [8091](https://github.com/airbytehq/airbyte/pull/8091)   | Adjust slice ranges for email streams                                                                        |
+| 0.1.12  | 2021-11-09 | [7780](https://github.com/airbytehq/airbyte/pull/7780)   | Split EmailSend stream into slices to fix premature connection close error                                   |
+| 0.1.11  | 2021-11-03 | [7619](https://github.com/airbytehq/airbyte/pull/7619)   | Bugfix type error while incrementally loading the `Templates` stream                                         |
+| 0.1.10  | 2021-11-03 | [7591](https://github.com/airbytehq/airbyte/pull/7591)   | Optimize export streams memory consumption for large requests                                                |
+| 0.1.9   | 2021-10-06 | [5915](https://github.com/airbytehq/airbyte/pull/5915)   | Enable campaign_metrics stream                                                                               |
+| 0.1.8   | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915)   | Add new streams: campaign_metrics, events                                                                    |
+| 0.1.7   | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242)   | Updated schema for: campaigns, lists, templates, metadata                                                    |
diff --git a/docs/integrations/sources/jira.inapp.md b/docs/integrations/sources/jira.inapp.md
new file mode 100644
index 000000000000..671fba287cfc
--- /dev/null
+++ b/docs/integrations/sources/jira.inapp.md
@@ -0,0 +1,20 @@
+## Prerequisites
+
+- Access to a JIRA account
+- [JIRA API Token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)
+- JIRA Account Domain
+
+## Setup guide
+
+1. Enter a name for the connector.
+2. Enter the **API Token** that you have created. The **API Token** is used for Authorization to your account.
+2. Enter the **Domain** for your Jira account, e.g. `airbyte.atlassian.net`.
+3. Enter the **Email** for your Jira account which you used to generate the API token. This field is used for Authorization to your account.
+4. (Optional) Enter the list of **Projects** for which you need to replicate data. If empty, data from all projects will be replicated.
+5. (Optional) Enter the **Start Date** from which you'd like to replicate data for Jira in the format YYYY-MM-DDTHH:MM:SSZ. All data generated after this date will be replicated. If empty, all data will be replicated. Note that it will be used only in the following streams: `BoardIssues`, `IssueComments`, `IssueProperties`, `IssueRemoteLinks`, `IssueVotes`, `IssueWatchers`, `IssueWorklogs`, `Issues`, `PullRequests`, `SprintIssues`. For other streams, it will replicate all data.  
+9. Toggle **Expand Issue Changelog** to get a list of updates to every issue in the Issues stream. If the toggle is off, the changelog will not be pulled.
+10. Toggle **Render Issue Fields** to additionally return field values rendered in HTML format in the Issues stream. Issue fields will always be returned in JSON format.
+11. Toggle **Enable Experimental Streams** to enable syncing for undocumented internal JIRA API endpoints and may stop working if those enpoints undergo major changes. Currently, this only applies to the PullRequests stream. 
+10. Click **Set up source**
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [JIRA](https://docs.airbyte.com/integrations/sources/jira).
diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md
index 6d68adaba869..aea4d3cac20d 100644
--- a/docs/integrations/sources/jira.md
+++ b/docs/integrations/sources/jira.md
@@ -25,7 +25,7 @@ This page contains the setup guide and reference information for the Jira source
 5. Enter the **Domain** for your Jira account, e.g. `airbyteio.atlassian.net`.
 6. Enter the **Email** for your Jira account which you used to generate the API token. This field is used for Authorization to your account by BasicAuth.
 7. Enter the list of **Projects (Optional)** for which you need to replicate data, or leave it empty if you want to replicate data for all projects.
-8. Enter the **Start Date (Optional)** from which you'd like to replicate data for Jira in the format YYYY-MM-DDTHH:MM:SSZ. All data generated after this date will be replicated, or leave it empty if you want to replicate all data. Note that it will be used only in the following streams:BoardIssues, IssueComments, IssueProperties, IssueRemoteLinks, IssueVotes, IssueWatchers, IssueWorklogs, Issues, PullRequests, SprintIssues. For other streams it will replicate all data.  
+8. Enter the **Start Date (Optional)** from which you'd like to replicate data for Jira in the format YYYY-MM-DDTHH:MM:SSZ. All data generated after this date will be replicated, or leave it empty if you want to replicate all data. Note that it will be used only in the following streams:BoardIssues, IssueComments, IssueProperties, IssueRemoteLinks, IssueVotes, IssueWatchers, IssueWorklogs, Issues, PullRequests, SprintIssues. For other streams it will replicate all data.
 9. Toggle **Expand Issue Changelog** allows you to get a list of recent updates to every issue in the Issues stream.
 10. Toggle **Render Issue Fields** allows returning field values rendered in HTML format in the Issues stream.
 11. Toggle **Enable Experimental Streams** enables experimental PullRequests stream.
@@ -37,70 +37,70 @@ The Jira source connector supports the following [sync modes](https://docs.airby
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Troubleshooting
 
-Check out common troubleshooting issues for the Jira connector on our Discourse [here](https://discuss.airbyte.io/tags/c/connector/11/source-jira).
+Check out common troubleshooting issues for the Jira connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions).
 
 ## Supported Streams
 
 This connector outputs the following full refresh streams:
 
-* [Application roles](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-application-roles/#api-rest-api-3-applicationrole-get)
-* [Avatars](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-avatars/#api-rest-api-3-avatar-type-system-get)
-* [Boards](https://developer.atlassian.com/cloud/jira/software/rest/api-group-other-operations/#api-agile-1-0-board-get)
-* [Dashboards](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-dashboards/#api-rest-api-3-dashboard-get)
-* [Filters](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-filters/#api-rest-api-3-filter-search-get)
-* [Filter sharing](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-filter-sharing/#api-rest-api-3-filter-id-permission-get)
-* [Groups](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-groups/#api-rest-api-3-groups-picker-get)
-* [Issue fields](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-fields/#api-rest-api-3-field-get)
-* [Issue field configurations](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-field-configurations/#api-rest-api-3-fieldconfiguration-get)
-* [Issue custom field contexts](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-custom-field-contexts/#api-rest-api-3-field-fieldid-context-get)
-* [Issue link types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-link-types/#api-rest-api-3-issuelinktype-get)
-* [Issue navigator settings](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-navigator-settings/#api-rest-api-3-settings-columns-get)
-* [Issue notification schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-notification-schemes/#api-rest-api-3-notificationscheme-get)
-* [Issue priorities](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-priorities/#api-rest-api-3-priority-get)
-* [Issue properties](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-properties/#api-rest-api-3-issue-issueidorkey-properties-propertykey-get)
-* [Issue remote links](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-remote-links/#api-rest-api-3-issue-issueidorkey-remotelink-get)
-* [Issue resolutions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-resolutions/#api-rest-api-3-resolution-search-get)
-* [Issue security schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-security-schemes/#api-rest-api-3-issuesecurityschemes-get)
-* [Issue type schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-schemes/#api-rest-api-3-issuetypescheme-get)
-* [Issue type screen schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-screen-schemes/#api-rest-api-3-issuetypescreenscheme-get)
-* [Issue votes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-votes/#api-group-issue-votes)
-* [Issue watchers](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-watchers/#api-rest-api-3-issue-issueidorkey-watchers-get)
-* [Jira settings](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jira-settings/#api-rest-api-3-application-properties-get)
-* [Labels](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-labels/#api-rest-api-3-label-get)
-* [Permissions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-permissions/#api-rest-api-3-mypermissions-get)
-* [Permission schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-permission-schemes/#api-rest-api-3-permissionscheme-get)
-* [Projects](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-projects/#api-rest-api-3-project-search-get)
-* [Project avatars](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-avatars/#api-rest-api-3-project-projectidorkey-avatars-get)
-* [Project categories](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-categories/#api-rest-api-3-projectcategory-get)
-* [Project components](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-components/#api-rest-api-3-project-projectidorkey-component-get)
-* [Project email](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-email/#api-rest-api-3-project-projectid-email-get)
-* [Project permission schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-permission-schemes/#api-group-project-permission-schemes)
-* [Project types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-types/#api-rest-api-3-project-type-get)
-* [Project versions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-versions/#api-rest-api-3-project-projectidorkey-version-get)
-* [Screens](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screens/#api-rest-api-3-screens-get)
-* [Screen tabs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tabs/#api-rest-api-3-screens-screenid-tabs-get)
-* [Screen tab fields](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tab-fields/#api-rest-api-3-screens-screenid-tabs-tabid-fields-get)
-* [Screen schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-schemes/#api-rest-api-3-screenscheme-get)
-* [Sprints](https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-sprint-get)
-* [Time tracking](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-time-tracking/#api-rest-api-3-configuration-timetracking-list-get)
-* [Users](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-user-search/#api-rest-api-3-user-search-get)
-* [UsersGroupsDetailed](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-users/#api-rest-api-3-user-get)
-* [Workflows](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflows/#api-rest-api-3-workflow-search-get)
-* [Workflow schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-schemes/#api-rest-api-3-workflowscheme-get)
-* [Workflow statuses](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-statuses/#api-rest-api-3-status-get)
-* [Workflow status categories](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-status-categories/#api-rest-api-3-statuscategory-get)
+- [Application roles](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-application-roles/#api-rest-api-3-applicationrole-get)
+- [Avatars](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-avatars/#api-rest-api-3-avatar-type-system-get)
+- [Boards](https://developer.atlassian.com/cloud/jira/software/rest/api-group-other-operations/#api-agile-1-0-board-get)
+- [Dashboards](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-dashboards/#api-rest-api-3-dashboard-get)
+- [Filters](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-filters/#api-rest-api-3-filter-search-get)
+- [Filter sharing](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-filter-sharing/#api-rest-api-3-filter-id-permission-get)
+- [Groups](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-groups/#api-rest-api-3-groups-picker-get)
+- [Issue fields](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-fields/#api-rest-api-3-field-get)
+- [Issue field configurations](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-field-configurations/#api-rest-api-3-fieldconfiguration-get)
+- [Issue custom field contexts](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-custom-field-contexts/#api-rest-api-3-field-fieldid-context-get)
+- [Issue link types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-link-types/#api-rest-api-3-issuelinktype-get)
+- [Issue navigator settings](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-navigator-settings/#api-rest-api-3-settings-columns-get)
+- [Issue notification schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-notification-schemes/#api-rest-api-3-notificationscheme-get)
+- [Issue priorities](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-priorities/#api-rest-api-3-priority-get)
+- [Issue properties](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-properties/#api-rest-api-3-issue-issueidorkey-properties-propertykey-get)
+- [Issue remote links](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-remote-links/#api-rest-api-3-issue-issueidorkey-remotelink-get)
+- [Issue resolutions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-resolutions/#api-rest-api-3-resolution-search-get)
+- [Issue security schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-security-schemes/#api-rest-api-3-issuesecurityschemes-get)
+- [Issue type schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-schemes/#api-rest-api-3-issuetypescheme-get)
+- [Issue type screen schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-screen-schemes/#api-rest-api-3-issuetypescreenscheme-get)
+- [Issue votes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-votes/#api-group-issue-votes)
+- [Issue watchers](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-watchers/#api-rest-api-3-issue-issueidorkey-watchers-get)
+- [Jira settings](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jira-settings/#api-rest-api-3-application-properties-get)
+- [Labels](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-labels/#api-rest-api-3-label-get)
+- [Permissions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-permissions/#api-rest-api-3-mypermissions-get)
+- [Permission schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-permission-schemes/#api-rest-api-3-permissionscheme-get)
+- [Projects](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-projects/#api-rest-api-3-project-search-get)
+- [Project avatars](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-avatars/#api-rest-api-3-project-projectidorkey-avatars-get)
+- [Project categories](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-categories/#api-rest-api-3-projectcategory-get)
+- [Project components](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-components/#api-rest-api-3-project-projectidorkey-component-get)
+- [Project email](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-email/#api-rest-api-3-project-projectid-email-get)
+- [Project permission schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-permission-schemes/#api-group-project-permission-schemes)
+- [Project types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-types/#api-rest-api-3-project-type-get)
+- [Project versions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-versions/#api-rest-api-3-project-projectidorkey-version-get)
+- [Screens](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screens/#api-rest-api-3-screens-get)
+- [Screen tabs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tabs/#api-rest-api-3-screens-screenid-tabs-get)
+- [Screen tab fields](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-tab-fields/#api-rest-api-3-screens-screenid-tabs-tabid-fields-get)
+- [Screen schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screen-schemes/#api-rest-api-3-screenscheme-get)
+- [Sprints](https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-sprint-get)
+- [Time tracking](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-time-tracking/#api-rest-api-3-configuration-timetracking-list-get)
+- [Users](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-user-search/#api-rest-api-3-user-search-get)
+- [UsersGroupsDetailed](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-users/#api-rest-api-3-user-get)
+- [Workflows](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflows/#api-rest-api-3-workflow-search-get)
+- [Workflow schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-schemes/#api-rest-api-3-workflowscheme-get)
+- [Workflow statuses](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-statuses/#api-rest-api-3-status-get)
+- [Workflow status categories](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-workflow-status-categories/#api-rest-api-3-statuscategory-get)
 
 This connector outputs the following incremental streams:
 
-* [Board issues](https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-issue-get)
-* [Issue comments](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-comments/#api-rest-api-3-issue-issueidorkey-comment-get)
-* [Issue worklogs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-worklogs/#api-rest-api-3-issue-issueidorkey-worklog-get)
-* [Issues](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/#api-rest-api-3-search-get)
-* [Sprint issues](https://developer.atlassian.com/cloud/jira/software/rest/api-group-sprint/#api-rest-agile-1-0-sprint-sprintid-issue-get)
+- [Board issues](https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-issue-get)
+- [Issue comments](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-comments/#api-rest-api-3-issue-issueidorkey-comment-get)
+- [Issue worklogs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-worklogs/#api-rest-api-3-issue-issueidorkey-worklog-get)
+- [Issues](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/#api-rest-api-3-search-get)
+- [Sprint issues](https://developer.atlassian.com/cloud/jira/software/rest/api-group-sprint/#api-rest-agile-1-0-sprint-sprintid-issue-get)
 
 If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose)
 
@@ -112,11 +112,11 @@ While they will not cause a sync to fail, they may not be able to pull any data.
 Use the "Enable Experimental Streams" option when setting up the source to allow
 or disallow these tables to be selected when configuring a connection.
 
-* Pull Requests (currently only GitHub PRs are supported)
+- Pull Requests (currently only GitHub PRs are supported)
 
 ## Troubleshooting
 
-Check out common troubleshooting issues for the Jira connector on our Discourse [here](https://discuss.airbyte.io/tags/c/connector/11/source-jira).
+Check out common troubleshooting issues for the Jira connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions).
 
 ## Rate Limiting & Performance
 
@@ -125,7 +125,8 @@ The Jira connector should not run into Jira API limitations under normal usage.
 ## CHANGELOG
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                 |
-|:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------- |
+| 0.3.12  | 2023-06-01 | [\#26652](https://github.com/airbytehq/airbyte/pull/26652) | Expand on `leads` for `projects` stream                                                                                 |
 | 0.3.11  | 2023-06-01 | [\#26906](https://github.com/airbytehq/airbyte/pull/26906) | Handle project permissions error                                                                                        |
 | 0.3.10  | 2023-05-26 | [\#26652](https://github.com/airbytehq/airbyte/pull/26652) | Fixed bug when `board` doesn't support `sprints`                                                                        |
 | 0.3.9   | 2023-05-16 | [\#26114](https://github.com/airbytehq/airbyte/pull/26114) | Update fields info in docs and spec, update to latest airbyte-cdk                                                       |
@@ -156,6 +157,5 @@ The Jira connector should not run into Jira API limitations under normal usage.
 | 0.2.7   | 2021-07-19 | [\#4817](https://github.com/airbytehq/airbyte/pull/4817)   | Fixed `labels` schema properties issue.                                                                                 |
 | 0.2.6   | 2021-06-15 | [\#4113](https://github.com/airbytehq/airbyte/pull/4113)   | Fixed `user` stream with the correct endpoint and query param.                                                          |
 | 0.2.5   | 2021-06-09 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973)   | Added `AIRBYTE_ENTRYPOINT` in base Docker image for Kubernetes support.                                                 |
-| 0.2.4   |            |                                                            | Implementing base\_read acceptance test dived by stream groups.                                                         |
+| 0.2.4   |            |                                                            | Implementing base_read acceptance test dived by stream groups.                                                          |
 | 0.2.3   |            |                                                            | Implementing incremental sync. Migrated to airbyte-cdk. Adding all available entities in Jira Cloud.                    |
-
diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md
index bedcf0bdd13b..e8d1ce170503 100644
--- a/docs/integrations/sources/klaviyo.md
+++ b/docs/integrations/sources/klaviyo.md
@@ -6,7 +6,6 @@ This page contains the setup guide and reference information for the Klaviyo sou
 
 To set up the Klaviyo source connector, you'll need the [Klaviyo Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3).
 
-
 ## Set up the Klaviyo connector in Airbyte
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or navigate to the Airbyte Open Source dashboard.
@@ -14,27 +13,27 @@ To set up the Klaviyo source connector, you'll need the [Klaviyo Private API key
 3. On the Set up the source page, select **Klaviyo** from the Source type dropdown.
 4. Enter the name for the Klaviyo connector.
 5. For **Api Key**, enter the Klaviyo [Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3).
-6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. 
+6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated.
 7. Click **Set up source**.
 
 ## Supported sync modes
 
 The Klaviyo source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Campaigns](https://developers.klaviyo.com/en/v1-2/reference/get-campaigns#get-campaigns)
-* [Events](https://developers.klaviyo.com/en/v1-2/reference/metrics-timeline)
-* [GlobalExclusions](https://developers.klaviyo.com/en/v1-2/reference/get-global-exclusions)
-* [Lists](https://developers.klaviyo.com/en/v1-2/reference/get-lists)
-* [Metrics](https://developers.klaviyo.com/en/v1-2/reference/get-metrics)
-* [Flows](https://developers.klaviyo.com/en/reference/get_flows)
-* [Profiles](https://developers.klaviyo.com/en/reference/get_profiles)
+- [Campaigns](https://developers.klaviyo.com/en/v1-2/reference/get-campaigns#get-campaigns)
+- [Events](https://developers.klaviyo.com/en/v1-2/reference/metrics-timeline)
+- [GlobalExclusions](https://developers.klaviyo.com/en/v1-2/reference/get-global-exclusions)
+- [Lists](https://developers.klaviyo.com/en/v1-2/reference/get-lists)
+- [Metrics](https://developers.klaviyo.com/en/v1-2/reference/get-metrics)
+- [Flows](https://developers.klaviyo.com/en/reference/get_flows)
+- [Profiles](https://developers.klaviyo.com/en/reference/get_profiles)
 
 ## Performance considerations
 
@@ -45,7 +44,7 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `number`         | `number`     |       |
 | `array`          | `array`      |       |
@@ -54,16 +53,18 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u
 ## Changelog
 
 | Version  | Date       | Pull Request                                               | Subject                                                                                   |
-|:---------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------|
-| `0.3.0`  | 2023-02-18 | [23236](https://github.com/airbytehq/airbyte/pull/23236)   | Add ` Email Templates` stream 
-| `0.2.0` | 2023-03-13 | [22942](https://github.com/airbytehq/airbyte/pull/23968)   | Add `Profiles` stream
-| `0.1.13` | 2023-02-13 | [22942](https://github.com/airbytehq/airbyte/pull/22942)   | Specified date formatting in specification                                                                |
+| :------- | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------- |
+| `0.3.2`  | 2023-06-20 | [27498](https://github.com/airbytehq/airbyte/pull/27498)   | Do not store state in the future                                                          |
+| `0.3.1`  | 2023-06-08 | [27162](https://github.com/airbytehq/airbyte/pull/27162)   | Anonymize check connection error message                                                  |
+| `0.3.0`  | 2023-02-18 | [23236](https://github.com/airbytehq/airbyte/pull/23236)   | Add ` Email Templates` stream                                                             |
+| `0.2.0`  | 2023-03-13 | [22942](https://github.com/airbytehq/airbyte/pull/23968)   | Add `Profiles` stream                                                                     |
+| `0.1.13` | 2023-02-13 | [22942](https://github.com/airbytehq/airbyte/pull/22942)   | Specified date formatting in specification                                                |
 | `0.1.12` | 2023-01-30 | [22071](https://github.com/airbytehq/airbyte/pull/22071)   | Fix `Events` stream schema                                                                |
 | `0.1.11` | 2023-01-27 | [22012](https://github.com/airbytehq/airbyte/pull/22012)   | Set `AvailabilityStrategy` for streams explicitly to `None`                               |
 | `0.1.10` | 2022-09-29 | [17422](https://github.com/airbytehq/airbyte/issues/17422) | Update CDK dependency                                                                     |
-| `0.1.9`  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state.                                                              |                                                      
+| `0.1.9`  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state.                                                              |
 | `0.1.6`  | 2022-07-20 | [14872](https://github.com/airbytehq/airbyte/issues/14872) | Increase test coverage                                                                    |
-| `0.1.5`  | 2022-07-12 | [14617](https://github.com/airbytehq/airbyte/issues/14617) | Set max\_retries = 10 for `lists` stream.                                                 |
+| `0.1.5`  | 2022-07-12 | [14617](https://github.com/airbytehq/airbyte/issues/14617) | Set max_retries = 10 for `lists` stream.                                                  |
 | `0.1.4`  | 2022-04-15 | [11723](https://github.com/airbytehq/airbyte/issues/11723) | Enhance klaviyo source for flows stream and update to events stream.                      |
 | `0.1.3`  | 2021-12-09 | [8592](https://github.com/airbytehq/airbyte/pull/8592)     | Improve performance, make Global Exclusions stream incremental and enable Metrics stream. |
 | `0.1.2`  | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952)     | Update schema validation in SAT                                                           |
diff --git a/docs/integrations/sources/kyve.md b/docs/integrations/sources/kyve.md
new file mode 100644
index 000000000000..fbb3ff6f694d
--- /dev/null
+++ b/docs/integrations/sources/kyve.md
@@ -0,0 +1,23 @@
+# Kyve Source
+
+This page contains the setup guide and reference information for the **KYVE** source connector.
+
+The KYVE Data Pipeline enables easy import of KYVE data into any data warehouse or destination
+supported by [Airbyte](https://airbyte.com/). With the `ELT` format, data analysts and engineers can now confidently source KYVE data without worrying about its validity or reliability.
+
+For information about how to setup an end to end pipeline with this connector, see [the documentation](https://docs.kyve.network/data_engineers/accessing_data/elt_pipeline/overview).
+
+## Source configuration setup
+
+1. In order to  create an ELT pipeline with KYVE source you should specify the **`Pool-ID`** of [Kyve storage pool](https://app.kyve.network/#/pools) from which you want to retrieve data.
+
+2. You can specify a specific **`Bundle-Start-ID`** in case you want to narrow the records that will be retrieved from the pool. You can find the valid bundles of in the KYVE app (e.g. [Moonbeam pool bundles](https://app.kyve.network/#/pools/0/bundles)).
+
+## Multiple pools
+You can fetch with one source configuration more than one pool simultaneously. You just need to specify the **`Pool-IDs`** and the **`Bundle-Start-ID`** for the KYVE storage pool you want to archive separated with comma.
+
+## Changelog
+
+| Version | Date | Pull Request | Subject          |
+|:--------| :--- | :----------- | :--------------- |
+| 0.1.0   | 25-05-29  | [26299](https://github.com/airbytehq/airbyte/pull/26299) | Initial release of KYVE source connector|
diff --git a/docs/integrations/sources/lever-hiring.md b/docs/integrations/sources/lever-hiring.md
index 4c7b57715dc6..d364368a96ad 100644
--- a/docs/integrations/sources/lever-hiring.md
+++ b/docs/integrations/sources/lever-hiring.md
@@ -43,7 +43,7 @@ The Lever Hiring connector should not run into Lever Hiring API limitations unde
 
 | Version | Date       | Pull Request                                             | Subject                           |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------|
-| 0.2.0   | 2023-05-25 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add Basic Auth management         |
+| 0.2.0   | 2023-05-25 | [26564](https://github.com/airbytehq/airbyte/pull/26564) | Migrate to advancedAuth           |
 | 0.1.3   | 2022-10-14 | [17996](https://github.com/airbytehq/airbyte/pull/17996) | Add Basic Auth management         |
 | 0.1.2   | 2021-12-30 | [9214](https://github.com/airbytehq/airbyte/pull/9214)   | Update title and descriptions     |
 | 0.1.1   | 2021-12-16 | [7677](https://github.com/airbytehq/airbyte/pull/7677)   | OAuth Automated Authentication    |
diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md
index 8abe1ac80a4d..0b752afae000 100644
--- a/docs/integrations/sources/linkedin-ads.md
+++ b/docs/integrations/sources/linkedin-ads.md
@@ -1,168 +1,145 @@
 # LinkedIn Ads
 
-This page guides you through the process of setting up the LinkedIn Ads source connector.
+This page contains the setup guide and reference information for the LinkedIn Ads source connector.
 
 ## Prerequisites
 
+- A LinkedIn Ads account with permission to access data from accounts you want to sync.
+
+## Setup guide
+
 
-**For Airbyte Cloud:**
 
-* The LinkedIn Ads account with permission to access data from accounts you want to sync.
+We recommend using **Oauth2.0** authentication for Airbyte Cloud, as this significantly simplifies the setup process, and allows you to authenticate your account directly from the Airbyte UI.
+
 
 
 
-**For Airbyte Open Source:**
-
-* The LinkedIn Ads account with permission to access data from accounts you want to sync.
-* Authentication Options:
-   * OAuth2.0:
-      * `Client ID` from your `Developer Application`
-      * `Client Secret` from your `Developer Application`
-      * `Refresh Token` obtained from successful authorization with `Client ID` + `Client Secret`
-   * Access Token:
-      * `Access Token` obtained from successful authorization with `Client ID` + `Client Secret`
-
 
-## Step 1: Set up LinkedIn Ads
+### Set up LinkedIn Ads authentication (Airbyte Open Source)
 
-1. [Login to LinkedIn](https://developer.linkedin.com/) with a developer account.
-2. Click the **Create App** icon on the center of the page or [use here](https://www.linkedin.com/developers/apps). Fill in the required fields:  
-    1. For **App Name**, enter a name.
-    2. For **LinkedIn Page**, enter your company's name or LinkedIn Company Page URL.
-    3. For **Privacy policy URL**, enter the link to your company's privacy policy.
-    4. For **App logo**, upload your company's logo.
-    5. For **Legal Agreement**, select **I have read and agree to these terms**.
-    6. Click **Create App**, on the bottom right of the screen. LinkedIn redirects you to a page showing the details of your application.
+To authenticate the connector in Airbyte Open Source, you will need to create a Linkedin developer application and obtain one of the following credentials:
 
-3. Verify your app. You can verify your app using the following steps:
-    1. To display the settings page, click the **Settings** tab. On the **App Settings** section, click **Verify** under **Company**. A popup window will be displayed. To generate the verification URL, click on **Generate URL**, then copy and send the URL to the Page Admin (this may be you). Click on **I'm done**.
-    If you are the administrator of your Page, simply run the URL in a new tab (if not, an administrator will have to do the next step). Click on **Verify**. Finally, Refresh the tab of app creation, the app should now be associated with your Page.
+1. OAuth2.0 credentials, consisting of:
 
-    2.  To display the Products page, click the **Product** tab. For **Marketing Developer Platform** click on the **Request access**. A popup window will be displayed. Review and Select **I have read and agree to these terms**. Finally, click **Request access**. 
-    
-    3. To authorize your application, click the **Auth** tab. The authentication page is displayed. Copy the **client_id** and **client_secret** (for later steps). For **Oauth 2.0 settings**, Provide a **redirect_uri** (for later steps).
-    
-    4. Click and review the **Analytics** tab. This page shows the daily application and user/member limits with the percent used for each resource endpoint.
+   - Client ID
+   - Client Secret
+   - Refresh Token (expires after 12 months)
 
+2. Access Token (expires after 60 days)
 
-4. (Optional for Airbyte Cloud) Authorize your app. In case your authorization expires:
+You can follow the steps laid out below to create the application and obtain the necessary credentials. For an overview of the LinkedIn authentication process, see the [official documentation](https://learn.microsoft.com/en-us/linkedin/shared/authentication/authentication?context=linkedin%2Fcontext).
 
-   The authorization token `lasts 60-days before expiring`. The connector app will need to be reauthorized when the authorization token expires.
-   Create an Authorization URL with the following steps:
+#### Create a LinkedIn developer application
 
-   1. Replace the highlighted parameters `YOUR_CLIENT_ID` and `YOUR_REDIRECT_URI` in the URL (`https://www.linkedin.com/oauth/v2/authorization?response_type=code&client_id=YOUR_CLIENT_ID&redirect_uri=YOUR_REDIRECT_URI&scope=r_emailaddress,r_liteprofile,r_ads,r_ads_reporting,r_organization_social`) from the scope obtain below.
+1. [Log in to LinkedIn](https://developer.linkedin.com/) with a developer account.
+2. Navigate to the [Apps page](https://www.linkedin.com/developers/apps) and click the **Create App** icon. Fill in the fields below:
+    1. For **App Name**, enter a name.
+    2. For **LinkedIn Page**, enter your company's name or LinkedIn Company Page URL.
+    3. For **Privacy policy URL**, enter the link to your company's privacy policy.
+    4. For **App logo**, upload your company's logo.
+    5. Check **I have read and agree to these terms**, then click **Create App**. LinkedIn redirects you to a page showing the details of your application.
 
-   2. Set up permissions for the following scopes `r_emailaddress,r_liteprofile,r_ads,r_ads_reporting,r_organization_social`. For **OAuth2.0**, copy the `Client ID`, and `Client Secret` from your `Developer Application`. And copy the `Refresh Token` obtained from successful authorization with `Client ID` + `Client Secret`
+3. You can verify your app using the following steps:
+    1. Click the **Settings** tab. On the **App Settings** section, click **Verify** under **Company**. A popup window will be displayed. To generate the verification URL, click on **Generate URL**, then copy and send the URL to the Page Admin (this may be you). Click on **I'm done**. If you are the administrator of your Page, simply run the URL in a new tab (if not, an administrator will have to do the next step). Click on **Verify**.
 
-   3. Enter the modified `URL` in the browser. You will be redirected.
+    2. To display the Products page, click the **Product** tab. For **Marketing Developer Platform**, click **Request access**. A popup window will be displayed. Review and Select **I have read and agree to these terms**. Finally, click **Request access**.
 
-   4. To authorize the app, click **Allow**.
+#### Authorize your app
 
-   5. Copy the `code` parameter listed in the redirect URL in the Browser header URL.
-   
-5. (Optional for Airbyte Cloud) Run the following curl command using `Terminal` or `Command line` with the parameters replaced to return your `access_token`. The `access_token` expires in 2-months.
+1. To authorize your application, click the **Auth** tab. Copy the **Client ID** and **Client Secret** (click the open eye icon to reveal the client secret). In the **Oauth 2.0 settings**, click the pencil icon and provide a redirect URL for your app.
 
-   ```text
-    curl -0 -v -X POST https://www.linkedin.com/oauth/v2/accessToken\
-    -H "Accept: application/json"\
-    -H "application/x-www-form-urlencoded"\
-    -d "grant_type=authorization_code"\
-    -d "code=YOUR_CODE"\
-    -d "client_id=YOUR_CLIENT_ID"\
-    -d "client_secret=YOUR_CLIENT_SECRET"\
-    -d "redirect_uri=YOUR_REDIRECT_URI"
-   ```
+2. Click the **OAuth 2.0 tools** link in the **Understanding authentication and OAuth 2.0** section on the right side of the page.
+3. Click **Create token**.
+4. Select the scopes you want to use for your app. We recommend using the following scopes:
+    - `r_emailaddress`
+    - `r_liteprofile`
+    - `r_ads`
+    - `r_ads_reporting`
+    - `r_organization_social`
+5. Click **Request access token**. You will be redirected to an authorization page. Use your LinkedIn credentials to log in and authorize your app and obtain your **Access Token** and **Refresh Token**.
 
-6. (Optional for Airbyte Cloud) Use the `access_token`. Same as the approach in  `Step 5` to authorize LinkedIn Ads connector.
+:::caution
+These tokens will not be displayed again, so make sure to copy them and store them securely.
+:::
 
+:::tip
+If either of your tokens expire, you can generate new ones by returning to LinkedIn's [Token Generator](https://www.linkedin.com/developers/tools/oauth/token-generator). You can also check on the status of your tokens using the [Token Inspector](https://www.linkedin.com/developers/tools/oauth/token-inspector).
+:::
 
-### Notes:
+
 
-The API user account should be assigned the following permissions for the API endpoints:
-Endpoints such as: `Accounts`, `Account Users`, `Ad Direct Sponsored Contents`, `Campaign Groups`, `Campaigns`, and `Creatives` requires the following permissions set:
+### Set up the LinkedIn Ads connector in Airbyte
 
-- `r_ads`: read ads \(Recommended\), `rw_ads`: read-write ads
-  Endpoints such as: `Ad Analytics by Campaign`, and `Ad Analytics by Creatives` requires the following permissions set:
-- `r_ads_reporting`: read ads reporting
-  The complete set of permissions is as follows:
-- `r_emailaddress,r_liteprofile,r_ads,r_ads_reporting,r_organization_social`
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **LinkedIn Ads** from the list of available sources.
+4. For **Source name**, enter a name for the LinkedIn Ads connector.
+5. To authenticate:
 
-The API user account should be assigned one of the following roles:
+
+#### For Airbyte Cloud
 
-- ACCOUNT_BILLING_ADMIN
-- ACCOUNT_MANAGER
-- CAMPAIGN_MANAGER
-- CREATIVE_MANAGER
-- VIEWER \(Recommended\)
+- Select **OAuth2.0** from the Authentication dropdown, then click **Authenticate your LinkedIn Ads account**. Sign in to your account and click **Allow**.
+
 
-To edit these roles, sign in to Campaign Manager and follow [these instructions](https://www.linkedin.com/help/lms/answer/a496075).
+
+#### For Airbyte Open Source
 
-## Step 2: Set up the source connector in Airbyte
+- Select an option from the Authentication dropdown:
+  1. **OAuth2.0:** Enter your **Client ID**, **Client Secret** and **Refresh Token**. Please note that the refresh token expires after 12 months.
+  2. **Access Token:** Enter your **Access Token**. Please note that the access token expires after 60 days.
+
 
-
-**For Airbyte Cloud:**
+6. For **Start Date**, use the provided datepicker or enter a date programmatically in the format YYYY-MM-DD. Any data before this date will not be replicated.
+7. (Optional) For **Account IDs**, you may optionally provide a space separated list of Account IDs to pull data from. If you do not specify any account IDs, the connector will replicate data from all accounts accessible using your credentials.
+8. (Optional) For **Custom Ad Analytics Reports**, you may optionally provide one or more custom reports to query the LinkedIn Ads API for. By defining custom reports, you can better align the data pulled form LinkedIn Ads with your particular needs. To add a custom report:
+   1. Click on **Add**.
+   2. Enter a **Report Name**. This will be used as the stream name during replication.
+   3. Select a **Pivot Category** from the dropdown. This defines the main dimension by which the report data will be grouped or segmented.
+   4. Select a **Time Granularity** to group the data in your report by time. The options are:
+      - `ALL`: Data is not grouped by time, providing a cumulative view.
+      - `DAILY`: Returns data grouped by day. Useful for closely monitoring short-term changes and effects.
+      - `MONTHLY`: Returns data grouped by month. Ideal for evaluating monthly goals or observing seasonal patterns.
+      - `YEARLY`: Returns data grouped by year. Ideal for high-level analysis of long-term trends and year-over-year comparisons.
+9. Click **Set up source** and wait for the tests to complete.
+
 
-1. [Login to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
-3. On the source setup page, select **LinkedIn Ads** from the Source type dropdown and enter a name for this connector.
-4. Add `Start Date` - the starting point for your data replication.
-5. Add your `Account IDs (Optional)` if required.
-6. Click **Authenticate your account**.
-7. Login and Authorize the LinkedIn Ads account
-8. Click **Set up source**.
+## Supported sync modes
 
-
-**For Airbyte Open Source:**
-
-1. Go to the local Airbyte page.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
-3. On the Set up the source page, enter the name for the connector and select **LinkedIn Ads** from the Source type dropdown.
-4. Add `Start Date` - the starting point for your data replication.
-5. Add your `Account IDs (Optional)` if required.
-6. Choose between Authentication Options:
-    1. For **OAuth2.0:** Copy and paste info (**Client ID**, **Client Secret**) from your **LinkedIn Ads developer application**, and obtain the **Refresh Token** using **Set up LinkedIn Ads**  guide steps and paste it into the corresponding field.
-    2. For **Access Token:** Obtain the **Access Token** using **Set up LinkedIn Ads**  guide steps and paste it into the corresponding field.
-7. Click **Set up source**.
-
+The LinkedIn Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-## Supported Streams and Sync Modes
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
-This Source is capable of syncing the following data as streams:
+## Supported streams
 
 - [Accounts](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-accounts?tabs=http&view=li-lms-2023-05#search-for-accounts)
 - [Account Users](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-account-users?tabs=http&view=li-lms-2023-05#find-ad-account-users-by-accounts)
 - [Campaign Groups](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaign-groups?tabs=http&view=li-lms-2023-05#search-for-campaign-groups)
 - [Campaigns](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-campaigns?tabs=http&view=li-lms-2023-05#search-for-campaigns)
 - [Creatives](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-creatives?tabs=http%2Chttp-update-a-creative&view=li-lms-2023-05#search-for-creatives)
+- [Conversions](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversion-tracking?view=li-lms-2023-05&tabs=curl#find-conversions-by-ad-account)
 - [Ad Analytics by Campaign](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
 - [Ad Analytics by Creative](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Impression Device](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Company Size](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Country](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Job Function](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Job Title](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Industry](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Region](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
+- [Ad Analytics by Member Company](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#ad-analytics)
 
-| Sync Mode                                 | Supported?\(Yes/No\) |
-|:------------------------------------------|:---------------------|
-| Full Refresh Overwrite Sync               | Yes                  |
-| Full Refresh Append Sync                  | Yes                  |
-| Incremental - Append Sync                 | Yes                  |
-| Incremental - Append + Deduplication Sync | Yes                  |
-
-### NOTE:
-
-The `Ad Direct Sponsored Contents` stream includes information about VIDEO ADS, as well as `SINGLE IMAGE ADS` and other directly sponsored ads your account might have.
+:::info
 
 For Analytics Streams such as `Ad Analytics by Campaign` and `Ad Analytics by Creative`, the `pivot` column name is renamed to `_pivot` to handle the data normalization correctly and avoid name conflicts with certain destinations.
 
-### Data type mapping
-
-| Integration Type | Airbyte Type | Notes                       |
-|:-----------------|:-------------|:----------------------------|
-| `number`         | `number`     | float number                |
-| `integer`        | `integer`    | whole number                |
-| `date`           | `string`     | FORMAT YYYY-MM-DD           |
-| `datetime`       | `string`     | FORMAT YYYY-MM-DDThh:mm: ss |
-| `array`          | `array`      |                             |
-| `boolean`        | `boolean`    | True/False                  |
-| `string`         | `string`     |                             |
+:::
 
-### Performance considerations
+## Performance considerations
 
 LinkedIn Ads has Official Rate Limits for API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. These limits reset at midnight UTC every day. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases, you will receive the following error message:
 
@@ -178,10 +155,28 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro
 
 After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose your app > Analytics.
 
+## Data type map
+
+| Integration Type | Airbyte Type | Notes                       |
+|:-----------------|:-------------|:----------------------------|
+| `number`         | `number`     | float number                |
+| `integer`        | `integer`    | whole number                |
+| `date`           | `string`     | FORMAT YYYY-MM-DD           |
+| `datetime`       | `string`     | FORMAT YYYY-MM-DDThh:mm: ss |
+| `array`          | `array`      |                             |
+| `boolean`        | `boolean`    | True/False                  |
+| `string`         | `string`     |                             |
+
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 0.6.1   | 2023-08-23 | [29600](https://github.com/airbytehq/airbyte/pull/29600) | Update field descriptions                                                                                       |
+| 0.6.0   | 2023-08-22 | [29721](https://github.com/airbytehq/airbyte/pull/29721) | Add `Conversions` stream                                                                                        |
+| 0.5.0   | 2023-08-14 | [29175](https://github.com/airbytehq/airbyte/pull/29175) | Add Custom report Constructor                                                                                   |
+| 0.4.0   | 2023-08-08 | [29175](https://github.com/airbytehq/airbyte/pull/29175) | Add analytics streams                                                                                           |
+| 0.3.1   | 2023-08-08 | [29189](https://github.com/airbytehq/airbyte/pull/29189) | Fix empty accounts field                                                                                        |
+| 0.3.0   | 2023-08-07 | [29045](https://github.com/airbytehq/airbyte/pull/29045) | Add new fields to schemas; convert datetime fields to `rfc3339`                                                 |
 | 0.2.1   | 2023-05-30 | [26780](https://github.com/airbytehq/airbyte/pull/26780) | Reduce records limit for Creatives Stream                                                                       |
 | 0.2.0   | 2023-05-23 | [26372](https://github.com/airbytehq/airbyte/pull/26372) | Migrate to LinkedIn API version: May 2023                                                                       |
 | 0.1.16  | 2023-05-24 | [26512](https://github.com/airbytehq/airbyte/pull/26512) | Removed authSpecification from spec.json in favour of advancedAuth                                              |
diff --git a/docs/integrations/sources/mailgun.md b/docs/integrations/sources/mailgun.md
index cdba0820a72f..5afbe7eaf37c 100644
--- a/docs/integrations/sources/mailgun.md
+++ b/docs/integrations/sources/mailgun.md
@@ -1,10 +1,70 @@
 # MailGun
 
-The Airbyte Source for [MailGun](https://www.mailgun.com/)
+This page contains the setup guide and reference information for the [MailGun](https://www.mailgun.com/) source connector.
+
+## Prerequisites
+
+Api key is mandate for this connector to work, It could be seen at Mailgun dashboard at settings, Navigate through API Keys section and click on the eye icon next to Private API key [See reference](https://documentation.mailgun.com/en/latest/api-intro.html#authentication-1).
+Just pass the generated API key for establishing the connection.
+
+## Setup guide
+
+### Step 1: Set up MailGun connection
+
+- Generate an API key (Example: 12345)
+- Params (If specific info is needed)
+- Available params
+  - domain_region: Domain region code. 'EU' or 'US' are possible values. The default is 'US'.
+  - start_date: UTC date and time in the format 2020-10-01 00:00:00. Any data before this date will not be replicated. If omitted, defaults to 3 days ago.
+
+## Step 2: Set up the MailGun connector in Airbyte
+
+### For Airbyte Cloud:
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**.
+3. On the Set up the source page, enter the name for the MailGun connector and select **MailGun** from the Source type dropdown.
+4. Enter your api_key as `private_key`.
+5. Enter the params configuration if needed. Supported params are: domain_region, start_date.
+6. Click **Set up source**.
+
+### For Airbyte OSS:
+
+1. Navigate to the Airbyte Open Source dashboard.
+2. Set the name for your source.
+3. Enter your api_key as `pivate_key`.
+4. Enter the params configuration if needed. Supported params are: domain_region, start_date.
+5. Click **Set up source**.
+
+## Supported sync modes
+
+The MailGun source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+
+| Feature                       | Supported? |
+| :---------------------------- | :--------- |
+| Full Refresh Sync             | Yes        |
+| Incremental Sync              | Yes        |
+| Replicate Incremental Deletes | No         |
+| SSL connection                | Yes        |
+| Namespaces                    | No         |
+
+## Supported Streams
+
+- domains
+- events
+
+## API method example
+
+`GET https://api.mailgun.net/v3/domains`
+
+## Performance considerations
+
+MailGun's [API reference](https://documentation.mailgun.com/en/latest/api_reference.html) has v3 at present and v4 is at development. The connector as default uses v3.
 
 ## Changelog
 
-| Version | Date       | Pull Request                                           | Subject             |
-| :------ | :--------- | :----------------------------------------------------- | :------------------ |
-| 0.1.1   | 2023-02-13 | [22939](https://github.com/airbytehq/airbyte/pull/22939) | Specified date formatting in specification |
-| 0.1.0   | 2021-11-09 | [8056](https://github.com/airbytehq/airbyte/pull/8056) | New Source: Mailgun |
+| Version | Date       | Pull Request                                             | Subject                                     |
+| :------ | :--------- | :------------------------------------------------------  | :------------------------------------------ |
+| 0.2.0   | 2023-08-05 | [29122](https://github.com/airbytehq/airbyte/pull/29122) | Migrate to Low Code                         |
+| 0.1.1   | 2023-02-13 | [22939](https://github.com/airbytehq/airbyte/pull/22939) | Specified date formatting in specification  |
+| 0.1.0   | 2021-11-09 | [8056](https://github.com/airbytehq/airbyte/pull/8056)   | New Source: Mailgun                         |
diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md
index b34e3a95fec0..b179e2515660 100644
--- a/docs/integrations/sources/marketo.md
+++ b/docs/integrations/sources/marketo.md
@@ -4,14 +4,15 @@ This page contains the setup guide and reference information for the Marketo sou
 
 ## Prerequisites
 
-* \(Optional\) Whitelist Airbyte's IP address if needed
-* An API-only Marketo User Role
-* An Airbyte Marketo API-only user
-* A Marketo API Custom Service
-* Marketo Client ID & Client Secret
-* Marketo Base URL
+- \(Optional\) Whitelist Airbyte's IP address if needed
+- An API-only Marketo User Role
+- An Airbyte Marketo API-only user
+- A Marketo API Custom Service
+- Marketo Client ID & Client Secret
+- Marketo Base URL
 
 ## Setup guide
+
 ### Step 1: Set up Marketo
 
 #### Step 1.1: \(Optional\) whitelist Airbyte's IP address
@@ -45,6 +46,7 @@ We're almost there! Armed with your Endpoint & Identity URLs and your Client ID
 ## Step 2: Set up the Marketo connector in Airbyte
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -55,6 +57,7 @@ We're almost there! Armed with your Endpoint & Identity URLs and your Client ID
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Navigate to the Airbyte Open Source dashboard
@@ -68,22 +71,23 @@ We're almost there! Armed with your Endpoint & Identity URLs and your Client ID
 ## Supported sync modes
 
 The Marketo source connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
- - Full Refresh | Overwrite
- - Full Refresh | Append
- - Incremental  | Append
- - Incremental  | Deduped
+
+- Full Refresh | Overwrite
+- Full Refresh | Append
+- Incremental | Append
+- Incremental | Deduped
 
 ## Supported Streams
 
 This connector can be used to sync the following tables from Marketo:
 
-* **activities\_X** where X is an activity type contains information about lead activities of the type X. For example, activities\_send\_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means.
-* **activity\_types.** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns.
-* **campaigns.** Contains info about your Marketo campaigns. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET).
-* **leads.** Contains info about your Marketo leads. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET).
-* **lists.** Contains info about your Marketo static lists. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET).
-* **programs.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET).
-* **segmentations.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET).
+- **activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means.
+- **activity_types.** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns.
+- **campaigns.** Contains info about your Marketo campaigns. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET).
+- **leads.** Contains info about your Marketo leads. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET).
+- **lists.** Contains info about your Marketo static lists. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET).
+- **programs.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET).
+- **segmentations.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET).
 
 ## Performance considerations
 
@@ -96,7 +100,7 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes                                                                           |
-|:-----------------|:-------------|:--------------------------------------------------------------------------------|
+| :--------------- | :----------- | :------------------------------------------------------------------------------ |
 | `array`          | `array`      | primitive arrays are converted into arrays of the types described in this table |
 | `int`, `long`    | `number`     |                                                                                 |
 | `object`         | `object`     |                                                                                 |
@@ -105,23 +109,24 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa
 
 ## Changelog
 
-| Version  | Date       | Pull Request                                             | Subject                                                                                       |
-|:---------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------|
-| `1.1.0`  | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream                                                                     |
-| `1.0.4`  | 2023-04-25 | [25481](https://github.com/airbytehq/airbyte/pull/25481) | Minor fix for bug caused  by `<=` producing additional API call when there is a single date slice                                                                     |
-| `1.0.3`  | 2023-02-13 | [22938](https://github.com/airbytehq/airbyte/pull/22938) | Specified date formatting in specification                                                                     |
-| `1.0.2`  | 2023-02-01 | [22203](https://github.com/airbytehq/airbyte/pull/22203) | Handle Null cursor values                                                                     |
-| `1.0.1`  | 2023-01-31 | [22015](https://github.com/airbytehq/airbyte/pull/22015) | Set `AvailabilityStrategy` for streams explicitly to `None`                                   |
-| `1.0.0`  | 2023-01-25 | [21790](https://github.com/airbytehq/airbyte/pull/21790) | Fix `activities_*` stream schemas                                                             |
-| `0.1.12` | 2023-01-19 | [20973](https://github.com/airbytehq/airbyte/pull/20973) | Fix encoding error (note: this change is not in version 1.0.0, but is in later versions       |
-| `0.1.11` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Do not use temporary files for memory optimization                                            |
-| `0.1.10` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Optimize memory consumption                                                                   |
-| `0.1.9`  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream sate.                                                                   |
-| `0.1.7`  | 2022-08-23 | [15817](https://github.com/airbytehq/airbyte/pull/15817) | Improved unit test coverage                                                                   |
-| `0.1.6`  | 2022-08-21 | [15824](https://github.com/airbytehq/airbyte/pull/15824) | Fix semi incremental streams: do not ignore start date, make one api call instead of multiple |
-| `0.1.5`  | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it                                         |
-| `0.1.4`  | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs                                                       |
-| `0.1.3`  | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578)   | Updated titles and descriptions                                                               |
-| `0.1.2`  | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483)   | Improve field conversion to conform schema                                                    |
-| `0.1.1`  | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000)   | Fix timestamp value format issue                                                              |
-| `0.1.0`  | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863)   | Release Marketo CDK Connector                                                                 |
+| Version  | Date       | Pull Request                                             | Subject                                                                                          |
+| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------- |
+| `1.2.0`  | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2                                                                             |
+| `1.1.0`  | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream                                                                       |
+| `1.0.4`  | 2023-04-25 | [25481](https://github.com/airbytehq/airbyte/pull/25481) | Minor fix for bug caused by `<=` producing additional API call when there is a single date slice |
+| `1.0.3`  | 2023-02-13 | [22938](https://github.com/airbytehq/airbyte/pull/22938) | Specified date formatting in specification                                                       |
+| `1.0.2`  | 2023-02-01 | [22203](https://github.com/airbytehq/airbyte/pull/22203) | Handle Null cursor values                                                                        |
+| `1.0.1`  | 2023-01-31 | [22015](https://github.com/airbytehq/airbyte/pull/22015) | Set `AvailabilityStrategy` for streams explicitly to `None`                                      |
+| `1.0.0`  | 2023-01-25 | [21790](https://github.com/airbytehq/airbyte/pull/21790) | Fix `activities_*` stream schemas                                                                |
+| `0.1.12` | 2023-01-19 | [20973](https://github.com/airbytehq/airbyte/pull/20973) | Fix encoding error (note: this change is not in version 1.0.0, but is in later versions          |
+| `0.1.11` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Do not use temporary files for memory optimization                                               |
+| `0.1.10` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Optimize memory consumption                                                                      |
+| `0.1.9`  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream sate.                                                                      |
+| `0.1.7`  | 2022-08-23 | [15817](https://github.com/airbytehq/airbyte/pull/15817) | Improved unit test coverage                                                                      |
+| `0.1.6`  | 2022-08-21 | [15824](https://github.com/airbytehq/airbyte/pull/15824) | Fix semi incremental streams: do not ignore start date, make one api call instead of multiple    |
+| `0.1.5`  | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it                                            |
+| `0.1.4`  | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs                                                          |
+| `0.1.3`  | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578)   | Updated titles and descriptions                                                                  |
+| `0.1.2`  | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483)   | Improve field conversion to conform schema                                                       |
+| `0.1.1`  | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000)   | Fix timestamp value format issue                                                                 |
+| `0.1.0`  | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863)   | Release Marketo CDK Connector                                                                    |
diff --git a/docs/integrations/sources/metabase.md b/docs/integrations/sources/metabase.md
index 2272fcdd2ca8..40e05a5f6223 100644
--- a/docs/integrations/sources/metabase.md
+++ b/docs/integrations/sources/metabase.md
@@ -40,7 +40,6 @@ The Metabase source connector supports the following [sync modes](https://docs.a
 * [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 
 ## Supported Streams
-* [Activity](https://www.metabase.com/docs/latest/api/activity.html#get-apiactivity)
 * [Card](https://www.metabase.com/docs/latest/api/card.html#get-apicard)
 * [Collections](https://www.metabase.com/docs/latest/api/collection.html#get-apicollection)
 * [Dashboard](https://www.metabase.com/docs/latest/api/dashboard.html#get-apidashboard)
@@ -71,6 +70,8 @@ The Metabase source connector supports the following [sync modes](https://docs.a
 
 | Version | Date       | Pull Request                                             | Subject                    |
 |:--------|:-----------|:---------------------------------------------------------|:---------------------------|
+| 1.0.1   | 2023-07-20 | [28470](https://github.com/airbytehq/airbyte/pull/27777) | Update CDK to 0.47.0       |
+| 1.0.0   | 2023-06-27 | [27777](https://github.com/airbytehq/airbyte/pull/27777) | Remove Activity Stream     |
 | 0.3.1   | 2022-12-15 | [20535](https://github.com/airbytehq/airbyte/pull/20535) | Run on CDK 0.15.0          |
 | 0.3.0   | 2022-12-13 | [19236](https://github.com/airbytehq/airbyte/pull/19236) | Migrate to YAML.           |
 | 0.2.0   | 2022-10-28 | [18607](https://github.com/airbytehq/airbyte/pull/18607) | Disallow using `http` URLs |
diff --git a/docs/integrations/sources/microsoft-dataverse.md b/docs/integrations/sources/microsoft-dataverse.md
index a64a55e1c086..4e3138cff796 100644
--- a/docs/integrations/sources/microsoft-dataverse.md
+++ b/docs/integrations/sources/microsoft-dataverse.md
@@ -61,5 +61,6 @@ https://blog.magnetismsolutions.com/blog/paulnieuwelaar/2021/9/21/setting-up-an-
 
 | Version | Date       | Pull Request                                             | Subject                                         |
 | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- |
+| 0.1.2   | 2023-08-24 | [29732](https://github.com/airbytehq/airbyte/pull/29732) | 🐛 Source Microsoft Dataverse: Adjust source_default_cursor when modifiedon not exists |
 | 0.1.1   | 2023-03-16 | [22805](https://github.com/airbytehq/airbyte/pull/22805) | Fixed deduped cursor field value update |
 | 0.1.0   | 2022-11-14 | [18646](https://github.com/airbytehq/airbyte/pull/18646) | 🎉 New Source: Microsoft Dataverse [python cdk] |
diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md
index 1efa5cfa5151..b0f31a65d224 100644
--- a/docs/integrations/sources/mixpanel.md
+++ b/docs/integrations/sources/mixpanel.md
@@ -17,30 +17,31 @@ To set up the Mixpanel source connector, you'll need a Mixpanel [Service Account
 7. For **Attribution Window**, enter the number of days for the length of the attribution window.
 8. For **Project Timezone**, enter the [timezone](https://help.mixpanel.com/hc/en-us/articles/115004547203-Manage-Timezones-for-Projects-in-Mixpanel) for your Mixpanel project.
 9. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If left blank, the connector will replicate data from up to one year ago by default.
-10. For **End Date**, enter the date in YYYY-MM-DD format. 
+10. For **End Date**, enter the date in YYYY-MM-DD format.
 11. For **Region**, enter the [region](https://help.mixpanel.com/hc/en-us/articles/360039135652-Data-Residency-in-EU) for your Mixpanel project.
 12. For **Date slicing window**, enter the number of days to slice through data. If you encounter RAM usage issues due to a huge amount of data in each window, try using a lower value for this parameter.
 13. Click **Set up source**.
 
 ## Supported sync modes
+
 The Mixpanel source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 Note: Incremental sync returns duplicated \(old records\) for the state date due to API filter limitation, which is granular to the whole day only.
 
 ### Supported Streams
 
-* [Export](https://developer.mixpanel.com/reference/raw-event-export) \(Incremental\)
-* [Engage](https://developer.mixpanel.com/reference/engage-query) \(Incremental\)
-* [Funnels](https://developer.mixpanel.com/reference/funnels-query) \(Incremental\)
-* [Revenue](https://developer.mixpanel.com/reference/engage-query) \(Incremental\)
-* [Annotations](https://developer.mixpanel.com/reference/overview-1) \(Full table\)
-* [Cohorts](https://developer.mixpanel.com/reference/cohorts-list) \(Incremental\)
-* [Cohort Members](https://developer.mixpanel.com/reference/engage-query) \(Incremental\)
+- [Export](https://developer.mixpanel.com/reference/raw-event-export) \(Incremental\)
+- [Engage](https://developer.mixpanel.com/reference/engage-query) \(Incremental\)
+- [Funnels](https://developer.mixpanel.com/reference/funnels-query) \(Incremental\)
+- [Revenue](https://developer.mixpanel.com/reference/engage-query) \(Incremental\)
+- [Annotations](https://developer.mixpanel.com/reference/overview-1) \(Full table\)
+- [Cohorts](https://developer.mixpanel.com/reference/cohorts-list) \(Incremental\)
+- [Cohort Members](https://developer.mixpanel.com/reference/engage-query) \(Incremental\)
 
 ## Performance considerations
 
@@ -49,7 +50,11 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits
 ## CHANGELOG
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                     |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------- |
+| 0.1.37  | 2022-07-20 | [27932](https://github.com/airbytehq/airbyte/pull/27932) | Fix spec: change start/end date format to `date`                                                            |
+| 0.1.36  | 2022-06-27 | [27752](https://github.com/airbytehq/airbyte/pull/27752) | Partially revert version 0.1.32; Use exponential backoff;                                                   |
+| 0.1.35  | 2022-06-12 | [27252](https://github.com/airbytehq/airbyte/pull/27252) | Add should_retry False for 402 error                                                                        |
+| 0.1.34  | 2022-05-15 | [21837](https://github.com/airbytehq/airbyte/pull/21837) | Add "insert_id" field to "export" stream schema                                                             |
 | 0.1.33  | 2023-04-25 | [25543](https://github.com/airbytehq/airbyte/pull/25543) | Set should_retry for 104 error in stream export                                                             |
 | 0.1.32  | 2023-04-11 | [25056](https://github.com/airbytehq/airbyte/pull/25056) | Set HttpAvailabilityStrategy, add exponential backoff, streams export and annotations add undeclared fields |
 | 0.1.31  | 2023-02-13 | [22936](https://github.com/airbytehq/airbyte/pull/22936) | Specified date formatting in specification                                                                  |
@@ -83,5 +88,3 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits
 | 0.1.2   | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate                                        |
 | 0.1.1   | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region                                                                       |
 | 0.1.0   | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector                                                                       |
-
-
diff --git a/docs/integrations/sources/monday.md b/docs/integrations/sources/monday.md
index a69967de5892..1b9694f43172 100644
--- a/docs/integrations/sources/monday.md
+++ b/docs/integrations/sources/monday.md
@@ -31,7 +31,7 @@ The Monday supports full refresh syncs
 | Feature           | Supported? |
 |:------------------|:-----------|
 | Full Refresh Sync | Yes        |
-| Incremental Sync  | No         |
+| Incremental Sync  | Yes        |
 | SSL connection    | No         |
 | Namespaces        | No         |
 
@@ -39,14 +39,28 @@ The Monday supports full refresh syncs
 
 Several output streams are available from this source:
 
+* [Activity logs](https://developer.monday.com/api-reference/docs/activity-logs)
 * [Items](https://developer.monday.com/api-reference/docs/items-queries)
 * [Boards](https://developer.monday.com/api-reference/docs/groups-queries#groups-queries)
 * [Teams](https://developer.monday.com/api-reference/docs/teams-queries)
 * [Updates](https://developer.monday.com/api-reference/docs/updates-queries)
 * [Users](https://developer.monday.com/api-reference/docs/users-queries-1)
+* [Tags](https://developer.monday.com/api-reference/docs/tags-queries)
+* [Workspaces](https://developer.monday.com/api-reference/docs/workspaces)
 
+Important Notes:
+* `Columns` are available from the `Boards` stream. By syncing the `Boards` stream you will get the `Columns` for each `Board` synced in the database
+The typical name of the table depends on the `destination` you use like `boards.columns`, for instance.
+
+* `Column Values` are  available from the `Items` stream. By syncing  the `Items` stream you will get the `Column Values` for each `Item` (row) of the board.
+The typical name of the table depends on the `destination` you use like `items.column_values`, for instance.
 If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose)
 
+* Incremental sync for `Items` and `Boards` streams is done using the `Activity logs` stream.
+Ids of boards and items are extracted from activity logs events and used to selectively sync boards and items.
+Some data may be lost if the time between incremental syncs is longer than the activity logs retention time for your plan.
+Check your Monday plan at https://monday.com/pricing.
+
 
 ## Performance considerations
 
@@ -55,16 +69,21 @@ The Monday connector should not run into Monday API limitations under normal usa
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                         |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------|
-| 0.2.5   | 2023-05-22 | [225881](https://github.com/airbytehq/airbyte/pull/25881) | Fix pagination for the items stream |  
-| 0.2.4   | 2023-04-26 | [25277](https://github.com/airbytehq/airbyte/pull/25277) | Increase row limit to 100 |
-| 0.2.3   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version         |
-| 0.2.2   | 2023-01-04 | [20996](https://github.com/airbytehq/airbyte/pull/20996) | Fix json schema loader                          |
-| 0.2.1   | 2022-12-15 | [20533](https://github.com/airbytehq/airbyte/pull/20533) | Bump CDK version                                |
-| 0.2.0   | 2022-12-13 | [19586](https://github.com/airbytehq/airbyte/pull/19586) | Migrate to low-code                             |
-| 0.1.4   | 2022-06-06 | [14443](https://github.com/airbytehq/airbyte/pull/14443) | Increase retry_factor for Items stream          |
-| 0.1.3   | 2021-12-23 | [8172](https://github.com/airbytehq/airbyte/pull/8172)   | Add oauth2.0 support                            |
-| 0.1.2   | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Update titles and descriptions                  |
-| 0.1.1   | 2021-11-18 | [8016](https://github.com/airbytehq/airbyte/pull/8016)   | 🐛 Source Monday: fix pagination and schema bug |
-| 0.1.0   | 2021-11-07 | [7168](https://github.com/airbytehq/airbyte/pull/7168)   | 🎉 New Source: Monday                           |
+| Version | Date       | Pull Request                                              | Subject                                                                 |
+|:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------|
+| 1.1.2   | 2023-08-23 | [29777](https://github.com/airbytehq/airbyte/pull/29777)  | Add retry for `502` error                                               |
+| 1.1.1   | 2023-08-15 | [29429](https://github.com/airbytehq/airbyte/pull/29429)  | Ignore `null` records in response                                       |
+| 1.1.0   | 2023-07-05 | [27944](https://github.com/airbytehq/airbyte/pull/27944)  | Add incremental sync for Items and Boards streams                       |
+| 1.0.0   | 2023-06-20 | [27410](https://github.com/airbytehq/airbyte/pull/27410)  | Add new streams: Tags, Workspaces. Add new fields for existing streams. |
+| 0.2.6   | 2023-06-12 | [27244](https://github.com/airbytehq/airbyte/pull/27244)  | Added http error handling for `403` and `500` HTTP errors               |  
+| 0.2.5   | 2023-05-22 | [225881](https://github.com/airbytehq/airbyte/pull/25881) | Fix pagination for the items stream                                     |  
+| 0.2.4   | 2023-04-26 | [25277](https://github.com/airbytehq/airbyte/pull/25277)  | Increase row limit to 100                                               |
+| 0.2.3   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231)  | Publish using low-code CDK Beta version                                 |
+| 0.2.2   | 2023-01-04 | [20996](https://github.com/airbytehq/airbyte/pull/20996)  | Fix json schema loader                                                  |
+| 0.2.1   | 2022-12-15 | [20533](https://github.com/airbytehq/airbyte/pull/20533)  | Bump CDK version                                                        |
+| 0.2.0   | 2022-12-13 | [19586](https://github.com/airbytehq/airbyte/pull/19586)  | Migrate to low-code                                                     |
+| 0.1.4   | 2022-06-06 | [14443](https://github.com/airbytehq/airbyte/pull/14443)  | Increase retry_factor for Items stream                                  |
+| 0.1.3   | 2021-12-23 | [8172](https://github.com/airbytehq/airbyte/pull/8172)    | Add oauth2.0 support                                                    |
+| 0.1.2   | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8429)    | Update titles and descriptions                                          |
+| 0.1.1   | 2021-11-18 | [8016](https://github.com/airbytehq/airbyte/pull/8016)    | 🐛 Source Monday: fix pagination and schema bug                         |
+| 0.1.0   | 2021-11-07 | [7168](https://github.com/airbytehq/airbyte/pull/7168)    | 🎉 New Source: Monday                                                   |
diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md
index ddea115267d3..9fca57ea3805 100644
--- a/docs/integrations/sources/mongodb-v2.md
+++ b/docs/integrations/sources/mongodb-v2.md
@@ -16,12 +16,12 @@ For each property found, connector determines its type, if all the selected valu
 
 ## Features
 
-| Feature | Supported |
-| :--- | :--- |
-| Full Refresh Sync | Yes |
-| Incremental - Append Sync | Yes |
-| Replicate Incremental Deletes | No |
-| Namespaces | No |
+| Feature                       | Supported |
+| :---------------------------- | :-------- |
+| Full Refresh Sync             | Yes       |
+| Incremental - Append Sync     | Yes       |
+| Replicate Incremental Deletes | No        |
+| Namespaces                    | No        |
 
 ### Full Refresh sync
 
@@ -35,8 +35,8 @@ Cursor should **never** be blank. In case cursor is blank - the incremental sync
 
 Only `datetime` and `number` cursor types are supported. Cursor type is determined based on the cursor field name:
 
-* `datetime` - if cursor field name contains a string from: `time`, `date`, `_at`, `timestamp`, `ts`
-* `number` - otherwise
+- `datetime` - if cursor field name contains a string from: `time`, `date`, `_at`, `timestamp`, `ts`
+- `number` - otherwise
 
 ## Getting started
 
@@ -82,42 +82,48 @@ It is recommended to use encrypted connection. Connection with TLS/SSL security
 
 ### Сonfiguration Parameters
 
-* Database: database name
-* Authentication Source: specifies the database that the supplied credentials should be validated against. Defaults to `admin`.
-* User: username to use when connecting
-* Password: used to authenticate the user
-* **Standalone MongoDb instance**
-  * Host: URL of the database
-  * Port: Port to use for connecting to the database
-  * TLS: indicates whether to create encrypted connection
-* **Replica Set**
-  * Server addresses: the members of a replica set
-  * Replica Set: A replica set name
-* **MongoDb Atlas Cluster**
-  * Cluster URL: URL of a cluster to connect to
+- Database: database name
+- Authentication Source: specifies the database that the supplied credentials should be validated against. Defaults to `admin`.
+- User: username to use when connecting
+- Password: used to authenticate the user
+- **Standalone MongoDb instance**
+  - Host: URL of the database
+  - Port: Port to use for connecting to the database
+  - TLS: indicates whether to create encrypted connection
+- **Replica Set**
+  - Server addresses: the members of a replica set
+  - Replica Set: A replica set name
+- **MongoDb Atlas Cluster**
+  - Cluster URL: URL of a cluster to connect to
 
 For more information regarding configuration parameters, please see [MongoDb Documentation](https://docs.mongodb.com/drivers/java/sync/v4.3/fundamentals/connection/).
 
 ## Changelog
 
-| Version | Date       | Pull Request | Subject                                                                                                   |
-|:--------|:-----------| :--- |:----------------------------------------------------------------------------------------------------------|
-| 0.1.19  | 2022-10-07 | [17614](https://github.com/airbytehq/airbyte/pull/17614) | Increased discover performance                             |
-| 0.1.18  | 2022-10-05 | [17590](https://github.com/airbytehq/airbyte/pull/17590) | Add ability to enforce SSL in MongoDB connector and check logic _                                         |
-| 0.1.17  | 2022-09-08 | [16401](https://github.com/airbytehq/airbyte/pull/16401) | Fixed bug with empty strings in fields with __aibyte_transform_                                           |
+| Version | Date       | Pull Request                                             | Subject                                                                                                   |
+| :------ | :--------- |:---------------------------------------------------------| :-------------------------------------------------------------------------------------------------------- |
+| 0.2.5   | 2023-07-27 | [28815](https://github.com/airbytehq/airbyte/pull/28815) | Revert back to version 0.2.0                                                                              |
+| 0.2.4   | 2023-07-26 | [28760](https://github.com/airbytehq/airbyte/pull/28760) | Fix bug preventing some syncs from succeeding when collecting stats                                       |
+| 0.2.3   | 2023-07-26 | [28733](https://github.com/airbytehq/airbyte/pull/28733) | Fix bug preventing syncs from discovering field types                                                     |
+| 0.2.2   | 2023-07-25 | [28692](https://github.com/airbytehq/airbyte/pull/28692) | Fix bug preventing statistics retrieval from views                                                        |
+| 0.2.1   | 2023-07-21 | [28527](https://github.com/airbytehq/airbyte/pull/28527) | Log server information                                                                                    |
+| 0.2.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2                                                                                      |
+| 0.1.19  | 2022-10-07 | [17614](https://github.com/airbytehq/airbyte/pull/17614) | Increased discover performance                                                                            |
+| 0.1.18  | 2022-10-05 | [17590](https://github.com/airbytehq/airbyte/pull/17590) | Add ability to enforce SSL in MongoDB connector and check logic                                           |
+| 0.1.17  | 2022-09-08 | [16401](https://github.com/airbytehq/airbyte/pull/16401) | Fixed bug with empty strings in fields with _aibyte_transform_                                            |
 | 0.1.16  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
 | 0.1.15  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors                                                    |
 | 0.1.14  | 2022-05-05 | [12428](https://github.com/airbytehq/airbyte/pull/12428) | JsonSchema: Add properties to fields with type 'object'                                                   |
 | 0.1.13  | 2022-02-21 | [10276](https://github.com/airbytehq/airbyte/pull/10276) | Create a custom codec registry to handle DBRef MongoDB objects                                            |
 | 0.1.12  | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                |
-| 0.1.11  | 2022-01-10 | [9238](https://github.com/airbytehq/airbyte/pull/9238) | Return only those collections for which the user has privileges                                           |
-| 0.1.10  | 2021-12-30 | [9202](https://github.com/airbytehq/airbyte/pull/9202) | Update connector fields title/description                                                                 |
-| 0.1.9   | 2021-12-07 | [8491](https://github.com/airbytehq/airbyte/pull/8491) | Configure 10000 limit doc reading during Discovery step                                                   |
-| 0.1.8   | 2021-11-29 | [8306](https://github.com/airbytehq/airbyte/pull/8306) | Added milliseconds for date format for cursor                                                             |
-| 0.1.7   | 2021-11-22 | [8161](https://github.com/airbytehq/airbyte/pull/8161) | Updated Performance and updated cursor for timestamp type                                                 |
-| 0.1.5   | 2021-11-17 | [8046](https://github.com/airbytehq/airbyte/pull/8046) | Added milliseconds to convert timestamp to datetime format                                                |
-| 0.1.4   | 2021-11-15 | [7982](https://github.com/airbytehq/airbyte/pull/7982) | Updated Performance                                                                                       |
-| 0.1.3   | 2021-10-19 | [7160](https://github.com/airbytehq/airbyte/pull/7160) | Fixed nested document parsing                                                                             |
-| 0.1.2   | 2021-10-07 | [6860](https://github.com/airbytehq/airbyte/pull/6860) | Added filter to avoid MongoDb system collections                                                          |
-| 0.1.1   | 2021-09-21 | [6364](https://github.com/airbytehq/airbyte/pull/6364) | Source MongoDb: added support via TLS/SSL                                                                 |
-| 0.1.0   | 2021-08-30 | [5530](https://github.com/airbytehq/airbyte/pull/5530) | New source: MongoDb ported to java                                                                        |
+| 0.1.11  | 2022-01-10 | [9238](https://github.com/airbytehq/airbyte/pull/9238)   | Return only those collections for which the user has privileges                                           |
+| 0.1.10  | 2021-12-30 | [9202](https://github.com/airbytehq/airbyte/pull/9202)   | Update connector fields title/description                                                                 |
+| 0.1.9   | 2021-12-07 | [8491](https://github.com/airbytehq/airbyte/pull/8491)   | Configure 10000 limit doc reading during Discovery step                                                   |
+| 0.1.8   | 2021-11-29 | [8306](https://github.com/airbytehq/airbyte/pull/8306)   | Added milliseconds for date format for cursor                                                             |
+| 0.1.7   | 2021-11-22 | [8161](https://github.com/airbytehq/airbyte/pull/8161)   | Updated Performance and updated cursor for timestamp type                                                 |
+| 0.1.5   | 2021-11-17 | [8046](https://github.com/airbytehq/airbyte/pull/8046)   | Added milliseconds to convert timestamp to datetime format                                                |
+| 0.1.4   | 2021-11-15 | [7982](https://github.com/airbytehq/airbyte/pull/7982)   | Updated Performance                                                                                       |
+| 0.1.3   | 2021-10-19 | [7160](https://github.com/airbytehq/airbyte/pull/7160)   | Fixed nested document parsing                                                                             |
+| 0.1.2   | 2021-10-07 | [6860](https://github.com/airbytehq/airbyte/pull/6860)   | Added filter to avoid MongoDb system collections                                                          |
+| 0.1.1   | 2021-09-21 | [6364](https://github.com/airbytehq/airbyte/pull/6364)   | Source MongoDb: added support via TLS/SSL                                                                 |
+| 0.1.0   | 2021-08-30 | [5530](https://github.com/airbytehq/airbyte/pull/5530)   | New source: MongoDb ported to java                                                                        |
diff --git a/docs/integrations/sources/mssql-migrations.md b/docs/integrations/sources/mssql-migrations.md
new file mode 100644
index 000000000000..c2271160bab3
--- /dev/null
+++ b/docs/integrations/sources/mssql-migrations.md
@@ -0,0 +1,4 @@
+# Microsoft SQL Server (MSSQL) Migration Guide
+
+## Upgrading to 2.0.0
+CDC syncs now has default cursor field called `_ab_cdc_cursor`. You will need to force normalization to rebuild your destination tables by manually dropping the SCD tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. 
\ No newline at end of file
diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md
index a092b87ca3b1..28d3fe16e5c0 100644
--- a/docs/integrations/sources/mssql.md
+++ b/docs/integrations/sources/mssql.md
@@ -3,7 +3,7 @@
 ## Features
 
 | Feature                       | Supported | Notes              |
-|:------------------------------|:----------|:-------------------|
+| :---------------------------- | :-------- | :----------------- |
 | Full Refresh Sync             | Yes       |                    |
 | Incremental Sync - Append     | Yes       |                    |
 | Replicate Incremental Deletes | Yes       |                    |
@@ -21,7 +21,6 @@ You may run into an issue where the connector provides wrong values for some dat
 Note: Currently hierarchyid and sql_variant are not processed in CDC migration type (not supported by debezium). For more details please check
 [this ticket](https://github.com/airbytehq/airbyte/issues/14411)
 
-
 ## Getting Started \(Airbyte Cloud\)
 
 On Airbyte Cloud, only TLS connections to your MSSQL instance are supported in source configuration. Other than that, you can proceed with the open-source instructions below.
@@ -50,40 +49,40 @@ _Coming soon: suggestions on how to create this user._
 
 We use [SQL Server's change data capture feature](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-2017) to capture row-level `INSERT`, `UPDATE` and `DELETE` operations that occur on cdc-enabled tables.
 
-Some extra setup requiring at least _db\_owner_ permissions on the database\(s\) you intend to sync from will be required \(detailed [below](mssql.md#setting-up-cdc-for-mssql)\).
+Some extra setup requiring at least _db_owner_ permissions on the database\(s\) you intend to sync from will be required \(detailed [below](mssql.md#setting-up-cdc-for-mssql)\).
 
 Please read the [CDC docs](../../understanding-airbyte/cdc.md) for an overview of how Airbyte approaches CDC.
 
 ### Should I use CDC for MSSQL?
 
-* If you need a record of deletions and can accept the limitations posted below, CDC is the way to go!
-* If your data set is small and/or you just want a snapshot of your table in the destination, consider using Full Refresh replication for your table instead of CDC.
-* If the limitations below prevent you from using CDC and your goal is to maintain a snapshot of your table in the destination, consider using non-CDC incremental and occasionally reset the data and re-sync.
-* If your table has a primary key but doesn't have a reasonable cursor field for incremental syncing \(i.e. `updated_at`\), CDC allows you to sync your table incrementally.
+- If you need a record of deletions and can accept the limitations posted below, CDC is the way to go!
+- If your data set is small and/or you just want a snapshot of your table in the destination, consider using Full Refresh replication for your table instead of CDC.
+- If the limitations below prevent you from using CDC and your goal is to maintain a snapshot of your table in the destination, consider using non-CDC incremental and occasionally reset the data and re-sync.
+- If your table has a primary key but doesn't have a reasonable cursor field for incremental syncing \(i.e. `updated_at`\), CDC allows you to sync your table incrementally.
 
 ### CDC Config
 
 | Parameter                |                     Type                     |      Default       | Description                                                                                                                                                                                                                                                                                                                                                                                                                                               |
-|:-------------------------|:--------------------------------------------:|:------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| :----------------------- | :------------------------------------------: | :----------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | Data to Sync             | Enum: `Existing and New`, `New Changes Only` | `Existing and New` | What data should be synced under the CDC. `Existing and New` will read existing data as a snapshot, and sync new changes through CDC. `New Changes Only` will skip the initial snapshot, and only sync new changes through CDC. See documentation [here](https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#sqlserver-property-snapshot-mode) for details. Under the hood, this parameter sets the `snapshot.mode` in Debezium. |
 | Snapshot Isolation Level |      Enum: `Snapshot`, `Read Committed`      |     `Snapshot`     | Mode to control which transaction isolation level is used and how long the connector locks tables that are designated for capture. If you don't know which one to choose, just use the default one. See documentation [here](https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#sqlserver-property-snapshot-isolation-mode) for details. Under the hood, this parameter sets the `snapshot.isolation.mode` in Debezium.         |
 
 #### CDC Limitations
 
-* Make sure to read our [CDC docs](../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication.
-* There are some critical issues regarding certain datatypes. Please find detailed info in [this Github issue](https://github.com/airbytehq/airbyte/issues/4542).
-* CDC is only available for SQL Server 2016 Service Pack 1 \(SP1\) and later.
-* _db\_owner_ \(or higher\) permissions are required to perform the [neccessary setup](mssql.md#setting-up-cdc-for-mssql) for CDC.
-* If you set `Initial Snapshot Isolation Level` to `Snapshot`, you must enable [snapshot isolation mode](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/snapshot-isolation-in-sql-server) on the database\(s\) you want to sync. This is used for retrieving an initial snapshot without locking tables.
-* For SQL Server Always On read-only replica, only `Snapshot` initial snapshot isolation level is supported.
-* On Linux, CDC is not supported on versions earlier than SQL Server 2017 CU18 \(SQL Server 2019 is supported\).
-* Change data capture cannot be enabled on tables with a clustered columnstore index. \(It can be enabled on tables with a _non-clustered_ columnstore index\).
-* The SQL Server CDC feature processes changes that occur in user-created tables only. You cannot enable CDC on the SQL Server master database.
-* Using variables with partition switching on databases or tables with change data capture \(CDC\) is not supported for the `ALTER TABLE` ... `SWITCH TO` ... `PARTITION` ... statement
-* Our implementation has not been tested with managed instances, such as Azure SQL Database \(we welcome any feedback from users who try this!\)
-  * If you do want to try this, CDC can only be enabled on Azure SQL databases tiers above Standard 3 \(S3+\). Basic, S0, S1 and S2 tiers are not supported for CDC.
-* Our CDC implementation uses at least once delivery for all change records.
-* Read more on CDC limitations in the [Microsoft docs](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-2017#limitations).
+- Make sure to read our [CDC docs](../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication.
+- There are some critical issues regarding certain datatypes. Please find detailed info in [this Github issue](https://github.com/airbytehq/airbyte/issues/4542).
+- CDC is only available for SQL Server 2016 Service Pack 1 \(SP1\) and later.
+- _db_owner_ \(or higher\) permissions are required to perform the [neccessary setup](mssql.md#setting-up-cdc-for-mssql) for CDC.
+- If you set `Initial Snapshot Isolation Level` to `Snapshot`, you must enable [snapshot isolation mode](https://docs.microsoft.com/en-us/dotnet/framework/data/adonet/sql/snapshot-isolation-in-sql-server) on the database\(s\) you want to sync. This is used for retrieving an initial snapshot without locking tables.
+- For SQL Server Always On read-only replica, only `Snapshot` initial snapshot isolation level is supported.
+- On Linux, CDC is not supported on versions earlier than SQL Server 2017 CU18 \(SQL Server 2019 is supported\).
+- Change data capture cannot be enabled on tables with a clustered columnstore index. \(It can be enabled on tables with a _non-clustered_ columnstore index\).
+- The SQL Server CDC feature processes changes that occur in user-created tables only. You cannot enable CDC on the SQL Server master database.
+- Using variables with partition switching on databases or tables with change data capture \(CDC\) is not supported for the `ALTER TABLE` ... `SWITCH TO` ... `PARTITION` ... statement
+- Our implementation has not been tested with managed instances, such as Azure SQL Database \(we welcome any feedback from users who try this!\)
+  - If you do want to try this, CDC can only be enabled on Azure SQL databases tiers above Standard 3 \(S3+\). Basic, S0, S1 and S2 tiers are not supported for CDC.
+- Our CDC implementation uses at least once delivery for all change records.
+- Read more on CDC limitations in the [Microsoft docs](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-2017#limitations).
 
 ### Setting up CDC for MSSQL
 
@@ -91,7 +90,7 @@ Please read the [CDC docs](../../understanding-airbyte/cdc.md) for an overview o
 
 MS SQL Server provides some built-in stored procedures to enable CDC.
 
-* To enable CDC, a SQL Server administrator with the necessary privileges \(_db\_owner_ or _sysadmin_\) must first run a query to enable CDC at the database level.
+- To enable CDC, a SQL Server administrator with the necessary privileges \(_db_owner_ or _sysadmin_\) must first run a query to enable CDC at the database level.
 
   ```text
   USE {database name}
@@ -100,7 +99,7 @@ MS SQL Server provides some built-in stored procedures to enable CDC.
   GO
   ```
 
-* The administrator must then enable CDC for each table that you want to capture. Here's an example:
+- The administrator must then enable CDC for each table that you want to capture. Here's an example:
 
   ```text
   USE {database name}
@@ -115,18 +114,18 @@ MS SQL Server provides some built-in stored procedures to enable CDC.
   GO
   ```
 
-  * \[1\] Specifies a role which will gain `SELECT` permission on the captured columns of the source table. We suggest putting a value here so you can use this role in the next step but you can also set the value of @role\_name to `NULL` to allow only _sysadmin_ and _db\_owner_ to have access. Be sure that the credentials used to connect to the source in Airbyte align with this role so that Airbyte can access the cdc tables.
-  * \[2\] Specifies the filegroup where SQL Server places the change table. We recommend creating a separate filegroup for CDC but you can leave this parameter out to use the default filegroup.
-  * \[3\] If 0, only the support functions to query for all changes are generated. If 1, the functions that are needed to query for net changes are also generated. If supports\_net\_changes is set to 1, index\_name must be specified, or the source table must have a defined primary key.
+  - \[1\] Specifies a role which will gain `SELECT` permission on the captured columns of the source table. We suggest putting a value here so you can use this role in the next step but you can also set the value of @role*name to `NULL` to allow only \_sysadmin* and _db_owner_ to have access. Be sure that the credentials used to connect to the source in Airbyte align with this role so that Airbyte can access the cdc tables.
+  - \[2\] Specifies the filegroup where SQL Server places the change table. We recommend creating a separate filegroup for CDC but you can leave this parameter out to use the default filegroup.
+  - \[3\] If 0, only the support functions to query for all changes are generated. If 1, the functions that are needed to query for net changes are also generated. If supports_net_changes is set to 1, index_name must be specified, or the source table must have a defined primary key.
 
-* \(For more details on parameters, see the [Microsoft doc page](https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sys-sp-cdc-enable-table-transact-sql?view=sql-server-ver15) for this stored procedure\).
-* If you have many tables to enable CDC on and would like to avoid having to run this query one-by-one for every table, [this script](http://www.techbrothersit.com/2013/06/change-data-capture-cdc-sql-server_69.html) might help!
+- \(For more details on parameters, see the [Microsoft doc page](https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sys-sp-cdc-enable-table-transact-sql?view=sql-server-ver15) for this stored procedure\).
+- If you have many tables to enable CDC on and would like to avoid having to run this query one-by-one for every table, [this script](http://www.techbrothersit.com/2013/06/change-data-capture-cdc-sql-server_69.html) might help!
 
 For further detail, see the [Microsoft docs on enabling and disabling CDC](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/enable-and-disable-change-data-capture-sql-server?view=sql-server-ver15).
 
 #### 2. Enable snapshot isolation
 
-* When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. To avoid acquiring table locks, Airbyte uses _snapshot isolation_, allowing simultaneous writes by other database clients. This must be enabled on the database like so:
+- When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. To avoid acquiring table locks, Airbyte uses _snapshot isolation_, allowing simultaneous writes by other database clients. This must be enabled on the database like so:
 
   ```text
   ALTER DATABASE {database name}
@@ -135,7 +134,7 @@ For further detail, see the [Microsoft docs on enabling and disabling CDC](https
 
 #### 3. Create a user and grant appropriate permissions
 
-* Rather than use _sysadmin_ or _db\_owner_ credentials, we recommend creating a new user with the relevant CDC access for use with Airbyte. First let's create the login and user and add to the [db\_datareader](https://docs.microsoft.com/en-us/sql/relational-databases/security/authentication-access/database-level-roles?view=sql-server-ver15) role:
+- Rather than use _sysadmin_ or _db_owner_ credentials, we recommend creating a new user with the relevant CDC access for use with Airbyte. First let's create the login and user and add to the [db_datareader](https://docs.microsoft.com/en-us/sql/relational-databases/security/authentication-access/database-level-roles?view=sql-server-ver15) role:
 
   ```text
   USE {database name};
@@ -145,20 +144,20 @@ For further detail, see the [Microsoft docs on enabling and disabling CDC](https
   EXEC sp_addrolemember 'db_datareader', '{user name}';
   ```
 
-  * Add the user to the role specified earlier when enabling cdc on the table\(s\):
+  - Add the user to the role specified earlier when enabling cdc on the table\(s\):
 
     ```text
     EXEC sp_addrolemember '{role name}', '{user name}';
     ```
 
-  * This should be enough access, but if you run into problems, try also directly granting the user `SELECT` access on the cdc schema:
+  - This should be enough access, but if you run into problems, try also directly granting the user `SELECT` access on the cdc schema:
 
     ```text
     USE {database name};
     GRANT SELECT ON SCHEMA :: [cdc] TO {user name};
     ```
 
-  * If feasible, granting this user 'VIEW SERVER STATE' permissions will allow Airbyte to check whether or not the [SQL Server Agent](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-ver15#relationship-with-log-reader-agent) is running. This is preferred as it ensures syncs will fail if the CDC tables are not being updated by the Agent in the source database.
+  - If feasible, granting this user 'VIEW SERVER STATE' permissions will allow Airbyte to check whether or not the [SQL Server Agent](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-ver15#relationship-with-log-reader-agent) is running. This is preferred as it ensures syncs will fail if the CDC tables are not being updated by the Agent in the source database.
 
     ```text
     USE master;
@@ -167,15 +166,15 @@ For further detail, see the [Microsoft docs on enabling and disabling CDC](https
 
 #### 4. Extend the retention period of CDC data
 
-* In SQL Server, by default, only three days of data are retained in the change tables. Unless you are running very frequent syncs, we suggest increasing this retention so that in case of a failure in sync or if the sync is paused, there is still some bandwidth to start from the last point in incremental sync.
-* These settings can be changed using the stored procedure [sys.sp\_cdc\_change\_job](https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sys-sp-cdc-change-job-transact-sql?view=sql-server-ver15) as below:
+- In SQL Server, by default, only three days of data are retained in the change tables. Unless you are running very frequent syncs, we suggest increasing this retention so that in case of a failure in sync or if the sync is paused, there is still some bandwidth to start from the last point in incremental sync.
+- These settings can be changed using the stored procedure [sys.sp_cdc_change_job](https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sys-sp-cdc-change-job-transact-sql?view=sql-server-ver15) as below:
 
   ```text
   -- we recommend 14400 minutes (10 days) as retention period
   EXEC sp_cdc_change_job @job_type='cleanup', @retention = {minutes}
   ```
 
-* After making this change, a restart of the cleanup job is required:
+- After making this change, a restart of the cleanup job is required:
 
 ```text
   EXEC sys.sp_cdc_stop_job @job_type = 'cleanup';
@@ -185,7 +184,7 @@ For further detail, see the [Microsoft docs on enabling and disabling CDC](https
 
 #### 5. Ensure the SQL Server Agent is running
 
-* MSSQL uses the SQL Server Agent
+- MSSQL uses the SQL Server Agent
 
   to [run the jobs necessary](https://docs.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=sql-server-ver15#agent-jobs)
 
@@ -197,7 +196,7 @@ For further detail, see the [Microsoft docs on enabling and disabling CDC](https
   EXEC xp_servicecontrol 'QueryState', N'SQLServerAGENT';
 ```
 
-* If you see something other than 'Running.' please follow
+- If you see something other than 'Running.' please follow
 
   the [Microsoft docs](https://docs.microsoft.com/en-us/sql/ssms/agent/start-stop-or-pause-the-sql-server-agent-service?view=sql-server-ver15)
 
@@ -267,7 +266,7 @@ This produces the private key in pem format, and the public key remains in the s
 MSSQL data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceComprehensiveTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test!
 
 | MSSQL Type                                              | Resulting Type | Notes |
-|:--------------------------------------------------------|:---------------|:------|
+| :------------------------------------------------------ | :------------- | :---- |
 | `bigint`                                                | number         |       |
 | `binary`                                                | string         |       |
 | `bit`                                                   | boolean        |       |
@@ -303,6 +302,7 @@ MSSQL data types are mapped to the following data types when synchronizing data.
 If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings.
 
 ## Upgrading from 0.4.17 and older versions to 0.4.18 and newer versions
+
 There is a backwards incompatible spec change between Microsoft SQL Source connector versions 0.4.17 and 0.4.18. As part of that spec change
 `replication_method` configuration parameter was changed to `object` from `string`.
 
@@ -313,6 +313,7 @@ In Microsoft SQL source connector versions 0.4.17 and older, `replication_method
 ```
 
 Starting with version 0.4.18, `replication_method` configuration parameter is saved as follows:
+
 ```
 "replication_method": {
     "method": "STANDARD"
@@ -330,7 +331,7 @@ update public.actor set configuration =jsonb_set(configuration, '{replication_me
 WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configuration->>'replication_method' = 'STANDARD');
 ```
 
-If you have connections with Microsoft SQL Source using _Logicai Replication (CDC)_ method,  run this SQL:
+If you have connections with Microsoft SQL Source using _Logicai Replication (CDC)_ method, run this SQL:
 
 ```sql
 update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "CDC"}', true)
@@ -340,7 +341,12 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
 ## Changelog
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                         |
-|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
+|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------| :---------------------------------------------------------------------------------------------------------------------------------------------- |
+| 2.0.0   | 2023-08-22 | [29493](https://github.com/airbytehq/airbyte/pull/29493)                                                          | Set a default cursor for Cdc mode                                                                                                               |
+| 1.1.1   | 2023-07-24 | [28545](https://github.com/airbytehq/airbyte/pull/28545)                                                          | Support Read Committed snapshot isolation level                                                                                                 |
+| 1.1.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737)                                                          | License Update: Elv2                                                                                                                            |
+| 1.0.19  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212)                                                          | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                        |
+| 1.0.18  | 2023-06-14 | [27335](https://github.com/airbytehq/airbyte/pull/27335)                                                          | Remove noisy debug logs                                                                                                                         |
 | 1.0.17  | 2023-05-25 | [26473](https://github.com/airbytehq/airbyte/pull/26473)                                                          | CDC : Limit queue size                                                                                                                          |
 | 1.0.16  | 2023-05-01 | [25740](https://github.com/airbytehq/airbyte/pull/25740)                                                          | Disable index logging                                                                                                                           |
 | 1.0.15  | 2023-04-26 | [25401](https://github.com/airbytehq/airbyte/pull/25401)                                                          | CDC : Upgrade Debezium to version 2.2.0                                                                                                         |
@@ -348,7 +354,7 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
 | 1.0.13  | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582)                                                          | CDC : refactor for performance improvement                                                                                                      |
 | 1.0.12  | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220)                                                          | Logging changes : Log additional metadata & clean up noisy logs                                                                                 |
 | 1.0.11  | 2023-04-11 | [24656](https://github.com/airbytehq/airbyte/pull/24656)                                                          | CDC minor refactor                                                                                                                              |
-| 1.0.10  | 2023-04-06 | [24820](https://github.com/airbytehq/airbyte/pull/24820)                                                          | Fix data loss bug during an initial failed non-CDC incremental sync                                                                             | 
+| 1.0.10  | 2023-04-06 | [24820](https://github.com/airbytehq/airbyte/pull/24820)                                                          | Fix data loss bug during an initial failed non-CDC incremental sync                                                                             |
 | 1.0.9   | 2023-04-04 | [24833](https://github.com/airbytehq/airbyte/pull/24833)                                                          | Fix Debezium retry policy configuration                                                                                                         |
 | 1.0.8   | 2023-03-28 | [24166](https://github.com/airbytehq/airbyte/pull/24166)                                                          | Fix InterruptedException bug during Debezium shutdown                                                                                           |
 | 1.0.7   | 2023-03-27 | [24529](https://github.com/airbytehq/airbyte/pull/24373)                                                          | Preparing the connector for CDC checkpointing                                                                                                   |
@@ -399,13 +405,13 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
 | 0.3.12  | 2021-12-30 | [9206](https://github.com/airbytehq/airbyte/pull/9206)                                                            | Update connector fields title/description                                                                                                       |
 | 0.3.11  | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958)                                                            | Add support for JdbcType.ARRAY                                                                                                                  |
 | 0.3.10  | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371)                                                            | Fixed incorrect handling "\n" in ssh key                                                                                                        |
-| 0.3.9   | 2021-11-09 | [7386](https://github.com/airbytehq/airbyte/pull/7386)                                                            | Improve support for binary and varbinary data types                                                                                             |
+| 0.3.9   | 2021-11-09 | [7748](https://github.com/airbytehq/airbyte/pull/7748)                                                            | Improve support for binary and varbinary data types                                                                                             |
 | 0.3.8   | 2021-10-26 | [7386](https://github.com/airbytehq/airbyte/pull/7386)                                                            | Fixed data type (smalldatetime, smallmoney) conversion from mssql source                                                                        |
 | 0.3.7   | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585)                                                            | Improved SSH Tunnel key generation steps                                                                                                        |
 | 0.3.6   | 2021-09-17 | [6318](https://github.com/airbytehq/airbyte/pull/6318)                                                            | Added option to connect to DB via SSH                                                                                                           |
 | 0.3.4   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699)                                                            | Added json config validator                                                                                                                     |
 | 0.3.3   | 2021-07-05 | [4689](https://github.com/airbytehq/airbyte/pull/4689)                                                            | Add CDC support                                                                                                                                 |
-| 0.3.2   | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973)                                                            | Add AIRBYTE\_ENTRYPOINT for Kubernetes support                                                                                                  |
+| 0.3.2   | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973)                                                            | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                                                                                   |
 | 0.3.1   | 2021-06-08 | [3893](https://github.com/airbytehq/airbyte/pull/3893)                                                            | Enable SSL connection                                                                                                                           |
 | 0.3.0   | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990)                                                            | Support namespaces                                                                                                                              |
 | 0.2.3   | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600)                                                            | Add NCHAR and NVCHAR support to DB and cursor type casting                                                                                      |
@@ -420,4 +426,3 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura
 | 0.1.6   | 2020-12-09 | [1172](https://github.com/airbytehq/airbyte/pull/1172)                                                            | Support incremental sync                                                                                                                        |
 | 0.1.5   | 2020-11-30 | [1038](https://github.com/airbytehq/airbyte/pull/1038)                                                            | Change JDBC sources to discover more than standard schemas                                                                                      |
 | 0.1.4   | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046)                                                            | Add connectors using an index YAML file                                                                                                         |
-
diff --git a/docs/integrations/sources/mysql-migrations.md b/docs/integrations/sources/mysql-migrations.md
new file mode 100644
index 000000000000..b593d3af36dc
--- /dev/null
+++ b/docs/integrations/sources/mysql-migrations.md
@@ -0,0 +1,4 @@
+# MySQL Migration Guide
+
+## Upgrading to 3.0.0
+CDC syncs now has default cursor field called `_ab_cdc_cursor`. You will need to force normalization to rebuild your destination tables by manually dropping the SCD tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. 
\ No newline at end of file
diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md
index eef4a1c99638..a388ce447596 100644
--- a/docs/integrations/sources/mysql.md
+++ b/docs/integrations/sources/mysql.md
@@ -3,7 +3,7 @@
 ## Features
 
 | Feature                       | Supported | Notes                             |
-|:------------------------------|:----------|:----------------------------------|
+| :---------------------------- | :-------- | :-------------------------------- |
 | Full Refresh Sync             | Yes       |                                   |
 | Incremental - Append Sync     | Yes       |                                   |
 | Replicate Incremental Deletes | Yes       |                                   |
@@ -137,11 +137,13 @@ In this case, you can configure the server timezone to the equivalent IANA timez
 
 When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. Airbyte doesn't acquire any table locks \(for tables defined with MyISAM engine, the tables would still be locked\) while creating the snapshot to allow writes by other database clients. But in order for the sync to work without any error/unexpected behaviour, it is assumed that no schema changes are happening while the snapshot is running.
 
-If seeing `EventDataDeserializationException` errors intermittently with root cause `EOFException` or `SocketException`, you may need to extend the following *MySql server* timeout values by running:
+If seeing `EventDataDeserializationException` errors intermittently with root cause `EOFException` or `SocketException`, you may need to extend the following _MySql server_ timeout values by running:
+
 ```
 set global slave_net_timeout = 120;
 set global thread_pool_idle_timeout = 120;
 ```
+
 ## Connection via SSH Tunnel
 
 Airbyte has the ability to connect to a MySQl instance via an SSH Tunnel. The reason you might want to do this because it is not possible \(or against security policy\) to connect to the database directly \(e.g. it does not have a public IP address\).
@@ -180,10 +182,10 @@ This produces the private key in pem format, and the public key remains in the s
 
 MySQL data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test!
 
-Any database or table encoding combination of charset and collation is supported. Charset setting however will not be carried over to destination and data will be encoded with whatever is configured by the destination. 
+Any database or table encoding combination of charset and collation is supported. Charset setting however will not be carried over to destination and data will be encoded with whatever is configured by the destination.
 
 | MySQL Type                                | Resulting Type         | Notes                                                                                                          |
-|:------------------------------------------|:-----------------------|:---------------------------------------------------------------------------------------------------------------|
+| :---------------------------------------- | :--------------------- | :------------------------------------------------------------------------------------------------------------- |
 | `bit(1)`                                  | boolean                |                                                                                                                |
 | `bit(>1)`                                 | base64 binary string   |                                                                                                                |
 | `boolean`                                 | boolean                |                                                                                                                |
@@ -260,114 +262,120 @@ WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configura
 
 ## Changelog
 
-| Version | Date       | Pull Request                                                   | Subject                                                                                                                                         |
-|:--------|:-----------|:---------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
-| 2.0.24  | 2023-05-25 | [26473](https://github.com/airbytehq/airbyte/pull/26473)       | CDC : Limit queue size                                                                                                                          |
-| 2.0.23  | 2023-05-24 | [25586](https://github.com/airbytehq/airbyte/pull/25586)       | No need to base64 encode strings on databases sorted with binary collation                                                                      |
-| 2.0.22  | 2023-05-22 | [25859](https://github.com/airbytehq/airbyte/pull/25859)       | Allow adding sessionVariables JDBC parameters                                                                                                   |
-| 2.0.21  | 2023-05-10 | [25460](https://github.com/airbytehq/airbyte/pull/25460)       | Handle a decimal number with 0 decimal points as an integer                                                                                     |
-| 2.0.20  | 2023-05-01 | [25740](https://github.com/airbytehq/airbyte/pull/25740)       | Disable index logging                                                                                                                           |
-| 2.0.19  | 2023-04-26 | [25401](https://github.com/airbytehq/airbyte/pull/25401)       | CDC : Upgrade Debezium to version 2.2.0                                                                                                         |
-| 2.0.18  | 2023-04-19 | [25345](https://github.com/airbytehq/airbyte/pull/25345)       | Logging : Log database indexes per stream                                                                                                       |
-| 2.0.17  | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582)       | CDC : refactor for performance improvement                                                                                                      |
-| 2.0.16  | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220)       | Logging changes : Log additional metadata & clean up noisy logs                                                                                 |
-| 2.0.15  | 2023-04-12 | [25131](https://github.com/airbytehq/airbyte/pull/25131)       | Make Client Certificate and Client Key always show                                                                                              |
-| 2.0.14  | 2023-04-11 | [24656](https://github.com/airbytehq/airbyte/pull/24656)       | CDC minor refactor                                                                                                                              |
-| 2.0.13  | 2023-04-06 | [24820](https://github.com/airbytehq/airbyte/pull/24820)       | Fix data loss bug during an initial failed non-CDC incremental sync                                                                             |
-| 2.0.12  | 2023-04-04 | [24833](https://github.com/airbytehq/airbyte/pull/24833)       | Fix Debezium retry policy configuration                                                                                                         |
-| 2.0.11  | 2023-03-28 | [24166](https://github.com/airbytehq/airbyte/pull/24166)       | Fix InterruptedException bug during Debezium shutdown                                                                                           |
-| 2.0.10  | 2023-03-27 | [24529](https://github.com/airbytehq/airbyte/pull/24373)       | Preparing the connector for CDC checkpointing                                                                                                   |
-| 2.0.9   | 2023-03-24 | [24529](https://github.com/airbytehq/airbyte/pull/24529)       | Set SSL Mode to required on strict-encrypt variant                                                                                              |
-| 2.0.8   | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760)       | Removed redundant date-time datatypes formatting                                                                                                |
-| 2.0.7   | 2023-03-21 | [24207](https://github.com/airbytehq/airbyte/pull/24207)       | Fix incorrect schema change warning in CDC mode                                                                                                 |
-| 2.0.6   | 2023-03-21 | [23984](https://github.com/airbytehq/airbyte/pull/23984)       | Support CDC heartbeats                                                                                                                          |
-| 2.0.5   | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275)       | Fix error with CDC checkpointing                                                                                                                |
-| 2.0.4   | 2023-03-20 | [24147](https://github.com/airbytehq/airbyte/pull/24147)       | Support different table structure during "DESCRIBE" query                                                                                       |
-| 2.0.3   | 2023-03-15 | [24082](https://github.com/airbytehq/airbyte/pull/24082)       | Fixed NPE during cursor values validation                                                                                                       |
-| 2.0.2   | 2023-03-14 | [23908](https://github.com/airbytehq/airbyte/pull/23908)       | Log warning on null cursor values                                                                                                               |
-| 2.0.1   | 2023-03-10 | [23939](https://github.com/airbytehq/airbyte/pull/23939)       | For network isolation, source connector accepts a list of hosts it is allowed to connect                                                        |
-| 2.0.0   | 2023-03-06 | [23112](https://github.com/airbytehq/airbyte/pull/23112)       | Upgrade Debezium version to 2.1.2                                                                                                               |
-| 1.0.21  | 2023-01-25 | [20939](https://github.com/airbytehq/airbyte/pull/20939)       | Adjust batch selection memory limits databases.                                                                                                 |
-| 1.0.20  | 2023-01-24 | [20593](https://github.com/airbytehq/airbyte/pull/20593)       | Handle ssh time out exception                                                                                                                   |
-| 1.0.19  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)       | Consolidate date/time values mapping for JDBC sources                                                                                           |
-| 1.0.18  | 2022-12-14 | [20378](https://github.com/airbytehq/airbyte/pull/20378)       | Improve descriptions                                                                                                                            |
-| 1.0.17  | 2022-12-13 | [20289](https://github.com/airbytehq/airbyte/pull/20289)       | Mark unknown column exception as config error                                                                                                   |
-| 1.0.16  | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959)       | CDC : Don't timeout if snapshot is not complete.                                                                                                |
-| 1.0.15  | 2022-12-06 | [20000](https://github.com/airbytehq/airbyte/pull/20000)       | Add check and better messaging when user does not have permission to access binary log in CDC mode                                              |
-| 1.0.14  | 2022-11-22 | [19514](https://github.com/airbytehq/airbyte/pull/19514)       | Adjust batch selection memory limits databases.                                                                                                 |
-| 1.0.13  | 2022-11-14 | [18956](https://github.com/airbytehq/airbyte/pull/18956)       | Clean up Tinyint Unsigned data type identification                                                                                              |
-| 1.0.12  | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025)       | Stop enforce SSL if ssl mode is disabled                                                                                                        |
-| 1.0.11  | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851)       | Fix bug with unencrypted CDC connections                                                                                                        |
-| 1.0.10  | 2022-11-02 | [18619](https://github.com/airbytehq/airbyte/pull/18619)       | Fix bug with handling Tinyint(1) Unsigned values as boolean                                                                                     |
-| 1.0.9   | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538)       | Encode database name                                                                                                                            |
-| 1.0.8   | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383)       | Better SSH error handling + messages                                                                                                            |
-| 1.0.7   | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263)       | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations |
-| 1.0.6   | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087)       | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor)                                                       |
-| 1.0.5   | 2022-10-17 | [18041](https://github.com/airbytehq/airbyte/pull/18041)       | Fixes bug introduced 2022-09-12 with SshTunnel, handles iterator exception properly                                                             |
-|         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238)       | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode       |
-| 1.0.4   | 2022-10-11 | [17815](https://github.com/airbytehq/airbyte/pull/17815)       | Expose setting server timezone for CDC syncs                                                                                                    |
-| 1.0.3   | 2022-10-07 | [17236](https://github.com/airbytehq/airbyte/pull/17236)       | Fix large table issue by fetch size                                                                                                             |
-| 1.0.2   | 2022-10-03 | [17170](https://github.com/airbytehq/airbyte/pull/17170)       | Make initial CDC waiting time configurable                                                                                                      |
-| 1.0.1   | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459)       | Upgrade debezium version to 1.9.6 from 1.9.2                                                                                                    |
-| 1.0.0   | 2022-09-27 | [17164](https://github.com/airbytehq/airbyte/pull/17164)       | Certify MySQL Source as Beta                                                                                                                    |
-| 0.6.15  | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299)       | Improve error handling for strict-encrypt mysql source                                                                                          |
-| 0.6.14  | 2022-09-26 | [16954](https://github.com/airbytehq/airbyte/pull/16954)       | Implement support for snapshot of new tables in CDC mode                                                                                        |
-| 0.6.13  | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668)       | Wrap logs in AirbyteLogMessage                                                                                                                  |
-| 0.6.12  | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657)       | Improve CDC record queueing performance                                                                                                         |
-| 0.6.11  | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202)       | Adds error messaging factory to UI                                                                                                              |
-| 0.6.10  | 2022-09-08 | [16007](https://github.com/airbytehq/airbyte/pull/16007)       | Implement per stream state support.                                                                                                             |
-| 0.6.9   | 2022-09-03 | [16216](https://github.com/airbytehq/airbyte/pull/16216)       | Standardize spec for CDC replication. See upgrade instructions [above](#upgrading-from-0.6.8-and-older-versions-to-0.6.9-and-later-versions).   |
-| 0.6.8   | 2022-09-01 | [16259](https://github.com/airbytehq/airbyte/pull/16259)       | Emit state messages more frequently                                                                                                             |
-| 0.6.7   | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114)       | Prevent traffic going on an unsecured channel in strict-encryption version of source mysql                                                      |
-| 0.6.6   | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993)       | Improved support for connecting over SSL                                                                                                        |
-| 0.6.5   | 2022-08-25 | [15917](https://github.com/airbytehq/airbyte/pull/15917)       | Fix temporal data type default value bug                                                                                                        |
-| 0.6.4   | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356)       | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field                                       |
-| 0.6.3   | 2022-08-12 | [15044](https://github.com/airbytehq/airbyte/pull/15044)       | Added the ability to connect using different SSL modes and SSL certificates                                                                     |
-| 0.6.2   | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538)       | Allow additional properties in db stream state                                                                                                  |
-| 0.6.1   | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801)       | Fix multiple log bindings                                                                                                                       |
-| 0.6.0   | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362)       | Integral columns are now discovered as int64 fields.                                                                                            |
-| 0.5.17  | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714)       | Clarified error message when invalid cursor column selected                                                                                     |
-| 0.5.16  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574)       | Removed additionalProperties:false from JDBC source connectors                                                                                  |
-| 0.5.15  | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077)       | Use the new state management                                                                                                                    |
-| 0.5.13  | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945)       | Aligned datatype test                                                                                                                           |
-| 0.5.12  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864)       | Updated stacktrace format for any trace message errors                                                                                          |
-| 0.5.11  | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544)       | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads.                                               |
-| 0.5.10  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480)       | Query tables with adaptive fetch size to optimize JDBC memory consumption                                                                       |
-| 0.5.9   | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729)       | Bump mina-sshd from 2.7.0 to 2.8.0                                                                                                              |
-| 0.5.6   | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242)       | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats                                          |
-| 0.5.5   | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242)       | Updated timestamp transformation with microseconds                                                                                              |
-| 0.5.4   | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251)     | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column                                                                  |
-| 0.5.2   | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425)       | MySQL CDC sync fails because starting binlog position not found in DB                                                                           |
-| 0.5.1   | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582)         | Update connector fields title/description                                                                                                       |
-| 0.5.0   | 2021-12-11 | [7970](https://github.com/airbytehq/airbyte/pull/7970)         | Support all MySQL types                                                                                                                         |
-| 0.4.13  | 2021-12-03 | [8335](https://github.com/airbytehq/airbyte/pull/8335)         | Source-MySql: do not check cdc required param binlog_row_image for standard replication                                                         |
-| 0.4.12  | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371)         | Fixed incorrect handling "\n" in ssh key                                                                                                        |
-| 0.4.11  | 2021-11-19 | [8047](https://github.com/airbytehq/airbyte/pull/8047)         | Source MySQL: transform binary data base64 format                                                                                               |
-| 0.4.10  | 2021-11-15 | [7820](https://github.com/airbytehq/airbyte/pull/7820)         | Added basic performance test                                                                                                                    |
-| 0.4.9   | 2021-11-02 | [7559](https://github.com/airbytehq/airbyte/pull/7559)         | Correctly process large unsigned short integer values which may fall outside java's `Short` data type capability                                |
-| 0.4.8   | 2021-09-16 | [6093](https://github.com/airbytehq/airbyte/pull/6093)         | Improve reliability of processing various data types like decimals, dates, datetime, binary, and text                                           |
-| 0.4.7   | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585)         | Improved SSH Tunnel key generation steps                                                                                                        |
-| 0.4.6   | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510)         | Support SSL connection                                                                                                                          |
-| 0.4.5   | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146)         | Added option to connect to DB via SSH                                                                                                           |
-| 0.4.1   | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956)         | Fix log link                                                                                                                                    |
-| 0.3.7   | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973)         | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                                                                                   |
-| 0.3.6   | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966)         | Fix excessive logging for CDC method                                                                                                            |
-| 0.3.5   | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890)         | Fix CDC handle tinyint\(1\) and boolean types                                                                                                   |
-| 0.3.4   | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846)         | Fix max integer value failure                                                                                                                   |
-| 0.3.3   | 2021-06-02 | [3789](https://github.com/airbytehq/airbyte/pull/3789)         | MySQL CDC poll wait 5 minutes when not received a single record                                                                                 |
-| 0.3.2   | 2021-06-01 | [3757](https://github.com/airbytehq/airbyte/pull/3757)         | MySQL CDC poll 5s to 5 min                                                                                                                      |
-| 0.3.1   | 2021-06-01 | [3505](https://github.com/airbytehq/airbyte/pull/3505)         | Implemented MySQL CDC                                                                                                                           |
-| 0.3.0   | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990)         | Support namespaces                                                                                                                              |
-| 0.2.5   | 2021-04-15 | [2899](https://github.com/airbytehq/airbyte/pull/2899)         | Fix bug in tests                                                                                                                                |
-| 0.2.4   | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600)         | Add NCHAR and NVCHAR support to DB and cursor type casting                                                                                      |
-| 0.2.3   | 2021-03-26 | [2611](https://github.com/airbytehq/airbyte/pull/2611)         | Add an optional `jdbc_url_params` in parameters                                                                                                 |
-| 0.2.2   | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460)         | Destination supports destination sync mode                                                                                                      |
-| 0.2.1   | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488)         | Sources support primary keys                                                                                                                    |
-| 0.2.0   | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238)         | Protocol allows future/unknown properties                                                                                                       |
-| 0.1.10  | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887)         | Migrate AbstractJdbcSource to use iterators                                                                                                     |
-| 0.1.9   | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746)         | Fix NPE in State Decorator                                                                                                                      |
-| 0.1.8   | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724)         | Fix JdbcSource handling of tables with same names in different schemas                                                                          |
-| 0.1.7   | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655)         | Fix JdbcSource OOM                                                                                                                              |
-| 0.1.6   | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307)         | Migrate Postgres and MySQL to use new JdbcSource                                                                                                |
-| 0.1.5   | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267)         | Support incremental sync                                                                                                                        |
-| 0.1.4   | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046)         | Add connectors using an index YAML file                                                                                                         |
+| Version | Date       | Pull Request                                               | Subject                                                                                                                                         |
+|:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.0.1   | 2023-08-21 | [29308](https://github.com/airbytehq/airbyte/pull/29308)   | CDC: Enable frequent state emissions during incremental runs                                                                                    |
+| 3.0.0   | 2023-08-08 | [28756](https://github.com/airbytehq/airbyte/pull/28756)   | CDC: Set a default cursor                                                                                                                       |
+| 2.1.2   | 2023-08-08 | [29220](https://github.com/airbytehq/airbyte/pull/29220)   | Add indicator that CDC is the recommended update method                                                                                         |
+| 2.1.1   | 2023-07-31 | [28882](https://github.com/airbytehq/airbyte/pull/28882)   | Improve replication method labels and descriptions                                                                                              |
+| 2.1.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737)   | License Update: Elv2                                                                                                                            |
+| 2.0.25  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212)   | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                        |
+| 2.0.24  | 2023-05-25 | [26473](https://github.com/airbytehq/airbyte/pull/26473)   | CDC : Limit queue size                                                                                                                          |
+| 2.0.23  | 2023-05-24 | [25586](https://github.com/airbytehq/airbyte/pull/25586)   | No need to base64 encode strings on databases sorted with binary collation                                                                      |
+| 2.0.22  | 2023-05-22 | [25859](https://github.com/airbytehq/airbyte/pull/25859)   | Allow adding sessionVariables JDBC parameters                                                                                                   |
+| 2.0.21  | 2023-05-10 | [25460](https://github.com/airbytehq/airbyte/pull/25460)   | Handle a decimal number with 0 decimal points as an integer                                                                                     |
+| 2.0.20  | 2023-05-01 | [25740](https://github.com/airbytehq/airbyte/pull/25740)   | Disable index logging                                                                                                                           |
+| 2.0.19  | 2023-04-26 | [25401](https://github.com/airbytehq/airbyte/pull/25401)   | CDC : Upgrade Debezium to version 2.2.0                                                                                                         |
+| 2.0.18  | 2023-04-19 | [25345](https://github.com/airbytehq/airbyte/pull/25345)   | Logging : Log database indexes per stream                                                                                                       |
+| 2.0.17  | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582)   | CDC : refactor for performance improvement                                                                                                      |
+| 2.0.16  | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220)   | Logging changes : Log additional metadata & clean up noisy logs                                                                                 |
+| 2.0.15  | 2023-04-12 | [25131](https://github.com/airbytehq/airbyte/pull/25131)   | Make Client Certificate and Client Key always show                                                                                              |
+| 2.0.14  | 2023-04-11 | [24656](https://github.com/airbytehq/airbyte/pull/24656)   | CDC minor refactor                                                                                                                              |
+| 2.0.13  | 2023-04-06 | [24820](https://github.com/airbytehq/airbyte/pull/24820)   | Fix data loss bug during an initial failed non-CDC incremental sync                                                                             |
+| 2.0.12  | 2023-04-04 | [24833](https://github.com/airbytehq/airbyte/pull/24833)   | Fix Debezium retry policy configuration                                                                                                         |
+| 2.0.11  | 2023-03-28 | [24166](https://github.com/airbytehq/airbyte/pull/24166)   | Fix InterruptedException bug during Debezium shutdown                                                                                           |
+| 2.0.10  | 2023-03-27 | [24529](https://github.com/airbytehq/airbyte/pull/24373)   | Preparing the connector for CDC checkpointing                                                                                                   |
+| 2.0.9   | 2023-03-24 | [24529](https://github.com/airbytehq/airbyte/pull/24529)   | Set SSL Mode to required on strict-encrypt variant                                                                                              |
+| 2.0.8   | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760)   | Removed redundant date-time datatypes formatting                                                                                                |
+| 2.0.7   | 2023-03-21 | [24207](https://github.com/airbytehq/airbyte/pull/24207)   | Fix incorrect schema change warning in CDC mode                                                                                                 |
+| 2.0.6   | 2023-03-21 | [23984](https://github.com/airbytehq/airbyte/pull/23984)   | Support CDC heartbeats                                                                                                                          |
+| 2.0.5   | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275)   | Fix error with CDC checkpointing                                                                                                                |
+| 2.0.4   | 2023-03-20 | [24147](https://github.com/airbytehq/airbyte/pull/24147)   | Support different table structure during "DESCRIBE" query                                                                                       |
+| 2.0.3   | 2023-03-15 | [24082](https://github.com/airbytehq/airbyte/pull/24082)   | Fixed NPE during cursor values validation                                                                                                       |
+| 2.0.2   | 2023-03-14 | [23908](https://github.com/airbytehq/airbyte/pull/23908)   | Log warning on null cursor values                                                                                                               |
+| 2.0.1   | 2023-03-10 | [23939](https://github.com/airbytehq/airbyte/pull/23939)   | For network isolation, source connector accepts a list of hosts it is allowed to connect                                                        |
+| 2.0.0   | 2023-03-06 | [23112](https://github.com/airbytehq/airbyte/pull/23112)   | Upgrade Debezium version to 2.1.2                                                                                                               |
+| 1.0.21  | 2023-01-25 | [20939](https://github.com/airbytehq/airbyte/pull/20939)   | Adjust batch selection memory limits databases.                                                                                                 |
+| 1.0.20  | 2023-01-24 | [20593](https://github.com/airbytehq/airbyte/pull/20593)   | Handle ssh time out exception                                                                                                                   |
+| 1.0.19  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)   | Consolidate date/time values mapping for JDBC sources                                                                                           |
+| 1.0.18  | 2022-12-14 | [20378](https://github.com/airbytehq/airbyte/pull/20378)   | Improve descriptions                                                                                                                            |
+| 1.0.17  | 2022-12-13 | [20289](https://github.com/airbytehq/airbyte/pull/20289)   | Mark unknown column exception as config error                                                                                                   |
+| 1.0.16  | 2022-12-12 | [18959](https://github.com/airbytehq/airbyte/pull/18959)   | CDC : Don't timeout if snapshot is not complete.                                                                                                |
+| 1.0.15  | 2022-12-06 | [20000](https://github.com/airbytehq/airbyte/pull/20000)   | Add check and better messaging when user does not have permission to access binary log in CDC mode                                              |
+| 1.0.14  | 2022-11-22 | [19514](https://github.com/airbytehq/airbyte/pull/19514)   | Adjust batch selection memory limits databases.                                                                                                 |
+| 1.0.13  | 2022-11-14 | [18956](https://github.com/airbytehq/airbyte/pull/18956)   | Clean up Tinyint Unsigned data type identification                                                                                              |
+| 1.0.12  | 2022-11-07 | [19025](https://github.com/airbytehq/airbyte/pull/19025)   | Stop enforce SSL if ssl mode is disabled                                                                                                        |
+| 1.0.11  | 2022-11-03 | [18851](https://github.com/airbytehq/airbyte/pull/18851)   | Fix bug with unencrypted CDC connections                                                                                                        |
+| 1.0.10  | 2022-11-02 | [18619](https://github.com/airbytehq/airbyte/pull/18619)   | Fix bug with handling Tinyint(1) Unsigned values as boolean                                                                                     |
+| 1.0.9   | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538)   | Encode database name                                                                                                                            |
+| 1.0.8   | 2022-10-25 | [18383](https://github.com/airbytehq/airbyte/pull/18383)   | Better SSH error handling + messages                                                                                                            |
+| 1.0.7   | 2022-10-21 | [18263](https://github.com/airbytehq/airbyte/pull/18263)   | Fixes bug introduced in [15833](https://github.com/airbytehq/airbyte/pull/15833) and adds better error messaging for SSH tunnel in Destinations |
+| 1.0.6   | 2022-10-19 | [18087](https://github.com/airbytehq/airbyte/pull/18087)   | Better error messaging for configuration errors (SSH configs, choosing an invalid cursor)                                                       |
+| 1.0.5   | 2022-10-17 | [18041](https://github.com/airbytehq/airbyte/pull/18041)   | Fixes bug introduced 2022-09-12 with SshTunnel, handles iterator exception properly                                                             |
+|         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238)   | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode       |
+| 1.0.4   | 2022-10-11 | [17815](https://github.com/airbytehq/airbyte/pull/17815)   | Expose setting server timezone for CDC syncs                                                                                                    |
+| 1.0.3   | 2022-10-07 | [17236](https://github.com/airbytehq/airbyte/pull/17236)   | Fix large table issue by fetch size                                                                                                             |
+| 1.0.2   | 2022-10-03 | [17170](https://github.com/airbytehq/airbyte/pull/17170)   | Make initial CDC waiting time configurable                                                                                                      |
+| 1.0.1   | 2022-10-01 | [17459](https://github.com/airbytehq/airbyte/pull/17459)   | Upgrade debezium version to 1.9.6 from 1.9.2                                                                                                    |
+| 1.0.0   | 2022-09-27 | [17164](https://github.com/airbytehq/airbyte/pull/17164)   | Certify MySQL Source as Beta                                                                                                                    |
+| 0.6.15  | 2022-09-27 | [17299](https://github.com/airbytehq/airbyte/pull/17299)   | Improve error handling for strict-encrypt mysql source                                                                                          |
+| 0.6.14  | 2022-09-26 | [16954](https://github.com/airbytehq/airbyte/pull/16954)   | Implement support for snapshot of new tables in CDC mode                                                                                        |
+| 0.6.13  | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668)   | Wrap logs in AirbyteLogMessage                                                                                                                  |
+| 0.6.12  | 2022-09-13 | [16657](https://github.com/airbytehq/airbyte/pull/16657)   | Improve CDC record queueing performance                                                                                                         |
+| 0.6.11  | 2022-09-08 | [16202](https://github.com/airbytehq/airbyte/pull/16202)   | Adds error messaging factory to UI                                                                                                              |
+| 0.6.10  | 2022-09-08 | [16007](https://github.com/airbytehq/airbyte/pull/16007)   | Implement per stream state support.                                                                                                             |
+| 0.6.9   | 2022-09-03 | [16216](https://github.com/airbytehq/airbyte/pull/16216)   | Standardize spec for CDC replication. See upgrade instructions [above](#upgrading-from-0.6.8-and-older-versions-to-0.6.9-and-later-versions).   |
+| 0.6.8   | 2022-09-01 | [16259](https://github.com/airbytehq/airbyte/pull/16259)   | Emit state messages more frequently                                                                                                             |
+| 0.6.7   | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114)   | Prevent traffic going on an unsecured channel in strict-encryption version of source mysql                                                      |
+| 0.6.6   | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993)   | Improved support for connecting over SSL                                                                                                        |
+| 0.6.5   | 2022-08-25 | [15917](https://github.com/airbytehq/airbyte/pull/15917)   | Fix temporal data type default value bug                                                                                                        |
+| 0.6.4   | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356)   | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field                                       |
+| 0.6.3   | 2022-08-12 | [15044](https://github.com/airbytehq/airbyte/pull/15044)   | Added the ability to connect using different SSL modes and SSL certificates                                                                     |
+| 0.6.2   | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538)   | Allow additional properties in db stream state                                                                                                  |
+| 0.6.1   | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801)   | Fix multiple log bindings                                                                                                                       |
+| 0.6.0   | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362)   | Integral columns are now discovered as int64 fields.                                                                                            |
+| 0.5.17  | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714)   | Clarified error message when invalid cursor column selected                                                                                     |
+| 0.5.16  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574)   | Removed additionalProperties:false from JDBC source connectors                                                                                  |
+| 0.5.15  | 2022-06-23 | [14077](https://github.com/airbytehq/airbyte/pull/14077)   | Use the new state management                                                                                                                    |
+| 0.5.13  | 2022-06-21 | [13945](https://github.com/airbytehq/airbyte/pull/13945)   | Aligned datatype test                                                                                                                           |
+| 0.5.12  | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864)   | Updated stacktrace format for any trace message errors                                                                                          |
+| 0.5.11  | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544)   | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads.                                               |
+| 0.5.10  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480)   | Query tables with adaptive fetch size to optimize JDBC memory consumption                                                                       |
+| 0.5.9   | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729)   | Bump mina-sshd from 2.7.0 to 2.8.0                                                                                                              |
+| 0.5.6   | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242)   | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats                                          |
+| 0.5.5   | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242)   | Updated timestamp transformation with microseconds                                                                                              |
+| 0.5.4   | 2022-02-11 | [10251](https://github.com/airbytehq/airbyte/issues/10251) | bug Source MySQL CDC: sync failed when has Zero-date value in mandatory column                                                                  |
+| 0.5.2   | 2021-12-14 | [6425](https://github.com/airbytehq/airbyte/issues/6425)   | MySQL CDC sync fails because starting binlog position not found in DB                                                                           |
+| 0.5.1   | 2021-12-13 | [8582](https://github.com/airbytehq/airbyte/pull/8582)     | Update connector fields title/description                                                                                                       |
+| 0.5.0   | 2021-12-11 | [7970](https://github.com/airbytehq/airbyte/pull/7970)     | Support all MySQL types                                                                                                                         |
+| 0.4.13  | 2021-12-03 | [8335](https://github.com/airbytehq/airbyte/pull/8335)     | Source-MySql: do not check cdc required param binlog_row_image for standard replication                                                         |
+| 0.4.12  | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371)     | Fixed incorrect handling "\n" in ssh key                                                                                                        |
+| 0.4.11  | 2021-11-19 | [8047](https://github.com/airbytehq/airbyte/pull/8047)     | Source MySQL: transform binary data base64 format                                                                                               |
+| 0.4.10  | 2021-11-15 | [7820](https://github.com/airbytehq/airbyte/pull/7820)     | Added basic performance test                                                                                                                    |
+| 0.4.9   | 2021-11-02 | [7559](https://github.com/airbytehq/airbyte/pull/7559)     | Correctly process large unsigned short integer values which may fall outside java's `Short` data type capability                                |
+| 0.4.8   | 2021-09-16 | [6093](https://github.com/airbytehq/airbyte/pull/6093)     | Improve reliability of processing various data types like decimals, dates, datetime, binary, and text                                           |
+| 0.4.7   | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585)     | Improved SSH Tunnel key generation steps                                                                                                        |
+| 0.4.6   | 2021-09-29 | [6510](https://github.com/airbytehq/airbyte/pull/6510)     | Support SSL connection                                                                                                                          |
+| 0.4.5   | 2021-09-17 | [6146](https://github.com/airbytehq/airbyte/pull/6146)     | Added option to connect to DB via SSH                                                                                                           |
+| 0.4.1   | 2021-07-23 | [4956](https://github.com/airbytehq/airbyte/pull/4956)     | Fix log link                                                                                                                                    |
+| 0.3.7   | 2021-06-09 | [3179](https://github.com/airbytehq/airbyte/pull/3973)     | Add AIRBYTE_ENTRYPOINT for Kubernetes support                                                                                                   |
+| 0.3.6   | 2021-06-09 | [3966](https://github.com/airbytehq/airbyte/pull/3966)     | Fix excessive logging for CDC method                                                                                                            |
+| 0.3.5   | 2021-06-07 | [3890](https://github.com/airbytehq/airbyte/pull/3890)     | Fix CDC handle tinyint\(1\) and boolean types                                                                                                   |
+| 0.3.4   | 2021-06-04 | [3846](https://github.com/airbytehq/airbyte/pull/3846)     | Fix max integer value failure                                                                                                                   |
+| 0.3.3   | 2021-06-02 | [3789](https://github.com/airbytehq/airbyte/pull/3789)     | MySQL CDC poll wait 5 minutes when not received a single record                                                                                 |
+| 0.3.2   | 2021-06-01 | [3757](https://github.com/airbytehq/airbyte/pull/3757)     | MySQL CDC poll 5s to 5 min                                                                                                                      |
+| 0.3.1   | 2021-06-01 | [3505](https://github.com/airbytehq/airbyte/pull/3505)     | Implemented MySQL CDC                                                                                                                           |
+| 0.3.0   | 2021-04-21 | [2990](https://github.com/airbytehq/airbyte/pull/2990)     | Support namespaces                                                                                                                              |
+| 0.2.5   | 2021-04-15 | [2899](https://github.com/airbytehq/airbyte/pull/2899)     | Fix bug in tests                                                                                                                                |
+| 0.2.4   | 2021-03-28 | [2600](https://github.com/airbytehq/airbyte/pull/2600)     | Add NCHAR and NVCHAR support to DB and cursor type casting                                                                                      |
+| 0.2.3   | 2021-03-26 | [2611](https://github.com/airbytehq/airbyte/pull/2611)     | Add an optional `jdbc_url_params` in parameters                                                                                                 |
+| 0.2.2   | 2021-03-26 | [2460](https://github.com/airbytehq/airbyte/pull/2460)     | Destination supports destination sync mode                                                                                                      |
+| 0.2.1   | 2021-03-18 | [2488](https://github.com/airbytehq/airbyte/pull/2488)     | Sources support primary keys                                                                                                                    |
+| 0.2.0   | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238)     | Protocol allows future/unknown properties                                                                                                       |
+| 0.1.10  | 2021-02-02 | [1887](https://github.com/airbytehq/airbyte/pull/1887)     | Migrate AbstractJdbcSource to use iterators                                                                                                     |
+| 0.1.9   | 2021-01-25 | [1746](https://github.com/airbytehq/airbyte/pull/1746)     | Fix NPE in State Decorator                                                                                                                      |
+| 0.1.8   | 2021-01-19 | [1724](https://github.com/airbytehq/airbyte/pull/1724)     | Fix JdbcSource handling of tables with same names in different schemas                                                                          |
+| 0.1.7   | 2021-01-14 | [1655](https://github.com/airbytehq/airbyte/pull/1655)     | Fix JdbcSource OOM                                                                                                                              |
+| 0.1.6   | 2021-01-08 | [1307](https://github.com/airbytehq/airbyte/pull/1307)     | Migrate Postgres and MySQL to use new JdbcSource                                                                                                |
+| 0.1.5   | 2020-12-11 | [1267](https://github.com/airbytehq/airbyte/pull/1267)     | Support incremental sync                                                                                                                        |
+| 0.1.4   | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046)     | Add connectors using an index YAML file                                                                                                         |
diff --git a/docs/integrations/sources/notion.inapp.md b/docs/integrations/sources/notion.inapp.md
index fce475fa78a4..019885f44abf 100644
--- a/docs/integrations/sources/notion.inapp.md
+++ b/docs/integrations/sources/notion.inapp.md
@@ -4,10 +4,30 @@
 ​
 ## Setup guide
 
-1. Choose the method of authentication:
-   * To use OAuth2.0 authorization, click **Authenticate your Notion account**.
-   * If you select Access Token, create a new integration in our [full documentation](https://docs.airbyte.com/integrations/sources/notion)
-2. (Optional) Enter the Start Date in `YYYY-MM-DDTHH:mm:ss.SSSZ`. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-3. Click **Set up source**.
+1. Enter a **Source name** to help you identify this source in Airbyte.
+2. Choose the method of authentication:
+
+
+:::note
+We highly recommend using OAuth2.0 authorization to connect to Notion, as this method significantly simplifies the setup process. If you use OAuth2.0 authorization, you do _not_ need to create and configure a new integration in Notion. Instead, you can authenticate your Notion account directly in Airbyte Cloud.
+:::
+
+- **OAuth2.0** (Recommended): Click **Authenticate your Notion account**. When the popup appears, click **Select pages**. Check the pages you want to give Airbyte access to, and click **Allow access**.
+- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page. For more information on how to create and configure an integration in Notion, refer to our 
+[full documentation](https://docs.airbyte.io/integrations/sources/notion#setup-guide).
+
+
+
+- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page.
+- **OAuth2.0**: Copy and paste the Client ID, Client Secret and Access Token you acquired.
+
+To obtain the necessary authorization credentials, you need to create and configure an integration in Notion. For more information on how to create and configure an integration in Notion, refer to our
+[full documentation](https://docs.airbyte.io/integrations/sources/notion#setup-guide).
+
+
+3. Enter the **Start Date** using the provided datepicker, or by programmatically entering a UTC date and time in the format: `YYYY-MM-DDTHH:mm:ss.SSSZ`. All data generated after this date will be replicated.
+4. Click **Set up source** and wait for the tests to complete.
 ​
-For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Notion](https://docs.airbyte.com/integrations/sources/notion).
\ No newline at end of file
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the 
+[full documentation for Notion](https://docs.airbyte.com/integrations/sources/notion).
diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md
index b0d59308794f..b465bb9c4786 100644
--- a/docs/integrations/sources/notion.md
+++ b/docs/integrations/sources/notion.md
@@ -2,73 +2,96 @@
 
 This page contains the setup guide and reference information for the Notion source connector.
 
+## Prerequisites
+
+- Access to a Notion workspace
+
 ## Setup guide​
 
-### Step 1: Set up Notion​
+To authenticate the Notion source connector, you need to use **one** of the following two methods:
 
-1. Create a new integration on the [My integrations](https://www.notion.so/my-integrations) page.
+- OAuth2.0 authorization (recommended for Airbyte Cloud)
+- Access Token
 
 :::note
+**For Airbyte Cloud users:** We highly recommend using OAuth2.0 authorization to connect to Notion, as this method significantly simplifies the setup process. If you use OAuth2.0 authorization in Airbyte Cloud, you do **not** need to create and configure a new integration in Notion. Instead, you can proceed straight to
+[setting up the connector in Airbyte](#step-3-set-up-the-notion-connector-in-airbyte).
+:::
 
-You must be the owner of a Notion workspace to create a new integration.
+We have provided a quick setup guide for creating an integration in Notion below. If you would like more detailed information and context on Notion integrations, or experience any difficulties with the integration setup process, please refer to the
+[official Notion documentation](https://developers.notion.com/docs).
 
+### Step 1: Create an integration in Notion​
+
+1. Log in to your Notion workspace and navigate to the [My integrations](https://www.notion.so/my-integrations) page. Select **+ New integration**.
+
+:::note
+You must be the owner of the Notion workspace to create a new integration associated with it.
 :::
 
-2. Fill out the form. Make sure to check **Read content** and check any other [capabilities](https://developers.notion.com/reference/capabilities) you want to authorize.
-3. Click **Submit**.
-4. In the **Integration type** section, select either **Internal integration** (token authorization) or **Public integration** (OAuth2.0 authentication).
-5. Check the capabilities you want to authorize.
-6. If you select **Public integration**, fill out the fields in the **OAuth Domain & URIs** section.
-7. Click **Save changes**.
-8. Copy the Internal Access Token if you are using the [internal integration](https://developers.notion.com/docs/authorization#authorizing-internal-integrations), or copy the `access_token`, `client_id`, and `client_secret` if you are using the [public integration](https://developers.notion.com/docs/authorization#authorizing-public-integrations).
-
-### Step 2: Set up the Notion connector in Airbyte
-
-
-**For Airbyte Cloud:**
-
-1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Notion** from the **Source type** dropdown.
-4. Enter a name for your source.
-5. Choose the method of authentication:
-      * If you select **Access Token**, paste the access token from [Step 8](#step-1-set-up-notion​).
-      * If you select **OAuth2.0** authorization, click **Authenticate your Notion account**.
-          * Log in and Authorize the Notion account. Select the permissions you want to allow Airbyte.
-6. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-7. Click **Set up source**.
-
+2. Enter a **Name** for your integration. Make sure you have selected the workspace containing your data to replicate from the **Associated workspace** dropdown menu, and click **Submit**.
+3. In the navbar, select **Capabilities** and make sure to check the **Read content** checkbox to authorize Airbyte to read the content of your pages. You may also wish to check the **Read comments** box, as well as set a User capability to allow access to user information. For more details on the capabilities you can enable, please refer to the [Notion documentation on capabilities](https://developers.notion.com/reference/capabilities).
+
+### Step 2: Set permissions and acquire authorization credentials
+
+#### Access Token (Cloud and Open Source)
+
+If you are authenticating via Access Token, you will need to manually set permissions for each page you want to share with Airbyte.
+
+1. Navigate to the page(s) you want to share with Airbyte. Click the **•••** menu at the top right of the page, select **Add connections**, and choose the integration you created in Step 1.
+2. Once you have selected all the pages to share, you can find and copy the Access Token from the **Secrets** tab of your Notion integration's page. Then proceed to
+   [setting up the connector in Airbyte](#step-2-set-up-the-notion-connector-in-airbyte).
 
 
-**For Airbyte Open Source:**
-
-1. Log in to your Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Notion** from the **Source type** dropdown.
-4. Enter a name for your source.
-5. Choose the method of authentication:
-      * If you select **Access Token**, paste the access token from [Step 8](#step-1-set-up-notion​).
-      * If you select **OAuth2.0** authorization, paste the client ID, access token, and client secret from [Step 8](#step-1-set-up-notion​).
-6. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-7. Click **Set up source**.
+
+#### OAuth2.0 (Open Source only)
+
+If you are authenticating via OAuth2.0 for Airbyte Open Source, you will need to make your integration public and acquire your Client ID, Client Secret and Access Token.
+
+1. Navigate to the **Distribution** tab in your integration page, and toggle the switch to make the integration public.
+2. Fill out the required fields in the **Organization information** and **OAuth Domain & URIs** section, then click **Submit**.
+3. Navigate to the **Secrets** tab to find your Client ID and Client Secret.
+4. You need to use your integration's authorization URL to set the necessary page permissions and send a request to obtain your Access Token. A thorough explanation of the necessary steps is provided in the [official Notion documentation](https://developers.notion.com/docs/authorization#public-integration-auth-flow-set-up). Once you have your Client ID, Client Secret and Access Token, you are ready to proceed to the next step.
 
 
+### Step 3: Set up the Notion connector in Airbyte
+
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Notion** from the list of available sources.
+4. Enter a **Source name** of your choosing.
+5. Choose the method of authentication from the dropdown menu:
+
+#### Authentication for Airbyte Cloud
+
+- **OAuth2.0** (Recommended): Click **Authenticate your Notion account**. When the popup appears, click **Select pages**. Check the pages you want to give Airbyte access to, and click **Allow access**.
+- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page.
+
+#### Authentication for Airbyte Open Source
+
+- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page.
+- **OAuth2.0**: Copy and paste the Client ID, Client Secret and Access Token you acquired.
+
+6. Enter the **Start Date** using the provided datepicker, or by programmatically entering a UTC date and time in the format: `YYYY-MM-DDTHH:mm:ss.SSSZ`. All data generated after this date will be replicated.
+7. Click **Set up source** and wait for the tests to complete.
+
 ## Supported sync modes
 
 The Notion source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) (partially)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) (partially)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
 The Notion source connector supports the following streams. For more information, see the [Notion API](https://developers.notion.com/reference/intro).
 
-* [blocks](https://developers.notion.com/reference/retrieve-a-block)
-* [databases](https://developers.notion.com/reference/retrieve-a-database)
-* [pages](https://developers.notion.com/reference/retrieve-a-page)
-* [users](https://developers.notion.com/reference/get-user)
+- [blocks](https://developers.notion.com/reference/retrieve-a-block)
+- [databases](https://developers.notion.com/reference/retrieve-a-database)
+- [pages](https://developers.notion.com/reference/retrieve-a-page)
+- [users](https://developers.notion.com/reference/get-user)
 
 :::note
 
@@ -82,23 +105,28 @@ The connector is restricted by Notion [request limits](https://developers.notion
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                         |
-|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------|
-| 1.0.6   | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream                           |
-| 1.0.5   | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream                          |
-| 1.0.4   | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search                          |
-| 1.0.3   | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification                      |
-| 1.0.2   | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error)                      |
-| 1.0.1   | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None`     |
-| 1.0.0   | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema                                       |
-| 0.1.10  | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff                            |
-| 0.1.9   | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                     |
-| 0.1.8   | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example    |
-| 0.1.7   | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas |
-| 0.1.6   | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec       |
-| 0.1.5   | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication                                   |
-| 0.1.4   | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through                 |
-| 0.1.3   | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream                                  |
-| 0.1.2   | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084)   | Fix documentation URL                                           |
-| 0.1.1   | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207)   | Update connector fields title/description                       |
-| 0.1.0   | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092)   | Initial Release                                                 |
+| Version | Date       | Pull Request                                             | Subject                                                                      |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------- |
+| 1.1.1   | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth`                |
+| 1.1.0   | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema                                                  |
+| 1.0.9   | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error                               |
+| 1.0.8   | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks`                               |
+| 1.0.7   | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream                                    |
+| 1.0.6   | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream                                        |
+| 1.0.5   | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream |
+| 1.0.4   | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search                                       |
+| 1.0.3   | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification                                   |
+| 1.0.2   | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error)                                   |
+| 1.0.1   | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None`                  |
+| 1.0.0   | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema                                                    |
+| 0.1.10  | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff                                         |
+| 0.1.9   | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                                  |
+| 0.1.8   | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example                 |
+| 0.1.7   | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas              |
+| 0.1.6   | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec                    |
+| 0.1.5   | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication                                                |
+| 0.1.4   | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through                              |
+| 0.1.3   | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream                                               |
+| 0.1.2   | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084)   | Fix documentation URL                                                        |
+| 0.1.1   | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207)   | Update connector fields title/description                                    |
+| 0.1.0   | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092)   | Initial Release                                                              |
diff --git a/docs/integrations/sources/okta.md b/docs/integrations/sources/okta.md
index b4c95ad5572b..b4da7c3b1a83 100644
--- a/docs/integrations/sources/okta.md
+++ b/docs/integrations/sources/okta.md
@@ -79,6 +79,8 @@ The connector is restricted by normal Okta [requests limitation](https://develop
 
 | Version | Date       | Pull Request                                             | Subject                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------|
+| 0.1.16  | 2023-07-07 | [20833](https://github.com/airbytehq/airbyte/pull/20833) | Fix infinite loop for GroupMembers stream                                      |
+| 0.1.15  | 2023-06-20 | [27533](https://github.com/airbytehq/airbyte/pull/27533) | Fixed group member stream and resource sets stream pagination                                                     |
 | 0.1.14  | 2022-12-24 | [20877](https://github.com/airbytehq/airbyte/pull/20877) | Disabled OAuth2.0 authorization method                                         |
 | 0.1.13  | 2022-08-12 | [14700](https://github.com/airbytehq/airbyte/pull/14700) | Add resource sets                                                              |
 | 0.1.12  | 2022-08-05 | [15050](https://github.com/airbytehq/airbyte/pull/15050) | Add parameter `start_date` for Logs stream                                     |
diff --git a/docs/integrations/sources/onesignal.md b/docs/integrations/sources/onesignal.md
index 1e0c708f8587..40e2ba112941 100644
--- a/docs/integrations/sources/onesignal.md
+++ b/docs/integrations/sources/onesignal.md
@@ -1,14 +1,15 @@
 # OneSignal
+
 This page contains the setup guide and reference information for the OneSignal source connector.
 
 ## Prerequisites
 
-* [User Auth Key](https://documentation.onesignal.com/docs/accounts-and-keys#user-auth-key)
-* Applications [credentials](https://documentation.onesignal.com/docs/accounts-and-keys) \(App Id & REST API Key\)
+- [User Auth Key](https://documentation.onesignal.com/docs/accounts-and-keys#user-auth-key)
+- Applications [credentials](https://documentation.onesignal.com/docs/accounts-and-keys) \(App Id & REST API Key\)
 
 ## Setup guide
-### Step 1: Set up OneSignal
 
+### Step 1: Set up OneSignal
 
 ### Step 2: Set up the OneSignal connector in Airbyte
 
@@ -26,7 +27,6 @@ This page contains the setup guide and reference information for the OneSignal s
 7. Enter the Start Date in format `YYYY-MM-DDTHH:mm:ssZ`
 8. Enter Outcome names as comma separated values, e.g. `os__session_duration.count,os__click.count,` see the [API docs](https://documentation.onesignal.com/reference/view-outcomes) for more details.
 
-
 #### For Airbyte Open Source:
 
 1. Navigate to the Airbyte Open Source dashboard.
@@ -41,22 +41,21 @@ This page contains the setup guide and reference information for the OneSignal s
 7. Enter the Start Date in format `YYYY-MM-DDTHH:mm:ssZ`
 8. Enter Outcome names as comma separated values, e.g. `os__session_duration.count,os__click.count,` see the [API docs](https://documentation.onesignal.com/reference/view-outcomes) for more details.
 
-
 ## Supported sync modes
 
 The OneSignal source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Apps](https://documentation.onesignal.com/reference/view-apps-apps)
-* [Devices](https://documentation.onesignal.com/reference/view-devices) \(Incremental\)
-* [Notifications](https://documentation.onesignal.com/reference/view-notification) \(Incremental\)
-* [Outcomes](https://documentation.onesignal.com/reference/view-outcomes)
+- [Apps](https://documentation.onesignal.com/reference/view-apps-apps)
+- [Devices](https://documentation.onesignal.com/reference/view-devices) \(Incremental\)
+- [Notifications](https://documentation.onesignal.com/reference/view-notification) \(Incremental\)
+- [Outcomes](https://documentation.onesignal.com/reference/view-outcomes)
 
 ## Performance considerations
 
@@ -65,20 +64,19 @@ The connector is restricted by normal OneSignal [rate limits](https://documentat
 ## Data type mapping
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `integer`        | `integer`    |       |
 | `number`         | `number`     |       |
 | `array`          | `array`      |       |
 | `object`         | `object`     |       |
 
-
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                      |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------- |
+| 1.0.1   | 2023-03-14 | [24076](https://github.com/airbytehq/airbyte/pull/24076) | Fix schema and add additionalProperties true |
 | 1.0.0   | 2023-03-14 | [24076](https://github.com/airbytehq/airbyte/pull/24076) | Update connectors spec; fix incremental sync |
 | 0.1.2   | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582)   | Update connector fields title/description    |
 | 0.1.1   | 2021-11-10 | [7617](https://github.com/airbytehq/airbyte/pull/7617)   | Fix get_update state                         |
 | 0.1.0   | 2021-10-13 | [6998](https://github.com/airbytehq/airbyte/pull/6998)   | Initial Release                              |
-
diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md
index 319e5763e941..1e81b7c73fed 100644
--- a/docs/integrations/sources/oracle.md
+++ b/docs/integrations/sources/oracle.md
@@ -2,17 +2,17 @@
 
 ## Features
 
-| Feature | Supported | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Replicate Incremental Deletes | Coming soon |  |
-| Logical Replication \(WAL\) | Coming soon |  |
-| SSL Support | Coming soon |  |
-| SSH Tunnel Connection | Yes |  |
-| LogMiner | Coming soon |  |
-| Flashback | Coming soon |  |
-| Namespaces | Yes | Enabled by default |
+| Feature                       | Supported   | Notes              |
+| :---------------------------- | :---------- | :----------------- |
+| Full Refresh Sync             | Yes         |                    |
+| Incremental - Append Sync     | Yes         |                    |
+| Replicate Incremental Deletes | Coming soon |                    |
+| Logical Replication \(WAL\)   | Coming soon |                    |
+| SSL Support                   | Coming soon |                    |
+| SSH Tunnel Connection         | Yes         |                    |
+| LogMiner                      | Coming soon |                    |
+| Flashback                     | Coming soon |                    |
+| Namespaces                    | Yes         | Enabled by default |
 
 The Oracle source does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details.
 
@@ -94,31 +94,31 @@ This produces the private key in pem format, and the public key remains in the s
 
 Oracle data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceComprehensiveTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test!
 
-| Oracle Type | Resulting Type | Notes |
-| :--- | :--- | :--- |
-| `binary_double` | number |  |
-| `binary_float` | number |  |
-| `blob` | string |  |
-| `char` | string |  |
-| `char(3 char)` | string |  |
-| `clob` | string |  |
-| `date` | string |  |
-| `decimal` | number |  |
-| `float` | number |  |
-| `float(5)` | number |  |
-| `integer` | number |  |
-| `interval year to month` | string |  |
-| `long raw` | string |  |
-| `number` | number |  |
-| `number(6, 2)` | number |  |
-| `nvarchar(3)` | string |  |
-| `raw` | string |  |
-| `timestamp` | string |  |
-| `timestamp with local time zone` | string |  |
-| `timestamp with time zone` | string |  |
-| `varchar2` | string |  |
-| `varchar2(256)` | string |  |
-| `xmltype` | string |  |
+| Oracle Type                      | Resulting Type | Notes |
+| :------------------------------- | :------------- | :---- |
+| `binary_double`                  | number         |       |
+| `binary_float`                   | number         |       |
+| `blob`                           | string         |       |
+| `char`                           | string         |       |
+| `char(3 char)`                   | string         |       |
+| `clob`                           | string         |       |
+| `date`                           | string         |       |
+| `decimal`                        | number         |       |
+| `float`                          | number         |       |
+| `float(5)`                       | number         |       |
+| `integer`                        | number         |       |
+| `interval year to month`         | string         |       |
+| `long raw`                       | string         |       |
+| `number`                         | number         |       |
+| `number(6, 2)`                   | number         |       |
+| `nvarchar(3)`                    | string         |       |
+| `raw`                            | string         |       |
+| `timestamp`                      | string         |       |
+| `timestamp with local time zone` | string         |       |
+| `timestamp with time zone`       | string         |       |
+| `varchar2`                       | string         |       |
+| `varchar2(256)`                  | string         |       |
+| `xmltype`                        | string         |       |
 
 If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings.
 
@@ -126,33 +126,35 @@ If you do not see a type in this list, assume that it is coerced into a string.
 
 Airbyte has the ability to connect to the Oracle source with 3 network connectivity options:
 
-1.`Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form. 2.`Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. In this case, the _SQLNET.ENCRYPTION\_CLIENT_ option will always be set as _REQUIRED_ by default: The client or server will only accept encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs. 3.`TLS Encrypted` \(verify certificate\) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification. To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field
+1.`Unencrypted` the connection will be made using the TCP protocol. In this case, all data over the network will be transmitted in unencrypted form. 2.`Native network encryption` gives you the ability to encrypt database connections, without the configuration overhead of TCP / IP and SSL / TLS and without the need to open and listen on different ports. In this case, the _SQLNET.ENCRYPTION_CLIENT_ option will always be set as _REQUIRED_ by default: The client or server will only accept encrypted traffic, but the user has the opportunity to choose an `Encryption algorithm` according to the security policies he needs. 3.`TLS Encrypted` \(verify certificate\) - if this option is selected, data transfer will be transfered using the TLS protocol, taking into account the handshake procedure and certificate verification. To use this option, insert the content of the certificate issued by the server into the `SSL PEM file` field
 
 ## Changelog
 
-| Version | Date       | Pull Request | Subject                                         |
-|:--------|:-----------| :--- |:------------------------------------------------|
-| 0.3.24  | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760)   | Removed redundant date-time datatypes formatting   |
-| 0.3.23  | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455)   | For network isolation, source connector accepts a list of hosts it is allowed to connect to |
-| 0.3.22  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)   | Consolidate date/time values mapping for JDBC sources                          |
+| Version | Date       | Pull Request                                             | Subject                                                                                                                                   |
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.4.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2                                                                                                                      |
+| 0.3.25  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                  |
+| 0.3.24  | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting                                                                                          |
+| 0.3.23  | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to                                               |
+| 0.3.22  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources                                                                                     |
 |         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
-| 0.3.21  | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
-| 0.3.20  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
-| 0.3.19  | 2022-08-03 | [14953](https://github.com/airbytehq/airbyte/pull/14953) | Use Service Name to connect to database |
-| 0.3.18  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors |
-| 0.3.17  | 2022-06-24 | [14092](https://github.com/airbytehq/airbyte/pull/14092) | Introduced a custom jdbc param field |
-| 0.3.16  | 2022-06-22 | [13997](https://github.com/airbytehq/airbyte/pull/13997) | Fixed tests |
-| 0.3.15  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |
-| 0.3.14  | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats |
-| 0.3.13  | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds |
-| 0.3.12  | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
-| 0.3.11  | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY |
-| 0.3.10  | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description |
-| 0.3.9   | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key      |
-| 0.3.8   | 2021-10-13 | [7125](https://github.com/airbytehq/airbyte/pull/7125) | Fix incorrect handling of LONG RAW data type    |
-| 0.3.7   | 2021-10-01 | [6616](https://github.com/airbytehq/airbyte/pull/6616) | Added network encryption options                |
-| 0.3.6   | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585) | Improved SSH Tunnel key generation steps        |
-| 0.3.5   | 2021-09-22 | [6356](https://github.com/airbytehq/airbyte/pull/6356) | Added option to connect to DB via SSH.          |
-| 0.3.4   | 2021-09-01 | [6038](https://github.com/airbytehq/airbyte/pull/6038) | Remove automatic filtering of system schemas.   |
-| 0.3.3   | 2021-09-01 | [5779](https://github.com/airbytehq/airbyte/pull/5779) | Ability to only discover certain schemas.       |
-| 0.3.2   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator.                    |
+| 0.3.21  | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently                                                                                                       |
+| 0.3.20  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field                                 |
+| 0.3.19  | 2022-08-03 | [14953](https://github.com/airbytehq/airbyte/pull/14953) | Use Service Name to connect to database                                                                                                   |
+| 0.3.18  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors                                                                            |
+| 0.3.17  | 2022-06-24 | [14092](https://github.com/airbytehq/airbyte/pull/14092) | Introduced a custom jdbc param field                                                                                                      |
+| 0.3.16  | 2022-06-22 | [13997](https://github.com/airbytehq/airbyte/pull/13997) | Fixed tests                                                                                                                               |
+| 0.3.15  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption                                                                 |
+| 0.3.14  | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats                                    |
+| 0.3.13  | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds                                                                                        |
+| 0.3.12  | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                                              |
+| 0.3.11  | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958)   | Add support for JdbcType.ARRAY                                                                                                            |
+| 0.3.10  | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582)   | Update connector fields title/description                                                                                                 |
+| 0.3.9   | 2021-12-01 | [8371](https://github.com/airbytehq/airbyte/pull/8371)   | Fixed incorrect handling "\n" in ssh key                                                                                                  |
+| 0.3.8   | 2021-10-13 | [7125](https://github.com/airbytehq/airbyte/pull/7125)   | Fix incorrect handling of LONG RAW data type                                                                                              |
+| 0.3.7   | 2021-10-01 | [6616](https://github.com/airbytehq/airbyte/pull/6616)   | Added network encryption options                                                                                                          |
+| 0.3.6   | 2021-09-30 | [6585](https://github.com/airbytehq/airbyte/pull/6585)   | Improved SSH Tunnel key generation steps                                                                                                  |
+| 0.3.5   | 2021-09-22 | [6356](https://github.com/airbytehq/airbyte/pull/6356)   | Added option to connect to DB via SSH.                                                                                                    |
+| 0.3.4   | 2021-09-01 | [6038](https://github.com/airbytehq/airbyte/pull/6038)   | Remove automatic filtering of system schemas.                                                                                             |
+| 0.3.3   | 2021-09-01 | [5779](https://github.com/airbytehq/airbyte/pull/5779)   | Ability to only discover certain schemas.                                                                                                 |
+| 0.3.2   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699)   | Added json config validator.                                                                                                              |
diff --git a/docs/integrations/sources/orb.md b/docs/integrations/sources/orb.md
index bdb98098cb94..aa219864f4d3 100644
--- a/docs/integrations/sources/orb.md
+++ b/docs/integrations/sources/orb.md
@@ -9,11 +9,11 @@ will only read and output new records based on their `created_at` timestamp.
 
 This Source is capable of syncing the following core resources, each of which has a separate Stream. Note that all of the streams are incremental:
 
-* [Subscriptions](https://docs.withorb.com/docs/orb-docs/api-reference/operations/list-subscriptions)
-* [Plans](https://docs.withorb.com/docs/orb-docs/api-reference/operations/list-plans)
-* [Customers](https://docs.withorb.com/docs/orb-docs/api-reference/operations/list-customers)
-* [Credits Ledger Entries](https://docs.withorb.com/docs/orb-docs/api-reference/operations/get-a-customer-credit-ledger)
-* [Subscription Usage](https://docs.withorb.com/docs/orb-docs/api-reference/operations/get-a-subscription-usage)
+* [Subscriptions](https://docs.withorb.com/reference/list-subscriptions)
+* [Plans](https://docs.withorb.com/reference/list-plans)
+* [Customers](https://docs.withorb.com/reference/list-customers)
+* [Credits Ledger Entries](https://docs.withorb.com/reference/fetch-customer-credits-ledger)
+* [Subscription Usage](https://docs.withorb.com/reference/fetch-subscription-usage)
 
 As a caveat, the Credits Ledger Entries must read all Customers for an incremental sync, but will only incrementally return new ledger entries for each customers.
 
diff --git a/docs/integrations/sources/outbrain-amplify.md b/docs/integrations/sources/outbrain-amplify.md
new file mode 100644
index 000000000000..3ebe25b2b8f1
--- /dev/null
+++ b/docs/integrations/sources/outbrain-amplify.md
@@ -0,0 +1,67 @@
+# Outbrain Amplify
+
+## Sync overview
+
+This source can sync data for the [Outbrain Amplify API](https://amplifyv01.docs.apiary.io/#reference/authentications). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run.
+
+### Output schema
+
+This Source is capable of syncing the following core Streams:
+
+* marketers stream.
+* campaigns by marketers stream.-Incremental
+* campaigns geo-location stream.
+* promoted links for campaigns stream.
+* promoted links sequence for campaigns stream.
+* budgets for marketers stream.
+* performance report campaigns by marketers stream.
+* performance report periodic by marketers stream.
+* performance report periodic by marketers campaign stream.
+* performance report periodic content by promoted links campaign stream.
+* performance report marketers by publisher stream.
+* performance report publishers by campaigns stream.
+* performance report marketers by platforms stream.
+* performance report marketers campaigns by platforms stream.
+* performance report marketers by geo performance stream.
+* performance report marketers campaigns by geo stream.
+* performance report marketers by Interest stream.
+
+### Data type mapping
+
+| Integration Type | Airbyte Type | Notes |
+| :--- | :--- | :--- |
+| `string` | `string` |  |
+| `integer` | `integer` |  |
+| `number` | `number` |  |
+| `array` | `array` |  |
+| `object` | `object` |  |
+
+### Features
+
+| Feature | Supported?\(Yes/No\) | Notes |
+| :--- | :--- | :--- |
+| Full Refresh Sync | Yes |  |
+| Incremental Sync | Yes |  |
+| Namespaces | No |  |
+
+### Performance considerations
+
+The Outbrain Amplify connector should not run into Outbrain Amplify API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully.
+
+## Getting started
+
+### Requirements
+
+* Credentials and start-date.
+
+### Setup guide
+
+Specify credentials and a start date.
+
+## Changelog
+
+| Version | Date | Pull Request | Subject |
+| :--- | :--- | :--- | :--- |
+| 0.1.2 | 2022-08-25 | [15667](https://github.com/airbytehq/airbyte/pull/15667) | Add message when no data available |
+| 0.1.1 | 2022-05-30 | [11732](https://github.com/airbytehq/airbyte/pull/11732) | Fix docs |
+| 0.1.0 | 2022-05-30 | [11732](https://github.com/airbytehq/airbyte/pull/11732) | Initial Release |
diff --git a/docs/integrations/sources/outreach.md b/docs/integrations/sources/outreach.md
index ac0786555067..631ef791998c 100644
--- a/docs/integrations/sources/outreach.md
+++ b/docs/integrations/sources/outreach.md
@@ -44,12 +44,16 @@ List of available streams:
 - Personas
 - Mailboxes
 - Stages
+- Users
+- Tasks
+- Templates
+- Snippets
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-
-| :------ | :-------- | :----- | :------ |
+| Version | Date       | Pull Request | Subject |
+| :------ |:-----------| :----- | :------ |
+| 0.4.0 | 2023-06-14 | [27343](https://github.com/airbytehq/airbyte/pull/27343) | Add Users, Tasks, Templates, Snippets streams
 | 0.3.0 | 2023-05-17 | [26211](https://github.com/airbytehq/airbyte/pull/26211) | Add SequenceStates Stream
 | 0.2.0 | 2022-10-27 | [17385](https://github.com/airbytehq/airbyte/pull/17385) | Add new streams + page size variable + relationship data |
 | 0.1.2 | 2022-07-04 | [14386](https://github.com/airbytehq/airbyte/pull/14386) | Fix stream schema and cursor field |
diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md
index 6f2079344afc..85277858c465 100644
--- a/docs/integrations/sources/paypal-transaction.md
+++ b/docs/integrations/sources/paypal-transaction.md
@@ -87,7 +87,9 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                      |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------|
-| 0.1.13  | 2023-02-20 | [22916](https://github.com/airbytehq/airbyte/pull/22916) | Specified date formatting in specification                                                      |
+| 2.0.0   | 2023-07-05 | [27916](https://github.com/airbytehq/airbyte/pull/27916) | Update `Balances` schema                                                                                                     |
+| 1.0.0   | 2023-07-03 | [27968](https://github.com/airbytehq/airbyte/pull/27968) | mark `Client ID` and `Client Secret` as required fields                                                                      |
+| 0.1.13  | 2023-02-20 | [22916](https://github.com/airbytehq/airbyte/pull/22916) | Specified date formatting in specification                                                                                   |
 | 0.1.12  | 2023-02-18 | [23211](https://github.com/airbytehq/airbyte/pull/23211) | Fix error handler                                                                                                            |
 | 0.1.11  | 2023-01-27 | [22019](https://github.com/airbytehq/airbyte/pull/22019) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                  |
 | 0.1.10  | 2022-09-04 | [17554](https://github.com/airbytehq/airbyte/pull/17554) | Made the spec and source config to be consistent                                                                             |
diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md
index d08817482255..a5ddc684e83c 100644
--- a/docs/integrations/sources/pinterest.md
+++ b/docs/integrations/sources/pinterest.md
@@ -9,6 +9,7 @@ To set up the Pinterest source connector with Airbyte Open Source, you'll need y
 ## Setup guide
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -21,6 +22,7 @@ To set up the Pinterest source connector with Airbyte Open Source, you'll need y
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Navigate to the Airbyte Open Source dashboard.
@@ -36,32 +38,33 @@ To set up the Pinterest source connector with Airbyte Open Source, you'll need y
 
 The Pinterest source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Account analytics](https://developers.pinterest.com/docs/api/v5/#operation/user_account/analytics) \(Incremental\)
-* [Boards](https://developers.pinterest.com/docs/api/v5/#operation/boards/list) \(Full table\)
-  * [Board sections](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list) \(Full table\)
-    * [Pins on board section](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list_pins) \(Full table\)
-  * [Pins on board](https://developers.pinterest.com/docs/api/v5/#operation/boards/list_pins) \(Full table\)
-* [Ad accounts](https://developers.pinterest.com/docs/api/v5/#operation/ad_accounts/list) \(Full table\)
-  * [Ad account analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_account/analytics) \(Incremental\)
-  * [Campaigns](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\)
-    * [Campaign analytics](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\)
-  * [Ad groups](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/list) \(Incremental\)
-    * [Ad group analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\)
-  * [Ads](https://developers.pinterest.com/docs/api/v5/#operation/ads/list) \(Incremental\)
-    * [Ad analytics](https://developers.pinterest.com/docs/api/v5/#operation/ads/analytics) \(Incremental\)
+- [Account analytics](https://developers.pinterest.com/docs/api/v5/#operation/user_account/analytics) \(Incremental\)
+- [Boards](https://developers.pinterest.com/docs/api/v5/#operation/boards/list) \(Full table\)
+  - [Board sections](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list) \(Full table\)
+    - [Pins on board section](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list_pins) \(Full table\)
+  - [Pins on board](https://developers.pinterest.com/docs/api/v5/#operation/boards/list_pins) \(Full table\)
+- [Ad accounts](https://developers.pinterest.com/docs/api/v5/#operation/ad_accounts/list) \(Full table\)
+  - [Ad account analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_account/analytics) \(Incremental\)
+  - [Campaigns](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\)
+    - [Campaign analytics](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\)
+  - [Campaign Analytics Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\)
+  - [Ad groups](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/list) \(Incremental\)
+    - [Ad group analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\)
+  - [Ads](https://developers.pinterest.com/docs/api/v5/#operation/ads/list) \(Incremental\)
+    - [Ad analytics](https://developers.pinterest.com/docs/api/v5/#operation/ads/analytics) \(Incremental\)
 
 ## Performance considerations
 
 The connector is restricted by the Pinterest [requests limitation](https://developers.pinterest.com/docs/api/v5/#tag/Rate-limits).
 
-#####  Rate Limits
+##### Rate Limits
 
 - Analytics streams: 300 calls per day / per user \
 - Ad accounts streams (Campaigns, Ad groups, Ads): 1000 calls per min / per user / per app \
@@ -69,27 +72,29 @@ The connector is restricted by the Pinterest [requests limitation](https://devel
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                                                 |
-|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------|
-| 0.5.2   | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property                                                         |
-| 0.5.1   | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date                                                                              |
-| 0.5.0   | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream                                                      |
-| 0.4.0   | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream                                                       |
-| 0.3.0   | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream                                                     |
-| 0.2.6   | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fixed `format` issue for `boards` stream schema for fields with `date-time`                             |
-| 0.2.5   | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days                                                   |
-| 0.2.4   | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Adding missing columns for analytics streams for pinterest source                                       |
-| 0.2.3   | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days                                        |
-| 0.2.2   | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None`                                             |
-| 0.2.1   | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version                                                                                        |
-| 0.2.0   | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Added data-type normalization up to the schemas declared                                                |
-| 0.1.9   | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Added filter based on statuses                                                                          |
-| 0.1.8   | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date`                                                                                |
-| 0.1.7   | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions.                                                 |
-| 0.1.6   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89                                                                                          |
-| 0.1.5   | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                                                             |
-| 0.1.4   | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Added ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` |
-| 0.1.3   | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Added support of `OAuth2.0` authentication method                                                       |
-| 0.1.2   | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields                                                        |
-| 0.1.1   | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043)   | Update connector fields title/description                                                               |
-| 0.1.0   | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493)   | Release Pinterest CDK Connector                                                                         |
+| Version | Date       | Pull Request                                             | Subject                                                                                               |
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------- |
+| 0.6.0   | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level                                                                |
+| 0.5.3   | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream                                               |
+| 0.5.2   | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property                                                        |
+| 0.5.1   | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date                                                                            |
+| 0.5.0   | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream                                                    |
+| 0.4.0   | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream                                                     |
+| 0.3.0   | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream                                                   |
+| 0.2.6   | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time`                             |
+| 0.2.5   | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days                                                 |
+| 0.2.4   | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source                                        |
+| 0.2.3   | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days                                      |
+| 0.2.2   | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None`                                           |
+| 0.2.1   | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version                                                                                      |
+| 0.2.0   | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared                                                |
+| 0.1.9   | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses                                                                          |
+| 0.1.8   | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date`                                                                              |
+| 0.1.7   | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions.                                               |
+| 0.1.6   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89                                                                                        |
+| 0.1.5   | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                                                           |
+| 0.1.4   | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` |
+| 0.1.3   | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method                                                       |
+| 0.1.2   | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields                                                      |
+| 0.1.1   | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043)   | Update connector fields title/description                                                             |
+| 0.1.0   | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493)   | Release Pinterest CDK Connector                                                                       |
diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md
index c021f2e635e0..534f992294fb 100644
--- a/docs/integrations/sources/pipedrive.md
+++ b/docs/integrations/sources/pipedrive.md
@@ -110,23 +110,25 @@ The Pipedrive connector will gracefully handle rate limits. For more information
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                        |
-| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------- |
-| 0.1.18  | 2023-06-02 | [26892](https://github.com/airbytehq/airbyte/pull/26892) | Update `DialFields` schema with `pipeline_ids` property        |
-| 0.1.17  | 2023-03-21 | [24282](https://github.com/airbytehq/airbyte/pull/24282) | Bugfix handle missed `cursor_field`                            |
-| 0.1.16  | 2023-03-08 | [23789](https://github.com/airbytehq/airbyte/pull/23789) | Add 11 new streams                                             |
-| 0.1.15  | 2023-03-02 | [23705](https://github.com/airbytehq/airbyte/pull/23705) | Disable OAuth                                                  |
-| 0.1.14  | 2023-03-01 | [23539](https://github.com/airbytehq/airbyte/pull/23539) | Fix schema for "activities", "check" works if empty "deals"    |
-| 0.1.13  | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                    |
-| 0.1.12  | 2022-05-12 | [12806](https://github.com/airbytehq/airbyte/pull/12806) | Remove date-time format from schemas                           |
-| 0.1.10  | 2022-04-26 | [11870](https://github.com/airbytehq/airbyte/pull/11870) | Add 3 streams: DealFields, OrganizationFields and PersonFields |
-| 0.1.9   | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582)   | Update connector fields title/description                      |
-| 0.1.8   | 2021-11-16 | [7875](https://github.com/airbytehq/airbyte/pull/7875)   | Extend schema for "persons" stream                             |
-| 0.1.7   | 2021-11-15 | [7968](https://github.com/airbytehq/airbyte/pull/7968)   | Update oAuth flow config                                       |
-| 0.1.6   | 2021-10-05 | [6821](https://github.com/airbytehq/airbyte/pull/6821)   | Add OAuth support                                              |
-| 0.1.5   | 2021-09-27 | [6441](https://github.com/airbytehq/airbyte/pull/6441)   | Fix normalization error                                        |
-| 0.1.4   | 2021-08-26 | [5943](https://github.com/airbytehq/airbyte/pull/5943)   | Add organizations stream                                       |
-| 0.1.3   | 2021-08-26 | [5642](https://github.com/airbytehq/airbyte/pull/5642)   | Remove date-time from deals stream                             |
-| 0.1.2   | 2021-07-23 | [4912](https://github.com/airbytehq/airbyte/pull/4912)   | Update money type to support floating point                    |
-| 0.1.1   | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686)   | Update spec.json                                               |
-| 0.1.0   | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686)   | 🎉 New source: Pipedrive connector                             |
+| Version | Date       | Pull Request                                             | Subject                                                                    |
+|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------|
+| 1.0.0   | 2023-06-29 | [27832](https://github.com/airbytehq/airbyte/pull/27832) | Remove `followers_count` field from `Products` stream                      |
+| 0.1.19  | 2023-07-05 | [27967](https://github.com/airbytehq/airbyte/pull/27967) | Update `OrganizationFields` and `ProductFields` with `display_field` field |
+| 0.1.18  | 2023-06-02 | [26892](https://github.com/airbytehq/airbyte/pull/26892) | Update `DialFields` schema with `pipeline_ids` property                    |
+| 0.1.17  | 2023-03-21 | [24282](https://github.com/airbytehq/airbyte/pull/24282) | Bugfix handle missed `cursor_field`                                        |
+| 0.1.16  | 2023-03-08 | [23789](https://github.com/airbytehq/airbyte/pull/23789) | Add 11 new streams                                                         |
+| 0.1.15  | 2023-03-02 | [23705](https://github.com/airbytehq/airbyte/pull/23705) | Disable OAuth                                                              |
+| 0.1.14  | 2023-03-01 | [23539](https://github.com/airbytehq/airbyte/pull/23539) | Fix schema for "activities", "check" works if empty "deals"                |
+| 0.1.13  | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state                                                |
+| 0.1.12  | 2022-05-12 | [12806](https://github.com/airbytehq/airbyte/pull/12806) | Remove date-time format from schemas                                       |
+| 0.1.10  | 2022-04-26 | [11870](https://github.com/airbytehq/airbyte/pull/11870) | Add 3 streams: DealFields, OrganizationFields and PersonFields             |
+| 0.1.9   | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582)   | Update connector fields title/description                                  |
+| 0.1.8   | 2021-11-16 | [7875](https://github.com/airbytehq/airbyte/pull/7875)   | Extend schema for "persons" stream                                         |
+| 0.1.7   | 2021-11-15 | [7968](https://github.com/airbytehq/airbyte/pull/7968)   | Update oAuth flow config                                                   |
+| 0.1.6   | 2021-10-05 | [6821](https://github.com/airbytehq/airbyte/pull/6821)   | Add OAuth support                                                          |
+| 0.1.5   | 2021-09-27 | [6441](https://github.com/airbytehq/airbyte/pull/6441)   | Fix normalization error                                                    |
+| 0.1.4   | 2021-08-26 | [5943](https://github.com/airbytehq/airbyte/pull/5943)   | Add organizations stream                                                   |
+| 0.1.3   | 2021-08-26 | [5642](https://github.com/airbytehq/airbyte/pull/5642)   | Remove date-time from deals stream                                         |
+| 0.1.2   | 2021-07-23 | [4912](https://github.com/airbytehq/airbyte/pull/4912)   | Update money type to support floating point                                |
+| 0.1.1   | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686)   | Update spec.json                                                           |
+| 0.1.0   | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686)   | 🎉 New source: Pipedrive connector                                         |
diff --git a/docs/integrations/sources/plaid.md b/docs/integrations/sources/plaid.md
index 889ffb39ea84..eab40b8c6196 100644
--- a/docs/integrations/sources/plaid.md
+++ b/docs/integrations/sources/plaid.md
@@ -47,7 +47,7 @@ This guide will walk through how to create the credentials you need to run this
           --data-raw '{
               "client_id": "",
               "secret": "",
-              "institution_id": "ins_43",
+              "institution_id": "ins_127287",
               "initial_products": ["auth", "transactions"]
           }'
     ```
@@ -70,6 +70,7 @@ This guide will walk through how to create the credentials you need to run this
 
 | Version | Date       | Pull Request                                             | Subject                                                       |
 | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------ |
+| 0.4.0   | 2023-08-17 | [29127](https://github.com/airbytehq/airbyte/pull/29127) | Rewrote connector to no-code SDK                              |
 | 0.3.2   | 2022-08-02 | [15231](https://github.com/airbytehq/airbyte/pull/15231) | Added min_last_updated_datetime support for Capital One items |
 | 0.3.1   | 2022-03-31 | [11104](https://github.com/airbytehq/airbyte/pull/11104) | Fix 100 record limit and added start_date                     |
 | 0.3.0   | 2022-01-05 | [7977](https://github.com/airbytehq/airbyte/pull/7977)   | Migrate to Python CDK + add transaction stream                |
diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md
index 5d5acce90a9e..21819531de08 100644
--- a/docs/integrations/sources/postgres.md
+++ b/docs/integrations/sources/postgres.md
@@ -1,237 +1,157 @@
 # Postgres
 
-This page contains the setup guide and reference information for the Postgres source connector for CDC and non-CDC workflows.
+Airbyte's certified Postgres connector offers the following features:
+* Multiple methods of keeping your data fresh, including [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) and replication using the [xmin system column](#xmin). 
+* All available [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes), providing flexibility in how data is delivered to your destination.
+* Reliable replication at any table size with [checkpointing](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#state--checkpointing) and chunking of database reads.
 
-## When to use Postgres with CDC
+The contents below include a 'Quick Start' guide, advanced setup steps, and reference information (data type mapping, and changelogs). See [here](https://docs.airbyte.com/integrations/sources/postgres/postgres-troubleshooting) to troubleshooting issues with the Postgres connector.
 
-Configure Postgres with CDC if:
+![Airbyte Postgres Connection](https://raw.githubusercontent.com/airbytehq/airbyte/c078e8ed6703020a584d9362efa5665fbe8db77f/docs/integrations/sources/postgres/assets/airbyte_postgres_source.png?raw=true)
 
-- You need a record of deletions
-- Your table has a primary key but doesn't have a reasonable cursor field for incremental syncing (`updated_at`). CDC allows you to sync your table incrementally
+## Quick Start
 
-If your goal is to maintain a snapshot of your table in the destination but the limitations prevent you from using CDC, consider using [non-CDC incremental sync](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) and occasionally reset the data and re-sync.
+Here is an outline of the minimum required steps to configure a Postgres connector:
+1. Create a dedicated read-only Postgres user with permissions for replicating data
+2. Create a new Postgres source in the Airbyte UI using `xmin` system column
+3. (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs
 
-If your dataset is small and you just want a snapshot of your table in the destination, consider using [Full Refresh replication](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite) for your table instead of CDC.
+Once this is complete, you will be able to select Postgres as a source for replicating data.
 
-## Prerequisites
+#### Step 1: Create a dedicated read-only Postgres user
 
-- For Airbyte Open Source users, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer
-- Use Postgres v9.3.x or above for non-CDC workflows and Postgres v10 or above for CDC workflows
-- For Airbyte Cloud (and optionally for Airbyte Open Source), ensure SSL is enabled in your environment
+These steps create a dedicated read-only user for replicating data. Alternatively, you can use an existing Postgres user in your database.
 
-## Setup guide
+The following commands will create a new user:
 
-### Step 1: (Optional) Create a dedicated read-only user
-
-We recommend creating a dedicated read-only user for better permission control and auditing. Alternatively, you can use an existing Postgres user in your database.
-
-To create a dedicated user, run the following command:
-
-```
+```roomsql
 CREATE USER  PASSWORD 'your_password_here';
 ```
 
-Grant access to the relevant schema:
-
-```
-GRANT USAGE ON SCHEMA  TO 
-```
-
-:::note
-To replicate data from multiple Postgres schemas, re-run the command to grant access to all the relevant schemas. Note that you'll need to set up multiple Airbyte sources connecting to the same Postgres database on multiple schemas.
-:::
-
-Grant the user read-only access to the relevant tables:
+Now, provide this user with read-only access to relevant schemas and tables. Re-run this command for each schema you expect to replicate data from:
 
-```
+```roomsql
+GRANT USAGE ON SCHEMA  TO ;
 GRANT SELECT ON ALL TABLES IN SCHEMA  TO ;
-```
-
-Allow user to see tables created in the future:
-
-```
 ALTER DEFAULT PRIVILEGES IN SCHEMA  GRANT SELECT ON TABLES TO ;
 ```
 
-Additionally, if you plan to configure CDC for the Postgres source connector, grant `REPLICATION` permissions to the user:
-
-```
-ALTER USER  REPLICATION;
-```
+#### Step 2: Create a new Postgres source in Airbyte UI
 
+From your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account, select `Sources` from the left navigation bar, search for `Postgres`, then create a new Postgres source.
 
-**Syncing a subset of columns​**
+![Create an Airbyte source](https://github.com/airbytehq/airbyte/blob/c078e8ed6703020a584d9362efa5665fbe8db77f/docs/integrations/sources/postgres/assets/airbyte_source_selection.png?raw=true)
 
-Currently, there is no way to sync a subset of columns using the Postgres source connector:
+To fill out the required information:
+1. Enter the hostname, port number, and name for your Postgres database.
+2. You may optionally opt to list each of the schemas you want to sync. These are case-sensitive, and multiple schemas may be entered. By default, `public` is the only selected schema.
+3. Enter the username and password you created in [Step 1](#step-1-create-a-dedicated-read-only-postgres-user).
+4. Select an SSL mode. You will most frequently choose `require` or `verify-ca`. Both of these always require encryption. `verify-ca` also requires certificates from your Postgres database. See here to learn about other SSL modes and SSH tunneling.
+5. Select `Standard (xmin)` from available replication methods. This uses the [xmin system column](#xmin) to reliably replicate data from your database.
+   1. If your database is particularly large (> 500 GB), you will benefit from [configuring your Postgres source using logical replication (CDC)](#cdc).
 
-- When setting up a connection, you can only choose which tables to sync, but not columns.
-- If the user can only access a subset of columns, the connection check will pass. However, the data sync will fail with a permission denied exception.
+#### Step 3: (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs.
 
-The workaround for partial table syncing is to create a view on the specific columns, and grant the user read access to that view:
-
-```
-CREATE VIEW  as SELECT  FROM 
    ;
-```
+If you are on Airbyte Cloud, you will always need to modify your database configuration to allow inbound traffic from Airbyte IPs. These are:
 
+```roomsql
+34.106.109.131
+34.106.196.165
+34.106.60.246
+34.106.229.69
+34.106.127.139
+34.106.218.58
+34.106.115.240
+34.106.225.141
+13.37.4.46
+13.37.142.60
+35.181.124.238
 ```
-GRANT SELECT ON TABLE  IN SCHEMA  to ;
-```
-
-**Note:** The workaround works only for non-CDC setups since CDC requires data to be in tables and not views.
-This issue is tracked in [#9771](https://github.com/airbytehq/airbyte/issues/9771).
-
-### Step 2: Set up the Postgres connector in Airbyte
-
-1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Postgres** from the Source type dropdown.
-4. Enter a name for your source.
-5. For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your Postgres database.
-6. List the **Schemas** you want to sync.
-   :::note
-   The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing.
-   :::
-7. For **User** and **Password**, enter the username and password you created in [Step 1](#step-1-optional-create-a-dedicated-read-only-user).
-8. To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://jdbc.postgresql.org/documentation/head/connect.html) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field.
-
-   Example: key1=value1&key2=value2&key3=value3
-
-   These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your Postgres database.
-
-   The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available.
-
-   **Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte:
-   `currentSchema`, `user`, `password`, `ssl`, and `sslmode`.
-
-   :::warning
-   This is an advanced configuration option. Users are advised to use it with caution.
-   :::
-
-9. For Airbyte Open Source, toggle the switch to connect using SSL. For Airbyte Cloud uses SSL by default.
-10. For SSL Modes, select:
-    - **disable** to disable encrypted communication between Airbyte and the source
-    - **allow** to enable encrypted communication only when required by the source
-    - **prefer** to allow unencrypted communication only when the source doesn't support encryption
-    - **require** to always require encryption. Note: The connection will fail if the source doesn't support encryption.
-    - **verify-ca** to always require encryption and verify that the source has a valid SSL certificate
-    - **verify-full** to always require encryption and verify the identity of the source
-11. For Replication Method, select Standard or [Logical CDC](https://www.postgresql.org/docs/10/logical-replication.html) from the dropdown. Refer to [Configuring Postgres connector with Change Data Capture (CDC)](#configuring-postgres-connector-with-change-data-capture-cdc) for more information.
-12. For SSH Tunnel Method, select:
 
-    - **No Tunnel** for a direct connection to the database
-    - **SSH Key Authentication** to use an RSA Private as your secret for establishing the SSH tunnel
-    - **Password Authentication** to use a password as your secret for establishing the SSH tunnel
+Now, click `Set up source` in the Airbyte UI. Airbyte will now test connecting to your database. Once this succeeds, you've configured an Airbyte Postgres source!
 
-    :::warning
-    Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **disable**, **allow**, or **prefer** as the **SSL Mode**; otherwise, the connection will fail.
-    :::
+## Advanced Configuration
 
-Refer to [Connect via SSH Tunnel](#connect-via-ssh-tunnel​) for more information. 13. Click **Set up source**.
+### Setup using CDC
 
-### Connect via SSH Tunnel​
+Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-replication.html) of the Postgres write-ahead log (WAL) to incrementally capture deletes using a replication plugin:
+* See [here](https://docs.airbyte.com/understanding-airbyte/cdc) to learn more on how Airbyte implements CDC.
+* See [here](https://docs.airbyte.com/integrations/sources/postgres/postgres-troubleshooting#cdc-requirements) to learn more about Postgres CDC requirements and limitations.
 
-You can connect to a Postgres instance via an SSH tunnel.
+We recommend configuring your Postgres source with CDC when:
+- You need a record of deletions.
+- You have a very large database (500 GB or more).
+- Your table has a primary key but doesn't have a reasonable cursor field for incremental syncing (`updated_at`).
 
-When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server (also called a bastion or a jump server) that has direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server.
+These are the additional steps required (after following the [quick start](#quick-start)) to configure your Postgres source using CDC:
+1. Provide additional `REPLICATION` permissions to read-only user
+2. Enable logical replication on your Postgres database
+3. Create a replication slot on your Postgres database
+4. Create publication and replication identities for each Postgres table
+5. Enable CDC replication in the Airbyte UI
 
-To connect to a Postgres instance via an SSH tunnel:
+#### Step 1: Prepopulate your Postgres source configuration
 
-1. While [setting up](#setup-guide) the Postgres source connector, from the SSH tunnel dropdown, select:
-   - SSH Key Authentication to use a private as your secret for establishing the SSH tunnel
-   - Password Authentication to use a password as your secret for establishing the SSH Tunnel
-2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to.
-3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22.
-4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the Postgres username.
-5. For authentication:
-   - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [private Key](#generating-a-private-key​) that you are using to create the SSH connection.
-   - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the Postgres password.
+We recommend following the steps in the [quick start](#quick-start) section to confirm that Airbyte can connect to your Postgres database prior to configuring CDC settings.
 
-#### Generating a private Key​
+For CDC, you must connect to primary/master databases. Pointing the connector configuration to replica database hosts for CDC will lead to failures.
 
-The connector supports any SSH compatible key format such as RSA or Ed25519. To generate an RSA key, for example, run:
+#### Step 2: Provide additional permissions to read-only user
 
+To configure CDC for the Postgres source connector, grant `REPLICATION` permissions to the user created in [step 1 of the quick start](#step-1-create-a-dedicated-read-only-postgres-user):
 ```
-ssh-keygen -t rsa -m PEM -f myuser_rsa
+ALTER USER  REPLICATION;
 ```
 
-The command produces the private key in PEM format and the public key remains in the standard format used by the `authorized_keys` file on your bastion server. Add the public key to your bastion host to the user you want to use with Airbyte. The private key is provided via copy-and-paste to the Airbyte connector configuration screen to allow it to log into the bastion server.
-
-## Configuring Postgres connector with Change Data Capture (CDC)
-
-Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-replication.html) of the Postgres write-ahead log (WAL) to incrementally capture deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/)
-
-### CDC Considerations
-
-- Incremental sync is only supported for tables with primary keys. For tables without primary keys, use [Full Refresh sync](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite).
-- Data must be in tables and not views. If you require data synchronization from a view, you would need to create a new connection with `Standard` as `Replication Method`.
-- The modifications you want to capture must be made using `DELETE`/`INSERT`/`UPDATE`. For example, changes made using `TRUNCATE`/`ALTER` will not appear in logs and therefore in your destination.
-- Schema changes are not supported automatically for CDC sources. Reset and resync data if you make a schema change.
-- The records produced by `DELETE` statements only contain primary keys. All other data fields are unset.
-- Log-based replication only works for master instances of Postgres.  CDC cannot be run from a read-replica of your primary database.
-- Using logical replication increases disk space used on the database server. The additional data is stored until it is consumed.
-  - Set frequent syncs for CDC to ensure that the data doesn't fill up your disk space.
-  - If you stop syncing a CDC-configured Postgres instance with Airbyte, delete the replication slot. Otherwise, it may fill up your disk space.
-
-:::note Connector configuration are supported only on primary/master db host/servers. Do not point connector configuration to replica db hosts, it will not work.. :::
-
-### Setting up CDC for Postgres​
+#### Step 3: Enable logical replication on your Postgres database
 
-Airbyte requires a replication slot configured only for its use. Only one source should be configured that uses this replication slot. See Setting up CDC for Postgres for instructions.
+To enable logical replication on bare metal, VMs (EC2/GCE/etc), or Docker, configure the following parameters in the postgresql.conf file for your Postgres database:
 
-#### Step 1: Enable logical replication​
+| Parameter             | Description                                                                    | Set value to                                                                                                                         |
+|-----------------------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------|
+| wal_level             | Type of coding used within the Postgres write-ahead log                        | `logical `                                                                                                                           |
+| max_wal_senders       | The maximum number of processes used for handling WAL changes                  | `min: 1`                                                                                                                             |
+| max_replication_slots | The maximum number of replication slots that are allowed to stream WAL changes | `1` (if Airbyte is the only service reading subscribing to WAL changes. More than 1 if other services are also reading from the WAL) |
 
-To enable logical replication on bare metal, VMs (EC2/GCE/etc), or Docker, configure the following parameters in the [postgresql.conf file](https://www.postgresql.org/docs/current/config-setting.html) for your Postgres database:
+To enable logical replication on AWS Postgres RDS or Aurora:
+* Go to the Configuration tab for your DB cluster.
+* Find your cluster parameter group. Either edit the parameters for this group or create a copy of this parameter group to edit. If you create a copy, change your cluster's parameter group before restarting.
+* Within the parameter group page, search for `rds.logical_replication`. Select this row and click Edit parameters. Set this value to 1.
+* Wait for a maintenance window to automatically restart the instance or restart it manually.
 
-| Parameter             | Description                                                                    | Set value to                                                                                                                       |
-|-----------------------|--------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------|
-| wal_level             | Type of coding used within the Postgres write-ahead log                        | logical                                                                                                                            |
-| max_wal_senders       | The maximum number of processes used for handling WAL changes                  | Min: 1                                                                                                                             |
-| max_replication_slots | The maximum number of replication slots that are allowed to stream WAL changes | 1 (if Airbyte is the only service reading subscribing to WAL changes. More than 1 if other services are also reading from the WAL) |
-
-To enable logical replication on AWS Postgres RDS or Aurora​:
-
-1. Go to the Configuration tab for your DB cluster.
-2. Find your cluster parameter group. Either edit the parameters for this group or create a copy of this parameter group to edit. If you create a copy, change your cluster's parameter group before restarting.
-3. Within the parameter group page, search for `rds.logical_replication`. Select this row and click Edit parameters. Set this value to 1.
-4. Wait for a maintenance window to automatically restart the instance or restart it manually.
-
-To enable logical replication on Azure Database for Postgres​:
-
-Change the replication mode of your Postgres DB on Azure to `logical` using the **Replication** menu of your PostgreSQL instance in the Azure Portal. Alternatively, use the Azure CLI to run the following command:
+To enable logical replication on Azure Database for Postgres, change the replication mode of your Postgres DB on Azure to `logical` using the replication menu of your PostgreSQL instance in the Azure Portal. Alternatively, use the Azure CLI to run the following command:
 
 ```
 az postgres server configuration set --resource-group group --server-name server --name azure.replication_support --value logical
-```
-
-```
 az postgres server restart --resource-group group --name server
 ```
 
-#### Step 2: Create replication slot​
+#### Step 4: Create a replication slot on your Postgres database
+
+Airbyte requires a replication slot configured only for its use. Only one source should be configured that uses this replication slot.
 
-Airbyte currently supports pgoutput plugin only. To create a replication slot called `airbyte_slot` using pgoutput, run:
+For this step, Airbyte requires use of the pgoutput plugin. To create a replication slot called `airbyte_slot` using pgoutput, run as the user with the newly granted `REPLICATION` role:
 
 ```
 SELECT pg_create_logical_replication_slot('airbyte_slot', 'pgoutput');
 ```
 
-#### Step 3: Create publications and replication identities for tables​
+The output of this command will include the name of the replication slot to fill into the Airbyte source setup page.
 
-For each table you want to replicate with CDC, add the replication identity (the method of distinguishing between rows) first:
+#### Step 5: Create publication and replication identities for each Postgres table
 
-To use primary keys to distinguish between rows for tables that don't have a large amount of data per row, run:
+For each table you want to replicate with CDC, follow the steps below:
+
+1. Add the replication identity (the method of distinguishing between rows) for each table you want to replicate:
 
 ```
 ALTER TABLE tbl1 REPLICA IDENTITY DEFAULT;
 ```
 
-In case your tables use data types that support [TOAST](https://www.postgresql.org/docs/current/storage-toast.html) and have very large field values, use:
+In rare cases, if your tables use data types that support [TOAST](https://www.postgresql.org/docs/current/storage-toast.html) or have very large field values, consider instead using replica identity type full: `
+ALTER TABLE tbl1 REPLICA IDENTITY FULL;`.
 
-```
-ALTER TABLE tbl1 REPLICA IDENTITY FULL;
-```
-
-After setting the replication identity, run:
+2. Create the Postgres publication. You should include all tables you want to replicate as part of the publication:
 
 ```
 CREATE PUBLICATION airbyte_publication FOR TABLE ;`
@@ -240,165 +160,160 @@ CREATE PUBLICATION airbyte_publication FOR TABLE ;`
 The publication name is customizable. Refer to the [Postgres docs](https://www.postgresql.org/docs/10/sql-alterpublication.html) if you need to add or remove tables from your publication in the future.
 
 :::note
-You must add the replication identity before creating the publication. Otherwise, `ALTER`/`UPDATE`/`DELETE` statements may fail if Postgres cannot determine how to uniquely identify rows.
-Also, the publication should include all the tables and only the tables that need to be synced. Otherwise, data from these tables may not be replicated correctly.
-:::
-
-:::warning
 The Airbyte UI currently allows selecting any tables for CDC. If a table is selected that is not part of the publication, it will not be replicated even though it is selected. If a table is part of the publication but does not have a replication identity, that replication identity will be created automatically on the first run if the Airbyte user has the necessary permissions.
 :::
 
-#### Step 4: [Optional] Set up initial waiting time
+#### Step 6: Enable CDC replication in Airbyte UI
 
-:::warning
-This is an advanced feature. Use it if absolutely necessary.
-:::
+In your Postgres source, change the replication mode to `Logical Replication (CDC)`, and enter the replication slot and publication you just created.
 
-The Postgres connector may need some time to start processing the data in the CDC mode in the following scenarios:
+## Postgres Replication Methods
 
-- When the connection is set up for the first time and a snapshot is needed
-- When the connector has a lot of change logs to process
+The Postgres source currently offers 3 methods of replicating updates to your destination: CDC, xmin and standard (with a user defined cursor). Both CDC and xmin are the **most reliable methods** of updating your data.
 
-The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 120 seconds to 1200 seconds.
+#### CDC
 
-If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs.
+Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-replication.html) of the Postgres write-ahead log (WAL) to incrementally capture deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/). We recommend configuring your Postgres source with CDC when:
+- You need a record of deletions.
+- You have a very large database (500 GB or more).
+- Your table has a primary key but doesn't have a reasonable cursor field for incremental syncing (`updated_at`).
 
-#### Step 5: Set up the Postgres source connector
+If your goal is to maintain a snapshot of your table in the destination but the limitations prevent you from using CDC, consider using the xmin replication method.
 
-In [Step 2](#step-2-set-up-the-postgres-connector-in-airbyte) of the connector setup guide, enter the replication slot and publication you just created.
+#### Xmin
 
-## Supported sync modes
+Xmin replication is the new cursor-less replication method for Postgres. Cursorless syncs enable syncing new or updated rows without explicitly choosing a cursor field. The xmin system column which (available in all Postgres databases) is used to track inserts and updates to your source data.
 
-The Postgres source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+This is a good solution if:
+- There is not a well-defined cursor candidate to use for Standard incremental mode.
+- You want to replace a previously configured full-refresh sync.
+- You are replicating Postgres tables less than 500GB.
+- You are not replicating non-materialized views. Non-materialized views are not supported by xmin replication. 
 
-- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+## Connecting with SSL or SSH Tunneling
 
-## Supported cursors
+### SSL Modes
 
-- `TIMESTAMP`
-- `TIMESTAMP_WITH_TIMEZONE`
-- `TIME`
-- `TIME_WITH_TIMEZONE`
-- `DATE`
-- `BIT`
-- `BOOLEAN`
-- `TINYINT/SMALLINT`
-- `INTEGER`
-- `BIGINT`
-- `FLOAT/DOUBLE`
-- `REAL`
-- `NUMERIC/DECIMAL`
-- `CHAR/NCHAR/NVARCHAR/VARCHAR/LONGVARCHAR`
-- `BINARY/BLOB`
+Airbyte Cloud uses SSL by default. You are not permitted to `disable` SSL while using Airbyte Cloud. 
 
-## Data type mapping
+Here is a breakdown of available SSL connection modes:
+- `disable` to disable encrypted communication between Airbyte and the source
+- `allow` to enable encrypted communication only when required by the source
+- `prefer` to allow unencrypted communication only when the source doesn't support encryption
+- `require` to always require encryption. Note: The connection will fail if the source doesn't support encryption.
+- `verify-ca` to always require encryption and verify that the source has a valid SSL certificate
+- `verify-full` to always require encryption and verify the identity of the source
 
-According to Postgres [documentation](https://www.postgresql.org/docs/14/datatype.html), Postgres data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test!
+### SSH Tunneling
 
-| Postgres Type                         | Resulting Type | Notes                                                                                                                                                 |
-|---------------------------------------|----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `bigint`                              | number         |                                                                                                                                                       |
-| `bigserial`, `serial8`                | number         |                                                                                                                                                       |
-| `bit`                                 | string         | Fixed-length bit string (e.g. "0100").                                                                                                                |
-| `bit varying`, `varbit`               | string         | Variable-length bit string (e.g. "0100").                                                                                                             |
-| `boolean`, `bool`                     | boolean        |                                                                                                                                                       |
-| `box`                                 | string         |                                                                                                                                                       |
-| `bytea`                               | string         | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a").                                                            |
-| `character`, `char`                   | string         |                                                                                                                                                       |
-| `character varying`, `varchar`        | string         |                                                                                                                                                       |
-| `cidr`                                | string         |                                                                                                                                                       |
-| `circle`                              | string         |                                                                                                                                                       |
-| `date`                                | string         | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590)  |
-| `double precision`, `float`, `float8` | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).       |
-| `hstore`                              | string         |                                                                                                                                                       |
-| `inet`                                | string         |                                                                                                                                                       |
-| `integer`, `int`, `int4`              | number         |                                                                                                                                                       |
-| `interval`                            | string         |                                                                                                                                                       |
-| `json`                                | string         |                                                                                                                                                       |
-| `jsonb`                               | string         |                                                                                                                                                       |
-| `line`                                | string         |                                                                                                                                                       |
-| `lseg`                                | string         |                                                                                                                                                       |
-| `macaddr`                             | string         |                                                                                                                                                       |
-| `macaddr8`                            | string         |                                                                                                                                                       |
-| `money`                               | number         |                                                                                                                                                       |
-| `numeric`, `decimal`                  | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).       |
-| `path`                                | string         |                                                                                                                                                       |
-| `pg_lsn`                              | string         |                                                                                                                                                       |
-| `point`                               | string         |                                                                                                                                                       |
-| `polygon`                             | string         |                                                                                                                                                       |
-| `real`, `float4`                      | number         |                                                                                                                                                       |
-| `smallint`, `int2`                    | number         |                                                                                                                                                       |
-| `smallserial`, `serial2`              | number         |                                                                                                                                                       |
-| `serial`, `serial4`                   | number         |                                                                                                                                                       |
-| `text`                                | string         |                                                                                                                                                       |
-| `time`                                | string         | Parsed as a time string without a time-zone in the ISO-8601 calendar system.                                                                          |
-| `timetz`                              | string         | Parsed as a time string with time-zone in the ISO-8601 calendar system.                                                                               |
-| `timestamp`                           | string         | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system.                                                                     |
-| `timestamptz`                         | string         | Parsed as a date-time string with time-zone in the ISO-8601 calendar system.                                                                          |
-| `tsquery`                             | string         |                                                                                                                                                       |
-| `tsvector`                            | string         |                                                                                                                                                       |
-| `uuid`                                | string         |                                                                                                                                                       |
-| `xml`                                 | string         |                                                                                                                                                       |
-| `enum`                                | string         |                                                                                                                                                       |
-| `tsrange`                             | string         |                                                                                                                                                       |
-| `array`                               | array          | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]".                                                                                                     |
-| composite type                        | string         |                                                                                                                                                       |
-
-## Limitations
-
-- The Postgres source connector currently does not handle schemas larger than 4MB.
-- The Postgres source connector does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details.
-- The following two schema evolution actions are currently supported:
-  - Adding/removing tables without resetting the entire connection at the destination
-    Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes.
-  - Resetting a single table within the connection without resetting the rest of the destination tables in that connection
-- Changing a column data type or removing a column might break connections.
-
-## Troubleshooting
-
-### Sync data from Postgres hot standby server
-
-When the connector is reading from a Postgres replica that is configured as a Hot Standby, any update from the primary server will terminate queries on the replica after a certain amount of time, default to 30 seconds. This default waiting time is not enough to sync any meaning amount of data. See the `Handling Query Conflicts` section in the Postgres [documentation](https://www.postgresql.org/docs/14/hot-standby.html#HOT-STANDBY-CONFLICT) for detailed explanation.
-
-Here is the typical exception:
+If you are using SSH tunneling, as Airbyte Cloud requires encrypted communication, select `SSH Key Authentication` or `Password Authentication` if you selected `disable`, `allow`, or `prefer` as the SSL Mode; otherwise, the connection will fail.
 
-```
-Caused by: org.postgresql.util.PSQLException: FATAL: terminating connection due to conflict with recovery
-    Detail: User query might have needed to see row versions that must be removed.
-    Hint: In a moment you should be able to reconnect to the database and repeat your command.
-```
+For SSH Tunnel Method, select:
+- `No Tunnel` for a direct connection to the database
+- `SSH Key Authentication` to use an RSA Private as your secret for establishing the SSH tunnel
+- `Password Authentication` to use a password as your secret for establishing the SSH tunnel
+
+#### Connect via SSH Tunnel
+
+You can connect to a Postgres instance via an SSH tunnel.
+
+When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server (also called a bastion or a jump server) that has direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server.
+
+To connect to a Postgres instance via an SSH tunnel:
+
+1. While [setting up](#step-2-create-a-new-postgres-source-in-airbyte-ui) the Postgres source connector, from the SSH tunnel dropdown, select:
+    - SSH Key Authentication to use a private as your secret for establishing the SSH tunnel
+    - Password Authentication to use a password as your secret for establishing the SSH Tunnel
+2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to.
+3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22.
+4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the Postgres username.
+5. For authentication:
+    - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [private Key](#generating-a-private-key-for-ssh-tunneling) that you are using to create the SSH connection.
+    - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the Postgres password.
 
-Possible solutions include:
+#### Generating a private key for SSH Tunneling
 
-- [Recommended] Set [`hot_standby_feedback`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-HOT-STANDBY-FEEDBACK) to `true` on the replica server. This parameter will prevent the primary server from deleting the write-ahead logs when the replica is busy serving user queries. However, the downside is that the write-ahead log will increase in size.
-- [Recommended] Sync data when there is no update running in the primary server, or sync data from the primary server.
-- [Not Recommended] Increase [`max_standby_archive_delay`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-MAX-STANDBY-ARCHIVE-DELAY) and [`max_standby_streaming_delay`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-MAX-STANDBY-STREAMING-DELAY) to be larger than the amount of time needed to complete the data sync. However, it is usually hard to tell how much time it will take to sync all the data. This approach is not very practical.
+The connector supports any SSH compatible key format such as RSA or Ed25519. To generate an RSA key, for example, run:
 
-### Under CDC incremental mode, there are still full refresh syncs
+```
+ssh-keygen -t rsa -m PEM -f myuser_rsa
+```
 
-Normally under the CDC mode, the Postgres source will first run a full refresh sync to read the snapshot of all the existing data, and all subsequent runs will only be incremental syncs reading from the write-ahead logs (WAL). However, occasionally, you may see full refresh syncs after the initial run. When this happens, you will see the following log:
+The command produces the private key in PEM format and the public key remains in the standard format used by the `authorized_keys` file on your bastion server. Add the public key to your bastion host to the user you want to use with Airbyte. The private key is provided via copy-and-paste to the Airbyte connector configuration screen to allow it to log into the bastion server.
 
-> Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch
+## Limitations & Troubleshooting
 
-The root causes is that the WALs needed for the incremental sync has been removed by Postgres. This can occur under the following scenarios:
-        
-- When there are lots of database updates resulting in more WAL files than allowed in the `pg_wal` directory, Postgres will purge or archive the WAL files. This scenario is preventable. Possible solutions include:
-  - Sync the data source more frequently. The downside is that more computation resources will be consumed, leading to a higher Airbyte bill.
-  - Set a higher `wal_keep_size`. If no unit is provided, it is in megabytes, and the default is `0`. See detailed documentation [here](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-KEEP-SIZE). The downside of this approach is that more disk space will be needed.
-- When the Postgres connector successfully reads the WAL and acknowledges it to Postgres, but the destination connector fails to consume the data, the Postgres connector will try to read the same WAL again, which may have been removed by Postgres, since the WAL record is already acknowledged. This scenario is rare, because it can happen, and currently there is no way to prevent it. The correct behavior is to perform a full refresh.
+To see connector limitations, or troubleshoot your Postgres connector, see more [in our Postgres troubleshooting guide](https://docs.airbyte.com/integrations/sources/postgres/postgres-troubleshooting).
 
-### Temporary File Size Limit
+## Data type mapping
 
-Some larger tables may encounter an error related to the temporary file size limit such as `temporary file size exceeds temp_file_limit`. To correct this error increase the [temp_file_limit](https://postgresqlco.nf/doc/en/param/temp_file_limit/).
+According to Postgres [documentation](https://www.postgresql.org/docs/14/datatype.html), Postgres data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test!
 
+| Postgres Type                         | Resulting Type | Notes                                                                                                                                                |
+|---------------------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `bigint`                              | number         |                                                                                                                                                      |
+| `bigserial`, `serial8`                | number         |                                                                                                                                                      |
+| `bit`                                 | string         | Fixed-length bit string (e.g. "0100").                                                                                                               |
+| `bit varying`, `varbit`               | string         | Variable-length bit string (e.g. "0100").                                                                                                            |
+| `boolean`, `bool`                     | boolean        |                                                                                                                                                      |
+| `box`                                 | string         |                                                                                                                                                      |
+| `bytea`                               | string         | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a").                                                           |
+| `character`, `char`                   | string         |                                                                                                                                                      |
+| `character varying`, `varchar`        | string         |                                                                                                                                                      |
+| `cidr`                                | string         |                                                                                                                                                      |
+| `circle`                              | string         |                                                                                                                                                      |
+| `date`                                | string         | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590) |
+| `double precision`, `float`, `float8` | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).      |
+| `hstore`                              | string         |                                                                                                                                                      |
+| `inet`                                | string         |                                                                                                                                                      |
+| `integer`, `int`, `int4`              | number         |                                                                                                                                                      |
+| `interval`                            | string         |                                                                                                                                                      |
+| `json`                                | string         |                                                                                                                                                      |
+| `jsonb`                               | string         |                                                                                                                                                      |
+| `line`                                | string         |                                                                                                                                                      |
+| `lseg`                                | string         |                                                                                                                                                      |
+| `macaddr`                             | string         |                                                                                                                                                      |
+| `macaddr8`                            | string         |                                                                                                                                                      |
+| `money`                               | number         |                                                                                                                                                      |
+| `numeric`, `decimal`                  | number         | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902).      |
+| `path`                                | string         |                                                                                                                                                      |
+| `pg_lsn`                              | string         |                                                                                                                                                      |
+| `point`                               | string         |                                                                                                                                                      |
+| `polygon`                             | string         |                                                                                                                                                      |
+| `real`, `float4`                      | number         |                                                                                                                                                      |
+| `smallint`, `int2`                    | number         |                                                                                                                                                      |
+| `smallserial`, `serial2`              | number         |                                                                                                                                                      |
+| `serial`, `serial4`                   | number         |                                                                                                                                                      |
+| `text`                                | string         |                                                                                                                                                      |
+| `time`                                | string         | Parsed as a time string without a time-zone in the ISO-8601 calendar system.                                                                         |
+| `timetz`                              | string         | Parsed as a time string with time-zone in the ISO-8601 calendar system.                                                                              |
+| `timestamp`                           | string         | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system.                                                                    |
+| `timestamptz`                         | string         | Parsed as a date-time string with time-zone in the ISO-8601 calendar system.                                                                         |
+| `tsquery`                             | string         |                                                                                                                                                      |
+| `tsvector`                            | string         |                                                                                                                                                      |
+| `uuid`                                | string         |                                                                                                                                                      |
+| `xml`                                 | string         |                                                                                                                                                      |
+| `enum`                                | string         |                                                                                                                                                      |
+| `tsrange`                             | string         |                                                                                                                                                      |
+| `array`                               | array          | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]".                                                                                                    |
+| composite type                        | string         |                                                                                                                                                      |
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.1.5   | 2023-08-22 | [29534](https://github.com/airbytehq/airbyte/pull/29534) | Support "options" JDBC URL parameter                                                                                                                                       |
+| 3.1.4   | 2023-08-21 | [28687](https://github.com/airbytehq/airbyte/pull/28687) | Under the hood: Add dependency on Java CDK v0.0.2.                                                                                                                          |
+| 3.1.3   | 2023-08-03 | [28708](https://github.com/airbytehq/airbyte/pull/28708) | Enable checkpointing snapshots in CDC connections                                                                                                                          |
+| 3.1.2   | 2023-08-01 | [28954](https://github.com/airbytehq/airbyte/pull/28954) | Fix an issue that prevented use of tables with names containing uppercase letters                                                                                          |
+| 3.1.1   | 2023-07-31 | [28892](https://github.com/airbytehq/airbyte/pull/28892) | Fix an issue that prevented use of cursor columns with names containing uppercase letters                                                                                  |
+| 3.1.0   | 2023-07-25 | [28339](https://github.com/airbytehq/airbyte/pull/28339) | Checkpointing initial load for incremental syncs: enabled for xmin and cursor based only.                                                                                  |
+| 3.0.2   | 2023-07-18 | [28336](https://github.com/airbytehq/airbyte/pull/28336) | Add full-refresh mode back to Xmin syncs.                                                                                                                                  |
+| 3.0.1   | 2023-07-14 | [28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild                                                                                                                                       |
+| 3.0.0   | 2023-07-12 | [27442](https://github.com/airbytehq/airbyte/pull/27442) | Set \_ab_cdc_lsn as the source defined cursor for CDC mode to prepare for Destination v2 normalization                                                                     |
+| 2.1.1   | 2023-07-06 | [26723](https://github.com/airbytehq/airbyte/pull/26723) | Add new xmin replication method.                                                                                                                                           |
+| 2.1.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2                                                                                                                                                       |
+| 2.0.34  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                                                   |
 | 2.0.33  | 2023-06-01 | [26873](https://github.com/airbytehq/airbyte/pull/26873) | Add prepareThreshold=0 to JDBC url to mitigate PGBouncer prepared statement [X] already exists.                                                                            |
 | 2.0.32  | 2023-05-31 | [26810](https://github.com/airbytehq/airbyte/pull/26810) | Remove incremental sync estimate from Postgres to increase performance.                                                                                                    |
 | 2.0.31  | 2023-05-25 | [26633](https://github.com/airbytehq/airbyte/pull/26633) | Collect and log information related to full vacuum operation in db                                                                                                         |
@@ -497,7 +412,7 @@ Some larger tables may encounter an error related to the temporary file size lim
 | 0.4.39  | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings                                                                                                                                                  |
 | 0.4.38  | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields.                                                                                                                       |
 | 0.4.37  | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected                                                                                                                |
-| 0.4.36  | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead)                                                              | |
+| 0.4.36  | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead)                                                              |     |
 | 0.4.35  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors                                                                                                             |
 | 0.4.34  | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates.                                                                                               |
 | 0.4.33  | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters                                                                                                                                        |
@@ -526,7 +441,7 @@ Some larger tables may encounter an error related to the temporary file size lim
 | 0.4.8   | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats                                                                     |
 | 0.4.7   | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds                                                                                                                         |
 | 0.4.6   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                                                                 |
-| 0.4.5   | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved  discovering tables in case if user does not have permissions to any table                                                                                        |
+| 0.4.5   | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table                                                                                         |
 | 0.4.4   | 2022-01-26 | [9807](https://github.com/airbytehq/airbyte/pull/9807)   | Update connector fields title/description                                                                                                                                  |
 | 0.4.3   | 2022-01-24 | [9554](https://github.com/airbytehq/airbyte/pull/9554)   | Allow handling of java sql date in CDC                                                                                                                                     |
 | 0.4.2   | 2022-01-13 | [9360](https://github.com/airbytehq/airbyte/pull/9360)   | Added schema selection                                                                                                                                                     |
diff --git a/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_add_network.png b/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_add_network.png
new file mode 100644
index 000000000000..134273ddbd5c
Binary files /dev/null and b/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_add_network.png differ
diff --git a/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_db.png b/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_db.png
new file mode 100644
index 000000000000..817da0a78532
Binary files /dev/null and b/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_db.png differ
diff --git a/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_logical_replication_flag.png b/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_logical_replication_flag.png
new file mode 100644
index 000000000000..8372afc93b8e
Binary files /dev/null and b/docs/integrations/sources/postgres/assets/airbyte_cloud_sql_postgres_logical_replication_flag.png differ
diff --git a/docs/integrations/sources/postgres/assets/airbyte_postgres_source.png b/docs/integrations/sources/postgres/assets/airbyte_postgres_source.png
new file mode 100644
index 000000000000..4e9db9c4067c
Binary files /dev/null and b/docs/integrations/sources/postgres/assets/airbyte_postgres_source.png differ
diff --git a/docs/integrations/sources/postgres/assets/airbyte_source_selection.png b/docs/integrations/sources/postgres/assets/airbyte_source_selection.png
new file mode 100644
index 000000000000..901bed4ab9f2
Binary files /dev/null and b/docs/integrations/sources/postgres/assets/airbyte_source_selection.png differ
diff --git a/docs/integrations/sources/postgres/cloud-sql-postgres.md b/docs/integrations/sources/postgres/cloud-sql-postgres.md
new file mode 100644
index 000000000000..0dd9bcf5ee3e
--- /dev/null
+++ b/docs/integrations/sources/postgres/cloud-sql-postgres.md
@@ -0,0 +1,169 @@
+# Cloud SQL for PostgreSQL
+
+Airbyte's certified Postgres connector offers the following features:
+* Multiple methods of keeping your data fresh, including [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) and replication using the [xmin system column](https://docs.airbyte.com/integrations/sources/postgres#xmin).
+* All available [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes), providing flexibility in how data is delivered to your destination.
+* Reliable replication at any table size with [checkpointing](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#state--checkpointing) and chunking of database reads.
+
+![Airbyte Postgres Connection](https://raw.githubusercontent.com/airbytehq/airbyte/c078e8ed6703020a584d9362efa5665fbe8db77f/docs/integrations/sources/postgres/assets/airbyte_postgres_source.png?raw=true)
+
+## Quick Start
+
+![Cloud SQL for PostgreSQL](./assets/airbyte_cloud_sql_postgres_db.png)
+
+Here is an outline of the minimum required steps to configure a connection to Postgres on Google Cloud SQL:
+1. Create a dedicated read-only Postgres user with permissions for replicating data
+2. Create a new Postgres source in the Airbyte UI using `xmin` system column
+3. (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs
+
+Once this is complete, you will be able to select Postgres as a source for replicating data.
+
+#### Step 1: Create a dedicated read-only Postgres user
+
+These steps create a dedicated read-only user for replicating data. Alternatively, you can use an existing Postgres user in your database. To create a user, first [connect to your database](https://cloud.google.com/sql/docs/postgres/connect-overview#external-connection-methods). If you are getting started, you can use [Cloud Shell to connect directly from the UI](https://cloud.google.com/sql/docs/postgres/connect-instance-cloud-shell).
+
+The following commands will create a new user:
+
+```roomsql
+CREATE USER  PASSWORD 'your_password_here';
+```
+
+Now, provide this user with read-only access to relevant schemas and tables. Re-run this command for each schema you expect to replicate data from (e.g. `public`):
+
+```roomsql
+GRANT USAGE ON SCHEMA  TO ;
+GRANT SELECT ON ALL TABLES IN SCHEMA  TO ;
+ALTER DEFAULT PRIVILEGES IN SCHEMA  GRANT SELECT ON TABLES TO ;
+```
+
+#### Step 2: Create a new Postgres source in Airbyte UI
+
+From your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account, select `Sources` from the left navigation bar, search for `Postgres`, then create a new Postgres source.
+
+![Create an Airbyte source](https://github.com/airbytehq/airbyte/blob/c078e8ed6703020a584d9362efa5665fbe8db77f/docs/integrations/sources/postgres/assets/airbyte_source_selection.png?raw=true)
+
+To fill out the required information:
+1. Enter the hostname, port number, and name for your Postgres database.
+2. You may optionally opt to list each of the schemas you want to sync. These are case-sensitive, and multiple schemas may be entered. By default, `public` is the only selected schema.
+3. Enter the username and password you created in [Step 1](#step-1-create-a-dedicated-read-only-postgres-user).
+4. Select an SSL mode. You will most frequently choose `require` or `verify-ca`. Both of these always require encryption. `verify-ca` also requires certificates from your Postgres database. See here to learn about other SSL modes and SSH tunneling.
+5. Select `Standard (xmin)` from available replication methods. This uses the [xmin system column](https://docs.airbyte.com/integrations/sources/postgres#xmin) to reliably replicate data from your database.
+    1. If your database is particularly large (> 500 GB), you will benefit from [configuring your Postgres source using logical replication (CDC)](https://docs.airbyte.com/integrations/sources/postgres#cdc).
+
+#### Step 3: (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs.
+
+If you are on Airbyte Cloud, you will always need to modify your database configuration to allow inbound traffic from Airbyte IPs. To allowlist IPs in Cloud SQL:
+1. In your Google Cloud SQL database dashboard, select `Connections` from the left menu. Then, select `Add Network` under the `Connectivity` section.
+
+![Add a Network](./assets/airbyte_cloud_sql_postgres_add_network.png)
+
+2. Add a new network, and enter Airbyte's IPs:
+
+```roomsql
+34.106.109.131
+34.106.196.165
+34.106.60.246
+34.106.229.69
+34.106.127.139
+34.106.218.58
+34.106.115.240
+34.106.225.141
+13.37.4.46
+13.37.142.60
+35.181.124.238
+```
+
+Now, click `Set up source` in the Airbyte UI. Airbyte will now test connecting to your database. Once this succeeds, you've configured an Airbyte Postgres source!
+
+## Advanced Configuration
+
+### Setup using CDC
+
+Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-replication.html) of the Postgres write-ahead log (WAL) to incrementally capture deletes using a replication plugin:
+* See [here](https://docs.airbyte.com/understanding-airbyte/cdc) to learn more on how Airbyte implements CDC.
+* See [here](https://docs.airbyte.com/integrations/sources/postgres/postgres-troubleshooting#cdc-requirements) to learn more about Postgres CDC requirements and limitations.
+
+We recommend configuring your Postgres source with CDC when:
+- You need a record of deletions.
+- You have a very large database (500 GB or more).
+- Your table has a primary key but doesn't have a reasonable cursor field for incremental syncing (`updated_at`).
+
+These are the additional steps required (after following the [quick start](#quick-start)) to configure your Postgres source using CDC:
+1. Provide additional `REPLICATION` permissions to read-only user
+2. Enable logical replication on your Postgres database
+3. Create a replication slot on your Postgres database
+4. Create publication and replication identities for each Postgres table
+5. Enable CDC replication in the Airbyte UI
+
+#### Step 1: Prepopulate your Postgres source configuration
+
+We recommend following the steps in the [quick start](#quick-start) section to confirm that Airbyte can connect to your Postgres database prior to configuring CDC settings.
+
+For CDC, you must connect to primary/master databases. Pointing the connector configuration to replica database hosts for CDC will lead to failures.
+
+#### Step 2: Provide additional permissions to read-only user
+
+To configure CDC for the Postgres source connector, grant `REPLICATION` permissions to the user created in [step 1 of the quick start](#step-1-create-a-dedicated-read-only-postgres-user):
+```
+ALTER USER  REPLICATION;
+```
+
+#### Step 3: Enable logical replication on your Postgres database
+
+To enable logical replication on Cloud SQL for Postgres, set the `cloudsql.logical_decoding` flag to on. You can find the `Flags` section in the `Edit Configuration` view of your database:
+
+![Enable Logical Decoding](./assets/airbyte_cloud_sql_postgres_logical_replication_flag.png)
+
+#### Step 4: Create a replication slot on your Postgres database
+
+Airbyte requires a replication slot configured only for its use. Only one source should be configured that uses this replication slot.
+
+For this step, Airbyte requires use of the pgoutput plugin. To create a replication slot called `airbyte_slot` using pgoutput, provide the instance superuser (default `postgres`) with `REPLICATION` permissions, and run the following:
+
+```
+ALTER user postgres with REPLICATION;
+SELECT pg_create_logical_replication_slot('airbyte_slot', 'pgoutput');
+```
+
+The output of this command will include the name of the replication slot to fill into the Airbyte source setup page.
+
+#### Step 5: Create publication and replication identities for each Postgres table
+
+For each table you want to replicate with CDC, follow the steps below:
+
+1. Add the replication identity (the method of distinguishing between rows) for each table you want to replicate:
+
+```
+ALTER TABLE tbl1 REPLICA IDENTITY DEFAULT;
+```
+
+In rare cases, if your tables use data types that support [TOAST](https://www.postgresql.org/docs/current/storage-toast.html) or have very large field values, consider instead using replica identity type full: `
+ALTER TABLE tbl1 REPLICA IDENTITY FULL;`.
+
+2. Create the Postgres publication. You should include all tables you want to replicate as part of the publication:
+
+```
+CREATE PUBLICATION airbyte_publication FOR TABLE tbl1, tbl2, tbl3;`
+```
+
+The publication name is customizable. Refer to the [Postgres docs](https://www.postgresql.org/docs/10/sql-alterpublication.html) if you need to add or remove tables from your publication in the future.
+
+:::note
+The Airbyte UI currently allows selecting any tables for CDC. If a table is selected that is not part of the publication, it will not be replicated even though it is selected. If a table is part of the publication but does not have a replication identity, that replication identity will be created automatically on the first run if the Airbyte user has the necessary permissions.
+:::
+
+#### Step 6: Enable CDC replication in Airbyte UI
+
+In your Postgres source, change the replication mode to `Logical Replication (CDC)`, and enter the replication slot and publication you just created.
+
+## Postgres Replication Methods
+
+The Postgres source currently offers 3 methods of replicating updates to your destination: CDC, xmin and standard (with a user defined cursor). See [here](https://docs.airbyte.com/integrations/sources/postgres#postgres-replication-methods) for more details.
+
+## Connecting with SSL or SSH Tunnel
+
+See [these instructions](https://docs.airbyte.com/integrations/sources/postgres#connecting-with-ssl-or-ssh-tunneling) to learn more about SSL modes and connecting via SSH tunnel.
+
+## Limitations & Troubleshooting
+
+To see connector limitations, or troubleshoot your Postgres connector, see more [in our Postgres troubleshooting guide](https://docs.airbyte.com/integrations/sources/postgres/postgres-troubleshooting).
diff --git a/docs/integrations/sources/postgres/postgres-troubleshooting.md b/docs/integrations/sources/postgres/postgres-troubleshooting.md
new file mode 100644
index 000000000000..556118a6c1d1
--- /dev/null
+++ b/docs/integrations/sources/postgres/postgres-troubleshooting.md
@@ -0,0 +1,113 @@
+# Troubleshooting Postgres Sources
+
+## Connector Limitations
+
+### General Limitations
+
+- The Postgres source connector currently does not handle schemas larger than 4MB.
+- The Postgres source connector does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details.
+- The following two schema evolution actions are currently supported:
+    - Adding/removing tables without resetting the entire connection at the destination
+      Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes.
+    - Resetting a single table within the connection without resetting the rest of the destination tables in that connection
+- Changing a column data type or removing a column might break connections.
+
+### Version Requirements
+
+- For Airbyte Open Source users, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer
+- Use Postgres v9.3.x or above for non-CDC workflows and Postgres v10 or above for CDC workflows
+- For Airbyte Cloud (and optionally for Airbyte Open Source), ensure SSL is enabled in your environment
+
+### CDC Requirements
+
+- Incremental sync is only supported for tables with primary keys. For tables without primary keys, use [Full Refresh sync](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite).
+- Data must be in tables and not views. If you require data synchronization from a view, you would need to create a new connection with `Standard` as `Replication Method`.
+- The modifications you want to capture must be made using `DELETE`/`INSERT`/`UPDATE`. For example, changes made using `TRUNCATE`/`ALTER` will not appear in logs and therefore in your destination.
+- Schema changes are not supported automatically for CDC sources. Reset and resync data if you make a schema change.
+- The records produced by `DELETE` statements only contain primary keys. All other data fields are unset.
+- Log-based replication only works for master instances of Postgres. CDC cannot be run from a read-replica of your primary database.
+- An Airbyte database source using CDC replication can only be used with a single Airbyte destination. This is due to how Postgres CDC is implemented - each destination would recieve only part of the data available in the replication slot.
+- Using logical replication increases disk space used on the database server. The additional data is stored until it is consumed.
+    - Set frequent syncs for CDC to ensure that the data doesn't fill up your disk space.
+    - If you stop syncing a CDC-configured Postgres instance with Airbyte, delete the replication slot. Otherwise, it may fill up your disk space.
+
+### Supported cursors
+
+- `TIMESTAMP`
+- `TIMESTAMP_WITH_TIMEZONE`
+- `TIME`
+- `TIME_WITH_TIMEZONE`
+- `DATE`
+- `BIT`
+- `BOOLEAN`
+- `TINYINT/SMALLINT`
+- `INTEGER`
+- `BIGINT`
+- `FLOAT/DOUBLE`
+- `REAL`
+- `NUMERIC/DECIMAL`
+- `CHAR/NCHAR/NVARCHAR/VARCHAR/LONGVARCHAR`
+- `BINARY/BLOB`
+
+## Troubleshooting
+
+### Sync data from Postgres hot standby server
+
+When the connector is reading from a Postgres replica that is configured as a Hot Standby, any update from the primary server will terminate queries on the replica after a certain amount of time, default to 30 seconds. This default waiting time is not enough to sync any meaning amount of data. See the `Handling Query Conflicts` section in the Postgres [documentation](https://www.postgresql.org/docs/14/hot-standby.html#HOT-STANDBY-CONFLICT) for detailed explanation.
+
+Here is the typical exception:
+
+```
+Caused by: org.postgresql.util.PSQLException: FATAL: terminating connection due to conflict with recovery
+    Detail: User query might have needed to see row versions that must be removed.
+    Hint: In a moment you should be able to reconnect to the database and repeat your command.
+```
+
+Possible solutions include:
+
+- [Recommended] Set [`hot_standby_feedback`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-HOT-STANDBY-FEEDBACK) to `true` on the replica server. This parameter will prevent the primary server from deleting the write-ahead logs when the replica is busy serving user queries. However, the downside is that the write-ahead log will increase in size.
+- [Recommended] Sync data when there is no update running in the primary server, or sync data from the primary server.
+- [Not Recommended] Increase [`max_standby_archive_delay`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-MAX-STANDBY-ARCHIVE-DELAY) and [`max_standby_streaming_delay`](https://www.postgresql.org/docs/14/runtime-config-replication.html#GUC-MAX-STANDBY-STREAMING-DELAY) to be larger than the amount of time needed to complete the data sync. However, it is usually hard to tell how much time it will take to sync all the data. This approach is not very practical.
+
+### Under CDC incremental mode, there are still full refresh syncs
+
+Normally under the CDC mode, the Postgres source will first run a full refresh sync to read the snapshot of all the existing data, and all subsequent runs will only be incremental syncs reading from the write-ahead logs (WAL). However, occasionally, you may see full refresh syncs after the initial run. When this happens, you will see the following log:
+
+> Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch
+
+The root causes is that the WALs needed for the incremental sync has been removed by Postgres. This can occur under the following scenarios:
+
+- When there are lots of database updates resulting in more WAL files than allowed in the `pg_wal` directory, Postgres will purge or archive the WAL files. This scenario is preventable. Possible solutions include:
+    - Sync the data source more frequently. The downside is that more computation resources will be consumed, leading to a higher Airbyte bill.
+    - Set a higher `wal_keep_size`. If no unit is provided, it is in megabytes, and the default is `0`. See detailed documentation [here](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-KEEP-SIZE). The downside of this approach is that more disk space will be needed.
+- When the Postgres connector successfully reads the WAL and acknowledges it to Postgres, but the destination connector fails to consume the data, the Postgres connector will try to read the same WAL again, which may have been removed by Postgres, since the WAL record is already acknowledged. This scenario is rare, because it can happen, and currently there is no way to prevent it. The correct behavior is to perform a full refresh.
+
+### Temporary File Size Limit
+
+Some larger tables may encounter an error related to the temporary file size limit such as `temporary file size exceeds temp_file_limit`. To correct this error increase the [temp_file_limit](https://postgresqlco.nf/doc/en/param/temp_file_limit/).
+
+
+### (Advanced) Custom JDBC Connection Strings
+
+To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://jdbc.postgresql.org/documentation/head/connect.html) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field.
+
+Example: key1=value1&key2=value2&key3=value3
+
+These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your Postgres database.
+
+The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available.
+
+**Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte:
+`currentSchema`, `user`, `password`, `ssl`, and `sslmode`.
+
+### (Advanced) Setting up initial CDC waiting time
+
+The Postgres connector may need some time to start processing the data in the CDC mode in the following scenarios:
+- When the connection is set up for the first time and a snapshot is needed
+- When the connector has a lot of change logs to process
+
+The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 120 seconds to 1200 seconds.
+
+If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs.
+
+
diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md
index cb4d0444a7c2..3ce96d515681 100644
--- a/docs/integrations/sources/posthog.md
+++ b/docs/integrations/sources/posthog.md
@@ -52,7 +52,10 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                            |
-| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------|
+| 0.1.13  | 2023-07-19 | [28461](https://github.com/airbytehq/airbyte/pull/28461) | Fixed EventsSimpleRetriever declaration                                                                            |
+| 0.1.12  | 2023-06-28 | [27764](https://github.com/airbytehq/airbyte/pull/27764) | Update following state breaking changes                                                                            |
+| 0.1.11  | 2023-06-09 | [27135](https://github.com/airbytehq/airbyte/pull/27135) | Fix custom EventsSimpleRetriever                                                                                   |
 | 0.1.10  | 2023-04-15 | [24084](https://github.com/airbytehq/airbyte/pull/24084) | Increase `events` streams batch size                                                                               |
 | 0.1.9   | 2023-02-13 | [22906](https://github.com/airbytehq/airbyte/pull/22906) | Specified date formatting in specification                                                                         |
 | 0.1.8   | 2022-11-11 | [18993](https://github.com/airbytehq/airbyte/pull/18993) | connector migrated to low-code, added projects,insights streams, added project based slicing for all other streams |
diff --git a/docs/integrations/sources/prestashop.md b/docs/integrations/sources/prestashop.md
index 1a56e90bec88..3a6fa7a7ff04 100644
--- a/docs/integrations/sources/prestashop.md
+++ b/docs/integrations/sources/prestashop.md
@@ -33,7 +33,7 @@ The PrestaShop source connector supports the following [ sync modes](https://doc
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
@@ -104,7 +104,8 @@ If there are more endpoints you'd like Airbyte to support, please [create an iss
 
 | Version | Date       | Pull Request                                              | Subject                                              |
 | :------ | :--------- | :-------------------------------------------------------- | :--------------------------------------------------- |
-| 0.3.1   | 2023-02-13 | [22905](https://github.com/airbytehq/airbyte/pull/22905) | Specified date formatting in specification |
+| 1.0.0   | 2023-06-26 | [27716](https://github.com/airbytehq/airbyte/pull/27716)  | update schema; remove empty datetime fields          |
+| 0.3.1   | 2023-02-13 | [22905](https://github.com/airbytehq/airbyte/pull/22905)  | Specified date formatting in specification           |
 | 0.3.0   | 2022-11-08 | [#18927](https://github.com/airbytehq/airbyte/pull/18927) | Migrate connector from Alpha (Python) to Beta (YAML) |
 | 0.2.0   | 2022-10-31 | [#18599](https://github.com/airbytehq/airbyte/pull/18599) | Only https scheme is allowed                         |
 | 0.1.0   | 2021-07-02 | [#4465](https://github.com/airbytehq/airbyte/pull/4465)   | Initial implementation                               |
diff --git a/docs/integrations/sources/quickbooks.md b/docs/integrations/sources/quickbooks.md
index 12b349bd4633..f5f905cd95ed 100644
--- a/docs/integrations/sources/quickbooks.md
+++ b/docs/integrations/sources/quickbooks.md
@@ -37,6 +37,7 @@ This page contains the setup guide and reference information for the QuickBooks
 
 
 
+
 **For Airbyte Open Source:**
 
 1. **Client ID** - The OAuth2.0 application ID
@@ -53,10 +54,10 @@ This page contains the setup guide and reference information for the QuickBooks
 
 The Quickbooks Source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
@@ -94,7 +95,7 @@ This Source is capable of syncing the following [Streams](https://developer.intu
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `number`         | `number`     |       |
 | `array`          | `array`      |       |
@@ -102,12 +103,15 @@ This Source is capable of syncing the following [Streams](https://developer.intu
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                  |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------|
-| `2.0.1` | 2023-05-28 | [26722](https://github.com/airbytehq/airbyte/pull/26722) | Change datatype for undisclosed amount field in payments |
-| `2.0.0` | 2023-04-11 | [25045](https://github.com/airbytehq/airbyte/pull/25045) | Fix datetime format, disable OAuth button in cloud       |
-| `1.0.0` | 2023-03-20 | [24324](https://github.com/airbytehq/airbyte/pull/24324) | Migrate to Low-Code                                      |
-| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks`                |
-| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960)   | Update connector fields title/description                |
-| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986)   | Using number data type for decimal fields instead string |
-| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539)   | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support          |
+| Version | Date       | Pull Request                                             | Subject                                                            |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------- |
+| `2.0.4` | 2023-06-28 | [27803](https://github.com/airbytehq/airbyte/pull/27803) | Update following state breaking changes                            |
+| `2.0.3` | 2023-06-08 | [27148](https://github.com/airbytehq/airbyte/pull/27148) | Update description and example values of a Start Date in spec.json |
+| `2.0.2` | 2023-06-07 | [26722](https://github.com/airbytehq/airbyte/pull/27053) | Update CDK version and adjust authenticator configuration          |
+| `2.0.1` | 2023-05-28 | [26722](https://github.com/airbytehq/airbyte/pull/26722) | Change datatype for undisclosed amount field in payments           |
+| `2.0.0` | 2023-04-11 | [25045](https://github.com/airbytehq/airbyte/pull/25045) | Fix datetime format, disable OAuth button in cloud                 |
+| `1.0.0` | 2023-03-20 | [24324](https://github.com/airbytehq/airbyte/pull/24324) | Migrate to Low-Code                                                |
+| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks`                          |
+| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960)   | Update connector fields title/description                          |
+| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986)   | Using number data type for decimal fields instead string           |
+| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539)   | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support                    |
diff --git a/docs/integrations/sources/rd-station-marketing.md b/docs/integrations/sources/rd-station-marketing.md
index 193192d0eb33..bf649567fca7 100644
--- a/docs/integrations/sources/rd-station-marketing.md
+++ b/docs/integrations/sources/rd-station-marketing.md
@@ -39,7 +39,8 @@ Each endpoint has its own performance limitations, which also consider the accou
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                         |
-| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------- |
-| 0.1.1   | 2022-11-01 | [18826](https://github.com/airbytehq/airbyte/pull/18826) | Fix stream analytics_conversions                                |
-| 0.1.0   | 2022-10-23 | [18348](https://github.com/airbytehq/airbyte/pull/18348) | Initial Release                                                 |
+| Version | Date       | Pull Request                                              | Subject                          |
+|:--------|:-----------|:----------------------------------------------------------|:---------------------------------|
+| 0.1.2   | 2022-07-06 | [28009](https://github.com/airbytehq/airbyte/pull/28009/) | Migrated to advancedOAuth        |
+| 0.1.1   | 2022-11-01 | [18826](https://github.com/airbytehq/airbyte/pull/18826)  | Fix stream analytics_conversions |
+| 0.1.0   | 2022-10-23 | [18348](https://github.com/airbytehq/airbyte/pull/18348)  | Initial Release                  |
diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md
index 19909f5ddcc4..ce15a8559e2a 100644
--- a/docs/integrations/sources/recharge.md
+++ b/docs/integrations/sources/recharge.md
@@ -74,23 +74,25 @@ The Recharge connector should gracefully handle Recharge API limitations under n
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------|
-| 0.2.9   | 2023-04-10 | [25009](https://github.com/airbytehq/airbyte/pull/25009) | Fix owner slicing for `Metafields` stream                                                  |      
-| 0.2.8   | 2023-04-07 | [24990](https://github.com/airbytehq/airbyte/pull/24990) | Add slicing to connector                                                                   |      
-| 0.2.7   | 2023-02-13 | [22901](https://github.com/airbytehq/airbyte/pull/22901) | Specified date formatting in specification                                                 |      
-| 0.2.6   | 2023-02-21 | [22473](https://github.com/airbytehq/airbyte/pull/22473) | Use default availability strategy                                                          |              
-| 0.2.5   | 2023-01-27 | [22021](https://github.com/airbytehq/airbyte/pull/22021) | Set `AvailabilityStrategy` for streams explicitly to `None`                                |
-| 0.2.4   | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry`                                                        |
-| 0.2.3   | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry`                                                        |
-| 0.2.2   | 2022-10-05 | [17608](https://github.com/airbytehq/airbyte/pull/17608) | Skip stream if we receive 403 error                                                        |
-| 0.2.2   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state.                                                               |
-| 0.2.1   | 2022-09-23 | [17080](https://github.com/airbytehq/airbyte/pull/17080) | Fix `total_weight` value to be `int` instead of `float`                                    |
-| 0.2.0   | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types                      |
-| 0.1.8   | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer                                                        |
-| 0.1.7   | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities                     |
-| 0.1.6   | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint  |
-| 0.1.5   | 2022-01-26 | [9808](https://github.com/airbytehq/airbyte/pull/9808)   | Update connector fields title/description                                                  |
-| 0.1.4   | 2021-11-05 | [7626](https://github.com/airbytehq/airbyte/pull/7626)   | Improve 'backoff' for HTTP requests                                                        |
-| 0.1.3   | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149)   | Update `discount` and `order` schema                                                       |
-| 0.1.2   | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149)   | Change `cursor_field` for Incremental streams                                              |
+| Version | Date       | Pull Request                                             | Subject                                                                                   |
+|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------|
+| 1.0.0   | 2023-06-22 | [27612](https://github.com/airbytehq/airbyte/pull/27612) | Change data type of the `shopify_variant_id_not_found` field of the `Charges` stream      |
+| 0.2.10  | 2023-06-20 | [27503](https://github.com/airbytehq/airbyte/pull/27503) | Update API version to 2021-11                                                             |      
+| 0.2.9   | 2023-04-10 | [25009](https://github.com/airbytehq/airbyte/pull/25009) | Fix owner slicing for `Metafields` stream                                                 |      
+| 0.2.8   | 2023-04-07 | [24990](https://github.com/airbytehq/airbyte/pull/24990) | Add slicing to connector                                                                  |      
+| 0.2.7   | 2023-02-13 | [22901](https://github.com/airbytehq/airbyte/pull/22901) | Specified date formatting in specification                                                |      
+| 0.2.6   | 2023-02-21 | [22473](https://github.com/airbytehq/airbyte/pull/22473) | Use default availability strategy                                                         |              
+| 0.2.5   | 2023-01-27 | [22021](https://github.com/airbytehq/airbyte/pull/22021) | Set `AvailabilityStrategy` for streams explicitly to `None`                               |
+| 0.2.4   | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry`                                                       |
+| 0.2.3   | 2022-10-11 | [17822](https://github.com/airbytehq/airbyte/pull/17822) | Do not parse JSON in `should_retry`                                                       |
+| 0.2.2   | 2022-10-05 | [17608](https://github.com/airbytehq/airbyte/pull/17608) | Skip stream if we receive 403 error                                                       |
+| 0.2.2   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state.                                                              |
+| 0.2.1   | 2022-09-23 | [17080](https://github.com/airbytehq/airbyte/pull/17080) | Fix `total_weight` value to be `int` instead of `float`                                   |
+| 0.2.0   | 2022-09-21 | [16959](https://github.com/airbytehq/airbyte/pull/16959) | Use TypeTransformer to reliably convert to schema declared data types                     |
+| 0.1.8   | 2022-08-27 | [16045](https://github.com/airbytehq/airbyte/pull/16045) | Force total_weight to be an integer                                                       |
+| 0.1.7   | 2022-07-24 | [14978](https://github.com/airbytehq/airbyte/pull/14978) | Set `additionalProperties` to True, to guarantee backward cababilities                    |
+| 0.1.6   | 2022-07-21 | [14902](https://github.com/airbytehq/airbyte/pull/14902) | Increased test coverage, fixed broken `charges`, `orders` schemas, added state checkpoint |
+| 0.1.5   | 2022-01-26 | [9808](https://github.com/airbytehq/airbyte/pull/9808)   | Update connector fields title/description                                                 |
+| 0.1.4   | 2021-11-05 | [7626](https://github.com/airbytehq/airbyte/pull/7626)   | Improve 'backoff' for HTTP requests                                                       |
+| 0.1.3   | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149)   | Update `discount` and `order` schema                                                      |
+| 0.1.2   | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149)   | Change `cursor_field` for Incremental streams                                             |
diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md
index 8c7ca149e69d..dafe396d2684 100644
--- a/docs/integrations/sources/redshift.md
+++ b/docs/integrations/sources/redshift.md
@@ -14,20 +14,22 @@ The Redshift source does not alter the schema present in your warehouse. Dependi
 
 ### Features
 
-| Feature | Supported | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental Sync | Coming soon |  |
-| Replicate Incremental Deletes | Coming soon |  |
-| Logical Replication \(WAL\) | Coming soon |  |
-| SSL Support | Yes |  |
-| SSH Tunnel Connection | Coming soon |  |
-| Namespaces | Yes | Enabled by default |
-| Schema Selection | Yes | Multiple schemas may be used at one time. Keep empty to process all of existing schemas |
+| Feature                       | Supported                 | Notes                                                                                   |
+| :---------------------------- | :------------------------ | :-------------------------------------------------------------------------------------- |
+| Full Refresh Sync             | Yes                       |                                                                                         |
+| Incremental Sync              | Yes                       | Cursor-based, using `ORDER BY` on a user-defined cursor column                          |
+| Replicate Incremental Deletes | Not supported in Redshift |                                                                                         |
+| Logical Replication \(WAL\)   | Not supported in Redshift |                                                                                         |
+| SSL Support                   | Yes                       |                                                                                         |
+| SSH Tunnel Connection         | No                        |                                                                                         |
+| Namespaces                    | Yes                       | Enabled by default                                                                      |
+| Schema Selection              | Yes                       | Multiple schemas may be used at one time. Keep empty to process all of existing schemas |
 
 #### Incremental Sync
 
-Incremental sync \(copying only the data that has changed\) for this source is coming soon.
+The Redshift source connector supports incremental syncs. To setup an incremental sync for a table in Redshift in the Airbyte UI, you must setup a [user-defined cursor field](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append/#user-defined-cursor) such as an `updated_at` column. The connector relies on this column to know which records were updated since the last sync it ran. See the [incremental sync docs](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) for more information.
+
+Defining a cursor field allows you to run incremental-append syncs. To run [incremental-dedupe](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) syncs, you'll need to tell the connector which column(s) to use as a primary key. See the [incremental-dedupe sync docs](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) for more information.
 
 ## Getting started
 
@@ -52,20 +54,22 @@ All Redshift connections are encrypted using SSL
 
 ## Changelog
 
-| Version | Date       | Pull Request | Subject                                                                                                   |
-|:--------|:-----------| :-----       |:----------------------------------------------------------------------------------------------------------|
-| 0.3.16  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)   | Consolidate date/time values mapping for JDBC sources                          |
+| Version | Date       | Pull Request                                             | Subject                                                                                                                                   |
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.4.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2                                                                                                                      |
+| 0.3.17  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                  |
+| 0.3.16  | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources                                                                                     |
 | 0.3.15  | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
-| 0.3.14  | 2022-09-01 | [16258](https://github.com/airbytehq/airbyte/pull/16258) | Emit state messages more frequently |
-| 0.3.13  | 2022-05-25 |  | Added JDBC URL params                                                                                     |
-| 0.3.12  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field |
-| 0.3.11  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors                                            |
-| 0.3.10  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption                                 |0
-| 0.3.9   | 2022-02-21 | [9744](https://github.com/airbytehq/airbyte/pull/9744) | List only the tables on which the user has SELECT permissions.                                            
-| 0.3.8   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                              |
-| 0.3.7   | 2022-01-26 | [9721](https://github.com/airbytehq/airbyte/pull/9721) | Added schema selection                                                                                    |
-| 0.3.6   | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617) | Update connector fields title/description                                                                 |
-| 0.3.5   | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY                                                                            |
-| 0.3.4   | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234) | Allow SSL traffic only                                                                                    |
-| 0.3.3   | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965) | Added SSL Support                                                                                         |
-| 0.3.2   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator                                                                               |
+| 0.3.14  | 2022-09-01 | [16258](https://github.com/airbytehq/airbyte/pull/16258) | Emit state messages more frequently                                                                                                       |
+| 0.3.13  | 2022-05-25 |                                                          | Added JDBC URL params                                                                                                                     |
+| 0.3.12  | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field                                 |
+| 0.3.11  | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors                                                                            |
+| 0.3.10  | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption                                                                 |
+| 0.3.9   | 2022-02-21 | [9744](https://github.com/airbytehq/airbyte/pull/9744)   | List only the tables on which the user has SELECT permissions.                                                                            |
+| 0.3.8   | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option                                                                                              |
+| 0.3.7   | 2022-01-26 | [9721](https://github.com/airbytehq/airbyte/pull/9721)   | Added schema selection                                                                                                                    |
+| 0.3.6   | 2022-01-20 | [8617](https://github.com/airbytehq/airbyte/pull/8617)   | Update connector fields title/description                                                                                                 |
+| 0.3.5   | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958)   | Add support for JdbcType.ARRAY                                                                                                            |
+| 0.3.4   | 2021-10-21 | [7234](https://github.com/airbytehq/airbyte/pull/7234)   | Allow SSL traffic only                                                                                                                    |
+| 0.3.3   | 2021-10-12 | [6965](https://github.com/airbytehq/airbyte/pull/6965)   | Added SSL Support                                                                                                                         |
+| 0.3.2   | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699)   | Added json config validator                                                                                                               |
diff --git a/docs/integrations/sources/retently.md b/docs/integrations/sources/retently.md
index c89ce4e127ba..6d885a03c4c3 100644
--- a/docs/integrations/sources/retently.md
+++ b/docs/integrations/sources/retently.md
@@ -11,6 +11,11 @@ Several output streams are available from this source:
 - [Customers](https://www.retently.com/api/#api-get-customers-get)
 - [Companies](https://www.retently.com/api/#api-get-companies-get)
 - [Reports](https://www.retently.com/api/#api-get-reports-get)
+- [Campaigns](https://www.retently.com/api/#api-get-campaigns)
+- [Feedback](https://www.retently.com/api/#api-get-feedback-get)
+- [NPS](https://www.retently.com/api/#api-get-latest-score)
+- [Outbox](https://www.retently.com/api/#api-get-sent-surveys)
+- [Templates](https://www.retently.com/api/#api-get-templates-get)
 
 If there are more endpoints you'd like Airbyte to support, please [create an issue](https://github.com/airbytehq/airbyte/issues/new/choose).
 
@@ -41,6 +46,7 @@ OAuth application is [here](https://app.retently.com/settings/oauth).
 
 | Version | Date       | Pull Request                                             | Subject                                              |
 | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------- |
+| 0.2.0   | 2023-08-03 | [29040](https://github.com/airbytehq/airbyte/pull/29040) | Migrate to Low-Code CDK                              |
 | 0.1.6   | 2023-05-10 | [25714](https://github.com/airbytehq/airbyte/pull/25714) | Fix invalid json schema for nps stream               |
 | 0.1.5   | 2023-05-08 | [25900](https://github.com/airbytehq/airbyte/pull/25900) | Fix integration tests                                |
 | 0.1.4   | 2023-05-08 | [25900](https://github.com/airbytehq/airbyte/pull/25900) | Fix integration tests                                |
diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md
index 264a7f688ca3..70ae53a88630 100644
--- a/docs/integrations/sources/s3.md
+++ b/docs/integrations/sources/s3.md
@@ -3,45 +3,66 @@
 This page contains the setup guide and reference information for the Amazon S3 source connector.
 
 :::info
-Cloud storage may incur egress costs. Egress refers to data that is transferred out of the cloud storage system, such as when you download files or access them from a different location. For more information, see the [Amazon S3 pricing guide](https://aws.amazon.com/s3/pricing/).
+Please note that using cloud storage may incur egress costs. Egress refers to data that is transferred out of the cloud storage system, such as when you download files or access them from a different location. For detailed information on egress costs, please consult the [Amazon S3 pricing guide](https://aws.amazon.com/s3/pricing/).
 :::
 
 ## Prerequisites
 
-Define file pattern, see the [Path Patterns section](s3.md#path-patterns)
+- Access to the S3 bucket containing the files to replicate.
 
 ## Setup guide
 
 ### Step 1: Set up Amazon S3
 
-* If syncing from a private bucket, the credentials you use for the connection must have have both `read` and `list` access on the S3 bucket. `list` is required to discover files based on the provided pattern\(s\).
+**If you are syncing from a private bucket**, you will need to provide your `AWS Access Key ID` and `AWS Secret Access Key` to authenticate the connection, and ensure that the IAM user associated with the credentials has `read` and `list` permissions for the bucket. If you are unfamiliar with configuring AWS permissions, you can follow these steps to obtain the necessary permissions and credentials:
 
-### Step 2: Set up the Amazon S3 connector in Airbyte
+1. Log in to your Amazon AWS account and open the [IAM console](https://console.aws.amazon.com/iam/home#home).
+2. In the IAM dashboard, select **Policies**, then click **Create Policy**.
+3. Select the **JSON** tab, then paste the following JSON into the Policy editor (be sure to substitute in your bucket name):
+
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+        "Effect": "Allow",
+        "Action": [
+                "s3:GetObject",
+                "s3:ListBucket"
+        ],
+        "Resource": [
+                "arn:aws:s3:::{your-bucket-name}/*",
+                "arn:aws:s3:::{your-bucket-name}"
+        ]
+        }
+    ]
+}
+```
 
-
-**For Airbyte Cloud:**
+4. Give your policy a descriptive name, then click **Create policy**.
+5. In the IAM dashboard, click **Users**. Select an existing IAM user or create a new one by clicking **Add users**.
+6. If you are using an _existing_ IAM user, click the **Add permissions** dropdown menu and select **Add permissions**. If you are creating a _new_ user, you will be taken to the Permissions screen after selecting a name.
+7. Select **Attach policies directly**, then find and check the box for your new policy. Click **Next**, then **Add permissions**.
+8. After successfully creating your user, select the **Security credentials** tab and click **Create access key**. You will be prompted to select a use case and add optional tags to your access key. Click **Create access key** to generate the keys.
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click ****. In the top-right corner, click **+new source/destination**.
-3. On the Set up the source/destination page, enter the name for the `connector name` connector and select **connector name** from the `Source/Destination` type dropdown.
-4. Set `dataset` appropriately. This will be the name of the table in the destination.
-5. If your bucket contains _only_ files containing data for this table, use `**` as path\_pattern. See the [Path Patterns section](s3.md#path-patterns) for more specific pattern matching.
-6. Leave schema as `{}` to automatically infer it from the file\(s\). For details on providing a schema, see the [User Schema section](s3.md#user-schema).
-7. Fill in the fields within the provider box appropriately. If your bucket is not public, add [credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) with sufficient permissions under `aws_access_key_id` and `aws_secret_access_key`.
-8. Choose the format corresponding to the format of your files and fill in fields as required. If unsure about values, try out the defaults and come back if needed. Find details on these settings [here](s3.md#file-format-settings).
-
+:::caution
+Your `Secret Access Key` will only be visible once upon creation. Be sure to copy and store it securely for future use.
+:::
 
-
-**For Airbyte Open Source:**
+For more information on managing your access keys, please refer to the 
+[official AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html).
 
-1. Create a new S3 source with a suitable name. Since each S3 source maps to just a single table, it may be worth including that in the name.
-2. Set `dataset` appropriately. This will be the name of the table in the destination.
-3. If your bucket contains _only_ files containing data for this table, use `**` as path\_pattern. See the [Path Patterns section](s3.md#path-patterns) for more specific pattern matching.
-4. Leave schema as `{}` to automatically infer it from the file\(s\). For details on providing a schema, see the [User Schema section](s3.md#user-schema).
-5. Fill in the fields within the provider box appropriately. If your bucket is not public, add [credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) with sufficient permissions under `aws_access_key_id` and `aws_secret_access_key`.
-6. Choose the format corresponding to the format of your files and fill in fields as required. If unsure about values, try out the defaults and come back if needed. Find details on these settings [here](s3.md#file-format-settings).
-
+### Step 2: Set up the Amazon S3 connector in Airbyte
 
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **S3** from the list of available sources.
+4. Enter the **Output Stream Name**. This will be the name of the table in the destination (can contain letters, numbers and underscores).
+5. Enter the **Pattern of files to replicate**. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below.
+6. Enter the name of the **Bucket** containing your files to replicate.
+7. Toggle the **Optional fields** under the **Bucket** field to expand additional configuration options. **If you are syncing from a private bucket**, you must fill the **AWS Access Key ID** and **AWS Secret Access Key** fields with the appropriate credentials to authenticate the connection. All other fields are optional and can be left empty. Refer to the [S3 Provider Settings section](#s3-provider-settings) below for more information on each field.
+8. In the **File Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **File Format** box will allow you to enter additional configurations based on the selected format.  For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below.
+6. (Optional) - If you want to enforce a specific schema, you can enter a **Manually enforced data schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema).
 
 ## Supported sync modes
 
@@ -56,7 +77,6 @@ The Amazon S3 source connector supports the following [sync modes](https://docs.
 | Replicate Multiple Streams \(distinct tables\) | No         |
 | Namespaces                                     | No         |
 
-
 ## File Compressions
 
 | Compression | Supported? |
@@ -70,15 +90,14 @@ The Amazon S3 source connector supports the following [sync modes](https://docs.
 
 Please let us know any specific compressions you'd like to see support for next!
 
-
 ## Path Patterns
 
 \(tl;dr -> path pattern syntax using [wcmatch.glob](https://facelessuser.github.io/wcmatch/glob/). GLOBSTAR and SPLIT flags are enabled.\)
 
 This connector can sync multiple files by using glob-style patterns, rather than requiring a specific path for every file. This enables:
 
-* Referencing many files with just one pattern, e.g. `**` would indicate every file in the bucket.
-* Referencing future files that don't exist yet \(and therefore don't have a specific path\).
+- Referencing many files with just one pattern, e.g. `**` would indicate every file in the bucket.
+- Referencing future files that don't exist yet \(and therefore don't have a specific path\).
 
 You must provide a path pattern. You can also provide many patterns split with \| for more complex directory layouts.
 
@@ -86,15 +105,15 @@ Each path pattern is a reference from the _root_ of the bucket, so don't include
 
 Some example patterns:
 
-* `**` : match everything.
-* `**/*.csv` : match all files with specific extension.
-* `myFolder/**/*.csv` : match all csv files anywhere under myFolder.
-* `*/**` : match everything at least one folder deep.
-* `*/*/*/**` : match everything at least three folders deep.
-* `**/file.*|**/file` : match every file called "file" with any extension \(or no extension\).
-* `x/*/y/*` : match all files that sit in folder x -> any folder -> folder y.
-* `**/prefix*.csv` : match all csv files with specific prefix.
-* `**/prefix*.parquet` : match all parquet files with specific prefix.
+- `**` : match everything.
+- `**/*.csv` : match all files with specific extension.
+- `myFolder/**/*.csv` : match all csv files anywhere under myFolder.
+- `*/**` : match everything at least one folder deep.
+- `*/*/*/**` : match everything at least three folders deep.
+- `**/file.*|**/file` : match every file called "file" with any extension \(or no extension\).
+- `x/*/y/*` : match all files that sit in folder x -> any folder -> folder y.
+- `**/prefix*.csv` : match all csv files with specific prefix.
+- `**/prefix*.parquet` : match all parquet files with specific prefix.
 
 Let's look at a specific example, matching the following bucket layout:
 
@@ -112,39 +131,38 @@ myBucket
             -> another_part1.csv
 ```
 
-We want to pick up part1.csv, part2.csv and part3.csv \(excluding another\_part1.csv for now\). We could do this a few different ways:
+We want to pick up part1.csv, part2.csv and part3.csv \(excluding another_part1.csv for now\). We could do this a few different ways:
 
-* We could pick up every csv file called "partX" with the single pattern `**/part*.csv`.
-* To be a bit more robust, we could use the dual pattern `some_table_files/*.csv|more_table_files/*.csv` to pick up relevant files only from those exact folders.
-* We could achieve the above in a single pattern by using the pattern `*table_files/*.csv`. This could however cause problems in the future if new unexpected folders started being created.
-* We can also recursively wildcard, so adding the pattern `extras/**/*.csv` would pick up any csv files nested in folders below "extras", such as "extras/misc/another\_part1.csv".
+- We could pick up every csv file called "partX" with the single pattern `**/part*.csv`.
+- To be a bit more robust, we could use the dual pattern `some_table_files/*.csv|more_table_files/*.csv` to pick up relevant files only from those exact folders.
+- We could achieve the above in a single pattern by using the pattern `*table_files/*.csv`. This could however cause problems in the future if new unexpected folders started being created.
+- We can also recursively wildcard, so adding the pattern `extras/**/*.csv` would pick up any csv files nested in folders below "extras", such as "extras/misc/another_part1.csv".
 
 As you can probably tell, there are many ways to achieve the same goal with path patterns. We recommend using a pattern that ensures clarity and is robust against future additions to the directory structure.
 
-
 ## User Schema
 
 Providing a schema allows for more control over the output of this stream. Without a provided schema, columns and datatypes will be inferred from the first created file in the bucket matching your path pattern and suffix. This will probably be fine in most cases but there may be situations you want to enforce a schema instead, e.g.:
 
-* You only care about a specific known subset of the columns. The other columns would all still be included, but packed into the `_ab_additional_properties` map.
-* Your initial dataset is quite small \(in terms of number of records\), and you think the automatic type inference from this sample might not be representative of the data in the future.
-* You want to purposely define types for every column.
-* You know the names of columns that will be added to future data and want to include these in the core schema as columns rather than have them appear in the `_ab_additional_properties` map.
+- You only care about a specific known subset of the columns. The other columns would all still be included, but packed into the `_ab_additional_properties` map.
+- Your initial dataset is quite small \(in terms of number of records\), and you think the automatic type inference from this sample might not be representative of the data in the future.
+- You want to purposely define types for every column.
+- You know the names of columns that will be added to future data and want to include these in the core schema as columns rather than have them appear in the `_ab_additional_properties` map.
 
 Or any other reason! The schema must be provided as valid JSON as a map of `{"column": "datatype"}` where each datatype is one of:
 
-* string
-* number
-* integer
-* object
-* array
-* boolean
-* null
+- string
+- number
+- integer
+- object
+- array
+- boolean
+- null
 
 For example:
 
-* {"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}
-* {"username": "string", "friends": "array", "information": "object"}
+- {"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}
+- {"username": "string", "friends": "array", "information": "object"}
 
 :::note
 
@@ -152,73 +170,131 @@ Please note, the S3 Source connector used to infer schemas from all the availabl
 
 :::
 
-
 ## S3 Provider Settings
 
-* `bucket` : name of the bucket your files are in
-* `aws_access_key_id` : one half of the [required credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) for accessing a private bucket.
-* `aws_secret_access_key` : other half of the [required credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) for accessing a private bucket.
-* `path_prefix` : an optional string that limits the files returned by AWS when listing files to only that those starting with this prefix. This is different to path\_pattern as it gets pushed down to the API call made to S3 rather than filtered in Airbyte and it does not accept pattern-style symbols \(like wildcards `*`\). We recommend using this if your bucket has many folders and files that are unrelated to this stream and all the relevant files will always sit under this chosen prefix.
-  * Together with `path_pattern`, there are multiple ways to specify the files to sync. For example, all the following configs are equivalent:
-    * `path_prefix` = ``, `path_pattern` = `path1/path2/myFolder/**/*`.
-    * `path_prefix` = `path1/`, `path_pattern` = `path2/myFolder/**/*.csv`.
-    * `path_prefix` = `path1/path2/` and `path_pattern` = `myFolder/**/*.csv`
-    * `path_prefix` = `path1/path2/myFolder/`, `path_pattern` = `**/*.csv`. This is the most efficient one because the directories are filtered earlier in the S3 API call. However, the difference in efficiency is usually negligible.
-  * The rationale of having both `path_prefix` and `path_pattern` is to accommodate as many use cases as possible. If you found them confusing, feel free to ignore `path_prefix` and just set the `path_pattern`.
-* `endpoint` : optional parameter that allow using of non Amazon S3 compatible services. Leave it blank for using default Amazon serivce.
-* `use_ssl` : Allows using custom servers that configured to use plain http. Ignored in case of using Amazon service.
-* `verify_ssl_cert` : Skip ssl validity check in case of using custom servers with self signed certificates. Ignored in case of using Amazon service.
+- **AWS Access Key ID**: One half of the [required credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) for accessing a private bucket.
+- **AWS Secret Access Key**: The other half of the [required credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) for accessing a private bucket.
+- **Path Prefix**: An optional string that limits the files returned by AWS when listing files to only those starting with the specified prefix. This is different than the **Path Pattern**, as the prefix is applied directly to the API call made to S3, rather than being filtered within Airbyte. **This is not a regular expression** and does not accept pattern-style symbols like wildcards (`*`). We recommend using this filter to improve performance if the connector if your bucket has many folders and files that are unrelated to the data you want to replicate, and all the relevant files will always reside under the specified prefix.
+  - Together with the **Path Pattern**, there are multiple ways to specify the files to sync. For example, all the following configurations are equivalent:
+    - **Prefix** = ``, **Pattern** = `path1/path2/myFolder/**/*`
+    - **Prefix** = `path1/`, **Pattern** = `path2/myFolder/**/*.csv`
+    - **Prefix** = `path1/path2/`, **Pattern** = `myFolder/**/*.csv`
+    - **Prefix** = `path1/path2/myFolder/`, **Pattern** = `**/*.csv`
 
-  **File Format Settings**
+  - The ability to individually configure the prefix and pattern has been included to accommodate situations where you do not want to replicate the majority of the files in the bucket. If you are unsure of the best approach, you can safely leave the **Path Prefix** field empty and just [set the Path Pattern](#path-patterns) to meet your requirements.
+- **Endpoint**: An optional parameter that enables the use of non-Amazon S3 compatible services. If you are using the default Amazon service, leave this field blank.
+- **Start Date**: An optional parameter that marks a starting date and time in UTC for data replication. Any files that have _not_ been modified since this specified date/time will _not_ be replicated. Use the provided datepicker (recommended) or enter the desired date programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. Leaving this field blank will replicate data from all files that have not been excluded by the **Path Pattern** and **Path Prefix**.
 
-  The Reader in charge of loading the file format is currently based on [PyArrow](https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html) \(Apache Arrow\).
+## File Format Settings
 
-  Note that all files within one stream must adhere to the same read options for every provided format.
+The Reader in charge of loading the file format is currently based on [PyArrow](https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html) \(Apache Arrow\).
+
+:::note
+All files within one stream must adhere to the same read options for every provided format.
+:::
 
 ### CSV
 
 Since CSV files are effectively plain text, providing specific reader options is often required for correct parsing of the files. These settings are applied when a CSV is created or exported so please ensure that this process happens consistently over time.
 
-* `delimiter` : Even though CSV is an acronymn for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator.
-* `quote_char` : In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can allow this behaviour by wrapping a value in defined quote characters so that on read it can parse it correctly.
-* `escape_char` : An escape character can be used to prefix a reserved character and allow correct parsing.
-* `encoding` : Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings).
-* `double_quote` : Whether two quotes in a quoted CSV value denote a single quote in the data.
-* `newlines_in_values` : Sometimes referred to as `multiline`. In most cases, newline characters signal the end of a row in a CSV, however text data may contain newline characters within it. Setting this to True allows correct parsing in this case.
-* `block_size` : This is the number of bytes to process in memory at a time while reading files. The default value here is usually fine but if your table is particularly wide \(lots of columns / data in fields is large\) then raising this might solve failures on detecting schema. Since this defines how much data to read into memory, raising this too high could cause Out Of Memory issues so use with caution.
-* `additional_reader_options` : This allows for editing the less commonly required CSV [ConvertOptions](https://arrow.apache.org/docs/python/generated/pyarrow.csv.ConvertOptions.html#pyarrow.csv.ConvertOptions). The value must be a valid JSON string, e.g.:
+- **Delimiter**: Even though CSV is an acronym for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator. To use [tab-delimiters](https://en.wikipedia.org/wiki/Tab-separated_values), you can set this value to `\t`. By default, this value is set to `,`.
+- **Infer Datatypes**: This option determines whether a schema for the source should be inferred from the current data. When set to False and a custom schema is provided, the manually enforced schema takes precedence. If no custom schema is set and this option is set to False, all fields will be read as strings. Set to True by default.
+- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`.
+- **Escape Character**: An escape character can be used to prefix a reserved character and ensure correct parsing. A commonly used character is the backslash (`\`). For example, given the following data:
 
-    ```text
-    {"timestamp_parsers": ["%m/%d/%Y %H:%M", "%Y/%m/%d %H:%M"], "strings_can_be_null": true, "null_values": ["NA", "NULL"]}
-    ```
-* `advanced_options` : This allows for editing the less commonly required CSV [ReadOptions](https://arrow.apache.org/docs/python/generated/pyarrow.csv.ReadOptions.html#pyarrow.csv.ReadOptions). The value must be a valid JSON string. One use case for this is when your CSV has no header, or you want to use custom column names, you can specify `column_names` using this option.
+```
+Product,Description,Price
+Jeans,"Navy Blue, Bootcut, 34\"",49.99
+```
 
-    ```test
-    {"column_names": ["column1", "column2", "column3"]}
-    ```
+The backslash (`\`) is used directly before the second double quote (`"`) to indicate that it is _not_ the closing quote for the field, but rather a literal double quote character that should be included in the value (in this example, denoting the size of the jeans in inches: `34"` ). 
+
+Leaving this field blank (default option) will disallow escaping.
+
+- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`.
+- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default.
+- **Allow newlines in values**: Also known as _multiline_, this option addresses situations where newline characters occur within text data. Typically, newline characters signify the end of a row in a CSV, but when this option is set to True, parsing correctly handles newlines within values. Set to False by default.
+- **Additional Reader Options**: This allows for editing the less commonly used CSV [ConvertOptions](https://arrow.apache.org/docs/python/generated/pyarrow.csv.ConvertOptions.html#pyarrow.csv.ConvertOptions). The value must be a valid JSON string, e.g.:
+
+  ```
+  {
+    "timestamp_parsers": [
+    "%m/%d/%Y %H:%M", "%Y/%m/%d %H:%M"
+    ], 
+    "strings_can_be_null": true, 
+    "null_values": [
+      "NA", "NULL"
+    ]
+  }
+  ```
+
+- **Advanced Options**: This allows for editing the less commonly used CSV [ReadOptions](https://arrow.apache.org/docs/python/generated/pyarrow.csv.ReadOptions.html#pyarrow.csv.ReadOptions). The value must be a valid JSON string. One use case for this is when your CSV has no header, or if you want to use custom column names. You can specify `column_names` using this option. For example:
+
+  ```
+  {
+    "column_names": [
+      "column1", "column2", "column3"
+    ]
+  }
+  ```
+
+- **Block Size**: This is the number of bytes to process in memory at a time while reading files. The default value of `10000` is usually suitable, but if your files are particularly wide (lots of columns, or the values in the columns are particularly large), increasing this might help avoid schema detection failures.
+
+:::caution
+Be cautious when raising this value too high, as it may result in Out Of Memory issues due to increased memory usage.
+:::
 
 ### Parquet
 
-Apache Parquet file is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. For now, the solution involves iterating through individual files at the abstract level thus partitioned parquet datasets are unsupported. The following settings are available:
+Apache Parquet is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. At the moment, partitioned parquet datasets are unsupported. The following settings are available:
 
-* `buffer_size` : If positive, perform read buffering when deserializing individual column chunks. Otherwise IO calls are unbuffered.
-* `columns` : If not None, only these columns will be read from the file.
-* `batch_size` : Maximum number of records per batch. Batches may be smaller if there aren’t enough rows in the file.
+- **Selected Columns**: If you only want to sync a subset of the columns from the file(s), enter the desired columns here as a comma-delimited list. Leave this field empty to sync all columns.
+- **Record Batch Size**: Sets the maximum number of records per batch. Batches may be smaller if there aren’t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide. Set to `65536` by default.
+- **Buffer Size**: If set to a positive number, read buffering is performed during the deserializing of individual column chunks. Otherwise I/O calls are unbuffered. Set to `2` by default.
 
-You can find details on [here](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html#pyarrow.parquet.ParquetFile.iter_batches).
+For more information on these fields, please refer to the [Apache documentation](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html#pyarrow.parquet.ParquetFile.iter_batches).
 
 ### Avro
 
-The avro parser uses [fastavro](https://fastavro.readthedocs.io/en/latest/). Currently, no additional options are supported.
+The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/latest/). Currently, no additional options are supported.
+
+### JSONL
+
+The JSONL parser uses the PyArrow library, which only supports the line-delimited JSON format. For more detailed info, please refer to the [official docs](https://arrow.apache.org/docs/python/json.html).
 
-### Jsonl
+- **Allow newlines in values**: While JSONL typically has each JSON object on a separate line, there are cases where newline characters may appear within JSON values, such as multiline strings. This option enables the parser to correctly interpret and treat newline characters within values. Please note that setting this option to True may affect performance. Set to False by default.
 
-The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is supported.For more detailed info, please refer to the [docs](https://arrow.apache.org/docs/python/generated/pyarrow.json.read_json.html)
+- **Unexpected field behavior**: This parameter determines how any JSON fields outside of the explicit schema (if defined) are handled. Possible behaviors include:
+
+  - `ignore`: Unexpected JSON fields are ignored.
+  - `error`: Error out on unexpected JSON fields.
+  - `infer`: Unexpected JSON fields are type-inferred and included in the output.
+
+Set to `infer` by default.
+
+- **Block Size**: This sets the number of bytes to process in memory at a time while reading files. The default value of `10000` is usually suitable, but if your files are particularly wide (lots of columns or the values in the columns are particularly large), increasing this might help avoid schema detection failures.
+
+:::caution
+Be cautious when raising this value too high, as it may result in Out Of Memory issues due to increased memory usage.
+:::
 
 ## Changelog
 
 | Version | Date       | Pull Request                                                                                                    | Subject                                                                                                              |
 |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------|
+| 3.1.9   | 2023-08-23 | [29753](https://github.com/airbytehq/airbyte/pull/29753)                                                        | Feature parity update for V4 release                                                                                 |
+| 3.1.8   | 2023-08-17 | [29520](https://github.com/airbytehq/airbyte/pull/29520)                                                        | Update legacy state and error handling                                                                               |
+| 3.1.7   | 2023-08-17 | [29505](https://github.com/airbytehq/airbyte/pull/29505)                                                        | v4 StreamReader and Cursor fixes                                                                                     |
+| 3.1.6   | 2023-08-16 | [29480](https://github.com/airbytehq/airbyte/pull/29480)                                                        | update Pyarrow to version 12.0.1                                                                                     |
+| 3.1.5   | 2023-08-15 | [29418](https://github.com/airbytehq/airbyte/pull/29418)                                                        | Avoid duplicate syncs when migrating from v3 to v4                                                                   |
+| 3.1.4   | 2023-08-15 | [29382](https://github.com/airbytehq/airbyte/pull/29382)                                                        | Handle legacy path prefix & path pattern                                                                             |
+| 3.1.3   | 2023-08-05 | [29028](https://github.com/airbytehq/airbyte/pull/29028)                                                        | Update v3 & v4 connector to handle either state message                                                              |
+| 3.1.2   | 2023-07-29 | [28786](https://github.com/airbytehq/airbyte/pull/28786)                                                        | Add a codepath for using the file-based CDK                                                                          |
+| 3.1.1   | 2023-07-26 | [28730](https://github.com/airbytehq/airbyte/pull/28730)                                                        | Add human readable error message and improve validation for encoding field when it empty                             |
+| 3.1.0   | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725)                                                        | License Update: Elv2                                                                                                 |
+| 3.0.3   | 2023-06-23 | [27651](https://github.com/airbytehq/airbyte/pull/27651)                                                        | Handle Bucket Access Errors                                                                                          |
+| 3.0.2   | 2023-06-22 | [27611](https://github.com/airbytehq/airbyte/pull/27611)                                                        | Fix start date                                                                                                       |
+| 3.0.1   | 2023-06-22 | [27604](https://github.com/airbytehq/airbyte/pull/27604)                                                        | Add logging for file reading                                                                                         |
 | 3.0.0   | 2023-05-02 | [25127](https://github.com/airbytehq/airbyte/pull/25127)                                                        | Remove ab_additional column; Use platform-handled schema evolution                                                   |
 | 2.2.0   | 2023-05-10 | [25937](https://github.com/airbytehq/airbyte/pull/25937)                                                        | Add support for Parquet Dataset                                                                                      |
 | 2.1.4   | 2023-05-01 | [25361](https://github.com/airbytehq/airbyte/pull/25361)                                                        | Parse nested avro schemas                                                                                            |
diff --git a/docs/integrations/sources/salesforce.inapp.md b/docs/integrations/sources/salesforce.inapp.md
new file mode 100644
index 000000000000..d60a552e465a
--- /dev/null
+++ b/docs/integrations/sources/salesforce.inapp.md
@@ -0,0 +1,88 @@
+## Prerequisites
+
+- [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased
+- (Optional, Recommended) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US)
+
+- (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials
+
+
+## Setup guide
+
+### Step 1: (Optional, Recommended) Create a read-only Salesforce user
+
+While you can set up the Salesforce connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the data Airbyte can read.
+
+To create a dedicated read only Salesforce user:
+
+1. [Log in to Salesforce](https://login.salesforce.com/) with an admin account.
+2. On the top right of the screen, click the gear icon and then click **Setup**.
+3. In the left navigation bar, under Administration, click **Users** > **Profiles**. The Profiles page is displayed. Click **New profile**.
+4. For Existing Profile, select **Read only**. For Profile Name, enter **Airbyte Read Only User**.
+5. Click **Save**. The Profiles page is displayed. Click **Edit**.
+6. Scroll down to the **Standard Object Permissions** and **Custom Object Permissions** and enable the **Read** checkbox for objects that you want to replicate via Airbyte.
+7. Scroll to the top and click **Save**.
+8. On the left side, under Administration, click **Users** > **Users**. The All Users page is displayed. Click **New User**.
+9. Fill out the required fields:
+   1. For License, select **Salesforce**.
+   2. For Profile, select **Airbyte Read Only User**.
+   3. For Email, make sure to use an email address that you can access.
+10. Click **Save**.
+11. Copy the Username and keep it accessible.
+12. Log into the email you used above and verify your new Salesforce account user. You'll need to set a password as part of this process. Keep this password accessible.
+
+
+
+### For Airbyte Open Source only: Obtain Salesforce OAuth credentials
+
+If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate:
+
+- Client ID
+- Client Secret
+- Refresh Token
+
+To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications:
+
+   1. If your Salesforce URL is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`.
+   2. When running a curl command, run it with the `-L` option to follow any redirects.
+   3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens.
+
+
+
+### Step 2: Set up the Salesforce connector in Airbyte
+
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Salesforce** from the list of available sources.
+4. Enter a **Source name** of your choosing to help you identify this source.
+5. To authenticate:
+
+**For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account.
+
+
+**For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token.
+
+6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account.
+7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/).
+8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data.
+9. Click **Set up source** and wait for the tests to complete.
+
+### Supported Objects
+
+The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm).
+
+Airbyte fetches and handles all the possible and available streams dynamically based on:
+
+* If the authenticated Salesforce user has the Role and Permissions to read and fetch objects
+
+* If the object has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with.
+
+### Incremental Deletes
+
+The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the `isDeleted=true` value in the respective field.
+
+### Syncing Formula Fields
+
+The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream to pull in all the updated values of the field. 
+
+For detailed information on supported sync modes, supported streams and performance considerations, refer to the 
+[full documentation for Salesforce](https://docs.airbyte.com/integrations/sources/google-analytics-v4).
diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md
index 23ee22e33337..ca9b189c2107 100644
--- a/docs/integrations/sources/salesforce.md
+++ b/docs/integrations/sources/salesforce.md
@@ -1,17 +1,22 @@
 # Salesforce
 
-Setting up the Salesforce source connector involves creating a read-only Salesforce user and configuring the Salesforce connector through the Airbyte UI.
-
-This page guides you through the process of setting up the Salesforce source connector.
+This page contains the setup guide and reference information for the Salesforce source connector.
 
 ## Prerequisites
 
-* [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased
-* Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) (optional)
+- [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased
+- (Optional, Recommended) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US)
 
-* (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials
+- (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials
 
 
+
+:::tip
+
+To use this connector, you'll need at least the Enterprise edition of Salesforce or the Professional Edition with API access purchased as an add-on. Reference the [Salesforce docs about API access](https://help.salesforce.com/s/articleView?id=000385436&type=1) for more information.
+
+:::
+
 ## Setup guide
 
 ### Step 1: (Optional, Recommended) Create a read-only Salesforce user
@@ -20,7 +25,7 @@ While you can set up the Salesforce connector using any Salesforce user with rea
 
 To create a dedicated read only Salesforce user:
 
-1. [Log into Salesforce](https://login.salesforce.com/) with an admin account.
+1. [Log in to Salesforce](https://login.salesforce.com/) with an admin account.
 2. On the top right of the screen, click the gear icon and then click **Setup**.
 3. In the left navigation bar, under Administration, click **Users** > **Profiles**. The Profiles page is displayed. Click **New profile**.
 4. For Existing Profile, select **Read only**. For Profile Name, enter **Airbyte Read Only User**.
@@ -29,173 +34,194 @@ To create a dedicated read only Salesforce user:
 7. Scroll to the top and click **Save**.
 8. On the left side, under Administration, click **Users** > **Users**. The All Users page is displayed. Click **New User**.
 9. Fill out the required fields:
-    1. For License, select **Salesforce**.
-    2. For Profile, select **Airbyte Read Only User**.
-    3. For Email, make sure to use an email address that you can access.
+   1. For License, select **Salesforce**.
+   2. For Profile, select **Airbyte Read Only User**.
+   3. For Email, make sure to use an email address that you can access.
 10. Click **Save**.
 11. Copy the Username and keep it accessible.
 12. Log into the email you used above and verify your new Salesforce account user. You'll need to set a password as part of this process. Keep this password accessible.
 
-### Step 2: Set up Salesforce as a Source in Airbyte
+
 
-
-**For Airbyte Cloud:**
+### For Airbyte Open Source only: Obtain Salesforce OAuth credentials
 
-To set up Salesforce as a source in Airbyte Cloud:
+If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate:
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
-3. On the Set up the source page, select **Salesforce** from the **Source type** dropdown.
-4. For Name, enter a name for the Salesforce connector.
-5. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account.
-6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate the data for last two years.
-7. (Optional) In the Salesforce Object filtering criteria section, click **Add**. From the Search criteria dropdown, select the criteria relevant to you. For Search value, add the search terms relevant to you. If this field is blank, Airbyte will replicate all data.
-8. Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Make sure you are logged into the right account.
-9. Click **Set up source**.
-
+- Client ID
+- Client Secret
+- Refresh Token
 
-
-**For Airbyte Open Source:**
+To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications:
 
-To set up Salesforce as a source in Airbyte Open Source:
+1.  If your Salesforce URL is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`.
+2.  When running a curl command, run it with the `-L` option to follow any redirects.
+3.  If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens.
 
-1. Follow this [walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications:
+
 
-    1. If your Salesforce URL’s is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`.
-    2. When running a curl command, run it with the `-L` option to follow any redirects.
-    3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens.
+### Step 2: Set up the Salesforce connector in Airbyte
 
-2. Navigate to the Airbute Open Source dashboard and follow the same steps as [setting up Salesforce as a source in Airbyte Cloud](#for-airbyte-cloud).
-
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Salesforce** from the list of available sources.
+4. Enter a **Source name** of your choosing to help you identify this source.
+5. To authenticate:
+   
+   **For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account.
+   
+   
+   **For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token.
+   
+6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account.
+7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/).
+8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data.
+9. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
 The Salesforce source connector supports the following sync modes:
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* (Recommended)[ Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- (Recommended)[ Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
-### Incremental Deletes Sync
+### Incremental Deletes sync
 
-The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the field `isDeleted=true` value.
+The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with `isDeleted=true`.
 
 ## Performance considerations
 
-The Salesforce connector is restricted by Salesforce’s [Daily Rate Limits](https://developer.salesforce.com/docs/atlas.en-us.salesforce_app_limits_cheatsheet.meta/salesforce_app_limits_cheatsheet/salesforce_app_limits_platform_api.htm). The connector syncs data until it hits the daily rate limit, then ends the sync early with success status, and starts the next sync from where it left off. Note that picking up from where it ends will work only for incremental sync, which is why we recommend using the [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) sync mode.
+The Salesforce connector is restricted by Salesforce’s [Daily Rate Limits](https://developer.salesforce.com/docs/atlas.en-us.salesforce_app_limits_cheatsheet.meta/salesforce_app_limits_cheatsheet/salesforce_app_limits_platform_api.htm). The connector syncs data until it hits the daily rate limit, then ends the sync early with success status, and starts the next sync from where it left off. Note that picking up from where it ends will work only for incremental sync, which is why we recommend using the [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) sync mode.
 
 ## Supported Objects
 
 The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm).
 
-Airbyte fetches and handles all the possible and available streams dynamically based on:
-
-* If the authenticated Salesforce user has the Role and Permissions to read and fetch objects
-
-* If the stream has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with in Step 2.
-
-**Note:** [BULK API](https://developer.salesforce.com/docs/atlas.en-us.api_asynch.meta/api_asynch/asynch_api_intro.htm) cannot be used to receive data from the following streams due to Salesforce API limitations. The Salesforce connector syncs them using the REST API which will occasionally cost more of your API quota:
-
-* AcceptedEventRelation
-* Attachment
-* CaseStatus
-* ContractStatus
-* DeclinedEventRelation
-* FieldSecurityClassification
-* KnowledgeArticle
-* KnowledgeArticleVersion
-* KnowledgeArticleVersionHistory
-* KnowledgeArticleViewStat
-* KnowledgeArticleVoteStat
-* OrderStatus
-* PartnerRole
-* RecentlyViewed
-* ServiceAppointmentStatus
-* ShiftStatus
-* SolutionStatus
-* TaskPriority
-* TaskStatus
-* UndecidedEventRelation
-
-## Salesforce tutorials
+Airbyte allows exporting all available Salesforce objects dynamically based on:
+
+- If the authenticated Salesforce user has the Role and Permissions to read and fetch objects
+- If the salesforce object has the queryable property set to true. Airbyte can only fetch objects which are queryable. If you don’t see an object available via Airbyte, and it is queryable, check if it is API-accessible to the Salesforce user you authenticated with.
+
+### A note on the BULK API vs REST API and their limitations
+
+Salesforce allows extracting data using either the [BULK API](https://developer.salesforce.com/docs/atlas.en-us.236.0.api_asynch.meta/api_asynch/asynch_api_intro.htm) or [REST API](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_what_is_rest_api.htm). To achieve fast performance, Salesforce recommends using the BULK API for extracting larger amounts of data (more than 2,000 records). For this reason, the Salesforce connector uses the BULK API by default to extract any Salesforce objects, unless any of the following conditions are met:
+
+- The Salesforce object has columns which are unsupported by the BULK API, like columns with a `base64` or `complexvalue` type
+- The Salesforce object is not supported by BULK API. In this case we sync the objects via the REST API which will occasionalyl cost more of your API quota. This list of objects was obtained experimentally, and includes the following objects: 
+   - AcceptedEventRelation
+   - Attachment
+   - CaseStatus
+   - ContractStatus
+   - DeclinedEventRelation
+   - FieldSecurityClassification
+   - KnowledgeArticle
+   - KnowledgeArticleVersion
+   - KnowledgeArticleVersionHistory
+   - KnowledgeArticleViewStat
+   - KnowledgeArticleVoteStat
+   - OrderStatus
+   - PartnerRole
+   - RecentlyViewed
+   - ServiceAppointmentStatus
+   - ShiftStatus
+   - SolutionStatus
+   - TaskPriority
+   - TaskStatus
+   - UndecidedEventRelation
+
+More information on the differences between various Salesforce APIs can be found [here](https://help.salesforce.com/s/articleView?id=sf.integrate_what_is_api.htm&type=5).
+
+:::info Force Using Bulk API
+
+If you set the `Force Use Bulk API` option to `true`, the connector will ignore unsupported properties and sync Stream using BULK API.
+
+:::
+
+
+## Tutorials
 
 Now that you have set up the Salesforce source connector, check out the following Salesforce tutorials:
 
-* [Replicate Salesforce data to BigQuery](https://airbyte.com/tutorials/replicate-salesforce-data-to-bigquery)
-* [Replicate Salesforce and Zendesk data to Keen for unified analytics](https://airbyte.com/tutorials/salesforce-zendesk-analytics)
+- [Replicate Salesforce data to BigQuery](https://airbyte.com/tutorials/replicate-salesforce-data-to-bigquery)
+- [Replicate Salesforce and Zendesk data to Keen for unified analytics](https://airbyte.com/tutorials/salesforce-zendesk-analytics)
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                                                                          |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------|
-| 2.0.14  | 2023-05-04 | [25794](https://github.com/airbytehq/airbyte/pull/25794) | Avoid pandas inferring wrong data types by forcing all data type as object                                            |
-| 2.0.13  | 2023-04-30 | [25700](https://github.com/airbytehq/airbyte/pull/25700) | Remove pagination and query limits                                                                                               |
-| 2.0.12  | 2023-04-25 | [25507](https://github.com/airbytehq/airbyte/pull/25507) | Update API version to 57                                                                                                         |
-| 2.0.11  | 2023-04-20 | [25352](https://github.com/airbytehq/airbyte/pull/25352) | Update API version to 53                                                                                                         |
-| 2.0.10  | 2023-04-05 | [24888](https://github.com/airbytehq/airbyte/pull/24888) | Add more frequent checkpointing                                                                                                  |
-| 2.0.9   | 2023-03-29 | [24660](https://github.com/airbytehq/airbyte/pull/24660) | Set default start_date. Sync for last two years if start date is not present in config                                           |
-| 2.0.8   | 2023-03-30 | [24690](https://github.com/airbytehq/airbyte/pull/24690) | Handle rate limit for bulk operations                                                                                            |
-| 2.0.7   | 2023-03-14 | [24071](https://github.com/airbytehq/airbyte/pull/24071) | Remove regex pattern for start_date, use format validation instead                                                               |
-| 2.0.6   | 2023-03-03 | [22891](https://github.com/airbytehq/airbyte/pull/22891) | Specified date formatting in specification                                                                                       |
-| 2.0.5   | 2023-03-01 | [23610](https://github.com/airbytehq/airbyte/pull/23610) | Handle different Salesforce page size for different queries                                                                      |
-| 2.0.4   | 2023-02-24 | [22636](https://github.com/airbytehq/airbyte/pull/22636) | Turn on default HttpAvailabilityStrategy for all streams that are not of class BulkSalesforceStream                              |
-| 2.0.3   | 2023-02-17 | [23190](https://github.com/airbytehq/airbyte/pull/23190) | In case properties are chunked, fetch primary key in every chunk                                                                 |
-| 2.0.2   | 2023-02-13 | [22896](https://github.com/airbytehq/airbyte/pull/22896) | Count the URL length based on encoded params                                                                                     |
-| 2.0.1   | 2023-02-08 | [22597](https://github.com/airbytehq/airbyte/pull/22597) | Make multiple requests if a REST stream has too many properties                                                                  |
-| 2.0.0   | 2023-02-02 | [22322](https://github.com/airbytehq/airbyte/pull/22322) | Remove `ActivityMetricRollup` stream                                                                                             |
-| 1.0.30  | 2023-01-27 | [22016](https://github.com/airbytehq/airbyte/pull/22016) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                      |
-| 1.0.29  | 2023-01-05 | [20886](https://github.com/airbytehq/airbyte/pull/20886) | Remove `ActivityMetric` stream                                                                                                   |
-| 1.0.28  | 2022-12-29 | [20927](https://github.com/airbytehq/airbyte/pull/20927) | Fix tests; add expected records                                                                                                  |
-| 1.0.27  | 2022-11-29 | [19869](https://github.com/airbytehq/airbyte/pull/19869) | Remove `AccountHistory` from unsupported BULK streams                                                                            |
-| 1.0.26  | 2022-11-15 | [19286](https://github.com/airbytehq/airbyte/pull/19286) | Bugfix: fallback to REST API if entity is not supported by BULK API                                                              |
-| 1.0.25  | 2022-11-13 | [19294](https://github.com/airbytehq/airbyte/pull/19294) | Use the correct encoding for non UTF-8 objects and data                                                                          |
-| 1.0.24  | 2022-11-01 | [18799](https://github.com/airbytehq/airbyte/pull/18799) | Update list of unsupported Bulk API objects                                                                                      |
-| 1.0.23  | 2022-11-01 | [18753](https://github.com/airbytehq/airbyte/pull/18753) | Add error_display_message for ConnectionError                                                                                    |
-| 1.0.22  | 2022-10-12 | [17615](https://github.com/airbytehq/airbyte/pull/17615) | Make paging work, if `cursor_field` is not changed inside one page                                                               |
-| 1.0.21  | 2022-10-10 | [17778](https://github.com/airbytehq/airbyte/pull/17778) | Add `EventWhoRelation` to the list of unsupported Bulk API objects.                                                              |
-| 1.0.20  | 2022-09-30 | [17453](https://github.com/airbytehq/airbyte/pull/17453) | Check objects that are not supported by the Bulk API (v52.0)                                                                     |
-| 1.0.19  | 2022-09-29 | [17314](https://github.com/airbytehq/airbyte/pull/17314) | Fixed bug with decoding response                                                                                                 |
-| 1.0.18  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states.                                                                                                    |
-| 1.0.17  | 2022-09-23 | [17094](https://github.com/airbytehq/airbyte/pull/17094) | Tune connection check: fetch a list of available streams                                                                         |
-| 1.0.16  | 2022-09-21 | [17001](https://github.com/airbytehq/airbyte/pull/17001) | Improve writing file of decode                                                                                                   |
-| 1.0.15  | 2022-08-30 | [16086](https://github.com/airbytehq/airbyte/pull/16086) | Improve API type detection                                                                                                       |
-| 1.0.14  | 2022-08-29 | [16119](https://github.com/airbytehq/airbyte/pull/16119) | Exclude `KnowledgeArticleVersion` from using bulk API                                                                            |
-| 1.0.13  | 2022-08-23 | [15901](https://github.com/airbytehq/airbyte/pull/15901) | Exclude `KnowledgeArticle` from using bulk API                                                                                   |
-| 1.0.12  | 2022-08-09 | [15444](https://github.com/airbytehq/airbyte/pull/15444) | Fixed bug when `Bulk Job` was timeout by the connector, but remained running on the server                                       |
-| 1.0.11  | 2022-07-07 | [13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field descriptions                                                                                         |
-| 1.0.10  | 2022-06-09 | [13658](https://github.com/airbytehq/airbyte/pull/13658) | Correct logic to sync stream larger than page size                                                                               |
-| 1.0.9   | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions                                                     |
-| 1.0.8   | 2022-05-04 | [12576](https://github.com/airbytehq/airbyte/pull/12576) | Decode responses as utf-8 and fallback to ISO-8859-1 if needed                                                                   |
-| 1.0.7   | 2022-05-03 | [12552](https://github.com/airbytehq/airbyte/pull/12552) | Decode responses as ISO-8859-1 instead of utf-8                                                                                  |
-| 1.0.6   | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep                                                          |
-| 1.0.5   | 2022-04-25 | [12304](https://github.com/airbytehq/airbyte/pull/12304) | Add `Describe` stream                                                                                                            |
-| 1.0.4   | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml`                                                                                            |
-| 1.0.3   | 2022-04-04 | [11692](https://github.com/airbytehq/airbyte/pull/11692) | Optimised memory usage for `BULK` API calls                                                                                      |
-| 1.0.2   | 2022-03-01 | [10751](https://github.com/airbytehq/airbyte/pull/10751) | Fix broken link anchor in connector configuration                                                                                |
-| 1.0.1   | 2022-02-27 | [10679](https://github.com/airbytehq/airbyte/pull/10679) | Reorganize input parameter order on the UI                                                                                       |
-| 1.0.0   | 2022-02-27 | [10516](https://github.com/airbytehq/airbyte/pull/10516) | Speed up schema discovery by using parallelism                                                                                   |
-| 0.1.23  | 2022-02-10 | [10141](https://github.com/airbytehq/airbyte/pull/10141) | Processing of failed jobs                                                                                                        |
-| 0.1.22  | 2022-02-02 | [10012](https://github.com/airbytehq/airbyte/pull/10012) | Increase CSV field_size_limit                                                                                                    |
-| 0.1.21  | 2022-01-28 | [9499](https://github.com/airbytehq/airbyte/pull/9499)   | If a sync reaches daily rate limit it ends the sync early with success status. Read more in `Performance considerations` section |
-| 0.1.20  | 2022-01-26 | [9757](https://github.com/airbytehq/airbyte/pull/9757)   | Parse CSV with "unix" dialect                                                                                                    |
-| 0.1.19  | 2022-01-25 | [8617](https://github.com/airbytehq/airbyte/pull/8617)   | Update connector fields title/description                                                                                        |
-| 0.1.18  | 2022-01-20 | [9478](https://github.com/airbytehq/airbyte/pull/9478)   | Add available stream filtering by `queryable` flag                                                                               |
-| 0.1.17  | 2022-01-19 | [9302](https://github.com/airbytehq/airbyte/pull/9302)   | Deprecate API Type parameter                                                                                                     |
-| 0.1.16  | 2022-01-18 | [9151](https://github.com/airbytehq/airbyte/pull/9151)   | Fix pagination in REST API streams                                                                                               |
-| 0.1.15  | 2022-01-11 | [9409](https://github.com/airbytehq/airbyte/pull/9409)   | Correcting the presence of an extra `else` handler in the error handling                                                         |
-| 0.1.14  | 2022-01-11 | [9386](https://github.com/airbytehq/airbyte/pull/9386)   | Handling 400 error, while `sobject` doesn't support `query` or `queryAll` requests                                               |
-| 0.1.13  | 2022-01-11 | [8797](https://github.com/airbytehq/airbyte/pull/8797)   | Switched from authSpecification to advanced_auth in specefication                                                                |
-| 0.1.12  | 2021-12-23 | [8871](https://github.com/airbytehq/airbyte/pull/8871)   | Fix `examples` for new field in specification                                                                                    |
-| 0.1.11  | 2021-12-23 | [8871](https://github.com/airbytehq/airbyte/pull/8871)   | Add the ability to filter streams by user                                                                                        |
-| 0.1.10  | 2021-12-23 | [9005](https://github.com/airbytehq/airbyte/pull/9005)   | Handling 400 error when a stream is not queryable                                                                                |
-| 0.1.9   | 2021-12-07 | [8405](https://github.com/airbytehq/airbyte/pull/8405)   | Filter 'null' byte(s) in HTTP responses                                                                                          |
-| 0.1.8   | 2021-11-30 | [8191](https://github.com/airbytehq/airbyte/pull/8191)   | Make `start_date` optional and change its format to `YYYY-MM-DD`                                                                 |
-| 0.1.7   | 2021-11-24 | [8206](https://github.com/airbytehq/airbyte/pull/8206)   | Handling 400 error when trying to create a job for sync using Bulk API.                                                          |
-| 0.1.6   | 2021-11-16 | [8009](https://github.com/airbytehq/airbyte/pull/8009)   | Fix retring of BULK jobs                                                                                                         |
-| 0.1.5   | 2021-11-15 | [7885](https://github.com/airbytehq/airbyte/pull/7885)   | Add `Transform` for output records                                                                                               |
-| 0.1.4   | 2021-11-09 | [7778](https://github.com/airbytehq/airbyte/pull/7778)   | Fix types for `anyType` fields                                                                                                   |
-| 0.1.3   | 2021-11-06 | [7592](https://github.com/airbytehq/airbyte/pull/7592)   | Fix getting `anyType` fields using BULK API                                                                                      |
-| 0.1.2   | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438)   | Annotate Oauth2 flow initialization parameters in connector specification                                                        |
-| 0.1.1   | 2021-09-21 | [6209](https://github.com/airbytehq/airbyte/pull/6209)   | Fix bug with pagination for BULK API                                                                                             |
-| 0.1.0   | 2021-09-08 | [5619](https://github.com/airbytehq/airbyte/pull/5619)   | Salesforce Aitbyte-Native Connector                                                                                              |
+| Version | Date       | Pull Request                                             | Subject                                                                                                                              |
+|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
+| 2.1.4   | 2023-08-17 | [29538](https://github.com/airbytehq/airbyte/pull/29538) | Fix encoding guess                                                                                                                   |
+| 2.1.3   | 2023-08-17 | [29500](https://github.com/airbytehq/airbyte/pull/29500) | handle expired refresh token error                                                                                                   |
+| 2.1.2   | 2023-08-10 | [28781](https://github.com/airbytehq/airbyte/pull/28781) | Fix pagination for BULK API jobs; Add option to force use BULK API                                                                   |
+| 2.1.1   | 2023-07-06 | [28021](https://github.com/airbytehq/airbyte/pull/28021) | Several Vulnerabilities Fixes; switched to use alpine instead of slim, CVE-2022-40897, CVE-2023-29383, CVE-2023-31484, CVE-2016-2781 |
+| 2.1.0   | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2                                                                                                                 |
+| 2.0.14  | 2023-05-04 | [25794](https://github.com/airbytehq/airbyte/pull/25794) | Avoid pandas inferring wrong data types by forcing all data type as object                                                           |
+| 2.0.13  | 2023-04-30 | [25700](https://github.com/airbytehq/airbyte/pull/25700) | Remove pagination and query limits                                                                                                   |
+| 2.0.12  | 2023-04-25 | [25507](https://github.com/airbytehq/airbyte/pull/25507) | Update API version to 57                                                                                                             |
+| 2.0.11  | 2023-04-20 | [25352](https://github.com/airbytehq/airbyte/pull/25352) | Update API version to 53                                                                                                             |
+| 2.0.10  | 2023-04-05 | [24888](https://github.com/airbytehq/airbyte/pull/24888) | Add more frequent checkpointing                                                                                                      |
+| 2.0.9   | 2023-03-29 | [24660](https://github.com/airbytehq/airbyte/pull/24660) | Set default start_date. Sync for last two years if start date is not present in config                                               |
+| 2.0.8   | 2023-03-30 | [24690](https://github.com/airbytehq/airbyte/pull/24690) | Handle rate limit for bulk operations                                                                                                |
+| 2.0.7   | 2023-03-14 | [24071](https://github.com/airbytehq/airbyte/pull/24071) | Remove regex pattern for start_date, use format validation instead                                                                   |
+| 2.0.6   | 2023-03-03 | [22891](https://github.com/airbytehq/airbyte/pull/22891) | Specified date formatting in specification                                                                                           |
+| 2.0.5   | 2023-03-01 | [23610](https://github.com/airbytehq/airbyte/pull/23610) | Handle different Salesforce page size for different queries                                                                          |
+| 2.0.4   | 2023-02-24 | [22636](https://github.com/airbytehq/airbyte/pull/22636) | Turn on default HttpAvailabilityStrategy for all streams that are not of class BulkSalesforceStream                                  |
+| 2.0.3   | 2023-02-17 | [23190](https://github.com/airbytehq/airbyte/pull/23190) | In case properties are chunked, fetch primary key in every chunk                                                                     |
+| 2.0.2   | 2023-02-13 | [22896](https://github.com/airbytehq/airbyte/pull/22896) | Count the URL length based on encoded params                                                                                         |
+| 2.0.1   | 2023-02-08 | [22597](https://github.com/airbytehq/airbyte/pull/22597) | Make multiple requests if a REST stream has too many properties                                                                      |
+| 2.0.0   | 2023-02-02 | [22322](https://github.com/airbytehq/airbyte/pull/22322) | Remove `ActivityMetricRollup` stream                                                                                                 |
+| 1.0.30  | 2023-01-27 | [22016](https://github.com/airbytehq/airbyte/pull/22016) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                                          |
+| 1.0.29  | 2023-01-05 | [20886](https://github.com/airbytehq/airbyte/pull/20886) | Remove `ActivityMetric` stream                                                                                                       |
+| 1.0.28  | 2022-12-29 | [20927](https://github.com/airbytehq/airbyte/pull/20927) | Fix tests; add expected records                                                                                                      |
+| 1.0.27  | 2022-11-29 | [19869](https://github.com/airbytehq/airbyte/pull/19869) | Remove `AccountHistory` from unsupported BULK streams                                                                                |
+| 1.0.26  | 2022-11-15 | [19286](https://github.com/airbytehq/airbyte/pull/19286) | Bugfix: fallback to REST API if entity is not supported by BULK API                                                                  |
+| 1.0.25  | 2022-11-13 | [19294](https://github.com/airbytehq/airbyte/pull/19294) | Use the correct encoding for non UTF-8 objects and data                                                                              |
+| 1.0.24  | 2022-11-01 | [18799](https://github.com/airbytehq/airbyte/pull/18799) | Update list of unsupported Bulk API objects                                                                                          |
+| 1.0.23  | 2022-11-01 | [18753](https://github.com/airbytehq/airbyte/pull/18753) | Add error_display_message for ConnectionError                                                                                        |
+| 1.0.22  | 2022-10-12 | [17615](https://github.com/airbytehq/airbyte/pull/17615) | Make paging work, if `cursor_field` is not changed inside one page                                                                   |
+| 1.0.21  | 2022-10-10 | [17778](https://github.com/airbytehq/airbyte/pull/17778) | Add `EventWhoRelation` to the list of unsupported Bulk API objects.                                                                  |
+| 1.0.20  | 2022-09-30 | [17453](https://github.com/airbytehq/airbyte/pull/17453) | Check objects that are not supported by the Bulk API (v52.0)                                                                         |
+| 1.0.19  | 2022-09-29 | [17314](https://github.com/airbytehq/airbyte/pull/17314) | Fixed bug with decoding response                                                                                                     |
+| 1.0.18  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states.                                                                                                        |
+| 1.0.17  | 2022-09-23 | [17094](https://github.com/airbytehq/airbyte/pull/17094) | Tune connection check: fetch a list of available streams                                                                             |
+| 1.0.16  | 2022-09-21 | [17001](https://github.com/airbytehq/airbyte/pull/17001) | Improve writing file of decode                                                                                                       |
+| 1.0.15  | 2022-08-30 | [16086](https://github.com/airbytehq/airbyte/pull/16086) | Improve API type detection                                                                                                           |
+| 1.0.14  | 2022-08-29 | [16119](https://github.com/airbytehq/airbyte/pull/16119) | Exclude `KnowledgeArticleVersion` from using bulk API                                                                                |
+| 1.0.13  | 2022-08-23 | [15901](https://github.com/airbytehq/airbyte/pull/15901) | Exclude `KnowledgeArticle` from using bulk API                                                                                       |
+| 1.0.12  | 2022-08-09 | [15444](https://github.com/airbytehq/airbyte/pull/15444) | Fixed bug when `Bulk Job` was timeout by the connector, but remained running on the server                                           |
+| 1.0.11  | 2022-07-07 | [13729](https://github.com/airbytehq/airbyte/pull/13729) | Improve configuration field descriptions                                                                                             |
+| 1.0.10  | 2022-06-09 | [13658](https://github.com/airbytehq/airbyte/pull/13658) | Correct logic to sync stream larger than page size                                                                                   |
+| 1.0.9   | 2022-05-06 | [12685](https://github.com/airbytehq/airbyte/pull/12685) | Update CDK to v0.1.56 to emit an `AirbyeTraceMessage` on uncaught exceptions                                                         |
+| 1.0.8   | 2022-05-04 | [12576](https://github.com/airbytehq/airbyte/pull/12576) | Decode responses as utf-8 and fallback to ISO-8859-1 if needed                                                                       |
+| 1.0.7   | 2022-05-03 | [12552](https://github.com/airbytehq/airbyte/pull/12552) | Decode responses as ISO-8859-1 instead of utf-8                                                                                      |
+| 1.0.6   | 2022-04-27 | [12335](https://github.com/airbytehq/airbyte/pull/12335) | Adding fixtures to mock time.sleep for connectors that explicitly sleep                                                              |
+| 1.0.5   | 2022-04-25 | [12304](https://github.com/airbytehq/airbyte/pull/12304) | Add `Describe` stream                                                                                                                |
+| 1.0.4   | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml`                                                                                                |
+| 1.0.3   | 2022-04-04 | [11692](https://github.com/airbytehq/airbyte/pull/11692) | Optimised memory usage for `BULK` API calls                                                                                          |
+| 1.0.2   | 2022-03-01 | [10751](https://github.com/airbytehq/airbyte/pull/10751) | Fix broken link anchor in connector configuration                                                                                    |
+| 1.0.1   | 2022-02-27 | [10679](https://github.com/airbytehq/airbyte/pull/10679) | Reorganize input parameter order on the UI                                                                                           |
+| 1.0.0   | 2022-02-27 | [10516](https://github.com/airbytehq/airbyte/pull/10516) | Speed up schema discovery by using parallelism                                                                                       |
+| 0.1.23  | 2022-02-10 | [10141](https://github.com/airbytehq/airbyte/pull/10141) | Processing of failed jobs                                                                                                            |
+| 0.1.22  | 2022-02-02 | [10012](https://github.com/airbytehq/airbyte/pull/10012) | Increase CSV field_size_limit                                                                                                        |
+| 0.1.21  | 2022-01-28 | [9499](https://github.com/airbytehq/airbyte/pull/9499)   | If a sync reaches daily rate limit it ends the sync early with success status. Read more in `Performance considerations` section     |
+| 0.1.20  | 2022-01-26 | [9757](https://github.com/airbytehq/airbyte/pull/9757)   | Parse CSV with "unix" dialect                                                                                                        |
+| 0.1.19  | 2022-01-25 | [8617](https://github.com/airbytehq/airbyte/pull/8617)   | Update connector fields title/description                                                                                            |
+| 0.1.18  | 2022-01-20 | [9478](https://github.com/airbytehq/airbyte/pull/9478)   | Add available stream filtering by `queryable` flag                                                                                   |
+| 0.1.17  | 2022-01-19 | [9302](https://github.com/airbytehq/airbyte/pull/9302)   | Deprecate API Type parameter                                                                                                         |
+| 0.1.16  | 2022-01-18 | [9151](https://github.com/airbytehq/airbyte/pull/9151)   | Fix pagination in REST API streams                                                                                                   |
+| 0.1.15  | 2022-01-11 | [9409](https://github.com/airbytehq/airbyte/pull/9409)   | Correcting the presence of an extra `else` handler in the error handling                                                             |
+| 0.1.14  | 2022-01-11 | [9386](https://github.com/airbytehq/airbyte/pull/9386)   | Handling 400 error, while `sobject` doesn't support `query` or `queryAll` requests                                                   |
+| 0.1.13  | 2022-01-11 | [8797](https://github.com/airbytehq/airbyte/pull/8797)   | Switched from authSpecification to advanced_auth in specefication                                                                    |
+| 0.1.12  | 2021-12-23 | [8871](https://github.com/airbytehq/airbyte/pull/8871)   | Fix `examples` for new field in specification                                                                                        |
+| 0.1.11  | 2021-12-23 | [8871](https://github.com/airbytehq/airbyte/pull/8871)   | Add the ability to filter streams by user                                                                                            |
+| 0.1.10  | 2021-12-23 | [9005](https://github.com/airbytehq/airbyte/pull/9005)   | Handling 400 error when a stream is not queryable                                                                                    |
+| 0.1.9   | 2021-12-07 | [8405](https://github.com/airbytehq/airbyte/pull/8405)   | Filter 'null' byte(s) in HTTP responses                                                                                              |
+| 0.1.8   | 2021-11-30 | [8191](https://github.com/airbytehq/airbyte/pull/8191)   | Make `start_date` optional and change its format to `YYYY-MM-DD`                                                                     |
+| 0.1.7   | 2021-11-24 | [8206](https://github.com/airbytehq/airbyte/pull/8206)   | Handling 400 error when trying to create a job for sync using Bulk API.                                                              |
+| 0.1.6   | 2021-11-16 | [8009](https://github.com/airbytehq/airbyte/pull/8009)   | Fix retring of BULK jobs                                                                                                             |
+| 0.1.5   | 2021-11-15 | [7885](https://github.com/airbytehq/airbyte/pull/7885)   | Add `Transform` for output records                                                                                                   |
+| 0.1.4   | 2021-11-09 | [7778](https://github.com/airbytehq/airbyte/pull/7778)   | Fix types for `anyType` fields                                                                                                       |
+| 0.1.3   | 2021-11-06 | [7592](https://github.com/airbytehq/airbyte/pull/7592)   | Fix getting `anyType` fields using BULK API                                                                                          |
+| 0.1.2   | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438)   | Annotate Oauth2 flow initialization parameters in connector specification                                                            |
+| 0.1.1   | 2021-09-21 | [6209](https://github.com/airbytehq/airbyte/pull/6209)   | Fix bug with pagination for BULK API                                                                                                 |
+| 0.1.0   | 2021-09-08 | [5619](https://github.com/airbytehq/airbyte/pull/5619)   | Salesforce Aitbyte-Native Connector                                                                                                  |
diff --git a/docs/integrations/sources/salesloft.md b/docs/integrations/sources/salesloft.md
index 9665fb61044e..be63f2cd8d7f 100644
--- a/docs/integrations/sources/salesloft.md
+++ b/docs/integrations/sources/salesloft.md
@@ -8,6 +8,7 @@ This page contains the setup guide and reference information for the Salesloft S
 - Start date
 
 
+
 **For Airbyte Open Source:**
 
 - Salesloft API Key (see [API Key Authentication](https://developers.salesloft.com/api.html#!/Topic/apikey))
@@ -20,12 +21,15 @@ This page contains the setup guide and reference information for the Salesloft S
 Create a [Salesloft Account](https://salesloft.com).
 
 
+
 **Airbyte Open Source additional setup steps**
 
 Log into [Salesloft](https://salesloft.com) and then generate an [API Key](https://developers.salesloft.com/api.html#!/Topic/apikey).
+
 
 
 
+
 ### Step 2: Set up the Salesloft connector in Airbyte
 
 **For Airbyte Cloud:**
@@ -41,6 +45,7 @@ Log into [Salesloft](https://salesloft.com) and then generate an [API Key](https
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Authenticate with **API Key**.
@@ -50,38 +55,44 @@ Log into [Salesloft](https://salesloft.com) and then generate an [API Key](https
 
 The Salesloft Source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
 This connector outputs the following streams:
 
-* [CadenceMemberships](https://developers.salesloft.com/api.html#!/Cadence_Memberships/get_v2_cadence_memberships_json)
-* [Cadences](https://developers.salesloft.com/api.html#!/Cadences/get_v2_cadences_json)
-* [People](https://developers.salesloft.com/api.html#!/People/get_v2_people_json)
-* [Users](https://developers.salesloft.com/api.html#!/Users/get_v2_users_json)
-* [Emails](https://developers.salesloft.com/api.html#!/Emails/get_v2_activities_emails_json)
-* [Account Stages](https://developers.salesloft.com/api.html#!/Account_Stages/get_v2_account_stages_json)
-* [Account Tiers](https://developers.salesloft.com/api.html#!/Account_Tiers/get_v2_account_tiers_json)
-* [Accounts](https://developers.salesloft.com/api.html#!/Accounts/get_v2_accounts_json)
-* [Actions](https://developers.salesloft.com/api.html#!/Actions/get_v2_actions_json)
-* [Calls](https://developers.salesloft.com/api.html#!/Calls/get_v2_activities_calls_json)
-* [Emails Templates](https://developers.salesloft.com/api.html#!/Email_Templates/get_v2_email_templates_json)
-* [Emails Template Attachements](https://developers.salesloft.com/api.html#!/Email_Template_Attachments/get_v2_email_template_attachments_json)
-* [Imports](https://developers.salesloft.com/api.html#!/Imports/get_v2_imports_json)
-* [Notes](https://developers.salesloft.com/api.html#!/Notes/get_v2_notes_json)
-* [Person Stages](https://developers.salesloft.com/api.html#!/Person_Stages/get_v2_person_stages_json)
-* [Phone Number Assignments](https://developers.salesloft.com/api.html#!/Phone_Number_Assignments/get_v2_phone_number_assignments_json)
-* [Steps](https://developers.salesloft.com/api.html#!/Steps/get_v2_steps_json)
-* [Team Templates](https://developers.salesloft.com/api.html#!/Team_Templates/get_v2_team_templates_json)
-* [Team Template Attachements](https://developers.salesloft.com/api.html#!/Team_Template_Attachments/get_v2_team_template_attachments_json)
-* [CRM Activities](https://developers.salesloft.com/api.html#!/CRM_Activities/get_v2_crm_activities_json)
-* [CRM Users](https://developers.salesloft.com/api.html#!/Crm_Users/get_v2_crm_users_json)
-* [Groups](https://developers.salesloft.com/api.html#!/Groups/get_v2_groups_json)
-* [Successes](https://developers.salesloft.com/api.html#!/Successes/get_v2_successes_json)
+- [CadenceMemberships](https://developers.salesloft.com/api.html#!/Cadence_Memberships/get_v2_cadence_memberships_json)
+- [Cadences](https://developers.salesloft.com/api.html#!/Cadences/get_v2_cadences_json)
+- [People](https://developers.salesloft.com/api.html#!/People/get_v2_people_json)
+- [Users](https://developers.salesloft.com/api.html#!/Users/get_v2_users_json)
+- [Emails](https://developers.salesloft.com/api.html#!/Emails/get_v2_activities_emails_json)
+- [Account Stages](https://developers.salesloft.com/api.html#!/Account_Stages/get_v2_account_stages_json)
+- [Account Tiers](https://developers.salesloft.com/api.html#!/Account_Tiers/get_v2_account_tiers_json)
+- [Accounts](https://developers.salesloft.com/api.html#!/Accounts/get_v2_accounts_json)
+- [Actions](https://developers.salesloft.com/api.html#!/Actions/get_v2_actions_json)
+- [Calls](https://developers.salesloft.com/api.html#!/Calls/get_v2_activities_calls_json)
+- [Emails Templates](https://developers.salesloft.com/api.html#!/Email_Templates/get_v2_email_templates_json)
+- [Emails Template Attachements](https://developers.salesloft.com/api.html#!/Email_Template_Attachments/get_v2_email_template_attachments_json)
+- [Imports](https://developers.salesloft.com/api.html#!/Imports/get_v2_imports_json)
+- [Notes](https://developers.salesloft.com/api.html#!/Notes/get_v2_notes_json)
+- [Person Stages](https://developers.salesloft.com/api.html#!/Person_Stages/get_v2_person_stages_json)
+- [Phone Number Assignments](https://developers.salesloft.com/api.html#!/Phone_Number_Assignments/get_v2_phone_number_assignments_json)
+- [Steps](https://developers.salesloft.com/api.html#!/Steps/get_v2_steps_json)
+- [Team Templates](https://developers.salesloft.com/api.html#!/Team_Templates/get_v2_team_templates_json)
+- [Team Template Attachements](https://developers.salesloft.com/api.html#!/Team_Template_Attachments/get_v2_team_template_attachments_json)
+- [CRM Activities](https://developers.salesloft.com/api.html#!/CRM_Activities/get_v2_crm_activities_json)
+- [CRM Users](https://developers.salesloft.com/api.html#!/Crm_Users/get_v2_crm_users_json)
+- [Groups](https://developers.salesloft.com/api.html#!/Groups/get_v2_groups_json)
+- [Successes](https://developers.salesloft.com/api.html#!/Successes/get_v2_successes_json)
+- [Call Data Records](https://developers.salesloft.com/api.html#!/Call_Data_Records/get_v2_call_data_records_json)
+- [Call Dispositions](https://developers.salesloft.com/api.html#!/Call_Dispositions/get_v2_call_dispositions_json)
+- [Call Sentiments](https://developers.salesloft.com/api.html#!/Call_Sentiments/get_v2_call_sentiments_json)
+- [Custom Fields](https://developers.salesloft.com/api.html#!/Custom_Fields/get_v2_custom_fields_json)
+- [Meetings](https://developers.salesloft.com/api.html#!/Meetings/get_v2_meetings_json)
+- [Searches](https://developers.salesloft.com/api.html#!/Searches/post_v2_searches_json)
 
 ## Performance considerations
 
@@ -90,7 +101,9 @@ Salesloft has the [rate limits](hhttps://developers.salesloft.com/api.html#!/Top
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                           |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------- |
+| 1.2.0   | 2023-06-20 | [27505](https://github.com/airbytehq/airbyte/pull/27505) | Added new streams (Call Data Records, Call Dispositions, ... )    |
+| 1.1.1   | 2023-06-17 | [27484](https://github.com/airbytehq/airbyte/pull/27484) | Bump version on py files updates                                  |
 | 1.1.0   | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Added `latest_active_date` field to the `Cadences` stream schema. |
 | 1.0.0   | 2023-03-08 | [23937](https://github.com/airbytehq/airbyte/pull/23937) | Certify to Beta                                                   |
 | 0.1.6   | 2023-03-07 | [22893](https://github.com/airbytehq/airbyte/pull/22893) | Specified date formatting in specification                        |
diff --git a/docs/integrations/sources/sendgrid.inapp.md b/docs/integrations/sources/sendgrid.inapp.md
new file mode 100644
index 000000000000..4c03aca5c0a0
--- /dev/null
+++ b/docs/integrations/sources/sendgrid.inapp.md
@@ -0,0 +1,23 @@
+## Prerequisites
+
+* [Sendgrid API Key]((https://docs.sendgrid.com/ui/account-and-settings/api-keys#creating-an-api-key)) with
+  * Read-only access to all resources
+  * Full access to marketing resources
+
+## Setup guide
+
+1. Enter a name for your Sendgridconnector.
+2. Enter your `api key`.
+3. (Optional) Enter the `start_time` in YYYY-MM-DDTHH:MM:SSZ format. Dataadded on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
+4. Click **Set up source**.
+
+### (Optional) Create a read-only API key
+
+While you can set up the Sendgrid connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the which resources Airbyte can read.
+
+The API key should be read-only on all resources except Marketing, where it needs Full Access.
+
+Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". **Legacy marketing campaigns are not supported by this source connector**. 
+If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it may be because your SendGrid account uses legacy marketing campaigns.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Sendgrid](https://docs.airbyte.com/integrations/sources/sendgrid).
\ No newline at end of file
diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md
index 2addcbb1ff8c..3e06271bebc8 100644
--- a/docs/integrations/sources/sentry.md
+++ b/docs/integrations/sources/sentry.md
@@ -22,17 +22,17 @@ To set up the Sentry source connector, you'll need the Sentry [project name](htt
 
 The Sentry source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Events](https://docs.sentry.io/api/events/list-a-projects-events/)
-* [Issues](https://docs.sentry.io/api/events/list-a-projects-issues/)
-* [Projects](https://docs.sentry.io/api/projects/list-your-projects/)
-* [Releases](https://docs.sentry.io/api/releases/list-an-organizations-releases/)
+- [Events](https://docs.sentry.io/api/events/list-a-projects-events/)
+- [Issues](https://docs.sentry.io/api/events/list-a-projects-issues/)
+- [Projects](https://docs.sentry.io/api/projects/list-your-projects/)
+- [Releases](https://docs.sentry.io/api/releases/list-an-organizations-releases/)
 
 ## Data type map
 
@@ -45,21 +45,23 @@ The Sentry source connector supports the following [sync modes](https://docs.air
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                     |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------|
-| 0.2.2   | 2023-05-02 | [25759](https://github.com/airbytehq/airbyte/pull/25759) | Change stream that used in check_connection                                         |
-| 0.2.1   | 2023-04-27 | [25602](https://github.com/airbytehq/airbyte/pull/25602) | Add validation of project and organization names during connector setup                                         |
-| 0.2.0   | 2023-04-03 | [23923](https://github.com/airbytehq/airbyte/pull/23923) | Add Releases stream                                         |
-| 0.1.12  | 2023-03-01 | [23619](https://github.com/airbytehq/airbyte/pull/23619) | Fix bug when `stream state` is `None` or any other bad value occurs                     |
-| 0.1.11  | 2023-02-02 | [22303](https://github.com/airbytehq/airbyte/pull/22303) | Turn ON default AvailabilityStrategy                        |
-| 0.1.10  | 2023-01-27 | [22041](https://github.com/airbytehq/airbyte/pull/22041) | Set `AvailabilityStrategy` for streams explicitly to `None` |
-| 0.1.9   | 2022-12-20 | [21864](https://github.com/airbytehq/airbyte/pull/21864) | Add state persistence to incremental sync                   |
-| 0.1.8   | 2022-12-20 | [20709](https://github.com/airbytehq/airbyte/pull/20709) | Add incremental sync                                        |
-| 0.1.7   | 2022-09-30 | [17466](https://github.com/airbytehq/airbyte/pull/17466) | Migrate to per-stream states                                |
-| 0.1.6   | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to the Python CDK                               |
-| 0.1.5   | 2022-08-24 | [15911](https://github.com/airbytehq/airbyte/pull/15911) | Bugfix to allowing reading schemas at runtime               |
-| 0.1.4   | 2022-08-19 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml at runtime              |
-| 0.1.3   | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator                  |
-| 0.1.2   | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework                           |
-| 0.1.1   | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628)   | Update fields in source-connectors specifications           |
-| 0.1.0   | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975)   | New Source: Sentry                                          |
+| Version | Date       | Pull Request                                             | Subject                                                                 |
+|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------|
+| 0.2.4   | 2023-08-14 | [29401](https://github.com/airbytehq/airbyte/pull/29401) | Fix `null` value in stream state                                        |
+| 0.2.3   | 2023-08-03 | [29023](https://github.com/airbytehq/airbyte/pull/29023) | Add incremental for `issues` stream                                     |
+| 0.2.2   | 2023-05-02 | [25759](https://github.com/airbytehq/airbyte/pull/25759) | Change stream that used in check_connection                             |
+| 0.2.1   | 2023-04-27 | [25602](https://github.com/airbytehq/airbyte/pull/25602) | Add validation of project and organization names during connector setup |
+| 0.2.0   | 2023-04-03 | [23923](https://github.com/airbytehq/airbyte/pull/23923) | Add Releases stream                                                     |
+| 0.1.12  | 2023-03-01 | [23619](https://github.com/airbytehq/airbyte/pull/23619) | Fix bug when `stream state` is `None` or any other bad value occurs     |
+| 0.1.11  | 2023-02-02 | [22303](https://github.com/airbytehq/airbyte/pull/22303) | Turn ON default AvailabilityStrategy                                    |
+| 0.1.10  | 2023-01-27 | [22041](https://github.com/airbytehq/airbyte/pull/22041) | Set `AvailabilityStrategy` for streams explicitly to `None`             |
+| 0.1.9   | 2022-12-20 | [21864](https://github.com/airbytehq/airbyte/pull/21864) | Add state persistence to incremental sync                               |
+| 0.1.8   | 2022-12-20 | [20709](https://github.com/airbytehq/airbyte/pull/20709) | Add incremental sync                                                    |
+| 0.1.7   | 2022-09-30 | [17466](https://github.com/airbytehq/airbyte/pull/17466) | Migrate to per-stream states                                            |
+| 0.1.6   | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to the Python CDK                                           |
+| 0.1.5   | 2022-08-24 | [15911](https://github.com/airbytehq/airbyte/pull/15911) | Bugfix to allowing reading schemas at runtime                           |
+| 0.1.4   | 2022-08-19 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml at runtime                          |
+| 0.1.3   | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator                              |
+| 0.1.2   | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework                                       |
+| 0.1.1   | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628)   | Update fields in source-connectors specifications                       |
+| 0.1.0   | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975)   | New Source: Sentry                                                      |
diff --git a/docs/integrations/sources/serpstat.md b/docs/integrations/sources/serpstat.md
new file mode 100644
index 000000000000..55ea29d539db
--- /dev/null
+++ b/docs/integrations/sources/serpstat.md
@@ -0,0 +1,52 @@
+# Serpstat
+
+This page contains the setup guide and reference information for the Serpstat source connector.
+
+## Setup guide
+### Step 1: Get Serpstat API key 
+
+#### For new Serpstat users
+
+1. Create a new [Serpstat account](https://serpstat.com/signup/?utm_source=).
+2. Go to [My account](https://serpstat.com/users/profile/) page and click **Get API key**.
+3. Follow the instructions to get the API key.
+4. Click **Copy** to copy the API key.
+
+#### For existing Serpstat users
+
+Go to [My account](https://serpstat.com/users/profile/) page and click **Copy** to copy the API key.
+
+### Step 2: Set up the Serpstat connector in Airbyte
+
+1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) or Airbyte Open Source account.
+2. Click **Sources** and then click **+ New source**.
+3. On the **Set up the source** page, select **Serpstat** from the **Source type** dropdown.
+4. Enter a name for your connector.
+5. Enter the API key.
+6. Expand **Optional fields** and fill them in. Each API response consumes API credits available to your Serpstat subscription plan. To limit the number of consumed API rows, decrease **Page size** and **Pages to fetch** options.
+7. Click **Set up source**.
+
+## Supported sync modes
+
+The Serpstat source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+
+* Full refresh
+
+## Supported Streams
+
+* [Domains summary](https://serpstat.com/api/412-summarnij-otchet-po-domenu-v4-serpstatdomainproceduregetdomainsinfo/)
+* [Domain history](https://serpstat.com/api/420-istoriya-po-domenu-v4-serpstatdomainproceduregetdomainshistory/)
+* [Domain keywords](https://serpstat.com/api/584-top-search-engine-keywords-by-v4-domain-serpstatdomainproceduregetdomainkeywords/)
+* [Domain keywords by region](https://serpstat.com/api/sorting-the-domain-by-keywords/)
+* [Domain competitors](https://serpstat.com/api/590-domain-competitors-in-v4-search-result-serpstatdomainproceduregetcompetitors/)
+* [Domain top pages](https://serpstat.com/api/588-domain-top-urls-v4-serpstatdomainproceduregettopurls/)
+
+## Performance considerations 
+    
+The maximum sync speed is limited by the number of requests per second per API key. See this limit in your [Serpstat account](https://serpstat.com/users/profile/).
+
+## Changelog
+
+| Version | Date       | Pull Request                                             | Subject                                                                                       |
+|:--------| :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------- |
+| 0.1.0   | 2023-08-21 | [28147](https://github.com/airbytehq/airbyte/pull/28147)   | Release Serpstat Connector                                                                |
diff --git a/docs/integrations/sources/sftp-bulk.md b/docs/integrations/sources/sftp-bulk.md
index c866bdd18bfa..8b1095a0a2c2 100644
--- a/docs/integrations/sources/sftp-bulk.md
+++ b/docs/integrations/sources/sftp-bulk.md
@@ -1,65 +1,125 @@
-# SFTP  Bulk
-This page contains the setup guide and reference information for the FTP source connector.
+# SFTP Bulk
 
-This connector allows you to:
-- Fetch files from an FTP server matching a folder path and define an optional file pattern to bulk ingest files into a single stream
-- Incrementally load files into your destination from an FTP server based on when files were last added or modified
-- Optionally load only the latest file matching a folder path and optional pattern and overwrite the data in your destination (helpful when a snapshot file gets added on a regular basis containing the latest data)
+This page contains the setup guide and reference information for the SFTP Bulk source connector.
+
+This connector provides the following features not found in the standard SFTP source connector:
+
+- **Bulk ingestion of files**: This connector can consolidate and process multiple files as a single data stream in your destination system.
+- **Incremental loading**: This connector supports incremental loading, allowing you to sync files from the SFTP server to your destination based on their creation or last modification time.
+- **Load most recent file**: You can choose to load only the most recent file from the designated folder path. This feature is particularly useful when dealing with snapshot files that are regularly added and contain the latest data.
 
 ## Prerequisites
 
-* The Server with FTP connection type support
-* The Server host
-* The Server port
-* Username-Password/Public Key Access Rights
+- Access to a remote server that supports SFTP
+- Host address
+- Valid username and password associated with the host server
 
 ## Setup guide
-### Step 1: Set up SFTP
-1. Use your username/password credential to connect the server.
-2. Alternatively generate Public Key Access
 
-The following simple steps are required to set up public key authentication:
+### Step 1: Set up SFTP authentication
+
+To set up the SFTP connector, you will need to select at least _one_ of the following authentication methods:
+
+- Your username and password credentials associated with the server.
+- A private/public key pair.
+
+To set up key pair authentication, you may use the following steps as a guide:
+
+1. Open your terminal or command prompt and use the `ssh-keygen` command to generate a new key pair.
+   :::note
+   If your operating system does not support the `ssh-keygen` command, you can use a third-party tool like [PuTTYgen](https://www.puttygen.com/) to generate the key pair instead.
+   :::
+
+2. You will be prompted for a location to save the keys, and a passphrase to secure the private key. You can press enter to accept the default location and opt out of a passphrase if desired. Your two keys will be generated in the designated location as two separate files. The private key will usually be saved as `id_rsa`, while the public key will be saved with the `.pub` extension (`id_rsa.pub`).
+
+3. Use the `ssh-copy-id` command in your terminal to copy the public key to the server.
+
+```
+ssh-copy-id @
+```
+
+Be sure to replace your specific values for your username and the server's IP address.
+:::note
+Depending on factors such as your operating system and the specific SSH implementation your remote server uses, you may not be able to use the `ssh-copy-id` command. If so, please consult your server administrator for the appropriate steps to copy the public key to the server.
+:::
+
+4. You should now be able to connect to the server via the private key. You can test this by using the `ssh` command:
+
+```
+ssh @
+```
 
-Key pair is created (typically by the user). This is typically done with ssh-keygen.
-Private key stays with the user (and only there), while the public key is sent to the server. Typically with the ssh-copy-id utility.
-Server stores the public key (and "marks" it as authorized).
-Server will now allow access to anyone who can prove they have the corresponding private key.
+For more information on SSH key pair authentication, please refer to the
+[official documentation](https://www.ssh.com/academy/ssh/keygen).
 
 ### Step 2: Set up the SFTP connector in Airbyte
 
-1. In the left navigation bar, click **`Sources`**. In the top-right corner, click **+new source**.
-2. On the Set up the source page, enter the name for the FTP connector and select **SFTP Bulk** from the Source type dropdown.
-3. Enter your `User Name`, `Host Address`, `Port`
-4. Enter authentication details for the FTP server (`Password` and/or `Private Key`)
-5. Choose a `File type`
-6. Enter `Folder Path` (Optional) to specify server folder for sync
-7. Enter `File Pattern` (Optional). e.g. ` log-([0-9]{4})([0-9]{2})([0-9]{2})`. Write your own [regex](https://docs.python.org/3/howto/regex.html)
-8. Check `Most recent file` (Optional) if you only want to sync the most recent file matching a folder path and optional file pattern
-9. Provide a `Start Date` for incremental syncs to only sync files modified/added after this date
-10. Click on `Check Connection` to finish configuring the FTP source.
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **SFTP** from the list of available sources.
+   
+   **For Airbyte Cloud users**: If you do not see the **SFTP Bulk** source listed, please make sure the **Alpha** checkbox at the top of the page is checked.
+   
+4. Enter a **Source name** of your choosing.
+5. Enter your **Username**, as well as the **Host Address** and **Port**. The default port for SFTP is 22. If your remote server is using a different port, please enter it here.
+6. Enter your authentication credentials for the SFTP server (**Password** or **Private Key**). If you are authenticating with a private key, you can upload the file containing the private key (usually named `rsa_id`) using the Upload file button.
+7. Enter a **Stream Name**. This will be the name of the stream that will be outputted to your destination.
+8. Use the dropdown menu to select the **File Type** you wish to sync. Currently, only CSV and JSON formats are supported.
+9. Provide a **Start Date** using the provided datepicker, or by programmatically entering the date in the format `YYYY-MM-DDT00:00:00Z`. Incremental syncs will only sync files modified/added after this date.
+10. If you wish to configure additional optional settings, please refer to the next section. Otherwise, click **Set up source** and wait for the tests to complete.
 
-## Supported sync modes
+## Optional fields
+
+The **Optional fields** can be used to further configure the SFTP source connector. If you do not wish to set additional configurations, these fields can be left at their default settings.
+
+1. **CSV Separator**: If you selected `csv` as the file type, you can use this field to specify a custom separator. The default value is `,`.
+
+2. **Folder Path**: Enter a folder path to specify the directory on the remote server to be synced. For example, given the file structure:
+
+```
+Root
+| - logs
+|   | - 2021
+|   | - 2022
+|
+| - files
+|   | - 2021
+|   | - 2022
+```
+
+An input of `/logs/2022` will only replicate data contained within the specified folder, ignoring the `/files` and `/logs/2021` folders. Leaving this field blank will replicate all applicable files in the remote server's designated entry point.
 
-The FTP source connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+3. **File Pattern**: Enter a [regular expression](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) to specify a naming pattern for the files to be replicated. Consider the following example:
+
+```
+log-([0-9]{4})([0-9]{2})([0-9]{2})
+```
+
+This pattern will filter for files that match the format `log-YYYYMMDD`, where `YYYY`, `MM`, and `DD` represented four-digit, two-digit, and two-digit numbers, respectively. For example, `log-20230713`. Leaving this field blank will replicate all files not filtered by the previous two fields.
+
+4. **Most Recent File**: Toggle this option if you only want to sync the most recent file located in the folder path. This may be useful when dealing with data sources that generate frequent updates, such as log files or real-time data feeds. Set to False by default.
+
+## Supported sync modes
 
-| Feature                       | Support  | Notes                                                                                 |
-|:------------------------------|:--------:|:--------------------------------------------------------------------------------------|
-| Full Refresh - Overwrite      |    ✅    |                                                                                      |
-| Full Refresh - Append Sync    |    ✅    |                                                                                      |
-| Incremental - Append          |    ✅    |                                                                                      |
-| Incremental - Deduped History |    ❌    |                                                                                      |
-| Namespaces                    |    ❌    |                                                                                      |
+The SFTP Bulk source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
+| Feature                        | Support | Notes |
+| :----------------------------- | :-----: | :---- |
+| Full Refresh - Overwrite       |   ✅    |       |
+| Full Refresh - Append Sync     |   ✅    |       |
+| Incremental - Append           |   ✅    |       |
+| Incremental - Append + Deduped |   ❌    |       |
+| Namespaces                     |   ❌    |       |
 
-## Supported Streams
+## Supported streams
 
-This source provides a single stream per file with a dynamic schema. The current supported type file: `.csv` and `.json`
+This source provides a single stream per file with a dynamic schema. The current supported type files are CSV and JSON.
 More formats \(e.g. Apache Avro\) will be supported in the future.
 
 ## Changelog
 
-| Version | Date       | Pull Request | Subject         |
-|:--------|:-----------|:-------------|:----------------|
+| Version | Date       | Pull Request                                              | Subject                       |
+| :------ | :--------- | :-------------------------------------------------------- | :---------------------------- |
 | 0.1.2   | 2023-04-19 | [#19224](https://github.com/airbytehq/airbyte/pull/19224) | Support custom CSV separators |
-| 0.1.1   | 2023-03-17 | [#24180](https://github.com/airbytehq/airbyte/pull/24180) | Fix field order |
-| 0.1.0   | 2021-24-05 |              | Initial version |
+| 0.1.1   | 2023-03-17 | [#24180](https://github.com/airbytehq/airbyte/pull/24180) | Fix field order               |
+| 0.1.0   | 2021-24-05 |                                                           | Initial version               |
diff --git a/docs/integrations/sources/sftp.md b/docs/integrations/sources/sftp.md
index ca19d406af57..8d6d84e942d9 100644
--- a/docs/integrations/sources/sftp.md
+++ b/docs/integrations/sources/sftp.md
@@ -1,62 +1,112 @@
 # SFTP
+
 This page contains the setup guide and reference information for the SFTP source connector.
 
 ## Prerequisites
 
-* The Server with SFTP connection type support
-* The Server host
-* The Server port
-* Username-Password/Public Key Access Rights
+- Access to a remote server that supports SFTP
+- Host address
+- Valid username and password associated with the host server
 
 ## Setup guide
-### Step 1: Set up SFTP 
-1. Use your username/password credential to connect the server.
-2. Alternatively generate Public Key Access
 
-The following simple steps are required to set up public key authentication:
+### Step 1: Set up SFTP authentication
+
+To set up the SFTP connector, you will need to select _one_ of the following authentication methods:
+
+- Your username and password credentials associated with the server.
+- A private/public key pair.
+
+To set up key pair authentication, you may use the following steps as a guide:
+
+1. Open your terminal or command prompt and use the `ssh-keygen` command to generate a new key pair.
+   :::note
+   If your operating system does not support the `ssh-keygen` command, you can use a third-party tool like [PuTTYgen](https://www.puttygen.com/) to generate the key pair instead.
+   :::
+
+2. You will be prompted for a location to save the keys, and a passphrase to secure the private key. You can press enter to accept the default location and opt out of a passphrase if desired. Your two keys will be generated in the designated location as two separate files. The private key will usually be saved as `id_rsa`, while the public key will be saved with the `.pub` extension (`id_rsa.pub`).
+
+3. Use the `ssh-copy-id` command in your terminal to copy the public key to the server.
+
+```
+ssh-copy-id @
+```
+
+Be sure to replace your specific values for your username and the server's IP address.
+:::note
+Depending on factors such as your operating system and the specific SSH implementation your remote server uses, you may not be able to use the `ssh-copy-id` command. If so, please consult your server administrator for the appropriate steps to copy the public key to the server.
+:::
 
-Key pair is created (typically by the user). This is typically done with ssh-keygen.
-Private key stays with the user (and only there), while the public key is sent to the server. Typically with the ssh-copy-id utility.
-Server stores the public key (and "marks" it as authorized).
-Server will now allow access to anyone who can prove they have the corresponding private key.
+4. You should now be able to connect to the server via the private key. You can test this by using the `ssh` command:
+
+```
+ssh @
+```
+
+For more information on SSH key pair authentication, please refer to the
+[official documentation](https://www.ssh.com/academy/ssh/keygen).
 
 ### Step 2: Set up the SFTP connector in Airbyte
 
-#### For Airbyte Cloud:
+1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **SFTP** from the list of available sources.
+   
+   **For Airbyte Cloud users**: If you do not see the **SFTP** source listed, please make sure the **Alpha** checkbox at the top of the page is checked.
+   
+4. Enter a **Source name** of your choosing.
+5. Enter your **Username**, as well as the **Host Address** and **Port**. The default port for SFTP is 22. If your remote server is using a different port, please enter it here.
+6. In the **Authentication** section, use the dropdown menu to select **Password Authentication** or **SSH Key Authentication**, then fill in the required credentials. If you are authenticating with a private key, you can upload the file containing the private key (usually named `rsa_id`) using the **Upload file** button.
+7. If you wish to configure additional optional settings, please refer to the next section. Otherwise, click **Set up source** and wait for the tests to complete.
 
-1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **`Sources`**. In the top-right corner, click **+new source**.
-3. On the Set up the source page, enter the name for the SFTP connector and select **SFTP** from the Source type dropdown.
-4. Enter your `User Name`, `Host Address`, `Port`
-5. Choose the `Authentication` type `Password Authentication` or `Key Authentication`
-6. Type `File type` (temporary comma separated)
-7. Enter `Folder Path` (Optional) to specify server folder for sync
-8. Enter `File Pattern` (Optional). e.g. ` log-([0-9]{4})([0-9]{2})([0-9]{2})`. Write your own [regex](https://www.tutorialspoint.com/java/java_regular_expressions.htm)    
-9. Click on `Check Connection` to finish configuring the Amplitude source.
+## Optional fields
 
-## Supported sync modes
+The **Optional fields** can be used to further configure the SFTP source connector. If you do not wish to set additional configurations, these fields can be left at their default settings.
+
+1. **File Types**: Enter the desired file types to replicate as comma-separated values. Currently, only CSV and JSON are supported. The default value is `csv,json`.
+2. **Folder Path**: Enter a folder path to specify the directory on the remote server to be synced. For example, given the file structure:
 
-The SFTP source connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
+```
+Root
+| - logs
+|   | - 2021
+|   | - 2022
+|
+| - files
+|   | - 2021
+|   | - 2022
+```
 
-| Feature                       | Support | Notes                                                                                |
-|:------------------------------|:-------:|:-------------------------------------------------------------------------------------|
-| Full Refresh - Overwrite      |    ✅    | Warning: this mode deletes all previously synced data in the configured bucket path. |
-| Full Refresh - Append Sync    |    ❌    |                                                                                      |
-| Incremental - Append          |    ❌    |                                                                                      |
-| Incremental - Deduped History |    ❌    |                                                                                      |
-| Namespaces                    |    ❌    |                                                                                      |
+An input of `/logs/2022` will only replicate data contained within the specified folder, ignoring the `/files` and `/logs/2021` folders. Leaving this field blank will replicate all applicable files in the remote server's designated entry point.
 
+3. **File Pattern**: Enter a [regular expression](https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html) to specify a naming pattern for the files to be replicated. Consider the following example:
+
+```
+log-([0-9]{4})([0-9]{2})([0-9]{2})
+```
+
+This pattern will filter for files that match the format `log-YYYYMMDD`, where `YYYY`, `MM`, and `DD` represented four-digit, two-digit, and two-digit numbers, respectively. For example, `log-20230713`. Leaving this field blank will replicate all files not filtered by the previous two fields.
+
+## Supported sync modes
 
+The SFTP source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
+| Feature                        | Support | Notes                                                                                |
+| :----------------------------- | :-----: | :----------------------------------------------------------------------------------- |
+| Full Refresh - Overwrite       |   ✅    | Warning: this mode deletes all previously synced data in the configured bucket path. |
+| Full Refresh - Append Sync     |   ❌    |                                                                                      |
+| Incremental - Append           |   ❌    |                                                                                      |
+| Incremental - Append + Deduped |   ❌    |                                                                                      |
+| Namespaces                     |   ❌    |                                                                                      |
 
-## Supported Streams
+## Supported streams
 
-This source provides a single stream per file with a dynamic schema. The current supported type file: `.csv` and `.json`
+This source provides a single stream per file with a dynamic schema. The current supported file types are CSV and JSON.
 More formats \(e.g. Apache Avro\) will be supported in the future.
 
 ## Changelog
 
-| Version | Date       | Pull Request | Subject         |
-|:--------|:-----------|:-------------|:----------------|
-| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
-| 0.1.0   | 2021-24-05 |              | Initial version |
+| Version | Date       | Pull Request                                             | Subject                                                |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------- |
+| 0.1.2   | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
+| 0.1.0   | 2021-24-05 |                                                          | Initial version                                        |
diff --git a/docs/integrations/sources/shopify.inapp.md b/docs/integrations/sources/shopify.inapp.md
new file mode 100644
index 000000000000..6907ba41080b
--- /dev/null
+++ b/docs/integrations/sources/shopify.inapp.md
@@ -0,0 +1,65 @@
+## Prerequisites
+
+* An Active [Shopify store](https://www.shopify.com) 
+* The Admin API access token of your [Custom App](https://help.shopify.com/en/manual/apps/app-types/custom-apps).
+
+:::note
+
+Our Shopify Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Shopify is critical to your business, [please reach out to us](mailto:product@airbyte.io) to discuss how we may be able to partner on this effort.
+ 
+:::
+
+## Setup guide
+
+1. Name your source.
+2. Enter your Store name. You can find this in your URL when logged in to Shopify or within the Store details section of your Settings.
+3. Enter your Admin API access token. To set up the access token, you will need to set up a custom application. See instructions below on creating a custom app.
+4. Click Set up source
+
+## Creating a Custom App
+Authentication to the Shopify API requies a [custom application](https://help.shopify.com/en/manual/apps/app-types/custom-apps). Follow these instructions to create a custom app and find your Admin API Access Token.
+
+1. Navigate to Settings > App and sales channels > Develop apps > Create an app
+2. Name your new app
+3. Select **Configure Admin API scopes**
+4. Tick all the scopes prefixed with `read_` (e.g. `read_locations`,`read_price_rules`, etc ) and save. See below for the full list of scopes to allow.
+5. Click **Install app** to give this app access to your data. 
+6. Once installed, go to **API Credentials** to copy the **Admin API Access Token**. 
+
+### Scopes Required for Custom App
+Add the following scopes to your custom app to ensure Airbyte can sync all available data. To see a list of streams this source supports, see our full [Shopify documentation](https://docs.airbyte.com/integrations/sources/shopify/).
+* `read_analytics`
+* `read_assigned_fulfillment_orders`
+* `read_gdpr_data_request`
+* `read_locations`
+* `read_price_rules` 
+* `read_product_listings` 
+* `read_products` 
+* `read_reports` 
+* `read_resource_feedbacks` 
+* `read_script_tags` 
+* `read_shipping`
+* `read_locales`
+* `read_shopify_payments_accounts` 
+* `read_shopify_payments_bank_accounts` 
+* `read_shopify_payments_disputes`
+* `read_shopify_payments_payouts`
+* `read_content`
+* `read_themes`
+* `read_third_party_fulfillment_orders`
+* `read_translations`
+* `read_customers`
+* `read_discounts` 
+* `read_draft_orders` 
+* `read_fulfillments` 
+* `read_gift_cards`
+* `read_inventory`
+* `read_legal_policies` 
+* `read_marketing_events` 
+* `read_merchant_managed_fulfillment_orders` 
+* `read_online_store_pages`
+* `read_order_edits`
+* `read_orders`
+
+​
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Shopify](https://docs.airbyte.com/integrations/sources/shopify).
\ No newline at end of file
diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md
index b3e6891379df..7f256b67ae7d 100644
--- a/docs/integrations/sources/shopify.md
+++ b/docs/integrations/sources/shopify.md
@@ -5,80 +5,15 @@ description: >-
 
 # Shopify
 
-
 :::note
 
 Our Shopify Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Shopify is critical to your business, [please reach out to us](mailto:product@airbyte.io) to discuss how we may be able to partner on this effort.
 
 :::
 
-## Sync overview
-
-The Shopify source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run.
-
-This source can sync data for the [Shopify REST API](https://shopify.dev/api/admin-rest) and the [Shopify GraphQl API](https://shopify.dev/api/admin-graphql).
-
-## Troubleshooting
-
-Check out common troubleshooting issues for the Shopify source connector on our Discourse [here](https://discuss.airbyte.io/tags/c/connector/11/source-shopify).
-
-### Output schema
-
-This Source is capable of syncing the following core Streams:
-
-* [Abandoned Checkouts](https://help.shopify.com/en/api/reference/orders/abandoned_checkouts)
-* [Collects](https://help.shopify.com/en/api/reference/products/collect)
-* [Custom Collections](https://help.shopify.com/en/api/reference/products/customcollection)
-* [Customers](https://help.shopify.com/en/api/reference/customers)
-* [Draft Orders](https://help.shopify.com/en/api/reference/orders/draftorder)
-* [Discount Codes](https://shopify.dev/docs/admin-api/rest/reference/discounts/discountcode)
-* [Metafields](https://help.shopify.com/en/api/reference/metafield)
-* [Orders](https://help.shopify.com/en/api/reference/order)
-* [Orders Refunds](https://shopify.dev/api/admin/rest/reference/orders/refund)
-* [Orders Risks](https://shopify.dev/api/admin/rest/reference/orders/order-risk)
-* [Products](https://help.shopify.com/en/api/reference/products)
-* [Products (GraphQL)](https://shopify.dev/api/admin-graphql/2022-10/queries/products)
-* [Transactions](https://help.shopify.com/en/api/reference/orders/transaction)
-* [Balance Transactions](https://shopify.dev/api/admin-rest/2021-07/resources/transactions)
-* [Pages](https://help.shopify.com/en/api/reference/online-store/page)
-* [Price Rules](https://help.shopify.com/en/api/reference/discounts/pricerule)
-* [Locations](https://shopify.dev/api/admin-rest/2021-10/resources/location)
-* [InventoryItems](https://shopify.dev/api/admin-rest/2021-10/resources/inventoryItem)
-* [InventoryLevels](https://shopify.dev/api/admin-rest/2021-10/resources/inventorylevel)
-* [Fulfillment Orders](https://shopify.dev/api/admin-rest/2021-07/resources/fulfillmentorder)
-* [Fulfillments](https://shopify.dev/api/admin-rest/2021-07/resources/fulfillment)
-* [Shop](https://shopify.dev/api/admin-rest/2021-07/resources/shop)
-
-#### NOTE
-
-For better experience with `Incremental Refresh` the following is recommended:
-
-* `Order Refunds`, `Order Risks`, `Transactions` should be synced along with `Orders` stream.
-* `Discount Codes` should be synced along with `Price Rules` stream.
-
-If child streams are synced alone from the parent stream - the full sync will take place, and the records are filtered out afterwards.
-
-### Data type mapping
-
-| Integration Type | Airbyte Type |
-| :--- | :--- |
-| `string` | `string` |
-| `number` | `number` |
-| `array` | `array` |
-| `object` | `object` |
-| `boolean` | `boolean` |
-
-### Features
-
-| Feature | Supported?\(Yes/No\) |
-| :--- | :--- |
-| Full Refresh Sync | Yes |
-| Incremental - Append Sync | Yes |
-| Namespaces | No |
-
 ## Getting started
 
-This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applications) athentication methods.
+This connector supports the `API PASSWORD` (for private applications) athentication methods.
 
 ### Connect using `API PASSWORD` option
 
@@ -86,53 +21,122 @@ This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applica
 2. Enable private development if it isn't enabled.
 3. Create a private application.
 4. Select the resources you want to allow access to. Airbyte only needs read-level access.
-   * Note: The UI will show all possible data sources and will show errors when syncing if it doesn't have permissions to access a resource.
+   - Note: The UI will show all possible data sources and will show errors when syncing if it doesn't have permissions to access a resource.
 5. The password under the `Admin API` section is what you'll use as the `API PASSWORD` for the integration.
 6. You're ready to set up Shopify in Airbyte!
 
-### Output Streams Schemas
+### Scopes Required for Custom App
+
+Add the following scopes to your custom app to ensure Airbyte can sync all available data. To see a list of streams this source supports, see our full [Shopify documentation](https://docs.airbyte.com/integrations/sources/shopify/).
+
+- `read_analytics`
+- `read_assigned_fulfillment_orders`
+- `read_gdpr_data_request`
+- `read_locations`
+- `read_price_rules`
+- `read_product_listings`
+- `read_products`
+- `read_reports`
+- `read_resource_feedbacks`
+- `read_script_tags`
+- `read_shipping`
+- `read_locales`
+- `read_shopify_payments_accounts`
+- `read_shopify_payments_bank_accounts`
+- `read_shopify_payments_disputes`
+- `read_shopify_payments_payouts`
+- `read_content`
+- `read_themes`
+- `read_third_party_fulfillment_orders`
+- `read_translations`
+- `read_customers`
+- `read_discounts`
+- `read_draft_orders`
+- `read_fulfillments`
+- `read_gift_cards`
+- `read_inventory`
+- `read_legal_policies`
+- `read_marketing_events`
+- `read_merchant_managed_fulfillment_orders`
+- `read_online_store_pages`
+- `read_order_edits`
+- `read_orders`
+
+## Supported sync modes
+
+The Shopify source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run.
+
+This source can sync data for the [Shopify REST API](https://shopify.dev/api/admin-rest) and the [Shopify GraphQl API](https://shopify.dev/api/admin-graphql).
+
+## Troubleshooting tips
+
+Check out common troubleshooting issues for the Shopify source connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions).
+
+## Supported Streams
 
 This Source is capable of syncing the following core Streams:
 
-* [Articles](https://shopify.dev/api/admin-rest/2022-01/resources/article)
-* [Blogs](https://shopify.dev/api/admin-rest/2022-01/resources/blog)
-* [Abandoned Checkouts](https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts#top)
-* [Collects](https://shopify.dev/api/admin-rest/2022-01/resources/collect#top)
-* [Collections](https://shopify.dev/api/admin-rest/2022-01/resources/collection)
-* [Custom Collections](https://shopify.dev/api/admin-rest/2022-01/resources/customcollection#top)
-* [Smart Collections](https://shopify.dev/api/admin-rest/2022-01/resources/smartcollection)
-* [Customers](https://shopify.dev/api/admin-rest/2022-01/resources/customer#top)
-* [Draft Orders](https://shopify.dev/api/admin-rest/2022-01/resources/draftorder#top)
-* [Discount Codes](https://shopify.dev/api/admin-rest/2022-01/resources/discountcode#top)
-* [Metafields](https://shopify.dev/api/admin-rest/2022-01/resources/metafield#top)
-* [Orders](https://shopify.dev/api/admin-rest/2022-01/resources/order#top)
-* [Orders Refunds](https://shopify.dev/api/admin-rest/2022-01/resources/refund#top)
-* [Orders Risks](https://shopify.dev/api/admin-rest/2022-01/resources/order-risk#top)
-* [Products](https://shopify.dev/api/admin-rest/2022-01/resources/product#top)
-* [Products (GraphQL)](https://shopify.dev/api/admin-graphql/2022-10/queries/products)
-* [Product Images](https://shopify.dev/api/admin-rest/2022-01/resources/product-image)
-* [Product Variants](https://shopify.dev/api/admin-rest/2022-01/resources/product-variant)
-* [Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/transaction#top)
-* [Tender Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/tendertransaction)
-* [Pages](https://shopify.dev/api/admin-rest/2022-01/resources/page#top)
-* [Price Rules](https://shopify.dev/api/admin-rest/2022-01/resources/pricerule#top)
-* [Locations](https://shopify.dev/api/admin-rest/2022-01/resources/location)
-* [InventoryItems](https://shopify.dev/api/admin-rest/2022-01/resources/inventoryItem)
-* [InventoryLevels](https://shopify.dev/api/admin-rest/2021-01/resources/inventorylevel)
-* [Fulfillment Orders](https://shopify.dev/api/admin-rest/2022-01/resources/fulfillmentorder)
-* [Fulfillments](https://shopify.dev/api/admin-rest/2022-01/resources/fulfillment)
-* [Shop](https://shopify.dev/api/admin-rest/2022-01/resources/shop)
-
-#### Notes
+- [Articles](https://shopify.dev/api/admin-rest/2022-01/resources/article)
+- [Blogs](https://shopify.dev/api/admin-rest/2022-01/resources/blog)
+- [Abandoned Checkouts](https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts#top)
+- [Collects](https://shopify.dev/api/admin-rest/2022-01/resources/collect#top)
+- [Collections](https://shopify.dev/api/admin-rest/2022-01/resources/collection)
+- [Countries](https://shopify.dev/docs/api/admin-rest/2023-04/resources/country)
+- [Custom Collections](https://shopify.dev/api/admin-rest/2022-01/resources/customcollection#top)
+- [CustomerAddress](https://shopify.dev/docs/api/admin-rest/2023-04/resources/customer-address)
+- [CustomerSavedSearch](https://shopify.dev/docs/api/admin-rest/2023-04/resources/customersavedsearch)
+- [Smart Collections](https://shopify.dev/api/admin-rest/2022-01/resources/smartcollection)
+- [Customers](https://shopify.dev/api/admin-rest/2022-01/resources/customer#top)
+- [Draft Orders](https://shopify.dev/api/admin-rest/2022-01/resources/draftorder#top)
+- [Discount Codes](https://shopify.dev/api/admin-rest/2022-01/resources/discountcode#top)
+- [Disputes](https://shopify.dev/docs/api/admin-rest/2023-07/resources/dispute)
+- [Metafields](https://shopify.dev/api/admin-rest/2022-01/resources/metafield#top)
+- [Orders](https://shopify.dev/api/admin-rest/2022-01/resources/order#top)
+- [Orders Refunds](https://shopify.dev/api/admin-rest/2022-01/resources/refund#top)
+- [Orders Risks](https://shopify.dev/api/admin-rest/2022-01/resources/order-risk#top)
+- [Products](https://shopify.dev/api/admin-rest/2022-01/resources/product#top)
+- [Products (GraphQL)](https://shopify.dev/api/admin-graphql/2022-10/queries/products)
+- [Product Images](https://shopify.dev/api/admin-rest/2022-01/resources/product-image)
+- [Product Variants](https://shopify.dev/api/admin-rest/2022-01/resources/product-variant)
+- [Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/transaction#top)
+- [Tender Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/tendertransaction)
+- [Pages](https://shopify.dev/api/admin-rest/2022-01/resources/page#top)
+- [Price Rules](https://shopify.dev/api/admin-rest/2022-01/resources/pricerule#top)
+- [Locations](https://shopify.dev/api/admin-rest/2022-01/resources/location)
+- [InventoryItems](https://shopify.dev/api/admin-rest/2022-01/resources/inventoryItem)
+- [InventoryLevels](https://shopify.dev/api/admin-rest/2021-01/resources/inventorylevel)
+- [Fulfillment Orders](https://shopify.dev/api/admin-rest/2022-01/resources/fulfillmentorder)
+- [Fulfillments](https://shopify.dev/api/admin-rest/2022-01/resources/fulfillment)
+- [Shop](https://shopify.dev/api/admin-rest/2022-01/resources/shop)
+
+### Stream sync recommendations
 
 For better experience with `Incremental Refresh` the following is recommended:
 
-* `Order Refunds`, `Order Risks`, `Transactions` should be synced along with `Orders` stream.
-* `Discount Codes` should be synced along with `Price Rules` stream.
+- `Order Refunds`, `Order Risks`, `Transactions` should be synced along with `Orders` stream.
+- `Discount Codes` should be synced along with `Price Rules` stream.
 
 If child streams are synced alone from the parent stream - the full sync will take place, and the records are filtered out afterwards.
 
-### Performance considerations
+## Data type mapping
+
+| Integration Type | Airbyte Type |
+| :--------------- | :----------- |
+| `string`         | `string`     |
+| `number`         | `number`     |
+| `array`          | `array`      |
+| `object`         | `object`     |
+| `boolean`        | `boolean`    |
+
+## Features
+
+| Feature                   | Supported?\(Yes/No\) |
+| :------------------------ | :------------------- |
+| Full Refresh Sync         | Yes                  |
+| Incremental - Append Sync | Yes                  |
+| Namespaces                | No                   |
+
+## Performance considerations
 
 Shopify has some [rate limit restrictions](https://shopify.dev/concepts/about-apis/rate-limits). Typically, there should not be issues with throttling or exceeding the rate limits but in some edge cases, user can receive the warning message as follows:
 
@@ -144,48 +148,54 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro
 
 ## Changelog
 
-| Version | Date       | Pull Request                                              | Subject                                                                                                   |
-|:--------|:-----------|:----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------|
-| 0.3.4   | 2023-05-10 | [25961](https://github.com/airbytehq/airbyte/pull/25961)  | Added validation for `shop` in input configuration (accepts non-url-like inputs) |
-| 0.3.3   | 2023-04-12 | [25110](https://github.com/airbytehq/airbyte/pull/25110)  | Fix issue when `cursor_field` is `"None"`, added missing properties to stream schemas, fixed `access_scopes` validation error |
-| 0.3.2   | 2023-02-27 | [23473](https://github.com/airbytehq/airbyte/pull/23473)  | Fixed OOM / Memory leak issue for Airbyte Cloud                                                           |
-| 0.3.1   | 2023-01-16 | [21461](https://github.com/airbytehq/airbyte/pull/21461)  | Add `discount_applications` to `orders` stream                                                            |
-| 0.3.0   | 2022-11-16 | [19492](https://github.com/airbytehq/airbyte/pull/19492)  | Add support for graphql and add a graphql products stream                                                 |
-| 0.2.0   | 2022-10-21 | [18298](https://github.com/airbytehq/airbyte/pull/18298)  | Updated API version to the `2022-10`, make stream schemas backward cpmpatible                             |
-| 0.1.39  | 2022-10-13 | [17962](https://github.com/airbytehq/airbyte/pull/17962)  | Add metafield streams; support for nested list streams                                                    |
-| 0.1.38  | 2022-10-10 | [17777](https://github.com/airbytehq/airbyte/pull/17777)  | Fixed `404` for configured streams, fix missing `cursor` error for old records                            |
-| 0.1.37  | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500)  | Improve input configuration copy                                                                          |
-| 0.1.36  | 2022-03-22 | [9850](https://github.com/airbytehq/airbyte/pull/9850)    | Added `BalanceTransactions` stream                                                                        |
-| 0.1.35  | 2022-03-07 | [10915](https://github.com/airbytehq/airbyte/pull/10915)  | Fix a bug which caused `full-refresh` syncs of child REST entities configured for `incremental`           |
-| 0.1.34  | 2022-03-02 | [10794](https://github.com/airbytehq/airbyte/pull/10794)  | Minor specification re-order, fixed links in documentation                                                |
-| 0.1.33  | 2022-02-17 | [10419](https://github.com/airbytehq/airbyte/pull/10419)  | Fixed wrong field type for tax_exemptions for `Abandoned_checkouts` stream                                |
-| 0.1.32  | 2022-02-18 | [10449](https://github.com/airbytehq/airbyte/pull/10449)  | Added `tender_transactions` stream                                                                        |
-| 0.1.31  | 2022-02-08 | [10175](https://github.com/airbytehq/airbyte/pull/10175)  | Fixed compatibility issues for legacy user config                                                         |
-| 0.1.30  | 2022-01-24 | [9648](https://github.com/airbytehq/airbyte/pull/9648)    | Added permission validation before sync                                                                   |
-| 0.1.29  | 2022-01-20 | [9049](https://github.com/airbytehq/airbyte/pull/9248)    | Added `shop_url` to the record for all streams                                                            |
-| 0.1.28  | 2022-01-19 | [9591](https://github.com/airbytehq/airbyte/pull/9591)    | Implemented `OAuth2.0` authentication method for Airbyte Cloud                                            |
-| 0.1.27  | 2021-12-22 | [9049](https://github.com/airbytehq/airbyte/pull/9049)    | Update connector fields title/description                                                                 |
-| 0.1.26  | 2021-12-14 | [8597](https://github.com/airbytehq/airbyte/pull/8597)    | Fix `mismatched number of tables` for base-normalization, increased performance of `order_refunds` stream |
-| 0.1.25  | 2021-12-02 | [8297](https://github.com/airbytehq/airbyte/pull/8297)    | Added Shop stream                                                                                         |
-| 0.1.24  | 2021-11-30 | [7783](https://github.com/airbytehq/airbyte/pull/7783)    | Reviewed and corrected schemas for all streams                                                            |
-| 0.1.23  | 2021-11-15 | [7973](https://github.com/airbytehq/airbyte/pull/7973)    | Added `InventoryItems`                                                                                    |
-| 0.1.22  | 2021-10-18 | [7101](https://github.com/airbytehq/airbyte/pull/7107)    | Added FulfillmentOrders, Fulfillments streams                                                             |
-| 0.1.21  | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382)    | Fixed `InventoryLevels` primary key                                                                       |
-| 0.1.20  | 2021-10-14 | [7063](https://github.com/airbytehq/airbyte/pull/7063)    | Added `Location` and `InventoryLevels` as streams                                                         |
-| 0.1.19  | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951)    | Added support of `OAuth 2.0` authorisation option                                                         |
-| 0.1.18  | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056)    | Added `pre_tax_price` to the `orders/line_items` schema                                                   |
-| 0.1.17  | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244)    | Created data type enforcer for converting prices into numbers                                             |
-| 0.1.16  | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945)    | Fixed the connector's performance for `Incremental refresh`                                               |
-| 0.1.15  | 2021-09-02 | [5853](https://github.com/airbytehq/airbyte/pull/5853)    | Fixed `amount` type in `order_refund` schema                                                              |
-| 0.1.14  | 2021-09-02 | [5801](https://github.com/airbytehq/airbyte/pull/5801)    | Fixed `line_items/discount allocations` & `duties` parts of `orders` schema                               |
-| 0.1.13  | 2021-08-17 | [5470](https://github.com/airbytehq/airbyte/pull/5470)    | Fixed rate limits throttling                                                                              |
-| 0.1.12  | 2021-08-09 | [5276](https://github.com/airbytehq/airbyte/pull/5276)    | Add status property to product schema                                                                     |
-| 0.1.11  | 2021-07-23 | [4943](https://github.com/airbytehq/airbyte/pull/4943)    | Fix products schema up to API 2021-07                                                                     |
-| 0.1.10  | 2021-07-19 | [4830](https://github.com/airbytehq/airbyte/pull/4830)    | Fix for streams json schemas, upgrade to API version 2021-07                                              |
-| 0.1.9   | 2021-07-04 | [4472](https://github.com/airbytehq/airbyte/pull/4472)    | Incremental sync is now using updated\_at instead of since\_id by default                                 |
-| 0.1.8   | 2021-06-29 | [4121](https://github.com/airbytehq/airbyte/pull/4121)    | Add draft orders stream                                                                                   |
-| 0.1.7   | 2021-06-26 | [4290](https://github.com/airbytehq/airbyte/pull/4290)    | Fixed the bug when limiting output records to 1 caused infinity loop                                      |
-| 0.1.6   | 2021-06-24 | [4009](https://github.com/airbytehq/airbyte/pull/4009)    | Add pages, price rules and discount codes streams                                                         |
-| 0.1.5   | 2021-06-10 | [3973](https://github.com/airbytehq/airbyte/pull/3973)    | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support                                                           |
-| 0.1.4   | 2021-06-09 | [3926](https://github.com/airbytehq/airbyte/pull/3926)    | New attributes to Orders schema                                                                           |
-| 0.1.3   | 2021-06-08 | [3787](https://github.com/airbytehq/airbyte/pull/3787)    | Add Native Shopify Source Connector                                                                       |
+| Version | Date       | Pull Request                                             | Subject                                                                                                                         |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------ |
+| 0.6.2   | 2023-08-09 | [29302](https://github.com/airbytehq/airbyte/pull/29302) | Handle the `Internal Server Error` when entity could be fetched                 |
+| 0.6.1   | 2023-08-08 | [28291](https://github.com/airbytehq/airbyte/pull/28291) | Allow `shop` field to accept `*.myshopify.com` shop names, updated `OAuth Spec`                  |
+| 0.6.0   | 2023-08-02 | [28770](https://github.com/airbytehq/airbyte/pull/28770) | Added `Disputes` stream  |
+| 0.5.1   | 2023-07-13 | [28700](https://github.com/airbytehq/airbyte/pull/28700) | Improved `error messages` with more user-friendly description, refactored code  |
+| 0.5.0   | 2023-06-13 | [27732](https://github.com/airbytehq/airbyte/pull/27732) | License Update: Elv2                                                                                                            |
+| 0.4.0   | 2023-06-13 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Added `CustomerSavedSearch`, `CustomerAddress` and `Countries` streams                                                          |
+| 0.3.4   | 2023-05-10 | [25961](https://github.com/airbytehq/airbyte/pull/25961) | Added validation for `shop` in input configuration (accepts non-url-like inputs)                                                |
+| 0.3.3   | 2023-04-12 | [25110](https://github.com/airbytehq/airbyte/pull/25110) | Fixed issue when `cursor_field` is `"None"`, added missing properties to stream schemas, fixed `access_scopes` validation error |
+| 0.3.2   | 2023-02-27 | [23473](https://github.com/airbytehq/airbyte/pull/23473) | Fixed OOM / Memory leak issue for Airbyte Cloud                                                                                 |
+| 0.3.1   | 2023-01-16 | [21461](https://github.com/airbytehq/airbyte/pull/21461) | Added `discount_applications` to `orders` stream                                                                                |
+| 0.3.0   | 2022-11-16 | [19492](https://github.com/airbytehq/airbyte/pull/19492) | Added support for graphql and add a graphql products stream                                                                     |
+| 0.2.0   | 2022-10-21 | [18298](https://github.com/airbytehq/airbyte/pull/18298) | Updated API version to the `2022-10`, make stream schemas backward cpmpatible                                                   |
+| 0.1.39  | 2022-10-13 | [17962](https://github.com/airbytehq/airbyte/pull/17962) | Added metafield streams; support for nested list streams                                                                        |
+| 0.1.38  | 2022-10-10 | [17777](https://github.com/airbytehq/airbyte/pull/17777) | Fixed `404` for configured streams, fix missing `cursor` error for old records                                                  |
+| 0.1.37  | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy                                                                                                |
+| 0.1.36  | 2022-03-22 | [9850](https://github.com/airbytehq/airbyte/pull/9850)   | Added `BalanceTransactions` stream                                                                                              |
+| 0.1.35  | 2022-03-07 | [10915](https://github.com/airbytehq/airbyte/pull/10915) | Fixed a bug which caused `full-refresh` syncs of child REST entities configured for `incremental`                               |
+| 0.1.34  | 2022-03-02 | [10794](https://github.com/airbytehq/airbyte/pull/10794) | Minor specification re-order, fixed links in documentation                                                                      |
+| 0.1.33  | 2022-02-17 | [10419](https://github.com/airbytehq/airbyte/pull/10419) | Fixed wrong field type for tax_exemptions for `Abandoned_checkouts` stream                                                      |
+| 0.1.32  | 2022-02-18 | [10449](https://github.com/airbytehq/airbyte/pull/10449) | Added `tender_transactions` stream                                                                                              |
+| 0.1.31  | 2022-02-08 | [10175](https://github.com/airbytehq/airbyte/pull/10175) | Fixed compatibility issues for legacy user config                                                                               |
+| 0.1.30  | 2022-01-24 | [9648](https://github.com/airbytehq/airbyte/pull/9648)   | Added permission validation before sync                                                                                         |
+| 0.1.29  | 2022-01-20 | [9049](https://github.com/airbytehq/airbyte/pull/9248)   | Added `shop_url` to the record for all streams                                                                                  |
+| 0.1.28  | 2022-01-19 | [9591](https://github.com/airbytehq/airbyte/pull/9591)   | Implemented `OAuth2.0` authentication method for Airbyte Cloud                                                                  |
+| 0.1.27  | 2021-12-22 | [9049](https://github.com/airbytehq/airbyte/pull/9049)   | Updated connector fields title/description                                                                                      |
+| 0.1.26  | 2021-12-14 | [8597](https://github.com/airbytehq/airbyte/pull/8597)   | Fixed `mismatched number of tables` for base-normalization, increased performance of `order_refunds` stream                     |
+| 0.1.25  | 2021-12-02 | [8297](https://github.com/airbytehq/airbyte/pull/8297)   | Added Shop stream                                                                                                               |
+| 0.1.24  | 2021-11-30 | [7783](https://github.com/airbytehq/airbyte/pull/7783)   | Reviewed and corrected schemas for all streams                                                                                  |
+| 0.1.23  | 2021-11-15 | [7973](https://github.com/airbytehq/airbyte/pull/7973)   | Added `InventoryItems`                                                                                                          |
+| 0.1.22  | 2021-10-18 | [7101](https://github.com/airbytehq/airbyte/pull/7107)   | Added FulfillmentOrders, Fulfillments streams                                                                                   |
+| 0.1.21  | 2021-10-14 | [7382](https://github.com/airbytehq/airbyte/pull/7382)   | Fixed `InventoryLevels` primary key                                                                                             |
+| 0.1.20  | 2021-10-14 | [7063](https://github.com/airbytehq/airbyte/pull/7063)   | Added `Location` and `InventoryLevels` as streams                                                                               |
+| 0.1.19  | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951)   | Added support of `OAuth 2.0` authorisation option                                                                               |
+| 0.1.18  | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056)   | Added `pre_tax_price` to the `orders/line_items` schema                                                                         |
+| 0.1.17  | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244)   | Created data type enforcer for converting prices into numbers                                                                   |
+| 0.1.16  | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945)   | Fixed the connector's performance for `Incremental refresh`                                                                     |
+| 0.1.15  | 2021-09-02 | [5853](https://github.com/airbytehq/airbyte/pull/5853)   | Fixed `amount` type in `order_refund` schema                                                                                    |
+| 0.1.14  | 2021-09-02 | [5801](https://github.com/airbytehq/airbyte/pull/5801)   | Fixed `line_items/discount allocations` & `duties` parts of `orders` schema                                                     |
+| 0.1.13  | 2021-08-17 | [5470](https://github.com/airbytehq/airbyte/pull/5470)   | Fixed rate limits throttling                                                                                                    |
+| 0.1.12  | 2021-08-09 | [5276](https://github.com/airbytehq/airbyte/pull/5276)   | Added status property to product schema                                                                                         |
+| 0.1.11  | 2021-07-23 | [4943](https://github.com/airbytehq/airbyte/pull/4943)   | Fixed products schema up to API 2021-07                                                                                         |
+| 0.1.10  | 2021-07-19 | [4830](https://github.com/airbytehq/airbyte/pull/4830)   | Fixed for streams json schemas, upgrade to API version 2021-07                                                                  |
+| 0.1.9   | 2021-07-04 | [4472](https://github.com/airbytehq/airbyte/pull/4472)   | Incremental sync is now using updated_at instead of since_id by default                                                         |
+| 0.1.8   | 2021-06-29 | [4121](https://github.com/airbytehq/airbyte/pull/4121)   | Added draft orders stream                                                                                                       |
+| 0.1.7   | 2021-06-26 | [4290](https://github.com/airbytehq/airbyte/pull/4290)   | Fixed the bug when limiting output records to 1 caused infinity loop                                                            |
+| 0.1.6   | 2021-06-24 | [4009](https://github.com/airbytehq/airbyte/pull/4009)   | Added pages, price rules and discount codes streams                                                                             |
+| 0.1.5   | 2021-06-10 | [3973](https://github.com/airbytehq/airbyte/pull/3973)   | Added `AIRBYTE_ENTRYPOINT` for Kubernetes support                                                                               |
+| 0.1.4   | 2021-06-09 | [3926](https://github.com/airbytehq/airbyte/pull/3926)   | New attributes to Orders schema                                                                                                 |
+| 0.1.3   | 2021-06-08 | [3787](https://github.com/airbytehq/airbyte/pull/3787)   | Added Native Shopify Source Connector                                                                                           |
diff --git a/docs/integrations/sources/shortio.md b/docs/integrations/sources/shortio.md
index 4bf26cee51fe..ad2f692dfd12 100644
--- a/docs/integrations/sources/shortio.md
+++ b/docs/integrations/sources/shortio.md
@@ -39,10 +39,11 @@ This Source is capable of syncing the following Streams:
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.1.3 | 2022-08-01 | [15066](https://github.com/airbytehq/airbyte/pull/15066) | Update primary key to `idString` |
-| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications |
-| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies |
-| 0.1.0 | 2021-08-16 | [3787](https://github.com/airbytehq/airbyte/pull/5418) | Add Native Shortio Source Connector |
+| Version | Date       | Pull Request                                             | Subject                                                            |
+| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------- |
+|  0.2.0  | 2023-08-02 | [28950](https://github.com/airbytehq/airbyte/pull/28950) | Migrate to Low-Code CDK                                            |
+|  0.1.3  | 2022-08-01 | [15066](https://github.com/airbytehq/airbyte/pull/15066) | Update primary key to `idString`                                   |
+|  0.1.2  | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628)   | Update fields in source-connectors specifications                  |
+|  0.1.1  | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                    |
+|  0.1.0  | 2021-08-16 | [3787](https://github.com/airbytehq/airbyte/pull/5418)   | Add Native Shortio Source Connector                                |
 
diff --git a/docs/integrations/sources/slack.inapp.md b/docs/integrations/sources/slack.inapp.md
new file mode 100644
index 000000000000..fabdd013ab8a
--- /dev/null
+++ b/docs/integrations/sources/slack.inapp.md
@@ -0,0 +1,58 @@
+## Prerequisites
+- Access to Slack via OAuth or API Token (via Slack App or Legacy API Key)
+
+
+## Setup Guide
+
+1. Enter a name for your connector
+2. Select `Authenticate your Slack account` (preferred) and authorize into the Slack account. To use an API token instead, see the instructions below on creating one.
+3. Toggle on **Join all channels** (recommended) to join all channels the user has access to or to sync data only from channels the app (if using API token) is already in. If false, you'll need to manually add all the channels from which you'd like to sync messages.
+4. (Optional) Enter a **Threads Lookback Window (Days)** to set how far back to look for messages in threads from when each sync start.
+5. (Optional) Enter a **Start Date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. Data created on and after this date will be replicated.
+8. (Optional) Enter your `Channel name filter` to filter the list of channels Airbyte can access. If none are entered, Airbyte will sync all channels. It can be helpful to only sync required channels to avoid Slack's [requests limits](https://api.slack.com/docs/rate-limits).
+
+
+9. Click **Set up source**.
+
+### Creating an API token
+
+You can no longer create "Legacy" API Keys, but if you already have one, you can use it with this source as the API key and skip setting up an application.
+
+In order to pull data out of your Slack instance, you need to create a Slack App. This may sound daunting, but it is actually pretty straight forward. Slack supplies [documentation](https://api.slack.com/start) on how to build apps. Feel free to follow that if you want to do something fancy. We'll describe the steps we followed to creat the Slack App for this tutorial.
+
+:::info
+This tutorial assumes that you are an administrator on your slack instance. If you are not, you will need to coordinate with your administrator on the steps that require setting permissions for your app.
+:::
+
+1. Go to the [apps page](https://api.slack.com/apps)
+2. Click "Create New App"
+3. It will request a name and the slack instance you want to create the app for. Make sure you select the instance form which you want to pull data.
+4. Completing that form will take you to the "Basic Information" page for your app.
+5. Now we need to grant the correct permissions to the app. \(This is the part that requires you to be an administrator\). Go to "Permissions". Then under "Bot Token Scopes" click on "Add an OAuth Scope". We will now need to add the following scopes:
+
+   ```text
+    channels:history
+    channels:join
+    channels:read
+    files:read
+    groups:read
+    links:read
+    reactions:read
+    remote_files:read
+    team:read
+    usergroups:read
+    users.profile:read
+    users:read
+   ```
+
+   This may look daunting, but the search functionality in the dropdown should make this part go pretty quick.
+
+6. Scroll to the top of the page and click "Install to Workspace". This will generate a "Bot User OAuth Access Token". We will need this in a moment.
+7. Now go to your slack instance. For any public channel go to info => more => add apps. In the search bar search for the name of your app. \(If using the desktop version of slack, you may need to restart Slack for it to pick up the new app\). Airbyte will only replicate messages from channels that the Slack bot has been added to.
+
+   ![](../../.gitbook/assets/slack-add-apps.png)
+
+8. In Airbyte, create a Slack source. The "Bot User OAuth Access Token" from the earlier should be used as the token.
+9. You can now pull data from your slack instance!
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Slack](https://docs.airbyte.com/integrations/sources/slack).
diff --git a/docs/integrations/sources/smartsheets.md b/docs/integrations/sources/smartsheets.md
index 308018ef5751..4d9e62d85997 100644
--- a/docs/integrations/sources/smartsheets.md
+++ b/docs/integrations/sources/smartsheets.md
@@ -110,7 +110,8 @@ The remaining column datatypes supported by Smartsheets are more complex types (
 
 | Version | Date       | Pull Request                                             | Subject                                                   |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------|
-| 1.1.0   | 2023-06-02 | [22382](https://github.com/airbytehq/airbyte/pull/22382) | Add support for ingesting metadata fields |
+| 1.1.1   | 2023-06-06 | [27096](https://github.com/airbytehq/airbyte/pull/27096) | Fix error when optional metadata fields are not set       |
+| 1.1.0   | 2023-06-02 | [22382](https://github.com/airbytehq/airbyte/pull/22382) | Add support for ingesting metadata fields                 |
 | 1.0.2   | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Fix dependencies conflict                                 |
 | 1.0.1   | 2023-04-27 | [25562](https://github.com/airbytehq/airbyte/pull/25562) | Update testing dependencies                               |
 | 1.0.0   | 2023-02-19 | [23237](https://github.com/airbytehq/airbyte/pull/23237) | Fix OAuth2.0 token refresh                                |
diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md
index f1eab3a6fa3c..73c07998cbe1 100644
--- a/docs/integrations/sources/snapchat-marketing.md
+++ b/docs/integrations/sources/snapchat-marketing.md
@@ -111,22 +111,23 @@ Snapchat Marketing API has limitations to 1000 items per page.
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                     |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------|
-| 0.2.0   | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce new field in the `Campaigns` stream schema        |
-| 0.1.16  | 2023-04-20 | [20897](https://github.com/airbytehq/airbyte/pull/20897) | Add missing fields to Basic Stats schema                    |    
-| 0.1.15  | 2023-03-02 | [22869](https://github.com/airbytehq/airbyte/pull/22869) | Specified date formatting in specification                  |                                        
-| 0.1.14  | 2023-02-10 | [22808](https://github.com/airbytehq/airbyte/pull/22808) | Enable default `AvailabilityStrategy`                       |
-| 0.1.13  | 2023-01-27 | [22023](https://github.com/airbytehq/airbyte/pull/22023) | Set `AvailabilityStrategy` for streams explicitly to `None` |
-| 0.1.12  | 2023-01-11 | [21267](https://github.com/airbytehq/airbyte/pull/21267) | Fix parse empty error response                              |
-| 0.1.11  | 2022-12-23 | [20865](https://github.com/airbytehq/airbyte/pull/20865) | Handle 403 permission error                                 |
-| 0.1.10  | 2022-12-15 | [20537](https://github.com/airbytehq/airbyte/pull/20537) | Run on CDK 0.15.0                                           |
-| 0.1.9   | 2022-12-14 | [20498](https://github.com/airbytehq/airbyte/pull/20498) | Fix output state when no records are read                   |
-| 0.1.8   | 2022-10-05 | [17596](https://github.com/airbytehq/airbyte/pull/17596) | Retry 429 and 5xx errors when refreshing access token       |
-| 0.1.6   | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from specs              |
-| 0.1.5   | 2022-07-13 | [14577](https://github.com/airbytehq/airbyte/pull/14577) | Added stats streams hourly, daily, lifetime                 |
-| 0.1.4   | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Update titles and descriptions                              |
-| 0.1.3   | 2021-11-10 | [7811](https://github.com/airbytehq/airbyte/pull/7811)   | Add oauth2.0, fix stream_state                              |
-| 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                             |
-| 0.1.1   | 2021-07-29 | [5072](https://github.com/airbytehq/airbyte/pull/5072)   | Fix bug with incorrect stream\_state value                  |
-| 0.1.0   | 2021-07-26 | [4843](https://github.com/airbytehq/airbyte/pull/4843)   | Initial release supporting the Snapchat Marketing API       |
\ No newline at end of file
+| Version | Date       | Pull Request                                             | Subject                                                       |
+|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------|
+| 0.3.0   | 2023-05-22 | [26358](https://github.com/airbytehq/airbyte/pull/26358) | Remove deprecated authSpecification in favour of advancedAuth |
+| 0.2.0   | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce new field in the `Campaigns` stream schema          |
+| 0.1.16  | 2023-04-20 | [20897](https://github.com/airbytehq/airbyte/pull/20897) | Add missing fields to Basic Stats schema                      |    
+| 0.1.15  | 2023-03-02 | [22869](https://github.com/airbytehq/airbyte/pull/22869) | Specified date formatting in specification                    |                                        
+| 0.1.14  | 2023-02-10 | [22808](https://github.com/airbytehq/airbyte/pull/22808) | Enable default `AvailabilityStrategy`                         |
+| 0.1.13  | 2023-01-27 | [22023](https://github.com/airbytehq/airbyte/pull/22023) | Set `AvailabilityStrategy` for streams explicitly to `None`   |
+| 0.1.12  | 2023-01-11 | [21267](https://github.com/airbytehq/airbyte/pull/21267) | Fix parse empty error response                                |
+| 0.1.11  | 2022-12-23 | [20865](https://github.com/airbytehq/airbyte/pull/20865) | Handle 403 permission error                                   |
+| 0.1.10  | 2022-12-15 | [20537](https://github.com/airbytehq/airbyte/pull/20537) | Run on CDK 0.15.0                                             |
+| 0.1.9   | 2022-12-14 | [20498](https://github.com/airbytehq/airbyte/pull/20498) | Fix output state when no records are read                     |
+| 0.1.8   | 2022-10-05 | [17596](https://github.com/airbytehq/airbyte/pull/17596) | Retry 429 and 5xx errors when refreshing access token         |
+| 0.1.6   | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from specs                |
+| 0.1.5   | 2022-07-13 | [14577](https://github.com/airbytehq/airbyte/pull/14577) | Added stats streams hourly, daily, lifetime                   |
+| 0.1.4   | 2021-12-07 | [8429](https://github.com/airbytehq/airbyte/pull/8429)   | Update titles and descriptions                                |
+| 0.1.3   | 2021-11-10 | [7811](https://github.com/airbytehq/airbyte/pull/7811)   | Add oauth2.0, fix stream_state                                |
+| 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                               |
+| 0.1.1   | 2021-07-29 | [5072](https://github.com/airbytehq/airbyte/pull/5072)   | Fix bug with incorrect stream\_state value                    |
+| 0.1.0   | 2021-07-26 | [4843](https://github.com/airbytehq/airbyte/pull/4843)   | Initial release supporting the Snapchat Marketing API         |
\ No newline at end of file
diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md
index 27fc7ea155e7..0e93412be2e1 100644
--- a/docs/integrations/sources/snowflake.md
+++ b/docs/integrations/sources/snowflake.md
@@ -4,7 +4,7 @@
 
 The Snowflake source allows you to sync data from Snowflake. It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run.
 
-This Snowflake source connector is built on top of the source-jdbc code base and is configured to rely on JDBC  3.13.22 [Snowflake driver](https://github.com/snowflakedb/snowflake-jdbc) as described in Snowflake [documentation](https://docs.snowflake.com/en/user-guide/jdbc.html).
+This Snowflake source connector is built on top of the source-jdbc code base and is configured to rely on JDBC 3.13.22 [Snowflake driver](https://github.com/snowflakedb/snowflake-jdbc) as described in Snowflake [documentation](https://docs.snowflake.com/en/user-guide/jdbc.html).
 
 #### Resulting schema
 
@@ -12,11 +12,11 @@ The Snowflake source does not alter the schema present in your warehouse. Depend
 
 #### Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental - Append Sync | Yes |  |
-| Namespaces | Yes |  |
+| Feature                   | Supported?\(Yes/No\) | Notes |
+| :------------------------ | :------------------- | :---- |
+| Full Refresh Sync         | Yes                  |       |
+| Incremental - Append Sync | Yes                  |       |
+| Namespaces                | Yes                  |       |
 
 ## Getting started
 
@@ -73,31 +73,33 @@ You can limit this grant down to specific schemas instead of the whole database.
 Your database user should now be ready for use with Airbyte.
 
 ###Authentication
+
 #### There are 2 way ways of oauth supported: login\pass and oauth2.
 
 ### Login and Password
-| Field | Description                                                                                                                                                                                       |
-|---|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` |
-| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                           |
-| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE`                                                                                                   |
-| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE`                                                                                                     |
-| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The schema whose tables this replication is targeting. If no schema is specified, all tables with permission will be presented regardless of their schema.                                        |
-| Username | The username you created in Step 2 to allow Airbyte to access the database. Example: `AIRBYTE_USER`                                                                                               |
-| Password | The password associated with the username.                                                                                                                                                        |
-| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`   |
 
+| Field                                                                                                 | Description                                                                                                                                                                                       |
+| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html)                        | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` |
+| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles)          | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                           |
+| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE`                                                                                                   |
+| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)   | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE`                                                                                                     |
+| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)     | The schema whose tables this replication is targeting. If no schema is specified, all tables with permission will be presented regardless of their schema.                                        |
+| Username                                                                                              | The username you created in Step 2 to allow Airbyte to access the database. Example: `AIRBYTE_USER`                                                                                               |
+| Password                                                                                              | The password associated with the username.                                                                                                                                                        |
+| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional)           | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`   |
 
 ### OAuth 2.0
-Field | Description                                                                                                                                                                                       |
-|---|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` |
-| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                           |
+
+| Field                                                                                                 | Description                                                                                                                                                                                       |
+| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html)                        | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` |
+| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles)          | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE`                                                                                                           |
 | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE`                                                                                                   |
-| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE`                                                                                                     |
-| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The schema whose tables this replication is targeting. If no schema is specified, all tables with permission will be presented regardless of their schema.                                        |
-| OAuth2 | The Login name and password to obtain auth token.                                                                                                                                                 |
-| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`   |
+| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)   | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE`                                                                                                     |
+| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl)     | The schema whose tables this replication is targeting. If no schema is specified, all tables with permission will be presented regardless of their schema.                                        |
+| OAuth2                                                                                                | The Login name and password to obtain auth token.                                                                                                                                                 |
+| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional)           | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3`   |
 
 ### Network policies
 
@@ -117,11 +119,13 @@ To determine whether a network policy is set on your account or for a specific u
 
 To read more please check official [Snowflake documentation](https://docs.snowflake.com/en/user-guide/network-policies.html#)
 
-
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                   |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- |
+| 0.2.0   | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2                                                                                                                      |
+| 0.1.36  | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                  |
+| 0.1.35  | 2023-06-14 | [27335](https://github.com/airbytehq/airbyte/pull/27335) | Remove noisy debug logs                                                                                                                   |
 | 0.1.34  | 2023-03-30 | [24693](https://github.com/airbytehq/airbyte/pull/24693) | Fix failure with TIMESTAMP_WITH_TIMEZONE column being used as cursor                                                                      |
 | 0.1.33  | 2023-03-29 | [24667](https://github.com/airbytehq/airbyte/pull/24667) | Fix bug which wont allow TIMESTAMP_WITH_TIMEZONE column to be used as a cursor                                                            |
 | 0.1.32  | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting                                                                                          |
diff --git a/docs/integrations/sources/square.md b/docs/integrations/sources/square.md
index 90a8046066f9..2dfa772284b3 100644
--- a/docs/integrations/sources/square.md
+++ b/docs/integrations/sources/square.md
@@ -6,8 +6,8 @@ This page contains the setup guide and reference information for the Square sour
 
 To set up the Square source connector with Airbyte, you'll need to create your Square Application and use Personal token or Oauth access token.
 
-
 ## Setup guide
+
 ### Step 1: Set up Square
 
 1. Create [Square Application](https://developer.squareup.com/apps)
@@ -21,25 +21,26 @@ To set up the Square source connector with Airbyte, you'll need to create your S
 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
 3. On the Set up the source page, enter the name for the Square connector and select **Square** from the Source type dropdown.
 4. Choose authentication method:
-   * Api-Key
-     * Fill in API key token with "Access token" from Square Application settings page (Credentials on the left)
-   * Oauth authentication
-     * Fill in Client ID and Client secret with data from Square Application settings page (Oauth on the left)
-     * Fill in refresh token with one obtained during the authentication process
+   - Api-Key
+     - Fill in API key token with "Access token" from Square Application settings page (Credentials on the left)
+   - Oauth authentication
+     - Fill in Client ID and Client secret with data from Square Application settings page (Oauth on the left)
+     - Fill in refresh token with one obtained during the authentication process
 5. Choose if your account is sandbox
 6. Choose start date
 7. Choose if you would like to include Deleted objects (for streams: Items, Categories, Discounts, Taxes)
 
 ### For Airbyte OSS:
+
 1. Navigate to the Airbyte Open Source dashboard.
-2. Set the name for your source. 
+2. Set the name for your source.
 3. On the Set up the source page, enter the name for the Square connector and select **Square** from the Source type dropdown.
 4. Choose authentication method:
-   * Api-Key
-     * Fill in API key token with "Access token" from Square Application settings page (Credentials on the left)
-   * Oauth authentication
-     * Fill in Client ID and Client secret with data from Square Application settings page (Oauth on the left)
-     * Fill in refresh token with one obtained during the authentication process
+   - Api-Key
+     - Fill in API key token with "Access token" from Square Application settings page (Credentials on the left)
+   - Oauth authentication
+     - Fill in Client ID and Client secret with data from Square Application settings page (Oauth on the left)
+     - Fill in refresh token with one obtained during the authentication process
 5. Choose if your account is sandbox
 6. Choose start date
 7. Choose if you would like to include Deleted objects (for streams: Items, Categories, Discounts, Taxes)
@@ -47,35 +48,35 @@ To set up the Square source connector with Airbyte, you'll need to create your S
 ## Supported sync modes
 
 The Square source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Items](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
-* [Categories](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
-* [Discounts](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
-* [Taxes](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
-* [ModifierLists](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
-* [Payments](https://developer.squareup.com/reference/square_2022-10-19/payments-api/list-payments) \(Incremental\)
-* [Refunds](https://developer.squareup.com/reference/square_2022-10-19/refunds-api/list-payment-refunds) \(Incremental\)
-* [Locations](https://developer.squareup.com/explorer/square/locations-api/list-locations) 
-* [Team Members](https://developer.squareup.com/reference/square_2022-10-19/team-api/search-team-members)
-* [List Team Member Wages](https://developer.squareup.com/explorer/square/labor-api/list-team-member-wages) 
-* [Customers](https://developer.squareup.com/explorer/square/customers-api/list-customers) 
-* [Shifts](https://developer.squareup.com/reference/square/labor-api/search-shifts) 
-* [Orders](https://developer.squareup.com/reference/square/orders-api/search-orders) 
+- [Items](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
+- [Categories](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
+- [Discounts](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
+- [Taxes](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
+- [ModifierLists](https://developer.squareup.com/explorer/square/catalog-api/search-catalog-objects) \(Incremental\)
+- [Payments](https://developer.squareup.com/reference/square_2022-10-19/payments-api/list-payments) \(Incremental\)
+- [Refunds](https://developer.squareup.com/reference/square_2022-10-19/refunds-api/list-payment-refunds) \(Incremental\)
+- [Locations](https://developer.squareup.com/explorer/square/locations-api/list-locations)
+- [Team Members](https://developer.squareup.com/reference/square_2022-10-19/team-api/search-team-members)
+- [List Team Member Wages](https://developer.squareup.com/explorer/square/labor-api/list-team-member-wages)
+- [Customers](https://developer.squareup.com/explorer/square/customers-api/list-customers)
+- [Shifts](https://developer.squareup.com/reference/square/labor-api/search-shifts)
+- [Orders](https://developer.squareup.com/reference/square/orders-api/search-orders)
 
 ## Connector-specific features & highlights
 
 Useful links:
 
-* [Square API Explorer](https://developer.squareup.com/explorer/square)
-* [Square API Docs](https://developer.squareup.com/reference/square)
-* [Square Developer Dashboard](https://developer.squareup.com/apps)
-
+- [Square API Explorer](https://developer.squareup.com/explorer/square)
+- [Square API Docs](https://developer.squareup.com/reference/square)
+- [Square Developer Dashboard](https://developer.squareup.com/apps)
 
 ## Performance considerations (if any)
 
@@ -85,26 +86,27 @@ Exponential [Backoff](https://developer.squareup.com/forums/t/current-square-api
 ## Data type map
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `integer`        | `integer`    |       |
 | `array`          | `array`      |       |
 | `object`         | `object`     |       |
 | `boolean`        | `boolean`    |       |
 
-
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------|
-| 1.1.0   | 2023-05-24 | [26485](https://github.com/airbytehq/airbyte/pull/26485) | Remove deprecated authSpecification in favour of advancedAuth              |
-| 1.0.1   | 2023-05-03 | [25784](https://github.com/airbytehq/airbyte/pull/25784) | Fix Authenticator                                                          |
-| 1.0.0   | 2023-05-03 | [25784](https://github.com/airbytehq/airbyte/pull/25784) | Fix Authenticator                                                          |
-| 0.2.2   | 2023-03-22 | [22867](https://github.com/airbytehq/airbyte/pull/22867) | Specified date formatting in specification                                 |
-| 0.2.1   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                                    |
-| 0.2.0   | 2022-11-14 | [19369](https://github.com/airbytehq/airbyte/pull/19369) | Migrate to low code (YAML); update API to version  2022-10-19; update docs |
-| 0.1.4   | 2021-12-02 | [6842](https://github.com/airbytehq/airbyte/pull/6842)   | Added oauth support                                                        |
-| 0.1.3   | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425)   | Update title, description fields in spec                                   |
-| 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                            |
-| 0.1.1   | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645)   | Update \_send\_request method due to Airbyte CDK changes                   |
-| 0.1.0   | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439)   | Initial release supporting the Square API                                  |
+| Version | Date       | Pull Request                                             | Subject                                                                   |
+| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------ |
+| 1.1.2   | 2023-07-10 | [28019](https://github.com/airbytehq/airbyte/pull/28019) | fix display order of spec fields                                          |
+| 1.1.1   | 2023-06-28 | [27762](https://github.com/airbytehq/airbyte/pull/27762) | Update following state breaking changes                                   |
+| 1.1.0   | 2023-05-24 | [26485](https://github.com/airbytehq/airbyte/pull/26485) | Remove deprecated authSpecification in favour of advancedAuth             |
+| 1.0.1   | 2023-05-03 | [25784](https://github.com/airbytehq/airbyte/pull/25784) | Fix Authenticator                                                         |
+| 1.0.0   | 2023-05-03 | [25784](https://github.com/airbytehq/airbyte/pull/25784) | Fix Authenticator                                                         |
+| 0.2.2   | 2023-03-22 | [22867](https://github.com/airbytehq/airbyte/pull/22867) | Specified date formatting in specification                                |
+| 0.2.1   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                                   |
+| 0.2.0   | 2022-11-14 | [19369](https://github.com/airbytehq/airbyte/pull/19369) | Migrate to low code (YAML); update API to version 2022-10-19; update docs |
+| 0.1.4   | 2021-12-02 | [6842](https://github.com/airbytehq/airbyte/pull/6842)   | Added oauth support                                                       |
+| 0.1.3   | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425)   | Update title, description fields in spec                                  |
+| 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                           |
+| 0.1.1   | 2021-07-09 | [4645](https://github.com/airbytehq/airbyte/pull/4645)   | Update \_send_request method due to Airbyte CDK changes                   |
+| 0.1.0   | 2021-06-30 | [4439](https://github.com/airbytehq/airbyte/pull/4439)   | Initial release supporting the Square API                                 |
diff --git a/docs/integrations/sources/strava.md b/docs/integrations/sources/strava.md
index 2d8669942f8d..c2d187919952 100644
--- a/docs/integrations/sources/strava.md
+++ b/docs/integrations/sources/strava.md
@@ -5,74 +5,79 @@ This page guides you through the process of setting up the Strava source connect
 ## Prerequisites
 
 Scopes:
-* `activity:read_all`
+
+- `activity:read_all`
 
 ## Setup guide
+
 ### Step 1: Set up Strava
 
 
+
 **For Airbyte Open Source:**
 
 Follow these steps to get the required credentials and inputs:
-* `client_id` and `client_secret`
-    * [Create a Strava account](https://developers.strava.com/docs/getting-started/#account)
-    * Continue to follow the instructions from the doc above to obtain `client_id` and `client_secret`
-* `refresh_token`
-    * Enter this URL into your browser (make sure to add your `client_id` from previous step:
-        * `https://www.strava.com/oauth/authorize?client_id=[REPLACE_WITH_YOUR_CLIENT_ID]&response_type=code&redirect_uri=https://localhost/exchange_token&approval_prompt=force&scope=activity:read_all`
-    * Authorize through the UI
-    * Browser will redirect you to an empty page with a URL similar to `https://localhost/exchange_token?state=&code=b55003496d87a9f0b694ca1680cd5690d27d9d28&scope=activity:read_all`
-    * Copy the authorization code above (in this example it would be `b55003496d87a9f0b694ca1680cd5690d27d9d28`)
-    * Make a cURL request to exchange the authorization code and scope for a refresh token:
-    * ```
-      curl -X POST https://www.strava.com/oauth/token \
-      -F client_id=YOUR_CLIENT_ID \
-      -F client_secret=YOUR_CLIENT_SECRET \
-      -F code=AUTHORIZATION_CODE \
-      -F grant_type=authorization_code
-      ```
-    * The resulting json will contain the `refresh_token`
-    * Example Result:
-    * ```
-        {
-            "token_type": "Bearer",
-            "expires_at": 1562908002,
-            "expires_in": 21600,
-            "refresh_token": "REFRESHTOKEN",
-            "access_token": "ACCESSTOKEN",
-            "athlete": {
-                "id": 123456,
-                "username": "MeowTheCat",
-                "resource_state": 2,
-                "firstname": "Meow",
-                "lastname": "TheCat",
-                "city": "",
-                "state": "",
-                "country": null,
-                ...
-            }
-        }
-      ```
-    * Refer to Strava's [Getting Started - Oauth](https://developers.strava.com/docs/getting-started/#oauth) or [Authentication](https://developers.strava.com/docs/authentication/) documents for more information
-* `athlete_id`
-    * Go to your athlete page by clicking your name on the [Strava dashboard](https://www.strava.com/dashboard) or click on "My Profile" on the drop down after hovering on your top bar icon
-    * The number at the end of the url will be your `athlete_id`. For example `17831421` would be the `athlete_id` for https://www.strava.com/athletes/17831421
+
+- `client_id` and `client_secret`
+  - [Create a Strava account](https://developers.strava.com/docs/getting-started/#account)
+  - Continue to follow the instructions from the doc above to obtain `client_id` and `client_secret`
+- `refresh_token`
+  - Enter this URL into your browser (make sure to add your `client_id` from previous step:
+    - `https://www.strava.com/oauth/authorize?client_id=[REPLACE_WITH_YOUR_CLIENT_ID]&response_type=code&redirect_uri=https://localhost/exchange_token&approval_prompt=force&scope=activity:read_all`
+  - Authorize through the UI
+  - Browser will redirect you to an empty page with a URL similar to `https://localhost/exchange_token?state=&code=b55003496d87a9f0b694ca1680cd5690d27d9d28&scope=activity:read_all`
+  - Copy the authorization code above (in this example it would be `b55003496d87a9f0b694ca1680cd5690d27d9d28`)
+  - Make a cURL request to exchange the authorization code and scope for a refresh token:
+  - ```
+    curl -X POST https://www.strava.com/oauth/token \
+    -F client_id=YOUR_CLIENT_ID \
+    -F client_secret=YOUR_CLIENT_SECRET \
+    -F code=AUTHORIZATION_CODE \
+    -F grant_type=authorization_code
+    ```
+  - The resulting json will contain the `refresh_token`
+  - Example Result:
+  - ```
+      {
+          "token_type": "Bearer",
+          "expires_at": 1562908002,
+          "expires_in": 21600,
+          "refresh_token": "REFRESHTOKEN",
+          "access_token": "ACCESSTOKEN",
+          "athlete": {
+              "id": 123456,
+              "username": "MeowTheCat",
+              "resource_state": 2,
+              "firstname": "Meow",
+              "lastname": "TheCat",
+              "city": "",
+              "state": "",
+              "country": null,
+              ...
+          }
+      }
+    ```
+  - Refer to Strava's [Getting Started - Oauth](https://developers.strava.com/docs/getting-started/#oauth) or [Authentication](https://developers.strava.com/docs/authentication/) documents for more information
+- `athlete_id`
+  - Go to your athlete page by clicking your name on the [Strava dashboard](https://www.strava.com/dashboard) or click on "My Profile" on the drop down after hovering on your top bar icon
+  - The number at the end of the url will be your `athlete_id`. For example `17831421` would be the `athlete_id` for https://www.strava.com/athletes/17831421
 
 
 
 
+
 **For Airbyte Cloud:**
 
-* `athlete_id`
-    * Go to your athlete page by clicking your name on the [Strava dashboard](https://www.strava.com/dashboard) or click on "My Profile" on the drop down after hovering on your top bar icon
-    * The number at the end of the url will be your `athlete_id`. For example `17831421` would be the `athlete_id` for https://www.strava.com/athletes/17831421
+- `athlete_id`
+  - Go to your athlete page by clicking your name on the [Strava dashboard](https://www.strava.com/dashboard) or click on "My Profile" on the drop down after hovering on your top bar icon
+  - The number at the end of the url will be your `athlete_id`. For example `17831421` would be the `athlete_id` for https://www.strava.com/athletes/17831421
 
 
 
-
 ### Step 2: Set up the source connector in Airbyte
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -85,12 +90,13 @@ Follow these steps to get the required credentials and inputs:
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Go to local Airbyte page.
 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
 3. On the source setup page, select **Strava** from the Source type dropdown and enter a name for this connector.
-4. Add **Client ID**,  **Client Secret** and **Refresh Token**
+4. Add **Client ID**, **Client Secret** and **Refresh Token**
 5. Set required **Athlete ID** and **Start Date**
 6. Click `Set up source`.
 
@@ -102,12 +108,12 @@ The Strava source connector supports the following [sync modes](https://docs.air
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported streams
 
-* [Athlete Stats](https://developers.strava.com/docs/reference/#api-Athletes-getStats)
-* [Activities](https://developers.strava.com/docs/reference/#api-Activities-getLoggedInAthleteActivities) \(Incremental\)
+- [Athlete Stats](https://developers.strava.com/docs/reference/#api-Athletes-getStats)
+- [Activities](https://developers.strava.com/docs/reference/#api-Activities-getLoggedInAthleteActivities) \(Incremental\)
 
 ## Performance considerations
 
@@ -117,10 +123,9 @@ More information about Strava rate limits and adjustments to those limits can be
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                    |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- |
 | 0.1.4   | 2023-03-23 | [24368](https://github.com/airbytehq/airbyte/pull/24368) | Add date-time format for input                             |
 | 0.1.3   | 2023-03-15 | [24101](https://github.com/airbytehq/airbyte/pull/24101) | certified to beta, fixed spec, fixed SAT, added unit tests |
 | 0.1.2   | 2021-12-15 | [8799](https://github.com/airbytehq/airbyte/pull/8799)   | Implement OAuth 2.0 support                                |
 | 0.1.1   | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425)   | Update title, description fields in spec                   |
 | 0.1.0   | 2021-10-18 | [7151](https://github.com/airbytehq/airbyte/pull/7151)   | Initial release supporting Strava API                      |
-
diff --git a/docs/integrations/sources/stripe.inapp.md b/docs/integrations/sources/stripe.inapp.md
new file mode 100644
index 000000000000..5988ba81922d
--- /dev/null
+++ b/docs/integrations/sources/stripe.inapp.md
@@ -0,0 +1,35 @@
+## Prerequisites
+
+- Access to the Stripe account containing the data to replicate
+
+## Setup Guide
+
+:::note
+To authenticate the Stripe connector, you need an API key with **Read** privileges for the data to replicate. For steps on obtaining and setting permissions for this key, refer to our [full Stripe documentation](https://docs.airbyte.com/integrations/sources/stripe#setup-guide).
+:::
+
+1. For **Source name**, enter a name to help you identify this source.
+2. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account).
+3. For **Secret Key**, enter your Stripe API key, which can be found at your Stripe [API keys page](https://dashboard.stripe.com/apikeys).
+4. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated.
+5. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments (such as tips, refunds, chargebacks, etc).
+
+    - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced.
+    - Setting the **Lookback Window** to 1 means Airbyte will re-export and capture any data changes within the last day.
+    - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week.
+
+6. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process.
+
+    For example, if you are replicating three years worth of data:
+
+    - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted.
+    - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs.
+
+    If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days.
+7. Click **Set up source** and wait for the tests to complete.
+
+### Stripe API limitations
+
+- When syncing `events` data from Stripe, data is only [returned for the last 30 days](https://stripe.com/docs/api/events). Using the Full Refresh (Overwrite) sync from Airbyte will delete the events data older than 30 days from your target destination. Use an Append sync mode to ensure historical data is retained.
+
+For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Stripe](https://docs.airbyte.com/integrations/sources/stripe).
diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md
index 13fcbf2e3007..9a69d906d11a 100644
--- a/docs/integrations/sources/stripe.md
+++ b/docs/integrations/sources/stripe.md
@@ -1,33 +1,50 @@
-:::warning
-Stripe API Restriction: Access to the events endpoint is [guaranteed only for the last 30 days](https://stripe.com/docs/api/events). Using the full-refresh-overwrite sync from Airbyte will delete the events data older than 30 days from your target destination.
-:::
-
 # Stripe
 
-This page guides you through the process of setting up the Stripe source connector.
+This page contains the setup guide and reference information for the Stripe source connector.
 
 ## Prerequisites
 
-- Your [Stripe `Account ID`](https://dashboard.stripe.com/settings/account)
-- Your [Stripe `Secret Key`](https://dashboard.stripe.com/apikeys)
+- Access to the Stripe account containing the data you wish to replicate
+
+## Setup Guide
+
+To authenticate the Stripe connector, you need to use a Stripe API key. Although you may use an existing key, we recommend that you create a new restricted key specifically for Airbyte and grant it **Read** privileges only. We also recommend granting **Read** privileges to all available permissions, and configuring the specific data you would like to replicate within Airbyte.
+
+### Create a Stripe Secret Key
+
+1. Log in to your [Stripe account](https://dashboard.stripe.com/login).
+2. In the top navigation bar, click **Developers**.
+3. In the top-left corner, click **API keys**.
+4. Click **+ Create restricted key**.
+5. Choose a **Key name**, and select **Read** for all available permissions.
+6. Click **Create key**. You may be prompted to enter a confirmation code sent to your email address.
+
+For more information on Stripe API Keys, see the [Stripe documentation](https://stripe.com/docs/keys).
 
-## Set up the Stripe source connector
+### Set up the Stripe source connector in Airbyte
 
-1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Stripe** from the Source type dropdown.
-4. Enter a name for your source.
-5. For **Account ID**, enter your [Stripe `Account ID`](https://dashboard.stripe.com/settings/account).
-6. For **Secret Key**, enter your [Stripe `Secret Key`](https://dashboard.stripe.com/apikeys)
+1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Stripe** from the list of available sources.
+4. For **Source name**, enter a name to help you identify this source.
+5. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account).
+6. For **Secret Key**, enter the restricted key you created for the connection.
+7. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated.
+8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments (such as tips, refunds, chargebacks, etc).
 
-   We recommend creating a secret key specifically for Airbyte to control which resources Airbyte can access. For ease of use, we recommend granting read permission to all resources and configuring which resource to replicate in the Airbyte UI. You can also use the API keys for the [test mode](https://stripe.com/docs/keys#obtain-api-keys) to try out the Stripe integration with Airbyte.
+    - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced.
+    - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours.
+    - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week.
 
-7. For **Replication start date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated.
-8. For **Lookback Window in days (Optional)**, select the number of days the value in days prior to the start date that you to sync your data with. If your data is updated after setting up this connector, you can use the this option to reload data from the past N days. Example: If the Replication start date is set to `2021-01-01T00:00:00Z`, then:
-   - If you leave the Lookback Window in days parameter to its the default value of 0, Airbyte will sync data from the Replication start date `2021-01-01T00:00:00Z`
-   - If the Lookback Window in days value is set to 1, Airbyte will consider the Replication start date to be `2020-12-31T00:00:00Z`
-   - If the Lookback Window in days value is set to 7, Airbyte will sync data from `2020-12-25T00:00:00Z`
-9. Click **Set up source**.
+9. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process.
+
+    For example, if you are replicating three years worth of data:
+
+    - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted.
+    - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs.
+
+    If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days.
+10. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
@@ -40,45 +57,68 @@ The Stripe source connector supports the following [sync modes](https://docs.air
 Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the `created` field to query for new data in your Stripe account.
 :::
 
-## Supported Streams
+## Supported streams
 
 The Stripe source connector supports the following streams:
 
+- [Accounts](https://stripe.com/docs/api/accounts/list) \(Incremental\)
 - [Application Fees](https://stripe.com/docs/api/application_fees) \(Incremental\)
 - [Application Fee Refunds](https://stripe.com/docs/api/fee_refunds/list)
+- [Authorizations](https://stripe.com/docs/api/issuing/authorizations/list) \(Incremental\)
 - [Balance Transactions](https://stripe.com/docs/api/balance_transactions/list) \(Incremental\)
 - [Bank accounts](https://stripe.com/docs/api/customer_bank_accounts/list)
+- [Cardholders](https://stripe.com/docs/api/issuing/cardholders/list) \(Incremental\)
+- [Cards](https://stripe.com/docs/api/issuing/cards/list) \(Incremental\)
 - [Charges](https://stripe.com/docs/api/charges/list) \(Incremental\)
-  - The `amount` column defaults to the smallest currency unit. (See [charge object](https://stripe.com/docs/api/charges/object) for more details)
+  :::note
+  The `amount` column defaults to the smallest currency unit. Check [the Stripe docs](https://stripe.com/docs/api/charges/object) for more details.
+  :::
 - [Checkout Sessions](https://stripe.com/docs/api/checkout/sessions/list)
 - [Checkout Sessions Line Items](https://stripe.com/docs/api/checkout/sessions/line_items)
 - [Coupons](https://stripe.com/docs/api/coupons/list) \(Incremental\)
+- [Credit Notes](https://stripe.com/docs/api/credit_notes/list) \(Full Refresh\)
 - [Customer Balance Transactions](https://stripe.com/docs/api/customer_balance_transactions/list)
 - [Customers](https://stripe.com/docs/api/customers/list) \(Incremental\)
-  - This endpoint does not include deleted customers
+  :::note
+  This endpoint does _not_ include deleted customers
+  :::
 - [Disputes](https://stripe.com/docs/api/disputes/list) \(Incremental\)
 - [Early Fraud Warnings](https://stripe.com/docs/api/radar/early_fraud_warnings/list) \(Incremental\)
 - [Events](https://stripe.com/docs/api/events/list) \(Incremental\)
-  - The Stripe API does not guarantee access to events older than 30 days, so this stream will only pull events created from the 30 days prior to the initial sync and not from the Replication start date.
+- [File Links](https://stripe.com/docs/api/file_links/list) \(Incremental\)
+- [Files](https://stripe.com/docs/api/files/list) \(Incremental\)
 - [Invoice Items](https://stripe.com/docs/api/invoiceitems/list) \(Incremental\)
 - [Invoice Line Items](https://stripe.com/docs/api/invoices/invoice_lines)
 - [Invoices](https://stripe.com/docs/api/invoices/list) \(Incremental\)
-- [PaymentIntents](https://stripe.com/docs/api/payment_intents/list) \(Incremental\)
+- [Payment Intents](https://stripe.com/docs/api/payment_intents/list) \(Incremental\)
+- [Payment Methods](https://stripe.com/docs/api/payment_methods/list)
 - [Payouts](https://stripe.com/docs/api/payouts/list) \(Incremental\)
 - [Promotion Code](https://stripe.com/docs/api/promotion_codes/list) \(Incremental\)
+- [Persons](https://stripe.com/docs/api/persons/list) \(Incremental\)
 - [Plans](https://stripe.com/docs/api/plans/list) \(Incremental\)
+- [Prices](https://stripe.com/docs/api/prices/list) \(Incremental\)
 - [Products](https://stripe.com/docs/api/products/list) \(Incremental\)
 - [Refunds](https://stripe.com/docs/api/refunds/list) \(Incremental\)
-- [SetupIntents](https://stripe.com/docs/api/setup_intents/list) \(Incremental\)
+- [Reviews](https://stripe.com/docs/api/radar/reviews/list) \(Incremental\)
+- [Setup Attempts](https://stripe.com/docs/api/setup_attempts/list) \(Incremental\)
+- [Setup Intents](https://stripe.com/docs/api/setup_intents/list) \(Incremental\)
+- [Shipping Rates](https://stripe.com/docs/api/shipping_rates/list) \(Incremental\)
 - [Subscription Items](https://stripe.com/docs/api/subscription_items/list)
 - [Subscription Schedule](https://stripe.com/docs/api/subscription_schedules) \(Incremental\)
 - [Subscriptions](https://stripe.com/docs/api/subscriptions/list) \(Incremental\)
+- [Top Ups](https://stripe.com/docs/api/topups/list) \(Incremental\)
+- [Transactions](https://stripe.com/docs/api/transfers/list) \(Incremental\)
 - [Transfers](https://stripe.com/docs/api/transfers/list) \(Incremental\)
-- [Accounts](https://stripe.com/docs/api/accounts/list) \(Incremental\)
+- [Transfer Reversals](https://stripe.com/docs/api/transfer_reversals/list)
+- [Usage Records](https://stripe.com/docs/api/usage_records/subscription_item_summary_list)
+
+:::warning
+**Stripe API Restriction on Events Data**: Access to the events endpoint is [guaranteed only for the last 30 days](https://stripe.com/docs/api/events) by Stripe. If you use the Full Refresh Overwrite sync, be aware that any events data older than 30 days will be **deleted** from your target destination and replaced with the data from the last 30 days only. Use an Append sync mode to ensure historical data is retained.
+:::
 
 ### Data type mapping
 
-The [Stripe API](https://stripe.com/docs/api) uses the same [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector.
+The [Stripe API](https://stripe.com/docs/api) uses the same [JSON Schema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector.
 
 ### Performance considerations
 
@@ -87,9 +127,24 @@ The Stripe connector should not run into Stripe API limitations under normal usa
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                              |
-| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
-| 3.6.0   | 2023-05-24 | [23963](https://github.com/airbytehq/airbyte/pull/)      | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees`                                                                                    |
-| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.17.4  | 2023-08-15 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Revert 3.17.3                                                                                                                                        |
+| 3.17.3  | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property                                                                                                               |
+| 3.17.2  | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling                                                                                                 |
+| 3.17.1  | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema                                                                                                                                |
+| 3.17.0  | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream                                                                                                                                  |
+| 3.16.0  | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas                                                                                                                     |
+| 3.15.0  | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams                                                                                                                             |
+| 3.14.0  | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream                                                                                                                           |
+| 3.13.0  | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version                                                                                                                               |
+| 3.12.0  | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons`                                                                                                                             |
+| 3.11.0  | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream                                                                                                                          |
+| 3.10.0  | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream                                                                                                                             |
+| 3.9.1   | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting                                                                                                                                       |
+| 3.9.0   | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions                                                                     |
+| 3.8.0   | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream                                                                                      |
+| 3.7.0   | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews                                                                        |
+| 3.6.0   | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees`                                                                                    |
 | 3.5.0   | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings`                                                                                                                    |
 | 3.4.3   | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types                                                                                                                     |
 | 3.4.2   | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types                                                                             |
diff --git a/docs/integrations/sources/surveycto.md b/docs/integrations/sources/surveycto.md
index b04fafb0597b..70a373ad8617 100644
--- a/docs/integrations/sources/surveycto.md
+++ b/docs/integrations/sources/surveycto.md
@@ -11,20 +11,22 @@ This page guides you through the process of setting up the SurveyCTO source conn
 - Start Date `Start Date default`
 
 ## How to setup a SurveyCTO Account
+
 - create the account
 - create your form
 - publish your form
 - give your user an API consumer permission to the existing role or create a user with that role and permission.
 
 ## Set up the SurveyCTO source connection
+
 1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
 2. Click **Sources** and then click **+ New source**.
 3. On the Set up the source page, select **Survey CTO** from the Source type dropdown.
 4. Enter a name for your source.
-5. Enter a Server name for your SurveyCTO account. 
+5. Enter a Server name for your SurveyCTO account.
 6. Enter a Username for SurveyCTO account.
 7. Enter a Password for SurveyCTO account.
-8. Form ID's (We can multiple forms id here to pull from) 
+8. Form ID's (We can multiple forms id here to pull from)
 9. Start Date (This can be pass to pull the data from particular date)
 10. Click **Set up source**.
 
@@ -32,10 +34,10 @@ This page guides you through the process of setting up the SurveyCTO source conn
 
 The SurveyCTO source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* (Recommended)[ Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- (Recommended)[ Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
@@ -46,5 +48,6 @@ The SurveyCTO source connector supports the following streams:
 ## Changelog
 
 | Version | Date | Pull Request | Subject |
-| 0.1.1   | 2023-04-25 | [24784](https://github.com/airbytehq/airbyte/pull/24784)   | Fix incremental sync       |
-| 0.1.0   | 2022-11-16 | [19371](https://github.com/airbytehq/airbyte/pull/19371)   | SurveyCTO Source Connector |
+| 0.1.2 | 2023-07-27 | [28512](https://github.com/airbytehq/airbyte/pull/28512) | Added Check Connection |
+| 0.1.1 | 2023-04-25 | [24784](https://github.com/airbytehq/airbyte/pull/24784) | Fix incremental sync |
+| 0.1.0 | 2022-11-16 | [19371](https://github.com/airbytehq/airbyte/pull/19371) | SurveyCTO Source Connector |
diff --git a/docs/integrations/sources/tempo.md b/docs/integrations/sources/tempo.md
index 7e9b57e6ea1e..68f40b69be28 100644
--- a/docs/integrations/sources/tempo.md
+++ b/docs/integrations/sources/tempo.md
@@ -4,16 +4,16 @@ This page contains the setup guide and reference information for the Tempo sourc
 
 ## Prerequisites
 
-* API Token
+- API Token
 
 ## Setup guide
+
 ### Step 1: Set up Tempo
 
 Source Tempo is designed to interact with the data your permissions give you access to. To do so, you will need to generate a Tempo OAuth 2.0 token for an individual user.
 
 Go to **Tempo > Settings**, scroll down to **Data Access** and select **API integration**.
 
-
 ## Step 2: Set up the Tempo connector in Airbyte
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -22,31 +22,30 @@ Go to **Tempo > Settings**, scroll down to **Data Access** and select **API i
 4. Enter your API token that you obtained from Tempo.
 5. Click **Set up source**.
 
-
 ## Supported sync modes
 
 The Tempo source connector supports the following [ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
 This connector outputs the following streams:
 
-* [Accounts](https://apidocs.tempo.io/#tag/Accounts)
-* [Customers](https://apidocs.tempo.io/#tag/Customers)
-* [Worklogs](https://apidocs.tempo.io/#tag/Worklogs)
-* [Workload Schemes](https://apidocs.tempo.io/#tag/Workload-Schemes)
+- [Accounts](https://apidocs.tempo.io/#tag/Accounts)
+- [Customers](https://apidocs.tempo.io/#tag/Customers)
+- [Worklogs](https://apidocs.tempo.io/#tag/Worklogs)
+- [Workload Schemes](https://apidocs.tempo.io/#tag/Workload-Schemes)
 
 If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose)
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                   |
-|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- |
 | 0.3.1   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                   |
 | 0.3.0   | 2022-11-02 | [18936](https://github.com/airbytehq/airbyte/pull/18936) | Migrate to low code + certify to Beta + migrate to API v4 |
 | 0.2.6   | 2022-09-08 | [16361](https://github.com/airbytehq/airbyte/pull/16361) | Avoid infinite loop for non-paginated APIs                |
diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md
index efc44043cd19..3007c8b0de44 100644
--- a/docs/integrations/sources/tidb.md
+++ b/docs/integrations/sources/tidb.md
@@ -127,16 +127,17 @@ Now that you have set up the TiDB source connector, check out the following TiDB
 ## Changelog
 
 | Version | Date | Pull Request | Subject |
-| :------ | :--- | :----------- | ------- |
-| 0.2.4 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting   |
-| 0.2.3 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455)   | For network isolation, source connector accepts a list of hosts it is allowed to connect to |
-| 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)   | Consolidate date/time values mapping for JDBC sources                          |
-|       | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
-| 0.2.1 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
-| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. |
-| 0.1.5 | 2022-07-25 | [14996](https://github.com/airbytehq/airbyte/pull/14996) | Removed additionalProperties:false from spec |
-| 0.1.4 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected |
-| 0.1.3 | 2022-07-04 | [14243](https://github.com/airbytehq/airbyte/pull/14243) | Update JDBC string builder |
-| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
+|:--------| :--- | :----------- | ------- |
+| 0.2.5   | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase                                                                                                                  |
+| 0.2.4   | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting   |
+| 0.2.3   | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455)   | For network isolation, source connector accepts a list of hosts it is allowed to connect to |
+| 0.2.2   | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346)   | Consolidate date/time values mapping for JDBC sources                          |
+|         | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode |
+| 0.2.1   | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently |
+| 0.2.0   | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. |
+| 0.1.5   | 2022-07-25 | [14996](https://github.com/airbytehq/airbyte/pull/14996) | Removed additionalProperties:false from spec |
+| 0.1.4   | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected |
+| 0.1.3   | 2022-07-04 | [14243](https://github.com/airbytehq/airbyte/pull/14243) | Update JDBC string builder |
+| 0.1.2   | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors |
 | 0.1.1   | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |
 | 0.1.0   | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release |
diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md
index d35ccdf16b62..f0c772467d16 100644
--- a/docs/integrations/sources/tiktok-marketing.md
+++ b/docs/integrations/sources/tiktok-marketing.md
@@ -5,21 +5,25 @@ This page guides you through the process of setting up the TikTok Marketing sour
 ## Prerequisites
 
 
+
 **For Airbyte Cloud:**
 
-* A Tiktok Ads Business account with permission to access data from accounts you want to sync
+- A Tiktok Ads Business account with permission to access data from accounts you want to sync
 
 
 
+
 **For Airbyte Open Source:**
 For the Production environment:
-* Access token
-* Secret
-* App ID
+
+- Access token
+- Secret
+- App ID
 
 To access the Sandbox environment:
-* Access token
-* Advertiser ID
+
+- Access token
+- Advertiser ID
 
 
 ## Setup guide
@@ -33,6 +37,7 @@ To access the Sandbox environment:
 ### Step 2: Set up the source connector in Airbyte
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -45,6 +50,7 @@ To access the Sandbox environment:
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Go to local Airbyte page.
@@ -58,7 +64,7 @@ To access the Sandbox environment:
 ## Supported streams and sync modes
 
 | Stream                                    | Environment  | Key                                        | Incremental |
-|:------------------------------------------|--------------|--------------------------------------------|:------------|
+| :---------------------------------------- | ------------ | ------------------------------------------ | :---------- |
 | Advertisers                               | Prod,Sandbox | advertiser_id                              | No          |
 | AdGroups                                  | Prod,Sandbox | adgroup_id                                 | Yes         |
 | Ads                                       | Prod,Sandbox | ad_id                                      | Yes         |
@@ -97,10 +103,13 @@ It is recommended to use higher values of attribution window (used in Incrementa
 :::
 
 ### Report Aggregation
+
 Reports synced by this connector can use either hourly, daily, or lifetime granularities for aggregating performance data. For example, if you select the daily-aggregation flavor of a report, the report will contain a row for each day for the duration of the report. Each row will indicate the number of impressions recorded on that day.
 
 ### Output Schemas
+
 **[Advertisers](https://ads.tiktok.com/marketing_api/docs?id=1708503202263042) Stream**
+
 ```
 {
   "contacter": "Ai***te",
@@ -135,6 +144,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **[AdGroups](https://ads.tiktok.com/marketing_api/docs?id=1708503489590273) Stream**
+
 ```
 {
   "placement_type": "PLACEMENT_TYPE_AUTOMATIC",
@@ -219,6 +229,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **[Ads](https://ads.tiktok.com/marketing_api/docs?id=1708572923161602) Stream**
+
 ```
 {
   "vast_moat": false,
@@ -266,6 +277,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **[Campaigns](https://ads.tiktok.com/marketing_api/docs?id=1708582970809346) Stream**
+
 ```
 {
   "create_time": "2021-10-19 18:18:08",
@@ -288,6 +300,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **AdsReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)**
+
 ```
 {
   "dimensions": {
@@ -336,6 +349,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **AdvertisersReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)**
+
 ```
 {
   "metrics": {
@@ -360,6 +374,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **AdGroupsReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)**
+
 ```
 {
   "metrics": {
@@ -405,6 +420,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **CampaignsReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)**
+
 ```
 {
   "metrics": {
@@ -428,6 +444,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **AdsAudienceReportsDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)**
+
 ```
 {
   {
@@ -472,6 +489,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **AdvertisersAudienceReportsDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)**
+
 ```
 {
   "dimensions": {
@@ -492,6 +510,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **AdGroupAudienceReportsDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)**
+
 ```
 {
   "dimensions": {
@@ -533,6 +552,7 @@ Reports synced by this connector can use either hourly, daily, or lifetime granu
 ```
 
 **CampaignsAudienceReportsByCountryDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)**
+
 ```
 {
   "metrics": {
@@ -561,10 +581,14 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark
 
 | Version | Date       | Pull Request                                             | Subject                                                                                       |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------|
+| 3.4.1   | 2023-08-04 | [29083](https://github.com/airbytehq/airbyte/pull/29083) | Added new `is_smart_performance_campaign` property to `ad groups` stream schema |
+| 3.4.0   | 2023-07-13 | [27910](https://github.com/airbytehq/airbyte/pull/27910) | Added `include_deleted` config param - include deleted `ad_groups`, `ad`, `campaigns` to reports |
+| 3.3.1   | 2023-07-06 | [25423](https://github.com/airbytehq/airbyte/pull/25423) | add new fields to ad reports streams                                                          |
+| 3.3.0   | 2023-07-05 | [27988](https://github.com/airbytehq/airbyte/pull/27988) | Add `category_exclusion_ids` field to `ad_groups` schema. |
 | 3.2.1   | 2023-05-26 | [26569](https://github.com/airbytehq/airbyte/pull/26569) | Fixed syncs with `advertiser_id` provided in input configuration                              |
 | 3.2.0   | 2023-05-25 | [26565](https://github.com/airbytehq/airbyte/pull/26565) | Change default value for `attribution window` to 3 days; add min/max validation               |
-| 3.1.0   | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Updated the  `Ads` stream schema                                                              |
-| 3.0.1   | 2023-04-07 | [24712](https://github.com/airbytehq/airbyte/pull/24712) | Added `attribution window` for *-reports streams                                              |
+| 3.1.0   | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Updated the `Ads` stream schema                                                               |
+| 3.0.1   | 2023-04-07 | [24712](https://github.com/airbytehq/airbyte/pull/24712) | Added `attribution window` for \*-reports streams                                             |
 | 3.0.0   | 2023-03-29 | [24630](https://github.com/airbytehq/airbyte/pull/24630) | Migrate to v1.3 API                                                                           |
 | 2.0.6   | 2023-03-30 | [22134](https://github.com/airbytehq/airbyte/pull/22134) | Add `country_code` and `platform` audience reports.                                           |
 | 2.0.5   | 2023-03-29 | [22863](https://github.com/airbytehq/airbyte/pull/22863) | Specified date formatting in specification                                                    |
@@ -592,4 +616,4 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark
 | 0.1.3   | 2021-12-10 | [8425](https://github.com/airbytehq/airbyte/pull/8425)   | Update title, description fields in spec                                                      |
 | 0.1.2   | 2021-12-02 | [8292](https://github.com/airbytehq/airbyte/pull/8292)   | Support reports                                                                               |
 | 0.1.1   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                                               |
-| 0.1.0   | 2021-09-18 | [5887](https://github.com/airbytehq/airbyte/pull/5887)   | Release TikTok Marketing CDK Connector                                                        |
\ No newline at end of file
+| 0.1.0   | 2021-09-18 | [5887](https://github.com/airbytehq/airbyte/pull/5887)   | Release TikTok Marketing CDK Connector                                                        |
diff --git a/docs/integrations/sources/trello.md b/docs/integrations/sources/trello.md
index 3f0307f87841..de088670a6e2 100644
--- a/docs/integrations/sources/trello.md
+++ b/docs/integrations/sources/trello.md
@@ -74,14 +74,16 @@ The Trello connector should not run into Trello API limitations under normal usa
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-| :--- | :--- | :--- | :--- |
-| 0.3.2 | 2023-05-05 | [25870](https://github.com/airbytehq/airbyte/pull/25870) | Added `CDK typeTransformer` to guarantee JSON schema types |
-| 0.3.1 | 2023-03-21 | [24266](https://github.com/airbytehq/airbyte/pull/24266) | Get board ids also from organizations |
-| 0.3.0 | 2023-03-17 | [24141](https://github.com/airbytehq/airbyte/pull/24141) | Certify to Beta |
-| 0.2.0 | 2023-03-15 | [24045](https://github.com/airbytehq/airbyte/pull/24045) | Fix schema for boards and cards streams |
-| 0.1.6 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Updated fields in source-connector specifications |
-| 0.1.3 | 2021-11-25 | [8183](https://github.com/airbytehq/airbyte/pull/8183) | Enable specifying board ids in configuration |
-| 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies |
-| 0.1.1 | 2021-10-12 | [6968](https://github.com/airbytehq/airbyte/pull/6968) | Add oAuth flow support |
-| 0.1.0 | 2021-08-18 | [5501](https://github.com/airbytehq/airbyte/pull/5501) | Release Trello CDK Connector |
+| Version | Date       | Pull Request                                             | Subject                                                    |
+|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------|
+| 0.3.4   | 2023-07-31 | [28734](https://github.com/airbytehq/airbyte/pull/28734) | Updated `expected records` for CAT test and fixed `advancedAuth` broken references                                |
+| 0.3.3   | 2023-06-19 | [27470](https://github.com/airbytehq/airbyte/pull/27470) | Update Organizations schema                                |
+| 0.3.2   | 2023-05-05 | [25870](https://github.com/airbytehq/airbyte/pull/25870) | Added `CDK typeTransformer` to guarantee JSON schema types |
+| 0.3.1   | 2023-03-21 | [24266](https://github.com/airbytehq/airbyte/pull/24266) | Get board ids also from organizations                      |
+| 0.3.0   | 2023-03-17 | [24141](https://github.com/airbytehq/airbyte/pull/24141) | Certify to Beta                                            |
+| 0.2.0   | 2023-03-15 | [24045](https://github.com/airbytehq/airbyte/pull/24045) | Fix schema for boards and cards streams                    |
+| 0.1.6   | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628)   | Updated fields in source-connector specifications          |
+| 0.1.3   | 2021-11-25 | [8183](https://github.com/airbytehq/airbyte/pull/8183)   | Enable specifying board ids in configuration               |
+| 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                            |
+| 0.1.1   | 2021-10-12 | [6968](https://github.com/airbytehq/airbyte/pull/6968)   | Add oAuth flow support                                     |
+| 0.1.0   | 2021-08-18 | [5501](https://github.com/airbytehq/airbyte/pull/5501)   | Release Trello CDK Connector                               |
diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md
index d8117557eb5d..d8b8dae0faf6 100644
--- a/docs/integrations/sources/twilio.md
+++ b/docs/integrations/sources/twilio.md
@@ -76,11 +76,15 @@ The Twilio source connector supports the following [sync modes](https://docs.air
 * [Queues](https://www.twilio.com/docs/voice/api/queue-resource#read-multiple-queue-resources)
 * [Recordings](https://www.twilio.com/docs/voice/api/recording#read-multiple-recording-resources) \(Incremental\)
 * [Services](https://www.twilio.com/docs/chat/rest/service-resource#read-multiple-service-resources)
+* [Step](https://www.twilio.com/docs/studio/rest-api/v2/step#read-a-list-of-step-resources)
 * [Roles](https://www.twilio.com/docs/chat/rest/role-resource#read-multiple-role-resources)
 * [Transcriptions](https://www.twilio.com/docs/voice/api/recording-transcription?code-sample=code-read-list-all-transcriptions&code-language=curl&code-sdk-version=json#read-multiple-transcription-resources)
 * [Trunks](https://www.twilio.com/docs/sip-trunking/api/trunk-resource#trunk-properties)
 * [Usage Records](https://www.twilio.com/docs/usage/api/usage-record#read-multiple-usagerecord-resources) \(Incremental\)
 * [Usage Triggers](https://www.twilio.com/docs/usage/api/usage-trigger#read-multiple-usagetrigger-resources)
+* [Users](https://www.twilio.com/docs/conversations/api/user-resource)
+* [UserConversations](https://www.twilio.com/docs/conversations/api/user-conversation-resource#list-all-of-a-users-conversations)
+* [VerifyServices](https://www.twilio.com/docs/verify/api/service#maincontent)
 
 ## Performance considerations
 
@@ -91,8 +95,12 @@ For more information, see [the Twilio docs for rate limitations](https://support
 
 | Version | Date       | Pull Request                                              | Subject                                                                                                 |
 |:--------|:-----------|:----------------------------------------------------------|:--------------------------------------------------------------------------------------------------------|
-| 0.7.0   | 2023-05-03 | [25781](https://github.com/airbytehq/airbyte/pull/25781)      | Add new stream `Trunks`                                                              |
-| 0.6.0   | 2023-05-03 | [](https://github.com/airbytehq/airbyte/pull/)      | Add new stream `Roles` with parent `Services`                                                              |
+| 0.10.0  | 2023-07-28 | [27323](https://github.com/airbytehq/airbyte/pull/27323)  | Add new stream `Step`                                                                                  |
+| 0.9.0   | 2023-06-27 | [27221](https://github.com/airbytehq/airbyte/pull/27221)  | Add new stream `UserConversations` with parent `Users`                                                  |
+| 0.8.1   | 2023-07-12 | [28216](https://github.com/airbytehq/airbyte/pull/28216)  | Add property `channel_metadata` to `ConversationMessages` schema                                                                         |
+| 0.8.0   | 2023-06-11 | [27231](https://github.com/airbytehq/airbyte/pull/27231)  | Add new stream `VerifyServices`                                                                         |
+| 0.7.0   | 2023-05-03 | [25781](https://github.com/airbytehq/airbyte/pull/25781)  | Add new stream `Trunks`                                                                                 |
+| 0.6.0   | 2023-05-03 | [25783](https://github.com/airbytehq/airbyte/pull/25783)  | Add new stream `Roles` with parent `Services`                                                           |
 | 0.5.0   | 2023-03-21 | [23995](https://github.com/airbytehq/airbyte/pull/23995)  | Add new stream `Conversation Participants`                                                              |
 | 0.4.0   | 2023-03-18 | [23995](https://github.com/airbytehq/airbyte/pull/23995)  | Add new stream `Conversation Messages`                                                                  |
 | 0.3.0   | 2023-03-18 | [22874](https://github.com/airbytehq/airbyte/pull/22874)  | Add new stream `Executions` with parent `Flows`                                                         |
diff --git a/docs/integrations/sources/typeform.md b/docs/integrations/sources/typeform.md
index a893853df5b5..16c94f432aa0 100644
--- a/docs/integrations/sources/typeform.md
+++ b/docs/integrations/sources/typeform.md
@@ -4,23 +4,40 @@ This page guides you through the process of setting up the Typeform source conne
 
 ## Prerequisites
 
-* token - The Typeform API key token.
-* start\_date - Date to start fetching Responses stream data from.
-* form_ids (Optional) - List of Form Ids to sync. If not passed - sync all account`s forms.
+- [Typeform Account](https://www.typeform.com/)
+- Form IDs (Optional) - If you want to sync data for specific forms, you'll need to have the IDs of those forms. If you want to sync data for all forms in your account you don't need any IDs. Form IDs can be found in the URLs to the forms in Typeform Admin Panel (for example, for URL `https://admin.typeform.com/form/12345/` a `12345` part would your Form ID)
+
+**For Airbyte Cloud:**
+
+- OAuth
+
+
+
+**For Airbyte Open Source:**
+
+- Personal Access Token (see [personal access token](https://www.typeform.com/developers/get-started/personal-access-token/))
+
 
 ## Setup guide
 
-### Step 1: Set up Typeform
+### Step 1: Obtain an API token 
 
+
+**For Airbyte Open Source:**
 To get the API token for your application follow this [steps](https://developer.typeform.com/get-started/personal-access-token/)
-
 * Log in to your account at Typeform.
 * In the upper-right corner, in the drop-down menu next to your profile photo, click My Account.
 * In the left menu, click Personal tokens.
 * Click Generate a new token.
 * In the Token name field, type a name for the token to help you identify it.
-* Choose needed scopes \(API actions this token can perform - or permissions it has\). See here for more details on scopes.
+* Choose needed scopes \(API actions this token can perform - or permissions it has\). See [here](https://www.typeform.com/developers/get-started/scopes/) for more details on scopes.
 * Click Generate token.
+
+
+
+**For Airbyte Cloud:**
+This step is not needed in Airbyte Cloud. Skip to the next step. 
+
 
 ### Step 2: Set up the source connector in Airbyte
 
@@ -28,20 +45,23 @@ To get the API token for your application follow this [steps](https://developer.
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New Source**.
 3. On the source setup page, select **Typeform** from the Source type dropdown and enter a name for this connector.
-4. Fill-in 'API Token' and 'Start Date'
-5. click `Set up source`.
+4. Click `Authenticate your Typeform account` by selecting Oauth or Personal Access Token for Authentication.
+5. Log in and Authorize to the Typeform account.
+6. **Start date (Optional)** - Date to start fetching Responses stream data from. If start date is not set, Responses stream will fetch data from a year ago from today.
+7. **Form IDs (Optional)** - List of Form Ids to sync. If not passed - sync all account`s forms.
+8. Click **Set up source**.
 
 
 
 **For Airbyte Open Source:**
 
 1. Go to local Airbyte page.
-2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
-3. On the Set up the source page, enter the name for the connector and select **Tiktok Marketing** from the Source type dropdown.
-4. Fill-in 'API Token' and 'Start Date'
-5. click `Set up source`.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New Source**.
+3. On the Set up the source page, enter the name for the connector and select **Typeform** from the Source type dropdown.
+4. Fill-in **API Token** and **Start Date**
+5. click **Set up source**
 
 
 ## Supported streams and sync modes
@@ -70,6 +90,9 @@ API rate limits \(2 requests per second\): [https://developer.typeform.com/get-s
 
 | Version | Date       | Pull Request                                             | Subject                                                                 |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------|
+| 1.0.0  | 2023-06-26 | [27240](https://github.com/airbytehq/airbyte/pull/27240) | Add OAuth support                                                   |
+| 0.3.0  | 2023-06-23 | [27653](https://github.com/airbytehq/airbyte/pull/27653) | Add `form_id` to records of `responses` stream                                                   |
+| 0.2.0  | 2023-06-17 | [27455](https://github.com/airbytehq/airbyte/pull/27455) | Add missing schema fields in `forms`, `themes`, `images`, `workspaces`, and `responses` streams                                                   |
 | 0.1.12  | 2023-02-21 | [22824](https://github.com/airbytehq/airbyte/pull/22824) | Specified date formatting in specification                                                   |
 | 0.1.11  | 2023-02-20 | [23248](https://github.com/airbytehq/airbyte/pull/23248) | Store cursor value as a string                                          |
 | 0.1.10  | 2023-01-07 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Certification to Beta                                                   |
diff --git a/docs/integrations/sources/us-census.md b/docs/integrations/sources/us-census.md
index 6a3b5d5b1a53..2242f3fceda5 100644
--- a/docs/integrations/sources/us-census.md
+++ b/docs/integrations/sources/us-census.md
@@ -3,7 +3,7 @@
 ## Overview
 
 This connector syncs data from the [US Census API](https://www.census.gov/data/developers/guidance/api-user-guide.Example_API_Queries.html)
-
+
 ### Output schema
 
 This source always outputs a single stream, `us_census_stream`. The output of the stream depends on the configuration of the connector.
@@ -16,7 +16,7 @@ This source always outputs a single stream, `us_census_stream`. The output of th
 | Incremental Sync  | No         |
 | SSL connection    | Yes        |
 | Namespaces        | No         |
-
+
 ## Getting started
 
 ### Requirements
diff --git a/docs/integrations/sources/waiteraid.md b/docs/integrations/sources/waiteraid.md
index 278cf94fbd63..9e568f634068 100644
--- a/docs/integrations/sources/waiteraid.md
+++ b/docs/integrations/sources/waiteraid.md
@@ -17,7 +17,7 @@ You can find or create authentication tokens within [Waiteraid](https://app.wait
 4. Enter your `auth_token` - Waiteraid Authentication Token.
 5. Enter your `restaurant ID` - The Waiteraid ID of the Restaurant you wanto sync. 
 6. Click **Set up source**.
-
+
 ### For Airbyte OSS:
 
 1. Navigate to the Airbyte Open Source dashboard.
@@ -36,7 +36,7 @@ The Waiteraid source connector supports the following [sync modes](https://docs.
 | Incremental Sync  | No         |
 | SSL connection    | No         |
 | Namespaces        | No         |
-
+
 ## Supported Streams
 
 * [Bookings](https://app.waiteraid.com/api-docs/index.html#api_get_bookings)
diff --git a/docs/integrations/sources/webflow.md b/docs/integrations/sources/webflow.md
index cabeae06101b..7a47e7158af9 100644
--- a/docs/integrations/sources/webflow.md
+++ b/docs/integrations/sources/webflow.md
@@ -8,7 +8,7 @@ Webflow is a CMS system that is used for publishing websites and blogs. This con
 
 Webflow uses [Collections](https://developers.webflow.com/#collections) to store different kinds of information. A collection can be "Blog Posts", or "Blog Authors", etc. Collection names are not pre-defined, the number of collections is not known in advance, and the schema for each collection may be different.
 
-This connector dynamically figures our which collections are available, creates the schema for each collection based on data extracted from Webflow, and creates an [Airbyte Stream](https://docs.airbyte.com/connector-development/cdk-python/full-refresh-stream/) for each collection.
+This connector dynamically figures out which collections are available, creates the schema for each collection based on data extracted from Webflow, and creates an [Airbyte Stream](https://docs.airbyte.com/connector-development/cdk-python/full-refresh-stream/) for each collection.
 
 # Webflow credentials
 
@@ -29,7 +29,7 @@ Which should respond with something similar to:
 ```
 
 You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site.
-
+
 # Related tutorial
 
 If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api).
@@ -41,3 +41,5 @@ If you are interested in learning more about the Webflow API and implementation
 | 0.1.2   | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow add ids to streams    |
 | 0.1.1   | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL |
 | 0.1.0   | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release               |
+
+
\ No newline at end of file
diff --git a/docs/integrations/sources/woocommerce.md b/docs/integrations/sources/woocommerce.md
index b649fbda32b2..e97eb3a6749d 100644
--- a/docs/integrations/sources/woocommerce.md
+++ b/docs/integrations/sources/woocommerce.md
@@ -32,6 +32,7 @@ You will need to generate new API key with read permissions and use `Customer ke
 5. Fill in `Shop Name`. For `https://EXAMPLE.com`, the shop name is 'EXAMPLE.com'.
 6. Choose start date you want to start sync from.
 7. (Optional) Fill in Conversion Window.
+
 
 ### For Airbyte OSS:
 
@@ -52,7 +53,8 @@ following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-s
 - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
 - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
 - [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-- [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
+
 
 ## Supported Streams
 
@@ -97,11 +99,11 @@ Useful links:
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                                 |
-| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------- |
-| 0.2.3   | 2023-06-02 | [26955](https://github.com/airbytehq/airbyte/pull/26955) | Added `block_context` and `author` properties to the `Products` stream  |
-| 0.2.2   | 2023-03-03 | [23599](https://github.com/airbytehq/airbyte/pull/23599) | Fix pagination and removed lookback window                              |
-| 0.2.1   | 2023-02-10 | [22821](https://github.com/airbytehq/airbyte/pull/22821) | Specified date formatting in specification                              |
-| 0.2.0   | 2022-11-30 | [19903](https://github.com/airbytehq/airbyte/pull/19903) | Migrate to low-code; Certification to Beta                              |
-| 0.1.1   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                         |
-| 0.1.0   | 2021-09-09 | [5955](https://github.com/airbytehq/airbyte/pull/5955)   | Initial Release. Source WooCommerce                                     |
+| Version | Date       | Pull Request                                             | Subject                                                                |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------- |
+| 0.2.3   | 2023-06-02 | [26955](https://github.com/airbytehq/airbyte/pull/26955) | Added `block_context` and `author` properties to the `Products` stream |
+| 0.2.2   | 2023-03-03 | [23599](https://github.com/airbytehq/airbyte/pull/23599) | Fix pagination and removed lookback window                             |
+| 0.2.1   | 2023-02-10 | [22821](https://github.com/airbytehq/airbyte/pull/22821) | Specified date formatting in specification                             |
+| 0.2.0   | 2022-11-30 | [19903](https://github.com/airbytehq/airbyte/pull/19903) | Migrate to low-code; Certification to Beta                             |
+| 0.1.1   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499)   | Remove base-python dependencies                                        |
+| 0.1.0   | 2021-09-09 | [5955](https://github.com/airbytehq/airbyte/pull/5955)   | Initial Release. Source WooCommerce                                    |
diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md
index ac4a5919ae47..2be1f4413618 100644
--- a/docs/integrations/sources/xero.md
+++ b/docs/integrations/sources/xero.md
@@ -90,7 +90,9 @@ The connector is restricted by Xero [API rate limits](https://developer.xero.com
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                           |
-| :------ | :--------- | :------------------------------------------------------- | :-------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------|:----------------------------------|
+| 0.2.3   | 2023-06-19 | [27471](https://github.com/airbytehq/airbyte/pull/27471) | Update CDK to 0.40                |
+| 0.2.2   | 2023-06-06 | [27007](https://github.com/airbytehq/airbyte/pull/27007) | Update CDK                        |
 | 0.2.1   | 2023-03-20 | [24217](https://github.com/airbytehq/airbyte/pull/24217) | Certify to Beta                   |
 | 0.2.0   | 2023-03-14 | [24005](https://github.com/airbytehq/airbyte/pull/24005) | Enable in Cloud                   |
 | 0.1.0   | 2021-11-11 | [18666](https://github.com/airbytehq/airbyte/pull/18666) | 🎉 New Source - Xero [python cdk] |
diff --git a/docs/integrations/sources/yahoo-finance-price.md b/docs/integrations/sources/yahoo-finance-price.md
index 8162eb58e3ea..63abbb1b0fe3 100644
--- a/docs/integrations/sources/yahoo-finance-price.md
+++ b/docs/integrations/sources/yahoo-finance-price.md
@@ -6,4 +6,5 @@ The Airbyte Source for [Yahoo Finance Price](https://finance.yahoo.com/)
 
 | Version | Date       | Pull Request                                             | Subject                       |
 | :------ | :--------- | :------------------------------------------------------- | :---------------------------- |
+| 0.2.0   | 2023-08-22 | [29355](https://github.com/airbytehq/airbyte/pull/29355) | Migrate to no-code framework |
 | 0.1.3   | 2022-03-23 | [10563](https://github.com/airbytehq/airbyte/pull/10563) | 🎉 Source Yahoo Finance Price |
diff --git a/docs/integrations/sources/yandex-metrica.md b/docs/integrations/sources/yandex-metrica.md
index d988711a3ca6..e38e7c8404b0 100644
--- a/docs/integrations/sources/yandex-metrica.md
+++ b/docs/integrations/sources/yandex-metrica.md
@@ -34,7 +34,6 @@ This page contains the setup guide and reference information for the Yandex Metr
 7. Enter the Start Date in format `YYYY-MM-DD`.
 8. Enter the End Date in format `YYYY-MM-DD` (Optional).
 
-
 #### For Airbyte Open Source:
 
 1. Navigate to the Airbyte Open Source dashboard.
@@ -50,15 +49,15 @@ This page contains the setup guide and reference information for the Yandex Metr
 
 The Yandex Metrica source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Views](https://yandex.com/dev/metrika/doc/api2/logs/fields/hits.html) \(Incremental\).
-* [Sessions](https://yandex.com/dev/metrika/doc/api2/logs/fields/visits.html) \(Incremental\).
+- [Views](https://yandex.com/dev/metrika/doc/api2/logs/fields/hits.html) \(Incremental\).
+- [Sessions](https://yandex.com/dev/metrika/doc/api2/logs/fields/visits.html) \(Incremental\).
 
 ## Performance considerations
 
@@ -79,7 +78,7 @@ Because of the way API works some syncs may take a long time to finish. Timeout
 ## Data type mapping
 
 | Integration Type | Airbyte Type | Notes |
-|:-----------------|:-------------|:------|
+| :--------------- | :----------- | :---- |
 | `string`         | `string`     |       |
 | `integer`        | `integer`    |       |
 | `number`         | `number`     |       |
@@ -89,6 +88,6 @@ Because of the way API works some syncs may take a long time to finish. Timeout
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                 |
-|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------|
+| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------- |
 | 1.0.0   | 2023-03-20 | [24188](https://github.com/airbytehq/airbyte/pull/24188) | Migrate to Beta; Change state structure |
 | 0.1.0   | 2022-09-09 | [15061](https://github.com/airbytehq/airbyte/pull/15061) | 🎉 New Source: Yandex metrica           |
diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md
index 098b276c8491..eab821953fd8 100644
--- a/docs/integrations/sources/zendesk-chat.md
+++ b/docs/integrations/sources/zendesk-chat.md
@@ -12,6 +12,7 @@ This page contains the setup guide and reference information for the Zendesk Cha
 ## Setup guide
 
 
+
 **For Airbyte Cloud:**
 
 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account.
@@ -25,6 +26,7 @@ This page contains the setup guide and reference information for the Zendesk Cha
 
 
 
+
 **For Airbyte Open Source:**
 
 1. Navigate to the Airbyte Open Source dashboard.
@@ -41,25 +43,25 @@ This page contains the setup guide and reference information for the Zendesk Cha
 
 The Zendesk Chat source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes):
 
-* [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
-* [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
-* [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
-* [Incremental - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history)
+- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/)
+- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append)
+- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append)
+- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped)
 
 ## Supported Streams
 
-* [Accounts](https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account)
-* [Agents](https://developer.zendesk.com/rest_api/docs/chat/agents#list-agents) \(Incremental\)
-* [Agent Timelines](https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export) \(Incremental\)
-* [Chats](https://developer.zendesk.com/rest_api/docs/chat/chats#list-chats)
-* [Shortcuts](https://developer.zendesk.com/rest_api/docs/chat/shortcuts#list-shortcuts)
-* [Triggers](https://developer.zendesk.com/rest_api/docs/chat/triggers#list-triggers)
-* [Bans](https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans) \(Incremental\)
-* [Departments](https://developer.zendesk.com/rest_api/docs/chat/departments#list-departments)
-* [Goals](https://developer.zendesk.com/rest_api/docs/chat/goals#list-goals)
-* [Skills](https://developer.zendesk.com/rest_api/docs/chat/skills#list-skills)
-* [Roles](https://developer.zendesk.com/rest_api/docs/chat/roles#list-roles)
-* [Routing Settings](https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings)
+- [Accounts](https://developer.zendesk.com/rest_api/docs/chat/accounts#show-account)
+- [Agents](https://developer.zendesk.com/rest_api/docs/chat/agents#list-agents) \(Incremental\)
+- [Agent Timelines](https://developer.zendesk.com/rest_api/docs/chat/incremental_export#incremental-agent-timeline-export) \(Incremental\)
+- [Chats](https://developer.zendesk.com/rest_api/docs/chat/chats#list-chats)
+- [Shortcuts](https://developer.zendesk.com/rest_api/docs/chat/shortcuts#list-shortcuts)
+- [Triggers](https://developer.zendesk.com/rest_api/docs/chat/triggers#list-triggers)
+- [Bans](https://developer.zendesk.com/rest_api/docs/chat/bans#list-bans) \(Incremental\)
+- [Departments](https://developer.zendesk.com/rest_api/docs/chat/departments#list-departments)
+- [Goals](https://developer.zendesk.com/rest_api/docs/chat/goals#list-goals)
+- [Skills](https://developer.zendesk.com/rest_api/docs/chat/skills#list-skills)
+- [Roles](https://developer.zendesk.com/rest_api/docs/chat/roles#list-roles)
+- [Routing Settings](https://developer.zendesk.com/rest_api/docs/chat/routing_settings#show-account-routing-settings)
 
 ## Performance considerations
 
@@ -77,11 +79,11 @@ The connector is restricted by Zendesk's [requests limitation](https://developer
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                          |
-|:--------| :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- |
-| 0.1.14  | 2023-02-10 | [24190](https://github.com/airbytehq/airbyte/pull/24190) | Fix remove too high min/max from account stream  |
-| 0.1.13  | 2023-02-10 | [22819](https://github.com/airbytehq/airbyte/pull/22819) | Specified date formatting in specification                                                     |
-| 0.1.12  | 2023-01-27 | [22026](https://github.com/airbytehq/airbyte/pull/22026) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                     |
-| 0.1.11  | 2022-10-18 | [17745](https://github.com/airbytehq/airbyte/pull/17745) | Add Engagements Stream and fix infity looping                                                                            |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- |
+| 0.1.14  | 2023-02-10 | [24190](https://github.com/airbytehq/airbyte/pull/24190) | Fix remove too high min/max from account stream                                                                  |
+| 0.1.13  | 2023-02-10 | [22819](https://github.com/airbytehq/airbyte/pull/22819) | Specified date formatting in specification                                                                       |
+| 0.1.12  | 2023-01-27 | [22026](https://github.com/airbytehq/airbyte/pull/22026) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                      |
+| 0.1.11  | 2022-10-18 | [17745](https://github.com/airbytehq/airbyte/pull/17745) | Add Engagements Stream and fix infity looping                                                                    |
 | 0.1.10  | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states.                                                                                    |
 | 0.1.9   | 2022-08-23 | [15879](https://github.com/airbytehq/airbyte/pull/15879) | Corrected specification and stream schemas to support backward capability                                        |
 | 0.1.8   | 2022-06-28 | [13387](https://github.com/airbytehq/airbyte/pull/13387) | Add state checkpoint to allow long runs                                                                          |
diff --git a/docs/integrations/sources/zendesk-sunshine.md b/docs/integrations/sources/zendesk-sunshine.md
index c12b22d9371f..0b957eee840e 100644
--- a/docs/integrations/sources/zendesk-sunshine.md
+++ b/docs/integrations/sources/zendesk-sunshine.md
@@ -64,6 +64,8 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.2.0 | 2023-08-22 | [29310](https://github.com/airbytehq/airbyte/pull/29310) | Migrate Python CDK to Low Code |
+| 0.1.2 | 2023-08-15 | [7976](https://github.com/airbytehq/airbyte/pull/7976) | Fix schemas and tests |
 | 0.1.1 | 2021-11-15 | [7976](https://github.com/airbytehq/airbyte/pull/7976) | Add oauth2.0 support |
 | 0.1.0 | 2021-07-08 | [4359](https://github.com/airbytehq/airbyte/pull/4359) | Initial Release |
 
diff --git a/docs/integrations/sources/zendesk-support-migrations.md b/docs/integrations/sources/zendesk-support-migrations.md
new file mode 100644
index 000000000000..e6d28f511b32
--- /dev/null
+++ b/docs/integrations/sources/zendesk-support-migrations.md
@@ -0,0 +1,6 @@
+# Zendesk Support Migration Guide
+
+## Upgrading to 1.0.0
+
+`cursor_field` for `Tickets` stream is changed to `generated_timestamp`.
+For a smooth migration, data reset and schema refresh are needed.
\ No newline at end of file
diff --git a/docs/integrations/sources/zendesk-support.inapp.md b/docs/integrations/sources/zendesk-support.inapp.md
new file mode 100644
index 000000000000..e62c88b8a182
--- /dev/null
+++ b/docs/integrations/sources/zendesk-support.inapp.md
@@ -0,0 +1,27 @@
+## Prerequisites
+
+- A Zendesk account with an Administrator role.
+
+## Setup Guide
+
+
+
+For **Airbyte Open Source** users, we recommend using an API token to authenticate your Zendesk account. You can follow the steps in our [full documentation](https://docs.airbyte.com/integrations/sources/zendesk-support#setup-guide) to generate this token.
+
+
+
+1. For **Source name**, enter a name to help you identify this source.
+2. You can use OAuth or an API token to authenticate your Zendesk account. We recommend using OAuth for Airbyte Cloud and an API token for Airbyte Open Source.
+
+   
+   - **For Airbyte Cloud:** To authenticate using OAuth, select **OAuth2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk and authorize your account.
+   
+   
+   - **For Airbyte Open Source**: To authenticate using an API token, select **API Token** from the Authentication dropdown and enter the [token you generated](https://docs.airbyte.com/integrations/sources/zendesk-support#setup-guide), as well as the email address associated with your Zendesk account.
+   
+
+3. For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated.
+4. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain.
+5. Click **Set up source** and wait for the tests to complete.
+
+For detailed information on supported sync modes, supported streams and performance considerations, refer to the [full documentation for Zendesk Support](https://docs.airbyte.com/integrations/sources/zendesk-support).
diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md
index e1ad0c7750d4..e25f589a9ada 100644
--- a/docs/integrations/sources/zendesk-support.md
+++ b/docs/integrations/sources/zendesk-support.md
@@ -1,24 +1,61 @@
 # Zendesk Support
 
-This page guides you through setting up the Zendesk Support source connector.
+This page contains the setup guide and reference information for the Zendesk Support source connector.
 
 ## Prerequisites
 
-- Locate your Zendesk subdomain found in your account URL. For example, if your account URL is `https://{MY_SUBDOMAIN}.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain.
-- (For Airbyte Open Source) Find the email address associated with your Zendesk account. Also, generate an [API token](https://support.zendesk.com/hc/en-us/articles/4408889192858-Generating-a-new-API-token) for the account.
+- A Zendesk account with an Administrator role.
 
-## Set up the Zendesk Support source connector
+## Setup guide
 
-1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
-2. Click **Sources** and then click **+ New source**.
-3. On the Set up the source page, select **Zendesk Support** from the Source type dropdown.
-4. Enter a name for your source.
-5. For **Subdomain**, enter your [Zendesk subdomain](#prerequisites).
-6. For **Start date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data.
-7. You can use OAuth or an API key to authenticate your Zendesk Support account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source.
-   - To authenticate using OAuth for Airbyte Cloud, click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account.
-   - To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter your [API key](#prerequisites). Enter the **Email** associated with your Zendesk Support account.
-8. Click **Set up source**.
+The Zendesk Support source connector supports two authentication methods:
+
+- OAuth 2.0
+- API token
+
+
+For **Airbyte Cloud** users, we highly recommend using OAuth to authenticate your Zendesk Support account, as it simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-zendesk-support-source-connector).
+
+
+For **Airbyte Open Source** users, we recommend using an API token to authenticate your Zendesk Support account. Please follow the steps below to generate this key.
+
+:::note
+If you prefer to authenticate with OAuth for **Airbyte Open Source**, you can follow the steps laid out in [this Zendesk article](https://support.zendesk.com/hc/en-us/articles/4408845965210) to obtain your client ID, client secret and access token. Please ensure you set the scope to `read` when generating the access token.
+:::
+
+### (Airbyte Open Source) Enable API token access and generate a token
+
+1. Log in to your Zendesk account.
+2. Click the **Zendesk Products** icon (four squares) in the top-right corner, then select **Admin Center**.
+3. In the left navbar, click **Apps and Integrations**, then select **APIs** > **Zendesk API**.
+4. In the **Settings** tab, toggle the option to enable token access.
+5. Click the **Add API token** button. You may optionally provide a token description.
+
+   :::caution
+   Be sure to copy the token and save it in a secure location. You will not be able to access the token's value after you close the page.
+   :::
+
+6. Click **Save**.
+
+
+### Set up the Zendesk Support source connector
+
+1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account.
+2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**.
+3. Find and select **Zendesk Support** from the list of available sources.
+4. For **Source name**, enter a name to help you identify this source.
+5. You can use OAuth or an API token to authenticate your Zendesk Support account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source.
+
+   
+   - **For Airbyte Cloud**: To authenticate using OAuth, select **OAuth 2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account.
+   
+   
+   - **For Airbyte Open Source**: To authenticate using an API key, select **API Token** from the Authentication dropdown and enter the API token you generated, as well as the email address associated with your Zendesk Support account.
+   
+
+6. For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated.
+7. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain.
+8. Click **Set up source** and wait for the tests to complete.
 
 ## Supported sync modes
 
@@ -36,17 +73,24 @@ There are two types of incremental sync:
 
 1. Incremental (standard server-side, where API returns only the data updated or generated since the last sync)
 2. Client-Side Incremental (API returns all available data and connector filters out only new records)
-   :::
+:::
 
 The Zendesk Support source connector supports the following streams:
 
-- [AuditLogs](https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/#list-audit-logs)\(Incremental\) (Only available for enterprise accounts)
+- [Account Attributes](https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-account-attributes)
+- [Attribute Definitions](https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-routing-attribute-definitions)
+- [Audit Logs](https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/#list-audit-logs)\(Incremental\) (Only available for enterprise accounts)
 - [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands)
 - [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles)
 - [Groups](https://developer.zendesk.com/rest_api/docs/support/groups) \(Incremental\)
 - [Group Memberships](https://developer.zendesk.com/rest_api/docs/support/group_memberships) \(Incremental\)
 - [Macros](https://developer.zendesk.com/rest_api/docs/support/macros) \(Incremental\)
 - [Organizations](https://developer.zendesk.com/rest_api/docs/support/organizations) \(Incremental\)
+- [Organization Memberships](https://developer.zendesk.com/api-reference/ticketing/organizations/organization_memberships/) \(Incremental\)
+- [Posts](https://developer.zendesk.com/api-reference/help_center/help-center-api/posts/#list-posts) \(Incremental\)
+- [Post Comments](https://developer.zendesk.com/api-reference/help_center/help-center-api/post_comments/#list-comments)
+- [Post Comment Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes)
+- [Post Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes)
 - [Satisfaction Ratings](https://developer.zendesk.com/rest_api/docs/support/satisfaction_ratings) \(Incremental\)
 - [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules)
 - [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies)
@@ -58,6 +102,8 @@ The Zendesk Support source connector supports the following streams:
 - [Ticket Forms](https://developer.zendesk.com/rest_api/docs/support/ticket_forms) \(Incremental\)
 - [Ticket Metrics](https://developer.zendesk.com/rest_api/docs/support/ticket_metrics) \(Incremental\)
 - [Ticket Metric Events](https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_metric_events/) \(Incremental\)
+- [Topics](https://developer.zendesk.com/api-reference/help_center/help-center-api/topics/#list-topics) \(Incremental\)
+- [Ticket Skips](https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_skips/) \(Incremental\)
 - [Users](https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-user-export) \(Incremental\)
 
 ## Performance considerations
@@ -69,8 +115,25 @@ The Zendesk connector ideally should not run into Zendesk API limitations under
 ## Changelog
 
 | Version  | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                            |
-| :------- | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `0.3.1`  | 2023-06-02 | [26945](https://github.com/airbytehq/airbyte/pull/26945) | Make `Ticket Metrics` stream to use cursor pagination  |
+|:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `1.0.0`  | 2023-07-27 | [28774](https://github.com/airbytehq/airbyte/pull/28774) | fix retry logic & update cursor for `Tickets` stream                                                                                                                                                                               |
+| `0.11.0` | 2023-08-10 | [27208](https://github.com/airbytehq/airbyte/pull/27208) | Add stream `Topics`                                                                                                                                                                                                                |
+| `0.10.7` | 2023-08-09 | [29256](https://github.com/airbytehq/airbyte/pull/29256) | Update tooltip descriptions in spec                                                                                                                                                                                                |
+| `0.10.6` | 2023-08-04 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes                                                                                                                                                                                               |
+| `0.10.5` | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references                                                                                                                                                                                           |
+| `0.10.4` | 2023-07-25 | [28397](https://github.com/airbytehq/airbyte/pull/28397) | Handle 404 Error                                                                                                                                                                                                                   |
+| `0.10.3` | 2023-07-24 | [28612](https://github.com/airbytehq/airbyte/pull/28612) | Fix pagination for stream `TicketMetricEvents`                                                                                                                                                                                     |
+| `0.10.2` | 2023-07-19 | [28487](https://github.com/airbytehq/airbyte/pull/28487) | Remove extra page from params                                                                                                                                                                                                      |
+| `0.10.1` | 2023-07-10 | [28096](https://github.com/airbytehq/airbyte/pull/28096) | Replace `offset` pagination with `cursor` pagination                                                                                                                                                                               |
+| `0.10.0` | 2023-07-06 | [27991](https://github.com/airbytehq/airbyte/pull/27991) | Add streams: `PostVotes`, `PostCommentVotes`                                                                                                                                                                                       |
+| `0.9.0`  | 2023-07-05 | [27961](https://github.com/airbytehq/airbyte/pull/27961) | Add stream: `Post Comments`                                                                                                                                                                                                        |
+| `0.8.1`  | 2023-06-27 | [27765](https://github.com/airbytehq/airbyte/pull/27765) | Bugfix: Nonetype error while syncing more then 100000 organizations                                                                                                                                                                |
+| `0.8.0`  | 2023-06-09 | [27156](https://github.com/airbytehq/airbyte/pull/27156) | Add stream `Posts`                                                                                                                                                                                                                 |
+| `0.7.0`  | 2023-06-27 | [27436](https://github.com/airbytehq/airbyte/pull/27436) | Add Ticket Skips stream                                                                                                                                                                                                            |
+| `0.6.0`  | 2023-06-27 | [27450](https://github.com/airbytehq/airbyte/pull/27450) | Add Skill Based Routing streams                                                                                                                                                                                                    |
+| `0.5.0`  | 2023-06-26 | [27735](https://github.com/airbytehq/airbyte/pull/27735) | License Update: Elv2 stream stream                                                                                                                                                                                                 |
+| `0.4.0`  | 2023-06-16 | [27431](https://github.com/airbytehq/airbyte/pull/27431) | Add Organization Memberships stream                                                                                                                                                                                                |
+| `0.3.1`  | 2023-06-02 | [26945](https://github.com/airbytehq/airbyte/pull/26945) | Make `Ticket Metrics` stream to use cursor pagination                                                                                                                                                                              |
 | `0.3.0`  | 2023-05-23 | [26347](https://github.com/airbytehq/airbyte/pull/26347) | Add stream `Audit Logs` logs`                                                                                                                                                                                                      |
 | `0.2.30` | 2023-05-23 | [26414](https://github.com/airbytehq/airbyte/pull/26414) | Added missing handlers when `empty json` or `JSONDecodeError` is received                                                                                                                                                          |
 | `0.2.29` | 2023-04-18 | [25214](https://github.com/airbytehq/airbyte/pull/25214) | Add missing fields to `Tickets` stream                                                                                                                                                                                             |
diff --git a/docs/integrations/sources/zendesk-talk.md b/docs/integrations/sources/zendesk-talk.md
index f3c42baf2a8a..16b202874c4e 100644
--- a/docs/integrations/sources/zendesk-talk.md
+++ b/docs/integrations/sources/zendesk-talk.md
@@ -76,6 +76,8 @@ The Zendesk connector should not run into Zendesk API limitations under normal u
 
 | Version | Date       | Pull Request | Subject                           |
 |:--------|:-----------| :-----       |:----------------------------------|
+| `0.1.9` | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes |
+| `0.1.8` | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references |
 | `0.1.7` | 2023-02-10 | [22815](https://github.com/airbytehq/airbyte/pull/22815) | Specified date formatting in specification                                                     |
 | `0.1.6` | 2023-01-27 | [22028](https://github.com/airbytehq/airbyte/pull/22028) | Set `AvailabilityStrategy` for streams explicitly to `None`                                                     |
 | `0.1.5` | 2022-09-29 | [17362](https://github.com/airbytehq/airbyte/pull/17362) | always use the latest CDK version |
diff --git a/docs/integrations/sources/zenloop.md b/docs/integrations/sources/zenloop.md
index 905185309c6d..54cb310d8e9c 100644
--- a/docs/integrations/sources/zenloop.md
+++ b/docs/integrations/sources/zenloop.md
@@ -69,12 +69,16 @@ The Zenloop connector should not run into Zenloop API limitations under normal u
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                                 |
-|:--------| :--------- | :------------------------------------------------------- |:--------------------------------------------------------|
-| 0.1.6   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                 |
-| 0.1.5   | 2023-02-08 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Fix unhashable type in ZenloopSubstreamSlicer component |
-| 0.1.4   | 2022-11-18 | [19624](https://github.com/airbytehq/airbyte/pull/19624) | Migrate to low code                                     |
-| 0.1.3   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states                            |
-| 0.1.2   | 2022-08-22 | [15843](https://github.com/airbytehq/airbyte/pull/15843) | Adds Properties stream                                  |
-| 0.1.1   | 2021-10-26 | [8299](https://github.com/airbytehq/airbyte/pull/8299)   | Fix missing seed files                                  |
-| 0.1.0   | 2021-10-26 | [7380](https://github.com/airbytehq/airbyte/pull/7380)   | Initial Release                                         |
+| Version | Date       | Pull Request                                             | Subject                                                             |
+|:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------|
+| 0.1.10  | 2023-06-29 | [27838](https://github.com/airbytehq/airbyte/pull/27838) | Update CDK version to avoid bug introduced during data feed release |
+| 0.1.9   | 2023-06-28 | [27761](https://github.com/airbytehq/airbyte/pull/27761) | Update following state breaking changes                             |
+| 0.1.8   | 2023-06-22 | [27243](https://github.com/airbytehq/airbyte/pull/27243) | Improving error message on state discrepancy                        |
+| 0.1.7   | 2023-06-22 | [27243](https://github.com/airbytehq/airbyte/pull/27243) | State per partition (breaking change - require reset)               |
+| 0.1.6   | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version                             |
+| 0.1.5   | 2023-02-08 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Fix unhashable type in ZenloopSubstreamSlicer component             |
+| 0.1.4   | 2022-11-18 | [19624](https://github.com/airbytehq/airbyte/pull/19624) | Migrate to low code                                                 |
+| 0.1.3   | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states                                        |
+| 0.1.2   | 2022-08-22 | [15843](https://github.com/airbytehq/airbyte/pull/15843) | Adds Properties stream                                              |
+| 0.1.1   | 2021-10-26 | [8299](https://github.com/airbytehq/airbyte/pull/8299)   | Fix missing seed files                                              |
+| 0.1.0   | 2021-10-26 | [7380](https://github.com/airbytehq/airbyte/pull/7380)   | Initial Release                                                     |
diff --git a/docs/integrations/sources/zoom.md b/docs/integrations/sources/zoom.md
index aeba329c79ec..f7b9764239ae 100644
--- a/docs/integrations/sources/zoom.md
+++ b/docs/integrations/sources/zoom.md
@@ -53,15 +53,22 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see
 
 ### Requirements
 
-* Zoom JWT Token
+* Zoom Server-to-Server Oauth App
 
 ### Setup guide
+Please read [How to generate your Server-to-Server OAuth app ](https://developers.zoom.us/docs/internal-apps/s2s-oauth/).
+
+:::info
+
+JWT Tokens are deprecated, only Server-to-Server works now. [link to Zoom](https://developers.zoom.us/docs/internal-apps/jwt-faq/)
+
+:::
 
-Please read [How to generate your JWT Token](https://marketplace.zoom.us/docs/guides/build/jwt-app).
 
 ## Changelog
 
 | Version | Date       | Pull Request                                             | Subject                                                                |
-| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------- |
+|:--------|:-----------|:---------------------------------------------------------| :--------------------------------------------------------------------- |
+| 1.0.0   | 2023-7-28  | [25308](https://github.com/airbytehq/airbyte/pull/25308) | Replace JWT Auth methods with server-to-server Oauth |
 | 0.1.1   | 2022-11-30 | [19939](https://github.com/airbytehq/airbyte/pull/19939) | Upgrade CDK version to fix bugs with SubStreamSlicer |
 | 0.1.0   | 2022-10-25 | [18179](https://github.com/airbytehq/airbyte/pull/18179) | Initial Release                                                        |
diff --git a/docs/operator-guides/assets/DatadogAirbyteIntegration_OutOfTheBox_Dashboard.png b/docs/operator-guides/assets/DatadogAirbyteIntegration_OutOfTheBox_Dashboard.png
new file mode 100644
index 000000000000..52b2baa4123c
Binary files /dev/null and b/docs/operator-guides/assets/DatadogAirbyteIntegration_OutOfTheBox_Dashboard.png differ
diff --git a/docs/operator-guides/collecting-metrics.md b/docs/operator-guides/collecting-metrics.md
index 09406df8e564..db1c6665b2c4 100644
--- a/docs/operator-guides/collecting-metrics.md
+++ b/docs/operator-guides/collecting-metrics.md
@@ -1,270 +1,140 @@
-# Collecting Metrics
+# Monitoring Airbyte
+
+Airbyte offers you various ways to monitor your ELT pipelines. These options range from using open-source tools to integrating with enterprise-grade SaaS platforms.
+
+Here's a quick overview:
+* Connection Logging: All Airbyte instances provide extensive logs for each connector, giving detailed reports on the data synchronization process. This is available across all Airbyte offerings.
+* [Airbyte Datadog Integration](#airbyte-datadog-integration): Airbyte Enterprise customers can leverage our integration with Datadog. This lets you monitor and analyze your data pipelines right within your Datadog dashboards at no additional cost.
+* Airbyte OpenTelemetry (OTEL) Integration: Coming soon, this will allow you to push metrics to your self-hosted monitoring solution using OpenTelemetry.
+
+Please browse the sections below for more details on each option and how to set it up.
+
+## Airbyte Datadog Integration
+
+![Datadog's Airbyte Integration Dashboard](assets/DatadogAirbyteIntegration_OutOfTheBox_Dashboard.png)
+
+_This integration is available for **Airbyte Enterprise users**. Airbyte Enterprise includes premium support with SLAs for your critical pipelines, security features including SSO, RBAC and audit logging, in addition to reliability, scalability and compliance features.
+Please reach out to [our team](https://airbyte.com/talk-to-sales) if you want to learn more._
+
+Airbyte's new integration with Datadog brings the convenience of monitoring and analyzing your Airbyte data pipelines directly within your Datadog dashboards. 
+This integration brings forth new `airbyte.*` metrics along with new dashboards. The list of metrics is found [here](https://docs.datadoghq.com/integrations/airbyte/#data-collected).
+
+### Setup Instructions
+
+Setting up this integration for Airbyte instances deployed with Docker involves five straightforward steps:
+
+
+1. **Set Datadog Airbyte Config:** Create or configure the `datadog.yaml` file with the contents below:
+
+```yaml
+dogstatsd_mapper_profiles:
+  - name: airbyte_worker
+    prefix: "worker."
+    mappings:
+      - match: "worker.temporal_workflow_*"
+        name: "airbyte.worker.temporal_workflow.$1"
+      - match: "worker.worker_*"
+        name: "airbyte.worker.$1"
+      - match: "worker.state_commit_*"
+        name: "airbyte.worker.state_commit.$1"
+      - match: "worker.job_*"
+        name: "airbyte.worker.job.$1"
+      - match: "worker.attempt_*"
+        name: "airbyte.worker.attempt.$1"
+      - match: "worker.activity_*"
+        name: "airbyte.worker.activity.$1"
+      - match: "worker.*"
+        name: "airbyte.worker.$1"
+  - name: airbyte_cron
+    prefix: "cron."
+    mappings:
+      - match: "cron.cron_jobs_run"
+        name: "airbyte.cron.jobs_run"
+      - match: "cron.*"
+        name: "airbyte.cron.$1"
+  - name: airbyte_metrics_reporter
+    prefix: "metrics-reporter."
+    mappings:
+      - match: "metrics-reporter.*"
+        name: "airbyte.metrics_reporter.$1"
+  - name: airbyte_orchestrator
+    prefix: "orchestrator."
+    mappings:
+      - match: "orchestrator.*"
+        name: "airbyte.orchestrator.$1"
+  - name: airbyte_server
+    prefix: "server."
+    mappings:
+      - match: "server.*"
+        name: "airbyte.server.$1"
+  - name: airbyte_general
+    prefix: "airbyte."
+    mappings:
+      - match: "airbyte.worker.temporal_workflow_*"
+        name: "airbyte.worker.temporal_workflow.$1"
+      - match: "airbyte.worker.worker_*"
+        name: "airbyte.worker.$1"
+      - match: "airbyte.worker.state_commit_*"
+        name: "airbyte.worker.state_commit.$1"
+      - match: "airbyte.worker.job_*"
+        name: "airbyte.worker.job.$1"
+      - match: "airbyte.worker.attempt_*"
+        name: "airbyte.worker.attempt.$1"
+      - match: "airbyte.worker.activity_*"
+        name: "airbyte.worker.activity.$1"
+      - match: "airbyte.cron.cron_jobs_run"
+        name: "airbyte.cron.jobs_run"
+```
+
+2. **Add Datadog Agent and Mount Config:** If the Datadog Agent is not yet deployed to your instances running Airbyte, you can modify the provided  `docker-compose.yaml` file in the Airbyte repository to include the Datadog Agent. For the Datadog agent to submit metrics, you will need to add an [API key](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token). Then, be sure to properly mount your `datadog.yaml` file as a Docker volume:
+
+```yaml
+  dd-agent:
+    container_name: dd-agent
+    image: gcr.io/datadoghq/agent:7
+    pid: host
+    environment:
+      - DD_API_KEY={REPLACE-WITH-DATADOG-API-KEY}
+      - DD_SITE=datadoghq.com
+      - DD_HOSTNAME={REPLACE-WITH-DATADOG-HOSTNAME}
+      - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup:/host/sys/fs/cgroup:ro
+      - {REPLACE-WITH-PATH-TO}/datadog.yaml:/etc/datadog-agent/datadog.yaml
+    networks:
+      - airbyte_internal
+```
+
+3. **Update Docker Compose Configuration**: Modify your `docker-compose.yaml` file in the Airbyte repository to include the `metrics-reporter` container. This submits Airbyte metrics to the Datadog Agent:
+
+```yaml
+  metric-reporter:
+    image: airbyte/metrics-reporter:${VERSION}
+    container_name: metric-reporter
+    networks:
+      - airbyte_internal
+    environment:
+      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
+      - DATABASE_URL=${DATABASE_URL}
+      - DATABASE_USER=${DATABASE_USER}
+      - DD_AGENT_HOST=${DD_AGENT_HOST}
+      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
+      - METRIC_CLIENT=${METRIC_CLIENT}
+      - PUBLISH_METRICS=${PUBLISH_METRICS}
+```
+
+4. **Set Environment Variables**: Amend your `.env` file with the correct values needed by `docker-compose.yaml`:
+
+```yaml
+PUBLISH_METRICS=true
+METRIC_CLIENT=datadog
+DD_AGENT_HOST=dd-agent
+DD_DOGSTATSD_PORT=8125
+```
+
+5. **Re-deploy Airbyte and the Datadog Agent**: With the updated configurations, you're ready to deploy your Airbyte application by running `docker compose up`.
 
-Airbyte supports two ways to collect metrics - using datadog or open telemetry. 
-Fill in `METRIC_CLIENT` field in `.env` file to get started!
 
-**Prerequisite:** 
-In order to get metrics from airbyte we need to deploy a container / pod called metrics-reporter like below 
-```
-airbyte-metrics: 
-  image: airbyte/metrics-reporter:${VERSION} 
-  container_name: airbyte-metrics 
-  environment: 
-    - METRIC_CLIENT=${METRIC_CLIENT} 
-    - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT}
-```
-
-
-# Open Telemetry
-
-1. In `.env` change `METRIC_CLIENT` to `otel`. 
-2. Similarly, configure `OTEL_COLLECTOR_ENDPOINT` to tell Airbyte where to send metrics RPC to.
-
-## Example
-
-### Run Opentelemetry and Airbyte locally
-
-In this example we will run Airbyte locally along with an Open Telemetry Collector. The Open telemetry collector
-will expose port 4317 to the localhost as the receiving endpoint.
-
-![](../.gitbook/assets/open_telemetry_example.png)
-
-Steps:
-
-1. Setting up Open telemetry. In this example we will use the repository from `opentelemetry-java-docs`. 
-Run the following commands to have it up and running.
-
-```bash
-  git clone https://github.com/open-telemetry/opentelemetry-java-docs
-  cd opentelemetry-java-docs/otlp/docker
-  docker-compose up
-```
-
-2. Configure Airbyte `.env` file. 
-   1. Change `METRIC_CLIENT` to `otel` to indicate Airbyte to use Open telemetry to emit metric data.
-   2. Change `OTEL_COLLECTOR_ENDPOINT` to `"http://host.docker.internal:4317"` because Open Telemetry 
-   Collector has enabled port forward from localhost:4317 to container port 4317. To send data to Collector container port 4317, we want to need to export data to physical machine's localhost:4317, which in docker will be represented as `http://host.docker.internal:4317`. 
-   > Do *not* use `localhost:4317` or you will send data to the same container where Airbyte Worker is running.
-   3. Start Airbyte server by running `docker compose up` under airbyte repository. Go to `localhost:8000` to visit Airbyte and start a sync, then go to `localhost:9090` to access Prometheus - you should be able to see the metrics there. Alternatively,
-
-### Run Opentelemetry and Airbyte on kubernetes
-
-> **Prerequisite:** Read https://github.com/airbytehq/airbyte/blob/master/docs/deploying-airbyte/on-kubernetes.md to understand how to start Airbyte on Kubernetes
-
-We will use `stable` in this example.
-
-Steps:
-1. Run open telemetry collector in the same Kubernetes context. Here we follow example in [OpenTelemetry doc](https://opentelemetry.io/docs/collector/getting-started/#kubernetes)
-2. edit `kube/overlays/stable/.env` and add the following lines:
-
-```aidl
-METRIC_CLIENT=otel
-OTEL_COLLECTOR_ENDPOINT=
    
-```
-
-If you started open telemetry collector in the link above, the address should be `http://otel-collector:4317`. 
-Note the format - unlike the base `.env`, there is no quote in `.env` file under kubernetes.
-
-
-## Tutorial 
-
-Deploy the airbyte metric pod :
-```
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ .Release.Name }}-metrics
-  namespace: {{ .Release.Namespace }}
-  labels: {{ set . "component" "metrics" | include "labels" | nindent 4 }}
-spec:
-  selector:
-    matchLabels: {{ set . "component" "metrics" | include "labels" | nindent 6 }}
-  template:
-    metadata:
-      labels: {{ set . "component" "metrics" | include "labels" | nindent 8 }}
-    spec:
-      containers:
-      - name: airbyte-metrics
-        image: "airbyte/metrics-reporter:latest"
-        imagePullPolicy: IfNotPresent
-        env:
-        - name: AIRBYTE_VERSION
-          value: latest
-        - name: DATABASE_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.database.secret.name" . }}
-              key: DATABASE_PASSWORD
-        - name: DATABASE_URL
-          value: {{ include "airbyte.database.url" . | quote }}
-        - name: DATABASE_USER
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-secrets
-              key: DATABASE_USER
-        - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION
-          value: 0.35.15.001
-        - name: METRIC_CLIENT
-          value: otel
-        - name: OTEL_COLLECTOR_ENDPOINT
-          value: http://otel-collector:4317
-```
-
-
-Deploy an Open telemetry pod like below :
-
-```
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: otel-collector
-  namespace: {{ .Release.Namespace }}
-  labels: {{ set . "component" "otel-collector" | include "labels" | nindent 4 }} 
-spec:
-  selector:
-    matchLabels: {{ set . "component" "otel-collector" | include "labels" | nindent 6 }} 
-  replicas: 1
-  template:
-    metadata:
-      labels: {{ set . "component" "otel-collector" | include "labels" | nindent 8 }} 
-    spec:
-      containers:
-      - command:
-          - "/otelcol"
-          - "--config=/conf/otel-collector-config.yaml"
-        image: "otel/opentelemetry-collector:latest"
-        name: otel-collector
-        ports:
-        - containerPort: 4317 # Default endpoint for OpenTelemetry receiver.
-        - containerPort: 8889 # Port for Prometheus instance
-        volumeMounts:
-        - name: config
-          mountPath: /conf
-      volumes:
-        - configMap:
-            name: otel-collector-conf
-            items:
-              - key: otel-collector-config
-                path: otel-collector-config.yaml
-          name: config
-```
-
-WIth this Config Map :
-
-```
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: otel-collector-conf
-  namespace: {{ .Release.Namespace }}
-  labels: {{ set . "component" "otel-collector" | include "labels" | nindent 4 }} 
-data:
-  otel-collector-config: |
-    receivers:
-      otlp:
-        protocols:
-          grpc:
-           endpoint: "0.0.0.0:4317"
-    processors:
-      batch:
-      memory_limiter:
-        limit_mib: 1500
-        spike_limit_mib: 512
-        check_interval: 5s
-    exporters:
-      prometheus:
-        endpoint: "0.0.0.0:8889"
-        namespace: "default"
-    service:
-      pipelines:
-        metrics:
-          receivers: [otlp]
-          processors: [memory_limiter, batch]
-          exporters: [prometheus]
-```
-
-Then we need a service to be able to access both open telemetry GRPC and Prometheus
-```
-apiVersion: v1
-kind: Service
-metadata:
-  name: otel-collector
-  namespace: {{ .Release.Namespace }}
-  labels: {{ set . "component" "otel-collector" | include "labels" | nindent 4 }} 
-spec:
-  ports:
-  - name: otlp-grpc # Default endpoint for OpenTelemetry gRPC receiver.
-    port: 4317
-    protocol: TCP
-    targetPort: 4317
-  - name: prometheus
-    port: 8889
-  selector: {{ set . "component" "otel-collector" | include "labels" | nindent 6 }}
-```
-
-And finally We can add a service monitor to receive metrics in prometheus and optionally add some prometheus rules to generate alerts. 
-You can replace with your prometheus name.
-```
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: {{ .Release.Name }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{ set . "component" "metrics" | include "labels" | nindent 4 }}
-    prometheus: 
-spec:
-  endpoints:
-    - interval: 30s
-      port: prometheus
-      path: /metrics
-      relabelings:
-      - action: labeldrop
-        regex: (service|endpoint|namespace|container)
-  selector:
-    matchLabels: {{ set . "component" "otel-collector" | include "labels" | nindent 6 }}
-```
-
-One rule example :
-```
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: {{ .Release.Name }}-rules
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{ set . "component" "prometheus-rules" | include "labels" | nindent 4 }}
-    prometheus: 
-spec:
-  groups:
-    - name: airbyte
-      rules:
-        - alert: AirbyteJobFail
-          for: 0m
-          expr: min(airbyte_job_failed_by_release_stage) > 0
-          labels:
-            priority: P2
-          annotations:
-            summary: {{ `"An Airbyte Job has failed"` }}     
-```
-
-
-# Datadog
-The set up for Datadog is pretty straightforward. 
-
-Set two env vars:
-
-1) `METRIC_CLIENT` to `datadog`.
-2) `PUBLISH_METRICS` to `true`.
-
-Configure two additional env vars to the Datadog endpoint:
-
-1) Set `DD_AGENT_HOST` to the IP where the Datadog agent is running.
-2) Set `DD_DOGSTATSD_PORT` to the port the Datadog agent is using.
-
-## Metrics
-Visit [OssMetricsRegistry.java](https://github.com/airbytehq/airbyte-platform/blob/master/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java) to get a complete list of metrics Airbyte is sending.
 
-## Additional information
-Suppose you are looking for a non-production way of collecting metrics with dbt and Metabase, the tutorial [Airbyte Monitoring with dbt and Metabase](https://airbyte.com/blog/airbyte-monitoring-with-dbt-and-metabase) by accessing Airbyte's Postgres DB. The source code is open on [airbytehq/open-data-stack](https://github.com/airbytehq/open-data-stack). Think of it as an exploratory for data analysts and data engineers of building a dashboard on top of the existing Airbyte Postgres database versus the Prometheus more for DevOps engineers in production.
diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md
index 69a06c1bab6f..db54ef40991a 100644
--- a/docs/operator-guides/configuring-airbyte.md
+++ b/docs/operator-guides/configuring-airbyte.md
@@ -84,12 +84,19 @@ Set to empty values, e.g. "" to disable basic auth. **Be sure to change these va
 
 #### Jobs
 
-1. `SYNC_JOB_MAX_ATTEMPTS` - Define the number of attempts a sync will attempt before failing.
-2. `SYNC_JOB_MAX_TIMEOUT_DAYS` - Define the number of days a sync job will execute for before timing out.
-3. `JOB_MAIN_CONTAINER_CPU_REQUEST` - Define the job container's minimum CPU usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
-4. `JOB_MAIN_CONTAINER_CPU_LIMIT` - Define the job container's maximum CPU usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
-5. `JOB_MAIN_CONTAINER_MEMORY_REQUEST` - Define the job container's minimum RAM usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
-6. `JOB_MAIN_CONTAINER_MEMORY_LIMIT` - Define the job container's maximum RAM usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
+1. `SYNC_JOB_MAX_ATTEMPTS` - Define the number of attempts a sync will attempt before failing. *Legacy - this is superseded by the values below*
+2. `SYNC_JOB_RETRIES_COMPLETE_FAILURES_MAX_SUCCESSIVE` - Defines the max number of successive attempts in which no data was synchronized before failing the job.
+3. `SYNC_JOB_RETRIES_COMPLETE_FAILURES_MAX_TOTAL` - Defines the max number of attempts in which no data was synchronized before failing the job.
+4. `SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_MIN_INTERVAL_S` - Defines the minimum backoff interval in seconds between failed attempts in which no data was synchronized.
+5. `SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_MAX_INTERVAL_S` - Defines the maximum backoff interval in seconds between failed attempts in which no data was synchronized.
+6. `SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_BASE` - Defines the exponential base of the backoff interval between failed attempts in which no data was synchronized.
+7. `SYNC_JOB_RETRIES_PARTIAL_FAILURES_MAX_SUCCESSIVE` - Defines the max number of attempts in which some data was synchronized before failing the job.
+8. `SYNC_JOB_RETRIES_PARTIAL_FAILURES_MAX_TOTAL` - Defines the max number of attempts in which some data was synchronized before failing the job.
+9. `SYNC_JOB_MAX_TIMEOUT_DAYS` - Define the number of days a sync job will execute for before timing out.
+10. `JOB_MAIN_CONTAINER_CPU_REQUEST` - Define the job container's minimum CPU usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
+11. `JOB_MAIN_CONTAINER_CPU_LIMIT` - Define the job container's maximum CPU usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
+12. `JOB_MAIN_CONTAINER_MEMORY_REQUEST` - Define the job container's minimum RAM usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
+13. `JOB_MAIN_CONTAINER_MEMORY_LIMIT` - Define the job container's maximum RAM usage. Units follow either Docker or Kubernetes, depending on the deployment. Defaults to none.
 
 #### Logging
 
@@ -109,11 +116,10 @@ Set to empty values, e.g. "" to disable basic auth. **Be sure to change these va
 2. `MAX_CHECK_WORKERS` - Define the maximum number of Check workers each Airbyte Worker container can support. Defaults to 5.
 3. `MAX_SYNC_WORKERS` - Define the maximum number of Sync workers each Airbyte Worker container can support. Defaults to 5.
 4. `MAX_DISCOVER_WORKERS` - Define the maximum number of Discover workers each Airbyte Worker container can support. Defaults to 5.
-5. `SENTRY_DSN` - Define the [DSN](https://docs.sentry.io/product/sentry-basics/dsn-explainer/) of necessary Sentry instance. Defaults to empty. Integration with Sentry is explained [here](./sentry-integration.md)
 
 #### Data Retention
 
-1. `TEMPORAL_HISTORY_RETENTION_IN_DAYS` - Define the retention period of the job history in Temporal, defaults to 30 days. When running in docker, 
+1. `TEMPORAL_HISTORY_RETENTION_IN_DAYS` - Define the retention period of the job history in Temporal, defaults to 30 days. When running in docker,
    this same value is applied to the log retention.
 
 ### Docker-Only
@@ -156,7 +162,7 @@ A job specific variable overwrites the default sync job variable defined above.
 
 Note that Airbyte does not support logging to separate Cloud Storage providers.
 
-Please see [here](https://docs.airbyte.com/deploying-airbyte/on-kubernetes#configure-logs) for more information on configuring Kubernetes logging.
+Please see [here](https://docs.airbyte.com/deploying-airbyte/on-kubernetes-via-helm#configure-logs) for more information on configuring Kubernetes logging.
 
 1. `GCS_LOG_BUCKET` - Define the GCS bucket to store logs.
 2. `S3_BUCKET` - Define the S3 bucket to store logs.
diff --git a/docs/operator-guides/contact-support.md b/docs/operator-guides/contact-support.md
new file mode 100644
index 000000000000..659a35e0b854
--- /dev/null
+++ b/docs/operator-guides/contact-support.md
@@ -0,0 +1,63 @@
+# Airbyte Support
+
+Hold up! Have you looked at [our docs](https://docs.airbyte.com/) yet? We recommend searching the wealth of knowledge in our documentation as many times the answer you are looking for is there! 
+
+## Airbyte Open Source Support
+
+Running Airbyte Open Source and have questions that our docs could not clear up? Join our community Slack and forum to connect with other Airbyte users.
+
+**Join our Slack community** [HERE](https://slack.airbyte.com/?_gl=1*1h8mjfe*_gcl_au*MTc4MjAxMDQzOS4xNjgyOTczMDYy*_ga*MTc1OTkyOTYzNi4xNjQxMjQyMjA0*_ga_HDBMVFQGBH*MTY4Nzg4OTQ4MC4zMjUuMS4xNjg3ODkwMjE1LjAuMC4w&_ga=2.58571491.813788522.1687789276-1759929636.1641242204)!
+
+If you're experiencing connection issues and need assistance, send your question to the **#help-connections-issue** channel.
+
+For questions related to building connectors, ask in the **#help-connector-development** channel.
+
+If you need help with infrastructure and deployment, head over to the **#help-infrastructure-deployment** channel.
+
+For assistance with the API, CLI, or setting up orchestration, visit the **#help-api-cli-orchestration** channel.
+
+We also hold daily office hours, Monday to Friday, where you can engage in live discussions with the community and Airbyte team. Join the **#office-hours** channel to participate and view the schedule.
+
+If you require personalized support, reach out to our sales team to inquire about [Airbyte Enterprise](https://airbyte.com/airbyte-enterprise).
+
+## Airbyte Cloud Support
+
+If you have questions about connector setup, error resolution, or want to report a bug, Airbyte Support is available to assist you. We recommend checking [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us) before opening a support ticket.
+
+If you couldn't find the information you need in our docs or Help Center, open a ticket within the Airbyte Cloud platform by selecting the "Support" icon in the lower left navigation bar. Alternatively, you can submit a ticket through our [Help Center](https://support.airbyte.com/hc/en-us) by completing an Airbyte Cloud Support Request.
+
+If you're unsure about the supported connectors, refer to our [connector catalog](https://docs.airbyte.com/integrations/) & [release phases](https://docs.airbyte.com/project-overview/product-release-stages/).
+
+For account or credit-related inquiries, contact our [sales team](https://airbyte.com/talk-to-sales).
+
+If you don't see a connector you need, you can submit a [connector request](https://airbyte.com/connector-requests).
+
+To stay updated on Airbyte's future plans, take a look at [our roadmap](https://github.com/orgs/airbytehq/projects/37/views/1).
+
+## Airbyte Enterprise (self-hosted) Support
+
+If you're running Airbyte Open Source with Airbyte Enterprise or have an OSS support package, we're here to help you with upgrading Airbyte versions, debugging connector issues, or troubleshooting schema changes.
+
+Before opening a support ticket, we recommend consulting [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us). If your question remains unanswered, please submit a ticket through our Help Center. We suggest creating an [Airbyte Help Center account](https://airbyte1416.zendesk.com/auth/v2/login/signin?return_to=https%3A%2F%2Fsupport.airbyte.com%2Fhc%2Fen-us&theme=hc&locale=en-us&brand_id=15365055240347&auth_origin=15365055240347%2Ctrue%2Ctrue) to access your organization's support requests.
+
+Submitting a Pull Request for review?
+
+* Be sure to follow our [contribution guidelines](https://docs.airbyte.com/contributing-to-airbyte/) laid out here on our doc. Highlights include:
+  * PRs should be limited to a single change-set
+* Submit the PR as a PR Request through the Help Center Open Source Enterprise Support Request form
+* If you are submitting a Platform PR we accept Platform PRs in the areas below:
+  * Helm
+  * Environment variable configurations
+  * Bug Fixes 
+  * Security version bumps
+  * **If outside these areas, please open up an issue to help the team understand the need and if we are able to consider a PR**
+
+Submitting a PR does not guarantee its merge. The Airbyte support team will conduct an initial review, and if the PR aligns with Airbyte's roadmap, it will be prioritized based on team capacities and priorities.
+
+Although we strive to offer our utmost assistance, there are certain requests that we are unable to support. Currently, we do not provide assistance for these particular items:
+* Question/troubleshooting assistance with forked versions of Airbyte
+* Configuring using Octavia CLI
+* Creating and configuring custom transformation using dbt
+* Curating unique documentation and training materials
+* Configuring Airbyte to meet security requirements
+
diff --git a/docs/operator-guides/scaling-airbyte.md b/docs/operator-guides/scaling-airbyte.md
index 5019757f4860..062cbd33d715 100644
--- a/docs/operator-guides/scaling-airbyte.md
+++ b/docs/operator-guides/scaling-airbyte.md
@@ -65,44 +65,3 @@ is capped by `SQL_MAX_CONNS`.
 The advice here is best-effort and by no means comprehensive. Please reach out on Slack if anything doesn't make sense or if something can be improved.
 
 If you've been running Airbyte in production and have more tips up your sleeve, we welcome contributions!
-
-## Metrics
-Airbyte supports exporting built-in metrics to Datadog or [OpenTelemetry](https://docs.airbyte.com/operator-guides/collecting-metrics/)
-
-### Key Metrics 
-
-| Key Metrics                     | Description                                                                                                                                      |
-|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|
-| ``oldest_pending_job_age_secs`` | Shows how long a pending job waits before it is scheduled. If a job is in pending state for a long time, more workers may be required.           |
-| ``oldest_running_job_age_secs`` | Shows how long the oldest job has been running. A running job that is too large can indicate stuck jobs. This is relative to each job’s runtime. |
-| ``job_failed_by_release_stage`` | Shows jobs that have failed in that release stage and is tagged as alpha, beta, or GA.                                                           |
-      
-:::note 
-    
-Metrics with ``by_release_stage`` in their name are tagged by connector release stage (alpha, beta, or GA). These tags allow you to filter by release stage. Alpha and beta connectors are less stable and have a higher failure rate than GA connectors, so filtering by those release stages can help you find failed jobs.  
-
-:::
-
-### Recommended Metrics
-
-| Recommended Metrics          | Description                                                                                                                           |
-|-----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
-| ``num_running_jobs & num_pending_jobs`` | Shows how many jobs are currently running and how many jobs are in pending state. These metrics help you understand the general system state. |
-| ``job_succeeded_by_release_stage``      | Shows successful jobs in that release stage and is tagged as alpha, beta, or GA.                                                      |
-| ``job_created_by_release_stage``        | Shows the jobs created in that release stage and is tagged as alpha, beta, or GA.                                                     |
-
-### Example  
-If a job was created for an Alpha source to a Beta destination and the outcome of the job is a success, the following metrics are displayed:
-
-```
-job_created_by_release_stage[“alpha”] = 1;  
-job_created_by_release_stage[“beta”] = 1;  
-job_failed_by_release_stage[“alpha”] = 1;  
-job_succeeded_by_release_stage[“beta”] = 1;  
-```
-
-:::note 
-    
-Each job has a source and destination, so each metric is counted twice — once for source and once for destination.
-
-:::
diff --git a/docs/operator-guides/sentry-integration.md b/docs/operator-guides/sentry-integration.md
deleted file mode 100644
index 7e452ee1cfb4..000000000000
--- a/docs/operator-guides/sentry-integration.md
+++ /dev/null
@@ -1,11 +0,0 @@
-# Sentry Integration
-
-Airbyte provides an opportunity to aggregate connectors' exceptions and errors via [Sentry](https://sentry.io/).
-By default, this option is off. There are 2 possible mechanisms for its activation:
-1. Define the `SENTRY_DSN` environment variable into Dockerfile of necessary connectors.
-2. Define the `SENTRY_DSN` into the workspace environment file (`.env`). Workers will add this variable to all docker connectors automatically.
-
-Most connectors written using the Airbyte Python or Java CDKs automatically detect this environment variable and activate Sentry profiling accordingly. 
-
-## UML diagram
-![](../.gitbook/assets/sentry-flow-v1.png)
diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md
index 21424172f143..d2a4d9b86b76 100644
--- a/docs/operator-guides/upgrading-airbyte.md
+++ b/docs/operator-guides/upgrading-airbyte.md
@@ -12,7 +12,7 @@ If you need help from our team for your upgrades, we offer premium support to ou
 
 Airbyte intelligently performs upgrades automatically based off of your version defined in your `.env` file and will handle data migration for you.
 
-If you are running [Airbyte on Kubernetes](../deploying-airbyte/on-kubernetes.md), you will need to use one of the two processes defined [here](#upgrading-on-k8s-0270-alpha-and-above) that differ based on your Airbyte version.
+If you are running [Airbyte on Kubernetes](../deploying-airbyte/on-kubernetes-via-helm.md), you will need to use one of the two processes defined [here](#upgrading-on-k8s-0270-alpha-and-above) that differ based on your Airbyte version.
 
 ## Mandatory Intermediate Upgrade
 
@@ -51,7 +51,13 @@ Airbyte version 0.40.27 or later requires [Docker Compose V2](https://docs.docke
 
    ii. If you are running Airbyte from downloaded `docker-compose.yaml` and `.env` files without a GitHub repo, run `wget https://raw.githubusercontent.com/airbytehq/airbyte/master/run-ab-platform.sh` to download the installation script.
 
-3. Bring Airbyte back online, optionally with the `-b` flag to run the containers in the background (docker detached mode).
+3. Remove previous local `docker-compose.yaml` and `.env`
+
+   ```bash
+   ./run-ab-platform.sh -r
+   ```
+
+4. Bring Airbyte back online, optionally with the `-b` flag to run the containers in the background (docker detached mode).
 
    ```bash
    ./run-ab-platform.sh -b
@@ -111,7 +117,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers
    Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`.
 
    ```bash
-   docker run --rm -v /tmp:/config airbyte/migration:0.44.12 --\
+   docker run --rm -v /tmp:/config airbyte/migration:0.50.21 --\
    --input /config/airbyte_archive.tar.gz\
    --output /config/airbyte_archive_migrated.tar.gz
    ```
diff --git a/docs/operator-guides/using-kestra-plugin.md b/docs/operator-guides/using-kestra-plugin.md
new file mode 100644
index 000000000000..3c27e8797c2d
--- /dev/null
+++ b/docs/operator-guides/using-kestra-plugin.md
@@ -0,0 +1,101 @@
+---
+description: Using the Kestra Plugin to Orchestrate Airbyte
+---
+
+# Using the Kestra Plugin
+
+Kestra has an official plugin for Airbyte, including support for self-hosted Airbyte and Airbyte Cloud. This plugin allows you to trigger data replication jobs (`Syncs`) and wait for their completion before proceeding with any downstream tasks. Alternatively, you may also run those syncs in a fire-and-forget way by setting the `wait` argument to `false`. 
+
+After Airbyte tasks successfully ingest raw data, you can easily start running downstream data transformations with dbt, Python, SQL, Spark, and many more, using a variety of available plugins. Check the [plugin documentation](https://kestra.io/plugins/) for a list of all supported integrations. 
+
+## Available tasks
+
+These are the two main tasks to orchestrate Airbyte syncs:
+
+1) The `io.kestra.plugin.airbyte.connections.Sync` task will sync connections for a self-hosted Airbyte instance
+
+2) The `io.kestra.plugin.airbyte.cloud.jobs.Sync` task will sync connections for Airbyte Cloud 
+
+## **1. Set up the tools**
+
+First, make sure you have Docker installed. We'll be using the `docker-compose` command, so your installation should contain `docker-compose`. When you use [Docker Desktop](https://docs.docker.com/compose/install/#scenario-one-install-docker-desktop), Docker Compose is already included.
+
+### Start Airbyte
+
+If this is your first time using Airbyte, we suggest following the [Quickstart Guide](https://github.com/airbytehq/airbyte/tree/e378d40236b6a34e1c1cb481c8952735ec687d88/docs/quickstart/getting-started.md). When creating Airbyte connections intended to be orchestrated with Kestra, set your Connection's **sync frequency** to **manual**. Kestra will automate triggering Airbyte jobs in response to external events or based on a schedule you’ll provide.
+
+### Install Kestra
+
+If you haven’t started Kestra yet, download [the Docker Compose file](https://raw.githubusercontent.com/kestra-io/kestra/develop/docker-compose.yml):
+
+```yaml
+curl -o docker-compose.yml https://raw.githubusercontent.com/kestra-io/kestra/develop/docker-compose.yml
+```
+
+Then, run `docker compose up -d` and [navigate to the UI](http://localhost:8080/). You can start [building your first flows](https://kestra.io/docs/getting-started) using the integrated code editor in the UI.
+
+![airbyte_kestra_CLI](../.gitbook/assets/airbyte_kestra_1.gif)
+
+
+## 2. Create a flow from the UI
+
+Kestra UI provides a wide range of Blueprints to help you get started. 
+
+Navigate to Blueprints. Then type "Airbyte" in the search bar to find the desired integration. This way, you can easily accomplish fairly standardized data orchestration tasks, such as the following:
+
+1. [Run a single Airbyte sync](https://demo.kestra.io/ui/blueprints/community/61) on a schedule
+2. [Run multiple Airbyte syncs in parallel](https://demo.kestra.io/ui/blueprints/community/18)
+3. [Run multiple Airbyte syncs in parallel, then clone a Git repository with dbt code and trigger dbt CLI commands](https://demo.kestra.io/ui/blueprints/community/30)
+4. [Run a single Airbyte Cloud sync](https://demo.kestra.io/ui/blueprints/community/62) on a schedule
+5. [Run multiple Airbyte Cloud syncs in parallel](https://demo.kestra.io/ui/blueprints/community/63)
+6. [Run multiple Airbyte Cloud syncs in parallel, then clone a Git repository with dbt code and trigger dbt CLI commands](https://demo.kestra.io/ui/blueprints/community/64)
+7. [Run multiple Airbyte Cloud syncs in parallel, then run a dbt Cloud job](https://demo.kestra.io/ui/blueprints/community/31)
+
+Select a blueprint matching your use case and click "Use".
+
+![airbyte_kestra_UI](../.gitbook/assets/airbyte_kestra_2.gif)
+
+
+Then, within the editor, adjust the connection ID and task names and click "Save". Finally, trigger your flow. 
+
+## 3. Simple demo
+
+Here is an example flow that triggers multiple Airbyte connections in parallel to sync data for multiple **Pokémon**. 
+
+```yaml
+id: airbyteSyncs
+namespace: dev
+description: Gotta catch ‘em all!
+
+tasks:
+  - id: data-ingestion
+    type: io.kestra.core.tasks.flows.Parallel
+    tasks:
+      - id: charizard
+        type: io.kestra.plugin.airbyte.connections.Sync
+        connectionId: 9bb96539-73e7-4b9a-9937-6ce861b49cb9
+      - id: pikachu
+        type: io.kestra.plugin.airbyte.connections.Sync
+        connectionId: 39c38950-b0b9-4fce-a303-06ced3dbfa75
+      - id: psyduck
+        type: io.kestra.plugin.airbyte.connections.Sync
+        connectionId: 4de8ab1e-50ef-4df0-aa01-7f21491081f1
+
+taskDefaults:
+  - type: io.kestra.plugin.airbyte.connections.Sync
+    values:
+      url: http://host.docker.internal:8000/
+      username: "{{envs.airbyte_username}}"
+      password: "{{envs.airbyte_password}}"
+
+triggers:
+  - id: everyMinute
+    type: io.kestra.core.models.triggers.types.Schedule
+    cron: "*/1 * * * *"      
+```
+
+## Next steps
+
+If you liked that demo, check out [the blog post](https://airbyte.com/blog/everything-as-code-for-data-infrastructure-with-airbyte-and-kestra-terraform-providers) about using Airbyte and Kestra Terraform providers together to manage Everything as Code.
+
+If you encounter anything unexpected while reproducing this tutorial, you can open [a GitHub issue](https://github.com/kestra-io/kestra) or [ask via Kestra Community Slack](https://kestra.io/slack). Lastly, give Kestra [a GitHub star](https://github.com/kestra-io/kestra) if you like the project.
diff --git a/docs/contributing-to-airbyte/code-of-conduct.md b/docs/project-overview/code-of-conduct.md
similarity index 100%
rename from docs/contributing-to-airbyte/code-of-conduct.md
rename to docs/project-overview/code-of-conduct.md
diff --git a/docs/quickstart/deploy-airbyte.md b/docs/quickstart/deploy-airbyte.md
index 330ebd8ba225..4df34e9aa05a 100644
--- a/docs/quickstart/deploy-airbyte.md
+++ b/docs/quickstart/deploy-airbyte.md
@@ -15,11 +15,11 @@ Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000]
 
 Alternatively, if you have an Airbyte Cloud invite, just follow [these steps.](../deploying-airbyte/on-cloud.md)
 
-If you need direct access to our team for any kind of assistance, don't hesitate to [talk to our team](https://airbyte.com/talk-to-sales-premium-support) to discuss about our premium support offers."
+If you need direct access to our team for any kind of assistance, don't hesitate to [talk to our team](https://airbyte.com/talk-to-sales-premium-support) to discuss about our premium support offers.
 
 ## FAQ
 
-If you have any questions about the Airbyte Open-Source setup and deployment process, head over to our [Getting Started FAQ](https://discuss.airbyte.io/c/faq/15) on our Discourse that answers the following questions and more:
+If you have any questions about the Airbyte Open-Source setup and deployment process, head over to our [Getting Started FAQ](https://github.com/airbytehq/airbyte/discussions/categories/questions) on our Airbyte Forum that answers the following questions and more:
 
 - How long does it take to set up Airbyte?
 - Where can I see my data once I've run a sync?
diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html
index 62089cd1e2ff..0a7a2d70b2d0 100644
--- a/docs/reference/api/generated-api-html/index.html
+++ b/docs/reference/api/generated-api-html/index.html
@@ -182,6 +182,18 @@
   
    Airbyte Configuration API
    
     
    Airbyte Configuration API
 https://airbyte.io.
    
+
    The Configuration API is an internal Airbyte API that is designed for communications between different Airbyte components.
    
+
      
+
    • It's main purpose is to enable the Airbyte Engineering team to configure the internal state of Airbyte Cloud
      • 
+
    • It is also sometimes used by OSS users to configure their own Self-Hosted Airbyte deployment (internal state, etc)
      • 
+
    
+
    WARNING
    
+
      
+
    • Airbyte does NOT have active commitments to support this API long-term.
      • 
+
    • OSS users can utilize the Configuration API, but at their own risk.
      • 
+
    • This API is utilized internally by the Airbyte Engineering team and may be modified in the future if the need arises.
      • 
+
    • Modifications by the Airbyte Engineering team could create breaking changes and OSS users would need to update their code to catch up to any backwards incompatible changes in the API.
      • 
+
    
 
    This API is a collection of HTTP RPC-style methods. While it is not a REST API, those familiar with REST should find the conventions of this API recognizable.
    
 
    Here are some conventions that this API follows:
    
 
+  
    ConnectorBuilderProject
    
+  
+  
    DeclarativeSourceDefinitions
    
+  
   
    Destination
    
   
@@ -258,6 +288,7 @@ 
    DestinationDefinition
    
   
    DestinationDefinitionSpecification
    
   
   
    DestinationOauth
    
   
@@ -317,6 +349,7 @@ 
    Scheduler
    
   
   
    Source
    
   
+  
    StreamStatuses
    
+  
+  
    Streams
    
+  
   
    WebBackend
    
   
      
   
    • post /v1/web_backend/state/get_type
      • 
@@ -593,6 +641,7 @@ 
      Example data
      
   },
   "breakingChange" : true,
   "notifySchemaChanges" : true,
+  "notifySchemaChangesByEmail" : true,
   "name" : "name",
   "syncCatalog" : {
     "streams" : [ {
@@ -653,7 +702,8 @@ 
      Example data
      
       "units" : 6,
       "timeUnit" : "minutes"
     }
-  }
+  },
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
 
     
      Produces
      
@@ -772,6 +822,7 @@ 
      Example data
      
   },
   "breakingChange" : true,
   "notifySchemaChanges" : true,
+  "notifySchemaChangesByEmail" : true,
   "name" : "name",
   "syncCatalog" : {
     "streams" : [ {
@@ -832,7 +883,8 @@ 
      Example data
      
       "units" : 6,
       "timeUnit" : "minutes"
     }
-  }
+  },
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
 
     
      Produces
      
@@ -907,6 +959,7 @@ 
      Example data
      
     },
     "breakingChange" : true,
     "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
     "name" : "name",
     "syncCatalog" : {
       "streams" : [ {
@@ -967,7 +1020,8 @@ 
      Example data
      
         "units" : 6,
         "timeUnit" : "minutes"
       }
-    }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
     "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
@@ -985,6 +1039,7 @@ 
      Example data
      
     },
     "breakingChange" : true,
     "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
     "name" : "name",
     "syncCatalog" : {
       "streams" : [ {
@@ -1045,7 +1100,8 @@ 
      Example data
      
         "units" : 6,
         "timeUnit" : "minutes"
       }
-    }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }
 
@@ -1068,12 +1124,12 @@ 
      422
      
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/connections/list
    
-    
    Returns all connections for a workspace. (listConnectionsForWorkspace)
    
-    
    List connections for workspace. Does not return deleted connections.
    
+    
    post /v1/connections/list_by_actor_definition
    
+    
    List all connections that use the provided actor definition (listConnectionsByActorDefinition)
    
+    
    
 
 
     
    Consumes
    
@@ -1084,7 +1140,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
+      
    ActorDefinitionRequestBody ActorDefinitionRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -1121,6 +1177,7 @@ 
    Example data
    
     },
     "breakingChange" : true,
     "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
     "name" : "name",
     "syncCatalog" : {
       "streams" : [ {
@@ -1181,7 +1238,8 @@ 
    Example data
    
         "units" : 6,
         "timeUnit" : "minutes"
       }
-    }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
     "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
@@ -1199,6 +1257,7 @@ 
    Example data
    
     },
     "breakingChange" : true,
     "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
     "name" : "name",
     "syncCatalog" : {
       "streams" : [ {
@@ -1259,199 +1318,8 @@ 
    Example data
    
         "units" : 6,
         "timeUnit" : "minutes"
       }
-    }
-  } ]
-}
-
-    
    Produces
    
-    This API call produces the following media types according to the Accept request header;
-    the media type will be conveyed by the Content-Type response header.
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Responses
    
-    
    200
    
-    Successful operation
-        ConnectionReadList
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
-  
     
-  
    
-  
    
-    
    
-    Up
-    
    post /v1/connections/reset
    
-    
    Reset the data for the connection. Deletes data generated by the connection in the destination. Resets any cursors back to initial state. (resetConnection)
    
-    
    
-
-
-    
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Request body
    
-    
    
-      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
-
-      
    Body Parameter —  
    
-
-    
      
-
-
-
-
-    
    Return type
    
-    
    
-      JobInfoRead
-      
-    
    
-
-    
-
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "job" : {
-    "createdAt" : 6,
-    "configId" : "configId",
-    "startedAt" : 5,
-    "id" : 0,
-    "resetConfig" : {
-      "streamsToReset" : [ {
-        "name" : "name",
-        "namespace" : "namespace"
-      }, {
-        "name" : "name",
-        "namespace" : "namespace"
-      } ]
-    },
-    "updatedAt" : 1
-  },
-  "attempts" : [ {
-    "attempt" : {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  }, {
-    "attempt" : {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
     },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }
    
 
@@ -1465,7 +1333,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        JobInfoRead
+        ConnectionReadList
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -1474,12 +1342,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/connections/search
    
-    
    Search connections (searchConnections)
    
-    
    
+    
    post /v1/connections/list
    
+    
    Returns all connections for a workspace. (listConnectionsForWorkspace)
    
+    
    List connections for workspace. Does not return deleted connections.
    
 
 
     
    Consumes
    
@@ -1490,7 +1358,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionSearch ConnectionSearch (required)
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -1527,6 +1395,7 @@ 
    Example data
    
     },
     "breakingChange" : true,
     "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
     "name" : "name",
     "syncCatalog" : {
       "streams" : [ {
@@ -1587,7 +1456,8 @@ 
    Example data
    
         "units" : 6,
         "timeUnit" : "minutes"
       }
-    }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
     "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
@@ -1605,6 +1475,7 @@ 
    Example data
    
     },
     "breakingChange" : true,
     "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
     "name" : "name",
     "syncCatalog" : {
       "streams" : [ {
@@ -1665,7 +1536,8 @@ 
    Example data
    
         "units" : 6,
         "timeUnit" : "minutes"
       }
-    }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }
 
@@ -1680,16 +1552,19 @@ 
    Responses
    
     
    200
    
     Successful operation
         ConnectionReadList
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/connections/sync
    
-    
    Trigger a manual sync of the connection (syncConnection)
    
+    
    post /v1/connections/reset
    
+    
    Reset the data for the connection. Deletes data generated by the connection in the destination. Resets any cursors back to initial state. (resetConnection)
    
     
    
 
 
@@ -1723,6 +1598,13 @@ 
    Example data
    
     
    {
   "job" : {
     "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
     "configId" : "configId",
     "startedAt" : 5,
     "id" : 0,
@@ -1743,9 +1625,10 @@ 
    Example data
    
         "stateMessagesEmitted" : 1,
         "recordsCommitted" : 1,
         "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
       },
       "failureSummary" : {
         "failures" : [ {
@@ -1753,13 +1636,13 @@ 
    Example data
    
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         }, {
           "retryable" : true,
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         } ],
         "partialSuccess" : true
       },
@@ -1771,9 +1654,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -1782,9 +1666,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -1802,9 +1687,10 @@ 
    Example data
    
         "stateMessagesEmitted" : 1,
         "recordsCommitted" : 1,
         "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
       },
       "failureSummary" : {
         "failures" : [ {
@@ -1812,13 +1698,13 @@ 
    Example data
    
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         }, {
           "retryable" : true,
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         } ],
         "partialSuccess" : true
       },
@@ -1830,9 +1716,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -1841,9 +1728,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -1877,15 +1765,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/connections/update
    
-    
    Update a connection (updateConnection)
    
-    
    Apply a patch-style update to a connection. Only fields present on the update request body will be updated.
-Note that if a catalog is present in the request body, the connection's entire catalog will be replaced
-with the catalog from the request. This means that to modify a single stream, the entire new catalog
-containing the updated stream needs to be sent.
    
+    
    post /v1/connections/reset/stream
    
+    
    Reset the data for a specific stream in the connection. Deletes data generated by the stream in the destination. Resets any cursors back to initial state. (resetConnectionStream)
    
+    
    
 
 
     
    Consumes
    
@@ -1896,7 +1781,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionUpdate ConnectionUpdate (required)
    
+      
    ConnectionStreamRequestBody ConnectionStreamRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -1907,7 +1792,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      ConnectionRead
+      JobInfoRead
       
     
    
 
@@ -1916,151 +1801,154 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "prefix" : "prefix",
-  "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "resourceRequirements" : {
-    "cpu_limit" : "cpu_limit",
-    "memory_request" : "memory_request",
-    "memory_limit" : "memory_limit",
-    "cpu_request" : "cpu_request"
-  },
-  "schedule" : {
-    "units" : 0,
-    "timeUnit" : "minutes"
-  },
-  "breakingChange" : true,
-  "notifySchemaChanges" : true,
-  "name" : "name",
-  "syncCatalog" : {
-    "streams" : [ {
-      "stream" : {
-        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
-        "supportedSyncModes" : [ null, null ],
-        "sourceDefinedCursor" : true,
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
         "name" : "name",
-        "namespace" : "namespace",
-        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  },
+  "attempts" : [ {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
       },
-      "config" : {
-        "aliasName" : "aliasName",
-        "suggested" : true,
-        "fieldSelectionEnabled" : true,
-        "selectedFields" : [ {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
         }, {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
         } ],
-        "cursorField" : [ "cursorField", "cursorField" ],
-        "selected" : true,
-        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
-      }
-    }, {
-      "stream" : {
-        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
-        "supportedSyncModes" : [ null, null ],
-        "sourceDefinedCursor" : true,
-        "name" : "name",
-        "namespace" : "namespace",
-        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+        "partialSuccess" : true
       },
-      "config" : {
-        "aliasName" : "aliasName",
-        "suggested" : true,
-        "fieldSelectionEnabled" : true,
-        "selectedFields" : [ {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  }, {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
         }, {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
         } ],
-        "cursorField" : [ "cursorField", "cursorField" ],
-        "selected" : true,
-        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
-      }
-    } ]
-  },
-  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "namespaceFormat" : "${SOURCE_NAMESPACE}",
-  "operationIds" : [ null, null ],
-  "scheduleData" : {
-    "cron" : {
-      "cronExpression" : "cronExpression",
-      "cronTimeZone" : "cronTimeZone"
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
     },
-    "basicSchedule" : {
-      "units" : 6,
-      "timeUnit" : "minutes"
-    }
-  }
-}
    
-
-    
    Produces
    
-    This API call produces the following media types according to the Accept request header;
-    the media type will be conveyed by the Content-Type response header.
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Responses
    
-    
    200
    
-    Successful operation
-        ConnectionRead
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
-  
     
-  
    
-  
    Destination
    
-  
    
-    
    
-    Up
-    
    post /v1/destinations/check_connection
    
-    
    Check connection to the destination (checkConnectionToDestination)
    
-    
    
-
-
-    
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Request body
    
-    
    
-      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
-
-      
    Body Parameter —  
    
-
-    
      
-
-
-
-
-    
    Return type
    
-    
    
-      CheckConnectionRead
-      
-    
    
-
-    
-
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "message" : "message",
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "logs" : {
       "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
-  },
-  "status" : "succeeded"
+    }
+  } ]
 }
    
 
     
    Produces
    
@@ -2073,7 +1961,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        CheckConnectionRead
+        JobInfoRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -2082,11 +1970,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/check_connection_for_update
    
-    
    Check connection for a proposed update to a destination (checkConnectionToDestinationForUpdate)
    
+    
    post /v1/connections/search
    
+    
    Search connections (searchConnections)
    
     
    
 
 
@@ -2098,7 +1986,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationUpdate DestinationUpdate (required)
    
+      
    ConnectionSearch ConnectionSearch (required)
    
 
       
    Body Parameter —  
    
 
@@ -2109,7 +1997,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      CheckConnectionRead
+      ConnectionReadList
       
     
    
 
@@ -2118,14 +2006,1125 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "message" : "message",
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
+  "connections" : [ {
+    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "prefix" : "prefix",
+    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "resourceRequirements" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "schedule" : {
+      "units" : 0,
+      "timeUnit" : "minutes"
+    },
+    "breakingChange" : true,
+    "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
+    "name" : "name",
+    "syncCatalog" : {
+      "streams" : [ {
+        "stream" : {
+          "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+          "supportedSyncModes" : [ null, null ],
+          "sourceDefinedCursor" : true,
+          "name" : "name",
+          "namespace" : "namespace",
+          "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+        },
+        "config" : {
+          "aliasName" : "aliasName",
+          "suggested" : true,
+          "fieldSelectionEnabled" : true,
+          "selectedFields" : [ {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          }, {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          } ],
+          "cursorField" : [ "cursorField", "cursorField" ],
+          "selected" : true,
+          "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+        }
+      }, {
+        "stream" : {
+          "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+          "supportedSyncModes" : [ null, null ],
+          "sourceDefinedCursor" : true,
+          "name" : "name",
+          "namespace" : "namespace",
+          "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+        },
+        "config" : {
+          "aliasName" : "aliasName",
+          "suggested" : true,
+          "fieldSelectionEnabled" : true,
+          "selectedFields" : [ {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          }, {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          } ],
+          "cursorField" : [ "cursorField", "cursorField" ],
+          "selected" : true,
+          "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+        }
+      } ]
+    },
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "namespaceFormat" : "${SOURCE_NAMESPACE}",
+    "operationIds" : [ null, null ],
+    "scheduleData" : {
+      "cron" : {
+        "cronExpression" : "cronExpression",
+        "cronTimeZone" : "cronTimeZone"
+      },
+      "basicSchedule" : {
+        "units" : 6,
+        "timeUnit" : "minutes"
+      }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "prefix" : "prefix",
+    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "resourceRequirements" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "schedule" : {
+      "units" : 0,
+      "timeUnit" : "minutes"
+    },
+    "breakingChange" : true,
+    "notifySchemaChanges" : true,
+    "notifySchemaChangesByEmail" : true,
+    "name" : "name",
+    "syncCatalog" : {
+      "streams" : [ {
+        "stream" : {
+          "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+          "supportedSyncModes" : [ null, null ],
+          "sourceDefinedCursor" : true,
+          "name" : "name",
+          "namespace" : "namespace",
+          "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+        },
+        "config" : {
+          "aliasName" : "aliasName",
+          "suggested" : true,
+          "fieldSelectionEnabled" : true,
+          "selectedFields" : [ {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          }, {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          } ],
+          "cursorField" : [ "cursorField", "cursorField" ],
+          "selected" : true,
+          "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+        }
+      }, {
+        "stream" : {
+          "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+          "supportedSyncModes" : [ null, null ],
+          "sourceDefinedCursor" : true,
+          "name" : "name",
+          "namespace" : "namespace",
+          "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+        },
+        "config" : {
+          "aliasName" : "aliasName",
+          "suggested" : true,
+          "fieldSelectionEnabled" : true,
+          "selectedFields" : [ {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          }, {
+            "fieldPath" : [ "fieldPath", "fieldPath" ]
+          } ],
+          "cursorField" : [ "cursorField", "cursorField" ],
+          "selected" : true,
+          "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+        }
+      } ]
+    },
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "namespaceFormat" : "${SOURCE_NAMESPACE}",
+    "operationIds" : [ null, null ],
+    "scheduleData" : {
+      "cron" : {
+        "cronExpression" : "cronExpression",
+        "cronTimeZone" : "cronTimeZone"
+      },
+      "basicSchedule" : {
+        "units" : 6,
+        "timeUnit" : "minutes"
+      }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        ConnectionReadList
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connections/sync
    
+    
    Trigger a manual sync of the connection (syncConnection)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      JobInfoRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  },
+  "attempts" : [ {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  }, {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  } ]
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        JobInfoRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connections/update
    
+    
    Update a connection (updateConnection)
    
+    
    Apply a patch-style update to a connection. Only fields present on the update request body will be updated.
+Note that if a catalog is present in the request body, the connection's entire catalog will be replaced
+with the catalog from the request. This means that to modify a single stream, the entire new catalog
+containing the updated stream needs to be sent.
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectionUpdate ConnectionUpdate (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      ConnectionRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "sourceCatalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "prefix" : "prefix",
+  "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "resourceRequirements" : {
+    "cpu_limit" : "cpu_limit",
+    "memory_request" : "memory_request",
+    "memory_limit" : "memory_limit",
+    "cpu_request" : "cpu_request"
+  },
+  "schedule" : {
+    "units" : 0,
+    "timeUnit" : "minutes"
+  },
+  "breakingChange" : true,
+  "notifySchemaChanges" : true,
+  "notifySchemaChangesByEmail" : true,
+  "name" : "name",
+  "syncCatalog" : {
+    "streams" : [ {
+      "stream" : {
+        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+        "supportedSyncModes" : [ null, null ],
+        "sourceDefinedCursor" : true,
+        "name" : "name",
+        "namespace" : "namespace",
+        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+      },
+      "config" : {
+        "aliasName" : "aliasName",
+        "suggested" : true,
+        "fieldSelectionEnabled" : true,
+        "selectedFields" : [ {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        }, {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        } ],
+        "cursorField" : [ "cursorField", "cursorField" ],
+        "selected" : true,
+        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+      }
+    }, {
+      "stream" : {
+        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+        "supportedSyncModes" : [ null, null ],
+        "sourceDefinedCursor" : true,
+        "name" : "name",
+        "namespace" : "namespace",
+        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+      },
+      "config" : {
+        "aliasName" : "aliasName",
+        "suggested" : true,
+        "fieldSelectionEnabled" : true,
+        "selectedFields" : [ {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        }, {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        } ],
+        "cursorField" : [ "cursorField", "cursorField" ],
+        "selected" : true,
+        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+      }
+    } ]
+  },
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "namespaceFormat" : "${SOURCE_NAMESPACE}",
+  "operationIds" : [ null, null ],
+  "scheduleData" : {
+    "cron" : {
+      "cronExpression" : "cronExpression",
+      "cronTimeZone" : "cronTimeZone"
+    },
+    "basicSchedule" : {
+      "units" : 6,
+      "timeUnit" : "minutes"
+    }
+  },
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        ConnectionRead
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    ConnectorBuilderProject
    
+  
    
+    
    
+    Up
+    
    post /v1/connector_builder_projects/create
    
+    
    Create new connector builder project (createConnectorBuilderProject)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectorBuilderProjectWithWorkspaceId ConnectorBuilderProjectWithWorkspaceId (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      ConnectorBuilderProjectIdWithWorkspaceId
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "builderProjectId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "version" : 0,
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    201
    
+    Successful operation
+        ConnectorBuilderProjectIdWithWorkspaceId
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connector_builder_projects/delete
    
+    
    Deletes connector builder project (deleteConnectorBuilderProject)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectorBuilderProjectIdWithWorkspaceId ConnectorBuilderProjectIdWithWorkspaceId (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+
+    
    Responses
    
+    
    204
    
+    Successful operation
+        
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connector_builder_projects/get_with_manifest
    
+    
    Get a connector builder project with draft manifest (getConnectorBuilderProject)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectorBuilderProjectIdWithWorkspaceId ConnectorBuilderProjectIdWithWorkspaceId (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      ConnectorBuilderProjectRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "declarativeManifest" : {
+    "manifest" : "{}",
+    "isDraft" : true,
+    "description" : "description",
+    "version" : 0
+  },
+  "builderProject" : {
+    "hasDraft" : true,
+    "activeDeclarativeManifestVersion" : 0,
+    "builderProjectId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "name" : "name",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        ConnectorBuilderProjectRead
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connector_builder_projects/list
    
+    
    List connector builder projects for workspace (listConnectorBuilderProjects)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      ConnectorBuilderProjectReadList
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "projects" : [ {
+    "hasDraft" : true,
+    "activeDeclarativeManifestVersion" : 0,
+    "builderProjectId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "name" : "name",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "hasDraft" : true,
+    "activeDeclarativeManifestVersion" : 0,
+    "builderProjectId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "name" : "name",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        ConnectorBuilderProjectReadList
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connector_builder_projects/publish
    
+    
    Publish a connector to the workspace (publishConnectorBuilderProject)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectorBuilderPublishRequestBody ConnectorBuilderPublishRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      SourceDefinitionIdBody
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        SourceDefinitionIdBody
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/connector_builder_projects/update
    
+    
    Update connector builder project (updateConnectorBuilderProject)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ExistingConnectorBuilderProjectWithWorkspaceId ExistingConnectorBuilderProjectWithWorkspaceId (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+
+    
    Responses
    
+    
    204
    
+    Successful operation
+        
+  
     
+  
    
+  
    DeclarativeSourceDefinitions
    
+  
    
+    
    
+    Up
+    
    post /v1/declarative_source_definitions/create_manifest
    
+    
    Create a declarative manifest to be used by the specified source definition (createDeclarativeSourceDefinitionManifest)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DeclarativeSourceDefinitionCreateManifestRequestBody DeclarativeSourceDefinitionCreateManifestRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    201
    
+    Successful operation
+        
+    
    400
    
+    Definition is not declarative source
+        
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    409
    
+    Version already exists for definition id
+        
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/declarative_source_definitions/list_manifests
    
+    
    List all available declarative manifest versions of a declarative source definition (listDeclarativeManifests)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ListDeclarativeManifestsRequestBody ListDeclarativeManifestsRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      DeclarativeManifestsReadList
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "manifestVersions" : [ {
+    "description" : "description",
+    "isActive" : true,
+    "version" : 0
+  }, {
+    "description" : "description",
+    "isActive" : true,
+    "version" : 0
+  } ]
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        DeclarativeManifestsReadList
+    
    400
    
+    Definition is not declarative source
+        
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/declarative_source_definitions/update_active_manifest
    
+    
    Update the declarative manifest version for a source (updateDeclarativeManifestVersion)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    UpdateActiveManifestRequestBody UpdateActiveManifestRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    204
    
+    Successful operation
+        
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+  
     
+  
    
+  
    Destination
    
+  
    
+    
    
+    Up
+    
    post /v1/destinations/check_connection
    
+    
    Check connection to the destination (checkConnectionToDestination)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      CheckConnectionRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "message" : "message",
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
+    },
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    },
+    "succeeded" : true
+  },
+  "status" : "succeeded"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        CheckConnectionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destinations/check_connection_for_update
    
+    
    Check connection for a proposed update to a destination (checkConnectionToDestinationForUpdate)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationUpdate DestinationUpdate (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      CheckConnectionRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "message" : "message",
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
+    },
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
       "logLines" : [ "logLines", "logLines" ]
     },
     "succeeded" : true
@@ -2141,9 +3140,186 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        CheckConnectionRead
+    
    200
    
+    Successful operation
+        CheckConnectionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destinations/clone
    
+    
    Clone destination (cloneDestination)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationCloneRequestBody DestinationCloneRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      DestinationRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "connectionConfiguration" : {
+    "user" : "charles"
+  },
+  "destinationName" : "destinationName",
+  "name" : "name",
+  "icon" : "icon",
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        DestinationRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destinations/create
    
+    
    Create a destination (createDestination)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationCreate DestinationCreate (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      DestinationRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "connectionConfiguration" : {
+    "user" : "charles"
+  },
+  "destinationName" : "destinationName",
+  "name" : "name",
+  "icon" : "icon",
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        DestinationRead
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destinations/delete
    
+    
    Delete the destination (deleteDestination)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    204
    
+    The resource was deleted successfully.
+        
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -2152,11 +3328,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/clone
    
-    
    Clone destination (cloneDestination)
    
+    
    post /v1/destinations/get
    
+    
    Get configured destination (getDestination)
    
     
    
 
 
@@ -2168,7 +3344,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationCloneRequestBody DestinationCloneRequestBody (required)
    
+      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -2218,11 +3394,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/create
    
-    
    Create a destination (createDestination)
    
+    
    post /v1/destinations/list
    
+    
    List configured destinations for a workspace (listDestinationsForWorkspace)
    
     
    
 
 
@@ -2234,7 +3410,85 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationCreate DestinationCreate (required)
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      DestinationReadList
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "destinations" : [ {
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "destinationName" : "destinationName",
+    "name" : "name",
+    "icon" : "icon",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "destinationName" : "destinationName",
+    "name" : "name",
+    "icon" : "icon",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        DestinationReadList
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destinations/partial_update
    
+    
    Update a destination partially (partialUpdateDestination)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    PartialDestinationUpdate PartialDestinationUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -2281,11 +3535,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/delete
    
-    
    Delete the destination (deleteDestination)
    
+    
    post /v1/destinations/search
    
+    
    Search destinations (searchDestinations)
    
     
    
 
 
@@ -2297,7 +3551,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
+      
    DestinationSearch DestinationSearch (required)
    
 
       
    Body Parameter —  
    
 
@@ -2306,9 +3560,39 @@ 
    Request body
    
 
 
 
+    
    Return type
    
+    
    
+      DestinationReadList
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "destinations" : [ {
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "destinationName" : "destinationName",
+    "name" : "name",
+    "icon" : "icon",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "destinationName" : "destinationName",
+    "name" : "name",
+    "icon" : "icon",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -2318,22 +3602,19 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    204
    
-    The resource was deleted successfully.
-        
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+    
    200
    
+    Successful operation
+        DestinationReadList
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/get
    
-    
    Get configured destination (getDestination)
    
+    
    post /v1/destinations/update
    
+    
    Update a destination (updateDestination)
    
     
    
 
 
@@ -2345,7 +3626,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
+      
    DestinationUpdate DestinationUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -2387,19 +3668,109 @@ 
    Responses
    
     
    200
    
     Successful operation
         DestinationRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    DestinationDefinition
    
+  
    
+    
    
+    Up
+    
    post /v1/destination_definitions/create_custom
    
+    
    Creates a custom destinationDefinition for the given workspace (createCustomDestinationDefinition)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    CustomDestinationDefinitionCreate CustomDestinationDefinitionCreate (optional)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      DestinationDefinitionRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "supportsDbt" : true,
+  "name" : "name",
+  "icon" : "icon",
+  "normalizationConfig" : {
+    "normalizationIntegrationType" : "normalizationIntegrationType",
+    "normalizationRepository" : "normalizationRepository",
+    "normalizationTag" : "normalizationTag",
+    "supported" : false
+  },
+  "protocolVersion" : "protocolVersion",
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        DestinationDefinitionRead
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/list
    
-    
    List configured destinations for a workspace (listDestinationsForWorkspace)
    
+    
    post /v1/destination_definitions/delete
    
+    
    Delete a destination definition (deleteDestinationDefinition)
    
     
    
 
 
@@ -2411,7 +3782,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
+      
    DestinationDefinitionIdRequestBody DestinationDefinitionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -2420,39 +3791,9 @@ 
    Request body
    
 
 
 
-    
    Return type
    
-    
    
-      DestinationReadList
-      
-    
    
 
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "destinations" : [ {
-    "connectionConfiguration" : {
-      "user" : "charles"
-    },
-    "destinationName" : "destinationName",
-    "name" : "name",
-    "icon" : "icon",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "connectionConfiguration" : {
-      "user" : "charles"
-    },
-    "destinationName" : "destinationName",
-    "name" : "name",
-    "icon" : "icon",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -2462,9 +3803,9 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        DestinationReadList
+    
    204
    
+    The resource was deleted successfully.
+        
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -2473,11 +3814,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/search
    
-    
    Search destinations (searchDestinations)
    
+    
    post /v1/destination_definitions/get
    
+    
    Get destinationDefinition (getDestinationDefinition)
    
     
    
 
 
@@ -2489,7 +3830,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationSearch DestinationSearch (required)
    
+      
    DestinationDefinitionIdRequestBody DestinationDefinitionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -2500,7 +3841,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationReadList
+      DestinationDefinitionRead
       
     
    
 
@@ -2509,27 +3850,44 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "destinations" : [ {
-    "connectionConfiguration" : {
-      "user" : "charles"
-    },
-    "destinationName" : "destinationName",
-    "name" : "name",
-    "icon" : "icon",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "connectionConfiguration" : {
-      "user" : "charles"
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
     },
-    "destinationName" : "destinationName",
-    "name" : "name",
-    "icon" : "icon",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "supportsDbt" : true,
+  "name" : "name",
+  "icon" : "icon",
+  "normalizationConfig" : {
+    "normalizationIntegrationType" : "normalizationIntegrationType",
+    "normalizationRepository" : "normalizationRepository",
+    "normalizationTag" : "normalizationTag",
+    "supported" : false
+  },
+  "protocolVersion" : "protocolVersion",
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -2542,17 +3900,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationReadList
+        DestinationDefinitionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destinations/update
    
-    
    Update a destination (updateDestination)
    
+    
    post /v1/destination_definitions/get_for_workspace
    
+    
    Get a destinationDefinition that is configured for the given workspace (getDestinationDefinitionForWorkspace)
    
     
    
 
 
@@ -2564,7 +3925,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationUpdate DestinationUpdate (required)
    
+      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
 
       
    Body Parameter —  
    
 
@@ -2575,7 +3936,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationRead
+      DestinationDefinitionRead
       
     
    
 
@@ -2584,15 +3945,44 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "connectionConfiguration" : {
-    "user" : "charles"
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
   },
-  "destinationName" : "destinationName",
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "supportsDbt" : true,
   "name" : "name",
   "icon" : "icon",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "normalizationConfig" : {
+    "normalizationIntegrationType" : "normalizationIntegrationType",
+    "normalizationRepository" : "normalizationRepository",
+    "normalizationTag" : "normalizationTag",
+    "supported" : false
+  },
+  "protocolVersion" : "protocolVersion",
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -2605,18 +3995,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationRead
+        DestinationDefinitionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    DestinationDefinition
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/create_custom
    
-    
    Creates a custom destinationDefinition for the given workspace (createCustomDestinationDefinition)
    
+    
    post /v1/destination_definitions/grant_definition
    
+    
    grant a private, non-custom destinationDefinition to a given workspace (grantDestinationDefinitionToWorkspace)
    
     
    
 
 
@@ -2628,7 +4020,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    CustomDestinationDefinitionCreate CustomDestinationDefinitionCreate (optional)
    
+      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
 
       
    Body Parameter —  
    
 
@@ -2639,7 +4031,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationDefinitionRead
+      PrivateDestinationDefinitionRead
       
     
    
 
@@ -2648,44 +4040,47 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      }
-    }, {
-      "resourceRequirements" : {
+  "destinationDefinition" : {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    } ]
-  },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "supportsDbt" : true,
-  "name" : "name",
-  "icon" : "icon",
-  "normalizationConfig" : {
-    "normalizationIntegrationType" : "normalizationIntegrationType",
-    "normalizationRepository" : "normalizationRepository",
-    "normalizationTag" : "normalizationTag",
-    "supported" : false
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   },
-  "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "granted" : true
 }
    
 
     
    Produces
    
@@ -2698,40 +4093,119 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionRead
+        PrivateDestinationDefinitionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/delete
    
-    
    Delete a destination definition (deleteDestinationDefinition)
    
+    
    post /v1/destination_definitions/list
    
+    
    List all the destinationDefinitions the current Airbyte deployment is configured to use (listDestinationDefinitions)
    
     
    
 
 
-    
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Request body
    
-    
    
-      
    DestinationDefinitionIdRequestBody DestinationDefinitionIdRequestBody (required)
    
-
-      
    Body Parameter —  
    
 
-    
      
 
 
 
 
+    
    Return type
    
+    
    
+      DestinationDefinitionReadList
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "destinationDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -2741,22 +4215,16 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    204
    
-    The resource was deleted successfully.
-        
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    
    200
    
+    Successful operation
+        DestinationDefinitionReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/get
    
-    
    Get destinationDefinition (getDestinationDefinition)
    
+    
    post /v1/destination_definitions/list_for_workspace
    
+    
    List all the destinationDefinitions the given workspace is configured to use (listDestinationDefinitionsForWorkspace)
    
     
    
 
 
@@ -2768,7 +4236,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationDefinitionIdRequestBody DestinationDefinitionIdRequestBody (required)
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -2779,7 +4247,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationDefinitionRead
+      DestinationDefinitionReadList
       
     
    
 
@@ -2788,44 +4256,85 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
+  "destinationDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    }, {
-      "resourceRequirements" : {
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    } ]
-  },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "supportsDbt" : true,
-  "name" : "name",
-  "icon" : "icon",
-  "normalizationConfig" : {
-    "normalizationIntegrationType" : "normalizationIntegrationType",
-    "normalizationRepository" : "normalizationRepository",
-    "normalizationTag" : "normalizationTag",
-    "supported" : false
-  },
-  "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
 }
    
 
     
    Produces
    
@@ -2838,43 +4347,25 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+        DestinationDefinitionReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/get_for_workspace
    
-    
    Get a destinationDefinition that is configured for the given workspace (getDestinationDefinitionForWorkspace)
    
-    
    
-
-
-    
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
+    
    post /v1/destination_definitions/list_latest
    
+    
    List the latest destinationDefinitions Airbyte supports (listLatestDestinationDefinitions)
    
+    
    Guaranteed to retrieve the latest information on supported destinations.
    
 
-    
    Request body
    
-    
    
-      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
 
-      
    Body Parameter —  
    
 
-    
      
 
 
 
 
     
    Return type
    
     
    
-      DestinationDefinitionRead
+      DestinationDefinitionReadList
       
     
    
 
@@ -2883,44 +4374,85 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
+  "destinationDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    }, {
-      "resourceRequirements" : {
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    } ]
-  },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "supportsDbt" : true,
-  "name" : "name",
-  "icon" : "icon",
-  "normalizationConfig" : {
-    "normalizationIntegrationType" : "normalizationIntegrationType",
-    "normalizationRepository" : "normalizationRepository",
-    "normalizationTag" : "normalizationTag",
-    "supported" : false
-  },
-  "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "dockerRepository" : "dockerRepository",
+    "supportsDbt" : true,
+    "name" : "name",
+    "icon" : "icon",
+    "normalizationConfig" : {
+      "normalizationIntegrationType" : "normalizationIntegrationType",
+      "normalizationRepository" : "normalizationRepository",
+      "normalizationTag" : "normalizationTag",
+      "supported" : false
+    },
+    "protocolVersion" : "protocolVersion",
+    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
 }
    
 
     
    Produces
    
@@ -2933,20 +4465,14 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+        DestinationDefinitionReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/grant_definition
    
-    
    grant a private, non-custom destinationDefinition to a given workspace (grantDestinationDefinitionToWorkspace)
    
+    
    post /v1/destination_definitions/list_private
    
+    
    List all private, non-custom destinationDefinitions, and for each indicate whether the given workspace has a grant for using the definition. Used by admins to view and modify a given workspace's grants. (listPrivateDestinationDefinitions)
    
     
    
 
 
@@ -2958,7 +4484,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -2969,7 +4495,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      PrivateDestinationDefinitionRead
+      PrivateDestinationDefinitionReadList
       
     
    
 
@@ -2978,47 +4504,91 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "destinationDefinition" : {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
+  "destinationDefinitions" : [ {
+    "destinationDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
           "cpu_limit" : "cpu_limit",
           "memory_request" : "memory_request",
           "memory_limit" : "memory_limit",
           "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
+      "documentationUrl" : "https://openapi-generator.tech",
+      "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
+      "supportsDbt" : true,
+      "name" : "name",
+      "icon" : "icon",
+      "normalizationConfig" : {
+        "normalizationIntegrationType" : "normalizationIntegrationType",
+        "normalizationRepository" : "normalizationRepository",
+        "normalizationTag" : "normalizationTag",
+        "supported" : false
+      },
+      "protocolVersion" : "protocolVersion",
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    },
+    "granted" : true
+  }, {
+    "destinationDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
           "cpu_limit" : "cpu_limit",
           "memory_request" : "memory_request",
           "memory_limit" : "memory_limit",
           "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
+      "documentationUrl" : "https://openapi-generator.tech",
+      "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
+      "supportsDbt" : true,
+      "name" : "name",
+      "icon" : "icon",
+      "normalizationConfig" : {
+        "normalizationIntegrationType" : "normalizationIntegrationType",
+        "normalizationRepository" : "normalizationRepository",
+        "normalizationTag" : "normalizationTag",
+        "supported" : false
+      },
+      "protocolVersion" : "protocolVersion",
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  },
-  "granted" : true
+    "granted" : true
+  } ]
 }
    
 
     
    Produces
    
@@ -3029,9 +4599,51 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        PrivateDestinationDefinitionRead
+    
    200
    
+    Successful operation
+        PrivateDestinationDefinitionReadList
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destination_definitions/revoke_definition
    
+    
    revoke a grant to a private, non-custom destinationDefinition from a given workspace (revokeDestinationDefinitionFromWorkspace)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    204
    
+    The resource was deleted successfully.
+        
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -3040,22 +4652,34 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/list
    
-    
    List all the destinationDefinitions the current Airbyte deployment is configured to use (listDestinationDefinitions)
    
+    
    post /v1/destination_definitions/update
    
+    
    Update destinationDefinition (updateDestinationDefinition)
    
     
    
 
 
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationDefinitionUpdate DestinationDefinitionUpdate (required)
    
+
+      
    Body Parameter —  
    
 
+    
      
 
 
 
 
     
    Return type
    
     
    
-      DestinationDefinitionReadList
+      DestinationDefinitionRead
       
     
    
 
@@ -3064,85 +4688,44 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "destinationDefinitions" : [ {
-    "resourceRequirements" : {
-      "default" : {
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
-    },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "resourceRequirements" : {
-      "default" : {
+      }
+    }, {
+      "resourceRequirements" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
-    },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+      }
+    } ]
+  },
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "dockerRepository" : "dockerRepository",
+  "supportsDbt" : true,
+  "name" : "name",
+  "icon" : "icon",
+  "normalizationConfig" : {
+    "normalizationIntegrationType" : "normalizationIntegrationType",
+    "normalizationRepository" : "normalizationRepository",
+    "normalizationTag" : "normalizationTag",
+    "supported" : false
+  },
+  "protocolVersion" : "protocolVersion",
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -3155,14 +4738,21 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionReadList
+        DestinationDefinitionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    DestinationDefinitionSpecification
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/list_for_workspace
    
-    
    List all the destinationDefinitions the given workspace is configured to use (listDestinationDefinitionsForWorkspace)
    
+    
    post /v1/destination_definition_specifications/get
    
+    
    Get specification for a destinationDefinition (getDestinationDefinitionSpecification)
    
     
    
 
 
@@ -3174,7 +4764,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
+      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
 
       
    Body Parameter —  
    
 
@@ -3185,7 +4775,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationDefinitionReadList
+      DestinationDefinitionSpecificationRead
       
     
    
 
@@ -3194,85 +4784,38 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "destinationDefinitions" : [ {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
-    },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
+  "documentationUrl" : "documentationUrl",
+  "connectionSpecification" : {
+    "user" : {
+      "type" : "string"
+    }
+  },
+  "supportedDestinationSyncModes" : [ null, null ],
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "advancedAuth" : {
+    "predicateValue" : "predicateValue",
+    "oauthConfigSpecification" : { },
+    "predicateKey" : [ "predicateKey", "predicateKey" ],
+    "authFlowType" : "oauth2.0"
+  },
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
     },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
     },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "succeeded" : true
+  }
 }
    
 
     
    Produces
    
@@ -3285,25 +4828,43 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionReadList
+        DestinationDefinitionSpecificationRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/list_latest
    
-    
    List the latest destinationDefinitions Airbyte supports (listLatestDestinationDefinitions)
    
-    
    Guaranteed to retrieve the latest information on supported destinations.
    
+    
    post /v1/destination_definition_specifications/get_for_destination
    
+    
    Get specification for a destination (getSpecificationForDestinationId)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
 
+    
    Request body
    
+    
    
+      
    DestinationIdRequestBody DestinationIdRequestBody (required)
    
 
+      
    Body Parameter —  
    
 
+    
      
 
 
 
 
     
    Return type
    
     
    
-      DestinationDefinitionReadList
+      DestinationDefinitionSpecificationRead
       
     
    
 
@@ -3312,85 +4873,38 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "destinationDefinitions" : [ {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
-    },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
+  "documentationUrl" : "documentationUrl",
+  "connectionSpecification" : {
+    "user" : {
+      "type" : "string"
+    }
+  },
+  "supportedDestinationSyncModes" : [ null, null ],
+  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "advancedAuth" : {
+    "predicateValue" : "predicateValue",
+    "oauthConfigSpecification" : { },
+    "predicateKey" : [ "predicateKey", "predicateKey" ],
+    "authFlowType" : "oauth2.0"
+  },
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
     },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "dockerRepository" : "dockerRepository",
-    "supportsDbt" : true,
-    "name" : "name",
-    "icon" : "icon",
-    "normalizationConfig" : {
-      "normalizationIntegrationType" : "normalizationIntegrationType",
-      "normalizationRepository" : "normalizationRepository",
-      "normalizationTag" : "normalizationTag",
-      "supported" : false
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
     },
-    "protocolVersion" : "protocolVersion",
-    "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "succeeded" : true
+  }
 }
    
 
     
    Produces
    
@@ -3403,14 +4917,21 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionReadList
+        DestinationDefinitionSpecificationRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    DestinationOauth
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/list_private
    
-    
    List all private, non-custom destinationDefinitions, and for each indicate whether the given workspace has a grant for using the definition. Used by admins to view and modify a given workspace's grants. (listPrivateDestinationDefinitions)
    
+    
    post /v1/destination_oauths/complete_oauth
    
+    
    Given a destination def ID generate an access/refresh token etc. (completeDestinationOAuth)
    
     
    
 
 
@@ -3422,7 +4943,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
+      
    CompleteDestinationOAuthRequest CompleteDestinationOAuthRequest (required)
    
 
       
    Body Parameter —  
    
 
@@ -3433,7 +4954,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      PrivateDestinationDefinitionReadList
+      CompleteOAuthResponse
       
     
    
 
@@ -3442,91 +4963,69 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "destinationDefinitions" : [ {
-    "destinationDefinition" : {
-      "resourceRequirements" : {
-        "default" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        },
-        "jobSpecific" : [ {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        }, {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        } ]
-      },
-      "documentationUrl" : "https://openapi-generator.tech",
-      "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
-      "supportsDbt" : true,
-      "name" : "name",
-      "icon" : "icon",
-      "normalizationConfig" : {
-        "normalizationIntegrationType" : "normalizationIntegrationType",
-        "normalizationRepository" : "normalizationRepository",
-        "normalizationTag" : "normalizationTag",
-        "supported" : false
-      },
-      "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-    },
-    "granted" : true
-  }, {
-    "destinationDefinition" : {
-      "resourceRequirements" : {
-        "default" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        },
-        "jobSpecific" : [ {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        }, {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        } ]
-      },
-      "documentationUrl" : "https://openapi-generator.tech",
-      "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
-      "supportsDbt" : true,
-      "name" : "name",
-      "icon" : "icon",
-      "normalizationConfig" : {
-        "normalizationIntegrationType" : "normalizationIntegrationType",
-        "normalizationRepository" : "normalizationRepository",
-        "normalizationTag" : "normalizationTag",
-        "supported" : false
-      },
-      "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-    },
-    "granted" : true
-  } ]
+  "auth_payload" : {
+    "key" : ""
+  },
+  "request_error" : "request_error",
+  "request_succeeded" : true
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        CompleteOAuthResponse
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/destination_oauths/get_consent_url
    
+    
    Given a destination connector definition ID, return the URL to the consent screen where to redirect the user to. (getDestinationOAuthConsent)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationOauthConsentRequest DestinationOauthConsentRequest (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      OAuthConsentRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "consentUrl" : "consentUrl"
 }
    
 
     
    Produces
    
@@ -3539,14 +5038,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        PrivateDestinationDefinitionReadList
+        OAuthConsentRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/revoke_definition
    
-    
    revoke a grant to a private, non-custom destinationDefinition from a given workspace (revokeDestinationDefinitionFromWorkspace)
    
+    
    post /v1/destination_oauths/oauth_params/create
    
+    
    Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know about these variables. (setInstancewideDestinationOauthParams)
    
     
    
 
 
@@ -3558,7 +5063,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
+      
    SetInstancewideDestinationOauthParamsRequestBody SetInstancewideDestinationOauthParamsRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -3579,22 +5084,64 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    204
    
-    The resource was deleted successfully.
+    
    200
    
+    Successful
         
+    
    400
    
+    Exception occurred; see message for details.
+        KnownExceptionInfo
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    Health
    
+  
    
     
    
     Up
-    
    post /v1/destination_definitions/update
    
-    
    Update destinationDefinition (updateDestinationDefinition)
    
+    
    get /v1/health
    
+    
    Health Check (getHealthCheck)
    
+    
    
+
+
+
+
+
+
+
+    
    Return type
    
+    
    
+      HealthCheckRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "available" : true
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        HealthCheckRead
+  
     
+  
    
+  
    Internal
    
+  
    
+    
    
+    Up
+    
    post /v1/state/create_or_update
    
+    
    Create or update the state for a connection. (createOrUpdateState)
    
     
    
 
 
@@ -3606,7 +5153,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationDefinitionUpdate DestinationDefinitionUpdate (required)
    
+      
    ConnectionStateCreateOrUpdate ConnectionStateCreateOrUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -3617,7 +5164,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationDefinitionRead
+      ConnectionState
       
     
    
 
@@ -3626,44 +5173,31 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
+  "globalState" : {
+    "streamStates" : [ {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
       }
     }, {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "dockerRepository" : "dockerRepository",
-  "supportsDbt" : true,
-  "name" : "name",
-  "icon" : "icon",
-  "normalizationConfig" : {
-    "normalizationIntegrationType" : "normalizationIntegrationType",
-    "normalizationRepository" : "normalizationRepository",
-    "normalizationTag" : "normalizationTag",
-    "supported" : false
-  },
-  "protocolVersion" : "protocolVersion",
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamState" : [ {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  }, {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  } ]
 }
    
 
     
    Produces
    
@@ -3676,7 +5210,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        DestinationDefinitionRead
+        ConnectionState
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -3685,12 +5219,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    DestinationDefinitionSpecification
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_definition_specifications/get
    
-    
    Get specification for a destinationDefinition (getDestinationDefinitionSpecification)
    
+    
    post /v1/jobs/get_normalization_status
    
+    
    Get normalization status to determine if we can bypass normalization phase (getAttemptNormalizationStatusesForJob)
    
     
    
 
 
@@ -3702,7 +5235,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationDefinitionIdWithWorkspaceId DestinationDefinitionIdWithWorkspaceId (required)
    
+      
    JobIdRequestBody JobIdRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -3713,7 +5246,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DestinationDefinitionSpecificationRead
+      AttemptNormalizationStatusReadList
       
     
    
 
@@ -3722,39 +5255,69 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "documentationUrl" : "documentationUrl",
-  "connectionSpecification" : {
-    "user" : {
-      "type" : "string"
-    }
-  },
-  "supportedDestinationSyncModes" : [ null, null ],
-  "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "advancedAuth" : {
-    "predicateValue" : "predicateValue",
-    "oauthConfigSpecification" : { },
-    "predicateKey" : [ "predicateKey", "predicateKey" ],
-    "authFlowType" : "oauth2.0"
-  },
-  "authSpecification" : {
-    "auth_type" : "oauth2.0",
-    "oauth2Specification" : {
-      "oauthFlowOutputParameters" : [ [ "oauthFlowOutputParameters", "oauthFlowOutputParameters" ], [ "oauthFlowOutputParameters", "oauthFlowOutputParameters" ] ],
-      "rootObject" : [ "path", 1 ],
-      "oauthFlowInitParameters" : [ [ "oauthFlowInitParameters", "oauthFlowInitParameters" ], [ "oauthFlowInitParameters", "oauthFlowInitParameters" ] ]
-    }
-  },
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
-  }
+  "attemptNormalizationStatuses" : [ {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
+  }, {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
+  } ]
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        AttemptNormalizationStatusReadList
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/attempt/save_stats
    
+    
    For worker to set sync stats of a running attempt. (saveStats)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    SaveStatsRequestBody SaveStatsRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      InternalOperationResult
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "succeeded" : true
 }
    
 
     
    Produces
    
@@ -3766,22 +5329,15 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful operation
-        DestinationDefinitionSpecificationRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    Successful Operation
+        InternalOperationResult
   
     
   
    
-  
    DestinationOauth
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_oauths/complete_oauth
    
-    
    Given a destination def ID generate an access/refresh token etc. (completeDestinationOAuth)
    
+    
    post /v1/attempt/save_sync_config
    
+    
    For worker to save the AttemptSyncConfig for an attempt. (saveSyncConfig)
    
     
    
 
 
@@ -3793,7 +5349,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    CompleteDestinationOAuthRequest CompleteDestinationOAuthRequest (required)
    
+      
    SaveAttemptSyncConfigRequestBody SaveAttemptSyncConfigRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -3804,12 +5360,17 @@ 
    Request body
    
 
     
    Return type
    
     
    
+      InternalOperationResult
       
-      map[String, oas_any_type_not_mapped]
     
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "succeeded" : true
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -3820,21 +5381,15 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful operation
-        
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    Successful Operation
+        InternalOperationResult
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_oauths/get_consent_url
    
-    
    Given a destination connector definition ID, return the URL to the consent screen where to redirect the user to. (getDestinationOAuthConsent)
    
+    
    post /v1/attempt/set_workflow_in_attempt
    
+    
    For worker to register the workflow id in attempt. (setWorkflowInAttempt)
    
     
    
 
 
@@ -3846,7 +5401,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationOauthConsentRequest DestinationOauthConsentRequest (required)
    
+      
    SetWorkflowInAttemptRequestBody SetWorkflowInAttemptRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -3857,7 +5412,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      OAuthConsentRead
+      InternalOperationResult
       
     
    
 
@@ -3866,7 +5421,7 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "consentUrl" : "consentUrl"
+  "succeeded" : true
 }
    
 
     
    Produces
    
@@ -3878,21 +5433,15 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful operation
-        OAuthConsentRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    Successful Operation
+        InternalOperationResult
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/destination_oauths/oauth_params/create
    
-    
    Sets instancewide variables to be used for the oauth flow when creating this destination. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know about these variables. (setInstancewideDestinationOauthParams)
    
+    
    post /v1/sources/write_discover_catalog_result
    
+    
    Should only called from worker, to write result from discover activity back to DB. (writeDiscoverCatalogResult)
    
     
    
 
 
@@ -3904,7 +5453,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SetInstancewideDestinationOauthParamsRequestBody SetInstancewideDestinationOauthParamsRequestBody (required)
    
+      
    SourceDiscoverSchemaWriteRequestBody SourceDiscoverSchemaWriteRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -3913,9 +5462,19 @@ 
    Request body
    
 
 
 
+    
    Return type
    
+    
    
+      DiscoverCatalogResult
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -3926,33 +5485,39 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful
-        
-    
    400
    
-    Exception occurred; see message for details.
-        KnownExceptionInfo
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+    Successful Operation
+        DiscoverCatalogResult
   
     
   
    
-  
    Health
    
-  
    
+  
    Jobs
    
+  
    
     
    
     Up
-    
    get /v1/health
    
-    
    Health Check (getHealthCheck)
    
+    
    post /v1/jobs/cancel
    
+    
    Cancels a job (cancelJob)
    
     
    
 
 
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    JobIdRequestBody JobIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
 
+    
      
 
 
 
 
     
    Return type
    
     
    
-      HealthCheckRead
+      JobInfoRead
       
     
    
 
@@ -3961,7 +5526,154 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "available" : true
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  },
+  "attempts" : [ {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  }, {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  } ]
 }
    
 
     
    Produces
    
@@ -3974,15 +5686,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        HealthCheckRead
+        JobInfoRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
-  
    
-  
    Internal
    
-  
    
+  
    
+  
    
     
    
     Up
-    
    post /v1/state/create_or_update
    
-    
    Create or update the state for a connection. (createOrUpdateState)
    
+    
    post /v1/jobs/get_normalization_status
    
+    
    Get normalization status to determine if we can bypass normalization phase (getAttemptNormalizationStatusesForJob)
    
     
    
 
 
@@ -3994,7 +5711,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionStateCreateOrUpdate ConnectionStateCreateOrUpdate (required)
    
+      
    JobIdRequestBody JobIdRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -4005,7 +5722,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      ConnectionState
+      AttemptNormalizationStatusReadList
       
     
    
 
@@ -4014,30 +5731,16 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "globalState" : {
-    "streamStates" : [ {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    }, {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    } ]
-  },
-  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "streamState" : [ {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
+  "attemptNormalizationStatuses" : [ {
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
   }, {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
+    "attemptNumber" : 0,
+    "recordsCommitted" : 6,
+    "hasRecordsCommitted" : true,
+    "hasNormalizationFailed" : true
   } ]
 }
    
 
@@ -4051,20 +5754,14 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        ConnectionState
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+        AttemptNormalizationStatusReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/jobs/get_normalization_status
    
-    
    Get normalization status to determine if we can bypass normalization phase (getAttemptNormalizationStatusesForJob)
    
+    
    post /v1/jobs/get_debug_info
    
+    
    Gets all information needed to debug this job (getJobDebugInfo)
    
     
    
 
 
@@ -4076,7 +5773,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobIdRequestBody JobIdRequestBody (optional)
    
+      
    JobIdRequestBody JobIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4087,7 +5784,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      AttemptNormalizationStatusReadList
+      JobDebugInfoRead
       
     
    
 
@@ -4096,16 +5793,213 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "attemptNormalizationStatuses" : [ {
-    "attemptNumber" : 0,
-    "recordsCommitted" : 6,
-    "hasRecordsCommitted" : true,
-    "hasNormalizationFailed" : true
+  "workflowState" : {
+    "running" : true
+  },
+  "job" : {
+    "configId" : "configId",
+    "sourceDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
+      "documentationUrl" : "https://openapi-generator.tech",
+      "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "sourceType" : "api",
+      "dockerRepository" : "dockerRepository",
+      "name" : "name",
+      "icon" : "icon",
+      "protocolVersion" : "protocolVersion",
+      "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "maxSecondsBetweenMessages" : 0
+    },
+    "airbyteVersion" : "airbyteVersion",
+    "id" : 0,
+    "destinationDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
+      "documentationUrl" : "https://openapi-generator.tech",
+      "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "dockerRepository" : "dockerRepository",
+      "supportsDbt" : true,
+      "name" : "name",
+      "icon" : "icon",
+      "normalizationConfig" : {
+        "normalizationIntegrationType" : "normalizationIntegrationType",
+        "normalizationRepository" : "normalizationRepository",
+        "normalizationTag" : "normalizationTag",
+        "supported" : false
+      },
+      "protocolVersion" : "protocolVersion",
+      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    }
+  },
+  "attempts" : [ {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
   }, {
-    "attemptNumber" : 0,
-    "recordsCommitted" : 6,
-    "hasRecordsCommitted" : true,
-    "hasNormalizationFailed" : true
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
   } ]
 }
    
 
@@ -4119,14 +6013,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        AttemptNormalizationStatusReadList
+        JobDebugInfoRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/attempt/save_stats
    
-    
    For worker to set sync stats of a running attempt. (saveStats)
    
+    
    post /v1/jobs/get
    
+    
    Get information about a job (getJobInfo)
    
     
    
 
 
@@ -4138,7 +6038,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SaveStatsRequestBody SaveStatsRequestBody (required)
    
+      
    JobIdRequestBody JobIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4149,7 +6049,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      InternalOperationResult
+      JobInfoRead
       
     
    
 
@@ -4158,7 +6058,154 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "succeeded" : true
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  },
+  "attempts" : [ {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  }, {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  } ]
 }
    
 
     
    Produces
    
@@ -4170,15 +6217,21 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful Operation
-        InternalOperationResult
+    Successful operation
+        JobInfoRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/attempt/save_sync_config
    
-    
    For worker to save the AttemptSyncConfig for an attempt. (saveSyncConfig)
    
+    
    post /v1/jobs/get_light
    
+    
    Get information about a job excluding attempt info and logs (getJobInfoLight)
    
     
    
 
 
@@ -4190,7 +6243,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SaveAttemptSyncConfigRequestBody SaveAttemptSyncConfigRequestBody (required)
    
+      
    JobIdRequestBody JobIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4201,7 +6254,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      InternalOperationResult
+      JobInfoLightRead
       
     
    
 
@@ -4210,7 +6263,29 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "succeeded" : true
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  }
 }
    
 
     
    Produces
    
@@ -4222,15 +6297,21 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful Operation
-        InternalOperationResult
+    Successful operation
+        JobInfoLightRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/attempt/set_workflow_in_attempt
    
-    
    For worker to register the workflow id in attempt. (setWorkflowInAttempt)
    
+    
    post /v1/jobs/get_without_logs
    
+    
    Get information about a job excluding logs (getJobInfoWithoutLogs)
    
     
    
 
 
@@ -4242,7 +6323,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SetWorkflowInAttemptRequestBody SetWorkflowInAttemptRequestBody (required)
    
+      
    JobIdRequestBody JobIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4253,7 +6334,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      InternalOperationResult
+      JobInfoRead
       
     
    
 
@@ -4262,7 +6343,154 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "succeeded" : true
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  },
+  "attempts" : [ {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  }, {
+    "attempt" : {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    },
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    }
+  } ]
 }
    
 
     
    Produces
    
@@ -4274,15 +6502,21 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful Operation
-        InternalOperationResult
+    Successful operation
+        JobInfoRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/write_discover_catalog_result
    
-    
    Should only called from worker, to write result from discover activity back to DB. (writeDiscoverCatalogResult)
    
+    
    post /v1/jobs/get_last_replication_job
    
+    
     (getLastReplicationJob)
    
     
    
 
 
@@ -4294,7 +6528,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDiscoverSchemaWriteRequestBody SourceDiscoverSchemaWriteRequestBody (required)
    
+      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4305,7 +6539,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DiscoverCatalogResult
+      JobOptionalRead
       
     
    
 
@@ -4314,7 +6548,29 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "job" : {
+    "createdAt" : 6,
+    "enabledStreams" : [ {
+      "name" : "name",
+      "namespace" : "namespace"
+    }, {
+      "name" : "name",
+      "namespace" : "namespace"
+    } ],
+    "configId" : "configId",
+    "startedAt" : 5,
+    "id" : 0,
+    "resetConfig" : {
+      "streamsToReset" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ]
+    },
+    "updatedAt" : 1
+  }
 }
    
 
     
    Produces
    
@@ -4326,16 +6582,21 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful Operation
-        DiscoverCatalogResult
+    Successful operation
+        JobOptionalRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    Jobs
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/jobs/cancel
    
-    
    Cancels a job (cancelJob)
    
+    
    post /v1/jobs/list
    
+    
    Returns recent jobs for a connection. Jobs are returned in descending order by createdAt. (listJobsFor)
    
     
    
 
 
@@ -4347,7 +6608,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobIdRequestBody JobIdRequestBody (required)
    
+      
    JobListRequestBody JobListRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4358,7 +6619,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      JobInfoRead
+      JobReadList
       
     
    
 
@@ -4367,31 +6628,179 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "job" : {
-    "createdAt" : 6,
-    "configId" : "configId",
-    "startedAt" : 5,
-    "id" : 0,
-    "resetConfig" : {
-      "streamsToReset" : [ {
+  "totalJobCount" : 0,
+  "jobs" : [ {
+    "job" : {
+      "createdAt" : 6,
+      "enabledStreams" : [ {
         "name" : "name",
         "namespace" : "namespace"
       }, {
         "name" : "name",
         "namespace" : "namespace"
-      } ]
+      } ],
+      "configId" : "configId",
+      "startedAt" : 5,
+      "id" : 0,
+      "resetConfig" : {
+        "streamsToReset" : [ {
+          "name" : "name",
+          "namespace" : "namespace"
+        }, {
+          "name" : "name",
+          "namespace" : "namespace"
+        } ]
+      },
+      "updatedAt" : 1
     },
-    "updatedAt" : 1
-  },
-  "attempts" : [ {
-    "attempt" : {
+    "attempts" : [ {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    }, {
+      "totalStats" : {
+        "stateMessagesEmitted" : 1,
+        "recordsCommitted" : 1,
+        "bytesEmitted" : 7,
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
+      },
+      "failureSummary" : {
+        "failures" : [ {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        }, {
+          "retryable" : true,
+          "stacktrace" : "stacktrace",
+          "internalMessage" : "internalMessage",
+          "externalMessage" : "externalMessage",
+          "timestamp" : 1
+        } ],
+        "partialSuccess" : true
+      },
+      "createdAt" : 2,
+      "bytesSynced" : 3,
+      "endedAt" : 9,
+      "streamStats" : [ {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      }, {
+        "stats" : {
+          "stateMessagesEmitted" : 1,
+          "recordsCommitted" : 1,
+          "bytesEmitted" : 7,
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
+        },
+        "streamNamespace" : "streamNamespace",
+        "streamName" : "streamName"
+      } ],
+      "id" : 5,
+      "recordsSynced" : 2,
+      "updatedAt" : 7
+    } ]
+  }, {
+    "job" : {
+      "createdAt" : 6,
+      "enabledStreams" : [ {
+        "name" : "name",
+        "namespace" : "namespace"
+      }, {
+        "name" : "name",
+        "namespace" : "namespace"
+      } ],
+      "configId" : "configId",
+      "startedAt" : 5,
+      "id" : 0,
+      "resetConfig" : {
+        "streamsToReset" : [ {
+          "name" : "name",
+          "namespace" : "namespace"
+        }, {
+          "name" : "name",
+          "namespace" : "namespace"
+        } ]
+      },
+      "updatedAt" : 1
+    },
+    "attempts" : [ {
       "totalStats" : {
         "stateMessagesEmitted" : 1,
         "recordsCommitted" : 1,
         "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
       },
       "failureSummary" : {
         "failures" : [ {
@@ -4399,13 +6808,13 @@ 
    Example data
    
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         }, {
           "retryable" : true,
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         } ],
         "partialSuccess" : true
       },
@@ -4417,9 +6826,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -4428,9 +6838,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -4438,19 +6849,15 @@ 
    Example data
    
       "id" : 5,
       "recordsSynced" : 2,
       "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  }, {
-    "attempt" : {
+    }, {
       "totalStats" : {
         "stateMessagesEmitted" : 1,
         "recordsCommitted" : 1,
         "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
+        "estimatedBytes" : 7,
+        "estimatedRecords" : 6,
+        "recordsEmitted" : 4,
+        "bytesCommitted" : 1
       },
       "failureSummary" : {
         "failures" : [ {
@@ -4458,13 +6865,13 @@ 
    Example data
    
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         }, {
           "retryable" : true,
           "stacktrace" : "stacktrace",
           "internalMessage" : "internalMessage",
           "externalMessage" : "externalMessage",
-          "timestamp" : 7
+          "timestamp" : 1
         } ],
         "partialSuccess" : true
       },
@@ -4476,9 +6883,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -4487,9 +6895,10 @@ 
    Example data
    
           "stateMessagesEmitted" : 1,
           "recordsCommitted" : 1,
           "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
+          "estimatedBytes" : 7,
+          "estimatedRecords" : 6,
+          "recordsEmitted" : 4,
+          "bytesCommitted" : 1
         },
         "streamNamespace" : "streamNamespace",
         "streamName" : "streamName"
@@ -4497,10 +6906,7 @@ 
    Example data
    
       "id" : 5,
       "recordsSynced" : 2,
       "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
+    } ]
   } ]
 }
    
 
@@ -4514,7 +6920,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        JobInfoRead
+        JobReadList
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -4523,11 +6929,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    Logs
    
+  
    
     
    
     Up
-    
    post /v1/jobs/get_normalization_status
    
-    
    Get normalization status to determine if we can bypass normalization phase (getAttemptNormalizationStatusesForJob)
    
+    
    post /v1/logs/get
    
+    
    Get logs (getLogs)
    
     
    
 
 
@@ -4539,7 +6946,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobIdRequestBody JobIdRequestBody (optional)
    
+      
    LogsRequestBody LogsRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -4550,46 +6957,39 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      AttemptNormalizationStatusReadList
       
+      File
     
    
 
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "attemptNormalizationStatuses" : [ {
-    "attemptNumber" : 0,
-    "recordsCommitted" : 6,
-    "hasRecordsCommitted" : true,
-    "hasNormalizationFailed" : true
-  }, {
-    "attemptNumber" : 0,
-    "recordsCommitted" : 6,
-    "hasRecordsCommitted" : true,
-    "hasNormalizationFailed" : true
-  } ]
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
     the media type will be conveyed by the Content-Type response header.
     
      
+      
    • text/plain
      • 
       
    • application/json
      • 
     
    
 
     
    Responses
    
     
    200
    
-    Successful operation
-        AttemptNormalizationStatusReadList
+    Returns the log file
+        File
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    Notifications
    
+  
    
     
    
     Up
-    
    post /v1/jobs/get_debug_info
    
-    
    Gets all information needed to debug this job (getJobDebugInfo)
    
+    
    post /v1/notifications/try
    
+    
    Try sending a notifications (tryNotificationConfig)
    
     
    
 
 
@@ -4601,227 +7001,28 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobIdRequestBody JobIdRequestBody (required)
    
+      
    Notification Notification (required)
    
 
       
    Body Parameter —  
    
-
-    
      
-
-
-
-
-    
    Return type
    
-    
    
-      JobDebugInfoRead
-      
-    
    
-
-    
-
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "workflowState" : {
-    "running" : true
-  },
-  "job" : {
-    "configId" : "configId",
-    "sourceDefinition" : {
-      "resourceRequirements" : {
-        "default" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        },
-        "jobSpecific" : [ {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        }, {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        } ]
-      },
-      "documentationUrl" : "https://openapi-generator.tech",
-      "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "sourceType" : "api",
-      "dockerRepository" : "dockerRepository",
-      "name" : "name",
-      "icon" : "icon",
-      "protocolVersion" : "protocolVersion",
-      "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-    },
-    "airbyteVersion" : "airbyteVersion",
-    "id" : 0,
-    "destinationDefinition" : {
-      "resourceRequirements" : {
-        "default" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        },
-        "jobSpecific" : [ {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        }, {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        } ]
-      },
-      "documentationUrl" : "https://openapi-generator.tech",
-      "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "dockerRepository" : "dockerRepository",
-      "supportsDbt" : true,
-      "name" : "name",
-      "icon" : "icon",
-      "normalizationConfig" : {
-        "normalizationIntegrationType" : "normalizationIntegrationType",
-        "normalizationRepository" : "normalizationRepository",
-        "normalizationTag" : "normalizationTag",
-        "supported" : false
-      },
-      "protocolVersion" : "protocolVersion",
-      "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-    }
-  },
-  "attempts" : [ {
-    "attempt" : {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  }, {
-    "attempt" : {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  } ]
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      NotificationRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "message" : "message",
+  "status" : "succeeded"
 }
    
 
     
    Produces
    
@@ -4834,7 +7035,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        JobDebugInfoRead
+        NotificationRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -4843,11 +7044,48 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    Openapi
    
+  
    
     
    
     Up
-    
    post /v1/jobs/get
    
-    
    Get information about a job (getJobInfo)
    
+    
    get /v1/openapi
    
+    
    Returns the openapi specification (getOpenApiSpec)
    
+    
    
+
+
+
+
+
+
+
+    
    Return type
    
+    
    
+      
+      File
+    
    
+
+    
+
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • text/plain
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Returns the openapi specification file
+        File
+  
     
+  
    
+  
    Operation
    
+  
    
+    
    
+    Up
+    
    post /v1/operations/check
    
+    
    Check if an operation to be created is valid (checkOperation)
    
     
    
 
 
@@ -4859,7 +7097,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobIdRequestBody JobIdRequestBody (required)
    
+      
    OperatorConfiguration OperatorConfiguration (required)
    
 
       
    Body Parameter —  
    
 
@@ -4870,7 +7108,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      JobInfoRead
+      CheckOperationRead
       
     
    
 
@@ -4879,141 +7117,8 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "job" : {
-    "createdAt" : 6,
-    "configId" : "configId",
-    "startedAt" : 5,
-    "id" : 0,
-    "resetConfig" : {
-      "streamsToReset" : [ {
-        "name" : "name",
-        "namespace" : "namespace"
-      }, {
-        "name" : "name",
-        "namespace" : "namespace"
-      } ]
-    },
-    "updatedAt" : 1
-  },
-  "attempts" : [ {
-    "attempt" : {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  }, {
-    "attempt" : {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  } ]
+  "message" : "message",
+  "status" : "succeeded"
 }
    
 
     
    Produces
    
@@ -5026,20 +7131,17 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        JobInfoRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+        CheckOperationRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/jobs/get_light
    
-    
    Get information about a job excluding attempt info and logs (getJobInfoLight)
    
+    
    post /v1/operations/create
    
+    
    Create an operation to be applied as part of a connection pipeline (createOperation)
    
     
    
 
 
@@ -5051,7 +7153,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobIdRequestBody JobIdRequestBody (required)
    
+      
    OperationCreate OperationCreate (required)
    
 
       
    Body Parameter —  
    
 
@@ -5062,7 +7164,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      JobInfoLightRead
+      OperationRead
       
     
    
 
@@ -5071,22 +7173,30 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "job" : {
-    "createdAt" : 6,
-    "configId" : "configId",
-    "startedAt" : 5,
-    "id" : 0,
-    "resetConfig" : {
-      "streamsToReset" : [ {
-        "name" : "name",
-        "namespace" : "namespace"
-      }, {
-        "name" : "name",
-        "namespace" : "namespace"
-      } ]
+  "name" : "name",
+  "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "operatorConfiguration" : {
+    "webhook" : {
+      "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "dbtCloud" : {
+        "accountId" : 0,
+        "jobId" : 6
+      },
+      "executionUrl" : "executionUrl",
+      "executionBody" : "executionBody",
+      "webhookType" : "dbtCloud"
     },
-    "updatedAt" : 1
-  }
+    "normalization" : {
+      "option" : "basic"
+    },
+    "dbt" : {
+      "gitRepoBranch" : "gitRepoBranch",
+      "dockerImage" : "dockerImage",
+      "dbtArguments" : "dbtArguments",
+      "gitRepoUrl" : "gitRepoUrl"
+    }
+  },
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -5099,20 +7209,17 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        JobInfoLightRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+        OperationRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/jobs/get_last_replication_job
    
-    
     (getLastReplicationJob)
    
+    
    post /v1/operations/delete
    
+    
    Delete an operation (deleteOperation)
    
     
    
 
 
@@ -5124,7 +7231,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
+      
    OperationIdRequestBody OperationIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -5132,35 +7239,10 @@ 
    Request body
    
 
 
 
-
-    
    Return type
    
-    
    
-      JobOptionalRead
-      
-    
    
-
+
+
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "job" : {
-    "createdAt" : 6,
-    "configId" : "configId",
-    "startedAt" : 5,
-    "id" : 0,
-    "resetConfig" : {
-      "streamsToReset" : [ {
-        "name" : "name",
-        "namespace" : "namespace"
-      }, {
-        "name" : "name",
-        "namespace" : "namespace"
-      } ]
-    },
-    "updatedAt" : 1
-  }
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -5170,9 +7252,9 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        JobOptionalRead
+    
    204
    
+    The resource was deleted successfully.
+        
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -5181,11 +7263,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/jobs/list
    
-    
    Returns recent jobs for a connection. Jobs are returned in descending order by createdAt. (listJobsFor)
    
+    
    post /v1/operations/get
    
+    
    Returns an operation (getOperation)
    
     
    
 
 
@@ -5197,7 +7279,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    JobListRequestBody JobListRequestBody (required)
    
+      
    OperationIdRequestBody OperationIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -5208,7 +7290,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      JobReadList
+      OperationRead
       
     
    
 
@@ -5217,260 +7299,30 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "totalJobCount" : 0,
-  "jobs" : [ {
-    "job" : {
-      "createdAt" : 6,
-      "configId" : "configId",
-      "startedAt" : 5,
-      "id" : 0,
-      "resetConfig" : {
-        "streamsToReset" : [ {
-          "name" : "name",
-          "namespace" : "namespace"
-        }, {
-          "name" : "name",
-          "namespace" : "namespace"
-        } ]
-      },
-      "updatedAt" : 1
-    },
-    "attempts" : [ {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    }, {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    } ]
-  }, {
-    "job" : {
-      "createdAt" : 6,
-      "configId" : "configId",
-      "startedAt" : 5,
-      "id" : 0,
-      "resetConfig" : {
-        "streamsToReset" : [ {
-          "name" : "name",
-          "namespace" : "namespace"
-        }, {
-          "name" : "name",
-          "namespace" : "namespace"
-        } ]
-      },
-      "updatedAt" : 1
-    },
-    "attempts" : [ {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
-      },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    }, {
-      "totalStats" : {
-        "stateMessagesEmitted" : 1,
-        "recordsCommitted" : 1,
-        "bytesEmitted" : 7,
-        "estimatedBytes" : 6,
-        "estimatedRecords" : 1,
-        "recordsEmitted" : 4
-      },
-      "failureSummary" : {
-        "failures" : [ {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        }, {
-          "retryable" : true,
-          "stacktrace" : "stacktrace",
-          "internalMessage" : "internalMessage",
-          "externalMessage" : "externalMessage",
-          "timestamp" : 7
-        } ],
-        "partialSuccess" : true
+  "name" : "name",
+  "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "operatorConfiguration" : {
+    "webhook" : {
+      "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "dbtCloud" : {
+        "accountId" : 0,
+        "jobId" : 6
       },
-      "createdAt" : 2,
-      "bytesSynced" : 3,
-      "endedAt" : 9,
-      "streamStats" : [ {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      }, {
-        "stats" : {
-          "stateMessagesEmitted" : 1,
-          "recordsCommitted" : 1,
-          "bytesEmitted" : 7,
-          "estimatedBytes" : 6,
-          "estimatedRecords" : 1,
-          "recordsEmitted" : 4
-        },
-        "streamNamespace" : "streamNamespace",
-        "streamName" : "streamName"
-      } ],
-      "id" : 5,
-      "recordsSynced" : 2,
-      "updatedAt" : 7
-    } ]
-  } ]
+      "executionUrl" : "executionUrl",
+      "executionBody" : "executionBody",
+      "webhookType" : "dbtCloud"
+    },
+    "normalization" : {
+      "option" : "basic"
+    },
+    "dbt" : {
+      "gitRepoBranch" : "gitRepoBranch",
+      "dockerImage" : "dockerImage",
+      "dbtArguments" : "dbtArguments",
+      "gitRepoUrl" : "gitRepoUrl"
+    }
+  },
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -5483,7 +7335,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        JobReadList
+        OperationRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -5492,13 +7344,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    Logs
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/logs/get
    
-    
    Get logs (getLogs)
    
-    
    
+    
    post /v1/operations/list
    
+    
    Returns all operations for a connection. (listOperationsForConnection)
    
+    
    List operations for connection.
    
 
 
     
    Consumes
    
@@ -5509,7 +7360,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    LogsRequestBody LogsRequestBody (required)
    
+      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -5520,25 +7371,79 @@ 
    Request body
    
 
     
    Return type
    
     
    
+      OperationReadList
       
-      File
     
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "operations" : [ {
+    "name" : "name",
+    "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "operatorConfiguration" : {
+      "webhook" : {
+        "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+        "dbtCloud" : {
+          "accountId" : 0,
+          "jobId" : 6
+        },
+        "executionUrl" : "executionUrl",
+        "executionBody" : "executionBody",
+        "webhookType" : "dbtCloud"
+      },
+      "normalization" : {
+        "option" : "basic"
+      },
+      "dbt" : {
+        "gitRepoBranch" : "gitRepoBranch",
+        "dockerImage" : "dockerImage",
+        "dbtArguments" : "dbtArguments",
+        "gitRepoUrl" : "gitRepoUrl"
+      }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "name" : "name",
+    "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "operatorConfiguration" : {
+      "webhook" : {
+        "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+        "dbtCloud" : {
+          "accountId" : 0,
+          "jobId" : 6
+        },
+        "executionUrl" : "executionUrl",
+        "executionBody" : "executionBody",
+        "webhookType" : "dbtCloud"
+      },
+      "normalization" : {
+        "option" : "basic"
+      },
+      "dbt" : {
+        "gitRepoBranch" : "gitRepoBranch",
+        "dockerImage" : "dockerImage",
+        "dbtArguments" : "dbtArguments",
+        "gitRepoUrl" : "gitRepoUrl"
+      }
+    },
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
     the media type will be conveyed by the Content-Type response header.
     
      
-      
    • text/plain
      • 
       
    • application/json
      • 
     
    
 
     
    Responses
    
     
    200
    
-    Returns the log file
-        File
+    Successful operation
+        OperationReadList
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -5547,12 +7452,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    Notifications
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/notifications/try
    
-    
    Try sending a notifications (tryNotificationConfig)
    
+    
    post /v1/operations/update
    
+    
    Update an operation (updateOperation)
    
     
    
 
 
@@ -5564,7 +7468,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    Notification Notification (required)
    
+      
    OperationUpdate OperationUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -5575,7 +7479,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      NotificationRead
+      OperationRead
       
     
    
 
@@ -5584,8 +7488,30 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "message" : "message",
-  "status" : "succeeded"
+  "name" : "name",
+  "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "operatorConfiguration" : {
+    "webhook" : {
+      "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "dbtCloud" : {
+        "accountId" : 0,
+        "jobId" : 6
+      },
+      "executionUrl" : "executionUrl",
+      "executionBody" : "executionBody",
+      "webhookType" : "dbtCloud"
+    },
+    "normalization" : {
+      "option" : "basic"
+    },
+    "dbt" : {
+      "gitRepoBranch" : "gitRepoBranch",
+      "dockerImage" : "dockerImage",
+      "dbtArguments" : "dbtArguments",
+      "gitRepoUrl" : "gitRepoUrl"
+    }
+  },
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -5598,57 +7524,92 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        NotificationRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+        OperationRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    Openapi
    
-  
    
+  
    Scheduler
    
+  
    
     
    
     Up
-    
    get /v1/openapi
    
-    
    Returns the openapi specification (getOpenApiSpec)
    
+    
    post /v1/scheduler/destinations/check_connection
    
+    
    Run check connection for a given destination configuration (executeDestinationCheckConnection)
    
     
    
 
 
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    DestinationCoreConfig DestinationCoreConfig (required)
    
+
+      
    Body Parameter —  
    
 
+    
      
 
 
 
 
     
    Return type
    
     
    
+      CheckConnectionRead
       
-      File
     
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "message" : "message",
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
+    },
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    },
+    "succeeded" : true
+  },
+  "status" : "succeeded"
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
     the media type will be conveyed by the Content-Type response header.
     
      
-      
    • text/plain
      • 
+      
    • application/json
      • 
     
    
 
     
    Responses
    
     
    200
    
-    Returns the openapi specification file
-        File
+    Successful operation
+        CheckConnectionRead
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    Operation
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/operations/check
    
-    
    Check if an operation to be created is valid (checkOperation)
    
+    
    post /v1/scheduler/sources/check_connection
    
+    
    Run check connection for a given source configuration (executeSourceCheckConnection)
    
     
    
 
 
@@ -5660,7 +7621,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    OperatorConfiguration OperatorConfiguration (required)
    
+      
    SourceCoreConfig SourceCoreConfig (required)
    
 
       
    Body Parameter —  
    
 
@@ -5671,7 +7632,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      CheckOperationRead
+      CheckConnectionRead
       
     
    
 
@@ -5681,6 +7642,24 @@ 
    Example data
    
     
    Content-Type: application/json
    
     
    {
   "message" : "message",
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
+    },
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    },
+    "succeeded" : true
+  },
   "status" : "succeeded"
 }
    
 
@@ -5694,17 +7673,17 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        CheckOperationRead
+        CheckConnectionRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/operations/create
    
-    
    Create an operation to be applied as part of a connection pipeline (createOperation)
    
+    
    post /v1/scheduler/sources/discover_schema
    
+    
    Run discover schema for a given source a source configuration (executeSourceDiscoverSchema)
    
     
    
 
 
@@ -5716,7 +7695,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    OperationCreate OperationCreate (required)
    
+      
    SourceCoreConfig SourceCoreConfig (required)
    
 
       
    Body Parameter —  
    
 
@@ -5727,7 +7706,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      OperationRead
+      SourceDiscoverSchemaRead
       
     
    
 
@@ -5736,30 +7715,118 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "name" : "name",
-  "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "operatorConfiguration" : {
-    "webhook" : {
-      "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "dbtCloud" : {
-        "accountId" : 0,
-        "jobId" : 6
+  "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "breakingChange" : true,
+  "catalog" : {
+    "streams" : [ {
+      "stream" : {
+        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+        "supportedSyncModes" : [ null, null ],
+        "sourceDefinedCursor" : true,
+        "name" : "name",
+        "namespace" : "namespace",
+        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
+      },
+      "config" : {
+        "aliasName" : "aliasName",
+        "suggested" : true,
+        "fieldSelectionEnabled" : true,
+        "selectedFields" : [ {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        }, {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        } ],
+        "cursorField" : [ "cursorField", "cursorField" ],
+        "selected" : true,
+        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+      }
+    }, {
+      "stream" : {
+        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
+        "supportedSyncModes" : [ null, null ],
+        "sourceDefinedCursor" : true,
+        "name" : "name",
+        "namespace" : "namespace",
+        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
       },
-      "executionUrl" : "executionUrl",
-      "executionBody" : "executionBody",
-      "webhookType" : "dbtCloud"
+      "config" : {
+        "aliasName" : "aliasName",
+        "suggested" : true,
+        "fieldSelectionEnabled" : true,
+        "selectedFields" : [ {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        }, {
+          "fieldPath" : [ "fieldPath", "fieldPath" ]
+        } ],
+        "cursorField" : [ "cursorField", "cursorField" ],
+        "selected" : true,
+        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
+      }
+    } ]
+  },
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
     },
-    "normalization" : {
-      "option" : "basic"
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
     },
-    "dbt" : {
-      "gitRepoBranch" : "gitRepoBranch",
-      "dockerImage" : "dockerImage",
-      "dbtArguments" : "dbtArguments",
-      "gitRepoUrl" : "gitRepoUrl"
-    }
+    "succeeded" : true
   },
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "catalogDiff" : {
+    "transforms" : [ {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      },
+      "transformType" : "add_stream",
+      "updateStream" : [ {
+        "updateFieldSchema" : { },
+        "fieldName" : [ "fieldName", "fieldName" ],
+        "addField" : { },
+        "transformType" : "add_field",
+        "removeField" : { },
+        "breaking" : true
+      }, {
+        "updateFieldSchema" : { },
+        "fieldName" : [ "fieldName", "fieldName" ],
+        "addField" : { },
+        "transformType" : "add_field",
+        "removeField" : { },
+        "breaking" : true
+      } ]
+    }, {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      },
+      "transformType" : "add_stream",
+      "updateStream" : [ {
+        "updateFieldSchema" : { },
+        "fieldName" : [ "fieldName", "fieldName" ],
+        "addField" : { },
+        "transformType" : "add_field",
+        "removeField" : { },
+        "breaking" : true
+      }, {
+        "updateFieldSchema" : { },
+        "fieldName" : [ "fieldName", "fieldName" ],
+        "addField" : { },
+        "transformType" : "add_field",
+        "removeField" : { },
+        "breaking" : true
+      } ]
+    } ]
+  }
 }
    
 
     
    Produces
    
@@ -5772,17 +7839,18 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        OperationRead
+        SourceDiscoverSchemaRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    Source
    
+  
    
     
    
     Up
-    
    post /v1/operations/delete
    
-    
    Delete an operation (deleteOperation)
    
+    
    post /v1/sources/apply_schema_changes
    
+    
    Auto propagate the change on a catalog to a catalog saved in the DB. It will fetch all the connections linked to a source id and apply the provided diff to their catalog. (applySchemaChangeForSource)
    
     
    
 
 
@@ -5794,7 +7862,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    OperationIdRequestBody OperationIdRequestBody (required)
    
+      
    SourceAutoPropagateChange SourceAutoPropagateChange (required)
    
 
       
    Body Parameter —  
    
 
@@ -5816,7 +7884,7 @@ 
    Produces
    
 
     
    Responses
    
     
    204
    
-    The resource was deleted successfully.
+    The schema was properly auto propagate
         
     
    404
    
     Object with given id was not found.
@@ -5826,11 +7894,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/operations/get
    
-    
    Returns an operation (getOperation)
    
+    
    post /v1/sources/check_connection
    
+    
    Check connection to the source (checkConnectionToSource)
    
     
    
 
 
@@ -5842,7 +7910,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    OperationIdRequestBody OperationIdRequestBody (required)
    
+      
    SourceIdRequestBody SourceIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -5853,7 +7921,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      OperationRead
+      CheckConnectionRead
       
     
    
 
@@ -5862,30 +7930,26 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "name" : "name",
-  "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "operatorConfiguration" : {
-    "webhook" : {
-      "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "dbtCloud" : {
-        "accountId" : 0,
-        "jobId" : 6
-      },
-      "executionUrl" : "executionUrl",
-      "executionBody" : "executionBody",
-      "webhookType" : "dbtCloud"
+  "message" : "message",
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
     },
-    "normalization" : {
-      "option" : "basic"
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
     },
-    "dbt" : {
-      "gitRepoBranch" : "gitRepoBranch",
-      "dockerImage" : "dockerImage",
-      "dbtArguments" : "dbtArguments",
-      "gitRepoUrl" : "gitRepoUrl"
-    }
+    "succeeded" : true
   },
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "status" : "succeeded"
 }
    
 
     
    Produces
    
@@ -5898,7 +7962,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        OperationRead
+        CheckConnectionRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -5907,12 +7971,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/operations/list
    
-    
    Returns all operations for a connection. (listOperationsForConnection)
    
-    
    List operations for connection.
    
+    
    post /v1/sources/check_connection_for_update
    
+    
    Check connection for a proposed update to a source (checkConnectionToSourceForUpdate)
    
+    
    
 
 
     
    Consumes
    
@@ -5923,7 +7987,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
+      
    SourceUpdate SourceUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -5934,7 +7998,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      OperationReadList
+      CheckConnectionRead
       
     
    
 
@@ -5943,57 +8007,26 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "operations" : [ {
-    "name" : "name",
-    "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "operatorConfiguration" : {
-      "webhook" : {
-        "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-        "dbtCloud" : {
-          "accountId" : 0,
-          "jobId" : 6
-        },
-        "executionUrl" : "executionUrl",
-        "executionBody" : "executionBody",
-        "webhookType" : "dbtCloud"
-      },
-      "normalization" : {
-        "option" : "basic"
-      },
-      "dbt" : {
-        "gitRepoBranch" : "gitRepoBranch",
-        "dockerImage" : "dockerImage",
-        "dbtArguments" : "dbtArguments",
-        "gitRepoUrl" : "gitRepoUrl"
-      }
+  "message" : "message",
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
     },
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "name" : "name",
-    "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "operatorConfiguration" : {
-      "webhook" : {
-        "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-        "dbtCloud" : {
-          "accountId" : 0,
-          "jobId" : 6
-        },
-        "executionUrl" : "executionUrl",
-        "executionBody" : "executionBody",
-        "webhookType" : "dbtCloud"
-      },
-      "normalization" : {
-        "option" : "basic"
-      },
-      "dbt" : {
-        "gitRepoBranch" : "gitRepoBranch",
-        "dockerImage" : "dockerImage",
-        "dbtArguments" : "dbtArguments",
-        "gitRepoUrl" : "gitRepoUrl"
-      }
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
     },
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "succeeded" : true
+  },
+  "status" : "succeeded"
 }
    
 
     
    Produces
    
@@ -6006,7 +8039,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        OperationReadList
+        CheckConnectionRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -6015,11 +8048,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/operations/update
    
-    
    Update an operation (updateOperation)
    
+    
    post /v1/sources/clone
    
+    
    Clone source (cloneSource)
    
     
    
 
 
@@ -6031,49 +8064,34 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    OperationUpdate OperationUpdate (required)
    
-
-      
    Body Parameter —  
    
-
-    
      
-
-
-
-
-    
    Return type
    
-    
    
-      OperationRead
-      
-    
    
-
-    
+      
    SourceCloneRequestBody SourceCloneRequestBody (required)
    
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "name" : "name",
-  "operationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "operatorConfiguration" : {
-    "webhook" : {
-      "webhookConfigId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-      "dbtCloud" : {
-        "accountId" : 0,
-        "jobId" : 6
-      },
-      "executionUrl" : "executionUrl",
-      "executionBody" : "executionBody",
-      "webhookType" : "dbtCloud"
-    },
-    "normalization" : {
-      "option" : "basic"
-    },
-    "dbt" : {
-      "gitRepoBranch" : "gitRepoBranch",
-      "dockerImage" : "dockerImage",
-      "dbtArguments" : "dbtArguments",
-      "gitRepoUrl" : "gitRepoUrl"
-    }
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      SourceRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "connectionConfiguration" : {
+    "user" : "charles"
   },
+  "name" : "name",
+  "icon" : "icon",
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "sourceName" : "sourceName",
   "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
@@ -6087,18 +8105,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        OperationRead
+        SourceRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    Scheduler
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/scheduler/destinations/check_connection
    
-    
    Run check connection for a given destination configuration (executeDestinationCheckConnection)
    
+    
    post /v1/sources/create
    
+    
    Create a source (createSource)
    
     
    
 
 
@@ -6110,7 +8130,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    DestinationCoreConfig DestinationCoreConfig (required)
    
+      
    SourceCreate SourceCreate (required)
    
 
       
    Body Parameter —  
    
 
@@ -6121,7 +8141,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      CheckConnectionRead
+      SourceRead
       
     
    
 
@@ -6130,19 +8150,15 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "message" : "message",
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
+  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "connectionConfiguration" : {
+    "user" : "charles"
   },
-  "status" : "succeeded"
+  "name" : "name",
+  "icon" : "icon",
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "sourceName" : "sourceName",
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -6155,17 +8171,17 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        CheckConnectionRead
+        SourceRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/scheduler/sources/check_connection
    
-    
    Run check connection for a given source configuration (executeSourceCheckConnection)
    
+    
    post /v1/sources/delete
    
+    
    Delete a source (deleteSource)
    
     
    
 
 
@@ -6177,7 +8193,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceCoreConfig SourceCoreConfig (required)
    
+      
    SourceIdRequestBody SourceIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -6186,31 +8202,9 @@ 
    Request body
    
 
 
 
-    
    Return type
    
-    
    
-      CheckConnectionRead
-      
-    
    
 
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "message" : "message",
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
-  },
-  "status" : "succeeded"
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -6220,19 +8214,22 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        CheckConnectionRead
+    
    204
    
+    The resource was deleted successfully.
+        
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/scheduler/sources/discover_schema
    
-    
    Run discover schema for a given source a source configuration (executeSourceDiscoverSchema)
    
+    
    post /v1/sources/discover_schema
    
+    
    Discover the schema catalog of the source (discoverSchemaForSource)
    
     
    
 
 
@@ -6244,7 +8241,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceCoreConfig SourceCoreConfig (required)
    
+      
    SourceDiscoverSchemaRequestBody SourceDiscoverSchemaRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -6318,6 +8315,13 @@ 
    Example data
    
     "connectorConfigurationUpdated" : false,
     "configId" : "configId",
     "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
+    },
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
     "logs" : {
       "logLines" : [ "logLines", "logLines" ]
@@ -6382,17 +8386,19 @@ 
    Responses
    
     
    200
    
     Successful operation
         SourceDiscoverSchemaRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    Source
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/check_connection
    
-    
    Check connection to the source (checkConnectionToSource)
    
+    
    post /v1/sources/most_recent_source_actor_catalog
    
+    
    Get most recent ActorCatalog for source (getMostRecentSourceActorCatalog)
    
     
    
 
 
@@ -6415,7 +8421,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      CheckConnectionRead
+      ActorCatalogWithUpdatedAt
       
     
    
 
@@ -6424,19 +8430,152 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "message" : "message",
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
+  "catalog" : "{}",
+  "updatedAt" : 0
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        ActorCatalogWithUpdatedAt
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/sources/get
    
+    
    Get source (getSource)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    SourceIdRequestBody SourceIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      SourceRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "connectionConfiguration" : {
+    "user" : "charles"
+  },
+  "name" : "name",
+  "icon" : "icon",
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "sourceName" : "sourceName",
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful operation
+        SourceRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
+  
     
+  
    
+  
    
+    
    
+    Up
+    
    post /v1/sources/list
    
+    
    List sources for workspace (listSourcesForWorkspace)
    
+    
    List sources for workspace. Does not return deleted sources.
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      SourceReadList
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "sources" : [ {
+    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "connectionConfiguration" : {
+      "user" : "charles"
     },
-    "succeeded" : true
-  },
-  "status" : "succeeded"
+    "name" : "name",
+    "icon" : "icon",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "sourceName" : "sourceName",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "name" : "name",
+    "icon" : "icon",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "sourceName" : "sourceName",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
 }
    
 
     
    Produces
    
@@ -6449,7 +8588,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        CheckConnectionRead
+        SourceReadList
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -6458,11 +8597,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/check_connection_for_update
    
-    
    Check connection for a proposed update to a source (checkConnectionToSourceForUpdate)
    
+    
    post /v1/sources/partial_update
    
+    
    Partially update a source (partialUpdateSource)
    
     
    
 
 
@@ -6474,7 +8613,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceUpdate SourceUpdate (required)
    
+      
    PartialSourceUpdate PartialSourceUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -6485,7 +8624,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      CheckConnectionRead
+      SourceRead
       
     
    
 
@@ -6494,19 +8633,15 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "message" : "message",
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
+  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "connectionConfiguration" : {
+    "user" : "charles"
   },
-  "status" : "succeeded"
+  "name" : "name",
+  "icon" : "icon",
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "sourceName" : "sourceName",
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -6519,7 +8654,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        CheckConnectionRead
+        SourceRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -6528,11 +8663,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/clone
    
-    
    Clone source (cloneSource)
    
+    
    post /v1/sources/search
    
+    
    Search sources (searchSources)
    
     
    
 
 
@@ -6544,7 +8679,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceCloneRequestBody SourceCloneRequestBody (required)
    
+      
    SourceSearch SourceSearch (required)
    
 
       
    Body Parameter —  
    
 
@@ -6555,7 +8690,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceRead
+      SourceReadList
       
     
    
 
@@ -6564,15 +8699,27 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "connectionConfiguration" : {
-    "user" : "charles"
-  },
-  "name" : "name",
-  "icon" : "icon",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "sourceName" : "sourceName",
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "sources" : [ {
+    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "name" : "name",
+    "icon" : "icon",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "sourceName" : "sourceName",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "connectionConfiguration" : {
+      "user" : "charles"
+    },
+    "name" : "name",
+    "icon" : "icon",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "sourceName" : "sourceName",
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
 }
    
 
     
    Produces
    
@@ -6585,20 +8732,17 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+        SourceReadList
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/create
    
-    
    Create a source (createSource)
    
+    
    post /v1/sources/update
    
+    
    Update a source (updateSource)
    
     
    
 
 
@@ -6610,7 +8754,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceCreate SourceCreate (required)
    
+      
    SourceUpdate SourceUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -6652,16 +8796,19 @@ 
    Responses
    
     
    200
    
     Successful operation
         SourceRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/delete
    
-    
    Delete a source (deleteSource)
    
+    
    post /v1/sources/write_discover_catalog_result
    
+    
    Should only called from worker, to write result from discover activity back to DB. (writeDiscoverCatalogResult)
    
     
    
 
 
@@ -6673,7 +8820,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceIdRequestBody SourceIdRequestBody (required)
    
+      
    SourceDiscoverSchemaWriteRequestBody SourceDiscoverSchemaWriteRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -6682,9 +8829,19 @@ 
    Request body
    
 
 
 
+    
    Return type
    
+    
    
+      DiscoverCatalogResult
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -6694,22 +8851,17 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    204
    
-    The resource was deleted successfully.
-        
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    
    200
    
+    Successful Operation
+        DiscoverCatalogResult
   
     
   
    
-  
    
+  
    SourceDefinition
    
+  
    
     
    
     Up
-    
    post /v1/sources/discover_schema
    
-    
    Discover the schema catalog of the source (discoverSchemaForSource)
    
+    
    post /v1/source_definitions/create_custom
    
+    
    Creates a custom sourceDefinition for the given workspace (createCustomSourceDefinition)
    
     
    
 
 
@@ -6721,7 +8873,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDiscoverSchemaRequestBody SourceDiscoverSchemaRequestBody (required)
    
+      
    CustomSourceDefinitionCreate CustomSourceDefinitionCreate (optional)
    
 
       
    Body Parameter —  
    
 
@@ -6732,120 +8884,48 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceDiscoverSchemaRead
+      SourceDefinitionRead
       
     
    
 
     
 
     
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "breakingChange" : true,
-  "catalog" : {
-    "streams" : [ {
-      "stream" : {
-        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
-        "supportedSyncModes" : [ null, null ],
-        "sourceDefinedCursor" : true,
-        "name" : "name",
-        "namespace" : "namespace",
-        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
-      },
-      "config" : {
-        "aliasName" : "aliasName",
-        "suggested" : true,
-        "fieldSelectionEnabled" : true,
-        "selectedFields" : [ {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
-        }, {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
-        } ],
-        "cursorField" : [ "cursorField", "cursorField" ],
-        "selected" : true,
-        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
-      }
-    }, {
-      "stream" : {
-        "sourceDefinedPrimaryKey" : [ [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ], [ "sourceDefinedPrimaryKey", "sourceDefinedPrimaryKey" ] ],
-        "supportedSyncModes" : [ null, null ],
-        "sourceDefinedCursor" : true,
-        "name" : "name",
-        "namespace" : "namespace",
-        "defaultCursorField" : [ "defaultCursorField", "defaultCursorField" ]
-      },
-      "config" : {
-        "aliasName" : "aliasName",
-        "suggested" : true,
-        "fieldSelectionEnabled" : true,
-        "selectedFields" : [ {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
-        }, {
-          "fieldPath" : [ "fieldPath", "fieldPath" ]
-        } ],
-        "cursorField" : [ "cursorField", "cursorField" ],
-        "selected" : true,
-        "primaryKey" : [ [ "primaryKey", "primaryKey" ], [ "primaryKey", "primaryKey" ] ]
-      }
-    } ]
-  },
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
-  },
-  "catalogDiff" : {
-    "transforms" : [ {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      },
-      "transformType" : "add_stream",
-      "updateStream" : [ {
-        "updateFieldSchema" : { },
-        "fieldName" : [ "fieldName", "fieldName" ],
-        "addField" : { },
-        "transformType" : "add_field",
-        "removeField" : { },
-        "breaking" : true
-      }, {
-        "updateFieldSchema" : { },
-        "fieldName" : [ "fieldName", "fieldName" ],
-        "addField" : { },
-        "transformType" : "add_field",
-        "removeField" : { },
-        "breaking" : true
-      } ]
-    }, {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      },
-      "transformType" : "add_stream",
-      "updateStream" : [ {
-        "updateFieldSchema" : { },
-        "fieldName" : [ "fieldName", "fieldName" ],
-        "addField" : { },
-        "transformType" : "add_field",
-        "removeField" : { },
-        "breaking" : true
-      }, {
-        "updateFieldSchema" : { },
-        "fieldName" : [ "fieldName", "fieldName" ],
-        "addField" : { },
-        "transformType" : "add_field",
-        "removeField" : { },
-        "breaking" : true
-      } ]
+    
    Content-Type: application/json
    
+    
    {
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
     } ]
-  }
+  },
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "sourceType" : "api",
+  "dockerRepository" : "dockerRepository",
+  "name" : "name",
+  "icon" : "icon",
+  "protocolVersion" : "protocolVersion",
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "maxSecondsBetweenMessages" : 0
 }
    
 
     
    Produces
    
@@ -6858,20 +8938,17 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceDiscoverSchemaRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+        SourceDefinitionRead
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/most_recent_source_actor_catalog
    
-    
    Get most recent ActorCatalog for source (getMostRecentSourceActorCatalog)
    
+    
    post /v1/source_definitions/delete
    
+    
    Delete a source definition (deleteSourceDefinition)
    
     
    
 
 
@@ -6883,7 +8960,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceIdRequestBody SourceIdRequestBody (required)
    
+      
    SourceDefinitionIdRequestBody SourceDefinitionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -6892,20 +8969,9 @@ 
    Request body
    
 
 
 
-    
    Return type
    
-    
    
-      ActorCatalogWithUpdatedAt
-      
-    
    
 
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "catalog" : "{}",
-  "updatedAt" : 0
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -6915,9 +8981,9 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        ActorCatalogWithUpdatedAt
+    
    204
    
+    The resource was deleted successfully.
+        
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -6926,11 +8992,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/get
    
-    
    Get source (getSource)
    
+    
    post /v1/source_definitions/get
    
+    
    Get source (getSourceDefinition)
    
     
    
 
 
@@ -6942,7 +9008,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceIdRequestBody SourceIdRequestBody (required)
    
+      
    SourceDefinitionIdRequestBody SourceDefinitionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -6953,7 +9019,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceRead
+      SourceDefinitionRead
       
     
    
 
@@ -6962,15 +9028,39 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "connectionConfiguration" : {
-    "user" : "charles"
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
+    },
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
   },
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "sourceType" : "api",
+  "dockerRepository" : "dockerRepository",
   "name" : "name",
   "icon" : "icon",
+  "protocolVersion" : "protocolVersion",
   "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "sourceName" : "sourceName",
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "maxSecondsBetweenMessages" : 0
 }
    
 
     
    Produces
    
@@ -6983,7 +9073,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceRead
+        SourceDefinitionRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -6992,12 +9082,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/list
    
-    
    List sources for workspace (listSourcesForWorkspace)
    
-    
    List sources for workspace. Does not return deleted sources.
    
+    
    post /v1/source_definitions/get_for_workspace
    
+    
    Get a sourceDefinition that is configured for the given workspace (getSourceDefinitionForWorkspace)
    
+    
    
 
 
     
    Consumes
    
@@ -7008,7 +9098,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    
+      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
 
       
    Body Parameter —  
    
 
@@ -7019,7 +9109,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceReadList
+      SourceDefinitionRead
       
     
    
 
@@ -7028,27 +9118,39 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sources" : [ {
-    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "connectionConfiguration" : {
-      "user" : "charles"
-    },
-    "name" : "name",
-    "icon" : "icon",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "sourceName" : "sourceName",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "connectionConfiguration" : {
-      "user" : "charles"
+  "resourceRequirements" : {
+    "default" : {
+      "cpu_limit" : "cpu_limit",
+      "memory_request" : "memory_request",
+      "memory_limit" : "memory_limit",
+      "cpu_request" : "cpu_request"
     },
-    "name" : "name",
-    "icon" : "icon",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "sourceName" : "sourceName",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "jobSpecific" : [ {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    }, {
+      "resourceRequirements" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      }
+    } ]
+  },
+  "documentationUrl" : "https://openapi-generator.tech",
+  "dockerImageTag" : "dockerImageTag",
+  "releaseDate" : "2000-01-23",
+  "sourceType" : "api",
+  "dockerRepository" : "dockerRepository",
+  "name" : "name",
+  "icon" : "icon",
+  "protocolVersion" : "protocolVersion",
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "maxSecondsBetweenMessages" : 0
 }
    
 
     
    Produces
    
@@ -7061,7 +9163,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceReadList
+        SourceDefinitionRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -7070,11 +9172,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/search
    
-    
    Search sources (searchSources)
    
+    
    post /v1/source_definitions/grant_definition
    
+    
    grant a private, non-custom sourceDefinition to a given workspace (grantSourceDefinitionToWorkspace)
    
     
    
 
 
@@ -7086,7 +9188,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceSearch SourceSearch (required)
    
+      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
 
       
    Body Parameter —  
    
 
@@ -7097,7 +9199,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceReadList
+      PrivateSourceDefinitionRead
       
     
    
 
@@ -7106,27 +9208,42 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sources" : [ {
-    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "connectionConfiguration" : {
-      "user" : "charles"
-    },
-    "name" : "name",
-    "icon" : "icon",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "sourceName" : "sourceName",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "connectionConfiguration" : {
-      "user" : "charles"
+  "sourceDefinition" : {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
     },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
     "name" : "name",
     "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
     "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "sourceName" : "sourceName",
-    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "maxSecondsBetweenMessages" : 0
+  },
+  "granted" : true
 }
    
 
     
    Produces
    
@@ -7139,40 +9256,31 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceReadList
+        PrivateSourceDefinitionRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
     
    422
    
     Input failed validation
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/update
    
-    
    Update a source (updateSource)
    
-    
    
-
-
-    
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
+    
    post /v1/source_definitions/list_latest
    
+    
    List the latest sourceDefinitions Airbyte supports (listLatestSourceDefinitions)
    
+    
    Guaranteed to retrieve the latest information on supported sources.
    
 
-    
    Request body
    
-    
    
-      
    SourceUpdate SourceUpdate (required)
    
 
-      
    Body Parameter —  
    
 
-    
      
 
 
 
 
     
    Return type
    
     
    
-      SourceRead
+      SourceDefinitionReadList
       
     
    
 
@@ -7181,15 +9289,75 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "connectionConfiguration" : {
-    "user" : "charles"
-  },
-  "name" : "name",
-  "icon" : "icon",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "sourceName" : "sourceName",
-  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "sourceDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
+    "name" : "name",
+    "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "maxSecondsBetweenMessages" : 0
+  }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
+    "name" : "name",
+    "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "maxSecondsBetweenMessages" : 0
+  } ]
 }
    
 
     
    Produces
    
@@ -7202,20 +9370,14 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+        SourceDefinitionReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/sources/write_discover_catalog_result
    
-    
    Should only called from worker, to write result from discover activity back to DB. (writeDiscoverCatalogResult)
    
+    
    post /v1/source_definitions/list_private
    
+    
    List all private, non-custom sourceDefinitions, and for each indicate whether the given workspace has a grant for using the definition. Used by admins to view and modify a given workspace's grants. (listPrivateSourceDefinitions)
    
     
    
 
 
@@ -7227,7 +9389,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDiscoverSchemaWriteRequestBody SourceDiscoverSchemaWriteRequestBody (required)
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -7238,7 +9400,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      DiscoverCatalogResult
+      PrivateSourceDefinitionReadList
       
     
    
 
@@ -7247,7 +9409,81 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "sourceDefinitions" : [ {
+    "sourceDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
+      "documentationUrl" : "https://openapi-generator.tech",
+      "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "sourceType" : "api",
+      "dockerRepository" : "dockerRepository",
+      "name" : "name",
+      "icon" : "icon",
+      "protocolVersion" : "protocolVersion",
+      "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "maxSecondsBetweenMessages" : 0
+    },
+    "granted" : true
+  }, {
+    "sourceDefinition" : {
+      "resourceRequirements" : {
+        "default" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        },
+        "jobSpecific" : [ {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        }, {
+          "resourceRequirements" : {
+            "cpu_limit" : "cpu_limit",
+            "memory_request" : "memory_request",
+            "memory_limit" : "memory_limit",
+            "cpu_request" : "cpu_request"
+          }
+        } ]
+      },
+      "documentationUrl" : "https://openapi-generator.tech",
+      "dockerImageTag" : "dockerImageTag",
+      "releaseDate" : "2000-01-23",
+      "sourceType" : "api",
+      "dockerRepository" : "dockerRepository",
+      "name" : "name",
+      "icon" : "icon",
+      "protocolVersion" : "protocolVersion",
+      "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+      "maxSecondsBetweenMessages" : 0
+    },
+    "granted" : true
+  } ]
 }
    
 
     
    Produces
    
@@ -7259,39 +9495,26 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful Operation
-        DiscoverCatalogResult
+    Successful operation
+        PrivateSourceDefinitionReadList
   
     
   
    
-  
    SourceDefinition
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/create_custom
    
-    
    Creates a custom sourceDefinition for the given workspace (createCustomSourceDefinition)
    
+    
    post /v1/source_definitions/list
    
+    
    List all the sourceDefinitions the current Airbyte deployment is configured to use (listSourceDefinitions)
    
     
    
 
 
-    
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Request body
    
-    
    
-      
    CustomSourceDefinitionCreate CustomSourceDefinitionCreate (optional)
    
-
-      
    Body Parameter —  
    
 
-    
      
 
 
 
 
     
    Return type
    
     
    
-      SourceDefinitionRead
+      SourceDefinitionReadList
       
     
    
 
@@ -7300,38 +9523,75 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
+  "sourceDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    }, {
-      "resourceRequirements" : {
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
+    "name" : "name",
+    "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "maxSecondsBetweenMessages" : 0
+  }, {
+    "resourceRequirements" : {
+      "default" : {
         "cpu_limit" : "cpu_limit",
         "memory_request" : "memory_request",
         "memory_limit" : "memory_limit",
         "cpu_request" : "cpu_request"
-      }
-    } ]
-  },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "sourceType" : "api",
-  "dockerRepository" : "dockerRepository",
-  "name" : "name",
-  "icon" : "icon",
-  "protocolVersion" : "protocolVersion",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
+    "name" : "name",
+    "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "maxSecondsBetweenMessages" : 0
+  } ]
 }
    
 
     
    Produces
    
@@ -7344,17 +9604,14 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceDefinitionRead
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+        SourceDefinitionReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/delete
    
-    
    Delete a source definition (deleteSourceDefinition)
    
+    
    post /v1/source_definitions/list_for_workspace
    
+    
    List all the sourceDefinitions the given workspace is configured to use (listSourceDefinitionsForWorkspace)
    
     
    
 
 
@@ -7366,7 +9623,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDefinitionIdRequestBody SourceDefinitionIdRequestBody (required)
    
+      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -7375,9 +9632,87 @@ 
    Request body
    
 
 
 
+    
    Return type
    
+    
    
+      SourceDefinitionReadList
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "sourceDefinitions" : [ {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
+    "name" : "name",
+    "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "maxSecondsBetweenMessages" : 0
+  }, {
+    "resourceRequirements" : {
+      "default" : {
+        "cpu_limit" : "cpu_limit",
+        "memory_request" : "memory_request",
+        "memory_limit" : "memory_limit",
+        "cpu_request" : "cpu_request"
+      },
+      "jobSpecific" : [ {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      }, {
+        "resourceRequirements" : {
+          "cpu_limit" : "cpu_limit",
+          "memory_request" : "memory_request",
+          "memory_limit" : "memory_limit",
+          "cpu_request" : "cpu_request"
+        }
+      } ]
+    },
+    "documentationUrl" : "https://openapi-generator.tech",
+    "dockerImageTag" : "dockerImageTag",
+    "releaseDate" : "2000-01-23",
+    "sourceType" : "api",
+    "dockerRepository" : "dockerRepository",
+    "name" : "name",
+    "icon" : "icon",
+    "protocolVersion" : "protocolVersion",
+    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "maxSecondsBetweenMessages" : 0
+  } ]
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -7387,22 +9722,16 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    204
    
-    The resource was deleted successfully.
-        
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    
    200
    
+    Successful operation
+        SourceDefinitionReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/get
    
-    
    Get source (getSourceDefinition)
    
+    
    post /v1/source_definitions/revoke_definition
    
+    
    revoke a grant to a private, non-custom sourceDefinition from a given workspace (revokeSourceDefinitionFromWorkspace)
    
     
    
 
 
@@ -7414,7 +9743,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDefinitionIdRequestBody SourceDefinitionIdRequestBody (required)
    
+      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
 
       
    Body Parameter —  
    
 
@@ -7423,50 +9752,9 @@ 
    Request body
    
 
 
 
-    
    Return type
    
-    
    
-      SourceDefinitionRead
-      
-    
    
 
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      }
-    }, {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      }
-    } ]
-  },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "sourceType" : "api",
-  "dockerRepository" : "dockerRepository",
-  "name" : "name",
-  "icon" : "icon",
-  "protocolVersion" : "protocolVersion",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -7476,9 +9764,9 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        SourceDefinitionRead
+    
    204
    
+    The resource was deleted successfully.
+        
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -7487,11 +9775,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/get_for_workspace
    
-    
    Get a sourceDefinition that is configured for the given workspace (getSourceDefinitionForWorkspace)
    
+    
    post /v1/source_definitions/update
    
+    
    Update a sourceDefinition (updateSourceDefinition)
    
     
    
 
 
@@ -7503,7 +9791,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
+      
    SourceDefinitionUpdate SourceDefinitionUpdate (optional)
    
 
       
    Body Parameter —  
    
 
@@ -7554,7 +9842,8 @@ 
    Example data
    
   "name" : "name",
   "icon" : "icon",
   "protocolVersion" : "protocolVersion",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "maxSecondsBetweenMessages" : 0
 }
 
     
    Produces
    
@@ -7576,77 +9865,74 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    SourceDefinitionSpecification
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/grant_definition
    
-    
    grant a private, non-custom sourceDefinition to a given workspace (grantSourceDefinitionToWorkspace)
    
+    
    post /v1/source_definition_specifications/get
    
+    
    Get specification for a SourceDefinition. (getSourceDefinitionSpecification)
    
     
    
 
 
     
    Consumes
    
-    This API call consumes the following media types via the Content-Type request header:
-    
      
-      
    • application/json
      • 
-    
    
-
-    
    Request body
    
-    
    
-      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
-
-      
    Body Parameter —  
    
-
-    
      
-
-
-
-
-    
    Return type
    
-    
    
-      PrivateSourceDefinitionRead
-      
-    
    
-
-    
-
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "sourceDefinition" : {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+    
    Return type
    
+    
    
+      SourceDefinitionSpecificationRead
+      
+    
    
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "documentationUrl" : "documentationUrl",
+  "connectionSpecification" : {
+    "user" : {
+      "type" : "string"
+    }
   },
-  "granted" : true
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "advancedAuth" : {
+    "predicateValue" : "predicateValue",
+    "oauthConfigSpecification" : { },
+    "predicateKey" : [ "predicateKey", "predicateKey" ],
+    "authFlowType" : "oauth2.0"
+  },
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
+    },
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
+    },
+    "succeeded" : true
+  }
 }
    
 
     
    Produces
    
@@ -7659,7 +9945,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        PrivateSourceDefinitionRead
+        SourceDefinitionSpecificationRead
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -7668,22 +9954,34 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/list_latest
    
-    
    List the latest sourceDefinitions Airbyte supports (listLatestSourceDefinitions)
    
-    
    Guaranteed to retrieve the latest information on supported sources.
    
+    
    post /v1/source_definition_specifications/get_for_source
    
+    
    Get specification for a source. (getSpecificationForSourceId)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
 
+    
    Request body
    
+    
    
+      
    SourceIdRequestBody SourceIdRequestBody (required)
    
 
+      
    Body Parameter —  
    
 
+    
      
 
 
 
 
     
    Return type
    
     
    
-      SourceDefinitionReadList
+      SourceDefinitionSpecificationRead
       
     
    
 
@@ -7692,73 +9990,37 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sourceDefinitions" : [ {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
+  "documentationUrl" : "documentationUrl",
+  "connectionSpecification" : {
+    "user" : {
+      "type" : "string"
+    }
+  },
+  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "advancedAuth" : {
+    "predicateValue" : "predicateValue",
+    "oauthConfigSpecification" : { },
+    "predicateKey" : [ "predicateKey", "predicateKey" ],
+    "authFlowType" : "oauth2.0"
+  },
+  "jobInfo" : {
+    "createdAt" : 0,
+    "connectorConfigurationUpdated" : false,
+    "configId" : "configId",
+    "endedAt" : 6,
+    "failureReason" : {
+      "retryable" : true,
+      "stacktrace" : "stacktrace",
+      "internalMessage" : "internalMessage",
+      "externalMessage" : "externalMessage",
+      "timestamp" : 1
     },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "logs" : {
+      "logLines" : [ "logLines", "logLines" ]
     },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+    "succeeded" : true
+  }
 }
    
 
     
    Produces
    
@@ -7771,14 +10033,21 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceDefinitionReadList
+        SourceDefinitionSpecificationRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    SourceOauth
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/list_private
    
-    
    List all private, non-custom sourceDefinitions, and for each indicate whether the given workspace has a grant for using the definition. Used by admins to view and modify a given workspace's grants. (listPrivateSourceDefinitions)
    
+    
    post /v1/source_oauths/complete_oauth
    
+    
    Given a source def ID generate an access/refresh token etc. (completeSourceOAuth)
    
     
    
 
 
@@ -7790,7 +10059,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
+      
    CompleteSourceOauthRequest CompleteSourceOauthRequest (required)
    
 
       
    Body Parameter —  
    
 
@@ -7801,88 +10070,20 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      PrivateSourceDefinitionReadList
+      CompleteOAuthResponse
       
     
    
-
-    
-
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "sourceDefinitions" : [ {
-    "sourceDefinition" : {
-      "resourceRequirements" : {
-        "default" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        },
-        "jobSpecific" : [ {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        }, {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        } ]
-      },
-      "documentationUrl" : "https://openapi-generator.tech",
-      "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "sourceType" : "api",
-      "dockerRepository" : "dockerRepository",
-      "name" : "name",
-      "icon" : "icon",
-      "protocolVersion" : "protocolVersion",
-      "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-    },
-    "granted" : true
-  }, {
-    "sourceDefinition" : {
-      "resourceRequirements" : {
-        "default" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        },
-        "jobSpecific" : [ {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        }, {
-          "resourceRequirements" : {
-            "cpu_limit" : "cpu_limit",
-            "memory_request" : "memory_request",
-            "memory_limit" : "memory_limit",
-            "cpu_request" : "cpu_request"
-          }
-        } ]
-      },
-      "documentationUrl" : "https://openapi-generator.tech",
-      "dockerImageTag" : "dockerImageTag",
-      "releaseDate" : "2000-01-23",
-      "sourceType" : "api",
-      "dockerRepository" : "dockerRepository",
-      "name" : "name",
-      "icon" : "icon",
-      "protocolVersion" : "protocolVersion",
-      "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-    },
-    "granted" : true
-  } ]
+
+    
+
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "auth_payload" : {
+    "key" : ""
+  },
+  "request_error" : "request_error",
+  "request_succeeded" : true
 }
    
 
     
    Produces
    
@@ -7895,25 +10096,43 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        PrivateSourceDefinitionReadList
+        CompleteOAuthResponse
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/list
    
-    
    List all the sourceDefinitions the current Airbyte deployment is configured to use (listSourceDefinitions)
    
+    
    post /v1/source_oauths/get_consent_url
    
+    
    Given a source connector definition ID, return the URL to the consent screen where to redirect the user to. (getSourceOAuthConsent)
    
     
    
 
 
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    SourceOauthConsentRequest SourceOauthConsentRequest (required)
    
+
+      
    Body Parameter —  
    
 
+    
      
 
 
 
 
     
    Return type
    
     
    
-      SourceDefinitionReadList
+      OAuthConsentRead
       
     
    
 
@@ -7922,73 +10141,7 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "sourceDefinitions" : [ {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
+  "consentUrl" : "consentUrl"
 }
    
 
     
    Produces
    
@@ -8001,14 +10154,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceDefinitionReadList
+        OAuthConsentRead
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+    
    422
    
+    Input failed validation
+        InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/list_for_workspace
    
-    
    List all the sourceDefinitions the given workspace is configured to use (listSourceDefinitionsForWorkspace)
    
+    
    post /v1/source_oauths/revoke
    
+    
    Given a source definition ID and workspace ID revoke access/refresh token etc. (revokeSourceOAuthTokens)
    
     
    
 
 
@@ -8020,7 +10179,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    WorkspaceIdRequestBody WorkspaceIdRequestBody (optional)
    
+      
    RevokeSourceOauthTokensRequest RevokeSourceOauthTokensRequest (required)
    
 
       
    Body Parameter —  
    
 
@@ -8029,85 +10188,9 @@ 
    Request body
    
 
 
 
-    
    Return type
    
-    
    
-      SourceDefinitionReadList
-      
-    
    
 
     
 
-    
    Example data
    
-    
    Content-Type: application/json
    
-    
    {
-  "sourceDefinitions" : [ {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  }, {
-    "resourceRequirements" : {
-      "default" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
-      },
-      "jobSpecific" : [ {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      }, {
-        "resourceRequirements" : {
-          "cpu_limit" : "cpu_limit",
-          "memory_request" : "memory_request",
-          "memory_limit" : "memory_limit",
-          "cpu_request" : "cpu_request"
-        }
-      } ]
-    },
-    "documentationUrl" : "https://openapi-generator.tech",
-    "dockerImageTag" : "dockerImageTag",
-    "releaseDate" : "2000-01-23",
-    "sourceType" : "api",
-    "dockerRepository" : "dockerRepository",
-    "name" : "name",
-    "icon" : "icon",
-    "protocolVersion" : "protocolVersion",
-    "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
-  } ]
-}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -8119,14 +10202,20 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceDefinitionReadList
+        
+    
    400
    
+    Exception occurred; see message for details.
+        KnownExceptionInfo
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/revoke_definition
    
-    
    revoke a grant to a private, non-custom sourceDefinition from a given workspace (revokeSourceDefinitionFromWorkspace)
    
+    
    post /v1/source_oauths/oauth_params/create
    
+    
    Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know about these variables. (setInstancewideSourceOauthParams)
    
     
    
 
 
@@ -8138,7 +10227,56 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
+      
    SetInstancewideSourceOauthParamsRequestBody SetInstancewideSourceOauthParamsRequestBody (required)
    
+
+      
    Body Parameter —  
    
+
+    
      
+
+
+
+
+
+    
+
+
+    
    Produces
    
+    This API call produces the following media types according to the Accept request header;
+    the media type will be conveyed by the Content-Type response header.
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Responses
    
+    
    200
    
+    Successful
+        
+    
    400
    
+    Exception occurred; see message for details.
+        KnownExceptionInfo
+    
    404
    
+    Object with given id was not found.
+        NotFoundKnownExceptionInfo
+  
     
+  
    
+  
    State
    
+  
    
+    
    
+    Up
+    
    post /v1/state/create_or_update
    
+    
    Create or update the state for a connection. (createOrUpdateState)
    
+    
    
+
+
+    
    Consumes
    
+    This API call consumes the following media types via the Content-Type request header:
+    
      
+      
    • application/json
      • 
+    
    
+
+    
    Request body
    
+    
    
+      
    ConnectionStateCreateOrUpdate ConnectionStateCreateOrUpdate (required)
    
 
       
    Body Parameter —  
    
 
@@ -8147,9 +10285,43 @@ 
    Request body
    
 
 
 
+    
    Return type
    
+    
    
+      ConnectionState
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "globalState" : {
+    "streamStates" : [ {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      }
+    }, {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
+      }
+    } ]
+  },
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamState" : [ {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  }, {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  } ]
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -8159,9 +10331,9 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    204
    
-    The resource was deleted successfully.
-        
+    
    200
    
+    Successful operation
+        ConnectionState
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -8170,11 +10342,11 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_definitions/update
    
-    
    Update a sourceDefinition (updateSourceDefinition)
    
+    
    post /v1/state/get
    
+    
    Fetch the current state for a connection. (getState)
    
     
    
 
 
@@ -8186,7 +10358,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDefinitionUpdate SourceDefinitionUpdate (optional)
    
+      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
 
       
    Body Parameter —  
    
 
@@ -8197,7 +10369,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceDefinitionRead
+      ConnectionState
       
     
    
 
@@ -8206,38 +10378,31 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "resourceRequirements" : {
-    "default" : {
-      "cpu_limit" : "cpu_limit",
-      "memory_request" : "memory_request",
-      "memory_limit" : "memory_limit",
-      "cpu_request" : "cpu_request"
-    },
-    "jobSpecific" : [ {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
+  "globalState" : {
+    "streamStates" : [ {
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
       }
     }, {
-      "resourceRequirements" : {
-        "cpu_limit" : "cpu_limit",
-        "memory_request" : "memory_request",
-        "memory_limit" : "memory_limit",
-        "cpu_request" : "cpu_request"
+      "streamDescriptor" : {
+        "name" : "name",
+        "namespace" : "namespace"
       }
     } ]
   },
-  "documentationUrl" : "https://openapi-generator.tech",
-  "dockerImageTag" : "dockerImageTag",
-  "releaseDate" : "2000-01-23",
-  "sourceType" : "api",
-  "dockerRepository" : "dockerRepository",
-  "name" : "name",
-  "icon" : "icon",
-  "protocolVersion" : "protocolVersion",
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamState" : [ {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  }, {
+    "streamDescriptor" : {
+      "name" : "name",
+      "namespace" : "namespace"
+    }
+  } ]
 }
    
 
     
    Produces
    
@@ -8250,7 +10415,7 @@ 
    Produces
    
     
    Responses
    
     
    200
    
     Successful operation
-        SourceDefinitionRead
+        ConnectionState
     
    404
    
     Object with given id was not found.
         NotFoundKnownExceptionInfo
@@ -8259,12 +10424,12 @@ 
    422
    
         InvalidInputExceptionInfo
   
     
   
    
-  
    SourceDefinitionSpecification
    
-  
    
+  
    StreamStatuses
    
+  
    
     
    
     Up
-    
    post /v1/source_definition_specifications/get
    
-    
    Get specification for a SourceDefinition. (getSourceDefinitionSpecification)
    
+    
    post /v1/stream_statuses/create
    
+    
    Creates a stream status. (createStreamStatus)
    
     
    
 
 
@@ -8276,7 +10441,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceDefinitionIdWithWorkspaceId SourceDefinitionIdWithWorkspaceId (required)
    
+      
    StreamStatusCreateRequestBody StreamStatusCreateRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -8287,7 +10452,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      SourceDefinitionSpecificationRead
+      StreamStatusRead
       
     
    
 
@@ -8296,38 +10461,14 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "documentationUrl" : "documentationUrl",
-  "connectionSpecification" : {
-    "user" : {
-      "type" : "string"
-    }
-  },
-  "sourceDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "advancedAuth" : {
-    "predicateValue" : "predicateValue",
-    "oauthConfigSpecification" : { },
-    "predicateKey" : [ "predicateKey", "predicateKey" ],
-    "authFlowType" : "oauth2.0"
-  },
-  "authSpecification" : {
-    "auth_type" : "oauth2.0",
-    "oauth2Specification" : {
-      "oauthFlowOutputParameters" : [ [ "oauthFlowOutputParameters", "oauthFlowOutputParameters" ], [ "oauthFlowOutputParameters", "oauthFlowOutputParameters" ] ],
-      "rootObject" : [ "path", 1 ],
-      "oauthFlowInitParameters" : [ [ "oauthFlowInitParameters", "oauthFlowInitParameters" ], [ "oauthFlowInitParameters", "oauthFlowInitParameters" ] ]
-    }
-  },
-  "jobInfo" : {
-    "createdAt" : 0,
-    "connectorConfigurationUpdated" : false,
-    "configId" : "configId",
-    "endedAt" : 6,
-    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    },
-    "succeeded" : true
-  }
+  "attemptNumber" : 0,
+  "jobId" : 6,
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamNamespace" : "streamNamespace",
+  "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamName" : "streamName",
+  "transitionedAt" : 1,
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -8338,23 +10479,16 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful operation
-        SourceDefinitionSpecificationRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    
    201
    
+    Successfully created stream status.
+        StreamStatusRead
   
     
   
    
-  
    SourceOauth
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_oauths/complete_oauth
    
-    
    Given a source def ID generate an access/refresh token etc. (completeSourceOAuth)
    
+    
    post /v1/stream_statuses/list
    
+    
    Gets a list of stream statuses filtered by parameters (with AND semantics). (getStreamStatuses)
    
     
    
 
 
@@ -8366,7 +10500,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    CompleteSourceOauthRequest CompleteSourceOauthRequest (required)
    
+      
    StreamStatusListRequestBody StreamStatusListRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -8377,12 +10511,35 @@ 
    Request body
    
 
     
    Return type
    
     
    
+      StreamStatusReadList
       
-      map[String, oas_any_type_not_mapped]
     
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "streamStatuses" : [ {
+    "attemptNumber" : 0,
+    "jobId" : 6,
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamNamespace" : "streamNamespace",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamName" : "streamName",
+    "transitionedAt" : 1,
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  }, {
+    "attemptNumber" : 0,
+    "jobId" : 6,
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamNamespace" : "streamNamespace",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamName" : "streamName",
+    "transitionedAt" : 1,
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+  } ]
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -8393,21 +10550,15 @@ 
    Produces
    
 
     
    Responses
    
     
    200
    
-    Successful operation
-        
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    Successfully queried stream statuses.
+        StreamStatusReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/source_oauths/get_consent_url
    
-    
    Given a source connector definition ID, return the URL to the consent screen where to redirect the user to. (getSourceOAuthConsent)
    
+    
    post /v1/stream_statuses/update
    
+    
    Updates a stream status. (updateStreamStatus)
    
     
    
 
 
@@ -8419,7 +10570,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SourceOauthConsentRequest SourceOauthConsentRequest (required)
    
+      
    StreamStatusUpdateRequestBody StreamStatusUpdateRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -8430,7 +10581,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      OAuthConsentRead
+      StreamStatusRead
       
     
    
 
@@ -8439,7 +10590,14 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "consentUrl" : "consentUrl"
+  "attemptNumber" : 0,
+  "jobId" : 6,
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamNamespace" : "streamNamespace",
+  "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamName" : "streamName",
+  "transitionedAt" : 1,
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -8450,22 +10608,20 @@ 
    Produces
    
     
 
     
    Responses
    
+    
    201
    
+    Successfully created stream status.
+        StreamStatusRead
     
    200
    
-    Successful operation
-        OAuthConsentRead
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    Successfully updated stream status.
+        StreamStatusRead
   
     
   
    
-  
    
+  
    Streams
    
+  
    
     
    
     Up
-    
    post /v1/source_oauths/oauth_params/create
    
-    
    Sets instancewide variables to be used for the oauth flow when creating this source. When set, these variables will be injected into a connector's configuration before any interaction with the connector image itself. This enables running oauth flows with consistent variables e.g: the company's Google Ads developer_token, client_id, and client_secret without the user having to know about these variables. (setInstancewideSourceOauthParams)
    
+    
    post /v1/stream_statuses/create
    
+    
    Creates a stream status. (createStreamStatus)
    
     
    
 
 
@@ -8477,7 +10633,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    SetInstancewideSourceOauthParamsRequestBody SetInstancewideSourceOauthParamsRequestBody (required)
    
+      
    StreamStatusCreateRequestBody StreamStatusCreateRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -8486,9 +10642,26 @@ 
    Request body
    
 
 
 
+    
    Return type
    
+    
    
+      StreamStatusRead
+      
+    
    
 
     
 
+    
    Example data
    
+    
    Content-Type: application/json
    
+    
    {
+  "attemptNumber" : 0,
+  "jobId" : 6,
+  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamNamespace" : "streamNamespace",
+  "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamName" : "streamName",
+  "transitionedAt" : 1,
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
+}
    
 
     
    Produces
    
     This API call produces the following media types according to the Accept request header;
@@ -8498,23 +10671,16 @@ 
    Produces
    
     
 
     
    Responses
    
-    
    200
    
-    Successful
-        
-    
    400
    
-    Exception occurred; see message for details.
-        KnownExceptionInfo
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
+    
    201
    
+    Successfully created stream status.
+        StreamStatusRead
   
     
   
    
-  
    State
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/state/create_or_update
    
-    
    Create or update the state for a connection. (createOrUpdateState)
    
+    
    post /v1/stream_statuses/list
    
+    
    Gets a list of stream statuses filtered by parameters (with AND semantics). (getStreamStatuses)
    
     
    
 
 
@@ -8526,7 +10692,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionStateCreateOrUpdate ConnectionStateCreateOrUpdate (required)
    
+      
    StreamStatusListRequestBody StreamStatusListRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -8537,7 +10703,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      ConnectionState
+      StreamStatusReadList
       
     
    
 
@@ -8546,30 +10712,24 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "globalState" : {
-    "streamStates" : [ {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    }, {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    } ]
-  },
-  "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "streamState" : [ {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
+  "streamStatuses" : [ {
+    "attemptNumber" : 0,
+    "jobId" : 6,
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamNamespace" : "streamNamespace",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamName" : "streamName",
+    "transitionedAt" : 1,
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   }, {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
+    "attemptNumber" : 0,
+    "jobId" : 6,
+    "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamNamespace" : "streamNamespace",
+    "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+    "streamName" : "streamName",
+    "transitionedAt" : 1,
+    "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ]
 }
    
 
@@ -8577,26 +10737,20 @@ 
    Produces
    
     This API call produces the following media types according to the Accept request header;
     the media type will be conveyed by the Content-Type response header.
     
      
-      
    • application/json
      • 
-    
    
-
-    
    Responses
    
-    
    200
    
-    Successful operation
-        ConnectionState
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+      
  • application/json
    • 
+    
+
+    
    Responses
    
+    
    200
    
+    Successfully queried stream statuses.
+        StreamStatusReadList
   
     
   
    
-  
    
+  
    
     
    
     Up
-    
    post /v1/state/get
    
-    
    Fetch the current state for a connection. (getState)
    
+    
    post /v1/stream_statuses/update
    
+    
    Updates a stream status. (updateStreamStatus)
    
     
    
 
 
@@ -8608,7 +10762,7 @@ 
    Consumes
    
 
     
    Request body
    
     
    
-      
    ConnectionIdRequestBody ConnectionIdRequestBody (required)
    
+      
    StreamStatusUpdateRequestBody StreamStatusUpdateRequestBody (optional)
    
 
       
    Body Parameter —  
    
 
@@ -8619,7 +10773,7 @@ 
    Request body
    
 
     
    Return type
    
     
    
-      ConnectionState
+      StreamStatusRead
       
     
    
 
@@ -8628,31 +10782,14 @@ 
    Return type
    
     
    Example data
    
     
    Content-Type: application/json
    
     
    {
-  "globalState" : {
-    "streamStates" : [ {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    }, {
-      "streamDescriptor" : {
-        "name" : "name",
-        "namespace" : "namespace"
-      }
-    } ]
-  },
+  "attemptNumber" : 0,
+  "jobId" : 6,
   "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "streamState" : [ {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
-  }, {
-    "streamDescriptor" : {
-      "name" : "name",
-      "namespace" : "namespace"
-    }
-  } ]
+  "streamNamespace" : "streamNamespace",
+  "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
+  "streamName" : "streamName",
+  "transitionedAt" : 1,
+  "workspaceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
 }
    
 
     
    Produces
    
@@ -8663,15 +10800,12 @@ 
    Produces
    
     
 
     
    Responses
    
+    
    201
    
+    Successfully created stream status.
+        StreamStatusRead
     
    200
    
-    Successful operation
-        ConnectionState
-    
    404
    
-    Object with given id was not found.
-        NotFoundKnownExceptionInfo
-    
    422
    
-    Input failed validation
-        InvalidInputExceptionInfo
+    Successfully updated stream status.
+        StreamStatusRead
   
     
   
    
   
    WebBackend
    
@@ -8933,6 +11067,7 @@ 
    Example data
    
   } ],
   "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
   "notifySchemaChanges" : true,
+  "notifySchemaChangesByEmail" : true,
   "namespaceFormat" : "${SOURCE_NAMESPACE}",
   "scheduleData" : {
     "cron" : {
@@ -9183,6 +11318,7 @@ 
    Example data
    
   } ],
   "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
   "notifySchemaChanges" : true,
+  "notifySchemaChangesByEmail" : true,
   "namespaceFormat" : "${SOURCE_NAMESPACE}",
   "scheduleData" : {
     "cron" : {
@@ -9661,6 +11797,7 @@ 
    Example data
    
   } ],
   "catalogId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
   "notifySchemaChanges" : true,
+  "notifySchemaChangesByEmail" : true,
   "namespaceFormat" : "${SOURCE_NAMESPACE}",
   "scheduleData" : {
     "cron" : {
@@ -9795,6 +11932,50 @@ 
    Example data
    
     "name" : "name",
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ],
+  "notificationSettings" : {
+    "sendOnConnectionUpdateActionRequired" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabledWarning" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSuccess" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnFailure" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnConnectionUpdate" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabled" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    }
+  },
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -9931,6 +12112,50 @@ 
    Example data
    
     "name" : "name",
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ],
+  "notificationSettings" : {
+    "sendOnConnectionUpdateActionRequired" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabledWarning" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSuccess" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnFailure" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnConnectionUpdate" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabled" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    }
+  },
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -10022,6 +12247,50 @@ 
    Example data
    
     "name" : "name",
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ],
+  "notificationSettings" : {
+    "sendOnConnectionUpdateActionRequired" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabledWarning" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSuccess" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnFailure" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnConnectionUpdate" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabled" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    }
+  },
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -10113,6 +12382,50 @@ 
    Example data
    
     "name" : "name",
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ],
+  "notificationSettings" : {
+    "sendOnConnectionUpdateActionRequired" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabledWarning" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSuccess" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnFailure" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnConnectionUpdate" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabled" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    }
+  },
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -10193,6 +12506,50 @@ 
    Example data
    
       "name" : "name",
       "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     } ],
+    "notificationSettings" : {
+      "sendOnConnectionUpdateActionRequired" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnSyncDisabledWarning" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnSuccess" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnFailure" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnConnectionUpdate" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnSyncDisabled" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      }
+    },
     "displaySetupWizard" : true,
     "initialSetupComplete" : true,
     "anonymousDataCollection" : true,
@@ -10228,6 +12585,50 @@ 
    Example data
    
       "name" : "name",
       "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
     } ],
+    "notificationSettings" : {
+      "sendOnConnectionUpdateActionRequired" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnSyncDisabledWarning" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnSuccess" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnFailure" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnConnectionUpdate" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      },
+      "sendOnSyncDisabled" : {
+        "slackConfiguration" : {
+          "webhook" : "webhook"
+        },
+        "customerioConfiguration" : "{}",
+        "notificationType" : [ null, null ]
+      }
+    },
     "displaySetupWizard" : true,
     "initialSetupComplete" : true,
     "anonymousDataCollection" : true,
@@ -10314,6 +12715,50 @@ 
    Example data
    
     "name" : "name",
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ],
+  "notificationSettings" : {
+    "sendOnConnectionUpdateActionRequired" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabledWarning" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSuccess" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnFailure" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnConnectionUpdate" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabled" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    }
+  },
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -10450,6 +12895,50 @@ 
    Example data
    
     "name" : "name",
     "id" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91"
   } ],
+  "notificationSettings" : {
+    "sendOnConnectionUpdateActionRequired" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabledWarning" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSuccess" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnFailure" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnConnectionUpdate" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    },
+    "sendOnSyncDisabled" : {
+      "slackConfiguration" : {
+        "webhook" : "webhook"
+      },
+      "customerioConfiguration" : "{}",
+      "notificationType" : [ null, null ]
+    }
+  },
   "displaySetupWizard" : true,
   "initialSetupComplete" : true,
   "anonymousDataCollection" : true,
@@ -10504,16 +12993,15 @@ 
    Models
    
   
    Table of Contents
    
   
      
     
    1. ActorCatalogWithUpdatedAt - 
      2. 
+    
    2. ActorDefinitionRequestBody - 
      3. 
     
    3. ActorDefinitionResourceRequirements - 
      4. 
+    
    4. ActorType - 
      5. 
     
    5. AdvancedAuth - 
      6. 
     
    6. AirbyteCatalog - 
      7. 
     
    7. AirbyteStream - 
      8. 
     
    8. AirbyteStreamAndConfiguration - 
      9. 
     
    9. AirbyteStreamConfiguration - 
      10. 
-    
    10. AttemptFailureOrigin - 
      11. 
-    
    11. AttemptFailureReason - 
      12. 
     
    12. AttemptFailureSummary - 
      13. 
-    
    13. AttemptFailureType - 
      14. 
     
    14. AttemptInfoRead - 
      15. 
     
    15. AttemptNormalizationStatusRead - 
      16. 
     
    16. AttemptNormalizationStatusReadList - 
      17. 
@@ -10522,11 +13010,11 @@ 
      Table of Contents
      
     
    17. AttemptStatus - 
      18. 
     
    18. AttemptStreamStats - 
      19. 
     
    19. AttemptSyncConfig - 
      20. 
-    
    20. AuthSpecification - 
      21. 
     
    21. CatalogDiff - 
      22. 
     
    22. CheckConnectionRead - 
      23. 
     
    23. CheckOperationRead - 
      24. 
     
    24. CompleteDestinationOAuthRequest - 
      25. 
+    
    25. CompleteOAuthResponse - 
      26. 
     
    26. CompleteSourceOauthRequest - 
      27. 
     
    27. ConnectionCreate - 
      28. 
     
    28. ConnectionIdRequestBody - 
      29. 
@@ -10542,7 +13030,16 @@ 
      Table of Contents
      
     
    29. ConnectionStateCreateOrUpdate - 
      30. 
     
    30. ConnectionStateType - 
      31. 
     
    31. ConnectionStatus - 
      32. 
+    
    32. ConnectionStream - 
      33. 
+    
    33. ConnectionStreamRequestBody - 
      34. 
     
    34. ConnectionUpdate - 
      35. 
+    
    35. ConnectorBuilderProjectDetails - 
      36. 
+    
    36. ConnectorBuilderProjectDetailsRead - 
      37. 
+    
    37. ConnectorBuilderProjectIdWithWorkspaceId - 
      38. 
+    
    38. ConnectorBuilderProjectRead - 
      39. 
+    
    39. ConnectorBuilderProjectReadList - 
      40. 
+    
    40. ConnectorBuilderProjectWithWorkspaceId - 
      41. 
+    
    41. ConnectorBuilderPublishRequestBody - 
      42. 
     
    42. CustomDestinationDefinitionCreate - 
      43. 
     
    43. CustomSourceDefinitionCreate - 
      44. 
     
    44. DataType - 
      45. 
@@ -10551,6 +13048,11 @@ 
      Table of Contents
      
     
    45. DbMigrationReadList - 
      46. 
     
    46. DbMigrationRequestBody - 
      47. 
     
    47. DbMigrationState - 
      48. 
+    
    48. DeclarativeManifestRead - 
      49. 
+    
    49. DeclarativeManifestVersionRead - 
      50. 
+    
    50. DeclarativeManifestsReadList - 
      51. 
+    
    51. DeclarativeSourceDefinitionCreateManifestRequestBody - 
      52. 
+    
    52. DeclarativeSourceManifest - 
      53. 
     
    53. DestinationCloneConfiguration - 
      54. 
     
    54. DestinationCloneRequestBody - 
      55. 
     
    55. DestinationCoreConfig - 
      56. 
@@ -10571,6 +13073,10 @@ 
      Table of Contents
      
     
    56. DestinationSyncMode - 
      57. 
     
    57. DestinationUpdate - 
      58. 
     
    58. DiscoverCatalogResult - 
      59. 
+    
    59. ExistingConnectorBuilderProjectWithWorkspaceId - 
      60. 
+    
    60. FailureOrigin - 
      61. 
+    
    61. FailureReason - 
      62. 
+    
    62. FailureType - 
      63. 
     
    63. FieldAdd - 
      64. 
     
    64. FieldRemove - 
      65. 
     
    65. FieldSchemaUpdate - 
      66. 
@@ -10598,6 +13104,9 @@ 
      Table of Contents
      
     
    66. JobTypeResourceLimit - 
      67. 
     
    67. JobWithAttemptsRead - 
      68. 
     
    68. KnownExceptionInfo - 
      69. 
+    
    69. ListConnectionsForWorkspacesRequestBody - 
      70. 
+    
    70. ListDeclarativeManifestsRequestBody - 
      71. 
+    
    71. ListResourcesForWorkspacesRequestBody - 
      72. 
     
    72. LogRead - 
      73. 
     
    73. LogType - 
      74. 
     
    74. LogsRequestBody - 
      75. 
@@ -10606,9 +13115,10 @@ 
      Table of Contents
      
     
    75. NormalizationDestinationDefinitionConfig - 
      76. 
     
    76. NotFoundKnownExceptionInfo - 
      77. 
     
    77. Notification - 
      78. 
+    
    78. NotificationItem - 
      79. 
     
    79. NotificationRead - 
      80. 
+    
    80. NotificationSettings - 
      81. 
     
    81. NotificationType - 
      82. 
-    
    82. OAuth2Specification - 
      83. 
     
    83. OAuthConfigSpecification - 
      84. 
     
    84. OAuthConsentRead - 
      85. 
     
    85. OperationCreate - 
      86. 
@@ -10623,6 +13133,8 @@ 
      Table of Contents
      
     
    86. OperatorWebhook - 
      87. 
     
    87. OperatorWebhook_dbtCloud - 
      88. 
     
    88. Pagination - 
      89. 
+    
    89. PartialDestinationUpdate - 
      90. 
+    
    90. PartialSourceUpdate - 
      91. 
     
    91. PrivateDestinationDefinitionRead - 
      92. 
     
    92. PrivateDestinationDefinitionReadList - 
      93. 
     
    93. PrivateSourceDefinitionRead - 
      94. 
@@ -10630,6 +13142,7 @@ 
      Table of Contents
      
     
    94. ReleaseStage - 
      95. 
     
    95. ResetConfig - 
      96. 
     
    96. ResourceRequirements - 
      97. 
+    
    97. RevokeSourceOauthTokensRequest - 
      98. 
     
    98. SaveAttemptSyncConfigRequestBody - 
      99. 
     
    99. SaveStatsRequestBody - 
      100. 
     
    100. SchemaChange - 
      101. 
@@ -10639,11 +13152,13 @@ 
      Table of Contents
      
     
    101. SetWorkflowInAttemptRequestBody - 
      102. 
     
    102. SlackNotificationConfiguration - 
      103. 
     
    103. SlugRequestBody - 
      104. 
+    
    104. SourceAutoPropagateChange - 
      105. 
     
    105. SourceCloneConfiguration - 
      106. 
     
    106. SourceCloneRequestBody - 
      107. 
     
    107. SourceCoreConfig - 
      108. 
     
    108. SourceCreate - 
      109. 
     
    109. SourceDefinitionCreate - 
      110. 
+    
    110. SourceDefinitionIdBody - 
      111. 
     
    111. SourceDefinitionIdRequestBody - 
      112. 
     
    112. SourceDefinitionIdWithWorkspaceId - 
      113. 
     
    113. SourceDefinitionRead - 
      114. 
@@ -10662,9 +13177,18 @@ 
      Table of Contents
      
     
    114. SourceUpdate - 
      115. 
     
    115. StreamDescriptor - 
      116. 
     
    116. StreamState - 
      117. 
+    
    117. StreamStatusCreateRequestBody - 
      118. 
+    
    118. StreamStatusIncompleteRunCause - 
      119. 
+    
    119. StreamStatusJobType - 
      120. 
+    
    120. StreamStatusListRequestBody - 
      121. 
+    
    121. StreamStatusRead - 
      122. 
+    
    122. StreamStatusReadList - 
      123. 
+    
    123. StreamStatusRunState - 
      124. 
+    
    124. StreamStatusUpdateRequestBody - 
      125. 
     
    125. StreamTransform - 
      126. 
     
    126. SyncMode - 
      127. 
     
    127. SynchronousJobRead - 
      128. 
+    
    128. UpdateActiveManifestRequestBody - 
      129. 
     
    129. UploadRead - 
      130. 
     
    130. WebBackendCheckUpdatesRead - 
      131. 
     
    131. WebBackendConnectionCreate - 
      132. 
@@ -10684,6 +13208,7 @@ 
      Table of Contents
      
     
    132. WorkspaceCreate - 
      133. 
     
    133. WorkspaceGiveFeedback - 
      134. 
     
    134. WorkspaceIdRequestBody - 
      135. 
+    
    135. WorkspaceOverrideOauthParamsRequestBody - 
      136. 
     
    136. WorkspaceRead - 
      137. 
     
    137. WorkspaceReadList - 
      138. 
     
    138. WorkspaceUpdate - 
      139. 
@@ -10698,6 +13223,14 @@ 
      ActorCatalogWithUpdatedAt -
 
      catalog (optional)
      Object  
      
     
      
   
    
+  
    
+    
    ActorDefinitionRequestBody -  Up
    
+    
    
+    
    
+      
    actorDefinitionId 
    UUID  format: uuid
    
+
    actorType 
    ActorType  
    
+    
      
+  
    
   
    
     
    ActorDefinitionResourceRequirements -  Up
    
     
    actor definition specific resource requirements. if default is set, these are the requirements that should be set for ALL jobs run for this actor definition. it is overriden by the job type specific configurations. if not set, the platform will use defaults. these values will be overriden by configuration at the connection level.
    
@@ -10706,6 +13239,12 @@ 
    ActorDefinitionResourceR
 
    jobSpecific (optional)
    array[JobTypeResourceLimit]  
    
     
      
   
    
+  
    
+    
    ActorType -  Up
    
+    
    
+    
    
+          
      
+  
    
   
    
     
    AdvancedAuth -  Up
    
     
    
@@ -10761,39 +13300,14 @@ 
    AirbyteStreamConfiguration
 
    selectedFields (optional)
    array[SelectedFieldInfo] Paths to the fields that will be included in the configured catalog. This must be set if fieldSelectedEnabled is set. An empty list indicates that no properties will be included. 
    
     
      
   
    
-  
    
-    
    AttemptFailureOrigin -  Up
    
-    
    Indicates where the error originated. If not set, the origin of error is not well known.
    
-    
    
-          
      
-  
    
-  
    
-    
    AttemptFailureReason -  Up
    
-    
    
-    
    
-      
    failureOrigin (optional)
    AttemptFailureOrigin  
    
-
    failureType (optional)
    AttemptFailureType  
    
-
    externalMessage (optional)
    String  
    
-
    internalMessage (optional)
    String  
    
-
    stacktrace (optional)
    String  
    
-
    retryable (optional)
    Boolean True if it is known that retrying may succeed, e.g. for a transient failure. False if it is known that a retry will not succeed, e.g. for a configuration issue. If not set, retryable status is not well known. 
    
-
    timestamp 
    Long  format: int64
    
-    
      
-  
    
   
    
     
    AttemptFailureSummary -  Up
    
     
    
     
    
-      
    failures 
    array[AttemptFailureReason]  
    
+      
    failures 
    array[FailureReason]  
    
 
    partialSuccess (optional)
    Boolean True if the number of committed records for this attempt was greater than 0. False if 0 records were committed. If not set, the number of committed records is unknown. 
    
     
      
   
    
-  
    
-    
    AttemptFailureType -  Up
    
-    
    Categorizes well known errors into types for programmatic handling. If not set, the type of error is not well known.
    
-    
    
-          
      
-  
    
   
    
     
    AttemptInfoRead -  Up
    
     
    
@@ -10842,6 +13356,7 @@ 
    AttemptStats -  recordsEmitted (optional)
    Long  format: int64
    
 
    bytesEmitted (optional)
    Long  format: int64
    
 
    stateMessagesEmitted (optional)
    Long  format: int64
    
+
    bytesCommitted (optional)
    Long  format: int64
    
 
    recordsCommitted (optional)
    Long  format: int64
    
 
    estimatedRecords (optional)
    Long  format: int64
    
 
    estimatedBytes (optional)
    Long  format: int64
    
@@ -10868,17 +13383,7 @@ 
    AttemptSyncConfig -  
       
    sourceConfiguration 
    SourceConfiguration  
    
 
    destinationConfiguration 
    DestinationConfiguration  
    
-
    state (optional)
    ConnectionState  
    
-    
      
-  
    
-  
    
-    
    AuthSpecification -  Up
    
-    
    
-    
    
-      
    auth_type (optional)
    String  
    
-        
    Enum:
    
-        
    oauth2.0
    
-
    oauth2Specification (optional)
    OAuth2Specification  
    
+
    state (optional)
    ConnectionState  
    
     
      
   
    
   
    
@@ -10892,7 +13397,7 @@ 
    CatalogDiff -  CheckConnectionRead -  Up
    
     
    
     
    
-      
    status 
    String  
    
+      
    status (optional)
    String  
    
         
    Enum:
    
         
    succeeded
    failed
    
 
    message (optional)
    String  
    
@@ -10921,6 +13426,15 @@ 
    CompleteDestinationOAuthRequ
 
    destinationId (optional)
    UUID  format: uuid
    
     
      
   
    
+  
    
+    
    CompleteOAuthResponse -  Up
    
+    
    
+    
    
+      
    request_succeeded 
    Boolean  
    
+
    request_error (optional)
    String  
    
+
    auth_payload 
    map[String, oas_any_type_not_mapped]  
    
+    
      
+  
    
   
    
     
    CompleteSourceOauthRequest -  Up
    
     
    
@@ -10930,6 +13444,7 @@ 
    CompleteSourceOauthRequest
 
    redirectUrl (optional)
    String When completing OAuth flow to gain an access token, some API sometimes requires to verify that the app re-send the redirectUrl that was used when consent was given. 
    
 
    queryParams (optional)
    map[String, oas_any_type_not_mapped] The query parameters present in the redirect URL after a user granted consent e.g auth code 
    
 
    oAuthInputConfiguration (optional)
    OAuthConfiguration  
    
+
    returnSecretCoordinate (optional)
    Boolean If set to true, returns a secret coordinate which references the stored tokens. By default, returns raw tokens. 
    
 
    sourceId (optional)
    UUID  format: uuid
    
     
      
   
    
@@ -10953,6 +13468,7 @@ 
    ConnectionCreate -  sourceCatalogId (optional)
    UUID  format: uuid
    
 
    geography (optional)
    Geography  
    
 
    notifySchemaChanges (optional)
    Boolean  
    
+
    notifySchemaChangesByEmail (optional)
    Boolean  
    
 
    nonBreakingChangesPreference (optional)
    NonBreakingChangesPreference  
    
     
      
   
    
@@ -10985,7 +13501,9 @@ 
    ConnectionRead -  geography (optional)
    Geography  
    
 
    breakingChange 
    Boolean  
    
 
    notifySchemaChanges (optional)
    Boolean  
    
+
    notifySchemaChangesByEmail (optional)
    Boolean  
    
 
    nonBreakingChangesPreference (optional)
    NonBreakingChangesPreference  
    
+
    workspaceId (optional)
    UUID  format: uuid
    
     
      
   
    
   
    
@@ -11087,6 +13605,22 @@ 
    ConnectionStatus -  
           
      
   
    
+  
    
+    
    ConnectionStream -  Up
    
+    
    
+    
    
+      
    streamName 
    String  
    
+
    streamNamespace 
    String  
    
+    
      
+  
    
+  
    
+    
    ConnectionStreamRequestBody -  Up
    
+    
    
+    
    
+      
    connectionId 
    UUID  format: uuid
    
+
    streams 
    array[ConnectionStream]  
    
+    
      
+  
    
   
    
     
    ConnectionUpdate -  Up
    
     
    Used to apply a patch-style update to a connection, which means that null properties remain unchanged
    
@@ -11106,10 +13640,72 @@ 
    ConnectionUpdate -  sourceCatalogId (optional)
    UUID  format: uuid
    
 
    geography (optional)
    Geography  
    
 
    notifySchemaChanges (optional)
    Boolean  
    
+
    notifySchemaChangesByEmail (optional)
    Boolean  
    
 
    nonBreakingChangesPreference (optional)
    NonBreakingChangesPreference  
    
 
    breakingChange (optional)
    Boolean  
    
     
      
   
    
+  
    
+    
    ConnectorBuilderProjectDetails -  Up
    
+    
    
+    
    
+      
    name 
    String  
    
+
    draftManifest (optional)
    Object Low code CDK manifest JSON object 
    
+    
      
+  
    
+  
    
+    
    ConnectorBuilderProjectDetailsRead -  Up
    
+    
    
+    
    
+      
    name 
    String  
    
+
    builderProjectId 
    UUID  format: uuid
    
+
    sourceDefinitionId (optional)
    UUID  format: uuid
    
+
    activeDeclarativeManifestVersion (optional)
    Long  format: int64
    
+
    hasDraft 
    Boolean  
    
+    
      
+  
    
+  
    
+    
    ConnectorBuilderProjectIdWithWorkspaceId -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    builderProjectId 
    UUID  format: uuid
    
+
    version (optional)
    Long  format: int64
    
+    
      
+  
    
+  
    
+    
    ConnectorBuilderProjectRead -  Up
    
+    
    
+    
    
+      
    builderProject 
    ConnectorBuilderProjectDetailsRead  
    
+
    declarativeManifest (optional)
    DeclarativeManifestRead  
    
+    
      
+  
    
+  
    
+    
    ConnectorBuilderProjectReadList -  Up
    
+    
    
+    
    
+      
    projects 
    array[ConnectorBuilderProjectDetailsRead]  
    
+    
      
+  
    
+  
    
+    
    ConnectorBuilderProjectWithWorkspaceId -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    builderProject 
    ConnectorBuilderProjectDetails  
    
+    
      
+  
    
+  
    
+    
    ConnectorBuilderPublishRequestBody -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    builderProjectId 
    UUID  format: uuid
    
+
    name 
    String  
    
+
    initialDeclarativeManifest 
    DeclarativeSourceManifest  
    
+    
      
+  
    
   
    
     
    CustomDestinationDefinitionCreate -  Up
    
     
    
@@ -11174,6 +13770,52 @@ 
    DbMigrationState -  
           
      
   
    
+  
    
+    
    DeclarativeManifestRead -  Up
    
+    
    
+    
    
+      
    manifest (optional)
    Object Low code CDK manifest JSON object 
    
+
    isDraft (optional)
    Boolean  
    
+
    version (optional)
    Long  format: int64
    
+
    description (optional)
    String  
    
+    
      
+  
    
+  
    
+    
    DeclarativeManifestVersionRead -  Up
    
+    
    
+    
    
+      
    version 
    Long  format: int64
    
+
    isActive 
    Boolean  
    
+
    description 
    String  
    
+    
      
+  
    
+  
    
+    
    DeclarativeManifestsReadList -  Up
    
+    
    
+    
    
+      
    manifestVersions 
    array[DeclarativeManifestVersionRead]  
    
+    
      
+  
    
+  
    
+    
    DeclarativeSourceDefinitionCreateManifestRequestBody -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    sourceDefinitionId 
    UUID  format: uuid
    
+
    setAsActiveManifest 
    Boolean  
    
+
    declarativeManifest 
    DeclarativeSourceManifest  
    
+    
      
+  
    
+  
    
+    
    DeclarativeSourceManifest -  Up
    
+    
    
+    
    
+      
    description 
    String  
    
+
    manifest 
    Object Low code CDK manifest JSON object 
    
+
    spec 
    SourceDefinitionSpecification  
    
+
    version 
    Long  format: int64
    
+    
      
+  
    
   
    
     
    DestinationCloneConfiguration -  Up
    
     
    
@@ -11269,7 +13911,6 @@ 
    DestinationDefinition
       
    destinationDefinitionId 
    UUID  format: uuid
    
 
    documentationUrl (optional)
    String  
    
 
    connectionSpecification (optional)
    DestinationDefinitionSpecification  
    
-
    authSpecification (optional)
    AuthSpecification  
    
 
    advancedAuth (optional)
    AdvancedAuth  
    
 
    jobInfo 
    SynchronousJobRead  
    
 
    supportedDestinationSyncModes (optional)
    array[DestinationSyncMode]  
    
@@ -11367,6 +14008,40 @@ 
    DiscoverCatalogResult -  catalogId 
    UUID  format: uuid
    
     
      
   
    
+  
    
+    
    ExistingConnectorBuilderProjectWithWorkspaceId -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    builderProjectId 
    UUID  format: uuid
    
+
    builderProject 
    ConnectorBuilderProjectDetails  
    
+    
      
+  
    
+  
    
+    
    FailureOrigin -  Up
    
+    
    Indicates where the error originated. If not set, the origin of error is not well known.
    
+    
    
+          
      
+  
    
+  
    
+    
    FailureReason -  Up
    
+    
    
+    
    
+      
    failureOrigin (optional)
    FailureOrigin  
    
+
    failureType (optional)
    FailureType  
    
+
    externalMessage (optional)
    String  
    
+
    internalMessage (optional)
    String  
    
+
    stacktrace (optional)
    String  
    
+
    retryable (optional)
    Boolean True if it is known that retrying may succeed, e.g. for a transient failure. False if it is known that a retry will not succeed, e.g. for a configuration issue. If not set, retryable status is not well known. 
    
+
    timestamp 
    Long  format: int64
    
+    
      
+  
    
+  
    
+    
    FailureType -  Up
    
+    
    Categorizes well known errors into types for programmatic handling. If not set, the type of error is not well known.
    
+    
    
+          
      
+  
    
   
    
     
    FieldAdd -  Up
    
     
    
@@ -11542,6 +14217,7 @@ 
    JobRead -  id 
    Long  format: int64
    
 
    configType 
    JobConfigType  
    
 
    configId 
    String  
    
+
    enabledStreams (optional)
    array[StreamDescriptor]  
    
 
    createdAt 
    Long  format: int64
    
 
    updatedAt 
    Long  format: int64
    
 
    startedAt (optional)
    Long  format: int64
    
@@ -11596,6 +14272,33 @@ 
    KnownExceptionInfo -  rootCauseExceptionStack (optional)
    array[String]  
    
     
      
   
    
+  
    
+    
    ListConnectionsForWorkspacesRequestBody -  Up
    
+    
    
+    
    
+      
    workspaceIds 
    array[UUID]  format: uuid
    
+
    userId 
    UUID  format: uuid
    
+
    pagination (optional)
    Pagination  
    
+
    includeDeleted (optional)
    Boolean  
    
+    
      
+  
    
+  
    
+    
    ListDeclarativeManifestsRequestBody -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    sourceDefinitionId 
    UUID  format: uuid
    
+    
      
+  
    
+  
    
+    
    ListResourcesForWorkspacesRequestBody -  Up
    
+    
    
+    
    
+      
    workspaceIds 
    array[UUID]  format: uuid
    
+
    includeDeleted (optional)
    Boolean  
    
+
    pagination (optional)
    Pagination  
    
+    
      
+  
    
   
    
     
    LogRead -  Up
    
     
    
@@ -11658,6 +14361,15 @@ 
    Notification -  sendOnSuccess 
    Boolean  
    
 
    sendOnFailure 
    Boolean  
    
 
    slackConfiguration (optional)
    SlackNotificationConfiguration  
    
+
    customerioConfiguration (optional)
    Object  
    
+    
      
+  
    
+  
    
+    
    NotificationItem -  Up
    
+    
    
+    
    
+      
    notificationType (optional)
    array[NotificationType]  
    
+
    slackConfiguration (optional)
    SlackNotificationConfiguration  
    
 
    customerioConfiguration (optional)
    Object  
    
     
      
   
    
@@ -11672,21 +14384,22 @@ 
    NotificationRead -  
-    
    NotificationType -  Up
    
+    
    NotificationSettings -  Up
    
     
    
     
    
-          
      
+      
    sendOnSuccess (optional)
    NotificationItem  
    
+
    sendOnFailure (optional)
    NotificationItem  
    
+
    sendOnSyncDisabled (optional)
    NotificationItem  
    
+
    sendOnSyncDisabledWarning (optional)
    NotificationItem  
    
+
    sendOnConnectionUpdate (optional)
    NotificationItem  
    
+
    sendOnConnectionUpdateActionRequired (optional)
    NotificationItem  
    
+    
      
   
    
   
    
-    
    OAuth2Specification -  Up
    
-    
    An object containing any metadata needed to describe this connector's Oauth flow
    
+    
    NotificationType -  Up
    
+    
    
     
    
-      
    rootObject 
    array[oas_any_type_not_mapped] A list of strings representing a pointer to the root object which contains any oauth parameters in the ConnectorSpecification.
-Examples:
-if oauth parameters were contained inside the top level, rootObject=[] If they were nested inside another object {'credentials': {'app_id' etc...}, rootObject=['credentials'] If they were inside a oneOf {'switch': {oneOf: [{client_id...}, {non_oauth_param]}},  rootObject=['switch', 0] 
    
-
    oauthFlowInitParameters 
    array[array[String]] Pointers to the fields in the rootObject needed to obtain the initial refresh/access tokens for the OAuth flow. Each inner array represents the path in the rootObject of the referenced field. For example. Assume the rootObject contains params 'app_secret', 'app_id' which are needed to get the initial refresh token. If they are not nested in the rootObject, then the array would look like this [['app_secret'], ['app_id']] If they are nested inside an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']] 
    
-
    oauthFlowOutputParameters 
    array[array[String]] Pointers to the fields in the rootObject which can be populated from successfully completing the oauth flow using the init parameters. This is typically a refresh/access token. Each inner array represents the path in the rootObject of the referenced field. 
    
-    
      
+          
      
   
    
   
    
     
    OAuthConfigSpecification -  Up
    
@@ -11811,6 +14524,25 @@ 
    Pagination -  rowOffset (optional)
    Integer  
    
     
      
   
    
+  
    
+    
    PartialDestinationUpdate -  Up
    
+    
    
+    
    
+      
    destinationId (optional)
    UUID  format: uuid
    
+
    connectionConfiguration (optional)
    DestinationConfiguration  
    
+
    name (optional)
    String  
    
+    
      
+  
    
+  
    
+    
    PartialSourceUpdate -  Up
    
+    
    
+    
    
+      
    sourceId 
    UUID  format: uuid
    
+
    connectionConfiguration (optional)
    SourceConfiguration  
    
+
    name (optional)
    String  
    
+
    secretId (optional)
    String  
    
+    
      
+  
    
   
    
     
    PrivateDestinationDefinitionRead -  Up
    
     
    
@@ -11864,6 +14596,15 @@ 
    ResourceRequirements -  memory_limit (optional)
    String  
    
     
      
   
    
+  
    
+    
    RevokeSourceOauthTokensRequest -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    sourceId 
    UUID  format: uuid
    
+
    sourceDefinitionId 
    UUID  format: uuid
    
+    
      
+  
    
   
    
     
    SaveAttemptSyncConfigRequestBody -  Up
    
     
    
@@ -11936,6 +14677,16 @@ 
    SlugRequestBody -  slug 
    String  
    
     
      
   
    
+  
    
+    
    SourceAutoPropagateChange -  Up
    
+    
    Input of the source propagation, it contains the discovered catalog and a list of diff that need to be applied to the existing catalog.
    
+    
    
+      
    catalog 
    AirbyteCatalog  
    
+
    catalogId 
    UUID  format: uuid
    
+
    sourceId 
    UUID  format: uuid
    
+
    workspaceId 
    UUID  format: uuid
    
+    
      
+  
    
   
    
     
    SourceCloneConfiguration -  Up
    
     
    
@@ -11970,6 +14721,7 @@ 
    SourceCreate -  connectionConfiguration 
    SourceConfiguration  
    
 
    workspaceId 
    UUID  format: uuid
    
 
    name 
    String  
    
+
    secretId (optional)
    String  
    
     
      
   
    
   
    
@@ -11984,6 +14736,13 @@ 
    SourceDefinitionCreate - 
 
    resourceRequirements (optional)
    ActorDefinitionResourceRequirements  
    
     
      
   
    
+  
    
+    
    SourceDefinitionIdBody -  Up
    
+    
    
+    
    
+      
    sourceDefinitionId 
    UUID  format: uuid
    
+    
      
+  
    
   
    
     
    SourceDefinitionIdRequestBody -  Up
    
     
    
@@ -12016,6 +14775,7 @@ 
    SourceDefinitionRead -  Enum:
    
         
    api
    file
    database
    custom
    
 
    resourceRequirements (optional)
    ActorDefinitionResourceRequirements  
    
+
    maxSecondsBetweenMessages (optional)
    Long Number of seconds allowed between 2 airbyte protocol messages. The source will timeout if this delay is reach format: int64
    
     
      
   
    
   
    
@@ -12032,7 +14792,6 @@ 
    SourceDefinitionSpecificat
       
    sourceDefinitionId 
    UUID  format: uuid
    
 
    documentationUrl (optional)
    String  
    
 
    connectionSpecification (optional)
    SourceDefinitionSpecification  
    
-
    authSpecification (optional)
    AuthSpecification  
    
 
    advancedAuth (optional)
    AdvancedAuth  
    
 
    jobInfo 
    SynchronousJobRead  
    
     
      
@@ -12146,6 +14905,7 @@ 
    SourceUpdate -  sourceId 
    UUID  format: uuid
    
 
    connectionConfiguration 
    SourceConfiguration  
    
 
    name 
    String  
    
+
    secretId (optional)
    String  
    
     
      
   
    
   
    
@@ -12164,6 +14924,108 @@ 
    StreamState -  streamState (optional)
    StateBlob  
    
     
      
   
    
+  
    
+    
    StreamStatusCreateRequestBody -  Up
    
+    
    
+    
    
+      
    attemptNumber 
    Integer  format: int32
    
+
    connectionId 
    UUID  format: uuid
    
+
    jobId 
    Long  format: int64
    
+
    incompleteRunCause (optional)
    StreamStatusIncompleteRunCause  
    
+
    jobType 
    StreamStatusJobType  
    
+
    runState 
    StreamStatusRunState  
    
+
    streamName 
    String  
    
+
    streamNamespace (optional)
    String  
    
+
    transitionedAt 
    Long  format: int64
    
+
    workspaceId 
    UUID  format: uuid
    
+    
      
+  
    
+  
    
+    
    StreamStatusIncompleteRunCause -  Up
    
+    
    Values:
    
+
      
+
    • FAILED - A failure has occurred
      • 
+
    • CANCELED - The run has been canceled
      • 
+
    
+
    
+    
    
+          
      
+  
    
+  
    
+    
    StreamStatusJobType -  Up
    
+    
    
+    
    
+          
      
+  
    
+  
    
+    
    StreamStatusListRequestBody -  Up
    
+    
    
+    
    
+      
    attemptNumber (optional)
    Integer  format: int32
    
+
    connectionId (optional)
    UUID  format: uuid
    
+
    jobId (optional)
    Long  format: int64
    
+
    jobType (optional)
    StreamStatusJobType  
    
+
    pagination 
    Pagination  
    
+
    streamName (optional)
    String  
    
+
    streamNamespace (optional)
    String  
    
+
    workspaceId 
    UUID  format: uuid
    
+    
      
+  
    
+  
    
+    
    StreamStatusRead -  Up
    
+    
    
+    
    
+      
    attemptNumber 
    Integer  format: int32
    
+
    connectionId 
    UUID  format: uuid
    
+
    id 
    UUID  format: uuid
    
+
    jobId 
    Long  format: int64
    
+
    incompleteRunCause (optional)
    StreamStatusIncompleteRunCause  
    
+
    jobType 
    StreamStatusJobType  
    
+
    runState 
    StreamStatusRunState  
    
+
    streamName 
    String  
    
+
    streamNamespace 
    String  
    
+
    transitionedAt 
    Long  format: int64
    
+
    workspaceId 
    UUID  format: uuid
    
+    
      
+  
    
+  
    
+    
    StreamStatusReadList -  Up
    
+    
    
+    
    
+      
    streamStatuses (optional)
    array[StreamStatusRead]  
    
+    
      
+  
    
+  
    
+    
    StreamStatusRunState -  Up
    
+    
    Values:
    
+
      
+
    • PENDING - The stream operation has been selected to run
      • 
+
    • RUNNING - The stream operation is running
      • 
+
    • COMPLETE - The stream operation ran successfully to completion
      • 
+
    • INCOMPLETE - The stream operation has terminated in an incomplete state.
+See StreamStatusIncompleteRunCause for more details.
      • 
+
    
+
    
+    
    
+          
      
+  
    
+  
    
+    
    StreamStatusUpdateRequestBody -  Up
    
+    
    
+    
    
+      
    id 
    UUID  format: uuid
    
+
    attemptNumber 
    Integer  format: int32
    
+
    connectionId 
    UUID  format: uuid
    
+
    jobId 
    Long  format: int64
    
+
    incompleteRunCause (optional)
    StreamStatusIncompleteRunCause  
    
+
    jobType 
    StreamStatusJobType  
    
+
    runState 
    StreamStatusRunState  
    
+
    streamName 
    String  
    
+
    streamNamespace (optional)
    String  
    
+
    transitionedAt 
    Long  format: int64
    
+
    workspaceId 
    UUID  format: uuid
    
+    
      
+  
    
   
    
     
    StreamTransform -  Up
    
     
    
@@ -12193,6 +15055,16 @@ 
    SynchronousJobRead -  succeeded 
    Boolean  
    
 
    connectorConfigurationUpdated (optional)
    Boolean  
    
 
    logs (optional)
    LogRead  
    
+
    failureReason (optional)
    FailureReason  
    
+    
      
+  
    
+  
    
+    
    UpdateActiveManifestRequestBody -  Up
    
+    
    
+    
    
+      
    workspaceId 
    UUID  format: uuid
    
+
    sourceDefinitionId 
    UUID  format: uuid
    
+
    version 
    Long  format: int64
    
     
      
   
    
   
    
@@ -12291,6 +15163,7 @@ 
    WebBackendConnectionRead - <
 
    geography (optional)
    Geography  
    
 
    schemaChange 
    SchemaChange  
    
 
    notifySchemaChanges 
    Boolean  
    
+
    notifySchemaChangesByEmail 
    Boolean  
    
 
    nonBreakingChangesPreference 
    NonBreakingChangesPreference  
    
     
      
   
    
@@ -12329,6 +15202,7 @@ 
    WebBackendConnectionUpdate
 
    sourceCatalogId (optional)
    UUID  format: uuid
    
 
    geography (optional)
    Geography  
    
 
    notifySchemaChanges (optional)
    Boolean  
    
+
    notifySchemaChangesByEmail (optional)
    Boolean  
    
 
    nonBreakingChangesPreference (optional)
    NonBreakingChangesPreference  
    
     
      
   
    
@@ -12399,6 +15273,7 @@ 
    WorkspaceCreate -  news (optional)
    Boolean  
    
 
    securityUpdates (optional)
    Boolean  
    
 
    notifications (optional)
    array[Notification]  
    
+
    notificationSettings (optional)
    NotificationSettings  
    
 
    displaySetupWizard (optional)
    Boolean  
    
 
    defaultGeography (optional)
    Geography  
    
 
    webhookConfigs (optional)
    array[WebhookConfigWrite]  
    
@@ -12418,6 +15293,16 @@ 
    WorkspaceIdRequestBody - 
       
    workspaceId 
    UUID  format: uuid
    
     
      
   
    
+  
    
+    
    WorkspaceOverrideOauthParamsRequestBody -  Up
    
+    
    
+    
    
+      
    definitionId 
    UUID  format: uuid
    
+
    params 
    map[String, oas_any_type_not_mapped]  
    
+
    workspaceId 
    UUID  format: uuid
    
+
    actorType 
    ActorType  
    
+    
      
+  
    
   
    
     
    WorkspaceRead -  Up
    
     
    
@@ -12433,6 +15318,7 @@ 
    WorkspaceRead -  news (optional)
    Boolean  
    
 
    securityUpdates (optional)
    Boolean  
    
 
    notifications (optional)
    array[Notification]  
    
+
    notificationSettings (optional)
    NotificationSettings  
    
 
    firstCompletedSync (optional)
    Boolean  
    
 
    feedbackDone (optional)
    Boolean  
    
 
    defaultGeography (optional)
    Geography  
    
@@ -12458,6 +15344,7 @@ 
    WorkspaceUpdate -  news (optional)
    Boolean  
    
 
    securityUpdates (optional)
    Boolean  
    
 
    notifications (optional)
    array[Notification]  
    
+
    notificationSettings (optional)
    NotificationSettings  
    
 
    defaultGeography (optional)
    Geography  
    
 
    webhookConfigs (optional)
    array[WebhookConfigWrite]  
    
     
      
diff --git a/docs/release_notes/assets/airbyte_connection_update_state.png b/docs/release_notes/assets/airbyte_connection_update_state.png
new file mode 100644
index 000000000000..001680bed656
Binary files /dev/null and b/docs/release_notes/assets/airbyte_connection_update_state.png differ
diff --git a/docs/release_notes/assets/airbyte_destinations_v2_upgrade_prompt.png b/docs/release_notes/assets/airbyte_destinations_v2_upgrade_prompt.png
new file mode 100644
index 000000000000..b5d0ec78b4ba
Binary files /dev/null and b/docs/release_notes/assets/airbyte_destinations_v2_upgrade_prompt.png differ
diff --git a/docs/release_notes/assets/airbyte_dual_destinations.png b/docs/release_notes/assets/airbyte_dual_destinations.png
new file mode 100644
index 000000000000..58dc44e3f596
Binary files /dev/null and b/docs/release_notes/assets/airbyte_dual_destinations.png differ
diff --git a/docs/release_notes/assets/airbyte_inherit_state.png b/docs/release_notes/assets/airbyte_inherit_state.png
new file mode 100644
index 000000000000..95620f696416
Binary files /dev/null and b/docs/release_notes/assets/airbyte_inherit_state.png differ
diff --git a/docs/release_notes/assets/airbyte_legacy_normalization.png b/docs/release_notes/assets/airbyte_legacy_normalization.png
new file mode 100644
index 000000000000..f387ef8fa8ff
Binary files /dev/null and b/docs/release_notes/assets/airbyte_legacy_normalization.png differ
diff --git a/docs/release_notes/assets/airbyte_version_upgrade.png b/docs/release_notes/assets/airbyte_version_upgrade.png
new file mode 100644
index 000000000000..79eb5b04f2e5
Binary files /dev/null and b/docs/release_notes/assets/airbyte_version_upgrade.png differ
diff --git a/docs/release_notes/assets/destinations-v2-column-changes.png b/docs/release_notes/assets/destinations-v2-column-changes.png
new file mode 100644
index 000000000000..ac15f0b292c3
Binary files /dev/null and b/docs/release_notes/assets/destinations-v2-column-changes.png differ
diff --git a/docs/release_notes/destinations_v2.js b/docs/release_notes/destinations_v2.js
new file mode 100644
index 000000000000..ce8d4c19c5e0
--- /dev/null
+++ b/docs/release_notes/destinations_v2.js
@@ -0,0 +1,149 @@
+import React, { useState } from 'react';
+import CodeBlock from '@theme/CodeBlock';
+
+function concatenateRawTableName(namespace, name) {
+  let plainConcat = namespace + name;
+  // Pretend we always have at least one underscore, so that we never generate `_raw_stream_`
+  let longestUnderscoreRun = 1;
+  for (let i = 0; i < plainConcat.length; i++) {
+    // If we've found an underscore, count the number of consecutive underscores
+    let underscoreRun = 0;
+    while (i < plainConcat.length && plainConcat.charAt(i) === '_') {
+        underscoreRun++;
+        i++;
+    }
+    longestUnderscoreRun = Math.max(longestUnderscoreRun, underscoreRun);
+  }
+  return namespace + "_raw" + "_".repeat(longestUnderscoreRun + 1) + "stream_" + name;
+}
+
+// Taken from StandardNameTransformer
+function convertStreamName(str) {
+  return str.normalize('NFKD')
+    .replaceAll(/\p{M}/gu, "")
+    .replaceAll(/\s+/g, "_")
+    .replaceAll(/[^A-Za-z0-9_]/g, "_");
+}
+
+export const BigQueryMigrationGenerator = () => {
+  // See BigQuerySQLNameTransformer
+  function bigqueryConvertStreamName(str) {
+    str = convertStreamName(str);
+    if (str.charAt(0).match(/[A-Za-z_]/)) {
+      return str;
+    } else {
+      return "_" + str;
+    }
+  }
+  function escapeNamespace(namespace) {
+    namespace = convertStreamName(namespace);
+    if (!namespace.charAt(0).match(/[A-Za-z0-9]/)) {
+      namespace = "n" + namespace;
+    }
+    return namespace;
+  }
+
+  function generateSql(namespace, name, raw_dataset) {
+    let v2RawTableName = '`' + bigqueryConvertStreamName(concatenateRawTableName(namespace, name)) + '`';
+    let v1namespace = '`' + escapeNamespace(namespace) + '`';
+    let v1name = '`' + bigqueryConvertStreamName("_airbyte_raw_" + name) + '`';
+    return `CREATE SCHEMA IF NOT EXISTS ${raw_dataset};
+CREATE OR REPLACE TABLE \`${raw_dataset}\`.${v2RawTableName} (
+  _airbyte_raw_id STRING,
+  _airbyte_extracted_at TIMESTAMP,
+  _airbyte_loaded_at TIMESTAMP,
+  _airbyte_data JSON)
+PARTITION BY DATE(_airbyte_extracted_at)
+CLUSTER BY _airbyte_extracted_at
+AS (
+    SELECT
+        _airbyte_ab_id AS _airbyte_raw_id,
+        _airbyte_emitted_at AS _airbyte_extracted_at,
+        CAST(NULL AS TIMESTAMP) AS _airbyte_loaded_at,
+        PARSE_JSON(_airbyte_data) AS _airbyte_data
+    FROM ${v1namespace}.${v1name}
+)`;
+  }
+
+  return (
+    
+  );
+}
+
+export const SnowflakeMigrationGenerator = () => {
+  // See SnowflakeSQLNameTransformer
+  function snowflakeConvertStreamName(str) {
+    str = convertStreamName(str);
+    if (str.charAt(0).match(/[A-Za-z_]/)) {
+      return str;
+    } else {
+      return "_" + str;
+    }
+  }
+  function generateSql(namespace, name, raw_schema) {
+    let v2RawTableName = '"' + concatenateRawTableName(namespace, name) + '"';
+    let v1namespace = snowflakeConvertStreamName(namespace);
+    let v1name = snowflakeConvertStreamName("_airbyte_raw_" + name);
+    return `CREATE SCHEMA IF NOT EXISTS "${raw_schema}";
+CREATE OR REPLACE TABLE "${raw_schema}".${v2RawTableName} (
+  "_airbyte_raw_id" STRING NOT NULL PRIMARY KEY,
+  "_airbyte_extracted_at" TIMESTAMP_TZ DEFAULT CURRENT_TIMESTAMP(),
+  "_airbyte_loaded_at" TIMESTAMP_TZ,
+  "_airbyte_data" VARIANT)
+AS (
+    SELECT
+        _airbyte_ab_id AS "_airbyte_raw_id",
+        _airbyte_emitted_at AS "_airbyte_extracted_at",
+        CAST(NULL AS TIMESTAMP_TZ) AS "_airbyte_loaded_at",
+        _airbyte_data AS "_airbyte_data"
+    FROM ${v1namespace}.${v1name}
+)`;
+  }
+  return (
+    
+  );
+}
+
+export const MigrationGenerator = ({destination, generateSql}) => {
+  const defaultMessage =
+`Enter your stream's name and namespace to see the SQL output.
+If your stream has no namespace, take the default value from the destination connector's settings.`;
+  const [message, updateMessage] = useState({
+    'message': defaultMessage,
+    'language': 'text'
+  });
+  function updateSql(event) {
+    let namespace = document.getElementById("stream_namespace_" + destination).value;
+    let name = document.getElementById("stream_name_" + destination).value;
+    var raw_dataset = document.getElementById("raw_dataset_" + destination).value;
+    if (raw_dataset === '') {
+      raw_dataset = 'airbyte_internal';
+    }
+    let sql = generateSql(namespace, name, raw_dataset);
+    if (namespace !== "" && name !== "") {
+      updateMessage({
+        'message': sql,
+        'language': 'sql'
+      });
+    } else {
+      updateMessage({
+        'message': defaultMessage,
+        'language': 'text'
+      });
+    }
+  }
+
+  return (
+    
    
+      
+      
    
+      
+      
    
+      
+      
    
+      
+        { message['message'] }
+      
+    
    
+  );
+}
diff --git a/docs/release_notes/july_2022.md b/docs/release_notes/july_2022.md
index 0cf73f7d4b7b..0c6cbc35e004 100644
--- a/docs/release_notes/july_2022.md
+++ b/docs/release_notes/july_2022.md
@@ -26,7 +26,7 @@ This page includes new features and improvements to the Airbyte Cloud and Airbyt
 
 * Improved Airbyte Open Source self-hosting by refactoring and publishing Helm charts according to best practices as we prepare to formally support Helm deployments. [#14794](https://github.com/airbytehq/airbyte/pull/14794)  
 
-* Improved Airbyte Open Source by supporting the [OpenTelemetry (OTEL) Collector](https://docs.airbyte.com/operator-guides/collecting-metrics/). Airbyte Open Source now sends telemetry data to the OTEL collector, and we included a set of [recommended metrics](https://docs.airbyte.com/operator-guides/scaling-airbyte/#metrics) to export to OTEL when running Airbyte Open Source at scale. [#12908](https://github.com/airbytehq/airbyte/issues/12908)  
+* Improved Airbyte Open Source by supporting the OpenTelemetry (OTEL) Collector. Airbyte Open Source now sends telemetry data to the OTEL collector, and we included a set of [recommended metrics](https://docs.airbyte.com/operator-guides/scaling-airbyte/#metrics) to export to OTEL when running Airbyte Open Source at scale. [#12908](https://github.com/airbytehq/airbyte/issues/12908)
 
 * Improved the [Airbyte Connector Development Kit (CDK)](https://airbyte.com/connector-development-kit) by enabling detailed bug logs from the command line. In addition to the preset CDK debug logs, you can also create custom debug statements and display custom debug logs in the command line. [#14521](https://github.com/airbytehq/airbyte/pull/14521)  
 
diff --git a/docs/release_notes/july_2023.md b/docs/release_notes/july_2023.md
new file mode 100644
index 000000000000..f4f80e835393
--- /dev/null
+++ b/docs/release_notes/july_2023.md
@@ -0,0 +1,18 @@
+# July 2023
+## airbyte v0.50.6 to v0.50.11
+
+This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms.
+
+## **✨ New and improved features**
+
+- **New Sources and Promotions**: July has been a month of expansion and innovation for us. We've rolled out several new connectors and promotions to enhance our offerings. One of the significant additions is the **[Datadog connector](https://github.com/airbytehq/airbyte/pull/27906)** to the cloud, which promises to streamline data integration from Datadog. Another noteworthy introduction is the destination for **[Vector Database powered by LangChain](https://github.com/airbytehq/airbyte/pull/26184)**, a powerful database solution. We've also ventured into new territories with the introduction of the **[Gainsight-px source](https://github.com/airbytehq/airbyte/pull/26998)**. Additionally, our commitment to keeping our tools updated is reflected in the upgrade of the **[Iceberg destination](https://github.com/airbytehq/airbyte/pull/23201)**.
+- **New Features for Existing Connectors**: Our existing connectors have received a lot of love this month. We've made OAuth the exclusive option for **[Shopify-Oauth](https://chat.openai.com/c/e3dcdfa7-a2d3-46b5-9976-2bb866e1bb2a#6457)**, enhancing security and simplifying the authentication process. The **[Datadog source](https://github.com/airbytehq/airbyte/pull/27804)** now supports multiple query streams, offering more flexibility in data retrieval. We've also enriched our **[Twilio](https://github.com/airbytehq/airbyte/pull/27231)** and **[Zendesk Support](https://github.com/airbytehq/airbyte/pull/27156)** connectors with new streams, broadening the scope of data that can be accessed. Furthermore, several sources have been migrated to advanced authentication, reinforcing security and streamlining the user experience.
+- **New Features in Airbyte Platform**: On the platform front, we've introduced aesthetic and functional enhancements. Users can now enjoy a visually pleasing **[Dark Mode](https://chat.openai.com/c/e3dcdfa7-a2d3-46b5-9976-2bb866e1bb2a#6632)**, reducing eye strain during extended usage. We've also made it easier to manage data synchronization with the addition of a **[sync mode dropdown](https://chat.openai.com/c/e3dcdfa7-a2d3-46b5-9976-2bb866e1bb2a#7688)** in the stream details panel. For those building connectors, the new **[global requests UI](https://chat.openai.com/c/e3dcdfa7-a2d3-46b5-9976-2bb866e1bb2a#7699)** in the Connector Builder promises a more intuitive experience.
+
+## **🚨 Security & Breaking changes**
+
+- Ensuring the security and reliability of our platform is paramount. This month, we've addressed several issues and made breaking changes to enhance performance. The incremental for the **[Hubspot engagement stream](https://github.com/airbytehq/airbyte/pull/27161)** has been fixed, ensuring accurate data retrieval. We've also taken the step to remove the **`Activity`** Stream from **[MetaBase](https://github.com/airbytehq/airbyte/pull/27777)** for optimization. The **[Facebook Marketing source](https://github.com/airbytehq/airbyte/pull/27563)** has been updated to use SDK v17, ensuring compatibility and enhanced performance.
+
+## **🐛 Bug fixes**
+
+- Our commitment to delivering a bug-free experience is unwavering. July saw us addressing a myriad of issues across our platform. We've rectified the **[custom connector creation flow](https://chat.openai.com/c/e3dcdfa7-a2d3-46b5-9976-2bb866e1bb2a#8018)**, ensuring a smoother user experience. Several sources, including **[Square](https://github.com/airbytehq/airbyte/pull/27762)** and **[Greenhouse](https://github.com/airbytehq/airbyte/pull/27773)**, have been updated following state management changes in the CDK. We've also tackled specific issues in connectors like **[Google Ads](https://github.com/airbytehq/airbyte/pull/27711)** and **[Datadog](https://github.com/airbytehq/airbyte/pull/27784)**, ensuring they function optimally.
\ No newline at end of file
diff --git a/docs/release_notes/june_2023.md b/docs/release_notes/june_2023.md
new file mode 100644
index 000000000000..99ca0ca7ac6d
--- /dev/null
+++ b/docs/release_notes/june_2023.md
@@ -0,0 +1,23 @@
+# June 2023
+## airbyte v0.44.12 to v0.50.5
+
+This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms.
+
+### **New and Improved Features:**
+
+1. **New Sources and Promotions:** Our data reach now extends with new destination connectors for **[Xata.io](https://github.com/airbytehq/airbyte/pull/24192)** and **[Timeplus](https://github.com/airbytehq/airbyte/pull/21226)**. Also, we've introduced a new source - **[KYVE Network](https://github.com/airbytehq/airbyte/pull/27373)**, further enhancing the variety of data you can harness.
+2. **New Features for Existing Connectors:** We've added new streams for users, tasks, templates, and snippets to our **[Source Outreach](https://github.com/airbytehq/airbyte/pull/27343)**. Furthermore, **[Destination Databricks](https://github.com/airbytehq/airbyte/pull/26942)** now supports schema evolution, allowing you more flexibility with your data structures.
+3. **New Features in Airbyte Platform:** Several updates to our platform include highlighting schema errors for test records (**[#6916](https://github.com/airbytehq/airbyte/pull/6916)**), enabling autopropagation in the UI by default (**[#7124](https://github.com/airbytehq/airbyte/pull/7124)**), and making cursor granularity optional (**[#7158](https://github.com/airbytehq/airbyte/pull/7158)**). Plus, our connector builder now auto-imports schemas (**[#7113](https://github.com/airbytehq/airbyte/pull/7113)**) and adds request options for the API key authenticator (**[#7009](https://github.com/airbytehq/airbyte/pull/7009)**).
+
+### **Bug Fixes:**
+
+We've addressed various bugs for smoother user experience:
+
+- Added handler for 402 error in **[Source Mixpanel](https://github.com/airbytehq/airbyte/pull/27252)**
+- Migrated **[Source Notion](https://github.com/airbytehq/airbyte/pull/26535)** and **[Source Google Ads](https://github.com/airbytehq/airbyte/pull/26905)** to advancedAuth
+- Improved error handling for **[Source Monday](https://github.com/airbytehq/airbyte/pull/27244)**, **[Source Drift](https://github.com/airbytehq/airbyte/pull/27202)**, and **[Source Snapchat Marketing](https://github.com/airbytehq/airbyte/pull/26358)**
+- Made various improvements to **[Source Gitlab](https://github.com/airbytehq/airbyte/pull/27346)**, including OAuth token expiry date fix (**[#27351](https://github.com/airbytehq/airbyte/pull/27351)**)
+- Fixed **`data_state`** config typo in **[Source Google Search Console](https://github.com/airbytehq/airbyte/pull/27307)**
+- Addressed issues with **[Source Amazon Seller Partner](https://github.com/airbytehq/airbyte/pull/27110)**, **[Facebook Marketing](https://github.com/airbytehq/airbyte/pull/27201)**, **[Quickbooks](https://github.com/airbytehq/airbyte/pull/27148)**, **[Smartsheets](https://github.com/airbytehq/airbyte/pull/27096)**, and others.
+
+We've also made significant improvements to our connector builder, including reloading diff view on stream change (**[#6974](https://github.com/airbytehq/airbyte/pull/6974)**)
\ No newline at end of file
diff --git a/docs/release_notes/upgrading_to_destinations_v2.md b/docs/release_notes/upgrading_to_destinations_v2.md
new file mode 100644
index 000000000000..30461beae755
--- /dev/null
+++ b/docs/release_notes/upgrading_to_destinations_v2.md
@@ -0,0 +1,196 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import {SnowflakeMigrationGenerator, BigQueryMigrationGenerator} from './destinations_v2.js'
+
+# Upgrading to Destinations V2
+
+## What is Destinations V2?
+
+Starting today, Airbyte Destinations V2 provides you with:
+
+- One-to-one table mapping: Data in one stream will always be mapped to one table in your data warehouse. No more sub-tables.
+- Improved error handling with `_airbyte_meta`: Airbyte will now populate typing errors in the `_airbyte_meta` column instead of failing your sync. You can query these results to audit misformatted or unexpected data.
+- Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables in the `airbyte_internal` schema. We no longer clutter your destination schema with raw data tables.
+- Incremental delivery for large syncs: Data will be incrementally delivered to your final tables. No more waiting hours to see the first rows in your destination table.
+
+To see more details and examples on the contents of the Destinations V2 release, see this [guide](understanding-airbyte/typing-deduping.md). The remainder of this page will walk you through upgrading connectors from legacy normalization to Destinations V2.
+
+## Deprecating Legacy Normalization
+
+The upgrade to Destinations V2 is handled by moving your connections to use [updated versions of Airbyte destinations](#destinations-v2-compatible-versions). Existing normalization options, both `Raw data (JSON)` and `Normalized tabular data` will be unsupported starting **Nov 1, 2023**.
+
+![Legacy Normalization](./assets/airbyte_legacy_normalization.png)
+
+As a Cloud user, existing connections using legacy normalization will be paused on **Oct 1, 2023**. As an Open Source user, you may choose to upgrade at your convenience. However, destination connector versions prior to Destinations V2 will no longer be supported as of **Nov 1, 2023**.
+
+### Breakdown of Breaking Changes
+
+The following table details the delivered data modified by Destinations V2:
+
+| Current Normalization Setting | Source Type                           | Impacted Data (Breaking Changes)                         |
+| ----------------------------- | ------------------------------------- | -------------------------------------------------------- |
+| Raw JSON                      | All                                   | `_airbyte` metadata columns, raw table location          |
+| Normalized tabular data       | API Source                            | Unnested tables, `_airbyte` metadata columns, SCD tables |
+| Normalized tabular data       | Tabular Source (database, file, etc.) | `_airbyte` metadata columns, SCD tables                  |
+
+![Airbyte Destinations V2 Column Changes](./assets/destinations-v2-column-changes.png)
+
+Whenever possible, we've taken this opportunity to use the best data type for storing JSON for your querying convenience. For example, `destination-bigquery` now loads `JSON` blobs as type `JSON` in BigQuery (introduced last [year](https://cloud.google.com/blog/products/data-analytics/bigquery-now-natively-supports-semi-structured-data)), instead of type `string`.
+
+## Quick Start to Upgrading
+
+The quickest path to upgrading is to click upgrade on any out-of-date connection in the UI:
+
+![Upgrade Path](./assets/airbyte_destinations_v2_upgrade_prompt.png)
+
+After upgrading the out-of-date destination to a [Destinations V2 compatible version](#destinations-v2-effective-versions), the following will occur at the next sync **for each connection** sending data to the updated destination:
+
+1. Existing raw tables replicated to this destination will be copied to a new `airbyte_internal` schema.
+2. The new raw tables will be updated to the new Destinations V2 format.
+3. The new raw tables will be updated with any new data since the last sync, like normal.
+4. The new raw tables will be typed and de-duplicated according to the Destinations V2 format.
+5. Once typing and de-duplication has completed successfully, your previous final table will be replaced with the updated data.
+
+:::caution
+
+Due to the amount of operations to be completed, this first sync after upgrading to Destination V2 **will be longer than normal**. Once your first sync has completed successfully, you may need to make changes to downstream models (dbt, sql, etc.) transforming data. See this [walkthrough of top changes to expect for more details](#updating-downstream-transformations).
+
+:::
+
+Pre-existing raw tables, SCD tables and "unnested" tables will always be left untouched. You can delete these at your convenience, but these tables will no longer be kept up-to-date by Airbyte syncs.
+Each destination version is managed separately, so if you have multiple destinations, they all need to be upgraded one by one.
+
+
+
+Versions are tied to the destination. When you update the destination, **all connections tied to that destination will be sending data in the Destinations V2 format**. For upgrade paths that will minimize disruption to existing dashboards, see:
+
+- [Upgrading Connections One by One with Dual-Writing](#upgrading-connections-one-by-one-with-dual-writing)
+- [Testing Destinations V2 on a Single Connection](#testing-destinations-v2-for-a-single-connection)
+- [Upgrading Connections One by One Using CDC](#upgrade-paths-for-connections-using-cdc)
+- [Upgrading as a User of Raw Tables](#upgrading-as-a-user-of-raw-tables)
+- [Rolling back to Legacy Normalization](#oss-only-rolling-back-to-legacy-normalization)
+
+## Advanced Upgrade Paths
+
+### Upgrading Connections One by One with Dual-Writing
+
+Dual writing is a method employed during upgrades where new incoming data is written simultaneously to both the old and new systems, facilitating a smooth transition between versions. We recommend this approach for connections where you are especially worried about breaking changes or downtime in downstream systems.
+
+#### Steps to Follow for All Sync Modes
+
+1. **[Open Source]** Update the default destination version for your workspace to a [Destinations V2 compatible version](#destinations-v2-effective-versions). This sets the default version for any newly created destination. All existing syncs will remain on their current version.
+
+![Upgrade your default destination version](assets/airbyte_version_upgrade.png)
+
+2. Create and configure a new destination connecting to the same database as your existing destination except for `Default Schema`, which you should update to a new value to avoid collisions.
+
+![Create a new destination](assets/airbyte_dual_destinations.png)
+
+3. Create a new connection leveraging your existing source and the newly created destination. Match the settings of your pre-existing connection.
+4. If the streams you are looking to replicate are in **full refresh** mode, enabling the connection will now provide a parallel copy of the data in the updated format for testing. If any of the streams in the connection are in an **incremental** sync mode, follow the steps below before enabling the connection.
+
+#### Additional Steps for Incremental Sync Modes
+
+These steps allow you to dual-write for connections incrementally syncing data without re-syncing historical data you've already replicated:
+
+1. Copy the raw data you've already replicated to the new schema being used by your newly created connection. You need to do this for every stream in the connection with an incremental sync mode. Sample SQL you can run in your data warehouse:
+
+
+  
+    
+  
+  
+    
+  
+
+
+2. Navigate to the existing connection you are duplicating, and navigate to the `Settings` tab. Open the `Advanced` settings to see the connection state (which manages incremental syncs). Copy the state to your clipboard.
+
+![img.png](assets/airbyte_connection_update_state.png)
+
+3. Go to your newly created connection, replace the state with the copied contents in the previous step, then click `Update State`. This will ensure historical data is not replicated again.
+4. Enabling the connection will now provide a parallel copy of all streams in the updated format.
+5. You can move your dashboards to rely on the new tables, then pause the out-of-date connection.
+
+### Testing Destinations V2 for a Single Connection
+
+You may want to verify the format of updated data for a single connection. To do this:
+
+1. If all of the streams you are looking to test with are in **full refresh mode**, follow the [steps for upgrading connections one by one](#steps-to-follow-for-all-sync-modes). Ensure any connections you create have a `Manual` replication frequency.
+2. For any streams in **incremental** sync modes, follow the [steps for upgrading incremental syncs](#additional-steps-for-incremental-sync-modes). For testing, you do not need to copy pre-existing raw data. By solely inheriting state from a pre-existing connection, enabling a sync will provide a sample of the most recent data in the updated format for testing.
+
+When you are done testing, you can disable or delete this testing connection, and [upgrade your pre-existing connections in place](#quick-start-to-upgrading) or [upgrade one-by-one with dual writing](#upgrading-connections-one-by-one-with-dual-writing).
+
+### Upgrading as a User of Raw Tables
+
+If you have written downstream transformations directly from the output of raw tables, or use the "Raw JSON" normalization setting, you should know that:
+
+- Multiple column names are being updated (from `airbyte_ab_id` to `airbyte_raw_id`, and `airbyte_emitted_at` to `airbyte_extracted_at`).
+- The location of raw tables will from now on default to an `airbyte` schema in your destination.
+- When you upgrade to a [Destinations V2 compatible version](#destinations-v2-effective-versions) of your destination, we will never alter your existing raw data. Although existing downstream dashboards will go stale, they will never be broken.
+- You can dual write by following the [steps above](#upgrading-connections-one-by-one-with-dual-writing) and copying your raw data to the schema of your newly created connection.
+
+We may make further changes to raw tables in the future, as these tables are intended to be a staging ground for Airbyte to optimize the performance of your syncs. We cannot guarantee the same level of stability as for final tables in your destination schema.
+
+### Upgrade Paths for Connections using CDC
+
+For each [CDC-supported](https://docs.airbyte.com/understanding-airbyte/cdc) source connector, we recommend the following:
+
+| CDC Source | Recommendation                                               | Notes                                                                                                                                                                                                                                                |
+| ---------- | ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Postgres   | [Upgrade connection in place](#quick-start-to-upgrading)     | You can optionally dual write, but this requires resyncing historical data from the source. You must create a new Postgres source with a different replication slot than your existing source to preserve the integrity of your existing connection. |
+| MySQL      | [All above upgrade paths supported](#advanced-upgrade-paths) | You can upgrade the connection in place, or dual write. When dual writing, Airbyte can leverage the state of an existing, active connection to ensure historical data is not re-replicated from MySQL.                                               |
+| SQL Server | [Upgrade connection in place](#quick-start-to-upgrading)     | You can optionally dual write, but this requires resyncing historical data from the SQL Server source.                                                                                                                                               |
+
+## Destinations V2 Compatible Versions
+
+For each destination connector, Destinations V2 is effective as of the following versions:
+
+| Destination Connector | Safe Rollback Version | Destinations V2 Compatible |
+| --------------------- | --------------------- | -------------------------- |
+| BigQuery              | 1.4.4                 | 2.0.0+                     |
+| Snowflake             | 0.4.1                 | 2.0.0+                     |
+| Redshift              | 0.4.8                 | 2.0.0+                     |
+| MSSQL                 | 0.1.24                | 2.0.0+                     |
+| MySQL                 | 0.1.20                | 2.0.0+                     |
+| Oracle                | 0.1.19                | 2.0.0+                     |
+| TiDB                  | 0.1.3                 | 2.0.0+                     |
+| DuckDB                | 0.1.0                 | 2.0.0+                     |
+| Clickhouse            | 0.2.3                 | 2.0.0+                     |
+
+## Destinations V2 Implementation Differences 
+
+In addition to the changes which apply for all destinations described above, there are some per-destination fixes and updates included in Destinations V2:
+
+### BigQuery
+
+1. [Object and array properties](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) are properly stored as JSON columns.  Previously, we had used TEXT, which made querying sub-properties more difficult.
+     * In certain cases, numbers within sub-properties with long decimal values will need to be converted to float representations due to a *quirk* of Bigquery.  Learn more [here](https://github.com/airbytehq/airbyte/issues/29594). 
+
+### Snowflake
+
+1. `destination-snowflake` is now case-sensitive, and was not previously.  This means that if you have a source stream "users", `destination-snowflake` would have previously created a "USERS" table in your data warehouse.  We now correctly create a "users" table.
+    * Note that to properly query case-sensitive tables and columns in Snowflake, you will need to quote your table and column names, e.g. `select "first_name" from "users";`
+    * If you are migrating from Destinations v1 to Destinations V2, we will leave your old "USERS" table, and create a new "users" table - please note the case sensitivity.
+
+## Updating Downstream Transformations
+
+_This section is targeted towards analysts updating downstream models after you've successfully upgraded to Destinations V2._
+
+See here for a [breakdown of changes](#breakdown-of-breaking-changes). Your models will often require updates for the following changes:
+
+#### Column Name Changes
+
+1. `_airbyte_emitted_at_` and `_airbyte_extracted_at` are exactly the same, only the column name changed. You can replace all instances of `_airbyte_emitted_at` with `_airbyte_extracted_at`.
+2. `_airbyte_ab_id` and `_airbyte_raw_id` are exactly the same, only the column name changed. You can replace all instances of `_airbyte_ab_id` with `_airbyte_raw_id`.
+3. Since `_airbyte_normalized_at` is no longer in the final table. We now recommend using `_airbyte_extracted_at` instead.
+
+#### Data Type Changes
+
+You'll get data type errors in downstream models where previously `string` columns are now JSON. In BigQuery, nested JSON values originating from API sources were previously delivered in type `string`. These are now delivered in type `JSON`.
+
+Example: In dbt, you may now get errors with functions such as `regexp_replace`. You can attempt prepending these with `json_extract_array(...)` or `to_json_string(...)` where appropriate.
+
+#### Stale Tables
+
+Unnested tables (e.g. `public.users_address`) do not get deleted during the migration, and are no longer updated. Your downstream models will not throw errors until you drop these tables. Until then, dashboards reliant on these tables will be stale.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index d56a7526c995..b9a5d7d12472 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -47,15 +47,13 @@ You can access Airbyte Slack [here](https://slack.airbyte.com/).
 * `#help-contributions`: for any questions about contributing to Airbyte’s codebase
  
 ## Airbyte Forum
-We are driving our community support from our [forum](https://discuss.airbyte.io/).
+We are driving our community support from our [forum](https://github.com/airbytehq/airbyte/discussions).
 
 **Before posting on this forum please first check if a similar question was already answered.**
 
 **The existing categories**:
-* [Connector Issues](https://discuss.airbyte.io/c/issues/11): Support requests on connector issues.
-* [Platform, Deploy & Infra Issues](https://discuss.airbyte.io/c/deploy-infra-issues/24): Discussion about Airbyte Platform or deploys/infrastructure issues.
-* [API, CLI & Orchestrations Issues](https://discuss.airbyte.io/c/api-cli-orchestration-issues/25): Discussion about how to use or issues with the API, CLI or orchestration
-* [Connector Development](https://discuss.airbyte.io/c/connector-development/16): Ask help when you're developing a new connector.
-* [Q&A](https://discuss.airbyte.io/c/faq/15): Ask anything that doesn’t belong to the other categories.
-* [Guides](https://discuss.airbyte.io/c/guides/17): Small tutorials and guides solving deployments or workarounds using connectors.
-* [Frequently Asked Questions](https://discuss.airbyte.io/c/read-answers-by-the-airbyte-team-to-commonly-asked-questions-from-our-community/22): Read answers from the Airbyte team for commonly asked questions from the community.
\ No newline at end of file
+* 🙏 Questions: Ask the community for help on your question. As a reminder, the Airbyte team won’t provide help here, as our support is part of our Airbyte Cloud and Airbyte Enterprise offers.
+* 💡 Ideas: Share ideas for new features, improvements, or feedback.
+* 🙌 Show & Tell: Share projects, tutorials, videos, and articles you are working on.
+* 🫶 Kind words: Show off something you love about Airbyte
+* 🐙 General: For anything that doesn’t fit in the above categories
diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md
index 1e0eb605f975..bbebf577fdd7 100644
--- a/docs/understanding-airbyte/basic-normalization.md
+++ b/docs/understanding-airbyte/basic-normalization.md
@@ -25,6 +25,7 @@ Basic Normalization uses a fixed set of rules to map a json object from a source
 ```
 
 The destination connectors produce the following raw table in the destination database:
+
 ```sql
 CREATE TABLE "_airbyte_raw_cars" (
     -- metadata added by airbyte
@@ -53,6 +54,7 @@ CREATE TABLE "cars" (
 ## Normalization metadata columns
 
 You'll notice that some metadata are added to keep track of important information about each record.
+
 - Some are introduced at the destination connector level: These are propagated by the normalization process from the raw table to the final table
   - `_airbyte_ab_id`: uuid value assigned by connectors to each row of the data written in the destination.
   - `_airbyte_emitted_at`: time at which the record was emitted and recorded by destination connector.
@@ -61,6 +63,7 @@ You'll notice that some metadata are added to keep track of important informatio
   - `_airbyte_normalized_at`: time at which the record was last normalized (useful to track when incremental transformations are performed)
 
 Additional metadata columns can be added on some tables depending on the usage:
+
 - On the Slowly Changing Dimension (SCD) tables:
   - `_airbyte_start_at`: equivalent to the cursor column defined on the table, denotes when the row was first seen
   - `_airbyte_end_at`: denotes until when the row was seen with these particular values. If this column is not NULL, then the record has been updated and is no longer the most up to date one. If NULL, then the row is the latest version for the record.
@@ -89,19 +92,19 @@ To summarize, we can represent the ELT process in the diagram below. These are s
 
 In Airbyte, the current normalization option is implemented using a dbt Transformer composed of:
 
-* Airbyte base-normalization python package to generate dbt SQL models files
-* dbt to compile and executes the models on top of the data in the destinations that supports it.
+- Airbyte base-normalization python package to generate dbt SQL models files
+- dbt to compile and executes the models on top of the data in the destinations that supports it.
 
 ## Destinations that Support Basic Normalization
 
-* [BigQuery](../integrations/destinations/bigquery.md)
-* [MS Server SQL](../integrations/destinations/mssql.md)
-* [MySQL](../integrations/destinations/mysql.md)
-  * The server must support the `WITH` keyword.
-  * Require MySQL >= 8.0, or MariaDB >= 10.2.1.
-* [Postgres](../integrations/destinations/postgres.md)
-* [Redshift](../integrations/destinations/redshift.md)
-* [Snowflake](../integrations/destinations/snowflake.md)
+- [BigQuery](../integrations/destinations/bigquery.md)
+- [MS Server SQL](../integrations/destinations/mssql.md)
+- [MySQL](../integrations/destinations/mysql.md)
+  - The server must support the `WITH` keyword.
+  - Require MySQL >= 8.0, or MariaDB >= 10.2.1.
+- [Postgres](../integrations/destinations/postgres.md)
+- [Redshift](../integrations/destinations/redshift.md)
+- [Snowflake](../integrations/destinations/snowflake.md)
 
 Basic Normalization can be configured when you're creating the connection between your Connection Setup and after in the Transformation Tab.
 Select the option: **Normalized tabular data**.
@@ -114,16 +117,16 @@ Airbyte tracks types using JsonSchema's primitive types. Here is how these types
 
 Airbyte uses the types described in the catalog to determine the correct type for each column. It does not try to use the values themselves to infer the type.
 
-| JsonSchema Type | Resulting Type | Notes |
-| :--- | :--- | :--- |
-| `number` | float |  |
-| `integer` | integer |  |
-| `string` | string |  |
-| `bit` | boolean |  |
-| `boolean` | boolean |  |
-| `string` with format label `date-time`| timestamp with timezone |  |
-| `array` | new table | see [nesting](basic-normalization.md#Nesting) |
-| `object` | new table | see [nesting](basic-normalization.md#Nesting) |
+| JsonSchema Type                        | Resulting Type          | Notes                                         |
+| :------------------------------------- | :---------------------- | :-------------------------------------------- |
+| `number`                               | float                   |                                               |
+| `integer`                              | integer                 |                                               |
+| `string`                               | string                  |                                               |
+| `bit`                                  | boolean                 |                                               |
+| `boolean`                              | boolean                 |                                               |
+| `string` with format label `date-time` | timestamp with timezone |                                               |
+| `array`                                | new table               | see [nesting](basic-normalization.md#Nesting) |
+| `object`                               | new table               | see [nesting](basic-normalization.md#Nesting) |
 
 ### Nesting
 
@@ -254,11 +257,11 @@ For example, if we had a `cars` table with a nested column `cars` containing an
 
 The expanded table would have a conflict in terms of naming since both are named `cars`. To avoid name collisions and ensure a more consistent naming scheme, Basic Normalization chooses the expanded name as follows:
 
-* `cars` for the original parent table
-* `cars_da3_cars` for the expanded nested columns following this naming scheme in 3 parts: `__`
-* Json path: The entire json path string with '\_' characters used as delimiters to reach the table that contains the nested column name.
-* Hash: Hash of the entire json path to reach the nested column reduced to 3 characters. This is to make sure we have a unique name \(in case part of the name gets truncated, see below\)
-* Nested column name: name of the column being expanded into its own table.
+- `cars` for the original parent table
+- `cars_da3_cars` for the expanded nested columns following this naming scheme in 3 parts: `__`
+- Json path: The entire json path string with '\_' characters used as delimiters to reach the table that contains the nested column name.
+- Hash: Hash of the entire json path to reach the nested column reduced to 3 characters. This is to make sure we have a unique name \(in case part of the name gets truncated, see below\)
+- Nested column name: name of the column being expanded into its own table.
 
 By following this strategy, nested columns should "never" collide with other table names. If it does, an exception will probably be thrown either by the normalization process or by dbt that runs afterward.
 
@@ -300,18 +303,18 @@ However, in the rare cases where these limits are reached:
 
 As an example from the hubspot source, we could have the following tables with nested columns:
 
-| Description | Example 1 | Example 2 |
-| :--- | :--- | :--- |
-| Original Stream Name | companies | deals |
-| Json path to the nested column | `companies/property_engagements_last_meeting_booked_campaign` | `deals/properties/engagements_last_meeting_booked_medium` |
-| Final table name of expanded nested column on BigQuery | companies\_2e8\_property\_engag**ements\_last\_meeting\_bo**oked\_campaign | deals\_prop**erties**\_6e6\_engagements\_l**ast\_meeting\_**booked\_medium |
-| Final table name of expanded nested column on Postgres | companies\_2e8\_property\_engag**\_\_**oked\_campaign | deals\_prop\_6e6\_engagements\_l**\_\_**booked\_medium |
+| Description                                            | Example 1                                                           | Example 2                                                             |
+| :----------------------------------------------------- | :------------------------------------------------------------------ | :-------------------------------------------------------------------- |
+| Original Stream Name                                   | companies                                                           | deals                                                                 |
+| Json path to the nested column                         | `companies/property_engagements_last_meeting_booked_campaign`       | `deals/properties/engagements_last_meeting_booked_medium`             |
+| Final table name of expanded nested column on BigQuery | companies_2e8_property_engag**ements_last_meeting_bo**oked_campaign | deals_prop**erties**\_6e6_engagements_l**ast_meeting\_**booked_medium |
+| Final table name of expanded nested column on Postgres | companies_2e8_property_engag**\_\_**oked_campaign                   | deals_prop_6e6_engagements_l**\_\_**booked_medium                     |
 
 As mentioned in the overview:
 
-* Airbyte places the json blob version of your data in a table called `_airbyte_raw_`.
-* If basic normalization is turned on, it will place a separate copy of the data in a table called ``.
-* In certain pathological cases, basic normalization is required to generate large models with many columns and multiple intermediate transformation steps for a stream. This may break down the "ephemeral" materialization strategy and require the use of additional intermediate views or tables instead. As a result, you may notice additional temporary tables being generated in the destination to handle these checkpoints.
+- Airbyte places the json blob version of your data in a table called `_airbyte_raw_`.
+- If basic normalization is turned on, it will place a separate copy of the data in a table called ``.
+- In certain pathological cases, basic normalization is required to generate large models with many columns and multiple intermediate transformation steps for a stream. This may break down the "ephemeral" materialization strategy and require the use of additional intermediate views or tables instead. As a result, you may notice additional temporary tables being generated in the destination to handle these checkpoints.
 
 ## UI Configurations
 
@@ -321,7 +324,7 @@ To enable basic normalization \(which is optional\), you can toggle it on or dis
 
 ## Incremental runs
 
-When the source is configured with sync modes compatible with incremental transformations (using append on destination) such as ( [full_refresh_append](connections/full-refresh-append.md), [incremental append](connections/incremental-append.md) or  [incremental deduped history](connections/incremental-deduped-history.md)), only rows that have changed in the source are transferred over the network and written by the destination connector.
+When the source is configured with sync modes compatible with incremental transformations (using append on destination) such as ( [full_refresh_append](connections/full-refresh-append.md), [incremental append](connections/incremental-append.md) or [incremental deduped history](connections/incremental-append-deduped.md)), only rows that have changed in the source are transferred over the network and written by the destination connector.
 Normalization will then try to build the normalized tables incrementally as the rows in the raw tables that have been created or updated since the last time dbt ran. As such, on each dbt run, the models get built incrementally. This limits the amount of data that needs to be transformed, vastly reducing the runtime of the transformations. This improves warehouse performance and reduces compute costs.
 Because normalization can be either run incrementally and, or, in full refresh, a technical column `_airbyte_normalized_at` can serve to track when was the last time a record has been transformed and written by normalization.
 This may greatly diverge from the `_airbyte_emitted_at` value as the normalized tables could be totally re-built at a latter time from the data stored in the `_airbyte_raw` tables.
@@ -333,15 +336,15 @@ Normalization produces tables that are partitioned, clustered, sorted or indexed
 In general, normalization needs to do lookup on the last emitted_at column to know if a record is freshly produced and need to be
 incrementally processed or not. But in certain models, such as SCD tables for example, we also need to retrieve older data to update their type 2 SCD end_date and active_row flags, thus a different partitioning scheme is used to optimize that use case.
 
-On Postgres destination, an additional table suffixed with `_stg` for every stream replicated in  [incremental deduped history](connections/incremental-deduped-history.md) needs to be persisted (in a different staging schema) for incremental transformations to work because of a [limitation](https://github.com/dbt-labs/docs.getdbt.com/issues/335#issuecomment-694199569).
+On Postgres destination, an additional table suffixed with `_stg` for every stream replicated in [incremental deduped history](connections/incremental-append-deduped.md) needs to be persisted (in a different staging schema) for incremental transformations to work because of a [limitation](https://github.com/dbt-labs/docs.getdbt.com/issues/335#issuecomment-694199569).
 
 ## Extending Basic Normalization
 
 Note that all the choices made by Normalization as described in this documentation page in terms of naming (and more) could be overridden by your own custom choices. To do so, you can follow the following tutorials:
 
-* to build a [custom SQL view](../operator-guides/transformation-and-normalization/transformations-with-sql.md) with your own naming conventions
-* to export, edit and run [custom dbt normalization](../operator-guides/transformation-and-normalization/transformations-with-dbt.md) yourself
-* or further, you can configure the use of a custom dbt project within Airbyte by following [this guide](../operator-guides/transformation-and-normalization/transformations-with-airbyte.md).
+- to build a [custom SQL view](../operator-guides/transformation-and-normalization/transformations-with-sql.md) with your own naming conventions
+- to export, edit and run [custom dbt normalization](../operator-guides/transformation-and-normalization/transformations-with-dbt.md) yourself
+- or further, you can configure the use of a custom dbt project within Airbyte by following [this guide](../operator-guides/transformation-and-normalization/transformations-with-airbyte.md).
 
 ## CHANGELOG
 
@@ -351,71 +354,71 @@ Note that Basic Normalization is packaged in a docker image `airbyte/normalizati
 
 Therefore, in order to "upgrade" to the desired normalization version, you need to use the corresponding Airbyte version that it's being released in:
 
-| Airbyte Version | Normalization Version      | Date       | Pull Request                                                                                                                         | Subject                                                                                                              |
-|:----------------|:---------------------------|:-----------|:-------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------|
+| Airbyte Version | Normalization Version      | Date       | Pull Request                                                                                                                         | Subject                                                                                                                                                             |
+| :-------------- | :------------------------- | :--------- | :----------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 |                 | 0.4.3                      | 2023-05-11 | [\#25993](https://github.com/airbytehq/airbyte/pull/25993)                                                                           | Fix bug in source-postgres CDC for multiple updates on a single PK in a single transaction (destinations MySQL, MSSQL, TiDB may still be affected in certain cases) |
-|                 | 0.4.2                      | 2023-05-03 | [\#25771](https://github.com/airbytehq/airbyte/pull/25771)                                                                           | Remove old VARCHAR to SUPER migration functionality for destination Redshift                                         |
-|                 | 0.4.1                      | 2023-04-26 | [\#25591](https://github.com/airbytehq/airbyte/pull/25591)                                                                           | Pin MarkupSafe library for Oracle normalization to fix build.                                                        |
-|                 | 0.4.0                      | 2023-03-23 | [\#22381](https://github.com/airbytehq/airbyte/pull/22381)                                                                           | Prevent normalization from creating unnecessary duplicates in nested tables.                                         |
-|                 | 0.2.27                     | 2023-03-15 | [\#24077](https://github.com/airbytehq/airbyte/pull/24077)                                                                           | Add more bigquery reserved words                                                                                     |
-|                 | 0.2.26                     | 2023-02-15 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573)                                                                           | Update Clickhouse dbt version to 1.4.0                                                                               |
-|                 | 0.3.2 (broken, do not use) | 2023-01-31 | [\#22165](https://github.com/airbytehq/airbyte/pull/22165)                                                                           | Fix support for non-object top-level schemas                                                                         |
-|                 | 0.3.1 (broken, do not use) | 2023-01-31 | [\#22161](https://github.com/airbytehq/airbyte/pull/22161)                                                                           | Fix handling for combined primitive types                                                                            |
-|                 | 0.3.0 (broken, do not use) | 2023-01-30 | [\#19721](https://github.com/airbytehq/airbyte/pull/19721)                                                                           | Update normalization to airbyte-protocol v1.0.0                                                                      |
-|                 | 0.2.25                     | 2022-12-05 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573)                                                                           | Update Clickhouse dbt version                                                                                        |
-|                 | 0.2.24                     | 2022-11-01 | [\#18015](https://github.com/airbytehq/airbyte/pull/18015)                                                                           | Add a drop table hook that drops *_scd tables after overwrite/reset                                                  |
-|                 | 0.2.23                     | 2022-10-12 | [\#17483](https://github.com/airbytehq/airbyte/pull/17483) (published in [\#17896](https://github.com/airbytehq/airbyte/pull/17896)) | Remove unnecessary `Native Port` config option                                                                       |
-|                 | 0.2.22                     | 2022-09-05 | [\#16339](https://github.com/airbytehq/airbyte/pull/16339)                                                                           | Update Clickhouse DBT to 1.1.8                                                                                       |
-|                 | 0.2.21                     | 2022-09-09 | [\#15833](https://github.com/airbytehq/airbyte/pull/15833/)                                                                          | SSH Tunnel: allow using OPENSSH key format (published in [\#16545](https://github.com/airbytehq/airbyte/pull/16545)) |
-|                 | 0.2.20                     | 2022-08-30 | [\#15592](https://github.com/airbytehq/airbyte/pull/15592)                                                                           | Add TiDB support                                                                                                     |
-|                 | 0.2.19                     | 2022-08-21 | [\#14897](https://github.com/airbytehq/airbyte/pull/14897)                                                                           | Update Clickhouse DBT to 1.1.7                                                                                       |
-|                 | 0.2.16                     | 2022-08-04 | [\#14295](https://github.com/airbytehq/airbyte/pull/14295)                                                                           | Fixed SSH tunnel port usage                                                                                          |
-|                 | 0.2.14                     | 2022-08-01 | [\#14790](https://github.com/airbytehq/airbyte/pull/14790)                                                                           | Add and persist job failures for Normalization                                                                       |
-|                 | 0.2.13                     | 2022-07-27 | [\#14683](https://github.com/airbytehq/airbyte/pull/14683)                                                                           | Quote schema name to allow reserved keywords                                                                         |
-|                 | 0.2.12                     | 2022-07-26 | [\#14362](https://github.com/airbytehq/airbyte/pull/14362)                                                                           | Handle timezone in date-time format. Parse date correct in clickhouse.                                               |
-|                 | 0.2.11                     | 2022-07-26 | [\#13591](https://github.com/airbytehq/airbyte/pull/13591)                                                                           | Updated support for integer columns.                                                                                 |
-|                 | 0.2.10                     | 2022-07-18 | [\#14792](https://github.com/airbytehq/airbyte/pull/14792)                                                                           | Add support for key pair auth for snowflake                                                                          |
-|                 | 0.2.9                      | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485)                                                                           | BigQuery partition pruning otimization                                                                               |
-|                 | 0.2.8                      | 2022-07-13 | [\#14522](https://github.com/airbytehq/airbyte/pull/14522)                                                                           | BigQuery replaces `NULL` array entries with the string value `"NULL"`                                                |
-|                 | 0.2.7                      | 2022-07-05 | [\#11694](https://github.com/airbytehq/airbyte/pull/11694)                                                                           | Do not return NULL for MySQL column values > 512 chars                                                               |
-|                 | 0.2.6                      | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894)                                                                           | Fix incorrect jinja2 macro `json_extract_array` call                                                                 |
-|                 | 0.2.5                      | 2022-06-15 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470)                                                                           | Upgrade MySQL to dbt 1.0.0                                                                                           |
-|                 | 0.2.4                      | 2022-06-14 | [\#12846](https://github.com/airbytehq/airbyte/pull/12846)                                                                           | CDC correctly deletes propagates deletions to final tables                                                           |
-|                 | 0.2.3                      | 2022-06-10 | [\#11204](https://github.com/airbytehq/airbyte/pull/11204)                                                                           | MySQL: add support for SSh tunneling                                                                                 |
-|                 | 0.2.2                      | 2022-06-02 | [\#13289](https://github.com/airbytehq/airbyte/pull/13289)                                                                           | BigQuery use `json_extract_string_array` for array of simple type elements                                           |
-|                 | 0.2.1                      | 2022-05-17 | [\#12924](https://github.com/airbytehq/airbyte/pull/12924)                                                                           | Fixed checking --event-buffer-size on old dbt crashed entrypoint.sh                                                  |
-|                 | 0.2.0                      | 2022-05-15 | [\#12745](https://github.com/airbytehq/airbyte/pull/12745)                                                                           | Snowflake: add datetime without timezone                                                                             |
-|                 | 0.1.78                     | 2022-05-06 | [\#12305](https://github.com/airbytehq/airbyte/pull/12305)                                                                           | Mssql: use NVARCHAR and datetime2 by default                                                                         |
-| 0.36.2-alpha    | 0.1.77                     | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064)                                                                           | Add support redshift SUPER type                                                                                      |
-| 0.35.65-alpha   | 0.1.75                     | 2022-04-09 | [\#11511](https://github.com/airbytehq/airbyte/pull/11511)                                                                           | Move DBT modules from `/tmp/dbt_modules` to `/dbt`                                                                   |
-| 0.35.61-alpha   | 0.1.74                     | 2022-03-24 | [\#10905](https://github.com/airbytehq/airbyte/pull/10905)                                                                           | Update clickhouse dbt version                                                                                        |
-| 0.35.60-alpha   | 0.1.73                     | 2022-03-25 | [\#11267](https://github.com/airbytehq/airbyte/pull/11267)                                                                           | Set `--event-buffer-size` to reduce memory usage                                                                     |
-| 0.35.59-alpha   | 0.1.72                     | 2022-03-24 | [\#11093](https://github.com/airbytehq/airbyte/pull/11093)                                                                           | Added Snowflake OAuth2.0 support                                                                                     |
-| 0.35.53-alpha   | 0.1.71                     | 2022-03-14 | [\#11077](https://github.com/airbytehq/airbyte/pull/11077)                                                                           | Enable BigQuery to handle project ID embedded inside dataset ID                                                      |
-| 0.35.49-alpha   | 0.1.70                     | 2022-03-11 | [\#11051](https://github.com/airbytehq/airbyte/pull/11051)                                                                           | Upgrade dbt to 1.0.0 (except for MySQL and Oracle)                                                                   |
-| 0.35.45-alpha   | 0.1.69                     | 2022-03-04 | [\#10754](https://github.com/airbytehq/airbyte/pull/10754)                                                                           | Enable Clickhouse normalization over SSL                                                                             |
-| 0.35.32-alpha   | 0.1.68                     | 2022-02-20 | [\#10485](https://github.com/airbytehq/airbyte/pull/10485)                                                                           | Fix row size too large for table with numerous `string` fields                                                       |
-|                 | 0.1.66                     | 2022-02-04 | [\#9341](https://github.com/airbytehq/airbyte/pull/9341)                                                                             | Fix normalization for bigquery datasetId and tables                                                                  |
-| 0.35.13-alpha   | 0.1.65                     | 2021-01-28 | [\#9846](https://github.com/airbytehq/airbyte/pull/9846)                                                                             | Tweak dbt multi-thread parameter down                                                                                |
-| 0.35.12-alpha   | 0.1.64                     | 2021-01-28 | [\#9793](https://github.com/airbytehq/airbyte/pull/9793)                                                                             | Support PEM format for ssh-tunnel keys                                                                               |
-| 0.35.4-alpha    | 0.1.63                     | 2021-01-07 | [\#9301](https://github.com/airbytehq/airbyte/pull/9301)                                                                             | Fix Snowflake prefix tables starting with numbers                                                                    |
-|                 | 0.1.62                     | 2021-01-07 | [\#9340](https://github.com/airbytehq/airbyte/pull/9340)                                                                             | Use TCP-port support for clickhouse                                                                                  |
-|                 | 0.1.62                     | 2021-01-07 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063)                                                                             | Change Snowflake-specific materialization settings                                                                   |
-|                 | 0.1.62                     | 2021-01-07 | [\#9317](https://github.com/airbytehq/airbyte/pull/9317)                                                                             | Fix issue with quoted & case sensitive columns                                                                       |
-|                 | 0.1.62                     | 2021-01-07 | [\#9281](https://github.com/airbytehq/airbyte/pull/9281)                                                                             | Fix SCD partition by float columns in BigQuery                                                                       |
-| 0.32.11-alpha   | 0.1.61                     | 2021-12-02 | [\#8394](https://github.com/airbytehq/airbyte/pull/8394)                                                                             | Fix incremental queries not updating empty tables                                                                    |
-|                 | 0.1.61                     | 2021-12-01 | [\#8378](https://github.com/airbytehq/airbyte/pull/8378)                                                                             | Fix un-nesting queries and add proper ref hints                                                                      |
-| 0.32.5-alpha    | 0.1.60                     | 2021-11-22 | [\#8088](https://github.com/airbytehq/airbyte/pull/8088)                                                                             | Speed-up incremental queries for SCD table on Snowflake                                                              |
-| 0.30.32-alpha   | 0.1.59                     | 2021-11-08 | [\#7669](https://github.com/airbytehq/airbyte/pull/7169)                                                                             | Fix nested incremental dbt                                                                                           |
-| 0.30.24-alpha   | 0.1.57                     | 2021-10-26 | [\#7162](https://github.com/airbytehq/airbyte/pull/7162)                                                                             | Implement incremental dbt updates                                                                                    |
-| 0.30.16-alpha   | 0.1.52                     | 2021-10-07 | [\#6379](https://github.com/airbytehq/airbyte/pull/6379)                                                                             | Handle empty string for date and date-time format                                                                    |
-|                 | 0.1.51                     | 2021-10-08 | [\#6799](https://github.com/airbytehq/airbyte/pull/6799)                                                                             | Added support for ad\_cdc\_log\_pos while normalization                                                              |
-|                 | 0.1.50                     | 2021-10-07 | [\#6079](https://github.com/airbytehq/airbyte/pull/6079)                                                                             | Added support for MS SQL Server normalization                                                                        |
-|                 | 0.1.49                     | 2021-10-06 | [\#6709](https://github.com/airbytehq/airbyte/pull/6709)                                                                             | Forward destination dataset location to dbt profiles                                                                 |
-| 0.29.17-alpha   | 0.1.47                     | 2021-09-20 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317)                                                                             | MySQL: updated MySQL normalization with using SSH tunnel                                                             |
-|                 | 0.1.45                     | 2021-09-18 | [\#6052](https://github.com/airbytehq/airbyte/pull/6052)                                                                             | Snowflake: accept any date-time format                                                                               |
-| 0.29.8-alpha    | 0.1.40                     | 2021-08-18 | [\#5433](https://github.com/airbytehq/airbyte/pull/5433)                                                                             | Allow optional credentials\_json for BigQuery                                                                        |
-| 0.29.5-alpha    | 0.1.39                     | 2021-08-11 | [\#4557](https://github.com/airbytehq/airbyte/pull/4557)                                                                             | Handle date times and solve conflict name btw stream/field                                                           |
-| 0.28.2-alpha    | 0.1.38                     | 2021-07-28 | [\#5027](https://github.com/airbytehq/airbyte/pull/5027)                                                                             | Handle quotes in column names when parsing JSON blob                                                                 |
-| 0.27.5-alpha    | 0.1.37                     | 2021-07-22 | [\#3947](https://github.com/airbytehq/airbyte/pull/4881/)                                                                            | Handle `NULL` cursor field values when deduping                                                                      |
-| 0.27.2-alpha    | 0.1.36                     | 2021-07-09 | [\#3947](https://github.com/airbytehq/airbyte/pull/4163/)                                                                            | Enable normalization for MySQL destination                                                                           |
+|                 | 0.4.2                      | 2023-05-03 | [\#25771](https://github.com/airbytehq/airbyte/pull/25771)                                                                           | Remove old VARCHAR to SUPER migration functionality for destination Redshift                                                                                        |
+|                 | 0.4.1                      | 2023-04-26 | [\#25591](https://github.com/airbytehq/airbyte/pull/25591)                                                                           | Pin MarkupSafe library for Oracle normalization to fix build.                                                                                                       |
+|                 | 0.4.0                      | 2023-03-23 | [\#22381](https://github.com/airbytehq/airbyte/pull/22381)                                                                           | Prevent normalization from creating unnecessary duplicates in nested tables.                                                                                        |
+|                 | 0.2.27                     | 2023-03-15 | [\#24077](https://github.com/airbytehq/airbyte/pull/24077)                                                                           | Add more bigquery reserved words                                                                                                                                    |
+|                 | 0.2.26                     | 2023-02-15 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573)                                                                           | Update Clickhouse dbt version to 1.4.0                                                                                                                              |
+|                 | 0.3.2 (broken, do not use) | 2023-01-31 | [\#22165](https://github.com/airbytehq/airbyte/pull/22165)                                                                           | Fix support for non-object top-level schemas                                                                                                                        |
+|                 | 0.3.1 (broken, do not use) | 2023-01-31 | [\#22161](https://github.com/airbytehq/airbyte/pull/22161)                                                                           | Fix handling for combined primitive types                                                                                                                           |
+|                 | 0.3.0 (broken, do not use) | 2023-01-30 | [\#19721](https://github.com/airbytehq/airbyte/pull/19721)                                                                           | Update normalization to airbyte-protocol v1.0.0                                                                                                                     |
+|                 | 0.2.25                     | 2022-12-05 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573)                                                                           | Update Clickhouse dbt version                                                                                                                                       |
+|                 | 0.2.24                     | 2022-11-01 | [\#18015](https://github.com/airbytehq/airbyte/pull/18015)                                                                           | Add a drop table hook that drops \*\_scd tables after overwrite/reset                                                                                               |
+|                 | 0.2.23                     | 2022-10-12 | [\#17483](https://github.com/airbytehq/airbyte/pull/17483) (published in [\#17896](https://github.com/airbytehq/airbyte/pull/17896)) | Remove unnecessary `Native Port` config option                                                                                                                      |
+|                 | 0.2.22                     | 2022-09-05 | [\#16339](https://github.com/airbytehq/airbyte/pull/16339)                                                                           | Update Clickhouse DBT to 1.1.8                                                                                                                                      |
+|                 | 0.2.21                     | 2022-09-09 | [\#15833](https://github.com/airbytehq/airbyte/pull/15833/)                                                                          | SSH Tunnel: allow using OPENSSH key format (published in [\#16545](https://github.com/airbytehq/airbyte/pull/16545))                                                |
+|                 | 0.2.20                     | 2022-08-30 | [\#15592](https://github.com/airbytehq/airbyte/pull/15592)                                                                           | Add TiDB support                                                                                                                                                    |
+|                 | 0.2.19                     | 2022-08-21 | [\#14897](https://github.com/airbytehq/airbyte/pull/14897)                                                                           | Update Clickhouse DBT to 1.1.7                                                                                                                                      |
+|                 | 0.2.16                     | 2022-08-04 | [\#14295](https://github.com/airbytehq/airbyte/pull/14295)                                                                           | Fixed SSH tunnel port usage                                                                                                                                         |
+|                 | 0.2.14                     | 2022-08-01 | [\#14790](https://github.com/airbytehq/airbyte/pull/14790)                                                                           | Add and persist job failures for Normalization                                                                                                                      |
+|                 | 0.2.13                     | 2022-07-27 | [\#14683](https://github.com/airbytehq/airbyte/pull/14683)                                                                           | Quote schema name to allow reserved keywords                                                                                                                        |
+|                 | 0.2.12                     | 2022-07-26 | [\#14362](https://github.com/airbytehq/airbyte/pull/14362)                                                                           | Handle timezone in date-time format. Parse date correct in clickhouse.                                                                                              |
+|                 | 0.2.11                     | 2022-07-26 | [\#13591](https://github.com/airbytehq/airbyte/pull/13591)                                                                           | Updated support for integer columns.                                                                                                                                |
+|                 | 0.2.10                     | 2022-07-18 | [\#14792](https://github.com/airbytehq/airbyte/pull/14792)                                                                           | Add support for key pair auth for snowflake                                                                                                                         |
+|                 | 0.2.9                      | 2022-07-06 | [\#14485](https://github.com/airbytehq/airbyte/pull/14485)                                                                           | BigQuery partition pruning otimization                                                                                                                              |
+|                 | 0.2.8                      | 2022-07-13 | [\#14522](https://github.com/airbytehq/airbyte/pull/14522)                                                                           | BigQuery replaces `NULL` array entries with the string value `"NULL"`                                                                                               |
+|                 | 0.2.7                      | 2022-07-05 | [\#11694](https://github.com/airbytehq/airbyte/pull/11694)                                                                           | Do not return NULL for MySQL column values > 512 chars                                                                                                              |
+|                 | 0.2.6                      | 2022-06-16 | [\#13894](https://github.com/airbytehq/airbyte/pull/13894)                                                                           | Fix incorrect jinja2 macro `json_extract_array` call                                                                                                                |
+|                 | 0.2.5                      | 2022-06-15 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470)                                                                           | Upgrade MySQL to dbt 1.0.0                                                                                                                                          |
+|                 | 0.2.4                      | 2022-06-14 | [\#12846](https://github.com/airbytehq/airbyte/pull/12846)                                                                           | CDC correctly deletes propagates deletions to final tables                                                                                                          |
+|                 | 0.2.3                      | 2022-06-10 | [\#11204](https://github.com/airbytehq/airbyte/pull/11204)                                                                           | MySQL: add support for SSh tunneling                                                                                                                                |
+|                 | 0.2.2                      | 2022-06-02 | [\#13289](https://github.com/airbytehq/airbyte/pull/13289)                                                                           | BigQuery use `json_extract_string_array` for array of simple type elements                                                                                          |
+|                 | 0.2.1                      | 2022-05-17 | [\#12924](https://github.com/airbytehq/airbyte/pull/12924)                                                                           | Fixed checking --event-buffer-size on old dbt crashed entrypoint.sh                                                                                                 |
+|                 | 0.2.0                      | 2022-05-15 | [\#12745](https://github.com/airbytehq/airbyte/pull/12745)                                                                           | Snowflake: add datetime without timezone                                                                                                                            |
+|                 | 0.1.78                     | 2022-05-06 | [\#12305](https://github.com/airbytehq/airbyte/pull/12305)                                                                           | Mssql: use NVARCHAR and datetime2 by default                                                                                                                        |
+| 0.36.2-alpha    | 0.1.77                     | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064)                                                                           | Add support redshift SUPER type                                                                                                                                     |
+| 0.35.65-alpha   | 0.1.75                     | 2022-04-09 | [\#11511](https://github.com/airbytehq/airbyte/pull/11511)                                                                           | Move DBT modules from `/tmp/dbt_modules` to `/dbt`                                                                                                                  |
+| 0.35.61-alpha   | 0.1.74                     | 2022-03-24 | [\#10905](https://github.com/airbytehq/airbyte/pull/10905)                                                                           | Update clickhouse dbt version                                                                                                                                       |
+| 0.35.60-alpha   | 0.1.73                     | 2022-03-25 | [\#11267](https://github.com/airbytehq/airbyte/pull/11267)                                                                           | Set `--event-buffer-size` to reduce memory usage                                                                                                                    |
+| 0.35.59-alpha   | 0.1.72                     | 2022-03-24 | [\#11093](https://github.com/airbytehq/airbyte/pull/11093)                                                                           | Added Snowflake OAuth2.0 support                                                                                                                                    |
+| 0.35.53-alpha   | 0.1.71                     | 2022-03-14 | [\#11077](https://github.com/airbytehq/airbyte/pull/11077)                                                                           | Enable BigQuery to handle project ID embedded inside dataset ID                                                                                                     |
+| 0.35.49-alpha   | 0.1.70                     | 2022-03-11 | [\#11051](https://github.com/airbytehq/airbyte/pull/11051)                                                                           | Upgrade dbt to 1.0.0 (except for MySQL and Oracle)                                                                                                                  |
+| 0.35.45-alpha   | 0.1.69                     | 2022-03-04 | [\#10754](https://github.com/airbytehq/airbyte/pull/10754)                                                                           | Enable Clickhouse normalization over SSL                                                                                                                            |
+| 0.35.32-alpha   | 0.1.68                     | 2022-02-20 | [\#10485](https://github.com/airbytehq/airbyte/pull/10485)                                                                           | Fix row size too large for table with numerous `string` fields                                                                                                      |
+|                 | 0.1.66                     | 2022-02-04 | [\#9341](https://github.com/airbytehq/airbyte/pull/9341)                                                                             | Fix normalization for bigquery datasetId and tables                                                                                                                 |
+| 0.35.13-alpha   | 0.1.65                     | 2021-01-28 | [\#9846](https://github.com/airbytehq/airbyte/pull/9846)                                                                             | Tweak dbt multi-thread parameter down                                                                                                                               |
+| 0.35.12-alpha   | 0.1.64                     | 2021-01-28 | [\#9793](https://github.com/airbytehq/airbyte/pull/9793)                                                                             | Support PEM format for ssh-tunnel keys                                                                                                                              |
+| 0.35.4-alpha    | 0.1.63                     | 2021-01-07 | [\#9301](https://github.com/airbytehq/airbyte/pull/9301)                                                                             | Fix Snowflake prefix tables starting with numbers                                                                                                                   |
+|                 | 0.1.62                     | 2021-01-07 | [\#9340](https://github.com/airbytehq/airbyte/pull/9340)                                                                             | Use TCP-port support for clickhouse                                                                                                                                 |
+|                 | 0.1.62                     | 2021-01-07 | [\#9063](https://github.com/airbytehq/airbyte/pull/9063)                                                                             | Change Snowflake-specific materialization settings                                                                                                                  |
+|                 | 0.1.62                     | 2021-01-07 | [\#9317](https://github.com/airbytehq/airbyte/pull/9317)                                                                             | Fix issue with quoted & case sensitive columns                                                                                                                      |
+|                 | 0.1.62                     | 2021-01-07 | [\#9281](https://github.com/airbytehq/airbyte/pull/9281)                                                                             | Fix SCD partition by float columns in BigQuery                                                                                                                      |
+| 0.32.11-alpha   | 0.1.61                     | 2021-12-02 | [\#8394](https://github.com/airbytehq/airbyte/pull/8394)                                                                             | Fix incremental queries not updating empty tables                                                                                                                   |
+|                 | 0.1.61                     | 2021-12-01 | [\#8378](https://github.com/airbytehq/airbyte/pull/8378)                                                                             | Fix un-nesting queries and add proper ref hints                                                                                                                     |
+| 0.32.5-alpha    | 0.1.60                     | 2021-11-22 | [\#8088](https://github.com/airbytehq/airbyte/pull/8088)                                                                             | Speed-up incremental queries for SCD table on Snowflake                                                                                                             |
+| 0.30.32-alpha   | 0.1.59                     | 2021-11-08 | [\#7669](https://github.com/airbytehq/airbyte/pull/7169)                                                                             | Fix nested incremental dbt                                                                                                                                          |
+| 0.30.24-alpha   | 0.1.57                     | 2021-10-26 | [\#7162](https://github.com/airbytehq/airbyte/pull/7162)                                                                             | Implement incremental dbt updates                                                                                                                                   |
+| 0.30.16-alpha   | 0.1.52                     | 2021-10-07 | [\#6379](https://github.com/airbytehq/airbyte/pull/6379)                                                                             | Handle empty string for date and date-time format                                                                                                                   |
+|                 | 0.1.51                     | 2021-10-08 | [\#6799](https://github.com/airbytehq/airbyte/pull/6799)                                                                             | Added support for ad_cdc_log_pos while normalization                                                                                                                |
+|                 | 0.1.50                     | 2021-10-07 | [\#6079](https://github.com/airbytehq/airbyte/pull/6079)                                                                             | Added support for MS SQL Server normalization                                                                                                                       |
+|                 | 0.1.49                     | 2021-10-06 | [\#6709](https://github.com/airbytehq/airbyte/pull/6709)                                                                             | Forward destination dataset location to dbt profiles                                                                                                                |
+| 0.29.17-alpha   | 0.1.47                     | 2021-09-20 | [\#6317](https://github.com/airbytehq/airbyte/pull/6317)                                                                             | MySQL: updated MySQL normalization with using SSH tunnel                                                                                                            |
+|                 | 0.1.45                     | 2021-09-18 | [\#6052](https://github.com/airbytehq/airbyte/pull/6052)                                                                             | Snowflake: accept any date-time format                                                                                                                              |
+| 0.29.8-alpha    | 0.1.40                     | 2021-08-18 | [\#5433](https://github.com/airbytehq/airbyte/pull/5433)                                                                             | Allow optional credentials_json for BigQuery                                                                                                                        |
+| 0.29.5-alpha    | 0.1.39                     | 2021-08-11 | [\#4557](https://github.com/airbytehq/airbyte/pull/4557)                                                                             | Handle date times and solve conflict name btw stream/field                                                                                                          |
+| 0.28.2-alpha    | 0.1.38                     | 2021-07-28 | [\#5027](https://github.com/airbytehq/airbyte/pull/5027)                                                                             | Handle quotes in column names when parsing JSON blob                                                                                                                |
+| 0.27.5-alpha    | 0.1.37                     | 2021-07-22 | [\#3947](https://github.com/airbytehq/airbyte/pull/4881/)                                                                            | Handle `NULL` cursor field values when deduping                                                                                                                     |
+| 0.27.2-alpha    | 0.1.36                     | 2021-07-09 | [\#3947](https://github.com/airbytehq/airbyte/pull/4163/)                                                                            | Enable normalization for MySQL destination                                                                                                                          |
diff --git a/docs/understanding-airbyte/connections/README.md b/docs/understanding-airbyte/connections/README.md
index 401f139dbd7b..49a0756a43d9 100644
--- a/docs/understanding-airbyte/connections/README.md
+++ b/docs/understanding-airbyte/connections/README.md
@@ -2,11 +2,11 @@
 
 A connection is a configuration for syncing data between a source and a destination. To setup a connection, a user must configure things such as:
 
-* Sync schedule: when to trigger a sync of the data.
-* Destination [Namespace](../namespaces.md) and stream names: where the data will end up being written.
-* A catalog selection: which [streams and fields](../airbyte-protocol.md#catalog) to replicate from the source
-* Sync mode: how streams should be replicated \(read and write\):
-* Optional transformations: how to convert Airbyte protocol messages \(raw JSON blob\) data into some other data representations. 
+- Sync schedule: when to trigger a sync of the data.
+- Destination [Namespace](../namespaces.md) and stream names: where the data will end up being written.
+- A catalog selection: which [streams and fields](../airbyte-protocol.md#catalog) to replicate from the source
+- Sync mode: how streams should be replicated \(read and write\):
+- Optional transformations: how to convert Airbyte protocol messages \(raw JSON blob\) data into some other data representations.
 
 ## Sync schedules
 
@@ -14,17 +14,17 @@ Sync schedules are explained below. For information about catalog selections, se
 
 Syncs will be triggered by either:
 
-* A manual request \(i.e: clicking the "Sync Now" button in the UI\)
-* A schedule
+- A manual request \(i.e: clicking the "Sync Now" button in the UI\)
+- A schedule
 
 When a scheduled connection is first created, a sync is executed as soon as possible. After that, a sync is run once the time since the last sync \(whether it was triggered manually or due to a schedule\) has exceeded the schedule interval. For example, consider the following illustrative scenario:
 
-* **October 1st, 2pm**, a user sets up a connection to sync data every 24 hours.
-* **October 1st, 2:01pm**: sync job runs
-* **October 2nd, 2:01pm:** 24 hours have passed since the last sync, so a sync is triggered.
-* **October 2nd, 5pm**: The user manually triggers a sync from the UI
-* **October 3rd, 2:01pm:** since the last sync was less than 24 hours ago, no sync is run
-* **October 3rd, 5:01pm:** It has been more than 24 hours since the last sync, so a sync is run
+- **October 1st, 2pm**, a user sets up a connection to sync data every 24 hours.
+- **October 1st, 2:01pm**: sync job runs
+- **October 2nd, 2:01pm:** 24 hours have passed since the last sync, so a sync is triggered.
+- **October 2nd, 5pm**: The user manually triggers a sync from the UI
+- **October 3rd, 2:01pm:** since the last sync was less than 24 hours ago, no sync is run
+- **October 3rd, 5:01pm:** It has been more than 24 hours since the last sync, so a sync is run
 
 ## Destination namespace
 
@@ -46,8 +46,8 @@ A sync mode governs how Airbyte reads from a source and writes to a destination.
 
 1. The first part of the name denotes how the source connector reads data from the source:
    1. Incremental: Read records added to the source since the last sync job. \(The first sync using Incremental is equivalent to a Full Refresh\)
-      * Method 1: Using a cursor. Generally supported by all connectors whose data source allows extracting records incrementally.
-      * Method 2: Using change data capture. Only supported by some sources. See [CDC](../cdc.md) for more info.
+      - Method 1: Using a cursor. Generally supported by all connectors whose data source allows extracting records incrementally.
+      - Method 2: Using change data capture. Only supported by some sources. See [CDC](../cdc.md) for more info.
    2. Full Refresh: Read everything in the source.
 2. The second part of the sync mode name denotes how the destination connector writes data. This is not affected by how the source connector produced the data:
    1. Overwrite: Overwrite by first deleting existing data in the destination.
@@ -56,10 +56,10 @@ A sync mode governs how Airbyte reads from a source and writes to a destination.
 
 A sync mode is therefore, a combination of a source and destination mode together. The UI exposes the following options, whenever both source and destination connectors are capable to support it for the corresponding stream:
 
-* [Full Refresh Overwrite](full-refresh-overwrite.md): Sync the whole stream and replace data in destination by overwriting it.
-* [Full Refresh Append](full-refresh-append.md): Sync the whole stream and append data in destination.
-* [Incremental Append](incremental-append.md): Sync new records from stream and append data in destination.
-* [Incremental Deduped History](incremental-deduped-history.md): Sync new records from stream and append data in destination, also provides a de-duplicated view mirroring the state of the stream in the source.
+- [Full Refresh Overwrite](full-refresh-overwrite.md): Sync the whole stream and replace data in destination by overwriting it.
+- [Full Refresh Append](full-refresh-append.md): Sync the whole stream and append data in destination.
+- [Incremental Append](incremental-append.md): Sync new records from stream and append data in destination.
+- [Incremental Append + Deduped](incremental-append-deduped.md): Sync new records from stream and append data in destination, also provides a de-duplicated view mirroring the state of the stream in the source.
 
 ## Optional operations
 
@@ -69,9 +69,9 @@ As described by the [Airbyte Protocol from the Airbyte Specifications](../airbyt
 
 On top of this replication, Airbyte provides the option to enable or disable an additional transformation step at the end of the sync called [basic normalization](../basic-normalization.md). This operation is:
 
-* Only available for destinations that support dbt execution
-* Automatically generates a pipeline or DAG of dbt transformation models to convert JSON blob objects into normalized tables
-* Runs and applies these dbt models to the data written in the destination
+- Only available for destinations that support dbt execution
+- Automatically generates a pipeline or DAG of dbt transformation models to convert JSON blob objects into normalized tables
+- Runs and applies these dbt models to the data written in the destination
 
 :::note
 
@@ -82,4 +82,3 @@ Normalizing data may cause an increase in your destination's compute cost. This
 ### Custom sync operations
 
 Further operations can be included in a sync on top of Airbyte basic normalization \(or even to replace it completely\). See [operations](../operations.md) for more details.
-
diff --git a/docs/understanding-airbyte/connections/incremental-append-deduped.md b/docs/understanding-airbyte/connections/incremental-append-deduped.md
new file mode 100644
index 000000000000..beff559938b0
--- /dev/null
+++ b/docs/understanding-airbyte/connections/incremental-append-deduped.md
@@ -0,0 +1,121 @@
+# Incremental Sync - Append + Deduped
+
+## High-Level Context
+
+This connector syncs data **incrementally**, which means that only new or modified data will be synced. In contrast with the [Incremental Append mode](./incremental-append.md), this mode updates rows that have been modified instead of adding a new version of the row with the updated data. Simply put, if you've synced a row before and it has since been updated, this mode will combine the two rows
+in the destination and use the most recent data. On the other hand, the [Incremental Append mode](./incremental-append.md) would just add a new row with the updated data.
+
+## Overview
+
+Airbyte supports syncing data in **Incremental Append Deduped** mode i.e:
+
+1. **Incremental** means syncing only replicate _new_ or _modified_ data. This prevents re-fetching data that you have already replicated from a source. If the sync is running for the first time, it is equivalent to a [Full Refresh](full-refresh-append.md) since all data will be considered as _new_.
+2. **Append** means taht this incremental data is added to existing tables in your data warehouse.
+3. **Deduped** means that data in the final table will be unique per primary key \(unlike [Append modes](incremental-append.md)\). This is determined by sorting the data using the cursor field and keeping only the latest de-duplicated data row.
+
+Records in the final destination can potentially be deleted as they are de-duplicated, and if your source supports emitting deleting records (e.g. an CDC database source). You should not find multiple copies of the same primary key as these should be unique in that table.
+
+## Definitions
+
+A `cursor` is the value used to track whether a record should be replicated in an incremental sync. A common example of a `cursor` would be a timestamp from an `updated_at` column in a database table.
+
+A `cursor field` is the _field_ or _column_ in the data where that cursor can be found. Extending the above example, the `updated_at` column in the database would be the `cursor field`, while the `cursor` is the actual timestamp _value_ used to determine if a record should be replicated.
+
+We will refer to the set of records that the source identifies as being new or updated as a `delta`.
+
+A `primary key` is one or multiple \(called `composite primary keys`\) _fields_ or _columns_ that is used to identify the unique entities of a table. Only one row per primary key value is permitted in a database table. In the data warehouse, just like in [incremental - Append](incremental-append.md), multiple rows for the same primary key can be found in the history table. The unique records per primary key behavior is mirrored in the final table with **incremental deduped** sync mode. The primary key is then used to refer to the entity which values should be updated.
+
+## Rules
+
+As mentioned above, the delta from a sync will be _appended_ to the existing history data in the data warehouse. In addition, it will update the associated record in the final table. Let's walk through a few examples.
+
+### Newly Created Record
+
+Assume that `updated_at` is our `cursor_field` and `name` is the `primary_key`. Let's say the following data already exists into our data warehouse.
+
+| name             | deceased | updated_at |
+| :--------------- | :------- | :--------- |
+| Louis XVI        | false    | 1754       |
+| Marie Antoinette | false    | 1755       |
+
+In the next sync, the delta contains the following record:
+
+| name      | deceased | updated_at |
+| :-------- | :------- | :--------- |
+| Louis XVI | false    | 1785       |
+
+At the end of this incremental sync, the data warehouse would now contain:
+
+| name             | deceased | updated_at |
+| :--------------- | :------- | :--------- |
+| Marie Antoinette | false    | 1755       |
+| Louis XVI        | false    | 1785       |
+
+### Updating a Record
+
+Let's assume that our warehouse contains all the data that it did at the end of the previous section. Now, unfortunately the king and queen lose their heads. Let's see that delta:
+
+| name             | deceased | updated_at |
+| :--------------- | :------- | :--------- |
+| Louis XVI        | true     | 1793       |
+| Marie Antoinette | true     | 1793       |
+
+In the final de-duplicated table:
+
+| name             | deceased | updated_at |
+| :--------------- | :------- | :--------- |
+| Louis XVI        | true     | 1793       |
+| Marie Antoinette | true     | 1793       |
+
+## Source-Defined Cursor
+
+Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI.
+
+![](../../.gitbook/assets/incremental_source_defined.png)
+
+\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\).
+
+## User-Defined Cursor
+
+Some sources cannot define the cursor without user input. For example, in the [postgres source](../../integrations/sources/postgres.md), the user needs to choose which column in a database table they want to use as the `cursor field`. In these cases, select the column in the sync settings dropdown that should be used as the `cursor field`.
+
+![](../../.gitbook/assets/incremental_user_defined.png)
+
+\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\).
+
+## Source-Defined Primary key
+
+Some sources are able to determine the primary key that they use without any user input. For example, in the \(JDBC\) Database sources, primary key can be defined in the table's metadata.
+
+## User-Defined Primary key
+
+Some sources cannot define the cursor without user input or the user may want to specify their own primary key on the destination that is different from the source definitions. In these cases, select the column in the sync settings dropdown that should be used as the `primary key` or `composite primary keys`.
+
+![](../../.gitbook/assets/primary_key_user_defined.png)
+
+In this example, we selected both the `campaigns.id` and `campaigns.name` as the composite primary key of our `campaigns` table.
+
+Note that in **Incremental Deduped History**, the size of the data in your warehouse increases monotonically since an updated record in the source is appended to the destination history table rather than updated in-place as it is done with the final table. If you only care about having the latest snapshot of your data, you may want to periodically run cleanup jobs which retain only the latest instance of each record in the history tables.
+
+## Inclusive Cursors
+
+When replicating data incrementally, Airbyte provides an at-least-once delivery guarantee. This means that it is acceptable for sources to re-send some data when ran incrementally. One case where this is particularly relevant is when a source's cursor is not very granular. For example, if a cursor field has the granularity of a day \(but not hours, seconds, etc\), then if that source is run twice in the same day, there is no way for the source to know which records that are that date were already replicated earlier that day. By convention, sources should prefer resending data if the cursor field is ambiguous.
+
+## Known Limitations
+
+Due to the use of a cursor column, if modifications to the underlying records are made without properly updating the cursor field, then the updated records won't be picked up by the **Incremental** sync as expected since the source connectors extract delta rows using a SQL query looking like:
+
+```sql
+select * from table where cursor_field > 'last_sync_max_cursor_field_value'
+```
+
+## Related information
+
+- [An overview of Airbyte’s replication modes](https://airbyte.com/blog/understanding-data-replication-modes).
+- [Explore Airbyte’s incremental data synchronization](https://airbyte.com/tutorials/incremental-data-synchronization).
+
+---
+
+**Note**:
+
+Previous versions of Airbyte destinations supported SCD tables, which would sore every entry seen for a record. This was removed with Destinations V2 and [Typing and Deduplication](/understanding-airbyte/typing-deduping.md).
diff --git a/docs/understanding-airbyte/connections/incremental-append.md b/docs/understanding-airbyte/connections/incremental-append.md
index a779d0b1d13c..c380d2226912 100644
--- a/docs/understanding-airbyte/connections/incremental-append.md
+++ b/docs/understanding-airbyte/connections/incremental-append.md
@@ -80,7 +80,7 @@ Some sources cannot define the cursor without user input. For example, in the [p
 
 As demonstrated in the examples above, with **Incremental Append,** a record which was updated in the source will be appended to the destination rather than updated in-place. This means that if data in the source uses a primary key \(e.g: `user_id` in the `users` table\), then the destination will end up having multiple records with the same primary key value.
 
-However, some use cases require only the latest snapshot of the data. This is available by using other flavors of sync modes such as [Incremental - Deduped History](incremental-deduped-history.md) instead.
+However, some use cases require only the latest snapshot of the data. This is available by using other flavors of sync modes such as [Incremental - Append + Deduped](incremental-append-deduped.md) instead.
 
 Note that in **Incremental Append**, the size of the data in your warehouse increases monotonically since an updated record in the source is appended to the destination rather than updated in-place.
 
diff --git a/docs/understanding-airbyte/connections/incremental-deduped-history.md b/docs/understanding-airbyte/connections/incremental-deduped-history.md
deleted file mode 100644
index be02105b8bfe..000000000000
--- a/docs/understanding-airbyte/connections/incremental-deduped-history.md
+++ /dev/null
@@ -1,161 +0,0 @@
-# Incremental Sync - Deduped History
-
-## High-Level Context
-
-This connector syncs data **incrementally**, which means that only new or modified data will be synced. In contrast with the [Incremental Append mode](./incremental-append.md), this mode updates rows that have been modified instead of adding a new version of the row with the updated data. Simply put, if you've synced a row before and it has since been updated, this mode will combine the two rows
-in the destination and use the updated data. On the other hand, the [Incremental Append mode](./incremental-append.md) would just add a new row with the updated data.
-
-## Overview
-
-Airbyte supports syncing data in **Incremental Deduped History** mode i.e:
-
-1. **Incremental** means syncing only replicate _new_ or _modified_ data. This prevents re-fetching data that you have already replicated from a source. If the sync is running for the first time, it is equivalent to a [Full Refresh](full-refresh-append.md) since all data will be considered as _new_.
-2. **Deduped** means that data in the final table will be unique per primary key \(unlike [Append modes](incremental-append.md)\). This is determined by sorting the data using the cursor field and keeping only the latest de-duplicated data row. In dimensional data warehouse jargon defined by Ralph Kimball, this is referred as a Slowly Changing Dimension \(SCD\) table of type 1.
-3. **History** means that an additional intermediate table is created in which data is being continuously appended to \(with duplicates exactly like [Append modes](incremental-append.md)\). With the use of primary key fields, it is identifying effective `start` and `end` dates of each row of a record. In dimensional data warehouse jargon, this is referred as a Slowly Changing Dimension \(SCD\) table of type 2.
-
-In this flavor of incremental, records in the warehouse destination will never be deleted in the history tables \(named with a `_scd` suffix\), but might not exist in the final table. A copy of each new or updated record is _appended_ to the history data in the warehouse. Only the `end` date column is mutated when a new version of the same record is inserted to denote effective date ranges of a row. This means you can find multiple copies of the same record in the destination warehouse. We provide an "at least once" guarantee of replicating each record that is present when the sync runs.
-
-On the other hand, records in the final destination can potentially be deleted as they are de-duplicated. You should not find multiple copies of the same primary key as these should be unique in that table.
-
-## Definitions
-
-A `cursor` is the value used to track whether a record should be replicated in an incremental sync. A common example of a `cursor` would be a timestamp from an `updated_at` column in a database table.
-
-A `cursor field` is the _field_ or _column_ in the data where that cursor can be found. Extending the above example, the `updated_at` column in the database would be the `cursor field`, while the `cursor` is the actual timestamp _value_ used to determine if a record should be replicated.
-
-We will refer to the set of records that the source identifies as being new or updated as a `delta`.
-
-A `primary key` is one or multiple \(called `composite primary keys`\) _fields_ or _columns_ that is used to identify the unique entities of a table. Only one row per primary key value is permitted in a database table. In the data warehouse, just like in [incremental - Append](incremental-append.md), multiple rows for the same primary key can be found in the history table. The unique records per primary key behavior is mirrored in the final table with **incremental deduped** sync mode. The primary key is then used to refer to the entity which values should be updated.
-
-## Rules
-
-As mentioned above, the delta from a sync will be _appended_ to the existing history data in the data warehouse. In addition, it will update the associated record in the final table. Let's walk through a few examples.
-
-### Newly Created Record
-
-Assume that `updated_at` is our `cursor_field` and `name` is the `primary_key`. Let's say the following data already exists into our data warehouse.
-
-| name             | deceased | updated_at |
-| :--------------- | :------- | :--------- |
-| Louis XVI        | false    | 1754       |
-| Marie Antoinette | false    | 1755       |
-
-In the next sync, the delta contains the following record:
-
-| name       | deceased | updated_at |
-| :--------- | :------- | :--------- |
-| Louis XVII | false    | 1785       |
-
-At the end of this incremental sync, the data warehouse would now contain:
-
-| name             | deceased | updated_at |
-| :--------------- | :------- | :--------- |
-| Louis XVI        | false    | 1754       |
-| Marie Antoinette | false    | 1755       |
-| Louis XVII       | false    | 1785       |
-
-### Updating a Record
-
-Let's assume that our warehouse contains all the data that it did at the end of the previous section. Now, unfortunately the king and queen lose their heads. Let's see that delta:
-
-| name             | deceased | updated_at |
-| :--------------- | :------- | :--------- |
-| Louis XVI        | true     | 1793       |
-| Marie Antoinette | true     | 1793       |
-
-The output we expect to see in the warehouse is as follows:
-
-In the history table:
-
-| name             | deceased | updated_at | start_at | end_at |
-| :--------------- | :------- | :--------- | :------- | :----- |
-| Louis XVI        | false    | 1754       | 1754     | 1793   |
-| Louis XVI        | true     | 1793       | 1793     | NULL   |
-| Louis XVII       | false    | 1785       | 1785     | NULL   |
-| Marie Antoinette | false    | 1755       | 1755     | 1793   |
-| Marie Antoinette | true     | 1793       | 1793     | NULL   |
-
-In the final de-duplicated table:
-
-| name             | deceased | updated_at |
-| :--------------- | :------- | :--------- |
-| Louis XVI        | true     | 1793       |
-| Louis XVII       | false    | 1785       |
-| Marie Antoinette | true     | 1793       |
-
-## Source-Defined Cursor
-
-Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI.
-
-![](../../.gitbook/assets/incremental_source_defined.png)
-
-\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\).
-
-## User-Defined Cursor
-
-Some sources cannot define the cursor without user input. For example, in the [postgres source](../../integrations/sources/postgres.md), the user needs to choose which column in a database table they want to use as the `cursor field`. In these cases, select the column in the sync settings dropdown that should be used as the `cursor field`.
-
-![](../../.gitbook/assets/incremental_user_defined.png)
-
-\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\).
-
-## Source-Defined Primary key
-
-Some sources are able to determine the primary key that they use without any user input. For example, in the \(JDBC\) Database sources, primary key can be defined in the table's metadata.
-
-## User-Defined Primary key
-
-Some sources cannot define the cursor without user input or the user may want to specify their own primary key on the destination that is different from the source definitions. In these cases, select the column in the sync settings dropdown that should be used as the `primary key` or `composite primary keys`.
-
-![](../../.gitbook/assets/primary_key_user_defined.png)
-
-In this example, we selected both the `campaigns.id` and `campaigns.name` as the composite primary key of our `campaigns` table.
-
-Note that in **Incremental Deduped History**, the size of the data in your warehouse increases monotonically since an updated record in the source is appended to the destination history table rather than updated in-place as it is done with the final table. If you only care about having the latest snapshot of your data, you may want to periodically run cleanup jobs which retain only the latest instance of each record in the history tables.
-
-## Inclusive Cursors
-
-When replicating data incrementally, Airbyte provides an at-least-once delivery guarantee. This means that it is acceptable for sources to re-send some data when ran incrementally. One case where this is particularly relevant is when a source's cursor is not very granular. For example, if a cursor field has the granularity of a day \(but not hours, seconds, etc\), then if that source is run twice in the same day, there is no way for the source to know which records that are that date were already replicated earlier that day. By convention, sources should prefer resending data if the cursor field is ambiguous.
-
-## Known Limitations
-
-Due to the use of a cursor column, if modifications to the underlying records are made without properly updating the cursor field, then the updated records won't be picked up by the **Incremental** sync as expected since the source connectors extract delta rows using a SQL query looking like:
-
-```sql
-select * from table where cursor_field > 'last_sync_max_cursor_field_value'
-```
-
-Let's say the following data already exists into our data warehouse.
-
-| name             | deceased | updated_at |
-| :--------------- | :------- | :--------- |
-| Louis XVI        | false    | 1754       |
-| Marie Antoinette | false    | 1755       |
-
-At the start of the next sync, the source data contains the following new record:
-
-| name      | deceased | updated_at |
-| :-------- | :------- | :--------- |
-| Louis XVI | true     | 1754       |
-
-At the end of the second incremental sync, the data warehouse would still contain data from the first sync because the delta record did not provide a valid value for the cursor field \(the cursor field is not greater than last sync's max value, `1754 < 1755`\), so it is not emitted by the source as a new or modified record.
-
-| name             | deceased | updated_at |
-| :--------------- | :------- | :--------- |
-| Louis XVI        | false    | 1754       |
-| Marie Antoinette | false    | 1755       |
-
-Similarly, if multiple modifications are made during the same day to the same records. If the frequency of the sync is not granular enough \(for example, set for every 24h\), then intermediate modifications to the data are not going to be detected and emitted. Only the state of data at the time the sync runs will be reflected in the destination.
-
-Those concerns could be solved by using a different incremental approach based on binary logs, Write-Ahead-Logs \(WAL\), or also called [Change Data Capture \(CDC\)](../cdc.md).
-
-The current behavior of **Incremental** is not able to handle source schema changes yet, for example, when a column is added, renamed or deleted from an existing table etc. It is recommended to trigger a [Full refresh - Overwrite](full-refresh-overwrite.md) to correctly replicate the data to the destination with the new schema changes.
-
-Additionally, this sync mode is only supported for destinations where dbt/normalization is possible for the moment. The de-duplicating logic is indeed implemented as dbt models as part of a sequence of transformations applied after the Extract and Load activities \(thus, an ELT approach\). Nevertheless, it is theoretically possible that destinations can handle directly this logic \(maybe in the future\) before actually writing records to the destination \(as in traditional ETL manner\), but that's not the way it is implemented at this time.
-
-If you are not satisfied with how transformations are applied on top of the appended data, you can find more relevant SQL transformations you might need to do on your data in the [Connecting EL with T using SQL \(part 1/2\)](../../operator-guides/transformation-and-normalization/transformations-with-sql.md)
-
-## Related information
-
-- [An overview of Airbyte’s replication modes](https://airbyte.com/blog/understanding-data-replication-modes).
-- [Explore Airbyte’s incremental data synchronization](https://airbyte.com/tutorials/incremental-data-synchronization).
diff --git a/docs/understanding-airbyte/database-data-catalog.md b/docs/understanding-airbyte/database-data-catalog.md
index 69bf8803c7c6..fa0b7dfd3dc0 100644
--- a/docs/understanding-airbyte/database-data-catalog.md
+++ b/docs/understanding-airbyte/database-data-catalog.md
@@ -7,7 +7,7 @@
   * Each record represents a connector that Airbyte supports, e.g. Postgres. This table represents all the connectors that is supported by the current running platform.
   * The `actor_type` column tells us whether the record represents a Source or a Destination.
   * The `spec` column is a JSON blob. The schema of this JSON blob matches the [spec](airbyte-protocol.md#actor-specification) model in the Airbyte Protocol. Because the protocol object is JSON, this has to be a JSON blob.
-  * The `release_stage` describes the certification level of the connector (e.g. Alpha, Beta, Generally Available).
+  * The `support_level` describes the support level of the connector (e.g. community, certified).
   * The `docker_repository` field is the name of the docker image associated with the connector definition. `docker_image_tag` is the tag of the docker image and the version of the connector definition.
   * The `source_type` field is only used for Sources, and represents the category of the connector definition (e.g. API, Database).
   * The `resource_requirements` field sets a default resource requirement for any connector of this type. This overrides the default we set for all connector definitions, and it can be overridden by a connection-specific resource requirement. The column is a JSON blob with the schema defined in [ActorDefinitionResourceRequirements.yaml](https://github.com/airbytehq/airbyte/blob/master/airbyte-config-oss/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml)
@@ -34,7 +34,7 @@
 * `connection`
   * Each record in this table configures a connection (`source_id`, `destination_id`, and relevant configuration).
   * The `resource_requirements` field sets a default resource requirement for the connection. This overrides the default we set for all connector definitions and the default set for the connector definitions. The column is a JSON blob with the schema defined in [ResourceRequirements.yaml](https://github.com/airbytehq/airbyte/blob/master/airbyte-config-oss/config-models-oss/src/main/resources/types/ResourceRequirements.yaml).
-  * The `source_catalog_id` column is a foreign key to the `sourc_catalog` table and represents the catalog that was used to configure the connection. This should not be confused with the `catalog` column which contains the [ConfiguredCatalog](airbyte-protocol.md#catalog) for the connection.
+  * The `source_catalog_id` column is a foreign key that refers to `id` column in `actor_catalog` table and represents the catalog that was used to configure the connection. This should not be confused with the `catalog` column which contains the [ConfiguredCatalog](airbyte-protocol.md#catalog) for the connection.
   * The `schedule_type` column defines what type of schedule is being used. If the `type` is manual, then `schedule_data` will be null. Otherwise, `schedule_data` column is a JSON blob with the schema of [StandardSync#scheduleData](https://github.com/airbytehq/airbyte/blob/master/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSync.yaml#L74) that defines the actual schedule. The columns `manual` and `schedule` are deprecated and should be ignored (they will be dropped soon).
   * The `namespace_type` column configures whether the namespace for the connection should use that defined by the source, the destination, or a user-defined format (`custom`). If `custom` the `namespace_format` column defines the string that will be used as the namespace.
   * The `status` column describes the activity level of the connector: `active` - current schedule is respected, `inactive` - current schedule is ignored (the connection does not run) but it could be switched back to active, and `deprecated` - the connection is permanently off (cannot be moved to active or inactive).
@@ -55,11 +55,11 @@
   * If the `operation` is `normalization`, then the `operator_dbt` column will be populated with a JSON blob with the scehma from [OperatorNormalization](https://github.com/airbytehq/airbyte/blob/master/airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorNormalization.yaml).
   * Operations are scoped by workspace, using the `workspace_id` column.
 * `connection_operation`
-  * This table joins the `operation` table to the `connection` for which it is configured. 
+  * This table joins the `operation` table to the `connection` for which it is configured.
 * `workspace_service_account`
   * This table is a WIP for an unfinished feature.
 * `actor_oauth_parameter`
-  * The name of this table is misleading. It refers to parameters to be used for any instance of an `actor_definition` (not an `actor`) within a given workspace. For OAuth, the model is that a user is provisioning access to their data to a third party tool (in this case the Airbyte Platform). Each record represents information (e.g. client id, client secret) for that third party that is getting access. 
+  * The name of this table is misleading. It refers to parameters to be used for any instance of an `actor_definition` (not an `actor`) within a given workspace. For OAuth, the model is that a user is provisioning access to their data to a third party tool (in this case the Airbyte Platform). Each record represents information (e.g. client id, client secret) for that third party that is getting access.
   * These parameters can be scoped by workspace. If `workspace_id` is not present, then the scope of the parameters is to the whole deployment of the platform (e.g. all workspaces).
   * The `actor_type` column tells us whether the record represents a Source or a Destination.
   * The `configuration` column is a JSON blob. The schema of this JSON blob matches the schema specified in the `spec` column in the `advanced_auth` field of the JSON blob. Keep in mind this schema is specific to each connector (e.g. the schema of Hubspot and Salesforce are different), which is why this column has to be a JSON blob.
diff --git a/docs/understanding-airbyte/jobs.md b/docs/understanding-airbyte/jobs.md
index e482d27d848a..0b577a72a5a2 100644
--- a/docs/understanding-airbyte/jobs.md
+++ b/docs/understanding-airbyte/jobs.md
@@ -11,14 +11,177 @@ Thus, there are generally 4 types of workers.
 
 **Note: Workers here refers to Airbyte workers. Temporal, which Airbyte uses under the hood for scheduling, has its own worker concept. This distinction is important.**
 
-## Job State Machine
+## Sync Jobs
 
-Jobs have the following state machine.
+At a high level, a sync job is an individual invocation of the Airbyte pipeline to synchronize data from a source to a destination data store.
+
+### Sync Job State Machine
+
+Sync jobs have the following state machine.
 
 ![Job state machine](../.gitbook/assets/job-state-machine.png)
 
 [Image Source](https://docs.google.com/drawings/d/1cp8LRZs6UnhAt3jbQ4h40nstcNB0OBOnNRdMFwOJL8I/edit)
 
+### Attempts and Retries
+
+In the event of a failure, the Airbyte platform will retry the pipeline. Each of these sub-invocations of a job is called an attempt.
+
+### Retry Rules
+
+Based on the outcome of previous attempts, the number of permitted attempts per job changes. By default, Airbyte is configured to allow the following:
+
+* 5 subsequent attempts where no data was synchronized
+* 10 total attempts where no data was synchronized
+* 10 total attempts where some data was synchronized
+
+For oss users, these values are configurable. See [Configuring Airbyte](../operator-guides/configuring-airbyte.md#jobs) for more details.
+
+### Retry Backoff
+
+After an attempt where no data was synchronized, we implement a short backoff period before starting a new attempt. This will increase with each successive complete failure—a partially successful attempt will reset this value.
+
+By default, Airbyte is configured to backoff with the following values:
+* 10 seconds after the first complete failure
+* 30 seconds after the second
+* 90 seconds after the third
+* 4 minutes and 30 seconds after the fourth
+
+For oss users, these values are configurable. See [Configuring Airbyte](../operator-guides/configuring-airbyte.md#jobs) for more details.
+
+The duration of expected backoff between attempts can be viewed in the logs accessible from the job history UI.
+
+### Retry examples
+
+To help illustrate what is possible, below are a couple examples of how the retry rules may play out under more elaborate circumstances. 
+
+
    
+    
+        
+            
+        
+        
+            
+            
+        
+    
+    
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+        
+    
+
    Job #1
    Attempt NumberSynced data?
    1No
    10 second backoff
    2No
    30 second backoff
    3Yes
    4Yes
    5Yes
    6No
    10 second backoff
    7Yes
    Job succeeds — all data synced
    
+
+
+    
+        
+            
+        
+        
+            
+            
+        
+    
+    
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+        
+        
+            
+            
+        
+        
+            
+        
+    
+
    Job #2
    Attempt NumberSynced data?
    1Yes
    2Yes
    3Yes
    4Yes
    5Yes
    6Yes
    7No
    10 second backoff
    8No
    30 second backoff
    9No
    90 second backoff
    10No
    4 minute 30 second backoff
    11No
    Job Fails — successive failure limit reached
    
+
 ## Worker Responsibilities
 
 The worker has the following responsibilities.
@@ -99,11 +262,15 @@ The Cloud Storage store is treated as the source-of-truth of execution state.
 
 The Container Orchestrator is only available for Airbyte Kubernetes today and automatically enabled when running the Airbyte Helm charts/Kustomize deploys.
 
+![Orchestrator Lifecycle](../.gitbook/assets/orchestrator-lifecycle.png)
+
+[Image Source](https://whimsical.com/sync-lifecycle-Vays9o1YaxCKPhUEEKmqHM@2bsEvpTYSt1HjEZjriPY9jiAqCmgJ41MmyY)
+
 Users running Airbyte Docker should be aware of the above pitfalls.
 
-## Configuring Workers
+## Configuring Jobs & Workers
 
-Details on configuring workers can be found [here](../operator-guides/configuring-airbyte.md).
+Details on configuring jobs & workers can be found [here](../operator-guides/configuring-airbyte.md).
 
 ### Worker Parallization
 Airbyte exposes the following environment variable to change the maximum number of each type of worker allowed to run in parallel. 
diff --git a/docs/understanding-airbyte/supported-data-types.md b/docs/understanding-airbyte/supported-data-types.md
index ca84b13759fd..f588436e6aa6 100644
--- a/docs/understanding-airbyte/supported-data-types.md
+++ b/docs/understanding-airbyte/supported-data-types.md
@@ -10,20 +10,20 @@ This type system does not constrain values. However, destinations may not fully
 
 This table summarizes the available types. See the [Specific Types](#specific-types) section for explanation of optional parameters.
 
-| Airbyte type                                                   | JSON Schema                                                                               | Examples                                                                        |
-| -------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- |
-| String                                                         | `{"type": "string""}`                                                                     | `"foo bar"`                                                                     |
-| Boolean                                                        | `{"type": "boolean"}`                                                                     | `true` or `false`                                                               |
-| Date                                                           | `{"type": "string", "format": "date"}`                                                    | `"2021-01-23"`, `"2021-01-23 BC"`                                               |
-| Timestamp with timezone                                        | `{"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}`    | `"2022-11-22T01:23:45.123456+05:00"`, `"2022-11-22T01:23:45Z BC"`               |
-| Timestamp without timezone                                     | `{"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}` | `"2022-11-22T01:23:45"`, `"2022-11-22T01:23:45.123456 BC"`                      |
-| Time without timezone                                          | `{"type": "string", "airbyte_type": "time_with_timezone"}`                                | `"01:23:45.123456"`, `"01:23:45"`                                               |
-| Time with timezone                                             | `{"type": "string", "airbyte_type": "time_without_timezone"}`                             | `"01:23:45.123456+05:00"`, `"01:23:45Z"`                                        |
-| Integer                                                        | `{"type": "integer"}` or `{"type": "number", "airbyte_type": "integer"}`                  | `42`                                                                            |
-| Number                                                         | `{"type": "number"}`                                                                      | `1234.56`                                                                       |
-| Array                                                          | `{"type": "array"}`; optionally `items`                                                   | `[1, 2, 3]`                                                                     |
-| Object                                                         | `{"type": "object"}`; optionally `properties`                                             | `{"foo": "bar"}`                                                                |
-| Union                                                          | `{"oneOf": [...]}`                                                                        |                                                                                 |ß
+| Airbyte type               | JSON Schema                                                                                         | Examples                                                          |
+|----------------------------|-----------------------------------------------------------------------------------------------------|-------------------------------------------------------------------|
+| String                     | `{"type": "string"}`                                                                                | `"foo bar"`                                                       |
+| Boolean                    | `{"type": "boolean"}`                                                                               | `true` or `false`                                                 |
+| Date                       | `{"type": "string", "format": "date"}`                                                              | `"2021-01-23"`, `"2021-01-23 BC"`                                 |
+| Timestamp without timezone | `{"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}`           | `"2022-11-22T01:23:45"`, `"2022-11-22T01:23:45.123456 BC"`        |
+| Timestamp with timezone    | `{"type": "string", "format": "date-time"}`; optionally `"airbyte_type": "timestamp_with_timezone"` | `"2022-11-22T01:23:45.123456+05:00"`, `"2022-11-22T01:23:45Z BC"` |
+| Time without timezone      | `{"type": "string", "format": "time", "airbyte_type": "time_without_timezone"}`                     | `"01:23:45.123456"`, `"01:23:45"`                                 |
+| Time with timezone         | `{"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}`                        | `"01:23:45.123456+05:00"`, `"01:23:45Z"`                          |
+| Integer                    | `{"type": "integer"}` or `{"type": "number", "airbyte_type": "integer"}`                            | `42`                                                              |
+| Number                     | `{"type": "number"}`                                                                                | `1234.56`                                                         |
+| Array                      | `{"type": "array"}`; optionally `items`                                                             | `[1, 2, 3]`                                                       |
+| Object                     | `{"type": "object"}`; optionally `properties`                                                       | `{"foo": "bar"}`                                                  |
+| Union                      | `{"oneOf": [...]}`                                                                                  |                                                                   |
 
 ### Record structure
 As a reminder, sources expose a `discover` command, which returns a list of [`AirbyteStreams`](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L122), and a `read` method, which emits a series of [`AirbyteRecordMessages`](https://github.com/airbytehq/airbyte/blob/111131a193359027d0081de1290eb4bb846662ef/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L46-L66). The type system determines what a valid `json_schema` is for an `AirbyteStream`, which in turn dictates what messages `read` is allowed to emit.
@@ -46,7 +46,7 @@ For example, a source could produce this `AirbyteStream` (remember that the `jso
         "items": {
           "type": "string",
           "format": "date-time",
-          "airbyte_type": "timestampt_with_timezone"
+          "airbyte_type": "timestamp_with_timezone"
         }
       }
     }
diff --git a/docs/understanding-airbyte/typing-deduping.md b/docs/understanding-airbyte/typing-deduping.md
new file mode 100644
index 000000000000..257bca566884
--- /dev/null
+++ b/docs/understanding-airbyte/typing-deduping.md
@@ -0,0 +1,90 @@
+# Typing and Deduping
+
+This page refers to new functionality currently available in **early access**. Typing and deduping will become the new default method of transforming datasets within data warehouse and database destinations after they've been replicated. This functionality is going live with [Destinations V2](/release_notes/upgrading_to_destinations_v2/), which is now in early access for BigQuery.
+
+You will eventually be required to upgrade your connections to use the new destination versions. We are building tools for you to copy your connector’s configuration to a new version to make testing new destinations easier. These will be available in the next few weeks.
+
+## What is Destinations V2?
+
+At launch, [Airbyte Destinations V2](/release_notes/upgrading_to_destinations_v2) will provide:
+
+- One-to-one table mapping: Data in one stream will always be mapped to one table in your data warehouse. No more sub-tables.
+- Improved per-row error handling with `_airbyte_meta`: Airbyte will now populate typing errors in the `_airbyte_meta` column instead of failing your sync. You can query these results to audit misformatted or unexpected data.
+- Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables in the `airbyte_internal` schema. We no longer clutter your desired schema with raw data tables.
+- Incremental delivery for large syncs: Data will be incrementally delivered to your final tables when possible. No more waiting hours to see the first rows in your destination table.
+
+## `_airbyte_meta` Errors
+
+"Per-row error handling" is a new paradigm for Airbyte which provides greater flexibility for our users.  Airbyte now separates `data-moving problems` from `data-content problems`. Prior to Destinations V2, both types of errors were handled the same way: by failing the sync. Now, a failing sync means that Airbyte could not _move_ all of your data. You can query the `_airbyte_meta` column to see which rows failed for _content_ reasons, and why. This is a more flexible approach, as you can now decide how to handle rows with errors on a case-by-case basis.
+
+:::tip
+When using data downstream from Airbyte, we generally recommend you only include rows which do not have an error, e.g:
+
+```sql
+-- postgres syntax
+SELECT COUNT(*) FROM _table_ WHERE json_array_length(_airbyte_meta ->> errors) = 0
+```
+
+:::
+
+The types of errors which will be stored in `_airbyte_meta.errors` include:
+
+- **Typing errors**: the source declared that the type of the column `id` should be an integer, but a string value was returned.
+- **Size errors**: the source returned content which cannot be stored within this this row or column (e.g. [a Redshift Super column has a 16mb limit](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html)).
+
+Depending on your use-case, it may still be valuable to consider rows with errors, especially for aggregations. For example, you may have a table `user_reviews`, and you would like to know the count of new reviews received today. You can choose to include reviews regardless of whether your data warehouse had difficulty storing the full contents of the `message` column. For this use case, `SELECT COUNT(*) from user_reviews WHERE DATE(created_at) = DATE(NOW())` is still valid.
+
+## Destinations V2 Example
+
+Consider the following [source schema](https://docs.airbyte.com/integrations/sources/faker) for stream `users`:
+
+```json
+{
+  "id": "number",
+  "first_name": "string",
+  "age": "number",
+  "address": {
+    "city": "string",
+    "zip": "string"
+  }
+}
+```
+
+The data from one stream will now be mapped to one table in your schema as below:
+
+#### Destination Table Name: _public.users_
+
+| _(note, not in actual table)_                | \_airbyte_raw_id | \_airbyte_extracted_at | \_airbyte_meta                                               | id  | first_name | age  | address                                 |
+| -------------------------------------------- | ---------------- | ---------------------- | ------------------------------------------------------------ | --- | ---------- | ---- | --------------------------------------- |
+| Successful typing and de-duping ⟶            | xxx-xxx-xxx      | 2022-01-01 12:00:00    | {}                                                           | 1   | sarah      | 39   | { city: “San Francisco”, zip: “94131” } |
+| Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy      | 2022-01-01 12:00:00    | { errors: {[“fish” is not a valid integer for column “age”]} | 2   | evan       | NULL | { city: “Menlo Park”, zip: “94002” }    |
+| Not-yet-typed ⟶                              |                  |                        |                                                              |     |            |      |                                         |
+
+In legacy normalization, columns of [Airbyte type](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) `Object` in the Destination were "unnested" into separate tables. In this example, with Destinations V2, the previously unnested `public.users_address` table with columns `city` and `zip` will no longer be generated.
+
+#### Destination Table Name: _airbyte.raw_public_users_ (`airbyte.{namespace}_{stream}`)
+
+| _(note, not in actual table)_                | \_airbyte_raw_id | \_airbyte_data                                                                           | \_airbyte_loaded_at  | \_airbyte_extracted_at |
+| -------------------------------------------- | ---------------- | ----------------------------------------------------------------------------------------- | -------------------- | ---------------------- |
+| Successful typing and de-duping ⟶            | xxx-xxx-xxx      | { id: 1, first_name: “sarah”, age: 39, address: { city: “San Francisco”, zip: “94131” } } | 2022-01-01 12:00:001 | 2022-01-01 12:00:00   |
+| Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy      | { id: 2, first_name: “evan”, age: “fish”, address: { city: “Menlo Park”, zip: “94002” } } | 2022-01-01 12:00:001 | 2022-01-01 12:00:00   |
+| Not-yet-typed ⟶                              | zzz-zzz-zzz      | { id: 3, first_name: “edward”, age: 35, address: { city: “Sunnyvale”, zip: “94003” } }    | NULL                 | 2022-01-01 13:00:00   |
+
+You also now see the following changes in Airbyte-provided columns:
+
+![Airbyte Destinations V2 Column Changes](../release_notes/assets/destinations-v2-column-changes.png)
+
+## Participating in Early Access
+
+You can start using Destinations V2 for BigQuery or Snowflake in early access by following the below instructions:
+
+1. **Upgrade your Destination**: If you are using Airbyte Open Source, update your destination version to the latest version. If you are a Cloud customer, this step will already be completed on your behalf.
+2. **Enabling Destinations V2**: Create a new destination, and enable the Destinations V2 option under `Advanced` settings. You will need your data warehouse credentials for this step. For this early release, we ask that you enable Destinations V2 on a new destination using new connections. When Destinations V2 is fully available, there will be additional migration paths for upgrading your destination without resetting any of your existing connections.
+   1. If your previous BigQuery destination is using “GCS Staging”, you can reuse the same staging bucket.
+   2. Do not enable Destinations V2 on your previous / existing destinations during early release. It will cause your existing connections to fail.
+3. **Create a New Connection**: Create connections using the new BigQuery destination. These will automatically use Destinations V2.
+   1. If your new destination has the same default namespace, you may want to add a stream prefix to avoid collisions in the final tables.
+   2. Do not modify the ‘Transformation’ settings. These will be ignored.
+4. **Monitor your Sync**: Wait at least 20 minutes, or until your sync is complete. Verify the data in your destination is correct. Congratulations, you have successfully upgraded your connection to Destinations V2!
+
+Once you’ve completed the setup for Destinations V2, we ask that you pay special attention to the data delivered in your destination. Let us know immediately if you see any unexpected data: table and column name changes, missing columns, or columns with incorrect types.
diff --git a/docusaurus/README.md b/docusaurus/README.md
index 7c8b5cf0a0e9..f03e15d4c705 100644
--- a/docusaurus/README.md
+++ b/docusaurus/README.md
@@ -28,13 +28,13 @@ yarn start # any changes will automatically be reflected in your browser!
 
 ## Making Changes
 
-All the content for docs.airbute.com lives in the `/docs` directory in this repo. All files are markdown. Make changes or add new files, and you should see them in your browser!
+All the content for docs.airbyte.com lives in the `/docs` directory in this repo. All files are markdown. Make changes or add new files, and you should see them in your browser!
 
 If you have created any new files, be sure to add them manually to the table of contents found here in this [file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js)
 
 ## Plugin Client Redirects
 
-A silly name, but a useful plugin that adds redirect functionality to docusuaurs
+A silly name, but a useful plugin that adds redirect functionality to docusaurus
 [Official documentation here](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-client-redirects)
 
 You will need to edit [this docusaurus file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22)
diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js
index d1035e5f2a64..e44a6ff2b2fe 100644
--- a/docusaurus/docusaurus.config.js
+++ b/docusaurus/docusaurus.config.js
@@ -20,13 +20,16 @@ const config = {
   projectName: "airbyte", // Usually your repo name.
 
   plugins: [
-    [require.resolve("@cmfcmf/docusaurus-search-local"), { indexBlog: false }],
     [
       "@docusaurus/plugin-client-redirects",
       {
         fromExtensions: ["html", "htm"], // /myPage.html -> /myPage
         redirects: [
           // /docs/oldDoc -> /docs/newDoc
+          {
+            from: "/airbyte-pro",
+            to: "/airbyte-enterprise"
+          },
           {
             from: "/upgrading-airbyte",
             to: "/operator-guides/upgrading-airbyte",
@@ -139,6 +142,11 @@ const config = {
           autoCollapseCategories: true,
         },
       },
+      algolia: {
+          appId: 'OYKDBC51MU',
+          apiKey: '15c487fd9f7722282efd8fcb76746fce', // Public API key: it is safe to commit it
+          indexName: 'airbyte',
+      },
       navbar: {
         title: "",
         logo: {
@@ -165,7 +173,7 @@ const config = {
             position: "left",
           },
           {
-            href: "https://discuss.airbyte.io/",
+            href: "https://support.airbyte.com/",
             label: "Support",
             position: "left",
           },
diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js
index 761c41403eb3..beea11a01839 100644
--- a/docusaurus/sidebars.js
+++ b/docusaurus/sidebars.js
@@ -1,243 +1,261 @@
-const fs = require('fs');
-const path = require('path');
+const fs = require("fs");
+const path = require("path");
 
-const connectorsDocsRoot = '../docs/integrations';
+const connectorsDocsRoot = "../docs/integrations";
 const sourcesDocs = `${connectorsDocsRoot}/sources`;
 const destinationDocs = `${connectorsDocsRoot}/destinations`;
 
 function getFilenamesInDir(prefix, dir, excludes) {
   return fs
     .readdirSync(dir)
-    .filter((fileName) => !fileName.endsWith('.inapp.md'))
-    .map((fileName) => fileName.replace('.md', ''))
+    .filter(
+      (fileName) =>
+        !(fileName.endsWith(".inapp.md") || fileName.endsWith("-migrations.md") || fileName.endsWith("postgres.md"))
+    )
+    .map((fileName) => fileName.replace(".md", ""))
     .filter((fileName) => excludes.indexOf(fileName.toLowerCase()) === -1)
     .map((filename) => {
-      return { type: 'doc', id: path.join(prefix, filename) };
+      return { type: "doc", id: path.join(prefix, filename) };
     });
 }
 
 function getSourceConnectors() {
-  return getFilenamesInDir('integrations/sources/', sourcesDocs, ['readme']);
+  return getFilenamesInDir("integrations/sources/", sourcesDocs, ["readme"]);
 }
 
 function getDestinationConnectors() {
-  return getFilenamesInDir('integrations/destinations/', destinationDocs, [
-    'readme',
+  return getFilenamesInDir("integrations/destinations/", destinationDocs, [
+    "readme",
   ]);
 }
 
+const sourcePostgres = {
+   type: 'category',
+   label: 'Postgres',
+   link: {
+       type: 'doc',
+       id: 'integrations/sources/postgres',
+   },
+   items: [
+        {
+          type: "doc",
+          label: "Cloud SQL for Postgres",
+          id: "integrations/sources/postgres/cloud-sql-postgres",
+        },
+        {
+        type: "doc",
+        label: "Troubleshooting",
+        id: "integrations/sources/postgres/postgres-troubleshooting",
+        }
+   ],
+};
+
 const sectionHeader = (title) => ({
-  type: 'html',
+  type: "html",
   value: title,
-  className: 'navbar__category',
+  className: "navbar__category",
 });
 
 const buildAConnector = {
-  type: 'category',
-  label: 'Build a Connector',
+  type: "category",
+  label: "Build a Connector",
   items: [
     {
-      type: 'doc',
-      label: 'Overview',
-      id: 'connector-development/README',
+      type: "doc",
+      label: "Overview",
+      id: "connector-development/README",
     },
     {
-      type: 'category',
-      label: 'Connector Builder',
+      type: "category",
+      label: "Connector Builder",
       items: [
-        'connector-development/connector-builder-ui/overview',
-        'connector-development/connector-builder-ui/connector-builder-compatibility',
-        'connector-development/connector-builder-ui/tutorial',
+        "connector-development/connector-builder-ui/overview",
+        "connector-development/connector-builder-ui/connector-builder-compatibility",
+        "connector-development/connector-builder-ui/tutorial",
         {
-          type: 'category',
-          label: 'Concepts',
+          type: "category",
+          label: "Concepts",
           items: [
-            'connector-development/connector-builder-ui/authentication',
-            'connector-development/connector-builder-ui/record-processing',
-            'connector-development/connector-builder-ui/pagination',
-            'connector-development/connector-builder-ui/incremental-sync',
-            'connector-development/connector-builder-ui/partitioning',
-            'connector-development/connector-builder-ui/error-handling',
+            "connector-development/connector-builder-ui/authentication",
+            "connector-development/connector-builder-ui/record-processing",
+            "connector-development/connector-builder-ui/pagination",
+            "connector-development/connector-builder-ui/incremental-sync",
+            "connector-development/connector-builder-ui/partitioning",
+            "connector-development/connector-builder-ui/error-handling",
           ],
         },
       ],
     },
     {
-      type: 'category',
-      label: 'Low-code connector development',
+      type: "category",
+      label: "Low-code connector development",
       items: [
         {
-          label: 'Low-code CDK Intro',
-          type: 'doc',
-          id: 'connector-development/config-based/low-code-cdk-overview',
+          label: "Low-code CDK Intro",
+          type: "doc",
+          id: "connector-development/config-based/low-code-cdk-overview",
         },
         {
-          type: 'category',
-          label: 'Tutorial',
+          type: "category",
+          label: "Tutorial",
           items: [
-            'connector-development/config-based/tutorial/getting-started',
-            'connector-development/config-based/tutorial/create-source',
-            'connector-development/config-based/tutorial/install-dependencies',
-            'connector-development/config-based/tutorial/connecting-to-the-API-source',
-            'connector-development/config-based/tutorial/reading-data',
-            'connector-development/config-based/tutorial/incremental-reads',
-            'connector-development/config-based/tutorial/testing',
+            "connector-development/config-based/tutorial/getting-started",
+            "connector-development/config-based/tutorial/create-source",
+            "connector-development/config-based/tutorial/install-dependencies",
+            "connector-development/config-based/tutorial/connecting-to-the-API-source",
+            "connector-development/config-based/tutorial/reading-data",
+            "connector-development/config-based/tutorial/incremental-reads",
+            "connector-development/config-based/tutorial/testing",
           ],
         },
         {
-          type: 'category',
-          label: 'Understanding the YAML file',
+          type: "category",
+          label: "Understanding the YAML file",
           link: {
-            type: 'doc',
-            id: 'connector-development/config-based/understanding-the-yaml-file/yaml-overview',
+            type: "doc",
+            id: "connector-development/config-based/understanding-the-yaml-file/yaml-overview",
           },
           items: [
             {
               type: `category`,
               label: `Requester`,
               link: {
-                type: 'doc',
-                id: 'connector-development/config-based/understanding-the-yaml-file/requester',
+                type: "doc",
+                id: "connector-development/config-based/understanding-the-yaml-file/requester",
               },
               items: [
-                'connector-development/config-based/understanding-the-yaml-file/request-options',
-                'connector-development/config-based/understanding-the-yaml-file/authentication',
-                'connector-development/config-based/understanding-the-yaml-file/error-handling',
+                "connector-development/config-based/understanding-the-yaml-file/request-options",
+                "connector-development/config-based/understanding-the-yaml-file/authentication",
+                "connector-development/config-based/understanding-the-yaml-file/error-handling",
               ],
             },
-            'connector-development/config-based/understanding-the-yaml-file/incremental-syncs',
-            'connector-development/config-based/understanding-the-yaml-file/pagination',
-            'connector-development/config-based/understanding-the-yaml-file/partition-router',
-            'connector-development/config-based/understanding-the-yaml-file/record-selector',
-            'connector-development/config-based/understanding-the-yaml-file/reference',
+            "connector-development/config-based/understanding-the-yaml-file/incremental-syncs",
+            "connector-development/config-based/understanding-the-yaml-file/pagination",
+            "connector-development/config-based/understanding-the-yaml-file/partition-router",
+            "connector-development/config-based/understanding-the-yaml-file/record-selector",
+            "connector-development/config-based/understanding-the-yaml-file/reference",
           ],
         },
-        'connector-development/config-based/advanced-topics',
+        "connector-development/config-based/advanced-topics",
       ],
     },
 
     {
-      type: 'category',
-      label: 'Connector Development Kit',
+      type: "category",
+      label: "Connector Development Kit",
       link: {
-        type: 'doc',
-        id: 'connector-development/cdk-python/README',
+        type: "doc",
+        id: "connector-development/cdk-python/README",
       },
       items: [
-        'connector-development/cdk-python/basic-concepts',
-        'connector-development/cdk-python/schemas',
-        'connector-development/cdk-python/full-refresh-stream',
-        'connector-development/cdk-python/incremental-stream',
-        'connector-development/cdk-python/http-streams',
-        'connector-development/cdk-python/python-concepts',
-        'connector-development/cdk-python/stream-slices',
+        "connector-development/cdk-python/basic-concepts",
+        "connector-development/cdk-python/schemas",
+        "connector-development/cdk-python/full-refresh-stream",
+        "connector-development/cdk-python/incremental-stream",
+        "connector-development/cdk-python/http-streams",
+        "connector-development/cdk-python/python-concepts",
+        "connector-development/cdk-python/stream-slices",
       ],
     },
     {
-      type: 'category',
-      label: 'Testing Connectors',
+      type: "category",
+      label: "Testing Connectors",
       link: {
-        type: 'doc',
-        id: 'connector-development/testing-connectors/README',
+        type: "doc",
+        id: "connector-development/testing-connectors/README",
       },
       items: [
-        'connector-development/testing-connectors/connector-acceptance-tests-reference',
-        'connector-development/testing-connectors/testing-a-local-catalog-in-development',
+        "connector-development/testing-connectors/connector-acceptance-tests-reference",
+        "connector-development/testing-connectors/testing-a-local-catalog-in-development",
       ],
     },
     {
-      type: 'category',
-      label: 'Tutorials',
+      type: "category",
+      label: "Tutorials",
       items: [
-        'connector-development/tutorials/cdk-speedrun',
+        "connector-development/tutorials/cdk-speedrun",
         {
-          type: 'category',
-          label: 'Python CDK: Creating a HTTP API Source',
+          type: "category",
+          label: "Python CDK: Creating a HTTP API Source",
           items: [
-            'connector-development/tutorials/cdk-tutorial-python-http/getting-started',
-            'connector-development/tutorials/cdk-tutorial-python-http/creating-the-source',
-            'connector-development/tutorials/cdk-tutorial-python-http/install-dependencies',
-            'connector-development/tutorials/cdk-tutorial-python-http/define-inputs',
-            'connector-development/tutorials/cdk-tutorial-python-http/connection-checking',
-            'connector-development/tutorials/cdk-tutorial-python-http/declare-schema',
-            'connector-development/tutorials/cdk-tutorial-python-http/read-data',
-            'connector-development/tutorials/cdk-tutorial-python-http/use-connector-in-airbyte',
-            'connector-development/tutorials/cdk-tutorial-python-http/test-your-connector',
+            "connector-development/tutorials/cdk-tutorial-python-http/getting-started",
+            "connector-development/tutorials/cdk-tutorial-python-http/creating-the-source",
+            "connector-development/tutorials/cdk-tutorial-python-http/install-dependencies",
+            "connector-development/tutorials/cdk-tutorial-python-http/define-inputs",
+            "connector-development/tutorials/cdk-tutorial-python-http/connection-checking",
+            "connector-development/tutorials/cdk-tutorial-python-http/declare-schema",
+            "connector-development/tutorials/cdk-tutorial-python-http/read-data",
+            "connector-development/tutorials/cdk-tutorial-python-http/use-connector-in-airbyte",
+            "connector-development/tutorials/cdk-tutorial-python-http/test-your-connector",
           ],
         },
-        'connector-development/tutorials/building-a-python-source',
-        'connector-development/tutorials/building-a-python-destination',
-        'connector-development/tutorials/building-a-java-destination',
-        'connector-development/tutorials/profile-java-connector-memory',
+        "connector-development/tutorials/building-a-python-source",
+        "connector-development/tutorials/building-a-python-destination",
+        "connector-development/tutorials/building-a-java-destination",
+        "connector-development/tutorials/profile-java-connector-memory",
       ],
     },
-    'connector-development/connector-specification-reference',
-    'connector-development/schema-reference',
-    'connector-development/connector-metadata-file',
-    'connector-development/best-practices',
-    'connector-development/ux-handbook',
+    "connector-development/connector-specification-reference",
+    "connector-development/schema-reference",
+    "connector-development/connector-metadata-file",
+    "connector-development/best-practices",
+    "connector-development/ux-handbook",
   ],
 };
 
 const connectorCatalog = {
-  type: 'category',
-  label: 'Connector Catalog',
+  type: "category",
+  label: "Connector Catalog",
   link: {
-    type: 'doc',
-    id: 'integrations/README',
+    type: "doc",
+    id: "integrations/README",
   },
   items: [
     {
-      type: 'category',
-      label: 'Sources',
+      type: "category",
+      label: "Sources",
       link: {
-        type: 'generated-index',
+        type: "generated-index",
       },
-      items: getSourceConnectors(),
+      items: [sourcePostgres, getSourceConnectors()],
     },
     {
-      type: 'category',
-      label: 'Destinations',
+      type: "category",
+      label: "Destinations",
       link: {
-        type: 'generated-index',
+        type: "generated-index",
       },
       items: getDestinationConnectors(),
     },
     {
-      type: 'doc',
-      id: 'integrations/custom-connectors',
+      type: "doc",
+      id: "integrations/custom-connectors",
     },
   ],
 };
 
 const contributeToAirbyte = {
-  type: 'category',
-  label: 'Contribute to Airbyte',
+  type: "category",
+  label: "Contribute to Airbyte",
   link: {
-    type: 'doc',
-    id: 'contributing-to-airbyte/README',
+    type: "doc",
+    id: "contributing-to-airbyte/README",
   },
   items: [
-    'contributing-to-airbyte/code-of-conduct',
-    'contributing-to-airbyte/issues-and-pull-requests',
-    'contributing-to-airbyte/developing-locally',
-    'contributing-to-airbyte/developing-on-docker',
-    'contributing-to-airbyte/python-gradle-setup',
-    'contributing-to-airbyte/code-style',
-    'contributing-to-airbyte/gradle',
+    "contributing-to-airbyte/issues-and-requests",
+    "contributing-to-airbyte/change-cdk-connector",
+    "contributing-to-airbyte/submit-new-connector",
+    "contributing-to-airbyte/writing-docs",
     {
-      type: 'category',
-      label: 'Updating documentation',
-      link: {
-        type: 'doc',
-        id: 'contributing-to-airbyte/contribute-documentation',
-      },
+      type: "category",
+      label: "Resources",
       items: [
-        {
-          type: 'link',
-          label: 'Connector doc template',
-          href: 'https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw',
-        },
+        "contributing-to-airbyte/resources/pull-requests-handbook",
+        "contributing-to-airbyte/resources/code-style",
+        "contributing-to-airbyte/resources/developing-locally",
+        "contributing-to-airbyte/resources/developing-on-docker",
+        "contributing-to-airbyte/resources/gradle",
+        "contributing-to-airbyte/resources/python-gradle-setup",
       ],
     },
   ],
@@ -245,271 +263,274 @@ const contributeToAirbyte = {
 
 const airbyteCloud = [
   {
-    type: 'doc',
-    label: 'Getting Started',
-    id: 'cloud/getting-started-with-airbyte-cloud',
+    type: "doc",
+    label: "Getting Started",
+    id: "cloud/getting-started-with-airbyte-cloud",
   },
-  'cloud/core-concepts',
+  "cloud/core-concepts",
   {
-    type: 'category',
-    label: 'Using Airbyte Cloud',
+    type: "category",
+    label: "Using Airbyte Cloud",
     link: {
-      type: 'generated-index',
+      type: "generated-index",
     },
     items: [
-      'cloud/managing-airbyte-cloud/edit-stream-configuration',
-      'cloud/managing-airbyte-cloud/manage-schema-changes',
-      'cloud/managing-airbyte-cloud/manage-data-residency',
-      'cloud/managing-airbyte-cloud/manage-credits',
-      'cloud/managing-airbyte-cloud/review-sync-summary',
-      'cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications',
-      'cloud/managing-airbyte-cloud/dbt-cloud-integration',
-      'cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace',
-      'cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits',
-      'cloud/managing-airbyte-cloud/review-connection-state',
+      "cloud/managing-airbyte-cloud/edit-stream-configuration",
+      "cloud/managing-airbyte-cloud/manage-schema-changes",
+      "cloud/managing-airbyte-cloud/manage-data-residency",
+      "cloud/managing-airbyte-cloud/manage-credits",
+      "cloud/managing-airbyte-cloud/review-sync-summary",
+      "cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications",
+      "cloud/managing-airbyte-cloud/dbt-cloud-integration",
+      "cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace",
+      "cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits",
+      "cloud/managing-airbyte-cloud/review-connection-state",
     ],
   },
 ];
 
 const ossGettingStarted = {
-  type: 'category',
-  label: 'Getting Started',
+  type: "category",
+  label: "Getting Started",
   link: {
-    type: 'generated-index',
+    type: "generated-index",
   },
   items: [
-    'quickstart/deploy-airbyte',
-    'quickstart/add-a-source',
-    'quickstart/add-a-destination',
-    'quickstart/set-up-a-connection',
+    "quickstart/deploy-airbyte",
+    "quickstart/add-a-source",
+    "quickstart/add-a-destination",
+    "quickstart/set-up-a-connection",
   ],
 };
 
 const deployAirbyte = {
-  type: 'category',
-  label: 'Deploy Airbyte',
+  type: "category",
+  label: "Deploy Airbyte",
   link: {
-    type: 'generated-index',
+    type: "generated-index",
   },
   items: [
     {
-      type: 'doc',
-      label: 'On your local machine',
-      id: 'deploying-airbyte/local-deployment',
+      type: "doc",
+      label: "On your local machine",
+      id: "deploying-airbyte/local-deployment",
     },
     {
-      type: 'doc',
-      label: 'On AWS EC2',
-      id: 'deploying-airbyte/on-aws-ec2',
+      type: "doc",
+      label: "On AWS EC2",
+      id: "deploying-airbyte/on-aws-ec2",
     },
 
     {
-      type: 'doc',
-      label: 'On Azure',
-      id: 'deploying-airbyte/on-azure-vm-cloud-shell',
-    },
-    {
-      type: 'doc',
-      label: 'On Google (GCP)',
-      id: 'deploying-airbyte/on-gcp-compute-engine',
+      type: "doc",
+      label: "On Azure",
+      id: "deploying-airbyte/on-azure-vm-cloud-shell",
     },
     {
-      type: 'doc',
-      label: 'On Kubernetes using Kustomize',
-      id: 'deploying-airbyte/on-kubernetes',
+      type: "doc",
+      label: "On Google (GCP)",
+      id: "deploying-airbyte/on-gcp-compute-engine",
     },
     {
-      type: 'doc',
-      label: 'On Kubernetes using Helm',
-      id: 'deploying-airbyte/on-kubernetes-via-helm',
+      type: "doc",
+      label: "On Kubernetes using Helm",
+      id: "deploying-airbyte/on-kubernetes-via-helm",
     },
     {
-      type: 'doc',
-      label: 'On Restack',
-      id: 'deploying-airbyte/on-restack',
+      type: "doc",
+      label: "On Restack",
+      id: "deploying-airbyte/on-restack",
     },
     {
-      type: 'doc',
-      label: 'On Plural',
-      id: 'deploying-airbyte/on-plural',
+      type: "doc",
+      label: "On Plural",
+      id: "deploying-airbyte/on-plural",
     },
     {
-      type: 'doc',
-      label: 'On Oracle Cloud',
-      id: 'deploying-airbyte/on-oci-vm',
+      type: "doc",
+      label: "On Oracle Cloud",
+      id: "deploying-airbyte/on-oci-vm",
     },
     {
-      type: 'doc',
-      label: 'On DigitalOcean',
-      id: 'deploying-airbyte/on-digitalocean-droplet',
+      type: "doc",
+      label: "On DigitalOcean",
+      id: "deploying-airbyte/on-digitalocean-droplet",
     },
   ],
 };
 
 const operatorGuide = {
-  type: 'category',
-  label: 'Manage Airbyte',
+  type: "category",
+  label: "Manage Airbyte",
   link: {
-    type: 'generated-index',
+    type: "generated-index",
   },
   items: [
-    'operator-guides/upgrading-airbyte',
-    'operator-guides/reset',
-    'operator-guides/configuring-airbyte-db',
-    'operator-guides/configuring-connector-resources',
-    'operator-guides/browsing-output-logs',
-    'operator-guides/using-the-airflow-airbyte-operator',
-    'operator-guides/using-prefect-task',
-    'operator-guides/using-dagster-integration',
-    'operator-guides/locating-files-local-destination',
+    "operator-guides/upgrading-airbyte",
+    "operator-guides/reset",
+    "operator-guides/configuring-airbyte-db",
+    "operator-guides/configuring-connector-resources",
+    "operator-guides/browsing-output-logs",
+    "operator-guides/using-the-airflow-airbyte-operator",
+    "operator-guides/using-prefect-task",
+    "operator-guides/using-dagster-integration",
+    "operator-guides/using-kestra-plugin",
+    "operator-guides/locating-files-local-destination",
+    "operator-guides/collecting-metrics",
     {
-      type: 'category',
-      label: 'Transformations and Normalization',
+      type: "category",
+      label: "Transformations and Normalization",
       items: [
-        'operator-guides/transformation-and-normalization/transformations-with-sql',
-        'operator-guides/transformation-and-normalization/transformations-with-dbt',
-        'operator-guides/transformation-and-normalization/transformations-with-airbyte',
+        "operator-guides/transformation-and-normalization/transformations-with-sql",
+        "operator-guides/transformation-and-normalization/transformations-with-dbt",
+        "operator-guides/transformation-and-normalization/transformations-with-airbyte",
       ],
     },
-    {
-      type: 'category',
-      label: 'Configuring Airbyte',
-      link: {
-        type: 'doc',
-        id: 'operator-guides/configuring-airbyte',
-      },
-      items: ['operator-guides/sentry-integration'],
-    },
-    'operator-guides/using-custom-connectors',
-    'operator-guides/scaling-airbyte',
-    'operator-guides/configuring-sync-notifications',
-    'operator-guides/collecting-metrics',
+    "operator-guides/configuring-airbyte",
+    "operator-guides/using-custom-connectors",
+    "operator-guides/scaling-airbyte",
+    "operator-guides/configuring-sync-notifications",
   ],
 };
 
 const understandingAirbyte = {
-  type: 'category',
-  label: 'Understand Airbyte',
+  type: "category",
+  label: "Understand Airbyte",
   items: [
-    'understanding-airbyte/beginners-guide-to-catalog',
-    'understanding-airbyte/airbyte-protocol',
-    'understanding-airbyte/airbyte-protocol-docker',
-    'understanding-airbyte/basic-normalization',
+    "understanding-airbyte/beginners-guide-to-catalog",
+    "understanding-airbyte/airbyte-protocol",
+    "understanding-airbyte/airbyte-protocol-docker",
+    "understanding-airbyte/basic-normalization",
+    "understanding-airbyte/typing-deduping",
     {
-      type: 'category',
-      label: 'Connections and Sync Modes',
+      type: "category",
+      label: "Connections and Sync Modes",
       items: [
         {
-          type: 'doc',
-          label: 'Connections Overview',
-          id: 'understanding-airbyte/connections/README',
+          type: "doc",
+          label: "Connections Overview",
+          id: "understanding-airbyte/connections/README",
         },
-        'understanding-airbyte/connections/full-refresh-overwrite',
-        'understanding-airbyte/connections/full-refresh-append',
-        'understanding-airbyte/connections/incremental-append',
-        'understanding-airbyte/connections/incremental-deduped-history',
+        "understanding-airbyte/connections/full-refresh-overwrite",
+        "understanding-airbyte/connections/full-refresh-append",
+        "understanding-airbyte/connections/incremental-append",
+        "understanding-airbyte/connections/incremental-append-deduped",
       ],
     },
-    'understanding-airbyte/operations',
-    'understanding-airbyte/high-level-view',
-    'understanding-airbyte/jobs',
-    'understanding-airbyte/tech-stack',
-    'understanding-airbyte/cdc',
-    'understanding-airbyte/namespaces',
-    'understanding-airbyte/supported-data-types',
-    'understanding-airbyte/json-avro-conversion',
-    'understanding-airbyte/database-data-catalog',
+    "understanding-airbyte/operations",
+    "understanding-airbyte/high-level-view",
+    "understanding-airbyte/jobs",
+    "understanding-airbyte/tech-stack",
+    "understanding-airbyte/cdc",
+    "understanding-airbyte/namespaces",
+    "understanding-airbyte/supported-data-types",
+    "understanding-airbyte/json-avro-conversion",
+    "understanding-airbyte/database-data-catalog",
   ],
 };
 
 const security = {
-  type: 'doc',
-  id: 'operator-guides/security',
+  type: "doc",
+  id: "operator-guides/security",
+};
+
+const support = {
+  type: "doc",
+  id: "operator-guides/contact-support",
 };
 
 module.exports = {
   mySidebar: [
     {
-      type: 'doc',
-      label: 'Start here',
-      id: 'readme',
+      type: "doc",
+      label: "Start here",
+      id: "readme",
     },
-    sectionHeader('Airbyte Connectors'),
+    sectionHeader("Airbyte Connectors"),
     connectorCatalog,
     buildAConnector,
-    sectionHeader('Airbyte Cloud'),
+    sectionHeader("Airbyte Cloud"),
     ...airbyteCloud,
-    sectionHeader('Airbyte Open Source (OSS)'),
+    sectionHeader("Airbyte Open Source (OSS)"),
     ossGettingStarted,
     deployAirbyte,
     operatorGuide,
     {
-      type: 'doc',
-      id: 'troubleshooting',
+      type: "doc",
+      id: "troubleshooting",
     },
-    sectionHeader('Developer Guides'),
     {
       type: 'doc',
-      id: 'api-documentation',
+      id: 'airbyte-enterprise',
     },
+    sectionHeader("Developer Guides"),
     {
-      type: 'doc',
-      id: 'cli-documentation',
+      type: "doc",
+      id: "api-documentation",
+    },
+    {
+      type: "doc",
+      id: "cli-documentation",
     },
     understandingAirbyte,
     contributeToAirbyte,
-    sectionHeader('Resources'),
+    sectionHeader("Resources"),
+    support,
     security,
     {
-      type: 'category',
-      label: 'Project Overview',
+      type: "category",
+      label: "Project Overview",
       items: [
         {
-          type: 'link',
-          label: 'Roadmap',
-          href: 'https://go.airbyte.com/roadmap',
+          type: "link",
+          label: "Roadmap",
+          href: "https://go.airbyte.com/roadmap",
         },
-        'project-overview/product-release-stages',
-        'project-overview/slack-code-of-conduct',
+        "project-overview/product-release-stages",
+        "project-overview/slack-code-of-conduct",
+        "project-overview/code-of-conduct",
         {
-          type: 'link',
-          label: 'Airbyte Repository',
-          href: 'https://github.com/airbytehq/airbyte',
+          type: "link",
+          label: "Airbyte Repository",
+          href: "https://github.com/airbytehq/airbyte",
         },
         {
-          type: 'category',
-          label: 'Licenses',
+          type: "category",
+          label: "Licenses",
           link: {
-            type: 'doc',
-            id: 'project-overview/licenses/README',
+            type: "doc",
+            id: "project-overview/licenses/README",
           },
           items: [
-            'project-overview/licenses/license-faq',
-            'project-overview/licenses/elv2-license',
-            'project-overview/licenses/mit-license',
-            'project-overview/licenses/examples',
+            "project-overview/licenses/license-faq",
+            "project-overview/licenses/elv2-license",
+            "project-overview/licenses/mit-license",
+            "project-overview/licenses/examples",
           ],
         },
       ],
     },
     {
-      type: 'category',
-      label: 'Release Notes',
+      type: "category",
+      label: "Release Notes",
       link: {
-        type: 'generated-index',
+        type: "generated-index",
       },
       items: [
-        'release_notes/may_2023',
-        'release_notes/april_2023',
-        'release_notes/march_2023',
-        'release_notes/february_2023',
-        'release_notes/january_2023',
-        'release_notes/december_2022',
-        'release_notes/november_2022',
-        'release_notes/october_2022',
-        'release_notes/september_2022',
-        'release_notes/august_2022',
-        'release_notes/july_2022',
+        "release_notes/upgrading_to_destinations_v2",
+        "release_notes/july_2023",
+        "release_notes/june_2023",
+        "release_notes/may_2023",
+        "release_notes/april_2023",
+        "release_notes/march_2023",
+        "release_notes/february_2023",
+        "release_notes/january_2023",
+        "release_notes/december_2022",
+        "release_notes/november_2022",
+        "release_notes/october_2022",
+        "release_notes/september_2022",
+        "release_notes/august_2022",
+        "release_notes/july_2022",
       ],
     },
   ],
diff --git a/docusaurus/src/css/custom.css b/docusaurus/src/css/custom.css
index 34f139bafdca..8e48effb6770 100644
--- a/docusaurus/src/css/custom.css
+++ b/docusaurus/src/css/custom.css
@@ -23,6 +23,8 @@
   --ifm-alert-shadow: none;
   --ifm-menu-color: var(--ifm-color-emphasis-900);
   --ifm-hover-overlay: rgb(0 0 0 / 2%);
+  --color-active-nav-item-background: #f4f4ff;
+  --color-active-nav-item-text: var(--ifm-color-primary-darker);
 }
 
 /* For readability concerns, you should choose a lighter palette in dark mode. */
@@ -35,6 +37,8 @@ html[data-theme="dark"] {
   --ifm-color-primary-light: #8381ff;
   --ifm-color-primary-lighter: #9492ff;
   --ifm-color-primary-lightest: #c8c7ff;
+  --color-active-nav-item-background: #272729;
+  --color-active-nav-item-text: var(--ifm-color-primary-lighter);
 }
 
 .docusaurus-highlight-code-line {
@@ -75,10 +79,12 @@ html[data-theme="dark"] .docusaurus-highlight-code-line {
 }
 
 .navbar__category {
-  font-weight: 200;
+  font-weight: 700;
   font-size: 0.8em;
-  opacity: 0.8;
-  padding: 1em var(--ifm-menu-link-padding-horizontal) 0;
+  padding: .4em 0 .4em .4em;
+  margin-top: 1.1em;
+  color: var(--docsearch-text-color);
+  background-color: var(--ifm-hover-overlay);
 }
 
 .codeBlockContainer_I0IT {
@@ -99,3 +105,15 @@ img {
   background: var(--ifm-menu-link-sublist-icon) 50%/2rem 2rem;
   min-width: 1.25rem;
 }
+
+ /* These properties set the color of the active item in the navbar (background and text).
+ The variables for them have been added to :root at the top of this file */
+ 
+ .menu__link--active,
+ .menu__link--active:not(.menu__link--sublist),
+ .menu__list-item-collapsible--active,
+ .menu__list-item-collapsible--active:hover {
+   background: var(--color-active-nav-item-background);
+   color: var(--color-active-nav-item-text);
+ }
+ 
\ No newline at end of file
diff --git a/gradle.properties b/gradle.properties
index 9bf03c37ce8a..f491bf7dc2e3 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,4 +1,4 @@
-VERSION=0.44.12
+VERSION=0.50.21
 
 # NOTE: some of these values are overwritten in CI!
 # NOTE: if you want to override this for your local machine, set overrides in ~/.gradle/gradle.properties
diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile
index cead95c55fd1..16eb123219d9 100644
--- a/octavia-cli/Dockerfile
+++ b/octavia-cli/Dockerfile
@@ -14,5 +14,5 @@ USER octavia-cli
 WORKDIR /home/octavia-project
 ENTRYPOINT ["octavia"]
 
-LABEL io.airbyte.version=0.44.12
+LABEL io.airbyte.version=0.50.0
 LABEL io.airbyte.name=airbyte/octavia-cli
diff --git a/octavia-cli/README.md b/octavia-cli/README.md
index a74c770ed6c7..2529e1dbf94c 100644
--- a/octavia-cli/README.md
+++ b/octavia-cli/README.md
@@ -27,7 +27,7 @@ These are non-exhaustive use cases `octavia` can be convenient for:
 - Integrating the Airbyte configuration deployment in a dev ops tooling stack: Helm, Ansible etc.
 - Streamlining the deployment of Airbyte configurations to multiple Airbyte instance.
 
-Feel free to share your use cases with the community in [#octavia-cli](https://airbytehq.slack.com/archives/C02RRUG9CP5) or on [Discourse](https://discuss.airbyte.io/).
+Feel free to share your use cases with the community in [#octavia-cli](https://airbytehq.slack.com/archives/C02RRUG9CP5) or on [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions).
 
 ## Table of content
 
@@ -104,7 +104,7 @@ This script:
 ```bash
 touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container
 mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored.
-docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.44.12
+docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.50.0
 ```
 
 ### Using `docker-compose`
diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh
index 472ad316762b..bb87917e46b8 100755
--- a/octavia-cli/install.sh
+++ b/octavia-cli/install.sh
@@ -3,7 +3,7 @@
 # This install scripts currently only works for ZSH and Bash profiles.
 # It creates an octavia alias in your profile bound to a docker run command and your current user.
 
-VERSION=0.44.12
+VERSION=0.44.4
 OCTAVIA_ENV_FILE=${HOME}/.octavia
 
 detect_profile() {
@@ -41,7 +41,7 @@ delete_previous_alias() {
 
 pull_image() {
     echo "🐙 - Pulling image for octavia ${VERSION}"
-    docker pull airbyte/octavia-cli:${VERSION} > /dev/null 2>&1
+    docker pull airbyte/octavia-cli:${VERSION} > /dev/null
     echo "🐙 - 🎉 octavia ${VERSION} image was pulled"
 }
 
diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py
index 13b9b5776d00..0356abd4732d 100644
--- a/octavia-cli/octavia_cli/apply/resources.py
+++ b/octavia-cli/octavia_cli/apply/resources.py
@@ -40,13 +40,16 @@
 from airbyte_api_client.model.destination_read import DestinationRead
 from airbyte_api_client.model.destination_sync_mode import DestinationSyncMode
 from airbyte_api_client.model.destination_update import DestinationUpdate
+from airbyte_api_client.model.geography import Geography
 from airbyte_api_client.model.namespace_definition_type import NamespaceDefinitionType
+from airbyte_api_client.model.non_breaking_changes_preference import NonBreakingChangesPreference
 from airbyte_api_client.model.operation_create import OperationCreate
 from airbyte_api_client.model.operator_configuration import OperatorConfiguration
 from airbyte_api_client.model.operator_dbt import OperatorDbt
 from airbyte_api_client.model.operator_normalization import OperatorNormalization
 from airbyte_api_client.model.operator_type import OperatorType
 from airbyte_api_client.model.resource_requirements import ResourceRequirements
+from airbyte_api_client.model.selected_field_info import SelectedFieldInfo
 from airbyte_api_client.model.source_create import SourceCreate
 from airbyte_api_client.model.source_definition_id_request_body import SourceDefinitionIdRequestBody
 from airbyte_api_client.model.source_definition_id_with_workspace_id import SourceDefinitionIdWithWorkspaceId
@@ -630,6 +633,16 @@ def _deserialize_raw_configuration(self):
         self._check_for_legacy_connection_configuration_keys(configuration)
         configuration["sync_catalog"] = self._create_configured_catalog(configuration["sync_catalog"])
         configuration["namespace_definition"] = NamespaceDefinitionType(configuration["namespace_definition"])
+        if "non_breaking_changes_preference" in configuration:
+            configuration["non_breaking_changes_preference"] = NonBreakingChangesPreference(
+                configuration["non_breaking_changes_preference"]
+            )
+        else:
+            configuration["non_breaking_changes_preference"] = NonBreakingChangesPreference("ignore")
+        if "geography" in configuration:
+            configuration["geography"] = Geography(configuration["geography"])
+        else:
+            configuration["geography"] = Geography("auto")
 
         if "schedule_type" in configuration:
             # If schedule type is manual we do not expect a schedule_data field to be set
@@ -742,7 +755,10 @@ def _create_configured_catalog(sync_catalog: dict) -> AirbyteCatalog:
             stream["stream"]["supported_sync_modes"] = [SyncMode(sm) for sm in stream["stream"]["supported_sync_modes"]]
             stream["config"]["sync_mode"] = SyncMode(stream["config"]["sync_mode"])
             stream["config"]["destination_sync_mode"] = DestinationSyncMode(stream["config"]["destination_sync_mode"])
-
+            if "selected_fields" in stream["config"]:
+                stream["config"]["selected_fields"] = [
+                    SelectedFieldInfo(field_path=selected_field["field_path"]) for selected_field in stream["config"]["selected_fields"]
+                ]
             streams_and_configurations.append(
                 AirbyteStreamAndConfiguration(
                     stream=AirbyteStream(**stream["stream"]), config=AirbyteStreamConfiguration(**stream["config"])
diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py
index f1852172f624..628defaeac7c 100644
--- a/octavia-cli/setup.py
+++ b/octavia-cli/setup.py
@@ -15,7 +15,7 @@
 
 setup(
     name="octavia-cli",
-    version="0.44.12",
+    version="0.50.0",
     description="A command line interface to manage Airbyte configurations",
     long_description=README,
     author="Airbyte",
diff --git a/octavia-cli/unit_tests/test_apply/test_resources.py b/octavia-cli/unit_tests/test_apply/test_resources.py
index 2b34ee7e41f1..2aa59a7d0f61 100644
--- a/octavia-cli/unit_tests/test_apply/test_resources.py
+++ b/octavia-cli/unit_tests/test_apply/test_resources.py
@@ -847,6 +847,8 @@ def test__deserialize_raw_configuration(self, mock_api_client, connection_config
             "schedule_data",
             "status",
             "resource_requirements",
+            "non_breaking_changes_preference",
+            "geography",
         ]
 
         resource = resources.Connection(mock_api_client, "workspace_id", connection_configuration_with_manual_schedule, "bar.yaml")
diff --git a/pyproject.toml b/pyproject.toml
index 93bc56a58624..80a70a5bd99e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -82,6 +82,6 @@ error_summary = true
 
 [tool.pytest.ini_options]
 minversion = "6.2.5"
-addopts ="-r a --capture=no -vv --log-level=DEBUG --color=yes"
+addopts ="-r a --capture=no -vv --color=yes"
 
 
diff --git a/resources/examples/airflow/dags/dag_airbyte_example.py b/resources/examples/airflow/dags/dag_airbyte_example.py
index 9af59791a5f5..9bd2f8e23329 100644
--- a/resources/examples/airflow/dags/dag_airbyte_example.py
+++ b/resources/examples/airflow/dags/dag_airbyte_example.py
@@ -1,21 +1,23 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 from airflow import DAG
-from airflow.utils.dates import days_ago
-from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator
 from airflow.models import Variable
+from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator
+from airflow.utils.dates import days_ago
 
 airbyte_connection_id = Variable.get("AIRBYTE_CONNECTION_ID")
 
-with DAG(dag_id='trigger_airbyte_job_example',
-         default_args={'owner': 'airflow'},
-         schedule_interval='@daily',
-         start_date=days_ago(1)
-         ) as dag:
+with DAG(
+    dag_id="trigger_airbyte_job_example", default_args={"owner": "airflow"}, schedule_interval="@daily", start_date=days_ago(1)
+) as dag:
 
     example_sync = AirbyteTriggerSyncOperator(
-        task_id='airbyte_example',
-        airbyte_conn_id='airbyte_example',
+        task_id="airbyte_example",
+        airbyte_conn_id="airbyte_example",
         connection_id=airbyte_connection_id,
         asynchronous=False,
         timeout=3600,
-        wait_seconds=3
-    )
\ No newline at end of file
+        wait_seconds=3,
+    )
diff --git a/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py b/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py
index a3ca8b1ec6c9..623bcdd5bcfc 100644
--- a/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py
+++ b/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py
@@ -38,9 +38,7 @@ def get_env_variable(var_name, default=None):
         if default is not None:
             return default
         else:
-            error_msg = "The environment variable {} was missing, abort...".format(
-                var_name
-            )
+            error_msg = "The environment variable {} was missing, abort...".format(var_name)
             raise EnvironmentError(error_msg)
 
 
@@ -106,8 +104,6 @@ class CeleryConfig(object):
     import superset_config_docker
     from superset_config_docker import *  # noqa
 
-    logger.info(
-        f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"
-    )
+    logger.info(f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]")
 except ImportError:
     logger.info("Using default Docker config...")
diff --git a/run-ab-platform.sh b/run-ab-platform.sh
index fb21242276cf..d50aa7242bbd 100755
--- a/run-ab-platform.sh
+++ b/run-ab-platform.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION=0.44.12
+VERSION=0.50.21
 # Run away from anything even a little scary
 set -o nounset # -u exit if a variable is not set
 set -o errexit # -f exit for any command failure"
@@ -40,8 +40,9 @@ docker_compose_debug_yaml="docker-compose.debug.yaml"
                   dot_env=".env"
               dot_env_dev=".env.dev"
                      flags="flags.yml"
+             temporal_yaml="temporal/dynamicconfig/development.yaml"
 # any string is an array to POSIX shell. Space seperates values
-all_files="$docker_compose_yaml $docker_compose_debug_yaml $dot_env $dot_env_dev $flags"
+all_files="$docker_compose_yaml $docker_compose_debug_yaml $dot_env $dot_env_dev $flags $temporal_yaml"
 
 base_github_url="https://raw.githubusercontent.com/airbytehq/airbyte-platform/v$VERSION/"
 
@@ -52,6 +53,10 @@ Download()
 {
   ########## Check if we already have the assets we are looking for ##########
   for file in $all_files; do
+    # Account for the case where the file is in a subdirectory.
+    # Make sure the directory exists to keep curl happy.
+    dir_path=$(dirname "${file}")
+    mkdir -p "${dir_path}"
     if test -f $file; then
       # Check if the assets are old.  A possibly sharp corner
       if test $(find $file -type f -mtime +60 > /dev/null); then
diff --git a/settings.gradle b/settings.gradle
index 54bee84c8a99..370b82463b14 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,3 +1,6 @@
+import com.gradle.scan.plugin.PublishedBuildScan
+
+
 pluginManagement {
     repositories {
         gradlePluginPortal()
@@ -28,9 +31,14 @@ gradleEnterprise {
     buildScan {
         termsOfServiceUrl = "https://gradle.com/terms-of-service"
         termsOfServiceAgree = "yes"
+        buildScanPublished { PublishedBuildScan scan ->
+            file("scan-journal.log") << "${new Date()} - ${scan.buildScanId} - ${scan.buildScanUri}\n"
+        }
     }
 }
 
+
+
 ext.isCiServer = System.getenv().containsKey("CI")
 
 buildCache {
@@ -100,10 +108,13 @@ if (!System.getenv().containsKey("SUB_BUILD") || (System.getenv().containsKey("S
 // connectors base
 if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "CONNECTORS_BASE" || System.getenv().get("SUB_BUILD") == "ALL_CONNECTORS") {
     include ':airbyte-cdk:python'
+    include ':airbyte-cdk:java:airbyte-cdk'
     include ':airbyte-integrations:bases:base'
     include ':airbyte-integrations:bases:base-java'
     include ':airbyte-integrations:bases:base-java-s3'
     include ':airbyte-integrations:bases:base-normalization'
+    include ':airbyte-integrations:bases:base-typing-deduping'
+    include ':airbyte-integrations:bases:base-typing-deduping-test'
     include ':airbyte-integrations:bases:bases-destination-jdbc' // needs to be lexicographically after base-java and base-normalization to avoid race condition
     include ':airbyte-integrations:bases:base-standard-source-test-file'
     include ':airbyte-integrations:bases:connector-acceptance-test'
diff --git a/tools/bin/build_report.py b/tools/bin/build_report.py
deleted file mode 100644
index 821efc2d39cb..000000000000
--- a/tools/bin/build_report.py
+++ /dev/null
@@ -1,270 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""
-Report Connector Build Status to Slack
-
-All invocations of this script must be run from the Airbyte repository root.
-
-BEFORE RUNNING THIS SCRIPT:
-- Ensure you have read the documentation on how this system works: https://internal-docs.airbyte.io/Generated-Reports/Build-Status-Reports
-
-To Run tests:
-pytest ./tools/bin/build_report.py
-
-To run the script:
-pip install slack-sdk pyyaml
-python ./tools/bin/build_report.py
-"""
-
-import os
-import pathlib
-import re
-import sys
-from typing import Dict, List, Optional
-
-import requests
-from slack_sdk import WebhookClient
-from slack_sdk.errors import SlackApiError
-
-
-# Global statics
-CONNECTOR_REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
-CONNECTORS_ROOT_PATH = "./airbyte-integrations/connectors"
-RELEVANT_BASE_MODULES = ["base-normalization", "connector-acceptance-test"]
-CONNECTOR_BUILD_OUTPUT_URL = "https://dnsgjos7lj2fu.cloudfront.net/tests/summary/connectors"
-
-# Global vars
-TESTED_SOURCE = []
-TESTED_DESTINATION = []
-SUCCESS_SOURCE = []
-SUCCESS_DESTINATION = []
-NO_TESTS = []
-FAILED_LAST = []
-FAILED_2_LAST = []
-
-
-def download_and_parse_registry_json():
-    response = requests.get(CONNECTOR_REGISTRY_URL)
-    response.raise_for_status()
-    return response.json()
-
-
-def get_status_page(connector) -> str:
-    response = requests.get(f"{CONNECTOR_BUILD_OUTPUT_URL}/{connector}/index.html")
-    if response.status_code == 200:
-        return response.text
-
-
-def parse(page) -> list:
-    history = []
-    for row in re.findall(r"(.*?)", page):
-        cols = re.findall(r"(.*?)", row)
-        if not cols or len(cols) != 3:
-            continue
-        history.append(
-            {
-                "date": cols[0],
-                "status": re.findall(r" (\S+)", cols[1])[0],
-                "link": re.findall(r'href="(.*?)"', cols[2])[0],
-            }
-        )
-    return history
-
-
-def check_module(connector):
-    status_page = get_status_page(connector)
-
-    # check if connector is tested
-    if not status_page:
-        NO_TESTS.append(connector)
-        print("F", end="", flush=True)
-        return
-
-    print(".", end="", flush=True)
-
-    if connector.startswith("source"):
-        TESTED_SOURCE.append(connector)
-    elif connector.startswith("destination"):
-        TESTED_DESTINATION.append(connector)
-
-    # order: recent values goes first
-    history = parse(status_page)
-    # order: recent values goes last
-    short_status = "".join(["✅" if build["status"] == "success" else "❌" for build in history[::-1]])  # ex: ❌✅✅❌✅✅❌❌
-
-    # check latest build status
-    last_build = history[0]
-    if last_build["status"] == "success":
-        if connector.startswith("source"):
-            SUCCESS_SOURCE.append(connector)
-        elif connector.startswith("destination"):
-            SUCCESS_DESTINATION.append(connector)
-    else:
-        failed_today = [connector, short_status, last_build["link"], last_build["date"]]
-
-        if len(history) > 1 and history[1]["status"] != "success":
-            FAILED_2_LAST.append(failed_today)
-            return
-
-        FAILED_LAST.append(failed_today)
-
-
-def failed_report(failed_report) -> str:
-    max_name_len = max([len(connector[0]) for connector in failed_report])
-    max_status_len = max(len(connector[1]) for connector in failed_report)
-    for connector in failed_report:
-        connector[0] = connector[0].ljust(max_name_len, " ")
-        connector[1] = connector[1].rjust(max_status_len, " ")
-    return "\n".join([" ".join(connector) for connector in failed_report])
-
-
-def create_report(connectors, statuses: List[str]) -> str:
-    sources_len = len([name for name in connectors if name.startswith("source")])
-    destinations_len = len([name for name in connectors if name.startswith("destination")])
-
-    report = f"""
-CONNECTORS:   total: {len(connectors)} {" & ".join(statuses)} connectors
-Sources:      total: {sources_len} / tested: {len(TESTED_SOURCE)} / success: {len(SUCCESS_SOURCE)} ({round(len(SUCCESS_SOURCE) / sources_len * 100, 1)}%)
-Destinations: total: {destinations_len} / tested: {len(TESTED_DESTINATION)} / success: {len(SUCCESS_DESTINATION)} ({round(len(SUCCESS_DESTINATION) / destinations_len * 100, 1)}%)
-
-"""
-    if FAILED_LAST:
-        report += f"FAILED LAST BUILD ONLY - {len(FAILED_LAST)} connectors:\n" + failed_report(FAILED_LAST) + "\n\n"
-
-    if FAILED_2_LAST:
-        report += f"FAILED TWO LAST BUILDS - {len(FAILED_2_LAST)} connectors:\n" + failed_report(FAILED_2_LAST) + "\n\n"
-
-    if NO_TESTS:
-        report += f"NO TESTS - {len(NO_TESTS)} connectors:\n" + "\n".join(NO_TESTS) + "\n"
-
-    return report
-
-
-def send_report(report):
-    webhook = WebhookClient(os.environ["SLACK_BUILD_REPORT"])
-    try:
-
-        def chunk_messages(report):
-            """split report into messages with no more than 4000 chars each (slack limitation)"""
-            msg = ""
-            for line in report.splitlines():
-                msg += line + "\n"
-                if len(msg) > 3500:
-                    yield msg
-                    msg = ""
-            yield msg
-
-        for msg in chunk_messages(report):
-            webhook.send(text=f"```{msg}```")
-        print("Report has been sent")
-    except SlackApiError as e:
-        print("Unable to send report")
-        assert e.response["error"]
-
-
-def parse_dockerfile_repository_label(dockerfile_contents: str) -> Optional[str]:
-    parsed_label = re.findall(r"LABEL io.airbyte.name=(.*)[\s\n]*", dockerfile_contents)
-    if len(parsed_label) == 1:
-        return parsed_label[0]
-    elif len(parsed_label) == 0:
-        return None
-    else:
-        raise Exception(f"found more than one label in dockerfile: {dockerfile_contents}")
-
-
-def get_docker_label_to_connector_directory(base_directory: str, connector_module_names: List[str]) -> Dict[str, str]:
-    result = {}
-    for connector in connector_module_names:
-        # parse the dockerfile label if the dockerfile exists
-        dockerfile_path = pathlib.Path(base_directory, connector, "Dockerfile")
-        if os.path.isfile(dockerfile_path):
-            print(f"Reading {dockerfile_path}")
-            with open(dockerfile_path, "r") as file:
-                dockerfile_contents = file.read()
-                label = parse_dockerfile_repository_label(dockerfile_contents)
-                if label:
-                    result[label] = connector
-                else:
-                    print(f"Couldn't find a connector label in {dockerfile_path}")
-        else:
-            print(f"Couldn't find a dockerfile at {dockerfile_path}")
-    return result
-
-
-def get_connectors_with_release_stage(definitions_yaml: List, stages: List[str]) -> List[str]:
-    """returns e.g: ['airbyte/source-salesforce', ...] when given 'generally_available' as input"""
-    return [definition["dockerRepository"] for definition in definitions_yaml if definition.get("releaseStage", "alpha") in stages]
-
-
-def get_connectors_with_release_stages(base_directory: str, connectors: List[str], relevant_stages=["beta", "generally_available"]):
-    # TODO currently this also excludes shared libs like source-jdbc, we probably shouldn't do that, so we can get the build status of those
-    #  modules as well.
-    connector_label_to_connector_directory = get_docker_label_to_connector_directory(base_directory, connectors)
-
-    registry_data = download_and_parse_registry_json()
-
-    connectors_with_desired_status = get_connectors_with_release_stage(
-        registry_data["sources"], relevant_stages
-    ) + get_connectors_with_release_stage(registry_data["destinations"], relevant_stages)
-    # return appropriate directory names
-    return [
-        connector_label_to_connector_directory[label]
-        for label in connectors_with_desired_status
-        if label in connector_label_to_connector_directory
-    ]
-
-
-def setup_module():
-    global pytest
-    global mock
-
-
-if __name__ == "__main__":
-
-    # find all connectors and filter to beta and GA
-    connectors = sorted(os.listdir(CONNECTORS_ROOT_PATH))
-    relevant_stages = ["beta", "generally_available"]
-    relevant_connectors = get_connectors_with_release_stages(CONNECTORS_ROOT_PATH, connectors, relevant_stages)
-    print(f"Checking {len(relevant_connectors)} relevant connectors out of {len(connectors)} total connectors")
-
-    # analyse build results for each connector
-    [check_module(connector) for connector in relevant_connectors]
-    [check_module(base) for base in RELEVANT_BASE_MODULES]
-
-    report = create_report(relevant_connectors, relevant_stages)
-    print(report)
-    send_report(report)
-    print("Finish")
-elif "pytest" in sys.argv[0]:
-    import unittest
-
-    class Tests(unittest.TestCase):
-        def test_filter_definitions_yaml(self):
-            mock_def_yaml = [
-                {"releaseStage": "alpha", "dockerRepository": "alpha_connector"},
-                {"releaseStage": "beta", "dockerRepository": "beta_connector"},
-                {"releaseStage": "generally_available", "dockerRepository": "GA_connector"},
-            ]
-            assert ["alpha_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["alpha"])
-            assert ["alpha_connector", "beta_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["alpha", "beta"])
-            assert ["beta_connector", "GA_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["beta", "generally_available"])
-            assert ["GA_connector"] == get_connectors_with_release_stage(mock_def_yaml, ["generally_available"])
-
-        def test_parse_dockerfile_label(self):
-            mock_dockerfile = """
-ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
-
-LABEL io.airbyte.version=1.0.8
-LABEL io.airbyte.name=airbyte/source-salesforce"""
-            assert "airbyte/source-salesforce" == parse_dockerfile_repository_label(mock_dockerfile)
-
-        def test_download_and_parse_registry_json(self):
-            registry_data = download_and_parse_registry_json()
-            assert len(registry_data["sources"]) > 20
-            assert len(registry_data["destinations"]) > 20
-
-            # Assert that the dockerRepository is not empty
-            assert registry_data["sources"][0]["dockerRepository"]
-            assert registry_data["destinations"][0]["dockerRepository"]
diff --git a/tools/bin/bump_version.sh b/tools/bin/bump_version.sh
index 0337344843d8..56aeb95271d9 100755
--- a/tools/bin/bump_version.sh
+++ b/tools/bin/bump_version.sh
@@ -22,14 +22,20 @@ fi
 GIT_REVISION=$(git rev-parse HEAD)
 
 pip install bumpversion
-bumpversion "$PART_TO_BUMP" # PART_TO_BUMP comes from the Github action (patch,major,minor)
+if [ -z "${OVERRIDE_VERSION:-}" ]; then
+  # No override, so bump the version normally
+  bumpversion "$PART_TO_BUMP"
+else
+  # We have an override version, so use it directly
+  bumpversion --current-version $PREV_VERSION --new-version $OVERRIDE_VERSION "$PART_TO_BUMP"
+fi
 
 if [ "$REPO" == "airbyte" ]; then
   NEW_VERSION=$(grep -w 'VERSION=[0-9]\+\(\.[0-9]\+\)\+' run-ab-platform.sh | cut -d"=" -f2)
-  echo "Bumping version for Airbyte"
+  echo "Bumped version for Airbyte"
 else
   NEW_VERSION=$(grep -w VERSION .env | cut -d"=" -f2)
-  echo "Bumping version for Airbyte Platform"
+  echo "Bumped version for Airbyte Platform"
 fi
 
 export VERSION=$NEW_VERSION # for safety, since lib.sh exports a VERSION that is now outdated
diff --git a/tools/bin/ci_check_dependency.py b/tools/bin/ci_check_dependency.py
deleted file mode 100644
index 3d534ae0bdb2..000000000000
--- a/tools/bin/ci_check_dependency.py
+++ /dev/null
@@ -1,253 +0,0 @@
-import sys
-import os
-import os.path
-import re
-from typing import Any, Dict, Text, List
-import requests
-
-CONNECTOR_REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
-CONNECTORS_PATH = "./airbyte-integrations/connectors/"
-NORMALIZATION_PATH = "./airbyte-integrations/bases/base-normalization/"
-DOC_PATH = "docs/integrations/"
-IGNORE_LIST = [
-    # Java
-    "/src/test/","/src/test-integration/", "/src/testFixtures/",
-    # Python
-    "/integration_tests/", "/unit_tests/",
-    # Common
-    "acceptance-test-config.yml", "acceptance-test-docker.sh", ".md", ".dockerignore", ".gitignore", "requirements.txt"]
-IGNORED_SOURCES = [
-    re.compile("^source-e2e-test-cloud$"),
-    re.compile("^source-mongodb$"),
-    re.compile("^source-python-http-tutorial$"),
-    re.compile("^source-relational-db$"),
-    re.compile("^source-stock-ticker-api-tutorial$"),
-    re.compile("source-jdbc$"),
-    re.compile("^source-scaffold-.*$"),
-    re.compile(".*-secure$"),
-]
-IGNORED_DESTINATIONS = [
-    re.compile(".*-strict-encrypt$"),
-    re.compile("^destination-dev-null$"),
-    re.compile("^destination-scaffold-destination-python$"),
-    re.compile("^bases-destination-jdbc$")
-]
-COMMENT_TEMPLATE_PATH = ".github/comment_templates/connector_dependency_template.md"
-
-
-def download_and_parse_registry_json():
-    response = requests.get(CONNECTOR_REGISTRY_URL)
-    response.raise_for_status()
-    return response.json()
-
-
-def main():
-    # Used git diff checks airbyte-integrations/ folder only
-    # See .github/workflows/report-connectors-dependency.yml file
-    git_diff_file_path = ' '.join(sys.argv[1:])
-
-    if git_diff_file_path == None or git_diff_file_path == "":
-        raise Exception("No changefile provided")
-
-    # Get changed files
-    changed_files = get_changed_files(git_diff_file_path)
-    # Get changed modules. e.g. connector-acceptance-test from airbyte-integrations/bases/
-    # or destination-mysql from airbyte-integrations/connectors/
-    changed_modules = get_changed_modules(changed_files)
-
-    # Get all existing connectors
-    all_connectors = get_all_connectors()
-
-    # Getting all build.gradle file
-    build_gradle_files = {}
-    for connector in all_connectors:
-        connector_path = CONNECTORS_PATH + connector + "/"
-        build_gradle_files.update(get_gradle_file_for_path(connector_path))
-    build_gradle_files.update(get_gradle_file_for_path(NORMALIZATION_PATH))
-
-    # Try to find dependency in build.gradle file
-    dependent_modules = list(set(get_dependent_modules(changed_modules, build_gradle_files)))
-
-    # Create comment body to post on pull request
-    if dependent_modules:
-        write_report(dependent_modules)
-
-
-def get_changed_files(path):
-    changed_connectors_files = []
-    with open(path) as file:
-        for line in file:
-            changed_connectors_files.append(line)
-    return changed_connectors_files
-
-
-def get_changed_modules(changed_files):
-    changed_modules = []
-    for changed_file in changed_files:
-        # Check if this file should be ignored
-        if not any(ignor in changed_file for ignor in IGNORE_LIST):
-            split_changed_file = changed_file.split("/")
-            changed_modules.append(split_changed_file[1] + ":" + split_changed_file[2])
-    return list(set(changed_modules))
-
-
-def get_all_connectors():
-    walk = os.walk(CONNECTORS_PATH)
-    return [connector for connector in next(walk)[1]]
-
-
-def get_gradle_file_for_path(path: str) -> Dict[Text, Any]:
-    if not path.endswith("/"):
-        path = path + "/"
-    build_gradle_file = find_file("build.gradle", path)
-    module = path.split("/")[-2]
-    return {module: build_gradle_file}
-
-
-def find_file(name, path):
-    for root, dirs, files in os.walk(path):
-        if name in files:
-            return os.path.join(root, name)
-
-
-def get_dependent_modules(changed_modules, all_build_gradle_files):
-    dependent_modules = []
-    for changed_module in changed_modules:
-        for module, gradle_file in all_build_gradle_files.items():
-            if gradle_file is None:
-                continue
-            with open(gradle_file) as file:
-                if changed_module in file.read():
-                    dependent_modules.append(module)
-    return dependent_modules
-
-
-def get_connector_version(connector):
-    with open(f"{CONNECTORS_PATH}/{connector}/Dockerfile") as f:
-        for line in f:
-            if "io.airbyte.version" in line:
-                return line.split("=")[1].strip()
-
-
-def get_connector_version_status(connector, version):
-    if "strict-encrypt" not in connector:
-        return f"`{version}`"
-    if connector == "source-mongodb-strict-encrypt":
-        base_variant_version = get_connector_version("source-mongodb-v2")
-    else:
-        base_variant_version = get_connector_version(connector.replace("-strict-encrypt", ""))
-    if base_variant_version == version:
-        return f"`{version}`"
-    else:
-        return f"❌ `{version}`
    (mismatch: `{base_variant_version}`)"
-
-
-def get_connector_changelog_status(connector: str, version) -> str:
-    type, name = connector.replace("-strict-encrypt", "").replace("-denormalized", "").split("-", 1)
-    doc_path = f"{DOC_PATH}{type}s/{name}.md"
-
-    if any(regex.match(connector) for regex in IGNORED_SOURCES):
-        return "🔵
    (ignored)"
-    if any(regex.match(connector) for regex in IGNORED_DESTINATIONS):
-        return "🔵
    (ignored)"
-    if not os.path.exists(doc_path):
-        return "⚠
    (doc not found)"
-
-    with open(doc_path) as f:
-        after_changelog = False
-        for line in f:
-            if "# changelog" in line.lower():
-                after_changelog = True
-            if after_changelog and version in line:
-                return "✅"
-
-    return "❌
    (changelog missing)"
-
-
-def as_bulleted_markdown_list(items):
-    text = ""
-    for item in items:
-        text += f"- {item}\n"
-    return text
-
-
-def as_markdown_table_rows(connectors: List[str], definitions) -> str:
-    text = ""
-    for connector in connectors:
-        version = get_connector_version(connector)
-        version_status = get_connector_version_status(connector, version)
-        changelog_status = get_connector_changelog_status(connector, version)
-        definition = next((x for x in definitions if x["dockerRepository"].endswith(connector)), None)
-        if any(regex.match(connector) for regex in IGNORED_SOURCES):
-            publish_status = "🔵
    (ignored)"
-        elif any(regex.match(connector) for regex in IGNORED_DESTINATIONS):
-            publish_status = "🔵
    (ignored)"
-        elif definition is None:
-            publish_status = "⚠
    (not in seed)"
-        elif definition["dockerImageTag"] == version:
-            publish_status = "✅"
-        else:
-            publish_status = "❌
    (diff seed version)"
-        text += f"| `{connector}` | {version_status} | {changelog_status} | {publish_status} |\n"
-    return text
-
-
-def get_status_summary(rows: str) -> str:
-    if "❌" in rows:
-        return "❌"
-    elif "⚠" in rows:
-        return "⚠"
-    else:
-        return "✅"
-
-
-def write_report(depended_connectors):
-    affected_sources = []
-    affected_destinations = []
-    affected_others = []
-    for depended_connector in depended_connectors:
-        if depended_connector.startswith("source"):
-            affected_sources.append(depended_connector)
-        elif depended_connector.startswith("destination"):
-            affected_destinations.append(depended_connector)
-        else:
-            affected_others.append(depended_connector)
-
-    with open(COMMENT_TEMPLATE_PATH, "r") as f:
-        template = f.read()
-
-    registry_data = download_and_parse_registry_json()
-    source_definitions = registry_data["sources"]
-    destination_definitions = registry_data["destinations"]
-
-    affected_sources.sort()
-    affected_destinations.sort()
-    affected_others.sort()
-
-    source_rows = as_markdown_table_rows(affected_sources, source_definitions)
-    destination_rows = as_markdown_table_rows(affected_destinations, destination_definitions)
-
-    other_status_summary = "✅" if len(affected_others) == 0 else "👀"
-    source_status_summary = get_status_summary(source_rows)
-    destination_status_summary = get_status_summary(destination_rows)
-
-    comment = template.format(
-        source_open="open" if source_status_summary == "❌" else "closed",
-        destination_open="open" if destination_status_summary == "❌" else "closed",
-        source_status_summary=source_status_summary,
-        destination_status_summary=destination_status_summary,
-        other_status_summary=other_status_summary,
-        source_rows=source_rows,
-        destination_rows=destination_rows,
-        others_rows=as_bulleted_markdown_list(affected_others),
-        num_sources=len(affected_sources),
-        num_destinations=len(affected_destinations),
-        num_others=len(affected_others),
-    )
-
-    with open("comment_body.md", "w") as f:
-        f.write(comment)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh
index b64d203c630c..a13a8611d92f 100755
--- a/tools/bin/ci_integration_test.sh
+++ b/tools/bin/ci_integration_test.sh
@@ -21,11 +21,6 @@ else
     # avoid schema conflicts when multiple tests for normalization are run concurrently
     export RANDOM_TEST_SCHEMA="true"
     ./gradlew --no-daemon --scan airbyteDocker
-  elif [[ "$connector" == *"connector-acceptance-test"* ]]; then
-    connector_name=$(echo $connector | cut -d / -f 2)
-    selected_integration_test="connector-acceptance-test"
-    integrationTestCommand="$(_to_gradle_path "airbyte-integrations/bases/$connector_name" integrationTest)"
-    export SUB_BUILD="CONNECTORS_BASE"
   elif [[ "$connector" == *"bases"* ]]; then
     connector_name=$(echo $connector | cut -d / -f 2)
     selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector_name$" || echo "")
diff --git a/tools/bin/ci_integration_workflow_launcher.py b/tools/bin/ci_integration_workflow_launcher.py
deleted file mode 100644
index e1604db16344..000000000000
--- a/tools/bin/ci_integration_workflow_launcher.py
+++ /dev/null
@@ -1,239 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import datetime
-import logging
-import os
-import re
-import subprocess
-import sys
-import time
-import uuid
-from functools import lru_cache
-from urllib.parse import parse_qsl, urljoin, urlparse
-
-import requests
-
-ORGANIZATION = "airbytehq"
-REPOSITORY = "airbyte"
-LOGGING_FORMAT = "%(asctime)-15s %(levelname)s %(message)s"
-API_URL = "https://api.github.com"
-BRANCH = "master"
-WORKFLOW_PATH = ".github/workflows/test-command.yml"
-RUN_UUID_REGEX = re.compile("^UUID ([0-9a-f-]+)$")
-SLEEP = 1200
-CONNECTOR_REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
-STAGES = ["alpha", "beta", "generally_available"]
-
-
-GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
-if not GITHUB_TOKEN:
-    logging.error("GITHUB_TOKEN not set...")
-    sys.exit(1)
-
-
-def download_and_parse_registry_json():
-    response = requests.get(CONNECTOR_REGISTRY_URL)
-    response.raise_for_status()
-    return response.json()
-
-
-def check_start_aws_runner_failed(jobs):
-    """
-    !!! WARNING !!! WARNING !!! WARNING !!!
-    !!! WARNING !!! WARNING !!! WARNING !!!
-    !!! WARNING !!! WARNING !!! WARNING !!!
-
-    If workflow {WORKFLOW_PATH} structure will change in future
-    there is a chance that we would need to update this function too.
-    """
-    return (
-        len(jobs) >= 2
-        and len(jobs[1]["steps"]) >= 3
-        and jobs[1]["steps"][2]["name"] == "Start AWS Runner"
-        and jobs[1]["steps"][2]["conclusion"] == "failure"
-    )
-
-
-def get_run_uuid(jobs):
-    """
-    This function relies on assumption that the first step of the first job
-
-    - name: UUID ${{ github.event.inputs.uuid }}
-      run: true
-    """
-    if jobs and len(jobs[0]["steps"]) >= 2:
-        name = jobs[0]["steps"][1]["name"]
-        m = re.match(RUN_UUID_REGEX, name)
-        if m:
-            return m.groups()[0]
-
-
-def get_response(url_or_path, params=None):
-    url = urljoin(API_URL, url_or_path)
-    response = requests.get(url, params=params, headers={"Authorization": "Bearer " + GITHUB_TOKEN})
-    response.raise_for_status()
-    return response
-
-
-def get_response_json(url_or_path, params=None):
-    response = get_response(url_or_path, params=params)
-    return response.json()
-
-
-def get_workflow_id(owner, repo, path):
-    response_json = get_response_json(f"/repos/{owner}/{repo}/actions/workflows")
-    for workflow in response_json["workflows"]:
-        if workflow["path"] == path:
-            return workflow["id"]
-
-
-def workflow_dispatch(owner, repo, workflow_id, connector):
-    run_uuid = str(uuid.uuid4())
-    url = urljoin(API_URL, f"/repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches")
-    response = requests.post(
-        url, headers={"Authorization": "Bearer " + GITHUB_TOKEN}, json={"ref": BRANCH, "inputs": {"connector": connector, "uuid": run_uuid}}
-    )
-    response.raise_for_status()
-    return run_uuid
-
-
-@lru_cache
-def get_gradlew_integrations():
-    process = subprocess.run(["./gradlew", "integrationTest", "--dry-run"], check=True, capture_output=True, universal_newlines=True)
-    res = []
-    for line in process.stdout.splitlines():
-        parts = line.split(":")
-        if (
-            len(parts) >= 4
-            and parts[1] == "airbyte-integrations"
-            and parts[2] in ["connectors", "bases"]
-            and parts[-1] == "integrationTest SKIPPED"
-        ):
-            res.append(parts[3])
-    return res
-
-
-@lru_cache
-def get_definitions(definition_type):
-    assert definition_type in ["source", "destination"]
-
-    plural_key = definition_type + "s"
-
-    registry_data = download_and_parse_registry_json()
-    return registry_data[plural_key]
-
-
-
-def normalize_stage(stage):
-    stage = stage.lower()
-    if stage == "ga":
-        stage = "generally_available"
-    return stage
-
-
-def get_integrations(names):
-    res = set()
-    for name in names:
-        parts = name.split(":")
-        if len(parts) == 2:
-            definition_type, stage = parts
-            stage = normalize_stage(stage)
-            if stage == "all":
-                for integration in get_gradlew_integrations():
-                    if integration.startswith(definition_type + "-"):
-                        res.add(integration)
-            elif stage in STAGES:
-                for definition in get_definitions(definition_type):
-                    if definition.get("releaseStage", "alpha") == stage:
-                        res.add(definition["dockerRepository"].partition("/")[2])
-            else:
-                logging.warning(f"unknown stage: '{stage}'")
-        else:
-            integration = parts[0]
-            airbyte_integrations = get_gradlew_integrations()
-            if integration in airbyte_integrations:
-                res.add(integration)
-            else:
-                logging.warning(f"integration not found: {integration}")
-    return res
-
-
-def iter_workflow_runs(owner, repo, per_page=100):
-    path = f"/repos/{owner}/{repo}/actions/runs"
-    page = None
-    while True:
-        params = {"per_page": per_page}
-        if page:
-            params["page"] = page
-        response = get_response(path, params=params)
-        response_json = response.json()
-        for workflow_run in response_json["workflow_runs"]:
-            yield workflow_run
-        if "next" not in response.links:
-            break
-        page = dict(parse_qsl(urlparse(response.links["next"]["url"]).query))["page"]
-
-
-def search_failed_workflow_runs(owner, repo, workflow_id, run_uuids):
-    run_uuids = set(run_uuids)
-    now = datetime.datetime.utcnow()
-    res = set()
-    for workflow_run in iter_workflow_runs(owner, repo):
-        if not run_uuids:
-            break
-
-        created_at = datetime.datetime.strptime(workflow_run["created_at"], "%Y-%m-%dT%H:%M:%SZ")
-        period = now - created_at
-        if period.seconds > 10800:
-            break
-
-        if workflow_run["workflow_id"] != workflow_id:
-            continue
-        if workflow_run["head_branch"] != BRANCH:
-            continue
-        if workflow_run["conclusion"] != "failure":
-            continue
-
-        response_json = get_response_json(workflow_run["jobs_url"])
-        run_uuid = get_run_uuid(response_json["jobs"])
-        if not run_uuid:
-            continue
-
-        if run_uuid in run_uuids:
-            run_uuids.remove(run_uuid)
-            if check_start_aws_runner_failed(response_json["jobs"]):
-                res.add(run_uuid)
-    return res
-
-
-def main():
-    workflow_id = get_workflow_id(ORGANIZATION, REPOSITORY, WORKFLOW_PATH)
-    if not workflow_id:
-        logging.error(f"Cannot find workflow path '{WORKFLOW_PATH}'")
-        sys.exit(1)
-
-    integration_names = get_integrations(sys.argv[1:])
-    run_uuid_to_name = {}
-    for integration_name in integration_names:
-        run_uuid = workflow_dispatch(ORGANIZATION, REPOSITORY, workflow_id, integration_name)
-        logging.info(f"Dispatch workflow for connector {integration_name}, UUID: {run_uuid}")
-        run_uuid_to_name[run_uuid] = integration_name
-        # to avoid overloading system
-        time.sleep(1)
-
-    logging.info(f"Sleeping {SLEEP} seconds")
-    time.sleep(SLEEP)
-
-    run_uuids = search_failed_workflow_runs(ORGANIZATION, REPOSITORY, workflow_id, run_uuid_to_name.keys())
-    for run_uuid in run_uuids:
-        integration_name = run_uuid_to_name[run_uuid]
-        run_uuid = workflow_dispatch(ORGANIZATION, REPOSITORY, workflow_id, integration_name)
-        logging.info(f"Re-dispatch workflow for connector {integration_name}, UUID: {run_uuid}")
-
-
-if __name__ == "__main__":
-    logging.basicConfig(format=LOGGING_FORMAT, level=logging.INFO)
-    main()
diff --git a/tools/bin/cleanup-workflow-runs.py b/tools/bin/cleanup-workflow-runs.py
index 993d67e662b0..d2ef9a38cbbc 100644
--- a/tools/bin/cleanup-workflow-runs.py
+++ b/tools/bin/cleanup-workflow-runs.py
@@ -1,14 +1,19 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import argparse
 import os
-import subprocess
 import re
+import subprocess
 from datetime import datetime, timedelta
+
 from github import Github
 
 DAYS_TO_KEEP_ORPHANED_JOBS = 90
 
 
-'''
+"""
 
 This script is intended to be run in conjuction with identify-dormant-workflows.py to keep GH actions clean.
 
@@ -23,14 +28,21 @@
 We don't want to delete workflow runs newer than 90 days on GH actions, even if the workflow doesn't exist.
 it's possible that people might test things off the master branch and we don't want to delete their recent runs
 
-'''
+"""
 
 # Initiate the parser
 parser = argparse.ArgumentParser()
 
 # Add long and short argument
 parser.add_argument("--pat", "-p", help="Set github personal access token")
-parser.add_argument("--delete", "-d", action='store', nargs='*', help="By default, the script will only print runs that will be deleted. Pass --delete to actually delete them")
+parser.add_argument(
+    "--delete",
+    "-d",
+    action="store",
+    nargs="*",
+    help="By default, the script will only print runs that will be deleted. Pass --delete to actually delete them",
+)
+
 
 def main():
     # Read arguments from the command line
@@ -43,19 +55,19 @@ def main():
     if args.pat:
         token = args.pat
     else:
-        token = os.getenv('GITHUB_TOKEN')
+        token = os.getenv("GITHUB_TOKEN")
     if not token:
         raise Exception("Github personal access token not provided via args and not available in GITHUB_TOKEN variable")
 
     g = Github(token)
 
-    git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True)  
+    git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True)
 
     # will match both forms (git and https url) of github e.g.
     # git@github.com:airbytehq/airbyte.git
     # https://github.com/airbytehq/airbyte.git
 
-    git_url_regex = re.compile(r'(?:git@|https://)github\.com[:/](.*?)(\.git|$)') 
+    git_url_regex = re.compile(r"(?:git@|https://)github\.com[:/](.*?)(\.git|$)")
     re_match = git_url_regex.match(git_url.stdout.decode("utf-8"))
 
     repo = g.get_repo(re_match.group(1))
@@ -64,22 +76,24 @@ def main():
     runs_to_delete = []
 
     for workflow in workflows:
-        if not os.path.exists(workflow.path): # it's not in the current branch 
+        if not os.path.exists(workflow.path):  # it's not in the current branch
             runs = workflow.get_runs()
             for run in runs:
-                if run.updated_at > datetime.now() - timedelta(days=DAYS_TO_KEEP_ORPHANED_JOBS): 
-                    break # don't clean up if it has a run newer than 90 days
+                if run.updated_at > datetime.now() - timedelta(days=DAYS_TO_KEEP_ORPHANED_JOBS):
+                    break  # don't clean up if it has a run newer than 90 days
                 if args.delete is not None:
                     print("Deleting run id " + str(run.id))
-                    run._requester.requestJson("DELETE", run.url) # normally we would use run.delete() but even though it's been merged it's not yet in pypi: https://github.com/PyGithub/PyGithub/pull/2078
+                    run._requester.requestJson(
+                        "DELETE", run.url
+                    )  # normally we would use run.delete() but even though it's been merged it's not yet in pypi: https://github.com/PyGithub/PyGithub/pull/2078
                 else:
                     runs_to_delete.append((workflow.name, run.id, run.created_at.strftime("%m/%d/%Y, %H:%M:%S")))
-                
+
     if args.delete is None:
         print("[DRY RUN] A total of " + str(len(runs_to_delete)) + " runs would be deleted: ")
         for run in runs_to_delete:
             print(run)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()
diff --git a/tools/bin/identify-dormant-workflows.py b/tools/bin/identify-dormant-workflows.py
index 64a7b65ab886..5dfa954c6056 100644
--- a/tools/bin/identify-dormant-workflows.py
+++ b/tools/bin/identify-dormant-workflows.py
@@ -1,9 +1,14 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import argparse
+import json
 import os
-import subprocess
 import re
-import json
+import subprocess
 from datetime import datetime, timedelta
+
 from github import Github
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
@@ -12,7 +17,7 @@
 SLACK_CHANNEL_FOR_NOTIFICATIONS = "infra-alerts"
 
 
-'''
+"""
 
 This script is intended to be run in conjuction with cleanup-workflow-runs.py to keep GH actions clean.
 
@@ -27,7 +32,7 @@
 We don't want to delete workflow runs newer than 90 days on GH actions, even if the workflow doesn't exist.
 it's possible that people might test things off the master branch and we don't want to delete their recent runs
 
-'''
+"""
 
 # Initiate the parser
 parser = argparse.ArgumentParser()
@@ -36,11 +41,11 @@
 parser.add_argument("--pat", "-p", help="Set github personal access token")
 parser.add_argument("--sat", "-s", help="Set slack api token. Optional. If not passed, will just print to console")
 
+
 def main():
     # Read arguments from the command line
     args = parser.parse_args()
 
-
     # Check for user supplied PAT. If not supplied, assume we are running in actions
     # and pull from environment
     gh_token = None
@@ -49,25 +54,24 @@ def main():
     if args.pat:
         gh_token = args.pat
     else:
-        gh_token = os.getenv('GITHUB_TOKEN')
+        gh_token = os.getenv("GITHUB_TOKEN")
     if not gh_token:
         raise Exception("Github personal access token not provided via args and not available in GITHUB_TOKEN variable")
-    
+
     if args.sat:
         slack_token = args.sat
     else:
-        slack_token = os.getenv('SLACK_TOKEN')
+        slack_token = os.getenv("SLACK_TOKEN")
 
     g = Github(gh_token)
 
-    git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True)  
+    git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True)
 
     # will match both forms (git and https url) of github e.g.
     # git@github.com:airbytehq/airbyte.git
     # https://github.com/airbytehq/airbyte.git
 
-
-    git_url_regex = re.compile(r'(?:git@|https://)github\.com[:/](.*?)(\.git|$)')
+    git_url_regex = re.compile(r"(?:git@|https://)github\.com[:/](.*?)(\.git|$)")
     re_match = git_url_regex.match(git_url.stdout.decode("utf-8"))
 
     repo_name = re_match.group(1)
@@ -82,19 +86,28 @@ def main():
         for run in runs:
             # check and see if a workflow exists but is not actively being triggered/run
             if os.path.exists(workflow.path) and run.updated_at < datetime.now() - timedelta(days=DAYS_TO_KEEP_ORPHANED_JOBS):
-                message = "The Github Workflow '" + workflow.name + "' exists in " + repo_name + " but has no run newer than 90 days old. URL: " + workflow.html_url
-                print(message)  
+                message = (
+                    "The Github Workflow '"
+                    + workflow.name
+                    + "' exists in "
+                    + repo_name
+                    + " but has no run newer than 90 days old. URL: "
+                    + workflow.html_url
+                )
+                print(message)
 
                 if slack_token:
 
                     print("Sending Slack notification...")
                     client = WebClient(slack_token)
 
-                    try:  response = client.chat_postMessage(channel = SLACK_CHANNEL_FOR_NOTIFICATIONS, text = message)
-                    except SlackApiError as e:  
-                        print(e, '\n\n')
+                    try:
+                        response = client.chat_postMessage(channel=SLACK_CHANNEL_FOR_NOTIFICATIONS, text=message)
+                    except SlackApiError as e:
+                        print(e, "\n\n")
                         raise Exception("Error calling the Slack API")
             break
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main()
diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py
index d1f015ea9423..d044a45bebce 100644
--- a/tools/bin/prep_test_results_for_gcs.py
+++ b/tools/bin/prep_test_results_for_gcs.py
@@ -1,24 +1,28 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import argparse
 import json
 import os
 
-
-'''
+"""
 
 This script is intended to be run in conjuction with https://github.com/EnricoMi/publish-unit-test-result-action to upload trimmed
 test results from the output to a GCS bucket for further analysis.
 
 The script takes as input the filename of the json output by the aforementioned action, trims it, and writes it out in jsonl format with ".jsonl" filename 
 
-'''
+"""
 
 # Initiate the parser
 parser = argparse.ArgumentParser()
 
 # Add long and short argument
 parser.add_argument("--json", "-j", help="Path to the result json output by https://github.com/EnricoMi/publish-unit-test-result-action")
-parser.add_argument("--runid", "-r", help="Run id of the action") # this can be derived from checks api, but it's easier to derive it here
-parser.add_argument("--jobid", "-c", help="Job id of the action") # this can be derived from checks api, but it's easier to derive it here
+parser.add_argument("--runid", "-r", help="Run id of the action")  # this can be derived from checks api, but it's easier to derive it here
+parser.add_argument("--jobid", "-c", help="Job id of the action")  # this can be derived from checks api, but it's easier to derive it here
+
 
 def main():
     # Read arguments from the command line
@@ -27,32 +31,32 @@ def main():
     f = open(args.json)
     d = json.load(f)
     out = []
-    
+
     check_run_id = int(d["check_url"].split("/")[-1])
 
-    for elem in d['cases']:
-        for conclusion in ('success', 'failure', 'skipped'):
-            if conclusion not in elem['states']: 
+    for elem in d["cases"]:
+        for conclusion in ("success", "failure", "skipped"):
+            if conclusion not in elem["states"]:
                 continue
-            for i in range(len(elem['states'][conclusion])):
+            for i in range(len(elem["states"][conclusion])):
                 output = {
-                    "test_name": elem['states'][conclusion][i]['test_name'],
-                    "class_name": elem['states'][conclusion][i]['class_name'],
-                    "result_file": elem['states'][conclusion][i]['result_file'],
-                    "time": elem['states'][conclusion][i]['time'],
+                    "test_name": elem["states"][conclusion][i]["test_name"],
+                    "class_name": elem["states"][conclusion][i]["class_name"],
+                    "result_file": elem["states"][conclusion][i]["result_file"],
+                    "time": elem["states"][conclusion][i]["time"],
                     "state": conclusion,
                     "check_run_id": check_run_id,
                     "workflow_run_id": args.runid,
                     "job_id": args.jobid,
-                    "repo": "airbytehq/airbyte"
+                    "repo": "airbytehq/airbyte",
                 }
                 out.append(output)
 
-    with open(args.json + "l", 'w') as f:
+    with open(args.json + "l", "w") as f:
         for o in out:
             json.dump(o, f)
-            f.write('\n')
+            f.write("\n")
 
 
-if __name__ == '__main__':
-    main()
\ No newline at end of file
+if __name__ == "__main__":
+    main()
diff --git a/tools/bin/update_intellij_venv.py b/tools/bin/update_intellij_venv.py
index 36b955c9386b..dfb259ee9255 100644
--- a/tools/bin/update_intellij_venv.py
+++ b/tools/bin/update_intellij_venv.py
@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
 import argparse
 import os
 import subprocess
@@ -31,13 +35,12 @@ def add_venv_to_xml_root(module: str, module_full_path: str, xml_root):
         print(f"{environment_name} already exists. Skipping...")
         return
 
-    jdk_node = ET.SubElement(table, 'jdk', {"version": "2"})
+    jdk_node = ET.SubElement(table, "jdk", {"version": "2"})
 
     ET.SubElement(jdk_node, "name", {"value": environment_name})
     ET.SubElement(jdk_node, "type", {"value": "Python SDK"})
     ET.SubElement(jdk_node, "version", {"value": f"{python_version}"})
-    ET.SubElement(jdk_node, "homePath",
-                  {"value": f"{module_full_path}/.venv/bin/python"})
+    ET.SubElement(jdk_node, "homePath", {"value": f"{module_full_path}/.venv/bin/python"})
 
     roots = ET.SubElement(jdk_node, "roots")
     annotationsPath = ET.SubElement(roots, "annotationsPath")
@@ -46,10 +49,7 @@ def add_venv_to_xml_root(module: str, module_full_path: str, xml_root):
     classPath = ET.SubElement(roots, "classPath")
     classPathRoot = ET.SubElement(classPath, "root", {"type": "composite"})
 
-    ET.SubElement(classPathRoot, "root", {"url":
-                                              f"file://{path_to_lib}{python_version}/site-packages",
-                                          "type": "simple"
-                                          })
+    ET.SubElement(classPathRoot, "root", {"url": f"file://{path_to_lib}{python_version}/site-packages", "type": "simple"})
 
 
 def get_output_path(input_path, output_path):
@@ -72,7 +72,8 @@ def get_input_path(input_from_args, version, home_directory):
             intellij_version_to_update = intellij_versions[0]
         else:
             raise RuntimeError(
-                f"Please select which version of Intellij to update with the `{INTELLIJ_VERSION_FLAG}` flag. Options are: {intellij_versions}")
+                f"Please select which version of Intellij to update with the `{INTELLIJ_VERSION_FLAG}` flag. Options are: {intellij_versions}"
+            )
         return f"{path_to_intellij_settings}{intellij_version_to_update}/options/jdk.table.xml"
 
 
@@ -91,12 +92,12 @@ def get_default_airbyte_path():
 def create_parser():
     parser = argparse.ArgumentParser(description="Prepare Python virtual environments for Python connectors")
     actions_group = parser.add_argument_group("actions")
-    actions_group.add_argument("--install-venv", action="store_true",
-                               help="Create virtual environment and install the module's dependencies")
+    actions_group.add_argument(
+        "--install-venv", action="store_true", help="Create virtual environment and install the module's dependencies"
+    )
     actions_group.add_argument("--update-intellij", action="store_true", help="Add interpreter to IntelliJ's list of known interpreters")
 
-    parser.add_argument("-airbyte", default=get_default_airbyte_path(),
-                        help="Path to Airbyte root directory")
+    parser.add_argument("-airbyte", default=get_default_airbyte_path(), help="Path to Airbyte root directory")
 
     modules_group = parser.add_mutually_exclusive_group(required=True)
     modules_group.add_argument("-modules", nargs="?", help="Comma separated list of modules to add (eg source-strava,source-stripe)")
@@ -150,7 +151,7 @@ def parse_args(args):
         input_path = get_input_path(args.input, args.intellij_version, home_directory)
 
         output_path = get_output_path(input_path, args.output)
-        with open(input_path, 'r') as f:
+        with open(input_path, "r") as f:
             root = ET.fromstring(f.read())
 
             for module in modules:
@@ -173,7 +174,6 @@ def setup_module():
 if "pytest" in sys.argv[0]:
     import unittest
 
-
     class TestNoneTypeError(unittest.TestCase):
         def test_output_is_input_if_not_set(self):
             input_path = "/input_path"
@@ -187,8 +187,7 @@ def test_get_output_path(self):
 
         @unittest.mock.patch("os.walk")
         def test_input_is_selected(self, mock_os):
-            os.walk.return_value = iter(
-                (("./test1", ["consentOptions", "IdeaIC2021.3", "PyCharmCE2021.3"], []),))
+            os.walk.return_value = iter((("./test1", ["consentOptions", "IdeaIC2021.3", "PyCharmCE2021.3"], []),))
             os.getenv.return_value = "{HOME}"
             input_from_args = None
             version = "IdeaIC2021.3"
@@ -197,8 +196,7 @@ def test_input_is_selected(self, mock_os):
 
         @unittest.mock.patch("os.walk")
         def test_input_single_intellij_version(self, mock_os):
-            os.walk.return_value = iter(
-                (("./test1", ["consentOptions", "IdeaIC2021.3"], []),))
+            os.walk.return_value = iter((("./test1", ["consentOptions", "IdeaIC2021.3"], []),))
             input_from_args = None
 
             version = None
@@ -207,8 +205,7 @@ def test_input_single_intellij_version(self, mock_os):
 
         @unittest.mock.patch("os.walk")
         def test_input_multiple_intellij_versions(self, mock_os):
-            os.walk.return_value = iter(
-                (('./test1', ['consentOptions', 'IdeaIC2021.3', "PyCharmCE2021.3"], []),))
+            os.walk.return_value = iter((("./test1", ["consentOptions", "IdeaIC2021.3", "PyCharmCE2021.3"], []),))
             input_from_args = None
 
             version = None
diff --git a/tools/ci_code_validator/ci_changes_detection/main.py b/tools/ci_code_validator/ci_changes_detection/main.py
deleted file mode 100644
index 0724080bb396..000000000000
--- a/tools/ci_code_validator/ci_changes_detection/main.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
-import json
-import sys
-from pathlib import Path
-from typing import Dict, List
-
-from ci_common_utils import Logger
-from ci_sonar_qube import ROOT_DIR
-
-LOGGER = Logger()
-
-AVAILABLE_SCAN_FOLDERS = (
-    "airbyte-integrations/connectors",
-    "airbyte-cdk/python",
-    "airbyte-integrations/bases/connector-acceptance-test",
-)
-
-
-def get_module_folder(dir_path: Path) -> Path:
-    while dir_path and str(dir_path) != dir_path.root and dir_path != dir_path.parent:
-        parent_path = dir_path.parent
-        if dir_path.is_dir():
-            for available_folder in AVAILABLE_SCAN_FOLDERS:
-                if str(parent_path).endswith(available_folder):
-                    """first child of known folder"""
-                    return dir_path
-        """keep looking up"""
-        dir_path = dir_path.parent
-
-    return None
-
-
-def get_module_type(dir_path: Path) -> Path:
-    """All Java connectors have a folder src/main/java into own folders"""
-    required_java_dir = dir_path / "src/main/java"
-    if required_java_dir.is_dir():
-        return "java"
-
-    """All Python connectors have setup.py file into own software folders"""
-    setup_py_file = dir_path / "setup.py"
-    if setup_py_file.is_file():
-        return "py"
-
-    return None
-
-
-def list_changed_modules(changed_files: List[str]) -> List[Dict[str, str]]:
-    """
-    changed_filed are the list of files which were modified in current branch.
-    E.g. changed_files = ["tools/ci_static_check_reports/__init__.py", "tools/ci_static_check_reports/setup.py", ...]
-    """
-    module_folders = {}
-    for file_path in changed_files:
-        if not file_path.startswith("/"):
-            file_path = ROOT_DIR / file_path
-        else:
-            file_path = Path(file_path)
-
-        module_folder = get_module_folder(file_path)
-        if module_folder:
-            module_type = get_module_type(module_folder)
-            if not module_type:
-                LOGGER.info(f"skip the folder {module_folder}...")
-            else:
-                module_folders[module_folder] = module_type
-
-    modules = []
-    for module_folder, lang in module_folders.items():
-        module_folder = str(module_folder)
-        parts = module_folder.split("/")
-        module_name = "/".join(parts[-2:])
-        modules.append({"folder": module_folder, "lang": lang, "module": module_name})
-        LOGGER.info(f"Detected the module: {module_name}({lang}) in the folder: {module_folder}")
-        # _, file_extension = os.path.splitext(file_path)
-        # find_base_path(file_path, modules, file_ext=file_extension, unique_modules=unique_modules)
-    return modules
-
-
-def main() -> int:
-    changed_modules = list_changed_modules(sys.argv[1:])
-    print(json.dumps(changed_modules))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/tools/ci_code_validator/ci_sonar_qube/__init__.py b/tools/ci_code_validator/ci_sonar_qube/__init__.py
deleted file mode 100644
index a6e83a6386e8..000000000000
--- a/tools/ci_code_validator/ci_sonar_qube/__init__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
-import os
-from pathlib import Path
-
-from ci_common_utils import Logger
-
-LOGGER = Logger()
-
-ROOT_DIR = Path(os.getcwd())
-while str(ROOT_DIR) != "/" and not (ROOT_DIR / "gradlew").is_file():
-    ROOT_DIR = ROOT_DIR.parent
-if str(ROOT_DIR) == "/":
-    LOGGER.critical("this script must be executed into the Airbyte repo only")
diff --git a/tools/ci_code_validator/ci_sonar_qube/log_parsers.py b/tools/ci_code_validator/ci_sonar_qube/log_parsers.py
deleted file mode 100644
index 56fd978a1603..000000000000
--- a/tools/ci_code_validator/ci_sonar_qube/log_parsers.py
+++ /dev/null
@@ -1,311 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from dataclasses import dataclass
-from enum import Enum
-from pathlib import Path
-from typing import Callable, TextIO, List, Optional, Mapping, Any
-
-from mypy.errorcodes import error_codes as mypy_error_codes, ErrorCode
-from unidiff import PatchSet
-
-from .sonar_qube_api import SonarQubeApi
-
-HERE = Path(os.getcwd())
-RE_MYPY_LINE = re.compile(r"^(.+):(\d+):(\d+):")
-RE_MYPY_LINE_WO_COORDINATES = re.compile(r"^(.+): error: (.+)")
-
-FORMAT_TIP = "Please go to the repo root and run the command: './gradlew --no-daemon " \
-             ":airbyte-integrations:connectors::airbytePythonFormat'"
-
-
-class IssueSeverity(Enum):
-    blocker = "BLOCKER"
-    critical = "CRITICAL"
-    major = "MAJOR"
-    minor = "MINOR"
-    info = "INFO"
-
-
-@dataclass
-class Rule:
-    class Type(Enum):
-        code_smell = "CODE_SMELL"
-        bug = "BUG"
-        vulnerability = "VULNERABILITY"
-        security_hotspot = "SECURITY_HOTSPOT"
-
-    rule_type: Type
-    key: str
-    name: str
-    description: str
-    tool_name: str
-    template: str
-    severity: IssueSeverity
-
-    @property
-    def unique_key(self):
-        return f"{self.tool_name}_{self.key}".replace("-", "_")
-
-    @property
-    def sq_key(self):
-        lang_part = self.template.split(":")[0]
-        return f"{lang_part}:{self.tool_name}_{self.key}".replace("-", "_")
-
-
-def generate_mypy_rules() -> Mapping[str, Rule]:
-    try:
-        addl_code = ErrorCode(
-            code="unknown",
-            description="Unknown error",
-            category="General",
-        )
-    except NameError:
-        return []
-    return {f"[{err.code}]": Rule(
-        rule_type=Rule.Type.code_smell,
-        key=err.code,
-        name=err.code.replace("-", " ").capitalize() + " (mypy)",
-        description=err.description,
-        tool_name="mypy",
-        severity=IssueSeverity.minor,
-        template="python:CommentRegularExpression"
-    ) for err in list(mypy_error_codes.values()) + [addl_code]}
-
-
-class LogParser(SonarQubeApi):
-    _mypy_rules: Mapping[str, Rule] = generate_mypy_rules()
-    _black_rule = Rule(
-        rule_type=Rule.Type.code_smell,
-        key="need_format",
-        name="Should be formatted (black)",
-        description=FORMAT_TIP,
-        tool_name="black",
-        severity=IssueSeverity.minor,
-        template="python:CommentRegularExpression"
-    )
-
-    _isort_rule = Rule(
-        rule_type=Rule.Type.code_smell,
-        key="need_format",
-        name="Should be formatted (isort)",
-        description=FORMAT_TIP,
-        tool_name="isort",
-        severity=IssueSeverity.minor,
-        template="python:CommentRegularExpression"
-    )
-
-    @dataclass
-    class Issue:
-        path: str
-
-        rule: Rule
-        description: str
-
-        line_number: int = None  # 1-indexed
-        column_number: int = None  # 1-indexed
-
-        def to_json(self):
-            data = {
-                "engineId": self.rule.tool_name,
-                "ruleId": self.rule.sq_key,
-                "severity": self.rule.severity.value,
-                "type": self.rule.rule_type.value,
-                "primaryLocation": {
-                    "message": self.description,
-                    "filePath": self.checked_path,
-                }
-            }
-            if self.line_number is not None:
-                data["primaryLocation"]["textRange"] = {
-                    "startLine": self.line_number,
-                    "endLine": self.line_number,
-                    "startColumn": self.column_number - 1,  # 0-indexed
-                    "endColumn": self.column_number,  # 0-indexed
-                }
-            return data
-
-        @property
-        def checked_path(self):
-            if self.path.startswith(str(HERE) + "/"):
-                # remove a parent part of path
-                return self.path[len(str(HERE) + "/"):].strip()
-            return self.path.strip()
-
-    def __init__(self, output_file: str, host: str, token: str):
-        super().__init__(host=host, token=token, pr_name="0")
-        self.output_file = output_file
-
-    def prepare_file(func: Callable) -> Callable:
-        def intra(self, input_file: str) -> int:
-            if not os.path.exists(input_file):
-                self.logger.critical(f"not found input file: {input_file}")
-            with open(input_file, "r") as file:
-                issues = func(self, file)
-                self._save_all_rules(issues)
-                data = self._issues2dict(issues)
-                with open(self.output_file, "w") as output_file:
-                    output_file.write(json.dumps(data))
-                self.logger.info(f"the file {self.output_file} was updated with {len(issues)} issues")
-                return 0
-            return 1
-
-        return intra
-
-    def _save_all_rules(self, issues: List[Issue]) -> bool:
-        """Checks and create SQ rules if needed"""
-        if not issues:
-            return False
-        rules = defaultdict(list)
-        for issue in issues:
-            rules[issue.rule.tool_name].append(issue.rule)
-        for tool_name, tool_rules in rules.items():
-            exist_rules = [rule["key"] for rule in self._get_list(f"rules/search?include_external=true&q={tool_name}", "rules")]
-            grouped_rules = {rule.sq_key: rule for rule in tool_rules}
-            for sq_key, rule in grouped_rules.items():
-                if sq_key in exist_rules:
-                    # was created before
-                    continue
-                self.logger.info(f"try to create the rule: {sq_key}")
-                body = {
-                    "custom_key": rule.unique_key,
-                    "markdown_description": rule.description,
-                    "name": rule.name,
-                    "severity": rule.severity.value,
-                    "type": rule.rule_type.value,
-                    "template_key": rule.template
-                }
-                self._post("rules/create", body)
-                self.logger.info(f"the rule {sq_key} was created")
-        return True
-
-    def _issues2dict(self, issues: List[Issue]) -> Mapping[str, Any]:
-        """
-         {
-            "issues": [
-                {
-                    "engineId": "test",
-                    "ruleId": "rule1",
-                    "severity":"BLOCKER",
-                    "type":"CODE_SMELL",
-                    "primaryLocation": {
-                        "message": "fully-fleshed issue",
-                        "filePath": "sources/A.java",
-                        "textRange": {
-                            "startLine": 30,
-                            "endLine": 30,
-                            "startColumn": 9,
-                            "endColumn": 14
-                        }
-                    }
-                },
-                ...
-        ]}"""
-        return {
-            "issues": [issue.to_json() for issue in issues]
-        }
-
-    @prepare_file
-    def from_mypy(self, file: TextIO) -> List[Issue]:
-        buff = None
-        items = []
-
-        for line in file:
-            line = line.strip()
-            if RE_MYPY_LINE.match(line):
-                if buff:
-                    items.append(self.__parse_mypy_issue(buff))
-                buff = []
-            if buff is not None:
-                buff.append(line)
-        if buff is None:
-            # mypy can return an error without line/column values
-            file.seek(0)
-            for line in file:
-                m = RE_MYPY_LINE_WO_COORDINATES.match(line.strip())
-                if not m:
-                    continue
-                items.append(self.Issue(
-                    path=m.group(1).strip(),
-                    description=m.group(2).strip(),
-                    rule=self._mypy_rules["[unknown]"],
-                ))
-                self.logger.info(f"detected an error without coordinates: {line}")
-
-        items.append(self.__parse_mypy_issue(buff))
-        return [i for i in items if i]
-
-    @classmethod
-    def __parse_mypy_issue(cls, lines: List[str]) -> Optional[Issue]:
-        """"
-        An example of log response:
-            source_airtable/helpers.py:8:1: error: Library stubs not installed for
-            "requests" (or incompatible with Python 3.7)  [import]
-            import requests
-            ^
-            source_airtable/helpers.py:8:1: note: Hint: "python3 -m pip install types-requests"
-        """
-        if not lines:
-            return None
-        path, line_number, column_number, error_or_note, *others = " ".join(lines).split(":")
-        if "test" in Path(path).name:
-            cls.logger.info(f"skip the test file: {path}")
-            return None
-        if error_or_note.strip() == "note":
-            return None
-        others = ":".join(others)
-        rule = None
-        for code in cls._mypy_rules:
-            if code in others:
-                rule = cls._mypy_rules[code]
-                others = re.sub(r"\s+", " ", others.replace(code, ". Code line: "))
-                break
-        if not rule:
-            cls.logger.warning(f"couldn't find the  rule with '{others}' and lines: {lines}, available rules: {cls._mypy_rules}")
-            return None
-
-        description = others.split("^")[0]
-
-        return cls.Issue(
-            path=path.strip(),
-            line_number=int(line_number.strip()),
-            column_number=int(column_number.strip()),
-            description=description.strip(),
-            rule=rule,
-        )
-
-    @staticmethod
-    def __parse_diff(lines: List[str]) -> Mapping[str, int]:
-        """Converts diff lines to mapping:
-           {file1: , file2: }
-        """
-        patch = PatchSet(lines, metadata_only=True)
-        return {updated_file.path: len(updated_file) for updated_file in patch}
-
-    @prepare_file
-    def from_black(self, file: TextIO) -> List[Issue]:
-        return [self.Issue(
-            path=path,
-            description=f"{count} code part(s) should be updated.",
-            rule=self._black_rule,
-        ) for path, count in self.__parse_diff(file.readlines()).items()]
-
-    @prepare_file
-    def from_isort(self, file: TextIO) -> List[Issue]:
-        changes = defaultdict(lambda: 0)
-        for path, count in self.__parse_diff(file.readlines()).items():
-            # check path value
-            # path in isort diff file has the following format
-            # :before|after
-            if path.endswith(":before"):
-                path = path[:-len(":before")]
-            elif path.endswith(":after"):
-                path = path[:-len(":after")]
-            changes[path] += count
-
-        return [self.Issue(
-            path=path,
-            description=f"{count} code part(s) should be updated.",
-            rule=self._isort_rule,
-        ) for path, count in changes.items()]
diff --git a/tools/ci_code_validator/ci_sonar_qube/main.py b/tools/ci_code_validator/ci_sonar_qube/main.py
deleted file mode 100644
index 17c8370f1afd..000000000000
--- a/tools/ci_code_validator/ci_sonar_qube/main.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import argparse
-import sys
-
-from .log_parsers import LogParser
-from .sonar_qube_api import SonarQubeApi
-
-
-def main() -> int:
-    convert_key = len(set(["--mypy_log", "--black_diff", "--isort_diff"]) & set(sys.argv)) > 0
-    need_print_key = "--print_key" in sys.argv
-
-    parser = argparse.ArgumentParser(description='Working with SonarQube instance.')
-    parser.add_argument('--host', help='SonarQube host', required=not need_print_key, type=str)
-    parser.add_argument('--token', help='SonarQube token', required=not need_print_key, type=str)
-    parser.add_argument('--pr', help='PR unique name. Example: airbyte/1231', type=str, default=None)
-
-    name_value = parser.add_mutually_exclusive_group(required=not convert_key)
-    name_value.add_argument('--project', help='Name of future project', type=str)
-    name_value.add_argument('--module', help='Name of future module project', type=str)
-
-    command = parser.add_mutually_exclusive_group(required=not convert_key)
-    command.add_argument('--print_key', help='Return a generate SonarQube key', action="store_true")
-    command.add_argument('--report', help='generate .md file with current issues of a project')
-    command.add_argument('--create', help='create a project', action="store_true")
-    command.add_argument('--remove', help='remove project', action="store_true")
-
-    parser.add_argument('--mypy_log', help='Path to MyPy Logs', required=False, type=str)
-    parser.add_argument('--black_diff', help='Path to Black Diff', required=False, type=str)
-    parser.add_argument('--isort_diff', help='Path to iSort Diff', required=False, type=str)
-    parser.add_argument('--output_file', help='Path of output file', required=convert_key, type=str)
-
-    args = parser.parse_args()
-    if convert_key:
-        parser = LogParser(output_file=args.output_file, host=args.host, token=args.token)
-        if args.mypy_log:
-            return parser.from_mypy(args.mypy_log)
-        if args.black_diff:
-            return parser.from_black(args.black_diff)
-        if args.isort_diff:
-            return parser.from_isort(args.isort_diff)
-    api = SonarQubeApi(host=args.host, token=args.token, pr_name=args.pr)
-
-    project_name = api.module2project(args.module) if args.module else args.project
-
-    if args.create:
-        return 0 if api.create_project(project_name=project_name) else 1
-    elif args.remove:
-        return 0 if api.remove_project(project_name=project_name) else 1
-    elif args.print_key:
-        data = api.prepare_project_settings(project_name)
-        print(data["project"], file=sys.stdout)
-        return 0
-    elif args.report:
-        return 0 if api.generate_report(project_name=project_name, report_file=args.report) else 1
-    api.logger.critical("not set any action...")
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main())
diff --git a/tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py b/tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py
deleted file mode 100644
index bcbb42dd371e..000000000000
--- a/tools/ci_code_validator/ci_sonar_qube/sonar_qube_api.py
+++ /dev/null
@@ -1,348 +0,0 @@
-import itertools
-import re
-from functools import reduce
-from typing import Mapping, Any, Optional, List
-from urllib.parse import urljoin
-
-import requests
-from mdutils.mdutils import MdUtils
-from requests.auth import HTTPBasicAuth
-
-from ci_common_utils import Logger
-
-AIRBYTE_PROJECT_PREFIX = "airbyte"
-RE_RULE_NAME = re.compile(r"(.+):[A-Za-z]+(\d+)")
-
-REPORT_METRICS = (
-    "alert_status",
-    # "quality_gate_details",
-    "bugs", "new_bugs",
-    "reliability_rating", "new_reliability_rating",
-    "vulnerabilities", "new_vulnerabilities",
-    "security_rating", "new_security_rating",
-    # "security_hotspots", "new_security_hotspots",
-    # "security_hotspots_reviewed", "new_security_hotspots_reviewed",
-    # "security_review_rating", "new_security_review_rating",
-    "code_smells", "new_code_smells",
-    # "sqale_rating", "new_maintainability_rating",
-    # "sqale_index", "new_technical_debt",
-    "coverage", "new_coverage",
-    "lines_to_cover", "new_lines_to_cover",
-    "tests",
-    "duplicated_lines_density", "new_duplicated_lines_density",
-    "duplicated_blocks",
-    "ncloc",
-    # "ncloc_language_distribution",
-    # "projects",
-    # "lines", "new_lines"
-)
-
-RATINGS = {
-    1.0: "A",
-    2.0: "B",
-    3.0: "C",
-    4.0: "D",
-    5.0: "F",
-}
-
-
-class SonarQubeApi:
-    """https://sonarcloud.io/web_api"""
-    logger = Logger()
-
-    def __init__(self, host: str, token: str, pr_name: str):
-
-        self._host = host
-        self._token = token
-
-        # split the latest name part
-        self._pr_id = (pr_name or '').split("/")[-1]
-        if not self._pr_id.isdigit():
-            self.logger.critical(f"PR id should be integer. Current value: {pr_name}")
-
-        self._pr_id = int(self._pr_id)
-        # check token
-        # https://sonarcloud.io/web_api/api/authentication/validate
-        if not self._host:
-            return
-        resp = self._get("authentication/validate")
-        if not resp["valid"]:
-            self.logger.critical("provided token is not valid")
-
-    @property
-    def __auth(self):
-        return HTTPBasicAuth(self._token, '')
-
-    def __parse_response(self, url: str, response: requests.Response) -> Mapping[str, Any]:
-        if response.status_code == 204:
-            # empty response
-            return {}
-        elif response.status_code != 200:
-            self.logger.critical(f"API error for {url}: [{response.status_code}] {response.json()['errors']}")
-        return response.json()
-
-    def generate_url(self, endpoint: str) -> str:
-        return reduce(urljoin, [self._host, "/api/", endpoint])
-
-    def _post(self, endpoint: str, json: Mapping[str, Any]) -> Mapping[str, Any]:
-        url = self.generate_url(endpoint)
-        return self.__parse_response(url, requests.post(url, auth=self.__auth, params=json, json=json))
-
-    def _get(self, endpoint: str) -> Mapping[str, Any]:
-        url = self.generate_url(endpoint)
-        return self.__parse_response(url, requests.get(url, auth=self.__auth))
-
-    def _get_list(self, endpoint: str, list_name: str) -> List[Mapping[str, Any]]:
-
-        page = 0
-        items = []
-        while True:
-            page += 1
-            url = endpoint + "&" if "?" in endpoint else "?" + f"p={page}"
-            data = self._get(url)
-            items += data[list_name]
-            total = data.get("total") or data.get("paging", {}).get("total", 0)
-            if len(items) >= total:
-                break
-        return items
-
-    @classmethod
-    def module2project(cls, module_name: str) -> str:
-        """"""
-        parts = module_name.split("/")
-        if len(parts) != 2:
-            cls.logger.critical("module name must have the format: component/module")
-        return f"{AIRBYTE_PROJECT_PREFIX}:{parts[0].lower()}:{parts[1].lower().replace('_', '-')}"
-
-    def __correct_project_name(self, project_name: str) -> str:
-        return f"pr:{self._pr_id}:{project_name}" if self._pr_id else f"master:{project_name}"
-
-    def __search_project(self, project_name: str) -> Optional[Mapping[str, Any]]:
-        """https://sonarcloud.io/web_api/api/projects/search"""
-        data = self._get(f"projects/search?q={project_name}")
-        exists_projects = data["components"]
-        if len(exists_projects) > 1:
-            self.logger.critical(f"there are several projects with the name '{project_name}'")
-        elif len(exists_projects) == 0:
-            return None
-        return exists_projects[0]
-
-    def prepare_project_settings(self, project_name: str) -> Mapping[str, str]:
-        title = re.sub('[:_-]', ' ', project_name).replace("connectors_", "").title()
-        if self._pr_id:
-            title += f"(#{self._pr_id})"
-
-        project_name = self.__correct_project_name(project_name)
-        return {
-            "name": title,
-            "project": project_name,
-            "visibility": "private",
-        }
-
-    def create_project(self, project_name: str) -> bool:
-        """https://sonarcloud.io/web_api/api/projects/create"""
-        data = self.prepare_project_settings(project_name)
-        project_name = data["project"]
-        exists_project = self.__search_project(project_name)
-        if exists_project:
-            self.logger.info(f"The project '{project_name}' was created before")
-            return True
-
-        self._post("projects/create", data)
-        self.logger.info(f"The project '{project_name}' was created")
-        return True
-
-    def remove_project(self, project_name: str) -> bool:
-        """https://sonarcloud.io/web_api/api/projects/delete"""
-        project_name = self.prepare_project_settings(project_name)["project"]
-
-        exists_project = self.__search_project(project_name)
-        if exists_project is None:
-            self.logger.info(f"not found the project '{project_name}'")
-            return True
-        body = {
-            "project": project_name
-        }
-        self._post("projects/delete", body)
-        self.logger.info(f"The project '{project_name}' was removed")
-        return True
-
-    def generate_report(self, project_name: str, report_file: str) -> bool:
-        project_data = self.prepare_project_settings(project_name)
-
-        md_file = MdUtils(file_name=report_file)
-        md_file.new_line("
      SonarQube Report ")
-        md_file.new_line("
    ")
-        md_file.new_line("")
-        md_file.new_line(f'### SonarQube report for {project_data["name"]}')
-
-        project_name = project_data["project"]
-
-        issues = self._get_list(f"issues/search?componentKeys={project_name}&additionalFields=_all", "issues")
-        rules = {}
-        for rule_key in set(issue["rule"] for issue in issues):
-            key_parts = rule_key.split(":")
-            while len(key_parts) > 2:
-                key_parts.pop(0)
-            key = ":".join(key_parts)
-
-            data = self._get(f"rules/search?rule_key={key}")["rules"]
-            if not data:
-                data = self._get(f"rules/show?key={rule_key}")["rule"]
-            else:
-                data = data[0]
-
-            description = data["name"]
-            public_name = key
-            link = None
-            if rule_key.startswith("external_"):
-                public_name = key.replace("external_", "")
-                if not data["isExternal"]:
-                    # this is custom rule
-                    description = data["htmlDesc"]
-                if public_name.startswith("flake"):
-                    # single link for all descriptions
-                    link = "https://flake8.pycqa.org/en/latest/user/error-codes.html"
-                elif "isort_" in public_name:
-                    link = "https://pycqa.github.io/isort/index.html"
-                elif "black_" in public_name:
-                    link = "https://black.readthedocs.io/en/stable/the_black_code_style/index.html"
-            else:
-                # link's example
-                # https://rules.sonarsource.com/python/RSPEC-6287
-                m = RE_RULE_NAME.match(public_name)
-                if not m:
-                    # for local server
-                    link = f"{self._host}coding_rules?open={key}&rule_key={key}"
-                else:
-                    # to public SQ docs
-                    link = f"https://rules.sonarsource.com/{m.group(1)}/RSPEC-{m.group(2)}"
-            if link:
-                public_name = md_file.new_inline_link(
-                    link=link,
-                    text=public_name
-                )
-
-            rules[rule_key] = (public_name, description)
-
-        data = self._get(f"measures/component?component={project_name}&additionalFields=metrics&metricKeys={','.join(REPORT_METRICS)}")
-        measures = {}
-        total_coverage = None
-        for measure in data["component"]["measures"]:
-            metric = measure["metric"]
-            if measure["metric"].startswith("new_") and measure.get("periods"):
-                # we need to show values for last sync period only
-                last_period = max(measure["periods"], key=lambda period: period["index"])
-                value = last_period["value"]
-            else:
-                value = measure.get("value")
-            measures[metric] = value
-        # group overall and latest values
-        measures = {metric: (value, measures.get(f"new_{metric}")) for metric, value in measures.items() if
-                    not metric.startswith("new_")}
-        metrics = {}
-        for metric in data["metrics"]:
-            # if metric["key"] not in measures:
-            #     continue
-            metrics[metric["key"]] = (metric["name"], metric["type"])
-
-        md_file.new_line('#### Measures')
-
-        values = []
-        for metric, (overall_value, latest_value) in measures.items():
-            if metric not in metrics:
-                continue
-            name, metric_type = metrics[metric]
-            value = overall_value if (latest_value is None or latest_value == "0") else latest_value
-
-            if metric_type == "PERCENT":
-                value = str(round(float(value), 1))
-            elif metric_type == "INT":
-                value = int(float(value))
-            elif metric_type == "LEVEL":
-                pass
-            elif metric_type == "RATING":
-                value = int(float(value))
-                for k, v in RATINGS.items():
-                    if value <= k:
-                        value = v
-                        break
-            if metric == "coverage":
-                total_coverage = value
-            values.append([name, value])
-
-        values += [
-            ("Blocker Issues", sum(map(lambda i: i["severity"] == "BLOCKER", issues))),
-            ("Critical Issues", sum(map(lambda i: i["severity"] == "CRITICAL", issues))),
-            ("Major Issues", sum(map(lambda i: i["severity"] == "MAJOR", issues))),
-            ("Minor Issues", sum(map(lambda i: i["severity"] == "MINOR", issues))),
-        ]
-
-        while len(values) % 3:
-            values.append(("", ""))
-        table_items = ["Name", "Value"] * 3 + list(itertools.chain.from_iterable(values))
-        md_file.new_table(columns=6, rows=int(len(values) / 3 + 1), text=table_items, text_align='left')
-        md_file.new_line()
-        if issues:
-            md_file.new_line('#### Detected Issues')
-            table_items = [
-                "Rule", "File", "Description", "Message"
-            ]
-            for issue in issues:
-                rule_name, description = rules[issue["rule"]]
-                path = issue["component"].split(":")[-1].split("/")
-                # need to show only 2 last path parts
-                while len(path) > 2:
-                    path.pop(0)
-                path = "/".join(path)
-
-                # add line number in the end
-                if issue.get("line"):
-                    path += f':{issue["line"]}'
-                table_items += [
-                    f'{rule_name} ({issue["severity"]})',
-                    path,
-                    description,
-                    issue["message"],
-                ]
-
-            md_file.new_table(columns=4, rows=len(issues) + 1, text=table_items, text_align='left')
-        coverage_files = [(k, v) for k, v in self.load_coverage_component(project_name).items()]
-        if total_coverage is not None:
-            md_file.new_line(f'#### Coverage ({total_coverage}%)')
-            while len(coverage_files) % 2:
-                coverage_files.append(("", ""))
-            table_items = ["File", "Coverage"] * 2 + list(itertools.chain.from_iterable(coverage_files))
-            md_file.new_table(columns=4, rows=int(len(coverage_files) / 2 + 1), text=table_items, text_align='left')
-        md_file.new_line("")
-        md_file.new_line("
    ")
-        md_file.new_line("
    ")
-        md_file.create_md_file()
-        self.logger.info(f"The {report_file} was generated")
-        return True
-
-    def load_coverage_component(self, base_component: str, dir_path: str = None) -> Mapping[str, Any]:
-
-        page = 0
-        coverage_files = {}
-        read_count = 0
-        while True:
-            page += 1
-            component = base_component
-            if dir_path:
-                component += f":{dir_path}"
-            url = f"measures/component_tree?p={page}&component={component}&additionalFields=metrics&metricKeys=coverage,uncovered_lines,uncovered_conditions&strategy=children"
-            data = self._get(url)
-            read_count += len(data["components"])
-            for component in data["components"]:
-                if component["qualifier"] == "DIR":
-                    coverage_files.update(self.load_coverage_component(base_component, component["path"]))
-                    continue
-                elif not component["measures"]:
-                    continue
-                elif component["qualifier"] == "FIL":
-                    coverage_files[component["path"]] = [m["value"] for m in component["measures"] if m["metric"] == "coverage"][0]
-            if data["paging"]["total"] <= read_count:
-                break
-
-        return coverage_files
diff --git a/tools/ci_code_validator/setup.py b/tools/ci_code_validator/setup.py
deleted file mode 100644
index 62e2cd978846..000000000000
--- a/tools/ci_code_validator/setup.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
-
-
-from setuptools import find_packages, setup
-
-MAIN_REQUIREMENTS = [
-    "requests",
-    "ci_common_utils",
-    "unidiff",
-    "mdutils~=1.3.1",
-    "mypy==0.930",
-]
-
-TEST_REQUIREMENTS = ["requests-mock", "pytest", "black", "lxml", "isort"]
-
-setup(
-    version="0.0.0",
-    name="ci_code_validator",
-    description="Load and extract CI secrets for test suites",
-    author="Airbyte",
-    author_email="contact@airbyte.io",
-    packages=find_packages(),
-    install_requires=MAIN_REQUIREMENTS,
-    python_requires=">=3.9",
-    extras_require={
-        "tests": TEST_REQUIREMENTS,
-    },
-    entry_points={
-        "console_scripts": [
-            "ci_sonar_qube = ci_sonar_qube.main:main",
-            "ci_changes_detection = ci_changes_detection.main:main",
-        ],
-    },
-)
diff --git a/tools/ci_code_validator/tests/simple_files/black_smell_package_report.json b/tools/ci_code_validator/tests/simple_files/black_smell_package_report.json
deleted file mode 100644
index f5ba0fa01f79..000000000000
--- a/tools/ci_code_validator/tests/simple_files/black_smell_package_report.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "issues": [
-    {
-      "engineId": "black",
-      "ruleId": "python:black_need_format",
-      "severity": "MINOR",
-      "type": "CODE_SMELL",
-      "primaryLocation": {
-        "message": "1 code part(s) should be updated.",
-        "filePath": "simple_smell_package/invalid_file.py"
-      }
-    }
-  ]
-}
diff --git a/tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json b/tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json
deleted file mode 100644
index 34c4c7c63f61..000000000000
--- a/tools/ci_code_validator/tests/simple_files/isort_smell_package_report.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "issues": [
-    {
-      "engineId": "isort",
-      "ruleId": "python:isort_need_format",
-      "severity": "MINOR",
-      "type": "CODE_SMELL",
-      "primaryLocation": {
-        "message": "1 code part(s) should be updated.",
-        "filePath": "simple_smell_package/invalid_file.py"
-      }
-    }
-  ]
-}
diff --git a/tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json b/tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json
deleted file mode 100644
index f9fda9491a3b..000000000000
--- a/tools/ci_code_validator/tests/simple_files/mypy_smell_package_report.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
-  "issues": [
-    {
-      "engineId": "mypy",
-      "primaryLocation": {
-        "filePath": "simple_smell_package/invalid_file.py",
-        "message": "Incompatible return value type (got \"int\", expected \"str\") . Code line: return 1000",
-        "textRange": {
-          "endColumn": 12,
-          "endLine": 11,
-          "startColumn": 11,
-          "startLine": 11
-        }
-      },
-      "ruleId": "python:mypy_return_value",
-      "severity": "MINOR",
-      "type": "CODE_SMELL"
-    },
-    {
-      "engineId": "mypy",
-      "primaryLocation": {
-        "filePath": "simple_smell_package/invalid_file.py",
-        "message": "Name \"fake_func\" already defined on line 10 . Code line: def fake_func(i):",
-        "textRange": {
-          "endColumn": 1,
-          "endLine": 14,
-          "startColumn": 0,
-          "startLine": 14
-        }
-      },
-      "ruleId": "python:mypy_no_redef",
-      "severity": "MINOR",
-      "type": "CODE_SMELL"
-    }
-  ]
-}
\ No newline at end of file
diff --git a/tools/ci_code_validator/tests/simple_files/without_issues_report.json b/tools/ci_code_validator/tests/simple_files/without_issues_report.json
deleted file mode 100644
index 2eb23288a67d..000000000000
--- a/tools/ci_code_validator/tests/simple_files/without_issues_report.json
+++ /dev/null
@@ -1 +0,0 @@
-{"issues": []}
\ No newline at end of file
diff --git a/tools/ci_code_validator/tests/simple_package/valid_file.py b/tools/ci_code_validator/tests/simple_package/valid_file.py
deleted file mode 100644
index 5ad71b1c936f..000000000000
--- a/tools/ci_code_validator/tests/simple_package/valid_file.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# don't valid it by auto-linters becaise this file is used for testing
-import os
-from pathlib import Path
-
-LONG_STRING = """aaaaaaaaaaaaaaaa"""
-
-
-def func() -> bool:
-    return Path(os.getcwd()).is_dir() is True
-
-
-def func2(i: int) -> int:
-    return i * 10
diff --git a/tools/ci_code_validator/tests/simple_smell_package/invalid_file.py b/tools/ci_code_validator/tests/simple_smell_package/invalid_file.py
deleted file mode 100644
index b99a26f14856..000000000000
--- a/tools/ci_code_validator/tests/simple_smell_package/invalid_file.py
+++ /dev/null
@@ -1,15 +0,0 @@
-# don't valid it by auto-linters because this file is used for testing
-import pathlib
-import os
-
-
-
-LONG_STRING = """aaaaaaaaaaaaaaaaaaaaaaaaaawwwwwwwwwwwwwwwwwwwwwwwww mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm mmmmmmmmmmmmmm      wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww"""
-
-
-def fake_func() -> str:
-    return 1000
-
-
-def fake_func(i):
-   return i * 10
diff --git a/tools/ci_code_validator/tests/test_detect_changed_modules.py b/tools/ci_code_validator/tests/test_detect_changed_modules.py
deleted file mode 100644
index cd1956fc17fd..000000000000
--- a/tools/ci_code_validator/tests/test_detect_changed_modules.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#
-# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
-#
-from typing import List, Set
-
-import pytest
-from ci_changes_detection.main import list_changed_modules
-from ci_sonar_qube import ROOT_DIR
-
-
-@pytest.mark.parametrize(
-    "changed_files,changed_modules",
-    [
-        (["path/to/file1", "file2.txt", "path/to/file3.txt"], []),
-        (
-                [
-                    "airbyte-integrations/connectors/source-asana/source_asana/streams.py",
-                    "airbyte-integrations/connectors/source-asana/source_asana/source.py",
-                    "airbyte-integrations/connectors/source-braintree/integration_tests/abnormal_state.json",
-                ],
-                [
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-asana"), "lang": "py",
-                     "module": "connectors/source-asana"},
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-braintree"), "lang": "py",
-                     "module": "connectors/source-braintree"},
-                ],
-        ),
-        (
-                [
-                    "airbyte-integrations/connectors/destination-mongodb/build.gradle",
-                    "airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java",
-                    "airbyte-integrations/connectors/destination-s3/Dockerfile",
-                ],
-                [
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-mongodb"), "lang": "java",
-                     "module": "connectors/destination-mongodb"},
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-s3"), "lang": "java",
-                     "module": "connectors/destination-s3"},
-                ],
-        ),
-        (
-                [
-                    "airbyte-integrations/connectors/source-s3/Dockerfile",
-                    "airbyte-integrations/connectors/destination-s3/Dockerfile",
-                    "tools/ci_code_validator"
-                ],
-                [
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-s3"), "lang": "py",
-                     "module": "connectors/source-s3"},
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-s3"), "lang": "java",
-                     "module": "connectors/destination-s3"},
-                ],
-        ),
-        (
-                [
-                    "airbyte-integrations/connectors/source-s3/Dockerfile",
-                    "airbyte-integrations/connectors/destination-s3/Dockerfile",
-                    "tools/ci_code_validator"
-                ],
-                [
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/source-s3"), "lang": "py",
-                     "module": "connectors/source-s3"},
-                    {"folder": str(ROOT_DIR / "airbyte-integrations/connectors/destination-s3"), "lang": "java",
-                     "module": "connectors/destination-s3"},
-                ],
-        ),
-
-    ],
-    ids=["incorrect_files", "py_modules_only", "java_modules_only", "mix_modules", "absolute_paths"],
-)
-def test_list_changed_modules(changed_files: List[str], changed_modules: Set[str]) -> None:
-    calculated_changed_modules = list_changed_modules(changed_files)
-
-    assert calculated_changed_modules == changed_modules
diff --git a/tools/ci_code_validator/tests/test_sq_project.py b/tools/ci_code_validator/tests/test_sq_project.py
deleted file mode 100644
index 4fb0ea1b3814..000000000000
--- a/tools/ci_code_validator/tests/test_sq_project.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import pytest
-import requests_mock
-from ci_sonar_qube.sonar_qube_api import SonarQubeApi
-
-
-@pytest.mark.parametrize(
-    "module_name,pr, expected_title, expected_key",
-    [
-        ("connectors/source-s3", "airbyte/1234", "Airbyte Connectors Source S3(#1234)", "pr:1234:airbyte:connectors:source-s3"),
-        ("tools/ci_code_validator", "airbyte/1111", "Airbyte Tools Ci Code Validator(#1111)", "pr:1111:airbyte:tools:ci-code-validator"),
-        ("airbyte-cdk/python", "0", "Airbyte Airbyte Cdk Python", "master:airbyte:airbyte-cdk:python"),
-    ]
-)
-def test_module2project(module_name, pr, expected_title, expected_key):
-    with requests_mock.Mocker() as m:
-        m.get('/api/authentication/validate', json={"valid": True})
-        api = SonarQubeApi(host="http://fake.com/", token="", pr_name=pr)
-        project_settings = api.prepare_project_settings(api.module2project(module_name))
-    assert project_settings["name"] == expected_title
-    assert project_settings["project"] == expected_key
diff --git a/tools/ci_code_validator/tests/test_tools.py b/tools/ci_code_validator/tests/test_tools.py
deleted file mode 100644
index 05df61495788..000000000000
--- a/tools/ci_code_validator/tests/test_tools.py
+++ /dev/null
@@ -1,102 +0,0 @@
-import json
-import os
-import shutil
-import subprocess
-from pathlib import Path
-
-import pytest
-import requests_mock
-
-from ci_code_validator.ci_sonar_qube.log_parsers import LogParser
-
-HERE = Path(__file__).parent
-PACKAGE_DIR = HERE / "simple_package"
-SMELL_PACKAGE_DIR = HERE / "simple_smell_package"
-SIMPLE_FILES = HERE / "simple_files"
-WITHOUT_ISSUE_REPORT = SIMPLE_FILES / "without_issues_report.json"
-
-ISORT_CMD = """isort --diff {package_dir}"""  # config file should be in a started folder
-BLACK_CMD = r"""black --config {toml_config_file} --diff {package_dir}"""
-MYPY_CMD = r"""mypy {package_dir} --config-file={toml_config_file}"""
-
-
-@pytest.fixture(scope="session")
-def toml_config_file() -> Path:
-    root_dir = HERE
-    while str(root_dir) != root_dir.root:
-        config_file = root_dir / "pyproject.toml"
-        if config_file.is_file():
-            return config_file
-        root_dir = root_dir.parent
-    raise Exception("can't found pyproject.toml")
-
-
-@pytest.fixture(autouse=True)
-def prepare_toml_file(toml_config_file):
-    pyproject_toml = Path(os.getcwd()) / "pyproject.toml"
-    if toml_config_file != pyproject_toml and not pyproject_toml.is_file():
-        shutil.copy(toml_config_file, pyproject_toml)
-    yield
-    if toml_config_file != pyproject_toml and pyproject_toml.is_file():
-        os.remove(str(pyproject_toml))
-
-
-@pytest.mark.parametrize(
-    "cmd,package_dir,expected_file",
-    [
-        (
-                "mypy {package_dir} --config-file={toml_config_file}",
-                SMELL_PACKAGE_DIR,
-                SIMPLE_FILES / "mypy_smell_package_report.json"
-        ),
-        (
-                "mypy {package_dir} --config-file={toml_config_file}",
-                PACKAGE_DIR,
-                WITHOUT_ISSUE_REPORT
-        ),
-        (
-                "black --config {toml_config_file} --diff {package_dir}",
-                SMELL_PACKAGE_DIR,
-                HERE / "simple_files/black_smell_package_report.json"
-        ),
-        (
-                "black --config {toml_config_file} --diff {package_dir}",
-                PACKAGE_DIR,
-                WITHOUT_ISSUE_REPORT
-        ),
-        (
-                ISORT_CMD,
-                SMELL_PACKAGE_DIR,
-                HERE / "simple_files/isort_smell_package_report.json"
-        ),
-        (
-                ISORT_CMD,
-                PACKAGE_DIR,
-                WITHOUT_ISSUE_REPORT,
-        ),
-    ],
-    ids=["mypy_failed", "mypy_pass", "black_failed", "black_pass", "isort_failed", "isort_pass"]
-)
-def test_tool(tmp_path, toml_config_file, cmd, package_dir, expected_file):
-    cmd = cmd.format(package_dir=package_dir, toml_config_file=toml_config_file)
-
-    proc = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, _ = proc.communicate()
-    file_log = tmp_path / "temp.log"
-    file_log.write_bytes(out)
-    assert file_log.is_file() is True
-    issues_file = tmp_path / "issues.json"
-    with requests_mock.Mocker() as m:
-        m.get('/api/authentication/validate', json={"valid": True})
-        m.get("/api/rules/search", json={"rules": []})
-        m.post("/api/rules/create", json={})
-        parser = LogParser(issues_file, host="http://fake.com/", token="fake_token")
-        assert getattr(parser, f'from_{cmd.split(" ")[0]}')(file_log) == 0
-
-        assert issues_file.is_file() is True
-        data = json.loads(issues_file.read_text())
-        for issue in data["issues"]:
-            issue["primaryLocation"]["filePath"] = "/".join(issue["primaryLocation"]["filePath"].split("/")[-2:])
-
-        expected_data = json.loads(Path(expected_file).read_text())
-        assert json.dumps(data, sort_keys=True, separators=(',', ': ')) == json.dumps(expected_data, sort_keys=True, separators=(',', ': '))
diff --git a/tools/ci_common_utils/setup.py b/tools/ci_common_utils/setup.py
deleted file mode 100644
index d0e178981fbc..000000000000
--- a/tools/ci_common_utils/setup.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-from setuptools import find_packages, setup
-
-MAIN_REQUIREMENTS = ["cryptography", "requests", "pyjwt~=2.6.0"]
-
-TEST_REQUIREMENTS = ["requests-mock"]
-
-setup(
-    version="0.0.0",
-    name="ci_common_utils",
-    description="Suite of all often used classes and common functions",
-    author="Airbyte",
-    author_email="contact@airbyte.io",
-    packages=find_packages(),
-    install_requires=MAIN_REQUIREMENTS,
-    python_requires=">=3.9",
-    extras_require={
-        "tests": TEST_REQUIREMENTS,
-    },
-)
diff --git a/tools/ci_common_utils/tests/test_logger.py b/tools/ci_common_utils/tests/test_logger.py
deleted file mode 100644
index e45464550edf..000000000000
--- a/tools/ci_common_utils/tests/test_logger.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import re
-from datetime import datetime, timedelta
-
-import pytest
-
-from ci_common_utils import Logger
-
-LOG_RE = re.compile(
-    r'^\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\.\d{6})\] -'
-    r'\s+(\w+)\s+- \[.*tests/test_logger.py:(\d+)\] # (.+)')
-LOGGER = Logger()
-TEST_MESSAGE = 'sbhY=)9\'v-}LT=)jjF66(XrZh=]>7Xp"?/zCz,=eu8K47u8'
-
-
-def check_output(msg: str, expected_line_number: int, expected_log_level: str):
-    m = LOG_RE.match(msg)
-    assert m is not None, f"incorrect message format, pattern: {LOG_RE.pattern}"
-    date_time, log_level, line_number, msg = m.groups()
-
-    assert int(line_number) == expected_line_number
-    assert expected_log_level == log_level
-    assert expected_log_level == log_level
-    dt = datetime.strptime(date_time, '%d/%m/%Y %H:%M:%S.%f')
-    now = datetime.now()
-    delta = timedelta(seconds=1)
-    assert now - delta < dt < now
-
-
-@pytest.mark.parametrize('log_func,expected_log_level,expected_code', (
-        (LOGGER.debug, 'DEBUG', 0),
-        (LOGGER.warning, 'WARNING', 0),
-        (LOGGER.info, 'INFO', 0),
-        (LOGGER.error, 'ERROR', 1)
-))
-def test_log_message(capfd, log_func, expected_log_level, expected_code):
-    assert log_func(TEST_MESSAGE) == expected_code
-    _, err = capfd.readouterr()
-    check_output(err, 36, expected_log_level)
-
-
-def test_critical_message(capfd):
-    with pytest.raises(SystemExit) as (err):
-        LOGGER.critical(TEST_MESSAGE)
-    _, err = capfd.readouterr()
-    check_output(err, 43, 'CRITICAL')
diff --git a/tools/ci_connector_ops/README.md b/tools/ci_connector_ops/README.md
deleted file mode 100644
index f2dfd02abead..000000000000
--- a/tools/ci_connector_ops/README.md
+++ /dev/null
@@ -1,35 +0,0 @@
-# CI_CONNECTOR_OPS
-
-A collection of tools and checks run by Github Actions
-
-## Running Locally
-
-From this directory, create a virtual environment:
-
-```
-python3 -m venv .venv
-```
-
-This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your
-development environment of choice. To activate it from the terminal, run:
-
-```bash
-source .venv/bin/activate
-pip install -e . # assuming you are in the ./tools/ci_connector_ops directory
-```
-
-pip will make binaries for all the commands in setup.py, so you can run `allowed-hosts-checks` directly from the virtual-env
-
-## Testing Locally
-
-To install requirements to run unit tests, use:
-
-```
-pip install -e ".[tests]"
-```
-
-Unit tests are currently configured to be run from the base `airbyte` directory. You can run the tests from that directory with the following command:
-
-```
-pytest -s tools/ci_connector_ops/tests
-```
\ No newline at end of file
diff --git a/tools/ci_connector_ops/ci_connector_ops/acceptance_test_config_checks.py b/tools/ci_connector_ops/ci_connector_ops/acceptance_test_config_checks.py
deleted file mode 100644
index cd9de2a63ce5..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/acceptance_test_config_checks.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import logging
-import sys
-from typing import List, Dict, Union, Set
-import yaml
-
-from ci_connector_ops import utils
-
-RELEASE_STAGE_TO_STRICTNESS_LEVEL_MAPPING = {"generally_available": "high"}
-BACKWARD_COMPATIBILITY_REVIEWERS = {"connector-operations", "connector-extensibility"}
-TEST_STRICTNESS_LEVEL_REVIEWERS = {"connector-operations"}
-GA_BYPASS_REASON_REVIEWERS = {"connector-operations"}
-GA_CONNECTOR_REVIEWERS = {"gl-python"}
-REVIEW_REQUIREMENTS_FILE_PATH = ".github/connector_org_review_requirements.yaml"
-
-
-def find_connectors_with_bad_strictness_level() -> List[utils.Connector]:
-    """Check if changed connectors have the expected connector acceptance test strictness level according to their release stage.
-    1. Identify changed connectors
-    2. Retrieve their release stage from the catalog
-    3. Parse their acceptance test config file
-    4. Check if the test strictness level matches the strictness level expected for their release stage.
-
-    Returns:
-        List[utils.Connector]: List of changed connector that are not matching test strictness level expectations.
-    """
-    connectors_with_bad_strictness_level = []
-    changed_connector = utils.get_changed_connectors()
-    for connector in changed_connector:
-        expected_test_strictness_level = RELEASE_STAGE_TO_STRICTNESS_LEVEL_MAPPING.get(connector.release_stage)
-        can_check_strictness_level = all(
-            [item is not None for item in [connector.release_stage, expected_test_strictness_level, connector.acceptance_test_config]]
-        )
-        if can_check_strictness_level:
-            try:
-                assert connector.acceptance_test_config.get("test_strictness_level") == expected_test_strictness_level
-            except AssertionError:
-                connectors_with_bad_strictness_level.append(connector)
-    return connectors_with_bad_strictness_level
-
-
-def find_changed_ga_connectors() -> Set[utils.Connector]:
-    """Find GA connectors modified on the current branch.
-
-    Returns:
-        Set[utils.Connector]: The set of GA connectors that were modified on the current branch.
-    """
-    changed_connectors = utils.get_changed_connectors()
-    return {connector for connector in changed_connectors if connector.release_stage == "generally_available"}
-
-
-def get_ga_bypass_reason_changes() -> Set[utils.Connector]:
-    """Find GA connectors that have modified bypass_reasons.
-
-    Returns:
-        Set[str]: Set of connector names e.g {"source-github"}: The set of GA connectors that have changed bypass_reasons.
-    """
-    bypass_reason_changes = utils.get_changed_acceptance_test_config(diff_regex="bypass_reason")
-    return bypass_reason_changes.intersection(find_changed_ga_connectors())
-
-
-def find_mandatory_reviewers() -> List[Union[str, Dict[str, List]]]:
-    ga_connector_changes = find_changed_ga_connectors()
-    backward_compatibility_changes = utils.get_changed_acceptance_test_config(diff_regex="disable_for_version")
-    test_strictness_level_changes = utils.get_changed_acceptance_test_config(diff_regex="test_strictness_level")
-    ga_bypass_reason_changes = get_ga_bypass_reason_changes()
-
-    if backward_compatibility_changes:
-        return [{"any-of": list(BACKWARD_COMPATIBILITY_REVIEWERS)}]
-    if test_strictness_level_changes:
-        return [{"any-of": list(TEST_STRICTNESS_LEVEL_REVIEWERS)}]
-    if ga_bypass_reason_changes:
-        return [{"any-of": list(GA_BYPASS_REASON_REVIEWERS)}]
-    if ga_connector_changes:
-        return list(GA_CONNECTOR_REVIEWERS)
-    return []
-
-
-def check_test_strictness_level():
-    connectors_with_bad_strictness_level = find_connectors_with_bad_strictness_level()
-    if connectors_with_bad_strictness_level:
-        logging.error(
-            f"The following GA connectors must enable high test strictness level: {connectors_with_bad_strictness_level}. Please check this documentation for details: https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/#strictness-level"
-        )
-        sys.exit(1)
-    else:
-        sys.exit(0)
-
-
-def write_review_requirements_file():
-    mandatory_reviewers = find_mandatory_reviewers()
-
-    if mandatory_reviewers:
-        requirements_file_content = [
-            {"name": "Required reviewers from the connector org teams", "paths": "unmatched", "teams": mandatory_reviewers}
-        ]
-        with open(REVIEW_REQUIREMENTS_FILE_PATH, "w") as requirements_file:
-            yaml.safe_dump(requirements_file_content, requirements_file)
-        print("CREATED_REQUIREMENTS_FILE=true")
-    else:
-        print("CREATED_REQUIREMENTS_FILE=false")
-
-
-def print_mandatory_reviewers():
-    teams = []
-    mandatory_reviewers = find_mandatory_reviewers()
-    for mandatory_reviewer in mandatory_reviewers:
-        if isinstance(mandatory_reviewer, dict):
-            teams += mandatory_reviewer["any-of"]
-        else:
-            teams.append(mandatory_reviewer)
-    print(f"MANDATORY_REVIEWERS=A review is required from these teams: {','.join(teams)}")
diff --git a/tools/ci_connector_ops/ci_connector_ops/allowed_hosts_checks.py b/tools/ci_connector_ops/ci_connector_ops/allowed_hosts_checks.py
deleted file mode 100644
index 2613bab0729b..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/allowed_hosts_checks.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import logging
-import sys
-from typing import List
-
-from ci_connector_ops import utils
-
-RELEASE_STAGES_TO_CHECK = ["generally_available", "beta"]
-
-
-def get_connectors_missing_allowed_hosts() -> List[utils.Connector]:
-    connectors_missing_allowed_hosts: List[utils.Connector] = []
-    changed_connectors = utils.get_changed_connectors()
-
-    for connector in changed_connectors:
-        if connector.release_stage in RELEASE_STAGES_TO_CHECK:
-            missing = not connector_has_allowed_hosts(connector)
-            if missing:
-                connectors_missing_allowed_hosts.append(connector)
-
-    return connectors_missing_allowed_hosts
-
-
-def connector_has_allowed_hosts(connector: utils.Connector) -> bool:
-    return connector.allowed_hosts is not None
-
-
-def check_allowed_hosts():
-    connectors_missing_allowed_hosts = get_connectors_missing_allowed_hosts()
-    if connectors_missing_allowed_hosts:
-        logging.error(f"The following {RELEASE_STAGES_TO_CHECK} connectors must include allowedHosts: {connectors_missing_allowed_hosts}")
-        sys.exit(1)
-    else:
-        sys.exit(0)
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/README.md b/tools/ci_connector_ops/ci_connector_ops/pipelines/README.md
deleted file mode 100644
index d678eff84bc9..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/README.md
+++ /dev/null
@@ -1,316 +0,0 @@
-# Airbyte CI CLI 
-
-## What is it?
-`airbyte-ci` is a command line interface to run CI/CD pipelines.
-The goal of this CLI is to offer developers a tool to run these pipelines locally and in a CI context with the same guarantee.
-It can prevent unnecessary commit -> push cycles developers typically go through when they when to test their changes against a remote CI.
-This is made possible thanks to the use of [Dagger](https://dagger.io), a CI/CD engine relying on Docker Buildkit to provide reproducible builds.
-Our pipeline are declared with Python code, the main entrypoint is [here](https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/airbyte_ci.py).
-This documentation should be helpful for both local and CI use of the CLI. We indeed [power connector testing in the CI with this CLI](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/connector_integration_test_single_dagger.yml#L78). 
-
-## How to install
-### Requirements
-* A running Docker engine
-* Python >= 3.10
-
-### Install
-```bash
-# Make sure that the current Python version is >= 3.10
-pyenv shell 3.10
-pip install "ci-connector-ops[pipelines] @ git+https://github.com/airbytehq/airbyte.git@master#subdirectory=tools/ci_connector_ops"
-cd airbyte
-airbyte-ci
-```
-
-If you face any installation problem feel free to reach out the Airbyte Connectors Operations team.
-N.B: This project will eventually be moved to `airbyte-ci` root directory.
-
-
-## Commands reference
-- [`airbyte-ci` command group](#airbyte-ci)
-  * [Options](#options)
-- [`connectors` command subgroup](#connectors-command-subgroup)
-  * [Options](#options-1)
-- [`connectors list` command](#connectors-list-command)
-- [`connectors test` command](#connectors-test-command)
-  * [Examples](#examples-)
-  * [What it runs](#what-it-runs-)
-- [`connectors build` command](#connectors-build-command)
-  * [What it runs](#what-it-runs)
-- [`connectors publish` command](#connectors-publish-command)
-- [Examples](#examples)
-- [Options](#options-2)
-  * [What it runs](#what-it-runs-1)
-- [`metadata` command subgroup](#metadata-command-subgroup)
-- [`metadata validate` command](#metadata-validate-command)
-  * [Example](#example)
-  * [Options](#options-3)
-- [`metadata upload` command](#metadata-upload-command)
-  * [Example](#example-1)
-  * [Options](#options-4)
-- [`metadata deploy orchestrator` command](#metadata-deploy-orchestrator-command)
-  * [Example](#example-2)
-  * [What it runs](#what-it-runs--1)
-- [`metadata test lib` command](#metadata-test-lib-command)
-  * [Example](#example-3)
-- [`metadata test orchestrator` command](#metadata-test-orchestrator-command)
-  * [Example](#example-4)
-
-### `airbyte-ci` command group
-**The main command group option has sensible defaults. In local use cases you're not likely to pass options to the `airbyte-ci` command group.**
-
-#### Options
-
-| Option                       | Default value                   | Mapped environment variable   | Description                                                                                 |
-| ---------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- |
-| `--is-local/--is-ci`         | `--is-local`                    |                               | Determines the environment in which the CLI runs: local environment or CI environment.      |
-| `--git-branch`               | The checked out git branch name | `CI_GIT_BRANCH`               | The git branch on which the pipelines will run.                                             |
-| `--git-revision`             | The current branch head         | `CI_GIT_REVISION`             | The commit hash on which the pipelines will run.                                            |
-| `--diffed-branch`            | `origin/master`                 |                               | Branch to which the git diff will happen to detect new or modified files.                   |
-| `--gha-workflow-run-id`      |                                 |                               | GHA CI only - The run id of the GitHub action workflow                                      |
-| `--ci-context`               | `manual`                        |                               | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
-| `--pipeline-start-timestamp` | Current epoch time              | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation.       |
-
-
-### `connectors` command subgroup
-
-Available commands:
-* `airbyte-ci connectors test`: Run tests for one or multiple connectors.
-* `airbyte-ci connectors build`: Build docker images for one or multiple connectors. 
-* `airbyte-ci connectors publish`: Publish a connector to Airbyte's DockerHub.
-
-#### Options
-| Option                 | Multiple | Default value | Description                                                                                                                                                                                                                                                                                           |
-| ---------------------- | -------- | ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--use-remote-secrets` | False    | True          | If True, connectors configuration will be pulled from Google Secret Manager. Requires the GCP_GSM_CREDENTIALS environment variable to be set with a service account with permission to read GSM secrets. If False the connector configuration will be read from the local connector `secrets` folder. |
-| `--name`               | True     |               | Select a specific connector for which the pipeline will run. Can be used multiple time to select multiple connectors. The expected name is the connector technical name. e.g. `source-pokeapi`                                                                                                        |
-| `--release-stage`      | True     |               | Select connectors with a specific release stage: `alpha`, `beta`, `generally_available`.  Can be used multiple times to select multiple release stages.                                                                                                                                               |
-| `--language`           | True     |               | Select connectors with a specific language: `python`, `low-code`, `java`. Can be used multiple times to select multiple languages.                                                                                                                                                                    |
-| `--modified`           | False    | False         | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation).                                                                                                                                                                           |
-| `--concurrency`        | False    | 5             | Control the number of connector pipelines that can run in parallel. Useful to speed up pipelines or control their resource usage.                                                                                                                                                                     |
-
-### `connectors list` command
-Retrieve the list of connectors satisfying the provided filters.
-
-#### Examples
-List all connectors:
-
-`airbyte-ci connectors list`
-
-List generally available connectors:
-
-`airbyte-ci connectors --release-stage=generally_available list`
-
-List connectors changed on the current branch:
-
-`airbyte-ci connectors --modified list`
-
-List connectors with a specific language:
-
-`airbyte-ci connectors --language=python list`
-
-List connectors with multiple filters:
-
-`airbyte-ci connectors --language=low-code --release-stage=generally_available list`
-
-### `connectors test` command
-Run a test pipeline for one or multiple connectors.
-
-#### Examples
-
-Test a single connector:
-`airbyte-ci connectors --name=source-pokeapi test`
-
-Test multiple connectors:
-`airbyte-ci connectors --name=source-pokeapi --name=source-bigquery test`
-
-Test generally available connectors:
-`airbyte-ci connectors --release-stage=generally_available test`
-
-Test connectors changed on the current branch:
-`airbyte-ci connectors --modified test`
-
-#### What it runs
-```mermaid
-flowchart TD
-    entrypoint[[For each selected connector]]
-    subgraph static ["Static code analysis"]
-      qa[Run QA checks]
-      fmt[Run code format checks]
-      sem["Check version follows semantic versionning"]
-      incr["Check version is incremented"]
-      metadata_validation["Run metadata validation on metadata.yaml"]
-      sem --> incr
-    end
-    subgraph tests ["Tests"]
-        build[Build connector docker image]
-        unit[Run unit tests]
-        integration[Run integration tests]
-        cat[Run connector acceptance tests]
-        secret[Load connector configuration]
-
-        unit-->secret
-        unit-->build
-        secret-->integration
-        secret-->cat
-        build-->integration
-        build-->cat
-    end
-    entrypoint-->static
-    entrypoint-->tests
-    report["Build test report"]
-    tests-->report
-    static-->report
-```
-
-### `connectors build` command
-Run a build pipeline for one or multiple connectors and export the built docker image to the local docker host.
-It's mainly purposed for local use.
-
-Build a single connector:
-`airbyte-ci connectors --name=source-pokeapi build`
-
-Build multiple connectors:
-`airbyte-ci connectors --name=source-pokeapi --name=source-bigquery build`
-
-Build generally available connectors:
-`airbyte-ci connectors --release-stage=generally_available build`
-
-Build connectors changed on the current branch:
-`airbyte-ci connectors --modified build`
-
-#### What it runs
-
-For Python and Low Code connectors:
-
-```mermaid
-flowchart TD
-    arch(For each platform amd64/arm64)
-    connector[Build connector image]
-    load[Load to docker host with :dev tag, current platform]
-    spec[Get spec]
-    arch-->connector-->spec--"if success"-->load
-```
-
-For Java connectors:
-```mermaid
-flowchart TD
-    arch(For each platform amd64/arm64)
-    distTar[Gradle distTar task run]
-    base[Build integration base]
-    java_base[Build integration base Java]
-    normalization[Build Normalization]
-    connector[Build connector image]
-    
-    arch-->base-->java_base-->connector
-    distTar-->connector
-    normalization--"if supports normalization"-->connector
-
-    load[Load to docker host with :dev tag, current platform]
-    spec[Get spec]
-    connector-->spec--"if success"-->load
-```
-
-### `connectors publish` command
-Run a publish pipeline for one or multiple connectors.
-It's mainly purposed for CI use to release a connector update.
-
-### Examples
-Publish all connectors modified in the head commit: `airbyte-ci connectors --modified publish`
-
-### Options
-
-| Option                               | Required | Default         | Mapped environment variable        | Description                                                                                                                                                                      |
-| ------------------------------------ | -------- | --------------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--pre-release/--main-release`       | False    | `--pre-release` |                                    | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. |
-| `--docker-hub-username`              | True     |                 | `DOCKER_HUB_USERNAME`              | Your username to connect to DockerHub.                                                                                                                                           |
-| `--docker-hub-password`              | True     |                 | `DOCKER_HUB_PASSWORD`              | Your password to connect to DockerHub.                                                                                                                                           |
-| `--spec-cache-gcs-credentials`       | False    |                 | `SPEC_CACHE_GCS_CREDENTIALS`       | The service account key to upload files to the GCS bucket hosting spec cache.                                                                                                    |
-| `--spec-cache-bucket-name`           | False    |                 | `SPEC_CACHE_BUCKET_NAME`           | The name of the GCS bucket where specs will be cached.                                                                                                                           |
-| `--metadata-service-gcs-credentials` | False    |                 | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files.                                                                                            |
-| `--metadata-service-bucket-name`     | False    |                 | `METADATA_SERVICE_BUCKET_NAME`     | The name of the GCS bucket where metadata files will be uploaded.                                                                                                                |
-| `--slack-webhook`                    | False    |                 | `SLACK_WEBHOOK`                    | The Slack webhook URL to send notifications to.                                                                                                                                  |
-| `--slack-channel`                    | False    |                 | `SLACK_CHANNEL`                    | The Slack channel name to send notifications to.                                                                                                                                 |
-
-I've added an empty "Default" column, and you can fill in the default values as needed.
-#### What it runs
-```mermaid
-flowchart TD
-    validate[Validate the metadata file]
-    check[Check if the connector image already exists]
-    build[Build the connector image for all platform variants]
-    upload_spec[Upload connector spec to the spec cache bucket]
-    push[Push the connector image from DockerHub, with platform variants]
-    pull[Pull the connector image from DockerHub to check SPEC can be run and the image layers are healthy]
-    upload_metadata[Upload its metadata file to the metadata service bucket]
-
-    validate-->check-->build-->upload_spec-->push-->pull-->upload_metadata
-```
-
-### `metadata` command subgroup
-
-Available commands:
-* `airbyte-ci metadata validate`
-* `airbyte-ci metadata upload`
-* `airbyte-ci metadata test lib`
-* `airbyte-ci metadata test orchestrator`
-* `airbyte-ci metadata deploy orchestrator`
-
-### `metadata validate` command
-This commands validates the modified `metadata.yaml` files in the head commit, or all the `metadata.yaml` files.
-
-#### Example
-Validate all `metadata.yaml` files in the repo:
-`airbyte-ci metadata validate --all`
-
-#### Options
-| Option             | Default      | Description                                                                                                                |
-| ------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------------- |
-| `--modified/--all` | `--modified` | Flag to run validation of `metadata.yaml` files on the modified files in the head commit or all the `metadata.yaml` files. |
-
-### `metadata upload` command
-This command upload the modified `metadata.yaml` files in the head commit, or all the `metadata.yaml` files, to a GCS bucket.
-
-#### Example
-Upload all the `metadata.yaml` files to a GCS bucket:
-`airbyte-ci metadata upload --all `
-
-#### Options
-| Option              | Required | Default      | Mapped environment variable | Description                                                                                                              |
-| ------------------- | -------- | ------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
-| `--gcs-credentials` | True     |              | `GCS_CREDENTIALS`           | Service account credentials in JSON format with permission to get and upload on the GCS bucket                           |
-| `--modified/--all`  | True     | `--modified` |                             | Flag to upload the modified `metadata.yaml` files in the head commit or all the  `metadata.yaml`  files to a GCS bucket. |
-
-### `metadata deploy orchestrator` command
-This command deploys the metadata service orchestrator to production.
-The `DAGSTER_CLOUD_METADATA_API_TOKEN` environment variable must be set.
-
-#### Example
-`airbyte-ci metadata deploy orchestrator`
-
-#### What it runs
-```mermaid
-flowchart TD
-    test[Run orchestrator tests] --> deploy[Deploy orchestrator to Dagster Cloud]
-```
-
-### `metadata test lib` command
-This command runs tests for the metadata service library.
-
-#### Example
-`airbyte-ci metadata test lib`
-
-### `metadata test orchestrator` command
-This command runs tests for the metadata service orchestrator.
-
-#### Example
-`airbyte-ci metadata test orchestrator`
-
-## Changelog 
-| Version | PR  | Description                                                                                |
-| ------- | --- | ------------------------------------------------------------------------------------------ |
-| 0.1.0   |     | Alpha version not in production yet. All the commands described in this doc are available. |
-
-## More info
-This project is owned by the Connectors Operations team.
-We share project updates and remaining stories before its release to production in this [EPIC](https://github.com/airbytehq/airbyte/issues/24403).
\ No newline at end of file
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/__init__.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/__init__.py
deleted file mode 100644
index 59e1ee767b75..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""The pipelines package."""
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/__init__.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/__init__.py
deleted file mode 100644
index f9e32611dfed..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/__init__.py
+++ /dev/null
@@ -1,61 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-"""The actions package is made to declare reusable pipeline components."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List, Tuple, Union
-
-import asyncer
-from ci_connector_ops.pipelines.bases import Step, StepStatus
-
-if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.bases import StepResult
-
-
-async def run_steps(
-    steps_and_run_args: List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]], results: List[StepResult] = []
-) -> List[StepResult]:
-    """Run multiple steps sequentially, or in parallel if steps are wrapped into a sublist.
-
-    Args:
-        steps_and_run_args (List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]]): List of steps to run, if steps are wrapped in a sublist they will be executed in parallel. run function arguments can be passed as a tuple along the Step instance.
-        results (List[StepResult], optional): List of step results, used for recursion.
-
-    Returns:
-        List[StepResult]: List of step results.
-    """
-    # If there are no steps to run, return the results
-    if not steps_and_run_args:
-        return results
-
-    # If any of the previous steps failed, skip the remaining steps
-    if any(result.status is StepStatus.FAILURE for result in results):
-        skipped_results = []
-        for step_and_run_args in steps_and_run_args:
-            if isinstance(step_and_run_args, Tuple):
-                skipped_results.append(step_and_run_args[0].skip())
-            else:
-                skipped_results.append(step_and_run_args.skip())
-        return results + skipped_results
-
-    # Pop the next step to run
-    steps_to_run, remaining_steps = steps_and_run_args[0], steps_and_run_args[1:]
-
-    # wrap the step in a list if it is not already (allows for parallel steps)
-    if not isinstance(steps_to_run, list):
-        steps_to_run = [steps_to_run]
-
-    async with asyncer.create_task_group() as task_group:
-        tasks = []
-        for step in steps_to_run:
-            if isinstance(step, Step):
-                tasks.append(task_group.soonify(step.run)())
-            elif isinstance(step, Tuple) and isinstance(step[0], Step) and isinstance(step[1], Tuple):
-                step, run_args = step
-                tasks.append(task_group.soonify(step.run)(*run_args))
-
-    new_results = [task.value for task in tasks]
-
-    return await run_steps(remaining_steps, results + new_results)
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
deleted file mode 100644
index 6c40766ca6d7..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py
+++ /dev/null
@@ -1,810 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This modules groups functions made to create reusable environments packaged in dagger containers."""
-
-from __future__ import annotations
-
-import importlib.util
-import uuid
-from typing import TYPE_CHECKING, List, Optional, Tuple
-
-from ci_connector_ops.pipelines import consts
-from ci_connector_ops.pipelines.consts import (
-    CI_CONNECTOR_OPS_SOURCE_PATH,
-    CI_CREDENTIALS_SOURCE_PATH,
-    CONNECTOR_TESTING_REQUIREMENTS,
-    DEFAULT_PYTHON_EXCLUDE,
-    PYPROJECT_TOML_FILE_PATH,
-)
-from ci_connector_ops.pipelines.utils import get_file_contents, slugify, with_exit_code
-from dagger import CacheSharingMode, CacheVolume, Container, Directory, File, Platform, Secret
-from dagger.engine._version import CLI_VERSION as dagger_engine_version
-
-if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.contexts import ConnectorContext, PipelineContext
-
-
-def with_python_base(context: PipelineContext, python_image_name: str = "python:3.9-slim") -> Container:
-    """Build a Python container with a cache volume for pip cache.
-
-    Args:
-        context (PipelineContext): The current test context, providing a dagger client and a repository directory.
-        python_image_name (str, optional): The python image to use to build the python base environment. Defaults to "python:3.9-slim".
-
-    Raises:
-        ValueError: Raised if the python_image_name is not a python image.
-
-    Returns:
-        Container: The python base environment container.
-    """
-    if not python_image_name.startswith("python:3"):
-        raise ValueError("You have to use a python image to build the python base environment")
-    pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache")
-
-    base_container = (
-        context.dagger_client.container()
-        .from_(python_image_name)
-        .with_mounted_cache("/root/.cache/pip", pip_cache)
-        .with_mounted_directory("/tools", context.get_repo_dir("tools", include=["ci_credentials", "ci_common_utils"]))
-        .with_exec(["pip", "install", "--upgrade", "pip"])
-    )
-
-    return base_container
-
-
-def with_testing_dependencies(context: PipelineContext) -> Container:
-    """Build a testing environment by installing testing dependencies on top of a python base environment.
-
-    Args:
-        context (PipelineContext): The current test context, providing a dagger client and a repository directory.
-
-    Returns:
-        Container: The testing environment container.
-    """
-    python_environment: Container = with_python_base(context)
-    pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH)
-    return python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS).with_file(
-        f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file
-    )
-
-
-def with_python_package(
-    context: PipelineContext,
-    python_environment: Container,
-    package_source_code_path: str,
-    exclude: Optional[List] = None,
-) -> Container:
-    """Load a python package source code to a python environment container.
-
-    Args:
-        context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled.
-        python_environment (Container): An existing python environment in which the package will be installed.
-        package_source_code_path (str): The local path to the package source code.
-        additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None.
-        exclude (Optional[List]): A list of file or directory to exclude from the python package source code.
-
-    Returns:
-        Container: A python environment container with the python package source code.
-    """
-    if exclude:
-        exclude = DEFAULT_PYTHON_EXCLUDE + exclude
-    else:
-        exclude = DEFAULT_PYTHON_EXCLUDE
-    package_source_code_directory: Directory = context.get_repo_dir(package_source_code_path, exclude=exclude)
-    container = python_environment.with_mounted_directory("/" + package_source_code_path, package_source_code_directory).with_workdir(
-        "/" + package_source_code_path
-    )
-    return container
-
-
-async def with_installed_python_package(
-    context: PipelineContext,
-    python_environment: Container,
-    package_source_code_path: str,
-    additional_dependency_groups: Optional[List] = None,
-    exclude: Optional[List] = None,
-) -> Container:
-    """Install a python package in a python environment container.
-
-    Args:
-        context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled.
-        python_environment (Container): An existing python environment in which the package will be installed.
-        package_source_code_path (str): The local path to the package source code.
-        additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None.
-        exclude (Optional[List]): A list of file or directory to exclude from the python package source code.
-
-    Returns:
-        Container: A python environment container with the python package installed.
-    """
-    install_local_requirements_cmd = ["python", "-m", "pip", "install", "-r", "requirements.txt"]
-    install_connector_package_cmd = ["python", "-m", "pip", "install", "."]
-
-    container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude)
-    if requirements_txt := await get_file_contents(container, "requirements.txt"):
-        for line in requirements_txt.split("\n"):
-            if line.startswith("-e ."):
-                local_dependency_path = package_source_code_path + "/" + line[3:]
-                container = container.with_mounted_directory(
-                    "/" + local_dependency_path, context.get_repo_dir(local_dependency_path, exclude=DEFAULT_PYTHON_EXCLUDE)
-                )
-        container = container.with_exec(install_local_requirements_cmd)
-
-    container = container.with_exec(install_connector_package_cmd)
-
-    if additional_dependency_groups:
-        container = container.with_exec(
-            install_connector_package_cmd[:-1] + [install_connector_package_cmd[-1] + f"[{','.join(additional_dependency_groups)}]"]
-        )
-
-    return container
-
-
-def with_python_connector_installed(context: ConnectorContext) -> Container:
-    """Load an airbyte connector source code in a testing environment.
-
-    Args:
-        context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled.
-    Returns:
-        Container: A python environment container (with the connector source code).
-    """
-    connector_source_path = str(context.connector.code_directory)
-    testing_environment: Container = with_testing_dependencies(context)
-    return with_python_package(context, testing_environment, connector_source_path, exclude=["secrets"])
-
-
-async def with_installed_airbyte_connector(context: ConnectorContext) -> Container:
-    """Install an airbyte connector python package in a testing environment.
-
-    Args:
-        context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled.
-    Returns:
-        Container: A python environment container (with the connector installed).
-    """
-    connector_source_path = str(context.connector.code_directory)
-    testing_environment: Container = with_testing_dependencies(context)
-    return await with_installed_python_package(
-        context, testing_environment, connector_source_path, additional_dependency_groups=["dev", "tests", "main"], exclude=["secrets"]
-    )
-
-
-async def with_ci_credentials(context: PipelineContext, gsm_secret: Secret) -> Container:
-    """Install the ci_credentials package in a python environment.
-
-    Args:
-        context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled.
-        gsm_secret (Secret): The secret holding GCP_GSM_CREDENTIALS env variable value.
-
-    Returns:
-        Container: A python environment with the ci_credentials package installed.
-    """
-    python_base_environment: Container = with_python_base(context)
-    ci_credentials = await with_installed_python_package(context, python_base_environment, CI_CREDENTIALS_SOURCE_PATH)
-
-    return ci_credentials.with_env_variable("VERSION", "dev").with_secret_variable("GCP_GSM_CREDENTIALS", gsm_secret).with_workdir("/")
-
-
-def with_alpine_packages(base_container: Container, packages_to_install: List[str]) -> Container:
-    """Installs packages using apk-get.
-    Args:
-        context (Container): A alpine based container.
-
-    Returns:
-        Container: A container with the packages installed.
-
-    """
-    package_install_command = ["apk", "add"]
-    return base_container.with_exec(package_install_command + packages_to_install)
-
-
-def with_debian_packages(base_container: Container, packages_to_install: List[str]) -> Container:
-    """Installs packages using apt-get.
-    Args:
-        context (Container): A alpine based container.
-
-    Returns:
-        Container: A container with the packages installed.
-
-    """
-    update_packages_command = ["apt-get", "update"]
-    package_install_command = ["apt-get", "install", "-y"]
-    return base_container.with_exec(update_packages_command).with_exec(package_install_command + packages_to_install)
-
-
-def with_pip_packages(base_container: Container, packages_to_install: List[str]) -> Container:
-    """Installs packages using pip
-    Args:
-        context (Container): A container with python installed
-
-    Returns:
-        Container: A container with the pip packages installed.
-
-    """
-    package_install_command = ["pip", "install"]
-    return base_container.with_exec(package_install_command + packages_to_install)
-
-
-async def with_ci_connector_ops(context: PipelineContext) -> Container:
-    """Installs the ci_connector_ops package in a Container running Python > 3.10 with git..
-
-    Args:
-        context (PipelineContext): The current test context, providing the repository directory from which the ci_connector_sources sources will be pulled.
-
-    Returns:
-        Container: A python environment container with ci_connector_ops installed.
-    """
-    python_base_environment: Container = with_python_base(context, "python:3-alpine")
-    python_with_git = with_alpine_packages(python_base_environment, ["gcc", "libffi-dev", "musl-dev", "git"])
-    return await with_installed_python_package(context, python_with_git, CI_CONNECTOR_OPS_SOURCE_PATH, exclude=["pipelines"])
-
-
-def with_dockerd_service(
-    context: ConnectorContext, shared_volume: Optional(Tuple[str, CacheVolume]) = None, docker_service_name: Optional[str] = None
-) -> Container:
-    """Create a container running dockerd, exposing its 2375 port, can be used as the docker host for docker-in-docker use cases.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-        shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be mounted to the dockerd container. Defaults to None.
-        docker_service_name (Optional[str], optional): The name of the docker service, appended to volume name, useful context isolation. Defaults to None.
-
-    Returns:
-        Container: The container running dockerd as a service.
-    """
-    docker_lib_volume_name = f"{slugify(context.connector.technical_name)}-docker-lib"
-    if docker_service_name:
-        docker_lib_volume_name = f"{docker_lib_volume_name}-{slugify(docker_service_name)}"
-    dind = (
-        context.dagger_client.container()
-        .from_(consts.DOCKER_DIND_IMAGE)
-        .with_mounted_cache(
-            "/var/lib/docker",
-            context.dagger_client.cache_volume(docker_lib_volume_name),
-            sharing=CacheSharingMode.SHARED,
-        )
-    )
-    if shared_volume is not None:
-        dind = dind.with_mounted_cache(*shared_volume)
-    return dind.with_exposed_port(2375).with_exec(
-        ["dockerd", "--log-level=error", "--host=tcp://0.0.0.0:2375", "--tls=false"], insecure_root_capabilities=True
-    )
-
-
-def with_bound_docker_host(
-    context: ConnectorContext,
-    container: Container,
-    shared_volume: Optional(Tuple[str, CacheVolume]) = None,
-    docker_service_name: Optional[str] = None,
-) -> Container:
-    """Bind a container to a docker host. It will use the dockerd service as a docker host.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-        container (Container): The container to bind to the docker host.
-        shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be both mounted to the container and the dockerd container. Defaults to None.
-        docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to None.
-
-    Returns:
-        Container: The container bound to the docker host.
-    """
-    dockerd = with_dockerd_service(context, shared_volume, docker_service_name)
-    docker_hostname = f"dockerhost-{slugify(context.connector.technical_name)}"
-    if docker_service_name:
-        docker_hostname = f"{docker_hostname}-{slugify(docker_service_name)}"
-    bound = container.with_env_variable("DOCKER_HOST", f"tcp://{docker_hostname}:2375").with_service_binding(docker_hostname, dockerd)
-    if shared_volume:
-        bound = bound.with_mounted_cache(*shared_volume)
-    return bound
-
-
-def with_docker_cli(
-    context: ConnectorContext, shared_volume: Optional(Tuple[str, CacheVolume]) = None, docker_service_name: Optional[str] = None
-) -> Container:
-    """Create a container with the docker CLI installed and bound to a persistent docker host.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-        shared_volume (Optional, optional): A tuple in the form of (mounted path, cache volume) that will be both mounted to the container and the dockerd container. Defaults to None.
-        docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to None.
-
-    Returns:
-        Container: A docker cli container bound to a docker host.
-    """
-    docker_cli = context.dagger_client.container().from_(consts.DOCKER_CLI_IMAGE)
-    return with_bound_docker_host(context, docker_cli, shared_volume, docker_service_name)
-
-
-async def with_connector_acceptance_test(context: ConnectorContext, connector_under_test_image_tar: File) -> Container:
-    """Create a container to run connector acceptance tests, bound to a persistent docker host.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-        connector_under_test_image_tar (File): The file containing the tar archive the image of the connector under test.
-    Returns:
-        Container: A container with connector acceptance tests installed.
-    """
-    connector_under_test_image_name = context.connector.acceptance_test_config["connector_image"]
-    await load_image_to_docker_host(context, connector_under_test_image_tar, connector_under_test_image_name, docker_service_name="cat")
-
-    if context.connector_acceptance_test_image.endswith(":dev"):
-        cat_container = context.connector_acceptance_test_source_dir.docker_build()
-    else:
-        cat_container = context.dagger_client.container().from_(context.connector_acceptance_test_image)
-    shared_tmp_volume = ("/tmp", context.dagger_client.cache_volume("share-tmp-cat"))
-
-    return (
-        with_bound_docker_host(context, cat_container, shared_tmp_volume, docker_service_name="cat")
-        .with_entrypoint([])
-        .with_exec(["pip", "install", "pytest-custom_exit_code"])
-        .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-        .with_mounted_directory("/test_input", context.get_connector_dir(exclude=["secrets", ".venv"]))
-        .with_directory("/test_input/secrets", context.secrets_dir)
-        .with_workdir("/test_input")
-        .with_entrypoint(["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "--suppress-tests-failed-exit-code"])
-        .with_exec(["--acceptance-test-config", "/test_input"])
-    )
-
-
-def with_gradle(
-    context: ConnectorContext,
-    sources_to_include: List[str] = None,
-    bind_to_docker_host: bool = True,
-    docker_service_name: Optional[str] = "gradle",
-) -> Container:
-    """Create a container with Gradle installed and bound to a persistent docker host.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-        sources_to_include (List[str], optional): List of additional source path to mount to the container. Defaults to None.
-        bind_to_docker_host (bool): Whether to bind the gradle container to a docker host.
-        docker_service_name (Optional[str], optional): The name of the docker service, useful context isolation. Defaults to "gradle".
-
-    Returns:
-        Container: A container with Gradle installed and Java sources from the repository.
-    """
-
-    include = [
-        ".root",
-        ".env",
-        "build.gradle",
-        "deps.toml",
-        "gradle.properties",
-        "gradle",
-        "gradlew",
-        "LICENSE_SHORT",
-        "publish-repositories.gradle",
-        "settings.gradle",
-        "build.gradle",
-        "tools/gradle",
-        "spotbugs-exclude-filter-file.xml",
-        "buildSrc",
-        "tools/bin/build_image.sh",
-        "tools/lib/lib.sh",
-    ]
-
-    if sources_to_include:
-        include += sources_to_include
-
-    gradle_dependency_cache: CacheVolume = context.dagger_client.cache_volume("gradle-dependencies-caching")
-    gradle_build_cache: CacheVolume = context.dagger_client.cache_volume(f"{context.connector.technical_name}-gradle-build-cache")
-
-    shared_tmp_volume = ("/tmp", context.dagger_client.cache_volume("share-tmp-gradle"))
-
-    openjdk_with_docker = (
-        context.dagger_client.container()
-        .from_("openjdk:17.0.1-jdk-slim")
-        .with_exec(["apt-get", "update"])
-        .with_exec(["apt-get", "install", "-y", "curl", "jq", "rsync"])
-        .with_env_variable("VERSION", consts.DOCKER_VERSION)
-        .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"])
-        .with_env_variable("GRADLE_HOME", "/root/.gradle")
-        .with_exec(["mkdir", "/airbyte"])
-        .with_workdir("/airbyte")
-        .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include))
-        .with_exec(["mkdir", "-p", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH])
-        .with_mounted_cache(consts.GRADLE_BUILD_CACHE_PATH, gradle_build_cache, sharing=CacheSharingMode.LOCKED)
-        .with_mounted_cache(consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH, gradle_dependency_cache)
-        .with_env_variable("GRADLE_RO_DEP_CACHE", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH)
-    )
-
-    if bind_to_docker_host:
-        return with_bound_docker_host(context, openjdk_with_docker, shared_tmp_volume, docker_service_name=docker_service_name)
-    else:
-        return openjdk_with_docker
-
-
-async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str, docker_service_name: Optional[str] = None):
-    """Load a docker image tar archive to the docker host.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-        tar_file (File): The file object holding the docker image tar archive.
-        image_tag (str): The tag to create on the image if it has no tag.
-        docker_service_name (str): Name of the docker service, useful for context isolation.
-    """
-    # Hacky way to make sure the image is always loaded
-    tar_name = f"{str(uuid.uuid4())}.tar"
-    docker_cli = with_docker_cli(context, docker_service_name=docker_service_name).with_mounted_file(tar_name, tar_file)
-
-    # Remove a previously existing image with the same tag if any.
-    docker_image_rm_exit_code = await with_exit_code(
-        docker_cli.with_env_variable("CACHEBUSTER", tar_name).with_exec(["docker", "image", "rm", image_tag])
-    )
-    if docker_image_rm_exit_code == 0:
-        context.logger.info(f"Removed an existing image tagged {image_tag}")
-    image_load_output = await docker_cli.with_exec(["docker", "load", "--input", tar_name]).stdout()
-    context.logger.info(image_load_output)
-    # Not tagged images only have a sha256 id the load output shares.
-    if "sha256:" in image_load_output:
-        image_id = image_load_output.replace("\n", "").replace("Loaded image ID: sha256:", "")
-        docker_tag_output = await docker_cli.with_exec(["docker", "tag", image_id, image_tag]).stdout()
-        context.logger.info(docker_tag_output)
-
-
-def with_poetry(context: PipelineContext) -> Container:
-    """Install poetry in a python environment.
-
-    Args:
-        context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled.
-    Returns:
-        Container: A python environment with poetry installed.
-    """
-    python_base_environment: Container = with_python_base(context, "python:3.9")
-    python_with_git = with_debian_packages(python_base_environment, ["git"])
-    python_with_poetry = with_pip_packages(python_with_git, ["poetry"])
-
-    # poetry_cache: CacheVolume = context.dagger_client.cache_volume("poetry_cache")
-    # poetry_with_cache = python_with_poetry.with_mounted_cache("/root/.cache/pypoetry", poetry_cache, sharing=CacheSharingMode.SHARED)
-
-    return python_with_poetry
-
-
-def with_poetry_module(context: PipelineContext, parent_dir: Directory, module_path: str) -> Container:
-    """Sets up a Poetry module.
-
-    Args:
-        context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled.
-    Returns:
-        Container: A python environment with dependencies installed using poetry.
-    """
-    poetry_install_dependencies_cmd = ["poetry", "install"]
-
-    python_with_poetry = with_poetry(context)
-    return (
-        python_with_poetry.with_mounted_directory("/src", parent_dir)
-        .with_workdir(f"/src/{module_path}")
-        .with_exec(poetry_install_dependencies_cmd)
-        .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-    )
-
-
-def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container:
-    return (
-        context.dagger_client.container(platform=build_platform)
-        .from_("amazonlinux:2022.0.20220831.1")
-        .with_workdir("/airbyte")
-        .with_file("base.sh", context.get_repo_dir("airbyte-integrations/bases/base", include=["base.sh"]).file("base.sh"))
-        .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh")
-        .with_label("io.airbyte.version", "0.1.0")
-        .with_label("io.airbyte.name", "airbyte/integration-base")
-    )
-
-
-def with_integration_base_java(context: PipelineContext, build_platform: Platform, jdk_version: str = "17.0.4") -> Container:
-    integration_base = with_integration_base(context, build_platform)
-    return (
-        context.dagger_client.container(platform=build_platform)
-        .from_(f"amazoncorretto:{jdk_version}")
-        .with_directory("/airbyte", integration_base.directory("/airbyte"))
-        .with_exec(["yum", "install", "-y", "tar", "openssl"])
-        .with_exec(["yum", "clean", "all"])
-        .with_workdir("/airbyte")
-        .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer"))
-        .with_file("javabase.sh", context.get_repo_dir("airbyte-integrations/bases/base-java", include=["javabase.sh"]).file("javabase.sh"))
-        .with_env_variable("AIRBYTE_SPEC_CMD", "/airbyte/javabase.sh --spec")
-        .with_env_variable("AIRBYTE_CHECK_CMD", "/airbyte/javabase.sh --check")
-        .with_env_variable("AIRBYTE_DISCOVER_CMD", "/airbyte/javabase.sh --discover")
-        .with_env_variable("AIRBYTE_READ_CMD", "/airbyte/javabase.sh --read")
-        .with_env_variable("AIRBYTE_WRITE_CMD", "/airbyte/javabase.sh --write")
-        .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh")
-        .with_label("io.airbyte.version", "0.1.2")
-        .with_label("io.airbyte.name", "airbyte/integration-base-java")
-    )
-
-
-BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = {
-    "destination-bigquery": {
-        "dockerfile": "Dockerfile",
-        "dbt_adapter": "dbt-bigquery==1.0.0",
-        "integration_name": "bigquery",
-        "supports_in_connector_normalization": True,
-        "yum_packages": [],
-    },
-    "destination-clickhouse": {
-        "dockerfile": "clickhouse.Dockerfile",
-        "dbt_adapter": "dbt-clickhouse>=1.4.0",
-        "integration_name": "clickhouse",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-    "destination-duckdb": {
-        "dockerfile": "duckdb.Dockerfile",
-        "dbt_adapter": "dbt-duckdb==1.0.1",
-        "integration_name": "duckdb",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-    "destination-mssql": {
-        "dockerfile": "mssql.Dockerfile",
-        "dbt_adapter": "dbt-sqlserver==1.0.0",
-        "integration_name": "mssql",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-    "destination-mysql": {
-        "dockerfile": "mysql.Dockerfile",
-        "dbt_adapter": "dbt-mysql==1.0.0",
-        "integration_name": "mysql",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-    "destination-oracle": {
-        "dockerfile": "oracle.Dockerfile",
-        "dbt_adapter": "dbt-oracle==0.4.3",
-        "integration_name": "oracle",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-    "destination-postgres": {
-        "dockerfile": "Dockerfile",
-        "dbt_adapter": "dbt-postgres==1.0.0",
-        "integration_name": "postgres",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-    "destination-redshift": {
-        "dockerfile": "redshift.Dockerfile",
-        "dbt_adapter": "dbt-redshift==1.0.0",
-        "integration_name": "redshift",
-        "supports_in_connector_normalization": True,
-        "yum_packages": [],
-    },
-    "destination-snowflake": {
-        "dockerfile": "snowflake.Dockerfile",
-        "dbt_adapter": "dbt-snowflake==1.0.0",
-        "integration_name": "snowflake",
-        "supports_in_connector_normalization": True,
-        "yum_packages": ["gcc-c++"],
-    },
-    "destination-tidb": {
-        "dockerfile": "tidb.Dockerfile",
-        "dbt_adapter": "dbt-tidb==1.0.1",
-        "integration_name": "tidb",
-        "supports_in_connector_normalization": False,
-        "yum_packages": [],
-    },
-}
-
-DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = {
-    **BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION,
-    **{f"{k}-strict-encrypt": v for k, v in BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION.items()},
-}
-
-
-def with_normalization(context: ConnectorContext) -> Container:
-    normalization_directory = context.get_repo_dir("airbyte-integrations/bases/base-normalization")
-    sshtunneling_file = context.get_repo_dir(
-        "airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources", include="sshtunneling.sh"
-    ).file("sshtunneling.sh")
-    normalization_directory_with_build = normalization_directory.with_new_directory("build")
-    normalization_directory_with_sshtunneling = normalization_directory_with_build.with_file("build/sshtunneling.sh", sshtunneling_file)
-    normalization_dockerfile_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["dockerfile"]
-    return normalization_directory_with_sshtunneling.docker_build(normalization_dockerfile_name)
-
-
-def with_integration_base_java_and_normalization(context: PipelineContext, build_platform: Platform) -> Container:
-    yum_packages_to_install = [
-        "python3",
-        "python3-devel",
-        "jq",
-        "sshpass",
-        "git",
-    ]
-
-    additional_yum_packages = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["yum_packages"]
-    yum_packages_to_install += additional_yum_packages
-
-    dbt_adapter_package = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["dbt_adapter"]
-    normalization_integration_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["integration_name"]
-
-    pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache")
-
-    return (
-        with_integration_base_java(context, build_platform)
-        .with_exec(["yum", "install", "-y"] + yum_packages_to_install)
-        .with_exec(["alternatives", "--install", "/usr/bin/python", "python", "/usr/bin/python3", "60"])
-        .with_mounted_cache("/root/.cache/pip", pip_cache)
-        .with_exec(["python", "-m", "ensurepip", "--upgrade"])
-        .with_exec(["pip3", "install", dbt_adapter_package])
-        .with_directory("airbyte_normalization", with_normalization(context).directory("/airbyte"))
-        .with_workdir("airbyte_normalization")
-        .with_exec(["sh", "-c", "mv * .."])
-        .with_workdir("/airbyte")
-        .with_exec(["rm", "-rf", "airbyte_normalization"])
-        .with_workdir("/airbyte/base_python_structs")
-        .with_exec(["pip3", "install", "."])
-        .with_workdir("/airbyte/normalization_code")
-        .with_exec(["pip3", "install", "."])
-        .with_workdir("/airbyte/normalization_code/dbt-template/")
-        # amazon linux 2 isn't compatible with urllib3 2.x, so force 1.x
-        .with_exec(["pip3", "install", "urllib3<2"])
-        .with_exec(["dbt", "deps"])
-        .with_workdir("/airbyte")
-        .with_file(
-            "run_with_normalization.sh",
-            context.get_repo_dir("airbyte-integrations/bases/base-java", include=["run_with_normalization.sh"]).file(
-                "run_with_normalization.sh"
-            ),
-        )
-        .with_env_variable("AIRBYTE_NORMALIZATION_INTEGRATION", normalization_integration_name)
-        .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/run_with_normalization.sh")
-    )
-
-
-async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container:
-    application = context.connector.technical_name
-
-    build_stage = (
-        with_integration_base_java(context, build_platform)
-        .with_workdir("/airbyte")
-        .with_env_variable("APPLICATION", context.connector.technical_name)
-        .with_file(f"{application}.tar", connector_java_tar_file)
-        .with_exec(["tar", "xf", f"{application}.tar", "--strip-components=1"])
-        .with_exec(["rm", "-rf", f"{application}.tar"])
-    )
-
-    if (
-        context.connector.supports_normalization
-        and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"]
-    ):
-        base = with_integration_base_java_and_normalization(context, build_platform)
-        entrypoint = ["/airbyte/run_with_normalization.sh"]
-    else:
-        base = with_integration_base_java(context, build_platform)
-        entrypoint = ["/airbyte/base.sh"]
-
-    connector_container = (
-        base.with_workdir("/airbyte")
-        .with_env_variable("APPLICATION", application)
-        .with_mounted_directory("builts_artifacts", build_stage.directory("/airbyte"))
-        .with_exec(["sh", "-c", "mv builts_artifacts/* ."])
-        .with_label("io.airbyte.version", context.metadata["dockerImageTag"])
-        .with_label("io.airbyte.name", context.metadata["dockerRepository"])
-        .with_entrypoint(entrypoint)
-    )
-    return await finalize_build(context, connector_container)
-
-
-async def get_cdk_version_from_python_connector(python_connector: Container) -> Optional[str]:
-    pip_freeze_stdout = await python_connector.with_entrypoint("pip").with_exec(["freeze"]).stdout()
-    pip_dependencies = [dep.split("==") for dep in pip_freeze_stdout.split("\n")]
-    for package_name, package_version in pip_dependencies:
-        if package_name == "airbyte-cdk":
-            return package_version
-    return None
-
-
-async def with_airbyte_python_connector(context: ConnectorContext, build_platform: Platform) -> Container:
-    pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache")
-    connector_container = (
-        context.dagger_client.container(platform=build_platform)
-        .with_mounted_cache("/root/.cache/pip", pip_cache)
-        .build(context.get_connector_dir())
-        .with_label("io.airbyte.version", context.metadata["dockerImageTag"])
-        .with_label("io.airbyte.name", context.metadata["dockerRepository"])
-    )
-    cdk_version = await get_cdk_version_from_python_connector(connector_container)
-    if cdk_version:
-        connector_container = connector_container.with_label("io.airbyte.cdk_version", cdk_version)
-        context.cdk_version = cdk_version
-    return await finalize_build(context, connector_container)
-
-
-async def finalize_build(context: ConnectorContext, connector_container: Container) -> Container:
-    """Finalize build by adding dagger engine version label and running finalize_build.sh or finalize_build.py if present in the connector directory."""
-    connector_container = connector_container.with_label("io.dagger.engine_version", dagger_engine_version)
-    connector_dir_with_finalize_script = context.get_connector_dir(include=["finalize_build.sh", "finalize_build.py"])
-    finalize_scripts = await connector_dir_with_finalize_script.entries()
-    if not finalize_scripts:
-        return connector_container
-
-    # We don't want finalize scripts to override the entrypoint so we keep it in memory to reset it after finalization
-    original_entrypoint = await connector_container.entrypoint()
-
-    has_finalize_bash_script = "finalize_build.sh" in finalize_scripts
-    has_finalize_python_script = "finalize_build.py" in finalize_scripts
-    if has_finalize_python_script and has_finalize_bash_script:
-        raise Exception("Connector has both finalize_build.sh and finalize_build.py, please remove one of them")
-
-    if has_finalize_python_script:
-        context.logger.info(f"{context.connector.technical_name} has a finalize_build.py script, running it to finalize build...")
-        module_path = context.connector.code_directory / "finalize_build.py"
-        connector_finalize_module_spec = importlib.util.spec_from_file_location(
-            f"{context.connector.code_directory.name}_finalize", module_path
-        )
-        connector_finalize_module = importlib.util.module_from_spec(connector_finalize_module_spec)
-        connector_finalize_module_spec.loader.exec_module(connector_finalize_module)
-        try:
-            connector_container = await connector_finalize_module.finalize_build(context, connector_container)
-        except AttributeError:
-            raise Exception("Connector has a finalize_build.py script but it doesn't have a finalize_build function.")
-
-    if has_finalize_bash_script:
-        context.logger.info(f"{context.connector.technical_name} has finalize_build.sh script, running it to finalize build...")
-        connector_container = (
-            connector_container.with_file("/tmp/finalize_build.sh", connector_dir_with_finalize_script.file("finalize_build.sh"))
-            .with_entrypoint("sh")
-            .with_exec(["/tmp/finalize_build.sh"])
-        )
-
-    return connector_container.with_entrypoint(original_entrypoint)
-
-
-# This function is not used at the moment as we decided to use Python connectors dockerfile instead of building it with dagger.
-# Some python connectors use alpine base image, other use debian... We should unify this.
-def with_airbyte_python_connector_full_dagger(context: ConnectorContext, build_platform: Platform) -> Container:
-    pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache")
-    base = context.dagger_client.container(platform=build_platform).from_("python:3.9.11-alpine3.15")
-    snake_case_name = context.connector.technical_name.replace("-", "_")
-    entrypoint = ["python", "/airbyte/integration_code/main.py"]
-    builder = (
-        base.with_workdir("/airbyte/integration_code")
-        .with_exec(["apk", "--no-cache", "upgrade"])
-        .with_mounted_cache("/root/.cache/pip", pip_cache)
-        .with_exec(["pip", "install", "--upgrade", "pip"])
-        .with_exec(["apk", "--no-cache", "add", "tzdata", "build-base"])
-        .with_file("setup.py", context.get_connector_dir(include="setup.py").file("setup.py"))
-        .with_exec(["pip", "install", "--prefix=/install", "."])
-    )
-    return (
-        base.with_workdir("/airbyte/integration_code")
-        .with_directory("/usr/local", builder.directory("/install"))
-        .with_file("/usr/localtime", builder.file("/usr/share/zoneinfo/Etc/UTC"))
-        .with_new_file("/etc/timezone", "Etc/UTC")
-        .with_exec(["apk", "--no-cache", "add", "bash"])
-        .with_file("main.py", context.get_connector_dir(include="main.py").file("main.py"))
-        .with_directory(snake_case_name, context.get_connector_dir(include=snake_case_name).directory(snake_case_name))
-        .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint))
-        .with_entrypoint(entrypoint)
-        .with_label("io.airbyte.version", context.metadata["dockerImageTag"])
-        .with_label("io.airbyte.name", context.metadata["dockerRepository"])
-    )
-
-
-def with_crane(
-    context: PipelineContext,
-) -> Container:
-    """Crane is a tool to analyze and manipulate container images.
-    We can use it to extract the image manifest and the list of layers or list the existing tags on an image repository.
-    https://github.com/google/go-containerregistry/tree/main/cmd/crane
-    """
-
-    # We use the debug image as it contains a shell which we need to properly use environment variables
-    # https://github.com/google/go-containerregistry/tree/main/cmd/crane#images
-    base_container = context.dagger_client.container().from_("gcr.io/go-containerregistry/crane/debug:v0.15.1")
-
-    if context.docker_hub_username_secret and context.docker_hub_password_secret:
-        base_container = (
-            base_container.with_secret_variable("DOCKER_HUB_USERNAME", context.docker_hub_username_secret).with_secret_variable(
-                "DOCKER_HUB_PASSWORD", context.docker_hub_password_secret
-            )
-            # We need to use skip_entrypoint=True to avoid the entrypoint to be overridden by the crane command
-            # We use sh -c to be able to use environment variables in the command
-            # This is a workaround as the default crane entrypoint doesn't support environment variables
-            .with_exec(
-                ["sh", "-c", "crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"], skip_entrypoint=True
-            )
-        )
-
-    return base_container
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/remote_storage.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/remote_storage.py
deleted file mode 100644
index d4d22735f670..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/remote_storage.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-"""This module groups functions to interact with remote storage services like S3 or GCS."""
-
-import uuid
-from pathlib import Path
-from typing import List, Optional, Tuple
-
-import asyncer
-from ci_connector_ops.pipelines.utils import with_exit_code, with_stderr, with_stdout
-from dagger import Client, File, Secret
-
-GOOGLE_CLOUD_SDK_TAG = "425.0.0-slim"
-
-
-async def upload_to_s3(dagger_client: Client, file_to_upload_path: Path, key: str, bucket: str) -> int:
-    """Upload a local file to S3 using the AWS CLI docker image and running aws s3 cp command.
-
-    Args:
-        dagger_client (Client): The dagger client.
-        file_to_upload_path (Path): The local path to the file to upload.
-        key (str): The key that will be written on the S3 bucket.
-        bucket (str): The S3 bucket name.
-
-    Returns:
-        int: Exit code of the upload process.
-    """
-    s3_uri = f"s3://{bucket}/{key}"
-    file_to_upload: File = dagger_client.host().directory(".", include=[str(file_to_upload_path)]).file(str(file_to_upload_path))
-    aws_access_key_id: Secret = dagger_client.host().env_variable("AWS_ACCESS_KEY_ID").secret()
-    aws_secret_access_key: Secret = dagger_client.host().env_variable("AWS_SECRET_ACCESS_KEY").secret()
-    aws_region: Secret = dagger_client.host().env_variable("AWS_DEFAULT_REGION").secret()
-    return await with_exit_code(
-        dagger_client.container()
-        .from_("amazon/aws-cli:latest")
-        .with_file(str(file_to_upload_path), file_to_upload)
-        .with_secret_variable("AWS_ACCESS_KEY_ID", aws_access_key_id)
-        .with_secret_variable("AWS_SECRET_ACCESS_KEY", aws_secret_access_key)
-        .with_secret_variable("AWS_DEFAULT_REGION", aws_region)
-        .with_exec(["s3", "cp", str(file_to_upload_path), s3_uri])
-    )
-
-
-async def upload_to_gcs(
-    dagger_client: Client,
-    file_to_upload: File,
-    key: str,
-    bucket: str,
-    gcs_credentials: Secret,
-    flags: Optional[List] = None,
-    cache_upload: bool = False,
-) -> Tuple[int, str, str]:
-    """Upload a local file to GCS using the AWS CLI docker image and running aws s3 cp command.
-    Args:
-        dagger_client (Client): The dagger client.
-        file_to_upload_path (File): The dagger File to upload.
-        key (str): The key that will be written on the S3 bucket.
-        bucket (str): The S3 bucket name.
-        gcs_credentials (Secret): The dagger secret holding the credentials to get and upload the targeted GCS bucket.
-        flags (List[str]): Flags to be passed to the 'gcloud storage cp' command.
-        cache_upload (bool): If false, the gcloud commands will be executed on each call.
-    Returns:
-        Tuple[int, str, str]: Exit code, stdout, stderr
-    """
-    flags = [] if flags is None else flags
-    gcs_uri = f"gs://{bucket}/{key}"
-    dagger_client = dagger_client.pipeline(f"Upload file to {gcs_uri}")
-    cp_command = ["gcloud", "storage", "cp"] + flags + ["to_upload", gcs_uri]
-
-    gcloud_container = (
-        dagger_client.container()
-        .from_(f"google/cloud-sdk:{GOOGLE_CLOUD_SDK_TAG}")
-        .with_workdir("/upload")
-        .with_new_file("credentials.json", await gcs_credentials.plaintext())
-        .with_env_variable("GOOGLE_APPLICATION_CREDENTIALS", "/upload/credentials.json")
-        .with_file("to_upload", file_to_upload)
-    )
-    if not cache_upload:
-        gcloud_container = gcloud_container.with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-    else:
-        gcloud_container = gcloud_container.without_env_variable("CACHEBUSTER")
-
-    gcloud_auth_container = gcloud_container.with_exec(["gcloud", "auth", "login", "--cred-file=credentials.json"])
-    if (await with_exit_code(gcloud_auth_container)) == 1:
-        gcloud_auth_container = gcloud_container.with_exec(["gcloud", "auth", "activate-service-account", "--key-file", "credentials.json"])
-
-    gcloud_cp_container = gcloud_auth_container.with_exec(cp_command)
-
-    async with asyncer.create_task_group() as task_group:
-        soon_exit_code = task_group.soonify(with_exit_code)(gcloud_cp_container)
-        soon_stderr = task_group.soonify(with_stderr)(gcloud_cp_container)
-        soon_stdout = task_group.soonify(with_stdout)(gcloud_cp_container)
-    return soon_exit_code.value, soon_stdout.value, soon_stderr.value
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py
deleted file mode 100644
index 99196a7b1891..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/secrets.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This modules groups functions made to download/upload secrets from/to a remote secret service and provide these secret in a dagger Directory."""
-from __future__ import annotations
-
-import datetime
-from typing import TYPE_CHECKING
-
-import anyio
-from ci_connector_ops.pipelines.actions import environments
-from dagger import Directory
-
-if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.contexts import ConnectorContext
-
-
-async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Directory:
-    """Use the ci-credentials tool to download the secrets stored for a specific connector to a Directory.
-
-    Args:
-        context (ConnectorContext): The context providing a connector object.
-        gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS".
-
-    Returns:
-        Directory: A directory with the downloaded secrets.
-    """
-    gsm_secret = context.dagger_client.host().env_variable(gcp_gsm_env_variable_name).secret()
-    secrets_path = f"/{context.connector.code_directory}/secrets"
-
-    ci_credentials = await environments.with_ci_credentials(context, gsm_secret)
-    return (
-        ci_credentials.with_exec(["mkdir", "-p", secrets_path])
-        .with_env_variable(
-            "CACHEBUSTER", datetime.datetime.now().isoformat()
-        )  # Secrets can be updated on GSM anytime, we can't cache this step...
-        .with_exec(["ci_credentials", context.connector.technical_name, "write-to-storage"])
-        .directory(secrets_path)
-    )
-
-
-async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> int:
-    """Use the ci-credentials tool to upload the secrets stored in the context's updated_secrets-dir.
-
-    Args:
-        context (ConnectorContext): The context providing a connector object and the update secrets dir.
-        gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS".
-
-    Returns:
-        int: The exit code of the ci-credentials update-secrets command.
-    """
-    gsm_secret = context.dagger_client.host().env_variable(gcp_gsm_env_variable_name).secret()
-    secrets_path = f"/{context.connector.code_directory}/secrets"
-
-    ci_credentials = await environments.with_ci_credentials(context, gsm_secret)
-
-    return await (
-        ci_credentials.with_directory(secrets_path, context.updated_secrets_dir)
-        .with_exec(["ci_credentials", context.connector.technical_name, "update-secrets"])
-        .exit_code()
-    )
-
-
-async def get_connector_secret_dir(context: ConnectorContext) -> Directory:
-    """Download the secrets from GSM or use the local secrets directory for a connector.
-
-    Args:
-        context (ConnectorContext): The context providing the connector directory and the use_remote_secrets flag.
-
-    Returns:
-        Directory: A directory with the downloaded connector secrets.
-    """
-    if context.use_remote_secrets:
-        secrets_dir = await download(context)
-    else:
-        local_secrets_dir = anyio.Path(context.connector.code_directory) / "secrets"
-        await local_secrets_dir.mkdir(exist_ok=True)
-        secrets_dir = context.get_connector_dir(include=["secrets"]).directory("secrets")
-    return secrets_dir
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py
deleted file mode 100644
index 5b504a652f0b..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/bases.py
+++ /dev/null
@@ -1,420 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module declare base / abstract models to be reused in a pipeline lifecycle."""
-
-from __future__ import annotations
-
-import json
-from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
-from datetime import datetime
-from enum import Enum
-from typing import TYPE_CHECKING, Any, ClassVar, List, Optional
-
-import anyio
-import asyncer
-from ci_connector_ops.pipelines.consts import PYPROJECT_TOML_FILE_PATH
-from ci_connector_ops.pipelines.utils import check_path_in_workdir, with_exit_code, with_stderr, with_stdout
-from ci_connector_ops.utils import console
-from dagger import Container, QueryError
-from rich.console import Group
-from rich.panel import Panel
-from rich.style import Style
-from rich.table import Table
-from rich.text import Text
-from tabulate import tabulate
-
-if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.contexts import ConnectorContext, PipelineContext
-
-
-class CIContext(str, Enum):
-    """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"]."""
-
-    MANUAL = "manual"
-    PULL_REQUEST = "pull_request"
-    NIGHTLY_BUILDS = "nightly_builds"
-    MASTER = "master"
-
-
-class StepStatus(Enum):
-    """An Enum to characterize the success, failure or skipping of a Step."""
-
-    SUCCESS = "Successful"
-    FAILURE = "Failed"
-    SKIPPED = "Skipped"
-
-    def from_exit_code(exit_code: int) -> StepStatus:
-        """Map an exit code to a step status.
-
-        Args:
-            exit_code (int): A process exit code.
-
-        Raises:
-            ValueError: Raised if the exit code is not mapped to a step status.
-
-        Returns:
-            StepStatus: The step status inferred from the exit code.
-        """
-        if exit_code == 0:
-            return StepStatus.SUCCESS
-        # pytest returns a 5 exit code when no test is found.
-        elif exit_code == 5:
-            return StepStatus.SKIPPED
-        else:
-            return StepStatus.FAILURE
-
-    def get_rich_style(self) -> Style:
-        """Match color used in the console output to the step status."""
-        if self is StepStatus.SUCCESS:
-            return Style(color="green")
-        if self is StepStatus.FAILURE:
-            return Style(color="red", bold=True)
-        if self is StepStatus.SKIPPED:
-            return Style(color="yellow")
-
-    def get_emoji(self) -> str:
-        """Match emoji used in the console output to the step status."""
-        if self is StepStatus.SUCCESS:
-            return "✅"
-        if self is StepStatus.FAILURE:
-            return "❌"
-        if self is StepStatus.SKIPPED:
-            return "🟡"
-
-    def __str__(self) -> str:  # noqa D105
-        return self.value
-
-
-class Step(ABC):
-    """An abstract class to declare and run pipeline step."""
-
-    title: ClassVar[str]
-    started_at: ClassVar[datetime]
-    retry: ClassVar[bool] = False
-    max_retries: ClassVar[int] = 0
-
-    def __init__(self, context: ConnectorContext) -> None:  # noqa D107
-        self.context = context
-        self.retry_count = 0
-
-    async def run(self, *args, **kwargs) -> StepResult:
-        """Public method to run the step. It output a step result.
-
-        If an unexpected dagger error happens it outputs a failed step result with the exception payload.
-
-        Returns:
-            StepResult: The step result following the step run.
-        """
-        self.started_at = datetime.utcnow()
-        try:
-            result = await self._run(*args, **kwargs)
-            if result.status is StepStatus.FAILURE and self.retry_count <= self.max_retries:
-                self.retry_count += 1
-                await anyio.sleep(10)
-                self.context.logger.warn(
-                    f"Retry #{self.retry_count} for {self.title} step on connector {self.context.connector.technical_name}"
-                )
-                return await self.run(*args, **kwargs)
-            return result
-        except QueryError as e:
-            return StepResult(self, StepStatus.FAILURE, stderr=str(e))
-
-    @abstractmethod
-    async def _run(self, *args, **kwargs) -> StepResult:
-        """Implement the execution of the step and return a step result.
-
-        Returns:
-            StepResult: The result of the step run.
-        """
-        ...
-
-    def skip(self, reason: str = None) -> StepResult:
-        """Declare a step as skipped.
-
-        Args:
-            reason (str, optional): Reason why the step was skipped.
-
-        Returns:
-            StepResult: A skipped step result.
-        """
-        return StepResult(self, StepStatus.SKIPPED, stdout=reason)
-
-    async def get_step_result(self, container: Container) -> StepResult:
-        """Concurrent retrieval of exit code, stdout and stdout of a container.
-
-        Create a StepResult object from these objects.
-
-        Args:
-            container (Container): The container from which we want to infer a step result/
-
-        Returns:
-            StepResult: Failure or success with stdout and stderr.
-        """
-        async with asyncer.create_task_group() as task_group:
-            soon_exit_code = task_group.soonify(with_exit_code)(container)
-            soon_stderr = task_group.soonify(with_stderr)(container)
-            soon_stdout = task_group.soonify(with_stdout)(container)
-        return StepResult(
-            self,
-            StepStatus.from_exit_code(soon_exit_code.value),
-            stderr=soon_stderr.value,
-            stdout=soon_stdout.value,
-            output_artifact=container,
-        )
-
-
-class PytestStep(Step, ABC):
-    """An abstract class to run pytest tests and evaluate success or failure according to pytest logs."""
-
-    # TODO this is not very robust if pytest crashes and does not outputs its expected last log line.
-    def pytest_logs_to_step_result(self, logs: str) -> StepResult:
-        """Parse pytest log and infer failure, success or skipping.
-
-        Args:
-            logs (str): The pytest logs.
-
-        Returns:
-            StepResult: The inferred step result according to the log.
-        """
-        last_log_line = logs.split("\n")[-2]
-        if "failed" in last_log_line or "errors" in last_log_line:
-            return StepResult(self, StepStatus.FAILURE, stderr=logs)
-        elif "no tests ran" in last_log_line:
-            return StepResult(self, StepStatus.SKIPPED, stdout=logs)
-        else:
-            return StepResult(self, StepStatus.SUCCESS, stdout=logs)
-
-    async def _run_tests_in_directory(self, connector_under_test: Container, test_directory: str) -> StepResult:
-        """Run the pytest tests in the test_directory that was passed.
-
-        A StepStatus.SKIPPED is returned if no tests were discovered.
-
-        Args:
-            connector_under_test (Container): The connector under test container.
-            test_directory (str): The directory in which the python test modules are declared
-
-        Returns:
-            Tuple[StepStatus, Optional[str], Optional[str]]: Tuple of StepStatus, stderr and stdout.
-        """
-        test_config = "pytest.ini" if await check_path_in_workdir(connector_under_test, "pytest.ini") else "/" + PYPROJECT_TOML_FILE_PATH
-        if await check_path_in_workdir(connector_under_test, test_directory):
-            tester = connector_under_test.with_exec(
-                [
-                    "python",
-                    "-m",
-                    "pytest",
-                    "--suppress-tests-failed-exit-code",
-                    "--suppress-no-test-exit-code",
-                    "-s",
-                    test_directory,
-                    "-c",
-                    test_config,
-                ]
-            )
-            return self.pytest_logs_to_step_result(await tester.stdout())
-
-        else:
-            return StepResult(self, StepStatus.SKIPPED)
-
-
-@dataclass(frozen=True)
-class StepResult:
-    """A dataclass to capture the result of a step."""
-
-    step: Step
-    status: StepStatus
-    created_at: datetime = field(default_factory=datetime.utcnow)
-    stderr: Optional[str] = None
-    stdout: Optional[str] = None
-    output_artifact: Any = None
-
-    def __repr__(self) -> str:  # noqa D105
-        return f"{self.step.title}: {self.status.value}"
-
-
-@dataclass(frozen=True)
-class Report:
-    """A dataclass to build reports to share pipelines executions results with the user."""
-
-    pipeline_context: PipelineContext
-    steps_results: List[StepResult]
-    created_at: datetime = field(default_factory=datetime.utcnow)
-    name: str = "REPORT"
-
-    @property
-    def failed_steps(self) -> List[StepResult]:  # noqa D102
-        return [step_result for step_result in self.steps_results if step_result.status is StepStatus.FAILURE]
-
-    @property
-    def successful_steps(self) -> List[StepResult]:  # noqa D102
-        return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SUCCESS]
-
-    @property
-    def skipped_steps(self) -> List[StepResult]:  # noqa D102
-        return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SKIPPED]
-
-    @property
-    def success(self) -> bool:  # noqa D102
-        return len(self.failed_steps) == 0
-
-    @property
-    def run_duration(self) -> int:  # noqa D102
-        return (self.created_at - self.pipeline_context.created_at).total_seconds()
-
-    def to_json(self) -> str:
-        """Create a JSON representation of the report.
-
-        Returns:
-            str: The JSON representation of the report.
-        """
-        return json.dumps(
-            {
-                "pipeline_name": self.pipeline_context.pipeline_name,
-                "run_timestamp": self.created_at.isoformat(),
-                "run_duration": self.run_duration,
-                "success": self.success,
-                "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps],
-                "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps],
-                "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps],
-                "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url,
-                "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp,
-                "pipeline_end_timestamp": round(self.created_at.timestamp()),
-                "pipeline_duration": round(self.created_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp,
-                "git_branch": self.pipeline_context.git_branch,
-                "git_revision": self.pipeline_context.git_revision,
-                "ci_context": self.pipeline_context.ci_context,
-                "pull_request_url": self.pipeline_context.pull_request.html_url if self.pipeline_context.pull_request else None,
-            }
-        )
-
-    def print(self):
-        """Print the test report to the console in a nice way."""
-        pipeline_name = self.pipeline_context.pipeline_name
-        main_panel_title = Text(f"{pipeline_name.upper()} - {self.name}")
-        main_panel_title.stylize(Style(color="blue", bold=True))
-        duration_subtitle = Text(f"⏲️  Total pipeline duration for {pipeline_name}: {round(self.run_duration)} seconds")
-        step_results_table = Table(title="Steps results")
-        step_results_table.add_column("Step")
-        step_results_table.add_column("Result")
-        step_results_table.add_column("Finished after")
-
-        for step_result in self.steps_results:
-            step = Text(step_result.step.title)
-            step.stylize(step_result.status.get_rich_style())
-            result = Text(step_result.status.value)
-            result.stylize(step_result.status.get_rich_style())
-
-            if step_result.status is StepStatus.SKIPPED:
-                step_results_table.add_row(step, result, "N/A")
-            else:
-                run_time_seconds = round((step_result.created_at - step_result.step.started_at).total_seconds())
-                step_results_table.add_row(step, result, f"{run_time_seconds}s")
-
-        to_render = [step_results_table]
-        if self.failed_steps:
-            sub_panels = []
-            for failed_step in self.failed_steps:
-                errors = Text(failed_step.stderr)
-                panel_title = Text(f"{pipeline_name} {failed_step.step.title.lower()} failures")
-                panel_title.stylize(Style(color="red", bold=True))
-                sub_panel = Panel(errors, title=panel_title)
-                sub_panels.append(sub_panel)
-            failures_group = Group(*sub_panels)
-            to_render.append(failures_group)
-
-        main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle)
-        console.print(main_panel)
-
-
-@dataclass(frozen=True)
-class ConnectorReport(Report):
-    """A dataclass to build connector test reports to share pipelines executions results with the user."""
-
-    @property
-    def should_be_saved(self) -> bool:  # noqa D102
-        return self.pipeline_context.is_ci
-
-    @property
-    def should_be_commented_on_pr(self) -> bool:  # noqa D102
-        return self.pipeline_context.is_ci and self.pipeline_context.pull_request and self.pipeline_context.PRODUCTION
-
-    def to_json(self) -> str:
-        """Create a JSON representation of the connector test report.
-
-        Returns:
-            str: The JSON representation of the report.
-        """
-        return json.dumps(
-            {
-                "connector_technical_name": self.pipeline_context.connector.technical_name,
-                "connector_version": self.pipeline_context.connector.version,
-                "run_timestamp": self.created_at.isoformat(),
-                "run_duration": self.run_duration,
-                "success": self.success,
-                "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps],
-                "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps],
-                "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps],
-                "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url,
-                "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp,
-                "pipeline_end_timestamp": round(self.created_at.timestamp()),
-                "pipeline_duration": round(self.created_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp,
-                "git_branch": self.pipeline_context.git_branch,
-                "git_revision": self.pipeline_context.git_revision,
-                "ci_context": self.pipeline_context.ci_context,
-                "cdk_version": self.pipeline_context.cdk_version,
-            }
-        )
-
-    def post_comment_on_pr(self) -> None:
-        icon_url = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg"
-        global_status_emoji = "✅" if self.success else "❌"
-        commit_url = f"{self.pipeline_context.pull_request.html_url}/commits/{self.pipeline_context.git_revision}"
-        markdown_comment = f'##  {self.pipeline_context.connector.technical_name} test report (commit [`{self.pipeline_context.git_revision[:10]}`]({commit_url})) - {global_status_emoji}\n\n'
-        markdown_comment += f"⏲️  Total pipeline duration: {round(self.run_duration)} seconds\n\n"
-        report_data = [
-            [step_result.step.title, step_result.status.get_emoji()]
-            for step_result in self.steps_results
-            if step_result.status is not StepStatus.SKIPPED
-        ]
-        markdown_comment += tabulate(report_data, headers=["Step", "Result"], tablefmt="pipe") + "\n\n"
-        markdown_comment += f"🔗 [View the logs here]({self.pipeline_context.gha_workflow_run_url})\n\n"
-        markdown_comment += "*Please note that tests are only run on PR ready for review. Please set your PR to draft mode to not flood the CI engine and upstream service on following commits.*\n"
-        markdown_comment += "**You can run the same pipeline locally on this branch with the [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/README.md) tool with the following command**\n"
-        markdown_comment += f"```bash\nairbyte-ci connectors --name={self.pipeline_context.connector.technical_name} test\n```\n\n"
-        self.pipeline_context.pull_request.create_issue_comment(markdown_comment)
-
-    def print(self):
-        """Print the test report to the console in a nice way."""
-        connector_name = self.pipeline_context.connector.technical_name
-        main_panel_title = Text(f"{connector_name.upper()} - {self.name}")
-        main_panel_title.stylize(Style(color="blue", bold=True))
-        duration_subtitle = Text(f"⏲️  Total pipeline duration for {connector_name}: {round(self.run_duration)} seconds")
-        step_results_table = Table(title="Steps results")
-        step_results_table.add_column("Step")
-        step_results_table.add_column("Result")
-        step_results_table.add_column("Finished after")
-
-        for step_result in self.steps_results:
-            step = Text(step_result.step.title)
-            step.stylize(step_result.status.get_rich_style())
-            result = Text(step_result.status.value)
-            result.stylize(step_result.status.get_rich_style())
-            step_results_table.add_row(step, result, f"{round((self.created_at - step_result.created_at).total_seconds())}s")
-
-        to_render = [step_results_table]
-        if self.failed_steps:
-            sub_panels = []
-            for failed_step in self.failed_steps:
-                errors = Text(failed_step.stderr)
-                panel_title = Text(f"{connector_name} {failed_step.step.title.lower()} failures")
-                panel_title.stylize(Style(color="red", bold=True))
-                sub_panel = Panel(errors, title=panel_title)
-                sub_panels.append(sub_panel)
-            failures_group = Group(*sub_panels)
-            to_render.append(failures_group)
-
-        main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle)
-        console.print(main_panel)
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/__init__.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/__init__.py
deleted file mode 100644
index 0acf25be22e9..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/__init__.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-"""This module groups factory like functions to dispatch builds steps according to the connector language."""
-
-from __future__ import annotations
-
-import platform
-
-import anyio
-from ci_connector_ops.pipelines.bases import ConnectorReport, StepResult
-from ci_connector_ops.pipelines.builds import common, java_connectors, python_connectors
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from ci_connector_ops.utils import ConnectorLanguage
-from dagger import Platform
-
-
-class NoBuildStepForLanguageError(Exception):
-    pass
-
-
-LANGUAGE_BUILD_CONNECTOR_MAPPING = {
-    ConnectorLanguage.PYTHON: python_connectors.run_connector_build,
-    ConnectorLanguage.LOW_CODE: python_connectors.run_connector_build,
-    ConnectorLanguage.JAVA: java_connectors.run_connector_build,
-}
-
-BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
-LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
-
-
-async def run_connector_build(context: ConnectorContext) -> StepResult:
-    """Run a build pipeline for a single connector."""
-    if context.connector.language not in LANGUAGE_BUILD_CONNECTOR_MAPPING:
-        raise NoBuildStepForLanguageError(f"No build step for connector language {context.connector.language}.")
-    return await LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language](context)
-
-
-async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport:
-    """Run a build pipeline for a single connector.
-
-    Args:
-        context (ConnectorContext): The initialized connector context.
-
-    Returns:
-        ConnectorReport: The reports holding builds results.
-    """
-    step_results = []
-    async with semaphore:
-        async with context:
-            build_result = await run_connector_build(context)
-            step_results.append(build_result)
-            if context.is_local and build_result.status is common.StepStatus.SUCCESS:
-                connector_to_load_to_local_docker_host = build_result.output_artifact[LOCAL_BUILD_PLATFORM]
-                load_image_result = await common.LoadContainerToLocalDockerHost(context, connector_to_load_to_local_docker_host).run()
-                step_results.append(load_image_result)
-            context.report = ConnectorReport(context, step_results, name="BUILD RESULTS")
-        return context.report
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/common.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/common.py
deleted file mode 100644
index 637c439af70a..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/common.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-from abc import ABC
-from typing import Tuple
-
-import docker
-from ci_connector_ops.pipelines.bases import Step, StepResult, StepStatus
-from ci_connector_ops.pipelines.consts import BUILD_PLATFORMS
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from ci_connector_ops.pipelines.utils import export_container_to_tarball
-from dagger import Container, Platform
-
-
-class BuildConnectorImageBase(Step, ABC):
-    @property
-    def title(self):
-        return f"Build {self.context.connector.technical_name} docker image for platform {self.build_platform}"
-
-    def __init__(self, context: ConnectorContext, build_platform: Platform) -> None:
-        self.build_platform = build_platform
-        super().__init__(context)
-
-
-class BuildConnectorImageForAllPlatformsBase(Step, ABC):
-
-    ALL_PLATFORMS = BUILD_PLATFORMS
-
-    title = f"Build connector image for {BUILD_PLATFORMS}"
-
-    def get_success_result(self, build_results_per_platform: dict[Platform, Container]) -> StepResult:
-        return StepResult(
-            self,
-            StepStatus.SUCCESS,
-            stdout="The connector image was successfully built for all platforms.",
-            output_artifact=build_results_per_platform,
-        )
-
-
-class LoadContainerToLocalDockerHost(Step):
-    IMAGE_TAG = "dev"
-
-    def __init__(self, context: ConnectorContext, container: Container) -> None:
-        super().__init__(context)
-        self.container = container
-
-    @property
-    def title(self):
-        return f"Load {self.image_name}:{self.IMAGE_TAG} to the local docker host."
-
-    @property
-    def image_name(self) -> Tuple:
-        return f"airbyte/{self.context.connector.technical_name}"
-
-    async def _run(self) -> StepResult:
-        _, exported_tarball_path = await export_container_to_tarball(self.context, self.container)
-        client = docker.from_env()
-        try:
-            with open(exported_tarball_path, "rb") as tarball_content:
-                new_image = client.images.load(tarball_content.read())[0]
-            new_image.tag(self.image_name, tag=self.IMAGE_TAG)
-            return StepResult(self, StepStatus.SUCCESS)
-        except ConnectionError:
-            return StepResult(self, StepStatus.FAILURE, stderr="The connection to the local docker host failed.")
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py
deleted file mode 100644
index 032e113d0de5..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/java_connectors.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from ci_connector_ops.pipelines.actions import environments
-from ci_connector_ops.pipelines.bases import StepResult, StepStatus
-from ci_connector_ops.pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from ci_connector_ops.pipelines.gradle import GradleTask
-from ci_connector_ops.pipelines.utils import with_exit_code
-from dagger import File, QueryError
-
-
-class BuildConnectorDistributionTar(GradleTask):
-
-    title = "Build connector tar"
-    gradle_task_name = "distTar"
-
-    async def _run(self) -> StepResult:
-        with_built_tar = (
-            environments.with_gradle(
-                self.context,
-                self.build_include,
-                docker_service_name=self.docker_service_name,
-            )
-            .with_mounted_directory(str(self.context.connector.code_directory), await self._get_patched_connector_dir())
-            .with_exec(self._get_gradle_command())
-            .with_workdir(f"{self.context.connector.code_directory}/build/distributions")
-        )
-        distributions = await with_built_tar.directory(".").entries()
-        tar_files = [f for f in distributions if f.endswith(".tar")]
-        await self._export_gradle_dependency_cache(with_built_tar)
-        if len(tar_files) == 1:
-            return StepResult(
-                self,
-                StepStatus.SUCCESS,
-                stdout="The tar file for the current connector was successfully built.",
-                output_artifact=with_built_tar.file(tar_files[0]),
-            )
-        else:
-            return StepResult(
-                self,
-                StepStatus.FAILURE,
-                stderr="The distributions directory contains multiple connector tar files. We can't infer which one should be used. Please review and delete any unnecessary tar files.",
-            )
-
-
-class BuildConnectorImage(BuildConnectorImageBase):
-    """
-    A step to build a Java connector image using the distTar Gradle task.
-    """
-
-    async def _run(self, distribution_tar: File) -> StepResult:
-        try:
-            java_connector = await environments.with_airbyte_java_connector(self.context, distribution_tar, self.build_platform)
-            spec_exit_code = await with_exit_code(java_connector.with_exec(["spec"]))
-            if spec_exit_code != 0:
-                return StepResult(
-                    self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {self.build_platform}."
-                )
-            return StepResult(
-                self, StepStatus.SUCCESS, stdout="The connector image was successfully built.", output_artifact=java_connector
-            )
-        except QueryError as e:
-            return StepResult(self, StepStatus.FAILURE, stderr=str(e))
-
-
-class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase):
-    """Build a Java connector image for all platforms."""
-
-    async def _run(self, distribution_tar: File) -> StepResult:
-        build_results_per_platform = {}
-        for platform in self.ALL_PLATFORMS:
-            build_connector_step_result = await BuildConnectorImage(self.context, platform).run(distribution_tar)
-            if build_connector_step_result.status is not StepStatus.SUCCESS:
-                return build_connector_step_result
-            build_results_per_platform[platform] = build_connector_step_result.output_artifact
-        return self.get_success_result(build_results_per_platform)
-
-
-async def run_connector_build(context: ConnectorContext) -> StepResult:
-    """Create the java connector distribution tar file and build the connector image."""
-
-    build_connector_tar_result = await BuildConnectorDistributionTar(context).run()
-    if build_connector_tar_result.status is not StepStatus.SUCCESS:
-        return build_connector_tar_result
-
-    return await BuildConnectorImageForAllPlatforms(context).run(build_connector_tar_result.output_artifact)
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/python_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/python_connectors.py
deleted file mode 100644
index 7c072664a5dc..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/builds/python_connectors.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from ci_connector_ops.pipelines.actions.environments import with_airbyte_python_connector
-from ci_connector_ops.pipelines.bases import StepResult, StepStatus
-from ci_connector_ops.pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from dagger import QueryError
-
-
-class BuildConnectorImage(BuildConnectorImageBase):
-    """
-    A step to build a Python connector image.
-    A spec command is run on the container to validate it was built successfully.
-    """
-
-    async def _run(self) -> StepResult:
-        connector = await with_airbyte_python_connector(self.context, self.build_platform)
-        try:
-            return await self.get_step_result(connector.with_exec(["spec"]))
-        except QueryError as e:
-            return StepResult(self, StepStatus.FAILURE, stderr=str(e))
-
-
-class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase):
-    """Build a Python connector image for all platforms."""
-
-    async def _run(self) -> StepResult:
-        build_results_per_platform = {}
-        for platform in self.ALL_PLATFORMS:
-            build_connector_step_result = await BuildConnectorImage(self.context, platform).run()
-            if build_connector_step_result.status is not StepStatus.SUCCESS:
-                return build_connector_step_result
-            build_results_per_platform[platform] = build_connector_step_result.output_artifact
-        return self.get_success_result(build_results_per_platform)
-
-
-async def run_connector_build(context: ConnectorContext) -> StepResult:
-    return await BuildConnectorImageForAllPlatforms(context).run()
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/airbyte_ci.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/airbyte_ci.py
deleted file mode 100644
index 6e409a864fbe..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/airbyte_ci.py
+++ /dev/null
@@ -1,113 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module is the CLI entrypoint to the airbyte-ci commands."""
-
-from typing import List
-
-import click
-from ci_connector_ops.pipelines import github
-from ci_connector_ops.pipelines.bases import CIContext
-from ci_connector_ops.pipelines.utils import (
-    get_current_epoch_time,
-    get_current_git_branch,
-    get_current_git_revision,
-    get_modified_files_in_branch,
-    get_modified_files_in_commit,
-    get_modified_files_in_pull_request,
-)
-from github import PullRequest
-
-from .groups.connectors import connectors
-from .groups.metadata import metadata
-
-
-def get_modified_files(
-    git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, pull_request: PullRequest
-) -> List[str]:
-    """Get the list of modified files in the current git branch.
-    If the current branch is master, it will return the list of modified files in the head commit.
-    The head commit on master should be the merge commit of the latest merged pull request as we squash commits on merge.
-    Pipelines like "publish on merge" are triggered on each new commit on master.
-
-    If the CI context is a pull request, it will return the list of modified files in the pull request, without using git diff.
-    If the current branch is not master, it will return the list of modified files in the current branch.
-    This latest case is the one we encounter when running the pipeline locally, on a local branch, or manually on GHA with a workflow dispatch event.
-    """
-    if ci_context is CIContext.MASTER or ci_context is CIContext.NIGHTLY_BUILDS:
-        return get_modified_files_in_commit(git_branch, git_revision, is_local)
-    if ci_context is CIContext.PULL_REQUEST and pull_request is not None:
-        return get_modified_files_in_pull_request(pull_request)
-    if ci_context is CIContext.MANUAL:
-        if git_branch == "master":
-            return get_modified_files_in_commit(git_branch, git_revision, is_local)
-        else:
-            return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local)
-    return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local)
-
-
-@click.group(help="Airbyte CI top-level command group.")
-@click.option("--is-local/--is-ci", default=True)
-@click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH")
-@click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION")
-@click.option(
-    "--diffed-branch",
-    help="Branch to which the git diff will happen to detect new or modified connectors",
-    default="origin/master",
-    type=str,
-)
-@click.option("--gha-workflow-run-id", help="[CI Only] The run id of the GitHub action workflow", default=None, type=str)
-@click.option("--ci-context", default=CIContext.MANUAL, envvar="CI_CONTEXT", type=click.Choice(CIContext))
-@click.option("--pipeline-start-timestamp", default=get_current_epoch_time, envvar="CI_PIPELINE_START_TIMESTAMP", type=int)
-@click.option("--pull-request-number", envvar="PULL_REQUEST_NUMBER", type=int)
-@click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str)
-@click.pass_context
-def airbyte_ci(
-    ctx: click.Context,
-    is_local: bool,
-    git_branch: str,
-    git_revision: str,
-    diffed_branch: str,
-    gha_workflow_run_id: str,
-    ci_context: str,
-    pipeline_start_timestamp: int,
-    pull_request_number: int,
-    ci_github_access_token: str,
-):  # noqa D103
-    ctx.ensure_object(dict)
-    ctx.obj["is_local"] = is_local
-    ctx.obj["is_ci"] = not is_local
-    ctx.obj["git_branch"] = git_branch
-    ctx.obj["git_revision"] = git_revision
-    ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id
-    ctx.obj["gha_workflow_run_url"] = (
-        f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None
-    )
-    ctx.obj["ci_context"] = ci_context
-    ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp
-
-    if pull_request_number and ci_github_access_token:
-        ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token)
-    else:
-        ctx.obj["pull_request"] = None
-
-    ctx.obj["modified_files"] = get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"])
-
-    if not is_local:
-        click.echo("Running airbyte-ci in CI mode.")
-        click.echo(f"CI Context: {ci_context}")
-        click.echo(f"Git Branch: {git_branch}")
-        click.echo(f"Git Revision: {git_revision}")
-        click.echo(f"GitHub Workflow Run ID: {gha_workflow_run_id}")
-        click.echo(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}")
-        click.echo(f"Pull Request Number: {pull_request_number}")
-        click.echo(f"Pipeline Start Timestamp: {pipeline_start_timestamp}")
-        click.echo(f"Modified Files: {ctx.obj['modified_files']}")
-
-
-airbyte_ci.add_command(connectors)
-airbyte_ci.add_command(metadata)
-
-if __name__ == "__main__":
-    airbyte_ci()
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/connectors.py
deleted file mode 100644
index 9c27bd6c7e1f..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/connectors.py
+++ /dev/null
@@ -1,391 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module declares the CLI commands to run the connectors CI pipelines."""
-
-import logging
-import os
-import sys
-from pathlib import Path
-from typing import Any, Dict, Tuple
-
-import anyio
-import click
-from ci_connector_ops.pipelines.builds import run_connector_build_pipeline
-from ci_connector_ops.pipelines.contexts import ConnectorContext, ContextState, PublishConnectorContext
-from ci_connector_ops.pipelines.github import update_global_commit_status_check_for_tests
-from ci_connector_ops.pipelines.pipelines.connectors import run_connectors_pipelines
-from ci_connector_ops.pipelines.publish import reorder_contexts, run_connector_publish_pipeline
-from ci_connector_ops.pipelines.tests import run_connector_test_pipeline
-from ci_connector_ops.pipelines.utils import DaggerPipelineCommand, get_modified_connectors, get_modified_metadata_files
-from ci_connector_ops.utils import ConnectorLanguage, console, get_all_released_connectors
-from rich.logging import RichHandler
-from rich.table import Table
-from rich.text import Text
-
-logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True)])
-
-logger = logging.getLogger(__name__)
-
-
-# HELPERS
-
-
-def validate_environment(is_local: bool, use_remote_secrets: bool):
-    """Check if the required environment variables exist."""
-    if is_local:
-        if not (os.getcwd().endswith("/airbyte") and Path(".git").is_dir()):
-            raise click.UsageError("You need to run this command from the airbyte repository root.")
-    else:
-        required_env_vars_for_ci = [
-            "GCP_GSM_CREDENTIALS",
-            "AWS_ACCESS_KEY_ID",
-            "AWS_SECRET_ACCESS_KEY",
-            "AWS_DEFAULT_REGION",
-            "TEST_REPORTS_BUCKET_NAME",
-            "CI_GITHUB_ACCESS_TOKEN",
-        ]
-        for required_env_var in required_env_vars_for_ci:
-            if os.getenv(required_env_var) is None:
-                raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.")
-    if use_remote_secrets and os.getenv("GCP_GSM_CREDENTIALS") is None:
-        raise click.UsageError(
-            "You have to set the GCP_GSM_CREDENTIALS if you want to download secrets from GSM. Set the --use-remote-secrets option to false otherwise."
-        )
-
-
-# COMMANDS
-
-
-@click.group(help="Commands related to connectors and connector acceptance tests.")
-@click.option("--use-remote-secrets", default=True)  # specific to connectors
-@click.option(
-    "--name", "names", multiple=True, help="Only test a specific connector. Use its technical name. e.g source-pokeapi.", type=str
-)
-@click.option("--language", "languages", multiple=True, help="Filter connectors to test by language.", type=click.Choice(ConnectorLanguage))
-@click.option(
-    "--release-stage",
-    "release_stages",
-    multiple=True,
-    help="Filter connectors to test by release stage.",
-    type=click.Choice(["alpha", "beta", "generally_available"]),
-)
-@click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool)
-@click.option("--concurrency", help="Number of connector tests pipeline to run in parallel.", default=5, type=int)
-@click.option(
-    "--execute-timeout",
-    help="The maximum time in seconds for the execution of a Dagger request before an ExecuteTimeoutError is raised. Passing None results in waiting forever.",
-    default=None,
-    type=int,
-)
-@click.pass_context
-def connectors(
-    ctx: click.Context,
-    use_remote_secrets: str,
-    names: Tuple[str],
-    languages: Tuple[ConnectorLanguage],
-    release_stages: Tuple[str],
-    modified: bool,
-    concurrency: int,
-    execute_timeout: int,
-):
-    """Group all the connectors-ci command."""
-    validate_environment(ctx.obj["is_local"], use_remote_secrets)
-
-    ctx.ensure_object(dict)
-    ctx.obj["use_remote_secrets"] = use_remote_secrets
-    ctx.obj["connector_names"] = names
-    ctx.obj["connector_languages"] = languages
-    ctx.obj["release_states"] = release_stages
-    ctx.obj["modified"] = modified
-    ctx.obj["concurrency"] = concurrency
-    ctx.obj["execute_timeout"] = execute_timeout
-
-    all_connectors = get_all_released_connectors()
-
-    modified_connectors_and_files = get_modified_connectors(ctx.obj["modified_files"])
-    # We select all connectors by default
-    selected_connectors_and_files = {connector: modified_connectors_and_files.get(connector, []) for connector in all_connectors}
-
-    if names:
-        selected_connectors_and_files = {
-            connector: selected_connectors_and_files[connector]
-            for connector in selected_connectors_and_files
-            if connector.technical_name in names
-        }
-    if languages:
-        selected_connectors_and_files = {
-            connector: selected_connectors_and_files[connector]
-            for connector in selected_connectors_and_files
-            if connector.language in languages
-        }
-    if release_stages:
-        selected_connectors_and_files = {
-            connector: selected_connectors_and_files[connector]
-            for connector in selected_connectors_and_files
-            if connector.release_stage in release_stages
-        }
-    if modified:
-        selected_connectors_and_files = {
-            connector: modified_files for connector, modified_files in selected_connectors_and_files.items() if modified_files
-        }
-
-    ctx.obj["selected_connectors_and_files"] = selected_connectors_and_files
-    ctx.obj["selected_connectors_names"] = [c.technical_name for c in selected_connectors_and_files.keys()]
-
-
-@connectors.command(cls=DaggerPipelineCommand, help="Test all the selected connectors.")
-@click.pass_context
-def test(
-    ctx: click.Context,
-) -> bool:
-    """Runs a test pipeline for the selected connectors.
-
-    Args:
-        ctx (click.Context): The click context.
-    """
-    if ctx.obj["is_ci"] and ctx.obj["pull_request"] and ctx.obj["pull_request"].draft:
-        click.echo("Skipping connectors tests for draft pull request.")
-        sys.exit(0)
-
-    click.secho(f"Will run the test pipeline for the following connectors: {', '.join(ctx.obj['selected_connectors_names'])}.", fg="green")
-    if ctx.obj["selected_connectors_and_files"]:
-        update_global_commit_status_check_for_tests(ctx.obj, "pending")
-    else:
-        click.secho("No connector were selected for testing.", fg="yellow")
-        update_global_commit_status_check_for_tests(ctx.obj, "success")
-        return True
-
-    connectors_tests_contexts = [
-        ConnectorContext(
-            pipeline_name=f"Testing connector {connector.technical_name}",
-            connector=connector,
-            is_local=ctx.obj["is_local"],
-            git_branch=ctx.obj["git_branch"],
-            git_revision=ctx.obj["git_revision"],
-            modified_files=modified_files,
-            s3_report_key="python-poc/tests/history/",
-            use_remote_secrets=ctx.obj["use_remote_secrets"],
-            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
-            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
-            ci_context=ctx.obj.get("ci_context"),
-            pull_request=ctx.obj.get("pull_request"),
-        )
-        for connector, modified_files in ctx.obj["selected_connectors_and_files"].items()
-    ]
-    try:
-        anyio.run(
-            run_connectors_pipelines,
-            connectors_tests_contexts,
-            run_connector_test_pipeline,
-            "Test Pipeline",
-            ctx.obj["concurrency"],
-            ctx.obj["execute_timeout"],
-        )
-    except Exception as e:
-        click.secho(str(e), err=True, fg="red")
-        update_global_commit_status_check_for_tests(ctx.obj, "failure")
-        return False
-    global_success = all(connector_context.state is ContextState.SUCCESSFUL for connector_context in connectors_tests_contexts)
-    update_global_commit_status_check_for_tests(ctx.obj, "success" if global_success else "failure")
-    # If we reach this point, it means that all the connectors have been tested so the pipeline did its job and can exit with success.
-    return True
-
-
-@connectors.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.")
-@click.pass_context
-def build(ctx: click.Context) -> bool:
-    click.secho(f"Will build the following connectors: {', '.join(ctx.obj['selected_connectors_names'])}.", fg="green")
-    connectors_contexts = [
-        ConnectorContext(
-            pipeline_name="Build connector {connector.technical_name}",
-            connector=connector,
-            is_local=ctx.obj["is_local"],
-            git_branch=ctx.obj["git_branch"],
-            git_revision=ctx.obj["git_revision"],
-            modified_files=modified_files,
-            s3_report_key="python-poc/build/history/",
-            use_remote_secrets=ctx.obj["use_remote_secrets"],
-            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
-            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
-            ci_context=ctx.obj.get("ci_context"),
-        )
-        for connector, modified_files in ctx.obj["selected_connectors_and_files"].items()
-    ]
-    anyio.run(
-        run_connectors_pipelines,
-        connectors_contexts,
-        run_connector_build_pipeline,
-        "Build Pipeline",
-        ctx.obj["concurrency"],
-        ctx.obj["execute_timeout"],
-    )
-
-    return True
-
-
-@connectors.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.")
-@click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool)
-@click.option(
-    "--spec-cache-gcs-credentials",
-    help="The service account key to upload files to the GCS bucket hosting spec cache.",
-    type=click.STRING,
-    required=False,  # Not required for pre-release pipelines, downstream validation happens for main release pipelines
-    envvar="SPEC_CACHE_GCS_CREDENTIALS",
-)
-@click.option(
-    "--spec-cache-bucket-name",
-    help="The name of the GCS bucket where specs will be cached.",
-    type=click.STRING,
-    required=False,  # Not required for pre-release pipelines, downstream validation happens for main release pipelines
-    envvar="SPEC_CACHE_BUCKET_NAME",
-)
-@click.option(
-    "--metadata-service-gcs-credentials",
-    help="The service account key to upload files to the GCS bucket hosting the metadata files.",
-    type=click.STRING,
-    required=False,  # Not required for pre-release pipelines, downstream validation happens for main release pipelines
-    envvar="METADATA_SERVICE_GCS_CREDENTIALS",
-)
-@click.option(
-    "--metadata-service-bucket-name",
-    help="The name of the GCS bucket where metadata files will be uploaded.",
-    type=click.STRING,
-    required=False,  # Not required for pre-release pipelines, downstream validation happens for main release pipelines
-    envvar="METADATA_SERVICE_BUCKET_NAME",
-)
-@click.option(
-    "--docker-hub-username",
-    help="Your username to connect to DockerHub.",
-    type=click.STRING,
-    required=True,
-    envvar="DOCKER_HUB_USERNAME",
-)
-@click.option(
-    "--docker-hub-password",
-    help="Your password to connect to DockerHub.",
-    type=click.STRING,
-    required=True,
-    envvar="DOCKER_HUB_PASSWORD",
-)
-@click.option(
-    "--slack-webhook",
-    help="The Slack webhook URL to send notifications to.",
-    type=click.STRING,
-    envvar="SLACK_WEBHOOK",
-)
-@click.option(
-    "--slack-channel",
-    help="The Slack webhook URL to send notifications to.",
-    type=click.STRING,
-    envvar="SLACK_CHANNEL",
-    default="#publish-on-merge-updates",
-)
-@click.pass_context
-def publish(
-    ctx: click.Context,
-    pre_release: bool,
-    spec_cache_gcs_credentials: str,
-    spec_cache_bucket_name: str,
-    metadata_service_bucket_name: str,
-    metadata_service_gcs_credentials: str,
-    docker_hub_username: str,
-    docker_hub_password: str,
-    slack_webhook: str,
-    slack_channel: str,
-):
-    ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials
-    ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name
-    ctx.obj["metadata_service_bucket_name"] = metadata_service_bucket_name
-    ctx.obj["metadata_service_gcs_credentials"] = metadata_service_gcs_credentials
-    validate_publish_options(pre_release, ctx.obj)
-    if ctx.obj["is_local"]:
-        click.confirm(
-            "Publishing from a local environment is not recommend and requires to be logged in Airbyte's DockerHub registry, do you want to continue?",
-            abort=True,
-        )
-    if ctx.obj["modified"]:
-        selected_connectors_and_files = get_modified_connectors(get_modified_metadata_files(ctx.obj["modified_files"]))
-        selected_connectors_names = [connector.technical_name for connector in selected_connectors_and_files.keys()]
-    else:
-        selected_connectors_and_files = ctx.obj["selected_connectors_and_files"]
-        selected_connectors_names = ctx.obj["selected_connectors_names"]
-
-    click.secho(f"Will publish the following connectors: {', '.join(selected_connectors_names)}.", fg="green")
-
-    publish_connector_contexts = reorder_contexts([
-        PublishConnectorContext(
-            connector,
-            pre_release,
-            modified_files,
-            spec_cache_gcs_credentials,
-            spec_cache_bucket_name,
-            metadata_service_gcs_credentials,
-            metadata_service_bucket_name,
-            docker_hub_username,
-            docker_hub_password,
-            slack_webhook,
-            slack_channel,
-            ctx.obj["is_local"],
-            ctx.obj["git_branch"],
-            ctx.obj["git_revision"],
-            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
-            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
-            ci_context=ctx.obj.get("ci_context"),
-            pull_request=ctx.obj.get("pull_request"),
-        )
-        for connector, modified_files in selected_connectors_and_files.items()
-    ])
-
-    click.secho("Concurrency is forced to 1. For stability reasons we disable parallel publish pipelines.", fg="yellow")
-    ctx.obj["concurrency"] = 1
-    publish_connector_contexts = anyio.run(
-        run_connectors_pipelines,
-        publish_connector_contexts,
-        run_connector_publish_pipeline,
-        "Publish pipeline",
-        ctx.obj["concurrency"],
-        ctx.obj["execute_timeout"],
-    )
-    return all(context.state is ContextState.SUCCESSFUL for context in publish_connector_contexts)
-
-
-def validate_publish_options(pre_release: bool, context_object: Dict[str, Any]):
-    """Validate that the publish options are set correctly."""
-    for k in ["spec_cache_bucket_name", "spec_cache_gcs_credentials", "metadata_service_bucket_name", "metadata_service_gcs_credentials"]:
-        if not pre_release and context_object.get(k) is None:
-            click.Abort(f'The --{k.replace("_", "-")} option is required when running a main release publish pipeline.')
-
-
-@connectors.command(cls=DaggerPipelineCommand, help="List all selected connectors.")
-@click.pass_context
-def list(
-    ctx: click.Context,
-):
-    selected_connectors = [
-        (connector, bool(modified_files))
-        for connector, modified_files in ctx.obj["selected_connectors_and_files"].items()
-        if connector.metadata
-    ]
-
-    selected_connectors = sorted(selected_connectors, key=lambda x: x[0].technical_name)
-    table = Table(title=f"{len(selected_connectors)} selected connectors")
-    table.add_column("Modified")
-    table.add_column("Connector")
-    table.add_column("Language")
-    table.add_column("Release stage")
-    table.add_column("Version")
-    table.add_column("Folder")
-
-    for connector, modified in selected_connectors:
-        modified = "X" if modified else ""
-        connector_name = Text(connector.technical_name)
-        language = Text(connector.language.value) if connector.language else "N/A"
-        release_stage = Text(connector.release_stage)
-        version = Text(connector.version)
-        folder = Text(str(connector.code_directory))
-        table.add_row(modified, connector_name, language, release_stage, version, folder)
-
-    console.print(table)
-    return True
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/metadata.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/metadata.py
deleted file mode 100644
index 1328e81fc62c..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/commands/groups/metadata.py
+++ /dev/null
@@ -1,150 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import logging
-
-import anyio
-import click
-from ci_connector_ops.pipelines.bases import CIContext
-from ci_connector_ops.pipelines.pipelines.metadata import (
-    run_metadata_lib_test_pipeline,
-    run_metadata_orchestrator_deploy_pipeline,
-    run_metadata_orchestrator_test_pipeline,
-    run_metadata_upload_pipeline,
-    run_metadata_validation_pipeline,
-)
-from ci_connector_ops.pipelines.utils import DaggerPipelineCommand, get_all_metadata_files, get_modified_metadata_files
-from rich.logging import RichHandler
-
-logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s", datefmt="[%X]", handlers=[RichHandler(rich_tracebacks=True)])
-logger = logging.getLogger(__name__)
-
-
-# MAIN GROUP
-
-
-@click.group(help="Commands related to the metadata service.")
-@click.pass_context
-def metadata(ctx: click.Context):
-    pass
-
-
-# VALIDATE COMMAND
-
-
-@metadata.command(cls=DaggerPipelineCommand, help="Commands related to validating the metadata files.")
-@click.option("--modified-only/--all", default=True)
-@click.pass_context
-def validate(ctx: click.Context, modified_only: bool) -> bool:
-    if modified_only:
-        metadata_to_validate = get_modified_metadata_files(ctx.obj["modified_files"])
-    else:
-        click.secho("Will run metadata validation on all the metadata files found in the repo.")
-        metadata_to_validate = get_all_metadata_files()
-
-    click.secho(f"Will validate {len(metadata_to_validate)} metadata files.")
-
-    return anyio.run(
-        run_metadata_validation_pipeline,
-        ctx.obj["is_local"],
-        ctx.obj["git_branch"],
-        ctx.obj["git_revision"],
-        ctx.obj.get("gha_workflow_run_url"),
-        ctx.obj.get("pipeline_start_timestamp"),
-        ctx.obj.get("ci_context"),
-        metadata_to_validate,
-    )
-
-
-# UPLOAD COMMAND
-
-
-@metadata.command(cls=DaggerPipelineCommand, help="Commands related to uploading the metadata files to remote storage.")
-@click.argument("gcs-bucket-name", type=click.STRING)
-@click.option("--modified-only/--all", default=True)
-@click.pass_context
-def upload(ctx: click.Context, gcs_bucket_name: str, modified_only: bool) -> bool:
-    if modified_only:
-        if ctx.obj["ci_context"] is not CIContext.MASTER and ctx.obj["git_branch"] != "master":
-            click.secho("Not on the master branch. Skipping metadata upload.")
-            return True
-        metadata_to_upload = get_modified_metadata_files(ctx.obj["modified_files"])
-        if not metadata_to_upload:
-            click.secho("No modified metadata found. Skipping metadata upload.")
-            return True
-    else:
-        metadata_to_upload = get_all_metadata_files()
-
-    click.secho(f"Will upload {len(metadata_to_upload)} metadata files.")
-
-    return anyio.run(
-        run_metadata_upload_pipeline,
-        ctx.obj["is_local"],
-        ctx.obj["git_branch"],
-        ctx.obj["git_revision"],
-        ctx.obj.get("gha_workflow_run_url"),
-        ctx.obj.get("pipeline_start_timestamp"),
-        ctx.obj.get("ci_context"),
-        metadata_to_upload,
-        gcs_bucket_name,
-    )
-
-
-# DEPLOY GROUP
-
-
-@metadata.group(help="Commands related to deploying components of the metadata service.")
-@click.pass_context
-def deploy(ctx: click.Context):
-    pass
-
-
-@deploy.command(cls=DaggerPipelineCommand, name="orchestrator", help="Deploy the metadata service orchestrator to production")
-@click.pass_context
-def deploy_orchestrator(ctx: click.Context) -> bool:
-    return anyio.run(
-        run_metadata_orchestrator_deploy_pipeline,
-        ctx.obj["is_local"],
-        ctx.obj["git_branch"],
-        ctx.obj["git_revision"],
-        ctx.obj.get("gha_workflow_run_url"),
-        ctx.obj.get("pipeline_start_timestamp"),
-        ctx.obj.get("ci_context"),
-    )
-
-
-# TEST GROUP
-
-
-@metadata.group(help="Commands related to testing the metadata service.")
-@click.pass_context
-def test(ctx: click.Context):
-    pass
-
-
-@test.command(cls=DaggerPipelineCommand, name="lib", help="Run tests for the metadata service library.")
-@click.pass_context
-def test_lib(ctx: click.Context) -> bool:
-    return anyio.run(
-        run_metadata_lib_test_pipeline,
-        ctx.obj["is_local"],
-        ctx.obj["git_branch"],
-        ctx.obj["git_revision"],
-        ctx.obj.get("gha_workflow_run_url"),
-        ctx.obj.get("pipeline_start_timestamp"),
-        ctx.obj.get("ci_context"),
-    )
-
-
-@test.command(cls=DaggerPipelineCommand, name="orchestrator", help="Run tests for the metadata service orchestrator.")
-@click.pass_context
-def test_orchestrator(ctx: click.Context) -> bool:
-    return anyio.run(
-        run_metadata_orchestrator_test_pipeline,
-        ctx.obj["is_local"],
-        ctx.obj["git_branch"],
-        ctx.obj["git_revision"],
-        ctx.obj.get("gha_workflow_run_url"),
-        ctx.obj.get("pipeline_start_timestamp"),
-        ctx.obj.get("ci_context"),
-    )
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py
deleted file mode 100644
index ff545df3bc65..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/consts.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import platform
-
-from dagger import Platform
-
-PYPROJECT_TOML_FILE_PATH = "pyproject.toml"
-
-CONNECTOR_TESTING_REQUIREMENTS = [
-    "pip==21.3.1",
-    "mccabe==0.6.1",
-    "flake8==4.0.1",
-    "pyproject-flake8==0.0.1a2",
-    "black==22.3.0",
-    "isort==5.6.4",
-    "pytest==6.2.5",
-    "coverage[toml]==6.3.1",
-    "pytest-custom_exit_code",
-]
-
-DEFAULT_PYTHON_EXCLUDE = ["**/.venv", "**/__pycache__"]
-CI_CREDENTIALS_SOURCE_PATH = "tools/ci_credentials"
-CI_CONNECTOR_OPS_SOURCE_PATH = "tools/ci_connector_ops"
-BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")]
-LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}")
-DOCKER_VERSION = "20.10.23"
-DOCKER_DIND_IMAGE = "docker:20-dind"
-DOCKER_CLI_IMAGE = "docker:20-cli"
-GRADLE_CACHE_PATH = "/root/.gradle/caches"
-GRADLE_BUILD_CACHE_PATH = f"{GRADLE_CACHE_PATH}/build-cache-1"
-GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH = "/root/gradle_dependency_cache"
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/github.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/github.py
deleted file mode 100644
index d1bddbc7f6e7..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/github.py
+++ /dev/null
@@ -1,102 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""Module grouping functions interacting with the GitHub API."""
-
-from __future__ import annotations
-
-import os
-from typing import TYPE_CHECKING, Optional
-
-from ci_connector_ops.pipelines.bases import CIContext
-from ci_connector_ops.utils import console
-
-if TYPE_CHECKING:
-    from logging import Logger
-
-from github import Github, PullRequest
-
-AIRBYTE_GITHUB_REPO = "airbytehq/airbyte"
-GITHUB_GLOBAL_CONTEXT_FOR_TESTS = "Connectors CI tests"
-GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS = "Running connectors tests"
-
-
-def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> None:
-    """Log a message to a logger if one is available, otherwise print to the console."""
-    if logger:
-        log_method = getattr(logger, level.lower())
-        log_method(message)
-    else:
-        console.print(message)
-
-
-def update_commit_status_check(
-    sha: str, state: str, target_url: str, description: str, context: str, is_optional=False, should_send=True, logger: Logger = None
-):
-    """Call the GitHub API to create commit status check.
-
-    Args:
-        sha (str): Hash of the commit for which you want to create a status check.
-        state (str): The check state (success, failure, pending)
-        target_url (str): The URL to attach to the commit check for details.
-        description (str): Description of the check that is run.
-        context (str): Name of the Check context e.g: source-pokeapi tests
-        should_send (bool, optional): Whether the commit check should actually be sent to GitHub API. Defaults to True.
-        logger (Logger, optional): A logger to log info about updates. Defaults to None.
-    """
-    if not should_send:
-        return
-
-    safe_log(logger, f"Attempting to create {state} status for commit {sha} on Github in {context} context.")
-    try:
-        github_client = Github(os.environ["CI_GITHUB_ACCESS_TOKEN"])
-        airbyte_repo = github_client.get_repo(AIRBYTE_GITHUB_REPO)
-    except Exception as e:
-        if logger:
-            logger.error("No commit status check sent, the connection to Github API failed", exc_info=True)
-        else:
-            console.print(e)
-        return
-
-    # If the check is optional, we don't want to fail the build if it fails.
-    # Instead, we want to mark it as a warning.
-    # Unfortunately, Github doesn't have a warning state, so we use success instead.
-    if is_optional and state == "failure":
-        state = "success"
-        description = f"[WARNING] optional check failed {context}: {description}"
-
-    context = context if bool(os.environ.get("PRODUCTION", False)) is True else f"[please ignore] {context}"
-    airbyte_repo.get_commit(sha=sha).create_status(
-        state=state,
-        target_url=target_url,
-        description=description,
-        context=context,
-    )
-    safe_log(logger, f"Created {state} status for commit {sha} on Github in {context} context with desc: {description}.")
-
-
-def get_pull_request(pull_request_number: int, github_access_token: str) -> PullRequest:
-    """Get a pull request object from its number.
-
-    Args:
-        pull_request_number (str): The number of the pull request to get.
-        github_access_token (str): The GitHub access token to use to authenticate.
-    Returns:
-        PullRequest: The pull request object.
-    """
-    github_client = Github(github_access_token)
-    airbyte_repo = github_client.get_repo(AIRBYTE_GITHUB_REPO)
-    return airbyte_repo.get_pull(pull_request_number)
-
-
-def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Logger = None):
-    update_commit_status_check(
-        click_context["git_revision"],
-        github_state,
-        click_context["gha_workflow_run_url"],
-        GITHUB_GLOBAL_DESCRIPTION_FOR_TESTS,
-        GITHUB_GLOBAL_CONTEXT_FOR_TESTS,
-        should_send=click_context.get("ci_context") == CIContext.PULL_REQUEST,
-        logger=logger,
-    )
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py
deleted file mode 100644
index 6a28a7180c49..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/gradle.py
+++ /dev/null
@@ -1,122 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from __future__ import annotations
-
-from abc import ABC
-from typing import ClassVar, List, Tuple
-
-from ci_connector_ops.pipelines import consts
-from ci_connector_ops.pipelines.actions import environments
-from ci_connector_ops.pipelines.bases import Step, StepResult
-from ci_connector_ops.pipelines.utils import slugify
-from dagger import CacheVolume, Container, Directory
-
-
-class GradleTask(Step, ABC):
-    """
-    A step to run a Gradle task.
-
-    Attributes:
-        task_name (str): The Gradle task name to run.
-        title (str): The step title.
-    """
-
-    DEFAULT_TASKS_TO_EXCLUDE = ["airbyteDocker"]
-    BIND_TO_DOCKER_HOST = True
-    gradle_task_name: ClassVar
-
-    # These are the lines we remove from the connector gradle file to ignore specific tasks / plugins.
-    LINES_TO_REMOVE_FROM_GRADLE_FILE = [
-        # Do not build normalization with Gradle - we build normalization with Dagger in the BuildOrPullNormalization step.
-        "project(':airbyte-integrations:bases:base-normalization').airbyteDocker.output",
-    ]
-
-    @property
-    def docker_service_name(self) -> str:
-        return slugify(f"gradle-{self.title}")
-
-    @property
-    def connector_java_build_cache(self) -> CacheVolume:
-        return self.context.dagger_client.cache_volume("connector_java_build_cache")
-
-    @property
-    def build_include(self) -> List[str]:
-        """Retrieve the list of source code directory required to run a Java connector Gradle task.
-
-        The list is different according to the connector type.
-
-        Returns:
-            List[str]: List of directories or files to be mounted to the container to run a Java connector Gradle task.
-        """
-        return [
-            str(dependency_directory)
-            for dependency_directory in self.context.connector.get_local_dependencies_paths(with_test_dependencies=True)
-        ]
-
-    async def _get_patched_connector_dir(self) -> Directory:
-        """Patch the build.gradle file of the connector under test by removing the lines declared in LINES_TO_REMOVE_FROM_GRADLE_FILE.
-
-        Returns:
-            Directory: The patched connector directory
-        """
-
-        gradle_file_content = await self.context.get_connector_dir(include=["build.gradle"]).file("build.gradle").contents()
-        patched_file_content = ""
-        for line in gradle_file_content.split("\n"):
-            if not any(line_to_remove in line for line_to_remove in self.LINES_TO_REMOVE_FROM_GRADLE_FILE):
-                patched_file_content += line + "\n"
-        return self.context.get_connector_dir().with_new_file("build.gradle", patched_file_content)
-
-    def _get_gradle_command(self, extra_options: Tuple[str] = ("--no-daemon", "--scan", "--build-cache")) -> List:
-        command = (
-            ["./gradlew"]
-            + list(extra_options)
-            + [f":airbyte-integrations:connectors:{self.context.connector.technical_name}:{self.gradle_task_name}"]
-        )
-        for task in self.DEFAULT_TASKS_TO_EXCLUDE:
-            command += ["-x", task]
-        return command
-
-    async def _run(self) -> StepResult:
-        connector_under_test = (
-            environments.with_gradle(
-                self.context, self.build_include, docker_service_name=self.docker_service_name, bind_to_docker_host=self.BIND_TO_DOCKER_HOST
-            )
-            .with_mounted_directory(str(self.context.connector.code_directory), await self._get_patched_connector_dir())
-            # Disable the Ryuk container because it needs privileged docker access that does not work:
-            .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true")
-            .with_directory(f"{self.context.connector.code_directory}/secrets", self.context.secrets_dir)
-            .with_exec(self._get_gradle_command())
-        )
-        results = await self.get_step_result(connector_under_test)
-        await self._export_gradle_dependency_cache(connector_under_test)
-        return results
-
-    async def _export_gradle_dependency_cache(self, gradle_container: Container) -> Container:
-        """Export the Gradle writable dependency cache to the read-only dependency cache path.
-        The read-only dependency cache is persisted thanks to mounted cache volumes in environments.with_gradle().
-        You can read more about Shared readonly cache here: https://docs.gradle.org/current/userguide/dependency_resolution.html#sub:shared-readonly-cache
-        Args:
-            gradle_container (Container): The Gradle container.
-
-        Returns:
-            Container: The Gradle container, with the updated cache.
-        """
-        with_cache = gradle_container.with_exec(
-            [
-                "rsync",
-                "--archive",
-                "--quiet",
-                "--times",
-                "--exclude",
-                "*.lock",
-                "--exclude",
-                "gc.properties",
-                f"{consts.GRADLE_CACHE_PATH}/modules-2/",
-                f"{consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH}/modules-2/",
-            ]
-        )
-        await with_cache.exit_code()
-        return with_cache
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py
deleted file mode 100644
index 2f3f3f6ed38d..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/connectors.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-"""This module groups the functions to run full pipelines for connector testing."""
-
-import sys
-from typing import Callable, List, Optional
-
-import anyio
-import dagger
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from dagger import Config
-
-GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI"
-GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests"
-
-
-async def run_connectors_pipelines(
-    contexts: List[ConnectorContext],
-    connector_pipeline: Callable,
-    pipeline_name: str,
-    concurrency: int,
-    execute_timeout: Optional[int],
-    *args,
-) -> List[ConnectorContext]:
-    """Run a connector pipeline for all the connector contexts."""
-    semaphore = anyio.Semaphore(concurrency)
-    async with dagger.Connection(Config(log_output=sys.stderr, execute_timeout=execute_timeout)) as dagger_client:
-        async with anyio.create_task_group() as tg:
-            for context in contexts:
-                context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}")
-                tg.start_soon(connector_pipeline, context, semaphore, *args)
-    return contexts
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/metadata.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/metadata.py
deleted file mode 100644
index f18bbd876667..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/pipelines/metadata.py
+++ /dev/null
@@ -1,326 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-import uuid
-from pathlib import Path
-from typing import Optional, Set
-
-import dagger
-from ci_connector_ops.pipelines.actions import run_steps
-from ci_connector_ops.pipelines.actions.environments import with_pip_packages, with_poetry_module, with_python_base
-from ci_connector_ops.pipelines.bases import Report, Step, StepResult
-from ci_connector_ops.pipelines.contexts import PipelineContext
-from ci_connector_ops.pipelines.utils import DAGGER_CONFIG, METADATA_FILE_NAME, METADATA_ICON_FILE_NAME, execute_concurrently
-
-METADATA_DIR = "airbyte-ci/connectors/metadata_service"
-METADATA_LIB_MODULE_PATH = "lib"
-METADATA_ORCHESTRATOR_MODULE_PATH = "orchestrator"
-
-# HELPERS
-
-
-def get_metadata_file_from_path(context: PipelineContext, metadata_path: Path) -> dagger.File:
-    if metadata_path.is_file() and metadata_path.name != METADATA_FILE_NAME:
-        raise ValueError(f"The metadata file name is not {METADATA_FILE_NAME}, it is {metadata_path.name} .")
-    if metadata_path.is_dir():
-        metadata_path = metadata_path / METADATA_FILE_NAME
-    if not metadata_path.exists():
-        raise FileNotFoundError(f"{str(metadata_path)} does not exist.")
-    return context.get_repo_dir(str(metadata_path.parent), include=[METADATA_FILE_NAME]).file(METADATA_FILE_NAME)
-
-
-def get_metadata_icon_file_from_path(context: PipelineContext, metadata_icon_path: Path) -> dagger.File:
-    return context.get_repo_dir(str(metadata_icon_path.parent), include=[METADATA_ICON_FILE_NAME]).file(METADATA_ICON_FILE_NAME)
-
-
-# STEPS
-
-
-class PoetryRun(Step):
-    def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, module_path: str):
-        self.title = title
-        super().__init__(context)
-        self.parent_dir = self.context.get_repo_dir(parent_dir_path)
-        self.module_path = module_path
-        self.poetry_run_container = with_poetry_module(self.context, self.parent_dir, self.module_path).with_entrypoint(["poetry", "run"])
-
-    async def _run(self, poetry_run_args: list) -> StepResult:
-        poetry_run_exec = self.poetry_run_container.with_exec(poetry_run_args)
-        return await self.get_step_result(poetry_run_exec)
-
-
-class MetadataValidation(PoetryRun):
-    def __init__(self, context: PipelineContext, metadata_path: Path):
-        title = f"Validate {metadata_path}"
-        super().__init__(context, title, METADATA_DIR, METADATA_LIB_MODULE_PATH)
-        self.poetry_run_container = self.poetry_run_container.with_mounted_file(
-            METADATA_FILE_NAME, get_metadata_file_from_path(context, metadata_path)
-        )
-
-    async def _run(self) -> StepResult:
-        return await super()._run(["metadata_service", "validate", METADATA_FILE_NAME])
-
-
-class MetadataUpload(PoetryRun):
-    def __init__(
-        self,
-        context: PipelineContext,
-        metadata_path: Path,
-        metadata_bucket_name: str,
-        metadata_service_gcs_credentials_secret: dagger.Secret,
-        docker_hub_username_secret: dagger.Secret,
-        docker_hub_password_secret: dagger.Secret,
-    ):
-        title = f"Upload {metadata_path}"
-        self.gcs_bucket_name = metadata_bucket_name
-        super().__init__(context, title, METADATA_DIR, METADATA_LIB_MODULE_PATH)
-
-        # Ensure the icon file is included in the upload
-        base_container = self.poetry_run_container.with_file(METADATA_FILE_NAME, get_metadata_file_from_path(context, metadata_path))
-        metadata_icon_path = metadata_path.parent / METADATA_ICON_FILE_NAME
-        if metadata_icon_path.exists():
-            base_container = base_container.with_file(
-                METADATA_ICON_FILE_NAME, get_metadata_icon_file_from_path(context, metadata_icon_path)
-            )
-
-        self.poetry_run_container = (
-            base_container.with_secret_variable("DOCKER_HUB_USERNAME", docker_hub_username_secret)
-            .with_secret_variable("DOCKER_HUB_PASSWORD", docker_hub_password_secret)
-            .with_secret_variable("GCS_CREDENTIALS", metadata_service_gcs_credentials_secret)
-            # The cache buster ensures we always run the upload command (in case of remote bucket change)
-            .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
-        )
-
-    async def _run(self) -> StepResult:
-        return await super()._run(
-            [
-                "metadata_service",
-                "upload",
-                METADATA_FILE_NAME,
-                self.gcs_bucket_name,
-            ]
-        )
-
-
-class DeployOrchestrator(Step):
-    title = "Deploy Metadata Orchestrator to Dagster Cloud"
-    deploy_dagster_command = [
-        "dagster-cloud",
-        "serverless",
-        "deploy-python-executable",
-        "--location-name",
-        "metadata_service_orchestrator",
-        "--location-file",
-        "dagster_cloud.yaml",
-        "--organization",
-        "airbyte-connectors",
-        "--deployment",
-        "prod",
-        "--python-version",
-        "3.9",
-    ]
-
-    async def _run(self) -> StepResult:
-        parent_dir = self.context.get_repo_dir(METADATA_DIR)
-        python_base = with_python_base(self.context)
-        python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud==1.2.6", "poetry2setup==1.1.0"])
-        dagster_cloud_api_token_secret: dagger.Secret = (
-            self.context.dagger_client.host().env_variable("DAGSTER_CLOUD_METADATA_API_TOKEN").secret()
-        )
-
-        container_to_run = (
-            python_with_dependencies.with_mounted_directory("/src", parent_dir)
-            .with_secret_variable("DAGSTER_CLOUD_API_TOKEN", dagster_cloud_api_token_secret)
-            .with_workdir(f"/src/{METADATA_ORCHESTRATOR_MODULE_PATH}")
-            .with_exec(["/bin/sh", "-c", "poetry2setup >> setup.py"])
-            .with_exec(self.deploy_dagster_command)
-        )
-        return await self.get_step_result(container_to_run)
-
-
-class TestOrchestrator(PoetryRun):
-    def __init__(self, context: PipelineContext):
-        super().__init__(
-            context=context,
-            title="Test Metadata Orchestrator",
-            parent_dir_path=METADATA_DIR,
-            module_path=METADATA_ORCHESTRATOR_MODULE_PATH,
-        )
-
-    async def _run(self) -> StepResult:
-        return await super()._run(["pytest"])
-
-
-# PIPELINES
-
-
-async def run_metadata_validation_pipeline(
-    is_local: bool,
-    git_branch: str,
-    git_revision: str,
-    gha_workflow_run_url: Optional[str],
-    pipeline_start_timestamp: Optional[int],
-    ci_context: Optional[str],
-    metadata_to_validate: Set[Path],
-) -> bool:
-    metadata_pipeline_context = PipelineContext(
-        pipeline_name="Validate metadata.yaml files",
-        is_local=is_local,
-        git_branch=git_branch,
-        git_revision=git_revision,
-        gha_workflow_run_url=gha_workflow_run_url,
-        pipeline_start_timestamp=pipeline_start_timestamp,
-        ci_context=ci_context,
-    )
-
-    async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
-        metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name)
-        async with metadata_pipeline_context:
-            validation_steps = [MetadataValidation(metadata_pipeline_context, metadata_path).run for metadata_path in metadata_to_validate]
-
-            results = await execute_concurrently(validation_steps, concurrency=10)
-            metadata_pipeline_context.report = Report(
-                pipeline_context=metadata_pipeline_context, steps_results=results, name="METADATA VALIDATION RESULTS"
-            )
-
-        return metadata_pipeline_context.report.success
-
-
-async def run_metadata_lib_test_pipeline(
-    is_local: bool,
-    git_branch: str,
-    git_revision: str,
-    gha_workflow_run_url: Optional[str],
-    pipeline_start_timestamp: Optional[int],
-    ci_context: Optional[str],
-) -> bool:
-    metadata_pipeline_context = PipelineContext(
-        pipeline_name="Metadata Service Lib Unit Test Pipeline",
-        is_local=is_local,
-        git_branch=git_branch,
-        git_revision=git_revision,
-        gha_workflow_run_url=gha_workflow_run_url,
-        pipeline_start_timestamp=pipeline_start_timestamp,
-        ci_context=ci_context,
-    )
-
-    async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
-        metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name)
-        async with metadata_pipeline_context:
-            test_lib_step = PoetryRun(
-                context=metadata_pipeline_context,
-                title="Test Metadata Service Lib",
-                parent_dir_path=METADATA_DIR,
-                module_path=METADATA_LIB_MODULE_PATH,
-            )
-            result = await test_lib_step.run(["pytest"])
-            metadata_pipeline_context.report = Report(
-                pipeline_context=metadata_pipeline_context, steps_results=[result], name="METADATA LIB TEST RESULTS"
-            )
-
-    return metadata_pipeline_context.report.success
-
-
-async def run_metadata_orchestrator_test_pipeline(
-    is_local: bool,
-    git_branch: str,
-    git_revision: str,
-    gha_workflow_run_url: Optional[str],
-    pipeline_start_timestamp: Optional[int],
-    ci_context: Optional[str],
-) -> bool:
-    metadata_pipeline_context = PipelineContext(
-        pipeline_name="Metadata Service Orchestrator Unit Test Pipeline",
-        is_local=is_local,
-        git_branch=git_branch,
-        git_revision=git_revision,
-        gha_workflow_run_url=gha_workflow_run_url,
-        pipeline_start_timestamp=pipeline_start_timestamp,
-        ci_context=ci_context,
-    )
-
-    async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
-        metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name)
-        async with metadata_pipeline_context:
-            test_orch_step = TestOrchestrator(context=metadata_pipeline_context)
-            result = await test_orch_step.run()
-            metadata_pipeline_context.report = Report(
-                pipeline_context=metadata_pipeline_context, steps_results=[result], name="METADATA ORCHESTRATOR TEST RESULTS"
-            )
-
-    return metadata_pipeline_context.report.success
-
-
-async def run_metadata_upload_pipeline(
-    is_local: bool,
-    git_branch: str,
-    git_revision: str,
-    gha_workflow_run_url: Optional[str],
-    pipeline_start_timestamp: Optional[int],
-    ci_context: Optional[str],
-    metadata_to_upload: Set[Path],
-    gcs_bucket_name: str,
-) -> bool:
-    pipeline_context = PipelineContext(
-        pipeline_name="Metadata Upload Pipeline",
-        is_local=is_local,
-        git_branch=git_branch,
-        git_revision=git_revision,
-        gha_workflow_run_url=gha_workflow_run_url,
-        pipeline_start_timestamp=pipeline_start_timestamp,
-        ci_context=ci_context,
-    )
-
-    async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
-        pipeline_context.dagger_client = dagger_client.pipeline(pipeline_context.pipeline_name)
-        async with pipeline_context:
-            gcs_credentials_secret: dagger.Secret = pipeline_context.dagger_client.host().env_variable("GCS_CREDENTIALS").secret()
-            docker_hub_username_secret: dagger.Secret = pipeline_context.dagger_client.host().env_variable("DOCKER_HUB_USERNAME").secret()
-            docker_hub_password_secret: dagger.Secret = pipeline_context.dagger_client.host().env_variable("DOCKER_HUB_PASSWORD").secret()
-
-            results = await execute_concurrently(
-                [
-                    MetadataUpload(
-                        context=pipeline_context,
-                        metadata_service_gcs_credentials_secret=gcs_credentials_secret,
-                        docker_hub_username_secret=docker_hub_username_secret,
-                        docker_hub_password_secret=docker_hub_password_secret,
-                        metadata_bucket_name=gcs_bucket_name,
-                        metadata_path=metadata_path,
-                    ).run
-                    for metadata_path in metadata_to_upload
-                ]
-            )
-            pipeline_context.report = Report(pipeline_context, results, name="METADATA UPLOAD RESULTS")
-
-        return pipeline_context.report.success
-
-
-async def run_metadata_orchestrator_deploy_pipeline(
-    is_local: bool,
-    git_branch: str,
-    git_revision: str,
-    gha_workflow_run_url: Optional[str],
-    pipeline_start_timestamp: Optional[int],
-    ci_context: Optional[str],
-) -> bool:
-    metadata_pipeline_context = PipelineContext(
-        pipeline_name="Metadata Service Orchestrator Unit Test Pipeline",
-        is_local=is_local,
-        git_branch=git_branch,
-        git_revision=git_revision,
-        gha_workflow_run_url=gha_workflow_run_url,
-        pipeline_start_timestamp=pipeline_start_timestamp,
-        ci_context=ci_context,
-    )
-
-    async with dagger.Connection(DAGGER_CONFIG) as dagger_client:
-        metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name)
-
-        async with metadata_pipeline_context:
-            steps = [TestOrchestrator(context=metadata_pipeline_context), DeployOrchestrator(context=metadata_pipeline_context)]
-            steps_results = await run_steps(steps)
-            metadata_pipeline_context.report = Report(
-                pipeline_context=metadata_pipeline_context, steps_results=steps_results, name="METADATA ORCHESTRATOR DEPLOY RESULTS"
-            )
-    return metadata_pipeline_context.report.success
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/slack.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/slack.py
deleted file mode 100644
index f2e6f5597251..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/slack.py
+++ /dev/null
@@ -1,14 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import json
-
-import requests
-
-
-def send_message_to_webhook(message: str, channel: str, webhook: str) -> dict:
-    payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message}
-    response = requests.post(webhook, data={"payload": json.dumps(payload)})
-    response.raise_for_status()
-    return response
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py
deleted file mode 100644
index a8b41bafa887..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/__init__.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-"""This module groups factory like functions to dispatch tests steps according to the connector under test language."""
-
-import itertools
-from typing import List
-
-import anyio
-import asyncer
-from ci_connector_ops.pipelines.bases import ConnectorReport, StepResult
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from ci_connector_ops.pipelines.pipelines.metadata import MetadataValidation
-from ci_connector_ops.pipelines.tests import java_connectors, python_connectors
-from ci_connector_ops.pipelines.tests.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck
-from ci_connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage
-
-LANGUAGE_MAPPING = {
-    "run_all_tests": {
-        ConnectorLanguage.PYTHON: python_connectors.run_all_tests,
-        ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests,
-        ConnectorLanguage.JAVA: java_connectors.run_all_tests,
-    },
-    "run_code_format_checks": {
-        ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks,
-        ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks,
-        # ConnectorLanguage.JAVA: java_connectors.run_code_format_checks
-    },
-}
-
-
-async def run_metadata_validation(context: ConnectorContext) -> List[StepResult]:
-    """Run the metadata validation on a connector.
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of the metadata validation steps.
-    """
-    context.logger.info("Run metadata validation.")
-    return [await MetadataValidation(context, context.connector.code_directory / METADATA_FILE_NAME).run()]
-
-
-async def run_version_checks(context: ConnectorContext) -> List[StepResult]:
-    """Run the version checks on a connector.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of the version checks steps.
-    """
-    context.logger.info("Run version checks.")
-    return [await VersionFollowsSemverCheck(context).run(), await VersionIncrementCheck(context).run()]
-
-
-async def run_qa_checks(context: ConnectorContext) -> List[StepResult]:
-    """Run the QA checks on a connector.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of the QA checks steps.
-    """
-    context.logger.info("Run QA checks.")
-    return [await QaChecks(context).run()]
-
-
-async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]:
-    """Run the code format checks according to the connector language.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of the code format checks steps.
-    """
-    if _run_code_format_checks := LANGUAGE_MAPPING["run_code_format_checks"].get(context.connector.language):
-        context.logger.info("Run code format checks.")
-        return await _run_code_format_checks(context)
-    else:
-        context.logger.warning(f"No code format checks defined for connector language {context.connector.language}!")
-        return []
-
-
-async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
-    """Run all the tests steps according to the connector language.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of the tests steps.
-    """
-    if _run_all_tests := LANGUAGE_MAPPING["run_all_tests"].get(context.connector.language):
-        return await _run_all_tests(context)
-    else:
-        context.logger.warning(f"No tests defined for connector language {context.connector.language}!")
-        return []
-
-
-async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport:
-    """Run a test pipeline for a single connector.
-
-    A visual DAG can be found on the README.md file of the pipelines modules.
-
-    Args:
-        context (ConnectorContext): The initialized connector context.
-
-    Returns:
-        ConnectorReport: The test reports holding tests results.
-    """
-    async with semaphore:
-        async with context:
-            async with asyncer.create_task_group() as task_group:
-                tasks = [
-                    task_group.soonify(run_metadata_validation)(context),
-                    task_group.soonify(run_version_checks)(context),
-                    task_group.soonify(run_qa_checks)(context),
-                    task_group.soonify(run_code_format_checks)(context),
-                    task_group.soonify(run_all_tests)(context),
-                ]
-            results = list(itertools.chain(*(task.value for task in tasks)))
-            context.report = ConnectorReport(context, steps_results=results, name="TEST RESULTS")
-
-        return context.report
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py
deleted file mode 100644
index ea9dca1f0f4d..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/common.py
+++ /dev/null
@@ -1,197 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module groups steps made to run tests agnostic to a connector language."""
-
-from abc import ABC, abstractmethod
-from functools import cached_property
-from typing import ClassVar, Optional
-
-import asyncer
-import requests
-import semver
-import yaml
-from ci_connector_ops.pipelines.actions import environments
-from ci_connector_ops.pipelines.bases import CIContext, PytestStep, Step, StepResult, StepStatus
-from ci_connector_ops.pipelines.utils import METADATA_FILE_NAME
-from ci_connector_ops.utils import Connector
-from dagger import File
-
-
-class VersionCheck(Step, ABC):
-    """A step to validate the connector version was bumped if files were modified"""
-
-    GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors"
-    failure_message: ClassVar
-    should_run = True
-
-    @property
-    def github_master_metadata_url(self):
-        return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}"
-
-    @cached_property
-    def master_metadata(self) -> dict:
-        response = requests.get(self.github_master_metadata_url)
-        response.raise_for_status()
-        return yaml.safe_load(response.text)
-
-    @property
-    def master_connector_version(self) -> semver.Version:
-        metadata = self.master_metadata
-        return semver.Version.parse(str(metadata["data"]["dockerImageTag"]))
-
-    @property
-    def current_connector_version(self) -> semver.Version:
-        return semver.Version.parse(str(self.context.metadata["dockerImageTag"]))
-
-    @property
-    def success_result(self) -> StepResult:
-        return StepResult(self, status=StepStatus.SUCCESS)
-
-    @property
-    def failure_result(self) -> StepResult:
-        return StepResult(self, status=StepStatus.FAILURE, stderr=self.failure_message)
-
-    @abstractmethod
-    def validate(self) -> StepResult:
-        raise NotImplementedError()
-
-    async def _run(self) -> StepResult:
-        if not self.should_run:
-            return StepResult(self, status=StepStatus.SKIPPED, stdout="No modified files required a version bump.")
-        if self.context.ci_context in [CIContext.MASTER, CIContext.NIGHTLY_BUILDS]:
-            return StepResult(self, status=StepStatus.SKIPPED, stdout="Version check are not running in master context.")
-        try:
-            return self.validate()
-        except (requests.HTTPError, ValueError, TypeError) as e:
-            return StepResult(self, status=StepStatus.FAILURE, stderr=str(e))
-
-
-class VersionIncrementCheck(VersionCheck):
-    title = "Connector version increment check."
-
-    BYPASS_CHECK_FOR = [
-        METADATA_FILE_NAME,
-        "acceptance-test-config.yml",
-        "README.md",
-        "bootstrap.md",
-        ".dockerignore",
-        "unit_tests",
-        "integration_tests",
-        "src/test",
-        "src/test-integration",
-        "src/test-performance",
-        "build.gradle",
-    ]
-
-    @property
-    def failure_message(self) -> str:
-        return f"The dockerImageTag in {METADATA_FILE_NAME} was not incremented. The files you modified should lead to a version bump. Master version is {self.master_connector_version}, current version is {self.current_connector_version}"
-
-    @property
-    def should_run(self) -> bool:
-        for filename in self.context.modified_files:
-            relative_path = filename.replace(str(self.context.connector.code_directory) + "/", "")
-            if not any([relative_path.startswith(to_bypass) for to_bypass in self.BYPASS_CHECK_FOR]):
-                return True
-        return False
-
-    def validate(self) -> StepResult:
-        if not self.current_connector_version > self.master_connector_version:
-            return self.failure_result
-        return self.success_result
-
-
-class VersionFollowsSemverCheck(VersionCheck):
-    title = "Connector version semver check."
-
-    @property
-    def failure_message(self) -> str:
-        return f"The dockerImageTag in {METADATA_FILE_NAME} is not following semantic versioning or was decremented. Master version is {self.master_connector_version}, current version is {self.current_connector_version}"
-
-    def validate(self) -> StepResult:
-        try:
-            if not self.current_connector_version >= self.master_connector_version:
-                return self.failure_result
-        except ValueError:
-            return self.failure_result
-        return self.success_result
-
-
-class QaChecks(Step):
-    """A step to run QA checks for a connector."""
-
-    title = "QA checks"
-
-    async def _run(self) -> StepResult:
-        """Run QA checks on a connector.
-
-        The QA checks are defined in this module:
-        https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/qa_checks.py
-
-        Args:
-            context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory.
-        Returns:
-            StepResult: Failure or success of the QA checks with stdout and stderr.
-        """
-        ci_connector_ops = await environments.with_ci_connector_ops(self.context)
-        include = [
-            str(self.context.connector.code_directory),
-            str(self.context.connector.documentation_file_path),
-            str(self.context.connector.icon_path),
-        ]
-        if (
-            self.context.connector.technical_name.endswith("strict-encrypt")
-            or self.context.connector.technical_name == "source-file-secure"
-        ):
-            original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", ""))
-            include += [
-                str(original_connector.code_directory),
-                str(original_connector.documentation_file_path),
-                str(original_connector.icon_path),
-            ]
-
-        filtered_repo = self.context.get_repo_dir(
-            include=include,
-        )
-
-        qa_checks = (
-            ci_connector_ops.with_mounted_directory("/airbyte", filtered_repo)
-            .with_workdir("/airbyte")
-            .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"])
-        )
-        return await self.get_step_result(qa_checks)
-
-
-class AcceptanceTests(PytestStep):
-    """A step to run acceptance tests for a connector if it has an acceptance test config file."""
-
-    title = "Acceptance tests"
-
-    async def _run(self, connector_under_test_image_tar: Optional[File]) -> StepResult:
-        """Run the acceptance test suite on a connector dev image. Build the connector acceptance test image if the tag is :dev.
-
-        Args:
-            connector_under_test_image_tar (File): The file holding the tar archive of the connector image.
-
-        Returns:
-            StepResult: Failure or success of the acceptances tests with stdout and stderr.
-        """
-        if not self.context.connector.acceptance_test_config:
-            return StepResult(self, StepStatus.SKIPPED)
-
-        cat_container = await environments.with_connector_acceptance_test(self.context, connector_under_test_image_tar)
-        secret_dir = cat_container.directory("/test_input/secrets")
-
-        async with asyncer.create_task_group() as task_group:
-            soon_secret_files = task_group.soonify(secret_dir.entries)()
-            soon_cat_container_stdout = task_group.soonify(cat_container.stdout)()
-
-        if secret_files := soon_secret_files.value:
-            for file_path in secret_files:
-                if file_path.startswith("updated_configurations"):
-                    self.context.updated_secrets_dir = secret_dir
-                    break
-
-        return self.pytest_logs_to_step_result(soon_cat_container_stdout.value)
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py
deleted file mode 100644
index ca969b5fe29d..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/java_connectors.py
+++ /dev/null
@@ -1,113 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module groups steps made to run tests for a specific Java connector given a test context."""
-
-from typing import List, Optional
-
-import anyio
-from ci_connector_ops.pipelines.actions import environments, run_steps, secrets
-from ci_connector_ops.pipelines.bases import StepResult, StepStatus
-from ci_connector_ops.pipelines.builds import LOCAL_BUILD_PLATFORM
-from ci_connector_ops.pipelines.builds.java_connectors import BuildConnectorDistributionTar, BuildConnectorImage
-from ci_connector_ops.pipelines.builds.normalization import BuildOrPullNormalization
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from ci_connector_ops.pipelines.gradle import GradleTask
-from ci_connector_ops.pipelines.tests.common import AcceptanceTests
-from ci_connector_ops.pipelines.utils import export_container_to_tarball
-from dagger import File, QueryError
-
-
-class UnitTests(GradleTask):
-    title = "Unit tests"
-    gradle_task_name = "test"
-
-
-class IntegrationTestJava(GradleTask):
-    """A step to run integrations tests for Java connectors using the integrationTestJava Gradle task."""
-
-    title = "Integration tests"
-    gradle_task_name = "integrationTestJava"
-
-    async def _load_normalization_image(self, normalization_tar_file: File):
-        normalization_image_tag = f"{self.context.connector.normalization_repository}:dev"
-        self.context.logger.info("Load the normalization image to the docker host.")
-        await environments.load_image_to_docker_host(
-            self.context, normalization_tar_file, normalization_image_tag, docker_service_name=self.docker_service_name
-        )
-        self.context.logger.info("Successfully loaded the normalization image to the docker host.")
-
-    async def _load_connector_image(self, connector_tar_file: File):
-        connector_image_tag = f"airbyte/{self.context.connector.technical_name}:dev"
-        self.context.logger.info("Load the connector image to the docker host")
-        await environments.load_image_to_docker_host(
-            self.context, connector_tar_file, connector_image_tag, docker_service_name=self.docker_service_name
-        )
-        self.context.logger.info("Successfully loaded the connector image to the docker host.")
-
-    async def _run(self, connector_tar_file: File, normalization_tar_file: Optional[File]) -> StepResult:
-        try:
-            async with anyio.create_task_group() as tg:
-                if normalization_tar_file:
-                    tg.start_soon(self._load_normalization_image, normalization_tar_file)
-                tg.start_soon(self._load_connector_image, connector_tar_file)
-            return await super()._run()
-        except QueryError as e:
-            return StepResult(self, StepStatus.FAILURE, stderr=str(e))
-
-
-async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
-    """Run all tests for a Java connectors.
-
-    - Build the normalization image if the connector supports it.
-    - Run unit tests with Gradle.
-    - Build connector image with Gradle.
-    - Run integration and acceptance test in parallel using the built connector and normalization images.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of all the tests steps.
-    """
-    context.secrets_dir = await secrets.get_connector_secret_dir(context)
-    step_results = []
-    build_distribution_tar_results = await BuildConnectorDistributionTar(context).run()
-    step_results.append(build_distribution_tar_results)
-    if build_distribution_tar_results.status is StepStatus.FAILURE:
-        return step_results
-
-    build_connector_image_results = await BuildConnectorImage(context, LOCAL_BUILD_PLATFORM).run(
-        build_distribution_tar_results.output_artifact
-    )
-    step_results.append(build_connector_image_results)
-    if build_connector_image_results.status is StepStatus.FAILURE:
-        return step_results
-
-    unit_tests_results = await UnitTests(context).run()
-    step_results.append(unit_tests_results)
-    if unit_tests_results.status is StepStatus.FAILURE:
-        return step_results
-
-    if context.connector.supports_normalization:
-        normalization_image = f"{context.connector.normalization_repository}:dev"
-        context.logger.info(f"This connector supports normalization: will build {normalization_image}.")
-        build_normalization_results = await BuildOrPullNormalization(context, normalization_image).run()
-        normalization_container = build_normalization_results.output_artifact
-        normalization_tar_file, _ = await export_container_to_tarball(
-            context, normalization_container, tar_file_name=f"{context.connector.normalization_repository}_{context.git_revision}.tar"
-        )
-        step_results.append(build_normalization_results)
-    else:
-        normalization_tar_file = None
-
-    connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact)
-
-    return await run_steps(
-        [
-            (IntegrationTestJava(context), (connector_image_tar_file, normalization_tar_file)),
-            (AcceptanceTests(context), (connector_image_tar_file)),
-        ],
-        results=step_results,
-    )
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py
deleted file mode 100644
index 5b82cd53e2e7..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/tests/python_connectors.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module groups steps made to run tests for a specific Python connector given a test context."""
-
-from typing import List
-
-import asyncer
-from ci_connector_ops.pipelines.actions import environments, run_steps, secrets
-from ci_connector_ops.pipelines.bases import Step, StepResult, StepStatus
-from ci_connector_ops.pipelines.builds import LOCAL_BUILD_PLATFORM
-from ci_connector_ops.pipelines.builds.python_connectors import BuildConnectorImage
-from ci_connector_ops.pipelines.contexts import ConnectorContext
-from ci_connector_ops.pipelines.tests.common import AcceptanceTests, PytestStep
-from ci_connector_ops.pipelines.utils import export_container_to_tarball
-from dagger import Container
-
-
-class CodeFormatChecks(Step):
-    """A step to run the code format checks on a Python connector using Black, Isort and Flake."""
-
-    title = "Code format checks"
-
-    RUN_BLACK_CMD = ["python", "-m", "black", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "--check", "."]
-    RUN_ISORT_CMD = ["python", "-m", "isort", f"--settings-file=/{environments.PYPROJECT_TOML_FILE_PATH}", "--check-only", "--diff", "."]
-    RUN_FLAKE_CMD = ["python", "-m", "pflake8", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "."]
-
-    async def _run(self) -> StepResult:
-        """Run a code format check on the container source code.
-
-        We call black, isort and flake commands:
-        - Black formats the code: fails if the code is not formatted.
-        - Isort checks the import orders: fails if the import are not properly ordered.
-        - Flake enforces style-guides: fails if the style-guide is not followed.
-
-        Args:
-            context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory.
-            step (Step): The step in which the code format checks are run. Defaults to Step.CODE_FORMAT_CHECKS
-        Returns:
-            StepResult: Failure or success of the code format checks with stdout and stderr.
-        """
-        connector_under_test = environments.with_python_connector_installed(self.context)
-        formatter = (
-            connector_under_test.with_exec(["echo", "Running black"])
-            .with_exec(self.RUN_BLACK_CMD)
-            .with_exec(["echo", "Running Isort"])
-            .with_exec(self.RUN_ISORT_CMD)
-            .with_exec(["echo", "Running Flake"])
-            .with_exec(self.RUN_FLAKE_CMD)
-        )
-        return await self.get_step_result(formatter)
-
-
-class ConnectorPackageInstall(Step):
-    """A step to install the Python connector package in a container."""
-
-    title = "Connector package install"
-    max_retries = 3
-
-    async def _run(self) -> StepResult:
-        """Install the connector under test package in a Python container.
-
-        Returns:
-            StepResult: Failure or success of the package installation and the connector under test container (with the connector package installed).
-        """
-        connector_under_test = await environments.with_installed_airbyte_connector(self.context)
-        return await self.get_step_result(connector_under_test)
-
-
-class UnitTests(PytestStep):
-    """A step to run the connector unit tests with Pytest."""
-
-    title = "Unit tests"
-
-    async def _run(self, connector_under_test: Container) -> StepResult:
-        """Run all pytest tests declared in the unit_tests directory of the connector code.
-
-        Args:
-            connector_under_test (Container): The connector under test container.
-
-        Returns:
-            StepResult: Failure or success of the unit tests with stdout and stdout.
-        """
-        connector_under_test_with_secrets = connector_under_test.with_directory("secrets", self.context.secrets_dir)
-        return await self._run_tests_in_directory(connector_under_test_with_secrets, "unit_tests")
-
-
-class IntegrationTests(PytestStep):
-    """A step to run the connector integration tests with Pytest."""
-
-    title = "Integration tests"
-
-    async def _run(self, connector_under_test: Container) -> StepResult:
-        """Run all pytest tests declared in the integration_tests directory of the connector code.
-
-        Args:
-            connector_under_test (Container): The connector under test container.
-
-        Returns:
-            StepResult: Failure or success of the integration tests with stdout and stdout.
-        """
-        connector_under_test_with_secrets = connector_under_test.with_directory("secrets", self.context.secrets_dir)
-
-        return await self._run_tests_in_directory(connector_under_test_with_secrets, "integration_tests")
-
-
-async def run_all_tests(context: ConnectorContext) -> List[StepResult]:
-    """Run all tests for a Python connector.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: The results of all the steps that ran or were skipped.
-    """
-
-    step_results = await run_steps(
-        [
-            ConnectorPackageInstall(context),
-            BuildConnectorImage(context, LOCAL_BUILD_PLATFORM),
-        ]
-    )
-    if any([step_result.status is StepStatus.FAILURE for step_result in step_results]):
-        return step_results
-    connector_package_install_results, build_connector_image_results = step_results[0], step_results[1]
-    connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact)
-    connector_container = connector_package_install_results.output_artifact
-
-    context.secrets_dir = await secrets.get_connector_secret_dir(context)
-
-    unit_test_results = await UnitTests(context).run(connector_container)
-    if unit_test_results.status is StepStatus.FAILURE:
-        return step_results + [unit_test_results]
-    step_results.append(unit_test_results)
-    async with asyncer.create_task_group() as task_group:
-        tasks = [
-            task_group.soonify(IntegrationTests(context).run)(connector_container),
-            task_group.soonify(AcceptanceTests(context).run)(connector_image_tar_file),
-        ]
-
-    return step_results + [task.value for task in tasks]
-
-
-async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]:
-    """Run the code format check steps for Python connectors.
-
-    Args:
-        context (ConnectorContext): The current connector context.
-
-    Returns:
-        List[StepResult]: Results of the code format checks.
-    """
-    return [await CodeFormatChecks(context).run()]
diff --git a/tools/ci_connector_ops/ci_connector_ops/pipelines/utils.py b/tools/ci_connector_ops/ci_connector_ops/pipelines/utils.py
deleted file mode 100644
index e810232e90b0..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/pipelines/utils.py
+++ /dev/null
@@ -1,389 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-"""This module groups util function used in pipelines."""
-from __future__ import annotations
-
-import datetime
-import json
-import re
-import sys
-import unicodedata
-from glob import glob
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Set, Tuple, Union
-
-import anyio
-import asyncer
-import click
-import git
-from ci_connector_ops.utils import get_all_released_connectors
-from dagger import Config, Connection, Container, DaggerError, File, ImageLayerCompression, QueryError
-from more_itertools import chunked
-
-if TYPE_CHECKING:
-    from ci_connector_ops.pipelines.contexts import ConnectorContext
-    from github import PullRequest
-
-DAGGER_CONFIG = Config(log_output=sys.stderr)
-AIRBYTE_REPO_URL = "https://github.com/airbytehq/airbyte.git"
-METADATA_FILE_NAME = "metadata.yaml"
-METADATA_ICON_FILE_NAME = "icon.svg"
-DIFF_FILTER = "MADRT"  # Modified, Added, Deleted, Renamed, Type changed
-
-
-# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented
-async def check_path_in_workdir(container: Container, path: str) -> bool:
-    """Check if a local path is mounted to the working directory of a container.
-
-    Args:
-        container (Container): The container on which we want the check the path existence.
-        path (str): Directory or file path we want to check the existence in the container working directory.
-
-    Returns:
-        bool: Whether the path exists in the container working directory.
-    """
-    workdir = (await container.with_exec(["pwd"]).stdout()).strip()
-    mounts = await container.mounts()
-    if workdir in mounts:
-        expected_file_path = Path(workdir[1:]) / path
-        return expected_file_path.is_file() or expected_file_path.is_dir()
-    else:
-        return False
-
-
-# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented
-async def get_file_contents(container: Container, path: str) -> Optional[str]:
-    """Retrieve a container file contents.
-
-    Args:
-        container (Container): The container hosting the file you want to read.
-        path (str): Path, in the container, to the file you want to read.
-
-    Returns:
-        Optional[str]: The file content if the file exists in the container, None otherwise.
-    """
-    try:
-        return await container.file(path).contents()
-    except QueryError as e:
-        if "no such file or directory" not in str(e):
-            # this is the hicky bit of the stopgap because
-            # this error could come from a network issue
-            raise
-    return None
-
-
-# This is a stop-gap solution to capture non 0 exit code on Containers
-# The original issue is tracked here https://github.com/dagger/dagger/issues/3192
-async def with_exit_code(container: Container) -> int:
-    """Read the container exit code.
-
-    If the exit code is not 0 a QueryError is raised. We extract the non-zero exit code from the QueryError message.
-
-    Args:
-        container (Container): The container from which you want to read the exit code.
-
-    Returns:
-        int: The exit code.
-    """
-    try:
-        await container.exit_code()
-    except QueryError as e:
-        error_message = str(e)
-        if "exit code: " in error_message:
-            exit_code = re.search(r"exit code: (\d+)", error_message)
-            if exit_code:
-                return int(exit_code.group(1))
-            else:
-                return 1
-        raise
-    return 0
-
-
-# This is a stop-gap solution to capture non 0 exit code on Containers
-# The original issue is tracked here https://github.com/dagger/dagger/issues/3192
-async def with_stderr(container: Container) -> str:
-    """Retrieve the stderr of a container and handle unexpected errors."""
-    try:
-        return await container.stderr()
-    except QueryError as e:
-        return str(e)
-
-
-# This is a stop-gap solution to capture non 0 exit code on Containers
-# The original issue is tracked here https://github.com/dagger/dagger/issues/3192
-async def with_stdout(container: Container) -> str:
-    """Retrieve the stdout of a container and handle unexpected errors."""
-    try:
-        return await container.stdout()
-    except QueryError as e:
-        return str(e)
-
-
-def get_current_git_branch() -> str:  # noqa D103
-    return git.Repo().active_branch.name
-
-
-def get_current_git_revision() -> str:  # noqa D103
-    return git.Repo().head.object.hexsha
-
-
-def get_current_epoch_time() -> int:  # noqa D103
-    return round(datetime.datetime.utcnow().timestamp())
-
-
-async def get_modified_files_in_branch_remote(
-    current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master"
-) -> Set[str]:
-    """Use git diff to spot the modified files on the remote branch."""
-    async with Connection(DAGGER_CONFIG) as dagger_client:
-        modified_files = await (
-            dagger_client.container()
-            .from_("alpine/git:latest")
-            .with_workdir("/repo")
-            .with_exec(["init"])
-            .with_env_variable("CACHEBUSTER", current_git_revision)
-            .with_exec(
-                [
-                    "remote",
-                    "add",
-                    "--fetch",
-                    "--track",
-                    diffed_branch.split("/")[-1],
-                    "--track",
-                    current_git_branch,
-                    "origin",
-                    AIRBYTE_REPO_URL,
-                ]
-            )
-            .with_exec(["checkout", "-t", f"origin/{current_git_branch}"])
-            .with_exec(["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_revision}"])
-            .stdout()
-        )
-    return set(modified_files.split("\n"))
-
-
-def get_modified_files_in_branch_local(current_git_revision: str, diffed_branch: str = "master") -> Set[str]:
-    """Use git diff and git status to spot the modified files on the local branch."""
-    airbyte_repo = git.Repo()
-    modified_files = airbyte_repo.git.diff(
-        f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_revision}"
-    ).split("\n")
-    status_output = airbyte_repo.git.status("--porcelain")
-    for not_committed_change in status_output.split("\n"):
-        file_path = not_committed_change.strip().split(" ")[-1]
-        if file_path:
-            modified_files.append(file_path)
-    return set(modified_files)
-
-
-def get_modified_files_in_branch(current_git_branch: str, current_git_revision: str, diffed_branch: str, is_local: bool = True) -> Set[str]:
-    """Retrieve the list of modified files on the branch."""
-    if is_local:
-        return get_modified_files_in_branch_local(current_git_revision, diffed_branch)
-    else:
-        return anyio.run(get_modified_files_in_branch_remote, current_git_branch, current_git_revision, diffed_branch)
-
-
-async def get_modified_files_in_commit_remote(current_git_branch: str, current_git_revision: str) -> Set[str]:
-    async with Connection(DAGGER_CONFIG) as dagger_client:
-        modified_files = await (
-            dagger_client.container()
-            .from_("alpine/git:latest")
-            .with_workdir("/repo")
-            .with_exec(["init"])
-            .with_env_variable("CACHEBUSTER", current_git_revision)
-            .with_exec(
-                [
-                    "remote",
-                    "add",
-                    "--fetch",
-                    "--track",
-                    current_git_branch,
-                    "origin",
-                    AIRBYTE_REPO_URL,
-                ]
-            )
-            .with_exec(["checkout", "-t", f"origin/{current_git_branch}"])
-            .with_exec(["diff-tree", "--no-commit-id", "--name-only", current_git_revision, "-r"])
-            .stdout()
-        )
-    return set(modified_files.split("\n"))
-
-
-def get_modified_files_in_commit_local(current_git_revision: str) -> Set[str]:
-    airbyte_repo = git.Repo()
-    modified_files = airbyte_repo.git.diff_tree("--no-commit-id", "--name-only", current_git_revision, "-r").split("\n")
-    return set(modified_files)
-
-
-def get_modified_files_in_commit(current_git_branch: str, current_git_revision: str, is_local: bool = True) -> Set[str]:
-    if is_local:
-        return get_modified_files_in_commit_local(current_git_revision)
-    else:
-        return anyio.run(get_modified_files_in_commit_remote, current_git_branch, current_git_revision)
-
-
-def get_modified_files_in_pull_request(pull_request: PullRequest) -> List[str]:
-    """Retrieve the list of modified files in a pull request."""
-    return [f.filename for f in pull_request.get_files()]
-
-
-def get_modified_connectors(modified_files: Set[Union[str, Path]]) -> dict:
-    """Create a mapping of modified connectors (key) and modified files (value).
-    As we call connector.get_local_dependencies_paths() any modification to a dependency will trigger connector pipeline for all connectors that depend on it.
-    The get_local_dependencies_paths function currently computes dependencies for Java connectors only.
-    It's especially useful to trigger tests of strict-encrypt variant when a change is made to the base connector.
-    Or to tests all jdbc connectors when a change is made to source-jdbc or base-java.
-    We'll consider extending the dependency resolution to Python connectors once we confirm that it's needed and feasible in term of scale.
-    """
-    modified_connectors = {}
-    all_connector_dependencies = [(connector, connector.get_local_dependencies_paths()) for connector in get_all_released_connectors()]
-    for modified_file in modified_files:
-        for connector, connector_dependencies in all_connector_dependencies:
-            for connector_dependency in connector_dependencies:
-                connector_dependency_parts = connector_dependency.parts
-                modified_file_parts = Path(modified_file).parts
-                # The modified file is a dependency of the connector if the modified file path starts with the connector dependency path.
-                if modified_file_parts[: len(connector_dependency_parts)] == connector_dependency_parts:
-                    modified_connectors.setdefault(connector, []).append(modified_file)
-    return modified_connectors
-
-
-def get_modified_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]:
-    return {
-        Path(str(f))
-        for f in modified_files
-        if str(f).endswith(METADATA_FILE_NAME) and str(f).startswith("airbyte-integrations/connectors") and "-scaffold-" not in str(f)
-    }
-
-
-def get_all_metadata_files() -> Set[Path]:
-    return {
-        Path(metadata_file)
-        for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
-        if "-scaffold-" not in metadata_file
-    }
-
-
-def slugify(value: Any, allow_unicode: bool = False):
-    """
-    Taken from https://github.com/django/django/blob/master/django/utils/text.py.
-
-    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
-    dashes to single dashes. Remove characters that aren't alphanumerics,
-    underscores, or hyphens. Convert to lowercase. Also strip leading and
-    trailing whitespace, dashes, and underscores.
-    """
-    value = str(value)
-    if allow_unicode:
-        value = unicodedata.normalize("NFKC", value)
-    else:
-        value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii")
-    value = re.sub(r"[^\w\s-]", "", value.lower())
-    return re.sub(r"[-\s]+", "-", value).strip("-_")
-
-
-def key_value_text_to_dict(text: str) -> dict:
-    kv = {}
-    for line in text.split("\n"):
-        if "=" in line:
-            try:
-                k, v = line.split("=")
-            except ValueError:
-                continue
-            kv[k] = v
-    return kv
-
-
-async def key_value_file_to_dict(file: File) -> dict:
-    return key_value_text_to_dict(await file.contents())
-
-
-async def get_dockerfile_labels(dockerfile: File) -> dict:
-    return {k.replace("LABEL ", ""): v for k, v in (await key_value_file_to_dict(dockerfile)).items() if k.startswith("LABEL")}
-
-
-async def get_version_from_dockerfile(dockerfile: File) -> str:
-    dockerfile_labels = await get_dockerfile_labels(dockerfile)
-    try:
-        return dockerfile_labels["io.airbyte.version"]
-    except KeyError:
-        raise Exception("Could not get the version from the Dockerfile labels.")
-
-
-class DaggerPipelineCommand(click.Command):
-    def invoke(self, ctx: click.Context) -> Any:
-        """Wrap parent invoke in a try catch suited to handle pipeline failures.
-        Args:
-            ctx (click.Context): The invocation context.
-        Raises:
-            e: Raise whatever exception that was caught.
-        Returns:
-            Any: The invocation return value.
-        """
-        command_name = self.name
-        click.secho(f"Running Dagger Command {command_name}...")
-        click.secho(
-            "If you're running this command for the first time the Dagger engine image will be pulled, it can take a short minute..."
-        )
-        try:
-            pipeline_success = super().invoke(ctx)
-            if not pipeline_success:
-                raise DaggerError(f"Dagger Command {command_name} failed.")
-        except DaggerError as e:
-            click.secho(str(e), err=True, fg="red")
-            sys.exit(1)
-
-
-async def execute_concurrently(steps: List[Callable], concurrency=5):
-    tasks = []
-    # Asyncer does not have builtin semaphore, so control concurrency via chunks of steps
-    # Anyio has semaphores but does not have the soonify method which allow access to results via the value task attribute.
-    for chunk in chunked(steps, concurrency):
-        async with asyncer.create_task_group() as task_group:
-            tasks += [task_group.soonify(step)() for step in chunk]
-    return [task.value for task in tasks]
-
-
-async def export_container_to_tarball(
-    context: ConnectorContext, container: Container, tar_file_name: Optional[str] = None
-) -> Tuple[Optional[File], Optional[Path]]:
-    """Save the container image to the host filesystem as a tar archive.
-
-    Exporting a container image as a tar archive allows user to have a dagger built container image available on their host filesystem.
-    They can load this tar file to their main docker host with 'docker load'.
-    This mechanism is also used to share dagger built containers with other steps like AcceptanceTest that have their own dockerd service.
-    We 'docker load' this tar file to AcceptanceTest's docker host to make sure the container under test image is available for testing.
-
-    Returns:
-        Tuple[Optional[File], Optional[Path]]: A tuple with the file object holding the tar archive on the host and its path.
-    """
-    if tar_file_name is None:
-        tar_file_name = f"{context.connector.technical_name}_{context.git_revision}.tar"
-    tar_file_name = slugify(tar_file_name)
-    local_path = Path(f"{context.host_image_export_dir_path}/{tar_file_name}")
-    export_success = await container.export(str(local_path), forced_compression=ImageLayerCompression.Gzip)
-    if export_success:
-        exported_file = (
-            context.dagger_client.host().directory(context.host_image_export_dir_path, include=[tar_file_name]).file(tar_file_name)
-        )
-        return exported_file, local_path
-    else:
-        return None, None
-
-
-def sanitize_gcs_credentials(raw_value: Optional[str]) -> Optional[str]:
-    """Try to parse the raw string input that should contain a json object with the GCS credentials.
-    It will raise an exception if the parsing fails and help us to fail fast on invalid credentials input.
-
-    Args:
-        raw_value (str): A string representing a json object with the GCS credentials.
-
-    Returns:
-        str: The raw value string if it was successfully parsed.
-    """
-    if raw_value is None:
-        return None
-    return json.dumps(json.loads(raw_value))
diff --git a/tools/ci_connector_ops/ci_connector_ops/utils.py b/tools/ci_connector_ops/ci_connector_ops/utils.py
deleted file mode 100644
index b61572d6507b..000000000000
--- a/tools/ci_connector_ops/ci_connector_ops/utils.py
+++ /dev/null
@@ -1,321 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-import functools
-import logging
-import os
-import re
-from dataclasses import dataclass
-from enum import Enum
-from glob import glob
-from pathlib import Path
-from typing import List, Optional, Set, Tuple
-
-import git
-import requests
-import yaml
-from ci_credentials import SecretsManager
-from rich.console import Console
-
-console = Console()
-
-DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master")
-OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
-CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors"
-SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-"
-DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-"
-ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml"
-AIRBYTE_DOCKER_REPO = "airbyte"
-AIRBYTE_REPO_DIRECTORY_NAME = "airbyte"
-GRADLE_PROJECT_RE_PATTERN = r"project\((['\"])(.+?)\1\)"
-TEST_GRADLE_DEPENDENCIES = ["testImplementation", "integrationTestJavaImplementation", "performanceTestJavaImplementation"]
-
-
-def download_catalog(catalog_url):
-    response = requests.get(catalog_url)
-    return response.json()
-
-
-OSS_CATALOG = download_catalog(OSS_CATALOG_URL)
-METADATA_FILE_NAME = "metadata.yaml"
-ICON_FILE_NAME = "icon.svg"
-
-
-class ConnectorInvalidNameError(Exception):
-    pass
-
-
-class ConnectorVersionNotFound(Exception):
-    pass
-
-
-def get_connector_name_from_path(path):
-    return path.split("/")[2]
-
-
-def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[str]:
-    """Retrieve the set of connectors for which the acceptance_test_config file was changed in the current branch (compared to master).
-
-    Args:
-        diff_regex (str): Find the edited files that contain the following regex in their change.
-
-    Returns:
-        Set[Connector]: Set of connectors that were changed
-    """
-    airbyte_repo = git.Repo(search_parent_directories=True)
-
-    if diff_regex is None:
-        diff_command_args = ("--name-only", DIFFED_BRANCH)
-    else:
-        diff_command_args = ("--name-only", f"-G{diff_regex}", DIFFED_BRANCH)
-
-    changed_acceptance_test_config_paths = {
-        file_path
-        for file_path in airbyte_repo.git.diff(*diff_command_args).split("\n")
-        if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX) and file_path.endswith(ACCEPTANCE_TEST_CONFIG_FILE_NAME)
-    }
-    return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths}
-
-
-def get_gradle_dependencies_block(build_file: Path) -> str:
-    """Get the dependencies block of a Gradle file.
-
-    Args:
-        build_file (Path): Path to the build.gradle file of the project.
-
-    Returns:
-        str: The dependencies block of the Gradle file.
-    """
-    contents = build_file.read_text().split("\n")
-    dependency_block = []
-    in_dependencies_block = False
-    for line in contents:
-        if line.strip().startswith("dependencies"):
-            in_dependencies_block = True
-            continue
-        if in_dependencies_block:
-            if line.startswith("}"):
-                in_dependencies_block = False
-                break
-            else:
-                dependency_block.append(line)
-    dependencies_block = "\n".join(dependency_block)
-    return dependencies_block
-
-
-def parse_gradle_dependencies(build_file: Path) -> Tuple[List[Path], List[Path]]:
-    """Parse the dependencies block of a Gradle file and return the list of project dependencies and test dependencies.
-
-    Args:
-        build_file (Path): _description_
-
-    Returns:
-        Tuple[List[Tuple[str, Path]], List[Tuple[str, Path]]]: _description_
-    """
-
-    dependencies_block = get_gradle_dependencies_block(build_file)
-
-    project_dependencies: List[Tuple[str, Path]] = []
-    test_dependencies: List[Tuple[str, Path]] = []
-
-    # Find all matches for test dependencies and regular dependencies
-    matches = re.findall(
-        r"(testImplementation|integrationTestJavaImplementation|performanceTestJavaImplementation|implementation).*?project\(['\"](.*?)['\"]\)",
-        dependencies_block,
-    )
-    if matches:
-        # Iterate through each match
-        for match in matches:
-            dependency_type, project_path = match
-            path_parts = project_path.split(":")
-            path = Path(*path_parts)
-
-            if dependency_type in TEST_GRADLE_DEPENDENCIES:
-                test_dependencies.append(path)
-            else:
-                project_dependencies.append(path)
-    return project_dependencies, test_dependencies
-
-
-def get_all_gradle_dependencies(
-    build_file: Path, with_test_dependencies: bool = True, found_dependencies: Optional[List[Path]] = None
-) -> List[Path]:
-    """Recursively retrieve all transitive dependencies of a Gradle project.
-
-    Args:
-        build_file (Path): Path to the build.gradle file of the project.
-        found_dependencies (List[Path]): List of dependencies that have already been found. Defaults to None.
-
-    Returns:
-        List[Path]: All dependencies of the project.
-    """
-    if found_dependencies is None:
-        found_dependencies = []
-    project_dependencies, test_dependencies = parse_gradle_dependencies(build_file)
-    all_dependencies = project_dependencies + test_dependencies if with_test_dependencies else project_dependencies
-    for dependency_path in all_dependencies:
-        if dependency_path not in found_dependencies and Path(dependency_path / "build.gradle").exists():
-            found_dependencies.append(dependency_path)
-            get_all_gradle_dependencies(dependency_path / "build.gradle", with_test_dependencies, found_dependencies)
-
-    return found_dependencies
-
-
-class ConnectorLanguage(str, Enum):
-    PYTHON = "python"
-    JAVA = "java"
-    LOW_CODE = "low-code"
-
-
-class ConnectorLanguageError(Exception):
-    pass
-
-
-@dataclass(frozen=True)
-class Connector:
-    """Utility class to gather metadata about a connector."""
-
-    technical_name: str
-
-    def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]:
-        if "-" not in self.technical_name:
-            raise ConnectorInvalidNameError(f"Connector type and name could not be inferred from {self.technical_name}")
-        _type = self.technical_name.split("-")[0]
-        name = self.technical_name[len(_type) + 1 :]
-        return _type, name
-
-    @property
-    def name(self):
-        return self._get_type_and_name_from_technical_name()[1]
-
-    @property
-    def connector_type(self) -> str:
-        return self._get_type_and_name_from_technical_name()[0]
-
-    @property
-    def documentation_file_path(self) -> Path:
-        return Path(f"./docs/integrations/{self.connector_type}s/{self.name}.md")
-
-    @property
-    def icon_path(self) -> Path:
-        file_path = self.code_directory / ICON_FILE_NAME
-        return file_path
-
-    @property
-    def code_directory(self) -> Path:
-        return Path(f"./airbyte-integrations/connectors/{self.technical_name}")
-
-    @property
-    def metadata(self) -> Optional[dict]:
-        file_path = self.code_directory / METADATA_FILE_NAME
-        if not file_path.is_file():
-            return None
-        return yaml.safe_load((self.code_directory / METADATA_FILE_NAME).read_text())["data"]
-
-    @property
-    def language(self) -> ConnectorLanguage:
-        if Path(self.code_directory / self.technical_name.replace("-", "_") / "manifest.yaml").is_file():
-            return ConnectorLanguage.LOW_CODE
-        if Path(self.code_directory / "setup.py").is_file():
-            return ConnectorLanguage.PYTHON
-        try:
-            with open(self.code_directory / "Dockerfile") as dockerfile:
-                if "FROM airbyte/integration-base-java" in dockerfile.read():
-                    return ConnectorLanguage.JAVA
-        except FileNotFoundError:
-            pass
-        return None
-        # raise ConnectorLanguageError(f"We could not infer {self.technical_name} connector language")
-
-    @property
-    def version(self) -> str:
-        if self.metadata is None:
-            return self.version_in_dockerfile_label
-        return self.metadata["dockerImageTag"]
-
-    @property
-    def version_in_dockerfile_label(self) -> str:
-        with open(self.code_directory / "Dockerfile") as f:
-            for line in f:
-                if "io.airbyte.version" in line:
-                    return line.split("=")[1].strip()
-        raise ConnectorVersionNotFound(
-            """
-            Could not find the connector version from its Dockerfile.
-            The io.airbyte.version tag is missing.
-            """
-        )
-
-    @property
-    def release_stage(self) -> Optional[str]:
-        return self.metadata.get("releaseStage") if self.metadata else None
-
-    @property
-    def allowed_hosts(self) -> Optional[List[str]]:
-        return self.metadata.get("allowedHosts") if self.metadata else None
-
-    @property
-    def suggested_streams(self) -> Optional[List[str]]:
-        return self.metadata.get("suggestedStreams") if self.metadata else None
-
-    @property
-    def acceptance_test_config_path(self) -> Path:
-        return self.code_directory / ACCEPTANCE_TEST_CONFIG_FILE_NAME
-
-    @property
-    def acceptance_test_config(self) -> Optional[dict]:
-        try:
-            with open(self.acceptance_test_config_path) as acceptance_test_config_file:
-                return yaml.safe_load(acceptance_test_config_file)
-        except FileNotFoundError:
-            logging.warning(f"No {ACCEPTANCE_TEST_CONFIG_FILE_NAME} file found for {self.technical_name}")
-            return None
-
-    @property
-    def supports_normalization(self) -> bool:
-        return self.metadata and self.metadata.get("normalizationConfig") is not None
-
-    @property
-    def normalization_repository(self) -> Optional[str]:
-        if self.supports_normalization:
-            return f"{self.metadata['normalizationConfig']['normalizationRepository']}"
-
-    @property
-    def normalization_tag(self) -> Optional[str]:
-        if self.supports_normalization:
-            return f"{self.metadata['normalizationConfig']['normalizationTag']}"
-
-    def get_secret_manager(self, gsm_credentials: str):
-        return SecretsManager(connector_name=self.technical_name, gsm_credentials=gsm_credentials)
-
-    def __repr__(self) -> str:
-        return self.technical_name
-
-    @functools.lru_cache(maxsize=2)
-    def get_local_dependencies_paths(self, with_test_dependencies: bool = True) -> Set[Path]:
-        dependencies_paths = [self.code_directory]
-        if self.language == ConnectorLanguage.JAVA:
-            dependencies_paths += get_all_gradle_dependencies(
-                self.code_directory / "build.gradle", with_test_dependencies=with_test_dependencies
-            )
-        return set(dependencies_paths)
-
-
-def get_changed_connectors() -> Set[Connector]:
-    """Retrieve a set of Connectors that were changed in the current branch (compared to master)."""
-    airbyte_repo = git.Repo(search_parent_directories=True)
-    changed_source_connector_files = {
-        file_path
-        for file_path in airbyte_repo.git.diff("--name-only", DIFFED_BRANCH).split("\n")
-        if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX)
-    }
-    return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files}
-
-
-def get_all_released_connectors() -> Set:
-    return {
-        Connector(Path(metadata_file).parent.name)
-        for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True)
-        if "-scaffold-" not in metadata_file
-    }
diff --git a/tools/ci_connector_ops/setup.py b/tools/ci_connector_ops/setup.py
deleted file mode 100644
index 730b09cd505c..000000000000
--- a/tools/ci_connector_ops/setup.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-MAIN_REQUIREMENTS = [
-    "click~=8.1.3",
-    "requests",
-    "PyYAML~=6.0",
-    "GitPython~=3.1.29",
-    "pydantic~=1.9",
-    "PyGithub~=1.58.0",
-    "rich",
-]
-
-
-def local_pkg(name: str) -> str:
-    """Returns a path to a local package."""
-    return f"{name} @ file://{Path.cwd().parent / name}"
-
-
-# These internal packages are not yet published to a Pypi repository.
-LOCAL_REQUIREMENTS = [local_pkg("ci_credentials")]
-
-TEST_REQUIREMENTS = [
-    "pytest~=6.2.5",
-    "pytest-mock~=3.10.0",
-    "freezegun",
-]
-
-DEV_REQUIREMENTS = ["pyinstrument"]
-# It is hard to containerize Pandas, it's only used in the QA engine, so I declared it as an extra requires
-# TODO update the GHA that install the QA engine to install this extra
-QA_ENGINE_REQUIREMENTS = [
-    "pandas~=1.5.3",
-    "pandas-gbq~=0.19.0",
-    "fsspec~=2023.1.0",
-    "gcsfs~=2023.1.0",
-    "pytablewriter~=0.64.2",
-]
-
-PIPELINES_REQUIREMENTS = [
-    "dagger-io==0.5.4",
-    "asyncer",
-    "anyio",
-    "more-itertools",
-    "docker",
-    "requests",
-    "semver",
-    "airbyte-protocol-models",
-    "tabulate",
-]
-
-setup(
-    version="0.2.1",
-    name="ci_connector_ops",
-    description="Packaged maintained by the connector operations team to perform CI for connectors",
-    author="Airbyte",
-    author_email="contact@airbyte.io",
-    packages=find_packages(),
-    install_requires=MAIN_REQUIREMENTS + LOCAL_REQUIREMENTS,
-    extras_require={
-        "tests": QA_ENGINE_REQUIREMENTS + TEST_REQUIREMENTS,
-        "dev": QA_ENGINE_REQUIREMENTS + TEST_REQUIREMENTS + DEV_REQUIREMENTS,
-        "pipelines": MAIN_REQUIREMENTS + PIPELINES_REQUIREMENTS,
-        "qa_engine": MAIN_REQUIREMENTS + QA_ENGINE_REQUIREMENTS,
-    },
-    # python_requires=">=3.10", TODO upgrade all our CI packages + GHA env to 3.10
-    package_data={"ci_connector_ops.qa_engine": ["connector_adoption.sql"]},
-    entry_points={
-        "console_scripts": [
-            "check-test-strictness-level = ci_connector_ops.acceptance_test_config_checks:check_test_strictness_level",
-            "write-review-requirements-file = ci_connector_ops.acceptance_test_config_checks:write_review_requirements_file",
-            "print-mandatory-reviewers = ci_connector_ops.acceptance_test_config_checks:print_mandatory_reviewers",
-            "allowed-hosts-checks = ci_connector_ops.allowed_hosts_checks:check_allowed_hosts",
-            "run-qa-engine = ci_connector_ops.qa_engine.main:main",
-            "run-qa-checks = ci_connector_ops.qa_checks:run_qa_checks",
-            "airbyte-ci = ci_connector_ops.pipelines.commands.airbyte_ci:airbyte_ci",
-        ],
-    },
-)
diff --git a/tools/ci_connector_ops/tests/conftest.py b/tools/ci_connector_ops/tests/conftest.py
deleted file mode 100644
index 60380444fd37..000000000000
--- a/tools/ci_connector_ops/tests/conftest.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-from datetime import datetime
-import pandas as pd
-import pytest
-
-from ci_connector_ops.qa_engine.constants import OSS_CATALOG_URL, CLOUD_CATALOG_URL
-from ci_connector_ops.qa_engine.inputs import fetch_remote_catalog
-
-@pytest.fixture(scope="module")
-def oss_catalog():
-    return fetch_remote_catalog(OSS_CATALOG_URL)
-
-@pytest.fixture(scope="module")
-def cloud_catalog():
-    return fetch_remote_catalog(CLOUD_CATALOG_URL)
-
-@pytest.fixture(scope="module")
-def adoption_metrics_per_connector_version():
-    return pd.DataFrame([{
-        "connector_definition_id": "dfd88b22-b603-4c3d-aad7-3701784586b1",
-        "connector_version": "2.0.0",
-        "number_of_connections": 0,
-        "number_of_users": 0,
-        "succeeded_syncs_count": 0,
-        "failed_syncs_count": 0,
-        "total_syncs_count": 0,
-        "sync_success_rate": 0.0,
-    }])
-
-@pytest.fixture
-def dummy_qa_report() -> pd.DataFrame:
-    return pd.DataFrame([
-        {
-            "connector_type": "source",
-            "connector_name": "test",
-            "connector_technical_name": "source-test",
-            "connector_definition_id": "foobar",
-            "connector_version": "0.0.0",
-            "release_stage": "alpha",
-            "is_on_cloud": False,
-            "is_appropriate_for_cloud_use": True,
-            "latest_build_is_successful": True,
-            "documentation_is_available": False,
-            "number_of_connections": 0,
-            "number_of_users": 0,
-            "sync_success_rate": .99,
-            "total_syncs_count": 0,
-            "failed_syncs_count": 0,
-            "succeeded_syncs_count": 0,
-            "is_eligible_for_promotion_to_cloud": True,
-            "report_generation_datetime": datetime.utcnow()
-        }
-    ])
diff --git a/tools/ci_connector_ops/tests/test_qa_checks.py b/tools/ci_connector_ops/tests/test_qa_checks.py
deleted file mode 100644
index 7f7761d32137..000000000000
--- a/tools/ci_connector_ops/tests/test_qa_checks.py
+++ /dev/null
@@ -1,183 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-from pathlib import Path
-
-import pytest
-from ci_connector_ops import qa_checks, utils
-
-
-@pytest.mark.parametrize(
-    "connector, expect_exists",
-    [
-        (utils.Connector("source-faker"), True),
-        (utils.Connector("source-foobar"), False),
-    ],
-)
-def test_check_documentation_file_exists(connector, expect_exists):
-    assert qa_checks.check_documentation_file_exists(connector) == expect_exists
-
-
-def test_check_changelog_entry_is_updated_missing_doc(mocker):
-    mocker.patch.object(qa_checks, "check_documentation_file_exists", mocker.Mock(return_value=False))
-    assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) is False
-
-
-def test_check_changelog_entry_is_updated_no_changelog_section(mocker, tmp_path):
-    mock_documentation_file_path = Path(tmp_path / "doc.md")
-    mock_documentation_file_path.touch()
-
-    mocker.patch.object(qa_checks.Connector, "documentation_file_path", mock_documentation_file_path)
-    assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) is False
-
-
-def test_check_changelog_entry_is_updated_version_not_in_changelog(mocker, tmp_path):
-    mock_documentation_file_path = Path(tmp_path / "doc.md")
-    with open(mock_documentation_file_path, "w") as f:
-        f.write("# Changelog")
-
-    mocker.patch.object(qa_checks.Connector, "documentation_file_path", mock_documentation_file_path)
-
-    mocker.patch.object(qa_checks.Connector, "version", "0.0.0")
-
-    assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar")) is False
-
-
-def test_check_changelog_entry_is_updated_version_in_changelog(mocker, tmp_path):
-    mock_documentation_file_path = Path(tmp_path / "doc.md")
-    with open(mock_documentation_file_path, "w") as f:
-        f.write("# Changelog\n0.0.0")
-
-    mocker.patch.object(qa_checks.Connector, "documentation_file_path", mock_documentation_file_path)
-
-    mocker.patch.object(qa_checks.Connector, "version", "0.0.0")
-    assert qa_checks.check_changelog_entry_is_updated(qa_checks.Connector("source-foobar"))
-
-
-@pytest.mark.parametrize(
-    "connector, expect_exists",
-    [
-        (utils.Connector("source-faker"), True),
-        (utils.Connector("source-foobar"), False),
-    ],
-)
-def test_check_connector_icon_is_available(connector, expect_exists):
-    assert qa_checks.check_connector_icon_is_available(connector) == expect_exists
-
-
-@pytest.mark.parametrize(
-    "user_input, expect_qa_checks_to_run",
-    [
-        ("not-a-connector", False),
-        ("connectors/source-faker", True),
-        ("source-faker", True),
-    ],
-)
-def test_run_qa_checks_success(capsys, mocker, user_input, expect_qa_checks_to_run):
-    mocker.patch.object(qa_checks.sys, "argv", ["", user_input])
-    mocker.patch.object(qa_checks, "Connector")
-    mock_qa_check = mocker.Mock(return_value=True, __name__="mock_qa_check")
-    if expect_qa_checks_to_run:
-        mocker.patch.object(qa_checks, "QA_CHECKS", [mock_qa_check])
-    with pytest.raises(SystemExit) as wrapped_error:
-        qa_checks.run_qa_checks()
-    assert wrapped_error.value.code == 0
-    if not expect_qa_checks_to_run:
-        qa_checks.Connector.assert_not_called()
-        stdout, _ = capsys.readouterr()
-        assert "No QA check to run" in stdout
-    else:
-        expected_connector_technical_name = user_input.split("/")[-1]
-        qa_checks.Connector.assert_called_with(expected_connector_technical_name)
-        mock_qa_check.assert_called_with(qa_checks.Connector.return_value)
-        stdout, _ = capsys.readouterr()
-        assert f"Running QA checks for {expected_connector_technical_name}" in stdout
-        assert f"All QA checks succeeded for {expected_connector_technical_name}" in stdout
-
-
-def test_run_qa_checks_error(capsys, mocker):
-    mocker.patch.object(qa_checks.sys, "argv", ["", "source-faker"])
-    mocker.patch.object(qa_checks, "Connector")
-    mock_qa_check = mocker.Mock(return_value=False, __name__="mock_qa_check")
-    mocker.patch.object(qa_checks, "QA_CHECKS", [mock_qa_check])
-    with pytest.raises(SystemExit) as wrapped_error:
-        qa_checks.run_qa_checks()
-    assert wrapped_error.value.code == 1
-    stdout, _ = capsys.readouterr()
-    assert "QA checks failed for source-faker" in stdout
-    assert "❌ - mock_qa_check" in stdout
-
-
-@pytest.mark.parametrize(
-    "file_name, file_line, expected_in_stdout",
-    [
-        ("file_with_http_url.foo", "http://foo.bar", True),
-        ("file_without_https_url.foo", "", False),
-        ("file_with_https_url.foo", "https://airbyte.com", False),
-        ("file_with_http_url_and_ignored.foo", "http://localhost http://airbyte.com", True),
-        ("file_with_ignored_url.foo", "http://localhost", False),
-        ("file_with_http_url_in_comment.py", "# http://dev.foo", False),
-        ("file_with_http_url_in_comment.yml", "# http://dev.foo", False),
-        ("file_with_http_url_in_comment.yaml", "# http://dev.foo", False),
-        ("file_with_http_url_in_comment.java", "// http://dev.foo", False),
-        ("file_with_http_url_in_comment.md", "